From aa3cced06116c0160f5e84ed70f88e8cc8a2bc29 Mon Sep 17 00:00:00 2001 From: Eino-Ville Talvala Date: Wed, 29 Nov 2017 15:40:18 -0800 Subject: [PATCH] Camera2 NDK: Add MOTION_TRACKING capability, and lens poseReference - MOTION_TRACKING capability defines a camera with accurate lens calibration values, accurate timestamps relative to the inertial sensors, and ability to limit exposure time to 20ms max. - Add android.lens.poseReference for supporting gyroscope-centered lens calibration data for MOTION_TRACKING. Test: Builds Bug: 63629224 Change-Id: I99f209afae930a7b59a2a84163d19de5f824cc96 --- camera/ndk/include/camera/NdkCameraDevice.h | 31 +++- .../include/camera/NdkCameraMetadataTags.h | 159 ++++++++++++++---- 2 files changed, 158 insertions(+), 32 deletions(-) diff --git a/camera/ndk/include/camera/NdkCameraDevice.h b/camera/ndk/include/camera/NdkCameraDevice.h index 61deb4647a..c0eb5c173a 100644 --- a/camera/ndk/include/camera/NdkCameraDevice.h +++ b/camera/ndk/include/camera/NdkCameraDevice.h @@ -251,6 +251,36 @@ typedef enum { * @see ACameraDevice_createCaptureRequest */ TEMPLATE_MANUAL = 6, + + /** + * A template for selecting camera parameters that match TEMPLATE_PREVIEW as closely as + * possible while improving the camera output for motion tracking use cases. + * + *

This template is best used by applications that are frequently switching between motion + * tracking use cases and regular still capture use cases, to minimize the IQ changes + * when swapping use cases.

+ * + *

This template is guaranteed to be supported on camera devices that support the + * {@link ACAMERA_REQUEST_AVAILABLE_CAPABILITIES_MOTION_TRACKING} capability.

+ * + * @see ACameraDevice_createCaptureRequest + */ + TEMPLATE_MOTION_TRACKING_PREVIEW = 7, + + /** + * A template for selecting camera parameters that maximize the quality of camera output for + * motion tracking use cases. + * + *

This template is best used by applications dedicated to motion tracking applications, + * which aren't concerned about fast switches between motion tracking and other use cases.

+ * + *

This template is guaranteed to be supported on camera devices that support the + * {@link ACAMERA_REQUEST_AVAILABLE_CAPABILITIES_MOTION_TRACKING} capability.

+ * + * @see ACameraDevice_createCaptureRequest + */ + TEMPLATE_MOTION_TRACKING_BEST = 8, + } ACameraDevice_request_template; /** @@ -760,4 +790,3 @@ __END_DECLS #endif /* _NDK_CAMERA_DEVICE_H */ /** @} */ - diff --git a/camera/ndk/include/camera/NdkCameraMetadataTags.h b/camera/ndk/include/camera/NdkCameraMetadataTags.h index e5a6f7d02f..2c144b79e8 100644 --- a/camera/ndk/include/camera/NdkCameraMetadataTags.h +++ b/camera/ndk/include/camera/NdkCameraMetadataTags.h @@ -837,10 +837,13 @@ typedef enum acamera_metadata_tag { * *

This control (except for MANUAL) is only effective if * ACAMERA_CONTROL_MODE != OFF and any 3A routine is active.

- *

ZERO_SHUTTER_LAG will be supported if ACAMERA_REQUEST_AVAILABLE_CAPABILITIES - * contains PRIVATE_REPROCESSING or YUV_REPROCESSING. MANUAL will be supported if - * ACAMERA_REQUEST_AVAILABLE_CAPABILITIES contains MANUAL_SENSOR. Other intent values are - * always supported.

+ *

All intents are supported by all devices, except that: + * * ZERO_SHUTTER_LAG will be supported if ACAMERA_REQUEST_AVAILABLE_CAPABILITIES contains + * PRIVATE_REPROCESSING or YUV_REPROCESSING. + * * MANUAL will be supported if ACAMERA_REQUEST_AVAILABLE_CAPABILITIES contains + * MANUAL_SENSOR. + * * MOTION_TRACKING will be supported if ACAMERA_REQUEST_AVAILABLE_CAPABILITIES contains + * MOTION_TRACKING.

* * @see ACAMERA_CONTROL_MODE * @see ACAMERA_REQUEST_AVAILABLE_CAPABILITIES @@ -2235,34 +2238,31 @@ typedef enum acamera_metadata_tag { *

* *

The position of the camera device's lens optical center, - * as a three-dimensional vector (x,y,z), relative to the - * optical center of the largest camera device facing in the - * same direction as this camera, in the Android sensor coordinate - * axes. Note that only the axis definitions are shared with - * the sensor coordinate system, but not the origin.

- *

If this device is the largest or only camera device with a - * given facing, then this position will be (0, 0, 0); a - * camera device with a lens optical center located 3 cm from - * the main sensor along the +X axis (to the right from the - * user's perspective) will report (0.03, 0, 0).

- *

To transform a pixel coordinates between two cameras - * facing the same direction, first the source camera - * ACAMERA_LENS_RADIAL_DISTORTION must be corrected for. Then - * the source camera ACAMERA_LENS_INTRINSIC_CALIBRATION needs - * to be applied, followed by the ACAMERA_LENS_POSE_ROTATION - * of the source camera, the translation of the source camera - * relative to the destination camera, the - * ACAMERA_LENS_POSE_ROTATION of the destination camera, and - * finally the inverse of ACAMERA_LENS_INTRINSIC_CALIBRATION - * of the destination camera. This obtains a - * radial-distortion-free coordinate in the destination - * camera pixel coordinates.

- *

To compare this against a real image from the destination - * camera, the destination camera image then needs to be - * corrected for radial distortion before comparison or - * sampling.

+ * as a three-dimensional vector (x,y,z).

+ *

Prior to Android P, or when ACAMERA_LENS_POSE_REFERENCE is PRIMARY_CAMERA, this position + * is relative to the optical center of the largest camera device facing in the same + * direction as this camera, in the Android sensor + * coordinate axes. Note that only the axis definitions are shared with the sensor + * coordinate system, but not the origin.

+ *

If this device is the largest or only camera device with a given facing, then this + * position will be (0, 0, 0); a camera device with a lens optical center located 3 cm + * from the main sensor along the +X axis (to the right from the user's perspective) will + * report (0.03, 0, 0).

+ *

To transform a pixel coordinates between two cameras facing the same direction, first + * the source camera ACAMERA_LENS_RADIAL_DISTORTION must be corrected for. Then the source + * camera ACAMERA_LENS_INTRINSIC_CALIBRATION needs to be applied, followed by the + * ACAMERA_LENS_POSE_ROTATION of the source camera, the translation of the source camera + * relative to the destination camera, the ACAMERA_LENS_POSE_ROTATION of the destination + * camera, and finally the inverse of ACAMERA_LENS_INTRINSIC_CALIBRATION of the destination + * camera. This obtains a radial-distortion-free coordinate in the destination camera pixel + * coordinates.

+ *

To compare this against a real image from the destination camera, the destination camera + * image then needs to be corrected for radial distortion before comparison or sampling.

+ *

When ACAMERA_LENS_POSE_REFERENCE is GYROSCOPE, then this position is relative to + * the center of the primary gyroscope on the device.

* * @see ACAMERA_LENS_INTRINSIC_CALIBRATION + * @see ACAMERA_LENS_POSE_REFERENCE * @see ACAMERA_LENS_POSE_ROTATION * @see ACAMERA_LENS_RADIAL_DISTORTION */ @@ -2433,6 +2433,26 @@ typedef enum acamera_metadata_tag { */ ACAMERA_LENS_RADIAL_DISTORTION = // float[6] ACAMERA_LENS_START + 11, + /** + *

The origin for ACAMERA_LENS_POSE_TRANSLATION.

+ * + * @see ACAMERA_LENS_POSE_TRANSLATION + * + *

Type: byte (acamera_metadata_enum_android_lens_pose_reference_t)

+ * + *

This tag may appear in: + *

+ * + *

Different calibration methods and use cases can produce better or worse results + * depending on the selected coordinate origin.

+ *

For devices designed to support the MOTION_TRACKING capability, the GYROSCOPE origin + * makes device calibration and later usage by applications combining camera and gyroscope + * information together simpler.

+ */ + ACAMERA_LENS_POSE_REFERENCE = // byte (acamera_metadata_enum_android_lens_pose_reference_t) + ACAMERA_LENS_START + 12, ACAMERA_LENS_END, /** @@ -2895,7 +2915,7 @@ typedef enum acamera_metadata_tag { * time-consuming hardware re-configuration or internal camera pipeline * change. For performance reasons we advise clients to pass their initial * values as part of - * {@link ACameraDevice_createCaptureSessionWithSessionParameters }.i + * {@link ACameraDevice_createCaptureSessionWithSessionParameters }. * Once the camera capture session is enabled it is also recommended to avoid * changing them from their initial values set in * {@link ACameraDevice_createCaptureSessionWithSessionParameters }. @@ -5717,6 +5737,15 @@ typedef enum acamera_metadata_enum_acamera_control_capture_intent { */ ACAMERA_CONTROL_CAPTURE_INTENT_MANUAL = 6, + /** + *

This request is for a motion tracking use case, where + * the application will use camera and inertial sensor data to + * locate and track objects in the world.

+ *

The camera device auto-exposure routine will limit the exposure time + * of the camera to no more than 20 milliseconds, to minimize motion blur.

+ */ + ACAMERA_CONTROL_CAPTURE_INTENT_MOTION_TRACKING = 7, + } acamera_metadata_enum_android_control_capture_intent_t; // ACAMERA_CONTROL_EFFECT_MODE @@ -6428,6 +6457,28 @@ typedef enum acamera_metadata_enum_acamera_lens_state { } acamera_metadata_enum_android_lens_state_t; +// ACAMERA_LENS_POSE_REFERENCE +typedef enum acamera_metadata_enum_acamera_lens_pose_reference { + /** + *

The value of ACAMERA_LENS_POSE_TRANSLATION is relative to the optical center of + * the largest camera device facing the same direction as this camera.

+ *

This default value for API levels before Android P.

+ * + * @see ACAMERA_LENS_POSE_TRANSLATION + */ + ACAMERA_LENS_POSE_REFERENCE_PRIMARY_CAMERA = 0, + + /** + *

The value of ACAMERA_LENS_POSE_TRANSLATION is relative to the position of the + * primary gyroscope of this Android device.

+ *

This is the value reported by all devices that support the MOTION_TRACKING capability.

+ * + * @see ACAMERA_LENS_POSE_TRANSLATION + */ + ACAMERA_LENS_POSE_REFERENCE_GYROSCOPE = 1, + +} acamera_metadata_enum_android_lens_pose_reference_t; + // ACAMERA_LENS_INFO_FOCUS_DISTANCE_CALIBRATION typedef enum acamera_metadata_enum_acamera_lens_info_focus_distance_calibration { @@ -6760,6 +6811,7 @@ typedef enum acamera_metadata_enum_acamera_request_available_capabilities { * * *
  • The ACAMERA_DEPTH_DEPTH_IS_EXCLUSIVE entry is listed by this device.
    • + *
  • As of Android P, the ACAMERA_LENS_POSE_REFERENCE entry is listed by this device.
    • *
  • A LIMITED camera with only the DEPTH_OUTPUT capability does not have to support * normal YUV_420_888, JPEG, and PRIV-format outputs. It only has to support the DEPTH16 * format.
    • @@ -6775,12 +6827,57 @@ typedef enum acamera_metadata_enum_acamera_request_available_capabilities { * @see ACAMERA_DEPTH_DEPTH_IS_EXCLUSIVE * @see ACAMERA_LENS_FACING * @see ACAMERA_LENS_INTRINSIC_CALIBRATION + * @see ACAMERA_LENS_POSE_REFERENCE * @see ACAMERA_LENS_POSE_ROTATION * @see ACAMERA_LENS_POSE_TRANSLATION * @see ACAMERA_LENS_RADIAL_DISTORTION */ ACAMERA_REQUEST_AVAILABLE_CAPABILITIES_DEPTH_OUTPUT = 8, + /** + *

    The device supports controls and metadata required for accurate motion tracking for + * use cases such as augmented reality, electronic image stabilization, and so on.

    + *

    This means this camera device has accurate optical calibration and timestamps relative + * to the inertial sensors.

    + *

    This capability requires the camera device to support the following:

    + * + * + * @see ACAMERA_CONTROL_AE_AVAILABLE_TARGET_FPS_RANGES + * @see ACAMERA_CONTROL_CAPTURE_INTENT + * @see ACAMERA_LENS_INTRINSIC_CALIBRATION + * @see ACAMERA_LENS_POSE_REFERENCE + * @see ACAMERA_LENS_POSE_ROTATION + * @see ACAMERA_LENS_POSE_TRANSLATION + * @see ACAMERA_LENS_RADIAL_DISTORTION + * @see ACAMERA_SENSOR_EXPOSURE_TIME + * @see ACAMERA_SENSOR_INFO_TIMESTAMP_SOURCE + * @see ACAMERA_SENSOR_ROLLING_SHUTTER_SKEW + */ + ACAMERA_REQUEST_AVAILABLE_CAPABILITIES_MOTION_TRACKING = 10, + } acamera_metadata_enum_android_request_available_capabilities_t;