Camera: Add support for dynamic depth if possible

Composite dynamic depth streams will be supported
in case camera devices include:
- Dense depth streams which are not exclusive.
- Jpeg/Blob output with sizes that either match depth
  or are with similar aspect ratio.
Make a guesstimate regarding the dynamic depth minimum
frame and stall durations.

Bug: 109735087
Test: Manual using application,
Camera CTS

Change-Id: I8a89c7895cf57ce4408e41b1afae9c85d48c4e07

camera/ndk/include/camera/NdkCameraMetadataTags.h

@@ -5549,6 +5549,73 @@ typedef enum acamera_metadata_tag {
     ACAMERA_DEPTH_AVAILABLE_RECOMMENDED_DEPTH_STREAM_CONFIGURATIONS = 
                                                                 // int32[n*5]
             ACAMERA_DEPTH_START + 5,
+    /**
+     * <p>The available dynamic depth dataspace stream
+     * configurations that this camera device supports
+     * (i.e. format, width, height, output/input stream).</p>
+     *
+     * <p>Type: int32[n*4] (acamera_metadata_enum_android_depth_available_dynamic_depth_stream_configurations_t)</p>
+     *
+     * <p>This tag may appear in:
+     * <ul>
+     *   <li>ACameraMetadata from ACameraManager_getCameraCharacteristics</li>
+     * </ul></p>
+     *
+     * <p>These are output stream configurations for use with
+     * dataSpace DYNAMIC_DEPTH. The configurations are
+     * listed as <code>(format, width, height, input?)</code> tuples.</p>
+     * <p>Only devices that support depth output for at least
+     * the HAL_PIXEL_FORMAT_Y16 dense depth map along with
+     * HAL_PIXEL_FORMAT_BLOB with the same size or size with
+     * the same aspect ratio can have dynamic depth dataspace
+     * stream configuration. ACAMERA_DEPTH_DEPTH_IS_EXCLUSIVE also
+     * needs to be set to FALSE.</p>
+     *
+     * @see ACAMERA_DEPTH_DEPTH_IS_EXCLUSIVE
+     */
+    ACAMERA_DEPTH_AVAILABLE_DYNAMIC_DEPTH_STREAM_CONFIGURATIONS = 
+                                                                // int32[n*4] (acamera_metadata_enum_android_depth_available_dynamic_depth_stream_configurations_t)
+            ACAMERA_DEPTH_START + 6,
+    /**
+     * <p>This lists the minimum frame duration for each
+     * format/size combination for dynamic depth output streams.</p>
+     *
+     * <p>Type: int64[4*n]</p>
+     *
+     * <p>This tag may appear in:
+     * <ul>
+     *   <li>ACameraMetadata from ACameraManager_getCameraCharacteristics</li>
+     * </ul></p>
+     *
+     * <p>This should correspond to the frame duration when only that
+     * stream is active, with all processing (typically in android.*.mode)
+     * set to either OFF or FAST.</p>
+     * <p>When multiple streams are used in a request, the minimum frame
+     * duration will be max(individual stream min durations).</p>
+     * <p>The minimum frame duration of a stream (of a particular format, size)
+     * is the same regardless of whether the stream is input or output.</p>
+     */
+    ACAMERA_DEPTH_AVAILABLE_DYNAMIC_DEPTH_MIN_FRAME_DURATIONS = // int64[4*n]
+            ACAMERA_DEPTH_START + 7,
+    /**
+     * <p>This lists the maximum stall duration for each
+     * output format/size combination for dynamic depth streams.</p>
+     *
+     * <p>Type: int64[4*n]</p>
+     *
+     * <p>This tag may appear in:
+     * <ul>
+     *   <li>ACameraMetadata from ACameraManager_getCameraCharacteristics</li>
+     * </ul></p>
+     *
+     * <p>A stall duration is how much extra time would get added
+     * to the normal minimum frame duration for a repeating request
+     * that has streams with non-zero stall.</p>
+     * <p>All dynamic depth output streams may have a nonzero stall
+     * duration.</p>
+     */
+    ACAMERA_DEPTH_AVAILABLE_DYNAMIC_DEPTH_STALL_DURATIONS =     // int64[4*n]
+            ACAMERA_DEPTH_START + 8,
     ACAMERA_DEPTH_END,
 
     /**
@@ -8246,6 +8313,16 @@ typedef enum acamera_metadata_enum_acamera_depth_depth_is_exclusive {
 
 } acamera_metadata_enum_android_depth_depth_is_exclusive_t;
 
+// ACAMERA_DEPTH_AVAILABLE_DYNAMIC_DEPTH_STREAM_CONFIGURATIONS
+typedef enum acamera_metadata_enum_acamera_depth_available_dynamic_depth_stream_configurations {
+    ACAMERA_DEPTH_AVAILABLE_DYNAMIC_DEPTH_STREAM_CONFIGURATIONS_OUTPUT
+                                                                      = 0,
+
+    ACAMERA_DEPTH_AVAILABLE_DYNAMIC_DEPTH_STREAM_CONFIGURATIONS_INPUT
+                                                                      = 1,
+
+} acamera_metadata_enum_android_depth_available_dynamic_depth_stream_configurations_t;
+
 
 // ACAMERA_LOGICAL_MULTI_CAMERA_SENSOR_SYNC_TYPE
 typedef enum acamera_metadata_enum_acamera_logical_multi_camera_sensor_sync_type {

services/camera/libcameraservice/common/CameraProviderManager.cpp

@@ -500,6 +500,248 @@ void CameraProviderManager::ProviderInfo::DeviceInfo3::queryPhysicalCameraIds()
     }
 }
 
+void CameraProviderManager::ProviderInfo::DeviceInfo3::getSupportedSizes(
+        const CameraMetadata& ch, uint32_t tag, android_pixel_format_t format,
+        std::vector<std::tuple<size_t, size_t>> *sizes/*out*/) {
+    if (sizes == nullptr) {
+        return;
+    }
+
+    auto scalerDims = ch.find(tag);
+    if (scalerDims.count > 0) {
+        // Scaler entry contains 4 elements (format, width, height, type)
+        for (size_t i = 0; i < scalerDims.count; i += 4) {
+            if ((scalerDims.data.i32[i] == format) &&
+                    (scalerDims.data.i32[i+3] ==
+                     ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS_OUTPUT)) {
+                sizes->push_back(std::make_tuple(scalerDims.data.i32[i+1],
+                            scalerDims.data.i32[i+2]));
+            }
+        }
+    }
+}
+
+void CameraProviderManager::ProviderInfo::DeviceInfo3::getSupportedDurations(
+        const CameraMetadata& ch, uint32_t tag, android_pixel_format_t format,
+        const std::vector<std::tuple<size_t, size_t>>& sizes,
+        std::vector<int64_t> *durations/*out*/) {
+    if (durations == nullptr) {
+        return;
+    }
+
+    auto availableDurations = ch.find(tag);
+    if (availableDurations.count > 0) {
+        // Duration entry contains 4 elements (format, width, height, duration)
+        for (size_t i = 0; i < availableDurations.count; i += 4) {
+            for (const auto& size : sizes) {
+                int64_t width = std::get<0>(size);
+                int64_t height = std::get<1>(size);
+                if ((availableDurations.data.i64[i] == format) &&
+                        (availableDurations.data.i64[i+1] == width) &&
+                        (availableDurations.data.i64[i+2] == height)) {
+                    durations->push_back(availableDurations.data.i64[i+3]);
+                }
+            }
+        }
+    }
+}
+void CameraProviderManager::ProviderInfo::DeviceInfo3::getSupportedDynamicDepthDurations(
+        const std::vector<int64_t>& depthDurations, const std::vector<int64_t>& blobDurations,
+        std::vector<int64_t> *dynamicDepthDurations /*out*/) {
+    if ((dynamicDepthDurations == nullptr) || (depthDurations.size() != blobDurations.size())) {
+        return;
+    }
+
+    // Unfortunately there is no direct way to calculate the dynamic depth stream duration.
+    // Processing time on camera service side can vary greatly depending on multiple
+    // variables which are not under our control. Make a guesstimate by taking the maximum
+    // corresponding duration value from depth and blob.
+    auto depthDuration = depthDurations.begin();
+    auto blobDuration = blobDurations.begin();
+    dynamicDepthDurations->reserve(depthDurations.size());
+    while ((depthDuration != depthDurations.end()) && (blobDuration != blobDurations.end())) {
+        dynamicDepthDurations->push_back(std::max(*depthDuration, *blobDuration));
+        depthDuration++; blobDuration++;
+    }
+}
+
+void CameraProviderManager::ProviderInfo::DeviceInfo3::getSupportedDynamicDepthSizes(
+        const std::vector<std::tuple<size_t, size_t>>& blobSizes,
+        const std::vector<std::tuple<size_t, size_t>>& depthSizes,
+        std::vector<std::tuple<size_t, size_t>> *dynamicDepthSizes /*out*/,
+        std::vector<std::tuple<size_t, size_t>> *internalDepthSizes /*out*/) {
+    if (dynamicDepthSizes == nullptr || internalDepthSizes == nullptr) {
+        return;
+    }
+
+    // The dynamic depth spec. does not mention how close the AR ratio should be.
+    // Try using something appropriate.
+    float ARTolerance = .01f;
+
+    //TODO: Remove this before merging! This is for testing purposes only
+    ARTolerance = 10.f;
+
+    for (const auto& blobSize : blobSizes) {
+        float jpegAR = static_cast<float> (std::get<0>(blobSize)) /
+                static_cast<float>(std::get<1>(blobSize));
+        bool found = false;
+        for (const auto& depthSize : depthSizes) {
+            if (depthSize == blobSize) {
+                internalDepthSizes->push_back(depthSize);
+                found = true;
+                break;
+            } else {
+                float depthAR = static_cast<float> (std::get<0>(depthSize)) /
+                    static_cast<float>(std::get<1>(depthSize));
+                if (std::fabs(jpegAR - depthAR) <= ARTolerance) {
+                    internalDepthSizes->push_back(depthSize);
+                    found = true;
+                    break;
+                }
+            }
+        }
+
+        if (found) {
+            dynamicDepthSizes->push_back(blobSize);
+        }
+    }
+}
+
+status_t CameraProviderManager::ProviderInfo::DeviceInfo3::addDynamicDepthTags() {
+    uint32_t depthExclTag = ANDROID_DEPTH_DEPTH_IS_EXCLUSIVE;
+    uint32_t depthSizesTag = ANDROID_DEPTH_AVAILABLE_DEPTH_STREAM_CONFIGURATIONS;
+    auto& c = mCameraCharacteristics;
+    std::vector<std::tuple<size_t, size_t>> supportedBlobSizes, supportedDepthSizes,
+            supportedDynamicDepthSizes, internalDepthSizes;
+    auto chTags = c.find(ANDROID_REQUEST_AVAILABLE_CHARACTERISTICS_KEYS);
+    if (chTags.count == 0) {
+        ALOGE("%s: Supported camera characteristics is empty!", __FUNCTION__);
+        return BAD_VALUE;
+    }
+
+    bool isDepthExclusivePresent = std::find(chTags.data.i32, chTags.data.i32 + chTags.count,
+            depthExclTag) != (chTags.data.i32 + chTags.count);
+    bool isDepthSizePresent = std::find(chTags.data.i32, chTags.data.i32 + chTags.count,
+            depthExclTag) != (chTags.data.i32 + chTags.count);
+    if (!(isDepthExclusivePresent && isDepthSizePresent)) {
+        // No depth support, nothing more to do.
+        return OK;
+    }
+
+    auto depthExclusiveEntry = c.find(depthExclTag);
+    if (depthExclusiveEntry.count > 0) {
+        if (depthExclusiveEntry.data.u8[0] != ANDROID_DEPTH_DEPTH_IS_EXCLUSIVE_FALSE) {
+            // Depth support is exclusive, nothing more to do.
+            return OK;
+        }
+    } else {
+        ALOGE("%s: Advertised depth exclusive tag but value is not present!", __FUNCTION__);
+        return BAD_VALUE;
+    }
+
+    getSupportedSizes(c, ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS, HAL_PIXEL_FORMAT_BLOB,
+            &supportedBlobSizes);
+    getSupportedSizes(c, depthSizesTag, HAL_PIXEL_FORMAT_Y16, &supportedDepthSizes);
+    if (supportedBlobSizes.empty() || supportedDepthSizes.empty()) {
+        // Nothing to do in this case.
+        return OK;
+    }
+
+    getSupportedDynamicDepthSizes(supportedBlobSizes, supportedDepthSizes,
+            &supportedDynamicDepthSizes, &internalDepthSizes);
+    if (supportedDynamicDepthSizes.empty()) {
+        ALOGE("%s: No dynamic depth size matched!", __func__);
+        // Nothing more to do.
+        return OK;
+    }
+
+    std::vector<int32_t> dynamicDepthEntries;
+    for (const auto& it : supportedDynamicDepthSizes) {
+        int32_t entry[4] = {HAL_PIXEL_FORMAT_BLOB, static_cast<int32_t> (std::get<0>(it)),
+                static_cast<int32_t> (std::get<1>(it)),
+                ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS_OUTPUT };
+        dynamicDepthEntries.insert(dynamicDepthEntries.end(), entry, entry + 4);
+    }
+
+    std::vector<int64_t> depthMinDurations, depthStallDurations;
+    std::vector<int64_t> blobMinDurations, blobStallDurations;
+    std::vector<int64_t> dynamicDepthMinDurations, dynamicDepthStallDurations;
+
+    getSupportedDurations(c, ANDROID_DEPTH_AVAILABLE_DEPTH_MIN_FRAME_DURATIONS,
+            HAL_PIXEL_FORMAT_Y16, internalDepthSizes, &depthMinDurations);
+    getSupportedDurations(c, ANDROID_SCALER_AVAILABLE_MIN_FRAME_DURATIONS,
+            HAL_PIXEL_FORMAT_BLOB, supportedDynamicDepthSizes, &blobMinDurations);
+    if (blobMinDurations.empty() || depthMinDurations.empty() ||
+            (depthMinDurations.size() != blobMinDurations.size())) {
+        ALOGE("%s: Unexpected number of available depth min durations! %zu vs. %zu",
+                __FUNCTION__, depthMinDurations.size(), blobMinDurations.size());
+        return BAD_VALUE;
+    }
+
+    getSupportedDurations(c, ANDROID_DEPTH_AVAILABLE_DEPTH_STALL_DURATIONS,
+            HAL_PIXEL_FORMAT_Y16, internalDepthSizes, &depthStallDurations);
+    getSupportedDurations(c, ANDROID_SCALER_AVAILABLE_STALL_DURATIONS,
+            HAL_PIXEL_FORMAT_BLOB, supportedDynamicDepthSizes, &blobStallDurations);
+    if (blobStallDurations.empty() || depthStallDurations.empty() ||
+            (depthStallDurations.size() != blobStallDurations.size())) {
+        ALOGE("%s: Unexpected number of available depth stall durations! %zu vs. %zu",
+                __FUNCTION__, depthStallDurations.size(), blobStallDurations.size());
+        return BAD_VALUE;
+    }
+
+    getSupportedDynamicDepthDurations(depthMinDurations, blobMinDurations,
+            &dynamicDepthMinDurations);
+    getSupportedDynamicDepthDurations(depthStallDurations, blobStallDurations,
+            &dynamicDepthStallDurations);
+    if (dynamicDepthMinDurations.empty() || dynamicDepthStallDurations.empty() ||
+            (dynamicDepthMinDurations.size() != dynamicDepthStallDurations.size())) {
+        ALOGE("%s: Unexpected number of dynamic depth stall/min durations! %zu vs. %zu",
+                __FUNCTION__, dynamicDepthMinDurations.size(), dynamicDepthStallDurations.size());
+        return BAD_VALUE;
+    }
+
+    std::vector<int64_t> dynamicDepthMinDurationEntries;
+    auto itDuration = dynamicDepthMinDurations.begin();
+    auto itSize = supportedDynamicDepthSizes.begin();
+    while (itDuration != dynamicDepthMinDurations.end()) {
+        int64_t entry[4] = {HAL_PIXEL_FORMAT_BLOB, static_cast<int32_t> (std::get<0>(*itSize)),
+                static_cast<int32_t> (std::get<1>(*itSize)), *itDuration};
+        dynamicDepthMinDurationEntries.insert(dynamicDepthMinDurationEntries.end(), entry,
+                entry + 4);
+        itDuration++; itSize++;
+    }
+
+    std::vector<int64_t> dynamicDepthStallDurationEntries;
+    itDuration = dynamicDepthStallDurations.begin();
+    itSize = supportedDynamicDepthSizes.begin();
+    while (itDuration != dynamicDepthStallDurations.end()) {
+        int64_t entry[4] = {HAL_PIXEL_FORMAT_BLOB, static_cast<int32_t> (std::get<0>(*itSize)),
+                static_cast<int32_t> (std::get<1>(*itSize)), *itDuration};
+        dynamicDepthStallDurationEntries.insert(dynamicDepthStallDurationEntries.end(), entry,
+                entry + 4);
+        itDuration++; itSize++;
+    }
+
+    c.update(ANDROID_DEPTH_AVAILABLE_DYNAMIC_DEPTH_STREAM_CONFIGURATIONS,
+            dynamicDepthEntries.data(), dynamicDepthEntries.size());
+    c.update(ANDROID_DEPTH_AVAILABLE_DYNAMIC_DEPTH_MIN_FRAME_DURATIONS,
+            dynamicDepthMinDurationEntries.data(), dynamicDepthMinDurationEntries.size());
+    c.update(ANDROID_DEPTH_AVAILABLE_DYNAMIC_DEPTH_STALL_DURATIONS,
+            dynamicDepthStallDurationEntries.data(), dynamicDepthStallDurationEntries.size());
+
+    std::vector<int32_t> supportedChTags;
+    supportedChTags.reserve(chTags.count + 3);
+    supportedChTags.insert(supportedChTags.end(), chTags.data.i32,
+            chTags.data.i32 + chTags.count);
+    supportedChTags.push_back(ANDROID_DEPTH_AVAILABLE_DYNAMIC_DEPTH_STREAM_CONFIGURATIONS);
+    supportedChTags.push_back(ANDROID_DEPTH_AVAILABLE_DYNAMIC_DEPTH_MIN_FRAME_DURATIONS);
+    supportedChTags.push_back(ANDROID_DEPTH_AVAILABLE_DYNAMIC_DEPTH_STALL_DURATIONS);
+    c.update(ANDROID_REQUEST_AVAILABLE_CHARACTERISTICS_KEYS, supportedChTags.data(),
+            supportedChTags.size());
+
+    return OK;
+}
+
 status_t CameraProviderManager::ProviderInfo::DeviceInfo3::fixupMonochromeTags() {
     status_t res = OK;
     auto& c = mCameraCharacteristics;
@@ -1442,6 +1684,12 @@ CameraProviderManager::ProviderInfo::DeviceInfo3::DeviceInfo3(const std::string&
                 __FUNCTION__, strerror(-res), res);
         return;
     }
+    res = addDynamicDepthTags();
+    if (OK != res) {
+        ALOGE("%s: Failed appending dynamic depth tags: %s (%d)", __FUNCTION__, strerror(-res),
+                res);
+        return;
+    }
     camera_metadata_entry flashAvailable =
             mCameraCharacteristics.find(ANDROID_FLASH_INFO_AVAILABLE);
     if (flashAvailable.count == 1 &&

services/camera/libcameraservice/common/CameraProviderManager.h

@@ -470,6 +470,22 @@ private:
             std::unordered_map<std::string, CameraMetadata> mPhysicalCameraCharacteristics;
             void queryPhysicalCameraIds();
             status_t fixupMonochromeTags();
+            status_t addDynamicDepthTags();
+            static void getSupportedSizes(const CameraMetadata& ch, uint32_t tag,
+                    android_pixel_format_t format,
+                    std::vector<std::tuple<size_t, size_t>> *sizes /*out*/);
+            void getSupportedDurations( const CameraMetadata& ch, uint32_t tag,
+                    android_pixel_format_t format,
+                    const std::vector<std::tuple<size_t, size_t>>& sizes,
+                    std::vector<int64_t> *durations/*out*/);
+            void getSupportedDynamicDepthDurations(const std::vector<int64_t>& depthDurations,
+                    const std::vector<int64_t>& blobDurations,
+                    std::vector<int64_t> *dynamicDepthDurations /*out*/);
+            static void getSupportedDynamicDepthSizes(
+                    const std::vector<std::tuple<size_t, size_t>>& blobSizes,
+                    const std::vector<std::tuple<size_t, size_t>>& depthSizes,
+                    std::vector<std::tuple<size_t, size_t>> *dynamicDepthSizes /*out*/,
+                    std::vector<std::tuple<size_t, size_t>> *internalDepthSizes /*out*/);
             status_t removeAvailableKeys(CameraMetadata& c, const std::vector<uint32_t>& keys,
                     uint32_t keyTag);
         };