Camera: Integrate dynamic depth processing

Integrate dynamic depth processing as part of
the camera service library.
Dynamic linking is no longer required as legacy
devices with small system partitions are not
supported.

Bug: 132449311
Test:
atest
cts/tests/camera/src/android/hardware/camera2/cts/StillCaptureTest.java#testDynamicDepthCapture
atest
cts/tests/camera/src/android/hardware/camera2/cts/ImageReaderTest.java#testDynamicDepth
atest
cts/tests/camera/src/android/hardware/camera2/cts/ExtendedCameraCharacteristicsTest.java#testDepthOutputCharacteristics
cameraservice_test --gtest_filter=DepthProcessorTest.*

Change-Id: Ie8befc5c0635e3e08c7ad8cac7b056cdf5aa3548

services/camera/libcameraservice/Android.bp

@@ -28,6 +28,7 @@ cc_library_shared {
         "common/CameraDeviceBase.cpp",
         "common/CameraOfflineSessionBase.cpp",
         "common/CameraProviderManager.cpp",
+        "common/DepthPhotoProcessor.cpp",
         "common/FrameProcessorBase.cpp",
         "api1/CameraClient.cpp",
         "api1/Camera2Client.cpp",
@@ -91,10 +92,12 @@ cc_library_shared {
         "libmediautils",
         "libcamera_client",
         "libcamera_metadata",
+        "libdynamic_depth",
         "libfmq",
         "libgui",
         "libhardware",
         "libhidlbase",
+        "libimage_io",
         "libjpeg",
         "libmedia_codeclist",
         "libmedia_omx",
@@ -102,6 +105,7 @@ cc_library_shared {
         "libsensorprivacy",
         "libstagefright",
         "libstagefright_foundation",
+        "libxml2",
         "libyuv",
         "android.frameworks.cameraservice.common@2.0",
         "android.frameworks.cameraservice.service@2.0",
@@ -143,40 +147,3 @@ cc_library_shared {
 
 }
 
-cc_library_shared {
-    name: "libdepthphoto",
-
-    srcs: [
-        "utils/ExifUtils.cpp",
-        "common/DepthPhotoProcessor.cpp",
-    ],
-
-    shared_libs: [
-        "libimage_io",
-        "libdynamic_depth",
-        "libxml2",
-        "liblog",
-        "libutilscallstack",
-        "libutils",
-        "libcutils",
-        "libjpeg",
-        "libmemunreachable",
-        "libexif",
-        "libcamera_client",
-    ],
-
-    include_dirs: [
-        "external/dynamic_depth/includes",
-        "external/dynamic_depth/internal",
-    ],
-
-    export_include_dirs: ["."],
-
-    cflags: [
-        "-Wall",
-        "-Wextra",
-        "-Werror",
-        "-Wno-ignored-qualifiers",
-    ],
-
-}

services/camera/libcameraservice/api2/DepthCompositeStream.cpp

@@ -20,7 +20,6 @@
 
 #include "api1/client2/JpegProcessor.h"
 #include "common/CameraProviderManager.h"
-#include "dlfcn.h"
 #include <gui/Surface.h>
 #include <utils/Log.h>
 #include <utils/Trace.h>
@@ -43,9 +42,7 @@ DepthCompositeStream::DepthCompositeStream(wp<CameraDeviceBase> device,
         mBlobBufferAcquired(false),
         mProducerListener(new ProducerListener()),
         mMaxJpegSize(-1),
-        mIsLogicalCamera(false),
+        mIsLogicalCamera(false) {
-        mDepthPhotoLibHandle(nullptr),
-        mDepthPhotoProcess(nullptr) {
     sp<CameraDeviceBase> cameraDevice = device.promote();
     if (cameraDevice.get() != nullptr) {
         CameraMetadata staticInfo = cameraDevice->info();
@@ -83,19 +80,6 @@ DepthCompositeStream::DepthCompositeStream(wp<CameraDeviceBase> device,
         }
 
         getSupportedDepthSizes(staticInfo, &mSupportedDepthSizes);
-
-        mDepthPhotoLibHandle = dlopen(camera3::kDepthPhotoLibrary, RTLD_NOW | RTLD_LOCAL);
-        if (mDepthPhotoLibHandle != nullptr) {
-            mDepthPhotoProcess = reinterpret_cast<camera3::process_depth_photo_frame> (
-                    dlsym(mDepthPhotoLibHandle, camera3::kDepthPhotoProcessFunction));
-            if (mDepthPhotoProcess == nullptr) {
-                ALOGE("%s: Failed to link to depth photo process function: %s", __FUNCTION__,
-                        dlerror());
-            }
-        } else {
-            ALOGE("%s: Failed to link to depth photo library: %s", __FUNCTION__, dlerror());
-        }
-
     }
 }
 
@@ -108,11 +92,6 @@ DepthCompositeStream::~DepthCompositeStream() {
     mDepthSurface.clear();
     mDepthConsumer = nullptr;
     mDepthSurface = nullptr;
-    if (mDepthPhotoLibHandle != nullptr) {
-        dlclose(mDepthPhotoLibHandle);
-        mDepthPhotoLibHandle = nullptr;
-    }
-    mDepthPhotoProcess = nullptr;
 }
 
 void DepthCompositeStream::compilePendingInputLocked() {
@@ -356,7 +335,7 @@ status_t DepthCompositeStream::processInputFrame(nsecs_t ts, const InputFrame &i
     }
 
     size_t actualJpegSize = 0;
-    res = mDepthPhotoProcess(depthPhoto, finalJpegBufferSize, dstBuffer, &actualJpegSize);
+    res = processDepthPhotoFrame(depthPhoto, finalJpegBufferSize, dstBuffer, &actualJpegSize);
     if (res != 0) {
         ALOGE("%s: Depth photo processing failed: %s (%d)", __FUNCTION__, strerror(-res), res);
         outputANW->cancelBuffer(mOutputSurface.get(), anb, /*fence*/ -1);
@@ -583,11 +562,6 @@ status_t DepthCompositeStream::configureStream() {
         return NO_ERROR;
     }
 
-    if ((mDepthPhotoLibHandle == nullptr) || (mDepthPhotoProcess == nullptr)) {
-        ALOGE("%s: Depth photo library is not present!", __FUNCTION__);
-        return NO_INIT;
-    }
-
     if (mOutputSurface.get() == nullptr) {
         ALOGE("%s: No valid output surface set!", __FUNCTION__);
         return NO_INIT;

services/camera/libcameraservice/api2/DepthCompositeStream.h

@@ -126,8 +126,6 @@ private:
     std::vector<std::tuple<size_t, size_t>> mSupportedDepthSizes;
     std::vector<float>   mIntrinsicCalibration, mLensDistortion;
     bool                 mIsLogicalCamera;
-    void*                mDepthPhotoLibHandle;
-    process_depth_photo_frame mDepthPhotoProcess;
 
     // Keep all incoming Depth buffer timestamps pending further processing.
     std::vector<int64_t> mInputDepthBuffers;

services/camera/libcameraservice/common/CameraProviderManager.cpp

@@ -716,31 +716,6 @@ void CameraProviderManager::ProviderInfo::DeviceInfo3::getSupportedDynamicDepthS
     }
 }
 
-bool CameraProviderManager::ProviderInfo::DeviceInfo3::isDepthPhotoLibraryPresent() {
-    static bool libraryPresent = false;
-    static bool initialized = false;
-    if (initialized) {
-        return libraryPresent;
-    } else {
-        initialized = true;
-    }
-
-    void* depthLibHandle = dlopen(camera3::kDepthPhotoLibrary, RTLD_NOW | RTLD_LOCAL);
-    if (depthLibHandle == nullptr) {
-        return false;
-    }
-
-    auto processFunc = dlsym(depthLibHandle, camera3::kDepthPhotoProcessFunction);
-    if (processFunc != nullptr) {
-        libraryPresent = true;
-    } else {
-        libraryPresent = false;
-    }
-    dlclose(depthLibHandle);
-
-    return libraryPresent;
-}
-
 status_t CameraProviderManager::ProviderInfo::DeviceInfo3::addDynamicDepthTags() {
     uint32_t depthExclTag = ANDROID_DEPTH_DEPTH_IS_EXCLUSIVE;
     uint32_t depthSizesTag = ANDROID_DEPTH_AVAILABLE_DEPTH_STREAM_CONFIGURATIONS;
@@ -788,11 +763,6 @@ status_t CameraProviderManager::ProviderInfo::DeviceInfo3::addDynamicDepthTags()
         return OK;
     }
 
-    if(!isDepthPhotoLibraryPresent()) {
-        // Depth photo processing library is not present, nothing more to do.
-        return OK;
-    }
-
     std::vector<int32_t> dynamicDepthEntries;
     for (const auto& it : supportedDynamicDepthSizes) {
         int32_t entry[4] = {HAL_PIXEL_FORMAT_BLOB, static_cast<int32_t> (std::get<0>(it)),

services/camera/libcameraservice/common/CameraProviderManager.h

@@ -545,7 +545,6 @@ private:
             void getSupportedDynamicDepthDurations(const std::vector<int64_t>& depthDurations,
                     const std::vector<int64_t>& blobDurations,
                     std::vector<int64_t> *dynamicDepthDurations /*out*/);
-            static bool isDepthPhotoLibraryPresent();
             static void getSupportedDynamicDepthSizes(
                     const std::vector<std::tuple<size_t, size_t>>& blobSizes,
                     const std::vector<std::tuple<size_t, size_t>>& depthSizes,

services/camera/libcameraservice/common/DepthPhotoProcessor.cpp

@@ -410,7 +410,7 @@ std::unique_ptr<dynamic_depth::DepthMap> processDepthMapFrame(DepthPhotoInputFra
     return DepthMap::FromData(depthParams, items);
 }
 
-extern "C" int processDepthPhotoFrame(DepthPhotoInputFrame inputFrame, size_t depthPhotoBufferSize,
+int processDepthPhotoFrame(DepthPhotoInputFrame inputFrame, size_t depthPhotoBufferSize,
         void* depthPhotoBuffer /*out*/, size_t* depthPhotoActualSize /*out*/) {
     if ((inputFrame.mMainJpegBuffer == nullptr) || (inputFrame.mDepthMapBuffer == nullptr) ||
             (depthPhotoBuffer == nullptr) || (depthPhotoActualSize == nullptr)) {

services/camera/libcameraservice/common/DepthPhotoProcessor.h

@@ -64,9 +64,7 @@ struct DepthPhotoInputFrame {
             mOrientation(DepthPhotoOrientation::DEPTH_ORIENTATION_0_DEGREES) {}
 };
 
-static const char *kDepthPhotoLibrary = "libdepthphoto.so";
+int processDepthPhotoFrame(DepthPhotoInputFrame /*inputFrame*/,
-static const char *kDepthPhotoProcessFunction = "processDepthPhotoFrame";
-typedef int (*process_depth_photo_frame) (DepthPhotoInputFrame /*inputFrame*/,
         size_t /*depthPhotoBufferSize*/, void* /*depthPhotoBuffer out*/,
         size_t* /*depthPhotoActualSize out*/);
 

services/camera/libcameraservice/tests/DepthProcessorTest.cpp

@@ -20,7 +20,6 @@
 #include <array>
 #include <random>
 
-#include <dlfcn.h>
 #include <gtest/gtest.h>
 
 #include "../common/DepthPhotoProcessor.h"
@@ -36,19 +35,6 @@ static const size_t kTestBufferNV12Size ((((kTestBufferWidth) * (kTestBufferHeig
 static const size_t kTestBufferDepthSize (kTestBufferWidth * kTestBufferHeight);
 static const size_t kSeed = 1234;
 
-void linkToDepthPhotoLibrary(void **libHandle /*out*/,
-        process_depth_photo_frame *processFrameFunc /*out*/) {
-    ASSERT_NE(libHandle, nullptr);
-    ASSERT_NE(processFrameFunc, nullptr);
-
-    *libHandle = dlopen(kDepthPhotoLibrary, RTLD_NOW | RTLD_LOCAL);
-    if (*libHandle != nullptr) {
-        *processFrameFunc = reinterpret_cast<camera3::process_depth_photo_frame> (
-                dlsym(*libHandle, kDepthPhotoProcessFunction));
-        ASSERT_NE(*processFrameFunc, nullptr);
-    }
-}
-
 void generateColorJpegBuffer(int jpegQuality, ExifOrientation orientationValue, bool includeExif,
         bool switchDimensions, std::vector<uint8_t> *colorJpegBuffer /*out*/) {
     ASSERT_NE(colorJpegBuffer, nullptr);
@@ -91,26 +77,9 @@ void generateDepth16Buffer(std::array<uint16_t, kTestBufferDepthSize> *depth16Bu
     }
 }
 
-TEST(DepthProcessorTest, LinkToLibray) {
-    void *libHandle;
-    process_depth_photo_frame processFunc;
-    linkToDepthPhotoLibrary(&libHandle, &processFunc);
-    if (libHandle != nullptr) {
-        dlclose(libHandle);
-    }
-}
-
 TEST(DepthProcessorTest, BadInput) {
-    void *libHandle;
     int jpegQuality = 95;
 
-    process_depth_photo_frame processFunc;
-    linkToDepthPhotoLibrary(&libHandle, &processFunc);
-    if (libHandle == nullptr) {
-        // Depth library no present, nothing more to test.
-        return;
-    }
-
     DepthPhotoInputFrame inputFrame;
     // Worst case both depth and confidence maps have the same size as the main color image.
     inputFrame.mMaxJpegSize = inputFrame.mMainJpegSize * 3;
@@ -128,37 +97,27 @@ TEST(DepthProcessorTest, BadInput) {
     inputFrame.mMainJpegWidth = kTestBufferWidth;
     inputFrame.mMainJpegHeight = kTestBufferHeight;
     inputFrame.mJpegQuality = jpegQuality;
-    ASSERT_NE(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
+    ASSERT_NE(processDepthPhotoFrame(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
                 &actualDepthPhotoSize), 0);
 
     inputFrame.mMainJpegBuffer = reinterpret_cast<const char*> (colorJpegBuffer.data());
     inputFrame.mMainJpegSize = colorJpegBuffer.size();
-    ASSERT_NE(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
+    ASSERT_NE(processDepthPhotoFrame(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
                 &actualDepthPhotoSize), 0);
 
     inputFrame.mDepthMapBuffer = depth16Buffer.data();
     inputFrame.mDepthMapWidth = inputFrame.mDepthMapStride = kTestBufferWidth;
     inputFrame.mDepthMapHeight = kTestBufferHeight;
-    ASSERT_NE(processFunc(inputFrame, depthPhotoBuffer.size(), nullptr,
+    ASSERT_NE(processDepthPhotoFrame(inputFrame, depthPhotoBuffer.size(), nullptr,
                 &actualDepthPhotoSize), 0);
 
-    ASSERT_NE(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(), nullptr),
+    ASSERT_NE(processDepthPhotoFrame(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
-            0);
+                nullptr), 0);
-
-    dlclose(libHandle);
 }
 
 TEST(DepthProcessorTest, BasicDepthPhotoValidation) {
-    void *libHandle;
     int jpegQuality = 95;
 
-    process_depth_photo_frame processFunc;
-    linkToDepthPhotoLibrary(&libHandle, &processFunc);
-    if (libHandle == nullptr) {
-        // Depth library no present, nothing more to test.
-        return;
-    }
-
     std::vector<uint8_t> colorJpegBuffer;
     generateColorJpegBuffer(jpegQuality, ExifOrientation::ORIENTATION_UNDEFINED,
             /*includeExif*/ false, /*switchDimensions*/ false, &colorJpegBuffer);
@@ -180,7 +139,7 @@ TEST(DepthProcessorTest, BasicDepthPhotoValidation) {
 
     std::vector<uint8_t> depthPhotoBuffer(inputFrame.mMaxJpegSize);
     size_t actualDepthPhotoSize = 0;
-    ASSERT_EQ(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
+    ASSERT_EQ(processDepthPhotoFrame(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
                 &actualDepthPhotoSize), 0);
     ASSERT_TRUE((actualDepthPhotoSize > 0) && (depthPhotoBuffer.size() >= actualDepthPhotoSize));
 
@@ -196,21 +155,11 @@ TEST(DepthProcessorTest, BasicDepthPhotoValidation) {
     ASSERT_EQ(NV12Compressor::findJpegSize(depthPhotoBuffer.data() + mainJpegSize,
                 actualDepthPhotoSize - mainJpegSize, &depthMapSize), OK);
     ASSERT_TRUE((depthMapSize > 0) && (depthMapSize < (actualDepthPhotoSize - mainJpegSize)));
-
-    dlclose(libHandle);
 }
 
 TEST(DepthProcessorTest, TestDepthPhotoExifOrientation) {
-    void *libHandle;
     int jpegQuality = 95;
 
-    process_depth_photo_frame processFunc;
-    linkToDepthPhotoLibrary(&libHandle, &processFunc);
-    if (libHandle == nullptr) {
-        // Depth library no present, nothing more to test.
-        return;
-    }
-
     ExifOrientation exifOrientations[] = { ExifOrientation::ORIENTATION_UNDEFINED,
             ExifOrientation::ORIENTATION_0_DEGREES, ExifOrientation::ORIENTATION_90_DEGREES,
             ExifOrientation::ORIENTATION_180_DEGREES, ExifOrientation::ORIENTATION_270_DEGREES };
@@ -242,8 +191,8 @@ TEST(DepthProcessorTest, TestDepthPhotoExifOrientation) {
 
         std::vector<uint8_t> depthPhotoBuffer(inputFrame.mMaxJpegSize);
         size_t actualDepthPhotoSize = 0;
-        ASSERT_EQ(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
+        ASSERT_EQ(processDepthPhotoFrame(inputFrame, depthPhotoBuffer.size(),
-                &actualDepthPhotoSize), 0);
+                    depthPhotoBuffer.data(), &actualDepthPhotoSize), 0);
         ASSERT_TRUE((actualDepthPhotoSize > 0) &&
                 (depthPhotoBuffer.size() >= actualDepthPhotoSize));
 
@@ -281,21 +230,11 @@ TEST(DepthProcessorTest, TestDepthPhotoExifOrientation) {
             ASSERT_EQ(confidenceJpegExifOrientation, exifOrientation);
         }
     }
-
-    dlclose(libHandle);
 }
 
 TEST(DepthProcessorTest, TestDephtPhotoPhysicalRotation) {
-    void *libHandle;
     int jpegQuality = 95;
 
-    process_depth_photo_frame processFunc;
-    linkToDepthPhotoLibrary(&libHandle, &processFunc);
-    if (libHandle == nullptr) {
-        // Depth library no present, nothing more to test.
-        return;
-    }
-
     // In case of physical rotation, the EXIF orientation must always be 0.
     auto exifOrientation = ExifOrientation::ORIENTATION_0_DEGREES;
     DepthPhotoOrientation depthOrientations[] = {
@@ -339,8 +278,8 @@ TEST(DepthProcessorTest, TestDephtPhotoPhysicalRotation) {
 
         std::vector<uint8_t> depthPhotoBuffer(inputFrame.mMaxJpegSize);
         size_t actualDepthPhotoSize = 0;
-        ASSERT_EQ(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
+        ASSERT_EQ(processDepthPhotoFrame(inputFrame, depthPhotoBuffer.size(),
-                &actualDepthPhotoSize), 0);
+                    depthPhotoBuffer.data(), &actualDepthPhotoSize), 0);
         ASSERT_TRUE((actualDepthPhotoSize > 0) &&
                 (depthPhotoBuffer.size() >= actualDepthPhotoSize));
 
@@ -377,6 +316,4 @@ TEST(DepthProcessorTest, TestDephtPhotoPhysicalRotation) {
         ASSERT_EQ(confidenceMapWidth, expectedWidth);
         ASSERT_EQ(confidenceMapHeight, expectedHeight);
     }
-
-    dlclose(libHandle);
 }