Merge "Camera: Integrate dynamic depth processing"

services/camera/libcameraservice/Android.bp

@@ -28,6 +28,7 @@ cc_library_shared {
         "common/CameraDeviceBase.cpp",
         "common/CameraOfflineSessionBase.cpp",
         "common/CameraProviderManager.cpp",
+        "common/DepthPhotoProcessor.cpp",
         "common/FrameProcessorBase.cpp",
         "api1/CameraClient.cpp",
         "api1/Camera2Client.cpp",
@@ -91,10 +92,12 @@ cc_library_shared {
         "libmediautils",
         "libcamera_client",
         "libcamera_metadata",
+        "libdynamic_depth",
         "libfmq",
         "libgui",
         "libhardware",
         "libhidlbase",
+        "libimage_io",
         "libjpeg",
         "libmedia_codeclist",
         "libmedia_omx",
@@ -102,6 +105,7 @@ cc_library_shared {
         "libsensorprivacy",
         "libstagefright",
         "libstagefright_foundation",
+        "libxml2",
         "libyuv",
         "android.frameworks.cameraservice.common@2.0",
         "android.frameworks.cameraservice.service@2.0",
@@ -143,40 +147,3 @@ cc_library_shared {
 
 }
 
-cc_library_shared {
-    name: "libdepthphoto",
-
-    srcs: [
-        "utils/ExifUtils.cpp",
-        "common/DepthPhotoProcessor.cpp",
-    ],
-
-    shared_libs: [
-        "libimage_io",
-        "libdynamic_depth",
-        "libxml2",
-        "liblog",
-        "libutilscallstack",
-        "libutils",
-        "libcutils",
-        "libjpeg",
-        "libmemunreachable",
-        "libexif",
-        "libcamera_client",
-    ],
-
-    include_dirs: [
-        "external/dynamic_depth/includes",
-        "external/dynamic_depth/internal",
-    ],
-
-    export_include_dirs: ["."],
-
-    cflags: [
-        "-Wall",
-        "-Wextra",
-        "-Werror",
-        "-Wno-ignored-qualifiers",
-    ],
-
-}

services/camera/libcameraservice/api2/DepthCompositeStream.cpp

@@ -20,7 +20,6 @@
 
 #include "api1/client2/JpegProcessor.h"
 #include "common/CameraProviderManager.h"
-#include "dlfcn.h"
 #include <gui/Surface.h>
 #include <utils/Log.h>
 #include <utils/Trace.h>
@@ -43,9 +42,7 @@ DepthCompositeStream::DepthCompositeStream(wp<CameraDeviceBase> device,
         mBlobBufferAcquired(false),
         mProducerListener(new ProducerListener()),
         mMaxJpegSize(-1),
-        mIsLogicalCamera(false),
+        mIsLogicalCamera(false) {
-        mDepthPhotoLibHandle(nullptr),
-        mDepthPhotoProcess(nullptr) {
     sp<CameraDeviceBase> cameraDevice = device.promote();
     if (cameraDevice.get() != nullptr) {
         CameraMetadata staticInfo = cameraDevice->info();
@@ -83,19 +80,6 @@ DepthCompositeStream::DepthCompositeStream(wp<CameraDeviceBase> device,
         }
 
         getSupportedDepthSizes(staticInfo, &mSupportedDepthSizes);
-
-        mDepthPhotoLibHandle = dlopen(camera3::kDepthPhotoLibrary, RTLD_NOW | RTLD_LOCAL);
-        if (mDepthPhotoLibHandle != nullptr) {
-            mDepthPhotoProcess = reinterpret_cast<camera3::process_depth_photo_frame> (
-                    dlsym(mDepthPhotoLibHandle, camera3::kDepthPhotoProcessFunction));
-            if (mDepthPhotoProcess == nullptr) {
-                ALOGE("%s: Failed to link to depth photo process function: %s", __FUNCTION__,
-                        dlerror());
-            }
-        } else {
-            ALOGE("%s: Failed to link to depth photo library: %s", __FUNCTION__, dlerror());
-        }
-
     }
 }
 
@@ -108,11 +92,6 @@ DepthCompositeStream::~DepthCompositeStream() {
     mDepthSurface.clear();
     mDepthConsumer = nullptr;
     mDepthSurface = nullptr;
-    if (mDepthPhotoLibHandle != nullptr) {
-        dlclose(mDepthPhotoLibHandle);
-        mDepthPhotoLibHandle = nullptr;
-    }
-    mDepthPhotoProcess = nullptr;
 }
 
 void DepthCompositeStream::compilePendingInputLocked() {
@@ -356,7 +335,7 @@ status_t DepthCompositeStream::processInputFrame(nsecs_t ts, const InputFrame &i
     }
 
     size_t actualJpegSize = 0;
-    res = mDepthPhotoProcess(depthPhoto, finalJpegBufferSize, dstBuffer, &actualJpegSize);
+    res = processDepthPhotoFrame(depthPhoto, finalJpegBufferSize, dstBuffer, &actualJpegSize);
     if (res != 0) {
         ALOGE("%s: Depth photo processing failed: %s (%d)", __FUNCTION__, strerror(-res), res);
         outputANW->cancelBuffer(mOutputSurface.get(), anb, /*fence*/ -1);
@@ -583,11 +562,6 @@ status_t DepthCompositeStream::configureStream() {
         return NO_ERROR;
     }
 
-    if ((mDepthPhotoLibHandle == nullptr) || (mDepthPhotoProcess == nullptr)) {
-        ALOGE("%s: Depth photo library is not present!", __FUNCTION__);
-        return NO_INIT;
-    }
-
     if (mOutputSurface.get() == nullptr) {
         ALOGE("%s: No valid output surface set!", __FUNCTION__);
         return NO_INIT;

services/camera/libcameraservice/api2/DepthCompositeStream.h

@@ -126,8 +126,6 @@ private:
     std::vector<std::tuple<size_t, size_t>> mSupportedDepthSizes;
     std::vector<float>   mIntrinsicCalibration, mLensDistortion;
     bool                 mIsLogicalCamera;
-    void*                mDepthPhotoLibHandle;
-    process_depth_photo_frame mDepthPhotoProcess;
 
     // Keep all incoming Depth buffer timestamps pending further processing.
     std::vector<int64_t> mInputDepthBuffers;

services/camera/libcameraservice/common/CameraProviderManager.cpp

@@ -716,31 +716,6 @@ void CameraProviderManager::ProviderInfo::DeviceInfo3::getSupportedDynamicDepthS
     }
 }
 
-bool CameraProviderManager::ProviderInfo::DeviceInfo3::isDepthPhotoLibraryPresent() {
-    static bool libraryPresent = false;
-    static bool initialized = false;
-    if (initialized) {
-        return libraryPresent;
-    } else {
-        initialized = true;
-    }
-
-    void* depthLibHandle = dlopen(camera3::kDepthPhotoLibrary, RTLD_NOW | RTLD_LOCAL);
-    if (depthLibHandle == nullptr) {
-        return false;
-    }
-
-    auto processFunc = dlsym(depthLibHandle, camera3::kDepthPhotoProcessFunction);
-    if (processFunc != nullptr) {
-        libraryPresent = true;
-    } else {
-        libraryPresent = false;
-    }
-    dlclose(depthLibHandle);
-
-    return libraryPresent;
-}
-
 status_t CameraProviderManager::ProviderInfo::DeviceInfo3::addDynamicDepthTags() {
     uint32_t depthExclTag = ANDROID_DEPTH_DEPTH_IS_EXCLUSIVE;
     uint32_t depthSizesTag = ANDROID_DEPTH_AVAILABLE_DEPTH_STREAM_CONFIGURATIONS;
@@ -788,11 +763,6 @@ status_t CameraProviderManager::ProviderInfo::DeviceInfo3::addDynamicDepthTags()
         return OK;
     }
 
-    if(!isDepthPhotoLibraryPresent()) {
-        // Depth photo processing library is not present, nothing more to do.
-        return OK;
-    }
-
     std::vector<int32_t> dynamicDepthEntries;
     for (const auto& it : supportedDynamicDepthSizes) {
         int32_t entry[4] = {HAL_PIXEL_FORMAT_BLOB, static_cast<int32_t> (std::get<0>(it)),

services/camera/libcameraservice/common/CameraProviderManager.h

@@ -545,7 +545,6 @@ private:
             void getSupportedDynamicDepthDurations(const std::vector<int64_t>& depthDurations,
                     const std::vector<int64_t>& blobDurations,
                     std::vector<int64_t> *dynamicDepthDurations /*out*/);
-            static bool isDepthPhotoLibraryPresent();
             static void getSupportedDynamicDepthSizes(
                     const std::vector<std::tuple<size_t, size_t>>& blobSizes,
                     const std::vector<std::tuple<size_t, size_t>>& depthSizes,

services/camera/libcameraservice/common/DepthPhotoProcessor.cpp

@@ -410,7 +410,7 @@ std::unique_ptr<dynamic_depth::DepthMap> processDepthMapFrame(DepthPhotoInputFra
     return DepthMap::FromData(depthParams, items);
 }
 
-extern "C" int processDepthPhotoFrame(DepthPhotoInputFrame inputFrame, size_t depthPhotoBufferSize,
+int processDepthPhotoFrame(DepthPhotoInputFrame inputFrame, size_t depthPhotoBufferSize,
         void* depthPhotoBuffer /*out*/, size_t* depthPhotoActualSize /*out*/) {
     if ((inputFrame.mMainJpegBuffer == nullptr) || (inputFrame.mDepthMapBuffer == nullptr) ||
             (depthPhotoBuffer == nullptr) || (depthPhotoActualSize == nullptr)) {

services/camera/libcameraservice/common/DepthPhotoProcessor.h

@@ -64,9 +64,7 @@ struct DepthPhotoInputFrame {
             mOrientation(DepthPhotoOrientation::DEPTH_ORIENTATION_0_DEGREES) {}
 };
 
-static const char *kDepthPhotoLibrary = "libdepthphoto.so";
+int processDepthPhotoFrame(DepthPhotoInputFrame /*inputFrame*/,
-static const char *kDepthPhotoProcessFunction = "processDepthPhotoFrame";
-typedef int (*process_depth_photo_frame) (DepthPhotoInputFrame /*inputFrame*/,
         size_t /*depthPhotoBufferSize*/, void* /*depthPhotoBuffer out*/,
         size_t* /*depthPhotoActualSize out*/);
 

services/camera/libcameraservice/tests/DepthProcessorTest.cpp

@@ -20,7 +20,6 @@
 #include <array>
 #include <random>
 
-#include <dlfcn.h>
 #include <gtest/gtest.h>
 
 #include "../common/DepthPhotoProcessor.h"
@@ -36,19 +35,6 @@ static const size_t kTestBufferNV12Size ((((kTestBufferWidth) * (kTestBufferHeig
 static const size_t kTestBufferDepthSize (kTestBufferWidth * kTestBufferHeight);
 static const size_t kSeed = 1234;
 
-void linkToDepthPhotoLibrary(void **libHandle /*out*/,
-        process_depth_photo_frame *processFrameFunc /*out*/) {
-    ASSERT_NE(libHandle, nullptr);
-    ASSERT_NE(processFrameFunc, nullptr);
-
-    *libHandle = dlopen(kDepthPhotoLibrary, RTLD_NOW | RTLD_LOCAL);
-    if (*libHandle != nullptr) {
-        *processFrameFunc = reinterpret_cast<camera3::process_depth_photo_frame> (
-                dlsym(*libHandle, kDepthPhotoProcessFunction));
-        ASSERT_NE(*processFrameFunc, nullptr);
-    }
-}
-
 void generateColorJpegBuffer(int jpegQuality, ExifOrientation orientationValue, bool includeExif,
         bool switchDimensions, std::vector<uint8_t> *colorJpegBuffer /*out*/) {
     ASSERT_NE(colorJpegBuffer, nullptr);
@@ -91,26 +77,9 @@ void generateDepth16Buffer(std::array<uint16_t, kTestBufferDepthSize> *depth16Bu
     }
 }
 
-TEST(DepthProcessorTest, LinkToLibray) {
-    void *libHandle;
-    process_depth_photo_frame processFunc;
-    linkToDepthPhotoLibrary(&libHandle, &processFunc);
-    if (libHandle != nullptr) {
-        dlclose(libHandle);
-    }
-}
-
 TEST(DepthProcessorTest, BadInput) {
-    void *libHandle;
     int jpegQuality = 95;
 
-    process_depth_photo_frame processFunc;
-    linkToDepthPhotoLibrary(&libHandle, &processFunc);
-    if (libHandle == nullptr) {
-        // Depth library no present, nothing more to test.
-        return;
-    }
-
     DepthPhotoInputFrame inputFrame;
     // Worst case both depth and confidence maps have the same size as the main color image.
     inputFrame.mMaxJpegSize = inputFrame.mMainJpegSize * 3;
@@ -128,37 +97,27 @@ TEST(DepthProcessorTest, BadInput) {
     inputFrame.mMainJpegWidth = kTestBufferWidth;
     inputFrame.mMainJpegHeight = kTestBufferHeight;
     inputFrame.mJpegQuality = jpegQuality;
-    ASSERT_NE(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
+    ASSERT_NE(processDepthPhotoFrame(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
                 &actualDepthPhotoSize), 0);
 
     inputFrame.mMainJpegBuffer = reinterpret_cast<const char*> (colorJpegBuffer.data());
     inputFrame.mMainJpegSize = colorJpegBuffer.size();
-    ASSERT_NE(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
+    ASSERT_NE(processDepthPhotoFrame(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
                 &actualDepthPhotoSize), 0);
 
     inputFrame.mDepthMapBuffer = depth16Buffer.data();
     inputFrame.mDepthMapWidth = inputFrame.mDepthMapStride = kTestBufferWidth;
     inputFrame.mDepthMapHeight = kTestBufferHeight;
-    ASSERT_NE(processFunc(inputFrame, depthPhotoBuffer.size(), nullptr,
+    ASSERT_NE(processDepthPhotoFrame(inputFrame, depthPhotoBuffer.size(), nullptr,
                 &actualDepthPhotoSize), 0);
 
-    ASSERT_NE(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(), nullptr),
+    ASSERT_NE(processDepthPhotoFrame(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
-            0);
+                nullptr), 0);
-
-    dlclose(libHandle);
 }
 
 TEST(DepthProcessorTest, BasicDepthPhotoValidation) {
-    void *libHandle;
     int jpegQuality = 95;
 
-    process_depth_photo_frame processFunc;
-    linkToDepthPhotoLibrary(&libHandle, &processFunc);
-    if (libHandle == nullptr) {
-        // Depth library no present, nothing more to test.
-        return;
-    }
-
     std::vector<uint8_t> colorJpegBuffer;
     generateColorJpegBuffer(jpegQuality, ExifOrientation::ORIENTATION_UNDEFINED,
             /*includeExif*/ false, /*switchDimensions*/ false, &colorJpegBuffer);
@@ -180,7 +139,7 @@ TEST(DepthProcessorTest, BasicDepthPhotoValidation) {
 
     std::vector<uint8_t> depthPhotoBuffer(inputFrame.mMaxJpegSize);
     size_t actualDepthPhotoSize = 0;
-    ASSERT_EQ(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
+    ASSERT_EQ(processDepthPhotoFrame(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
                 &actualDepthPhotoSize), 0);
     ASSERT_TRUE((actualDepthPhotoSize > 0) && (depthPhotoBuffer.size() >= actualDepthPhotoSize));
 
@@ -196,21 +155,11 @@ TEST(DepthProcessorTest, BasicDepthPhotoValidation) {
     ASSERT_EQ(NV12Compressor::findJpegSize(depthPhotoBuffer.data() + mainJpegSize,
                 actualDepthPhotoSize - mainJpegSize, &depthMapSize), OK);
     ASSERT_TRUE((depthMapSize > 0) && (depthMapSize < (actualDepthPhotoSize - mainJpegSize)));
-
-    dlclose(libHandle);
 }
 
 TEST(DepthProcessorTest, TestDepthPhotoExifOrientation) {
-    void *libHandle;
     int jpegQuality = 95;
 
-    process_depth_photo_frame processFunc;
-    linkToDepthPhotoLibrary(&libHandle, &processFunc);
-    if (libHandle == nullptr) {
-        // Depth library no present, nothing more to test.
-        return;
-    }
-
     ExifOrientation exifOrientations[] = { ExifOrientation::ORIENTATION_UNDEFINED,
             ExifOrientation::ORIENTATION_0_DEGREES, ExifOrientation::ORIENTATION_90_DEGREES,
             ExifOrientation::ORIENTATION_180_DEGREES, ExifOrientation::ORIENTATION_270_DEGREES };
@@ -242,8 +191,8 @@ TEST(DepthProcessorTest, TestDepthPhotoExifOrientation) {
 
         std::vector<uint8_t> depthPhotoBuffer(inputFrame.mMaxJpegSize);
         size_t actualDepthPhotoSize = 0;
-        ASSERT_EQ(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
+        ASSERT_EQ(processDepthPhotoFrame(inputFrame, depthPhotoBuffer.size(),
-                &actualDepthPhotoSize), 0);
+                    depthPhotoBuffer.data(), &actualDepthPhotoSize), 0);
         ASSERT_TRUE((actualDepthPhotoSize > 0) &&
                 (depthPhotoBuffer.size() >= actualDepthPhotoSize));
 
@@ -281,21 +230,11 @@ TEST(DepthProcessorTest, TestDepthPhotoExifOrientation) {
             ASSERT_EQ(confidenceJpegExifOrientation, exifOrientation);
         }
     }
-
-    dlclose(libHandle);
 }
 
 TEST(DepthProcessorTest, TestDephtPhotoPhysicalRotation) {
-    void *libHandle;
     int jpegQuality = 95;
 
-    process_depth_photo_frame processFunc;
-    linkToDepthPhotoLibrary(&libHandle, &processFunc);
-    if (libHandle == nullptr) {
-        // Depth library no present, nothing more to test.
-        return;
-    }
-
     // In case of physical rotation, the EXIF orientation must always be 0.
     auto exifOrientation = ExifOrientation::ORIENTATION_0_DEGREES;
     DepthPhotoOrientation depthOrientations[] = {
@@ -339,8 +278,8 @@ TEST(DepthProcessorTest, TestDephtPhotoPhysicalRotation) {
 
         std::vector<uint8_t> depthPhotoBuffer(inputFrame.mMaxJpegSize);
         size_t actualDepthPhotoSize = 0;
-        ASSERT_EQ(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
+        ASSERT_EQ(processDepthPhotoFrame(inputFrame, depthPhotoBuffer.size(),
-                &actualDepthPhotoSize), 0);
+                    depthPhotoBuffer.data(), &actualDepthPhotoSize), 0);
         ASSERT_TRUE((actualDepthPhotoSize > 0) &&
                 (depthPhotoBuffer.size() >= actualDepthPhotoSize));
 
@@ -377,6 +316,4 @@ TEST(DepthProcessorTest, TestDephtPhotoPhysicalRotation) {
         ASSERT_EQ(confidenceMapWidth, expectedWidth);
         ASSERT_EQ(confidenceMapHeight, expectedHeight);
     }
-
-    dlclose(libHandle);
 }