From 29e9ec182d20f44ee2e8a15de1940cb4ef29663e Mon Sep 17 00:00:00 2001 From: Emilian Peev Date: Thu, 2 Jan 2020 12:43:50 -0800 Subject: [PATCH] Camera: Integrate dynamic depth processing Integrate dynamic depth processing as part of the camera service library. Dynamic linking is no longer required as legacy devices with small system partitions are not supported. Bug: 132449311 Test: atest cts/tests/camera/src/android/hardware/camera2/cts/StillCaptureTest.java#testDynamicDepthCapture atest cts/tests/camera/src/android/hardware/camera2/cts/ImageReaderTest.java#testDynamicDepth atest cts/tests/camera/src/android/hardware/camera2/cts/ExtendedCameraCharacteristicsTest.java#testDepthOutputCharacteristics cameraservice_test --gtest_filter=DepthProcessorTest.* Change-Id: Ie8befc5c0635e3e08c7ad8cac7b056cdf5aa3548 --- services/camera/libcameraservice/Android.bp | 41 +-------- .../api2/DepthCompositeStream.cpp | 30 +------ .../api2/DepthCompositeStream.h | 2 - .../common/CameraProviderManager.cpp | 30 ------- .../common/CameraProviderManager.h | 1 - .../common/DepthPhotoProcessor.cpp | 2 +- .../common/DepthPhotoProcessor.h | 4 +- .../tests/DepthProcessorTest.cpp | 83 +++---------------- 8 files changed, 18 insertions(+), 175 deletions(-) diff --git a/services/camera/libcameraservice/Android.bp b/services/camera/libcameraservice/Android.bp index 6052a06ca5..c63feb2810 100644 --- a/services/camera/libcameraservice/Android.bp +++ b/services/camera/libcameraservice/Android.bp @@ -28,6 +28,7 @@ cc_library_shared { "common/CameraDeviceBase.cpp", "common/CameraOfflineSessionBase.cpp", "common/CameraProviderManager.cpp", + "common/DepthPhotoProcessor.cpp", "common/FrameProcessorBase.cpp", "api1/CameraClient.cpp", "api1/Camera2Client.cpp", @@ -91,10 +92,12 @@ cc_library_shared { "libmediautils", "libcamera_client", "libcamera_metadata", + "libdynamic_depth", "libfmq", "libgui", "libhardware", "libhidlbase", + "libimage_io", "libjpeg", "libmedia_codeclist", "libmedia_omx", @@ -102,6 +105,7 @@ cc_library_shared { "libsensorprivacy", "libstagefright", "libstagefright_foundation", + "libxml2", "libyuv", "android.frameworks.cameraservice.common@2.0", "android.frameworks.cameraservice.service@2.0", @@ -143,40 +147,3 @@ cc_library_shared { } -cc_library_shared { - name: "libdepthphoto", - - srcs: [ - "utils/ExifUtils.cpp", - "common/DepthPhotoProcessor.cpp", - ], - - shared_libs: [ - "libimage_io", - "libdynamic_depth", - "libxml2", - "liblog", - "libutilscallstack", - "libutils", - "libcutils", - "libjpeg", - "libmemunreachable", - "libexif", - "libcamera_client", - ], - - include_dirs: [ - "external/dynamic_depth/includes", - "external/dynamic_depth/internal", - ], - - export_include_dirs: ["."], - - cflags: [ - "-Wall", - "-Wextra", - "-Werror", - "-Wno-ignored-qualifiers", - ], - -} diff --git a/services/camera/libcameraservice/api2/DepthCompositeStream.cpp b/services/camera/libcameraservice/api2/DepthCompositeStream.cpp index 0b9101619d..ac6f8d6e26 100644 --- a/services/camera/libcameraservice/api2/DepthCompositeStream.cpp +++ b/services/camera/libcameraservice/api2/DepthCompositeStream.cpp @@ -20,7 +20,6 @@ #include "api1/client2/JpegProcessor.h" #include "common/CameraProviderManager.h" -#include "dlfcn.h" #include #include #include @@ -43,9 +42,7 @@ DepthCompositeStream::DepthCompositeStream(wp device, mBlobBufferAcquired(false), mProducerListener(new ProducerListener()), mMaxJpegSize(-1), - mIsLogicalCamera(false), - mDepthPhotoLibHandle(nullptr), - mDepthPhotoProcess(nullptr) { + mIsLogicalCamera(false) { sp cameraDevice = device.promote(); if (cameraDevice.get() != nullptr) { CameraMetadata staticInfo = cameraDevice->info(); @@ -83,19 +80,6 @@ DepthCompositeStream::DepthCompositeStream(wp device, } getSupportedDepthSizes(staticInfo, &mSupportedDepthSizes); - - mDepthPhotoLibHandle = dlopen(camera3::kDepthPhotoLibrary, RTLD_NOW | RTLD_LOCAL); - if (mDepthPhotoLibHandle != nullptr) { - mDepthPhotoProcess = reinterpret_cast ( - dlsym(mDepthPhotoLibHandle, camera3::kDepthPhotoProcessFunction)); - if (mDepthPhotoProcess == nullptr) { - ALOGE("%s: Failed to link to depth photo process function: %s", __FUNCTION__, - dlerror()); - } - } else { - ALOGE("%s: Failed to link to depth photo library: %s", __FUNCTION__, dlerror()); - } - } } @@ -108,11 +92,6 @@ DepthCompositeStream::~DepthCompositeStream() { mDepthSurface.clear(); mDepthConsumer = nullptr; mDepthSurface = nullptr; - if (mDepthPhotoLibHandle != nullptr) { - dlclose(mDepthPhotoLibHandle); - mDepthPhotoLibHandle = nullptr; - } - mDepthPhotoProcess = nullptr; } void DepthCompositeStream::compilePendingInputLocked() { @@ -356,7 +335,7 @@ status_t DepthCompositeStream::processInputFrame(nsecs_t ts, const InputFrame &i } size_t actualJpegSize = 0; - res = mDepthPhotoProcess(depthPhoto, finalJpegBufferSize, dstBuffer, &actualJpegSize); + res = processDepthPhotoFrame(depthPhoto, finalJpegBufferSize, dstBuffer, &actualJpegSize); if (res != 0) { ALOGE("%s: Depth photo processing failed: %s (%d)", __FUNCTION__, strerror(-res), res); outputANW->cancelBuffer(mOutputSurface.get(), anb, /*fence*/ -1); @@ -583,11 +562,6 @@ status_t DepthCompositeStream::configureStream() { return NO_ERROR; } - if ((mDepthPhotoLibHandle == nullptr) || (mDepthPhotoProcess == nullptr)) { - ALOGE("%s: Depth photo library is not present!", __FUNCTION__); - return NO_INIT; - } - if (mOutputSurface.get() == nullptr) { ALOGE("%s: No valid output surface set!", __FUNCTION__); return NO_INIT; diff --git a/services/camera/libcameraservice/api2/DepthCompositeStream.h b/services/camera/libcameraservice/api2/DepthCompositeStream.h index 28a7826c5d..60c6b1e715 100644 --- a/services/camera/libcameraservice/api2/DepthCompositeStream.h +++ b/services/camera/libcameraservice/api2/DepthCompositeStream.h @@ -126,8 +126,6 @@ private: std::vector> mSupportedDepthSizes; std::vector mIntrinsicCalibration, mLensDistortion; bool mIsLogicalCamera; - void* mDepthPhotoLibHandle; - process_depth_photo_frame mDepthPhotoProcess; // Keep all incoming Depth buffer timestamps pending further processing. std::vector mInputDepthBuffers; diff --git a/services/camera/libcameraservice/common/CameraProviderManager.cpp b/services/camera/libcameraservice/common/CameraProviderManager.cpp index 23f78842f7..c8c31f3baa 100644 --- a/services/camera/libcameraservice/common/CameraProviderManager.cpp +++ b/services/camera/libcameraservice/common/CameraProviderManager.cpp @@ -716,31 +716,6 @@ void CameraProviderManager::ProviderInfo::DeviceInfo3::getSupportedDynamicDepthS } } -bool CameraProviderManager::ProviderInfo::DeviceInfo3::isDepthPhotoLibraryPresent() { - static bool libraryPresent = false; - static bool initialized = false; - if (initialized) { - return libraryPresent; - } else { - initialized = true; - } - - void* depthLibHandle = dlopen(camera3::kDepthPhotoLibrary, RTLD_NOW | RTLD_LOCAL); - if (depthLibHandle == nullptr) { - return false; - } - - auto processFunc = dlsym(depthLibHandle, camera3::kDepthPhotoProcessFunction); - if (processFunc != nullptr) { - libraryPresent = true; - } else { - libraryPresent = false; - } - dlclose(depthLibHandle); - - return libraryPresent; -} - status_t CameraProviderManager::ProviderInfo::DeviceInfo3::addDynamicDepthTags() { uint32_t depthExclTag = ANDROID_DEPTH_DEPTH_IS_EXCLUSIVE; uint32_t depthSizesTag = ANDROID_DEPTH_AVAILABLE_DEPTH_STREAM_CONFIGURATIONS; @@ -788,11 +763,6 @@ status_t CameraProviderManager::ProviderInfo::DeviceInfo3::addDynamicDepthTags() return OK; } - if(!isDepthPhotoLibraryPresent()) { - // Depth photo processing library is not present, nothing more to do. - return OK; - } - std::vector dynamicDepthEntries; for (const auto& it : supportedDynamicDepthSizes) { int32_t entry[4] = {HAL_PIXEL_FORMAT_BLOB, static_cast (std::get<0>(it)), diff --git a/services/camera/libcameraservice/common/CameraProviderManager.h b/services/camera/libcameraservice/common/CameraProviderManager.h index 651b8a1093..58df0e845b 100644 --- a/services/camera/libcameraservice/common/CameraProviderManager.h +++ b/services/camera/libcameraservice/common/CameraProviderManager.h @@ -545,7 +545,6 @@ private: void getSupportedDynamicDepthDurations(const std::vector& depthDurations, const std::vector& blobDurations, std::vector *dynamicDepthDurations /*out*/); - static bool isDepthPhotoLibraryPresent(); static void getSupportedDynamicDepthSizes( const std::vector>& blobSizes, const std::vector>& depthSizes, diff --git a/services/camera/libcameraservice/common/DepthPhotoProcessor.cpp b/services/camera/libcameraservice/common/DepthPhotoProcessor.cpp index 94541d87a9..c9956702dc 100644 --- a/services/camera/libcameraservice/common/DepthPhotoProcessor.cpp +++ b/services/camera/libcameraservice/common/DepthPhotoProcessor.cpp @@ -410,7 +410,7 @@ std::unique_ptr processDepthMapFrame(DepthPhotoInputFra return DepthMap::FromData(depthParams, items); } -extern "C" int processDepthPhotoFrame(DepthPhotoInputFrame inputFrame, size_t depthPhotoBufferSize, +int processDepthPhotoFrame(DepthPhotoInputFrame inputFrame, size_t depthPhotoBufferSize, void* depthPhotoBuffer /*out*/, size_t* depthPhotoActualSize /*out*/) { if ((inputFrame.mMainJpegBuffer == nullptr) || (inputFrame.mDepthMapBuffer == nullptr) || (depthPhotoBuffer == nullptr) || (depthPhotoActualSize == nullptr)) { diff --git a/services/camera/libcameraservice/common/DepthPhotoProcessor.h b/services/camera/libcameraservice/common/DepthPhotoProcessor.h index ba5ca9ec34..09b693515c 100644 --- a/services/camera/libcameraservice/common/DepthPhotoProcessor.h +++ b/services/camera/libcameraservice/common/DepthPhotoProcessor.h @@ -64,9 +64,7 @@ struct DepthPhotoInputFrame { mOrientation(DepthPhotoOrientation::DEPTH_ORIENTATION_0_DEGREES) {} }; -static const char *kDepthPhotoLibrary = "libdepthphoto.so"; -static const char *kDepthPhotoProcessFunction = "processDepthPhotoFrame"; -typedef int (*process_depth_photo_frame) (DepthPhotoInputFrame /*inputFrame*/, +int processDepthPhotoFrame(DepthPhotoInputFrame /*inputFrame*/, size_t /*depthPhotoBufferSize*/, void* /*depthPhotoBuffer out*/, size_t* /*depthPhotoActualSize out*/); diff --git a/services/camera/libcameraservice/tests/DepthProcessorTest.cpp b/services/camera/libcameraservice/tests/DepthProcessorTest.cpp index 2162514d14..673c149f7f 100644 --- a/services/camera/libcameraservice/tests/DepthProcessorTest.cpp +++ b/services/camera/libcameraservice/tests/DepthProcessorTest.cpp @@ -20,7 +20,6 @@ #include #include -#include #include #include "../common/DepthPhotoProcessor.h" @@ -36,19 +35,6 @@ static const size_t kTestBufferNV12Size ((((kTestBufferWidth) * (kTestBufferHeig static const size_t kTestBufferDepthSize (kTestBufferWidth * kTestBufferHeight); static const size_t kSeed = 1234; -void linkToDepthPhotoLibrary(void **libHandle /*out*/, - process_depth_photo_frame *processFrameFunc /*out*/) { - ASSERT_NE(libHandle, nullptr); - ASSERT_NE(processFrameFunc, nullptr); - - *libHandle = dlopen(kDepthPhotoLibrary, RTLD_NOW | RTLD_LOCAL); - if (*libHandle != nullptr) { - *processFrameFunc = reinterpret_cast ( - dlsym(*libHandle, kDepthPhotoProcessFunction)); - ASSERT_NE(*processFrameFunc, nullptr); - } -} - void generateColorJpegBuffer(int jpegQuality, ExifOrientation orientationValue, bool includeExif, bool switchDimensions, std::vector *colorJpegBuffer /*out*/) { ASSERT_NE(colorJpegBuffer, nullptr); @@ -91,26 +77,9 @@ void generateDepth16Buffer(std::array *depth16Bu } } -TEST(DepthProcessorTest, LinkToLibray) { - void *libHandle; - process_depth_photo_frame processFunc; - linkToDepthPhotoLibrary(&libHandle, &processFunc); - if (libHandle != nullptr) { - dlclose(libHandle); - } -} - TEST(DepthProcessorTest, BadInput) { - void *libHandle; int jpegQuality = 95; - process_depth_photo_frame processFunc; - linkToDepthPhotoLibrary(&libHandle, &processFunc); - if (libHandle == nullptr) { - // Depth library no present, nothing more to test. - return; - } - DepthPhotoInputFrame inputFrame; // Worst case both depth and confidence maps have the same size as the main color image. inputFrame.mMaxJpegSize = inputFrame.mMainJpegSize * 3; @@ -128,37 +97,27 @@ TEST(DepthProcessorTest, BadInput) { inputFrame.mMainJpegWidth = kTestBufferWidth; inputFrame.mMainJpegHeight = kTestBufferHeight; inputFrame.mJpegQuality = jpegQuality; - ASSERT_NE(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(), + ASSERT_NE(processDepthPhotoFrame(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(), &actualDepthPhotoSize), 0); inputFrame.mMainJpegBuffer = reinterpret_cast (colorJpegBuffer.data()); inputFrame.mMainJpegSize = colorJpegBuffer.size(); - ASSERT_NE(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(), + ASSERT_NE(processDepthPhotoFrame(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(), &actualDepthPhotoSize), 0); inputFrame.mDepthMapBuffer = depth16Buffer.data(); inputFrame.mDepthMapWidth = inputFrame.mDepthMapStride = kTestBufferWidth; inputFrame.mDepthMapHeight = kTestBufferHeight; - ASSERT_NE(processFunc(inputFrame, depthPhotoBuffer.size(), nullptr, + ASSERT_NE(processDepthPhotoFrame(inputFrame, depthPhotoBuffer.size(), nullptr, &actualDepthPhotoSize), 0); - ASSERT_NE(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(), nullptr), - 0); - - dlclose(libHandle); + ASSERT_NE(processDepthPhotoFrame(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(), + nullptr), 0); } TEST(DepthProcessorTest, BasicDepthPhotoValidation) { - void *libHandle; int jpegQuality = 95; - process_depth_photo_frame processFunc; - linkToDepthPhotoLibrary(&libHandle, &processFunc); - if (libHandle == nullptr) { - // Depth library no present, nothing more to test. - return; - } - std::vector colorJpegBuffer; generateColorJpegBuffer(jpegQuality, ExifOrientation::ORIENTATION_UNDEFINED, /*includeExif*/ false, /*switchDimensions*/ false, &colorJpegBuffer); @@ -180,7 +139,7 @@ TEST(DepthProcessorTest, BasicDepthPhotoValidation) { std::vector depthPhotoBuffer(inputFrame.mMaxJpegSize); size_t actualDepthPhotoSize = 0; - ASSERT_EQ(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(), + ASSERT_EQ(processDepthPhotoFrame(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(), &actualDepthPhotoSize), 0); ASSERT_TRUE((actualDepthPhotoSize > 0) && (depthPhotoBuffer.size() >= actualDepthPhotoSize)); @@ -196,21 +155,11 @@ TEST(DepthProcessorTest, BasicDepthPhotoValidation) { ASSERT_EQ(NV12Compressor::findJpegSize(depthPhotoBuffer.data() + mainJpegSize, actualDepthPhotoSize - mainJpegSize, &depthMapSize), OK); ASSERT_TRUE((depthMapSize > 0) && (depthMapSize < (actualDepthPhotoSize - mainJpegSize))); - - dlclose(libHandle); } TEST(DepthProcessorTest, TestDepthPhotoExifOrientation) { - void *libHandle; int jpegQuality = 95; - process_depth_photo_frame processFunc; - linkToDepthPhotoLibrary(&libHandle, &processFunc); - if (libHandle == nullptr) { - // Depth library no present, nothing more to test. - return; - } - ExifOrientation exifOrientations[] = { ExifOrientation::ORIENTATION_UNDEFINED, ExifOrientation::ORIENTATION_0_DEGREES, ExifOrientation::ORIENTATION_90_DEGREES, ExifOrientation::ORIENTATION_180_DEGREES, ExifOrientation::ORIENTATION_270_DEGREES }; @@ -242,8 +191,8 @@ TEST(DepthProcessorTest, TestDepthPhotoExifOrientation) { std::vector depthPhotoBuffer(inputFrame.mMaxJpegSize); size_t actualDepthPhotoSize = 0; - ASSERT_EQ(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(), - &actualDepthPhotoSize), 0); + ASSERT_EQ(processDepthPhotoFrame(inputFrame, depthPhotoBuffer.size(), + depthPhotoBuffer.data(), &actualDepthPhotoSize), 0); ASSERT_TRUE((actualDepthPhotoSize > 0) && (depthPhotoBuffer.size() >= actualDepthPhotoSize)); @@ -281,21 +230,11 @@ TEST(DepthProcessorTest, TestDepthPhotoExifOrientation) { ASSERT_EQ(confidenceJpegExifOrientation, exifOrientation); } } - - dlclose(libHandle); } TEST(DepthProcessorTest, TestDephtPhotoPhysicalRotation) { - void *libHandle; int jpegQuality = 95; - process_depth_photo_frame processFunc; - linkToDepthPhotoLibrary(&libHandle, &processFunc); - if (libHandle == nullptr) { - // Depth library no present, nothing more to test. - return; - } - // In case of physical rotation, the EXIF orientation must always be 0. auto exifOrientation = ExifOrientation::ORIENTATION_0_DEGREES; DepthPhotoOrientation depthOrientations[] = { @@ -339,8 +278,8 @@ TEST(DepthProcessorTest, TestDephtPhotoPhysicalRotation) { std::vector depthPhotoBuffer(inputFrame.mMaxJpegSize); size_t actualDepthPhotoSize = 0; - ASSERT_EQ(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(), - &actualDepthPhotoSize), 0); + ASSERT_EQ(processDepthPhotoFrame(inputFrame, depthPhotoBuffer.size(), + depthPhotoBuffer.data(), &actualDepthPhotoSize), 0); ASSERT_TRUE((actualDepthPhotoSize > 0) && (depthPhotoBuffer.size() >= actualDepthPhotoSize)); @@ -377,6 +316,4 @@ TEST(DepthProcessorTest, TestDephtPhotoPhysicalRotation) { ASSERT_EQ(confidenceMapWidth, expectedWidth); ASSERT_EQ(confidenceMapHeight, expectedHeight); } - - dlclose(libHandle); }