gugelfrei
/
android_frameworks_av
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Camera: Integrate dynamic depth processing

Browse Source 
Integrate dynamic depth processing as part of
the camera service library.
Dynamic linking is no longer required as legacy
devices with small system partitions are not
supported.

Bug: 132449311
Test:
atest
cts/tests/camera/src/android/hardware/camera2/cts/StillCaptureTest.java#testDynamicDepthCapture
atest
cts/tests/camera/src/android/hardware/camera2/cts/ImageReaderTest.java#testDynamicDepth
atest
cts/tests/camera/src/android/hardware/camera2/cts/ExtendedCameraCharacteristicsTest.java#testDepthOutputCharacteristics
cameraservice_test --gtest_filter=DepthProcessorTest.*

Change-Id: Ie8befc5c0635e3e08c7ad8cac7b056cdf5aa3548
gugelfrei
Emilian Peev 5 years ago
parent 684e475866
commit 29e9ec182d
8 changed files with 18 additions and 175 deletions
Show Stats Download Patch File Download Diff File

41
services/camera/libcameraservice/Android.bp
Unescape View File

@ -28,6 +28,7 @@ cc_library_shared {
"common/CameraDeviceBase.cpp",
"common/CameraOfflineSessionBase.cpp",
"common/CameraProviderManager.cpp",
"common/DepthPhotoProcessor.cpp",
"common/FrameProcessorBase.cpp",
"api1/CameraClient.cpp",
"api1/Camera2Client.cpp",
@ -91,10 +92,12 @@ cc_library_shared {
"libmediautils",
"libcamera_client",
"libcamera_metadata",
"libdynamic_depth",
"libfmq",
"libgui",
"libhardware",
"libhidlbase",
"libimage_io",
"libjpeg",
"libmedia_codeclist",
"libmedia_omx",
@ -102,6 +105,7 @@ cc_library_shared {
"libsensorprivacy",
"libstagefright",
"libstagefright_foundation",
"libxml2",
"libyuv",
"android.frameworks.cameraservice.common@2.0",
"android.frameworks.cameraservice.service@2.0",
@ -143,40 +147,3 @@ cc_library_shared {
}
cc_library_shared {
name: "libdepthphoto",
srcs: [
"utils/ExifUtils.cpp",
"common/DepthPhotoProcessor.cpp",
],
shared_libs: [
"libimage_io",
"libdynamic_depth",
"libxml2",
"liblog",
"libutilscallstack",
"libutils",
"libcutils",
"libjpeg",
"libmemunreachable",
"libexif",
"libcamera_client",
],
include_dirs: [
"external/dynamic_depth/includes",
"external/dynamic_depth/internal",
],
export_include_dirs: ["."],
cflags: [
"-Wall",
"-Wextra",
"-Werror",
"-Wno-ignored-qualifiers",
],
}

30
services/camera/libcameraservice/api2/DepthCompositeStream.cpp
Unescape View File

@ -20,7 +20,6 @@
#include "api1/client2/JpegProcessor.h"
#include "common/CameraProviderManager.h"
#include "dlfcn.h"
#include <gui/Surface.h>
#include <utils/Log.h>
#include <utils/Trace.h>
@ -43,9 +42,7 @@ DepthCompositeStream::DepthCompositeStream(wp<CameraDeviceBase> device,
mBlobBufferAcquired(false),
mProducerListener(new ProducerListener()),
mMaxJpegSize(-1),
mIsLogicalCamera(false),
mDepthPhotoLibHandle(nullptr),
mDepthPhotoProcess(nullptr) {
mIsLogicalCamera(false) {
sp<CameraDeviceBase> cameraDevice = device.promote();
if (cameraDevice.get() != nullptr) {
CameraMetadata staticInfo = cameraDevice->info();
@ -83,19 +80,6 @@ DepthCompositeStream::DepthCompositeStream(wp<CameraDeviceBase> device,
}
getSupportedDepthSizes(staticInfo, &mSupportedDepthSizes);
mDepthPhotoLibHandle = dlopen(camera3::kDepthPhotoLibrary, RTLD_NOW | RTLD_LOCAL);
if (mDepthPhotoLibHandle != nullptr) {
mDepthPhotoProcess = reinterpret_cast<camera3::process_depth_photo_frame> (
dlsym(mDepthPhotoLibHandle, camera3::kDepthPhotoProcessFunction));
if (mDepthPhotoProcess == nullptr) {
ALOGE("%s: Failed to link to depth photo process function: %s", __FUNCTION__,
dlerror());
}
} else {
ALOGE("%s: Failed to link to depth photo library: %s", __FUNCTION__, dlerror());
}
}
}
@ -108,11 +92,6 @@ DepthCompositeStream::~DepthCompositeStream() {
mDepthSurface.clear();
mDepthConsumer = nullptr;
mDepthSurface = nullptr;
if (mDepthPhotoLibHandle != nullptr) {
dlclose(mDepthPhotoLibHandle);
mDepthPhotoLibHandle = nullptr;
}
mDepthPhotoProcess = nullptr;
}
void DepthCompositeStream::compilePendingInputLocked() {
@ -356,7 +335,7 @@ status_t DepthCompositeStream::processInputFrame(nsecs_t ts, const InputFrame &i
}
size_t actualJpegSize = 0;
res = mDepthPhotoProcess(depthPhoto, finalJpegBufferSize, dstBuffer, &actualJpegSize);
res = processDepthPhotoFrame(depthPhoto, finalJpegBufferSize, dstBuffer, &actualJpegSize);
if (res != 0) {
ALOGE("%s: Depth photo processing failed: %s (%d)", __FUNCTION__, strerror(-res), res);
outputANW->cancelBuffer(mOutputSurface.get(), anb, /*fence*/ -1);
@ -583,11 +562,6 @@ status_t DepthCompositeStream::configureStream() {
return NO_ERROR;
}
if ((mDepthPhotoLibHandle == nullptr) || (mDepthPhotoProcess == nullptr)) {
ALOGE("%s: Depth photo library is not present!", __FUNCTION__);
return NO_INIT;
}
if (mOutputSurface.get() == nullptr) {
ALOGE("%s: No valid output surface set!", __FUNCTION__);
return NO_INIT;

2
services/camera/libcameraservice/api2/DepthCompositeStream.h
Unescape View File

@ -126,8 +126,6 @@ private:
std::vector<std::tuple<size_t, size_t>> mSupportedDepthSizes;
std::vector<float> mIntrinsicCalibration, mLensDistortion;
bool mIsLogicalCamera;
void* mDepthPhotoLibHandle;
process_depth_photo_frame mDepthPhotoProcess;
// Keep all incoming Depth buffer timestamps pending further processing.
std::vector<int64_t> mInputDepthBuffers;

30
services/camera/libcameraservice/common/CameraProviderManager.cpp
Unescape View File

@ -716,31 +716,6 @@ void CameraProviderManager::ProviderInfo::DeviceInfo3::getSupportedDynamicDepthS
}
}
bool CameraProviderManager::ProviderInfo::DeviceInfo3::isDepthPhotoLibraryPresent() {
static bool libraryPresent = false;
static bool initialized = false;
if (initialized) {
return libraryPresent;
} else {
initialized = true;
}
void* depthLibHandle = dlopen(camera3::kDepthPhotoLibrary, RTLD_NOW | RTLD_LOCAL);
if (depthLibHandle == nullptr) {
return false;
}
auto processFunc = dlsym(depthLibHandle, camera3::kDepthPhotoProcessFunction);
if (processFunc != nullptr) {
libraryPresent = true;
} else {
libraryPresent = false;
}
dlclose(depthLibHandle);
return libraryPresent;
}
status_t CameraProviderManager::ProviderInfo::DeviceInfo3::addDynamicDepthTags() {
uint32_t depthExclTag = ANDROID_DEPTH_DEPTH_IS_EXCLUSIVE;
uint32_t depthSizesTag = ANDROID_DEPTH_AVAILABLE_DEPTH_STREAM_CONFIGURATIONS;
@ -788,11 +763,6 @@ status_t CameraProviderManager::ProviderInfo::DeviceInfo3::addDynamicDepthTags()
return OK;
}
if(!isDepthPhotoLibraryPresent()) {
// Depth photo processing library is not present, nothing more to do.
return OK;
}
std::vector<int32_t> dynamicDepthEntries;
for (const auto& it : supportedDynamicDepthSizes) {
int32_t entry[4] = {HAL_PIXEL_FORMAT_BLOB, static_cast<int32_t> (std::get<0>(it)),

1
services/camera/libcameraservice/common/CameraProviderManager.h
Unescape View File

@ -545,7 +545,6 @@ private:
void getSupportedDynamicDepthDurations(const std::vector<int64_t>& depthDurations,
const std::vector<int64_t>& blobDurations,
std::vector<int64_t> *dynamicDepthDurations /*out*/);
static bool isDepthPhotoLibraryPresent();
static void getSupportedDynamicDepthSizes(
const std::vector<std::tuple<size_t, size_t>>& blobSizes,
const std::vector<std::tuple<size_t, size_t>>& depthSizes,

2
services/camera/libcameraservice/common/DepthPhotoProcessor.cpp
Unescape View File

@ -410,7 +410,7 @@ std::unique_ptr<dynamic_depth::DepthMap> processDepthMapFrame(DepthPhotoInputFra
return DepthMap::FromData(depthParams, items);
}
extern "C" int processDepthPhotoFrame(DepthPhotoInputFrame inputFrame, size_t depthPhotoBufferSize,
int processDepthPhotoFrame(DepthPhotoInputFrame inputFrame, size_t depthPhotoBufferSize,
void* depthPhotoBuffer /*out*/, size_t* depthPhotoActualSize /*out*/) {
if ((inputFrame.mMainJpegBuffer == nullptr) || (inputFrame.mDepthMapBuffer == nullptr) ||
(depthPhotoBuffer == nullptr) || (depthPhotoActualSize == nullptr)) {

4
services/camera/libcameraservice/common/DepthPhotoProcessor.h
Unescape View File

@ -64,9 +64,7 @@ struct DepthPhotoInputFrame {
mOrientation(DepthPhotoOrientation::DEPTH_ORIENTATION_0_DEGREES) {}
};
static const char *kDepthPhotoLibrary = "libdepthphoto.so";
static const char *kDepthPhotoProcessFunction = "processDepthPhotoFrame";
typedef int (*process_depth_photo_frame) (DepthPhotoInputFrame /*inputFrame*/,
int processDepthPhotoFrame(DepthPhotoInputFrame /*inputFrame*/,
size_t /*depthPhotoBufferSize*/, void* /*depthPhotoBuffer out*/,
size_t* /*depthPhotoActualSize out*/);

83
services/camera/libcameraservice/tests/DepthProcessorTest.cpp
Unescape View File

@ -20,7 +20,6 @@
#include <array>
#include <random>
#include <dlfcn.h>
#include <gtest/gtest.h>
#include "../common/DepthPhotoProcessor.h"
@ -36,19 +35,6 @@ static const size_t kTestBufferNV12Size ((((kTestBufferWidth) * (kTestBufferHeig
static const size_t kTestBufferDepthSize (kTestBufferWidth * kTestBufferHeight);
static const size_t kSeed = 1234;
void linkToDepthPhotoLibrary(void **libHandle /*out*/,
process_depth_photo_frame *processFrameFunc /*out*/) {
ASSERT_NE(libHandle, nullptr);
ASSERT_NE(processFrameFunc, nullptr);
*libHandle = dlopen(kDepthPhotoLibrary, RTLD_NOW | RTLD_LOCAL);
if (*libHandle != nullptr) {
*processFrameFunc = reinterpret_cast<camera3::process_depth_photo_frame> (
dlsym(*libHandle, kDepthPhotoProcessFunction));
ASSERT_NE(*processFrameFunc, nullptr);
}
}
void generateColorJpegBuffer(int jpegQuality, ExifOrientation orientationValue, bool includeExif,
bool switchDimensions, std::vector<uint8_t> *colorJpegBuffer /*out*/) {
ASSERT_NE(colorJpegBuffer, nullptr);
@ -91,26 +77,9 @@ void generateDepth16Buffer(std::array<uint16_t, kTestBufferDepthSize> *depth16Bu
}
}
TEST(DepthProcessorTest, LinkToLibray) {
void *libHandle;
process_depth_photo_frame processFunc;
linkToDepthPhotoLibrary(&libHandle, &processFunc);
if (libHandle != nullptr) {
dlclose(libHandle);
}
}
TEST(DepthProcessorTest, BadInput) {
void *libHandle;
int jpegQuality = 95;
process_depth_photo_frame processFunc;
linkToDepthPhotoLibrary(&libHandle, &processFunc);
if (libHandle == nullptr) {
// Depth library no present, nothing more to test.
return;
}
DepthPhotoInputFrame inputFrame;
// Worst case both depth and confidence maps have the same size as the main color image.
inputFrame.mMaxJpegSize = inputFrame.mMainJpegSize * 3;
@ -128,37 +97,27 @@ TEST(DepthProcessorTest, BadInput) {
inputFrame.mMainJpegWidth = kTestBufferWidth;
inputFrame.mMainJpegHeight = kTestBufferHeight;
inputFrame.mJpegQuality = jpegQuality;
ASSERT_NE(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
ASSERT_NE(processDepthPhotoFrame(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
&actualDepthPhotoSize), 0);
inputFrame.mMainJpegBuffer = reinterpret_cast<const char*> (colorJpegBuffer.data());
inputFrame.mMainJpegSize = colorJpegBuffer.size();
ASSERT_NE(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
ASSERT_NE(processDepthPhotoFrame(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
&actualDepthPhotoSize), 0);
inputFrame.mDepthMapBuffer = depth16Buffer.data();
inputFrame.mDepthMapWidth = inputFrame.mDepthMapStride = kTestBufferWidth;
inputFrame.mDepthMapHeight = kTestBufferHeight;
ASSERT_NE(processFunc(inputFrame, depthPhotoBuffer.size(), nullptr,
ASSERT_NE(processDepthPhotoFrame(inputFrame, depthPhotoBuffer.size(), nullptr,
&actualDepthPhotoSize), 0);
ASSERT_NE(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(), nullptr),
0);
dlclose(libHandle);
ASSERT_NE(processDepthPhotoFrame(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
nullptr), 0);
}
TEST(DepthProcessorTest, BasicDepthPhotoValidation) {
void *libHandle;
int jpegQuality = 95;
process_depth_photo_frame processFunc;
linkToDepthPhotoLibrary(&libHandle, &processFunc);
if (libHandle == nullptr) {
// Depth library no present, nothing more to test.
return;
}
std::vector<uint8_t> colorJpegBuffer;
generateColorJpegBuffer(jpegQuality, ExifOrientation::ORIENTATION_UNDEFINED,
/*includeExif*/ false, /*switchDimensions*/ false, &colorJpegBuffer);
@ -180,7 +139,7 @@ TEST(DepthProcessorTest, BasicDepthPhotoValidation) {
std::vector<uint8_t> depthPhotoBuffer(inputFrame.mMaxJpegSize);
size_t actualDepthPhotoSize = 0;
ASSERT_EQ(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
ASSERT_EQ(processDepthPhotoFrame(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
&actualDepthPhotoSize), 0);
ASSERT_TRUE((actualDepthPhotoSize > 0) && (depthPhotoBuffer.size() >= actualDepthPhotoSize));
@ -196,21 +155,11 @@ TEST(DepthProcessorTest, BasicDepthPhotoValidation) {
ASSERT_EQ(NV12Compressor::findJpegSize(depthPhotoBuffer.data() + mainJpegSize,
actualDepthPhotoSize - mainJpegSize, &depthMapSize), OK);
ASSERT_TRUE((depthMapSize > 0) && (depthMapSize < (actualDepthPhotoSize - mainJpegSize)));
dlclose(libHandle);
}
TEST(DepthProcessorTest, TestDepthPhotoExifOrientation) {
void *libHandle;
int jpegQuality = 95;
process_depth_photo_frame processFunc;
linkToDepthPhotoLibrary(&libHandle, &processFunc);
if (libHandle == nullptr) {
// Depth library no present, nothing more to test.
return;
}
ExifOrientation exifOrientations[] = { ExifOrientation::ORIENTATION_UNDEFINED,
ExifOrientation::ORIENTATION_0_DEGREES, ExifOrientation::ORIENTATION_90_DEGREES,
ExifOrientation::ORIENTATION_180_DEGREES, ExifOrientation::ORIENTATION_270_DEGREES };
@ -242,8 +191,8 @@ TEST(DepthProcessorTest, TestDepthPhotoExifOrientation) {
std::vector<uint8_t> depthPhotoBuffer(inputFrame.mMaxJpegSize);
size_t actualDepthPhotoSize = 0;
ASSERT_EQ(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
&actualDepthPhotoSize), 0);
ASSERT_EQ(processDepthPhotoFrame(inputFrame, depthPhotoBuffer.size(),
depthPhotoBuffer.data(), &actualDepthPhotoSize), 0);
ASSERT_TRUE((actualDepthPhotoSize > 0) &&
(depthPhotoBuffer.size() >= actualDepthPhotoSize));
@ -281,21 +230,11 @@ TEST(DepthProcessorTest, TestDepthPhotoExifOrientation) {
ASSERT_EQ(confidenceJpegExifOrientation, exifOrientation);
}
}
dlclose(libHandle);
}
TEST(DepthProcessorTest, TestDephtPhotoPhysicalRotation) {
void *libHandle;
int jpegQuality = 95;
process_depth_photo_frame processFunc;
linkToDepthPhotoLibrary(&libHandle, &processFunc);
if (libHandle == nullptr) {
// Depth library no present, nothing more to test.
return;
}
// In case of physical rotation, the EXIF orientation must always be 0.
auto exifOrientation = ExifOrientation::ORIENTATION_0_DEGREES;
DepthPhotoOrientation depthOrientations[] = {
@ -339,8 +278,8 @@ TEST(DepthProcessorTest, TestDephtPhotoPhysicalRotation) {
std::vector<uint8_t> depthPhotoBuffer(inputFrame.mMaxJpegSize);
size_t actualDepthPhotoSize = 0;
ASSERT_EQ(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
&actualDepthPhotoSize), 0);
ASSERT_EQ(processDepthPhotoFrame(inputFrame, depthPhotoBuffer.size(),
depthPhotoBuffer.data(), &actualDepthPhotoSize), 0);
ASSERT_TRUE((actualDepthPhotoSize > 0) &&
(depthPhotoBuffer.size() >= actualDepthPhotoSize));
@ -377,6 +316,4 @@ TEST(DepthProcessorTest, TestDephtPhotoPhysicalRotation) {
ASSERT_EQ(confidenceMapWidth, expectedWidth);
ASSERT_EQ(confidenceMapHeight, expectedHeight);
}
dlclose(libHandle);
}

Write Preview
Loading…
Cancel
Save