gugelfrei
/
android_frameworks_av
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge "Camera: Integrate dynamic depth processing"

Browse Source
gugelfrei
TreeHugger Robot 5 years ago committed by Android (Google) Code Review
parent c60fccb3b3 29e9ec182d
commit 336ffa96f5
8 changed files with 18 additions and 175 deletions
Show Stats Download Patch File Download Diff File

41
services/camera/libcameraservice/Android.bp
Unescape View File

@ -28,6 +28,7 @@ cc_library_shared {
"common/CameraDeviceBase.cpp",
"common/CameraOfflineSessionBase.cpp",
"common/CameraProviderManager.cpp",
"common/DepthPhotoProcessor.cpp",
"common/FrameProcessorBase.cpp",
"api1/CameraClient.cpp",
"api1/Camera2Client.cpp",
@ -91,10 +92,12 @@ cc_library_shared {
"libmediautils",
"libcamera_client",
"libcamera_metadata",
"libdynamic_depth",
"libfmq",
"libgui",
"libhardware",
"libhidlbase",
"libimage_io",
"libjpeg",
"libmedia_codeclist",
"libmedia_omx",
@ -102,6 +105,7 @@ cc_library_shared {
"libsensorprivacy",
"libstagefright",
"libstagefright_foundation",
"libxml2",
"libyuv",
"android.frameworks.cameraservice.common@2.0",
"android.frameworks.cameraservice.service@2.0",
@ -143,40 +147,3 @@ cc_library_shared {
}
cc_library_shared {
name: "libdepthphoto",
srcs: [
"utils/ExifUtils.cpp",
"common/DepthPhotoProcessor.cpp",
],
shared_libs: [
"libimage_io",
"libdynamic_depth",
"libxml2",
"liblog",
"libutilscallstack",
"libutils",
"libcutils",
"libjpeg",
"libmemunreachable",
"libexif",
"libcamera_client",
],
include_dirs: [
"external/dynamic_depth/includes",
"external/dynamic_depth/internal",
],
export_include_dirs: ["."],
cflags: [
"-Wall",
"-Wextra",
"-Werror",
"-Wno-ignored-qualifiers",
],
}

30
services/camera/libcameraservice/api2/DepthCompositeStream.cpp
Unescape View File

@ -20,7 +20,6 @@
#include "api1/client2/JpegProcessor.h"
#include "common/CameraProviderManager.h"
#include "dlfcn.h"
#include <gui/Surface.h>
#include <utils/Log.h>
#include <utils/Trace.h>
@ -43,9 +42,7 @@ DepthCompositeStream::DepthCompositeStream(wp<CameraDeviceBase> device,
mBlobBufferAcquired(false),
mProducerListener(new ProducerListener()),
mMaxJpegSize(-1),
mIsLogicalCamera(false),
mDepthPhotoLibHandle(nullptr),
mDepthPhotoProcess(nullptr) {
mIsLogicalCamera(false) {
sp<CameraDeviceBase> cameraDevice = device.promote();
if (cameraDevice.get() != nullptr) {
CameraMetadata staticInfo = cameraDevice->info();
@ -83,19 +80,6 @@ DepthCompositeStream::DepthCompositeStream(wp<CameraDeviceBase> device,
}
getSupportedDepthSizes(staticInfo, &mSupportedDepthSizes);
mDepthPhotoLibHandle = dlopen(camera3::kDepthPhotoLibrary, RTLD_NOW | RTLD_LOCAL);
if (mDepthPhotoLibHandle != nullptr) {
mDepthPhotoProcess = reinterpret_cast<camera3::process_depth_photo_frame> (
dlsym(mDepthPhotoLibHandle, camera3::kDepthPhotoProcessFunction));
if (mDepthPhotoProcess == nullptr) {
ALOGE("%s: Failed to link to depth photo process function: %s", __FUNCTION__,
dlerror());
}
} else {
ALOGE("%s: Failed to link to depth photo library: %s", __FUNCTION__, dlerror());
}
}
}
@ -108,11 +92,6 @@ DepthCompositeStream::~DepthCompositeStream() {
mDepthSurface.clear();
mDepthConsumer = nullptr;
mDepthSurface = nullptr;
if (mDepthPhotoLibHandle != nullptr) {
dlclose(mDepthPhotoLibHandle);
mDepthPhotoLibHandle = nullptr;
}
mDepthPhotoProcess = nullptr;
}
void DepthCompositeStream::compilePendingInputLocked() {
@ -356,7 +335,7 @@ status_t DepthCompositeStream::processInputFrame(nsecs_t ts, const InputFrame &i
}
size_t actualJpegSize = 0;
res = mDepthPhotoProcess(depthPhoto, finalJpegBufferSize, dstBuffer, &actualJpegSize);
res = processDepthPhotoFrame(depthPhoto, finalJpegBufferSize, dstBuffer, &actualJpegSize);
if (res != 0) {
ALOGE("%s: Depth photo processing failed: %s (%d)", __FUNCTION__, strerror(-res), res);
outputANW->cancelBuffer(mOutputSurface.get(), anb, /*fence*/ -1);
@ -583,11 +562,6 @@ status_t DepthCompositeStream::configureStream() {
return NO_ERROR;
}
if ((mDepthPhotoLibHandle == nullptr) || (mDepthPhotoProcess == nullptr)) {
ALOGE("%s: Depth photo library is not present!", __FUNCTION__);
return NO_INIT;
}
if (mOutputSurface.get() == nullptr) {
ALOGE("%s: No valid output surface set!", __FUNCTION__);
return NO_INIT;

2
services/camera/libcameraservice/api2/DepthCompositeStream.h
Unescape View File

@ -126,8 +126,6 @@ private:
std::vector<std::tuple<size_t, size_t>> mSupportedDepthSizes;
std::vector<float> mIntrinsicCalibration, mLensDistortion;
bool mIsLogicalCamera;
void* mDepthPhotoLibHandle;
process_depth_photo_frame mDepthPhotoProcess;
// Keep all incoming Depth buffer timestamps pending further processing.
std::vector<int64_t> mInputDepthBuffers;

30
services/camera/libcameraservice/common/CameraProviderManager.cpp
Unescape View File

@ -716,31 +716,6 @@ void CameraProviderManager::ProviderInfo::DeviceInfo3::getSupportedDynamicDepthS
}
}
bool CameraProviderManager::ProviderInfo::DeviceInfo3::isDepthPhotoLibraryPresent() {
static bool libraryPresent = false;
static bool initialized = false;
if (initialized) {
return libraryPresent;
} else {
initialized = true;
}
void* depthLibHandle = dlopen(camera3::kDepthPhotoLibrary, RTLD_NOW | RTLD_LOCAL);
if (depthLibHandle == nullptr) {
return false;
}
auto processFunc = dlsym(depthLibHandle, camera3::kDepthPhotoProcessFunction);
if (processFunc != nullptr) {
libraryPresent = true;
} else {
libraryPresent = false;
}
dlclose(depthLibHandle);
return libraryPresent;
}
status_t CameraProviderManager::ProviderInfo::DeviceInfo3::addDynamicDepthTags() {
uint32_t depthExclTag = ANDROID_DEPTH_DEPTH_IS_EXCLUSIVE;
uint32_t depthSizesTag = ANDROID_DEPTH_AVAILABLE_DEPTH_STREAM_CONFIGURATIONS;
@ -788,11 +763,6 @@ status_t CameraProviderManager::ProviderInfo::DeviceInfo3::addDynamicDepthTags()
return OK;
}
if(!isDepthPhotoLibraryPresent()) {
// Depth photo processing library is not present, nothing more to do.
return OK;
}
std::vector<int32_t> dynamicDepthEntries;
for (const auto& it : supportedDynamicDepthSizes) {
int32_t entry[4] = {HAL_PIXEL_FORMAT_BLOB, static_cast<int32_t> (std::get<0>(it)),

1
services/camera/libcameraservice/common/CameraProviderManager.h
Unescape View File

@ -545,7 +545,6 @@ private:
void getSupportedDynamicDepthDurations(const std::vector<int64_t>& depthDurations,
const std::vector<int64_t>& blobDurations,
std::vector<int64_t> *dynamicDepthDurations /*out*/);
static bool isDepthPhotoLibraryPresent();
static void getSupportedDynamicDepthSizes(
const std::vector<std::tuple<size_t, size_t>>& blobSizes,
const std::vector<std::tuple<size_t, size_t>>& depthSizes,

2
services/camera/libcameraservice/common/DepthPhotoProcessor.cpp
Unescape View File

@ -410,7 +410,7 @@ std::unique_ptr<dynamic_depth::DepthMap> processDepthMapFrame(DepthPhotoInputFra
return DepthMap::FromData(depthParams, items);
}
extern "C" int processDepthPhotoFrame(DepthPhotoInputFrame inputFrame, size_t depthPhotoBufferSize,
int processDepthPhotoFrame(DepthPhotoInputFrame inputFrame, size_t depthPhotoBufferSize,
void* depthPhotoBuffer /*out*/, size_t* depthPhotoActualSize /*out*/) {
if ((inputFrame.mMainJpegBuffer == nullptr) || (inputFrame.mDepthMapBuffer == nullptr) ||
(depthPhotoBuffer == nullptr) || (depthPhotoActualSize == nullptr)) {

4
services/camera/libcameraservice/common/DepthPhotoProcessor.h
Unescape View File

@ -64,9 +64,7 @@ struct DepthPhotoInputFrame {
mOrientation(DepthPhotoOrientation::DEPTH_ORIENTATION_0_DEGREES) {}
};
static const char *kDepthPhotoLibrary = "libdepthphoto.so";
static const char *kDepthPhotoProcessFunction = "processDepthPhotoFrame";
typedef int (*process_depth_photo_frame) (DepthPhotoInputFrame /*inputFrame*/,
int processDepthPhotoFrame(DepthPhotoInputFrame /*inputFrame*/,
size_t /*depthPhotoBufferSize*/, void* /*depthPhotoBuffer out*/,
size_t* /*depthPhotoActualSize out*/);

83
services/camera/libcameraservice/tests/DepthProcessorTest.cpp
Unescape View File

@ -20,7 +20,6 @@
#include <array>
#include <random>
#include <dlfcn.h>
#include <gtest/gtest.h>
#include "../common/DepthPhotoProcessor.h"
@ -36,19 +35,6 @@ static const size_t kTestBufferNV12Size ((((kTestBufferWidth) * (kTestBufferHeig
static const size_t kTestBufferDepthSize (kTestBufferWidth * kTestBufferHeight);
static const size_t kSeed = 1234;
void linkToDepthPhotoLibrary(void **libHandle /*out*/,
process_depth_photo_frame *processFrameFunc /*out*/) {
ASSERT_NE(libHandle, nullptr);
ASSERT_NE(processFrameFunc, nullptr);
*libHandle = dlopen(kDepthPhotoLibrary, RTLD_NOW | RTLD_LOCAL);
if (*libHandle != nullptr) {
*processFrameFunc = reinterpret_cast<camera3::process_depth_photo_frame> (
dlsym(*libHandle, kDepthPhotoProcessFunction));
ASSERT_NE(*processFrameFunc, nullptr);
}
}
void generateColorJpegBuffer(int jpegQuality, ExifOrientation orientationValue, bool includeExif,
bool switchDimensions, std::vector<uint8_t> *colorJpegBuffer /*out*/) {
ASSERT_NE(colorJpegBuffer, nullptr);
@ -91,26 +77,9 @@ void generateDepth16Buffer(std::array<uint16_t, kTestBufferDepthSize> *depth16Bu
}
}
TEST(DepthProcessorTest, LinkToLibray) {
void *libHandle;
process_depth_photo_frame processFunc;
linkToDepthPhotoLibrary(&libHandle, &processFunc);
if (libHandle != nullptr) {
dlclose(libHandle);
}
}
TEST(DepthProcessorTest, BadInput) {
void *libHandle;
int jpegQuality = 95;
process_depth_photo_frame processFunc;
linkToDepthPhotoLibrary(&libHandle, &processFunc);
if (libHandle == nullptr) {
// Depth library no present, nothing more to test.
return;
}
DepthPhotoInputFrame inputFrame;
// Worst case both depth and confidence maps have the same size as the main color image.
inputFrame.mMaxJpegSize = inputFrame.mMainJpegSize * 3;
@ -128,37 +97,27 @@ TEST(DepthProcessorTest, BadInput) {
inputFrame.mMainJpegWidth = kTestBufferWidth;
inputFrame.mMainJpegHeight = kTestBufferHeight;
inputFrame.mJpegQuality = jpegQuality;
ASSERT_NE(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
ASSERT_NE(processDepthPhotoFrame(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
&actualDepthPhotoSize), 0);
inputFrame.mMainJpegBuffer = reinterpret_cast<const char*> (colorJpegBuffer.data());
inputFrame.mMainJpegSize = colorJpegBuffer.size();
ASSERT_NE(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
ASSERT_NE(processDepthPhotoFrame(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
&actualDepthPhotoSize), 0);
inputFrame.mDepthMapBuffer = depth16Buffer.data();
inputFrame.mDepthMapWidth = inputFrame.mDepthMapStride = kTestBufferWidth;
inputFrame.mDepthMapHeight = kTestBufferHeight;
ASSERT_NE(processFunc(inputFrame, depthPhotoBuffer.size(), nullptr,
ASSERT_NE(processDepthPhotoFrame(inputFrame, depthPhotoBuffer.size(), nullptr,
&actualDepthPhotoSize), 0);
ASSERT_NE(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(), nullptr),
0);
dlclose(libHandle);
ASSERT_NE(processDepthPhotoFrame(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
nullptr), 0);
}
TEST(DepthProcessorTest, BasicDepthPhotoValidation) {
void *libHandle;
int jpegQuality = 95;
process_depth_photo_frame processFunc;
linkToDepthPhotoLibrary(&libHandle, &processFunc);
if (libHandle == nullptr) {
// Depth library no present, nothing more to test.
return;
}
std::vector<uint8_t> colorJpegBuffer;
generateColorJpegBuffer(jpegQuality, ExifOrientation::ORIENTATION_UNDEFINED,
/*includeExif*/ false, /*switchDimensions*/ false, &colorJpegBuffer);
@ -180,7 +139,7 @@ TEST(DepthProcessorTest, BasicDepthPhotoValidation) {
std::vector<uint8_t> depthPhotoBuffer(inputFrame.mMaxJpegSize);
size_t actualDepthPhotoSize = 0;
ASSERT_EQ(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
ASSERT_EQ(processDepthPhotoFrame(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
&actualDepthPhotoSize), 0);
ASSERT_TRUE((actualDepthPhotoSize > 0) && (depthPhotoBuffer.size() >= actualDepthPhotoSize));
@ -196,21 +155,11 @@ TEST(DepthProcessorTest, BasicDepthPhotoValidation) {
ASSERT_EQ(NV12Compressor::findJpegSize(depthPhotoBuffer.data() + mainJpegSize,
actualDepthPhotoSize - mainJpegSize, &depthMapSize), OK);
ASSERT_TRUE((depthMapSize > 0) && (depthMapSize < (actualDepthPhotoSize - mainJpegSize)));
dlclose(libHandle);
}
TEST(DepthProcessorTest, TestDepthPhotoExifOrientation) {
void *libHandle;
int jpegQuality = 95;
process_depth_photo_frame processFunc;
linkToDepthPhotoLibrary(&libHandle, &processFunc);
if (libHandle == nullptr) {
// Depth library no present, nothing more to test.
return;
}
ExifOrientation exifOrientations[] = { ExifOrientation::ORIENTATION_UNDEFINED,
ExifOrientation::ORIENTATION_0_DEGREES, ExifOrientation::ORIENTATION_90_DEGREES,
ExifOrientation::ORIENTATION_180_DEGREES, ExifOrientation::ORIENTATION_270_DEGREES };
@ -242,8 +191,8 @@ TEST(DepthProcessorTest, TestDepthPhotoExifOrientation) {
std::vector<uint8_t> depthPhotoBuffer(inputFrame.mMaxJpegSize);
size_t actualDepthPhotoSize = 0;
ASSERT_EQ(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
&actualDepthPhotoSize), 0);
ASSERT_EQ(processDepthPhotoFrame(inputFrame, depthPhotoBuffer.size(),
depthPhotoBuffer.data(), &actualDepthPhotoSize), 0);
ASSERT_TRUE((actualDepthPhotoSize > 0) &&
(depthPhotoBuffer.size() >= actualDepthPhotoSize));
@ -281,21 +230,11 @@ TEST(DepthProcessorTest, TestDepthPhotoExifOrientation) {
ASSERT_EQ(confidenceJpegExifOrientation, exifOrientation);
}
}
dlclose(libHandle);
}
TEST(DepthProcessorTest, TestDephtPhotoPhysicalRotation) {
void *libHandle;
int jpegQuality = 95;
process_depth_photo_frame processFunc;
linkToDepthPhotoLibrary(&libHandle, &processFunc);
if (libHandle == nullptr) {
// Depth library no present, nothing more to test.
return;
}
// In case of physical rotation, the EXIF orientation must always be 0.
auto exifOrientation = ExifOrientation::ORIENTATION_0_DEGREES;
DepthPhotoOrientation depthOrientations[] = {
@ -339,8 +278,8 @@ TEST(DepthProcessorTest, TestDephtPhotoPhysicalRotation) {
std::vector<uint8_t> depthPhotoBuffer(inputFrame.mMaxJpegSize);
size_t actualDepthPhotoSize = 0;
ASSERT_EQ(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
&actualDepthPhotoSize), 0);
ASSERT_EQ(processDepthPhotoFrame(inputFrame, depthPhotoBuffer.size(),
depthPhotoBuffer.data(), &actualDepthPhotoSize), 0);
ASSERT_TRUE((actualDepthPhotoSize > 0) &&
(depthPhotoBuffer.size() >= actualDepthPhotoSize));
@ -377,6 +316,4 @@ TEST(DepthProcessorTest, TestDephtPhotoPhysicalRotation) {
ASSERT_EQ(confidenceMapWidth, expectedWidth);
ASSERT_EQ(confidenceMapHeight, expectedHeight);
}
dlclose(libHandle);
}

Write Preview
Loading…
Cancel
Save