From cbf174b59307290e4c80947232feedb4cc9c0c26 Mon Sep 17 00:00:00 2001 From: Emilian Peev Date: Fri, 25 Jan 2019 14:38:59 -0800 Subject: [PATCH] Camera: Link dynamically to Depth photo library Move all depth photo specific processing in a separate library and link to it dynamically. Bug: 109735087 Test: Camera CTS Change-Id: I00a20b26fc9a1d127ad962a36b5b554dd36f0d41 --- services/camera/libcameraservice/Android.bp | 39 +- .../api2/DepthCompositeStream.cpp | 390 ++++-------------- .../api2/DepthCompositeStream.h | 3 + .../common/CameraProviderManager.cpp | 32 ++ .../common/CameraProviderManager.h | 1 + .../common/DepthPhotoProcessor.cpp | 340 +++++++++++++++ .../common/DepthPhotoProcessor.h | 67 +++ 7 files changed, 562 insertions(+), 310 deletions(-) create mode 100644 services/camera/libcameraservice/common/DepthPhotoProcessor.cpp create mode 100644 services/camera/libcameraservice/common/DepthPhotoProcessor.h diff --git a/services/camera/libcameraservice/Android.bp b/services/camera/libcameraservice/Android.bp index 3b362d6707..a090479183 100644 --- a/services/camera/libcameraservice/Android.bp +++ b/services/camera/libcameraservice/Android.bp @@ -67,9 +67,7 @@ cc_library_shared { ], shared_libs: [ - "libimage_io", - "libdynamic_depth", - "libxml2", + "libdl", "libui", "liblog", "libutilscallstack", @@ -127,3 +125,38 @@ cc_library_shared { ], } + +cc_library_shared { + name: "libdepthphoto", + + srcs: [ + "common/DepthPhotoProcessor.cpp", + ], + + shared_libs: [ + "libimage_io", + "libdynamic_depth", + "libxml2", + "liblog", + "libutilscallstack", + "libutils", + "libcutils", + "libjpeg", + "libmemunreachable", + ], + + include_dirs: [ + "external/dynamic_depth/includes", + "external/dynamic_depth/internal", + ], + + export_include_dirs: ["."], + + cflags: [ + "-Wall", + "-Wextra", + "-Werror", + "-Wno-ignored-qualifiers", + ], + +} diff --git a/services/camera/libcameraservice/api2/DepthCompositeStream.cpp b/services/camera/libcameraservice/api2/DepthCompositeStream.cpp index b12bb50f09..1cf9529476 100644 --- a/services/camera/libcameraservice/api2/DepthCompositeStream.cpp +++ b/services/camera/libcameraservice/api2/DepthCompositeStream.cpp @@ -20,45 +20,13 @@ #include "api1/client2/JpegProcessor.h" #include "common/CameraProviderManager.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - +#include "dlfcn.h" #include #include #include #include "DepthCompositeStream.h" -using dynamic_depth::Camera; -using dynamic_depth::Cameras; -using dynamic_depth::CameraParams; -using dynamic_depth::Container; -using dynamic_depth::DepthFormat; -using dynamic_depth::DepthMapParams; -using dynamic_depth::DepthUnits; -using dynamic_depth::Device; -using dynamic_depth::DeviceParams; -using dynamic_depth::Dimension; -using dynamic_depth::Image; -using dynamic_depth::ImagingModelParams; -using dynamic_depth::Pose; -using dynamic_depth::Profile; -using dynamic_depth::Profiles; - namespace android { namespace camera3 { @@ -75,7 +43,9 @@ DepthCompositeStream::DepthCompositeStream(wp device, mBlobBufferAcquired(false), mProducerListener(new ProducerListener()), mMaxJpegSize(-1), - mIsLogicalCamera(false) { + mIsLogicalCamera(false), + mDepthPhotoLibHandle(nullptr), + mDepthPhotoProcess(nullptr) { sp cameraDevice = device.promote(); if (cameraDevice.get() != nullptr) { CameraMetadata staticInfo = cameraDevice->info(); @@ -113,6 +83,19 @@ DepthCompositeStream::DepthCompositeStream(wp device, } getSupportedDepthSizes(staticInfo, &mSupportedDepthSizes); + + mDepthPhotoLibHandle = dlopen(camera3::kDepthPhotoLibrary, RTLD_NOW | RTLD_LOCAL); + if (mDepthPhotoLibHandle != nullptr) { + mDepthPhotoProcess = reinterpret_cast ( + dlsym(mDepthPhotoLibHandle, camera3::kDepthPhotoProcessFunction)); + if (mDepthPhotoProcess == nullptr) { + ALOGE("%s: Failed to link to depth photo process function: %s", __FUNCTION__, + dlerror()); + } + } else { + ALOGE("%s: Failed to link to depth photo library: %s", __FUNCTION__, dlerror()); + } + } } @@ -125,6 +108,11 @@ DepthCompositeStream::~DepthCompositeStream() { mDepthSurface.clear(); mDepthConsumer = nullptr; mDepthSurface = nullptr; + if (mDepthPhotoLibHandle != nullptr) { + dlclose(mDepthPhotoLibHandle); + mDepthPhotoLibHandle = nullptr; + } + mDepthPhotoProcess = nullptr; } void DepthCompositeStream::compilePendingInputLocked() { @@ -259,200 +247,12 @@ int64_t DepthCompositeStream::getNextFailingInputLocked(int64_t *currentTs /*ino return ret; } -status_t DepthCompositeStream::encodeGrayscaleJpeg(size_t width, size_t height, uint8_t *in, - void *out, const size_t maxOutSize, uint8_t jpegQuality, size_t &actualSize) { - status_t ret; - // libjpeg is a C library so we use C-style "inheritance" by - // putting libjpeg's jpeg_destination_mgr first in our custom - // struct. This allows us to cast jpeg_destination_mgr* to - // CustomJpegDestMgr* when we get it passed to us in a callback. - struct CustomJpegDestMgr : public jpeg_destination_mgr { - JOCTET *mBuffer; - size_t mBufferSize; - size_t mEncodedSize; - bool mSuccess; - } dmgr; - - jpeg_compress_struct cinfo = {}; - jpeg_error_mgr jerr; - - // Initialize error handling with standard callbacks, but - // then override output_message (to print to ALOG) and - // error_exit to set a flag and print a message instead - // of killing the whole process. - cinfo.err = jpeg_std_error(&jerr); - - cinfo.err->output_message = [](j_common_ptr cinfo) { - char buffer[JMSG_LENGTH_MAX]; - - /* Create the message */ - (*cinfo->err->format_message)(cinfo, buffer); - ALOGE("libjpeg error: %s", buffer); - }; - - cinfo.err->error_exit = [](j_common_ptr cinfo) { - (*cinfo->err->output_message)(cinfo); - if(cinfo->client_data) { - auto & dmgr = *static_cast(cinfo->client_data); - dmgr.mSuccess = false; - } - }; - - // Now that we initialized some callbacks, let's create our compressor - jpeg_create_compress(&cinfo); - dmgr.mBuffer = static_cast(out); - dmgr.mBufferSize = maxOutSize; - dmgr.mEncodedSize = 0; - dmgr.mSuccess = true; - cinfo.client_data = static_cast(&dmgr); - - // These lambdas become C-style function pointers and as per C++11 spec - // may not capture anything. - dmgr.init_destination = [](j_compress_ptr cinfo) { - auto & dmgr = static_cast(*cinfo->dest); - dmgr.next_output_byte = dmgr.mBuffer; - dmgr.free_in_buffer = dmgr.mBufferSize; - ALOGV("%s:%d jpeg start: %p [%zu]", - __FUNCTION__, __LINE__, dmgr.mBuffer, dmgr.mBufferSize); - }; - - dmgr.empty_output_buffer = [](j_compress_ptr cinfo __unused) { - ALOGV("%s:%d Out of buffer", __FUNCTION__, __LINE__); - return 0; - }; - - dmgr.term_destination = [](j_compress_ptr cinfo) { - auto & dmgr = static_cast(*cinfo->dest); - dmgr.mEncodedSize = dmgr.mBufferSize - dmgr.free_in_buffer; - ALOGV("%s:%d Done with jpeg: %zu", __FUNCTION__, __LINE__, dmgr.mEncodedSize); - }; - cinfo.dest = reinterpret_cast(&dmgr); - cinfo.image_width = width; - cinfo.image_height = height; - cinfo.input_components = 1; - cinfo.in_color_space = JCS_GRAYSCALE; - - // Initialize defaults and then override what we want - jpeg_set_defaults(&cinfo); - - jpeg_set_quality(&cinfo, jpegQuality, 1); - jpeg_set_colorspace(&cinfo, JCS_GRAYSCALE); - cinfo.raw_data_in = 0; - cinfo.dct_method = JDCT_IFAST; - - cinfo.comp_info[0].h_samp_factor = 1; - cinfo.comp_info[1].h_samp_factor = 1; - cinfo.comp_info[2].h_samp_factor = 1; - cinfo.comp_info[0].v_samp_factor = 1; - cinfo.comp_info[1].v_samp_factor = 1; - cinfo.comp_info[2].v_samp_factor = 1; - - jpeg_start_compress(&cinfo, TRUE); - - for (size_t i = 0; i < cinfo.image_height; i++) { - auto currentRow = static_cast(in + i*width); - jpeg_write_scanlines(&cinfo, ¤tRow, /*num_lines*/1); - } - - jpeg_finish_compress(&cinfo); - - actualSize = dmgr.mEncodedSize; - if (dmgr.mSuccess) { - ret = NO_ERROR; - } else { - ret = UNKNOWN_ERROR; - } - - return ret; -} - -std::unique_ptr DepthCompositeStream::processDepthMapFrame( - const CpuConsumer::LockedBuffer &depthMapBuffer, size_t maxJpegSize, uint8_t jpegQuality, - std::vector> *items /*out*/) { - std::vector points, confidence; - - size_t pointCount = depthMapBuffer.width * depthMapBuffer.height; - points.reserve(pointCount); - confidence.reserve(pointCount); - float near = UINT16_MAX; - float far = .0f; - uint16_t *data = reinterpret_cast (depthMapBuffer.data); - for (size_t i = 0; i < depthMapBuffer.height; i++) { - for (size_t j = 0; j < depthMapBuffer.width; j++) { - // Android densely packed depth map. The units for the range are in - // millimeters and need to be scaled to meters. - // The confidence value is encoded in the 3 most significant bits. - // The confidence data needs to be additionally normalized with - // values 1.0f, 0.0f representing maximum and minimum confidence - // respectively. - auto value = data[i*depthMapBuffer.stride + j]; - auto point = static_cast(value & 0x1FFF) / 1000.f; - points.push_back(point); - - auto conf = (value >> 13) & 0x7; - float normConfidence = (conf == 0) ? 1.f : (static_cast(conf) - 1) / 7.f; - confidence.push_back(normConfidence); - - if (near > point) { - near = point; - } - if (far < point) { - far = point; - } - } - } - - if (near == far) { - ALOGE("%s: Near and far range values must not match!", __FUNCTION__); - return nullptr; - } - - std::vector pointsQuantized, confidenceQuantized; - pointsQuantized.reserve(pointCount); confidenceQuantized.reserve(pointCount); - auto pointIt = points.begin(); - auto confidenceIt = confidence.begin(); - while ((pointIt != points.end()) && (confidenceIt != confidence.end())) { - pointsQuantized.push_back(floorf(((far * (*pointIt - near)) / - (*pointIt * (far - near))) * 255.0f)); - confidenceQuantized.push_back(floorf(*confidenceIt * 255.0f)); - confidenceIt++; pointIt++; - } - - DepthMapParams depthParams(DepthFormat::kRangeInverse, near, far, DepthUnits::kMeters, - "android/depthmap"); - depthParams.confidence_uri = "android/confidencemap"; - depthParams.mime = "image/jpeg"; - depthParams.depth_image_data.resize(maxJpegSize); - depthParams.confidence_data.resize(maxJpegSize); - size_t actualJpegSize; - auto ret = encodeGrayscaleJpeg(depthMapBuffer.width, depthMapBuffer.height, - pointsQuantized.data(), depthParams.depth_image_data.data(), maxJpegSize, jpegQuality, - actualJpegSize); - if (ret != NO_ERROR) { - ALOGE("%s: Depth map compression failed!", __FUNCTION__); - return nullptr; - } - depthParams.depth_image_data.resize(actualJpegSize); - - ret = encodeGrayscaleJpeg(depthMapBuffer.width, depthMapBuffer.height, - confidenceQuantized.data(), depthParams.confidence_data.data(), maxJpegSize, - jpegQuality, actualJpegSize); - if (ret != NO_ERROR) { - ALOGE("%s: Confidence map compression failed!", __FUNCTION__); - return nullptr; - } - depthParams.confidence_data.resize(actualJpegSize); - - return DepthMap::FromData(depthParams, items); -} - status_t DepthCompositeStream::processInputFrame(const InputFrame &inputFrame) { status_t res; sp outputANW = mOutputSurface; ANativeWindowBuffer *anb; int fenceFd; void *dstBuffer; - auto imgBuffer = inputFrame.jpegBuffer; auto jpegSize = android::camera2::JpegProcessor::findJpegSize(inputFrame.jpegBuffer.data, inputFrame.jpegBuffer.width); @@ -461,15 +261,6 @@ status_t DepthCompositeStream::processInputFrame(const InputFrame &inputFrame) { jpegSize = inputFrame.jpegBuffer.width; } - std::vector> items; - std::vector> cameraList; - auto image = Image::FromDataForPrimaryImage("android/mainimage", &items); - std::unique_ptr cameraParams(new CameraParams(std::move(image))); - if (cameraParams == nullptr) { - ALOGE("%s: Failed to initialize camera parameters", __FUNCTION__); - return BAD_VALUE; - } - size_t maxDepthJpegSize; if (mMaxJpegSize > 0) { maxDepthJpegSize = mMaxJpegSize; @@ -482,49 +273,16 @@ status_t DepthCompositeStream::processInputFrame(const InputFrame &inputFrame) { if (entry.count > 0) { jpegQuality = entry.data.u8[0]; } - cameraParams->depth_map = processDepthMapFrame(inputFrame.depthBuffer, maxDepthJpegSize, - jpegQuality, &items); - if (cameraParams->depth_map == nullptr) { - ALOGE("%s: Depth map processing failed!", __FUNCTION__); - return BAD_VALUE; - } - cameraParams->imaging_model = getImagingModel(); - - if (mIsLogicalCamera) { - cameraParams->trait = dynamic_depth::CameraTrait::LOGICAL; - } else { - cameraParams->trait = dynamic_depth::CameraTrait::PHYSICAL; - } - cameraList.emplace_back(Camera::FromData(std::move(cameraParams))); + // The final depth photo will consist of the main jpeg buffer, the depth map buffer (also in + // jpeg format) and confidence map (jpeg as well). Assume worst case that all 3 jpeg need + // max jpeg size. + size_t finalJpegBufferSize = maxDepthJpegSize * 3; - auto deviceParams = std::make_unique (Cameras::FromCameraArray(&cameraList)); - deviceParams->container = Container::FromItems(&items); - std::vector> profileList; - profileList.emplace_back(Profile::FromData("DepthPhoto", {0})); - deviceParams->profiles = Profiles::FromProfileArray(&profileList); - std::unique_ptr device = Device::FromData(std::move(deviceParams)); - if (device == nullptr) { - ALOGE("%s: Failed to initialize camera device", __FUNCTION__); - return BAD_VALUE; - } - - std::istringstream inputJpegStream(std::string(reinterpret_cast (imgBuffer.data), - jpegSize)); - std::ostringstream outputJpegStream; - if (!WriteImageAndMetadataAndContainer(&inputJpegStream, device.get(), &outputJpegStream)) { - ALOGE("%s: Failed writing depth output", __FUNCTION__); - return BAD_VALUE; - } - - size_t finalJpegSize = static_cast (outputJpegStream.tellp()) + - sizeof(struct camera3_jpeg_blob); - - ALOGV("%s: Final jpeg size: %zu", __func__, finalJpegSize); - if ((res = native_window_set_buffers_dimensions(mOutputSurface.get(), finalJpegSize, 1)) + if ((res = native_window_set_buffers_dimensions(mOutputSurface.get(), finalJpegBufferSize, 1)) != OK) { ALOGE("%s: Unable to configure stream buffer dimensions" - " %zux%u for stream %d", __FUNCTION__, finalJpegSize, 1U, mBlobStreamId); + " %zux%u for stream %d", __FUNCTION__, finalJpegBufferSize, 1U, mBlobStreamId); return res; } @@ -544,20 +302,65 @@ status_t DepthCompositeStream::processInputFrame(const InputFrame &inputFrame) { return res; } - if ((gb->getWidth() < finalJpegSize) || (gb->getHeight() != 1)) { + if ((gb->getWidth() < finalJpegBufferSize) || (gb->getHeight() != 1)) { ALOGE("%s: Blob buffer size mismatch, expected %dx%d received %zux%u", __FUNCTION__, - gb->getWidth(), gb->getHeight(), finalJpegSize, 1U); + gb->getWidth(), gb->getHeight(), finalJpegBufferSize, 1U); outputANW->cancelBuffer(mOutputSurface.get(), anb, /*fence*/ -1); return BAD_VALUE; } - // Copy final jpeg with embedded depth data in the composite stream output buffer + DepthPhotoInputFrame depthPhoto; + depthPhoto.mMainJpegBuffer = reinterpret_cast (inputFrame.jpegBuffer.data); + depthPhoto.mMainJpegWidth = mBlobWidth; + depthPhoto.mMainJpegHeight = mBlobHeight; + depthPhoto.mMainJpegSize = jpegSize; + depthPhoto.mDepthMapBuffer = reinterpret_cast (inputFrame.depthBuffer.data); + depthPhoto.mDepthMapWidth = inputFrame.depthBuffer.width; + depthPhoto.mDepthMapHeight = inputFrame.depthBuffer.height; + depthPhoto.mDepthMapStride = inputFrame.depthBuffer.stride; + depthPhoto.mJpegQuality = jpegQuality; + depthPhoto.mIsLogical = mIsLogicalCamera; + depthPhoto.mMaxJpegSize = maxDepthJpegSize; + // The camera intrinsic calibration layout is as follows: + // [focalLengthX, focalLengthY, opticalCenterX, opticalCenterY, skew] + if (mInstrinsicCalibration.size() == 5) { + memcpy(depthPhoto.mInstrinsicCalibration, mInstrinsicCalibration.data(), + sizeof(depthPhoto.mInstrinsicCalibration)); + depthPhoto.mIsInstrinsicCalibrationValid = 1; + } else { + depthPhoto.mIsInstrinsicCalibrationValid = 0; + } + // The camera lens distortion contains the following lens correction coefficients. + // [kappa_1, kappa_2, kappa_3 kappa_4, kappa_5] + if (mLensDistortion.size() == 5) { + memcpy(depthPhoto.mLensDistortion, mLensDistortion.data(), + sizeof(depthPhoto.mLensDistortion)); + depthPhoto.mIsLensDistortionValid = 1; + } else { + depthPhoto.mIsLensDistortionValid = 0; + } + + size_t actualJpegSize = 0; + res = mDepthPhotoProcess(depthPhoto, finalJpegBufferSize, dstBuffer, &actualJpegSize); + if (res != 0) { + ALOGE("%s: Depth photo processing failed: %s (%d)", __FUNCTION__, strerror(-res), res); + outputANW->cancelBuffer(mOutputSurface.get(), anb, /*fence*/ -1); + return res; + } + + size_t finalJpegSize = actualJpegSize + sizeof(struct camera3_jpeg_blob); + if (finalJpegSize > finalJpegBufferSize) { + ALOGE("%s: Final jpeg buffer not large enough for the jpeg blob header", __FUNCTION__); + outputANW->cancelBuffer(mOutputSurface.get(), anb, /*fence*/ -1); + return NO_MEMORY; + } + + ALOGV("%s: Final jpeg size: %zu", __func__, finalJpegSize); uint8_t* header = static_cast (dstBuffer) + (gb->getWidth() - sizeof(struct camera3_jpeg_blob)); struct camera3_jpeg_blob *blob = reinterpret_cast (header); blob->jpeg_blob_id = CAMERA3_JPEG_BLOB_ID; - blob->jpeg_size = static_cast (outputJpegStream.tellp()); - memcpy(dstBuffer, outputJpegStream.str().c_str(), blob->jpeg_size); + blob->jpeg_size = actualJpegSize; outputANW->queueBuffer(mOutputSurface.get(), anb, /*fence*/ -1); return res; @@ -758,6 +561,11 @@ status_t DepthCompositeStream::configureStream() { return NO_ERROR; } + if ((mDepthPhotoLibHandle == nullptr) || (mDepthPhotoProcess == nullptr)) { + ALOGE("%s: Depth photo library is not present!", __FUNCTION__); + return NO_INIT; + } + if (mOutputSurface.get() == nullptr) { ALOGE("%s: No valid output surface set!", __FUNCTION__); return NO_INIT; @@ -990,37 +798,5 @@ status_t DepthCompositeStream::getCompositeStreamInfo(const OutputStreamInfo &st return NO_ERROR; } -std::unique_ptr DepthCompositeStream::getImagingModel() { - // It is not possible to generate an imaging model without instrinsic calibration. - if (mInstrinsicCalibration.empty() || mInstrinsicCalibration.size() != 5) { - return nullptr; - } - - // The camera intrinsic calibration layout is as follows: - // [focalLengthX, focalLengthY, opticalCenterX, opticalCenterY, skew] - const dynamic_depth::Point focalLength(mInstrinsicCalibration[0], - mInstrinsicCalibration[1]); - const Dimension imageSize(mBlobWidth, mBlobHeight); - ImagingModelParams params(focalLength, imageSize); - params.principal_point.x = mInstrinsicCalibration[2]; - params.principal_point.y = mInstrinsicCalibration[3]; - params.skew = mInstrinsicCalibration[4]; - - // The camera lens distortion contains the following lens correction coefficients. - // [kappa_1, kappa_2, kappa_3 kappa_4, kappa_5] - if (mLensDistortion.size() == 5) { - // According to specification the lens distortion coefficients should be ordered - // as [1, kappa_4, kappa_1, kappa_5, kappa_2, 0, kappa_3, 0] - float distortionData[] = {1.f, mLensDistortion[3], mLensDistortion[0], mLensDistortion[4], - mLensDistortion[1], 0.f, mLensDistortion[2], 0.f}; - auto distortionDataLength = sizeof(distortionData) / sizeof(distortionData[0]); - params.distortion.reserve(distortionDataLength); - params.distortion.insert(params.distortion.end(), distortionData, - distortionData + distortionDataLength); - } - - return ImagingModel::FromData(params); -} - }; // namespace camera3 }; // namespace android diff --git a/services/camera/libcameraservice/api2/DepthCompositeStream.h b/services/camera/libcameraservice/api2/DepthCompositeStream.h index 129d5385e6..e8fe517e13 100644 --- a/services/camera/libcameraservice/api2/DepthCompositeStream.h +++ b/services/camera/libcameraservice/api2/DepthCompositeStream.h @@ -17,6 +17,7 @@ #ifndef ANDROID_SERVERS_CAMERA_CAMERA3_DEPTH_COMPOSITE_STREAM_H #define ANDROID_SERVERS_CAMERA_CAMERA3_DEPTH_COMPOSITE_STREAM_H +#include "common/DepthPhotoProcessor.h" #include #include @@ -131,6 +132,8 @@ private: std::vector> mSupportedDepthSizes; std::vector mInstrinsicCalibration, mLensDistortion; bool mIsLogicalCamera; + void* mDepthPhotoLibHandle; + process_depth_photo_frame mDepthPhotoProcess; // Keep all incoming Depth buffer timestamps pending further processing. std::vector mInputDepthBuffers; diff --git a/services/camera/libcameraservice/common/CameraProviderManager.cpp b/services/camera/libcameraservice/common/CameraProviderManager.cpp index d6d34e95f2..3059b0737d 100644 --- a/services/camera/libcameraservice/common/CameraProviderManager.cpp +++ b/services/camera/libcameraservice/common/CameraProviderManager.cpp @@ -24,6 +24,8 @@ #include #include +#include "common/DepthPhotoProcessor.h" +#include #include #include #include @@ -606,6 +608,31 @@ void CameraProviderManager::ProviderInfo::DeviceInfo3::getSupportedDynamicDepthS } } +bool CameraProviderManager::ProviderInfo::DeviceInfo3::isDepthPhotoLibraryPresent() { + static bool libraryPresent = false; + static bool initialized = false; + if (initialized) { + return libraryPresent; + } else { + initialized = true; + } + + void* depthLibHandle = dlopen(camera3::kDepthPhotoLibrary, RTLD_NOW | RTLD_LOCAL); + if (depthLibHandle == nullptr) { + return false; + } + + auto processFunc = dlsym(depthLibHandle, camera3::kDepthPhotoProcessFunction); + if (processFunc != nullptr) { + libraryPresent = true; + } else { + libraryPresent = false; + } + dlclose(depthLibHandle); + + return libraryPresent; +} + status_t CameraProviderManager::ProviderInfo::DeviceInfo3::addDynamicDepthTags() { uint32_t depthExclTag = ANDROID_DEPTH_DEPTH_IS_EXCLUSIVE; uint32_t depthSizesTag = ANDROID_DEPTH_AVAILABLE_DEPTH_STREAM_CONFIGURATIONS; @@ -654,6 +681,11 @@ status_t CameraProviderManager::ProviderInfo::DeviceInfo3::addDynamicDepthTags() return OK; } + if(!isDepthPhotoLibraryPresent()) { + // Depth photo processing library is not present, nothing more to do. + return OK; + } + std::vector dynamicDepthEntries; for (const auto& it : supportedDynamicDepthSizes) { int32_t entry[4] = {HAL_PIXEL_FORMAT_BLOB, static_cast (std::get<0>(it)), diff --git a/services/camera/libcameraservice/common/CameraProviderManager.h b/services/camera/libcameraservice/common/CameraProviderManager.h index 80ec130e98..fbd7d2e8f8 100644 --- a/services/camera/libcameraservice/common/CameraProviderManager.h +++ b/services/camera/libcameraservice/common/CameraProviderManager.h @@ -483,6 +483,7 @@ private: void getSupportedDynamicDepthDurations(const std::vector& depthDurations, const std::vector& blobDurations, std::vector *dynamicDepthDurations /*out*/); + static bool isDepthPhotoLibraryPresent(); static void getSupportedDynamicDepthSizes( const std::vector>& blobSizes, const std::vector>& depthSizes, diff --git a/services/camera/libcameraservice/common/DepthPhotoProcessor.cpp b/services/camera/libcameraservice/common/DepthPhotoProcessor.cpp new file mode 100644 index 0000000000..a945aca13b --- /dev/null +++ b/services/camera/libcameraservice/common/DepthPhotoProcessor.cpp @@ -0,0 +1,340 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "Camera3-DepthPhotoProcessor" +#define ATRACE_TAG ATRACE_TAG_CAMERA +//#define LOG_NDEBUG 0 +// + +#include "DepthPhotoProcessor.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using dynamic_depth::Camera; +using dynamic_depth::Cameras; +using dynamic_depth::CameraParams; +using dynamic_depth::Container; +using dynamic_depth::DepthFormat; +using dynamic_depth::DepthMap; +using dynamic_depth::DepthMapParams; +using dynamic_depth::DepthUnits; +using dynamic_depth::Device; +using dynamic_depth::DeviceParams; +using dynamic_depth::Dimension; +using dynamic_depth::Image; +using dynamic_depth::ImagingModel; +using dynamic_depth::ImagingModelParams; +using dynamic_depth::Item; +using dynamic_depth::Pose; +using dynamic_depth::Profile; +using dynamic_depth::Profiles; + +namespace android { +namespace camera3 { + +status_t encodeGrayscaleJpeg(size_t width, size_t height, uint8_t *in, void *out, + const size_t maxOutSize, uint8_t jpegQuality, size_t &actualSize) { + status_t ret; + // libjpeg is a C library so we use C-style "inheritance" by + // putting libjpeg's jpeg_destination_mgr first in our custom + // struct. This allows us to cast jpeg_destination_mgr* to + // CustomJpegDestMgr* when we get it passed to us in a callback. + struct CustomJpegDestMgr : public jpeg_destination_mgr { + JOCTET *mBuffer; + size_t mBufferSize; + size_t mEncodedSize; + bool mSuccess; + } dmgr; + + jpeg_compress_struct cinfo = {}; + jpeg_error_mgr jerr; + + // Initialize error handling with standard callbacks, but + // then override output_message (to print to ALOG) and + // error_exit to set a flag and print a message instead + // of killing the whole process. + cinfo.err = jpeg_std_error(&jerr); + + cinfo.err->output_message = [](j_common_ptr cinfo) { + char buffer[JMSG_LENGTH_MAX]; + + /* Create the message */ + (*cinfo->err->format_message)(cinfo, buffer); + ALOGE("libjpeg error: %s", buffer); + }; + + cinfo.err->error_exit = [](j_common_ptr cinfo) { + (*cinfo->err->output_message)(cinfo); + if(cinfo->client_data) { + auto & dmgr = *static_cast(cinfo->client_data); + dmgr.mSuccess = false; + } + }; + + // Now that we initialized some callbacks, let's create our compressor + jpeg_create_compress(&cinfo); + dmgr.mBuffer = static_cast(out); + dmgr.mBufferSize = maxOutSize; + dmgr.mEncodedSize = 0; + dmgr.mSuccess = true; + cinfo.client_data = static_cast(&dmgr); + + // These lambdas become C-style function pointers and as per C++11 spec + // may not capture anything. + dmgr.init_destination = [](j_compress_ptr cinfo) { + auto & dmgr = static_cast(*cinfo->dest); + dmgr.next_output_byte = dmgr.mBuffer; + dmgr.free_in_buffer = dmgr.mBufferSize; + ALOGV("%s:%d jpeg start: %p [%zu]", + __FUNCTION__, __LINE__, dmgr.mBuffer, dmgr.mBufferSize); + }; + + dmgr.empty_output_buffer = [](j_compress_ptr cinfo __unused) { + ALOGV("%s:%d Out of buffer", __FUNCTION__, __LINE__); + return 0; + }; + + dmgr.term_destination = [](j_compress_ptr cinfo) { + auto & dmgr = static_cast(*cinfo->dest); + dmgr.mEncodedSize = dmgr.mBufferSize - dmgr.free_in_buffer; + ALOGV("%s:%d Done with jpeg: %zu", __FUNCTION__, __LINE__, dmgr.mEncodedSize); + }; + cinfo.dest = reinterpret_cast(&dmgr); + cinfo.image_width = width; + cinfo.image_height = height; + cinfo.input_components = 1; + cinfo.in_color_space = JCS_GRAYSCALE; + + // Initialize defaults and then override what we want + jpeg_set_defaults(&cinfo); + + jpeg_set_quality(&cinfo, jpegQuality, 1); + jpeg_set_colorspace(&cinfo, JCS_GRAYSCALE); + cinfo.raw_data_in = 0; + cinfo.dct_method = JDCT_IFAST; + + cinfo.comp_info[0].h_samp_factor = 1; + cinfo.comp_info[1].h_samp_factor = 1; + cinfo.comp_info[2].h_samp_factor = 1; + cinfo.comp_info[0].v_samp_factor = 1; + cinfo.comp_info[1].v_samp_factor = 1; + cinfo.comp_info[2].v_samp_factor = 1; + + jpeg_start_compress(&cinfo, TRUE); + + for (size_t i = 0; i < cinfo.image_height; i++) { + auto currentRow = static_cast(in + i*width); + jpeg_write_scanlines(&cinfo, ¤tRow, /*num_lines*/1); + } + + jpeg_finish_compress(&cinfo); + + actualSize = dmgr.mEncodedSize; + if (dmgr.mSuccess) { + ret = NO_ERROR; + } else { + ret = UNKNOWN_ERROR; + } + + return ret; +} + +std::unique_ptr processDepthMapFrame(DepthPhotoInputFrame inputFrame, + std::vector> *items /*out*/) { + std::vector points, confidence; + + size_t pointCount = inputFrame.mDepthMapWidth * inputFrame.mDepthMapHeight; + points.reserve(pointCount); + confidence.reserve(pointCount); + float near = UINT16_MAX; + float far = .0f; + for (size_t i = 0; i < inputFrame.mDepthMapHeight; i++) { + for (size_t j = 0; j < inputFrame.mDepthMapWidth; j++) { + // Android densely packed depth map. The units for the range are in + // millimeters and need to be scaled to meters. + // The confidence value is encoded in the 3 most significant bits. + // The confidence data needs to be additionally normalized with + // values 1.0f, 0.0f representing maximum and minimum confidence + // respectively. + auto value = inputFrame.mDepthMapBuffer[i*inputFrame.mDepthMapStride + j]; + auto point = static_cast(value & 0x1FFF) / 1000.f; + points.push_back(point); + + auto conf = (value >> 13) & 0x7; + float normConfidence = (conf == 0) ? 1.f : (static_cast(conf) - 1) / 7.f; + confidence.push_back(normConfidence); + + if (near > point) { + near = point; + } + if (far < point) { + far = point; + } + } + } + + if (near == far) { + ALOGE("%s: Near and far range values must not match!", __FUNCTION__); + return nullptr; + } + + std::vector pointsQuantized, confidenceQuantized; + pointsQuantized.reserve(pointCount); confidenceQuantized.reserve(pointCount); + auto pointIt = points.begin(); + auto confidenceIt = confidence.begin(); + while ((pointIt != points.end()) && (confidenceIt != confidence.end())) { + pointsQuantized.push_back(floorf(((far * (*pointIt - near)) / + (*pointIt * (far - near))) * 255.0f)); + confidenceQuantized.push_back(floorf(*confidenceIt * 255.0f)); + confidenceIt++; pointIt++; + } + + DepthMapParams depthParams(DepthFormat::kRangeInverse, near, far, DepthUnits::kMeters, + "android/depthmap"); + depthParams.confidence_uri = "android/confidencemap"; + depthParams.mime = "image/jpeg"; + depthParams.depth_image_data.resize(inputFrame.mMaxJpegSize); + depthParams.confidence_data.resize(inputFrame.mMaxJpegSize); + size_t actualJpegSize; + auto ret = encodeGrayscaleJpeg(inputFrame.mDepthMapWidth, inputFrame.mDepthMapHeight, + pointsQuantized.data(), depthParams.depth_image_data.data(), inputFrame.mMaxJpegSize, + inputFrame.mJpegQuality, actualJpegSize); + if (ret != NO_ERROR) { + ALOGE("%s: Depth map compression failed!", __FUNCTION__); + return nullptr; + } + depthParams.depth_image_data.resize(actualJpegSize); + + ret = encodeGrayscaleJpeg(inputFrame.mDepthMapWidth, inputFrame.mDepthMapHeight, + confidenceQuantized.data(), depthParams.confidence_data.data(), inputFrame.mMaxJpegSize, + inputFrame.mJpegQuality, actualJpegSize); + if (ret != NO_ERROR) { + ALOGE("%s: Confidence map compression failed!", __FUNCTION__); + return nullptr; + } + depthParams.confidence_data.resize(actualJpegSize); + + return DepthMap::FromData(depthParams, items); +} + +extern "C" int processDepthPhotoFrame(DepthPhotoInputFrame inputFrame, size_t depthPhotoBufferSize, + void* depthPhotoBuffer /*out*/, size_t* depthPhotoActualSize /*out*/) { + if ((inputFrame.mMainJpegBuffer == nullptr) || (inputFrame.mDepthMapBuffer == nullptr) || + (depthPhotoBuffer == nullptr) || (depthPhotoActualSize == nullptr)) { + return BAD_VALUE; + } + + std::vector> items; + std::vector> cameraList; + auto image = Image::FromDataForPrimaryImage("android/mainimage", &items); + std::unique_ptr cameraParams(new CameraParams(std::move(image))); + if (cameraParams == nullptr) { + ALOGE("%s: Failed to initialize camera parameters", __FUNCTION__); + return BAD_VALUE; + } + + cameraParams->depth_map = processDepthMapFrame(inputFrame, &items); + if (cameraParams->depth_map == nullptr) { + ALOGE("%s: Depth map processing failed!", __FUNCTION__); + return BAD_VALUE; + } + + // It is not possible to generate an imaging model without instrinsic calibration. + if (inputFrame.mIsInstrinsicCalibrationValid) { + // The camera intrinsic calibration layout is as follows: + // [focalLengthX, focalLengthY, opticalCenterX, opticalCenterY, skew] + const dynamic_depth::Point focalLength(inputFrame.mInstrinsicCalibration[0], + inputFrame.mInstrinsicCalibration[1]); + const Dimension imageSize(inputFrame.mMainJpegWidth, inputFrame.mMainJpegHeight); + ImagingModelParams imagingParams(focalLength, imageSize); + imagingParams.principal_point.x = inputFrame.mInstrinsicCalibration[2]; + imagingParams.principal_point.y = inputFrame.mInstrinsicCalibration[3]; + imagingParams.skew = inputFrame.mInstrinsicCalibration[4]; + + // The camera lens distortion contains the following lens correction coefficients. + // [kappa_1, kappa_2, kappa_3 kappa_4, kappa_5] + if (inputFrame.mIsLensDistortionValid) { + // According to specification the lens distortion coefficients should be ordered + // as [1, kappa_4, kappa_1, kappa_5, kappa_2, 0, kappa_3, 0] + float distortionData[] = {1.f, inputFrame.mLensDistortion[3], + inputFrame.mLensDistortion[0], inputFrame.mLensDistortion[4], + inputFrame.mLensDistortion[1], 0.f, inputFrame.mLensDistortion[2], 0.f}; + auto distortionDataLength = sizeof(distortionData) / sizeof(distortionData[0]); + imagingParams.distortion.reserve(distortionDataLength); + imagingParams.distortion.insert(imagingParams.distortion.end(), distortionData, + distortionData + distortionDataLength); + } + + cameraParams->imaging_model = ImagingModel::FromData(imagingParams); + } + + if (inputFrame.mIsLogical) { + cameraParams->trait = dynamic_depth::CameraTrait::LOGICAL; + } else { + cameraParams->trait = dynamic_depth::CameraTrait::PHYSICAL; + } + + cameraList.emplace_back(Camera::FromData(std::move(cameraParams))); + + auto deviceParams = std::make_unique (Cameras::FromCameraArray(&cameraList)); + deviceParams->container = Container::FromItems(&items); + std::vector> profileList; + profileList.emplace_back(Profile::FromData("DepthPhoto", {0})); + deviceParams->profiles = Profiles::FromProfileArray(&profileList); + std::unique_ptr device = Device::FromData(std::move(deviceParams)); + if (device == nullptr) { + ALOGE("%s: Failed to initialize camera device", __FUNCTION__); + return BAD_VALUE; + } + + std::istringstream inputJpegStream( + std::string(inputFrame.mMainJpegBuffer, inputFrame.mMainJpegSize)); + std::ostringstream outputJpegStream; + if (!WriteImageAndMetadataAndContainer(&inputJpegStream, device.get(), &outputJpegStream)) { + ALOGE("%s: Failed writing depth output", __FUNCTION__); + return BAD_VALUE; + } + + *depthPhotoActualSize = static_cast (outputJpegStream.tellp()); + if (*depthPhotoActualSize > depthPhotoBufferSize) { + ALOGE("%s: Depth photo output buffer not sufficient, needed %zu actual %zu", __FUNCTION__, + *depthPhotoActualSize, depthPhotoBufferSize); + return NO_MEMORY; + } + + memcpy(depthPhotoBuffer, outputJpegStream.str().c_str(), *depthPhotoActualSize); + + return 0; +} + +}; // namespace camera3 +}; // namespace android diff --git a/services/camera/libcameraservice/common/DepthPhotoProcessor.h b/services/camera/libcameraservice/common/DepthPhotoProcessor.h new file mode 100644 index 0000000000..19889a1f3d --- /dev/null +++ b/services/camera/libcameraservice/common/DepthPhotoProcessor.h @@ -0,0 +1,67 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ANDROID_SERVERS_CAMERA_CAMERA3_DEPTH_PROCESSOR_H +#define ANDROID_SERVERS_CAMERA_CAMERA3_DEPTH_PROCESSOR_H + +#include +#include + +namespace android { +namespace camera3 { + +struct DepthPhotoInputFrame { + const char* mMainJpegBuffer; + size_t mMainJpegSize; + size_t mMainJpegWidth, mMainJpegHeight; + uint16_t* mDepthMapBuffer; + size_t mDepthMapWidth, mDepthMapHeight, mDepthMapStride; + size_t mMaxJpegSize; + uint8_t mJpegQuality; + uint8_t mIsLogical; + float mInstrinsicCalibration[5]; + uint8_t mIsInstrinsicCalibrationValid; + float mLensDistortion[5]; + uint8_t mIsLensDistortionValid; + + DepthPhotoInputFrame() : + mMainJpegBuffer(nullptr), + mMainJpegSize(0), + mMainJpegWidth(0), + mMainJpegHeight(0), + mDepthMapBuffer(nullptr), + mDepthMapWidth(0), + mDepthMapHeight(0), + mDepthMapStride(0), + mMaxJpegSize(0), + mJpegQuality(100), + mIsLogical(0), + mInstrinsicCalibration{0.f}, + mIsInstrinsicCalibrationValid(0), + mLensDistortion{0.f}, + mIsLensDistortionValid(0) {} +}; + +static const char *kDepthPhotoLibrary = "libdepthphoto.so"; +static const char *kDepthPhotoProcessFunction = "processDepthPhotoFrame"; +typedef int (*process_depth_photo_frame) (DepthPhotoInputFrame /*inputFrame*/, + size_t /*depthPhotoBufferSize*/, void* /*depthPhotoBuffer out*/, + size_t* /*depthPhotoActualSize out*/); + +}; // namespace camera3 +}; // namespace android + +#endif