Move all depth photo specific processing in a separate library and link to it dynamically. Bug: 109735087 Test: Camera CTS Change-Id: I00a20b26fc9a1d127ad962a36b5b554dd36f0d41gugelfrei
Emilian Peev
6 years ago
parent
538c90e79d
commit
cbf174b593
@ -0,0 +1,340 @@
|
||||
/*
|
||||
* Copyright (C) 2019 The Android Open Source Project
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#define LOG_TAG "Camera3-DepthPhotoProcessor"
|
||||
#define ATRACE_TAG ATRACE_TAG_CAMERA
|
||||
//#define LOG_NDEBUG 0
|
||||
//
|
||||
|
||||
#include "DepthPhotoProcessor.h"
|
||||
|
||||
#include <dynamic_depth/camera.h>
|
||||
#include <dynamic_depth/cameras.h>
|
||||
#include <dynamic_depth/container.h>
|
||||
#include <dynamic_depth/device.h>
|
||||
#include <dynamic_depth/dimension.h>
|
||||
#include <dynamic_depth/dynamic_depth.h>
|
||||
#include <dynamic_depth/point.h>
|
||||
#include <dynamic_depth/pose.h>
|
||||
#include <dynamic_depth/profile.h>
|
||||
#include <dynamic_depth/profiles.h>
|
||||
#include <jpeglib.h>
|
||||
#include <math.h>
|
||||
#include <sstream>
|
||||
#include <utils/Errors.h>
|
||||
#include <utils/Log.h>
|
||||
#include <xmpmeta/xmp_data.h>
|
||||
#include <xmpmeta/xmp_writer.h>
|
||||
|
||||
using dynamic_depth::Camera;
|
||||
using dynamic_depth::Cameras;
|
||||
using dynamic_depth::CameraParams;
|
||||
using dynamic_depth::Container;
|
||||
using dynamic_depth::DepthFormat;
|
||||
using dynamic_depth::DepthMap;
|
||||
using dynamic_depth::DepthMapParams;
|
||||
using dynamic_depth::DepthUnits;
|
||||
using dynamic_depth::Device;
|
||||
using dynamic_depth::DeviceParams;
|
||||
using dynamic_depth::Dimension;
|
||||
using dynamic_depth::Image;
|
||||
using dynamic_depth::ImagingModel;
|
||||
using dynamic_depth::ImagingModelParams;
|
||||
using dynamic_depth::Item;
|
||||
using dynamic_depth::Pose;
|
||||
using dynamic_depth::Profile;
|
||||
using dynamic_depth::Profiles;
|
||||
|
||||
namespace android {
|
||||
namespace camera3 {
|
||||
|
||||
status_t encodeGrayscaleJpeg(size_t width, size_t height, uint8_t *in, void *out,
|
||||
const size_t maxOutSize, uint8_t jpegQuality, size_t &actualSize) {
|
||||
status_t ret;
|
||||
// libjpeg is a C library so we use C-style "inheritance" by
|
||||
// putting libjpeg's jpeg_destination_mgr first in our custom
|
||||
// struct. This allows us to cast jpeg_destination_mgr* to
|
||||
// CustomJpegDestMgr* when we get it passed to us in a callback.
|
||||
struct CustomJpegDestMgr : public jpeg_destination_mgr {
|
||||
JOCTET *mBuffer;
|
||||
size_t mBufferSize;
|
||||
size_t mEncodedSize;
|
||||
bool mSuccess;
|
||||
} dmgr;
|
||||
|
||||
jpeg_compress_struct cinfo = {};
|
||||
jpeg_error_mgr jerr;
|
||||
|
||||
// Initialize error handling with standard callbacks, but
|
||||
// then override output_message (to print to ALOG) and
|
||||
// error_exit to set a flag and print a message instead
|
||||
// of killing the whole process.
|
||||
cinfo.err = jpeg_std_error(&jerr);
|
||||
|
||||
cinfo.err->output_message = [](j_common_ptr cinfo) {
|
||||
char buffer[JMSG_LENGTH_MAX];
|
||||
|
||||
/* Create the message */
|
||||
(*cinfo->err->format_message)(cinfo, buffer);
|
||||
ALOGE("libjpeg error: %s", buffer);
|
||||
};
|
||||
|
||||
cinfo.err->error_exit = [](j_common_ptr cinfo) {
|
||||
(*cinfo->err->output_message)(cinfo);
|
||||
if(cinfo->client_data) {
|
||||
auto & dmgr = *static_cast<CustomJpegDestMgr*>(cinfo->client_data);
|
||||
dmgr.mSuccess = false;
|
||||
}
|
||||
};
|
||||
|
||||
// Now that we initialized some callbacks, let's create our compressor
|
||||
jpeg_create_compress(&cinfo);
|
||||
dmgr.mBuffer = static_cast<JOCTET*>(out);
|
||||
dmgr.mBufferSize = maxOutSize;
|
||||
dmgr.mEncodedSize = 0;
|
||||
dmgr.mSuccess = true;
|
||||
cinfo.client_data = static_cast<void*>(&dmgr);
|
||||
|
||||
// These lambdas become C-style function pointers and as per C++11 spec
|
||||
// may not capture anything.
|
||||
dmgr.init_destination = [](j_compress_ptr cinfo) {
|
||||
auto & dmgr = static_cast<CustomJpegDestMgr&>(*cinfo->dest);
|
||||
dmgr.next_output_byte = dmgr.mBuffer;
|
||||
dmgr.free_in_buffer = dmgr.mBufferSize;
|
||||
ALOGV("%s:%d jpeg start: %p [%zu]",
|
||||
__FUNCTION__, __LINE__, dmgr.mBuffer, dmgr.mBufferSize);
|
||||
};
|
||||
|
||||
dmgr.empty_output_buffer = [](j_compress_ptr cinfo __unused) {
|
||||
ALOGV("%s:%d Out of buffer", __FUNCTION__, __LINE__);
|
||||
return 0;
|
||||
};
|
||||
|
||||
dmgr.term_destination = [](j_compress_ptr cinfo) {
|
||||
auto & dmgr = static_cast<CustomJpegDestMgr&>(*cinfo->dest);
|
||||
dmgr.mEncodedSize = dmgr.mBufferSize - dmgr.free_in_buffer;
|
||||
ALOGV("%s:%d Done with jpeg: %zu", __FUNCTION__, __LINE__, dmgr.mEncodedSize);
|
||||
};
|
||||
cinfo.dest = reinterpret_cast<struct jpeg_destination_mgr*>(&dmgr);
|
||||
cinfo.image_width = width;
|
||||
cinfo.image_height = height;
|
||||
cinfo.input_components = 1;
|
||||
cinfo.in_color_space = JCS_GRAYSCALE;
|
||||
|
||||
// Initialize defaults and then override what we want
|
||||
jpeg_set_defaults(&cinfo);
|
||||
|
||||
jpeg_set_quality(&cinfo, jpegQuality, 1);
|
||||
jpeg_set_colorspace(&cinfo, JCS_GRAYSCALE);
|
||||
cinfo.raw_data_in = 0;
|
||||
cinfo.dct_method = JDCT_IFAST;
|
||||
|
||||
cinfo.comp_info[0].h_samp_factor = 1;
|
||||
cinfo.comp_info[1].h_samp_factor = 1;
|
||||
cinfo.comp_info[2].h_samp_factor = 1;
|
||||
cinfo.comp_info[0].v_samp_factor = 1;
|
||||
cinfo.comp_info[1].v_samp_factor = 1;
|
||||
cinfo.comp_info[2].v_samp_factor = 1;
|
||||
|
||||
jpeg_start_compress(&cinfo, TRUE);
|
||||
|
||||
for (size_t i = 0; i < cinfo.image_height; i++) {
|
||||
auto currentRow = static_cast<JSAMPROW>(in + i*width);
|
||||
jpeg_write_scanlines(&cinfo, ¤tRow, /*num_lines*/1);
|
||||
}
|
||||
|
||||
jpeg_finish_compress(&cinfo);
|
||||
|
||||
actualSize = dmgr.mEncodedSize;
|
||||
if (dmgr.mSuccess) {
|
||||
ret = NO_ERROR;
|
||||
} else {
|
||||
ret = UNKNOWN_ERROR;
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
std::unique_ptr<dynamic_depth::DepthMap> processDepthMapFrame(DepthPhotoInputFrame inputFrame,
|
||||
std::vector<std::unique_ptr<Item>> *items /*out*/) {
|
||||
std::vector<float> points, confidence;
|
||||
|
||||
size_t pointCount = inputFrame.mDepthMapWidth * inputFrame.mDepthMapHeight;
|
||||
points.reserve(pointCount);
|
||||
confidence.reserve(pointCount);
|
||||
float near = UINT16_MAX;
|
||||
float far = .0f;
|
||||
for (size_t i = 0; i < inputFrame.mDepthMapHeight; i++) {
|
||||
for (size_t j = 0; j < inputFrame.mDepthMapWidth; j++) {
|
||||
// Android densely packed depth map. The units for the range are in
|
||||
// millimeters and need to be scaled to meters.
|
||||
// The confidence value is encoded in the 3 most significant bits.
|
||||
// The confidence data needs to be additionally normalized with
|
||||
// values 1.0f, 0.0f representing maximum and minimum confidence
|
||||
// respectively.
|
||||
auto value = inputFrame.mDepthMapBuffer[i*inputFrame.mDepthMapStride + j];
|
||||
auto point = static_cast<float>(value & 0x1FFF) / 1000.f;
|
||||
points.push_back(point);
|
||||
|
||||
auto conf = (value >> 13) & 0x7;
|
||||
float normConfidence = (conf == 0) ? 1.f : (static_cast<float>(conf) - 1) / 7.f;
|
||||
confidence.push_back(normConfidence);
|
||||
|
||||
if (near > point) {
|
||||
near = point;
|
||||
}
|
||||
if (far < point) {
|
||||
far = point;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (near == far) {
|
||||
ALOGE("%s: Near and far range values must not match!", __FUNCTION__);
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
std::vector<uint8_t> pointsQuantized, confidenceQuantized;
|
||||
pointsQuantized.reserve(pointCount); confidenceQuantized.reserve(pointCount);
|
||||
auto pointIt = points.begin();
|
||||
auto confidenceIt = confidence.begin();
|
||||
while ((pointIt != points.end()) && (confidenceIt != confidence.end())) {
|
||||
pointsQuantized.push_back(floorf(((far * (*pointIt - near)) /
|
||||
(*pointIt * (far - near))) * 255.0f));
|
||||
confidenceQuantized.push_back(floorf(*confidenceIt * 255.0f));
|
||||
confidenceIt++; pointIt++;
|
||||
}
|
||||
|
||||
DepthMapParams depthParams(DepthFormat::kRangeInverse, near, far, DepthUnits::kMeters,
|
||||
"android/depthmap");
|
||||
depthParams.confidence_uri = "android/confidencemap";
|
||||
depthParams.mime = "image/jpeg";
|
||||
depthParams.depth_image_data.resize(inputFrame.mMaxJpegSize);
|
||||
depthParams.confidence_data.resize(inputFrame.mMaxJpegSize);
|
||||
size_t actualJpegSize;
|
||||
auto ret = encodeGrayscaleJpeg(inputFrame.mDepthMapWidth, inputFrame.mDepthMapHeight,
|
||||
pointsQuantized.data(), depthParams.depth_image_data.data(), inputFrame.mMaxJpegSize,
|
||||
inputFrame.mJpegQuality, actualJpegSize);
|
||||
if (ret != NO_ERROR) {
|
||||
ALOGE("%s: Depth map compression failed!", __FUNCTION__);
|
||||
return nullptr;
|
||||
}
|
||||
depthParams.depth_image_data.resize(actualJpegSize);
|
||||
|
||||
ret = encodeGrayscaleJpeg(inputFrame.mDepthMapWidth, inputFrame.mDepthMapHeight,
|
||||
confidenceQuantized.data(), depthParams.confidence_data.data(), inputFrame.mMaxJpegSize,
|
||||
inputFrame.mJpegQuality, actualJpegSize);
|
||||
if (ret != NO_ERROR) {
|
||||
ALOGE("%s: Confidence map compression failed!", __FUNCTION__);
|
||||
return nullptr;
|
||||
}
|
||||
depthParams.confidence_data.resize(actualJpegSize);
|
||||
|
||||
return DepthMap::FromData(depthParams, items);
|
||||
}
|
||||
|
||||
extern "C" int processDepthPhotoFrame(DepthPhotoInputFrame inputFrame, size_t depthPhotoBufferSize,
|
||||
void* depthPhotoBuffer /*out*/, size_t* depthPhotoActualSize /*out*/) {
|
||||
if ((inputFrame.mMainJpegBuffer == nullptr) || (inputFrame.mDepthMapBuffer == nullptr) ||
|
||||
(depthPhotoBuffer == nullptr) || (depthPhotoActualSize == nullptr)) {
|
||||
return BAD_VALUE;
|
||||
}
|
||||
|
||||
std::vector<std::unique_ptr<Item>> items;
|
||||
std::vector<std::unique_ptr<Camera>> cameraList;
|
||||
auto image = Image::FromDataForPrimaryImage("android/mainimage", &items);
|
||||
std::unique_ptr<CameraParams> cameraParams(new CameraParams(std::move(image)));
|
||||
if (cameraParams == nullptr) {
|
||||
ALOGE("%s: Failed to initialize camera parameters", __FUNCTION__);
|
||||
return BAD_VALUE;
|
||||
}
|
||||
|
||||
cameraParams->depth_map = processDepthMapFrame(inputFrame, &items);
|
||||
if (cameraParams->depth_map == nullptr) {
|
||||
ALOGE("%s: Depth map processing failed!", __FUNCTION__);
|
||||
return BAD_VALUE;
|
||||
}
|
||||
|
||||
// It is not possible to generate an imaging model without instrinsic calibration.
|
||||
if (inputFrame.mIsInstrinsicCalibrationValid) {
|
||||
// The camera intrinsic calibration layout is as follows:
|
||||
// [focalLengthX, focalLengthY, opticalCenterX, opticalCenterY, skew]
|
||||
const dynamic_depth::Point<double> focalLength(inputFrame.mInstrinsicCalibration[0],
|
||||
inputFrame.mInstrinsicCalibration[1]);
|
||||
const Dimension imageSize(inputFrame.mMainJpegWidth, inputFrame.mMainJpegHeight);
|
||||
ImagingModelParams imagingParams(focalLength, imageSize);
|
||||
imagingParams.principal_point.x = inputFrame.mInstrinsicCalibration[2];
|
||||
imagingParams.principal_point.y = inputFrame.mInstrinsicCalibration[3];
|
||||
imagingParams.skew = inputFrame.mInstrinsicCalibration[4];
|
||||
|
||||
// The camera lens distortion contains the following lens correction coefficients.
|
||||
// [kappa_1, kappa_2, kappa_3 kappa_4, kappa_5]
|
||||
if (inputFrame.mIsLensDistortionValid) {
|
||||
// According to specification the lens distortion coefficients should be ordered
|
||||
// as [1, kappa_4, kappa_1, kappa_5, kappa_2, 0, kappa_3, 0]
|
||||
float distortionData[] = {1.f, inputFrame.mLensDistortion[3],
|
||||
inputFrame.mLensDistortion[0], inputFrame.mLensDistortion[4],
|
||||
inputFrame.mLensDistortion[1], 0.f, inputFrame.mLensDistortion[2], 0.f};
|
||||
auto distortionDataLength = sizeof(distortionData) / sizeof(distortionData[0]);
|
||||
imagingParams.distortion.reserve(distortionDataLength);
|
||||
imagingParams.distortion.insert(imagingParams.distortion.end(), distortionData,
|
||||
distortionData + distortionDataLength);
|
||||
}
|
||||
|
||||
cameraParams->imaging_model = ImagingModel::FromData(imagingParams);
|
||||
}
|
||||
|
||||
if (inputFrame.mIsLogical) {
|
||||
cameraParams->trait = dynamic_depth::CameraTrait::LOGICAL;
|
||||
} else {
|
||||
cameraParams->trait = dynamic_depth::CameraTrait::PHYSICAL;
|
||||
}
|
||||
|
||||
cameraList.emplace_back(Camera::FromData(std::move(cameraParams)));
|
||||
|
||||
auto deviceParams = std::make_unique<DeviceParams> (Cameras::FromCameraArray(&cameraList));
|
||||
deviceParams->container = Container::FromItems(&items);
|
||||
std::vector<std::unique_ptr<Profile>> profileList;
|
||||
profileList.emplace_back(Profile::FromData("DepthPhoto", {0}));
|
||||
deviceParams->profiles = Profiles::FromProfileArray(&profileList);
|
||||
std::unique_ptr<Device> device = Device::FromData(std::move(deviceParams));
|
||||
if (device == nullptr) {
|
||||
ALOGE("%s: Failed to initialize camera device", __FUNCTION__);
|
||||
return BAD_VALUE;
|
||||
}
|
||||
|
||||
std::istringstream inputJpegStream(
|
||||
std::string(inputFrame.mMainJpegBuffer, inputFrame.mMainJpegSize));
|
||||
std::ostringstream outputJpegStream;
|
||||
if (!WriteImageAndMetadataAndContainer(&inputJpegStream, device.get(), &outputJpegStream)) {
|
||||
ALOGE("%s: Failed writing depth output", __FUNCTION__);
|
||||
return BAD_VALUE;
|
||||
}
|
||||
|
||||
*depthPhotoActualSize = static_cast<size_t> (outputJpegStream.tellp());
|
||||
if (*depthPhotoActualSize > depthPhotoBufferSize) {
|
||||
ALOGE("%s: Depth photo output buffer not sufficient, needed %zu actual %zu", __FUNCTION__,
|
||||
*depthPhotoActualSize, depthPhotoBufferSize);
|
||||
return NO_MEMORY;
|
||||
}
|
||||
|
||||
memcpy(depthPhotoBuffer, outputJpegStream.str().c_str(), *depthPhotoActualSize);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
}; // namespace camera3
|
||||
}; // namespace android
|
||||
@ -0,0 +1,67 @@
|
||||
/*
|
||||
* Copyright (C) 2019 The Android Open Source Project
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef ANDROID_SERVERS_CAMERA_CAMERA3_DEPTH_PROCESSOR_H
|
||||
#define ANDROID_SERVERS_CAMERA_CAMERA3_DEPTH_PROCESSOR_H
|
||||
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
|
||||
namespace android {
|
||||
namespace camera3 {
|
||||
|
||||
struct DepthPhotoInputFrame {
|
||||
const char* mMainJpegBuffer;
|
||||
size_t mMainJpegSize;
|
||||
size_t mMainJpegWidth, mMainJpegHeight;
|
||||
uint16_t* mDepthMapBuffer;
|
||||
size_t mDepthMapWidth, mDepthMapHeight, mDepthMapStride;
|
||||
size_t mMaxJpegSize;
|
||||
uint8_t mJpegQuality;
|
||||
uint8_t mIsLogical;
|
||||
float mInstrinsicCalibration[5];
|
||||
uint8_t mIsInstrinsicCalibrationValid;
|
||||
float mLensDistortion[5];
|
||||
uint8_t mIsLensDistortionValid;
|
||||
|
||||
DepthPhotoInputFrame() :
|
||||
mMainJpegBuffer(nullptr),
|
||||
mMainJpegSize(0),
|
||||
mMainJpegWidth(0),
|
||||
mMainJpegHeight(0),
|
||||
mDepthMapBuffer(nullptr),
|
||||
mDepthMapWidth(0),
|
||||
mDepthMapHeight(0),
|
||||
mDepthMapStride(0),
|
||||
mMaxJpegSize(0),
|
||||
mJpegQuality(100),
|
||||
mIsLogical(0),
|
||||
mInstrinsicCalibration{0.f},
|
||||
mIsInstrinsicCalibrationValid(0),
|
||||
mLensDistortion{0.f},
|
||||
mIsLensDistortionValid(0) {}
|
||||
};
|
||||
|
||||
static const char *kDepthPhotoLibrary = "libdepthphoto.so";
|
||||
static const char *kDepthPhotoProcessFunction = "processDepthPhotoFrame";
|
||||
typedef int (*process_depth_photo_frame) (DepthPhotoInputFrame /*inputFrame*/,
|
||||
size_t /*depthPhotoBufferSize*/, void* /*depthPhotoBuffer out*/,
|
||||
size_t* /*depthPhotoActualSize out*/);
|
||||
|
||||
}; // namespace camera3
|
||||
}; // namespace android
|
||||
|
||||
#endif
|
||||
Loading…
Reference in new issue