diff --git a/services/camera/libcameraservice/Android.bp b/services/camera/libcameraservice/Android.bp index 9ae7fd9fa4..7ec0e4ca7c 100644 --- a/services/camera/libcameraservice/Android.bp +++ b/services/camera/libcameraservice/Android.bp @@ -138,6 +138,7 @@ cc_library_shared { name: "libdepthphoto", srcs: [ + "utils/ExifUtils.cpp", "common/DepthPhotoProcessor.cpp", ], @@ -151,6 +152,8 @@ cc_library_shared { "libcutils", "libjpeg", "libmemunreachable", + "libexif", + "libcamera_client", ], include_dirs: [ diff --git a/services/camera/libcameraservice/api2/DepthCompositeStream.cpp b/services/camera/libcameraservice/api2/DepthCompositeStream.cpp index 2eec0f7f9c..9525ad2635 100644 --- a/services/camera/libcameraservice/api2/DepthCompositeStream.cpp +++ b/services/camera/libcameraservice/api2/DepthCompositeStream.cpp @@ -339,6 +339,21 @@ status_t DepthCompositeStream::processInputFrame(const InputFrame &inputFrame) { } else { depthPhoto.mIsLensDistortionValid = 0; } + entry = inputFrame.result.find(ANDROID_JPEG_ORIENTATION); + if (entry.count > 0) { + // The camera jpeg orientation values must be within [0, 90, 180, 270]. + switch (entry.data.i32[0]) { + case 0: + case 90: + case 180: + case 270: + depthPhoto.mOrientation = static_cast (entry.data.i32[0]); + break; + default: + ALOGE("%s: Unexpected jpeg orientation value: %d, default to 0 degrees", + __FUNCTION__, entry.data.i32[0]); + } + } size_t actualJpegSize = 0; res = mDepthPhotoProcess(depthPhoto, finalJpegBufferSize, dstBuffer, &actualJpegSize); diff --git a/services/camera/libcameraservice/common/DepthPhotoProcessor.cpp b/services/camera/libcameraservice/common/DepthPhotoProcessor.cpp index a945aca13b..6d96163b42 100644 --- a/services/camera/libcameraservice/common/DepthPhotoProcessor.cpp +++ b/services/camera/libcameraservice/common/DepthPhotoProcessor.cpp @@ -32,9 +32,12 @@ #include #include #include +#include +#include #include #include #include +#include #include #include #include @@ -61,8 +64,44 @@ using dynamic_depth::Profiles; namespace android { namespace camera3 { +ExifOrientation getExifOrientation(const unsigned char *jpegBuffer, size_t jpegBufferSize) { + if ((jpegBuffer == nullptr) || (jpegBufferSize == 0)) { + return ExifOrientation::ORIENTATION_UNDEFINED; + } + + auto exifData = exif_data_new(); + exif_data_load_data(exifData, jpegBuffer, jpegBufferSize); + ExifEntry *orientation = exif_content_get_entry(exifData->ifd[EXIF_IFD_0], + EXIF_TAG_ORIENTATION); + if ((orientation == nullptr) || (orientation->size != sizeof(ExifShort))) { + ALOGV("%s: Orientation EXIF entry invalid!", __FUNCTION__); + exif_data_unref(exifData); + return ExifOrientation::ORIENTATION_0_DEGREES; + } + + auto orientationValue = exif_get_short(orientation->data, exif_data_get_byte_order(exifData)); + ExifOrientation ret; + switch (orientationValue) { + case ExifOrientation::ORIENTATION_0_DEGREES: + case ExifOrientation::ORIENTATION_90_DEGREES: + case ExifOrientation::ORIENTATION_180_DEGREES: + case ExifOrientation::ORIENTATION_270_DEGREES: + ret = static_cast (orientationValue); + break; + default: + ALOGE("%s: Unexpected EXIF orientation value: %d, defaulting to 0 degrees", + __FUNCTION__, orientationValue); + ret = ExifOrientation::ORIENTATION_0_DEGREES; + } + + exif_data_unref(exifData); + + return ret; +} + status_t encodeGrayscaleJpeg(size_t width, size_t height, uint8_t *in, void *out, - const size_t maxOutSize, uint8_t jpegQuality, size_t &actualSize) { + const size_t maxOutSize, uint8_t jpegQuality, ExifOrientation exifOrientation, + size_t &actualSize) { status_t ret; // libjpeg is a C library so we use C-style "inheritance" by // putting libjpeg's jpeg_destination_mgr first in our custom @@ -151,6 +190,23 @@ status_t encodeGrayscaleJpeg(size_t width, size_t height, uint8_t *in, void *out jpeg_start_compress(&cinfo, TRUE); + if (exifOrientation != ExifOrientation::ORIENTATION_UNDEFINED) { + std::unique_ptr utils(ExifUtils::create()); + utils->initializeEmpty(); + utils->setImageWidth(width); + utils->setImageHeight(height); + utils->setOrientationValue(exifOrientation); + + if (utils->generateApp1()) { + const uint8_t* exifBuffer = utils->getApp1Buffer(); + size_t exifBufferSize = utils->getApp1Length(); + jpeg_write_marker(&cinfo, JPEG_APP0 + 1, static_cast(exifBuffer), + exifBufferSize); + } else { + ALOGE("%s: Unable to generate App1 buffer", __FUNCTION__); + } + } + for (size_t i = 0; i < cinfo.image_height; i++) { auto currentRow = static_cast(in + i*width); jpeg_write_scanlines(&cinfo, ¤tRow, /*num_lines*/1); @@ -168,8 +224,106 @@ status_t encodeGrayscaleJpeg(size_t width, size_t height, uint8_t *in, void *out return ret; } +inline void unpackDepth16(uint16_t value, std::vector *points /*out*/, + std::vector *confidence /*out*/, float *near /*out*/, float *far /*out*/) { + // Android densely packed depth map. The units for the range are in + // millimeters and need to be scaled to meters. + // The confidence value is encoded in the 3 most significant bits. + // The confidence data needs to be additionally normalized with + // values 1.0f, 0.0f representing maximum and minimum confidence + // respectively. + auto point = static_cast(value & 0x1FFF) / 1000.f; + points->push_back(point); + + auto conf = (value >> 13) & 0x7; + float normConfidence = (conf == 0) ? 1.f : (static_cast(conf) - 1) / 7.f; + confidence->push_back(normConfidence); + + if (*near > point) { + *near = point; + } + if (*far < point) { + *far = point; + } +} + +// Trivial case, read forward from top,left corner. +void rotate0AndUnpack(DepthPhotoInputFrame inputFrame, std::vector *points /*out*/, + std::vector *confidence /*out*/, float *near /*out*/, float *far /*out*/) { + for (size_t i = 0; i < inputFrame.mDepthMapHeight; i++) { + for (size_t j = 0; j < inputFrame.mDepthMapWidth; j++) { + unpackDepth16(inputFrame.mDepthMapBuffer[i*inputFrame.mDepthMapStride + j], points, + confidence, near, far); + } + } +} + +// 90 degrees CW rotation can be applied by starting to read from bottom, left corner +// transposing rows and columns. +void rotate90AndUnpack(DepthPhotoInputFrame inputFrame, std::vector *points /*out*/, + std::vector *confidence /*out*/, float *near /*out*/, float *far /*out*/) { + for (size_t i = 0; i < inputFrame.mDepthMapWidth; i++) { + for (ssize_t j = inputFrame.mDepthMapHeight-1; j >= 0; j--) { + unpackDepth16(inputFrame.mDepthMapBuffer[j*inputFrame.mDepthMapStride + i], points, + confidence, near, far); + } + } +} + +// 180 CW degrees rotation can be applied by starting to read backwards from bottom, right corner. +void rotate180AndUnpack(DepthPhotoInputFrame inputFrame, std::vector *points /*out*/, + std::vector *confidence /*out*/, float *near /*out*/, float *far /*out*/) { + for (ssize_t i = inputFrame.mDepthMapHeight-1; i >= 0; i--) { + for (ssize_t j = inputFrame.mDepthMapWidth-1; j >= 0; j--) { + unpackDepth16(inputFrame.mDepthMapBuffer[i*inputFrame.mDepthMapStride + j], points, + confidence, near, far); + } + } +} + +// 270 degrees CW rotation can be applied by starting to read from top, right corner +// transposing rows and columns. +void rotate270AndUnpack(DepthPhotoInputFrame inputFrame, std::vector *points /*out*/, + std::vector *confidence /*out*/, float *near /*out*/, float *far /*out*/) { + for (ssize_t i = inputFrame.mDepthMapWidth-1; i >= 0; i--) { + for (size_t j = 0; j < inputFrame.mDepthMapHeight; j++) { + unpackDepth16(inputFrame.mDepthMapBuffer[j*inputFrame.mDepthMapStride + i], points, + confidence, near, far); + } + } +} + +bool rotateAndUnpack(DepthPhotoInputFrame inputFrame, std::vector *points /*out*/, + std::vector *confidence /*out*/, float *near /*out*/, float *far /*out*/) { + switch (inputFrame.mOrientation) { + case DepthPhotoOrientation::DEPTH_ORIENTATION_0_DEGREES: + rotate0AndUnpack(inputFrame, points, confidence, near, far); + return false; + case DepthPhotoOrientation::DEPTH_ORIENTATION_90_DEGREES: + rotate90AndUnpack(inputFrame, points, confidence, near, far); + return true; + case DepthPhotoOrientation::DEPTH_ORIENTATION_180_DEGREES: + rotate180AndUnpack(inputFrame, points, confidence, near, far); + return false; + case DepthPhotoOrientation::DEPTH_ORIENTATION_270_DEGREES: + rotate270AndUnpack(inputFrame, points, confidence, near, far); + return true; + default: + ALOGE("%s: Unsupported depth photo rotation: %d, default to 0", __FUNCTION__, + inputFrame.mOrientation); + rotate0AndUnpack(inputFrame, points, confidence, near, far); + } + + return false; +} + std::unique_ptr processDepthMapFrame(DepthPhotoInputFrame inputFrame, - std::vector> *items /*out*/) { + ExifOrientation exifOrientation, std::vector> *items /*out*/, + bool *switchDimensions /*out*/) { + if ((items == nullptr) || (switchDimensions == nullptr)) { + return nullptr; + } + std::vector points, confidence; size_t pointCount = inputFrame.mDepthMapWidth * inputFrame.mDepthMapHeight; @@ -177,29 +331,21 @@ std::unique_ptr processDepthMapFrame(DepthPhotoInputFra confidence.reserve(pointCount); float near = UINT16_MAX; float far = .0f; - for (size_t i = 0; i < inputFrame.mDepthMapHeight; i++) { - for (size_t j = 0; j < inputFrame.mDepthMapWidth; j++) { - // Android densely packed depth map. The units for the range are in - // millimeters and need to be scaled to meters. - // The confidence value is encoded in the 3 most significant bits. - // The confidence data needs to be additionally normalized with - // values 1.0f, 0.0f representing maximum and minimum confidence - // respectively. - auto value = inputFrame.mDepthMapBuffer[i*inputFrame.mDepthMapStride + j]; - auto point = static_cast(value & 0x1FFF) / 1000.f; - points.push_back(point); - - auto conf = (value >> 13) & 0x7; - float normConfidence = (conf == 0) ? 1.f : (static_cast(conf) - 1) / 7.f; - confidence.push_back(normConfidence); - - if (near > point) { - near = point; - } - if (far < point) { - far = point; - } - } + *switchDimensions = false; + // Physical rotation of depth and confidence maps may be needed in case + // the EXIF orientation is set to 0 degrees and the depth photo orientation + // (source color image) has some different value. + if (exifOrientation == ExifOrientation::ORIENTATION_0_DEGREES) { + *switchDimensions = rotateAndUnpack(inputFrame, &points, &confidence, &near, &far); + } else { + rotate0AndUnpack(inputFrame, &points, &confidence, &near, &far); + } + + size_t width = inputFrame.mDepthMapWidth; + size_t height = inputFrame.mDepthMapHeight; + if (*switchDimensions) { + width = inputFrame.mDepthMapHeight; + height = inputFrame.mDepthMapWidth; } if (near == far) { @@ -225,18 +371,18 @@ std::unique_ptr processDepthMapFrame(DepthPhotoInputFra depthParams.depth_image_data.resize(inputFrame.mMaxJpegSize); depthParams.confidence_data.resize(inputFrame.mMaxJpegSize); size_t actualJpegSize; - auto ret = encodeGrayscaleJpeg(inputFrame.mDepthMapWidth, inputFrame.mDepthMapHeight, - pointsQuantized.data(), depthParams.depth_image_data.data(), inputFrame.mMaxJpegSize, - inputFrame.mJpegQuality, actualJpegSize); + auto ret = encodeGrayscaleJpeg(width, height, pointsQuantized.data(), + depthParams.depth_image_data.data(), inputFrame.mMaxJpegSize, + inputFrame.mJpegQuality, exifOrientation, actualJpegSize); if (ret != NO_ERROR) { ALOGE("%s: Depth map compression failed!", __FUNCTION__); return nullptr; } depthParams.depth_image_data.resize(actualJpegSize); - ret = encodeGrayscaleJpeg(inputFrame.mDepthMapWidth, inputFrame.mDepthMapHeight, - confidenceQuantized.data(), depthParams.confidence_data.data(), inputFrame.mMaxJpegSize, - inputFrame.mJpegQuality, actualJpegSize); + ret = encodeGrayscaleJpeg(width, height, confidenceQuantized.data(), + depthParams.confidence_data.data(), inputFrame.mMaxJpegSize, + inputFrame.mJpegQuality, exifOrientation, actualJpegSize); if (ret != NO_ERROR) { ALOGE("%s: Confidence map compression failed!", __FUNCTION__); return nullptr; @@ -262,7 +408,12 @@ extern "C" int processDepthPhotoFrame(DepthPhotoInputFrame inputFrame, size_t de return BAD_VALUE; } - cameraParams->depth_map = processDepthMapFrame(inputFrame, &items); + ExifOrientation exifOrientation = getExifOrientation( + reinterpret_cast (inputFrame.mMainJpegBuffer), + inputFrame.mMainJpegSize); + bool switchDimensions; + cameraParams->depth_map = processDepthMapFrame(inputFrame, exifOrientation, &items, + &switchDimensions); if (cameraParams->depth_map == nullptr) { ALOGE("%s: Depth map processing failed!", __FUNCTION__); return BAD_VALUE; @@ -274,7 +425,13 @@ extern "C" int processDepthPhotoFrame(DepthPhotoInputFrame inputFrame, size_t de // [focalLengthX, focalLengthY, opticalCenterX, opticalCenterY, skew] const dynamic_depth::Point focalLength(inputFrame.mInstrinsicCalibration[0], inputFrame.mInstrinsicCalibration[1]); - const Dimension imageSize(inputFrame.mMainJpegWidth, inputFrame.mMainJpegHeight); + size_t width = inputFrame.mMainJpegWidth; + size_t height = inputFrame.mMainJpegHeight; + if (switchDimensions) { + width = inputFrame.mMainJpegHeight; + height = inputFrame.mMainJpegWidth; + } + const Dimension imageSize(width, height); ImagingModelParams imagingParams(focalLength, imageSize); imagingParams.principal_point.x = inputFrame.mInstrinsicCalibration[2]; imagingParams.principal_point.y = inputFrame.mInstrinsicCalibration[3]; diff --git a/services/camera/libcameraservice/common/DepthPhotoProcessor.h b/services/camera/libcameraservice/common/DepthPhotoProcessor.h index 19889a1f3d..6a2fbffb6b 100644 --- a/services/camera/libcameraservice/common/DepthPhotoProcessor.h +++ b/services/camera/libcameraservice/common/DepthPhotoProcessor.h @@ -23,19 +23,27 @@ namespace android { namespace camera3 { +enum DepthPhotoOrientation { + DEPTH_ORIENTATION_0_DEGREES = 0, + DEPTH_ORIENTATION_90_DEGREES = 90, + DEPTH_ORIENTATION_180_DEGREES = 180, + DEPTH_ORIENTATION_270_DEGREES = 270, +}; + struct DepthPhotoInputFrame { - const char* mMainJpegBuffer; - size_t mMainJpegSize; - size_t mMainJpegWidth, mMainJpegHeight; - uint16_t* mDepthMapBuffer; - size_t mDepthMapWidth, mDepthMapHeight, mDepthMapStride; - size_t mMaxJpegSize; - uint8_t mJpegQuality; - uint8_t mIsLogical; - float mInstrinsicCalibration[5]; - uint8_t mIsInstrinsicCalibrationValid; - float mLensDistortion[5]; - uint8_t mIsLensDistortionValid; + const char* mMainJpegBuffer; + size_t mMainJpegSize; + size_t mMainJpegWidth, mMainJpegHeight; + uint16_t* mDepthMapBuffer; + size_t mDepthMapWidth, mDepthMapHeight, mDepthMapStride; + size_t mMaxJpegSize; + uint8_t mJpegQuality; + uint8_t mIsLogical; + float mInstrinsicCalibration[5]; + uint8_t mIsInstrinsicCalibrationValid; + float mLensDistortion[5]; + uint8_t mIsLensDistortionValid; + DepthPhotoOrientation mOrientation; DepthPhotoInputFrame() : mMainJpegBuffer(nullptr), @@ -52,7 +60,8 @@ struct DepthPhotoInputFrame { mInstrinsicCalibration{0.f}, mIsInstrinsicCalibrationValid(0), mLensDistortion{0.f}, - mIsLensDistortionValid(0) {} + mIsLensDistortionValid(0), + mOrientation(DepthPhotoOrientation::DEPTH_ORIENTATION_0_DEGREES) {} }; static const char *kDepthPhotoLibrary = "libdepthphoto.so"; diff --git a/services/camera/libcameraservice/tests/Android.mk b/services/camera/libcameraservice/tests/Android.mk index d777ca1cb1..b4e7c32bef 100644 --- a/services/camera/libcameraservice/tests/Android.mk +++ b/services/camera/libcameraservice/tests/Android.mk @@ -27,6 +27,8 @@ LOCAL_SHARED_LIBRARIES := \ libcamera_client \ libcamera_metadata \ libutils \ + libjpeg \ + libexif \ android.hardware.camera.common@1.0 \ android.hardware.camera.provider@2.4 \ android.hardware.camera.provider@2.5 \ @@ -36,6 +38,8 @@ LOCAL_SHARED_LIBRARIES := \ LOCAL_C_INCLUDES += \ system/media/private/camera/include \ + external/dynamic_depth/includes \ + external/dynamic_depth/internal \ LOCAL_CFLAGS += -Wall -Wextra -Werror diff --git a/services/camera/libcameraservice/tests/DepthProcessorTest.cpp b/services/camera/libcameraservice/tests/DepthProcessorTest.cpp new file mode 100644 index 0000000000..2162514d14 --- /dev/null +++ b/services/camera/libcameraservice/tests/DepthProcessorTest.cpp @@ -0,0 +1,382 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_NDEBUG 0 +#define LOG_TAG "DepthProcessorTest" + +#include +#include + +#include +#include + +#include "../common/DepthPhotoProcessor.h" +#include "../utils/ExifUtils.h" +#include "NV12Compressor.h" + +using namespace android; +using namespace android::camera3; + +static const size_t kTestBufferWidth = 640; +static const size_t kTestBufferHeight = 480; +static const size_t kTestBufferNV12Size ((((kTestBufferWidth) * (kTestBufferHeight)) * 3) / 2); +static const size_t kTestBufferDepthSize (kTestBufferWidth * kTestBufferHeight); +static const size_t kSeed = 1234; + +void linkToDepthPhotoLibrary(void **libHandle /*out*/, + process_depth_photo_frame *processFrameFunc /*out*/) { + ASSERT_NE(libHandle, nullptr); + ASSERT_NE(processFrameFunc, nullptr); + + *libHandle = dlopen(kDepthPhotoLibrary, RTLD_NOW | RTLD_LOCAL); + if (*libHandle != nullptr) { + *processFrameFunc = reinterpret_cast ( + dlsym(*libHandle, kDepthPhotoProcessFunction)); + ASSERT_NE(*processFrameFunc, nullptr); + } +} + +void generateColorJpegBuffer(int jpegQuality, ExifOrientation orientationValue, bool includeExif, + bool switchDimensions, std::vector *colorJpegBuffer /*out*/) { + ASSERT_NE(colorJpegBuffer, nullptr); + + std::array colorSourceBuffer; + std::default_random_engine gen(kSeed); + std::uniform_int_distribution uniDist(0, UINT8_MAX - 1); + for (size_t i = 0; i < colorSourceBuffer.size(); i++) { + colorSourceBuffer[i] = uniDist(gen); + } + + size_t width = kTestBufferWidth; + size_t height = kTestBufferHeight; + if (switchDimensions) { + width = kTestBufferHeight; + height = kTestBufferWidth; + } + + NV12Compressor jpegCompressor; + if (includeExif) { + ASSERT_TRUE(jpegCompressor.compressWithExifOrientation( + reinterpret_cast (colorSourceBuffer.data()), width, height, + jpegQuality, orientationValue)); + } else { + ASSERT_TRUE(jpegCompressor.compress( + reinterpret_cast (colorSourceBuffer.data()), width, height, + jpegQuality)); + } + + *colorJpegBuffer = std::move(jpegCompressor.getCompressedData()); + ASSERT_FALSE(colorJpegBuffer->empty()); +} + +void generateDepth16Buffer(std::array *depth16Buffer /*out*/) { + ASSERT_NE(depth16Buffer, nullptr); + std::default_random_engine gen(kSeed+1); + std::uniform_int_distribution uniDist(0, UINT16_MAX - 1); + for (size_t i = 0; i < depth16Buffer->size(); i++) { + (*depth16Buffer)[i] = uniDist(gen); + } +} + +TEST(DepthProcessorTest, LinkToLibray) { + void *libHandle; + process_depth_photo_frame processFunc; + linkToDepthPhotoLibrary(&libHandle, &processFunc); + if (libHandle != nullptr) { + dlclose(libHandle); + } +} + +TEST(DepthProcessorTest, BadInput) { + void *libHandle; + int jpegQuality = 95; + + process_depth_photo_frame processFunc; + linkToDepthPhotoLibrary(&libHandle, &processFunc); + if (libHandle == nullptr) { + // Depth library no present, nothing more to test. + return; + } + + DepthPhotoInputFrame inputFrame; + // Worst case both depth and confidence maps have the same size as the main color image. + inputFrame.mMaxJpegSize = inputFrame.mMainJpegSize * 3; + + std::vector colorJpegBuffer; + generateColorJpegBuffer(jpegQuality, ExifOrientation::ORIENTATION_UNDEFINED, + /*includeExif*/ false, /*switchDimensions*/ false, &colorJpegBuffer); + + std::array depth16Buffer; + generateDepth16Buffer(&depth16Buffer); + + std::vector depthPhotoBuffer(inputFrame.mMaxJpegSize); + size_t actualDepthPhotoSize = 0; + + inputFrame.mMainJpegWidth = kTestBufferWidth; + inputFrame.mMainJpegHeight = kTestBufferHeight; + inputFrame.mJpegQuality = jpegQuality; + ASSERT_NE(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(), + &actualDepthPhotoSize), 0); + + inputFrame.mMainJpegBuffer = reinterpret_cast (colorJpegBuffer.data()); + inputFrame.mMainJpegSize = colorJpegBuffer.size(); + ASSERT_NE(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(), + &actualDepthPhotoSize), 0); + + inputFrame.mDepthMapBuffer = depth16Buffer.data(); + inputFrame.mDepthMapWidth = inputFrame.mDepthMapStride = kTestBufferWidth; + inputFrame.mDepthMapHeight = kTestBufferHeight; + ASSERT_NE(processFunc(inputFrame, depthPhotoBuffer.size(), nullptr, + &actualDepthPhotoSize), 0); + + ASSERT_NE(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(), nullptr), + 0); + + dlclose(libHandle); +} + +TEST(DepthProcessorTest, BasicDepthPhotoValidation) { + void *libHandle; + int jpegQuality = 95; + + process_depth_photo_frame processFunc; + linkToDepthPhotoLibrary(&libHandle, &processFunc); + if (libHandle == nullptr) { + // Depth library no present, nothing more to test. + return; + } + + std::vector colorJpegBuffer; + generateColorJpegBuffer(jpegQuality, ExifOrientation::ORIENTATION_UNDEFINED, + /*includeExif*/ false, /*switchDimensions*/ false, &colorJpegBuffer); + + std::array depth16Buffer; + generateDepth16Buffer(&depth16Buffer); + + DepthPhotoInputFrame inputFrame; + inputFrame.mMainJpegBuffer = reinterpret_cast (colorJpegBuffer.data()); + inputFrame.mMainJpegSize = colorJpegBuffer.size(); + // Worst case both depth and confidence maps have the same size as the main color image. + inputFrame.mMaxJpegSize = inputFrame.mMainJpegSize * 3; + inputFrame.mMainJpegWidth = kTestBufferWidth; + inputFrame.mMainJpegHeight = kTestBufferHeight; + inputFrame.mJpegQuality = jpegQuality; + inputFrame.mDepthMapBuffer = depth16Buffer.data(); + inputFrame.mDepthMapWidth = inputFrame.mDepthMapStride = kTestBufferWidth; + inputFrame.mDepthMapHeight = kTestBufferHeight; + + std::vector depthPhotoBuffer(inputFrame.mMaxJpegSize); + size_t actualDepthPhotoSize = 0; + ASSERT_EQ(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(), + &actualDepthPhotoSize), 0); + ASSERT_TRUE((actualDepthPhotoSize > 0) && (depthPhotoBuffer.size() >= actualDepthPhotoSize)); + + // The final depth photo must consist of three jpeg images: + // - the main color image + // - the depth map image + // - the confidence map image + size_t mainJpegSize = 0; + ASSERT_EQ(NV12Compressor::findJpegSize(depthPhotoBuffer.data(), actualDepthPhotoSize, + &mainJpegSize), OK); + ASSERT_TRUE((mainJpegSize > 0) && (mainJpegSize < actualDepthPhotoSize)); + size_t depthMapSize = 0; + ASSERT_EQ(NV12Compressor::findJpegSize(depthPhotoBuffer.data() + mainJpegSize, + actualDepthPhotoSize - mainJpegSize, &depthMapSize), OK); + ASSERT_TRUE((depthMapSize > 0) && (depthMapSize < (actualDepthPhotoSize - mainJpegSize))); + + dlclose(libHandle); +} + +TEST(DepthProcessorTest, TestDepthPhotoExifOrientation) { + void *libHandle; + int jpegQuality = 95; + + process_depth_photo_frame processFunc; + linkToDepthPhotoLibrary(&libHandle, &processFunc); + if (libHandle == nullptr) { + // Depth library no present, nothing more to test. + return; + } + + ExifOrientation exifOrientations[] = { ExifOrientation::ORIENTATION_UNDEFINED, + ExifOrientation::ORIENTATION_0_DEGREES, ExifOrientation::ORIENTATION_90_DEGREES, + ExifOrientation::ORIENTATION_180_DEGREES, ExifOrientation::ORIENTATION_270_DEGREES }; + for (auto exifOrientation : exifOrientations) { + std::vector colorJpegBuffer; + generateColorJpegBuffer(jpegQuality, exifOrientation, /*includeExif*/ true, + /*switchDimensions*/ false, &colorJpegBuffer); + if (exifOrientation != ExifOrientation::ORIENTATION_UNDEFINED) { + auto jpegExifOrientation = ExifOrientation::ORIENTATION_UNDEFINED; + ASSERT_EQ(NV12Compressor::getExifOrientation(colorJpegBuffer.data(), + colorJpegBuffer.size(), &jpegExifOrientation), OK); + ASSERT_EQ(exifOrientation, jpegExifOrientation); + } + + std::array depth16Buffer; + generateDepth16Buffer(&depth16Buffer); + + DepthPhotoInputFrame inputFrame; + inputFrame.mMainJpegBuffer = reinterpret_cast (colorJpegBuffer.data()); + inputFrame.mMainJpegSize = colorJpegBuffer.size(); + // Worst case both depth and confidence maps have the same size as the main color image. + inputFrame.mMaxJpegSize = inputFrame.mMainJpegSize * 3; + inputFrame.mMainJpegWidth = kTestBufferWidth; + inputFrame.mMainJpegHeight = kTestBufferHeight; + inputFrame.mJpegQuality = jpegQuality; + inputFrame.mDepthMapBuffer = depth16Buffer.data(); + inputFrame.mDepthMapWidth = inputFrame.mDepthMapStride = kTestBufferWidth; + inputFrame.mDepthMapHeight = kTestBufferHeight; + + std::vector depthPhotoBuffer(inputFrame.mMaxJpegSize); + size_t actualDepthPhotoSize = 0; + ASSERT_EQ(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(), + &actualDepthPhotoSize), 0); + ASSERT_TRUE((actualDepthPhotoSize > 0) && + (depthPhotoBuffer.size() >= actualDepthPhotoSize)); + + size_t mainJpegSize = 0; + ASSERT_EQ(NV12Compressor::findJpegSize(depthPhotoBuffer.data(), actualDepthPhotoSize, + &mainJpegSize), OK); + ASSERT_TRUE((mainJpegSize > 0) && (mainJpegSize < actualDepthPhotoSize)); + size_t depthMapSize = 0; + ASSERT_EQ(NV12Compressor::findJpegSize(depthPhotoBuffer.data() + mainJpegSize, + actualDepthPhotoSize - mainJpegSize, &depthMapSize), OK); + ASSERT_TRUE((depthMapSize > 0) && (depthMapSize < (actualDepthPhotoSize - mainJpegSize))); + size_t confidenceMapSize = actualDepthPhotoSize - (mainJpegSize + depthMapSize); + + //Depth and confidence images must have the same EXIF orientation as the source + auto depthJpegExifOrientation = ExifOrientation::ORIENTATION_UNDEFINED; + ASSERT_EQ(NV12Compressor::getExifOrientation(depthPhotoBuffer.data() + mainJpegSize, + depthMapSize, &depthJpegExifOrientation), OK); + if (exifOrientation == ORIENTATION_UNDEFINED) { + // In case of undefined or missing EXIF orientation, always expect 0 degrees in the + // depth map. + ASSERT_EQ(depthJpegExifOrientation, ExifOrientation::ORIENTATION_0_DEGREES); + } else { + ASSERT_EQ(depthJpegExifOrientation, exifOrientation); + } + + auto confidenceJpegExifOrientation = ExifOrientation::ORIENTATION_UNDEFINED; + ASSERT_EQ(NV12Compressor::getExifOrientation( + depthPhotoBuffer.data() + mainJpegSize + depthMapSize, + confidenceMapSize, &confidenceJpegExifOrientation), OK); + if (exifOrientation == ORIENTATION_UNDEFINED) { + // In case of undefined or missing EXIF orientation, always expect 0 degrees in the + // confidence map. + ASSERT_EQ(confidenceJpegExifOrientation, ExifOrientation::ORIENTATION_0_DEGREES); + } else { + ASSERT_EQ(confidenceJpegExifOrientation, exifOrientation); + } + } + + dlclose(libHandle); +} + +TEST(DepthProcessorTest, TestDephtPhotoPhysicalRotation) { + void *libHandle; + int jpegQuality = 95; + + process_depth_photo_frame processFunc; + linkToDepthPhotoLibrary(&libHandle, &processFunc); + if (libHandle == nullptr) { + // Depth library no present, nothing more to test. + return; + } + + // In case of physical rotation, the EXIF orientation must always be 0. + auto exifOrientation = ExifOrientation::ORIENTATION_0_DEGREES; + DepthPhotoOrientation depthOrientations[] = { + DepthPhotoOrientation::DEPTH_ORIENTATION_0_DEGREES, + DepthPhotoOrientation::DEPTH_ORIENTATION_90_DEGREES, + DepthPhotoOrientation::DEPTH_ORIENTATION_180_DEGREES, + DepthPhotoOrientation::DEPTH_ORIENTATION_270_DEGREES }; + for (auto depthOrientation : depthOrientations) { + std::vector colorJpegBuffer; + bool switchDimensions = false; + size_t expectedWidth = kTestBufferWidth; + size_t expectedHeight = kTestBufferHeight; + if ((depthOrientation == DepthPhotoOrientation::DEPTH_ORIENTATION_90_DEGREES) || + (depthOrientation == DepthPhotoOrientation::DEPTH_ORIENTATION_270_DEGREES)) { + switchDimensions = true; + expectedWidth = kTestBufferHeight; + expectedHeight = kTestBufferWidth; + } + generateColorJpegBuffer(jpegQuality, exifOrientation, /*includeExif*/ true, + switchDimensions, &colorJpegBuffer); + auto jpegExifOrientation = ExifOrientation::ORIENTATION_UNDEFINED; + ASSERT_EQ(NV12Compressor::getExifOrientation(colorJpegBuffer.data(), colorJpegBuffer.size(), + &jpegExifOrientation), OK); + ASSERT_EQ(exifOrientation, jpegExifOrientation); + + std::array depth16Buffer; + generateDepth16Buffer(&depth16Buffer); + + DepthPhotoInputFrame inputFrame; + inputFrame.mMainJpegBuffer = reinterpret_cast (colorJpegBuffer.data()); + inputFrame.mMainJpegSize = colorJpegBuffer.size(); + // Worst case both depth and confidence maps have the same size as the main color image. + inputFrame.mMaxJpegSize = inputFrame.mMainJpegSize * 3; + inputFrame.mMainJpegWidth = kTestBufferWidth; + inputFrame.mMainJpegHeight = kTestBufferHeight; + inputFrame.mJpegQuality = jpegQuality; + inputFrame.mDepthMapBuffer = depth16Buffer.data(); + inputFrame.mDepthMapWidth = inputFrame.mDepthMapStride = kTestBufferWidth; + inputFrame.mDepthMapHeight = kTestBufferHeight; + inputFrame.mOrientation = depthOrientation; + + std::vector depthPhotoBuffer(inputFrame.mMaxJpegSize); + size_t actualDepthPhotoSize = 0; + ASSERT_EQ(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(), + &actualDepthPhotoSize), 0); + ASSERT_TRUE((actualDepthPhotoSize > 0) && + (depthPhotoBuffer.size() >= actualDepthPhotoSize)); + + size_t mainJpegSize = 0; + ASSERT_EQ(NV12Compressor::findJpegSize(depthPhotoBuffer.data(), actualDepthPhotoSize, + &mainJpegSize), OK); + ASSERT_TRUE((mainJpegSize > 0) && (mainJpegSize < actualDepthPhotoSize)); + size_t depthMapSize = 0; + ASSERT_EQ(NV12Compressor::findJpegSize(depthPhotoBuffer.data() + mainJpegSize, + actualDepthPhotoSize - mainJpegSize, &depthMapSize), OK); + ASSERT_TRUE((depthMapSize > 0) && (depthMapSize < (actualDepthPhotoSize - mainJpegSize))); + size_t confidenceMapSize = actualDepthPhotoSize - (mainJpegSize + depthMapSize); + + //Depth and confidence images must have the same EXIF orientation as the source + auto depthJpegExifOrientation = ExifOrientation::ORIENTATION_UNDEFINED; + ASSERT_EQ(NV12Compressor::getExifOrientation(depthPhotoBuffer.data() + mainJpegSize, + depthMapSize, &depthJpegExifOrientation), OK); + ASSERT_EQ(depthJpegExifOrientation, exifOrientation); + size_t depthMapWidth, depthMapHeight; + ASSERT_EQ(NV12Compressor::getJpegImageDimensions(depthPhotoBuffer.data() + mainJpegSize, + depthMapSize, &depthMapWidth, &depthMapHeight), OK); + ASSERT_EQ(depthMapWidth, expectedWidth); + ASSERT_EQ(depthMapHeight, expectedHeight); + + auto confidenceJpegExifOrientation = ExifOrientation::ORIENTATION_UNDEFINED; + ASSERT_EQ(NV12Compressor::getExifOrientation( + depthPhotoBuffer.data() + mainJpegSize + depthMapSize, confidenceMapSize, + &confidenceJpegExifOrientation), OK); + ASSERT_EQ(confidenceJpegExifOrientation, exifOrientation); + size_t confidenceMapWidth, confidenceMapHeight; + ASSERT_EQ(NV12Compressor::getJpegImageDimensions( + depthPhotoBuffer.data() + mainJpegSize + depthMapSize, confidenceMapSize, + &confidenceMapWidth, &confidenceMapHeight), OK); + ASSERT_EQ(confidenceMapWidth, expectedWidth); + ASSERT_EQ(confidenceMapHeight, expectedHeight); + } + + dlclose(libHandle); +} diff --git a/services/camera/libcameraservice/tests/NV12Compressor.cpp b/services/camera/libcameraservice/tests/NV12Compressor.cpp new file mode 100644 index 0000000000..0a41a1ff82 --- /dev/null +++ b/services/camera/libcameraservice/tests/NV12Compressor.cpp @@ -0,0 +1,379 @@ +/* +* Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_NDEBUG 0 +#define LOG_TAG "Test_NV12Compressor" + +#include "NV12Compressor.h" + +#include +#include + +using namespace android; +using namespace android::camera3; + +namespace std { +template <> +struct default_delete { + inline void operator()(ExifEntry* entry) const { exif_entry_unref(entry); } +}; + +template <> +struct default_delete { + inline void operator()(ExifData* data) const { exif_data_unref(data); } +}; + +} // namespace std + +bool NV12Compressor::compress(const unsigned char* data, int width, int height, int quality) { + if (!configureCompressor(width, height, quality)) { + // the method will have logged a more detailed error message than we can + // provide here so just return. + return false; + } + + return compressData(data, /*exifData*/ nullptr); +} + +bool NV12Compressor::compressWithExifOrientation(const unsigned char* data, int width, int height, + int quality, android::camera3::ExifOrientation exifValue) { + std::unique_ptr exifData(exif_data_new()); + if (exifData.get() == nullptr) { + return false; + } + + exif_data_set_option(exifData.get(), EXIF_DATA_OPTION_FOLLOW_SPECIFICATION); + exif_data_set_data_type(exifData.get(), EXIF_DATA_TYPE_COMPRESSED); + exif_data_set_byte_order(exifData.get(), EXIF_BYTE_ORDER_INTEL); + std::unique_ptr exifEntry(exif_entry_new()); + if (exifEntry.get() == nullptr) { + return false; + } + + exifEntry->tag = EXIF_TAG_ORIENTATION; + exif_content_add_entry(exifData->ifd[EXIF_IFD_0], exifEntry.get()); + exif_entry_initialize(exifEntry.get(), exifEntry->tag); + exif_set_short(exifEntry->data, EXIF_BYTE_ORDER_INTEL, exifValue); + + if (!configureCompressor(width, height, quality)) { + return false; + } + + return compressData(data, exifData.get()); +} + +const std::vector& NV12Compressor::getCompressedData() const { + return mDestManager.mBuffer; +} + +bool NV12Compressor::configureCompressor(int width, int height, int quality) { + mCompressInfo.err = jpeg_std_error(&mErrorManager); + // NOTE! DANGER! Do not construct any non-trivial objects below setjmp! + // The compiler will not generate code to destroy them during the return + // below so they will leak. Additionally, do not place any calls to libjpeg + // that can fail above this line or any error will cause undefined behavior. + if (setjmp(mErrorManager.mJumpBuffer)) { + // This is where the error handler will jump in case setup fails + // The error manager will ALOG an appropriate error message + return false; + } + + jpeg_create_compress(&mCompressInfo); + + mCompressInfo.image_width = width; + mCompressInfo.image_height = height; + mCompressInfo.input_components = 3; + mCompressInfo.in_color_space = JCS_YCbCr; + jpeg_set_defaults(&mCompressInfo); + + jpeg_set_quality(&mCompressInfo, quality, TRUE); + // It may seem weird to set color space here again but this will also set + // other fields. These fields might be overwritten by jpeg_set_defaults + jpeg_set_colorspace(&mCompressInfo, JCS_YCbCr); + mCompressInfo.raw_data_in = TRUE; + mCompressInfo.dct_method = JDCT_IFAST; + // Set sampling factors + mCompressInfo.comp_info[0].h_samp_factor = 2; + mCompressInfo.comp_info[0].v_samp_factor = 2; + mCompressInfo.comp_info[1].h_samp_factor = 1; + mCompressInfo.comp_info[1].v_samp_factor = 1; + mCompressInfo.comp_info[2].h_samp_factor = 1; + mCompressInfo.comp_info[2].v_samp_factor = 1; + + mCompressInfo.dest = &mDestManager; + + return true; +} + +static void deinterleave(const uint8_t* vuPlanar, std::vector& uRows, + std::vector& vRows, int rowIndex, int width, int height, int stride) { + int numRows = (height - rowIndex) / 2; + if (numRows > 8) numRows = 8; + for (int row = 0; row < numRows; ++row) { + int offset = ((rowIndex >> 1) + row) * stride; + const uint8_t* vu = vuPlanar + offset; + for (int i = 0; i < (width >> 1); ++i) { + int index = row * (width >> 1) + i; + uRows[index] = vu[1]; + vRows[index] = vu[0]; + vu += 2; + } + } +} + +bool NV12Compressor::compressData(const unsigned char* data, ExifData* exifData) { + const uint8_t* y[16]; + const uint8_t* cb[8]; + const uint8_t* cr[8]; + const uint8_t** planes[3] = { y, cb, cr }; + + int i, offset; + int width = mCompressInfo.image_width; + int height = mCompressInfo.image_height; + const uint8_t* yPlanar = data; + const uint8_t* vuPlanar = data + (width * height); + std::vector uRows(8 * (width >> 1)); + std::vector vRows(8 * (width >> 1)); + + // NOTE! DANGER! Do not construct any non-trivial objects below setjmp! + // The compiler will not generate code to destroy them during the return + // below so they will leak. Additionally, do not place any calls to libjpeg + // that can fail above this line or any error will cause undefined behavior. + if (setjmp(mErrorManager.mJumpBuffer)) { + // This is where the error handler will jump in case compression fails + // The error manager will ALOG an appropriate error message + return false; + } + + jpeg_start_compress(&mCompressInfo, TRUE); + + attachExifData(exifData); + + // process 16 lines of Y and 8 lines of U/V each time. + while (mCompressInfo.next_scanline < mCompressInfo.image_height) { + //deinterleave u and v + deinterleave(vuPlanar, uRows, vRows, mCompressInfo.next_scanline, + width, height, width); + + // Jpeg library ignores the rows whose indices are greater than height. + for (i = 0; i < 16; i++) { + // y row + y[i] = yPlanar + (mCompressInfo.next_scanline + i) * width; + + // construct u row and v row + if ((i & 1) == 0) { + // height and width are both halved because of downsampling + offset = (i >> 1) * (width >> 1); + cb[i/2] = &uRows[offset]; + cr[i/2] = &vRows[offset]; + } + } + jpeg_write_raw_data(&mCompressInfo, const_cast(planes), 16); + } + + jpeg_finish_compress(&mCompressInfo); + jpeg_destroy_compress(&mCompressInfo); + + return true; +} + +bool NV12Compressor::attachExifData(ExifData* exifData) { + if (exifData == nullptr) { + // This is not an error, we don't require EXIF data + return true; + } + + // Save the EXIF data to memory + unsigned char* rawData = nullptr; + unsigned int size = 0; + exif_data_save_data(exifData, &rawData, &size); + if (rawData == nullptr) { + ALOGE("Failed to create EXIF data block"); + return false; + } + + jpeg_write_marker(&mCompressInfo, JPEG_APP0 + 1, rawData, size); + free(rawData); + return true; +} + +NV12Compressor::ErrorManager::ErrorManager() { + error_exit = &onJpegError; +} + +void NV12Compressor::ErrorManager::onJpegError(j_common_ptr cinfo) { + // NOTE! Do not construct any non-trivial objects in this method at the top + // scope. Their destructors will not be called. If you do need such an + // object create a local scope that does not include the longjmp call, + // that ensures the object is destroyed before longjmp is called. + ErrorManager* errorManager = reinterpret_cast(cinfo->err); + + // Format and log error message + char errorMessage[JMSG_LENGTH_MAX]; + (*errorManager->format_message)(cinfo, errorMessage); + errorMessage[sizeof(errorMessage) - 1] = '\0'; + ALOGE("JPEG compression error: %s", errorMessage); + jpeg_destroy(cinfo); + + // And through the looking glass we go + longjmp(errorManager->mJumpBuffer, 1); +} + +NV12Compressor::DestinationManager::DestinationManager() { + init_destination = &initDestination; + empty_output_buffer = &emptyOutputBuffer; + term_destination = &termDestination; +} + +void NV12Compressor::DestinationManager::initDestination(j_compress_ptr cinfo) { + auto manager = reinterpret_cast(cinfo->dest); + + // Start out with some arbitrary but not too large buffer size + manager->mBuffer.resize(16 * 1024); + manager->next_output_byte = &manager->mBuffer[0]; + manager->free_in_buffer = manager->mBuffer.size(); +} + +boolean NV12Compressor::DestinationManager::emptyOutputBuffer( + j_compress_ptr cinfo) { + auto manager = reinterpret_cast(cinfo->dest); + + // Keep doubling the size of the buffer for a very low, amortized + // performance cost of the allocations + size_t oldSize = manager->mBuffer.size(); + manager->mBuffer.resize(oldSize * 2); + manager->next_output_byte = &manager->mBuffer[oldSize]; + manager->free_in_buffer = manager->mBuffer.size() - oldSize; + return manager->free_in_buffer != 0; +} + +void NV12Compressor::DestinationManager::termDestination(j_compress_ptr cinfo) { + auto manager = reinterpret_cast(cinfo->dest); + + // Resize down to the exact size of the output, that is remove as many + // bytes as there are left in the buffer + manager->mBuffer.resize(manager->mBuffer.size() - manager->free_in_buffer); +} + +status_t NV12Compressor::findJpegSize(uint8_t *jpegBuffer, size_t maxSize, size_t *size /*out*/) { + if ((size == nullptr) || (jpegBuffer == nullptr)) { + return BAD_VALUE; + } + + if (checkJpegStart(jpegBuffer) == 0) { + return BAD_VALUE; + } + + // Read JFIF segment markers, skip over segment data + *size = kMarkerLength; //jump to Start Of Image + while (*size <= maxSize - kMarkerLength) { + segment_t *segment = (segment_t*)(jpegBuffer + *size); + uint8_t type = checkJpegMarker(segment->marker); + if (type == 0) { // invalid marker, no more segments, begin JPEG data + break; + } + if (type == kEndOfImage || *size > maxSize - sizeof(segment_t)) { + return BAD_VALUE; + } + + size_t length = ntohs(segment->length); + *size += length + kMarkerLength; + } + + // Find End of Image + // Scan JPEG buffer until End of Image + bool foundEnd = false; + for ( ; *size <= maxSize - kMarkerLength; (*size)++) { + if (checkJpegEnd(jpegBuffer + *size)) { + foundEnd = true; + *size += kMarkerLength; + break; + } + } + + if (!foundEnd) { + return BAD_VALUE; + } + + if (*size > maxSize) { + *size = maxSize; + } + + return OK; +} + +status_t NV12Compressor::getJpegImageDimensions(uint8_t *jpegBuffer, + size_t jpegBufferSize, size_t *width /*out*/, size_t *height /*out*/) { + if ((jpegBuffer == nullptr) || (width == nullptr) || (height == nullptr) || + (jpegBufferSize == 0u)) { + return BAD_VALUE; + } + + // Scan JPEG buffer until Start of Frame + bool foundSOF = false; + size_t currentPos; + for (currentPos = 0; currentPos <= jpegBufferSize - kMarkerLength; currentPos++) { + if (checkStartOfFrame(jpegBuffer + currentPos)) { + foundSOF = true; + currentPos += kMarkerLength; + break; + } + } + + if (!foundSOF) { + ALOGE("%s: Start of Frame not found", __func__); + return BAD_VALUE; + } + + sof_t *startOfFrame = reinterpret_cast (jpegBuffer + currentPos); + *width = ntohs(startOfFrame->width); + *height = ntohs(startOfFrame->height); + + return OK; +} + +status_t NV12Compressor::getExifOrientation(uint8_t *jpegBuffer, size_t jpegBufferSize, + ExifOrientation *exifValue /*out*/) { + if ((jpegBuffer == nullptr) || (exifValue == nullptr) || (jpegBufferSize == 0u)) { + return BAD_VALUE; + } + + std::unique_ptr exifData(exif_data_new()); + exif_data_load_data(exifData.get(), jpegBuffer, jpegBufferSize); + ExifEntry *orientation = exif_content_get_entry(exifData->ifd[EXIF_IFD_0], + EXIF_TAG_ORIENTATION); + if ((orientation == nullptr) || (orientation->size != sizeof(ExifShort))) { + return BAD_VALUE; + } + + auto orientationValue = exif_get_short(orientation->data, + exif_data_get_byte_order(exifData.get())); + status_t ret; + switch (orientationValue) { + case ExifOrientation::ORIENTATION_0_DEGREES: + case ExifOrientation::ORIENTATION_90_DEGREES: + case ExifOrientation::ORIENTATION_180_DEGREES: + case ExifOrientation::ORIENTATION_270_DEGREES: + *exifValue = static_cast (orientationValue); + ret = OK; + break; + default: + ALOGE("%s: Unexpected EXIF orientation value: %u", __FUNCTION__, orientationValue); + ret = BAD_VALUE; + } + + return ret; +} diff --git a/services/camera/libcameraservice/tests/NV12Compressor.h b/services/camera/libcameraservice/tests/NV12Compressor.h new file mode 100644 index 0000000000..ee22d5e282 --- /dev/null +++ b/services/camera/libcameraservice/tests/NV12Compressor.h @@ -0,0 +1,134 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef TEST_CAMERA_JPEG_STUB_NV12_COMPRESSOR_H +#define TEST_CAMERA_JPEG_STUB_NV12_COMPRESSOR_H + +#include +#include +extern "C" { +#include +#include +} + +#include +#include + +#include "../utils/ExifUtils.h" + +struct _ExifData; +typedef _ExifData ExifData; + +class NV12Compressor { +public: + NV12Compressor() {} + + /* Compress |data| which represents raw NV21 encoded data of dimensions + * |width| * |height|. + */ + bool compress(const unsigned char* data, int width, int height, int quality); + bool compressWithExifOrientation(const unsigned char* data, int width, int height, int quality, + android::camera3::ExifOrientation exifValue); + + /* Get a reference to the compressed data, this will return an empty vector + * if compress has not been called yet + */ + const std::vector& getCompressedData() const; + + // Utility methods + static android::status_t findJpegSize(uint8_t *jpegBuffer, size_t maxSize, + size_t *size /*out*/); + + static android::status_t getExifOrientation(uint8_t *jpegBuffer, + size_t jpegBufferSize, android::camera3::ExifOrientation *exifValue /*out*/); + + /* Get Jpeg image dimensions from the first Start Of Frame. Please note that due to the + * way the jpeg buffer is scanned if the image contains a thumbnail, then the size returned + * will be of the thumbnail and not the main image. + */ + static android::status_t getJpegImageDimensions(uint8_t *jpegBuffer, size_t jpegBufferSize, + size_t *width /*out*/, size_t *height /*out*/); + +private: + + struct DestinationManager : jpeg_destination_mgr { + DestinationManager(); + + static void initDestination(j_compress_ptr cinfo); + static boolean emptyOutputBuffer(j_compress_ptr cinfo); + static void termDestination(j_compress_ptr cinfo); + + std::vector mBuffer; + }; + + struct ErrorManager : jpeg_error_mgr { + ErrorManager(); + + static void onJpegError(j_common_ptr cinfo); + + jmp_buf mJumpBuffer; + }; + + static const size_t kMarkerLength = 2; // length of a marker + static const uint8_t kMarker = 0xFF; // First byte of marker + static const uint8_t kStartOfImage = 0xD8; // Start of Image + static const uint8_t kEndOfImage = 0xD9; // End of Image + static const uint8_t kStartOfFrame = 0xC0; // Start of Frame + + struct __attribute__((packed)) segment_t { + uint8_t marker[kMarkerLength]; + uint16_t length; + }; + + struct __attribute__((packed)) sof_t { + uint16_t length; + uint8_t precision; + uint16_t height; + uint16_t width; + }; + + // check for start of image marker + static bool checkStartOfFrame(uint8_t* buf) { + return buf[0] == kMarker && buf[1] == kStartOfFrame; + } + + // check for start of image marker + static bool checkJpegStart(uint8_t* buf) { + return buf[0] == kMarker && buf[1] == kStartOfImage; + } + + // check for End of Image marker + static bool checkJpegEnd(uint8_t *buf) { + return buf[0] == kMarker && buf[1] == kEndOfImage; + } + + // check for arbitrary marker, returns marker type (second byte) + // returns 0 if no marker found. Note: 0x00 is not a valid marker type + static uint8_t checkJpegMarker(uint8_t *buf) { + return (buf[0] == kMarker) ? buf[1] : 0; + } + + jpeg_compress_struct mCompressInfo; + DestinationManager mDestManager; + ErrorManager mErrorManager; + + bool configureCompressor(int width, int height, int quality); + bool compressData(const unsigned char* data, ExifData* exifData); + bool attachExifData(ExifData* exifData); +}; + +#endif // TEST_CAMERA_JPEG_STUB_NV12_COMPRESSOR_H + diff --git a/services/camera/libcameraservice/utils/ExifUtils.cpp b/services/camera/libcameraservice/utils/ExifUtils.cpp index 4dea8b5d3b..c0afdc18ca 100644 --- a/services/camera/libcameraservice/utils/ExifUtils.cpp +++ b/services/camera/libcameraservice/utils/ExifUtils.cpp @@ -55,6 +55,7 @@ public: // Initialize() can be called multiple times. The setting of Exif tags will be // cleared. virtual bool initialize(const unsigned char *app1Segment, size_t app1SegmentSize); + virtual bool initializeEmpty(); // set all known fields from a metadata structure virtual bool setFromMetadata(const CameraMetadata& metadata, @@ -150,7 +151,11 @@ public: // sets image orientation. // Returns false if memory allocation fails. - virtual bool setOrientation(uint16_t orientation); + virtual bool setOrientation(uint16_t degrees); + + // sets image orientation. + // Returns false if memory allocation fails. + virtual bool setOrientationValue(ExifOrientation orientationValue); // sets the shutter speed. // Returns false if memory allocation fails. @@ -314,6 +319,26 @@ bool ExifUtilsImpl::initialize(const unsigned char *app1Segment, size_t app1Segm return true; } +bool ExifUtilsImpl::initializeEmpty() { + reset(); + exif_data_ = exif_data_new(); + if (exif_data_ == nullptr) { + ALOGE("%s: allocate memory for exif_data_ failed", __FUNCTION__); + return false; + } + // set the image options. + exif_data_set_option(exif_data_, EXIF_DATA_OPTION_FOLLOW_SPECIFICATION); + exif_data_set_data_type(exif_data_, EXIF_DATA_TYPE_COMPRESSED); + exif_data_set_byte_order(exif_data_, EXIF_BYTE_ORDER_INTEL); + + // set exif version to 2.2. + if (!setExifVersion("0220")) { + return false; + } + + return true; +} + bool ExifUtilsImpl::setAperture(float aperture) { float apexValue = convertToApex(aperture); SET_RATIONAL(EXIF_IFD_EXIF, EXIF_TAG_APERTURE_VALUE, @@ -609,32 +634,26 @@ bool ExifUtilsImpl::setExposureBias(int32_t ev, return true; } -bool ExifUtilsImpl::setOrientation(uint16_t orientation) { - /* - * Orientation value: - * 1 2 3 4 5 6 7 8 - * - * 888888 888888 88 88 8888888888 88 88 8888888888 - * 88 88 88 88 88 88 88 88 88 88 88 88 - * 8888 8888 8888 8888 88 8888888888 8888888888 88 - * 88 88 88 88 - * 88 88 888888 888888 - */ - int value = 1; - switch (orientation) { +bool ExifUtilsImpl::setOrientation(uint16_t degrees) { + ExifOrientation value = ExifOrientation::ORIENTATION_0_DEGREES; + switch (degrees) { case 90: - value = 6; + value = ExifOrientation::ORIENTATION_90_DEGREES; break; case 180: - value = 3; + value = ExifOrientation::ORIENTATION_180_DEGREES; break; case 270: - value = 8; + value = ExifOrientation::ORIENTATION_270_DEGREES; break; default: break; } - SET_SHORT(EXIF_IFD_0, EXIF_TAG_ORIENTATION, value); + return setOrientationValue(value); +} + +bool ExifUtilsImpl::setOrientationValue(ExifOrientation orientationValue) { + SET_SHORT(EXIF_IFD_0, EXIF_TAG_ORIENTATION, orientationValue); return true; } diff --git a/services/camera/libcameraservice/utils/ExifUtils.h b/services/camera/libcameraservice/utils/ExifUtils.h index c78bab9961..f1d0205759 100644 --- a/services/camera/libcameraservice/utils/ExifUtils.h +++ b/services/camera/libcameraservice/utils/ExifUtils.h @@ -22,6 +22,24 @@ namespace android { namespace camera3 { +/* + * Orientation value: + * 1 2 3 4 5 6 7 8 + * + * 888888 888888 88 88 8888888888 88 88 8888888888 + * 88 88 88 88 88 88 88 88 88 88 88 88 + * 8888 8888 8888 8888 88 8888888888 8888888888 88 + * 88 88 88 88 + * 88 88 888888 888888 + */ +enum ExifOrientation : uint16_t { + ORIENTATION_UNDEFINED = 0x0, + ORIENTATION_0_DEGREES = 0x1, + ORIENTATION_90_DEGREES = 0x6, + ORIENTATION_180_DEGREES = 0x3, + ORIENTATION_270_DEGREES = 0x8, +}; + // This is based on the camera HIDL shim implementation, which was in turned // based on original ChromeOS ARC implementation of a V4L2 HAL @@ -49,6 +67,7 @@ public: // Initialize() can be called multiple times. The setting of Exif tags will be // cleared. virtual bool initialize(const unsigned char *app1Segment, size_t app1SegmentSize) = 0; + virtual bool initializeEmpty() = 0; // Set all known fields from a metadata structure virtual bool setFromMetadata(const CameraMetadata& metadata, @@ -142,7 +161,11 @@ public: // Sets image orientation. // Returns false if memory allocation fails. - virtual bool setOrientation(uint16_t orientation) = 0; + virtual bool setOrientation(uint16_t degrees) = 0; + + // Sets image orientation. + // Returns false if memory allocation fails. + virtual bool setOrientationValue(ExifOrientation orientationValue) = 0; // Sets the shutter speed. // Returns false if memory allocation fails.