From 4b08d5dce8d3a8aef64b493da1fa3e4bf114b347 Mon Sep 17 00:00:00 2001 From: Emilian Peev Date: Fri, 1 Feb 2019 09:53:14 -0800 Subject: [PATCH] Camera: Keep Depth EXIF orientation consistent Depth and confidence maps should always use the same EXIF orientation as the main color image. Bug: 123699590 Test: Manual using application, Camera CTS, adb shell /data/nativetest64/cameraservice_test/cameraservice_test --gtest_filter=DepthProcessorTest.* Change-Id: I0d887798e8717cdff81aba10d595dc3ccfe99197 --- services/camera/libcameraservice/Android.bp | 3 + .../common/DepthPhotoProcessor.cpp | 69 +++- .../camera/libcameraservice/tests/Android.mk | 4 + .../tests/DepthProcessorTest.cpp | 280 ++++++++++++++ .../libcameraservice/tests/NV12Compressor.cpp | 349 ++++++++++++++++++ .../libcameraservice/tests/NV12Compressor.h | 115 ++++++ .../libcameraservice/utils/ExifUtils.cpp | 55 ++- .../camera/libcameraservice/utils/ExifUtils.h | 25 +- 8 files changed, 876 insertions(+), 24 deletions(-) create mode 100644 services/camera/libcameraservice/tests/DepthProcessorTest.cpp create mode 100644 services/camera/libcameraservice/tests/NV12Compressor.cpp create mode 100644 services/camera/libcameraservice/tests/NV12Compressor.h diff --git a/services/camera/libcameraservice/Android.bp b/services/camera/libcameraservice/Android.bp index 2ca83561a2..3ce11aeab5 100644 --- a/services/camera/libcameraservice/Android.bp +++ b/services/camera/libcameraservice/Android.bp @@ -137,6 +137,7 @@ cc_library_shared { name: "libdepthphoto", srcs: [ + "utils/ExifUtils.cpp", "common/DepthPhotoProcessor.cpp", ], @@ -150,6 +151,8 @@ cc_library_shared { "libcutils", "libjpeg", "libmemunreachable", + "libexif", + "libcamera_client", ], include_dirs: [ diff --git a/services/camera/libcameraservice/common/DepthPhotoProcessor.cpp b/services/camera/libcameraservice/common/DepthPhotoProcessor.cpp index a945aca13b..3af422064f 100644 --- a/services/camera/libcameraservice/common/DepthPhotoProcessor.cpp +++ b/services/camera/libcameraservice/common/DepthPhotoProcessor.cpp @@ -32,9 +32,12 @@ #include #include #include +#include +#include #include #include #include +#include #include #include #include @@ -61,8 +64,44 @@ using dynamic_depth::Profiles; namespace android { namespace camera3 { +ExifOrientation getExifOrientation(const unsigned char *jpegBuffer, size_t jpegBufferSize) { + if ((jpegBuffer == nullptr) || (jpegBufferSize == 0)) { + return ExifOrientation::ORIENTATION_UNDEFINED; + } + + auto exifData = exif_data_new(); + exif_data_load_data(exifData, jpegBuffer, jpegBufferSize); + ExifEntry *orientation = exif_content_get_entry(exifData->ifd[EXIF_IFD_0], + EXIF_TAG_ORIENTATION); + if ((orientation == nullptr) || (orientation->size != sizeof(ExifShort))) { + ALOGV("%s: Orientation EXIF entry invalid!", __FUNCTION__); + exif_data_unref(exifData); + return ExifOrientation::ORIENTATION_0_DEGREES; + } + + auto orientationValue = exif_get_short(orientation->data, exif_data_get_byte_order(exifData)); + ExifOrientation ret; + switch (orientationValue) { + case ExifOrientation::ORIENTATION_0_DEGREES: + case ExifOrientation::ORIENTATION_90_DEGREES: + case ExifOrientation::ORIENTATION_180_DEGREES: + case ExifOrientation::ORIENTATION_270_DEGREES: + ret = static_cast (orientationValue); + break; + default: + ALOGE("%s: Unexpected EXIF orientation value: %d, defaulting to 0 degrees", + __FUNCTION__, orientationValue); + ret = ExifOrientation::ORIENTATION_0_DEGREES; + } + + exif_data_unref(exifData); + + return ret; +} + status_t encodeGrayscaleJpeg(size_t width, size_t height, uint8_t *in, void *out, - const size_t maxOutSize, uint8_t jpegQuality, size_t &actualSize) { + const size_t maxOutSize, uint8_t jpegQuality, ExifOrientation exifOrientation, + size_t &actualSize) { status_t ret; // libjpeg is a C library so we use C-style "inheritance" by // putting libjpeg's jpeg_destination_mgr first in our custom @@ -151,6 +190,23 @@ status_t encodeGrayscaleJpeg(size_t width, size_t height, uint8_t *in, void *out jpeg_start_compress(&cinfo, TRUE); + if (exifOrientation != ExifOrientation::ORIENTATION_UNDEFINED) { + std::unique_ptr utils(ExifUtils::create()); + utils->initializeEmpty(); + utils->setImageWidth(width); + utils->setImageHeight(height); + utils->setOrientationValue(exifOrientation); + + if (utils->generateApp1()) { + const uint8_t* exifBuffer = utils->getApp1Buffer(); + size_t exifBufferSize = utils->getApp1Length(); + jpeg_write_marker(&cinfo, JPEG_APP0 + 1, static_cast(exifBuffer), + exifBufferSize); + } else { + ALOGE("%s: Unable to generate App1 buffer", __FUNCTION__); + } + } + for (size_t i = 0; i < cinfo.image_height; i++) { auto currentRow = static_cast(in + i*width); jpeg_write_scanlines(&cinfo, ¤tRow, /*num_lines*/1); @@ -169,7 +225,7 @@ status_t encodeGrayscaleJpeg(size_t width, size_t height, uint8_t *in, void *out } std::unique_ptr processDepthMapFrame(DepthPhotoInputFrame inputFrame, - std::vector> *items /*out*/) { + ExifOrientation exifOrientation, std::vector> *items /*out*/) { std::vector points, confidence; size_t pointCount = inputFrame.mDepthMapWidth * inputFrame.mDepthMapHeight; @@ -227,7 +283,7 @@ std::unique_ptr processDepthMapFrame(DepthPhotoInputFra size_t actualJpegSize; auto ret = encodeGrayscaleJpeg(inputFrame.mDepthMapWidth, inputFrame.mDepthMapHeight, pointsQuantized.data(), depthParams.depth_image_data.data(), inputFrame.mMaxJpegSize, - inputFrame.mJpegQuality, actualJpegSize); + inputFrame.mJpegQuality, exifOrientation, actualJpegSize); if (ret != NO_ERROR) { ALOGE("%s: Depth map compression failed!", __FUNCTION__); return nullptr; @@ -236,7 +292,7 @@ std::unique_ptr processDepthMapFrame(DepthPhotoInputFra ret = encodeGrayscaleJpeg(inputFrame.mDepthMapWidth, inputFrame.mDepthMapHeight, confidenceQuantized.data(), depthParams.confidence_data.data(), inputFrame.mMaxJpegSize, - inputFrame.mJpegQuality, actualJpegSize); + inputFrame.mJpegQuality, exifOrientation, actualJpegSize); if (ret != NO_ERROR) { ALOGE("%s: Confidence map compression failed!", __FUNCTION__); return nullptr; @@ -262,7 +318,10 @@ extern "C" int processDepthPhotoFrame(DepthPhotoInputFrame inputFrame, size_t de return BAD_VALUE; } - cameraParams->depth_map = processDepthMapFrame(inputFrame, &items); + ExifOrientation exifOrientation = getExifOrientation( + reinterpret_cast (inputFrame.mMainJpegBuffer), + inputFrame.mMainJpegSize); + cameraParams->depth_map = processDepthMapFrame(inputFrame, exifOrientation, &items); if (cameraParams->depth_map == nullptr) { ALOGE("%s: Depth map processing failed!", __FUNCTION__); return BAD_VALUE; diff --git a/services/camera/libcameraservice/tests/Android.mk b/services/camera/libcameraservice/tests/Android.mk index d777ca1cb1..b4e7c32bef 100644 --- a/services/camera/libcameraservice/tests/Android.mk +++ b/services/camera/libcameraservice/tests/Android.mk @@ -27,6 +27,8 @@ LOCAL_SHARED_LIBRARIES := \ libcamera_client \ libcamera_metadata \ libutils \ + libjpeg \ + libexif \ android.hardware.camera.common@1.0 \ android.hardware.camera.provider@2.4 \ android.hardware.camera.provider@2.5 \ @@ -36,6 +38,8 @@ LOCAL_SHARED_LIBRARIES := \ LOCAL_C_INCLUDES += \ system/media/private/camera/include \ + external/dynamic_depth/includes \ + external/dynamic_depth/internal \ LOCAL_CFLAGS += -Wall -Wextra -Werror diff --git a/services/camera/libcameraservice/tests/DepthProcessorTest.cpp b/services/camera/libcameraservice/tests/DepthProcessorTest.cpp new file mode 100644 index 0000000000..9898122e9d --- /dev/null +++ b/services/camera/libcameraservice/tests/DepthProcessorTest.cpp @@ -0,0 +1,280 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_NDEBUG 0 +#define LOG_TAG "DepthProcessorTest" + +#include +#include + +#include +#include + +#include "../common/DepthPhotoProcessor.h" +#include "../utils/ExifUtils.h" +#include "NV12Compressor.h" + +using namespace android; +using namespace android::camera3; + +static const size_t kTestBufferWidth = 640; +static const size_t kTestBufferHeight = 480; +static const size_t kTestBufferNV12Size ((((kTestBufferWidth) * (kTestBufferHeight)) * 3) / 2); +static const size_t kTestBufferDepthSize (kTestBufferWidth * kTestBufferHeight); +static const size_t kSeed = 1234; + +void linkToDepthPhotoLibrary(void **libHandle /*out*/, + process_depth_photo_frame *processFrameFunc /*out*/) { + ASSERT_NE(libHandle, nullptr); + ASSERT_NE(processFrameFunc, nullptr); + + *libHandle = dlopen(kDepthPhotoLibrary, RTLD_NOW | RTLD_LOCAL); + if (*libHandle != nullptr) { + *processFrameFunc = reinterpret_cast ( + dlsym(*libHandle, kDepthPhotoProcessFunction)); + ASSERT_NE(*processFrameFunc, nullptr); + } +} + +void generateColorJpegBuffer(int jpegQuality, ExifOrientation orientationValue, bool includeExif, + std::vector *colorJpegBuffer /*out*/) { + ASSERT_NE(colorJpegBuffer, nullptr); + + std::array colorSourceBuffer; + std::default_random_engine gen(kSeed); + std::uniform_int_distribution uniDist(0, UINT8_MAX - 1); + for (size_t i = 0; i < colorSourceBuffer.size(); i++) { + colorSourceBuffer[i] = uniDist(gen); + } + NV12Compressor jpegCompressor; + if (includeExif) { + ASSERT_TRUE(jpegCompressor.compressWithExifOrientation( + reinterpret_cast (colorSourceBuffer.data()), + kTestBufferWidth, kTestBufferHeight, jpegQuality, orientationValue)); + } else { + ASSERT_TRUE(jpegCompressor.compress( + reinterpret_cast (colorSourceBuffer.data()), + kTestBufferWidth, kTestBufferHeight, jpegQuality)); + } + + *colorJpegBuffer = std::move(jpegCompressor.getCompressedData()); + ASSERT_FALSE(colorJpegBuffer->empty()); +} + +void generateDepth16Buffer(std::array *depth16Buffer /*out*/) { + ASSERT_NE(depth16Buffer, nullptr); + std::default_random_engine gen(kSeed+1); + std::uniform_int_distribution uniDist(0, UINT16_MAX - 1); + for (size_t i = 0; i < depth16Buffer->size(); i++) { + (*depth16Buffer)[i] = uniDist(gen); + } +} + +TEST(DepthProcessorTest, LinkToLibray) { + void *libHandle; + process_depth_photo_frame processFunc; + linkToDepthPhotoLibrary(&libHandle, &processFunc); + if (libHandle != nullptr) { + dlclose(libHandle); + } +} + +TEST(DepthProcessorTest, BadInput) { + void *libHandle; + int jpegQuality = 95; + + process_depth_photo_frame processFunc; + linkToDepthPhotoLibrary(&libHandle, &processFunc); + if (libHandle == nullptr) { + // Depth library no present, nothing more to test. + return; + } + + DepthPhotoInputFrame inputFrame; + // Worst case both depth and confidence maps have the same size as the main color image. + inputFrame.mMaxJpegSize = inputFrame.mMainJpegSize * 3; + + std::vector colorJpegBuffer; + generateColorJpegBuffer(jpegQuality, ExifOrientation::ORIENTATION_UNDEFINED, + /*includeExif*/ false, &colorJpegBuffer); + + std::array depth16Buffer; + generateDepth16Buffer(&depth16Buffer); + + std::vector depthPhotoBuffer(inputFrame.mMaxJpegSize); + size_t actualDepthPhotoSize = 0; + + inputFrame.mMainJpegWidth = kTestBufferWidth; + inputFrame.mMainJpegHeight = kTestBufferHeight; + inputFrame.mJpegQuality = jpegQuality; + ASSERT_NE(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(), + &actualDepthPhotoSize), 0); + + inputFrame.mMainJpegBuffer = reinterpret_cast (colorJpegBuffer.data()); + inputFrame.mMainJpegSize = colorJpegBuffer.size(); + ASSERT_NE(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(), + &actualDepthPhotoSize), 0); + + inputFrame.mDepthMapBuffer = depth16Buffer.data(); + inputFrame.mDepthMapWidth = inputFrame.mDepthMapStride = kTestBufferWidth; + inputFrame.mDepthMapHeight = kTestBufferHeight; + ASSERT_NE(processFunc(inputFrame, depthPhotoBuffer.size(), nullptr, + &actualDepthPhotoSize), 0); + + ASSERT_NE(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(), nullptr), + 0); + + dlclose(libHandle); +} + +TEST(DepthProcessorTest, BasicDepthPhotoValidation) { + void *libHandle; + int jpegQuality = 95; + + process_depth_photo_frame processFunc; + linkToDepthPhotoLibrary(&libHandle, &processFunc); + if (libHandle == nullptr) { + // Depth library no present, nothing more to test. + return; + } + + std::vector colorJpegBuffer; + generateColorJpegBuffer(jpegQuality, ExifOrientation::ORIENTATION_UNDEFINED, + /*includeExif*/ false, &colorJpegBuffer); + + std::array depth16Buffer; + generateDepth16Buffer(&depth16Buffer); + + DepthPhotoInputFrame inputFrame; + inputFrame.mMainJpegBuffer = reinterpret_cast (colorJpegBuffer.data()); + inputFrame.mMainJpegSize = colorJpegBuffer.size(); + // Worst case both depth and confidence maps have the same size as the main color image. + inputFrame.mMaxJpegSize = inputFrame.mMainJpegSize * 3; + inputFrame.mMainJpegWidth = kTestBufferWidth; + inputFrame.mMainJpegHeight = kTestBufferHeight; + inputFrame.mJpegQuality = jpegQuality; + inputFrame.mDepthMapBuffer = depth16Buffer.data(); + inputFrame.mDepthMapWidth = inputFrame.mDepthMapStride = kTestBufferWidth; + inputFrame.mDepthMapHeight = kTestBufferHeight; + + std::vector depthPhotoBuffer(inputFrame.mMaxJpegSize); + size_t actualDepthPhotoSize = 0; + ASSERT_EQ(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(), + &actualDepthPhotoSize), 0); + ASSERT_TRUE((actualDepthPhotoSize > 0) && (depthPhotoBuffer.size() >= actualDepthPhotoSize)); + + // The final depth photo must consist of three jpeg images: + // - the main color image + // - the depth map image + // - the confidence map image + size_t mainJpegSize = 0; + ASSERT_EQ(NV12Compressor::findJpegSize(depthPhotoBuffer.data(), actualDepthPhotoSize, + &mainJpegSize), OK); + ASSERT_TRUE((mainJpegSize > 0) && (mainJpegSize < actualDepthPhotoSize)); + size_t depthMapSize = 0; + ASSERT_EQ(NV12Compressor::findJpegSize(depthPhotoBuffer.data() + mainJpegSize, + actualDepthPhotoSize - mainJpegSize, &depthMapSize), OK); + ASSERT_TRUE((depthMapSize > 0) && (depthMapSize < (actualDepthPhotoSize - mainJpegSize))); + + dlclose(libHandle); +} + +TEST(DepthProcessorTest, TestDepthPhotoExifOrientation) { + void *libHandle; + int jpegQuality = 95; + + process_depth_photo_frame processFunc; + linkToDepthPhotoLibrary(&libHandle, &processFunc); + if (libHandle == nullptr) { + // Depth library no present, nothing more to test. + return; + } + + ExifOrientation exifOrientations[] = { ExifOrientation::ORIENTATION_UNDEFINED, + ExifOrientation::ORIENTATION_0_DEGREES, ExifOrientation::ORIENTATION_90_DEGREES, + ExifOrientation::ORIENTATION_180_DEGREES, ExifOrientation::ORIENTATION_270_DEGREES }; + for (auto exifOrientation : exifOrientations) { + std::vector colorJpegBuffer; + generateColorJpegBuffer(jpegQuality, exifOrientation, /*includeExif*/ true, + &colorJpegBuffer); + if (exifOrientation != ExifOrientation::ORIENTATION_UNDEFINED) { + auto jpegExifOrientation = ExifOrientation::ORIENTATION_UNDEFINED; + ASSERT_EQ(NV12Compressor::getExifOrientation( + reinterpret_cast (colorJpegBuffer.data()), + colorJpegBuffer.size(), &jpegExifOrientation), OK); + ASSERT_EQ(exifOrientation, jpegExifOrientation); + } + + std::array depth16Buffer; + generateDepth16Buffer(&depth16Buffer); + + DepthPhotoInputFrame inputFrame; + inputFrame.mMainJpegBuffer = reinterpret_cast (colorJpegBuffer.data()); + inputFrame.mMainJpegSize = colorJpegBuffer.size(); + // Worst case both depth and confidence maps have the same size as the main color image. + inputFrame.mMaxJpegSize = inputFrame.mMainJpegSize * 3; + inputFrame.mMainJpegWidth = kTestBufferWidth; + inputFrame.mMainJpegHeight = kTestBufferHeight; + inputFrame.mJpegQuality = jpegQuality; + inputFrame.mDepthMapBuffer = depth16Buffer.data(); + inputFrame.mDepthMapWidth = inputFrame.mDepthMapStride = kTestBufferWidth; + inputFrame.mDepthMapHeight = kTestBufferHeight; + + std::vector depthPhotoBuffer(inputFrame.mMaxJpegSize); + size_t actualDepthPhotoSize = 0; + ASSERT_EQ(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(), + &actualDepthPhotoSize), 0); + ASSERT_TRUE((actualDepthPhotoSize > 0) && + (depthPhotoBuffer.size() >= actualDepthPhotoSize)); + + size_t mainJpegSize = 0; + ASSERT_EQ(NV12Compressor::findJpegSize(depthPhotoBuffer.data(), actualDepthPhotoSize, + &mainJpegSize), OK); + ASSERT_TRUE((mainJpegSize > 0) && (mainJpegSize < actualDepthPhotoSize)); + size_t depthMapSize = 0; + ASSERT_EQ(NV12Compressor::findJpegSize(depthPhotoBuffer.data() + mainJpegSize, + actualDepthPhotoSize - mainJpegSize, &depthMapSize), OK); + ASSERT_TRUE((depthMapSize > 0) && (depthMapSize < (actualDepthPhotoSize - mainJpegSize))); + size_t confidenceMapSize = actualDepthPhotoSize - (mainJpegSize + depthMapSize); + + //Depth and confidence images must have the same EXIF orientation as the source + auto depthJpegExifOrientation = ExifOrientation::ORIENTATION_UNDEFINED; + ASSERT_EQ(NV12Compressor::getExifOrientation( + reinterpret_cast (depthPhotoBuffer.data() + mainJpegSize), + depthMapSize, &depthJpegExifOrientation), OK); + if (exifOrientation == ORIENTATION_UNDEFINED) { + // In case of undefined or missing EXIF orientation, always expect 0 degrees in the + // depth map. + ASSERT_EQ(depthJpegExifOrientation, ExifOrientation::ORIENTATION_0_DEGREES); + } else { + ASSERT_EQ(depthJpegExifOrientation, exifOrientation); + } + + auto confidenceJpegExifOrientation = ExifOrientation::ORIENTATION_UNDEFINED; + ASSERT_EQ(NV12Compressor::getExifOrientation( + reinterpret_cast (depthPhotoBuffer.data() + mainJpegSize + + depthMapSize), confidenceMapSize, &confidenceJpegExifOrientation), OK); + if (exifOrientation == ORIENTATION_UNDEFINED) { + // In case of undefined or missing EXIF orientation, always expect 0 degrees in the + // confidence map. + ASSERT_EQ(confidenceJpegExifOrientation, ExifOrientation::ORIENTATION_0_DEGREES); + } else { + ASSERT_EQ(confidenceJpegExifOrientation, exifOrientation); + } + } + + dlclose(libHandle); +} diff --git a/services/camera/libcameraservice/tests/NV12Compressor.cpp b/services/camera/libcameraservice/tests/NV12Compressor.cpp new file mode 100644 index 0000000000..b9f27faa57 --- /dev/null +++ b/services/camera/libcameraservice/tests/NV12Compressor.cpp @@ -0,0 +1,349 @@ +/* +* Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_NDEBUG 0 +#define LOG_TAG "Test_NV12Compressor" + +#include "NV12Compressor.h" + +#include +#include + +using namespace android; +using namespace android::camera3; + +namespace std { +template <> +struct default_delete { + inline void operator()(ExifEntry* entry) const { exif_entry_unref(entry); } +}; + +template <> +struct default_delete { + inline void operator()(ExifData* data) const { exif_data_unref(data); } +}; + +} // namespace std + +bool NV12Compressor::compress(const unsigned char* data, int width, int height, int quality) { + if (!configureCompressor(width, height, quality)) { + // the method will have logged a more detailed error message than we can + // provide here so just return. + return false; + } + + return compressData(data, /*exifData*/ nullptr); +} + +bool NV12Compressor::compressWithExifOrientation(const unsigned char* data, int width, int height, + int quality, android::camera3::ExifOrientation exifValue) { + std::unique_ptr exifData(exif_data_new()); + if (exifData.get() == nullptr) { + return false; + } + + exif_data_set_option(exifData.get(), EXIF_DATA_OPTION_FOLLOW_SPECIFICATION); + exif_data_set_data_type(exifData.get(), EXIF_DATA_TYPE_COMPRESSED); + exif_data_set_byte_order(exifData.get(), EXIF_BYTE_ORDER_INTEL); + std::unique_ptr exifEntry(exif_entry_new()); + if (exifEntry.get() == nullptr) { + return false; + } + + exifEntry->tag = EXIF_TAG_ORIENTATION; + exif_content_add_entry(exifData->ifd[EXIF_IFD_0], exifEntry.get()); + exif_entry_initialize(exifEntry.get(), exifEntry->tag); + exif_set_short(exifEntry->data, EXIF_BYTE_ORDER_INTEL, exifValue); + + if (!configureCompressor(width, height, quality)) { + return false; + } + + return compressData(data, exifData.get()); +} + +const std::vector& NV12Compressor::getCompressedData() const { + return mDestManager.mBuffer; +} + +bool NV12Compressor::configureCompressor(int width, int height, int quality) { + mCompressInfo.err = jpeg_std_error(&mErrorManager); + // NOTE! DANGER! Do not construct any non-trivial objects below setjmp! + // The compiler will not generate code to destroy them during the return + // below so they will leak. Additionally, do not place any calls to libjpeg + // that can fail above this line or any error will cause undefined behavior. + if (setjmp(mErrorManager.mJumpBuffer)) { + // This is where the error handler will jump in case setup fails + // The error manager will ALOG an appropriate error message + return false; + } + + jpeg_create_compress(&mCompressInfo); + + mCompressInfo.image_width = width; + mCompressInfo.image_height = height; + mCompressInfo.input_components = 3; + mCompressInfo.in_color_space = JCS_YCbCr; + jpeg_set_defaults(&mCompressInfo); + + jpeg_set_quality(&mCompressInfo, quality, TRUE); + // It may seem weird to set color space here again but this will also set + // other fields. These fields might be overwritten by jpeg_set_defaults + jpeg_set_colorspace(&mCompressInfo, JCS_YCbCr); + mCompressInfo.raw_data_in = TRUE; + mCompressInfo.dct_method = JDCT_IFAST; + // Set sampling factors + mCompressInfo.comp_info[0].h_samp_factor = 2; + mCompressInfo.comp_info[0].v_samp_factor = 2; + mCompressInfo.comp_info[1].h_samp_factor = 1; + mCompressInfo.comp_info[1].v_samp_factor = 1; + mCompressInfo.comp_info[2].h_samp_factor = 1; + mCompressInfo.comp_info[2].v_samp_factor = 1; + + mCompressInfo.dest = &mDestManager; + + return true; +} + +static void deinterleave(const uint8_t* vuPlanar, std::vector& uRows, + std::vector& vRows, int rowIndex, int width, int height, int stride) { + int numRows = (height - rowIndex) / 2; + if (numRows > 8) numRows = 8; + for (int row = 0; row < numRows; ++row) { + int offset = ((rowIndex >> 1) + row) * stride; + const uint8_t* vu = vuPlanar + offset; + for (int i = 0; i < (width >> 1); ++i) { + int index = row * (width >> 1) + i; + uRows[index] = vu[1]; + vRows[index] = vu[0]; + vu += 2; + } + } +} + +bool NV12Compressor::compressData(const unsigned char* data, ExifData* exifData) { + const uint8_t* y[16]; + const uint8_t* cb[8]; + const uint8_t* cr[8]; + const uint8_t** planes[3] = { y, cb, cr }; + + int i, offset; + int width = mCompressInfo.image_width; + int height = mCompressInfo.image_height; + const uint8_t* yPlanar = data; + const uint8_t* vuPlanar = data + (width * height); + std::vector uRows(8 * (width >> 1)); + std::vector vRows(8 * (width >> 1)); + + // NOTE! DANGER! Do not construct any non-trivial objects below setjmp! + // The compiler will not generate code to destroy them during the return + // below so they will leak. Additionally, do not place any calls to libjpeg + // that can fail above this line or any error will cause undefined behavior. + if (setjmp(mErrorManager.mJumpBuffer)) { + // This is where the error handler will jump in case compression fails + // The error manager will ALOG an appropriate error message + return false; + } + + jpeg_start_compress(&mCompressInfo, TRUE); + + attachExifData(exifData); + + // process 16 lines of Y and 8 lines of U/V each time. + while (mCompressInfo.next_scanline < mCompressInfo.image_height) { + //deinterleave u and v + deinterleave(vuPlanar, uRows, vRows, mCompressInfo.next_scanline, + width, height, width); + + // Jpeg library ignores the rows whose indices are greater than height. + for (i = 0; i < 16; i++) { + // y row + y[i] = yPlanar + (mCompressInfo.next_scanline + i) * width; + + // construct u row and v row + if ((i & 1) == 0) { + // height and width are both halved because of downsampling + offset = (i >> 1) * (width >> 1); + cb[i/2] = &uRows[offset]; + cr[i/2] = &vRows[offset]; + } + } + jpeg_write_raw_data(&mCompressInfo, const_cast(planes), 16); + } + + jpeg_finish_compress(&mCompressInfo); + jpeg_destroy_compress(&mCompressInfo); + + return true; +} + +bool NV12Compressor::attachExifData(ExifData* exifData) { + if (exifData == nullptr) { + // This is not an error, we don't require EXIF data + return true; + } + + // Save the EXIF data to memory + unsigned char* rawData = nullptr; + unsigned int size = 0; + exif_data_save_data(exifData, &rawData, &size); + if (rawData == nullptr) { + ALOGE("Failed to create EXIF data block"); + return false; + } + + jpeg_write_marker(&mCompressInfo, JPEG_APP0 + 1, rawData, size); + free(rawData); + return true; +} + +NV12Compressor::ErrorManager::ErrorManager() { + error_exit = &onJpegError; +} + +void NV12Compressor::ErrorManager::onJpegError(j_common_ptr cinfo) { + // NOTE! Do not construct any non-trivial objects in this method at the top + // scope. Their destructors will not be called. If you do need such an + // object create a local scope that does not include the longjmp call, + // that ensures the object is destroyed before longjmp is called. + ErrorManager* errorManager = reinterpret_cast(cinfo->err); + + // Format and log error message + char errorMessage[JMSG_LENGTH_MAX]; + (*errorManager->format_message)(cinfo, errorMessage); + errorMessage[sizeof(errorMessage) - 1] = '\0'; + ALOGE("JPEG compression error: %s", errorMessage); + jpeg_destroy(cinfo); + + // And through the looking glass we go + longjmp(errorManager->mJumpBuffer, 1); +} + +NV12Compressor::DestinationManager::DestinationManager() { + init_destination = &initDestination; + empty_output_buffer = &emptyOutputBuffer; + term_destination = &termDestination; +} + +void NV12Compressor::DestinationManager::initDestination(j_compress_ptr cinfo) { + auto manager = reinterpret_cast(cinfo->dest); + + // Start out with some arbitrary but not too large buffer size + manager->mBuffer.resize(16 * 1024); + manager->next_output_byte = &manager->mBuffer[0]; + manager->free_in_buffer = manager->mBuffer.size(); +} + +boolean NV12Compressor::DestinationManager::emptyOutputBuffer( + j_compress_ptr cinfo) { + auto manager = reinterpret_cast(cinfo->dest); + + // Keep doubling the size of the buffer for a very low, amortized + // performance cost of the allocations + size_t oldSize = manager->mBuffer.size(); + manager->mBuffer.resize(oldSize * 2); + manager->next_output_byte = &manager->mBuffer[oldSize]; + manager->free_in_buffer = manager->mBuffer.size() - oldSize; + return manager->free_in_buffer != 0; +} + +void NV12Compressor::DestinationManager::termDestination(j_compress_ptr cinfo) { + auto manager = reinterpret_cast(cinfo->dest); + + // Resize down to the exact size of the output, that is remove as many + // bytes as there are left in the buffer + manager->mBuffer.resize(manager->mBuffer.size() - manager->free_in_buffer); +} + +status_t NV12Compressor::findJpegSize(uint8_t *jpegBuffer, size_t maxSize, size_t *size /*out*/) { + if ((size == nullptr) || (jpegBuffer == nullptr)) { + return BAD_VALUE; + } + + if (checkJpegStart(jpegBuffer) == 0) { + return BAD_VALUE; + } + + // Read JFIF segment markers, skip over segment data + *size = kMarkerLength; //jump to Start Of Image + while (*size <= maxSize - kMarkerLength) { + segment_t *segment = (segment_t*)(jpegBuffer + *size); + uint8_t type = checkJpegMarker(segment->marker); + if (type == 0) { // invalid marker, no more segments, begin JPEG data + break; + } + if (type == kEndOfImage || *size > maxSize - sizeof(segment_t)) { + return BAD_VALUE; + } + + size_t length = ntohs(segment->length); + *size += length + kMarkerLength; + } + + // Find End of Image + // Scan JPEG buffer until End of Image + bool foundEnd = false; + for ( ; *size <= maxSize - kMarkerLength; (*size)++) { + if (checkJpegEnd(jpegBuffer + *size)) { + foundEnd = true; + *size += kMarkerLength; + break; + } + } + + if (!foundEnd) { + return BAD_VALUE; + } + + if (*size > maxSize) { + *size = maxSize; + } + + return OK; +} + +status_t NV12Compressor::getExifOrientation(const unsigned char *jpegBuffer, size_t jpegBufferSize, + ExifOrientation *exifValue /*out*/) { + if ((jpegBuffer == nullptr) || (exifValue == nullptr) || (jpegBufferSize == 0u)) { + return BAD_VALUE; + } + + std::unique_ptr exifData(exif_data_new()); + exif_data_load_data(exifData.get(), jpegBuffer, jpegBufferSize); + ExifEntry *orientation = exif_content_get_entry(exifData->ifd[EXIF_IFD_0], + EXIF_TAG_ORIENTATION); + if ((orientation == nullptr) || (orientation->size != sizeof(ExifShort))) { + return BAD_VALUE; + } + + auto orientationValue = exif_get_short(orientation->data, + exif_data_get_byte_order(exifData.get())); + status_t ret; + switch (orientationValue) { + case ExifOrientation::ORIENTATION_0_DEGREES: + case ExifOrientation::ORIENTATION_90_DEGREES: + case ExifOrientation::ORIENTATION_180_DEGREES: + case ExifOrientation::ORIENTATION_270_DEGREES: + *exifValue = static_cast (orientationValue); + ret = OK; + break; + default: + ALOGE("%s: Unexpected EXIF orientation value: %u", __FUNCTION__, orientationValue); + ret = BAD_VALUE; + } + + return ret; +} diff --git a/services/camera/libcameraservice/tests/NV12Compressor.h b/services/camera/libcameraservice/tests/NV12Compressor.h new file mode 100644 index 0000000000..92804c10af --- /dev/null +++ b/services/camera/libcameraservice/tests/NV12Compressor.h @@ -0,0 +1,115 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef TEST_CAMERA_JPEG_STUB_NV12_COMPRESSOR_H +#define TEST_CAMERA_JPEG_STUB_NV12_COMPRESSOR_H + +#include +#include +extern "C" { +#include +#include +} + +#include +#include + +#include "../utils/ExifUtils.h" + +struct _ExifData; +typedef _ExifData ExifData; + +class NV12Compressor { +public: + NV12Compressor() {} + + /* Compress |data| which represents raw NV21 encoded data of dimensions + * |width| * |height|. + */ + bool compress(const unsigned char* data, int width, int height, int quality); + bool compressWithExifOrientation(const unsigned char* data, int width, int height, int quality, + android::camera3::ExifOrientation exifValue); + + /* Get a reference to the compressed data, this will return an empty vector + * if compress has not been called yet + */ + const std::vector& getCompressedData() const; + + static android::status_t findJpegSize(uint8_t *jpegBuffer, size_t maxSize, + size_t *size /*out*/); + + + static android::status_t getExifOrientation(const unsigned char *jpegBuffer, + size_t jpegBufferSize, android::camera3::ExifOrientation *exifValue /*out*/); + +private: + + struct DestinationManager : jpeg_destination_mgr { + DestinationManager(); + + static void initDestination(j_compress_ptr cinfo); + static boolean emptyOutputBuffer(j_compress_ptr cinfo); + static void termDestination(j_compress_ptr cinfo); + + std::vector mBuffer; + }; + + struct ErrorManager : jpeg_error_mgr { + ErrorManager(); + + static void onJpegError(j_common_ptr cinfo); + + jmp_buf mJumpBuffer; + }; + + static const size_t kMarkerLength = 2; // length of a marker + static const uint8_t kMarker = 0xFF; // First byte of marker + static const uint8_t kStartOfImage = 0xD8; // Start of Image + static const uint8_t kEndOfImage = 0xD9; // End of Image + + struct __attribute__((packed)) segment_t { + uint8_t marker[kMarkerLength]; + uint16_t length; + }; + + + // check for Start of Image marker + static bool checkJpegStart(uint8_t* buf) { + return buf[0] == kMarker && buf[1] == kStartOfImage; + } + + // check for End of Image marker + static bool checkJpegEnd(uint8_t *buf) { + return buf[0] == kMarker && buf[1] == kEndOfImage; + } + + // check for arbitrary marker, returns marker type (second byte) + // returns 0 if no marker found. Note: 0x00 is not a valid marker type + static uint8_t checkJpegMarker(uint8_t *buf) { + return (buf[0] == kMarker) ? buf[1] : 0; + } + + jpeg_compress_struct mCompressInfo; + DestinationManager mDestManager; + ErrorManager mErrorManager; + + bool configureCompressor(int width, int height, int quality); + bool compressData(const unsigned char* data, ExifData* exifData); + bool attachExifData(ExifData* exifData); +}; + +#endif // TEST_CAMERA_JPEG_STUB_NV12_COMPRESSOR_H + diff --git a/services/camera/libcameraservice/utils/ExifUtils.cpp b/services/camera/libcameraservice/utils/ExifUtils.cpp index 4dea8b5d3b..c0afdc18ca 100644 --- a/services/camera/libcameraservice/utils/ExifUtils.cpp +++ b/services/camera/libcameraservice/utils/ExifUtils.cpp @@ -55,6 +55,7 @@ public: // Initialize() can be called multiple times. The setting of Exif tags will be // cleared. virtual bool initialize(const unsigned char *app1Segment, size_t app1SegmentSize); + virtual bool initializeEmpty(); // set all known fields from a metadata structure virtual bool setFromMetadata(const CameraMetadata& metadata, @@ -150,7 +151,11 @@ public: // sets image orientation. // Returns false if memory allocation fails. - virtual bool setOrientation(uint16_t orientation); + virtual bool setOrientation(uint16_t degrees); + + // sets image orientation. + // Returns false if memory allocation fails. + virtual bool setOrientationValue(ExifOrientation orientationValue); // sets the shutter speed. // Returns false if memory allocation fails. @@ -314,6 +319,26 @@ bool ExifUtilsImpl::initialize(const unsigned char *app1Segment, size_t app1Segm return true; } +bool ExifUtilsImpl::initializeEmpty() { + reset(); + exif_data_ = exif_data_new(); + if (exif_data_ == nullptr) { + ALOGE("%s: allocate memory for exif_data_ failed", __FUNCTION__); + return false; + } + // set the image options. + exif_data_set_option(exif_data_, EXIF_DATA_OPTION_FOLLOW_SPECIFICATION); + exif_data_set_data_type(exif_data_, EXIF_DATA_TYPE_COMPRESSED); + exif_data_set_byte_order(exif_data_, EXIF_BYTE_ORDER_INTEL); + + // set exif version to 2.2. + if (!setExifVersion("0220")) { + return false; + } + + return true; +} + bool ExifUtilsImpl::setAperture(float aperture) { float apexValue = convertToApex(aperture); SET_RATIONAL(EXIF_IFD_EXIF, EXIF_TAG_APERTURE_VALUE, @@ -609,32 +634,26 @@ bool ExifUtilsImpl::setExposureBias(int32_t ev, return true; } -bool ExifUtilsImpl::setOrientation(uint16_t orientation) { - /* - * Orientation value: - * 1 2 3 4 5 6 7 8 - * - * 888888 888888 88 88 8888888888 88 88 8888888888 - * 88 88 88 88 88 88 88 88 88 88 88 88 - * 8888 8888 8888 8888 88 8888888888 8888888888 88 - * 88 88 88 88 - * 88 88 888888 888888 - */ - int value = 1; - switch (orientation) { +bool ExifUtilsImpl::setOrientation(uint16_t degrees) { + ExifOrientation value = ExifOrientation::ORIENTATION_0_DEGREES; + switch (degrees) { case 90: - value = 6; + value = ExifOrientation::ORIENTATION_90_DEGREES; break; case 180: - value = 3; + value = ExifOrientation::ORIENTATION_180_DEGREES; break; case 270: - value = 8; + value = ExifOrientation::ORIENTATION_270_DEGREES; break; default: break; } - SET_SHORT(EXIF_IFD_0, EXIF_TAG_ORIENTATION, value); + return setOrientationValue(value); +} + +bool ExifUtilsImpl::setOrientationValue(ExifOrientation orientationValue) { + SET_SHORT(EXIF_IFD_0, EXIF_TAG_ORIENTATION, orientationValue); return true; } diff --git a/services/camera/libcameraservice/utils/ExifUtils.h b/services/camera/libcameraservice/utils/ExifUtils.h index c78bab9961..f1d0205759 100644 --- a/services/camera/libcameraservice/utils/ExifUtils.h +++ b/services/camera/libcameraservice/utils/ExifUtils.h @@ -22,6 +22,24 @@ namespace android { namespace camera3 { +/* + * Orientation value: + * 1 2 3 4 5 6 7 8 + * + * 888888 888888 88 88 8888888888 88 88 8888888888 + * 88 88 88 88 88 88 88 88 88 88 88 88 + * 8888 8888 8888 8888 88 8888888888 8888888888 88 + * 88 88 88 88 + * 88 88 888888 888888 + */ +enum ExifOrientation : uint16_t { + ORIENTATION_UNDEFINED = 0x0, + ORIENTATION_0_DEGREES = 0x1, + ORIENTATION_90_DEGREES = 0x6, + ORIENTATION_180_DEGREES = 0x3, + ORIENTATION_270_DEGREES = 0x8, +}; + // This is based on the camera HIDL shim implementation, which was in turned // based on original ChromeOS ARC implementation of a V4L2 HAL @@ -49,6 +67,7 @@ public: // Initialize() can be called multiple times. The setting of Exif tags will be // cleared. virtual bool initialize(const unsigned char *app1Segment, size_t app1SegmentSize) = 0; + virtual bool initializeEmpty() = 0; // Set all known fields from a metadata structure virtual bool setFromMetadata(const CameraMetadata& metadata, @@ -142,7 +161,11 @@ public: // Sets image orientation. // Returns false if memory allocation fails. - virtual bool setOrientation(uint16_t orientation) = 0; + virtual bool setOrientation(uint16_t degrees) = 0; + + // Sets image orientation. + // Returns false if memory allocation fails. + virtual bool setOrientationValue(ExifOrientation orientationValue) = 0; // Sets the shutter speed. // Returns false if memory allocation fails.