diff --git a/services/camera/libcameraservice/Android.bp b/services/camera/libcameraservice/Android.bp index 851dd690d9..3b362d6707 100644 --- a/services/camera/libcameraservice/Android.bp +++ b/services/camera/libcameraservice/Android.bp @@ -39,6 +39,8 @@ cc_library_shared { "api1/client2/CaptureSequencer.cpp", "api1/client2/ZslProcessor.cpp", "api2/CameraDeviceClient.cpp", + "api2/CompositeStream.cpp", + "api2/DepthCompositeStream.cpp", "device1/CameraHardwareInterface.cpp", "device3/Camera3Device.cpp", "device3/Camera3Stream.cpp", @@ -65,6 +67,9 @@ cc_library_shared { ], shared_libs: [ + "libimage_io", + "libdynamic_depth", + "libxml2", "libui", "liblog", "libutilscallstack", @@ -108,6 +113,8 @@ cc_library_shared { "system/media/private/camera/include", "frameworks/native/include/media/openmax", "frameworks/av/media/ndk", + "external/dynamic_depth/includes", + "external/dynamic_depth/internal", ], export_include_dirs: ["."], @@ -116,6 +123,7 @@ cc_library_shared { "-Wall", "-Wextra", "-Werror", + "-Wno-ignored-qualifiers", ], } diff --git a/services/camera/libcameraservice/api1/client2/JpegProcessor.cpp b/services/camera/libcameraservice/api1/client2/JpegProcessor.cpp index b7020fe312..e6f75f4d96 100755 --- a/services/camera/libcameraservice/api1/client2/JpegProcessor.cpp +++ b/services/camera/libcameraservice/api1/client2/JpegProcessor.cpp @@ -62,6 +62,10 @@ void JpegProcessor::onFrameAvailable(const BufferItem& /*item*/) { } } +void JpegProcessor::onBufferRequestForFrameNumber(uint64_t /*frameNumber*/, int /*streamId*/) { + // Intentionally left empty +} + void JpegProcessor::onBufferAcquired(const BufferInfo& /*bufferInfo*/) { // Intentionally left empty } diff --git a/services/camera/libcameraservice/api1/client2/JpegProcessor.h b/services/camera/libcameraservice/api1/client2/JpegProcessor.h index 7187ad94a4..2ee930e02b 100644 --- a/services/camera/libcameraservice/api1/client2/JpegProcessor.h +++ b/services/camera/libcameraservice/api1/client2/JpegProcessor.h @@ -25,6 +25,7 @@ #include #include "camera/CameraMetadata.h" +#include "device3/Camera3StreamBufferListener.h" namespace android { @@ -53,12 +54,16 @@ class JpegProcessor: // Camera3StreamBufferListener implementation void onBufferAcquired(const BufferInfo& bufferInfo) override; void onBufferReleased(const BufferInfo& bufferInfo) override; + void onBufferRequestForFrameNumber(uint64_t frameNumber, int streamId) override; status_t updateStream(const Parameters ¶ms); status_t deleteStream(); int getStreamId() const; void dump(int fd, const Vector& args) const; + + static size_t findJpegSize(uint8_t* jpegBuffer, size_t maxSize); + private: static const nsecs_t kWaitDuration = 10000000; // 10 ms wp mDevice; @@ -82,7 +87,6 @@ class JpegProcessor: virtual bool threadLoop(); status_t processNewCapture(bool captureSuccess); - size_t findJpegSize(uint8_t* jpegBuffer, size_t maxSize); }; diff --git a/services/camera/libcameraservice/api2/CameraDeviceClient.cpp b/services/camera/libcameraservice/api2/CameraDeviceClient.cpp index 46fbc3e3c7..9e203da4d4 100644 --- a/services/camera/libcameraservice/api2/CameraDeviceClient.cpp +++ b/services/camera/libcameraservice/api2/CameraDeviceClient.cpp @@ -33,6 +33,8 @@ #include +#include "DepthCompositeStream.h" + // Convenience methods for constructing binder::Status objects for error returns #define STATUS_ERROR(errorCode, errorString) \ @@ -143,6 +145,7 @@ binder::Status CameraDeviceClient::submitRequest( binder::Status CameraDeviceClient::insertGbpLocked(const sp& gbp, SurfaceMap* outSurfaceMap, Vector* outputStreamIds, int32_t *currentStreamId) { + int compositeIdx; int idx = mStreamMap.indexOfKey(IInterface::asBinder(gbp)); // Trying to submit request with surface that wasn't created @@ -152,6 +155,11 @@ binder::Status CameraDeviceClient::insertGbpLocked(const spinsertGbp(outSurfaceMap, outputStreamIds, + currentStreamId); + return binder::Status::ok(); } const StreamSurfaceId& streamSurfaceId = mStreamMap.valueAt(idx); @@ -489,6 +497,17 @@ binder::Status CameraDeviceClient::endConfigure(int operatingMode, mCameraIdStr.string(), strerror(-err), err); ALOGE("%s: %s", __FUNCTION__, msg.string()); res = STATUS_ERROR(CameraService::ERROR_INVALID_OPERATION, msg.string()); + } else { + for (size_t i = 0; i < mCompositeStreamMap.size(); ++i) { + err = mCompositeStreamMap.valueAt(i)->configureStream(); + if (err != OK ) { + String8 msg = String8::format("Camera %s: Error configuring composite " + "streams: %s (%d)", mCameraIdStr.string(), strerror(-err), err); + ALOGE("%s: %s", __FUNCTION__, msg.string()); + res = STATUS_ERROR(CameraService::ERROR_INVALID_OPERATION, msg.string()); + break; + } + } } return res; @@ -692,8 +711,35 @@ binder::Status CameraDeviceClient::isSessionConfigurationSupported( return res; if (!isStreamInfoValid) { - mapStreamInfo(streamInfo, static_cast (it.getRotation()), - physicalCameraId, &streamConfiguration.streams[streamIdx++]); + if (camera3::DepthCompositeStream::isDepthCompositeStream(surface)) { + // We need to take in to account that composite streams can have + // additional internal camera streams. + std::vector compositeStreams; + ret = camera3::DepthCompositeStream::getCompositeStreamInfo(streamInfo, + mDevice->info(), &compositeStreams); + if (ret != OK) { + String8 msg = String8::format( + "Camera %s: Failed adding depth composite streams: %s (%d)", + mCameraIdStr.string(), strerror(-ret), ret); + ALOGE("%s: %s", __FUNCTION__, msg.string()); + return STATUS_ERROR(CameraService::ERROR_ILLEGAL_ARGUMENT, msg.string()); + } + + if (compositeStreams.size() > 1) { + streamCount += compositeStreams.size() - 1; + streamConfiguration.streams.resize(streamCount); + } + + for (const auto& compositeStream : compositeStreams) { + mapStreamInfo(compositeStream, + static_cast (it.getRotation()), + physicalCameraId, &streamConfiguration.streams[streamIdx++]); + } + } else { + mapStreamInfo(streamInfo, + static_cast (it.getRotation()), + physicalCameraId, &streamConfiguration.streams[streamIdx++]); + } isStreamInfoValid = true; } } @@ -743,6 +789,7 @@ binder::Status CameraDeviceClient::deleteStream(int streamId) { bool isInput = false; std::vector> surfaces; ssize_t dIndex = NAME_NOT_FOUND; + ssize_t compositeIndex = NAME_NOT_FOUND; if (mInputStream.configured && mInputStream.id == streamId) { isInput = true; @@ -762,6 +809,13 @@ binder::Status CameraDeviceClient::deleteStream(int streamId) { } } + for (size_t i = 0; i < mCompositeStreamMap.size(); ++i) { + if (streamId == mCompositeStreamMap.valueAt(i)->getStreamId()) { + compositeIndex = i; + break; + } + } + if (surfaces.empty() && dIndex == NAME_NOT_FOUND) { String8 msg = String8::format("Camera %s: Invalid stream ID (%d) specified, no such" " stream created yet", mCameraIdStr.string(), streamId); @@ -791,6 +845,19 @@ binder::Status CameraDeviceClient::deleteStream(int streamId) { if (dIndex != NAME_NOT_FOUND) { mDeferredStreams.removeItemsAt(dIndex); } + + if (compositeIndex != NAME_NOT_FOUND) { + status_t ret; + if ((ret = mCompositeStreamMap.valueAt(compositeIndex)->deleteStream()) + != OK) { + String8 msg = String8::format("Camera %s: Unexpected error %s (%d) when " + "deleting composite stream %d", mCameraIdStr.string(), strerror(-err), err, + streamId); + ALOGE("%s: %s", __FUNCTION__, msg.string()); + res = STATUS_ERROR(CameraService::ERROR_INVALID_OPERATION, msg.string()); + } + mCompositeStreamMap.removeItemsAt(compositeIndex); + } } } @@ -870,11 +937,25 @@ binder::Status CameraDeviceClient::createStream( int streamId = camera3::CAMERA3_STREAM_ID_INVALID; std::vector surfaceIds; - err = mDevice->createStream(surfaces, deferredConsumer, streamInfo.width, - streamInfo.height, streamInfo.format, streamInfo.dataSpace, - static_cast(outputConfiguration.getRotation()), - &streamId, physicalCameraId, &surfaceIds, outputConfiguration.getSurfaceSetID(), - isShared); + if (!camera3::DepthCompositeStream::isDepthCompositeStream(surfaces[0])) { + err = mDevice->createStream(surfaces, deferredConsumer, streamInfo.width, + streamInfo.height, streamInfo.format, streamInfo.dataSpace, + static_cast(outputConfiguration.getRotation()), + &streamId, physicalCameraId, &surfaceIds, outputConfiguration.getSurfaceSetID(), + isShared); + } else { + sp compositeStream = new camera3::DepthCompositeStream(mDevice, + getRemoteCallback()); + err = compositeStream->createStream(surfaces, deferredConsumer, streamInfo.width, + streamInfo.height, streamInfo.format, + static_cast(outputConfiguration.getRotation()), + &streamId, physicalCameraId, &surfaceIds, outputConfiguration.getSurfaceSetID(), + isShared); + if (err == OK) { + mCompositeStreamMap.add(IInterface::asBinder(surfaces[0]->getIGraphicBufferProducer()), + compositeStream); + } + } if (err != OK) { res = STATUS_ERROR_FMT(CameraService::ERROR_INVALID_OPERATION, @@ -1808,7 +1889,14 @@ void CameraDeviceClient::notifyError(int32_t errorCode, // Thread safe. Don't bother locking. sp remoteCb = getRemoteCallback(); - if (remoteCb != 0) { + // Composites can have multiple internal streams. Error notifications coming from such internal + // streams may need to remain within camera service. + bool skipClientNotification = false; + for (size_t i = 0; i < mCompositeStreamMap.size(); i++) { + skipClientNotification |= mCompositeStreamMap.valueAt(i)->onError(errorCode, resultExtras); + } + + if ((remoteCb != 0) && (!skipClientNotification)) { remoteCb->onDeviceError(errorCode, resultExtras); } } @@ -1901,6 +1989,10 @@ void CameraDeviceClient::onResultAvailable(const CaptureResult& result) { remoteCb->onResultReceived(result.mMetadata, result.mResultExtras, result.mPhysicalMetadatas); } + + for (size_t i = 0; i < mCompositeStreamMap.size(); i++) { + mCompositeStreamMap.valueAt(i)->onResultAvailable(result); + } } binder::Status CameraDeviceClient::checkPidStatus(const char* checkLocation) { diff --git a/services/camera/libcameraservice/api2/CameraDeviceClient.h b/services/camera/libcameraservice/api2/CameraDeviceClient.h index 17a0983498..1c5abb034d 100644 --- a/services/camera/libcameraservice/api2/CameraDeviceClient.h +++ b/services/camera/libcameraservice/api2/CameraDeviceClient.h @@ -26,8 +26,10 @@ #include "CameraService.h" #include "common/FrameProcessorBase.h" #include "common/Camera2ClientBase.h" +#include "CompositeStream.h" using android::camera3::OutputStreamInfo; +using android::camera3::CompositeStream; namespace android { @@ -314,6 +316,8 @@ private: // stream ID -> outputStreamInfo mapping std::unordered_map mStreamInfoMap; + KeyedVector, sp> mCompositeStreamMap; + static const int32_t MAX_SURFACES_PER_STREAM = 4; sp mProviderManager; }; diff --git a/services/camera/libcameraservice/api2/CompositeStream.cpp b/services/camera/libcameraservice/api2/CompositeStream.cpp new file mode 100644 index 0000000000..796bf42ae0 --- /dev/null +++ b/services/camera/libcameraservice/api2/CompositeStream.cpp @@ -0,0 +1,203 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "Camera3-CompositeStream" +#define ATRACE_TAG ATRACE_TAG_CAMERA +//#define LOG_NDEBUG 0 + +#include +#include + +#include "common/CameraDeviceBase.h" +#include "CameraDeviceClient.h" +#include "CompositeStream.h" + +namespace android { +namespace camera3 { + +CompositeStream::CompositeStream(wp device, + wp cb) : + mDevice(device), + mRemoteCallback(cb), + mNumPartialResults(1), + mErrorState(false) { + sp cameraDevice = device.promote(); + if (cameraDevice.get() != nullptr) { + CameraMetadata staticInfo = cameraDevice->info(); + camera_metadata_entry_t entry = staticInfo.find(ANDROID_REQUEST_PARTIAL_RESULT_COUNT); + if (entry.count > 0) { + mNumPartialResults = entry.data.i32[0]; + } + } +} + +status_t CompositeStream::createStream(const std::vector>& consumers, + bool hasDeferredConsumer, uint32_t width, uint32_t height, int format, + camera3_stream_rotation_t rotation, int * id, const String8& physicalCameraId, + std::vector * surfaceIds, int streamSetId, bool isShared) { + if (hasDeferredConsumer) { + ALOGE("%s: Deferred consumers not supported in case of composite streams!", + __FUNCTION__); + return BAD_VALUE; + } + + if (streamSetId != camera3::CAMERA3_STREAM_ID_INVALID) { + ALOGE("%s: Surface groups not supported in case of composite streams!", + __FUNCTION__); + return BAD_VALUE; + } + + if (isShared) { + ALOGE("%s: Shared surfaces not supported in case of composite streams!", + __FUNCTION__); + return BAD_VALUE; + } + + return createInternalStreams(consumers, hasDeferredConsumer, width, height, format, rotation, id, + physicalCameraId, surfaceIds, streamSetId, isShared); +} + +status_t CompositeStream::deleteStream() { + { + Mutex::Autolock l(mMutex); + mPendingCaptureResults.clear(); + mCaptureResults.clear(); + mFrameNumberMap.clear(); + mErrorFrameNumbers.clear(); + } + + return deleteInternalStreams(); +} + +void CompositeStream::onBufferRequestForFrameNumber(uint64_t frameNumber, int streamId) { + Mutex::Autolock l(mMutex); + if (!mErrorState && (streamId == getStreamId())) { + mPendingCaptureResults.emplace(frameNumber, CameraMetadata()); + } +} + +void CompositeStream::onBufferReleased(const BufferInfo& bufferInfo) { + Mutex::Autolock l(mMutex); + if (!mErrorState && !bufferInfo.mError) { + mFrameNumberMap.emplace(bufferInfo.mFrameNumber, bufferInfo.mTimestamp); + mInputReadyCondition.signal(); + } +} + +void CompositeStream::eraseResult(int64_t frameNumber) { + Mutex::Autolock l(mMutex); + + auto it = mPendingCaptureResults.find(frameNumber); + if (it == mPendingCaptureResults.end()) { + return; + } + + it = mPendingCaptureResults.erase(it); +} + +void CompositeStream::onResultAvailable(const CaptureResult& result) { + bool resultError = false; + { + Mutex::Autolock l(mMutex); + + uint64_t frameNumber = result.mResultExtras.frameNumber; + bool resultReady = false; + auto it = mPendingCaptureResults.find(frameNumber); + if (it != mPendingCaptureResults.end()) { + it->second.append(result.mMetadata); + if (result.mResultExtras.partialResultCount >= mNumPartialResults) { + auto entry = it->second.find(ANDROID_SENSOR_TIMESTAMP); + if (entry.count == 1) { + auto ts = entry.data.i64[0]; + mCaptureResults.emplace(ts, std::make_tuple(frameNumber, it->second)); + resultReady = true; + } else { + ALOGE("%s: Timestamp metadata entry missing for frameNumber: %" PRIu64, + __FUNCTION__, frameNumber); + resultError = true; + } + mPendingCaptureResults.erase(it); + } + } + + if (resultReady) { + mInputReadyCondition.signal(); + } + } + + if (resultError) { + onResultError(result.mResultExtras); + } +} + +void CompositeStream::flagAnErrorFrameNumber(int64_t frameNumber) { + Mutex::Autolock l(mMutex); + mErrorFrameNumbers.emplace(frameNumber); + mInputReadyCondition.signal(); +} + +status_t CompositeStream::registerCompositeStreamListener(int32_t streamId) { + sp device = mDevice.promote(); + if (device.get() == nullptr) { + return NO_INIT; + } + + auto ret = device->addBufferListenerForStream(streamId, this); + if (ret != OK) { + ALOGE("%s: Failed to register composite stream listener!", __FUNCTION__); + } + + return ret; +} + +bool CompositeStream::onError(int32_t errorCode, const CaptureResultExtras& resultExtras) { + auto ret = false; + switch (errorCode) { + case hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_RESULT: + onResultError(resultExtras); + break; + case hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_BUFFER: + ret = onStreamBufferError(resultExtras); + break; + case hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_REQUEST: + // Invalid request, this shouldn't affect composite streams. + break; + default: + ALOGE("%s: Unrecoverable error: %d detected!", __FUNCTION__, errorCode); + Mutex::Autolock l(mMutex); + mErrorState = true; + break; + } + + return ret; +} + +void CompositeStream::notifyError(int64_t frameNumber) { + sp remoteCb = + mRemoteCallback.promote(); + + if ((frameNumber >= 0) && (remoteCb.get() != nullptr)) { + CaptureResultExtras extras; + extras.errorStreamId = getStreamId(); + extras.frameNumber = frameNumber; + remoteCb->onDeviceError( + hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_BUFFER, + extras); + } +} + +}; // namespace camera3 +}; // namespace android diff --git a/services/camera/libcameraservice/api2/CompositeStream.h b/services/camera/libcameraservice/api2/CompositeStream.h new file mode 100644 index 0000000000..583774550b --- /dev/null +++ b/services/camera/libcameraservice/api2/CompositeStream.h @@ -0,0 +1,120 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ANDROID_SERVERS_CAMERA_CAMERA3_COMPOSITE_STREAM_H +#define ANDROID_SERVERS_CAMERA_CAMERA3_COMPOSITE_STREAM_H + +#include +#include + +#include +#include +#include +#include "common/CameraDeviceBase.h" +#include "device3/Camera3StreamInterface.h" + +namespace android { + +class CameraDeviceClient; +class CameraMetadata; +class Surface; + +namespace camera3 { + +class CompositeStream : public camera3::Camera3StreamBufferListener { + +public: + CompositeStream(wp device, wp cb); + virtual ~CompositeStream() {} + + status_t createStream(const std::vector>& consumers, + bool hasDeferredConsumer, uint32_t width, uint32_t height, int format, + camera3_stream_rotation_t rotation, int *id, const String8& physicalCameraId, + std::vector *surfaceIds, int streamSetId, bool isShared); + + status_t deleteStream(); + + // Create and register all internal camera streams. + virtual status_t createInternalStreams(const std::vector>& consumers, + bool hasDeferredConsumer, uint32_t width, uint32_t height, int format, + camera3_stream_rotation_t rotation, int *id, const String8& physicalCameraId, + std::vector *surfaceIds, int streamSetId, bool isShared) = 0; + + // Release all internal streams and corresponding resources. + virtual status_t deleteInternalStreams() = 0; + + // Stream configuration completed. + virtual status_t configureStream() = 0; + + // Insert the internal composite stream id in the user capture request. + virtual status_t insertGbp(SurfaceMap* /*out*/outSurfaceMap, + Vector* /*out*/outputStreamIds, int32_t* /*out*/currentStreamId) = 0; + + // Return composite stream id. + virtual int getStreamId() = 0; + + void onResultAvailable(const CaptureResult& result); + bool onError(int32_t errorCode, const CaptureResultExtras& resultExtras); + + // Camera3StreamBufferListener implementation + void onBufferAcquired(const BufferInfo& /*bufferInfo*/) override { /*Empty for now */ } + void onBufferReleased(const BufferInfo& bufferInfo) override; + void onBufferRequestForFrameNumber(uint64_t frameNumber, int streamId) override; + +protected: + status_t registerCompositeStreamListener(int32_t streamId); + void eraseResult(int64_t frameNumber); + void flagAnErrorFrameNumber(int64_t frameNumber); + void notifyError(int64_t frameNumber); + + // Subclasses should check for buffer errors from internal streams and return 'true' in + // case the error notification should remain within camera service. + virtual bool onStreamBufferError(const CaptureResultExtras& resultExtras) = 0; + + // Subclasses can decide how to handle result errors depending on whether or not the + // internal processing needs result data. + virtual void onResultError(const CaptureResultExtras& resultExtras) = 0; + + // Device and/or service is in unrecoverable error state. + // Composite streams should behave accordingly. + void enableErrorState(); + + wp mDevice; + wp mRemoteCallback; + + mutable Mutex mMutex; + Condition mInputReadyCondition; + int32_t mNumPartialResults; + bool mErrorState; + + // Frame number to capture result map of partial pending request results. + std::unordered_map mPendingCaptureResults; + + // Timestamp to capture (frame number, result) map of completed pending request results. + std::unordered_map> mCaptureResults; + + // Frame number to timestamp map + std::unordered_map mFrameNumberMap; + + // Keeps a set buffer/result frame numbers for any errors detected during processing. + std::set mErrorFrameNumbers; + +}; + +}; //namespace camera3 +}; //namespace android + +#endif diff --git a/services/camera/libcameraservice/api2/DepthCompositeStream.cpp b/services/camera/libcameraservice/api2/DepthCompositeStream.cpp new file mode 100644 index 0000000000..b12bb50f09 --- /dev/null +++ b/services/camera/libcameraservice/api2/DepthCompositeStream.cpp @@ -0,0 +1,1026 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "Camera3-DepthCompositeStream" +#define ATRACE_TAG ATRACE_TAG_CAMERA +//#define LOG_NDEBUG 0 + +#include "api1/client2/JpegProcessor.h" +#include "common/CameraProviderManager.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include + +#include "DepthCompositeStream.h" + +using dynamic_depth::Camera; +using dynamic_depth::Cameras; +using dynamic_depth::CameraParams; +using dynamic_depth::Container; +using dynamic_depth::DepthFormat; +using dynamic_depth::DepthMapParams; +using dynamic_depth::DepthUnits; +using dynamic_depth::Device; +using dynamic_depth::DeviceParams; +using dynamic_depth::Dimension; +using dynamic_depth::Image; +using dynamic_depth::ImagingModelParams; +using dynamic_depth::Pose; +using dynamic_depth::Profile; +using dynamic_depth::Profiles; + +namespace android { +namespace camera3 { + +DepthCompositeStream::DepthCompositeStream(wp device, + wp cb) : + CompositeStream(device, cb), + mBlobStreamId(-1), + mBlobSurfaceId(-1), + mDepthStreamId(-1), + mDepthSurfaceId(-1), + mBlobWidth(0), + mBlobHeight(0), + mDepthBufferAcquired(false), + mBlobBufferAcquired(false), + mProducerListener(new ProducerListener()), + mMaxJpegSize(-1), + mIsLogicalCamera(false) { + sp cameraDevice = device.promote(); + if (cameraDevice.get() != nullptr) { + CameraMetadata staticInfo = cameraDevice->info(); + auto entry = staticInfo.find(ANDROID_JPEG_MAX_SIZE); + if (entry.count > 0) { + mMaxJpegSize = entry.data.i32[0]; + } else { + ALOGW("%s: Maximum jpeg size absent from camera characteristics", __FUNCTION__); + } + + entry = staticInfo.find(ANDROID_LENS_INTRINSIC_CALIBRATION); + if (entry.count == 5) { + mInstrinsicCalibration.reserve(5); + mInstrinsicCalibration.insert(mInstrinsicCalibration.end(), entry.data.f, + entry.data.f + 5); + } else { + ALOGW("%s: Intrinsic calibration absent from camera characteristics!", __FUNCTION__); + } + + entry = staticInfo.find(ANDROID_LENS_DISTORTION); + if (entry.count == 5) { + mLensDistortion.reserve(5); + mLensDistortion.insert(mLensDistortion.end(), entry.data.f, entry.data.f + 5); + } else { + ALOGW("%s: Lens distortion absent from camera characteristics!", __FUNCTION__); + } + + entry = staticInfo.find(ANDROID_REQUEST_AVAILABLE_CAPABILITIES); + for (size_t i = 0; i < entry.count; ++i) { + uint8_t capability = entry.data.u8[i]; + if (capability == ANDROID_REQUEST_AVAILABLE_CAPABILITIES_LOGICAL_MULTI_CAMERA) { + mIsLogicalCamera = true; + break; + } + } + + getSupportedDepthSizes(staticInfo, &mSupportedDepthSizes); + } +} + +DepthCompositeStream::~DepthCompositeStream() { + mBlobConsumer.clear(), + mBlobSurface.clear(), + mBlobStreamId = -1; + mBlobSurfaceId = -1; + mDepthConsumer.clear(); + mDepthSurface.clear(); + mDepthConsumer = nullptr; + mDepthSurface = nullptr; +} + +void DepthCompositeStream::compilePendingInputLocked() { + CpuConsumer::LockedBuffer imgBuffer; + + while (!mInputJpegBuffers.empty() && !mBlobBufferAcquired) { + auto it = mInputJpegBuffers.begin(); + auto res = mBlobConsumer->lockNextBuffer(&imgBuffer); + if (res == NOT_ENOUGH_DATA) { + // Can not lock any more buffers. + break; + } else if (res != OK) { + ALOGE("%s: Error locking blob image buffer: %s (%d)", __FUNCTION__, + strerror(-res), res); + mPendingInputFrames[*it].error = true; + mInputDepthBuffers.erase(it); + continue; + } + + if (*it != imgBuffer.timestamp) { + ALOGW("%s: Expecting jpeg buffer with time stamp: %" PRId64 " received buffer with " + "time stamp: %" PRId64, __FUNCTION__, *it, imgBuffer.timestamp); + } + + if ((mPendingInputFrames.find(imgBuffer.timestamp) != mPendingInputFrames.end()) && + (mPendingInputFrames[imgBuffer.timestamp].error)) { + mBlobConsumer->unlockBuffer(imgBuffer); + } else { + mPendingInputFrames[imgBuffer.timestamp].jpegBuffer = imgBuffer; + mBlobBufferAcquired = true; + } + mInputJpegBuffers.erase(it); + } + + while (!mInputDepthBuffers.empty() && !mDepthBufferAcquired) { + auto it = mInputDepthBuffers.begin(); + auto res = mDepthConsumer->lockNextBuffer(&imgBuffer); + if (res == NOT_ENOUGH_DATA) { + // Can not lock any more buffers. + break; + } else if (res != OK) { + ALOGE("%s: Error receiving depth image buffer: %s (%d)", __FUNCTION__, + strerror(-res), res); + mPendingInputFrames[*it].error = true; + mInputDepthBuffers.erase(it); + continue; + } + + if (*it != imgBuffer.timestamp) { + ALOGW("%s: Expecting depth buffer with time stamp: %" PRId64 " received buffer with " + "time stamp: %" PRId64, __FUNCTION__, *it, imgBuffer.timestamp); + } + + if ((mPendingInputFrames.find(imgBuffer.timestamp) != mPendingInputFrames.end()) && + (mPendingInputFrames[imgBuffer.timestamp].error)) { + mDepthConsumer->unlockBuffer(imgBuffer); + } else { + mPendingInputFrames[imgBuffer.timestamp].depthBuffer = imgBuffer; + mDepthBufferAcquired = true; + } + mInputDepthBuffers.erase(it); + } + + while (!mCaptureResults.empty()) { + auto it = mCaptureResults.begin(); + // Negative timestamp indicates that something went wrong during the capture result + // collection process. + if (it->first >= 0) { + mPendingInputFrames[it->first].frameNumber = std::get<0>(it->second); + mPendingInputFrames[it->first].result = std::get<1>(it->second); + } + mCaptureResults.erase(it); + } + + while (!mFrameNumberMap.empty()) { + auto it = mFrameNumberMap.begin(); + mPendingInputFrames[it->second].frameNumber = it->first; + mFrameNumberMap.erase(it); + } + + auto it = mErrorFrameNumbers.begin(); + while (it != mErrorFrameNumbers.end()) { + bool frameFound = false; + for (auto &inputFrame : mPendingInputFrames) { + if (inputFrame.second.frameNumber == *it) { + inputFrame.second.error = true; + frameFound = true; + break; + } + } + + if (frameFound) { + it = mErrorFrameNumbers.erase(it); + } else { + ALOGW("%s: Not able to find failing input with frame number: %" PRId64, __FUNCTION__, + *it); + it++; + } + } +} + +bool DepthCompositeStream::getNextReadyInputLocked(int64_t *currentTs /*inout*/) { + if (currentTs == nullptr) { + return false; + } + + bool newInputAvailable = false; + for (const auto& it : mPendingInputFrames) { + if ((!it.second.error) && (it.second.depthBuffer.data != nullptr) && + (it.second.jpegBuffer.data != nullptr) && (it.first < *currentTs)) { + *currentTs = it.first; + newInputAvailable = true; + } + } + + return newInputAvailable; +} + +int64_t DepthCompositeStream::getNextFailingInputLocked(int64_t *currentTs /*inout*/) { + int64_t ret = -1; + if (currentTs == nullptr) { + return ret; + } + + for (const auto& it : mPendingInputFrames) { + if (it.second.error && !it.second.errorNotified && (it.first < *currentTs)) { + *currentTs = it.first; + ret = it.second.frameNumber; + } + } + + return ret; +} + +status_t DepthCompositeStream::encodeGrayscaleJpeg(size_t width, size_t height, uint8_t *in, + void *out, const size_t maxOutSize, uint8_t jpegQuality, size_t &actualSize) { + status_t ret; + // libjpeg is a C library so we use C-style "inheritance" by + // putting libjpeg's jpeg_destination_mgr first in our custom + // struct. This allows us to cast jpeg_destination_mgr* to + // CustomJpegDestMgr* when we get it passed to us in a callback. + struct CustomJpegDestMgr : public jpeg_destination_mgr { + JOCTET *mBuffer; + size_t mBufferSize; + size_t mEncodedSize; + bool mSuccess; + } dmgr; + + jpeg_compress_struct cinfo = {}; + jpeg_error_mgr jerr; + + // Initialize error handling with standard callbacks, but + // then override output_message (to print to ALOG) and + // error_exit to set a flag and print a message instead + // of killing the whole process. + cinfo.err = jpeg_std_error(&jerr); + + cinfo.err->output_message = [](j_common_ptr cinfo) { + char buffer[JMSG_LENGTH_MAX]; + + /* Create the message */ + (*cinfo->err->format_message)(cinfo, buffer); + ALOGE("libjpeg error: %s", buffer); + }; + + cinfo.err->error_exit = [](j_common_ptr cinfo) { + (*cinfo->err->output_message)(cinfo); + if(cinfo->client_data) { + auto & dmgr = *static_cast(cinfo->client_data); + dmgr.mSuccess = false; + } + }; + + // Now that we initialized some callbacks, let's create our compressor + jpeg_create_compress(&cinfo); + dmgr.mBuffer = static_cast(out); + dmgr.mBufferSize = maxOutSize; + dmgr.mEncodedSize = 0; + dmgr.mSuccess = true; + cinfo.client_data = static_cast(&dmgr); + + // These lambdas become C-style function pointers and as per C++11 spec + // may not capture anything. + dmgr.init_destination = [](j_compress_ptr cinfo) { + auto & dmgr = static_cast(*cinfo->dest); + dmgr.next_output_byte = dmgr.mBuffer; + dmgr.free_in_buffer = dmgr.mBufferSize; + ALOGV("%s:%d jpeg start: %p [%zu]", + __FUNCTION__, __LINE__, dmgr.mBuffer, dmgr.mBufferSize); + }; + + dmgr.empty_output_buffer = [](j_compress_ptr cinfo __unused) { + ALOGV("%s:%d Out of buffer", __FUNCTION__, __LINE__); + return 0; + }; + + dmgr.term_destination = [](j_compress_ptr cinfo) { + auto & dmgr = static_cast(*cinfo->dest); + dmgr.mEncodedSize = dmgr.mBufferSize - dmgr.free_in_buffer; + ALOGV("%s:%d Done with jpeg: %zu", __FUNCTION__, __LINE__, dmgr.mEncodedSize); + }; + cinfo.dest = reinterpret_cast(&dmgr); + cinfo.image_width = width; + cinfo.image_height = height; + cinfo.input_components = 1; + cinfo.in_color_space = JCS_GRAYSCALE; + + // Initialize defaults and then override what we want + jpeg_set_defaults(&cinfo); + + jpeg_set_quality(&cinfo, jpegQuality, 1); + jpeg_set_colorspace(&cinfo, JCS_GRAYSCALE); + cinfo.raw_data_in = 0; + cinfo.dct_method = JDCT_IFAST; + + cinfo.comp_info[0].h_samp_factor = 1; + cinfo.comp_info[1].h_samp_factor = 1; + cinfo.comp_info[2].h_samp_factor = 1; + cinfo.comp_info[0].v_samp_factor = 1; + cinfo.comp_info[1].v_samp_factor = 1; + cinfo.comp_info[2].v_samp_factor = 1; + + jpeg_start_compress(&cinfo, TRUE); + + for (size_t i = 0; i < cinfo.image_height; i++) { + auto currentRow = static_cast(in + i*width); + jpeg_write_scanlines(&cinfo, ¤tRow, /*num_lines*/1); + } + + jpeg_finish_compress(&cinfo); + + actualSize = dmgr.mEncodedSize; + if (dmgr.mSuccess) { + ret = NO_ERROR; + } else { + ret = UNKNOWN_ERROR; + } + + return ret; +} + +std::unique_ptr DepthCompositeStream::processDepthMapFrame( + const CpuConsumer::LockedBuffer &depthMapBuffer, size_t maxJpegSize, uint8_t jpegQuality, + std::vector> *items /*out*/) { + std::vector points, confidence; + + size_t pointCount = depthMapBuffer.width * depthMapBuffer.height; + points.reserve(pointCount); + confidence.reserve(pointCount); + float near = UINT16_MAX; + float far = .0f; + uint16_t *data = reinterpret_cast (depthMapBuffer.data); + for (size_t i = 0; i < depthMapBuffer.height; i++) { + for (size_t j = 0; j < depthMapBuffer.width; j++) { + // Android densely packed depth map. The units for the range are in + // millimeters and need to be scaled to meters. + // The confidence value is encoded in the 3 most significant bits. + // The confidence data needs to be additionally normalized with + // values 1.0f, 0.0f representing maximum and minimum confidence + // respectively. + auto value = data[i*depthMapBuffer.stride + j]; + auto point = static_cast(value & 0x1FFF) / 1000.f; + points.push_back(point); + + auto conf = (value >> 13) & 0x7; + float normConfidence = (conf == 0) ? 1.f : (static_cast(conf) - 1) / 7.f; + confidence.push_back(normConfidence); + + if (near > point) { + near = point; + } + if (far < point) { + far = point; + } + } + } + + if (near == far) { + ALOGE("%s: Near and far range values must not match!", __FUNCTION__); + return nullptr; + } + + std::vector pointsQuantized, confidenceQuantized; + pointsQuantized.reserve(pointCount); confidenceQuantized.reserve(pointCount); + auto pointIt = points.begin(); + auto confidenceIt = confidence.begin(); + while ((pointIt != points.end()) && (confidenceIt != confidence.end())) { + pointsQuantized.push_back(floorf(((far * (*pointIt - near)) / + (*pointIt * (far - near))) * 255.0f)); + confidenceQuantized.push_back(floorf(*confidenceIt * 255.0f)); + confidenceIt++; pointIt++; + } + + DepthMapParams depthParams(DepthFormat::kRangeInverse, near, far, DepthUnits::kMeters, + "android/depthmap"); + depthParams.confidence_uri = "android/confidencemap"; + depthParams.mime = "image/jpeg"; + depthParams.depth_image_data.resize(maxJpegSize); + depthParams.confidence_data.resize(maxJpegSize); + size_t actualJpegSize; + auto ret = encodeGrayscaleJpeg(depthMapBuffer.width, depthMapBuffer.height, + pointsQuantized.data(), depthParams.depth_image_data.data(), maxJpegSize, jpegQuality, + actualJpegSize); + if (ret != NO_ERROR) { + ALOGE("%s: Depth map compression failed!", __FUNCTION__); + return nullptr; + } + depthParams.depth_image_data.resize(actualJpegSize); + + ret = encodeGrayscaleJpeg(depthMapBuffer.width, depthMapBuffer.height, + confidenceQuantized.data(), depthParams.confidence_data.data(), maxJpegSize, + jpegQuality, actualJpegSize); + if (ret != NO_ERROR) { + ALOGE("%s: Confidence map compression failed!", __FUNCTION__); + return nullptr; + } + depthParams.confidence_data.resize(actualJpegSize); + + return DepthMap::FromData(depthParams, items); +} + +status_t DepthCompositeStream::processInputFrame(const InputFrame &inputFrame) { + status_t res; + sp outputANW = mOutputSurface; + ANativeWindowBuffer *anb; + int fenceFd; + void *dstBuffer; + auto imgBuffer = inputFrame.jpegBuffer; + + auto jpegSize = android::camera2::JpegProcessor::findJpegSize(inputFrame.jpegBuffer.data, + inputFrame.jpegBuffer.width); + if (jpegSize == 0) { + ALOGW("%s: Failed to find input jpeg size, default to using entire buffer!", __FUNCTION__); + jpegSize = inputFrame.jpegBuffer.width; + } + + std::vector> items; + std::vector> cameraList; + auto image = Image::FromDataForPrimaryImage("android/mainimage", &items); + std::unique_ptr cameraParams(new CameraParams(std::move(image))); + if (cameraParams == nullptr) { + ALOGE("%s: Failed to initialize camera parameters", __FUNCTION__); + return BAD_VALUE; + } + + size_t maxDepthJpegSize; + if (mMaxJpegSize > 0) { + maxDepthJpegSize = mMaxJpegSize; + } else { + maxDepthJpegSize = std::max (jpegSize, + inputFrame.depthBuffer.width * inputFrame.depthBuffer.height * 3 / 2); + } + uint8_t jpegQuality = 100; + auto entry = inputFrame.result.find(ANDROID_JPEG_QUALITY); + if (entry.count > 0) { + jpegQuality = entry.data.u8[0]; + } + cameraParams->depth_map = processDepthMapFrame(inputFrame.depthBuffer, maxDepthJpegSize, + jpegQuality, &items); + if (cameraParams->depth_map == nullptr) { + ALOGE("%s: Depth map processing failed!", __FUNCTION__); + return BAD_VALUE; + } + cameraParams->imaging_model = getImagingModel(); + + if (mIsLogicalCamera) { + cameraParams->trait = dynamic_depth::CameraTrait::LOGICAL; + } else { + cameraParams->trait = dynamic_depth::CameraTrait::PHYSICAL; + } + + cameraList.emplace_back(Camera::FromData(std::move(cameraParams))); + + auto deviceParams = std::make_unique (Cameras::FromCameraArray(&cameraList)); + deviceParams->container = Container::FromItems(&items); + std::vector> profileList; + profileList.emplace_back(Profile::FromData("DepthPhoto", {0})); + deviceParams->profiles = Profiles::FromProfileArray(&profileList); + std::unique_ptr device = Device::FromData(std::move(deviceParams)); + if (device == nullptr) { + ALOGE("%s: Failed to initialize camera device", __FUNCTION__); + return BAD_VALUE; + } + + std::istringstream inputJpegStream(std::string(reinterpret_cast (imgBuffer.data), + jpegSize)); + std::ostringstream outputJpegStream; + if (!WriteImageAndMetadataAndContainer(&inputJpegStream, device.get(), &outputJpegStream)) { + ALOGE("%s: Failed writing depth output", __FUNCTION__); + return BAD_VALUE; + } + + size_t finalJpegSize = static_cast (outputJpegStream.tellp()) + + sizeof(struct camera3_jpeg_blob); + + ALOGV("%s: Final jpeg size: %zu", __func__, finalJpegSize); + if ((res = native_window_set_buffers_dimensions(mOutputSurface.get(), finalJpegSize, 1)) + != OK) { + ALOGE("%s: Unable to configure stream buffer dimensions" + " %zux%u for stream %d", __FUNCTION__, finalJpegSize, 1U, mBlobStreamId); + return res; + } + + res = outputANW->dequeueBuffer(mOutputSurface.get(), &anb, &fenceFd); + if (res != OK) { + ALOGE("%s: Error retrieving output buffer: %s (%d)", __FUNCTION__, strerror(-res), + res); + return res; + } + + sp gb = GraphicBuffer::from(anb); + res = gb->lockAsync(GRALLOC_USAGE_SW_WRITE_OFTEN, &dstBuffer, fenceFd); + if (res != OK) { + ALOGE("%s: Error trying to lock output buffer fence: %s (%d)", __FUNCTION__, + strerror(-res), res); + outputANW->cancelBuffer(mOutputSurface.get(), anb, /*fence*/ -1); + return res; + } + + if ((gb->getWidth() < finalJpegSize) || (gb->getHeight() != 1)) { + ALOGE("%s: Blob buffer size mismatch, expected %dx%d received %zux%u", __FUNCTION__, + gb->getWidth(), gb->getHeight(), finalJpegSize, 1U); + outputANW->cancelBuffer(mOutputSurface.get(), anb, /*fence*/ -1); + return BAD_VALUE; + } + + // Copy final jpeg with embedded depth data in the composite stream output buffer + uint8_t* header = static_cast (dstBuffer) + + (gb->getWidth() - sizeof(struct camera3_jpeg_blob)); + struct camera3_jpeg_blob *blob = reinterpret_cast (header); + blob->jpeg_blob_id = CAMERA3_JPEG_BLOB_ID; + blob->jpeg_size = static_cast (outputJpegStream.tellp()); + memcpy(dstBuffer, outputJpegStream.str().c_str(), blob->jpeg_size); + outputANW->queueBuffer(mOutputSurface.get(), anb, /*fence*/ -1); + + return res; +} + +void DepthCompositeStream::releaseInputFrameLocked(InputFrame *inputFrame /*out*/) { + if (inputFrame == nullptr) { + return; + } + + if (inputFrame->depthBuffer.data != nullptr) { + mDepthConsumer->unlockBuffer(inputFrame->depthBuffer); + inputFrame->depthBuffer.data = nullptr; + mDepthBufferAcquired = false; + } + + if (inputFrame->jpegBuffer.data != nullptr) { + mBlobConsumer->unlockBuffer(inputFrame->jpegBuffer); + inputFrame->jpegBuffer.data = nullptr; + mBlobBufferAcquired = false; + } + + if ((inputFrame->error || mErrorState) && !inputFrame->errorNotified) { + notifyError(inputFrame->frameNumber); + inputFrame->errorNotified = true; + } +} + +void DepthCompositeStream::releaseInputFramesLocked(int64_t currentTs) { + auto it = mPendingInputFrames.begin(); + while (it != mPendingInputFrames.end()) { + if (it->first <= currentTs) { + releaseInputFrameLocked(&it->second); + it = mPendingInputFrames.erase(it); + } else { + it++; + } + } +} + +bool DepthCompositeStream::threadLoop() { + int64_t currentTs = INT64_MAX; + bool newInputAvailable = false; + + { + Mutex::Autolock l(mMutex); + + if (mErrorState) { + // In case we landed in error state, return any pending buffers and + // halt all further processing. + compilePendingInputLocked(); + releaseInputFramesLocked(currentTs); + return false; + } + + while (!newInputAvailable) { + compilePendingInputLocked(); + newInputAvailable = getNextReadyInputLocked(¤tTs); + if (!newInputAvailable) { + auto failingFrameNumber = getNextFailingInputLocked(¤tTs); + if (failingFrameNumber >= 0) { + // We cannot erase 'mPendingInputFrames[currentTs]' at this point because it is + // possible for two internal stream buffers to fail. In such scenario the + // composite stream should notify the client about a stream buffer error only + // once and this information is kept within 'errorNotified'. + // Any present failed input frames will be removed on a subsequent call to + // 'releaseInputFramesLocked()'. + releaseInputFrameLocked(&mPendingInputFrames[currentTs]); + currentTs = INT64_MAX; + } + + auto ret = mInputReadyCondition.waitRelative(mMutex, kWaitDuration); + if (ret == TIMED_OUT) { + return true; + } else if (ret != OK) { + ALOGE("%s: Timed wait on condition failed: %s (%d)", __FUNCTION__, + strerror(-ret), ret); + return false; + } + } + } + } + + auto res = processInputFrame(mPendingInputFrames[currentTs]); + Mutex::Autolock l(mMutex); + if (res != OK) { + ALOGE("%s: Failed processing frame with timestamp: %" PRIu64 ": %s (%d)", __FUNCTION__, + currentTs, strerror(-res), res); + mPendingInputFrames[currentTs].error = true; + } + + releaseInputFramesLocked(currentTs); + + return true; +} + +bool DepthCompositeStream::isDepthCompositeStream(const sp &surface) { + ANativeWindow *anw = surface.get(); + status_t err; + int format; + if ((err = anw->query(anw, NATIVE_WINDOW_FORMAT, &format)) != OK) { + String8 msg = String8::format("Failed to query Surface format: %s (%d)", strerror(-err), + err); + ALOGE("%s: %s", __FUNCTION__, msg.string()); + return false; + } + + int dataspace; + if ((err = anw->query(anw, NATIVE_WINDOW_DEFAULT_DATASPACE, &dataspace)) != OK) { + String8 msg = String8::format("Failed to query Surface dataspace: %s (%d)", strerror(-err), + err); + ALOGE("%s: %s", __FUNCTION__, msg.string()); + return false; + } + + if ((format == HAL_PIXEL_FORMAT_BLOB) && (dataspace == HAL_DATASPACE_DYNAMIC_DEPTH)) { + return true; + } + + return false; +} + +status_t DepthCompositeStream::createInternalStreams(const std::vector>& consumers, + bool /*hasDeferredConsumer*/, uint32_t width, uint32_t height, int format, + camera3_stream_rotation_t rotation, int *id, const String8& physicalCameraId, + std::vector *surfaceIds, int /*streamSetId*/, bool /*isShared*/) { + if (mSupportedDepthSizes.empty()) { + ALOGE("%s: This camera device doesn't support any depth map streams!", __FUNCTION__); + return INVALID_OPERATION; + } + + size_t depthWidth, depthHeight; + auto ret = getMatchingDepthSize(width, height, mSupportedDepthSizes, &depthWidth, &depthHeight); + if (ret != OK) { + ALOGE("%s: Failed to find an appropriate depth stream size!", __FUNCTION__); + return ret; + } + + sp device = mDevice.promote(); + if (!device.get()) { + ALOGE("%s: Invalid camera device!", __FUNCTION__); + return NO_INIT; + } + + sp producer; + sp consumer; + BufferQueue::createBufferQueue(&producer, &consumer); + mBlobConsumer = new CpuConsumer(consumer, /*maxLockedBuffers*/1, /*controlledByApp*/ true); + mBlobConsumer->setFrameAvailableListener(this); + mBlobConsumer->setName(String8("Camera3-JpegCompositeStream")); + mBlobSurface = new Surface(producer); + + ret = device->createStream(mBlobSurface, width, height, format, kJpegDataSpace, rotation, + id, physicalCameraId, surfaceIds); + if (ret == OK) { + mBlobStreamId = *id; + mBlobSurfaceId = (*surfaceIds)[0]; + mOutputSurface = consumers[0]; + } else { + return ret; + } + + BufferQueue::createBufferQueue(&producer, &consumer); + mDepthConsumer = new CpuConsumer(consumer, /*maxLockedBuffers*/ 1, /*controlledByApp*/ true); + mDepthConsumer->setFrameAvailableListener(this); + mDepthConsumer->setName(String8("Camera3-DepthCompositeStream")); + mDepthSurface = new Surface(producer); + std::vector depthSurfaceId; + ret = device->createStream(mDepthSurface, depthWidth, depthHeight, kDepthMapPixelFormat, + kDepthMapDataSpace, rotation, &mDepthStreamId, physicalCameraId, &depthSurfaceId); + if (ret == OK) { + mDepthSurfaceId = depthSurfaceId[0]; + } else { + return ret; + } + + ret = registerCompositeStreamListener(getStreamId()); + if (ret != OK) { + ALOGE("%s: Failed to register blob stream listener!", __FUNCTION__); + return ret; + } + + ret = registerCompositeStreamListener(mDepthStreamId); + if (ret != OK) { + ALOGE("%s: Failed to register depth stream listener!", __FUNCTION__); + return ret; + } + + mBlobWidth = width; + mBlobHeight = height; + + return ret; +} + +status_t DepthCompositeStream::configureStream() { + if (isRunning()) { + // Processing thread is already running, nothing more to do. + return NO_ERROR; + } + + if (mOutputSurface.get() == nullptr) { + ALOGE("%s: No valid output surface set!", __FUNCTION__); + return NO_INIT; + } + + auto res = mOutputSurface->connect(NATIVE_WINDOW_API_CAMERA, mProducerListener); + if (res != OK) { + ALOGE("%s: Unable to connect to native window for stream %d", + __FUNCTION__, mBlobStreamId); + return res; + } + + if ((res = native_window_set_buffers_format(mOutputSurface.get(), HAL_PIXEL_FORMAT_BLOB)) + != OK) { + ALOGE("%s: Unable to configure stream buffer format for stream %d", __FUNCTION__, + mBlobStreamId); + return res; + } + + int maxProducerBuffers; + ANativeWindow *anw = mBlobSurface.get(); + if ((res = anw->query(anw, NATIVE_WINDOW_MIN_UNDEQUEUED_BUFFERS, &maxProducerBuffers)) != OK) { + ALOGE("%s: Unable to query consumer undequeued" + " buffer count for stream %d", __FUNCTION__, mBlobStreamId); + return res; + } + + ANativeWindow *anwConsumer = mOutputSurface.get(); + int maxConsumerBuffers; + if ((res = anwConsumer->query(anwConsumer, NATIVE_WINDOW_MIN_UNDEQUEUED_BUFFERS, + &maxConsumerBuffers)) != OK) { + ALOGE("%s: Unable to query consumer undequeued" + " buffer count for stream %d", __FUNCTION__, mBlobStreamId); + return res; + } + + if ((res = native_window_set_buffer_count( + anwConsumer, maxProducerBuffers + maxConsumerBuffers)) != OK) { + ALOGE("%s: Unable to set buffer count for stream %d", __FUNCTION__, mBlobStreamId); + return res; + } + + run("DepthCompositeStreamProc"); + + return NO_ERROR; +} + +status_t DepthCompositeStream::deleteInternalStreams() { + // The 'CameraDeviceClient' parent will delete the blob stream + requestExit(); + + auto ret = join(); + if (ret != OK) { + ALOGE("%s: Failed to join with the main processing thread: %s (%d)", __FUNCTION__, + strerror(-ret), ret); + } + + sp device = mDevice.promote(); + if (!device.get()) { + ALOGE("%s: Invalid camera device!", __FUNCTION__); + return NO_INIT; + } + + if (mDepthStreamId >= 0) { + ret = device->deleteStream(mDepthStreamId); + mDepthStreamId = -1; + } + + return ret; +} + +void DepthCompositeStream::onFrameAvailable(const BufferItem& item) { + if (item.mDataSpace == kJpegDataSpace) { + ALOGV("%s: Jpeg buffer with ts: %" PRIu64 " ms. arrived!", + __func__, ns2ms(item.mTimestamp)); + + Mutex::Autolock l(mMutex); + if (!mErrorState) { + mInputJpegBuffers.push_back(item.mTimestamp); + mInputReadyCondition.signal(); + } + } else if (item.mDataSpace == kDepthMapDataSpace) { + ALOGV("%s: Depth buffer with ts: %" PRIu64 " ms. arrived!", __func__, + ns2ms(item.mTimestamp)); + + Mutex::Autolock l(mMutex); + if (!mErrorState) { + mInputDepthBuffers.push_back(item.mTimestamp); + mInputReadyCondition.signal(); + } + } else { + ALOGE("%s: Unexpected data space: 0x%x", __FUNCTION__, item.mDataSpace); + } +} + +status_t DepthCompositeStream::insertGbp(SurfaceMap* /*out*/outSurfaceMap, + Vector * /*out*/outputStreamIds, int32_t* /*out*/currentStreamId) { + if (outSurfaceMap->find(mDepthStreamId) == outSurfaceMap->end()) { + (*outSurfaceMap)[mDepthStreamId] = std::vector(); + outputStreamIds->push_back(mDepthStreamId); + } + (*outSurfaceMap)[mDepthStreamId].push_back(mDepthSurfaceId); + + if (outSurfaceMap->find(mBlobStreamId) == outSurfaceMap->end()) { + (*outSurfaceMap)[mBlobStreamId] = std::vector(); + outputStreamIds->push_back(mBlobStreamId); + } + (*outSurfaceMap)[mBlobStreamId].push_back(mBlobSurfaceId); + + if (currentStreamId != nullptr) { + *currentStreamId = mBlobStreamId; + } + + return NO_ERROR; +} + +void DepthCompositeStream::onResultError(const CaptureResultExtras& resultExtras) { + // Processing can continue even in case of result errors. + // At the moment depth composite stream processing relies mainly on static camera + // characteristics data. The actual result data can be used for the jpeg quality but + // in case it is absent we can default to maximum. + eraseResult(resultExtras.frameNumber); +} + +bool DepthCompositeStream::onStreamBufferError(const CaptureResultExtras& resultExtras) { + bool ret = false; + // Buffer errors concerning internal composite streams should not be directly visible to + // camera clients. They must only receive a single buffer error with the public composite + // stream id. + if ((resultExtras.errorStreamId == mDepthStreamId) || + (resultExtras.errorStreamId == mBlobStreamId)) { + flagAnErrorFrameNumber(resultExtras.frameNumber); + ret = true; + } + + return ret; +} + +status_t DepthCompositeStream::getMatchingDepthSize(size_t width, size_t height, + const std::vector>& supporedDepthSizes, + size_t *depthWidth /*out*/, size_t *depthHeight /*out*/) { + if ((depthWidth == nullptr) || (depthHeight == nullptr)) { + return BAD_VALUE; + } + + float arTol = CameraProviderManager::kDepthARTolerance; + *depthWidth = *depthHeight = 0; + + float aspectRatio = static_cast (width) / static_cast (height); + for (const auto& it : supporedDepthSizes) { + auto currentWidth = std::get<0>(it); + auto currentHeight = std::get<1>(it); + if ((currentWidth == width) && (currentHeight == height)) { + *depthWidth = width; + *depthHeight = height; + break; + } else { + float currentRatio = static_cast (currentWidth) / + static_cast (currentHeight); + auto currentSize = currentWidth * currentHeight; + auto oldSize = (*depthWidth) * (*depthHeight); + if ((fabs(aspectRatio - currentRatio) <= arTol) && (currentSize > oldSize)) { + *depthWidth = currentWidth; + *depthHeight = currentHeight; + } + } + } + + return ((*depthWidth > 0) && (*depthHeight > 0)) ? OK : BAD_VALUE; +} + +void DepthCompositeStream::getSupportedDepthSizes(const CameraMetadata& ch, + std::vector>* depthSizes /*out*/) { + if (depthSizes == nullptr) { + return; + } + + auto entry = ch.find(ANDROID_DEPTH_AVAILABLE_DEPTH_STREAM_CONFIGURATIONS); + if (entry.count > 0) { + // Depth stream dimensions have four int32_t components + // (pixelformat, width, height, type) + size_t entryCount = entry.count / 4; + depthSizes->reserve(entryCount); + for (size_t i = 0; i < entry.count; i += 4) { + if ((entry.data.i32[i] == kDepthMapPixelFormat) && + (entry.data.i32[i+3] == + ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS_OUTPUT)) { + depthSizes->push_back(std::make_tuple(entry.data.i32[i+1], + entry.data.i32[i+2])); + } + } + } +} + +status_t DepthCompositeStream::getCompositeStreamInfo(const OutputStreamInfo &streamInfo, + const CameraMetadata& ch, std::vector* compositeOutput /*out*/) { + if (compositeOutput == nullptr) { + return BAD_VALUE; + } + + std::vector> depthSizes; + getSupportedDepthSizes(ch, &depthSizes); + if (depthSizes.empty()) { + ALOGE("%s: No depth stream configurations present", __FUNCTION__); + return BAD_VALUE; + } + + size_t depthWidth, depthHeight; + auto ret = getMatchingDepthSize(streamInfo.width, streamInfo.height, depthSizes, &depthWidth, + &depthHeight); + if (ret != OK) { + ALOGE("%s: No matching depth stream size found", __FUNCTION__); + return ret; + } + + compositeOutput->clear(); + compositeOutput->insert(compositeOutput->end(), 2, streamInfo); + + // Jpeg/Blob stream info + (*compositeOutput)[0].dataSpace = kJpegDataSpace; + (*compositeOutput)[0].consumerUsage = GRALLOC_USAGE_SW_READ_OFTEN; + + // Depth stream info + (*compositeOutput)[1].width = depthWidth; + (*compositeOutput)[1].height = depthHeight; + (*compositeOutput)[1].format = kDepthMapPixelFormat; + (*compositeOutput)[1].dataSpace = kDepthMapDataSpace; + (*compositeOutput)[1].consumerUsage = GRALLOC_USAGE_SW_READ_OFTEN; + + return NO_ERROR; +} + +std::unique_ptr DepthCompositeStream::getImagingModel() { + // It is not possible to generate an imaging model without instrinsic calibration. + if (mInstrinsicCalibration.empty() || mInstrinsicCalibration.size() != 5) { + return nullptr; + } + + // The camera intrinsic calibration layout is as follows: + // [focalLengthX, focalLengthY, opticalCenterX, opticalCenterY, skew] + const dynamic_depth::Point focalLength(mInstrinsicCalibration[0], + mInstrinsicCalibration[1]); + const Dimension imageSize(mBlobWidth, mBlobHeight); + ImagingModelParams params(focalLength, imageSize); + params.principal_point.x = mInstrinsicCalibration[2]; + params.principal_point.y = mInstrinsicCalibration[3]; + params.skew = mInstrinsicCalibration[4]; + + // The camera lens distortion contains the following lens correction coefficients. + // [kappa_1, kappa_2, kappa_3 kappa_4, kappa_5] + if (mLensDistortion.size() == 5) { + // According to specification the lens distortion coefficients should be ordered + // as [1, kappa_4, kappa_1, kappa_5, kappa_2, 0, kappa_3, 0] + float distortionData[] = {1.f, mLensDistortion[3], mLensDistortion[0], mLensDistortion[4], + mLensDistortion[1], 0.f, mLensDistortion[2], 0.f}; + auto distortionDataLength = sizeof(distortionData) / sizeof(distortionData[0]); + params.distortion.reserve(distortionDataLength); + params.distortion.insert(params.distortion.end(), distortionData, + distortionData + distortionDataLength); + } + + return ImagingModel::FromData(params); +} + +}; // namespace camera3 +}; // namespace android diff --git a/services/camera/libcameraservice/api2/DepthCompositeStream.h b/services/camera/libcameraservice/api2/DepthCompositeStream.h new file mode 100644 index 0000000000..129d5385e6 --- /dev/null +++ b/services/camera/libcameraservice/api2/DepthCompositeStream.h @@ -0,0 +1,148 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ANDROID_SERVERS_CAMERA_CAMERA3_DEPTH_COMPOSITE_STREAM_H +#define ANDROID_SERVERS_CAMERA_CAMERA3_DEPTH_COMPOSITE_STREAM_H + +#include +#include + +#include +#include + +#include "CompositeStream.h" + +using dynamic_depth::DepthMap; +using dynamic_depth::Item; +using dynamic_depth::ImagingModel; + +namespace android { + +class CameraDeviceClient; +class CameraMetadata; +class Surface; + +namespace camera3 { + +class DepthCompositeStream : public CompositeStream, public Thread, + public CpuConsumer::FrameAvailableListener { + +public: + DepthCompositeStream(wp device, + wp cb); + ~DepthCompositeStream() override; + + static bool isDepthCompositeStream(const sp &surface); + + // CompositeStream overrides + status_t createInternalStreams(const std::vector>& consumers, + bool hasDeferredConsumer, uint32_t width, uint32_t height, int format, + camera3_stream_rotation_t rotation, int *id, const String8& physicalCameraId, + std::vector *surfaceIds, int streamSetId, bool isShared) override; + status_t deleteInternalStreams() override; + status_t configureStream() override; + status_t insertGbp(SurfaceMap* /*out*/outSurfaceMap, Vector* /*out*/outputStreamIds, + int32_t* /*out*/currentStreamId) override; + int getStreamId() override { return mBlobStreamId; } + + // CpuConsumer listener implementation + void onFrameAvailable(const BufferItem& item) override; + + // Return stream information about the internal camera streams + static status_t getCompositeStreamInfo(const OutputStreamInfo &streamInfo, + const CameraMetadata& ch, std::vector* compositeOutput /*out*/); + +protected: + + bool threadLoop() override; + bool onStreamBufferError(const CaptureResultExtras& resultExtras) override; + void onResultError(const CaptureResultExtras& resultExtras) override; + +private: + struct InputFrame { + CpuConsumer::LockedBuffer depthBuffer; + CpuConsumer::LockedBuffer jpegBuffer; + CameraMetadata result; + bool error; + bool errorNotified; + int64_t frameNumber; + + InputFrame() : error(false), errorNotified(false), frameNumber(-1) { } + }; + + // Helper methods + static void getSupportedDepthSizes(const CameraMetadata& ch, + std::vector>* depthSizes /*out*/); + static status_t getMatchingDepthSize(size_t width, size_t height, + const std::vector>& supporedDepthSizes, + size_t *depthWidth /*out*/, size_t *depthHeight /*out*/); + + // Dynamic depth processing + status_t encodeGrayscaleJpeg(size_t width, size_t height, uint8_t *in, void *out, + const size_t maxOutSize, uint8_t jpegQuality, size_t &actualSize); + std::unique_ptr processDepthMapFrame(const CpuConsumer::LockedBuffer &depthMapBuffer, + size_t maxJpegSize, uint8_t jpegQuality, + std::vector>* items /*out*/); + std::unique_ptr getImagingModel(); + status_t processInputFrame(const InputFrame &inputFrame); + + // Buffer/Results handling + void compilePendingInputLocked(); + void releaseInputFrameLocked(InputFrame *inputFrame /*out*/); + void releaseInputFramesLocked(int64_t currentTs); + + // Find first complete and valid frame with smallest timestamp + bool getNextReadyInputLocked(int64_t *currentTs /*inout*/); + + // Find next failing frame number with smallest timestamp and return respective frame number + int64_t getNextFailingInputLocked(int64_t *currentTs /*inout*/); + + static const nsecs_t kWaitDuration = 10000000; // 10 ms + static const auto kDepthMapPixelFormat = HAL_PIXEL_FORMAT_Y16; + static const auto kDepthMapDataSpace = HAL_DATASPACE_DEPTH; + static const auto kJpegDataSpace = HAL_DATASPACE_V0_JFIF; + + struct ProducerListener : public BnProducerListener { + // ProducerListener implementation + void onBufferReleased() override { /*No impl. for now*/ }; + }; + + int mBlobStreamId, mBlobSurfaceId, mDepthStreamId, mDepthSurfaceId; + size_t mBlobWidth, mBlobHeight; + sp mBlobConsumer, mDepthConsumer; + bool mDepthBufferAcquired, mBlobBufferAcquired; + sp mDepthSurface, mBlobSurface, mOutputSurface; + sp mProducerListener; + + ssize_t mMaxJpegSize; + std::vector> mSupportedDepthSizes; + std::vector mInstrinsicCalibration, mLensDistortion; + bool mIsLogicalCamera; + + // Keep all incoming Depth buffer timestamps pending further processing. + std::vector mInputDepthBuffers; + + // Keep all incoming Jpeg/Blob buffer timestamps pending further processing. + std::vector mInputJpegBuffers; + + // Map of all input frames pending further processing. + std::unordered_map mPendingInputFrames; +}; + +}; //namespace camera3 +}; //namespace android + +#endif diff --git a/services/camera/libcameraservice/common/CameraProviderManager.cpp b/services/camera/libcameraservice/common/CameraProviderManager.cpp index cee4a84433..d6d34e95f2 100644 --- a/services/camera/libcameraservice/common/CameraProviderManager.cpp +++ b/services/camera/libcameraservice/common/CameraProviderManager.cpp @@ -58,6 +58,8 @@ const std::chrono::system_clock::duration kCameraKeepAliveDelay = 3s; } // anonymous namespace +const float CameraProviderManager::kDepthARTolerance = .1f; + CameraProviderManager::HardwareServiceInteractionProxy CameraProviderManager::sHardwareServiceInteractionProxy{}; @@ -576,10 +578,7 @@ void CameraProviderManager::ProviderInfo::DeviceInfo3::getSupportedDynamicDepthS // The dynamic depth spec. does not mention how close the AR ratio should be. // Try using something appropriate. - float ARTolerance = .01f; - - //TODO: Remove this before merging! This is for testing purposes only - ARTolerance = 10.f; + float ARTolerance = kDepthARTolerance; for (const auto& blobSize : blobSizes) { float jpegAR = static_cast (std::get<0>(blobSize)) / diff --git a/services/camera/libcameraservice/common/CameraProviderManager.h b/services/camera/libcameraservice/common/CameraProviderManager.h index 1f34f2fc46..80ec130e98 100644 --- a/services/camera/libcameraservice/common/CameraProviderManager.h +++ b/services/camera/libcameraservice/common/CameraProviderManager.h @@ -265,6 +265,8 @@ public: bool isLogicalCamera(const std::string& id, std::vector* physicalCameraIds); bool isHiddenPhysicalCamera(const std::string& cameraId); + + static const float kDepthARTolerance; private: // All private members, unless otherwise noted, expect mInterfaceMutex to be locked before use mutable std::mutex mInterfaceMutex; diff --git a/services/camera/libcameraservice/device3/Camera3Device.cpp b/services/camera/libcameraservice/device3/Camera3Device.cpp index 12fbf82f7c..99b804394a 100644 --- a/services/camera/libcameraservice/device3/Camera3Device.cpp +++ b/services/camera/libcameraservice/device3/Camera3Device.cpp @@ -3119,10 +3119,12 @@ void Camera3Device::returnOutputBuffers( status_t res = OK; if (it != outputSurfaces.end()) { res = stream->returnBuffer( - outputBuffers[i], timestamp, timestampIncreasing, it->second); + outputBuffers[i], timestamp, timestampIncreasing, it->second, + inResultExtras.frameNumber); } else { res = stream->returnBuffer( - outputBuffers[i], timestamp, timestampIncreasing); + outputBuffers[i], timestamp, timestampIncreasing, std::vector (), + inResultExtras.frameNumber); } // Note: stream may be deallocated at this point, if this buffer was @@ -3139,7 +3141,8 @@ void Camera3Device::returnOutputBuffers( // cancel the buffer camera3_stream_buffer_t sb = outputBuffers[i]; sb.status = CAMERA3_BUFFER_STATUS_ERROR; - stream->returnBuffer(sb, /*timestamp*/0, timestampIncreasing); + stream->returnBuffer(sb, /*timestamp*/0, timestampIncreasing, std::vector (), + inResultExtras.frameNumber); // notify client buffer error sp listener; @@ -3279,7 +3282,8 @@ void Camera3Device::flushInflightRequests() { streamBuffer.stream = halStream; switch (halStream->stream_type) { case CAMERA3_STREAM_OUTPUT: - res = stream->returnBuffer(streamBuffer, /*timestamp*/ 0); + res = stream->returnBuffer(streamBuffer, /*timestamp*/ 0, + /*timestampIncreasing*/true, std::vector (), frameNumber); if (res != OK) { ALOGE("%s: Can't return output buffer for frame %d to" " stream %d: %s (%d)", __FUNCTION__, @@ -5469,6 +5473,8 @@ status_t Camera3Device::RequestThread::prepareHalRequests() { return TIMED_OUT; } } + outputStream->fireBufferRequestForFrameNumber( + captureRequest->mResultExtras.frameNumber); String8 physicalCameraId = outputStream->getPhysicalCameraId(); @@ -5692,7 +5698,9 @@ void Camera3Device::RequestThread::cleanUpFailedRequests(bool sendRequestError) outputBuffers->editItemAt(i).acquire_fence = -1; } outputBuffers->editItemAt(i).status = CAMERA3_BUFFER_STATUS_ERROR; - captureRequest->mOutputStreams.editItemAt(i)->returnBuffer((*outputBuffers)[i], 0); + captureRequest->mOutputStreams.editItemAt(i)->returnBuffer((*outputBuffers)[i], 0, + /*timestampIncreasing*/true, std::vector (), + captureRequest->mResultExtras.frameNumber); } if (sendRequestError) { diff --git a/services/camera/libcameraservice/device3/Camera3Stream.cpp b/services/camera/libcameraservice/device3/Camera3Stream.cpp index 24d1c1bd24..b296513245 100644 --- a/services/camera/libcameraservice/device3/Camera3Stream.cpp +++ b/services/camera/libcameraservice/device3/Camera3Stream.cpp @@ -656,7 +656,7 @@ void Camera3Stream::removeOutstandingBuffer(const camera3_stream_buffer &buffer) status_t Camera3Stream::returnBuffer(const camera3_stream_buffer &buffer, nsecs_t timestamp, bool timestampIncreasing, - const std::vector& surface_ids) { + const std::vector& surface_ids, uint64_t frameNumber) { ATRACE_CALL(); Mutex::Autolock l(mLock); @@ -687,7 +687,7 @@ status_t Camera3Stream::returnBuffer(const camera3_stream_buffer &buffer, */ status_t res = returnBufferLocked(b, timestamp, surface_ids); if (res == OK) { - fireBufferListenersLocked(b, /*acquired*/false, /*output*/true); + fireBufferListenersLocked(b, /*acquired*/false, /*output*/true, timestamp, frameNumber); } // Even if returning the buffer failed, we still want to signal whoever is waiting for the @@ -763,8 +763,21 @@ status_t Camera3Stream::getInputBufferProducer(sp *produ return getInputBufferProducerLocked(producer); } +void Camera3Stream::fireBufferRequestForFrameNumber(uint64_t frameNumber) { + ATRACE_CALL(); + Mutex::Autolock l(mLock); + + for (auto &it : mBufferListenerList) { + sp listener = it.promote(); + if (listener.get() != nullptr) { + listener->onBufferRequestForFrameNumber(frameNumber, getId()); + } + } +} + void Camera3Stream::fireBufferListenersLocked( - const camera3_stream_buffer& buffer, bool acquired, bool output) { + const camera3_stream_buffer& buffer, bool acquired, bool output, nsecs_t timestamp, + uint64_t frameNumber) { List >::iterator it, end; // TODO: finish implementing @@ -773,6 +786,8 @@ void Camera3Stream::fireBufferListenersLocked( Camera3StreamBufferListener::BufferInfo(); info.mOutput = output; info.mError = (buffer.status == CAMERA3_BUFFER_STATUS_ERROR); + info.mFrameNumber = frameNumber; + info.mTimestamp = timestamp; // TODO: rest of fields for (it = mBufferListenerList.begin(), end = mBufferListenerList.end(); diff --git a/services/camera/libcameraservice/device3/Camera3Stream.h b/services/camera/libcameraservice/device3/Camera3Stream.h index ddba9f63e5..06deba9fa9 100644 --- a/services/camera/libcameraservice/device3/Camera3Stream.h +++ b/services/camera/libcameraservice/device3/Camera3Stream.h @@ -332,7 +332,8 @@ class Camera3Stream : */ status_t returnBuffer(const camera3_stream_buffer &buffer, nsecs_t timestamp, bool timestampIncreasing, - const std::vector& surface_ids = std::vector()); + const std::vector& surface_ids = std::vector(), + uint64_t frameNumber = 0); /** * Fill in the camera3_stream_buffer with the next valid buffer for this @@ -430,6 +431,11 @@ class Camera3Stream : */ status_t restoreConfiguredState(); + /** + * Notify buffer stream listeners about incoming request with particular frame number. + */ + void fireBufferRequestForFrameNumber(uint64_t frameNumber) override; + protected: const int mId; /** @@ -538,7 +544,7 @@ class Camera3Stream : static const nsecs_t kWaitForBufferDuration = 3000000000LL; // 3000 ms void fireBufferListenersLocked(const camera3_stream_buffer& buffer, - bool acquired, bool output); + bool acquired, bool output, nsecs_t timestamp = 0, uint64_t frameNumber = 0); List > mBufferListenerList; status_t cancelPrepareLocked(); diff --git a/services/camera/libcameraservice/device3/Camera3StreamBufferListener.h b/services/camera/libcameraservice/device3/Camera3StreamBufferListener.h index 2db333db07..0e6104e7ff 100644 --- a/services/camera/libcameraservice/device3/Camera3StreamBufferListener.h +++ b/services/camera/libcameraservice/device3/Camera3StreamBufferListener.h @@ -41,6 +41,8 @@ public: virtual void onBufferAcquired(const BufferInfo& bufferInfo) = 0; // Buffer was released by the HAL virtual void onBufferReleased(const BufferInfo& bufferInfo) = 0; + // Notify about incoming buffer request frame number + virtual void onBufferRequestForFrameNumber(uint64_t frameNumber, int streamId) = 0; }; }; //namespace camera3 diff --git a/services/camera/libcameraservice/device3/Camera3StreamInterface.h b/services/camera/libcameraservice/device3/Camera3StreamInterface.h index a84720bfe2..7b80cbd687 100644 --- a/services/camera/libcameraservice/device3/Camera3StreamInterface.h +++ b/services/camera/libcameraservice/device3/Camera3StreamInterface.h @@ -259,7 +259,8 @@ class Camera3StreamInterface : public virtual RefBase { */ virtual status_t returnBuffer(const camera3_stream_buffer &buffer, nsecs_t timestamp, bool timestampIncreasing = true, - const std::vector& surface_ids = std::vector()) = 0; + const std::vector& surface_ids = std::vector(), + uint64_t frameNumber = 0) = 0; /** * Fill in the camera3_stream_buffer with the next valid buffer for this @@ -341,6 +342,11 @@ class Camera3StreamInterface : public virtual RefBase { * Camera3Stream. */ virtual void setBufferFreedListener(wp listener) = 0; + + /** + * Notify buffer stream listeners about incoming request with particular frame number. + */ + virtual void fireBufferRequestForFrameNumber(uint64_t frameNumber) = 0; }; } // namespace camera3