From 4697b64a3b853898ec26e6397eacc3d8bb486473 Mon Sep 17 00:00:00 2001 From: Emilian Peev Date: Tue, 19 Nov 2019 17:11:14 -0800 Subject: [PATCH] Camera: Initial support for composite streams in offline mode Composite streams must be able to switch to offline processing mode. To support them the offline session client must continue passing all necessary callbacks and events similar to regular camera clients. Test: Camera CTS Bug: 135142453 Change-Id: I498681af16ad072e3df01d0279b4cfe76b48f9ec --- .../api2/CameraDeviceClient.cpp | 25 +++++++--- .../api2/CameraOfflineSessionClient.cpp | 49 ++++++++++++++++++- .../api2/CameraOfflineSessionClient.h | 14 +++++- .../libcameraservice/api2/CompositeStream.h | 4 ++ .../api2/DepthCompositeStream.cpp | 12 +++++ .../api2/DepthCompositeStream.h | 1 + .../api2/HeicCompositeStream.cpp | 12 +++++ .../api2/HeicCompositeStream.h | 2 + 8 files changed, 110 insertions(+), 9 deletions(-) diff --git a/services/camera/libcameraservice/api2/CameraDeviceClient.cpp b/services/camera/libcameraservice/api2/CameraDeviceClient.cpp index f618b2e764..e7e72b2f61 100644 --- a/services/camera/libcameraservice/api2/CameraDeviceClient.cpp +++ b/services/camera/libcameraservice/api2/CameraDeviceClient.cpp @@ -1930,6 +1930,7 @@ binder::Status CameraDeviceClient::switchToOffline( } std::vector offlineStreamIds(offlineOutputIds.size()); + KeyedVector, sp> offlineCompositeStreamMap; for (const auto& streamId : offlineOutputIds) { ssize_t index = mConfiguredOutputs.indexOfKey(streamId); if (index == NAME_NOT_FOUND) { @@ -1944,11 +1945,23 @@ binder::Status CameraDeviceClient::switchToOffline( sp s = new Surface(gbp, false /*controlledByApp*/); isCompositeStream = camera3::DepthCompositeStream::isDepthCompositeStream(s) | camera3::HeicCompositeStream::isHeicCompositeStream(s); + if (isCompositeStream) { + auto compositeIdx = mCompositeStreamMap.indexOfKey(IInterface::asBinder(gbp)); + if (compositeIdx == NAME_NOT_FOUND) { + ALOGE("%s: Unknown composite stream", __FUNCTION__); + return STATUS_ERROR(CameraService::ERROR_ILLEGAL_ARGUMENT, + "Unknown composite stream"); + } + + mCompositeStreamMap.valueAt(compositeIdx)->insertCompositeStreamIds( + &offlineStreamIds); + offlineCompositeStreamMap.add(mCompositeStreamMap.keyAt(compositeIdx), + mCompositeStreamMap.valueAt(compositeIdx)); + break; + } } - if (isCompositeStream) { - // TODO: Add composite specific handling - } else { + if (!isCompositeStream) { offlineStreamIds.push_back(streamId); } } @@ -1962,11 +1975,11 @@ binder::Status CameraDeviceClient::switchToOffline( } sp offlineClient = new CameraOfflineSessionClient(sCameraService, - offlineSession, cameraCb, mClientPackageName, mClientFeatureId, mCameraIdStr, - mCameraFacing, mClientPid, mClientUid, mServicePid); + offlineSession, offlineCompositeStreamMap, cameraCb, mClientPackageName, + mClientFeatureId, mCameraIdStr, mCameraFacing, mClientPid, mClientUid, mServicePid); ret = sCameraService->addOfflineClient(mCameraIdStr, offlineClient); if (ret == OK) { - // TODO: We need to update mStreamMap, mConfiguredOutputs + // TODO: We need to update mStreamMap, mConfiguredOutputs, mCompositeStreams } else { switch(ret) { case BAD_VALUE: diff --git a/services/camera/libcameraservice/api2/CameraOfflineSessionClient.cpp b/services/camera/libcameraservice/api2/CameraOfflineSessionClient.cpp index 974e10acc2..f713419c4c 100644 --- a/services/camera/libcameraservice/api2/CameraOfflineSessionClient.cpp +++ b/services/camera/libcameraservice/api2/CameraOfflineSessionClient.cpp @@ -59,6 +59,15 @@ binder::Status CameraOfflineSessionClient::disconnect() { // client shouldn't be able to call into us anymore mClientPid = 0; + for (size_t i = 0; i < mCompositeStreamMap.size(); i++) { + auto ret = mCompositeStreamMap.valueAt(i)->deleteInternalStreams(); + if (ret != OK) { + ALOGE("%s: Failed removing composite stream %s (%d)", __FUNCTION__, + strerror(-ret), ret); + } + } + mCompositeStreamMap.clear(); + return res; } @@ -66,8 +75,15 @@ void CameraOfflineSessionClient::notifyError(int32_t errorCode, const CaptureResultExtras& resultExtras) { // Thread safe. Don't bother locking. sp remoteCb = getRemoteCallback(); - // TODO: handle composite streams - if ((remoteCb != 0)) { + // + // Composites can have multiple internal streams. Error notifications coming from such internal + // streams may need to remain within camera service. + bool skipClientNotification = false; + for (size_t i = 0; i < mCompositeStreamMap.size(); i++) { + skipClientNotification |= mCompositeStreamMap.valueAt(i)->onError(errorCode, resultExtras); + } + + if ((remoteCb != 0) && (!skipClientNotification)) { remoteCb->onDeviceError(errorCode, resultExtras); } } @@ -136,5 +152,34 @@ status_t CameraOfflineSessionClient::finishCameraOps() { return OK; } +void CameraOfflineSessionClient::onResultAvailable(const CaptureResult& result) { + ATRACE_CALL(); + ALOGV("%s", __FUNCTION__); + + // Thread-safe. No lock necessary. + sp remoteCb = mRemoteCallback; + if (remoteCb != NULL) { + remoteCb->onResultReceived(result.mMetadata, result.mResultExtras, + result.mPhysicalMetadatas); + } + + for (size_t i = 0; i < mCompositeStreamMap.size(); i++) { + mCompositeStreamMap.valueAt(i)->onResultAvailable(result); + } +} + +void CameraOfflineSessionClient::notifyShutter(const CaptureResultExtras& resultExtras, + nsecs_t timestamp) { + // Thread safe. Don't bother locking. + sp remoteCb = getRemoteCallback(); + if (remoteCb != 0) { + remoteCb->onCaptureStarted(resultExtras, timestamp); + } + + for (size_t i = 0; i < mCompositeStreamMap.size(); i++) { + mCompositeStreamMap.valueAt(i)->onShutter(resultExtras, timestamp); + } +} + // ---------------------------------------------------------------------------- }; // namespace android diff --git a/services/camera/libcameraservice/api2/CameraOfflineSessionClient.h b/services/camera/libcameraservice/api2/CameraOfflineSessionClient.h index 0bb14008ab..e92a05cedf 100644 --- a/services/camera/libcameraservice/api2/CameraOfflineSessionClient.h +++ b/services/camera/libcameraservice/api2/CameraOfflineSessionClient.h @@ -20,10 +20,12 @@ #include #include #include "CameraService.h" +#include "CompositeStream.h" namespace android { using android::hardware::camera2::ICameraDeviceCallbacks; +using camera3::CompositeStream; // Client for offline session. Note that offline session client does not affect camera service's // client arbitration logic. It is camera HAL's decision to decide whether a normal camera @@ -40,6 +42,7 @@ public: CameraOfflineSessionClient( const sp& cameraService, sp session, + const KeyedVector, sp>& offlineCompositeStreamMap, const sp& remoteCallback, const String16& clientPackageName, const std::unique_ptr& clientFeatureId, @@ -50,7 +53,8 @@ public: IInterface::asBinder(remoteCallback), clientPackageName, clientFeatureId, cameraIdStr, cameraFacing, clientPid, clientUid, servicePid), - mRemoteCallback(remoteCallback), mOfflineSession(session) {} + mRemoteCallback(remoteCallback), mOfflineSession(session), + mCompositeStreamMap(offlineCompositeStreamMap) {} virtual ~CameraOfflineSessionClient() {} @@ -74,6 +78,11 @@ public: virtual status_t startCameraOps() override; virtual status_t finishCameraOps() override; + // TODO: Those will be introduced when we implement FilteredListener and the device + // callbacks respectively. Just adding for now. + void onResultAvailable(const CaptureResult& result); + void notifyShutter(const CaptureResultExtras& resultExtras, nsecs_t timestamp); + private: const sp& getRemoteCallback() { @@ -83,6 +92,9 @@ private: sp mRemoteCallback; sp mOfflineSession; + + // Offline composite streams + KeyedVector, sp> mCompositeStreamMap; }; } // namespace android diff --git a/services/camera/libcameraservice/api2/CompositeStream.h b/services/camera/libcameraservice/api2/CompositeStream.h index a401a8258c..de894f318f 100644 --- a/services/camera/libcameraservice/api2/CompositeStream.h +++ b/services/camera/libcameraservice/api2/CompositeStream.h @@ -64,6 +64,10 @@ public: virtual status_t insertGbp(SurfaceMap* /*out*/outSurfaceMap, Vector* /*out*/outputStreamIds, int32_t* /*out*/currentStreamId) = 0; + // Attach the internal composite stream ids. + virtual status_t insertCompositeStreamIds( + std::vector* compositeStreamIds /*out*/) = 0; + // Return composite stream id. virtual int getStreamId() = 0; diff --git a/services/camera/libcameraservice/api2/DepthCompositeStream.cpp b/services/camera/libcameraservice/api2/DepthCompositeStream.cpp index ac6f8d6e26..acad8c6a3a 100644 --- a/services/camera/libcameraservice/api2/DepthCompositeStream.cpp +++ b/services/camera/libcameraservice/api2/DepthCompositeStream.cpp @@ -683,6 +683,18 @@ status_t DepthCompositeStream::insertGbp(SurfaceMap* /*out*/outSurfaceMap, return NO_ERROR; } +status_t DepthCompositeStream::insertCompositeStreamIds( + std::vector* compositeStreamIds /*out*/) { + if (compositeStreamIds == nullptr) { + return BAD_VALUE; + } + + compositeStreamIds->push_back(mDepthStreamId); + compositeStreamIds->push_back(mBlobStreamId); + + return OK; +} + void DepthCompositeStream::onResultError(const CaptureResultExtras& resultExtras) { // Processing can continue even in case of result errors. // At the moment depth composite stream processing relies mainly on static camera diff --git a/services/camera/libcameraservice/api2/DepthCompositeStream.h b/services/camera/libcameraservice/api2/DepthCompositeStream.h index 60c6b1e715..1bf714da09 100644 --- a/services/camera/libcameraservice/api2/DepthCompositeStream.h +++ b/services/camera/libcameraservice/api2/DepthCompositeStream.h @@ -56,6 +56,7 @@ public: status_t configureStream() override; status_t insertGbp(SurfaceMap* /*out*/outSurfaceMap, Vector* /*out*/outputStreamIds, int32_t* /*out*/currentStreamId) override; + status_t insertCompositeStreamIds(std::vector* compositeStreamIds /*out*/) override; int getStreamId() override { return mBlobStreamId; } // CpuConsumer listener implementation diff --git a/services/camera/libcameraservice/api2/HeicCompositeStream.cpp b/services/camera/libcameraservice/api2/HeicCompositeStream.cpp index 9cdad700fe..d25e467fb7 100644 --- a/services/camera/libcameraservice/api2/HeicCompositeStream.cpp +++ b/services/camera/libcameraservice/api2/HeicCompositeStream.cpp @@ -504,6 +504,18 @@ status_t HeicCompositeStream::insertGbp(SurfaceMap* /*out*/outSurfaceMap, return NO_ERROR; } +status_t HeicCompositeStream::insertCompositeStreamIds( + std::vector* compositeStreamIds /*out*/) { + if (compositeStreamIds == nullptr) { + return BAD_VALUE; + } + + compositeStreamIds->push_back(mAppSegmentStreamId); + compositeStreamIds->push_back(mMainImageStreamId); + + return OK; +} + void HeicCompositeStream::onShutter(const CaptureResultExtras& resultExtras, nsecs_t timestamp) { Mutex::Autolock l(mMutex); if (mErrorState) { diff --git a/services/camera/libcameraservice/api2/HeicCompositeStream.h b/services/camera/libcameraservice/api2/HeicCompositeStream.h index e88471d74d..8fc521e69d 100644 --- a/services/camera/libcameraservice/api2/HeicCompositeStream.h +++ b/services/camera/libcameraservice/api2/HeicCompositeStream.h @@ -55,6 +55,8 @@ public: status_t insertGbp(SurfaceMap* /*out*/outSurfaceMap, Vector* /*out*/outputStreamIds, int32_t* /*out*/currentStreamId) override; + status_t insertCompositeStreamIds(std::vector* compositeStreamIds /*out*/) override; + void onShutter(const CaptureResultExtras& resultExtras, nsecs_t timestamp) override; int getStreamId() override { return mMainImageStreamId; }