Camera: Initial support for composite streams in offline mode

Composite streams must be able to switch to offline processing
mode. To support them the offline session client must continue
passing all necessary callbacks and events similar to regular
camera clients.

Test: Camera CTS
Bug: 135142453
Change-Id: I498681af16ad072e3df01d0279b4cfe76b48f9ec

services/camera/libcameraservice/api2/CameraDeviceClient.cpp

@@ -1930,6 +1930,7 @@ binder::Status CameraDeviceClient::switchToOffline(
     }
 
     std::vector<int32_t> offlineStreamIds(offlineOutputIds.size());
+    KeyedVector<sp<IBinder>, sp<CompositeStream>> offlineCompositeStreamMap;
     for (const auto& streamId : offlineOutputIds) {
         ssize_t index = mConfiguredOutputs.indexOfKey(streamId);
         if (index == NAME_NOT_FOUND) {
@@ -1944,11 +1945,23 @@ binder::Status CameraDeviceClient::switchToOffline(
             sp<Surface> s = new Surface(gbp, false /*controlledByApp*/);
             isCompositeStream = camera3::DepthCompositeStream::isDepthCompositeStream(s) |
                 camera3::HeicCompositeStream::isHeicCompositeStream(s);
+            if (isCompositeStream) {
+                auto compositeIdx = mCompositeStreamMap.indexOfKey(IInterface::asBinder(gbp));
+                if (compositeIdx == NAME_NOT_FOUND) {
+                    ALOGE("%s: Unknown composite stream", __FUNCTION__);
+                    return STATUS_ERROR(CameraService::ERROR_ILLEGAL_ARGUMENT,
+                            "Unknown composite stream");
+                }
+
+                mCompositeStreamMap.valueAt(compositeIdx)->insertCompositeStreamIds(
+                        &offlineStreamIds);
+                offlineCompositeStreamMap.add(mCompositeStreamMap.keyAt(compositeIdx),
+                        mCompositeStreamMap.valueAt(compositeIdx));
+                break;
+            }
         }
 
-        if (isCompositeStream) {
-            // TODO: Add composite specific handling
-        } else {
+        if (!isCompositeStream) {
             offlineStreamIds.push_back(streamId);
         }
     }
@@ -1962,11 +1975,11 @@ binder::Status CameraDeviceClient::switchToOffline(
     }
 
     sp<CameraOfflineSessionClient> offlineClient = new CameraOfflineSessionClient(sCameraService,
-            offlineSession, cameraCb, mClientPackageName, mClientFeatureId, mCameraIdStr,
-            mCameraFacing, mClientPid, mClientUid, mServicePid);
+            offlineSession, offlineCompositeStreamMap, cameraCb, mClientPackageName,
+            mClientFeatureId, mCameraIdStr, mCameraFacing, mClientPid, mClientUid, mServicePid);
     ret = sCameraService->addOfflineClient(mCameraIdStr, offlineClient);
     if (ret == OK) {
-        // TODO: We need to update mStreamMap, mConfiguredOutputs
+        // TODO: We need to update mStreamMap, mConfiguredOutputs, mCompositeStreams
     } else {
         switch(ret) {
             case BAD_VALUE:

services/camera/libcameraservice/api2/CameraOfflineSessionClient.cpp

@@ -59,6 +59,15 @@ binder::Status CameraOfflineSessionClient::disconnect() {
     // client shouldn't be able to call into us anymore
     mClientPid = 0;
 
+    for (size_t i = 0; i < mCompositeStreamMap.size(); i++) {
+        auto ret = mCompositeStreamMap.valueAt(i)->deleteInternalStreams();
+        if (ret != OK) {
+            ALOGE("%s: Failed removing composite stream  %s (%d)", __FUNCTION__,
+                    strerror(-ret), ret);
+        }
+    }
+    mCompositeStreamMap.clear();
+
     return res;
 }
 
@@ -66,8 +75,15 @@ void CameraOfflineSessionClient::notifyError(int32_t errorCode,
         const CaptureResultExtras& resultExtras) {
     // Thread safe. Don't bother locking.
     sp<hardware::camera2::ICameraDeviceCallbacks> remoteCb = getRemoteCallback();
-    // TODO: handle composite streams
-    if ((remoteCb != 0)) {
+    //
+    // Composites can have multiple internal streams. Error notifications coming from such internal
+    // streams may need to remain within camera service.
+    bool skipClientNotification = false;
+    for (size_t i = 0; i < mCompositeStreamMap.size(); i++) {
+        skipClientNotification |= mCompositeStreamMap.valueAt(i)->onError(errorCode, resultExtras);
+    }
+
+    if ((remoteCb != 0) && (!skipClientNotification)) {
         remoteCb->onDeviceError(errorCode, resultExtras);
     }
 }
@@ -136,5 +152,34 @@ status_t CameraOfflineSessionClient::finishCameraOps() {
     return OK;
 }
 
+void CameraOfflineSessionClient::onResultAvailable(const CaptureResult& result) {
+    ATRACE_CALL();
+    ALOGV("%s", __FUNCTION__);
+
+    // Thread-safe. No lock necessary.
+    sp<hardware::camera2::ICameraDeviceCallbacks> remoteCb = mRemoteCallback;
+    if (remoteCb != NULL) {
+        remoteCb->onResultReceived(result.mMetadata, result.mResultExtras,
+                result.mPhysicalMetadatas);
+    }
+
+    for (size_t i = 0; i < mCompositeStreamMap.size(); i++) {
+        mCompositeStreamMap.valueAt(i)->onResultAvailable(result);
+    }
+}
+
+void CameraOfflineSessionClient::notifyShutter(const CaptureResultExtras& resultExtras,
+        nsecs_t timestamp) {
+    // Thread safe. Don't bother locking.
+    sp<hardware::camera2::ICameraDeviceCallbacks> remoteCb = getRemoteCallback();
+    if (remoteCb != 0) {
+        remoteCb->onCaptureStarted(resultExtras, timestamp);
+    }
+
+    for (size_t i = 0; i < mCompositeStreamMap.size(); i++) {
+        mCompositeStreamMap.valueAt(i)->onShutter(resultExtras, timestamp);
+    }
+}
+
 // ----------------------------------------------------------------------------
 }; // namespace android

services/camera/libcameraservice/api2/CameraOfflineSessionClient.h

@@ -20,10 +20,12 @@
 #include <android/hardware/camera2/BnCameraOfflineSession.h>
 #include <android/hardware/camera2/ICameraDeviceCallbacks.h>
 #include "CameraService.h"
+#include "CompositeStream.h"
 
 namespace android {
 
 using android::hardware::camera2::ICameraDeviceCallbacks;
+using camera3::CompositeStream;
 
 // Client for offline session. Note that offline session client does not affect camera service's
 // client arbitration logic. It is camera HAL's decision to decide whether a normal camera
@@ -40,6 +42,7 @@ public:
     CameraOfflineSessionClient(
             const sp<CameraService>& cameraService,
             sp<CameraOfflineSessionBase> session,
+            const KeyedVector<sp<IBinder>, sp<CompositeStream>>& offlineCompositeStreamMap,
             const sp<ICameraDeviceCallbacks>& remoteCallback,
             const String16& clientPackageName,
             const std::unique_ptr<String16>& clientFeatureId,
@@ -50,7 +53,8 @@ public:
                     IInterface::asBinder(remoteCallback),
                     clientPackageName, clientFeatureId,
                     cameraIdStr, cameraFacing, clientPid, clientUid, servicePid),
-            mRemoteCallback(remoteCallback), mOfflineSession(session) {}
+            mRemoteCallback(remoteCallback), mOfflineSession(session),
+            mCompositeStreamMap(offlineCompositeStreamMap) {}
 
     virtual ~CameraOfflineSessionClient() {}
 
@@ -74,6 +78,11 @@ public:
     virtual status_t startCameraOps() override;
     virtual status_t finishCameraOps() override;
 
+    // TODO: Those will be introduced when we implement FilteredListener and the device
+    // callbacks respectively. Just adding for now.
+    void onResultAvailable(const CaptureResult& result);
+    void notifyShutter(const CaptureResultExtras& resultExtras, nsecs_t timestamp);
+
 private:
 
     const sp<hardware::camera2::ICameraDeviceCallbacks>& getRemoteCallback() {
@@ -83,6 +92,9 @@ private:
     sp<hardware::camera2::ICameraDeviceCallbacks> mRemoteCallback;
 
     sp<CameraOfflineSessionBase> mOfflineSession;
+
+    // Offline composite streams
+    KeyedVector<sp<IBinder>, sp<CompositeStream>> mCompositeStreamMap;
 };
 
 } // namespace android

services/camera/libcameraservice/api2/CompositeStream.h

@@ -64,6 +64,10 @@ public:
     virtual status_t insertGbp(SurfaceMap* /*out*/outSurfaceMap,
             Vector<int32_t>* /*out*/outputStreamIds, int32_t* /*out*/currentStreamId) = 0;
 
+    // Attach the internal composite stream ids.
+    virtual status_t insertCompositeStreamIds(
+            std::vector<int32_t>* compositeStreamIds /*out*/) = 0;
+
     // Return composite stream id.
     virtual int getStreamId() = 0;
 

services/camera/libcameraservice/api2/DepthCompositeStream.cpp

@@ -683,6 +683,18 @@ status_t DepthCompositeStream::insertGbp(SurfaceMap* /*out*/outSurfaceMap,
     return NO_ERROR;
 }
 
+status_t DepthCompositeStream::insertCompositeStreamIds(
+        std::vector<int32_t>* compositeStreamIds /*out*/) {
+    if (compositeStreamIds == nullptr) {
+        return BAD_VALUE;
+    }
+
+    compositeStreamIds->push_back(mDepthStreamId);
+    compositeStreamIds->push_back(mBlobStreamId);
+
+    return OK;
+}
+
 void DepthCompositeStream::onResultError(const CaptureResultExtras& resultExtras) {
     // Processing can continue even in case of result errors.
     // At the moment depth composite stream processing relies mainly on static camera

services/camera/libcameraservice/api2/DepthCompositeStream.h

@@ -56,6 +56,7 @@ public:
     status_t configureStream() override;
     status_t insertGbp(SurfaceMap* /*out*/outSurfaceMap, Vector<int32_t>* /*out*/outputStreamIds,
             int32_t* /*out*/currentStreamId) override;
+    status_t insertCompositeStreamIds(std::vector<int32_t>* compositeStreamIds /*out*/) override;
     int getStreamId() override { return mBlobStreamId; }
 
     // CpuConsumer listener implementation

services/camera/libcameraservice/api2/HeicCompositeStream.cpp

@@ -504,6 +504,18 @@ status_t HeicCompositeStream::insertGbp(SurfaceMap* /*out*/outSurfaceMap,
     return NO_ERROR;
 }
 
+status_t HeicCompositeStream::insertCompositeStreamIds(
+        std::vector<int32_t>* compositeStreamIds /*out*/) {
+    if (compositeStreamIds == nullptr) {
+        return BAD_VALUE;
+    }
+
+    compositeStreamIds->push_back(mAppSegmentStreamId);
+    compositeStreamIds->push_back(mMainImageStreamId);
+
+    return OK;
+}
+
 void HeicCompositeStream::onShutter(const CaptureResultExtras& resultExtras, nsecs_t timestamp) {
     Mutex::Autolock l(mMutex);
     if (mErrorState) {

services/camera/libcameraservice/api2/HeicCompositeStream.h

@@ -55,6 +55,8 @@ public:
     status_t insertGbp(SurfaceMap* /*out*/outSurfaceMap, Vector<int32_t>* /*out*/outputStreamIds,
             int32_t* /*out*/currentStreamId) override;
 
+    status_t insertCompositeStreamIds(std::vector<int32_t>* compositeStreamIds /*out*/) override;
+
     void onShutter(const CaptureResultExtras& resultExtras, nsecs_t timestamp) override;
 
     int getStreamId() override { return mMainImageStreamId; }