Camera: handle HAL buffer manager + stream sharing case

Test: CTS MultiViewTest
Bug: 109829698
Change-Id: I98a7438d6bb9c01f95ae69c7556d7ed965629410

services/camera/libcameraservice/device3/Camera3Device.cpp

@@ -1084,6 +1084,11 @@ hardware::Return<void> Camera3Device::requestStreamBuffers(
                             __FUNCTION__, streamId, strerror(-res), res);
                 }
             }
+            for (size_t b = 0; b < numAllocatedBuffers; b++) {
+                camera3_stream_buffer_t& sb = streamBuffers[b];
+                sb.acquire_fence = -1;
+                sb.status = CAMERA3_BUFFER_STATUS_ERROR;
+            }
             returnOutputBuffers(streamBuffers.data(), numAllocatedBuffers, 0);
         }
     }
@@ -1744,7 +1749,8 @@ status_t Camera3Device::createStream(const std::vector<sp<Surface>>& consumers,
     } else if (isShared) {
         newStream = new Camera3SharedOutputStream(mNextStreamId, consumers,
                 width, height, format, consumerUsage, dataSpace, rotation,
-                mTimestampOffset, physicalCameraId, streamSetId);
+                mTimestampOffset, physicalCameraId, streamSetId,
+                mUseHalBufManager);
     } else if (consumers.size() == 0 && hasDeferredConsumer) {
         newStream = new Camera3OutputStream(mNextStreamId,
                 width, height, format, consumerUsage, dataSpace, rotation,
@@ -3030,13 +3036,14 @@ status_t Camera3Device::registerInFlight(uint32_t frameNumber,
         int32_t numBuffers, CaptureResultExtras resultExtras, bool hasInput,
         bool hasAppCallback, nsecs_t maxExpectedDuration,
         std::set<String8>& physicalCameraIds, bool isStillCapture,
-        bool isZslCapture) {
+        bool isZslCapture, const SurfaceMap& outputSurfaces) {
     ATRACE_CALL();
     Mutex::Autolock l(mInFlightLock);
 
     ssize_t res;
     res = mInFlightMap.add(frameNumber, InFlightRequest(numBuffers, resultExtras, hasInput,
-            hasAppCallback, maxExpectedDuration, physicalCameraIds, isStillCapture, isZslCapture));
+            hasAppCallback, maxExpectedDuration, physicalCameraIds, isStillCapture, isZslCapture,
+            outputSurfaces));
     if (res < 0) return res;
 
     if (mInFlightMap.size() == 1) {
@@ -3054,18 +3061,55 @@ status_t Camera3Device::registerInFlight(uint32_t frameNumber,
 
 void Camera3Device::returnOutputBuffers(
         const camera3_stream_buffer_t *outputBuffers, size_t numBuffers,
-        nsecs_t timestamp, bool timestampIncreasing) {
+        nsecs_t timestamp, bool timestampIncreasing,
+        const SurfaceMap& outputSurfaces,
+        const CaptureResultExtras &inResultExtras) {
 
     for (size_t i = 0; i < numBuffers; i++)
     {
-        Camera3Stream *stream = Camera3Stream::cast(outputBuffers[i].stream);
-        status_t res = stream->returnBuffer(outputBuffers[i], timestamp, timestampIncreasing);
+        Camera3StreamInterface *stream = Camera3Stream::cast(outputBuffers[i].stream);
+        int streamId = stream->getId();
+        const auto& it = outputSurfaces.find(streamId);
+        status_t res = OK;
+        if (it != outputSurfaces.end()) {
+            res = stream->returnBuffer(
+                    outputBuffers[i], timestamp, timestampIncreasing, it->second);
+        } else {
+            res = stream->returnBuffer(
+                    outputBuffers[i], timestamp, timestampIncreasing);
+        }
+
         // Note: stream may be deallocated at this point, if this buffer was
         // the last reference to it.
         if (res != OK) {
             ALOGE("Can't return buffer to its stream: %s (%d)",
                 strerror(-res), res);
         }
+
+        // Long processing consumers can cause returnBuffer timeout for shared stream
+        // If that happens, cancel the buffer and send a buffer error to client
+        if (it != outputSurfaces.end() && res == TIMED_OUT &&
+                outputBuffers[i].status == CAMERA3_BUFFER_STATUS_OK) {
+            // cancel the buffer
+            camera3_stream_buffer_t sb = outputBuffers[i];
+            sb.status = CAMERA3_BUFFER_STATUS_ERROR;
+            stream->returnBuffer(sb, /*timestamp*/0, timestampIncreasing);
+
+            // notify client buffer error
+            sp<NotificationListener> listener;
+            {
+                Mutex::Autolock l(mOutputLock);
+                listener = mListener.promote();
+            }
+
+            if (listener != nullptr) {
+                CaptureResultExtras extras = inResultExtras;
+                extras.errorStreamId = streamId;
+                listener->notifyError(
+                        hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_BUFFER,
+                        extras);
+            }
+        }
     }
 }
 
@@ -3124,7 +3168,8 @@ void Camera3Device::removeInFlightRequestIfReadyLocked(int idx) {
         assert(request.requestStatus != OK ||
                request.pendingOutputBuffers.size() == 0);
         returnOutputBuffers(request.pendingOutputBuffers.array(),
-            request.pendingOutputBuffers.size(), 0);
+            request.pendingOutputBuffers.size(), 0, /*timestampIncreasing*/true,
+            request.outputSurfaces, request.resultExtras);
 
         removeInFlightMapEntryLocked(idx);
         ALOGVV("%s: removed frame %d from InFlightMap", __FUNCTION__, frameNumber);
@@ -3148,7 +3193,9 @@ void Camera3Device::flushInflightRequests() {
         for (size_t idx = 0; idx < mInFlightMap.size(); idx++) {
             const InFlightRequest &request = mInFlightMap.valueAt(idx);
             returnOutputBuffers(request.pendingOutputBuffers.array(),
-                request.pendingOutputBuffers.size(), 0);
+                request.pendingOutputBuffers.size(), 0,
+                /*timestampIncreasing*/true, request.outputSurfaces,
+                request.resultExtras);
         }
         mInFlightMap.clear();
         mExpectedInflightDuration = 0;
@@ -3506,7 +3553,8 @@ void Camera3Device::processCaptureResult(const camera3_capture_result *result) {
         } else {
             bool timestampIncreasing = !(request.zslCapture || request.hasInputBuffer);
             returnOutputBuffers(result->output_buffers,
-                result->num_output_buffers, shutterTimestamp, timestampIncreasing);
+                result->num_output_buffers, shutterTimestamp, timestampIncreasing,
+                request.outputSurfaces, request.resultExtras);
         }
 
         if (result->result != NULL && !isPartialResult) {
@@ -3639,6 +3687,9 @@ void Camera3Device::notifyError(const camera3_error_msg_t &msg,
                         // In case of missing result check whether the buffers
                         // returned. If they returned, then remove inflight
                         // request.
+                        // TODO: should we call this for ERROR_CAMERA_REQUEST as well?
+                        //       otherwise we are depending on HAL to send the buffers back after
+                        //       calling notifyError. Not sure if that's in the spec.
                         removeInFlightRequestIfReadyLocked(idx);
                     }
                 } else {
@@ -3714,7 +3765,8 @@ void Camera3Device::notifyShutter(const camera3_shutter_msg_t &msg,
             }
             bool timestampIncreasing = !(r.zslCapture || r.hasInputBuffer);
             returnOutputBuffers(r.pendingOutputBuffers.array(),
-                r.pendingOutputBuffers.size(), r.shutterTimestamp, timestampIncreasing);
+                    r.pendingOutputBuffers.size(), r.shutterTimestamp, timestampIncreasing,
+                    r.outputSurfaces, r.resultExtras);
             r.pendingOutputBuffers.clear();
 
             removeInFlightRequestIfReadyLocked(idx);
@@ -5266,8 +5318,11 @@ status_t Camera3Device::RequestThread::prepareHalRequests() {
         }
         nsecs_t waitDuration = kBaseGetBufferWait + parent->getExpectedInFlightDuration();
 
+        SurfaceMap uniqueSurfaceIdMap;
         for (size_t j = 0; j < captureRequest->mOutputStreams.size(); j++) {
-            sp<Camera3OutputStreamInterface> outputStream = captureRequest->mOutputStreams.editItemAt(j);
+            sp<Camera3OutputStreamInterface> outputStream =
+                    captureRequest->mOutputStreams.editItemAt(j);
+            int streamId = outputStream->getId();
 
             // Prepare video buffers for high speed recording on the first video request.
             if (mPrepareVideoStream && outputStream->isVideoStream()) {
@@ -5286,6 +5341,20 @@ status_t Camera3Device::RequestThread::prepareHalRequests() {
                 }
             }
 
+            std::vector<size_t> uniqueSurfaceIds;
+            res = outputStream->getUniqueSurfaceIds(
+                    captureRequest->mOutputSurfaces[streamId],
+                    &uniqueSurfaceIds);
+            // INVALID_OPERATION is normal output for streams not supporting surfaceIds
+            if (res != OK && res != INVALID_OPERATION) {
+                ALOGE("%s: failed to query stream %d unique surface IDs",
+                        __FUNCTION__, streamId);
+                return res;
+            }
+            if (res == OK) {
+                uniqueSurfaceIdMap.insert({streamId, std::move(uniqueSurfaceIds)});
+            }
+
             if (mUseHalBufManager) {
                 // HAL will request buffer through requestStreamBuffer API
                 camera3_stream_buffer_t& buffer = outputBuffers->editItemAt(j);
@@ -5297,7 +5366,7 @@ status_t Camera3Device::RequestThread::prepareHalRequests() {
             } else {
                 res = outputStream->getBuffer(&outputBuffers->editItemAt(j),
                         waitDuration,
-                        captureRequest->mOutputSurfaces[outputStream->getId()]);
+                        captureRequest->mOutputSurfaces[streamId]);
                 if (res != OK) {
                     // Can't get output buffer from gralloc queue - this could be due to
                     // abandoned queue or other consumer misbehavior, so not a fatal
@@ -5351,7 +5420,9 @@ status_t Camera3Device::RequestThread::prepareHalRequests() {
                 /*hasInput*/halRequest->input_buffer != NULL,
                 hasCallback,
                 calculateMaxExpectedDuration(halRequest->settings),
-                requestedPhysicalCameras, isStillCapture, isZslCapture);
+                requestedPhysicalCameras, isStillCapture, isZslCapture,
+                (mUseHalBufManager) ? uniqueSurfaceIdMap :
+                                      SurfaceMap{});
         ALOGVV("%s: registered in flight requestId = %" PRId32 ", frameNumber = %" PRId64
                ", burstId = %" PRId32 ".",
                 __FUNCTION__,
@@ -5427,7 +5498,7 @@ bool Camera3Device::RequestThread::isOutputSurfacePending(int streamId, size_t s
             if (s.first == streamId) {
                 const auto &it = std::find(s.second.begin(), s.second.end(), surfaceId);
                 if (it != s.second.end()) {
-                  return true;
+                    return true;
                 }
             }
         }
@@ -5438,7 +5509,7 @@ bool Camera3Device::RequestThread::isOutputSurfacePending(int streamId, size_t s
             if (s.first == streamId) {
                 const auto &it = std::find(s.second.begin(), s.second.end(), surfaceId);
                 if (it != s.second.end()) {
-                  return true;
+                    return true;
                 }
             }
         }

services/camera/libcameraservice/device3/Camera3Device.h

@@ -1031,6 +1031,9 @@ class Camera3Device :
         // Indicates a ZSL capture request
         bool zslCapture;
 
+        // What shared surfaces an output should go to
+        SurfaceMap outputSurfaces;
+
         // Default constructor needed by KeyedVector
         InFlightRequest() :
                 shutterTimestamp(0),
@@ -1049,7 +1052,8 @@ class Camera3Device :
         InFlightRequest(int numBuffers, CaptureResultExtras extras, bool hasInput,
                 bool hasAppCallback, nsecs_t maxDuration,
                 const std::set<String8>& physicalCameraIdSet, bool isStillCapture,
-                bool isZslCapture) :
+                bool isZslCapture,
+                const SurfaceMap& outSurfaces = SurfaceMap{}) :
                 shutterTimestamp(0),
                 sensorTimestamp(0),
                 requestStatus(OK),
@@ -1062,7 +1066,8 @@ class Camera3Device :
                 skipResultMetadata(false),
                 physicalCameraIds(physicalCameraIdSet),
                 stillCapture(isStillCapture),
-                zslCapture(isZslCapture) {
+                zslCapture(isZslCapture),
+                outputSurfaces(outSurfaces) {
         }
     };
 
@@ -1079,7 +1084,8 @@ class Camera3Device :
     status_t registerInFlight(uint32_t frameNumber,
             int32_t numBuffers, CaptureResultExtras resultExtras, bool hasInput,
             bool callback, nsecs_t maxExpectedDuration, std::set<String8>& physicalCameraIds,
-            bool isStillCapture, bool isZslCapture);
+            bool isStillCapture, bool isZslCapture,
+            const SurfaceMap& outputSurfaces);
 
     /**
      * Returns the maximum expected time it'll take for all currently in-flight
@@ -1189,7 +1195,11 @@ class Camera3Device :
 
     // helper function to return the output buffers to the streams.
     void returnOutputBuffers(const camera3_stream_buffer_t *outputBuffers,
-            size_t numBuffers, nsecs_t timestamp, bool timestampIncreasing = true);
+            size_t numBuffers, nsecs_t timestamp, bool timestampIncreasing = true,
+            // The following arguments are only meant for surface sharing use case
+            const SurfaceMap& outputSurfaces = SurfaceMap{},
+            // Used to send buffer error callback when failing to return buffer
+            const CaptureResultExtras &resultExtras = CaptureResultExtras{});
 
     // Send a partial capture result.
     void sendPartialCaptureResult(const camera_metadata_t * partialResult,

services/camera/libcameraservice/device3/Camera3DummyStream.cpp

@@ -48,7 +48,7 @@ status_t Camera3DummyStream::getBufferLocked(camera3_stream_buffer *,
 
 status_t Camera3DummyStream::returnBufferLocked(
         const camera3_stream_buffer &,
-        nsecs_t) {
+        nsecs_t, const std::vector<size_t>&) {
     ATRACE_CALL();
     ALOGE("%s: Stream %d: Dummy stream cannot return buffers!", __FUNCTION__, mId);
     return INVALID_OPERATION;
@@ -58,6 +58,7 @@ status_t Camera3DummyStream::returnBufferCheckedLocked(
             const camera3_stream_buffer &,
             nsecs_t,
             bool,
+            const std::vector<size_t>&,
             /*out*/
             sp<Fence>*) {
     ATRACE_CALL();

services/camera/libcameraservice/device3/Camera3DummyStream.h

@@ -87,6 +87,9 @@ class Camera3DummyStream :
      */
     virtual ssize_t getSurfaceId(const sp<Surface> &/*surface*/) { return 0; }
 
+    virtual status_t getUniqueSurfaceIds(const std::vector<size_t>&,
+            /*out*/std::vector<size_t>*) { return INVALID_OPERATION; };
+
     /**
      * Update the stream output surfaces.
      */
@@ -104,6 +107,7 @@ class Camera3DummyStream :
             const camera3_stream_buffer &buffer,
             nsecs_t timestamp,
             bool output,
+            const std::vector<size_t>& surface_ids,
             /*out*/
             sp<Fence> *releaseFenceOut);
 
@@ -128,7 +132,7 @@ class Camera3DummyStream :
             const std::vector<size_t>& surface_ids = std::vector<size_t>());
     virtual status_t returnBufferLocked(
             const camera3_stream_buffer &buffer,
-            nsecs_t timestamp);
+            nsecs_t timestamp, const std::vector<size_t>& surface_ids);
 
     virtual status_t configureQueueLocked();
 

services/camera/libcameraservice/device3/Camera3IOStreamBase.cpp

@@ -219,7 +219,8 @@ status_t Camera3IOStreamBase::returnBufferPreconditionCheckLocked() const {
 status_t Camera3IOStreamBase::returnAnyBufferLocked(
         const camera3_stream_buffer &buffer,
         nsecs_t timestamp,
-        bool output) {
+        bool output,
+        const std::vector<size_t>& surface_ids) {
     status_t res;
 
     // returnBuffer may be called from a raw pointer, not a sp<>, and we'll be
@@ -235,7 +236,7 @@ status_t Camera3IOStreamBase::returnAnyBufferLocked(
     }
 
     sp<Fence> releaseFence;
-    res = returnBufferCheckedLocked(buffer, timestamp, output,
+    res = returnBufferCheckedLocked(buffer, timestamp, output, surface_ids,
                                     &releaseFence);
     // Res may be an error, but we still want to decrement our owned count
     // to enable clean shutdown. So we'll just return the error but otherwise

services/camera/libcameraservice/device3/Camera3IOStreamBase.h

@@ -66,12 +66,14 @@ class Camera3IOStreamBase :
     status_t         returnAnyBufferLocked(
             const camera3_stream_buffer &buffer,
             nsecs_t timestamp,
-            bool output);
+            bool output,
+            const std::vector<size_t>& surface_ids = std::vector<size_t>());
 
     virtual status_t returnBufferCheckedLocked(
             const camera3_stream_buffer &buffer,
             nsecs_t timestamp,
             bool output,
+            const std::vector<size_t>& surface_ids,
             /*out*/
             sp<Fence> *releaseFenceOut) = 0;
 

services/camera/libcameraservice/device3/Camera3InputStream.cpp

@@ -98,6 +98,7 @@ status_t Camera3InputStream::returnBufferCheckedLocked(
             const camera3_stream_buffer &buffer,
             nsecs_t timestamp,
             bool output,
+            const std::vector<size_t>&,
             /*out*/
             sp<Fence> *releaseFenceOut) {
 

services/camera/libcameraservice/device3/Camera3InputStream.h

@@ -62,6 +62,7 @@ class Camera3InputStream : public Camera3IOStreamBase,
             const camera3_stream_buffer &buffer,
             nsecs_t timestamp,
             bool output,
+            const std::vector<size_t>& surface_ids,
             /*out*/
             sp<Fence> *releaseFenceOut);
 

services/camera/libcameraservice/device3/Camera3OutputStream.cpp

@@ -187,16 +187,17 @@ status_t Camera3OutputStream::getBufferLocked(camera3_stream_buffer *buffer,
 }
 
 status_t Camera3OutputStream::queueBufferToConsumer(sp<ANativeWindow>& consumer,
-            ANativeWindowBuffer* buffer, int anwReleaseFence) {
+            ANativeWindowBuffer* buffer, int anwReleaseFence,
+            const std::vector<size_t>&) {
     return consumer->queueBuffer(consumer.get(), buffer, anwReleaseFence);
 }
 
 status_t Camera3OutputStream::returnBufferLocked(
         const camera3_stream_buffer &buffer,
-        nsecs_t timestamp) {
+        nsecs_t timestamp, const std::vector<size_t>& surface_ids) {
     ATRACE_CALL();
 
-    status_t res = returnAnyBufferLocked(buffer, timestamp, /*output*/true);
+    status_t res = returnAnyBufferLocked(buffer, timestamp, /*output*/true, surface_ids);
 
     if (res != OK) {
         return res;
@@ -212,6 +213,7 @@ status_t Camera3OutputStream::returnBufferCheckedLocked(
             const camera3_stream_buffer &buffer,
             nsecs_t timestamp,
             bool output,
+            const std::vector<size_t>& surface_ids,
             /*out*/
             sp<Fence> *releaseFenceOut) {
 
@@ -281,7 +283,7 @@ status_t Camera3OutputStream::returnBufferCheckedLocked(
             return res;
         }
 
-        res = queueBufferToConsumer(currentConsumer, anwBuffer, anwReleaseFence);
+        res = queueBufferToConsumer(currentConsumer, anwBuffer, anwReleaseFence, surface_ids);
         if (res != OK) {
             ALOGE("%s: Stream %d: Error queueing buffer to native window: "
                   "%s (%d)", __FUNCTION__, mId, strerror(-res), res);

services/camera/libcameraservice/device3/Camera3OutputStream.h

@@ -190,6 +190,9 @@ class Camera3OutputStream :
      */
     virtual ssize_t getSurfaceId(const sp<Surface> &/*surface*/) { return 0; }
 
+    virtual status_t getUniqueSurfaceIds(const std::vector<size_t>&,
+            /*out*/std::vector<size_t>*) { return INVALID_OPERATION; };
+
     /**
      * Update the stream output surfaces.
      */
@@ -213,6 +216,7 @@ class Camera3OutputStream :
             const camera3_stream_buffer &buffer,
             nsecs_t timestamp,
             bool output,
+            const std::vector<size_t>& surface_ids,
             /*out*/
             sp<Fence> *releaseFenceOut);
 
@@ -285,10 +289,11 @@ class Camera3OutputStream :
 
     virtual status_t returnBufferLocked(
             const camera3_stream_buffer &buffer,
-            nsecs_t timestamp);
+            nsecs_t timestamp, const std::vector<size_t>& surface_ids);
 
     virtual status_t queueBufferToConsumer(sp<ANativeWindow>& consumer,
-            ANativeWindowBuffer* buffer, int anwReleaseFence);
+            ANativeWindowBuffer* buffer, int anwReleaseFence,
+            const std::vector<size_t>& surface_ids);
 
     virtual status_t configureQueueLocked();
 

services/camera/libcameraservice/device3/Camera3OutputStreamInterface.h

@@ -66,6 +66,18 @@ class Camera3OutputStreamInterface : public virtual Camera3StreamInterface {
      */
     virtual ssize_t getSurfaceId(const sp<Surface> &surface) = 0;
 
+    /**
+     * Query the unique surface IDs of current surfaceIds.
+     * When passing unique surface IDs in returnBuffer(), if the
+     * surfaceId has been removed from the stream, the output corresponding to
+     * the unique surface ID will be ignored and not delivered to client.
+     *
+     * Return INVALID_OPERATION if and only if the stream does not support
+     * surface sharing.
+     */
+    virtual status_t getUniqueSurfaceIds(const std::vector<size_t>& surfaceIds,
+            /*out*/std::vector<size_t>* outUniqueIds) = 0;
+
     /**
      * Update the stream output surfaces.
      */

services/camera/libcameraservice/device3/Camera3SharedOutputStream.cpp

@@ -14,6 +14,10 @@
  * limitations under the License.
  */
 
+#define LOG_TAG "Camera3-SharedOuStrm"
+#define ATRACE_TAG ATRACE_TAG_CAMERA
+//#define LOG_NDEBUG 0
+
 #include "Camera3SharedOutputStream.h"
 
 namespace android {
@@ -28,16 +32,17 @@ Camera3SharedOutputStream::Camera3SharedOutputStream(int id,
         uint64_t consumerUsage, android_dataspace dataSpace,
         camera3_stream_rotation_t rotation,
         nsecs_t timestampOffset, const String8& physicalCameraId,
-        int setId) :
+        int setId, bool useHalBufManager) :
         Camera3OutputStream(id, CAMERA3_STREAM_OUTPUT, width, height,
                             format, dataSpace, rotation, physicalCameraId,
-                            consumerUsage, timestampOffset, setId) {
+                            consumerUsage, timestampOffset, setId),
+        mUseHalBufManager(useHalBufManager) {
     size_t consumerCount = std::min(surfaces.size(), kMaxOutputs);
     if (surfaces.size() > consumerCount) {
         ALOGE("%s: Trying to add more consumers than the maximum ", __func__);
     }
     for (size_t i = 0; i < consumerCount; i++) {
-        mSurfaces[i] = surfaces[i];
+        mSurfaceUniqueIds[i] = std::make_pair(surfaces[i], mNextUniqueSurfaceId++);
     }
 }
 
@@ -48,15 +53,15 @@ Camera3SharedOutputStream::~Camera3SharedOutputStream() {
 status_t Camera3SharedOutputStream::connectStreamSplitterLocked() {
     status_t res = OK;
 
-    mStreamSplitter = new Camera3StreamSplitter();
+    mStreamSplitter = new Camera3StreamSplitter(mUseHalBufManager);
 
     uint64_t usage;
     getEndpointUsage(&usage);
 
     std::unordered_map<size_t, sp<Surface>> initialSurfaces;
     for (size_t i = 0; i < kMaxOutputs; i++) {
-        if (mSurfaces[i] != nullptr) {
-            initialSurfaces.emplace(i, mSurfaces[i]);
+        if (mSurfaceUniqueIds[i].first != nullptr) {
+            initialSurfaces.emplace(i, mSurfaceUniqueIds[i].first);
         }
     }
 
@@ -71,6 +76,31 @@ status_t Camera3SharedOutputStream::connectStreamSplitterLocked() {
     return res;
 }
 
+status_t Camera3SharedOutputStream::attachBufferToSplitterLocked(
+        ANativeWindowBuffer* anb,
+        const std::vector<size_t>& surface_ids) {
+    status_t res = OK;
+
+    // Attach the buffer to the splitter output queues. This could block if
+    // the output queue doesn't have any empty slot. So unlock during the course
+    // of attachBufferToOutputs.
+    sp<Camera3StreamSplitter> splitter = mStreamSplitter;
+    mLock.unlock();
+    res = splitter->attachBufferToOutputs(anb, surface_ids);
+    mLock.lock();
+    if (res != OK) {
+        ALOGE("%s: Stream %d: Cannot attach stream splitter buffer to outputs: %s (%d)",
+                __FUNCTION__, mId, strerror(-res), res);
+        // Only transition to STATE_ABANDONED from STATE_CONFIGURED. (If it is STATE_PREPARING,
+        // let prepareNextBuffer handle the error.)
+        if (res == NO_INIT && mState == STATE_CONFIGURED) {
+            mState = STATE_ABANDONED;
+        }
+    }
+    return res;
+}
+
+
 status_t Camera3SharedOutputStream::notifyBufferReleased(ANativeWindowBuffer *anwBuffer) {
     Mutex::Autolock l(mLock);
     status_t res = OK;
@@ -89,7 +119,7 @@ bool Camera3SharedOutputStream::isConsumerConfigurationDeferred(size_t surface_i
         return true;
     }
 
-    return (mSurfaces[surface_id] == nullptr);
+    return (mSurfaceUniqueIds[surface_id].first == nullptr);
 }
 
 status_t Camera3SharedOutputStream::setConsumers(const std::vector<sp<Surface>>& surfaces) {
@@ -112,7 +142,7 @@ status_t Camera3SharedOutputStream::setConsumers(const std::vector<sp<Surface>>&
             return NO_MEMORY;
         }
 
-        mSurfaces[id] = surface;
+        mSurfaceUniqueIds[id] = std::make_pair(surface, mNextUniqueSurfaceId++);
 
         // Only call addOutput if the splitter has been connected.
         if (mStreamSplitter != nullptr) {
@@ -128,7 +158,7 @@ status_t Camera3SharedOutputStream::setConsumers(const std::vector<sp<Surface>>&
 }
 
 status_t Camera3SharedOutputStream::getBufferLocked(camera3_stream_buffer *buffer,
-        const std::vector<size_t>& surface_ids) {
+        const std::vector<size_t>& surfaceIds) {
     ANativeWindowBuffer* anb;
     int fenceFd = -1;
 
@@ -138,27 +168,11 @@ status_t Camera3SharedOutputStream::getBufferLocked(camera3_stream_buffer *buffe
         return res;
     }
 
-    // TODO: need to refactor this to support requestStreamBuffers API
-    // Need to wait until processCaptureResult to decide the source buffer
-    // to attach to output...
-
-    // Attach the buffer to the splitter output queues. This could block if
-    // the output queue doesn't have any empty slot. So unlock during the course
-    // of attachBufferToOutputs.
-    sp<Camera3StreamSplitter> splitter = mStreamSplitter;
-    mLock.unlock();
-    res = splitter->attachBufferToOutputs(anb, surface_ids);
-    mLock.lock();
-    if (res != OK) {
-        ALOGE("%s: Stream %d: Cannot attach stream splitter buffer to outputs: %s (%d)",
-                __FUNCTION__, mId, strerror(-res), res);
-        // Only transition to STATE_ABANDONED from STATE_CONFIGURED. (If it is STATE_PREPARING,
-        // let prepareNextBuffer handle the error.)
-        if (res == NO_INIT && mState == STATE_CONFIGURED) {
-            mState = STATE_ABANDONED;
+    if (!mUseHalBufManager) {
+        res = attachBufferToSplitterLocked(anb, surfaceIds);
+        if (res != OK) {
+            return res;
         }
-
-        return res;
     }
 
     /**
@@ -172,8 +186,38 @@ status_t Camera3SharedOutputStream::getBufferLocked(camera3_stream_buffer *buffe
 }
 
 status_t Camera3SharedOutputStream::queueBufferToConsumer(sp<ANativeWindow>& consumer,
-            ANativeWindowBuffer* buffer, int anwReleaseFence) {
-    status_t res = consumer->queueBuffer(consumer.get(), buffer, anwReleaseFence);
+            ANativeWindowBuffer* buffer, int anwReleaseFence,
+            const std::vector<size_t>& uniqueSurfaceIds) {
+    status_t res = OK;
+    if (mUseHalBufManager) {
+        if (uniqueSurfaceIds.size() == 0) {
+            ALOGE("%s: uniqueSurfaceIds must not be empty!", __FUNCTION__);
+            return BAD_VALUE;
+        }
+        Mutex::Autolock l(mLock);
+        std::vector<size_t> surfaceIds;
+        for (const auto& uniqueId : uniqueSurfaceIds) {
+            bool uniqueIdFound = false;
+            for (size_t i = 0; i < kMaxOutputs; i++) {
+                if (mSurfaceUniqueIds[i].second == uniqueId) {
+                    surfaceIds.push_back(i);
+                    uniqueIdFound = true;
+                    break;
+                }
+            }
+            if (!uniqueIdFound) {
+                ALOGV("%s: unknown unique surface ID %zu for stream %d: "
+                        "output might have been removed.",
+                        __FUNCTION__, uniqueId, mId);
+            }
+        }
+        res = attachBufferToSplitterLocked(buffer, surfaceIds);
+        if (res != OK) {
+            return res;
+        }
+    }
+
+    res = consumer->queueBuffer(consumer.get(), buffer, anwReleaseFence);
 
     // After queuing buffer to the internal consumer queue, check whether the buffer is
     // successfully queued to the output queues.
@@ -232,8 +276,8 @@ status_t Camera3SharedOutputStream::getEndpointUsage(uint64_t *usage) const {
         *usage = getPresetConsumerUsage();
 
         for (size_t id = 0; id < kMaxOutputs; id++) {
-            if (mSurfaces[id] != nullptr) {
-                res = getEndpointUsageForSurface(&u, mSurfaces[id]);
+            if (mSurfaceUniqueIds[id].first != nullptr) {
+                res = getEndpointUsageForSurface(&u, mSurfaceUniqueIds[id].first);
                 *usage |= u;
             }
         }
@@ -249,7 +293,7 @@ status_t Camera3SharedOutputStream::getEndpointUsage(uint64_t *usage) const {
 ssize_t Camera3SharedOutputStream::getNextSurfaceIdLocked() {
     ssize_t id = -1;
     for (size_t i = 0; i < kMaxOutputs; i++) {
-        if (mSurfaces[i] == nullptr) {
+        if (mSurfaceUniqueIds[i].first == nullptr) {
             id = i;
             break;
         }
@@ -262,7 +306,7 @@ ssize_t Camera3SharedOutputStream::getSurfaceId(const sp<Surface> &surface) {
     Mutex::Autolock l(mLock);
     ssize_t id = -1;
     for (size_t i = 0; i < kMaxOutputs; i++) {
-        if (mSurfaces[i] == surface) {
+        if (mSurfaceUniqueIds[i].first == surface) {
             id = i;
             break;
         }
@@ -271,6 +315,26 @@ ssize_t Camera3SharedOutputStream::getSurfaceId(const sp<Surface> &surface) {
     return id;
 }
 
+status_t Camera3SharedOutputStream::getUniqueSurfaceIds(
+        const std::vector<size_t>& surfaceIds,
+        /*out*/std::vector<size_t>* outUniqueIds) {
+    Mutex::Autolock l(mLock);
+    if (outUniqueIds == nullptr || surfaceIds.size() > kMaxOutputs) {
+        return BAD_VALUE;
+    }
+
+    outUniqueIds->clear();
+    outUniqueIds->reserve(surfaceIds.size());
+
+    for (const auto& surfaceId : surfaceIds) {
+        if (surfaceId >= kMaxOutputs) {
+            return BAD_VALUE;
+        }
+        outUniqueIds->push_back(mSurfaceUniqueIds[surfaceId].second);
+    }
+    return OK;
+}
+
 status_t Camera3SharedOutputStream::revertPartialUpdateLocked(
         const KeyedVector<sp<Surface>, size_t> &removedSurfaces,
         const KeyedVector<sp<Surface>, size_t> &attachedSurfaces) {
@@ -284,7 +348,7 @@ status_t Camera3SharedOutputStream::revertPartialUpdateLocked(
                 return UNKNOWN_ERROR;
             }
         }
-        mSurfaces[index] = nullptr;
+        mSurfaceUniqueIds[index] = std::make_pair(nullptr, mNextUniqueSurfaceId++);
     }
 
     for (size_t i = 0; i < removedSurfaces.size(); i++) {
@@ -295,7 +359,8 @@ status_t Camera3SharedOutputStream::revertPartialUpdateLocked(
                 return UNKNOWN_ERROR;
             }
         }
-        mSurfaces[index] = removedSurfaces.keyAt(i);
+        mSurfaceUniqueIds[index] = std::make_pair(
+                removedSurfaces.keyAt(i), mNextUniqueSurfaceId++);
     }
 
     return ret;
@@ -347,8 +412,8 @@ status_t Camera3SharedOutputStream::updateStream(const std::vector<sp<Surface>>
 
             }
         }
-        mSurfaces[it] = nullptr;
-        removedSurfaces.add(mSurfaces[it], it);
+        removedSurfaces.add(mSurfaceUniqueIds[it].first, it);
+        mSurfaceUniqueIds[it] = std::make_pair(nullptr, mNextUniqueSurfaceId++);
     }
 
     //Next add the new outputs
@@ -373,7 +438,7 @@ status_t Camera3SharedOutputStream::updateStream(const std::vector<sp<Surface>>
                 return ret;
             }
         }
-        mSurfaces[surfaceId] = it;
+        mSurfaceUniqueIds[surfaceId] = std::make_pair(it, mNextUniqueSurfaceId++);
         outputMap->add(it, surfaceId);
     }
 

services/camera/libcameraservice/device3/Camera3SharedOutputStream.h

@@ -17,6 +17,7 @@
 #ifndef ANDROID_SERVERS_CAMERA3_SHARED_OUTPUT_STREAM_H
 #define ANDROID_SERVERS_CAMERA3_SHARED_OUTPUT_STREAM_H
 
+#include <array>
 #include "Camera3StreamSplitter.h"
 #include "Camera3OutputStream.h"
 
@@ -37,7 +38,8 @@ public:
             uint64_t consumerUsage, android_dataspace dataSpace,
             camera3_stream_rotation_t rotation, nsecs_t timestampOffset,
             const String8& physicalCameraId,
-            int setId = CAMERA3_STREAM_SET_ID_INVALID);
+            int setId = CAMERA3_STREAM_SET_ID_INVALID,
+            bool useHalBufManager = false);
 
     virtual ~Camera3SharedOutputStream();
 
@@ -49,6 +51,15 @@ public:
 
     virtual ssize_t getSurfaceId(const sp<Surface> &surface);
 
+    /**
+     * Query the unique surface IDs of current surfaceIds.
+     * When passing unique surface IDs in returnBuffer(), if the
+     * surfaceId has been removed from the stream, the output corresponding to
+     * the unique surface ID will be ignored and not delivered to client.
+     */
+    virtual status_t getUniqueSurfaceIds(const std::vector<size_t>& surfaceIds,
+            /*out*/std::vector<size_t>* outUniqueIds) override;
+
     virtual status_t updateStream(const std::vector<sp<Surface>> &outputSurfaces,
             const std::vector<OutputStreamInfo> &outputInfo,
             const std::vector<size_t> &removedSurfaceIds,
@@ -58,8 +69,17 @@ private:
 
     static const size_t kMaxOutputs = 4;
 
-    // Map surfaceId -> output surfaces
-    sp<Surface> mSurfaces[kMaxOutputs];
+    // Whether HAL is in control for buffer management. Surface sharing behavior
+    // depends on this flag.
+    const bool mUseHalBufManager;
+
+    // Pair of an output Surface and its unique ID
+    typedef std::pair<sp<Surface>, size_t> SurfaceUniqueId;
+
+    // Map surfaceId -> (output surface, unique surface ID)
+    std::array<SurfaceUniqueId, kMaxOutputs> mSurfaceUniqueIds;
+
+    size_t mNextUniqueSurfaceId = 0;
 
     ssize_t getNextSurfaceIdLocked();
 
@@ -77,6 +97,16 @@ private:
      */
     status_t connectStreamSplitterLocked();
 
+    /**
+     * Attach the output buffer to stream splitter.
+     * When camera service is doing buffer management, this method will be called
+     * before the buffer is handed out to HAL in request thread.
+     * When HAL is doing buffer management, this method will be called when
+     * the buffer is returned from HAL in hwbinder callback thread.
+     */
+    status_t attachBufferToSplitterLocked(ANativeWindowBuffer* anb,
+            const std::vector<size_t>& surface_ids);
+
     /**
      * Internal Camera3Stream interface
      */
@@ -84,7 +114,8 @@ private:
             const std::vector<size_t>& surface_ids);
 
     virtual status_t queueBufferToConsumer(sp<ANativeWindow>& consumer,
-            ANativeWindowBuffer* buffer, int anwReleaseFence);
+            ANativeWindowBuffer* buffer, int anwReleaseFence,
+            const std::vector<size_t>& uniqueSurfaceIds);
 
     virtual status_t configureQueueLocked();
 

services/camera/libcameraservice/device3/Camera3Stream.cpp

@@ -655,7 +655,8 @@ void Camera3Stream::removeOutstandingBuffer(const camera3_stream_buffer &buffer)
 }
 
 status_t Camera3Stream::returnBuffer(const camera3_stream_buffer &buffer,
-        nsecs_t timestamp, bool timestampIncreasing) {
+        nsecs_t timestamp, bool timestampIncreasing,
+         const std::vector<size_t>& surface_ids) {
     ATRACE_CALL();
     Mutex::Autolock l(mLock);
 
@@ -684,7 +685,7 @@ status_t Camera3Stream::returnBuffer(const camera3_stream_buffer &buffer,
      *
      * Do this for getBuffer as well.
      */
-    status_t res = returnBufferLocked(b, timestamp);
+    status_t res = returnBufferLocked(b, timestamp, surface_ids);
     if (res == OK) {
         fireBufferListenersLocked(b, /*acquired*/false, /*output*/true);
     }
@@ -843,7 +844,7 @@ status_t Camera3Stream::getBufferLocked(camera3_stream_buffer *,
     return INVALID_OPERATION;
 }
 status_t Camera3Stream::returnBufferLocked(const camera3_stream_buffer &,
-                                           nsecs_t) {
+                                           nsecs_t, const std::vector<size_t>&) {
     ALOGE("%s: This type of stream does not support output", __FUNCTION__);
     return INVALID_OPERATION;
 }

services/camera/libcameraservice/device3/Camera3Stream.h

@@ -322,11 +322,17 @@ class Camera3Stream :
     /**
      * Return a buffer to the stream after use by the HAL.
      *
+     * Multiple surfaces could share the same HAL stream, but a request may
+     * be only for a subset of surfaces. In this case, the
+     * Camera3StreamInterface object needs the surface ID information to attach
+     * buffers for those surfaces.
+     *
      * This method may only be called for buffers provided by getBuffer().
      * For bidirectional streams, this method applies to the output-side buffers
      */
     status_t         returnBuffer(const camera3_stream_buffer &buffer,
-            nsecs_t timestamp, bool timestampIncreasing);
+            nsecs_t timestamp, bool timestampIncreasing,
+            const std::vector<size_t>& surface_ids = std::vector<size_t>());
 
     /**
      * Fill in the camera3_stream_buffer with the next valid buffer for this
@@ -478,7 +484,8 @@ class Camera3Stream :
     virtual status_t getBufferLocked(camera3_stream_buffer *buffer,
             const std::vector<size_t>& surface_ids = std::vector<size_t>());
     virtual status_t returnBufferLocked(const camera3_stream_buffer &buffer,
-            nsecs_t timestamp);
+            nsecs_t timestamp,
+            const std::vector<size_t>& surface_ids = std::vector<size_t>());
     virtual status_t getInputBufferLocked(camera3_stream_buffer *buffer);
     virtual status_t returnInputBufferLocked(
             const camera3_stream_buffer &buffer);

services/camera/libcameraservice/device3/Camera3StreamInterface.h

@@ -248,11 +248,18 @@ class Camera3StreamInterface : public virtual RefBase {
     /**
      * Return a buffer to the stream after use by the HAL.
      *
+     * Multiple surfaces could share the same HAL stream, but a request may
+     * be only for a subset of surfaces. In this case, the
+     * Camera3StreamInterface object needs the surface ID information to attach
+     * buffers for those surfaces. For the case of single surface for a HAL
+     * stream, surface_ids parameter has no effect.
+     *
      * This method may only be called for buffers provided by getBuffer().
      * For bidirectional streams, this method applies to the output-side buffers
      */
     virtual status_t returnBuffer(const camera3_stream_buffer &buffer,
-            nsecs_t timestamp, bool timestampIncreasing = true) = 0;
+            nsecs_t timestamp, bool timestampIncreasing = true,
+            const std::vector<size_t>& surface_ids = std::vector<size_t>()) = 0;
 
     /**
      * Fill in the camera3_stream_buffer with the next valid buffer for this

services/camera/libcameraservice/device3/Camera3StreamSplitter.cpp

@@ -150,6 +150,8 @@ void Camera3StreamSplitter::disconnect() {
     SP_LOGV("%s: Disconnected", __FUNCTION__);
 }
 
+Camera3StreamSplitter::Camera3StreamSplitter(bool useHalBufManager) :
+        mUseHalBufManager(useHalBufManager) {}
 
 Camera3StreamSplitter::~Camera3StreamSplitter() {
     disconnect();
@@ -237,7 +239,9 @@ status_t Camera3StreamSplitter::addOutputLocked(size_t surfaceId, const sp<Surfa
     uint64_t usage = 0;
     res = native_window_get_consumer_usage(static_cast<ANativeWindow*>(outputQueue.get()), &usage);
     if (!(usage & (GRALLOC_USAGE_HW_COMPOSER | GRALLOC_USAGE_HW_TEXTURE))) {
-        outputQueue->setDequeueTimeout(kDequeueBufferTimeout);
+        nsecs_t timeout = mUseHalBufManager ?
+                kHalBufMgrDequeueBufferTimeout : kNormalDequeueBufferTimeout;
+        outputQueue->setDequeueTimeout(timeout);
     }
 
     res = gbp->allowAllocation(false);
@@ -430,8 +434,9 @@ status_t Camera3StreamSplitter::attachBufferToOutputs(ANativeWindowBuffer* anb,
         res = gbp->attachBuffer(&slot, gb);
         mMutex.lock();
         if (res != OK) {
-            SP_LOGE("%s: Cannot acquireBuffer from GraphicBufferProducer %p: %s (%d)",
+            SP_LOGE("%s: Cannot attachBuffer from GraphicBufferProducer %p: %s (%d)",
                     __FUNCTION__, gbp.get(), strerror(-res), res);
+            // TODO: might need to detach/cleanup the already attached buffers before return?
             return res;
         }
         if ((slot < 0) || (slot > BufferQueue::NUM_BUFFER_SLOTS)) {

services/camera/libcameraservice/device3/Camera3StreamSplitter.h

@@ -49,7 +49,7 @@ class Camera3StreamSplitter : public BnConsumerListener {
 public:
 
     // Constructor
-    Camera3StreamSplitter() = default;
+    Camera3StreamSplitter(bool useHalBufManager = false);
 
     // Connect to the stream splitter by creating buffer queue and connecting it
     // with output surfaces.
@@ -226,7 +226,10 @@ private:
     android::PixelFormat mFormat = android::PIXEL_FORMAT_NONE;
     uint64_t mProducerUsage = 0;
 
-    static const nsecs_t kDequeueBufferTimeout   = s2ns(1); // 1 sec
+    // The attachBuffer call will happen on different thread according to mUseHalBufManager and have
+    // different timing constraint.
+    static const nsecs_t kNormalDequeueBufferTimeout    = s2ns(1);  // 1 sec
+    static const nsecs_t kHalBufMgrDequeueBufferTimeout = ms2ns(1); // 1 msec
 
     Mutex mMutex;
 
@@ -270,6 +273,8 @@ private:
     std::atomic<status_t> mOnFrameAvailableRes{0};
 
     String8 mConsumerName;
+
+    const bool mUseHalBufManager;
 };
 
 } // namespace android