diff --git a/services/camera/libcameraservice/device3/Camera3Device.cpp b/services/camera/libcameraservice/device3/Camera3Device.cpp index fa4e036b15..f0d94750ee 100644 --- a/services/camera/libcameraservice/device3/Camera3Device.cpp +++ b/services/camera/libcameraservice/device3/Camera3Device.cpp @@ -1084,6 +1084,11 @@ hardware::Return Camera3Device::requestStreamBuffers( __FUNCTION__, streamId, strerror(-res), res); } } + for (size_t b = 0; b < numAllocatedBuffers; b++) { + camera3_stream_buffer_t& sb = streamBuffers[b]; + sb.acquire_fence = -1; + sb.status = CAMERA3_BUFFER_STATUS_ERROR; + } returnOutputBuffers(streamBuffers.data(), numAllocatedBuffers, 0); } } @@ -1744,7 +1749,8 @@ status_t Camera3Device::createStream(const std::vector>& consumers, } else if (isShared) { newStream = new Camera3SharedOutputStream(mNextStreamId, consumers, width, height, format, consumerUsage, dataSpace, rotation, - mTimestampOffset, physicalCameraId, streamSetId); + mTimestampOffset, physicalCameraId, streamSetId, + mUseHalBufManager); } else if (consumers.size() == 0 && hasDeferredConsumer) { newStream = new Camera3OutputStream(mNextStreamId, width, height, format, consumerUsage, dataSpace, rotation, @@ -3030,13 +3036,14 @@ status_t Camera3Device::registerInFlight(uint32_t frameNumber, int32_t numBuffers, CaptureResultExtras resultExtras, bool hasInput, bool hasAppCallback, nsecs_t maxExpectedDuration, std::set& physicalCameraIds, bool isStillCapture, - bool isZslCapture) { + bool isZslCapture, const SurfaceMap& outputSurfaces) { ATRACE_CALL(); Mutex::Autolock l(mInFlightLock); ssize_t res; res = mInFlightMap.add(frameNumber, InFlightRequest(numBuffers, resultExtras, hasInput, - hasAppCallback, maxExpectedDuration, physicalCameraIds, isStillCapture, isZslCapture)); + hasAppCallback, maxExpectedDuration, physicalCameraIds, isStillCapture, isZslCapture, + outputSurfaces)); if (res < 0) return res; if (mInFlightMap.size() == 1) { @@ -3054,18 +3061,55 @@ status_t Camera3Device::registerInFlight(uint32_t frameNumber, void Camera3Device::returnOutputBuffers( const camera3_stream_buffer_t *outputBuffers, size_t numBuffers, - nsecs_t timestamp, bool timestampIncreasing) { + nsecs_t timestamp, bool timestampIncreasing, + const SurfaceMap& outputSurfaces, + const CaptureResultExtras &inResultExtras) { for (size_t i = 0; i < numBuffers; i++) { - Camera3Stream *stream = Camera3Stream::cast(outputBuffers[i].stream); - status_t res = stream->returnBuffer(outputBuffers[i], timestamp, timestampIncreasing); + Camera3StreamInterface *stream = Camera3Stream::cast(outputBuffers[i].stream); + int streamId = stream->getId(); + const auto& it = outputSurfaces.find(streamId); + status_t res = OK; + if (it != outputSurfaces.end()) { + res = stream->returnBuffer( + outputBuffers[i], timestamp, timestampIncreasing, it->second); + } else { + res = stream->returnBuffer( + outputBuffers[i], timestamp, timestampIncreasing); + } + // Note: stream may be deallocated at this point, if this buffer was // the last reference to it. if (res != OK) { ALOGE("Can't return buffer to its stream: %s (%d)", strerror(-res), res); } + + // Long processing consumers can cause returnBuffer timeout for shared stream + // If that happens, cancel the buffer and send a buffer error to client + if (it != outputSurfaces.end() && res == TIMED_OUT && + outputBuffers[i].status == CAMERA3_BUFFER_STATUS_OK) { + // cancel the buffer + camera3_stream_buffer_t sb = outputBuffers[i]; + sb.status = CAMERA3_BUFFER_STATUS_ERROR; + stream->returnBuffer(sb, /*timestamp*/0, timestampIncreasing); + + // notify client buffer error + sp listener; + { + Mutex::Autolock l(mOutputLock); + listener = mListener.promote(); + } + + if (listener != nullptr) { + CaptureResultExtras extras = inResultExtras; + extras.errorStreamId = streamId; + listener->notifyError( + hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_BUFFER, + extras); + } + } } } @@ -3124,7 +3168,8 @@ void Camera3Device::removeInFlightRequestIfReadyLocked(int idx) { assert(request.requestStatus != OK || request.pendingOutputBuffers.size() == 0); returnOutputBuffers(request.pendingOutputBuffers.array(), - request.pendingOutputBuffers.size(), 0); + request.pendingOutputBuffers.size(), 0, /*timestampIncreasing*/true, + request.outputSurfaces, request.resultExtras); removeInFlightMapEntryLocked(idx); ALOGVV("%s: removed frame %d from InFlightMap", __FUNCTION__, frameNumber); @@ -3148,7 +3193,9 @@ void Camera3Device::flushInflightRequests() { for (size_t idx = 0; idx < mInFlightMap.size(); idx++) { const InFlightRequest &request = mInFlightMap.valueAt(idx); returnOutputBuffers(request.pendingOutputBuffers.array(), - request.pendingOutputBuffers.size(), 0); + request.pendingOutputBuffers.size(), 0, + /*timestampIncreasing*/true, request.outputSurfaces, + request.resultExtras); } mInFlightMap.clear(); mExpectedInflightDuration = 0; @@ -3506,7 +3553,8 @@ void Camera3Device::processCaptureResult(const camera3_capture_result *result) { } else { bool timestampIncreasing = !(request.zslCapture || request.hasInputBuffer); returnOutputBuffers(result->output_buffers, - result->num_output_buffers, shutterTimestamp, timestampIncreasing); + result->num_output_buffers, shutterTimestamp, timestampIncreasing, + request.outputSurfaces, request.resultExtras); } if (result->result != NULL && !isPartialResult) { @@ -3639,6 +3687,9 @@ void Camera3Device::notifyError(const camera3_error_msg_t &msg, // In case of missing result check whether the buffers // returned. If they returned, then remove inflight // request. + // TODO: should we call this for ERROR_CAMERA_REQUEST as well? + // otherwise we are depending on HAL to send the buffers back after + // calling notifyError. Not sure if that's in the spec. removeInFlightRequestIfReadyLocked(idx); } } else { @@ -3714,7 +3765,8 @@ void Camera3Device::notifyShutter(const camera3_shutter_msg_t &msg, } bool timestampIncreasing = !(r.zslCapture || r.hasInputBuffer); returnOutputBuffers(r.pendingOutputBuffers.array(), - r.pendingOutputBuffers.size(), r.shutterTimestamp, timestampIncreasing); + r.pendingOutputBuffers.size(), r.shutterTimestamp, timestampIncreasing, + r.outputSurfaces, r.resultExtras); r.pendingOutputBuffers.clear(); removeInFlightRequestIfReadyLocked(idx); @@ -5266,8 +5318,11 @@ status_t Camera3Device::RequestThread::prepareHalRequests() { } nsecs_t waitDuration = kBaseGetBufferWait + parent->getExpectedInFlightDuration(); + SurfaceMap uniqueSurfaceIdMap; for (size_t j = 0; j < captureRequest->mOutputStreams.size(); j++) { - sp outputStream = captureRequest->mOutputStreams.editItemAt(j); + sp outputStream = + captureRequest->mOutputStreams.editItemAt(j); + int streamId = outputStream->getId(); // Prepare video buffers for high speed recording on the first video request. if (mPrepareVideoStream && outputStream->isVideoStream()) { @@ -5286,6 +5341,20 @@ status_t Camera3Device::RequestThread::prepareHalRequests() { } } + std::vector uniqueSurfaceIds; + res = outputStream->getUniqueSurfaceIds( + captureRequest->mOutputSurfaces[streamId], + &uniqueSurfaceIds); + // INVALID_OPERATION is normal output for streams not supporting surfaceIds + if (res != OK && res != INVALID_OPERATION) { + ALOGE("%s: failed to query stream %d unique surface IDs", + __FUNCTION__, streamId); + return res; + } + if (res == OK) { + uniqueSurfaceIdMap.insert({streamId, std::move(uniqueSurfaceIds)}); + } + if (mUseHalBufManager) { // HAL will request buffer through requestStreamBuffer API camera3_stream_buffer_t& buffer = outputBuffers->editItemAt(j); @@ -5297,7 +5366,7 @@ status_t Camera3Device::RequestThread::prepareHalRequests() { } else { res = outputStream->getBuffer(&outputBuffers->editItemAt(j), waitDuration, - captureRequest->mOutputSurfaces[outputStream->getId()]); + captureRequest->mOutputSurfaces[streamId]); if (res != OK) { // Can't get output buffer from gralloc queue - this could be due to // abandoned queue or other consumer misbehavior, so not a fatal @@ -5351,7 +5420,9 @@ status_t Camera3Device::RequestThread::prepareHalRequests() { /*hasInput*/halRequest->input_buffer != NULL, hasCallback, calculateMaxExpectedDuration(halRequest->settings), - requestedPhysicalCameras, isStillCapture, isZslCapture); + requestedPhysicalCameras, isStillCapture, isZslCapture, + (mUseHalBufManager) ? uniqueSurfaceIdMap : + SurfaceMap{}); ALOGVV("%s: registered in flight requestId = %" PRId32 ", frameNumber = %" PRId64 ", burstId = %" PRId32 ".", __FUNCTION__, @@ -5427,7 +5498,7 @@ bool Camera3Device::RequestThread::isOutputSurfacePending(int streamId, size_t s if (s.first == streamId) { const auto &it = std::find(s.second.begin(), s.second.end(), surfaceId); if (it != s.second.end()) { - return true; + return true; } } } @@ -5438,7 +5509,7 @@ bool Camera3Device::RequestThread::isOutputSurfacePending(int streamId, size_t s if (s.first == streamId) { const auto &it = std::find(s.second.begin(), s.second.end(), surfaceId); if (it != s.second.end()) { - return true; + return true; } } } diff --git a/services/camera/libcameraservice/device3/Camera3Device.h b/services/camera/libcameraservice/device3/Camera3Device.h index 4f5be6bd7a..3fe48fbc25 100644 --- a/services/camera/libcameraservice/device3/Camera3Device.h +++ b/services/camera/libcameraservice/device3/Camera3Device.h @@ -1031,6 +1031,9 @@ class Camera3Device : // Indicates a ZSL capture request bool zslCapture; + // What shared surfaces an output should go to + SurfaceMap outputSurfaces; + // Default constructor needed by KeyedVector InFlightRequest() : shutterTimestamp(0), @@ -1049,7 +1052,8 @@ class Camera3Device : InFlightRequest(int numBuffers, CaptureResultExtras extras, bool hasInput, bool hasAppCallback, nsecs_t maxDuration, const std::set& physicalCameraIdSet, bool isStillCapture, - bool isZslCapture) : + bool isZslCapture, + const SurfaceMap& outSurfaces = SurfaceMap{}) : shutterTimestamp(0), sensorTimestamp(0), requestStatus(OK), @@ -1062,7 +1066,8 @@ class Camera3Device : skipResultMetadata(false), physicalCameraIds(physicalCameraIdSet), stillCapture(isStillCapture), - zslCapture(isZslCapture) { + zslCapture(isZslCapture), + outputSurfaces(outSurfaces) { } }; @@ -1079,7 +1084,8 @@ class Camera3Device : status_t registerInFlight(uint32_t frameNumber, int32_t numBuffers, CaptureResultExtras resultExtras, bool hasInput, bool callback, nsecs_t maxExpectedDuration, std::set& physicalCameraIds, - bool isStillCapture, bool isZslCapture); + bool isStillCapture, bool isZslCapture, + const SurfaceMap& outputSurfaces); /** * Returns the maximum expected time it'll take for all currently in-flight @@ -1189,7 +1195,11 @@ class Camera3Device : // helper function to return the output buffers to the streams. void returnOutputBuffers(const camera3_stream_buffer_t *outputBuffers, - size_t numBuffers, nsecs_t timestamp, bool timestampIncreasing = true); + size_t numBuffers, nsecs_t timestamp, bool timestampIncreasing = true, + // The following arguments are only meant for surface sharing use case + const SurfaceMap& outputSurfaces = SurfaceMap{}, + // Used to send buffer error callback when failing to return buffer + const CaptureResultExtras &resultExtras = CaptureResultExtras{}); // Send a partial capture result. void sendPartialCaptureResult(const camera_metadata_t * partialResult, diff --git a/services/camera/libcameraservice/device3/Camera3DummyStream.cpp b/services/camera/libcameraservice/device3/Camera3DummyStream.cpp index fb1ff779d2..b637160306 100644 --- a/services/camera/libcameraservice/device3/Camera3DummyStream.cpp +++ b/services/camera/libcameraservice/device3/Camera3DummyStream.cpp @@ -48,7 +48,7 @@ status_t Camera3DummyStream::getBufferLocked(camera3_stream_buffer *, status_t Camera3DummyStream::returnBufferLocked( const camera3_stream_buffer &, - nsecs_t) { + nsecs_t, const std::vector&) { ATRACE_CALL(); ALOGE("%s: Stream %d: Dummy stream cannot return buffers!", __FUNCTION__, mId); return INVALID_OPERATION; @@ -58,6 +58,7 @@ status_t Camera3DummyStream::returnBufferCheckedLocked( const camera3_stream_buffer &, nsecs_t, bool, + const std::vector&, /*out*/ sp*) { ATRACE_CALL(); diff --git a/services/camera/libcameraservice/device3/Camera3DummyStream.h b/services/camera/libcameraservice/device3/Camera3DummyStream.h index 462754810c..4b67ea5357 100644 --- a/services/camera/libcameraservice/device3/Camera3DummyStream.h +++ b/services/camera/libcameraservice/device3/Camera3DummyStream.h @@ -87,6 +87,9 @@ class Camera3DummyStream : */ virtual ssize_t getSurfaceId(const sp &/*surface*/) { return 0; } + virtual status_t getUniqueSurfaceIds(const std::vector&, + /*out*/std::vector*) { return INVALID_OPERATION; }; + /** * Update the stream output surfaces. */ @@ -104,6 +107,7 @@ class Camera3DummyStream : const camera3_stream_buffer &buffer, nsecs_t timestamp, bool output, + const std::vector& surface_ids, /*out*/ sp *releaseFenceOut); @@ -128,7 +132,7 @@ class Camera3DummyStream : const std::vector& surface_ids = std::vector()); virtual status_t returnBufferLocked( const camera3_stream_buffer &buffer, - nsecs_t timestamp); + nsecs_t timestamp, const std::vector& surface_ids); virtual status_t configureQueueLocked(); diff --git a/services/camera/libcameraservice/device3/Camera3IOStreamBase.cpp b/services/camera/libcameraservice/device3/Camera3IOStreamBase.cpp index 18b8c4d613..2e909a04ad 100644 --- a/services/camera/libcameraservice/device3/Camera3IOStreamBase.cpp +++ b/services/camera/libcameraservice/device3/Camera3IOStreamBase.cpp @@ -219,7 +219,8 @@ status_t Camera3IOStreamBase::returnBufferPreconditionCheckLocked() const { status_t Camera3IOStreamBase::returnAnyBufferLocked( const camera3_stream_buffer &buffer, nsecs_t timestamp, - bool output) { + bool output, + const std::vector& surface_ids) { status_t res; // returnBuffer may be called from a raw pointer, not a sp<>, and we'll be @@ -235,7 +236,7 @@ status_t Camera3IOStreamBase::returnAnyBufferLocked( } sp releaseFence; - res = returnBufferCheckedLocked(buffer, timestamp, output, + res = returnBufferCheckedLocked(buffer, timestamp, output, surface_ids, &releaseFence); // Res may be an error, but we still want to decrement our owned count // to enable clean shutdown. So we'll just return the error but otherwise diff --git a/services/camera/libcameraservice/device3/Camera3IOStreamBase.h b/services/camera/libcameraservice/device3/Camera3IOStreamBase.h index 48e9bbf846..750f64d181 100644 --- a/services/camera/libcameraservice/device3/Camera3IOStreamBase.h +++ b/services/camera/libcameraservice/device3/Camera3IOStreamBase.h @@ -66,12 +66,14 @@ class Camera3IOStreamBase : status_t returnAnyBufferLocked( const camera3_stream_buffer &buffer, nsecs_t timestamp, - bool output); + bool output, + const std::vector& surface_ids = std::vector()); virtual status_t returnBufferCheckedLocked( const camera3_stream_buffer &buffer, nsecs_t timestamp, bool output, + const std::vector& surface_ids, /*out*/ sp *releaseFenceOut) = 0; diff --git a/services/camera/libcameraservice/device3/Camera3InputStream.cpp b/services/camera/libcameraservice/device3/Camera3InputStream.cpp index 017d7beac9..fc83684601 100644 --- a/services/camera/libcameraservice/device3/Camera3InputStream.cpp +++ b/services/camera/libcameraservice/device3/Camera3InputStream.cpp @@ -98,6 +98,7 @@ status_t Camera3InputStream::returnBufferCheckedLocked( const camera3_stream_buffer &buffer, nsecs_t timestamp, bool output, + const std::vector&, /*out*/ sp *releaseFenceOut) { diff --git a/services/camera/libcameraservice/device3/Camera3InputStream.h b/services/camera/libcameraservice/device3/Camera3InputStream.h index 0732464c44..97a627a250 100644 --- a/services/camera/libcameraservice/device3/Camera3InputStream.h +++ b/services/camera/libcameraservice/device3/Camera3InputStream.h @@ -62,6 +62,7 @@ class Camera3InputStream : public Camera3IOStreamBase, const camera3_stream_buffer &buffer, nsecs_t timestamp, bool output, + const std::vector& surface_ids, /*out*/ sp *releaseFenceOut); diff --git a/services/camera/libcameraservice/device3/Camera3OutputStream.cpp b/services/camera/libcameraservice/device3/Camera3OutputStream.cpp index ecbcf76c68..219cc240dc 100644 --- a/services/camera/libcameraservice/device3/Camera3OutputStream.cpp +++ b/services/camera/libcameraservice/device3/Camera3OutputStream.cpp @@ -187,16 +187,17 @@ status_t Camera3OutputStream::getBufferLocked(camera3_stream_buffer *buffer, } status_t Camera3OutputStream::queueBufferToConsumer(sp& consumer, - ANativeWindowBuffer* buffer, int anwReleaseFence) { + ANativeWindowBuffer* buffer, int anwReleaseFence, + const std::vector&) { return consumer->queueBuffer(consumer.get(), buffer, anwReleaseFence); } status_t Camera3OutputStream::returnBufferLocked( const camera3_stream_buffer &buffer, - nsecs_t timestamp) { + nsecs_t timestamp, const std::vector& surface_ids) { ATRACE_CALL(); - status_t res = returnAnyBufferLocked(buffer, timestamp, /*output*/true); + status_t res = returnAnyBufferLocked(buffer, timestamp, /*output*/true, surface_ids); if (res != OK) { return res; @@ -212,6 +213,7 @@ status_t Camera3OutputStream::returnBufferCheckedLocked( const camera3_stream_buffer &buffer, nsecs_t timestamp, bool output, + const std::vector& surface_ids, /*out*/ sp *releaseFenceOut) { @@ -281,7 +283,7 @@ status_t Camera3OutputStream::returnBufferCheckedLocked( return res; } - res = queueBufferToConsumer(currentConsumer, anwBuffer, anwReleaseFence); + res = queueBufferToConsumer(currentConsumer, anwBuffer, anwReleaseFence, surface_ids); if (res != OK) { ALOGE("%s: Stream %d: Error queueing buffer to native window: " "%s (%d)", __FUNCTION__, mId, strerror(-res), res); diff --git a/services/camera/libcameraservice/device3/Camera3OutputStream.h b/services/camera/libcameraservice/device3/Camera3OutputStream.h index 6f36f922fa..410905dc12 100644 --- a/services/camera/libcameraservice/device3/Camera3OutputStream.h +++ b/services/camera/libcameraservice/device3/Camera3OutputStream.h @@ -190,6 +190,9 @@ class Camera3OutputStream : */ virtual ssize_t getSurfaceId(const sp &/*surface*/) { return 0; } + virtual status_t getUniqueSurfaceIds(const std::vector&, + /*out*/std::vector*) { return INVALID_OPERATION; }; + /** * Update the stream output surfaces. */ @@ -213,6 +216,7 @@ class Camera3OutputStream : const camera3_stream_buffer &buffer, nsecs_t timestamp, bool output, + const std::vector& surface_ids, /*out*/ sp *releaseFenceOut); @@ -285,10 +289,11 @@ class Camera3OutputStream : virtual status_t returnBufferLocked( const camera3_stream_buffer &buffer, - nsecs_t timestamp); + nsecs_t timestamp, const std::vector& surface_ids); virtual status_t queueBufferToConsumer(sp& consumer, - ANativeWindowBuffer* buffer, int anwReleaseFence); + ANativeWindowBuffer* buffer, int anwReleaseFence, + const std::vector& surface_ids); virtual status_t configureQueueLocked(); diff --git a/services/camera/libcameraservice/device3/Camera3OutputStreamInterface.h b/services/camera/libcameraservice/device3/Camera3OutputStreamInterface.h index a711a6d838..2bde94957a 100644 --- a/services/camera/libcameraservice/device3/Camera3OutputStreamInterface.h +++ b/services/camera/libcameraservice/device3/Camera3OutputStreamInterface.h @@ -66,6 +66,18 @@ class Camera3OutputStreamInterface : public virtual Camera3StreamInterface { */ virtual ssize_t getSurfaceId(const sp &surface) = 0; + /** + * Query the unique surface IDs of current surfaceIds. + * When passing unique surface IDs in returnBuffer(), if the + * surfaceId has been removed from the stream, the output corresponding to + * the unique surface ID will be ignored and not delivered to client. + * + * Return INVALID_OPERATION if and only if the stream does not support + * surface sharing. + */ + virtual status_t getUniqueSurfaceIds(const std::vector& surfaceIds, + /*out*/std::vector* outUniqueIds) = 0; + /** * Update the stream output surfaces. */ diff --git a/services/camera/libcameraservice/device3/Camera3SharedOutputStream.cpp b/services/camera/libcameraservice/device3/Camera3SharedOutputStream.cpp index 1c13950950..e3b74d7a44 100644 --- a/services/camera/libcameraservice/device3/Camera3SharedOutputStream.cpp +++ b/services/camera/libcameraservice/device3/Camera3SharedOutputStream.cpp @@ -14,6 +14,10 @@ * limitations under the License. */ +#define LOG_TAG "Camera3-SharedOuStrm" +#define ATRACE_TAG ATRACE_TAG_CAMERA +//#define LOG_NDEBUG 0 + #include "Camera3SharedOutputStream.h" namespace android { @@ -28,16 +32,17 @@ Camera3SharedOutputStream::Camera3SharedOutputStream(int id, uint64_t consumerUsage, android_dataspace dataSpace, camera3_stream_rotation_t rotation, nsecs_t timestampOffset, const String8& physicalCameraId, - int setId) : + int setId, bool useHalBufManager) : Camera3OutputStream(id, CAMERA3_STREAM_OUTPUT, width, height, format, dataSpace, rotation, physicalCameraId, - consumerUsage, timestampOffset, setId) { + consumerUsage, timestampOffset, setId), + mUseHalBufManager(useHalBufManager) { size_t consumerCount = std::min(surfaces.size(), kMaxOutputs); if (surfaces.size() > consumerCount) { ALOGE("%s: Trying to add more consumers than the maximum ", __func__); } for (size_t i = 0; i < consumerCount; i++) { - mSurfaces[i] = surfaces[i]; + mSurfaceUniqueIds[i] = std::make_pair(surfaces[i], mNextUniqueSurfaceId++); } } @@ -48,15 +53,15 @@ Camera3SharedOutputStream::~Camera3SharedOutputStream() { status_t Camera3SharedOutputStream::connectStreamSplitterLocked() { status_t res = OK; - mStreamSplitter = new Camera3StreamSplitter(); + mStreamSplitter = new Camera3StreamSplitter(mUseHalBufManager); uint64_t usage; getEndpointUsage(&usage); std::unordered_map> initialSurfaces; for (size_t i = 0; i < kMaxOutputs; i++) { - if (mSurfaces[i] != nullptr) { - initialSurfaces.emplace(i, mSurfaces[i]); + if (mSurfaceUniqueIds[i].first != nullptr) { + initialSurfaces.emplace(i, mSurfaceUniqueIds[i].first); } } @@ -71,6 +76,31 @@ status_t Camera3SharedOutputStream::connectStreamSplitterLocked() { return res; } +status_t Camera3SharedOutputStream::attachBufferToSplitterLocked( + ANativeWindowBuffer* anb, + const std::vector& surface_ids) { + status_t res = OK; + + // Attach the buffer to the splitter output queues. This could block if + // the output queue doesn't have any empty slot. So unlock during the course + // of attachBufferToOutputs. + sp splitter = mStreamSplitter; + mLock.unlock(); + res = splitter->attachBufferToOutputs(anb, surface_ids); + mLock.lock(); + if (res != OK) { + ALOGE("%s: Stream %d: Cannot attach stream splitter buffer to outputs: %s (%d)", + __FUNCTION__, mId, strerror(-res), res); + // Only transition to STATE_ABANDONED from STATE_CONFIGURED. (If it is STATE_PREPARING, + // let prepareNextBuffer handle the error.) + if (res == NO_INIT && mState == STATE_CONFIGURED) { + mState = STATE_ABANDONED; + } + } + return res; +} + + status_t Camera3SharedOutputStream::notifyBufferReleased(ANativeWindowBuffer *anwBuffer) { Mutex::Autolock l(mLock); status_t res = OK; @@ -89,7 +119,7 @@ bool Camera3SharedOutputStream::isConsumerConfigurationDeferred(size_t surface_i return true; } - return (mSurfaces[surface_id] == nullptr); + return (mSurfaceUniqueIds[surface_id].first == nullptr); } status_t Camera3SharedOutputStream::setConsumers(const std::vector>& surfaces) { @@ -112,7 +142,7 @@ status_t Camera3SharedOutputStream::setConsumers(const std::vector>& return NO_MEMORY; } - mSurfaces[id] = surface; + mSurfaceUniqueIds[id] = std::make_pair(surface, mNextUniqueSurfaceId++); // Only call addOutput if the splitter has been connected. if (mStreamSplitter != nullptr) { @@ -128,7 +158,7 @@ status_t Camera3SharedOutputStream::setConsumers(const std::vector>& } status_t Camera3SharedOutputStream::getBufferLocked(camera3_stream_buffer *buffer, - const std::vector& surface_ids) { + const std::vector& surfaceIds) { ANativeWindowBuffer* anb; int fenceFd = -1; @@ -138,27 +168,11 @@ status_t Camera3SharedOutputStream::getBufferLocked(camera3_stream_buffer *buffe return res; } - // TODO: need to refactor this to support requestStreamBuffers API - // Need to wait until processCaptureResult to decide the source buffer - // to attach to output... - - // Attach the buffer to the splitter output queues. This could block if - // the output queue doesn't have any empty slot. So unlock during the course - // of attachBufferToOutputs. - sp splitter = mStreamSplitter; - mLock.unlock(); - res = splitter->attachBufferToOutputs(anb, surface_ids); - mLock.lock(); - if (res != OK) { - ALOGE("%s: Stream %d: Cannot attach stream splitter buffer to outputs: %s (%d)", - __FUNCTION__, mId, strerror(-res), res); - // Only transition to STATE_ABANDONED from STATE_CONFIGURED. (If it is STATE_PREPARING, - // let prepareNextBuffer handle the error.) - if (res == NO_INIT && mState == STATE_CONFIGURED) { - mState = STATE_ABANDONED; + if (!mUseHalBufManager) { + res = attachBufferToSplitterLocked(anb, surfaceIds); + if (res != OK) { + return res; } - - return res; } /** @@ -172,8 +186,38 @@ status_t Camera3SharedOutputStream::getBufferLocked(camera3_stream_buffer *buffe } status_t Camera3SharedOutputStream::queueBufferToConsumer(sp& consumer, - ANativeWindowBuffer* buffer, int anwReleaseFence) { - status_t res = consumer->queueBuffer(consumer.get(), buffer, anwReleaseFence); + ANativeWindowBuffer* buffer, int anwReleaseFence, + const std::vector& uniqueSurfaceIds) { + status_t res = OK; + if (mUseHalBufManager) { + if (uniqueSurfaceIds.size() == 0) { + ALOGE("%s: uniqueSurfaceIds must not be empty!", __FUNCTION__); + return BAD_VALUE; + } + Mutex::Autolock l(mLock); + std::vector surfaceIds; + for (const auto& uniqueId : uniqueSurfaceIds) { + bool uniqueIdFound = false; + for (size_t i = 0; i < kMaxOutputs; i++) { + if (mSurfaceUniqueIds[i].second == uniqueId) { + surfaceIds.push_back(i); + uniqueIdFound = true; + break; + } + } + if (!uniqueIdFound) { + ALOGV("%s: unknown unique surface ID %zu for stream %d: " + "output might have been removed.", + __FUNCTION__, uniqueId, mId); + } + } + res = attachBufferToSplitterLocked(buffer, surfaceIds); + if (res != OK) { + return res; + } + } + + res = consumer->queueBuffer(consumer.get(), buffer, anwReleaseFence); // After queuing buffer to the internal consumer queue, check whether the buffer is // successfully queued to the output queues. @@ -232,8 +276,8 @@ status_t Camera3SharedOutputStream::getEndpointUsage(uint64_t *usage) const { *usage = getPresetConsumerUsage(); for (size_t id = 0; id < kMaxOutputs; id++) { - if (mSurfaces[id] != nullptr) { - res = getEndpointUsageForSurface(&u, mSurfaces[id]); + if (mSurfaceUniqueIds[id].first != nullptr) { + res = getEndpointUsageForSurface(&u, mSurfaceUniqueIds[id].first); *usage |= u; } } @@ -249,7 +293,7 @@ status_t Camera3SharedOutputStream::getEndpointUsage(uint64_t *usage) const { ssize_t Camera3SharedOutputStream::getNextSurfaceIdLocked() { ssize_t id = -1; for (size_t i = 0; i < kMaxOutputs; i++) { - if (mSurfaces[i] == nullptr) { + if (mSurfaceUniqueIds[i].first == nullptr) { id = i; break; } @@ -262,7 +306,7 @@ ssize_t Camera3SharedOutputStream::getSurfaceId(const sp &surface) { Mutex::Autolock l(mLock); ssize_t id = -1; for (size_t i = 0; i < kMaxOutputs; i++) { - if (mSurfaces[i] == surface) { + if (mSurfaceUniqueIds[i].first == surface) { id = i; break; } @@ -271,6 +315,26 @@ ssize_t Camera3SharedOutputStream::getSurfaceId(const sp &surface) { return id; } +status_t Camera3SharedOutputStream::getUniqueSurfaceIds( + const std::vector& surfaceIds, + /*out*/std::vector* outUniqueIds) { + Mutex::Autolock l(mLock); + if (outUniqueIds == nullptr || surfaceIds.size() > kMaxOutputs) { + return BAD_VALUE; + } + + outUniqueIds->clear(); + outUniqueIds->reserve(surfaceIds.size()); + + for (const auto& surfaceId : surfaceIds) { + if (surfaceId >= kMaxOutputs) { + return BAD_VALUE; + } + outUniqueIds->push_back(mSurfaceUniqueIds[surfaceId].second); + } + return OK; +} + status_t Camera3SharedOutputStream::revertPartialUpdateLocked( const KeyedVector, size_t> &removedSurfaces, const KeyedVector, size_t> &attachedSurfaces) { @@ -284,7 +348,7 @@ status_t Camera3SharedOutputStream::revertPartialUpdateLocked( return UNKNOWN_ERROR; } } - mSurfaces[index] = nullptr; + mSurfaceUniqueIds[index] = std::make_pair(nullptr, mNextUniqueSurfaceId++); } for (size_t i = 0; i < removedSurfaces.size(); i++) { @@ -295,7 +359,8 @@ status_t Camera3SharedOutputStream::revertPartialUpdateLocked( return UNKNOWN_ERROR; } } - mSurfaces[index] = removedSurfaces.keyAt(i); + mSurfaceUniqueIds[index] = std::make_pair( + removedSurfaces.keyAt(i), mNextUniqueSurfaceId++); } return ret; @@ -347,8 +412,8 @@ status_t Camera3SharedOutputStream::updateStream(const std::vector> } } - mSurfaces[it] = nullptr; - removedSurfaces.add(mSurfaces[it], it); + removedSurfaces.add(mSurfaceUniqueIds[it].first, it); + mSurfaceUniqueIds[it] = std::make_pair(nullptr, mNextUniqueSurfaceId++); } //Next add the new outputs @@ -373,7 +438,7 @@ status_t Camera3SharedOutputStream::updateStream(const std::vector> return ret; } } - mSurfaces[surfaceId] = it; + mSurfaceUniqueIds[surfaceId] = std::make_pair(it, mNextUniqueSurfaceId++); outputMap->add(it, surfaceId); } diff --git a/services/camera/libcameraservice/device3/Camera3SharedOutputStream.h b/services/camera/libcameraservice/device3/Camera3SharedOutputStream.h index 02b1c09621..b5e37c2ffd 100644 --- a/services/camera/libcameraservice/device3/Camera3SharedOutputStream.h +++ b/services/camera/libcameraservice/device3/Camera3SharedOutputStream.h @@ -17,6 +17,7 @@ #ifndef ANDROID_SERVERS_CAMERA3_SHARED_OUTPUT_STREAM_H #define ANDROID_SERVERS_CAMERA3_SHARED_OUTPUT_STREAM_H +#include #include "Camera3StreamSplitter.h" #include "Camera3OutputStream.h" @@ -37,7 +38,8 @@ public: uint64_t consumerUsage, android_dataspace dataSpace, camera3_stream_rotation_t rotation, nsecs_t timestampOffset, const String8& physicalCameraId, - int setId = CAMERA3_STREAM_SET_ID_INVALID); + int setId = CAMERA3_STREAM_SET_ID_INVALID, + bool useHalBufManager = false); virtual ~Camera3SharedOutputStream(); @@ -49,6 +51,15 @@ public: virtual ssize_t getSurfaceId(const sp &surface); + /** + * Query the unique surface IDs of current surfaceIds. + * When passing unique surface IDs in returnBuffer(), if the + * surfaceId has been removed from the stream, the output corresponding to + * the unique surface ID will be ignored and not delivered to client. + */ + virtual status_t getUniqueSurfaceIds(const std::vector& surfaceIds, + /*out*/std::vector* outUniqueIds) override; + virtual status_t updateStream(const std::vector> &outputSurfaces, const std::vector &outputInfo, const std::vector &removedSurfaceIds, @@ -58,8 +69,17 @@ private: static const size_t kMaxOutputs = 4; - // Map surfaceId -> output surfaces - sp mSurfaces[kMaxOutputs]; + // Whether HAL is in control for buffer management. Surface sharing behavior + // depends on this flag. + const bool mUseHalBufManager; + + // Pair of an output Surface and its unique ID + typedef std::pair, size_t> SurfaceUniqueId; + + // Map surfaceId -> (output surface, unique surface ID) + std::array mSurfaceUniqueIds; + + size_t mNextUniqueSurfaceId = 0; ssize_t getNextSurfaceIdLocked(); @@ -77,6 +97,16 @@ private: */ status_t connectStreamSplitterLocked(); + /** + * Attach the output buffer to stream splitter. + * When camera service is doing buffer management, this method will be called + * before the buffer is handed out to HAL in request thread. + * When HAL is doing buffer management, this method will be called when + * the buffer is returned from HAL in hwbinder callback thread. + */ + status_t attachBufferToSplitterLocked(ANativeWindowBuffer* anb, + const std::vector& surface_ids); + /** * Internal Camera3Stream interface */ @@ -84,7 +114,8 @@ private: const std::vector& surface_ids); virtual status_t queueBufferToConsumer(sp& consumer, - ANativeWindowBuffer* buffer, int anwReleaseFence); + ANativeWindowBuffer* buffer, int anwReleaseFence, + const std::vector& uniqueSurfaceIds); virtual status_t configureQueueLocked(); diff --git a/services/camera/libcameraservice/device3/Camera3Stream.cpp b/services/camera/libcameraservice/device3/Camera3Stream.cpp index 0a30a97631..24d1c1bd24 100644 --- a/services/camera/libcameraservice/device3/Camera3Stream.cpp +++ b/services/camera/libcameraservice/device3/Camera3Stream.cpp @@ -655,7 +655,8 @@ void Camera3Stream::removeOutstandingBuffer(const camera3_stream_buffer &buffer) } status_t Camera3Stream::returnBuffer(const camera3_stream_buffer &buffer, - nsecs_t timestamp, bool timestampIncreasing) { + nsecs_t timestamp, bool timestampIncreasing, + const std::vector& surface_ids) { ATRACE_CALL(); Mutex::Autolock l(mLock); @@ -684,7 +685,7 @@ status_t Camera3Stream::returnBuffer(const camera3_stream_buffer &buffer, * * Do this for getBuffer as well. */ - status_t res = returnBufferLocked(b, timestamp); + status_t res = returnBufferLocked(b, timestamp, surface_ids); if (res == OK) { fireBufferListenersLocked(b, /*acquired*/false, /*output*/true); } @@ -843,7 +844,7 @@ status_t Camera3Stream::getBufferLocked(camera3_stream_buffer *, return INVALID_OPERATION; } status_t Camera3Stream::returnBufferLocked(const camera3_stream_buffer &, - nsecs_t) { + nsecs_t, const std::vector&) { ALOGE("%s: This type of stream does not support output", __FUNCTION__); return INVALID_OPERATION; } diff --git a/services/camera/libcameraservice/device3/Camera3Stream.h b/services/camera/libcameraservice/device3/Camera3Stream.h index e29c3e00ae..ddba9f63e5 100644 --- a/services/camera/libcameraservice/device3/Camera3Stream.h +++ b/services/camera/libcameraservice/device3/Camera3Stream.h @@ -322,11 +322,17 @@ class Camera3Stream : /** * Return a buffer to the stream after use by the HAL. * + * Multiple surfaces could share the same HAL stream, but a request may + * be only for a subset of surfaces. In this case, the + * Camera3StreamInterface object needs the surface ID information to attach + * buffers for those surfaces. + * * This method may only be called for buffers provided by getBuffer(). * For bidirectional streams, this method applies to the output-side buffers */ status_t returnBuffer(const camera3_stream_buffer &buffer, - nsecs_t timestamp, bool timestampIncreasing); + nsecs_t timestamp, bool timestampIncreasing, + const std::vector& surface_ids = std::vector()); /** * Fill in the camera3_stream_buffer with the next valid buffer for this @@ -478,7 +484,8 @@ class Camera3Stream : virtual status_t getBufferLocked(camera3_stream_buffer *buffer, const std::vector& surface_ids = std::vector()); virtual status_t returnBufferLocked(const camera3_stream_buffer &buffer, - nsecs_t timestamp); + nsecs_t timestamp, + const std::vector& surface_ids = std::vector()); virtual status_t getInputBufferLocked(camera3_stream_buffer *buffer); virtual status_t returnInputBufferLocked( const camera3_stream_buffer &buffer); diff --git a/services/camera/libcameraservice/device3/Camera3StreamInterface.h b/services/camera/libcameraservice/device3/Camera3StreamInterface.h index 866b7224f8..a84720bfe2 100644 --- a/services/camera/libcameraservice/device3/Camera3StreamInterface.h +++ b/services/camera/libcameraservice/device3/Camera3StreamInterface.h @@ -248,11 +248,18 @@ class Camera3StreamInterface : public virtual RefBase { /** * Return a buffer to the stream after use by the HAL. * + * Multiple surfaces could share the same HAL stream, but a request may + * be only for a subset of surfaces. In this case, the + * Camera3StreamInterface object needs the surface ID information to attach + * buffers for those surfaces. For the case of single surface for a HAL + * stream, surface_ids parameter has no effect. + * * This method may only be called for buffers provided by getBuffer(). * For bidirectional streams, this method applies to the output-side buffers */ virtual status_t returnBuffer(const camera3_stream_buffer &buffer, - nsecs_t timestamp, bool timestampIncreasing = true) = 0; + nsecs_t timestamp, bool timestampIncreasing = true, + const std::vector& surface_ids = std::vector()) = 0; /** * Fill in the camera3_stream_buffer with the next valid buffer for this diff --git a/services/camera/libcameraservice/device3/Camera3StreamSplitter.cpp b/services/camera/libcameraservice/device3/Camera3StreamSplitter.cpp index 8bc208dfa7..13a1567eaa 100644 --- a/services/camera/libcameraservice/device3/Camera3StreamSplitter.cpp +++ b/services/camera/libcameraservice/device3/Camera3StreamSplitter.cpp @@ -150,6 +150,8 @@ void Camera3StreamSplitter::disconnect() { SP_LOGV("%s: Disconnected", __FUNCTION__); } +Camera3StreamSplitter::Camera3StreamSplitter(bool useHalBufManager) : + mUseHalBufManager(useHalBufManager) {} Camera3StreamSplitter::~Camera3StreamSplitter() { disconnect(); @@ -237,7 +239,9 @@ status_t Camera3StreamSplitter::addOutputLocked(size_t surfaceId, const sp(outputQueue.get()), &usage); if (!(usage & (GRALLOC_USAGE_HW_COMPOSER | GRALLOC_USAGE_HW_TEXTURE))) { - outputQueue->setDequeueTimeout(kDequeueBufferTimeout); + nsecs_t timeout = mUseHalBufManager ? + kHalBufMgrDequeueBufferTimeout : kNormalDequeueBufferTimeout; + outputQueue->setDequeueTimeout(timeout); } res = gbp->allowAllocation(false); @@ -430,8 +434,9 @@ status_t Camera3StreamSplitter::attachBufferToOutputs(ANativeWindowBuffer* anb, res = gbp->attachBuffer(&slot, gb); mMutex.lock(); if (res != OK) { - SP_LOGE("%s: Cannot acquireBuffer from GraphicBufferProducer %p: %s (%d)", + SP_LOGE("%s: Cannot attachBuffer from GraphicBufferProducer %p: %s (%d)", __FUNCTION__, gbp.get(), strerror(-res), res); + // TODO: might need to detach/cleanup the already attached buffers before return? return res; } if ((slot < 0) || (slot > BufferQueue::NUM_BUFFER_SLOTS)) { diff --git a/services/camera/libcameraservice/device3/Camera3StreamSplitter.h b/services/camera/libcameraservice/device3/Camera3StreamSplitter.h index fea1bdb3d6..8c7181c7c4 100644 --- a/services/camera/libcameraservice/device3/Camera3StreamSplitter.h +++ b/services/camera/libcameraservice/device3/Camera3StreamSplitter.h @@ -49,7 +49,7 @@ class Camera3StreamSplitter : public BnConsumerListener { public: // Constructor - Camera3StreamSplitter() = default; + Camera3StreamSplitter(bool useHalBufManager = false); // Connect to the stream splitter by creating buffer queue and connecting it // with output surfaces. @@ -226,7 +226,10 @@ private: android::PixelFormat mFormat = android::PIXEL_FORMAT_NONE; uint64_t mProducerUsage = 0; - static const nsecs_t kDequeueBufferTimeout = s2ns(1); // 1 sec + // The attachBuffer call will happen on different thread according to mUseHalBufManager and have + // different timing constraint. + static const nsecs_t kNormalDequeueBufferTimeout = s2ns(1); // 1 sec + static const nsecs_t kHalBufMgrDequeueBufferTimeout = ms2ns(1); // 1 msec Mutex mMutex; @@ -270,6 +273,8 @@ private: std::atomic mOnFrameAvailableRes{0}; String8 mConsumerName; + + const bool mUseHalBufManager; }; } // namespace android