diff --git a/camera/aidl/android/hardware/camera2/ICameraDeviceUser.aidl b/camera/aidl/android/hardware/camera2/ICameraDeviceUser.aidl index cdb7ac3a9d..b183ccc822 100644 --- a/camera/aidl/android/hardware/camera2/ICameraDeviceUser.aidl +++ b/camera/aidl/android/hardware/camera2/ICameraDeviceUser.aidl @@ -83,8 +83,9 @@ interface ICameraDeviceUser * @param operatingMode The kind of session to create; either NORMAL_MODE or * CONSTRAINED_HIGH_SPEED_MODE. Must be a non-negative value. * @param sessionParams Session wide camera parameters + * @return a list of stream ids that can be used in offline mode via "switchToOffline" */ - void endConfigure(int operatingMode, in CameraMetadataNative sessionParams); + int[] endConfigure(int operatingMode, in CameraMetadataNative sessionParams); /** * Check whether a particular session configuration has camera device @@ -189,5 +190,5 @@ interface ICameraDeviceUser * @return Offline session object. */ ICameraOfflineSession switchToOffline(in ICameraDeviceCallbacks callbacks, - in Surface[] offlineOutputs); + in int[] offlineOutputIds); } diff --git a/camera/ndk/impl/ACameraDevice.cpp b/camera/ndk/impl/ACameraDevice.cpp index 46a8dae317..0d7180abb5 100644 --- a/camera/ndk/impl/ACameraDevice.cpp +++ b/camera/ndk/impl/ACameraDevice.cpp @@ -710,7 +710,8 @@ CameraDevice::configureStreamsLocked(const ACaptureSessionOutputContainer* outpu if ((sessionParameters != nullptr) && (sessionParameters->settings != nullptr)) { params.append(sessionParameters->settings->getInternalData()); } - remoteRet = mRemote->endConfigure(/*isConstrainedHighSpeed*/ false, params); + std::vector offlineStreamIds; + remoteRet = mRemote->endConfigure(/*isConstrainedHighSpeed*/ false, params, &offlineStreamIds); if (remoteRet.serviceSpecificErrorCode() == hardware::ICameraService::ERROR_ILLEGAL_ARGUMENT) { ALOGE("Camera device %s cannnot support app output configuration: %s", getId(), remoteRet.toString8().string()); diff --git a/camera/tests/CameraBinderTests.cpp b/camera/tests/CameraBinderTests.cpp index 9a18b10222..cd5bdd1cb9 100644 --- a/camera/tests/CameraBinderTests.cpp +++ b/camera/tests/CameraBinderTests.cpp @@ -498,7 +498,9 @@ TEST_F(CameraClientBinderTest, CheckBinderCameraDeviceUser) { EXPECT_TRUE(res.isOk()) << res; EXPECT_LE(0, streamId); CameraMetadata sessionParams; - res = device->endConfigure(/*isConstrainedHighSpeed*/ false, sessionParams); + std::vector offlineStreamIds; + res = device->endConfigure(/*isConstrainedHighSpeed*/ false, sessionParams, + &offlineStreamIds); EXPECT_TRUE(res.isOk()) << res; EXPECT_FALSE(callbacks->hadError()); @@ -609,7 +611,8 @@ TEST_F(CameraClientBinderTest, CheckBinderCameraDeviceUser) { EXPECT_TRUE(res.isOk()) << res; res = device->deleteStream(streamId); EXPECT_TRUE(res.isOk()) << res; - res = device->endConfigure(/*isConstrainedHighSpeed*/ false, sessionParams); + res = device->endConfigure(/*isConstrainedHighSpeed*/ false, sessionParams, + &offlineStreamIds); EXPECT_TRUE(res.isOk()) << res; sleep(/*second*/1); // allow some time for errors to show up, if any diff --git a/services/camera/libcameraservice/Android.bp b/services/camera/libcameraservice/Android.bp index c63feb2810..496a21b6dc 100644 --- a/services/camera/libcameraservice/Android.bp +++ b/services/camera/libcameraservice/Android.bp @@ -41,11 +41,13 @@ cc_library_shared { "api1/client2/CaptureSequencer.cpp", "api1/client2/ZslProcessor.cpp", "api2/CameraDeviceClient.cpp", + "api2/CameraOfflineSessionClient.cpp", "api2/CompositeStream.cpp", "api2/DepthCompositeStream.cpp", "api2/HeicEncoderInfoManager.cpp", "api2/HeicCompositeStream.cpp", "device1/CameraHardwareInterface.cpp", + "device3/BufferUtils.cpp", "device3/Camera3Device.cpp", "device3/Camera3OfflineSession.cpp", "device3/Camera3Stream.cpp", @@ -60,6 +62,8 @@ cc_library_shared { "device3/CoordinateMapper.cpp", "device3/DistortionMapper.cpp", "device3/ZoomRatioMapper.cpp", + "device3/Camera3OutputStreamInterface.cpp", + "device3/Camera3OutputUtils.cpp", "gui/RingBufferConsumer.cpp", "utils/CameraThreadState.cpp", "hidl/AidlCameraDeviceCallbacks.cpp", diff --git a/services/camera/libcameraservice/CameraService.cpp b/services/camera/libcameraservice/CameraService.cpp index a110c33e42..2fe717939b 100644 --- a/services/camera/libcameraservice/CameraService.cpp +++ b/services/camera/libcameraservice/CameraService.cpp @@ -128,6 +128,7 @@ static const String16 static constexpr int32_t kVendorClientScore = 200; // Matches with PROCESS_STATE_PERSISTENT_UI in ActivityManager.java static constexpr int32_t kVendorClientState = 1; +const String8 CameraService::kOfflineDevice("offline-"); Mutex CameraService::sProxyMutex; sp CameraService::sCameraServiceProxy; @@ -394,7 +395,7 @@ void CameraService::onDeviceStatusChanged(const String8& id, // to this device until the status changes updateStatus(StatusInternal::NOT_PRESENT, id); - sp clientToDisconnect; + sp clientToDisconnectOnline, clientToDisconnectOffline; { // Don't do this in updateStatus to avoid deadlock over mServiceLock Mutex::Autolock lock(mServiceLock); @@ -402,23 +403,14 @@ void CameraService::onDeviceStatusChanged(const String8& id, // Remove cached shim parameters state->setShimParams(CameraParameters()); - // Remove the client from the list of active clients, if there is one - clientToDisconnect = removeClientLocked(id); + // Remove online as well as offline client from the list of active clients, + // if they are present + clientToDisconnectOnline = removeClientLocked(id); + clientToDisconnectOffline = removeClientLocked(kOfflineDevice + id); } - // Disconnect client - if (clientToDisconnect.get() != nullptr) { - ALOGI("%s: Client for camera ID %s evicted due to device status change from HAL", - __FUNCTION__, id.string()); - // Notify the client of disconnection - clientToDisconnect->notifyError( - hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_DISCONNECTED, - CaptureResultExtras{}); - // Ensure not in binder RPC so client disconnect PID checks work correctly - LOG_ALWAYS_FATAL_IF(CameraThreadState::getCallingPid() != getpid(), - "onDeviceStatusChanged must be called from the camera service process!"); - clientToDisconnect->disconnect(); - } + disconnectClient(id, clientToDisconnectOnline); + disconnectClient(kOfflineDevice + id, clientToDisconnectOffline); removeStates(id); } else { @@ -431,6 +423,21 @@ void CameraService::onDeviceStatusChanged(const String8& id, } +void CameraService::disconnectClient(const String8& id, sp clientToDisconnect) { + if (clientToDisconnect.get() != nullptr) { + ALOGI("%s: Client for camera ID %s evicted due to device status change from HAL", + __FUNCTION__, id.string()); + // Notify the client of disconnection + clientToDisconnect->notifyError( + hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_DISCONNECTED, + CaptureResultExtras{}); + // Ensure not in binder RPC so client disconnect PID checks work correctly + LOG_ALWAYS_FATAL_IF(CameraThreadState::getCallingPid() != getpid(), + "onDeviceStatusChanged must be called from the camera service process!"); + clientToDisconnect->disconnect(); + } +} + void CameraService::onTorchStatusChanged(const String8& cameraId, TorchModeStatus newStatus) { Mutex::Autolock al(mTorchStatusMutex); @@ -1696,6 +1703,77 @@ Status CameraService::connectHelper(const sp& cameraCb, const String8& return ret; } +status_t CameraService::addOfflineClient(String8 cameraId, sp offlineClient) { + if (offlineClient.get() == nullptr) { + return BAD_VALUE; + } + + { + // Acquire mServiceLock and prevent other clients from connecting + std::unique_ptr lock = + AutoConditionLock::waitAndAcquire(mServiceLockWrapper, DEFAULT_CONNECT_TIMEOUT_NS); + + if (lock == nullptr) { + ALOGE("%s: (PID %d) rejected (too many other clients connecting)." + , __FUNCTION__, offlineClient->getClientPid()); + return TIMED_OUT; + } + + auto onlineClientDesc = mActiveClientManager.get(cameraId); + if (onlineClientDesc.get() == nullptr) { + ALOGE("%s: No active online client using camera id: %s", __FUNCTION__, + cameraId.c_str()); + return BAD_VALUE; + } + + // Offline clients do not evict or conflict with other online devices. Resource sharing + // conflicts are handled by the camera provider which will either succeed or fail before + // reaching this method. + const auto& onlinePriority = onlineClientDesc->getPriority(); + auto offlineClientDesc = CameraClientManager::makeClientDescriptor( + kOfflineDevice + onlineClientDesc->getKey(), offlineClient, /*cost*/ 0, + /*conflictingKeys*/ std::set(), onlinePriority.getScore(), + onlineClientDesc->getOwnerId(), onlinePriority.getState()); + + // Allow only one offline device per camera + auto incompatibleClients = mActiveClientManager.getIncompatibleClients(offlineClientDesc); + if (!incompatibleClients.empty()) { + ALOGE("%s: Incompatible offline clients present!", __FUNCTION__); + return BAD_VALUE; + } + + auto err = offlineClient->initialize(mCameraProviderManager, mMonitorTags); + if (err != OK) { + ALOGE("%s: Could not initialize offline client.", __FUNCTION__); + return err; + } + + auto evicted = mActiveClientManager.addAndEvict(offlineClientDesc); + if (evicted.size() > 0) { + for (auto& i : evicted) { + ALOGE("%s: Invalid state: Offline client for camera %s was not removed ", + __FUNCTION__, i->getKey().string()); + } + + LOG_ALWAYS_FATAL("%s: Invalid state for CameraService, offline clients not evicted " + "properly", __FUNCTION__); + + return BAD_VALUE; + } + + logConnectedOffline(offlineClientDesc->getKey(), + static_cast(offlineClientDesc->getOwnerId()), + String8(offlineClient->getPackageName())); + + sp remoteCallback = offlineClient->getRemote(); + if (remoteCallback != nullptr) { + remoteCallback->linkToDeath(this); + } + } // lock is destroyed, allow further connect calls + + return OK; +} + Status CameraService::setTorchMode(const String16& cameraId, bool enabled, const sp& clientBinder) { Mutex::Autolock lock(mServiceLock); @@ -2300,6 +2378,13 @@ void CameraService::logDisconnected(const char* cameraId, int clientPid, clientPackage, clientPid)); } +void CameraService::logDisconnectedOffline(const char* cameraId, int clientPid, + const char* clientPackage) { + // Log the clients evicted + logEvent(String8::format("DISCONNECT offline device %s client for package %s (PID %d)", + cameraId, clientPackage, clientPid)); +} + void CameraService::logConnected(const char* cameraId, int clientPid, const char* clientPackage) { // Log the clients evicted @@ -2307,6 +2392,13 @@ void CameraService::logConnected(const char* cameraId, int clientPid, clientPackage, clientPid)); } +void CameraService::logConnectedOffline(const char* cameraId, int clientPid, + const char* clientPackage) { + // Log the clients evicted + logEvent(String8::format("CONNECT offline device %s client for package %s (PID %d)", cameraId, + clientPackage, clientPid)); +} + void CameraService::logRejected(const char* cameraId, int clientPid, const char* clientPackage, const char* reason) { // Log the client rejected @@ -2744,6 +2836,7 @@ void CameraService::BasicClient::opChanged(int32_t op, const String16&) { if (mAppOpsManager == nullptr) { return; } + // TODO : add offline camera session case if (op != AppOpsManager::OP_CAMERA) { ALOGW("Unexpected app ops notification received: %d", op); return; @@ -2778,20 +2871,6 @@ void CameraService::BasicClient::block() { // ---------------------------------------------------------------------------- -sp CameraService::OfflineClient::sCameraService; - -status_t CameraService::OfflineClient::startCameraOps() { - // TODO - return OK; -} - -status_t CameraService::OfflineClient::finishCameraOps() { - // TODO - return OK; -} - -// ---------------------------------------------------------------------------- - void CameraService::Client::notifyError(int32_t errorCode, const CaptureResultExtras& resultExtras) { (void) resultExtras; diff --git a/services/camera/libcameraservice/CameraService.h b/services/camera/libcameraservice/CameraService.h index dae9c0960f..726cb0f4a1 100644 --- a/services/camera/libcameraservice/CameraService.h +++ b/services/camera/libcameraservice/CameraService.h @@ -68,6 +68,7 @@ class CameraService : { friend class BinderService; friend class CameraClient; + friend class CameraOfflineSessionClient; public: class Client; class BasicClient; @@ -185,6 +186,9 @@ public: // Monitored UIDs availability notification void notifyMonitoredUids(); + // Register an offline client for a given active camera id + status_t addOfflineClient(String8 cameraId, sp offlineClient); + ///////////////////////////////////////////////////////////////////// // Client functionality @@ -310,10 +314,9 @@ public: sp mRemoteBinder; // immutable after constructor // permissions management - status_t startCameraOps(); - status_t finishCameraOps(); + virtual status_t startCameraOps(); + virtual status_t finishCameraOps(); - private: std::unique_ptr mAppOpsManager = nullptr; class OpsCallback : public BnAppOpsCallback { @@ -402,87 +405,6 @@ public: int mCameraId; // All API1 clients use integer camera IDs }; // class Client - - // Client for offline session. Note that offline session client does not affect camera service's - // client arbitration logic. It is camera HAL's decision to decide whether a normal camera - // client is conflicting with existing offline client(s). - // The other distinctive difference between offline clients and normal clients is that normal - // clients are created through ICameraService binder calls, while the offline session client - // is created through ICameraDeviceUser::switchToOffline call. - class OfflineClient : public virtual RefBase { - - virtual status_t dump(int fd, const Vector& args) = 0; - - // Block the client form using the camera - virtual void block() = 0; - - // Return the package name for this client - virtual String16 getPackageName() const = 0; - - // Notify client about a fatal error - // TODO: maybe let impl notify within block? - virtual void notifyError(int32_t errorCode, - const CaptureResultExtras& resultExtras) = 0; - - // Get the UID of the application client using this - virtual uid_t getClientUid() const = 0; - - // Get the PID of the application client using this - virtual int getClientPid() const = 0; - - protected: - OfflineClient(const sp& cameraService, - const String16& clientPackageName, - const String8& cameraIdStr, - int clientPid, - uid_t clientUid, - int servicePid): mCameraIdStr(cameraIdStr), - mClientPackageName(clientPackageName), mClientPid(clientPid), - mClientUid(clientUid), mServicePid(servicePid) { - if (sCameraService == nullptr) { - sCameraService = cameraService; - } - } - - virtual ~OfflineClient() { /*TODO*/ } - - // these are initialized in the constructor. - static sp sCameraService; - const String8 mCameraIdStr; - String16 mClientPackageName; - pid_t mClientPid; - const uid_t mClientUid; - const pid_t mServicePid; - bool mDisconnected; - - // - The app-side Binder interface to receive callbacks from us - sp mRemoteBinder; // immutable after constructor - - // permissions management - status_t startCameraOps(); - status_t finishCameraOps(); - - private: - std::unique_ptr mAppOpsManager = nullptr; - - class OpsCallback : public BnAppOpsCallback { - public: - explicit OpsCallback(wp client) : mClient(client) {} - virtual void opChanged(int32_t /*op*/, const String16& /*packageName*/) { - //TODO - } - - private: - wp mClient; - - }; // class OpsCallback - - sp mOpsCallback; - - // IAppOpsCallback interface, indirected through opListener - // virtual void opChanged(int32_t op, const String16& packageName); - }; // class OfflineClient - /** * A listener class that implements the LISTENER interface for use with a ClientManager, and * implements the following methods: @@ -871,6 +793,17 @@ private: */ void logDisconnected(const char* cameraId, int clientPid, const char* clientPackage); + /** + * Add an event log message that a client has been disconnected from offline device. + */ + void logDisconnectedOffline(const char* cameraId, int clientPid, const char* clientPackage); + + /** + * Add an event log message that an offline client has been connected. + */ + void logConnectedOffline(const char* cameraId, int clientPid, + const char* clientPackage); + /** * Add an event log message that a client has been connected. */ @@ -1095,6 +1028,12 @@ private: void broadcastTorchModeStatus(const String8& cameraId, hardware::camera::common::V1_0::TorchModeStatus status); + void disconnectClient(const String8& id, sp clientToDisconnect); + + // Regular online and offline devices must not be in conflict at camera service layer. + // Use separate keys for offline devices. + static const String8 kOfflineDevice; + // TODO: right now each BasicClient holds one AppOpsManager instance. // We can refactor the code so all of clients share this instance AppOpsManager mAppOps; diff --git a/services/camera/libcameraservice/api2/CameraDeviceClient.cpp b/services/camera/libcameraservice/api2/CameraDeviceClient.cpp index 28421ba90d..9deb662aeb 100644 --- a/services/camera/libcameraservice/api2/CameraDeviceClient.cpp +++ b/services/camera/libcameraservice/api2/CameraDeviceClient.cpp @@ -470,7 +470,8 @@ binder::Status CameraDeviceClient::beginConfigure() { } binder::Status CameraDeviceClient::endConfigure(int operatingMode, - const hardware::camera2::impl::CameraMetadataNative& sessionParams) { + const hardware::camera2::impl::CameraMetadataNative& sessionParams, + std::vector* offlineStreamIds /*out*/) { ATRACE_CALL(); ALOGV("%s: ending configure (%d input stream, %zu output surfaces)", __FUNCTION__, mInputStream.configured ? 1 : 0, @@ -479,6 +480,12 @@ binder::Status CameraDeviceClient::endConfigure(int operatingMode, binder::Status res; if (!(res = checkPidStatus(__FUNCTION__)).isOk()) return res; + if (offlineStreamIds == nullptr) { + String8 msg = String8::format("Invalid offline stream ids"); + ALOGE("%s: %s", __FUNCTION__, msg.string()); + return STATUS_ERROR(CameraService::ERROR_ILLEGAL_ARGUMENT, msg.string()); + } + Mutex::Autolock icl(mBinderSerializationLock); if (!mDevice.get()) { @@ -502,15 +509,41 @@ binder::Status CameraDeviceClient::endConfigure(int operatingMode, ALOGE("%s: %s", __FUNCTION__, msg.string()); res = STATUS_ERROR(CameraService::ERROR_INVALID_OPERATION, msg.string()); } else { + offlineStreamIds->clear(); + mDevice->getOfflineStreamIds(offlineStreamIds); + for (size_t i = 0; i < mCompositeStreamMap.size(); ++i) { err = mCompositeStreamMap.valueAt(i)->configureStream(); - if (err != OK ) { + if (err != OK) { String8 msg = String8::format("Camera %s: Error configuring composite " "streams: %s (%d)", mCameraIdStr.string(), strerror(-err), err); ALOGE("%s: %s", __FUNCTION__, msg.string()); res = STATUS_ERROR(CameraService::ERROR_INVALID_OPERATION, msg.string()); break; } + + // Composite streams can only support offline mode in case all individual internal + // streams are also supported. + std::vector internalStreams; + mCompositeStreamMap.valueAt(i)->insertCompositeStreamIds(&internalStreams); + std::remove_if(offlineStreamIds->begin(), offlineStreamIds->end(), + [&internalStreams] (int streamId) { + auto it = std::find(internalStreams.begin(), internalStreams.end(), + streamId); + if (it != internalStreams.end()) { + internalStreams.erase(it); + return true; + } + + return false; + }); + if (internalStreams.empty()) { + offlineStreamIds->push_back(mCompositeStreamMap.valueAt(i)->getStreamId()); + } + } + + for (const auto& offlineStreamId : *offlineStreamIds) { + mStreamInfoMap[offlineStreamId].supportsOffline = true; } } @@ -1903,7 +1936,7 @@ binder::Status CameraDeviceClient::getGlobalAudioRestriction(/*out*/ int32_t* ou binder::Status CameraDeviceClient::switchToOffline( const sp& cameraCb, - const std::vector& offlineOutputs, + const std::vector& offlineOutputIds, /*out*/ sp* session) { ATRACE_CALL(); @@ -1917,8 +1950,8 @@ binder::Status CameraDeviceClient::switchToOffline( return STATUS_ERROR(CameraService::ERROR_DISCONNECTED, "Camera device no longer alive"); } - if (offlineOutputs.empty()) { - String8 msg = String8::format("Offline outputs must not be empty"); + if (offlineOutputIds.empty()) { + String8 msg = String8::format("Offline surfaces must not be empty"); ALOGE("%s: %s", __FUNCTION__, msg.string()); return STATUS_ERROR(CameraService::ERROR_ILLEGAL_ARGUMENT, msg.string()); } @@ -1929,24 +1962,47 @@ binder::Status CameraDeviceClient::switchToOffline( return STATUS_ERROR(CameraService::ERROR_ILLEGAL_ARGUMENT, msg.string()); } - std::vector offlineStreamIds(offlineOutputs.size()); - for (auto& surface : offlineOutputs) { - sp binder = IInterface::asBinder(surface.graphicBufferProducer); - ssize_t index = mStreamMap.indexOfKey(binder); + std::vector offlineStreamIds(offlineOutputIds.size()); + KeyedVector, sp> offlineCompositeStreamMap; + for (const auto& streamId : offlineOutputIds) { + ssize_t index = mConfiguredOutputs.indexOfKey(streamId); if (index == NAME_NOT_FOUND) { - String8 msg = String8::format("Offline output is invalid"); + String8 msg = String8::format("Offline surface with id: %d is not registered", + streamId); ALOGE("%s: %s", __FUNCTION__, msg.string()); return STATUS_ERROR(CameraService::ERROR_ILLEGAL_ARGUMENT, msg.string()); } - // TODO: Also check whether the offline output is supported by Hal for offline mode. - sp s = new Surface(surface.graphicBufferProducer); - bool isCompositeStream = camera3::DepthCompositeStream::isDepthCompositeStream(s); - isCompositeStream |= camera3::HeicCompositeStream::isHeicCompositeStream(s); - if (isCompositeStream) { - // TODO: Add composite specific handling - } else { - offlineStreamIds.push_back(mStreamMap.valueAt(index).streamId()); + if (!mStreamInfoMap[streamId].supportsOffline) { + String8 msg = String8::format("Offline surface with id: %d doesn't support " + "offline mode", streamId); + ALOGE("%s: %s", __FUNCTION__, msg.string()); + return STATUS_ERROR(CameraService::ERROR_ILLEGAL_ARGUMENT, msg.string()); + } + + bool isCompositeStream = false; + for (const auto& gbp : mConfiguredOutputs[streamId].getGraphicBufferProducers()) { + sp s = new Surface(gbp, false /*controlledByApp*/); + isCompositeStream = camera3::DepthCompositeStream::isDepthCompositeStream(s) | + camera3::HeicCompositeStream::isHeicCompositeStream(s); + if (isCompositeStream) { + auto compositeIdx = mCompositeStreamMap.indexOfKey(IInterface::asBinder(gbp)); + if (compositeIdx == NAME_NOT_FOUND) { + ALOGE("%s: Unknown composite stream", __FUNCTION__); + return STATUS_ERROR(CameraService::ERROR_ILLEGAL_ARGUMENT, + "Unknown composite stream"); + } + + mCompositeStreamMap.valueAt(compositeIdx)->insertCompositeStreamIds( + &offlineStreamIds); + offlineCompositeStreamMap.add(mCompositeStreamMap.keyAt(compositeIdx), + mCompositeStreamMap.valueAt(compositeIdx)); + break; + } + } + + if (!isCompositeStream) { + offlineStreamIds.push_back(streamId); } } @@ -1958,16 +2014,36 @@ binder::Status CameraDeviceClient::switchToOffline( mCameraIdStr.string(), strerror(ret), ret); } - sp offlineClient = new CameraOfflineSessionClient(sCameraService, - offlineSession, cameraCb, mClientPackageName, mCameraIdStr, mClientPid, mClientUid, - mServicePid); - ret = offlineClient->initialize(); + sp offlineClient; + if (offlineSession.get() != nullptr) { + offlineClient = new CameraOfflineSessionClient(sCameraService, + offlineSession, offlineCompositeStreamMap, cameraCb, mClientPackageName, + mClientFeatureId, mCameraIdStr, mCameraFacing, mClientPid, mClientUid, mServicePid); + ret = sCameraService->addOfflineClient(mCameraIdStr, offlineClient); + } + if (ret == OK) { - // TODO: We need to update mStreamMap, mConfiguredOutputs + // A successful offline session switch must reset the current camera client + // and release any resources occupied by previously configured streams. + mStreamMap.clear(); + mConfiguredOutputs.clear(); + mDeferredStreams.clear(); + mStreamInfoMap.clear(); + mCompositeStreamMap.clear(); + mInputStream = {false, 0, 0, 0, 0}; } else { - return STATUS_ERROR_FMT(CameraService::ERROR_ILLEGAL_ARGUMENT, - "Camera %s: Failed to initilize offline session: %s (%d)", - mCameraIdStr.string(), strerror(ret), ret); + switch(ret) { + case BAD_VALUE: + return STATUS_ERROR_FMT(CameraService::ERROR_ILLEGAL_ARGUMENT, + "Illegal argument to HAL module for camera \"%s\"", mCameraIdStr.c_str()); + case TIMED_OUT: + return STATUS_ERROR_FMT(CameraService::ERROR_CAMERA_IN_USE, + "Camera \"%s\" is already open", mCameraIdStr.c_str()); + default: + return STATUS_ERROR_FMT(CameraService::ERROR_INVALID_OPERATION, + "Failed to initialize camera \"%s\": %s (%d)", mCameraIdStr.c_str(), + strerror(-ret), ret); + } } *session = offlineClient; diff --git a/services/camera/libcameraservice/api2/CameraDeviceClient.h b/services/camera/libcameraservice/api2/CameraDeviceClient.h index 0a8f37777c..a3a3189b88 100644 --- a/services/camera/libcameraservice/api2/CameraDeviceClient.h +++ b/services/camera/libcameraservice/api2/CameraDeviceClient.h @@ -92,7 +92,9 @@ public: virtual binder::Status beginConfigure() override; virtual binder::Status endConfigure(int operatingMode, - const hardware::camera2::impl::CameraMetadataNative& sessionParams) override; + const hardware::camera2::impl::CameraMetadataNative& sessionParams, + /*out*/ + std::vector* offlineStreamIds) override; // Verify specific session configuration. virtual binder::Status isSessionConfigurationSupported( @@ -160,7 +162,7 @@ public: virtual binder::Status switchToOffline( const sp& cameraCb, - const std::vector& offlineOutputs, + const std::vector& offlineOutputIds, /*out*/ sp* session) override; diff --git a/services/camera/libcameraservice/api2/CameraOfflineSessionClient.cpp b/services/camera/libcameraservice/api2/CameraOfflineSessionClient.cpp new file mode 100644 index 0000000000..af7b9e18dc --- /dev/null +++ b/services/camera/libcameraservice/api2/CameraOfflineSessionClient.cpp @@ -0,0 +1,287 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "CameraOfflineClient" +#define ATRACE_TAG ATRACE_TAG_CAMERA +//#define LOG_NDEBUG 0 + +#include "CameraOfflineSessionClient.h" +#include "utils/CameraThreadState.h" +#include + +namespace android { + +using binder::Status; + +status_t CameraOfflineSessionClient::initialize(sp, const String8&) { + ATRACE_CALL(); + + // Verify ops permissions + auto res = startCameraOps(); + if (res != OK) { + return res; + } + + if (mOfflineSession.get() == nullptr) { + ALOGE("%s: Camera %s: No valid offline session", + __FUNCTION__, mCameraIdStr.string()); + return NO_INIT; + } + + wp weakThis(this); + res = mOfflineSession->initialize(weakThis); + if (res != OK) { + ALOGE("%s: Camera %s: unable to initialize device: %s (%d)", + __FUNCTION__, mCameraIdStr.string(), strerror(-res), res); + return res; + } + + return OK; +} + +status_t CameraOfflineSessionClient::dump(int fd, const Vector& args) { + return BasicClient::dump(fd, args); +} + +status_t CameraOfflineSessionClient::dumpClient(int fd, const Vector& /*args*/) { + String8 result; + + result = " Offline session dump:\n"; + write(fd, result.string(), result.size()); + + if (mOfflineSession.get() == nullptr) { + result = " *** Offline session is detached\n"; + write(fd, result.string(), result.size()); + return NO_ERROR; + } + + auto res = mOfflineSession->dump(fd); + if (res != OK) { + result = String8::format(" Error dumping offline session: %s (%d)", + strerror(-res), res); + write(fd, result.string(), result.size()); + } + + return OK; +} + +binder::Status CameraOfflineSessionClient::disconnect() { + Mutex::Autolock icl(mBinderSerializationLock); + + binder::Status res = Status::ok(); + if (mDisconnected) { + return res; + } + // Allow both client and the media server to disconnect at all times + int callingPid = CameraThreadState::getCallingPid(); + if (callingPid != mClientPid && + callingPid != mServicePid) { + return res; + } + + mDisconnected = true; + + sCameraService->removeByClient(this); + sCameraService->logDisconnectedOffline(mCameraIdStr, mClientPid, String8(mClientPackageName)); + + sp remote = getRemote(); + if (remote != nullptr) { + remote->unlinkToDeath(sCameraService); + } + + finishCameraOps(); + ALOGI("%s: Disconnected client for offline camera %s for PID %d", __FUNCTION__, + mCameraIdStr.string(), mClientPid); + + // client shouldn't be able to call into us anymore + mClientPid = 0; + + if (mOfflineSession.get() != nullptr) { + auto ret = mOfflineSession->disconnect(); + if (ret != OK) { + ALOGE("%s: Failed disconnecting from offline session %s (%d)", __FUNCTION__, + strerror(-ret), ret); + } + mOfflineSession = nullptr; + } + + for (size_t i = 0; i < mCompositeStreamMap.size(); i++) { + auto ret = mCompositeStreamMap.valueAt(i)->deleteInternalStreams(); + if (ret != OK) { + ALOGE("%s: Failed removing composite stream %s (%d)", __FUNCTION__, + strerror(-ret), ret); + } + } + mCompositeStreamMap.clear(); + + return res; +} + +void CameraOfflineSessionClient::notifyError(int32_t errorCode, + const CaptureResultExtras& resultExtras) { + // Thread safe. Don't bother locking. + // Composites can have multiple internal streams. Error notifications coming from such internal + // streams may need to remain within camera service. + bool skipClientNotification = false; + for (size_t i = 0; i < mCompositeStreamMap.size(); i++) { + skipClientNotification |= mCompositeStreamMap.valueAt(i)->onError(errorCode, resultExtras); + } + + if ((mRemoteCallback.get() != nullptr) && (!skipClientNotification)) { + mRemoteCallback->onDeviceError(errorCode, resultExtras); + } +} + +status_t CameraOfflineSessionClient::startCameraOps() { + ATRACE_CALL(); + { + ALOGV("%s: Start camera ops, package name = %s, client UID = %d", + __FUNCTION__, String8(mClientPackageName).string(), mClientUid); + } + + if (mAppOpsManager != nullptr) { + // Notify app ops that the camera is not available + mOpsCallback = new OpsCallback(this); + int32_t res; + // TODO : possibly change this to OP_OFFLINE_CAMERA_SESSION + mAppOpsManager->startWatchingMode(AppOpsManager::OP_CAMERA, + mClientPackageName, mOpsCallback); + // TODO : possibly change this to OP_OFFLINE_CAMERA_SESSION + res = mAppOpsManager->startOpNoThrow(AppOpsManager::OP_CAMERA, + mClientUid, mClientPackageName, /*startIfModeDefault*/ false); + + if (res == AppOpsManager::MODE_ERRORED) { + ALOGI("Offline Camera %s: Access for \"%s\" has been revoked", + mCameraIdStr.string(), String8(mClientPackageName).string()); + return PERMISSION_DENIED; + } + + if (res == AppOpsManager::MODE_IGNORED) { + ALOGI("Offline Camera %s: Access for \"%s\" has been restricted", + mCameraIdStr.string(), String8(mClientPackageName).string()); + // Return the same error as for device policy manager rejection + return -EACCES; + } + } + + mOpsActive = true; + + // Transition device state to OPEN + sCameraService->mUidPolicy->registerMonitorUid(mClientUid); + + return OK; +} + +status_t CameraOfflineSessionClient::finishCameraOps() { + ATRACE_CALL(); + + // Check if startCameraOps succeeded, and if so, finish the camera op + if (mOpsActive) { + // Notify app ops that the camera is available again + if (mAppOpsManager != nullptr) { + // TODO : possibly change this to OP_OFFLINE_CAMERA_SESSION + mAppOpsManager->finishOp(AppOpsManager::OP_CAMERA, mClientUid, + mClientPackageName); + mOpsActive = false; + } + } + // Always stop watching, even if no camera op is active + if (mOpsCallback != nullptr && mAppOpsManager != nullptr) { + mAppOpsManager->stopWatchingMode(mOpsCallback); + } + mOpsCallback.clear(); + + sCameraService->mUidPolicy->unregisterMonitorUid(mClientUid); + + return OK; +} + +void CameraOfflineSessionClient::onResultAvailable(const CaptureResult& result) { + ATRACE_CALL(); + ALOGV("%s", __FUNCTION__); + + if (mRemoteCallback.get() != NULL) { + mRemoteCallback->onResultReceived(result.mMetadata, result.mResultExtras, + result.mPhysicalMetadatas); + } + + for (size_t i = 0; i < mCompositeStreamMap.size(); i++) { + mCompositeStreamMap.valueAt(i)->onResultAvailable(result); + } +} + +void CameraOfflineSessionClient::notifyShutter(const CaptureResultExtras& resultExtras, + nsecs_t timestamp) { + + if (mRemoteCallback.get() != nullptr) { + mRemoteCallback->onCaptureStarted(resultExtras, timestamp); + } + + for (size_t i = 0; i < mCompositeStreamMap.size(); i++) { + mCompositeStreamMap.valueAt(i)->onShutter(resultExtras, timestamp); + } +} + +void CameraOfflineSessionClient::notifyIdle() { + if (mRemoteCallback.get() != nullptr) { + mRemoteCallback->onDeviceIdle(); + } +} + +void CameraOfflineSessionClient::notifyAutoFocus(uint8_t newState, int triggerId) { + (void)newState; + (void)triggerId; + + ALOGV("%s: Autofocus state now %d, last trigger %d", + __FUNCTION__, newState, triggerId); +} + +void CameraOfflineSessionClient::notifyAutoExposure(uint8_t newState, int triggerId) { + (void)newState; + (void)triggerId; + + ALOGV("%s: Autoexposure state now %d, last trigger %d", + __FUNCTION__, newState, triggerId); +} + +void CameraOfflineSessionClient::notifyAutoWhitebalance(uint8_t newState, int triggerId) { + (void)newState; + (void)triggerId; + + ALOGV("%s: Auto-whitebalance state now %d, last trigger %d", __FUNCTION__, newState, + triggerId); +} + +void CameraOfflineSessionClient::notifyPrepared(int /*streamId*/) { + ALOGE("%s: Unexpected stream prepare notification in offline mode!", __FUNCTION__); + notifyError(hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_DEVICE, + CaptureResultExtras()); +} + +void CameraOfflineSessionClient::notifyRequestQueueEmpty() { + if (mRemoteCallback.get() != nullptr) { + mRemoteCallback->onRequestQueueEmpty(); + } +} + +void CameraOfflineSessionClient::notifyRepeatingRequestError(long /*lastFrameNumber*/) { + ALOGE("%s: Unexpected repeating request error in offline mode!", __FUNCTION__); + notifyError(hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_DEVICE, + CaptureResultExtras()); +} + +// ---------------------------------------------------------------------------- +}; // namespace android diff --git a/services/camera/libcameraservice/api2/CameraOfflineSessionClient.h b/services/camera/libcameraservice/api2/CameraOfflineSessionClient.h index cb83e2907a..b0f000da7f 100644 --- a/services/camera/libcameraservice/api2/CameraOfflineSessionClient.h +++ b/services/camera/libcameraservice/api2/CameraOfflineSessionClient.h @@ -19,65 +19,88 @@ #include #include +#include "common/FrameProcessorBase.h" +#include "common/CameraDeviceBase.h" #include "CameraService.h" +#include "CompositeStream.h" namespace android { using android::hardware::camera2::ICameraDeviceCallbacks; - +using camera3::CompositeStream; + +// Client for offline session. Note that offline session client does not affect camera service's +// client arbitration logic. It is camera HAL's decision to decide whether a normal camera +// client is conflicting with existing offline client(s). +// The other distinctive difference between offline clients and normal clients is that normal +// clients are created through ICameraService binder calls, while the offline session client +// is created through ICameraDeviceUser::switchToOffline call. class CameraOfflineSessionClient : - public CameraService::OfflineClient, - public hardware::camera2::BnCameraOfflineSession - // public camera2::FrameProcessorBase::FilteredListener? + public CameraService::BasicClient, + public hardware::camera2::BnCameraOfflineSession, + public camera2::FrameProcessorBase::FilteredListener, + public NotificationListener { public: CameraOfflineSessionClient( const sp& cameraService, sp session, + const KeyedVector, sp>& offlineCompositeStreamMap, const sp& remoteCallback, const String16& clientPackageName, - const String8& cameraIdStr, + const std::unique_ptr& clientFeatureId, + const String8& cameraIdStr, int cameraFacing, int clientPid, uid_t clientUid, int servicePid) : - CameraService::OfflineClient(cameraService, clientPackageName, - cameraIdStr, clientPid, clientUid, servicePid), - mRemoteCallback(remoteCallback), mOfflineSession(session) {} - - ~CameraOfflineSessionClient() {} + CameraService::BasicClient( + cameraService, + IInterface::asBinder(remoteCallback), + clientPackageName, clientFeatureId, + cameraIdStr, cameraFacing, clientPid, clientUid, servicePid), + mRemoteCallback(remoteCallback), mOfflineSession(session), + mCompositeStreamMap(offlineCompositeStreamMap) {} + + virtual ~CameraOfflineSessionClient() {} + + sp asBinderWrapper() override { + return IInterface::asBinder(this); + } - virtual binder::Status disconnect() override { return binder::Status::ok(); } + binder::Status disconnect() override; - virtual status_t dump(int /*fd*/, const Vector& /*args*/) override { - return OK; - } + status_t dump(int /*fd*/, const Vector& /*args*/) override; - // Block the client form using the camera - virtual void block() override {}; + status_t dumpClient(int /*fd*/, const Vector& /*args*/) override; - // Return the package name for this client - virtual String16 getPackageName() const override { String16 ret; return ret; }; + status_t initialize(sp /*manager*/, + const String8& /*monitorTags*/) override; - // Notify client about a fatal error - // TODO: maybe let impl notify within block? - virtual void notifyError(int32_t /*errorCode*/, - const CaptureResultExtras& /*resultExtras*/) override {} + // permissions management + status_t startCameraOps() override; + status_t finishCameraOps() override; - // Get the UID of the application client using this - virtual uid_t getClientUid() const override { return 0; } + // FilteredResultListener API + void onResultAvailable(const CaptureResult& result) override; - // Get the PID of the application client using this - virtual int getClientPid() const override { return 0; } + // NotificationListener API + void notifyError(int32_t errorCode, const CaptureResultExtras& resultExtras) override; + void notifyShutter(const CaptureResultExtras& resultExtras, nsecs_t timestamp) override; + void notifyIdle() override; + void notifyAutoFocus(uint8_t newState, int triggerId) override; + void notifyAutoExposure(uint8_t newState, int triggerId) override; + void notifyAutoWhitebalance(uint8_t newState, int triggerId) override; + void notifyPrepared(int streamId) override; + void notifyRequestQueueEmpty() override; + void notifyRepeatingRequestError(long lastFrameNumber) override; - status_t initialize() { - // TODO: Talk to camera service to add the offline session client book keeping - return OK; - } private: - sp mSession; + mutable Mutex mBinderSerializationLock; sp mRemoteCallback; - // This class is responsible to convert HAL callbacks to AIDL callbacks sp mOfflineSession; + + // Offline composite stream map, output surface -> composite stream + KeyedVector, sp> mCompositeStreamMap; }; } // namespace android diff --git a/services/camera/libcameraservice/api2/CompositeStream.h b/services/camera/libcameraservice/api2/CompositeStream.h index a401a8258c..de894f318f 100644 --- a/services/camera/libcameraservice/api2/CompositeStream.h +++ b/services/camera/libcameraservice/api2/CompositeStream.h @@ -64,6 +64,10 @@ public: virtual status_t insertGbp(SurfaceMap* /*out*/outSurfaceMap, Vector* /*out*/outputStreamIds, int32_t* /*out*/currentStreamId) = 0; + // Attach the internal composite stream ids. + virtual status_t insertCompositeStreamIds( + std::vector* compositeStreamIds /*out*/) = 0; + // Return composite stream id. virtual int getStreamId() = 0; diff --git a/services/camera/libcameraservice/api2/DepthCompositeStream.cpp b/services/camera/libcameraservice/api2/DepthCompositeStream.cpp index ac6f8d6e26..acad8c6a3a 100644 --- a/services/camera/libcameraservice/api2/DepthCompositeStream.cpp +++ b/services/camera/libcameraservice/api2/DepthCompositeStream.cpp @@ -683,6 +683,18 @@ status_t DepthCompositeStream::insertGbp(SurfaceMap* /*out*/outSurfaceMap, return NO_ERROR; } +status_t DepthCompositeStream::insertCompositeStreamIds( + std::vector* compositeStreamIds /*out*/) { + if (compositeStreamIds == nullptr) { + return BAD_VALUE; + } + + compositeStreamIds->push_back(mDepthStreamId); + compositeStreamIds->push_back(mBlobStreamId); + + return OK; +} + void DepthCompositeStream::onResultError(const CaptureResultExtras& resultExtras) { // Processing can continue even in case of result errors. // At the moment depth composite stream processing relies mainly on static camera diff --git a/services/camera/libcameraservice/api2/DepthCompositeStream.h b/services/camera/libcameraservice/api2/DepthCompositeStream.h index 60c6b1e715..1bf714da09 100644 --- a/services/camera/libcameraservice/api2/DepthCompositeStream.h +++ b/services/camera/libcameraservice/api2/DepthCompositeStream.h @@ -56,6 +56,7 @@ public: status_t configureStream() override; status_t insertGbp(SurfaceMap* /*out*/outSurfaceMap, Vector* /*out*/outputStreamIds, int32_t* /*out*/currentStreamId) override; + status_t insertCompositeStreamIds(std::vector* compositeStreamIds /*out*/) override; int getStreamId() override { return mBlobStreamId; } // CpuConsumer listener implementation diff --git a/services/camera/libcameraservice/api2/HeicCompositeStream.cpp b/services/camera/libcameraservice/api2/HeicCompositeStream.cpp index 9cdad700fe..d25e467fb7 100644 --- a/services/camera/libcameraservice/api2/HeicCompositeStream.cpp +++ b/services/camera/libcameraservice/api2/HeicCompositeStream.cpp @@ -504,6 +504,18 @@ status_t HeicCompositeStream::insertGbp(SurfaceMap* /*out*/outSurfaceMap, return NO_ERROR; } +status_t HeicCompositeStream::insertCompositeStreamIds( + std::vector* compositeStreamIds /*out*/) { + if (compositeStreamIds == nullptr) { + return BAD_VALUE; + } + + compositeStreamIds->push_back(mAppSegmentStreamId); + compositeStreamIds->push_back(mMainImageStreamId); + + return OK; +} + void HeicCompositeStream::onShutter(const CaptureResultExtras& resultExtras, nsecs_t timestamp) { Mutex::Autolock l(mMutex); if (mErrorState) { diff --git a/services/camera/libcameraservice/api2/HeicCompositeStream.h b/services/camera/libcameraservice/api2/HeicCompositeStream.h index e88471d74d..8fc521e69d 100644 --- a/services/camera/libcameraservice/api2/HeicCompositeStream.h +++ b/services/camera/libcameraservice/api2/HeicCompositeStream.h @@ -55,6 +55,8 @@ public: status_t insertGbp(SurfaceMap* /*out*/outSurfaceMap, Vector* /*out*/outputStreamIds, int32_t* /*out*/currentStreamId) override; + status_t insertCompositeStreamIds(std::vector* compositeStreamIds /*out*/) override; + void onShutter(const CaptureResultExtras& resultExtras, nsecs_t timestamp) override; int getStreamId() override { return mMainImageStreamId; } diff --git a/services/camera/libcameraservice/common/Camera2ClientBase.cpp b/services/camera/libcameraservice/common/Camera2ClientBase.cpp index 8792f9a160..0a41776a3e 100644 --- a/services/camera/libcameraservice/common/Camera2ClientBase.cpp +++ b/services/camera/libcameraservice/common/Camera2ClientBase.cpp @@ -111,7 +111,7 @@ status_t Camera2ClientBase::initializeImpl(TProviderPtr providerPtr return res; } - wp weakThis(this); + wp weakThis(this); res = mDevice->setNotifyCallback(weakThis); return OK; diff --git a/services/camera/libcameraservice/common/Camera2ClientBase.h b/services/camera/libcameraservice/common/Camera2ClientBase.h index 12cba0b1dc..042f5aa3b7 100644 --- a/services/camera/libcameraservice/common/Camera2ClientBase.h +++ b/services/camera/libcameraservice/common/Camera2ClientBase.h @@ -29,7 +29,7 @@ class CameraService; template class Camera2ClientBase : public TClientBase, - public CameraDeviceBase::NotificationListener + public NotificationListener { public: typedef typename TClientBase::TCamCallbacks TCamCallbacks; @@ -61,7 +61,7 @@ public: virtual status_t dumpClient(int fd, const Vector& args); /** - * CameraDeviceBase::NotificationListener implementation + * NotificationListener implementation */ virtual void notifyError(int32_t errorCode, diff --git a/services/camera/libcameraservice/common/CameraDeviceBase.cpp b/services/camera/libcameraservice/common/CameraDeviceBase.cpp index 6c4e87f630..0efe4bcc4d 100644 --- a/services/camera/libcameraservice/common/CameraDeviceBase.cpp +++ b/services/camera/libcameraservice/common/CameraDeviceBase.cpp @@ -24,7 +24,4 @@ namespace android { CameraDeviceBase::~CameraDeviceBase() { } -CameraDeviceBase::NotificationListener::~NotificationListener() { -} - } // namespace android diff --git a/services/camera/libcameraservice/common/CameraDeviceBase.h b/services/camera/libcameraservice/common/CameraDeviceBase.h index 1026fdf2b3..2f011984f1 100644 --- a/services/camera/libcameraservice/common/CameraDeviceBase.h +++ b/services/camera/libcameraservice/common/CameraDeviceBase.h @@ -234,6 +234,12 @@ class CameraDeviceBase : public virtual RefBase { virtual status_t configureStreams(const CameraMetadata& sessionParams, int operatingMode = 0) = 0; + /** + * Retrieve a list of all stream ids that were advertised as capable of + * supporting offline processing mode by Hal after the last stream configuration. + */ + virtual void getOfflineStreamIds(std::vector *offlineStreamIds) = 0; + // get the buffer producer of the input stream virtual status_t getInputBufferProducer( sp *producer) = 0; @@ -258,35 +264,6 @@ class CameraDeviceBase : public virtual RefBase { */ virtual ssize_t getJpegBufferSize(uint32_t width, uint32_t height) const = 0; - /** - * Abstract class for HAL notification listeners - */ - class NotificationListener : public virtual RefBase { - public: - // The set of notifications is a merge of the notifications required for - // API1 and API2. - - // Required for API 1 and 2 - virtual void notifyError(int32_t errorCode, - const CaptureResultExtras &resultExtras) = 0; - - // Required only for API2 - virtual void notifyIdle() = 0; - virtual void notifyShutter(const CaptureResultExtras &resultExtras, - nsecs_t timestamp) = 0; - virtual void notifyPrepared(int streamId) = 0; - virtual void notifyRequestQueueEmpty() = 0; - - // Required only for API1 - virtual void notifyAutoFocus(uint8_t newState, int triggerId) = 0; - virtual void notifyAutoExposure(uint8_t newState, int triggerId) = 0; - virtual void notifyAutoWhitebalance(uint8_t newState, - int triggerId) = 0; - virtual void notifyRepeatingRequestError(long lastFrameNumber) = 0; - protected: - virtual ~NotificationListener(); - }; - /** * Connect HAL notifications to a listener. Overwrites previous * listener. Set to NULL to stop receiving notifications. diff --git a/services/camera/libcameraservice/common/CameraOfflineSessionBase.h b/services/camera/libcameraservice/common/CameraOfflineSessionBase.h index 38625218f3..05ea7fb2b2 100644 --- a/services/camera/libcameraservice/common/CameraOfflineSessionBase.h +++ b/services/camera/libcameraservice/common/CameraOfflineSessionBase.h @@ -25,10 +25,42 @@ namespace android { +/** + * Abstract class for HAL notification listeners + */ +class NotificationListener : public virtual RefBase { + public: + // The set of notifications is a merge of the notifications required for + // API1 and API2. + + // Required for API 1 and 2 + virtual void notifyError(int32_t errorCode, + const CaptureResultExtras &resultExtras) = 0; + + // Required only for API2 + virtual void notifyIdle() = 0; + virtual void notifyShutter(const CaptureResultExtras &resultExtras, + nsecs_t timestamp) = 0; + virtual void notifyPrepared(int streamId) = 0; + virtual void notifyRequestQueueEmpty() = 0; + + // Required only for API1 + virtual void notifyAutoFocus(uint8_t newState, int triggerId) = 0; + virtual void notifyAutoExposure(uint8_t newState, int triggerId) = 0; + virtual void notifyAutoWhitebalance(uint8_t newState, + int triggerId) = 0; + virtual void notifyRepeatingRequestError(long lastFrameNumber) = 0; + protected: + virtual ~NotificationListener() {} +}; + class CameraOfflineSessionBase : public virtual RefBase { public: virtual ~CameraOfflineSessionBase(); + virtual status_t initialize( + wp listener) = 0; + // The session's original camera ID virtual const String8& getId() const = 0; @@ -36,8 +68,6 @@ class CameraOfflineSessionBase : public virtual RefBase { virtual status_t dump(int fd) = 0; - virtual status_t abort() = 0; - /** * Capture result passing */ diff --git a/services/camera/libcameraservice/device3/BufferUtils.cpp b/services/camera/libcameraservice/device3/BufferUtils.cpp new file mode 100644 index 0000000000..cc2939062e --- /dev/null +++ b/services/camera/libcameraservice/device3/BufferUtils.cpp @@ -0,0 +1,277 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "Camera3-BufUtils" +#define ATRACE_TAG ATRACE_TAG_CAMERA +//#define LOG_NDEBUG 0 +//#define LOG_NNDEBUG 0 // Per-frame verbose logging + +#include + +#include + +#include "device3/BufferUtils.h" + +namespace android { +namespace camera3 { + +camera3_buffer_status_t mapHidlBufferStatus(hardware::camera::device::V3_2::BufferStatus status) { + using hardware::camera::device::V3_2::BufferStatus; + + switch (status) { + case BufferStatus::OK: return CAMERA3_BUFFER_STATUS_OK; + case BufferStatus::ERROR: return CAMERA3_BUFFER_STATUS_ERROR; + } + return CAMERA3_BUFFER_STATUS_ERROR; +} + +void BufferRecords::takeInflightBufferMap(BufferRecords& other) { + std::lock_guard oLock(other.mInflightLock); + std::lock_guard lock(mInflightLock); + if (mInflightBufferMap.size() > 0) { + ALOGE("%s: inflight map is set in non-empty state!", __FUNCTION__); + } + mInflightBufferMap = std::move(other.mInflightBufferMap); + other.mInflightBufferMap.clear(); +} + +void BufferRecords::takeRequestedBufferMap(BufferRecords& other) { + std::lock_guard oLock(other.mRequestedBuffersLock); + std::lock_guard lock(mRequestedBuffersLock); + if (mRequestedBufferMap.size() > 0) { + ALOGE("%s: requested buffer map is set in non-empty state!", __FUNCTION__); + } + mRequestedBufferMap = std::move(other.mRequestedBufferMap); + other.mRequestedBufferMap.clear(); +} + +void BufferRecords::takeBufferCaches(BufferRecords& other, const std::vector& streams) { + std::lock_guard oLock(other.mBufferIdMapLock); + std::lock_guard lock(mBufferIdMapLock); + if (mBufferIdMaps.size() > 0) { + ALOGE("%s: buffer ID map is set in non-empty state!", __FUNCTION__); + } + for (auto streamId : streams) { + mBufferIdMaps.insert({streamId, std::move(other.mBufferIdMaps.at(streamId))}); + } + other.mBufferIdMaps.clear(); +} + +std::pair BufferRecords::getBufferId( + const buffer_handle_t& buf, int streamId) { + std::lock_guard lock(mBufferIdMapLock); + + BufferIdMap& bIdMap = mBufferIdMaps.at(streamId); + auto it = bIdMap.find(buf); + if (it == bIdMap.end()) { + bIdMap[buf] = mNextBufferId++; + ALOGV("stream %d now have %zu buffer caches, buf %p", + streamId, bIdMap.size(), buf); + return std::make_pair(true, mNextBufferId - 1); + } else { + return std::make_pair(false, it->second); + } +} + +void BufferRecords::tryCreateBufferCache(int streamId) { + std::lock_guard lock(mBufferIdMapLock); + if (mBufferIdMaps.count(streamId) == 0) { + mBufferIdMaps.emplace(streamId, BufferIdMap{}); + } +} + +void BufferRecords::removeInactiveBufferCaches(const std::set& activeStreams) { + std::lock_guard lock(mBufferIdMapLock); + for(auto it = mBufferIdMaps.begin(); it != mBufferIdMaps.end();) { + int streamId = it->first; + bool active = activeStreams.count(streamId) > 0; + if (!active) { + it = mBufferIdMaps.erase(it); + } else { + ++it; + } + } +} + +uint64_t BufferRecords::removeOneBufferCache(int streamId, const native_handle_t* handle) { + std::lock_guard lock(mBufferIdMapLock); + uint64_t bufferId = BUFFER_ID_NO_BUFFER; + auto mapIt = mBufferIdMaps.find(streamId); + if (mapIt == mBufferIdMaps.end()) { + // streamId might be from a deleted stream here + ALOGI("%s: stream %d has been removed", + __FUNCTION__, streamId); + return BUFFER_ID_NO_BUFFER; + } + BufferIdMap& bIdMap = mapIt->second; + auto it = bIdMap.find(handle); + if (it == bIdMap.end()) { + ALOGW("%s: cannot find buffer %p in stream %d", + __FUNCTION__, handle, streamId); + return BUFFER_ID_NO_BUFFER; + } else { + bufferId = it->second; + bIdMap.erase(it); + ALOGV("%s: stream %d now have %zu buffer caches after removing buf %p", + __FUNCTION__, streamId, bIdMap.size(), handle); + } + return bufferId; +} + +std::vector BufferRecords::clearBufferCaches(int streamId) { + std::lock_guard lock(mBufferIdMapLock); + std::vector ret; + auto mapIt = mBufferIdMaps.find(streamId); + if (mapIt == mBufferIdMaps.end()) { + ALOGE("%s: streamId %d not found!", __FUNCTION__, streamId); + return ret; + } + BufferIdMap& bIdMap = mapIt->second; + ret.reserve(bIdMap.size()); + for (const auto& it : bIdMap) { + ret.push_back(it.second); + } + bIdMap.clear(); + return ret; +} + +bool BufferRecords::isStreamCached(int streamId) { + std::lock_guard lock(mBufferIdMapLock); + return mBufferIdMaps.find(streamId) != mBufferIdMaps.end(); +} + +bool BufferRecords::verifyBufferIds( + int32_t streamId, std::vector& bufIds) { + std::lock_guard lock(mBufferIdMapLock); + camera3::BufferIdMap& bIdMap = mBufferIdMaps.at(streamId); + if (bIdMap.size() != bufIds.size()) { + ALOGE("%s: stream ID %d buffer cache number mismatch: %zu/%zu (service/HAL)", + __FUNCTION__, streamId, bIdMap.size(), bufIds.size()); + return false; + } + std::vector internalBufIds; + internalBufIds.reserve(bIdMap.size()); + for (const auto& pair : bIdMap) { + internalBufIds.push_back(pair.second); + } + std::sort(bufIds.begin(), bufIds.end()); + std::sort(internalBufIds.begin(), internalBufIds.end()); + for (size_t i = 0; i < bufIds.size(); i++) { + if (bufIds[i] != internalBufIds[i]) { + ALOGE("%s: buffer cache mismatch! Service %" PRIu64 ", HAL %" PRIu64, + __FUNCTION__, internalBufIds[i], bufIds[i]); + return false; + } + } + return true; +} + +void BufferRecords::getInflightBufferKeys( + std::vector>* out) { + std::lock_guard lock(mInflightLock); + out->clear(); + out->reserve(mInflightBufferMap.size()); + for (auto& pair : mInflightBufferMap) { + uint64_t key = pair.first; + int32_t streamId = key & 0xFFFFFFFF; + int32_t frameNumber = (key >> 32) & 0xFFFFFFFF; + out->push_back(std::make_pair(frameNumber, streamId)); + } + return; +} + +status_t BufferRecords::pushInflightBuffer( + int32_t frameNumber, int32_t streamId, buffer_handle_t *buffer) { + std::lock_guard lock(mInflightLock); + uint64_t key = static_cast(frameNumber) << 32 | static_cast(streamId); + mInflightBufferMap[key] = buffer; + return OK; +} + +status_t BufferRecords::popInflightBuffer( + int32_t frameNumber, int32_t streamId, + /*out*/ buffer_handle_t **buffer) { + std::lock_guard lock(mInflightLock); + + uint64_t key = static_cast(frameNumber) << 32 | static_cast(streamId); + auto it = mInflightBufferMap.find(key); + if (it == mInflightBufferMap.end()) return NAME_NOT_FOUND; + if (buffer != nullptr) { + *buffer = it->second; + } + mInflightBufferMap.erase(it); + return OK; +} + +void BufferRecords::popInflightBuffers( + const std::vector>& buffers) { + for (const auto& pair : buffers) { + int32_t frameNumber = pair.first; + int32_t streamId = pair.second; + popInflightBuffer(frameNumber, streamId, nullptr); + } +} + +status_t BufferRecords::pushInflightRequestBuffer( + uint64_t bufferId, buffer_handle_t* buf, int32_t streamId) { + std::lock_guard lock(mRequestedBuffersLock); + auto pair = mRequestedBufferMap.insert({bufferId, {streamId, buf}}); + if (!pair.second) { + ALOGE("%s: bufId %" PRIu64 " is already inflight!", + __FUNCTION__, bufferId); + return BAD_VALUE; + } + return OK; +} + +// Find and pop a buffer_handle_t based on bufferId +status_t BufferRecords::popInflightRequestBuffer( + uint64_t bufferId, + /*out*/ buffer_handle_t** buffer, + /*optional out*/ int32_t* streamId) { + if (buffer == nullptr) { + ALOGE("%s: buffer (%p) must not be null", __FUNCTION__, buffer); + return BAD_VALUE; + } + std::lock_guard lock(mRequestedBuffersLock); + auto it = mRequestedBufferMap.find(bufferId); + if (it == mRequestedBufferMap.end()) { + ALOGE("%s: bufId %" PRIu64 " is not inflight!", + __FUNCTION__, bufferId); + return BAD_VALUE; + } + *buffer = it->second.second; + if (streamId != nullptr) { + *streamId = it->second.first; + } + mRequestedBufferMap.erase(it); + return OK; +} + +void BufferRecords::getInflightRequestBufferKeys( + std::vector* out) { + std::lock_guard lock(mRequestedBuffersLock); + out->clear(); + out->reserve(mRequestedBufferMap.size()); + for (auto& pair : mRequestedBufferMap) { + out->push_back(pair.first); + } + return; +} + + +} // camera3 +} // namespace android diff --git a/services/camera/libcameraservice/device3/BufferUtils.h b/services/camera/libcameraservice/device3/BufferUtils.h new file mode 100644 index 0000000000..452a9080c7 --- /dev/null +++ b/services/camera/libcameraservice/device3/BufferUtils.h @@ -0,0 +1,167 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ANDROID_SERVERS_CAMERA3_BUFFER_UTILS_H +#define ANDROID_SERVERS_CAMERA3_BUFFER_UTILS_H + +#include +#include +#include + +#include + +#include + +// TODO: remove legacy camera3.h references +#include "hardware/camera3.h" + +#include + +namespace android { + +namespace camera3 { + + struct BufferHasher { + size_t operator()(const buffer_handle_t& buf) const { + if (buf == nullptr) + return 0; + + size_t result = 1; + result = 31 * result + buf->numFds; + for (int i = 0; i < buf->numFds; i++) { + result = 31 * result + buf->data[i]; + } + return result; + } + }; + + struct BufferComparator { + bool operator()(const buffer_handle_t& buf1, const buffer_handle_t& buf2) const { + if (buf1->numFds == buf2->numFds) { + for (int i = 0; i < buf1->numFds; i++) { + if (buf1->data[i] != buf2->data[i]) { + return false; + } + } + return true; + } + return false; + } + }; + + // Per stream buffer native handle -> bufId map + typedef std::unordered_map BufferIdMap; + + // streamId -> BufferIdMap + typedef std::unordered_map BufferIdMaps; + + // Map of inflight buffers sent along in capture requests. + // Key is composed by (frameNumber << 32 | streamId) + typedef std::unordered_map InflightBufferMap; + + // Map of inflight buffers dealt by requestStreamBuffers API + typedef std::unordered_map> RequestedBufferMap; + + // A struct containing all buffer tracking information like inflight buffers + // and buffer ID caches + class BufferRecords : public BufferRecordsInterface { + + public: + BufferRecords() {} + + BufferRecords(BufferRecords&& other) : + mBufferIdMaps(other.mBufferIdMaps), + mNextBufferId(other.mNextBufferId), + mInflightBufferMap(other.mInflightBufferMap), + mRequestedBufferMap(other.mRequestedBufferMap) {} + + virtual ~BufferRecords() {} + + // Helper methods to help moving buffer records + void takeInflightBufferMap(BufferRecords& other); + void takeRequestedBufferMap(BufferRecords& other); + void takeBufferCaches(BufferRecords& other, const std::vector& streams); + + // method to extract buffer's unique ID + // return pair of (newlySeenBuffer?, bufferId) + virtual std::pair getBufferId( + const buffer_handle_t& buf, int streamId) override; + + void tryCreateBufferCache(int streamId); + + void removeInactiveBufferCaches(const std::set& activeStreams); + + // Return the removed buffer ID if input cache is found. + // Otherwise return BUFFER_ID_NO_BUFFER + uint64_t removeOneBufferCache(int streamId, const native_handle_t* handle); + + // Clear all caches for input stream, but do not remove the stream + // Removed buffers' ID are returned + std::vector clearBufferCaches(int streamId); + + bool isStreamCached(int streamId); + + // Return true if the input caches match what we have; otherwise false + bool verifyBufferIds(int32_t streamId, std::vector& inBufIds); + + // Get a vector of (frameNumber, streamId) pair of currently inflight + // buffers + void getInflightBufferKeys(std::vector>* out); + + status_t pushInflightBuffer(int32_t frameNumber, int32_t streamId, + buffer_handle_t *buffer); + + // Find a buffer_handle_t based on frame number and stream ID + virtual status_t popInflightBuffer(int32_t frameNumber, int32_t streamId, + /*out*/ buffer_handle_t **buffer) override; + + // Pop inflight buffers based on pairs of (frameNumber,streamId) + void popInflightBuffers(const std::vector>& buffers); + + // Get a vector of bufferId of currently inflight buffers + void getInflightRequestBufferKeys(std::vector* out); + + // Register a bufId (streamId, buffer_handle_t) to inflight request buffer + virtual status_t pushInflightRequestBuffer( + uint64_t bufferId, buffer_handle_t* buf, int32_t streamId) override; + + // Find a buffer_handle_t based on bufferId + virtual status_t popInflightRequestBuffer(uint64_t bufferId, + /*out*/ buffer_handle_t** buffer, + /*optional out*/ int32_t* streamId = nullptr) override; + + private: + std::mutex mBufferIdMapLock; + BufferIdMaps mBufferIdMaps; + uint64_t mNextBufferId = 1; // 0 means no buffer + + std::mutex mInflightLock; + InflightBufferMap mInflightBufferMap; + + std::mutex mRequestedBuffersLock; + RequestedBufferMap mRequestedBufferMap; + }; // class BufferRecords + + static const uint64_t BUFFER_ID_NO_BUFFER = 0; + + camera3_buffer_status_t mapHidlBufferStatus( + hardware::camera::device::V3_2::BufferStatus status); +} // namespace camera3 + +} // namespace android + +#endif diff --git a/services/camera/libcameraservice/device3/Camera3Device.cpp b/services/camera/libcameraservice/device3/Camera3Device.cpp index 92ba5e4bd6..23e26ce65a 100644 --- a/services/camera/libcameraservice/device3/Camera3Device.cpp +++ b/services/camera/libcameraservice/device3/Camera3Device.cpp @@ -58,10 +58,10 @@ #include "device3/Camera3InputStream.h" #include "device3/Camera3DummyStream.h" #include "device3/Camera3SharedOutputStream.h" -#include "device3/Camera3OfflineSession.h" #include "CameraService.h" #include "utils/CameraThreadState.h" +#include #include using namespace android::camera3; @@ -574,14 +574,6 @@ status_t Camera3Device::mapToStreamConfigurationMode( return OK; } -camera3_buffer_status_t Camera3Device::mapHidlBufferStatus(BufferStatus status) { - switch (status) { - case BufferStatus::OK: return CAMERA3_BUFFER_STATUS_OK; - case BufferStatus::ERROR: return CAMERA3_BUFFER_STATUS_ERROR; - } - return CAMERA3_BUFFER_STATUS_ERROR; -} - int Camera3Device::mapToFrameworkFormat( hardware::graphics::common::V1_0::PixelFormat pixelFormat) { return static_cast(pixelFormat); @@ -1001,230 +993,18 @@ status_t Camera3Device::submitRequestsHelper( hardware::Return Camera3Device::requestStreamBuffers( const hardware::hidl_vec& bufReqs, requestStreamBuffers_cb _hidl_cb) { - using hardware::camera::device::V3_5::BufferRequestStatus; - using hardware::camera::device::V3_5::StreamBufferRet; - using hardware::camera::device::V3_5::StreamBufferRequestError; - - std::lock_guard lock(mRequestBufferInterfaceLock); - - hardware::hidl_vec bufRets; - if (!mUseHalBufManager) { - ALOGE("%s: Camera %s does not support HAL buffer management", - __FUNCTION__, mId.string()); - _hidl_cb(BufferRequestStatus::FAILED_ILLEGAL_ARGUMENTS, bufRets); - return hardware::Void(); - } - - SortedVector streamIds; - ssize_t sz = streamIds.setCapacity(bufReqs.size()); - if (sz < 0 || static_cast(sz) != bufReqs.size()) { - ALOGE("%s: failed to allocate memory for %zu buffer requests", - __FUNCTION__, bufReqs.size()); - _hidl_cb(BufferRequestStatus::FAILED_ILLEGAL_ARGUMENTS, bufRets); - return hardware::Void(); - } - - if (bufReqs.size() > mOutputStreams.size()) { - ALOGE("%s: too many buffer requests (%zu > # of output streams %zu)", - __FUNCTION__, bufReqs.size(), mOutputStreams.size()); - _hidl_cb(BufferRequestStatus::FAILED_ILLEGAL_ARGUMENTS, bufRets); - return hardware::Void(); - } - - // Check for repeated streamId - for (const auto& bufReq : bufReqs) { - if (streamIds.indexOf(bufReq.streamId) != NAME_NOT_FOUND) { - ALOGE("%s: Stream %d appear multiple times in buffer requests", - __FUNCTION__, bufReq.streamId); - _hidl_cb(BufferRequestStatus::FAILED_ILLEGAL_ARGUMENTS, bufRets); - return hardware::Void(); - } - streamIds.add(bufReq.streamId); - } - - if (!mRequestBufferSM.startRequestBuffer()) { - ALOGE("%s: request buffer disallowed while camera service is configuring", - __FUNCTION__); - _hidl_cb(BufferRequestStatus::FAILED_CONFIGURING, bufRets); - return hardware::Void(); - } - - bufRets.resize(bufReqs.size()); - - bool allReqsSucceeds = true; - bool oneReqSucceeds = false; - for (size_t i = 0; i < bufReqs.size(); i++) { - const auto& bufReq = bufReqs[i]; - auto& bufRet = bufRets[i]; - int32_t streamId = bufReq.streamId; - sp outputStream = mOutputStreams.get(streamId); - if (outputStream == nullptr) { - ALOGE("%s: Output stream id %d not found!", __FUNCTION__, streamId); - hardware::hidl_vec emptyBufRets; - _hidl_cb(BufferRequestStatus::FAILED_ILLEGAL_ARGUMENTS, emptyBufRets); - mRequestBufferSM.endRequestBuffer(); - return hardware::Void(); - } - - if (outputStream->isAbandoned()) { - bufRet.val.error(StreamBufferRequestError::STREAM_DISCONNECTED); - allReqsSucceeds = false; - continue; - } - - bufRet.streamId = streamId; - size_t handOutBufferCount = outputStream->getOutstandingBuffersCount(); - uint32_t numBuffersRequested = bufReq.numBuffersRequested; - size_t totalHandout = handOutBufferCount + numBuffersRequested; - uint32_t maxBuffers = outputStream->asHalStream()->max_buffers; - if (totalHandout > maxBuffers) { - // Not able to allocate enough buffer. Exit early for this stream - ALOGE("%s: request too much buffers for stream %d: at HAL: %zu + requesting: %d" - " > max: %d", __FUNCTION__, streamId, handOutBufferCount, - numBuffersRequested, maxBuffers); - bufRet.val.error(StreamBufferRequestError::MAX_BUFFER_EXCEEDED); - allReqsSucceeds = false; - continue; - } - - hardware::hidl_vec tmpRetBuffers(numBuffersRequested); - bool currentReqSucceeds = true; - std::vector streamBuffers(numBuffersRequested); - size_t numAllocatedBuffers = 0; - size_t numPushedInflightBuffers = 0; - for (size_t b = 0; b < numBuffersRequested; b++) { - camera3_stream_buffer_t& sb = streamBuffers[b]; - // Since this method can run concurrently with request thread - // We need to update the wait duration everytime we call getbuffer - nsecs_t waitDuration = kBaseGetBufferWait + getExpectedInFlightDuration(); - status_t res = outputStream->getBuffer(&sb, waitDuration); - if (res != OK) { - if (res == NO_INIT || res == DEAD_OBJECT) { - ALOGV("%s: Can't get output buffer for stream %d: %s (%d)", - __FUNCTION__, streamId, strerror(-res), res); - bufRet.val.error(StreamBufferRequestError::STREAM_DISCONNECTED); - } else { - ALOGE("%s: Can't get output buffer for stream %d: %s (%d)", - __FUNCTION__, streamId, strerror(-res), res); - if (res == TIMED_OUT || res == NO_MEMORY) { - bufRet.val.error(StreamBufferRequestError::NO_BUFFER_AVAILABLE); - } else { - bufRet.val.error(StreamBufferRequestError::UNKNOWN_ERROR); - } - } - currentReqSucceeds = false; - break; - } - numAllocatedBuffers++; - - buffer_handle_t *buffer = sb.buffer; - auto pair = mInterface->getBufferId(*buffer, streamId); - bool isNewBuffer = pair.first; - uint64_t bufferId = pair.second; - StreamBuffer& hBuf = tmpRetBuffers[b]; - - hBuf.streamId = streamId; - hBuf.bufferId = bufferId; - hBuf.buffer = (isNewBuffer) ? *buffer : nullptr; - hBuf.status = BufferStatus::OK; - hBuf.releaseFence = nullptr; - - native_handle_t *acquireFence = nullptr; - if (sb.acquire_fence != -1) { - acquireFence = native_handle_create(1,0); - acquireFence->data[0] = sb.acquire_fence; - } - hBuf.acquireFence.setTo(acquireFence, /*shouldOwn*/true); - hBuf.releaseFence = nullptr; - - res = mInterface->pushInflightRequestBuffer(bufferId, buffer, streamId); - if (res != OK) { - ALOGE("%s: Can't get register request buffers for stream %d: %s (%d)", - __FUNCTION__, streamId, strerror(-res), res); - bufRet.val.error(StreamBufferRequestError::UNKNOWN_ERROR); - currentReqSucceeds = false; - break; - } - numPushedInflightBuffers++; - } - if (currentReqSucceeds) { - bufRet.val.buffers(std::move(tmpRetBuffers)); - oneReqSucceeds = true; - } else { - allReqsSucceeds = false; - for (size_t b = 0; b < numPushedInflightBuffers; b++) { - StreamBuffer& hBuf = tmpRetBuffers[b]; - buffer_handle_t* buffer; - status_t res = mInterface->popInflightRequestBuffer(hBuf.bufferId, &buffer); - if (res != OK) { - SET_ERR("%s: popInflightRequestBuffer failed for stream %d: %s (%d)", - __FUNCTION__, streamId, strerror(-res), res); - } - } - for (size_t b = 0; b < numAllocatedBuffers; b++) { - camera3_stream_buffer_t& sb = streamBuffers[b]; - sb.acquire_fence = -1; - sb.status = CAMERA3_BUFFER_STATUS_ERROR; - } - returnOutputBuffers(streamBuffers.data(), numAllocatedBuffers, 0); - } - } - - _hidl_cb(allReqsSucceeds ? BufferRequestStatus::OK : - oneReqSucceeds ? BufferRequestStatus::FAILED_PARTIAL : - BufferRequestStatus::FAILED_UNKNOWN, - bufRets); - mRequestBufferSM.endRequestBuffer(); + RequestBufferStates states { + mId, mRequestBufferInterfaceLock, mUseHalBufManager, mOutputStreams, + *this, *mInterface, *this}; + camera3::requestStreamBuffers(states, bufReqs, _hidl_cb); return hardware::Void(); } hardware::Return Camera3Device::returnStreamBuffers( const hardware::hidl_vec& buffers) { - if (!mUseHalBufManager) { - ALOGE("%s: Camera %s does not support HAL buffer managerment", - __FUNCTION__, mId.string()); - return hardware::Void(); - } - - for (const auto& buf : buffers) { - if (buf.bufferId == HalInterface::BUFFER_ID_NO_BUFFER) { - ALOGE("%s: cannot return a buffer without bufferId", __FUNCTION__); - continue; - } - - buffer_handle_t* buffer; - status_t res = mInterface->popInflightRequestBuffer(buf.bufferId, &buffer); - - if (res != OK) { - ALOGE("%s: cannot find in-flight buffer %" PRIu64 " for stream %d", - __FUNCTION__, buf.bufferId, buf.streamId); - continue; - } - - camera3_stream_buffer_t streamBuffer; - streamBuffer.buffer = buffer; - streamBuffer.status = CAMERA3_BUFFER_STATUS_ERROR; - streamBuffer.acquire_fence = -1; - streamBuffer.release_fence = -1; - - if (buf.releaseFence == nullptr) { - streamBuffer.release_fence = -1; - } else if (buf.releaseFence->numFds == 1) { - streamBuffer.release_fence = dup(buf.releaseFence->data[0]); - } else { - ALOGE("%s: Invalid release fence, fd count is %d, not 1", - __FUNCTION__, buf.releaseFence->numFds); - continue; - } - - sp stream = mOutputStreams.get(buf.streamId); - if (stream == nullptr) { - ALOGE("%s: Output stream id %d not found!", __FUNCTION__, buf.streamId); - continue; - } - streamBuffer.stream = stream->asHalStream(); - returnOutputBuffers(&streamBuffer, /*size*/1, /*timestamp*/ 0); - } + ReturnBufferStates states { + mId, mUseHalBufManager, mOutputStreams, *mInterface}; + camera3::returnStreamBuffers(states, buffers); return hardware::Void(); } @@ -1241,6 +1021,12 @@ hardware::Return Camera3Device::processCaptureResult_3_4( ALOGW("%s: received capture result in error state.", __FUNCTION__); } + sp listener; + { + std::lock_guard l(mOutputLock); + listener = mListener.promote(); + } + if (mProcessCaptureResultLock.tryLock() != OK) { // This should never happen; it indicates a wrong client implementation // that doesn't follow the contract. But, we can be tolerant here. @@ -1253,8 +1039,22 @@ hardware::Return Camera3Device::processCaptureResult_3_4( return hardware::Void(); } } + CaptureOutputStates states { + mId, + mInFlightLock, mInFlightMap, + mOutputLock, mResultQueue, mResultSignal, + mNextShutterFrameNumber, + mNextReprocessShutterFrameNumber, mNextZslStillShutterFrameNumber, + mNextResultFrameNumber, + mNextReprocessResultFrameNumber, mNextZslStillResultFrameNumber, + mUseHalBufManager, mUsePartialResult, mNeedFixupMonochromeTags, + mNumPartialResults, mVendorTagId, mDeviceInfo, mPhysicalDeviceInfoMap, + mResultMetadataQueue, mDistortionMappers, mZoomRatioMappers, mTagMonitor, + mInputStream, mOutputStreams, listener, *this, *this, *mInterface + }; + for (const auto& result : results) { - processOneCaptureResultLocked(result.v3_2, result.physicalCameraMetadata); + processOneCaptureResultLocked(states, result.v3_2, result.physicalCameraMetadata); } mProcessCaptureResultLock.unlock(); return hardware::Void(); @@ -1277,6 +1077,12 @@ hardware::Return Camera3Device::processCaptureResult( ALOGW("%s: received capture result in error state.", __FUNCTION__); } + sp listener; + { + std::lock_guard l(mOutputLock); + listener = mListener.promote(); + } + if (mProcessCaptureResultLock.tryLock() != OK) { // This should never happen; it indicates a wrong client implementation // that doesn't follow the contract. But, we can be tolerant here. @@ -1289,186 +1095,28 @@ hardware::Return Camera3Device::processCaptureResult( return hardware::Void(); } } + + CaptureOutputStates states { + mId, + mInFlightLock, mInFlightMap, + mOutputLock, mResultQueue, mResultSignal, + mNextShutterFrameNumber, + mNextReprocessShutterFrameNumber, mNextZslStillShutterFrameNumber, + mNextResultFrameNumber, + mNextReprocessResultFrameNumber, mNextZslStillResultFrameNumber, + mUseHalBufManager, mUsePartialResult, mNeedFixupMonochromeTags, + mNumPartialResults, mVendorTagId, mDeviceInfo, mPhysicalDeviceInfoMap, + mResultMetadataQueue, mDistortionMappers, mZoomRatioMappers, mTagMonitor, + mInputStream, mOutputStreams, listener, *this, *this, *mInterface + }; + for (const auto& result : results) { - processOneCaptureResultLocked(result, noPhysMetadata); + processOneCaptureResultLocked(states, result, noPhysMetadata); } mProcessCaptureResultLock.unlock(); return hardware::Void(); } -status_t Camera3Device::readOneCameraMetadataLocked( - uint64_t fmqResultSize, hardware::camera::device::V3_2::CameraMetadata& resultMetadata, - const hardware::camera::device::V3_2::CameraMetadata& result) { - if (fmqResultSize > 0) { - resultMetadata.resize(fmqResultSize); - if (mResultMetadataQueue == nullptr) { - return NO_MEMORY; // logged in initialize() - } - if (!mResultMetadataQueue->read(resultMetadata.data(), fmqResultSize)) { - ALOGE("%s: Cannot read camera metadata from fmq, size = %" PRIu64, - __FUNCTION__, fmqResultSize); - return INVALID_OPERATION; - } - } else { - resultMetadata.setToExternal(const_cast(result.data()), - result.size()); - } - - if (resultMetadata.size() != 0) { - status_t res; - const camera_metadata_t* metadata = - reinterpret_cast(resultMetadata.data()); - size_t expected_metadata_size = resultMetadata.size(); - if ((res = validate_camera_metadata_structure(metadata, &expected_metadata_size)) != OK) { - ALOGE("%s: Invalid camera metadata received by camera service from HAL: %s (%d)", - __FUNCTION__, strerror(-res), res); - return INVALID_OPERATION; - } - } - - return OK; -} - -void Camera3Device::processOneCaptureResultLocked( - const hardware::camera::device::V3_2::CaptureResult& result, - const hardware::hidl_vec< - hardware::camera::device::V3_4::PhysicalCameraMetadata> physicalCameraMetadata) { - camera3_capture_result r; - status_t res; - r.frame_number = result.frameNumber; - - // Read and validate the result metadata. - hardware::camera::device::V3_2::CameraMetadata resultMetadata; - res = readOneCameraMetadataLocked(result.fmqResultSize, resultMetadata, result.result); - if (res != OK) { - ALOGE("%s: Frame %d: Failed to read capture result metadata", - __FUNCTION__, result.frameNumber); - return; - } - r.result = reinterpret_cast(resultMetadata.data()); - - // Read and validate physical camera metadata - size_t physResultCount = physicalCameraMetadata.size(); - std::vector physCamIds(physResultCount); - std::vector phyCamMetadatas(physResultCount); - std::vector physResultMetadata; - physResultMetadata.resize(physResultCount); - for (size_t i = 0; i < physicalCameraMetadata.size(); i++) { - res = readOneCameraMetadataLocked(physicalCameraMetadata[i].fmqMetadataSize, - physResultMetadata[i], physicalCameraMetadata[i].metadata); - if (res != OK) { - ALOGE("%s: Frame %d: Failed to read capture result metadata for camera %s", - __FUNCTION__, result.frameNumber, - physicalCameraMetadata[i].physicalCameraId.c_str()); - return; - } - physCamIds[i] = physicalCameraMetadata[i].physicalCameraId.c_str(); - phyCamMetadatas[i] = reinterpret_cast( - physResultMetadata[i].data()); - } - r.num_physcam_metadata = physResultCount; - r.physcam_ids = physCamIds.data(); - r.physcam_metadata = phyCamMetadatas.data(); - - std::vector outputBuffers(result.outputBuffers.size()); - std::vector outputBufferHandles(result.outputBuffers.size()); - for (size_t i = 0; i < result.outputBuffers.size(); i++) { - auto& bDst = outputBuffers[i]; - const StreamBuffer &bSrc = result.outputBuffers[i]; - - sp stream = mOutputStreams.get(bSrc.streamId); - if (stream == nullptr) { - ALOGE("%s: Frame %d: Buffer %zu: Invalid output stream id %d", - __FUNCTION__, result.frameNumber, i, bSrc.streamId); - return; - } - bDst.stream = stream->asHalStream(); - - bool noBufferReturned = false; - buffer_handle_t *buffer = nullptr; - if (mUseHalBufManager) { - // This is suspicious most of the time but can be correct during flush where HAL - // has to return capture result before a buffer is requested - if (bSrc.bufferId == HalInterface::BUFFER_ID_NO_BUFFER) { - if (bSrc.status == BufferStatus::OK) { - ALOGE("%s: Frame %d: Buffer %zu: No bufferId for stream %d", - __FUNCTION__, result.frameNumber, i, bSrc.streamId); - // Still proceeds so other buffers can be returned - } - noBufferReturned = true; - } - if (noBufferReturned) { - res = OK; - } else { - res = mInterface->popInflightRequestBuffer(bSrc.bufferId, &buffer); - } - } else { - res = mInterface->popInflightBuffer(result.frameNumber, bSrc.streamId, &buffer); - } - - if (res != OK) { - ALOGE("%s: Frame %d: Buffer %zu: No in-flight buffer for stream %d", - __FUNCTION__, result.frameNumber, i, bSrc.streamId); - return; - } - - bDst.buffer = buffer; - bDst.status = mapHidlBufferStatus(bSrc.status); - bDst.acquire_fence = -1; - if (bSrc.releaseFence == nullptr) { - bDst.release_fence = -1; - } else if (bSrc.releaseFence->numFds == 1) { - if (noBufferReturned) { - ALOGE("%s: got releaseFence without output buffer!", __FUNCTION__); - } - bDst.release_fence = dup(bSrc.releaseFence->data[0]); - } else { - ALOGE("%s: Frame %d: Invalid release fence for buffer %zu, fd count is %d, not 1", - __FUNCTION__, result.frameNumber, i, bSrc.releaseFence->numFds); - return; - } - } - r.num_output_buffers = outputBuffers.size(); - r.output_buffers = outputBuffers.data(); - - camera3_stream_buffer_t inputBuffer; - if (result.inputBuffer.streamId == -1) { - r.input_buffer = nullptr; - } else { - if (mInputStream->getId() != result.inputBuffer.streamId) { - ALOGE("%s: Frame %d: Invalid input stream id %d", __FUNCTION__, - result.frameNumber, result.inputBuffer.streamId); - return; - } - inputBuffer.stream = mInputStream->asHalStream(); - buffer_handle_t *buffer; - res = mInterface->popInflightBuffer(result.frameNumber, result.inputBuffer.streamId, - &buffer); - if (res != OK) { - ALOGE("%s: Frame %d: Input buffer: No in-flight buffer for stream %d", - __FUNCTION__, result.frameNumber, result.inputBuffer.streamId); - return; - } - inputBuffer.buffer = buffer; - inputBuffer.status = mapHidlBufferStatus(result.inputBuffer.status); - inputBuffer.acquire_fence = -1; - if (result.inputBuffer.releaseFence == nullptr) { - inputBuffer.release_fence = -1; - } else if (result.inputBuffer.releaseFence->numFds == 1) { - inputBuffer.release_fence = dup(result.inputBuffer.releaseFence->data[0]); - } else { - ALOGE("%s: Frame %d: Invalid release fence for input buffer, fd count is %d, not 1", - __FUNCTION__, result.frameNumber, result.inputBuffer.releaseFence->numFds); - return; - } - r.input_buffer = &inputBuffer; - } - - r.partial_result = result.partialResult; - - processCaptureResult(&r); -} - hardware::Return Camera3Device::notify( const hardware::hidl_vec& msgs) { // Ideally we should grab mLock, but that can lead to deadlock, and @@ -1481,53 +1129,29 @@ hardware::Return Camera3Device::notify( ALOGW("%s: received notify message in error state.", __FUNCTION__); } - for (const auto& msg : msgs) { - notify(msg); + sp listener; + { + std::lock_guard l(mOutputLock); + listener = mListener.promote(); } - return hardware::Void(); -} -void Camera3Device::notify( - const hardware::camera::device::V3_2::NotifyMsg& msg) { - - camera3_notify_msg m; - switch (msg.type) { - case MsgType::ERROR: - m.type = CAMERA3_MSG_ERROR; - m.message.error.frame_number = msg.msg.error.frameNumber; - if (msg.msg.error.errorStreamId >= 0) { - sp stream = mOutputStreams.get(msg.msg.error.errorStreamId); - if (stream == nullptr) { - ALOGE("%s: Frame %d: Invalid error stream id %d", __FUNCTION__, - m.message.error.frame_number, msg.msg.error.errorStreamId); - return; - } - m.message.error.error_stream = stream->asHalStream(); - } else { - m.message.error.error_stream = nullptr; - } - switch (msg.msg.error.errorCode) { - case ErrorCode::ERROR_DEVICE: - m.message.error.error_code = CAMERA3_MSG_ERROR_DEVICE; - break; - case ErrorCode::ERROR_REQUEST: - m.message.error.error_code = CAMERA3_MSG_ERROR_REQUEST; - break; - case ErrorCode::ERROR_RESULT: - m.message.error.error_code = CAMERA3_MSG_ERROR_RESULT; - break; - case ErrorCode::ERROR_BUFFER: - m.message.error.error_code = CAMERA3_MSG_ERROR_BUFFER; - break; - } - break; - case MsgType::SHUTTER: - m.type = CAMERA3_MSG_SHUTTER; - m.message.shutter.frame_number = msg.msg.shutter.frameNumber; - m.message.shutter.timestamp = msg.msg.shutter.timestamp; - break; + CaptureOutputStates states { + mId, + mInFlightLock, mInFlightMap, + mOutputLock, mResultQueue, mResultSignal, + mNextShutterFrameNumber, + mNextReprocessShutterFrameNumber, mNextZslStillShutterFrameNumber, + mNextResultFrameNumber, + mNextReprocessResultFrameNumber, mNextZslStillResultFrameNumber, + mUseHalBufManager, mUsePartialResult, mNeedFixupMonochromeTags, + mNumPartialResults, mVendorTagId, mDeviceInfo, mPhysicalDeviceInfoMap, + mResultMetadataQueue, mDistortionMappers, mZoomRatioMappers, mTagMonitor, + mInputStream, mOutputStreams, listener, *this, *this, *mInterface + }; + for (const auto& msg : msgs) { + camera3::notify(states, msg); } - notify(&m); + return hardware::Void(); } status_t Camera3Device::captureList(const List &requestsList, @@ -1683,56 +1307,6 @@ status_t Camera3Device::createInputStream( return OK; } -status_t Camera3Device::StreamSet::add( - int streamId, sp stream) { - if (stream == nullptr) { - ALOGE("%s: cannot add null stream", __FUNCTION__); - return BAD_VALUE; - } - std::lock_guard lock(mLock); - return mData.add(streamId, stream); -} - -ssize_t Camera3Device::StreamSet::remove(int streamId) { - std::lock_guard lock(mLock); - return mData.removeItem(streamId); -} - -sp -Camera3Device::StreamSet::get(int streamId) { - std::lock_guard lock(mLock); - ssize_t idx = mData.indexOfKey(streamId); - if (idx == NAME_NOT_FOUND) { - return nullptr; - } - return mData.editValueAt(idx); -} - -sp -Camera3Device::StreamSet::operator[] (size_t index) { - std::lock_guard lock(mLock); - return mData.editValueAt(index); -} - -size_t Camera3Device::StreamSet::size() const { - std::lock_guard lock(mLock); - return mData.size(); -} - -void Camera3Device::StreamSet::clear() { - std::lock_guard lock(mLock); - return mData.clear(); -} - -std::vector Camera3Device::StreamSet::getStreamIds() { - std::lock_guard lock(mLock); - std::vector streamIds(mData.size()); - for (size_t i = 0; i < mData.size(); i++) { - streamIds[i] = mData.keyAt(i); - } - return streamIds; -} - status_t Camera3Device::createStream(sp consumer, uint32_t width, uint32_t height, int format, android_dataspace dataSpace, camera3_stream_rotation_t rotation, int *id, @@ -2296,7 +1870,7 @@ status_t Camera3Device::waitUntilStateThenRelock(bool active, nsecs_t timeout) { status_t Camera3Device::setNotifyCallback(wp listener) { ATRACE_CALL(); - Mutex::Autolock l(mOutputLock); + std::lock_guard l(mOutputLock); if (listener != NULL && mListener != NULL) { ALOGW("%s: Replacing old callback listener", __FUNCTION__); @@ -2314,17 +1888,12 @@ bool Camera3Device::willNotify3A() { status_t Camera3Device::waitForNextFrame(nsecs_t timeout) { ATRACE_CALL(); - status_t res; - Mutex::Autolock l(mOutputLock); + std::unique_lock l(mOutputLock); while (mResultQueue.empty()) { - res = mResultSignal.waitRelative(mOutputLock, timeout); - if (res == TIMED_OUT) { - return res; - } else if (res != OK) { - ALOGW("%s: Camera %s: No frame in %" PRId64 " ns: %s (%d)", - __FUNCTION__, mId.string(), timeout, strerror(-res), res); - return res; + auto st = mResultSignal.wait_for(l, std::chrono::nanoseconds(timeout)); + if (st == std::cv_status::timeout) { + return TIMED_OUT; } } return OK; @@ -2332,7 +1901,7 @@ status_t Camera3Device::waitForNextFrame(nsecs_t timeout) { status_t Camera3Device::getNextResult(CaptureResult *frame) { ATRACE_CALL(); - Mutex::Autolock l(mOutputLock); + std::lock_guard l(mOutputLock); if (mResultQueue.empty()) { return NOT_ENOUGH_DATA; @@ -2528,7 +2097,7 @@ void Camera3Device::notifyStatus(bool idle) { sp listener; { - Mutex::Autolock l(mOutputLock); + std::lock_guard l(mOutputLock); listener = mListener.promote(); } if (idle && listener != NULL) { @@ -2727,17 +2296,6 @@ sp Camera3Device::createCaptureRequest( return newRequest; } -bool Camera3Device::isOpaqueInputSizeSupported(uint32_t width, uint32_t height) { - for (uint32_t i = 0; i < mSupportedOpaqueInputSizes.size(); i++) { - Size size = mSupportedOpaqueInputSizes[i]; - if (size.width == width && size.height == height) { - return true; - } - } - - return false; -} - void Camera3Device::cancelStreamsConfigurationLocked() { int res = OK; if (mInputStream != NULL && mInputStream->isConfiguring()) { @@ -3175,7 +2733,7 @@ status_t Camera3Device::registerInFlight(uint32_t frameNumber, bool isZslCapture, const std::set& cameraIdsWithZoom, const SurfaceMap& outputSurfaces) { ATRACE_CALL(); - Mutex::Autolock l(mInFlightLock); + std::lock_guard l(mInFlightLock); ssize_t res; res = mInFlightMap.add(frameNumber, InFlightRequest(numBuffers, resultExtras, hasInput, @@ -3188,888 +2746,71 @@ status_t Camera3Device::registerInFlight(uint32_t frameNumber, // avoid a deadlock during reprocess requests. Mutex::Autolock l(mTrackerLock); if (mStatusTracker != nullptr) { - mStatusTracker->markComponentActive(mInFlightStatusId); - } - } - - mExpectedInflightDuration += maxExpectedDuration; - return OK; -} - -void Camera3Device::returnOutputBuffers( - const camera3_stream_buffer_t *outputBuffers, size_t numBuffers, - nsecs_t timestamp, bool timestampIncreasing, - const SurfaceMap& outputSurfaces, - const CaptureResultExtras &inResultExtras) { - - for (size_t i = 0; i < numBuffers; i++) - { - if (outputBuffers[i].buffer == nullptr) { - if (!mUseHalBufManager) { - // With HAL buffer management API, HAL sometimes will have to return buffers that - // has not got a output buffer handle filled yet. This is though illegal if HAL - // buffer management API is not being used. - ALOGE("%s: cannot return a null buffer!", __FUNCTION__); - } - continue; - } - - Camera3StreamInterface *stream = Camera3Stream::cast(outputBuffers[i].stream); - int streamId = stream->getId(); - const auto& it = outputSurfaces.find(streamId); - status_t res = OK; - if (it != outputSurfaces.end()) { - res = stream->returnBuffer( - outputBuffers[i], timestamp, timestampIncreasing, it->second, - inResultExtras.frameNumber); - } else { - res = stream->returnBuffer( - outputBuffers[i], timestamp, timestampIncreasing, std::vector (), - inResultExtras.frameNumber); - } - - // Note: stream may be deallocated at this point, if this buffer was - // the last reference to it. - if (res == NO_INIT || res == DEAD_OBJECT) { - ALOGV("Can't return buffer to its stream: %s (%d)", strerror(-res), res); - } else if (res != OK) { - ALOGE("Can't return buffer to its stream: %s (%d)", strerror(-res), res); - } - - // Long processing consumers can cause returnBuffer timeout for shared stream - // If that happens, cancel the buffer and send a buffer error to client - if (it != outputSurfaces.end() && res == TIMED_OUT && - outputBuffers[i].status == CAMERA3_BUFFER_STATUS_OK) { - // cancel the buffer - camera3_stream_buffer_t sb = outputBuffers[i]; - sb.status = CAMERA3_BUFFER_STATUS_ERROR; - stream->returnBuffer(sb, /*timestamp*/0, timestampIncreasing, std::vector (), - inResultExtras.frameNumber); - - // notify client buffer error - sp listener; - { - Mutex::Autolock l(mOutputLock); - listener = mListener.promote(); - } - - if (listener != nullptr) { - CaptureResultExtras extras = inResultExtras; - extras.errorStreamId = streamId; - listener->notifyError( - hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_BUFFER, - extras); - } - } - } -} - -void Camera3Device::removeInFlightMapEntryLocked(int idx) { - ATRACE_CALL(); - nsecs_t duration = mInFlightMap.valueAt(idx).maxExpectedDuration; - mInFlightMap.removeItemsAt(idx, 1); - - // Indicate idle inFlightMap to the status tracker - if (mInFlightMap.size() == 0) { - mRequestBufferSM.onInflightMapEmpty(); - // Hold a separate dedicated tracker lock to prevent race with disconnect and also - // avoid a deadlock during reprocess requests. - Mutex::Autolock l(mTrackerLock); - if (mStatusTracker != nullptr) { - mStatusTracker->markComponentIdle(mInFlightStatusId, Fence::NO_FENCE); - } - } - mExpectedInflightDuration -= duration; -} - -void Camera3Device::removeInFlightRequestIfReadyLocked(int idx) { - - const InFlightRequest &request = mInFlightMap.valueAt(idx); - const uint32_t frameNumber = mInFlightMap.keyAt(idx); - - nsecs_t sensorTimestamp = request.sensorTimestamp; - nsecs_t shutterTimestamp = request.shutterTimestamp; - - // Check if it's okay to remove the request from InFlightMap: - // In the case of a successful request: - // all input and output buffers, all result metadata, shutter callback - // arrived. - // In the case of a unsuccessful request: - // all input and output buffers arrived. - if (request.numBuffersLeft == 0 && - (request.skipResultMetadata || - (request.haveResultMetadata && shutterTimestamp != 0))) { - if (request.stillCapture) { - ATRACE_ASYNC_END("still capture", frameNumber); - } - - ATRACE_ASYNC_END("frame capture", frameNumber); - - // Sanity check - if sensor timestamp matches shutter timestamp in the - // case of request having callback. - if (request.hasCallback && request.requestStatus == OK && - sensorTimestamp != shutterTimestamp) { - SET_ERR("sensor timestamp (%" PRId64 - ") for frame %d doesn't match shutter timestamp (%" PRId64 ")", - sensorTimestamp, frameNumber, shutterTimestamp); - } - - // for an unsuccessful request, it may have pending output buffers to - // return. - assert(request.requestStatus != OK || - request.pendingOutputBuffers.size() == 0); - returnOutputBuffers(request.pendingOutputBuffers.array(), - request.pendingOutputBuffers.size(), 0, /*timestampIncreasing*/true, - request.outputSurfaces, request.resultExtras); - - removeInFlightMapEntryLocked(idx); - ALOGVV("%s: removed frame %d from InFlightMap", __FUNCTION__, frameNumber); - } - - // Sanity check - if we have too many in-flight frames with long total inflight duration, - // something has likely gone wrong. This might still be legit only if application send in - // a long burst of long exposure requests. - if (mExpectedInflightDuration > kMinWarnInflightDuration) { - if (!mIsConstrainedHighSpeedConfiguration && mInFlightMap.size() > kInFlightWarnLimit) { - CLOGW("In-flight list too large: %zu, total inflight duration %" PRIu64, - mInFlightMap.size(), mExpectedInflightDuration); - } else if (mIsConstrainedHighSpeedConfiguration && mInFlightMap.size() > - kInFlightWarnLimitHighSpeed) { - CLOGW("In-flight list too large for high speed configuration: %zu," - "total inflight duration %" PRIu64, - mInFlightMap.size(), mExpectedInflightDuration); - } - } -} - -void Camera3Device::flushInflightRequests() { - ATRACE_CALL(); - { // First return buffers cached in mInFlightMap - Mutex::Autolock l(mInFlightLock); - for (size_t idx = 0; idx < mInFlightMap.size(); idx++) { - const InFlightRequest &request = mInFlightMap.valueAt(idx); - returnOutputBuffers(request.pendingOutputBuffers.array(), - request.pendingOutputBuffers.size(), 0, - /*timestampIncreasing*/true, request.outputSurfaces, - request.resultExtras); - } - mInFlightMap.clear(); - mExpectedInflightDuration = 0; - } - - // Then return all inflight buffers not returned by HAL - std::vector> inflightKeys; - mInterface->getInflightBufferKeys(&inflightKeys); - - // Inflight buffers for HAL buffer manager - std::vector inflightRequestBufferKeys; - mInterface->getInflightRequestBufferKeys(&inflightRequestBufferKeys); - - // (streamId, frameNumber, buffer_handle_t*) tuple for all inflight buffers. - // frameNumber will be -1 for buffers from HAL buffer manager - std::vector> inflightBuffers; - inflightBuffers.reserve(inflightKeys.size() + inflightRequestBufferKeys.size()); - - for (auto& pair : inflightKeys) { - int32_t frameNumber = pair.first; - int32_t streamId = pair.second; - buffer_handle_t* buffer; - status_t res = mInterface->popInflightBuffer(frameNumber, streamId, &buffer); - if (res != OK) { - ALOGE("%s: Frame %d: No in-flight buffer for stream %d", - __FUNCTION__, frameNumber, streamId); - continue; - } - inflightBuffers.push_back(std::make_tuple(streamId, frameNumber, buffer)); - } - - for (auto& bufferId : inflightRequestBufferKeys) { - int32_t streamId = -1; - buffer_handle_t* buffer = nullptr; - status_t res = mInterface->popInflightRequestBuffer(bufferId, &buffer, &streamId); - if (res != OK) { - ALOGE("%s: cannot find in-flight buffer %" PRIu64, __FUNCTION__, bufferId); - continue; - } - inflightBuffers.push_back(std::make_tuple(streamId, /*frameNumber*/-1, buffer)); - } - - int32_t inputStreamId = (mInputStream != nullptr) ? mInputStream->getId() : -1; - for (auto& tuple : inflightBuffers) { - status_t res = OK; - int32_t streamId = std::get<0>(tuple); - int32_t frameNumber = std::get<1>(tuple); - buffer_handle_t* buffer = std::get<2>(tuple); - - camera3_stream_buffer_t streamBuffer; - streamBuffer.buffer = buffer; - streamBuffer.status = CAMERA3_BUFFER_STATUS_ERROR; - streamBuffer.acquire_fence = -1; - streamBuffer.release_fence = -1; - - // First check if the buffer belongs to deleted stream - bool streamDeleted = false; - for (auto& stream : mDeletedStreams) { - if (streamId == stream->getId()) { - streamDeleted = true; - // Return buffer to deleted stream - camera3_stream* halStream = stream->asHalStream(); - streamBuffer.stream = halStream; - switch (halStream->stream_type) { - case CAMERA3_STREAM_OUTPUT: - res = stream->returnBuffer(streamBuffer, /*timestamp*/ 0, - /*timestampIncreasing*/true, std::vector (), frameNumber); - if (res != OK) { - ALOGE("%s: Can't return output buffer for frame %d to" - " stream %d: %s (%d)", __FUNCTION__, - frameNumber, streamId, strerror(-res), res); - } - break; - case CAMERA3_STREAM_INPUT: - res = stream->returnInputBuffer(streamBuffer); - if (res != OK) { - ALOGE("%s: Can't return input buffer for frame %d to" - " stream %d: %s (%d)", __FUNCTION__, - frameNumber, streamId, strerror(-res), res); - } - break; - default: // Bi-direcitonal stream is deprecated - ALOGE("%s: stream %d has unknown stream type %d", - __FUNCTION__, streamId, halStream->stream_type); - break; - } - break; - } - } - if (streamDeleted) { - continue; - } - - // Then check against configured streams - if (streamId == inputStreamId) { - streamBuffer.stream = mInputStream->asHalStream(); - res = mInputStream->returnInputBuffer(streamBuffer); - if (res != OK) { - ALOGE("%s: Can't return input buffer for frame %d to" - " stream %d: %s (%d)", __FUNCTION__, - frameNumber, streamId, strerror(-res), res); - } - } else { - sp stream = mOutputStreams.get(streamId); - if (stream == nullptr) { - ALOGE("%s: Output stream id %d not found!", __FUNCTION__, streamId); - continue; - } - streamBuffer.stream = stream->asHalStream(); - returnOutputBuffers(&streamBuffer, /*size*/1, /*timestamp*/ 0); - } - } -} - -void Camera3Device::insertResultLocked(CaptureResult *result, - uint32_t frameNumber) { - if (result == nullptr) return; - - camera_metadata_t *meta = const_cast( - result->mMetadata.getAndLock()); - set_camera_metadata_vendor_id(meta, mVendorTagId); - result->mMetadata.unlock(meta); - - if (result->mMetadata.update(ANDROID_REQUEST_FRAME_COUNT, - (int32_t*)&frameNumber, 1) != OK) { - SET_ERR("Failed to set frame number %d in metadata", frameNumber); - return; - } - - if (result->mMetadata.update(ANDROID_REQUEST_ID, &result->mResultExtras.requestId, 1) != OK) { - SET_ERR("Failed to set request ID in metadata for frame %d", frameNumber); - return; - } - - // Update vendor tag id for physical metadata - for (auto& physicalMetadata : result->mPhysicalMetadatas) { - camera_metadata_t *pmeta = const_cast( - physicalMetadata.mPhysicalCameraMetadata.getAndLock()); - set_camera_metadata_vendor_id(pmeta, mVendorTagId); - physicalMetadata.mPhysicalCameraMetadata.unlock(pmeta); - } - - // Valid result, insert into queue - List::iterator queuedResult = - mResultQueue.insert(mResultQueue.end(), CaptureResult(*result)); - ALOGVV("%s: result requestId = %" PRId32 ", frameNumber = %" PRId64 - ", burstId = %" PRId32, __FUNCTION__, - queuedResult->mResultExtras.requestId, - queuedResult->mResultExtras.frameNumber, - queuedResult->mResultExtras.burstId); - - mResultSignal.signal(); -} - - -void Camera3Device::sendPartialCaptureResult(const camera_metadata_t * partialResult, - const CaptureResultExtras &resultExtras, uint32_t frameNumber) { - ATRACE_CALL(); - Mutex::Autolock l(mOutputLock); - - CaptureResult captureResult; - captureResult.mResultExtras = resultExtras; - captureResult.mMetadata = partialResult; - - // Fix up result metadata for monochrome camera. - status_t res = fixupMonochromeTags(mDeviceInfo, captureResult.mMetadata); - if (res != OK) { - SET_ERR("Failed to override result metadata: %s (%d)", strerror(-res), res); - return; - } - - insertResultLocked(&captureResult, frameNumber); -} - - -void Camera3Device::sendCaptureResult(CameraMetadata &pendingMetadata, - CaptureResultExtras &resultExtras, - CameraMetadata &collectedPartialResult, - uint32_t frameNumber, - bool reprocess, bool zslStillCapture, const std::set& cameraIdsWithZoom, - const std::vector& physicalMetadatas) { - ATRACE_CALL(); - if (pendingMetadata.isEmpty()) - return; - - Mutex::Autolock l(mOutputLock); - - // TODO: need to track errors for tighter bounds on expected frame number - if (reprocess) { - if (frameNumber < mNextReprocessResultFrameNumber) { - SET_ERR("Out-of-order reprocess capture result metadata submitted! " - "(got frame number %d, expecting %d)", - frameNumber, mNextReprocessResultFrameNumber); - return; - } - mNextReprocessResultFrameNumber = frameNumber + 1; - } else if (zslStillCapture) { - if (frameNumber < mNextZslStillResultFrameNumber) { - SET_ERR("Out-of-order ZSL still capture result metadata submitted! " - "(got frame number %d, expecting %d)", - frameNumber, mNextZslStillResultFrameNumber); - return; - } - mNextZslStillResultFrameNumber = frameNumber + 1; - } else { - if (frameNumber < mNextResultFrameNumber) { - SET_ERR("Out-of-order capture result metadata submitted! " - "(got frame number %d, expecting %d)", - frameNumber, mNextResultFrameNumber); - return; - } - mNextResultFrameNumber = frameNumber + 1; - } - - CaptureResult captureResult; - captureResult.mResultExtras = resultExtras; - captureResult.mMetadata = pendingMetadata; - captureResult.mPhysicalMetadatas = physicalMetadatas; - - // Append any previous partials to form a complete result - if (mUsePartialResult && !collectedPartialResult.isEmpty()) { - captureResult.mMetadata.append(collectedPartialResult); - } - - captureResult.mMetadata.sort(); - - // Check that there's a timestamp in the result metadata - camera_metadata_entry timestamp = captureResult.mMetadata.find(ANDROID_SENSOR_TIMESTAMP); - if (timestamp.count == 0) { - SET_ERR("No timestamp provided by HAL for frame %d!", - frameNumber); - return; - } - for (auto& physicalMetadata : captureResult.mPhysicalMetadatas) { - camera_metadata_entry timestamp = - physicalMetadata.mPhysicalCameraMetadata.find(ANDROID_SENSOR_TIMESTAMP); - if (timestamp.count == 0) { - SET_ERR("No timestamp provided by HAL for physical camera %s frame %d!", - String8(physicalMetadata.mPhysicalCameraId).c_str(), frameNumber); - return; - } - } - - // Fix up some result metadata to account for HAL-level distortion correction - status_t res = - mDistortionMappers[mId.c_str()].correctCaptureResult(&captureResult.mMetadata); - if (res != OK) { - SET_ERR("Unable to correct capture result metadata for frame %d: %s (%d)", - frameNumber, strerror(-res), res); - return; - } - - // Fix up result metadata to account for zoom ratio availabilities between - // HAL and app. - bool zoomRatioIs1 = cameraIdsWithZoom.find(mId.c_str()) == cameraIdsWithZoom.end(); - res = mZoomRatioMappers[mId.c_str()].updateCaptureResult( - &captureResult.mMetadata, zoomRatioIs1); - if (res != OK) { - SET_ERR("Failed to update capture result zoom ratio metadata for frame %d: %s (%d)", - frameNumber, strerror(-res), res); - return; - } - - for (auto& physicalMetadata : captureResult.mPhysicalMetadatas) { - String8 cameraId8(physicalMetadata.mPhysicalCameraId); - if (mDistortionMappers.find(cameraId8.c_str()) != mDistortionMappers.end()) { - res = mDistortionMappers[cameraId8.c_str()].correctCaptureResult( - &physicalMetadata.mPhysicalCameraMetadata); - if (res != OK) { - SET_ERR("Unable to correct physical capture result metadata for frame %d: %s (%d)", - frameNumber, strerror(-res), res); - return; - } - } - - zoomRatioIs1 = cameraIdsWithZoom.find(cameraId8.c_str()) == cameraIdsWithZoom.end(); - res = mZoomRatioMappers[cameraId8.c_str()].updateCaptureResult( - &physicalMetadata.mPhysicalCameraMetadata, zoomRatioIs1); - if (res != OK) { - SET_ERR("Failed to update camera %s's physical zoom ratio metadata for " - "frame %d: %s(%d)", cameraId8.c_str(), frameNumber, strerror(-res), res); - return; - } - } - - // Fix up result metadata for monochrome camera. - res = fixupMonochromeTags(mDeviceInfo, captureResult.mMetadata); - if (res != OK) { - SET_ERR("Failed to override result metadata: %s (%d)", strerror(-res), res); - return; - } - for (auto& physicalMetadata : captureResult.mPhysicalMetadatas) { - String8 cameraId8(physicalMetadata.mPhysicalCameraId); - res = fixupMonochromeTags(mPhysicalDeviceInfoMap.at(cameraId8.c_str()), - physicalMetadata.mPhysicalCameraMetadata); - if (res != OK) { - SET_ERR("Failed to override result metadata: %s (%d)", strerror(-res), res); - return; - } - } - - std::unordered_map monitoredPhysicalMetadata; - for (auto& m : physicalMetadatas) { - monitoredPhysicalMetadata.emplace(String8(m.mPhysicalCameraId).string(), - CameraMetadata(m.mPhysicalCameraMetadata)); - } - mTagMonitor.monitorMetadata(TagMonitor::RESULT, - frameNumber, timestamp.data.i64[0], captureResult.mMetadata, - monitoredPhysicalMetadata); - - insertResultLocked(&captureResult, frameNumber); -} - -/** - * Camera HAL device callback methods - */ - -void Camera3Device::processCaptureResult(const camera3_capture_result *result) { - ATRACE_CALL(); - - status_t res; - - uint32_t frameNumber = result->frame_number; - if (result->result == NULL && result->num_output_buffers == 0 && - result->input_buffer == NULL) { - SET_ERR("No result data provided by HAL for frame %d", - frameNumber); - return; - } - - if (!mUsePartialResult && - result->result != NULL && - result->partial_result != 1) { - SET_ERR("Result is malformed for frame %d: partial_result %u must be 1" - " if partial result is not supported", - frameNumber, result->partial_result); - return; - } - - bool isPartialResult = false; - CameraMetadata collectedPartialResult; - bool hasInputBufferInRequest = false; - - // Get shutter timestamp and resultExtras from list of in-flight requests, - // where it was added by the shutter notification for this frame. If the - // shutter timestamp isn't received yet, append the output buffers to the - // in-flight request and they will be returned when the shutter timestamp - // arrives. Update the in-flight status and remove the in-flight entry if - // all result data and shutter timestamp have been received. - nsecs_t shutterTimestamp = 0; - - { - Mutex::Autolock l(mInFlightLock); - ssize_t idx = mInFlightMap.indexOfKey(frameNumber); - if (idx == NAME_NOT_FOUND) { - SET_ERR("Unknown frame number for capture result: %d", - frameNumber); - return; - } - InFlightRequest &request = mInFlightMap.editValueAt(idx); - ALOGVV("%s: got InFlightRequest requestId = %" PRId32 - ", frameNumber = %" PRId64 ", burstId = %" PRId32 - ", partialResultCount = %d, hasCallback = %d", - __FUNCTION__, request.resultExtras.requestId, - request.resultExtras.frameNumber, request.resultExtras.burstId, - result->partial_result, request.hasCallback); - // Always update the partial count to the latest one if it's not 0 - // (buffers only). When framework aggregates adjacent partial results - // into one, the latest partial count will be used. - if (result->partial_result != 0) - request.resultExtras.partialResultCount = result->partial_result; - - // Check if this result carries only partial metadata - if (mUsePartialResult && result->result != NULL) { - if (result->partial_result > mNumPartialResults || result->partial_result < 1) { - SET_ERR("Result is malformed for frame %d: partial_result %u must be in" - " the range of [1, %d] when metadata is included in the result", - frameNumber, result->partial_result, mNumPartialResults); - return; - } - isPartialResult = (result->partial_result < mNumPartialResults); - if (isPartialResult && result->num_physcam_metadata) { - SET_ERR("Result is malformed for frame %d: partial_result not allowed for" - " physical camera result", frameNumber); - return; - } - if (isPartialResult) { - request.collectedPartialResult.append(result->result); - } - - if (isPartialResult && request.hasCallback) { - // Send partial capture result - sendPartialCaptureResult(result->result, request.resultExtras, - frameNumber); - } - } - - shutterTimestamp = request.shutterTimestamp; - hasInputBufferInRequest = request.hasInputBuffer; - - // Did we get the (final) result metadata for this capture? - if (result->result != NULL && !isPartialResult) { - if (request.physicalCameraIds.size() != result->num_physcam_metadata) { - SET_ERR("Expected physical Camera metadata count %d not equal to actual count %d", - request.physicalCameraIds.size(), result->num_physcam_metadata); - return; - } - if (request.haveResultMetadata) { - SET_ERR("Called multiple times with metadata for frame %d", - frameNumber); - return; - } - for (uint32_t i = 0; i < result->num_physcam_metadata; i++) { - String8 physicalId(result->physcam_ids[i]); - std::set::iterator cameraIdIter = - request.physicalCameraIds.find(physicalId); - if (cameraIdIter != request.physicalCameraIds.end()) { - request.physicalCameraIds.erase(cameraIdIter); - } else { - SET_ERR("Total result for frame %d has already returned for camera %s", - frameNumber, physicalId.c_str()); - return; - } - } - if (mUsePartialResult && - !request.collectedPartialResult.isEmpty()) { - collectedPartialResult.acquire( - request.collectedPartialResult); - } - request.haveResultMetadata = true; - } - - uint32_t numBuffersReturned = result->num_output_buffers; - if (result->input_buffer != NULL) { - if (hasInputBufferInRequest) { - numBuffersReturned += 1; - } else { - ALOGW("%s: Input buffer should be NULL if there is no input" - " buffer sent in the request", - __FUNCTION__); - } - } - request.numBuffersLeft -= numBuffersReturned; - if (request.numBuffersLeft < 0) { - SET_ERR("Too many buffers returned for frame %d", - frameNumber); - return; - } - - camera_metadata_ro_entry_t entry; - res = find_camera_metadata_ro_entry(result->result, - ANDROID_SENSOR_TIMESTAMP, &entry); - if (res == OK && entry.count == 1) { - request.sensorTimestamp = entry.data.i64[0]; - } - - // If shutter event isn't received yet, append the output buffers to - // the in-flight request. Otherwise, return the output buffers to - // streams. - if (shutterTimestamp == 0) { - request.pendingOutputBuffers.appendArray(result->output_buffers, - result->num_output_buffers); - } else { - bool timestampIncreasing = !(request.zslCapture || request.hasInputBuffer); - returnOutputBuffers(result->output_buffers, - result->num_output_buffers, shutterTimestamp, timestampIncreasing, - request.outputSurfaces, request.resultExtras); - } - - if (result->result != NULL && !isPartialResult) { - for (uint32_t i = 0; i < result->num_physcam_metadata; i++) { - CameraMetadata physicalMetadata; - physicalMetadata.append(result->physcam_metadata[i]); - request.physicalMetadatas.push_back({String16(result->physcam_ids[i]), - physicalMetadata}); - } - if (shutterTimestamp == 0) { - request.pendingMetadata = result->result; - request.collectedPartialResult = collectedPartialResult; - } else if (request.hasCallback) { - CameraMetadata metadata; - metadata = result->result; - sendCaptureResult(metadata, request.resultExtras, - collectedPartialResult, frameNumber, - hasInputBufferInRequest, request.zslCapture && request.stillCapture, - request.cameraIdsWithZoom, request.physicalMetadatas); - } - } - - removeInFlightRequestIfReadyLocked(idx); - } // scope for mInFlightLock - - if (result->input_buffer != NULL) { - if (hasInputBufferInRequest) { - Camera3Stream *stream = - Camera3Stream::cast(result->input_buffer->stream); - res = stream->returnInputBuffer(*(result->input_buffer)); - // Note: stream may be deallocated at this point, if this buffer was the - // last reference to it. - if (res != OK) { - ALOGE("%s: RequestThread: Can't return input buffer for frame %d to" - " its stream:%s (%d)", __FUNCTION__, - frameNumber, strerror(-res), res); - } - } else { - ALOGW("%s: Input buffer should be NULL if there is no input" - " buffer sent in the request, skipping input buffer return.", - __FUNCTION__); + mStatusTracker->markComponentActive(mInFlightStatusId); } } -} -void Camera3Device::notify(const camera3_notify_msg *msg) { - ATRACE_CALL(); - sp listener; - { - Mutex::Autolock l(mOutputLock); - listener = mListener.promote(); - } + mExpectedInflightDuration += maxExpectedDuration; + return OK; +} - if (msg == NULL) { - SET_ERR("HAL sent NULL notify message!"); - return; +void Camera3Device::onInflightEntryRemovedLocked(nsecs_t duration) { + // Indicate idle inFlightMap to the status tracker + if (mInFlightMap.size() == 0) { + mRequestBufferSM.onInflightMapEmpty(); + // Hold a separate dedicated tracker lock to prevent race with disconnect and also + // avoid a deadlock during reprocess requests. + Mutex::Autolock l(mTrackerLock); + if (mStatusTracker != nullptr) { + mStatusTracker->markComponentIdle(mInFlightStatusId, Fence::NO_FENCE); + } } + mExpectedInflightDuration -= duration; +} - switch (msg->type) { - case CAMERA3_MSG_ERROR: { - notifyError(msg->message.error, listener); - break; - } - case CAMERA3_MSG_SHUTTER: { - notifyShutter(msg->message.shutter, listener); - break; +void Camera3Device::checkInflightMapLengthLocked() { + // Sanity check - if we have too many in-flight frames with long total inflight duration, + // something has likely gone wrong. This might still be legit only if application send in + // a long burst of long exposure requests. + if (mExpectedInflightDuration > kMinWarnInflightDuration) { + if (!mIsConstrainedHighSpeedConfiguration && mInFlightMap.size() > kInFlightWarnLimit) { + CLOGW("In-flight list too large: %zu, total inflight duration %" PRIu64, + mInFlightMap.size(), mExpectedInflightDuration); + } else if (mIsConstrainedHighSpeedConfiguration && mInFlightMap.size() > + kInFlightWarnLimitHighSpeed) { + CLOGW("In-flight list too large for high speed configuration: %zu," + "total inflight duration %" PRIu64, + mInFlightMap.size(), mExpectedInflightDuration); } - default: - SET_ERR("Unknown notify message from HAL: %d", - msg->type); } } -void Camera3Device::notifyError(const camera3_error_msg_t &msg, - sp listener) { - ATRACE_CALL(); - // Map camera HAL error codes to ICameraDeviceCallback error codes - // Index into this with the HAL error code - static const int32_t halErrorMap[CAMERA3_MSG_NUM_ERRORS] = { - // 0 = Unused error code - hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_INVALID_ERROR, - // 1 = CAMERA3_MSG_ERROR_DEVICE - hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_DEVICE, - // 2 = CAMERA3_MSG_ERROR_REQUEST - hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_REQUEST, - // 3 = CAMERA3_MSG_ERROR_RESULT - hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_RESULT, - // 4 = CAMERA3_MSG_ERROR_BUFFER - hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_BUFFER - }; +void Camera3Device::onInflightMapFlushedLocked() { + mExpectedInflightDuration = 0; +} - int32_t errorCode = - ((msg.error_code >= 0) && - (msg.error_code < CAMERA3_MSG_NUM_ERRORS)) ? - halErrorMap[msg.error_code] : - hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_INVALID_ERROR; - - int streamId = 0; - String16 physicalCameraId; - if (msg.error_stream != NULL) { - Camera3Stream *stream = - Camera3Stream::cast(msg.error_stream); - streamId = stream->getId(); - physicalCameraId = String16(stream->physicalCameraId()); - } - ALOGV("Camera %s: %s: HAL error, frame %d, stream %d: %d", - mId.string(), __FUNCTION__, msg.frame_number, - streamId, msg.error_code); - - CaptureResultExtras resultExtras; - switch (errorCode) { - case hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_DEVICE: - // SET_ERR calls notifyError - SET_ERR("Camera HAL reported serious device error"); - break; - case hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_REQUEST: - case hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_RESULT: - case hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_BUFFER: - { - Mutex::Autolock l(mInFlightLock); - ssize_t idx = mInFlightMap.indexOfKey(msg.frame_number); - if (idx >= 0) { - InFlightRequest &r = mInFlightMap.editValueAt(idx); - r.requestStatus = msg.error_code; - resultExtras = r.resultExtras; - bool logicalDeviceResultError = false; - if (hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_RESULT == - errorCode) { - if (physicalCameraId.size() > 0) { - String8 cameraId(physicalCameraId); - auto iter = r.physicalCameraIds.find(cameraId); - if (iter == r.physicalCameraIds.end()) { - ALOGE("%s: Reported result failure for physical camera device: %s " - " which is not part of the respective request!", - __FUNCTION__, cameraId.string()); - break; - } - r.physicalCameraIds.erase(iter); - resultExtras.errorPhysicalCameraId = physicalCameraId; - } else { - logicalDeviceResultError = true; - } - } +void Camera3Device::removeInFlightMapEntryLocked(int idx) { + ATRACE_CALL(); + nsecs_t duration = mInFlightMap.valueAt(idx).maxExpectedDuration; + mInFlightMap.removeItemsAt(idx, 1); - if (logicalDeviceResultError - || hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_REQUEST == - errorCode) { - r.skipResultMetadata = true; - } - if (logicalDeviceResultError) { - // In case of missing result check whether the buffers - // returned. If they returned, then remove inflight - // request. - // TODO: should we call this for ERROR_CAMERA_REQUEST as well? - // otherwise we are depending on HAL to send the buffers back after - // calling notifyError. Not sure if that's in the spec. - removeInFlightRequestIfReadyLocked(idx); - } - } else { - resultExtras.frameNumber = msg.frame_number; - ALOGE("Camera %s: %s: cannot find in-flight request on " - "frame %" PRId64 " error", mId.string(), __FUNCTION__, - resultExtras.frameNumber); - } - } - resultExtras.errorStreamId = streamId; - if (listener != NULL) { - listener->notifyError(errorCode, resultExtras); - } else { - ALOGE("Camera %s: %s: no listener available", mId.string(), __FUNCTION__); - } - break; - default: - // SET_ERR calls notifyError - SET_ERR("Unknown error message from HAL: %d", msg.error_code); - break; - } + onInflightEntryRemovedLocked(duration); } -void Camera3Device::notifyShutter(const camera3_shutter_msg_t &msg, - sp listener) { - ATRACE_CALL(); - ssize_t idx; - // Set timestamp for the request in the in-flight tracking - // and get the request ID to send upstream +void Camera3Device::flushInflightRequests() { + ATRACE_CALL(); + sp listener; { - Mutex::Autolock l(mInFlightLock); - idx = mInFlightMap.indexOfKey(msg.frame_number); - if (idx >= 0) { - InFlightRequest &r = mInFlightMap.editValueAt(idx); - - // Verify ordering of shutter notifications - { - Mutex::Autolock l(mOutputLock); - // TODO: need to track errors for tighter bounds on expected frame number. - if (r.hasInputBuffer) { - if (msg.frame_number < mNextReprocessShutterFrameNumber) { - SET_ERR("Reprocess shutter notification out-of-order. Expected " - "notification for frame %d, got frame %d", - mNextReprocessShutterFrameNumber, msg.frame_number); - return; - } - mNextReprocessShutterFrameNumber = msg.frame_number + 1; - } else if (r.zslCapture && r.stillCapture) { - if (msg.frame_number < mNextZslStillShutterFrameNumber) { - SET_ERR("ZSL still capture shutter notification out-of-order. Expected " - "notification for frame %d, got frame %d", - mNextZslStillShutterFrameNumber, msg.frame_number); - return; - } - mNextZslStillShutterFrameNumber = msg.frame_number + 1; - } else { - if (msg.frame_number < mNextShutterFrameNumber) { - SET_ERR("Shutter notification out-of-order. Expected " - "notification for frame %d, got frame %d", - mNextShutterFrameNumber, msg.frame_number); - return; - } - mNextShutterFrameNumber = msg.frame_number + 1; - } - } + std::lock_guard l(mOutputLock); + listener = mListener.promote(); + } - r.shutterTimestamp = msg.timestamp; - if (r.hasCallback) { - ALOGVV("Camera %s: %s: Shutter fired for frame %d (id %d) at %" PRId64, - mId.string(), __FUNCTION__, - msg.frame_number, r.resultExtras.requestId, msg.timestamp); - // Call listener, if any - if (listener != NULL) { - listener->notifyShutter(r.resultExtras, msg.timestamp); - } - // send pending result and buffers - sendCaptureResult(r.pendingMetadata, r.resultExtras, - r.collectedPartialResult, msg.frame_number, - r.hasInputBuffer, r.zslCapture && r.stillCapture, - r.cameraIdsWithZoom, r.physicalMetadatas); - } - bool timestampIncreasing = !(r.zslCapture || r.hasInputBuffer); - returnOutputBuffers(r.pendingOutputBuffers.array(), - r.pendingOutputBuffers.size(), r.shutterTimestamp, timestampIncreasing, - r.outputSurfaces, r.resultExtras); - r.pendingOutputBuffers.clear(); + FlushInflightReqStates states { + mId, mInFlightLock, mInFlightMap, mUseHalBufManager, + listener, *this, *mInterface, *this}; - removeInFlightRequestIfReadyLocked(idx); - } - } - if (idx < 0) { - SET_ERR("Shutter notification for non-existent frame number %d", - msg.frame_number); - } + camera3::flushInflightRequests(states); } CameraMetadata Camera3Device::getLatestRequestLocked() { @@ -4084,7 +2825,6 @@ CameraMetadata Camera3Device::getLatestRequestLocked() { return retVal; } - void Camera3Device::monitorMetadata(TagMonitor::eventSource source, int64_t frameNumber, nsecs_t timestamp, const CameraMetadata& metadata, const std::unordered_map& physicalMetadata) { @@ -4319,20 +3059,10 @@ status_t Camera3Device::HalInterface::configureStreams(const camera_metadata_t * activeStreams.insert(streamId); // Create Buffer ID map if necessary - if (mBufferIdMaps.count(streamId) == 0) { - mBufferIdMaps.emplace(streamId, BufferIdMap{}); - } + mBufferRecords.tryCreateBufferCache(streamId); } // remove BufferIdMap for deleted streams - for(auto it = mBufferIdMaps.begin(); it != mBufferIdMaps.end();) { - int streamId = it->first; - bool active = activeStreams.count(streamId) > 0; - if (!active) { - it = mBufferIdMaps.erase(it); - } else { - ++it; - } - } + mBufferRecords.removeInactiveBufferCaches(activeStreams); StreamConfigurationMode operationMode; res = mapToStreamConfigurationMode( @@ -4350,6 +3080,7 @@ status_t Camera3Device::HalInterface::configureStreams(const camera_metadata_t * // Invoke configureStreams device::V3_3::HalStreamConfiguration finalConfiguration; device::V3_4::HalStreamConfiguration finalConfiguration3_4; + device::V3_6::HalStreamConfiguration finalConfiguration3_6; common::V1_0::Status status; auto configStream34Cb = [&status, &finalConfiguration3_4] @@ -4358,6 +3089,12 @@ status_t Camera3Device::HalInterface::configureStreams(const camera_metadata_t * status = s; }; + auto configStream36Cb = [&status, &finalConfiguration3_6] + (common::V1_0::Status s, const device::V3_6::HalStreamConfiguration& halConfiguration) { + finalConfiguration3_6 = halConfiguration; + status = s; + }; + auto postprocConfigStream34 = [&finalConfiguration, &finalConfiguration3_4] (hardware::Return& err) -> status_t { if (!err.isOk()) { @@ -4371,8 +3108,32 @@ status_t Camera3Device::HalInterface::configureStreams(const camera_metadata_t * return OK; }; + auto postprocConfigStream36 = [&finalConfiguration, &finalConfiguration3_6] + (hardware::Return& err) -> status_t { + if (!err.isOk()) { + ALOGE("%s: Transaction error: %s", __FUNCTION__, err.description().c_str()); + return DEAD_OBJECT; + } + finalConfiguration.streams.resize(finalConfiguration3_6.streams.size()); + for (size_t i = 0; i < finalConfiguration3_6.streams.size(); i++) { + finalConfiguration.streams[i] = finalConfiguration3_6.streams[i].v3_4.v3_3; + } + return OK; + }; + // See which version of HAL we have - if (mHidlSession_3_5 != nullptr) { + if (mHidlSession_3_6 != nullptr) { + ALOGV("%s: v3.6 device found", __FUNCTION__); + device::V3_5::StreamConfiguration requestedConfiguration3_5; + requestedConfiguration3_5.v3_4 = requestedConfiguration3_4; + requestedConfiguration3_5.streamConfigCounter = mNextStreamConfigCounter++; + auto err = mHidlSession_3_6->configureStreams_3_6( + requestedConfiguration3_5, configStream36Cb); + res = postprocConfigStream36(err); + if (res != OK) { + return res; + } + } else if (mHidlSession_3_5 != nullptr) { ALOGV("%s: v3.5 device found", __FUNCTION__); device::V3_5::StreamConfiguration requestedConfiguration3_5; requestedConfiguration3_5.v3_4 = requestedConfiguration3_4; @@ -4454,11 +3215,16 @@ status_t Camera3Device::HalInterface::configureStreams(const camera_metadata_t * return INVALID_OPERATION; } device::V3_3::HalStream &src = finalConfiguration.streams[realIdx]; + device::V3_6::HalStream &src_36 = finalConfiguration3_6.streams[realIdx]; Camera3Stream* dstStream = Camera3Stream::cast(dst); int overrideFormat = mapToFrameworkFormat(src.v3_2.overrideFormat); android_dataspace overrideDataSpace = mapToFrameworkDataspace(src.overrideDataSpace); + if (mHidlSession_3_6 != nullptr) { + dstStream->setOfflineProcessingSupport(src_36.supportOffline); + } + if (dstStream->getOriginalFormat() != HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED) { dstStream->setFormatOverride(false); dstStream->setDataSpaceOverride(false); @@ -4522,7 +3288,6 @@ status_t Camera3Device::HalInterface::wrapAsHidlRequest(camera3_capture_request_ captureRequest->fmqSettingsSize = 0; { - std::lock_guard lock(mInflightLock); if (request->input_buffer != nullptr) { int32_t streamId = Camera3Stream::cast(request->input_buffer->stream)->getId(); buffer_handle_t buf = *(request->input_buffer->buffer); @@ -4542,7 +3307,7 @@ status_t Camera3Device::HalInterface::wrapAsHidlRequest(camera3_capture_request_ captureRequest->inputBuffer.acquireFence = acquireFence; captureRequest->inputBuffer.releaseFence = nullptr; - pushInflightBufferLocked(captureRequest->frameNumber, streamId, + mBufferRecords.pushInflightBuffer(captureRequest->frameNumber, streamId, request->input_buffer->buffer); inflightBuffers->push_back(std::make_pair(captureRequest->frameNumber, streamId)); } else { @@ -4583,7 +3348,8 @@ status_t Camera3Device::HalInterface::wrapAsHidlRequest(camera3_capture_request_ // Output buffers are empty when using HAL buffer manager if (!mUseHalBufManager) { - pushInflightBufferLocked(captureRequest->frameNumber, streamId, src->buffer); + mBufferRecords.pushInflightBuffer( + captureRequest->frameNumber, streamId, src->buffer); inflightBuffers->push_back(std::make_pair(captureRequest->frameNumber, streamId)); } } @@ -4640,7 +3406,7 @@ status_t Camera3Device::HalInterface::processBatchCaptureRequests( /*out*/&handlesCreated, /*out*/&inflightBuffers); } if (res != OK) { - popInflightBuffers(inflightBuffers); + mBufferRecords.popInflightBuffers(inflightBuffers); cleanupNativeHandles(&handlesCreated); return res; } @@ -4648,10 +3414,10 @@ status_t Camera3Device::HalInterface::processBatchCaptureRequests( std::vector cachesToRemove; { - std::lock_guard lock(mBufferIdMapLock); + std::lock_guard lock(mFreedBuffersLock); for (auto& pair : mFreedBuffers) { // The stream might have been removed since onBufferFreed - if (mBufferIdMaps.find(pair.first) != mBufferIdMaps.end()) { + if (mBufferRecords.isStreamCached(pair.first)) { cachesToRemove.push_back({pair.first, pair.second}); } } @@ -4758,7 +3524,7 @@ status_t Camera3Device::HalInterface::processBatchCaptureRequests( cleanupNativeHandles(&handlesCreated); } } else { - popInflightBuffers(inflightBuffers); + mBufferRecords.popInflightBuffers(inflightBuffers); cleanupNativeHandles(&handlesCreated); } return res; @@ -4820,20 +3586,20 @@ void Camera3Device::HalInterface::signalPipelineDrain(const std::vector& st status_t Camera3Device::HalInterface::switchToOffline( const std::vector& streamsToKeep, /*out*/hardware::camera::device::V3_6::CameraOfflineSessionInfo* offlineSessionInfo, - /*out*/sp* offlineSession) { + /*out*/sp* offlineSession, + /*out*/camera3::BufferRecords* bufferRecords) { ATRACE_NAME("CameraHal::switchToOffline"); if (!valid() || mHidlSession_3_6 == nullptr) { ALOGE("%s called on invalid camera!", __FUNCTION__); return INVALID_OPERATION; } - if (offlineSessionInfo == nullptr || offlineSession == nullptr) { - ALOGE("%s: offlineSessionInfo and offlineSession must not be null!", __FUNCTION__); + if (offlineSessionInfo == nullptr || offlineSession == nullptr || bufferRecords == nullptr) { + ALOGE("%s: output arguments must not be null!", __FUNCTION__); return INVALID_OPERATION; } common::V1_0::Status status = common::V1_0::Status::INTERNAL_ERROR; - auto resultCallback = [&status, &offlineSessionInfo, &offlineSession] (auto s, auto info, auto session) { status = s; @@ -4846,75 +3612,65 @@ status_t Camera3Device::HalInterface::switchToOffline( ALOGE("%s: Transaction error: %s", __FUNCTION__, err.description().c_str()); return DEAD_OBJECT; } - return CameraProviderManager::mapToStatusT(status); + + status_t ret = CameraProviderManager::mapToStatusT(status); + if (ret != OK) { + return ret; + } + + // TODO: assert no ongoing requestBuffer/returnBuffer call here + // TODO: update RequestBufferStateMachine to block requestBuffer/returnBuffer once HAL + // returns from switchToOffline. + + + // Validate buffer caches + std::vector streams; + streams.reserve(offlineSessionInfo->offlineStreams.size()); + for (auto offlineStream : offlineSessionInfo->offlineStreams) { + int32_t id = offlineStream.id; + streams.push_back(id); + // Verify buffer caches + std::vector bufIds(offlineStream.circulatingBufferIds.begin(), + offlineStream.circulatingBufferIds.end()); + if (!verifyBufferIds(id, bufIds)) { + ALOGE("%s: stream ID %d buffer cache records mismatch!", __FUNCTION__, id); + return UNKNOWN_ERROR; + } + } + + // Move buffer records + bufferRecords->takeBufferCaches(mBufferRecords, streams); + bufferRecords->takeInflightBufferMap(mBufferRecords); + bufferRecords->takeRequestedBufferMap(mBufferRecords); + return ret; } void Camera3Device::HalInterface::getInflightBufferKeys( std::vector>* out) { - std::lock_guard lock(mInflightLock); - out->clear(); - out->reserve(mInflightBufferMap.size()); - for (auto& pair : mInflightBufferMap) { - uint64_t key = pair.first; - int32_t streamId = key & 0xFFFFFFFF; - int32_t frameNumber = (key >> 32) & 0xFFFFFFFF; - out->push_back(std::make_pair(frameNumber, streamId)); - } + mBufferRecords.getInflightBufferKeys(out); return; } void Camera3Device::HalInterface::getInflightRequestBufferKeys( std::vector* out) { - std::lock_guard lock(mRequestedBuffersLock); - out->clear(); - out->reserve(mRequestedBuffers.size()); - for (auto& pair : mRequestedBuffers) { - out->push_back(pair.first); - } + mBufferRecords.getInflightRequestBufferKeys(out); return; } -status_t Camera3Device::HalInterface::pushInflightBufferLocked( - int32_t frameNumber, int32_t streamId, buffer_handle_t *buffer) { - uint64_t key = static_cast(frameNumber) << 32 | static_cast(streamId); - mInflightBufferMap[key] = buffer; - return OK; +bool Camera3Device::HalInterface::verifyBufferIds( + int32_t streamId, std::vector& bufIds) { + return mBufferRecords.verifyBufferIds(streamId, bufIds); } status_t Camera3Device::HalInterface::popInflightBuffer( int32_t frameNumber, int32_t streamId, /*out*/ buffer_handle_t **buffer) { - std::lock_guard lock(mInflightLock); - - uint64_t key = static_cast(frameNumber) << 32 | static_cast(streamId); - auto it = mInflightBufferMap.find(key); - if (it == mInflightBufferMap.end()) return NAME_NOT_FOUND; - if (buffer != nullptr) { - *buffer = it->second; - } - mInflightBufferMap.erase(it); - return OK; -} - -void Camera3Device::HalInterface::popInflightBuffers( - const std::vector>& buffers) { - for (const auto& pair : buffers) { - int32_t frameNumber = pair.first; - int32_t streamId = pair.second; - popInflightBuffer(frameNumber, streamId, nullptr); - } + return mBufferRecords.popInflightBuffer(frameNumber, streamId, buffer); } status_t Camera3Device::HalInterface::pushInflightRequestBuffer( uint64_t bufferId, buffer_handle_t* buf, int32_t streamId) { - std::lock_guard lock(mRequestedBuffersLock); - auto pair = mRequestedBuffers.insert({bufferId, {streamId, buf}}); - if (!pair.second) { - ALOGE("%s: bufId %" PRIu64 " is already inflight!", - __FUNCTION__, bufferId); - return BAD_VALUE; - } - return OK; + return mBufferRecords.pushInflightRequestBuffer(bufferId, buf, streamId); } // Find and pop a buffer_handle_t based on bufferId @@ -4922,81 +3678,29 @@ status_t Camera3Device::HalInterface::popInflightRequestBuffer( uint64_t bufferId, /*out*/ buffer_handle_t** buffer, /*optional out*/ int32_t* streamId) { - if (buffer == nullptr) { - ALOGE("%s: buffer (%p) must not be null", __FUNCTION__, buffer); - return BAD_VALUE; - } - std::lock_guard lock(mRequestedBuffersLock); - auto it = mRequestedBuffers.find(bufferId); - if (it == mRequestedBuffers.end()) { - ALOGE("%s: bufId %" PRIu64 " is not inflight!", - __FUNCTION__, bufferId); - return BAD_VALUE; - } - *buffer = it->second.second; - if (streamId != nullptr) { - *streamId = it->second.first; - } - mRequestedBuffers.erase(it); - return OK; + return mBufferRecords.popInflightRequestBuffer(bufferId, buffer, streamId); } std::pair Camera3Device::HalInterface::getBufferId( const buffer_handle_t& buf, int streamId) { - std::lock_guard lock(mBufferIdMapLock); - - BufferIdMap& bIdMap = mBufferIdMaps.at(streamId); - auto it = bIdMap.find(buf); - if (it == bIdMap.end()) { - bIdMap[buf] = mNextBufferId++; - ALOGV("stream %d now have %zu buffer caches, buf %p", - streamId, bIdMap.size(), buf); - return std::make_pair(true, mNextBufferId - 1); - } else { - return std::make_pair(false, it->second); - } + return mBufferRecords.getBufferId(buf, streamId); } void Camera3Device::HalInterface::onBufferFreed( int streamId, const native_handle_t* handle) { - std::lock_guard lock(mBufferIdMapLock); - uint64_t bufferId = BUFFER_ID_NO_BUFFER; - auto mapIt = mBufferIdMaps.find(streamId); - if (mapIt == mBufferIdMaps.end()) { - // streamId might be from a deleted stream here - ALOGI("%s: stream %d has been removed", - __FUNCTION__, streamId); - return; - } - BufferIdMap& bIdMap = mapIt->second; - auto it = bIdMap.find(handle); - if (it == bIdMap.end()) { - ALOGW("%s: cannot find buffer %p in stream %d", - __FUNCTION__, handle, streamId); - return; - } else { - bufferId = it->second; - bIdMap.erase(it); - ALOGV("%s: stream %d now have %zu buffer caches after removing buf %p", - __FUNCTION__, streamId, bIdMap.size(), handle); + uint32_t bufferId = mBufferRecords.removeOneBufferCache(streamId, handle); + std::lock_guard lock(mFreedBuffersLock); + if (bufferId != BUFFER_ID_NO_BUFFER) { + mFreedBuffers.push_back(std::make_pair(streamId, bufferId)); } - mFreedBuffers.push_back(std::make_pair(streamId, bufferId)); } void Camera3Device::HalInterface::onStreamReConfigured(int streamId) { - std::lock_guard lock(mBufferIdMapLock); - auto mapIt = mBufferIdMaps.find(streamId); - if (mapIt == mBufferIdMaps.end()) { - ALOGE("%s: streamId %d not found!", __FUNCTION__, streamId); - return; - } - - BufferIdMap& bIdMap = mapIt->second; - for (const auto& it : bIdMap) { - uint64_t bufferId = it.second; + std::vector bufIds = mBufferRecords.clearBufferCaches(streamId); + std::lock_guard lock(mFreedBuffersLock); + for (auto bufferId : bufIds) { mFreedBuffers.push_back(std::make_pair(streamId, bufferId)); } - bIdMap.clear(); } /** @@ -6030,17 +4734,22 @@ void Camera3Device::RequestThread::signalPipelineDrain(const std::vector& s status_t Camera3Device::RequestThread::switchToOffline( const std::vector& streamsToKeep, /*out*/hardware::camera::device::V3_6::CameraOfflineSessionInfo* offlineSessionInfo, - /*out*/sp* offlineSession) { + /*out*/sp* offlineSession, + /*out*/camera3::BufferRecords* bufferRecords) { Mutex::Autolock l(mRequestLock); clearRepeatingRequestsLocked(/*lastFrameNumber*/nullptr); - // Theoretically we should also check for mRepeatingRequests.empty(), but the API interface - // is serialized by mInterfaceLock so skip that check. + // Wait until request thread is fully stopped + // TBD: check if request thread is being paused by other APIs (shouldn't be) + + // We could also check for mRepeatingRequests.empty(), but the API interface + // is serialized by Camera3Device::mInterfaceLock so no one should be able to submit any + // new requests during the call; hence skip that check. bool queueEmpty = mNextRequests.empty() && mRequestQueue.empty(); while (!queueEmpty) { status_t res = mRequestSubmittedSignal.waitRelative(mRequestLock, kRequestSubmitTimeout); if (res == TIMED_OUT) { - ALOGE("%s: request thread failed to submit a request within timeout!", __FUNCTION__); + ALOGE("%s: request thread failed to submit one request within timeout!", __FUNCTION__); return res; } else if (res != OK) { ALOGE("%s: request thread failed to submit a request: %s (%d)!", @@ -6049,13 +4758,14 @@ status_t Camera3Device::RequestThread::switchToOffline( } queueEmpty = mNextRequests.empty() && mRequestQueue.empty(); } + return mInterface->switchToOffline( - streamsToKeep, offlineSessionInfo, offlineSession); + streamsToKeep, offlineSessionInfo, offlineSession, bufferRecords); } nsecs_t Camera3Device::getExpectedInFlightDuration() { ATRACE_CALL(); - Mutex::Autolock al(mInFlightLock); + std::lock_guard l(mInFlightLock); return mExpectedInflightDuration > kMinInflightDuration ? mExpectedInflightDuration : kMinInflightDuration; } @@ -6141,7 +4851,7 @@ void Camera3Device::RequestThread::cleanUpFailedRequests(bool sendRequestError) { sp parent = mParent.promote(); if (parent != NULL) { - Mutex::Autolock l(parent->mInFlightLock); + std::lock_guard l(parent->mInFlightLock); ssize_t idx = parent->mInFlightMap.indexOfKey(captureRequest->mResultExtras.frameNumber); if (idx >= 0) { ALOGV("%s: Remove inflight request from queue: frameNumber %" PRId64, @@ -6830,6 +5540,7 @@ void Camera3Device::RequestBufferStateMachine::endRequestBuffer() { void Camera3Device::RequestBufferStateMachine::onStreamsConfigured() { std::lock_guard lock(mLock); + mSwitchedToOffline = false; mStatus = RB_STATUS_READY; return; } @@ -6872,6 +5583,20 @@ void Camera3Device::RequestBufferStateMachine::onWaitUntilIdle() { return; } +bool Camera3Device::RequestBufferStateMachine::onSwitchToOfflineSuccess() { + std::lock_guard lock(mLock); + if (mRequestBufferOngoing) { + ALOGE("%s: HAL must not be requesting buffer after HAL returns switchToOffline!", + __FUNCTION__); + return false; + } + mSwitchedToOffline = true; + mInflightMapEmpty = true; + mRequestThreadPaused = true; + mStatus = RB_STATUS_STOPPED; + return true; +} + void Camera3Device::RequestBufferStateMachine::notifyTrackerLocked(bool active) { sp statusTracker = mStatusTracker.promote(); if (statusTracker != nullptr) { @@ -6891,75 +5616,40 @@ bool Camera3Device::RequestBufferStateMachine::checkSwitchToStopLocked() { return false; } -status_t Camera3Device::fixupMonochromeTags(const CameraMetadata& deviceInfo, - CameraMetadata& resultMetadata) { - status_t res = OK; - if (!mNeedFixupMonochromeTags) { - return res; - } +bool Camera3Device::startRequestBuffer() { + return mRequestBufferSM.startRequestBuffer(); +} - // Remove tags that are not applicable to monochrome camera. - int32_t tagsToRemove[] = { - ANDROID_SENSOR_GREEN_SPLIT, - ANDROID_SENSOR_NEUTRAL_COLOR_POINT, - ANDROID_COLOR_CORRECTION_MODE, - ANDROID_COLOR_CORRECTION_TRANSFORM, - ANDROID_COLOR_CORRECTION_GAINS, - }; - for (auto tag : tagsToRemove) { - res = resultMetadata.erase(tag); - if (res != OK) { - ALOGE("%s: Failed to remove tag %d for monochrome camera", __FUNCTION__, tag); - return res; - } - } +void Camera3Device::endRequestBuffer() { + mRequestBufferSM.endRequestBuffer(); +} - // ANDROID_SENSOR_DYNAMIC_BLACK_LEVEL - camera_metadata_entry blEntry = resultMetadata.find(ANDROID_SENSOR_DYNAMIC_BLACK_LEVEL); - for (size_t i = 1; i < blEntry.count; i++) { - blEntry.data.f[i] = blEntry.data.f[0]; - } +nsecs_t Camera3Device::getWaitDuration() { + return kBaseGetBufferWait + getExpectedInFlightDuration(); +} - // ANDROID_SENSOR_NOISE_PROFILE - camera_metadata_entry npEntry = resultMetadata.find(ANDROID_SENSOR_NOISE_PROFILE); - if (npEntry.count > 0 && npEntry.count % 2 == 0) { - double np[] = {npEntry.data.d[0], npEntry.data.d[1]}; - res = resultMetadata.update(ANDROID_SENSOR_NOISE_PROFILE, np, 2); - if (res != OK) { - ALOGE("%s: Failed to update SENSOR_NOISE_PROFILE: %s (%d)", - __FUNCTION__, strerror(-res), res); - return res; - } - } +void Camera3Device::getInflightBufferKeys(std::vector>* out) { + mInterface->getInflightBufferKeys(out); +} - // ANDROID_STATISTICS_LENS_SHADING_MAP - camera_metadata_ro_entry lsSizeEntry = deviceInfo.find(ANDROID_LENS_INFO_SHADING_MAP_SIZE); - camera_metadata_entry lsEntry = resultMetadata.find(ANDROID_STATISTICS_LENS_SHADING_MAP); - if (lsSizeEntry.count == 2 && lsEntry.count > 0 - && (int32_t)lsEntry.count == 4 * lsSizeEntry.data.i32[0] * lsSizeEntry.data.i32[1]) { - for (int32_t i = 0; i < lsSizeEntry.data.i32[0] * lsSizeEntry.data.i32[1]; i++) { - lsEntry.data.f[4*i+1] = lsEntry.data.f[4*i]; - lsEntry.data.f[4*i+2] = lsEntry.data.f[4*i]; - lsEntry.data.f[4*i+3] = lsEntry.data.f[4*i]; - } - } +void Camera3Device::getInflightRequestBufferKeys(std::vector* out) { + mInterface->getInflightRequestBufferKeys(out); +} - // ANDROID_TONEMAP_CURVE_BLUE - // ANDROID_TONEMAP_CURVE_GREEN - // ANDROID_TONEMAP_CURVE_RED - camera_metadata_entry tcbEntry = resultMetadata.find(ANDROID_TONEMAP_CURVE_BLUE); - camera_metadata_entry tcgEntry = resultMetadata.find(ANDROID_TONEMAP_CURVE_GREEN); - camera_metadata_entry tcrEntry = resultMetadata.find(ANDROID_TONEMAP_CURVE_RED); - if (tcbEntry.count > 0 - && tcbEntry.count == tcgEntry.count - && tcbEntry.count == tcrEntry.count) { - for (size_t i = 0; i < tcbEntry.count; i++) { - tcbEntry.data.f[i] = tcrEntry.data.f[i]; - tcgEntry.data.f[i] = tcrEntry.data.f[i]; - } +std::vector> Camera3Device::getAllStreams() { + std::vector> ret; + bool hasInputStream = mInputStream != nullptr; + ret.reserve(mOutputStreams.size() + mDeletedStreams.size() + ((hasInputStream) ? 1 : 0)); + if (hasInputStream) { + ret.push_back(mInputStream); } - - return res; + for (size_t i = 0; i < mOutputStreams.size(); i++) { + ret.push_back(mOutputStreams[i]); + } + for (size_t i = 0; i < mDeletedStreams.size(); i++) { + ret.push_back(mDeletedStreams[i]); + } + return ret; } status_t Camera3Device::switchToOffline( @@ -6972,24 +5662,231 @@ status_t Camera3Device::switchToOffline( } Mutex::Autolock il(mInterfaceLock); - Mutex::Autolock l(mLock); - // 1. Stop repeating request, wait until request thread submitted all requests, then - // call HAL switchToOffline + bool hasInputStream = mInputStream != nullptr; + int32_t inputStreamId = hasInputStream ? mInputStream->getId() : -1; + bool inputStreamSupportsOffline = hasInputStream ? + mInputStream->getOfflineProcessingSupport() : false; + auto outputStreamIds = mOutputStreams.getStreamIds(); + auto streamIds = outputStreamIds; + if (hasInputStream) { + streamIds.push_back(mInputStream->getId()); + } + + // Check all streams in streamsToKeep supports offline mode + for (auto id : streamsToKeep) { + if (std::find(streamIds.begin(), streamIds.end(), id) == streamIds.end()) { + ALOGE("%s: Unknown stream ID %d", __FUNCTION__, id); + return BAD_VALUE; + } else if (id == inputStreamId) { + if (!inputStreamSupportsOffline) { + ALOGE("%s: input stream %d cannot be switched to offline", + __FUNCTION__, id); + return BAD_VALUE; + } + } else { + sp stream = mOutputStreams.get(id); + if (!stream->getOfflineProcessingSupport()) { + ALOGE("%s: output stream %d cannot be switched to offline", + __FUNCTION__, id); + return BAD_VALUE; + } + } + } + + // TODO: block surface sharing and surface group streams until we can support them + + // Stop repeating request, wait until all remaining requests are submitted, then call into + // HAL switchToOffline hardware::camera::device::V3_6::CameraOfflineSessionInfo offlineSessionInfo; sp offlineSession; + camera3::BufferRecords bufferRecords; + status_t ret = mRequestThread->switchToOffline( + streamsToKeep, &offlineSessionInfo, &offlineSession, &bufferRecords); + + if (ret != OK) { + SET_ERR("Switch to offline failed: %s (%d)", strerror(-ret), ret); + return ret; + } + + bool succ = mRequestBufferSM.onSwitchToOfflineSuccess(); + if (!succ) { + SET_ERR("HAL must not be calling requestStreamBuffers call"); + // TODO: block ALL callbacks from HAL till app configured new streams? + return UNKNOWN_ERROR; + } + + // Verify offlineSessionInfo + std::vector offlineStreamIds; + offlineStreamIds.reserve(offlineSessionInfo.offlineStreams.size()); + for (auto offlineStream : offlineSessionInfo.offlineStreams) { + // verify stream IDs + int32_t id = offlineStream.id; + if (std::find(streamIds.begin(), streamIds.end(), id) == streamIds.end()) { + SET_ERR("stream ID %d not found!", id); + return UNKNOWN_ERROR; + } + + // When not using HAL buf manager, only allow streams requested by app to be preserved + if (!mUseHalBufManager) { + if (std::find(streamsToKeep.begin(), streamsToKeep.end(), id) == streamsToKeep.end()) { + SET_ERR("stream ID %d must not be switched to offline!", id); + return UNKNOWN_ERROR; + } + } + + offlineStreamIds.push_back(id); + sp stream = (id == inputStreamId) ? + static_cast>(mInputStream) : + static_cast>(mOutputStreams.get(id)); + // Verify number of outstanding buffers + if (stream->getOutstandingBuffersCount() != offlineStream.numOutstandingBuffers) { + SET_ERR("Offline stream %d # of remaining buffer mismatch: (%zu,%d) (service/HAL)", + id, stream->getOutstandingBuffersCount(), offlineStream.numOutstandingBuffers); + return UNKNOWN_ERROR; + } + } + + // Verify all streams to be deleted don't have any outstanding buffers + if (hasInputStream && std::find(offlineStreamIds.begin(), offlineStreamIds.end(), + inputStreamId) == offlineStreamIds.end()) { + if (mInputStream->hasOutstandingBuffers()) { + SET_ERR("Input stream %d still has %zu outstanding buffer!", + inputStreamId, mInputStream->getOutstandingBuffersCount()); + return UNKNOWN_ERROR; + } + } + + for (const auto& outStreamId : outputStreamIds) { + if (std::find(offlineStreamIds.begin(), offlineStreamIds.end(), + outStreamId) == offlineStreamIds.end()) { + auto outStream = mOutputStreams.get(outStreamId); + if (outStream->hasOutstandingBuffers()) { + SET_ERR("Output stream %d still has %zu outstanding buffer!", + outStreamId, outStream->getOutstandingBuffersCount()); + return UNKNOWN_ERROR; + } + } + } - mRequestThread->switchToOffline(streamsToKeep, &offlineSessionInfo, &offlineSession); + InFlightRequestMap offlineReqs; + // Verify inflight requests and their pending buffers + { + std::lock_guard l(mInFlightLock); + for (auto offlineReq : offlineSessionInfo.offlineRequests) { + int idx = mInFlightMap.indexOfKey(offlineReq.frameNumber); + if (idx == NAME_NOT_FOUND) { + SET_ERR("Offline request frame number %d not found!", offlineReq.frameNumber); + return UNKNOWN_ERROR; + } + const auto& inflightReq = mInFlightMap.valueAt(idx); + // TODO: check specific stream IDs + size_t numBuffersLeft = static_cast(inflightReq.numBuffersLeft); + if (numBuffersLeft != offlineReq.pendingStreams.size()) { + SET_ERR("Offline request # of remaining buffer mismatch: (%d,%d) (service/HAL)", + inflightReq.numBuffersLeft, offlineReq.pendingStreams.size()); + return UNKNOWN_ERROR; + } + offlineReqs.add(offlineReq.frameNumber, inflightReq); + } + } - // 2. Verify offlineSessionInfo - // 3. create Camera3OfflineSession and transfer object ownership + // Create Camera3OfflineSession and transfer object ownership // (streams, inflight requests, buffer caches) - *session = new Camera3OfflineSession(mId); + camera3::StreamSet offlineStreamSet; + sp inputStream; + for (auto offlineStream : offlineSessionInfo.offlineStreams) { + int32_t id = offlineStream.id; + if (mInputStream != nullptr && id == mInputStream->getId()) { + inputStream = mInputStream; + } else { + offlineStreamSet.add(id, mOutputStreams.get(id)); + } + } + + // TODO: check if we need to lock before copying states + // though technically no other thread should be talking to Camera3Device at this point + Camera3OfflineStates offlineStates( + mTagMonitor, mVendorTagId, mUseHalBufManager, mNeedFixupMonochromeTags, + mUsePartialResult, mNumPartialResults, mNextResultFrameNumber, + mNextReprocessResultFrameNumber, mNextZslStillResultFrameNumber, + mNextShutterFrameNumber, mNextReprocessShutterFrameNumber, + mNextZslStillShutterFrameNumber, mDeviceInfo, mPhysicalDeviceInfoMap, + mDistortionMappers, mZoomRatioMappers); + + *session = new Camera3OfflineSession(mId, inputStream, offlineStreamSet, + std::move(bufferRecords), offlineReqs, offlineStates, offlineSession); + + // Delete all streams that has been transferred to offline session + Mutex::Autolock l(mLock); + for (auto offlineStream : offlineSessionInfo.offlineStreams) { + int32_t id = offlineStream.id; + if (mInputStream != nullptr && id == mInputStream->getId()) { + mInputStream.clear(); + } else { + mOutputStreams.remove(id); + } + } + + // disconnect all other streams and switch to UNCONFIGURED state + if (mInputStream != nullptr) { + ret = mInputStream->disconnect(); + if (ret != OK) { + SET_ERR_L("disconnect input stream failed!"); + return UNKNOWN_ERROR; + } + } + + for (auto streamId : mOutputStreams.getStreamIds()) { + sp stream = mOutputStreams.get(streamId); + ret = stream->disconnect(); + if (ret != OK) { + SET_ERR_L("disconnect output stream %d failed!", streamId); + return UNKNOWN_ERROR; + } + } + + mInputStream.clear(); + mOutputStreams.clear(); + mNeedConfig = true; + internalUpdateStatusLocked(STATUS_UNCONFIGURED); + mOperatingMode = NO_MODE; + mIsConstrainedHighSpeedConfiguration = false; - // 4. Delete unneeded streams - // 5. Verify Camera3Device is in unconfigured state return OK; + // TO be done by CameraDeviceClient/Camera3OfflineSession + // register the offline client to camera service + // Setup result passthing threads etc + // Initialize offline session so HAL can start sending callback to it (result Fmq) + // TODO: check how many onIdle callback will be sent + // Java side to make sure the CameraCaptureSession is properly closed +} + +void Camera3Device::getOfflineStreamIds(std::vector *offlineStreamIds) { + ATRACE_CALL(); + + if (offlineStreamIds == nullptr) { + return; + } + + Mutex::Autolock il(mInterfaceLock); + + auto streamIds = mOutputStreams.getStreamIds(); + bool hasInputStream = mInputStream != nullptr; + if (hasInputStream && mInputStream->getOfflineProcessingSupport()) { + offlineStreamIds->push_back(mInputStream->getId()); + } + + for (const auto & streamId : streamIds) { + sp stream = mOutputStreams.get(streamId); + // Streams that use the camera buffer manager are currently not supported in + // offline mode + if (stream->getOfflineProcessingSupport() && + (stream->getStreamSetId() == CAMERA3_STREAM_SET_ID_INVALID)) { + offlineStreamIds->push_back(streamId); + } + } } }; // namespace android diff --git a/services/camera/libcameraservice/device3/Camera3Device.h b/services/camera/libcameraservice/device3/Camera3Device.h index 5faabd141d..7279baf5a5 100644 --- a/services/camera/libcameraservice/device3/Camera3Device.h +++ b/services/camera/libcameraservice/device3/Camera3Device.h @@ -43,10 +43,14 @@ #include #include "common/CameraDeviceBase.h" +#include "device3/BufferUtils.h" #include "device3/StatusTracker.h" #include "device3/Camera3BufferManager.h" #include "device3/DistortionMapper.h" #include "device3/ZoomRatioMapper.h" +#include "device3/InFlightRequest.h" +#include "device3/Camera3OutputInterface.h" +#include "device3/Camera3OfflineSession.h" #include "utils/TagMonitor.h" #include "utils/LatencyHistogram.h" #include @@ -69,7 +73,11 @@ class Camera3StreamInterface; */ class Camera3Device : public CameraDeviceBase, - virtual public hardware::camera::device::V3_5::ICameraDeviceCallback { + virtual public hardware::camera::device::V3_5::ICameraDeviceCallback, + public camera3::SetErrorInterface, + public camera3::InflightRequestUpdateInterface, + public camera3::RequestBufferInterface, + public camera3::FlushBufferInterface { public: explicit Camera3Device(const String8& id); @@ -142,6 +150,8 @@ class Camera3Device : status_t getInputBufferProducer( sp *producer) override; + void getOfflineStreamIds(std::vector *offlineStreamIds) override; + status_t createDefaultRequest(int templateId, CameraMetadata *request) override; // Transitions to the idle state on success @@ -201,6 +211,16 @@ class Camera3Device : status_t switchToOffline(const std::vector& streamsToKeep, /*out*/ sp* session) override; + // RequestBufferInterface + bool startRequestBuffer() override; + void endRequestBuffer() override; + nsecs_t getWaitDuration() override; + + // FlushBufferInterface + void getInflightBufferKeys(std::vector>* out) override; + void getInflightRequestBufferKeys(std::vector* out) override; + std::vector> getAllStreams() override; + /** * Helper functions to map between framework and HIDL values */ @@ -213,8 +233,6 @@ class Camera3Device : // Returns a negative error code if the passed-in operation mode is not valid. static status_t mapToStreamConfigurationMode(camera3_stream_configuration_mode_t operationMode, /*out*/ hardware::camera::device::V3_2::StreamConfigurationMode *mode); - static camera3_buffer_status_t mapHidlBufferStatus( - hardware::camera::device::V3_2::BufferStatus status); static int mapToFrameworkFormat(hardware::graphics::common::V1_0::PixelFormat pixelFormat); static android_dataspace mapToFrameworkDataspace( hardware::camera::device::V3_2::DataspaceFlags); @@ -229,7 +247,6 @@ class Camera3Device : // internal typedefs using RequestMetadataQueue = hardware::MessageQueue; - using ResultMetadataQueue = hardware::MessageQueue; static const size_t kDumpLockAttempts = 10; static const size_t kDumpSleepDuration = 100000; // 0.10 sec @@ -283,7 +300,8 @@ class Camera3Device : * Adapter for legacy HAL / HIDL HAL interface calls; calls either into legacy HALv3 or the * HIDL HALv3 interfaces. */ - class HalInterface : public camera3::Camera3StreamBufferFreedListener { + class HalInterface : public camera3::Camera3StreamBufferFreedListener, + public camera3::BufferRecordsInterface { public: HalInterface(sp &session, std::shared_ptr queue, @@ -321,27 +339,31 @@ class Camera3Device : bool isReconfigurationRequired(CameraMetadata& oldSessionParams, CameraMetadata& newSessionParams); + // Upon successful return, HalInterface will return buffer maps needed for offline + // processing, and clear all its internal buffer maps. status_t switchToOffline( const std::vector& streamsToKeep, /*out*/hardware::camera::device::V3_6::CameraOfflineSessionInfo* offlineSessionInfo, - /*out*/sp* offlineSession); + /*out*/sp* offlineSession, + /*out*/camera3::BufferRecords* bufferRecords); - // method to extract buffer's unique ID - // return pair of (newlySeenBuffer?, bufferId) - std::pair getBufferId(const buffer_handle_t& buf, int streamId); + ///////////////////////////////////////////////////////////////////// + // Implements BufferRecordsInterface + + std::pair getBufferId( + const buffer_handle_t& buf, int streamId) override; - // Find a buffer_handle_t based on frame number and stream ID status_t popInflightBuffer(int32_t frameNumber, int32_t streamId, - /*out*/ buffer_handle_t **buffer); + /*out*/ buffer_handle_t **buffer) override; - // Register a bufId (streamId, buffer_handle_t) to inflight request buffer status_t pushInflightRequestBuffer( - uint64_t bufferId, buffer_handle_t* buf, int32_t streamId); + uint64_t bufferId, buffer_handle_t* buf, int32_t streamId) override; - // Find a buffer_handle_t based on bufferId status_t popInflightRequestBuffer(uint64_t bufferId, /*out*/ buffer_handle_t** buffer, - /*optional out*/ int32_t* streamId = nullptr); + /*optional out*/ int32_t* streamId = nullptr) override; + + ///////////////////////////////////////////////////////////////////// // Get a vector of (frameNumber, streamId) pair of currently inflight // buffers @@ -352,7 +374,6 @@ class Camera3Device : void onStreamReConfigured(int streamId); - static const uint64_t BUFFER_ID_NO_BUFFER = 0; private: // Always valid sp mHidlSession; @@ -367,8 +388,6 @@ class Camera3Device : std::shared_ptr mRequestMetadataQueue; - std::mutex mInflightLock; - // The output HIDL request still depends on input camera3_capture_request_t // Do not free input camera3_capture_request_t before output HIDL request status_t wrapAsHidlRequest(camera3_capture_request_t* in, @@ -382,55 +401,20 @@ class Camera3Device : // Pop inflight buffers based on pairs of (frameNumber,streamId) void popInflightBuffers(const std::vector>& buffers); - // Cache of buffer handles keyed off (frameNumber << 32 | streamId) - std::unordered_map mInflightBufferMap; + // Return true if the input caches match what we have; otherwise false + bool verifyBufferIds(int32_t streamId, std::vector& inBufIds); // Delete and optionally close native handles and clear the input vector afterward static void cleanupNativeHandles( std::vector *handles, bool closeFd = false); - struct BufferHasher { - size_t operator()(const buffer_handle_t& buf) const { - if (buf == nullptr) - return 0; - - size_t result = 1; - result = 31 * result + buf->numFds; - for (int i = 0; i < buf->numFds; i++) { - result = 31 * result + buf->data[i]; - } - return result; - } - }; - - struct BufferComparator { - bool operator()(const buffer_handle_t& buf1, const buffer_handle_t& buf2) const { - if (buf1->numFds == buf2->numFds) { - for (int i = 0; i < buf1->numFds; i++) { - if (buf1->data[i] != buf2->data[i]) { - return false; - } - } - return true; - } - return false; - } - }; - - std::mutex mBufferIdMapLock; // protecting mBufferIdMaps and mNextBufferId - typedef std::unordered_map BufferIdMap; - // stream ID -> per stream buffer ID map - std::unordered_map mBufferIdMaps; - uint64_t mNextBufferId = 1; // 0 means no buffer - virtual void onBufferFreed(int streamId, const native_handle_t* handle) override; + std::mutex mFreedBuffersLock; std::vector> mFreedBuffers; - // Buffers given to HAL through requestStreamBuffer API - std::mutex mRequestedBuffersLock; - std::unordered_map> mRequestedBuffers; + // Keep track of buffer cache and inflight buffer records + camera3::BufferRecords mBufferRecords; uint32_t mNextStreamConfigCounter = 1; @@ -473,24 +457,7 @@ class Camera3Device : // Tracking cause of fatal errors when in STATUS_ERROR String8 mErrorCause; - // Synchronized mapping of stream IDs to stream instances - class StreamSet { - public: - status_t add(int streamId, sp); - ssize_t remove(int streamId); - sp get(int streamId); - // get by (underlying) vector index - sp operator[] (size_t index); - size_t size() const; - std::vector getStreamIds(); - void clear(); - - private: - mutable std::mutex mLock; - KeyedVector> mData; - }; - - StreamSet mOutputStreams; + camera3::StreamSet mOutputStreams; sp mInputStream; int mNextStreamId; bool mNeedConfig; @@ -579,15 +546,6 @@ class Camera3Device : const hardware::hidl_vec< hardware::camera::device::V3_2::StreamBuffer>& buffers) override; - // Handle one capture result. Assume that mProcessCaptureResultLock is held. - void processOneCaptureResultLocked( - const hardware::camera::device::V3_2::CaptureResult& result, - const hardware::hidl_vec< - hardware::camera::device::V3_4::PhysicalCameraMetadata> physicalCameraMetadata); - status_t readOneCameraMetadataLocked(uint64_t fmqResultSize, - hardware::camera::device::V3_2::CameraMetadata& resultMetadata, - const hardware::camera::device::V3_2::CameraMetadata& result); - // Handle one notify message void notify(const hardware::camera::device::V3_2::NotifyMsg& msg); @@ -710,23 +668,26 @@ class Camera3Device : * error message to indicate why. Only the first call's message will be * used. The message is also sent to the log. */ - void setErrorState(const char *fmt, ...); + void setErrorState(const char *fmt, ...) override; + void setErrorStateLocked(const char *fmt, ...) override; void setErrorStateV(const char *fmt, va_list args); - void setErrorStateLocked(const char *fmt, ...); void setErrorStateLockedV(const char *fmt, va_list args); + ///////////////////////////////////////////////////////////////////// + // Implements InflightRequestUpdateInterface + + void onInflightEntryRemovedLocked(nsecs_t duration) override; + void checkInflightMapLengthLocked() override; + void onInflightMapFlushedLocked() override; + + ///////////////////////////////////////////////////////////////////// + /** * Debugging trylock/spin method * Try to acquire a lock a few times with sleeps between before giving up. */ bool tryLockSpinRightRound(Mutex& lock); - /** - * Helper function to determine if an input size for implementation defined - * format is supported. - */ - bool isOpaqueInputSizeSupported(uint32_t width, uint32_t height); - /** * Helper function to get the largest Jpeg resolution (in area) * Return Size(0, 0) if static metatdata is invalid @@ -860,7 +821,8 @@ class Camera3Device : status_t switchToOffline( const std::vector& streamsToKeep, /*out*/hardware::camera::device::V3_6::CameraOfflineSessionInfo* offlineSessionInfo, - /*out*/sp* offlineSession); + /*out*/sp* offlineSession, + /*out*/camera3::BufferRecords* bufferRecords); protected: @@ -1021,119 +983,12 @@ class Camera3Device : /** * In-flight queue for tracking completion of capture requests. */ + std::mutex mInFlightLock; + camera3::InFlightRequestMap mInFlightMap; + nsecs_t mExpectedInflightDuration = 0; + // End of mInFlightLock protection scope - struct InFlightRequest { - // Set by notify() SHUTTER call. - nsecs_t shutterTimestamp; - // Set by process_capture_result(). - nsecs_t sensorTimestamp; - int requestStatus; - // Set by process_capture_result call with valid metadata - bool haveResultMetadata; - // Decremented by calls to process_capture_result with valid output - // and input buffers - int numBuffersLeft; - CaptureResultExtras resultExtras; - // If this request has any input buffer - bool hasInputBuffer; - - // The last metadata that framework receives from HAL and - // not yet send out because the shutter event hasn't arrived. - // It's added by process_capture_result and sent when framework - // receives the shutter event. - CameraMetadata pendingMetadata; - - // The metadata of the partial results that framework receives from HAL so far - // and has sent out. - CameraMetadata collectedPartialResult; - - // Buffers are added by process_capture_result when output buffers - // return from HAL but framework has not yet received the shutter - // event. They will be returned to the streams when framework receives - // the shutter event. - Vector pendingOutputBuffers; - - // Whether this inflight request's shutter and result callback are to be - // called. The policy is that if the request is the last one in the constrained - // high speed recording request list, this flag will be true. If the request list - // is not for constrained high speed recording, this flag will also be true. - bool hasCallback; - - // Maximum expected frame duration for this request. - // For manual captures, equal to the max of requested exposure time and frame duration - // For auto-exposure modes, equal to 1/(lower end of target FPS range) - nsecs_t maxExpectedDuration; - - // Whether the result metadata for this request is to be skipped. The - // result metadata should be skipped in the case of - // REQUEST/RESULT error. - bool skipResultMetadata; - - // The physical camera ids being requested. - std::set physicalCameraIds; - - // Map of physicalCameraId <-> Metadata - std::vector physicalMetadatas; - - // Indicates a still capture request. - bool stillCapture; - - // Indicates a ZSL capture request - bool zslCapture; - - // Requested camera ids (both logical and physical) with zoomRatio != 1.0f - std::set cameraIdsWithZoom; - - // What shared surfaces an output should go to - SurfaceMap outputSurfaces; - - // Default constructor needed by KeyedVector - InFlightRequest() : - shutterTimestamp(0), - sensorTimestamp(0), - requestStatus(OK), - haveResultMetadata(false), - numBuffersLeft(0), - hasInputBuffer(false), - hasCallback(true), - maxExpectedDuration(kDefaultExpectedDuration), - skipResultMetadata(false), - stillCapture(false), - zslCapture(false) { - } - - InFlightRequest(int numBuffers, CaptureResultExtras extras, bool hasInput, - bool hasAppCallback, nsecs_t maxDuration, - const std::set& physicalCameraIdSet, bool isStillCapture, - bool isZslCapture, const std::set& idsWithZoom, - const SurfaceMap& outSurfaces = SurfaceMap{}) : - shutterTimestamp(0), - sensorTimestamp(0), - requestStatus(OK), - haveResultMetadata(false), - numBuffersLeft(numBuffers), - resultExtras(extras), - hasInputBuffer(hasInput), - hasCallback(hasAppCallback), - maxExpectedDuration(maxDuration), - skipResultMetadata(false), - physicalCameraIds(physicalCameraIdSet), - stillCapture(isStillCapture), - zslCapture(isZslCapture), - cameraIdsWithZoom(idsWithZoom), - outputSurfaces(outSurfaces) { - } - }; - - // Map from frame number to the in-flight request state - typedef KeyedVector InFlightMap; - - - Mutex mInFlightLock; // Protects mInFlightMap and - // mExpectedInflightDuration - InFlightMap mInFlightMap; - nsecs_t mExpectedInflightDuration = 0; - int mInFlightStatusId; + int mInFlightStatusId; // const after initialize status_t registerInFlight(uint32_t frameNumber, int32_t numBuffers, CaptureResultExtras resultExtras, bool hasInput, @@ -1211,7 +1066,7 @@ class Camera3Device : */ // Lock for output side of device - Mutex mOutputLock; + std::mutex mOutputLock; /**** Scope for mOutputLock ****/ // the minimal frame number of the next non-reprocess result @@ -1226,61 +1081,18 @@ class Camera3Device : uint32_t mNextReprocessShutterFrameNumber; // the minimal frame number of the next ZSL still capture shutter uint32_t mNextZslStillShutterFrameNumber; - List mResultQueue; - Condition mResultSignal; - wp mListener; + List mResultQueue; + std::condition_variable mResultSignal; + wp mListener; /**** End scope for mOutputLock ****/ - /** - * Callback functions from HAL device - */ - void processCaptureResult(const camera3_capture_result *result); - - void notify(const camera3_notify_msg *msg); - - // Specific notify handlers - void notifyError(const camera3_error_msg_t &msg, - sp listener); - void notifyShutter(const camera3_shutter_msg_t &msg, - sp listener); - - // helper function to return the output buffers to the streams. - void returnOutputBuffers(const camera3_stream_buffer_t *outputBuffers, - size_t numBuffers, nsecs_t timestamp, bool timestampIncreasing = true, - // The following arguments are only meant for surface sharing use case - const SurfaceMap& outputSurfaces = SurfaceMap{}, - // Used to send buffer error callback when failing to return buffer - const CaptureResultExtras &resultExtras = CaptureResultExtras{}); - - // Send a partial capture result. - void sendPartialCaptureResult(const camera_metadata_t * partialResult, - const CaptureResultExtras &resultExtras, uint32_t frameNumber); - - // Send a total capture result given the pending metadata and result extras, - // partial results, and the frame number to the result queue. - void sendCaptureResult(CameraMetadata &pendingMetadata, - CaptureResultExtras &resultExtras, - CameraMetadata &collectedPartialResult, uint32_t frameNumber, - bool reprocess, bool zslStillCapture, - const std::set& cameraIdsWithZoom, - const std::vector& physicalMetadatas); - - bool isLastFullResult(const InFlightRequest& inFlightRequest); - - // Insert the result to the result queue after updating frame number and overriding AE - // trigger cancel. - // mOutputLock must be held when calling this function. - void insertResultLocked(CaptureResult *result, uint32_t frameNumber); - /**** Scope for mInFlightLock ****/ // Remove the in-flight map entry of the given index from mInFlightMap. // It must only be called with mInFlightLock held. void removeInFlightMapEntryLocked(int idx); - // Remove the in-flight request of the given index from mInFlightMap - // if it's no longer needed. It must only be called with mInFlightLock held. - void removeInFlightRequestIfReadyLocked(int idx); + // Remove all in-flight requests and return all buffers. // This is used after HAL interface is closed to cleanup any request/buffers // not returned by HAL. @@ -1378,6 +1190,10 @@ class Camera3Device : void onSubmittingRequest(); void onRequestThreadPaused(); + // Events triggered by successful switchToOffline call + // Return true is there is no ongoing requestBuffer call. + bool onSwitchToOfflineSuccess(); + private: void notifyTrackerLocked(bool active); @@ -1391,6 +1207,7 @@ class Camera3Device : bool mRequestThreadPaused = true; bool mInflightMapEmpty = true; bool mRequestBufferOngoing = false; + bool mSwitchedToOffline = false; wp mStatusTracker; int mRequestBufferStatusId; @@ -1398,7 +1215,6 @@ class Camera3Device : // Fix up result metadata for monochrome camera. bool mNeedFixupMonochromeTags; - status_t fixupMonochromeTags(const CameraMetadata& deviceInfo, CameraMetadata& resultMetadata); // Whether HAL supports offline processing capability. bool mSupportOfflineProcessing = false; diff --git a/services/camera/libcameraservice/device3/Camera3OfflineSession.cpp b/services/camera/libcameraservice/device3/Camera3OfflineSession.cpp index dc5cbb350c..0f056321e3 100644 --- a/services/camera/libcameraservice/device3/Camera3OfflineSession.cpp +++ b/services/camera/libcameraservice/device3/Camera3OfflineSession.cpp @@ -29,90 +29,435 @@ #include +#include + #include "device3/Camera3OfflineSession.h" #include "device3/Camera3OutputStream.h" #include "device3/Camera3InputStream.h" #include "device3/Camera3SharedOutputStream.h" +#include "utils/CameraTraces.h" using namespace android::camera3; using namespace android::hardware::camera; namespace android { -Camera3OfflineSession::Camera3OfflineSession(const String8 &id): - mId(id) -{ +Camera3OfflineSession::Camera3OfflineSession(const String8 &id, + const sp& inputStream, + const camera3::StreamSet& offlineStreamSet, + camera3::BufferRecords&& bufferRecords, + const camera3::InFlightRequestMap& offlineReqs, + const Camera3OfflineStates& offlineStates, + sp offlineSession) : + mId(id), + mInputStream(inputStream), + mOutputStreams(offlineStreamSet), + mBufferRecords(std::move(bufferRecords)), + mOfflineReqs(offlineReqs), + mSession(offlineSession), + mTagMonitor(offlineStates.mTagMonitor), + mVendorTagId(offlineStates.mVendorTagId), + mUseHalBufManager(offlineStates.mUseHalBufManager), + mNeedFixupMonochromeTags(offlineStates.mNeedFixupMonochromeTags), + mUsePartialResult(offlineStates.mUsePartialResult), + mNumPartialResults(offlineStates.mNumPartialResults), + mNextResultFrameNumber(offlineStates.mNextResultFrameNumber), + mNextReprocessResultFrameNumber(offlineStates.mNextReprocessResultFrameNumber), + mNextZslStillResultFrameNumber(offlineStates.mNextZslStillResultFrameNumber), + mNextShutterFrameNumber(offlineStates.mNextShutterFrameNumber), + mNextReprocessShutterFrameNumber(offlineStates.mNextReprocessShutterFrameNumber), + mNextZslStillShutterFrameNumber(offlineStates.mNextZslStillShutterFrameNumber), + mDeviceInfo(offlineStates.mDeviceInfo), + mPhysicalDeviceInfoMap(offlineStates.mPhysicalDeviceInfoMap), + mDistortionMappers(offlineStates.mDistortionMappers), + mZoomRatioMappers(offlineStates.mZoomRatioMappers), + mStatus(STATUS_UNINITIALIZED) { ATRACE_CALL(); ALOGV("%s: Created offline session for camera %s", __FUNCTION__, mId.string()); } -Camera3OfflineSession::~Camera3OfflineSession() -{ +Camera3OfflineSession::~Camera3OfflineSession() { ATRACE_CALL(); ALOGV("%s: Tearing down offline session for camera id %s", __FUNCTION__, mId.string()); + disconnectImpl(); } const String8& Camera3OfflineSession::getId() const { return mId; } -status_t Camera3OfflineSession::initialize( - sp /*hidlSession*/) { +status_t Camera3OfflineSession::initialize(wp listener) { ATRACE_CALL(); + + if (mSession == nullptr) { + ALOGE("%s: HIDL session is null!", __FUNCTION__); + return DEAD_OBJECT; + } + + { + std::lock_guard lock(mLock); + + mListener = listener; + + // setup result FMQ + std::unique_ptr& resQueue = mResultMetadataQueue; + auto resultQueueRet = mSession->getCaptureResultMetadataQueue( + [&resQueue](const auto& descriptor) { + resQueue = std::make_unique(descriptor); + if (!resQueue->isValid() || resQueue->availableToWrite() <= 0) { + ALOGE("HAL returns empty result metadata fmq, not use it"); + resQueue = nullptr; + // Don't use resQueue onwards. + } + }); + if (!resultQueueRet.isOk()) { + ALOGE("Transaction error when getting result metadata queue from camera session: %s", + resultQueueRet.description().c_str()); + return DEAD_OBJECT; + } + mStatus = STATUS_ACTIVE; + } + + mSession->setCallback(this); + return OK; } status_t Camera3OfflineSession::dump(int /*fd*/) { ATRACE_CALL(); + std::lock_guard il(mInterfaceLock); return OK; } -status_t Camera3OfflineSession::abort() { +status_t Camera3OfflineSession::disconnect() { ATRACE_CALL(); - return OK; + return disconnectImpl(); } -status_t Camera3OfflineSession::disconnect() { +status_t Camera3OfflineSession::disconnectImpl() { ATRACE_CALL(); + std::lock_guard il(mInterfaceLock); + + sp listener; + { + std::lock_guard lock(mLock); + if (mStatus == STATUS_CLOSED) { + return OK; // don't close twice + } else if (mStatus == STATUS_ERROR) { + ALOGE("%s: offline session %s shutting down in error state", + __FUNCTION__, mId.string()); + } + listener = mListener.promote(); + } + + ALOGV("%s: E", __FUNCTION__); + + { + std::lock_guard lock(mRequestBufferInterfaceLock); + mAllowRequestBuffer = false; + } + + std::vector> streams; + streams.reserve(mOutputStreams.size() + (mInputStream != nullptr ? 1 : 0)); + for (size_t i = 0; i < mOutputStreams.size(); i++) { + streams.push_back(mOutputStreams[i]); + } + if (mInputStream != nullptr) { + streams.push_back(mInputStream); + } + + mSession->close(); + + FlushInflightReqStates states { + mId, mOfflineReqsLock, mOfflineReqs, mUseHalBufManager, + listener, *this, mBufferRecords, *this}; + + camera3::flushInflightRequests(states); + + { + std::lock_guard lock(mLock); + mSession.clear(); + mOutputStreams.clear(); + mInputStream.clear(); + mStatus = STATUS_CLOSED; + } + + for (auto& weakStream : streams) { + sp stream = weakStream.promote(); + if (stream != nullptr) { + ALOGE("%s: Stream %d leaked! strong reference (%d)!", + __FUNCTION__, stream->getId(), stream->getStrongCount() - 1); + } + } + + ALOGV("%s: X", __FUNCTION__); return OK; } -status_t Camera3OfflineSession::waitForNextFrame(nsecs_t /*timeout*/) { +status_t Camera3OfflineSession::waitForNextFrame(nsecs_t timeout) { ATRACE_CALL(); + std::unique_lock lk(mOutputLock); + + while (mResultQueue.empty()) { + auto st = mResultSignal.wait_for(lk, std::chrono::nanoseconds(timeout)); + if (st == std::cv_status::timeout) { + return TIMED_OUT; + } + } return OK; } -status_t Camera3OfflineSession::getNextResult(CaptureResult* /*frame*/) { +status_t Camera3OfflineSession::getNextResult(CaptureResult* frame) { ATRACE_CALL(); + std::lock_guard l(mOutputLock); + + if (mResultQueue.empty()) { + return NOT_ENOUGH_DATA; + } + + if (frame == nullptr) { + ALOGE("%s: argument cannot be NULL", __FUNCTION__); + return BAD_VALUE; + } + + CaptureResult &result = *(mResultQueue.begin()); + frame->mResultExtras = result.mResultExtras; + frame->mMetadata.acquire(result.mMetadata); + frame->mPhysicalMetadatas = std::move(result.mPhysicalMetadatas); + mResultQueue.erase(mResultQueue.begin()); + return OK; } hardware::Return Camera3OfflineSession::processCaptureResult_3_4( const hardware::hidl_vec< - hardware::camera::device::V3_4::CaptureResult>& /*results*/) { + hardware::camera::device::V3_4::CaptureResult>& results) { + sp listener; + { + std::lock_guard lock(mLock); + if (mStatus != STATUS_ACTIVE) { + ALOGE("%s called in wrong state %d", __FUNCTION__, mStatus); + return hardware::Void(); + } + listener = mListener.promote(); + } + + CaptureOutputStates states { + mId, + mOfflineReqsLock, mOfflineReqs, + mOutputLock, mResultQueue, mResultSignal, + mNextShutterFrameNumber, + mNextReprocessShutterFrameNumber, mNextZslStillShutterFrameNumber, + mNextResultFrameNumber, + mNextReprocessResultFrameNumber, mNextZslStillResultFrameNumber, + mUseHalBufManager, mUsePartialResult, mNeedFixupMonochromeTags, + mNumPartialResults, mVendorTagId, mDeviceInfo, mPhysicalDeviceInfoMap, + mResultMetadataQueue, mDistortionMappers, mZoomRatioMappers, mTagMonitor, + mInputStream, mOutputStreams, listener, *this, *this, mBufferRecords + }; + + std::lock_guard lock(mProcessCaptureResultLock); + for (const auto& result : results) { + processOneCaptureResultLocked(states, result.v3_2, result.physicalCameraMetadata); + } return hardware::Void(); } hardware::Return Camera3OfflineSession::processCaptureResult( const hardware::hidl_vec< - hardware::camera::device::V3_2::CaptureResult>& /*results*/) { + hardware::camera::device::V3_2::CaptureResult>& results) { + // TODO: changed impl to call into processCaptureResult_3_4 instead? + // might need to figure how to reduce copy though. + sp listener; + { + std::lock_guard lock(mLock); + if (mStatus != STATUS_ACTIVE) { + ALOGE("%s called in wrong state %d", __FUNCTION__, mStatus); + return hardware::Void(); + } + listener = mListener.promote(); + } + + hardware::hidl_vec noPhysMetadata; + + CaptureOutputStates states { + mId, + mOfflineReqsLock, mOfflineReqs, + mOutputLock, mResultQueue, mResultSignal, + mNextShutterFrameNumber, + mNextReprocessShutterFrameNumber, mNextZslStillShutterFrameNumber, + mNextResultFrameNumber, + mNextReprocessResultFrameNumber, mNextZslStillResultFrameNumber, + mUseHalBufManager, mUsePartialResult, mNeedFixupMonochromeTags, + mNumPartialResults, mVendorTagId, mDeviceInfo, mPhysicalDeviceInfoMap, + mResultMetadataQueue, mDistortionMappers, mZoomRatioMappers, mTagMonitor, + mInputStream, mOutputStreams, listener, *this, *this, mBufferRecords + }; + + std::lock_guard lock(mProcessCaptureResultLock); + for (const auto& result : results) { + processOneCaptureResultLocked(states, result, noPhysMetadata); + } return hardware::Void(); } hardware::Return Camera3OfflineSession::notify( - const hardware::hidl_vec& /*msgs*/) { + const hardware::hidl_vec& msgs) { + sp listener; + { + std::lock_guard lock(mLock); + if (mStatus != STATUS_ACTIVE) { + ALOGE("%s called in wrong state %d", __FUNCTION__, mStatus); + return hardware::Void(); + } + listener = mListener.promote(); + } + + CaptureOutputStates states { + mId, + mOfflineReqsLock, mOfflineReqs, + mOutputLock, mResultQueue, mResultSignal, + mNextShutterFrameNumber, + mNextReprocessShutterFrameNumber, mNextZslStillShutterFrameNumber, + mNextResultFrameNumber, + mNextReprocessResultFrameNumber, mNextZslStillResultFrameNumber, + mUseHalBufManager, mUsePartialResult, mNeedFixupMonochromeTags, + mNumPartialResults, mVendorTagId, mDeviceInfo, mPhysicalDeviceInfoMap, + mResultMetadataQueue, mDistortionMappers, mZoomRatioMappers, mTagMonitor, + mInputStream, mOutputStreams, listener, *this, *this, mBufferRecords + }; + for (const auto& msg : msgs) { + camera3::notify(states, msg); + } return hardware::Void(); } hardware::Return Camera3OfflineSession::requestStreamBuffers( - const hardware::hidl_vec& /*bufReqs*/, - requestStreamBuffers_cb /*_hidl_cb*/) { + const hardware::hidl_vec& bufReqs, + requestStreamBuffers_cb _hidl_cb) { + { + std::lock_guard lock(mLock); + if (mStatus != STATUS_ACTIVE) { + ALOGE("%s called in wrong state %d", __FUNCTION__, mStatus); + return hardware::Void(); + } + } + + RequestBufferStates states { + mId, mRequestBufferInterfaceLock, mUseHalBufManager, mOutputStreams, + *this, mBufferRecords, *this}; + camera3::requestStreamBuffers(states, bufReqs, _hidl_cb); return hardware::Void(); } hardware::Return Camera3OfflineSession::returnStreamBuffers( - const hardware::hidl_vec& /*buffers*/) { + const hardware::hidl_vec& buffers) { + { + std::lock_guard lock(mLock); + if (mStatus != STATUS_ACTIVE) { + ALOGE("%s called in wrong state %d", __FUNCTION__, mStatus); + return hardware::Void(); + } + } + + ReturnBufferStates states { + mId, mUseHalBufManager, mOutputStreams, mBufferRecords}; + camera3::returnStreamBuffers(states, buffers); return hardware::Void(); } +void Camera3OfflineSession::setErrorState(const char *fmt, ...) { + ATRACE_CALL(); + std::lock_guard lock(mLock); + va_list args; + va_start(args, fmt); + + setErrorStateLockedV(fmt, args); + + va_end(args); + + //FIXME: automatically disconnect here? +} + +void Camera3OfflineSession::setErrorStateLocked(const char *fmt, ...) { + va_list args; + va_start(args, fmt); + + setErrorStateLockedV(fmt, args); + + va_end(args); +} + +void Camera3OfflineSession::setErrorStateLockedV(const char *fmt, va_list args) { + // Print out all error messages to log + String8 errorCause = String8::formatV(fmt, args); + ALOGE("Camera %s: %s", mId.string(), errorCause.string()); + + // But only do error state transition steps for the first error + if (mStatus == STATUS_ERROR || mStatus == STATUS_UNINITIALIZED) return; + + mErrorCause = errorCause; + + mStatus = STATUS_ERROR; + + // Notify upstream about a device error + sp listener = mListener.promote(); + if (listener != NULL) { + listener->notifyError(hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_DEVICE, + CaptureResultExtras()); + } + + // Save stack trace. View by dumping it later. + CameraTraces::saveTrace(); +} + +void Camera3OfflineSession::onInflightEntryRemovedLocked(nsecs_t /*duration*/) { + if (mOfflineReqs.size() == 0) { + std::lock_guard lock(mRequestBufferInterfaceLock); + mAllowRequestBuffer = false; + } +} + +void Camera3OfflineSession::checkInflightMapLengthLocked() { + // Intentional empty impl. +} + +void Camera3OfflineSession::onInflightMapFlushedLocked() { + // Intentional empty impl. +} + +bool Camera3OfflineSession::startRequestBuffer() { + return mAllowRequestBuffer; +} + +void Camera3OfflineSession::endRequestBuffer() { + // Intentional empty impl. +} + +nsecs_t Camera3OfflineSession::getWaitDuration() { + const nsecs_t kBaseGetBufferWait = 3000000000; // 3 sec. + return kBaseGetBufferWait; +} + +void Camera3OfflineSession::getInflightBufferKeys(std::vector>* out) { + mBufferRecords.getInflightBufferKeys(out); +} + +void Camera3OfflineSession::getInflightRequestBufferKeys(std::vector* out) { + mBufferRecords.getInflightRequestBufferKeys(out); +} + +std::vector> Camera3OfflineSession::getAllStreams() { + std::vector> ret; + bool hasInputStream = mInputStream != nullptr; + ret.reserve(mOutputStreams.size() + ((hasInputStream) ? 1 : 0)); + if (hasInputStream) { + ret.push_back(mInputStream); + } + for (size_t i = 0; i < mOutputStreams.size(); i++) { + ret.push_back(mOutputStreams[i]); + } + return ret; +} + }; // namespace android diff --git a/services/camera/libcameraservice/device3/Camera3OfflineSession.h b/services/camera/libcameraservice/device3/Camera3OfflineSession.h index 30e8c4f4b7..b6b8a7d501 100644 --- a/services/camera/libcameraservice/device3/Camera3OfflineSession.h +++ b/services/camera/libcameraservice/device3/Camera3OfflineSession.h @@ -17,16 +17,23 @@ #ifndef ANDROID_SERVERS_CAMERA3OFFLINESESSION_H #define ANDROID_SERVERS_CAMERA3OFFLINESESSION_H +#include +#include + #include #include #include + #include #include "common/CameraOfflineSessionBase.h" #include "device3/Camera3BufferManager.h" #include "device3/DistortionMapper.h" +#include "device3/InFlightRequest.h" +#include "device3/Camera3OutputUtils.h" +#include "device3/ZoomRatioMapper.h" #include "utils/TagMonitor.h" #include "utils/LatencyHistogram.h" #include @@ -41,26 +48,90 @@ class Camera3StreamInterface; } // namespace camera3 + +// An immutable struct containing general states that will be copied from Camera3Device to +// Camera3OfflineSession +struct Camera3OfflineStates { + Camera3OfflineStates( + const TagMonitor& tagMonitor, const metadata_vendor_id_t vendorTagId, + const bool useHalBufManager, const bool needFixupMonochromeTags, + const bool usePartialResult, const uint32_t numPartialResults, + const uint32_t nextResultFN, const uint32_t nextReprocResultFN, + const uint32_t nextZslResultFN, const uint32_t nextShutterFN, + const uint32_t nextReprocShutterFN, const uint32_t nextZslShutterFN, + const CameraMetadata& deviceInfo, + const std::unordered_map& physicalDeviceInfoMap, + const std::unordered_map& distortionMappers, + const std::unordered_map& zoomRatioMappers) : + mTagMonitor(tagMonitor), mVendorTagId(vendorTagId), + mUseHalBufManager(useHalBufManager), mNeedFixupMonochromeTags(needFixupMonochromeTags), + mUsePartialResult(usePartialResult), mNumPartialResults(numPartialResults), + mNextResultFrameNumber(nextResultFN), + mNextReprocessResultFrameNumber(nextReprocResultFN), + mNextZslStillResultFrameNumber(nextZslResultFN), + mNextShutterFrameNumber(nextShutterFN), + mNextReprocessShutterFrameNumber(nextReprocShutterFN), + mNextZslStillShutterFrameNumber(nextZslShutterFN), + mDeviceInfo(deviceInfo), + mPhysicalDeviceInfoMap(physicalDeviceInfoMap), + mDistortionMappers(distortionMappers), + mZoomRatioMappers(zoomRatioMappers) {} + + const TagMonitor& mTagMonitor; + const metadata_vendor_id_t mVendorTagId; + + const bool mUseHalBufManager; + const bool mNeedFixupMonochromeTags; + + const bool mUsePartialResult; + const uint32_t mNumPartialResults; + + // the minimal frame number of the next non-reprocess result + const uint32_t mNextResultFrameNumber; + // the minimal frame number of the next reprocess result + const uint32_t mNextReprocessResultFrameNumber; + // the minimal frame number of the next ZSL still capture result + const uint32_t mNextZslStillResultFrameNumber; + // the minimal frame number of the next non-reprocess shutter + const uint32_t mNextShutterFrameNumber; + // the minimal frame number of the next reprocess shutter + const uint32_t mNextReprocessShutterFrameNumber; + // the minimal frame number of the next ZSL still capture shutter + const uint32_t mNextZslStillShutterFrameNumber; + + const CameraMetadata& mDeviceInfo; + + const std::unordered_map& mPhysicalDeviceInfoMap; + + const std::unordered_map& mDistortionMappers; + + const std::unordered_map& mZoomRatioMappers; +}; + /** * Camera3OfflineSession for offline session defined in HIDL ICameraOfflineSession@3.6 or higher */ class Camera3OfflineSession : public CameraOfflineSessionBase, - virtual public hardware::camera::device::V3_5::ICameraDeviceCallback { - + virtual public hardware::camera::device::V3_5::ICameraDeviceCallback, + public camera3::SetErrorInterface, + public camera3::InflightRequestUpdateInterface, + public camera3::RequestBufferInterface, + public camera3::FlushBufferInterface { public: - // initialize by Camera3Device. Camera3Device must send all info in separate argument. - // monitored tags - // mUseHalBufManager - // mUsePartialResult - // mNumPartialResults - explicit Camera3OfflineSession(const String8& id); + // initialize by Camera3Device. + explicit Camera3OfflineSession(const String8& id, + const sp& inputStream, + const camera3::StreamSet& offlineStreamSet, + camera3::BufferRecords&& bufferRecords, + const camera3::InFlightRequestMap& offlineReqs, + const Camera3OfflineStates& offlineStates, + sp offlineSession); virtual ~Camera3OfflineSession(); - status_t initialize( - sp hidlSession); + virtual status_t initialize(wp listener) override; /** * CameraOfflineSessionBase interface @@ -71,8 +142,6 @@ class Camera3OfflineSession : status_t dump(int fd) override; - status_t abort() override; - // methods for capture result passing status_t waitForNextFrame(nsecs_t timeout) override; status_t getNextResult(CaptureResult *frame) override; @@ -115,10 +184,99 @@ class Camera3OfflineSession : */ private: - // Camera device ID const String8 mId; + sp mInputStream; + camera3::StreamSet mOutputStreams; + camera3::BufferRecords mBufferRecords; + + std::mutex mOfflineReqsLock; + camera3::InFlightRequestMap mOfflineReqs; + + sp mSession; + + TagMonitor mTagMonitor; + const metadata_vendor_id_t mVendorTagId; + + const bool mUseHalBufManager; + const bool mNeedFixupMonochromeTags; + + const bool mUsePartialResult; + const uint32_t mNumPartialResults; + + std::mutex mOutputLock; + List mResultQueue; + std::condition_variable mResultSignal; + // the minimal frame number of the next non-reprocess result + uint32_t mNextResultFrameNumber; + // the minimal frame number of the next reprocess result + uint32_t mNextReprocessResultFrameNumber; + // the minimal frame number of the next ZSL still capture result + uint32_t mNextZslStillResultFrameNumber; + // the minimal frame number of the next non-reprocess shutter + uint32_t mNextShutterFrameNumber; + // the minimal frame number of the next reprocess shutter + uint32_t mNextReprocessShutterFrameNumber; + // the minimal frame number of the next ZSL still capture shutter + uint32_t mNextZslStillShutterFrameNumber; + // End of mOutputLock scope + + const CameraMetadata mDeviceInfo; + std::unordered_map mPhysicalDeviceInfoMap; + + std::unordered_map mDistortionMappers; + + std::unordered_map mZoomRatioMappers; + + mutable std::mutex mLock; + + enum Status { + STATUS_UNINITIALIZED = 0, + STATUS_ACTIVE, + STATUS_ERROR, + STATUS_CLOSED + } mStatus; + + wp mListener; + // End of mLock protect scope + + std::mutex mProcessCaptureResultLock; + // FMQ to write result on. Must be guarded by mProcessCaptureResultLock. + std::unique_ptr mResultMetadataQueue; + + // Tracking cause of fatal errors when in STATUS_ERROR + String8 mErrorCause; + + // Lock to ensure requestStreamBuffers() callbacks are serialized + std::mutex mRequestBufferInterfaceLock; + // allow request buffer until all requests are processed or disconnectImpl is called + bool mAllowRequestBuffer = true; + + // For client methods such as disconnect/dump + std::mutex mInterfaceLock; + + // SetErrorInterface + void setErrorState(const char *fmt, ...) override; + void setErrorStateLocked(const char *fmt, ...) override; + + // InflightRequestUpdateInterface + void onInflightEntryRemovedLocked(nsecs_t duration) override; + void checkInflightMapLengthLocked() override; + void onInflightMapFlushedLocked() override; + + // RequestBufferInterface + bool startRequestBuffer() override; + void endRequestBuffer() override; + nsecs_t getWaitDuration() override; + + // FlushBufferInterface + void getInflightBufferKeys(std::vector>* out) override; + void getInflightRequestBufferKeys(std::vector* out) override; + std::vector> getAllStreams() override; + + void setErrorStateLockedV(const char *fmt, va_list args); + status_t disconnectImpl(); }; // class Camera3OfflineSession }; // namespace android diff --git a/services/camera/libcameraservice/device3/Camera3OutputInterface.h b/services/camera/libcameraservice/device3/Camera3OutputInterface.h new file mode 100644 index 0000000000..88178334e3 --- /dev/null +++ b/services/camera/libcameraservice/device3/Camera3OutputInterface.h @@ -0,0 +1,111 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ANDROID_SERVERS_CAMERA3_OUTPUT_INTERFACE_H +#define ANDROID_SERVERS_CAMERA3_OUTPUT_INTERFACE_H + +#include + +#include + +#include + +#include "device3/Camera3StreamInterface.h" + +namespace android { + +namespace camera3 { + + /** + * Interfaces used by result/notification path shared between Camera3Device and + * Camera3OfflineSession + */ + class SetErrorInterface { + public: + // Switch device into error state and send a ERROR_DEVICE notification + virtual void setErrorState(const char *fmt, ...) = 0; + // Same as setErrorState except this method assumes callers holds the main object lock + virtual void setErrorStateLocked(const char *fmt, ...) = 0; + + virtual ~SetErrorInterface() {} + }; + + // Interface used by callback path to update buffer records + class BufferRecordsInterface { + public: + // method to extract buffer's unique ID + // return pair of (newlySeenBuffer?, bufferId) + virtual std::pair getBufferId(const buffer_handle_t& buf, int streamId) = 0; + + // Find a buffer_handle_t based on frame number and stream ID + virtual status_t popInflightBuffer(int32_t frameNumber, int32_t streamId, + /*out*/ buffer_handle_t **buffer) = 0; + + // Register a bufId (streamId, buffer_handle_t) to inflight request buffer + virtual status_t pushInflightRequestBuffer( + uint64_t bufferId, buffer_handle_t* buf, int32_t streamId) = 0; + + // Find a buffer_handle_t based on bufferId + virtual status_t popInflightRequestBuffer(uint64_t bufferId, + /*out*/ buffer_handle_t** buffer, + /*optional out*/ int32_t* streamId = nullptr) = 0; + + virtual ~BufferRecordsInterface() {} + }; + + class InflightRequestUpdateInterface { + public: + // Caller must hold the lock proctecting InflightRequestMap + // duration: the maxExpectedDuration of the removed entry + virtual void onInflightEntryRemovedLocked(nsecs_t duration) = 0; + + virtual void checkInflightMapLengthLocked() = 0; + + virtual void onInflightMapFlushedLocked() = 0; + + virtual ~InflightRequestUpdateInterface() {} + }; + + class RequestBufferInterface { + public: + // Return if the state machine currently allows for requestBuffers. + // If this returns true, caller must call endRequestBuffer() later to signal end of a + // request buffer transaction. + virtual bool startRequestBuffer() = 0; + + virtual void endRequestBuffer() = 0; + + // Returns how long should implementation wait for a buffer returned + virtual nsecs_t getWaitDuration() = 0; + + virtual ~RequestBufferInterface() {} + }; + + class FlushBufferInterface { + public: + virtual void getInflightBufferKeys(std::vector>* out) = 0; + + virtual void getInflightRequestBufferKeys(std::vector* out) = 0; + + virtual std::vector> getAllStreams() = 0; + + virtual ~FlushBufferInterface() {} + }; +} // namespace camera3 + +} // namespace android + +#endif diff --git a/services/camera/libcameraservice/device3/Camera3OutputStreamInterface.cpp b/services/camera/libcameraservice/device3/Camera3OutputStreamInterface.cpp new file mode 100644 index 0000000000..64af91e510 --- /dev/null +++ b/services/camera/libcameraservice/device3/Camera3OutputStreamInterface.cpp @@ -0,0 +1,84 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "Camera3-OutStrmIntf" +#define ATRACE_TAG ATRACE_TAG_CAMERA +//#define LOG_NDEBUG 0 +//#define LOG_NNDEBUG 0 // Per-frame verbose logging + + +#include "Camera3OutputStreamInterface.h" + +namespace android { + +namespace camera3 { + +status_t StreamSet::add( + int streamId, sp stream) { + if (stream == nullptr) { + ALOGE("%s: cannot add null stream", __FUNCTION__); + return BAD_VALUE; + } + std::lock_guard lock(mLock); + return mData.add(streamId, stream); +} + +ssize_t StreamSet::remove(int streamId) { + std::lock_guard lock(mLock); + return mData.removeItem(streamId); +} + +sp StreamSet::get(int streamId) { + std::lock_guard lock(mLock); + ssize_t idx = mData.indexOfKey(streamId); + if (idx == NAME_NOT_FOUND) { + return nullptr; + } + return mData.editValueAt(idx); +} + +sp StreamSet::operator[] (size_t index) { + std::lock_guard lock(mLock); + return mData.editValueAt(index); +} + +size_t StreamSet::size() const { + std::lock_guard lock(mLock); + return mData.size(); +} + +void StreamSet::clear() { + std::lock_guard lock(mLock); + return mData.clear(); +} + +std::vector StreamSet::getStreamIds() { + std::lock_guard lock(mLock); + std::vector streamIds(mData.size()); + for (size_t i = 0; i < mData.size(); i++) { + streamIds[i] = mData.keyAt(i); + } + return streamIds; +} + +StreamSet::StreamSet(const StreamSet& other) { + std::lock_guard lock(other.mLock); + mData = other.mData; +} + +} // namespace camera3 + +} // namespace android diff --git a/services/camera/libcameraservice/device3/Camera3OutputStreamInterface.h b/services/camera/libcameraservice/device3/Camera3OutputStreamInterface.h index 2bde94957a..7f5c87a72f 100644 --- a/services/camera/libcameraservice/device3/Camera3OutputStreamInterface.h +++ b/services/camera/libcameraservice/device3/Camera3OutputStreamInterface.h @@ -97,6 +97,26 @@ class Camera3OutputStreamInterface : public virtual Camera3StreamInterface { virtual const String8& getPhysicalCameraId() const = 0; }; +// Helper class to organize a synchronized mapping of stream IDs to stream instances +class StreamSet { + public: + status_t add(int streamId, sp); + ssize_t remove(int streamId); + sp get(int streamId); + // get by (underlying) vector index + sp operator[] (size_t index); + size_t size() const; + std::vector getStreamIds(); + void clear(); + + StreamSet() {}; + StreamSet(const StreamSet& other); + + private: + mutable std::mutex mLock; + KeyedVector> mData; +}; + } // namespace camera3 } // namespace android diff --git a/services/camera/libcameraservice/device3/Camera3OutputUtils.cpp b/services/camera/libcameraservice/device3/Camera3OutputUtils.cpp new file mode 100644 index 0000000000..eb7b6668d3 --- /dev/null +++ b/services/camera/libcameraservice/device3/Camera3OutputUtils.cpp @@ -0,0 +1,1414 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "Camera3-OutputUtils" +#define ATRACE_TAG ATRACE_TAG_CAMERA +//#define LOG_NDEBUG 0 +//#define LOG_NNDEBUG 0 // Per-frame verbose logging + +#ifdef LOG_NNDEBUG +#define ALOGVV(...) ALOGV(__VA_ARGS__) +#else +#define ALOGVV(...) ((void)0) +#endif + +// Convenience macros for transitioning to the error state +#define SET_ERR(fmt, ...) states.setErrIntf.setErrorState( \ + "%s: " fmt, __FUNCTION__, \ + ##__VA_ARGS__) + +#include + +#include +#include +#include + +#include + +#include +#include +#include + +#include + +#include "device3/Camera3OutputUtils.h" + +using namespace android::camera3; +using namespace android::hardware::camera; + +namespace android { +namespace camera3 { + +status_t fixupMonochromeTags( + CaptureOutputStates& states, + const CameraMetadata& deviceInfo, + CameraMetadata& resultMetadata) { + status_t res = OK; + if (!states.needFixupMonoChrome) { + return res; + } + + // Remove tags that are not applicable to monochrome camera. + int32_t tagsToRemove[] = { + ANDROID_SENSOR_GREEN_SPLIT, + ANDROID_SENSOR_NEUTRAL_COLOR_POINT, + ANDROID_COLOR_CORRECTION_MODE, + ANDROID_COLOR_CORRECTION_TRANSFORM, + ANDROID_COLOR_CORRECTION_GAINS, + }; + for (auto tag : tagsToRemove) { + res = resultMetadata.erase(tag); + if (res != OK) { + ALOGE("%s: Failed to remove tag %d for monochrome camera", __FUNCTION__, tag); + return res; + } + } + + // ANDROID_SENSOR_DYNAMIC_BLACK_LEVEL + camera_metadata_entry blEntry = resultMetadata.find(ANDROID_SENSOR_DYNAMIC_BLACK_LEVEL); + for (size_t i = 1; i < blEntry.count; i++) { + blEntry.data.f[i] = blEntry.data.f[0]; + } + + // ANDROID_SENSOR_NOISE_PROFILE + camera_metadata_entry npEntry = resultMetadata.find(ANDROID_SENSOR_NOISE_PROFILE); + if (npEntry.count > 0 && npEntry.count % 2 == 0) { + double np[] = {npEntry.data.d[0], npEntry.data.d[1]}; + res = resultMetadata.update(ANDROID_SENSOR_NOISE_PROFILE, np, 2); + if (res != OK) { + ALOGE("%s: Failed to update SENSOR_NOISE_PROFILE: %s (%d)", + __FUNCTION__, strerror(-res), res); + return res; + } + } + + // ANDROID_STATISTICS_LENS_SHADING_MAP + camera_metadata_ro_entry lsSizeEntry = deviceInfo.find(ANDROID_LENS_INFO_SHADING_MAP_SIZE); + camera_metadata_entry lsEntry = resultMetadata.find(ANDROID_STATISTICS_LENS_SHADING_MAP); + if (lsSizeEntry.count == 2 && lsEntry.count > 0 + && (int32_t)lsEntry.count == 4 * lsSizeEntry.data.i32[0] * lsSizeEntry.data.i32[1]) { + for (int32_t i = 0; i < lsSizeEntry.data.i32[0] * lsSizeEntry.data.i32[1]; i++) { + lsEntry.data.f[4*i+1] = lsEntry.data.f[4*i]; + lsEntry.data.f[4*i+2] = lsEntry.data.f[4*i]; + lsEntry.data.f[4*i+3] = lsEntry.data.f[4*i]; + } + } + + // ANDROID_TONEMAP_CURVE_BLUE + // ANDROID_TONEMAP_CURVE_GREEN + // ANDROID_TONEMAP_CURVE_RED + camera_metadata_entry tcbEntry = resultMetadata.find(ANDROID_TONEMAP_CURVE_BLUE); + camera_metadata_entry tcgEntry = resultMetadata.find(ANDROID_TONEMAP_CURVE_GREEN); + camera_metadata_entry tcrEntry = resultMetadata.find(ANDROID_TONEMAP_CURVE_RED); + if (tcbEntry.count > 0 + && tcbEntry.count == tcgEntry.count + && tcbEntry.count == tcrEntry.count) { + for (size_t i = 0; i < tcbEntry.count; i++) { + tcbEntry.data.f[i] = tcrEntry.data.f[i]; + tcgEntry.data.f[i] = tcrEntry.data.f[i]; + } + } + + return res; +} + +void insertResultLocked(CaptureOutputStates& states, CaptureResult *result, uint32_t frameNumber) { + if (result == nullptr) return; + + camera_metadata_t *meta = const_cast( + result->mMetadata.getAndLock()); + set_camera_metadata_vendor_id(meta, states.vendorTagId); + result->mMetadata.unlock(meta); + + if (result->mMetadata.update(ANDROID_REQUEST_FRAME_COUNT, + (int32_t*)&frameNumber, 1) != OK) { + SET_ERR("Failed to set frame number %d in metadata", frameNumber); + return; + } + + if (result->mMetadata.update(ANDROID_REQUEST_ID, &result->mResultExtras.requestId, 1) != OK) { + SET_ERR("Failed to set request ID in metadata for frame %d", frameNumber); + return; + } + + // Update vendor tag id for physical metadata + for (auto& physicalMetadata : result->mPhysicalMetadatas) { + camera_metadata_t *pmeta = const_cast( + physicalMetadata.mPhysicalCameraMetadata.getAndLock()); + set_camera_metadata_vendor_id(pmeta, states.vendorTagId); + physicalMetadata.mPhysicalCameraMetadata.unlock(pmeta); + } + + // Valid result, insert into queue + List::iterator queuedResult = + states.resultQueue.insert(states.resultQueue.end(), CaptureResult(*result)); + ALOGVV("%s: result requestId = %" PRId32 ", frameNumber = %" PRId64 + ", burstId = %" PRId32, __FUNCTION__, + queuedResult->mResultExtras.requestId, + queuedResult->mResultExtras.frameNumber, + queuedResult->mResultExtras.burstId); + + states.resultSignal.notify_one(); +} + + +void sendPartialCaptureResult(CaptureOutputStates& states, + const camera_metadata_t * partialResult, + const CaptureResultExtras &resultExtras, uint32_t frameNumber) { + ATRACE_CALL(); + std::lock_guard l(states.outputLock); + + CaptureResult captureResult; + captureResult.mResultExtras = resultExtras; + captureResult.mMetadata = partialResult; + + // Fix up result metadata for monochrome camera. + status_t res = fixupMonochromeTags(states, states.deviceInfo, captureResult.mMetadata); + if (res != OK) { + SET_ERR("Failed to override result metadata: %s (%d)", strerror(-res), res); + return; + } + + insertResultLocked(states, &captureResult, frameNumber); +} + +void sendCaptureResult( + CaptureOutputStates& states, + CameraMetadata &pendingMetadata, + CaptureResultExtras &resultExtras, + CameraMetadata &collectedPartialResult, + uint32_t frameNumber, + bool reprocess, bool zslStillCapture, + const std::set& cameraIdsWithZoom, + const std::vector& physicalMetadatas) { + ATRACE_CALL(); + if (pendingMetadata.isEmpty()) + return; + + std::lock_guard l(states.outputLock); + + // TODO: need to track errors for tighter bounds on expected frame number + if (reprocess) { + if (frameNumber < states.nextReprocResultFrameNum) { + SET_ERR("Out-of-order reprocess capture result metadata submitted! " + "(got frame number %d, expecting %d)", + frameNumber, states.nextReprocResultFrameNum); + return; + } + states.nextReprocResultFrameNum = frameNumber + 1; + } else if (zslStillCapture) { + if (frameNumber < states.nextZslResultFrameNum) { + SET_ERR("Out-of-order ZSL still capture result metadata submitted! " + "(got frame number %d, expecting %d)", + frameNumber, states.nextZslResultFrameNum); + return; + } + states.nextZslResultFrameNum = frameNumber + 1; + } else { + if (frameNumber < states.nextResultFrameNum) { + SET_ERR("Out-of-order capture result metadata submitted! " + "(got frame number %d, expecting %d)", + frameNumber, states.nextResultFrameNum); + return; + } + states.nextResultFrameNum = frameNumber + 1; + } + + CaptureResult captureResult; + captureResult.mResultExtras = resultExtras; + captureResult.mMetadata = pendingMetadata; + captureResult.mPhysicalMetadatas = physicalMetadatas; + + // Append any previous partials to form a complete result + if (states.usePartialResult && !collectedPartialResult.isEmpty()) { + captureResult.mMetadata.append(collectedPartialResult); + } + + captureResult.mMetadata.sort(); + + // Check that there's a timestamp in the result metadata + camera_metadata_entry timestamp = captureResult.mMetadata.find(ANDROID_SENSOR_TIMESTAMP); + if (timestamp.count == 0) { + SET_ERR("No timestamp provided by HAL for frame %d!", + frameNumber); + return; + } + for (auto& physicalMetadata : captureResult.mPhysicalMetadatas) { + camera_metadata_entry timestamp = + physicalMetadata.mPhysicalCameraMetadata.find(ANDROID_SENSOR_TIMESTAMP); + if (timestamp.count == 0) { + SET_ERR("No timestamp provided by HAL for physical camera %s frame %d!", + String8(physicalMetadata.mPhysicalCameraId).c_str(), frameNumber); + return; + } + } + + // Fix up some result metadata to account for HAL-level distortion correction + status_t res = + states.distortionMappers[states.cameraId.c_str()].correctCaptureResult( + &captureResult.mMetadata); + if (res != OK) { + SET_ERR("Unable to correct capture result metadata for frame %d: %s (%d)", + frameNumber, strerror(-res), res); + return; + } + + // Fix up result metadata to account for zoom ratio availabilities between + // HAL and app. + bool zoomRatioIs1 = cameraIdsWithZoom.find(states.cameraId.c_str()) == cameraIdsWithZoom.end(); + res = states.zoomRatioMappers[states.cameraId.c_str()].updateCaptureResult( + &captureResult.mMetadata, zoomRatioIs1); + if (res != OK) { + SET_ERR("Failed to update capture result zoom ratio metadata for frame %d: %s (%d)", + frameNumber, strerror(-res), res); + return; + } + + for (auto& physicalMetadata : captureResult.mPhysicalMetadatas) { + String8 cameraId8(physicalMetadata.mPhysicalCameraId); + if (states.distortionMappers.find(cameraId8.c_str()) != states.distortionMappers.end()) { + res = states.distortionMappers[cameraId8.c_str()].correctCaptureResult( + &physicalMetadata.mPhysicalCameraMetadata); + if (res != OK) { + SET_ERR("Unable to correct physical capture result metadata for frame %d: %s (%d)", + frameNumber, strerror(-res), res); + return; + } + } + + zoomRatioIs1 = cameraIdsWithZoom.find(cameraId8.c_str()) == cameraIdsWithZoom.end(); + res = states.zoomRatioMappers[cameraId8.c_str()].updateCaptureResult( + &physicalMetadata.mPhysicalCameraMetadata, zoomRatioIs1); + if (res != OK) { + SET_ERR("Failed to update camera %s's physical zoom ratio metadata for " + "frame %d: %s(%d)", cameraId8.c_str(), frameNumber, strerror(-res), res); + return; + } + } + + // Fix up result metadata for monochrome camera. + res = fixupMonochromeTags(states, states.deviceInfo, captureResult.mMetadata); + if (res != OK) { + SET_ERR("Failed to override result metadata: %s (%d)", strerror(-res), res); + return; + } + for (auto& physicalMetadata : captureResult.mPhysicalMetadatas) { + String8 cameraId8(physicalMetadata.mPhysicalCameraId); + res = fixupMonochromeTags(states, + states.physicalDeviceInfoMap.at(cameraId8.c_str()), + physicalMetadata.mPhysicalCameraMetadata); + if (res != OK) { + SET_ERR("Failed to override result metadata: %s (%d)", strerror(-res), res); + return; + } + } + + std::unordered_map monitoredPhysicalMetadata; + for (auto& m : physicalMetadatas) { + monitoredPhysicalMetadata.emplace(String8(m.mPhysicalCameraId).string(), + CameraMetadata(m.mPhysicalCameraMetadata)); + } + states.tagMonitor.monitorMetadata(TagMonitor::RESULT, + frameNumber, timestamp.data.i64[0], captureResult.mMetadata, + monitoredPhysicalMetadata); + + insertResultLocked(states, &captureResult, frameNumber); +} + +// Reading one camera metadata from result argument via fmq or from the result +// Assuming the fmq is protected by a lock already +status_t readOneCameraMetadataLocked( + std::unique_ptr& fmq, + uint64_t fmqResultSize, + hardware::camera::device::V3_2::CameraMetadata& resultMetadata, + const hardware::camera::device::V3_2::CameraMetadata& result) { + if (fmqResultSize > 0) { + resultMetadata.resize(fmqResultSize); + if (fmq == nullptr) { + return NO_MEMORY; // logged in initialize() + } + if (!fmq->read(resultMetadata.data(), fmqResultSize)) { + ALOGE("%s: Cannot read camera metadata from fmq, size = %" PRIu64, + __FUNCTION__, fmqResultSize); + return INVALID_OPERATION; + } + } else { + resultMetadata.setToExternal(const_cast(result.data()), + result.size()); + } + + if (resultMetadata.size() != 0) { + status_t res; + const camera_metadata_t* metadata = + reinterpret_cast(resultMetadata.data()); + size_t expected_metadata_size = resultMetadata.size(); + if ((res = validate_camera_metadata_structure(metadata, &expected_metadata_size)) != OK) { + ALOGE("%s: Invalid camera metadata received by camera service from HAL: %s (%d)", + __FUNCTION__, strerror(-res), res); + return INVALID_OPERATION; + } + } + + return OK; +} + +void removeInFlightMapEntryLocked(CaptureOutputStates& states, int idx) { + ATRACE_CALL(); + InFlightRequestMap& inflightMap = states.inflightMap; + nsecs_t duration = inflightMap.valueAt(idx).maxExpectedDuration; + inflightMap.removeItemsAt(idx, 1); + + states.inflightIntf.onInflightEntryRemovedLocked(duration); +} + +void removeInFlightRequestIfReadyLocked(CaptureOutputStates& states, int idx) { + InFlightRequestMap& inflightMap = states.inflightMap; + const InFlightRequest &request = inflightMap.valueAt(idx); + const uint32_t frameNumber = inflightMap.keyAt(idx); + + nsecs_t sensorTimestamp = request.sensorTimestamp; + nsecs_t shutterTimestamp = request.shutterTimestamp; + + // Check if it's okay to remove the request from InFlightMap: + // In the case of a successful request: + // all input and output buffers, all result metadata, shutter callback + // arrived. + // In the case of a unsuccessful request: + // all input and output buffers arrived. + if (request.numBuffersLeft == 0 && + (request.skipResultMetadata || + (request.haveResultMetadata && shutterTimestamp != 0))) { + if (request.stillCapture) { + ATRACE_ASYNC_END("still capture", frameNumber); + } + + ATRACE_ASYNC_END("frame capture", frameNumber); + + // Sanity check - if sensor timestamp matches shutter timestamp in the + // case of request having callback. + if (request.hasCallback && request.requestStatus == OK && + sensorTimestamp != shutterTimestamp) { + SET_ERR("sensor timestamp (%" PRId64 + ") for frame %d doesn't match shutter timestamp (%" PRId64 ")", + sensorTimestamp, frameNumber, shutterTimestamp); + } + + // for an unsuccessful request, it may have pending output buffers to + // return. + assert(request.requestStatus != OK || + request.pendingOutputBuffers.size() == 0); + + returnOutputBuffers( + states.useHalBufManager, states.listener, + request.pendingOutputBuffers.array(), + request.pendingOutputBuffers.size(), 0, /*timestampIncreasing*/true, + request.outputSurfaces, request.resultExtras); + + removeInFlightMapEntryLocked(states, idx); + ALOGVV("%s: removed frame %d from InFlightMap", __FUNCTION__, frameNumber); + } + + states.inflightIntf.checkInflightMapLengthLocked(); +} + +void processCaptureResult(CaptureOutputStates& states, const camera3_capture_result *result) { + ATRACE_CALL(); + + status_t res; + + uint32_t frameNumber = result->frame_number; + if (result->result == NULL && result->num_output_buffers == 0 && + result->input_buffer == NULL) { + SET_ERR("No result data provided by HAL for frame %d", + frameNumber); + return; + } + + if (!states.usePartialResult && + result->result != NULL && + result->partial_result != 1) { + SET_ERR("Result is malformed for frame %d: partial_result %u must be 1" + " if partial result is not supported", + frameNumber, result->partial_result); + return; + } + + bool isPartialResult = false; + CameraMetadata collectedPartialResult; + bool hasInputBufferInRequest = false; + + // Get shutter timestamp and resultExtras from list of in-flight requests, + // where it was added by the shutter notification for this frame. If the + // shutter timestamp isn't received yet, append the output buffers to the + // in-flight request and they will be returned when the shutter timestamp + // arrives. Update the in-flight status and remove the in-flight entry if + // all result data and shutter timestamp have been received. + nsecs_t shutterTimestamp = 0; + { + std::lock_guard l(states.inflightLock); + ssize_t idx = states.inflightMap.indexOfKey(frameNumber); + if (idx == NAME_NOT_FOUND) { + SET_ERR("Unknown frame number for capture result: %d", + frameNumber); + return; + } + InFlightRequest &request = states.inflightMap.editValueAt(idx); + ALOGVV("%s: got InFlightRequest requestId = %" PRId32 + ", frameNumber = %" PRId64 ", burstId = %" PRId32 + ", partialResultCount = %d, hasCallback = %d", + __FUNCTION__, request.resultExtras.requestId, + request.resultExtras.frameNumber, request.resultExtras.burstId, + result->partial_result, request.hasCallback); + // Always update the partial count to the latest one if it's not 0 + // (buffers only). When framework aggregates adjacent partial results + // into one, the latest partial count will be used. + if (result->partial_result != 0) + request.resultExtras.partialResultCount = result->partial_result; + + // Check if this result carries only partial metadata + if (states.usePartialResult && result->result != NULL) { + if (result->partial_result > states.numPartialResults || result->partial_result < 1) { + SET_ERR("Result is malformed for frame %d: partial_result %u must be in" + " the range of [1, %d] when metadata is included in the result", + frameNumber, result->partial_result, states.numPartialResults); + return; + } + isPartialResult = (result->partial_result < states.numPartialResults); + if (isPartialResult && result->num_physcam_metadata) { + SET_ERR("Result is malformed for frame %d: partial_result not allowed for" + " physical camera result", frameNumber); + return; + } + if (isPartialResult) { + request.collectedPartialResult.append(result->result); + } + + if (isPartialResult && request.hasCallback) { + // Send partial capture result + sendPartialCaptureResult(states, result->result, request.resultExtras, + frameNumber); + } + } + + shutterTimestamp = request.shutterTimestamp; + hasInputBufferInRequest = request.hasInputBuffer; + + // Did we get the (final) result metadata for this capture? + if (result->result != NULL && !isPartialResult) { + if (request.physicalCameraIds.size() != result->num_physcam_metadata) { + SET_ERR("Expected physical Camera metadata count %d not equal to actual count %d", + request.physicalCameraIds.size(), result->num_physcam_metadata); + return; + } + if (request.haveResultMetadata) { + SET_ERR("Called multiple times with metadata for frame %d", + frameNumber); + return; + } + for (uint32_t i = 0; i < result->num_physcam_metadata; i++) { + String8 physicalId(result->physcam_ids[i]); + std::set::iterator cameraIdIter = + request.physicalCameraIds.find(physicalId); + if (cameraIdIter != request.physicalCameraIds.end()) { + request.physicalCameraIds.erase(cameraIdIter); + } else { + SET_ERR("Total result for frame %d has already returned for camera %s", + frameNumber, physicalId.c_str()); + return; + } + } + if (states.usePartialResult && + !request.collectedPartialResult.isEmpty()) { + collectedPartialResult.acquire( + request.collectedPartialResult); + } + request.haveResultMetadata = true; + } + + uint32_t numBuffersReturned = result->num_output_buffers; + if (result->input_buffer != NULL) { + if (hasInputBufferInRequest) { + numBuffersReturned += 1; + } else { + ALOGW("%s: Input buffer should be NULL if there is no input" + " buffer sent in the request", + __FUNCTION__); + } + } + request.numBuffersLeft -= numBuffersReturned; + if (request.numBuffersLeft < 0) { + SET_ERR("Too many buffers returned for frame %d", + frameNumber); + return; + } + + camera_metadata_ro_entry_t entry; + res = find_camera_metadata_ro_entry(result->result, + ANDROID_SENSOR_TIMESTAMP, &entry); + if (res == OK && entry.count == 1) { + request.sensorTimestamp = entry.data.i64[0]; + } + + // If shutter event isn't received yet, append the output buffers to + // the in-flight request. Otherwise, return the output buffers to + // streams. + if (shutterTimestamp == 0) { + request.pendingOutputBuffers.appendArray(result->output_buffers, + result->num_output_buffers); + } else { + bool timestampIncreasing = !(request.zslCapture || request.hasInputBuffer); + returnOutputBuffers(states.useHalBufManager, states.listener, + result->output_buffers, result->num_output_buffers, + shutterTimestamp, timestampIncreasing, + request.outputSurfaces, request.resultExtras); + } + + if (result->result != NULL && !isPartialResult) { + for (uint32_t i = 0; i < result->num_physcam_metadata; i++) { + CameraMetadata physicalMetadata; + physicalMetadata.append(result->physcam_metadata[i]); + request.physicalMetadatas.push_back({String16(result->physcam_ids[i]), + physicalMetadata}); + } + if (shutterTimestamp == 0) { + request.pendingMetadata = result->result; + request.collectedPartialResult = collectedPartialResult; + } else if (request.hasCallback) { + CameraMetadata metadata; + metadata = result->result; + sendCaptureResult(states, metadata, request.resultExtras, + collectedPartialResult, frameNumber, + hasInputBufferInRequest, request.zslCapture && request.stillCapture, + request.cameraIdsWithZoom, request.physicalMetadatas); + } + } + removeInFlightRequestIfReadyLocked(states, idx); + } // scope for states.inFlightLock + + if (result->input_buffer != NULL) { + if (hasInputBufferInRequest) { + Camera3Stream *stream = + Camera3Stream::cast(result->input_buffer->stream); + res = stream->returnInputBuffer(*(result->input_buffer)); + // Note: stream may be deallocated at this point, if this buffer was the + // last reference to it. + if (res != OK) { + ALOGE("%s: RequestThread: Can't return input buffer for frame %d to" + " its stream:%s (%d)", __FUNCTION__, + frameNumber, strerror(-res), res); + } + } else { + ALOGW("%s: Input buffer should be NULL if there is no input" + " buffer sent in the request, skipping input buffer return.", + __FUNCTION__); + } + } +} + +void processOneCaptureResultLocked( + CaptureOutputStates& states, + const hardware::camera::device::V3_2::CaptureResult& result, + const hardware::hidl_vec< + hardware::camera::device::V3_4::PhysicalCameraMetadata> physicalCameraMetadata) { + using hardware::camera::device::V3_2::StreamBuffer; + using hardware::camera::device::V3_2::BufferStatus; + std::unique_ptr& fmq = states.fmq; + BufferRecordsInterface& bufferRecords = states.bufferRecordsIntf; + camera3_capture_result r; + status_t res; + r.frame_number = result.frameNumber; + + // Read and validate the result metadata. + hardware::camera::device::V3_2::CameraMetadata resultMetadata; + res = readOneCameraMetadataLocked( + fmq, result.fmqResultSize, + resultMetadata, result.result); + if (res != OK) { + ALOGE("%s: Frame %d: Failed to read capture result metadata", + __FUNCTION__, result.frameNumber); + return; + } + r.result = reinterpret_cast(resultMetadata.data()); + + // Read and validate physical camera metadata + size_t physResultCount = physicalCameraMetadata.size(); + std::vector physCamIds(physResultCount); + std::vector phyCamMetadatas(physResultCount); + std::vector physResultMetadata; + physResultMetadata.resize(physResultCount); + for (size_t i = 0; i < physicalCameraMetadata.size(); i++) { + res = readOneCameraMetadataLocked(fmq, physicalCameraMetadata[i].fmqMetadataSize, + physResultMetadata[i], physicalCameraMetadata[i].metadata); + if (res != OK) { + ALOGE("%s: Frame %d: Failed to read capture result metadata for camera %s", + __FUNCTION__, result.frameNumber, + physicalCameraMetadata[i].physicalCameraId.c_str()); + return; + } + physCamIds[i] = physicalCameraMetadata[i].physicalCameraId.c_str(); + phyCamMetadatas[i] = reinterpret_cast( + physResultMetadata[i].data()); + } + r.num_physcam_metadata = physResultCount; + r.physcam_ids = physCamIds.data(); + r.physcam_metadata = phyCamMetadatas.data(); + + std::vector outputBuffers(result.outputBuffers.size()); + std::vector outputBufferHandles(result.outputBuffers.size()); + for (size_t i = 0; i < result.outputBuffers.size(); i++) { + auto& bDst = outputBuffers[i]; + const StreamBuffer &bSrc = result.outputBuffers[i]; + + sp stream = states.outputStreams.get(bSrc.streamId); + if (stream == nullptr) { + ALOGE("%s: Frame %d: Buffer %zu: Invalid output stream id %d", + __FUNCTION__, result.frameNumber, i, bSrc.streamId); + return; + } + bDst.stream = stream->asHalStream(); + + bool noBufferReturned = false; + buffer_handle_t *buffer = nullptr; + if (states.useHalBufManager) { + // This is suspicious most of the time but can be correct during flush where HAL + // has to return capture result before a buffer is requested + if (bSrc.bufferId == BUFFER_ID_NO_BUFFER) { + if (bSrc.status == BufferStatus::OK) { + ALOGE("%s: Frame %d: Buffer %zu: No bufferId for stream %d", + __FUNCTION__, result.frameNumber, i, bSrc.streamId); + // Still proceeds so other buffers can be returned + } + noBufferReturned = true; + } + if (noBufferReturned) { + res = OK; + } else { + res = bufferRecords.popInflightRequestBuffer(bSrc.bufferId, &buffer); + } + } else { + res = bufferRecords.popInflightBuffer(result.frameNumber, bSrc.streamId, &buffer); + } + + if (res != OK) { + ALOGE("%s: Frame %d: Buffer %zu: No in-flight buffer for stream %d", + __FUNCTION__, result.frameNumber, i, bSrc.streamId); + return; + } + + bDst.buffer = buffer; + bDst.status = mapHidlBufferStatus(bSrc.status); + bDst.acquire_fence = -1; + if (bSrc.releaseFence == nullptr) { + bDst.release_fence = -1; + } else if (bSrc.releaseFence->numFds == 1) { + if (noBufferReturned) { + ALOGE("%s: got releaseFence without output buffer!", __FUNCTION__); + } + bDst.release_fence = dup(bSrc.releaseFence->data[0]); + } else { + ALOGE("%s: Frame %d: Invalid release fence for buffer %zu, fd count is %d, not 1", + __FUNCTION__, result.frameNumber, i, bSrc.releaseFence->numFds); + return; + } + } + r.num_output_buffers = outputBuffers.size(); + r.output_buffers = outputBuffers.data(); + + camera3_stream_buffer_t inputBuffer; + if (result.inputBuffer.streamId == -1) { + r.input_buffer = nullptr; + } else { + if (states.inputStream->getId() != result.inputBuffer.streamId) { + ALOGE("%s: Frame %d: Invalid input stream id %d", __FUNCTION__, + result.frameNumber, result.inputBuffer.streamId); + return; + } + inputBuffer.stream = states.inputStream->asHalStream(); + buffer_handle_t *buffer; + res = bufferRecords.popInflightBuffer(result.frameNumber, result.inputBuffer.streamId, + &buffer); + if (res != OK) { + ALOGE("%s: Frame %d: Input buffer: No in-flight buffer for stream %d", + __FUNCTION__, result.frameNumber, result.inputBuffer.streamId); + return; + } + inputBuffer.buffer = buffer; + inputBuffer.status = mapHidlBufferStatus(result.inputBuffer.status); + inputBuffer.acquire_fence = -1; + if (result.inputBuffer.releaseFence == nullptr) { + inputBuffer.release_fence = -1; + } else if (result.inputBuffer.releaseFence->numFds == 1) { + inputBuffer.release_fence = dup(result.inputBuffer.releaseFence->data[0]); + } else { + ALOGE("%s: Frame %d: Invalid release fence for input buffer, fd count is %d, not 1", + __FUNCTION__, result.frameNumber, result.inputBuffer.releaseFence->numFds); + return; + } + r.input_buffer = &inputBuffer; + } + + r.partial_result = result.partialResult; + + processCaptureResult(states, &r); +} + +void returnOutputBuffers( + bool useHalBufManager, + sp listener, + const camera3_stream_buffer_t *outputBuffers, size_t numBuffers, + nsecs_t timestamp, bool timestampIncreasing, + const SurfaceMap& outputSurfaces, + const CaptureResultExtras &inResultExtras) { + + for (size_t i = 0; i < numBuffers; i++) + { + if (outputBuffers[i].buffer == nullptr) { + if (!useHalBufManager) { + // With HAL buffer management API, HAL sometimes will have to return buffers that + // has not got a output buffer handle filled yet. This is though illegal if HAL + // buffer management API is not being used. + ALOGE("%s: cannot return a null buffer!", __FUNCTION__); + } + continue; + } + + Camera3StreamInterface *stream = Camera3Stream::cast(outputBuffers[i].stream); + int streamId = stream->getId(); + const auto& it = outputSurfaces.find(streamId); + status_t res = OK; + if (it != outputSurfaces.end()) { + res = stream->returnBuffer( + outputBuffers[i], timestamp, timestampIncreasing, it->second, + inResultExtras.frameNumber); + } else { + res = stream->returnBuffer( + outputBuffers[i], timestamp, timestampIncreasing, std::vector (), + inResultExtras.frameNumber); + } + + // Note: stream may be deallocated at this point, if this buffer was + // the last reference to it. + if (res == NO_INIT || res == DEAD_OBJECT) { + ALOGV("Can't return buffer to its stream: %s (%d)", strerror(-res), res); + } else if (res != OK) { + ALOGE("Can't return buffer to its stream: %s (%d)", strerror(-res), res); + } + + // Long processing consumers can cause returnBuffer timeout for shared stream + // If that happens, cancel the buffer and send a buffer error to client + if (it != outputSurfaces.end() && res == TIMED_OUT && + outputBuffers[i].status == CAMERA3_BUFFER_STATUS_OK) { + // cancel the buffer + camera3_stream_buffer_t sb = outputBuffers[i]; + sb.status = CAMERA3_BUFFER_STATUS_ERROR; + stream->returnBuffer(sb, /*timestamp*/0, timestampIncreasing, std::vector (), + inResultExtras.frameNumber); + + if (listener != nullptr) { + CaptureResultExtras extras = inResultExtras; + extras.errorStreamId = streamId; + listener->notifyError( + hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_BUFFER, + extras); + } + } + } +} + +void notifyShutter(CaptureOutputStates& states, const camera3_shutter_msg_t &msg) { + ATRACE_CALL(); + ssize_t idx; + + // Set timestamp for the request in the in-flight tracking + // and get the request ID to send upstream + { + std::lock_guard l(states.inflightLock); + InFlightRequestMap& inflightMap = states.inflightMap; + idx = inflightMap.indexOfKey(msg.frame_number); + if (idx >= 0) { + InFlightRequest &r = inflightMap.editValueAt(idx); + + // Verify ordering of shutter notifications + { + std::lock_guard l(states.outputLock); + // TODO: need to track errors for tighter bounds on expected frame number. + if (r.hasInputBuffer) { + if (msg.frame_number < states.nextReprocShutterFrameNum) { + SET_ERR("Reprocess shutter notification out-of-order. Expected " + "notification for frame %d, got frame %d", + states.nextReprocShutterFrameNum, msg.frame_number); + return; + } + states.nextReprocShutterFrameNum = msg.frame_number + 1; + } else if (r.zslCapture && r.stillCapture) { + if (msg.frame_number < states.nextZslShutterFrameNum) { + SET_ERR("ZSL still capture shutter notification out-of-order. Expected " + "notification for frame %d, got frame %d", + states.nextZslShutterFrameNum, msg.frame_number); + return; + } + states.nextZslShutterFrameNum = msg.frame_number + 1; + } else { + if (msg.frame_number < states.nextShutterFrameNum) { + SET_ERR("Shutter notification out-of-order. Expected " + "notification for frame %d, got frame %d", + states.nextShutterFrameNum, msg.frame_number); + return; + } + states.nextShutterFrameNum = msg.frame_number + 1; + } + } + + r.shutterTimestamp = msg.timestamp; + if (r.hasCallback) { + ALOGVV("Camera %s: %s: Shutter fired for frame %d (id %d) at %" PRId64, + states.cameraId.string(), __FUNCTION__, + msg.frame_number, r.resultExtras.requestId, msg.timestamp); + // Call listener, if any + if (states.listener != nullptr) { + states.listener->notifyShutter(r.resultExtras, msg.timestamp); + } + // send pending result and buffers + sendCaptureResult(states, + r.pendingMetadata, r.resultExtras, + r.collectedPartialResult, msg.frame_number, + r.hasInputBuffer, r.zslCapture && r.stillCapture, + r.cameraIdsWithZoom, r.physicalMetadatas); + } + bool timestampIncreasing = !(r.zslCapture || r.hasInputBuffer); + returnOutputBuffers( + states.useHalBufManager, states.listener, + r.pendingOutputBuffers.array(), + r.pendingOutputBuffers.size(), r.shutterTimestamp, timestampIncreasing, + r.outputSurfaces, r.resultExtras); + r.pendingOutputBuffers.clear(); + + removeInFlightRequestIfReadyLocked(states, idx); + } + } + if (idx < 0) { + SET_ERR("Shutter notification for non-existent frame number %d", + msg.frame_number); + } +} + +void notifyError(CaptureOutputStates& states, const camera3_error_msg_t &msg) { + ATRACE_CALL(); + // Map camera HAL error codes to ICameraDeviceCallback error codes + // Index into this with the HAL error code + static const int32_t halErrorMap[CAMERA3_MSG_NUM_ERRORS] = { + // 0 = Unused error code + hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_INVALID_ERROR, + // 1 = CAMERA3_MSG_ERROR_DEVICE + hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_DEVICE, + // 2 = CAMERA3_MSG_ERROR_REQUEST + hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_REQUEST, + // 3 = CAMERA3_MSG_ERROR_RESULT + hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_RESULT, + // 4 = CAMERA3_MSG_ERROR_BUFFER + hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_BUFFER + }; + + int32_t errorCode = + ((msg.error_code >= 0) && + (msg.error_code < CAMERA3_MSG_NUM_ERRORS)) ? + halErrorMap[msg.error_code] : + hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_INVALID_ERROR; + + int streamId = 0; + String16 physicalCameraId; + if (msg.error_stream != nullptr) { + Camera3Stream *stream = + Camera3Stream::cast(msg.error_stream); + streamId = stream->getId(); + physicalCameraId = String16(stream->physicalCameraId()); + } + ALOGV("Camera %s: %s: HAL error, frame %d, stream %d: %d", + states.cameraId.string(), __FUNCTION__, msg.frame_number, + streamId, msg.error_code); + + CaptureResultExtras resultExtras; + switch (errorCode) { + case hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_DEVICE: + // SET_ERR calls into listener to notify application + SET_ERR("Camera HAL reported serious device error"); + break; + case hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_REQUEST: + case hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_RESULT: + case hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_BUFFER: + { + std::lock_guard l(states.inflightLock); + ssize_t idx = states.inflightMap.indexOfKey(msg.frame_number); + if (idx >= 0) { + InFlightRequest &r = states.inflightMap.editValueAt(idx); + r.requestStatus = msg.error_code; + resultExtras = r.resultExtras; + bool logicalDeviceResultError = false; + if (hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_RESULT == + errorCode) { + if (physicalCameraId.size() > 0) { + String8 cameraId(physicalCameraId); + auto iter = r.physicalCameraIds.find(cameraId); + if (iter == r.physicalCameraIds.end()) { + ALOGE("%s: Reported result failure for physical camera device: %s " + " which is not part of the respective request!", + __FUNCTION__, cameraId.string()); + break; + } + r.physicalCameraIds.erase(iter); + resultExtras.errorPhysicalCameraId = physicalCameraId; + } else { + logicalDeviceResultError = true; + } + } + + if (logicalDeviceResultError + || hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_REQUEST == + errorCode) { + r.skipResultMetadata = true; + } + if (logicalDeviceResultError) { + // In case of missing result check whether the buffers + // returned. If they returned, then remove inflight + // request. + // TODO: should we call this for ERROR_CAMERA_REQUEST as well? + // otherwise we are depending on HAL to send the buffers back after + // calling notifyError. Not sure if that's in the spec. + removeInFlightRequestIfReadyLocked(states, idx); + } + } else { + resultExtras.frameNumber = msg.frame_number; + ALOGE("Camera %s: %s: cannot find in-flight request on " + "frame %" PRId64 " error", states.cameraId.string(), __FUNCTION__, + resultExtras.frameNumber); + } + } + resultExtras.errorStreamId = streamId; + if (states.listener != nullptr) { + states.listener->notifyError(errorCode, resultExtras); + } else { + ALOGE("Camera %s: %s: no listener available", + states.cameraId.string(), __FUNCTION__); + } + break; + default: + // SET_ERR calls notifyError + SET_ERR("Unknown error message from HAL: %d", msg.error_code); + break; + } +} + +void notify(CaptureOutputStates& states, const camera3_notify_msg *msg) { + switch (msg->type) { + case CAMERA3_MSG_ERROR: { + notifyError(states, msg->message.error); + break; + } + case CAMERA3_MSG_SHUTTER: { + notifyShutter(states, msg->message.shutter); + break; + } + default: + SET_ERR("Unknown notify message from HAL: %d", + msg->type); + } +} + +void notify(CaptureOutputStates& states, + const hardware::camera::device::V3_2::NotifyMsg& msg) { + using android::hardware::camera::device::V3_2::MsgType; + using android::hardware::camera::device::V3_2::ErrorCode; + + ATRACE_CALL(); + camera3_notify_msg m; + switch (msg.type) { + case MsgType::ERROR: + m.type = CAMERA3_MSG_ERROR; + m.message.error.frame_number = msg.msg.error.frameNumber; + if (msg.msg.error.errorStreamId >= 0) { + sp stream = + states.outputStreams.get(msg.msg.error.errorStreamId); + if (stream == nullptr) { + ALOGE("%s: Frame %d: Invalid error stream id %d", __FUNCTION__, + m.message.error.frame_number, msg.msg.error.errorStreamId); + return; + } + m.message.error.error_stream = stream->asHalStream(); + } else { + m.message.error.error_stream = nullptr; + } + switch (msg.msg.error.errorCode) { + case ErrorCode::ERROR_DEVICE: + m.message.error.error_code = CAMERA3_MSG_ERROR_DEVICE; + break; + case ErrorCode::ERROR_REQUEST: + m.message.error.error_code = CAMERA3_MSG_ERROR_REQUEST; + break; + case ErrorCode::ERROR_RESULT: + m.message.error.error_code = CAMERA3_MSG_ERROR_RESULT; + break; + case ErrorCode::ERROR_BUFFER: + m.message.error.error_code = CAMERA3_MSG_ERROR_BUFFER; + break; + } + break; + case MsgType::SHUTTER: + m.type = CAMERA3_MSG_SHUTTER; + m.message.shutter.frame_number = msg.msg.shutter.frameNumber; + m.message.shutter.timestamp = msg.msg.shutter.timestamp; + break; + } + notify(states, &m); +} + +void requestStreamBuffers(RequestBufferStates& states, + const hardware::hidl_vec& bufReqs, + hardware::camera::device::V3_5::ICameraDeviceCallback::requestStreamBuffers_cb _hidl_cb) { + using android::hardware::camera::device::V3_2::BufferStatus; + using android::hardware::camera::device::V3_2::StreamBuffer; + using android::hardware::camera::device::V3_5::BufferRequestStatus; + using android::hardware::camera::device::V3_5::StreamBufferRet; + using android::hardware::camera::device::V3_5::StreamBufferRequestError; + + std::lock_guard lock(states.reqBufferLock); + + hardware::hidl_vec bufRets; + if (!states.useHalBufManager) { + ALOGE("%s: Camera %s does not support HAL buffer management", + __FUNCTION__, states.cameraId.string()); + _hidl_cb(BufferRequestStatus::FAILED_ILLEGAL_ARGUMENTS, bufRets); + return; + } + + SortedVector streamIds; + ssize_t sz = streamIds.setCapacity(bufReqs.size()); + if (sz < 0 || static_cast(sz) != bufReqs.size()) { + ALOGE("%s: failed to allocate memory for %zu buffer requests", + __FUNCTION__, bufReqs.size()); + _hidl_cb(BufferRequestStatus::FAILED_ILLEGAL_ARGUMENTS, bufRets); + return; + } + + if (bufReqs.size() > states.outputStreams.size()) { + ALOGE("%s: too many buffer requests (%zu > # of output streams %zu)", + __FUNCTION__, bufReqs.size(), states.outputStreams.size()); + _hidl_cb(BufferRequestStatus::FAILED_ILLEGAL_ARGUMENTS, bufRets); + return; + } + + // Check for repeated streamId + for (const auto& bufReq : bufReqs) { + if (streamIds.indexOf(bufReq.streamId) != NAME_NOT_FOUND) { + ALOGE("%s: Stream %d appear multiple times in buffer requests", + __FUNCTION__, bufReq.streamId); + _hidl_cb(BufferRequestStatus::FAILED_ILLEGAL_ARGUMENTS, bufRets); + return; + } + streamIds.add(bufReq.streamId); + } + + if (!states.reqBufferIntf.startRequestBuffer()) { + ALOGE("%s: request buffer disallowed while camera service is configuring", + __FUNCTION__); + _hidl_cb(BufferRequestStatus::FAILED_CONFIGURING, bufRets); + return; + } + + bufRets.resize(bufReqs.size()); + + bool allReqsSucceeds = true; + bool oneReqSucceeds = false; + for (size_t i = 0; i < bufReqs.size(); i++) { + const auto& bufReq = bufReqs[i]; + auto& bufRet = bufRets[i]; + int32_t streamId = bufReq.streamId; + sp outputStream = states.outputStreams.get(streamId); + if (outputStream == nullptr) { + ALOGE("%s: Output stream id %d not found!", __FUNCTION__, streamId); + hardware::hidl_vec emptyBufRets; + _hidl_cb(BufferRequestStatus::FAILED_ILLEGAL_ARGUMENTS, emptyBufRets); + states.reqBufferIntf.endRequestBuffer(); + return; + } + + if (outputStream->isAbandoned()) { + bufRet.val.error(StreamBufferRequestError::STREAM_DISCONNECTED); + allReqsSucceeds = false; + continue; + } + + bufRet.streamId = streamId; + size_t handOutBufferCount = outputStream->getOutstandingBuffersCount(); + uint32_t numBuffersRequested = bufReq.numBuffersRequested; + size_t totalHandout = handOutBufferCount + numBuffersRequested; + uint32_t maxBuffers = outputStream->asHalStream()->max_buffers; + if (totalHandout > maxBuffers) { + // Not able to allocate enough buffer. Exit early for this stream + ALOGE("%s: request too much buffers for stream %d: at HAL: %zu + requesting: %d" + " > max: %d", __FUNCTION__, streamId, handOutBufferCount, + numBuffersRequested, maxBuffers); + bufRet.val.error(StreamBufferRequestError::MAX_BUFFER_EXCEEDED); + allReqsSucceeds = false; + continue; + } + + hardware::hidl_vec tmpRetBuffers(numBuffersRequested); + bool currentReqSucceeds = true; + std::vector streamBuffers(numBuffersRequested); + size_t numAllocatedBuffers = 0; + size_t numPushedInflightBuffers = 0; + for (size_t b = 0; b < numBuffersRequested; b++) { + camera3_stream_buffer_t& sb = streamBuffers[b]; + // Since this method can run concurrently with request thread + // We need to update the wait duration everytime we call getbuffer + nsecs_t waitDuration = states.reqBufferIntf.getWaitDuration(); + status_t res = outputStream->getBuffer(&sb, waitDuration); + if (res != OK) { + if (res == NO_INIT || res == DEAD_OBJECT) { + ALOGV("%s: Can't get output buffer for stream %d: %s (%d)", + __FUNCTION__, streamId, strerror(-res), res); + bufRet.val.error(StreamBufferRequestError::STREAM_DISCONNECTED); + } else { + ALOGE("%s: Can't get output buffer for stream %d: %s (%d)", + __FUNCTION__, streamId, strerror(-res), res); + if (res == TIMED_OUT || res == NO_MEMORY) { + bufRet.val.error(StreamBufferRequestError::NO_BUFFER_AVAILABLE); + } else { + bufRet.val.error(StreamBufferRequestError::UNKNOWN_ERROR); + } + } + currentReqSucceeds = false; + break; + } + numAllocatedBuffers++; + + buffer_handle_t *buffer = sb.buffer; + auto pair = states.bufferRecordsIntf.getBufferId(*buffer, streamId); + bool isNewBuffer = pair.first; + uint64_t bufferId = pair.second; + StreamBuffer& hBuf = tmpRetBuffers[b]; + + hBuf.streamId = streamId; + hBuf.bufferId = bufferId; + hBuf.buffer = (isNewBuffer) ? *buffer : nullptr; + hBuf.status = BufferStatus::OK; + hBuf.releaseFence = nullptr; + + native_handle_t *acquireFence = nullptr; + if (sb.acquire_fence != -1) { + acquireFence = native_handle_create(1,0); + acquireFence->data[0] = sb.acquire_fence; + } + hBuf.acquireFence.setTo(acquireFence, /*shouldOwn*/true); + hBuf.releaseFence = nullptr; + + res = states.bufferRecordsIntf.pushInflightRequestBuffer(bufferId, buffer, streamId); + if (res != OK) { + ALOGE("%s: Can't get register request buffers for stream %d: %s (%d)", + __FUNCTION__, streamId, strerror(-res), res); + bufRet.val.error(StreamBufferRequestError::UNKNOWN_ERROR); + currentReqSucceeds = false; + break; + } + numPushedInflightBuffers++; + } + if (currentReqSucceeds) { + bufRet.val.buffers(std::move(tmpRetBuffers)); + oneReqSucceeds = true; + } else { + allReqsSucceeds = false; + for (size_t b = 0; b < numPushedInflightBuffers; b++) { + StreamBuffer& hBuf = tmpRetBuffers[b]; + buffer_handle_t* buffer; + status_t res = states.bufferRecordsIntf.popInflightRequestBuffer( + hBuf.bufferId, &buffer); + if (res != OK) { + SET_ERR("%s: popInflightRequestBuffer failed for stream %d: %s (%d)", + __FUNCTION__, streamId, strerror(-res), res); + } + } + for (size_t b = 0; b < numAllocatedBuffers; b++) { + camera3_stream_buffer_t& sb = streamBuffers[b]; + sb.acquire_fence = -1; + sb.status = CAMERA3_BUFFER_STATUS_ERROR; + } + returnOutputBuffers(states.useHalBufManager, /*listener*/nullptr, + streamBuffers.data(), numAllocatedBuffers, 0); + } + } + + _hidl_cb(allReqsSucceeds ? BufferRequestStatus::OK : + oneReqSucceeds ? BufferRequestStatus::FAILED_PARTIAL : + BufferRequestStatus::FAILED_UNKNOWN, + bufRets); + states.reqBufferIntf.endRequestBuffer(); +} + +void returnStreamBuffers(ReturnBufferStates& states, + const hardware::hidl_vec& buffers) { + if (!states.useHalBufManager) { + ALOGE("%s: Camera %s does not support HAL buffer managerment", + __FUNCTION__, states.cameraId.string()); + return; + } + + for (const auto& buf : buffers) { + if (buf.bufferId == BUFFER_ID_NO_BUFFER) { + ALOGE("%s: cannot return a buffer without bufferId", __FUNCTION__); + continue; + } + + buffer_handle_t* buffer; + status_t res = states.bufferRecordsIntf.popInflightRequestBuffer(buf.bufferId, &buffer); + + if (res != OK) { + ALOGE("%s: cannot find in-flight buffer %" PRIu64 " for stream %d", + __FUNCTION__, buf.bufferId, buf.streamId); + continue; + } + + camera3_stream_buffer_t streamBuffer; + streamBuffer.buffer = buffer; + streamBuffer.status = CAMERA3_BUFFER_STATUS_ERROR; + streamBuffer.acquire_fence = -1; + streamBuffer.release_fence = -1; + + if (buf.releaseFence == nullptr) { + streamBuffer.release_fence = -1; + } else if (buf.releaseFence->numFds == 1) { + streamBuffer.release_fence = dup(buf.releaseFence->data[0]); + } else { + ALOGE("%s: Invalid release fence, fd count is %d, not 1", + __FUNCTION__, buf.releaseFence->numFds); + continue; + } + + sp stream = states.outputStreams.get(buf.streamId); + if (stream == nullptr) { + ALOGE("%s: Output stream id %d not found!", __FUNCTION__, buf.streamId); + continue; + } + streamBuffer.stream = stream->asHalStream(); + returnOutputBuffers(states.useHalBufManager, /*listener*/nullptr, + &streamBuffer, /*size*/1, /*timestamp*/ 0); + } +} + +void flushInflightRequests(FlushInflightReqStates& states) { + ATRACE_CALL(); + { // First return buffers cached in mInFlightMap + std::lock_guard l(states.inflightLock); + for (size_t idx = 0; idx < states.inflightMap.size(); idx++) { + const InFlightRequest &request = states.inflightMap.valueAt(idx); + returnOutputBuffers( + states.useHalBufManager, states.listener, + request.pendingOutputBuffers.array(), + request.pendingOutputBuffers.size(), 0, + /*timestampIncreasing*/true, request.outputSurfaces, + request.resultExtras); + } + states.inflightMap.clear(); + states.inflightIntf.onInflightMapFlushedLocked(); + } + + // Then return all inflight buffers not returned by HAL + std::vector> inflightKeys; + states.flushBufferIntf.getInflightBufferKeys(&inflightKeys); + + // Inflight buffers for HAL buffer manager + std::vector inflightRequestBufferKeys; + states.flushBufferIntf.getInflightRequestBufferKeys(&inflightRequestBufferKeys); + + // (streamId, frameNumber, buffer_handle_t*) tuple for all inflight buffers. + // frameNumber will be -1 for buffers from HAL buffer manager + std::vector> inflightBuffers; + inflightBuffers.reserve(inflightKeys.size() + inflightRequestBufferKeys.size()); + + for (auto& pair : inflightKeys) { + int32_t frameNumber = pair.first; + int32_t streamId = pair.second; + buffer_handle_t* buffer; + status_t res = states.bufferRecordsIntf.popInflightBuffer(frameNumber, streamId, &buffer); + if (res != OK) { + ALOGE("%s: Frame %d: No in-flight buffer for stream %d", + __FUNCTION__, frameNumber, streamId); + continue; + } + inflightBuffers.push_back(std::make_tuple(streamId, frameNumber, buffer)); + } + + for (auto& bufferId : inflightRequestBufferKeys) { + int32_t streamId = -1; + buffer_handle_t* buffer = nullptr; + status_t res = states.bufferRecordsIntf.popInflightRequestBuffer( + bufferId, &buffer, &streamId); + if (res != OK) { + ALOGE("%s: cannot find in-flight buffer %" PRIu64, __FUNCTION__, bufferId); + continue; + } + inflightBuffers.push_back(std::make_tuple(streamId, /*frameNumber*/-1, buffer)); + } + + std::vector> streams = states.flushBufferIntf.getAllStreams(); + + for (auto& tuple : inflightBuffers) { + status_t res = OK; + int32_t streamId = std::get<0>(tuple); + int32_t frameNumber = std::get<1>(tuple); + buffer_handle_t* buffer = std::get<2>(tuple); + + camera3_stream_buffer_t streamBuffer; + streamBuffer.buffer = buffer; + streamBuffer.status = CAMERA3_BUFFER_STATUS_ERROR; + streamBuffer.acquire_fence = -1; + streamBuffer.release_fence = -1; + + for (auto& stream : streams) { + if (streamId == stream->getId()) { + // Return buffer to deleted stream + camera3_stream* halStream = stream->asHalStream(); + streamBuffer.stream = halStream; + switch (halStream->stream_type) { + case CAMERA3_STREAM_OUTPUT: + res = stream->returnBuffer(streamBuffer, /*timestamp*/ 0, + /*timestampIncreasing*/true, std::vector (), frameNumber); + if (res != OK) { + ALOGE("%s: Can't return output buffer for frame %d to" + " stream %d: %s (%d)", __FUNCTION__, + frameNumber, streamId, strerror(-res), res); + } + break; + case CAMERA3_STREAM_INPUT: + res = stream->returnInputBuffer(streamBuffer); + if (res != OK) { + ALOGE("%s: Can't return input buffer for frame %d to" + " stream %d: %s (%d)", __FUNCTION__, + frameNumber, streamId, strerror(-res), res); + } + break; + default: // Bi-direcitonal stream is deprecated + ALOGE("%s: stream %d has unknown stream type %d", + __FUNCTION__, streamId, halStream->stream_type); + break; + } + break; + } + } + } +} + +} // camera3 +} // namespace android diff --git a/services/camera/libcameraservice/device3/Camera3OutputUtils.h b/services/camera/libcameraservice/device3/Camera3OutputUtils.h new file mode 100644 index 0000000000..47d8095972 --- /dev/null +++ b/services/camera/libcameraservice/device3/Camera3OutputUtils.h @@ -0,0 +1,143 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ANDROID_SERVERS_CAMERA3_OUTPUT_UTILS_H +#define ANDROID_SERVERS_CAMERA3_OUTPUT_UTILS_H + +#include +#include + +#include + +#include + +#include + +#include "device3/BufferUtils.h" +#include "device3/DistortionMapper.h" +#include "device3/ZoomRatioMapper.h" +#include "device3/InFlightRequest.h" +#include "device3/Camera3Stream.h" +#include "device3/Camera3OutputStreamInterface.h" +#include "utils/TagMonitor.h" + +namespace android { + +using ResultMetadataQueue = hardware::MessageQueue; + +namespace camera3 { + + /** + * Helper methods shared between Camera3Device/Camera3OfflineSession for HAL callbacks + */ + // helper function to return the output buffers to output streams. + void returnOutputBuffers( + bool useHalBufManager, + sp listener, // Only needed when outputSurfaces is not empty + const camera3_stream_buffer_t *outputBuffers, + size_t numBuffers, nsecs_t timestamp, bool timestampIncreasing = true, + // The following arguments are only meant for surface sharing use case + const SurfaceMap& outputSurfaces = SurfaceMap{}, + // Used to send buffer error callback when failing to return buffer + const CaptureResultExtras &resultExtras = CaptureResultExtras{}); + + // Camera3Device/Camera3OfflineSession internal states used in notify/processCaptureResult + // callbacks + struct CaptureOutputStates { + const String8& cameraId; + std::mutex& inflightLock; + InFlightRequestMap& inflightMap; // end of inflightLock scope + std::mutex& outputLock; + List& resultQueue; + std::condition_variable& resultSignal; + uint32_t& nextShutterFrameNum; + uint32_t& nextReprocShutterFrameNum; + uint32_t& nextZslShutterFrameNum; + uint32_t& nextResultFrameNum; + uint32_t& nextReprocResultFrameNum; + uint32_t& nextZslResultFrameNum; // end of outputLock scope + const bool useHalBufManager; + const bool usePartialResult; + const bool needFixupMonoChrome; + const uint32_t numPartialResults; + const metadata_vendor_id_t vendorTagId; + const CameraMetadata& deviceInfo; + const std::unordered_map& physicalDeviceInfoMap; + std::unique_ptr& fmq; + std::unordered_map& distortionMappers; + std::unordered_map& zoomRatioMappers; + TagMonitor& tagMonitor; + sp inputStream; + StreamSet& outputStreams; + sp listener; + SetErrorInterface& setErrIntf; + InflightRequestUpdateInterface& inflightIntf; + BufferRecordsInterface& bufferRecordsIntf; + }; + + // Handle one capture result. Assume callers hold the lock to serialize all + // processCaptureResult calls + void processOneCaptureResultLocked( + CaptureOutputStates& states, + const hardware::camera::device::V3_2::CaptureResult& result, + const hardware::hidl_vec< + hardware::camera::device::V3_4::PhysicalCameraMetadata> physicalCameraMetadata); + + // Handle one notify message + void notify(CaptureOutputStates& states, + const hardware::camera::device::V3_2::NotifyMsg& msg); + + struct RequestBufferStates { + const String8& cameraId; + std::mutex& reqBufferLock; // lock to serialize request buffer calls + const bool useHalBufManager; + StreamSet& outputStreams; + SetErrorInterface& setErrIntf; + BufferRecordsInterface& bufferRecordsIntf; + RequestBufferInterface& reqBufferIntf; + }; + + void requestStreamBuffers(RequestBufferStates& states, + const hardware::hidl_vec& bufReqs, + hardware::camera::device::V3_5::ICameraDeviceCallback::requestStreamBuffers_cb _hidl_cb); + + struct ReturnBufferStates { + const String8& cameraId; + const bool useHalBufManager; + StreamSet& outputStreams; + BufferRecordsInterface& bufferRecordsIntf; + }; + + void returnStreamBuffers(ReturnBufferStates& states, + const hardware::hidl_vec& buffers); + + struct FlushInflightReqStates { + const String8& cameraId; + std::mutex& inflightLock; + InFlightRequestMap& inflightMap; // end of inflightLock scope + const bool useHalBufManager; + sp listener; + InflightRequestUpdateInterface& inflightIntf; + BufferRecordsInterface& bufferRecordsIntf; + FlushBufferInterface& flushBufferIntf; + }; + + void flushInflightRequests(FlushInflightReqStates& states); +} // namespace camera3 + +} // namespace android + +#endif diff --git a/services/camera/libcameraservice/device3/Camera3SharedOutputStream.h b/services/camera/libcameraservice/device3/Camera3SharedOutputStream.h index b5e37c2ffd..e645e05e38 100644 --- a/services/camera/libcameraservice/device3/Camera3SharedOutputStream.h +++ b/services/camera/libcameraservice/device3/Camera3SharedOutputStream.h @@ -65,6 +65,12 @@ public: const std::vector &removedSurfaceIds, KeyedVector, size_t> *outputMap/*out*/); + virtual bool getOfflineProcessingSupport() const { + // As per Camera spec. shared streams currently do not support + // offline mode. + return false; + } + private: static const size_t kMaxOutputs = 4; diff --git a/services/camera/libcameraservice/device3/Camera3Stream.cpp b/services/camera/libcameraservice/device3/Camera3Stream.cpp index f707ef835a..7916ddb91d 100644 --- a/services/camera/libcameraservice/device3/Camera3Stream.cpp +++ b/services/camera/libcameraservice/device3/Camera3Stream.cpp @@ -151,6 +151,14 @@ const String8& Camera3Stream::physicalCameraId() const { return mPhysicalCameraId; } +void Camera3Stream::setOfflineProcessingSupport(bool support) { + mSupportOfflineProcessing = support; +} + +bool Camera3Stream::getOfflineProcessingSupport() const { + return mSupportOfflineProcessing; +} + status_t Camera3Stream::forceToIdle() { ATRACE_CALL(); Mutex::Autolock l(mLock); diff --git a/services/camera/libcameraservice/device3/Camera3Stream.h b/services/camera/libcameraservice/device3/Camera3Stream.h index 805df82599..d768d3d332 100644 --- a/services/camera/libcameraservice/device3/Camera3Stream.h +++ b/services/camera/libcameraservice/device3/Camera3Stream.h @@ -167,6 +167,9 @@ class Camera3Stream : android_dataspace getOriginalDataSpace() const; const String8& physicalCameraId() const; + void setOfflineProcessingSupport(bool) override; + bool getOfflineProcessingSupport() const override; + camera3_stream* asHalStream() override { return this; } @@ -592,6 +595,8 @@ class Camera3Stream : String8 mPhysicalCameraId; nsecs_t mLastTimestamp; + + bool mSupportOfflineProcessing = false; }; // class Camera3Stream }; // namespace camera3 diff --git a/services/camera/libcameraservice/device3/Camera3StreamInterface.h b/services/camera/libcameraservice/device3/Camera3StreamInterface.h index 73f501a57b..667e3bb36e 100644 --- a/services/camera/libcameraservice/device3/Camera3StreamInterface.h +++ b/services/camera/libcameraservice/device3/Camera3StreamInterface.h @@ -23,6 +23,7 @@ #include "Camera3StreamBufferListener.h" #include "Camera3StreamBufferFreedListener.h" +struct camera3_stream; struct camera3_stream_buffer; namespace android { @@ -54,6 +55,7 @@ class OutputStreamInfo { android_dataspace dataSpace; uint64_t consumerUsage; bool finalized = false; + bool supportsOffline = false; OutputStreamInfo() : width(-1), height(-1), format(-1), dataSpace(HAL_DATASPACE_UNKNOWN), consumerUsage(0) {} @@ -98,6 +100,12 @@ class Camera3StreamInterface : public virtual RefBase { virtual bool isDataSpaceOverridden() const = 0; virtual android_dataspace getOriginalDataSpace() const = 0; + /** + * Offline processing + */ + virtual void setOfflineProcessingSupport(bool support) = 0; + virtual bool getOfflineProcessingSupport() const = 0; + /** * Get a HAL3 handle for the stream, without starting stream configuration. */ diff --git a/services/camera/libcameraservice/device3/DistortionMapper.h b/services/camera/libcameraservice/device3/DistortionMapper.h index a255003d9b..7dcb67b699 100644 --- a/services/camera/libcameraservice/device3/DistortionMapper.h +++ b/services/camera/libcameraservice/device3/DistortionMapper.h @@ -36,6 +36,15 @@ class DistortionMapper : private CoordinateMapper { public: DistortionMapper(); + DistortionMapper(const DistortionMapper& other) : + mValidMapping(other.mValidMapping), mValidGrids(other.mValidGrids), + mFx(other.mFx), mFy(other.mFy), mCx(other.mCx), mCy(other.mCy), mS(other.mS), + mInvFx(other.mInvFx), mInvFy(other.mInvFy), mK(other.mK), + mArrayWidth(other.mArrayWidth), mArrayHeight(other.mArrayHeight), + mActiveWidth(other.mActiveWidth), mActiveHeight(other.mActiveHeight), + mArrayDiffX(other.mArrayDiffX), mArrayDiffY(other.mArrayDiffY), + mCorrectedGrid(other.mCorrectedGrid), mDistortedGrid(other.mDistortedGrid) {} + /** * Check whether distortion correction is supported by the camera HAL */ @@ -173,7 +182,7 @@ class DistortionMapper : private CoordinateMapper { // pre-calculated inverses for speed float mInvFx, mInvFy; // radial/tangential distortion parameters - float mK[5]; + std::array mK; // pre-correction active array dimensions float mArrayWidth, mArrayHeight; diff --git a/services/camera/libcameraservice/device3/InFlightRequest.h b/services/camera/libcameraservice/device3/InFlightRequest.h new file mode 100644 index 0000000000..ceeaa71b61 --- /dev/null +++ b/services/camera/libcameraservice/device3/InFlightRequest.h @@ -0,0 +1,148 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ANDROID_SERVERS_CAMERA3_INFLIGHT_REQUEST_H +#define ANDROID_SERVERS_CAMERA3_INFLIGHT_REQUEST_H + +#include + +#include +#include +#include +#include + +#include "hardware/camera3.h" + +#include "common/CameraDeviceBase.h" + +namespace android { + +namespace camera3 { + +struct InFlightRequest { + // Set by notify() SHUTTER call. + nsecs_t shutterTimestamp; + // Set by process_capture_result(). + nsecs_t sensorTimestamp; + int requestStatus; + // Set by process_capture_result call with valid metadata + bool haveResultMetadata; + // Decremented by calls to process_capture_result with valid output + // and input buffers + int numBuffersLeft; + CaptureResultExtras resultExtras; + // If this request has any input buffer + bool hasInputBuffer; + + // The last metadata that framework receives from HAL and + // not yet send out because the shutter event hasn't arrived. + // It's added by process_capture_result and sent when framework + // receives the shutter event. + CameraMetadata pendingMetadata; + + // The metadata of the partial results that framework receives from HAL so far + // and has sent out. + CameraMetadata collectedPartialResult; + + // Buffers are added by process_capture_result when output buffers + // return from HAL but framework has not yet received the shutter + // event. They will be returned to the streams when framework receives + // the shutter event. + Vector pendingOutputBuffers; + + // Whether this inflight request's shutter and result callback are to be + // called. The policy is that if the request is the last one in the constrained + // high speed recording request list, this flag will be true. If the request list + // is not for constrained high speed recording, this flag will also be true. + bool hasCallback; + + // Maximum expected frame duration for this request. + // For manual captures, equal to the max of requested exposure time and frame duration + // For auto-exposure modes, equal to 1/(lower end of target FPS range) + nsecs_t maxExpectedDuration; + + // Whether the result metadata for this request is to be skipped. The + // result metadata should be skipped in the case of + // REQUEST/RESULT error. + bool skipResultMetadata; + + // The physical camera ids being requested. + std::set physicalCameraIds; + + // Map of physicalCameraId <-> Metadata + std::vector physicalMetadatas; + + // Indicates a still capture request. + bool stillCapture; + + // Indicates a ZSL capture request + bool zslCapture; + + // Requested camera ids (both logical and physical) with zoomRatio != 1.0f + std::set cameraIdsWithZoom; + + // What shared surfaces an output should go to + SurfaceMap outputSurfaces; + + // TODO: dedupe + static const nsecs_t kDefaultExpectedDuration = 100000000; // 100 ms + + // Default constructor needed by KeyedVector + InFlightRequest() : + shutterTimestamp(0), + sensorTimestamp(0), + requestStatus(OK), + haveResultMetadata(false), + numBuffersLeft(0), + hasInputBuffer(false), + hasCallback(true), + maxExpectedDuration(kDefaultExpectedDuration), + skipResultMetadata(false), + stillCapture(false), + zslCapture(false) { + } + + InFlightRequest(int numBuffers, CaptureResultExtras extras, bool hasInput, + bool hasAppCallback, nsecs_t maxDuration, + const std::set& physicalCameraIdSet, bool isStillCapture, + bool isZslCapture, const std::set& idsWithZoom, + const SurfaceMap& outSurfaces = SurfaceMap{}) : + shutterTimestamp(0), + sensorTimestamp(0), + requestStatus(OK), + haveResultMetadata(false), + numBuffersLeft(numBuffers), + resultExtras(extras), + hasInputBuffer(hasInput), + hasCallback(hasAppCallback), + maxExpectedDuration(maxDuration), + skipResultMetadata(false), + physicalCameraIds(physicalCameraIdSet), + stillCapture(isStillCapture), + zslCapture(isZslCapture), + cameraIdsWithZoom(idsWithZoom), + outputSurfaces(outSurfaces) { + } +}; + +// Map from frame number to the in-flight request state +typedef KeyedVector InFlightRequestMap; + +} // namespace camera3 + +} // namespace android + +#endif diff --git a/services/camera/libcameraservice/hidl/HidlCameraDeviceUser.cpp b/services/camera/libcameraservice/hidl/HidlCameraDeviceUser.cpp index 675ad24bcf..bf89ca5a7b 100644 --- a/services/camera/libcameraservice/hidl/HidlCameraDeviceUser.cpp +++ b/services/camera/libcameraservice/hidl/HidlCameraDeviceUser.cpp @@ -201,8 +201,9 @@ Return HidlCameraDeviceUser::endConfigure(StreamConfigurationMode opera return HStatus::ILLEGAL_ARGUMENT; } + std::vector offlineStreamIds; binder::Status ret = mDeviceRemote->endConfigure(convertFromHidl(operatingMode), - cameraMetadata); + cameraMetadata, &offlineStreamIds); return B2HStatus(ret); } diff --git a/services/camera/libcameraservice/utils/TagMonitor.cpp b/services/camera/libcameraservice/utils/TagMonitor.cpp index 4037a669cc..262f962c74 100644 --- a/services/camera/libcameraservice/utils/TagMonitor.cpp +++ b/services/camera/libcameraservice/utils/TagMonitor.cpp @@ -33,6 +33,16 @@ TagMonitor::TagMonitor(): mVendorTagId(CAMERA_METADATA_INVALID_VENDOR_ID) {} +TagMonitor::TagMonitor(const TagMonitor& other): + mMonitoringEnabled(other.mMonitoringEnabled.load()), + mMonitoredTagList(other.mMonitoredTagList), + mLastMonitoredRequestValues(other.mLastMonitoredRequestValues), + mLastMonitoredResultValues(other.mLastMonitoredResultValues), + mLastMonitoredPhysicalRequestKeys(other.mLastMonitoredPhysicalRequestKeys), + mLastMonitoredPhysicalResultKeys(other.mLastMonitoredPhysicalResultKeys), + mMonitoringEvents(other.mMonitoringEvents), + mVendorTagId(other.mVendorTagId) {} + const String16 TagMonitor::kMonitorOption = String16("-m"); const char* TagMonitor::k3aTags = diff --git a/services/camera/libcameraservice/utils/TagMonitor.h b/services/camera/libcameraservice/utils/TagMonitor.h index 1b7b0330fe..413f502020 100644 --- a/services/camera/libcameraservice/utils/TagMonitor.h +++ b/services/camera/libcameraservice/utils/TagMonitor.h @@ -50,6 +50,8 @@ class TagMonitor { TagMonitor(); + TagMonitor(const TagMonitor& other); + void initialize(metadata_vendor_id_t id) { mVendorTagId = id; } // Parse tag name list (comma-separated) and if valid, enable monitoring