From b978c389e811a957d173cca97e19dfb5cc49ba38 Mon Sep 17 00:00:00 2001 From: Yin-Chia Yeh Date: Wed, 30 Oct 2019 00:22:37 -0700 Subject: [PATCH] Camera: add offline session API skeleton Test: N/A, no implementation yet Bug: 135142453 Change-Id: I497e42433d4d49466fe37467d4ff66e8583c55df --- camera/Android.bp | 1 + .../hardware/camera2/ICameraDeviceUser.aidl | 13 ++ .../camera2/ICameraOfflineSession.aidl | 23 ++++ .../include/camera/NdkCameraMetadataTags.h | 2 +- services/camera/libcameraservice/Android.bp | 3 + .../camera/libcameraservice/CameraService.cpp | 14 ++ .../camera/libcameraservice/CameraService.h | 82 ++++++++++++ .../api2/CameraDeviceClient.cpp | 74 ++++++++++ .../api2/CameraDeviceClient.h | 7 + .../api2/CameraOfflineSessionClient.h | 85 ++++++++++++ .../common/CameraDeviceBase.h | 9 ++ .../common/CameraOfflineSessionBase.cpp | 27 ++++ .../common/CameraOfflineSessionBase.h | 53 ++++++++ .../device3/Camera3Device.cpp | 122 +++++++++++++++-- .../libcameraservice/device3/Camera3Device.h | 27 +++- .../device3/Camera3OfflineSession.cpp | 118 ++++++++++++++++ .../device3/Camera3OfflineSession.h | 126 ++++++++++++++++++ 17 files changed, 776 insertions(+), 10 deletions(-) create mode 100644 camera/aidl/android/hardware/camera2/ICameraOfflineSession.aidl create mode 100644 services/camera/libcameraservice/api2/CameraOfflineSessionClient.h create mode 100644 services/camera/libcameraservice/common/CameraOfflineSessionBase.cpp create mode 100644 services/camera/libcameraservice/common/CameraOfflineSessionBase.h create mode 100644 services/camera/libcameraservice/device3/Camera3OfflineSession.cpp create mode 100644 services/camera/libcameraservice/device3/Camera3OfflineSession.h diff --git a/camera/Android.bp b/camera/Android.bp index b288bcfce0..ea7259a6dc 100644 --- a/camera/Android.bp +++ b/camera/Android.bp @@ -85,6 +85,7 @@ filegroup { "aidl/android/hardware/ICameraServiceProxy.aidl", "aidl/android/hardware/camera2/ICameraDeviceCallbacks.aidl", "aidl/android/hardware/camera2/ICameraDeviceUser.aidl", + "aidl/android/hardware/camera2/ICameraOfflineSession.aidl", ], path: "aidl", } diff --git a/camera/aidl/android/hardware/camera2/ICameraDeviceUser.aidl b/camera/aidl/android/hardware/camera2/ICameraDeviceUser.aidl index 93549e05de..cdb7ac3a9d 100644 --- a/camera/aidl/android/hardware/camera2/ICameraDeviceUser.aidl +++ b/camera/aidl/android/hardware/camera2/ICameraDeviceUser.aidl @@ -17,6 +17,8 @@ package android.hardware.camera2; import android.hardware.camera2.CaptureRequest; +import android.hardware.camera2.ICameraDeviceCallbacks; +import android.hardware.camera2.ICameraOfflineSession; import android.hardware.camera2.impl.CameraMetadataNative; import android.hardware.camera2.params.OutputConfiguration; import android.hardware.camera2.params.SessionConfiguration; @@ -177,4 +179,15 @@ interface ICameraDeviceUser * @return the currently applied system-wide audio restriction mode */ int getGlobalAudioRestriction(); + + /** + * Offline processing main entry point + * + * @param callbacks Object that will receive callbacks from offline session + * @param offlineOutputIds The ID of streams that needs to be preserved in offline session + * + * @return Offline session object. + */ + ICameraOfflineSession switchToOffline(in ICameraDeviceCallbacks callbacks, + in Surface[] offlineOutputs); } diff --git a/camera/aidl/android/hardware/camera2/ICameraOfflineSession.aidl b/camera/aidl/android/hardware/camera2/ICameraOfflineSession.aidl new file mode 100644 index 0000000000..ab030abea9 --- /dev/null +++ b/camera/aidl/android/hardware/camera2/ICameraOfflineSession.aidl @@ -0,0 +1,23 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.hardware.camera2; + + /** @hide */ +interface ICameraOfflineSession +{ + void disconnect(); +} diff --git a/camera/ndk/include/camera/NdkCameraMetadataTags.h b/camera/ndk/include/camera/NdkCameraMetadataTags.h index 825f308efb..0652cbbad8 100644 --- a/camera/ndk/include/camera/NdkCameraMetadataTags.h +++ b/camera/ndk/include/camera/NdkCameraMetadataTags.h @@ -1311,7 +1311,7 @@ typedef enum acamera_metadata_tag { /** *

List of the maximum number of regions that can be used for metering in * auto-exposure (AE), auto-white balance (AWB), and auto-focus (AF); - * this corresponds to the the maximum number of elements in + * this corresponds to the maximum number of elements in * ACAMERA_CONTROL_AE_REGIONS, ACAMERA_CONTROL_AWB_REGIONS, * and ACAMERA_CONTROL_AF_REGIONS.

* diff --git a/services/camera/libcameraservice/Android.bp b/services/camera/libcameraservice/Android.bp index c50a3c621f..9cc15cdea2 100644 --- a/services/camera/libcameraservice/Android.bp +++ b/services/camera/libcameraservice/Android.bp @@ -26,6 +26,7 @@ cc_library_shared { "CameraFlashlight.cpp", "common/Camera2ClientBase.cpp", "common/CameraDeviceBase.cpp", + "common/CameraOfflineSessionBase.cpp", "common/CameraProviderManager.cpp", "common/FrameProcessorBase.cpp", "api1/CameraClient.cpp", @@ -45,6 +46,7 @@ cc_library_shared { "api2/HeicCompositeStream.cpp", "device1/CameraHardwareInterface.cpp", "device3/Camera3Device.cpp", + "device3/Camera3OfflineSession.cpp", "device3/Camera3Stream.cpp", "device3/Camera3IOStreamBase.cpp", "device3/Camera3InputStream.cpp", @@ -110,6 +112,7 @@ cc_library_shared { "android.hardware.camera.device@3.3", "android.hardware.camera.device@3.4", "android.hardware.camera.device@3.5", + "android.hardware.camera.device@3.6" ], export_shared_lib_headers: [ diff --git a/services/camera/libcameraservice/CameraService.cpp b/services/camera/libcameraservice/CameraService.cpp index b704573a32..18ed3a527a 100644 --- a/services/camera/libcameraservice/CameraService.cpp +++ b/services/camera/libcameraservice/CameraService.cpp @@ -2776,6 +2776,20 @@ void CameraService::BasicClient::block() { // ---------------------------------------------------------------------------- +sp CameraService::OfflineClient::sCameraService; + +status_t CameraService::OfflineClient::startCameraOps() { + // TODO + return OK; +} + +status_t CameraService::OfflineClient::finishCameraOps() { + // TODO + return OK; +} + +// ---------------------------------------------------------------------------- + void CameraService::Client::notifyError(int32_t errorCode, const CaptureResultExtras& resultExtras) { (void) resultExtras; diff --git a/services/camera/libcameraservice/CameraService.h b/services/camera/libcameraservice/CameraService.h index 1f40fc31d4..dae9c0960f 100644 --- a/services/camera/libcameraservice/CameraService.h +++ b/services/camera/libcameraservice/CameraService.h @@ -71,6 +71,7 @@ class CameraService : public: class Client; class BasicClient; + class OfflineClient; // The effective API level. The Camera2 API running in LEGACY mode counts as API_1. enum apiLevel { @@ -401,6 +402,87 @@ public: int mCameraId; // All API1 clients use integer camera IDs }; // class Client + + // Client for offline session. Note that offline session client does not affect camera service's + // client arbitration logic. It is camera HAL's decision to decide whether a normal camera + // client is conflicting with existing offline client(s). + // The other distinctive difference between offline clients and normal clients is that normal + // clients are created through ICameraService binder calls, while the offline session client + // is created through ICameraDeviceUser::switchToOffline call. + class OfflineClient : public virtual RefBase { + + virtual status_t dump(int fd, const Vector& args) = 0; + + // Block the client form using the camera + virtual void block() = 0; + + // Return the package name for this client + virtual String16 getPackageName() const = 0; + + // Notify client about a fatal error + // TODO: maybe let impl notify within block? + virtual void notifyError(int32_t errorCode, + const CaptureResultExtras& resultExtras) = 0; + + // Get the UID of the application client using this + virtual uid_t getClientUid() const = 0; + + // Get the PID of the application client using this + virtual int getClientPid() const = 0; + + protected: + OfflineClient(const sp& cameraService, + const String16& clientPackageName, + const String8& cameraIdStr, + int clientPid, + uid_t clientUid, + int servicePid): mCameraIdStr(cameraIdStr), + mClientPackageName(clientPackageName), mClientPid(clientPid), + mClientUid(clientUid), mServicePid(servicePid) { + if (sCameraService == nullptr) { + sCameraService = cameraService; + } + } + + virtual ~OfflineClient() { /*TODO*/ } + + // these are initialized in the constructor. + static sp sCameraService; + const String8 mCameraIdStr; + String16 mClientPackageName; + pid_t mClientPid; + const uid_t mClientUid; + const pid_t mServicePid; + bool mDisconnected; + + // - The app-side Binder interface to receive callbacks from us + sp mRemoteBinder; // immutable after constructor + + // permissions management + status_t startCameraOps(); + status_t finishCameraOps(); + + private: + std::unique_ptr mAppOpsManager = nullptr; + + class OpsCallback : public BnAppOpsCallback { + public: + explicit OpsCallback(wp client) : mClient(client) {} + virtual void opChanged(int32_t /*op*/, const String16& /*packageName*/) { + //TODO + } + + private: + wp mClient; + + }; // class OpsCallback + + sp mOpsCallback; + + // IAppOpsCallback interface, indirected through opListener + // virtual void opChanged(int32_t op, const String16& packageName); + }; // class OfflineClient + /** * A listener class that implements the LISTENER interface for use with a ClientManager, and * implements the following methods: diff --git a/services/camera/libcameraservice/api2/CameraDeviceClient.cpp b/services/camera/libcameraservice/api2/CameraDeviceClient.cpp index 27bebde10c..28421ba90d 100644 --- a/services/camera/libcameraservice/api2/CameraDeviceClient.cpp +++ b/services/camera/libcameraservice/api2/CameraDeviceClient.cpp @@ -1901,6 +1901,80 @@ binder::Status CameraDeviceClient::getGlobalAudioRestriction(/*out*/ int32_t* ou return binder::Status::ok(); } +binder::Status CameraDeviceClient::switchToOffline( + const sp& cameraCb, + const std::vector& offlineOutputs, + /*out*/ + sp* session) { + ATRACE_CALL(); + + binder::Status res; + if (!(res = checkPidStatus(__FUNCTION__)).isOk()) return res; + + Mutex::Autolock icl(mBinderSerializationLock); + + if (!mDevice.get()) { + return STATUS_ERROR(CameraService::ERROR_DISCONNECTED, "Camera device no longer alive"); + } + + if (offlineOutputs.empty()) { + String8 msg = String8::format("Offline outputs must not be empty"); + ALOGE("%s: %s", __FUNCTION__, msg.string()); + return STATUS_ERROR(CameraService::ERROR_ILLEGAL_ARGUMENT, msg.string()); + } + + if (session == nullptr) { + String8 msg = String8::format("Invalid offline session"); + ALOGE("%s: %s", __FUNCTION__, msg.string()); + return STATUS_ERROR(CameraService::ERROR_ILLEGAL_ARGUMENT, msg.string()); + } + + std::vector offlineStreamIds(offlineOutputs.size()); + for (auto& surface : offlineOutputs) { + sp binder = IInterface::asBinder(surface.graphicBufferProducer); + ssize_t index = mStreamMap.indexOfKey(binder); + if (index == NAME_NOT_FOUND) { + String8 msg = String8::format("Offline output is invalid"); + ALOGE("%s: %s", __FUNCTION__, msg.string()); + return STATUS_ERROR(CameraService::ERROR_ILLEGAL_ARGUMENT, msg.string()); + } + // TODO: Also check whether the offline output is supported by Hal for offline mode. + + sp s = new Surface(surface.graphicBufferProducer); + bool isCompositeStream = camera3::DepthCompositeStream::isDepthCompositeStream(s); + isCompositeStream |= camera3::HeicCompositeStream::isHeicCompositeStream(s); + if (isCompositeStream) { + // TODO: Add composite specific handling + } else { + offlineStreamIds.push_back(mStreamMap.valueAt(index).streamId()); + } + } + + sp offlineSession; + auto ret = mDevice->switchToOffline(offlineStreamIds, &offlineSession); + if (ret != OK) { + return STATUS_ERROR_FMT(CameraService::ERROR_ILLEGAL_ARGUMENT, + "Camera %s: Error switching to offline mode: %s (%d)", + mCameraIdStr.string(), strerror(ret), ret); + } + + sp offlineClient = new CameraOfflineSessionClient(sCameraService, + offlineSession, cameraCb, mClientPackageName, mCameraIdStr, mClientPid, mClientUid, + mServicePid); + ret = offlineClient->initialize(); + if (ret == OK) { + // TODO: We need to update mStreamMap, mConfiguredOutputs + } else { + return STATUS_ERROR_FMT(CameraService::ERROR_ILLEGAL_ARGUMENT, + "Camera %s: Failed to initilize offline session: %s (%d)", + mCameraIdStr.string(), strerror(ret), ret); + } + + *session = offlineClient; + + return binder::Status::ok(); +} + status_t CameraDeviceClient::dump(int fd, const Vector& args) { return BasicClient::dump(fd, args); } diff --git a/services/camera/libcameraservice/api2/CameraDeviceClient.h b/services/camera/libcameraservice/api2/CameraDeviceClient.h index 7efc5abe18..0a8f37777c 100644 --- a/services/camera/libcameraservice/api2/CameraDeviceClient.h +++ b/services/camera/libcameraservice/api2/CameraDeviceClient.h @@ -23,6 +23,7 @@ #include #include +#include "CameraOfflineSessionClient.h" #include "CameraService.h" #include "common/FrameProcessorBase.h" #include "common/Camera2ClientBase.h" @@ -157,6 +158,12 @@ public: virtual binder::Status getGlobalAudioRestriction(/*out*/int32_t* outMode) override; + virtual binder::Status switchToOffline( + const sp& cameraCb, + const std::vector& offlineOutputs, + /*out*/ + sp* session) override; + /** * Interface used by CameraService */ diff --git a/services/camera/libcameraservice/api2/CameraOfflineSessionClient.h b/services/camera/libcameraservice/api2/CameraOfflineSessionClient.h new file mode 100644 index 0000000000..cb83e2907a --- /dev/null +++ b/services/camera/libcameraservice/api2/CameraOfflineSessionClient.h @@ -0,0 +1,85 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ANDROID_SERVERS_CAMERA_PHOTOGRAPHY_CAMERAOFFLINESESSIONCLIENT_H +#define ANDROID_SERVERS_CAMERA_PHOTOGRAPHY_CAMERAOFFLINESESSIONCLIENT_H + +#include +#include +#include "CameraService.h" + +namespace android { + +using android::hardware::camera2::ICameraDeviceCallbacks; + +class CameraOfflineSessionClient : + public CameraService::OfflineClient, + public hardware::camera2::BnCameraOfflineSession + // public camera2::FrameProcessorBase::FilteredListener? +{ +public: + CameraOfflineSessionClient( + const sp& cameraService, + sp session, + const sp& remoteCallback, + const String16& clientPackageName, + const String8& cameraIdStr, + int clientPid, uid_t clientUid, int servicePid) : + CameraService::OfflineClient(cameraService, clientPackageName, + cameraIdStr, clientPid, clientUid, servicePid), + mRemoteCallback(remoteCallback), mOfflineSession(session) {} + + ~CameraOfflineSessionClient() {} + + virtual binder::Status disconnect() override { return binder::Status::ok(); } + + virtual status_t dump(int /*fd*/, const Vector& /*args*/) override { + return OK; + } + + // Block the client form using the camera + virtual void block() override {}; + + // Return the package name for this client + virtual String16 getPackageName() const override { String16 ret; return ret; }; + + // Notify client about a fatal error + // TODO: maybe let impl notify within block? + virtual void notifyError(int32_t /*errorCode*/, + const CaptureResultExtras& /*resultExtras*/) override {} + + // Get the UID of the application client using this + virtual uid_t getClientUid() const override { return 0; } + + // Get the PID of the application client using this + virtual int getClientPid() const override { return 0; } + + status_t initialize() { + // TODO: Talk to camera service to add the offline session client book keeping + return OK; + } +private: + sp mSession; + + sp mRemoteCallback; + // This class is responsible to convert HAL callbacks to AIDL callbacks + + sp mOfflineSession; +}; + +} // namespace android + +#endif // ANDROID_SERVERS_CAMERA_PHOTOGRAPHY_CAMERAOFFLINESESSIONCLIENT_H diff --git a/services/camera/libcameraservice/common/CameraDeviceBase.h b/services/camera/libcameraservice/common/CameraDeviceBase.h index 935bc37a62..1026fdf2b3 100644 --- a/services/camera/libcameraservice/common/CameraDeviceBase.h +++ b/services/camera/libcameraservice/common/CameraDeviceBase.h @@ -35,6 +35,8 @@ #include "device3/Camera3StreamInterface.h" #include "binder/Status.h" +#include "CameraOfflineSessionBase.h" + namespace android { class CameraProviderManager; @@ -389,6 +391,13 @@ class CameraDeviceBase : public virtual RefBase { * requests to complete, based on their settings */ virtual nsecs_t getExpectedInFlightDuration() = 0; + + /** + * switch to offline session + */ + virtual status_t switchToOffline( + const std::vector& streamsToKeep, + /*out*/ sp* session) = 0; }; }; // namespace android diff --git a/services/camera/libcameraservice/common/CameraOfflineSessionBase.cpp b/services/camera/libcameraservice/common/CameraOfflineSessionBase.cpp new file mode 100644 index 0000000000..ff673a91d2 --- /dev/null +++ b/services/camera/libcameraservice/common/CameraOfflineSessionBase.cpp @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "CameraOfflineSessionBase.h" + +namespace android { + +/** + * Base class destructors + */ +CameraOfflineSessionBase::~CameraOfflineSessionBase() { +} + +} // namespace android diff --git a/services/camera/libcameraservice/common/CameraOfflineSessionBase.h b/services/camera/libcameraservice/common/CameraOfflineSessionBase.h new file mode 100644 index 0000000000..38625218f3 --- /dev/null +++ b/services/camera/libcameraservice/common/CameraOfflineSessionBase.h @@ -0,0 +1,53 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ANDROID_SERVERS_CAMERA_CAMERAOFFLINESESSIONBASE_H +#define ANDROID_SERVERS_CAMERA_CAMERAOFFLINESESSIONBASE_H + +#include +#include +#include + +#include "camera/CaptureResult.h" + +namespace android { + +class CameraOfflineSessionBase : public virtual RefBase { + public: + virtual ~CameraOfflineSessionBase(); + + // The session's original camera ID + virtual const String8& getId() const = 0; + + virtual status_t disconnect() = 0; + + virtual status_t dump(int fd) = 0; + + virtual status_t abort() = 0; + + /** + * Capture result passing + */ + virtual status_t waitForNextFrame(nsecs_t timeout) = 0; + + virtual status_t getNextResult(CaptureResult *frame) = 0; + + // TODO: notification passing path +}; // class CameraOfflineSessionBase + +} // namespace android + +#endif diff --git a/services/camera/libcameraservice/device3/Camera3Device.cpp b/services/camera/libcameraservice/device3/Camera3Device.cpp index 254136543c..4e5c8d6241 100644 --- a/services/camera/libcameraservice/device3/Camera3Device.cpp +++ b/services/camera/libcameraservice/device3/Camera3Device.cpp @@ -58,6 +58,7 @@ #include "device3/Camera3InputStream.h" #include "device3/Camera3DummyStream.h" #include "device3/Camera3SharedOutputStream.h" +#include "device3/Camera3OfflineSession.h" #include "CameraService.h" #include "utils/CameraThreadState.h" @@ -206,7 +207,15 @@ status_t Camera3Device::initialize(sp manager, const Stri ANDROID_INFO_SUPPORTED_BUFFER_MANAGEMENT_VERSION_HIDL_DEVICE_3_5); } - mInterface = new HalInterface(session, queue, mUseHalBufManager); + camera_metadata_entry_t capabilities = mDeviceInfo.find(ANDROID_REQUEST_AVAILABLE_CAPABILITIES); + for (size_t i = 0; i < capabilities.count; i++) { + uint8_t capability = capabilities.data.u8[i]; + if (capability == ANDROID_REQUEST_AVAILABLE_CAPABILITIES_OFFLINE_PROCESSING) { + mSupportOfflineProcessing = true; + } + } + + mInterface = new HalInterface(session, queue, mUseHalBufManager, mSupportOfflineProcessing); std::string providerType; mVendorTagId = manager->getProviderTagIdLocked(mId.string()); mTagMonitor.initialize(mVendorTagId); @@ -227,9 +236,8 @@ status_t Camera3Device::initialize(sp manager, const Stri maxVersion.get_major(), maxVersion.get_minor()); bool isMonochrome = false; - camera_metadata_entry_t entry = mDeviceInfo.find(ANDROID_REQUEST_AVAILABLE_CAPABILITIES); - for (size_t i = 0; i < entry.count; i++) { - uint8_t capability = entry.data.u8[i]; + for (size_t i = 0; i < capabilities.count; i++) { + uint8_t capability = capabilities.data.u8[i]; if (capability == ANDROID_REQUEST_AVAILABLE_CAPABILITIES_MONOCHROME) { isMonochrome = true; } @@ -4045,13 +4053,18 @@ void Camera3Device::monitorMetadata(TagMonitor::eventSource source, Camera3Device::HalInterface::HalInterface( sp &session, std::shared_ptr queue, - bool useHalBufManager) : + bool useHalBufManager, bool supportOfflineProcessing) : mHidlSession(session), mRequestMetadataQueue(queue), mUseHalBufManager(useHalBufManager), - mIsReconfigurationQuerySupported(true) { + mIsReconfigurationQuerySupported(true), + mSupportOfflineProcessing(supportOfflineProcessing) { // Check with hardware service manager if we can downcast these interfaces // Somewhat expensive, so cache the results at startup + auto castResult_3_6 = device::V3_6::ICameraDeviceSession::castFrom(mHidlSession); + if (castResult_3_6.isOk()) { + mHidlSession_3_6 = castResult_3_6; + } auto castResult_3_5 = device::V3_5::ICameraDeviceSession::castFrom(mHidlSession); if (castResult_3_5.isOk()) { mHidlSession_3_5 = castResult_3_5; @@ -4066,18 +4079,22 @@ Camera3Device::HalInterface::HalInterface( } } -Camera3Device::HalInterface::HalInterface() : mUseHalBufManager(false) {} +Camera3Device::HalInterface::HalInterface() : + mUseHalBufManager(false), + mSupportOfflineProcessing(false) {} Camera3Device::HalInterface::HalInterface(const HalInterface& other) : mHidlSession(other.mHidlSession), mRequestMetadataQueue(other.mRequestMetadataQueue), - mUseHalBufManager(other.mUseHalBufManager) {} + mUseHalBufManager(other.mUseHalBufManager), + mSupportOfflineProcessing(other.mSupportOfflineProcessing) {} bool Camera3Device::HalInterface::valid() { return (mHidlSession != nullptr); } void Camera3Device::HalInterface::clear() { + mHidlSession_3_6.clear(); mHidlSession_3_5.clear(); mHidlSession_3_4.clear(); mHidlSession_3_3.clear(); @@ -4753,6 +4770,38 @@ void Camera3Device::HalInterface::signalPipelineDrain(const std::vector& st } } +status_t Camera3Device::HalInterface::switchToOffline( + const std::vector& streamsToKeep, + /*out*/hardware::camera::device::V3_6::CameraOfflineSessionInfo* offlineSessionInfo, + /*out*/sp* offlineSession) { + ATRACE_NAME("CameraHal::switchToOffline"); + if (!valid() || mHidlSession_3_6 == nullptr) { + ALOGE("%s called on invalid camera!", __FUNCTION__); + return INVALID_OPERATION; + } + + if (offlineSessionInfo == nullptr || offlineSession == nullptr) { + ALOGE("%s: offlineSessionInfo and offlineSession must not be null!", __FUNCTION__); + return INVALID_OPERATION; + } + + common::V1_0::Status status = common::V1_0::Status::INTERNAL_ERROR; + + auto resultCallback = + [&status, &offlineSessionInfo, &offlineSession] (auto s, auto info, auto session) { + status = s; + *offlineSessionInfo = info; + *offlineSession = session; + }; + auto err = mHidlSession_3_6->switchToOffline(streamsToKeep, resultCallback); + + if (!err.isOk()) { + ALOGE("%s: Transaction error: %s", __FUNCTION__, err.description().c_str()); + return DEAD_OBJECT; + } + return CameraProviderManager::mapToStatusT(status); +} + void Camera3Device::HalInterface::getInflightBufferKeys( std::vector>* out) { std::lock_guard lock(mInflightLock); @@ -5524,6 +5573,7 @@ bool Camera3Device::RequestThread::threadLoop() { Mutex::Autolock l(mRequestLock); mNextRequests.clear(); } + mRequestSubmittedSignal.signal(); return submitRequestSuccess; } @@ -5903,6 +5953,32 @@ void Camera3Device::RequestThread::signalPipelineDrain(const std::vector& s mStreamIdsToBeDrained = streamIds; } +status_t Camera3Device::RequestThread::switchToOffline( + const std::vector& streamsToKeep, + /*out*/hardware::camera::device::V3_6::CameraOfflineSessionInfo* offlineSessionInfo, + /*out*/sp* offlineSession) { + Mutex::Autolock l(mRequestLock); + clearRepeatingRequestsLocked(/*lastFrameNumber*/nullptr); + + // Theoretically we should also check for mRepeatingRequests.empty(), but the API interface + // is serialized by mInterfaceLock so skip that check. + bool queueEmpty = mNextRequests.empty() && mRequestQueue.empty(); + while (!queueEmpty) { + status_t res = mRequestSubmittedSignal.waitRelative(mRequestLock, kRequestSubmitTimeout); + if (res == TIMED_OUT) { + ALOGE("%s: request thread failed to submit a request within timeout!", __FUNCTION__); + return res; + } else if (res != OK) { + ALOGE("%s: request thread failed to submit a request: %s (%d)!", + __FUNCTION__, strerror(-res), res); + return res; + } + queueEmpty = mNextRequests.empty() && mRequestQueue.empty(); + } + return mInterface->switchToOffline( + streamsToKeep, offlineSessionInfo, offlineSession); +} + nsecs_t Camera3Device::getExpectedInFlightDuration() { ATRACE_CALL(); Mutex::Autolock al(mInFlightLock); @@ -6812,4 +6888,34 @@ status_t Camera3Device::fixupMonochromeTags(const CameraMetadata& deviceInfo, return res; } +status_t Camera3Device::switchToOffline( + const std::vector& streamsToKeep, + /*out*/ sp* session) { + ATRACE_CALL(); + if (session == nullptr) { + ALOGE("%s: session must not be null", __FUNCTION__); + return BAD_VALUE; + } + + Mutex::Autolock il(mInterfaceLock); + Mutex::Autolock l(mLock); + + // 1. Stop repeating request, wait until request thread submitted all requests, then + // call HAL switchToOffline + hardware::camera::device::V3_6::CameraOfflineSessionInfo offlineSessionInfo; + sp offlineSession; + + mRequestThread->switchToOffline(streamsToKeep, &offlineSessionInfo, &offlineSession); + + + // 2. Verify offlineSessionInfo + // 3. create Camera3OfflineSession and transfer object ownership + // (streams, inflight requests, buffer caches) + *session = new Camera3OfflineSession(mId); + + // 4. Delete unneeded streams + // 5. Verify Camera3Device is in unconfigured state + return OK; +} + }; // namespace android diff --git a/services/camera/libcameraservice/device3/Camera3Device.h b/services/camera/libcameraservice/device3/Camera3Device.h index 2573b4889c..eabc44dc5e 100644 --- a/services/camera/libcameraservice/device3/Camera3Device.h +++ b/services/camera/libcameraservice/device3/Camera3Device.h @@ -34,6 +34,7 @@ #include #include #include +#include #include #include #include @@ -196,6 +197,9 @@ class Camera3Device : nsecs_t getExpectedInFlightDuration() override; + status_t switchToOffline(const std::vector& streamsToKeep, + /*out*/ sp* session) override; + /** * Helper functions to map between framework and HIDL values */ @@ -282,7 +286,7 @@ class Camera3Device : public: HalInterface(sp &session, std::shared_ptr queue, - bool useHalBufManager); + bool useHalBufManager, bool supportOfflineProcessing); HalInterface(const HalInterface &other); HalInterface(); @@ -316,6 +320,11 @@ class Camera3Device : bool isReconfigurationRequired(CameraMetadata& oldSessionParams, CameraMetadata& newSessionParams); + status_t switchToOffline( + const std::vector& streamsToKeep, + /*out*/hardware::camera::device::V3_6::CameraOfflineSessionInfo* offlineSessionInfo, + /*out*/sp* offlineSession); + // method to extract buffer's unique ID // return pair of (newlySeenBuffer?, bufferId) std::pair getBufferId(const buffer_handle_t& buf, int streamId); @@ -352,6 +361,8 @@ class Camera3Device : sp mHidlSession_3_4; // Valid if ICameraDeviceSession is @3.5 or newer sp mHidlSession_3_5; + // Valid if ICameraDeviceSession is @3.6 or newer + sp mHidlSession_3_6; std::shared_ptr mRequestMetadataQueue; @@ -424,6 +435,8 @@ class Camera3Device : const bool mUseHalBufManager; bool mIsReconfigurationQuerySupported; + + const bool mSupportOfflineProcessing; }; sp mInterface; @@ -842,6 +855,11 @@ class Camera3Device : void signalPipelineDrain(const std::vector& streamIds); + status_t switchToOffline( + const std::vector& streamsToKeep, + /*out*/hardware::camera::device::V3_6::CameraOfflineSessionInfo* offlineSessionInfo, + /*out*/sp* offlineSession); + protected: virtual bool threadLoop(); @@ -862,6 +880,9 @@ class Camera3Device : static const nsecs_t kRequestTimeout = 50e6; // 50 ms + // TODO: does this need to be adjusted for long exposure requests? + static const nsecs_t kRequestSubmitTimeout = 200e6; // 200 ms + // Used to prepare a batch of requests. struct NextRequest { sp captureRequest; @@ -936,6 +957,7 @@ class Camera3Device : Mutex mRequestLock; Condition mRequestSignal; + Condition mRequestSubmittedSignal; RequestList mRequestQueue; RequestList mRepeatingRequests; // The next batch of requests being prepped for submission to the HAL, no longer @@ -1364,6 +1386,9 @@ class Camera3Device : // Fix up result metadata for monochrome camera. bool mNeedFixupMonochromeTags; status_t fixupMonochromeTags(const CameraMetadata& deviceInfo, CameraMetadata& resultMetadata); + + // Whether HAL supports offline processing capability. + bool mSupportOfflineProcessing = false; }; // class Camera3Device }; // namespace android diff --git a/services/camera/libcameraservice/device3/Camera3OfflineSession.cpp b/services/camera/libcameraservice/device3/Camera3OfflineSession.cpp new file mode 100644 index 0000000000..dc5cbb350c --- /dev/null +++ b/services/camera/libcameraservice/device3/Camera3OfflineSession.cpp @@ -0,0 +1,118 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "Camera3-OffLnSsn" +#define ATRACE_TAG ATRACE_TAG_CAMERA +//#define LOG_NDEBUG 0 +//#define LOG_NNDEBUG 0 // Per-frame verbose logging + +#ifdef LOG_NNDEBUG +#define ALOGVV(...) ALOGV(__VA_ARGS__) +#else +#define ALOGVV(...) ((void)0) +#endif + +#include + +#include + +#include "device3/Camera3OfflineSession.h" +#include "device3/Camera3OutputStream.h" +#include "device3/Camera3InputStream.h" +#include "device3/Camera3SharedOutputStream.h" + +using namespace android::camera3; +using namespace android::hardware::camera; + +namespace android { + +Camera3OfflineSession::Camera3OfflineSession(const String8 &id): + mId(id) +{ + ATRACE_CALL(); + ALOGV("%s: Created offline session for camera %s", __FUNCTION__, mId.string()); +} + +Camera3OfflineSession::~Camera3OfflineSession() +{ + ATRACE_CALL(); + ALOGV("%s: Tearing down offline session for camera id %s", __FUNCTION__, mId.string()); +} + +const String8& Camera3OfflineSession::getId() const { + return mId; +} + +status_t Camera3OfflineSession::initialize( + sp /*hidlSession*/) { + ATRACE_CALL(); + return OK; +} + +status_t Camera3OfflineSession::dump(int /*fd*/) { + ATRACE_CALL(); + return OK; +} + +status_t Camera3OfflineSession::abort() { + ATRACE_CALL(); + return OK; +} + +status_t Camera3OfflineSession::disconnect() { + ATRACE_CALL(); + return OK; +} + +status_t Camera3OfflineSession::waitForNextFrame(nsecs_t /*timeout*/) { + ATRACE_CALL(); + return OK; +} + +status_t Camera3OfflineSession::getNextResult(CaptureResult* /*frame*/) { + ATRACE_CALL(); + return OK; +} + +hardware::Return Camera3OfflineSession::processCaptureResult_3_4( + const hardware::hidl_vec< + hardware::camera::device::V3_4::CaptureResult>& /*results*/) { + return hardware::Void(); +} + +hardware::Return Camera3OfflineSession::processCaptureResult( + const hardware::hidl_vec< + hardware::camera::device::V3_2::CaptureResult>& /*results*/) { + return hardware::Void(); +} + +hardware::Return Camera3OfflineSession::notify( + const hardware::hidl_vec& /*msgs*/) { + return hardware::Void(); +} + +hardware::Return Camera3OfflineSession::requestStreamBuffers( + const hardware::hidl_vec& /*bufReqs*/, + requestStreamBuffers_cb /*_hidl_cb*/) { + return hardware::Void(); +} + +hardware::Return Camera3OfflineSession::returnStreamBuffers( + const hardware::hidl_vec& /*buffers*/) { + return hardware::Void(); +} + +}; // namespace android diff --git a/services/camera/libcameraservice/device3/Camera3OfflineSession.h b/services/camera/libcameraservice/device3/Camera3OfflineSession.h new file mode 100644 index 0000000000..30e8c4f4b7 --- /dev/null +++ b/services/camera/libcameraservice/device3/Camera3OfflineSession.h @@ -0,0 +1,126 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ANDROID_SERVERS_CAMERA3OFFLINESESSION_H +#define ANDROID_SERVERS_CAMERA3OFFLINESESSION_H + +#include +#include + +#include +#include + +#include "common/CameraOfflineSessionBase.h" + +#include "device3/Camera3BufferManager.h" +#include "device3/DistortionMapper.h" +#include "utils/TagMonitor.h" +#include "utils/LatencyHistogram.h" +#include + +namespace android { + +namespace camera3 { + +class Camera3Stream; +class Camera3OutputStreamInterface; +class Camera3StreamInterface; + +} // namespace camera3 + +/** + * Camera3OfflineSession for offline session defined in HIDL ICameraOfflineSession@3.6 or higher + */ +class Camera3OfflineSession : + public CameraOfflineSessionBase, + virtual public hardware::camera::device::V3_5::ICameraDeviceCallback { + + public: + + // initialize by Camera3Device. Camera3Device must send all info in separate argument. + // monitored tags + // mUseHalBufManager + // mUsePartialResult + // mNumPartialResults + explicit Camera3OfflineSession(const String8& id); + + virtual ~Camera3OfflineSession(); + + status_t initialize( + sp hidlSession); + + /** + * CameraOfflineSessionBase interface + */ + const String8& getId() const override; + + status_t disconnect() override; + + status_t dump(int fd) override; + + status_t abort() override; + + // methods for capture result passing + status_t waitForNextFrame(nsecs_t timeout) override; + status_t getNextResult(CaptureResult *frame) override; + + // TODO: methods for notification (error/idle/finished etc) passing + + /** + * End of CameraOfflineSessionBase interface + */ + + /** + * HIDL ICameraDeviceCallback interface + */ + + /** + * Implementation of android::hardware::camera::device::V3_5::ICameraDeviceCallback + */ + + hardware::Return processCaptureResult_3_4( + const hardware::hidl_vec< + hardware::camera::device::V3_4::CaptureResult>& results) override; + hardware::Return processCaptureResult( + const hardware::hidl_vec< + hardware::camera::device::V3_2::CaptureResult>& results) override; + hardware::Return notify( + const hardware::hidl_vec< + hardware::camera::device::V3_2::NotifyMsg>& msgs) override; + + hardware::Return requestStreamBuffers( + const hardware::hidl_vec< + hardware::camera::device::V3_5::BufferRequest>& bufReqs, + requestStreamBuffers_cb _hidl_cb) override; + + hardware::Return returnStreamBuffers( + const hardware::hidl_vec< + hardware::camera::device::V3_2::StreamBuffer>& buffers) override; + + /** + * End of CameraOfflineSessionBase interface + */ + + private: + + // Camera device ID + const String8 mId; + +}; // class Camera3OfflineSession + +}; // namespace android + +#endif