Add the necessary logic to support dynamic depth metadata. Bug: 109735087 Test: Manual using application, Camera CTS Change-Id: Ic4710872dc596bc718270e1c79d4da53fb850875gugelfrei
Emilian Peev
6 years ago
parent
4c6d2b5c41
commit
538c90e79d
@ -0,0 +1,203 @@
|
||||
/*
|
||||
* Copyright (C) 2018 The Android Open Source Project
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#define LOG_TAG "Camera3-CompositeStream"
|
||||
#define ATRACE_TAG ATRACE_TAG_CAMERA
|
||||
//#define LOG_NDEBUG 0
|
||||
|
||||
#include <utils/Log.h>
|
||||
#include <utils/Trace.h>
|
||||
|
||||
#include "common/CameraDeviceBase.h"
|
||||
#include "CameraDeviceClient.h"
|
||||
#include "CompositeStream.h"
|
||||
|
||||
namespace android {
|
||||
namespace camera3 {
|
||||
|
||||
CompositeStream::CompositeStream(wp<CameraDeviceBase> device,
|
||||
wp<hardware::camera2::ICameraDeviceCallbacks> cb) :
|
||||
mDevice(device),
|
||||
mRemoteCallback(cb),
|
||||
mNumPartialResults(1),
|
||||
mErrorState(false) {
|
||||
sp<CameraDeviceBase> cameraDevice = device.promote();
|
||||
if (cameraDevice.get() != nullptr) {
|
||||
CameraMetadata staticInfo = cameraDevice->info();
|
||||
camera_metadata_entry_t entry = staticInfo.find(ANDROID_REQUEST_PARTIAL_RESULT_COUNT);
|
||||
if (entry.count > 0) {
|
||||
mNumPartialResults = entry.data.i32[0];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
status_t CompositeStream::createStream(const std::vector<sp<Surface>>& consumers,
|
||||
bool hasDeferredConsumer, uint32_t width, uint32_t height, int format,
|
||||
camera3_stream_rotation_t rotation, int * id, const String8& physicalCameraId,
|
||||
std::vector<int> * surfaceIds, int streamSetId, bool isShared) {
|
||||
if (hasDeferredConsumer) {
|
||||
ALOGE("%s: Deferred consumers not supported in case of composite streams!",
|
||||
__FUNCTION__);
|
||||
return BAD_VALUE;
|
||||
}
|
||||
|
||||
if (streamSetId != camera3::CAMERA3_STREAM_ID_INVALID) {
|
||||
ALOGE("%s: Surface groups not supported in case of composite streams!",
|
||||
__FUNCTION__);
|
||||
return BAD_VALUE;
|
||||
}
|
||||
|
||||
if (isShared) {
|
||||
ALOGE("%s: Shared surfaces not supported in case of composite streams!",
|
||||
__FUNCTION__);
|
||||
return BAD_VALUE;
|
||||
}
|
||||
|
||||
return createInternalStreams(consumers, hasDeferredConsumer, width, height, format, rotation, id,
|
||||
physicalCameraId, surfaceIds, streamSetId, isShared);
|
||||
}
|
||||
|
||||
status_t CompositeStream::deleteStream() {
|
||||
{
|
||||
Mutex::Autolock l(mMutex);
|
||||
mPendingCaptureResults.clear();
|
||||
mCaptureResults.clear();
|
||||
mFrameNumberMap.clear();
|
||||
mErrorFrameNumbers.clear();
|
||||
}
|
||||
|
||||
return deleteInternalStreams();
|
||||
}
|
||||
|
||||
void CompositeStream::onBufferRequestForFrameNumber(uint64_t frameNumber, int streamId) {
|
||||
Mutex::Autolock l(mMutex);
|
||||
if (!mErrorState && (streamId == getStreamId())) {
|
||||
mPendingCaptureResults.emplace(frameNumber, CameraMetadata());
|
||||
}
|
||||
}
|
||||
|
||||
void CompositeStream::onBufferReleased(const BufferInfo& bufferInfo) {
|
||||
Mutex::Autolock l(mMutex);
|
||||
if (!mErrorState && !bufferInfo.mError) {
|
||||
mFrameNumberMap.emplace(bufferInfo.mFrameNumber, bufferInfo.mTimestamp);
|
||||
mInputReadyCondition.signal();
|
||||
}
|
||||
}
|
||||
|
||||
void CompositeStream::eraseResult(int64_t frameNumber) {
|
||||
Mutex::Autolock l(mMutex);
|
||||
|
||||
auto it = mPendingCaptureResults.find(frameNumber);
|
||||
if (it == mPendingCaptureResults.end()) {
|
||||
return;
|
||||
}
|
||||
|
||||
it = mPendingCaptureResults.erase(it);
|
||||
}
|
||||
|
||||
void CompositeStream::onResultAvailable(const CaptureResult& result) {
|
||||
bool resultError = false;
|
||||
{
|
||||
Mutex::Autolock l(mMutex);
|
||||
|
||||
uint64_t frameNumber = result.mResultExtras.frameNumber;
|
||||
bool resultReady = false;
|
||||
auto it = mPendingCaptureResults.find(frameNumber);
|
||||
if (it != mPendingCaptureResults.end()) {
|
||||
it->second.append(result.mMetadata);
|
||||
if (result.mResultExtras.partialResultCount >= mNumPartialResults) {
|
||||
auto entry = it->second.find(ANDROID_SENSOR_TIMESTAMP);
|
||||
if (entry.count == 1) {
|
||||
auto ts = entry.data.i64[0];
|
||||
mCaptureResults.emplace(ts, std::make_tuple(frameNumber, it->second));
|
||||
resultReady = true;
|
||||
} else {
|
||||
ALOGE("%s: Timestamp metadata entry missing for frameNumber: %" PRIu64,
|
||||
__FUNCTION__, frameNumber);
|
||||
resultError = true;
|
||||
}
|
||||
mPendingCaptureResults.erase(it);
|
||||
}
|
||||
}
|
||||
|
||||
if (resultReady) {
|
||||
mInputReadyCondition.signal();
|
||||
}
|
||||
}
|
||||
|
||||
if (resultError) {
|
||||
onResultError(result.mResultExtras);
|
||||
}
|
||||
}
|
||||
|
||||
void CompositeStream::flagAnErrorFrameNumber(int64_t frameNumber) {
|
||||
Mutex::Autolock l(mMutex);
|
||||
mErrorFrameNumbers.emplace(frameNumber);
|
||||
mInputReadyCondition.signal();
|
||||
}
|
||||
|
||||
status_t CompositeStream::registerCompositeStreamListener(int32_t streamId) {
|
||||
sp<CameraDeviceBase> device = mDevice.promote();
|
||||
if (device.get() == nullptr) {
|
||||
return NO_INIT;
|
||||
}
|
||||
|
||||
auto ret = device->addBufferListenerForStream(streamId, this);
|
||||
if (ret != OK) {
|
||||
ALOGE("%s: Failed to register composite stream listener!", __FUNCTION__);
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
bool CompositeStream::onError(int32_t errorCode, const CaptureResultExtras& resultExtras) {
|
||||
auto ret = false;
|
||||
switch (errorCode) {
|
||||
case hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_RESULT:
|
||||
onResultError(resultExtras);
|
||||
break;
|
||||
case hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_BUFFER:
|
||||
ret = onStreamBufferError(resultExtras);
|
||||
break;
|
||||
case hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_REQUEST:
|
||||
// Invalid request, this shouldn't affect composite streams.
|
||||
break;
|
||||
default:
|
||||
ALOGE("%s: Unrecoverable error: %d detected!", __FUNCTION__, errorCode);
|
||||
Mutex::Autolock l(mMutex);
|
||||
mErrorState = true;
|
||||
break;
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
void CompositeStream::notifyError(int64_t frameNumber) {
|
||||
sp<hardware::camera2::ICameraDeviceCallbacks> remoteCb =
|
||||
mRemoteCallback.promote();
|
||||
|
||||
if ((frameNumber >= 0) && (remoteCb.get() != nullptr)) {
|
||||
CaptureResultExtras extras;
|
||||
extras.errorStreamId = getStreamId();
|
||||
extras.frameNumber = frameNumber;
|
||||
remoteCb->onDeviceError(
|
||||
hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_BUFFER,
|
||||
extras);
|
||||
}
|
||||
}
|
||||
|
||||
}; // namespace camera3
|
||||
}; // namespace android
|
||||
@ -0,0 +1,120 @@
|
||||
/*
|
||||
* Copyright (C) 2018 The Android Open Source Project
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef ANDROID_SERVERS_CAMERA_CAMERA3_COMPOSITE_STREAM_H
|
||||
#define ANDROID_SERVERS_CAMERA_CAMERA3_COMPOSITE_STREAM_H
|
||||
|
||||
#include <set>
|
||||
#include <unordered_map>
|
||||
|
||||
#include <android/hardware/camera2/ICameraDeviceCallbacks.h>
|
||||
#include <camera/CameraMetadata.h>
|
||||
#include <camera/camera2/OutputConfiguration.h>
|
||||
#include "common/CameraDeviceBase.h"
|
||||
#include "device3/Camera3StreamInterface.h"
|
||||
|
||||
namespace android {
|
||||
|
||||
class CameraDeviceClient;
|
||||
class CameraMetadata;
|
||||
class Surface;
|
||||
|
||||
namespace camera3 {
|
||||
|
||||
class CompositeStream : public camera3::Camera3StreamBufferListener {
|
||||
|
||||
public:
|
||||
CompositeStream(wp<CameraDeviceBase> device, wp<hardware::camera2::ICameraDeviceCallbacks> cb);
|
||||
virtual ~CompositeStream() {}
|
||||
|
||||
status_t createStream(const std::vector<sp<Surface>>& consumers,
|
||||
bool hasDeferredConsumer, uint32_t width, uint32_t height, int format,
|
||||
camera3_stream_rotation_t rotation, int *id, const String8& physicalCameraId,
|
||||
std::vector<int> *surfaceIds, int streamSetId, bool isShared);
|
||||
|
||||
status_t deleteStream();
|
||||
|
||||
// Create and register all internal camera streams.
|
||||
virtual status_t createInternalStreams(const std::vector<sp<Surface>>& consumers,
|
||||
bool hasDeferredConsumer, uint32_t width, uint32_t height, int format,
|
||||
camera3_stream_rotation_t rotation, int *id, const String8& physicalCameraId,
|
||||
std::vector<int> *surfaceIds, int streamSetId, bool isShared) = 0;
|
||||
|
||||
// Release all internal streams and corresponding resources.
|
||||
virtual status_t deleteInternalStreams() = 0;
|
||||
|
||||
// Stream configuration completed.
|
||||
virtual status_t configureStream() = 0;
|
||||
|
||||
// Insert the internal composite stream id in the user capture request.
|
||||
virtual status_t insertGbp(SurfaceMap* /*out*/outSurfaceMap,
|
||||
Vector<int32_t>* /*out*/outputStreamIds, int32_t* /*out*/currentStreamId) = 0;
|
||||
|
||||
// Return composite stream id.
|
||||
virtual int getStreamId() = 0;
|
||||
|
||||
void onResultAvailable(const CaptureResult& result);
|
||||
bool onError(int32_t errorCode, const CaptureResultExtras& resultExtras);
|
||||
|
||||
// Camera3StreamBufferListener implementation
|
||||
void onBufferAcquired(const BufferInfo& /*bufferInfo*/) override { /*Empty for now */ }
|
||||
void onBufferReleased(const BufferInfo& bufferInfo) override;
|
||||
void onBufferRequestForFrameNumber(uint64_t frameNumber, int streamId) override;
|
||||
|
||||
protected:
|
||||
status_t registerCompositeStreamListener(int32_t streamId);
|
||||
void eraseResult(int64_t frameNumber);
|
||||
void flagAnErrorFrameNumber(int64_t frameNumber);
|
||||
void notifyError(int64_t frameNumber);
|
||||
|
||||
// Subclasses should check for buffer errors from internal streams and return 'true' in
|
||||
// case the error notification should remain within camera service.
|
||||
virtual bool onStreamBufferError(const CaptureResultExtras& resultExtras) = 0;
|
||||
|
||||
// Subclasses can decide how to handle result errors depending on whether or not the
|
||||
// internal processing needs result data.
|
||||
virtual void onResultError(const CaptureResultExtras& resultExtras) = 0;
|
||||
|
||||
// Device and/or service is in unrecoverable error state.
|
||||
// Composite streams should behave accordingly.
|
||||
void enableErrorState();
|
||||
|
||||
wp<CameraDeviceBase> mDevice;
|
||||
wp<hardware::camera2::ICameraDeviceCallbacks> mRemoteCallback;
|
||||
|
||||
mutable Mutex mMutex;
|
||||
Condition mInputReadyCondition;
|
||||
int32_t mNumPartialResults;
|
||||
bool mErrorState;
|
||||
|
||||
// Frame number to capture result map of partial pending request results.
|
||||
std::unordered_map<uint64_t, CameraMetadata> mPendingCaptureResults;
|
||||
|
||||
// Timestamp to capture (frame number, result) map of completed pending request results.
|
||||
std::unordered_map<int64_t, std::tuple<int64_t, CameraMetadata>> mCaptureResults;
|
||||
|
||||
// Frame number to timestamp map
|
||||
std::unordered_map<int64_t, int64_t> mFrameNumberMap;
|
||||
|
||||
// Keeps a set buffer/result frame numbers for any errors detected during processing.
|
||||
std::set<int64_t> mErrorFrameNumbers;
|
||||
|
||||
};
|
||||
|
||||
}; //namespace camera3
|
||||
}; //namespace android
|
||||
|
||||
#endif
|
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,148 @@
|
||||
/*
|
||||
* Copyright (C) 2018 The Android Open Source Project
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef ANDROID_SERVERS_CAMERA_CAMERA3_DEPTH_COMPOSITE_STREAM_H
|
||||
#define ANDROID_SERVERS_CAMERA_CAMERA3_DEPTH_COMPOSITE_STREAM_H
|
||||
|
||||
#include <dynamic_depth/imaging_model.h>
|
||||
#include <dynamic_depth/depth_map.h>
|
||||
|
||||
#include <gui/IProducerListener.h>
|
||||
#include <gui/CpuConsumer.h>
|
||||
|
||||
#include "CompositeStream.h"
|
||||
|
||||
using dynamic_depth::DepthMap;
|
||||
using dynamic_depth::Item;
|
||||
using dynamic_depth::ImagingModel;
|
||||
|
||||
namespace android {
|
||||
|
||||
class CameraDeviceClient;
|
||||
class CameraMetadata;
|
||||
class Surface;
|
||||
|
||||
namespace camera3 {
|
||||
|
||||
class DepthCompositeStream : public CompositeStream, public Thread,
|
||||
public CpuConsumer::FrameAvailableListener {
|
||||
|
||||
public:
|
||||
DepthCompositeStream(wp<CameraDeviceBase> device,
|
||||
wp<hardware::camera2::ICameraDeviceCallbacks> cb);
|
||||
~DepthCompositeStream() override;
|
||||
|
||||
static bool isDepthCompositeStream(const sp<Surface> &surface);
|
||||
|
||||
// CompositeStream overrides
|
||||
status_t createInternalStreams(const std::vector<sp<Surface>>& consumers,
|
||||
bool hasDeferredConsumer, uint32_t width, uint32_t height, int format,
|
||||
camera3_stream_rotation_t rotation, int *id, const String8& physicalCameraId,
|
||||
std::vector<int> *surfaceIds, int streamSetId, bool isShared) override;
|
||||
status_t deleteInternalStreams() override;
|
||||
status_t configureStream() override;
|
||||
status_t insertGbp(SurfaceMap* /*out*/outSurfaceMap, Vector<int32_t>* /*out*/outputStreamIds,
|
||||
int32_t* /*out*/currentStreamId) override;
|
||||
int getStreamId() override { return mBlobStreamId; }
|
||||
|
||||
// CpuConsumer listener implementation
|
||||
void onFrameAvailable(const BufferItem& item) override;
|
||||
|
||||
// Return stream information about the internal camera streams
|
||||
static status_t getCompositeStreamInfo(const OutputStreamInfo &streamInfo,
|
||||
const CameraMetadata& ch, std::vector<OutputStreamInfo>* compositeOutput /*out*/);
|
||||
|
||||
protected:
|
||||
|
||||
bool threadLoop() override;
|
||||
bool onStreamBufferError(const CaptureResultExtras& resultExtras) override;
|
||||
void onResultError(const CaptureResultExtras& resultExtras) override;
|
||||
|
||||
private:
|
||||
struct InputFrame {
|
||||
CpuConsumer::LockedBuffer depthBuffer;
|
||||
CpuConsumer::LockedBuffer jpegBuffer;
|
||||
CameraMetadata result;
|
||||
bool error;
|
||||
bool errorNotified;
|
||||
int64_t frameNumber;
|
||||
|
||||
InputFrame() : error(false), errorNotified(false), frameNumber(-1) { }
|
||||
};
|
||||
|
||||
// Helper methods
|
||||
static void getSupportedDepthSizes(const CameraMetadata& ch,
|
||||
std::vector<std::tuple<size_t, size_t>>* depthSizes /*out*/);
|
||||
static status_t getMatchingDepthSize(size_t width, size_t height,
|
||||
const std::vector<std::tuple<size_t, size_t>>& supporedDepthSizes,
|
||||
size_t *depthWidth /*out*/, size_t *depthHeight /*out*/);
|
||||
|
||||
// Dynamic depth processing
|
||||
status_t encodeGrayscaleJpeg(size_t width, size_t height, uint8_t *in, void *out,
|
||||
const size_t maxOutSize, uint8_t jpegQuality, size_t &actualSize);
|
||||
std::unique_ptr<DepthMap> processDepthMapFrame(const CpuConsumer::LockedBuffer &depthMapBuffer,
|
||||
size_t maxJpegSize, uint8_t jpegQuality,
|
||||
std::vector<std::unique_ptr<Item>>* items /*out*/);
|
||||
std::unique_ptr<ImagingModel> getImagingModel();
|
||||
status_t processInputFrame(const InputFrame &inputFrame);
|
||||
|
||||
// Buffer/Results handling
|
||||
void compilePendingInputLocked();
|
||||
void releaseInputFrameLocked(InputFrame *inputFrame /*out*/);
|
||||
void releaseInputFramesLocked(int64_t currentTs);
|
||||
|
||||
// Find first complete and valid frame with smallest timestamp
|
||||
bool getNextReadyInputLocked(int64_t *currentTs /*inout*/);
|
||||
|
||||
// Find next failing frame number with smallest timestamp and return respective frame number
|
||||
int64_t getNextFailingInputLocked(int64_t *currentTs /*inout*/);
|
||||
|
||||
static const nsecs_t kWaitDuration = 10000000; // 10 ms
|
||||
static const auto kDepthMapPixelFormat = HAL_PIXEL_FORMAT_Y16;
|
||||
static const auto kDepthMapDataSpace = HAL_DATASPACE_DEPTH;
|
||||
static const auto kJpegDataSpace = HAL_DATASPACE_V0_JFIF;
|
||||
|
||||
struct ProducerListener : public BnProducerListener {
|
||||
// ProducerListener implementation
|
||||
void onBufferReleased() override { /*No impl. for now*/ };
|
||||
};
|
||||
|
||||
int mBlobStreamId, mBlobSurfaceId, mDepthStreamId, mDepthSurfaceId;
|
||||
size_t mBlobWidth, mBlobHeight;
|
||||
sp<CpuConsumer> mBlobConsumer, mDepthConsumer;
|
||||
bool mDepthBufferAcquired, mBlobBufferAcquired;
|
||||
sp<Surface> mDepthSurface, mBlobSurface, mOutputSurface;
|
||||
sp<ProducerListener> mProducerListener;
|
||||
|
||||
ssize_t mMaxJpegSize;
|
||||
std::vector<std::tuple<size_t, size_t>> mSupportedDepthSizes;
|
||||
std::vector<float> mInstrinsicCalibration, mLensDistortion;
|
||||
bool mIsLogicalCamera;
|
||||
|
||||
// Keep all incoming Depth buffer timestamps pending further processing.
|
||||
std::vector<int64_t> mInputDepthBuffers;
|
||||
|
||||
// Keep all incoming Jpeg/Blob buffer timestamps pending further processing.
|
||||
std::vector<int64_t> mInputJpegBuffers;
|
||||
|
||||
// Map of all input frames pending further processing.
|
||||
std::unordered_map<int64_t, InputFrame> mPendingInputFrames;
|
||||
};
|
||||
|
||||
}; //namespace camera3
|
||||
}; //namespace android
|
||||
|
||||
#endif
|
||||
Loading…
Reference in new issue