gugelfrei
/
android_frameworks_av
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge "Capture HDR thumbnail from surface control"

Browse Source
gugelfrei
TreeHugger Robot 5 years ago committed by Android (Google) Code Review
parent 19dbc30932 34c0cdf9eb
commit 107fd5a178
2 changed files with 204 additions and 21 deletions
Show Stats Download Patch File Download Diff File

200
media/libstagefright/FrameDecoder.cpp
Unescape View File

@ -21,6 +21,7 @@
#include <binder/MemoryBase.h>
#include <binder/MemoryHeapBase.h>
#include <gui/Surface.h>
#include <gui/SurfaceComposerClient.h>
#include <inttypes.h>
#include <media/ICrypto.h>
#include <media/IMediaSource.h>
@ -28,6 +29,7 @@
#include <media/stagefright/foundation/avc_utils.h>
#include <media/stagefright/foundation/ADebug.h>
#include <media/stagefright/foundation/AMessage.h>
#include <media/stagefright/foundation/ColorUtils.h>
#include <media/stagefright/ColorConverter.h>
#include <media/stagefright/MediaBuffer.h>
#include <media/stagefright/MediaCodec.h>
@ -44,7 +46,7 @@ static const size_t kRetryCount = 50; // must be >0
sp<IMemory> allocVideoFrame(const sp<MetaData>& trackMeta,
int32_t width, int32_t height, int32_t tileWidth, int32_t tileHeight,
int32_t dstBpp, bool metaOnly = false) {
int32_t dstBpp, bool allocRotated, bool metaOnly) {
int32_t rotationAngle;
if (!trackMeta->findInt32(kKeyRotation, &rotationAngle)) {
rotationAngle = 0; // By default, no rotation
@ -74,6 +76,14 @@ sp<IMemory> allocVideoFrame(const sp<MetaData>& trackMeta,
displayHeight = height;
}
if (allocRotated && (rotationAngle == 90 || rotationAngle == 270)) {
int32_t tmp;
tmp = width; width = height; height = tmp;
tmp = displayWidth; displayWidth = displayHeight; displayHeight = tmp;
tmp = tileWidth; tileWidth = tileHeight; tileHeight = tmp;
rotationAngle = 0;
}
VideoFrame frame(width, height, displayWidth, displayHeight,
tileWidth, tileHeight, rotationAngle, dstBpp, !metaOnly, iccSize);
@ -94,6 +104,20 @@ sp<IMemory> allocVideoFrame(const sp<MetaData>& trackMeta,
return frameMem;
}
sp<IMemory> allocVideoFrame(const sp<MetaData>& trackMeta,
int32_t width, int32_t height, int32_t tileWidth, int32_t tileHeight,
int32_t dstBpp, bool allocRotated = false) {
return allocVideoFrame(trackMeta, width, height, tileWidth, tileHeight, dstBpp,
allocRotated, false /*metaOnly*/);
}
sp<IMemory> allocMetaFrame(const sp<MetaData>& trackMeta,
int32_t width, int32_t height, int32_t tileWidth, int32_t tileHeight,
int32_t dstBpp) {
return allocVideoFrame(trackMeta, width, height, tileWidth, tileHeight, dstBpp,
false /*allocRotated*/, true /*metaOnly*/);
}
bool findThumbnailInfo(
const sp<MetaData> &trackMeta, int32_t *width, int32_t *height,
uint32_t *type = NULL, const void **data = NULL, size_t *size = NULL) {
@ -117,23 +141,27 @@ bool findGridInfo(const sp<MetaData> &trackMeta,
bool getDstColorFormat(
android_pixel_format_t colorFormat,
OMX_COLOR_FORMATTYPE *dstFormat,
ui::PixelFormat *captureFormat,
int32_t *dstBpp) {
switch (colorFormat) {
case HAL_PIXEL_FORMAT_RGB_565:
{
*dstFormat = OMX_COLOR_Format16bitRGB565;
*captureFormat = ui::PixelFormat::RGB_565;
*dstBpp = 2;
return true;
}
case HAL_PIXEL_FORMAT_RGBA_8888:
{
*dstFormat = OMX_COLOR_Format32BitRGBA8888;
*captureFormat = ui::PixelFormat::RGBA_8888;
*dstBpp = 4;
return true;
}
case HAL_PIXEL_FORMAT_BGRA_8888:
{
*dstFormat = OMX_COLOR_Format32bitBGRA8888;
*captureFormat = ui::PixelFormat::BGRA_8888;
*dstBpp = 4;
return true;
}
@ -150,9 +178,10 @@ bool getDstColorFormat(
sp<IMemory> FrameDecoder::getMetadataOnly(
const sp<MetaData> &trackMeta, int colorFormat, bool thumbnail) {
OMX_COLOR_FORMATTYPE dstFormat;
ui::PixelFormat captureFormat;
int32_t dstBpp;
if (!getDstColorFormat(
(android_pixel_format_t)colorFormat, &dstFormat, &dstBpp)) {
if (!getDstColorFormat((android_pixel_format_t)colorFormat,
&dstFormat, &captureFormat, &dstBpp)) {
return NULL;
}
@ -170,8 +199,7 @@ sp<IMemory> FrameDecoder::getMetadataOnly(
tileWidth = tileHeight = 0;
}
}
return allocVideoFrame(trackMeta,
width, height, tileWidth, tileHeight, dstBpp, true /*metaOnly*/);
return allocMetaFrame(trackMeta, width, height, tileWidth, tileHeight, dstBpp);
}
FrameDecoder::FrameDecoder(
@ -194,15 +222,30 @@ FrameDecoder::~FrameDecoder() {
}
}
bool isHDR(const sp<AMessage> &format) {
uint32_t standard, range, transfer;
if (!format->findInt32("color-standard", (int32_t*)&standard)) {
standard = 0;
}
if (!format->findInt32("color-range", (int32_t*)&range)) {
range = 0;
}
if (!format->findInt32("color-transfer", (int32_t*)&transfer)) {
transfer = 0;
}
return standard == ColorUtils::kColorStandardBT2020 &&
transfer == ColorUtils::kColorTransferST2084;
}
status_t FrameDecoder::init(
int64_t frameTimeUs, int option, int colorFormat) {
if (!getDstColorFormat(
(android_pixel_format_t)colorFormat, &mDstFormat, &mDstBpp)) {
if (!getDstColorFormat((android_pixel_format_t)colorFormat,
&mDstFormat, &mCaptureFormat, &mDstBpp)) {
return ERROR_UNSUPPORTED;
}
sp<AMessage> videoFormat = onGetFormatAndSeekOptions(
frameTimeUs, option, &mReadOptions);
frameTimeUs, option, &mReadOptions, &mSurface);
if (videoFormat == NULL) {
ALOGE("video format or seek mode not supported");
return ERROR_UNSUPPORTED;
@ -219,7 +262,7 @@ status_t FrameDecoder::init(
}
err = decoder->configure(
videoFormat, NULL /* surface */, NULL /* crypto */, 0 /* flags */);
videoFormat, mSurface, NULL /* crypto */, 0 /* flags */);
if (err != OK) {
ALOGW("configure returned error %d (%s)", err, asString(err));
decoder->release();
@ -367,8 +410,13 @@ status_t FrameDecoder::extractInternal() {
ALOGE("failed to get output buffer %zu", index);
break;
}
err = onOutputReceived(videoFrameBuffer, mOutputFormat, ptsUs, &done);
mDecoder->releaseOutputBuffer(index);
if (mSurface != nullptr) {
mDecoder->renderOutputBufferAndRelease(index);
err = onOutputReceived(videoFrameBuffer, mOutputFormat, ptsUs, &done);
} else {
err = onOutputReceived(videoFrameBuffer, mOutputFormat, ptsUs, &done);
mDecoder->releaseOutputBuffer(index);
}
} else {
ALOGW("Received error %d (%s) instead of output", err, asString(err));
done = true;
@ -399,7 +447,9 @@ VideoFrameDecoder::VideoFrameDecoder(
}
sp<AMessage> VideoFrameDecoder::onGetFormatAndSeekOptions(
int64_t frameTimeUs, int seekMode, MediaSource::ReadOptions *options) {
int64_t frameTimeUs, int seekMode,
MediaSource::ReadOptions *options,
sp<Surface> *window) {
mSeekMode = static_cast<MediaSource::ReadOptions::SeekMode>(seekMode);
if (mSeekMode < MediaSource::ReadOptions::SEEK_PREVIOUS_SYNC ||
mSeekMode > MediaSource::ReadOptions::SEEK_FRAME_INDEX) {
@ -446,6 +496,16 @@ sp<AMessage> VideoFrameDecoder::onGetFormatAndSeekOptions(
videoFormat->setInt32("android._num-input-buffers", 1);
videoFormat->setInt32("android._num-output-buffers", 1);
}
if (isHDR(videoFormat)) {
*window = initSurfaceControl();
if (*window == NULL) {
ALOGE("Failed to init surface control for HDR, fallback to non-hdr");
} else {
videoFormat->setInt32("color-format", OMX_COLOR_FormatAndroidOpaque);
}
}
return videoFormat;
}
@ -490,13 +550,22 @@ status_t VideoFrameDecoder::onOutputReceived(
int32_t width, height, stride, srcFormat;
if (!outputFormat->findInt32("width", &width) ||
!outputFormat->findInt32("height", &height) ||
!outputFormat->findInt32("stride", &stride) ||
!outputFormat->findInt32("color-format", &srcFormat)) {
ALOGE("format missing dimension or color: %s",
outputFormat->debugString().c_str());
return ERROR_MALFORMED;
}
if (!outputFormat->findInt32("stride", &stride)) {
if (mSurfaceControl == NULL) {
ALOGE("format must have stride for byte buffer mode: %s",
outputFormat->debugString().c_str());
return ERROR_MALFORMED;
}
// for surface output, set stride to width, we don't actually need it.
stride = width;
}
int32_t crop_left, crop_top, crop_right, crop_bottom;
if (!outputFormat->findRect("crop", &crop_left, &crop_top, &crop_right, &crop_bottom)) {
crop_left = crop_top = 0;
@ -511,12 +580,17 @@ status_t VideoFrameDecoder::onOutputReceived(
(crop_bottom - crop_top + 1),
0,
0,
dstBpp());
dstBpp(),
mSurfaceControl != nullptr /*allocRotated*/);
mFrame = static_cast<VideoFrame*>(frameMem->pointer());
setFrame(frameMem);
}
if (mSurfaceControl != nullptr) {
return captureSurfaceControl();
}
ColorConverter converter((OMX_COLOR_FORMATTYPE)srcFormat, dstFormat());
uint32_t standard, range, transfer;
@ -547,6 +621,101 @@ status_t VideoFrameDecoder::onOutputReceived(
return ERROR_UNSUPPORTED;
}
sp<Surface> VideoFrameDecoder::initSurfaceControl() {
sp<SurfaceComposerClient> client = new SurfaceComposerClient();
if (client->initCheck() != NO_ERROR) {
ALOGE("failed to get SurfaceComposerClient");
return NULL;
}
// create a container layer to hold the capture layer, so that we can
// use full frame drop. If without the container, the crop will be set
// to display size.
sp<SurfaceControl> parent = client->createSurface(
String8("parent"),
0 /* width */, 0 /* height */,
PIXEL_FORMAT_RGBA_8888,
ISurfaceComposerClient::eFXSurfaceContainer );
if (!parent) {
ALOGE("failed to get surface control parent");
return NULL;
}
// create the surface with unknown size 1x1 for now, real size will
// be set before the capture when we have output format info.
sp<SurfaceControl> surfaceControl = client->createSurface(
String8("thumbnail"),
1 /* width */, 1 /* height */,
PIXEL_FORMAT_RGBA_8888,
ISurfaceComposerClient::eFXSurfaceBufferQueue,
parent.get());
if (!surfaceControl) {
ALOGE("failed to get surface control");
return NULL;
}
SurfaceComposerClient::Transaction t;
t.hide(parent)
.show(surfaceControl)
.apply(true);
mSurfaceControl = surfaceControl;
mParent = parent;
return surfaceControl->getSurface();
}
status_t VideoFrameDecoder::captureSurfaceControl() {
// set the layer size to the output size before the capture
SurfaceComposerClient::Transaction()
.setSize(mSurfaceControl, mFrame->mWidth, mFrame->mHeight)
.apply(true);
sp<GraphicBuffer> outBuffer;
status_t err = ScreenshotClient::captureChildLayers(
mParent->getHandle(),
ui::Dataspace::V0_SRGB,
captureFormat(),
Rect(0, 0, mFrame->mWidth, mFrame->mHeight),
{},
1.0f /*frameScale*/,
&outBuffer);
if (err != OK) {
ALOGE("failed to captureLayers: err %d", err);
return err;
}
ALOGV("capture: %dx%d, format %d, stride %d",
outBuffer->getWidth(),
outBuffer->getHeight(),
outBuffer->getPixelFormat(),
outBuffer->getStride());
uint8_t *base;
int32_t outBytesPerPixel, outBytesPerStride;
err = outBuffer->lock(
GraphicBuffer::USAGE_SW_READ_OFTEN,
reinterpret_cast<void**>(&base),
&outBytesPerPixel,
&outBytesPerStride);
if (err != OK) {
ALOGE("failed to lock graphic buffer: err %d", err);
return err;
}
uint8_t *dst = mFrame->getFlattenedData();
for (size_t y = 0 ; y < fmin(mFrame->mHeight, outBuffer->getHeight()) ; y++) {
memcpy(dst, base, fmin(mFrame->mWidth, outBuffer->getWidth()) * mFrame->mBytesPerPixel);
dst += mFrame->mRowBytes;
base += outBuffer->getStride() * mFrame->mBytesPerPixel;
}
outBuffer->unlock();
return OK;
}
////////////////////////////////////////////////////////////////////////
ImageDecoder::ImageDecoder(
@ -566,7 +735,8 @@ ImageDecoder::ImageDecoder(
}
sp<AMessage> ImageDecoder::onGetFormatAndSeekOptions(
int64_t frameTimeUs, int /*seekMode*/, MediaSource::ReadOptions *options) {
int64_t frameTimeUs, int /*seekMode*/,
MediaSource::ReadOptions *options, sp<Surface> * /*window*/) {
sp<MetaData> overrideMeta;
if (frameTimeUs < 0) {
uint32_t type;

25
media/libstagefright/include/FrameDecoder.h
Unescape View File

@ -24,15 +24,17 @@
#include <media/stagefright/foundation/ABase.h>
#include <media/MediaSource.h>
#include <media/openmax/OMX_Video.h>
#include <system/graphics-base.h>
#include <ui/GraphicTypes.h>
namespace android {
struct AMessage;
class MediaCodecBuffer;
struct MediaCodec;
class IMediaSource;
class MediaCodecBuffer;
class Surface;
class SurfaceControl;
class VideoFrame;
struct MediaCodec;
struct FrameRect {
int32_t left, top, right, bottom;
@ -57,7 +59,8 @@ protected:
virtual sp<AMessage> onGetFormatAndSeekOptions(
int64_t frameTimeUs,
int seekMode,
MediaSource::ReadOptions *options) = 0;
MediaSource::ReadOptions *options,
sp<Surface> *window) = 0;
virtual status_t onExtractRect(FrameRect *rect) = 0;
@ -75,6 +78,7 @@ protected:
sp<MetaData> trackMeta() const { return mTrackMeta; }
OMX_COLOR_FORMATTYPE dstFormat() const { return mDstFormat; }
ui::PixelFormat captureFormat() const { return mCaptureFormat; }
int32_t dstBpp() const { return mDstBpp; }
void setFrame(const sp<IMemory> &frameMem) { mFrameMemory = frameMem; }
@ -83,6 +87,7 @@ private:
sp<MetaData> mTrackMeta;
sp<IMediaSource> mSource;
OMX_COLOR_FORMATTYPE mDstFormat;
ui::PixelFormat mCaptureFormat;
int32_t mDstBpp;
sp<IMemory> mFrameMemory;
MediaSource::ReadOptions mReadOptions;
@ -90,6 +95,7 @@ private:
sp<AMessage> mOutputFormat;
bool mHaveMoreInputs;
bool mFirstSample;
sp<Surface> mSurface;
status_t extractInternal();
@ -106,7 +112,8 @@ protected:
virtual sp<AMessage> onGetFormatAndSeekOptions(
int64_t frameTimeUs,
int seekMode,
MediaSource::ReadOptions *options) override;
MediaSource::ReadOptions *options,
sp<Surface> *window) override;
virtual status_t onExtractRect(FrameRect *rect) override {
// Rect extraction for sequences is not supported for now.
@ -126,10 +133,15 @@ protected:
bool *done) override;
private:
sp<SurfaceControl> mSurfaceControl;
sp<SurfaceControl> mParent;
VideoFrame *mFrame;
bool mIsAvcOrHevc;
MediaSource::ReadOptions::SeekMode mSeekMode;
int64_t mTargetTimeUs;
sp<Surface> initSurfaceControl();
status_t captureSurfaceControl();
};
struct ImageDecoder : public FrameDecoder {
@ -142,7 +154,8 @@ protected:
virtual sp<AMessage> onGetFormatAndSeekOptions(
int64_t frameTimeUs,
int seekMode,
MediaSource::ReadOptions *options) override;
MediaSource::ReadOptions *options,
sp<Surface> *window) override;
virtual status_t onExtractRect(FrameRect *rect) override;

Write Preview
Loading…
Cancel
Save