From b18c1afb550ab5bfdfac93f6dd921831edf8dbaf Mon Sep 17 00:00:00 2001 From: Pawin Vongmasa Date: Sat, 11 Apr 2020 05:07:15 -0700 Subject: [PATCH] CCodecBufferChannel: Process output format when registering buffer Test: cts-tradefed run cts-dev -m \ CtsMediaTestCases --compatibility:module-arg \ CtsMediaTestCases:include-annotation:\ android.platform.test.annotations.RequiresDevice Test: cts-tradefed run cts -m \ CtsMediaTestCases -t android.media.cts.AdaptivePlaybackTest Test: cts-tradefed run cts -m \ CtsMediaTestCases -t android.media.cts.DecoderTest Test: cts-tradefed run cts -m \ CtsMediaTestCases -t android.media.cts.MediaCodecTest Bug: 149751672 Change-Id: I7befa892f0a09339126a03dc64e8ec91ae711bdc --- media/codec2/sfplugin/CCodecBufferChannel.cpp | 287 ++++++------------ media/codec2/sfplugin/CCodecBufferChannel.h | 42 --- media/codec2/sfplugin/CCodecBuffers.cpp | 208 ++++++++++++- media/codec2/sfplugin/CCodecBuffers.h | 256 +++++++++++++++- 4 files changed, 542 insertions(+), 251 deletions(-) diff --git a/media/codec2/sfplugin/CCodecBufferChannel.cpp b/media/codec2/sfplugin/CCodecBufferChannel.cpp index 6b389d58fc..e2be9911a2 100644 --- a/media/codec2/sfplugin/CCodecBufferChannel.cpp +++ b/media/codec2/sfplugin/CCodecBufferChannel.cpp @@ -127,97 +127,6 @@ void CCodecBufferChannel::QueueSync::stop() { count->value = -1; } -// CCodecBufferChannel::ReorderStash - -CCodecBufferChannel::ReorderStash::ReorderStash() { - clear(); -} - -void CCodecBufferChannel::ReorderStash::clear() { - mPending.clear(); - mStash.clear(); - mDepth = 0; - mKey = C2Config::ORDINAL; -} - -void CCodecBufferChannel::ReorderStash::flush() { - mPending.clear(); - mStash.clear(); -} - -void CCodecBufferChannel::ReorderStash::setDepth(uint32_t depth) { - mPending.splice(mPending.end(), mStash); - mDepth = depth; -} - -void CCodecBufferChannel::ReorderStash::setKey(C2Config::ordinal_key_t key) { - mPending.splice(mPending.end(), mStash); - mKey = key; -} - -bool CCodecBufferChannel::ReorderStash::pop(Entry *entry) { - if (mPending.empty()) { - return false; - } - entry->buffer = mPending.front().buffer; - entry->timestamp = mPending.front().timestamp; - entry->flags = mPending.front().flags; - entry->ordinal = mPending.front().ordinal; - mPending.pop_front(); - return true; -} - -void CCodecBufferChannel::ReorderStash::emplace( - const std::shared_ptr &buffer, - int64_t timestamp, - int32_t flags, - const C2WorkOrdinalStruct &ordinal) { - bool eos = flags & MediaCodec::BUFFER_FLAG_EOS; - if (!buffer && eos) { - // TRICKY: we may be violating ordering of the stash here. Because we - // don't expect any more emplace() calls after this, the ordering should - // not matter. - mStash.emplace_back(buffer, timestamp, flags, ordinal); - } else { - flags = flags & ~MediaCodec::BUFFER_FLAG_EOS; - auto it = mStash.begin(); - for (; it != mStash.end(); ++it) { - if (less(ordinal, it->ordinal)) { - break; - } - } - mStash.emplace(it, buffer, timestamp, flags, ordinal); - if (eos) { - mStash.back().flags = mStash.back().flags | MediaCodec::BUFFER_FLAG_EOS; - } - } - while (!mStash.empty() && mStash.size() > mDepth) { - mPending.push_back(mStash.front()); - mStash.pop_front(); - } -} - -void CCodecBufferChannel::ReorderStash::defer( - const CCodecBufferChannel::ReorderStash::Entry &entry) { - mPending.push_front(entry); -} - -bool CCodecBufferChannel::ReorderStash::hasPending() const { - return !mPending.empty(); -} - -bool CCodecBufferChannel::ReorderStash::less( - const C2WorkOrdinalStruct &o1, const C2WorkOrdinalStruct &o2) { - switch (mKey) { - case C2Config::ORDINAL: return o1.frameIndex < o2.frameIndex; - case C2Config::TIMESTAMP: return o1.timestamp < o2.timestamp; - case C2Config::CUSTOM: return o1.customOrdinal < o2.customOrdinal; - default: - ALOGD("Unrecognized key; default to timestamp"); - return o1.frameIndex < o2.frameIndex; - } -} - // Input CCodecBufferChannel::Input::Input() : extraBuffers("extra") {} @@ -707,7 +616,7 @@ void CCodecBufferChannel::feedInputBufferIfAvailable() { void CCodecBufferChannel::feedInputBufferIfAvailableInternal() { if (mInputMetEos || - mReorderStash.lock()->hasPending() || + mOutput.lock()->buffers->hasPending() || mPipelineWatcher.lock()->pipelineFull()) { return; } else { @@ -980,17 +889,6 @@ status_t CCodecBufferChannel::start( return UNKNOWN_ERROR; } - { - Mutexed::Locked reorder(mReorderStash); - reorder->clear(); - if (reorderDepth) { - reorder->setDepth(reorderDepth.value); - } - if (reorderKey) { - reorder->setKey(reorderKey.value); - } - } - uint32_t inputDelayValue = inputDelay ? inputDelay.value : 0; uint32_t pipelineDelayValue = pipelineDelay ? pipelineDelay.value : 0; uint32_t outputDelayValue = outputDelay ? outputDelay.value : 0; @@ -1259,6 +1157,13 @@ status_t CCodecBufferChannel::start( } output->buffers->setFormat(outputFormat); + output->buffers->clearStash(); + if (reorderDepth) { + output->buffers->setReorderDepth(reorderDepth.value); + } + if (reorderKey) { + output->buffers->setReorderKey(reorderKey.value); + } // Try to set output surface to created block pool if given. if (outputSurface) { @@ -1426,8 +1331,8 @@ void CCodecBufferChannel::flush(const std::list> &flushe { Mutexed::Locked output(mOutput); output->buffers->flush(flushedWork); + output->buffers->flushStash(); } - mReorderStash.lock()->flush(); mPipelineWatcher.lock()->flush(); } @@ -1463,45 +1368,34 @@ bool CCodecBufferChannel::handleWork( std::unique_ptr work, const sp &outputFormat, const C2StreamInitDataInfo::output *initData) { - if (outputFormat != nullptr) { - Mutexed::Locked output(mOutput); - ALOGD("[%s] onWorkDone: output format changed to %s", - mName, outputFormat->debugString().c_str()); - output->buffers->setFormat(outputFormat); + // Whether the output buffer should be reported to the client or not. + bool notifyClient = false; - AString mediaType; - if (outputFormat->findString(KEY_MIME, &mediaType) - && mediaType == MIMETYPE_AUDIO_RAW) { - int32_t channelCount; - int32_t sampleRate; - if (outputFormat->findInt32(KEY_CHANNEL_COUNT, &channelCount) - && outputFormat->findInt32(KEY_SAMPLE_RATE, &sampleRate)) { - output->buffers->updateSkipCutBuffer(sampleRate, channelCount); - } - } + if (work->result == C2_OK){ + notifyClient = true; + } else if (work->result == C2_NOT_FOUND) { + ALOGD("[%s] flushed work; ignored.", mName); + } else { + // C2_OK and C2_NOT_FOUND are the only results that we accept for processing + // the config update. + ALOGD("[%s] work failed to complete: %d", mName, work->result); + mCCodecCallback->onError(work->result, ACTION_CODE_FATAL); + return false; } - if ((work->input.ordinal.frameIndex - mFirstValidFrameIndex.load()).peek() < 0) { + if ((work->input.ordinal.frameIndex - + mFirstValidFrameIndex.load()).peek() < 0) { // Discard frames from previous generation. ALOGD("[%s] Discard frames from previous generation.", mName); - return false; + notifyClient = false; } if (mInputSurface == nullptr && (work->worklets.size() != 1u || !work->worklets.front() - || !(work->worklets.front()->output.flags & C2FrameData::FLAG_INCOMPLETE))) { - mPipelineWatcher.lock()->onWorkDone(work->input.ordinal.frameIndex.peeku()); - } - - if (work->result == C2_NOT_FOUND) { - ALOGD("[%s] flushed work; ignored.", mName); - return true; - } - - if (work->result != C2_OK) { - ALOGD("[%s] work failed to complete: %d", mName, work->result); - mCCodecCallback->onError(work->result, ACTION_CODE_FATAL); - return false; + || !(work->worklets.front()->output.flags & + C2FrameData::FLAG_INCOMPLETE))) { + mPipelineWatcher.lock()->onWorkDone( + work->input.ordinal.frameIndex.peeku()); } // NOTE: MediaCodec usage supposedly have only one worklet @@ -1537,8 +1431,10 @@ bool CCodecBufferChannel::handleWork( case C2PortReorderBufferDepthTuning::CORE_INDEX: { C2PortReorderBufferDepthTuning::output reorderDepth; if (reorderDepth.updateFrom(*param)) { - bool secure = mComponent->getName().find(".secure") != std::string::npos; - mReorderStash.lock()->setDepth(reorderDepth.value); + bool secure = mComponent->getName().find(".secure") != + std::string::npos; + mOutput.lock()->buffers->setReorderDepth( + reorderDepth.value); ALOGV("[%s] onWorkDone: updated reorder depth to %u", mName, reorderDepth.value); size_t numOutputSlots = mOutput.lock()->numSlots; @@ -1550,17 +1446,19 @@ bool CCodecBufferChannel::handleWork( output->maxDequeueBuffers += numInputSlots; } if (output->surface) { - output->surface->setMaxDequeuedBufferCount(output->maxDequeueBuffers); + output->surface->setMaxDequeuedBufferCount( + output->maxDequeueBuffers); } } else { - ALOGD("[%s] onWorkDone: failed to read reorder depth", mName); + ALOGD("[%s] onWorkDone: failed to read reorder depth", + mName); } break; } case C2PortReorderKeySetting::CORE_INDEX: { C2PortReorderKeySetting::output reorderKey; if (reorderKey.updateFrom(*param)) { - mReorderStash.lock()->setKey(reorderKey.value); + mOutput.lock()->buffers->setReorderKey(reorderKey.value); ALOGV("[%s] onWorkDone: updated reorder key to %u", mName, reorderKey.value); } else { @@ -1575,7 +1473,8 @@ bool CCodecBufferChannel::handleWork( ALOGV("[%s] onWorkDone: updating pipeline delay %u", mName, pipelineDelay.value); newPipelineDelay = pipelineDelay.value; - (void)mPipelineWatcher.lock()->pipelineDelay(pipelineDelay.value); + (void)mPipelineWatcher.lock()->pipelineDelay( + pipelineDelay.value); } } if (param->forInput()) { @@ -1584,7 +1483,8 @@ bool CCodecBufferChannel::handleWork( ALOGV("[%s] onWorkDone: updating input delay %u", mName, inputDelay.value); newInputDelay = inputDelay.value; - (void)mPipelineWatcher.lock()->inputDelay(inputDelay.value); + (void)mPipelineWatcher.lock()->inputDelay( + inputDelay.value); } } if (param->forOutput()) { @@ -1592,8 +1492,10 @@ bool CCodecBufferChannel::handleWork( if (outputDelay.updateFrom(*param)) { ALOGV("[%s] onWorkDone: updating output delay %u", mName, outputDelay.value); - bool secure = mComponent->getName().find(".secure") != std::string::npos; - (void)mPipelineWatcher.lock()->outputDelay(outputDelay.value); + bool secure = mComponent->getName().find(".secure") != + std::string::npos; + (void)mPipelineWatcher.lock()->outputDelay( + outputDelay.value); bool outputBuffersChanged = false; size_t numOutputSlots = 0; @@ -1601,7 +1503,8 @@ bool CCodecBufferChannel::handleWork( { Mutexed::Locked output(mOutput); output->outputDelay = outputDelay.value; - numOutputSlots = outputDelay.value + kSmoothnessFactor; + numOutputSlots = outputDelay.value + + kSmoothnessFactor; if (output->numSlots < numOutputSlots) { output->numSlots = numOutputSlots; if (output->buffers->isArrayMode()) { @@ -1620,7 +1523,7 @@ bool CCodecBufferChannel::handleWork( mCCodecCallback->onOutputBuffersChanged(); } - uint32_t depth = mReorderStash.lock()->depth(); + uint32_t depth = mOutput.lock()->buffers->getReorderDepth(); Mutexed::Locked output(mOutputSurface); output->maxDequeueBuffers = numOutputSlots + depth + kRenderingDepth; if (!secure) { @@ -1664,9 +1567,6 @@ bool CCodecBufferChannel::handleWork( ALOGV("[%s] onWorkDone: output EOS", mName); } - sp outBuffer; - size_t index; - // WORKAROUND: adjust output timestamp based on client input timestamp and codec // input timestamp. Codec output timestamp (in the timestamp field) shall correspond to // the codec input timestamp, but client output timestamp should (reported in timeUs) @@ -1687,8 +1587,18 @@ bool CCodecBufferChannel::handleWork( worklet->output.ordinal.timestamp.peekll(), timestamp.peekll()); + // csd cannot be re-ordered and will always arrive first. if (initData != nullptr) { Mutexed::Locked output(mOutput); + if (outputFormat) { + output->buffers->updateSkipCutBuffer(outputFormat); + output->buffers->setFormat(outputFormat); + } + if (!notifyClient) { + return false; + } + size_t index; + sp outBuffer; if (output->buffers->registerCsd(initData, &index, &outBuffer) == OK) { outBuffer->meta()->setInt64("timeUs", timestamp.peek()); outBuffer->meta()->setInt32("flags", MediaCodec::BUFFER_FLAG_CODECCONFIG); @@ -1704,10 +1614,10 @@ bool CCodecBufferChannel::handleWork( } } - if (!buffer && !flags) { + if (notifyClient && !buffer && !flags) { ALOGV("[%s] onWorkDone: Not reporting output buffer (%lld)", mName, work->input.ordinal.frameIndex.peekull()); - return true; + notifyClient = false; } if (buffer) { @@ -1726,63 +1636,60 @@ bool CCodecBufferChannel::handleWork( } { - Mutexed::Locked reorder(mReorderStash); - reorder->emplace(buffer, timestamp.peek(), flags, worklet->output.ordinal); - if (flags & MediaCodec::BUFFER_FLAG_EOS) { - // Flush reorder stash - reorder->setDepth(0); - } + Mutexed::Locked output(mOutput); + output->buffers->pushToStash( + buffer, + notifyClient, + timestamp.peek(), + flags, + outputFormat, + worklet->output.ordinal); } sendOutputBuffers(); return true; } void CCodecBufferChannel::sendOutputBuffers() { - ReorderStash::Entry entry; - sp outBuffer; + OutputBuffers::BufferAction action; size_t index; + sp outBuffer; + std::shared_ptr c2Buffer; while (true) { - Mutexed::Locked reorder(mReorderStash); - if (!reorder->hasPending()) { + Mutexed::Locked output(mOutput); + action = output->buffers->popFromStashAndRegister( + &c2Buffer, &index, &outBuffer); + switch (action) { + case OutputBuffers::SKIP: + return; + case OutputBuffers::DISCARD: break; - } - if (!reorder->pop(&entry)) { + case OutputBuffers::NOTIFY_CLIENT: + output.unlock(); + mCallback->onOutputBufferAvailable(index, outBuffer); break; - } - - Mutexed::Locked output(mOutput); - status_t err = output->buffers->registerBuffer(entry.buffer, &index, &outBuffer); - if (err != OK) { - bool outputBuffersChanged = false; - if (err != WOULD_BLOCK) { + case OutputBuffers::REALLOCATE: { if (!output->buffers->isArrayMode()) { - output->buffers = output->buffers->toArrayMode(output->numSlots); + output->buffers = + output->buffers->toArrayMode(output->numSlots); } - OutputBuffersArray *array = (OutputBuffersArray *)output->buffers.get(); - array->realloc(entry.buffer); - outputBuffersChanged = true; - } - ALOGV("[%s] sendOutputBuffers: unable to register output buffer", mName); - reorder->defer(entry); - - output.unlock(); - reorder.unlock(); - - if (outputBuffersChanged) { + static_cast(output->buffers.get())-> + realloc(c2Buffer); + output.unlock(); mCCodecCallback->onOutputBuffersChanged(); } return; + case OutputBuffers::RETRY: + ALOGV("[%s] sendOutputBuffers: unable to register output buffer", + mName); + return; + default: + LOG_ALWAYS_FATAL("[%s] sendOutputBuffers: " + "corrupted BufferAction value (%d) " + "returned from popFromStashAndRegister.", + mName, int(action)); + return; } - output.unlock(); - reorder.unlock(); - - outBuffer->meta()->setInt64("timeUs", entry.timestamp); - outBuffer->meta()->setInt32("flags", entry.flags); - ALOGV("[%s] sendOutputBuffers: out buffer index = %zu [%p] => %p + %zu (%lld)", - mName, index, outBuffer.get(), outBuffer->data(), outBuffer->size(), - (long long)entry.timestamp); - mCallback->onOutputBufferAvailable(index, outBuffer); } } diff --git a/media/codec2/sfplugin/CCodecBufferChannel.h b/media/codec2/sfplugin/CCodecBufferChannel.h index 0263211006..da15724bbe 100644 --- a/media/codec2/sfplugin/CCodecBufferChannel.h +++ b/media/codec2/sfplugin/CCodecBufferChannel.h @@ -296,48 +296,6 @@ private: Mutexed mPipelineWatcher; - class ReorderStash { - public: - struct Entry { - inline Entry() : buffer(nullptr), timestamp(0), flags(0), ordinal({0, 0, 0}) {} - inline Entry( - const std::shared_ptr &b, - int64_t t, - int32_t f, - const C2WorkOrdinalStruct &o) - : buffer(b), timestamp(t), flags(f), ordinal(o) {} - std::shared_ptr buffer; - int64_t timestamp; - int32_t flags; - C2WorkOrdinalStruct ordinal; - }; - - ReorderStash(); - - void clear(); - void flush(); - void setDepth(uint32_t depth); - void setKey(C2Config::ordinal_key_t key); - bool pop(Entry *entry); - void emplace( - const std::shared_ptr &buffer, - int64_t timestamp, - int32_t flags, - const C2WorkOrdinalStruct &ordinal); - void defer(const Entry &entry); - bool hasPending() const; - uint32_t depth() const { return mDepth; } - - private: - std::list mPending; - std::list mStash; - uint32_t mDepth; - C2Config::ordinal_key_t mKey; - - bool less(const C2WorkOrdinalStruct &o1, const C2WorkOrdinalStruct &o2); - }; - Mutexed mReorderStash; - std::atomic_bool mInputMetEos; std::once_flag mRenderWarningFlag; diff --git a/media/codec2/sfplugin/CCodecBuffers.cpp b/media/codec2/sfplugin/CCodecBuffers.cpp index d7cc175aea..4ce13aabe9 100644 --- a/media/codec2/sfplugin/CCodecBuffers.cpp +++ b/media/codec2/sfplugin/CCodecBuffers.cpp @@ -21,6 +21,7 @@ #include #include +#include #include #include @@ -149,16 +150,29 @@ void OutputBuffers::updateSkipCutBuffer(int32_t sampleRate, int32_t channelCount setSkipCutBuffer(delay, padding); } +void OutputBuffers::updateSkipCutBuffer( + const sp &format, bool notify) { + AString mediaType; + if (format->findString(KEY_MIME, &mediaType) + && mediaType == MIMETYPE_AUDIO_RAW) { + int32_t channelCount; + int32_t sampleRate; + if (format->findInt32(KEY_CHANNEL_COUNT, &channelCount) + && format->findInt32(KEY_SAMPLE_RATE, &sampleRate)) { + updateSkipCutBuffer(sampleRate, channelCount); + } + } + if (notify) { + mUnreportedFormat = nullptr; + } +} + void OutputBuffers::submit(const sp &buffer) { if (mSkipCutBuffer != nullptr) { mSkipCutBuffer->submit(buffer); } } -void OutputBuffers::transferSkipCutBuffer(const sp &scb) { - mSkipCutBuffer = scb; -} - void OutputBuffers::setSkipCutBuffer(int32_t skip, int32_t cut) { if (mSkipCutBuffer != nullptr) { size_t prevSize = mSkipCutBuffer->size(); @@ -169,6 +183,175 @@ void OutputBuffers::setSkipCutBuffer(int32_t skip, int32_t cut) { mSkipCutBuffer = new SkipCutBuffer(skip, cut, mChannelCount); } +void OutputBuffers::clearStash() { + mPending.clear(); + mReorderStash.clear(); + mDepth = 0; + mKey = C2Config::ORDINAL; + mUnreportedFormat = nullptr; +} + +void OutputBuffers::flushStash() { + for (StashEntry& e : mPending) { + e.notify = false; + } + for (StashEntry& e : mReorderStash) { + e.notify = false; + } +} + +uint32_t OutputBuffers::getReorderDepth() const { + return mDepth; +} + +void OutputBuffers::setReorderDepth(uint32_t depth) { + mPending.splice(mPending.end(), mReorderStash); + mDepth = depth; +} + +void OutputBuffers::setReorderKey(C2Config::ordinal_key_t key) { + mPending.splice(mPending.end(), mReorderStash); + mKey = key; +} + +void OutputBuffers::pushToStash( + const std::shared_ptr& buffer, + bool notify, + int64_t timestamp, + int32_t flags, + const sp& format, + const C2WorkOrdinalStruct& ordinal) { + bool eos = flags & MediaCodec::BUFFER_FLAG_EOS; + if (!buffer && eos) { + // TRICKY: we may be violating ordering of the stash here. Because we + // don't expect any more emplace() calls after this, the ordering should + // not matter. + mReorderStash.emplace_back( + buffer, notify, timestamp, flags, format, ordinal); + } else { + flags = flags & ~MediaCodec::BUFFER_FLAG_EOS; + auto it = mReorderStash.begin(); + for (; it != mReorderStash.end(); ++it) { + if (less(ordinal, it->ordinal)) { + break; + } + } + mReorderStash.emplace(it, + buffer, notify, timestamp, flags, format, ordinal); + if (eos) { + mReorderStash.back().flags = + mReorderStash.back().flags | MediaCodec::BUFFER_FLAG_EOS; + } + } + while (!mReorderStash.empty() && mReorderStash.size() > mDepth) { + mPending.push_back(mReorderStash.front()); + mReorderStash.pop_front(); + } + ALOGV("[%s] %s: pushToStash -- pending size = %zu", mName, __func__, mPending.size()); +} + +OutputBuffers::BufferAction OutputBuffers::popFromStashAndRegister( + std::shared_ptr* c2Buffer, + size_t* index, + sp* outBuffer) { + if (mPending.empty()) { + return SKIP; + } + + // Retrieve the first entry. + StashEntry &entry = mPending.front(); + + *c2Buffer = entry.buffer; + sp outputFormat = entry.format; + + // The output format can be processed without a registered slot. + if (outputFormat) { + ALOGD("[%s] popFromStashAndRegister: output format changed to %s", + mName, outputFormat->debugString().c_str()); + updateSkipCutBuffer(outputFormat, entry.notify); + } + + if (entry.notify) { + if (outputFormat) { + setFormat(outputFormat); + } else if (mUnreportedFormat) { + outputFormat = mUnreportedFormat->dup(); + setFormat(outputFormat); + } + mUnreportedFormat = nullptr; + } else { + if (outputFormat) { + mUnreportedFormat = outputFormat; + } else if (!mUnreportedFormat) { + mUnreportedFormat = mFormat; + } + } + + // Flushing mReorderStash because no other buffers should come after output + // EOS. + if (entry.flags & MediaCodec::BUFFER_FLAG_EOS) { + // Flush reorder stash + setReorderDepth(0); + } + + if (!entry.notify) { + mPending.pop_front(); + return DISCARD; + } + + // Try to register the buffer. + status_t err = registerBuffer(*c2Buffer, index, outBuffer); + if (err != OK) { + if (err != WOULD_BLOCK) { + return REALLOCATE; + } + return RETRY; + } + + // Append information from the front stash entry to outBuffer. + (*outBuffer)->meta()->setInt64("timeUs", entry.timestamp); + (*outBuffer)->meta()->setInt32("flags", entry.flags); + ALOGV("[%s] popFromStashAndRegister: " + "out buffer index = %zu [%p] => %p + %zu (%lld)", + mName, *index, outBuffer->get(), + (*outBuffer)->data(), (*outBuffer)->size(), + (long long)entry.timestamp); + + // The front entry of mPending will be removed now that the registration + // succeeded. + mPending.pop_front(); + return NOTIFY_CLIENT; +} + +bool OutputBuffers::popPending(StashEntry *entry) { + if (mPending.empty()) { + return false; + } + *entry = mPending.front(); + mPending.pop_front(); + return true; +} + +void OutputBuffers::deferPending(const OutputBuffers::StashEntry &entry) { + mPending.push_front(entry); +} + +bool OutputBuffers::hasPending() const { + return !mPending.empty(); +} + +bool OutputBuffers::less( + const C2WorkOrdinalStruct &o1, const C2WorkOrdinalStruct &o2) const { + switch (mKey) { + case C2Config::ORDINAL: return o1.frameIndex < o2.frameIndex; + case C2Config::TIMESTAMP: return o1.timestamp < o2.timestamp; + case C2Config::CUSTOM: return o1.customOrdinal < o2.customOrdinal; + default: + ALOGD("Unrecognized key; default to timestamp"); + return o1.frameIndex < o2.frameIndex; + } +} + // LocalBufferPool std::shared_ptr LocalBufferPool::Create(size_t poolCapacity) { @@ -968,6 +1151,16 @@ void OutputBuffersArray::grow(size_t newSize) { mImpl.grow(newSize, mAlloc); } +void OutputBuffersArray::transferFrom(OutputBuffers* source) { + mFormat = source->mFormat; + mSkipCutBuffer = source->mSkipCutBuffer; + mUnreportedFormat = source->mUnreportedFormat; + mPending = std::move(source->mPending); + mReorderStash = std::move(source->mReorderStash); + mDepth = source->mDepth; + mKey = source->mKey; +} + // FlexOutputBuffers status_t FlexOutputBuffers::registerBuffer( @@ -1010,13 +1203,12 @@ void FlexOutputBuffers::flush( // track of the flushed work. } -std::unique_ptr FlexOutputBuffers::toArrayMode(size_t size) { +std::unique_ptr FlexOutputBuffers::toArrayMode(size_t size) { std::unique_ptr array(new OutputBuffersArray(mComponentName.c_str())); - array->setFormat(mFormat); - array->transferSkipCutBuffer(mSkipCutBuffer); + array->transferFrom(this); std::function()> alloc = getAlloc(); array->initialize(mImpl, size, alloc); - return std::move(array); + return array; } size_t FlexOutputBuffers::numClientBuffers() const { diff --git a/media/codec2/sfplugin/CCodecBuffers.h b/media/codec2/sfplugin/CCodecBuffers.h index 85ca5d5ac3..cadc4d8a65 100644 --- a/media/codec2/sfplugin/CCodecBuffers.h +++ b/media/codec2/sfplugin/CCodecBuffers.h @@ -154,6 +154,8 @@ private: DISALLOW_EVIL_CONSTRUCTORS(InputBuffers); }; +class OutputBuffersArray; + class OutputBuffers : public CCodecBuffers { public: OutputBuffers(const char *componentName, const char *name = "Output") @@ -162,8 +164,12 @@ public: /** * Register output C2Buffer from the component and obtain corresponding - * index and MediaCodecBuffer object. Returns false if registration - * fails. + * index and MediaCodecBuffer object. + * + * Returns: + * OK if registration succeeds. + * NO_MEMORY if all buffers are available but not compatible. + * WOULD_BLOCK if there are compatible buffers, but they are all in use. */ virtual status_t registerBuffer( const std::shared_ptr &buffer, @@ -198,7 +204,7 @@ public: * shall retain the internal state so that it will honor index and * buffer from previous calls of registerBuffer(). */ - virtual std::unique_ptr toArrayMode(size_t size) = 0; + virtual std::unique_ptr toArrayMode(size_t size) = 0; /** * Initialize SkipCutBuffer object. @@ -207,24 +213,175 @@ public: int32_t delay, int32_t padding, int32_t sampleRate, int32_t channelCount); /** - * Update the SkipCutBuffer object. No-op if it's never initialized. + * Update SkipCutBuffer from format. The @p format must not be null. + * @p notify determines whether the format comes with a buffer that should + * be reported to the client or not. */ - void updateSkipCutBuffer(int32_t sampleRate, int32_t channelCount); + void updateSkipCutBuffer(const sp &format, bool notify = true); /** - * Submit buffer to SkipCutBuffer object, if initialized. + * Output Stash + * ============ + * + * The output stash is a place to hold output buffers temporarily before + * they are registered to output slots. It has 2 main functions: + * 1. Allow reordering of output frames as the codec may produce frames in a + * different order. + * 2. Act as a "buffer" between the codec and the client because the codec + * may produce more buffers than available slots. This excess of codec's + * output buffers should be registered to slots later, after the client + * has released some slots. + * + * The stash consists of 2 lists of buffers: mPending and mReorderStash. + * mPending is a normal FIFO queue with not size limit, while mReorderStash + * is a sorted list with size limit mDepth. + * + * The normal flow of a non-csd output buffer is as follows: + * + * |----------------OutputBuffers---------------| + * |----------Output stash----------| | + * Codec --|-> mReorderStash --> mPending --|-> slots --|-> client + * | | | + * pushToStash() popFromStashAndRegister() + * + * The buffer that comes from the codec first enters mReorderStash. The + * first buffer in mReorderStash gets moved to mPending when mReorderStash + * overflows. Buffers in mPending are registered to slots and given to the + * client as soon as slots are available. + * + * Every output buffer that is not a csd buffer should be put on the stash + * by calling pushToStash(), then later registered to a slot by calling + * popFromStashAndRegister() before notifying the client with + * onOutputBufferAvailable(). + * + * Reordering + * ========== + * + * mReorderStash is a sorted list with a specified size limit. The size + * limit can be set by calling setReorderDepth(). + * + * Every buffer in mReorderStash has a C2WorkOrdinalStruct, which contains 3 + * members, all of which are comparable. Which member of C2WorkOrdinalStruct + * should be used for reordering can be chosen by calling setReorderKey(). */ - void submit(const sp &buffer); /** - * Transfer SkipCutBuffer object to the other Buffers object. + * Return the reorder depth---the size of mReorderStash. + */ + uint32_t getReorderDepth() const; + + /** + * Set the reorder depth. + */ + void setReorderDepth(uint32_t depth); + + /** + * Set the type of "key" to use in comparisons. + */ + void setReorderKey(C2Config::ordinal_key_t key); + + /** + * Return whether the output stash has any pending buffers. + */ + bool hasPending() const; + + /** + * Flush the stash and reset the depth and the key to their default values. + */ + void clearStash(); + + /** + * Flush the stash. + */ + void flushStash(); + + /** + * Push a buffer to the reorder stash. + * + * @param buffer C2Buffer object from the returned work. + * @param notify Whether the returned work contains a buffer that should + * be reported to the client. This may be false if the + * caller wants to process the buffer without notifying the + * client. + * @param timestamp Buffer timestamp to report to the client. + * @param flags Buffer flags to report to the client. + * @param format Buffer format to report to the client. + * @param ordinal Ordinal used in reordering. This determines when the + * buffer will be popped from the output stash by + * `popFromStashAndRegister()`. + */ + void pushToStash( + const std::shared_ptr& buffer, + bool notify, + int64_t timestamp, + int32_t flags, + const sp& format, + const C2WorkOrdinalStruct& ordinal); + + enum BufferAction : int { + SKIP, + DISCARD, + NOTIFY_CLIENT, + REALLOCATE, + RETRY, + }; + + /** + * Try to atomically pop the first buffer from the reorder stash and + * register it to an output slot. The function returns a value that + * indicates a recommended course of action for the caller. + * + * If the stash is empty, the function will return `SKIP`. + * + * If the stash is not empty, the function will peek at the first (oldest) + * entry in mPending process the buffer in the entry as follows: + * - If the buffer should not be sent to the client, the function will + * return `DISCARD`. The stash entry will be removed. + * - If the buffer should be sent to the client, the function will attempt + * to register the buffer to a slot. The registration may have 3 outcomes + * corresponding to the following return values: + * - `NOTIFY_CLIENT`: The buffer is successfully registered to a slot. The + * output arguments @p index and @p outBuffer will contain valid values + * that the caller can use to call onOutputBufferAvailable(). The stash + * entry will be removed. + * - `REALLOCATE`: The buffer is not registered because it is not + * compatible with the current slots (which are available). The caller + * should reallocate the OutputBuffers with slots that can fit the + * returned @p c2Buffer. The stash entry will not be removed + * - `RETRY`: All slots are currently occupied by the client. The caller + * should try to call this function again after the client has released + * some slots. + * + * @return What the caller should do afterwards. + * + * @param[out] c2Buffer Underlying C2Buffer associated to the first buffer + * on the stash. This value is guaranteed to be valid + * unless the return value is `SKIP`. + * @param[out] index Slot index. This value is valid only if the return + * value is `NOTIFY_CLIENT`. + * @param[out] outBuffer Registered buffer. This value is valid only if the + * return valu is `NOTIFY_CLIENT`. */ - void transferSkipCutBuffer(const sp &scb); + BufferAction popFromStashAndRegister( + std::shared_ptr* c2Buffer, + size_t* index, + sp* outBuffer); protected: sp mSkipCutBuffer; + /** + * Update the SkipCutBuffer object. No-op if it's never initialized. + */ + void updateSkipCutBuffer(int32_t sampleRate, int32_t channelCount); + + /** + * Submit buffer to SkipCutBuffer object, if initialized. + */ + void submit(const sp &buffer); + private: + // SkipCutBuffer int32_t mDelay; int32_t mPadding; int32_t mSampleRate; @@ -232,7 +389,78 @@ private: void setSkipCutBuffer(int32_t skip, int32_t cut); + // Output stash + + // Output format that has not been made available to the client. + sp mUnreportedFormat; + + // Struct for an entry in the output stash (mPending and mReorderStash) + struct StashEntry { + inline StashEntry() + : buffer(nullptr), + notify(false), + timestamp(0), + flags(0), + format(), + ordinal({0, 0, 0}) {} + inline StashEntry( + const std::shared_ptr &b, + bool n, + int64_t t, + int32_t f, + const sp &fmt, + const C2WorkOrdinalStruct &o) + : buffer(b), + notify(n), + timestamp(t), + flags(f), + format(fmt), + ordinal(o) {} + std::shared_ptr buffer; + bool notify; + int64_t timestamp; + int32_t flags; + sp format; + C2WorkOrdinalStruct ordinal; + }; + + /** + * FIFO queue of stash entries. + */ + std::list mPending; + /** + * Sorted list of stash entries. + */ + std::list mReorderStash; + /** + * Size limit of mReorderStash. + */ + uint32_t mDepth{0}; + /** + * Choice of key to use in ordering of stash entries in mReorderStash. + */ + C2Config::ordinal_key_t mKey{C2Config::ORDINAL}; + + /** + * Return false if mPending is empty; otherwise, pop the first entry from + * mPending and return true. + */ + bool popPending(StashEntry *entry); + + /** + * Push an entry as the first entry of mPending. + */ + void deferPending(const StashEntry &entry); + + /** + * Comparison of C2WorkOrdinalStruct based on mKey. + */ + bool less(const C2WorkOrdinalStruct &o1, + const C2WorkOrdinalStruct &o2) const; + DISALLOW_EVIL_CONSTRUCTORS(OutputBuffers); + + friend OutputBuffersArray; }; /** @@ -772,7 +1000,7 @@ public: bool isArrayMode() const final { return true; } - std::unique_ptr toArrayMode(size_t) final { + std::unique_ptr toArrayMode(size_t) final { return nullptr; } @@ -811,6 +1039,12 @@ public: */ void grow(size_t newSize); + /** + * Transfer the SkipCutBuffer and the output stash from another + * OutputBuffers. + */ + void transferFrom(OutputBuffers* source); + private: BuffersArrayImpl mImpl; std::function()> mAlloc; @@ -839,7 +1073,7 @@ public: void flush( const std::list> &flushedWork) override; - std::unique_ptr toArrayMode(size_t size) override; + std::unique_ptr toArrayMode(size_t size) override; size_t numClientBuffers() const final;