From dce8f8cc41cb6be63d13ad0099d61807746ff5d7 Mon Sep 17 00:00:00 2001 From: jiabin Date: Mon, 10 Dec 2018 17:49:31 -0800 Subject: [PATCH] Add buffer provider that can adjust channels. 1. Add adjust channels buffer provider, which will expand or contract sample data from one interleaved channel format to another. Expanded channels are filled with zeros and put at the end of each frames. Contracted channels are omitted from the end of each audio frame. 2. Add adjust channels non destructive buffer provider, which could expand or contract sample data from one interleaved channel format to another while not destructing the buffer. Extra expanded channels are interleaved in from the end of the input buffer. Contracted channels are copied to the end of the output buffer. Contracted channels could be written to an output buffer with certain audio format. Test: Manually Bug: 111454766 Change-Id: I3f963307e73b3f7aa662d4127f78f0c61ac84510 --- .../libaudioclient/include/media/AudioMixer.h | 33 ++++++- media/libaudioprocessing/AudioMixer.cpp | 93 +++++++++++++++++++ media/libaudioprocessing/BufferProviders.cpp | 79 ++++++++++++++++ .../libmedia/include/media/BufferProviders.h | 47 ++++++++++ 4 files changed, 248 insertions(+), 4 deletions(-) diff --git a/media/libaudioclient/include/media/AudioMixer.h b/media/libaudioclient/include/media/AudioMixer.h index aa036a8e1f..0532232073 100644 --- a/media/libaudioclient/include/media/AudioMixer.h +++ b/media/libaudioclient/include/media/AudioMixer.h @@ -137,6 +137,13 @@ public: void setBufferProvider(int name, AudioBufferProvider* bufferProvider); void process() { + for (const auto &pair : mTracks) { + // Clear contracted buffer before processing if contracted channels are saved + const std::shared_ptr &t = pair.second; + if (t->mKeepContractedChannels) { + t->clearContractedBuffer(); + } + } (this->*mHook)(); } @@ -235,6 +242,8 @@ private: mPostDownmixReformatBufferProvider.reset(nullptr); mDownmixerBufferProvider.reset(nullptr); mReformatBufferProvider.reset(nullptr); + mAdjustChannelsNonDestructiveBufferProvider.reset(nullptr); + mAdjustChannelsBufferProvider.reset(nullptr); } bool needsRamp() { return (volumeInc[0] | volumeInc[1] | auxInc) != 0; } @@ -249,6 +258,11 @@ private: void unprepareForDownmix(); status_t prepareForReformat(); void unprepareForReformat(); + status_t prepareForAdjustChannels(); + void unprepareForAdjustChannels(); + status_t prepareForAdjustChannelsNonDestructive(size_t frames); + void unprepareForAdjustChannelsNonDestructive(); + void clearContractedBuffer(); bool setPlaybackRate(const AudioPlaybackRate &playbackRate); void reconfigureBufferProviders(); @@ -302,17 +316,21 @@ private: * all pre-mixer track buffer conversions outside the AudioMixer class. * * 1) mInputBufferProvider: The AudioTrack buffer provider. - * 2) mReformatBufferProvider: If not NULL, performs the audio reformat to + * 2) mAdjustChannelsBufferProvider: Expend or contracts data + * 3) mAdjustChannelsNonDestructiveBufferProvider: Non-destructively adjust sample data + * 4) mReformatBufferProvider: If not NULL, performs the audio reformat to * match either mMixerInFormat or mDownmixRequiresFormat, if the downmixer * requires reformat. For example, it may convert floating point input to * PCM_16_bit if that's required by the downmixer. - * 3) mDownmixerBufferProvider: If not NULL, performs the channel remixing to match + * 5) mDownmixerBufferProvider: If not NULL, performs the channel remixing to match * the number of channels required by the mixer sink. - * 4) mPostDownmixReformatBufferProvider: If not NULL, performs reformatting from + * 6) mPostDownmixReformatBufferProvider: If not NULL, performs reformatting from * the downmixer requirements to the mixer engine input requirements. - * 5) mTimestretchBufferProvider: Adds timestretching for playback rate + * 7) mTimestretchBufferProvider: Adds timestretching for playback rate */ AudioBufferProvider* mInputBufferProvider; // externally provided buffer provider. + std::unique_ptr mAdjustChannelsBufferProvider; + std::unique_ptr mAdjustChannelsNonDestructiveBufferProvider; std::unique_ptr mReformatBufferProvider; std::unique_ptr mDownmixerBufferProvider; std::unique_ptr mPostDownmixReformatBufferProvider; @@ -341,6 +359,13 @@ private: AudioPlaybackRate mPlaybackRate; + // Haptic + uint32_t mAdjustInChannelCount; + uint32_t mAdjustOutChannelCount; + uint32_t mAdjustNonDestructiveInChannelCount; + uint32_t mAdjustNonDestructiveOutChannelCount; + bool mKeepContractedChannels; + private: // hooks void track__genericResample(int32_t* out, size_t numFrames, int32_t* temp, int32_t* aux); diff --git a/media/libaudioprocessing/AudioMixer.cpp b/media/libaudioprocessing/AudioMixer.cpp index af54b21986..365af75c1c 100644 --- a/media/libaudioprocessing/AudioMixer.cpp +++ b/media/libaudioprocessing/AudioMixer.cpp @@ -162,6 +162,12 @@ status_t AudioMixer::create( AUDIO_CHANNEL_REPRESENTATION_POSITION, AUDIO_CHANNEL_OUT_STEREO); t->mMixerChannelCount = audio_channel_count_from_out_mask(t->mMixerChannelMask); t->mPlaybackRate = AUDIO_PLAYBACK_RATE_DEFAULT; + // haptic + t->mAdjustInChannelCount = 0; + t->mAdjustOutChannelCount = 0; + t->mAdjustNonDestructiveInChannelCount = 0; + t->mAdjustNonDestructiveOutChannelCount = 0; + t->mKeepContractedChannels = false; // Check the downmixing (or upmixing) requirements. status_t status = t->prepareForDownmix(); if (status != OK) { @@ -171,6 +177,8 @@ status_t AudioMixer::create( // prepareForDownmix() may change mDownmixRequiresFormat ALOGVV("mMixerFormat:%#x mMixerInFormat:%#x\n", t->mMixerFormat, t->mMixerInFormat); t->prepareForReformat(); + t->prepareForAdjustChannelsNonDestructive(mFrameCount); + t->prepareForAdjustChannels(); mTracks[name] = t; return OK; @@ -212,6 +220,9 @@ bool AudioMixer::setChannelMasks(int name, // do it after downmix since track format may change! track->prepareForReformat(); + track->prepareForAdjustChannelsNonDestructive(mFrameCount); + track->prepareForAdjustChannels(); + if (track->mResampler.get() != nullptr) { // resampler channels may have changed. const uint32_t resetToSampleRate = track->sampleRate; @@ -335,10 +346,82 @@ status_t AudioMixer::Track::prepareForReformat() return NO_ERROR; } +void AudioMixer::Track::unprepareForAdjustChannels() +{ + ALOGV("AUDIOMIXER::unprepareForAdjustChannels"); + if (mAdjustChannelsBufferProvider.get() != nullptr) { + mAdjustChannelsBufferProvider.reset(nullptr); + reconfigureBufferProviders(); + } +} + +status_t AudioMixer::Track::prepareForAdjustChannels() +{ + ALOGV("AudioMixer::prepareForAdjustChannels(%p) with inChannelCount: %u, outChannelCount: %u", + this, mAdjustInChannelCount, mAdjustOutChannelCount); + unprepareForAdjustChannels(); + if (mAdjustInChannelCount != mAdjustOutChannelCount) { + mAdjustChannelsBufferProvider.reset(new AdjustChannelsBufferProvider( + mFormat, mAdjustInChannelCount, mAdjustOutChannelCount, kCopyBufferFrameCount)); + reconfigureBufferProviders(); + } + return NO_ERROR; +} + +void AudioMixer::Track::unprepareForAdjustChannelsNonDestructive() +{ + ALOGV("AUDIOMIXER::unprepareForAdjustChannelsNonDestructive"); + if (mAdjustChannelsNonDestructiveBufferProvider.get() != nullptr) { + mAdjustChannelsNonDestructiveBufferProvider.reset(nullptr); + reconfigureBufferProviders(); + } +} + +status_t AudioMixer::Track::prepareForAdjustChannelsNonDestructive(size_t frames) +{ + ALOGV("AudioMixer::prepareForAdjustChannelsNonDestructive(%p) with inChannelCount: %u, " + "outChannelCount: %u, keepContractedChannels: %d", + this, mAdjustNonDestructiveInChannelCount, mAdjustNonDestructiveOutChannelCount, + mKeepContractedChannels); + unprepareForAdjustChannelsNonDestructive(); + if (mAdjustNonDestructiveInChannelCount != mAdjustNonDestructiveOutChannelCount) { + uint8_t* buffer = mKeepContractedChannels + ? (uint8_t*)mainBuffer + frames * audio_bytes_per_frame( + mMixerChannelCount, mMixerFormat) + : NULL; + mAdjustChannelsNonDestructiveBufferProvider.reset( + new AdjustChannelsNonDestructiveBufferProvider( + mFormat, + mAdjustNonDestructiveInChannelCount, + mAdjustNonDestructiveOutChannelCount, + mKeepContractedChannels ? mMixerFormat : AUDIO_FORMAT_INVALID, + frames, + buffer)); + reconfigureBufferProviders(); + } + return NO_ERROR; +} + +void AudioMixer::Track::clearContractedBuffer() +{ + if (mAdjustChannelsNonDestructiveBufferProvider.get() != nullptr) { + static_cast( + mAdjustChannelsNonDestructiveBufferProvider.get())->clearContractedFrames(); + } +} + void AudioMixer::Track::reconfigureBufferProviders() { // configure from upstream to downstream buffer providers. bufferProvider = mInputBufferProvider; + if (mAdjustChannelsBufferProvider.get() != nullptr) { + mAdjustChannelsBufferProvider->setBufferProvider(bufferProvider); + bufferProvider = mAdjustChannelsBufferProvider.get(); + } + if (mAdjustChannelsNonDestructiveBufferProvider.get() != nullptr) { + mAdjustChannelsNonDestructiveBufferProvider->setBufferProvider(bufferProvider); + bufferProvider = mAdjustChannelsNonDestructiveBufferProvider.get(); + } if (mReformatBufferProvider.get() != nullptr) { mReformatBufferProvider->setBufferProvider(bufferProvider); bufferProvider = mReformatBufferProvider.get(); @@ -542,6 +625,9 @@ void AudioMixer::setParameter(int name, int target, int param, void *value) if (track->mainBuffer != valueBuf) { track->mainBuffer = valueBuf; ALOGV("setParameter(TRACK, MAIN_BUFFER, %p)", valueBuf); + if (track->mKeepContractedChannels) { + track->prepareForAdjustChannelsNonDestructive(mFrameCount); + } invalidate(); } break; @@ -571,6 +657,9 @@ void AudioMixer::setParameter(int name, int target, int param, void *value) if (track->mMixerFormat != format) { track->mMixerFormat = format; ALOGV("setParameter(TRACK, MIXER_FORMAT, %#x)", format); + if (track->mKeepContractedChannels) { + track->prepareForAdjustChannelsNonDestructive(mFrameCount); + } } } break; case MIXER_CHANNEL_MASK: { @@ -823,6 +912,10 @@ void AudioMixer::setBufferProvider(int name, AudioBufferProvider* bufferProvider track->mDownmixerBufferProvider->reset(); } else if (track->mReformatBufferProvider.get() != nullptr) { track->mReformatBufferProvider->reset(); + } else if (track->mAdjustChannelsNonDestructiveBufferProvider.get() != nullptr) { + track->mAdjustChannelsNonDestructiveBufferProvider->reset(); + } else if (track->mAdjustChannelsBufferProvider.get() != nullptr) { + track->mAdjustChannelsBufferProvider->reset(); } track->mInputBufferProvider = bufferProvider; diff --git a/media/libaudioprocessing/BufferProviders.cpp b/media/libaudioprocessing/BufferProviders.cpp index 2d9e1cb16b..8b9ee0b0ab 100644 --- a/media/libaudioprocessing/BufferProviders.cpp +++ b/media/libaudioprocessing/BufferProviders.cpp @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -609,5 +610,83 @@ void TimestretchBufferProvider::processFrames(void *dstBuffer, size_t *dstFrames } } } + +AdjustChannelsBufferProvider::AdjustChannelsBufferProvider(audio_format_t format, + size_t inChannelCount, size_t outChannelCount, size_t frameCount) : + CopyBufferProvider( + audio_bytes_per_frame(inChannelCount, format), + audio_bytes_per_frame(outChannelCount, format), + frameCount), + mFormat(format), + mInChannelCount(inChannelCount), + mOutChannelCount(outChannelCount), + mSampleSizeInBytes(audio_bytes_per_sample(format)) +{ + ALOGV("AdjustBufferProvider(%p)(%#x, %zu, %zu, %zu)", + this, format, inChannelCount, outChannelCount, frameCount); +} + +void AdjustChannelsBufferProvider::copyFrames(void *dst, const void *src, size_t frames) +{ + adjust_channels(src, mInChannelCount, dst, mOutChannelCount, mSampleSizeInBytes, + frames * mInChannelCount * mSampleSizeInBytes); +} + +AdjustChannelsNonDestructiveBufferProvider::AdjustChannelsNonDestructiveBufferProvider( + audio_format_t format, size_t inChannelCount, size_t outChannelCount, + audio_format_t contractedFormat, size_t contractedFrameCount, void* contractedBuffer) : + CopyBufferProvider( + audio_bytes_per_frame(inChannelCount, format), + audio_bytes_per_frame(outChannelCount, format), + 0 /*bufferFrameCount*/), + mFormat(format), + mInChannelCount(inChannelCount), + mOutChannelCount(outChannelCount), + mSampleSizeInBytes(audio_bytes_per_sample(format)), + mContractedChannelCount(inChannelCount - outChannelCount), + mContractedFormat(contractedFormat), + mContractedFrameCount(contractedFrameCount), + mContractedBuffer(contractedBuffer), + mContractedWrittenFrames(0) +{ + ALOGV("AdjustChannelsNonDestructiveBufferProvider(%p)(%#x, %zu, %zu, %#x, %p)", + this, format, inChannelCount, outChannelCount, contractedFormat, contractedBuffer); + if (mContractedFormat != AUDIO_FORMAT_INVALID && mInChannelCount > mOutChannelCount) { + mContractedFrameSize = audio_bytes_per_frame(mContractedChannelCount, mContractedFormat); + } +} + +status_t AdjustChannelsNonDestructiveBufferProvider::getNextBuffer( + AudioBufferProvider::Buffer* pBuffer) +{ + const size_t outFramesLeft = mContractedFrameCount - mContractedWrittenFrames; + if (outFramesLeft < pBuffer->frameCount) { + // Restrict the frame count so that we don't write over the size of the output buffer. + pBuffer->frameCount = outFramesLeft; + } + return CopyBufferProvider::getNextBuffer(pBuffer); +} + +void AdjustChannelsNonDestructiveBufferProvider::copyFrames( + void *dst, const void *src, size_t frames) +{ + adjust_channels_non_destructive(src, mInChannelCount, dst, mOutChannelCount, mSampleSizeInBytes, + frames * mInChannelCount * mSampleSizeInBytes); + if (mContractedFormat != AUDIO_FORMAT_INVALID && mContractedBuffer != NULL + && mInChannelCount > mOutChannelCount) { + const size_t contractedIdx = frames * mOutChannelCount * mSampleSizeInBytes; + memcpy_by_audio_format( + (uint8_t*)mContractedBuffer + mContractedWrittenFrames * mContractedFrameSize, + mContractedFormat, (uint8_t*)dst + contractedIdx, mFormat, + mContractedChannelCount * frames); + mContractedWrittenFrames += frames; + } +} + +void AdjustChannelsNonDestructiveBufferProvider::reset() +{ + mContractedWrittenFrames = 0; + CopyBufferProvider::reset(); +} // ---------------------------------------------------------------------------- } // namespace android diff --git a/media/libmedia/include/media/BufferProviders.h b/media/libmedia/include/media/BufferProviders.h index d6a9cfbb99..38603e3d26 100644 --- a/media/libmedia/include/media/BufferProviders.h +++ b/media/libmedia/include/media/BufferProviders.h @@ -216,6 +216,53 @@ private: bool mAudioPlaybackRateValid; // flag for current parameters validity }; +// AdjustBufferProvider derives from CopyBufferProvider to adjust sample data. +// Expands or contracts sample data from one interleaved channel format to another. +// Expanded channels are filled with zeros and put at the end of each audio frame. +// Contracted channels are omitted from the end of each audio frame. +class AdjustChannelsBufferProvider : public CopyBufferProvider { +public: + AdjustChannelsBufferProvider(audio_format_t format, size_t inChannelCount, + size_t outChannelCount, size_t frameCount); + //Overrides + void copyFrames(void *dst, const void *src, size_t frames) override; + +protected: + const audio_format_t mFormat; + const size_t mInChannelCount; + const size_t mOutChannelCount; + const size_t mSampleSizeInBytes; +}; + +// AdjustChannelsNonDestructiveBufferProvider derives from CopyBufferProvider to adjust sample data. +// Expands or contracts sample data from one interleaved channel format to another. +// Extra expanded channels are interleaved in from the end of the input buffer. +// Contracted channels are copied to the end of the output buffer. +// Contracted channels could be written to output buffer. +class AdjustChannelsNonDestructiveBufferProvider : public CopyBufferProvider { +public: + AdjustChannelsNonDestructiveBufferProvider(audio_format_t format, size_t inChannelCount, + size_t outChannelCount, audio_format_t contractedFormat, size_t contractedFrameCount, + void* contractedBuffer); + //Overrides + status_t getNextBuffer(Buffer* pBuffer) override; + void copyFrames(void *dst, const void *src, size_t frames) override; + void reset() override; + + void clearContractedFrames() { mContractedWrittenFrames = 0; } + +protected: + const audio_format_t mFormat; + const size_t mInChannelCount; + const size_t mOutChannelCount; + const size_t mSampleSizeInBytes; + const size_t mContractedChannelCount; + const audio_format_t mContractedFormat; + const size_t mContractedFrameCount; + void *mContractedBuffer; + size_t mContractedWrittenFrames; + size_t mContractedFrameSize; +}; // ---------------------------------------------------------------------------- } // namespace android