From c5cf9e2c9696fac73b70336deea73f2dfb537689 Mon Sep 17 00:00:00 2001 From: Judy Hsiao Date: Thu, 15 Aug 2019 11:32:02 +0800 Subject: [PATCH] AudioMixer: Expand mono track to multi-channel Extend MONO_HACK to multi-channel output. Mono track will have only one channel output on multi-channel devices. Since ARC++ exposed a 4-channel output device and downmix track in ChromeOS, we need to add this feature to support mono track sample expansion. Add support for both re-sample and non-resample path. For resample path, we need to add MIXTYPE_STEREOEXPAND in AudioMixerOps.h since AudioResampler will upmix mono track to stereo track. Bug: 120222604 Bug: 112341269 Bug: 117116052 Bug: crbug.com/890560 Test: Play mono tracks without re-sampling on ARC++ Test: Play mono tracks with re-sampling on ARC++ Test: Play normal stereo tracks on ARC++ (cherry picked from commit 9b79e0752e6536c31430aa31838a9de1b7b56f9f) Change-Id: I51f5914c41dd0196db9c6a2e1a99b44e5d87c0d6 --- media/libaudioprocessing/AudioMixer.cpp | 4 +- media/libaudioprocessing/AudioMixerBase.cpp | 32 ++++++++-- media/libaudioprocessing/AudioMixerOps.h | 59 +++++++++++++------ .../include/media/AudioMixerBase.h | 1 + 4 files changed, 72 insertions(+), 24 deletions(-) diff --git a/media/libaudioprocessing/AudioMixer.cpp b/media/libaudioprocessing/AudioMixer.cpp index c0b11a41d5..1a3142002b 100644 --- a/media/libaudioprocessing/AudioMixer.cpp +++ b/media/libaudioprocessing/AudioMixer.cpp @@ -162,10 +162,10 @@ status_t AudioMixer::Track::prepareForDownmix() // discard the previous downmixer if there was one unprepareForDownmix(); // MONO_HACK Only remix (upmix or downmix) if the track and mixer/device channel masks - // are not the same and not handled internally, as mono -> stereo currently is. + // are not the same and not handled internally, as mono for channel position masks is. if (channelMask == mMixerChannelMask || (channelMask == AUDIO_CHANNEL_OUT_MONO - && mMixerChannelMask == AUDIO_CHANNEL_OUT_STEREO)) { + && isAudioChannelPositionMask(mMixerChannelMask))) { return NO_ERROR; } // DownmixerBufferProvider is only used for position masks. diff --git a/media/libaudioprocessing/AudioMixerBase.cpp b/media/libaudioprocessing/AudioMixerBase.cpp index a169db9707..64f91fe9ab 100644 --- a/media/libaudioprocessing/AudioMixerBase.cpp +++ b/media/libaudioprocessing/AudioMixerBase.cpp @@ -643,8 +643,14 @@ void AudioMixerBase::process__validate() if (n & NEEDS_RESAMPLE) { all16BitsStereoNoResample = false; resampling = true; - if ((n & NEEDS_CHANNEL_COUNT__MASK) >= NEEDS_CHANNEL_2 - && t->useStereoVolume()) { + if ((n & NEEDS_CHANNEL_COUNT__MASK) == NEEDS_CHANNEL_1 + && t->channelMask == AUDIO_CHANNEL_OUT_MONO // MONO_HACK + && isAudioChannelPositionMask(t->mMixerChannelMask)) { + t->hook = TrackBase::getTrackHook( + TRACKTYPE_RESAMPLEMONO, t->mMixerChannelCount, + t->mMixerInFormat, t->mMixerFormat); + } else if ((n & NEEDS_CHANNEL_COUNT__MASK) >= NEEDS_CHANNEL_2 + && t->useStereoVolume()) { t->hook = TrackBase::getTrackHook( TRACKTYPE_RESAMPLESTEREO, t->mMixerChannelCount, t->mMixerInFormat, t->mMixerFormat); @@ -658,7 +664,7 @@ void AudioMixerBase::process__validate() } else { if ((n & NEEDS_CHANNEL_COUNT__MASK) == NEEDS_CHANNEL_1){ t->hook = TrackBase::getTrackHook( - (t->mMixerChannelMask == AUDIO_CHANNEL_OUT_STEREO // TODO: MONO_HACK + (isAudioChannelPositionMask(t->mMixerChannelMask) // TODO: MONO_HACK && t->channelMask == AUDIO_CHANNEL_OUT_MONO) ? TRACKTYPE_NORESAMPLEMONO : TRACKTYPE_NORESAMPLE, t->mMixerChannelCount, @@ -1494,7 +1500,8 @@ void AudioMixerBase::TrackBase::track__Resample(TO* out, size_t outFrameCount, T ALOGVV("track__Resample\n"); mResampler->setSampleRate(sampleRate); const bool ramp = needsRamp(); - if (ramp || aux != NULL) { + if (MIXTYPE == MIXTYPE_MONOEXPAND || MIXTYPE == MIXTYPE_STEREOEXPAND + || ramp || aux != NULL) { // if ramp: resample with unity gain to temp buffer and scale/mix in 2nd step. // if aux != NULL: resample with unity gain to temp buffer then apply send level. @@ -1629,6 +1636,23 @@ AudioMixerBase::hook_t AudioMixerBase::TrackBase::getTrackHook(int trackType, ui break; } break; + // RESAMPLEMONO needs MIXTYPE_STEREOEXPAND since resampler will upmix mono + // track to stereo track + case TRACKTYPE_RESAMPLEMONO: + switch (mixerInFormat) { + case AUDIO_FORMAT_PCM_FLOAT: + return (AudioMixerBase::hook_t) &TrackBase::track__Resample< + MIXTYPE_STEREOEXPAND, float /*TO*/, float /*TI*/, + TYPE_AUX>; + case AUDIO_FORMAT_PCM_16_BIT: + return (AudioMixerBase::hook_t) &TrackBase::track__Resample< + MIXTYPE_STEREOEXPAND, int32_t /*TO*/, int16_t /*TI*/, + TYPE_AUX>; + default: + LOG_ALWAYS_FATAL("bad mixerInFormat: %#x", mixerInFormat); + break; + } + break; case TRACKTYPE_NORESAMPLEMONO: switch (mixerInFormat) { case AUDIO_FORMAT_PCM_FLOAT: diff --git a/media/libaudioprocessing/AudioMixerOps.h b/media/libaudioprocessing/AudioMixerOps.h index d3a26ad1c8..2748182076 100644 --- a/media/libaudioprocessing/AudioMixerOps.h +++ b/media/libaudioprocessing/AudioMixerOps.h @@ -219,6 +219,7 @@ enum { MIXTYPE_MULTI_SAVEONLY_MONOVOL, MIXTYPE_MULTI_STEREOVOL, MIXTYPE_MULTI_SAVEONLY_STEREOVOL, + MIXTYPE_STEREOEXPAND, }; /* @@ -232,7 +233,8 @@ template 0 && NCHAN <= 8); static_assert(MIXTYPE == MIXTYPE_MULTI_STEREOVOL - || MIXTYPE == MIXTYPE_MULTI_SAVEONLY_STEREOVOL); + || MIXTYPE == MIXTYPE_MULTI_SAVEONLY_STEREOVOL + || MIXTYPE == MIXTYPE_STEREOEXPAND); auto proc = [](auto& a, const auto& b) { if constexpr (MIXTYPE == MIXTYPE_MULTI_STEREOVOL) { a += b; @@ -240,14 +242,22 @@ void stereoVolumeHelper(TO*& out, const TI*& in, const TV *vol, F f) { a = b; } }; + auto inp = [&in]() -> const TI& { + if constexpr (MIXTYPE == MIXTYPE_STEREOEXPAND) { + return *in; + } else { + return *in++; + } + }; + // HALs should only expose the canonical channel masks. - proc(*out++, f(*in++, vol[0])); // front left + proc(*out++, f(inp(), vol[0])); // front left if constexpr (NCHAN == 1) return; - proc(*out++, f(*in++, vol[1])); // front right + proc(*out++, f(inp(), vol[1])); // front right if constexpr (NCHAN == 2) return; if constexpr (NCHAN == 4) { - proc(*out++, f(*in++, vol[0])); // back left - proc(*out++, f(*in++, vol[1])); // back right + proc(*out++, f(inp(), vol[0])); // back left + proc(*out++, f(inp(), vol[1])); // back right return; } @@ -258,25 +268,25 @@ void stereoVolumeHelper(TO*& out, const TI*& in, const TV *vol, F f) { } else { center = (vol[0] >> 1) + (vol[1] >> 1); // rounds to 0. } - proc(*out++, f(*in++, center)); // center (or 2.1 LFE) + proc(*out++, f(inp(), center)); // center (or 2.1 LFE) if constexpr (NCHAN == 3) return; if constexpr (NCHAN == 5) { - proc(*out++, f(*in++, vol[0])); // back left - proc(*out++, f(*in++, vol[1])); // back right + proc(*out++, f(inp(), vol[0])); // back left + proc(*out++, f(inp(), vol[1])); // back right return; } - proc(*out++, f(*in++, center)); // lfe - proc(*out++, f(*in++, vol[0])); // back left - proc(*out++, f(*in++, vol[1])); // back right + proc(*out++, f(inp(), center)); // lfe + proc(*out++, f(inp(), vol[0])); // back left + proc(*out++, f(inp(), vol[1])); // back right if constexpr (NCHAN == 6) return; if constexpr (NCHAN == 7) { - proc(*out++, f(*in++, center)); // back center + proc(*out++, f(inp(), center)); // back center return; } // NCHAN == 8 - proc(*out++, f(*in++, vol[0])); // side left - proc(*out++, f(*in++, vol[1])); // side right + proc(*out++, f(inp(), vol[0])); // side left + proc(*out++, f(inp(), vol[1])); // side right } /* @@ -326,6 +336,11 @@ void stereoVolumeHelper(TO*& out, const TI*& in, const TV *vol, F f) { * MIXTYPE_MULTI_SAVEONLY_STEREOVOL: * Same as MIXTYPE_MULTI_SAVEONLY, but uses only volume[0] and volume[1]. * + * MIXTYPE_STEREOEXPAND: + * Stereo input channel. NCHAN represents number of output channels. + * Expand size 2 array "in" and "vol" to multi-channel output. Note + * that the 2 array is assumed to have replicated L+R. + * */ template ( out, in, vol, [&auxaccum] (auto &a, const auto &b) { return MixMulAux(a, b, &auxaccum); }); + if constexpr (MIXTYPE == MIXTYPE_STEREOEXPAND) in += 2; vol[0] += volinc[0]; vol[1] += volinc[1]; } else /* constexpr */ { @@ -409,10 +426,12 @@ inline void volumeRampMulti(TO* out, size_t frameCount, } vol[0] += volinc[0]; } else if constexpr (MIXTYPE == MIXTYPE_MULTI_STEREOVOL - || MIXTYPE == MIXTYPE_MULTI_SAVEONLY_STEREOVOL) { + || MIXTYPE == MIXTYPE_MULTI_SAVEONLY_STEREOVOL + || MIXTYPE == MIXTYPE_STEREOEXPAND) { stereoVolumeHelper(out, in, vol, [] (auto &a, const auto &b) { return MixMul(a, b); }); + if constexpr (MIXTYPE == MIXTYPE_STEREOEXPAND) in += 2; vol[0] += volinc[0]; vol[1] += volinc[1]; } else /* constexpr */ { @@ -455,11 +474,13 @@ inline void volumeMulti(TO* out, size_t frameCount, *out++ = MixMulAux(*in++, vol[0], &auxaccum); } } else if constexpr (MIXTYPE == MIXTYPE_MULTI_STEREOVOL - || MIXTYPE == MIXTYPE_MULTI_SAVEONLY_STEREOVOL) { + || MIXTYPE == MIXTYPE_MULTI_SAVEONLY_STEREOVOL + || MIXTYPE == MIXTYPE_STEREOEXPAND) { stereoVolumeHelper( out, in, vol, [&auxaccum] (auto &a, const auto &b) { return MixMulAux(a, b, &auxaccum); }); + if constexpr (MIXTYPE == MIXTYPE_STEREOEXPAND) in += 2; } else /* constexpr */ { static_assert(dependent_false, "invalid mixtype"); } @@ -490,10 +511,12 @@ inline void volumeMulti(TO* out, size_t frameCount, *out++ = MixMul(*in++, vol[0]); } } else if constexpr (MIXTYPE == MIXTYPE_MULTI_STEREOVOL - || MIXTYPE == MIXTYPE_MULTI_SAVEONLY_STEREOVOL) { + || MIXTYPE == MIXTYPE_MULTI_SAVEONLY_STEREOVOL + || MIXTYPE == MIXTYPE_STEREOEXPAND) { stereoVolumeHelper(out, in, vol, [] (auto &a, const auto &b) { return MixMul(a, b); }); + if constexpr (MIXTYPE == MIXTYPE_STEREOEXPAND) in += 2; } else /* constexpr */ { static_assert(dependent_false, "invalid mixtype"); } diff --git a/media/libaudioprocessing/include/media/AudioMixerBase.h b/media/libaudioprocessing/include/media/AudioMixerBase.h index e0fb4241bd..cf84b83784 100644 --- a/media/libaudioprocessing/include/media/AudioMixerBase.h +++ b/media/libaudioprocessing/include/media/AudioMixerBase.h @@ -188,6 +188,7 @@ public: enum { TRACKTYPE_NOP, TRACKTYPE_RESAMPLE, + TRACKTYPE_RESAMPLEMONO, TRACKTYPE_RESAMPLESTEREO, TRACKTYPE_NORESAMPLE, TRACKTYPE_NORESAMPLEMONO,