From 4c1ef4b64d113be6ee1106375bd10cdc643e80d8 Mon Sep 17 00:00:00 2001 From: Eric Laurent Date: Tue, 13 Nov 2018 16:46:26 -0800 Subject: [PATCH] audio policy: concurrent capture Implement concurrent capture in audio policy manager: - Attach AudioRecord client to already opened input when possible instead of systematically opening a new input for each client. - Always allow inputs to start even in case of concurrency. - Clients are selectively silenced based on their app state by audio policy service. - In case of concurrency on a given input stream, device and source is chosen based app states and source priority. Bug: 111438757 Test: Manual capture tests with solotester and Camera, Assistant and Duo Test: CTS tests for AudioRecord Change-Id: I302710ff545f67361d9aca89e81de40771ce7fb0 --- media/libaudioclient/AudioSystem.cpp | 4 +- media/libaudioclient/IAudioPolicyService.cpp | 9 +- .../include/media/AudioSystem.h | 3 +- .../include/media/IAudioPolicyService.h | 3 +- services/audioflinger/Threads.cpp | 23 +- services/audiopolicy/AudioPolicyInterface.h | 18 +- services/audiopolicy/common/include/policy.h | 25 -- .../include/AudioInputDescriptor.h | 5 +- .../include/ClientDescriptor.h | 11 +- .../managerdefinitions/include/IOProfile.h | 2 +- .../src/AudioInputDescriptor.cpp | 55 ++--- .../managerdefault/AudioPolicyManager.cpp | 216 ++++-------------- .../managerdefault/AudioPolicyManager.h | 10 +- .../service/AudioPolicyInterfaceImpl.cpp | 107 ++++----- .../service/AudioPolicyService.cpp | 117 +++++++++- .../audiopolicy/service/AudioPolicyService.h | 11 +- 16 files changed, 262 insertions(+), 357 deletions(-) diff --git a/media/libaudioclient/AudioSystem.cpp b/media/libaudioclient/AudioSystem.cpp index ec36ed7dc8..efe65bb299 100644 --- a/media/libaudioclient/AudioSystem.cpp +++ b/media/libaudioclient/AudioSystem.cpp @@ -917,11 +917,11 @@ status_t AudioSystem::getInputForAttr(const audio_attributes_t *attr, config, flags, selectedDeviceId, portId); } -status_t AudioSystem::startInput(audio_port_handle_t portId, bool *silenced) +status_t AudioSystem::startInput(audio_port_handle_t portId) { const sp& aps = AudioSystem::get_audio_policy_service(); if (aps == 0) return PERMISSION_DENIED; - return aps->startInput(portId, silenced); + return aps->startInput(portId); } status_t AudioSystem::stopInput(audio_port_handle_t portId) diff --git a/media/libaudioclient/IAudioPolicyService.cpp b/media/libaudioclient/IAudioPolicyService.cpp index a406658a16..7f2e5e5805 100644 --- a/media/libaudioclient/IAudioPolicyService.cpp +++ b/media/libaudioclient/IAudioPolicyService.cpp @@ -329,16 +329,13 @@ public: return NO_ERROR; } - virtual status_t startInput(audio_port_handle_t portId, - bool *silenced) + virtual status_t startInput(audio_port_handle_t portId) { Parcel data, reply; data.writeInterfaceToken(IAudioPolicyService::getInterfaceDescriptor()); data.writeInt32(portId); - data.writeInt32(*silenced ? 1 : 0); remote()->transact(START_INPUT, data, &reply); status_t status = static_cast (reply.readInt32()); - *silenced = reply.readInt32() == 1; return status; } @@ -1219,10 +1216,8 @@ status_t BnAudioPolicyService::onTransact( case START_INPUT: { CHECK_INTERFACE(IAudioPolicyService, data, reply); audio_port_handle_t portId = static_cast (data.readInt32()); - bool silenced = data.readInt32() == 1; - status_t status = startInput(portId, &silenced); + status_t status = startInput(portId); reply->writeInt32(static_cast (status)); - reply->writeInt32(silenced ? 1 : 0); return NO_ERROR; } break; diff --git a/media/libaudioclient/include/media/AudioSystem.h b/media/libaudioclient/include/media/AudioSystem.h index 76a79c9536..ca1879fc4e 100644 --- a/media/libaudioclient/include/media/AudioSystem.h +++ b/media/libaudioclient/include/media/AudioSystem.h @@ -241,8 +241,7 @@ public: audio_port_handle_t *selectedDeviceId, audio_port_handle_t *portId); - static status_t startInput(audio_port_handle_t portId, - bool *silenced); + static status_t startInput(audio_port_handle_t portId); static status_t stopInput(audio_port_handle_t portId); static void releaseInput(audio_port_handle_t portId); static status_t initStreamVolume(audio_stream_type_t stream, diff --git a/media/libaudioclient/include/media/IAudioPolicyService.h b/media/libaudioclient/include/media/IAudioPolicyService.h index a246df68ea..e5fcfb5c2f 100644 --- a/media/libaudioclient/include/media/IAudioPolicyService.h +++ b/media/libaudioclient/include/media/IAudioPolicyService.h @@ -78,8 +78,7 @@ public: audio_input_flags_t flags, audio_port_handle_t *selectedDeviceId, audio_port_handle_t *portId) = 0; - virtual status_t startInput(audio_port_handle_t portId, - bool *silenced) = 0; + virtual status_t startInput(audio_port_handle_t portId) = 0; virtual status_t stopInput(audio_port_handle_t portId) = 0; virtual void releaseInput(audio_port_handle_t portId) = 0; virtual status_t initStreamVolume(audio_stream_type_t stream, diff --git a/services/audioflinger/Threads.cpp b/services/audioflinger/Threads.cpp index 45d3a065aa..e70e567099 100644 --- a/services/audioflinger/Threads.cpp +++ b/services/audioflinger/Threads.cpp @@ -7367,8 +7367,7 @@ status_t AudioFlinger::RecordThread::start(RecordThread::RecordTrack* recordTrac status_t status = NO_ERROR; if (recordTrack->isExternalTrack()) { mLock.unlock(); - bool silenced; - status = AudioSystem::startInput(recordTrack->portId(), &silenced); + status = AudioSystem::startInput(recordTrack->portId()); mLock.lock(); if (recordTrack->isInvalid()) { recordTrack->clearSyncStartEvent(); @@ -7396,7 +7395,6 @@ status_t AudioFlinger::RecordThread::start(RecordThread::RecordTrack* recordTrac recordTrack->clearSyncStartEvent(); return status; } - recordTrack->setSilenced(silenced); } // Catch up with current buffer indices if thread is already running. // This is what makes a new client discard all buffered data. If the track's mRsmpInFront @@ -8346,11 +8344,10 @@ status_t AudioFlinger::MmapThread::start(const AudioClient& client, return BAD_VALUE; } - bool silenced = false; if (isOutput()) { ret = AudioSystem::startOutput(portId); } else { - ret = AudioSystem::startInput(portId, &silenced); + ret = AudioSystem::startInput(portId); } Mutex::Autolock _l(mLock); @@ -8371,21 +8368,21 @@ status_t AudioFlinger::MmapThread::start(const AudioClient& client, return PERMISSION_DENIED; } + // Given that MmapThread::mAttr is mutable, should a MmapTrack have attributes ? + sp track = new MmapTrack(this, mAttr, mSampleRate, mFormat, mChannelMask, mSessionId, + isOutput(), client.clientUid, client.clientPid, portId); + if (isOutput()) { // force volume update when a new track is added mHalVolFloat = -1.0f; - } else if (!silenced) { - for (const sp &track : mActiveTracks) { - if (track->isSilenced_l() && track->uid() != client.clientUid) - track->invalidate(); + } else if (!track->isSilenced_l()) { + for (const sp &t : mActiveTracks) { + if (t->isSilenced_l() && t->uid() != client.clientUid) + t->invalidate(); } } - // Given that MmapThread::mAttr is mutable, should a MmapTrack have attributes ? - sp track = new MmapTrack(this, mAttr, mSampleRate, mFormat, mChannelMask, mSessionId, - isOutput(), client.clientUid, client.clientPid, portId); - track->setSilenced_l(silenced); mActiveTracks.add(track); sp chain = getEffectChain_l(mSessionId); if (chain != 0) { diff --git a/services/audiopolicy/AudioPolicyInterface.h b/services/audiopolicy/AudioPolicyInterface.h index 3c3a82b38f..3fb505d7b2 100644 --- a/services/audiopolicy/AudioPolicyInterface.h +++ b/services/audiopolicy/AudioPolicyInterface.h @@ -65,20 +65,6 @@ public: API_INPUT_TELEPHONY_RX, // used for capture from telephony RX path } input_type_t; - enum { - API_INPUT_CONCURRENCY_NONE = 0, - API_INPUT_CONCURRENCY_CALL = (1 << 0), // Concurrency with a call - API_INPUT_CONCURRENCY_CAPTURE = (1 << 1), // Concurrency with another capture - API_INPUT_CONCURRENCY_HOTWORD = (1 << 2), // Concurrency with a hotword - API_INPUT_CONCURRENCY_PREEMPT = (1 << 3), // pre-empted someone - // NB: preempt is marked on a successful return, others are on failing calls - API_INPUT_CONCURRENCY_LAST = (1 << 4), - - API_INPUT_CONCURRENCY_ALL = (API_INPUT_CONCURRENCY_LAST - 1), - }; - - typedef uint32_t concurrency_type__mask_t; - public: virtual ~AudioPolicyInterface() {} // @@ -141,9 +127,7 @@ public: input_type_t *inputType, audio_port_handle_t *portId) = 0; // indicates to the audio policy manager that the input starts being used. - virtual status_t startInput(audio_port_handle_t portId, - bool silenced, - concurrency_type__mask_t *concurrency) = 0; + virtual status_t startInput(audio_port_handle_t portId) = 0; // indicates to the audio policy manager that the input stops being used. virtual status_t stopInput(audio_port_handle_t portId) = 0; // releases the input. diff --git a/services/audiopolicy/common/include/policy.h b/services/audiopolicy/common/include/policy.h index 9bd68e1ffa..30b0044440 100644 --- a/services/audiopolicy/common/include/policy.h +++ b/services/audiopolicy/common/include/policy.h @@ -31,14 +31,6 @@ static const audio_format_t gDynamicFormat = AUDIO_FORMAT_DEFAULT; // Do not limit channel count otherwise #define MAX_MIXER_CHANNEL_COUNT FCC_8 -/** - * A device mask for all audio input devices that are considered "virtual" when evaluating - * active inputs in getActiveInputs() - */ -#define APM_AUDIO_IN_DEVICE_VIRTUAL_ALL (AUDIO_DEVICE_IN_REMOTE_SUBMIX|\ - AUDIO_DEVICE_IN_BUS|AUDIO_DEVICE_IN_FM_TUNER) - - /** * A device mask for all audio input and output devices where matching inputs/outputs on device * type alone is not enough: the address must match too @@ -67,23 +59,6 @@ static inline bool is_state_in_call(int state) return (state == AUDIO_MODE_IN_CALL) || (state == AUDIO_MODE_IN_COMMUNICATION); } -/** - * Check if the input device given is considered as a virtual device. - * - * @param[in] device to consider - * - * @return true if the device is a virtual one, false otherwise. - */ -static inline bool is_virtual_input_device(audio_devices_t device) -{ - if ((device & AUDIO_DEVICE_BIT_IN) != 0) { - device &= ~AUDIO_DEVICE_BIT_IN; - if ((popcount(device) == 1) && ((device & ~APM_AUDIO_IN_DEVICE_VIRTUAL_ALL) == 0)) - return true; - } - return false; -} - /** * Check whether the device type is one * where addresses are used to distinguish between one connected device and another diff --git a/services/audiopolicy/common/managerdefinitions/include/AudioInputDescriptor.h b/services/audiopolicy/common/managerdefinitions/include/AudioInputDescriptor.h index 6e4c044bb5..9f8b8c0580 100644 --- a/services/audiopolicy/common/managerdefinitions/include/AudioInputDescriptor.h +++ b/services/audiopolicy/common/managerdefinitions/include/AudioInputDescriptor.h @@ -58,9 +58,8 @@ public: void clearPreemptedSessions(); bool isActive() const { return mGlobalActiveCount > 0; } bool isSourceActive(audio_source_t source) const; - audio_source_t inputSource(bool activeOnly = false) const; + audio_source_t source() const; bool isSoundTrigger() const; - audio_source_t getHighestPrioritySource(bool activeOnly) const; void setClientActive(const sp& client, bool active); int32_t activeCount() { return mGlobalActiveCount; } @@ -121,7 +120,7 @@ public: * Only considers inputs from physical devices (e.g. main mic, headset mic) when * ignoreVirtualInputs is true. */ - Vector > getActiveInputs(bool ignoreVirtualInputs = true); + Vector > getActiveInputs(); audio_devices_t getSupportedDevices(audio_io_handle_t handle) const; diff --git a/services/audiopolicy/common/managerdefinitions/include/ClientDescriptor.h b/services/audiopolicy/common/managerdefinitions/include/ClientDescriptor.h index 030bf4bd31..986d109122 100644 --- a/services/audiopolicy/common/managerdefinitions/include/ClientDescriptor.h +++ b/services/audiopolicy/common/managerdefinitions/include/ClientDescriptor.h @@ -22,6 +22,7 @@ #include #include +#include #include #include #include @@ -106,7 +107,7 @@ public: audio_port_handle_t preferredDeviceId, audio_source_t source, audio_input_flags_t flags, bool isSoundTrigger) : ClientDescriptor(portId, uid, sessionId, attributes, config, preferredDeviceId), - mSource(source), mFlags(flags), mIsSoundTrigger(isSoundTrigger), mSilenced(false) {} + mSource(source), mFlags(flags), mIsSoundTrigger(isSoundTrigger), mAppState(APP_STATE_IDLE) {} ~RecordClientDescriptor() override = default; using ClientDescriptor::dump; @@ -115,14 +116,16 @@ public: audio_source_t source() const { return mSource; } audio_input_flags_t flags() const { return mFlags; } bool isSoundTrigger() const { return mIsSoundTrigger; } - void setSilenced(bool silenced) { mSilenced = silenced; } - bool isSilenced() const { return mSilenced; } + void setAppState(app_state_t appState) { mAppState = appState; } + app_state_t appState() { return mAppState; } + bool isSilenced() const { return mAppState == APP_STATE_IDLE; } private: const audio_source_t mSource; const audio_input_flags_t mFlags; const bool mIsSoundTrigger; - bool mSilenced; + app_state_t mAppState; + }; class SourceClientDescriptor: public TrackClientDescriptor diff --git a/services/audiopolicy/common/managerdefinitions/include/IOProfile.h b/services/audiopolicy/common/managerdefinitions/include/IOProfile.h index eb329597da..8ff823860f 100644 --- a/services/audiopolicy/common/managerdefinitions/include/IOProfile.h +++ b/services/audiopolicy/common/managerdefinitions/include/IOProfile.h @@ -35,7 +35,7 @@ class IOProfile : public AudioPort public: IOProfile(const String8 &name, audio_port_role_t role) : AudioPort(name, AUDIO_PORT_TYPE_MIX, role), - maxOpenCount((role == AUDIO_PORT_ROLE_SOURCE) ? 1 : 0), + maxOpenCount(1), curOpenCount(0), maxActiveCount(1), curActiveCount(0) {} diff --git a/services/audiopolicy/common/managerdefinitions/src/AudioInputDescriptor.cpp b/services/audiopolicy/common/managerdefinitions/src/AudioInputDescriptor.cpp index 1f29874b5c..559274f8c3 100644 --- a/services/audiopolicy/common/managerdefinitions/src/AudioInputDescriptor.cpp +++ b/services/audiopolicy/common/managerdefinitions/src/AudioInputDescriptor.cpp @@ -53,9 +53,32 @@ audio_port_handle_t AudioInputDescriptor::getId() const return mId; } -audio_source_t AudioInputDescriptor::inputSource(bool activeOnly) const +audio_source_t AudioInputDescriptor::source() const { - return getHighestPrioritySource(activeOnly); + audio_source_t source = AUDIO_SOURCE_DEFAULT; + + for (bool activeOnly : { true, false }) { + int32_t topPriority = -1; + app_state_t topState = APP_STATE_IDLE; + for (const auto &client : getClientIterable()) { + if (activeOnly && !client->active()) { + continue; + } + app_state_t curState = client->appState(); + if (curState >= topState) { + int32_t curPriority = source_priority(client->source()); + if (curPriority > topPriority) { + source = client->source(); + topPriority = curPriority; + } + topState = curState; + } + } + if (source != AUDIO_SOURCE_DEFAULT) { + break; + } + } + return source; } void AudioInputDescriptor::toAudioPortConfig(struct audio_port_config *dstConfig, @@ -76,7 +99,7 @@ void AudioInputDescriptor::toAudioPortConfig(struct audio_port_config *dstConfig dstConfig->type = AUDIO_PORT_TYPE_MIX; dstConfig->ext.mix.hw_module = getModuleHandle(); dstConfig->ext.mix.handle = mIoHandle; - dstConfig->ext.mix.usecase.source = inputSource(); + dstConfig->ext.mix.usecase.source = source(); } void AudioInputDescriptor::toAudioPort(struct audio_port *port) const @@ -125,24 +148,6 @@ bool AudioInputDescriptor::isSourceActive(audio_source_t source) const return false; } -audio_source_t AudioInputDescriptor::getHighestPrioritySource(bool activeOnly) const -{ - audio_source_t source = AUDIO_SOURCE_DEFAULT; - int32_t priority = -1; - - for (const auto &client : getClientIterable()) { - if (activeOnly && !client->active() ) { - continue; - } - int32_t curPriority = source_priority(client->source()); - if (curPriority > priority) { - priority = curPriority; - source = client->source(); - } - } - return source; -} - bool AudioInputDescriptor::isSoundTrigger() const { // sound trigger and non sound trigger clients are not mixed on a given input // so check only first client @@ -224,7 +229,7 @@ status_t AudioInputDescriptor::open(const audio_config_t *config, status_t AudioInputDescriptor::start() { - if (mGlobalActiveCount == 1) { + if (!isActive()) { if (!mProfile->canStartNewIo()) { ALOGI("%s mProfile->curActiveCount %d", __func__, mProfile->curActiveCount); return INVALID_OPERATION; @@ -388,15 +393,13 @@ uint32_t AudioInputCollection::activeInputsCountOnDevices(audio_devices_t device return count; } -Vector > AudioInputCollection::getActiveInputs(bool ignoreVirtualInputs) +Vector > AudioInputCollection::getActiveInputs() { Vector > activeInputs; for (size_t i = 0; i < size(); i++) { const sp inputDescriptor = valueAt(i); - if ((inputDescriptor->isActive()) - && (!ignoreVirtualInputs || - !is_virtual_input_device(inputDescriptor->mDevice))) { + if (inputDescriptor->isActive()) { activeInputs.add(inputDescriptor); } } diff --git a/services/audiopolicy/managerdefault/AudioPolicyManager.cpp b/services/audiopolicy/managerdefault/AudioPolicyManager.cpp index 6ec6a767dd..1b088bb477 100644 --- a/services/audiopolicy/managerdefault/AudioPolicyManager.cpp +++ b/services/audiopolicy/managerdefault/AudioPolicyManager.cpp @@ -1876,7 +1876,38 @@ audio_io_handle_t AudioPolicyManager::getInputForDevice(audio_devices_t device, } if (!profile->canOpenNewIo()) { - return AUDIO_IO_HANDLE_NONE; + for (size_t i = 0; i < mInputs.size(); ) { + sp desc = mInputs.valueAt(i); + if (desc->mProfile != profile) { + continue; + } + // if sound trigger, reuse input if used by other sound trigger on same session + // else + // reuse input if active client app is not in IDLE state + // + RecordClientVector clients = desc->clientsList(); + bool doClose = false; + for (const auto& client : clients) { + if (isSoundTrigger != client->isSoundTrigger()) { + continue; + } + if (client->isSoundTrigger()) { + if (session == client->session()) { + return desc->mIoHandle; + } + continue; + } + if (client->active() && client->appState() != APP_STATE_IDLE) { + return desc->mIoHandle; + } + doClose = true; + } + if (doClose) { + closeInput(desc->mIoHandle); + } else { + i++; + } + } } sp inputDesc = new AudioInputDescriptor(profile, mpClientInterface); @@ -1917,55 +1948,8 @@ audio_io_handle_t AudioPolicyManager::getInputForDevice(audio_devices_t device, return input; } -//static -bool AudioPolicyManager::isConcurrentSource(audio_source_t source) -{ - return (source == AUDIO_SOURCE_HOTWORD) || - (source == AUDIO_SOURCE_VOICE_RECOGNITION) || - (source == AUDIO_SOURCE_FM_TUNER); -} - -// FIXME: remove when concurrent capture is ready. This is a hack to work around bug b/63083537. -bool AudioPolicyManager::soundTriggerSupportsConcurrentCapture() { - if (!mHasComputedSoundTriggerSupportsConcurrentCapture) { - bool soundTriggerSupportsConcurrentCapture = false; - unsigned int numModules = 0; - struct sound_trigger_module_descriptor* nModules = NULL; - - status_t status = SoundTrigger::listModules(nModules, &numModules); - if (status == NO_ERROR && numModules != 0) { - nModules = (struct sound_trigger_module_descriptor*) calloc( - numModules, sizeof(struct sound_trigger_module_descriptor)); - if (nModules == NULL) { - // We failed to malloc the buffer, so just say no for now, and hope that we have more - // ram the next time this function is called. - ALOGE("Failed to allocate buffer for module descriptors"); - return false; - } - - status = SoundTrigger::listModules(nModules, &numModules); - if (status == NO_ERROR) { - soundTriggerSupportsConcurrentCapture = true; - for (size_t i = 0; i < numModules; ++i) { - soundTriggerSupportsConcurrentCapture &= - nModules[i].properties.concurrent_capture; - } - } - free(nModules); - } - mSoundTriggerSupportsConcurrentCapture = soundTriggerSupportsConcurrentCapture; - mHasComputedSoundTriggerSupportsConcurrentCapture = true; - } - return mSoundTriggerSupportsConcurrentCapture; -} - - -status_t AudioPolicyManager::startInput(audio_port_handle_t portId, - bool silenced, - concurrency_type__mask_t *concurrency) +status_t AudioPolicyManager::startInput(audio_port_handle_t portId) { - *concurrency = API_INPUT_CONCURRENCY_NONE; - ALOGV("%s portId %d", __FUNCTION__, portId); sp inputDesc = mInputs.getInputForClient(portId); @@ -1982,106 +1966,16 @@ status_t AudioPolicyManager::startInput(audio_port_handle_t portId, audio_session_t session = client->session(); - ALOGV("%s input:%d, session:%d, silenced:%d, concurrency:%d)", - __FUNCTION__, input, session, silenced, *concurrency); + ALOGV("%s input:%d, session:%d)", __FUNCTION__, input, session); - if (!is_virtual_input_device(inputDesc->mDevice)) { - if (mCallTxPatch != 0 && - inputDesc->getModuleHandle() == mCallTxPatch->mPatch.sources[0].ext.device.hw_module) { - ALOGW("startInput(%d) failed: call in progress", input); - *concurrency |= API_INPUT_CONCURRENCY_CALL; - return INVALID_OPERATION; - } - - Vector> activeInputs = mInputs.getActiveInputs(); - - // If a UID is idle and records silence and another not silenced recording starts - // from another UID (idle or active) we stop the current idle UID recording in - // favor of the new one - "There can be only one" TM - if (!silenced) { - for (const auto& activeDesc : activeInputs) { - if ((activeDesc->getAudioPort()->getFlags() & AUDIO_INPUT_FLAG_MMAP_NOIRQ) != 0 && - activeDesc->getId() == inputDesc->getId()) { - continue; - } - - RecordClientVector activeClients = activeDesc->clientsList(true /*activeOnly*/); - for (const auto& activeClient : activeClients) { - if (activeClient->isSilenced()) { - closeClient(activeClient->portId()); - ALOGV("%s client %d stopping silenced client %d", __FUNCTION__, - portId, activeClient->portId()); - activeInputs = mInputs.getActiveInputs(); - } - } - } - } + Vector> activeInputs = mInputs.getActiveInputs(); - for (const auto& activeDesc : activeInputs) { - if ((client->flags() & AUDIO_INPUT_FLAG_MMAP_NOIRQ) != 0 && - activeDesc->getId() == inputDesc->getId()) { - continue; - } - - audio_source_t activeSource = activeDesc->inputSource(true); - if (client->source() == AUDIO_SOURCE_HOTWORD) { - if (activeSource == AUDIO_SOURCE_HOTWORD) { - if (activeDesc->hasPreemptedSession(session)) { - ALOGW("%s input %d failed for HOTWORD: " - "other input %d already started for HOTWORD", __FUNCTION__, - input, activeDesc->mIoHandle); - *concurrency |= API_INPUT_CONCURRENCY_HOTWORD; - return INVALID_OPERATION; - } - } else { - ALOGV("%s input %d failed for HOTWORD: other input %d already started", - __FUNCTION__, input, activeDesc->mIoHandle); - *concurrency |= API_INPUT_CONCURRENCY_CAPTURE; - return INVALID_OPERATION; - } - } else { - if (activeSource != AUDIO_SOURCE_HOTWORD) { - ALOGW("%s input %d failed: other input %d already started", __FUNCTION__, - input, activeDesc->mIoHandle); - *concurrency |= API_INPUT_CONCURRENCY_CAPTURE; - return INVALID_OPERATION; - } - } - } - - // We only need to check if the sound trigger session supports concurrent capture if the - // input is also a sound trigger input. Otherwise, we should preempt any hotword stream - // that's running. - const bool allowConcurrentWithSoundTrigger = - inputDesc->isSoundTrigger() ? soundTriggerSupportsConcurrentCapture() : false; - - // if capture is allowed, preempt currently active HOTWORD captures - for (const auto& activeDesc : activeInputs) { - if (allowConcurrentWithSoundTrigger && activeDesc->isSoundTrigger()) { - continue; - } - RecordClientVector activeHotwordClients = - activeDesc->clientsList(true, AUDIO_SOURCE_HOTWORD); - if (activeHotwordClients.size() > 0) { - SortedVector sessions = activeDesc->getPreemptedSessions(); - - for (const auto& activeClient : activeHotwordClients) { - *concurrency |= API_INPUT_CONCURRENCY_PREEMPT; - sessions.add(activeClient->session()); - closeClient(activeClient->portId()); - ALOGV("%s input %d for HOTWORD preempting HOTWORD input %d", __FUNCTION__, - input, activeDesc->mIoHandle); - } - - inputDesc->setPreemptedSessions(sessions); - } - } + status_t status = inputDesc->start(); + if (status != NO_ERROR) { + return status; } - // Make sure we start with the correct silence state - client->setSilenced(silenced); - - // increment activity count before calling getNewInputDevice() below as only active sessions + // increment activity count before calling getNewInputDevice() below as only active sessions // are considered for device selection inputDesc->setClientActive(client, true); @@ -2090,12 +1984,6 @@ status_t AudioPolicyManager::startInput(audio_port_handle_t portId, audio_devices_t device = getNewInputDevice(inputDesc); setInputDevice(input, device, true /* force */); - status_t status = inputDesc->start(); - if (status != NO_ERROR) { - inputDesc->setClientActive(client, false); - return status; - } - if (inputDesc->activeCount() == 1) { // if input maps to a dynamic policy with an activity listener, notify of state change if ((inputDesc->mPolicyMix != NULL) @@ -3344,7 +3232,7 @@ void AudioPolicyManager::clearSessionRoutes(uid_t uid) SortedVector inputsToClose; for (size_t i = 0; i < mInputs.size(); i++) { sp inputDesc = mInputs.valueAt(i); - if (affectedSources.indexOf(inputDesc->inputSource()) >= 0) { + if (affectedSources.indexOf(inputDesc->source()) >= 0) { inputsToClose.add(inputDesc->mIoHandle); } } @@ -3721,16 +3609,15 @@ status_t AudioPolicyManager::setSurroundFormatEnabled(audio_format_t audioFormat void AudioPolicyManager::setAppState(uid_t uid, app_state_t state) { Vector > activeInputs = mInputs.getActiveInputs(); - bool silenced = state == APP_STATE_IDLE; - ALOGV("AudioPolicyManager:setRecordSilenced(uid:%d, silenced:%d)", uid, silenced); + ALOGV("%s(uid:%d, state:%d)", __func__, uid, state); for (size_t i = 0; i < activeInputs.size(); i++) { sp activeDesc = activeInputs[i]; RecordClientVector clients = activeDesc->clientsList(true /*activeOnly*/); for (const auto& client : clients) { if (uid == client->uid()) { - client->setSilenced(silenced); + client->setAppState(state); } } } @@ -3840,8 +3727,7 @@ AudioPolicyManager::AudioPolicyManager(AudioPolicyClientInterface *clientInterfa mBeaconMuted(false), mTtsOutputAvailable(false), mMasterMono(false), - mMusicEffectOutput(AUDIO_IO_HANDLE_NONE), - mHasComputedSoundTriggerSupportsConcurrentCapture(false) + mMusicEffectOutput(AUDIO_IO_HANDLE_NONE) { } @@ -4894,7 +4780,7 @@ audio_devices_t AudioPolicyManager::getNewInputDevice(const spgetHighestPrioritySource(true /*activeOnly*/); + audio_source_t source = inputDesc->source(); if (source == AUDIO_SOURCE_DEFAULT && isInCall()) { source = AUDIO_SOURCE_VOICE_COMMUNICATION; } @@ -5233,20 +5119,6 @@ uint32_t AudioPolicyManager::setOutputDevice(const sp& ou } installPatch(__func__, patchHandle, outputDesc.get(), patchBuilder.patch(), delayMs); } - - // inform all input as well - for (size_t i = 0; i < mInputs.size(); i++) { - const sp inputDescriptor = mInputs.valueAt(i); - if (!is_virtual_input_device(inputDescriptor->mDevice)) { - AudioParameter inputCmd = AudioParameter(); - ALOGV("%s: inform input %d of device:%d", __func__, - inputDescriptor->mIoHandle, device); - inputCmd.addInt(String8(AudioParameter::keyRouting),device); - mpClientInterface->setParameters(inputDescriptor->mIoHandle, - inputCmd.toString(), - delayMs); - } - } } // update stream volumes according to new device diff --git a/services/audiopolicy/managerdefault/AudioPolicyManager.h b/services/audiopolicy/managerdefault/AudioPolicyManager.h index 0436b1d0f8..22bfab0915 100644 --- a/services/audiopolicy/managerdefault/AudioPolicyManager.h +++ b/services/audiopolicy/managerdefault/AudioPolicyManager.h @@ -134,9 +134,7 @@ public: audio_port_handle_t *portId); // indicates to the audio policy manager that the input starts being used. - virtual status_t startInput(audio_port_handle_t portId, - bool silenced, - concurrency_type__mask_t *concurrency); + virtual status_t startInput(audio_port_handle_t portId); // indicates to the audio policy manager that the input stops being used. virtual status_t stopInput(audio_port_handle_t portId); @@ -539,8 +537,6 @@ protected: void clearAudioSources(uid_t uid); - static bool isConcurrentSource(audio_source_t source); - static bool streamsMatchForvolume(audio_stream_type_t stream1, audio_stream_type_t stream2); @@ -704,10 +700,6 @@ private: int delayMs, uid_t uid, sp *patchDescPtr); - - bool soundTriggerSupportsConcurrentCapture(); - bool mSoundTriggerSupportsConcurrentCapture; - bool mHasComputedSoundTriggerSupportsConcurrentCapture; }; }; diff --git a/services/audiopolicy/service/AudioPolicyInterfaceImpl.cpp b/services/audiopolicy/service/AudioPolicyInterfaceImpl.cpp index 59c8f10075..c2ce754616 100644 --- a/services/audiopolicy/service/AudioPolicyInterfaceImpl.cpp +++ b/services/audiopolicy/service/AudioPolicyInterfaceImpl.cpp @@ -454,23 +454,6 @@ static std::string audioSourceString(audio_source_t value) { return rawbuffer; } -static std::string audioConcurrencyString( - AudioPolicyInterface::concurrency_type__mask_t concurrency) -{ - char buffer[64]; // oversized - if (concurrency & AudioPolicyInterface::API_INPUT_CONCURRENCY_ALL) { - snprintf(buffer, sizeof(buffer), "%s%s%s%s", - (concurrency & AudioPolicyInterface::API_INPUT_CONCURRENCY_CALL)? ",call":"", - (concurrency & AudioPolicyInterface::API_INPUT_CONCURRENCY_CAPTURE)? ",capture":"", - (concurrency & AudioPolicyInterface::API_INPUT_CONCURRENCY_HOTWORD)? ",hotword":"", - (concurrency & AudioPolicyInterface::API_INPUT_CONCURRENCY_PREEMPT)? ",preempt":""); - } else { - snprintf(buffer, sizeof(buffer), ",none"); - } - - return &buffer[1]; -} - std::string AudioPolicyService::getDeviceTypeStrForPortId(audio_port_handle_t portId) { std::string typeStr; struct audio_port port = {}; @@ -482,7 +465,7 @@ std::string AudioPolicyService::getDeviceTypeStrForPortId(audio_port_handle_t po return typeStr; } -status_t AudioPolicyService::startInput(audio_port_handle_t portId, bool *silenced) +status_t AudioPolicyService::startInput(audio_port_handle_t portId) { if (mAudioPolicyManager == NULL) { return NO_INIT; @@ -505,17 +488,16 @@ status_t AudioPolicyService::startInput(audio_port_handle_t portId, bool *silenc return PERMISSION_DENIED; } - // If UID inactive it records silence until becoming active - *silenced = !mUidPolicy->isUidActive(client->uid) && !client->isVirtualDevice; - Mutex::Autolock _l(mLock); - AudioPolicyInterface::concurrency_type__mask_t concurrency = - AudioPolicyInterface::API_INPUT_CONCURRENCY_NONE; + + client->active = true; + client->startTimeNs = systemTime(); + updateUidStates_l(); status_t status; { AutoCallerClear acc; - status = mAudioPolicyManager->startInput(portId, *silenced, &concurrency); + status = mAudioPolicyManager->startInput(portId); } @@ -524,7 +506,6 @@ status_t AudioPolicyService::startInput(audio_port_handle_t portId, bool *silenc static constexpr char kAudioPolicy[] = "audiopolicy"; - static constexpr char kAudioPolicyReason[] = "android.media.audiopolicy.reason"; static constexpr char kAudioPolicyStatus[] = "android.media.audiopolicy.status"; static constexpr char kAudioPolicyRqstSrc[] = "android.media.audiopolicy.rqst.src"; static constexpr char kAudioPolicyRqstPkg[] = "android.media.audiopolicy.rqst.pkg"; @@ -541,7 +522,6 @@ status_t AudioPolicyService::startInput(audio_port_handle_t portId, bool *silenc MediaAnalyticsItem *item = new MediaAnalyticsItem(kAudioPolicy); if (item != NULL) { - item->setCString(kAudioPolicyReason, audioConcurrencyString(concurrency).c_str()); item->setInt32(kAudioPolicyStatus, status); item->setCString(kAudioPolicyRqstSrc, @@ -556,54 +536,35 @@ status_t AudioPolicyService::startInput(audio_port_handle_t portId, bool *silenc item->setCString( kAudioPolicyRqstDevice, getDeviceTypeStrForPortId(client->deviceId).c_str()); - // figure out who is active - // NB: might the other party have given up the microphone since then? how sure. - // perhaps could have given up on it. - // we hold mLock, so perhaps we're safe for this looping - if (concurrency != AudioPolicyInterface::API_INPUT_CONCURRENCY_NONE) { - int count = mAudioRecordClients.size(); - for (int i = 0; i other = mAudioRecordClients.valueAt(i); - if (other->active) { - // keeps the last of the clients marked active - item->setCString(kAudioPolicyActiveSrc, - audioSourceString(other->attributes.source).c_str()); - item->setInt32(kAudioPolicyActiveSession, other->session); - if (other->opPackageName.size() != 0) { - item->setCString(kAudioPolicyActivePkg, - std::string(String8(other->opPackageName).string()).c_str()); - } else { - item->setCString(kAudioPolicyRqstPkg, - std::to_string(other->uid).c_str()); - } - item->setCString(kAudioPolicyActiveDevice, - getDeviceTypeStrForPortId(other->deviceId).c_str()); + int count = mAudioRecordClients.size(); + for (int i = 0; i < count ; i++) { + if (portId == mAudioRecordClients.keyAt(i)) { + continue; + } + sp other = mAudioRecordClients.valueAt(i); + if (other->active) { + // keeps the last of the clients marked active + item->setCString(kAudioPolicyActiveSrc, + audioSourceString(other->attributes.source).c_str()); + item->setInt32(kAudioPolicyActiveSession, other->session); + if (other->opPackageName.size() != 0) { + item->setCString(kAudioPolicyActivePkg, + std::string(String8(other->opPackageName).string()).c_str()); + } else { + item->setCString(kAudioPolicyRqstPkg, + std::to_string(other->uid).c_str()); } + item->setCString(kAudioPolicyActiveDevice, + getDeviceTypeStrForPortId(other->deviceId).c_str()); } } item->selfrecord(); delete item; item = NULL; } - } - - if (status == NO_ERROR) { - LOG_ALWAYS_FATAL_IF(concurrency & ~AudioPolicyInterface::API_INPUT_CONCURRENCY_ALL, - "startInput(): invalid concurrency type %d", (int)concurrency); - - // enforce permission (if any) required for each type of concurrency - if (concurrency & AudioPolicyInterface::API_INPUT_CONCURRENCY_CALL) { - //TODO: check incall capture permission - } - if (concurrency & AudioPolicyInterface::API_INPUT_CONCURRENCY_CAPTURE) { - //TODO: check concurrent capture permission - } - - client->active = true; - } else { + client->active = false; + client->startTimeNs = 0; + updateUidStates_l(); finishRecording(client->opPackageName, client->uid); } @@ -615,6 +576,7 @@ status_t AudioPolicyService::stopInput(audio_port_handle_t portId) if (mAudioPolicyManager == NULL) { return NO_INIT; } + Mutex::Autolock _l(mLock); ssize_t index = mAudioRecordClients.indexOfKey(portId); @@ -624,6 +586,9 @@ status_t AudioPolicyService::stopInput(audio_port_handle_t portId) sp client = mAudioRecordClients.valueAt(index); client->active = false; + client->startTimeNs = 0; + + updateUidStates_l(); // finish the recording app op finishRecording(client->opPackageName, client->uid); @@ -646,6 +611,14 @@ void AudioPolicyService::releaseInput(audio_port_handle_t portId) return; } client = mAudioRecordClients.valueAt(index); + + if (client->active) { + ALOGW("%s releasing active client portId %d", __FUNCTION__, portId); + client->active = false; + client->startTimeNs = 0; + updateUidStates_l(); + } + mAudioRecordClients.removeItem(portId); } if (client == 0) { diff --git a/services/audiopolicy/service/AudioPolicyService.cpp b/services/audiopolicy/service/AudioPolicyService.cpp index 78dbf5f16b..2893872346 100644 --- a/services/audiopolicy/service/AudioPolicyService.cpp +++ b/services/audiopolicy/service/AudioPolicyService.cpp @@ -348,11 +348,91 @@ void AudioPolicyService::updateUidStates() void AudioPolicyService::updateUidStates_l() { - //TODO: implement real concurrent capture policy: for now just apply each app state directly +// Go over all active clients and allow capture (does not force silence) in the +// following cases: +// - The client is the assistant AND +// an accessibility service is on TOP AND the source is VOICE_RECOGNITION or HOTWORD +// OR +// is on TOP AND uses VOICE_RECOGNITION +// OR uses HOTWORD AND there is no privacy sensitive active capture +// - The client is an accessibility service AND +// is on TOP AND the source is VOICE_RECOGNITION or HOTWORD +// - Any other client AND +// The assistant is not on TOP AND +// is on TOP OR latest started AND +// there is no privacy sensitive active capture +//TODO: mamanage pre processing effects according to use case priority + + sp topActive; + sp latestActive; + nsecs_t latestStartNs = 0; + sp latestSensitiveActive; + nsecs_t latestSensitiveStartNs = 0; + bool isA11yOnTop = mUidPolicy->isA11yOnTop(); + bool isAssistantOnTop = false; + bool isSensitiveActive = false; + for (size_t i =0; i < mAudioRecordClients.size(); i++) { sp current = mAudioRecordClients[i]; if (!current->active) continue; - setAppState_l(current->uid, apmStatFromAmState(mUidPolicy->getUidState(current->uid))); + if (isPrivacySensitive(current->attributes.source)) { + if (current->startTimeNs > latestSensitiveStartNs) { + latestSensitiveActive = current; + latestSensitiveStartNs = current->startTimeNs; + } + isSensitiveActive = true; + } + if (mUidPolicy->getUidState(current->uid) == ActivityManager::PROCESS_STATE_TOP) { + topActive = current; + latestActive = nullptr; + if (mUidPolicy->isAssistantUid(current->uid)) { + isAssistantOnTop = true; + } + } + if (current->startTimeNs > latestStartNs) { + latestActive = current; + latestStartNs = current->startTimeNs; + } + } + + if (topActive == nullptr && latestActive == nullptr) { + return; + } + + for (size_t i =0; i < mAudioRecordClients.size(); i++) { + sp current = mAudioRecordClients[i]; + if (!current->active) continue; + + audio_source_t source = current->attributes.source; + bool isOnTop = mUidPolicy->getUidState(current->uid) == ActivityManager::PROCESS_STATE_TOP; + bool isLatest = current == latestActive; + bool isLatestSensitive = current == latestSensitiveActive; + bool forceIdle = true; + if (mUidPolicy->isAssistantUid(current->uid)) { + if (isA11yOnTop) { + if (source == AUDIO_SOURCE_HOTWORD || source == AUDIO_SOURCE_VOICE_RECOGNITION) { + forceIdle = false; + } + } else { + if (((isOnTop && source == AUDIO_SOURCE_VOICE_RECOGNITION) || + source == AUDIO_SOURCE_HOTWORD) && !isSensitiveActive) { + forceIdle = false; + } + } + } else if (mUidPolicy->isA11yUid(current->uid)) { + if (isOnTop && + (source == AUDIO_SOURCE_VOICE_RECOGNITION || source == AUDIO_SOURCE_HOTWORD)) { + forceIdle = false; + } + } else { + if (!isAssistantOnTop && (isOnTop || (topActive == nullptr && isLatest)) && + (!isSensitiveActive || isLatestSensitive)) { + forceIdle = false; + } + } + setAppState_l(current->uid, + forceIdle ? APP_STATE_IDLE : + apmStatFromAmState(mUidPolicy->getUidState(current->uid))); } } @@ -369,6 +449,22 @@ app_state_t AudioPolicyService::apmStatFromAmState(int amState) { return APP_STATE_FOREGROUND; } +/* static */ +bool AudioPolicyService::isPrivacySensitive(audio_source_t source) +{ + switch (source) { + case AUDIO_SOURCE_VOICE_UPLINK: + case AUDIO_SOURCE_VOICE_DOWNLINK: + case AUDIO_SOURCE_VOICE_CALL: + case AUDIO_SOURCE_CAMCORDER: + case AUDIO_SOURCE_VOICE_COMMUNICATION: + return true; + default: + break; + } + return false; +} + void AudioPolicyService::setAppState_l(uid_t uid, app_state_t state) { AutoCallerClear acc; @@ -548,6 +644,7 @@ void AudioPolicyService::UidPolicy::registerSelf() { mObserverRegistered = true; } else { ALOGE("UidPolicy::registerSelf linkToDeath failed: %d", res); + am.unregisterUidObserver(this); } } @@ -650,6 +747,7 @@ int AudioPolicyService::UidPolicy::getUidState(uid_t uid) { mCachedUids.insert(std::pair>(uid, std::pair(active, state))); } + return state; } @@ -730,6 +828,21 @@ void AudioPolicyService::UidPolicy::updateUidLocked(std::unordered_map::iterator it = find(mA11yUids.begin(), mA11yUids.end(), uid.first); + if (it == mA11yUids.end()) { + continue; + } + if (uid.second.second == ActivityManager::PROCESS_STATE_TOP || + uid.second.second == ActivityManager::PROCESS_STATE_FOREGROUND_SERVICE || + uid.second.second == ActivityManager::PROCESS_STATE_BOUND_FOREGROUND_SERVICE) { + return true; + } + } + return false; +} + bool AudioPolicyService::UidPolicy::isA11yUid(uid_t uid) { std::vector::iterator it = find(mA11yUids.begin(), mA11yUids.end(), uid); diff --git a/services/audiopolicy/service/AudioPolicyService.h b/services/audiopolicy/service/AudioPolicyService.h index 4d7235fce8..dc5a36d34c 100644 --- a/services/audiopolicy/service/AudioPolicyService.h +++ b/services/audiopolicy/service/AudioPolicyService.h @@ -94,8 +94,7 @@ public: audio_input_flags_t flags, audio_port_handle_t *selectedDeviceId = NULL, audio_port_handle_t *portId = NULL); - virtual status_t startInput(audio_port_handle_t portId, - bool *silenced); + virtual status_t startInput(audio_port_handle_t portId); virtual status_t stopInput(audio_port_handle_t portId); virtual void releaseInput(audio_port_handle_t portId); virtual status_t initStreamVolume(audio_stream_type_t stream, @@ -276,6 +275,8 @@ private: void updateUidStates(); void updateUidStates_l(); + static bool isPrivacySensitive(audio_source_t source); + // If recording we need to make sure the UID is allowed to do that. If the UID is idle // then it cannot record and gets buffers with zeros - silence. As soon as the UID // transitions to an active state we will start reporting buffers with data. This approach @@ -299,6 +300,7 @@ private: bool isAssistantUid(uid_t uid) { return uid == mAssistantUid; } void setA11yUids(const std::vector& uids) { mA11yUids.clear(); mA11yUids = uids; } bool isA11yUid(uid_t uid); + bool isA11yOnTop(); // BnUidObserver implementation void onUidActive(uid_t uid) override; @@ -650,12 +652,11 @@ private: const audio_session_t session, const audio_port_handle_t deviceId, const String16& opPackageName) : AudioClient(attributes, io, uid, pid, session, deviceId), - opPackageName(opPackageName), isConcurrent(false), isVirtualDevice(false) {} + opPackageName(opPackageName), startTimeNs(0) {} ~AudioRecordClient() override = default; const String16 opPackageName; // client package name - bool isConcurrent; // is allowed to concurrent capture - bool isVirtualDevice; // uses virtual device: updated by APM::getInputForAttr() + nsecs_t startTimeNs; }; // --- AudioPlaybackClient ---