PatchPanel: store "downstream" patches for a device

When the system inserts an intermediate processing module between
audio flinger and hardware HAL, and they are connected via a
software patch, the threads of this patch need to receive
metadata passed to the thread serving the "upstream" device of
the processing module.

PatchPanel stores an association between "upstream" streams
and the software patch. This allows audio flinger to access
the patch threads.

For more info, see the comments in PatchPanel.h

Bug: 63901775
Test: manual
Change-Id: I60e578c37a1bc7385daf32d379ea143120584a31

services/audioflinger/AudioFlinger.cpp

@@ -1817,6 +1817,10 @@ audio_module_handle_t AudioFlinger::loadHwModule_l(const char *name)
 
         mHardwareStatus = AUDIO_HW_IDLE;
     }
+    if (strcmp(name, AUDIO_HAL_SERVICE_NAME_MSD) == 0) {
+        // An MSD module is inserted before hardware modules in order to mix encoded streams.
+        flags = static_cast<AudioHwDevice::Flags>(flags | AudioHwDevice::AHWD_IS_INSERT);
+    }
 
     audio_module_handle_t handle = (audio_module_handle_t) nextUniqueId(AUDIO_UNIQUE_ID_USE_MODULE);
     mAudioHwDevs.add(handle, new AudioHwDevice(handle, name, dev, flags));
@@ -2096,6 +2100,7 @@ sp<AudioFlinger::ThreadBase> AudioFlinger::openOutput_l(audio_module_handle_t mo
                       *output, thread.get());
             }
             mPlaybackThreads.add(*output, thread);
+            mPatchPanel.notifyStreamOpened(outHwDev, *output);
             return thread;
         }
     }
@@ -2231,6 +2236,7 @@ status_t AudioFlinger::closeOutput_nonvirtual(audio_io_handle_t output)
         const sp<AudioIoDescriptor> ioDesc = new AudioIoDescriptor();
         ioDesc->mIoHandle = output;
         ioConfigChanged(AUDIO_OUTPUT_CLOSED, ioDesc);
+        mPatchPanel.notifyStreamClosed(output);
     }
     // The thread entity (active unit of execution) is no longer running here,
     // but the ThreadBase container still exists.

services/audioflinger/AudioHwDevice.h

@@ -35,6 +35,9 @@ public:
     enum Flags {
         AHWD_CAN_SET_MASTER_VOLUME  = 0x1,
         AHWD_CAN_SET_MASTER_MUTE    = 0x2,
+        // Means that this isn't a terminal module, and software patches
+        // are used to transport audio data further.
+        AHWD_IS_INSERT              = 0x4,
     };
 
     AudioHwDevice(audio_module_handle_t handle,
@@ -55,6 +58,10 @@ public:
         return (0 != (mFlags & AHWD_CAN_SET_MASTER_MUTE));
     }
 
+    bool isInsert() const {
+        return (0 != (mFlags & AHWD_IS_INSERT));
+    }
+
     audio_module_handle_t handle() const { return mHandle; }
     const char *moduleName() const { return mModuleName; }
     sp<DeviceHalInterface> hwDevice() const { return mHwDevice; }

services/audioflinger/PatchPanel.cpp

@@ -83,6 +83,16 @@ status_t AudioFlinger::listAudioPatches(unsigned int *num_patches,
     return mPatchPanel.listAudioPatches(num_patches, patches);
 }
 
+status_t AudioFlinger::PatchPanel::SoftwarePatch::getLatencyMs_l(double *latencyMs) const
+{
+    const auto& iter = mPatchPanel.mPatches.find(mPatchHandle);
+    if (iter != mPatchPanel.mPatches.end()) {
+        return iter->second.getLatencyMs(latencyMs);
+    } else {
+        return BAD_VALUE;
+    }
+}
+
 /* List connected audio ports and their attributes */
 status_t AudioFlinger::PatchPanel::listAudioPorts(unsigned int *num_ports __unused,
                                 struct audio_port *ports __unused)
@@ -159,21 +169,21 @@ status_t AudioFlinger::PatchPanel::createAudioPatch(const struct audio_patch *pa
                 }
             }
             mPatches.erase(iter);
+            removeSoftwarePatchFromInsertedModules(*handle);
         }
     }
 
     Patch newPatch{*patch};
+    audio_module_handle_t insertedModule = AUDIO_MODULE_HANDLE_NONE;
 
     switch (patch->sources[0].type) {
         case AUDIO_PORT_TYPE_DEVICE: {
             audio_module_handle_t srcModule = patch->sources[0].ext.device.hw_module;
-            ssize_t index = mAudioFlinger.mAudioHwDevs.indexOfKey(srcModule);
-            if (index < 0) {
-                ALOGW("%s() bad src hw module %d", __func__, srcModule);
+            AudioHwDevice *audioHwDevice = findAudioHwDeviceByModule(srcModule);
+            if (!audioHwDevice) {
                 status = BAD_VALUE;
                 goto exit;
             }
-            AudioHwDevice *audioHwDevice = mAudioFlinger.mAudioHwDevs.valueAt(index);
             for (unsigned int i = 0; i < patch->num_sinks; i++) {
                 // support only one sink if connection to a mix or across HW modules
                 if ((patch->sinks[i].type == AUDIO_PORT_TYPE_MIX ||
@@ -285,6 +295,9 @@ status_t AudioFlinger::PatchPanel::createAudioPatch(const struct audio_patch *pa
                 if (status != NO_ERROR) {
                     goto exit;
                 }
+                if (audioHwDevice->isInsert()) {
+                    insertedModule = audioHwDevice->handle();
+                }
             } else {
                 if (patch->sinks[0].type == AUDIO_PORT_TYPE_MIX) {
                     sp<ThreadBase> thread = mAudioFlinger.checkRecordThread_l(
@@ -364,6 +377,9 @@ exit:
         *handle = (audio_patch_handle_t) mAudioFlinger.nextUniqueId(AUDIO_UNIQUE_ID_USE_PATCH);
         newPatch.mHalHandle = halHandle;
         mPatches.insert(std::make_pair(*handle, std::move(newPatch)));
+        if (insertedModule != AUDIO_MODULE_HANDLE_NONE) {
+            addSoftwarePatchToInsertedModules(insertedModule, *handle);
+        }
         ALOGV("%s() added new patch handle %d halHandle %d", __func__, *handle, halHandle);
     } else {
         newPatch.clearConnections(this);
@@ -511,21 +527,17 @@ status_t AudioFlinger::PatchPanel::Patch::getLatencyMs(double *latencyMs) const
     return OK;
 }
 
-String8 AudioFlinger::PatchPanel::Patch::dump(audio_patch_handle_t myHandle)
+String8 AudioFlinger::PatchPanel::Patch::dump(audio_patch_handle_t myHandle) const
 {
-    String8 result;
-
     // TODO: Consider table dump form for patches, just like tracks.
-    result.appendFormat("Patch %d: thread %p => thread %p",
-            myHandle, mRecord.thread().get(), mPlayback.thread().get());
+    String8 result = String8::format("Patch %d: thread %p => thread %p",
+            myHandle, mRecord.const_thread().get(), mPlayback.const_thread().get());
 
     // add latency if it exists
     double latencyMs;
     if (getLatencyMs(&latencyMs) == OK) {
         result.appendFormat("  latency: %.2lf", latencyMs);
     }
-
-    result.append("\n");
     return result;
 }
 
@@ -596,6 +608,7 @@ status_t AudioFlinger::PatchPanel::releaseAudioPatch(audio_patch_handle_t handle
     }
 
     mPatches.erase(iter);
+    removeSoftwarePatchFromInsertedModules(handle);
     return status;
 }
 
@@ -607,35 +620,114 @@ status_t AudioFlinger::PatchPanel::listAudioPatches(unsigned int *num_patches __
     return NO_ERROR;
 }
 
-sp<DeviceHalInterface> AudioFlinger::PatchPanel::findHwDeviceByModule(audio_module_handle_t module)
+status_t AudioFlinger::PatchPanel::getDownstreamSoftwarePatches(
+        audio_io_handle_t stream,
+        std::vector<AudioFlinger::PatchPanel::SoftwarePatch> *patches) const
+{
+    for (const auto& module : mInsertedModules) {
+        if (module.second.streams.count(stream)) {
+            for (const auto& patchHandle : module.second.sw_patches) {
+                const auto& patch_iter = mPatches.find(patchHandle);
+                if (patch_iter != mPatches.end()) {
+                    const Patch &patch = patch_iter->second;
+                    patches->emplace_back(*this, patchHandle,
+                            patch.mPlayback.const_thread()->id(),
+                            patch.mRecord.const_thread()->id());
+                } else {
+                    ALOGE("Stale patch handle in the cache: %d", patchHandle);
+                }
+            }
+            return OK;
+        }
+    }
+    // The stream is not associated with any of inserted modules.
+    return BAD_VALUE;
+}
+
+void AudioFlinger::PatchPanel::notifyStreamOpened(
+        AudioHwDevice *audioHwDevice, audio_io_handle_t stream)
+{
+    if (audioHwDevice->isInsert()) {
+        mInsertedModules[audioHwDevice->handle()].streams.insert(stream);
+    }
+}
+
+void AudioFlinger::PatchPanel::notifyStreamClosed(audio_io_handle_t stream)
+{
+    for (auto& module : mInsertedModules) {
+        module.second.streams.erase(stream);
+    }
+}
+
+AudioHwDevice* AudioFlinger::PatchPanel::findAudioHwDeviceByModule(audio_module_handle_t module)
 {
     if (module == AUDIO_MODULE_HANDLE_NONE) return nullptr;
     ssize_t index = mAudioFlinger.mAudioHwDevs.indexOfKey(module);
     if (index < 0) {
+        ALOGW("%s() bad hw module %d", __func__, module);
         return nullptr;
     }
-    return mAudioFlinger.mAudioHwDevs.valueAt(index)->hwDevice();
+    return mAudioFlinger.mAudioHwDevs.valueAt(index);
 }
 
-void AudioFlinger::PatchPanel::dump(int fd)
+sp<DeviceHalInterface> AudioFlinger::PatchPanel::findHwDeviceByModule(audio_module_handle_t module)
 {
+    AudioHwDevice *audioHwDevice = findAudioHwDeviceByModule(module);
+    return audioHwDevice ? audioHwDevice->hwDevice() : nullptr;
+}
+
+void AudioFlinger::PatchPanel::addSoftwarePatchToInsertedModules(
+        audio_module_handle_t module, audio_patch_handle_t handle)
+{
+    mInsertedModules[module].sw_patches.insert(handle);
+}
+
+void AudioFlinger::PatchPanel::removeSoftwarePatchFromInsertedModules(
+        audio_patch_handle_t handle)
+{
+    for (auto& module : mInsertedModules) {
+        module.second.sw_patches.erase(handle);
+    }
+}
+
+void AudioFlinger::PatchPanel::dump(int fd) const
+{
+    String8 patchPanelDump;
+    const char *indent = "  ";
+
     // Only dump software patches.
     bool headerPrinted = false;
-    for (auto& iter : mPatches) {
+    for (const auto& iter : mPatches) {
         if (iter.second.isSoftware()) {
             if (!headerPrinted) {
-                String8 header("\nSoftware patches:\n");
-                write(fd, header.string(), header.size());
+                patchPanelDump += "\nSoftware patches:\n";
+                headerPrinted = true;
+            }
+            patchPanelDump.appendFormat("%s%s\n", indent, iter.second.dump(iter.first).string());
+        }
+    }
+
+    headerPrinted = false;
+    for (const auto& module : mInsertedModules) {
+        if (!module.second.streams.empty() || !module.second.sw_patches.empty()) {
+            if (!headerPrinted) {
+                patchPanelDump += "\nTracked inserted modules:\n";
                 headerPrinted = true;
             }
-            String8 patchDump("  ");
-            patchDump.append(iter.second.dump(iter.first));
-            write(fd, patchDump.string(), patchDump.size());
+            String8 moduleDump = String8::format("Module %d: I/O handles: ", module.first);
+            for (const auto& stream : module.second.streams) {
+                moduleDump.appendFormat("%d ", stream);
+            }
+            moduleDump.append("; SW Patches: ");
+            for (const auto& patch : module.second.sw_patches) {
+                moduleDump.appendFormat("%d ", patch);
+            }
+            patchPanelDump.appendFormat("%s%s\n", indent, moduleDump.string());
         }
     }
-    if (headerPrinted) {
-        String8 trailing("\n");
-        write(fd, trailing.string(), trailing.size());
+
+    if (!patchPanelDump.isEmpty()) {
+        write(fd, patchPanelDump.string(), patchPanelDump.size());
     }
 }
 

services/audioflinger/PatchPanel.h

@@ -19,9 +19,31 @@
     #error This header file should only be included from AudioFlinger.h
 #endif
 
+
 // PatchPanel is concealed within AudioFlinger, their lifetimes are the same.
 class PatchPanel {
 public:
+    class SoftwarePatch {
+      public:
+        SoftwarePatch(const PatchPanel &patchPanel, audio_patch_handle_t patchHandle,
+                audio_io_handle_t playbackThreadHandle, audio_io_handle_t recordThreadHandle)
+                : mPatchPanel(patchPanel), mPatchHandle(patchHandle),
+                  mPlaybackThreadHandle(playbackThreadHandle),
+                  mRecordThreadHandle(recordThreadHandle) {}
+        SoftwarePatch(const SoftwarePatch&) = default;
+        SoftwarePatch& operator=(const SoftwarePatch&) = default;
+
+        // Must be called under AudioFlinger::mLock
+        status_t getLatencyMs_l(double *latencyMs) const;
+        audio_io_handle_t getPlaybackThreadHandle() const { return mPlaybackThreadHandle; };
+        audio_io_handle_t getRecordThreadHandle() const { return mRecordThreadHandle; };
+      private:
+        const PatchPanel &mPatchPanel;
+        const audio_patch_handle_t mPatchHandle;
+        const audio_io_handle_t mPlaybackThreadHandle;
+        const audio_io_handle_t mRecordThreadHandle;
+    };
+
     explicit PatchPanel(AudioFlinger* audioFlinger) : mAudioFlinger(*audioFlinger) {}
 
     /* List connected audio ports and their attributes */
@@ -42,7 +64,16 @@ public:
     status_t listAudioPatches(unsigned int *num_patches,
                                       struct audio_patch *patches);
 
-    void dump(int fd);
+    // Retrieves all currently estrablished software patches for a stream
+    // opened on an intermediate module.
+    status_t getDownstreamSoftwarePatches(audio_io_handle_t stream,
+            std::vector<SoftwarePatch> *patches) const;
+
+    // Notifies patch panel about all opened and closed streams.
+    void notifyStreamOpened(AudioHwDevice *audioHwDevice, audio_io_handle_t stream);
+    void notifyStreamClosed(audio_io_handle_t stream);
+
+    void dump(int fd) const;
 
 private:
     template<typename ThreadType, typename TrackType>
@@ -65,6 +96,7 @@ private:
         audio_patch_handle_t handle() const { return mHandle; }
         sp<ThreadType> thread() { return mThread; }
         sp<TrackType> track() { return mTrack; }
+        sp<const ThreadType> const_thread() const { return mThread; }
         sp<const TrackType> const_track() const { return mTrack; }
 
         void closeConnections(PatchPanel *panel) {
@@ -122,7 +154,7 @@ private:
         // returns the latency of the patch (from record to playback).
         status_t getLatencyMs(double *latencyMs) const;
 
-        String8 dump(audio_patch_handle_t myHandle);
+        String8 dump(audio_patch_handle_t myHandle) const;
 
         // Note that audio_patch::id is only unique within a HAL module
         struct audio_patch              mAudioPatch;
@@ -138,8 +170,38 @@ private:
         Endpoint<RecordThread, RecordThread::PatchRecord> mRecord;
     };
 
+    AudioHwDevice* findAudioHwDeviceByModule(audio_module_handle_t module);
     sp<DeviceHalInterface> findHwDeviceByModule(audio_module_handle_t module);
+    void addSoftwarePatchToInsertedModules(
+            audio_module_handle_t module, audio_patch_handle_t handle);
+    void removeSoftwarePatchFromInsertedModules(audio_patch_handle_t handle);
 
     AudioFlinger &mAudioFlinger;
     std::map<audio_patch_handle_t, Patch> mPatches;
+
+    // This map allows going from a thread to "downstream" software patches
+    // when a processing module inserted in between. Example:
+    //
+    //  from map value.streams                               map key
+    //  [Mixer thread] --> [Virtual output device] --> [Processing module] ---\
+    //       [Harware module] <-- [Physical output device] <-- [S/W Patch] <--/
+    //                                                 from map value.sw_patches
+    //
+    // This allows the mixer thread to look up the threads of the software patch
+    // for propagating timing info, parameters, etc.
+    //
+    // The current assumptions are:
+    //   1) The processing module acts as a mixer with several outputs which
+    //      represent differently downmixed and / or encoded versions of the same
+    //      mixed stream. There is no 1:1 correspondence between the input streams
+    //      and the software patches, but rather a N:N correspondence between
+    //      a group of streams and a group of patches.
+    //   2) There are only a couple of inserted processing modules in the system,
+    //      so when looking for a stream or patch handle we can iterate over
+    //      all modules.
+    struct ModuleConnections {
+        std::set<audio_io_handle_t> streams;
+        std::set<audio_patch_handle_t> sw_patches;
+    };
+    std::map<audio_module_handle_t, ModuleConnections> mInsertedModules;
 };