Fix multi-user and multi-storage with FUSE

Up until now, the FUSE mount logic has made two assumptions:
1. The primary external volume is an emulated volume on /data/media
2. Only the primary user is running, as user zero

These assumptions are fixed by the following changes
creating an EmulatedVolume per Android user and changing the
VolumeBase id format to append the user to the id, so
s/emulated/emulated-0/. This allows us mount separate volumes per user

Some additional refactorings to re-use/clean up code.

Test: adb shell sm set-virtual-disk and partition disk operations work
even after setting up a work profile
Bug: 135341433

Change-Id: Ifabaa12368e5a591fbcdce4ee71c83ff35fdac6b

Utils.cpp

@@ -985,26 +985,61 @@ bool writeStringToFile(const std::string& payload, const std::string& filename)
     return true;
 }
 
-int MountUserFuse(userid_t user_id, const std::string& relative_path,
-                  android::base::unique_fd* fuse_fd) {
-    std::string path(StringPrintf("/mnt/user/%d/%s", user_id, relative_path.c_str()));
+status_t MountUserFuse(userid_t user_id, const std::string& absolute_lower_path,
+                       const std::string& relative_upper_path, android::base::unique_fd* fuse_fd) {
+    std::string pre_fuse_path(StringPrintf("/mnt/user/%d", user_id));
+    std::string fuse_path(
+            StringPrintf("%s/%s", pre_fuse_path.c_str(), relative_upper_path.c_str()));
+
+    std::string pre_pass_through_path(StringPrintf("/mnt/pass_through/%d", user_id));
+    std::string pass_through_path(
+            StringPrintf("%s/%s", pre_pass_through_path.c_str(), relative_upper_path.c_str()));
 
     // Force remove the existing mount before we attempt to prepare the
     // directory. If we have a dangling mount, then PrepareDir may fail if the
     // indirection to FUSE doesn't work.
-    android::status_t result = android::vold::ForceUnmount(path);
+    android::status_t result = UnmountUserFuse(pass_through_path, fuse_path);
     if (result != android::OK) {
-        PLOG(ERROR) << "Failed to unmount " << path;
         return -1;
     }
 
     // Create directories.
-    result = android::vold::PrepareDir(path, 0700, AID_ROOT, AID_ROOT);
+    result = PrepareDir(pre_fuse_path, 0700, AID_ROOT, AID_ROOT);
+    if (result != android::OK) {
+        PLOG(ERROR) << "Failed to prepare directory " << pre_fuse_path;
+        return -1;
+    }
+
+    result = PrepareDir(fuse_path, 0700, AID_ROOT, AID_ROOT);
+    if (result != android::OK) {
+        PLOG(ERROR) << "Failed to prepare directory " << fuse_path;
+        return -1;
+    }
+
+    result = PrepareDir(pre_pass_through_path, 0755, AID_ROOT, AID_ROOT);
+    if (result != android::OK) {
+        PLOG(ERROR) << "Failed to prepare directory " << pre_pass_through_path;
+        return -1;
+    }
+
+    result = PrepareDir(pass_through_path, 0755, AID_ROOT, AID_ROOT);
     if (result != android::OK) {
-        PLOG(ERROR) << "Failed to prepare directory " << path;
+        PLOG(ERROR) << "Failed to prepare directory " << pass_through_path;
         return -1;
     }
 
+    if (relative_upper_path == "emulated") {
+        std::string target(StringPrintf("/mnt/user/%d/self", user_id));
+        result = PrepareDir(target, 0755, AID_ROOT, AID_ROOT);
+        if (result != android::OK) {
+            PLOG(ERROR) << "Failed to prepare directory " << target;
+            return -1;
+        }
+        target += "/primary";
+
+        Symlink(fuse_path + "/" + std::to_string(user_id), target);
+    }
+
     // Open fuse fd.
     fuse_fd->reset(open("/dev/fuse", O_RDWR | O_CLOEXEC));
     if (fuse_fd->get() == -1) {
@@ -1021,15 +1056,35 @@ int MountUserFuse(userid_t user_id, const std::string& relative_path,
         "user_id=0,group_id=0,",
         fuse_fd->get());
 
-    const int result_int =
-        TEMP_FAILURE_RETRY(mount("/dev/fuse", path.c_str(), "fuse",
-                                 MS_NOSUID | MS_NODEV | MS_NOEXEC | MS_NOATIME | MS_LAZYTIME,
-                                 opts.c_str()));
-    if (result_int != 0) {
-        PLOG(ERROR) << "Failed to mount " << path;
+    result = TEMP_FAILURE_RETRY(mount("/dev/fuse", fuse_path.c_str(), "fuse",
+                                      MS_NOSUID | MS_NODEV | MS_NOEXEC | MS_NOATIME | MS_LAZYTIME,
+                                      opts.c_str()));
+    if (result != 0) {
+        PLOG(ERROR) << "Failed to mount " << fuse_path;
         return -errno;
     }
-    return 0;
+    LOG(INFO) << "Bind mounting to " << absolute_lower_path;
+    return BindMount(absolute_lower_path, pass_through_path);
+}
+
+status_t UnmountUserFuse(const std::string& pass_through_path, const std::string& fuse_path) {
+    // Best effort unmount pass_through path
+    sSleepOnUnmount = false;
+    ForceUnmount(pass_through_path);
+    android::status_t result = ForceUnmount(fuse_path);
+    sSleepOnUnmount = true;
+    if (result != android::OK) {
+        // TODO(b/135341433): MNT_DETACH is needed for fuse because umount2 can fail with EBUSY.
+        // Figure out why we get EBUSY and remove this special casing if possible.
+        PLOG(ERROR) << "Failed to unmount. Trying MNT_DETACH " << fuse_path << " ...";
+        if (umount2(fuse_path.c_str(), UMOUNT_NOFOLLOW | MNT_DETACH) && errno != EINVAL &&
+            errno != ENOENT) {
+            PLOG(ERROR) << "Failed to unmount with MNT_DETACH " << fuse_path;
+            return -errno;
+        }
+        return android::OK;
+    }
+    return result;
 }
 
 }  // namespace vold

Utils.h

@@ -151,8 +151,10 @@ bool FsyncDirectory(const std::string& dirname);
 
 bool writeStringToFile(const std::string& payload, const std::string& filename);
 
-int MountUserFuse(userid_t user_id, const std::string& relative_path,
-                  android::base::unique_fd* fuse_fd);
+status_t MountUserFuse(userid_t user_id, const std::string& absolute_lower_path,
+                       const std::string& relative_upper_path, android::base::unique_fd* fuse_fd);
+
+status_t UnmountUserFuse(const std::string& pass_through_path, const std::string& fuse_path);
 
 }  // namespace vold
 }  // namespace android

VoldNativeService.cpp

@@ -107,7 +107,7 @@ binder::Status checkArgumentId(const std::string& id) {
         return exception(binder::Status::EX_ILLEGAL_ARGUMENT, "Missing ID");
     }
     for (const char& c : id) {
-        if (!std::isalnum(c) && c != ':' && c != ',') {
+        if (!std::isalnum(c) && c != ':' && c != ',' && c != ';') {
             return exception(binder::Status::EX_ILLEGAL_ARGUMENT,
                              StringPrintf("ID %s is malformed", id.c_str()));
         }

VolumeManager.cpp

@@ -31,7 +31,6 @@
 #include <sys/wait.h>
 #include <unistd.h>
 #include <array>
-#include <thread>
 
 #include <linux/kdev_t.h>
 
@@ -68,6 +67,7 @@
 #include "fs/Vfat.h"
 #include "model/EmulatedVolume.h"
 #include "model/ObbVolume.h"
+#include "model/PrivateVolume.h"
 #include "model/StubVolume.h"
 
 using android::OK;
@@ -80,10 +80,12 @@ using android::vold::BindMount;
 using android::vold::CreateDir;
 using android::vold::DeleteDirContents;
 using android::vold::DeleteDirContentsAndDir;
+using android::vold::PrivateVolume;
 using android::vold::Symlink;
 using android::vold::Unlink;
 using android::vold::UnmountTree;
 using android::vold::VoldNativeService;
+using android::vold::VolumeBase;
 
 static const char* kPathUserMount = "/mnt/user";
 static const char* kPathVirtualDisk = "/data/misc/vold/virtual_disk";
@@ -101,22 +103,6 @@ static const unsigned int kMajorBlockExperimentalMax = 254;
 
 VolumeManager* VolumeManager::sInstance = NULL;
 
-static void* symlinkPrimary(void* data) {
-    std::unique_ptr<std::pair<std::string, std::string>> linkInfo(
-        static_cast<std::pair<std::string, std::string>*>(data));
-    std::string* source = &linkInfo->first;
-    std::string* target = &linkInfo->second;
-
-    fs_prepare_dir(source->c_str(), 0755, AID_ROOT, AID_ROOT);
-    fs_prepare_dir(target->c_str(), 0755, AID_ROOT, AID_ROOT);
-    *target = *target + "/primary";
-
-    // Link source to target
-    LOG(DEBUG) << "Linking " << *source << " to " << *target;
-    Symlink(*source, *target);
-    return nullptr;
-}
-
 VolumeManager* VolumeManager::Instance() {
     if (!sInstance) sInstance = new VolumeManager();
     return sInstance;
@@ -191,10 +177,13 @@ int VolumeManager::start() {
 
     // Assume that we always have an emulated volume on internal
     // storage; the framework will decide if it should be mounted.
-    CHECK(mInternalEmulated == nullptr);
-    mInternalEmulated = std::shared_ptr<android::vold::VolumeBase>(
-        new android::vold::EmulatedVolume("/data/media"));
-    mInternalEmulated->create();
+    CHECK(mInternalEmulatedVolumes.empty());
+
+    auto vol = std::shared_ptr<android::vold::VolumeBase>(
+            new android::vold::EmulatedVolume("/data/media", 0));
+    vol->setMountUserId(0);
+    vol->create();
+    mInternalEmulatedVolumes.push_back(vol);
 
     // Consider creating a virtual disk
     updateVirtualDisk();
@@ -203,9 +192,12 @@ int VolumeManager::start() {
 }
 
 int VolumeManager::stop() {
-    CHECK(mInternalEmulated != nullptr);
-    mInternalEmulated->destroy();
-    mInternalEmulated = nullptr;
+    CHECK(!mInternalEmulatedVolumes.empty());
+    for (const auto& vol : mInternalEmulatedVolumes) {
+        vol->destroy();
+    }
+    mInternalEmulatedVolumes.clear();
+
     return 0;
 }
 
@@ -327,11 +319,10 @@ std::shared_ptr<android::vold::Disk> VolumeManager::findDisk(const std::string&
 }
 
 std::shared_ptr<android::vold::VolumeBase> VolumeManager::findVolume(const std::string& id) {
-    // Vold could receive "mount" after "shutdown" command in the extreme case.
-    // If this happens, mInternalEmulated will equal nullptr and
-    // we need to deal with it in order to avoid null pointer crash.
-    if (mInternalEmulated != nullptr && mInternalEmulated->getId() == id) {
-        return mInternalEmulated;
+    for (const auto& vol : mInternalEmulatedVolumes) {
+        if (vol->getId() == id) {
+            return vol;
+        }
     }
     for (const auto& disk : mDisks) {
         auto vol = disk->findVolume(id);
@@ -382,61 +373,129 @@ int VolumeManager::forgetPartition(const std::string& partGuid, const std::strin
 }
 
 int VolumeManager::linkPrimary(userid_t userId) {
-    bool isFuse = GetBoolProperty(android::vold::kPropFuseSnapshot, false);
-
-    if (isFuse) {
-        // Here we have to touch /mnt/user/userid>/<volumeid> which was already mounted as part of
-        // the boot sequence, requiring waiting till a fuse handler is available. If we do this work
-        // in foreground we could hang the caller, i.e. system server, which needs to start the fuse
-        // handler. So do it in the background.
-        std::string source(
-            StringPrintf("/mnt/user/%d/%s/%d", userId, mPrimary->getId().c_str(), userId));
-        std::string target(StringPrintf("/mnt/user/%d/self", userId));
-
-        auto symlinkInfo = new std::pair<std::string, std::string>(source, target);
-        std::thread(symlinkPrimary, symlinkInfo).detach();
-        return 0;
+    if (!GetBoolProperty(android::vold::kPropFuseSnapshot, false)) {
+        std::string source(mPrimary->getPath());
+        if (mPrimary->isEmulated()) {
+            source = StringPrintf("%s/%d", source.c_str(), userId);
+            fs_prepare_dir(source.c_str(), 0755, AID_ROOT, AID_ROOT);
+        }
+
+        std::string target(StringPrintf("/mnt/user/%d/primary", userId));
+        LOG(DEBUG) << "Linking " << source << " to " << target;
+        Symlink(source, target);
     }
+    return 0;
+}
 
-    std::string source(mPrimary->getPath());
-    if (mPrimary->isEmulated()) {
-        source = StringPrintf("%s/%d", source.c_str(), userId);
-        fs_prepare_dir(source.c_str(), 0755, AID_ROOT, AID_ROOT);
+void VolumeManager::destroyEmulatedVolumesForUser(userid_t userId) {
+    // Destroy and remove all unstacked EmulatedVolumes for the user
+    auto i = mInternalEmulatedVolumes.begin();
+    while (i != mInternalEmulatedVolumes.end()) {
+        auto vol = *i;
+        if (vol->getMountUserId() == userId) {
+            vol->destroy();
+            i = mInternalEmulatedVolumes.erase(i);
+        } else {
+            i++;
+        }
     }
 
-    std::string target(StringPrintf("/mnt/user/%d/primary", userId));
-    LOG(DEBUG) << "Linking " << source << " to " << target;
-    Symlink(source, target);
-    return 0;
+    // Destroy and remove all stacked EmulatedVolumes for the user on each mounted private volume
+    std::list<std::string> private_vols;
+    listVolumes(VolumeBase::Type::kPrivate, private_vols);
+    for (const std::string& id : private_vols) {
+        PrivateVolume* pvol = static_cast<PrivateVolume*>(findVolume(id).get());
+        std::list<std::shared_ptr<VolumeBase>> vols_to_remove;
+        if (pvol->getState() == VolumeBase::State::kMounted) {
+            for (const auto& vol : pvol->getVolumes()) {
+                if (vol->getMountUserId() == userId) {
+                    vols_to_remove.push_back(vol);
+                }
+            }
+            for (const auto& vol : vols_to_remove) {
+                vol->destroy();
+                pvol->removeVolume(vol);
+            }
+        }  // else EmulatedVolumes will be destroyed on VolumeBase#unmount
+    }
+}
+
+void VolumeManager::createEmulatedVolumesForUser(userid_t userId) {
+    // Create unstacked EmulatedVolumes for the user
+    auto vol = std::shared_ptr<android::vold::VolumeBase>(
+            new android::vold::EmulatedVolume("/data/media", userId));
+    vol->setMountUserId(userId);
+    mInternalEmulatedVolumes.push_back(vol);
+    vol->create();
+
+    // Create stacked EmulatedVolumes for the user on each PrivateVolume
+    std::list<std::string> private_vols;
+    listVolumes(VolumeBase::Type::kPrivate, private_vols);
+    for (const std::string& id : private_vols) {
+        PrivateVolume* pvol = static_cast<PrivateVolume*>(findVolume(id).get());
+        if (pvol->getState() == VolumeBase::State::kMounted) {
+            auto evol =
+                    std::shared_ptr<android::vold::VolumeBase>(new android::vold::EmulatedVolume(
+                            pvol->getPath() + "/media", pvol->getRawDevice(), pvol->getFsUuid(),
+                            userId));
+            evol->setMountUserId(userId);
+            pvol->addVolume(evol);
+            evol->create();
+        }  // else EmulatedVolumes will be created per user when on PrivateVolume#doMount
+    }
 }
 
 int VolumeManager::onUserAdded(userid_t userId, int userSerialNumber) {
+    LOG(INFO) << "onUserAdded: " << userId;
+
     mAddedUsers[userId] = userSerialNumber;
     return 0;
 }
 
 int VolumeManager::onUserRemoved(userid_t userId) {
+    LOG(INFO) << "onUserRemoved: " << userId;
+
+    if (GetBoolProperty(android::vold::kPropFuseSnapshot, false) &&
+        mAddedUsers.find(userId) != mAddedUsers.end()) {
+        destroyEmulatedVolumesForUser(userId);
+    }
+
     mAddedUsers.erase(userId);
+    mStartedUsers.erase(userId);
     return 0;
 }
 
 int VolumeManager::onUserStarted(userid_t userId) {
-    LOG(VERBOSE) << "onUserStarted: " << userId;
-    // Note that sometimes the system will spin up processes from Zygote
-    // before actually starting the user, so we're okay if Zygote
-    // already created this directory.
-    std::string path(StringPrintf("%s/%d", kPathUserMount, userId));
-    fs_prepare_dir(path.c_str(), 0755, AID_ROOT, AID_ROOT);
+    LOG(INFO) << "onUserStarted: " << userId;
 
-    mStartedUsers.insert(userId);
-    if (mPrimary) {
-        linkPrimary(userId);
+    if (GetBoolProperty(android::vold::kPropFuseSnapshot, false)) {
+        if (mStartedUsers.find(userId) == mStartedUsers.end()) {
+            createEmulatedVolumesForUser(userId);
+        }
+    } else {
+        // Note that sometimes the system will spin up processes from Zygote
+        // before actually starting the user, so we're okay if Zygote
+        // already created this directory.
+        std::string path(StringPrintf("%s/%d", kPathUserMount, userId));
+        fs_prepare_dir(path.c_str(), 0755, AID_ROOT, AID_ROOT);
+
+        if (mPrimary) {
+            linkPrimary(userId);
+        }
     }
+
+    mStartedUsers.insert(userId);
     return 0;
 }
 
 int VolumeManager::onUserStopped(userid_t userId) {
     LOG(VERBOSE) << "onUserStopped: " << userId;
+
+    if (GetBoolProperty(android::vold::kPropFuseSnapshot, false) &&
+        mStartedUsers.find(userId) != mStartedUsers.end()) {
+        destroyEmulatedVolumesForUser(userId);
+    }
+
     mStartedUsers.erase(userId);
     return 0;
 }
@@ -640,10 +699,17 @@ int VolumeManager::remountUid(uid_t uid, int32_t mountMode) {
 int VolumeManager::reset() {
     // Tear down all existing disks/volumes and start from a blank slate so
     // newly connected framework hears all events.
-    if (mInternalEmulated != nullptr) {
-        mInternalEmulated->destroy();
-        mInternalEmulated->create();
+    for (const auto& vol : mInternalEmulatedVolumes) {
+        vol->destroy();
     }
+    mInternalEmulatedVolumes.clear();
+    // Add user 0 cos it's always running and started
+    auto vol = std::shared_ptr<android::vold::VolumeBase>(
+            new android::vold::EmulatedVolume("/data/media", 0));
+    vol->setMountUserId(0);
+    vol->create();
+    mInternalEmulatedVolumes.push_back(vol);
+
     for (const auto& disk : mDisks) {
         disk->destroy();
         disk->create();
@@ -651,20 +717,25 @@ int VolumeManager::reset() {
     updateVirtualDisk();
     mAddedUsers.clear();
     mStartedUsers.clear();
+
+    mStartedUsers.insert(0);
     return 0;
 }
 
 // Can be called twice (sequentially) during shutdown. should be safe for that.
 int VolumeManager::shutdown() {
-    if (mInternalEmulated == nullptr) {
+    if (mInternalEmulatedVolumes.empty()) {
         return 0;  // already shutdown
     }
     android::vold::sSleepOnUnmount = false;
-    mInternalEmulated->destroy();
-    mInternalEmulated = nullptr;
+    for (const auto& vol : mInternalEmulatedVolumes) {
+        vol->destroy();
+    }
     for (const auto& disk : mDisks) {
         disk->destroy();
     }
+
+    mInternalEmulatedVolumes.clear();
     mStubVolumes.clear();
     mDisks.clear();
     mPendingDisks.clear();
@@ -677,8 +748,8 @@ int VolumeManager::unmountAll() {
     ATRACE_NAME("VolumeManager::unmountAll()");
 
     // First, try gracefully unmounting all known devices
-    if (mInternalEmulated != nullptr) {
-        mInternalEmulated->unmount();
+    for (const auto& vol : mInternalEmulatedVolumes) {
+        vol->unmount();
     }
     for (const auto& stub : mStubVolumes) {
         stub->unmount();

VolumeManager.h

@@ -84,6 +84,8 @@ class VolumeManager {
 
     void listVolumes(android::vold::VolumeBase::Type type, std::list<std::string>& list) const;
 
+    const std::unordered_set<userid_t>& getStartedUsers() const { return mStartedUsers; }
+
     int forgetPartition(const std::string& partGuid, const std::string& fsUuid);
 
     int onUserAdded(userid_t userId, int userSerialNumber);
@@ -137,6 +139,9 @@ class VolumeManager {
 
     int linkPrimary(userid_t userId);
 
+    void createEmulatedVolumesForUser(userid_t userId);
+    void destroyEmulatedVolumesForUser(userid_t userId);
+
     void handleDiskAdded(const std::shared_ptr<android::vold::Disk>& disk);
     void handleDiskChanged(dev_t device);
     void handleDiskRemoved(dev_t device);
@@ -151,13 +156,13 @@ class VolumeManager {
     std::list<std::shared_ptr<android::vold::Disk>> mPendingDisks;
     std::list<std::shared_ptr<android::vold::VolumeBase>> mObbVolumes;
     std::list<std::shared_ptr<android::vold::VolumeBase>> mStubVolumes;
+    std::list<std::shared_ptr<android::vold::VolumeBase>> mInternalEmulatedVolumes;
 
     std::unordered_map<userid_t, int> mAddedUsers;
     std::unordered_set<userid_t> mStartedUsers;
 
     std::string mVirtualDiskPath;
     std::shared_ptr<android::vold::Disk> mVirtualDisk;
-    std::shared_ptr<android::vold::VolumeBase> mInternalEmulated;
     std::shared_ptr<android::vold::VolumeBase> mPrimary;
 
     int mNextObbId;

binder/android/os/IVoldListener.aidl

@@ -25,7 +25,7 @@ oneway interface IVoldListener {
     void onDiskDestroyed(@utf8InCpp String diskId);
 
     void onVolumeCreated(@utf8InCpp String volId,
-            int type, @utf8InCpp String diskId, @utf8InCpp String partGuid);
+            int type, @utf8InCpp String diskId, @utf8InCpp String partGuid, int userId);
     void onVolumeStateChanged(@utf8InCpp String volId, int state);
     void onVolumeMetadataChanged(@utf8InCpp String volId,
             @utf8InCpp String fsType, @utf8InCpp String fsUuid, @utf8InCpp String fsLabel);

model/EmulatedVolume.cpp

@@ -42,16 +42,17 @@ namespace vold {
 
 static const char* kFusePath = "/system/bin/sdcard";
 
-EmulatedVolume::EmulatedVolume(const std::string& rawPath)
+EmulatedVolume::EmulatedVolume(const std::string& rawPath, int userId)
     : VolumeBase(Type::kEmulated), mFusePid(0) {
-    setId("emulated");
+    setId(StringPrintf("emulated;%u", userId));
     mRawPath = rawPath;
     mLabel = "emulated";
 }
 
-EmulatedVolume::EmulatedVolume(const std::string& rawPath, dev_t device, const std::string& fsUuid)
+EmulatedVolume::EmulatedVolume(const std::string& rawPath, dev_t device, const std::string& fsUuid,
+                               int userId)
     : VolumeBase(Type::kEmulated), mFusePid(0) {
-    setId(StringPrintf("emulated:%u,%u", major(device), minor(device)));
+    setId(StringPrintf("emulated:%u,%u;%u", major(device), minor(device), userId));
     mRawPath = rawPath;
     mLabel = fsUuid;
 }
@@ -90,17 +91,14 @@ status_t EmulatedVolume::doMount() {
         LOG(INFO) << "Mounting emulated fuse volume";
         android::base::unique_fd fd;
         int user_id = getMountUserId();
-        int result = MountUserFuse(user_id, label, &fd);
+        int result = MountUserFuse(user_id, getInternalPath(), label, &fd);
 
         if (result != 0) {
             PLOG(ERROR) << "Failed to mount emulated fuse volume";
             return -result;
         }
         setFuseFd(std::move(fd));
-
-        std::string pass_through_path(StringPrintf("/mnt/pass_through/%d/%s",
-                                                   user_id, label.c_str()));
-        return BindMount(getInternalPath(), pass_through_path);
+        return 0;
     }
 
     if (!(mFusePid = fork())) {
@@ -163,18 +161,17 @@ status_t EmulatedVolume::doUnmount() {
         if (getMountFlags() & MountFlags::kPrimary) {
             label = "emulated";
         }
-        std::string path(StringPrintf("/mnt/user/%d/%s", getMountUserId(), label.c_str()));
-        status_t result = ForceUnmount(path);
-        if (result != OK) {
-            // TODO(135341433): MNT_DETACH is needed for fuse because umount2 can fail with EBUSY.
-            // Figure out why we get EBUSY and remove this special casing if possible.
-            if (!umount2(path.c_str(), UMOUNT_NOFOLLOW | MNT_DETACH) || errno == EINVAL ||
-                errno == ENOENT) {
-                PLOG(INFO) << "ForceUnmount failed on emulated fuse volume";
-            }
+
+        std::string fuse_path(StringPrintf("/mnt/user/%d/%s", getMountUserId(), label.c_str()));
+        std::string pass_through_path(
+                StringPrintf("/mnt/pass_through/%d/%s", getMountUserId(), label.c_str()));
+        if (UnmountUserFuse(pass_through_path, fuse_path) != OK) {
+            PLOG(INFO) << "UnmountUserFuse failed on emulated fuse volume";
+            return -errno;
         }
 
-        rmdir(path.c_str());
+        rmdir(fuse_path.c_str());
+        rmdir(pass_through_path.c_str());
         setFuseFd(android::base::unique_fd());
         return OK;
     }

model/EmulatedVolume.h

@@ -37,8 +37,8 @@ namespace vold {
  */
 class EmulatedVolume : public VolumeBase {
   public:
-    explicit EmulatedVolume(const std::string& rawPath);
-    EmulatedVolume(const std::string& rawPath, dev_t device, const std::string& fsUuid);
+    explicit EmulatedVolume(const std::string& rawPath, int userId);
+    EmulatedVolume(const std::string& rawPath, dev_t device, const std::string& fsUuid, int userId);
     virtual ~EmulatedVolume();
 
   protected:

model/PrivateVolume.cpp

@@ -155,12 +155,18 @@ status_t PrivateVolume::doMount() {
         return -EIO;
     }
 
-    // Create a new emulated volume stacked above us, it will automatically
-    // be destroyed during unmount
+    auto vol_manager = VolumeManager::Instance();
     std::string mediaPath(mPath + "/media");
-    auto vol = std::shared_ptr<VolumeBase>(new EmulatedVolume(mediaPath, mRawDevice, mFsUuid));
-    addVolume(vol);
-    vol->create();
+
+    // Create a new emulated volume stacked above us for all added users, they will automatically
+    // be destroyed during unmount
+    for (userid_t user : vol_manager->getStartedUsers()) {
+        auto vol = std::shared_ptr<VolumeBase>(
+                new EmulatedVolume(mediaPath, mRawDevice, mFsUuid, user));
+        vol->setMountUserId(user);
+        addVolume(vol);
+        vol->create();
+    }
 
     return OK;
 }

model/PrivateVolume.h

@@ -42,6 +42,8 @@ class PrivateVolume : public VolumeBase {
     const std::string& getFsType() const { return mFsType; };
     const std::string& getRawDevPath() const { return mRawDevPath; };
     const std::string& getRawDmDevPath() const { return mDmDevPath; };
+    const std::string& getFsUuid() const { return mFsUuid; };
+    dev_t getRawDevice() const { return mRawDevice; };
 
   protected:
     status_t doCreate() override;

model/PublicVolume.cpp

@@ -170,22 +170,18 @@ status_t PublicVolume::doMount() {
     dev_t before = GetDevice(mFuseFull);
 
     bool isFuse = base::GetBoolProperty(kPropFuseSnapshot, false);
-
     if (isFuse) {
         LOG(INFO) << "Mounting public fuse volume";
         android::base::unique_fd fd;
         int user_id = getMountUserId();
-        int result = MountUserFuse(user_id, stableName, &fd);
+        int result = MountUserFuse(user_id, getInternalPath(), stableName, &fd);
 
         if (result != 0) {
             LOG(ERROR) << "Failed to mount public fuse volume";
             return -result;
         }
         setFuseFd(std::move(fd));
-
-        std::string pass_through_path(StringPrintf("/mnt/pass_through/%d/%s",
-                                                 user_id, stableName.c_str()));
-        return BindMount(getInternalPath(), pass_through_path);
+        return OK;
     }
 
     if (!(mFusePid = fork())) {
@@ -258,19 +254,24 @@ status_t PublicVolume::doUnmount() {
             stableName = mFsUuid;
         }
 
-        std::string path(StringPrintf("/mnt/user/%d/%s", getMountUserId(), stableName.c_str()));
-        status_t result = ForceUnmount(path);
-        if (result != OK) {
-            // TODO(135341433): MNT_DETACH is needed for fuse because umount2 can fail with EBUSY.
-            // Figure out why we get EBUSY and remove this special casing if possible.
-            if (!umount2(path.c_str(), UMOUNT_NOFOLLOW | MNT_DETACH) || errno == EINVAL ||
-                errno == ENOENT) {
-                PLOG(INFO) << "ForceUnmount failed on public fuse volume";
-            }
+        std::string fuse_path(
+                StringPrintf("/mnt/user/%d/%s", getMountUserId(), stableName.c_str()));
+        std::string pass_through_path(
+                StringPrintf("/mnt/pass_through/%d/%s", getMountUserId(), stableName.c_str()));
+        if (UnmountUserFuse(pass_through_path, fuse_path) != OK) {
+            PLOG(INFO) << "UnmountUserFuse failed on public fuse volume";
+            return -errno;
         }
+        ForceUnmount(kAsecPath);
+        ForceUnmount(mRawPath);
+
+        rmdir(fuse_path.c_str());
+        rmdir(pass_through_path.c_str());
+        rmdir(mRawPath.c_str());
+        mRawPath.clear();
 
-        rmdir(path.c_str());
         setFuseFd(android::base::unique_fd());
+
         return OK;
     }
 

model/VolumeBase.cpp

@@ -186,7 +186,8 @@ status_t VolumeBase::create() {
 
     auto listener = getListener();
     if (listener) {
-        listener->onVolumeCreated(getId(), static_cast<int32_t>(mType), mDiskId, mPartGuid);
+        listener->onVolumeCreated(getId(), static_cast<int32_t>(mType), mDiskId, mPartGuid,
+                                  mMountUserId);
     }
 
     setState(State::kUnmounted);

model/VolumeBase.h

@@ -88,6 +88,7 @@ class VolumeBase {
     const std::string& getPath() const { return mPath; }
     const std::string& getInternalPath() const { return mInternalPath; }
     const android::base::unique_fd& getFuseFd() const { return mFuseFd; }
+    const std::list<std::shared_ptr<VolumeBase>>& getVolumes() const { return mVolumes; }
 
     status_t setDiskId(const std::string& diskId);
     status_t setPartGuid(const std::string& partGuid);