Mount direct boot apps obb dir after fuse is ready.

- Remove bind mounting Android/ code as we want to bind mount obb dir
for each process instead.
- Set property "vold.vold.fuse_running_users" as an array of user id
for which fuse is ready to use.
- After fuse is ready for a user, fork a background process in vold
to bind mount all direct boot apps for that user so its direct boot
apps obb dir will be mounted to lower fs for imporoved performance.

Bug: 148049767
Bug: 137890172
Test: After flag is enabled, AdoptableHostTest still pass.
Change-Id: I90079fbeed1c91f9780ca71e37b0012884680b7c

Process.cpp

@@ -29,6 +29,7 @@
 #include <unistd.h>
 
 #include <fstream>
+#include <mntent.h>
 #include <unordered_set>
 
 #include <android-base/file.h>
@@ -81,6 +82,51 @@ static bool checkSymlink(const std::string& path, const std::string& prefix) {
     return false;
 }
 
+// TODO: Refactor the code with KillProcessesWithOpenFiles().
+int KillProcessesWithMounts(const std::string& prefix, int signal) {
+    std::unordered_set<pid_t> pids;
+
+    auto proc_d = std::unique_ptr<DIR, int (*)(DIR*)>(opendir("/proc"), closedir);
+    if (!proc_d) {
+        PLOG(ERROR) << "Failed to open proc";
+        return -1;
+    }
+
+    struct dirent* proc_de;
+    while ((proc_de = readdir(proc_d.get())) != nullptr) {
+        // We only care about valid PIDs
+        pid_t pid;
+        if (proc_de->d_type != DT_DIR) continue;
+        if (!android::base::ParseInt(proc_de->d_name, &pid)) continue;
+
+        // Look for references to prefix
+        std::string mounts_file(StringPrintf("/proc/%d/mounts", pid));
+        auto fp = std::unique_ptr<FILE, int (*)(FILE*)>(
+                setmntent(mounts_file.c_str(), "r"), endmntent);
+        if (!fp) {
+            PLOG(WARNING) << "Failed to open " << mounts_file;
+            continue;
+        }
+
+        // Check if obb directory is mounted, and get all packages of mounted app data directory.
+        mntent* mentry;
+        while ((mentry = getmntent(fp.get())) != nullptr) {
+            if (android::base::StartsWith(mentry->mnt_dir, prefix)) {
+                pids.insert(pid);
+                break;
+            }
+        }
+    }
+    if (signal != 0) {
+        for (const auto& pid : pids) {
+            LOG(WARNING) << "Killing pid "<< pid << " with signal " << strsignal(signal) <<
+                    " because it has a mount with prefix " << prefix;
+            kill(pid, signal);
+        }
+    }
+    return pids.size();
+}
+
 int KillProcessesWithOpenFiles(const std::string& prefix, int signal) {
     std::unordered_set<pid_t> pids;
 

Process.h

@@ -21,6 +21,7 @@ namespace android {
 namespace vold {
 
 int KillProcessesWithOpenFiles(const std::string& path, int signal);
+int KillProcessesWithMounts(const std::string& path, int signal);
 
 }  // namespace vold
 }  // namespace android

Utils.cpp

@@ -82,6 +82,9 @@ static const char* kAppDataDir = "/Android/data/";
 static const char* kAppMediaDir = "/Android/media/";
 static const char* kAppObbDir = "/Android/obb/";
 
+static const char* kMediaProviderCtx = "u:r:mediaprovider:";
+static const char* kMediaProviderAppCtx = "u:r:mediaprovider_app:";
+
 // Lock used to protect process-level SELinux changes from racing with each
 // other between multiple threads.
 static std::mutex kSecurityLock;
@@ -382,6 +385,31 @@ status_t ForceUnmount(const std::string& path) {
     return -errno;
 }
 
+status_t KillProcessesWithMountPrefix(const std::string& path) {
+    if (KillProcessesWithMounts(path, SIGINT) == 0) {
+        return OK;
+    }
+    if (sSleepOnUnmount) sleep(5);
+
+    if (KillProcessesWithMounts(path, SIGTERM) == 0) {
+        return OK;
+    }
+    if (sSleepOnUnmount) sleep(5);
+
+    if (KillProcessesWithMounts(path, SIGKILL) == 0) {
+        return OK;
+    }
+    if (sSleepOnUnmount) sleep(5);
+
+    // Send SIGKILL a second time to determine if we've
+    // actually killed everyone mount
+    if (KillProcessesWithMounts(path, SIGKILL) == 0) {
+        return OK;
+    }
+    PLOG(ERROR) << "Failed to kill processes using " << path;
+    return -EBUSY;
+}
+
 status_t KillProcessesUsingPath(const std::string& path) {
     if (KillProcessesWithOpenFiles(path, SIGINT) == 0) {
         return OK;
@@ -839,6 +867,19 @@ uint64_t GetTreeBytes(const std::string& path) {
     }
 }
 
+// TODO: Use a better way to determine if it's media provider app.
+bool IsFuseDaemon(const pid_t pid) {
+    auto path = StringPrintf("/proc/%d/mounts", pid);
+    char* tmp;
+    if (lgetfilecon(path.c_str(), &tmp) < 0) {
+        return false;
+    }
+    bool result = android::base::StartsWith(tmp, kMediaProviderAppCtx)
+            || android::base::StartsWith(tmp, kMediaProviderCtx);
+    freecon(tmp);
+    return result;
+}
+
 bool IsFilesystemSupported(const std::string& fsType) {
     std::string supported;
     if (!ReadFileToString(kProcFilesystems, &supported)) {
@@ -1198,6 +1239,16 @@ bool writeStringToFile(const std::string& payload, const std::string& filename)
     return true;
 }
 
+status_t EnsureDirExists(const std::string& path, mode_t mode, uid_t uid, gid_t gid) {
+    if (access(path.c_str(), F_OK) != 0) {
+        PLOG(WARNING) << "Dir does not exist: " << path;
+        if (fs_prepare_dir(path.c_str(), mode, uid, gid) != 0) {
+            return -errno;
+        }
+    }
+    return OK;
+}
+
 status_t MountUserFuse(userid_t user_id, const std::string& absolute_lower_path,
                        const std::string& relative_upper_path, android::base::unique_fd* fuse_fd) {
     std::string pre_fuse_path(StringPrintf("/mnt/user/%d", user_id));

Utils.h

@@ -35,6 +35,7 @@ namespace android {
 namespace vold {
 
 static const char* kPropFuse = "persist.sys.fuse";
+static const char* kVoldAppDataIsolationEnabled = "persist.sys.vold_app_data_isolation_enabled";
 
 /* SELinux contexts used depending on the block device type */
 extern security_context_t sBlkidContext;
@@ -68,6 +69,9 @@ status_t ForceUnmount(const std::string& path);
 /* Kills any processes using given path */
 status_t KillProcessesUsingPath(const std::string& path);
 
+/* Kills any processes using given mount prifix */
+status_t KillProcessesWithMountPrefix(const std::string& path);
+
 /* Creates bind mount from source to target */
 status_t BindMount(const std::string& source, const std::string& target);
 
@@ -119,6 +123,7 @@ uint64_t GetFreeBytes(const std::string& path);
 uint64_t GetTreeBytes(const std::string& path);
 
 bool IsFilesystemSupported(const std::string& fsType);
+bool IsFuseDaemon(const pid_t pid);
 
 /* Wipes contents of block device at given path */
 status_t WipeBlockDevice(const std::string& path);
@@ -142,6 +147,8 @@ std::string BuildDataUserDePath(const std::string& volumeUuid, userid_t userid);
 
 dev_t GetDevice(const std::string& path);
 
+status_t EnsureDirExists(const std::string& path, mode_t mode, uid_t uid, gid_t gid);
+
 status_t RestoreconRecursive(const std::string& path);
 
 // TODO: promote to android::base

VolumeManager.cpp

@@ -80,6 +80,7 @@ using android::vold::BindMount;
 using android::vold::CreateDir;
 using android::vold::DeleteDirContents;
 using android::vold::DeleteDirContentsAndDir;
+using android::vold::EnsureDirExists;
 using android::vold::IsFilesystemSupported;
 using android::vold::PrepareAndroidDirs;
 using android::vold::PrepareAppDirFromRoot;
@@ -104,6 +105,8 @@ static const unsigned int kMajorBlockMmc = 179;
 static const unsigned int kMajorBlockExperimentalMin = 240;
 static const unsigned int kMajorBlockExperimentalMax = 254;
 
+using ScanProcCallback = bool(*)(uid_t uid, pid_t pid, int nsFd, const char* name, void* params);
+
 VolumeManager* VolumeManager::sInstance = NULL;
 
 VolumeManager* VolumeManager::Instance() {
@@ -534,9 +537,9 @@ int VolumeManager::setPrimary(const std::shared_ptr<android::vold::VolumeBase>&
 // TODO: Get rid of this guesswork altogether and instead exec a process
 // immediately after fork to do our bindding for us.
 static bool childProcess(const char* storageSource, const char* userSource, int nsFd,
-                         struct dirent* de) {
+                         const char* name) {
     if (setns(nsFd, CLONE_NEWNS) != 0) {
-        async_safe_format_log(ANDROID_LOG_ERROR, "vold", "Failed to setns for %s :%s", de->d_name,
+        async_safe_format_log(ANDROID_LOG_ERROR, "vold", "Failed to setns for %s :%s", name,
                               strerror(errno));
         return false;
     }
@@ -553,59 +556,82 @@ static bool childProcess(const char* storageSource, const char* userSource, int
 
     if (TEMP_FAILURE_RETRY(mount(storageSource, "/storage", NULL, MS_BIND | MS_REC, NULL)) == -1) {
         async_safe_format_log(ANDROID_LOG_ERROR, "vold", "Failed to mount %s for %s :%s",
-                              storageSource, de->d_name, strerror(errno));
+                              storageSource, name, strerror(errno));
         return false;
     }
 
     if (TEMP_FAILURE_RETRY(mount(NULL, "/storage", NULL, MS_REC | MS_SLAVE, NULL)) == -1) {
         async_safe_format_log(ANDROID_LOG_ERROR, "vold",
-                              "Failed to set MS_SLAVE to /storage for %s :%s", de->d_name,
+                              "Failed to set MS_SLAVE to /storage for %s :%s", name,
                               strerror(errno));
         return false;
     }
 
     if (TEMP_FAILURE_RETRY(mount(userSource, "/storage/self", NULL, MS_BIND, NULL)) == -1) {
         async_safe_format_log(ANDROID_LOG_ERROR, "vold", "Failed to mount %s for %s :%s",
-                              userSource, de->d_name, strerror(errno));
+                              userSource, name, strerror(errno));
         return false;
     }
 
     return true;
 }
 
-int VolumeManager::remountUid(uid_t uid, int32_t mountMode) {
-    if (GetBoolProperty(android::vold::kPropFuse, false)) {
-        // TODO(135341433): Implement fuse specific logic.
-        return 0;
-    }
-    std::string mode;
+// Fork the process and remount storage
+bool forkAndRemountChild(uid_t uid, pid_t pid, int nsFd, const char* name, void* params) {
+    int32_t mountMode = *static_cast<int32_t*>(params);
+    std::string userSource;
+    std::string storageSource;
+    pid_t child;
+    // Need to fix these paths to account for when sdcardfs is gone
     switch (mountMode) {
         case VoldNativeService::REMOUNT_MODE_NONE:
-            mode = "none";
-            break;
+            return true;
         case VoldNativeService::REMOUNT_MODE_DEFAULT:
-            mode = "default";
+            storageSource = "/mnt/runtime/default";
             break;
         case VoldNativeService::REMOUNT_MODE_READ:
-            mode = "read";
+            storageSource = "/mnt/runtime/read";
             break;
         case VoldNativeService::REMOUNT_MODE_WRITE:
         case VoldNativeService::REMOUNT_MODE_LEGACY:
         case VoldNativeService::REMOUNT_MODE_INSTALLER:
-            mode = "write";
+            storageSource = "/mnt/runtime/write";
             break;
         case VoldNativeService::REMOUNT_MODE_FULL:
-            mode = "full";
+            storageSource = "/mnt/runtime/full";
             break;
         case VoldNativeService::REMOUNT_MODE_PASS_THROUGH:
-            mode = "pass_through";
-            break;
+            return true;
         default:
             PLOG(ERROR) << "Unknown mode " << std::to_string(mountMode);
-            return -1;
+            return false;
+    }
+    LOG(DEBUG) << "Remounting " << uid << " as " << storageSource;
+
+    // Fork a child to mount user-specific symlink helper into place
+    userSource = StringPrintf("/mnt/user/%d", multiuser_get_user_id(uid));
+    if (!(child = fork())) {
+        if (childProcess(storageSource.c_str(), userSource.c_str(), nsFd, name)) {
+            _exit(0);
+        } else {
+            _exit(1);
+        }
     }
-    LOG(DEBUG) << "Remounting " << uid << " as mode " << mode;
 
+    if (child == -1) {
+        PLOG(ERROR) << "Failed to fork";
+        return false;
+    } else {
+        TEMP_FAILURE_RETRY(waitpid(child, nullptr, 0));
+    }
+    return true;
+}
+
+// Helper function to scan all processes in /proc and call the callback if:
+// 1). pid belongs to an app process
+// 2). If input uid is 0 or it matches the process uid
+// 3). If userId is not -1 or userId matches the process userId
+bool scanProcProcesses(uid_t uid, userid_t userId, ScanProcCallback callback, void* params) {
     DIR* dir;
     struct dirent* de;
     std::string rootName;
@@ -613,24 +639,22 @@ int VolumeManager::remountUid(uid_t uid, int32_t mountMode) {
     int pidFd;
     int nsFd;
     struct stat sb;
-    pid_t child;
-    std::string userSource;
-    std::string storageSource;
 
     static bool apexUpdatable = android::sysprop::ApexProperties::updatable().value_or(false);
 
     if (!(dir = opendir("/proc"))) {
-        PLOG(ERROR) << "Failed to opendir";
-        return -1;
+        async_safe_format_log(ANDROID_LOG_ERROR, "vold", "Failed to opendir");
+        return false;
     }
 
     // Figure out root namespace to compare against below
     if (!android::vold::Readlinkat(dirfd(dir), "1/ns/mnt", &rootName)) {
-        PLOG(ERROR) << "Failed to read root namespace";
+        async_safe_format_log(ANDROID_LOG_ERROR, "vold", "Failed to read root namespace");
         closedir(dir);
-        return -1;
+        return false;
     }
 
+    async_safe_format_log(ANDROID_LOG_INFO, "vold", "Start scanning all processes");
     // Poke through all running PIDs look for apps running as UID
     while ((de = readdir(dir))) {
         pid_t pid;
@@ -645,21 +669,23 @@ int VolumeManager::remountUid(uid_t uid, int32_t mountMode) {
             goto next;
         }
         if (fstat(pidFd, &sb) != 0) {
-            PLOG(WARNING) << "Failed to stat " << de->d_name;
+            async_safe_format_log(ANDROID_LOG_ERROR, "vold", "Failed to stat %s", de->d_name);
             goto next;
         }
-        if (sb.st_uid != uid) {
+        if (uid != 0 && sb.st_uid != uid) {
+            goto next;
+        }
+        if (userId != static_cast<userid_t>(-1) && multiuser_get_user_id(sb.st_uid) != userId) {
             goto next;
         }
 
         // Matches so far, but refuse to touch if in root namespace
-        LOG(DEBUG) << "Found matching PID " << de->d_name;
         if (!android::vold::Readlinkat(pidFd, "ns/mnt", &pidName)) {
-            PLOG(WARNING) << "Failed to read namespace for " << de->d_name;
+            async_safe_format_log(ANDROID_LOG_ERROR, "vold",
+                    "Failed to read namespacefor %s", de->d_name);
             goto next;
         }
         if (rootName == pidName) {
-            LOG(WARNING) << "Skipping due to root namespace";
             goto next;
         }
 
@@ -674,11 +700,9 @@ int VolumeManager::remountUid(uid_t uid, int32_t mountMode) {
             // non-Java process whose UID is < AID_APP_START. (The UID condition
             // is required to not filter out child processes spawned by apps.)
             if (!android::vold::Readlinkat(pidFd, "exe", &exeName)) {
-                PLOG(WARNING) << "Failed to read exe name for " << de->d_name;
                 goto next;
             }
             if (!StartsWith(exeName, "/system/bin/app_process") && sb.st_uid < AID_APP_START) {
-                LOG(WARNING) << "Skipping due to native system process";
                 goto next;
             }
         }
@@ -687,49 +711,218 @@ int VolumeManager::remountUid(uid_t uid, int32_t mountMode) {
         // NOLINTNEXTLINE(android-cloexec-open): Deliberately not O_CLOEXEC
         nsFd = openat(pidFd, "ns/mnt", O_RDONLY);
         if (nsFd < 0) {
-            PLOG(WARNING) << "Failed to open namespace for " << de->d_name;
+            async_safe_format_log(ANDROID_LOG_ERROR, "vold",
+                    "Failed to open namespace for %s", de->d_name);
             goto next;
         }
 
-        if (mode == "default") {
-            storageSource = "/mnt/runtime/default";
-        } else if (mode == "read") {
-            storageSource = "/mnt/runtime/read";
-        } else if (mode == "write") {
-            storageSource = "/mnt/runtime/write";
-        } else if (mode == "full") {
-            storageSource = "/mnt/runtime/full";
-        } else {
-            // Sane default of no storage visible. No need to fork a child
-            // to remount uid.
-            goto next;
+        if (!callback(sb.st_uid, pid, nsFd, de->d_name, params)) {
+            async_safe_format_log(ANDROID_LOG_ERROR, "vold", "Failed in callback");
         }
 
-        // Mount user-specific symlink helper into place
-        userSource = StringPrintf("/mnt/user/%d", multiuser_get_user_id(uid));
-        if (!(child = fork())) {
-            if (childProcess(storageSource.c_str(), userSource.c_str(), nsFd, de)) {
-                _exit(0);
-            } else {
-                _exit(1);
-            }
+    next:
+        close(nsFd);
+        close(pidFd);
+    }
+    closedir(dir);
+    async_safe_format_log(ANDROID_LOG_INFO, "vold", "Finished scanning all processes");
+    return true;
+}
+
+int VolumeManager::remountUid(uid_t uid, int32_t mountMode) {
+    if (GetBoolProperty(android::vold::kPropFuse, false)) {
+        // TODO(135341433): Implement fuse specific logic.
+        return 0;
+    }
+    return scanProcProcesses(uid, static_cast<userid_t>(-1),
+            forkAndRemountChild, &mountMode) ? 0 : -1;
+}
+
+// Bind mount obb dir for an app if necessary.
+// How it works:
+// 1). Check if a pid is an app uid and not the FuseDaemon, if not then return.
+// 2). Get the mounts for that pid.
+// 3). If obb is already mounted then return, otherwise we need to mount obb for this pid.
+// 4). Get all packages and uid mounted for jit profile. These packages are all packages with
+// same uid or whitelisted apps.
+// 5a). If there's no package, it means it's not a process running app data isolation, so
+// just bind mount Android/obb dir.
+// 5b). Otherwise, for each package, create obb dir if it's not created and bind mount it.
+// TODO: Should we get some reliable data from system server instead of scanning /proc ?
+static bool bindMountAppObbDir(uid_t uid, pid_t pid, int nsFd, const char* name, void* params) {
+    if (uid < AID_APP_START || uid > AID_APP_END) {
+        return true;
+    }
+    if (android::vold::IsFuseDaemon(pid)) {
+        return true;
+    }
+    async_safe_format_log(ANDROID_LOG_ERROR, "vold", "Start mounting obb for uid:%d, pid:%d", uid,
+                          pid);
+
+    userid_t userId = multiuser_get_user_id(uid);
+    if (setns(nsFd, CLONE_NEWNS) != 0) {
+        async_safe_format_log(ANDROID_LOG_ERROR, "vold", "Failed to setns %s", strerror(errno));
+        return false;
+    }
+
+    std::string profiles_path(StringPrintf("/data/misc/profiles/cur/%d/", userId));
+    // We search both .../obb and .../obb/$PKG paths here.
+    std::string obb_path(StringPrintf("/storage/emulated/%d/Android/obb", userId));
+    int profiles_path_len = profiles_path.length();
+    int obb_path_len = obb_path.length();
+
+    // TODO: Refactor the code as a util function so we can reuse the mount parsing code.
+    std::string mounts_file(StringPrintf("/proc/%d/mounts", pid));
+    auto fp = std::unique_ptr<FILE, int (*)(FILE*)>(
+            setmntent(mounts_file.c_str(), "r"), endmntent);
+    if (!fp) {
+        async_safe_format_log(ANDROID_LOG_ERROR, "vold", "Error opening %s: %s",
+                              mounts_file.c_str(), strerror(errno));
+        return false;
+    }
+
+    // Check if obb directory is mounted, and get all packages of mounted app data directory.
+    bool obb_mounted = false;
+    std::vector<std::string> pkg_name_list;
+    mntent* mentry;
+    while ((mentry = getmntent(fp.get())) != nullptr) {
+        if (strncmp(mentry->mnt_dir, profiles_path.c_str(), profiles_path_len) == 0) {
+            pkg_name_list.push_back(std::string(mentry->mnt_dir + profiles_path_len));
         }
+        if (strncmp(mentry->mnt_dir, obb_path.c_str(), obb_path_len) == 0) {
+            obb_mounted = true;
+        }
+    }
 
-        if (child == -1) {
-            PLOG(ERROR) << "Failed to fork";
-            goto next;
+    // Obb mounted in zygote already, so skip it
+    if (obb_mounted) {
+        return true;
+    }
+
+    // Ensure obb parent directory exists
+    std::string obbSource;
+    if (IsFilesystemSupported("sdcardfs")) {
+        obbSource = StringPrintf("/mnt/runtime/default/emulated/%d/Android/obb", userId);
+    } else {
+        obbSource = StringPrintf("/mnt/pass_through/%d/emulated/%d/Android/obb", userId, userId);
+    }
+    std::string obbTarget(StringPrintf("/storage/emulated/%d/Android/obb", userId));
+    auto status = EnsureDirExists(obbSource, 0771, AID_MEDIA_RW, AID_MEDIA_RW);
+    if (status != OK) {
+        async_safe_format_log(ANDROID_LOG_ERROR, "vold", "Failed to create dir %s %s",
+                              obbSource.c_str(), strerror(-status));
+        return false;
+    }
+
+    // It means app data isolation is not applied to this, so we can just bind the whole obb
+    // directory instead.
+    if (pkg_name_list.empty()) {
+        async_safe_format_log(ANDROID_LOG_INFO, "vold",
+                              "Bind mounting whole obb directory for pid %d", pid);
+        status = BindMount(obbSource, obbTarget);
+        if (status != OK) {
+            async_safe_format_log(ANDROID_LOG_ERROR, "vold", "Failed to mount %s %s %s",
+                                  obbSource.c_str(), obbTarget.c_str(), strerror(-status));
+            return false;
+        }
+        return true;
+    }
+
+    // Bind mount each app's obb directory
+    for (const auto& pkg_name : pkg_name_list) {
+        std::string appObbSource;
+        if (IsFilesystemSupported("sdcardfs")) {
+            appObbSource = StringPrintf("/mnt/runtime/default/emulated/%d/Android/obb/%s",
+                    userId, pkg_name.c_str());
         } else {
-            TEMP_FAILURE_RETRY(waitpid(child, nullptr, 0));
+            appObbSource = StringPrintf("/mnt/pass_through/%d/emulated/%d/Android/obb/%s",
+                    userId, userId, pkg_name.c_str());
         }
+        std::string appObbTarget(StringPrintf("/storage/emulated/%d/Android/obb/%s",
+                userId, pkg_name.c_str()));
+
+        status = EnsureDirExists(appObbSource, 0770, uid, AID_MEDIA_RW);
+        if (status != OK) {
+            async_safe_format_log(ANDROID_LOG_INFO, "vold", "Failed to ensure dir %s exists",
+                                  appObbSource.c_str());
+            continue;
+        }
+        async_safe_format_log(ANDROID_LOG_INFO, "vold",
+                              "Bind mounting app obb directory(%s) for pid %d", pkg_name.c_str(),
+                              pid);
+        status = BindMount(appObbSource, appObbTarget);
+        if (status != OK) {
+            async_safe_format_log(ANDROID_LOG_ERROR, "vold", "Failed to mount %s %s %s",
+                                  obbSource.c_str(), obbTarget.c_str(), strerror(-status));
+            continue;
+        }
+    }
+    return true;
+}
 
-    next:
-        close(nsFd);
-        close(pidFd);
+int VolumeManager::remountAppObb(userid_t userId) {
+    if (!GetBoolProperty(android::vold::kPropFuse, false)) {
+        return 0;
+    }
+    LOG(INFO) << "Start remounting app obb";
+    pid_t child;
+    if (!(child = fork())) {
+        // Child process
+        if (daemon(0, 0) == -1) {
+            PLOG(FATAL) << "Cannot create daemon";
+        }
+        // TODO(149548518): Refactor the code so minimize the work after fork to prevent deadlock.
+        if (scanProcProcesses(0, userId, bindMountAppObbDir, nullptr)) {
+            // As some forked zygote processes may not setuid and recognized as an app yet, sleep
+            // 3s and try again to catch 'em all.
+            usleep(3 * 1000 * 1000);  // 3s
+            async_safe_format_log(ANDROID_LOG_ERROR, "vold", "Retry remounting app obb");
+            scanProcProcesses(0, userId, bindMountAppObbDir, nullptr);
+            _exit(0);
+        } else {
+            _exit(1);
+        }
+    }
+    if (child == -1) {
+        PLOG(ERROR) << "Failed to fork";
+        return -1;
+    } else if (child == 0) {
+        // Parent
+        int stat_loc;
+        for (;;) {
+            if (waitpid(child, &stat_loc, 0) != -1 || errno != EINTR) {
+                break;
+            }
+        }
     }
-    closedir(dir);
     return 0;
 }
 
+bool VolumeManager::updateFuseMountedProperty() {
+    if (mFuseMountedUsers.size() == 0) {
+        android::base::SetProperty("vold.fuse_running_users", "");
+        return true;
+    }
+    std::stringstream stream;
+    char const * sep = "";
+    for (const auto& userId : mFuseMountedUsers) {
+        stream << sep;
+        stream << userId;
+        sep = ", ";
+    }
+    return android::base::SetProperty("vold.fuse_running_users", stream.str());
+}
+
+bool VolumeManager::addFuseMountedUser(userid_t userId) {
+    mFuseMountedUsers.insert(userId);
+    return updateFuseMountedProperty();
+}
+
+bool VolumeManager::removeFuseMountedUser(userid_t userId) {
+    mFuseMountedUsers.erase(userId);
+    return updateFuseMountedProperty();
+}
+
 int VolumeManager::reset() {
     // Tear down all existing disks/volumes and start from a blank slate so
     // newly connected framework hears all events.
@@ -745,6 +938,8 @@ int VolumeManager::reset() {
     updateVirtualDisk();
     mAddedUsers.clear();
     mStartedUsers.clear();
+    mFuseMountedUsers.clear();
+    updateFuseMountedProperty();
     return 0;
 }
 
@@ -764,6 +959,8 @@ int VolumeManager::shutdown() {
     mInternalEmulatedVolumes.clear();
     mDisks.clear();
     mPendingDisks.clear();
+    mFuseMountedUsers.clear();
+    updateFuseMountedProperty();
     android::vold::sSleepOnUnmount = true;
     return 0;
 }

VolumeManager.h

@@ -118,6 +118,10 @@ class VolumeManager {
     int setPrimary(const std::shared_ptr<android::vold::VolumeBase>& vol);
 
     int remountUid(uid_t uid, int32_t remountMode);
+    int remountAppObb(userid_t userId);
+
+    bool addFuseMountedUser(userid_t userId);
+    bool removeFuseMountedUser(userid_t userId);
 
     /* Reset all internal state, typically during framework boot */
     int reset();
@@ -190,6 +194,8 @@ class VolumeManager {
     void handleDiskChanged(dev_t device);
     void handleDiskRemoved(dev_t device);
 
+    bool updateFuseMountedProperty();
+
     std::mutex mLock;
     std::mutex mCryptLock;
 
@@ -213,6 +219,9 @@ class VolumeManager {
     int mNextObbId;
     int mNextStubId;
     bool mSecureKeyguardShowing;
+
+    // Set of all user id that fuse is ready to use.
+    std::unordered_set<userid_t> mFuseMountedUsers;
 };
 
 #endif

model/EmulatedVolume.cpp

@@ -49,6 +49,7 @@ EmulatedVolume::EmulatedVolume(const std::string& rawPath, int userId)
     mLabel = "emulated";
     mFuseMounted = false;
     mUseSdcardFs = IsFilesystemSupported("sdcardfs");
+    mAppDataIsolationEnabled = base::GetBoolProperty(kVoldAppDataIsolationEnabled, false);
 }
 
 EmulatedVolume::EmulatedVolume(const std::string& rawPath, dev_t device, const std::string& fsUuid,
@@ -59,6 +60,7 @@ EmulatedVolume::EmulatedVolume(const std::string& rawPath, dev_t device, const s
     mLabel = fsUuid;
     mFuseMounted = false;
     mUseSdcardFs = IsFilesystemSupported("sdcardfs");
+    mAppDataIsolationEnabled = base::GetBoolProperty(kVoldAppDataIsolationEnabled, false);
 }
 
 EmulatedVolume::~EmulatedVolume() {}
@@ -94,14 +96,19 @@ status_t EmulatedVolume::mountFuseBindMounts() {
     } else {
         androidSource = StringPrintf("/%s/%d/Android", mRawPath.c_str(), userId);
     }
-    std::string androidTarget(
+
+    status_t status = OK;
+    // When app data isolation is enabled, obb/ will be mounted per app, otherwise we should
+    // bind mount the whole Android/ to speed up reading.
+    if (!mAppDataIsolationEnabled) {
+        std::string androidTarget(
             StringPrintf("/mnt/user/%d/%s/%d/Android", userId, label.c_str(), userId));
+        status = doFuseBindMount(androidSource, androidTarget);
+    }
 
-    auto status = doFuseBindMount(androidSource, androidTarget);
     if (status != OK) {
         return status;
     }
-
     // Installers get the same view as all other apps, with the sole exception that the
     // OBB dirs (Android/obb) are writable to them. On sdcardfs devices, this requires
     // a special bind mount, since app-private and OBB dirs share the same GID, but we
@@ -118,6 +125,19 @@ status_t EmulatedVolume::mountFuseBindMounts() {
     if (status != OK) {
         return status;
     }
+
+    if (mAppDataIsolationEnabled) {
+        // Starting from now, fuse is running, and zygote will bind app obb data directory
+        if (!VolumeManager::Instance()->addFuseMountedUser(userId)) {
+            return UNKNOWN_ERROR;
+        }
+
+        // As all new processes created by zygote will bind app obb data directory, we just need
+        // to have a snapshot of all existing processes and see if any existing process needs to
+        // remount obb data directory.
+        VolumeManager::Instance()->remountAppObb(userId);
+    }
+
     return OK;
 }
 
@@ -135,16 +155,22 @@ status_t EmulatedVolume::unmountFuseBindMounts() {
             // Intentional continue to try to unmount the other bind mount
         }
     }
+    // When app data isolation is enabled, kill all apps that obb/ is mounted, otherwise we should
+    // umount the whole Android/ dir.
+    if (mAppDataIsolationEnabled) {
+        std::string appObbDir(StringPrintf("%s/%d/Android/obb", getPath().c_str(), userId));
+        KillProcessesWithMountPrefix(appObbDir);
+    } else {
+        std::string androidTarget(
+                StringPrintf("/mnt/user/%d/%s/%d/Android", userId, label.c_str(), userId));
 
-    std::string androidTarget(
-            StringPrintf("/mnt/user/%d/%s/%d/Android", userId, label.c_str(), userId));
-
-    LOG(INFO) << "Unmounting " << androidTarget;
-    auto status = UnmountTree(androidTarget);
-    if (status != OK) {
-        return status;
+        LOG(INFO) << "Unmounting " << androidTarget;
+        auto status = UnmountTree(androidTarget);
+        if (status != OK) {
+            return status;
+        }
+        LOG(INFO) << "Unmounted " << androidTarget;
     }
-    LOG(INFO) << "Unmounted " << androidTarget;
 
     return OK;
 }
@@ -281,9 +307,15 @@ status_t EmulatedVolume::doUnmount() {
 
     if (mFuseMounted) {
         std::string label = getLabel();
+
+        // Update fuse mounted record
+        if (mAppDataIsolationEnabled &&
+                !VolumeManager::Instance()->removeFuseMountedUser(userId)) {
+            return UNKNOWN_ERROR;
+        }
+
         // Ignoring unmount return status because we do want to try to unmount
         // the rest cleanly.
-
         unmountFuseBindMounts();
         if (UnmountUserFuse(userId, getInternalPath(), label) != OK) {
             PLOG(INFO) << "UnmountUserFuse failed on emulated fuse volume";

model/EmulatedVolume.h

@@ -65,6 +65,9 @@ class EmulatedVolume : public VolumeBase {
     /* Whether to use sdcardfs for this volume */
     bool mUseSdcardFs;
 
+    /* Whether to use app data isolation is enabled tor this volume */
+    bool mAppDataIsolationEnabled;
+
     DISALLOW_COPY_AND_ASSIGN(EmulatedVolume);
 };