Allow callers to prepare CE/DE user storage.

Give callers the option of preparing CE and/or DE storage.  The
framework will only prepare CE storage after the CE keys have been
unlocked for that user.

When init is calling enablecrypto, kick off the work in a thread so
that we can make other calls back into vold without causing
deadlock.  Leaves blocking call intact for framework callers.

Clean up 'vdc' tool to send useful transaction numbers, and
actually watch for the matching result to come back.  This fixes
race conditions when there are multiple 'vdc' callers.

Also add other system and misc directories to match spec.

Bug: 25796509
Change-Id: Ie4f853db6e387916b845d2b5fb92925d743b063d

Android.mk

@@ -88,7 +88,7 @@ include $(BUILD_STATIC_LIBRARY)
 include $(CLEAR_VARS)
 
 LOCAL_ADDITIONAL_DEPENDENCIES := $(LOCAL_PATH)/Android.mk
-LOCAL_MODULE:= vold
+LOCAL_MODULE := vold
 LOCAL_CLANG := true
 LOCAL_SRC_FILES := \
 	main.cpp \
@@ -115,9 +115,9 @@ include $(CLEAR_VARS)
 
 LOCAL_ADDITIONAL_DEPENDENCIES := $(LOCAL_PATH)/Android.mk
 LOCAL_CLANG := true
-LOCAL_SRC_FILES:= vdc.cpp
-LOCAL_MODULE:= vdc
-LOCAL_SHARED_LIBRARIES := libcutils
+LOCAL_SRC_FILES := vdc.cpp
+LOCAL_MODULE := vdc
+LOCAL_SHARED_LIBRARIES := libcutils libbase
 LOCAL_CFLAGS := $(vold_cflags)
 LOCAL_CONLYFLAGS := $(vold_conlyflags)
 LOCAL_INIT_RC := vdc.rc

CryptCommandListener.cpp

@@ -30,6 +30,7 @@
 #include <inttypes.h>
 
 #include <algorithm>
+#include <thread>
 
 #define LOG_TAG "VoldCryptCmdListener"
 
@@ -144,6 +145,25 @@ static bool check_argc(SocketClient *cli, const std::string &subcommand, int arg
     return false;
 }
 
+static int do_enablecrypto(char** argv, int type, bool no_ui) {
+    int rc;
+    int tries;
+    for (tries = 0; tries < 2; ++tries) {
+        if (type == CRYPT_TYPE_DEFAULT) {
+            rc = cryptfs_enable_default(argv[2], no_ui);
+        } else {
+            rc = cryptfs_enable(argv[2], type, argv[4], no_ui);
+        }
+
+        if (rc == 0) {
+            return 0;
+        } else if (tries == 0) {
+            Process::killProcessesWithOpenFiles(DATA_MNT_POINT, SIGKILL);
+        }
+    }
+    return -1;
+}
+
 int CryptCommandListener::CryptfsCmd::runCommand(SocketClient *cli,
                                                  int argc, char **argv) {
     if ((cli->getUid() != 0) && (cli->getUid() != AID_SYSTEM)) {
@@ -166,7 +186,10 @@ int CryptCommandListener::CryptfsCmd::runCommand(SocketClient *cli,
     } else if (subcommand == "restart") {
         if (!check_argc(cli, subcommand, argc, 2, "")) return 0;
         dumpArgs(argc, argv, -1);
-        rc = cryptfs_restart();
+
+        // Spawn as thread so init can issue commands back to vold without
+        // causing deadlock, usually as a result of prep_data_fs.
+        std::thread(&cryptfs_restart).detach();
     } else if (subcommand == "cryptocomplete") {
         if (!check_argc(cli, subcommand, argc, 2, "")) return 0;
         dumpArgs(argc, argv, -1);
@@ -216,31 +239,16 @@ int CryptCommandListener::CryptfsCmd::runCommand(SocketClient *cli,
             }
         }
 
-        if (!valid ) {
+        if (!valid) {
             cli->sendMsg(ResponseCode::CommandSyntaxError, syntax, false);
             return 0;
         }
 
         dumpArgs(argc, argv, 4);
 
-        int tries;
-        for (tries = 0; tries < 2; ++tries) {
-            if (type == -1) {
-                cli->sendMsg(ResponseCode::CommandSyntaxError, syntax,
-                             false);
-                return 0;
-            } else if (type == CRYPT_TYPE_DEFAULT) {
-              rc = cryptfs_enable_default(argv[2], no_ui);
-            } else {
-                rc = cryptfs_enable(argv[2], type, argv[4], no_ui);
-            }
-
-            if (rc == 0) {
-                break;
-            } else if (tries == 0) {
-                Process::killProcessesWithOpenFiles(DATA_MNT_POINT, SIGKILL);
-            }
-        }
+        // Spawn as thread so init can issue commands back to vold without
+        // causing deadlock, usually as a result of prep_data_fs.
+        std::thread(&do_enablecrypto, argv, type, no_ui).detach();
     } else if (subcommand == "enablefilecrypto") {
         if (!check_argc(cli, subcommand, argc, 2, "")) return 0;
         dumpArgs(argc, argv, -1);
@@ -301,7 +309,10 @@ int CryptCommandListener::CryptfsCmd::runCommand(SocketClient *cli,
         if (!check_argc(cli, subcommand, argc, 2, "")) return 0;
         SLOGD("cryptfs mountdefaultencrypted");
         dumpArgs(argc, argv, -1);
-        rc = cryptfs_mount_default_encrypted();
+
+        // Spawn as thread so init can issue commands back to vold without
+        // causing deadlock, usually as a result of prep_data_fs.
+        std::thread(&cryptfs_mount_default_encrypted).detach();
     } else if (subcommand == "getpwtype") {
         if (!check_argc(cli, subcommand, argc, 2, "")) return 0;
         SLOGD("cryptfs getpwtype");
@@ -379,12 +390,12 @@ int CryptCommandListener::CryptfsCmd::runCommand(SocketClient *cli,
         return sendGenericOkFail(cli, e4crypt_lock_user_key(atoi(argv[2])));
 
     } else if (subcommand == "prepare_user_storage") {
-        if (!check_argc(cli, subcommand, argc, 6, "<uuid> <user> <serial> <ephemeral>")) return 0;
+        if (!check_argc(cli, subcommand, argc, 6, "<uuid> <user> <serial> <flags>")) return 0;
         return sendGenericOkFail(cli,
                                  e4crypt_prepare_user_storage(parseNull(argv[2]),
                                                               atoi(argv[3]),
                                                               atoi(argv[4]),
-                                                              atoi(argv[5]) != 0));
+                                                              atoi(argv[5])));
 
     } else {
         dumpArgs(argc, argv, -1);

Ext4Crypt.cpp

@@ -40,7 +40,7 @@
 
 #include "key_control.h"
 #include "cryptfs.h"
-#include "ext4_crypt_init_extensions.h"
+#include "ext4_crypt.h"
 
 #define LOG_TAG "Ext4Crypt"
 
@@ -56,6 +56,10 @@
 
 using android::base::StringPrintf;
 
+// NOTE: keep in sync with StorageManager
+static constexpr int FLAG_STORAGE_DE = 1 << 0;
+static constexpr int FLAG_STORAGE_CE = 1 << 1;
+
 static bool e4crypt_is_native() {
     char value[PROPERTY_VALUE_MAX];
     property_get("ro.crypto.type", value, "none");
@@ -66,6 +70,10 @@ static bool e4crypt_is_emulated() {
     return property_get_bool("persist.sys.emulate_fbe", false);
 }
 
+static const char* escape_null(const char* value) {
+    return (value == nullptr) ? "null" : value;
+}
+
 namespace {
     // Key length in bits
     const int key_length = 128;
@@ -282,8 +290,8 @@ static bool lookup_key_ref(const std::map<userid_t, std::string> &key_map,
     return true;
 }
 
-static bool set_policy(const std::string &raw_ref, const std::string& path) {
-    if (do_policy_set(path.c_str(), raw_ref.data(), raw_ref.size()) != 0) {
+static bool ensure_policy(const std::string &raw_ref, const std::string& path) {
+    if (e4crypt_policy_ensure(path.c_str(), raw_ref.data(), raw_ref.size()) != 0) {
         LOG(ERROR) << "Failed to set policy on: " << path;
         return false;
     }
@@ -395,13 +403,17 @@ int e4crypt_init_user0() {
             }
             if (!create_and_install_user_keys(0, false)) return -1;
         }
+        // TODO: switch to loading only DE_0 here once framework makes
+        // explicit calls to install DE keys for secondary users
         if (!load_all_de_keys()) return -1;
     }
-    // Ignore failures. FIXME this is horrid
-    // FIXME: we need an idempotent policy-setting call, which simply verifies the
-    // policy is already set on a second run, even if the directory is nonempty.
-    // Then we need to call it all the time.
-    e4crypt_prepare_user_storage(nullptr, 0, 0, false);
+    // We can only safely prepare DE storage here, since CE keys are probably
+    // entangled with user credentials.  The framework will always prepare CE
+    // storage once CE keys are installed.
+    if (e4crypt_prepare_user_storage(nullptr, 0, 0, FLAG_STORAGE_DE) != 0) {
+        LOG(ERROR) << "Failed to prepare user 0 storage";
+        return -1;
+    }
     return 0;
 }
 
@@ -484,6 +496,7 @@ static int emulated_unlock(const std::string& path, mode_t mode) {
     return 0;
 }
 
+// TODO: rename to 'install' for consistency, and take flags to know which keys to install
 int e4crypt_unlock_user_key(userid_t user_id, int serial, const char* token) {
     LOG(DEBUG) << "e4crypt_unlock_user_key " << user_id << " " << (token != nullptr);
     if (e4crypt_is_native()) {
@@ -505,6 +518,7 @@ int e4crypt_unlock_user_key(userid_t user_id, int serial, const char* token) {
     return 0;
 }
 
+// TODO: rename to 'evict' for consistency
 int e4crypt_lock_user_key(userid_t user_id) {
     if (e4crypt_is_native()) {
         // TODO: remove from kernel keyring
@@ -521,35 +535,48 @@ int e4crypt_lock_user_key(userid_t user_id) {
     return 0;
 }
 
-int e4crypt_prepare_user_storage(const char* volume_uuid,
-                                 userid_t user_id,
-                                 int serial,
-                                 bool ephemeral) {
-    if (volume_uuid) {
-        LOG(DEBUG) << "e4crypt_prepare_user_storage " << volume_uuid << " " << user_id;
-    } else {
-        LOG(DEBUG) << "e4crypt_prepare_user_storage, null volume " << user_id;
-    }
-    auto system_ce_path = android::vold::BuildDataSystemCePath(user_id);
-    auto media_ce_path = android::vold::BuildDataMediaPath(volume_uuid, user_id);
-    auto user_ce_path = android::vold::BuildDataUserPath(volume_uuid, user_id);
-    auto user_de_path = android::vold::BuildDataUserDePath(volume_uuid, user_id);
-
-    // FIXME: should this be 0770 or 0700?
-    if (!prepare_dir(system_ce_path, 0770, AID_SYSTEM, AID_SYSTEM)) return -1;
-    if (!prepare_dir(media_ce_path, 0770, AID_MEDIA_RW, AID_MEDIA_RW)) return -1;
-    if (!prepare_dir(user_ce_path, 0771, AID_SYSTEM, AID_SYSTEM)) return -1;
-    if (!prepare_dir(user_de_path, 0771, AID_SYSTEM, AID_SYSTEM)) return -1;
-
-    if (e4crypt_crypto_complete(DATA_MNT_POINT) == 0) {
-        std::string ce_raw_ref, de_raw_ref;
-        if (!lookup_key_ref(s_ce_key_raw_refs, user_id, ce_raw_ref)) return -1;
-        if (!lookup_key_ref(s_de_key_raw_refs, user_id, de_raw_ref)) return -1;
-        if (!set_policy(ce_raw_ref, system_ce_path)) return -1;
-        if (!set_policy(ce_raw_ref, media_ce_path)) return -1;
-        if (!set_policy(ce_raw_ref, user_ce_path)) return -1;
-        if (!set_policy(de_raw_ref, user_de_path)) return -1;
-        // FIXME I thought there were more DE directories than this
+int e4crypt_prepare_user_storage(const char* volume_uuid, userid_t user_id,
+        int serial, int flags) {
+    LOG(DEBUG) << "e4crypt_prepare_user_storage for volume " << escape_null(volume_uuid)
+            << ", user " << user_id << ", serial " << serial << ", flags " << flags;
+
+    if (flags & FLAG_STORAGE_DE) {
+        auto system_de_path = android::vold::BuildDataSystemDePath(user_id);
+        auto misc_de_path = android::vold::BuildDataMiscDePath(user_id);
+        auto user_de_path = android::vold::BuildDataUserDePath(volume_uuid, user_id);
+
+        if (!prepare_dir(system_de_path, 0770, AID_SYSTEM, AID_SYSTEM)) return -1;
+        if (!prepare_dir(misc_de_path, 01771, AID_SYSTEM, AID_MISC)) return -1;
+        if (!prepare_dir(user_de_path, 0771, AID_SYSTEM, AID_SYSTEM)) return -1;
+
+        if (e4crypt_crypto_complete(DATA_MNT_POINT) == 0) {
+            std::string de_raw_ref;
+            if (!lookup_key_ref(s_de_key_raw_refs, user_id, de_raw_ref)) return -1;
+            if (!ensure_policy(de_raw_ref, system_de_path)) return -1;
+            if (!ensure_policy(de_raw_ref, misc_de_path)) return -1;
+            if (!ensure_policy(de_raw_ref, user_de_path)) return -1;
+        }
+    }
+
+    if (flags & FLAG_STORAGE_CE) {
+        auto system_ce_path = android::vold::BuildDataSystemCePath(user_id);
+        auto misc_ce_path = android::vold::BuildDataMiscCePath(user_id);
+        auto media_ce_path = android::vold::BuildDataMediaPath(volume_uuid, user_id);
+        auto user_ce_path = android::vold::BuildDataUserPath(volume_uuid, user_id);
+
+        if (!prepare_dir(system_ce_path, 0770, AID_SYSTEM, AID_SYSTEM)) return -1;
+        if (!prepare_dir(misc_ce_path, 01771, AID_SYSTEM, AID_MISC)) return -1;
+        if (!prepare_dir(media_ce_path, 0770, AID_MEDIA_RW, AID_MEDIA_RW)) return -1;
+        if (!prepare_dir(user_ce_path, 0771, AID_SYSTEM, AID_SYSTEM)) return -1;
+
+        if (e4crypt_crypto_complete(DATA_MNT_POINT) == 0) {
+            std::string ce_raw_ref;
+            if (!lookup_key_ref(s_ce_key_raw_refs, user_id, ce_raw_ref)) return -1;
+            if (!ensure_policy(ce_raw_ref, system_ce_path)) return -1;
+            if (!ensure_policy(ce_raw_ref, misc_ce_path)) return -1;
+            if (!ensure_policy(ce_raw_ref, media_ce_path)) return -1;
+            if (!ensure_policy(ce_raw_ref, user_ce_path)) return -1;
+        }
     }
 
     return 0;

Ext4Crypt.h

@@ -32,9 +32,7 @@ int e4crypt_destroy_user_key(userid_t user_id);
 int e4crypt_unlock_user_key(userid_t user_id, int serial, const char* token);
 int e4crypt_lock_user_key(userid_t user_id);
 
-int e4crypt_prepare_user_storage(const char* volume_uuid,
-                                 userid_t user_id,
-                                 int serial,
-                                 bool ephemeral);
+int e4crypt_prepare_user_storage(const char* volume_uuid, userid_t user_id,
+        int serial, int flags);
 
 __END_DECLS

Utils.cpp

@@ -561,9 +561,19 @@ std::string BuildKeyPath(const std::string& partGuid) {
 }
 
 std::string BuildDataSystemCePath(userid_t userId) {
-    // TODO: unify with installd path generation logic
-    std::string data(BuildDataPath(nullptr));
-    return StringPrintf("%s/system_ce/%u", data.c_str(), userId);
+    return StringPrintf("%s/system_ce/%u", BuildDataPath(nullptr).c_str(), userId);
+}
+
+std::string BuildDataSystemDePath(userid_t userId) {
+    return StringPrintf("%s/system_de/%u", BuildDataPath(nullptr).c_str(), userId);
+}
+
+std::string BuildDataMiscCePath(userid_t userId) {
+    return StringPrintf("%s/misc_ce/%u", BuildDataPath(nullptr).c_str(), userId);
+}
+
+std::string BuildDataMiscDePath(userid_t userId) {
+    return StringPrintf("%s/misc_de/%u", BuildDataPath(nullptr).c_str(), userId);
 }
 
 std::string BuildDataPath(const char* volumeUuid) {

Utils.h

@@ -97,6 +97,9 @@ status_t WipeBlockDevice(const std::string& path);
 std::string BuildKeyPath(const std::string& partGuid);
 
 std::string BuildDataSystemCePath(userid_t userid);
+std::string BuildDataSystemDePath(userid_t userid);
+std::string BuildDataMiscCePath(userid_t userid);
+std::string BuildDataMiscDePath(userid_t userid);
 
 std::string BuildDataPath(const char* volumeUuid);
 std::string BuildDataMediaPath(const char* volumeUuid, userid_t userid);

cryptfs.c

@@ -1582,6 +1582,9 @@ static int prep_data_fs(void)
 {
     int i;
 
+    // NOTE: post_fs_data results in init calling back around to vold, so all
+    // callers to this method must be async
+
     /* Do the prep of the /data filesystem */
     property_set("vold.post_fs_data_done", "0");
     property_set("vold.decrypt", "trigger_post_fs_data");

vdc.cpp

@@ -22,6 +22,7 @@
 #include <errno.h>
 #include <fcntl.h>
 #include <stdlib.h>
+#include <poll.h>
 
 #include <sys/socket.h>
 #include <sys/select.h>
@@ -29,6 +30,8 @@
 #include <sys/types.h>
 #include <sys/un.h>
 
+#include <android-base/stringprintf.h>
+
 #include <cutils/sockets.h>
 #include <private/android_filesystem_config.h>
 
@@ -36,6 +39,8 @@ static void usage(char *progname);
 static int do_monitor(int sock, int stop_after_cmd);
 static int do_cmd(int sock, int argc, char **argv);
 
+static constexpr int kCommandTimeoutMs = 20 * 1000;
+
 int main(int argc, char **argv) {
     int sock;
     int wait_for_socket;
@@ -44,12 +49,12 @@ int main(int argc, char **argv) {
     progname = argv[0];
 
     wait_for_socket = argc > 1 && strcmp(argv[1], "--wait") == 0;
-    if(wait_for_socket) {
+    if (wait_for_socket) {
         argv++;
         argc--;
     }
 
-    if(argc < 2) {
+    if (argc < 2) {
         usage(progname);
         exit(5);
     }
@@ -62,8 +67,8 @@ int main(int argc, char **argv) {
     while ((sock = socket_local_client(sockname,
                                  ANDROID_SOCKET_NAMESPACE_RESERVED,
                                  SOCK_STREAM)) < 0) {
-        if(!wait_for_socket) {
-            fprintf(stderr, "Error connecting (%s)\n", strerror(errno));
+        if (!wait_for_socket) {
+            fprintf(stdout, "Error connecting to %s: %s\n", sockname, strerror(errno));
             exit(4);
         } else {
             usleep(10000);
@@ -78,97 +83,92 @@ int main(int argc, char **argv) {
 }
 
 static int do_cmd(int sock, int argc, char **argv) {
-    char final_cmd[255] = "0 "; /* 0 is a (now required) sequence number */
-
-    int i;
-    size_t ret;
+    int seq = getpid();
 
-    for (i = 1; i < argc; i++) {
-        char *cmp;
-
-        if (!strchr(argv[i], ' '))
-            asprintf(&cmp, "%s%s", argv[i], (i == (argc -1)) ? "" : " ");
-        else
-            asprintf(&cmp, "\"%s\"%s", argv[i], (i == (argc -1)) ? "" : " ");
+    std::string cmd(android::base::StringPrintf("%d ", seq));
+    for (int i = 1; i < argc; i++) {
+        if (!strchr(argv[i], ' ')) {
+            cmd.append(argv[i]);
+        } else {
+            cmd.push_back('\"');
+            cmd.append(argv[i]);
+            cmd.push_back('\"');
+        }
 
-        ret = strlcat(final_cmd, cmp, sizeof(final_cmd));
-        if (ret >= sizeof(final_cmd))
-            abort();
-        free(cmp);
+        if (i < argc - 1) {
+            cmd.push_back(' ');
+        }
     }
 
-    if (write(sock, final_cmd, strlen(final_cmd) + 1) < 0) {
-        perror("write");
+    if (TEMP_FAILURE_RETRY(write(sock, cmd.c_str(), cmd.length() + 1)) < 0) {
+        fprintf(stderr, "Failed to write command: %s\n", strerror(errno));
         return errno;
     }
 
-    return do_monitor(sock, 1);
+    return do_monitor(sock, seq);
 }
 
-static int do_monitor(int sock, int stop_after_cmd) {
-    char *buffer = (char *) malloc(4096);
-
-    if (!stop_after_cmd)
-        printf("[Connected to Vold]\n");
-
-    while(1) {
-        fd_set read_fds;
-        struct timeval to;
-        int rc = 0;
+static int do_monitor(int sock, int stop_after_seq) {
+    char buffer[4096];
+    int timeout = kCommandTimeoutMs;
 
-        to.tv_sec = 10;
-        to.tv_usec = 0;
-
-        FD_ZERO(&read_fds);
-        FD_SET(sock, &read_fds);
+    if (stop_after_seq == 0) {
+        fprintf(stderr, "Connected to vold\n");
+        timeout = -1;
+    }
 
-        if ((rc = select(sock +1, &read_fds, NULL, NULL, &to)) < 0) {
-            fprintf(stderr, "Error in select (%s)\n", strerror(errno));
-            free(buffer);
-            return errno;
-        } else if (!rc) {
-            continue;
-            fprintf(stderr, "[TIMEOUT]\n");
+    while (1) {
+        struct pollfd poll_sock = { sock, POLLIN, 0 };
+        int rc = TEMP_FAILURE_RETRY(poll(&poll_sock, 1, timeout));
+        if (rc == 0) {
+            fprintf(stderr, "Timeout waiting for %d\n", stop_after_seq);
             return ETIMEDOUT;
-        } else if (FD_ISSET(sock, &read_fds)) {
-            memset(buffer, 0, 4096);
-            if ((rc = read(sock, buffer, 4096)) <= 0) {
-                if (rc == 0)
-                    fprintf(stderr, "Lost connection to Vold - did it crash?\n");
-                else
-                    fprintf(stderr, "Error reading data (%s)\n", strerror(errno));
-                free(buffer);
-                if (rc == 0)
-                    return ECONNRESET;
-                return errno;
-            }
-
-            int offset = 0;
-            int i = 0;
+        } else if (rc < 0) {
+            fprintf(stderr, "Failed during poll: %s\n", strerror(errno));
+            return errno;
+        }
 
-            for (i = 0; i < rc; i++) {
-                if (buffer[i] == '\0') {
-                    int code;
-                    char tmp[4];
+        if (!(poll_sock.revents & POLLIN)) {
+            fprintf(stderr, "No data; trying again\n");
+            continue;
+        }
 
-                    strlcpy(tmp, buffer + offset, sizeof(tmp));
-                    code = atoi(tmp);
+        memset(buffer, 0, sizeof(buffer));
+        rc = TEMP_FAILURE_RETRY(read(sock, buffer, sizeof(buffer)));
+        if (rc == 0) {
+            fprintf(stderr, "Lost connection, did vold crash?\n");
+            return ECONNRESET;
+        } else if (rc < 0) {
+            fprintf(stderr, "Error reading data: %s\n", strerror(errno));
+            return errno;
+        }
 
-                    printf("%s\n", buffer + offset);
-                    if (stop_after_cmd) {
-                        if (code >= 200 && code < 600)
+        int offset = 0;
+        for (int i = 0; i < rc; i++) {
+            if (buffer[i] == '\0') {
+                char* res = buffer + offset;
+                fprintf(stdout, "%s\n", res);
+
+                int code = atoi(strtok(res, " "));
+                if (code >= 200 && code < 600) {
+                    int seq = atoi(strtok(nullptr, " "));
+                    if (seq == stop_after_seq) {
+                        if (code == 200) {
                             return 0;
+                        } else {
+                            return code;
+                        }
                     }
-                    offset = i + 1;
                 }
+
+                offset = i + 1;
             }
         }
     }
-    free(buffer);
-    return 0;
+    return EIO;
 }
 
 static void usage(char *progname) {
     fprintf(stderr,
             "Usage: %s [--wait] <monitor>|<cmd> [arg1] [arg2...]\n", progname);
- }
+}