Merge "v4 digest tree streaming" into rvc-dev am: 437fd07a1f

Change-Id: Ie2f95d0c514e55a82840db86ccd3574834571d9a

services/core/java/com/android/server/pm/PackageManagerShellCommand.java

@@ -105,6 +105,7 @@ import com.android.internal.util.ArrayUtils;
 import com.android.internal.util.IndentingPrintWriter;
 import com.android.server.LocalServices;
 import com.android.server.SystemConfig;
+import com.android.server.pm.PackageManagerShellCommandDataLoader.Metadata;
 
 import dalvik.system.DexFile;
 
@@ -118,7 +119,6 @@ import java.io.InputStream;
 import java.io.OutputStream;
 import java.io.PrintWriter;
 import java.net.URISyntaxException;
-import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Base64;
 import java.util.Collection;
@@ -3025,9 +3025,9 @@ class PackageManagerShellCommand extends ShellCommand {
             // 1. Single file from stdin.
             if (args.isEmpty() || STDIN_PATH.equals(args.get(0))) {
                 final String name = "base." + (isApex ? "apex" : "apk");
-                final String metadata = "-" + name;
+                final Metadata metadata = Metadata.forStdIn(name);
                 session.addFile(LOCATION_DATA_APP, name, sessionSizeBytes,
-                        metadata.getBytes(StandardCharsets.UTF_8), null);
+                        metadata.toByteArray(), null);
                 return 0;
             }
 
@@ -3056,9 +3056,10 @@ class PackageManagerShellCommand extends ShellCommand {
 
     private int processArgForStdin(String arg, PackageInstaller.Session session) {
         final String[] fileDesc = arg.split(":");
-        String name, metadata;
+        String name, fileId;
         long sizeBytes;
         byte[] signature = null;
+        int streamingVersion = 0;
 
         try {
             if (fileDesc.length < 2) {
@@ -3067,14 +3068,22 @@ class PackageManagerShellCommand extends ShellCommand {
             }
             name = fileDesc[0];
             sizeBytes = Long.parseUnsignedLong(fileDesc[1]);
-            metadata = name;
+            fileId = name;
 
             if (fileDesc.length > 2 && !TextUtils.isEmpty(fileDesc[2])) {
-                metadata = fileDesc[2];
+                fileId = fileDesc[2];
             }
             if (fileDesc.length > 3) {
                 signature = Base64.getDecoder().decode(fileDesc[3]);
             }
+            if (fileDesc.length > 4) {
+                streamingVersion = Integer.parseUnsignedInt(fileDesc[4]);
+                if (streamingVersion < 0 || streamingVersion > 1) {
+                    getErrPrintWriter().println(
+                            "Unsupported streaming version: " + streamingVersion);
+                    return 1;
+                }
+            }
         } catch (IllegalArgumentException e) {
             getErrPrintWriter().println(
                     "Unable to parse file parameters: " + arg + ", reason: " + e);
@@ -3086,9 +3095,14 @@ class PackageManagerShellCommand extends ShellCommand {
             return 1;
         }
 
+        final Metadata metadata;
+
         if (signature != null) {
-            // Streaming/adb mode.
+            // Streaming/adb mode. Versions:
-            metadata = "+" + metadata;
+            // 0: data only streaming, tree has to be fully available,
+            // 1: tree and data streaming.
+            metadata = (streamingVersion == 0) ? Metadata.forDataOnlyStreaming(fileId)
+                    : Metadata.forStreaming(fileId);
             try {
                 if (V4Signature.readFrom(signature) == null) {
                     getErrPrintWriter().println("V4 signature is invalid in: " + arg);
@@ -3101,11 +3115,10 @@ class PackageManagerShellCommand extends ShellCommand {
             }
         } else {
             // Single-shot read from stdin.
-            metadata = "-" + metadata;
+            metadata = Metadata.forStdIn(fileId);
         }
 
-        session.addFile(LOCATION_DATA_APP, name, sizeBytes,
+        session.addFile(LOCATION_DATA_APP, name, sizeBytes, metadata.toByteArray(), signature);
-                metadata.getBytes(StandardCharsets.UTF_8), signature);
         return 0;
     }
 
@@ -3115,7 +3128,7 @@ class PackageManagerShellCommand extends ShellCommand {
         final File file = new File(inPath);
         final String name = file.getName();
         final long size = file.length();
-        final byte[] metadata = inPath.getBytes(StandardCharsets.UTF_8);
+        final Metadata metadata = Metadata.forLocalFile(inPath);
 
         byte[] v4signatureBytes = null;
         // Try to load the v4 signature file for the APK; it might not exist.
@@ -3132,7 +3145,7 @@ class PackageManagerShellCommand extends ShellCommand {
             }
         }
 
-        session.addFile(LOCATION_DATA_APP, name, size, metadata, v4signatureBytes);
+        session.addFile(LOCATION_DATA_APP, name, size, metadata.toByteArray(), v4signatureBytes);
     }
 
     private int doWriteSplits(int sessionId, ArrayList<String> splitPaths, long sessionSizeBytes,

services/core/java/com/android/server/pm/PackageManagerShellCommandDataLoader.java

@@ -24,7 +24,6 @@ import android.content.pm.PackageInstaller;
 import android.os.ParcelFileDescriptor;
 import android.os.ShellCommand;
 import android.service.dataloader.DataLoaderService;
-import android.text.TextUtils;
 import android.util.Slog;
 import android.util.SparseArray;
 
@@ -114,6 +113,74 @@ public class PackageManagerShellCommandDataLoader extends DataLoaderService {
         }
     }
 
+    static class Metadata {
+        /**
+         * Full files read from stdin.
+         */
+        static final byte STDIN = 0;
+        /**
+         * Full files read from local file.
+         */
+        static final byte LOCAL_FILE = 1;
+        /**
+         * Signature tree read from stdin, data streamed.
+         */
+        static final byte DATA_ONLY_STREAMING = 2;
+        /**
+         * Everything streamed.
+         */
+        static final byte STREAMING = 3;
+
+        private final byte mMode;
+        private final String mData;
+
+        static Metadata forStdIn(String fileId) {
+            return new Metadata(STDIN, fileId);
+        }
+
+        static Metadata forLocalFile(String filePath) {
+            return new Metadata(LOCAL_FILE, filePath);
+        }
+
+        static Metadata forDataOnlyStreaming(String fileId) {
+            return new Metadata(DATA_ONLY_STREAMING, fileId);
+        }
+
+        static Metadata forStreaming(String fileId) {
+            return new Metadata(STREAMING, fileId);
+        }
+
+        private Metadata(byte mode, String data) {
+            this.mMode = mode;
+            this.mData = (data == null) ? "" : data;
+        }
+
+        static Metadata fromByteArray(byte[] bytes) throws IOException {
+            if (bytes == null || bytes.length == 0) {
+                return null;
+            }
+            byte mode = bytes[0];
+            String data = new String(bytes, 1, bytes.length - 1, StandardCharsets.UTF_8);
+            return new Metadata(mode, data);
+        }
+
+        byte[] toByteArray() {
+            byte[] dataBytes = this.mData.getBytes(StandardCharsets.UTF_8);
+            byte[] result = new byte[1 + dataBytes.length];
+            result[0] = this.mMode;
+            System.arraycopy(dataBytes, 0, result, 1, dataBytes.length);
+            return result;
+        }
+
+        byte getMode() {
+            return this.mMode;
+        }
+
+        String getData() {
+            return this.mData;
+        }
+    }
+
     private static class DataLoader implements DataLoaderService.DataLoader {
         private DataLoaderParams mParams = null;
         private FileSystemConnector mConnector = null;
@@ -136,19 +203,31 @@ public class PackageManagerShellCommandDataLoader extends DataLoaderService {
             }
             try {
                 for (InstallationFile file : addedFiles) {
-                    String filePath = new String(file.getMetadata(), StandardCharsets.UTF_8);
+                    Metadata metadata = Metadata.fromByteArray(file.getMetadata());
-                    if (TextUtils.isEmpty(filePath) || filePath.startsWith(STDIN_PATH)) {
+                    if (metadata == null) {
-                        final ParcelFileDescriptor inFd = getStdInPFD(shellCommand);
+                        Slog.e(TAG, "Invalid metadata for file: " + file.getName());
-                        mConnector.writeData(file.getName(), 0, file.getLengthBytes(), inFd);
+                        return false;
-                    } else {
+                    }
-                        ParcelFileDescriptor incomingFd = null;
+                    switch (metadata.getMode()) {
-                        try {
+                        case Metadata.STDIN: {
-                            incomingFd = getLocalFile(shellCommand, filePath);
+                            final ParcelFileDescriptor inFd = getStdInPFD(shellCommand);
-                            mConnector.writeData(file.getName(), 0, incomingFd.getStatSize(),
+                            mConnector.writeData(file.getName(), 0, file.getLengthBytes(), inFd);
-                                    incomingFd);
+                            break;
-                        } finally {
-                            IoUtils.closeQuietly(incomingFd);
                         }
+                        case Metadata.LOCAL_FILE: {
+                            ParcelFileDescriptor incomingFd = null;
+                            try {
+                                incomingFd = getLocalFile(shellCommand, metadata.getData());
+                                mConnector.writeData(file.getName(), 0, incomingFd.getStatSize(),
+                                        incomingFd);
+                            } finally {
+                                IoUtils.closeQuietly(incomingFd);
+                            }
+                            break;
+                        }
+                        default:
+                            Slog.e(TAG, "Unsupported metadata mode: " + metadata.getMode());
+                            return false;
                     }
                 }
                 return true;

services/core/jni/com_android_server_pm_PackageManagerShellCommandDataLoader.cpp

@@ -227,56 +227,40 @@ static inline unique_fd convertPfdToFdAndDup(JNIEnv* env, const JniIds& jni, job
     return result;
 }
 
+enum MetadataMode : int8_t {
+    STDIN = 0,
+    LOCAL_FILE = 1,
+    DATA_ONLY_STREAMING = 2,
+    STREAMING = 3,
+};
+
 struct InputDesc {
     unique_fd fd;
     IncFsSize size;
     IncFsBlockKind kind = INCFS_BLOCK_KIND_DATA;
     bool waitOnEof = false;
     bool streaming = false;
+    MetadataMode mode = STDIN;
 };
 using InputDescs = std::vector<InputDesc>;
 
-static inline InputDescs openInputs(JNIEnv* env, const JniIds& jni, jobject shellCommand,
+template <class T>
-                                    IncFsSize size, IncFsSpan metadata) {
+std::optional<T> read(IncFsSpan& data) {
+    if (data.size < (int32_t)sizeof(T)) {
+        return {};
+    }
+    T res;
+    memcpy(&res, data.data, sizeof(res));
+    data.data += sizeof(res);
+    data.size -= sizeof(res);
+    return res;
+}
+
+static inline InputDescs openLocalFile(JNIEnv* env, const JniIds& jni, jobject shellCommand,
+                                       IncFsSize size, const std::string& filePath) {
     InputDescs result;
     result.reserve(2);
 
-    if (metadata.size == 0 || *metadata.data == '-') {
-        // stdin
-        auto fd = convertPfdToFdAndDup(
-                env, jni,
-                env->CallStaticObjectMethod(jni.packageManagerShellCommandDataLoader,
-                                            jni.pmscdGetStdInPFD, shellCommand));
-        if (fd.ok()) {
-            result.push_back(InputDesc{
-                    .fd = std::move(fd),
-                    .size = size,
-                    .waitOnEof = true,
-            });
-        }
-        return result;
-    }
-    if (*metadata.data == '+') {
-        // verity tree from stdin, rest is streaming
-        auto fd = convertPfdToFdAndDup(
-                env, jni,
-                env->CallStaticObjectMethod(jni.packageManagerShellCommandDataLoader,
-                                            jni.pmscdGetStdInPFD, shellCommand));
-        if (fd.ok()) {
-            auto treeSize = verityTreeSizeForFile(size);
-            result.push_back(InputDesc{
-                    .fd = std::move(fd),
-                    .size = treeSize,
-                    .kind = INCFS_BLOCK_KIND_HASH,
-                    .waitOnEof = true,
-                    .streaming = true,
-            });
-        }
-        return result;
-    }
-
-    // local file and possibly signature
-    const std::string filePath(metadata.data, metadata.size);
     const std::string idsigPath = filePath + ".idsig";
 
     auto idsigFd = convertPfdToFdAndDup(
@@ -314,6 +298,59 @@ static inline InputDescs openInputs(JNIEnv* env, const JniIds& jni, jobject shel
     return result;
 }
 
+static inline InputDescs openInputs(JNIEnv* env, const JniIds& jni, jobject shellCommand,
+                                    IncFsSize size, IncFsSpan metadata) {
+    auto mode = read<int8_t>(metadata).value_or(STDIN);
+    if (mode == LOCAL_FILE) {
+        // local file and possibly signature
+        return openLocalFile(env, jni, shellCommand, size,
+                             std::string(metadata.data, metadata.size));
+    }
+
+    auto fd = convertPfdToFdAndDup(
+            env, jni,
+            env->CallStaticObjectMethod(jni.packageManagerShellCommandDataLoader,
+                                        jni.pmscdGetStdInPFD, shellCommand));
+    if (!fd.ok()) {
+        return {};
+    }
+
+    InputDescs result;
+    switch (mode) {
+        case STDIN: {
+            result.push_back(InputDesc{
+                    .fd = std::move(fd),
+                    .size = size,
+                    .waitOnEof = true,
+            });
+            break;
+        }
+        case DATA_ONLY_STREAMING: {
+            // verity tree from stdin, rest is streaming
+            auto treeSize = verityTreeSizeForFile(size);
+            result.push_back(InputDesc{
+                    .fd = std::move(fd),
+                    .size = treeSize,
+                    .kind = INCFS_BLOCK_KIND_HASH,
+                    .waitOnEof = true,
+                    .streaming = true,
+                    .mode = DATA_ONLY_STREAMING,
+            });
+            break;
+        }
+        case STREAMING: {
+            result.push_back(InputDesc{
+                    .fd = std::move(fd),
+                    .size = 0,
+                    .streaming = true,
+                    .mode = STREAMING,
+            });
+            break;
+        }
+    }
+    return result;
+}
+
 static inline JNIEnv* GetJNIEnvironment(JavaVM* vm) {
     JNIEnv* env;
     if (vm->GetEnv(reinterpret_cast<void**>(&env), JNI_VERSION_1_6) != JNI_OK) {
@@ -390,6 +427,7 @@ private:
         blocks.reserve(BLOCKS_COUNT);
 
         unique_fd streamingFd;
+        MetadataMode streamingMode;
         for (auto&& file : addedFiles) {
             auto inputs = openInputs(env, jni, shellCommand, file.size, file.metadata);
             if (inputs.empty()) {
@@ -411,6 +449,7 @@ private:
             for (auto&& input : inputs) {
                 if (input.streaming && !streamingFd.ok()) {
                     streamingFd.reset(dup(input.fd));
+                    streamingMode = input.mode;
                 }
                 if (!copyToIncFs(incfsFd, input.size, input.kind, input.fd, input.waitOnEof,
                                  &buffer, &blocks)) {
@@ -425,7 +464,7 @@ private:
 
         if (streamingFd.ok()) {
             ALOGE("onPrepareImage: done, proceeding to streaming.");
-            return initStreaming(std::move(streamingFd));
+            return initStreaming(std::move(streamingFd), streamingMode);
         }
 
         ALOGE("onPrepareImage: done.");
@@ -564,7 +603,7 @@ private:
     }
 
     // Streaming.
-    bool initStreaming(unique_fd inout) {
+    bool initStreaming(unique_fd inout, MetadataMode mode) {
         mEventFd.reset(eventfd(0, EFD_CLOEXEC));
         if (mEventFd < 0) {
             ALOGE("Failed to create eventfd.");
@@ -591,8 +630,8 @@ private:
             }
         }
 
-        mReceiverThread =
+        mReceiverThread = std::thread(
-                std::thread([this, io = std::move(inout)]() mutable { receiver(std::move(io)); });
+                [this, io = std::move(inout), mode]() mutable { receiver(std::move(io), mode); });
         ALOGI("Started streaming...");
         return true;
     }
@@ -624,7 +663,7 @@ private:
         }
     }
 
-    void receiver(unique_fd inout) {
+    void receiver(unique_fd inout, MetadataMode mode) {
         std::vector<uint8_t> data;
         std::vector<IncFsDataBlock> instructions;
         std::unordered_map<FileIdx, unique_fd> writeFds;
@@ -667,7 +706,7 @@ private:
                     break;
                 }
                 const FileIdx fileIdx = header.fileIdx;
-                const android::dataloader::FileId fileId = convertFileIndexToFileId(fileIdx);
+                const android::dataloader::FileId fileId = convertFileIndexToFileId(mode, fileIdx);
                 if (!android::incfs::isValidFileId(fileId)) {
                     ALOGE("Unknown data destination for file ID %d. "
                           "Ignore.",
@@ -679,7 +718,7 @@ private:
                 if (writeFd < 0) {
                     writeFd.reset(this->mIfs->openWrite(fileId));
                     if (writeFd < 0) {
-                        ALOGE("Failed to open file %d for writing (%d). Aboring.", header.fileIdx,
+                        ALOGE("Failed to open file %d for writing (%d). Aborting.", header.fileIdx,
                               -writeFd);
                         break;
                     }
@@ -716,9 +755,11 @@ private:
     }
 
     FileIdx convertFileIdToFileIndex(android::dataloader::FileId fileId) {
-        // FileId is a string in format '+FileIdx\0'.
+        // FileId has format '\2FileIdx'.
         const char* meta = (const char*)&fileId;
-        if (*meta != '+') {
+
+        int8_t mode = *meta;
+        if (mode != DATA_ONLY_STREAMING && mode != STREAMING) {
             return -1;
         }
 
@@ -732,10 +773,10 @@ private:
         return FileIdx(fileIdx);
     }
 
-    android::dataloader::FileId convertFileIndexToFileId(FileIdx fileIdx) {
+    android::dataloader::FileId convertFileIndexToFileId(MetadataMode mode, FileIdx fileIdx) {
         IncFsFileId fileId = {};
         char* meta = (char*)&fileId;
-        *meta = '+';
+        *meta = mode;
         if (auto [p, ec] = std::to_chars(meta + 1, meta + sizeof(fileId), fileIdx);
             ec != std::errc()) {
             return {};