Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge "v4 digest tree streaming" into rvc-dev

Browse Source
This commit is contained in:
Alex Buynytskyy
2020-03-27 22:26:01 +00:00
committed by Android (Google) Code Review
parent db11409edc 06970b9338
commit 437fd07a1f
3 changed files with 208 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
services/core/java/com/android/server/pm/PackageManagerShellCommand.java
View File

@@ -105,6 +105,7 @@ import com.android.internal.util.ArrayUtils;
import com.android.internal.util.IndentingPrintWriter;
import com.android.server.LocalServices;
import com.android.server.SystemConfig;
import com.android.server.pm.PackageManagerShellCommandDataLoader.Metadata;
import dalvik.system.DexFile;
@@ -118,7 +119,6 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.io.PrintWriter;
import java.net.URISyntaxException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Base64;
import java.util.Collection;
@@ -3025,9 +3025,9 @@ class PackageManagerShellCommand extends ShellCommand {
// 1. Single file from stdin.
if (args.isEmpty() || STDIN_PATH.equals(args.get(0))) {
final String name = "base." + (isApex ? "apex" : "apk");
final String metadata = "-" + name;
final Metadata metadata = Metadata.forStdIn(name);
session.addFile(LOCATION_DATA_APP, name, sessionSizeBytes,
metadata.getBytes(StandardCharsets.UTF_8), null);
metadata.toByteArray(), null);
return 0;
}
@@ -3056,9 +3056,10 @@ class PackageManagerShellCommand extends ShellCommand {
private int processArgForStdin(String arg, PackageInstaller.Session session) {
final String[] fileDesc = arg.split(":");
String name, metadata;
String name, fileId;
long sizeBytes;
byte[] signature = null;
int streamingVersion = 0;
try {
if (fileDesc.length < 2) {
@@ -3067,14 +3068,22 @@ class PackageManagerShellCommand extends ShellCommand {
}
name = fileDesc[0];
sizeBytes = Long.parseUnsignedLong(fileDesc[1]);
metadata = name;
fileId = name;
if (fileDesc.length > 2 && !TextUtils.isEmpty(fileDesc[2])) {
metadata = fileDesc[2];
fileId = fileDesc[2];
}
if (fileDesc.length > 3) {
signature = Base64.getDecoder().decode(fileDesc[3]);
}
if (fileDesc.length > 4) {
streamingVersion = Integer.parseUnsignedInt(fileDesc[4]);
if (streamingVersion < 0 || streamingVersion > 1) {
getErrPrintWriter().println(
"Unsupported streaming version: " + streamingVersion);
return 1;
}
}
} catch (IllegalArgumentException e) {
getErrPrintWriter().println(
"Unable to parse file parameters: " + arg + ", reason: " + e);
@@ -3086,9 +3095,14 @@ class PackageManagerShellCommand extends ShellCommand {
return 1;
}
final Metadata metadata;
if (signature != null) {
// Streaming/adb mode.
metadata = "+" + metadata;
// Streaming/adb mode. Versions:
// 0: data only streaming, tree has to be fully available,
// 1: tree and data streaming.
metadata = (streamingVersion == 0) ? Metadata.forDataOnlyStreaming(fileId)
: Metadata.forStreaming(fileId);
try {
if (V4Signature.readFrom(signature) == null) {
getErrPrintWriter().println("V4 signature is invalid in: " + arg);
@@ -3101,11 +3115,10 @@ class PackageManagerShellCommand extends ShellCommand {
}
} else {
// Single-shot read from stdin.
metadata = "-" + metadata;
metadata = Metadata.forStdIn(fileId);
}
session.addFile(LOCATION_DATA_APP, name, sizeBytes,
metadata.getBytes(StandardCharsets.UTF_8), signature);
session.addFile(LOCATION_DATA_APP, name, sizeBytes, metadata.toByteArray(), signature);
return 0;
}
@@ -3115,7 +3128,7 @@ class PackageManagerShellCommand extends ShellCommand {
final File file = new File(inPath);
final String name = file.getName();
final long size = file.length();
final byte[] metadata = inPath.getBytes(StandardCharsets.UTF_8);
final Metadata metadata = Metadata.forLocalFile(inPath);
byte[] v4signatureBytes = null;
// Try to load the v4 signature file for the APK; it might not exist.
@@ -3132,7 +3145,7 @@ class PackageManagerShellCommand extends ShellCommand {
}
}
session.addFile(LOCATION_DATA_APP, name, size, metadata, v4signatureBytes);
session.addFile(LOCATION_DATA_APP, name, size, metadata.toByteArray(), v4signatureBytes);
}
private int doWriteSplits(int sessionId, ArrayList<String> splitPaths, long sessionSizeBytes,

105
services/core/java/com/android/server/pm/PackageManagerShellCommandDataLoader.java
View File

@@ -24,7 +24,6 @@ import android.content.pm.PackageInstaller;
import android.os.ParcelFileDescriptor;
import android.os.ShellCommand;
import android.service.dataloader.DataLoaderService;
import android.text.TextUtils;
import android.util.Slog;
import android.util.SparseArray;
@@ -114,6 +113,74 @@ public class PackageManagerShellCommandDataLoader extends DataLoaderService {
}
}
static class Metadata {
/**
* Full files read from stdin.
*/
static final byte STDIN = 0;
/**
* Full files read from local file.
*/
static final byte LOCAL_FILE = 1;
/**
* Signature tree read from stdin, data streamed.
*/
static final byte DATA_ONLY_STREAMING = 2;
/**
* Everything streamed.
*/
static final byte STREAMING = 3;
private final byte mMode;
private final String mData;
static Metadata forStdIn(String fileId) {
return new Metadata(STDIN, fileId);
}
static Metadata forLocalFile(String filePath) {
return new Metadata(LOCAL_FILE, filePath);
}
static Metadata forDataOnlyStreaming(String fileId) {
return new Metadata(DATA_ONLY_STREAMING, fileId);
}
static Metadata forStreaming(String fileId) {
return new Metadata(STREAMING, fileId);
}
private Metadata(byte mode, String data) {
this.mMode = mode;
this.mData = (data == null) ? "" : data;
}
static Metadata fromByteArray(byte[] bytes) throws IOException {
if (bytes == null || bytes.length == 0) {
return null;
}
byte mode = bytes[0];
String data = new String(bytes, 1, bytes.length - 1, StandardCharsets.UTF_8);
return new Metadata(mode, data);
}
byte[] toByteArray() {
byte[] dataBytes = this.mData.getBytes(StandardCharsets.UTF_8);
byte[] result = new byte[1 + dataBytes.length];
result[0] = this.mMode;
System.arraycopy(dataBytes, 0, result, 1, dataBytes.length);
return result;
}
byte getMode() {
return this.mMode;
}
String getData() {
return this.mData;
}
}
private static class DataLoader implements DataLoaderService.DataLoader {
private DataLoaderParams mParams = null;
private FileSystemConnector mConnector = null;
@@ -136,19 +203,31 @@ public class PackageManagerShellCommandDataLoader extends DataLoaderService {
}
try {
for (InstallationFile file : addedFiles) {
String filePath = new String(file.getMetadata(), StandardCharsets.UTF_8);
if (TextUtils.isEmpty(filePath) || filePath.startsWith(STDIN_PATH)) {
final ParcelFileDescriptor inFd = getStdInPFD(shellCommand);
mConnector.writeData(file.getName(), 0, file.getLengthBytes(), inFd);
} else {
ParcelFileDescriptor incomingFd = null;
try {
incomingFd = getLocalFile(shellCommand, filePath);
mConnector.writeData(file.getName(), 0, incomingFd.getStatSize(),
incomingFd);
} finally {
IoUtils.closeQuietly(incomingFd);
Metadata metadata = Metadata.fromByteArray(file.getMetadata());
if (metadata == null) {
Slog.e(TAG, "Invalid metadata for file: " + file.getName());
return false;
}
switch (metadata.getMode()) {
case Metadata.STDIN: {
final ParcelFileDescriptor inFd = getStdInPFD(shellCommand);
mConnector.writeData(file.getName(), 0, file.getLengthBytes(), inFd);
break;
}
case Metadata.LOCAL_FILE: {
ParcelFileDescriptor incomingFd = null;
try {
incomingFd = getLocalFile(shellCommand, metadata.getData());
mConnector.writeData(file.getName(), 0, incomingFd.getStatSize(),
incomingFd);
} finally {
IoUtils.closeQuietly(incomingFd);
}
break;
}
default:
Slog.e(TAG, "Unsupported metadata mode: " + metadata.getMode());
return false;
}
}
return true;

139
services/core/jni/com_android_server_pm_PackageManagerShellCommandDataLoader.cpp
View File

@@ -227,56 +227,40 @@ static inline unique_fd convertPfdToFdAndDup(JNIEnv* env, const JniIds& jni, job
return result;
}
enum MetadataMode : int8_t {
STDIN = 0,
LOCAL_FILE = 1,
DATA_ONLY_STREAMING = 2,
STREAMING = 3,
};
struct InputDesc {
unique_fd fd;
IncFsSize size;
IncFsBlockKind kind = INCFS_BLOCK_KIND_DATA;
bool waitOnEof = false;
bool streaming = false;
MetadataMode mode = STDIN;
};
using InputDescs = std::vector<InputDesc>;
static inline InputDescs openInputs(JNIEnv* env, const JniIds& jni, jobject shellCommand,
IncFsSize size, IncFsSpan metadata) {
template <class T>
std::optional<T> read(IncFsSpan& data) {
if (data.size < (int32_t)sizeof(T)) {
return {};
}
T res;
memcpy(&res, data.data, sizeof(res));
data.data += sizeof(res);
data.size -= sizeof(res);
return res;
}
static inline InputDescs openLocalFile(JNIEnv* env, const JniIds& jni, jobject shellCommand,
IncFsSize size, const std::string& filePath) {
InputDescs result;
result.reserve(2);
if (metadata.size == 0 || *metadata.data == '-') {
// stdin
auto fd = convertPfdToFdAndDup(
env, jni,
env->CallStaticObjectMethod(jni.packageManagerShellCommandDataLoader,
jni.pmscdGetStdInPFD, shellCommand));
if (fd.ok()) {
result.push_back(InputDesc{
.fd = std::move(fd),
.size = size,
.waitOnEof = true,
});
}
return result;
}
if (*metadata.data == '+') {
// verity tree from stdin, rest is streaming
auto fd = convertPfdToFdAndDup(
env, jni,
env->CallStaticObjectMethod(jni.packageManagerShellCommandDataLoader,
jni.pmscdGetStdInPFD, shellCommand));
if (fd.ok()) {
auto treeSize = verityTreeSizeForFile(size);
result.push_back(InputDesc{
.fd = std::move(fd),
.size = treeSize,
.kind = INCFS_BLOCK_KIND_HASH,
.waitOnEof = true,
.streaming = true,
});
}
return result;
}
// local file and possibly signature
const std::string filePath(metadata.data, metadata.size);
const std::string idsigPath = filePath + ".idsig";
auto idsigFd = convertPfdToFdAndDup(
@@ -314,6 +298,59 @@ static inline InputDescs openInputs(JNIEnv* env, const JniIds& jni, jobject shel
return result;
}
static inline InputDescs openInputs(JNIEnv* env, const JniIds& jni, jobject shellCommand,
IncFsSize size, IncFsSpan metadata) {
auto mode = read<int8_t>(metadata).value_or(STDIN);
if (mode == LOCAL_FILE) {
// local file and possibly signature
return openLocalFile(env, jni, shellCommand, size,
std::string(metadata.data, metadata.size));
}
auto fd = convertPfdToFdAndDup(
env, jni,
env->CallStaticObjectMethod(jni.packageManagerShellCommandDataLoader,
jni.pmscdGetStdInPFD, shellCommand));
if (!fd.ok()) {
return {};
}
InputDescs result;
switch (mode) {
case STDIN: {
result.push_back(InputDesc{
.fd = std::move(fd),
.size = size,
.waitOnEof = true,
});
break;
}
case DATA_ONLY_STREAMING: {
// verity tree from stdin, rest is streaming
auto treeSize = verityTreeSizeForFile(size);
result.push_back(InputDesc{
.fd = std::move(fd),
.size = treeSize,
.kind = INCFS_BLOCK_KIND_HASH,
.waitOnEof = true,
.streaming = true,
.mode = DATA_ONLY_STREAMING,
});
break;
}
case STREAMING: {
result.push_back(InputDesc{
.fd = std::move(fd),
.size = 0,
.streaming = true,
.mode = STREAMING,
});
break;
}
}
return result;
}
static inline JNIEnv* GetJNIEnvironment(JavaVM* vm) {
JNIEnv* env;
if (vm->GetEnv(reinterpret_cast<void**>(&env), JNI_VERSION_1_6) != JNI_OK) {
@@ -390,6 +427,7 @@ private:
blocks.reserve(BLOCKS_COUNT);
unique_fd streamingFd;
MetadataMode streamingMode;
for (auto&& file : addedFiles) {
auto inputs = openInputs(env, jni, shellCommand, file.size, file.metadata);
if (inputs.empty()) {
@@ -411,6 +449,7 @@ private:
for (auto&& input : inputs) {
if (input.streaming && !streamingFd.ok()) {
streamingFd.reset(dup(input.fd));
streamingMode = input.mode;
}
if (!copyToIncFs(incfsFd, input.size, input.kind, input.fd, input.waitOnEof,
&buffer, &blocks)) {
@@ -425,7 +464,7 @@ private:
if (streamingFd.ok()) {
ALOGE("onPrepareImage: done, proceeding to streaming.");
return initStreaming(std::move(streamingFd));
return initStreaming(std::move(streamingFd), streamingMode);
}
ALOGE("onPrepareImage: done.");
@@ -564,7 +603,7 @@ private:
}
// Streaming.
bool initStreaming(unique_fd inout) {
bool initStreaming(unique_fd inout, MetadataMode mode) {
mEventFd.reset(eventfd(0, EFD_CLOEXEC));
if (mEventFd < 0) {
ALOGE("Failed to create eventfd.");
@@ -591,8 +630,8 @@ private:
}
}
mReceiverThread =
std::thread([this, io = std::move(inout)]() mutable { receiver(std::move(io)); });
mReceiverThread = std::thread(
[this, io = std::move(inout), mode]() mutable { receiver(std::move(io), mode); });
ALOGI("Started streaming...");
return true;
}
@@ -624,7 +663,7 @@ private:
}
}
void receiver(unique_fd inout) {
void receiver(unique_fd inout, MetadataMode mode) {
std::vector<uint8_t> data;
std::vector<IncFsDataBlock> instructions;
std::unordered_map<FileIdx, unique_fd> writeFds;
@@ -667,7 +706,7 @@ private:
break;
}
const FileIdx fileIdx = header.fileIdx;
const android::dataloader::FileId fileId = convertFileIndexToFileId(fileIdx);
const android::dataloader::FileId fileId = convertFileIndexToFileId(mode, fileIdx);
if (!android::incfs::isValidFileId(fileId)) {
ALOGE("Unknown data destination for file ID %d. "
"Ignore.",
@@ -679,7 +718,7 @@ private:
if (writeFd < 0) {
writeFd.reset(this->mIfs->openWrite(fileId));
if (writeFd < 0) {
ALOGE("Failed to open file %d for writing (%d). Aboring.", header.fileIdx,
ALOGE("Failed to open file %d for writing (%d). Aborting.", header.fileIdx,
-writeFd);
break;
}
@@ -716,9 +755,11 @@ private:
}
FileIdx convertFileIdToFileIndex(android::dataloader::FileId fileId) {
// FileId is a string in format '+FileIdx\0'.
// FileId has format '\2FileIdx'.
const char* meta = (const char*)&fileId;
if (*meta != '+') {
int8_t mode = *meta;
if (mode != DATA_ONLY_STREAMING && mode != STREAMING) {
return -1;
}
@@ -732,10 +773,10 @@ private:
return FileIdx(fileIdx);
}
android::dataloader::FileId convertFileIndexToFileId(FileIdx fileIdx) {
android::dataloader::FileId convertFileIndexToFileId(MetadataMode mode, FileIdx fileIdx) {
IncFsFileId fileId = {};
char* meta = (char*)&fileId;
*meta = '+';
*meta = mode;
if (auto [p, ec] = std::to_chars(meta + 1, meta + sizeof(fileId), fileIdx);
ec != std::errc()) {
return {};