From 7a38dcc93891862586b40d98c7f7851cbd1a0d00 Mon Sep 17 00:00:00 2001 From: Jeff Sharkey Date: Fri, 6 Nov 2020 20:40:59 -0700 Subject: [PATCH 1/4] CharsetUtils alternatives that avoid allocations. Internally String.getBytes() calls libcore.util.CharsetUtils methods for a handful of common charsets, but that path requires new memory allocations for every call. This change introduces alternative versions of those methods which attempt to encode data directly into an already-allocated memory region. If the destination is to small, callers can detect and pivot back to calling String.getBytes(). The included benchmarks reveal these raw performance improvements, in addition to the reduced GC load which is harder to measure: timeLocal_LargeBuffer[simple]_mean: 424 timeLocal_SmallBuffer[simple]_mean: 511 timeUpstream[simple]_mean: 800 timeLocal_LargeBuffer[complex]_mean: 977 timeLocal_SmallBuffer[complex]_mean: 1266 timeUpstream[complex]_mean: 1468 Bug: 171832118 Test: atest CorePerfTests:android.util.CharsetUtilsPerfTest Test: atest FrameworksCoreTests:android.util.CharsetUtilsTest Change-Id: Iac1151e7cb8e88bf82339cada64b0936e1a7578b --- .../android/util/CharsetUtilsPerfTest.java | 91 +++++++++++++++++++ core/java/android/util/CharsetUtils.java | 64 +++++++++++++ core/jni/Android.bp | 1 + core/jni/AndroidRuntime.cpp | 2 + core/jni/android_util_CharsetUtils.cpp | 54 +++++++++++ .../src/android/util/CharsetUtilsTest.java | 76 ++++++++++++++++ 6 files changed, 288 insertions(+) create mode 100644 apct-tests/perftests/core/src/android/util/CharsetUtilsPerfTest.java create mode 100644 core/java/android/util/CharsetUtils.java create mode 100644 core/jni/android_util_CharsetUtils.cpp create mode 100644 core/tests/coretests/src/android/util/CharsetUtilsTest.java diff --git a/apct-tests/perftests/core/src/android/util/CharsetUtilsPerfTest.java b/apct-tests/perftests/core/src/android/util/CharsetUtilsPerfTest.java new file mode 100644 index 0000000000000..2a538b2586639 --- /dev/null +++ b/apct-tests/perftests/core/src/android/util/CharsetUtilsPerfTest.java @@ -0,0 +1,91 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.util; + +import android.perftests.utils.BenchmarkState; +import android.perftests.utils.PerfStatusReporter; + +import androidx.test.filters.LargeTest; + +import dalvik.system.VMRuntime; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Collection; + +@LargeTest +@RunWith(Parameterized.class) +public class CharsetUtilsPerfTest { + @Rule + public PerfStatusReporter mPerfStatusReporter = new PerfStatusReporter(); + + @Parameterized.Parameter(0) + public String mName; + @Parameterized.Parameter(1) + public String mValue; + + @Parameterized.Parameters(name = "{0}") + public static Collection getParameters() { + return Arrays.asList(new Object[][] { + { "simple", "com.example.typical_package_name" }, + { "complex", "從不喜歡孤單一個 - 蘇永康／吳雨霏" }, + }); + } + + @Test + public void timeUpstream() { + final BenchmarkState state = mPerfStatusReporter.getBenchmarkState(); + while (state.keepRunning()) { + mValue.getBytes(StandardCharsets.UTF_8); + } + } + + /** + * Measure performance of writing into a small buffer where bounds checking + * requires careful measurement of encoded size. + */ + @Test + public void timeLocal_SmallBuffer() { + final byte[] dest = (byte[]) VMRuntime.getRuntime().newNonMovableArray(byte.class, 64); + final long destPtr = VMRuntime.getRuntime().addressOf(dest); + + final BenchmarkState state = mPerfStatusReporter.getBenchmarkState(); + while (state.keepRunning()) { + CharsetUtils.toUtf8Bytes(mValue, destPtr, 0, dest.length); + } + } + + /** + * Measure performance of writing into a large buffer where bounds checking + * only needs a simple worst-case 4-bytes-per-char check. + */ + @Test + public void timeLocal_LargeBuffer() { + final byte[] dest = (byte[]) VMRuntime.getRuntime().newNonMovableArray(byte.class, 1024); + final long destPtr = VMRuntime.getRuntime().addressOf(dest); + + final BenchmarkState state = mPerfStatusReporter.getBenchmarkState(); + while (state.keepRunning()) { + CharsetUtils.toUtf8Bytes(mValue, destPtr, 0, dest.length); + } + } +} diff --git a/core/java/android/util/CharsetUtils.java b/core/java/android/util/CharsetUtils.java new file mode 100644 index 0000000000000..80c205511a7b9 --- /dev/null +++ b/core/java/android/util/CharsetUtils.java @@ -0,0 +1,64 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.util; + +import android.annotation.NonNull; + +import dalvik.annotation.optimization.FastNative; + +/** + * Specializations of {@code libcore.util.CharsetUtils} which enable efficient + * in-place encoding without making any new allocations. + *

+ * These methods purposefully accept only non-movable byte array addresses to + * avoid extra JNI overhead. + * + * @hide + */ +public class CharsetUtils { + /** + * Attempt to encode the given string as UTF-8 into the destination byte + * array without making any new allocations. + * + * @param src string value to be encoded + * @param dest destination byte array to encode into + * @param destOff offset into destination where encoding should begin + * @param destLen length of destination + * @return the number of bytes written to the destination when encoded + * successfully, otherwise {@code -1} if not large enough + */ + public static int toUtf8Bytes(@NonNull String src, + long dest, int destOff, int destLen) { + return toUtf8Bytes(src, src.length(), dest, destOff, destLen); + } + + /** + * Attempt to encode the given string as UTF-8 into the destination byte + * array without making any new allocations. + * + * @param src string value to be encoded + * @param srcLen exact length of string to be encoded + * @param dest destination byte array to encode into + * @param destOff offset into destination where encoding should begin + * @param destLen length of destination + * @return the number of bytes written to the destination when encoded + * successfully, otherwise {@code -1} if not large enough + */ + @FastNative + private static native int toUtf8Bytes(@NonNull String src, int srcLen, + long dest, int destOff, int destLen); +} diff --git a/core/jni/Android.bp b/core/jni/Android.bp index dd1c87b68ca5b..dc059348657ae 100644 --- a/core/jni/Android.bp +++ b/core/jni/Android.bp @@ -134,6 +134,7 @@ cc_library_shared { "android_service_DataLoaderService.cpp", "android_util_AssetManager.cpp", "android_util_Binder.cpp", + "android_util_CharsetUtils.cpp", "android_util_MemoryIntArray.cpp", "android_util_Process.cpp", "android_media_AudioDeviceAttributes.cpp", diff --git a/core/jni/AndroidRuntime.cpp b/core/jni/AndroidRuntime.cpp index 27b23bd05e1f1..14e74a840b21a 100644 --- a/core/jni/AndroidRuntime.cpp +++ b/core/jni/AndroidRuntime.cpp @@ -105,6 +105,7 @@ namespace android { */ extern int register_android_app_admin_SecurityLog(JNIEnv* env); extern int register_android_content_AssetManager(JNIEnv* env); +extern int register_android_util_CharsetUtils(JNIEnv* env); extern int register_android_util_EventLog(JNIEnv* env); extern int register_android_util_Log(JNIEnv* env); extern int register_android_util_MemoryIntArray(JNIEnv* env); @@ -1449,6 +1450,7 @@ static const RegJNIRec gRegJNI[] = { REG_JNI(register_com_android_internal_os_RuntimeInit), REG_JNI(register_com_android_internal_os_ZygoteInit_nativeZygoteInit), REG_JNI(register_android_os_SystemClock), + REG_JNI(register_android_util_CharsetUtils), REG_JNI(register_android_util_EventLog), REG_JNI(register_android_util_Log), REG_JNI(register_android_util_MemoryIntArray), diff --git a/core/jni/android_util_CharsetUtils.cpp b/core/jni/android_util_CharsetUtils.cpp new file mode 100644 index 0000000000000..3e1d4a7049192 --- /dev/null +++ b/core/jni/android_util_CharsetUtils.cpp @@ -0,0 +1,54 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "core_jni_helpers.h" +#include "nativehelper/scoped_primitive_array.h" + +namespace android { + +static jint android_util_CharsetUtils_toUtf8Bytes(JNIEnv *env, jobject clazz, + jstring src, jint srcLen, jlong dest, jint destOff, jint destLen) { + char *destPtr = reinterpret_cast(dest); + + // Quickly check if destination has plenty of room for worst-case + // 4-bytes-per-char encoded size + if (destOff >= 0 && destOff + (srcLen * 4) < destLen) { + env->GetStringUTFRegion(src, 0, srcLen, destPtr + destOff); + return strlen(destPtr + destOff + srcLen) + srcLen; + } + + // String still might fit in destination, but we need to measure + // its actual encoded size to be sure + const size_t encodedLen = env->GetStringUTFLength(src); + if (destOff >= 0 && destOff + encodedLen < destLen) { + env->GetStringUTFRegion(src, 0, srcLen, destPtr + destOff); + return encodedLen; + } + + return -1; +} + +static const JNINativeMethod methods[] = { + // @FastNative + {"toUtf8Bytes", "(Ljava/lang/String;IJII)I", + (void*)android_util_CharsetUtils_toUtf8Bytes}, +}; + +int register_android_util_CharsetUtils(JNIEnv *env) { + return RegisterMethodsOrDie(env, "android/util/CharsetUtils", methods, NELEM(methods)); +} + +} diff --git a/core/tests/coretests/src/android/util/CharsetUtilsTest.java b/core/tests/coretests/src/android/util/CharsetUtilsTest.java new file mode 100644 index 0000000000000..04cb3d7dbf654 --- /dev/null +++ b/core/tests/coretests/src/android/util/CharsetUtilsTest.java @@ -0,0 +1,76 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.util; + +import static org.junit.Assert.assertEquals; + +import androidx.test.runner.AndroidJUnit4; + +import com.android.internal.util.HexDump; + +import dalvik.system.VMRuntime; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; + +@RunWith(AndroidJUnit4.class) +public class CharsetUtilsTest { + private byte[] dest; + private long destPtr; + + @Before + public void setUp() { + dest = (byte[]) VMRuntime.getRuntime().newNonMovableArray(byte.class, 8); + destPtr = VMRuntime.getRuntime().addressOf(dest); + } + + @Test + public void testUtf8_Empty() { + assertEquals(0, CharsetUtils.toUtf8Bytes("", destPtr, 0, dest.length)); + assertEquals("0000000000000000", HexDump.toHexString(dest)); + } + + @Test + public void testUtf8_Simple() { + assertEquals(7, CharsetUtils.toUtf8Bytes("example", destPtr, 0, dest.length)); + assertEquals("6578616D706C6500", HexDump.toHexString(dest)); + } + + @Test + public void testUtf8_Complex() { + assertEquals(3, CharsetUtils.toUtf8Bytes("☃", destPtr, 4, dest.length)); + assertEquals("00000000E2988300", HexDump.toHexString(dest)); + } + + @Test + public void testUtf8_Bounds() { + assertEquals(-1, CharsetUtils.toUtf8Bytes("foo", destPtr, 0, 0)); + assertEquals(-1, CharsetUtils.toUtf8Bytes("foo", destPtr, 0, 2)); + assertEquals(-1, CharsetUtils.toUtf8Bytes("foo", destPtr, -2, 8)); + assertEquals(-1, CharsetUtils.toUtf8Bytes("foo", destPtr, 6, 8)); + assertEquals(-1, CharsetUtils.toUtf8Bytes("foo", destPtr, 10, 8)); + } + + @Test + public void testUtf8_Overwrite() { + assertEquals(5, CharsetUtils.toUtf8Bytes("!!!!!", destPtr, 0, dest.length)); + assertEquals(3, CharsetUtils.toUtf8Bytes("...", destPtr, 0, dest.length)); + assertEquals(1, CharsetUtils.toUtf8Bytes("?", destPtr, 0, dest.length)); + assertEquals("3F002E0021000000", HexDump.toHexString(dest)); + } +} From 162f86ddce3704f490bb51d329c6511129506e48 Mon Sep 17 00:00:00 2001 From: Jeff Sharkey Date: Sat, 7 Nov 2020 12:19:44 -0700 Subject: [PATCH 2/4] More efficient alternatives to ByteBuffer. Some upcoming binary XML work needs to efficiently read and write raw bytes, and we initially started using ByteBuffer. However, that design had additional overhead since we were performing bounds checks twice (once to fill/drain buffers, then again to parse data). In addition, the upstream ByteBuffer makes per-byte method invocations internally, instead of going directly the the buffer. This change introduces FastDataInput/Output as local implementations of DataInput/Output which are focused on performance. They also handle fill/drain from an underlying Input/OutputStream, and the included benchmarks show reading 3x faster and writing 2x faster: timeRead_Upstream_mean: 5543730 timeRead_Local_mean: 1698602 timeWrite_Upstream_mean: 3731119 timeWrite_Local_mean: 1885983 We also use the new CharsetUtils methods to write UTF-8 values directly without additional allocations whenever possible. This requires using a non-movable buffer to avoid JNI overhead to gain the 30% benchmarked performance wins. Bug: 171832118 Test: atest CorePerfTests:com.android.internal.util.FastDataPerfTest Test: atest FrameworksCoreTests:com.android.internal.util.FastDataTest Change-Id: If28ee381adb528d03cc9851d78236d985b6ede16 --- .../internal/util/FastDataPerfTest.java | 132 +++++++ .../android/internal/util/FastDataInput.java | 256 +++++++++++++ .../android/internal/util/FastDataOutput.java | 228 ++++++++++++ .../android/internal/util/FastDataTest.java | 348 ++++++++++++++++++ 4 files changed, 964 insertions(+) create mode 100644 apct-tests/perftests/core/src/com/android/internal/util/FastDataPerfTest.java create mode 100644 core/java/com/android/internal/util/FastDataInput.java create mode 100644 core/java/com/android/internal/util/FastDataOutput.java create mode 100644 core/tests/coretests/src/com/android/internal/util/FastDataTest.java diff --git a/apct-tests/perftests/core/src/com/android/internal/util/FastDataPerfTest.java b/apct-tests/perftests/core/src/com/android/internal/util/FastDataPerfTest.java new file mode 100644 index 0000000000000..2700fff4cba10 --- /dev/null +++ b/apct-tests/perftests/core/src/com/android/internal/util/FastDataPerfTest.java @@ -0,0 +1,132 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.internal.util; + +import android.perftests.utils.BenchmarkState; +import android.perftests.utils.PerfStatusReporter; + +import androidx.test.filters.LargeTest; +import androidx.test.runner.AndroidJUnit4; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInput; +import java.io.DataInputStream; +import java.io.DataOutput; +import java.io.DataOutputStream; +import java.io.IOException; + +@LargeTest +@RunWith(AndroidJUnit4.class) +public class FastDataPerfTest { + @Rule + public PerfStatusReporter mPerfStatusReporter = new PerfStatusReporter(); + + private static final int OUTPUT_SIZE = 64000; + private static final int BUFFER_SIZE = 4096; + + @Test + public void timeWrite_Upstream() throws IOException { + final ByteArrayOutputStream os = new ByteArrayOutputStream(OUTPUT_SIZE); + final BenchmarkState state = mPerfStatusReporter.getBenchmarkState(); + while (state.keepRunning()) { + os.reset(); + final BufferedOutputStream bos = new BufferedOutputStream(os, BUFFER_SIZE); + final DataOutput out = new DataOutputStream(bos); + doWrite(out); + bos.flush(); + } + } + + @Test + public void timeWrite_Local() throws IOException { + final ByteArrayOutputStream os = new ByteArrayOutputStream(OUTPUT_SIZE); + final BenchmarkState state = mPerfStatusReporter.getBenchmarkState(); + while (state.keepRunning()) { + os.reset(); + final FastDataOutput out = new FastDataOutput(os, BUFFER_SIZE); + doWrite(out); + out.flush(); + } + } + + @Test + public void timeRead_Upstream() throws Exception { + final ByteArrayInputStream is = new ByteArrayInputStream(doWrite()); + final BenchmarkState state = mPerfStatusReporter.getBenchmarkState(); + while (state.keepRunning()) { + is.reset(); + final BufferedInputStream bis = new BufferedInputStream(is, BUFFER_SIZE); + final DataInput in = new DataInputStream(bis); + doRead(in); + } + } + + @Test + public void timeRead_Local() throws Exception { + final ByteArrayInputStream is = new ByteArrayInputStream(doWrite()); + final BenchmarkState state = mPerfStatusReporter.getBenchmarkState(); + while (state.keepRunning()) { + is.reset(); + final DataInput in = new FastDataInput(is, BUFFER_SIZE); + doRead(in); + } + } + + /** + * Since each iteration is around 64 bytes, we need to iterate many times to + * exercise the buffer logic. + */ + private static final int REPEATS = 1000; + + private static byte[] doWrite() throws IOException { + final ByteArrayOutputStream os = new ByteArrayOutputStream(OUTPUT_SIZE); + final DataOutput out = new DataOutputStream(os); + doWrite(out); + return os.toByteArray(); + } + + private static void doWrite(DataOutput out) throws IOException { + for (int i = 0; i < REPEATS; i++) { + out.writeByte(Byte.MAX_VALUE); + out.writeShort(Short.MAX_VALUE); + out.writeInt(Integer.MAX_VALUE); + out.writeLong(Long.MAX_VALUE); + out.writeFloat(Float.MAX_VALUE); + out.writeDouble(Double.MAX_VALUE); + out.writeUTF("com.example.typical_package_name"); + } + } + + private static void doRead(DataInput in) throws IOException { + for (int i = 0; i < REPEATS; i++) { + in.readByte(); + in.readShort(); + in.readInt(); + in.readLong(); + in.readFloat(); + in.readDouble(); + in.readUTF(); + } + } +} diff --git a/core/java/com/android/internal/util/FastDataInput.java b/core/java/com/android/internal/util/FastDataInput.java new file mode 100644 index 0000000000000..2e8cb473c8096 --- /dev/null +++ b/core/java/com/android/internal/util/FastDataInput.java @@ -0,0 +1,256 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.internal.util; + +import android.annotation.NonNull; + +import java.io.BufferedInputStream; +import java.io.Closeable; +import java.io.DataInput; +import java.io.DataInputStream; +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Objects; + +/** + * Optimized implementation of {@link DataInput} which buffers data in memory + * from the underlying {@link InputStream}. + *

+ * Benchmarks have demonstrated this class is 3x more efficient than using a + * {@link DataInputStream} with a {@link BufferedInputStream}. + */ +public class FastDataInput implements DataInput, Closeable { + private static final int MAX_UNSIGNED_SHORT = 65_535; + + private final InputStream mIn; + + private final byte[] mBuffer; + private final int mBufferCap; + + private int mBufferPos; + private int mBufferLim; + + /** + * Values that have been "interned" by {@link #readInternedUTF()}. + */ + private int mStringRefCount = 0; + private String[] mStringRefs = new String[32]; + + public FastDataInput(@NonNull InputStream in, int bufferSize) { + mIn = Objects.requireNonNull(in); + if (bufferSize < 8) { + throw new IllegalArgumentException(); + } + + mBuffer = new byte[bufferSize]; + mBufferCap = mBuffer.length; + } + + private void fill(int need) throws IOException { + final int remain = mBufferLim - mBufferPos; + System.arraycopy(mBuffer, mBufferPos, mBuffer, 0, remain); + mBufferPos = 0; + mBufferLim = remain; + need -= remain; + + while (need > 0) { + int c = mIn.read(mBuffer, mBufferLim, mBufferCap - mBufferLim); + if (c == -1) { + throw new EOFException(); + } else { + mBufferLim += c; + need -= c; + } + } + } + + @Override + public void close() throws IOException { + mIn.close(); + } + + @Override + public void readFully(byte[] b) throws IOException { + readFully(b, 0, b.length); + } + + @Override + public void readFully(byte[] b, int off, int len) throws IOException { + // Attempt to read directly from buffer space if there's enough room, + // otherwise fall back to chunking into place + if (mBufferCap >= len) { + if (mBufferLim - mBufferPos < len) fill(len); + System.arraycopy(mBuffer, mBufferPos, b, off, len); + mBufferPos += len; + } else { + final int remain = mBufferLim - mBufferPos; + System.arraycopy(mBuffer, mBufferPos, b, off, remain); + mBufferPos += remain; + off += remain; + len -= remain; + + while (len > 0) { + int c = mIn.read(b, off, len); + if (c == -1) { + throw new EOFException(); + } else { + off += c; + len -= c; + } + } + } + } + + @Override + public String readUTF() throws IOException { + // Attempt to read directly from buffer space if there's enough room, + // otherwise fall back to chunking into place + final int len = readUnsignedShort(); + if (mBufferCap >= len) { + if (mBufferLim - mBufferPos < len) fill(len); + final String res = new String(mBuffer, mBufferPos, len, StandardCharsets.UTF_8); + mBufferPos += len; + return res; + } else { + final byte[] tmp = new byte[len]; + readFully(tmp, 0, tmp.length); + return new String(tmp, StandardCharsets.UTF_8); + } + } + + /** + * Read a {@link String} value with the additional signal that the given + * value is a candidate for being canonicalized, similar to + * {@link String#intern()}. + *

+ * Canonicalization is implemented by writing each unique string value once + * the first time it appears, and then writing a lightweight {@code short} + * reference when that string is written again in the future. + * + * @see FastDataOutput#writeInternedUTF(String) + */ + public @NonNull String readInternedUTF() throws IOException { + final int ref = readUnsignedShort(); + if (ref == MAX_UNSIGNED_SHORT) { + final String s = readUTF(); + + // We can only safely intern when we have remaining values; if we're + // full we at least sent the string value above + if (mStringRefCount < MAX_UNSIGNED_SHORT) { + if (mStringRefCount == mStringRefs.length) { + mStringRefs = Arrays.copyOf(mStringRefs, + mStringRefCount + (mStringRefCount >> 1)); + } + mStringRefs[mStringRefCount++] = s; + } + + return s; + } else { + return mStringRefs[ref]; + } + } + + @Override + public boolean readBoolean() throws IOException { + return readByte() != 0; + } + + /** + * Returns the same decoded value as {@link #readByte()} but without + * actually consuming the underlying data. + */ + public byte peekByte() throws IOException { + if (mBufferLim - mBufferPos < 1) fill(1); + return mBuffer[mBufferPos]; + } + + @Override + public byte readByte() throws IOException { + if (mBufferLim - mBufferPos < 1) fill(1); + return mBuffer[mBufferPos++]; + } + + @Override + public int readUnsignedByte() throws IOException { + return Byte.toUnsignedInt(readByte()); + } + + @Override + public short readShort() throws IOException { + if (mBufferLim - mBufferPos < 2) fill(2); + return (short) (((mBuffer[mBufferPos++] & 0xff) << 8) | + ((mBuffer[mBufferPos++] & 0xff) << 0)); + } + + @Override + public int readUnsignedShort() throws IOException { + return Short.toUnsignedInt((short) readShort()); + } + + @Override + public char readChar() throws IOException { + return (char) readShort(); + } + + @Override + public int readInt() throws IOException { + if (mBufferLim - mBufferPos < 4) fill(4); + return (((mBuffer[mBufferPos++] & 0xff) << 24) | + ((mBuffer[mBufferPos++] & 0xff) << 16) | + ((mBuffer[mBufferPos++] & 0xff) << 8) | + ((mBuffer[mBufferPos++] & 0xff) << 0)); + } + + @Override + public long readLong() throws IOException { + if (mBufferLim - mBufferPos < 8) fill(8); + int h = ((mBuffer[mBufferPos++] & 0xff) << 24) | + ((mBuffer[mBufferPos++] & 0xff) << 16) | + ((mBuffer[mBufferPos++] & 0xff) << 8) | + ((mBuffer[mBufferPos++] & 0xff) << 0); + int l = ((mBuffer[mBufferPos++] & 0xff) << 24) | + ((mBuffer[mBufferPos++] & 0xff) << 16) | + ((mBuffer[mBufferPos++] & 0xff) << 8) | + ((mBuffer[mBufferPos++] & 0xff) << 0); + return (((long) h) << 32L) | ((long) l) & 0xffffffffL; + } + + @Override + public float readFloat() throws IOException { + return Float.intBitsToFloat(readInt()); + } + + @Override + public double readDouble() throws IOException { + return Double.longBitsToDouble(readLong()); + } + + @Override + public int skipBytes(int n) throws IOException { + // Callers should read data piecemeal + throw new UnsupportedOperationException(); + } + + @Override + public String readLine() throws IOException { + // Callers should read data piecemeal + throw new UnsupportedOperationException(); + } +} diff --git a/core/java/com/android/internal/util/FastDataOutput.java b/core/java/com/android/internal/util/FastDataOutput.java new file mode 100644 index 0000000000000..2530501bf85f7 --- /dev/null +++ b/core/java/com/android/internal/util/FastDataOutput.java @@ -0,0 +1,228 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.internal.util; + +import android.annotation.NonNull; +import android.util.CharsetUtils; + +import dalvik.system.VMRuntime; + +import java.io.BufferedOutputStream; +import java.io.Closeable; +import java.io.DataOutput; +import java.io.DataOutputStream; +import java.io.Flushable; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Objects; + +/** + * Optimized implementation of {@link DataOutput} which buffers data in memory + * before flushing to the underlying {@link OutputStream}. + *

+ * Benchmarks have demonstrated this class is 2x more efficient than using a + * {@link DataOutputStream} with a {@link BufferedOutputStream}. + */ +public class FastDataOutput implements DataOutput, Flushable, Closeable { + private static final int MAX_UNSIGNED_SHORT = 65_535; + + private final OutputStream mOut; + + private final byte[] mBuffer; + private final long mBufferPtr; + private final int mBufferCap; + + private int mBufferPos; + + /** + * Values that have been "interned" by {@link #writeInternedUTF(String)}. + */ + private HashMap mStringRefs = new HashMap<>(); + + public FastDataOutput(@NonNull OutputStream out, int bufferSize) { + mOut = Objects.requireNonNull(out); + if (bufferSize < 8) { + throw new IllegalArgumentException(); + } + + mBuffer = (byte[]) VMRuntime.getRuntime().newNonMovableArray(byte.class, bufferSize); + mBufferPtr = VMRuntime.getRuntime().addressOf(mBuffer); + mBufferCap = mBuffer.length; + } + + private void drain() throws IOException { + if (mBufferPos > 0) { + mOut.write(mBuffer, 0, mBufferPos); + mBufferPos = 0; + } + } + + @Override + public void flush() throws IOException { + drain(); + mOut.flush(); + } + + @Override + public void close() throws IOException { + mOut.close(); + } + + @Override + public void write(int b) throws IOException { + writeByte(b); + } + + @Override + public void write(byte[] b) throws IOException { + write(b, 0, b.length); + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + if (mBufferCap < len) { + drain(); + mOut.write(b, off, len); + } else { + if (mBufferCap - mBufferPos < len) drain(); + System.arraycopy(b, off, mBuffer, mBufferPos, len); + mBufferPos += len; + } + } + + @Override + public void writeUTF(String s) throws IOException { + // Attempt to write directly to buffer space if there's enough room, + // otherwise fall back to chunking into place + if (mBufferCap - mBufferPos < 2 + s.length()) drain(); + final int res = CharsetUtils.toUtf8Bytes(s, mBufferPtr, mBufferPos + 2, + mBufferCap - mBufferPos - 2); + if (res >= 0) { + if (res > MAX_UNSIGNED_SHORT) { + throw new IOException("UTF-8 length too large: " + res); + } + writeShort(res); + mBufferPos += res; + } else { + final byte[] tmp = s.getBytes(StandardCharsets.UTF_8); + if (tmp.length > MAX_UNSIGNED_SHORT) { + throw new IOException("UTF-8 length too large: " + res); + } + writeShort(tmp.length); + write(tmp, 0, tmp.length); + } + } + + /** + * Write a {@link String} value with the additional signal that the given + * value is a candidate for being canonicalized, similar to + * {@link String#intern()}. + *

+ * Canonicalization is implemented by writing each unique string value once + * the first time it appears, and then writing a lightweight {@code short} + * reference when that string is written again in the future. + * + * @see FastDataInput#readInternedUTF() + */ + public void writeInternedUTF(@NonNull String s) throws IOException { + Short ref = mStringRefs.get(s); + if (ref != null) { + writeShort(ref); + } else { + writeShort(MAX_UNSIGNED_SHORT); + writeUTF(s); + + // We can only safely intern when we have remaining values; if we're + // full we at least sent the string value above + ref = (short) mStringRefs.size(); + if (ref < MAX_UNSIGNED_SHORT) { + mStringRefs.put(s, ref); + } + } + } + + @Override + public void writeBoolean(boolean v) throws IOException { + writeByte(v ? 1 : 0); + } + + @Override + public void writeByte(int v) throws IOException { + if (mBufferCap - mBufferPos < 1) drain(); + mBuffer[mBufferPos++] = (byte) ((v >> 0) & 0xff); + } + + @Override + public void writeShort(int v) throws IOException { + if (mBufferCap - mBufferPos < 2) drain(); + mBuffer[mBufferPos++] = (byte) ((v >> 8) & 0xff); + mBuffer[mBufferPos++] = (byte) ((v >> 0) & 0xff); + } + + @Override + public void writeChar(int v) throws IOException { + writeShort((short) v); + } + + @Override + public void writeInt(int v) throws IOException { + if (mBufferCap - mBufferPos < 4) drain(); + mBuffer[mBufferPos++] = (byte) ((v >> 24) & 0xff); + mBuffer[mBufferPos++] = (byte) ((v >> 16) & 0xff); + mBuffer[mBufferPos++] = (byte) ((v >> 8) & 0xff); + mBuffer[mBufferPos++] = (byte) ((v >> 0) & 0xff); + } + + @Override + public void writeLong(long v) throws IOException { + if (mBufferCap - mBufferPos < 8) drain(); + int i = (int) (v >> 32); + mBuffer[mBufferPos++] = (byte) ((i >> 24) & 0xff); + mBuffer[mBufferPos++] = (byte) ((i >> 16) & 0xff); + mBuffer[mBufferPos++] = (byte) ((i >> 8) & 0xff); + mBuffer[mBufferPos++] = (byte) ((i >> 0) & 0xff); + i = (int) v; + mBuffer[mBufferPos++] = (byte) ((i >> 24) & 0xff); + mBuffer[mBufferPos++] = (byte) ((i >> 16) & 0xff); + mBuffer[mBufferPos++] = (byte) ((i >> 8) & 0xff); + mBuffer[mBufferPos++] = (byte) ((i >> 0) & 0xff); + } + + @Override + public void writeFloat(float v) throws IOException { + writeInt(Float.floatToIntBits(v)); + } + + @Override + public void writeDouble(double v) throws IOException { + writeLong(Double.doubleToLongBits(v)); + } + + @Override + public void writeBytes(String s) throws IOException { + // Callers should use writeUTF() + throw new UnsupportedOperationException(); + } + + @Override + public void writeChars(String s) throws IOException { + // Callers should use writeUTF() + throw new UnsupportedOperationException(); + } +} diff --git a/core/tests/coretests/src/com/android/internal/util/FastDataTest.java b/core/tests/coretests/src/com/android/internal/util/FastDataTest.java new file mode 100644 index 0000000000000..841d6597343f1 --- /dev/null +++ b/core/tests/coretests/src/com/android/internal/util/FastDataTest.java @@ -0,0 +1,348 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.internal.util; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import android.annotation.NonNull; +import android.util.ExceptionUtils; + +import androidx.test.ext.junit.runners.AndroidJUnit4; + +import org.junit.Test; +import org.junit.runner.RunWith; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.EOFException; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.function.Consumer; + +@RunWith(AndroidJUnit4.class) +public class FastDataTest { + private static final String TEST_SHORT_STRING = "a"; + private static final String TEST_LONG_STRING = "com☃example☃typical☃package☃name"; + private static final byte[] TEST_BYTES = TEST_LONG_STRING.getBytes(StandardCharsets.UTF_16LE); + + @Test + public void testEndOfFile_Int() throws Exception { + try (FastDataInput in = new FastDataInput(new ByteArrayInputStream( + new byte[] { 1 }), 1000)) { + assertThrows(EOFException.class, () -> in.readInt()); + } + try (FastDataInput in = new FastDataInput(new ByteArrayInputStream( + new byte[] { 1, 1, 1, 1 }), 1000)) { + assertEquals(1, in.readByte()); + assertThrows(EOFException.class, () -> in.readInt()); + } + } + + @Test + public void testEndOfFile_String() throws Exception { + try (FastDataInput in = new FastDataInput(new ByteArrayInputStream( + new byte[] { 1 }), 1000)) { + assertThrows(EOFException.class, () -> in.readUTF()); + } + try (FastDataInput in = new FastDataInput(new ByteArrayInputStream( + new byte[] { 1, 1, 1, 1 }), 1000)) { + assertThrows(EOFException.class, () -> in.readUTF()); + } + } + + @Test + public void testEndOfFile_Bytes_Small() throws Exception { + try (FastDataInput in = new FastDataInput(new ByteArrayInputStream( + new byte[] { 1, 1, 1, 1 }), 1000)) { + final byte[] tmp = new byte[10]; + assertThrows(EOFException.class, () -> in.readFully(tmp)); + } + try (FastDataInput in = new FastDataInput(new ByteArrayInputStream( + new byte[] { 1, 1, 1, 1 }), 1000)) { + final byte[] tmp = new byte[10_000]; + assertThrows(EOFException.class, () -> in.readFully(tmp)); + } + } + + @Test + public void testUTF_Bounds() throws Exception { + final char[] buf = new char[65_535]; + try (FastDataOutput out = new FastDataOutput(new ByteArrayOutputStream(), BOUNCE_SIZE)) { + // Writing simple string will fit fine + Arrays.fill(buf, '!'); + final String simple = new String(buf); + out.writeUTF(simple); + out.writeInternedUTF(simple); + + // Just one complex char will cause it to overflow + buf[0] = '☃'; + final String complex = new String(buf); + assertThrows(IOException.class, () -> out.writeUTF(complex)); + assertThrows(IOException.class, () -> out.writeInternedUTF(complex)); + } + } + + @Test + public void testBounce_Char() throws Exception { + doBounce((out) -> { + out.writeChar('\0'); + out.writeChar('☃'); + }, (in) -> { + assertEquals('\0', in.readChar()); + assertEquals('☃', in.readChar()); + }); + } + + @Test + public void testBounce_Short() throws Exception { + doBounce((out) -> { + out.writeShort(0); + out.writeShort((short) 0x0f0f); + out.writeShort((short) 0xf0f0); + out.writeShort(Short.MIN_VALUE); + out.writeShort(Short.MAX_VALUE); + }, (in) -> { + assertEquals(0, in.readShort()); + assertEquals((short) 0x0f0f, in.readShort()); + assertEquals((short) 0xf0f0, in.readShort()); + assertEquals(Short.MIN_VALUE, in.readShort()); + assertEquals(Short.MAX_VALUE, in.readShort()); + }); + } + + @Test + public void testBounce_Int() throws Exception { + doBounce((out) -> { + out.writeInt(0); + out.writeInt(0x0f0f0f0f); + out.writeInt(0xf0f0f0f0); + out.writeInt(Integer.MIN_VALUE); + out.writeInt(Integer.MAX_VALUE); + }, (in) -> { + assertEquals(0, in.readInt()); + assertEquals(0x0f0f0f0f, in.readInt()); + assertEquals(0xf0f0f0f0, in.readInt()); + assertEquals(Integer.MIN_VALUE, in.readInt()); + assertEquals(Integer.MAX_VALUE, in.readInt()); + }); + } + + @Test + public void testBounce_Long() throws Exception { + doBounce((out) -> { + out.writeLong(0); + out.writeLong(0x0f0f0f0f0f0f0f0fL); + out.writeLong(0xf0f0f0f0f0f0f0f0L); + out.writeLong(Long.MIN_VALUE); + out.writeLong(Long.MAX_VALUE); + }, (in) -> { + assertEquals(0, in.readLong()); + assertEquals(0x0f0f0f0f0f0f0f0fL, in.readLong()); + assertEquals(0xf0f0f0f0f0f0f0f0L, in.readLong()); + assertEquals(Long.MIN_VALUE, in.readLong()); + assertEquals(Long.MAX_VALUE, in.readLong()); + }); + } + + @Test + public void testBounce_UTF() throws Exception { + doBounce((out) -> { + out.writeUTF(""); + out.writeUTF("☃"); + out.writeUTF("example"); + }, (in) -> { + assertEquals("", in.readUTF()); + assertEquals("☃", in.readUTF()); + assertEquals("example", in.readUTF()); + }); + } + + @Test + public void testBounce_UTF_Exact() throws Exception { + final char[] expectedBuf = new char[BOUNCE_SIZE]; + Arrays.fill(expectedBuf, '!'); + final String expected = new String(expectedBuf); + + doBounce((out) -> { + out.writeUTF(expected); + }, (in) -> { + final String actual = in.readUTF(); + assertEquals(expected.length(), actual.length()); + assertEquals(expected, actual); + }); + } + + @Test + public void testBounce_UTF_Maximum() throws Exception { + final char[] expectedBuf = new char[65_535]; + Arrays.fill(expectedBuf, '!'); + final String expected = new String(expectedBuf); + + doBounce((out) -> { + out.writeUTF(expected); + }, (in) -> { + final String actual = in.readUTF(); + assertEquals(expected.length(), actual.length()); + assertEquals(expected, actual); + }, 1); + } + + @Test + public void testBounce_InternedUTF() throws Exception { + doBounce((out) -> { + out.writeInternedUTF("foo"); + out.writeInternedUTF("bar"); + out.writeInternedUTF("baz"); + out.writeInternedUTF("bar"); + out.writeInternedUTF("foo"); + }, (in) -> { + assertEquals("foo", in.readInternedUTF()); + assertEquals("bar", in.readInternedUTF()); + assertEquals("baz", in.readInternedUTF()); + assertEquals("bar", in.readInternedUTF()); + assertEquals("foo", in.readInternedUTF()); + }); + } + + /** + * Verify that when we overflow the maximum number of interned string + * references, we still transport the raw string values successfully. + */ + @Test + public void testBounce_InternedUTF_Maximum() throws Exception { + final int num = 70_000; + doBounce((out) -> { + for (int i = 0; i < num; i++) { + out.writeInternedUTF("foo" + i); + } + }, (in) -> { + for (int i = 0; i < num; i++) { + assertEquals("foo" + i, in.readInternedUTF()); + } + }, 1); + } + + @Test + public void testBounce_Bytes() throws Exception { + doBounce((out) -> { + out.write(TEST_BYTES, 8, 32); + out.writeInt(64); + }, (in) -> { + final byte[] tmp = new byte[128]; + in.readFully(tmp, 8, 32); + assertArrayEquals(Arrays.copyOfRange(TEST_BYTES, 8, 8 + 32), + Arrays.copyOfRange(tmp, 8, 8 + 32)); + assertEquals(64, in.readInt()); + }); + } + + @Test + public void testBounce_Mixed() throws Exception { + doBounce((out) -> { + out.writeBoolean(true); + out.writeBoolean(false); + out.writeByte(1); + out.writeShort(2); + out.writeInt(4); + out.writeUTF(TEST_SHORT_STRING); + out.writeUTF(TEST_LONG_STRING); + out.writeLong(8L); + out.writeFloat(16f); + out.writeDouble(32d); + }, (in) -> { + assertEquals(true, in.readBoolean()); + assertEquals(false, in.readBoolean()); + assertEquals(1, in.readByte()); + assertEquals(2, in.readShort()); + assertEquals(4, in.readInt()); + assertEquals(TEST_SHORT_STRING, in.readUTF()); + assertEquals(TEST_LONG_STRING, in.readUTF()); + assertEquals(8L, in.readLong()); + assertEquals(16f, in.readFloat(), 0.01); + assertEquals(32d, in.readDouble(), 0.01); + }); + } + + /** + * Buffer size to use for {@link #doBounce}; purposefully chosen to be a + * small prime number to help uncover edge cases. + */ + private static final int BOUNCE_SIZE = 11; + + /** + * Number of times to repeat message when bouncing; repeating is used to + * help uncover edge cases. + */ + private static final int BOUNCE_REPEAT = 1_000; + + /** + * Verify that some common data can be written and read back, effectively + * "bouncing" it through a serialized representation. + */ + private static void doBounce(@NonNull ThrowingConsumer out, + @NonNull ThrowingConsumer in) throws Exception { + doBounce(out, in, BOUNCE_REPEAT); + } + + private static void doBounce(@NonNull ThrowingConsumer out, + @NonNull ThrowingConsumer in, int count) throws Exception { + final ByteArrayOutputStream outStream = new ByteArrayOutputStream(); + final FastDataOutput outData = new FastDataOutput(outStream, BOUNCE_SIZE); + for (int i = 0; i < count; i++) { + out.accept(outData); + } + outData.flush(); + + final ByteArrayInputStream inStream = new ByteArrayInputStream(outStream.toByteArray()); + final FastDataInput inData = new FastDataInput(inStream, BOUNCE_SIZE); + for (int i = 0; i < count; i++) { + in.accept(inData); + } + } + + private static void assertThrows(Class clazz, ThrowingRunnable r) + throws Exception { + try { + r.run(); + fail("Expected " + clazz + " to be thrown"); + } catch (Exception e) { + if (!clazz.isAssignableFrom(e.getClass())) { + throw e; + } + } + } + + public interface ThrowingRunnable { + void run() throws Exception; + } + + public interface ThrowingConsumer extends Consumer { + void acceptOrThrow(T t) throws Exception; + + @Override + default void accept(T t) { + try { + acceptOrThrow(t); + } catch (Exception ex) { + throw ExceptionUtils.propagate(ex); + } + } + } +} From b5856365b40c454ed126210b99679d9d1a444b28 Mon Sep 17 00:00:00 2001 From: Jeff Sharkey Date: Wed, 28 Oct 2020 11:03:23 -0600 Subject: [PATCH 3/4] Progress towards efficient XML serialization. We've identified that XML writing and reading uses roughly 1.5% of all system_server CPU, and can generate many temporary objects. To set the stage for some long-term improvements, this change introduces new TypedXmlSerializer and TypedXmlPullParser interfaces which offer efficient access to primitive attributes and text. This change also updates XmlUtils to redirect primitive operations through these new interfaces. Bug: 171832118 Test: atest FrameworksCoreTests:android.util.XmlTest Change-Id: I5ba3ad87cf79ca10705a96b98855164a27fee021 --- .../java/android/util/TypedXmlPullParser.java | 186 ++++++++++++ .../java/android/util/TypedXmlSerializer.java | 103 +++++++ core/java/android/util/Xml.java | 83 +++++ .../internal/util/XmlPullParserWrapper.java | 189 ++++++++++++ .../internal/util/XmlSerializerWrapper.java | 140 +++++++++ .../com/android/internal/util/XmlUtils.java | 189 +++++++++++- .../coretests/src/android/util/XmlTest.java | 287 ++++++++++++++++++ 7 files changed, 1176 insertions(+), 1 deletion(-) create mode 100644 core/java/android/util/TypedXmlPullParser.java create mode 100644 core/java/android/util/TypedXmlSerializer.java create mode 100644 core/java/com/android/internal/util/XmlPullParserWrapper.java create mode 100644 core/java/com/android/internal/util/XmlSerializerWrapper.java create mode 100644 core/tests/coretests/src/android/util/XmlTest.java diff --git a/core/java/android/util/TypedXmlPullParser.java b/core/java/android/util/TypedXmlPullParser.java new file mode 100644 index 0000000000000..5ff7e5dd022fa --- /dev/null +++ b/core/java/android/util/TypedXmlPullParser.java @@ -0,0 +1,186 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.util; + +import android.annotation.NonNull; +import android.annotation.Nullable; + +import org.xmlpull.v1.XmlPullParser; + +import java.io.IOException; + +/** + * Specialization of {@link XmlPullParser} which adds explicit methods to + * support consistent and efficient conversion of primitive data types. + * + * @hide + */ +public interface TypedXmlPullParser extends XmlPullParser { + /** + * @return decoded strongly-typed {@link #getAttributeValue}, or + * {@code null} if malformed or undefined + */ + @Nullable byte[] getAttributeBytesHex(@Nullable String namespace, @NonNull String name) + throws IOException; + + /** + * @return decoded strongly-typed {@link #getAttributeValue}, or + * {@code null} if malformed or undefined + */ + @Nullable byte[] getAttributeBytesBase64(@Nullable String namespace, @NonNull String name) + throws IOException; + + /** + * @return decoded strongly-typed {@link #getAttributeValue} + * @throws IOException if the value is malformed or undefined + */ + int getAttributeInt(@Nullable String namespace, @NonNull String name) + throws IOException; + + /** + * @return decoded strongly-typed {@link #getAttributeValue} + * @throws IOException if the value is malformed or undefined + */ + int getAttributeIntHex(@Nullable String namespace, @NonNull String name) + throws IOException; + + /** + * @return decoded strongly-typed {@link #getAttributeValue} + * @throws IOException if the value is malformed or undefined + */ + long getAttributeLong(@Nullable String namespace, @NonNull String name) + throws IOException; + + /** + * @return decoded strongly-typed {@link #getAttributeValue} + * @throws IOException if the value is malformed or undefined + */ + long getAttributeLongHex(@Nullable String namespace, @NonNull String name) + throws IOException; + + /** + * @return decoded strongly-typed {@link #getAttributeValue} + * @throws IOException if the value is malformed or undefined + */ + float getAttributeFloat(@Nullable String namespace, @NonNull String name) + throws IOException; + + /** + * @return decoded strongly-typed {@link #getAttributeValue} + * @throws IOException if the value is malformed or undefined + */ + double getAttributeDouble(@Nullable String namespace, @NonNull String name) + throws IOException; + + /** + * @return decoded strongly-typed {@link #getAttributeValue} + * @throws IOException if the value is malformed or undefined + */ + boolean getAttributeBoolean(@Nullable String namespace, @NonNull String name) + throws IOException; + + /** + * @return decoded strongly-typed {@link #getAttributeValue}, otherwise + * default value if the value is malformed or undefined + */ + default int getAttributeInt(@Nullable String namespace, @NonNull String name, + int defaultValue) { + try { + return getAttributeInt(namespace, name); + } catch (Exception ignored) { + return defaultValue; + } + } + + /** + * @return decoded strongly-typed {@link #getAttributeValue}, otherwise + * default value if the value is malformed or undefined + */ + default int getAttributeIntHex(@Nullable String namespace, @NonNull String name, + int defaultValue) { + try { + return getAttributeIntHex(namespace, name); + } catch (Exception ignored) { + return defaultValue; + } + } + + /** + * @return decoded strongly-typed {@link #getAttributeValue}, otherwise + * default value if the value is malformed or undefined + */ + default long getAttributeLong(@Nullable String namespace, @NonNull String name, + long defaultValue) { + try { + return getAttributeLong(namespace, name); + } catch (Exception ignored) { + return defaultValue; + } + } + + /** + * @return decoded strongly-typed {@link #getAttributeValue}, otherwise + * default value if the value is malformed or undefined + */ + default long getAttributeLongHex(@Nullable String namespace, @NonNull String name, + long defaultValue) { + try { + return getAttributeLongHex(namespace, name); + } catch (Exception ignored) { + return defaultValue; + } + } + + /** + * @return decoded strongly-typed {@link #getAttributeValue}, otherwise + * default value if the value is malformed or undefined + */ + default float getAttributeFloat(@Nullable String namespace, @NonNull String name, + float defaultValue) { + try { + return getAttributeFloat(namespace, name); + } catch (Exception ignored) { + return defaultValue; + } + } + + /** + * @return decoded strongly-typed {@link #getAttributeValue}, otherwise + * default value if the value is malformed or undefined + */ + default double getAttributeDouble(@Nullable String namespace, @NonNull String name, + double defaultValue) { + try { + return getAttributeDouble(namespace, name); + } catch (Exception ignored) { + return defaultValue; + } + } + + /** + * @return decoded strongly-typed {@link #getAttributeValue}, otherwise + * default value if the value is malformed or undefined + */ + default boolean getAttributeBoolean(@Nullable String namespace, @NonNull String name, + boolean defaultValue) { + try { + return getAttributeBoolean(namespace, name); + } catch (Exception ignored) { + return defaultValue; + } + } +} diff --git a/core/java/android/util/TypedXmlSerializer.java b/core/java/android/util/TypedXmlSerializer.java new file mode 100644 index 0000000000000..fe5e3e6e9a52c --- /dev/null +++ b/core/java/android/util/TypedXmlSerializer.java @@ -0,0 +1,103 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.util; + +import android.annotation.NonNull; +import android.annotation.Nullable; + +import org.xmlpull.v1.XmlSerializer; + +import java.io.IOException; + +/** + * Specialization of {@link XmlSerializer} which adds explicit methods to + * support consistent and efficient conversion of primitive data types. + * + * @hide + */ +public interface TypedXmlSerializer extends XmlSerializer { + /** + * Functionally equivalent to {@link #attribute(String, String, String)} but + * with the additional signal that the given value is a candidate for being + * canonicalized, similar to {@link String#intern()}. + */ + @NonNull XmlSerializer attributeInterned(@Nullable String namespace, @NonNull String name, + @Nullable String value) throws IOException; + + /** + * Encode the given strongly-typed value and serialize using + * {@link #attribute(String, String, String)}. + */ + @NonNull XmlSerializer attributeBytesHex(@Nullable String namespace, @NonNull String name, + byte[] value) throws IOException; + + /** + * Encode the given strongly-typed value and serialize using + * {@link #attribute(String, String, String)}. + */ + @NonNull XmlSerializer attributeBytesBase64(@Nullable String namespace, @NonNull String name, + byte[] value) throws IOException; + + /** + * Encode the given strongly-typed value and serialize using + * {@link #attribute(String, String, String)}. + */ + @NonNull XmlSerializer attributeInt(@Nullable String namespace, @NonNull String name, + int value) throws IOException; + + /** + * Encode the given strongly-typed value and serialize using + * {@link #attribute(String, String, String)}. + */ + @NonNull XmlSerializer attributeIntHex(@Nullable String namespace, @NonNull String name, + int value) throws IOException; + + /** + * Encode the given strongly-typed value and serialize using + * {@link #attribute(String, String, String)}. + */ + @NonNull XmlSerializer attributeLong(@Nullable String namespace, @NonNull String name, + long value) throws IOException; + + /** + * Encode the given strongly-typed value and serialize using + * {@link #attribute(String, String, String)}. + */ + @NonNull XmlSerializer attributeLongHex(@Nullable String namespace, @NonNull String name, + long value) throws IOException; + + /** + * Encode the given strongly-typed value and serialize using + * {@link #attribute(String, String, String)}. + */ + @NonNull XmlSerializer attributeFloat(@Nullable String namespace, @NonNull String name, + float value) throws IOException; + + /** + * Encode the given strongly-typed value and serialize using + * {@link #attribute(String, String, String)}. + */ + @NonNull XmlSerializer attributeDouble(@Nullable String namespace, @NonNull String name, + double value) throws IOException; + + /** + * Encode the given strongly-typed value and serialize using + * {@link #attribute(String, String, String)}. + */ + @NonNull XmlSerializer attributeBoolean(@Nullable String namespace, @NonNull String name, + boolean value) throws IOException; +} diff --git a/core/java/android/util/Xml.java b/core/java/android/util/Xml.java index e3b8fec3559eb..9849c10c1604e 100644 --- a/core/java/android/util/Xml.java +++ b/core/java/android/util/Xml.java @@ -16,6 +16,11 @@ package android.util; +import android.annotation.NonNull; + +import com.android.internal.util.FastXmlSerializer; +import com.android.internal.util.XmlUtils; + import libcore.util.XmlObjectFactory; import org.xml.sax.ContentHandler; @@ -26,11 +31,15 @@ import org.xmlpull.v1.XmlPullParser; import org.xmlpull.v1.XmlPullParserException; import org.xmlpull.v1.XmlSerializer; +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; import java.io.Reader; import java.io.StringReader; import java.io.UnsupportedEncodingException; +import java.nio.charset.StandardCharsets; /** * XML utility methods. @@ -98,6 +107,45 @@ public class Xml { } } + /** + * Creates a new {@link TypedXmlPullParser} which is optimized for use + * inside the system, typically by supporting only a basic set of features. + *

+ * In particular, the returned parser does not support namespaces, prefixes, + * properties, or options. + * + * @hide + */ + public static @NonNull TypedXmlPullParser newFastPullParser() { + return XmlUtils.makeTyped(newPullParser()); + } + + /** + * Creates a new {@link XmlPullParser} which is optimized for use inside the + * system, typically by supporting only a basic set of features. + *

+ * This returned instance may be configured to read using an efficient + * binary format instead of a human-readable text format, depending on + * device feature flags. + *

+ * To ensure that both formats are detected and transparently handled + * correctly, you must shift to using both {@link #resolveSerializer} and + * {@link #resolvePullParser}. + * + * @hide + */ + public static @NonNull TypedXmlPullParser resolvePullParser(@NonNull InputStream in) + throws IOException { + // TODO: add support for binary format + final TypedXmlPullParser xml = newFastPullParser(); + try { + xml.setInput(in, StandardCharsets.UTF_8.name()); + } catch (XmlPullParserException e) { + throw new IOException(e); + } + return xml; + } + /** * Creates a new xml serializer. */ @@ -105,6 +153,41 @@ public class Xml { return XmlObjectFactory.newXmlSerializer(); } + /** + * Creates a new {@link XmlSerializer} which is optimized for use inside the + * system, typically by supporting only a basic set of features. + *

+ * In particular, the returned parser does not support namespaces, prefixes, + * properties, or options. + * + * @hide + */ + public static @NonNull TypedXmlSerializer newFastSerializer() { + return XmlUtils.makeTyped(new FastXmlSerializer()); + } + + /** + * Creates a new {@link XmlSerializer} which is optimized for use inside the + * system, typically by supporting only a basic set of features. + *

+ * This returned instance may be configured to write using an efficient + * binary format instead of a human-readable text format, depending on + * device feature flags. + *

+ * To ensure that both formats are detected and transparently handled + * correctly, you must shift to using both {@link #resolveSerializer} and + * {@link #resolvePullParser}. + * + * @hide + */ + public static @NonNull TypedXmlSerializer resolveSerializer(@NonNull OutputStream out) + throws IOException { + // TODO: add support for binary format + final TypedXmlSerializer xml = newFastSerializer(); + xml.setOutput(out, StandardCharsets.UTF_8.name()); + return xml; + } + /** * Supported character encodings. */ diff --git a/core/java/com/android/internal/util/XmlPullParserWrapper.java b/core/java/com/android/internal/util/XmlPullParserWrapper.java new file mode 100644 index 0000000000000..efa17ef4d9cbe --- /dev/null +++ b/core/java/com/android/internal/util/XmlPullParserWrapper.java @@ -0,0 +1,189 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.internal.util; + +import android.annotation.NonNull; + +import org.xmlpull.v1.XmlPullParser; +import org.xmlpull.v1.XmlPullParserException; + +import java.io.IOException; +import java.io.InputStream; +import java.io.Reader; +import java.util.Objects; + +/** + * Wrapper which delegates all calls through to the given {@link XmlPullParser}. + */ +public class XmlPullParserWrapper implements XmlPullParser { + private final XmlPullParser mWrapped; + + public XmlPullParserWrapper(@NonNull XmlPullParser wrapped) { + mWrapped = Objects.requireNonNull(wrapped); + } + + public void setFeature(String name, boolean state) throws XmlPullParserException { + mWrapped.setFeature(name, state); + } + + public boolean getFeature(String name) { + return mWrapped.getFeature(name); + } + + public void setProperty(String name, Object value) throws XmlPullParserException { + mWrapped.setProperty(name, value); + } + + public Object getProperty(String name) { + return mWrapped.getProperty(name); + } + + public void setInput(Reader in) throws XmlPullParserException { + mWrapped.setInput(in); + } + + public void setInput(InputStream inputStream, String inputEncoding) + throws XmlPullParserException { + mWrapped.setInput(inputStream, inputEncoding); + } + + public String getInputEncoding() { + return mWrapped.getInputEncoding(); + } + + public void defineEntityReplacementText(String entityName, String replacementText) + throws XmlPullParserException { + mWrapped.defineEntityReplacementText(entityName, replacementText); + } + + public int getNamespaceCount(int depth) throws XmlPullParserException { + return mWrapped.getNamespaceCount(depth); + } + + public String getNamespacePrefix(int pos) throws XmlPullParserException { + return mWrapped.getNamespacePrefix(pos); + } + + public String getNamespaceUri(int pos) throws XmlPullParserException { + return mWrapped.getNamespaceUri(pos); + } + + public String getNamespace(String prefix) { + return mWrapped.getNamespace(prefix); + } + + public int getDepth() { + return mWrapped.getDepth(); + } + + public String getPositionDescription() { + return mWrapped.getPositionDescription(); + } + + public int getLineNumber() { + return mWrapped.getLineNumber(); + } + + public int getColumnNumber() { + return mWrapped.getColumnNumber(); + } + + public boolean isWhitespace() throws XmlPullParserException { + return mWrapped.isWhitespace(); + } + + public String getText() { + return mWrapped.getText(); + } + + public char[] getTextCharacters(int[] holderForStartAndLength) { + return mWrapped.getTextCharacters(holderForStartAndLength); + } + + public String getNamespace() { + return mWrapped.getNamespace(); + } + + public String getName() { + return mWrapped.getName(); + } + + public String getPrefix() { + return mWrapped.getPrefix(); + } + + public boolean isEmptyElementTag() throws XmlPullParserException { + return mWrapped.isEmptyElementTag(); + } + + public int getAttributeCount() { + return mWrapped.getAttributeCount(); + } + + public String getAttributeNamespace(int index) { + return mWrapped.getAttributeNamespace(index); + } + + public String getAttributeName(int index) { + return mWrapped.getAttributeName(index); + } + + public String getAttributePrefix(int index) { + return mWrapped.getAttributePrefix(index); + } + + public String getAttributeType(int index) { + return mWrapped.getAttributeType(index); + } + + public boolean isAttributeDefault(int index) { + return mWrapped.isAttributeDefault(index); + } + + public String getAttributeValue(int index) { + return mWrapped.getAttributeValue(index); + } + + public String getAttributeValue(String namespace, String name) { + return mWrapped.getAttributeValue(namespace, name); + } + + public int getEventType() throws XmlPullParserException { + return mWrapped.getEventType(); + } + + public int next() throws XmlPullParserException, IOException { + return mWrapped.next(); + } + + public int nextToken() throws XmlPullParserException, IOException { + return mWrapped.nextToken(); + } + + public void require(int type, String namespace, String name) + throws XmlPullParserException, IOException { + mWrapped.require(type, namespace, name); + } + + public String nextText() throws XmlPullParserException, IOException { + return mWrapped.nextText(); + } + + public int nextTag() throws XmlPullParserException, IOException { + return mWrapped.nextTag(); + } +} diff --git a/core/java/com/android/internal/util/XmlSerializerWrapper.java b/core/java/com/android/internal/util/XmlSerializerWrapper.java new file mode 100644 index 0000000000000..2131db0cfb6fe --- /dev/null +++ b/core/java/com/android/internal/util/XmlSerializerWrapper.java @@ -0,0 +1,140 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.internal.util; + +import android.annotation.NonNull; + +import org.xmlpull.v1.XmlSerializer; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.Writer; +import java.util.Objects; + +/** + * Wrapper which delegates all calls through to the given {@link XmlSerializer}. + */ +public class XmlSerializerWrapper { + private final XmlSerializer mWrapped; + + public XmlSerializerWrapper(@NonNull XmlSerializer wrapped) { + mWrapped = Objects.requireNonNull(wrapped); + } + + public void setFeature(String name, boolean state) { + mWrapped.setFeature(name, state); + } + + public boolean getFeature(String name) { + return mWrapped.getFeature(name); + } + + public void setProperty(String name, Object value) { + mWrapped.setProperty(name, value); + } + + public Object getProperty(String name) { + return mWrapped.getProperty(name); + } + + public void setOutput(OutputStream os, String encoding) throws IOException { + mWrapped.setOutput(os, encoding); + } + + public void setOutput(Writer writer) + throws IOException, IllegalArgumentException, IllegalStateException { + mWrapped.setOutput(writer); + } + + public void startDocument(String encoding, Boolean standalone) throws IOException { + mWrapped.startDocument(encoding, standalone); + } + + public void endDocument() throws IOException { + mWrapped.endDocument(); + } + + public void setPrefix(String prefix, String namespace) throws IOException { + mWrapped.setPrefix(prefix, namespace); + } + + public String getPrefix(String namespace, boolean generatePrefix) { + return mWrapped.getPrefix(namespace, generatePrefix); + } + + public int getDepth() { + return mWrapped.getDepth(); + } + + public String getNamespace() { + return mWrapped.getNamespace(); + } + + public String getName() { + return mWrapped.getName(); + } + + public XmlSerializer startTag(String namespace, String name) throws IOException { + return mWrapped.startTag(namespace, name); + } + + public XmlSerializer attribute(String namespace, String name, String value) + throws IOException { + return mWrapped.attribute(namespace, name, value); + } + + public XmlSerializer endTag(String namespace, String name) throws IOException { + return mWrapped.endTag(namespace, name); + } + + public XmlSerializer text(String text) throws IOException{ + return mWrapped.text(text); + } + + public XmlSerializer text(char[] buf, int start, int len) throws IOException { + return mWrapped.text(buf, start, len); + } + + public void cdsect(String text) + throws IOException, IllegalArgumentException, IllegalStateException { + mWrapped.cdsect(text); + } + + public void entityRef(String text) throws IOException { + mWrapped.entityRef(text); + } + + public void processingInstruction(String text) throws IOException { + mWrapped.processingInstruction(text); + } + + public void comment(String text) throws IOException { + mWrapped.comment(text); + } + + public void docdecl(String text) throws IOException { + mWrapped.docdecl(text); + } + + public void ignorableWhitespace(String text) throws IOException { + mWrapped.ignorableWhitespace(text); + } + + public void flush() throws IOException { + mWrapped.flush(); + } +} diff --git a/core/java/com/android/internal/util/XmlUtils.java b/core/java/com/android/internal/util/XmlUtils.java index bd6b950623eb2..cdd0e04b0fad7 100644 --- a/core/java/com/android/internal/util/XmlUtils.java +++ b/core/java/com/android/internal/util/XmlUtils.java @@ -16,6 +16,7 @@ package com.android.internal.util; +import android.annotation.NonNull; import android.compat.annotation.UnsupportedAppUsage; import android.graphics.Bitmap; import android.graphics.Bitmap.CompressFormat; @@ -24,6 +25,8 @@ import android.net.Uri; import android.text.TextUtils; import android.util.ArrayMap; import android.util.Base64; +import android.util.TypedXmlPullParser; +import android.util.TypedXmlSerializer; import android.util.Xml; import libcore.util.HexEncoding; @@ -48,9 +51,193 @@ import java.util.Set; /** {@hide} */ public class XmlUtils { - private static final String STRING_ARRAY_SEPARATOR = ":"; + private static class ForcedTypedXmlSerializer extends XmlSerializerWrapper + implements TypedXmlSerializer { + public ForcedTypedXmlSerializer(XmlSerializer wrapped) { + super(wrapped); + } + + @Override + public XmlSerializer attributeInterned(String namespace, String name, String value) + throws IOException { + return attribute(namespace, name, value); + } + + @Override + public XmlSerializer attributeBytesHex(String namespace, String name, byte[] value) + throws IOException { + return attribute(namespace, name, HexDump.toHexString(value)); + } + + @Override + public XmlSerializer attributeBytesBase64(String namespace, String name, byte[] value) + throws IOException { + return attribute(namespace, name, Base64.encodeToString(value, Base64.NO_WRAP)); + } + + @Override + public XmlSerializer attributeInt(String namespace, String name, int value) + throws IOException { + return attribute(namespace, name, Integer.toString(value)); + } + + @Override + public XmlSerializer attributeIntHex(String namespace, String name, int value) + throws IOException { + return attribute(namespace, name, Integer.toString(value, 16)); + } + + @Override + public XmlSerializer attributeLong(String namespace, String name, long value) + throws IOException { + return attribute(namespace, name, Long.toString(value)); + } + + @Override + public XmlSerializer attributeLongHex(String namespace, String name, long value) + throws IOException { + return attribute(namespace, name, Long.toString(value, 16)); + } + + @Override + public XmlSerializer attributeFloat(String namespace, String name, float value) + throws IOException { + return attribute(namespace, name, Float.toString(value)); + } + + @Override + public XmlSerializer attributeDouble(String namespace, String name, double value) + throws IOException { + return attribute(namespace, name, Double.toString(value)); + } + + @Override + public XmlSerializer attributeBoolean(String namespace, String name, boolean value) + throws IOException { + return attribute(namespace, name, Boolean.toString(value)); + } + } + + /** + * Return a specialization of the given {@link XmlSerializer} which has + * explicit methods to support consistent and efficient conversion of + * primitive data types. + */ + public static @NonNull TypedXmlSerializer makeTyped(@NonNull XmlSerializer xml) { + if (xml instanceof TypedXmlSerializer) { + return (TypedXmlSerializer) xml; + } else { + return new ForcedTypedXmlSerializer(xml); + } + } + + private static class ForcedTypedXmlPullParser extends XmlPullParserWrapper + implements TypedXmlPullParser { + public ForcedTypedXmlPullParser(XmlPullParser wrapped) { + super(wrapped); + } + + @Override + public byte[] getAttributeBytesHex(String namespace, String name) throws IOException { + try { + return HexDump.hexStringToByteArray(getAttributeValue(namespace, name)); + } catch (Exception e) { + throw new IOException("Invalid attribute " + name, e); + } + } + + @Override + public byte[] getAttributeBytesBase64(String namespace, String name) throws IOException { + try { + return Base64.decode(getAttributeValue(namespace, name), Base64.NO_WRAP); + } catch (Exception e) { + throw new IOException("Invalid attribute " + name, e); + } + } + + @Override + public int getAttributeInt(String namespace, String name) throws IOException { + try { + return Integer.parseInt(getAttributeValue(namespace, name)); + } catch (NumberFormatException e) { + throw new IOException("Invalid attribute " + name, e); + } + } + + @Override + public int getAttributeIntHex(String namespace, String name) throws IOException { + try { + return Integer.parseInt(getAttributeValue(namespace, name), 16); + } catch (NumberFormatException e) { + throw new IOException("Invalid attribute " + name, e); + } + } + + @Override + public long getAttributeLong(String namespace, String name) throws IOException { + try { + return Long.parseLong(getAttributeValue(namespace, name)); + } catch (NumberFormatException e) { + throw new IOException("Invalid attribute " + name, e); + } + } + + @Override + public long getAttributeLongHex(String namespace, String name) throws IOException { + try { + return Long.parseLong(getAttributeValue(namespace, name), 16); + } catch (NumberFormatException e) { + throw new IOException("Invalid attribute " + name, e); + } + } + + @Override + public float getAttributeFloat(String namespace, String name) throws IOException { + try { + return Float.parseFloat(getAttributeValue(namespace, name)); + } catch (NumberFormatException e) { + throw new IOException("Invalid attribute " + name, e); + } + } + + @Override + public double getAttributeDouble(String namespace, String name) throws IOException { + try { + return Double.parseDouble(getAttributeValue(namespace, name)); + } catch (NumberFormatException e) { + throw new IOException("Invalid attribute " + name, e); + } + } + + @Override + public boolean getAttributeBoolean(String namespace, String name) throws IOException { + final String value = getAttributeValue(namespace, name); + if ("true".equalsIgnoreCase(value)) { + return true; + } else if ("false".equalsIgnoreCase(value)) { + return false; + } else { + throw new IOException("Invalid attribute " + name, + new IllegalArgumentException("For input string: \"" + value + "\"")); + } + } + } + + /** + * Return a specialization of the given {@link XmlPullParser} which has + * explicit methods to support consistent and efficient conversion of + * primitive data types. + */ + public static @NonNull TypedXmlPullParser makeTyped(@NonNull XmlPullParser xml) { + if (xml instanceof TypedXmlPullParser) { + return (TypedXmlPullParser) xml; + } else { + return new ForcedTypedXmlPullParser(xml); + } + } + @UnsupportedAppUsage public static void skipCurrentTag(XmlPullParser parser) throws XmlPullParserException, IOException { diff --git a/core/tests/coretests/src/android/util/XmlTest.java b/core/tests/coretests/src/android/util/XmlTest.java new file mode 100644 index 0000000000000..602b672a2b82d --- /dev/null +++ b/core/tests/coretests/src/android/util/XmlTest.java @@ -0,0 +1,287 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.util; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.xmlpull.v1.XmlPullParser.END_DOCUMENT; +import static org.xmlpull.v1.XmlPullParser.END_TAG; +import static org.xmlpull.v1.XmlPullParser.START_DOCUMENT; +import static org.xmlpull.v1.XmlPullParser.START_TAG; +import static org.xmlpull.v1.XmlPullParser.TEXT; + +import android.os.PersistableBundle; + +import androidx.test.runner.AndroidJUnit4; + +import com.android.internal.util.XmlUtils; + +import org.junit.Test; +import org.junit.runner.RunWith; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; + +@RunWith(AndroidJUnit4.class) +public class XmlTest { + @Test + public void testLargeValues_Normal() throws Exception { + doLargeValues(XmlUtils.makeTyped(Xml.newSerializer()), + XmlUtils.makeTyped(Xml.newPullParser())); + } + + @Test + public void testLargeValues_Fast() throws Exception { + doLargeValues(Xml.newFastSerializer(), + Xml.newFastPullParser()); + } + + /** + * Verify that we can write and read large {@link String} and {@code byte[]} + * without issues. + */ + private static void doLargeValues(TypedXmlSerializer out, TypedXmlPullParser in) + throws Exception { + final char[] chars = new char[32_768]; + Arrays.fill(chars, '!'); + + final String string = new String(chars); + final byte[] bytes = string.getBytes(); + assertEquals(chars.length, bytes.length); + + final ByteArrayOutputStream os = new ByteArrayOutputStream(); + out.setOutput(os, StandardCharsets.UTF_8.name()); + out.startTag(null, "tag"); + out.attribute(null, "string", string); + out.attributeBytesBase64(null, "bytes", bytes); + out.endTag(null, "tag"); + out.flush(); + + final ByteArrayInputStream is = new ByteArrayInputStream(os.toByteArray()); + in.setInput(is, StandardCharsets.UTF_8.name()); + assertNext(in, START_TAG, "tag", 1); + assertEquals(2, in.getAttributeCount()); + assertEquals(string, in.getAttributeValue(null, "string")); + assertArrayEquals(bytes, in.getAttributeBytesBase64(null, "bytes")); + } + + @Test + public void testPersistableBundle_Normal() throws Exception { + doPersistableBundle(XmlUtils.makeTyped(Xml.newSerializer()), + XmlUtils.makeTyped(Xml.newPullParser())); + } + + @Test + public void testPersistableBundle_Fast() throws Exception { + doPersistableBundle(Xml.newFastSerializer(), + Xml.newFastPullParser()); + } + + /** + * Verify that a complex {@link PersistableBundle} can be serialized out and + * then parsed in with the original structure intact. + */ + private static void doPersistableBundle(TypedXmlSerializer out, TypedXmlPullParser in) + throws Exception { + final PersistableBundle expected = buildPersistableBundle(); + final byte[] raw = doPersistableBundleWrite(out, expected); + + // Yes, this string-based check is fragile, but kindofEquals() is broken + // when working with nested objects and arrays + final PersistableBundle actual = doPersistableBundleRead(in, raw); + assertEquals(expected.toString(), actual.toString()); + } + + private static PersistableBundle buildPersistableBundle() { + final PersistableBundle outer = new PersistableBundle(); + + outer.putBoolean("boolean", true); + outer.putInt("int", 42); + outer.putLong("long", 43L); + outer.putDouble("double", 44d); + outer.putString("string", "com.example & more"); + + outer.putBooleanArray("boolean[]", new boolean[] { true, false, true }); + outer.putIntArray("int[]", new int[] { 42, 43, 44 }); + outer.putLongArray("long[]", new long[] { 43L, 44L, 45L }); + outer.putDoubleArray("double[]", new double[] { 43d, 44d, 45d }); + outer.putStringArray("string[]", new String[] { "foo", "bar", "baz" }); + + final PersistableBundle nested = new PersistableBundle(); + nested.putString("nested_key", "nested_value"); + outer.putPersistableBundle("nested", nested); + + return outer; + } + + private static byte[] doPersistableBundleWrite(TypedXmlSerializer out, PersistableBundle bundle) + throws Exception { + // We purposefully omit START/END_DOCUMENT events here to verify correct + // behavior of what PersistableBundle does internally + final ByteArrayOutputStream os = new ByteArrayOutputStream(); + out.setOutput(os, StandardCharsets.UTF_8.name()); + out.startTag(null, "bundle"); + bundle.saveToXml(out); + out.endTag(null, "bundle"); + out.flush(); + return os.toByteArray(); + } + + private static PersistableBundle doPersistableBundleRead(TypedXmlPullParser in, byte[] raw) + throws Exception { + final ByteArrayInputStream is = new ByteArrayInputStream(raw); + in.setInput(is, StandardCharsets.UTF_8.name()); + in.next(); + return PersistableBundle.restoreFromXml(in); + } + + @Test + public void testVerify_Normal() throws Exception { + doVerify(XmlUtils.makeTyped(Xml.newSerializer()), + XmlUtils.makeTyped(Xml.newPullParser())); + } + + @Test + public void testVerify_Fast() throws Exception { + doVerify(Xml.newFastSerializer(), + Xml.newFastPullParser()); + } + + /** + * Verify that example test data is correctly serialized and parsed + * end-to-end using the given objects. + */ + private static void doVerify(TypedXmlSerializer out, TypedXmlPullParser in) throws Exception { + final ByteArrayOutputStream os = new ByteArrayOutputStream(); + out.setOutput(os, StandardCharsets.UTF_8.name()); + doVerifyWrite(out); + out.flush(); + + final ByteArrayInputStream is = new ByteArrayInputStream(os.toByteArray()); + in.setInput(is, StandardCharsets.UTF_8.name()); + doVerifyRead(in); + } + + private static final String TEST_STRING = "com.example"; + private static final byte[] TEST_BYTES = new byte[] { 0, 1, 2, 3, 4, 3, 2, 1, 0 }; + + private static void doVerifyWrite(TypedXmlSerializer out) throws Exception { + out.startDocument(StandardCharsets.UTF_8.name(), true); + out.startTag(null, "one"); + { + out.startTag(null, "two"); + { + out.attribute(null, "string", TEST_STRING); + out.attribute(null, "stringNumber", "49"); + out.attributeBytesHex(null, "bytesHex", TEST_BYTES); + out.attributeBytesBase64(null, "bytesBase64", TEST_BYTES); + out.attributeInt(null, "int", 43); + out.attributeIntHex(null, "intHex", 44); + out.attributeLong(null, "long", 45L); + out.attributeLongHex(null, "longHex", 46L); + out.attributeFloat(null, "float", 47f); + out.attributeDouble(null, "double", 48d); + out.attributeBoolean(null, "boolean", true); + } + out.endTag(null, "two"); + + out.startTag(null, "three"); + { + out.text("foo"); + out.startTag(null, "four"); + { + } + out.endTag(null, "four"); + out.text("bar"); + out.text("baz"); + } + out.endTag(null, "three"); + } + out.endTag(null, "one"); + out.endDocument(); + } + + private static void doVerifyRead(TypedXmlPullParser in) throws Exception { + assertEquals(START_DOCUMENT, in.getEventType()); + assertNext(in, START_TAG, "one", 1); + { + assertNext(in, START_TAG, "two", 2); + { + assertEquals(11, in.getAttributeCount()); + assertEquals(TEST_STRING, in.getAttributeValue(null, "string")); + assertArrayEquals(TEST_BYTES, in.getAttributeBytesHex(null, "bytesHex")); + assertArrayEquals(TEST_BYTES, in.getAttributeBytesBase64(null, "bytesBase64")); + assertEquals(43, in.getAttributeInt(null, "int")); + assertEquals(44, in.getAttributeIntHex(null, "intHex")); + assertEquals(45L, in.getAttributeLong(null, "long")); + assertEquals(46L, in.getAttributeLongHex(null, "longHex")); + assertEquals(47f, in.getAttributeFloat(null, "float"), 0.01); + assertEquals(48d, in.getAttributeDouble(null, "double"), 0.01); + assertEquals(true, in.getAttributeBoolean(null, "boolean")); + + // Also verify that typed values are available as strings + assertEquals("000102030403020100", in.getAttributeValue(null, "bytesHex")); + assertEquals("AAECAwQDAgEA", in.getAttributeValue(null, "bytesBase64")); + assertEquals("43", in.getAttributeValue(null, "int")); + assertEquals("2c", in.getAttributeValue(null, "intHex")); + assertEquals("45", in.getAttributeValue(null, "long")); + assertEquals("2e", in.getAttributeValue(null, "longHex")); + assertEquals("true", in.getAttributeValue(null, "boolean")); + + // And that raw strings can be parsed too + assertEquals("49", in.getAttributeValue(null, "stringNumber")); + assertEquals(49, in.getAttributeInt(null, "stringNumber")); + } + assertNext(in, END_TAG, "two", 2); + + assertNext(in, START_TAG, "three", 2); + { + assertNext(in, TEXT); + assertEquals("foo", in.getText().trim()); + assertNext(in, START_TAG, "four", 3); + { + assertEquals(0, in.getAttributeCount()); + } + assertNext(in, END_TAG, "four", 3); + assertNext(in, TEXT); + assertEquals("barbaz", in.getText().trim()); + } + assertNext(in, END_TAG, "three", 2); + } + assertNext(in, END_TAG, "one", 1); + assertNext(in, END_DOCUMENT); + } + + private static void assertNext(TypedXmlPullParser in, int token) throws Exception { + // We're willing to skip over empty text regions, which some + // serializers emit transparently + int event; + while ((event = in.next()) == TEXT && in.getText().trim().length() == 0) { + } + assertEquals("next", token, event); + assertEquals("getEventType", token, in.getEventType()); + } + + private static void assertNext(TypedXmlPullParser in, int token, String name, int depth) + throws Exception { + assertNext(in, token); + assertEquals("getName", name, in.getName()); + assertEquals("getDepth", depth, in.getDepth()); + } +} From 4ccea8796991d678ead4399130ec31edf63ff4fa Mon Sep 17 00:00:00 2001 From: Jeff Sharkey Date: Fri, 30 Oct 2020 19:35:25 -0600 Subject: [PATCH 4/4] Custom binary XML wire protocol. We've identified that XML writing and reading uses roughly 1.5% of all system_server CPU, and can generate many temporary objects. Building on the recent TypedXmlSerializer/PullParser interfaces, this change introduces new BinaryXmlSerializer/PullParser implementations that store data using a custom binary wire protocol. Benchmarking of a typical packages.xml has shown this new binary approach can write 4.3x faster and read 8.5x faster, while using 2.4x less disk space: timeWrite_Fast_mean: 27946635 timeWrite_Binary_mean: 6519341 timeRead_Fast_mean: 59562531 timeRead_Binary_mean: 7020185 A major factor in choosing to invest in this new wire protocol is that it enables the long-tail of over 100 unique XML schemas used across the OS internals to be transparently upgraded to gain these benefits with only minimal changes, reducing the risks associated with rewriting those schemas. Finally, since the wire protocol is essentially a serialized event stream, it's trivial to transparently convert this new protocol into human-readable XML and vice-versa. The tests in this change demonstrate this translation working correctly, and future changes will introduce new shell tools to aid development work. Bug: 171832118 Test: atest FrameworksCoreTests:android.util.XmlTest Test: atest FrameworksCoreTests:android.util.BinaryXmlTest Test: atest CorePerfTests:android.util.XmlPerfTest Change-Id: Ib9390701f09562dca952b3786622675b9c68a462 --- .../core/src/android/util/XmlPerfTest.java | 292 ++++++ core/java/android/util/Xml.java | 103 ++ .../internal/util/BinaryXmlPullParser.java | 899 ++++++++++++++++++ .../internal/util/BinaryXmlSerializer.java | 396 ++++++++ .../src/android/util/BinaryXmlTest.java | 99 ++ .../coretests/src/android/util/XmlTest.java | 30 +- 6 files changed, 1813 insertions(+), 6 deletions(-) create mode 100644 apct-tests/perftests/core/src/android/util/XmlPerfTest.java create mode 100644 core/java/com/android/internal/util/BinaryXmlPullParser.java create mode 100644 core/java/com/android/internal/util/BinaryXmlSerializer.java create mode 100644 core/tests/coretests/src/android/util/BinaryXmlTest.java diff --git a/apct-tests/perftests/core/src/android/util/XmlPerfTest.java b/apct-tests/perftests/core/src/android/util/XmlPerfTest.java new file mode 100644 index 0000000000000..e05bd2aad20e4 --- /dev/null +++ b/apct-tests/perftests/core/src/android/util/XmlPerfTest.java @@ -0,0 +1,292 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.util; + +import static org.junit.Assert.assertEquals; + +import android.os.Bundle; +import android.os.Debug; +import android.perftests.utils.BenchmarkState; +import android.perftests.utils.PerfStatusReporter; + +import androidx.test.InstrumentationRegistry; +import androidx.test.filters.LargeTest; +import androidx.test.runner.AndroidJUnit4; + +import com.android.internal.util.HexDump; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.xmlpull.v1.XmlPullParser; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.function.Supplier; + +@RunWith(AndroidJUnit4.class) +@LargeTest +public class XmlPerfTest { + /** + * Since allocation measurement adds overhead, it's disabled by default for + * performance runs. It can be manually enabled to compare GC behavior. + */ + private static final boolean MEASURE_ALLOC = false; + + @Rule + public PerfStatusReporter mPerfStatusReporter = new PerfStatusReporter(); + + @Test + public void timeWrite_Fast() throws Exception { + doWrite(() -> Xml.newFastSerializer()); + } + + @Test + public void timeWrite_Binary() throws Exception { + doWrite(() -> Xml.newBinarySerializer()); + } + + private void doWrite(Supplier outFactory) throws Exception { + if (MEASURE_ALLOC) { + Debug.startAllocCounting(); + } + + int iterations = 0; + final BenchmarkState state = mPerfStatusReporter.getBenchmarkState(); + while (state.keepRunning()) { + iterations++; + try (ByteArrayOutputStream os = new ByteArrayOutputStream()) { + final TypedXmlSerializer out = outFactory.get(); + out.setOutput(os, StandardCharsets.UTF_8.name()); + write(out); + } + } + + if (MEASURE_ALLOC) { + Debug.stopAllocCounting(); + final Bundle results = new Bundle(); + results.putLong("threadAllocCount_mean", Debug.getThreadAllocCount() / iterations); + results.putLong("threadAllocSize_mean", Debug.getThreadAllocSize() / iterations); + InstrumentationRegistry.getInstrumentation().sendStatus(0, results); + } + } + + @Test + public void timeRead_Fast() throws Exception { + doRead(() -> Xml.newFastSerializer(), () -> Xml.newFastPullParser()); + } + + @Test + public void timeRead_Binary() throws Exception { + doRead(() -> Xml.newBinarySerializer(), () -> Xml.newBinaryPullParser()); + } + + private void doRead(Supplier outFactory, + Supplier inFactory) throws Exception { + final byte[] raw; + try (ByteArrayOutputStream os = new ByteArrayOutputStream()) { + TypedXmlSerializer out = outFactory.get(); + out.setOutput(os, StandardCharsets.UTF_8.name()); + write(out); + raw = os.toByteArray(); + } + + if (MEASURE_ALLOC) { + Debug.startAllocCounting(); + } + + int iterations = 0; + final BenchmarkState state = mPerfStatusReporter.getBenchmarkState(); + while (state.keepRunning()) { + iterations++; + try (ByteArrayInputStream is = new ByteArrayInputStream(raw)) { + TypedXmlPullParser xml = inFactory.get(); + xml.setInput(is, StandardCharsets.UTF_8.name()); + read(xml); + } + } + + if (MEASURE_ALLOC) { + Debug.stopAllocCounting(); + final Bundle results = new Bundle(); + results.putLong("sizeBytes", raw.length); + results.putLong("threadAllocCount_mean", Debug.getThreadAllocCount() / iterations); + results.putLong("threadAllocSize_mean", Debug.getThreadAllocSize() / iterations); + InstrumentationRegistry.getInstrumentation().sendStatus(0, results); + } else { + final Bundle results = new Bundle(); + results.putLong("sizeBytes", raw.length); + InstrumentationRegistry.getInstrumentation().sendStatus(0, results); + } + } + + /** + * Not even joking, this is a typical public key blob stored in + * {@code packages.xml}. + */ + private static final byte[] KEY_BLOB = HexDump.hexStringToByteArray("" + + "308204a830820390a003020102020900a1573d0f45bea193300d06092a864886f70d010105050030819" + + "4310b3009060355040613025553311330110603550408130a43616c69666f726e696131163014060355" + + "0407130d4d6f756e7461696e20566965773110300e060355040a1307416e64726f69643110300e06035" + + "5040b1307416e64726f69643110300e06035504031307416e64726f69643122302006092a864886f70d" + + "0109011613616e64726f696440616e64726f69642e636f6d301e170d3131303931393138343232355a1" + + "70d3339303230343138343232355a308194310b3009060355040613025553311330110603550408130a" + + "43616c69666f726e6961311630140603550407130d4d6f756e7461696e20566965773110300e0603550" + + "40a1307416e64726f69643110300e060355040b1307416e64726f69643110300e06035504031307416e" + + "64726f69643122302006092a864886f70d0109011613616e64726f696440616e64726f69642e636f6d3" + + "0820120300d06092a864886f70d01010105000382010d00308201080282010100de1b51336afc909d8b" + + "cca5920fcdc8940578ec5c253898930e985481cfdea75ba6fc54b1f7bb492a03d98db471ab4200103a8" + + "314e60ee25fef6c8b83bc1b2b45b084874cffef148fa2001bb25c672b6beba50b7ac026b546da762ea2" + + "23829a22b80ef286131f059d2c9b4ca71d54e515a8a3fd6bf5f12a2493dfc2619b337b032a7cf8bbd34" + + "b833f2b93aeab3d325549a93272093943bb59dfc0197ae4861ff514e019b73f5cf10023ad1a032adb4b" + + "9bbaeb4debecb4941d6a02381f1165e1ac884c1fca9525c5854dce2ad8ec839b8ce78442c16367efc07" + + "778a337d3ca2cdf9792ac722b95d67c345f1c00976ec372f02bfcbef0262cc512a6845e71cfea0d0201" + + "03a381fc3081f9301d0603551d0e0416041478a0fc4517fb70ff52210df33c8d32290a44b2bb3081c90" + + "603551d230481c13081be801478a0fc4517fb70ff52210df33c8d32290a44b2bba1819aa48197308194" + + "310b3009060355040613025553311330110603550408130a43616c69666f726e6961311630140603550" + + "407130d4d6f756e7461696e20566965773110300e060355040a1307416e64726f69643110300e060355" + + "040b1307416e64726f69643110300e06035504031307416e64726f69643122302006092a864886f70d0" + + "109011613616e64726f696440616e64726f69642e636f6d820900a1573d0f45bea193300c0603551d13" + + "040530030101ff300d06092a864886f70d01010505000382010100977302dfbf668d7c61841c9c78d25" + + "63bcda1b199e95e6275a799939981416909722713531157f3cdcfea94eea7bb79ca3ca972bd8058a36a" + + "d1919291df42d7190678d4ea47a4b9552c9dfb260e6d0d9129b44615cd641c1080580e8a990dd768c6a" + + "b500c3b964e185874e4105109d94c5bd8c405deb3cf0f7960a563bfab58169a956372167a7e2674a04c" + + "4f80015d8f7869a7a4139aecbbdca2abc294144ee01e4109f0e47a518363cf6e9bf41f7560e94bdd4a5" + + "d085234796b05c7a1389adfd489feec2a107955129d7991daa49afb3d327dc0dc4fe959789372b093a8" + + "9c8dbfa41554f771c18015a6cb242a17e04d19d55d3b4664eae12caf2a11cd2b836e"); + + /** + * Typical list of permissions referenced in {@code packages.xml}. + */ + private static final String[] PERMS = new String[] { + "android.permission.ACCESS_CACHE_FILESYSTEM", + "android.permission.WRITE_SETTINGS", + "android.permission.MANAGE_EXTERNAL_STORAGE", + "android.permission.SEND_DOWNLOAD_COMPLETED_INTENTS", + "android.permission.FOREGROUND_SERVICE", + "android.permission.RECEIVE_BOOT_COMPLETED", + "android.permission.WRITE_MEDIA_STORAGE", + "android.permission.INTERNET", + "android.permission.UPDATE_DEVICE_STATS", + "android.permission.RECEIVE_DEVICE_CUSTOMIZATION_READY", + "android.permission.MANAGE_USB", + "android.permission.ACCESS_ALL_DOWNLOADS", + "android.permission.ACCESS_DOWNLOAD_MANAGER", + "android.permission.MANAGE_USERS", + "android.permission.ACCESS_NETWORK_STATE", + "android.permission.ACCESS_MTP", + "android.permission.INTERACT_ACROSS_USERS", + "android.permission.CONNECTIVITY_USE_RESTRICTED_NETWORKS", + "android.permission.CLEAR_APP_CACHE", + "android.permission.CONNECTIVITY_INTERNAL", + "android.permission.START_ACTIVITIES_FROM_BACKGROUND", + "android.permission.QUERY_ALL_PACKAGES", + "android.permission.WAKE_LOCK", + "android.permission.UPDATE_APP_OPS_STATS", + }; + + /** + * Write a typical {@code packages.xml} file containing 100 applications, + * each of which defines signing key and permission information. + */ + private static void write(TypedXmlSerializer out) throws IOException { + out.startDocument(null, true); + out.startTag(null, "packages"); + for (int i = 0; i < 100; i++) { + out.startTag(null, "package"); + out.attribute(null, "name", "com.android.providers.media"); + out.attribute(null, "codePath", "/system/priv-app/MediaProviderLegacy"); + out.attribute(null, "nativeLibraryPath", "/system/priv-app/MediaProviderLegacy/lib"); + out.attributeLong(null, "publicFlags", 944258629L); + out.attributeLong(null, "privateFlags", -1946152952L); + out.attributeLong(null, "ft", 1603899064000L); + out.attributeLong(null, "it", 1603899064000L); + out.attributeLong(null, "ut", 1603899064000L); + out.attributeInt(null, "version", 1024); + out.attributeInt(null, "sharedUserId", 10100); + out.attributeBoolean(null, "isOrphaned", true); + + out.startTag(null, "sigs"); + out.startTag(null, "cert"); + out.attributeInt(null, "index", 10); + out.attributeBytesHex(null, "key", KEY_BLOB); + out.endTag(null, "cert"); + out.endTag(null, "sigs"); + + out.startTag(null, "perms"); + for (String perm : PERMS) { + out.startTag(null, "item"); + out.attributeInterned(null, "name", perm); + out.attributeBoolean(null, "granted", true); + out.attributeInt(null, "flags", 0); + out.endTag(null, "item"); + } + out.endTag(null, "perms"); + + out.endTag(null, "package"); + } + out.endTag(null, "packages"); + out.endDocument(); + } + + /** + * Read a typical {@code packages.xml} file containing 100 applications, and + * verify that data passes smell test. + */ + private static void read(TypedXmlPullParser xml) throws Exception { + int type; + int packages = 0; + int certs = 0; + int perms = 0; + while ((type = xml.next()) != XmlPullParser.END_DOCUMENT) { + final String tag = xml.getName(); + if (type == XmlPullParser.START_TAG) { + if ("package".equals(tag)) { + xml.getAttributeValue(null, "name"); + xml.getAttributeValue(null, "codePath"); + xml.getAttributeValue(null, "nativeLibraryPath"); + xml.getAttributeLong(null, "publicFlags"); + assertEquals(-1946152952L, xml.getAttributeLong(null, "privateFlags")); + xml.getAttributeLong(null, "ft"); + xml.getAttributeLong(null, "it"); + xml.getAttributeLong(null, "ut"); + xml.getAttributeInt(null, "version"); + xml.getAttributeInt(null, "sharedUserId"); + xml.getAttributeBoolean(null, "isOrphaned"); + packages++; + } else if ("cert".equals(tag)) { + xml.getAttributeInt(null, "index"); + xml.getAttributeBytesHex(null, "key"); + certs++; + } else if ("item".equals(tag)) { + xml.getAttributeValue(null, "name"); + xml.getAttributeBoolean(null, "granted"); + xml.getAttributeInt(null, "flags"); + perms++; + } + } else if (type == XmlPullParser.TEXT) { + xml.getText(); + } + } + + assertEquals(100, packages); + assertEquals(packages * 1, certs); + assertEquals(packages * PERMS.length, perms); + } +} diff --git a/core/java/android/util/Xml.java b/core/java/android/util/Xml.java index 9849c10c1604e..cc6ed2e4539eb 100644 --- a/core/java/android/util/Xml.java +++ b/core/java/android/util/Xml.java @@ -17,7 +17,10 @@ package android.util; import android.annotation.NonNull; +import android.annotation.Nullable; +import com.android.internal.util.BinaryXmlPullParser; +import com.android.internal.util.BinaryXmlSerializer; import com.android.internal.util.FastXmlSerializer; import com.android.internal.util.XmlUtils; @@ -120,6 +123,18 @@ public class Xml { return XmlUtils.makeTyped(newPullParser()); } + /** + * Creates a new {@link XmlPullParser} that reads XML documents using a + * custom binary wire protocol which benchmarking has shown to be 8.5x + * faster than {@code Xml.newFastPullParser()} for a typical + * {@code packages.xml}. + * + * @hide + */ + public static @NonNull TypedXmlPullParser newBinaryPullParser() { + return new BinaryXmlPullParser(); + } + /** * Creates a new {@link XmlPullParser} which is optimized for use inside the * system, typically by supporting only a basic set of features. @@ -166,6 +181,18 @@ public class Xml { return XmlUtils.makeTyped(new FastXmlSerializer()); } + /** + * Creates a new {@link XmlSerializer} that writes XML documents using a + * custom binary wire protocol which benchmarking has shown to be 4.4x + * faster and use 2.8x less disk space than {@code Xml.newFastSerializer()} + * for a typical {@code packages.xml}. + * + * @hide + */ + public static @NonNull TypedXmlSerializer newBinarySerializer() { + return new BinaryXmlSerializer(); + } + /** * Creates a new {@link XmlSerializer} which is optimized for use inside the * system, typically by supporting only a basic set of features. @@ -188,6 +215,82 @@ public class Xml { return xml; } + /** + * Copy the first XML document into the second document. + *

+ * Implemented by reading all events from the given {@link XmlPullParser} + * and writing them directly to the given {@link XmlSerializer}. This can be + * useful for transparently converting between underlying wire protocols. + * + * @hide + */ + public static void copy(@NonNull XmlPullParser in, @NonNull XmlSerializer out) + throws XmlPullParserException, IOException { + // Some parsers may have already consumed the event that starts the + // document, so we manually emit that event here for consistency + if (in.getEventType() == XmlPullParser.START_DOCUMENT) { + out.startDocument(in.getInputEncoding(), true); + } + + while (true) { + final int token = in.nextToken(); + switch (token) { + case XmlPullParser.START_DOCUMENT: + out.startDocument(in.getInputEncoding(), true); + break; + case XmlPullParser.END_DOCUMENT: + out.endDocument(); + return; + case XmlPullParser.START_TAG: + out.startTag(normalizeNamespace(in.getNamespace()), in.getName()); + for (int i = 0; i < in.getAttributeCount(); i++) { + out.attribute(normalizeNamespace(in.getAttributeNamespace(i)), + in.getAttributeName(i), in.getAttributeValue(i)); + } + break; + case XmlPullParser.END_TAG: + out.endTag(normalizeNamespace(in.getNamespace()), in.getName()); + break; + case XmlPullParser.TEXT: + out.text(in.getText()); + break; + case XmlPullParser.CDSECT: + out.cdsect(in.getText()); + break; + case XmlPullParser.ENTITY_REF: + out.entityRef(in.getName()); + break; + case XmlPullParser.IGNORABLE_WHITESPACE: + out.ignorableWhitespace(in.getText()); + break; + case XmlPullParser.PROCESSING_INSTRUCTION: + out.processingInstruction(in.getText()); + break; + case XmlPullParser.COMMENT: + out.comment(in.getText()); + break; + case XmlPullParser.DOCDECL: + out.docdecl(in.getText()); + break; + default: + throw new IllegalStateException("Unknown token " + token); + } + } + } + + /** + * Some parsers may return an empty string {@code ""} when a namespace in + * unsupported, which can confuse serializers. This method normalizes empty + * strings to be {@code null}. + */ + private static @Nullable String normalizeNamespace(@Nullable String namespace) { + if (namespace == null || namespace.isEmpty()) { + return null; + } else { + return namespace; + } + } + /** * Supported character encodings. */ diff --git a/core/java/com/android/internal/util/BinaryXmlPullParser.java b/core/java/com/android/internal/util/BinaryXmlPullParser.java new file mode 100644 index 0000000000000..da16eca5239c3 --- /dev/null +++ b/core/java/com/android/internal/util/BinaryXmlPullParser.java @@ -0,0 +1,899 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.internal.util; + +import static com.android.internal.util.BinaryXmlSerializer.ATTRIBUTE; +import static com.android.internal.util.BinaryXmlSerializer.PROTOCOL_MAGIC_VERSION_0; +import static com.android.internal.util.BinaryXmlSerializer.TYPE_BOOLEAN_FALSE; +import static com.android.internal.util.BinaryXmlSerializer.TYPE_BOOLEAN_TRUE; +import static com.android.internal.util.BinaryXmlSerializer.TYPE_BYTES_BASE64; +import static com.android.internal.util.BinaryXmlSerializer.TYPE_BYTES_HEX; +import static com.android.internal.util.BinaryXmlSerializer.TYPE_DOUBLE; +import static com.android.internal.util.BinaryXmlSerializer.TYPE_FLOAT; +import static com.android.internal.util.BinaryXmlSerializer.TYPE_INT; +import static com.android.internal.util.BinaryXmlSerializer.TYPE_INT_HEX; +import static com.android.internal.util.BinaryXmlSerializer.TYPE_LONG; +import static com.android.internal.util.BinaryXmlSerializer.TYPE_LONG_HEX; +import static com.android.internal.util.BinaryXmlSerializer.TYPE_NULL; +import static com.android.internal.util.BinaryXmlSerializer.TYPE_STRING; +import static com.android.internal.util.BinaryXmlSerializer.TYPE_STRING_INTERNED; + +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.text.TextUtils; +import android.util.Base64; +import android.util.TypedXmlPullParser; +import android.util.TypedXmlSerializer; + +import org.xmlpull.v1.XmlPullParser; +import org.xmlpull.v1.XmlPullParserException; + +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.io.Reader; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Objects; + +/** + * Parser that reads XML documents using a custom binary wire protocol which + * benchmarking has shown to be 8.5x faster than {@link Xml.newFastPullParser()} + * for a typical {@code packages.xml}. + *

+ * The high-level design of the wire protocol is to directly serialize the event + * stream, while efficiently and compactly writing strongly-typed primitives + * delivered through the {@link TypedXmlSerializer} interface. + *

+ * Each serialized event is a single byte where the lower half is a normal + * {@link XmlPullParser} token and the upper half is an optional data type + * signal, such as {@link #TYPE_INT}. + *

+ * This parser has some specific limitations: + *

+ */ +public final class BinaryXmlPullParser implements TypedXmlPullParser { + /** + * Default buffer size, which matches {@code FastXmlSerializer}. This should + * be kept in sync with {@link BinaryXmlPullParser}. + */ + private static final int BUFFER_SIZE = 32_768; + + private FastDataInput mIn; + + private int mCurrentToken = START_DOCUMENT; + private int mCurrentDepth = 0; + private String mCurrentName; + private String mCurrentText; + + /** + * Pool of attributes parsed for the currently tag. All interactions should + * be done via {@link #obtainAttribute()}, {@link #findAttribute(String)}, + * and {@link #resetAttributes()}. + */ + private int mAttributeCount = 0; + private Attribute[] mAttributes; + + @Override + public void setInput(InputStream is, String inputEncoding) throws XmlPullParserException { + if (inputEncoding != null && !StandardCharsets.UTF_8.name().equals(inputEncoding)) { + throw new UnsupportedOperationException(); + } + + mIn = new FastDataInput(is, BUFFER_SIZE); + + mCurrentToken = START_DOCUMENT; + mCurrentDepth = 0; + mCurrentName = null; + mCurrentText = null; + + mAttributeCount = 0; + mAttributes = new Attribute[8]; + for (int i = 0; i < mAttributes.length; i++) { + mAttributes[i] = new Attribute(); + } + + try { + final byte[] magic = new byte[4]; + mIn.readFully(magic); + if (!Arrays.equals(magic, PROTOCOL_MAGIC_VERSION_0)) { + throw new IOException("Unexpected magic " + bytesToHexString(magic)); + } + + // We're willing to immediately consume a START_DOCUMENT if present, + // but we're okay if it's missing + if (peekNextExternalToken() == START_DOCUMENT) { + consumeToken(); + } + } catch (IOException e) { + throw new XmlPullParserException(e.toString()); + } + } + + @Override + public void setInput(Reader in) throws XmlPullParserException { + throw new UnsupportedOperationException(); + } + + @Override + public int next() throws XmlPullParserException, IOException { + while (true) { + final int token = nextToken(); + switch (token) { + case START_TAG: + case END_TAG: + case END_DOCUMENT: + return token; + case TEXT: + consumeAdditionalText(); + // Per interface docs, empty text regions are skipped + if (mCurrentText == null || mCurrentText.length() == 0) { + continue; + } else { + return TEXT; + } + } + } + } + + @Override + public int nextToken() throws XmlPullParserException, IOException { + if (mCurrentToken == XmlPullParser.END_TAG) { + mCurrentDepth--; + } + + int token; + try { + token = peekNextExternalToken(); + consumeToken(); + } catch (EOFException e) { + token = END_DOCUMENT; + } + switch (token) { + case XmlPullParser.START_TAG: + // We need to peek forward to find the next external token so + // that we parse all pending INTERNAL_ATTRIBUTE tokens + peekNextExternalToken(); + mCurrentDepth++; + break; + } + mCurrentToken = token; + return token; + } + + /** + * Peek at the next "external" token without consuming it. + *

+ * External tokens, such as {@link #START_TAG}, are expected by typical + * {@link XmlPullParser} clients. In contrast, internal tokens, such as + * {@link #ATTRIBUTE}, are not expected by typical clients. + *

+ * This method consumes any internal events until it reaches the next + * external event. + */ + private int peekNextExternalToken() throws IOException, XmlPullParserException { + while (true) { + final int token = peekNextToken(); + switch (token) { + case ATTRIBUTE: + consumeToken(); + continue; + default: + return token; + } + } + } + + /** + * Peek at the next token in the underlying stream without consuming it. + */ + private int peekNextToken() throws IOException { + return mIn.peekByte() & 0x0f; + } + + /** + * Parse and consume the next token in the underlying stream. + */ + private void consumeToken() throws IOException, XmlPullParserException { + final int event = mIn.readByte(); + final int token = event & 0x0f; + final int type = event & 0xf0; + switch (token) { + case ATTRIBUTE: { + final Attribute attr = obtainAttribute(); + attr.name = mIn.readInternedUTF(); + attr.type = type; + switch (type) { + case TYPE_NULL: + case TYPE_BOOLEAN_TRUE: + case TYPE_BOOLEAN_FALSE: + // Nothing extra to fill in + break; + case TYPE_STRING: + attr.valueString = mIn.readUTF(); + break; + case TYPE_STRING_INTERNED: + attr.valueString = mIn.readInternedUTF(); + break; + case TYPE_BYTES_HEX: + case TYPE_BYTES_BASE64: + final int len = mIn.readUnsignedShort(); + final byte[] res = new byte[len]; + mIn.readFully(res); + attr.valueBytes = res; + break; + case TYPE_INT: + case TYPE_INT_HEX: + attr.valueInt = mIn.readInt(); + break; + case TYPE_LONG: + case TYPE_LONG_HEX: + attr.valueLong = mIn.readLong(); + break; + case TYPE_FLOAT: + attr.valueFloat = mIn.readFloat(); + break; + case TYPE_DOUBLE: + attr.valueDouble = mIn.readDouble(); + break; + default: + throw new IOException("Unexpected data type " + type); + } + break; + } + case XmlPullParser.START_DOCUMENT: { + break; + } + case XmlPullParser.END_DOCUMENT: { + break; + } + case XmlPullParser.START_TAG: { + mCurrentName = mIn.readInternedUTF(); + resetAttributes(); + break; + } + case XmlPullParser.END_TAG: { + mCurrentName = mIn.readInternedUTF(); + resetAttributes(); + break; + } + case XmlPullParser.TEXT: + case XmlPullParser.CDSECT: + case XmlPullParser.PROCESSING_INSTRUCTION: + case XmlPullParser.COMMENT: + case XmlPullParser.DOCDECL: + case XmlPullParser.IGNORABLE_WHITESPACE: { + mCurrentText = mIn.readUTF(); + break; + } + case XmlPullParser.ENTITY_REF: { + mCurrentName = mIn.readUTF(); + mCurrentText = resolveEntity(mCurrentName); + break; + } + default: { + throw new IOException("Unknown token " + token + " with type " + type); + } + } + } + + /** + * When the current tag is {@link #TEXT}, consume all subsequent "text" + * events, as described by {@link #next}. When finished, the current event + * will still be {@link #TEXT}. + */ + private void consumeAdditionalText() throws IOException, XmlPullParserException { + String combinedText = mCurrentText; + while (true) { + final int token = peekNextExternalToken(); + switch (token) { + case COMMENT: + case PROCESSING_INSTRUCTION: + // Quietly consumed + consumeToken(); + break; + case TEXT: + case CDSECT: + case ENTITY_REF: + // Additional text regions collected + consumeToken(); + combinedText += mCurrentText; + break; + default: + // Next token is something non-text, so wrap things up + mCurrentToken = TEXT; + mCurrentName = null; + mCurrentText = combinedText; + return; + } + } + } + + static @NonNull String resolveEntity(@NonNull String entity) + throws XmlPullParserException { + switch (entity) { + case "lt": return "<"; + case "gt": return ">"; + case "amp": return "&"; + case "apos": return "'"; + case "quot": return "\""; + } + if (entity.length() > 1 && entity.charAt(0) == '#') { + final char c = (char) Integer.parseInt(entity.substring(1)); + return new String(new char[] { c }); + } + throw new XmlPullParserException("Unknown entity " + entity); + } + + @Override + public void require(int type, String namespace, String name) + throws XmlPullParserException, IOException { + if (namespace != null && !namespace.isEmpty()) throw illegalNamespace(); + if (mCurrentToken != type || !Objects.equals(mCurrentName, name)) { + throw new XmlPullParserException(getPositionDescription()); + } + } + + @Override + public String nextText() throws XmlPullParserException, IOException { + if (getEventType() != START_TAG) { + throw new XmlPullParserException(getPositionDescription()); + } + int eventType = next(); + if (eventType == TEXT) { + String result = getText(); + eventType = next(); + if (eventType != END_TAG) { + throw new XmlPullParserException(getPositionDescription()); + } + return result; + } else if (eventType == END_TAG) { + return ""; + } else { + throw new XmlPullParserException(getPositionDescription()); + } + } + + @Override + public int nextTag() throws XmlPullParserException, IOException { + int eventType = next(); + if (eventType == TEXT && isWhitespace()) { + eventType = next(); + } + if (eventType != START_TAG && eventType != END_TAG) { + throw new XmlPullParserException(getPositionDescription()); + } + return eventType; + } + + /** + * Allocate and return a new {@link Attribute} associated with the tag being + * currently processed. This will automatically grow the internal pool as + * needed. + */ + private @NonNull Attribute obtainAttribute() { + if (mAttributeCount == mAttributes.length) { + final int before = mAttributes.length; + final int after = before + (before >> 1); + mAttributes = Arrays.copyOf(mAttributes, after); + for (int i = before; i < after; i++) { + mAttributes[i] = new Attribute(); + } + } + return mAttributes[mAttributeCount++]; + } + + /** + * Clear any {@link Attribute} instances that have been allocated by + * {@link #obtainAttribute()}, returning them into the pool for recycling. + */ + private void resetAttributes() { + for (int i = 0; i < mAttributeCount; i++) { + mAttributes[i].reset(); + } + mAttributeCount = 0; + } + + /** + * Search through the pool of currently allocated {@link Attribute} + * instances for one that matches the given name. + */ + private @NonNull Attribute findAttribute(@NonNull String name) throws IOException { + for (int i = 0; i < mAttributeCount; i++) { + if (Objects.equals(mAttributes[i].name, name)) { + return mAttributes[i]; + } + } + throw new IOException("Missing attribute " + name); + } + + @Override + public String getAttributeValue(String namespace, String name) { + if (namespace != null && !namespace.isEmpty()) throw illegalNamespace(); + try { + return findAttribute(name).getValueString(); + } catch (IOException e) { + // Missing attributes default to null + return null; + } + } + + @Override + public String getAttributeValue(int index) { + return mAttributes[index].getValueString(); + } + + @Override + public byte[] getAttributeBytesHex(String namespace, String name) throws IOException { + if (namespace != null && !namespace.isEmpty()) throw illegalNamespace(); + return findAttribute(name).getValueBytesHex(); + } + + @Override + public byte[] getAttributeBytesBase64(String namespace, String name) throws IOException { + if (namespace != null && !namespace.isEmpty()) throw illegalNamespace(); + return findAttribute(name).getValueBytesBase64(); + } + + @Override + public int getAttributeInt(String namespace, String name) throws IOException { + if (namespace != null && !namespace.isEmpty()) throw illegalNamespace(); + return findAttribute(name).getValueInt(); + } + + @Override + public int getAttributeIntHex(String namespace, String name) throws IOException { + if (namespace != null && !namespace.isEmpty()) throw illegalNamespace(); + return findAttribute(name).getValueIntHex(); + } + + @Override + public long getAttributeLong(String namespace, String name) throws IOException { + if (namespace != null && !namespace.isEmpty()) throw illegalNamespace(); + return findAttribute(name).getValueLong(); + } + + @Override + public long getAttributeLongHex(String namespace, String name) throws IOException { + if (namespace != null && !namespace.isEmpty()) throw illegalNamespace(); + return findAttribute(name).getValueLongHex(); + } + + @Override + public float getAttributeFloat(String namespace, String name) throws IOException { + if (namespace != null && !namespace.isEmpty()) throw illegalNamespace(); + return findAttribute(name).getValueFloat(); + } + + @Override + public double getAttributeDouble(String namespace, String name) throws IOException { + if (namespace != null && !namespace.isEmpty()) throw illegalNamespace(); + return findAttribute(name).getValueDouble(); + } + + @Override + public boolean getAttributeBoolean(String namespace, String name) throws IOException { + if (namespace != null && !namespace.isEmpty()) throw illegalNamespace(); + return findAttribute(name).getValueBoolean(); + } + + @Override + public String getText() { + return mCurrentText; + } + + @Override + public char[] getTextCharacters(int[] holderForStartAndLength) { + final char[] chars = mCurrentText.toCharArray(); + holderForStartAndLength[0] = 0; + holderForStartAndLength[1] = chars.length; + return chars; + } + + @Override + public String getInputEncoding() { + return StandardCharsets.UTF_8.name(); + } + + @Override + public int getDepth() { + return mCurrentDepth; + } + + @Override + public String getPositionDescription() { + // Not very helpful, but it's the best information we have + return "Token " + mCurrentToken + " at depth " + mCurrentDepth; + } + + @Override + public int getLineNumber() { + return -1; + } + + @Override + public int getColumnNumber() { + return -1; + } + + @Override + public boolean isWhitespace() throws XmlPullParserException { + switch (mCurrentToken) { + case IGNORABLE_WHITESPACE: + return true; + case TEXT: + case CDSECT: + return !TextUtils.isGraphic(mCurrentText); + default: + throw new XmlPullParserException("Not applicable for token " + mCurrentToken); + } + } + + @Override + public String getNamespace() { + switch (mCurrentToken) { + case START_TAG: + case END_TAG: + // Namespaces are unsupported + return NO_NAMESPACE; + default: + return null; + } + } + + @Override + public String getName() { + return mCurrentName; + } + + @Override + public String getPrefix() { + // Prefixes are not supported + return null; + } + + @Override + public boolean isEmptyElementTag() throws XmlPullParserException { + switch (mCurrentToken) { + case START_TAG: + try { + return (peekNextExternalToken() == END_TAG); + } catch (IOException e) { + throw new XmlPullParserException(e.toString()); + } + default: + throw new XmlPullParserException("Not at START_TAG"); + } + } + + @Override + public int getAttributeCount() { + return mAttributeCount; + } + + @Override + public String getAttributeNamespace(int index) { + // Namespaces are unsupported + return NO_NAMESPACE; + } + + @Override + public String getAttributeName(int index) { + return mAttributes[index].name; + } + + @Override + public String getAttributePrefix(int index) { + // Prefixes are not supported + return null; + } + + @Override + public String getAttributeType(int index) { + // Validation is not supported + return "CDATA"; + } + + @Override + public boolean isAttributeDefault(int index) { + // Validation is not supported + return false; + } + + @Override + public int getEventType() throws XmlPullParserException { + return mCurrentToken; + } + + @Override + public int getNamespaceCount(int depth) throws XmlPullParserException { + // Namespaces are unsupported + return 0; + } + + @Override + public String getNamespacePrefix(int pos) throws XmlPullParserException { + // Namespaces are unsupported + throw new UnsupportedOperationException(); + } + + @Override + public String getNamespaceUri(int pos) throws XmlPullParserException { + // Namespaces are unsupported + throw new UnsupportedOperationException(); + } + + @Override + public String getNamespace(String prefix) { + // Namespaces are unsupported + throw new UnsupportedOperationException(); + } + + @Override + public void defineEntityReplacementText(String entityName, String replacementText) + throws XmlPullParserException { + // Custom entities are not supported + throw new UnsupportedOperationException(); + } + + @Override + public void setFeature(String name, boolean state) throws XmlPullParserException { + // Features are not supported + throw new UnsupportedOperationException(); + } + + @Override + public boolean getFeature(String name) { + // Features are not supported + throw new UnsupportedOperationException(); + } + + @Override + public void setProperty(String name, Object value) throws XmlPullParserException { + // Properties are not supported + throw new UnsupportedOperationException(); + } + + @Override + public Object getProperty(String name) { + // Properties are not supported + throw new UnsupportedOperationException(); + } + + private static IllegalArgumentException illegalNamespace() { + throw new IllegalArgumentException("Namespaces are not supported"); + } + + /** + * Holder representing a single attribute. This design enables object + * recycling without resorting to autoboxing. + *

+ * To support conversion between human-readable XML and binary XML, the + * various accessor methods will transparently convert from/to + * human-readable values when needed. + */ + private static class Attribute { + public String name; + public int type; + + public String valueString; + public byte[] valueBytes; + public int valueInt; + public long valueLong; + public float valueFloat; + public double valueDouble; + + public void reset() { + name = null; + valueString = null; + valueBytes = null; + } + + public @Nullable String getValueString() { + switch (type) { + case TYPE_NULL: + return null; + case TYPE_STRING: + case TYPE_STRING_INTERNED: + return valueString; + case TYPE_BYTES_HEX: + return bytesToHexString(valueBytes); + case TYPE_BYTES_BASE64: + return Base64.encodeToString(valueBytes, Base64.NO_WRAP); + case TYPE_INT: + return Integer.toString(valueInt); + case TYPE_INT_HEX: + return Integer.toString(valueInt, 16); + case TYPE_LONG: + return Long.toString(valueLong); + case TYPE_LONG_HEX: + return Long.toString(valueLong, 16); + case TYPE_FLOAT: + return Float.toString(valueFloat); + case TYPE_DOUBLE: + return Double.toString(valueDouble); + case TYPE_BOOLEAN_TRUE: + return "true"; + case TYPE_BOOLEAN_FALSE: + return "false"; + default: + // Unknown data type; null is the best we can offer + return null; + } + } + + public @Nullable byte[] getValueBytesHex() throws IOException { + switch (type) { + case TYPE_NULL: + return null; + case TYPE_BYTES_HEX: + case TYPE_BYTES_BASE64: + return valueBytes; + case TYPE_STRING: + return hexStringToBytes(valueString); + default: + throw new IOException("Invalid conversion from " + type); + } + } + + public @Nullable byte[] getValueBytesBase64() throws IOException { + switch (type) { + case TYPE_NULL: + return null; + case TYPE_BYTES_HEX: + case TYPE_BYTES_BASE64: + return valueBytes; + case TYPE_STRING: + return Base64.decode(valueString, Base64.NO_WRAP); + default: + throw new IOException("Invalid conversion from " + type); + } + } + + public int getValueInt() throws IOException { + switch (type) { + case TYPE_INT: + case TYPE_INT_HEX: + return valueInt; + case TYPE_STRING: + return Integer.parseInt(valueString); + default: + throw new IOException("Invalid conversion from " + type); + } + } + + public int getValueIntHex() throws IOException { + switch (type) { + case TYPE_INT: + case TYPE_INT_HEX: + return valueInt; + case TYPE_STRING: + return Integer.parseInt(valueString, 16); + default: + throw new IOException("Invalid conversion from " + type); + } + } + + public long getValueLong() throws IOException { + switch (type) { + case TYPE_LONG: + case TYPE_LONG_HEX: + return valueLong; + case TYPE_STRING: + return Long.parseLong(valueString); + default: + throw new IOException("Invalid conversion from " + type); + } + } + + public long getValueLongHex() throws IOException { + switch (type) { + case TYPE_LONG: + case TYPE_LONG_HEX: + return valueLong; + case TYPE_STRING: + return Long.parseLong(valueString, 16); + default: + throw new IOException("Invalid conversion from " + type); + } + } + + public float getValueFloat() throws IOException { + switch (type) { + case TYPE_FLOAT: + return valueFloat; + case TYPE_STRING: + return Float.parseFloat(valueString); + default: + throw new IOException("Invalid conversion from " + type); + } + } + + public double getValueDouble() throws IOException { + switch (type) { + case TYPE_DOUBLE: + return valueDouble; + case TYPE_STRING: + return Double.parseDouble(valueString); + default: + throw new IOException("Invalid conversion from " + type); + } + } + + public boolean getValueBoolean() throws IOException { + switch (type) { + case TYPE_BOOLEAN_TRUE: + return true; + case TYPE_BOOLEAN_FALSE: + return false; + case TYPE_STRING: + if ("true".equalsIgnoreCase(valueString)) { + return true; + } else if ("false".equalsIgnoreCase(valueString)) { + return false; + } else { + throw new IOException("Invalid boolean: " + valueString); + } + default: + throw new IOException("Invalid conversion from " + type); + } + } + } + + // NOTE: To support unbundled clients, we include an inlined copy + // of hex conversion logic from HexDump below + private final static char[] HEX_DIGITS = + { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' }; + + private static int toByte(char c) throws IOException { + if (c >= '0' && c <= '9') return (c - '0'); + if (c >= 'A' && c <= 'F') return (c - 'A' + 10); + if (c >= 'a' && c <= 'f') return (c - 'a' + 10); + throw new IOException("Invalid hex char '" + c + "'"); + } + + static String bytesToHexString(byte[] value) { + final int length = value.length; + final char[] buf = new char[length * 2]; + int bufIndex = 0; + for (int i = 0; i < length; i++) { + byte b = value[i]; + buf[bufIndex++] = HEX_DIGITS[(b >>> 4) & 0x0F]; + buf[bufIndex++] = HEX_DIGITS[b & 0x0F]; + } + return new String(buf); + } + + static byte[] hexStringToBytes(String value) throws IOException { + final int length = value.length(); + if (length % 2 != 0) { + throw new IOException("Invalid hex length " + length); + } + byte[] buffer = new byte[length / 2]; + for (int i = 0; i < length; i += 2) { + buffer[i / 2] = (byte) ((toByte(value.charAt(i)) << 4) + | toByte(value.charAt(i + 1))); + } + return buffer; + } +} diff --git a/core/java/com/android/internal/util/BinaryXmlSerializer.java b/core/java/com/android/internal/util/BinaryXmlSerializer.java new file mode 100644 index 0000000000000..d3fcf71ba3996 --- /dev/null +++ b/core/java/com/android/internal/util/BinaryXmlSerializer.java @@ -0,0 +1,396 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.internal.util; + +import static org.xmlpull.v1.XmlPullParser.CDSECT; +import static org.xmlpull.v1.XmlPullParser.COMMENT; +import static org.xmlpull.v1.XmlPullParser.DOCDECL; +import static org.xmlpull.v1.XmlPullParser.END_DOCUMENT; +import static org.xmlpull.v1.XmlPullParser.END_TAG; +import static org.xmlpull.v1.XmlPullParser.ENTITY_REF; +import static org.xmlpull.v1.XmlPullParser.IGNORABLE_WHITESPACE; +import static org.xmlpull.v1.XmlPullParser.PROCESSING_INSTRUCTION; +import static org.xmlpull.v1.XmlPullParser.START_DOCUMENT; +import static org.xmlpull.v1.XmlPullParser.START_TAG; +import static org.xmlpull.v1.XmlPullParser.TEXT; + +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.util.TypedXmlSerializer; + +import org.xmlpull.v1.XmlPullParser; +import org.xmlpull.v1.XmlSerializer; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.Writer; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; + +/** + * Serializer that writes XML documents using a custom binary wire protocol + * which benchmarking has shown to be 4.3x faster and use 2.4x less disk space + * than {@code Xml.newFastSerializer()} for a typical {@code packages.xml}. + *

+ * The high-level design of the wire protocol is to directly serialize the event + * stream, while efficiently and compactly writing strongly-typed primitives + * delivered through the {@link TypedXmlSerializer} interface. + *

+ * Each serialized event is a single byte where the lower half is a normal + * {@link XmlPullParser} token and the upper half is an optional data type + * signal, such as {@link #TYPE_INT}. + *

+ * This serializer has some specific limitations: + *

+ */ +public final class BinaryXmlSerializer implements TypedXmlSerializer { + /** + * The wire protocol always begins with a well-known magic value of + * {@code ABX_}, representing "Android Binary XML." The final byte is a + * version number which may be incremented as the protocol changes. + */ + static final byte[] PROTOCOL_MAGIC_VERSION_0 = new byte[] { 0x41, 0x42, 0x58, 0x00 }; + + /** + * Internal token which represents an attribute associated with the most + * recent {@link #START_TAG} token. + */ + static final int ATTRIBUTE = 15; + + static final int TYPE_NULL = 1 << 4; + static final int TYPE_STRING = 2 << 4; + static final int TYPE_STRING_INTERNED = 3 << 4; + static final int TYPE_BYTES_HEX = 4 << 4; + static final int TYPE_BYTES_BASE64 = 5 << 4; + static final int TYPE_INT = 6 << 4; + static final int TYPE_INT_HEX = 7 << 4; + static final int TYPE_LONG = 8 << 4; + static final int TYPE_LONG_HEX = 9 << 4; + static final int TYPE_FLOAT = 10 << 4; + static final int TYPE_DOUBLE = 11 << 4; + static final int TYPE_BOOLEAN_TRUE = 12 << 4; + static final int TYPE_BOOLEAN_FALSE = 13 << 4; + + /** + * Default buffer size, which matches {@code FastXmlSerializer}. This should + * be kept in sync with {@link BinaryXmlPullParser}. + */ + private static final int BUFFER_SIZE = 32_768; + + private FastDataOutput mOut; + + /** + * Stack of tags which are currently active via {@link #startTag} and which + * haven't been terminated via {@link #endTag}. + */ + private int mTagCount = 0; + private String[] mTagNames; + + /** + * Write the given token and optional {@link String} into our buffer. + */ + private void writeToken(int token, @Nullable String text) throws IOException { + if (text != null) { + mOut.writeByte(token | TYPE_STRING); + mOut.writeUTF(text); + } else { + mOut.writeByte(token | TYPE_NULL); + } + } + + @Override + public void setOutput(@NonNull OutputStream os, @Nullable String encoding) throws IOException { + if (encoding != null && !StandardCharsets.UTF_8.name().equals(encoding)) { + throw new UnsupportedOperationException(); + } + + mOut = new FastDataOutput(os, BUFFER_SIZE); + mOut.write(PROTOCOL_MAGIC_VERSION_0); + + mTagCount = 0; + mTagNames = new String[8]; + } + + @Override + public void setOutput(Writer writer) { + throw new UnsupportedOperationException(); + } + + @Override + public void flush() throws IOException { + mOut.flush(); + } + + @Override + public void startDocument(@Nullable String encoding, @Nullable Boolean standalone) + throws IOException { + if (encoding != null && !StandardCharsets.UTF_8.name().equals(encoding)) { + throw new UnsupportedOperationException(); + } + mOut.writeByte(START_DOCUMENT | TYPE_NULL); + } + + @Override + public void endDocument() throws IOException { + mOut.writeByte(END_DOCUMENT | TYPE_NULL); + flush(); + } + + @Override + public int getDepth() { + return mTagCount; + } + + @Override + public String getNamespace() { + // Namespaces are unsupported + return XmlPullParser.NO_NAMESPACE; + } + + @Override + public String getName() { + return mTagNames[mTagCount - 1]; + } + + @Override + public XmlSerializer startTag(String namespace, String name) throws IOException { + if (namespace != null && !namespace.isEmpty()) throw illegalNamespace(); + if (mTagCount == mTagNames.length) { + mTagNames = Arrays.copyOf(mTagNames, mTagCount + (mTagCount >> 1)); + } + mTagNames[mTagCount++] = name; + mOut.writeByte(START_TAG | TYPE_STRING_INTERNED); + mOut.writeInternedUTF(name); + return this; + } + + @Override + public XmlSerializer endTag(String namespace, String name) throws IOException { + if (namespace != null && !namespace.isEmpty()) throw illegalNamespace(); + mTagCount--; + mOut.writeByte(END_TAG | TYPE_STRING_INTERNED); + mOut.writeInternedUTF(name); + return this; + } + + @Override + public XmlSerializer attribute(String namespace, String name, String value) throws IOException { + if (namespace != null && !namespace.isEmpty()) throw illegalNamespace(); + mOut.writeByte(ATTRIBUTE | TYPE_STRING); + mOut.writeInternedUTF(name); + mOut.writeUTF(value); + return this; + } + + @Override + public XmlSerializer attributeInterned(String namespace, String name, String value) + throws IOException { + if (namespace != null && !namespace.isEmpty()) throw illegalNamespace(); + mOut.writeByte(ATTRIBUTE | TYPE_STRING_INTERNED); + mOut.writeInternedUTF(name); + mOut.writeInternedUTF(value); + return this; + } + + @Override + public XmlSerializer attributeBytesHex(String namespace, String name, byte[] value) + throws IOException { + if (namespace != null && !namespace.isEmpty()) throw illegalNamespace(); + mOut.writeByte(ATTRIBUTE | TYPE_BYTES_HEX); + mOut.writeInternedUTF(name); + mOut.writeShort(value.length); + mOut.write(value); + return this; + } + + @Override + public XmlSerializer attributeBytesBase64(String namespace, String name, byte[] value) + throws IOException { + if (namespace != null && !namespace.isEmpty()) throw illegalNamespace(); + mOut.writeByte(ATTRIBUTE | TYPE_BYTES_BASE64); + mOut.writeInternedUTF(name); + mOut.writeShort(value.length); + mOut.write(value); + return this; + } + + @Override + public XmlSerializer attributeInt(String namespace, String name, int value) + throws IOException { + if (namespace != null && !namespace.isEmpty()) throw illegalNamespace(); + mOut.writeByte(ATTRIBUTE | TYPE_INT); + mOut.writeInternedUTF(name); + mOut.writeInt(value); + return this; + } + + @Override + public XmlSerializer attributeIntHex(String namespace, String name, int value) + throws IOException { + if (namespace != null && !namespace.isEmpty()) throw illegalNamespace(); + mOut.writeByte(ATTRIBUTE | TYPE_INT_HEX); + mOut.writeInternedUTF(name); + mOut.writeInt(value); + return this; + } + + @Override + public XmlSerializer attributeLong(String namespace, String name, long value) + throws IOException { + if (namespace != null && !namespace.isEmpty()) throw illegalNamespace(); + mOut.writeByte(ATTRIBUTE | TYPE_LONG); + mOut.writeInternedUTF(name); + mOut.writeLong(value); + return this; + } + + @Override + public XmlSerializer attributeLongHex(String namespace, String name, long value) + throws IOException { + if (namespace != null && !namespace.isEmpty()) throw illegalNamespace(); + mOut.writeByte(ATTRIBUTE | TYPE_LONG_HEX); + mOut.writeInternedUTF(name); + mOut.writeLong(value); + return this; + } + + @Override + public XmlSerializer attributeFloat(String namespace, String name, float value) + throws IOException { + if (namespace != null && !namespace.isEmpty()) throw illegalNamespace(); + mOut.writeByte(ATTRIBUTE | TYPE_FLOAT); + mOut.writeInternedUTF(name); + mOut.writeFloat(value); + return this; + } + + @Override + public XmlSerializer attributeDouble(String namespace, String name, double value) + throws IOException { + if (namespace != null && !namespace.isEmpty()) throw illegalNamespace(); + mOut.writeByte(ATTRIBUTE | TYPE_DOUBLE); + mOut.writeInternedUTF(name); + mOut.writeDouble(value); + return this; + } + + @Override + public XmlSerializer attributeBoolean(String namespace, String name, boolean value) + throws IOException { + if (namespace != null && !namespace.isEmpty()) throw illegalNamespace(); + if (value) { + mOut.writeByte(ATTRIBUTE | TYPE_BOOLEAN_TRUE); + mOut.writeInternedUTF(name); + } else { + mOut.writeByte(ATTRIBUTE | TYPE_BOOLEAN_FALSE); + mOut.writeInternedUTF(name); + } + return this; + } + + @Override + public XmlSerializer text(char[] buf, int start, int len) throws IOException { + writeToken(TEXT, new String(buf, start, len)); + return this; + } + + @Override + public XmlSerializer text(String text) throws IOException { + writeToken(TEXT, text); + return this; + } + + @Override + public void cdsect(String text) throws IOException { + writeToken(CDSECT, text); + } + + @Override + public void entityRef(String text) throws IOException { + writeToken(ENTITY_REF, text); + } + + @Override + public void processingInstruction(String text) throws IOException { + writeToken(PROCESSING_INSTRUCTION, text); + } + + @Override + public void comment(String text) throws IOException { + writeToken(COMMENT, text); + } + + @Override + public void docdecl(String text) throws IOException { + writeToken(DOCDECL, text); + } + + @Override + public void ignorableWhitespace(String text) throws IOException { + writeToken(IGNORABLE_WHITESPACE, text); + } + + @Override + public void setFeature(String name, boolean state) { + // Quietly handle no-op features + if ("http://xmlpull.org/v1/doc/features.html#indent-output".equals(name)) { + return; + } + // Features are not supported + throw new UnsupportedOperationException(); + } + + @Override + public boolean getFeature(String name) { + // Features are not supported + throw new UnsupportedOperationException(); + } + + @Override + public void setProperty(String name, Object value) { + // Properties are not supported + throw new UnsupportedOperationException(); + } + + @Override + public Object getProperty(String name) { + // Properties are not supported + throw new UnsupportedOperationException(); + } + + @Override + public void setPrefix(String prefix, String namespace) { + // Prefixes are not supported + throw new UnsupportedOperationException(); + } + + @Override + public String getPrefix(String namespace, boolean generatePrefix) { + // Prefixes are not supported + throw new UnsupportedOperationException(); + } + + private static IllegalArgumentException illegalNamespace() { + throw new IllegalArgumentException("Namespaces are not supported"); + } +} diff --git a/core/tests/coretests/src/android/util/BinaryXmlTest.java b/core/tests/coretests/src/android/util/BinaryXmlTest.java new file mode 100644 index 0000000000000..be63a0ecc65fb --- /dev/null +++ b/core/tests/coretests/src/android/util/BinaryXmlTest.java @@ -0,0 +1,99 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.util; + +import static android.util.XmlTest.assertNext; +import static android.util.XmlTest.buildPersistableBundle; +import static android.util.XmlTest.doPersistableBundleRead; +import static android.util.XmlTest.doPersistableBundleWrite; + +import static org.junit.Assert.assertEquals; +import static org.xmlpull.v1.XmlPullParser.START_TAG; + +import android.os.PersistableBundle; + +import androidx.test.runner.AndroidJUnit4; + +import org.junit.Test; +import org.junit.runner.RunWith; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; + +@RunWith(AndroidJUnit4.class) +public class BinaryXmlTest { + /** + * Verify that we can write and read large numbers of interned + * {@link String} values. + */ + @Test + public void testLargeInterned_Binary() throws Exception { + // We're okay with the tag itself being interned + final int count = (1 << 16) - 2; + + final TypedXmlSerializer out = Xml.newBinarySerializer(); + final ByteArrayOutputStream os = new ByteArrayOutputStream(); + out.setOutput(os, StandardCharsets.UTF_8.name()); + out.startTag(null, "tag"); + for (int i = 0; i < count; i++) { + out.attribute(null, "name" + i, "value"); + } + out.endTag(null, "tag"); + out.flush(); + + final TypedXmlPullParser in = Xml.newBinaryPullParser(); + final ByteArrayInputStream is = new ByteArrayInputStream(os.toByteArray()); + in.setInput(is, StandardCharsets.UTF_8.name()); + assertNext(in, START_TAG, "tag", 1); + assertEquals(count, in.getAttributeCount()); + } + + @Test + public void testTranscode_FastToBinary() throws Exception { + doTranscode(Xml.newFastSerializer(), Xml.newFastPullParser(), + Xml.newBinarySerializer(), Xml.newBinaryPullParser()); + } + + @Test + public void testTranscode_BinaryToFast() throws Exception { + doTranscode(Xml.newBinarySerializer(), Xml.newBinaryPullParser(), + Xml.newFastSerializer(), Xml.newFastPullParser()); + } + + /** + * Verify that a complex {@link PersistableBundle} can be transcoded using + * the two given formats with the original structure intact. + */ + private static void doTranscode(TypedXmlSerializer firstOut, TypedXmlPullParser firstIn, + TypedXmlSerializer secondOut, TypedXmlPullParser secondIn) throws Exception { + final PersistableBundle expected = buildPersistableBundle(); + final byte[] firstRaw = doPersistableBundleWrite(firstOut, expected); + + // Perform actual transcoding between the two formats + final ByteArrayInputStream is = new ByteArrayInputStream(firstRaw); + firstIn.setInput(is, StandardCharsets.UTF_8.name()); + final ByteArrayOutputStream os = new ByteArrayOutputStream(); + secondOut.setOutput(os, StandardCharsets.UTF_8.name()); + Xml.copy(firstIn, secondOut); + + // Yes, this string-based check is fragile, but kindofEquals() is broken + // when working with nested objects and arrays + final PersistableBundle actual = doPersistableBundleRead(secondIn, os.toByteArray()); + assertEquals(expected.toString(), actual.toString()); + } +} diff --git a/core/tests/coretests/src/android/util/XmlTest.java b/core/tests/coretests/src/android/util/XmlTest.java index 602b672a2b82d..2ae9cdfedb11f 100644 --- a/core/tests/coretests/src/android/util/XmlTest.java +++ b/core/tests/coretests/src/android/util/XmlTest.java @@ -52,13 +52,19 @@ public class XmlTest { Xml.newFastPullParser()); } + @Test + public void testLargeValues_Binary() throws Exception { + doLargeValues(Xml.newBinarySerializer(), + Xml.newBinaryPullParser()); + } + /** * Verify that we can write and read large {@link String} and {@code byte[]} * without issues. */ private static void doLargeValues(TypedXmlSerializer out, TypedXmlPullParser in) throws Exception { - final char[] chars = new char[32_768]; + final char[] chars = new char[(1 << 16) - 1]; Arrays.fill(chars, '!'); final String string = new String(chars); @@ -93,6 +99,12 @@ public class XmlTest { Xml.newFastPullParser()); } + @Test + public void testPersistableBundle_Binary() throws Exception { + doPersistableBundle(Xml.newBinarySerializer(), + Xml.newBinaryPullParser()); + } + /** * Verify that a complex {@link PersistableBundle} can be serialized out and * then parsed in with the original structure intact. @@ -108,7 +120,7 @@ public class XmlTest { assertEquals(expected.toString(), actual.toString()); } - private static PersistableBundle buildPersistableBundle() { + static PersistableBundle buildPersistableBundle() { final PersistableBundle outer = new PersistableBundle(); outer.putBoolean("boolean", true); @@ -130,7 +142,7 @@ public class XmlTest { return outer; } - private static byte[] doPersistableBundleWrite(TypedXmlSerializer out, PersistableBundle bundle) + static byte[] doPersistableBundleWrite(TypedXmlSerializer out, PersistableBundle bundle) throws Exception { // We purposefully omit START/END_DOCUMENT events here to verify correct // behavior of what PersistableBundle does internally @@ -143,7 +155,7 @@ public class XmlTest { return os.toByteArray(); } - private static PersistableBundle doPersistableBundleRead(TypedXmlPullParser in, byte[] raw) + static PersistableBundle doPersistableBundleRead(TypedXmlPullParser in, byte[] raw) throws Exception { final ByteArrayInputStream is = new ByteArrayInputStream(raw); in.setInput(is, StandardCharsets.UTF_8.name()); @@ -163,6 +175,12 @@ public class XmlTest { Xml.newFastPullParser()); } + @Test + public void testVerify_Binary() throws Exception { + doVerify(Xml.newBinarySerializer(), + Xml.newBinaryPullParser()); + } + /** * Verify that example test data is correctly serialized and parsed * end-to-end using the given objects. @@ -268,7 +286,7 @@ public class XmlTest { assertNext(in, END_DOCUMENT); } - private static void assertNext(TypedXmlPullParser in, int token) throws Exception { + static void assertNext(TypedXmlPullParser in, int token) throws Exception { // We're willing to skip over empty text regions, which some // serializers emit transparently int event; @@ -278,7 +296,7 @@ public class XmlTest { assertEquals("getEventType", token, in.getEventType()); } - private static void assertNext(TypedXmlPullParser in, int token, String name, int depth) + static void assertNext(TypedXmlPullParser in, int token, String name, int depth) throws Exception { assertNext(in, token); assertEquals("getName", name, in.getName());