diff --git a/services/backup/java/com/android/server/backup/encryption/chunking/ByteRange.java b/services/backup/java/com/android/server/backup/encryption/chunking/ByteRange.java new file mode 100644 index 0000000000000..004d9e3b45f16 --- /dev/null +++ b/services/backup/java/com/android/server/backup/encryption/chunking/ByteRange.java @@ -0,0 +1,80 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.backup.encryption.chunking; + +import com.android.internal.util.Preconditions; + +/** Representation of a range of bytes to be downloaded. */ +final class ByteRange { + private final long mStart; + private final long mEnd; + + /** Creates a range of bytes which includes {@code mStart} and {@code mEnd}. */ + ByteRange(long start, long end) { + Preconditions.checkArgument(start >= 0); + Preconditions.checkArgument(end >= start); + mStart = start; + mEnd = end; + } + + /** Returns the start of the {@code ByteRange}. The start is included in the range. */ + long getStart() { + return mStart; + } + + /** Returns the end of the {@code ByteRange}. The end is included in the range. */ + long getEnd() { + return mEnd; + } + + /** Returns the number of bytes included in the {@code ByteRange}. */ + int getLength() { + return (int) (mEnd - mStart + 1); + } + + /** Creates a new {@link ByteRange} from {@code mStart} to {@code mEnd + length}. */ + ByteRange extend(long length) { + Preconditions.checkArgument(length > 0); + return new ByteRange(mStart, mEnd + length); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof ByteRange)) { + return false; + } + + ByteRange byteRange = (ByteRange) o; + return (mEnd == byteRange.mEnd && mStart == byteRange.mStart); + } + + @Override + public int hashCode() { + int result = 17; + result = 31 * result + (int) (mStart ^ (mStart >>> 32)); + result = 31 * result + (int) (mEnd ^ (mEnd >>> 32)); + return result; + } + + @Override + public String toString() { + return String.format("ByteRange{mStart=%d, mEnd=%d}", mStart, mEnd); + } +} diff --git a/services/backup/java/com/android/server/backup/encryption/chunking/DiffScriptBackupWriter.java b/services/backup/java/com/android/server/backup/encryption/chunking/DiffScriptBackupWriter.java new file mode 100644 index 0000000000000..69fb5cbf606d3 --- /dev/null +++ b/services/backup/java/com/android/server/backup/encryption/chunking/DiffScriptBackupWriter.java @@ -0,0 +1,75 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.backup.encryption.chunking; + +import com.android.internal.annotations.VisibleForTesting; + +import java.io.IOException; +import java.io.OutputStream; + +/** Writes backup data to a diff script, using a {@link SingleStreamDiffScriptWriter}. */ +public class DiffScriptBackupWriter implements BackupWriter { + /** + * The maximum size of a chunk in the diff script. The diff script writer {@code mWriter} will + * buffer this many bytes in memory. + */ + private static final int ENCRYPTION_DIFF_SCRIPT_MAX_CHUNK_SIZE_BYTES = 1024 * 1024; + + private final SingleStreamDiffScriptWriter mWriter; + private long mBytesWritten; + + /** + * Constructs a new writer which writes the diff script to the given output stream, using the + * maximum new chunk size {@code ENCRYPTION_DIFF_SCRIPT_MAX_CHUNK_SIZE_BYTES}. + */ + public static DiffScriptBackupWriter newInstance(OutputStream outputStream) { + SingleStreamDiffScriptWriter writer = + new SingleStreamDiffScriptWriter( + outputStream, ENCRYPTION_DIFF_SCRIPT_MAX_CHUNK_SIZE_BYTES); + return new DiffScriptBackupWriter(writer); + } + + @VisibleForTesting + DiffScriptBackupWriter(SingleStreamDiffScriptWriter writer) { + mWriter = writer; + } + + @Override + public void writeBytes(byte[] bytes) throws IOException { + for (byte b : bytes) { + mWriter.writeByte(b); + } + + mBytesWritten += bytes.length; + } + + @Override + public void writeChunk(long start, int length) throws IOException { + mWriter.writeChunk(start, length); + mBytesWritten += length; + } + + @Override + public long getBytesWritten() { + return mBytesWritten; + } + + @Override + public void flush() throws IOException { + mWriter.flush(); + } +} diff --git a/services/backup/java/com/android/server/backup/encryption/chunking/DiffScriptWriter.java b/services/backup/java/com/android/server/backup/encryption/chunking/DiffScriptWriter.java new file mode 100644 index 0000000000000..49d15712d4ccc --- /dev/null +++ b/services/backup/java/com/android/server/backup/encryption/chunking/DiffScriptWriter.java @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.backup.encryption.chunking; + +import java.io.IOException; +import java.io.OutputStream; + +/** Writer that formats a Diff Script and writes it to an output source. */ +interface DiffScriptWriter { + /** Adds a new byte to the diff script. */ + void writeByte(byte b) throws IOException; + + /** Adds a known chunk to the diff script. */ + void writeChunk(long chunkStart, int chunkLength) throws IOException; + + /** Indicates that no more bytes or chunks will be added to the diff script. */ + void flush() throws IOException; + + interface Factory { + DiffScriptWriter create(OutputStream outputStream); + } +} diff --git a/services/backup/java/com/android/server/backup/encryption/chunking/OutputStreamWrapper.java b/services/backup/java/com/android/server/backup/encryption/chunking/OutputStreamWrapper.java new file mode 100644 index 0000000000000..4aea60121810a --- /dev/null +++ b/services/backup/java/com/android/server/backup/encryption/chunking/OutputStreamWrapper.java @@ -0,0 +1,25 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.backup.encryption.chunking; + +import java.io.OutputStream; + +/** An interface that wraps one {@link OutputStream} with another for filtration purposes. */ +public interface OutputStreamWrapper { + /** Wraps a given {@link OutputStream}. */ + OutputStream wrap(OutputStream outputStream); +} diff --git a/services/backup/java/com/android/server/backup/encryption/chunking/SingleStreamDiffScriptWriter.java b/services/backup/java/com/android/server/backup/encryption/chunking/SingleStreamDiffScriptWriter.java new file mode 100644 index 0000000000000..0e4bd58345d55 --- /dev/null +++ b/services/backup/java/com/android/server/backup/encryption/chunking/SingleStreamDiffScriptWriter.java @@ -0,0 +1,130 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.backup.encryption.chunking; + +import android.annotation.Nullable; + +import com.android.internal.util.Preconditions; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.Charset; +import java.util.Locale; + +/** + * A {@link DiffScriptWriter} that writes an entire diff script to a single {@link OutputStream}. + */ +public class SingleStreamDiffScriptWriter implements DiffScriptWriter { + static final byte LINE_SEPARATOR = 0xA; + private static final Charset UTF_8 = Charset.forName("UTF-8"); + + private final int mMaxNewByteChunkSize; + private final OutputStream mOutputStream; + private final byte[] mByteBuffer; + private int mBufferSize = 0; + // Each chunk could be written immediately to the output stream. However, + // it is possible that chunks may overlap. We therefore cache the most recent + // reusable chunk and try to merge it with future chunks. + private ByteRange mReusableChunk; + + public SingleStreamDiffScriptWriter(OutputStream outputStream, int maxNewByteChunkSize) { + mOutputStream = outputStream; + mMaxNewByteChunkSize = maxNewByteChunkSize; + mByteBuffer = new byte[maxNewByteChunkSize]; + } + + @Override + public void writeByte(byte b) throws IOException { + if (mReusableChunk != null) { + writeReusableChunk(); + } + mByteBuffer[mBufferSize++] = b; + if (mBufferSize == mMaxNewByteChunkSize) { + writeByteBuffer(); + } + } + + @Override + public void writeChunk(long chunkStart, int chunkLength) throws IOException { + Preconditions.checkArgument(chunkStart >= 0); + Preconditions.checkArgument(chunkLength > 0); + if (mBufferSize != 0) { + writeByteBuffer(); + } + + if (mReusableChunk != null && mReusableChunk.getEnd() + 1 == chunkStart) { + // The new chunk overlaps the old, so combine them into a single byte range. + mReusableChunk = mReusableChunk.extend(chunkLength); + } else { + writeReusableChunk(); + mReusableChunk = new ByteRange(chunkStart, chunkStart + chunkLength - 1); + } + } + + @Override + public void flush() throws IOException { + Preconditions.checkState(!(mBufferSize != 0 && mReusableChunk != null)); + if (mBufferSize != 0) { + writeByteBuffer(); + } + if (mReusableChunk != null) { + writeReusableChunk(); + } + mOutputStream.flush(); + } + + private void writeByteBuffer() throws IOException { + mOutputStream.write(Integer.toString(mBufferSize).getBytes(UTF_8)); + mOutputStream.write(LINE_SEPARATOR); + mOutputStream.write(mByteBuffer, 0, mBufferSize); + mOutputStream.write(LINE_SEPARATOR); + mBufferSize = 0; + } + + private void writeReusableChunk() throws IOException { + if (mReusableChunk != null) { + mOutputStream.write( + String.format( + Locale.US, + "%d-%d", + mReusableChunk.getStart(), + mReusableChunk.getEnd()) + .getBytes(UTF_8)); + mOutputStream.write(LINE_SEPARATOR); + mReusableChunk = null; + } + } + + /** A factory that creates {@link SingleStreamDiffScriptWriter}s. */ + public static class Factory implements DiffScriptWriter.Factory { + private final int mMaxNewByteChunkSize; + private final OutputStreamWrapper mOutputStreamWrapper; + + public Factory(int maxNewByteChunkSize, @Nullable OutputStreamWrapper outputStreamWrapper) { + mMaxNewByteChunkSize = maxNewByteChunkSize; + mOutputStreamWrapper = outputStreamWrapper; + } + + @Override + public SingleStreamDiffScriptWriter create(OutputStream outputStream) { + if (mOutputStreamWrapper != null) { + outputStream = mOutputStreamWrapper.wrap(outputStream); + } + return new SingleStreamDiffScriptWriter(outputStream, mMaxNewByteChunkSize); + } + } +} diff --git a/services/robotests/src/com/android/server/backup/encryption/chunking/ByteRangeTest.java b/services/robotests/src/com/android/server/backup/encryption/chunking/ByteRangeTest.java new file mode 100644 index 0000000000000..8df08262c9faa --- /dev/null +++ b/services/robotests/src/com/android/server/backup/encryption/chunking/ByteRangeTest.java @@ -0,0 +1,57 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.backup.encryption.chunking; + +import static org.junit.Assert.assertEquals; +import static org.testng.Assert.assertThrows; + +import android.platform.test.annotations.Presubmit; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.robolectric.RobolectricTestRunner; + +/** Tests for {@link ByteRange}. */ +@RunWith(RobolectricTestRunner.class) +@Presubmit +public class ByteRangeTest { + @Test + public void getLength_includesEnd() throws Exception { + ByteRange byteRange = new ByteRange(5, 10); + + int length = byteRange.getLength(); + + assertEquals(6, length); + } + + @Test + public void constructor_rejectsNegativeStart() { + assertThrows(IllegalArgumentException.class, () -> new ByteRange(-1, 10)); + } + + @Test + public void constructor_rejectsEndBeforeStart() { + assertThrows(IllegalArgumentException.class, () -> new ByteRange(10, 9)); + } + + @Test + public void extend_withZeroLength_throwsException() { + ByteRange byteRange = new ByteRange(5, 10); + + assertThrows(IllegalArgumentException.class, () -> byteRange.extend(0)); + } +} diff --git a/services/robotests/src/com/android/server/backup/encryption/chunking/DiffScriptBackupWriterTest.java b/services/robotests/src/com/android/server/backup/encryption/chunking/DiffScriptBackupWriterTest.java new file mode 100644 index 0000000000000..2af6f2bee8ffb --- /dev/null +++ b/services/robotests/src/com/android/server/backup/encryption/chunking/DiffScriptBackupWriterTest.java @@ -0,0 +1,90 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.backup.encryption.chunking; + +import static com.google.common.truth.Truth.assertThat; + +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; + +import android.platform.test.annotations.Presubmit; + +import com.google.common.primitives.Bytes; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.robolectric.RobolectricTestRunner; + +import java.io.IOException; + +/** Tests for {@link DiffScriptBackupWriter}. */ +@RunWith(RobolectricTestRunner.class) +@Presubmit +public class DiffScriptBackupWriterTest { + private static final byte[] TEST_BYTES = {1, 2, 3, 4, 5, 6, 7, 8, 9}; + + @Captor private ArgumentCaptor mBytesCaptor; + @Mock private SingleStreamDiffScriptWriter mDiffScriptWriter; + private BackupWriter mBackupWriter; + + @Before + public void setUp() { + mDiffScriptWriter = mock(SingleStreamDiffScriptWriter.class); + mBackupWriter = new DiffScriptBackupWriter(mDiffScriptWriter); + mBytesCaptor = ArgumentCaptor.forClass(Byte.class); + } + + @Test + public void writeBytes_writesBytesToWriter() throws Exception { + mBackupWriter.writeBytes(TEST_BYTES); + + verify(mDiffScriptWriter, atLeastOnce()).writeByte(mBytesCaptor.capture()); + assertThat(mBytesCaptor.getAllValues()) + .containsExactlyElementsIn(Bytes.asList(TEST_BYTES)) + .inOrder(); + } + + @Test + public void writeChunk_writesChunkToWriter() throws Exception { + mBackupWriter.writeChunk(0, 10); + + verify(mDiffScriptWriter).writeChunk(0, 10); + } + + @Test + public void getBytesWritten_returnsTotalSum() throws Exception { + mBackupWriter.writeBytes(TEST_BYTES); + mBackupWriter.writeBytes(TEST_BYTES); + mBackupWriter.writeChunk(/*start=*/ 0, /*length=*/ 10); + + long bytesWritten = mBackupWriter.getBytesWritten(); + + assertThat(bytesWritten).isEqualTo(2 * TEST_BYTES.length + 10); + } + + @Test + public void flush_flushesWriter() throws IOException { + mBackupWriter.flush(); + + verify(mDiffScriptWriter).flush(); + } +} diff --git a/services/robotests/src/com/android/server/backup/encryption/chunking/SingleStreamDiffScriptWriterTest.java b/services/robotests/src/com/android/server/backup/encryption/chunking/SingleStreamDiffScriptWriterTest.java new file mode 100644 index 0000000000000..73baf80a2c708 --- /dev/null +++ b/services/robotests/src/com/android/server/backup/encryption/chunking/SingleStreamDiffScriptWriterTest.java @@ -0,0 +1,128 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.backup.encryption.chunking; + +import static com.google.common.truth.Truth.assertThat; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.testng.Assert.assertThrows; + +import android.platform.test.annotations.Presubmit; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.robolectric.RobolectricTestRunner; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.Locale; + +/** Tests for {@link SingleStreamDiffScriptWriter}. */ +@RunWith(RobolectricTestRunner.class) +@Presubmit +public class SingleStreamDiffScriptWriterTest { + private static final int MAX_CHUNK_SIZE_IN_BYTES = 256; + /** By default this Locale does not use Arabic numbers for %d formatting. */ + private static final Locale HINDI = new Locale("hi", "IN"); + + private Locale mDefaultLocale; + private ByteArrayOutputStream mOutputStream; + private SingleStreamDiffScriptWriter mDiffScriptWriter; + + @Before + public void setUp() { + mDefaultLocale = Locale.getDefault(); + mOutputStream = new ByteArrayOutputStream(); + mDiffScriptWriter = + new SingleStreamDiffScriptWriter(mOutputStream, MAX_CHUNK_SIZE_IN_BYTES); + } + + @After + public void tearDown() { + Locale.setDefault(mDefaultLocale); + } + + @Test + public void writeChunk_withNegativeStart_throwsException() { + assertThrows( + IllegalArgumentException.class, + () -> mDiffScriptWriter.writeChunk(-1, 50)); + } + + @Test + public void writeChunk_withZeroLength_throwsException() { + assertThrows( + IllegalArgumentException.class, + () -> mDiffScriptWriter.writeChunk(0, 0)); + } + + @Test + public void writeChunk_withExistingBytesInBuffer_writesBufferFirst() + throws IOException { + String testString = "abcd"; + writeStringAsBytesToWriter(testString, mDiffScriptWriter); + + mDiffScriptWriter.writeChunk(0, 20); + mDiffScriptWriter.flush(); + + // Expected format: length of abcd, newline, abcd, newline, chunk start - chunk end + assertThat(mOutputStream.toString("UTF-8")).isEqualTo( + String.format("%d\n%s\n%d-%d\n", testString.length(), testString, 0, 19)); + } + + @Test + public void writeChunk_overlappingPreviousChunk_combinesChunks() throws IOException { + mDiffScriptWriter.writeChunk(3, 4); + + mDiffScriptWriter.writeChunk(7, 5); + mDiffScriptWriter.flush(); + + assertThat(mOutputStream.toString("UTF-8")).isEqualTo(String.format("3-11\n")); + } + + @Test + public void writeChunk_formatsByteIndexesUsingArabicNumbers() throws Exception { + Locale.setDefault(HINDI); + + mDiffScriptWriter.writeChunk(0, 12345); + mDiffScriptWriter.flush(); + + assertThat(mOutputStream.toString("UTF-8")).isEqualTo("0-12344\n"); + } + + @Test + public void flush_flushesOutputStream() throws IOException { + ByteArrayOutputStream mockOutputStream = mock(ByteArrayOutputStream.class); + SingleStreamDiffScriptWriter diffScriptWriter = + new SingleStreamDiffScriptWriter(mockOutputStream, MAX_CHUNK_SIZE_IN_BYTES); + + diffScriptWriter.flush(); + + verify(mockOutputStream).flush(); + } + + private void writeStringAsBytesToWriter(String string, SingleStreamDiffScriptWriter writer) + throws IOException { + byte[] bytes = string.getBytes("UTF-8"); + for (int i = 0; i < bytes.length; i++) { + writer.writeByte(bytes[i]); + } + } +}