Merge "Ports DiffScriptBackupWriter from gmscore to AOSP."

services/backup/java/com/android/server/backup/encryption/chunking/ByteRange.java

@@ -0,0 +1,80 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.server.backup.encryption.chunking;
+
+import com.android.internal.util.Preconditions;
+
+/** Representation of a range of bytes to be downloaded. */
+final class ByteRange {
+    private final long mStart;
+    private final long mEnd;
+
+    /** Creates a range of bytes which includes {@code mStart} and {@code mEnd}. */
+    ByteRange(long start, long end) {
+        Preconditions.checkArgument(start >= 0);
+        Preconditions.checkArgument(end >= start);
+        mStart = start;
+        mEnd = end;
+    }
+
+    /** Returns the start of the {@code ByteRange}. The start is included in the range. */
+    long getStart() {
+        return mStart;
+    }
+
+    /** Returns the end of the {@code ByteRange}. The end is included in the range. */
+    long getEnd() {
+        return mEnd;
+    }
+
+    /** Returns the number of bytes included in the {@code ByteRange}. */
+    int getLength() {
+        return (int) (mEnd - mStart + 1);
+    }
+
+    /** Creates a new {@link ByteRange} from {@code mStart} to {@code mEnd + length}. */
+    ByteRange extend(long length) {
+        Preconditions.checkArgument(length > 0);
+        return new ByteRange(mStart, mEnd + length);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (!(o instanceof ByteRange)) {
+            return false;
+        }
+
+        ByteRange byteRange = (ByteRange) o;
+        return (mEnd == byteRange.mEnd && mStart == byteRange.mStart);
+    }
+
+    @Override
+    public int hashCode() {
+        int result = 17;
+        result = 31 * result + (int) (mStart ^ (mStart >>> 32));
+        result = 31 * result + (int) (mEnd ^ (mEnd >>> 32));
+        return result;
+    }
+
+    @Override
+    public String toString() {
+        return String.format("ByteRange{mStart=%d, mEnd=%d}", mStart, mEnd);
+    }
+}

services/backup/java/com/android/server/backup/encryption/chunking/DiffScriptBackupWriter.java

@@ -0,0 +1,75 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.server.backup.encryption.chunking;
+
+import com.android.internal.annotations.VisibleForTesting;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+/** Writes backup data to a diff script, using a {@link SingleStreamDiffScriptWriter}. */
+public class DiffScriptBackupWriter implements BackupWriter {
+    /**
+     * The maximum size of a chunk in the diff script. The diff script writer {@code mWriter} will
+     * buffer this many bytes in memory.
+     */
+    private static final int ENCRYPTION_DIFF_SCRIPT_MAX_CHUNK_SIZE_BYTES = 1024 * 1024;
+
+    private final SingleStreamDiffScriptWriter mWriter;
+    private long mBytesWritten;
+
+    /**
+     * Constructs a new writer which writes the diff script to the given output stream, using the
+     * maximum new chunk size {@code ENCRYPTION_DIFF_SCRIPT_MAX_CHUNK_SIZE_BYTES}.
+     */
+    public static DiffScriptBackupWriter newInstance(OutputStream outputStream) {
+        SingleStreamDiffScriptWriter writer =
+                new SingleStreamDiffScriptWriter(
+                        outputStream, ENCRYPTION_DIFF_SCRIPT_MAX_CHUNK_SIZE_BYTES);
+        return new DiffScriptBackupWriter(writer);
+    }
+
+    @VisibleForTesting
+    DiffScriptBackupWriter(SingleStreamDiffScriptWriter writer) {
+        mWriter = writer;
+    }
+
+    @Override
+    public void writeBytes(byte[] bytes) throws IOException {
+        for (byte b : bytes) {
+            mWriter.writeByte(b);
+        }
+
+        mBytesWritten += bytes.length;
+    }
+
+    @Override
+    public void writeChunk(long start, int length) throws IOException {
+        mWriter.writeChunk(start, length);
+        mBytesWritten += length;
+    }
+
+    @Override
+    public long getBytesWritten() {
+        return mBytesWritten;
+    }
+
+    @Override
+    public void flush() throws IOException {
+        mWriter.flush();
+    }
+}

services/backup/java/com/android/server/backup/encryption/chunking/DiffScriptWriter.java

@@ -0,0 +1,36 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.server.backup.encryption.chunking;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+/** Writer that formats a Diff Script and writes it to an output source. */
+interface DiffScriptWriter {
+    /** Adds a new byte to the diff script. */
+    void writeByte(byte b) throws IOException;
+
+    /** Adds a known chunk to the diff script. */
+    void writeChunk(long chunkStart, int chunkLength) throws IOException;
+
+    /** Indicates that no more bytes or chunks will be added to the diff script. */
+    void flush() throws IOException;
+
+    interface Factory {
+        DiffScriptWriter create(OutputStream outputStream);
+    }
+}

services/backup/java/com/android/server/backup/encryption/chunking/OutputStreamWrapper.java

@@ -0,0 +1,25 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.server.backup.encryption.chunking;
+
+import java.io.OutputStream;
+
+/** An interface that wraps one {@link OutputStream} with another for filtration purposes. */
+public interface OutputStreamWrapper {
+    /** Wraps a given {@link OutputStream}. */
+    OutputStream wrap(OutputStream outputStream);
+}

services/backup/java/com/android/server/backup/encryption/chunking/SingleStreamDiffScriptWriter.java

@@ -0,0 +1,130 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.server.backup.encryption.chunking;
+
+import android.annotation.Nullable;
+
+import com.android.internal.util.Preconditions;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.charset.Charset;
+import java.util.Locale;
+
+/**
+ * A {@link DiffScriptWriter} that writes an entire diff script to a single {@link OutputStream}.
+ */
+public class SingleStreamDiffScriptWriter implements DiffScriptWriter {
+    static final byte LINE_SEPARATOR = 0xA;
+    private static final Charset UTF_8 = Charset.forName("UTF-8");
+
+    private final int mMaxNewByteChunkSize;
+    private final OutputStream mOutputStream;
+    private final byte[] mByteBuffer;
+    private int mBufferSize = 0;
+    // Each chunk could be written immediately to the output stream. However,
+    // it is possible that chunks may overlap. We therefore cache the most recent
+    // reusable chunk and try to merge it with future chunks.
+    private ByteRange mReusableChunk;
+
+    public SingleStreamDiffScriptWriter(OutputStream outputStream, int maxNewByteChunkSize) {
+        mOutputStream = outputStream;
+        mMaxNewByteChunkSize = maxNewByteChunkSize;
+        mByteBuffer = new byte[maxNewByteChunkSize];
+    }
+
+    @Override
+    public void writeByte(byte b) throws IOException {
+        if (mReusableChunk != null) {
+            writeReusableChunk();
+        }
+        mByteBuffer[mBufferSize++] = b;
+        if (mBufferSize == mMaxNewByteChunkSize) {
+            writeByteBuffer();
+        }
+    }
+
+    @Override
+    public void writeChunk(long chunkStart, int chunkLength) throws IOException {
+        Preconditions.checkArgument(chunkStart >= 0);
+        Preconditions.checkArgument(chunkLength > 0);
+        if (mBufferSize != 0) {
+            writeByteBuffer();
+        }
+
+        if (mReusableChunk != null && mReusableChunk.getEnd() + 1 == chunkStart) {
+            // The new chunk overlaps the old, so combine them into a single byte range.
+            mReusableChunk = mReusableChunk.extend(chunkLength);
+        } else {
+            writeReusableChunk();
+            mReusableChunk = new ByteRange(chunkStart, chunkStart + chunkLength - 1);
+        }
+    }
+
+    @Override
+    public void flush() throws IOException {
+        Preconditions.checkState(!(mBufferSize != 0 && mReusableChunk != null));
+        if (mBufferSize != 0) {
+            writeByteBuffer();
+        }
+        if (mReusableChunk != null) {
+            writeReusableChunk();
+        }
+        mOutputStream.flush();
+    }
+
+    private void writeByteBuffer() throws IOException {
+        mOutputStream.write(Integer.toString(mBufferSize).getBytes(UTF_8));
+        mOutputStream.write(LINE_SEPARATOR);
+        mOutputStream.write(mByteBuffer, 0, mBufferSize);
+        mOutputStream.write(LINE_SEPARATOR);
+        mBufferSize = 0;
+    }
+
+    private void writeReusableChunk() throws IOException {
+        if (mReusableChunk != null) {
+            mOutputStream.write(
+                    String.format(
+                                    Locale.US,
+                                    "%d-%d",
+                                    mReusableChunk.getStart(),
+                                    mReusableChunk.getEnd())
+                            .getBytes(UTF_8));
+            mOutputStream.write(LINE_SEPARATOR);
+            mReusableChunk = null;
+        }
+    }
+
+    /** A factory that creates {@link SingleStreamDiffScriptWriter}s. */
+    public static class Factory implements DiffScriptWriter.Factory {
+        private final int mMaxNewByteChunkSize;
+        private final OutputStreamWrapper mOutputStreamWrapper;
+
+        public Factory(int maxNewByteChunkSize, @Nullable OutputStreamWrapper outputStreamWrapper) {
+            mMaxNewByteChunkSize = maxNewByteChunkSize;
+            mOutputStreamWrapper = outputStreamWrapper;
+        }
+
+        @Override
+        public SingleStreamDiffScriptWriter create(OutputStream outputStream) {
+            if (mOutputStreamWrapper != null) {
+                outputStream = mOutputStreamWrapper.wrap(outputStream);
+            }
+            return new SingleStreamDiffScriptWriter(outputStream, mMaxNewByteChunkSize);
+        }
+    }
+}

services/robotests/src/com/android/server/backup/encryption/chunking/ByteRangeTest.java

@@ -0,0 +1,57 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.server.backup.encryption.chunking;
+
+import static org.junit.Assert.assertEquals;
+import static org.testng.Assert.assertThrows;
+
+import android.platform.test.annotations.Presubmit;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.robolectric.RobolectricTestRunner;
+
+/** Tests for {@link ByteRange}. */
+@RunWith(RobolectricTestRunner.class)
+@Presubmit
+public class ByteRangeTest {
+    @Test
+    public void getLength_includesEnd() throws Exception {
+        ByteRange byteRange = new ByteRange(5, 10);
+
+        int length = byteRange.getLength();
+
+        assertEquals(6, length);
+    }
+
+    @Test
+    public void constructor_rejectsNegativeStart() {
+        assertThrows(IllegalArgumentException.class, () -> new ByteRange(-1, 10));
+    }
+
+    @Test
+    public void constructor_rejectsEndBeforeStart() {
+        assertThrows(IllegalArgumentException.class, () -> new ByteRange(10, 9));
+    }
+
+    @Test
+    public void extend_withZeroLength_throwsException() {
+        ByteRange byteRange = new ByteRange(5, 10);
+
+        assertThrows(IllegalArgumentException.class, () -> byteRange.extend(0));
+    }
+}

services/robotests/src/com/android/server/backup/encryption/chunking/DiffScriptBackupWriterTest.java

@@ -0,0 +1,90 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.server.backup.encryption.chunking;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import static org.mockito.Mockito.atLeastOnce;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+
+import android.platform.test.annotations.Presubmit;
+
+import com.google.common.primitives.Bytes;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Captor;
+import org.mockito.Mock;
+import org.robolectric.RobolectricTestRunner;
+
+import java.io.IOException;
+
+/** Tests for {@link DiffScriptBackupWriter}. */
+@RunWith(RobolectricTestRunner.class)
+@Presubmit
+public class DiffScriptBackupWriterTest {
+    private static final byte[] TEST_BYTES = {1, 2, 3, 4, 5, 6, 7, 8, 9};
+
+    @Captor private ArgumentCaptor<Byte> mBytesCaptor;
+    @Mock private SingleStreamDiffScriptWriter mDiffScriptWriter;
+    private BackupWriter mBackupWriter;
+
+    @Before
+    public void setUp() {
+        mDiffScriptWriter = mock(SingleStreamDiffScriptWriter.class);
+        mBackupWriter = new DiffScriptBackupWriter(mDiffScriptWriter);
+        mBytesCaptor = ArgumentCaptor.forClass(Byte.class);
+    }
+
+    @Test
+    public void writeBytes_writesBytesToWriter() throws Exception {
+        mBackupWriter.writeBytes(TEST_BYTES);
+
+        verify(mDiffScriptWriter, atLeastOnce()).writeByte(mBytesCaptor.capture());
+        assertThat(mBytesCaptor.getAllValues())
+                .containsExactlyElementsIn(Bytes.asList(TEST_BYTES))
+                .inOrder();
+    }
+
+    @Test
+    public void writeChunk_writesChunkToWriter() throws Exception {
+        mBackupWriter.writeChunk(0, 10);
+
+        verify(mDiffScriptWriter).writeChunk(0, 10);
+    }
+
+    @Test
+    public void getBytesWritten_returnsTotalSum() throws Exception {
+        mBackupWriter.writeBytes(TEST_BYTES);
+        mBackupWriter.writeBytes(TEST_BYTES);
+        mBackupWriter.writeChunk(/*start=*/ 0, /*length=*/ 10);
+
+        long bytesWritten = mBackupWriter.getBytesWritten();
+
+        assertThat(bytesWritten).isEqualTo(2 * TEST_BYTES.length + 10);
+    }
+
+    @Test
+    public void flush_flushesWriter() throws IOException {
+        mBackupWriter.flush();
+
+        verify(mDiffScriptWriter).flush();
+    }
+}

services/robotests/src/com/android/server/backup/encryption/chunking/SingleStreamDiffScriptWriterTest.java

@@ -0,0 +1,128 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.server.backup.encryption.chunking;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.testng.Assert.assertThrows;
+
+import android.platform.test.annotations.Presubmit;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.robolectric.RobolectricTestRunner;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.Locale;
+
+/** Tests for {@link SingleStreamDiffScriptWriter}. */
+@RunWith(RobolectricTestRunner.class)
+@Presubmit
+public class SingleStreamDiffScriptWriterTest {
+    private static final int MAX_CHUNK_SIZE_IN_BYTES = 256;
+    /** By default this Locale does not use Arabic numbers for %d formatting. */
+    private static final Locale HINDI = new Locale("hi", "IN");
+
+    private Locale mDefaultLocale;
+    private ByteArrayOutputStream mOutputStream;
+    private SingleStreamDiffScriptWriter mDiffScriptWriter;
+
+    @Before
+    public void setUp() {
+        mDefaultLocale = Locale.getDefault();
+        mOutputStream = new ByteArrayOutputStream();
+        mDiffScriptWriter =
+                new SingleStreamDiffScriptWriter(mOutputStream, MAX_CHUNK_SIZE_IN_BYTES);
+    }
+
+    @After
+    public void tearDown() {
+        Locale.setDefault(mDefaultLocale);
+    }
+
+    @Test
+    public void writeChunk_withNegativeStart_throwsException() {
+        assertThrows(
+                IllegalArgumentException.class,
+                () -> mDiffScriptWriter.writeChunk(-1, 50));
+    }
+
+    @Test
+    public void writeChunk_withZeroLength_throwsException() {
+        assertThrows(
+                IllegalArgumentException.class,
+                () -> mDiffScriptWriter.writeChunk(0, 0));
+    }
+
+    @Test
+    public void writeChunk_withExistingBytesInBuffer_writesBufferFirst()
+            throws IOException {
+        String testString = "abcd";
+        writeStringAsBytesToWriter(testString, mDiffScriptWriter);
+
+        mDiffScriptWriter.writeChunk(0, 20);
+        mDiffScriptWriter.flush();
+
+        // Expected format: length of abcd, newline, abcd, newline, chunk start - chunk end
+        assertThat(mOutputStream.toString("UTF-8")).isEqualTo(
+                String.format("%d\n%s\n%d-%d\n", testString.length(), testString, 0, 19));
+    }
+
+    @Test
+    public void writeChunk_overlappingPreviousChunk_combinesChunks() throws IOException {
+        mDiffScriptWriter.writeChunk(3, 4);
+
+        mDiffScriptWriter.writeChunk(7, 5);
+        mDiffScriptWriter.flush();
+
+        assertThat(mOutputStream.toString("UTF-8")).isEqualTo(String.format("3-11\n"));
+    }
+
+    @Test
+    public void writeChunk_formatsByteIndexesUsingArabicNumbers() throws Exception {
+        Locale.setDefault(HINDI);
+
+        mDiffScriptWriter.writeChunk(0, 12345);
+        mDiffScriptWriter.flush();
+
+        assertThat(mOutputStream.toString("UTF-8")).isEqualTo("0-12344\n");
+    }
+
+    @Test
+    public void flush_flushesOutputStream() throws IOException {
+        ByteArrayOutputStream mockOutputStream = mock(ByteArrayOutputStream.class);
+        SingleStreamDiffScriptWriter diffScriptWriter =
+                new SingleStreamDiffScriptWriter(mockOutputStream, MAX_CHUNK_SIZE_IN_BYTES);
+
+        diffScriptWriter.flush();
+
+        verify(mockOutputStream).flush();
+    }
+
+    private void writeStringAsBytesToWriter(String string, SingleStreamDiffScriptWriter writer)
+            throws IOException {
+        byte[] bytes = string.getBytes("UTF-8");
+        for (int i = 0; i < bytes.length; i++) {
+            writer.writeByte(bytes[i]);
+        }
+    }
+}