Add deduplicate_entry_values mode to TableFlattener.

This CL introduces DeduplicateItemsResEntryWriter which stores unique
pairs of entry and value and deduplicates a new pair based of this data
before writing it into the buffer.

Bug: b/249793372
Test: ResEntriesWriter_test, TableFlattener_test
Change-Id: I938a3425a87bfad9853e59193680de2050a95bd2

tools/aapt2/format/binary/ResEntryWriter.cpp

@@ -222,19 +222,57 @@ int32_t WriteMapToBuffer(const FlatEntry* map_entry, BigBuffer* buffer) {
   return offset;
 }
 
+void WriteItemToPair(const FlatEntry* item_entry, ResEntryValuePair* out_pair) {
+  static_assert(sizeof(ResEntryValuePair) == sizeof(ResTable_entry) + sizeof(Res_value),
+                "ResEntryValuePair must not have padding between entry and value.");
+
+  WriteEntry<ResTable_entry>(item_entry, &out_pair->entry);
+
+  CHECK(ValueCast<Item>(item_entry->value)->Flatten(&out_pair->value)) << "flatten failed";
+  out_pair->value.size = android::util::HostToDevice16(sizeof(out_pair->value));
+}
+
 int32_t SequentialResEntryWriter::WriteMap(const FlatEntry* entry) {
   return WriteMapToBuffer(entry, entries_buffer_);
 }
 
 int32_t SequentialResEntryWriter::WriteItem(const FlatEntry* entry) {
   int32_t offset = entries_buffer_->size();
-  ResTable_entry* out_entry = entries_buffer_->NextBlock<ResTable_entry>();
-  Res_value* out_value = entries_buffer_->NextBlock<Res_value>();
-  out_value->size = android::util::HostToDevice16(sizeof(*out_value));
-
-  WriteEntry<ResTable_entry>(entry, out_entry);
-  CHECK(ValueCast<Item>(entry->value)->Flatten(out_value)) << "flatten failed";
+  auto* out_pair = entries_buffer_->NextBlock<ResEntryValuePair>();
+  WriteItemToPair(entry, out_pair);
   return offset;
 }
 
+std::size_t ResEntryValuePairContentHasher::operator()(const ResEntryValuePairRef& ref) const {
+  return android::JenkinsHashMixBytes(0, ref.ptr, sizeof(ResEntryValuePair));
+}
+
+bool ResEntryValuePairContentEqualTo::operator()(const ResEntryValuePairRef& a,
+                                                 const ResEntryValuePairRef& b) const {
+  return std::memcmp(a.ptr, b.ptr, sizeof(ResEntryValuePair)) == 0;
+}
+
+int32_t DeduplicateItemsResEntryWriter::WriteMap(const FlatEntry* entry) {
+  return WriteMapToBuffer(entry, entries_buffer_);
+}
+
+int32_t DeduplicateItemsResEntryWriter::WriteItem(const FlatEntry* entry) {
+  int32_t initial_offset = entries_buffer_->size();
+
+  auto* out_pair = entries_buffer_->NextBlock<ResEntryValuePair>();
+  WriteItemToPair(entry, out_pair);
+
+  auto ref = ResEntryValuePairRef{*out_pair};
+  auto [it, inserted] = entry_offsets.insert({ref, initial_offset});
+  if (inserted) {
+    // If inserted just return a new offset as this is a first time we store
+    // this entry.
+    return initial_offset;
+  }
+  // If not inserted this means that this is a duplicate, backup allocated block to the buffer
+  // and return offset of previously stored entry.
+  entries_buffer_->BackUp(sizeof(ResEntryValuePair));
+  return it->second;
+}
+
 }  // namespace aapt

tools/aapt2/format/binary/ResEntryWriter.h

@@ -17,10 +17,13 @@
 #ifndef AAPT_FORMAT_BINARY_RESENTRY_SERIALIZER_H
 #define AAPT_FORMAT_BINARY_RESENTRY_SERIALIZER_H
 
+#include <unordered_map>
+
 #include "ResourceTable.h"
 #include "ValueVisitor.h"
 #include "android-base/macros.h"
 #include "androidfw/BigBuffer.h"
+#include "androidfw/ResourceTypes.h"
 
 namespace aapt {
 
@@ -32,6 +35,35 @@ struct FlatEntry {
   uint32_t entry_key;
 };
 
+// Pair of ResTable_entry and Res_value. These pairs are stored sequentially in values buffer.
+// We introduce this structure for ResEntryWriter to a have single allocation using
+// BigBuffer::NextBlock which allows to return it back with BigBuffer::Backup.
+struct ResEntryValuePair {
+  android::ResTable_entry entry;
+  android::Res_value value;
+};
+
+// References ResEntryValuePair object stored in BigBuffer used as a key in std::unordered_map.
+// Allows access to memory address where ResEntryValuePair is stored.
+union ResEntryValuePairRef {
+  const std::reference_wrapper<const ResEntryValuePair> pair;
+  const u_char* ptr;
+
+  explicit ResEntryValuePairRef(const ResEntryValuePair& ref) : pair(ref) {
+  }
+};
+
+// Hasher which computes hash of ResEntryValuePair using its bytes representation in memory.
+struct ResEntryValuePairContentHasher {
+  std::size_t operator()(const ResEntryValuePairRef& ref) const;
+};
+
+// Equaler which compares ResEntryValuePairs using theirs bytes representation in memory.
+struct ResEntryValuePairContentEqualTo {
+  bool operator()(const ResEntryValuePairRef& a, const ResEntryValuePairRef& b) const;
+};
+
+// Base class that allows to write FlatEntries into entries_buffer.
 class ResEntryWriter {
  public:
   virtual ~ResEntryWriter() = default;
@@ -59,6 +91,8 @@ class ResEntryWriter {
   DISALLOW_COPY_AND_ASSIGN(ResEntryWriter);
 };
 
+// ResEntryWriter which writes FlatEntries sequentially into entries_buffer.
+// Next entry is always written right after previous one in the buffer.
 class SequentialResEntryWriter : public ResEntryWriter {
  public:
   explicit SequentialResEntryWriter(android::BigBuffer* entries_buffer)
@@ -74,6 +108,28 @@ class SequentialResEntryWriter : public ResEntryWriter {
   DISALLOW_COPY_AND_ASSIGN(SequentialResEntryWriter);
 };
 
+// ResEntryWriter that writes only unique entry and value pairs into entries_buffer.
+// Next entry is written into buffer only if there is no entry with the same bytes representation
+// in memory written before. Otherwise returns offset of already written entry.
+class DeduplicateItemsResEntryWriter : public ResEntryWriter {
+ public:
+  explicit DeduplicateItemsResEntryWriter(android::BigBuffer* entries_buffer)
+      : ResEntryWriter(entries_buffer) {
+  }
+  ~DeduplicateItemsResEntryWriter() override = default;
+
+  int32_t WriteItem(const FlatEntry* entry) override;
+
+  int32_t WriteMap(const FlatEntry* entry) override;
+
+ private:
+  DISALLOW_COPY_AND_ASSIGN(DeduplicateItemsResEntryWriter);
+
+  std::unordered_map<ResEntryValuePairRef, int32_t, ResEntryValuePairContentHasher,
+                     ResEntryValuePairContentEqualTo>
+      entry_offsets;
+};
+
 }  // namespace aapt
 
 #endif

tools/aapt2/format/binary/ResEntryWriter_test.cpp

@@ -0,0 +1,138 @@
+/*
+ * Copyright (C) 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "format/binary/ResEntryWriter.h"
+
+#include "androidfw/BigBuffer.h"
+#include "format/binary/ResourceTypeExtensions.h"
+#include "test/Test.h"
+#include "util/Util.h"
+
+using ::android::BigBuffer;
+using ::android::Res_value;
+using ::android::ResTable_map;
+using ::testing::Eq;
+using ::testing::Ge;
+using ::testing::IsNull;
+using ::testing::Ne;
+using ::testing::NotNull;
+
+namespace aapt {
+
+using SequentialResEntryWriterTest = CommandTestFixture;
+using DeduplicateItemsResEntryWriterTest = CommandTestFixture;
+
+std::vector<int32_t> WriteAllEntries(const ResourceTableView& table, ResEntryWriter& writer) {
+  std::vector<int32_t> result = {};
+  for (const auto& type : table.packages[0].types) {
+    for (const auto& entry : type.entries) {
+      for (const auto& value : entry.values) {
+        auto flat_entry = FlatEntry{&entry, value->value.get(), 0};
+        result.push_back(writer.Write(&flat_entry));
+      }
+    }
+  }
+  return result;
+}
+
+TEST_F(SequentialResEntryWriterTest, WriteEntriesOneByOne) {
+  std::unique_ptr<ResourceTable> table =
+      test::ResourceTableBuilder()
+          .AddSimple("com.app.test:id/id1", ResourceId(0x7f010000))
+          .AddSimple("com.app.test:id/id2", ResourceId(0x7f010001))
+          .AddSimple("com.app.test:id/id3", ResourceId(0x7f010002))
+          .Build();
+
+  BigBuffer out(512);
+  SequentialResEntryWriter writer(&out);
+  auto offsets = WriteAllEntries(table->GetPartitionedView(), writer);
+
+  std::vector<int32_t> expected_offsets{0, sizeof(ResEntryValuePair),
+                                        2 * sizeof(ResEntryValuePair)};
+  EXPECT_EQ(out.size(), 3 * sizeof(ResEntryValuePair));
+  EXPECT_EQ(offsets, expected_offsets);
+};
+
+TEST_F(SequentialResEntryWriterTest, WriteMapEntriesOneByOne) {
+  std::unique_ptr<Array> array1 = util::make_unique<Array>();
+  array1->elements.push_back(
+      util::make_unique<BinaryPrimitive>(uint8_t(Res_value::TYPE_INT_DEC), 1u));
+  array1->elements.push_back(
+      util::make_unique<BinaryPrimitive>(uint8_t(Res_value::TYPE_INT_DEC), 2u));
+  std::unique_ptr<Array> array2 = util::make_unique<Array>();
+  array2->elements.push_back(
+      util::make_unique<BinaryPrimitive>(uint8_t(Res_value::TYPE_INT_DEC), 1u));
+  array2->elements.push_back(
+      util::make_unique<BinaryPrimitive>(uint8_t(Res_value::TYPE_INT_DEC), 2u));
+
+  std::unique_ptr<ResourceTable> table = test::ResourceTableBuilder()
+                                             .AddValue("com.app.test:array/arr1", std::move(array1))
+                                             .AddValue("com.app.test:array/arr2", std::move(array2))
+                                             .Build();
+
+  BigBuffer out(512);
+  SequentialResEntryWriter writer(&out);
+  auto offsets = WriteAllEntries(table->GetPartitionedView(), writer);
+
+  std::vector<int32_t> expected_offsets{0, sizeof(ResTable_entry_ext) + 2 * sizeof(ResTable_map)};
+  EXPECT_EQ(out.size(), 2 * (sizeof(ResTable_entry_ext) + 2 * sizeof(ResTable_map)));
+  EXPECT_EQ(offsets, expected_offsets);
+};
+
+TEST_F(DeduplicateItemsResEntryWriterTest, DeduplicateItemEntries) {
+  std::unique_ptr<ResourceTable> table =
+      test::ResourceTableBuilder()
+          .AddSimple("com.app.test:id/id1", ResourceId(0x7f010000))
+          .AddSimple("com.app.test:id/id2", ResourceId(0x7f010001))
+          .AddSimple("com.app.test:id/id3", ResourceId(0x7f010002))
+          .Build();
+
+  BigBuffer out(512);
+  DeduplicateItemsResEntryWriter writer(&out);
+  auto offsets = WriteAllEntries(table->GetPartitionedView(), writer);
+
+  std::vector<int32_t> expected_offsets{0, 0, 0};
+  EXPECT_EQ(out.size(), sizeof(ResEntryValuePair));
+  EXPECT_EQ(offsets, expected_offsets);
+};
+
+TEST_F(DeduplicateItemsResEntryWriterTest, WriteMapEntriesOneByOne) {
+  std::unique_ptr<Array> array1 = util::make_unique<Array>();
+  array1->elements.push_back(
+      util::make_unique<BinaryPrimitive>(uint8_t(Res_value::TYPE_INT_DEC), 1u));
+  array1->elements.push_back(
+      util::make_unique<BinaryPrimitive>(uint8_t(Res_value::TYPE_INT_DEC), 2u));
+  std::unique_ptr<Array> array2 = util::make_unique<Array>();
+  array2->elements.push_back(
+      util::make_unique<BinaryPrimitive>(uint8_t(Res_value::TYPE_INT_DEC), 1u));
+  array2->elements.push_back(
+      util::make_unique<BinaryPrimitive>(uint8_t(Res_value::TYPE_INT_DEC), 2u));
+
+  std::unique_ptr<ResourceTable> table = test::ResourceTableBuilder()
+                                             .AddValue("com.app.test:array/arr1", std::move(array1))
+                                             .AddValue("com.app.test:array/arr2", std::move(array2))
+                                             .Build();
+
+  BigBuffer out(512);
+  DeduplicateItemsResEntryWriter writer(&out);
+  auto offsets = WriteAllEntries(table->GetPartitionedView(), writer);
+
+  std::vector<int32_t> expected_offsets{0, sizeof(ResTable_entry_ext) + 2 * sizeof(ResTable_map)};
+  EXPECT_EQ(out.size(), 2 * (sizeof(ResTable_entry_ext) + 2 * sizeof(ResTable_map)));
+  EXPECT_EQ(offsets, expected_offsets);
+};
+
+}  // namespace aapt

tools/aapt2/format/binary/TableFlattener.cpp

@@ -16,15 +16,13 @@
 
 #include "format/binary/TableFlattener.h"
 
-#include <algorithm>
-#include <numeric>
 #include <sstream>
 #include <type_traits>
+#include <variant>
 
 #include "ResourceTable.h"
 #include "ResourceValues.h"
 #include "SdkConstants.h"
-#include "ValueVisitor.h"
 #include "android-base/logging.h"
 #include "android-base/macros.h"
 #include "android-base/stringprintf.h"
@@ -70,14 +68,16 @@ class PackageFlattener {
   PackageFlattener(IAaptContext* context, const ResourceTablePackageView& package,
                    const std::map<size_t, std::string>* shared_libs,
                    SparseEntriesMode sparse_entries, bool collapse_key_stringpool,
-                   const std::set<ResourceName>& name_collapse_exemptions)
+                   const std::set<ResourceName>& name_collapse_exemptions,
+                   bool deduplicate_entry_values)
       : context_(context),
         diag_(context->GetDiagnostics()),
         package_(package),
         shared_libs_(shared_libs),
         sparse_entries_(sparse_entries),
         collapse_key_stringpool_(collapse_key_stringpool),
-        name_collapse_exemptions_(name_collapse_exemptions) {
+        name_collapse_exemptions_(name_collapse_exemptions),
+        deduplicate_entry_values_(deduplicate_entry_values) {
   }
 
   bool FlattenPackage(BigBuffer* buffer) {
@@ -151,10 +151,18 @@ class PackageFlattener {
     offsets.resize(num_total_entries, 0xffffffffu);
 
     android::BigBuffer values_buffer(512);
-    SequentialResEntryWriter res_entry_writer(&values_buffer);
+    std::variant<std::monostate, DeduplicateItemsResEntryWriter, SequentialResEntryWriter>
+        writer_variant;
+    ResEntryWriter* res_entry_writer;
+    if (deduplicate_entry_values_) {
+      res_entry_writer = &writer_variant.emplace<DeduplicateItemsResEntryWriter>(&values_buffer);
+    } else {
+      res_entry_writer = &writer_variant.emplace<SequentialResEntryWriter>(&values_buffer);
+    }
+
     for (FlatEntry& flat_entry : *entries) {
       CHECK(static_cast<size_t>(flat_entry.entry->id.value()) < num_total_entries);
-      offsets[flat_entry.entry->id.value()] = res_entry_writer.Write(&flat_entry);
+      offsets[flat_entry.entry->id.value()] = res_entry_writer->Write(&flat_entry);
     }
 
     bool sparse_encode = sparse_entries_ == SparseEntriesMode::Enabled ||
@@ -510,6 +518,7 @@ class PackageFlattener {
   bool collapse_key_stringpool_;
   const std::set<ResourceName>& name_collapse_exemptions_;
   std::map<uint32_t, uint32_t> aliases_;
+  bool deduplicate_entry_values_;
 };
 
 }  // namespace
@@ -561,7 +570,8 @@ bool TableFlattener::Consume(IAaptContext* context, ResourceTable* table) {
 
     PackageFlattener flattener(context, package, &table->included_packages_,
                                options_.sparse_entries, options_.collapse_key_stringpool,
-                               options_.name_collapse_exemptions);
+                               options_.name_collapse_exemptions,
+                               options_.deduplicate_entry_values);
     if (!flattener.FlattenPackage(&package_buffer)) {
       return false;
     }

tools/aapt2/format/binary/TableFlattener.h

@@ -54,6 +54,20 @@ struct TableFlattenerOptions {
 
   // Map from original resource paths to shortened resource paths.
   std::map<std::string, std::string> shortened_path_map;
+
+  // When enabled, only unique pairs of entry and value are stored in type chunks.
+  //
+  // By default, all such pairs are unique because a reference to resource name in the string pool
+  // is a part of the pair. But when resource names are collapsed (using 'collapse_key_stringpool'
+  // flag or manually) the same data might be duplicated multiple times in the same type chunk.
+  //
+  // For example: an application has 3 boolean resources with collapsed names and 3 'true' values
+  // are defined for these resources in 'default' configuration. All pairs of entry and value for
+  // these resources will have the same binary representation and stored only once in type chunk
+  // instead of three times when this flag is disabled.
+  //
+  // This applies only to simple entries (entry->flags & ResTable_entry::FLAG_COMPLEX == 0).
+  bool deduplicate_entry_values = false;
 };
 
 class TableFlattener : public IResourceTableConsumer {

tools/aapt2/format/binary/TableFlattener_test.cpp

@@ -669,6 +669,87 @@ TEST_F(TableFlattenerTest, ObfuscatingResourceNamesNoNameCollapseExemptionsSucce
                      ResourceId(0x7f050000), {}, Res_value::TYPE_STRING, (uint32_t)*idx, 0u));
 }
 
+TEST_F(TableFlattenerTest, ObfuscatingResourceNamesWithDeduplicationSucceeds) {
+  std::unique_ptr<ResourceTable> table =
+      test::ResourceTableBuilder()
+          .AddSimple("com.app.test:id/one", ResourceId(0x7f020000))
+          .AddSimple("com.app.test:id/two", ResourceId(0x7f020001))
+          .AddValue("com.app.test:id/three", ResourceId(0x7f020002),
+                    test::BuildReference("com.app.test:id/one", ResourceId(0x7f020000)))
+          .AddValue("com.app.test:integer/one", ResourceId(0x7f030000),
+                    util::make_unique<BinaryPrimitive>(uint8_t(Res_value::TYPE_INT_DEC), 1u))
+          .AddValue("com.app.test:integer/one", test::ParseConfigOrDie("v1"),
+                    ResourceId(0x7f030000),
+                    util::make_unique<BinaryPrimitive>(uint8_t(Res_value::TYPE_INT_DEC), 2u))
+          .AddString("com.app.test:string/test1", ResourceId(0x7f040000), "foo")
+          .AddString("com.app.test:string/test2", ResourceId(0x7f040001), "foo")
+          .AddString("com.app.test:string/test3", ResourceId(0x7f040002), "bar")
+          .AddString("com.app.test:string/test4", ResourceId(0x7f040003), "foo")
+          .AddString("com.app.test:layout/bar1", ResourceId(0x7f050000), "res/layout/bar.xml")
+          .AddString("com.app.test:layout/bar2", ResourceId(0x7f050001), "res/layout/bar.xml")
+          .Build();
+
+  TableFlattenerOptions options;
+  options.collapse_key_stringpool = true;
+  options.deduplicate_entry_values = true;
+
+  ResTable res_table;
+
+  ASSERT_TRUE(Flatten(context_.get(), options, table.get(), &res_table));
+
+  EXPECT_TRUE(Exists(&res_table, "com.app.test:id/0_resource_name_obfuscated",
+                     ResourceId(0x7f020000), {}, Res_value::TYPE_INT_BOOLEAN, 0u, 0u));
+
+  EXPECT_TRUE(Exists(&res_table, "com.app.test:id/0_resource_name_obfuscated",
+                     ResourceId(0x7f020001), {}, Res_value::TYPE_INT_BOOLEAN, 0u, 0u));
+
+  EXPECT_TRUE(Exists(&res_table, "com.app.test:id/0_resource_name_obfuscated",
+                     ResourceId(0x7f020002), {}, Res_value::TYPE_REFERENCE, 0x7f020000u, 0u));
+
+  EXPECT_TRUE(Exists(&res_table, "com.app.test:integer/0_resource_name_obfuscated",
+                     ResourceId(0x7f030000), {}, Res_value::TYPE_INT_DEC, 1u,
+                     ResTable_config::CONFIG_VERSION));
+
+  EXPECT_TRUE(Exists(&res_table, "com.app.test:integer/0_resource_name_obfuscated",
+                     ResourceId(0x7f030000), test::ParseConfigOrDie("v1"), Res_value::TYPE_INT_DEC,
+                     2u, ResTable_config::CONFIG_VERSION));
+
+  std::u16string foo_str = u"foo";
+  std::u16string bar_str = u"bar";
+  auto foo_idx = res_table.getTableStringBlock(0)->indexOfString(foo_str.data(), foo_str.size());
+  auto bar_idx = res_table.getTableStringBlock(0)->indexOfString(bar_str.data(), bar_str.size());
+  ASSERT_TRUE(foo_idx.has_value());
+  EXPECT_TRUE(Exists(&res_table, "com.app.test:string/0_resource_name_obfuscated",
+                     ResourceId(0x7f040000), {}, Res_value::TYPE_STRING, (uint32_t)*foo_idx, 0u));
+  EXPECT_TRUE(Exists(&res_table, "com.app.test:string/0_resource_name_obfuscated",
+                     ResourceId(0x7f040001), {}, Res_value::TYPE_STRING, (uint32_t)*foo_idx, 0u));
+  EXPECT_TRUE(Exists(&res_table, "com.app.test:string/0_resource_name_obfuscated",
+                     ResourceId(0x7f040002), {}, Res_value::TYPE_STRING, (uint32_t)*bar_idx, 0u));
+  EXPECT_TRUE(Exists(&res_table, "com.app.test:string/0_resource_name_obfuscated",
+                     ResourceId(0x7f040003), {}, Res_value::TYPE_STRING, (uint32_t)*foo_idx, 0u));
+
+  std::u16string bar_path = u"res/layout/bar.xml";
+  auto bar_path_idx =
+      res_table.getTableStringBlock(0)->indexOfString(bar_path.data(), bar_path.size());
+  ASSERT_TRUE(bar_path_idx.has_value());
+  EXPECT_TRUE(Exists(&res_table, "com.app.test:layout/0_resource_name_obfuscated",
+                     ResourceId(0x7f050000), {}, Res_value::TYPE_STRING, (uint32_t)*bar_path_idx,
+                     0u));
+  EXPECT_TRUE(Exists(&res_table, "com.app.test:layout/0_resource_name_obfuscated",
+                     ResourceId(0x7f050001), {}, Res_value::TYPE_STRING, (uint32_t)*bar_path_idx,
+                     0u));
+
+  std::string deduplicated_output;
+  std::string sequential_output;
+  Flatten(context_.get(), options, table.get(), &deduplicated_output);
+  options.deduplicate_entry_values = false;
+  Flatten(context_.get(), options, table.get(), &sequential_output);
+
+  // We have 4 duplicates: 0x7f020001 id, 0x7f040001 string, 0x7f040003 string, 0x7f050001 layout.
+  EXPECT_EQ(sequential_output.size(),
+            deduplicated_output.size() + 4 * (sizeof(ResTable_entry) + sizeof(Res_value)));
+}
+
 TEST_F(TableFlattenerTest, ObfuscatingResourceNamesWithNameCollapseExemptionsSucceeds) {
   std::unique_ptr<ResourceTable> table =
       test::ResourceTableBuilder()