From 8b33e7adb86de38321620b8fd2d5491a10d70b4e Mon Sep 17 00:00:00 2001 From: Iurii Makhno Date: Fri, 30 Sep 2022 19:30:46 +0000 Subject: [PATCH] Add deduplicate_entry_values mode to TableFlattener. This CL introduces DeduplicateItemsResEntryWriter which stores unique pairs of entry and value and deduplicates a new pair based of this data before writing it into the buffer. Bug: b/249793372 Test: ResEntriesWriter_test, TableFlattener_test Change-Id: I938a3425a87bfad9853e59193680de2050a95bd2 --- tools/aapt2/format/binary/ResEntryWriter.cpp | 50 ++++++- tools/aapt2/format/binary/ResEntryWriter.h | 56 +++++++ .../format/binary/ResEntryWriter_test.cpp | 138 ++++++++++++++++++ tools/aapt2/format/binary/TableFlattener.cpp | 26 +++- tools/aapt2/format/binary/TableFlattener.h | 14 ++ .../format/binary/TableFlattener_test.cpp | 81 ++++++++++ 6 files changed, 351 insertions(+), 14 deletions(-) create mode 100644 tools/aapt2/format/binary/ResEntryWriter_test.cpp diff --git a/tools/aapt2/format/binary/ResEntryWriter.cpp b/tools/aapt2/format/binary/ResEntryWriter.cpp index 7b64619be2ba8..8832c24842eea 100644 --- a/tools/aapt2/format/binary/ResEntryWriter.cpp +++ b/tools/aapt2/format/binary/ResEntryWriter.cpp @@ -222,19 +222,57 @@ int32_t WriteMapToBuffer(const FlatEntry* map_entry, BigBuffer* buffer) { return offset; } +void WriteItemToPair(const FlatEntry* item_entry, ResEntryValuePair* out_pair) { + static_assert(sizeof(ResEntryValuePair) == sizeof(ResTable_entry) + sizeof(Res_value), + "ResEntryValuePair must not have padding between entry and value."); + + WriteEntry(item_entry, &out_pair->entry); + + CHECK(ValueCast(item_entry->value)->Flatten(&out_pair->value)) << "flatten failed"; + out_pair->value.size = android::util::HostToDevice16(sizeof(out_pair->value)); +} + int32_t SequentialResEntryWriter::WriteMap(const FlatEntry* entry) { return WriteMapToBuffer(entry, entries_buffer_); } int32_t SequentialResEntryWriter::WriteItem(const FlatEntry* entry) { int32_t offset = entries_buffer_->size(); - ResTable_entry* out_entry = entries_buffer_->NextBlock(); - Res_value* out_value = entries_buffer_->NextBlock(); - out_value->size = android::util::HostToDevice16(sizeof(*out_value)); - - WriteEntry(entry, out_entry); - CHECK(ValueCast(entry->value)->Flatten(out_value)) << "flatten failed"; + auto* out_pair = entries_buffer_->NextBlock(); + WriteItemToPair(entry, out_pair); return offset; } +std::size_t ResEntryValuePairContentHasher::operator()(const ResEntryValuePairRef& ref) const { + return android::JenkinsHashMixBytes(0, ref.ptr, sizeof(ResEntryValuePair)); +} + +bool ResEntryValuePairContentEqualTo::operator()(const ResEntryValuePairRef& a, + const ResEntryValuePairRef& b) const { + return std::memcmp(a.ptr, b.ptr, sizeof(ResEntryValuePair)) == 0; +} + +int32_t DeduplicateItemsResEntryWriter::WriteMap(const FlatEntry* entry) { + return WriteMapToBuffer(entry, entries_buffer_); +} + +int32_t DeduplicateItemsResEntryWriter::WriteItem(const FlatEntry* entry) { + int32_t initial_offset = entries_buffer_->size(); + + auto* out_pair = entries_buffer_->NextBlock(); + WriteItemToPair(entry, out_pair); + + auto ref = ResEntryValuePairRef{*out_pair}; + auto [it, inserted] = entry_offsets.insert({ref, initial_offset}); + if (inserted) { + // If inserted just return a new offset as this is a first time we store + // this entry. + return initial_offset; + } + // If not inserted this means that this is a duplicate, backup allocated block to the buffer + // and return offset of previously stored entry. + entries_buffer_->BackUp(sizeof(ResEntryValuePair)); + return it->second; +} + } // namespace aapt \ No newline at end of file diff --git a/tools/aapt2/format/binary/ResEntryWriter.h b/tools/aapt2/format/binary/ResEntryWriter.h index 10bd3125091b6..a36ceec2613b4 100644 --- a/tools/aapt2/format/binary/ResEntryWriter.h +++ b/tools/aapt2/format/binary/ResEntryWriter.h @@ -17,10 +17,13 @@ #ifndef AAPT_FORMAT_BINARY_RESENTRY_SERIALIZER_H #define AAPT_FORMAT_BINARY_RESENTRY_SERIALIZER_H +#include + #include "ResourceTable.h" #include "ValueVisitor.h" #include "android-base/macros.h" #include "androidfw/BigBuffer.h" +#include "androidfw/ResourceTypes.h" namespace aapt { @@ -32,6 +35,35 @@ struct FlatEntry { uint32_t entry_key; }; +// Pair of ResTable_entry and Res_value. These pairs are stored sequentially in values buffer. +// We introduce this structure for ResEntryWriter to a have single allocation using +// BigBuffer::NextBlock which allows to return it back with BigBuffer::Backup. +struct ResEntryValuePair { + android::ResTable_entry entry; + android::Res_value value; +}; + +// References ResEntryValuePair object stored in BigBuffer used as a key in std::unordered_map. +// Allows access to memory address where ResEntryValuePair is stored. +union ResEntryValuePairRef { + const std::reference_wrapper pair; + const u_char* ptr; + + explicit ResEntryValuePairRef(const ResEntryValuePair& ref) : pair(ref) { + } +}; + +// Hasher which computes hash of ResEntryValuePair using its bytes representation in memory. +struct ResEntryValuePairContentHasher { + std::size_t operator()(const ResEntryValuePairRef& ref) const; +}; + +// Equaler which compares ResEntryValuePairs using theirs bytes representation in memory. +struct ResEntryValuePairContentEqualTo { + bool operator()(const ResEntryValuePairRef& a, const ResEntryValuePairRef& b) const; +}; + +// Base class that allows to write FlatEntries into entries_buffer. class ResEntryWriter { public: virtual ~ResEntryWriter() = default; @@ -59,6 +91,8 @@ class ResEntryWriter { DISALLOW_COPY_AND_ASSIGN(ResEntryWriter); }; +// ResEntryWriter which writes FlatEntries sequentially into entries_buffer. +// Next entry is always written right after previous one in the buffer. class SequentialResEntryWriter : public ResEntryWriter { public: explicit SequentialResEntryWriter(android::BigBuffer* entries_buffer) @@ -74,6 +108,28 @@ class SequentialResEntryWriter : public ResEntryWriter { DISALLOW_COPY_AND_ASSIGN(SequentialResEntryWriter); }; +// ResEntryWriter that writes only unique entry and value pairs into entries_buffer. +// Next entry is written into buffer only if there is no entry with the same bytes representation +// in memory written before. Otherwise returns offset of already written entry. +class DeduplicateItemsResEntryWriter : public ResEntryWriter { + public: + explicit DeduplicateItemsResEntryWriter(android::BigBuffer* entries_buffer) + : ResEntryWriter(entries_buffer) { + } + ~DeduplicateItemsResEntryWriter() override = default; + + int32_t WriteItem(const FlatEntry* entry) override; + + int32_t WriteMap(const FlatEntry* entry) override; + + private: + DISALLOW_COPY_AND_ASSIGN(DeduplicateItemsResEntryWriter); + + std::unordered_map + entry_offsets; +}; + } // namespace aapt #endif \ No newline at end of file diff --git a/tools/aapt2/format/binary/ResEntryWriter_test.cpp b/tools/aapt2/format/binary/ResEntryWriter_test.cpp new file mode 100644 index 0000000000000..56ca1332ec5da --- /dev/null +++ b/tools/aapt2/format/binary/ResEntryWriter_test.cpp @@ -0,0 +1,138 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "format/binary/ResEntryWriter.h" + +#include "androidfw/BigBuffer.h" +#include "format/binary/ResourceTypeExtensions.h" +#include "test/Test.h" +#include "util/Util.h" + +using ::android::BigBuffer; +using ::android::Res_value; +using ::android::ResTable_map; +using ::testing::Eq; +using ::testing::Ge; +using ::testing::IsNull; +using ::testing::Ne; +using ::testing::NotNull; + +namespace aapt { + +using SequentialResEntryWriterTest = CommandTestFixture; +using DeduplicateItemsResEntryWriterTest = CommandTestFixture; + +std::vector WriteAllEntries(const ResourceTableView& table, ResEntryWriter& writer) { + std::vector result = {}; + for (const auto& type : table.packages[0].types) { + for (const auto& entry : type.entries) { + for (const auto& value : entry.values) { + auto flat_entry = FlatEntry{&entry, value->value.get(), 0}; + result.push_back(writer.Write(&flat_entry)); + } + } + } + return result; +} + +TEST_F(SequentialResEntryWriterTest, WriteEntriesOneByOne) { + std::unique_ptr table = + test::ResourceTableBuilder() + .AddSimple("com.app.test:id/id1", ResourceId(0x7f010000)) + .AddSimple("com.app.test:id/id2", ResourceId(0x7f010001)) + .AddSimple("com.app.test:id/id3", ResourceId(0x7f010002)) + .Build(); + + BigBuffer out(512); + SequentialResEntryWriter writer(&out); + auto offsets = WriteAllEntries(table->GetPartitionedView(), writer); + + std::vector expected_offsets{0, sizeof(ResEntryValuePair), + 2 * sizeof(ResEntryValuePair)}; + EXPECT_EQ(out.size(), 3 * sizeof(ResEntryValuePair)); + EXPECT_EQ(offsets, expected_offsets); +}; + +TEST_F(SequentialResEntryWriterTest, WriteMapEntriesOneByOne) { + std::unique_ptr array1 = util::make_unique(); + array1->elements.push_back( + util::make_unique(uint8_t(Res_value::TYPE_INT_DEC), 1u)); + array1->elements.push_back( + util::make_unique(uint8_t(Res_value::TYPE_INT_DEC), 2u)); + std::unique_ptr array2 = util::make_unique(); + array2->elements.push_back( + util::make_unique(uint8_t(Res_value::TYPE_INT_DEC), 1u)); + array2->elements.push_back( + util::make_unique(uint8_t(Res_value::TYPE_INT_DEC), 2u)); + + std::unique_ptr table = test::ResourceTableBuilder() + .AddValue("com.app.test:array/arr1", std::move(array1)) + .AddValue("com.app.test:array/arr2", std::move(array2)) + .Build(); + + BigBuffer out(512); + SequentialResEntryWriter writer(&out); + auto offsets = WriteAllEntries(table->GetPartitionedView(), writer); + + std::vector expected_offsets{0, sizeof(ResTable_entry_ext) + 2 * sizeof(ResTable_map)}; + EXPECT_EQ(out.size(), 2 * (sizeof(ResTable_entry_ext) + 2 * sizeof(ResTable_map))); + EXPECT_EQ(offsets, expected_offsets); +}; + +TEST_F(DeduplicateItemsResEntryWriterTest, DeduplicateItemEntries) { + std::unique_ptr table = + test::ResourceTableBuilder() + .AddSimple("com.app.test:id/id1", ResourceId(0x7f010000)) + .AddSimple("com.app.test:id/id2", ResourceId(0x7f010001)) + .AddSimple("com.app.test:id/id3", ResourceId(0x7f010002)) + .Build(); + + BigBuffer out(512); + DeduplicateItemsResEntryWriter writer(&out); + auto offsets = WriteAllEntries(table->GetPartitionedView(), writer); + + std::vector expected_offsets{0, 0, 0}; + EXPECT_EQ(out.size(), sizeof(ResEntryValuePair)); + EXPECT_EQ(offsets, expected_offsets); +}; + +TEST_F(DeduplicateItemsResEntryWriterTest, WriteMapEntriesOneByOne) { + std::unique_ptr array1 = util::make_unique(); + array1->elements.push_back( + util::make_unique(uint8_t(Res_value::TYPE_INT_DEC), 1u)); + array1->elements.push_back( + util::make_unique(uint8_t(Res_value::TYPE_INT_DEC), 2u)); + std::unique_ptr array2 = util::make_unique(); + array2->elements.push_back( + util::make_unique(uint8_t(Res_value::TYPE_INT_DEC), 1u)); + array2->elements.push_back( + util::make_unique(uint8_t(Res_value::TYPE_INT_DEC), 2u)); + + std::unique_ptr table = test::ResourceTableBuilder() + .AddValue("com.app.test:array/arr1", std::move(array1)) + .AddValue("com.app.test:array/arr2", std::move(array2)) + .Build(); + + BigBuffer out(512); + DeduplicateItemsResEntryWriter writer(&out); + auto offsets = WriteAllEntries(table->GetPartitionedView(), writer); + + std::vector expected_offsets{0, sizeof(ResTable_entry_ext) + 2 * sizeof(ResTable_map)}; + EXPECT_EQ(out.size(), 2 * (sizeof(ResTable_entry_ext) + 2 * sizeof(ResTable_map))); + EXPECT_EQ(offsets, expected_offsets); +}; + +} // namespace aapt \ No newline at end of file diff --git a/tools/aapt2/format/binary/TableFlattener.cpp b/tools/aapt2/format/binary/TableFlattener.cpp index bb3d034b71a32..7dc9d26f91083 100644 --- a/tools/aapt2/format/binary/TableFlattener.cpp +++ b/tools/aapt2/format/binary/TableFlattener.cpp @@ -16,15 +16,13 @@ #include "format/binary/TableFlattener.h" -#include -#include #include #include +#include #include "ResourceTable.h" #include "ResourceValues.h" #include "SdkConstants.h" -#include "ValueVisitor.h" #include "android-base/logging.h" #include "android-base/macros.h" #include "android-base/stringprintf.h" @@ -70,14 +68,16 @@ class PackageFlattener { PackageFlattener(IAaptContext* context, const ResourceTablePackageView& package, const std::map* shared_libs, SparseEntriesMode sparse_entries, bool collapse_key_stringpool, - const std::set& name_collapse_exemptions) + const std::set& name_collapse_exemptions, + bool deduplicate_entry_values) : context_(context), diag_(context->GetDiagnostics()), package_(package), shared_libs_(shared_libs), sparse_entries_(sparse_entries), collapse_key_stringpool_(collapse_key_stringpool), - name_collapse_exemptions_(name_collapse_exemptions) { + name_collapse_exemptions_(name_collapse_exemptions), + deduplicate_entry_values_(deduplicate_entry_values) { } bool FlattenPackage(BigBuffer* buffer) { @@ -151,10 +151,18 @@ class PackageFlattener { offsets.resize(num_total_entries, 0xffffffffu); android::BigBuffer values_buffer(512); - SequentialResEntryWriter res_entry_writer(&values_buffer); + std::variant + writer_variant; + ResEntryWriter* res_entry_writer; + if (deduplicate_entry_values_) { + res_entry_writer = &writer_variant.emplace(&values_buffer); + } else { + res_entry_writer = &writer_variant.emplace(&values_buffer); + } + for (FlatEntry& flat_entry : *entries) { CHECK(static_cast(flat_entry.entry->id.value()) < num_total_entries); - offsets[flat_entry.entry->id.value()] = res_entry_writer.Write(&flat_entry); + offsets[flat_entry.entry->id.value()] = res_entry_writer->Write(&flat_entry); } bool sparse_encode = sparse_entries_ == SparseEntriesMode::Enabled || @@ -510,6 +518,7 @@ class PackageFlattener { bool collapse_key_stringpool_; const std::set& name_collapse_exemptions_; std::map aliases_; + bool deduplicate_entry_values_; }; } // namespace @@ -561,7 +570,8 @@ bool TableFlattener::Consume(IAaptContext* context, ResourceTable* table) { PackageFlattener flattener(context, package, &table->included_packages_, options_.sparse_entries, options_.collapse_key_stringpool, - options_.name_collapse_exemptions); + options_.name_collapse_exemptions, + options_.deduplicate_entry_values); if (!flattener.FlattenPackage(&package_buffer)) { return false; } diff --git a/tools/aapt2/format/binary/TableFlattener.h b/tools/aapt2/format/binary/TableFlattener.h index c6d30335ed53d..6151b7e33b3fc 100644 --- a/tools/aapt2/format/binary/TableFlattener.h +++ b/tools/aapt2/format/binary/TableFlattener.h @@ -54,6 +54,20 @@ struct TableFlattenerOptions { // Map from original resource paths to shortened resource paths. std::map shortened_path_map; + + // When enabled, only unique pairs of entry and value are stored in type chunks. + // + // By default, all such pairs are unique because a reference to resource name in the string pool + // is a part of the pair. But when resource names are collapsed (using 'collapse_key_stringpool' + // flag or manually) the same data might be duplicated multiple times in the same type chunk. + // + // For example: an application has 3 boolean resources with collapsed names and 3 'true' values + // are defined for these resources in 'default' configuration. All pairs of entry and value for + // these resources will have the same binary representation and stored only once in type chunk + // instead of three times when this flag is disabled. + // + // This applies only to simple entries (entry->flags & ResTable_entry::FLAG_COMPLEX == 0). + bool deduplicate_entry_values = false; }; class TableFlattener : public IResourceTableConsumer { diff --git a/tools/aapt2/format/binary/TableFlattener_test.cpp b/tools/aapt2/format/binary/TableFlattener_test.cpp index b69daddf1c7ae..2097a6372adbb 100644 --- a/tools/aapt2/format/binary/TableFlattener_test.cpp +++ b/tools/aapt2/format/binary/TableFlattener_test.cpp @@ -669,6 +669,87 @@ TEST_F(TableFlattenerTest, ObfuscatingResourceNamesNoNameCollapseExemptionsSucce ResourceId(0x7f050000), {}, Res_value::TYPE_STRING, (uint32_t)*idx, 0u)); } +TEST_F(TableFlattenerTest, ObfuscatingResourceNamesWithDeduplicationSucceeds) { + std::unique_ptr table = + test::ResourceTableBuilder() + .AddSimple("com.app.test:id/one", ResourceId(0x7f020000)) + .AddSimple("com.app.test:id/two", ResourceId(0x7f020001)) + .AddValue("com.app.test:id/three", ResourceId(0x7f020002), + test::BuildReference("com.app.test:id/one", ResourceId(0x7f020000))) + .AddValue("com.app.test:integer/one", ResourceId(0x7f030000), + util::make_unique(uint8_t(Res_value::TYPE_INT_DEC), 1u)) + .AddValue("com.app.test:integer/one", test::ParseConfigOrDie("v1"), + ResourceId(0x7f030000), + util::make_unique(uint8_t(Res_value::TYPE_INT_DEC), 2u)) + .AddString("com.app.test:string/test1", ResourceId(0x7f040000), "foo") + .AddString("com.app.test:string/test2", ResourceId(0x7f040001), "foo") + .AddString("com.app.test:string/test3", ResourceId(0x7f040002), "bar") + .AddString("com.app.test:string/test4", ResourceId(0x7f040003), "foo") + .AddString("com.app.test:layout/bar1", ResourceId(0x7f050000), "res/layout/bar.xml") + .AddString("com.app.test:layout/bar2", ResourceId(0x7f050001), "res/layout/bar.xml") + .Build(); + + TableFlattenerOptions options; + options.collapse_key_stringpool = true; + options.deduplicate_entry_values = true; + + ResTable res_table; + + ASSERT_TRUE(Flatten(context_.get(), options, table.get(), &res_table)); + + EXPECT_TRUE(Exists(&res_table, "com.app.test:id/0_resource_name_obfuscated", + ResourceId(0x7f020000), {}, Res_value::TYPE_INT_BOOLEAN, 0u, 0u)); + + EXPECT_TRUE(Exists(&res_table, "com.app.test:id/0_resource_name_obfuscated", + ResourceId(0x7f020001), {}, Res_value::TYPE_INT_BOOLEAN, 0u, 0u)); + + EXPECT_TRUE(Exists(&res_table, "com.app.test:id/0_resource_name_obfuscated", + ResourceId(0x7f020002), {}, Res_value::TYPE_REFERENCE, 0x7f020000u, 0u)); + + EXPECT_TRUE(Exists(&res_table, "com.app.test:integer/0_resource_name_obfuscated", + ResourceId(0x7f030000), {}, Res_value::TYPE_INT_DEC, 1u, + ResTable_config::CONFIG_VERSION)); + + EXPECT_TRUE(Exists(&res_table, "com.app.test:integer/0_resource_name_obfuscated", + ResourceId(0x7f030000), test::ParseConfigOrDie("v1"), Res_value::TYPE_INT_DEC, + 2u, ResTable_config::CONFIG_VERSION)); + + std::u16string foo_str = u"foo"; + std::u16string bar_str = u"bar"; + auto foo_idx = res_table.getTableStringBlock(0)->indexOfString(foo_str.data(), foo_str.size()); + auto bar_idx = res_table.getTableStringBlock(0)->indexOfString(bar_str.data(), bar_str.size()); + ASSERT_TRUE(foo_idx.has_value()); + EXPECT_TRUE(Exists(&res_table, "com.app.test:string/0_resource_name_obfuscated", + ResourceId(0x7f040000), {}, Res_value::TYPE_STRING, (uint32_t)*foo_idx, 0u)); + EXPECT_TRUE(Exists(&res_table, "com.app.test:string/0_resource_name_obfuscated", + ResourceId(0x7f040001), {}, Res_value::TYPE_STRING, (uint32_t)*foo_idx, 0u)); + EXPECT_TRUE(Exists(&res_table, "com.app.test:string/0_resource_name_obfuscated", + ResourceId(0x7f040002), {}, Res_value::TYPE_STRING, (uint32_t)*bar_idx, 0u)); + EXPECT_TRUE(Exists(&res_table, "com.app.test:string/0_resource_name_obfuscated", + ResourceId(0x7f040003), {}, Res_value::TYPE_STRING, (uint32_t)*foo_idx, 0u)); + + std::u16string bar_path = u"res/layout/bar.xml"; + auto bar_path_idx = + res_table.getTableStringBlock(0)->indexOfString(bar_path.data(), bar_path.size()); + ASSERT_TRUE(bar_path_idx.has_value()); + EXPECT_TRUE(Exists(&res_table, "com.app.test:layout/0_resource_name_obfuscated", + ResourceId(0x7f050000), {}, Res_value::TYPE_STRING, (uint32_t)*bar_path_idx, + 0u)); + EXPECT_TRUE(Exists(&res_table, "com.app.test:layout/0_resource_name_obfuscated", + ResourceId(0x7f050001), {}, Res_value::TYPE_STRING, (uint32_t)*bar_path_idx, + 0u)); + + std::string deduplicated_output; + std::string sequential_output; + Flatten(context_.get(), options, table.get(), &deduplicated_output); + options.deduplicate_entry_values = false; + Flatten(context_.get(), options, table.get(), &sequential_output); + + // We have 4 duplicates: 0x7f020001 id, 0x7f040001 string, 0x7f040003 string, 0x7f050001 layout. + EXPECT_EQ(sequential_output.size(), + deduplicated_output.size() + 4 * (sizeof(ResTable_entry) + sizeof(Res_value))); +} + TEST_F(TableFlattenerTest, ObfuscatingResourceNamesWithNameCollapseExemptionsSucceeds) { std::unique_ptr table = test::ResourceTableBuilder()