From cf844a3e8cde155a10eb14f174b57300c732cda5 Mon Sep 17 00:00:00 2001 From: Iurii Makhno Date: Thu, 29 Sep 2022 16:45:51 +0000 Subject: [PATCH 1/2] Intoduce ResEntryWriter unit. ResEntryWriter is responsible to write pairs of entry+value into binary buffers and allows easy customization how and where entries are written. This is no-op refactoring and all existing tests should pass. Bug: b/249793372 Test: Existing tests Change-Id: Ica98a25aeb49cc96cad28547c462b36316a8adb6 --- tools/aapt2/Android.bp | 1 + tools/aapt2/format/binary/ResEntryWriter.cpp | 240 +++++++++++++++++++ tools/aapt2/format/binary/ResEntryWriter.h | 79 ++++++ tools/aapt2/format/binary/TableFlattener.cpp | 216 +---------------- 4 files changed, 323 insertions(+), 213 deletions(-) create mode 100644 tools/aapt2/format/binary/ResEntryWriter.cpp create mode 100644 tools/aapt2/format/binary/ResEntryWriter.h diff --git a/tools/aapt2/Android.bp b/tools/aapt2/Android.bp index 7ddbe95aa79bd..aa337e5f9691c 100644 --- a/tools/aapt2/Android.bp +++ b/tools/aapt2/Android.bp @@ -105,6 +105,7 @@ cc_library_host_static { "format/Container.cpp", "format/binary/BinaryResourceParser.cpp", "format/binary/ResChunkPullParser.cpp", + "format/binary/ResEntryWriter.cpp", "format/binary/TableFlattener.cpp", "format/binary/XmlFlattener.cpp", "format/proto/ProtoDeserialize.cpp", diff --git a/tools/aapt2/format/binary/ResEntryWriter.cpp b/tools/aapt2/format/binary/ResEntryWriter.cpp new file mode 100644 index 0000000000000..7b64619be2ba8 --- /dev/null +++ b/tools/aapt2/format/binary/ResEntryWriter.cpp @@ -0,0 +1,240 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "format/binary/ResEntryWriter.h" + +#include "ValueVisitor.h" +#include "androidfw/BigBuffer.h" +#include "androidfw/ResourceTypes.h" +#include "androidfw/Util.h" +#include "format/binary/ResourceTypeExtensions.h" + +namespace aapt { + +using android::BigBuffer; +using android::Res_value; +using android::ResTable_entry; +using android::ResTable_map; + +struct less_style_entries { + bool operator()(const Style::Entry* a, const Style::Entry* b) const { + if (a->key.id) { + if (b->key.id) { + return cmp_ids_dynamic_after_framework(a->key.id.value(), b->key.id.value()); + } + return true; + } + if (!b->key.id) { + return a->key.name.value() < b->key.name.value(); + } + return false; + } +}; + +class MapFlattenVisitor : public ConstValueVisitor { + public: + using ConstValueVisitor::Visit; + + MapFlattenVisitor(ResTable_entry_ext* out_entry, BigBuffer* buffer) + : out_entry_(out_entry), buffer_(buffer) { + } + + void Visit(const Attribute* attr) override { + { + Reference key = Reference(ResourceId(ResTable_map::ATTR_TYPE)); + BinaryPrimitive val(Res_value::TYPE_INT_DEC, attr->type_mask); + FlattenEntry(&key, &val); + } + + if (attr->min_int != std::numeric_limits::min()) { + Reference key = Reference(ResourceId(ResTable_map::ATTR_MIN)); + BinaryPrimitive val(Res_value::TYPE_INT_DEC, static_cast(attr->min_int)); + FlattenEntry(&key, &val); + } + + if (attr->max_int != std::numeric_limits::max()) { + Reference key = Reference(ResourceId(ResTable_map::ATTR_MAX)); + BinaryPrimitive val(Res_value::TYPE_INT_DEC, static_cast(attr->max_int)); + FlattenEntry(&key, &val); + } + + for (const Attribute::Symbol& s : attr->symbols) { + BinaryPrimitive val(s.type, s.value); + FlattenEntry(&s.symbol, &val); + } + } + + void Visit(const Style* style) override { + if (style->parent) { + const Reference& parent_ref = style->parent.value(); + CHECK(bool(parent_ref.id)) << "parent has no ID"; + out_entry_->parent.ident = android::util::HostToDevice32(parent_ref.id.value().id); + } + + // Sort the style. + std::vector sorted_entries; + for (const auto& entry : style->entries) { + sorted_entries.emplace_back(&entry); + } + + std::sort(sorted_entries.begin(), sorted_entries.end(), less_style_entries()); + + for (const Style::Entry* entry : sorted_entries) { + FlattenEntry(&entry->key, entry->value.get()); + } + } + + void Visit(const Styleable* styleable) override { + for (auto& attr_ref : styleable->entries) { + BinaryPrimitive val(Res_value{}); + FlattenEntry(&attr_ref, &val); + } + } + + void Visit(const Array* array) override { + const size_t count = array->elements.size(); + for (size_t i = 0; i < count; i++) { + Reference key(android::ResTable_map::ATTR_MIN + i); + FlattenEntry(&key, array->elements[i].get()); + } + } + + void Visit(const Plural* plural) override { + const size_t count = plural->values.size(); + for (size_t i = 0; i < count; i++) { + if (!plural->values[i]) { + continue; + } + + ResourceId q; + switch (i) { + case Plural::Zero: + q.id = android::ResTable_map::ATTR_ZERO; + break; + + case Plural::One: + q.id = android::ResTable_map::ATTR_ONE; + break; + + case Plural::Two: + q.id = android::ResTable_map::ATTR_TWO; + break; + + case Plural::Few: + q.id = android::ResTable_map::ATTR_FEW; + break; + + case Plural::Many: + q.id = android::ResTable_map::ATTR_MANY; + break; + + case Plural::Other: + q.id = android::ResTable_map::ATTR_OTHER; + break; + + default: + LOG(FATAL) << "unhandled plural type"; + break; + } + + Reference key(q); + FlattenEntry(&key, plural->values[i].get()); + } + } + + /** + * Call this after visiting a Value. This will finish any work that + * needs to be done to prepare the entry. + */ + void Finish() { + out_entry_->count = android::util::HostToDevice32(entry_count_); + } + + private: + DISALLOW_COPY_AND_ASSIGN(MapFlattenVisitor); + + void FlattenKey(const Reference* key, ResTable_map* out_entry) { + CHECK(bool(key->id)) << "key has no ID"; + out_entry->name.ident = android::util::HostToDevice32(key->id.value().id); + } + + void FlattenValue(const Item* value, ResTable_map* out_entry) { + CHECK(value->Flatten(&out_entry->value)) << "flatten failed"; + } + + void FlattenEntry(const Reference* key, Item* value) { + ResTable_map* out_entry = buffer_->NextBlock(); + FlattenKey(key, out_entry); + FlattenValue(value, out_entry); + out_entry->value.size = android::util::HostToDevice16(sizeof(out_entry->value)); + entry_count_++; + } + + ResTable_entry_ext* out_entry_; + BigBuffer* buffer_; + size_t entry_count_ = 0; +}; + +template +void WriteEntry(const FlatEntry* entry, T* out_result) { + static_assert(std::is_same_v || std::is_same_v, + "T must be ResTable_entry or ResTable_entry_ext"); + + ResTable_entry* out_entry = (ResTable_entry*)out_result; + if (entry->entry->visibility.level == Visibility::Level::kPublic) { + out_entry->flags |= ResTable_entry::FLAG_PUBLIC; + } + + if (entry->value->IsWeak()) { + out_entry->flags |= ResTable_entry::FLAG_WEAK; + } + + if constexpr (std::is_same_v) { + out_entry->flags |= ResTable_entry::FLAG_COMPLEX; + } + + out_entry->flags = android::util::HostToDevice16(out_entry->flags); + out_entry->key.index = android::util::HostToDevice32(entry->entry_key); + out_entry->size = android::util::HostToDevice16(sizeof(T)); +} + +int32_t WriteMapToBuffer(const FlatEntry* map_entry, BigBuffer* buffer) { + int32_t offset = buffer->size(); + ResTable_entry_ext* out_entry = buffer->NextBlock(); + WriteEntry(map_entry, out_entry); + + MapFlattenVisitor visitor(out_entry, buffer); + map_entry->value->Accept(&visitor); + visitor.Finish(); + return offset; +} + +int32_t SequentialResEntryWriter::WriteMap(const FlatEntry* entry) { + return WriteMapToBuffer(entry, entries_buffer_); +} + +int32_t SequentialResEntryWriter::WriteItem(const FlatEntry* entry) { + int32_t offset = entries_buffer_->size(); + ResTable_entry* out_entry = entries_buffer_->NextBlock(); + Res_value* out_value = entries_buffer_->NextBlock(); + out_value->size = android::util::HostToDevice16(sizeof(*out_value)); + + WriteEntry(entry, out_entry); + CHECK(ValueCast(entry->value)->Flatten(out_value)) << "flatten failed"; + return offset; +} + +} // namespace aapt \ No newline at end of file diff --git a/tools/aapt2/format/binary/ResEntryWriter.h b/tools/aapt2/format/binary/ResEntryWriter.h new file mode 100644 index 0000000000000..10bd3125091b6 --- /dev/null +++ b/tools/aapt2/format/binary/ResEntryWriter.h @@ -0,0 +1,79 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef AAPT_FORMAT_BINARY_RESENTRY_SERIALIZER_H +#define AAPT_FORMAT_BINARY_RESENTRY_SERIALIZER_H + +#include "ResourceTable.h" +#include "ValueVisitor.h" +#include "android-base/macros.h" +#include "androidfw/BigBuffer.h" + +namespace aapt { + +struct FlatEntry { + const ResourceTableEntryView* entry; + const Value* value; + + // The entry string pool index to the entry's name. + uint32_t entry_key; +}; + +class ResEntryWriter { + public: + virtual ~ResEntryWriter() = default; + + // Writes resource table entry and its value into 'entries_buffer_' and returns offset + // in the buffer where entry was written. + int32_t Write(const FlatEntry* entry) { + if (ValueCast(entry->value) != nullptr) { + return WriteItem(entry); + } else { + return WriteMap(entry); + } + } + + protected: + ResEntryWriter(android::BigBuffer* entries_buffer) : entries_buffer_(entries_buffer) { + } + android::BigBuffer* entries_buffer_; + + virtual int32_t WriteItem(const FlatEntry* entry) = 0; + + virtual int32_t WriteMap(const FlatEntry* entry) = 0; + + private: + DISALLOW_COPY_AND_ASSIGN(ResEntryWriter); +}; + +class SequentialResEntryWriter : public ResEntryWriter { + public: + explicit SequentialResEntryWriter(android::BigBuffer* entries_buffer) + : ResEntryWriter(entries_buffer) { + } + ~SequentialResEntryWriter() override = default; + + int32_t WriteItem(const FlatEntry* entry) override; + + int32_t WriteMap(const FlatEntry* entry) override; + + private: + DISALLOW_COPY_AND_ASSIGN(SequentialResEntryWriter); +}; + +} // namespace aapt + +#endif \ No newline at end of file diff --git a/tools/aapt2/format/binary/TableFlattener.cpp b/tools/aapt2/format/binary/TableFlattener.cpp index 22f278cc0b222..bb3d034b71a32 100644 --- a/tools/aapt2/format/binary/TableFlattener.cpp +++ b/tools/aapt2/format/binary/TableFlattener.cpp @@ -31,6 +31,7 @@ #include "androidfw/BigBuffer.h" #include "androidfw/ResourceUtils.h" #include "format/binary/ChunkWriter.h" +#include "format/binary/ResEntryWriter.h" #include "format/binary/ResourceTypeExtensions.h" #include "trace/TraceBuffer.h" @@ -58,170 +59,6 @@ static void strcpy16_htod(uint16_t* dst, size_t len, const StringPiece16& src) { dst[i] = 0; } -static bool cmp_style_entries(const Style::Entry* a, const Style::Entry* b) { - if (a->key.id) { - if (b->key.id) { - return cmp_ids_dynamic_after_framework(a->key.id.value(), b->key.id.value()); - } - return true; - } else if (!b->key.id) { - return a->key.name.value() < b->key.name.value(); - } - return false; -} - -struct FlatEntry { - const ResourceTableEntryView* entry; - const Value* value; - - // The entry string pool index to the entry's name. - uint32_t entry_key; -}; - -class MapFlattenVisitor : public ConstValueVisitor { - public: - using ConstValueVisitor::Visit; - - MapFlattenVisitor(ResTable_entry_ext* out_entry, BigBuffer* buffer) - : out_entry_(out_entry), buffer_(buffer) { - } - - void Visit(const Attribute* attr) override { - { - Reference key = Reference(ResourceId(ResTable_map::ATTR_TYPE)); - BinaryPrimitive val(Res_value::TYPE_INT_DEC, attr->type_mask); - FlattenEntry(&key, &val); - } - - if (attr->min_int != std::numeric_limits::min()) { - Reference key = Reference(ResourceId(ResTable_map::ATTR_MIN)); - BinaryPrimitive val(Res_value::TYPE_INT_DEC, static_cast(attr->min_int)); - FlattenEntry(&key, &val); - } - - if (attr->max_int != std::numeric_limits::max()) { - Reference key = Reference(ResourceId(ResTable_map::ATTR_MAX)); - BinaryPrimitive val(Res_value::TYPE_INT_DEC, static_cast(attr->max_int)); - FlattenEntry(&key, &val); - } - - for (const Attribute::Symbol& s : attr->symbols) { - BinaryPrimitive val(s.type, s.value); - FlattenEntry(&s.symbol, &val); - } - } - - void Visit(const Style* style) override { - if (style->parent) { - const Reference& parent_ref = style->parent.value(); - CHECK(bool(parent_ref.id)) << "parent has no ID"; - out_entry_->parent.ident = android::util::HostToDevice32(parent_ref.id.value().id); - } - - // Sort the style. - std::vector sorted_entries; - for (const auto& entry : style->entries) { - sorted_entries.emplace_back(&entry); - } - - std::sort(sorted_entries.begin(), sorted_entries.end(), cmp_style_entries); - - for (const Style::Entry* entry : sorted_entries) { - FlattenEntry(&entry->key, entry->value.get()); - } - } - - void Visit(const Styleable* styleable) override { - for (auto& attr_ref : styleable->entries) { - BinaryPrimitive val(Res_value{}); - FlattenEntry(&attr_ref, &val); - } - } - - void Visit(const Array* array) override { - const size_t count = array->elements.size(); - for (size_t i = 0; i < count; i++) { - Reference key(android::ResTable_map::ATTR_MIN + i); - FlattenEntry(&key, array->elements[i].get()); - } - } - - void Visit(const Plural* plural) override { - const size_t count = plural->values.size(); - for (size_t i = 0; i < count; i++) { - if (!plural->values[i]) { - continue; - } - - ResourceId q; - switch (i) { - case Plural::Zero: - q.id = android::ResTable_map::ATTR_ZERO; - break; - - case Plural::One: - q.id = android::ResTable_map::ATTR_ONE; - break; - - case Plural::Two: - q.id = android::ResTable_map::ATTR_TWO; - break; - - case Plural::Few: - q.id = android::ResTable_map::ATTR_FEW; - break; - - case Plural::Many: - q.id = android::ResTable_map::ATTR_MANY; - break; - - case Plural::Other: - q.id = android::ResTable_map::ATTR_OTHER; - break; - - default: - LOG(FATAL) << "unhandled plural type"; - break; - } - - Reference key(q); - FlattenEntry(&key, plural->values[i].get()); - } - } - - /** - * Call this after visiting a Value. This will finish any work that - * needs to be done to prepare the entry. - */ - void Finish() { - out_entry_->count = android::util::HostToDevice32(entry_count_); - } - - private: - DISALLOW_COPY_AND_ASSIGN(MapFlattenVisitor); - - void FlattenKey(const Reference* key, ResTable_map* out_entry) { - CHECK(bool(key->id)) << "key has no ID"; - out_entry->name.ident = android::util::HostToDevice32(key->id.value().id); - } - - void FlattenValue(const Item* value, ResTable_map* out_entry) { - CHECK(value->Flatten(&out_entry->value)) << "flatten failed"; - } - - void FlattenEntry(const Reference* key, Item* value) { - ResTable_map* out_entry = buffer_->NextBlock(); - FlattenKey(key, out_entry); - FlattenValue(value, out_entry); - out_entry->value.size = android::util::HostToDevice16(sizeof(out_entry->value)); - entry_count_++; - } - - ResTable_entry_ext* out_entry_; - BigBuffer* buffer_; - size_t entry_count_ = 0; -}; - struct OverlayableChunk { std::string actor; android::Source source; @@ -298,47 +135,6 @@ class PackageFlattener { private: DISALLOW_COPY_AND_ASSIGN(PackageFlattener); - template - T* WriteEntry(FlatEntry* entry, BigBuffer* buffer) { - static_assert( - std::is_same::value || std::is_same::value, - "T must be ResTable_entry or ResTable_entry_ext"); - - T* result = buffer->NextBlock(); - ResTable_entry* out_entry = (ResTable_entry*)result; - if (entry->entry->visibility.level == Visibility::Level::kPublic) { - out_entry->flags |= ResTable_entry::FLAG_PUBLIC; - } - - if (entry->value->IsWeak()) { - out_entry->flags |= ResTable_entry::FLAG_WEAK; - } - - if (!IsItem) { - out_entry->flags |= ResTable_entry::FLAG_COMPLEX; - } - - out_entry->flags = android::util::HostToDevice16(out_entry->flags); - out_entry->key.index = android::util::HostToDevice32(entry->entry_key); - out_entry->size = android::util::HostToDevice16(sizeof(T)); - return result; - } - - bool FlattenValue(FlatEntry* entry, BigBuffer* buffer) { - if (const Item* item = ValueCast(entry->value)) { - WriteEntry(entry, buffer); - Res_value* outValue = buffer->NextBlock(); - CHECK(item->Flatten(outValue)) << "flatten failed"; - outValue->size = android::util::HostToDevice16(sizeof(*outValue)); - } else { - ResTable_entry_ext* out_entry = WriteEntry(entry, buffer); - MapFlattenVisitor visitor(out_entry, buffer); - entry->value->Accept(&visitor); - visitor.Finish(); - } - return true; - } - bool FlattenConfig(const ResourceTableTypeView& type, const ConfigDescription& config, const size_t num_total_entries, std::vector* entries, BigBuffer* buffer) { @@ -355,16 +151,10 @@ class PackageFlattener { offsets.resize(num_total_entries, 0xffffffffu); android::BigBuffer values_buffer(512); + SequentialResEntryWriter res_entry_writer(&values_buffer); for (FlatEntry& flat_entry : *entries) { CHECK(static_cast(flat_entry.entry->id.value()) < num_total_entries); - offsets[flat_entry.entry->id.value()] = values_buffer.size(); - if (!FlattenValue(&flat_entry, &values_buffer)) { - diag_->Error(android::DiagMessage() - << "failed to flatten resource '" - << ResourceNameRef(package_.name, type.named_type, flat_entry.entry->name) - << "' for configuration '" << config << "'"); - return false; - } + offsets[flat_entry.entry->id.value()] = res_entry_writer.Write(&flat_entry); } bool sparse_encode = sparse_entries_ == SparseEntriesMode::Enabled || From 8b33e7adb86de38321620b8fd2d5491a10d70b4e Mon Sep 17 00:00:00 2001 From: Iurii Makhno Date: Fri, 30 Sep 2022 19:30:46 +0000 Subject: [PATCH 2/2] Add deduplicate_entry_values mode to TableFlattener. This CL introduces DeduplicateItemsResEntryWriter which stores unique pairs of entry and value and deduplicates a new pair based of this data before writing it into the buffer. Bug: b/249793372 Test: ResEntriesWriter_test, TableFlattener_test Change-Id: I938a3425a87bfad9853e59193680de2050a95bd2 --- tools/aapt2/format/binary/ResEntryWriter.cpp | 50 ++++++- tools/aapt2/format/binary/ResEntryWriter.h | 56 +++++++ .../format/binary/ResEntryWriter_test.cpp | 138 ++++++++++++++++++ tools/aapt2/format/binary/TableFlattener.cpp | 26 +++- tools/aapt2/format/binary/TableFlattener.h | 14 ++ .../format/binary/TableFlattener_test.cpp | 81 ++++++++++ 6 files changed, 351 insertions(+), 14 deletions(-) create mode 100644 tools/aapt2/format/binary/ResEntryWriter_test.cpp diff --git a/tools/aapt2/format/binary/ResEntryWriter.cpp b/tools/aapt2/format/binary/ResEntryWriter.cpp index 7b64619be2ba8..8832c24842eea 100644 --- a/tools/aapt2/format/binary/ResEntryWriter.cpp +++ b/tools/aapt2/format/binary/ResEntryWriter.cpp @@ -222,19 +222,57 @@ int32_t WriteMapToBuffer(const FlatEntry* map_entry, BigBuffer* buffer) { return offset; } +void WriteItemToPair(const FlatEntry* item_entry, ResEntryValuePair* out_pair) { + static_assert(sizeof(ResEntryValuePair) == sizeof(ResTable_entry) + sizeof(Res_value), + "ResEntryValuePair must not have padding between entry and value."); + + WriteEntry(item_entry, &out_pair->entry); + + CHECK(ValueCast(item_entry->value)->Flatten(&out_pair->value)) << "flatten failed"; + out_pair->value.size = android::util::HostToDevice16(sizeof(out_pair->value)); +} + int32_t SequentialResEntryWriter::WriteMap(const FlatEntry* entry) { return WriteMapToBuffer(entry, entries_buffer_); } int32_t SequentialResEntryWriter::WriteItem(const FlatEntry* entry) { int32_t offset = entries_buffer_->size(); - ResTable_entry* out_entry = entries_buffer_->NextBlock(); - Res_value* out_value = entries_buffer_->NextBlock(); - out_value->size = android::util::HostToDevice16(sizeof(*out_value)); - - WriteEntry(entry, out_entry); - CHECK(ValueCast(entry->value)->Flatten(out_value)) << "flatten failed"; + auto* out_pair = entries_buffer_->NextBlock(); + WriteItemToPair(entry, out_pair); return offset; } +std::size_t ResEntryValuePairContentHasher::operator()(const ResEntryValuePairRef& ref) const { + return android::JenkinsHashMixBytes(0, ref.ptr, sizeof(ResEntryValuePair)); +} + +bool ResEntryValuePairContentEqualTo::operator()(const ResEntryValuePairRef& a, + const ResEntryValuePairRef& b) const { + return std::memcmp(a.ptr, b.ptr, sizeof(ResEntryValuePair)) == 0; +} + +int32_t DeduplicateItemsResEntryWriter::WriteMap(const FlatEntry* entry) { + return WriteMapToBuffer(entry, entries_buffer_); +} + +int32_t DeduplicateItemsResEntryWriter::WriteItem(const FlatEntry* entry) { + int32_t initial_offset = entries_buffer_->size(); + + auto* out_pair = entries_buffer_->NextBlock(); + WriteItemToPair(entry, out_pair); + + auto ref = ResEntryValuePairRef{*out_pair}; + auto [it, inserted] = entry_offsets.insert({ref, initial_offset}); + if (inserted) { + // If inserted just return a new offset as this is a first time we store + // this entry. + return initial_offset; + } + // If not inserted this means that this is a duplicate, backup allocated block to the buffer + // and return offset of previously stored entry. + entries_buffer_->BackUp(sizeof(ResEntryValuePair)); + return it->second; +} + } // namespace aapt \ No newline at end of file diff --git a/tools/aapt2/format/binary/ResEntryWriter.h b/tools/aapt2/format/binary/ResEntryWriter.h index 10bd3125091b6..a36ceec2613b4 100644 --- a/tools/aapt2/format/binary/ResEntryWriter.h +++ b/tools/aapt2/format/binary/ResEntryWriter.h @@ -17,10 +17,13 @@ #ifndef AAPT_FORMAT_BINARY_RESENTRY_SERIALIZER_H #define AAPT_FORMAT_BINARY_RESENTRY_SERIALIZER_H +#include + #include "ResourceTable.h" #include "ValueVisitor.h" #include "android-base/macros.h" #include "androidfw/BigBuffer.h" +#include "androidfw/ResourceTypes.h" namespace aapt { @@ -32,6 +35,35 @@ struct FlatEntry { uint32_t entry_key; }; +// Pair of ResTable_entry and Res_value. These pairs are stored sequentially in values buffer. +// We introduce this structure for ResEntryWriter to a have single allocation using +// BigBuffer::NextBlock which allows to return it back with BigBuffer::Backup. +struct ResEntryValuePair { + android::ResTable_entry entry; + android::Res_value value; +}; + +// References ResEntryValuePair object stored in BigBuffer used as a key in std::unordered_map. +// Allows access to memory address where ResEntryValuePair is stored. +union ResEntryValuePairRef { + const std::reference_wrapper pair; + const u_char* ptr; + + explicit ResEntryValuePairRef(const ResEntryValuePair& ref) : pair(ref) { + } +}; + +// Hasher which computes hash of ResEntryValuePair using its bytes representation in memory. +struct ResEntryValuePairContentHasher { + std::size_t operator()(const ResEntryValuePairRef& ref) const; +}; + +// Equaler which compares ResEntryValuePairs using theirs bytes representation in memory. +struct ResEntryValuePairContentEqualTo { + bool operator()(const ResEntryValuePairRef& a, const ResEntryValuePairRef& b) const; +}; + +// Base class that allows to write FlatEntries into entries_buffer. class ResEntryWriter { public: virtual ~ResEntryWriter() = default; @@ -59,6 +91,8 @@ class ResEntryWriter { DISALLOW_COPY_AND_ASSIGN(ResEntryWriter); }; +// ResEntryWriter which writes FlatEntries sequentially into entries_buffer. +// Next entry is always written right after previous one in the buffer. class SequentialResEntryWriter : public ResEntryWriter { public: explicit SequentialResEntryWriter(android::BigBuffer* entries_buffer) @@ -74,6 +108,28 @@ class SequentialResEntryWriter : public ResEntryWriter { DISALLOW_COPY_AND_ASSIGN(SequentialResEntryWriter); }; +// ResEntryWriter that writes only unique entry and value pairs into entries_buffer. +// Next entry is written into buffer only if there is no entry with the same bytes representation +// in memory written before. Otherwise returns offset of already written entry. +class DeduplicateItemsResEntryWriter : public ResEntryWriter { + public: + explicit DeduplicateItemsResEntryWriter(android::BigBuffer* entries_buffer) + : ResEntryWriter(entries_buffer) { + } + ~DeduplicateItemsResEntryWriter() override = default; + + int32_t WriteItem(const FlatEntry* entry) override; + + int32_t WriteMap(const FlatEntry* entry) override; + + private: + DISALLOW_COPY_AND_ASSIGN(DeduplicateItemsResEntryWriter); + + std::unordered_map + entry_offsets; +}; + } // namespace aapt #endif \ No newline at end of file diff --git a/tools/aapt2/format/binary/ResEntryWriter_test.cpp b/tools/aapt2/format/binary/ResEntryWriter_test.cpp new file mode 100644 index 0000000000000..56ca1332ec5da --- /dev/null +++ b/tools/aapt2/format/binary/ResEntryWriter_test.cpp @@ -0,0 +1,138 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "format/binary/ResEntryWriter.h" + +#include "androidfw/BigBuffer.h" +#include "format/binary/ResourceTypeExtensions.h" +#include "test/Test.h" +#include "util/Util.h" + +using ::android::BigBuffer; +using ::android::Res_value; +using ::android::ResTable_map; +using ::testing::Eq; +using ::testing::Ge; +using ::testing::IsNull; +using ::testing::Ne; +using ::testing::NotNull; + +namespace aapt { + +using SequentialResEntryWriterTest = CommandTestFixture; +using DeduplicateItemsResEntryWriterTest = CommandTestFixture; + +std::vector WriteAllEntries(const ResourceTableView& table, ResEntryWriter& writer) { + std::vector result = {}; + for (const auto& type : table.packages[0].types) { + for (const auto& entry : type.entries) { + for (const auto& value : entry.values) { + auto flat_entry = FlatEntry{&entry, value->value.get(), 0}; + result.push_back(writer.Write(&flat_entry)); + } + } + } + return result; +} + +TEST_F(SequentialResEntryWriterTest, WriteEntriesOneByOne) { + std::unique_ptr table = + test::ResourceTableBuilder() + .AddSimple("com.app.test:id/id1", ResourceId(0x7f010000)) + .AddSimple("com.app.test:id/id2", ResourceId(0x7f010001)) + .AddSimple("com.app.test:id/id3", ResourceId(0x7f010002)) + .Build(); + + BigBuffer out(512); + SequentialResEntryWriter writer(&out); + auto offsets = WriteAllEntries(table->GetPartitionedView(), writer); + + std::vector expected_offsets{0, sizeof(ResEntryValuePair), + 2 * sizeof(ResEntryValuePair)}; + EXPECT_EQ(out.size(), 3 * sizeof(ResEntryValuePair)); + EXPECT_EQ(offsets, expected_offsets); +}; + +TEST_F(SequentialResEntryWriterTest, WriteMapEntriesOneByOne) { + std::unique_ptr array1 = util::make_unique(); + array1->elements.push_back( + util::make_unique(uint8_t(Res_value::TYPE_INT_DEC), 1u)); + array1->elements.push_back( + util::make_unique(uint8_t(Res_value::TYPE_INT_DEC), 2u)); + std::unique_ptr array2 = util::make_unique(); + array2->elements.push_back( + util::make_unique(uint8_t(Res_value::TYPE_INT_DEC), 1u)); + array2->elements.push_back( + util::make_unique(uint8_t(Res_value::TYPE_INT_DEC), 2u)); + + std::unique_ptr table = test::ResourceTableBuilder() + .AddValue("com.app.test:array/arr1", std::move(array1)) + .AddValue("com.app.test:array/arr2", std::move(array2)) + .Build(); + + BigBuffer out(512); + SequentialResEntryWriter writer(&out); + auto offsets = WriteAllEntries(table->GetPartitionedView(), writer); + + std::vector expected_offsets{0, sizeof(ResTable_entry_ext) + 2 * sizeof(ResTable_map)}; + EXPECT_EQ(out.size(), 2 * (sizeof(ResTable_entry_ext) + 2 * sizeof(ResTable_map))); + EXPECT_EQ(offsets, expected_offsets); +}; + +TEST_F(DeduplicateItemsResEntryWriterTest, DeduplicateItemEntries) { + std::unique_ptr table = + test::ResourceTableBuilder() + .AddSimple("com.app.test:id/id1", ResourceId(0x7f010000)) + .AddSimple("com.app.test:id/id2", ResourceId(0x7f010001)) + .AddSimple("com.app.test:id/id3", ResourceId(0x7f010002)) + .Build(); + + BigBuffer out(512); + DeduplicateItemsResEntryWriter writer(&out); + auto offsets = WriteAllEntries(table->GetPartitionedView(), writer); + + std::vector expected_offsets{0, 0, 0}; + EXPECT_EQ(out.size(), sizeof(ResEntryValuePair)); + EXPECT_EQ(offsets, expected_offsets); +}; + +TEST_F(DeduplicateItemsResEntryWriterTest, WriteMapEntriesOneByOne) { + std::unique_ptr array1 = util::make_unique(); + array1->elements.push_back( + util::make_unique(uint8_t(Res_value::TYPE_INT_DEC), 1u)); + array1->elements.push_back( + util::make_unique(uint8_t(Res_value::TYPE_INT_DEC), 2u)); + std::unique_ptr array2 = util::make_unique(); + array2->elements.push_back( + util::make_unique(uint8_t(Res_value::TYPE_INT_DEC), 1u)); + array2->elements.push_back( + util::make_unique(uint8_t(Res_value::TYPE_INT_DEC), 2u)); + + std::unique_ptr table = test::ResourceTableBuilder() + .AddValue("com.app.test:array/arr1", std::move(array1)) + .AddValue("com.app.test:array/arr2", std::move(array2)) + .Build(); + + BigBuffer out(512); + DeduplicateItemsResEntryWriter writer(&out); + auto offsets = WriteAllEntries(table->GetPartitionedView(), writer); + + std::vector expected_offsets{0, sizeof(ResTable_entry_ext) + 2 * sizeof(ResTable_map)}; + EXPECT_EQ(out.size(), 2 * (sizeof(ResTable_entry_ext) + 2 * sizeof(ResTable_map))); + EXPECT_EQ(offsets, expected_offsets); +}; + +} // namespace aapt \ No newline at end of file diff --git a/tools/aapt2/format/binary/TableFlattener.cpp b/tools/aapt2/format/binary/TableFlattener.cpp index bb3d034b71a32..7dc9d26f91083 100644 --- a/tools/aapt2/format/binary/TableFlattener.cpp +++ b/tools/aapt2/format/binary/TableFlattener.cpp @@ -16,15 +16,13 @@ #include "format/binary/TableFlattener.h" -#include -#include #include #include +#include #include "ResourceTable.h" #include "ResourceValues.h" #include "SdkConstants.h" -#include "ValueVisitor.h" #include "android-base/logging.h" #include "android-base/macros.h" #include "android-base/stringprintf.h" @@ -70,14 +68,16 @@ class PackageFlattener { PackageFlattener(IAaptContext* context, const ResourceTablePackageView& package, const std::map* shared_libs, SparseEntriesMode sparse_entries, bool collapse_key_stringpool, - const std::set& name_collapse_exemptions) + const std::set& name_collapse_exemptions, + bool deduplicate_entry_values) : context_(context), diag_(context->GetDiagnostics()), package_(package), shared_libs_(shared_libs), sparse_entries_(sparse_entries), collapse_key_stringpool_(collapse_key_stringpool), - name_collapse_exemptions_(name_collapse_exemptions) { + name_collapse_exemptions_(name_collapse_exemptions), + deduplicate_entry_values_(deduplicate_entry_values) { } bool FlattenPackage(BigBuffer* buffer) { @@ -151,10 +151,18 @@ class PackageFlattener { offsets.resize(num_total_entries, 0xffffffffu); android::BigBuffer values_buffer(512); - SequentialResEntryWriter res_entry_writer(&values_buffer); + std::variant + writer_variant; + ResEntryWriter* res_entry_writer; + if (deduplicate_entry_values_) { + res_entry_writer = &writer_variant.emplace(&values_buffer); + } else { + res_entry_writer = &writer_variant.emplace(&values_buffer); + } + for (FlatEntry& flat_entry : *entries) { CHECK(static_cast(flat_entry.entry->id.value()) < num_total_entries); - offsets[flat_entry.entry->id.value()] = res_entry_writer.Write(&flat_entry); + offsets[flat_entry.entry->id.value()] = res_entry_writer->Write(&flat_entry); } bool sparse_encode = sparse_entries_ == SparseEntriesMode::Enabled || @@ -510,6 +518,7 @@ class PackageFlattener { bool collapse_key_stringpool_; const std::set& name_collapse_exemptions_; std::map aliases_; + bool deduplicate_entry_values_; }; } // namespace @@ -561,7 +570,8 @@ bool TableFlattener::Consume(IAaptContext* context, ResourceTable* table) { PackageFlattener flattener(context, package, &table->included_packages_, options_.sparse_entries, options_.collapse_key_stringpool, - options_.name_collapse_exemptions); + options_.name_collapse_exemptions, + options_.deduplicate_entry_values); if (!flattener.FlattenPackage(&package_buffer)) { return false; } diff --git a/tools/aapt2/format/binary/TableFlattener.h b/tools/aapt2/format/binary/TableFlattener.h index c6d30335ed53d..6151b7e33b3fc 100644 --- a/tools/aapt2/format/binary/TableFlattener.h +++ b/tools/aapt2/format/binary/TableFlattener.h @@ -54,6 +54,20 @@ struct TableFlattenerOptions { // Map from original resource paths to shortened resource paths. std::map shortened_path_map; + + // When enabled, only unique pairs of entry and value are stored in type chunks. + // + // By default, all such pairs are unique because a reference to resource name in the string pool + // is a part of the pair. But when resource names are collapsed (using 'collapse_key_stringpool' + // flag or manually) the same data might be duplicated multiple times in the same type chunk. + // + // For example: an application has 3 boolean resources with collapsed names and 3 'true' values + // are defined for these resources in 'default' configuration. All pairs of entry and value for + // these resources will have the same binary representation and stored only once in type chunk + // instead of three times when this flag is disabled. + // + // This applies only to simple entries (entry->flags & ResTable_entry::FLAG_COMPLEX == 0). + bool deduplicate_entry_values = false; }; class TableFlattener : public IResourceTableConsumer { diff --git a/tools/aapt2/format/binary/TableFlattener_test.cpp b/tools/aapt2/format/binary/TableFlattener_test.cpp index b69daddf1c7ae..2097a6372adbb 100644 --- a/tools/aapt2/format/binary/TableFlattener_test.cpp +++ b/tools/aapt2/format/binary/TableFlattener_test.cpp @@ -669,6 +669,87 @@ TEST_F(TableFlattenerTest, ObfuscatingResourceNamesNoNameCollapseExemptionsSucce ResourceId(0x7f050000), {}, Res_value::TYPE_STRING, (uint32_t)*idx, 0u)); } +TEST_F(TableFlattenerTest, ObfuscatingResourceNamesWithDeduplicationSucceeds) { + std::unique_ptr table = + test::ResourceTableBuilder() + .AddSimple("com.app.test:id/one", ResourceId(0x7f020000)) + .AddSimple("com.app.test:id/two", ResourceId(0x7f020001)) + .AddValue("com.app.test:id/three", ResourceId(0x7f020002), + test::BuildReference("com.app.test:id/one", ResourceId(0x7f020000))) + .AddValue("com.app.test:integer/one", ResourceId(0x7f030000), + util::make_unique(uint8_t(Res_value::TYPE_INT_DEC), 1u)) + .AddValue("com.app.test:integer/one", test::ParseConfigOrDie("v1"), + ResourceId(0x7f030000), + util::make_unique(uint8_t(Res_value::TYPE_INT_DEC), 2u)) + .AddString("com.app.test:string/test1", ResourceId(0x7f040000), "foo") + .AddString("com.app.test:string/test2", ResourceId(0x7f040001), "foo") + .AddString("com.app.test:string/test3", ResourceId(0x7f040002), "bar") + .AddString("com.app.test:string/test4", ResourceId(0x7f040003), "foo") + .AddString("com.app.test:layout/bar1", ResourceId(0x7f050000), "res/layout/bar.xml") + .AddString("com.app.test:layout/bar2", ResourceId(0x7f050001), "res/layout/bar.xml") + .Build(); + + TableFlattenerOptions options; + options.collapse_key_stringpool = true; + options.deduplicate_entry_values = true; + + ResTable res_table; + + ASSERT_TRUE(Flatten(context_.get(), options, table.get(), &res_table)); + + EXPECT_TRUE(Exists(&res_table, "com.app.test:id/0_resource_name_obfuscated", + ResourceId(0x7f020000), {}, Res_value::TYPE_INT_BOOLEAN, 0u, 0u)); + + EXPECT_TRUE(Exists(&res_table, "com.app.test:id/0_resource_name_obfuscated", + ResourceId(0x7f020001), {}, Res_value::TYPE_INT_BOOLEAN, 0u, 0u)); + + EXPECT_TRUE(Exists(&res_table, "com.app.test:id/0_resource_name_obfuscated", + ResourceId(0x7f020002), {}, Res_value::TYPE_REFERENCE, 0x7f020000u, 0u)); + + EXPECT_TRUE(Exists(&res_table, "com.app.test:integer/0_resource_name_obfuscated", + ResourceId(0x7f030000), {}, Res_value::TYPE_INT_DEC, 1u, + ResTable_config::CONFIG_VERSION)); + + EXPECT_TRUE(Exists(&res_table, "com.app.test:integer/0_resource_name_obfuscated", + ResourceId(0x7f030000), test::ParseConfigOrDie("v1"), Res_value::TYPE_INT_DEC, + 2u, ResTable_config::CONFIG_VERSION)); + + std::u16string foo_str = u"foo"; + std::u16string bar_str = u"bar"; + auto foo_idx = res_table.getTableStringBlock(0)->indexOfString(foo_str.data(), foo_str.size()); + auto bar_idx = res_table.getTableStringBlock(0)->indexOfString(bar_str.data(), bar_str.size()); + ASSERT_TRUE(foo_idx.has_value()); + EXPECT_TRUE(Exists(&res_table, "com.app.test:string/0_resource_name_obfuscated", + ResourceId(0x7f040000), {}, Res_value::TYPE_STRING, (uint32_t)*foo_idx, 0u)); + EXPECT_TRUE(Exists(&res_table, "com.app.test:string/0_resource_name_obfuscated", + ResourceId(0x7f040001), {}, Res_value::TYPE_STRING, (uint32_t)*foo_idx, 0u)); + EXPECT_TRUE(Exists(&res_table, "com.app.test:string/0_resource_name_obfuscated", + ResourceId(0x7f040002), {}, Res_value::TYPE_STRING, (uint32_t)*bar_idx, 0u)); + EXPECT_TRUE(Exists(&res_table, "com.app.test:string/0_resource_name_obfuscated", + ResourceId(0x7f040003), {}, Res_value::TYPE_STRING, (uint32_t)*foo_idx, 0u)); + + std::u16string bar_path = u"res/layout/bar.xml"; + auto bar_path_idx = + res_table.getTableStringBlock(0)->indexOfString(bar_path.data(), bar_path.size()); + ASSERT_TRUE(bar_path_idx.has_value()); + EXPECT_TRUE(Exists(&res_table, "com.app.test:layout/0_resource_name_obfuscated", + ResourceId(0x7f050000), {}, Res_value::TYPE_STRING, (uint32_t)*bar_path_idx, + 0u)); + EXPECT_TRUE(Exists(&res_table, "com.app.test:layout/0_resource_name_obfuscated", + ResourceId(0x7f050001), {}, Res_value::TYPE_STRING, (uint32_t)*bar_path_idx, + 0u)); + + std::string deduplicated_output; + std::string sequential_output; + Flatten(context_.get(), options, table.get(), &deduplicated_output); + options.deduplicate_entry_values = false; + Flatten(context_.get(), options, table.get(), &sequential_output); + + // We have 4 duplicates: 0x7f020001 id, 0x7f040001 string, 0x7f040003 string, 0x7f050001 layout. + EXPECT_EQ(sequential_output.size(), + deduplicated_output.size() + 4 * (sizeof(ResTable_entry) + sizeof(Res_value))); +} + TEST_F(TableFlattenerTest, ObfuscatingResourceNamesWithNameCollapseExemptionsSucceeds) { std::unique_ptr table = test::ResourceTableBuilder()