From c000962349494f96cfd2c568c1aab97e0f7967b7 Mon Sep 17 00:00:00 2001 From: Donald Chai Date: Wed, 22 Apr 2020 19:19:22 -0700 Subject: [PATCH 1/2] Define a unified protobuf message for "aapt2 optimize" metadata. This helps solve three problems: * obfuscation maps are now documented. * obfuscation maps will not require hand-rolled parsers. * no additional plumbing will be required for future optimizations---plumbing individual mapping files in Bazel for ProGuard, aapt2, etc. has been a pain. Bug: 153201806 Test: make aapt2 Change-Id: Ie3829834527e8875d768ae28242d3aeca28c6db2 --- tools/aapt2/Android.bp | 2 ++ tools/aapt2/ResourceMetadata.proto | 49 ++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 tools/aapt2/ResourceMetadata.proto diff --git a/tools/aapt2/Android.bp b/tools/aapt2/Android.bp index 7efe3c3472fa2..a4199b6b828b7 100644 --- a/tools/aapt2/Android.bp +++ b/tools/aapt2/Android.bp @@ -161,6 +161,7 @@ cc_library_host_static { "ApkInfo.proto", "Configuration.proto", "Resources.proto", + "ResourceMetadata.proto", "ResourcesInternal.proto", "ValueTransformer.cpp", ], @@ -218,6 +219,7 @@ genrule { srcs: [ "Configuration.proto", "ResourcesInternal.proto", + "ResourceMetadata.proto", "Resources.proto", ], out: ["aapt2-protos.zip"], diff --git a/tools/aapt2/ResourceMetadata.proto b/tools/aapt2/ResourceMetadata.proto new file mode 100644 index 0000000000000..8eca54c4da5e7 --- /dev/null +++ b/tools/aapt2/ResourceMetadata.proto @@ -0,0 +1,49 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +package aapt.pb; + +option java_package = "com.android.aapt"; +option java_multiple_files = true; + +message ResourceMappings { + ShortenedPathsMap shortened_paths = 1; + CollapsedNamesMap collapsed_names = 2; +} + +// Metadata relating to "aapt2 optimize --shorten-resource-paths" +message ShortenedPathsMap { + // Maps shorted paths (e.g. "res/foo.xml") to their original names (e.g. + // "res/xml/file_with_long_name.xml"). + message ResourcePathMapping { + string shortened_path = 1; + string original_path = 2; + } + repeated ResourcePathMapping resource_paths = 1; +} + +// Metadata relating to "aapt2 optimize --collapse-resource-names" +message CollapsedNamesMap { + // Maps resource IDs (e.g. 0x7f123456) to their original names (e.g. + // "package:type/entry"). + message ResourceNameMapping { + uint32 id = 1; + string name = 2; + } + repeated ResourceNameMapping resource_names = 1; +} From 4bdd3ac44c77465b5653dd7d3e254fea7afdc385 Mon Sep 17 00:00:00 2001 From: felkachang Date: Tue, 13 Sep 2022 10:58:49 +0800 Subject: [PATCH 2/2] Rename ResourcePathShortener to Obfuscator ResourcePathShortener is a part of Obfuscator. Obfuscator will not only shorten the path but also handle obfuscating resource names. Format C++ code by clang-format. Bug: 228192695 Test: atest aapt2_test idmap2_test Change-Id: I27b4b1af44061e3cd8a6717ffd46b8a187cdc41d --- tools/aapt2/Android.bp | 2 +- tools/aapt2/cmd/Optimize.cpp | 14 ++++-- ...sourcePathShortener.cpp => Obfuscator.cpp} | 33 ++++++------- .../{ResourcePathShortener.h => Obfuscator.h} | 16 +++---- ...Shortener_test.cpp => Obfuscator_test.cpp} | 47 +++++++++---------- 5 files changed, 55 insertions(+), 57 deletions(-) rename tools/aapt2/optimize/{ResourcePathShortener.cpp => Obfuscator.cpp} (83%) rename tools/aapt2/optimize/{ResourcePathShortener.h => Obfuscator.h} (72%) rename tools/aapt2/optimize/{ResourcePathShortener_test.cpp => Obfuscator_test.cpp} (81%) diff --git a/tools/aapt2/Android.bp b/tools/aapt2/Android.bp index a4199b6b828b7..7ddbe95aa79bd 100644 --- a/tools/aapt2/Android.bp +++ b/tools/aapt2/Android.bp @@ -130,7 +130,7 @@ cc_library_host_static { "optimize/MultiApkGenerator.cpp", "optimize/ResourceDeduper.cpp", "optimize/ResourceFilter.cpp", - "optimize/ResourcePathShortener.cpp", + "optimize/Obfuscator.cpp", "optimize/VersionCollapser.cpp", "process/SymbolTable.cpp", "split/TableSplitter.cpp", diff --git a/tools/aapt2/cmd/Optimize.cpp b/tools/aapt2/cmd/Optimize.cpp index e37c2d450b732..9feaf524eaf18 100644 --- a/tools/aapt2/cmd/Optimize.cpp +++ b/tools/aapt2/cmd/Optimize.cpp @@ -16,7 +16,11 @@ #include "Optimize.h" +#include #include +#include +#include +#include #include #include "Diagnostics.h" @@ -38,9 +42,9 @@ #include "io/BigBufferStream.h" #include "io/Util.h" #include "optimize/MultiApkGenerator.h" +#include "optimize/Obfuscator.h" #include "optimize/ResourceDeduper.h" #include "optimize/ResourceFilter.h" -#include "optimize/ResourcePathShortener.h" #include "optimize/VersionCollapser.h" #include "split/TableSplitter.h" #include "util/Files.h" @@ -114,11 +118,11 @@ class OptimizeContext : public IAaptContext { } private: - DISALLOW_COPY_AND_ASSIGN(OptimizeContext); - StdErrDiagnostics diagnostics_; bool verbose_ = false; int sdk_version_ = 0; + + DISALLOW_COPY_AND_ASSIGN(OptimizeContext); }; class Optimizer { @@ -151,8 +155,8 @@ class Optimizer { } if (options_.shorten_resource_paths) { - ResourcePathShortener shortener(options_.table_flattener_options.shortened_path_map); - if (!shortener.Consume(context_, apk->GetResourceTable())) { + Obfuscator obfuscator(options_.table_flattener_options.shortened_path_map); + if (!obfuscator.Consume(context_, apk->GetResourceTable())) { context_->GetDiagnostics()->Error(android::DiagMessage() << "failed shortening resource paths"); return 1; diff --git a/tools/aapt2/optimize/ResourcePathShortener.cpp b/tools/aapt2/optimize/Obfuscator.cpp similarity index 83% rename from tools/aapt2/optimize/ResourcePathShortener.cpp rename to tools/aapt2/optimize/Obfuscator.cpp index 7ff9bf5aa8df1..f704f26bfd291 100644 --- a/tools/aapt2/optimize/ResourcePathShortener.cpp +++ b/tools/aapt2/optimize/Obfuscator.cpp @@ -14,28 +14,25 @@ * limitations under the License. */ -#include "optimize/ResourcePathShortener.h" +#include "optimize/Obfuscator.h" #include +#include #include -#include "androidfw/StringPiece.h" - #include "ResourceTable.h" #include "ValueVisitor.h" +#include "androidfw/StringPiece.h" #include "util/Util.h" - -static const std::string base64_chars = - "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - "abcdefghijklmnopqrstuvwxyz" - "0123456789-_"; +static const char base64_chars[] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz" + "0123456789-_"; namespace aapt { -ResourcePathShortener::ResourcePathShortener( - std::map& path_map_out) - : path_map_(path_map_out) { +Obfuscator::Obfuscator(std::map& path_map_out) : path_map_(path_map_out) { } std::string ShortenFileName(const android::StringPiece& file_path, int output_length) { @@ -50,7 +47,6 @@ std::string ShortenFileName(const android::StringPiece& file_path, int output_le return result; } - // Return the optimal hash length such that at most 10% of resources collide in // their shortened path. // Reference: http://matt.might.net/articles/counting-hash-collisions/ @@ -63,7 +59,7 @@ int OptimalShortenedLength(int num_resources) { } std::string GetShortenedPath(const android::StringPiece& shortened_filename, - const android::StringPiece& extension, int collision_count) { + const android::StringPiece& extension, int collision_count) { std::string shortened_path = "res/" + shortened_filename.to_string(); if (collision_count > 0) { shortened_path += std::to_string(collision_count); @@ -76,12 +72,12 @@ std::string GetShortenedPath(const android::StringPiece& shortened_filename, // underlying filepath as key rather than the integer address. This is to ensure // determinism of output for colliding files. struct PathComparator { - bool operator() (const FileReference* lhs, const FileReference* rhs) const { - return lhs->path->compare(*rhs->path); - } + bool operator()(const FileReference* lhs, const FileReference* rhs) const { + return lhs->path->compare(*rhs->path); + } }; -bool ResourcePathShortener::Consume(IAaptContext* context, ResourceTable* table) { +bool Obfuscator::Consume(IAaptContext* context, ResourceTable* table) { // used to detect collisions std::unordered_set shortened_paths; std::set file_refs; @@ -103,8 +99,7 @@ bool ResourcePathShortener::Consume(IAaptContext* context, ResourceTable* table) util::ExtractResFilePathParts(*file_ref->path, &res_subdir, &actual_filename, &extension); // Android detects ColorStateLists via pathname, skip res/color* - if (util::StartsWith(res_subdir, "res/color")) - continue; + if (util::StartsWith(res_subdir, "res/color")) continue; std::string shortened_filename = ShortenFileName(*file_ref->path, num_chars); int collision_count = 0; diff --git a/tools/aapt2/optimize/ResourcePathShortener.h b/tools/aapt2/optimize/Obfuscator.h similarity index 72% rename from tools/aapt2/optimize/ResourcePathShortener.h rename to tools/aapt2/optimize/Obfuscator.h index f1074ef083bd0..1ea32db128158 100644 --- a/tools/aapt2/optimize/ResourcePathShortener.h +++ b/tools/aapt2/optimize/Obfuscator.h @@ -14,13 +14,13 @@ * limitations under the License. */ -#ifndef AAPT_OPTIMIZE_RESOURCEPATHSHORTENER_H -#define AAPT_OPTIMIZE_RESOURCEPATHSHORTENER_H +#ifndef TOOLS_AAPT2_OPTIMIZE_OBFUSCATOR_H_ +#define TOOLS_AAPT2_OPTIMIZE_OBFUSCATOR_H_ #include +#include #include "android-base/macros.h" - #include "process/IResourceTableConsumer.h" namespace aapt { @@ -28,17 +28,17 @@ namespace aapt { class ResourceTable; // Maps resources in the apk to shortened paths. -class ResourcePathShortener : public IResourceTableConsumer { +class Obfuscator : public IResourceTableConsumer { public: - explicit ResourcePathShortener(std::map& path_map_out); + explicit Obfuscator(std::map& path_map_out); bool Consume(IAaptContext* context, ResourceTable* table) override; private: - DISALLOW_COPY_AND_ASSIGN(ResourcePathShortener); std::map& path_map_; + DISALLOW_COPY_AND_ASSIGN(Obfuscator); }; -} // namespace aapt +} // namespace aapt -#endif // AAPT_OPTIMIZE_RESOURCEPATHSHORTENER_H +#endif // TOOLS_AAPT2_OPTIMIZE_OBFUSCATOR_H_ diff --git a/tools/aapt2/optimize/ResourcePathShortener_test.cpp b/tools/aapt2/optimize/Obfuscator_test.cpp similarity index 81% rename from tools/aapt2/optimize/ResourcePathShortener_test.cpp rename to tools/aapt2/optimize/Obfuscator_test.cpp index f5a02be0ea5ef..a3339d486d4a6 100644 --- a/tools/aapt2/optimize/ResourcePathShortener_test.cpp +++ b/tools/aapt2/optimize/Obfuscator_test.cpp @@ -14,15 +14,18 @@ * limitations under the License. */ -#include "optimize/ResourcePathShortener.h" +#include "optimize/Obfuscator.h" + +#include +#include #include "ResourceTable.h" #include "test/Test.h" using ::aapt::test::GetValue; +using ::testing::Eq; using ::testing::Not; using ::testing::NotNull; -using ::testing::Eq; android::StringPiece GetExtension(android::StringPiece path) { auto iter = std::find(path.begin(), path.end(), '.'); @@ -30,16 +33,15 @@ android::StringPiece GetExtension(android::StringPiece path) { } void FillTable(aapt::test::ResourceTableBuilder& builder, int start, int end) { - for (int i=start; i context = test::ContextBuilder().Build(); std::unique_ptr table = @@ -50,7 +52,7 @@ TEST(ResourcePathShortenerTest, FileRefPathsChangedInResourceTable) { .Build(); std::map path_map; - ASSERT_TRUE(ResourcePathShortener(path_map).Consume(context.get(), table.get())); + ASSERT_TRUE(Obfuscator(path_map).Consume(context.get(), table.get())); // Expect that the path map is populated ASSERT_THAT(path_map.find("res/drawables/xmlfile.xml"), Not(Eq(path_map.end()))); @@ -64,39 +66,36 @@ TEST(ResourcePathShortenerTest, FileRefPathsChangedInResourceTable) { EXPECT_THAT(path_map["res/drawables/xmlfile.xml"], Not(Eq(path_map["res/drawables/xmlfile2.xml"]))); - FileReference* ref = - GetValue(table.get(), "android:drawable/xmlfile"); + FileReference* ref = GetValue(table.get(), "android:drawable/xmlfile"); ASSERT_THAT(ref, NotNull()); // The map correctly points to the new location of the file EXPECT_THAT(path_map["res/drawables/xmlfile.xml"], Eq(*ref->path)); // Strings should not be affected, only file paths - EXPECT_THAT( - *GetValue(table.get(), "android:string/string")->value, + EXPECT_THAT(*GetValue(table.get(), "android:string/string")->value, Eq("res/should/still/be/the/same.png")); EXPECT_THAT(path_map.find("res/should/still/be/the/same.png"), Eq(path_map.end())); } -TEST(ResourcePathShortenerTest, SkipColorFileRefPaths) { +TEST(ObfuscatorTest, SkipColorFileRefPaths) { std::unique_ptr context = test::ContextBuilder().Build(); std::unique_ptr table = test::ResourceTableBuilder() .AddFileReference("android:color/colorlist", "res/color/colorlist.xml") - .AddFileReference("android:color/colorlist", - "res/color-mdp-v21/colorlist.xml", + .AddFileReference("android:color/colorlist", "res/color-mdp-v21/colorlist.xml", test::ParseConfigOrDie("mdp-v21")) .Build(); std::map path_map; - ASSERT_TRUE(ResourcePathShortener(path_map).Consume(context.get(), table.get())); + ASSERT_TRUE(Obfuscator(path_map).Consume(context.get(), table.get())); // Expect that the path map to not contain the ColorStateList ASSERT_THAT(path_map.find("res/color/colorlist.xml"), Eq(path_map.end())); ASSERT_THAT(path_map.find("res/color-mdp-v21/colorlist.xml"), Eq(path_map.end())); } -TEST(ResourcePathShortenerTest, KeepExtensions) { +TEST(ObfuscatorTest, KeepExtensions) { std::unique_ptr context = test::ContextBuilder().Build(); std::string original_xml_path = "res/drawable/xmlfile.xml"; @@ -109,7 +108,7 @@ TEST(ResourcePathShortenerTest, KeepExtensions) { .Build(); std::map path_map; - ASSERT_TRUE(ResourcePathShortener(path_map).Consume(context.get(), table.get())); + ASSERT_TRUE(Obfuscator(path_map).Consume(context.get(), table.get())); // Expect that the path map is populated ASSERT_THAT(path_map.find("res/drawable/xmlfile.xml"), Not(Eq(path_map.end()))); @@ -122,7 +121,7 @@ TEST(ResourcePathShortenerTest, KeepExtensions) { EXPECT_THAT(GetExtension(path_map[original_png_path]), Eq(android::StringPiece(".png"))); } -TEST(ResourcePathShortenerTest, DeterministicallyHandleCollisions) { +TEST(ObfuscatorTest, DeterministicallyHandleCollisions) { std::unique_ptr context = test::ContextBuilder().Build(); // 4000 resources is the limit at which the hash space is expanded to 3 @@ -135,27 +134,27 @@ TEST(ResourcePathShortenerTest, DeterministicallyHandleCollisions) { FillTable(builder1, 0, kNumResources); std::unique_ptr table1 = builder1.Build(); std::map expected_mapping; - ASSERT_TRUE(ResourcePathShortener(expected_mapping).Consume(context.get(), table1.get())); + ASSERT_TRUE(Obfuscator(expected_mapping).Consume(context.get(), table1.get())); // We are trying to ensure lack of non-determinism, it is not simple to prove // a negative, thus we must try the test a few times so that the test itself // is non-flaky. Basically create the pathmap 5 times from the same set of // resources but a different order of addition and then ensure they are always // mapped to the same short path. - for (int i=0; i table2 = builder2.Build(); std::map actual_mapping; - ASSERT_TRUE(ResourcePathShortener(actual_mapping).Consume(context.get(), table2.get())); + ASSERT_TRUE(Obfuscator(actual_mapping).Consume(context.get(), table2.get())); for (auto& item : actual_mapping) { ASSERT_THAT(expected_mapping[item.first], Eq(item.second)); @@ -163,4 +162,4 @@ TEST(ResourcePathShortenerTest, DeterministicallyHandleCollisions) { } } -} // namespace aapt +} // namespace aapt