Merge "Remove raw values from proto APK"

This commit is contained in:
Ryan Mitchell
2019-01-04 16:43:22 +00:00
committed by Android (Google) Code Review
13 changed files with 480 additions and 19 deletions

View File

@@ -170,6 +170,7 @@ cc_test_host {
srcs: [
"test/Builders.cpp",
"test/Common.cpp",
"test/Fixture.cpp",
"**/*_test.cpp",
] + toolSources,
static_libs: [
@@ -177,7 +178,10 @@ cc_test_host {
"libgmock",
],
defaults: ["aapt2_defaults"],
data: ["integration-tests/CompileTest/**/*"],
data: [
"integration-tests/CompileTest/**/*",
"integration-tests/CommandTests/**/*"
],
}
// ==========================================================

View File

@@ -60,16 +60,17 @@ class IApkSerializer {
class BinaryApkSerializer : public IApkSerializer {
public:
BinaryApkSerializer(IAaptContext* context, const Source& source,
const TableFlattenerOptions& options)
: IApkSerializer(context, source), tableFlattenerOptions_(options) {}
const TableFlattenerOptions& table_flattener_options,
const XmlFlattenerOptions& xml_flattener_options)
: IApkSerializer(context, source),
table_flattener_options_(table_flattener_options),
xml_flattener_options_(xml_flattener_options) {}
bool SerializeXml(const xml::XmlResource* xml, const std::string& path, bool utf16,
IArchiveWriter* writer, uint32_t compression_flags) override {
BigBuffer buffer(4096);
XmlFlattenerOptions options = {};
options.use_utf16 = utf16;
options.keep_raw_values = true;
XmlFlattener flattener(&buffer, options);
xml_flattener_options_.use_utf16 = utf16;
XmlFlattener flattener(&buffer, xml_flattener_options_);
if (!flattener.Consume(context_, xml)) {
return false;
}
@@ -80,7 +81,7 @@ class BinaryApkSerializer : public IApkSerializer {
bool SerializeTable(ResourceTable* table, IArchiveWriter* writer) override {
BigBuffer buffer(4096);
TableFlattener table_flattener(tableFlattenerOptions_, &buffer);
TableFlattener table_flattener(table_flattener_options_, &buffer);
if (!table_flattener.Consume(context_, table)) {
return false;
}
@@ -136,7 +137,8 @@ class BinaryApkSerializer : public IApkSerializer {
}
private:
TableFlattenerOptions tableFlattenerOptions_;
TableFlattenerOptions table_flattener_options_;
XmlFlattenerOptions xml_flattener_options_;
DISALLOW_COPY_AND_ASSIGN(BinaryApkSerializer);
};
@@ -252,13 +254,15 @@ class Context : public IAaptContext {
};
int Convert(IAaptContext* context, LoadedApk* apk, IArchiveWriter* output_writer,
ApkFormat output_format, TableFlattenerOptions& options) {
ApkFormat output_format, TableFlattenerOptions table_flattener_options,
XmlFlattenerOptions xml_flattener_options) {
// Do not change the ordering of strings in the values string pool
options.sort_stringpool_entries = false;
table_flattener_options.sort_stringpool_entries = false;
unique_ptr<IApkSerializer> serializer;
if (output_format == ApkFormat::kBinary) {
serializer.reset(new BinaryApkSerializer(context, apk->GetSource(), options));
serializer.reset(new BinaryApkSerializer(context, apk->GetSource(), table_flattener_options,
xml_flattener_options));
} else if (output_format == ApkFormat::kProto) {
serializer.reset(new ProtoApkSerializer(context, apk->GetSource()));
} else {
@@ -378,7 +382,8 @@ int ConvertCommand::Action(const std::vector<std::string>& args) {
return 1;
}
return Convert(&context, apk.get(), writer.get(), format, options_);
return Convert(&context, apk.get(), writer.get(), format, table_flattener_options_,
xml_flattener_options_);
}
} // namespace aapt

View File

@@ -20,6 +20,7 @@
#include "Command.h"
#include "LoadedApk.h"
#include "format/binary/TableFlattener.h"
#include "format/binary/XmlFlattener.h"
namespace aapt {
@@ -33,8 +34,12 @@ class ConvertCommand : public Command {
kOutputFormatProto, kOutputFormatBinary, kOutputFormatBinary), &output_format_);
AddOptionalSwitch("--enable-sparse-encoding",
"Enables encoding sparse entries using a binary search tree.\n"
"This decreases APK size at the cost of resource retrieval performance.",
&options_.use_sparse_entries);
"This decreases APK size at the cost of resource retrieval performance.",
&table_flattener_options_.use_sparse_entries);
AddOptionalSwitch("--keep-raw-values",
android::base::StringPrintf("Preserve raw attribute values in xml files when using the"
" '%s' output format", kOutputFormatBinary),
&xml_flattener_options_.keep_raw_values);
AddOptionalSwitch("-v", "Enables verbose logging", &verbose_);
}
@@ -44,14 +49,16 @@ class ConvertCommand : public Command {
const static char* kOutputFormatProto;
const static char* kOutputFormatBinary;
TableFlattenerOptions options_;
TableFlattenerOptions table_flattener_options_;
XmlFlattenerOptions xml_flattener_options_;
std::string output_path_;
Maybe<std::string> output_format_;
bool verbose_ = false;
};
int Convert(IAaptContext* context, LoadedApk* input, IArchiveWriter* output_writer,
ApkFormat output_format, TableFlattenerOptions& options);
ApkFormat output_format,TableFlattenerOptions table_flattener_options,
XmlFlattenerOptions xml_flattener_options);
} // namespace aapt

View File

@@ -0,0 +1,98 @@
/*
* Copyright (C) 2018 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "Convert.h"
#include "LoadedApk.h"
#include "test/Test.h"
using testing::Eq;
using testing::Ne;
namespace aapt {
using ConvertTest = CommandTestFixture;
TEST_F(ConvertTest, RemoveRawXmlStrings) {
StdErrDiagnostics diag;
const std::string compiled_files_dir = GetTestPath("compiled");
ASSERT_TRUE(CompileFile(GetTestPath("res/xml/test.xml"), R"(<Item AgentCode="007"/>)",
compiled_files_dir, &diag));
const std::string out_apk = GetTestPath("out.apk");
std::vector<std::string> link_args = {
"--manifest", GetDefaultManifest(),
"-o", out_apk,
"--keep-raw-values",
"--proto-format"
};
ASSERT_TRUE(Link(link_args, compiled_files_dir, &diag));
const std::string out_convert_apk = GetTestPath("out_convert.apk");
std::vector<android::StringPiece> convert_args = {
"-o", out_convert_apk,
"--output-format", "binary",
out_apk,
};
ASSERT_THAT(ConvertCommand().Execute(convert_args, &std::cerr), Eq(0));
// Load the binary xml tree
android::ResXMLTree tree;
std::unique_ptr<LoadedApk> apk = LoadedApk::LoadApkFromPath(out_convert_apk, &diag);
AssertLoadXml(apk.get(), "res/xml/test.xml", &tree);
// Check that the raw string index has not been assigned
EXPECT_THAT(tree.getAttributeValueStringID(0), Eq(-1));
}
TEST_F(ConvertTest, KeepRawXmlStrings) {
StdErrDiagnostics diag;
const std::string compiled_files_dir = GetTestPath("compiled");
ASSERT_TRUE(CompileFile(GetTestPath("res/xml/test.xml"), R"(<Item AgentCode="007"/>)",
compiled_files_dir, &diag));
const std::string out_apk = GetTestPath("out.apk");
std::vector<std::string> link_args = {
"--manifest", GetDefaultManifest(),
"-o", out_apk,
"--keep-raw-values",
"--proto-format"
};
ASSERT_TRUE(Link(link_args, compiled_files_dir, &diag));
const std::string out_convert_apk = GetTestPath("out_convert.apk");
std::vector<android::StringPiece> convert_args = {
"-o", out_convert_apk,
"--output-format", "binary",
"--keep-raw-values",
out_apk,
};
ASSERT_THAT(ConvertCommand().Execute(convert_args, &std::cerr), Eq(0));
// Load the binary xml tree
android::ResXMLTree tree;
std::unique_ptr<LoadedApk> apk = LoadedApk::LoadApkFromPath(out_convert_apk, &diag);
AssertLoadXml(apk.get(), "res/xml/test.xml", &tree);
// Check that the raw string index has been set to the correct string pool entry
int32_t raw_index = tree.getAttributeValueStringID(0);
ASSERT_THAT(raw_index, Ne(-1));
EXPECT_THAT(util::GetString(tree.getStrings(), static_cast<size_t>(raw_index)), Eq("007"));
}
} // namespace aapt

View File

@@ -1545,7 +1545,8 @@ class Linker {
// to the IArchiveWriter.
bool WriteApk(IArchiveWriter* writer, proguard::KeepSet* keep_set, xml::XmlResource* manifest,
ResourceTable* table) {
const bool keep_raw_values = context_->GetPackageType() == PackageType::kStaticLib;
const bool keep_raw_values = (context_->GetPackageType() == PackageType::kStaticLib)
|| options_.keep_raw_values;
bool result = FlattenXml(context_, *manifest, kAndroidManifestPath, keep_raw_values,
true /*utf16*/, options_.output_format, writer);
if (!result) {

View File

@@ -75,6 +75,7 @@ struct LinkOptions {
// Flattening options.
TableFlattenerOptions table_flattener_options;
bool keep_raw_values = false;
// Split APK options.
TableSplitterOptions table_splitter_options;
@@ -244,6 +245,8 @@ class LinkCommand : public Command {
&options_.extensions_to_not_compress);
AddOptionalSwitch("--no-compress", "Do not compress any resources.",
&options_.do_not_compress_anything);
AddOptionalSwitch("--keep-raw-values", "Preserve raw attribute values in xml files.",
&options_.keep_raw_values);
AddOptionalSwitch("--warn-manifest-validation",
"Treat manifest validation errors as warnings.",
&options_.manifest_fixer_options.warn_validation);
@@ -252,7 +255,6 @@ class LinkCommand : public Command {
"Syntax: path/to/output.apk:<config>[,<config>[...]].\n"
"On Windows, use a semicolon ';' separator instead.",
&split_args_);
AddOptionalSwitch("-v", "Enables verbose logging.", &verbose_);
AddOptionalSwitch("--debug-mode",
"Inserts android:debuggable=\"true\" in to the application node of the\n"
"manifest, making the application debuggable even on production devices.",
@@ -260,6 +262,7 @@ class LinkCommand : public Command {
AddOptionalSwitch("--strict-visibility",
"Do not allow overlays with different visibility levels.",
&options_.strict_visibility);
AddOptionalSwitch("-v", "Enables verbose logging.", &verbose_);
}
int Action(const std::vector<std::string>& args) override;

View File

@@ -0,0 +1,78 @@
/*
* Copyright (C) 2018 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "Link.h"
#include "LoadedApk.h"
#include "test/Test.h"
using testing::Eq;
using testing::Ne;
namespace aapt {
using LinkTest = CommandTestFixture;
TEST_F(LinkTest, RemoveRawXmlStrings) {
StdErrDiagnostics diag;
const std::string compiled_files_dir = GetTestPath("compiled");
ASSERT_TRUE(CompileFile(GetTestPath("res/xml/test.xml"), R"(<Item AgentCode="007"/>)",
compiled_files_dir, &diag));
const std::string out_apk = GetTestPath("out.apk");
std::vector<std::string> link_args = {
"--manifest", GetDefaultManifest(),
"-o", out_apk,
};
ASSERT_TRUE(Link(link_args, compiled_files_dir, &diag));
// Load the binary xml tree
android::ResXMLTree tree;
std::unique_ptr<LoadedApk> apk = LoadedApk::LoadApkFromPath(out_apk, &diag);
AssertLoadXml(apk.get(), "res/xml/test.xml", &tree);
// Check that the raw string index has not been assigned
EXPECT_THAT(tree.getAttributeValueStringID(0), Eq(-1));
}
TEST_F(LinkTest, KeepRawXmlStrings) {
StdErrDiagnostics diag;
const std::string compiled_files_dir = GetTestPath("compiled");
ASSERT_TRUE(CompileFile(GetTestPath("res/xml/test.xml"), R"(<Item AgentCode="007"/>)",
compiled_files_dir, &diag));
const std::string out_apk = GetTestPath("out.apk");
std::vector<std::string> link_args = {
"--manifest", GetDefaultManifest(),
"-o", out_apk,
"--keep-raw-values"
};
ASSERT_TRUE(Link(link_args, compiled_files_dir, &diag));
// Load the binary xml tree
android::ResXMLTree tree;
std::unique_ptr<LoadedApk> apk = LoadedApk::LoadApkFromPath(out_apk, &diag);
AssertLoadXml(apk.get(), "res/xml/test.xml", &tree);
// Check that the raw string index has been set to the correct string pool entry
int32_t raw_index = tree.getAttributeValueStringID(0);
ASSERT_THAT(raw_index, Ne(-1));
EXPECT_THAT(util::GetString(tree.getStrings(), static_cast<size_t>(raw_index)), Eq("007"));
}
} // namespace aapt

View File

@@ -0,0 +1,154 @@
/*
* Copyright (C) 2018 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "test/Fixture.h"
#include <dirent.h>
#include "android-base/errors.h"
#include "android-base/file.h"
#include "android-base/stringprintf.h"
#include "android-base/utf8.h"
#include "androidfw/StringPiece.h"
#include "gmock/gmock.h"
#include "gtest/gtest.h"
#include "cmd/Compile.h"
#include "cmd/Link.h"
#include "io/FileStream.h"
#include "io/Util.h"
#include "util/Files.h"
using testing::Eq;
using testing::Ne;
namespace aapt {
void ClearDirectory(const android::StringPiece& path) {
const std::string root_dir = path.to_string();
std::unique_ptr<DIR, decltype(closedir)*> dir(opendir(root_dir.data()), closedir);
if (!dir) {
StdErrDiagnostics().Error(DiagMessage() << android::base::SystemErrorCodeToString(errno));
return;
}
while (struct dirent* entry = readdir(dir.get())) {
// Do not delete hidden files and do not recurse to the parent of this directory
if (util::StartsWith(entry->d_name, ".")) {
continue;
}
std::string full_path = file::BuildPath({root_dir, entry->d_name});
if (file::GetFileType(full_path) == file::FileType::kDirectory) {
ClearDirectory(full_path);
#ifdef _WIN32
_rmdir(full_path.c_str());
#else
rmdir(full_path.c_str());
#endif
} else {
android::base::utf8::unlink(full_path.c_str());
}
}
}
void TestDirectoryFixture::SetUp() {
temp_dir_ = file::BuildPath({android::base::GetExecutableDirectory(),
"_temp",
testing::UnitTest::GetInstance()->current_test_case()->name(),
testing::UnitTest::GetInstance()->current_test_info()->name()});
ASSERT_TRUE(file::mkdirs(temp_dir_));
ClearDirectory(temp_dir_);
}
void TestDirectoryFixture::TearDown() {
ClearDirectory(temp_dir_);
}
bool TestDirectoryFixture::WriteFile(const std::string& path, const std::string& contents) {
CHECK(util::StartsWith(path, temp_dir_))
<< "Attempting to create a file outside of test temporary directory.";
// Create any intermediate directories specified in the path
auto pos = std::find(path.rbegin(), path.rend(), file::sDirSep);
if (pos != path.rend()) {
std::string dirs = path.substr(0, (&*pos - path.data()));
file::mkdirs(dirs);
}
return android::base::WriteStringToFile(contents, path);
}
bool CommandTestFixture::CompileFile(const std::string& path, const std::string& contents,
const android::StringPiece& out_dir, IDiagnostics* diag) {
CHECK(WriteFile(path, contents));
CHECK(file::mkdirs(out_dir.data()));
return CompileCommand(diag).Execute({path, "-o", out_dir, "-v"}, &std::cerr) == 0;
}
bool CommandTestFixture::Link(const std::vector<std::string>& args,
const android::StringPiece& flat_dir, IDiagnostics* diag) {
std::vector<android::StringPiece> link_args;
for(const std::string& arg : args) {
link_args.emplace_back(arg);
}
// Link against the android SDK
std::string android_sdk = file::BuildPath({android::base::GetExecutableDirectory(),
"integration-tests", "CommandTests",
"android-28.jar"});
link_args.insert(link_args.end(), {"-I", android_sdk});
// Add the files from the compiled resources directory to the link file arguments
Maybe<std::vector<std::string>> compiled_files = file::FindFiles(flat_dir, diag);
if (compiled_files) {
for (std::string& compile_file : compiled_files.value()) {
compile_file = file::BuildPath({flat_dir, compile_file});
link_args.emplace_back(std::move(compile_file));
}
}
return LinkCommand(diag).Execute(link_args, &std::cerr) == 0;
}
std::string CommandTestFixture::GetDefaultManifest() {
const std::string manifest_file = GetTestPath("AndroidManifest.xml");
CHECK(WriteFile(manifest_file, R"(
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
package="com.aapt.command.test">
</manifest>)"));
return manifest_file;
}
void CommandTestFixture::AssertLoadXml(LoadedApk *apk, const android::StringPiece &xml_path,
android::ResXMLTree *out_tree) {
ASSERT_THAT(apk, Ne(nullptr));
io::IFile* file = apk->GetFileCollection()->FindFile(xml_path);
ASSERT_THAT(file, Ne(nullptr));
std::unique_ptr<io::IData> data = file->OpenAsData();
ASSERT_THAT(data, Ne(nullptr));
out_tree->setTo(data->data(), data->size());
ASSERT_THAT(out_tree->getError(), Eq(android::OK));
while (out_tree->next() != android::ResXMLTree::START_TAG) {
ASSERT_THAT(out_tree->getEventType(), Ne(android::ResXMLTree::BAD_DOCUMENT));
ASSERT_THAT(out_tree->getEventType(), Ne(android::ResXMLTree::END_DOCUMENT));
}
}
} // namespace aapt

View File

@@ -0,0 +1,96 @@
/*
* Copyright (C) 2018 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef AAPT_TEST_FIXTURE_H
#define AAPT_TEST_FIXTURE_H
#include "android-base/file.h"
#include "android-base/macros.h"
#include "androidfw/StringPiece.h"
#include "gmock/gmock.h"
#include "gtest/gtest.h"
#include "io/Util.h"
#include "util/Files.h"
#include "LoadedApk.h"
namespace aapt {
class TestDirectoryFixture : public ::testing::Test {
public:
TestDirectoryFixture() = default;
virtual ~TestDirectoryFixture() = default;
// Creates the test directory or clears its contents if it contains previously created files.
void SetUp() override;
// Clears the contents of the test directory.
void TearDown() override;
// Retrieve the test directory of the fixture.
const android::StringPiece GetTestDirectory() {
return temp_dir_;
}
// Retrieves the absolute path of the specified relative path in the test directory. Directories
// should be separated using forward slashes ('/'), and these slashes will be translated to
// backslashes when running Windows tests.
const std::string GetTestPath(const android::StringPiece& path) {
std::string base = temp_dir_;
for (android::StringPiece part : util::Split(path, '/')) {
file::AppendPath(&base, part);
}
return base;
}
// Creates a file with the specified contents, creates any intermediate directories in the
// process. The file path must be an absolute path within the test directory.
bool WriteFile(const std::string& path, const std::string& contents);
private:
std::string temp_dir_;
DISALLOW_COPY_AND_ASSIGN(TestDirectoryFixture);
};
class CommandTestFixture : public TestDirectoryFixture {
public:
CommandTestFixture() = default;
virtual ~CommandTestFixture() = default;
// Wries the contents of the file to the specified path. The file is compiled and the flattened
// file is written to the out directory.
bool CompileFile(const std::string& path, const std::string& contents,
const android::StringPiece& flat_out_dir, IDiagnostics* diag);
// Executes the link command with the specified arguments. The flattened files residing in the
// flat directory will be added to the link command as file arguments.
bool Link(const std::vector<std::string>& args, const android::StringPiece& flat_dir,
IDiagnostics* diag);
// Creates a minimal android manifest within the test directory and returns the file path.
std::string GetDefaultManifest();
// Asserts that loading the tree from the specified file in the apk succeeds.
void AssertLoadXml(LoadedApk* apk, const android::StringPiece& xml_path,
android::ResXMLTree* out_tree);
private:
DISALLOW_COPY_AND_ASSIGN(CommandTestFixture);
};
} // namespace aapt
#endif // AAPT_TEST_FIXTURE_H

View File

@@ -23,5 +23,6 @@
#include "test/Builders.h"
#include "test/Common.h"
#include "test/Context.h"
#include "test/Fixture.h"
#endif // AAPT_TEST_TEST_H

View File

@@ -180,6 +180,17 @@ void AppendPath(std::string* base, StringPiece part) {
base->append(part.data(), part.size());
}
std::string BuildPath(std::vector<const StringPiece>&& args) {
if (args.empty()) {
return "";
}
std::string out = args[0].to_string();
for (int i = 1; i < args.size(); i++) {
file::AppendPath(&out, args[i]);
}
return out;
}
std::string PackageToPath(const StringPiece& package) {
std::string out_path;
for (const StringPiece& part : util::Tokenize(package, '.')) {

View File

@@ -57,6 +57,9 @@ FileType GetFileType(const std::string& path);
// Appends a path to `base`, separated by the directory separator.
void AppendPath(std::string* base, android::StringPiece part);
// Concatenates the list of paths and separates each part with the directory separator.
std::string BuildPath(std::vector<const android::StringPiece>&& args);
// Makes all the directories in `path`. The last element in the path is interpreted as a directory.
bool mkdirs(const std::string& path);