Remove raw values from proto APK

Raw attribute values are not included in xml files when building binary
APKs but they are when building proto APKs. Remove raw attribute values
and create a new convert flag that (--keep-raw-values) that allows raw
value to be kept.

Also added flag to link to keep raw attribute values.

Bug: 115915884
Bug: 116700688
Test: aapt2_tests
Change-Id: Id50136b775b3f43d159ca06181282f6a7b8eaf40
This commit is contained in:
Ryan Mitchell
2019-01-02 17:15:39 -08:00
parent d692ddf66d
commit 479fa39d65
13 changed files with 480 additions and 19 deletions

View File

@@ -170,6 +170,7 @@ cc_test_host {
srcs: [
"test/Builders.cpp",
"test/Common.cpp",
"test/Fixture.cpp",
"**/*_test.cpp",
] + toolSources,
static_libs: [
@@ -177,7 +178,10 @@ cc_test_host {
"libgmock",
],
defaults: ["aapt2_defaults"],
data: ["integration-tests/CompileTest/**/*"],
data: [
"integration-tests/CompileTest/**/*",
"integration-tests/CommandTests/**/*"
],
}
// ==========================================================

View File

@@ -60,16 +60,17 @@ class IApkSerializer {
class BinaryApkSerializer : public IApkSerializer {
public:
BinaryApkSerializer(IAaptContext* context, const Source& source,
const TableFlattenerOptions& options)
: IApkSerializer(context, source), tableFlattenerOptions_(options) {}
const TableFlattenerOptions& table_flattener_options,
const XmlFlattenerOptions& xml_flattener_options)
: IApkSerializer(context, source),
table_flattener_options_(table_flattener_options),
xml_flattener_options_(xml_flattener_options) {}
bool SerializeXml(const xml::XmlResource* xml, const std::string& path, bool utf16,
IArchiveWriter* writer, uint32_t compression_flags) override {
BigBuffer buffer(4096);
XmlFlattenerOptions options = {};
options.use_utf16 = utf16;
options.keep_raw_values = true;
XmlFlattener flattener(&buffer, options);
xml_flattener_options_.use_utf16 = utf16;
XmlFlattener flattener(&buffer, xml_flattener_options_);
if (!flattener.Consume(context_, xml)) {
return false;
}
@@ -80,7 +81,7 @@ class BinaryApkSerializer : public IApkSerializer {
bool SerializeTable(ResourceTable* table, IArchiveWriter* writer) override {
BigBuffer buffer(4096);
TableFlattener table_flattener(tableFlattenerOptions_, &buffer);
TableFlattener table_flattener(table_flattener_options_, &buffer);
if (!table_flattener.Consume(context_, table)) {
return false;
}
@@ -136,7 +137,8 @@ class BinaryApkSerializer : public IApkSerializer {
}
private:
TableFlattenerOptions tableFlattenerOptions_;
TableFlattenerOptions table_flattener_options_;
XmlFlattenerOptions xml_flattener_options_;
DISALLOW_COPY_AND_ASSIGN(BinaryApkSerializer);
};
@@ -252,13 +254,15 @@ class Context : public IAaptContext {
};
int Convert(IAaptContext* context, LoadedApk* apk, IArchiveWriter* output_writer,
ApkFormat output_format, TableFlattenerOptions& options) {
ApkFormat output_format, TableFlattenerOptions table_flattener_options,
XmlFlattenerOptions xml_flattener_options) {
// Do not change the ordering of strings in the values string pool
options.sort_stringpool_entries = false;
table_flattener_options.sort_stringpool_entries = false;
unique_ptr<IApkSerializer> serializer;
if (output_format == ApkFormat::kBinary) {
serializer.reset(new BinaryApkSerializer(context, apk->GetSource(), options));
serializer.reset(new BinaryApkSerializer(context, apk->GetSource(), table_flattener_options,
xml_flattener_options));
} else if (output_format == ApkFormat::kProto) {
serializer.reset(new ProtoApkSerializer(context, apk->GetSource()));
} else {
@@ -378,7 +382,8 @@ int ConvertCommand::Action(const std::vector<std::string>& args) {
return 1;
}
return Convert(&context, apk.get(), writer.get(), format, options_);
return Convert(&context, apk.get(), writer.get(), format, table_flattener_options_,
xml_flattener_options_);
}
} // namespace aapt

View File

@@ -20,6 +20,7 @@
#include "Command.h"
#include "LoadedApk.h"
#include "format/binary/TableFlattener.h"
#include "format/binary/XmlFlattener.h"
namespace aapt {
@@ -33,8 +34,12 @@ class ConvertCommand : public Command {
kOutputFormatProto, kOutputFormatBinary, kOutputFormatBinary), &output_format_);
AddOptionalSwitch("--enable-sparse-encoding",
"Enables encoding sparse entries using a binary search tree.\n"
"This decreases APK size at the cost of resource retrieval performance.",
&options_.use_sparse_entries);
"This decreases APK size at the cost of resource retrieval performance.",
&table_flattener_options_.use_sparse_entries);
AddOptionalSwitch("--keep-raw-values",
android::base::StringPrintf("Preserve raw attribute values in xml files when using the"
" '%s' output format", kOutputFormatBinary),
&xml_flattener_options_.keep_raw_values);
AddOptionalSwitch("-v", "Enables verbose logging", &verbose_);
}
@@ -44,14 +49,16 @@ class ConvertCommand : public Command {
const static char* kOutputFormatProto;
const static char* kOutputFormatBinary;
TableFlattenerOptions options_;
TableFlattenerOptions table_flattener_options_;
XmlFlattenerOptions xml_flattener_options_;
std::string output_path_;
Maybe<std::string> output_format_;
bool verbose_ = false;
};
int Convert(IAaptContext* context, LoadedApk* input, IArchiveWriter* output_writer,
ApkFormat output_format, TableFlattenerOptions& options);
ApkFormat output_format,TableFlattenerOptions table_flattener_options,
XmlFlattenerOptions xml_flattener_options);
} // namespace aapt

View File

@@ -0,0 +1,98 @@
/*
* Copyright (C) 2018 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "Convert.h"
#include "LoadedApk.h"
#include "test/Test.h"
using testing::Eq;
using testing::Ne;
namespace aapt {
using ConvertTest = CommandTestFixture;
TEST_F(ConvertTest, RemoveRawXmlStrings) {
StdErrDiagnostics diag;
const std::string compiled_files_dir = GetTestPath("compiled");
ASSERT_TRUE(CompileFile(GetTestPath("res/xml/test.xml"), R"(<Item AgentCode="007"/>)",
compiled_files_dir, &diag));
const std::string out_apk = GetTestPath("out.apk");
std::vector<std::string> link_args = {
"--manifest", GetDefaultManifest(),
"-o", out_apk,
"--keep-raw-values",
"--proto-format"
};
ASSERT_TRUE(Link(link_args, compiled_files_dir, &diag));
const std::string out_convert_apk = GetTestPath("out_convert.apk");
std::vector<android::StringPiece> convert_args = {
"-o", out_convert_apk,
"--output-format", "binary",
out_apk,
};
ASSERT_THAT(ConvertCommand().Execute(convert_args, &std::cerr), Eq(0));
// Load the binary xml tree
android::ResXMLTree tree;
std::unique_ptr<LoadedApk> apk = LoadedApk::LoadApkFromPath(out_convert_apk, &diag);
AssertLoadXml(apk.get(), "res/xml/test.xml", &tree);
// Check that the raw string index has not been assigned
EXPECT_THAT(tree.getAttributeValueStringID(0), Eq(-1));
}
TEST_F(ConvertTest, KeepRawXmlStrings) {
StdErrDiagnostics diag;
const std::string compiled_files_dir = GetTestPath("compiled");
ASSERT_TRUE(CompileFile(GetTestPath("res/xml/test.xml"), R"(<Item AgentCode="007"/>)",
compiled_files_dir, &diag));
const std::string out_apk = GetTestPath("out.apk");
std::vector<std::string> link_args = {
"--manifest", GetDefaultManifest(),
"-o", out_apk,
"--keep-raw-values",
"--proto-format"
};
ASSERT_TRUE(Link(link_args, compiled_files_dir, &diag));
const std::string out_convert_apk = GetTestPath("out_convert.apk");
std::vector<android::StringPiece> convert_args = {
"-o", out_convert_apk,
"--output-format", "binary",
"--keep-raw-values",
out_apk,
};
ASSERT_THAT(ConvertCommand().Execute(convert_args, &std::cerr), Eq(0));
// Load the binary xml tree
android::ResXMLTree tree;
std::unique_ptr<LoadedApk> apk = LoadedApk::LoadApkFromPath(out_convert_apk, &diag);
AssertLoadXml(apk.get(), "res/xml/test.xml", &tree);
// Check that the raw string index has been set to the correct string pool entry
int32_t raw_index = tree.getAttributeValueStringID(0);
ASSERT_THAT(raw_index, Ne(-1));
EXPECT_THAT(util::GetString(tree.getStrings(), static_cast<size_t>(raw_index)), Eq("007"));
}
} // namespace aapt

View File

@@ -1545,7 +1545,8 @@ class Linker {
// to the IArchiveWriter.
bool WriteApk(IArchiveWriter* writer, proguard::KeepSet* keep_set, xml::XmlResource* manifest,
ResourceTable* table) {
const bool keep_raw_values = context_->GetPackageType() == PackageType::kStaticLib;
const bool keep_raw_values = (context_->GetPackageType() == PackageType::kStaticLib)
|| options_.keep_raw_values;
bool result = FlattenXml(context_, *manifest, kAndroidManifestPath, keep_raw_values,
true /*utf16*/, options_.output_format, writer);
if (!result) {

View File

@@ -75,6 +75,7 @@ struct LinkOptions {
// Flattening options.
TableFlattenerOptions table_flattener_options;
bool keep_raw_values = false;
// Split APK options.
TableSplitterOptions table_splitter_options;
@@ -242,6 +243,8 @@ class LinkCommand : public Command {
&options_.extensions_to_not_compress);
AddOptionalSwitch("--no-compress", "Do not compress any resources.",
&options_.do_not_compress_anything);
AddOptionalSwitch("--keep-raw-values", "Preserve raw attribute values in xml files.",
&options_.keep_raw_values);
AddOptionalSwitch("--warn-manifest-validation",
"Treat manifest validation errors as warnings.",
&options_.manifest_fixer_options.warn_validation);
@@ -250,7 +253,6 @@ class LinkCommand : public Command {
"Syntax: path/to/output.apk:<config>[,<config>[...]].\n"
"On Windows, use a semicolon ';' separator instead.",
&split_args_);
AddOptionalSwitch("-v", "Enables verbose logging.", &verbose_);
AddOptionalSwitch("--debug-mode",
"Inserts android:debuggable=\"true\" in to the application node of the\n"
"manifest, making the application debuggable even on production devices.",
@@ -258,6 +260,7 @@ class LinkCommand : public Command {
AddOptionalSwitch("--strict-visibility",
"Do not allow overlays with different visibility levels.",
&options_.strict_visibility);
AddOptionalSwitch("-v", "Enables verbose logging.", &verbose_);
}
int Action(const std::vector<std::string>& args) override;

View File

@@ -0,0 +1,78 @@
/*
* Copyright (C) 2018 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "Link.h"
#include "LoadedApk.h"
#include "test/Test.h"
using testing::Eq;
using testing::Ne;
namespace aapt {
using LinkTest = CommandTestFixture;
TEST_F(LinkTest, RemoveRawXmlStrings) {
StdErrDiagnostics diag;
const std::string compiled_files_dir = GetTestPath("compiled");
ASSERT_TRUE(CompileFile(GetTestPath("res/xml/test.xml"), R"(<Item AgentCode="007"/>)",
compiled_files_dir, &diag));
const std::string out_apk = GetTestPath("out.apk");
std::vector<std::string> link_args = {
"--manifest", GetDefaultManifest(),
"-o", out_apk,
};
ASSERT_TRUE(Link(link_args, compiled_files_dir, &diag));
// Load the binary xml tree
android::ResXMLTree tree;
std::unique_ptr<LoadedApk> apk = LoadedApk::LoadApkFromPath(out_apk, &diag);
AssertLoadXml(apk.get(), "res/xml/test.xml", &tree);
// Check that the raw string index has not been assigned
EXPECT_THAT(tree.getAttributeValueStringID(0), Eq(-1));
}
TEST_F(LinkTest, KeepRawXmlStrings) {
StdErrDiagnostics diag;
const std::string compiled_files_dir = GetTestPath("compiled");
ASSERT_TRUE(CompileFile(GetTestPath("res/xml/test.xml"), R"(<Item AgentCode="007"/>)",
compiled_files_dir, &diag));
const std::string out_apk = GetTestPath("out.apk");
std::vector<std::string> link_args = {
"--manifest", GetDefaultManifest(),
"-o", out_apk,
"--keep-raw-values"
};
ASSERT_TRUE(Link(link_args, compiled_files_dir, &diag));
// Load the binary xml tree
android::ResXMLTree tree;
std::unique_ptr<LoadedApk> apk = LoadedApk::LoadApkFromPath(out_apk, &diag);
AssertLoadXml(apk.get(), "res/xml/test.xml", &tree);
// Check that the raw string index has been set to the correct string pool entry
int32_t raw_index = tree.getAttributeValueStringID(0);
ASSERT_THAT(raw_index, Ne(-1));
EXPECT_THAT(util::GetString(tree.getStrings(), static_cast<size_t>(raw_index)), Eq("007"));
}
} // namespace aapt

View File

@@ -0,0 +1,154 @@
/*
* Copyright (C) 2018 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "test/Fixture.h"
#include <dirent.h>
#include "android-base/errors.h"
#include "android-base/file.h"
#include "android-base/stringprintf.h"
#include "android-base/utf8.h"
#include "androidfw/StringPiece.h"
#include "gmock/gmock.h"
#include "gtest/gtest.h"
#include "cmd/Compile.h"
#include "cmd/Link.h"
#include "io/FileStream.h"
#include "io/Util.h"
#include "util/Files.h"
using testing::Eq;
using testing::Ne;
namespace aapt {
void ClearDirectory(const android::StringPiece& path) {
const std::string root_dir = path.to_string();
std::unique_ptr<DIR, decltype(closedir)*> dir(opendir(root_dir.data()), closedir);
if (!dir) {
StdErrDiagnostics().Error(DiagMessage() << android::base::SystemErrorCodeToString(errno));
return;
}
while (struct dirent* entry = readdir(dir.get())) {
// Do not delete hidden files and do not recurse to the parent of this directory
if (util::StartsWith(entry->d_name, ".")) {
continue;
}
std::string full_path = file::BuildPath({root_dir, entry->d_name});
if (file::GetFileType(full_path) == file::FileType::kDirectory) {
ClearDirectory(full_path);
#ifdef _WIN32
_rmdir(full_path.c_str());
#else
rmdir(full_path.c_str());
#endif
} else {
android::base::utf8::unlink(full_path.c_str());
}
}
}
void TestDirectoryFixture::SetUp() {
temp_dir_ = file::BuildPath({android::base::GetExecutableDirectory(),
"_temp",
testing::UnitTest::GetInstance()->current_test_case()->name(),
testing::UnitTest::GetInstance()->current_test_info()->name()});
ASSERT_TRUE(file::mkdirs(temp_dir_));
ClearDirectory(temp_dir_);
}
void TestDirectoryFixture::TearDown() {
ClearDirectory(temp_dir_);
}
bool TestDirectoryFixture::WriteFile(const std::string& path, const std::string& contents) {
CHECK(util::StartsWith(path, temp_dir_))
<< "Attempting to create a file outside of test temporary directory.";
// Create any intermediate directories specified in the path
auto pos = std::find(path.rbegin(), path.rend(), file::sDirSep);
if (pos != path.rend()) {
std::string dirs = path.substr(0, (&*pos - path.data()));
file::mkdirs(dirs);
}
return android::base::WriteStringToFile(contents, path);
}
bool CommandTestFixture::CompileFile(const std::string& path, const std::string& contents,
const android::StringPiece& out_dir, IDiagnostics* diag) {
CHECK(WriteFile(path, contents));
CHECK(file::mkdirs(out_dir.data()));
return CompileCommand(diag).Execute({path, "-o", out_dir, "-v"}, &std::cerr) == 0;
}
bool CommandTestFixture::Link(const std::vector<std::string>& args,
const android::StringPiece& flat_dir, IDiagnostics* diag) {
std::vector<android::StringPiece> link_args;
for(const std::string& arg : args) {
link_args.emplace_back(arg);
}
// Link against the android SDK
std::string android_sdk = file::BuildPath({android::base::GetExecutableDirectory(),
"integration-tests", "CommandTests",
"android-28.jar"});
link_args.insert(link_args.end(), {"-I", android_sdk});
// Add the files from the compiled resources directory to the link file arguments
Maybe<std::vector<std::string>> compiled_files = file::FindFiles(flat_dir, diag);
if (compiled_files) {
for (std::string& compile_file : compiled_files.value()) {
compile_file = file::BuildPath({flat_dir, compile_file});
link_args.emplace_back(std::move(compile_file));
}
}
return LinkCommand(diag).Execute(link_args, &std::cerr) == 0;
}
std::string CommandTestFixture::GetDefaultManifest() {
const std::string manifest_file = GetTestPath("AndroidManifest.xml");
CHECK(WriteFile(manifest_file, R"(
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
package="com.aapt.command.test">
</manifest>)"));
return manifest_file;
}
void CommandTestFixture::AssertLoadXml(LoadedApk *apk, const android::StringPiece &xml_path,
android::ResXMLTree *out_tree) {
ASSERT_THAT(apk, Ne(nullptr));
io::IFile* file = apk->GetFileCollection()->FindFile(xml_path);
ASSERT_THAT(file, Ne(nullptr));
std::unique_ptr<io::IData> data = file->OpenAsData();
ASSERT_THAT(data, Ne(nullptr));
out_tree->setTo(data->data(), data->size());
ASSERT_THAT(out_tree->getError(), Eq(android::OK));
while (out_tree->next() != android::ResXMLTree::START_TAG) {
ASSERT_THAT(out_tree->getEventType(), Ne(android::ResXMLTree::BAD_DOCUMENT));
ASSERT_THAT(out_tree->getEventType(), Ne(android::ResXMLTree::END_DOCUMENT));
}
}
} // namespace aapt

View File

@@ -0,0 +1,96 @@
/*
* Copyright (C) 2018 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef AAPT_TEST_FIXTURE_H
#define AAPT_TEST_FIXTURE_H
#include "android-base/file.h"
#include "android-base/macros.h"
#include "androidfw/StringPiece.h"
#include "gmock/gmock.h"
#include "gtest/gtest.h"
#include "io/Util.h"
#include "util/Files.h"
#include "LoadedApk.h"
namespace aapt {
class TestDirectoryFixture : public ::testing::Test {
public:
TestDirectoryFixture() = default;
virtual ~TestDirectoryFixture() = default;
// Creates the test directory or clears its contents if it contains previously created files.
void SetUp() override;
// Clears the contents of the test directory.
void TearDown() override;
// Retrieve the test directory of the fixture.
const android::StringPiece GetTestDirectory() {
return temp_dir_;
}
// Retrieves the absolute path of the specified relative path in the test directory. Directories
// should be separated using forward slashes ('/'), and these slashes will be translated to
// backslashes when running Windows tests.
const std::string GetTestPath(const android::StringPiece& path) {
std::string base = temp_dir_;
for (android::StringPiece part : util::Split(path, '/')) {
file::AppendPath(&base, part);
}
return base;
}
// Creates a file with the specified contents, creates any intermediate directories in the
// process. The file path must be an absolute path within the test directory.
bool WriteFile(const std::string& path, const std::string& contents);
private:
std::string temp_dir_;
DISALLOW_COPY_AND_ASSIGN(TestDirectoryFixture);
};
class CommandTestFixture : public TestDirectoryFixture {
public:
CommandTestFixture() = default;
virtual ~CommandTestFixture() = default;
// Wries the contents of the file to the specified path. The file is compiled and the flattened
// file is written to the out directory.
bool CompileFile(const std::string& path, const std::string& contents,
const android::StringPiece& flat_out_dir, IDiagnostics* diag);
// Executes the link command with the specified arguments. The flattened files residing in the
// flat directory will be added to the link command as file arguments.
bool Link(const std::vector<std::string>& args, const android::StringPiece& flat_dir,
IDiagnostics* diag);
// Creates a minimal android manifest within the test directory and returns the file path.
std::string GetDefaultManifest();
// Asserts that loading the tree from the specified file in the apk succeeds.
void AssertLoadXml(LoadedApk* apk, const android::StringPiece& xml_path,
android::ResXMLTree* out_tree);
private:
DISALLOW_COPY_AND_ASSIGN(CommandTestFixture);
};
} // namespace aapt
#endif // AAPT_TEST_FIXTURE_H

View File

@@ -23,5 +23,6 @@
#include "test/Builders.h"
#include "test/Common.h"
#include "test/Context.h"
#include "test/Fixture.h"
#endif // AAPT_TEST_TEST_H

View File

@@ -180,6 +180,17 @@ void AppendPath(std::string* base, StringPiece part) {
base->append(part.data(), part.size());
}
std::string BuildPath(std::vector<const StringPiece>&& args) {
if (args.empty()) {
return "";
}
std::string out = args[0].to_string();
for (int i = 1; i < args.size(); i++) {
file::AppendPath(&out, args[i]);
}
return out;
}
std::string PackageToPath(const StringPiece& package) {
std::string out_path;
for (const StringPiece& part : util::Tokenize(package, '.')) {

View File

@@ -57,6 +57,9 @@ FileType GetFileType(const std::string& path);
// Appends a path to `base`, separated by the directory separator.
void AppendPath(std::string* base, android::StringPiece part);
// Concatenates the list of paths and separates each part with the directory separator.
std::string BuildPath(std::vector<const android::StringPiece>&& args);
// Makes all the directories in `path`. The last element in the path is interpreted as a directory.
bool mkdirs(const std::string& path);