AAPT2: Change proto format to reduce usage of StringPool

The StringPool class is a binary blob and makes it difficult to
modify the proto files from external tools (like bundle-tool).

Size increase of full build is negligible.

Test: make aapt2_tests
Bug: 65645766
Change-Id: I984755170c315730ab751b51133f8fb2f614f6af
Merged-In: I984755170c315730ab751b51133f8fb2f614f6af
This commit is contained in:
Adam Lesinski
2017-08-04 16:15:17 -07:00
parent ee0e486d36
commit e116ebe0bc
4 changed files with 135 additions and 168 deletions

View File

@@ -67,21 +67,12 @@ message CompiledFile {
// Top level message representing a resource table.
message ResourceTable {
// The string pool containing string values (strings that actually end up in the binary ARSC
// file) referenced throughout the resource table.
optional StringPool string_pool = 1;
// The string pool containing source paths referenced throughout the resource table. This does
// not end up in the final binary ARSC file.
optional StringPool source_pool = 2;
// The string pool containing the names of unresolved symbols. This does not end up in the final
// binary ARSC file. Unresolved symbols are just resource names that haven't had a resource ID
// assigned to them, therefore can't be referenced by resource ID.
optional StringPool symbol_pool = 3;
optional StringPool source_pool = 1;
// Resource definitions corresponding to an Android package.
repeated Package packages = 4;
repeated Package packages = 2;
}
// Defines resources for an Android package.
@@ -202,9 +193,10 @@ message Item {
optional Reference ref = 1;
optional String str = 2;
optional RawString raw_str = 3;
optional FileReference file = 4;
optional Id id = 5;
optional Primitive prim = 6;
optional StyledString styled_str = 4;
optional FileReference file = 5;
optional Id id = 6;
optional Primitive prim = 7;
}
// A CompoundValue is an abstract type. It represents a value that is a made of other values.
@@ -233,10 +225,8 @@ message Reference {
// The resource ID (0xPPTTEEEE) of the resource being referred.
optional uint32 id = 2;
// If the resource ID is not resolved, the index into the symbol string pool where the name of
// the reference is stored. The symbol string pool is located at the top level ResourceTable
// message.
optional uint32 symbol_idx = 3;
// The optional resource name.
optional string name = 3;
// Whether this reference is referencing a private resource (@*package:type/entry).
optional bool private = 4;
@@ -249,23 +239,41 @@ message Id {
// A value that is a string.
message String {
// The index into the values string pool, located at the top level ResourceTable message.
optional uint32 idx = 1;
optional string value = 1;
}
// A value that is a raw string, which is unescaped/uninterpreted. This is typically used to
// represent the value of a style attribute before the attribute is compiled and the set of
// allowed values is known.
message RawString {
// The index into the values string pool, located at the top level ResourceTable message.
optional uint32 idx = 1;
optional string value = 1;
}
// A string with styling information, like html tags that specify boldness, italics, etc.
message StyledString {
// The raw text of the string.
optional string value = 1;
// A Span marks a region of the string text that is styled.
message Span {
// The name of the tag, and its attributes, encoded as follows:
// tag_name;attr1=value1;attr2=value2;[...]
optional string tag = 1;
// The first character position this span applies to, in UTF-16 offset.
optional uint32 first_char = 2;
// The last character position this span applies to, in UTF-16 offset.
optional uint32 last_char = 3;
}
repeated Span span = 2;
}
// A value that is a reference to an external entity, like an XML file or a PNG.
message FileReference {
// The index into the values string pool, located at the top level ResourceTable message. This
// represents the path to the file within an APK (typically res/type-config/entry.ext).
optional uint32 path_idx = 1;
// Path to a file within the APK (typically res/type-config/entry.ext).
optional string path = 1;
}
// A value that represents a primitive data type (float, int, boolean, etc.).

View File

@@ -55,15 +55,10 @@ class ReferenceIdToNameVisitor : public ValueVisitor {
class PackagePbDeserializer {
public:
PackagePbDeserializer(const android::ResStringPool* valuePool,
const android::ResStringPool* sourcePool,
const android::ResStringPool* symbolPool,
const Source& source, IDiagnostics* diag)
: value_pool_(valuePool),
source_pool_(sourcePool),
symbol_pool_(symbolPool),
source_(source),
diag_(diag) {}
PackagePbDeserializer(const android::ResStringPool* sourcePool, const Source& source,
IDiagnostics* diag)
: source_pool_(sourcePool), source_(source), diag_(diag) {
}
public:
bool DeserializeFromPb(const pb::Package& pbPackage, ResourceTable* table) {
@@ -87,8 +82,7 @@ class PackagePbDeserializer {
for (const pb::Entry& pbEntry : pbType.entries()) {
ResourceEntry* entry = type->FindOrCreateEntry(pbEntry.name());
// Deserialize the symbol status (public/private with source and
// comments).
// Deserialize the symbol status (public/private with source and comments).
if (pbEntry.has_symbol_status()) {
const pb::SymbolStatus& pbStatus = pbEntry.symbol_status();
if (pbStatus.has_source()) {
@@ -161,8 +155,7 @@ class PackagePbDeserializer {
private:
std::unique_ptr<Item> DeserializeItemFromPb(const pb::Item& pb_item,
const ConfigDescription& config,
StringPool* pool) {
const ConfigDescription& config, StringPool* pool) {
if (pb_item.has_ref()) {
const pb::Reference& pb_ref = pb_item.ref();
std::unique_ptr<Reference> ref = util::make_unique<Reference>();
@@ -173,45 +166,32 @@ class PackagePbDeserializer {
} else if (pb_item.has_prim()) {
const pb::Primitive& pb_prim = pb_item.prim();
android::Res_value prim = {};
prim.dataType = static_cast<uint8_t>(pb_prim.type());
prim.data = pb_prim.data();
return util::make_unique<BinaryPrimitive>(prim);
return util::make_unique<BinaryPrimitive>(static_cast<uint8_t>(pb_prim.type()),
pb_prim.data());
} else if (pb_item.has_id()) {
return util::make_unique<Id>();
} else if (pb_item.has_str()) {
const uint32_t idx = pb_item.str().idx();
const std::string str = util::GetString(*value_pool_, idx);
const android::ResStringPool_span* spans = value_pool_->styleAt(idx);
if (spans && spans->name.index != android::ResStringPool_span::END) {
StyleString style_str = {str};
while (spans->name.index != android::ResStringPool_span::END) {
style_str.spans.push_back(
Span{util::GetString(*value_pool_, spans->name.index),
spans->firstChar, spans->lastChar});
spans++;
}
return util::make_unique<StyledString>(pool->MakeRef(
style_str, StringPool::Context(StringPool::Context::kNormalPriority, config)));
}
return util::make_unique<String>(
pool->MakeRef(str, StringPool::Context(config)));
pool->MakeRef(pb_item.str().value(), StringPool::Context(config)));
} else if (pb_item.has_raw_str()) {
const uint32_t idx = pb_item.raw_str().idx();
const std::string str = util::GetString(*value_pool_, idx);
return util::make_unique<RawString>(
pool->MakeRef(str, StringPool::Context(config)));
pool->MakeRef(pb_item.raw_str().value(), StringPool::Context(config)));
} else if (pb_item.has_styled_str()) {
const pb::StyledString& pb_str = pb_item.styled_str();
StyleString style_str{pb_str.value()};
for (const pb::StyledString::Span& pb_span : pb_str.span()) {
style_str.spans.push_back(Span{pb_span.tag(), pb_span.first_char(), pb_span.last_char()});
}
return util::make_unique<StyledString>(pool->MakeRef(
style_str, StringPool::Context(StringPool::Context::kNormalPriority, config)));
} else if (pb_item.has_file()) {
const uint32_t idx = pb_item.file().path_idx();
const std::string str = util::GetString(*value_pool_, idx);
return util::make_unique<FileReference>(pool->MakeRef(
str,
StringPool::Context(StringPool::Context::kHighPriority, config)));
pb_item.file().path(), StringPool::Context(StringPool::Context::kHighPriority, config)));
} else {
diag_->Error(DiagMessage(source_) << "unknown item");
@@ -255,15 +235,13 @@ class PackagePbDeserializer {
std::unique_ptr<Style> style = util::make_unique<Style>();
if (pb_style.has_parent()) {
style->parent = Reference();
if (!DeserializeReferenceFromPb(pb_style.parent(),
&style->parent.value())) {
if (!DeserializeReferenceFromPb(pb_style.parent(), &style->parent.value())) {
return {};
}
if (pb_style.has_parent_source()) {
Source parent_source;
DeserializeSourceFromPb(pb_style.parent_source(), *source_pool_,
&parent_source);
DeserializeSourceFromPb(pb_style.parent_source(), *source_pool_, &parent_source);
style->parent.value().SetSource(std::move(parent_source));
}
}
@@ -300,8 +278,7 @@ class PackagePbDeserializer {
const pb::Array& pb_array = pb_compound_value.array();
std::unique_ptr<Array> array = util::make_unique<Array>();
for (const pb::Array_Entry& pb_entry : pb_array.entries()) {
std::unique_ptr<Item> item =
DeserializeItemFromPb(pb_entry.item(), config, pool);
std::unique_ptr<Item> item = DeserializeItemFromPb(pb_entry.item(), config, pool);
if (!item) {
return {};
}
@@ -316,8 +293,7 @@ class PackagePbDeserializer {
std::unique_ptr<Plural> plural = util::make_unique<Plural>();
for (const pb::Plural_Entry& pb_entry : pb_plural.entries()) {
size_t pluralIdx = DeserializePluralEnumFromPb(pb_entry.arity());
plural->values[pluralIdx] =
DeserializeItemFromPb(pb_entry.item(), config, pool);
plural->values[pluralIdx] = DeserializeItemFromPb(pb_entry.item(), config, pool);
if (!plural->values[pluralIdx]) {
return {};
}
@@ -350,11 +326,10 @@ class PackagePbDeserializer {
out_ref->id = ResourceId(pb_ref.id());
}
if (pb_ref.has_symbol_idx()) {
const std::string str_symbol = util::GetString(*symbol_pool_, pb_ref.symbol_idx());
if (pb_ref.has_name()) {
ResourceNameRef name_ref;
if (!ResourceUtils::ParseResourceName(str_symbol, &name_ref, nullptr)) {
diag_->Error(DiagMessage(source_) << "invalid reference name '" << str_symbol << "'");
if (!ResourceUtils::ParseResourceName(pb_ref.name(), &name_ref, nullptr)) {
diag_->Error(DiagMessage(source_) << "invalid reference name '" << pb_ref.name() << "'");
return false;
}
@@ -377,60 +352,32 @@ class PackagePbDeserializer {
}
private:
const android::ResStringPool* value_pool_;
const android::ResStringPool* source_pool_;
const android::ResStringPool* symbol_pool_;
const Source source_;
IDiagnostics* diag_;
};
} // namespace
std::unique_ptr<ResourceTable> DeserializeTableFromPb(
const pb::ResourceTable& pb_table, const Source& source,
IDiagnostics* diag) {
// We import the android namespace because on Windows NO_ERROR is a macro, not
// an enum, which
std::unique_ptr<ResourceTable> DeserializeTableFromPb(const pb::ResourceTable& pb_table,
const Source& source, IDiagnostics* diag) {
// We import the android namespace because on Windows NO_ERROR is a macro, not an enum, which
// causes errors when qualifying it with android::
using namespace android;
std::unique_ptr<ResourceTable> table = util::make_unique<ResourceTable>();
if (!pb_table.has_string_pool()) {
diag->Error(DiagMessage(source) << "no string pool found");
return {};
}
ResStringPool value_pool;
status_t result = value_pool.setTo(pb_table.string_pool().data().data(),
pb_table.string_pool().data().size());
if (result != NO_ERROR) {
diag->Error(DiagMessage(source) << "invalid string pool");
return {};
}
ResStringPool source_pool;
if (pb_table.has_source_pool()) {
result = source_pool.setTo(pb_table.source_pool().data().data(),
pb_table.source_pool().data().size());
status_t result = source_pool.setTo(pb_table.source_pool().data().data(),
pb_table.source_pool().data().size());
if (result != NO_ERROR) {
diag->Error(DiagMessage(source) << "invalid source pool");
return {};
}
}
ResStringPool symbol_pool;
if (pb_table.has_symbol_pool()) {
result = symbol_pool.setTo(pb_table.symbol_pool().data().data(),
pb_table.symbol_pool().data().size());
if (result != NO_ERROR) {
diag->Error(DiagMessage(source) << "invalid symbol pool");
return {};
}
}
PackagePbDeserializer package_pb_deserializer(&value_pool, &source_pool,
&symbol_pool, source, diag);
PackagePbDeserializer package_pb_deserializer(&source_pool, source, diag);
for (const pb::Package& pb_package : pb_table.packages()) {
if (!package_pb_deserializer.DeserializeFromPb(pb_package, table.get())) {
return {};

View File

@@ -24,9 +24,9 @@
#include "android-base/logging.h"
using google::protobuf::io::CodedOutputStream;
using google::protobuf::io::CodedInputStream;
using google::protobuf::io::ZeroCopyOutputStream;
using ::google::protobuf::io::CodedInputStream;
using ::google::protobuf::io::CodedOutputStream;
using ::google::protobuf::io::ZeroCopyOutputStream;
namespace aapt {
@@ -36,46 +36,46 @@ class PbSerializerVisitor : public RawValueVisitor {
public:
using RawValueVisitor::Visit;
/**
* Constructor to use when expecting to serialize any value.
*/
PbSerializerVisitor(StringPool* source_pool, StringPool* symbol_pool,
pb::Value* out_pb_value)
: source_pool_(source_pool),
symbol_pool_(symbol_pool),
out_pb_value_(out_pb_value),
out_pb_item_(nullptr) {}
// Constructor to use when expecting to serialize any value.
PbSerializerVisitor(StringPool* source_pool, pb::Value* out_pb_value)
: source_pool_(source_pool), out_pb_value_(out_pb_value), out_pb_item_(nullptr) {
}
/**
* Constructor to use when expecting to serialize an Item.
*/
PbSerializerVisitor(StringPool* sourcePool, StringPool* symbolPool,
pb::Item* outPbItem)
: source_pool_(sourcePool),
symbol_pool_(symbolPool),
out_pb_value_(nullptr),
out_pb_item_(outPbItem) {}
// Constructor to use when expecting to serialize an Item.
PbSerializerVisitor(StringPool* sourcePool, pb::Item* outPbItem)
: source_pool_(sourcePool), out_pb_value_(nullptr), out_pb_item_(outPbItem) {
}
void Visit(Reference* ref) override {
SerializeReferenceToPb(*ref, pb_item()->mutable_ref());
}
void Visit(String* str) override {
pb_item()->mutable_str()->set_idx(str->value.index());
pb_item()->mutable_str()->set_value(*str->value);
}
void Visit(RawString* str) override {
pb_item()->mutable_raw_str()->set_value(*str->value);
}
void Visit(StyledString* str) override {
pb_item()->mutable_str()->set_idx(str->value.index());
pb::StyledString* pb_str = pb_item()->mutable_styled_str();
pb_str->set_value(str->value->value);
for (const StringPool::Span& span : str->value->spans) {
pb::StyledString::Span* pb_span = pb_str->add_span();
pb_span->set_tag(*span.name);
pb_span->set_first_char(span.first_char);
pb_span->set_last_char(span.last_char);
}
}
void Visit(FileReference* file) override {
pb_item()->mutable_file()->set_path_idx(file->path.index());
pb_item()->mutable_file()->set_path(*file->path);
}
void Visit(Id* id) override { pb_item()->mutable_id(); }
void Visit(RawString* raw_str) override {
pb_item()->mutable_raw_str()->set_idx(raw_str->value.index());
void Visit(Id* /*id*/) override {
pb_item()->mutable_id();
}
void Visit(BinaryPrimitive* prim) override {
@@ -119,7 +119,7 @@ class PbSerializerVisitor : public RawValueVisitor {
pb::Item* pb_item = pb_entry->mutable_item();
SerializeItemCommonToPb(entry.key, pb_entry);
PbSerializerVisitor sub_visitor(source_pool_, symbol_pool_, pb_item);
PbSerializerVisitor sub_visitor(source_pool_, pb_item);
entry.value->Accept(&sub_visitor);
}
}
@@ -138,8 +138,7 @@ class PbSerializerVisitor : public RawValueVisitor {
for (auto& value : array->items) {
pb::Array_Entry* pb_entry = pb_array->add_entries();
SerializeItemCommonToPb(*value, pb_entry);
PbSerializerVisitor sub_visitor(source_pool_, symbol_pool_,
pb_entry->mutable_item());
PbSerializerVisitor sub_visitor(source_pool_, pb_entry->mutable_item());
value->Accept(&sub_visitor);
}
}
@@ -157,7 +156,7 @@ class PbSerializerVisitor : public RawValueVisitor {
pb_entry->set_arity(SerializePluralEnumToPb(i));
pb::Item* pb_element = pb_entry->mutable_item();
SerializeItemCommonToPb(*plural->values[i], pb_entry);
PbSerializerVisitor sub_visitor(source_pool_, symbol_pool_, pb_element);
PbSerializerVisitor sub_visitor(source_pool_, pb_element);
plural->values[i]->Accept(&sub_visitor);
}
}
@@ -179,8 +178,7 @@ class PbSerializerVisitor : public RawValueVisitor {
template <typename T>
void SerializeItemCommonToPb(const Item& item, T* pb_item) {
SerializeSourceToPb(item.GetSource(), source_pool_,
pb_item->mutable_source());
SerializeSourceToPb(item.GetSource(), source_pool_, pb_item->mutable_source());
if (!item.GetComment().empty()) {
pb_item->set_comment(item.GetComment());
}
@@ -192,8 +190,7 @@ class PbSerializerVisitor : public RawValueVisitor {
}
if (ref.name) {
StringPool::Ref symbol_ref = symbol_pool_->MakeRef(ref.name.value().ToString());
pb_ref->set_symbol_idx(static_cast<uint32_t>(symbol_ref.index()));
pb_ref->set_name(ref.name.value().ToString());
}
pb_ref->set_private_(ref.private_reference);
@@ -201,7 +198,6 @@ class PbSerializerVisitor : public RawValueVisitor {
}
StringPool* source_pool_;
StringPool* symbol_pool_;
pb::Value* out_pb_value_;
pb::Item* out_pb_item_;
};
@@ -220,9 +216,7 @@ std::unique_ptr<pb::ResourceTable> SerializeTableToPb(ResourceTable* table) {
});
auto pb_table = util::make_unique<pb::ResourceTable>();
SerializeStringPoolToPb(table->string_pool, pb_table->mutable_string_pool());
StringPool source_pool, symbol_pool;
StringPool source_pool;
for (auto& package : table->packages) {
pb::Package* pb_package = pb_table->add_packages();
@@ -270,7 +264,7 @@ std::unique_ptr<pb::ResourceTable> SerializeTableToPb(ResourceTable* table) {
pb_value->set_weak(true);
}
PbSerializerVisitor visitor(&source_pool, &symbol_pool, pb_value);
PbSerializerVisitor visitor(&source_pool, pb_value);
config_value->value->Accept(&visitor);
}
}
@@ -278,7 +272,6 @@ std::unique_ptr<pb::ResourceTable> SerializeTableToPb(ResourceTable* table) {
}
SerializeStringPoolToPb(source_pool, pb_table->mutable_source_pool());
SerializeStringPoolToPb(symbol_pool, pb_table->mutable_symbol_pool());
return pb_table;
}

View File

@@ -20,7 +20,9 @@
#include "test/Test.h"
using ::google::protobuf::io::StringOutputStream;
using ::testing::Eq;
using ::testing::NotNull;
using ::testing::SizeIs;
namespace aapt {
@@ -38,12 +40,12 @@ TEST(TableProtoSerializer, SerializeSinglePackage) {
Symbol public_symbol;
public_symbol.state = SymbolState::kPublic;
ASSERT_TRUE(table->SetSymbolState(
test::ParseNameOrDie("com.app.a:layout/main"), ResourceId(0x7f020000),
public_symbol, context->GetDiagnostics()));
ASSERT_TRUE(table->SetSymbolState(test::ParseNameOrDie("com.app.a:layout/main"),
ResourceId(0x7f020000), public_symbol,
context->GetDiagnostics()));
Id* id = test::GetValue<Id>(table.get(), "com.app.a:id/foo");
ASSERT_NE(nullptr, id);
ASSERT_THAT(id, NotNull());
// Make a plural.
std::unique_ptr<Plural> plural = util::make_unique<Plural>();
@@ -52,6 +54,15 @@ TEST(TableProtoSerializer, SerializeSinglePackage) {
ConfigDescription{}, {}, std::move(plural),
context->GetDiagnostics()));
// Make a styled string.
StyleString style_string;
style_string.str = "hello";
style_string.spans.push_back(Span{"b", 0u, 4u});
ASSERT_TRUE(
table->AddResource(test::ParseNameOrDie("com.app.a:string/styled"), ConfigDescription{}, {},
util::make_unique<StyledString>(table->string_pool.MakeRef(style_string)),
context->GetDiagnostics()));
// Make a resource with different products.
ASSERT_TRUE(table->AddResource(
test::ParseNameOrDie("com.app.a:integer/one"),
@@ -65,9 +76,8 @@ TEST(TableProtoSerializer, SerializeSinglePackage) {
context->GetDiagnostics()));
// Make a reference with both resource name and resource ID.
// The reference should point to a resource outside of this table to test that
// both
// name and id get serialized.
// The reference should point to a resource outside of this table to test that both name and id
// get serialized.
Reference expected_ref;
expected_ref.name = test::ParseNameOrDie("android:layout/main");
expected_ref.id = ResourceId(0x01020000);
@@ -85,36 +95,45 @@ TEST(TableProtoSerializer, SerializeSinglePackage) {
Id* new_id = test::GetValue<Id>(new_table.get(), "com.app.a:id/foo");
ASSERT_THAT(new_id, NotNull());
EXPECT_EQ(id->IsWeak(), new_id->IsWeak());
EXPECT_THAT(new_id->IsWeak(), Eq(id->IsWeak()));
Maybe<ResourceTable::SearchResult> result =
new_table->FindResource(test::ParseNameOrDie("com.app.a:layout/main"));
ASSERT_TRUE(result);
EXPECT_EQ(SymbolState::kPublic, result.value().type->symbol_status.state);
EXPECT_EQ(SymbolState::kPublic, result.value().entry->symbol_status.state);
EXPECT_THAT(result.value().type->symbol_status.state, Eq(SymbolState::kPublic));
EXPECT_THAT(result.value().entry->symbol_status.state, Eq(SymbolState::kPublic));
result = new_table->FindResource(test::ParseNameOrDie("com.app.a:bool/foo"));
ASSERT_TRUE(result);
EXPECT_EQ(SymbolState::kUndefined, result.value().entry->symbol_status.state);
EXPECT_THAT(result.value().entry->symbol_status.state, Eq(SymbolState::kUndefined));
EXPECT_TRUE(result.value().entry->symbol_status.allow_new);
// Find the product-dependent values
BinaryPrimitive* prim = test::GetValueForConfigAndProduct<BinaryPrimitive>(
new_table.get(), "com.app.a:integer/one", test::ParseConfigOrDie("land"), "");
ASSERT_THAT(prim, NotNull());
EXPECT_EQ(123u, prim->value.data);
EXPECT_THAT(prim->value.data, Eq(123u));
prim = test::GetValueForConfigAndProduct<BinaryPrimitive>(
new_table.get(), "com.app.a:integer/one", test::ParseConfigOrDie("land"), "tablet");
ASSERT_THAT(prim, NotNull());
EXPECT_EQ(321u, prim->value.data);
EXPECT_THAT(prim->value.data, Eq(321u));
Reference* actual_ref = test::GetValue<Reference>(new_table.get(), "com.app.a:layout/abc");
ASSERT_THAT(actual_ref, NotNull());
ASSERT_TRUE(actual_ref->name);
ASSERT_TRUE(actual_ref->id);
EXPECT_EQ(expected_ref.name.value(), actual_ref->name.value());
EXPECT_EQ(expected_ref.id.value(), actual_ref->id.value());
EXPECT_THAT(*actual_ref, Eq(expected_ref));
StyledString* actual_styled_str =
test::GetValue<StyledString>(new_table.get(), "com.app.a:string/styled");
ASSERT_THAT(actual_styled_str, NotNull());
EXPECT_THAT(actual_styled_str->value->value, Eq("hello"));
ASSERT_THAT(actual_styled_str->value->spans, SizeIs(1u));
EXPECT_THAT(*actual_styled_str->value->spans[0].name, Eq("b"));
EXPECT_THAT(actual_styled_str->value->spans[0].first_char, Eq(0u));
EXPECT_THAT(actual_styled_str->value->spans[0].last_char, Eq(4u));
}
TEST(TableProtoSerializer, SerializeFileHeader) {