Merge "AAPT2: Port AAPT pseudolocalization to AAPT2"

This commit is contained in:
Adam Lesinski
2016-01-06 23:55:06 +00:00
committed by Android (Google) Code Review
12 changed files with 1177 additions and 23 deletions

View File

@@ -27,6 +27,8 @@ main := Main.cpp
sources := \
compile/IdAssigner.cpp \
compile/Png.cpp \
compile/PseudolocaleGenerator.cpp \
compile/Pseudolocalizer.cpp \
compile/XmlIdCollector.cpp \
flatten/Archive.cpp \
flatten/TableFlattener.cpp \
@@ -66,6 +68,8 @@ sources := \
testSources := \
compile/IdAssigner_test.cpp \
compile/PseudolocaleGenerator_test.cpp \
compile/Pseudolocalizer_test.cpp \
compile/XmlIdCollector_test.cpp \
flatten/FileExportWriter_test.cpp \
flatten/TableFlattener_test.cpp \

View File

@@ -564,8 +564,10 @@ bool ResourceParser::parseString(xml::XmlPullParser* parser, ParsedResource* out
return false;
}
if (formatted && translateable) {
if (String* stringValue = valueCast<String>(outResource->value.get())) {
if (String* stringValue = valueCast<String>(outResource->value.get())) {
stringValue->setTranslateable(translateable);
if (formatted && translateable) {
if (!util::verifyJavaStringFormat(*stringValue->value)) {
mDiag->error(DiagMessage(outResource->source)
<< "multiple substitutions specified in non-positional format; "
@@ -573,6 +575,9 @@ bool ResourceParser::parseString(xml::XmlPullParser* parser, ParsedResource* out
return false;
}
}
} else if (StyledString* stringValue = valueCast<StyledString>(outResource->value.get())) {
stringValue->setTranslateable(translateable);
}
return true;
}

View File

@@ -36,10 +36,6 @@ void BaseItem<Derived>::accept(RawValueVisitor* visitor) {
visitor->visit(static_cast<Derived*>(this));
}
bool Value::isWeak() const {
return false;
}
RawString::RawString(const StringPool::Ref& ref) : value(ref) {
}
@@ -101,10 +97,6 @@ void Reference::print(std::ostream* out) const {
}
}
bool Id::isWeak() const {
return true;
}
bool Id::flatten(android::Res_value* out) const {
out->dataType = android::Res_value::TYPE_INT_BOOLEAN;
out->data = util::hostToDevice32(0);
@@ -119,7 +111,15 @@ void Id::print(std::ostream* out) const {
*out << "(id)";
}
String::String(const StringPool::Ref& ref) : value(ref) {
String::String(const StringPool::Ref& ref) : value(ref), mTranslateable(true) {
}
void String::setTranslateable(bool val) {
mTranslateable = val;
}
bool String::isTranslateable() const {
return mTranslateable;
}
bool String::flatten(android::Res_value* outValue) const {
@@ -144,7 +144,15 @@ void String::print(std::ostream* out) const {
*out << "(string) \"" << *value << "\"";
}
StyledString::StyledString(const StringPool::StyleRef& ref) : value(ref) {
StyledString::StyledString(const StringPool::StyleRef& ref) : value(ref), mTranslateable(true) {
}
void StyledString::setTranslateable(bool val) {
mTranslateable = val;
}
bool StyledString::isTranslateable() const {
return mTranslateable;
}
bool StyledString::flatten(android::Res_value* outValue) const {
@@ -238,13 +246,10 @@ void BinaryPrimitive::print(std::ostream* out) const {
}
Attribute::Attribute(bool w, uint32_t t) :
weak(w), typeMask(t),
typeMask(t),
minInt(std::numeric_limits<int32_t>::min()),
maxInt(std::numeric_limits<int32_t>::max()) {
}
bool Attribute::isWeak() const {
return weak;
mWeak = w;
}
Attribute* Attribute::clone(StringPool* /*newPool*/) const {
@@ -359,7 +364,7 @@ void Attribute::print(std::ostream* out) const {
<< "]";
}
if (weak) {
if (isWeak()) {
*out << " [weak]";
}
}

View File

@@ -43,9 +43,15 @@ struct Value {
/**
* Whether this value is weak and can be overridden without
* warning or error. Default for base class is false.
* warning or error. Default is false.
*/
virtual bool isWeak() const;
bool isWeak() const {
return mWeak;
}
void setWeak(bool val) {
mWeak = val;
}
/**
* Returns the source where this value was defined.
@@ -95,6 +101,7 @@ struct Value {
protected:
Source mSource;
std::u16string mComment;
bool mWeak = false;
};
/**
@@ -159,7 +166,7 @@ struct Reference : public BaseItem<Reference> {
* An ID resource. Has no real value, just a place holder.
*/
struct Id : public BaseItem<Id> {
bool isWeak() const override;
Id() { mWeak = true; }
bool flatten(android::Res_value* out) const override;
Id* clone(StringPool* newPool) const override;
void print(std::ostream* out) const override;
@@ -185,9 +192,17 @@ struct String : public BaseItem<String> {
String(const StringPool::Ref& ref);
// Whether the string is marked as translateable. This does not persist when flattened.
// It is only used during compilation phase.
void setTranslateable(bool val);
bool isTranslateable() const;
bool flatten(android::Res_value* outValue) const override;
String* clone(StringPool* newPool) const override;
void print(std::ostream* out) const override;
private:
bool mTranslateable;
};
struct StyledString : public BaseItem<StyledString> {
@@ -195,9 +210,17 @@ struct StyledString : public BaseItem<StyledString> {
StyledString(const StringPool::StyleRef& ref);
// Whether the string is marked as translateable. This does not persist when flattened.
// It is only used during compilation phase.
void setTranslateable(bool val);
bool isTranslateable() const;
bool flatten(android::Res_value* outValue) const override;
StyledString* clone(StringPool* newPool) const override;
void print(std::ostream* out) const override;
private:
bool mTranslateable;
};
struct FileReference : public BaseItem<FileReference> {
@@ -232,7 +255,6 @@ struct Attribute : public BaseValue<Attribute> {
uint32_t value;
};
bool weak;
uint32_t typeMask;
int32_t minInt;
int32_t maxInt;
@@ -240,7 +262,6 @@ struct Attribute : public BaseValue<Attribute> {
Attribute(bool w, uint32_t t = 0u);
bool isWeak() const override;
Attribute* clone(StringPool* newPool) const override;
void printMask(std::ostream* out) const;
void print(std::ostream* out) const override;

View File

@@ -21,6 +21,7 @@
#include "ResourceTable.h"
#include "compile/IdAssigner.h"
#include "compile/Png.h"
#include "compile/PseudolocaleGenerator.h"
#include "compile/XmlIdCollector.h"
#include "flatten/Archive.h"
#include "flatten/FileExportWriter.h"
@@ -105,6 +106,7 @@ struct CompileOptions {
std::string outputPath;
Maybe<std::string> resDir;
Maybe<std::u16string> product;
bool pseudolocalize = false;
bool verbose = false;
};
@@ -203,6 +205,16 @@ static bool compileTable(IAaptContext* context, const CompileOptions& options,
fin.close();
}
if (options.pseudolocalize) {
// Generate pseudo-localized strings (en-XA and ar-XB).
// These are created as weak symbols, and are only generated from default configuration
// strings and plurals.
PseudolocaleGenerator pseudolocaleGenerator;
if (!pseudolocaleGenerator.consume(context, &table)) {
return false;
}
}
// Ensure we have the compilation package at least.
table.createPackage(context->getCompilationPackage());
@@ -423,6 +435,8 @@ int compile(const std::vector<StringPiece>& args) {
.requiredFlag("-o", "Output path", &options.outputPath)
.optionalFlag("--product", "Product type to compile", &product)
.optionalFlag("--dir", "Directory to scan for resources", &options.resDir)
.optionalSwitch("--pseudo-localize", "Generate resources for pseudo-locales "
"(en-XA and ar-XB)", &options.pseudolocalize)
.optionalSwitch("-v", "Enables verbose logging", &options.verbose);
if (!flags.parse("aapt2 compile", args, &std::cerr)) {
return 1;

View File

@@ -0,0 +1,261 @@
/*
* Copyright (C) 2016 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "ResourceTable.h"
#include "ResourceValues.h"
#include "ValueVisitor.h"
#include "compile/PseudolocaleGenerator.h"
#include "compile/Pseudolocalizer.h"
#include "util/Comparators.h"
namespace aapt {
std::unique_ptr<StyledString> pseudolocalizeStyledString(StyledString* string,
Pseudolocalizer::Method method,
StringPool* pool) {
Pseudolocalizer localizer(method);
const StringPiece16 originalText = *string->value->str;
StyleString localized;
// Copy the spans. We will update their offsets when we localize.
localized.spans.reserve(string->value->spans.size());
for (const StringPool::Span& span : string->value->spans) {
localized.spans.push_back(Span{ *span.name, span.firstChar, span.lastChar });
}
// The ranges are all represented with a single value. This is the start of one range and
// end of another.
struct Range {
size_t start;
// Once the new string is localized, these are the pointers to the spans to adjust.
// Since this struct represents the start of one range and end of another, we have
// the two pointers respectively.
uint32_t* updateStart;
uint32_t* updateEnd;
};
auto cmp = [](const Range& r, size_t index) -> bool {
return r.start < index;
};
// Construct the ranges. The ranges are represented like so: [0, 2, 5, 7]
// The ranges are the spaces in between. In this example, with a total string length of 9,
// the vector represents: (0,1], (2,4], (5,6], (7,9]
//
std::vector<Range> ranges;
ranges.push_back(Range{ 0 });
ranges.push_back(Range{ originalText.size() - 1 });
for (size_t i = 0; i < string->value->spans.size(); i++) {
const StringPool::Span& span = string->value->spans[i];
// Insert or update the Range marker for the start of this span.
auto iter = std::lower_bound(ranges.begin(), ranges.end(), span.firstChar, cmp);
if (iter != ranges.end() && iter->start == span.firstChar) {
iter->updateStart = &localized.spans[i].firstChar;
} else {
ranges.insert(iter,
Range{ span.firstChar, &localized.spans[i].firstChar, nullptr });
}
// Insert or update the Range marker for the end of this span.
iter = std::lower_bound(ranges.begin(), ranges.end(), span.lastChar, cmp);
if (iter != ranges.end() && iter->start == span.lastChar) {
iter->updateEnd = &localized.spans[i].lastChar;
} else {
ranges.insert(iter,
Range{ span.lastChar, nullptr, &localized.spans[i].lastChar });
}
}
localized.str += localizer.start();
// Iterate over the ranges and localize each section.
for (size_t i = 0; i < ranges.size(); i++) {
const size_t start = ranges[i].start;
size_t len = originalText.size() - start;
if (i + 1 < ranges.size()) {
len = ranges[i + 1].start - start;
}
if (ranges[i].updateStart) {
*ranges[i].updateStart = localized.str.size();
}
if (ranges[i].updateEnd) {
*ranges[i].updateEnd = localized.str.size();
}
localized.str += localizer.text(originalText.substr(start, len));
}
localized.str += localizer.end();
std::unique_ptr<StyledString> localizedString = util::make_unique<StyledString>(
pool->makeRef(localized));
localizedString->setSource(string->getSource());
return localizedString;
}
namespace {
struct Visitor : public RawValueVisitor {
StringPool* mPool;
Pseudolocalizer::Method mMethod;
Pseudolocalizer mLocalizer;
// Either value or item will be populated upon visiting the value.
std::unique_ptr<Value> mValue;
std::unique_ptr<Item> mItem;
Visitor(StringPool* pool, Pseudolocalizer::Method method) :
mPool(pool), mMethod(method), mLocalizer(method) {
}
void visit(Array* array) override {
std::unique_ptr<Array> localized = util::make_unique<Array>();
localized->items.resize(array->items.size());
for (size_t i = 0; i < array->items.size(); i++) {
Visitor subVisitor(mPool, mMethod);
array->items[i]->accept(&subVisitor);
if (subVisitor.mItem) {
localized->items[i] = std::move(subVisitor.mItem);
} else {
localized->items[i] = std::unique_ptr<Item>(array->items[i]->clone(mPool));
}
}
localized->setSource(array->getSource());
localized->setWeak(true);
mValue = std::move(localized);
}
void visit(Plural* plural) override {
std::unique_ptr<Plural> localized = util::make_unique<Plural>();
for (size_t i = 0; i < plural->values.size(); i++) {
Visitor subVisitor(mPool, mMethod);
if (plural->values[i]) {
plural->values[i]->accept(&subVisitor);
if (subVisitor.mValue) {
localized->values[i] = std::move(subVisitor.mItem);
} else {
localized->values[i] = std::unique_ptr<Item>(plural->values[i]->clone(mPool));
}
}
}
localized->setSource(plural->getSource());
localized->setWeak(true);
mValue = std::move(localized);
}
void visit(String* string) override {
if (!string->isTranslateable()) {
return;
}
std::u16string result = mLocalizer.start() + mLocalizer.text(*string->value) +
mLocalizer.end();
std::unique_ptr<String> localized = util::make_unique<String>(mPool->makeRef(result));
localized->setSource(string->getSource());
localized->setWeak(true);
mItem = std::move(localized);
}
void visit(StyledString* string) override {
if (!string->isTranslateable()) {
return;
}
mItem = pseudolocalizeStyledString(string, mMethod, mPool);
mItem->setWeak(true);
}
};
ConfigDescription modifyConfigForPseudoLocale(const ConfigDescription& base,
Pseudolocalizer::Method m) {
ConfigDescription modified = base;
switch (m) {
case Pseudolocalizer::Method::kAccent:
modified.language[0] = 'e';
modified.language[1] = 'n';
modified.country[0] = 'X';
modified.country[1] = 'A';
break;
case Pseudolocalizer::Method::kBidi:
modified.language[0] = 'a';
modified.language[1] = 'r';
modified.country[0] = 'X';
modified.country[1] = 'B';
break;
default:
break;
}
return modified;
}
void pseudolocalizeIfNeeded(std::vector<ResourceConfigValue>* configValues,
Pseudolocalizer::Method method, StringPool* pool, Value* value) {
Visitor visitor(pool, method);
value->accept(&visitor);
std::unique_ptr<Value> localizedValue;
if (visitor.mValue) {
localizedValue = std::move(visitor.mValue);
} else if (visitor.mItem) {
localizedValue = std::move(visitor.mItem);
}
if (localizedValue) {
ConfigDescription pseudolocalizedConfig = modifyConfigForPseudoLocale(ConfigDescription{},
method);
auto iter = std::lower_bound(configValues->begin(), configValues->end(),
pseudolocalizedConfig, cmp::lessThanConfig);
if (iter == configValues->end() || iter->config != pseudolocalizedConfig) {
// The pseudolocalized config doesn't exist, add it.
configValues->insert(iter, ResourceConfigValue{ pseudolocalizedConfig,
std::move(localizedValue) });
}
}
}
} // namespace
bool PseudolocaleGenerator::consume(IAaptContext* context, ResourceTable* table) {
for (auto& package : table->packages) {
for (auto& type : package->types) {
for (auto& entry : type->entries) {
auto iter = std::lower_bound(entry->values.begin(), entry->values.end(),
ConfigDescription{}, cmp::lessThanConfig);
if (iter != entry->values.end() && iter->config == ConfigDescription{}) {
// Only pseudolocalize the default configuration.
// The iterator will be invalidated, so grab a pointer to the value.
Value* originalValue = iter->value.get();
pseudolocalizeIfNeeded(&entry->values, Pseudolocalizer::Method::kAccent,
&table->stringPool, originalValue);
pseudolocalizeIfNeeded(&entry->values, Pseudolocalizer::Method::kBidi,
&table->stringPool, originalValue);
}
}
}
}
return true;
}
} // namespace aapt

View File

@@ -0,0 +1,36 @@
/*
* Copyright (C) 2016 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef AAPT_COMPILE_PSEUDOLOCALEGENERATOR_H
#define AAPT_COMPILE_PSEUDOLOCALEGENERATOR_H
#include "StringPool.h"
#include "compile/Pseudolocalizer.h"
#include "process/IResourceTableConsumer.h"
namespace aapt {
std::unique_ptr<StyledString> pseudolocalizeStyledString(StyledString* string,
Pseudolocalizer::Method method,
StringPool* pool);
struct PseudolocaleGenerator : public IResourceTableConsumer {
bool consume(IAaptContext* context, ResourceTable* table) override;
};
} // namespace aapt
#endif /* AAPT_COMPILE_PSEUDOLOCALEGENERATOR_H */

View File

@@ -0,0 +1,123 @@
/*
* Copyright (C) 2016 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "compile/PseudolocaleGenerator.h"
#include "test/Builders.h"
#include "test/Common.h"
#include "test/Context.h"
#include "util/Util.h"
#include <androidfw/ResourceTypes.h>
#include <gtest/gtest.h>
namespace aapt {
TEST(PseudolocaleGeneratorTest, PseudolocalizeStyledString) {
StringPool pool;
StyleString originalStyle;
originalStyle.str = u"Hello world!";
originalStyle.spans = { Span{ u"b", 2, 3 }, Span{ u"b", 6, 7 }, Span{ u"i", 1, 10 } };
std::unique_ptr<StyledString> newString = pseudolocalizeStyledString(
util::make_unique<StyledString>(pool.makeRef(originalStyle)).get(),
Pseudolocalizer::Method::kNone, &pool);
EXPECT_EQ(originalStyle.str, *newString->value->str);
ASSERT_EQ(originalStyle.spans.size(), newString->value->spans.size());
EXPECT_EQ(2u, newString->value->spans[0].firstChar);
EXPECT_EQ(3u, newString->value->spans[0].lastChar);
EXPECT_EQ(std::u16string(u"b"), *newString->value->spans[0].name);
EXPECT_EQ(6u, newString->value->spans[1].firstChar);
EXPECT_EQ(7u, newString->value->spans[1].lastChar);
EXPECT_EQ(std::u16string(u"b"), *newString->value->spans[1].name);
EXPECT_EQ(1u, newString->value->spans[2].firstChar);
EXPECT_EQ(10u, newString->value->spans[2].lastChar);
EXPECT_EQ(std::u16string(u"i"), *newString->value->spans[2].name);
originalStyle.spans.push_back(Span{ u"em", 0, 11u });
newString = pseudolocalizeStyledString(
util::make_unique<StyledString>(pool.makeRef(originalStyle)).get(),
Pseudolocalizer::Method::kAccent, &pool);
EXPECT_EQ(std::u16string(u"[Ĥéļļö ŵöŕļð¡ one two]"), *newString->value->str);
ASSERT_EQ(originalStyle.spans.size(), newString->value->spans.size());
EXPECT_EQ(3u, newString->value->spans[0].firstChar);
EXPECT_EQ(4u, newString->value->spans[0].lastChar);
EXPECT_EQ(7u, newString->value->spans[1].firstChar);
EXPECT_EQ(8u, newString->value->spans[1].lastChar);
EXPECT_EQ(2u, newString->value->spans[2].firstChar);
EXPECT_EQ(11u, newString->value->spans[2].lastChar);
EXPECT_EQ(1u, newString->value->spans[3].firstChar);
EXPECT_EQ(12u, newString->value->spans[3].lastChar);
}
TEST(PseudolocaleGeneratorTest, PseudolocalizeOnlyDefaultConfigs) {
std::unique_ptr<ResourceTable> table = test::ResourceTableBuilder()
.addString(u"@android:string/one", u"one")
.addString(u"@android:string/two", ResourceId{}, test::parseConfigOrDie("en"), u"two")
.addString(u"@android:string/three", u"three")
.addString(u"@android:string/three", ResourceId{}, test::parseConfigOrDie("en-rXA"),
u"three")
.addString(u"@android:string/four", u"four")
.build();
String* val = test::getValue<String>(table.get(), u"@android:string/four");
val->setTranslateable(false);
std::unique_ptr<IAaptContext> context = test::ContextBuilder().build();
PseudolocaleGenerator generator;
ASSERT_TRUE(generator.consume(context.get(), table.get()));
// Normal pseudolocalization should take place.
ASSERT_NE(nullptr, test::getValueForConfig<String>(table.get(), u"@android:string/one",
test::parseConfigOrDie("en-rXA")));
ASSERT_NE(nullptr, test::getValueForConfig<String>(table.get(), u"@android:string/one",
test::parseConfigOrDie("ar-rXB")));
// No default config for android:string/two, so no pseudlocales should exist.
ASSERT_EQ(nullptr, test::getValueForConfig<String>(table.get(), u"@android:string/two",
test::parseConfigOrDie("en-rXA")));
ASSERT_EQ(nullptr, test::getValueForConfig<String>(table.get(), u"@android:string/two",
test::parseConfigOrDie("ar-rXB")));
// Check that we didn't override manual pseudolocalization.
val = test::getValueForConfig<String>(table.get(), u"@android:string/three",
test::parseConfigOrDie("en-rXA"));
ASSERT_NE(nullptr, val);
EXPECT_EQ(std::u16string(u"three"), *val->value);
ASSERT_NE(nullptr, test::getValueForConfig<String>(table.get(), u"@android:string/three",
test::parseConfigOrDie("ar-rXB")));
// Check that four's translateable marker was honored.
ASSERT_EQ(nullptr, test::getValueForConfig<String>(table.get(), u"@android:string/four",
test::parseConfigOrDie("en-rXA")));
ASSERT_EQ(nullptr, test::getValueForConfig<String>(table.get(), u"@android:string/four",
test::parseConfigOrDie("ar-rXB")));
}
} // namespace aapt

View File

@@ -0,0 +1,394 @@
/*
* Copyright (C) 2015 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "compile/Pseudolocalizer.h"
#include "util/Util.h"
namespace aapt {
// String basis to generate expansion
static const std::u16string k_expansion_string = u"one two three "
"four five six seven eight nine ten eleven twelve thirteen "
"fourteen fiveteen sixteen seventeen nineteen twenty";
// Special unicode characters to override directionality of the words
static const std::u16string k_rlm = u"\u200f";
static const std::u16string k_rlo = u"\u202e";
static const std::u16string k_pdf = u"\u202c";
// Placeholder marks
static const std::u16string k_placeholder_open = u"\u00bb";
static const std::u16string k_placeholder_close = u"\u00ab";
static const char16_t k_arg_start = u'{';
static const char16_t k_arg_end = u'}';
class PseudoMethodNone : public PseudoMethodImpl {
public:
std::u16string text(const StringPiece16& text) override { return text.toString(); }
std::u16string placeholder(const StringPiece16& text) override { return text.toString(); }
};
class PseudoMethodBidi : public PseudoMethodImpl {
public:
std::u16string text(const StringPiece16& text) override;
std::u16string placeholder(const StringPiece16& text) override;
};
class PseudoMethodAccent : public PseudoMethodImpl {
public:
PseudoMethodAccent() : mDepth(0), mWordCount(0), mLength(0) {}
std::u16string start() override;
std::u16string end() override;
std::u16string text(const StringPiece16& text) override;
std::u16string placeholder(const StringPiece16& text) override;
private:
size_t mDepth;
size_t mWordCount;
size_t mLength;
};
Pseudolocalizer::Pseudolocalizer(Method method) : mLastDepth(0) {
setMethod(method);
}
void Pseudolocalizer::setMethod(Method method) {
switch (method) {
case Method::kNone:
mImpl = util::make_unique<PseudoMethodNone>();
break;
case Method::kAccent:
mImpl = util::make_unique<PseudoMethodAccent>();
break;
case Method::kBidi:
mImpl = util::make_unique<PseudoMethodBidi>();
break;
}
}
std::u16string Pseudolocalizer::text(const StringPiece16& text) {
std::u16string out;
size_t depth = mLastDepth;
size_t lastpos, pos;
const size_t length = text.size();
const char16_t* str = text.data();
bool escaped = false;
for (lastpos = pos = 0; pos < length; pos++) {
char16_t c = str[pos];
if (escaped) {
escaped = false;
continue;
}
if (c == '\'') {
escaped = true;
continue;
}
if (c == k_arg_start) {
depth++;
} else if (c == k_arg_end && depth) {
depth--;
}
if (mLastDepth != depth || pos == length - 1) {
bool pseudo = ((mLastDepth % 2) == 0);
size_t nextpos = pos;
if (!pseudo || depth == mLastDepth) {
nextpos++;
}
size_t size = nextpos - lastpos;
if (size) {
std::u16string chunk = text.substr(lastpos, size).toString();
if (pseudo) {
chunk = mImpl->text(chunk);
} else if (str[lastpos] == k_arg_start && str[nextpos - 1] == k_arg_end) {
chunk = mImpl->placeholder(chunk);
}
out.append(chunk);
}
if (pseudo && depth < mLastDepth) { // End of message
out.append(mImpl->end());
} else if (!pseudo && depth > mLastDepth) { // Start of message
out.append(mImpl->start());
}
lastpos = nextpos;
mLastDepth = depth;
}
}
return out;
}
static const char16_t* pseudolocalizeChar(const char16_t c) {
switch (c) {
case 'a': return u"\u00e5";
case 'b': return u"\u0253";
case 'c': return u"\u00e7";
case 'd': return u"\u00f0";
case 'e': return u"\u00e9";
case 'f': return u"\u0192";
case 'g': return u"\u011d";
case 'h': return u"\u0125";
case 'i': return u"\u00ee";
case 'j': return u"\u0135";
case 'k': return u"\u0137";
case 'l': return u"\u013c";
case 'm': return u"\u1e3f";
case 'n': return u"\u00f1";
case 'o': return u"\u00f6";
case 'p': return u"\u00fe";
case 'q': return u"\u0051";
case 'r': return u"\u0155";
case 's': return u"\u0161";
case 't': return u"\u0163";
case 'u': return u"\u00fb";
case 'v': return u"\u0056";
case 'w': return u"\u0175";
case 'x': return u"\u0445";
case 'y': return u"\u00fd";
case 'z': return u"\u017e";
case 'A': return u"\u00c5";
case 'B': return u"\u03b2";
case 'C': return u"\u00c7";
case 'D': return u"\u00d0";
case 'E': return u"\u00c9";
case 'G': return u"\u011c";
case 'H': return u"\u0124";
case 'I': return u"\u00ce";
case 'J': return u"\u0134";
case 'K': return u"\u0136";
case 'L': return u"\u013b";
case 'M': return u"\u1e3e";
case 'N': return u"\u00d1";
case 'O': return u"\u00d6";
case 'P': return u"\u00de";
case 'Q': return u"\u0071";
case 'R': return u"\u0154";
case 'S': return u"\u0160";
case 'T': return u"\u0162";
case 'U': return u"\u00db";
case 'V': return u"\u03bd";
case 'W': return u"\u0174";
case 'X': return u"\u00d7";
case 'Y': return u"\u00dd";
case 'Z': return u"\u017d";
case '!': return u"\u00a1";
case '?': return u"\u00bf";
case '$': return u"\u20ac";
default: return NULL;
}
}
static bool isPossibleNormalPlaceholderEnd(const char16_t c) {
switch (c) {
case 's': return true;
case 'S': return true;
case 'c': return true;
case 'C': return true;
case 'd': return true;
case 'o': return true;
case 'x': return true;
case 'X': return true;
case 'f': return true;
case 'e': return true;
case 'E': return true;
case 'g': return true;
case 'G': return true;
case 'a': return true;
case 'A': return true;
case 'b': return true;
case 'B': return true;
case 'h': return true;
case 'H': return true;
case '%': return true;
case 'n': return true;
default: return false;
}
}
static std::u16string pseudoGenerateExpansion(const unsigned int length) {
std::u16string result = k_expansion_string;
const char16_t* s = result.data();
if (result.size() < length) {
result += u" ";
result += pseudoGenerateExpansion(length - result.size());
} else {
int ext = 0;
// Should contain only whole words, so looking for a space
for (unsigned int i = length + 1; i < result.size(); ++i) {
++ext;
if (s[i] == ' ') {
break;
}
}
result = result.substr(0, length + ext);
}
return result;
}
std::u16string PseudoMethodAccent::start() {
std::u16string result;
if (mDepth == 0) {
result = u"[";
}
mWordCount = mLength = 0;
mDepth++;
return result;
}
std::u16string PseudoMethodAccent::end() {
std::u16string result;
if (mLength) {
result += u" ";
result += pseudoGenerateExpansion(mWordCount > 3 ? mLength : mLength / 2);
}
mWordCount = mLength = 0;
mDepth--;
if (mDepth == 0) {
result += u"]";
}
return result;
}
/**
* Converts characters so they look like they've been localized.
*
* Note: This leaves placeholder syntax untouched.
*/
std::u16string PseudoMethodAccent::text(const StringPiece16& source)
{
const char16_t* s = source.data();
std::u16string result;
const size_t I = source.size();
bool lastspace = true;
for (size_t i = 0; i < I; i++) {
char16_t c = s[i];
if (c == '%') {
// Placeholder syntax, no need to pseudolocalize
std::u16string chunk;
bool end = false;
chunk.append(&c, 1);
while (!end && i < I) {
++i;
c = s[i];
chunk.append(&c, 1);
if (isPossibleNormalPlaceholderEnd(c)) {
end = true;
} else if (c == 't') {
++i;
c = s[i];
chunk.append(&c, 1);
end = true;
}
}
// Treat chunk as a placeholder unless it ends with %.
result += ((c == '%') ? chunk : placeholder(chunk));
} else if (c == '<' || c == '&') {
// html syntax, no need to pseudolocalize
bool tag_closed = false;
while (!tag_closed && i < I) {
if (c == '&') {
std::u16string escapeText;
escapeText.append(&c, 1);
bool end = false;
size_t htmlCodePos = i;
while (!end && htmlCodePos < I) {
++htmlCodePos;
c = s[htmlCodePos];
escapeText.append(&c, 1);
// Valid html code
if (c == ';') {
end = true;
i = htmlCodePos;
}
// Wrong html code
else if (!((c == '#' ||
(c >= 'a' && c <= 'z') ||
(c >= 'A' && c <= 'Z') ||
(c >= '0' && c <= '9')))) {
end = true;
}
}
result += escapeText;
if (escapeText != u"&lt;") {
tag_closed = true;
}
continue;
}
if (c == '>') {
tag_closed = true;
result.append(&c, 1);
continue;
}
result.append(&c, 1);
i++;
c = s[i];
}
} else {
// This is a pure text that should be pseudolocalized
const char16_t* p = pseudolocalizeChar(c);
if (p != nullptr) {
result += p;
} else {
bool space = util::isspace16(c);
if (lastspace && !space) {
mWordCount++;
}
lastspace = space;
result.append(&c, 1);
}
// Count only pseudolocalizable chars and delimiters
mLength++;
}
}
return result;
}
std::u16string PseudoMethodAccent::placeholder(const StringPiece16& source) {
// Surround a placeholder with brackets
return k_placeholder_open + source.toString() + k_placeholder_close;
}
std::u16string PseudoMethodBidi::text(const StringPiece16& source) {
const char16_t* s = source.data();
std::u16string result;
bool lastspace = true;
bool space = true;
for (size_t i = 0; i < source.size(); i++) {
char16_t c = s[i];
space = util::isspace16(c);
if (lastspace && !space) {
// Word start
result += k_rlm + k_rlo;
} else if (!lastspace && space) {
// Word end
result += k_pdf + k_rlm;
}
lastspace = space;
result.append(&c, 1);
}
if (!lastspace) {
// End of last word
result += k_pdf + k_rlm;
}
return result;
}
std::u16string PseudoMethodBidi::placeholder(const StringPiece16& source) {
// Surround a placeholder with directionality change sequence
return k_rlm + k_rlo + source.toString() + k_pdf + k_rlm;
}
} // namespace aapt

View File

@@ -0,0 +1,58 @@
/*
* Copyright (C) 2015 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef AAPT_COMPILE_PSEUDOLOCALIZE_H
#define AAPT_COMPILE_PSEUDOLOCALIZE_H
#include "ResourceValues.h"
#include "StringPool.h"
#include "util/StringPiece.h"
#include <android-base/macros.h>
#include <memory>
namespace aapt {
class PseudoMethodImpl {
public:
virtual ~PseudoMethodImpl() {}
virtual std::u16string start() { return {}; }
virtual std::u16string end() { return {}; }
virtual std::u16string text(const StringPiece16& text) = 0;
virtual std::u16string placeholder(const StringPiece16& text) = 0;
};
class Pseudolocalizer {
public:
enum class Method {
kNone,
kAccent,
kBidi,
};
Pseudolocalizer(Method method);
void setMethod(Method method);
std::u16string start() { return mImpl->start(); }
std::u16string end() { return mImpl->end(); }
std::u16string text(const StringPiece16& text);
private:
std::unique_ptr<PseudoMethodImpl> mImpl;
size_t mLastDepth;
};
} // namespace aapt
#endif /* AAPT_COMPILE_PSEUDOLOCALIZE_H */

View File

@@ -0,0 +1,227 @@
/*
* Copyright (C) 2015 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "compile/Pseudolocalizer.h"
#include "util/Util.h"
#include <androidfw/ResourceTypes.h>
#include <gtest/gtest.h>
namespace aapt {
// In this context, 'Axis' represents a particular field in the configuration,
// such as language or density.
static ::testing::AssertionResult simpleHelper(const char* input, const char* expected,
Pseudolocalizer::Method method) {
Pseudolocalizer pseudo(method);
std::string result = util::utf16ToUtf8(
pseudo.start() + pseudo.text(util::utf8ToUtf16(input)) + pseudo.end());
if (StringPiece(expected) != result) {
return ::testing::AssertionFailure() << expected << " != " << result;
}
return ::testing::AssertionSuccess();
}
static ::testing::AssertionResult compoundHelper(const char* in1, const char* in2, const char *in3,
const char* expected,
Pseudolocalizer::Method method) {
Pseudolocalizer pseudo(method);
std::string result = util::utf16ToUtf8(pseudo.start() +
pseudo.text(util::utf8ToUtf16(in1)) +
pseudo.text(util::utf8ToUtf16(in2)) +
pseudo.text(util::utf8ToUtf16(in3)) +
pseudo.end());
if (StringPiece(expected) != result) {
return ::testing::AssertionFailure() << expected << " != " << result;
}
return ::testing::AssertionSuccess();
}
TEST(PseudolocalizerTest, NoPseudolocalization) {
EXPECT_TRUE(simpleHelper("", "", Pseudolocalizer::Method::kNone));
EXPECT_TRUE(simpleHelper("Hello, world", "Hello, world", Pseudolocalizer::Method::kNone));
EXPECT_TRUE(compoundHelper("Hello,", " world", "",
"Hello, world", Pseudolocalizer::Method::kNone));
}
TEST(PseudolocalizerTest, PlaintextAccent) {
EXPECT_TRUE(simpleHelper("", "[]", Pseudolocalizer::Method::kAccent));
EXPECT_TRUE(simpleHelper("Hello, world",
"[Ĥéļļö, ŵöŕļð one two]", Pseudolocalizer::Method::kAccent));
EXPECT_TRUE(simpleHelper("Hello, %1d",
"[Ĥéļļö, »%1d« one two]", Pseudolocalizer::Method::kAccent));
EXPECT_TRUE(simpleHelper("Battery %1d%%",
"[βåţţéŕý »%1d«%% one two]", Pseudolocalizer::Method::kAccent));
EXPECT_TRUE(compoundHelper("", "", "", "[]", Pseudolocalizer::Method::kAccent));
EXPECT_TRUE(compoundHelper("Hello,", " world", "",
"[Ĥéļļö, ŵöŕļð one two]", Pseudolocalizer::Method::kAccent));
}
TEST(PseudolocalizerTest, PlaintextBidi) {
EXPECT_TRUE(simpleHelper("", "", Pseudolocalizer::Method::kBidi));
EXPECT_TRUE(simpleHelper("word",
"\xe2\x80\x8f\xE2\x80\xaeword\xE2\x80\xac\xe2\x80\x8f",
Pseudolocalizer::Method::kBidi));
EXPECT_TRUE(simpleHelper(" word ",
" \xe2\x80\x8f\xE2\x80\xaeword\xE2\x80\xac\xe2\x80\x8f ",
Pseudolocalizer::Method::kBidi));
EXPECT_TRUE(simpleHelper(" word ",
" \xe2\x80\x8f\xE2\x80\xaeword\xE2\x80\xac\xe2\x80\x8f ",
Pseudolocalizer::Method::kBidi));
EXPECT_TRUE(simpleHelper("hello\n world\n",
"\xe2\x80\x8f\xE2\x80\xaehello\xE2\x80\xac\xe2\x80\x8f\n" \
" \xe2\x80\x8f\xE2\x80\xaeworld\xE2\x80\xac\xe2\x80\x8f\n",
Pseudolocalizer::Method::kBidi));
EXPECT_TRUE(compoundHelper("hello", "\n ", " world\n",
"\xe2\x80\x8f\xE2\x80\xaehello\xE2\x80\xac\xe2\x80\x8f\n" \
" \xe2\x80\x8f\xE2\x80\xaeworld\xE2\x80\xac\xe2\x80\x8f\n",
Pseudolocalizer::Method::kBidi));
}
TEST(PseudolocalizerTest, SimpleICU) {
// Single-fragment messages
EXPECT_TRUE(simpleHelper("{placeholder}", "[»{placeholder}«]",
Pseudolocalizer::Method::kAccent));
EXPECT_TRUE(simpleHelper("{USER} is offline",
"[»{USER}« îš öƒƒļîñé one two]", Pseudolocalizer::Method::kAccent));
EXPECT_TRUE(simpleHelper("Copy from {path1} to {path2}",
"[Çöþý ƒŕöḿ »{path1}« ţö »{path2}« one two three]",
Pseudolocalizer::Method::kAccent));
EXPECT_TRUE(simpleHelper("Today is {1,date} {1,time}",
"[Ţöðåý îš »{1,date}« »{1,time}« one two]",
Pseudolocalizer::Method::kAccent));
// Multi-fragment messages
EXPECT_TRUE(compoundHelper("{USER}", " ", "is offline",
"[»{USER}« îš öƒƒļîñé one two]",
Pseudolocalizer::Method::kAccent));
EXPECT_TRUE(compoundHelper("Copy from ", "{path1}", " to {path2}",
"[Çöþý ƒŕöḿ »{path1}« ţö »{path2}« one two three]",
Pseudolocalizer::Method::kAccent));
}
TEST(PseudolocalizerTest, ICUBidi) {
// Single-fragment messages
EXPECT_TRUE(simpleHelper("{placeholder}",
"\xe2\x80\x8f\xE2\x80\xae{placeholder}\xE2\x80\xac\xe2\x80\x8f",
Pseudolocalizer::Method::kBidi));
EXPECT_TRUE(simpleHelper(
"{COUNT, plural, one {one} other {other}}",
"{COUNT, plural, " \
"one {\xe2\x80\x8f\xE2\x80\xaeone\xE2\x80\xac\xe2\x80\x8f} " \
"other {\xe2\x80\x8f\xE2\x80\xaeother\xE2\x80\xac\xe2\x80\x8f}}",
Pseudolocalizer::Method::kBidi));
}
TEST(PseudolocalizerTest, Escaping) {
// Single-fragment messages
EXPECT_TRUE(simpleHelper("'{USER'} is offline",
"['{ÛŠÉŔ'} îš öƒƒļîñé one two three]",
Pseudolocalizer::Method::kAccent));
// Multi-fragment messages
EXPECT_TRUE(compoundHelper("'{USER}", " ", "''is offline",
"['{ÛŠÉŔ} ''îš öƒƒļîñé one two three]",
Pseudolocalizer::Method::kAccent));
}
TEST(PseudolocalizerTest, PluralsAndSelects) {
EXPECT_TRUE(simpleHelper(
"{COUNT, plural, one {Delete a file} other {Delete {COUNT} files}}",
"[{COUNT, plural, one {Ðéļéţé å ƒîļé one two} " \
"other {Ðéļéţé »{COUNT}« ƒîļéš one two}}]",
Pseudolocalizer::Method::kAccent));
EXPECT_TRUE(simpleHelper(
"Distance is {COUNT, plural, one {# mile} other {# miles}}",
"[Ðîšţåñçé îš {COUNT, plural, one {# ḿîļé one two} " \
"other {# ḿîļéš one two}}]",
Pseudolocalizer::Method::kAccent));
EXPECT_TRUE(simpleHelper(
"{1, select, female {{1} added you} " \
"male {{1} added you} other {{1} added you}}",
"[{1, select, female {»{1}« åððéð ýöû one two} " \
"male {»{1}« åððéð ýöû one two} other {»{1}« åððéð ýöû one two}}]",
Pseudolocalizer::Method::kAccent));
EXPECT_TRUE(compoundHelper(
"{COUNT, plural, one {Delete a file} " \
"other {Delete ", "{COUNT}", " files}}",
"[{COUNT, plural, one {Ðéļéţé å ƒîļé one two} " \
"other {Ðéļéţé »{COUNT}« ƒîļéš one two}}]",
Pseudolocalizer::Method::kAccent));
}
TEST(PseudolocalizerTest, NestedICU) {
EXPECT_TRUE(simpleHelper(
"{person, select, " \
"female {" \
"{num_circles, plural," \
"=0{{person} didn't add you to any of her circles.}" \
"=1{{person} added you to one of her circles.}" \
"other{{person} added you to her # circles.}}}" \
"male {" \
"{num_circles, plural," \
"=0{{person} didn't add you to any of his circles.}" \
"=1{{person} added you to one of his circles.}" \
"other{{person} added you to his # circles.}}}" \
"other {" \
"{num_circles, plural," \
"=0{{person} didn't add you to any of their circles.}" \
"=1{{person} added you to one of their circles.}" \
"other{{person} added you to their # circles.}}}}",
"[{person, select, " \
"female {" \
"{num_circles, plural," \
"=0{»{person}« ðîðñ'ţ åðð ýöû ţö åñý öƒ ĥéŕ çîŕçļéš." \
" one two three four five}" \
"=1{»{person}« åððéð ýöû ţö öñé öƒ ĥéŕ çîŕçļéš." \
" one two three four}" \
"other{»{person}« åððéð ýöû ţö ĥéŕ # çîŕçļéš." \
" one two three four}}}" \
"male {" \
"{num_circles, plural," \
"=0{»{person}« ðîðñ'ţ åðð ýöû ţö åñý öƒ ĥîš çîŕçļéš." \
" one two three four five}" \
"=1{»{person}« åððéð ýöû ţö öñé öƒ ĥîš çîŕçļéš." \
" one two three four}" \
"other{»{person}« åððéð ýöû ţö ĥîš # çîŕçļéš." \
" one two three four}}}" \
"other {{num_circles, plural," \
"=0{»{person}« ðîðñ'ţ åðð ýöû ţö åñý öƒ ţĥéîŕ çîŕçļéš." \
" one two three four five}" \
"=1{»{person}« åððéð ýöû ţö öñé öƒ ţĥéîŕ çîŕçļéš." \
" one two three four}" \
"other{»{person}« åððéð ýöû ţö ţĥéîŕ # çîŕçļéš." \
" one two three four}}}}]",
Pseudolocalizer::Method::kAccent));
}
TEST(PseudolocalizerTest, RedefineMethod) {
Pseudolocalizer pseudo(Pseudolocalizer::Method::kAccent);
std::u16string result = pseudo.text(u"Hello, ");
pseudo.setMethod(Pseudolocalizer::Method::kNone);
result += pseudo.text(u"world!");
ASSERT_EQ(StringPiece("Ĥéļļö, world!"), util::utf16ToUtf8(result));
}
} // namespace aapt

View File

@@ -68,6 +68,12 @@ public:
return addValue(name, id, util::make_unique<String>(mTable->stringPool.makeRef(str)));
}
ResourceTableBuilder& addString(const StringPiece16& name, const ResourceId id,
const ConfigDescription& config, const StringPiece16& str) {
return addValue(name, id, config,
util::make_unique<String>(mTable->stringPool.makeRef(str)));
}
ResourceTableBuilder& addFileReference(const StringPiece16& name, const StringPiece16& path) {
return addFileReference(name, {}, path);
}