Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge "Implement smarter locale resource selection"

Browse Source
This commit is contained in:
Roozbeh Pournader
2016-01-21 22:17:31 +00:00
committed by Android (Google) Code Review
parent 6f0324315b b927c559e1
commit 2d05094209
11 changed files with 2785 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
core/java/android/util/LocaleList.java
View File

@@ -301,10 +301,19 @@ public final class LocaleList implements Parcelable {
// is a pseudo-locale. So this is not a match.
return 0;
}
final String supportedScr = getLikelyScript(supported);
if (supportedScr.isEmpty()) {
// If we can't guess a script, we don't know enough about the locales' language to find
// if the locales match. So we fall back to old behavior of matching, which considered
// locales with different regions different.
final String supportedRegion = supported.getCountry();
return (supportedRegion.isEmpty() ||
supportedRegion.equals(desired.getCountry()))
? 1 : 0;
}
final String desiredScr = getLikelyScript(desired);
// There is no match if the two locales use different scripts. This will most imporantly
// take care of traditional vs simplified Chinese.
final String supportedScr = getLikelyScript(supported);
final String desiredScr = getLikelyScript(desired);
return supportedScr.equals(desiredScr) ? 1 : 0;
}

34
include/androidfw/LocaleData.h Normal file
View File

@@ -0,0 +1,34 @@
/*
* Copyright (C) 2016 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef _LIBS_UTILS_LOCALE_DATA_H
#define _LIBS_UTILS_LOCALE_DATA_H
#include <stddef.h>
#include <stdint.h>
namespace android {
int localeDataCompareRegions(
const char* left_region, const char* right_region,
const char* requested_language, const char* requested_script,
const char* requested_region);
void localeDataComputeScript(char out[4], const char* language, const char* region);
} // namespace android
#endif // _LIBS_UTILS_LOCALE_DATA_H

15
include/androidfw/ResourceTypes.h
View File

@@ -21,6 +21,7 @@
#define _LIBS_UTILS_RESOURCE_TYPES_H
#include <androidfw/Asset.h>
#include <androidfw/LocaleData.h>
#include <utils/ByteOrder.h>
#include <utils/Errors.h>
#include <utils/String16.h>
@@ -1127,8 +1128,9 @@ struct ResTable_config
// configuration. (eg. Hant, Latn, etc.). Interpreted in conjunction with
// the locale field.
char localeScript[4];
bool localeScriptWasProvided;
// A single BCP-47 variant subtag. Will vary in length between 5 and 8
// A single BCP-47 variant subtag. Will vary in length between 4 and 8
// chars. Interpreted in conjunction with the locale field.
char localeVariant[8];
@@ -1228,10 +1230,15 @@ struct ResTable_config
inline void clearLocale() {
locale = 0;
localeScriptWasProvided = false;
memset(localeScript, 0, sizeof(localeScript));
memset(localeVariant, 0, sizeof(localeVariant));
}
inline void computeScript() {
localeDataComputeScript(localeScript, language, country);
}
// Get the 2 or 3 letter language code of this configuration. Trailing
// bytes are set to '\0'.
size_t unpackLanguage(char language[4]) const;
@@ -1255,6 +1262,12 @@ struct ResTable_config
// and 0 if they're equally specific.
int isLocaleMoreSpecificThan(const ResTable_config &o) const;
// Return true if 'this' is a better locale match than 'o' for the
// 'requested' configuration. Similar to isBetterThan(), this assumes that
// match() has already been used to remove any configurations that don't
// match the requested configuration at all.
bool isLocaleBetterThan(const ResTable_config& o, const ResTable_config* requested) const;
String8 toString() const;
};

1
libs/androidfw/Android.mk
View File

@@ -21,6 +21,7 @@ commonSources := \
Asset.cpp \
AssetDir.cpp \
AssetManager.cpp \
LocaleData.cpp \
misc.cpp \
ObbFile.cpp \
ResourceTypes.cpp \

201
libs/androidfw/LocaleData.cpp Normal file
View File

@@ -0,0 +1,201 @@
/*
* Copyright (C) 2016 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <cstdint>
#include <cstdlib>
#include <cstring>
#include <string>
#include <unordered_map>
#include <unordered_set>
#include <androidfw/LocaleData.h>
namespace android {
#include "LocaleDataTables.cpp"
inline uint32_t packLocale(const char* language, const char* region) {
return (((uint8_t) language[0]) << 24u) | (((uint8_t) language[1]) << 16u) |
(((uint8_t) region[0]) << 8u) | ((uint8_t) region[1]);
}
inline uint32_t dropRegion(uint32_t packed_locale) {
return packed_locale & 0xFFFF0000lu;
}
inline bool hasRegion(uint32_t packed_locale) {
return (packed_locale & 0x0000FFFFlu) != 0;
}
const size_t SCRIPT_LENGTH = 4;
const size_t SCRIPT_PARENTS_COUNT = sizeof(SCRIPT_PARENTS)/sizeof(SCRIPT_PARENTS[0]);
const uint32_t PACKED_ROOT = 0; // to represent the root locale
uint32_t findParent(uint32_t packed_locale, const char* script) {
if (hasRegion(packed_locale)) {
for (size_t i = 0; i < SCRIPT_PARENTS_COUNT; i++) {
if (memcmp(script, SCRIPT_PARENTS[i].script, SCRIPT_LENGTH) == 0) {
auto map = SCRIPT_PARENTS[i].map;
auto lookup_result = map->find(packed_locale);
if (lookup_result != map->end()) {
return lookup_result->second;
}
break;
}
}
return dropRegion(packed_locale);
}
return PACKED_ROOT;
}
// Find the ancestors of a locale, and fill 'out' with it (assumes out has enough
// space). If any of the members of stop_list was seen, write it in the
// output but stop afterwards.
//
// This also outputs the index of the last written ancestor in the stop_list
// to stop_list_index, which will be -1 if it is not found in the stop_list.
//
// Returns the number of ancestors written in the output, which is always
// at least one.
size_t findAncestors(uint32_t* out, ssize_t* stop_list_index,
uint32_t packed_locale, const char* script,
const uint32_t* stop_list, size_t stop_set_length) {
uint32_t ancestor = packed_locale;
size_t count = 0;
do {
out[count++] = ancestor;
for (size_t i = 0; i < stop_set_length; i++) {
if (stop_list[i] == ancestor) {
*stop_list_index = (ssize_t) i;
return count;
}
}
ancestor = findParent(ancestor, script);
} while (ancestor != PACKED_ROOT);
*stop_list_index = (ssize_t) -1;
return count;
}
size_t findDistance(uint32_t supported,
const char* script,
const uint32_t* request_ancestors,
size_t request_ancestors_count) {
uint32_t supported_ancestors[MAX_PARENT_DEPTH+1];
ssize_t request_ancestors_index;
const size_t supported_ancestor_count = findAncestors(
supported_ancestors, &request_ancestors_index,
supported, script,
request_ancestors, request_ancestors_count);
// Since both locales share the same root, there will always be a shared
// ancestor, so the distance in the parent tree is the sum of the distance
// of 'supported' to the lowest common ancestor (number of ancestors
// written for 'supported' minus 1) plus the distance of 'request' to the
// lowest common ancestor (the index of the ancestor in request_ancestors).
return supported_ancestor_count + request_ancestors_index - 1;
}
inline bool isRepresentative(uint32_t language_and_region, const char* script) {
const uint64_t packed_locale = (
(((uint64_t) language_and_region) << 32u) |
(((uint64_t) script[0]) << 24u) |
(((uint64_t) script[1]) << 16u) |
(((uint64_t) script[2]) << 8u) |
((uint64_t) script[3]));
return (REPRESENTATIVE_LOCALES.count(packed_locale) != 0);
}
int localeDataCompareRegions(
const char* left_region, const char* right_region,
const char* requested_language, const char* requested_script,
const char* requested_region) {
if (left_region[0] == right_region[0] && left_region[1] == right_region[1]) {
return 0;
}
const uint32_t left = packLocale(requested_language, left_region);
const uint32_t right = packLocale(requested_language, right_region);
const uint32_t request = packLocale(requested_language, requested_region);
uint32_t request_ancestors[MAX_PARENT_DEPTH+1];
ssize_t left_right_index;
// Find the parents of the request, but stop as soon as we saw left or right
const uint32_t left_and_right[] = {left, right};
const size_t ancestor_count = findAncestors(
request_ancestors, &left_right_index,
request, requested_script,
left_and_right, sizeof(left_and_right)/sizeof(left_and_right[0]));
if (left_right_index == 0) { // We saw left earlier
return 1;
}
if (left_right_index == 1) { // We saw right earlier
return -1;
}
// If we are here, neither left nor right are an ancestor of the
// request. This means that all the ancestors have been computed and
// the last ancestor is just the language by itself. We will use the
// distance in the parent tree for determining the better match.
const size_t left_distance = findDistance(
left, requested_script, request_ancestors, ancestor_count);
const size_t right_distance = findDistance(
right, requested_script, request_ancestors, ancestor_count);
if (left_distance != right_distance) {
return (int) right_distance - (int) left_distance; // smaller distance is better
}
// If we are here, left and right are equidistant from the request. We will
// try and see if any of them is a representative locale.
const bool left_is_representative = isRepresentative(left, requested_script);
const bool right_is_representative = isRepresentative(right, requested_script);
if (left_is_representative != right_is_representative) {
return (int) left_is_representative - (int) right_is_representative;
}
// We have no way of figuring out which locale is a better match. For
// the sake of stability, we consider the locale with the lower region
// code (in dictionary order) better, with two-letter codes before
// three-digit codes (since two-letter codes are more specific).
return (int64_t) right - (int64_t) left;
}
void localeDataComputeScript(char out[4], const char* language, const char* region) {
if (language[0] == '\0') {
memset(out, '\0', SCRIPT_LENGTH);
return;
}
uint32_t lookup_key = packLocale(language, region);
auto lookup_result = LIKELY_SCRIPTS.find(lookup_key);
if (lookup_result == LIKELY_SCRIPTS.end()) {
// We couldn't find the locale. Let's try without the region
if (region[0] != '\0') {
lookup_key = dropRegion(lookup_key);
lookup_result = LIKELY_SCRIPTS.find(lookup_key);
if (lookup_result != LIKELY_SCRIPTS.end()) {
memcpy(out, SCRIPT_CODES[lookup_result->second], SCRIPT_LENGTH);
return;
}
}
// We don't know anything about the locale
memset(out, '\0', SCRIPT_LENGTH);
return;
} else {
// We found the locale.
memcpy(out, SCRIPT_CODES[lookup_result->second], SCRIPT_LENGTH);
}
}
} // namespace android

1701
libs/androidfw/LocaleDataTables.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

148
libs/androidfw/ResourceTypes.cpp
View File

@@ -1868,7 +1868,10 @@ void ResTable_config::swapHtoD() {
}
// The language & region are equal, so compare the scripts and variants.
int script = memcmp(l.localeScript, r.localeScript, sizeof(l.localeScript));
const char emptyScript[sizeof(l.localeScript)] = {'\0', '\0', '\0', '\0'};
const char *lScript = l.localeScriptWasProvided ? l.localeScript : emptyScript;
const char *rScript = r.localeScriptWasProvided ? r.localeScript : emptyScript;
int script = memcmp(lScript, rScript, sizeof(l.localeScript));
if (script) {
return script;
}
@@ -2012,10 +2015,10 @@ int ResTable_config::isLocaleMoreSpecificThan(const ResTable_config& o) const {
// scripts since it seems more useful to do so. We will consider
// "en-US-POSIX" to be more specific than "en-Latn-US".
const int score = ((localeScript[0] != 0) ? 1 : 0) +
const int score = (localeScriptWasProvided ? 1 : 0) +
((localeVariant[0] != 0) ? 2 : 0);
const int oScore = ((o.localeScript[0] != 0) ? 1 : 0) +
const int oScore = (o.localeScriptWasProvided ? 1 : 0) +
((o.localeVariant[0] != 0) ? 2 : 0);
return score - oScore;
@@ -2165,6 +2168,63 @@ bool ResTable_config::isMoreSpecificThan(const ResTable_config& o) const {
return false;
}
bool ResTable_config::isLocaleBetterThan(const ResTable_config& o,
const ResTable_config* requested) const {
if (requested->locale == 0) {
// The request doesn't have a locale, so no resource is better
// than the other.
return false;
}
if (locale == 0 && o.locale == 0) {
// The locales parts of both resources are empty, so no one is better
// than the other.
return false;
}
// Non-matching locales have been filtered out, so both resources
// match the requested locale.
//
// Because of the locale-related checks in match() and the checks, we know
// that:
// 1) The resource languages are either empty or match the request;
// and
// 2) If the request's script is known, the resource scripts are either
// unknown or match the request.
if (language[0] != o.language[0]) {
// The languages of the two resources are not the same. We can only
// assume that one of the two resources matched the request because one
// doesn't have a language and the other has a matching language.
return (language[0] != 0);
}
// If we are here, both the resources have the same non-empty language as
// the request.
//
// Because the languages are the same, computeScript() always
// returns a non-empty script for languages it knows about, and we have passed
// the script checks in match(), the scripts are either all unknown or are
// all the same. So we can't gain anything by checking the scripts. We need
// to check the region and variant.
// See if any of the regions is better than the other
const int region_comparison = localeDataCompareRegions(
country, o.country,
language, localeScript, requested->country);
if (region_comparison != 0) {
return (region_comparison > 0);
}
// The regions are the same. Try the variant.
if (requested->localeVariant[0] != '\0'
&& strncmp(localeVariant, requested->localeVariant, sizeof(localeVariant)) == 0) {
return (strncmp(o.localeVariant, requested->localeVariant, sizeof(localeVariant)) != 0);
}
return false;
}
bool ResTable_config::isBetterThan(const ResTable_config& o,
const ResTable_config* requested) const {
if (requested) {
@@ -2178,26 +2238,8 @@ bool ResTable_config::isBetterThan(const ResTable_config& o,
}
}
if (locale || o.locale) {
if ((language[0] != o.language[0]) && requested->language[0]) {
return (language[0]);
}
if ((country[0] != o.country[0]) && requested->country[0]) {
return (country[0]);
}
}
if (localeScript[0] || o.localeScript[0]) {
if (localeScript[0] != o.localeScript[0] && requested->localeScript[0]) {
return localeScript[0];
}
}
if (localeVariant[0] || o.localeVariant[0]) {
if (localeVariant[0] != o.localeVariant[0] && requested->localeVariant[0]) {
return localeVariant[0];
}
if (isLocaleBetterThan(o, requested)) {
return true;
}
if (screenLayout || o.screenLayout) {
@@ -2445,20 +2487,33 @@ bool ResTable_config::match(const ResTable_config& settings) const {
}
}
if (locale != 0) {
// Don't consider the script & variants when deciding matches.
// Don't consider country and variants when deciding matches.
// (Theoretically, the variant can also affect the script. For
// example, "ar-alalc97" probably implies the Latin script, but since
// CLDR doesn't support getting likely scripts for that, we'll assume
// the variant doesn't change the script.)
//
// If we two configs differ only in their script or language, they
// can be weeded out in the isMoreSpecificThan test.
if (language[0] != 0
&& (language[0] != settings.language[0]
|| language[1] != settings.language[1])) {
// If two configs differ only in their country and variant,
// they can be weeded out in the isMoreSpecificThan test.
if (language[0] != settings.language[0] || language[1] != settings.language[1]) {
return false;
}
if (country[0] != 0
&& (country[0] != settings.country[0]
|| country[1] != settings.country[1])) {
return false;
// For backward compatibility and supporting private-use locales, we
// fall back to old behavior if we couldn't determine the script for
// either of the desired locale or the provided locale.
if (localeScript[0] == '\0' || localeScript[1] == '\0') {
if (country[0] != '\0'
&& (country[0] != settings.country[0]
|| country[1] != settings.country[1])) {
return false;
}
} else {
// But if we could determine the scripts, they should be the same
// for the locales to match.
if (memcmp(localeScript, settings.localeScript, sizeof(localeScript)) != 0) {
return false;
}
}
}
@@ -2587,7 +2642,7 @@ void ResTable_config::appendDirLocale(String8& out) const {
return;
}
if (!localeScript[0] && !localeVariant[0]) {
if (!localeScriptWasProvided && !localeVariant[0]) {
// Legacy format.
if (out.size() > 0) {
out.append("-");
@@ -2605,7 +2660,7 @@ void ResTable_config::appendDirLocale(String8& out) const {
return;
}
// We are writing the modified bcp47 tag.
// We are writing the modified BCP 47 tag.
// It starts with 'b+' and uses '+' as a separator.
if (out.size() > 0) {
@@ -2617,7 +2672,7 @@ void ResTable_config::appendDirLocale(String8& out) const {
size_t len = unpackLanguage(buf);
out.append(buf, len);
if (localeScript[0]) {
if (localeScriptWasProvided) {
out.append("+");
out.append(localeScript, sizeof(localeScript));
}
@@ -2630,7 +2685,7 @@ void ResTable_config::appendDirLocale(String8& out) const {
if (localeVariant[0]) {
out.append("+");
out.append(localeVariant, sizeof(localeVariant));
out.append(localeVariant, strnlen(localeVariant, sizeof(localeVariant)));
}
}
@@ -2648,7 +2703,7 @@ void ResTable_config::getBcp47Locale(char str[RESTABLE_MAX_LOCALE_LEN]) const {
charsWritten += unpackLanguage(str);
}
if (localeScript[0]) {
if (localeScriptWasProvided) {
if (charsWritten) {
str[charsWritten++] = '-';
}
@@ -2682,11 +2737,16 @@ void ResTable_config::getBcp47Locale(char str[RESTABLE_MAX_LOCALE_LEN]) const {
config->language[0] ? config->packRegion(start) : config->packLanguage(start);
break;
case 4:
config->localeScript[0] = toupper(start[0]);
for (size_t i = 1; i < 4; ++i) {
config->localeScript[i] = tolower(start[i]);
if ('0' <= start[0] && start[0] <= '9') {
// this is a variant, so fall through
} else {
config->localeScript[0] = toupper(start[0]);
for (size_t i = 1; i < 4; ++i) {
config->localeScript[i] = tolower(start[i]);
}
config->localeScriptWasProvided = true;
break;
}
break;
case 5:
case 6:
case 7:
@@ -2704,6 +2764,7 @@ void ResTable_config::getBcp47Locale(char str[RESTABLE_MAX_LOCALE_LEN]) const {
void ResTable_config::setBcp47Locale(const char* in) {
locale = 0;
localeScriptWasProvided = false;
memset(localeScript, 0, sizeof(localeScript));
memset(localeVariant, 0, sizeof(localeVariant));
@@ -2720,6 +2781,9 @@ void ResTable_config::setBcp47Locale(const char* in) {
const size_t size = in + strlen(in) - start;
assignLocaleComponent(this, start, size);
if (localeScript[0] == '\0') {
computeScript();
};
}
String8 ResTable_config::toString() const {

408
libs/androidfw/tests/ConfigLocale_test.cpp
View File

@@ -14,6 +14,7 @@
* limitations under the License.
*/
#include <androidfw/LocaleData.h>
#include <androidfw/ResourceTypes.h>
#include <utils/Log.h>
#include <utils/String8.h>
@@ -28,7 +29,7 @@ TEST(ConfigLocaleTest, packAndUnpack2LetterLanguage) {
EXPECT_EQ('e', config.language[0]);
EXPECT_EQ('n', config.language[1]);
char out[4] = { 1, 1, 1, 1};
char out[4] = {1, 1, 1, 1};
config.unpackLanguage(out);
EXPECT_EQ('e', out[0]);
EXPECT_EQ('n', out[1]);
@@ -51,7 +52,7 @@ TEST(ConfigLocaleTest, packAndUnpack2LetterRegion) {
EXPECT_EQ('U', config.country[0]);
EXPECT_EQ('S', config.country[1]);
char out[4] = { 1, 1, 1, 1};
char out[4] = {1, 1, 1, 1};
config.unpackRegion(out);
EXPECT_EQ('U', out[0]);
EXPECT_EQ('S', out[1]);
@@ -67,7 +68,7 @@ TEST(ConfigLocaleTest, packAndUnpack3LetterLanguage) {
EXPECT_EQ('\x99', config.language[0]);
EXPECT_EQ('\xA4', config.language[1]);
char out[4] = { 1, 1, 1, 1};
char out[4] = {1, 1, 1, 1};
config.unpackLanguage(out);
EXPECT_EQ('e', out[0]);
EXPECT_EQ('n', out[1]);
@@ -91,7 +92,7 @@ TEST(ConfigLocaleTest, packAndUnpack3LetterLanguageAtOffset16) {
EXPECT_EQ(char(0xbc), config.language[0]);
EXPECT_EQ(char(0xd3), config.language[1]);
char out[4] = { 1, 1, 1, 1};
char out[4] = {1, 1, 1, 1};
config.unpackLanguage(out);
EXPECT_EQ('t', out[0]);
EXPECT_EQ('g', out[1]);
@@ -103,7 +104,7 @@ TEST(ConfigLocaleTest, packAndUnpack3LetterRegion) {
ResTable_config config;
config.packRegion("419");
char out[4] = { 1, 1, 1, 1};
char out[4] = {1, 1, 1, 1};
config.unpackRegion(out);
EXPECT_EQ('4', out[0]);
@@ -124,6 +125,10 @@ TEST(ConfigLocaleTest, packAndUnpack3LetterRegion) {
if (script != NULL) {
memcpy(out->localeScript, script, 4);
out->localeScriptWasProvided = true;
} else {
out->computeScript();
out->localeScriptWasProvided = false;
}
if (variant != NULL) {
@@ -177,11 +182,12 @@ TEST(ConfigLocaleTest, setLocale) {
EXPECT_EQ('n', test.language[1]);
EXPECT_EQ('U', test.country[0]);
EXPECT_EQ('S', test.country[1]);
EXPECT_EQ(0, test.localeScript[0]);
EXPECT_FALSE(test.localeScriptWasProvided);
EXPECT_EQ(0, memcmp("Latn", test.localeScript, 4));
EXPECT_EQ(0, test.localeVariant[0]);
test.setBcp47Locale("eng-419");
char out[4] = { 1, 1, 1, 1};
char out[4] = {1, 1, 1, 1};
test.unpackLanguage(out);
EXPECT_EQ('e', out[0]);
EXPECT_EQ('n', out[1]);
@@ -193,17 +199,397 @@ TEST(ConfigLocaleTest, setLocale) {
EXPECT_EQ('1', out[1]);
EXPECT_EQ('9', out[2]);
test.setBcp47Locale("en-Latn-419");
memset(out, 1, 4);
EXPECT_EQ('e', test.language[0]);
EXPECT_EQ('n', test.language[1]);
EXPECT_EQ(0, memcmp("Latn", test.localeScript, 4));
EXPECT_TRUE(test.localeScriptWasProvided);
memset(out, 1, 4);
test.unpackRegion(out);
EXPECT_EQ('4', out[0]);
EXPECT_EQ('1', out[1]);
EXPECT_EQ('9', out[2]);
test.setBcp47Locale("de-1901");
memset(out, 1, 4);
test.unpackLanguage(out);
EXPECT_EQ('d', out[0]);
EXPECT_EQ('e', out[1]);
EXPECT_EQ('\0', out[2]);
EXPECT_FALSE(test.localeScriptWasProvided);
EXPECT_EQ(0, memcmp("Latn", test.localeScript, 4));
memset(out, 1, 4);
test.unpackRegion(out);
EXPECT_EQ('\0', out[0]);
EXPECT_EQ(0, strcmp("1901", test.localeVariant));
test.setBcp47Locale("de-Latn-1901");
memset(out, 1, 4);
test.unpackLanguage(out);
EXPECT_EQ('d', out[0]);
EXPECT_EQ('e', out[1]);
EXPECT_EQ('\0', out[2]);
EXPECT_TRUE(test.localeScriptWasProvided);
EXPECT_EQ(0, memcmp("Latn", test.localeScript, 4));
memset(out, 1, 4);
test.unpackRegion(out);
EXPECT_EQ('\0', out[0]);
EXPECT_EQ(0, strcmp("1901", test.localeVariant));
}
} // namespace android.
TEST(ConfigLocaleTest, computeScript) {
ResTable_config config;
fillIn(NULL, NULL, NULL, NULL, &config);
EXPECT_EQ(0, memcmp("\0\0\0\0", config.localeScript, 4));
fillIn("zh", "TW", NULL, NULL, &config);
EXPECT_EQ(0, memcmp("Hant", config.localeScript, 4));
fillIn("zh", "CN", NULL, NULL, &config);
EXPECT_EQ(0, memcmp("Hans", config.localeScript, 4));
fillIn("az", NULL, NULL, NULL, &config);
EXPECT_EQ(0, memcmp("Latn", config.localeScript, 4));
fillIn("az", "AZ", NULL, NULL, &config);
EXPECT_EQ(0, memcmp("Latn", config.localeScript, 4));
fillIn("az", "IR", NULL, NULL, &config);
EXPECT_EQ(0, memcmp("Arab", config.localeScript, 4));
fillIn("peo", NULL, NULL, NULL, &config);
EXPECT_EQ(0, memcmp("Xpeo", config.localeScript, 4));
fillIn("qaa", NULL, NULL, NULL, &config);
EXPECT_EQ(0, memcmp("\0\0\0\0", config.localeScript, 4));
}
TEST(ConfigLocaleTest, getBcp47Locale_script) {
ResTable_config config;
fillIn("en", NULL, "Latn", NULL, &config);
char out[RESTABLE_MAX_LOCALE_LEN];
config.localeScriptWasProvided = true;
config.getBcp47Locale(out);
EXPECT_EQ(0, strcmp("en-Latn", out));
config.localeScriptWasProvided = false;
config.getBcp47Locale(out);
EXPECT_EQ(0, strcmp("en", out));
}
TEST(ConfigLocaleTest, match) {
ResTable_config supported, requested;
fillIn(NULL, NULL, NULL, NULL, &supported);
fillIn("fr", "CA", NULL, NULL, &requested);
// Empty locale matches everything (as a default).
EXPECT_TRUE(supported.match(requested));
fillIn("en", "CA", NULL, NULL, &supported);
fillIn("fr", "CA", NULL, NULL, &requested);
// Different languages don't match.
EXPECT_FALSE(supported.match(requested));
fillIn("qaa", "FR", NULL, NULL, &supported);
fillIn("qaa", "CA", NULL, NULL, &requested);
// If we can't infer the scripts, different regions don't match.
EXPECT_FALSE(supported.match(requested));
fillIn("qaa", "FR", "Latn", NULL, &supported);
fillIn("qaa", "CA", NULL, NULL, &requested);
// If we can't infer any of the scripts, different regions don't match.
EXPECT_FALSE(supported.match(requested));
fillIn("qaa", "FR", NULL, NULL, &supported);
fillIn("qaa", "CA", "Latn", NULL, &requested);
// If we can't infer any of the scripts, different regions don't match.
EXPECT_FALSE(supported.match(requested));
fillIn("qaa", NULL, NULL, NULL, &supported);
fillIn("qaa", "CA", NULL, NULL, &requested);
// language-only resources still support language+region requests, even if we can't infer the
// script.
EXPECT_TRUE(supported.match(requested));
fillIn("qaa", "CA", NULL, NULL, &supported);
fillIn("qaa", "CA", NULL, NULL, &requested);
// Even if we can't infer the scripts, exactly equal locales match.
EXPECT_TRUE(supported.match(requested));
fillIn("az", NULL, NULL, NULL, &supported);
fillIn("az", NULL, "Latn", NULL, &requested);
// If the resolved scripts are the same, it doesn't matter if they were explicitly provided
// or not, and they match.
EXPECT_TRUE(supported.match(requested));
fillIn("az", NULL, NULL, NULL, &supported);
fillIn("az", NULL, "Cyrl", NULL, &requested);
// If the resolved scripts are different, they don't match.
EXPECT_FALSE(supported.match(requested));
fillIn("az", NULL, NULL, NULL, &supported);
fillIn("az", "IR", NULL, NULL, &requested);
// If the resolved scripts are different, they don't match.
EXPECT_FALSE(supported.match(requested));
fillIn("az", "IR", NULL, NULL, &supported);
fillIn("az", NULL, "Arab", NULL, &requested);
// If the resolved scripts are the same, it doesn't matter if they were explicitly provided
// or not, and they match.
EXPECT_TRUE(supported.match(requested));
fillIn("en", NULL, NULL, NULL, &supported);
fillIn("en", "XA", NULL, NULL, &requested);
// en-XA is a pseudo-locale, and English resources are not a match for it.
EXPECT_FALSE(supported.match(requested));
fillIn("en", "XA", NULL, NULL, &supported);
fillIn("en", NULL, NULL, NULL, &requested);
// en-XA is a pseudo-locale, and its resources don't support English locales.
EXPECT_FALSE(supported.match(requested));
fillIn("en", "XA", NULL, NULL, &supported);
fillIn("en", "XA", NULL, NULL, &requested);
// Even if they are pseudo-locales, exactly equal locales match.
EXPECT_TRUE(supported.match(requested));
fillIn("ar", NULL, NULL, NULL, &supported);
fillIn("ar", "XB", NULL, NULL, &requested);
// ar-XB is a pseudo-locale, and Arabic resources are not a match for it.
EXPECT_FALSE(supported.match(requested));
fillIn("ar", "XB", NULL, NULL, &supported);
fillIn("ar", NULL, NULL, NULL, &requested);
// ar-XB is a pseudo-locale, and its resources don't support Arabic locales.
EXPECT_FALSE(supported.match(requested));
fillIn("ar", "XB", NULL, NULL, &supported);
fillIn("ar", "XB", NULL, NULL, &requested);
// Even if they are pseudo-locales, exactly equal locales match.
EXPECT_TRUE(supported.match(requested));
}
TEST(ConfigLocaleTest, isLocaleBetterThan_basics) {
ResTable_config config1, config2, request;
fillIn(NULL, NULL, NULL, NULL, &request);
fillIn("fr", "FR", NULL, NULL, &config1);
fillIn("fr", "CA", NULL, NULL, &config2);
EXPECT_FALSE(config1.isLocaleBetterThan(config2, &request));
EXPECT_FALSE(config2.isLocaleBetterThan(config1, &request));
fillIn("fr", "CA", NULL, NULL, &request);
fillIn(NULL, NULL, NULL, NULL, &config1);
fillIn(NULL, NULL, NULL, NULL, &config2);
EXPECT_FALSE(config1.isLocaleBetterThan(config2, &request));
EXPECT_FALSE(config2.isLocaleBetterThan(config1, &request));
fillIn("fr", "CA", NULL, NULL, &request);
fillIn("fr", "FR", NULL, NULL, &config1);
fillIn(NULL, NULL, NULL, NULL, &config2);
EXPECT_TRUE(config1.isLocaleBetterThan(config2, &request));
EXPECT_FALSE(config2.isLocaleBetterThan(config1, &request));
fillIn("de", "DE", NULL, NULL, &request);
fillIn("de", "DE", NULL, "1901", &config1);
fillIn("de", "DE", NULL, "1996", &config2);
EXPECT_FALSE(config1.isLocaleBetterThan(config2, &request));
EXPECT_FALSE(config2.isLocaleBetterThan(config1, &request));
fillIn("de", "DE", NULL, "1901", &request);
fillIn("de", "DE", NULL, "1901", &config1);
fillIn("de", "DE", NULL, NULL, &config2);
EXPECT_TRUE(config1.isLocaleBetterThan(config2, &request));
EXPECT_FALSE(config2.isLocaleBetterThan(config1, &request));
fillIn("de", "DE", NULL, "1901", &request);
fillIn("de", "DE", NULL, "1996", &config1);
fillIn("de", "DE", NULL, NULL, &config2);
EXPECT_FALSE(config1.isLocaleBetterThan(config2, &request));
EXPECT_FALSE(config2.isLocaleBetterThan(config1, &request));
}
TEST(ConfigLocaleTest, isLocaleBetterThan_regionComparison) {
ResTable_config config1, config2, request;
fillIn("es", "AR", NULL, NULL, &request);
fillIn("es", "419", NULL, NULL, &config1);
fillIn("es", "419", NULL, NULL, &config2);
// Both supported locales are the same, so none is better than the other.
EXPECT_FALSE(config1.isLocaleBetterThan(config2, &request));
EXPECT_FALSE(config2.isLocaleBetterThan(config1, &request));
fillIn("es", "AR", NULL, NULL, &request);
fillIn("es", "AR", NULL, NULL, &config1);
fillIn("es", "419", NULL, NULL, &config2);
// An exact locale match is better than a parent.
EXPECT_TRUE(config1.isLocaleBetterThan(config2, &request));
EXPECT_FALSE(config2.isLocaleBetterThan(config1, &request));
fillIn("es", "AR", NULL, NULL, &request);
fillIn("es", "419", NULL, NULL, &config1);
fillIn("es", NULL, NULL, NULL, &config2);
// A closer parent is better.
EXPECT_TRUE(config1.isLocaleBetterThan(config2, &request));
EXPECT_FALSE(config2.isLocaleBetterThan(config1, &request));
fillIn("es", "AR", NULL, NULL, &request);
fillIn("es", "419", NULL, NULL, &config1);
fillIn("es", "ES", NULL, NULL, &config2);
// A parent is better than a non-parent representative locale.
EXPECT_TRUE(config1.isLocaleBetterThan(config2, &request));
EXPECT_FALSE(config2.isLocaleBetterThan(config1, &request));
fillIn("es", "AR", NULL, NULL, &request);
fillIn("es", NULL, NULL, NULL, &config1);
fillIn("es", "ES", NULL, NULL, &config2);
// A parent is better than a non-parent representative locale.
EXPECT_TRUE(config1.isLocaleBetterThan(config2, &request));
EXPECT_FALSE(config2.isLocaleBetterThan(config1, &request));
fillIn("es", "AR", NULL, NULL, &request);
fillIn("es", "PE", NULL, NULL, &config1);
fillIn("es", "ES", NULL, NULL, &config2);
// A closer locale is better.
EXPECT_TRUE(config1.isLocaleBetterThan(config2, &request));
EXPECT_FALSE(config2.isLocaleBetterThan(config1, &request));
fillIn("es", "AR", NULL, NULL, &request);
fillIn("es", "MX", NULL, NULL, &config1);
fillIn("es", "BO", NULL, NULL, &config2);
// A representative locale is better if they are equidistant.
EXPECT_TRUE(config1.isLocaleBetterThan(config2, &request));
EXPECT_FALSE(config2.isLocaleBetterThan(config1, &request));
fillIn("es", "AR", NULL, NULL, &request);
fillIn("es", "US", NULL, NULL, &config1);
fillIn("es", "BO", NULL, NULL, &config2);
// A representative locale is better if they are equidistant.
EXPECT_TRUE(config1.isLocaleBetterThan(config2, &request));
EXPECT_FALSE(config2.isLocaleBetterThan(config1, &request));
fillIn("es", "AR", NULL, NULL, &request);
fillIn("es", "MX", NULL, NULL, &config1);
fillIn("es", "US", NULL, NULL, &config2);
// If all is equal, the locale earlier in the dictionary is better.
EXPECT_TRUE(config1.isLocaleBetterThan(config2, &request));
EXPECT_FALSE(config2.isLocaleBetterThan(config1, &request));
fillIn("es", "GQ", NULL, NULL, &request);
fillIn("es", "IC", NULL, NULL, &config1);
fillIn("es", "419", NULL, NULL, &config2);
// If all is equal, the locale earlier in the dictionary is better and
// letters are better than numbers.
EXPECT_TRUE(config1.isLocaleBetterThan(config2, &request));
EXPECT_FALSE(config2.isLocaleBetterThan(config1, &request));
fillIn("en", "GB", NULL, NULL, &request);
fillIn("en", "001", NULL, NULL, &config1);
fillIn("en", NULL, NULL, NULL, &config2);
// A closer parent is better.
EXPECT_TRUE(config1.isLocaleBetterThan(config2, &request));
EXPECT_FALSE(config2.isLocaleBetterThan(config1, &request));
fillIn("en", "PR", NULL, NULL, &request);
fillIn("en", NULL, NULL, NULL, &config1);
fillIn("en", "001", NULL, NULL, &config2);
// A parent is better than a non-parent.
EXPECT_TRUE(config1.isLocaleBetterThan(config2, &request));
EXPECT_FALSE(config2.isLocaleBetterThan(config1, &request));
fillIn("en", "DE", NULL, NULL, &request);
fillIn("en", "150", NULL, NULL, &config1);
fillIn("en", "001", NULL, NULL, &config2);
// A closer parent is better.
EXPECT_TRUE(config1.isLocaleBetterThan(config2, &request));
EXPECT_FALSE(config2.isLocaleBetterThan(config1, &request));
fillIn("en", "IN", NULL, NULL, &request);
fillIn("en", "AU", NULL, NULL, &config1);
fillIn("en", "US", NULL, NULL, &config2);
// A closer locale is better.
EXPECT_TRUE(config1.isLocaleBetterThan(config2, &request));
EXPECT_FALSE(config2.isLocaleBetterThan(config1, &request));
fillIn("en", "PR", NULL, NULL, &request);
fillIn("en", "001", NULL, NULL, &config1);
fillIn("en", "GB", NULL, NULL, &config2);
// A closer locale is better.
EXPECT_TRUE(config1.isLocaleBetterThan(config2, &request));
EXPECT_FALSE(config2.isLocaleBetterThan(config1, &request));
fillIn("en", "IN", NULL, NULL, &request);
fillIn("en", "GB", NULL, NULL, &config1);
fillIn("en", "AU", NULL, NULL, &config2);
// A representative locale is better if they are equidistant.
EXPECT_TRUE(config1.isLocaleBetterThan(config2, &request));
EXPECT_FALSE(config2.isLocaleBetterThan(config1, &request));
fillIn("en", "IN", NULL, NULL, &request);
fillIn("en", "AU", NULL, NULL, &config1);
fillIn("en", "CA", NULL, NULL, &config2);
// If all is equal, the locale earlier in the dictionary is better.
EXPECT_TRUE(config1.isLocaleBetterThan(config2, &request));
EXPECT_FALSE(config2.isLocaleBetterThan(config1, &request));
fillIn("pt", "MZ", NULL, NULL, &request);
fillIn("pt", "PT", NULL, NULL, &config1);
fillIn("pt", NULL, NULL, NULL, &config2);
// A closer parent is better.
EXPECT_TRUE(config1.isLocaleBetterThan(config2, &request));
EXPECT_FALSE(config2.isLocaleBetterThan(config1, &request));
fillIn("pt", "MZ", NULL, NULL, &request);
fillIn("pt", "PT", NULL, NULL, &config1);
fillIn("pt", "BR", NULL, NULL, &config2);
// A parent is better than a non-parent.
EXPECT_TRUE(config1.isLocaleBetterThan(config2, &request));
EXPECT_FALSE(config2.isLocaleBetterThan(config1, &request));
fillIn("zh", "MO", "Hant", NULL, &request);
fillIn("zh", "HK", "Hant", NULL, &config1);
fillIn("zh", "TW", "Hant", NULL, &config2);
// A parent is better than a non-parent.
EXPECT_TRUE(config1.isLocaleBetterThan(config2, &request));
EXPECT_FALSE(config2.isLocaleBetterThan(config1, &request));
fillIn("zh", "US", "Hant", NULL, &request);
fillIn("zh", "TW", "Hant", NULL, &config1);
fillIn("zh", "HK", "Hant", NULL, &config2);
// A representative locale is better if they are equidistant.
EXPECT_TRUE(config1.isLocaleBetterThan(config2, &request));
EXPECT_FALSE(config2.isLocaleBetterThan(config1, &request));
fillIn("ar", "DZ", NULL, NULL, &request);
fillIn("ar", "015", NULL, NULL, &config1);
fillIn("ar", NULL, NULL, NULL, &config2);
// A closer parent is better.
EXPECT_TRUE(config1.isLocaleBetterThan(config2, &request));
EXPECT_FALSE(config2.isLocaleBetterThan(config1, &request));
fillIn("ar", "EG", NULL, NULL, &request);
fillIn("ar", NULL, NULL, NULL, &config1);
fillIn("ar", "015", NULL, NULL, &config2);
// A parent is better than a non-parent.
EXPECT_TRUE(config1.isLocaleBetterThan(config2, &request));
EXPECT_FALSE(config2.isLocaleBetterThan(config1, &request));
fillIn("ar", "QA", NULL, NULL, &request);
fillIn("ar", "EG", NULL, NULL, &config1);
fillIn("ar", "BH", NULL, NULL, &config2);
// A representative locale is better if they are equidistant.
EXPECT_TRUE(config1.isLocaleBetterThan(config2, &request));
EXPECT_FALSE(config2.isLocaleBetterThan(config1, &request));
fillIn("ar", "QA", NULL, NULL, &request);
fillIn("ar", "SA", NULL, NULL, &config1);
fillIn("ar", "015", NULL, NULL, &config2);
// If all is equal, the locale earlier in the dictionary is better and
// letters are better than numbers.
EXPECT_TRUE(config1.isLocaleBetterThan(config2, &request));
EXPECT_FALSE(config2.isLocaleBetterThan(config1, &request));
}
} // namespace android

29
tools/aapt/AaptAssets.cpp
View File

@@ -235,7 +235,7 @@ bool AaptLocaleValue::initFromFilterString(const String8& str) {
setRegion(part2.string());
} else if (part2.length() == 4 && isAlpha(part2)) {
setScript(part2.string());
} else if (part2.length() >= 5 && part2.length() <= 8) {
} else if (part2.length() >= 4 && part2.length() <= 8) {
setVariant(part2.string());
} else {
valid = false;
@@ -250,7 +250,7 @@ bool AaptLocaleValue::initFromFilterString(const String8& str) {
if (((part3.length() == 2 && isAlpha(part3)) ||
(part3.length() == 3 && isNumber(part3))) && script[0]) {
setRegion(part3.string());
} else if (part3.length() >= 5 && part3.length() <= 8) {
} else if (part3.length() >= 4 && part3.length() <= 8) {
setVariant(part3.string());
} else {
valid = false;
@@ -261,7 +261,7 @@ bool AaptLocaleValue::initFromFilterString(const String8& str) {
}
const String8& part4 = parts[3];
if (part4.length() >= 5 && part4.length() <= 8) {
if (part4.length() >= 4 && part4.length() <= 8) {
setVariant(part4.string());
} else {
valid = false;
@@ -280,7 +280,7 @@ int AaptLocaleValue::initFromDirName(const Vector<String8>& parts, const int sta
String8 part = parts[currentIndex];
if (part[0] == 'b' && part[1] == '+') {
// This is a "modified" BCP-47 language tag. Same semantics as BCP-47 tags,
// This is a "modified" BCP 47 language tag. Same semantics as BCP 47 tags,
// except that the separator is "+" and not "-".
Vector<String8> subtags = AaptUtil::splitAndLowerCase(part, '+');
subtags.removeItemsAt(0);
@@ -296,8 +296,11 @@ int AaptLocaleValue::initFromDirName(const Vector<String8>& parts, const int sta
setRegion(subtags[1]);
break;
case 4:
setScript(subtags[1]);
break;
if (isAlpha(subtags[1])) {
setScript(subtags[1]);
break;
}
// This is not alphabetical, so we fall through to variant
case 5:
case 6:
case 7:
@@ -305,7 +308,7 @@ int AaptLocaleValue::initFromDirName(const Vector<String8>& parts, const int sta
setVariant(subtags[1]);
break;
default:
fprintf(stderr, "ERROR: Invalid BCP-47 tag in directory name %s\n",
fprintf(stderr, "ERROR: Invalid BCP 47 tag in directory name %s\n",
part.string());
return -1;
}
@@ -322,13 +325,13 @@ int AaptLocaleValue::initFromDirName(const Vector<String8>& parts, const int sta
setRegion(subtags[1]);
hasRegion = true;
} else {
fprintf(stderr, "ERROR: Invalid BCP-47 tag in directory name %s\n", part.string());
fprintf(stderr, "ERROR: Invalid BCP 47 tag in directory name %s\n", part.string());
return -1;
}
// The third tag can either be a region code (if the second tag was
// a script), else a variant code.
if (subtags[2].size() > 4) {
if (subtags[2].size() >= 4) {
setVariant(subtags[2]);
} else {
setRegion(subtags[2]);
@@ -339,7 +342,7 @@ int AaptLocaleValue::initFromDirName(const Vector<String8>& parts, const int sta
setRegion(subtags[2]);
setVariant(subtags[3]);
} else {
fprintf(stderr, "ERROR: Invalid BCP-47 tag in directory name: %s\n", part.string());
fprintf(stderr, "ERROR: Invalid BCP 47 tag in directory name: %s\n", part.string());
return -1;
}
@@ -370,7 +373,7 @@ int AaptLocaleValue::initFromDirName(const Vector<String8>& parts, const int sta
void AaptLocaleValue::initFromResTable(const ResTable_config& config) {
config.unpackLanguage(language);
config.unpackRegion(region);
if (config.localeScript[0]) {
if (config.localeScriptWasProvided) {
memcpy(script, config.localeScript, sizeof(config.localeScript));
}
@@ -385,6 +388,10 @@ void AaptLocaleValue::writeTo(ResTable_config* out) const {
if (script[0]) {
memcpy(out->localeScript, script, sizeof(out->localeScript));
out->localeScriptWasProvided = true;
} else {
out->computeScript();
out->localeScriptWasProvided = false;
}
if (variant[0]) {

24
tools/aapt2/Locale.cpp
View File

@@ -96,7 +96,7 @@ bool LocaleValue::initFromFilterString(const std::string& str) {
setRegion(part2.c_str());
} else if (part2.length() == 4 && isAlpha(part2)) {
setScript(part2.c_str());
} else if (part2.length() >= 5 && part2.length() <= 8) {
} else if (part2.length() >= 4 && part2.length() <= 8) {
setVariant(part2.c_str());
} else {
valid = false;
@@ -111,7 +111,7 @@ bool LocaleValue::initFromFilterString(const std::string& str) {
if (((part3.length() == 2 && isAlpha(part3)) ||
(part3.length() == 3 && isNumber(part3))) && script[0]) {
setRegion(part3.c_str());
} else if (part3.length() >= 5 && part3.length() <= 8) {
} else if (part3.length() >= 4 && part3.length() <= 8) {
setVariant(part3.c_str());
} else {
valid = false;
@@ -122,7 +122,7 @@ bool LocaleValue::initFromFilterString(const std::string& str) {
}
const std::string& part4 = parts[3];
if (part4.length() >= 5 && part4.length() <= 8) {
if (part4.length() >= 4 && part4.length() <= 8) {
setVariant(part4.c_str());
} else {
valid = false;
@@ -141,7 +141,7 @@ ssize_t LocaleValue::initFromParts(std::vector<std::string>::iterator iter,
std::string& part = *iter;
if (part[0] == 'b' && part[1] == '+') {
// This is a "modified" BCP-47 language tag. Same semantics as BCP-47 tags,
// This is a "modified" BCP 47 language tag. Same semantics as BCP 47 tags,
// except that the separator is "+" and not "-".
std::vector<std::string> subtags = util::splitAndLowercase(part, '+');
subtags.erase(subtags.begin());
@@ -157,8 +157,12 @@ ssize_t LocaleValue::initFromParts(std::vector<std::string>::iterator iter,
setRegion(subtags[1].c_str());
break;
case 4:
setScript(subtags[1].c_str());
break;
if ('0' <= subtags[1][0] && subtags[1][0] <= '9') {
// This is a variant: fall through
} else {
setScript(subtags[1].c_str());
break;
}
case 5:
case 6:
case 7:
@@ -184,7 +188,7 @@ ssize_t LocaleValue::initFromParts(std::vector<std::string>::iterator iter,
// The third tag can either be a region code (if the second tag was
// a script), else a variant code.
if (subtags[2].size() > 4) {
if (subtags[2].size() >= 4) {
setVariant(subtags[2].c_str());
} else {
setRegion(subtags[2].c_str());
@@ -249,7 +253,7 @@ std::string LocaleValue::toDirName() const {
void LocaleValue::initFromResTable(const ResTable_config& config) {
config.unpackLanguage(language);
config.unpackRegion(region);
if (config.localeScript[0]) {
if (config.localeScriptWasProvided) {
memcpy(script, config.localeScript, sizeof(config.localeScript));
}
@@ -264,6 +268,10 @@ void LocaleValue::writeTo(ResTable_config* out) const {
if (script[0]) {
memcpy(out->localeScript, script, sizeof(out->localeScript));
out->localeScriptWasProvided = true;
} else {
out->computeScript();
out->localeScriptWasProvided = false;
}
if (variant[0]) {

286
tools/localedata/extract_icu_data.py Executable file
View File

@@ -0,0 +1,286 @@
#!/usr/bin/env python
#
# Copyright 2016 The Android Open Source Project. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""Generate a C++ data table containing locale data."""
import collections
import glob
import os.path
import sys
def get_locale_parts(locale):
"""Split a locale into three parts, for langauge, script, and region."""
parts = locale.split('_')
if len(parts) == 1:
return (parts[0], None, None)
elif len(parts) == 2:
if len(parts[1]) == 4: # parts[1] is a script
return (parts[0], parts[1], None)
else:
return (parts[0], None, parts[1])
else:
assert len(parts) == 3
return tuple(parts)
def read_likely_subtags(input_file_name):
"""Read and parse ICU's likelySubtags.txt."""
with open(input_file_name) as input_file:
likely_script_dict = {
# Android's additions for pseudo-locales. These internal codes make
# sure that the pseudo-locales would not match other English or
# Arabic locales. (We can't use private-use ISO 15924 codes, since
# they may be used by apps for other purposes.)
"en_XA": "~~~A",
"ar_XB": "~~~B",
}
representative_locales = {
# Android's additions
"en_Latn_GB", # representative for en_Latn_001
"es_Latn_MX", # representative for es_Latn_419
"es_Latn_US", # representative for es_Latn_419 (not the best idea,
# but Android has been shipping with it for quite a
# while. Fortunately, MX < US, so if both exist, MX
# would be chosen.)
}
for line in input_file:
line = unicode(line, 'UTF-8').strip(u' \n\uFEFF').encode('UTF-8')
if line.startswith('//'):
continue
if '{' in line and '}' in line:
from_locale = line[:line.index('{')]
to_locale = line[line.index('"')+1:line.rindex('"')]
from_lang, from_scr, from_region = get_locale_parts(from_locale)
_, to_scr, to_region = get_locale_parts(to_locale)
if from_lang == 'und':
continue # not very useful for our purposes
if from_region is None and to_region != '001':
representative_locales.add(to_locale)
if from_scr is None:
likely_script_dict[from_locale] = to_scr
return likely_script_dict, frozenset(representative_locales)
# From packLanguageOrRegion() in ResourceTypes.cpp
def pack_language_or_region(inp, base):
"""Pack langauge or region in a two-byte tuple."""
if inp is None:
return (0, 0)
elif len(inp) == 2:
return ord(inp[0]), ord(inp[1])
else:
assert len(inp) == 3
base = ord(base)
first = ord(inp[0]) - base
second = ord(inp[1]) - base
third = ord(inp[2]) - base
return (0x80 | (third << 2) | (second >>3),
((second << 5) | first) & 0xFF)
# From packLanguage() in ResourceTypes.cpp
def pack_language(language):
"""Pack language in a two-byte tuple."""
return pack_language_or_region(language, 'a')
# From packRegion() in ResourceTypes.cpp
def pack_region(region):
"""Pack region in a two-byte tuple."""
return pack_language_or_region(region, '0')
def pack_to_uint32(locale):
"""Pack language+region of locale into a 32-bit unsigned integer."""
lang, _, region = get_locale_parts(locale)
plang = pack_language(lang)
pregion = pack_region(region)
return (plang[0] << 24) | (plang[1] << 16) | (pregion[0] << 8) | pregion[1]
def dump_script_codes(all_scripts):
"""Dump the SCRIPT_CODES table."""
print 'const char SCRIPT_CODES[][4] = {'
for index, script in enumerate(all_scripts):
print " /* %-2d */ {'%c', '%c', '%c', '%c'}," % (
index, script[0], script[1], script[2], script[3])
print '};'
print
def dump_script_data(likely_script_dict, all_scripts):
"""Dump the script data."""
print
print 'const std::unordered_map<uint32_t, uint8_t> LIKELY_SCRIPTS({'
for locale in sorted(likely_script_dict.keys()):
script = likely_script_dict[locale]
print ' {0x%08Xu, %2du}, // %s -> %s' % (
pack_to_uint32(locale),
all_scripts.index(script),
locale.replace('_', '-'),
script)
print '});'
def pack_to_uint64(locale):
"""Pack a full locale into a 64-bit unsigned integer."""
_, script, _ = get_locale_parts(locale)
return ((pack_to_uint32(locale) << 32) |
(ord(script[0]) << 24) |
(ord(script[1]) << 16) |
(ord(script[2]) << 8) |
ord(script[3]))
def dump_representative_locales(representative_locales):
"""Dump the set of representative locales."""
print
print 'std::unordered_set<uint64_t> REPRESENTATIVE_LOCALES({'
for locale in sorted(representative_locales):
print ' 0x%08Xllu, // %s' % (
pack_to_uint64(locale),
locale)
print '});'
def read_and_dump_likely_data(icu_data_dir):
"""Read and dump the likely-script data."""
likely_subtags_txt = os.path.join(icu_data_dir, 'misc', 'likelySubtags.txt')
likely_script_dict, representative_locales = read_likely_subtags(
likely_subtags_txt)
all_scripts = list(set(likely_script_dict.values()))
assert len(all_scripts) <= 256
all_scripts.sort()
dump_script_codes(all_scripts)
dump_script_data(likely_script_dict, all_scripts)
dump_representative_locales(representative_locales)
return likely_script_dict
def read_parent_data(icu_data_dir):
"""Read locale parent data from ICU data files."""
all_icu_data_files = glob.glob(os.path.join(icu_data_dir, '*', '*.txt'))
parent_dict = {}
for data_file in all_icu_data_files:
locale = os.path.splitext(os.path.basename(data_file))[0]
with open(data_file) as input_file:
for line in input_file:
if '%%Parent' in line:
parent = line[line.index('"')+1:line.rindex('"')]
if locale in parent_dict:
# Different files shouldn't have different parent info
assert parent_dict[locale] == parent
else:
parent_dict[locale] = parent
elif locale.startswith('ar_') and 'default{"latn"}' in line:
# Arabic parent overrides for ASCII digits. Since
# Unicode extensions are not supported in ResourceTypes,
# we will use ar-015 (Arabic, Northern Africa) instead
# of the more correct ar-u-nu-latn.
parent_dict[locale] = 'ar_015'
return parent_dict
def get_likely_script(locale, likely_script_dict):
"""Find the likely script for a locale, given the likely-script dictionary.
"""
if locale.count('_') == 2:
# it already has a script
return locale.split('_')[1]
elif locale in likely_script_dict:
return likely_script_dict[locale]
else:
language = locale.split('_')[0]
return likely_script_dict[language]
def dump_parent_data(script_organized_dict):
"""Dump information for parents of locales."""
sorted_scripts = sorted(script_organized_dict.keys())
print
for script in sorted_scripts:
parent_dict = script_organized_dict[script]
print ('const std::unordered_map<uint32_t, uint32_t> %s_PARENTS({'
% script.upper())
for locale in sorted(parent_dict.keys()):
parent = parent_dict[locale]
print ' {0x%08Xu, 0x%08Xu}, // %s -> %s' % (
pack_to_uint32(locale),
pack_to_uint32(parent),
locale.replace('_', '-'),
parent.replace('_', '-'))
print '});'
print
print 'const struct {'
print ' const char script[4];'
print ' const std::unordered_map<uint32_t, uint32_t>* map;'
print '} SCRIPT_PARENTS[] = {'
for script in sorted_scripts:
print " {{'%c', '%c', '%c', '%c'}, &%s_PARENTS}," % (
script[0], script[1], script[2], script[3],
script.upper())
print '};'
def dump_parent_tree_depth(parent_dict):
"""Find and dump the depth of the parent tree."""
max_depth = 1
for locale, _ in parent_dict.items():
depth = 1
while locale in parent_dict:
locale = parent_dict[locale]
depth += 1
max_depth = max(max_depth, depth)
assert max_depth < 5 # Our algorithms assume small max_depth
print
print 'const size_t MAX_PARENT_DEPTH = %d;' % max_depth
def read_and_dump_parent_data(icu_data_dir, likely_script_dict):
"""Read parent data from ICU and dump it."""
parent_dict = read_parent_data(icu_data_dir)
script_organized_dict = collections.defaultdict(dict)
for locale in parent_dict:
parent = parent_dict[locale]
if parent == 'root':
continue
script = get_likely_script(locale, likely_script_dict)
script_organized_dict[script][locale] = parent_dict[locale]
dump_parent_data(script_organized_dict)
dump_parent_tree_depth(parent_dict)
def main():
"""Read the data files from ICU and dump the output to a C++ file."""
source_root = sys.argv[1]
icu_data_dir = os.path.join(
source_root,
'external', 'icu', 'icu4c', 'source', 'data')
print '// Auto-generated by %s' % sys.argv[0]
print
likely_script_dict = read_and_dump_likely_data(icu_data_dir)
read_and_dump_parent_data(icu_data_dir, likely_script_dict)
if __name__ == '__main__':
main()