Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge "The 'localize' tool is dead." into jb-dev

Browse Source
This commit is contained in:
Elliott Hughes
2012-05-08 10:45:38 -07:00
committed by Android (Google) Code Review
parent 4a337ec99e 436bbd1a6c
commit 6d6d186cd1
48 changed files with 0 additions and 5802 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
tools/localize/Android.mk
View File

@@ -1,55 +0,0 @@
#
# Copyright 2006 The Android Open Source Project
#
# Android Asset Packaging Tool
#
LOCAL_PATH:= $(call my-dir)
include $(CLEAR_VARS)
LOCAL_SRC_FILES := \
file_utils.cpp \
localize.cpp \
merge_res_and_xliff.cpp \
res_check.cpp \
xmb.cpp \
Configuration.cpp \
Perforce.cpp \
SourcePos.cpp \
Values.cpp \
ValuesFile.cpp \
XLIFFFile.cpp \
XMLHandler.cpp
LOCAL_C_INCLUDES := \
external/expat/lib \
build/libs/host/include
LOCAL_CFLAGS += -g -O0
LOCAL_STATIC_LIBRARIES := \
libexpat \
libhost \
libutils \
libcutils
ifeq ($(HOST_OS),linux)
LOCAL_LDLIBS += -lrt -ldl -lpthread
endif
LOCAL_MODULE := localize
ifeq (a,a)
LOCAL_CFLAGS += -DLOCALIZE_WITH_TESTS
LOCAL_SRC_FILES += \
test.cpp \
localize_test.cpp \
merge_res_and_xliff_test.cpp \
Perforce_test.cpp \
ValuesFile_test.cpp \
XLIFFFile_test.cpp \
XMLHandler_test.cpp
endif
include $(BUILD_HOST_EXECUTABLE)

76
tools/localize/Configuration.cpp
View File

@@ -1,76 +0,0 @@
#include "Configuration.h"
#include <string.h>
int
Configuration::Compare(const Configuration& that) const
{
int n;
n = locale.compare(that.locale);
if (n != 0) return n;
n = vendor.compare(that.vendor);
if (n != 0) return n;
n = orientation.compare(that.orientation);
if (n != 0) return n;
n = density.compare(that.density);
if (n != 0) return n;
n = touchscreen.compare(that.touchscreen);
if (n != 0) return n;
n = keyboard.compare(that.keyboard);
if (n != 0) return n;
n = navigation.compare(that.navigation);
if (n != 0) return n;
n = screenSize.compare(that.screenSize);
if (n != 0) return n;
return 0;
}
string
Configuration::ToString() const
{
string s;
if (locale.length() > 0) {
if (s.length() > 0) {
s += "-";
}
s += locale;
}
return s;
}
bool
split_locale(const string& in, string* language, string* region)
{
const int len = in.length();
if (len == 2) {
if (isalpha(in[0]) && isalpha(in[1])) {
*language = in;
region->clear();
return true;
} else {
return false;
}
}
else if (len == 5) {
if (isalpha(in[0]) && isalpha(in[1]) && (in[2] == '_' || in[2] == '-')
&& isalpha(in[3]) && isalpha(in[4])) {
language->assign(in.c_str(), 2);
region->assign(in.c_str()+3, 2);
return true;
} else {
return false;
}
}
else {
return false;
}
}

38
tools/localize/Configuration.h
View File

@@ -1,38 +0,0 @@
#ifndef CONFIGURATION_H
#define CONFIGURATION_H
#include <string>
using namespace std;
struct Configuration
{
string locale;
string vendor;
string orientation;
string density;
string touchscreen;
string keyboard;
string navigation;
string screenSize;
// Compare two configurations
int Compare(const Configuration& that) const;
inline bool operator<(const Configuration& that) const { return Compare(that) < 0; }
inline bool operator<=(const Configuration& that) const { return Compare(that) <= 0; }
inline bool operator==(const Configuration& that) const { return Compare(that) == 0; }
inline bool operator!=(const Configuration& that) const { return Compare(that) != 0; }
inline bool operator>=(const Configuration& that) const { return Compare(that) >= 0; }
inline bool operator>(const Configuration& that) const { return Compare(that) > 0; }
// Parse a directory name, like "values-en-rUS". Return the first segment in resType.
bool ParseDiectoryName(const string& dir, string* resType);
string ToString() const;
};
bool split_locale(const string& in, string* language, string* region);
#endif // CONFIGURATION_H

234
tools/localize/Perforce.cpp
View File

@@ -1,234 +0,0 @@
#include "Perforce.h"
#include "log.h"
#include <string.h>
#include <cstdio>
#include <stdlib.h>
#include <sstream>
#include <sys/types.h>
#include <unistd.h>
#include <stdlib.h>
#include <string.h>
#include <sys/wait.h>
#include <cstdio>
using namespace std;
extern char** environ;
int
Perforce::RunCommand(const string& cmd, string* result, bool printOnFailure)
{
int err;
int outPipe[2];
int errPipe[2];
pid_t pid;
log_printf("Perforce::RunCommand: %s\n", cmd.c_str());
err = pipe(outPipe);
err |= pipe(errPipe);
if (err == -1) {
printf("couldn't create pipe. exiting.\n");
exit(1);
return -1;
}
pid = fork();
if (pid == -1) {
printf("couldn't fork. eixiting\n");
exit(1);
return -1;
}
else if (pid == 0) {
char const* args[] = {
"/bin/sh",
"-c",
cmd.c_str(),
NULL
};
close(outPipe[0]);
close(errPipe[0]);
dup2(outPipe[1], 1);
dup2(errPipe[1], 2);
execve(args[0], (char* const*)args, environ);
// done
}
close(outPipe[1]);
close(errPipe[1]);
result->clear();
char buf[1024];
// stdout
while (true) {
size_t amt = read(outPipe[0], buf, sizeof(buf));
result->append(buf, amt);
if (amt <= 0) {
break;
}
}
// stderr -- the messages are short so it ought to just fit in the buffer
string error;
while (true) {
size_t amt = read(errPipe[0], buf, sizeof(buf));
error.append(buf, amt);
if (amt <= 0) {
break;
}
}
close(outPipe[0]);
close(errPipe[0]);
waitpid(pid, &err, 0);
if (WIFEXITED(err)) {
err = WEXITSTATUS(err);
} else {
err = -1;
}
if (err != 0 && printOnFailure) {
write(2, error.c_str(), error.length());
}
return err;
}
int
Perforce::GetResourceFileNames(const string& version, const string& base,
const vector<string>& apps, vector<string>* results,
bool printOnFailure)
{
int err;
string text;
stringstream cmd;
cmd << "p4 files";
const size_t I = apps.size();
for (size_t i=0; i<I; i++) {
cmd << " \"" << base << '/' << apps[i] << "/res/values/strings.xml@" << version << '"';
}
err = RunCommand(cmd.str(), &text, printOnFailure);
const char* str = text.c_str();
while (*str) {
const char* lineend = strchr(str, '\n');
if (lineend == str) {
str++;
continue;
}
if (lineend-str > 1023) {
fprintf(stderr, "line too long!\n");
return 1;
}
string s(str, lineend-str);
char filename[1024];
char edit[1024];
int count = sscanf(str, "%[^#]#%*d - %s change %*d %*[^\n]\n", filename, edit);
if (count == 2 && 0 != strcmp("delete", edit)) {
results->push_back(string(filename));
}
str = lineend + 1;
}
return err;
}
int
Perforce::GetFile(const string& file, const string& version, string* result,
bool printOnFailure)
{
stringstream cmd;
cmd << "p4 print -q \"" << file << '@' << version << '"';
return RunCommand(cmd.str(), result, printOnFailure);
}
string
Perforce::GetCurrentChange(bool printOnFailure)
{
int err;
string text;
err = RunCommand("p4 changes -m 1 \\#have", &text, printOnFailure);
if (err != 0) {
return "";
}
long long n;
int count = sscanf(text.c_str(), "Change %lld on", &n);
if (count != 1) {
return "";
}
char result[100];
sprintf(result, "%lld", n);
return string(result);
}
static int
do_files(const string& op, const vector<string>& files, bool printOnFailure)
{
string text;
stringstream cmd;
cmd << "p4 " << op;
const size_t I = files.size();
for (size_t i=0; i<I; i++) {
cmd << " \"" << files[i] << "\"";
}
return Perforce::RunCommand(cmd.str(), &text, printOnFailure);
}
int
Perforce::EditFiles(const vector<string>& files, bool printOnFailure)
{
return do_files("edit", files, printOnFailure);
}
int
Perforce::AddFiles(const vector<string>& files, bool printOnFailure)
{
return do_files("add", files, printOnFailure);
}
int
Perforce::DeleteFiles(const vector<string>& files, bool printOnFailure)
{
return do_files("delete", files, printOnFailure);
}
string
Perforce::Where(const string& depotPath, bool printOnFailure)
{
int err;
string text;
string cmd = "p4 where ";
cmd += depotPath;
err = RunCommand(cmd, &text, printOnFailure);
if (err != 0) {
return "";
}
size_t index = text.find(' ');
if (index == text.npos) {
return "";
}
index = text.find(' ', index+1)+1;
if (index == text.npos) {
return "";
}
return text.substr(index, text.length()-index-1);
}

25
tools/localize/Perforce.h
View File

@@ -1,25 +0,0 @@
#ifndef PERFORCE_H
#define PERFORCE_H
#include <string>
#include <vector>
using namespace std;
class Perforce
{
public:
static int RunCommand(const string& cmd, string* result, bool printOnFailure);
static int GetResourceFileNames(const string& version, const string& base,
const vector<string>& apps, vector<string>* result,
bool printOnFailure);
static int GetFile(const string& file, const string& version, string* result,
bool printOnFailure);
static string GetCurrentChange(bool printOnFailure);
static int EditFiles(const vector<string>& filename, bool printOnFailure);
static int AddFiles(const vector<string>& files, bool printOnFailure);
static int DeleteFiles(const vector<string>& files, bool printOnFailure);
static string Where(const string& depotPath, bool printOnFailure);
};
#endif // PERFORCE_H

62
tools/localize/Perforce_test.cpp
View File

@@ -1,62 +0,0 @@
#include "Perforce.h"
#include <stdio.h>
static int
RunCommand_test()
{
string result;
int err = Perforce::RunCommand("p4 help csommands", &result, true);
printf("err=%d result=[[%s]]\n", err, result.c_str());
return 0;
}
static int
GetResourceFileNames_test()
{
vector<string> results;
vector<string> apps;
apps.push_back("apps/common");
apps.push_back("apps/Contacts");
int err = Perforce::GetResourceFileNames("43019", "//device", apps, &results, true);
if (err != 0) {
return err;
}
if (results.size() != 2) {
return 1;
}
if (results[0] != "//device/apps/common/res/values/strings.xml") {
return 1;
}
if (results[1] != "//device/apps/Contacts/res/values/strings.xml") {
return 1;
}
if (false) {
for (size_t i=0; i<results.size(); i++) {
printf("[%zd] '%s'\n", i, results[i].c_str());
}
}
return 0;
}
static int
GetFile_test()
{
string result;
int err = Perforce::GetFile("//device/Makefile", "296", &result, true);
printf("err=%d result=[[%s]]\n", err, result.c_str());
return 0;
}
int
Perforce_test()
{
bool all = false;
int err = 0;
if (all) err |= RunCommand_test();
if (all) err |= GetResourceFileNames_test();
if (all) err |= GetFile_test();
return err;
}

168
tools/localize/SourcePos.cpp
View File

@@ -1,168 +0,0 @@
#include "SourcePos.h"
#include <stdarg.h>
#include <cstdio>
#include <set>
#include <cstdio>
using namespace std;
const SourcePos GENERATED_POS("<generated>", -1);
// ErrorPos
// =============================================================================
struct ErrorPos
{
string file;
int line;
string error;
ErrorPos();
ErrorPos(const ErrorPos& that);
ErrorPos(const string& file, int line, const string& error);
~ErrorPos();
bool operator<(const ErrorPos& rhs) const;
bool operator==(const ErrorPos& rhs) const;
ErrorPos& operator=(const ErrorPos& rhs);
void Print(FILE* to) const;
};
static set<ErrorPos> g_errors;
ErrorPos::ErrorPos()
{
}
ErrorPos::ErrorPos(const ErrorPos& that)
:file(that.file),
line(that.line),
error(that.error)
{
}
ErrorPos::ErrorPos(const string& f, int l, const string& e)
:file(f),
line(l),
error(e)
{
}
ErrorPos::~ErrorPos()
{
}
bool
ErrorPos::operator<(const ErrorPos& rhs) const
{
if (this->file < rhs.file) return true;
if (this->file == rhs.file) {
if (this->line < rhs.line) return true;
if (this->line == rhs.line) {
if (this->error < rhs.error) return true;
}
}
return false;
}
bool
ErrorPos::operator==(const ErrorPos& rhs) const
{
return this->file == rhs.file
&& this->line == rhs.line
&& this->error == rhs.error;
}
ErrorPos&
ErrorPos::operator=(const ErrorPos& rhs)
{
this->file = rhs.file;
this->line = rhs.line;
this->error = rhs.error;
return *this;
}
void
ErrorPos::Print(FILE* to) const
{
if (this->line >= 0) {
fprintf(to, "%s:%d: %s\n", this->file.c_str(), this->line, this->error.c_str());
} else {
fprintf(to, "%s: %s\n", this->file.c_str(), this->error.c_str());
}
}
// SourcePos
// =============================================================================
SourcePos::SourcePos(const string& f, int l)
: file(f), line(l)
{
}
SourcePos::SourcePos(const SourcePos& that)
: file(that.file), line(that.line)
{
}
SourcePos::SourcePos()
: file("???", 0)
{
}
SourcePos::~SourcePos()
{
}
string
SourcePos::ToString() const
{
char buf[1024];
if (this->line >= 0) {
snprintf(buf, sizeof(buf)-1, "%s:%d", this->file.c_str(), this->line);
} else {
snprintf(buf, sizeof(buf)-1, "%s:", this->file.c_str());
}
buf[sizeof(buf)-1] = '\0';
return string(buf);
}
int
SourcePos::Error(const char* fmt, ...) const
{
int retval=0;
char buf[1024];
va_list ap;
va_start(ap, fmt);
retval = vsnprintf(buf, sizeof(buf), fmt, ap);
va_end(ap);
char* p = buf + retval - 1;
while (p > buf && *p == '\n') {
*p = '\0';
p--;
}
ErrorPos err(this->file, this->line, string(buf));
if (g_errors.find(err) == g_errors.end()) {
err.Print(stderr);
g_errors.insert(err);
}
return retval;
}
bool
SourcePos::HasErrors()
{
return g_errors.size() > 0;
}
void
SourcePos::PrintErrors(FILE* to)
{
set<ErrorPos>::const_iterator it;
for (it=g_errors.begin(); it!=g_errors.end(); it++) {
it->Print(to);
}
}

28
tools/localize/SourcePos.h
View File

@@ -1,28 +0,0 @@
#ifndef SOURCEPOS_H
#define SOURCEPOS_H
#include <string>
using namespace std;
class SourcePos
{
public:
string file;
int line;
SourcePos(const string& f, int l);
SourcePos(const SourcePos& that);
SourcePos();
~SourcePos();
string ToString() const;
int Error(const char* fmt, ...) const;
static bool HasErrors();
static void PrintErrors(FILE* to);
};
extern const SourcePos GENERATED_POS;
#endif // SOURCEPOS_H

135
tools/localize/Values.cpp
View File

@@ -1,135 +0,0 @@
#include "Values.h"
#include <stdlib.h>
#include <cstdio>
// =====================================================================================
StringResource::StringResource(const SourcePos& p, const string& f, const Configuration& c,
const string& i, int ix, XMLNode* v, const int ve, const string& vs,
const string& cmnt)
:pos(p),
file(f),
config(c),
id(i),
index(ix),
value(v),
version(ve),
versionString(vs),
comment(cmnt)
{
}
StringResource::StringResource()
:pos(),
file(),
config(),
id(),
index(-1),
value(NULL),
version(),
versionString(),
comment()
{
}
StringResource::StringResource(const StringResource& that)
:pos(that.pos),
file(that.file),
config(that.config),
id(that.id),
index(that.index),
value(that.value),
version(that.version),
versionString(that.versionString),
comment(that.comment)
{
}
int
StringResource::Compare(const StringResource& that) const
{
if (file != that.file) {
return file < that.file ? -1 : 1;
}
if (id != that.id) {
return id < that.id ? -1 : 1;
}
if (index != that.index) {
return index - that.index;
}
if (config != that.config) {
return config < that.config ? -1 : 1;
}
if (version != that.version) {
return version < that.version ? -1 : 1;
}
return 0;
}
string
StringResource::TypedID() const
{
string result;
if (index < 0) {
result = "string:";
} else {
char n[20];
sprintf(n, "%d:", index);
result = "array:";
result += n;
}
result += id;
return result;
}
static void
split(const string& raw, vector<string>*parts)
{
size_t index = 0;
while (true) {
size_t next = raw.find(':', index);
if (next != raw.npos) {
parts->push_back(string(raw, index, next-index));
index = next + 1;
} else {
parts->push_back(string(raw, index));
break;
}
}
}
bool
StringResource::ParseTypedID(const string& raw, string* id, int* index)
{
vector<string> parts;
split(raw, &parts);
const size_t N = parts.size();
for (size_t i=0; i<N; i++) {
if (parts[i].length() == 0) {
return false;
}
}
if (N == 2 && parts[0] == "string") {
*id = parts[1];
*index = -1;
return true;
}
else if (N == 3 && parts[0] == "array") {
char* p;
int n = (int)strtol(parts[1].c_str(), &p, 0);
if (*p == '\0') {
*id = parts[2];
*index = n;
return true;
} else {
return false;
}
}
else {
return false;
}
}

48
tools/localize/Values.h
View File

@@ -1,48 +0,0 @@
#ifndef VALUES_H
#define VALUES_H
#include "Configuration.h"
#include "XMLHandler.h"
#include <string>
using namespace std;
enum {
CURRENT_VERSION,
OLD_VERSION
};
struct StringResource
{
StringResource();
StringResource(const SourcePos& pos, const string& file, const Configuration& config,
const string& id, int index, XMLNode* value,
int version, const string& versionString, const string& comment = "");
StringResource(const StringResource& that);
// Compare two configurations
int Compare(const StringResource& that) const;
inline bool operator<(const StringResource& that) const { return Compare(that) < 0; }
inline bool operator<=(const StringResource& that) const { return Compare(that) <= 0; }
inline bool operator==(const StringResource& that) const { return Compare(that) == 0; }
inline bool operator!=(const StringResource& that) const { return Compare(that) != 0; }
inline bool operator>=(const StringResource& that) const { return Compare(that) >= 0; }
inline bool operator>(const StringResource& that) const { return Compare(that) > 0; }
string TypedID() const;
static bool ParseTypedID(const string& typed, string* id, int* index);
SourcePos pos;
string file;
Configuration config;
string id;
int index;
XMLNode* value;
int version;
string versionString;
string comment;
};
#endif // VALUES_H

266
tools/localize/ValuesFile.cpp
View File

@@ -1,266 +0,0 @@
#include "ValuesFile.h"
#include "XMLHandler.h"
#include <algorithm>
#include <fcntl.h>
#include <expat.h>
#include <unistd.h>
#include <errno.h>
using namespace std;
const char* const ANDROID_XMLNS = "http://schemas.android.com/apk/res/android";
const char* const XLIFF_XMLNS = "urn:oasis:names:tc:xliff:document:1.2";
const char *const NS_MAP[] = {
"android", ANDROID_XMLNS,
"xliff", XLIFF_XMLNS,
NULL, NULL
};
const XMLNamespaceMap ANDROID_NAMESPACES(NS_MAP);
// =====================================================================================
class ArrayHandler : public XMLHandler
{
public:
ArrayHandler(ValuesFile* vf, int version, const string& versionString, const string& id);
virtual int OnStartElement(const SourcePos& pos, const string& ns, const string& name,
const vector<XMLAttribute>& attrs, XMLHandler** next);
virtual int OnText(const SourcePos& pos, const string& text);
virtual int OnComment(const SourcePos& pos, const string& text);
private:
ValuesFile* m_vf;
int m_version;
int m_index;
string m_versionString;
string m_id;
string m_comment;
};
ArrayHandler::ArrayHandler(ValuesFile* vf, int version, const string& versionString,
const string& id)
:m_vf(vf),
m_version(version),
m_index(0),
m_versionString(versionString),
m_id(id)
{
}
int
ArrayHandler::OnStartElement(const SourcePos& pos, const string& ns, const string& name,
const vector<XMLAttribute>& attrs, XMLHandler** next)
{
if (ns == "" && name == "item") {
XMLNode* node = XMLNode::NewElement(pos, ns, name, attrs, XMLNode::EXACT);
m_vf->AddString(StringResource(pos, pos.file, m_vf->GetConfiguration(),
m_id, m_index, node, m_version, m_versionString,
trim_string(m_comment)));
*next = new NodeHandler(node, XMLNode::EXACT);
m_index++;
m_comment = "";
return 0;
} else {
pos.Error("invalid <%s> element inside <array>\n", name.c_str());
return 1;
}
}
int
ArrayHandler::OnText(const SourcePos& pos, const string& text)
{
return 0;
}
int
ArrayHandler::OnComment(const SourcePos& pos, const string& text)
{
m_comment += text;
return 0;
}
// =====================================================================================
class ValuesHandler : public XMLHandler
{
public:
ValuesHandler(ValuesFile* vf, int version, const string& versionString);
virtual int OnStartElement(const SourcePos& pos, const string& ns, const string& name,
const vector<XMLAttribute>& attrs, XMLHandler** next);
virtual int OnText(const SourcePos& pos, const string& text);
virtual int OnComment(const SourcePos& pos, const string& text);
private:
ValuesFile* m_vf;
int m_version;
string m_versionString;
string m_comment;
};
ValuesHandler::ValuesHandler(ValuesFile* vf, int version, const string& versionString)
:m_vf(vf),
m_version(version),
m_versionString(versionString)
{
}
int
ValuesHandler::OnStartElement(const SourcePos& pos, const string& ns, const string& name,
const vector<XMLAttribute>& attrs, XMLHandler** next)
{
if (ns == "" && name == "string") {
string id = XMLAttribute::Find(attrs, "", "name", "");
XMLNode* node = XMLNode::NewElement(pos, ns, name, attrs, XMLNode::EXACT);
m_vf->AddString(StringResource(pos, pos.file, m_vf->GetConfiguration(),
id, -1, node, m_version, m_versionString,
trim_string(m_comment)));
*next = new NodeHandler(node, XMLNode::EXACT);
}
else if (ns == "" && name == "array") {
string id = XMLAttribute::Find(attrs, "", "name", "");
*next = new ArrayHandler(m_vf, m_version, m_versionString, id);
}
m_comment = "";
return 0;
}
int
ValuesHandler::OnText(const SourcePos& pos, const string& text)
{
return 0;
}
int
ValuesHandler::OnComment(const SourcePos& pos, const string& text)
{
m_comment += text;
return 0;
}
// =====================================================================================
ValuesFile::ValuesFile(const Configuration& config)
:m_config(config),
m_strings(),
m_arrays()
{
}
ValuesFile::~ValuesFile()
{
}
ValuesFile*
ValuesFile::ParseFile(const string& filename, const Configuration& config,
int version, const string& versionString)
{
ValuesFile* result = new ValuesFile(config);
TopElementHandler top("", "resources", new ValuesHandler(result, version, versionString));
XMLHandler::ParseFile(filename, &top);
return result;
}
ValuesFile*
ValuesFile::ParseString(const string& filename, const string& text, const Configuration& config,
int version, const string& versionString)
{
ValuesFile* result = new ValuesFile(config);
TopElementHandler top("", "resources", new ValuesHandler(result, version, versionString));
XMLHandler::ParseString(filename, text, &top);
return result;
}
const Configuration&
ValuesFile::GetConfiguration() const
{
return m_config;
}
void
ValuesFile::AddString(const StringResource& str)
{
if (str.index < 0) {
m_strings.insert(str);
} else {
m_arrays[str.id].insert(str);
}
}
set<StringResource>
ValuesFile::GetStrings() const
{
set<StringResource> result = m_strings;
for (map<string,set<StringResource> >::const_iterator it = m_arrays.begin();
it != m_arrays.end(); it++) {
result.insert(it->second.begin(), it->second.end());
}
return result;
}
XMLNode*
ValuesFile::ToXMLNode() const
{
XMLNode* root;
// <resources>
{
vector<XMLAttribute> attrs;
ANDROID_NAMESPACES.AddToAttributes(&attrs);
root = XMLNode::NewElement(GENERATED_POS, "", "resources", attrs, XMLNode::PRETTY);
}
// <array>
for (map<string,set<StringResource> >::const_iterator it = m_arrays.begin();
it != m_arrays.end(); it++) {
vector<XMLAttribute> arrayAttrs;
arrayAttrs.push_back(XMLAttribute("", "name", it->first));
const set<StringResource>& items = it->second;
XMLNode* arrayNode = XMLNode::NewElement(items.begin()->pos, "", "array", arrayAttrs,
XMLNode::PRETTY);
root->EditChildren().push_back(arrayNode);
// <item>
for (set<StringResource>::const_iterator item = items.begin();
item != items.end(); item++) {
XMLNode* itemNode = item->value->Clone();
itemNode->SetName("", "item");
itemNode->EditAttributes().clear();
arrayNode->EditChildren().push_back(itemNode);
}
}
// <string>
for (set<StringResource>::const_iterator it=m_strings.begin(); it!=m_strings.end(); it++) {
const StringResource& str = *it;
vector<XMLAttribute> attrs;
XMLNode* strNode = str.value->Clone();
strNode->SetName("", "string");
strNode->EditAttributes().clear();
strNode->EditAttributes().push_back(XMLAttribute("", "name", str.id));
root->EditChildren().push_back(strNode);
}
return root;
}
string
ValuesFile::ToString() const
{
XMLNode* xml = ToXMLNode();
string s = "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n";
s += xml->ToString(ANDROID_NAMESPACES);
delete xml;
s += '\n';
return s;
}

52
tools/localize/ValuesFile.h
View File

@@ -1,52 +0,0 @@
#ifndef VALUES_FILE_H
#define VALUES_FILE_H
#include "SourcePos.h"
#include "Configuration.h"
#include "XMLHandler.h"
#include "Values.h"
#include <string>
#include <set>
using namespace std;
extern const XMLNamespaceMap ANDROID_NAMESPACES;
class ValuesFile
{
public:
ValuesFile(const Configuration& config);
static ValuesFile* ParseFile(const string& filename, const Configuration& config,
int version, const string& versionString);
static ValuesFile* ParseString(const string& filename, const string& text,
const Configuration& config,
int version, const string& versionString);
~ValuesFile();
const Configuration& GetConfiguration() const;
void AddString(const StringResource& str);
set<StringResource> GetStrings() const;
// exports this file as a n XMLNode, you own this object
XMLNode* ToXMLNode() const;
// writes the ValuesFile out to a string in the canonical format (i.e. writes the contents of
// ToXMLNode()).
string ToString() const;
private:
class ParseState;
friend class ValuesFile::ParseState;
friend class StringHandler;
ValuesFile();
Configuration m_config;
set<StringResource> m_strings;
map<string,set<StringResource> > m_arrays;
};
#endif // VALUES_FILE_H

54
tools/localize/ValuesFile_test.cpp
View File

@@ -1,54 +0,0 @@
#include "ValuesFile.h"
#include <stdio.h>
int
ValuesFile_test()
{
int err = 0;
Configuration config;
config.locale = "zz_ZZ";
ValuesFile* vf = ValuesFile::ParseFile("testdata/values/strings.xml", config,
OLD_VERSION, "1");
const set<StringResource>& strings = vf->GetStrings();
string canonical = vf->ToString();
if (false) {
printf("Strings (%zd)\n", strings.size());
for (set<StringResource>::const_iterator it=strings.begin();
it!=strings.end(); it++) {
const StringResource& str = *it;
printf("%s: '%s'[%d]='%s' (%s) <!-- %s -->\n", str.pos.ToString().c_str(),
str.id.c_str(), str.index,
str.value->ContentsToString(ANDROID_NAMESPACES).c_str(),
str.config.ToString().c_str(), str.comment.c_str());
}
printf("XML:[[%s]]\n", canonical.c_str());
}
const char * const EXPECTED =
"<?xml version=\"1.0\" encoding=\"utf-8\"?>\n"
"<resources xmlns:android=\"http://schemas.android.com/apk/res/android\"\n"
" xmlns:xliff=\"urn:oasis:names:tc:xliff:document:1.2\">\n"
" <array name=\"emailAddressTypes\">\n"
" <item>Email</item>\n"
" <item>Home</item>\n"
" <item>Work</item>\n"
" <item>Other\\u2026</item>\n"
" </array>\n"
" <string name=\"test1\">Discard</string>\n"
" <string name=\"test2\">a<b>b<i>c</i></b>d</string>\n"
" <string name=\"test3\">a<xliff:g a=\"b\" xliff:a=\"asdf\">bBb</xliff:g>C</string>\n"
"</resources>\n";
if (canonical != EXPECTED) {
fprintf(stderr, "ValuesFile_test failed\n");
fprintf(stderr, "canonical=[[%s]]\n", canonical.c_str());
fprintf(stderr, "EXPECTED=[[%s]]\n", EXPECTED);
err = 1;
}
delete vf;
return err;
}

610
tools/localize/XLIFFFile.cpp
View File

@@ -1,610 +0,0 @@
#include "XLIFFFile.h"
#include <algorithm>
#include <sys/time.h>
#include <time.h>
#include <cstdio>
const char* const XLIFF_XMLNS = "urn:oasis:names:tc:xliff:document:1.2";
const char *const NS_MAP[] = {
"", XLIFF_XMLNS,
"xml", XMLNS_XMLNS,
NULL, NULL
};
const XMLNamespaceMap XLIFF_NAMESPACES(NS_MAP);
int
XLIFFFile::File::Compare(const XLIFFFile::File& that) const
{
if (filename != that.filename) {
return filename < that.filename ? -1 : 1;
}
return 0;
}
// =====================================================================================
XLIFFFile::XLIFFFile()
{
}
XLIFFFile::~XLIFFFile()
{
}
static XMLNode*
get_unique_node(const XMLNode* parent, const string& ns, const string& name, bool required)
{
size_t count = parent->CountElementsByName(ns, name);
if (count == 1) {
return parent->GetElementByNameAt(ns, name, 0);
} else {
if (required) {
SourcePos pos = count == 0
? parent->Position()
: parent->GetElementByNameAt(XLIFF_XMLNS, name, 1)->Position();
pos.Error("<%s> elements must contain exactly one <%s> element",
parent->Name().c_str(), name.c_str());
}
return NULL;
}
}
XLIFFFile*
XLIFFFile::Parse(const string& filename)
{
XLIFFFile* result = new XLIFFFile();
XMLNode* root = NodeHandler::ParseFile(filename, XMLNode::PRETTY);
if (root == NULL) {
return NULL;
}
// <file>
vector<XMLNode*> files = root->GetElementsByName(XLIFF_XMLNS, "file");
for (size_t i=0; i<files.size(); i++) {
XMLNode* file = files[i];
string datatype = file->GetAttribute("", "datatype", "");
string originalFile = file->GetAttribute("", "original", "");
Configuration sourceConfig;
sourceConfig.locale = file->GetAttribute("", "source-language", "");
result->m_sourceConfig = sourceConfig;
Configuration targetConfig;
targetConfig.locale = file->GetAttribute("", "target-language", "");
result->m_targetConfig = targetConfig;
result->m_currentVersion = file->GetAttribute("", "build-num", "");
result->m_oldVersion = "old";
// <body>
XMLNode* body = get_unique_node(file, XLIFF_XMLNS, "body", true);
if (body == NULL) continue;
// <trans-unit>
vector<XMLNode*> transUnits = body->GetElementsByName(XLIFF_XMLNS, "trans-unit");
for (size_t j=0; j<transUnits.size(); j++) {
XMLNode* transUnit = transUnits[j];
string rawID = transUnit->GetAttribute("", "id", "");
if (rawID == "") {
transUnit->Position().Error("<trans-unit> tag requires an id");
continue;
}
string id;
int index;
if (!StringResource::ParseTypedID(rawID, &id, &index)) {
transUnit->Position().Error("<trans-unit> has invalid id '%s'\n", rawID.c_str());
continue;
}
// <source>
XMLNode* source = get_unique_node(transUnit, XLIFF_XMLNS, "source", false);
if (source != NULL) {
XMLNode* node = source->Clone();
node->SetPrettyRecursive(XMLNode::EXACT);
result->AddStringResource(StringResource(source->Position(), originalFile,
sourceConfig, id, index, node, CURRENT_VERSION,
result->m_currentVersion));
}
// <target>
XMLNode* target = get_unique_node(transUnit, XLIFF_XMLNS, "target", false);
if (target != NULL) {
XMLNode* node = target->Clone();
node->SetPrettyRecursive(XMLNode::EXACT);
result->AddStringResource(StringResource(target->Position(), originalFile,
targetConfig, id, index, node, CURRENT_VERSION,
result->m_currentVersion));
}
// <alt-trans>
XMLNode* altTrans = get_unique_node(transUnit, XLIFF_XMLNS, "alt-trans", false);
if (altTrans != NULL) {
// <source>
XMLNode* altSource = get_unique_node(altTrans, XLIFF_XMLNS, "source", false);
if (altSource != NULL) {
XMLNode* node = altSource->Clone();
node->SetPrettyRecursive(XMLNode::EXACT);
result->AddStringResource(StringResource(altSource->Position(),
originalFile, sourceConfig, id, index, node, OLD_VERSION,
result->m_oldVersion));
}
// <target>
XMLNode* altTarget = get_unique_node(altTrans, XLIFF_XMLNS, "target", false);
if (altTarget != NULL) {
XMLNode* node = altTarget->Clone();
node->SetPrettyRecursive(XMLNode::EXACT);
result->AddStringResource(StringResource(altTarget->Position(),
originalFile, targetConfig, id, index, node, OLD_VERSION,
result->m_oldVersion));
}
}
}
}
delete root;
return result;
}
XLIFFFile*
XLIFFFile::Create(const Configuration& sourceConfig, const Configuration& targetConfig,
const string& currentVersion)
{
XLIFFFile* result = new XLIFFFile();
result->m_sourceConfig = sourceConfig;
result->m_targetConfig = targetConfig;
result->m_currentVersion = currentVersion;
return result;
}
set<string>
XLIFFFile::Files() const
{
set<string> result;
for (vector<File>::const_iterator f = m_files.begin(); f != m_files.end(); f++) {
result.insert(f->filename);
}
return result;
}
void
XLIFFFile::AddStringResource(const StringResource& str)
{
string id = str.TypedID();
File* f = NULL;
const size_t I = m_files.size();
for (size_t i=0; i<I; i++) {
if (m_files[i].filename == str.file) {
f = &m_files[i];
break;
}
}
if (f == NULL) {
File file;
file.filename = str.file;
m_files.push_back(file);
f = &m_files[I];
}
const size_t J = f->transUnits.size();
TransUnit* g = NULL;
for (size_t j=0; j<J; j++) {
if (f->transUnits[j].id == id) {
g = &f->transUnits[j];
}
}
if (g == NULL) {
TransUnit group;
group.id = id;
f->transUnits.push_back(group);
g = &f->transUnits[J];
}
StringResource* res = find_string_res(*g, str);
if (res == NULL) {
return ;
}
if (res->id != "") {
str.pos.Error("Duplicate string resource: %s", res->id.c_str());
res->pos.Error("Previous definition here");
return ;
}
*res = str;
m_strings.insert(str);
}
void
XLIFFFile::Filter(bool (*func)(const string&,const TransUnit&,void*), void* cookie)
{
const size_t I = m_files.size();
for (size_t ix=0, i=I-1; ix<I; ix++, i--) {
File& file = m_files[i];
const size_t J = file.transUnits.size();
for (size_t jx=0, j=J-1; jx<J; jx++, j--) {
TransUnit& tu = file.transUnits[j];
bool keep = func(file.filename, tu, cookie);
if (!keep) {
if (tu.source.id != "") {
m_strings.erase(tu.source);
}
if (tu.target.id != "") {
m_strings.erase(tu.target);
}
if (tu.altSource.id != "") {
m_strings.erase(tu.altSource);
}
if (tu.altTarget.id != "") {
m_strings.erase(tu.altTarget);
}
file.transUnits.erase(file.transUnits.begin()+j);
}
}
if (file.transUnits.size() == 0) {
m_files.erase(m_files.begin()+i);
}
}
}
void
XLIFFFile::Map(void (*func)(const string&,TransUnit*,void*), void* cookie)
{
const size_t I = m_files.size();
for (size_t i=0; i<I; i++) {
File& file = m_files[i];
const size_t J = file.transUnits.size();
for (size_t j=0; j<J; j++) {
func(file.filename, &(file.transUnits[j]), cookie);
}
}
}
TransUnit*
XLIFFFile::EditTransUnit(const string& filename, const string& id)
{
const size_t I = m_files.size();
for (size_t ix=0, i=I-1; ix<I; ix++, i--) {
File& file = m_files[i];
if (file.filename == filename) {
const size_t J = file.transUnits.size();
for (size_t jx=0, j=J-1; jx<J; jx++, j--) {
TransUnit& tu = file.transUnits[j];
if (tu.id == id) {
return &tu;
}
}
}
}
return NULL;
}
StringResource*
XLIFFFile::find_string_res(TransUnit& g, const StringResource& str)
{
int index;
if (str.version == CURRENT_VERSION) {
index = 0;
}
else if (str.version == OLD_VERSION) {
index = 2;
}
else {
str.pos.Error("Internal Error %s:%d\n", __FILE__, __LINE__);
return NULL;
}
if (str.config == m_sourceConfig) {
// index += 0;
}
else if (str.config == m_targetConfig) {
index += 1;
}
else {
str.pos.Error("unknown config for string %s: %s", str.id.c_str(),
str.config.ToString().c_str());
return NULL;
}
switch (index) {
case 0:
return &g.source;
case 1:
return &g.target;
case 2:
return &g.altSource;
case 3:
return &g.altTarget;
}
str.pos.Error("Internal Error %s:%d\n", __FILE__, __LINE__);
return NULL;
}
int
convert_html_to_xliff(const XMLNode* original, const string& name, XMLNode* addTo, int* phID)
{
int err = 0;
if (original->Type() == XMLNode::TEXT) {
addTo->EditChildren().push_back(original->Clone());
return 0;
} else {
string ctype;
if (original->Namespace() == "") {
if (original->Name() == "b") {
ctype = "bold";
}
else if (original->Name() == "i") {
ctype = "italic";
}
else if (original->Name() == "u") {
ctype = "underline";
}
}
if (ctype != "") {
vector<XMLAttribute> attrs;
attrs.push_back(XMLAttribute(XLIFF_XMLNS, "ctype", ctype));
XMLNode* copy = XMLNode::NewElement(original->Position(), XLIFF_XMLNS, "g",
attrs, XMLNode::EXACT);
const vector<XMLNode*>& children = original->Children();
size_t I = children.size();
for (size_t i=0; i<I; i++) {
err |= convert_html_to_xliff(children[i], name, copy, phID);
}
return err;
}
else {
if (original->Namespace() == XLIFF_XMLNS) {
addTo->EditChildren().push_back(original->Clone());
return 0;
} else {
if (original->Namespace() == "") {
// flatten out the tag into ph tags -- but only if there is no namespace
// that's still unsupported because propagating the xmlns attribute is hard.
vector<XMLAttribute> attrs;
char idStr[30];
(*phID)++;
sprintf(idStr, "id-%d", *phID);
attrs.push_back(XMLAttribute(XLIFF_XMLNS, "id", idStr));
if (original->Children().size() == 0) {
XMLNode* ph = XMLNode::NewElement(original->Position(), XLIFF_XMLNS,
"ph", attrs, XMLNode::EXACT);
ph->EditChildren().push_back(
XMLNode::NewText(original->Position(),
original->ToString(XLIFF_NAMESPACES),
XMLNode::EXACT));
addTo->EditChildren().push_back(ph);
} else {
XMLNode* begin = XMLNode::NewElement(original->Position(), XLIFF_XMLNS,
"bpt", attrs, XMLNode::EXACT);
begin->EditChildren().push_back(
XMLNode::NewText(original->Position(),
original->OpenTagToString(XLIFF_NAMESPACES, XMLNode::EXACT),
XMLNode::EXACT));
XMLNode* end = XMLNode::NewElement(original->Position(), XLIFF_XMLNS,
"ept", attrs, XMLNode::EXACT);
string endText = "</";
endText += original->Name();
endText += ">";
end->EditChildren().push_back(XMLNode::NewText(original->Position(),
endText, XMLNode::EXACT));
addTo->EditChildren().push_back(begin);
const vector<XMLNode*>& children = original->Children();
size_t I = children.size();
for (size_t i=0; i<I; i++) {
err |= convert_html_to_xliff(children[i], name, addTo, phID);
}
addTo->EditChildren().push_back(end);
}
return err;
} else {
original->Position().Error("invalid <%s> element in <%s> tag\n",
original->Name().c_str(), name.c_str());
return 1;
}
}
}
}
}
XMLNode*
create_string_node(const StringResource& str, const string& name)
{
vector<XMLAttribute> attrs;
attrs.push_back(XMLAttribute(XMLNS_XMLNS, "space", "preserve"));
XMLNode* node = XMLNode::NewElement(str.pos, XLIFF_XMLNS, name, attrs, XMLNode::EXACT);
const vector<XMLNode*>& children = str.value->Children();
size_t I = children.size();
int err = 0;
for (size_t i=0; i<I; i++) {
int phID = 0;
err |= convert_html_to_xliff(children[i], name, node, &phID);
}
if (err != 0) {
delete node;
}
return node;
}
static bool
compare_id(const TransUnit& lhs, const TransUnit& rhs)
{
string lid, rid;
int lindex, rindex;
StringResource::ParseTypedID(lhs.id, &lid, &lindex);
StringResource::ParseTypedID(rhs.id, &rid, &rindex);
if (lid < rid) return true;
if (lid == rid && lindex < rindex) return true;
return false;
}
XMLNode*
XLIFFFile::ToXMLNode() const
{
XMLNode* root;
size_t N;
// <xliff>
{
vector<XMLAttribute> attrs;
XLIFF_NAMESPACES.AddToAttributes(&attrs);
attrs.push_back(XMLAttribute(XLIFF_XMLNS, "version", "1.2"));
root = XMLNode::NewElement(GENERATED_POS, XLIFF_XMLNS, "xliff", attrs, XMLNode::PRETTY);
}
vector<TransUnit> groups;
// <file>
vector<File> files = m_files;
sort(files.begin(), files.end());
const size_t I = files.size();
for (size_t i=0; i<I; i++) {
const File& file = files[i];
vector<XMLAttribute> fileAttrs;
fileAttrs.push_back(XMLAttribute(XLIFF_XMLNS, "datatype", "x-android-res"));
fileAttrs.push_back(XMLAttribute(XLIFF_XMLNS, "original", file.filename));
struct timeval tv;
struct timezone tz;
gettimeofday(&tv, &tz);
fileAttrs.push_back(XMLAttribute(XLIFF_XMLNS, "date", trim_string(ctime(&tv.tv_sec))));
fileAttrs.push_back(XMLAttribute(XLIFF_XMLNS, "source-language", m_sourceConfig.locale));
fileAttrs.push_back(XMLAttribute(XLIFF_XMLNS, "target-language", m_targetConfig.locale));
fileAttrs.push_back(XMLAttribute(XLIFF_XMLNS, "build-num", m_currentVersion));
XMLNode* fileNode = XMLNode::NewElement(GENERATED_POS, XLIFF_XMLNS, "file", fileAttrs,
XMLNode::PRETTY);
root->EditChildren().push_back(fileNode);
// <body>
XMLNode* bodyNode = XMLNode::NewElement(GENERATED_POS, XLIFF_XMLNS, "body",
vector<XMLAttribute>(), XMLNode::PRETTY);
fileNode->EditChildren().push_back(bodyNode);
// <trans-unit>
vector<TransUnit> transUnits = file.transUnits;
sort(transUnits.begin(), transUnits.end(), compare_id);
const size_t J = transUnits.size();
for (size_t j=0; j<J; j++) {
const TransUnit& transUnit = transUnits[j];
vector<XMLAttribute> tuAttrs;
// strings start with string:
tuAttrs.push_back(XMLAttribute(XLIFF_XMLNS, "id", transUnit.id));
XMLNode* transUnitNode = XMLNode::NewElement(GENERATED_POS, XLIFF_XMLNS, "trans-unit",
tuAttrs, XMLNode::PRETTY);
bodyNode->EditChildren().push_back(transUnitNode);
// <extradata>
if (transUnit.source.comment != "") {
vector<XMLAttribute> extradataAttrs;
XMLNode* extraNode = XMLNode::NewElement(GENERATED_POS, XLIFF_XMLNS, "extradata",
extradataAttrs, XMLNode::EXACT);
transUnitNode->EditChildren().push_back(extraNode);
extraNode->EditChildren().push_back(
XMLNode::NewText(GENERATED_POS, transUnit.source.comment,
XMLNode::PRETTY));
}
// <source>
if (transUnit.source.id != "") {
transUnitNode->EditChildren().push_back(
create_string_node(transUnit.source, "source"));
}
// <target>
if (transUnit.target.id != "") {
transUnitNode->EditChildren().push_back(
create_string_node(transUnit.target, "target"));
}
// <alt-trans>
if (transUnit.altSource.id != "" || transUnit.altTarget.id != ""
|| transUnit.rejectComment != "") {
vector<XMLAttribute> altTransAttrs;
XMLNode* altTransNode = XMLNode::NewElement(GENERATED_POS, XLIFF_XMLNS, "alt-trans",
altTransAttrs, XMLNode::PRETTY);
transUnitNode->EditChildren().push_back(altTransNode);
// <extradata>
if (transUnit.rejectComment != "") {
vector<XMLAttribute> extradataAttrs;
XMLNode* extraNode = XMLNode::NewElement(GENERATED_POS, XLIFF_XMLNS,
"extradata", extradataAttrs,
XMLNode::EXACT);
altTransNode->EditChildren().push_back(extraNode);
extraNode->EditChildren().push_back(
XMLNode::NewText(GENERATED_POS, transUnit.rejectComment,
XMLNode::PRETTY));
}
// <source>
if (transUnit.altSource.id != "") {
altTransNode->EditChildren().push_back(
create_string_node(transUnit.altSource, "source"));
}
// <target>
if (transUnit.altTarget.id != "") {
altTransNode->EditChildren().push_back(
create_string_node(transUnit.altTarget, "target"));
}
}
}
}
return root;
}
string
XLIFFFile::ToString() const
{
XMLNode* xml = ToXMLNode();
string s = "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n";
s += xml->ToString(XLIFF_NAMESPACES);
delete xml;
s += '\n';
return s;
}
Stats
XLIFFFile::GetStats(const string& config) const
{
Stats stat;
stat.config = config;
stat.files = m_files.size();
stat.toBeTranslated = 0;
stat.noComments = 0;
for (vector<File>::const_iterator file=m_files.begin(); file!=m_files.end(); file++) {
stat.toBeTranslated += file->transUnits.size();
for (vector<TransUnit>::const_iterator tu=file->transUnits.begin();
tu!=file->transUnits.end(); tu++) {
if (tu->source.comment == "") {
stat.noComments++;
}
}
}
stat.totalStrings = stat.toBeTranslated;
return stat;
}

98
tools/localize/XLIFFFile.h
View File

@@ -1,98 +0,0 @@
#ifndef XLIFF_FILE_H
#define XLIFF_FILE_H
#include "Values.h"
#include "Configuration.h"
#include <set>
using namespace std;
extern const XMLNamespaceMap XLIFF_NAMESPACES;
extern const char*const XLIFF_XMLNS;
struct Stats
{
string config;
size_t files;
size_t toBeTranslated;
size_t noComments;
size_t totalStrings;
};
struct TransUnit {
string id;
StringResource source;
StringResource target;
StringResource altSource;
StringResource altTarget;
string rejectComment;
};
class XLIFFFile
{
public:
static XLIFFFile* Parse(const string& filename);
static XLIFFFile* Create(const Configuration& sourceConfig, const Configuration& targetConfig,
const string& currentVersion);
~XLIFFFile();
inline const Configuration& SourceConfig() const { return m_sourceConfig; }
inline const Configuration& TargetConfig() const { return m_targetConfig; }
inline const string& CurrentVersion() const { return m_currentVersion; }
inline const string& OldVersion() const { return m_oldVersion; }
set<string> Files() const;
void AddStringResource(const StringResource& res);
inline set<StringResource> const& GetStringResources() const { return m_strings; }
bool FindStringResource(const string& filename, int version, bool source);
void Filter(bool (*func)(const string&,const TransUnit&,void*), void* cookie);
void Map(void (*func)(const string&,TransUnit*,void*), void* cookie);
TransUnit* EditTransUnit(const string& file, const string& id);
// exports this file as a n XMLNode, you own this object
XMLNode* ToXMLNode() const;
// writes the ValuesFile out to a string in the canonical format (i.e. writes the contents of
// ToXMLNode()).
string ToString() const;
Stats GetStats(const string& config) const;
private:
struct File {
int Compare(const File& that) const;
inline bool operator<(const File& that) const { return Compare(that) < 0; }
inline bool operator<=(const File& that) const { return Compare(that) <= 0; }
inline bool operator==(const File& that) const { return Compare(that) == 0; }
inline bool operator!=(const File& that) const { return Compare(that) != 0; }
inline bool operator>=(const File& that) const { return Compare(that) >= 0; }
inline bool operator>(const File& that) const { return Compare(that) > 0; }
string filename;
vector<TransUnit> transUnits;
};
XLIFFFile();
StringResource* find_string_res(TransUnit& g, const StringResource& str);
Configuration m_sourceConfig;
Configuration m_targetConfig;
string m_currentVersion;
string m_oldVersion;
set<StringResource> m_strings;
vector<File> m_files;
};
int convert_html_to_xliff(const XMLNode* original, const string& name, XMLNode* addTo, int* phID);
#endif // XLIFF_FILE_H

115
tools/localize/XLIFFFile_test.cpp
View File

@@ -1,115 +0,0 @@
#include "XLIFFFile.h"
#include <stdio.h>
#include "ValuesFile.h"
XMLNode* create_string_node(const StringResource& str, const string& name);
static int
Parse_test()
{
XLIFFFile* xf = XLIFFFile::Parse("testdata/xliff1.xliff");
if (xf == NULL) {
return 1;
}
set<StringResource> const& strings = xf->GetStringResources();
if (false) {
for (set<StringResource>::iterator it=strings.begin(); it!=strings.end(); it++) {
const StringResource& str = *it;
printf("STRING!!! id=%s index=%d value='%s' pos=%s file=%s version=%d(%s)\n",
str.id.c_str(), str.index,
str.value->ContentsToString(ANDROID_NAMESPACES).c_str(),
str.pos.ToString().c_str(), str.file.c_str(), str.version,
str.versionString.c_str());
}
printf("XML:[[%s]]\n", xf->ToString().c_str());
}
delete xf;
return 0;
}
static XMLNode*
add_html_tag(XMLNode* addTo, const string& tag)
{
vector<XMLAttribute> attrs;
XMLNode* node = XMLNode::NewElement(GENERATED_POS, "", tag, attrs, XMLNode::EXACT);
addTo->EditChildren().push_back(node);
return node;
}
static int
create_string_node_test()
{
int err = 0;
StringResource res;
vector<XMLAttribute> attrs;
res.value = XMLNode::NewElement(GENERATED_POS, "", "something", attrs, XMLNode::EXACT);
res.value->EditChildren().push_back(XMLNode::NewText(GENERATED_POS, " begin ", XMLNode::EXACT));
XMLNode* child;
child = add_html_tag(res.value, "b");
child->EditChildren().push_back(XMLNode::NewText(GENERATED_POS, "b", XMLNode::EXACT));
child = add_html_tag(res.value, "i");
child->EditChildren().push_back(XMLNode::NewText(GENERATED_POS, "i", XMLNode::EXACT));
child = add_html_tag(child, "b");
child->EditChildren().push_back(XMLNode::NewText(GENERATED_POS, "b", XMLNode::EXACT));
child = add_html_tag(res.value, "u");
child->EditChildren().push_back(XMLNode::NewText(GENERATED_POS, "u", XMLNode::EXACT));
res.value->EditChildren().push_back(XMLNode::NewText(GENERATED_POS, " end ", XMLNode::EXACT));
XMLNode* xliff = create_string_node(res, "blah");
string oldString = res.value->ToString(XLIFF_NAMESPACES);
string newString = xliff->ToString(XLIFF_NAMESPACES);
if (false) {
printf("OLD=\"%s\"\n", oldString.c_str());
printf("NEW=\"%s\"\n", newString.c_str());
}
const char* const EXPECTED_OLD
= "<something> begin <b>b</b><i>i<b>b</b></i><u>u</u> end </something>";
if (oldString != EXPECTED_OLD) {
fprintf(stderr, "oldString mismatch:\n");
fprintf(stderr, " expected='%s'\n", EXPECTED_OLD);
fprintf(stderr, " actual='%s'\n", oldString.c_str());
err |= 1;
}
const char* const EXPECTED_NEW
= "<blah xml:space=\"preserve\"> begin <g ctype=\"bold\">b</g>"
"<g ctype=\"italic\">i<g ctype=\"bold\">b</g></g><g ctype=\"underline\">u</g>"
" end </blah>";
if (newString != EXPECTED_NEW) {
fprintf(stderr, "newString mismatch:\n");
fprintf(stderr, " expected='%s'\n", EXPECTED_NEW);
fprintf(stderr, " actual='%s'\n", newString.c_str());
err |= 1;
}
if (err != 0) {
fprintf(stderr, "create_string_node_test failed\n");
}
return err;
}
int
XLIFFFile_test()
{
bool all = true;
int err = 0;
if (all) err |= Parse_test();
if (all) err |= create_string_node_test();
return err;
}

793
tools/localize/XMLHandler.cpp
View File

@@ -1,793 +0,0 @@
#include "XMLHandler.h"
#include <algorithm>
#include <expat.h>
#include <stdio.h>
#include <string.h>
#include <fcntl.h>
#include <unistd.h>
#include <errno.h>
#define NS_SEPARATOR 1
#define MORE_INDENT " "
static string
xml_text_escape(const string& s)
{
string result;
const size_t N = s.length();
for (size_t i=0; i<N; i++) {
char c = s[i];
switch (c) {
case '<':
result += "&lt;";
break;
case '>':
result += "&gt;";
break;
case '&':
result += "&amp;";
break;
default:
result += c;
break;
}
}
return result;
}
static string
xml_attr_escape(const string& s)
{
string result;
const size_t N = s.length();
for (size_t i=0; i<N; i++) {
char c = s[i];
switch (c) {
case '\"':
result += "&quot;";
break;
default:
result += c;
break;
}
}
return result;
}
XMLNamespaceMap::XMLNamespaceMap()
{
}
XMLNamespaceMap::XMLNamespaceMap(char const*const* nspaces)
{
while (*nspaces) {
m_map[nspaces[1]] = nspaces[0];
nspaces += 2;
}
}
string
XMLNamespaceMap::Get(const string& ns) const
{
if (ns == "xml") {
return ns;
}
map<string,string>::const_iterator it = m_map.find(ns);
if (it == m_map.end()) {
return "";
} else {
return it->second;
}
}
string
XMLNamespaceMap::GetPrefix(const string& ns) const
{
if (ns == "") {
return "";
}
map<string,string>::const_iterator it = m_map.find(ns);
if (it != m_map.end()) {
if (it->second == "") {
return "";
} else {
return it->second + ":";
}
} else {
return ":"; // invalid
}
}
void
XMLNamespaceMap::AddToAttributes(vector<XMLAttribute>* attrs) const
{
map<string,string>::const_iterator it;
for (it=m_map.begin(); it!=m_map.end(); it++) {
if (it->second == "xml") {
continue;
}
XMLAttribute attr;
if (it->second == "") {
attr.name = "xmlns";
} else {
attr.name = "xmlns:";
attr.name += it->second;
}
attr.value = it->first;
attrs->push_back(attr);
}
}
XMLAttribute::XMLAttribute()
{
}
XMLAttribute::XMLAttribute(const XMLAttribute& that)
:ns(that.ns),
name(that.name),
value(that.value)
{
}
XMLAttribute::XMLAttribute(string n, string na, string v)
:ns(n),
name(na),
value(v)
{
}
XMLAttribute::~XMLAttribute()
{
}
int
XMLAttribute::Compare(const XMLAttribute& that) const
{
if (ns != that.ns) {
return ns < that.ns ? -1 : 1;
}
if (name != that.name) {
return name < that.name ? -1 : 1;
}
return 0;
}
string
XMLAttribute::Find(const vector<XMLAttribute>& list, const string& ns, const string& name,
const string& def)
{
const size_t N = list.size();
for (size_t i=0; i<N; i++) {
const XMLAttribute& attr = list[i];
if (attr.ns == ns && attr.name == name) {
return attr.value;
}
}
return def;
}
struct xml_handler_data {
vector<XMLHandler*> stack;
XML_Parser parser;
vector<vector<XMLAttribute>*> attributes;
string filename;
};
XMLNode::XMLNode()
{
}
XMLNode::~XMLNode()
{
// for_each(m_children.begin(), m_children.end(), delete_object<XMLNode>);
}
XMLNode*
XMLNode::Clone() const
{
switch (m_type) {
case ELEMENT: {
XMLNode* e = XMLNode::NewElement(m_pos, m_ns, m_name, m_attrs, m_pretty);
const size_t N = m_children.size();
for (size_t i=0; i<N; i++) {
e->m_children.push_back(m_children[i]->Clone());
}
return e;
}
case TEXT: {
return XMLNode::NewText(m_pos, m_text, m_pretty);
}
default:
return NULL;
}
}
XMLNode*
XMLNode::NewElement(const SourcePos& pos, const string& ns, const string& name,
const vector<XMLAttribute>& attrs, int pretty)
{
XMLNode* node = new XMLNode();
node->m_type = ELEMENT;
node->m_pretty = pretty;
node->m_pos = pos;
node->m_ns = ns;
node->m_name = name;
node->m_attrs = attrs;
return node;
}
XMLNode*
XMLNode::NewText(const SourcePos& pos, const string& text, int pretty)
{
XMLNode* node = new XMLNode();
node->m_type = TEXT;
node->m_pretty = pretty;
node->m_pos = pos;
node->m_text = text;
return node;
}
void
XMLNode::SetPrettyRecursive(int value)
{
m_pretty = value;
const size_t N = m_children.size();
for (size_t i=0; i<N; i++) {
m_children[i]->SetPrettyRecursive(value);
}
}
string
XMLNode::ContentsToString(const XMLNamespaceMap& nspaces) const
{
return contents_to_string(nspaces, "");
}
string
XMLNode::ToString(const XMLNamespaceMap& nspaces) const
{
return to_string(nspaces, "");
}
string
XMLNode::OpenTagToString(const XMLNamespaceMap& nspaces, int pretty) const
{
return open_tag_to_string(nspaces, "", pretty);
}
string
XMLNode::contents_to_string(const XMLNamespaceMap& nspaces, const string& indent) const
{
string result;
const size_t N = m_children.size();
for (size_t i=0; i<N; i++) {
const XMLNode* child = m_children[i];
switch (child->Type()) {
case ELEMENT:
if (m_pretty == PRETTY) {
result += '\n';
result += indent;
}
case TEXT:
result += child->to_string(nspaces, indent);
break;
}
}
return result;
}
string
trim_string(const string& str)
{
const char* p = str.c_str();
while (*p && isspace(*p)) {
p++;
}
const char* q = str.c_str() + str.length() - 1;
while (q > p && isspace(*q)) {
q--;
}
q++;
return string(p, q-p);
}
string
XMLNode::open_tag_to_string(const XMLNamespaceMap& nspaces, const string& indent, int pretty) const
{
if (m_type != ELEMENT) {
return "";
}
string result = "<";
result += nspaces.GetPrefix(m_ns);
result += m_name;
vector<XMLAttribute> attrs = m_attrs;
sort(attrs.begin(), attrs.end());
const size_t N = attrs.size();
for (size_t i=0; i<N; i++) {
const XMLAttribute& attr = attrs[i];
if (i == 0 || m_pretty == EXACT || pretty == EXACT) {
result += ' ';
}
else {
result += "\n";
result += indent;
result += MORE_INDENT;
result += MORE_INDENT;
}
result += nspaces.GetPrefix(attr.ns);
result += attr.name;
result += "=\"";
result += xml_attr_escape(attr.value);
result += '\"';
}
if (m_children.size() > 0) {
result += '>';
} else {
result += " />";
}
return result;
}
string
XMLNode::to_string(const XMLNamespaceMap& nspaces, const string& indent) const
{
switch (m_type)
{
case TEXT: {
if (m_pretty == EXACT) {
return xml_text_escape(m_text);
} else {
return xml_text_escape(trim_string(m_text));
}
}
case ELEMENT: {
string result = open_tag_to_string(nspaces, indent, PRETTY);
if (m_children.size() > 0) {
result += contents_to_string(nspaces, indent + MORE_INDENT);
if (m_pretty == PRETTY && m_children.size() > 0) {
result += '\n';
result += indent;
}
result += "</";
result += nspaces.GetPrefix(m_ns);
result += m_name;
result += '>';
}
return result;
}
default:
return "";
}
}
string
XMLNode::CollapseTextContents() const
{
if (m_type == TEXT) {
return m_text;
}
else if (m_type == ELEMENT) {
string result;
const size_t N=m_children.size();
for (size_t i=0; i<N; i++) {
result += m_children[i]->CollapseTextContents();
}
return result;
}
else {
return "";
}
}
vector<XMLNode*>
XMLNode::GetElementsByName(const string& ns, const string& name) const
{
vector<XMLNode*> result;
const size_t N=m_children.size();
for (size_t i=0; i<N; i++) {
XMLNode* child = m_children[i];
if (child->m_type == ELEMENT && child->m_ns == ns && child->m_name == name) {
result.push_back(child);
}
}
return result;
}
XMLNode*
XMLNode::GetElementByNameAt(const string& ns, const string& name, size_t index) const
{
vector<XMLNode*> result;
const size_t N=m_children.size();
for (size_t i=0; i<N; i++) {
XMLNode* child = m_children[i];
if (child->m_type == ELEMENT && child->m_ns == ns && child->m_name == name) {
if (index == 0) {
return child;
} else {
index--;
}
}
}
return NULL;
}
size_t
XMLNode::CountElementsByName(const string& ns, const string& name) const
{
size_t result = 0;
const size_t N=m_children.size();
for (size_t i=0; i<N; i++) {
XMLNode* child = m_children[i];
if (child->m_type == ELEMENT && child->m_ns == ns && child->m_name == name) {
result++;
}
}
return result;
}
string
XMLNode::GetAttribute(const string& ns, const string& name, const string& def) const
{
return XMLAttribute::Find(m_attrs, ns, name, def);
}
static void
parse_namespace(const char* data, string* ns, string* name)
{
const char* p = strchr(data, NS_SEPARATOR);
if (p != NULL) {
ns->assign(data, p-data);
name->assign(p+1);
} else {
ns->assign("");
name->assign(data);
}
}
static void
convert_attrs(const char** in, vector<XMLAttribute>* out)
{
while (*in) {
XMLAttribute attr;
parse_namespace(in[0], &attr.ns, &attr.name);
attr.value = in[1];
out->push_back(attr);
in += 2;
}
}
static bool
list_contains(const vector<XMLHandler*>& stack, XMLHandler* handler)
{
const size_t N = stack.size();
for (size_t i=0; i<N; i++) {
if (stack[i] == handler) {
return true;
}
}
return false;
}
static void XMLCALL
start_element_handler(void *userData, const char *name, const char **attrs)
{
xml_handler_data* data = (xml_handler_data*)userData;
XMLHandler* handler = data->stack[data->stack.size()-1];
SourcePos pos(data->filename, (int)XML_GetCurrentLineNumber(data->parser));
string nsString;
string nameString;
XMLHandler* next = handler;
vector<XMLAttribute> attributes;
parse_namespace(name, &nsString, &nameString);
convert_attrs(attrs, &attributes);
handler->OnStartElement(pos, nsString, nameString, attributes, &next);
if (next == NULL) {
next = handler;
}
if (next != handler) {
next->elementPos = pos;
next->elementNamespace = nsString;
next->elementName = nameString;
next->elementAttributes = attributes;
}
data->stack.push_back(next);
}
static void XMLCALL
end_element_handler(void *userData, const char *name)
{
xml_handler_data* data = (xml_handler_data*)userData;
XMLHandler* handler = data->stack[data->stack.size()-1];
data->stack.pop_back();
SourcePos pos(data->filename, (int)XML_GetCurrentLineNumber(data->parser));
if (!list_contains(data->stack, handler)) {
handler->OnDone(pos);
if (data->stack.size() > 1) {
// not top one
delete handler;
}
}
handler = data->stack[data->stack.size()-1];
string nsString;
string nameString;
parse_namespace(name, &nsString, &nameString);
handler->OnEndElement(pos, nsString, nameString);
}
static void XMLCALL
text_handler(void *userData, const XML_Char *s, int len)
{
xml_handler_data* data = (xml_handler_data*)userData;
XMLHandler* handler = data->stack[data->stack.size()-1];
SourcePos pos(data->filename, (int)XML_GetCurrentLineNumber(data->parser));
handler->OnText(pos, string(s, len));
}
static void XMLCALL
comment_handler(void *userData, const char *comment)
{
xml_handler_data* data = (xml_handler_data*)userData;
XMLHandler* handler = data->stack[data->stack.size()-1];
SourcePos pos(data->filename, (int)XML_GetCurrentLineNumber(data->parser));
handler->OnComment(pos, string(comment));
}
bool
XMLHandler::ParseFile(const string& filename, XMLHandler* handler)
{
char buf[16384];
int fd = open(filename.c_str(), O_RDONLY);
if (fd < 0) {
SourcePos(filename, -1).Error("Unable to open file for read: %s", strerror(errno));
return false;
}
XML_Parser parser = XML_ParserCreateNS(NULL, NS_SEPARATOR);
xml_handler_data state;
state.stack.push_back(handler);
state.parser = parser;
state.filename = filename;
XML_SetUserData(parser, &state);
XML_SetElementHandler(parser, start_element_handler, end_element_handler);
XML_SetCharacterDataHandler(parser, text_handler);
XML_SetCommentHandler(parser, comment_handler);
ssize_t len;
bool done;
do {
len = read(fd, buf, sizeof(buf));
done = len < (ssize_t)sizeof(buf);
if (len < 0) {
SourcePos(filename, -1).Error("Error reading file: %s\n", strerror(errno));
close(fd);
return false;
}
if (XML_Parse(parser, buf, len, done) == XML_STATUS_ERROR) {
SourcePos(filename, (int)XML_GetCurrentLineNumber(parser)).Error(
"Error parsing XML: %s\n", XML_ErrorString(XML_GetErrorCode(parser)));
close(fd);
return false;
}
} while (!done);
XML_ParserFree(parser);
close(fd);
return true;
}
bool
XMLHandler::ParseString(const string& filename, const string& text, XMLHandler* handler)
{
XML_Parser parser = XML_ParserCreateNS(NULL, NS_SEPARATOR);
xml_handler_data state;
state.stack.push_back(handler);
state.parser = parser;
state.filename = filename;
XML_SetUserData(parser, &state);
XML_SetElementHandler(parser, start_element_handler, end_element_handler);
XML_SetCharacterDataHandler(parser, text_handler);
XML_SetCommentHandler(parser, comment_handler);
if (XML_Parse(parser, text.c_str(), text.size(), true) == XML_STATUS_ERROR) {
SourcePos(filename, (int)XML_GetCurrentLineNumber(parser)).Error(
"Error parsing XML: %s\n", XML_ErrorString(XML_GetErrorCode(parser)));
return false;
}
XML_ParserFree(parser);
return true;
}
XMLHandler::XMLHandler()
{
}
XMLHandler::~XMLHandler()
{
}
int
XMLHandler::OnStartElement(const SourcePos& pos, const string& ns, const string& name,
const vector<XMLAttribute>& attrs, XMLHandler** next)
{
return 0;
}
int
XMLHandler::OnEndElement(const SourcePos& pos, const string& ns, const string& name)
{
return 0;
}
int
XMLHandler::OnText(const SourcePos& pos, const string& text)
{
return 0;
}
int
XMLHandler::OnComment(const SourcePos& pos, const string& text)
{
return 0;
}
int
XMLHandler::OnDone(const SourcePos& pos)
{
return 0;
}
TopElementHandler::TopElementHandler(const string& ns, const string& name, XMLHandler* next)
:m_ns(ns),
m_name(name),
m_next(next)
{
}
int
TopElementHandler::OnStartElement(const SourcePos& pos, const string& ns, const string& name,
const vector<XMLAttribute>& attrs, XMLHandler** next)
{
*next = m_next;
return 0;
}
int
TopElementHandler::OnEndElement(const SourcePos& pos, const string& ns, const string& name)
{
return 0;
}
int
TopElementHandler::OnText(const SourcePos& pos, const string& text)
{
return 0;
}
int
TopElementHandler::OnDone(const SourcePos& pos)
{
return 0;
}
NodeHandler::NodeHandler(XMLNode* root, int pretty)
:m_root(root),
m_pretty(pretty)
{
if (root != NULL) {
m_nodes.push_back(root);
}
}
NodeHandler::~NodeHandler()
{
}
int
NodeHandler::OnStartElement(const SourcePos& pos, const string& ns, const string& name,
const vector<XMLAttribute>& attrs, XMLHandler** next)
{
int pretty;
if (XMLAttribute::Find(attrs, XMLNS_XMLNS, "space", "") == "preserve") {
pretty = XMLNode::EXACT;
} else {
if (m_root == NULL) {
pretty = m_pretty;
} else {
pretty = m_nodes[m_nodes.size()-1]->Pretty();
}
}
XMLNode* n = XMLNode::NewElement(pos, ns, name, attrs, pretty);
if (m_root == NULL) {
m_root = n;
} else {
m_nodes[m_nodes.size()-1]->EditChildren().push_back(n);
}
m_nodes.push_back(n);
return 0;
}
int
NodeHandler::OnEndElement(const SourcePos& pos, const string& ns, const string& name)
{
m_nodes.pop_back();
return 0;
}
int
NodeHandler::OnText(const SourcePos& pos, const string& text)
{
if (m_root == NULL) {
return 1;
}
XMLNode* n = XMLNode::NewText(pos, text, m_nodes[m_nodes.size()-1]->Pretty());
m_nodes[m_nodes.size()-1]->EditChildren().push_back(n);
return 0;
}
int
NodeHandler::OnComment(const SourcePos& pos, const string& text)
{
return 0;
}
int
NodeHandler::OnDone(const SourcePos& pos)
{
return 0;
}
XMLNode*
NodeHandler::ParseFile(const string& filename, int pretty)
{
NodeHandler handler(NULL, pretty);
if (!XMLHandler::ParseFile(filename, &handler)) {
fprintf(stderr, "error parsing file: %s\n", filename.c_str());
return NULL;
}
return handler.Root();
}
XMLNode*
NodeHandler::ParseString(const string& filename, const string& text, int pretty)
{
NodeHandler handler(NULL, pretty);
if (!XMLHandler::ParseString(filename, text, &handler)) {
fprintf(stderr, "error parsing file: %s\n", filename.c_str());
return NULL;
}
return handler.Root();
}

198
tools/localize/XMLHandler.h
View File

@@ -1,198 +0,0 @@
#ifndef XML_H
#define XML_H
#include "SourcePos.h"
#include <algorithm>
#include <string>
#include <vector>
#include <map>
#define XMLNS_XMLNS "http://www.w3.org/XML/1998/namespace"
using namespace std;
string trim_string(const string& str);
struct XMLAttribute
{
string ns;
string name;
string value;
XMLAttribute();
XMLAttribute(const XMLAttribute& that);
XMLAttribute(string ns, string name, string value);
~XMLAttribute();
int Compare(const XMLAttribute& that) const;
inline bool operator<(const XMLAttribute& that) const { return Compare(that) < 0; }
inline bool operator<=(const XMLAttribute& that) const { return Compare(that) <= 0; }
inline bool operator==(const XMLAttribute& that) const { return Compare(that) == 0; }
inline bool operator!=(const XMLAttribute& that) const { return Compare(that) != 0; }
inline bool operator>=(const XMLAttribute& that) const { return Compare(that) >= 0; }
inline bool operator>(const XMLAttribute& that) const { return Compare(that) > 0; }
static string Find(const vector<XMLAttribute>& list,
const string& ns, const string& name, const string& def);
};
class XMLNamespaceMap
{
public:
XMLNamespaceMap();
XMLNamespaceMap(char const*const* nspaces);
string Get(const string& ns) const;
string GetPrefix(const string& ns) const;
void AddToAttributes(vector<XMLAttribute>* attrs) const;
private:
map<string,string> m_map;
};
struct XMLNode
{
public:
enum {
EXACT = 0,
PRETTY = 1
};
enum {
ELEMENT = 0,
TEXT = 1
};
static XMLNode* NewElement(const SourcePos& pos, const string& ns, const string& name,
const vector<XMLAttribute>& attrs, int pretty);
static XMLNode* NewText(const SourcePos& pos, const string& text, int pretty);
~XMLNode();
// a deep copy
XMLNode* Clone() const;
inline int Type() const { return m_type; }
inline int Pretty() const { return m_pretty; }
void SetPrettyRecursive(int value);
string ContentsToString(const XMLNamespaceMap& nspaces) const;
string ToString(const XMLNamespaceMap& nspaces) const;
string OpenTagToString(const XMLNamespaceMap& nspaces, int pretty) const;
string CollapseTextContents() const;
inline const SourcePos& Position() const { return m_pos; }
// element
inline string Namespace() const { return m_ns; }
inline string Name() const { return m_name; }
inline void SetName(const string& ns, const string& n) { m_ns = ns; m_name = n; }
inline const vector<XMLAttribute>& Attributes() const { return m_attrs; }
inline vector<XMLAttribute>& EditAttributes() { return m_attrs; }
inline const vector<XMLNode*>& Children() const { return m_children; }
inline vector<XMLNode*>& EditChildren() { return m_children; }
vector<XMLNode*> GetElementsByName(const string& ns, const string& name) const;
XMLNode* GetElementByNameAt(const string& ns, const string& name, size_t index) const;
size_t CountElementsByName(const string& ns, const string& name) const;
string GetAttribute(const string& ns, const string& name, const string& def) const;
// text
inline string Text() const { return m_text; }
private:
XMLNode();
XMLNode(const XMLNode&);
string contents_to_string(const XMLNamespaceMap& nspaces, const string& indent) const;
string to_string(const XMLNamespaceMap& nspaces, const string& indent) const;
string open_tag_to_string(const XMLNamespaceMap& nspaces, const string& indent,
int pretty) const;
int m_type;
int m_pretty;
SourcePos m_pos;
// element
string m_ns;
string m_name;
vector<XMLAttribute> m_attrs;
vector<XMLNode*> m_children;
// text
string m_text;
};
class XMLHandler
{
public:
// information about the element that started us
SourcePos elementPos;
string elementNamespace;
string elementName;
vector<XMLAttribute> elementAttributes;
XMLHandler();
virtual ~XMLHandler();
XMLHandler* parent;
virtual int OnStartElement(const SourcePos& pos, const string& ns, const string& name,
const vector<XMLAttribute>& attrs, XMLHandler** next);
virtual int OnEndElement(const SourcePos& pos, const string& ns, const string& name);
virtual int OnText(const SourcePos& pos, const string& text);
virtual int OnComment(const SourcePos& pos, const string& text);
virtual int OnDone(const SourcePos& pos);
static bool ParseFile(const string& filename, XMLHandler* handler);
static bool ParseString(const string& filename, const string& text, XMLHandler* handler);
};
class TopElementHandler : public XMLHandler
{
public:
TopElementHandler(const string& ns, const string& name, XMLHandler* next);
virtual int OnStartElement(const SourcePos& pos, const string& ns, const string& name,
const vector<XMLAttribute>& attrs, XMLHandler** next);
virtual int OnEndElement(const SourcePos& pos, const string& ns, const string& name);
virtual int OnText(const SourcePos& pos, const string& text);
virtual int OnDone(const SourcePos& endPos);
private:
string m_ns;
string m_name;
XMLHandler* m_next;
};
class NodeHandler : public XMLHandler
{
public:
// after it's done, you own everything created and added to root
NodeHandler(XMLNode* root, int pretty);
~NodeHandler();
virtual int OnStartElement(const SourcePos& pos, const string& ns, const string& name,
const vector<XMLAttribute>& attrs, XMLHandler** next);
virtual int OnEndElement(const SourcePos& pos, const string& ns, const string& name);
virtual int OnText(const SourcePos& pos, const string& text);
virtual int OnComment(const SourcePos& pos, const string& text);
virtual int OnDone(const SourcePos& endPos);
inline XMLNode* Root() const { return m_root; }
static XMLNode* ParseFile(const string& filename, int pretty);
static XMLNode* ParseString(const string& filename, const string& text, int pretty);
private:
XMLNode* m_root;
int m_pretty;
vector<XMLNode*> m_nodes;
};
template <class T>
static void delete_object(T* obj)
{
delete obj;
}
#endif // XML_H

133
tools/localize/XMLHandler_test.cpp
View File

@@ -1,133 +0,0 @@
#include "XMLHandler.h"
#include <stdio.h>
#include <unistd.h>
#include <fcntl.h>
const char *const NS_MAP[] = {
"xml", XMLNS_XMLNS,
NULL, NULL
};
const XMLNamespaceMap NO_NAMESPACES(NS_MAP);
char const*const EXPECTED_EXACT =
"<ASDF>\n"
" <a id=\"system\" old-cl=\"1\" new-cl=\"43019\">\n"
" <app dir=\"apps/common\" />\n"
" </a>\n"
" <a id=\"samples\" old-cl=\"1\" new-cl=\"43019\">asdf\n"
" <app dir=\"samples/NotePad\" />\n"
" <app dir=\"samples/LunarLander\" />\n"
" <something>a<b>,</b>b </something>\n"
" <exact xml:space=\"preserve\">a<b>,</b>b </exact>\n"
" </a>\n"
"</ASDF>\n";
char const*const EXPECTED_PRETTY =
"<ASDF>\n"
" <a id=\"system\"\n"
" old-cl=\"1\"\n"
" new-cl=\"43019\">\n"
" <app dir=\"apps/common\" />\n"
" </a>\n"
" <a id=\"samples\"\n"
" old-cl=\"1\"\n"
" new-cl=\"43019\">asdf\n"
" <app dir=\"samples/NotePad\" />\n"
" <app dir=\"samples/LunarLander\" />\n"
" <something>a\n"
" <b>,\n"
" </b>b \n"
" </something>\n"
" <exact xml:space=\"preserve\">a<b>,</b>b </exact>\n"
" </a>\n"
"</ASDF>\n";
static string
read_file(const string& filename)
{
char buf[1024];
int fd = open(filename.c_str(), O_RDONLY);
if (fd < 0) {
return "";
}
string result;
while (true) {
ssize_t len = read(fd, buf, sizeof(buf)-1);
buf[len] = '\0';
if (len <= 0) {
break;
}
result.append(buf, len);
}
close(fd);
return result;
}
static int
ParseFile_EXACT_test()
{
XMLNode* root = NodeHandler::ParseFile("testdata/xml.xml", XMLNode::EXACT);
if (root == NULL) {
return 1;
}
string result = root->ToString(NO_NAMESPACES);
delete root;
//printf("[[%s]]\n", result.c_str());
return result == EXPECTED_EXACT;
}
static int
ParseFile_PRETTY_test()
{
XMLNode* root = NodeHandler::ParseFile("testdata/xml.xml", XMLNode::PRETTY);
if (root == NULL) {
return 1;
}
string result = root->ToString(NO_NAMESPACES);
delete root;
//printf("[[%s]]\n", result.c_str());
return result == EXPECTED_PRETTY;
}
static int
ParseString_EXACT_test()
{
string text = read_file("testdata/xml.xml");
XMLNode* root = NodeHandler::ParseString("testdata/xml.xml", text, XMLNode::EXACT);
if (root == NULL) {
return 1;
}
string result = root->ToString(NO_NAMESPACES);
delete root;
//printf("[[%s]]\n", result.c_str());
return result == EXPECTED_EXACT;
}
static int
ParseString_PRETTY_test()
{
string text = read_file("testdata/xml.xml");
XMLNode* root = NodeHandler::ParseString("testdata/xml.xml", text, XMLNode::PRETTY);
if (root == NULL) {
return 1;
}
string result = root->ToString(NO_NAMESPACES);
delete root;
//printf("[[%s]]\n", result.c_str());
return result == EXPECTED_PRETTY;
}
int
XMLHandler_test()
{
int err = 0;
bool all = true;
if (all) err |= ParseFile_EXACT_test();
if (all) err |= ParseFile_PRETTY_test();
if (all) err |= ParseString_EXACT_test();
if (all) err |= ParseString_PRETTY_test();
return err;
}

19
tools/localize/XMLNode.h
View File

@@ -1,19 +0,0 @@
#ifndef XMLNODE_H
#define XMLNODE_H
#include <string>
using namespace std;
struct XMLAttribute
{
string ns;
string name;
string value;
static string Find(const vector<XMLAttribute>& list,
const string& ns, const string& name, const string& def);
};
#endif // XMLNODE_H

181
tools/localize/file_utils.cpp
View File

@@ -1,181 +0,0 @@
#include <string.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include "file_utils.h"
#include "Perforce.h"
#include <utils/String8.h>
#include <sys/fcntl.h>
#include <sys/stat.h>
#include <errno.h>
#include <string.h>
#include <stdlib.h>
#include <cstdio>
#include "log.h"
using namespace android;
using namespace std;
static string
parent_dir(const string& path)
{
return string(String8(path.c_str()).getPathDir().string());
}
static int
mkdirs(const char* last)
{
String8 dest;
const char* s = last-1;
int err;
do {
s++;
if (s > last && (*s == '.' || *s == 0)) {
String8 part(last, s-last);
dest.appendPath(part);
#ifdef HAVE_MS_C_RUNTIME
err = _mkdir(dest.string());
#else
err = mkdir(dest.string(), S_IRUSR|S_IWUSR|S_IXUSR|S_IRGRP|S_IXGRP);
#endif
if (err != 0) {
return err;
}
last = s+1;
}
} while (*s);
return 0;
}
string
translated_file_name(const string& file, const string& locale)
{
const char* str = file.c_str();
const char* p = str + file.length();
const char* rest = NULL;
const char* values = p;
while (p > str) {
p--;
if (*p == '/') {
rest = values;
values = p;
if (0 == strncmp("values", values+1, rest-values-1)) {
break;
}
}
}
values++;
string result(str, values-str);
result.append(values, rest-values);
string language, region;
if (locale == "") {
language = "";
region = "";
}
else if (!split_locale(locale, &language, &region)) {
return "";
}
if (language != "") {
result += '-';
result += language;
}
if (region != "") {
result += "-r";
result += region;
}
result += rest;
return result;
}
ValuesFile*
get_values_file(const string& filename, const Configuration& configuration,
int version, const string& versionString, bool printOnFailure)
{
int err;
string text;
log_printf("get_values_file filename=%s\n", filename.c_str());
err = Perforce::GetFile(filename, versionString, &text, printOnFailure);
if (err != 0 || text == "") {
return NULL;
}
ValuesFile* result = ValuesFile::ParseString(filename, text, configuration, version,
versionString);
if (result == NULL) {
fprintf(stderr, "unable to parse file: %s\n", filename.c_str());
exit(1);
}
return result;
}
ValuesFile*
get_local_values_file(const string& filename, const Configuration& configuration,
int version, const string& versionString, bool printOnFailure)
{
int err;
string text;
char buf[2049];
int fd;
ssize_t amt;
fd = open(filename.c_str(), O_RDONLY);
if (fd == -1) {
fprintf(stderr, "unable to open file: %s\n", filename.c_str());
return NULL;
}
while ((amt = read(fd, buf, sizeof(buf)-1)) > 0) {
text.append(buf, amt);
}
close(fd);
if (text == "") {
return NULL;
}
ValuesFile* result = ValuesFile::ParseString(filename, text, configuration, version,
versionString);
if (result == NULL) {
fprintf(stderr, "unable to parse file: %s\n", filename.c_str());
exit(1);
}
return result;
}
void
print_file_status(size_t j, size_t J, const string& message)
{
printf("\r%s file %zd of %zd...", message.c_str(), j, J);
fflush(stdout);
}
int
write_to_file(const string& filename, const string& text)
{
mkdirs(parent_dir(filename).c_str());
int fd = open(filename.c_str(), O_RDWR | O_CREAT | O_TRUNC, 0666);
if (fd < 0) {
fprintf(stderr, "unable to open file for write (%s): %s\n", strerror(errno),
filename.c_str());
return -1;
}
ssize_t amt = write(fd, text.c_str(), text.length());
close(fd);
if (amt < 0) {
return amt;
}
return amt == (ssize_t)text.length() ? 0 : -1;
}

22
tools/localize/file_utils.h
View File

@@ -1,22 +0,0 @@
#ifndef FILE_UTILS_H
#define FILE_UTILS_H
#include "ValuesFile.h"
#include "Configuration.h"
#include <string>
#include <cstdio>
using namespace std;
string translated_file_name(const string& file, const string& locale);
ValuesFile* get_values_file(const string& filename, const Configuration& configuration,
int version, const string& versionString, bool printOnFailure);
ValuesFile* get_local_values_file(const string& filename, const Configuration& configuration,
int version, const string& versionString, bool printOnFailure);
void print_file_status(size_t j, size_t J, const string& message = "Reading");
int write_to_file(const string& filename, const string& text);
#endif // FILE_UTILS_H

768
tools/localize/localize.cpp
View File

@@ -1,768 +0,0 @@
#include "SourcePos.h"
#include "ValuesFile.h"
#include "XLIFFFile.h"
#include "Perforce.h"
#include "merge_res_and_xliff.h"
#include "localize.h"
#include "file_utils.h"
#include "res_check.h"
#include "xmb.h"
#include <host/pseudolocalize.h>
#include <stdlib.h>
#include <stdarg.h>
#include <sstream>
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
using namespace std;
FILE* g_logFile = NULL;
int test();
int
read_settings(const string& filename, map<string,Settings>* result, const string& rootDir)
{
XMLNode* root = NodeHandler::ParseFile(filename, XMLNode::PRETTY);
if (root == NULL) {
SourcePos(filename, -1).Error("Error reading file.");
return 1;
}
// <configuration>
vector<XMLNode*> configNodes = root->GetElementsByName("", "configuration");
const size_t I = configNodes.size();
for (size_t i=0; i<I; i++) {
const XMLNode* configNode = configNodes[i];
Settings settings;
settings.id = configNode->GetAttribute("", "id", "");
if (settings.id == "") {
configNode->Position().Error("<configuration> needs an id attribute.");
delete root;
return 1;
}
settings.oldVersion = configNode->GetAttribute("", "old-cl", "");
settings.currentVersion = configNode->GetAttribute("", "new-cl", "");
if (settings.currentVersion == "") {
configNode->Position().Error("<configuration> needs a new-cl attribute.");
delete root;
return 1;
}
// <app>
vector<XMLNode*> appNodes = configNode->GetElementsByName("", "app");
const size_t J = appNodes.size();
for (size_t j=0; j<J; j++) {
const XMLNode* appNode = appNodes[j];
string dir = appNode->GetAttribute("", "dir", "");
if (dir == "") {
appNode->Position().Error("<app> needs a dir attribute.");
delete root;
return 1;
}
settings.apps.push_back(dir);
}
// <reject>
vector<XMLNode*> rejectNodes = configNode->GetElementsByName("", "reject");
const size_t K = rejectNodes.size();
for (size_t k=0; k<K; k++) {
const XMLNode* rejectNode = rejectNodes[k];
Reject reject;
reject.file = rejectNode->GetAttribute("", "file", "");
if (reject.file == "") {
rejectNode->Position().Error("<reject> needs a file attribute.");
delete root;
return 1;
}
string f = reject.file;
reject.file = rootDir;
reject.file += '/';
reject.file += f;
reject.name = rejectNode->GetAttribute("", "name", "");
if (reject.name == "") {
rejectNode->Position().Error("<reject> needs a name attribute.");
delete root;
return 1;
}
reject.comment = trim_string(rejectNode->CollapseTextContents());
settings.reject.push_back(reject);
}
(*result)[settings.id] = settings;
}
delete root;
return 0;
}
static void
ValuesFile_to_XLIFFFile(const ValuesFile* values, XLIFFFile* xliff, const string& englishFilename)
{
const set<StringResource>& strings = values->GetStrings();
for (set<StringResource>::const_iterator it=strings.begin(); it!=strings.end(); it++) {
StringResource res = *it;
res.file = englishFilename;
xliff->AddStringResource(res);
}
}
static bool
contains_reject(const Settings& settings, const string& file, const TransUnit& tu)
{
const string name = tu.id;
const vector<Reject>& reject = settings.reject;
const size_t I = reject.size();
for (size_t i=0; i<I; i++) {
const Reject& r = reject[i];
if (r.file == file && r.name == name) {
return true;
}
}
return false;
}
/**
* If it's been rejected, then we keep whatever info we have.
*
* Implements this truth table:
*
* S AT AS Keep
* -----------------------
* 0 0 0 0 (this case can't happen)
* 0 0 1 0 (it was there, never translated, and removed)
* 0 1 0 0 (somehow it got translated, but it was removed)
* 0 1 1 0 (it was removed after having been translated)
*
* 1 0 0 1 (it was just added)
* 1 0 1 1 (it was added, has been changed, but it never got translated)
* 1 1 0 1 (somehow it got translated, but we don't know based on what)
* 1 1 1 0/1 (it's in both. 0 if S=AS b/c there's no need to retranslate if they're
* the same. 1 if S!=AS because S changed, so it should be retranslated)
*
* The first four are cases where, whatever happened in the past, the string isn't there
* now, so it shouldn't be in the XLIFF file.
*
* For cases 4 and 5, the string has never been translated, so get it translated.
*
* For case 6, it's unclear where the translated version came from, so we're conservative
* and send it back for them to have another shot at.
*
* For case 7, we have some data. We have two choices. We could rely on the translator's
* translation memory or tools to notice that the strings haven't changed, and populate the
* <target> field themselves. Or if the string hasn't changed since last time, we can just
* not even tell them about it. As the project nears the end, it will be convenient to see
* the xliff files reducing in size, so we pick the latter. Obviously, if the string has
* changed, then we need to get it retranslated.
*/
bool
keep_this_trans_unit(const string& file, const TransUnit& unit, void* cookie)
{
const Settings* settings = reinterpret_cast<const Settings*>(cookie);
if (contains_reject(*settings, file, unit)) {
return true;
}
if (unit.source.id == "") {
return false;
}
if (unit.altTarget.id == "" || unit.altSource.id == "") {
return true;
}
return unit.source.value->ContentsToString(XLIFF_NAMESPACES)
!= unit.altSource.value->ContentsToString(XLIFF_NAMESPACES);
}
int
validate_config(const string& settingsFile, const map<string,Settings>& settings,
const string& config)
{
if (settings.find(config) == settings.end()) {
SourcePos(settingsFile, -1).Error("settings file does not contain setting: %s\n",
config.c_str());
return 1;
}
return 0;
}
int
validate_configs(const string& settingsFile, const map<string,Settings>& settings,
const vector<string>& configs)
{
int err = 0;
for (size_t i=0; i<configs.size(); i++) {
string config = configs[i];
err |= validate_config(settingsFile, settings, config);
}
return err;
}
int
select_files(vector<string> *resFiles, const string& config,
const map<string,Settings>& settings, const string& rootDir)
{
int err;
vector<vector<string> > allResFiles;
vector<string> configs;
configs.push_back(config);
err = select_files(&allResFiles, configs, settings, rootDir);
if (err == 0) {
*resFiles = allResFiles[0];
}
return err;
}
int
select_files(vector<vector<string> > *allResFiles, const vector<string>& configs,
const map<string,Settings>& settings, const string& rootDir)
{
int err;
printf("Selecting files...");
fflush(stdout);
for (size_t i=0; i<configs.size(); i++) {
const string& config = configs[i];
const Settings& setting = settings.find(config)->second;
vector<string> resFiles;
err = Perforce::GetResourceFileNames(setting.currentVersion, rootDir,
setting.apps, &resFiles, true);
if (err != 0) {
fprintf(stderr, "error with perforce. bailing\n");
return err;
}
allResFiles->push_back(resFiles);
}
return 0;
}
static int
do_export(const string& settingsFile, const string& rootDir, const string& outDir,
const string& targetLocale, const vector<string>& configs)
{
bool success = true;
int err;
if (false) {
printf("settingsFile=%s\n", settingsFile.c_str());
printf("rootDir=%s\n", rootDir.c_str());
printf("outDir=%s\n", outDir.c_str());
for (size_t i=0; i<configs.size(); i++) {
printf("config[%zd]=%s\n", i, configs[i].c_str());
}
}
map<string,Settings> settings;
err = read_settings(settingsFile, &settings, rootDir);
if (err != 0) {
return err;
}
err = validate_configs(settingsFile, settings, configs);
if (err != 0) {
return err;
}
vector<vector<string> > allResFiles;
err = select_files(&allResFiles, configs, settings, rootDir);
if (err != 0) {
return err;
}
size_t totalFileCount = 0;
for (size_t i=0; i<allResFiles.size(); i++) {
totalFileCount += allResFiles[i].size();
}
totalFileCount *= 3; // we try all 3 versions of the file
size_t fileProgress = 0;
vector<Stats> stats;
vector<pair<string,XLIFFFile*> > xliffs;
for (size_t i=0; i<configs.size(); i++) {
const string& config = configs[i];
const Settings& setting = settings[config];
if (false) {
fprintf(stderr, "Configuration: %s (%zd of %zd)\n", config.c_str(), i+1,
configs.size());
fprintf(stderr, " Old CL: %s\n", setting.oldVersion.c_str());
fprintf(stderr, " Current CL: %s\n", setting.currentVersion.c_str());
}
Configuration english;
english.locale = "en_US";
Configuration translated;
translated.locale = targetLocale;
XLIFFFile* xliff = XLIFFFile::Create(english, translated, setting.currentVersion);
const vector<string>& resFiles = allResFiles[i];
const size_t J = resFiles.size();
for (size_t j=0; j<J; j++) {
string resFile = resFiles[j];
// parse the files into a ValuesFile
// pull out the strings and add them to the XLIFFFile
// current file
print_file_status(++fileProgress, totalFileCount);
ValuesFile* currentFile = get_values_file(resFile, english, CURRENT_VERSION,
setting.currentVersion, true);
if (currentFile != NULL) {
ValuesFile_to_XLIFFFile(currentFile, xliff, resFile);
//printf("currentFile=[%s]\n", currentFile->ToString().c_str());
} else {
fprintf(stderr, "error reading file %s@%s\n", resFile.c_str(),
setting.currentVersion.c_str());
success = false;
}
// old file
print_file_status(++fileProgress, totalFileCount);
ValuesFile* oldFile = get_values_file(resFile, english, OLD_VERSION,
setting.oldVersion, false);
if (oldFile != NULL) {
ValuesFile_to_XLIFFFile(oldFile, xliff, resFile);
//printf("oldFile=[%s]\n", oldFile->ToString().c_str());
}
// translated version
// (get the head of the tree for the most recent translation, but it's considered
// the old one because the "current" one hasn't been made yet, and this goes into
// the <alt-trans> tag if necessary
print_file_status(++fileProgress, totalFileCount);
string transFilename = translated_file_name(resFile, targetLocale);
ValuesFile* transFile = get_values_file(transFilename, translated, OLD_VERSION,
setting.currentVersion, false);
if (transFile != NULL) {
ValuesFile_to_XLIFFFile(transFile, xliff, resFile);
}
delete currentFile;
delete oldFile;
delete transFile;
}
Stats beforeFilterStats = xliff->GetStats(config);
// run through the XLIFFFile and strip out TransUnits that have identical
// old and current source values and are not in the reject list, or just
// old values and no source values
xliff->Filter(keep_this_trans_unit, (void*)&setting);
Stats afterFilterStats = xliff->GetStats(config);
afterFilterStats.totalStrings = beforeFilterStats.totalStrings;
// add the reject comments
for (vector<Reject>::const_iterator reject = setting.reject.begin();
reject != setting.reject.end(); reject++) {
TransUnit* tu = xliff->EditTransUnit(reject->file, reject->name);
tu->rejectComment = reject->comment;
}
// config-locale-current_cl.xliff
stringstream filename;
if (outDir != "") {
filename << outDir << '/';
}
filename << config << '-' << targetLocale << '-' << setting.currentVersion << ".xliff";
xliffs.push_back(pair<string,XLIFFFile*>(filename.str(), xliff));
stats.push_back(afterFilterStats);
}
// today is a good day to die
if (!success || SourcePos::HasErrors()) {
return 1;
}
// write the XLIFF files
printf("\nWriting %zd file%s...\n", xliffs.size(), xliffs.size() == 1 ? "" : "s");
for (vector<pair<string,XLIFFFile*> >::iterator it = xliffs.begin(); it != xliffs.end(); it++) {
const string& filename = it->first;
XLIFFFile* xliff = it->second;
string text = xliff->ToString();
write_to_file(filename, text);
}
// the stats
printf("\n"
" to without total\n"
" config files translate comments strings\n"
"-----------------------------------------------------------------------\n");
Stats totals;
totals.config = "total";
totals.files = 0;
totals.toBeTranslated = 0;
totals.noComments = 0;
totals.totalStrings = 0;
for (vector<Stats>::iterator it=stats.begin(); it!=stats.end(); it++) {
string cfg = it->config;
if (cfg.length() > 20) {
cfg.resize(20);
}
printf(" %-20s %-9zd %-9zd %-9zd %-19zd\n", cfg.c_str(), it->files,
it->toBeTranslated, it->noComments, it->totalStrings);
totals.files += it->files;
totals.toBeTranslated += it->toBeTranslated;
totals.noComments += it->noComments;
totals.totalStrings += it->totalStrings;
}
if (stats.size() > 1) {
printf("-----------------------------------------------------------------------\n"
" %-20s %-9zd %-9zd %-9zd %-19zd\n", totals.config.c_str(), totals.files,
totals.toBeTranslated, totals.noComments, totals.totalStrings);
}
printf("\n");
return 0;
}
struct PseudolocalizeSettings {
XLIFFFile* xliff;
bool expand;
};
string
pseudolocalize_string(const string& source, const PseudolocalizeSettings* settings)
{
return pseudolocalize_string(source);
}
static XMLNode*
pseudolocalize_xml_node(const XMLNode* source, const PseudolocalizeSettings* settings)
{
if (source->Type() == XMLNode::TEXT) {
return XMLNode::NewText(source->Position(), pseudolocalize_string(source->Text(), settings),
source->Pretty());
} else {
XMLNode* target;
if (source->Namespace() == XLIFF_XMLNS && source->Name() == "g") {
// XXX don't translate these
target = XMLNode::NewElement(source->Position(), source->Namespace(),
source->Name(), source->Attributes(), source->Pretty());
} else {
target = XMLNode::NewElement(source->Position(), source->Namespace(),
source->Name(), source->Attributes(), source->Pretty());
}
const vector<XMLNode*>& children = source->Children();
const size_t I = children.size();
for (size_t i=0; i<I; i++) {
target->EditChildren().push_back(pseudolocalize_xml_node(children[i], settings));
}
return target;
}
}
void
pseudolocalize_trans_unit(const string&file, TransUnit* unit, void* cookie)
{
const PseudolocalizeSettings* settings = (PseudolocalizeSettings*)cookie;
const StringResource& source = unit->source;
StringResource* target = &unit->target;
*target = source;
target->config = settings->xliff->TargetConfig();
delete target->value;
target->value = pseudolocalize_xml_node(source.value, settings);
}
int
pseudolocalize_xliff(XLIFFFile* xliff, bool expand)
{
PseudolocalizeSettings settings;
settings.xliff = xliff;
settings.expand = expand;
xliff->Map(pseudolocalize_trans_unit, &settings);
return 0;
}
static int
do_pseudo(const string& infile, const string& outfile, bool expand)
{
int err;
XLIFFFile* xliff = XLIFFFile::Parse(infile);
if (xliff == NULL) {
return 1;
}
pseudolocalize_xliff(xliff, expand);
err = write_to_file(outfile, xliff->ToString());
delete xliff;
return err;
}
void
log_printf(const char *fmt, ...)
{
int ret;
va_list ap;
if (g_logFile != NULL) {
va_start(ap, fmt);
ret = vfprintf(g_logFile, fmt, ap);
va_end(ap);
fflush(g_logFile);
}
}
void
close_log_file()
{
if (g_logFile != NULL) {
fclose(g_logFile);
}
}
void
open_log_file(const char* file)
{
g_logFile = fopen(file, "w");
printf("log file: %s -- %p\n", file, g_logFile);
atexit(close_log_file);
}
static int
usage()
{
fprintf(stderr,
"usage: localize export OPTIONS CONFIGS...\n"
" REQUIRED OPTIONS\n"
" --settings SETTINGS The settings file to use. See CONFIGS below.\n"
" --root TREE_ROOT The location in Perforce of the files. e.g. //device\n"
" --target LOCALE The target locale. See LOCALES below.\n"
"\n"
" OPTIONAL OPTIONS\n"
" --out DIR Directory to put the output files. Defaults to the\n"
" current directory if not supplied. Files are\n"
" named as follows:\n"
" CONFIG-LOCALE-CURRENT_CL.xliff\n"
"\n"
"\n"
"usage: localize import XLIFF_FILE...\n"
"\n"
"Import a translated XLIFF file back into the tree.\n"
"\n"
"\n"
"usage: localize xlb XMB_FILE VALUES_FILES...\n"
"\n"
"Read resource files from the tree file and write the corresponding XLB file\n"
"\n"
"Supply all of the android resource files (values files) to export after that.\n"
"\n"
"\n"
"\n"
"CONFIGS\n"
"\n"
"LOCALES\n"
"Locales are specified in the form en_US They will be processed correctly\n"
"to locate the resouce files in the tree.\n"
"\n"
"\n"
"usage: localize pseudo OPTIONS INFILE [OUTFILE]\n"
" OPTIONAL OPTIONS\n"
" --big Pad strings so they get longer.\n"
"\n"
"Read INFILE, an XLIFF file, and output a pseudotranslated version of that file. If\n"
"OUTFILE is specified, the results are written there; otherwise, the results are\n"
"written back to INFILE.\n"
"\n"
"\n"
"usage: localize rescheck FILES...\n"
"\n"
"Reads the base strings and prints warnings about bad resources from the given files.\n"
"\n");
return 1;
}
int
main(int argc, const char** argv)
{
//open_log_file("log.txt");
//g_logFile = stdout;
if (argc == 2 && 0 == strcmp(argv[1], "--test")) {
return test();
}
if (argc < 2) {
return usage();
}
int index = 1;
if (0 == strcmp("export", argv[index])) {
string settingsFile;
string rootDir;
string outDir;
string baseLocale = "en";
string targetLocale;
string language, region;
vector<string> configs;
index++;
while (index < argc) {
if (0 == strcmp("--settings", argv[index])) {
settingsFile = argv[index+1];
index += 2;
}
else if (0 == strcmp("--root", argv[index])) {
rootDir = argv[index+1];
index += 2;
}
else if (0 == strcmp("--out", argv[index])) {
outDir = argv[index+1];
index += 2;
}
else if (0 == strcmp("--target", argv[index])) {
targetLocale = argv[index+1];
index += 2;
}
else if (argv[index][0] == '-') {
fprintf(stderr, "unknown argument %s\n", argv[index]);
return usage();
}
else {
break;
}
}
for (; index<argc; index++) {
configs.push_back(argv[index]);
}
if (settingsFile == "" || rootDir == "" || configs.size() == 0 || targetLocale == "") {
return usage();
}
if (!split_locale(targetLocale, &language, &region)) {
fprintf(stderr, "illegal --target locale: '%s'\n", targetLocale.c_str());
return usage();
}
return do_export(settingsFile, rootDir, outDir, targetLocale, configs);
}
else if (0 == strcmp("import", argv[index])) {
vector<string> xliffFilenames;
index++;
for (; index<argc; index++) {
xliffFilenames.push_back(argv[index]);
}
return do_merge(xliffFilenames);
}
else if (0 == strcmp("xlb", argv[index])) {
string outfile;
vector<string> resFiles;
index++;
if (argc < index+1) {
return usage();
}
outfile = argv[index];
index++;
for (; index<argc; index++) {
resFiles.push_back(argv[index]);
}
return do_xlb_export(outfile, resFiles);
}
else if (0 == strcmp("pseudo", argv[index])) {
string infile;
string outfile;
bool big = false;
index++;
while (index < argc) {
if (0 == strcmp("--big", argv[index])) {
big = true;
index += 1;
}
else if (argv[index][0] == '-') {
fprintf(stderr, "unknown argument %s\n", argv[index]);
return usage();
}
else {
break;
}
}
if (index == argc-1) {
infile = argv[index];
outfile = argv[index];
}
else if (index == argc-2) {
infile = argv[index];
outfile = argv[index+1];
}
else {
fprintf(stderr, "unknown argument %s\n", argv[index]);
return usage();
}
return do_pseudo(infile, outfile, big);
}
else if (0 == strcmp("rescheck", argv[index])) {
vector<string> files;
index++;
while (index < argc) {
if (argv[index][0] == '-') {
fprintf(stderr, "unknown argument %s\n", argv[index]);
return usage();
}
else {
break;
}
}
for (; index<argc; index++) {
files.push_back(argv[index]);
}
if (files.size() == 0) {
return usage();
}
return do_rescheck(files);
}
else {
return usage();
}
if (SourcePos::HasErrors()) {
SourcePos::PrintErrors(stderr);
return 1;
}
return 0;
}

40
tools/localize/localize.h
View File

@@ -1,40 +0,0 @@
#ifndef LOCALIZE_H
#define LOCALIZE_H
#include "XLIFFFile.h"
#include <map>
#include <string>
using namespace std;
struct Reject
{
string file;
string name;
string comment;
};
struct Settings
{
string id;
string oldVersion;
string currentVersion;
vector<string> apps;
vector<Reject> reject;
};
int read_settings(const string& filename, map<string,Settings>* result, const string& rootDir);
string translated_file_name(const string& file, const string& locale);
bool keep_this_trans_unit(const string& file, const TransUnit& unit, void* cookie);
int validate_config(const string& settingsFile, const map<string,Settings>& settings,
const string& configs);
int validate_configs(const string& settingsFile, const map<string,Settings>& settings,
const vector<string>& configs);
int select_files(vector<string> *resFiles, const string& config,
const map<string,Settings>& settings, const string& rootDir);
int select_files(vector<vector<string> > *allResFiles, const vector<string>& configs,
const map<string,Settings>& settings, const string& rootDir);
#endif // LOCALIZE_H

221
tools/localize/localize_test.cpp
View File

@@ -1,221 +0,0 @@
#include <cstdio>
#include "XLIFFFile.h"
#include "ValuesFile.h"
#include "localize.h"
#include <stdio.h>
int pseudolocalize_xliff(XLIFFFile* xliff, bool expand);
static int
test_filename(const string& file, const string& locale, const string& expected)
{
string result = translated_file_name(file, locale);
if (result != expected) {
fprintf(stderr, "translated_file_name test failed\n");
fprintf(stderr, " locale='%s'\n", locale.c_str());
fprintf(stderr, " expected='%s'\n", expected.c_str());
fprintf(stderr, " result='%s'\n", result.c_str());
return 1;
} else {
if (false) {
fprintf(stderr, "translated_file_name test passed\n");
fprintf(stderr, " locale='%s'\n", locale.c_str());
fprintf(stderr, " expected='%s'\n", expected.c_str());
fprintf(stderr, " result='%s'\n", result.c_str());
}
return 0;
}
}
static int
translated_file_name_test()
{
bool all = true;
int err = 0;
if (all) err |= test_filename("//device/samples/NotePad/res/values/strings.xml", "zz_ZZ",
"//device/samples/NotePad/res/values-zz-rZZ/strings.xml");
if (all) err |= test_filename("//device/samples/NotePad/res/values/strings.xml", "zz",
"//device/samples/NotePad/res/values-zz/strings.xml");
if (all) err |= test_filename("//device/samples/NotePad/res/values/strings.xml", "",
"//device/samples/NotePad/res/values/strings.xml");
return err;
}
bool
return_false(const string&, const TransUnit& unit, void* cookie)
{
return false;
}
static int
delete_trans_units()
{
XLIFFFile* xliff = XLIFFFile::Parse("testdata/strip_xliff.xliff");
if (xliff == NULL) {
printf("couldn't read file\n");
return 1;
}
if (false) {
printf("XLIFF was [[%s]]\n", xliff->ToString().c_str());
}
xliff->Filter(return_false, NULL);
if (false) {
printf("XLIFF is [[%s]]\n", xliff->ToString().c_str());
set<StringResource> const& strings = xliff->GetStringResources();
printf("strings.size=%zd\n", strings.size());
for (set<StringResource>::iterator it=strings.begin(); it!=strings.end(); it++) {
const StringResource& str = *it;
printf("STRING!!! id=%s value='%s' pos=%s file=%s version=%d(%s)\n", str.id.c_str(),
str.value->ContentsToString(ANDROID_NAMESPACES).c_str(),
str.pos.ToString().c_str(), str.file.c_str(), str.version,
str.versionString.c_str());
}
}
return 0;
}
static int
filter_trans_units()
{
XLIFFFile* xliff = XLIFFFile::Parse("testdata/strip_xliff.xliff");
if (xliff == NULL) {
printf("couldn't read file\n");
return 1;
}
if (false) {
printf("XLIFF was [[%s]]\n", xliff->ToString().c_str());
}
Settings setting;
xliff->Filter(keep_this_trans_unit, &setting);
if (false) {
printf("XLIFF is [[%s]]\n", xliff->ToString().c_str());
set<StringResource> const& strings = xliff->GetStringResources();
printf("strings.size=%zd\n", strings.size());
for (set<StringResource>::iterator it=strings.begin(); it!=strings.end(); it++) {
const StringResource& str = *it;
printf("STRING!!! id=%s value='%s' pos=%s file=%s version=%d(%s)\n", str.id.c_str(),
str.value->ContentsToString(ANDROID_NAMESPACES).c_str(),
str.pos.ToString().c_str(), str.file.c_str(), str.version,
str.versionString.c_str());
}
}
return 0;
}
static int
settings_test()
{
int err;
map<string,Settings> settings;
map<string,Settings>::iterator it;
err = read_settings("testdata/config.xml", &settings, "//asdf");
if (err != 0) {
return err;
}
if (false) {
for (it=settings.begin(); it!=settings.end(); it++) {
const Settings& setting = it->second;
printf("CONFIG:\n");
printf(" id='%s'\n", setting.id.c_str());
printf(" oldVersion='%s'\n", setting.oldVersion.c_str());
printf(" currentVersion='%s'\n", setting.currentVersion.c_str());
int i=0;
for (vector<string>::const_iterator app=setting.apps.begin();
app!=setting.apps.end(); app++) {
printf(" apps[%02d]='%s'\n", i, app->c_str());
i++;
}
i=0;
for (vector<Reject>::const_iterator reject=setting.reject.begin();
reject!=setting.reject.end(); reject++) {
i++;
printf(" reject[%02d]=('%s','%s','%s')\n", i, reject->file.c_str(),
reject->name.c_str(), reject->comment.c_str());
}
}
}
for (it=settings.begin(); it!=settings.end(); it++) {
const Settings& setting = it->second;
if (it->first != setting.id) {
fprintf(stderr, "it->first='%s' setting.id='%s'\n", it->first.c_str(),
setting.id.c_str());
err |= 1;
}
}
return err;
}
static int
test_one_pseudo(bool big, const char* expected)
{
XLIFFFile* xliff = XLIFFFile::Parse("testdata/pseudo.xliff");
if (xliff == NULL) {
printf("couldn't read file\n");
return 1;
}
if (false) {
printf("XLIFF was [[%s]]\n", xliff->ToString().c_str());
}
pseudolocalize_xliff(xliff, big);
string newString = xliff->ToString();
delete xliff;
if (false) {
printf("XLIFF is [[%s]]\n", newString.c_str());
}
if (false && newString != expected) {
fprintf(stderr, "xliff didn't translate as expected\n");
fprintf(stderr, "newString=[[%s]]\n", newString.c_str());
fprintf(stderr, "expected=[[%s]]\n", expected);
return 1;
}
return 0;
}
static int
pseudolocalize_test()
{
int err = 0;
err |= test_one_pseudo(false, "");
//err |= test_one_pseudo(true, "");
return err;
}
int
localize_test()
{
bool all = true;
int err = 0;
if (all) err |= translated_file_name_test();
if (all) err |= delete_trans_units();
if (all) err |= filter_trans_units();
if (all) err |= settings_test();
if (all) err |= pseudolocalize_test();
return err;
}

7
tools/localize/log.h
View File

@@ -1,7 +0,0 @@
#ifndef LOG_H
#define LOG_H
void log_printf(const char* fmt, ...);
#endif // LOG_H

392
tools/localize/merge_res_and_xliff.cpp
View File

@@ -1,392 +0,0 @@
#include "merge_res_and_xliff.h"
#include "file_utils.h"
#include "Perforce.h"
#include "log.h"
#include <stdio.h>
static set<StringResource>::const_iterator
find_id(const set<StringResource>& s, const string& id, int index)
{
for (set<StringResource>::const_iterator it = s.begin(); it != s.end(); it++) {
if (it->id == id && it->index == index) {
return it;
}
}
return s.end();
}
static set<StringResource>::const_iterator
find_in_xliff(const set<StringResource>& s, const string& filename, const string& id, int index,
int version, const Configuration& config)
{
for (set<StringResource>::const_iterator it = s.begin(); it != s.end(); it++) {
if (it->file == filename && it->id == id && it->index == index && it->version == version
&& it->config == config) {
return it;
}
}
return s.end();
}
static void
printit(const set<StringResource>& s, const set<StringResource>::const_iterator& it)
{
if (it == s.end()) {
printf("(none)\n");
} else {
printf("id=%s index=%d config=%s file=%s value='%s'\n", it->id.c_str(), it->index,
it->config.ToString().c_str(), it->file.c_str(),
it->value->ToString(ANDROID_NAMESPACES).c_str());
}
}
StringResource
convert_resource(const StringResource& s, const string& file, const Configuration& config,
int version, const string& versionString)
{
return StringResource(s.pos, file, config, s.id, s.index, s.value ? s.value->Clone() : NULL,
version, versionString, s.comment);
}
static bool
resource_has_contents(const StringResource& res)
{
XMLNode* value = res.value;
if (value == NULL) {
return false;
}
string contents = value->ContentsToString(ANDROID_NAMESPACES);
return contents != "";
}
ValuesFile*
merge_res_and_xliff(const ValuesFile* en_currentFile,
const ValuesFile* xx_currentFile, const ValuesFile* xx_oldFile,
const string& filename, const XLIFFFile* xliffFile)
{
bool success = true;
Configuration en_config = xliffFile->SourceConfig();
Configuration xx_config = xliffFile->TargetConfig();
string currentVersion = xliffFile->CurrentVersion();
ValuesFile* result = new ValuesFile(xx_config);
set<StringResource> en_cur = en_currentFile->GetStrings();
set<StringResource> xx_cur = xx_currentFile->GetStrings();
set<StringResource> xx_old = xx_oldFile->GetStrings();
set<StringResource> xliff = xliffFile->GetStringResources();
// for each string in en_current
for (set<StringResource>::const_iterator en_c = en_cur.begin();
en_c != en_cur.end(); en_c++) {
set<StringResource>::const_iterator xx_c = find_id(xx_cur, en_c->id, en_c->index);
set<StringResource>::const_iterator xx_o = find_id(xx_old, en_c->id, en_c->index);
set<StringResource>::const_iterator xlf = find_in_xliff(xliff, en_c->file, en_c->id,
en_c->index, CURRENT_VERSION, xx_config);
if (false) {
printf("\nen_c: "); printit(en_cur, en_c);
printf("xx_c: "); printit(xx_cur, xx_c);
printf("xx_o: "); printit(xx_old, xx_o);
printf("xlf: "); printit(xliff, xlf);
}
// if it changed between xx_old and xx_current, use xx_current
// (someone changed it by hand)
if (xx_o != xx_old.end() && xx_c != xx_cur.end()) {
string xx_o_value = xx_o->value->ToString(ANDROID_NAMESPACES);
string xx_c_value = xx_c->value->ToString(ANDROID_NAMESPACES);
if (xx_o_value != xx_c_value && xx_c_value != "") {
StringResource r(convert_resource(*xx_c, filename, xx_config,
CURRENT_VERSION, currentVersion));
if (resource_has_contents(r)) {
result->AddString(r);
}
continue;
}
}
// if it is present in xliff, use that
// (it just got translated)
if (xlf != xliff.end() && xlf->value->ToString(ANDROID_NAMESPACES) != "") {
StringResource r(convert_resource(*xlf, filename, xx_config,
CURRENT_VERSION, currentVersion));
if (resource_has_contents(r)) {
result->AddString(r);
}
}
// if it is present in xx_current, use that
// (it was already translated, and not retranslated)
// don't filter out empty strings if they were added by hand, the above code just
// guarantees that this tool never adds an empty one.
if (xx_c != xx_cur.end()) {
StringResource r(convert_resource(*xx_c, filename, xx_config,
CURRENT_VERSION, currentVersion));
result->AddString(r);
}
// othwerwise, leave it out. The resource fall-through code will use the English
// one at runtime, and the xliff export code will pick it up for translation next time.
}
if (success) {
return result;
} else {
delete result;
return NULL;
}
}
struct MergedFile {
XLIFFFile* xliff;
string xliffFilename;
string original;
string translated;
ValuesFile* en_current;
ValuesFile* xx_current;
ValuesFile* xx_old;
ValuesFile* xx_new;
string xx_new_text;
string xx_new_filename;
bool new_file;
bool deleted_file;
MergedFile();
MergedFile(const MergedFile&);
};
struct compare_filenames {
bool operator()(const MergedFile& lhs, const MergedFile& rhs) const
{
return lhs.original < rhs.original;
}
};
MergedFile::MergedFile()
:xliff(NULL),
xliffFilename(),
original(),
translated(),
en_current(NULL),
xx_current(NULL),
xx_old(NULL),
xx_new(NULL),
xx_new_text(),
xx_new_filename(),
new_file(false),
deleted_file(false)
{
}
MergedFile::MergedFile(const MergedFile& that)
:xliff(that.xliff),
xliffFilename(that.xliffFilename),
original(that.original),
translated(that.translated),
en_current(that.en_current),
xx_current(that.xx_current),
xx_old(that.xx_old),
xx_new(that.xx_new),
xx_new_text(that.xx_new_text),
xx_new_filename(that.xx_new_filename),
new_file(that.new_file),
deleted_file(that.deleted_file)
{
}
typedef set<MergedFile, compare_filenames> MergedFileSet;
int
do_merge(const vector<string>& xliffFilenames)
{
int err = 0;
MergedFileSet files;
printf("\rPreparing..."); fflush(stdout);
string currentChange = Perforce::GetCurrentChange(true);
// for each xliff, make a MergedFile record and do a little error checking
for (vector<string>::const_iterator xliffFilename=xliffFilenames.begin();
xliffFilename!=xliffFilenames.end(); xliffFilename++) {
XLIFFFile* xliff = XLIFFFile::Parse(*xliffFilename);
if (xliff == NULL) {
fprintf(stderr, "localize import: unable to read file %s\n", xliffFilename->c_str());
err = 1;
continue;
}
set<string> xf = xliff->Files();
for (set<string>::const_iterator f=xf.begin(); f!=xf.end(); f++) {
MergedFile mf;
mf.xliff = xliff;
mf.xliffFilename = *xliffFilename;
mf.original = *f;
mf.translated = translated_file_name(mf.original, xliff->TargetConfig().locale);
log_printf("mf.translated=%s mf.original=%s locale=%s\n", mf.translated.c_str(),
mf.original.c_str(), xliff->TargetConfig().locale.c_str());
if (files.find(mf) != files.end()) {
fprintf(stderr, "%s: duplicate string resources for file %s\n",
xliffFilename->c_str(), f->c_str());
fprintf(stderr, "%s: previously defined here.\n",
files.find(mf)->xliffFilename.c_str());
err = 1;
continue;
}
files.insert(mf);
}
}
size_t deletedFileCount = 0;
size_t J = files.size() * 3;
size_t j = 1;
// Read all of the files from perforce.
for (MergedFileSet::iterator mf = files.begin(); mf != files.end(); mf++) {
MergedFile* file = const_cast<MergedFile*>(&(*mf));
// file->en_current
print_file_status(j++, J);
file->en_current = get_values_file(file->original, file->xliff->SourceConfig(),
CURRENT_VERSION, currentChange, true);
if (file->en_current == NULL) {
// deleted file
file->deleted_file = true;
deletedFileCount++;
continue;
}
// file->xx_current;
print_file_status(j++, J);
file->xx_current = get_values_file(file->translated, file->xliff->TargetConfig(),
CURRENT_VERSION, currentChange, false);
if (file->xx_current == NULL) {
file->xx_current = new ValuesFile(file->xliff->TargetConfig());
file->new_file = true;
}
// file->xx_old (note that the xliff's current version is our old version, because that
// was the current version when it was exported)
print_file_status(j++, J);
file->xx_old = get_values_file(file->translated, file->xliff->TargetConfig(),
OLD_VERSION, file->xliff->CurrentVersion(), false);
if (file->xx_old == NULL) {
file->xx_old = new ValuesFile(file->xliff->TargetConfig());
file->new_file = true;
}
}
// merge them
for (MergedFileSet::iterator mf = files.begin(); mf != files.end(); mf++) {
MergedFile* file = const_cast<MergedFile*>(&(*mf));
if (file->deleted_file) {
continue;
}
file->xx_new = merge_res_and_xliff(file->en_current, file->xx_current, file->xx_old,
file->original, file->xliff);
}
// now is a good time to stop if there was an error
if (err != 0) {
return err;
}
// locate the files
j = 1;
for (MergedFileSet::iterator mf = files.begin(); mf != files.end(); mf++) {
MergedFile* file = const_cast<MergedFile*>(&(*mf));
print_file_status(j++, J, "Locating");
file->xx_new_filename = Perforce::Where(file->translated, true);
if (file->xx_new_filename == "") {
fprintf(stderr, "\nWas not able to determine the location of depot file %s\n",
file->translated.c_str());
err = 1;
}
}
if (err != 0) {
return err;
}
// p4 edit the files
// only do this if it changed - no need to submit files that haven't changed meaningfully
vector<string> filesToEdit;
vector<string> filesToAdd;
vector<string> filesToDelete;
for (MergedFileSet::iterator mf = files.begin(); mf != files.end(); mf++) {
MergedFile* file = const_cast<MergedFile*>(&(*mf));
if (file->deleted_file) {
filesToDelete.push_back(file->xx_new_filename);
continue;
}
string xx_current_text = file->xx_current->ToString();
string xx_new_text = file->xx_new->ToString();
if (xx_current_text != xx_new_text) {
if (file->xx_new->GetStrings().size() == 0) {
file->deleted_file = true;
filesToDelete.push_back(file->xx_new_filename);
} else {
file->xx_new_text = xx_new_text;
if (file->new_file) {
filesToAdd.push_back(file->xx_new_filename);
} else {
filesToEdit.push_back(file->xx_new_filename);
}
}
}
}
if (filesToAdd.size() == 0 && filesToEdit.size() == 0 && deletedFileCount == 0) {
printf("\nAll of the files are the same. Nothing to change.\n");
return 0;
}
if (filesToEdit.size() > 0) {
printf("\np4 editing files...\n");
if (0 != Perforce::EditFiles(filesToEdit, true)) {
return 1;
}
}
printf("\n");
for (MergedFileSet::iterator mf = files.begin(); mf != files.end(); mf++) {
MergedFile* file = const_cast<MergedFile*>(&(*mf));
if (file->deleted_file) {
continue;
}
if (file->xx_new_text != "" && file->xx_new_filename != "") {
if (0 != write_to_file(file->xx_new_filename, file->xx_new_text)) {
err = 1;
}
}
}
if (err != 0) {
return err;
}
if (filesToAdd.size() > 0) {
printf("p4 adding %zd new files...\n", filesToAdd.size());
err = Perforce::AddFiles(filesToAdd, true);
}
if (filesToDelete.size() > 0) {
printf("p4 deleting %zd removed files...\n", filesToDelete.size());
err = Perforce::DeleteFiles(filesToDelete, true);
}
if (err != 0) {
return err;
}
printf("\n"
"Theoretically, this merge was successfull. Next you should\n"
"review the diffs, get a code review, and submit it. Enjoy.\n\n");
return 0;
}

13
tools/localize/merge_res_and_xliff.h
View File

@@ -1,13 +0,0 @@
#ifndef MERGE_RES_AND_XLIFF_H
#define MERGE_RES_AND_XLIFF_H
#include "ValuesFile.h"
#include "XLIFFFile.h"
ValuesFile* merge_res_and_xliff(const ValuesFile* en_current,
const ValuesFile* xx_current, const ValuesFile* xx_old,
const string& filename, const XLIFFFile* xliff);
int do_merge(const vector<string>& xliffFilenames);
#endif // MERGE_RES_AND_XLIFF_H

48
tools/localize/merge_res_and_xliff_test.cpp
View File

@@ -1,48 +0,0 @@
#include <cstdio>
#include "merge_res_and_xliff.h"
#include <stdio.h>
int
merge_test()
{
Configuration english;
english.locale = "en_US";
Configuration translated;
translated.locale = "zz_ZZ";
ValuesFile* en_current = ValuesFile::ParseFile("testdata/merge_en_current.xml", english,
CURRENT_VERSION, "3");
if (en_current == NULL) {
fprintf(stderr, "merge_test: unable to read testdata/merge_en_current.xml\n");
return 1;
}
ValuesFile* xx_current = ValuesFile::ParseFile("testdata/merge_xx_current.xml", translated,
CURRENT_VERSION, "3");
if (xx_current == NULL) {
fprintf(stderr, "merge_test: unable to read testdata/merge_xx_current.xml\n");
return 1;
}
ValuesFile* xx_old = ValuesFile::ParseFile("testdata/merge_xx_old.xml", translated,
OLD_VERSION, "2");
if (xx_old == NULL) {
fprintf(stderr, "merge_test: unable to read testdata/merge_xx_old.xml\n");
return 1;
}
XLIFFFile* xliff = XLIFFFile::Parse("testdata/merge.xliff");
ValuesFile* result = merge_res_and_xliff(en_current, xx_current, xx_old,
"//device/tools/localize/testdata/res/values/strings.xml", xliff);
if (result == NULL) {
fprintf(stderr, "merge_test: result is NULL\n");
return 1;
}
printf("======= RESULT =======\n%s===============\n", result->ToString().c_str());
return 0;
}

106
tools/localize/res_check.cpp
View File

@@ -1,106 +0,0 @@
#include "res_check.h"
#include "localize.h"
#include "file_utils.h"
#include "ValuesFile.h"
#include <stdio.h>
static int check_file(const ValuesFile* file);
static int check_value(const SourcePos& pos, const XMLNode* value);
static int scan_for_unguarded_format(const SourcePos& pos, const XMLNode* value, int depth = 0);
int
do_rescheck(const vector<string>& files)
{
int err;
Configuration english;
english.locale = "en_US";
for (size_t i=0; i<files.size(); i++) {
const string filename = files[i];
ValuesFile* valuesFile = get_local_values_file(filename, english, CURRENT_VERSION,
"0", true);
if (valuesFile != NULL) {
err |= check_file(valuesFile);
delete valuesFile;
} else {
err |= 1;
}
}
return err;
}
static int
check_file(const ValuesFile* file)
{
int err = 0;
set<StringResource> strings = file->GetStrings();
for (set<StringResource>::iterator it=strings.begin(); it!=strings.end(); it++) {
XMLNode* value = it->value;
if (value != NULL) {
err |= check_value(it->pos, value);
}
}
return err;
}
static bool
contains_percent(const string& str)
{
const size_t len = str.length();
for (size_t i=0; i<len; i++) {
char c = str[i];
if (c == '%') {
return true;
}
}
return false;
}
static int
check_value(const SourcePos& pos, const XMLNode* value)
{
int err = 0;
err |= scan_for_unguarded_format(pos, value);
return err;
}
static bool
is_xliff_block(const string& ns, const string& name)
{
if (ns == XLIFF_XMLNS) {
return name == "g";
} else {
return false;
}
}
static int
scan_for_unguarded_format(const SourcePos& pos, const string& string)
{
bool containsPercent = contains_percent(string);
if (containsPercent) {
pos.Error("unguarded percent: '%s'\n", string.c_str());
}
return 0;
}
static int
scan_for_unguarded_format(const SourcePos& pos, const XMLNode* value, int depth)
{
if (value->Type() == XMLNode::ELEMENT) {
int err = 0;
if (depth == 0 || !is_xliff_block(value->Namespace(), value->Name())) {
const vector<XMLNode*>& children = value->Children();
for (size_t i=0; i<children.size(); i++) {
err |= scan_for_unguarded_format(pos, children[i], depth+1);
}
}
return err;
} else {
return scan_for_unguarded_format(pos, value->Text());
}
}

12
tools/localize/res_check.h
View File

@@ -1,12 +0,0 @@
#ifndef RESCHECK_H
#define RESCHECK_H
#include <map>
#include <string>
#include <vector>
using namespace std;
int do_rescheck(const vector<string>& files);
#endif // RESCHECK_H

31
tools/localize/test.cpp
View File

@@ -1,31 +0,0 @@
#include "SourcePos.h"
#include <stdio.h>
int ValuesFile_test();
int XLIFFFile_test();
int XMLHandler_test();
int Perforce_test();
int localize_test();
int merge_test();
int
test()
{
bool all = true;
int err = 0;
if (all) err |= XMLHandler_test();
if (all) err |= ValuesFile_test();
if (all) err |= XLIFFFile_test();
if (all) err |= Perforce_test();
if (all) err |= localize_test();
if (all) err |= merge_test();
if (err != 0) {
fprintf(stderr, "some tests failed\n");
} else {
fprintf(stderr, "all tests passed\n");
}
return err;
}

15
tools/localize/testdata/config.xml vendored
View File

@@ -1,15 +0,0 @@
<localize-config>
<configuration id="system"
old-cl="1"
new-cl="43019">
<app dir="apps/common" />
</configuration>
<configuration id="samples"
old-cl="24801"
new-cl="43019">
<app dir="samples/NotePad" />
<reject file="samples/NotePad/res/values/strings.xml" name="string:menu_delete">
QA says this sounds <b>rude</b>.
</reject>
</configuration>
</localize-config>

72
tools/localize/testdata/import.xliff vendored
View File

@@ -1,72 +0,0 @@
<?xml version="1.0" encoding="UTF-8" ?>
<xliff xmlns="urn:oasis:names:tc:xliff:document:1.2"
version="1.2"
>
<file datatype="x-android-res"
original="//device/tools/localize/testdata/res/values/strings.xml"
product-version="1.0"
date="08:10:54 12/07/07 PST"
source-language="en_US"
product-name="kila"
target-language="zz_ZZ"
build-num="44391"
>
<body>
<trans-unit id="string:changed_in_xx">
<source>aaa</source>
<target>AAA</target>
</trans-unit>
<trans-unit id="string:first_translation">
<source>bbb</source>
<target>BBB</target>
</trans-unit>
<trans-unit id="string:deleted_string">
<source>ddd</source>
<target>DDDD</target>
</trans-unit>
<trans-unit id="array:0:growing_array">
<source>1-One</source>
<target>1-oNE</target>
</trans-unit>
<trans-unit id="array:1:growing_array">
<source>1-Two</source>
<target>1-tWO</target>
</trans-unit>
<trans-unit id="array:2:growing_array">
<source>1-Three</source>
<target>1-tHREE</target>
</trans-unit>
<trans-unit id="array:0:shrinking_array">
<source>2-One</source>
<target>2-oNE</target>
</trans-unit>
<trans-unit id="array:1:shrinking_array">
<source>2-Two</source>
<target>2-tWO</target>
</trans-unit>
<trans-unit id="array:2:shrinking_array">
<source>2-Three</source>
<target>2-tHREE</target>
</trans-unit>
<trans-unit id="array:3:shrinking_array">
<source>2-Four</source>
<target>2-fOUR</target>
</trans-unit>
<trans-unit id="array:0:deleted_array">
<source>4-One</source>
<target>4-oNE</target>
</trans-unit>
<trans-unit id="array:1:deleted_array">
<source>4-Two</source>
<target>4-tWO</target>
</trans-unit>
<trans-unit id="array:2:deleted_array">
<source>4-Three</source>
<target>4-tHREE</target>
</trans-unit>
</body>
</file>
</xliff>

72
tools/localize/testdata/merge.xliff vendored
View File

@@ -1,72 +0,0 @@
<?xml version="1.0" encoding="UTF-8" ?>
<xliff xmlns="urn:oasis:names:tc:xliff:document:1.2"
version="1.2"
>
<file datatype="x-android-res"
original="testdata/merge_en_current.xml"
product-version="1.0"
date="08:10:54 12/07/07 PST"
source-language="en-US"
product-name="kila"
target-language="zz-ZZ"
build-num="44391"
>
<body>
<trans-unit id="string:changed_in_xx">
<source>aaa</source>
<target>AAA</target>
</trans-unit>
<trans-unit id="string:first_translation">
<source>bbb</source>
<target>BBB</target>
</trans-unit>
<trans-unit id="string:deleted_string">
<source>ddd</source>
<target>DDDD</target>
</trans-unit>
<trans-unit id="array:0:growing_array">
<source>1-One</source>
<target>1-oNE</target>
</trans-unit>
<trans-unit id="array:1:growing_array">
<source>1-Two</source>
<target>1-tWO</target>
</trans-unit>
<trans-unit id="array:2:growing_array">
<source>1-Three</source>
<target>1-tHREE</target>
</trans-unit>
<trans-unit id="array:0:shrinking_array">
<source>2-One</source>
<target>2-oNE</target>
</trans-unit>
<trans-unit id="array:1:shrinking_array">
<source>2-Two</source>
<target>2-tWO</target>
</trans-unit>
<trans-unit id="array:2:shrinking_array">
<source>2-Three</source>
<target>2-tHREE</target>
</trans-unit>
<trans-unit id="array:3:shrinking_array">
<source>2-Four</source>
<target>2-fOUR</target>
</trans-unit>
<trans-unit id="array:0:deleted_array">
<source>4-One</source>
<target>4-oNE</target>
</trans-unit>
<trans-unit id="array:1:deleted_array">
<source>4-Two</source>
<target>4-tWO</target>
</trans-unit>
<trans-unit id="array:2:deleted_array">
<source>4-Three</source>
<target>4-tHREE</target>
</trans-unit>
</body>
</file>
</xliff>

44
tools/localize/testdata/merge_en_current.xml vendored
View File

@@ -1,44 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<!-- Copyright (C) 2007 The Android Open Source Project
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<resources xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2">
<string name="changed_in_xx">aaa</string>
<string name="first_translation">bbb</string>
<string name="previously_translated">ccc</string>
<string name="new_string">ccc</string>
<string name="formatted_string"><b>bold</b><i>italic<u>italic_underline</u></i><u>underline</u></string>
<array name="growing_array">
<!-- somebody wrote a comment! -->
<item>1-One</item>
<item>1-Two</item>
<item>1-Three</item>
<item>1-Four</item>
</array>
<array name="shrinking_array">
<!-- somebody wrote a comment! -->
<item>2-One</item>
<item>2-Two</item>
<item>2-Three</item>
</array>
<array name="new_array">
<!-- somebody wrote a comment! -->
<item>3-One</item>
<item>3-Two</item>
<item>3-Three</item>
</array>
</resources>

45
tools/localize/testdata/merge_en_old.xml vendored
View File

@@ -1,45 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<!-- Copyright (C) 2007 The Android Open Source Project
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<resources xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2">
<string name="changed_in_xx">aaa</string>
<string name="first_translation">bbb</string>
<string name="previously_translated">ccc</string>
<string name="deleted_string">ddd</string>
<string name="formatted_string"><b>bold</b><i>italic<u>italic_underline</u></i><u>underline</u></string>
<array name="growing_array">
<!-- somebody wrote a comment! -->
<item>1-One</item>
<item>1-Two</item>
<item>1-Three</item>
</array>
<array name="shrinking_array">
<!-- somebody wrote a comment! -->
<item>2-One</item>
<item>2-Two</item>
<item>2-Three</item>
<item>2-Four</item>
</array>
<array name="deleted_array">
<!-- somebody wrote a comment! -->
<item>4-One</item>
<item>4-Two</item>
<item>4-Three</item>
</array>
</resources>

22
tools/localize/testdata/merge_xx_current.xml vendored
View File

@@ -1,22 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<!-- Copyright (C) 2007 The Android Open Source Project
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<resources xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2">
<string name="changed_in_xx">AAAA</string>
<string name="previously_translated">CCC</string>
</resources>

21
tools/localize/testdata/merge_xx_old.xml vendored
View File

@@ -1,21 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<!-- Copyright (C) 2007 The Android Open Source Project
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<resources xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2">
<string name="changed_in_xx">aaa</string>
<string name="previously_translated">CCC</string>
</resources>

40
tools/localize/testdata/pseudo.xliff vendored
View File

@@ -1,40 +0,0 @@
<?xml version="1.0" encoding="UTF-8" ?>
<xliff xmlns="urn:oasis:names:tc:xliff:document:1.2"
version="1.2"
>
<file datatype="x-android-res"
original="//device/tools/localization/tests/res/values/strings.xml"
product-version="1.0"
date="08:10:54 12/07/07 PST"
source-language="en-US"
product-name="kila"
target-language="zz-ZZ"
build-num="32138"
>
<body>
<trans-unit id="string:complex">
<source>First <g id="string:complex:0" ctype="underline">underline</g>, <g id="string:complex:1" ctype="italic">italic<g id="string:complex:2" ctype="bold">italicbold</g></g> End </source>
</trans-unit>
<trans-unit id="string:complex-quoted">
<source xml:space="preserve">First <g id="string:complex-quoted:0" ctype="underline">underline</g>, <g id="string:complex-quoted:1" ctype="italic">italic<g id="string:complex-quoted:2" ctype="bold">italicbold</g></g> End</source>
</trans-unit>
<trans-unit id="string:simple">
<source>Simple</source>
</trans-unit>
<trans-unit id="array:0:simple">
<source>Simple</source>
</trans-unit>
<trans-unit id="array:1:simple">
<source>Simple</source>
</trans-unit>
<trans-unit id="string:simple-quoted">
<source xml:space="preserve"> Quote</source>
<alt-trans>
<source xml:lang="en" xml:space="preserve"> OLD Quote</source>
<target xml:lang="xx"> OLD Ờũỡŧę</target>
</alt-trans>
</trans-unit>
</body>
</file>
</xliff>

22
tools/localize/testdata/res/values-zz-rZZ/strings.xml vendored
View File

@@ -1,22 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<!-- Copyright (C) 2007 The Android Open Source Project
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<resources xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2">
<string name="changed_in_xx">AAAA</string>
<string name="previously_translated">CCC</string>
</resources>

44
tools/localize/testdata/res/values/strings.xml vendored
View File

@@ -1,44 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<!-- Copyright (C) 2007 The Android Open Source Project
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<resources xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2">
<string name="changed_in_xx">aaa</string>
<string name="first_translation">bbb</string>
<string name="previously_translated">ccc</string>
<string name="new_string">ccc</string>
<string name="formatted_string"><b>bold</b><i>italic<u>italic_underline</u></i><u>underline</u></string>
<array name="growing_array">
<!-- somebody wrote a comment! -->
<item>1-One</item>
<item>1-Two</item>
<item>1-Three</item>
<item>1-Four</item>
</array>
<array name="shrinking_array">
<!-- somebody wrote a comment! -->
<item>2-One</item>
<item>2-Two</item>
<item>2-Three</item>
</array>
<array name="new_array">
<!-- somebody wrote a comment! -->
<item>3-One</item>
<item>3-Two</item>
<item>3-Three</item>
</array>
</resources>

70
tools/localize/testdata/strip_xliff.xliff vendored
View File

@@ -1,70 +0,0 @@
<?xml version="1.0" encoding="UTF-8" ?>
<xliff xmlns="urn:oasis:names:tc:xliff:document:1.2"
version="1.2"
>
<file datatype="x-android-res"
original="//device/tools/localization/tests/res/values/strings.xml"
product-version="1.0"
date="08:10:54 12/07/07 PST"
source-language="en-US"
product-name="kila"
target-language="zz-ZZ"
build-num="32138"
>
<body>
<trans-unit id="string:string-000-0">
</trans-unit>
<trans-unit id="string:string-001-0">
<alt-trans>
<source xml:lang="en" xml:space="preserve">source</source>
</alt-trans>
</trans-unit>
<trans-unit id="string:string-010-0">
<alt-trans>
<target xml:lang="zz" xml:space="preserve">target</target>
</alt-trans>
</trans-unit>
<trans-unit id="string:string-011-0">
<alt-trans>
<source xml:lang="en" xml:space="preserve">source</source>
<target xml:lang="zz" xml:space="preserve">target</target>
</alt-trans>
</trans-unit>
<trans-unit id="string:string-100-1">
<source xml:space="preserve">source</source>
</trans-unit>
<trans-unit id="string:string-101-1">
<source xml:space="preserve">source</source>
<alt-trans>
<source xml:lang="en" xml:space="preserve">source</source>
</alt-trans>
</trans-unit>
<trans-unit id="string:string-110-1">
<source xml:space="preserve">source</source>
<alt-trans>
<target xml:lang="zz" xml:space="preserve">target</target>
</alt-trans>
</trans-unit>
<trans-unit id="string:string-111-0">
<source xml:space="preserve">source</source>
<alt-trans>
<source xml:lang="en" xml:space="preserve">source</source>
<target xml:lang="zz" xml:space="preserve">target</target>
</alt-trans>
</trans-unit>
<trans-unit id="string:string-111-1">
<source xml:space="preserve">source</source>
<alt-trans>
<source xml:lang="en" xml:space="preserve">alt-source</source>
<target xml:lang="zz" xml:space="preserve">target</target>
</alt-trans>
</trans-unit>
</body>
</file>
</xliff>

32
tools/localize/testdata/values/strings.xml vendored
View File

@@ -1,32 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<!-- Copyright (C) 2007 The Android Open Source Project
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<resources
xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2">
<string name="test1">Discard</string>
<!-- comment -->
<string name="test2">a<b>b<i>c</i></b>d</string>
<string name="test3">a<xliff:g a="b" xliff:a="asdf">bBb</xliff:g>C</string>
<!-- Email address types from android.provider.Contacts -->
<array name="emailAddressTypes">
<!-- somebody wrote a comment! -->
<item>Email</item>
<item>Home</item>
<item>Work</item>
<item>Other\u2026</item>
</array>
</resources>

46
tools/localize/testdata/xliff1.xliff vendored
View File

@@ -1,46 +0,0 @@
<?xml version="1.0" encoding="UTF-8" ?>
<xliff xmlns="urn:oasis:names:tc:xliff:document:1.2"
version="1.2"
>
<file datatype="x-android-res"
original="//device/tools/localization/tests/res/values/strings.xml"
product-version="1.0"
date="08:10:54 12/07/07 PST"
source-language="en-US"
product-name="kila"
target-language="zz-ZZ"
build-num="32138"
>
<body>
<trans-unit id="string:complex">
<source>First <g id="string:complex:0" ctype="underline">underline</g>, <g id="string:complex:1" ctype="italic">italic<g id="string:complex:2" ctype="bold">italicbold</g></g> End </source>
<target>Ḟịṙṩŧ , Ḛŋḋ </target>
</trans-unit>
<trans-unit id="string:complex-quoted">
<source xml:space="preserve">First <g id="string:complex-quoted:0" ctype="underline">underline</g>, <g id="string:complex-quoted:1" ctype="italic">italic<g id="string:complex-quoted:2" ctype="bold">italicbold</g></g> End</source>
<target>Ḟịṙṩŧ , Ḛŋḋ</target>
</trans-unit>
<trans-unit id="string:simple">
<source>Simple</source>
<target>Ṩịṃṕļę</target>
</trans-unit>
<trans-unit id="array:0:simple">
<source>Simple</source>
<target>Ṩịṃṕļę</target>
</trans-unit>
<trans-unit id="array:1:simple">
<source>Simple</source>
<target>Ṩịṃṕļę</target>
</trans-unit>
<trans-unit id="string:simple-quoted">
<source xml:space="preserve"> Quote</source>
<target> Ờũỡŧę</target>
<alt-trans>
<source xml:lang="en" xml:space="preserve"> OLD Quote</source>
<target xml:lang="xx"> OLD Ờũỡŧę</target>
</alt-trans>
</trans-unit>
</body>
</file>
</xliff>

16
tools/localize/testdata/xml.xml vendored
View File

@@ -1,16 +0,0 @@
<ASDF>
<a id="system"
old-cl="1"
new-cl="43019">
<app dir="apps/common" />
</a>
<a id="samples"
old-cl="1"
new-cl="43019">asdf
<app dir="samples/NotePad" />
<app dir="samples/LunarLander" />
<something>a<b>,</b>b </something>
<exact xml:space="preserve">a<b>,</b>b </exact>
</a>
</ASDF>

182
tools/localize/xmb.cpp
View File

@@ -1,182 +0,0 @@
#include "xmb.h"
#include "file_utils.h"
#include "localize.h"
#include "ValuesFile.h"
#include "XMLHandler.h"
#include "XLIFFFile.h"
#include <map>
#include <cstdio>
using namespace std;
const char *const NS_MAP[] = {
"xml", XMLNS_XMLNS,
NULL, NULL
};
set<string> g_tags;
static string
strip_newlines(const string& str)
{
string res;
const size_t N = str.length();
for (size_t i=0; i<N; i++) {
char c = str[i];
if (c != '\n' && c != '\r') {
res += c;
} else {
res += ' ';
}
}
return res;
}
static int
rename_id_attribute(XMLNode* node)
{
vector<XMLAttribute>& attrs = node->EditAttributes();
const size_t I = attrs.size();
for (size_t i=0; i<I; i++) {
XMLAttribute attr = attrs[i];
if (attr.name == "id") {
attr.name = "name";
attrs.erase(attrs.begin()+i);
attrs.push_back(attr);
return 0;
}
}
return 1;
}
static int
convert_xliff_to_ph(XMLNode* node, int* phID)
{
int err = 0;
if (node->Type() == XMLNode::ELEMENT) {
if (node->Namespace() == XLIFF_XMLNS) {
g_tags.insert(node->Name());
node->SetName("", "ph");
err = rename_id_attribute(node);
if (err != 0) {
char name[30];
(*phID)++;
sprintf(name, "id-%d", *phID);
node->EditAttributes().push_back(XMLAttribute("", "name", name));
err = 0;
}
}
vector<XMLNode*>& children = node->EditChildren();
const size_t I = children.size();
for (size_t i=0; i<I; i++) {
err |= convert_xliff_to_ph(children[i], phID);
}
}
return err;
}
XMLNode*
resource_to_xmb_msg(const StringResource& res)
{
// the msg element
vector<XMLAttribute> attrs;
string name = res.pos.file;
name += ":";
name += res.TypedID();
attrs.push_back(XMLAttribute("", "name", name));
attrs.push_back(XMLAttribute("", "desc", strip_newlines(res.comment)));
attrs.push_back(XMLAttribute(XMLNS_XMLNS, "space", "preserve"));
XMLNode* msg = XMLNode::NewElement(res.pos, "", "msg", attrs, XMLNode::EXACT);
// the contents are in xliff/html, convert it to xliff
int err = 0;
XMLNode* value = res.value;
string tag = value->Name();
int phID = 0;
for (vector<XMLNode*>::const_iterator it=value->Children().begin();
it!=value->Children().end(); it++) {
err |= convert_html_to_xliff(*it, tag, msg, &phID);
}
if (err != 0) {
return NULL;
}
// and then convert that to xmb
for (vector<XMLNode*>::iterator it=msg->EditChildren().begin();
it!=msg->EditChildren().end(); it++) {
err |= convert_xliff_to_ph(*it, &phID);
}
if (err == 0) {
return msg;
} else {
return NULL;
}
}
int
do_xlb_export(const string& outfile, const vector<string>& resFiles)
{
int err = 0;
size_t totalFileCount = resFiles.size();
Configuration english;
english.locale = "en_US";
set<StringResource> allResources;
const size_t J = resFiles.size();
for (size_t j=0; j<J; j++) {
string resFile = resFiles[j];
ValuesFile* valuesFile = get_local_values_file(resFile, english, CURRENT_VERSION, "", true);
if (valuesFile != NULL) {
set<StringResource> resources = valuesFile->GetStrings();
allResources.insert(resources.begin(), resources.end());
} else {
fprintf(stderr, "error reading file %s\n", resFile.c_str());
}
delete valuesFile;
}
// Construct the XLB xml
vector<XMLAttribute> attrs;
attrs.push_back(XMLAttribute("", "locale", "en"));
XMLNode* localizationbundle = XMLNode::NewElement(GENERATED_POS, "", "localizationbundle",
attrs, XMLNode::PRETTY);
for (set<StringResource>::iterator it=allResources.begin(); it!=allResources.end(); it++) {
XMLNode* msg = resource_to_xmb_msg(*it);
if (msg) {
localizationbundle->EditChildren().push_back(msg);
} else {
err = 1;
}
}
#if 0
for (set<string>::iterator it=g_tags.begin(); it!=g_tags.end(); it++) {
printf("tag: %s\n", it->c_str());
}
printf("err=%d\n", err);
#endif
if (err == 0) {
FILE* f = fopen(outfile.c_str(), "wb");
if (f == NULL) {
fprintf(stderr, "can't open outputfile: %s\n", outfile.c_str());
return 1;
}
fprintf(f, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
fprintf(f, "%s\n", localizationbundle->ToString(NS_MAP).c_str());
fclose(f);
}
return err;
}

11
tools/localize/xmb.h
View File

@@ -1,11 +0,0 @@
#ifndef XMB_H
#define XMB_H
#include <string>
#include <vector>
using namespace std;
int do_xlb_export(const string& outFile, const vector<string>& resFiles);
#endif // XMB_H