From d86ea58bddea7d5608e3539fc77e3d805c0af1d1 Mon Sep 17 00:00:00 2001
From: Ryan Mitchell <rtmitchell@google.com>
Date: Wed, 27 Jun 2018 11:57:18 -0700
Subject: [PATCH] AAPT2: Encode 4-byte strings in Modified UTF-8

Codepoints that are encoded to 4 bytes in UTF-8 are not allowed in
Modified UTF-8. They instead should be encoded as surrogate pairs in the
same way that CESU-8 allows for surrogate pairs. This will also cause 4
byte UTF-8 codes to be represented in 6 bytes.

Bug: 37140916
Test: aapt2_tests
Change-Id: I155dc24f166139d1d36a16bac088dcfcd59eb321
---
 tools/aapt2/StringPool.cpp      |  2 +-
 tools/aapt2/StringPool_test.cpp | 19 +++++++++++++
 tools/aapt2/util/Util.cpp       | 47 +++++++++++++++++++++++++++++++++
 tools/aapt2/util/Util.h         |  3 +++
 4 files changed, 70 insertions(+), 1 deletion(-)

diff --git a/tools/aapt2/StringPool.cpp b/tools/aapt2/StringPool.cpp
index b37e1fbd96938..8eabd3225d87a 100644
--- a/tools/aapt2/StringPool.cpp
+++ b/tools/aapt2/StringPool.cpp
@@ -367,7 +367,7 @@ const std::string kStringTooLarge = "STRING_TOO_LARGE";
 static bool EncodeString(const std::string& str, const bool utf8, BigBuffer* out,
                          IDiagnostics* diag) {
   if (utf8) {
-    const std::string& encoded = str;
+    const std::string& encoded = util::Utf8ToModifiedUtf8(str);
     const ssize_t utf16_length = utf8_to_utf16_length(
         reinterpret_cast<const uint8_t*>(encoded.data()), encoded.size());
     CHECK(utf16_length >= 0);
diff --git a/tools/aapt2/StringPool_test.cpp b/tools/aapt2/StringPool_test.cpp
index 4b3afe2bb962a..0778564ee0791 100644
--- a/tools/aapt2/StringPool_test.cpp
+++ b/tools/aapt2/StringPool_test.cpp
@@ -303,6 +303,25 @@ TEST(StringPoolTest, Flatten) {
   }
 }
 
+TEST(StringPoolTest, FlattenModifiedUTF8) {
+  using namespace android;  // For NO_ERROR on Windows.
+  StdErrDiagnostics diag;
+  StringPool pool;
+  StringPool::Ref ref_a = pool.MakeRef("\xF0\x90\x90\x80"); // 𐐀 (U+10400)
+  StringPool::Ref ref_b = pool.MakeRef("foo \xF0\x90\x90\xB7 bar"); // 𐐷 (U+10437)
+  StringPool::Ref ref_c = pool.MakeRef("\xF0\x90\x90\x80\xF0\x90\x90\xB7");
+
+  BigBuffer buffer(1024);
+  StringPool::FlattenUtf8(&buffer, pool, &diag);
+  std::unique_ptr<uint8_t[]> data = util::Copy(buffer);
+
+  // Check that the 4 byte utf-8 codepoint is encoded using 2 3 byte surrogate pairs
+  ResStringPool test;
+  ASSERT_EQ(test.setTo(data.get(), buffer.size()), NO_ERROR);
+  EXPECT_THAT(util::GetString(test, 0), Eq("\xED\xA0\x81\xED\xB0\x80"));
+  EXPECT_THAT(util::GetString(test, 1), Eq("foo \xED\xA0\x81\xED\xB0\xB7 bar"));
+  EXPECT_THAT(util::GetString(test, 2), Eq("\xED\xA0\x81\xED\xB0\x80\xED\xA0\x81\xED\xB0\xB7"));
+}
 
 TEST(StringPoolTest, MaxEncodingLength) {
   StdErrDiagnostics diag;
diff --git a/tools/aapt2/util/Util.cpp b/tools/aapt2/util/Util.cpp
index d1c9ca1644d99..9bef54e590c98 100644
--- a/tools/aapt2/util/Util.cpp
+++ b/tools/aapt2/util/Util.cpp
@@ -297,6 +297,53 @@ bool VerifyJavaStringFormat(const StringPiece& str) {
   return true;
 }
 
+std::string Utf8ToModifiedUtf8(const std::string& utf8) {
+  // Java uses Modified UTF-8 which only supports the 1, 2, and 3 byte formats of UTF-8. To encode
+  // 4 byte UTF-8 codepoints, Modified UTF-8 allows the use of surrogate pairs in the same format
+  // of CESU-8 surrogate pairs. Calculate the size of the utf8 string with all 4 byte UTF-8
+  // codepoints replaced with 2 3 byte surrogate pairs
+  size_t modified_size = 0;
+  const size_t size = utf8.size();
+  for (size_t i = 0; i < size; i++) {
+    if (((uint8_t) utf8[i] >> 4) == 0xF) {
+      modified_size += 6;
+      i += 3;
+    } else {
+      modified_size++;
+    }
+  }
+
+  // Early out if no 4 byte codepoints are found
+  if (size == modified_size) {
+    return utf8;
+  }
+
+  std::string output;
+  output.reserve(modified_size);
+  for (size_t i = 0; i < size; i++) {
+    if (((uint8_t) utf8[i] >> 4) == 0xF) {
+      auto codepoint = (char32_t) utf32_from_utf8_at(utf8.data(), size, i, nullptr);
+
+      // Calculate the high and low surrogates as UTF-16 would
+      char32_t high = ((codepoint - 0x10000) / 0x400) + 0xD800;
+      char32_t low = ((codepoint - 0x10000) % 0x400) + 0xDC00;
+
+      // Encode each surrogate in UTF-8
+      output.push_back((char) (0xE4 | ((high >> 12) & 0xF)));
+      output.push_back((char) (0x80 | ((high >> 6) & 0x3F)));
+      output.push_back((char) (0x80 | (high & 0x3F)));
+      output.push_back((char) (0xE4 | ((low >> 12) & 0xF)));
+      output.push_back((char) (0x80 | ((low >> 6) & 0x3F)));
+      output.push_back((char) (0x80 | (low & 0x3F)));
+      i += 3;
+    } else {
+      output.push_back(utf8[i]);
+    }
+  }
+
+  return output;
+}
+
 std::u16string Utf8ToUtf16(const StringPiece& utf8) {
   ssize_t utf16_length = utf8_to_utf16_length(
       reinterpret_cast<const uint8_t*>(utf8.data()), utf8.length());
diff --git a/tools/aapt2/util/Util.h b/tools/aapt2/util/Util.h
index 0eb35d18c06e0..36b733376e6fe 100644
--- a/tools/aapt2/util/Util.h
+++ b/tools/aapt2/util/Util.h
@@ -197,6 +197,9 @@ inline StringBuilder::operator bool() const {
   return error_.empty();
 }
 
+// Converts a UTF8 string into Modified UTF8
+std::string Utf8ToModifiedUtf8(const std::string& utf8);
+
 // Converts a UTF8 string to a UTF16 string.
 std::u16string Utf8ToUtf16(const android::StringPiece& utf8);
 std::string Utf16ToUtf8(const android::StringPiece16& utf16);