From a40e972fdaa68fc486ff90a319195515819068b2 Mon Sep 17 00:00:00 2001 From: Adam Lesinski Date: Tue, 24 Nov 2015 19:11:46 -0800 Subject: [PATCH] AAPT2: Support compiling a res/ directory and output to zip This allows us to compile an entire directory and output to a single file. This is important to support generated resources in the make build, since we may not know what resources get generated. The link step will accept the zip and read the contents of it as if they were passed in on the command line. Change-Id: If1a51b0abe772350c24074353eb4989953c2e0cb --- tools/aapt2/compile/Compile.cpp | 221 ++++++++++++++++------- tools/aapt2/flatten/Archive.cpp | 173 +++++++++--------- tools/aapt2/flatten/Archive.h | 15 +- tools/aapt2/io/Data.h | 85 +++++++++ tools/aapt2/io/File.h | 72 ++++++++ tools/aapt2/io/FileSystem.h | 78 ++++++++ tools/aapt2/io/ZipArchive.h | 143 +++++++++++++++ tools/aapt2/link/Link.cpp | 311 +++++++++++++++++++------------- 8 files changed, 818 insertions(+), 280 deletions(-) create mode 100644 tools/aapt2/io/Data.h create mode 100644 tools/aapt2/io/File.h create mode 100644 tools/aapt2/io/FileSystem.h create mode 100644 tools/aapt2/io/ZipArchive.h diff --git a/tools/aapt2/compile/Compile.cpp b/tools/aapt2/compile/Compile.cpp index 17a658ee27cf4..90e35d52788c4 100644 --- a/tools/aapt2/compile/Compile.cpp +++ b/tools/aapt2/compile/Compile.cpp @@ -22,6 +22,7 @@ #include "compile/IdAssigner.h" #include "compile/Png.h" #include "compile/XmlIdCollector.h" +#include "flatten/Archive.h" #include "flatten/FileExportWriter.h" #include "flatten/TableFlattener.h" #include "flatten/XmlFlattener.h" @@ -31,6 +32,7 @@ #include "xml/XmlDom.h" #include "xml/XmlPullParser.h" +#include #include #include @@ -90,7 +92,7 @@ static Maybe extractResourcePathData(const std::string& path, } return ResourcePathData{ - Source{ path }, + Source(path), util::utf8ToUtf16(dirStr), util::utf8ToUtf16(name), extension.toString(), @@ -101,25 +103,79 @@ static Maybe extractResourcePathData(const std::string& path, struct CompileOptions { std::string outputPath; + Maybe resDir; Maybe product; bool verbose = false; }; -static std::string buildIntermediateFilename(const std::string outDir, - const ResourcePathData& data) { +static std::string buildIntermediateFilename(const ResourcePathData& data) { std::stringstream name; name << data.resourceDir; if (!data.configStr.empty()) { name << "-" << data.configStr; } name << "_" << data.name << "." << data.extension << ".flat"; - std::string outPath = outDir; - file::appendPath(&outPath, name.str()); - return outPath; + return name.str(); +} + +static bool isHidden(const StringPiece& filename) { + return util::stringStartsWith(filename, "."); +} + +/** + * Walks the res directory structure, looking for resource files. + */ +static bool loadInputFilesFromDir(IAaptContext* context, const CompileOptions& options, + std::vector* outPathData) { + const std::string& rootDir = options.resDir.value(); + std::unique_ptr d(opendir(rootDir.data()), closedir); + if (!d) { + context->getDiagnostics()->error(DiagMessage() << strerror(errno)); + return false; + } + + while (struct dirent* entry = readdir(d.get())) { + if (isHidden(entry->d_name)) { + continue; + } + + std::string prefixPath = rootDir; + file::appendPath(&prefixPath, entry->d_name); + + if (file::getFileType(prefixPath) != file::FileType::kDirectory) { + continue; + } + + std::unique_ptr subDir(opendir(prefixPath.data()), closedir); + if (!subDir) { + context->getDiagnostics()->error(DiagMessage() << strerror(errno)); + return false; + } + + while (struct dirent* leafEntry = readdir(subDir.get())) { + if (isHidden(leafEntry->d_name)) { + continue; + } + + std::string fullPath = prefixPath; + file::appendPath(&fullPath, leafEntry->d_name); + + std::string errStr; + Maybe pathData = extractResourcePathData(fullPath, &errStr); + if (!pathData) { + context->getDiagnostics()->error(DiagMessage() << errStr); + return false; + } + + outPathData->push_back(std::move(pathData.value())); + } + } + return true; } static bool compileTable(IAaptContext* context, const CompileOptions& options, - const ResourcePathData& pathData, const std::string& outputPath) { + const ResourcePathData& pathData, IArchiveWriter* writer, + const std::string& outputPath) { ResourceTable table; { std::ifstream fin(pathData.source.path, std::ifstream::binary); @@ -150,6 +206,7 @@ static bool compileTable(IAaptContext* context, const CompileOptions& options, // Ensure we have the compilation package at least. table.createPackage(context->getCompilationPackage()); + // Assign an ID to any package that has resources. for (auto& pkg : table.packages) { if (!pkg->id) { // If no package ID was set while parsing (public identifiers), auto assign an ID. @@ -172,23 +229,24 @@ static bool compileTable(IAaptContext* context, const CompileOptions& options, return false; } - // Build the output filename. - std::ofstream fout(outputPath, std::ofstream::binary); - if (!fout) { - context->getDiagnostics()->error(DiagMessage(Source{ outputPath }) << strerror(errno)); + if (!writer->startEntry(outputPath, 0)) { + context->getDiagnostics()->error(DiagMessage(outputPath) << "failed to open"); return false; } - // Write it to disk. - if (!util::writeAll(fout, buffer)) { - context->getDiagnostics()->error(DiagMessage(Source{ outputPath }) << strerror(errno)); - return false; + if (writer->writeEntry(buffer)) { + if (writer->finishEntry()) { + return true; + } } - return true; + + context->getDiagnostics()->error(DiagMessage(outputPath) << "failed to write"); + return false; } static bool compileXml(IAaptContext* context, const CompileOptions& options, - const ResourcePathData& pathData, const std::string& outputPath) { + const ResourcePathData& pathData, IArchiveWriter* writer, + const std::string& outputPath) { std::unique_ptr xmlRes; @@ -214,7 +272,7 @@ static bool compileXml(IAaptContext* context, const CompileOptions& options, return false; } - xmlRes->file.name = ResourceName{ {}, *parseResourceType(pathData.resourceDir), pathData.name }; + xmlRes->file.name = ResourceName({}, *parseResourceType(pathData.resourceDir), pathData.name); xmlRes->file.config = pathData.config; xmlRes->file.source = pathData.source; @@ -230,25 +288,27 @@ static bool compileXml(IAaptContext* context, const CompileOptions& options, fileExportWriter.finish(); - std::ofstream fout(outputPath, std::ofstream::binary); - if (!fout) { - context->getDiagnostics()->error(DiagMessage(Source{ outputPath }) << strerror(errno)); + if (!writer->startEntry(outputPath, 0)) { + context->getDiagnostics()->error(DiagMessage(outputPath) << "failed to open"); return false; } - // Write it to disk. - if (!util::writeAll(fout, buffer)) { - context->getDiagnostics()->error(DiagMessage(Source{ outputPath }) << strerror(errno)); - return false; + if (writer->writeEntry(buffer)) { + if (writer->finishEntry()) { + return true; + } } - return true; + + context->getDiagnostics()->error(DiagMessage(outputPath) << "failed to write"); + return false; } static bool compilePng(IAaptContext* context, const CompileOptions& options, - const ResourcePathData& pathData, const std::string& outputPath) { + const ResourcePathData& pathData, IArchiveWriter* writer, + const std::string& outputPath) { BigBuffer buffer(4096); ResourceFile resFile; - resFile.name = ResourceName{ {}, *parseResourceType(pathData.resourceDir), pathData.name }; + resFile.name = ResourceName({}, *parseResourceType(pathData.resourceDir), pathData.name); resFile.config = pathData.config; resFile.source = pathData.source; @@ -269,24 +329,27 @@ static bool compilePng(IAaptContext* context, const CompileOptions& options, fileExportWriter.finish(); - std::ofstream fout(outputPath, std::ofstream::binary); - if (!fout) { - context->getDiagnostics()->error(DiagMessage(Source{ outputPath }) << strerror(errno)); + if (!writer->startEntry(outputPath, 0)) { + context->getDiagnostics()->error(DiagMessage(outputPath) << "failed to open"); return false; } - if (!util::writeAll(fout, buffer)) { - context->getDiagnostics()->error(DiagMessage(Source{ outputPath }) << strerror(errno)); - return false; + if (writer->writeEntry(buffer)) { + if (writer->finishEntry()) { + return true; + } } - return true; + + context->getDiagnostics()->error(DiagMessage(outputPath) << "failed to write"); + return false; } static bool compileFile(IAaptContext* context, const CompileOptions& options, - const ResourcePathData& pathData, const std::string& outputPath) { + const ResourcePathData& pathData, IArchiveWriter* writer, + const std::string& outputPath) { BigBuffer buffer(256); ResourceFile resFile; - resFile.name = ResourceName{ {}, *parseResourceType(pathData.resourceDir), pathData.name }; + resFile.name = ResourceName({}, *parseResourceType(pathData.resourceDir), pathData.name); resFile.config = pathData.config; resFile.source = pathData.source; @@ -299,9 +362,8 @@ static bool compileFile(IAaptContext* context, const CompileOptions& options, return false; } - std::ofstream fout(outputPath, std::ofstream::binary); - if (!fout) { - context->getDiagnostics()->error(DiagMessage(Source{ outputPath }) << strerror(errno)); + if (!writer->startEntry(outputPath, 0)) { + context->getDiagnostics()->error(DiagMessage(outputPath) << "failed to open"); return false; } @@ -309,16 +371,17 @@ static bool compileFile(IAaptContext* context, const CompileOptions& options, // the buffer the entire file. fileExportWriter.getChunkHeader()->size = util::hostToDevice32(buffer.size() + f.value().getDataLength()); - if (!util::writeAll(fout, buffer)) { - context->getDiagnostics()->error(DiagMessage(Source{ outputPath }) << strerror(errno)); - return false; + + if (writer->writeEntry(buffer)) { + if (writer->writeEntry(f.value().getDataPtr(), f.value().getDataLength())) { + if (writer->finishEntry()) { + return true; + } + } } - if (!fout.write((const char*) f.value().getDataPtr(), f.value().getDataLength())) { - context->getDiagnostics()->error(DiagMessage(Source{ outputPath }) << strerror(errno)); - return false; - } - return true; + context->getDiagnostics()->error(DiagMessage(outputPath) << "failed to write"); + return false; } class CompileContext : public IAaptContext { @@ -359,6 +422,7 @@ int compile(const std::vector& args) { Flags flags = Flags() .requiredFlag("-o", "Output path", &options.outputPath) .optionalFlag("--product", "Product type to compile", &product) + .optionalFlag("--dir", "Directory to scan for resources", &options.resDir) .optionalSwitch("-v", "Enables verbose logging", &options.verbose); if (!flags.parse("aapt2 compile", args, &std::cerr)) { return 1; @@ -369,19 +433,42 @@ int compile(const std::vector& args) { } CompileContext context; + std::unique_ptr archiveWriter; std::vector inputData; - inputData.reserve(flags.getArgs().size()); - - // Collect data from the path for each input file. - for (const std::string& arg : flags.getArgs()) { - std::string errorStr; - if (Maybe pathData = extractResourcePathData(arg, &errorStr)) { - inputData.push_back(std::move(pathData.value())); - } else { - context.getDiagnostics()->error(DiagMessage() << errorStr << " (" << arg << ")"); + if (options.resDir) { + if (!flags.getArgs().empty()) { + // Can't have both files and a resource directory. + context.getDiagnostics()->error(DiagMessage() << "files given but --dir specified"); + flags.usage("aapt2 compile", &std::cerr); return 1; } + + if (!loadInputFilesFromDir(&context, options, &inputData)) { + return 1; + } + + archiveWriter = createZipFileArchiveWriter(context.getDiagnostics(), options.outputPath); + + } else { + inputData.reserve(flags.getArgs().size()); + + // Collect data from the path for each input file. + for (const std::string& arg : flags.getArgs()) { + std::string errorStr; + if (Maybe pathData = extractResourcePathData(arg, &errorStr)) { + inputData.push_back(std::move(pathData.value())); + } else { + context.getDiagnostics()->error(DiagMessage() << errorStr << " (" << arg << ")"); + return 1; + } + } + + archiveWriter = createDirectoryArchiveWriter(context.getDiagnostics(), options.outputPath); + } + + if (!archiveWriter) { + return false; } bool error = false; @@ -394,32 +481,34 @@ int compile(const std::vector& args) { // Overwrite the extension. pathData.extension = "arsc"; - const std::string outputFilename = buildIntermediateFilename( - options.outputPath, pathData); - if (!compileTable(&context, options, pathData, outputFilename)) { + const std::string outputFilename = buildIntermediateFilename(pathData); + if (!compileTable(&context, options, pathData, archiveWriter.get(), outputFilename)) { error = true; } } else { - const std::string outputFilename = buildIntermediateFilename(options.outputPath, - pathData); + const std::string outputFilename = buildIntermediateFilename(pathData); if (const ResourceType* type = parseResourceType(pathData.resourceDir)) { if (*type != ResourceType::kRaw) { if (pathData.extension == "xml") { - if (!compileXml(&context, options, pathData, outputFilename)) { + if (!compileXml(&context, options, pathData, archiveWriter.get(), + outputFilename)) { error = true; } } else if (pathData.extension == "png" || pathData.extension == "9.png") { - if (!compilePng(&context, options, pathData, outputFilename)) { + if (!compilePng(&context, options, pathData, archiveWriter.get(), + outputFilename)) { error = true; } } else { - if (!compileFile(&context, options, pathData, outputFilename)) { + if (!compileFile(&context, options, pathData, archiveWriter.get(), + outputFilename)) { error = true; } } } else { - if (!compileFile(&context, options, pathData, outputFilename)) { + if (!compileFile(&context, options, pathData, archiveWriter.get(), + outputFilename)) { error = true; } } diff --git a/tools/aapt2/flatten/Archive.cpp b/tools/aapt2/flatten/Archive.cpp index 6db13b86fbfa4..3a244c05efecb 100644 --- a/tools/aapt2/flatten/Archive.cpp +++ b/tools/aapt2/flatten/Archive.cpp @@ -18,7 +18,7 @@ #include "util/Files.h" #include "util/StringPiece.h" -#include +#include #include #include #include @@ -30,70 +30,85 @@ namespace { struct DirectoryWriter : public IArchiveWriter { std::string mOutDir; - std::vector> mEntries; + std::unique_ptr mFile = { nullptr, fclose }; - explicit DirectoryWriter(const StringPiece& outDir) : mOutDir(outDir.toString()) { + bool open(IDiagnostics* diag, const StringPiece& outDir) { + mOutDir = outDir.toString(); + file::FileType type = file::getFileType(mOutDir); + if (type == file::FileType::kNonexistant) { + diag->error(DiagMessage() << "directory " << mOutDir << " does not exist"); + return false; + } else if (type != file::FileType::kDirectory) { + diag->error(DiagMessage() << mOutDir << " is not a directory"); + return false; + } + return true; } - ArchiveEntry* writeEntry(const StringPiece& path, uint32_t flags, - const BigBuffer& buffer) override { + bool startEntry(const StringPiece& path, uint32_t flags) override { + if (mFile) { + return false; + } + std::string fullPath = mOutDir; file::appendPath(&fullPath, path); file::mkdirs(file::getStem(fullPath)); - std::ofstream fout(fullPath, std::ofstream::binary); - if (!fout) { - return nullptr; + mFile = { fopen(fullPath.data(), "wb"), fclose }; + if (!mFile) { + return false; } - - if (!util::writeAll(fout, buffer)) { - return nullptr; - } - - mEntries.push_back(util::make_unique(fullPath, flags, buffer.size())); - return mEntries.back().get(); + return true; } - ArchiveEntry* writeEntry(const StringPiece& path, uint32_t flags, android::FileMap* fileMap, - size_t offset, size_t len) override { - std::string fullPath = mOutDir; - file::appendPath(&fullPath, path); - file::mkdirs(file::getStem(fullPath)); - - std::ofstream fout(fullPath, std::ofstream::binary); - if (!fout) { - return nullptr; + bool writeEntry(const BigBuffer& buffer) override { + if (!mFile) { + return false; } - if (!fout.write((const char*) fileMap->getDataPtr() + offset, len)) { - return nullptr; + for (const BigBuffer::Block& b : buffer) { + if (fwrite(b.buffer.get(), 1, b.size, mFile.get()) != b.size) { + mFile.reset(nullptr); + return false; + } } - - mEntries.push_back(util::make_unique(fullPath, flags, len)); - return mEntries.back().get(); + return true; } - virtual ~DirectoryWriter() { + bool writeEntry(const void* data, size_t len) override { + if (fwrite(data, 1, len, mFile.get()) != len) { + mFile.reset(nullptr); + return false; + } + return true; + } + bool finishEntry() override { + if (!mFile) { + return false; + } + mFile.reset(nullptr); + return true; } }; struct ZipFileWriter : public IArchiveWriter { - FILE* mFile; + std::unique_ptr mFile = { nullptr, fclose }; std::unique_ptr mWriter; - std::vector> mEntries; - explicit ZipFileWriter(const StringPiece& path) { - mFile = fopen(path.data(), "w+b"); - if (mFile) { - mWriter = util::make_unique(mFile); + bool open(IDiagnostics* diag, const StringPiece& path) { + mFile = { fopen(path.data(), "w+b"), fclose }; + if (!mFile) { + diag->error(DiagMessage() << "failed to open " << path << ": " << strerror(errno)); + return false; } + mWriter = util::make_unique(mFile.get()); + return true; } - ArchiveEntry* writeEntry(const StringPiece& path, uint32_t flags, - const BigBuffer& buffer) override { + bool startEntry(const StringPiece& path, uint32_t flags) override { if (!mWriter) { - return nullptr; + return false; } size_t zipFlags = 0; @@ -107,75 +122,63 @@ struct ZipFileWriter : public IArchiveWriter { int32_t result = mWriter->StartEntry(path.data(), zipFlags); if (result != 0) { - return nullptr; + return false; } + return true; + } + bool writeEntry(const void* data, size_t len) override { + int32_t result = mWriter->WriteBytes(data, len); + if (result != 0) { + return false; + } + return true; + } + + bool writeEntry(const BigBuffer& buffer) override { for (const BigBuffer::Block& b : buffer) { - result = mWriter->WriteBytes(reinterpret_cast(b.buffer.get()), b.size); + int32_t result = mWriter->WriteBytes(b.buffer.get(), b.size); if (result != 0) { - return nullptr; + return false; } } - - result = mWriter->FinishEntry(); - if (result != 0) { - return nullptr; - } - - mEntries.push_back(util::make_unique(path.toString(), flags, buffer.size())); - return mEntries.back().get(); + return true; } - ArchiveEntry* writeEntry(const StringPiece& path, uint32_t flags, android::FileMap* fileMap, - size_t offset, size_t len) override { - if (!mWriter) { - return nullptr; - } - - size_t zipFlags = 0; - if (flags & ArchiveEntry::kCompress) { - zipFlags |= ZipWriter::kCompress; - } - - if (flags & ArchiveEntry::kAlign) { - zipFlags |= ZipWriter::kAlign32; - } - - int32_t result = mWriter->StartEntry(path.data(), zipFlags); + bool finishEntry() override { + int32_t result = mWriter->FinishEntry(); if (result != 0) { - return nullptr; + return false; } - - result = mWriter->WriteBytes((const char*) fileMap->getDataPtr() + offset, len); - if (result != 0) { - return nullptr; - } - - result = mWriter->FinishEntry(); - if (result != 0) { - return nullptr; - } - - mEntries.push_back(util::make_unique(path.toString(), flags, len)); - return mEntries.back().get(); + return true; } virtual ~ZipFileWriter() { if (mWriter) { mWriter->Finish(); - fclose(mFile); } } }; } // namespace -std::unique_ptr createDirectoryArchiveWriter(const StringPiece& path) { - return util::make_unique(path); +std::unique_ptr createDirectoryArchiveWriter(IDiagnostics* diag, + const StringPiece& path) { + + std::unique_ptr writer = util::make_unique(); + if (!writer->open(diag, path)) { + return {}; + } + return std::move(writer); } -std::unique_ptr createZipFileArchiveWriter(const StringPiece& path) { - return util::make_unique(path); +std::unique_ptr createZipFileArchiveWriter(IDiagnostics* diag, + const StringPiece& path) { + std::unique_ptr writer = util::make_unique(); + if (!writer->open(diag, path)) { + return {}; + } + return std::move(writer); } } // namespace aapt diff --git a/tools/aapt2/flatten/Archive.h b/tools/aapt2/flatten/Archive.h index c4ddeb3163c01..6da1d2ac5620c 100644 --- a/tools/aapt2/flatten/Archive.h +++ b/tools/aapt2/flatten/Archive.h @@ -17,6 +17,7 @@ #ifndef AAPT_FLATTEN_ARCHIVE_H #define AAPT_FLATTEN_ARCHIVE_H +#include "Diagnostics.h" #include "util/BigBuffer.h" #include "util/Files.h" #include "util/StringPiece.h" @@ -42,15 +43,17 @@ struct ArchiveEntry { struct IArchiveWriter { virtual ~IArchiveWriter() = default; - virtual ArchiveEntry* writeEntry(const StringPiece& path, uint32_t flags, - const BigBuffer& buffer) = 0; - virtual ArchiveEntry* writeEntry(const StringPiece& path, uint32_t flags, - android::FileMap* fileMap, size_t offset, size_t len) = 0; + virtual bool startEntry(const StringPiece& path, uint32_t flags) = 0; + virtual bool writeEntry(const BigBuffer& buffer) = 0; + virtual bool writeEntry(const void* data, size_t len) = 0; + virtual bool finishEntry() = 0; }; -std::unique_ptr createDirectoryArchiveWriter(const StringPiece& path); +std::unique_ptr createDirectoryArchiveWriter(IDiagnostics* diag, + const StringPiece& path); -std::unique_ptr createZipFileArchiveWriter(const StringPiece& path); +std::unique_ptr createZipFileArchiveWriter(IDiagnostics* diag, + const StringPiece& path); } // namespace aapt diff --git a/tools/aapt2/io/Data.h b/tools/aapt2/io/Data.h new file mode 100644 index 0000000000000..9081c55fc6e12 --- /dev/null +++ b/tools/aapt2/io/Data.h @@ -0,0 +1,85 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef AAPT_IO_DATA_H +#define AAPT_IO_DATA_H + +#include + +#include + +namespace aapt { +namespace io { + +/** + * Interface for a block of contiguous memory. An instance of this interface owns the data. + */ +class IData { +public: + virtual ~IData() = default; + + virtual const void* data() const = 0; + virtual size_t size() const = 0; +}; + +/** + * Implementation of IData that exposes a memory mapped file. The mmapped file is owned by this + * object. + */ +class MmappedData : public IData { +public: + explicit MmappedData(android::FileMap&& map) : mMap(std::forward(map)) { + } + + const void* data() const override { + return mMap.getDataPtr(); + } + + size_t size() const override { + return mMap.getDataLength(); + } + +private: + android::FileMap mMap; +}; + +/** + * Implementation of IData that exposes a block of memory that was malloc'ed (new'ed). The + * memory is owned by this object. + */ +class MallocData : public IData { +public: + MallocData(std::unique_ptr data, size_t size) : + mData(std::move(data)), mSize(size) { + } + + const void* data() const override { + return mData.get(); + } + + size_t size() const override { + return mSize; + } + +private: + std::unique_ptr mData; + size_t mSize; +}; + +} // namespace io +} // namespace aapt + +#endif /* AAPT_IO_DATA_H */ diff --git a/tools/aapt2/io/File.h b/tools/aapt2/io/File.h new file mode 100644 index 0000000000000..9fca3980d964e --- /dev/null +++ b/tools/aapt2/io/File.h @@ -0,0 +1,72 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef AAPT_IO_FILE_H +#define AAPT_IO_FILE_H + +#include "Source.h" +#include "io/Data.h" + +#include +#include + +namespace aapt { +namespace io { + +/** + * Interface for a file, which could be a real file on the file system, or a file inside + * a ZIP archive. + */ +class IFile { +public: + virtual ~IFile() = default; + + /** + * Open the file and return it as a block of contiguous memory. How this occurs is + * implementation dependent. For example, if this is a file on the file system, it may + * simply mmap the contents. If this file represents a compressed file in a ZIP archive, + * it may need to inflate it to memory, incurring a copy. + * + * Returns nullptr on failure. + */ + virtual std::unique_ptr openAsData() = 0; + + /** + * Returns the source of this file. This is for presentation to the user and may not be a + * valid file system path (for example, it may contain a '@' sign to separate the files within + * a ZIP archive from the path to the containing ZIP archive. + */ + virtual const Source& getSource() const = 0; +}; + +/** + * Interface for a collection of files, all of which share a common source. That source may + * simply be the filesystem, or a ZIP archive. + */ +class IFileCollection { +public: + virtual ~IFileCollection() = default; + + using const_iterator = std::vector>::const_iterator; + + virtual const_iterator begin() const = 0; + virtual const_iterator end() const = 0; +}; + +} // namespace io +} // namespace aapt + +#endif /* AAPT_IO_FILE_H */ diff --git a/tools/aapt2/io/FileSystem.h b/tools/aapt2/io/FileSystem.h new file mode 100644 index 0000000000000..5dbefcc0f6873 --- /dev/null +++ b/tools/aapt2/io/FileSystem.h @@ -0,0 +1,78 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef AAPT_IO_FILESYSTEM_H +#define AAPT_IO_FILESYSTEM_H + +#include "io/File.h" +#include "util/Files.h" + +namespace aapt { +namespace io { + +/** + * A regular file from the file system. Uses mmap to open the data. + */ +class RegularFile : public IFile { +public: + RegularFile(const Source& source) : mSource(source) { + } + + std::unique_ptr openAsData() override { + android::FileMap map; + if (Maybe map = file::mmapPath(mSource.path, nullptr)) { + return util::make_unique(std::move(map.value())); + } + return {}; + } + + const Source& getSource() const override { + return mSource; + } + +private: + Source mSource; +}; + +/** + * An IFileCollection representing the file system. + */ +class FileCollection : public IFileCollection { +public: + /** + * Adds a file located at path. Returns the IFile representation of that file. + */ + IFile* insertFile(const StringPiece& path) { + mFiles.push_back(util::make_unique(Source(path))); + return mFiles.back().get(); + } + + const_iterator begin() const override { + return mFiles.begin(); + } + + const_iterator end() const override { + return mFiles.end(); + } + +private: + std::vector> mFiles; +}; + +} // namespace io +} // namespace aapt + +#endif // AAPT_IO_FILESYSTEM_H diff --git a/tools/aapt2/io/ZipArchive.h b/tools/aapt2/io/ZipArchive.h new file mode 100644 index 0000000000000..98afc498708ff --- /dev/null +++ b/tools/aapt2/io/ZipArchive.h @@ -0,0 +1,143 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef AAPT_IO_ZIPARCHIVE_H +#define AAPT_IO_ZIPARCHIVE_H + +#include "io/File.h" +#include "util/StringPiece.h" + +#include +#include + +namespace aapt { +namespace io { + +/** + * An IFile representing a file within a ZIP archive. If the file is compressed, it is uncompressed + * and copied into memory when opened. Otherwise it is mmapped from the ZIP archive. + */ +class ZipFile : public IFile { +public: + ZipFile(ZipArchiveHandle handle, const ZipEntry& entry, const Source& source) : + mZipHandle(handle), mZipEntry(entry), mSource(source) { + } + + std::unique_ptr openAsData() override { + if (mZipEntry.method == kCompressStored) { + int fd = GetFileDescriptor(mZipHandle); + + android::FileMap fileMap; + bool result = fileMap.create(nullptr, fd, mZipEntry.offset, + mZipEntry.uncompressed_length, true); + if (!result) { + return {}; + } + return util::make_unique(std::move(fileMap)); + + } else { + std::unique_ptr data = std::unique_ptr( + new uint8_t[mZipEntry.uncompressed_length]); + int32_t result = ExtractToMemory(mZipHandle, &mZipEntry, data.get(), + static_cast(mZipEntry.uncompressed_length)); + if (result != 0) { + return {}; + } + return util::make_unique(std::move(data), mZipEntry.uncompressed_length); + } + } + + const Source& getSource() const override { + return mSource; + } + +private: + ZipArchiveHandle mZipHandle; + ZipEntry mZipEntry; + Source mSource; +}; + +/** + * An IFileCollection that represents a ZIP archive and the entries within it. + */ +class ZipFileCollection : public IFileCollection { +public: + static std::unique_ptr create(const StringPiece& path, + std::string* outError) { + std::unique_ptr collection = std::unique_ptr( + new ZipFileCollection()); + + int32_t result = OpenArchive(path.data(), &collection->mHandle); + if (result != 0) { + if (outError) *outError = ErrorCodeString(result); + return {}; + } + + ZipString suffix(".flat"); + void* cookie = nullptr; + result = StartIteration(collection->mHandle, &cookie, nullptr, &suffix); + if (result != 0) { + if (outError) *outError = ErrorCodeString(result); + return {}; + } + + using IterationEnder = std::unique_ptr; + IterationEnder iterationEnder(cookie, EndIteration); + + ZipString zipEntryName; + ZipEntry zipData; + while ((result = Next(cookie, &zipData, &zipEntryName)) == 0) { + std::string nestedPath = path.toString(); + nestedPath += "@" + std::string(reinterpret_cast(zipEntryName.name), + zipEntryName.name_length); + collection->mFiles.push_back(util::make_unique(collection->mHandle, + zipData, + Source(nestedPath))); + } + + if (result != -1) { + if (outError) *outError = ErrorCodeString(result); + return {}; + } + return collection; + } + + const_iterator begin() const override { + return mFiles.begin(); + } + + const_iterator end() const override { + return mFiles.end(); + } + + ~ZipFileCollection() override { + if (mHandle) { + CloseArchive(mHandle); + } + } + +private: + ZipFileCollection() : mHandle(nullptr) { + } + + ZipArchiveHandle mHandle; + std::vector> mFiles; +}; + +} // namespace io +} // namespace aapt + +#endif /* AAPT_IO_ZIPARCHIVE_H */ diff --git a/tools/aapt2/link/Link.cpp b/tools/aapt2/link/Link.cpp index 9850ae5cf57b0..33d9272b39fdb 100644 --- a/tools/aapt2/link/Link.cpp +++ b/tools/aapt2/link/Link.cpp @@ -22,6 +22,8 @@ #include "flatten/Archive.h" #include "flatten/TableFlattener.h" #include "flatten/XmlFlattener.h" +#include "io/FileSystem.h" +#include "io/ZipArchive.h" #include "java/JavaClassGenerator.h" #include "java/ManifestClassGenerator.h" #include "java/ProguardRules.h" @@ -39,7 +41,6 @@ #include #include -#include #include namespace aapt { @@ -92,7 +93,15 @@ struct LinkContext : public IAaptContext { class LinkCommand { public: LinkCommand(const LinkOptions& options) : - mOptions(options), mContext(), mFinalTable() { + mOptions(options), mContext(), mFinalTable(), mFileCollection(nullptr) { + std::unique_ptr fileCollection = + util::make_unique(); + + // Get a pointer to the FileCollection for convenience, but it will be owned by the vector. + mFileCollection = fileCollection.get(); + + // Move it to the collection. + mCollections.push_back(std::move(fileCollection)); } std::string buildResourceFileName(const ResourceFile& resFile) { @@ -136,20 +145,9 @@ public: return builder.build(); } - /** - * Loads the resource table (not inside an apk) at the given path. - */ - std::unique_ptr loadTable(const std::string& input) { - std::string errorStr; - Maybe map = file::mmapPath(input, &errorStr); - if (!map) { - mContext.getDiagnostics()->error(DiagMessage(input) << errorStr); - return {}; - } - + std::unique_ptr loadTable(const Source& source, const void* data, size_t len) { std::unique_ptr table = util::make_unique(); - BinaryResourceParser parser(&mContext, table.get(), Source(input), - map.value().getDataPtr(), map.value().getDataLength()); + BinaryResourceParser parser(&mContext, table.get(), source, data, len); if (!parser.parse()) { return {}; } @@ -159,90 +157,79 @@ public: /** * Inflates an XML file from the source path. */ - std::unique_ptr loadXml(const std::string& path) { + static std::unique_ptr loadXml(const std::string& path, IDiagnostics* diag) { std::ifstream fin(path, std::ifstream::binary); if (!fin) { - mContext.getDiagnostics()->error(DiagMessage(path) << strerror(errno)); + diag->error(DiagMessage(path) << strerror(errno)); return {}; } - return xml::inflate(&fin, mContext.getDiagnostics(), Source(path)); + return xml::inflate(&fin, diag, Source(path)); } - /** - * Inflates a binary XML file from the source path. - */ - std::unique_ptr loadBinaryXmlSkipFileExport(const std::string& path) { - // Read header for symbol info and export info. + static std::unique_ptr loadBinaryXmlSkipFileExport( + const Source& source, + const void* data, size_t len, + IDiagnostics* diag) { std::string errorStr; - Maybe maybeF = file::mmapPath(path, &errorStr); - if (!maybeF) { - mContext.getDiagnostics()->error(DiagMessage(path) << errorStr); - return {}; - } - - ssize_t offset = getWrappedDataOffset(maybeF.value().getDataPtr(), - maybeF.value().getDataLength(), &errorStr); + ssize_t offset = getWrappedDataOffset(data, len, &errorStr); if (offset < 0) { - mContext.getDiagnostics()->error(DiagMessage(path) << errorStr); + diag->error(DiagMessage(source) << errorStr); return {}; } std::unique_ptr xmlRes = xml::inflate( - (const uint8_t*) maybeF.value().getDataPtr() + (size_t) offset, - maybeF.value().getDataLength() - offset, - mContext.getDiagnostics(), Source(path)); + reinterpret_cast(data) + static_cast(offset), + len - static_cast(offset), + diag, + source); if (!xmlRes) { return {}; } return xmlRes; } - Maybe loadFileExportHeader(const std::string& path) { - // Read header for symbol info and export info. + static std::unique_ptr loadFileExportHeader(const Source& source, + const void* data, size_t len, + IDiagnostics* diag) { + std::unique_ptr resFile = util::make_unique(); std::string errorStr; - Maybe maybeF = file::mmapPath(path, &errorStr); - if (!maybeF) { - mContext.getDiagnostics()->error(DiagMessage(path) << errorStr); - return {}; - } - - ResourceFile resFile; - ssize_t offset = unwrapFileExportHeader(maybeF.value().getDataPtr(), - maybeF.value().getDataLength(), - &resFile, &errorStr); + ssize_t offset = unwrapFileExportHeader(data, len, resFile.get(), &errorStr); if (offset < 0) { - mContext.getDiagnostics()->error(DiagMessage(path) << errorStr); + diag->error(DiagMessage(source) << errorStr); return {}; } - return std::move(resFile); + return resFile; } - bool copyFileToArchive(const std::string& path, const std::string& outPath, uint32_t flags, + bool copyFileToArchive(io::IFile* file, const std::string& outPath, uint32_t flags, IArchiveWriter* writer) { + std::unique_ptr data = file->openAsData(); + if (!data) { + mContext.getDiagnostics()->error(DiagMessage(file->getSource()) + << "failed to open file"); + return false; + } + std::string errorStr; - Maybe maybeF = file::mmapPath(path, &errorStr); - if (!maybeF) { - mContext.getDiagnostics()->error(DiagMessage(path) << errorStr); - return false; - } - - ssize_t offset = getWrappedDataOffset(maybeF.value().getDataPtr(), - maybeF.value().getDataLength(), - &errorStr); + ssize_t offset = getWrappedDataOffset(data->data(), data->size(), &errorStr); if (offset < 0) { - mContext.getDiagnostics()->error(DiagMessage(path) << errorStr); + mContext.getDiagnostics()->error(DiagMessage(file->getSource()) << errorStr); return false; } - ArchiveEntry* entry = writer->writeEntry(outPath, flags, &maybeF.value(), - offset, maybeF.value().getDataLength() - offset); - if (!entry) { - mContext.getDiagnostics()->error( - DiagMessage(mOptions.outputPath) << "failed to write file " << outPath); - return false; + if (writer->startEntry(outPath, flags)) { + if (writer->writeEntry(reinterpret_cast(data->data()) + offset, + data->size() - static_cast(offset))) { + if (writer->finishEntry()) { + return true; + } + } } - return true; + + mContext.getDiagnostics()->error( + DiagMessage(mOptions.outputPath) << "failed to write file " << outPath); + return false; } Maybe extractAppInfoFromManifest(xml::XmlResource* xmlRes) { @@ -285,9 +272,9 @@ public: std::unique_ptr makeArchiveWriter() { if (mOptions.outputToDirectory) { - return createDirectoryArchiveWriter(mOptions.outputPath); + return createDirectoryArchiveWriter(mContext.getDiagnostics(), mOptions.outputPath); } else { - return createZipFileArchiveWriter(mOptions.outputPath); + return createZipFileArchiveWriter(mContext.getDiagnostics(), mOptions.outputPath); } } @@ -300,13 +287,17 @@ public: return false; } - ArchiveEntry* entry = writer->writeEntry("resources.arsc", ArchiveEntry::kAlign, buffer); - if (!entry) { - mContext.getDiagnostics()->error( - DiagMessage() << "failed to write resources.arsc to archive"); - return false; + if (writer->startEntry("resources.arsc", ArchiveEntry::kAlign)) { + if (writer->writeEntry(buffer)) { + if (writer->finishEntry()) { + return true; + } + } } - return true; + + mContext.getDiagnostics()->error( + DiagMessage() << "failed to write resources.arsc to archive"); + return false; } bool flattenXml(xml::XmlResource* xmlRes, const StringPiece& path, Maybe maxSdkLevel, @@ -320,13 +311,17 @@ public: return false; } - ArchiveEntry* entry = writer->writeEntry(path, ArchiveEntry::kCompress, buffer); - if (!entry) { - mContext.getDiagnostics()->error( - DiagMessage() << "failed to write " << path << " to archive"); - return false; + + if (writer->startEntry(path, ArchiveEntry::kCompress)) { + if (writer->writeEntry(buffer)) { + if (writer->finishEntry()) { + return true; + } + } } - return true; + mContext.getDiagnostics()->error( + DiagMessage() << "failed to write " << path << " to archive"); + return false; } bool writeJavaFile(ResourceTable* table, const StringPiece16& packageNameToGenerate, @@ -412,34 +407,44 @@ public: return true; } - bool mergeResourceTable(const std::string& input, bool override) { + bool mergeResourceTable(io::IFile* file, bool override) { if (mOptions.verbose) { - mContext.getDiagnostics()->note(DiagMessage() << "linking " << input); + mContext.getDiagnostics()->note(DiagMessage() << "linking " << file->getSource()); } - std::unique_ptr table = loadTable(input); + std::unique_ptr data = file->openAsData(); + if (!data) { + mContext.getDiagnostics()->error(DiagMessage(file->getSource()) + << "failed to open file"); + return false; + } + + std::unique_ptr table = loadTable(file->getSource(), data->data(), + data->size()); if (!table) { return false; } - if (!mTableMerger->merge(Source(input), table.get(), override)) { + if (!mTableMerger->merge(file->getSource(), table.get(), override)) { return false; } return true; } - bool mergeCompiledFile(const std::string& input, ResourceFile&& file, bool override) { - if (file.name.package.empty()) { - file.name.package = mContext.getCompilationPackage().toString(); + bool mergeCompiledFile(io::IFile* file, std::unique_ptr fileDesc, bool override) { + // Apply the package name used for this compilation phase if none was specified. + if (fileDesc->name.package.empty()) { + fileDesc->name.package = mContext.getCompilationPackage().toString(); } - ResourceNameRef resName = file.name; - - Maybe mangledName = mContext.getNameMangler()->mangleName(file.name); + // Mangle the name if necessary. + ResourceNameRef resName = fileDesc->name; + Maybe mangledName = mContext.getNameMangler()->mangleName(fileDesc->name); if (mangledName) { resName = mangledName.value(); } + // If we are overriding resources, we supply a custom resolver function. std::function resolver; if (override) { resolver = [](Value* a, Value* b) -> int { @@ -456,14 +461,14 @@ public: } // Add this file to the table. - if (!mFinalTable.addFileReference(resName, file.config, file.source, - util::utf8ToUtf16(buildResourceFileName(file)), + if (!mFinalTable.addFileReference(resName, fileDesc->config, fileDesc->source, + util::utf8ToUtf16(buildResourceFileName(*fileDesc)), resolver, mContext.getDiagnostics())) { return false; } // Add the exports of this file to the table. - for (SourcedResourceName& exportedSymbol : file.exportedSymbols) { + for (SourcedResourceName& exportedSymbol : fileDesc->exportedSymbols) { if (exportedSymbol.name.package.empty()) { exportedSymbol.name.package = mContext.getCompilationPackage().toString(); } @@ -477,32 +482,78 @@ public: } std::unique_ptr id = util::make_unique(); - id->setSource(file.source.withLine(exportedSymbol.line)); + id->setSource(fileDesc->source.withLine(exportedSymbol.line)); bool result = mFinalTable.addResourceAllowMangled(resName, {}, std::move(id), - mContext.getDiagnostics()); + mContext.getDiagnostics()); if (!result) { return false; } } - mFilesToProcess.insert(FileToProcess{ std::move(file), Source(input) }); + // Now add this file for later processing. Once the table is assigned IDs, we can compile + // this file. + mFilesToProcess.insert(FileToProcess{ std::move(fileDesc), file }); return true; } - bool processFile(const std::string& input, bool override) { - if (util::stringEndsWith(input, ".apk")) { - return mergeStaticLibrary(input); - } else if (util::stringEndsWith(input, ".arsc.flat")) { - return mergeResourceTable(input, override); - } else if (Maybe maybeF = loadFileExportHeader(input)) { - return mergeCompiledFile(input, std::move(maybeF.value()), override); + /** + * Creates an io::IFileCollection from the ZIP archive and processes the files within. + */ + bool mergeArchive(const std::string& input, bool override) { + std::string errorStr; + std::unique_ptr collection = io::ZipFileCollection::create( + input, &errorStr); + if (!collection) { + mContext.getDiagnostics()->error(DiagMessage(input) << errorStr); + return false; + } + + bool error = false; + for (const std::unique_ptr& file : *collection) { + if (!processFile(file.get(), override)) { + error = true; + } + } + + // Make sure to move the collection into the set of IFileCollections. + mCollections.push_back(std::move(collection)); + return !error; + } + + bool processFile(const std::string& path, bool override) { + if (util::stringEndsWith(path, ".flata")) { + return mergeArchive(path, override); + } + + io::IFile* file = mFileCollection->insertFile(path); + return processFile(file, override); + } + + bool processFile(io::IFile* file, bool override) { + const Source& src = file->getSource(); + if (util::stringEndsWith(src.path, ".arsc.flat")) { + return mergeResourceTable(file, override); + } else { + // Try opening the file and looking for an Export header. + std::unique_ptr data = file->openAsData(); + if (!data) { + mContext.getDiagnostics()->error(DiagMessage(src) << "failed to open"); + return false; + } + + std::unique_ptr resourceFile = loadFileExportHeader( + src, data->data(), data->size(), mContext.getDiagnostics()); + if (resourceFile) { + return mergeCompiledFile(file, std::move(resourceFile), override); + } } return false; } int run(const std::vector& inputFiles) { // Load the AndroidManifest.xml - std::unique_ptr manifestXml = loadXml(mOptions.manifestPath); + std::unique_ptr manifestXml = loadXml(mOptions.manifestPath, + mContext.getDiagnostics()); if (!manifestXml) { return 1; } @@ -648,20 +699,30 @@ public: } for (const FileToProcess& file : mFilesToProcess) { - if (file.file.name.type != ResourceType::kRaw && - util::stringEndsWith(file.source.path, ".xml.flat")) { + const StringPiece path = file.file->getSource().path; + + if (file.fileExport->name.type != ResourceType::kRaw && + util::stringEndsWith(path, ".xml.flat")) { if (mOptions.verbose) { - mContext.getDiagnostics()->note(DiagMessage() - << "linking " << file.source.path); + mContext.getDiagnostics()->note(DiagMessage() << "linking " << path); + } + + std::unique_ptr data = file.file->openAsData(); + if (!data) { + mContext.getDiagnostics()->error(DiagMessage(file.file->getSource()) + << "failed to open file"); + return 1; } std::unique_ptr xmlRes = loadBinaryXmlSkipFileExport( - file.source.path); + file.file->getSource(), data->data(), data->size(), + mContext.getDiagnostics()); if (!xmlRes) { return 1; } - xmlRes->file = std::move(file.file); + // Move the file description over. + xmlRes->file = std::move(*file.fileExport); XmlReferenceLinker xmlLinker; if (xmlLinker.consume(&mContext, xmlRes.get())) { @@ -689,12 +750,13 @@ public: xmlRes->file.config, sdkLevel)) { xmlRes->file.config.sdkVersion = sdkLevel; - if (!mFinalTable.addFileReference(xmlRes->file.name, - xmlRes->file.config, - xmlRes->file.source, - util::utf8ToUtf16( - buildResourceFileName(xmlRes->file)), - mContext.getDiagnostics())) { + bool added = mFinalTable.addFileReference( + xmlRes->file.name, + xmlRes->file.config, + xmlRes->file.source, + util::utf8ToUtf16(buildResourceFileName(xmlRes->file)), + mContext.getDiagnostics()); + if (!added) { error = true; continue; } @@ -712,11 +774,10 @@ public: } } else { if (mOptions.verbose) { - mContext.getDiagnostics()->note(DiagMessage() << "copying " - << file.source.path); + mContext.getDiagnostics()->note(DiagMessage() << "copying " << path); } - if (!copyFileToArchive(file.source.path, buildResourceFileName(file.file), 0, + if (!copyFileToArchive(file.file, buildResourceFileName(*file.fileExport), 0, archiveWriter.get())) { error = true; } @@ -802,14 +863,18 @@ private: ResourceTable mFinalTable; std::unique_ptr mTableMerger; + io::FileCollection* mFileCollection; + std::vector> mCollections; + struct FileToProcess { - ResourceFile file; - Source source; + std::unique_ptr fileExport; + io::IFile* file; }; struct FileToProcessComparator { bool operator()(const FileToProcess& a, const FileToProcess& b) { - return std::tie(a.file.name, a.file.config) < std::tie(b.file.name, b.file.config); + return std::tie(a.fileExport->name, a.fileExport->config) < + std::tie(b.fileExport->name, b.fileExport->config); } };