Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

protolog: improve build time of frameworks/base/services

Browse Source 
Improves build time by speeding up the protolog source transformation
tool:
- parallelize the transformation to use all available cores
- remove dependency on lexicographically preserving printer
- disable comment attribution and language level validation

Speeds up the build step from ~30s to ~4s.

Test: atest protologtool-tests
Test: make services.core.protologsrc && edit file in fw/base/services/core && time make services.core.protologsrc
Change-Id: I052e9eb3cc791edf60c555c6fcf59f2e59f0ba68
This commit is contained in:
Adrian Roos
2019-10-17 22:50:08 +02:00
parent e522501adf
commit fbcd32e552
4 changed files with 124 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files

94
tools/protologtool/src/com/android/protolog/tool/ProtoLogTool.kt
View File

@@ -18,11 +18,14 @@ package com.android.protolog.tool
import com.android.protolog.tool.CommandOptions.Companion.USAGE
import com.github.javaparser.ParseProblemException
import com.github.javaparser.ParserConfiguration
import com.github.javaparser.StaticJavaParser
import com.github.javaparser.ast.CompilationUnit
import java.io.File
import java.io.FileInputStream
import java.io.FileOutputStream
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.jar.JarOutputStream
import java.util.zip.ZipEntry
import kotlin.system.exitProcess
@@ -45,30 +48,38 @@ object ProtoLogTool {
val outJar = JarOutputStream(out)
val processor = ProtoLogCallProcessor(command.protoLogClassNameArg,
command.protoLogGroupsClassNameArg, groups)
val transformer = SourceTransformer(command.protoLogImplClassNameArg, processor)
command.javaSourceArgs.forEach { path ->
val file = File(path)
val text = file.readText()
val newPath = path
val outSrc = try {
val code = tryParse(text, path)
if (containsProtoLogText(text, command.protoLogClassNameArg)) {
transformer.processClass(text, newPath, code)
} else {
val executor = newThreadPool()
command.javaSourceArgs.map { path ->
executor.submitCallable {
val transformer = SourceTransformer(command.protoLogImplClassNameArg, processor)
val file = File(path)
val text = file.readText()
val outSrc = try {
val code = tryParse(text, path)
if (containsProtoLogText(text, command.protoLogClassNameArg)) {
transformer.processClass(text, path, code)
} else {
text
}
} catch (ex: ParsingException) {
// If we cannot parse this file, skip it (and log why). Compilation will fail
// in a subsequent build step.
println("\n${ex.message}\n")
text
}
} catch (ex: ParsingException) {
// If we cannot parse this file, skip it (and log why). Compilation will fail
// in a subsequent build step.
println("\n${ex.message}\n")
text
path to outSrc
}
outJar.putNextEntry(ZipEntry(newPath))
}.map { future ->
val (path, outSrc) = future.get()
outJar.putNextEntry(ZipEntry(path))
outJar.write(outSrc.toByteArray())
outJar.closeEntry()
}
executor.shutdown()
outJar.close()
out.close()
}
@@ -92,23 +103,36 @@ object ProtoLogTool {
val processor = ProtoLogCallProcessor(command.protoLogClassNameArg,
command.protoLogGroupsClassNameArg, groups)
val builder = ViewerConfigBuilder(processor)
command.javaSourceArgs.forEach { path ->
val file = File(path)
val text = file.readText()
if (containsProtoLogText(text, command.protoLogClassNameArg)) {
try {
val code = tryParse(text, path)
val pack = if (code.packageDeclaration.isPresent) code.packageDeclaration
.get().nameAsString else ""
val newPath = pack.replace('.', '/') + '/' + file.name
builder.processClass(code, newPath)
} catch (ex: ParsingException) {
// If we cannot parse this file, skip it (and log why). Compilation will fail
// in a subsequent build step.
println("\n${ex.message}\n")
val executor = newThreadPool()
command.javaSourceArgs.map { path ->
executor.submitCallable {
val file = File(path)
val text = file.readText()
if (containsProtoLogText(text, command.protoLogClassNameArg)) {
try {
val code = tryParse(text, path)
val pack = if (code.packageDeclaration.isPresent) code.packageDeclaration
.get().nameAsString else ""
val newPath = pack.replace('.', '/') + '/' + file.name
builder.findLogCalls(code, newPath)
} catch (ex: ParsingException) {
// If we cannot parse this file, skip it (and log why). Compilation will fail
// in a subsequent build step.
println("\n${ex.message}\n")
null
}
} else {
null
}
}
}.forEach { future ->
builder.addLogCalls(future.get() ?: return@forEach)
}
executor.shutdown()
val out = FileOutputStream(command.viewerConfigJsonArg)
out.write(builder.build().toByteArray())
out.close()
@@ -122,6 +146,11 @@ object ProtoLogTool {
@JvmStatic
fun main(args: Array<String>) {
StaticJavaParser.setConfiguration(ParserConfiguration().apply {
setLanguageLevel(ParserConfiguration.LanguageLevel.RAW)
setAttributeComments(false)
})
try {
val command = CommandOptions(args)
when (command.command) {
@@ -138,3 +167,8 @@ object ProtoLogTool {
}
}
}
private fun <T> ExecutorService.submitCallable(f: () -> T) = submit(f)
private fun newThreadPool() = Executors.newFixedThreadPool(
Runtime.getRuntime().availableProcessors())

6
tools/protologtool/src/com/android/protolog/tool/SourceTransformer.kt
View File

@@ -42,7 +42,6 @@ import com.github.javaparser.ast.type.PrimitiveType
import com.github.javaparser.ast.type.Type
import com.github.javaparser.printer.PrettyPrinter
import com.github.javaparser.printer.PrettyPrinterConfiguration
import com.github.javaparser.printer.lexicalpreservation.LexicalPreservingPrinter
class SourceTransformer(
protoLogImplClassName: String,
@@ -135,7 +134,8 @@ class SourceTransformer(
// Inline the new statement.
val printedIfStmt = inlinePrinter.print(ifStmt)
// Append blank lines to preserve line numbering in file (to allow debugging)
val newLines = LexicalPreservingPrinter.print(parentStmt).count { c -> c == '\n' }
val parentRange = parentStmt.range.get()
val newLines = parentRange.end.line - parentRange.begin.line
val newStmt = printedIfStmt.substringBeforeLast('}') + ("\n".repeat(newLines)) + '}'
// pre-workaround code, see explanation below
/*
@@ -224,9 +224,7 @@ class SourceTransformer(
fileName = path
processedCode = code.split('\n').toMutableList()
offsets = IntArray(processedCode.size)
LexicalPreservingPrinter.setup(compilationUnit)
protoLogCallProcessor.process(compilationUnit, this, fileName)
// return LexicalPreservingPrinter.print(compilationUnit)
return processedCode.joinToString("\n")
}
}

55
tools/protologtool/src/com/android/protolog/tool/ViewerConfigBuilder.kt
View File

@@ -23,39 +23,52 @@ import com.github.javaparser.ast.expr.MethodCallExpr
import java.io.StringWriter
class ViewerConfigBuilder(
private val protoLogCallVisitor: ProtoLogCallProcessor
) : ProtoLogCallVisitor {
override fun processCall(
call: MethodCallExpr,
messageString: String,
level: LogLevel,
group: LogGroup
) {
private val processor: ProtoLogCallProcessor
) {
private fun addLogCall(logCall: LogCall, context: ParsingContext) {
val group = logCall.logGroup
val messageString = logCall.messageString
if (group.enabled) {
val position = fileName
val key = CodeUtils.hash(position, messageString, level, group)
val key = logCall.key()
if (statements.containsKey(key)) {
if (statements[key] != LogCall(messageString, level, group, position)) {
if (statements[key] != logCall) {
throw HashCollisionException(
"Please modify the log message \"$messageString\" " +
"or \"${statements[key]}\" - their hashes are equal.",
ParsingContext(fileName, call))
"or \"${statements[key]}\" - their hashes are equal.", context)
}
} else {
groups.add(group)
statements[key] = LogCall(messageString, level, group, position)
call.range.isPresent
statements[key] = logCall
}
}
}
private val statements: MutableMap<Int, LogCall> = mutableMapOf()
private val groups: MutableSet<LogGroup> = mutableSetOf()
private var fileName: String = ""
fun processClass(unit: CompilationUnit, fileName: String) {
this.fileName = fileName
protoLogCallVisitor.process(unit, this, fileName)
fun findLogCalls(unit: CompilationUnit, fileName: String): List<Pair<LogCall, ParsingContext>> {
val calls = mutableListOf<Pair<LogCall, ParsingContext>>()
val visitor = object : ProtoLogCallVisitor {
override fun processCall(
call: MethodCallExpr,
messageString: String,
level: LogLevel,
group: LogGroup
) {
val logCall = LogCall(messageString, level, group, fileName)
val context = ParsingContext(fileName, call)
calls.add(logCall to context)
}
}
processor.process(unit, visitor, fileName)
return calls
}
fun addLogCalls(calls: List<Pair<LogCall, ParsingContext>>) {
calls.forEach { (logCall, context) ->
addLogCall(logCall, context)
}
}
fun build(): String {
@@ -101,5 +114,7 @@ class ViewerConfigBuilder(
val logLevel: LogLevel,
val logGroup: LogGroup,
val position: String
)
) {
fun key() = CodeUtils.hash(position, messageString, logLevel, logGroup)
}
}

82
tools/protologtool/tests/com/android/protolog/tool/ViewerConfigBuilderTest.kt
View File

@@ -17,8 +17,7 @@
package com.android.protolog.tool
import com.android.json.stream.JsonReader
import com.github.javaparser.ast.CompilationUnit
import com.github.javaparser.ast.expr.MethodCallExpr
import com.android.protolog.tool.ViewerConfigBuilder.LogCall
import org.junit.Assert.assertEquals
import org.junit.Test
import org.mockito.Mockito
@@ -34,14 +33,12 @@ class ViewerConfigBuilderTest {
private val GROUP1 = LogGroup("TEST_GROUP", true, true, TAG1)
private val GROUP2 = LogGroup("DEBUG_GROUP", true, true, TAG2)
private val GROUP3 = LogGroup("DEBUG_GROUP", true, true, TAG2)
private val GROUP_DISABLED = LogGroup("DEBUG_GROUP", false, true, TAG2)
private val GROUP_TEXT_DISABLED = LogGroup("DEBUG_GROUP", true, false, TAG2)
private const val PATH = "/tmp/test.java"
}
private val processor: ProtoLogCallProcessor = Mockito.mock(ProtoLogCallProcessor::class.java)
private val configBuilder = ViewerConfigBuilder(processor)
private val dummyCompilationUnit = CompilationUnit()
private fun <T> any(type: Class<T>): T = Mockito.any<T>(type)
private val configBuilder = ViewerConfigBuilder(Mockito.mock(ProtoLogCallProcessor::class.java))
private fun parseConfig(json: String): Map<Int, ViewerConfigParser.ConfigEntry> {
return ViewerConfigParser().parseConfig(JsonReader(StringReader(json)))
@@ -49,22 +46,10 @@ class ViewerConfigBuilderTest {
@Test
fun processClass() {
Mockito.`when`(processor.process(any(CompilationUnit::class.java),
any(ProtoLogCallVisitor::class.java), any(String::class.java)))
.thenAnswer { invocation ->
val visitor = invocation.arguments[1] as ProtoLogCallVisitor
visitor.processCall(MethodCallExpr(), TEST1.messageString, LogLevel.INFO,
GROUP1)
visitor.processCall(MethodCallExpr(), TEST2.messageString, LogLevel.DEBUG,
GROUP2)
visitor.processCall(MethodCallExpr(), TEST3.messageString, LogLevel.ERROR,
GROUP3)
invocation.arguments[0] as CompilationUnit
}
configBuilder.processClass(dummyCompilationUnit, PATH)
configBuilder.addLogCalls(listOf(
LogCall(TEST1.messageString, LogLevel.INFO, GROUP1, PATH),
LogCall(TEST2.messageString, LogLevel.DEBUG, GROUP2, PATH),
LogCall(TEST3.messageString, LogLevel.ERROR, GROUP3, PATH)).withContext())
val parsedConfig = parseConfig(configBuilder.build())
assertEquals(3, parsedConfig.size)
@@ -78,22 +63,10 @@ class ViewerConfigBuilderTest {
@Test
fun processClass_nonUnique() {
Mockito.`when`(processor.process(any(CompilationUnit::class.java),
any(ProtoLogCallVisitor::class.java), any(String::class.java)))
.thenAnswer { invocation ->
val visitor = invocation.arguments[1] as ProtoLogCallVisitor
visitor.processCall(MethodCallExpr(), TEST1.messageString, LogLevel.INFO,
GROUP1)
visitor.processCall(MethodCallExpr(), TEST1.messageString, LogLevel.INFO,
GROUP1)
visitor.processCall(MethodCallExpr(), TEST1.messageString, LogLevel.INFO,
GROUP1)
invocation.arguments[0] as CompilationUnit
}
configBuilder.processClass(dummyCompilationUnit, PATH)
configBuilder.addLogCalls(listOf(
LogCall(TEST1.messageString, LogLevel.INFO, GROUP1, PATH),
LogCall(TEST1.messageString, LogLevel.INFO, GROUP1, PATH),
LogCall(TEST1.messageString, LogLevel.INFO, GROUP1, PATH)).withContext())
val parsedConfig = parseConfig(configBuilder.build())
assertEquals(1, parsedConfig.size)
@@ -103,28 +76,19 @@ class ViewerConfigBuilderTest {
@Test
fun processClass_disabled() {
Mockito.`when`(processor.process(any(CompilationUnit::class.java),
any(ProtoLogCallVisitor::class.java), any(String::class.java)))
.thenAnswer { invocation ->
val visitor = invocation.arguments[1] as ProtoLogCallVisitor
visitor.processCall(MethodCallExpr(), TEST1.messageString, LogLevel.INFO,
GROUP1)
visitor.processCall(MethodCallExpr(), TEST2.messageString, LogLevel.DEBUG,
LogGroup("DEBUG_GROUP", false, true, TAG2))
visitor.processCall(MethodCallExpr(), TEST3.messageString, LogLevel.ERROR,
LogGroup("DEBUG_GROUP", true, false, TAG2))
invocation.arguments[0] as CompilationUnit
}
configBuilder.processClass(dummyCompilationUnit, PATH)
configBuilder.addLogCalls(listOf(
LogCall(TEST1.messageString, LogLevel.INFO, GROUP1, PATH),
LogCall(TEST2.messageString, LogLevel.DEBUG, GROUP_DISABLED, PATH),
LogCall(TEST3.messageString, LogLevel.ERROR, GROUP_TEXT_DISABLED, PATH))
.withContext())
val parsedConfig = parseConfig(configBuilder.build())
assertEquals(2, parsedConfig.size)
assertEquals(TEST1, parsedConfig[CodeUtils.hash(PATH, TEST1.messageString,
LogLevel.INFO, GROUP1)])
assertEquals(TEST3, parsedConfig[CodeUtils.hash(PATH, TEST3.messageString,
LogLevel.ERROR, LogGroup("DEBUG_GROUP", true, false, TAG2))])
assertEquals(TEST1, parsedConfig[CodeUtils.hash(
PATH, TEST1.messageString, LogLevel.INFO, GROUP1)])
assertEquals(TEST3, parsedConfig[CodeUtils.hash(
PATH, TEST3.messageString, LogLevel.ERROR, GROUP_TEXT_DISABLED)])
}
private fun List<LogCall>.withContext() = map { it to ParsingContext() }
}