startop: add compiler type support to host and device switch.
When using device, the python modules for host are not loaded to be compatiable in tradefed. Test: pytest Test: python app_startup_runner.py --package com.google.android.GoogleCamera --readahead fadvise --inodes textcache --output output.txt -lc 1 -d Change-Id: I9fe54045fbabda5f66ebe1cfe6d0a5461fff0640
This commit is contained in:
Yan Wang
@@ -41,11 +41,12 @@ DIR = os.path.abspath(os.path.dirname(__file__))
|
||||
sys.path.append(os.path.dirname(DIR))
|
||||
import lib.cmd_utils as cmd_utils
|
||||
import lib.print_utils as print_utils
|
||||
import iorap.compiler as compiler
|
||||
from app_startup.run_app_with_prefetch import PrefetchAppRunner
|
||||
import app_startup.lib.args_utils as args_utils
|
||||
from app_startup.lib.data_frame import DataFrame
|
||||
from app_startup.lib.perfetto_trace_collector import PerfettoTraceCollector
|
||||
from iorap.compiler import CompilerType
|
||||
import iorap.compiler as compiler
|
||||
|
||||
# The following command line options participate in the combinatorial generation.
|
||||
# All other arguments have a global effect.
|
||||
@@ -58,8 +59,6 @@ _RUN_SCRIPT = os.path.join(os.path.dirname(os.path.realpath(__file__)),
|
||||
|
||||
CollectorPackageInfo = NamedTuple('CollectorPackageInfo',
|
||||
[('package', str), ('compiler_filter', str)])
|
||||
_COMPILER_SCRIPT = os.path.join(os.path.dirname(os.path.dirname(
|
||||
os.path.realpath(__file__))), 'iorap/compiler.py')
|
||||
# by 2; systrace starts up slowly.
|
||||
|
||||
_UNLOCK_SCREEN_SCRIPT = os.path.join(
|
||||
@@ -135,6 +134,10 @@ def parse_options(argv: List[str] = None):
|
||||
action='append',
|
||||
help='The trace duration (milliseconds) in '
|
||||
'compilation')
|
||||
optional_named.add_argument('--compiler-type', dest='compiler_type',
|
||||
type=CompilerType, choices=list(CompilerType),
|
||||
default=CompilerType.DEVICE,
|
||||
help='The type of compiler.')
|
||||
|
||||
return parser.parse_args(argv)
|
||||
|
||||
@@ -211,26 +214,26 @@ def parse_run_script_csv_file(csv_file: TextIO) -> DataFrame:
|
||||
|
||||
return DataFrame(d)
|
||||
|
||||
def compile_perfetto_trace(inodes_path: str,
|
||||
def build_ri_compiler_argv(inodes_path: str,
|
||||
perfetto_trace_file: str,
|
||||
trace_duration: Optional[timedelta]) -> TextIO:
|
||||
compiler_trace_file = tempfile.NamedTemporaryFile()
|
||||
argv = [_COMPILER_SCRIPT, '-i', inodes_path, '--perfetto-trace',
|
||||
perfetto_trace_file, '-o', compiler_trace_file.name]
|
||||
trace_duration: Optional[timedelta]
|
||||
) -> str:
|
||||
argv = ['-i', inodes_path, '--perfetto-trace',
|
||||
perfetto_trace_file]
|
||||
|
||||
if trace_duration is not None:
|
||||
argv += ['--duration', str(int(trace_duration.total_seconds()
|
||||
* PerfettoTraceCollector.MS_PER_SEC))]
|
||||
* PerfettoTraceCollector.MS_PER_SEC))]
|
||||
|
||||
print_utils.debug_print(argv)
|
||||
compiler.main(argv)
|
||||
return compiler_trace_file
|
||||
return argv
|
||||
|
||||
def execute_run_using_perfetto_trace(collector_info,
|
||||
run_combos: Iterable[RunCommandArgs],
|
||||
simulate: bool,
|
||||
inodes_path: str,
|
||||
timeout: int) -> DataFrame:
|
||||
timeout: int,
|
||||
compiler_type: CompilerType) -> DataFrame:
|
||||
""" Executes run based on perfetto trace. """
|
||||
passed, perfetto_trace_file = run_perfetto_collector(collector_info,
|
||||
timeout,
|
||||
@@ -244,9 +247,15 @@ def execute_run_using_perfetto_trace(collector_info,
|
||||
if simulate:
|
||||
compiler_trace_file = tempfile.NamedTemporaryFile()
|
||||
else:
|
||||
compiler_trace_file = compile_perfetto_trace(inodes_path,
|
||||
perfetto_trace_file.name,
|
||||
combos.trace_duration)
|
||||
ri_compiler_argv = build_ri_compiler_argv(inodes_path,
|
||||
perfetto_trace_file.name,
|
||||
combos.trace_duration)
|
||||
compiler_trace_file = compiler.compile(compiler_type,
|
||||
inodes_path,
|
||||
ri_compiler_argv,
|
||||
combos.package,
|
||||
combos.activity)
|
||||
|
||||
with compiler_trace_file:
|
||||
combos = combos._replace(input=compiler_trace_file.name)
|
||||
print_utils.debug_print(combos)
|
||||
@@ -261,7 +270,8 @@ def execute_run_combos(
|
||||
grouped_run_combos: Iterable[Tuple[CollectorPackageInfo, Iterable[RunCommandArgs]]],
|
||||
simulate: bool,
|
||||
inodes_path: str,
|
||||
timeout: int):
|
||||
timeout: int,
|
||||
compiler_type: CompilerType):
|
||||
# nothing will work if the screen isn't unlocked first.
|
||||
cmd_utils.execute_arbitrary_command([_UNLOCK_SCREEN_SCRIPT],
|
||||
timeout,
|
||||
@@ -273,7 +283,8 @@ def execute_run_combos(
|
||||
run_combos,
|
||||
simulate,
|
||||
inodes_path,
|
||||
timeout)
|
||||
timeout,
|
||||
compiler_type)
|
||||
|
||||
def gather_results(commands: Iterable[Tuple[DataFrame]],
|
||||
key_list: List[str], value_list: List[Tuple[str, ...]]):
|
||||
@@ -361,7 +372,8 @@ def main():
|
||||
exec = execute_run_combos(grouped_combos(),
|
||||
opts.simulate,
|
||||
opts.inodes,
|
||||
opts.timeout)
|
||||
opts.timeout,
|
||||
opts.compiler_type)
|
||||
|
||||
results = gather_results(exec, _COMBINATORIAL_OPTIONS, combos())
|
||||
|
||||
|
||||
@@ -92,7 +92,7 @@ def default_dict_for_parsed_args(**kwargs):
|
||||
"""
|
||||
d = {'compiler_filters': None, 'simulate': False, 'debug': False,
|
||||
'output': None, 'timeout': 10, 'loop_count': 1, 'inodes': None,
|
||||
'trace_duration': None}
|
||||
'trace_duration': None, 'compiler_type': asr.CompilerType.HOST}
|
||||
d.update(kwargs)
|
||||
return d
|
||||
|
||||
|
||||
346
startop/scripts/iorap/compiler.py
Executable file → Normal file
346
startop/scripts/iorap/compiler.py
Executable file → Normal file
@@ -1,323 +1,73 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
#
|
||||
# Copyright (C) 2019 The Android Open Source Project
|
||||
# Copyright 2019, The Android Open Source Project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
#
|
||||
# Dependencies:
|
||||
#
|
||||
# $> sudo apt-get install python3-pip
|
||||
# $> pip3 install --user protobuf sqlalchemy sqlite3
|
||||
#
|
||||
|
||||
import optparse
|
||||
import importlib
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from datetime import timedelta
|
||||
from typing import Iterable, Optional, List
|
||||
from enum import Enum
|
||||
from typing import TextIO, List
|
||||
|
||||
# local import
|
||||
DIR = os.path.abspath(os.path.dirname(__file__))
|
||||
sys.path.append(os.path.dirname(DIR))
|
||||
from iorap.generated.TraceFile_pb2 import *
|
||||
from iorap.lib.inode2filename import Inode2Filename
|
||||
import lib.print_utils as print_utils
|
||||
|
||||
parent_dir_name = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
|
||||
sys.path.append(parent_dir_name)
|
||||
from trace_analyzer.lib.trace2db import Trace2Db, MmFilemapAddToPageCache, \
|
||||
RawFtraceEntry
|
||||
import lib.cmd_utils as cmd_utils
|
||||
# Type of compiler.
|
||||
class CompilerType(Enum):
|
||||
HOST = 1 # iorap.cmd.compiler on host
|
||||
DEVICE = 2 # adb shell iorap.cmd.compiler
|
||||
RI = 3 # compiler.py
|
||||
|
||||
_PAGE_SIZE = 4096 # adb shell getconf PAGESIZE ## size of a memory page in bytes.
|
||||
ANDROID_BUILD_TOP = Path(parent_dir_name).parents[3]
|
||||
TRACECONV_BIN = ANDROID_BUILD_TOP.joinpath(
|
||||
'external/perfetto/tools/traceconv')
|
||||
def compile_perfetto_trace_ri(
|
||||
argv: List[str],
|
||||
compiler) -> TextIO:
|
||||
print_utils.debug_print('Compile using RI compiler.')
|
||||
compiler_trace_file = tempfile.NamedTemporaryFile()
|
||||
argv.extend(['-o', compiler_trace_file.name])
|
||||
print_utils.debug_print(argv)
|
||||
compiler.main([''] + argv)
|
||||
return compiler_trace_file
|
||||
|
||||
class PageRun:
|
||||
"""
|
||||
Intermediate representation for a run of one or more pages.
|
||||
"""
|
||||
def __init__(self, device_number: int, inode: int, offset: int, length: int):
|
||||
self.device_number = device_number
|
||||
self.inode = inode
|
||||
self.offset = offset
|
||||
self.length = length
|
||||
def compile_perfetto_trace_device(inodes_path: str,
|
||||
package: str,
|
||||
activity: str,
|
||||
compiler) -> TextIO:
|
||||
print_utils.debug_print('Compile using on-device compiler.')
|
||||
compiler_trace_file = tempfile.NamedTemporaryFile()
|
||||
compiler.main(inodes_path, package, activity, compiler_trace_file.name)
|
||||
return compiler_trace_file
|
||||
|
||||
def __str__(self):
|
||||
return "PageRun(device_number=%d, inode=%d, offset=%d, length=%d)" \
|
||||
%(self.device_number, self.inode, self.offset, self.length)
|
||||
def compile(compiler_type: CompilerType,
|
||||
inodes_path: str,
|
||||
ri_compiler_argv,
|
||||
package: str,
|
||||
activity: str) -> TextIO:
|
||||
if compiler_type == CompilerType.RI:
|
||||
compiler = importlib.import_module('iorap.compiler_ri')
|
||||
compiler_trace_file = compile_perfetto_trace_ri(ri_compiler_argv,
|
||||
compiler)
|
||||
return compiler_trace_file
|
||||
if compiler_type == CompilerType.DEVICE:
|
||||
compiler = importlib.import_module('iorap.compiler_device')
|
||||
compiler_trace_file = compile_perfetto_trace_device(inodes_path,
|
||||
package,
|
||||
activity,
|
||||
compiler)
|
||||
return compiler_trace_file
|
||||
|
||||
def debug_print(msg):
|
||||
#print(msg)
|
||||
pass
|
||||
|
||||
UNDER_LAUNCH = False
|
||||
|
||||
def page_cache_entries_to_runs(page_cache_entries: Iterable[MmFilemapAddToPageCache]):
|
||||
global _PAGE_SIZE
|
||||
|
||||
runs = [
|
||||
PageRun(device_number=pg_entry.dev, inode=pg_entry.ino, offset=pg_entry.ofs,
|
||||
length=_PAGE_SIZE)
|
||||
for pg_entry in page_cache_entries
|
||||
]
|
||||
|
||||
for r in runs:
|
||||
debug_print(r)
|
||||
|
||||
print("Stats: Page runs totaling byte length: %d" %(len(runs) * _PAGE_SIZE))
|
||||
|
||||
return runs
|
||||
|
||||
def optimize_page_runs(page_runs):
|
||||
new_entries = []
|
||||
last_entry = None
|
||||
for pg_entry in page_runs:
|
||||
if last_entry:
|
||||
if pg_entry.device_number == last_entry.device_number and pg_entry.inode == last_entry.inode:
|
||||
# we are dealing with a run for the same exact file as a previous run.
|
||||
if pg_entry.offset == last_entry.offset + last_entry.length:
|
||||
# trivially contiguous entries. merge them together.
|
||||
last_entry.length += pg_entry.length
|
||||
continue
|
||||
# Default: Add the run without merging it to a previous run.
|
||||
last_entry = pg_entry
|
||||
new_entries.append(pg_entry)
|
||||
return new_entries
|
||||
|
||||
def is_filename_matching_filter(file_name, filters=[]):
|
||||
"""
|
||||
Blacklist-style regular expression filters.
|
||||
|
||||
:return: True iff file_name has an RE match in one of the filters.
|
||||
"""
|
||||
for filt in filters:
|
||||
res = re.search(filt, file_name)
|
||||
if res:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def build_protobuf(page_runs, inode2filename, filters=[]):
|
||||
trace_file = TraceFile()
|
||||
trace_file_index = trace_file.index
|
||||
|
||||
file_id_counter = 0
|
||||
file_id_map = {} # filename -> id
|
||||
|
||||
stats_length_total = 0
|
||||
filename_stats = {} # filename -> total size
|
||||
|
||||
skipped_inode_map = {}
|
||||
filtered_entry_map = {} # filename -> count
|
||||
|
||||
for pg_entry in page_runs:
|
||||
fn = inode2filename.resolve(pg_entry.device_number, pg_entry.inode)
|
||||
if not fn:
|
||||
skipped_inode_map[pg_entry.inode] = skipped_inode_map.get(pg_entry.inode, 0) + 1
|
||||
continue
|
||||
|
||||
filename = fn
|
||||
|
||||
if filters and not is_filename_matching_filter(filename, filters):
|
||||
filtered_entry_map[filename] = filtered_entry_map.get(filename, 0) + 1
|
||||
continue
|
||||
|
||||
file_id = file_id_map.get(filename)
|
||||
if not file_id:
|
||||
file_id = file_id_counter
|
||||
file_id_map[filename] = file_id_counter
|
||||
file_id_counter = file_id_counter + 1
|
||||
|
||||
file_index_entry = trace_file_index.entries.add()
|
||||
file_index_entry.id = file_id
|
||||
file_index_entry.file_name = filename
|
||||
|
||||
# already in the file index, add the file entry.
|
||||
file_entry = trace_file.list.entries.add()
|
||||
file_entry.index_id = file_id
|
||||
file_entry.file_length = pg_entry.length
|
||||
stats_length_total += file_entry.file_length
|
||||
file_entry.file_offset = pg_entry.offset
|
||||
|
||||
filename_stats[filename] = filename_stats.get(filename, 0) + file_entry.file_length
|
||||
|
||||
for inode, count in skipped_inode_map.items():
|
||||
print("WARNING: Skip inode %s because it's not in inode map (%d entries)" %(inode, count))
|
||||
|
||||
print("Stats: Sum of lengths %d" %(stats_length_total))
|
||||
|
||||
if filters:
|
||||
print("Filter: %d total files removed." %(len(filtered_entry_map)))
|
||||
|
||||
for fn, count in filtered_entry_map.items():
|
||||
print("Filter: File '%s' removed '%d' entries." %(fn, count))
|
||||
|
||||
for filename, file_size in filename_stats.items():
|
||||
print("%s,%s" %(filename, file_size))
|
||||
|
||||
return trace_file
|
||||
|
||||
def calc_trace_end_time(trace2db: Trace2Db,
|
||||
trace_duration: Optional[timedelta]) -> float:
|
||||
"""
|
||||
Calculates the end time based on the trace duration.
|
||||
The start time is the first receiving mm file map event.
|
||||
The end time is the start time plus the trace duration.
|
||||
All of them are in milliseconds.
|
||||
"""
|
||||
# If the duration is not set, assume all time is acceptable.
|
||||
if trace_duration is None:
|
||||
# float('inf')
|
||||
return RawFtraceEntry.__table__.c.timestamp.type.python_type('inf')
|
||||
|
||||
first_event = trace2db.session.query(MmFilemapAddToPageCache).join(
|
||||
MmFilemapAddToPageCache.raw_ftrace_entry).order_by(
|
||||
RawFtraceEntry.timestamp).first()
|
||||
|
||||
# total_seconds() will return a float number.
|
||||
return first_event.raw_ftrace_entry.timestamp + trace_duration.total_seconds()
|
||||
|
||||
def query_add_to_page_cache(trace2db: Trace2Db, trace_duration: Optional[timedelta]):
|
||||
end_time = calc_trace_end_time(trace2db, trace_duration)
|
||||
# SELECT * FROM tbl ORDER BY id;
|
||||
return trace2db.session.query(MmFilemapAddToPageCache).join(
|
||||
MmFilemapAddToPageCache.raw_ftrace_entry).filter(
|
||||
RawFtraceEntry.timestamp <= end_time).order_by(
|
||||
MmFilemapAddToPageCache.id).all()
|
||||
|
||||
def transform_perfetto_trace_to_systrace(path_to_perfetto_trace: str,
|
||||
path_to_tmp_systrace: str) -> None:
|
||||
""" Transforms the systrace file from perfetto trace. """
|
||||
cmd_utils.run_command_nofail([str(TRACECONV_BIN),
|
||||
'systrace',
|
||||
path_to_perfetto_trace,
|
||||
path_to_tmp_systrace])
|
||||
|
||||
|
||||
def run(sql_db_path:str,
|
||||
trace_file:str,
|
||||
trace_duration:Optional[timedelta],
|
||||
output_file:str,
|
||||
inode_table:str,
|
||||
filter:List[str]) -> int:
|
||||
trace2db = Trace2Db(sql_db_path)
|
||||
# Speed optimization: Skip any entries that aren't mm_filemap_add_to_pagecache.
|
||||
trace2db.set_raw_ftrace_entry_filter(\
|
||||
lambda entry: entry['function'] == 'mm_filemap_add_to_page_cache')
|
||||
# TODO: parse multiple trace files here.
|
||||
parse_count = trace2db.parse_file_into_db(trace_file)
|
||||
|
||||
mm_filemap_add_to_page_cache_rows = query_add_to_page_cache(trace2db,
|
||||
trace_duration)
|
||||
print("DONE. Parsed %d entries into sql db." %(len(mm_filemap_add_to_page_cache_rows)))
|
||||
|
||||
page_runs = page_cache_entries_to_runs(mm_filemap_add_to_page_cache_rows)
|
||||
print("DONE. Converted %d entries" %(len(page_runs)))
|
||||
|
||||
# TODO: flags to select optimizations.
|
||||
optimized_page_runs = optimize_page_runs(page_runs)
|
||||
print("DONE. Optimized down to %d entries" %(len(optimized_page_runs)))
|
||||
|
||||
print("Build protobuf...")
|
||||
trace_file = build_protobuf(optimized_page_runs, inode_table, filter)
|
||||
|
||||
print("Write protobuf to file...")
|
||||
output_file = open(output_file, 'wb')
|
||||
output_file.write(trace_file.SerializeToString())
|
||||
output_file.close()
|
||||
|
||||
print("DONE")
|
||||
|
||||
# TODO: Silent running mode [no output except on error] for build runs.
|
||||
|
||||
return 0
|
||||
|
||||
def main(argv):
|
||||
parser = optparse.OptionParser(usage="Usage: %prog [options]", description="Compile systrace file into TraceFile.pb")
|
||||
parser.add_option('-i', dest='inode_data_file', metavar='FILE',
|
||||
help='Read cached inode data from a file saved earlier with pagecache.py -d')
|
||||
parser.add_option('-t', dest='trace_file', metavar='FILE',
|
||||
help='Path to systrace file (trace.html) that will be parsed')
|
||||
parser.add_option('--perfetto-trace', dest='perfetto_trace_file',
|
||||
metavar='FILE',
|
||||
help='Path to perfetto trace that will be parsed')
|
||||
|
||||
parser.add_option('--db', dest='sql_db', metavar='FILE',
|
||||
help='Path to intermediate sqlite3 database [default: in-memory].')
|
||||
|
||||
parser.add_option('-f', dest='filter', action="append", default=[],
|
||||
help="Add file filter. All file entries not matching one of the filters are discarded.")
|
||||
|
||||
parser.add_option('-l', dest='launch_lock', action="store_true", default=False,
|
||||
help="Exclude all events not inside launch_lock")
|
||||
|
||||
parser.add_option('-o', dest='output_file', metavar='FILE',
|
||||
help='Output protobuf file')
|
||||
|
||||
parser.add_option('--duration', dest='trace_duration', action="store",
|
||||
type=int, help='The duration of trace in milliseconds.')
|
||||
|
||||
options, categories = parser.parse_args(argv[1:])
|
||||
|
||||
# TODO: OptionParser should have some flags to make these mandatory.
|
||||
if not options.inode_data_file:
|
||||
parser.error("-i is required")
|
||||
if not options.trace_file and not options.perfetto_trace_file:
|
||||
parser.error("one of -t or --perfetto-trace is required")
|
||||
if options.trace_file and options.perfetto_trace_file:
|
||||
parser.error("please enter either -t or --perfetto-trace, not both")
|
||||
if not options.output_file:
|
||||
parser.error("-o is required")
|
||||
|
||||
if options.launch_lock:
|
||||
print("INFO: Launch lock flag (-l) enabled; filtering all events not inside launch_lock.")
|
||||
|
||||
inode_table = Inode2Filename.new_from_filename(options.inode_data_file)
|
||||
|
||||
sql_db_path = ":memory:"
|
||||
if options.sql_db:
|
||||
sql_db_path = options.sql_db
|
||||
|
||||
trace_duration = timedelta(milliseconds=options.trace_duration) if \
|
||||
options.trace_duration is not None else None
|
||||
|
||||
# if the input is systrace
|
||||
if options.trace_file:
|
||||
return run(sql_db_path,
|
||||
options.trace_file,
|
||||
trace_duration,
|
||||
options.output_file,
|
||||
inode_table,
|
||||
options.filter)
|
||||
|
||||
# if the input is perfetto trace
|
||||
# TODO python 3.7 switch to using nullcontext
|
||||
with tempfile.NamedTemporaryFile() as trace_file:
|
||||
transform_perfetto_trace_to_systrace(options.perfetto_trace_file,
|
||||
trace_file.name)
|
||||
return run(sql_db_path,
|
||||
trace_file.name,
|
||||
trace_duration,
|
||||
options.output_file,
|
||||
inode_table,
|
||||
options.filter)
|
||||
|
||||
if __name__ == '__main__':
|
||||
print(sys.argv)
|
||||
sys.exit(main(sys.argv))
|
||||
# Should not arrive here.
|
||||
raise ValueError('Unknown compiler type')
|
||||
|
||||
68
startop/scripts/iorap/compiler_device.py
Normal file
68
startop/scripts/iorap/compiler_device.py
Normal file
@@ -0,0 +1,68 @@
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
# Copyright 2019, The Android Open Source Project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
from typing import List
|
||||
|
||||
DIR = os.path.abspath(os.path.dirname(__file__))
|
||||
sys.path.append(os.path.dirname(DIR)) # framework/base/startop/script
|
||||
import lib.print_utils as print_utils
|
||||
import iorap.lib.iorapd_utils as iorapd_utils
|
||||
from app_startup.lib.app_runner import AppRunner
|
||||
|
||||
IORAP_COMMON_BASH_SCRIPT = os.path.join(DIR, 'common')
|
||||
|
||||
def parse_options(argv: List[str] = None):
|
||||
"""Parses command line arguments and returns an argparse Namespace object."""
|
||||
parser = argparse.ArgumentParser(description="Compile perfetto trace file")
|
||||
required_named = parser.add_argument_group('required named arguments')
|
||||
|
||||
required_named.add_argument('-i', dest='inodes', metavar='FILE',
|
||||
help='Read cached inode data from a file saved '
|
||||
'earlier with pagecache.py -d')
|
||||
required_named.add_argument('-p', dest='package',
|
||||
help='Package of the app to be compiled')
|
||||
|
||||
optional_named = parser.add_argument_group('optional named arguments')
|
||||
optional_named.add_argument('-o', dest='output',
|
||||
help='The compiled trace is stored into the output file')
|
||||
optional_named.add_argument('-a', dest='activity',
|
||||
help='Activity of the app to be compiled')
|
||||
optional_named.add_argument('-d', dest='debug', action='store_true'
|
||||
, help='Activity of the app to be compiled')
|
||||
|
||||
return parser.parse_args(argv)
|
||||
|
||||
def main(inodes, package, activity, output, **kwargs) -> int:
|
||||
"""Entries of the program."""
|
||||
if not activity:
|
||||
activity = AppRunner.get_activity(package)
|
||||
|
||||
passed = iorapd_utils.compile_perfetto_trace_on_device(package, activity,
|
||||
inodes)
|
||||
if passed and output:
|
||||
iorapd_utils.get_iorapd_compiler_trace(package, activity, output)
|
||||
|
||||
return 0
|
||||
|
||||
if __name__ == '__main__':
|
||||
opts = parse_options()
|
||||
if opts.debug:
|
||||
print_utils.DEBUG = opts.debug
|
||||
print_utils.debug_print(opts)
|
||||
sys.exit(main(**(vars(opts))))
|
||||
323
startop/scripts/iorap/compiler_ri.py
Executable file
323
startop/scripts/iorap/compiler_ri.py
Executable file
@@ -0,0 +1,323 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
#
|
||||
# Copyright (C) 2019 The Android Open Source Project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
#
|
||||
# Dependencies:
|
||||
#
|
||||
# $> sudo apt-get install python3-pip
|
||||
# $> pip3 install --user protobuf sqlalchemy sqlite3
|
||||
#
|
||||
|
||||
import optparse
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from datetime import timedelta
|
||||
from typing import Iterable, Optional, List
|
||||
|
||||
DIR = os.path.abspath(os.path.dirname(__file__))
|
||||
sys.path.append(os.path.dirname(DIR))
|
||||
from iorap.generated.TraceFile_pb2 import *
|
||||
from iorap.lib.inode2filename import Inode2Filename
|
||||
|
||||
parent_dir_name = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
|
||||
sys.path.append(parent_dir_name)
|
||||
from trace_analyzer.lib.trace2db import Trace2Db, MmFilemapAddToPageCache, \
|
||||
RawFtraceEntry
|
||||
import lib.cmd_utils as cmd_utils
|
||||
|
||||
_PAGE_SIZE = 4096 # adb shell getconf PAGESIZE ## size of a memory page in bytes.
|
||||
ANDROID_BUILD_TOP = Path(parent_dir_name).parents[3]
|
||||
TRACECONV_BIN = ANDROID_BUILD_TOP.joinpath(
|
||||
'external/perfetto/tools/traceconv')
|
||||
|
||||
class PageRun:
|
||||
"""
|
||||
Intermediate representation for a run of one or more pages.
|
||||
"""
|
||||
def __init__(self, device_number: int, inode: int, offset: int, length: int):
|
||||
self.device_number = device_number
|
||||
self.inode = inode
|
||||
self.offset = offset
|
||||
self.length = length
|
||||
|
||||
def __str__(self):
|
||||
return "PageRun(device_number=%d, inode=%d, offset=%d, length=%d)" \
|
||||
%(self.device_number, self.inode, self.offset, self.length)
|
||||
|
||||
def debug_print(msg):
|
||||
#print(msg)
|
||||
pass
|
||||
|
||||
UNDER_LAUNCH = False
|
||||
|
||||
def page_cache_entries_to_runs(page_cache_entries: Iterable[MmFilemapAddToPageCache]):
|
||||
global _PAGE_SIZE
|
||||
|
||||
runs = [
|
||||
PageRun(device_number=pg_entry.dev, inode=pg_entry.ino, offset=pg_entry.ofs,
|
||||
length=_PAGE_SIZE)
|
||||
for pg_entry in page_cache_entries
|
||||
]
|
||||
|
||||
for r in runs:
|
||||
debug_print(r)
|
||||
|
||||
print("Stats: Page runs totaling byte length: %d" %(len(runs) * _PAGE_SIZE))
|
||||
|
||||
return runs
|
||||
|
||||
def optimize_page_runs(page_runs):
|
||||
new_entries = []
|
||||
last_entry = None
|
||||
for pg_entry in page_runs:
|
||||
if last_entry:
|
||||
if pg_entry.device_number == last_entry.device_number and pg_entry.inode == last_entry.inode:
|
||||
# we are dealing with a run for the same exact file as a previous run.
|
||||
if pg_entry.offset == last_entry.offset + last_entry.length:
|
||||
# trivially contiguous entries. merge them together.
|
||||
last_entry.length += pg_entry.length
|
||||
continue
|
||||
# Default: Add the run without merging it to a previous run.
|
||||
last_entry = pg_entry
|
||||
new_entries.append(pg_entry)
|
||||
return new_entries
|
||||
|
||||
def is_filename_matching_filter(file_name, filters=[]):
|
||||
"""
|
||||
Blacklist-style regular expression filters.
|
||||
|
||||
:return: True iff file_name has an RE match in one of the filters.
|
||||
"""
|
||||
for filt in filters:
|
||||
res = re.search(filt, file_name)
|
||||
if res:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def build_protobuf(page_runs, inode2filename, filters=[]):
|
||||
trace_file = TraceFile()
|
||||
trace_file_index = trace_file.index
|
||||
|
||||
file_id_counter = 0
|
||||
file_id_map = {} # filename -> id
|
||||
|
||||
stats_length_total = 0
|
||||
filename_stats = {} # filename -> total size
|
||||
|
||||
skipped_inode_map = {}
|
||||
filtered_entry_map = {} # filename -> count
|
||||
|
||||
for pg_entry in page_runs:
|
||||
fn = inode2filename.resolve(pg_entry.device_number, pg_entry.inode)
|
||||
if not fn:
|
||||
skipped_inode_map[pg_entry.inode] = skipped_inode_map.get(pg_entry.inode, 0) + 1
|
||||
continue
|
||||
|
||||
filename = fn
|
||||
|
||||
if filters and not is_filename_matching_filter(filename, filters):
|
||||
filtered_entry_map[filename] = filtered_entry_map.get(filename, 0) + 1
|
||||
continue
|
||||
|
||||
file_id = file_id_map.get(filename)
|
||||
if not file_id:
|
||||
file_id = file_id_counter
|
||||
file_id_map[filename] = file_id_counter
|
||||
file_id_counter = file_id_counter + 1
|
||||
|
||||
file_index_entry = trace_file_index.entries.add()
|
||||
file_index_entry.id = file_id
|
||||
file_index_entry.file_name = filename
|
||||
|
||||
# already in the file index, add the file entry.
|
||||
file_entry = trace_file.list.entries.add()
|
||||
file_entry.index_id = file_id
|
||||
file_entry.file_length = pg_entry.length
|
||||
stats_length_total += file_entry.file_length
|
||||
file_entry.file_offset = pg_entry.offset
|
||||
|
||||
filename_stats[filename] = filename_stats.get(filename, 0) + file_entry.file_length
|
||||
|
||||
for inode, count in skipped_inode_map.items():
|
||||
print("WARNING: Skip inode %s because it's not in inode map (%d entries)" %(inode, count))
|
||||
|
||||
print("Stats: Sum of lengths %d" %(stats_length_total))
|
||||
|
||||
if filters:
|
||||
print("Filter: %d total files removed." %(len(filtered_entry_map)))
|
||||
|
||||
for fn, count in filtered_entry_map.items():
|
||||
print("Filter: File '%s' removed '%d' entries." %(fn, count))
|
||||
|
||||
for filename, file_size in filename_stats.items():
|
||||
print("%s,%s" %(filename, file_size))
|
||||
|
||||
return trace_file
|
||||
|
||||
def calc_trace_end_time(trace2db: Trace2Db,
|
||||
trace_duration: Optional[timedelta]) -> float:
|
||||
"""
|
||||
Calculates the end time based on the trace duration.
|
||||
The start time is the first receiving mm file map event.
|
||||
The end time is the start time plus the trace duration.
|
||||
All of them are in milliseconds.
|
||||
"""
|
||||
# If the duration is not set, assume all time is acceptable.
|
||||
if trace_duration is None:
|
||||
# float('inf')
|
||||
return RawFtraceEntry.__table__.c.timestamp.type.python_type('inf')
|
||||
|
||||
first_event = trace2db.session.query(MmFilemapAddToPageCache).join(
|
||||
MmFilemapAddToPageCache.raw_ftrace_entry).order_by(
|
||||
RawFtraceEntry.timestamp).first()
|
||||
|
||||
# total_seconds() will return a float number.
|
||||
return first_event.raw_ftrace_entry.timestamp + trace_duration.total_seconds()
|
||||
|
||||
def query_add_to_page_cache(trace2db: Trace2Db, trace_duration: Optional[timedelta]):
|
||||
end_time = calc_trace_end_time(trace2db, trace_duration)
|
||||
# SELECT * FROM tbl ORDER BY id;
|
||||
return trace2db.session.query(MmFilemapAddToPageCache).join(
|
||||
MmFilemapAddToPageCache.raw_ftrace_entry).filter(
|
||||
RawFtraceEntry.timestamp <= end_time).order_by(
|
||||
MmFilemapAddToPageCache.id).all()
|
||||
|
||||
def transform_perfetto_trace_to_systrace(path_to_perfetto_trace: str,
|
||||
path_to_tmp_systrace: str) -> None:
|
||||
""" Transforms the systrace file from perfetto trace. """
|
||||
cmd_utils.run_command_nofail([str(TRACECONV_BIN),
|
||||
'systrace',
|
||||
path_to_perfetto_trace,
|
||||
path_to_tmp_systrace])
|
||||
|
||||
|
||||
def run(sql_db_path:str,
|
||||
trace_file:str,
|
||||
trace_duration:Optional[timedelta],
|
||||
output_file:str,
|
||||
inode_table:str,
|
||||
filter:List[str]) -> int:
|
||||
trace2db = Trace2Db(sql_db_path)
|
||||
# Speed optimization: Skip any entries that aren't mm_filemap_add_to_pagecache.
|
||||
trace2db.set_raw_ftrace_entry_filter(\
|
||||
lambda entry: entry['function'] == 'mm_filemap_add_to_page_cache')
|
||||
# TODO: parse multiple trace files here.
|
||||
parse_count = trace2db.parse_file_into_db(trace_file)
|
||||
|
||||
mm_filemap_add_to_page_cache_rows = query_add_to_page_cache(trace2db,
|
||||
trace_duration)
|
||||
print("DONE. Parsed %d entries into sql db." %(len(mm_filemap_add_to_page_cache_rows)))
|
||||
|
||||
page_runs = page_cache_entries_to_runs(mm_filemap_add_to_page_cache_rows)
|
||||
print("DONE. Converted %d entries" %(len(page_runs)))
|
||||
|
||||
# TODO: flags to select optimizations.
|
||||
optimized_page_runs = optimize_page_runs(page_runs)
|
||||
print("DONE. Optimized down to %d entries" %(len(optimized_page_runs)))
|
||||
|
||||
print("Build protobuf...")
|
||||
trace_file = build_protobuf(optimized_page_runs, inode_table, filter)
|
||||
|
||||
print("Write protobuf to file...")
|
||||
output_file = open(output_file, 'wb')
|
||||
output_file.write(trace_file.SerializeToString())
|
||||
output_file.close()
|
||||
|
||||
print("DONE")
|
||||
|
||||
# TODO: Silent running mode [no output except on error] for build runs.
|
||||
|
||||
return 0
|
||||
|
||||
def main(argv):
|
||||
parser = optparse.OptionParser(usage="Usage: %prog [options]", description="Compile systrace file into TraceFile.pb")
|
||||
parser.add_option('-i', dest='inode_data_file', metavar='FILE',
|
||||
help='Read cached inode data from a file saved earlier with pagecache.py -d')
|
||||
parser.add_option('-t', dest='trace_file', metavar='FILE',
|
||||
help='Path to systrace file (trace.html) that will be parsed')
|
||||
parser.add_option('--perfetto-trace', dest='perfetto_trace_file',
|
||||
metavar='FILE',
|
||||
help='Path to perfetto trace that will be parsed')
|
||||
|
||||
parser.add_option('--db', dest='sql_db', metavar='FILE',
|
||||
help='Path to intermediate sqlite3 database [default: in-memory].')
|
||||
|
||||
parser.add_option('-f', dest='filter', action="append", default=[],
|
||||
help="Add file filter. All file entries not matching one of the filters are discarded.")
|
||||
|
||||
parser.add_option('-l', dest='launch_lock', action="store_true", default=False,
|
||||
help="Exclude all events not inside launch_lock")
|
||||
|
||||
parser.add_option('-o', dest='output_file', metavar='FILE',
|
||||
help='Output protobuf file')
|
||||
|
||||
parser.add_option('--duration', dest='trace_duration', action="store",
|
||||
type=int, help='The duration of trace in milliseconds.')
|
||||
|
||||
options, categories = parser.parse_args(argv[1:])
|
||||
|
||||
# TODO: OptionParser should have some flags to make these mandatory.
|
||||
if not options.inode_data_file:
|
||||
parser.error("-i is required")
|
||||
if not options.trace_file and not options.perfetto_trace_file:
|
||||
parser.error("one of -t or --perfetto-trace is required")
|
||||
if options.trace_file and options.perfetto_trace_file:
|
||||
parser.error("please enter either -t or --perfetto-trace, not both")
|
||||
if not options.output_file:
|
||||
parser.error("-o is required")
|
||||
|
||||
if options.launch_lock:
|
||||
print("INFO: Launch lock flag (-l) enabled; filtering all events not inside launch_lock.")
|
||||
|
||||
inode_table = Inode2Filename.new_from_filename(options.inode_data_file)
|
||||
|
||||
sql_db_path = ":memory:"
|
||||
if options.sql_db:
|
||||
sql_db_path = options.sql_db
|
||||
|
||||
trace_duration = timedelta(milliseconds=options.trace_duration) if \
|
||||
options.trace_duration is not None else None
|
||||
|
||||
# if the input is systrace
|
||||
if options.trace_file:
|
||||
return run(sql_db_path,
|
||||
options.trace_file,
|
||||
trace_duration,
|
||||
options.output_file,
|
||||
inode_table,
|
||||
options.filter)
|
||||
|
||||
# if the input is perfetto trace
|
||||
# TODO python 3.7 switch to using nullcontext
|
||||
with tempfile.NamedTemporaryFile() as trace_file:
|
||||
transform_perfetto_trace_to_systrace(options.perfetto_trace_file,
|
||||
trace_file.name)
|
||||
return run(sql_db_path,
|
||||
trace_file.name,
|
||||
trace_duration,
|
||||
options.output_file,
|
||||
inode_table,
|
||||
options.filter)
|
||||
|
||||
if __name__ == '__main__':
|
||||
print(sys.argv)
|
||||
sys.exit(main(sys.argv))
|
||||
@@ -30,7 +30,7 @@ See also https://docs.pytest.org/en/latest/usage.html
|
||||
"""
|
||||
import os
|
||||
|
||||
import compiler
|
||||
import compiler_host as compiler
|
||||
|
||||
DIR = os.path.abspath(os.path.dirname(__file__))
|
||||
TEXTCACHE = os.path.join(DIR, 'test_fixtures/compiler/common_textcache')
|
||||
|
||||
@@ -18,10 +18,9 @@
|
||||
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# up to two level, like '../../'
|
||||
sys.path.append(Path(os.path.abspath(__file__)).parents[2])
|
||||
# up to two level
|
||||
sys.path.append(os.path.join(os.path.abspath(__file__),'../..'))
|
||||
import lib.cmd_utils as cmd_utils
|
||||
|
||||
IORAPID_LIB_DIR = os.path.abspath(os.path.dirname(__file__))
|
||||
@@ -39,6 +38,22 @@ def _iorapd_path_to_data_file(package: str, activity: str, suffix: str) -> str:
|
||||
# Match logic of 'AppComponentName' in iorap::compiler C++ code.
|
||||
return '{}/{}%2F{}.{}'.format(IORAPD_DATA_PATH, package, activity, suffix)
|
||||
|
||||
def compile_perfetto_trace_on_device(package: str, activity: str,
|
||||
inodes: str) -> bool:
|
||||
"""Compiles the perfetto trace using on-device compiler."""
|
||||
passed, _ = cmd_utils.run_shell_func(IORAP_COMMON_BASH_SCRIPT,
|
||||
'iorapd_compiler_for_app_trace',
|
||||
[package, activity, inodes])
|
||||
return passed
|
||||
|
||||
def get_iorapd_compiler_trace(package: str, activity: str, dest: str) -> str:
|
||||
"""Gets compiler trace to dest file."""
|
||||
src = _iorapd_path_to_data_file(package, activity, 'compiled_trace.pb')
|
||||
passed, _ = cmd_utils.run_shell_command('adb pull "{}" "{}"'.format(src, dest))
|
||||
if not passed:
|
||||
return False
|
||||
return True
|
||||
|
||||
def iorapd_compiler_install_trace_file(package: str, activity: str,
|
||||
input_file: str) -> bool:
|
||||
"""Installs a compiled trace file.
|
||||
|
||||
Reference in New Issue
Block a user