Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

startop: Add support for trace duration.

Browse Source 
Test: pytest app_startup/
Bug: 138233470
Change-Id: Ic8e99b369d3385015d7c86624a61bb6d5e8fdc70
This commit is contained in:
Yan Wang
2019-07-26 10:53:45 -07:00
parent 59aab11f09
commit 6ddab32f68
5 changed files with 112 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files

143
startop/scripts/app_startup/app_startup_runner.py
View File

@@ -32,21 +32,25 @@ import itertools
import os
import sys
import tempfile
from datetime import timedelta
from typing import Any, Callable, Iterable, List, NamedTuple, TextIO, Tuple, \
TypeVar, Union, Optional
# local import
DIR = os.path.abspath(os.path.dirname(__file__))
sys.path.append(os.path.dirname(DIR))
import lib.cmd_utils as cmd_utils
import lib.print_utils as print_utils
import iorap.compiler as compiler
from app_startup.run_app_with_prefetch import PrefetchAppRunner
import app_startup.lib.args_utils as args_utils
from app_startup.lib.data_frame import DataFrame
import lib.cmd_utils as cmd_utils
import lib.print_utils as print_utils
from app_startup.lib.perfetto_trace_collector import PerfettoTraceCollector
# The following command line options participate in the combinatorial generation.
# All other arguments have a global effect.
_COMBINATORIAL_OPTIONS = ['package', 'readahead', 'compiler_filter', 'activity']
_COMBINATORIAL_OPTIONS = ['package', 'readahead', 'compiler_filter',
'activity', 'trace_duration']
_TRACING_READAHEADS = ['mlock', 'fadvise']
_FORWARD_OPTIONS = {'loop_count': '--count'}
_RUN_SCRIPT = os.path.join(os.path.dirname(os.path.realpath(__file__)),
@@ -54,9 +58,8 @@ _RUN_SCRIPT = os.path.join(os.path.dirname(os.path.realpath(__file__)),
CollectorPackageInfo = NamedTuple('CollectorPackageInfo',
[('package', str), ('compiler_filter', str)])
_COLLECTOR_SCRIPT = os.path.join(os.path.dirname(os.path.realpath(__file__)),
'../iorap/collector')
_COLLECTOR_TIMEOUT_MULTIPLIER = 10 # take the regular --timeout and multiply
_COMPILER_SCRIPT = os.path.join(os.path.dirname(os.path.dirname(
os.path.realpath(__file__))), 'iorap/compiler.py')
# by 2; systrace starts up slowly.
_UNLOCK_SCREEN_SCRIPT = os.path.join(
@@ -70,11 +73,14 @@ RunCommandArgs = NamedTuple('RunCommandArgs',
('timeout', Optional[int]),
('debug', bool),
('simulate', bool),
('input', Optional[str])])
('input', Optional[str]),
('trace_duration', Optional[timedelta])])
# This must be the only mutable global variable. All other global variables are constants to avoid magic literals.
_debug = False # See -d/--debug flag.
_DEBUG_FORCE = None # Ignore -d/--debug if this is not none.
_PERFETTO_TRACE_DURATION_MS = 5000 # milliseconds
_PERFETTO_TRACE_DURATION = timedelta(milliseconds=_PERFETTO_TRACE_DURATION_MS)
# Type hinting names.
T = TypeVar('T')
@@ -123,26 +129,15 @@ def parse_options(argv: List[str] = None):
optional_named.add_argument('-in', '--inodes', dest='inodes', type=str,
action='store',
help='Path to inodes file (system/extras/pagecache/pagecache.py -d inodes)')
optional_named.add_argument('--compiler-trace-duration-ms',
dest='trace_duration',
type=lambda ms_str: timedelta(milliseconds=int(ms_str)),
action='append',
help='The trace duration (milliseconds) in '
'compilation')
return parser.parse_args(argv)
def make_script_command_with_temp_output(script: str,
args: List[str],
**kwargs) -> Tuple[str, TextIO]:
"""
Create a command to run a script given the args.
Appends --count <loop_count> --output <tmp-file-name>.
Returns a tuple (cmd, tmp_file)
"""
tmp_output_file = tempfile.NamedTemporaryFile(mode='r')
cmd = [script] + args
for key, value in kwargs.items():
cmd += ['--%s' % (key), "%s" % (value)]
if _debug:
cmd += ['--verbose']
cmd = cmd + ["--output", tmp_output_file.name]
return cmd, tmp_output_file
def key_to_cmdline_flag(key: str) -> str:
"""Convert key into a command line flag, e.g. 'foo-bars' -> '--foo-bar' """
if key.endswith("s"):
@@ -163,26 +158,26 @@ def as_run_command(tpl: NamedTuple) -> List[Union[str, Any]]:
args.append(value)
return args
def run_collector_script(collector_info: CollectorPackageInfo,
inodes_path: str,
timeout: int,
simulate: bool) -> Tuple[bool, TextIO]:
"""Run collector to collect prefetching trace. """
# collector_args = ["--package", package_name]
collector_args = as_run_command(collector_info)
# TODO: forward --wait_time for how long systrace runs?
# TODO: forward --trace_buffer_size for size of systrace buffer size?
collector_cmd, collector_tmp_output_file = make_script_command_with_temp_output(
_COLLECTOR_SCRIPT, collector_args, inodes=inodes_path)
def run_perfetto_collector(collector_info: CollectorPackageInfo,
timeout: int,
simulate: bool) -> Tuple[bool, TextIO]:
"""Run collector to collect prefetching trace.
collector_timeout = timeout and _COLLECTOR_TIMEOUT_MULTIPLIER * timeout
(collector_passed, collector_script_output) = \
cmd_utils.execute_arbitrary_command(collector_cmd,
collector_timeout,
shell=False,
simulate=simulate)
Returns:
A tuple of whether the collection succeeds and the generated trace file.
"""
tmp_output_file = tempfile.NamedTemporaryFile()
return collector_passed, collector_tmp_output_file
collector = PerfettoTraceCollector(package=collector_info.package,
activity=None,
compiler_filter=collector_info.compiler_filter,
timeout=timeout,
simulate=simulate,
trace_duration=_PERFETTO_TRACE_DURATION,
save_destination_file_path=tmp_output_file.name)
result = collector.run()
return result is not None, tmp_output_file
def parse_run_script_csv_file(csv_file: TextIO) -> DataFrame:
"""Parse a CSV file full of integers into a DataFrame."""
@@ -216,6 +211,52 @@ def parse_run_script_csv_file(csv_file: TextIO) -> DataFrame:
return DataFrame(d)
def compile_perfetto_trace(inodes_path: str,
perfetto_trace_file: str,
trace_duration: Optional[timedelta]) -> TextIO:
compiler_trace_file = tempfile.NamedTemporaryFile()
argv = [_COMPILER_SCRIPT, '-i', inodes_path, '--perfetto-trace',
perfetto_trace_file, '-o', compiler_trace_file.name]
if trace_duration is not None:
argv += ['--duration', str(int(trace_duration.total_seconds()
* PerfettoTraceCollector.MS_PER_SEC))]
print_utils.debug_print(argv)
compiler.main(argv)
return compiler_trace_file
def execute_run_using_perfetto_trace(collector_info,
run_combos: Iterable[RunCommandArgs],
simulate: bool,
inodes_path: str,
timeout: int) -> DataFrame:
""" Executes run based on perfetto trace. """
passed, perfetto_trace_file = run_perfetto_collector(collector_info,
timeout,
simulate)
if not passed:
raise RuntimeError('Cannot run perfetto collector!')
with perfetto_trace_file:
for combos in run_combos:
if combos.readahead in _TRACING_READAHEADS:
if simulate:
compiler_trace_file = tempfile.NamedTemporaryFile()
else:
compiler_trace_file = compile_perfetto_trace(inodes_path,
perfetto_trace_file.name,
combos.trace_duration)
with compiler_trace_file:
combos = combos._replace(input=compiler_trace_file.name)
print_utils.debug_print(combos)
output = PrefetchAppRunner(**combos._asdict()).run()
else:
print_utils.debug_print(combos)
output = PrefetchAppRunner(**combos._asdict()).run()
yield DataFrame(dict((x, [y]) for x, y in output)) if output else None
def execute_run_combos(
grouped_run_combos: Iterable[Tuple[CollectorPackageInfo, Iterable[RunCommandArgs]]],
simulate: bool,
@@ -228,19 +269,11 @@ def execute_run_combos(
shell=False)
for collector_info, run_combos in grouped_run_combos:
for combos in run_combos:
args = as_run_command(combos)
if combos.readahead in _TRACING_READAHEADS:
passed, collector_tmp_output_file = run_collector_script(collector_info,
inodes_path,
timeout,
simulate)
combos = combos._replace(input=collector_tmp_output_file.name)
print_utils.debug_print(combos)
output = PrefetchAppRunner(**combos._asdict()).run()
yield DataFrame(dict((x, [y]) for x, y in output)) if output else None
yield from execute_run_using_perfetto_trace(collector_info,
run_combos,
simulate,
inodes_path,
timeout)
def gather_results(commands: Iterable[Tuple[DataFrame]],
key_list: List[str], value_list: List[Tuple[str, ...]]):

16
startop/scripts/app_startup/app_startup_runner_test.py
View File

@@ -91,7 +91,8 @@ def default_dict_for_parsed_args(**kwargs):
# Combine it with all of the "optional" parameters' default values.
"""
d = {'compiler_filters': None, 'simulate': False, 'debug': False,
'output': None, 'timeout': 10, 'loop_count': 1, 'inodes': None}
'output': None, 'timeout': 10, 'loop_count': 1, 'inodes': None,
'trace_duration': None}
d.update(kwargs)
return d
@@ -159,19 +160,6 @@ def test_key_to_cmdline_flag():
assert asr.key_to_cmdline_flag("ba_r") == "--ba-r"
assert asr.key_to_cmdline_flag("ba_zs") == "--ba-z"
def test_make_script_command_with_temp_output():
cmd_str, tmp_file = asr.make_script_command_with_temp_output("fake_script",
args=[], count=1)
with tmp_file:
assert cmd_str == ["fake_script", "--count", "1", "--output", tmp_file.name]
cmd_str, tmp_file = asr.make_script_command_with_temp_output("fake_script",
args=['a', 'b'],
count=2)
with tmp_file:
assert cmd_str == ["fake_script", "a", "b", "--count", "2", "--output",
tmp_file.name]
def test_parse_run_script_csv_file():
# empty file -> empty list
f = io.StringIO("")

3
startop/scripts/app_startup/lib/app_runner.py
View File

@@ -51,6 +51,7 @@ class AppRunnerListener(object):
returns:
a string in the format of "<metric>=<value>\n<metric>=<value>\n..."
for further parsing. For example "TotalTime=123\nDisplayedTime=121".
Return an empty string if no metrics need to be parsed further.
"""
pass
@@ -61,7 +62,7 @@ class AppRunner(object):
APP_STARTUP_DIR = os.path.dirname(DIR)
IORAP_COMMON_BASH_SCRIPT = os.path.realpath(os.path.join(DIR,
'../../iorap/common'))
DEFAULT_TIMEOUT = 30
DEFAULT_TIMEOUT = 30 # seconds
def __init__(self,
package: str,

23
startop/scripts/app_startup/lib/perfetto_trace_collector.py
View File

@@ -14,11 +14,11 @@
"""Class to collector perfetto trace."""
import datetime
from datetime import timedelta
import os
import re
import sys
import time
from datetime import timedelta
from typing import Optional, List, Tuple
# global variables
@@ -40,7 +40,9 @@ class PerfettoTraceCollector(AppRunnerListener):
"""
TRACE_FILE_SUFFIX = 'perfetto_trace.pb'
TRACE_DURATION_PROP = 'iorapd.perfetto.trace_duration_ms'
SECONDS_TO_MILLISECONDS = 1000
MS_PER_SEC = 1000
DEFAULT_TRACE_DURATION = timedelta(milliseconds=5000) # 5 seconds
_COLLECTOR_TIMEOUT_MULTIPLIER = 10 # take the regular timeout and multiply
def __init__(self,
package: str,
@@ -48,7 +50,7 @@ class PerfettoTraceCollector(AppRunnerListener):
compiler_filter: Optional[str],
timeout: Optional[int],
simulate: bool,
trace_duration: timedelta = timedelta(milliseconds=5000),
trace_duration: timedelta = DEFAULT_TRACE_DURATION,
save_destination_file_path: Optional[str] = None):
""" Initialize the perfetto trace collector. """
self.app_runner = AppRunner(package,
@@ -89,7 +91,7 @@ class PerfettoTraceCollector(AppRunnerListener):
# Set perfetto trace duration prop to milliseconds.
adb_utils.set_prop(PerfettoTraceCollector.TRACE_DURATION_PROP,
int(self.trace_duration.total_seconds()*
PerfettoTraceCollector.SECONDS_TO_MILLISECONDS))
PerfettoTraceCollector.MS_PER_SEC))
if not iorapd_utils.stop_iorapd():
raise RuntimeError('Cannot stop iorapd!')
@@ -122,7 +124,7 @@ class PerfettoTraceCollector(AppRunnerListener):
manner until all metrics have been found".
Returns:
An empty string.
An empty string because the metric needs no further parsing.
"""
if not self._wait_for_perfetto_trace(pre_launch_timestamp):
raise RuntimeError('Could not save perfetto app trace file!')
@@ -143,14 +145,19 @@ class PerfettoTraceCollector(AppRunnerListener):
format(self._get_remote_path()))
# The pre_launch_timestamp is longer than what the datetime can parse. Trim
# last three digits to make them align.
# last three digits to make them align. For example:
# 2019-07-02 23:20:06.972674825999 -> 2019-07-02 23:20:06.972674825
assert len(pre_launch_timestamp) == len('2019-07-02 23:20:06.972674825')
timestamp = datetime.datetime.strptime(pre_launch_timestamp[:-3],
'%Y-%m-%d %H:%M:%S.%f')
timeout_dt = timestamp + datetime.timedelta(0, self.app_runner.timeout)
# The timeout of perfetto trace is longer than the normal app run timeout.
timeout_dt = self.app_runner.timeout * PerfettoTraceCollector._COLLECTOR_TIMEOUT_MULTIPLIER
timeout_end = timestamp + datetime.timedelta(seconds=timeout_dt)
return logcat_utils.blocking_wait_for_logcat_pattern(timestamp,
pattern,
timeout_dt)
timeout_end)
def _get_remote_path(self):
# For example: android.music%2Fmusic.TopLevelActivity.perfetto_trace.pb

8
startop/scripts/iorap/compiler.py
View File

@@ -31,8 +31,10 @@ import tempfile
from pathlib import Path
from typing import Iterable, Optional, List
from generated.TraceFile_pb2 import *
from lib.inode2filename import Inode2Filename
DIR = os.path.abspath(os.path.dirname(__file__))
sys.path.append(os.path.dirname(DIR))
from iorap.generated.TraceFile_pb2 import *
from iorap.lib.inode2filename import Inode2Filename
parent_dir_name = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
sys.path.append(parent_dir_name)
@@ -305,7 +307,7 @@ def main(argv):
transform_perfetto_trace_to_systrace(options.perfetto_trace_file,
trace_file.name)
return run(sql_db_path,
options.trace_file,
trace_file.name,
options.trace_duration,
options.output_file,
inode_table,