Merge "startop: Add iorap compiler written in python"
This commit is contained in:
TreeHugger Robot
committed by
Android (Google) Code Review
Android (Google) Code Review
commit
7587d2a350
244
startop/scripts/iorap/compiler.py
Executable file
244
startop/scripts/iorap/compiler.py
Executable file
@@ -0,0 +1,244 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
#
|
||||
# Copyright (C) 2019 The Android Open Source Project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
#
|
||||
# Dependencies:
|
||||
#
|
||||
# $> sudo apt-get install python3-pip
|
||||
# $> pip3 install --user protobuf sqlalchemy sqlite3
|
||||
#
|
||||
|
||||
import collections
|
||||
import optparse
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
from typing import Iterable
|
||||
|
||||
from lib.inode2filename import Inode2Filename
|
||||
from generated.TraceFile_pb2 import *
|
||||
|
||||
parent_dir_name = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
|
||||
sys.path.append(parent_dir_name + "/trace_analyzer")
|
||||
from lib.trace2db import Trace2Db, MmFilemapAddToPageCache
|
||||
|
||||
_PAGE_SIZE = 4096 # adb shell getconf PAGESIZE ## size of a memory page in bytes.
|
||||
|
||||
class PageRun:
|
||||
"""
|
||||
Intermediate representation for a run of one or more pages.
|
||||
"""
|
||||
def __init__(self, device_number: int, inode: int, offset: int, length: int):
|
||||
self.device_number = device_number
|
||||
self.inode = inode
|
||||
self.offset = offset
|
||||
self.length = length
|
||||
|
||||
def __str__(self):
|
||||
return "PageRun(device_number=%d, inode=%d, offset=%d, length=%d)" \
|
||||
%(self.device_number, self.inode, self.offset, self.length)
|
||||
|
||||
def debug_print(msg):
|
||||
#print(msg)
|
||||
pass
|
||||
|
||||
UNDER_LAUNCH = False
|
||||
|
||||
def page_cache_entries_to_runs(page_cache_entries: Iterable[MmFilemapAddToPageCache]):
|
||||
global _PAGE_SIZE
|
||||
|
||||
runs = [
|
||||
PageRun(device_number=pg_entry.dev, inode=pg_entry.ino, offset=pg_entry.ofs,
|
||||
length=_PAGE_SIZE)
|
||||
for pg_entry in page_cache_entries
|
||||
]
|
||||
|
||||
for r in runs:
|
||||
debug_print(r)
|
||||
|
||||
print("Stats: Page runs totaling byte length: %d" %(len(runs) * _PAGE_SIZE))
|
||||
|
||||
return runs
|
||||
|
||||
def optimize_page_runs(page_runs):
|
||||
new_entries = []
|
||||
last_entry = None
|
||||
for pg_entry in page_runs:
|
||||
if last_entry:
|
||||
if pg_entry.device_number == last_entry.device_number and pg_entry.inode == last_entry.inode:
|
||||
# we are dealing with a run for the same exact file as a previous run.
|
||||
if pg_entry.offset == last_entry.offset + last_entry.length:
|
||||
# trivially contiguous entries. merge them together.
|
||||
last_entry.length += pg_entry.length
|
||||
continue
|
||||
# Default: Add the run without merging it to a previous run.
|
||||
last_entry = pg_entry
|
||||
new_entries.append(pg_entry)
|
||||
return new_entries
|
||||
|
||||
def is_filename_matching_filter(file_name, filters=[]):
|
||||
"""
|
||||
Blacklist-style regular expression filters.
|
||||
|
||||
:return: True iff file_name has an RE match in one of the filters.
|
||||
"""
|
||||
for filt in filters:
|
||||
res = re.search(filt, file_name)
|
||||
if res:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def build_protobuf(page_runs, inode2filename, filters=[]):
|
||||
trace_file = TraceFile()
|
||||
trace_file_index = trace_file.index
|
||||
|
||||
file_id_counter = 0
|
||||
file_id_map = {} # filename -> id
|
||||
|
||||
stats_length_total = 0
|
||||
filename_stats = {} # filename -> total size
|
||||
|
||||
skipped_inode_map = {}
|
||||
filtered_entry_map = {} # filename -> count
|
||||
|
||||
for pg_entry in page_runs:
|
||||
fn = inode2filename.resolve(pg_entry.device_number, pg_entry.inode)
|
||||
if not fn:
|
||||
skipped_inode_map[pg_entry.inode] = skipped_inode_map.get(pg_entry.inode, 0) + 1
|
||||
continue
|
||||
|
||||
filename = fn
|
||||
|
||||
if filters and not is_filename_matching_filter(filename, filters):
|
||||
filtered_entry_map[filename] = filtered_entry_map.get(filename, 0) + 1
|
||||
continue
|
||||
|
||||
file_id = file_id_map.get(filename)
|
||||
if not file_id:
|
||||
file_id = file_id_counter
|
||||
file_id_map[filename] = file_id_counter
|
||||
file_id_counter = file_id_counter + 1
|
||||
|
||||
file_index_entry = trace_file_index.entries.add()
|
||||
file_index_entry.id = file_id
|
||||
file_index_entry.file_name = filename
|
||||
|
||||
# already in the file index, add the file entry.
|
||||
file_entry = trace_file.list.entries.add()
|
||||
file_entry.index_id = file_id
|
||||
file_entry.file_length = pg_entry.length
|
||||
stats_length_total += file_entry.file_length
|
||||
file_entry.file_offset = pg_entry.offset
|
||||
|
||||
filename_stats[filename] = filename_stats.get(filename, 0) + file_entry.file_length
|
||||
|
||||
for inode, count in skipped_inode_map.items():
|
||||
print("WARNING: Skip inode %s because it's not in inode map (%d entries)" %(inode, count))
|
||||
|
||||
print("Stats: Sum of lengths %d" %(stats_length_total))
|
||||
|
||||
if filters:
|
||||
print("Filter: %d total files removed." %(len(filtered_entry_map)))
|
||||
|
||||
for fn, count in filtered_entry_map.items():
|
||||
print("Filter: File '%s' removed '%d' entries." %(fn, count))
|
||||
|
||||
for filename, file_size in filename_stats.items():
|
||||
print("%s,%s" %(filename, file_size))
|
||||
|
||||
return trace_file
|
||||
|
||||
def query_add_to_page_cache(trace2db: Trace2Db):
|
||||
# SELECT * FROM tbl ORDER BY id;
|
||||
return trace2db.session.query(MmFilemapAddToPageCache).order_by(MmFilemapAddToPageCache.id).all()
|
||||
|
||||
def main(argv):
|
||||
parser = optparse.OptionParser(usage="Usage: %prog [options]", description="Compile systrace file into TraceFile.pb")
|
||||
parser.add_option('-i', dest='inode_data_file', metavar='FILE',
|
||||
help='Read cached inode data from a file saved earlier with pagecache.py -d')
|
||||
parser.add_option('-t', dest='trace_file', metavar='FILE',
|
||||
help='Path to systrace file (trace.html) that will be parsed')
|
||||
|
||||
parser.add_option('--db', dest='sql_db', metavar='FILE',
|
||||
help='Path to intermediate sqlite3 database [default: in-memory].')
|
||||
|
||||
parser.add_option('-f', dest='filter', action="append", default=[],
|
||||
help="Add file filter. All file entries not matching one of the filters are discarded.")
|
||||
|
||||
parser.add_option('-l', dest='launch_lock', action="store_true", default=False,
|
||||
help="Exclude all events not inside launch_lock")
|
||||
|
||||
parser.add_option('-o', dest='output_file', metavar='FILE',
|
||||
help='Output protobuf file')
|
||||
|
||||
options, categories = parser.parse_args(argv[1:])
|
||||
|
||||
# TODO: OptionParser should have some flags to make these mandatory.
|
||||
if not options.inode_data_file:
|
||||
parser.error("-i is required")
|
||||
if not options.trace_file:
|
||||
parser.error("-t is required")
|
||||
if not options.output_file:
|
||||
parser.error("-o is required")
|
||||
|
||||
if options.launch_lock:
|
||||
print("INFO: Launch lock flag (-l) enabled; filtering all events not inside launch_lock.")
|
||||
|
||||
|
||||
inode_table = Inode2Filename.new_from_filename(options.inode_data_file)
|
||||
|
||||
trace_file = open(options.trace_file)
|
||||
|
||||
sql_db_path = ":memory:"
|
||||
if options.sql_db:
|
||||
sql_db_path = options.sql_db
|
||||
|
||||
trace2db = Trace2Db(sql_db_path)
|
||||
# Speed optimization: Skip any entries that aren't mm_filemap_add_to_pagecache.
|
||||
trace2db.set_raw_ftrace_entry_filter(\
|
||||
lambda entry: entry['function'] == 'mm_filemap_add_to_page_cache')
|
||||
# TODO: parse multiple trace files here.
|
||||
parse_count = trace2db.parse_file_into_db(options.trace_file)
|
||||
|
||||
mm_filemap_add_to_page_cache_rows = query_add_to_page_cache(trace2db)
|
||||
print("DONE. Parsed %d entries into sql db." %(len(mm_filemap_add_to_page_cache_rows)))
|
||||
|
||||
page_runs = page_cache_entries_to_runs(mm_filemap_add_to_page_cache_rows)
|
||||
print("DONE. Converted %d entries" %(len(page_runs)))
|
||||
|
||||
# TODO: flags to select optimizations.
|
||||
optimized_page_runs = optimize_page_runs(page_runs)
|
||||
print("DONE. Optimized down to %d entries" %(len(optimized_page_runs)))
|
||||
|
||||
print("Build protobuf...")
|
||||
trace_file = build_protobuf(optimized_page_runs, inode_table, options.filter)
|
||||
|
||||
print("Write protobuf to file...")
|
||||
output_file = open(options.output_file, 'wb')
|
||||
output_file.write(trace_file.SerializeToString())
|
||||
output_file.close()
|
||||
|
||||
print("DONE")
|
||||
|
||||
# TODO: Silent running mode [no output except on error] for build runs.
|
||||
|
||||
return 0
|
||||
|
||||
sys.exit(main(sys.argv))
|
||||
259
startop/scripts/iorap/generated/TraceFile_pb2.py
Normal file
259
startop/scripts/iorap/generated/TraceFile_pb2.py
Normal file
@@ -0,0 +1,259 @@
|
||||
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
||||
# source: TraceFile.proto
|
||||
|
||||
import sys
|
||||
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
|
||||
from google.protobuf import descriptor as _descriptor
|
||||
from google.protobuf import message as _message
|
||||
from google.protobuf import reflection as _reflection
|
||||
from google.protobuf import symbol_database as _symbol_database
|
||||
from google.protobuf import descriptor_pb2
|
||||
# @@protoc_insertion_point(imports)
|
||||
|
||||
_sym_db = _symbol_database.Default()
|
||||
|
||||
|
||||
|
||||
|
||||
DESCRIPTOR = _descriptor.FileDescriptor(
|
||||
name='TraceFile.proto',
|
||||
package='iorap.serialize.proto',
|
||||
syntax='proto2',
|
||||
serialized_pb=_b('\n\x0fTraceFile.proto\x12\x15iorap.serialize.proto\"u\n\tTraceFile\x12\x34\n\x05index\x18\x01 \x02(\x0b\x32%.iorap.serialize.proto.TraceFileIndex\x12\x32\n\x04list\x18\x02 \x02(\x0b\x32$.iorap.serialize.proto.TraceFileList\"M\n\x0eTraceFileIndex\x12;\n\x07\x65ntries\x18\x01 \x03(\x0b\x32*.iorap.serialize.proto.TraceFileIndexEntry\"4\n\x13TraceFileIndexEntry\x12\n\n\x02id\x18\x01 \x02(\x03\x12\x11\n\tfile_name\x18\x02 \x02(\t\"G\n\rTraceFileList\x12\x36\n\x07\x65ntries\x18\x01 \x03(\x0b\x32%.iorap.serialize.proto.TraceFileEntry\"L\n\x0eTraceFileEntry\x12\x10\n\x08index_id\x18\x01 \x02(\x03\x12\x13\n\x0b\x66ile_offset\x18\x02 \x02(\x03\x12\x13\n\x0b\x66ile_length\x18\x03 \x02(\x03\x42\x1c\n\x18\x63om.google.android.iorapH\x03')
|
||||
)
|
||||
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
|
||||
|
||||
|
||||
|
||||
|
||||
_TRACEFILE = _descriptor.Descriptor(
|
||||
name='TraceFile',
|
||||
full_name='iorap.serialize.proto.TraceFile',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='index', full_name='iorap.serialize.proto.TraceFile.index', index=0,
|
||||
number=1, type=11, cpp_type=10, label=2,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='list', full_name='iorap.serialize.proto.TraceFile.list', index=1,
|
||||
number=2, type=11, cpp_type=10, label=2,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
],
|
||||
extensions=[
|
||||
],
|
||||
nested_types=[],
|
||||
enum_types=[
|
||||
],
|
||||
options=None,
|
||||
is_extendable=False,
|
||||
syntax='proto2',
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=42,
|
||||
serialized_end=159,
|
||||
)
|
||||
|
||||
|
||||
_TRACEFILEINDEX = _descriptor.Descriptor(
|
||||
name='TraceFileIndex',
|
||||
full_name='iorap.serialize.proto.TraceFileIndex',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='entries', full_name='iorap.serialize.proto.TraceFileIndex.entries', index=0,
|
||||
number=1, type=11, cpp_type=10, label=3,
|
||||
has_default_value=False, default_value=[],
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
],
|
||||
extensions=[
|
||||
],
|
||||
nested_types=[],
|
||||
enum_types=[
|
||||
],
|
||||
options=None,
|
||||
is_extendable=False,
|
||||
syntax='proto2',
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=161,
|
||||
serialized_end=238,
|
||||
)
|
||||
|
||||
|
||||
_TRACEFILEINDEXENTRY = _descriptor.Descriptor(
|
||||
name='TraceFileIndexEntry',
|
||||
full_name='iorap.serialize.proto.TraceFileIndexEntry',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='id', full_name='iorap.serialize.proto.TraceFileIndexEntry.id', index=0,
|
||||
number=1, type=3, cpp_type=2, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='file_name', full_name='iorap.serialize.proto.TraceFileIndexEntry.file_name', index=1,
|
||||
number=2, type=9, cpp_type=9, label=2,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
],
|
||||
extensions=[
|
||||
],
|
||||
nested_types=[],
|
||||
enum_types=[
|
||||
],
|
||||
options=None,
|
||||
is_extendable=False,
|
||||
syntax='proto2',
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=240,
|
||||
serialized_end=292,
|
||||
)
|
||||
|
||||
|
||||
_TRACEFILELIST = _descriptor.Descriptor(
|
||||
name='TraceFileList',
|
||||
full_name='iorap.serialize.proto.TraceFileList',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='entries', full_name='iorap.serialize.proto.TraceFileList.entries', index=0,
|
||||
number=1, type=11, cpp_type=10, label=3,
|
||||
has_default_value=False, default_value=[],
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
],
|
||||
extensions=[
|
||||
],
|
||||
nested_types=[],
|
||||
enum_types=[
|
||||
],
|
||||
options=None,
|
||||
is_extendable=False,
|
||||
syntax='proto2',
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=294,
|
||||
serialized_end=365,
|
||||
)
|
||||
|
||||
|
||||
_TRACEFILEENTRY = _descriptor.Descriptor(
|
||||
name='TraceFileEntry',
|
||||
full_name='iorap.serialize.proto.TraceFileEntry',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='index_id', full_name='iorap.serialize.proto.TraceFileEntry.index_id', index=0,
|
||||
number=1, type=3, cpp_type=2, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='file_offset', full_name='iorap.serialize.proto.TraceFileEntry.file_offset', index=1,
|
||||
number=2, type=3, cpp_type=2, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='file_length', full_name='iorap.serialize.proto.TraceFileEntry.file_length', index=2,
|
||||
number=3, type=3, cpp_type=2, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
],
|
||||
extensions=[
|
||||
],
|
||||
nested_types=[],
|
||||
enum_types=[
|
||||
],
|
||||
options=None,
|
||||
is_extendable=False,
|
||||
syntax='proto2',
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=367,
|
||||
serialized_end=443,
|
||||
)
|
||||
|
||||
_TRACEFILE.fields_by_name['index'].message_type = _TRACEFILEINDEX
|
||||
_TRACEFILE.fields_by_name['list'].message_type = _TRACEFILELIST
|
||||
_TRACEFILEINDEX.fields_by_name['entries'].message_type = _TRACEFILEINDEXENTRY
|
||||
_TRACEFILELIST.fields_by_name['entries'].message_type = _TRACEFILEENTRY
|
||||
DESCRIPTOR.message_types_by_name['TraceFile'] = _TRACEFILE
|
||||
DESCRIPTOR.message_types_by_name['TraceFileIndex'] = _TRACEFILEINDEX
|
||||
DESCRIPTOR.message_types_by_name['TraceFileIndexEntry'] = _TRACEFILEINDEXENTRY
|
||||
DESCRIPTOR.message_types_by_name['TraceFileList'] = _TRACEFILELIST
|
||||
DESCRIPTOR.message_types_by_name['TraceFileEntry'] = _TRACEFILEENTRY
|
||||
|
||||
TraceFile = _reflection.GeneratedProtocolMessageType('TraceFile', (_message.Message,), dict(
|
||||
DESCRIPTOR = _TRACEFILE,
|
||||
__module__ = 'TraceFile_pb2'
|
||||
# @@protoc_insertion_point(class_scope:iorap.serialize.proto.TraceFile)
|
||||
))
|
||||
_sym_db.RegisterMessage(TraceFile)
|
||||
|
||||
TraceFileIndex = _reflection.GeneratedProtocolMessageType('TraceFileIndex', (_message.Message,), dict(
|
||||
DESCRIPTOR = _TRACEFILEINDEX,
|
||||
__module__ = 'TraceFile_pb2'
|
||||
# @@protoc_insertion_point(class_scope:iorap.serialize.proto.TraceFileIndex)
|
||||
))
|
||||
_sym_db.RegisterMessage(TraceFileIndex)
|
||||
|
||||
TraceFileIndexEntry = _reflection.GeneratedProtocolMessageType('TraceFileIndexEntry', (_message.Message,), dict(
|
||||
DESCRIPTOR = _TRACEFILEINDEXENTRY,
|
||||
__module__ = 'TraceFile_pb2'
|
||||
# @@protoc_insertion_point(class_scope:iorap.serialize.proto.TraceFileIndexEntry)
|
||||
))
|
||||
_sym_db.RegisterMessage(TraceFileIndexEntry)
|
||||
|
||||
TraceFileList = _reflection.GeneratedProtocolMessageType('TraceFileList', (_message.Message,), dict(
|
||||
DESCRIPTOR = _TRACEFILELIST,
|
||||
__module__ = 'TraceFile_pb2'
|
||||
# @@protoc_insertion_point(class_scope:iorap.serialize.proto.TraceFileList)
|
||||
))
|
||||
_sym_db.RegisterMessage(TraceFileList)
|
||||
|
||||
TraceFileEntry = _reflection.GeneratedProtocolMessageType('TraceFileEntry', (_message.Message,), dict(
|
||||
DESCRIPTOR = _TRACEFILEENTRY,
|
||||
__module__ = 'TraceFile_pb2'
|
||||
# @@protoc_insertion_point(class_scope:iorap.serialize.proto.TraceFileEntry)
|
||||
))
|
||||
_sym_db.RegisterMessage(TraceFileEntry)
|
||||
|
||||
|
||||
DESCRIPTOR.has_options = True
|
||||
DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\030com.google.android.iorapH\003'))
|
||||
# @@protoc_insertion_point(module_scope)
|
||||
35
startop/scripts/iorap/generated/codegen_protos
Executable file
35
startop/scripts/iorap/generated/codegen_protos
Executable file
@@ -0,0 +1,35 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Copyright 2019, The Android Open Source Project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||
APROTOC="$(which aprotoc)"
|
||||
|
||||
IORAP_SERIALIZE_DIR="${DIR}/../../../../../../system/iorap/src/serialize"
|
||||
IORAP_PROTOS=($IORAP_SERIALIZE_DIR/*.proto)
|
||||
|
||||
if [[ $? -ne 0 ]]; then
|
||||
echo "Fatal: Missing aprotoc. Set APROTOC=... or lunch build/envsetup.sh?" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! [[ -d $IORAP_SERIALIZE_DIR ]]; then
|
||||
echo "Fatal: Directory '$IORAP_SERIALIZE_DIR' does not exist." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# codegen the .py files into the same directory as this script.
|
||||
echo "$APROTOC" --proto_path="$IORAP_SERIALIZE_DIR" --python_out="$DIR" "${IORAP_PROTOS[@]}"
|
||||
"$APROTOC" --proto_path="$IORAP_SERIALIZE_DIR" --python_out="$DIR" "${IORAP_PROTOS[@]}"
|
||||
94
startop/scripts/iorap/lib/inode2filename.py
Normal file
94
startop/scripts/iorap/lib/inode2filename.py
Normal file
@@ -0,0 +1,94 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
#
|
||||
# Copyright (C) 2019 The Android Open Source Project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
from typing import Any, Callable, Dict, Generic, Iterable, List, NamedTuple, TextIO, Tuple, TypeVar, Optional, Union, TextIO
|
||||
|
||||
import re
|
||||
|
||||
class Inode2Filename:
|
||||
"""
|
||||
Parses a text file of the format
|
||||
"uint(dev_t) uint(ino_t) int(file_size) string(filepath)\\n"*
|
||||
|
||||
Lines not matching this format are ignored.
|
||||
"""
|
||||
|
||||
def __init__(self, inode_data_file: TextIO):
|
||||
"""
|
||||
Create an Inode2Filename that reads cached inode from a file saved earlier
|
||||
(e.g. with pagecache.py -d or with inode2filename --format=textcache)
|
||||
|
||||
:param inode_data_file: a file object (e.g. created with open or StringIO).
|
||||
|
||||
Lifetime: inode_data_file is only used during the construction of the object.
|
||||
"""
|
||||
self._inode_table = Inode2Filename.build_inode_lookup_table(inode_data_file)
|
||||
|
||||
@classmethod
|
||||
def new_from_filename(cls, textcache_filename: str) -> 'Inode2Filename':
|
||||
"""
|
||||
Create an Inode2Filename that reads cached inode from a file saved earlier
|
||||
(e.g. with pagecache.py -d or with inode2filename --format=textcache)
|
||||
|
||||
:param textcache_filename: path to textcache
|
||||
"""
|
||||
with open(textcache_filename) as inode_data_file:
|
||||
return cls(inode_data_file)
|
||||
|
||||
@staticmethod
|
||||
def build_inode_lookup_table(inode_data_file: TextIO) -> Dict[Tuple[int, int], Tuple[str, str]]:
|
||||
"""
|
||||
:return: map { (device_int, inode_int) -> (filename_str, size_str) }
|
||||
"""
|
||||
inode2filename = {}
|
||||
for line in inode_data_file:
|
||||
# stat -c "%d %i %s %n
|
||||
# device number, inode number, total size in bytes, file name
|
||||
result = re.match('([0-9]+)d? ([0-9]+) -?([0-9]+) (.*)', line)
|
||||
if result:
|
||||
inode2filename[(int(result.group(1)), int(result.group(2)))] = \
|
||||
(result.group(4), result.group(3))
|
||||
|
||||
return inode2filename
|
||||
|
||||
def resolve(self, dev_t: int, ino_t: int) -> Optional[str]:
|
||||
"""
|
||||
Return a filename (str) from a (dev_t, ino_t) inode pair.
|
||||
|
||||
Returns None if the lookup fails.
|
||||
"""
|
||||
maybe_result = self._inode_table.get((dev_t, ino_t))
|
||||
|
||||
if not maybe_result:
|
||||
return None
|
||||
|
||||
return maybe_result[0] # filename str
|
||||
|
||||
def __len__(self) -> int:
|
||||
"""
|
||||
:return: the number of inode entries parsed from the file.
|
||||
"""
|
||||
return len(self._inode_table)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
"""
|
||||
:return: string representation for debugging/test failures.
|
||||
"""
|
||||
return "Inode2Filename%s" %(repr(self._inode_table))
|
||||
|
||||
# end of class.
|
||||
83
startop/scripts/iorap/lib/inode2filename_test.py
Executable file
83
startop/scripts/iorap/lib/inode2filename_test.py
Executable file
@@ -0,0 +1,83 @@
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
# Copyright 2019, The Android Open Source Project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
"""
|
||||
Unit tests for inode2filename module.
|
||||
|
||||
Install:
|
||||
$> sudo apt-get install python3-pytest ## OR
|
||||
$> pip install -U pytest
|
||||
See also https://docs.pytest.org/en/latest/getting-started.html
|
||||
|
||||
Usage:
|
||||
$> ./inode2filename_test.py
|
||||
$> pytest inode2filename_test.py
|
||||
$> python -m pytest inode2filename_test.py
|
||||
|
||||
See also https://docs.pytest.org/en/latest/usage.html
|
||||
"""
|
||||
|
||||
# global imports
|
||||
from contextlib import contextmanager
|
||||
import io
|
||||
import shlex
|
||||
import sys
|
||||
import typing
|
||||
|
||||
# pip imports
|
||||
import pytest
|
||||
|
||||
# local imports
|
||||
from inode2filename import *
|
||||
|
||||
def create_inode2filename(*contents):
|
||||
buf = io.StringIO()
|
||||
|
||||
for c in contents:
|
||||
buf.write(c)
|
||||
buf.write("\n")
|
||||
|
||||
buf.seek(0)
|
||||
|
||||
i2f = Inode2Filename(buf)
|
||||
|
||||
buf.close()
|
||||
|
||||
return i2f
|
||||
|
||||
def test_inode2filename():
|
||||
a = create_inode2filename("")
|
||||
assert len(a) == 0
|
||||
assert a.resolve(1, 2) == None
|
||||
|
||||
a = create_inode2filename("1 2 3 foo.bar")
|
||||
assert len(a) == 1
|
||||
assert a.resolve(1, 2) == "foo.bar"
|
||||
assert a.resolve(4, 5) == None
|
||||
|
||||
a = create_inode2filename("1 2 3 foo.bar", "4 5 6 bar.baz")
|
||||
assert len(a) == 2
|
||||
assert a.resolve(1, 2) == "foo.bar"
|
||||
assert a.resolve(4, 5) == "bar.baz"
|
||||
|
||||
a = create_inode2filename("1567d 8910 -1 /a/b/c/", "4 5 6 bar.baz")
|
||||
assert len(a) == 2
|
||||
assert a.resolve(1567, 8910) == "/a/b/c/"
|
||||
assert a.resolve(4, 5) == "bar.baz"
|
||||
|
||||
if __name__ == '__main__':
|
||||
pytest.main()
|
||||
343
startop/scripts/trace_analyzer/lib/trace2db.py
Normal file
343
startop/scripts/trace_analyzer/lib/trace2db.py
Normal file
@@ -0,0 +1,343 @@
|
||||
#!/usr/bin/python3
|
||||
# Copyright (C) 2019 The Android Open Source Project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import re
|
||||
import sys
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy import Column, Date, Integer, Float, String, ForeignKey
|
||||
from sqlalchemy.ext.declarative import declarative_base
|
||||
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
import sqlalchemy
|
||||
|
||||
from typing import Optional, Tuple
|
||||
|
||||
_DEBUG = False # print sql commands to console
|
||||
_FLUSH_LIMIT = 10000 # how many entries are parsed before flushing to DB from memory
|
||||
|
||||
Base = declarative_base()
|
||||
|
||||
class RawFtraceEntry(Base):
|
||||
__tablename__ = 'raw_ftrace_entries'
|
||||
|
||||
id = Column(Integer, primary_key=True)
|
||||
task_name = Column(String, nullable=True) # <...> -> None.
|
||||
task_pid = Column(String, nullable=False)
|
||||
tgid = Column(Integer, nullable=True) # ----- -> None.
|
||||
cpu = Column(Integer, nullable=False)
|
||||
timestamp = Column(Float, nullable=False)
|
||||
function = Column(String, nullable=False)
|
||||
function_args = Column(String, nullable=False)
|
||||
|
||||
@staticmethod
|
||||
def parse_dict(line):
|
||||
# ' <...>-5521 (-----) [003] ...1 17148.446877: tracing_mark_write: trace_event_clock_sync: parent_ts=17148.447266'
|
||||
m = re.match('\s*(.*)-(\d+)\s+\(([^\)]+)\)\s+\[(\d+)\]\s+([\w.]{4})\s+(\d+[.]\d+):\s+(\w+):\s+(.*)', line)
|
||||
if not m:
|
||||
return None
|
||||
|
||||
groups = m.groups()
|
||||
# groups example:
|
||||
# ('<...>',
|
||||
# '5521',
|
||||
# '-----',
|
||||
# '003',
|
||||
# '...1',
|
||||
# '17148.446877',
|
||||
# 'tracing_mark_write',
|
||||
# 'trace_event_clock_sync: parent_ts=17148.447266')
|
||||
task_name = groups[0]
|
||||
if task_name == '<...>':
|
||||
task_name = None
|
||||
|
||||
task_pid = int(groups[1])
|
||||
tgid = groups[2]
|
||||
if tgid == '-----':
|
||||
tgid = None
|
||||
|
||||
cpu = int(groups[3])
|
||||
# irq_flags = groups[4]
|
||||
timestamp = float(groups[5])
|
||||
function = groups[6]
|
||||
function_args = groups[7]
|
||||
|
||||
return {'task_name': task_name, 'task_pid': task_pid, 'tgid': tgid, 'cpu': cpu, 'timestamp': timestamp, 'function': function, 'function_args': function_args}
|
||||
|
||||
class SchedSwitch(Base):
|
||||
__tablename__ = 'sched_switches'
|
||||
|
||||
id = Column(Integer, ForeignKey('raw_ftrace_entries.id'), primary_key=True)
|
||||
|
||||
prev_comm = Column(String, nullable=False)
|
||||
prev_pid = Column(Integer, nullable=False)
|
||||
prev_prio = Column(Integer, nullable=False)
|
||||
prev_state = Column(String, nullable=False)
|
||||
|
||||
next_comm = Column(String, nullable=False)
|
||||
next_pid = Column(Integer, nullable=False)
|
||||
next_prio = Column(Integer, nullable=False)
|
||||
|
||||
@staticmethod
|
||||
def parse_dict(function_args, id = None):
|
||||
# 'prev_comm=kworker/u16:5 prev_pid=13971 prev_prio=120 prev_state=S ==> next_comm=swapper/4 next_pid=0 next_prio=120'
|
||||
m = re.match("prev_comm=(.*) prev_pid=(\d+) prev_prio=(\d+) prev_state=(.*) ==> next_comm=(.*) next_pid=(\d+) next_prio=(\d+) ?", function_args)
|
||||
if not m:
|
||||
return None
|
||||
|
||||
groups = m.groups()
|
||||
# ('kworker/u16:5', '13971', '120', 'S', 'swapper/4', '0', '120')
|
||||
d = {}
|
||||
if id is not None:
|
||||
d['id'] = id
|
||||
d['prev_comm'] = groups[0]
|
||||
d['prev_pid'] = int(groups[1])
|
||||
d['prev_prio'] = int(groups[2])
|
||||
d['prev_state'] = groups[3]
|
||||
d['next_comm'] = groups[4]
|
||||
d['next_pid'] = int(groups[5])
|
||||
d['next_prio'] = int(groups[6])
|
||||
|
||||
return d
|
||||
|
||||
class SchedBlockedReason(Base):
|
||||
__tablename__ = 'sched_blocked_reasons'
|
||||
|
||||
id = Column(Integer, ForeignKey('raw_ftrace_entries.id'), primary_key=True)
|
||||
|
||||
pid = Column(Integer, nullable=False)
|
||||
iowait = Column(Integer, nullable=False)
|
||||
caller = Column(String, nullable=False)
|
||||
|
||||
@staticmethod
|
||||
def parse_dict(function_args, id = None):
|
||||
# 'pid=2289 iowait=1 caller=wait_on_page_bit_common+0x2a8/0x5f'
|
||||
m = re.match("pid=(\d+) iowait=(\d+) caller=(.*) ?", function_args)
|
||||
if not m:
|
||||
return None
|
||||
|
||||
groups = m.groups()
|
||||
# ('2289', '1', 'wait_on_page_bit_common+0x2a8/0x5f8')
|
||||
d = {}
|
||||
if id is not None:
|
||||
d['id'] = id
|
||||
d['pid'] = int(groups[0])
|
||||
d['iowait'] = int(groups[1])
|
||||
d['caller'] = groups[2]
|
||||
|
||||
return d
|
||||
|
||||
class MmFilemapAddToPageCache(Base):
|
||||
__tablename__ = 'mm_filemap_add_to_page_caches'
|
||||
|
||||
id = Column(Integer, ForeignKey('raw_ftrace_entries.id'), primary_key=True)
|
||||
|
||||
dev = Column(Integer, nullable=False) # decoded from ${major}:${minor} syntax.
|
||||
dev_major = Column(Integer, nullable=False) # original ${major} value.
|
||||
dev_minor = Column(Integer, nullable=False) # original ${minor} value.
|
||||
|
||||
ino = Column(Integer, nullable=False) # decoded from hex to base 10
|
||||
page = Column(Integer, nullable=False) # decoded from hex to base 10
|
||||
|
||||
pfn = Column(Integer, nullable=False)
|
||||
ofs = Column(Integer, nullable=False)
|
||||
|
||||
@staticmethod
|
||||
def parse_dict(function_args, id = None):
|
||||
# dev 253:6 ino b2c7 page=00000000ec787cd9 pfn=1478539 ofs=4096
|
||||
m = re.match("dev (\d+):(\d+) ino ([0-9a-fA-F]+) page=([0-9a-fA-F]+) pfn=(\d+) ofs=(\d+)", function_args)
|
||||
if not m:
|
||||
return None
|
||||
|
||||
groups = m.groups()
|
||||
# ('253', '6', 'b2c7', '00000000ec787cd9', '1478539', '4096')
|
||||
d = {}
|
||||
if id is not None:
|
||||
d['id'] = id
|
||||
|
||||
device_major = d['dev_major'] = int(groups[0])
|
||||
device_minor = d['dev_minor'] = int(groups[1])
|
||||
d['dev'] = device_major << 8 | device_minor
|
||||
d['ino'] = int(groups[2], 16)
|
||||
d['page'] = int(groups[3], 16)
|
||||
d['pfn'] = int(groups[4])
|
||||
d['ofs'] = int(groups[5])
|
||||
|
||||
return d
|
||||
|
||||
class Trace2Db:
|
||||
def __init__(self, db_filename: str):
|
||||
(s, e) = self._init_sqlalchemy(db_filename)
|
||||
self._session = s
|
||||
self._engine = e
|
||||
self._raw_ftrace_entry_filter = lambda x: True
|
||||
|
||||
def set_raw_ftrace_entry_filter(self, flt):
|
||||
"""
|
||||
Install a function dict(RawFtraceEntry) -> bool
|
||||
|
||||
If this returns 'false', then we skip adding the RawFtraceEntry to the database.
|
||||
"""
|
||||
self._raw_ftrace_entry_filter = flt
|
||||
|
||||
@staticmethod
|
||||
def _init_sqlalchemy(db_filename: str) -> Tuple[object, object]:
|
||||
global _DEBUG
|
||||
engine = create_engine('sqlite:///' + db_filename, echo=_DEBUG)
|
||||
|
||||
# CREATE ... (tables)
|
||||
Base.metadata.create_all(engine)
|
||||
|
||||
Session = sessionmaker(bind=engine)
|
||||
session = Session()
|
||||
return (session, engine)
|
||||
|
||||
def parse_file_into_db(self, filename: str, limit: Optional[int] = None):
|
||||
"""
|
||||
Parse the ftrace/systrace at 'filename',
|
||||
inserting the values into the current sqlite database.
|
||||
|
||||
:return: number of RawFtraceEntry inserted.
|
||||
"""
|
||||
return parse_file(filename, self._session, self._engine, self._raw_ftrace_entry_filter, limit)
|
||||
|
||||
def parse_file_buf_into_db(self, file_buf, limit: Optional[int] = None):
|
||||
"""
|
||||
Parse the ftrace/systrace at 'filename',
|
||||
inserting the values into the current sqlite database.
|
||||
|
||||
:return: number of RawFtraceEntry inserted.
|
||||
"""
|
||||
return parse_file_buf(file_buf, self._session, self._engine, self._raw_ftrace_entry_filter, limit)
|
||||
|
||||
|
||||
@property
|
||||
def session(self):
|
||||
return self._session
|
||||
|
||||
def insert_pending_entries(engine, kls, lst):
|
||||
if len(lst) > 0:
|
||||
# for some reason, it tries to generate an empty INSERT statement with len=0,
|
||||
# which of course violates the first non-null constraint.
|
||||
try:
|
||||
# Performance-sensitive parsing according to:
|
||||
# https://docs.sqlalchemy.org/en/13/faq/performance.html#i-m-inserting-400-000-rows-with-the-orm-and-it-s-really-slow
|
||||
engine.execute(kls.__table__.insert(), lst)
|
||||
lst.clear()
|
||||
except sqlalchemy.exc.IntegrityError as err:
|
||||
# possibly violating some SQL constraint, print data here.
|
||||
print(err)
|
||||
print(lst)
|
||||
raise
|
||||
|
||||
def parse_file(filename: str, *args, **kwargs) -> int:
|
||||
# use explicit encoding to avoid UnicodeDecodeError.
|
||||
with open(filename, encoding="ISO-8859-1") as f:
|
||||
return parse_file_buf(f, *args, **kwargs)
|
||||
|
||||
def parse_file_buf(filebuf, session, engine, raw_ftrace_entry_filter, limit=None) -> int:
|
||||
global _FLUSH_LIMIT
|
||||
count = 0
|
||||
|
||||
pending_entries = []
|
||||
pending_sched_switch = []
|
||||
pending_sched_blocked_reasons = []
|
||||
pending_mm_filemap_add_to_pagecaches = []
|
||||
|
||||
def insert_all_pending_entries():
|
||||
insert_pending_entries(engine, RawFtraceEntry, pending_entries)
|
||||
insert_pending_entries(engine, SchedSwitch, pending_sched_switch)
|
||||
insert_pending_entries(engine, SchedBlockedReason, pending_sched_blocked_reasons)
|
||||
insert_pending_entries(engine, MmFilemapAddToPageCache, pending_mm_filemap_add_to_pagecaches)
|
||||
|
||||
# for trace.html files produced by systrace,
|
||||
# the actual ftrace is in the 'second' trace-data script class.
|
||||
parsing_trace_data = 0
|
||||
parsing_systrace_file = False
|
||||
|
||||
f = filebuf
|
||||
for l in f:
|
||||
if parsing_trace_data == 0 and l == "<!DOCTYPE html>\n":
|
||||
parsing_systrace_file = True
|
||||
continue
|
||||
if parsing_trace_data != 2 and parsing_systrace_file:
|
||||
if l == ' <script class="trace-data" type="application/text">\n':
|
||||
parsing_trace_data = parsing_trace_data + 1
|
||||
continue
|
||||
|
||||
if parsing_systrace_file and parsing_trace_data != 2:
|
||||
continue
|
||||
elif parsing_systrace_file and parsing_trace_data == 2 and l == " </script>\n":
|
||||
# the rest of this file is just random html
|
||||
break
|
||||
|
||||
# now parsing the ftrace data.
|
||||
if len(l) > 1 and l[0] == '#':
|
||||
continue
|
||||
|
||||
count = count + 1
|
||||
|
||||
if limit and count >= limit:
|
||||
break
|
||||
|
||||
raw_ftrace_entry = RawFtraceEntry.parse_dict(l)
|
||||
if not raw_ftrace_entry:
|
||||
print("WARNING: Failed to parse raw ftrace entry: " + l)
|
||||
continue
|
||||
|
||||
if not raw_ftrace_entry_filter(raw_ftrace_entry):
|
||||
# Skip processing raw ftrace entries that don't match a filter.
|
||||
# This is an optimization for when Trace2Db is used programatically
|
||||
# to avoid having an overly large database.
|
||||
continue
|
||||
|
||||
pending_entries.append(raw_ftrace_entry)
|
||||
|
||||
if raw_ftrace_entry['function'] == 'sched_switch':
|
||||
sched_switch = SchedSwitch.parse_dict(raw_ftrace_entry['function_args'], count)
|
||||
|
||||
if not sched_switch:
|
||||
print("WARNING: Failed to parse sched_switch: " + l)
|
||||
else:
|
||||
pending_sched_switch.append(sched_switch)
|
||||
|
||||
elif raw_ftrace_entry['function'] == 'sched_blocked_reason':
|
||||
sbr = SchedBlockedReason.parse_dict(raw_ftrace_entry['function_args'], count)
|
||||
|
||||
if not sbr:
|
||||
print("WARNING: Failed to parse sched_blocked_reason: " + l)
|
||||
else:
|
||||
pending_sched_blocked_reasons.append(sbr)
|
||||
|
||||
elif raw_ftrace_entry['function'] == 'mm_filemap_add_to_page_cache':
|
||||
d = MmFilemapAddToPageCache.parse_dict(raw_ftrace_entry['function_args'], count)
|
||||
if not d:
|
||||
print("WARNING: Failed to parse mm_filemap_add_to_page_cache: " + l)
|
||||
else:
|
||||
pending_mm_filemap_add_to_pagecaches.append(d)
|
||||
|
||||
# Objects are cached in python memory, not yet sent to SQL database.
|
||||
|
||||
# Send INSERT/UPDATE/etc statements to the underlying SQL database.
|
||||
if count % _FLUSH_LIMIT == 0:
|
||||
insert_all_pending_entries()
|
||||
|
||||
insert_all_pending_entries()
|
||||
|
||||
# Ensure underlying database commits changes from memory to disk.
|
||||
session.commit()
|
||||
|
||||
return count
|
||||
202
startop/scripts/trace_analyzer/lib/trace2db_test.py
Executable file
202
startop/scripts/trace_analyzer/lib/trace2db_test.py
Executable file
@@ -0,0 +1,202 @@
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
# Copyright 2019, The Android Open Source Project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
"""
|
||||
Unit tests for inode2filename module.
|
||||
|
||||
Install:
|
||||
$> sudo apt-get install python3-pytest ## OR
|
||||
$> pip install -U pytest
|
||||
See also https://docs.pytest.org/en/latest/getting-started.html
|
||||
|
||||
Usage:
|
||||
$> ./inode2filename_test.py
|
||||
$> pytest inode2filename_test.py
|
||||
$> python -m pytest inode2filename_test.py
|
||||
|
||||
See also https://docs.pytest.org/en/latest/usage.html
|
||||
"""
|
||||
|
||||
# global imports
|
||||
from contextlib import contextmanager
|
||||
import io
|
||||
import shlex
|
||||
import sys
|
||||
import typing
|
||||
|
||||
from copy import deepcopy
|
||||
|
||||
# pip imports
|
||||
import pytest
|
||||
|
||||
# local imports
|
||||
from trace2db import *
|
||||
|
||||
# This pretty-prints the raw dictionary of the sqlalchemy object if it fails.
|
||||
class EqualsSqlAlchemyObject:
|
||||
# For convenience to write shorter tests, we also add 'ignore_fields' which allow us to specify
|
||||
# which fields to ignore when doing the comparison.
|
||||
def __init__(self_, self, ignore_fields=[]):
|
||||
self_.self = self
|
||||
self_.ignore_fields = ignore_fields
|
||||
|
||||
# Do field-by-field comparison.
|
||||
# It seems that SQLAlchemy does not implement __eq__ itself so we have to do it ourselves.
|
||||
def __eq__(self_, other):
|
||||
if isinstance(other, EqualsSqlAlchemyObject):
|
||||
other = other.self
|
||||
|
||||
self = self_.self
|
||||
|
||||
classes_match = isinstance(other, self.__class__)
|
||||
a, b = deepcopy(self.__dict__), deepcopy(other.__dict__)
|
||||
|
||||
#compare based on equality our attributes, ignoring SQLAlchemy internal stuff
|
||||
|
||||
a.pop('_sa_instance_state', None)
|
||||
b.pop('_sa_instance_state', None)
|
||||
|
||||
for f in self_.ignore_fields:
|
||||
a.pop(f, None)
|
||||
b.pop(f, None)
|
||||
|
||||
attrs_match = (a == b)
|
||||
return classes_match and attrs_match
|
||||
|
||||
def __repr__(self):
|
||||
return repr(self.self.__dict__)
|
||||
|
||||
|
||||
def assert_eq_ignore_id(left, right):
|
||||
# This pretty-prints the raw dictionary of the sqlalchemy object if it fails.
|
||||
# It does field-by-field comparison, but ignores the 'id' field.
|
||||
assert EqualsSqlAlchemyObject(left, ignore_fields=['id']) == EqualsSqlAlchemyObject(right)
|
||||
|
||||
def parse_trace_file_to_db(*contents):
|
||||
"""
|
||||
Make temporary in-memory sqlite3 database by parsing the string contents as a trace.
|
||||
|
||||
:return: Trace2Db instance
|
||||
"""
|
||||
buf = io.StringIO()
|
||||
|
||||
for c in contents:
|
||||
buf.write(c)
|
||||
buf.write("\n")
|
||||
|
||||
buf.seek(0)
|
||||
|
||||
t2d = Trace2Db(":memory:")
|
||||
t2d.parse_file_buf_into_db(buf)
|
||||
|
||||
buf.close()
|
||||
|
||||
return t2d
|
||||
|
||||
def test_ftrace_mm_filemap_add_to_pagecache():
|
||||
test_contents = """
|
||||
MediaStoreImpor-27212 (27176) [000] .... 16136.595194: mm_filemap_add_to_page_cache: dev 253:6 ino 7580 page=0000000060e990c7 pfn=677646 ofs=159744
|
||||
MediaStoreImpor-27212 (27176) [000] .... 16136.595920: mm_filemap_add_to_page_cache: dev 253:6 ino 7580 page=0000000048e2e156 pfn=677645 ofs=126976
|
||||
MediaStoreImpor-27212 (27176) [000] .... 16136.597793: mm_filemap_add_to_page_cache: dev 253:6 ino 7580 page=0000000051eabfb2 pfn=677644 ofs=122880
|
||||
MediaStoreImpor-27212 (27176) [000] .... 16136.597815: mm_filemap_add_to_page_cache: dev 253:6 ino 7580 page=00000000ce7cd606 pfn=677643 ofs=131072
|
||||
MediaStoreImpor-27212 (27176) [000] .... 16136.603732: mm_filemap_add_to_page_cache: dev 253:6 ino 1 page=000000008ffd3030 pfn=730119 ofs=186482688
|
||||
MediaStoreImpor-27212 (27176) [000] .... 16136.604126: mm_filemap_add_to_page_cache: dev 253:6 ino b1d8 page=0000000098d4d2e2 pfn=829676 ofs=0
|
||||
<...>-27197 (-----) [002] .... 16136.613471: mm_filemap_add_to_page_cache: dev 253:6 ino 7580 page=00000000aca88a97 pfn=743346 ofs=241664
|
||||
<...>-27197 (-----) [002] .... 16136.615979: mm_filemap_add_to_page_cache: dev 253:6 ino 7580 page=00000000351f2bc1 pfn=777799 ofs=106496
|
||||
<...>-27224 (-----) [006] .... 16137.400090: mm_filemap_add_to_page_cache: dev 253:6 ino 712d page=000000006ff7ffdb pfn=754861 ofs=0
|
||||
<...>-1396 (-----) [000] .... 16137.451660: mm_filemap_add_to_page_cache: dev 253:6 ino 1 page=00000000ba0cbb34 pfn=769173 ofs=187191296
|
||||
<...>-1396 (-----) [000] .... 16137.453020: mm_filemap_add_to_page_cache: dev 253:6 ino b285 page=00000000f6ef038e pfn=820291 ofs=0
|
||||
<...>-1396 (-----) [000] .... 16137.453067: mm_filemap_add_to_page_cache: dev 253:6 ino b285 page=0000000083ebc446 pfn=956463 ofs=4096
|
||||
<...>-1396 (-----) [000] .... 16137.453101: mm_filemap_add_to_page_cache: dev 253:6 ino b285 page=000000009dc2cd25 pfn=822813 ofs=8192
|
||||
<...>-1396 (-----) [000] .... 16137.453113: mm_filemap_add_to_page_cache: dev 253:6 ino b285 page=00000000a11167fb pfn=928650 ofs=12288
|
||||
<...>-1396 (-----) [000] .... 16137.453126: mm_filemap_add_to_page_cache: dev 253:6 ino b285 page=00000000c1c3311b pfn=621110 ofs=16384
|
||||
<...>-1396 (-----) [000] .... 16137.453139: mm_filemap_add_to_page_cache: dev 253:6 ino b285 page=000000009aa78342 pfn=689370 ofs=20480
|
||||
<...>-1396 (-----) [000] .... 16137.453151: mm_filemap_add_to_page_cache: dev 253:6 ino b285 page=0000000082cddcd6 pfn=755584 ofs=24576
|
||||
<...>-1396 (-----) [000] .... 16137.453162: mm_filemap_add_to_page_cache: dev 253:6 ino b285 page=00000000b0249bc7 pfn=691431 ofs=28672
|
||||
<...>-1396 (-----) [000] .... 16137.453183: mm_filemap_add_to_page_cache: dev 253:6 ino b285 page=000000006a776ff0 pfn=795084 ofs=32768
|
||||
<...>-1396 (-----) [000] .... 16137.453203: mm_filemap_add_to_page_cache: dev 253:6 ino b285 page=000000001a4918a7 pfn=806998 ofs=36864
|
||||
<...>-2578 (-----) [002] .... 16137.561871: mm_filemap_add_to_page_cache: dev 253:6 ino 1 page=00000000d65af9d2 pfn=719246 ofs=187015168
|
||||
<...>-2578 (-----) [002] .... 16137.562846: mm_filemap_add_to_page_cache: dev 253:6 ino b25a page=000000002f6ba74f pfn=864982 ofs=0
|
||||
<...>-2578 (-----) [000] .... 16138.104500: mm_filemap_add_to_page_cache: dev 253:6 ino 1 page=00000000f888d0f6 pfn=805812 ofs=192794624
|
||||
<...>-2578 (-----) [000] .... 16138.105836: mm_filemap_add_to_page_cache: dev 253:6 ino b7dd page=000000003749523b pfn=977196 ofs=0
|
||||
<...>-27215 (-----) [001] .... 16138.256881: mm_filemap_add_to_page_cache: dev 253:6 ino 758f page=000000001b375de1 pfn=755928 ofs=0
|
||||
<...>-27215 (-----) [001] .... 16138.257526: mm_filemap_add_to_page_cache: dev 253:6 ino 7591 page=000000004e039481 pfn=841534 ofs=0
|
||||
NonUserFacing6-5246 ( 1322) [005] .... 16138.356491: mm_filemap_add_to_page_cache: dev 253:6 ino 1 page=00000000d65af9d2 pfn=719246 ofs=161890304
|
||||
NonUserFacing6-5246 ( 1322) [005] .... 16138.357538: mm_filemap_add_to_page_cache: dev 253:6 ino 9a64 page=000000002f6ba74f pfn=864982 ofs=0
|
||||
NonUserFacing6-5246 ( 1322) [005] .... 16138.357581: mm_filemap_add_to_page_cache: dev 253:6 ino 9a64 page=000000006e0f8322 pfn=797894 ofs=4096
|
||||
<...>-27197 (-----) [005] .... 16140.143224: mm_filemap_add_to_page_cache: dev 253:6 ino 7580 page=00000000a42527c6 pfn=1076669 ofs=32768
|
||||
"""
|
||||
|
||||
t2d = parse_trace_file_to_db(test_contents)
|
||||
session = t2d.session
|
||||
|
||||
first_row = session.query(MmFilemapAddToPageCache).order_by(MmFilemapAddToPageCache.id).first()
|
||||
|
||||
#dev 253:6 ino 7580 page=0000000060e990c7 pfn=677646 ofs=159744
|
||||
assert_eq_ignore_id(MmFilemapAddToPageCache(dev=64774, dev_major=253, dev_minor=6,
|
||||
ino=0x7580, page=0x0000000060e990c7, pfn=677646, ofs=159744), first_row)
|
||||
|
||||
second_to_last_row = session.query(MmFilemapAddToPageCache).filter(MmFilemapAddToPageCache.page.in_([0x000000006e0f8322])).first()
|
||||
|
||||
# dev 253:6 ino 9a64 page=000000006e0f8322 pfn=797894 ofs=4096
|
||||
assert_eq_ignore_id(MmFilemapAddToPageCache(dev=64774, dev_major=253, dev_minor=6,
|
||||
ino=0x9a64, page=0x000000006e0f8322, pfn=797894, ofs=4096), second_to_last_row)
|
||||
|
||||
def test_systrace_mm_filemap_add_to_pagecache():
|
||||
test_contents = """
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head i18n-values="dir:textdirection;">
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
|
||||
<meta charset="utf-8"/>
|
||||
<title>Android System Trace</title>
|
||||
<script class="trace-data" type="application/text">
|
||||
PROCESS DUMP
|
||||
USER PID PPID VSZ RSS WCHAN PC S NAME COMM
|
||||
root 1 0 62148 5976 0 0 S init [init]
|
||||
root 2 0 0 0 0 0 S [kthreadd] [kthreadd]
|
||||
</script>
|
||||
|
||||
<script class="trace-data" type="application/text">
|
||||
MediaStoreImpor-27212 (27176) [000] .... 16136.595194: mm_filemap_add_to_page_cache: dev 253:6 ino 7580 page=0000000060e990c7 pfn=677646 ofs=159744
|
||||
NonUserFacing6-5246 ( 1322) [005] .... 16138.357581: mm_filemap_add_to_page_cache: dev 253:6 ino 9a64 page=000000006e0f8322 pfn=797894 ofs=4096
|
||||
</script>
|
||||
|
||||
<script class="trace-data" type="application/text">
|
||||
{"traceEvents": [{"category": "process_argv", "name": "process_argv", "args": {"argv": ["/mnt/ssd3/workspace/master/external/chromium-trace/systrace.py", "-t", "5", "pagecache"]}, "pid": 160383, "ts": 1037300940509.7991, "tid": 139628672526080, "ph": "M"}, {"category": "python", "name": "clock_sync", "args": {"issue_ts": 1037307346185.212, "sync_id": "9a7e4fe3-89ad-441f-8226-8fe533fe973e"}, "pid": 160383, "ts": 1037307351643.906, "tid": 139628726089536, "ph": "c"}], "metadata": {"clock-domain": "SYSTRACE"}}
|
||||
</script>
|
||||
<!-- END TRACE -->
|
||||
"""
|
||||
|
||||
t2d = parse_trace_file_to_db(test_contents)
|
||||
session = t2d.session
|
||||
|
||||
first_row = session.query(MmFilemapAddToPageCache).order_by(MmFilemapAddToPageCache.id).first()
|
||||
|
||||
#dev 253:6 ino 7580 page=0000000060e990c7 pfn=677646 ofs=159744
|
||||
assert_eq_ignore_id(MmFilemapAddToPageCache(dev=64774, dev_major=253, dev_minor=6,
|
||||
ino=0x7580, page=0x0000000060e990c7, pfn=677646, ofs=159744), first_row)
|
||||
|
||||
second_to_last_row = session.query(MmFilemapAddToPageCache).filter(MmFilemapAddToPageCache.page.in_([0x000000006e0f8322])).first()
|
||||
|
||||
# dev 253:6 ino 9a64 page=000000006e0f8322 pfn=797894 ofs=4096
|
||||
assert_eq_ignore_id(MmFilemapAddToPageCache(dev=64774, dev_major=253, dev_minor=6,
|
||||
ino=0x9a64, page=0x000000006e0f8322, pfn=797894, ofs=4096), second_to_last_row)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
pytest.main()
|
||||
@@ -15,335 +15,36 @@
|
||||
|
||||
import re
|
||||
import sys
|
||||
import argparse
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy import Column, Date, Integer, Float, String, ForeignKey
|
||||
from sqlalchemy.ext.declarative import declarative_base
|
||||
from lib.trace2db import Trace2Db
|
||||
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
import sqlalchemy
|
||||
|
||||
_DEBUG = False
|
||||
#_LIMIT = 100000
|
||||
_LIMIT = None
|
||||
_FLUSH_LIMIT = 10000
|
||||
|
||||
Base = declarative_base()
|
||||
|
||||
class RawFtraceEntry(Base):
|
||||
__tablename__ = 'raw_ftrace_entries'
|
||||
|
||||
id = Column(Integer, primary_key=True)
|
||||
task_name = Column(String, nullable=True) # <...> -> None.
|
||||
task_pid = Column(String, nullable=False)
|
||||
tgid = Column(Integer, nullable=True) # ----- -> None.
|
||||
cpu = Column(Integer, nullable=False)
|
||||
timestamp = Column(Float, nullable=False)
|
||||
function = Column(String, nullable=False)
|
||||
function_args = Column(String, nullable=False)
|
||||
|
||||
# __mapper_args__ = {
|
||||
# 'polymorphic_identity':'raw_ftrace_entry',
|
||||
# 'polymorphic_on':function
|
||||
# }
|
||||
|
||||
@staticmethod
|
||||
def parse(line):
|
||||
# ' <...>-5521 (-----) [003] ...1 17148.446877: tracing_mark_write: trace_event_clock_sync: parent_ts=17148.447266'
|
||||
m = re.match('\s*(.*)-(\d+)\s+\(([^\)]+)\)\s+\[(\d+)\]\s+([\w.]{4})\s+(\d+[.]\d+):\s+(\w+):\s+(.*)', line)
|
||||
if not m:
|
||||
return None
|
||||
|
||||
groups = m.groups()
|
||||
# groups example:
|
||||
# ('<...>',
|
||||
# '5521',
|
||||
# '-----',
|
||||
# '003',
|
||||
# '...1',
|
||||
# '17148.446877',
|
||||
# 'tracing_mark_write',
|
||||
# 'trace_event_clock_sync: parent_ts=17148.447266')
|
||||
task_name = groups[0]
|
||||
if task_name == '<...>':
|
||||
task_name = None
|
||||
|
||||
task_pid = int(groups[1])
|
||||
tgid = groups[2]
|
||||
if tgid == '-----':
|
||||
tgid = None
|
||||
|
||||
cpu = int(groups[3])
|
||||
# irq_flags = groups[4]
|
||||
timestamp = float(groups[5])
|
||||
function = groups[6]
|
||||
function_args = groups[7]
|
||||
|
||||
return RawFtraceEntry(task_name=task_name, task_pid=task_pid, tgid=tgid, cpu=cpu,
|
||||
timestamp=timestamp, function=function, function_args=function_args)
|
||||
|
||||
@staticmethod
|
||||
def parse_dict(line):
|
||||
# ' <...>-5521 (-----) [003] ...1 17148.446877: tracing_mark_write: trace_event_clock_sync: parent_ts=17148.447266'
|
||||
m = re.match('\s*(.*)-(\d+)\s+\(([^\)]+)\)\s+\[(\d+)\]\s+([\w.]{4})\s+(\d+[.]\d+):\s+(\w+):\s+(.*)', line)
|
||||
if not m:
|
||||
return None
|
||||
|
||||
groups = m.groups()
|
||||
# groups example:
|
||||
# ('<...>',
|
||||
# '5521',
|
||||
# '-----',
|
||||
# '003',
|
||||
# '...1',
|
||||
# '17148.446877',
|
||||
# 'tracing_mark_write',
|
||||
# 'trace_event_clock_sync: parent_ts=17148.447266')
|
||||
task_name = groups[0]
|
||||
if task_name == '<...>':
|
||||
task_name = None
|
||||
|
||||
task_pid = int(groups[1])
|
||||
tgid = groups[2]
|
||||
if tgid == '-----':
|
||||
tgid = None
|
||||
|
||||
cpu = int(groups[3])
|
||||
# irq_flags = groups[4]
|
||||
timestamp = float(groups[5])
|
||||
function = groups[6]
|
||||
function_args = groups[7]
|
||||
|
||||
return {'task_name': task_name, 'task_pid': task_pid, 'tgid': tgid, 'cpu': cpu, 'timestamp': timestamp, 'function': function, 'function_args': function_args}
|
||||
|
||||
#class TracingMarkWriteFtraceEntry(RawFtraceEntry):
|
||||
# __tablename__ = 'tracing_mark_write_ftrace_entries'
|
||||
# This script requires 'sqlalchemy' to access the sqlite3 database.
|
||||
#
|
||||
# id = Column(Integer, ForeignKey('raw_ftrace_entries.id'), primary_key=True)
|
||||
# mark_type = Column(String(1), nullable=False)
|
||||
# mark_id = Column(Integer, nullable=False)
|
||||
# message = Column(String)
|
||||
# $> sudo apt-get install python3-pip
|
||||
# $> pip3 install --user sqlalchemy
|
||||
#
|
||||
## __mapper_args__ = {
|
||||
## 'polymorphic_identity':'tracing_mark_write',
|
||||
## }
|
||||
#
|
||||
# @staticmethod
|
||||
# def decode(raw_ftrace_entry):
|
||||
# if raw_ftrace_entry.function != 'tracing_mark_write':
|
||||
# raise ValueError("raw_ftrace_entry must be function 'tracing_mark_write':" + raw_ftrace_entry)
|
||||
#
|
||||
# #"B|2446|(Paused)ClearCards|Foo"
|
||||
# match = re.match("([^|]*)\|([^|]*)\|(.*)", raw_ftrace_entry.function_args)
|
||||
#
|
||||
# if not match:
|
||||
# return None
|
||||
#
|
||||
# # ('B', '2446', '(Paused)ClearCards|Foo')
|
||||
# groups = match.groups()
|
||||
#
|
||||
# mark_type = groups[0]
|
||||
# mark_id = int(groups[1])
|
||||
# message = groups[2]
|
||||
#
|
||||
# return TracingMarkWriteFtraceEntry(id = raw_ftrace_entry.id,
|
||||
# mark_type = mark_type,
|
||||
# mark_id = mark_id,
|
||||
# message = message)
|
||||
|
||||
class SchedSwitch(Base):
|
||||
__tablename__ = 'sched_switches'
|
||||
|
||||
id = Column(Integer, ForeignKey('raw_ftrace_entries.id'), primary_key=True)
|
||||
|
||||
prev_comm = Column(String, nullable=False)
|
||||
prev_pid = Column(Integer, nullable=False)
|
||||
prev_prio = Column(Integer, nullable=False)
|
||||
prev_state = Column(String, nullable=False)
|
||||
|
||||
next_comm = Column(String, nullable=False)
|
||||
next_pid = Column(Integer, nullable=False)
|
||||
next_prio = Column(Integer, nullable=False)
|
||||
|
||||
# __mapper_args__ = {
|
||||
# 'polymorphic_identity':'raw_ftrace_entry',
|
||||
# 'polymorphic_on':function
|
||||
# }
|
||||
|
||||
@staticmethod
|
||||
def parse_dict(function_args, id = None):
|
||||
# 'prev_comm=kworker/u16:5 prev_pid=13971 prev_prio=120 prev_state=S ==> next_comm=swapper/4 next_pid=0 next_prio=120'
|
||||
m = re.match("prev_comm=(.*) prev_pid=(\d+) prev_prio=(\d+) prev_state=(.*) ==> next_comm=(.*) next_pid=(\d+) next_prio=(\d+) ?", function_args)
|
||||
if not m:
|
||||
return None
|
||||
|
||||
groups = m.groups()
|
||||
# ('kworker/u16:5', '13971', '120', 'S', 'swapper/4', '0', '120')
|
||||
d = {}
|
||||
if id is not None:
|
||||
d['id'] = id
|
||||
d['prev_comm'] = groups[0]
|
||||
d['prev_pid'] = int(groups[1])
|
||||
d['prev_prio'] = int(groups[2])
|
||||
d['prev_state'] = groups[3]
|
||||
d['next_comm'] = groups[4]
|
||||
d['next_pid'] = int(groups[5])
|
||||
d['next_prio'] = int(groups[6])
|
||||
|
||||
return d
|
||||
|
||||
class SchedBlockedReason(Base):
|
||||
__tablename__ = 'sched_blocked_reasons'
|
||||
|
||||
id = Column(Integer, ForeignKey('raw_ftrace_entries.id'), primary_key=True)
|
||||
|
||||
pid = Column(Integer, nullable=False)
|
||||
iowait = Column(Integer, nullable=False)
|
||||
caller = Column(String, nullable=False)
|
||||
|
||||
@staticmethod
|
||||
def parse_dict(function_args, id = None):
|
||||
# 'pid=2289 iowait=1 caller=wait_on_page_bit_common+0x2a8/0x5f'
|
||||
m = re.match("pid=(\d+) iowait=(\d+) caller=(.*) ?", function_args)
|
||||
if not m:
|
||||
return None
|
||||
|
||||
groups = m.groups()
|
||||
# ('2289', '1', 'wait_on_page_bit_common+0x2a8/0x5f8')
|
||||
d = {}
|
||||
if id is not None:
|
||||
d['id'] = id
|
||||
d['pid'] = int(groups[0])
|
||||
d['iowait'] = int(groups[1])
|
||||
d['caller'] = groups[2]
|
||||
|
||||
return d
|
||||
|
||||
def init_sqlalchemy(db_filename):
|
||||
global _DEBUG
|
||||
engine = create_engine('sqlite:///' + db_filename, echo=_DEBUG)
|
||||
|
||||
# DROP TABLES
|
||||
# Base.metadata.drop_all(engine)
|
||||
# CREATE ... (tables)
|
||||
Base.metadata.create_all(engine)
|
||||
|
||||
Session = sessionmaker(bind=engine)
|
||||
session = Session()
|
||||
return (session, engine)
|
||||
|
||||
def insert_pending_entries(engine, kls, lst):
|
||||
if len(lst) > 0:
|
||||
# for some reason, it tries to generate an empty INSERT statement with len=0,
|
||||
# which of course violates the first non-null constraint.
|
||||
try:
|
||||
# Performance-sensitive parsing according to:
|
||||
# https://docs.sqlalchemy.org/en/13/faq/performance.html#i-m-inserting-400-000-rows-with-the-orm-and-it-s-really-slow
|
||||
engine.execute(kls.__table__.insert(), lst)
|
||||
lst.clear()
|
||||
except sqlalchemy.exc.IntegrityError as err:
|
||||
# possibly violating some SQL constraint, print data here.
|
||||
print(err)
|
||||
print(lst)
|
||||
raise
|
||||
|
||||
def parse_file(filename, session, engine):
|
||||
global _LIMIT
|
||||
global _FLUSH_LIMIT
|
||||
count = 0
|
||||
|
||||
pending_entries = []
|
||||
pending_sched_switch = []
|
||||
pending_sched_blocked_reasons = []
|
||||
|
||||
def insert_all_pending_entries():
|
||||
insert_pending_entries(engine, RawFtraceEntry, pending_entries)
|
||||
insert_pending_entries(engine, SchedSwitch, pending_sched_switch)
|
||||
insert_pending_entries(engine, SchedBlockedReason, pending_sched_blocked_reasons)
|
||||
|
||||
# use explicit encoding to avoid UnicodeDecodeError.
|
||||
with open(filename, encoding="ISO-8859-1") as f:
|
||||
for l in f:
|
||||
|
||||
if len(l) > 1 and l[0] == '#':
|
||||
continue
|
||||
|
||||
count = count + 1
|
||||
|
||||
if _LIMIT and count >= _LIMIT:
|
||||
break
|
||||
|
||||
raw_ftrace_entry = RawFtraceEntry.parse_dict(l)
|
||||
if not raw_ftrace_entry:
|
||||
print("WARNING: Failed to parse raw ftrace entry: " + l)
|
||||
continue
|
||||
|
||||
pending_entries.append(raw_ftrace_entry)
|
||||
|
||||
if raw_ftrace_entry['function'] == 'sched_switch':
|
||||
sched_switch = SchedSwitch.parse_dict(raw_ftrace_entry['function_args'], count)
|
||||
|
||||
if not sched_switch:
|
||||
print("WARNING: Failed to parse sched_switch: " + l)
|
||||
else:
|
||||
pending_sched_switch.append(sched_switch)
|
||||
|
||||
elif raw_ftrace_entry['function'] == 'sched_blocked_reason':
|
||||
sbr = SchedBlockedReason.parse_dict(raw_ftrace_entry['function_args'], count)
|
||||
|
||||
if not sbr:
|
||||
print("WARNING: Failed to parse sched_blocked_reason: " + l)
|
||||
else:
|
||||
pending_sched_blocked_reasons.append(sbr)
|
||||
|
||||
# Objects are cached in python memory, not yet sent to SQL database.
|
||||
#session.add(raw_ftrace_entry)
|
||||
|
||||
# Send INSERT/UPDATE/etc statements to the underlying SQL database.
|
||||
if count % _FLUSH_LIMIT == 0:
|
||||
# session.flush()
|
||||
#session.bulk_save_objects(pending_entries)
|
||||
#session.bulk_insert_mappings(RawFtraceEntry, pending_entries)
|
||||
insert_all_pending_entries()
|
||||
|
||||
insert_all_pending_entries()
|
||||
|
||||
# Ensure underlying database commits changes from memory to disk.
|
||||
session.commit()
|
||||
|
||||
return count
|
||||
|
||||
#def decode_raw_traces(session, engine):
|
||||
# count = 0
|
||||
# global _FLUSH_LIMIT
|
||||
#
|
||||
# for tmw in session.query(RawFtraceEntry).filter_by(function = 'tracing_mark_write'):
|
||||
# print(tmw)
|
||||
# decoded = TracingMarkWriteFtraceEntry.decode(tmw)
|
||||
# session.add(decoded)
|
||||
#
|
||||
# if count % _FLUSH_LIMIT == 0:
|
||||
# session.flush()
|
||||
#
|
||||
# session.commit()
|
||||
#
|
||||
# return count
|
||||
|
||||
def main(argv):
|
||||
db_filename = sys.argv[1]
|
||||
trace_filename = sys.argv[2]
|
||||
parser = argparse.ArgumentParser(description='Convert ftrace/systrace file into sqlite3 db.')
|
||||
parser.add_argument('db_filename', metavar='sql_filename.db', type=str,
|
||||
help='path to sqlite3 db filename')
|
||||
parser.add_argument('trace_filename', metavar='systrace.ftrace', type=str,
|
||||
help='path to ftrace/systrace filename')
|
||||
parser.add_argument('--limit', type=int, help='limit the number of entries parsed [for debugging]')
|
||||
|
||||
session, engine = init_sqlalchemy(db_filename)
|
||||
args = parser.parse_args()
|
||||
|
||||
db_filename = args.db_filename
|
||||
trace_filename = args.trace_filename
|
||||
|
||||
trace2db = Trace2Db(db_filename)
|
||||
print("SQL Alchemy db initialized")
|
||||
|
||||
# parse 'raw_ftrace_entries' table
|
||||
count = parse_file(trace_filename, session, engine)
|
||||
count = trace2db.parse_file_into_db(trace_filename, limit=args.limit)
|
||||
print("Count was ", count)
|
||||
|
||||
# create other tables
|
||||
# count = decode_raw_traces(session, engine)
|
||||
|
||||
return 0
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
Reference in New Issue
Block a user