Merge "Add uid field annotation in atoms.proto and statd memory usage optimization." into pi-dev

This commit is contained in:
TreeHugger Robot
2018-03-20 17:12:25 +00:00
committed by Android (Google) Code Review
10 changed files with 185 additions and 123 deletions

View File

@@ -112,8 +112,8 @@ void updateUid(Value* value, int hostUid) {
}
void StatsLogProcessor::mapIsolatedUidToHostUidIfNecessaryLocked(LogEvent* event) const {
if (android::util::kAtomsWithAttributionChain.find(event->GetTagId()) !=
android::util::kAtomsWithAttributionChain.end()) {
if (android::util::AtomsInfo::kAtomsWithAttributionChain.find(event->GetTagId()) !=
android::util::AtomsInfo::kAtomsWithAttributionChain.end()) {
for (auto& value : *(event->getMutableValues())) {
if (value.mField.getPosAtDepth(0) > kAttributionField) {
break;
@@ -123,12 +123,20 @@ void StatsLogProcessor::mapIsolatedUidToHostUidIfNecessaryLocked(LogEvent* event
updateUid(&value.mValue, hostUid);
}
}
} else if (android::util::kAtomsWithUidField.find(event->GetTagId()) !=
android::util::kAtomsWithUidField.end() &&
event->getValues().size() > 0 && (event->getValues())[0].mValue.getType() == INT) {
Value& value = (*event->getMutableValues())[0].mValue;
const int hostUid = mUidMap->getHostUidOrSelf(value.int_value);
updateUid(&value, hostUid);
} else {
auto it = android::util::AtomsInfo::kAtomsWithUidField.find(event->GetTagId());
if (it != android::util::AtomsInfo::kAtomsWithUidField.end()) {
int uidField = it->second; // uidField is the field number in proto,
// starting from 1
if (uidField > 0 && (int)event->getValues().size() >= uidField &&
(event->getValues())[uidField - 1].mValue.getType() == INT) {
Value& value = (*event->getMutableValues())[uidField - 1].mValue;
const int hostUid = mUidMap->getHostUidOrSelf(value.int_value);
updateUid(&value, hostUid);
} else {
ALOGE("Malformed log, uid not found. %s", event->ToString().c_str());
}
}
}
}

View File

@@ -67,4 +67,7 @@ message StateAtomFieldOption {
extend google.protobuf.FieldOptions {
// Flags to decorate an atom that presents a state change.
optional StateAtomFieldOption stateFieldOption = 50000;
// Flags to decorate the uid fields in an atom.
optional bool is_uid = 50001 [default = false];
}

View File

@@ -209,7 +209,7 @@ message ScreenStateChanged {
* frameworks/base/services/core/java/com/android/server/am/BatteryStatsService.java
*/
message UidProcessStateChanged {
optional int32 uid = 1 [(stateFieldOption).option = PRIMARY];
optional int32 uid = 1 [(stateFieldOption).option = PRIMARY, (is_uid) = true];
// The state, from frameworks/base/core/proto/android/app/enums.proto.
optional android.app.ProcessStateEnum state = 2 [(stateFieldOption).option = EXCLUSIVE];
@@ -223,7 +223,7 @@ message UidProcessStateChanged {
*/
message ProcessLifeCycleStateChanged {
// TODO: should be a string tagged w/ uid annotation
optional int32 uid = 1;
optional int32 uid = 1 [(is_uid) = true];
// The process name (usually same as the app name).
optional string name = 2;
@@ -593,7 +593,7 @@ message WakeupAlarmOccurred {
*/
message MobileRadioPowerStateChanged {
// TODO: Add attribution instead of uid?
optional int32 uid = 1;
optional int32 uid = 1 [(is_uid) = true];
// Power state, from frameworks/base/core/proto/android/telephony/enums.proto.
optional android.telephony.DataConnectionPowerStateEnum state = 2;
@@ -608,7 +608,7 @@ message MobileRadioPowerStateChanged {
*/
message WifiRadioPowerStateChanged {
// TODO: Add attribution instead of uid?
optional int32 uid = 1;
optional int32 uid = 1 [(is_uid) = true];
// Power state, from frameworks/base/core/proto/android/telephony/enums.proto.
optional android.telephony.DataConnectionPowerStateEnum state = 2;
@@ -1130,7 +1130,8 @@ message ChargeCyclesReported {
*/
message DaveyOccurred {
// The UID that logged this atom.
optional int32 uid = 1;
optional int32 uid = 1 [(is_uid) = true];
;
// Amount of time it took to render the frame. Should be >=700ms.
optional int64 jank_duration_millis = 2;
@@ -1187,7 +1188,7 @@ message SettingChanged {
* frameworks/base/services/core/java/com/android/server/am/ActivityRecord.java
*/
message ActivityForegroundStateChanged {
optional int32 uid = 1;
optional int32 uid = 1 [(is_uid) = true];
optional string pkg_name = 2;
optional string class_name = 3;
@@ -1205,7 +1206,7 @@ message ActivityForegroundStateChanged {
*/
message DropboxErrorChanged {
// The uid if available. -1 means not available.
optional int32 uid = 1;
optional int32 uid = 1 [(is_uid) = true];
// Tag used when recording this error to dropbox. Contains data_ or system_ prefix.
optional string tag = 2;
@@ -1236,7 +1237,7 @@ message DropboxErrorChanged {
*/
message AppBreadcrumbReported {
// The uid of the application that sent this custom atom.
optional int32 uid = 1;
optional int32 uid = 1 [(is_uid) = true];
// An arbitrary label chosen by the developer. For Android P, the label should be in [0, 16).
optional int32 label = 2;
@@ -1260,7 +1261,7 @@ message AppBreadcrumbReported {
*/
message AnomalyDetected {
// Uid that owns the config whose anomaly detection alert fired.
optional int32 config_uid = 1;
optional int32 config_uid = 1 [(is_uid) = true];
// Id of the config whose anomaly detection alert fired.
optional int64 config_id = 2;
@@ -1271,7 +1272,7 @@ message AnomalyDetected {
message AppStartChanged {
// The uid if available. -1 means not available.
optional int32 uid = 1;
optional int32 uid = 1 [(is_uid) = true];
// The app package name.
optional string pkg_name = 2;
@@ -1318,7 +1319,7 @@ message AppStartChanged {
message AppStartCancelChanged {
// The uid if available. -1 means not available.
optional int32 uid = 1;
optional int32 uid = 1 [(is_uid) = true];
// The app package name.
optional string pkg_name = 2;
@@ -1338,7 +1339,7 @@ message AppStartCancelChanged {
message AppStartFullyDrawnChanged {
// The uid if available. -1 means not available.
optional int32 uid = 1;
optional int32 uid = 1 [(is_uid) = true];
// The app package name.
optional string pkg_name = 2;
@@ -1369,7 +1370,7 @@ message AppStartFullyDrawnChanged {
*/
message PictureInPictureStateChanged {
// -1 if it is not available
optional int32 uid = 1;
optional int32 uid = 1 [(is_uid) = true];
optional string short_name = 2;
@@ -1388,7 +1389,7 @@ message PictureInPictureStateChanged {
* services/core/java/com/android/server/wm/Session.java
*/
message OverlayStateChanged {
optional int32 uid = 1;
optional int32 uid = 1 [(is_uid) = true];
optional string package_name = 2;
@@ -1409,7 +1410,7 @@ message OverlayStateChanged {
* //frameworks/base/services/core/java/com/android/server/am/ActiveServices.java
*/
message ForegroundServiceStateChanged {
optional int32 uid = 1;
optional int32 uid = 1 [(is_uid) = true];
// package_name + "/" + class_name
optional string short_name = 2;
@@ -1429,6 +1430,7 @@ message ForegroundServiceStateChanged {
* frameworks/base/core/java/com/android/internal/os/BatteryStatsImpl.java
*/
message IsolatedUidChanged {
// NOTE: DO NOT annotate uid field in this atom. This atom is specially handled in statsd.
// The host UID. Generally, we should attribute metrics from the isolated uid to the host uid.
optional int32 parent_uid = 1;
@@ -1450,7 +1452,7 @@ message IsolatedUidChanged {
message PacketWakeupOccurred {
// The uid owning the socket into which the packet was delivered, or -1 if the packet was
// delivered nowhere.
optional int32 uid = 1;
optional int32 uid = 1 [(is_uid) = true];
// The interface name on which the packet was received.
optional string iface = 2;
// The ethertype value of the packet.
@@ -1478,7 +1480,7 @@ message PacketWakeupOccurred {
*/
message AppStartMemoryStateCaptured {
// The uid if available. -1 means not available.
optional int32 uid = 1;
optional int32 uid = 1 [(is_uid) = true];
// The process name.
optional string process_name = 2;
@@ -1524,7 +1526,7 @@ message LmkStateChanged {
*/
message LmkKillOccurred {
// The uid if available. -1 means not available.
optional int32 uid = 1;
optional int32 uid = 1 [(is_uid) = true];
// The process name.
optional string process_name = 2;
@@ -1556,7 +1558,7 @@ message LmkKillOccurred {
*/
message AppDied {
// timestamp(elapsedRealtime) of record creation
optional uint64 timestamp_millis = 1;
optional uint64 timestamp_millis = 1 [(stateFieldOption).option = EXCLUSIVE];
}
//////////////////////////////////////////////////////////////////////
@@ -1570,7 +1572,7 @@ message AppDied {
* StatsCompanionService (using BatteryStats to get which interfaces are wifi)
*/
message WifiBytesTransfer {
optional int32 uid = 1;
optional int32 uid = 1 [(is_uid) = true];
optional int64 rx_bytes = 2;
@@ -1588,9 +1590,10 @@ message WifiBytesTransfer {
* StatsCompanionService (using BatteryStats to get which interfaces are wifi)
*/
message WifiBytesTransferByFgBg {
optional int32 uid = 1;
optional int32 uid = 1 [(is_uid) = true];
// 1 denotes foreground and 0 denotes background. This is called Set in NetworkStats.
// 1 denotes foreground and 0 denotes background. This is called Set in
// NetworkStats.
optional int32 is_foreground = 2;
optional int64 rx_bytes = 3;
@@ -1609,7 +1612,7 @@ message WifiBytesTransferByFgBg {
* StatsCompanionService (using BatteryStats to get which interfaces are mobile data)
*/
message MobileBytesTransfer {
optional int32 uid = 1;
optional int32 uid = 1 [(is_uid) = true];
optional int64 rx_bytes = 2;
@@ -1627,9 +1630,10 @@ message MobileBytesTransfer {
* StatsCompanionService (using BatteryStats to get which interfaces are mobile data)
*/
message MobileBytesTransferByFgBg {
optional int32 uid = 1;
optional int32 uid = 1 [(is_uid) = true];
// 1 denotes foreground and 0 denotes background. This is called Set in NetworkStats.
// 1 denotes foreground and 0 denotes background. This is called Set in
// NetworkStats.
optional int32 is_foreground = 2;
optional int64 rx_bytes = 3;
@@ -1648,7 +1652,7 @@ message MobileBytesTransferByFgBg {
* StatsCompanionService
*/
message BluetoothBytesTransfer {
optional int32 uid = 1;
optional int32 uid = 1 [(is_uid) = true];
optional int64 rx_bytes = 2;
@@ -1708,7 +1712,7 @@ message CpuTimePerFreq {
* Note that isolated process uid time should be attributed to host uids.
*/
message CpuTimePerUid {
optional int32 uid = 1;
optional int32 uid = 1 [(is_uid) = true];
optional uint64 user_time_millis = 2;
optional uint64 sys_time_millis = 3;
}
@@ -1719,7 +1723,7 @@ message CpuTimePerUid {
* For each uid, we order the time by descending frequencies.
*/
message CpuTimePerUidFreq {
optional int32 uid = 1;
optional int32 uid = 1 [(is_uid) = true];
optional uint32 freq_index = 2;
optional uint64 time_millis = 3;
}
@@ -1801,7 +1805,7 @@ message BluetoothActivityInfo {
*/
message ProcessMemoryState {
// The uid if available. -1 means not available.
optional int32 uid = 1;
optional int32 uid = 1 [(is_uid) = true];
// The process name.
optional string process_name = 2;
@@ -1853,7 +1857,7 @@ message SystemUptime {
* The file contains a monotonically increasing count of time for a single boot.
*/
message CpuActiveTime {
optional int32 uid = 1;
optional int32 uid = 1 [(is_uid) = true];
optional uint64 time_millis = 2;
}
@@ -1867,7 +1871,7 @@ message CpuActiveTime {
* The file contains a monotonically increasing count of time for a single boot.
*/
message CpuClusterTime {
optional int32 uid = 1;
optional int32 uid = 1 [(is_uid) = true];
optional int32 cluster_index = 2;
optional uint64 time_millis = 3;
}

View File

@@ -112,7 +112,8 @@ void mergeIsolatedUidsToHostUid(vector<shared_ptr<LogEvent>>& data, const sp<Uid
VLOG("Unknown pull atom id %d", tagId);
return;
}
if (android::util::kAtomsWithUidField.find(tagId) == android::util::kAtomsWithUidField.end()) {
if (android::util::AtomsInfo::kAtomsWithUidField.find(tagId) ==
android::util::AtomsInfo::kAtomsWithUidField.end()) {
VLOG("No uid to merge for atom %d", tagId);
return;
}

View File

@@ -134,8 +134,8 @@ void EventMetricProducer::onMatchedLogEventInternalLocked(
uint64_t wrapperToken =
mProto->start(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_DATA);
const bool truncateTimestamp =
android::util::kNotTruncatingTimestampAtomWhiteList.find(event.GetTagId()) ==
android::util::kNotTruncatingTimestampAtomWhiteList.end();
android::util::AtomsInfo::kNotTruncatingTimestampAtomWhiteList.find(event.GetTagId()) ==
android::util::AtomsInfo::kNotTruncatingTimestampAtomWhiteList.end();
if (truncateTimestamp) {
mProto->write(FIELD_TYPE_INT64 | FIELD_ID_ELAPSED_TIMESTAMP_NANOS,
(long long)truncateTimestampNsToFiveMinutes(event.GetElapsedTimestampNs()));

View File

@@ -195,8 +195,9 @@ void GaugeMetricProducer::onDumpReportLocked(const uint64_t dumpTimeNs,
protoOutput->end(atomsToken);
}
const bool truncateTimestamp =
android::util::kNotTruncatingTimestampAtomWhiteList.find(mTagId) ==
android::util::kNotTruncatingTimestampAtomWhiteList.end();
android::util::AtomsInfo::kNotTruncatingTimestampAtomWhiteList.find(
mTagId) ==
android::util::AtomsInfo::kNotTruncatingTimestampAtomWhiteList.end();
const int64_t wall_clock_ns = truncateTimestamp ?
truncateTimestampNsToFiveMinutes(getWallClockNs()) : getWallClockNs();
for (const auto& atom : bucket.mGaugeAtoms) {

View File

@@ -170,9 +170,10 @@ bool isStateTracker(const SimplePredicate& simplePredicate, vector<Matcher>* pri
// 1. must not have "stop". must have "dimension"
if (!simplePredicate.has_stop() && simplePredicate.has_dimensions()) {
// TODO: need to check the start atom matcher too.
auto it = android::util::kStateAtomsFieldOptions.find(simplePredicate.dimensions().field());
auto it = android::util::AtomsInfo::kStateAtomsFieldOptions.find(
simplePredicate.dimensions().field());
// 2. must be based on a state atom.
if (it != android::util::kStateAtomsFieldOptions.end()) {
if (it != android::util::AtomsInfo::kStateAtomsFieldOptions.end()) {
// 3. dimension must be primary fields + state field IN ORDER
size_t expectedDimensionCount = it->second.primaryFields.size() + 1;
vector<Matcher> dimensions;

View File

@@ -46,7 +46,8 @@ AtomDecl::AtomDecl(const AtomDecl& that)
message(that.message),
fields(that.fields),
primaryFields(that.primaryFields),
exclusiveField(that.exclusiveField) {}
exclusiveField(that.exclusiveField),
uidField(that.uidField) {}
AtomDecl::AtomDecl(int c, const string& n, const string& m)
:code(c),
@@ -262,6 +263,18 @@ int collate_atom(const Descriptor *atom, AtomDecl *atomDecl,
errorCount++;
}
}
if (field->options().GetExtension(os::statsd::is_uid) == true) {
if (javaType != JAVA_TYPE_INT) {
errorCount++;
}
if (atomDecl->uidField == 0) {
atomDecl->uidField = it->first;
} else {
errorCount++;
}
}
}
return errorCount;

View File

@@ -84,6 +84,8 @@ struct AtomDecl {
vector<int> primaryFields;
int exclusiveField = 0;
int uidField = 0;
AtomDecl();
AtomDecl(const AtomDecl& that);
AtomDecl(int code, const string& name, const string& message);

View File

@@ -113,6 +113,98 @@ static int write_stats_log_cpp(FILE *out, const Atoms &atoms,
fprintf(out, "// the single event tag id for all stats logs\n");
fprintf(out, "const static int kStatsEventTag = 1937006964;\n");
std::set<string> kTruncatingAtomNames = {"mobile_radio_power_state_changed",
"audio_state_changed",
"call_state_changed",
"phone_signal_strength_changed",
"mobile_bytes_transfer_by_fg_bg",
"mobile_bytes_transfer"};
fprintf(out,
"const std::set<int> "
"AtomsInfo::kNotTruncatingTimestampAtomWhiteList = {\n");
for (set<AtomDecl>::const_iterator atom = atoms.decls.begin();
atom != atoms.decls.end(); atom++) {
if (kTruncatingAtomNames.find(atom->name) ==
kTruncatingAtomNames.end()) {
string constant = make_constant_name(atom->name);
fprintf(out, " %s,\n", constant.c_str());
}
}
fprintf(out, "};\n");
fprintf(out, "\n");
fprintf(out,
"const std::set<int> AtomsInfo::kAtomsWithAttributionChain = {\n");
for (set<AtomDecl>::const_iterator atom = atoms.decls.begin();
atom != atoms.decls.end(); atom++) {
for (vector<AtomField>::const_iterator field = atom->fields.begin();
field != atom->fields.end(); field++) {
if (field->javaType == JAVA_TYPE_ATTRIBUTION_CHAIN) {
string constant = make_constant_name(atom->name);
fprintf(out, " %s,\n", constant.c_str());
break;
}
}
}
fprintf(out, "};\n");
fprintf(out, "\n");
fprintf(out,
"static std::map<int, int> "
"getAtomUidField() {\n");
fprintf(out, " std::map<int, int> uidField;\n");
for (set<AtomDecl>::const_iterator atom = atoms.decls.begin();
atom != atoms.decls.end(); atom++) {
if (atom->uidField == 0) {
continue;
}
fprintf(out,
"\n // Adding uid field for atom "
"(%d)%s\n",
atom->code, atom->name.c_str());
fprintf(out, " uidField[static_cast<int>(%s)] = %d;\n",
make_constant_name(atom->name).c_str(), atom->uidField);
}
fprintf(out, " return uidField;\n");
fprintf(out, "};\n");
fprintf(out,
"const std::map<int, int> AtomsInfo::kAtomsWithUidField = "
"getAtomUidField();\n");
fprintf(out,
"static std::map<int, StateAtomFieldOptions> "
"getStateAtomFieldOptions() {\n");
fprintf(out, " std::map<int, StateAtomFieldOptions> options;\n");
fprintf(out, " StateAtomFieldOptions opt;\n");
for (set<AtomDecl>::const_iterator atom = atoms.decls.begin();
atom != atoms.decls.end(); atom++) {
if (atom->primaryFields.size() == 0 && atom->exclusiveField == 0) {
continue;
}
fprintf(out,
"\n // Adding primary and exclusive fields for atom "
"(%d)%s\n",
atom->code, atom->name.c_str());
fprintf(out, " opt.primaryFields.clear();\n");
for (const auto& field : atom->primaryFields) {
fprintf(out, " opt.primaryFields.push_back(%d);\n", field);
}
fprintf(out, " opt.exclusiveField = %d;\n", atom->exclusiveField);
fprintf(out, " options[static_cast<int>(%s)] = opt;\n",
make_constant_name(atom->name).c_str());
}
fprintf(out, " return options;\n");
fprintf(out, " }\n");
fprintf(out,
"const std::map<int, StateAtomFieldOptions> "
"AtomsInfo::kStateAtomsFieldOptions = "
"getStateAtomFieldOptions();\n");
// Print write methods
fprintf(out, "\n");
for (set<vector<java_type_t>>::const_iterator signature = atoms.signatures.begin();
@@ -366,89 +458,26 @@ write_stats_log_header(FILE* out, const Atoms& atoms, const AtomDecl &attributio
fprintf(out, "};\n");
fprintf(out, "\n");
std::set<string> kTruncatingAtomNames =
{ "mobile_radio_power_state_changed", "audio_state_changed", "call_state_changed",
"phone_signal_strength_changed", "mobile_bytes_transfer_by_fg_bg",
"mobile_bytes_transfer"};
fprintf(out, "const static std::set<int> kNotTruncatingTimestampAtomWhiteList = {\n");
for (set<AtomDecl>::const_iterator atom = atoms.decls.begin();
atom != atoms.decls.end(); atom++) {
if (kTruncatingAtomNames.find(atom->name) == kTruncatingAtomNames.end()) {
string constant = make_constant_name(atom->name);
fprintf(out, " %s,\n", constant.c_str());
}
}
fprintf(out, "};\n");
fprintf(out, "\n");
fprintf(out, "const static std::set<int> kAtomsWithUidField = {\n");
for (set<AtomDecl>::const_iterator atom = atoms.decls.begin();
atom != atoms.decls.end(); atom++) {
for (vector<AtomField>::const_iterator field = atom->fields.begin();
field != atom->fields.end(); field++) {
if (field->name == "uid") {
string constant = make_constant_name(atom->name);
fprintf(out, " %s,\n", constant.c_str());
break;
}
}
}
fprintf(out, "};\n");
fprintf(out, "\n");
fprintf(out, "const static std::set<int> kAtomsWithAttributionChain = {\n");
for (set<AtomDecl>::const_iterator atom = atoms.decls.begin();
atom != atoms.decls.end(); atom++) {
for (vector<AtomField>::const_iterator field = atom->fields.begin();
field != atom->fields.end(); field++) {
if (field->javaType == JAVA_TYPE_ATTRIBUTION_CHAIN) {
string constant = make_constant_name(atom->name);
fprintf(out, " %s,\n", constant.c_str());
break;
}
}
}
fprintf(out, "};\n");
fprintf(out, "\n");
fprintf(out, "const static int kMaxPushedAtomId = %d;\n\n", maxPushedAtomId);
fprintf(out, "struct StateAtomFieldOptions {\n");
fprintf(out, " std::vector<int> primaryFields;\n");
fprintf(out, " int exclusiveField;\n");
fprintf(out, "};\n");
fprintf(out, "\n");
fprintf(out, "struct AtomsInfo {\n");
fprintf(out,
" static std::map<int, StateAtomFieldOptions> "
"getStateAtomFieldOptions() {\n");
fprintf(out, " std::map<int, StateAtomFieldOptions> options;\n");
fprintf(out, " StateAtomFieldOptions opt;\n");
for (set<AtomDecl>::const_iterator atom = atoms.decls.begin();
atom != atoms.decls.end(); atom++) {
if (atom->primaryFields.size() == 0 && atom->exclusiveField == 0) {
continue;
}
fprintf(out,
"\n // Adding primary and exclusive fields for atom "
"(%d)%s\n",
atom->code, atom->name.c_str());
fprintf(out, " opt.primaryFields.clear();\n");
for (const auto& field : atom->primaryFields) {
fprintf(out, " opt.primaryFields.push_back(%d);\n", field);
}
fprintf(out, " opt.exclusiveField = %d;\n", atom->exclusiveField);
fprintf(out, " options[static_cast<int>(%s)] = opt;\n",
make_constant_name(atom->name).c_str());
}
fprintf(out, " return options;\n");
fprintf(out, " }\n");
" const static std::set<int> "
"kNotTruncatingTimestampAtomWhiteList;\n");
fprintf(out, " const static std::map<int, int> kAtomsWithUidField;\n");
fprintf(out,
" const static std::set<int> kAtomsWithAttributionChain;\n");
fprintf(out,
" const static std::map<int, StateAtomFieldOptions> "
"kStateAtomsFieldOptions;\n");
fprintf(out, "};\n");
fprintf(out,
"const static std::map<int, StateAtomFieldOptions> "
"kStateAtomsFieldOptions = "
"StateAtomFieldOptions::getStateAtomFieldOptions();\n");
fprintf(out, "const static int kMaxPushedAtomId = %d;\n\n",
maxPushedAtomId);
// Print write methods
fprintf(out, "//\n");