Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge changes Icd1b1e57,Ib9c6b9b4 into rvc-dev

Browse Source 
* changes:
  Load Metadata from Disk
  Save metadata to Disk
This commit is contained in:
Jeffrey Huang
2020-04-01 18:56:26 +00:00
committed by Android (Google) Code Review
parent b4b806dd7a 475677e771
commit 898dda3919
13 changed files with 639 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
cmds/statsd/Android.bp
View File

@@ -78,6 +78,7 @@ cc_defaults {
"src/matchers/EventMatcherWizard.cpp",
"src/matchers/matcher_util.cpp",
"src/matchers/SimpleLogMatchingTracker.cpp",
"src/metadata_util.cpp",
"src/metrics/CountMetricProducer.cpp",
"src/metrics/duration_helper/MaxDurationTracker.cpp",
"src/metrics/duration_helper/OringDurationTracker.cpp",
@@ -340,6 +341,7 @@ cc_test {
"tests/log_event/LogEventQueue_test.cpp",
"tests/LogEntryMatcher_test.cpp",
"tests/LogEvent_test.cpp",
"tests/metadata_util_test.cpp",
"tests/metrics/CountMetricProducer_test.cpp",
"tests/metrics/DurationMetricProducer_test.cpp",
"tests/metrics/EventMetricProducer_test.cpp",

105
cmds/statsd/src/StatsLogProcessor.cpp
View File

@@ -79,6 +79,7 @@ constexpr const char* kPermissionUsage = "android.permission.PACKAGE_USAGE_STATS
#define NS_PER_HOUR 3600 * NS_PER_SEC
#define STATS_ACTIVE_METRIC_DIR "/data/misc/stats-active-metric"
#define STATS_METADATA_DIR "/data/misc/stats-metadata"
// Cool down period for writing data to disk to avoid overwriting files.
#define WRITE_DATA_COOL_DOWN_SEC 5
@@ -852,6 +853,110 @@ void StatsLogProcessor::SaveActiveConfigsToDisk(int64_t currentTimeNs) {
proto.flush(fd.get());
}
void StatsLogProcessor::SaveMetadataToDisk(int64_t currentWallClockTimeNs,
int64_t systemElapsedTimeNs) {
std::lock_guard<std::mutex> lock(mMetricsMutex);
// Do not write to disk if we already have in the last few seconds.
if (static_cast<unsigned long long> (systemElapsedTimeNs) <
mLastMetadataWriteNs + WRITE_DATA_COOL_DOWN_SEC * NS_PER_SEC) {
ALOGI("Statsd skipping writing metadata to disk. Already wrote data in last %d seconds",
WRITE_DATA_COOL_DOWN_SEC);
return;
}
mLastMetadataWriteNs = systemElapsedTimeNs;
metadata::StatsMetadataList metadataList;
WriteMetadataToProtoLocked(
currentWallClockTimeNs, systemElapsedTimeNs, &metadataList);
string file_name = StringPrintf("%s/metadata", STATS_METADATA_DIR);
StorageManager::deleteFile(file_name.c_str());
if (metadataList.stats_metadata_size() == 0) {
// Skip the write if we have nothing to write.
return;
}
std::string data;
metadataList.SerializeToString(&data);
StorageManager::writeFile(file_name.c_str(), data.c_str(), data.size());
}
void StatsLogProcessor::WriteMetadataToProto(int64_t currentWallClockTimeNs,
int64_t systemElapsedTimeNs,
metadata::StatsMetadataList* metadataList) {
std::lock_guard<std::mutex> lock(mMetricsMutex);
WriteMetadataToProtoLocked(currentWallClockTimeNs, systemElapsedTimeNs, metadataList);
}
void StatsLogProcessor::WriteMetadataToProtoLocked(int64_t currentWallClockTimeNs,
int64_t systemElapsedTimeNs,
metadata::StatsMetadataList* metadataList) {
for (const auto& pair : mMetricsManagers) {
const sp<MetricsManager>& metricsManager = pair.second;
metadata::StatsMetadata* statsMetadata = metadataList->add_stats_metadata();
bool metadataWritten = metricsManager->writeMetadataToProto(currentWallClockTimeNs,
systemElapsedTimeNs, statsMetadata);
if (!metadataWritten) {
metadataList->mutable_stats_metadata()->RemoveLast();
}
}
}
void StatsLogProcessor::LoadMetadataFromDisk(int64_t currentWallClockTimeNs,
int64_t systemElapsedTimeNs) {
std::lock_guard<std::mutex> lock(mMetricsMutex);
string file_name = StringPrintf("%s/metadata", STATS_METADATA_DIR);
int fd = open(file_name.c_str(), O_RDONLY | O_CLOEXEC);
if (-1 == fd) {
VLOG("Attempt to read %s but failed", file_name.c_str());
StorageManager::deleteFile(file_name.c_str());
return;
}
string content;
if (!android::base::ReadFdToString(fd, &content)) {
ALOGE("Attempt to read %s but failed", file_name.c_str());
close(fd);
StorageManager::deleteFile(file_name.c_str());
return;
}
close(fd);
metadata::StatsMetadataList statsMetadataList;
if (!statsMetadataList.ParseFromString(content)) {
ALOGE("Attempt to read %s but failed; failed to metadata", file_name.c_str());
StorageManager::deleteFile(file_name.c_str());
return;
}
SetMetadataStateLocked(statsMetadataList, currentWallClockTimeNs, systemElapsedTimeNs);
StorageManager::deleteFile(file_name.c_str());
}
void StatsLogProcessor::SetMetadataState(const metadata::StatsMetadataList& statsMetadataList,
int64_t currentWallClockTimeNs,
int64_t systemElapsedTimeNs) {
std::lock_guard<std::mutex> lock(mMetricsMutex);
SetMetadataStateLocked(statsMetadataList, currentWallClockTimeNs, systemElapsedTimeNs);
}
void StatsLogProcessor::SetMetadataStateLocked(
const metadata::StatsMetadataList& statsMetadataList,
int64_t currentWallClockTimeNs,
int64_t systemElapsedTimeNs) {
for (const metadata::StatsMetadata& metadata : statsMetadataList.stats_metadata()) {
ConfigKey key(metadata.config_key().uid(), metadata.config_key().config_id());
auto it = mMetricsManagers.find(key);
if (it == mMetricsManagers.end()) {
ALOGE("No config found for configKey %s", key.ToString().c_str());
continue;
}
VLOG("Setting metadata %s", key.ToString().c_str());
it->second->loadMetadata(metadata, currentWallClockTimeNs, systemElapsedTimeNs);
}
VLOG("Successfully loaded %d metadata.", statsMetadataList.stats_metadata_size());
}
void StatsLogProcessor::WriteActiveConfigsToProtoOutputStream(
int64_t currentTimeNs, const DumpReportReason reason, ProtoOutputStream* proto) {
std::lock_guard<std::mutex> lock(mMetricsMutex);

33
cmds/statsd/src/StatsLogProcessor.h
View File

@@ -24,6 +24,7 @@
#include "external/StatsPullerManager.h"
#include "frameworks/base/cmds/statsd/src/statsd_config.pb.h"
#include "frameworks/base/cmds/statsd/src/statsd_metadata.pb.h"
#include <stdio.h>
#include <unordered_map>
@@ -89,6 +90,23 @@ public:
/* Load configs containing metrics with active activations from disk. */
void LoadActiveConfigsFromDisk();
/* Persist metadata for configs and metrics to disk. */
void SaveMetadataToDisk(int64_t currentWallClockTimeNs, int64_t systemElapsedTimeNs);
/* Writes the statsd metadata for all configs and metrics to StatsMetadataList. */
void WriteMetadataToProto(int64_t currentWallClockTimeNs,
int64_t systemElapsedTimeNs,
metadata::StatsMetadataList* metadataList);
/* Load stats metadata for configs and metrics from disk. */
void LoadMetadataFromDisk(int64_t currentWallClockTimeNs,
int64_t systemElapsedTimeNs);
/* Sets the metadata for all configs and metrics */
void SetMetadataState(const metadata::StatsMetadataList& statsMetadataList,
int64_t currentWallClockTimeNs,
int64_t systemElapsedTimeNs);
/* Sets the active status/ttl for all configs and metrics to the status in ActiveConfigList. */
void SetConfigsActiveState(const ActiveConfigList& activeConfigList, int64_t currentTimeNs);
@@ -173,8 +191,17 @@ private:
void SetConfigsActiveStateLocked(const ActiveConfigList& activeConfigList,
int64_t currentTimeNs);
void SetMetadataStateLocked(const metadata::StatsMetadataList& statsMetadataList,
int64_t currentWallClockTimeNs,
int64_t systemElapsedTimeNs);
void WriteMetadataToProtoLocked(int64_t currentWallClockTimeNs,
int64_t systemElapsedTimeNs,
metadata::StatsMetadataList* metadataList);
void WriteDataToDiskLocked(const DumpReportReason dumpReportReason,
const DumpLatency dumpLatency);
void WriteDataToDiskLocked(const ConfigKey& key, const int64_t timestampNs,
const DumpReportReason dumpReportReason,
const DumpLatency dumpLatency);
@@ -241,6 +268,9 @@ private:
// Last time we wrote active metrics to disk.
int64_t mLastActiveMetricsWriteNs = 0;
//Last time we wrote metadata to disk.
int64_t mLastMetadataWriteNs = 0;
#ifdef VERY_VERBOSE_PRINTING
bool mPrintAllLogs = false;
#endif
@@ -278,6 +308,9 @@ private:
FRIEND_TEST(AnomalyDetectionE2eTest, TestSlicedCountMetric_single_bucket);
FRIEND_TEST(AnomalyDetectionE2eTest, TestSlicedCountMetric_multiple_buckets);
FRIEND_TEST(AnomalyDetectionE2eTest, TestCountMetric_save_refractory_to_disk_no_data_written);
FRIEND_TEST(AnomalyDetectionE2eTest, TestCountMetric_save_refractory_to_disk);
FRIEND_TEST(AnomalyDetectionE2eTest, TestCountMetric_load_refractory_from_disk);
FRIEND_TEST(AnomalyDetectionE2eTest, TestDurationMetric_SUM_single_bucket);
FRIEND_TEST(AnomalyDetectionE2eTest, TestDurationMetric_SUM_multiple_buckets);
FRIEND_TEST(AnomalyDetectionE2eTest, TestDurationMetric_SUM_long_refractory_period);

14
cmds/statsd/src/StatsService.cpp
View File

@@ -1022,6 +1022,7 @@ Status StatsService::informDeviceShutdown() {
VLOG("StatsService::informDeviceShutdown");
mProcessor->WriteDataToDisk(DEVICE_SHUTDOWN, FAST);
mProcessor->SaveActiveConfigsToDisk(getElapsedRealtimeNs());
mProcessor->SaveMetadataToDisk(getWallClockNs(), getElapsedRealtimeNs());
return Status::ok();
}
@@ -1056,6 +1057,7 @@ Status StatsService::statsCompanionReady() {
void StatsService::Startup() {
mConfigManager->Startup();
mProcessor->LoadActiveConfigsFromDisk();
mProcessor->LoadMetadataFromDisk(getWallClockNs(), getElapsedRealtimeNs());
}
void StatsService::Terminate() {
@@ -1063,6 +1065,7 @@ void StatsService::Terminate() {
if (mProcessor != nullptr) {
mProcessor->WriteDataToDisk(TERMINATION_SIGNAL_RECEIVED, FAST);
mProcessor->SaveActiveConfigsToDisk(getElapsedRealtimeNs());
mProcessor->SaveMetadataToDisk(getWallClockNs(), getElapsedRealtimeNs());
}
}
@@ -1295,20 +1298,23 @@ void StatsService::statsCompanionServiceDiedImpl() {
if (mProcessor != nullptr) {
ALOGW("Reset statsd upon system server restarts.");
int64_t systemServerRestartNs = getElapsedRealtimeNs();
ProtoOutputStream proto;
ProtoOutputStream activeConfigsProto;
mProcessor->WriteActiveConfigsToProtoOutputStream(systemServerRestartNs,
STATSCOMPANION_DIED, &proto);
STATSCOMPANION_DIED, &activeConfigsProto);
metadata::StatsMetadataList metadataList;
mProcessor->WriteMetadataToProto(getWallClockNs(),
systemServerRestartNs, &metadataList);
mProcessor->WriteDataToDisk(STATSCOMPANION_DIED, FAST);
mProcessor->resetConfigs();
std::string serializedActiveConfigs;
if (proto.serializeToString(&serializedActiveConfigs)) {
if (activeConfigsProto.serializeToString(&serializedActiveConfigs)) {
ActiveConfigList activeConfigs;
if (activeConfigs.ParseFromString(serializedActiveConfigs)) {
mProcessor->SetConfigsActiveState(activeConfigs, systemServerRestartNs);
}
}
mProcessor->SetMetadataState(metadataList, getWallClockNs(), systemServerRestartNs);
}
mAnomalyAlarmMonitor->setStatsCompanionService(nullptr);
mPeriodicAlarmMonitor->setStatsCompanionService(nullptr);

56
cmds/statsd/src/anomaly/AnomalyTracker.cpp
View File

@@ -18,9 +18,11 @@
#include "Log.h"
#include "AnomalyTracker.h"
#include "subscriber_util.h"
#include "external/Perfetto.h"
#include "guardrail/StatsdStats.h"
#include "metadata_util.h"
#include "stats_log_util.h"
#include "subscriber_util.h"
#include "subscriber/IncidentdReporter.h"
#include "subscriber/SubscriberReporter.h"
@@ -262,6 +264,58 @@ void AnomalyTracker::informSubscribers(const MetricDimensionKey& key, int64_t me
triggerSubscribers(mAlert.id(), metric_id, key, metricValue, mConfigKey, mSubscriptions);
}
bool AnomalyTracker::writeAlertMetadataToProto(int64_t currentWallClockTimeNs,
int64_t systemElapsedTimeNs,
metadata::AlertMetadata* alertMetadata) {
bool metadataWritten = false;
if (mRefractoryPeriodEndsSec.empty()) {
return false;
}
for (const auto& it: mRefractoryPeriodEndsSec) {
// Do not write the timestamp to disk if it has already expired
if (it.second < systemElapsedTimeNs / NS_PER_SEC) {
continue;
}
metadataWritten = true;
if (alertMetadata->alert_dim_keyed_data_size() == 0) {
alertMetadata->set_alert_id(mAlert.id());
}
metadata::AlertDimensionKeyedData* keyedData = alertMetadata->add_alert_dim_keyed_data();
// We convert and write the refractory_end_sec to wall clock time because we do not know
// when statsd will start again.
int32_t refractoryEndWallClockSec = (int32_t) ((currentWallClockTimeNs / NS_PER_SEC) +
(it.second - systemElapsedTimeNs / NS_PER_SEC));
keyedData->set_last_refractory_ends_sec(refractoryEndWallClockSec);
writeMetricDimensionKeyToMetadataDimensionKey(
it.first, keyedData->mutable_dimension_key());
}
return metadataWritten;
}
void AnomalyTracker::loadAlertMetadata(
const metadata::AlertMetadata& alertMetadata,
int64_t currentWallClockTimeNs,
int64_t systemElapsedTimeNs) {
for (const metadata::AlertDimensionKeyedData& keyedData :
alertMetadata.alert_dim_keyed_data()) {
if ((uint64_t) keyedData.last_refractory_ends_sec() < currentWallClockTimeNs / NS_PER_SEC) {
// Do not update the timestamp if it has already expired.
continue;
}
MetricDimensionKey metricKey = loadMetricDimensionKeyFromProto(
keyedData.dimension_key());
int32_t refractoryPeriodEndsSec = (int32_t) keyedData.last_refractory_ends_sec() -
currentWallClockTimeNs / NS_PER_SEC + systemElapsedTimeNs / NS_PER_SEC;
mRefractoryPeriodEndsSec[metricKey] = refractoryPeriodEndsSec;
}
}
} // namespace statsd
} // namespace os
} // namespace android

12
cmds/statsd/src/anomaly/AnomalyTracker.h
View File

@@ -24,6 +24,7 @@
#include "AlarmMonitor.h"
#include "config/ConfigKey.h"
#include "frameworks/base/cmds/statsd/src/statsd_config.pb.h" // Alert
#include "frameworks/base/cmds/statsd/src/statsd_metadata.pb.h" // AlertMetadata
#include "stats_util.h" // HashableDimensionKey and DimToValMap
namespace android {
@@ -112,6 +113,17 @@ public:
return; // The base AnomalyTracker class doesn't have alarms.
}
// Writes metadata of the alert (refractory_period_end_sec) to AlertMetadata.
// Returns true if at least one element is written to alertMetadata.
bool writeAlertMetadataToProto(
int64_t currentWallClockTimeNs,
int64_t systemElapsedTimeNs, metadata::AlertMetadata* alertMetadata);
void loadAlertMetadata(
const metadata::AlertMetadata& alertMetadata,
int64_t currentWallClockTimeNs,
int64_t systemElapsedTimeNs);
protected:
// For testing only.
// Returns the alarm timestamp in seconds for the query dimension if it exists. Otherwise

122
cmds/statsd/src/metadata_util.cpp Normal file
View File

@@ -0,0 +1,122 @@
/*
* Copyright (C) 2020 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "FieldValue.h"
#include "metadata_util.h"
namespace android {
namespace os {
namespace statsd {
using google::protobuf::RepeatedPtrField;
void writeValueToProto(metadata::FieldValue* metadataFieldValue, const Value& value) {
std::string storage_value;
switch (value.getType()) {
case INT:
metadataFieldValue->set_value_int(value.int_value);
break;
case LONG:
metadataFieldValue->set_value_long(value.long_value);
break;
case FLOAT:
metadataFieldValue->set_value_float(value.float_value);
break;
case DOUBLE:
metadataFieldValue->set_value_double(value.double_value);
break;
case STRING:
metadataFieldValue->set_value_str(value.str_value.c_str());
break;
case STORAGE: // byte array
storage_value = ((char*) value.storage_value.data());
metadataFieldValue->set_value_storage(storage_value);
break;
default:
break;
}
}
void writeMetricDimensionKeyToMetadataDimensionKey(
const MetricDimensionKey& metricKey,
metadata::MetricDimensionKey* metadataMetricKey) {
for (const FieldValue& fieldValue : metricKey.getDimensionKeyInWhat().getValues()) {
metadata::FieldValue* metadataFieldValue = metadataMetricKey->add_dimension_key_in_what();
metadata::Field* metadataField = metadataFieldValue->mutable_field();
metadataField->set_tag(fieldValue.mField.getTag());
metadataField->set_field(fieldValue.mField.getField());
writeValueToProto(metadataFieldValue, fieldValue.mValue);
}
for (const FieldValue& fieldValue : metricKey.getStateValuesKey().getValues()) {
metadata::FieldValue* metadataFieldValue = metadataMetricKey->add_state_values_key();
metadata::Field* metadataField = metadataFieldValue->mutable_field();
metadataField->set_tag(fieldValue.mField.getTag());
metadataField->set_field(fieldValue.mField.getField());
writeValueToProto(metadataFieldValue, fieldValue.mValue);
}
}
void writeFieldValuesFromMetadata(
const RepeatedPtrField<metadata::FieldValue>& repeatedFieldValueList,
std::vector<FieldValue>* fieldValues) {
for (const metadata::FieldValue& metadataFieldValue : repeatedFieldValueList) {
Field field(metadataFieldValue.field().tag(), metadataFieldValue.field().field());
Value value;
switch (metadataFieldValue.value_case()) {
case metadata::FieldValue::ValueCase::kValueInt:
value = Value(metadataFieldValue.value_int());
break;
case metadata::FieldValue::ValueCase::kValueLong:
value = Value(metadataFieldValue.value_long());
break;
case metadata::FieldValue::ValueCase::kValueFloat:
value = Value(metadataFieldValue.value_float());
break;
case metadata::FieldValue::ValueCase::kValueDouble:
value = Value(metadataFieldValue.value_double());
break;
case metadata::FieldValue::ValueCase::kValueStr:
value = Value(metadataFieldValue.value_str());
break;
case metadata::FieldValue::ValueCase::kValueStorage:
value = Value(metadataFieldValue.value_storage());
break;
default:
break;
}
FieldValue fieldValue(field, value);
fieldValues->emplace_back(field, value);
}
}
MetricDimensionKey loadMetricDimensionKeyFromProto(
const metadata::MetricDimensionKey& metricDimensionKey) {
std::vector<FieldValue> dimKeyInWhatFieldValues;
writeFieldValuesFromMetadata(metricDimensionKey.dimension_key_in_what(),
&dimKeyInWhatFieldValues);
std::vector<FieldValue> stateValuesFieldValues;
writeFieldValuesFromMetadata(metricDimensionKey.state_values_key(), &stateValuesFieldValues);
HashableDimensionKey dimKeyInWhat(dimKeyInWhatFieldValues);
HashableDimensionKey stateValues(stateValuesFieldValues);
MetricDimensionKey metricKey(dimKeyInWhat, stateValues);
return metricKey;
}
} // namespace statsd
} // namespace os
} // namespace android

32
cmds/statsd/src/metadata_util.h Normal file
View File

@@ -0,0 +1,32 @@
/*
* Copyright (C) 2020 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "HashableDimensionKey.h"
#include "frameworks/base/cmds/statsd/src/statsd_metadata.pb.h" // AlertMetadata
namespace android {
namespace os {
namespace statsd {
void writeMetricDimensionKeyToMetadataDimensionKey(const MetricDimensionKey& metricKey,
metadata::MetricDimensionKey* metadataMetricKey);
MetricDimensionKey loadMetricDimensionKeyFromProto(
const metadata::MetricDimensionKey& metricDimensionKey);
} // namespace statsd
} // namespace os
} // namespace android

34
cmds/statsd/src/metrics/MetricsManager.cpp
View File

@@ -642,8 +642,40 @@ void MetricsManager::writeActiveConfigToProtoOutputStream(
}
}
bool MetricsManager::writeMetadataToProto(int64_t currentWallClockTimeNs,
int64_t systemElapsedTimeNs,
metadata::StatsMetadata* statsMetadata) {
bool metadataWritten = false;
metadata::ConfigKey* configKey = statsMetadata->mutable_config_key();
configKey->set_config_id(mConfigKey.GetId());
configKey->set_uid(mConfigKey.GetUid());
for (const auto& anomalyTracker : mAllAnomalyTrackers) {
metadata::AlertMetadata* alertMetadata = statsMetadata->add_alert_metadata();
bool alertWritten = anomalyTracker->writeAlertMetadataToProto(currentWallClockTimeNs,
systemElapsedTimeNs, alertMetadata);
if (!alertWritten) {
statsMetadata->mutable_alert_metadata()->RemoveLast();
}
metadataWritten |= alertWritten;
}
return metadataWritten;
}
void MetricsManager::loadMetadata(const metadata::StatsMetadata& metadata,
int64_t currentWallClockTimeNs,
int64_t systemElapsedTimeNs) {
for (const metadata::AlertMetadata& alertMetadata : metadata.alert_metadata()) {
int64_t alertId = alertMetadata.alert_id();
auto it = mAlertTrackerMap.find(alertId);
if (it == mAlertTrackerMap.end()) {
ALOGE("No anomalyTracker found for alertId %lld", (long long) alertId);
continue;
}
mAllAnomalyTrackers[it->second]->loadAlertMetadata(alertMetadata,
currentWallClockTimeNs,
systemElapsedTimeNs);
}
}
} // namespace statsd
} // namespace os

12
cmds/statsd/src/metrics/MetricsManager.h
View File

@@ -23,6 +23,7 @@
#include "config/ConfigKey.h"
#include "external/StatsPullerManager.h"
#include "frameworks/base/cmds/statsd/src/statsd_config.pb.h"
#include "frameworks/base/cmds/statsd/src/statsd_metadata.pb.h"
#include "logd/LogEvent.h"
#include "matchers/LogMatchingTracker.h"
#include "metrics/MetricProducer.h"
@@ -143,6 +144,14 @@ public:
void writeActiveConfigToProtoOutputStream(
int64_t currentTimeNs, const DumpReportReason reason, ProtoOutputStream* proto);
// Returns true if at least one piece of metadata is written.
bool writeMetadataToProto(int64_t currentWallClockTimeNs,
int64_t systemElapsedTimeNs,
metadata::StatsMetadata* statsMetadata);
void loadMetadata(const metadata::StatsMetadata& metadata,
int64_t currentWallClockTimeNs,
int64_t systemElapsedTimeNs);
private:
// For test only.
inline int64_t getTtlEndNs() const { return mTtlEndNs; }
@@ -285,6 +294,9 @@ private:
FRIEND_TEST(AnomalyDetectionE2eTest, TestSlicedCountMetric_single_bucket);
FRIEND_TEST(AnomalyDetectionE2eTest, TestSlicedCountMetric_multiple_buckets);
FRIEND_TEST(AnomalyDetectionE2eTest, TestCountMetric_save_refractory_to_disk_no_data_written);
FRIEND_TEST(AnomalyDetectionE2eTest, TestCountMetric_save_refractory_to_disk);
FRIEND_TEST(AnomalyDetectionE2eTest, TestCountMetric_load_refractory_from_disk);
FRIEND_TEST(AnomalyDetectionE2eTest, TestDurationMetric_SUM_single_bucket);
FRIEND_TEST(AnomalyDetectionE2eTest, TestDurationMetric_SUM_multiple_buckets);
FRIEND_TEST(AnomalyDetectionE2eTest, TestDurationMetric_SUM_long_refractory_period);

10
cmds/statsd/src/statsd_metadata.proto
View File

@@ -45,11 +45,15 @@ message MetricDimensionKey {
repeated FieldValue state_values_key = 2;
}
message AlertDimensionKeyedData {
// The earliest time the alert can be fired again in wall clock time.
optional int32 last_refractory_ends_sec = 1;
optional MetricDimensionKey dimension_key = 2;
}
message AlertMetadata {
optional int64 alert_id = 1;
// The earliest time the alert can be fired again in wall clock time.
optional int32 last_refractory_ends_sec = 2;
optional MetricDimensionKey dimension_key = 3;
repeated AlertDimensionKeyedData alert_dim_keyed_data = 2;
}
// All metadata for a config in statsd

153
cmds/statsd/tests/e2e/Anomaly_count_e2e_test.cpp
View File

@@ -14,6 +14,7 @@
#include <gtest/gtest.h>
#include "frameworks/base/cmds/statsd/src/statsd_metadata.pb.h"
#include "src/StatsLogProcessor.h"
#include "src/stats_log_util.h"
#include "tests/statsd_test_util.h"
@@ -28,7 +29,7 @@ namespace statsd {
namespace {
StatsdConfig CreateStatsdConfig(int num_buckets, int threshold) {
StatsdConfig CreateStatsdConfig(int num_buckets, int threshold, int refractory_period_sec) {
StatsdConfig config;
config.add_allowed_log_source("AID_ROOT"); // LogEvent defaults to UID of root.
auto wakelockAcquireMatcher = CreateAcquireWakelockAtomMatcher();
@@ -46,7 +47,7 @@ StatsdConfig CreateStatsdConfig(int num_buckets, int threshold) {
alert->set_id(StringToId("alert"));
alert->set_metric_id(123456);
alert->set_num_buckets(num_buckets);
alert->set_refractory_period_secs(10);
alert->set_refractory_period_secs(refractory_period_sec);
alert->set_trigger_if_sum_gt(threshold);
return config;
}
@@ -56,9 +57,9 @@ StatsdConfig CreateStatsdConfig(int num_buckets, int threshold) {
TEST(AnomalyDetectionE2eTest, TestSlicedCountMetric_single_bucket) {
const int num_buckets = 1;
const int threshold = 3;
auto config = CreateStatsdConfig(num_buckets, threshold);
const int refractory_period_sec = 10;
auto config = CreateStatsdConfig(num_buckets, threshold, refractory_period_sec);
const uint64_t alert_id = config.alert(0).id();
const uint32_t refractory_period_sec = config.alert(0).refractory_period_secs();
int64_t bucketStartTimeNs = 10000000000;
int64_t bucketSizeNs = TimeUnitToBucketSizeInMillis(config.count_metric(0).bucket()) * 1000000;
@@ -173,9 +174,9 @@ TEST(AnomalyDetectionE2eTest, TestSlicedCountMetric_single_bucket) {
TEST(AnomalyDetectionE2eTest, TestSlicedCountMetric_multiple_buckets) {
const int num_buckets = 3;
const int threshold = 3;
auto config = CreateStatsdConfig(num_buckets, threshold);
const int refractory_period_sec = 10;
auto config = CreateStatsdConfig(num_buckets, threshold, refractory_period_sec);
const uint64_t alert_id = config.alert(0).id();
const uint32_t refractory_period_sec = config.alert(0).refractory_period_secs();
int64_t bucketStartTimeNs = 10000000000;
int64_t bucketSizeNs = TimeUnitToBucketSizeInMillis(config.count_metric(0).bucket()) * 1000000;
@@ -240,6 +241,146 @@ TEST(AnomalyDetectionE2eTest, TestSlicedCountMetric_multiple_buckets) {
anomalyTracker->getRefractoryPeriodEndsSec(dimensionKey1));
}
TEST(AnomalyDetectionE2eTest, TestCountMetric_save_refractory_to_disk_no_data_written) {
const int num_buckets = 1;
const int threshold = 0;
const int refractory_period_sec = 86400 * 365; // 1 year
auto config = CreateStatsdConfig(num_buckets, threshold, refractory_period_sec);
const int64_t alert_id = config.alert(0).id();
int64_t bucketStartTimeNs = 10000000000;
int configUid = 2000;
int64_t configId = 1000;
ConfigKey cfgKey(configUid, configId);
auto processor = CreateStatsLogProcessor(bucketStartTimeNs, bucketStartTimeNs, config, cfgKey);
EXPECT_EQ(processor->mMetricsManagers.size(), 1u);
EXPECT_TRUE(processor->mMetricsManagers.begin()->second->isConfigValid());
EXPECT_EQ(1u, processor->mMetricsManagers.begin()->second->mAllAnomalyTrackers.size());
metadata::StatsMetadataList result;
int64_t mockWallClockNs = 1584991200 * NS_PER_SEC;
int64_t mockElapsedTimeNs = bucketStartTimeNs + 5000 * NS_PER_SEC;
processor->WriteMetadataToProto(mockWallClockNs, mockElapsedTimeNs, &result);
EXPECT_EQ(result.stats_metadata_size(), 0);
}
TEST(AnomalyDetectionE2eTest, TestCountMetric_save_refractory_to_disk) {
const int num_buckets = 1;
const int threshold = 0;
const int refractory_period_sec = 86400 * 365; // 1 year
auto config = CreateStatsdConfig(num_buckets, threshold, refractory_period_sec);
const int64_t alert_id = config.alert(0).id();
int64_t bucketStartTimeNs = 10000000000;
int configUid = 2000;
int64_t configId = 1000;
ConfigKey cfgKey(configUid, configId);
auto processor = CreateStatsLogProcessor(bucketStartTimeNs, bucketStartTimeNs, config, cfgKey);
EXPECT_EQ(processor->mMetricsManagers.size(), 1u);
EXPECT_TRUE(processor->mMetricsManagers.begin()->second->isConfigValid());
EXPECT_EQ(1u, processor->mMetricsManagers.begin()->second->mAllAnomalyTrackers.size());
sp<AnomalyTracker> anomalyTracker =
processor->mMetricsManagers.begin()->second->mAllAnomalyTrackers[0];
std::vector<int> attributionUids1 = {111};
std::vector<string> attributionTags1 = {"App1"};
std::vector<int> attributionUids2 = {111, 222};
std::vector<string> attributionTags2 = {"App1", "GMSCoreModule1"};
FieldValue fieldValue1(Field(util::WAKELOCK_STATE_CHANGED, (int32_t)0x02010101),
Value((int32_t)111));
HashableDimensionKey whatKey1({fieldValue1});
MetricDimensionKey dimensionKey1(whatKey1, DEFAULT_DIMENSION_KEY);
auto event = CreateAcquireWakelockEvent(bucketStartTimeNs + 2, attributionUids1,
attributionTags1, "wl1");
processor->OnLogEvent(event.get());
EXPECT_EQ(refractory_period_sec + (bucketStartTimeNs + 2) / NS_PER_SEC + 1,
anomalyTracker->getRefractoryPeriodEndsSec(dimensionKey1));
metadata::StatsMetadataList result;
int64_t mockWallClockNs = 1584991200 * NS_PER_SEC;
int64_t mockElapsedTimeNs = bucketStartTimeNs + 5000 * NS_PER_SEC;
processor->WriteMetadataToProto(mockWallClockNs, mockElapsedTimeNs, &result);
metadata::StatsMetadata statsMetadata = result.stats_metadata(0);
EXPECT_EQ(result.stats_metadata_size(), 1);
EXPECT_EQ(statsMetadata.config_key().config_id(), configId);
EXPECT_EQ(statsMetadata.config_key().uid(), configUid);
metadata::AlertMetadata alertMetadata = statsMetadata.alert_metadata(0);
EXPECT_EQ(statsMetadata.alert_metadata_size(), 1);
EXPECT_EQ(alertMetadata.alert_id(), alert_id);
metadata::AlertDimensionKeyedData keyedData = alertMetadata.alert_dim_keyed_data(0);
EXPECT_EQ(alertMetadata.alert_dim_keyed_data_size(), 1);
EXPECT_EQ(keyedData.last_refractory_ends_sec(),
anomalyTracker->getRefractoryPeriodEndsSec(dimensionKey1) -
mockElapsedTimeNs / NS_PER_SEC +
mockWallClockNs / NS_PER_SEC);
metadata::MetricDimensionKey metadataDimKey = keyedData.dimension_key();
metadata::FieldValue dimKeyInWhat = metadataDimKey.dimension_key_in_what(0);
EXPECT_EQ(dimKeyInWhat.field().tag(), fieldValue1.mField.getTag());
EXPECT_EQ(dimKeyInWhat.field().field(), fieldValue1.mField.getField());
EXPECT_EQ(dimKeyInWhat.value_int(), fieldValue1.mValue.int_value);
}
TEST(AnomalyDetectionE2eTest, TestCountMetric_load_refractory_from_disk) {
const int num_buckets = 1;
const int threshold = 0;
const int refractory_period_sec = 86400 * 365; // 1 year
auto config = CreateStatsdConfig(num_buckets, threshold, refractory_period_sec);
const int64_t alert_id = config.alert(0).id();
int64_t bucketStartTimeNs = 10000000000;
int configUid = 2000;
int64_t configId = 1000;
ConfigKey cfgKey(configUid, configId);
auto processor = CreateStatsLogProcessor(bucketStartTimeNs, bucketStartTimeNs, config, cfgKey);
EXPECT_EQ(processor->mMetricsManagers.size(), 1u);
EXPECT_TRUE(processor->mMetricsManagers.begin()->second->isConfigValid());
EXPECT_EQ(1u, processor->mMetricsManagers.begin()->second->mAllAnomalyTrackers.size());
sp<AnomalyTracker> anomalyTracker =
processor->mMetricsManagers.begin()->second->mAllAnomalyTrackers[0];
std::vector<int> attributionUids1 = {111};
std::vector<string> attributionTags1 = {"App1"};
std::vector<int> attributionUids2 = {111, 222};
std::vector<string> attributionTags2 = {"App1", "GMSCoreModule1"};
FieldValue fieldValue1(Field(util::WAKELOCK_STATE_CHANGED, (int32_t)0x02010101),
Value((int32_t)111));
HashableDimensionKey whatKey1({fieldValue1});
MetricDimensionKey dimensionKey1(whatKey1, DEFAULT_DIMENSION_KEY);
auto event = CreateAcquireWakelockEvent(bucketStartTimeNs + 2, attributionUids1,
attributionTags1, "wl1");
processor->OnLogEvent(event.get());
EXPECT_EQ(refractory_period_sec + (bucketStartTimeNs + 2) / NS_PER_SEC + 1,
anomalyTracker->getRefractoryPeriodEndsSec(dimensionKey1));
int64_t mockWallClockNs = 1584991200 * NS_PER_SEC;
int64_t mockElapsedTimeNs = bucketStartTimeNs + 5000 * NS_PER_SEC;
processor->SaveMetadataToDisk(mockWallClockNs, mockElapsedTimeNs);
auto processor2 = CreateStatsLogProcessor(bucketStartTimeNs, bucketStartTimeNs, config, cfgKey);
int64_t mockElapsedTimeSinceBoot = 10 * NS_PER_SEC;
processor2->LoadMetadataFromDisk(mockWallClockNs, mockElapsedTimeSinceBoot);
sp<AnomalyTracker> anomalyTracker2 =
processor2->mMetricsManagers.begin()->second->mAllAnomalyTrackers[0];
EXPECT_EQ(anomalyTracker2->getRefractoryPeriodEndsSec(dimensionKey1) -
mockElapsedTimeSinceBoot / NS_PER_SEC,
anomalyTracker->getRefractoryPeriodEndsSec(dimensionKey1) -
mockElapsedTimeNs / NS_PER_SEC);
}
#else
GTEST_LOG_(INFO) << "This test does nothing.\n";
#endif

69
cmds/statsd/tests/metadata_util_test.cpp Normal file
View File

@@ -0,0 +1,69 @@
/*
* Copyright (C) 2020 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <gtest/gtest.h>
#include "metadata_util.h"
#include "tests/statsd_test_util.h"
#ifdef __ANDROID__
namespace android {
namespace os {
namespace statsd {
TEST(MetadataUtilTest, TestWriteAndReadMetricDimensionKey) {
HashableDimensionKey dim;
HashableDimensionKey dim2;
int pos1[] = {1, 1, 1};
int pos2[] = {1, 1, 2};
int pos3[] = {1, 1, 3};
int pos4[] = {2, 0, 0};
Field field1(10, pos1, 2);
Field field2(10, pos2, 2);
Field field3(10, pos3, 2);
Field field4(10, pos4, 0);
Value value1((int32_t)10025);
Value value2("tag");
Value value3((int32_t)987654);
Value value4((int32_t)99999);
dim.addValue(FieldValue(field1, value1));
dim.addValue(FieldValue(field2, value2));
dim.addValue(FieldValue(field3, value3));
dim.addValue(FieldValue(field4, value4));
dim2.addValue(FieldValue(field1, value1));
dim2.addValue(FieldValue(field2, value2));
MetricDimensionKey dimKey(dim, dim2);
metadata::MetricDimensionKey metadataDimKey;
writeMetricDimensionKeyToMetadataDimensionKey(dimKey, &metadataDimKey);
MetricDimensionKey loadedDimKey = loadMetricDimensionKeyFromProto(metadataDimKey);
ASSERT_EQ(loadedDimKey, dimKey);
ASSERT_EQ(std::hash<MetricDimensionKey>{}(loadedDimKey),
std::hash<MetricDimensionKey>{}(dimKey));
}
} // namespace statsd
} // namespace os
} // namespace android
#else
GTEST_LOG_(INFO) << "This test does nothing.\n";
#endif