Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Load Metadata from Disk

Browse Source 
Bug: 148280505
Test: bit statsd_test:*
Test: Manual - Added Config, Tested System Server Crash
Test: Manual - Added Config, Tested Device restart
Change-Id: Icd1b1e57bbf24e4a5676820a2b955f2034d4b37d
This commit is contained in:
Jeffrey Huang
2020-03-30 19:52:07 -07:00
parent b8f5403d17
commit 475677e771
12 changed files with 290 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
cmds/statsd/Android.bp
View File

@@ -340,6 +340,7 @@ cc_test {
"tests/log_event/LogEventQueue_test.cpp",
"tests/LogEntryMatcher_test.cpp",
"tests/LogEvent_test.cpp",
"tests/metadata_util_test.cpp",
"tests/metrics/CountMetricProducer_test.cpp",
"tests/metrics/DurationMetricProducer_test.cpp",
"tests/metrics/EventMetricProducer_test.cpp",

54
cmds/statsd/src/StatsLogProcessor.cpp
View File

@@ -902,6 +902,60 @@ void StatsLogProcessor::WriteMetadataToProtoLocked(int64_t currentWallClockTimeN
}
}
void StatsLogProcessor::LoadMetadataFromDisk(int64_t currentWallClockTimeNs,
int64_t systemElapsedTimeNs) {
std::lock_guard<std::mutex> lock(mMetricsMutex);
string file_name = StringPrintf("%s/metadata", STATS_METADATA_DIR);
int fd = open(file_name.c_str(), O_RDONLY | O_CLOEXEC);
if (-1 == fd) {
VLOG("Attempt to read %s but failed", file_name.c_str());
StorageManager::deleteFile(file_name.c_str());
return;
}
string content;
if (!android::base::ReadFdToString(fd, &content)) {
ALOGE("Attempt to read %s but failed", file_name.c_str());
close(fd);
StorageManager::deleteFile(file_name.c_str());
return;
}
close(fd);
metadata::StatsMetadataList statsMetadataList;
if (!statsMetadataList.ParseFromString(content)) {
ALOGE("Attempt to read %s but failed; failed to metadata", file_name.c_str());
StorageManager::deleteFile(file_name.c_str());
return;
}
SetMetadataStateLocked(statsMetadataList, currentWallClockTimeNs, systemElapsedTimeNs);
StorageManager::deleteFile(file_name.c_str());
}
void StatsLogProcessor::SetMetadataState(const metadata::StatsMetadataList& statsMetadataList,
int64_t currentWallClockTimeNs,
int64_t systemElapsedTimeNs) {
std::lock_guard<std::mutex> lock(mMetricsMutex);
SetMetadataStateLocked(statsMetadataList, currentWallClockTimeNs, systemElapsedTimeNs);
}
void StatsLogProcessor::SetMetadataStateLocked(
const metadata::StatsMetadataList& statsMetadataList,
int64_t currentWallClockTimeNs,
int64_t systemElapsedTimeNs) {
for (const metadata::StatsMetadata& metadata : statsMetadataList.stats_metadata()) {
ConfigKey key(metadata.config_key().uid(), metadata.config_key().config_id());
auto it = mMetricsManagers.find(key);
if (it == mMetricsManagers.end()) {
ALOGE("No config found for configKey %s", key.ToString().c_str());
continue;
}
VLOG("Setting metadata %s", key.ToString().c_str());
it->second->loadMetadata(metadata, currentWallClockTimeNs, systemElapsedTimeNs);
}
VLOG("Successfully loaded %d metadata.", statsMetadataList.stats_metadata_size());
}
void StatsLogProcessor::WriteActiveConfigsToProtoOutputStream(
int64_t currentTimeNs, const DumpReportReason reason, ProtoOutputStream* proto) {
std::lock_guard<std::mutex> lock(mMetricsMutex);

14
cmds/statsd/src/StatsLogProcessor.h
View File

@@ -98,6 +98,15 @@ public:
int64_t systemElapsedTimeNs,
metadata::StatsMetadataList* metadataList);
/* Load stats metadata for configs and metrics from disk. */
void LoadMetadataFromDisk(int64_t currentWallClockTimeNs,
int64_t systemElapsedTimeNs);
/* Sets the metadata for all configs and metrics */
void SetMetadataState(const metadata::StatsMetadataList& statsMetadataList,
int64_t currentWallClockTimeNs,
int64_t systemElapsedTimeNs);
/* Sets the active status/ttl for all configs and metrics to the status in ActiveConfigList. */
void SetConfigsActiveState(const ActiveConfigList& activeConfigList, int64_t currentTimeNs);
@@ -182,6 +191,10 @@ private:
void SetConfigsActiveStateLocked(const ActiveConfigList& activeConfigList,
int64_t currentTimeNs);
void SetMetadataStateLocked(const metadata::StatsMetadataList& statsMetadataList,
int64_t currentWallClockTimeNs,
int64_t systemElapsedTimeNs);
void WriteMetadataToProtoLocked(int64_t currentWallClockTimeNs,
int64_t systemElapsedTimeNs,
metadata::StatsMetadataList* metadataList);
@@ -297,6 +310,7 @@ private:
FRIEND_TEST(AnomalyDetectionE2eTest, TestSlicedCountMetric_multiple_buckets);
FRIEND_TEST(AnomalyDetectionE2eTest, TestCountMetric_save_refractory_to_disk_no_data_written);
FRIEND_TEST(AnomalyDetectionE2eTest, TestCountMetric_save_refractory_to_disk);
FRIEND_TEST(AnomalyDetectionE2eTest, TestCountMetric_load_refractory_from_disk);
FRIEND_TEST(AnomalyDetectionE2eTest, TestDurationMetric_SUM_single_bucket);
FRIEND_TEST(AnomalyDetectionE2eTest, TestDurationMetric_SUM_multiple_buckets);
FRIEND_TEST(AnomalyDetectionE2eTest, TestDurationMetric_SUM_long_refractory_period);

2
cmds/statsd/src/StatsService.cpp
View File

@@ -1042,6 +1042,7 @@ Status StatsService::statsCompanionReady() {
void StatsService::Startup() {
mConfigManager->Startup();
mProcessor->LoadActiveConfigsFromDisk();
mProcessor->LoadMetadataFromDisk(getWallClockNs(), getElapsedRealtimeNs());
}
void StatsService::Terminate() {
@@ -1298,6 +1299,7 @@ void StatsService::statsCompanionServiceDiedImpl() {
mProcessor->SetConfigsActiveState(activeConfigs, systemServerRestartNs);
}
}
mProcessor->SetMetadataState(metadataList, getWallClockNs(), systemServerRestartNs);
}
mAnomalyAlarmMonitor->setStatsCompanionService(nullptr);
mPeriodicAlarmMonitor->setStatsCompanionService(nullptr);

18
cmds/statsd/src/anomaly/AnomalyTracker.cpp
View File

@@ -298,6 +298,24 @@ bool AnomalyTracker::writeAlertMetadataToProto(int64_t currentWallClockTimeNs,
return metadataWritten;
}
void AnomalyTracker::loadAlertMetadata(
const metadata::AlertMetadata& alertMetadata,
int64_t currentWallClockTimeNs,
int64_t systemElapsedTimeNs) {
for (const metadata::AlertDimensionKeyedData& keyedData :
alertMetadata.alert_dim_keyed_data()) {
if ((uint64_t) keyedData.last_refractory_ends_sec() < currentWallClockTimeNs / NS_PER_SEC) {
// Do not update the timestamp if it has already expired.
continue;
}
MetricDimensionKey metricKey = loadMetricDimensionKeyFromProto(
keyedData.dimension_key());
int32_t refractoryPeriodEndsSec = (int32_t) keyedData.last_refractory_ends_sec() -
currentWallClockTimeNs / NS_PER_SEC + systemElapsedTimeNs / NS_PER_SEC;
mRefractoryPeriodEndsSec[metricKey] = refractoryPeriodEndsSec;
}
}
} // namespace statsd
} // namespace os
} // namespace android

7
cmds/statsd/src/anomaly/AnomalyTracker.h
View File

@@ -24,7 +24,7 @@
#include "AlarmMonitor.h"
#include "config/ConfigKey.h"
#include "frameworks/base/cmds/statsd/src/statsd_config.pb.h" // Alert
#include "frameworks/base/cmds/statsd/src/statsd_metadata.pb.h" // AlertMetadata
#include "frameworks/base/cmds/statsd/src/statsd_metadata.pb.h" // AlertMetadata
#include "stats_util.h" // HashableDimensionKey and DimToValMap
namespace android {
@@ -119,6 +119,11 @@ public:
int64_t currentWallClockTimeNs,
int64_t systemElapsedTimeNs, metadata::AlertMetadata* alertMetadata);
void loadAlertMetadata(
const metadata::AlertMetadata& alertMetadata,
int64_t currentWallClockTimeNs,
int64_t systemElapsedTimeNs);
protected:
// For testing only.
// Returns the alarm timestamp in seconds for the query dimension if it exists. Otherwise

49
cmds/statsd/src/metadata_util.cpp
View File

@@ -21,6 +21,8 @@ namespace android {
namespace os {
namespace statsd {
using google::protobuf::RepeatedPtrField;
void writeValueToProto(metadata::FieldValue* metadataFieldValue, const Value& value) {
std::string storage_value;
switch (value.getType()) {
@@ -68,6 +70,53 @@ void writeMetricDimensionKeyToMetadataDimensionKey(
}
}
void writeFieldValuesFromMetadata(
const RepeatedPtrField<metadata::FieldValue>& repeatedFieldValueList,
std::vector<FieldValue>* fieldValues) {
for (const metadata::FieldValue& metadataFieldValue : repeatedFieldValueList) {
Field field(metadataFieldValue.field().tag(), metadataFieldValue.field().field());
Value value;
switch (metadataFieldValue.value_case()) {
case metadata::FieldValue::ValueCase::kValueInt:
value = Value(metadataFieldValue.value_int());
break;
case metadata::FieldValue::ValueCase::kValueLong:
value = Value(metadataFieldValue.value_long());
break;
case metadata::FieldValue::ValueCase::kValueFloat:
value = Value(metadataFieldValue.value_float());
break;
case metadata::FieldValue::ValueCase::kValueDouble:
value = Value(metadataFieldValue.value_double());
break;
case metadata::FieldValue::ValueCase::kValueStr:
value = Value(metadataFieldValue.value_str());
break;
case metadata::FieldValue::ValueCase::kValueStorage:
value = Value(metadataFieldValue.value_storage());
break;
default:
break;
}
FieldValue fieldValue(field, value);
fieldValues->emplace_back(field, value);
}
}
MetricDimensionKey loadMetricDimensionKeyFromProto(
const metadata::MetricDimensionKey& metricDimensionKey) {
std::vector<FieldValue> dimKeyInWhatFieldValues;
writeFieldValuesFromMetadata(metricDimensionKey.dimension_key_in_what(),
&dimKeyInWhatFieldValues);
std::vector<FieldValue> stateValuesFieldValues;
writeFieldValuesFromMetadata(metricDimensionKey.state_values_key(), &stateValuesFieldValues);
HashableDimensionKey dimKeyInWhat(dimKeyInWhatFieldValues);
HashableDimensionKey stateValues(stateValuesFieldValues);
MetricDimensionKey metricKey(dimKeyInWhat, stateValues);
return metricKey;
}
} // namespace statsd
} // namespace os
} // namespace android

3
cmds/statsd/src/metadata_util.h
View File

@@ -24,6 +24,9 @@ namespace statsd {
void writeMetricDimensionKeyToMetadataDimensionKey(const MetricDimensionKey& metricKey,
metadata::MetricDimensionKey* metadataMetricKey);
MetricDimensionKey loadMetricDimensionKeyFromProto(
const metadata::MetricDimensionKey& metricDimensionKey);
} // namespace statsd
} // namespace os
} // namespace android

16
cmds/statsd/src/metrics/MetricsManager.cpp
View File

@@ -579,6 +579,22 @@ bool MetricsManager::writeMetadataToProto(int64_t currentWallClockTimeNs,
return metadataWritten;
}
void MetricsManager::loadMetadata(const metadata::StatsMetadata& metadata,
int64_t currentWallClockTimeNs,
int64_t systemElapsedTimeNs) {
for (const metadata::AlertMetadata& alertMetadata : metadata.alert_metadata()) {
int64_t alertId = alertMetadata.alert_id();
auto it = mAlertTrackerMap.find(alertId);
if (it == mAlertTrackerMap.end()) {
ALOGE("No anomalyTracker found for alertId %lld", (long long) alertId);
continue;
}
mAllAnomalyTrackers[it->second]->loadAlertMetadata(alertMetadata,
currentWallClockTimeNs,
systemElapsedTimeNs);
}
}
} // namespace statsd
} // namespace os
} // namespace android

5
cmds/statsd/src/metrics/MetricsManager.h
View File

@@ -144,6 +144,10 @@ public:
bool writeMetadataToProto(int64_t currentWallClockTimeNs,
int64_t systemElapsedTimeNs,
metadata::StatsMetadata* statsMetadata);
void loadMetadata(const metadata::StatsMetadata& metadata,
int64_t currentWallClockTimeNs,
int64_t systemElapsedTimeNs);
private:
// For test only.
inline int64_t getTtlEndNs() const { return mTtlEndNs; }
@@ -270,6 +274,7 @@ private:
FRIEND_TEST(AnomalyDetectionE2eTest, TestSlicedCountMetric_multiple_buckets);
FRIEND_TEST(AnomalyDetectionE2eTest, TestCountMetric_save_refractory_to_disk_no_data_written);
FRIEND_TEST(AnomalyDetectionE2eTest, TestCountMetric_save_refractory_to_disk);
FRIEND_TEST(AnomalyDetectionE2eTest, TestCountMetric_load_refractory_from_disk);
FRIEND_TEST(AnomalyDetectionE2eTest, TestDurationMetric_SUM_single_bucket);
FRIEND_TEST(AnomalyDetectionE2eTest, TestDurationMetric_SUM_multiple_buckets);
FRIEND_TEST(AnomalyDetectionE2eTest, TestDurationMetric_SUM_long_refractory_period);

53
cmds/statsd/tests/e2e/Anomaly_count_e2e_test.cpp
View File

@@ -328,6 +328,59 @@ TEST(AnomalyDetectionE2eTest, TestCountMetric_save_refractory_to_disk) {
EXPECT_EQ(dimKeyInWhat.field().field(), fieldValue1.mField.getField());
EXPECT_EQ(dimKeyInWhat.value_int(), fieldValue1.mValue.int_value);
}
TEST(AnomalyDetectionE2eTest, TestCountMetric_load_refractory_from_disk) {
const int num_buckets = 1;
const int threshold = 0;
const int refractory_period_sec = 86400 * 365; // 1 year
auto config = CreateStatsdConfig(num_buckets, threshold, refractory_period_sec);
const int64_t alert_id = config.alert(0).id();
int64_t bucketStartTimeNs = 10000000000;
int configUid = 2000;
int64_t configId = 1000;
ConfigKey cfgKey(configUid, configId);
auto processor = CreateStatsLogProcessor(bucketStartTimeNs, bucketStartTimeNs, config, cfgKey);
EXPECT_EQ(processor->mMetricsManagers.size(), 1u);
EXPECT_TRUE(processor->mMetricsManagers.begin()->second->isConfigValid());
EXPECT_EQ(1u, processor->mMetricsManagers.begin()->second->mAllAnomalyTrackers.size());
sp<AnomalyTracker> anomalyTracker =
processor->mMetricsManagers.begin()->second->mAllAnomalyTrackers[0];
std::vector<int> attributionUids1 = {111};
std::vector<string> attributionTags1 = {"App1"};
std::vector<int> attributionUids2 = {111, 222};
std::vector<string> attributionTags2 = {"App1", "GMSCoreModule1"};
FieldValue fieldValue1(Field(util::WAKELOCK_STATE_CHANGED, (int32_t)0x02010101),
Value((int32_t)111));
HashableDimensionKey whatKey1({fieldValue1});
MetricDimensionKey dimensionKey1(whatKey1, DEFAULT_DIMENSION_KEY);
auto event = CreateAcquireWakelockEvent(bucketStartTimeNs + 2, attributionUids1,
attributionTags1, "wl1");
processor->OnLogEvent(event.get());
EXPECT_EQ(refractory_period_sec + (bucketStartTimeNs + 2) / NS_PER_SEC + 1,
anomalyTracker->getRefractoryPeriodEndsSec(dimensionKey1));
int64_t mockWallClockNs = 1584991200 * NS_PER_SEC;
int64_t mockElapsedTimeNs = bucketStartTimeNs + 5000 * NS_PER_SEC;
processor->SaveMetadataToDisk(mockWallClockNs, mockElapsedTimeNs);
auto processor2 = CreateStatsLogProcessor(bucketStartTimeNs, bucketStartTimeNs, config, cfgKey);
int64_t mockElapsedTimeSinceBoot = 10 * NS_PER_SEC;
processor2->LoadMetadataFromDisk(mockWallClockNs, mockElapsedTimeSinceBoot);
sp<AnomalyTracker> anomalyTracker2 =
processor2->mMetricsManagers.begin()->second->mAllAnomalyTrackers[0];
EXPECT_EQ(anomalyTracker2->getRefractoryPeriodEndsSec(dimensionKey1) -
mockElapsedTimeSinceBoot / NS_PER_SEC,
anomalyTracker->getRefractoryPeriodEndsSec(dimensionKey1) -
mockElapsedTimeNs / NS_PER_SEC);
}
#else
GTEST_LOG_(INFO) << "This test does nothing.\n";
#endif

69
cmds/statsd/tests/metadata_util_test.cpp Normal file
View File

@@ -0,0 +1,69 @@
/*
* Copyright (C) 2020 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <gtest/gtest.h>
#include "metadata_util.h"
#include "tests/statsd_test_util.h"
#ifdef __ANDROID__
namespace android {
namespace os {
namespace statsd {
TEST(MetadataUtilTest, TestWriteAndReadMetricDimensionKey) {
HashableDimensionKey dim;
HashableDimensionKey dim2;
int pos1[] = {1, 1, 1};
int pos2[] = {1, 1, 2};
int pos3[] = {1, 1, 3};
int pos4[] = {2, 0, 0};
Field field1(10, pos1, 2);
Field field2(10, pos2, 2);
Field field3(10, pos3, 2);
Field field4(10, pos4, 0);
Value value1((int32_t)10025);
Value value2("tag");
Value value3((int32_t)987654);
Value value4((int32_t)99999);
dim.addValue(FieldValue(field1, value1));
dim.addValue(FieldValue(field2, value2));
dim.addValue(FieldValue(field3, value3));
dim.addValue(FieldValue(field4, value4));
dim2.addValue(FieldValue(field1, value1));
dim2.addValue(FieldValue(field2, value2));
MetricDimensionKey dimKey(dim, dim2);
metadata::MetricDimensionKey metadataDimKey;
writeMetricDimensionKeyToMetadataDimensionKey(dimKey, &metadataDimKey);
MetricDimensionKey loadedDimKey = loadMetricDimensionKeyFromProto(metadataDimKey);
ASSERT_EQ(loadedDimKey, dimKey);
ASSERT_EQ(std::hash<MetricDimensionKey>{}(loadedDimKey),
std::hash<MetricDimensionKey>{}(dimKey));
}
} // namespace statsd
} // namespace os
} // namespace android
#else
GTEST_LOG_(INFO) << "This test does nothing.\n";
#endif