From c58750578fdf6e54500e9deabe7d5062afbd6e09 Mon Sep 17 00:00:00 2001 From: Chenjie Yu Date: Fri, 9 Mar 2018 10:13:11 -0800 Subject: [PATCH] Up the dimension key limit for CpuTimePerUidFreq This atom logs time spent on cpu per frequency per uid. On marlin, there are 27 frequency steps per uid (should be 50+ if flatten to both cores) On walleye, there are 52 frequency steps per uid. So it easily goes to 6k data. Soft limit set to 6k now. Hard limit set to 10k to accommodate future processors Memory impact: on marlin, start memory no config is 2,346K after using this atom in one guage metric, it is 3,067K Bug: 72505991 Test: cts test Change-Id: I067a32e54e4a457fdf9a25911aa16030e893ef4d --- cmds/statsd/src/guardrail/StatsdStats.cpp | 4 ++++ cmds/statsd/src/guardrail/StatsdStats.h | 3 +++ .../src/metrics/GaugeMetricProducer.cpp | 22 +++++++++++++------ cmds/statsd/src/metrics/GaugeMetricProducer.h | 4 ++++ .../src/metrics/ValueMetricProducer.cpp | 18 ++++++++++----- cmds/statsd/src/metrics/ValueMetricProducer.h | 4 ++++ 6 files changed, 43 insertions(+), 12 deletions(-) diff --git a/cmds/statsd/src/guardrail/StatsdStats.cpp b/cmds/statsd/src/guardrail/StatsdStats.cpp index 7a55f6065e880..ef637df68ed30 100644 --- a/cmds/statsd/src/guardrail/StatsdStats.cpp +++ b/cmds/statsd/src/guardrail/StatsdStats.cpp @@ -92,6 +92,10 @@ const int FIELD_ID_UID_MAP_BYTES_USED = 3; const int FIELD_ID_UID_MAP_DROPPED_SNAPSHOTS = 4; const int FIELD_ID_UID_MAP_DROPPED_CHANGES = 5; +const std::map> StatsdStats::kAtomDimensionKeySizeLimitMap = { + {android::util::CPU_TIME_PER_UID_FREQ, {6000, 10000}}, +}; + // TODO: add stats for pulled atoms. StatsdStats::StatsdStats() { mPushedAtomStats.resize(android::util::kMaxPushedAtomId + 1); diff --git a/cmds/statsd/src/guardrail/StatsdStats.h b/cmds/statsd/src/guardrail/StatsdStats.h index a4f64ddfd3a5e..767588808d45b 100644 --- a/cmds/statsd/src/guardrail/StatsdStats.h +++ b/cmds/statsd/src/guardrail/StatsdStats.h @@ -81,6 +81,9 @@ public: const static int kDimensionKeySizeSoftLimit = 300; const static int kDimensionKeySizeHardLimit = 500; + // Per atom dimension key size limit + static const std::map> kAtomDimensionKeySizeLimitMap; + const static int kMaxConfigCount = 10; const static int kMaxAlertCountPerConfig = 100; const static int kMaxConditionCountPerConfig = 200; diff --git a/cmds/statsd/src/metrics/GaugeMetricProducer.cpp b/cmds/statsd/src/metrics/GaugeMetricProducer.cpp index 49034ac25c648..f08d54a9f41b5 100644 --- a/cmds/statsd/src/metrics/GaugeMetricProducer.cpp +++ b/cmds/statsd/src/metrics/GaugeMetricProducer.cpp @@ -17,9 +17,9 @@ #define DEBUG false // STOPSHIP if true #include "Log.h" +#include "../guardrail/StatsdStats.h" #include "GaugeMetricProducer.h" -#include "guardrail/StatsdStats.h" -#include "stats_log_util.h" +#include "../stats_log_util.h" #include @@ -60,12 +60,20 @@ const int FIELD_ID_WALL_CLOCK_ATOM_TIMESTAMP = 5; GaugeMetricProducer::GaugeMetricProducer(const ConfigKey& key, const GaugeMetric& metric, const int conditionIndex, - const sp& wizard, - const int pullTagId, const uint64_t startTimeNs, + const sp& wizard, const int pullTagId, + const uint64_t startTimeNs, shared_ptr statsPullerManager) : MetricProducer(metric.id(), key, startTimeNs, conditionIndex, wizard), mStatsPullerManager(statsPullerManager), - mPullTagId(pullTagId) { + mPullTagId(pullTagId), + mDimensionSoftLimit(StatsdStats::kAtomDimensionKeySizeLimitMap.find(pullTagId) != + StatsdStats::kAtomDimensionKeySizeLimitMap.end() + ? StatsdStats::kAtomDimensionKeySizeLimitMap.at(pullTagId).first + : StatsdStats::kDimensionKeySizeSoftLimit), + mDimensionHardLimit(StatsdStats::kAtomDimensionKeySizeLimitMap.find(pullTagId) != + StatsdStats::kAtomDimensionKeySizeLimitMap.end() + ? StatsdStats::kAtomDimensionKeySizeLimitMap.at(pullTagId).second + : StatsdStats::kDimensionKeySizeHardLimit) { mCurrentSlicedBucket = std::make_shared(); mCurrentSlicedBucketForAnomaly = std::make_shared(); int64_t bucketSizeMills = 0; @@ -305,11 +313,11 @@ bool GaugeMetricProducer::hitGuardRailLocked(const MetricDimensionKey& newKey) { return false; } // 1. Report the tuple count if the tuple count > soft limit - if (mCurrentSlicedBucket->size() > StatsdStats::kDimensionKeySizeSoftLimit - 1) { + if (mCurrentSlicedBucket->size() > mDimensionSoftLimit - 1) { size_t newTupleCount = mCurrentSlicedBucket->size() + 1; StatsdStats::getInstance().noteMetricDimensionSize(mConfigKey, mMetricId, newTupleCount); // 2. Don't add more tuples, we are above the allowed threshold. Drop the data. - if (newTupleCount > StatsdStats::kDimensionKeySizeHardLimit) { + if (newTupleCount > mDimensionHardLimit) { ALOGE("GaugeMetric %lld dropping data for dimension key %s", (long long)mMetricId, newKey.toString().c_str()); return true; diff --git a/cmds/statsd/src/metrics/GaugeMetricProducer.h b/cmds/statsd/src/metrics/GaugeMetricProducer.h index dd6aff4130de9..4b543f808de22 100644 --- a/cmds/statsd/src/metrics/GaugeMetricProducer.h +++ b/cmds/statsd/src/metrics/GaugeMetricProducer.h @@ -150,6 +150,10 @@ private: static const size_t kBucketSize = sizeof(GaugeBucket{}); + const size_t mDimensionSoftLimit; + + const size_t mDimensionHardLimit; + FRIEND_TEST(GaugeMetricProducerTest, TestWithCondition); FRIEND_TEST(GaugeMetricProducerTest, TestNoCondition); FRIEND_TEST(GaugeMetricProducerTest, TestPushedEventsWithUpgrade); diff --git a/cmds/statsd/src/metrics/ValueMetricProducer.cpp b/cmds/statsd/src/metrics/ValueMetricProducer.cpp index 767260d263516..ed4d7e4ffc9c4 100644 --- a/cmds/statsd/src/metrics/ValueMetricProducer.cpp +++ b/cmds/statsd/src/metrics/ValueMetricProducer.cpp @@ -18,8 +18,8 @@ #include "Log.h" #include "ValueMetricProducer.h" -#include "guardrail/StatsdStats.h" -#include "stats_log_util.h" +#include "../guardrail/StatsdStats.h" +#include "../stats_log_util.h" #include #include @@ -68,7 +68,15 @@ ValueMetricProducer::ValueMetricProducer(const ConfigKey& key, const ValueMetric : MetricProducer(metric.id(), key, startTimeNs, conditionIndex, wizard), mValueField(metric.value_field()), mStatsPullerManager(statsPullerManager), - mPullTagId(pullTagId) { + mPullTagId(pullTagId), + mDimensionSoftLimit(StatsdStats::kAtomDimensionKeySizeLimitMap.find(pullTagId) != + StatsdStats::kAtomDimensionKeySizeLimitMap.end() + ? StatsdStats::kAtomDimensionKeySizeLimitMap.at(pullTagId).first + : StatsdStats::kDimensionKeySizeSoftLimit), + mDimensionHardLimit(StatsdStats::kAtomDimensionKeySizeLimitMap.find(pullTagId) != + StatsdStats::kAtomDimensionKeySizeLimitMap.end() + ? StatsdStats::kAtomDimensionKeySizeLimitMap.at(pullTagId).second + : StatsdStats::kDimensionKeySizeHardLimit) { // TODO: valuemetric for pushed events may need unlimited bucket length int64_t bucketSizeMills = 0; if (metric.has_bucket()) { @@ -266,11 +274,11 @@ bool ValueMetricProducer::hitGuardRailLocked(const MetricDimensionKey& newKey) { if (mCurrentSlicedBucket.find(newKey) != mCurrentSlicedBucket.end()) { return false; } - if (mCurrentSlicedBucket.size() > StatsdStats::kDimensionKeySizeSoftLimit - 1) { + if (mCurrentSlicedBucket.size() > mDimensionSoftLimit - 1) { size_t newTupleCount = mCurrentSlicedBucket.size() + 1; StatsdStats::getInstance().noteMetricDimensionSize(mConfigKey, mMetricId, newTupleCount); // 2. Don't add more tuples, we are above the allowed threshold. Drop the data. - if (newTupleCount > StatsdStats::kDimensionKeySizeHardLimit) { + if (newTupleCount > mDimensionHardLimit) { ALOGE("ValueMetric %lld dropping data for dimension key %s", (long long)mMetricId, newKey.toString().c_str()); return true; diff --git a/cmds/statsd/src/metrics/ValueMetricProducer.h b/cmds/statsd/src/metrics/ValueMetricProducer.h index be57183bd926b..c4477b3cff4e7 100644 --- a/cmds/statsd/src/metrics/ValueMetricProducer.h +++ b/cmds/statsd/src/metrics/ValueMetricProducer.h @@ -149,6 +149,10 @@ private: static const size_t kBucketSize = sizeof(ValueBucket{}); + const size_t mDimensionSoftLimit; + + const size_t mDimensionHardLimit; + FRIEND_TEST(ValueMetricProducerTest, TestNonDimensionalEvents); FRIEND_TEST(ValueMetricProducerTest, TestEventsWithNonSlicedCondition); FRIEND_TEST(ValueMetricProducerTest, TestPushedEventsWithUpgrade);