Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Slice by state in ValueMetricProducer

Browse Source 
- Added #onStateChanged logic to ValueMetricProducer
- #onDumpReport fills in slice_by_state values in ValueMetricProducer
- Base information is tracked in a separate data structure and keyed by
dimensions_in_what HashableDimensionKey. The current state key is also
stored in base info so we can close the interval for this state once we
get a new state key.
- Added unit tests for ValueMetricProducer onStateChange

Other changes:
- mCondition parameter was removed from #pullAndMatchEventsLocked and
 #accumulateEvents because it is unused in these functions.
- The event key is initialized in each metric's internal
 onMatchedLogEvent function. This allows ValueMetric to skip events that
 have primary keys that do not match the current state change primary
 key.

Test: bit statsd_test:*
Change-Id: I565c6d251262be57abf10fdef243b8bdc01f3772
This commit is contained in:
tsaichristine
2019-12-10 13:47:05 -08:00
parent 30959eba77
commit c876b492ce
26 changed files with 1422 additions and 259 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
cmds/statsd/src/HashableDimensionKey.cpp
View File

@@ -152,6 +152,10 @@ bool LessThan(const vector<FieldValue>& s1, const vector<FieldValue>& s2) {
return false;
}
bool HashableDimensionKey::operator!=(const HashableDimensionKey& that) const {
return !((*this) == that);
}
bool HashableDimensionKey::operator==(const HashableDimensionKey& that) const {
if (mValues.size() != that.getValues().size()) {
return false;

2
cmds/statsd/src/HashableDimensionKey.h
View File

@@ -71,6 +71,8 @@ public:
std::string toString() const;
bool operator!=(const HashableDimensionKey& that) const;
bool operator==(const HashableDimensionKey& that) const;
bool operator<(const HashableDimensionKey& that) const;

4
cmds/statsd/src/StatsLogProcessor.h
View File

@@ -284,6 +284,10 @@ private:
FRIEND_TEST(DurationMetricE2eTest, TestWithCondition);
FRIEND_TEST(DurationMetricE2eTest, TestWithSlicedCondition);
FRIEND_TEST(DurationMetricE2eTest, TestWithActivationAndSlicedCondition);
FRIEND_TEST(ValueMetricE2eTest, TestInitWithSlicedState);
FRIEND_TEST(ValueMetricE2eTest, TestInitWithSlicedState_WithDimensions);
FRIEND_TEST(ValueMetricE2eTest, TestInitWithSlicedState_WithIncorrectDimensions);
};
} // namespace statsd

4
cmds/statsd/src/metrics/CountMetricProducer.cpp
View File

@@ -277,8 +277,8 @@ bool CountMetricProducer::hitGuardRailLocked(const MetricDimensionKey& newKey) {
void CountMetricProducer::onMatchedLogEventInternalLocked(
const size_t matcherIndex, const MetricDimensionKey& eventKey,
const ConditionKey& conditionKey, bool condition,
const LogEvent& event) {
const ConditionKey& conditionKey, bool condition, const LogEvent& event,
const map<int, HashableDimensionKey>& statePrimaryKeys) {
int64_t eventTimeNs = event.GetElapsedTimestampNs();
flushIfNeededLocked(eventTimeNs);

4
cmds/statsd/src/metrics/CountMetricProducer.h
View File

@@ -59,8 +59,8 @@ public:
protected:
void onMatchedLogEventInternalLocked(
const size_t matcherIndex, const MetricDimensionKey& eventKey,
const ConditionKey& conditionKey, bool condition,
const LogEvent& event) override;
const ConditionKey& conditionKey, bool condition, const LogEvent& event,
const std::map<int, HashableDimensionKey>& statePrimaryKeys) override;
private:

4
cmds/statsd/src/metrics/DurationMetricProducer.cpp
View File

@@ -541,8 +541,8 @@ void DurationMetricProducer::handleStartEvent(const MetricDimensionKey& eventKey
void DurationMetricProducer::onMatchedLogEventInternalLocked(
const size_t matcherIndex, const MetricDimensionKey& eventKey,
const ConditionKey& conditionKeys, bool condition,
const LogEvent& event) {
const ConditionKey& conditionKeys, bool condition, const LogEvent& event,
const map<int, HashableDimensionKey>& statePrimaryKeys) {
ALOGW("Not used in duration tracker.");
}

4
cmds/statsd/src/metrics/DurationMetricProducer.h
View File

@@ -59,8 +59,8 @@ protected:
void onMatchedLogEventInternalLocked(
const size_t matcherIndex, const MetricDimensionKey& eventKey,
const ConditionKey& conditionKeys, bool condition,
const LogEvent& event) override;
const ConditionKey& conditionKeys, bool condition, const LogEvent& event,
const std::map<int, HashableDimensionKey>& statePrimaryKeys) override;
private:
void handleStartEvent(const MetricDimensionKey& eventKey, const ConditionKey& conditionKeys,

4
cmds/statsd/src/metrics/EventMetricProducer.cpp
View File

@@ -143,8 +143,8 @@ void EventMetricProducer::onConditionChangedLocked(const bool conditionMet,
void EventMetricProducer::onMatchedLogEventInternalLocked(
const size_t matcherIndex, const MetricDimensionKey& eventKey,
const ConditionKey& conditionKey, bool condition,
const LogEvent& event) {
const ConditionKey& conditionKey, bool condition, const LogEvent& event,
const map<int, HashableDimensionKey>& statePrimaryKeys) {
if (!condition) {
return;
}

4
cmds/statsd/src/metrics/EventMetricProducer.h
View File

@@ -47,8 +47,8 @@ public:
private:
void onMatchedLogEventInternalLocked(
const size_t matcherIndex, const MetricDimensionKey& eventKey,
const ConditionKey& conditionKey, bool condition,
const LogEvent& event) override;
const ConditionKey& conditionKey, bool condition, const LogEvent& event,
const std::map<int, HashableDimensionKey>& statePrimaryKeys) override;
void onDumpReportLocked(const int64_t dumpTimeNs,
const bool include_current_partial_bucket,

4
cmds/statsd/src/metrics/GaugeMetricProducer.cpp
View File

@@ -449,8 +449,8 @@ bool GaugeMetricProducer::hitGuardRailLocked(const MetricDimensionKey& newKey) {
void GaugeMetricProducer::onMatchedLogEventInternalLocked(
const size_t matcherIndex, const MetricDimensionKey& eventKey,
const ConditionKey& conditionKey, bool condition,
const LogEvent& event) {
const ConditionKey& conditionKey, bool condition, const LogEvent& event,
const map<int, HashableDimensionKey>& statePrimaryKeys) {
if (condition == false) {
return;
}

4
cmds/statsd/src/metrics/GaugeMetricProducer.h
View File

@@ -95,8 +95,8 @@ public:
protected:
void onMatchedLogEventInternalLocked(
const size_t matcherIndex, const MetricDimensionKey& eventKey,
const ConditionKey& conditionKey, bool condition,
const LogEvent& event) override;
const ConditionKey& conditionKey, bool condition, const LogEvent& event,
const std::map<int, HashableDimensionKey>& statePrimaryKeys) override;
private:
void onDumpReportLocked(const int64_t dumpTimeNs,

5
cmds/statsd/src/metrics/MetricProducer.cpp
View File

@@ -133,8 +133,8 @@ void MetricProducer::onMatchedLogEventLocked(const size_t matcherIndex, const Lo
HashableDimensionKey dimensionInWhat;
filterValues(mDimensionsInWhat, event.getValues(), &dimensionInWhat);
MetricDimensionKey metricKey(dimensionInWhat, stateValuesKey);
onMatchedLogEventInternalLocked(
matcherIndex, metricKey, conditionKey, condition, event);
onMatchedLogEventInternalLocked(matcherIndex, metricKey, conditionKey, condition, event,
statePrimaryKeys);
}
bool MetricProducer::evaluateActiveStateLocked(int64_t elapsedTimestampNs) {
@@ -269,6 +269,7 @@ void MetricProducer::getMappedStateValue(const int32_t atomId, const HashableDim
FieldValue* value) {
if (!StateManager::getInstance().getStateValue(atomId, queryKey, value)) {
value->mValue = Value(StateTracker::kStateUnknown);
value->mField.setTag(atomId);
ALOGW("StateTracker not found for state atom %d", atomId);
return;
}

8
cmds/statsd/src/metrics/MetricProducer.h
View File

@@ -330,8 +330,8 @@ protected:
*/
virtual void onMatchedLogEventInternalLocked(
const size_t matcherIndex, const MetricDimensionKey& eventKey,
const ConditionKey& conditionKey, bool condition,
const LogEvent& event) = 0;
const ConditionKey& conditionKey, bool condition, const LogEvent& event,
const map<int, HashableDimensionKey>& statePrimaryKeys) = 0;
// Consume the parsed stats log entry that already matched the "what" of the metric.
virtual void onMatchedLogEventLocked(const size_t matcherIndex, const LogEvent& event);
@@ -475,6 +475,10 @@ protected:
FRIEND_TEST(StatsLogProcessorTest,
TestActivationOnBootMultipleActivationsDifferentActivationTypes);
FRIEND_TEST(StatsLogProcessorTest, TestActivationsPersistAcrossSystemServerRestart);
FRIEND_TEST(ValueMetricE2eTest, TestInitWithSlicedState);
FRIEND_TEST(ValueMetricE2eTest, TestInitWithSlicedState_WithDimensions);
FRIEND_TEST(ValueMetricE2eTest, TestInitWithSlicedState_WithIncorrectDimensions);
};
} // namespace statsd

4
cmds/statsd/src/metrics/MetricsManager.h
View File

@@ -289,6 +289,10 @@ private:
FRIEND_TEST(DurationMetricE2eTest, TestWithCondition);
FRIEND_TEST(DurationMetricE2eTest, TestWithSlicedCondition);
FRIEND_TEST(DurationMetricE2eTest, TestWithActivationAndSlicedCondition);
FRIEND_TEST(ValueMetricE2eTest, TestInitWithSlicedState);
FRIEND_TEST(ValueMetricE2eTest, TestInitWithSlicedState_WithDimensions);
FRIEND_TEST(ValueMetricE2eTest, TestInitWithSlicedState_WithIncorrectDimensions);
};
} // namespace statsd

156
cmds/statsd/src/metrics/ValueMetricProducer.cpp
View File

@@ -66,6 +66,7 @@ const int FIELD_ID_DROP_TIME = 2;
const int FIELD_ID_DIMENSION_IN_WHAT = 1;
const int FIELD_ID_BUCKET_INFO = 3;
const int FIELD_ID_DIMENSION_LEAF_IN_WHAT = 4;
const int FIELD_ID_SLICE_BY_STATE = 6;
// for ValueBucketInfo
const int FIELD_ID_VALUE_INDEX = 1;
const int FIELD_ID_VALUE_LONG = 2;
@@ -146,6 +147,14 @@ ValueMetricProducer::ValueMetricProducer(
mConditionSliced = true;
}
for (const auto& stateLink : metric.state_link()) {
Metric2State ms;
ms.stateAtomId = stateLink.state_atom_id();
translateFieldMatcher(stateLink.fields_in_what(), &ms.metricFields);
translateFieldMatcher(stateLink.fields_in_state(), &ms.stateFields);
mMetric2StateLinks.push_back(ms);
}
int64_t numBucketsForward = calcBucketsForwardCount(startTimeNs);
mCurrentBucketNum += numBucketsForward;
@@ -181,6 +190,33 @@ ValueMetricProducer::~ValueMetricProducer() {
}
}
void ValueMetricProducer::onStateChanged(int64_t eventTimeNs, int32_t atomId,
const HashableDimensionKey& primaryKey, int oldState,
int newState) {
VLOG("ValueMetric %lld onStateChanged time %lld, State %d, key %s, %d -> %d",
(long long)mMetricId, (long long)eventTimeNs, atomId, primaryKey.toString().c_str(),
oldState, newState);
// If condition is not true, we do not need to pull for this state change.
if (mCondition != ConditionState::kTrue) {
return;
}
bool isEventLate = eventTimeNs < mCurrentBucketStartTimeNs;
if (isEventLate) {
VLOG("Skip event due to late arrival: %lld vs %lld", (long long)eventTimeNs,
(long long)mCurrentBucketStartTimeNs);
invalidateCurrentBucket(eventTimeNs, BucketDropReason::EVENT_IN_WRONG_BUCKET);
return;
}
mStateChangePrimaryKey.first = atomId;
mStateChangePrimaryKey.second = primaryKey;
if (mIsPulled) {
pullAndMatchEventsLocked(eventTimeNs);
}
mStateChangePrimaryKey.first = 0;
mStateChangePrimaryKey.second = DEFAULT_DIMENSION_KEY;
flushIfNeededLocked(eventTimeNs);
}
void ValueMetricProducer::onSlicedConditionMayChangeLocked(bool overallCondition,
const int64_t eventTime) {
VLOG("Metric %lld onSlicedConditionMayChange", (long long)mMetricId);
@@ -281,6 +317,14 @@ void ValueMetricProducer::onDumpReportLocked(const int64_t dumpTimeNs,
FIELD_ID_DIMENSION_LEAF_IN_WHAT, str_set, protoOutput);
}
// Then fill slice_by_state.
for (auto state : dimensionKey.getStateValuesKey().getValues()) {
uint64_t stateToken = protoOutput->start(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED |
FIELD_ID_SLICE_BY_STATE);
writeStateToProto(state, protoOutput);
protoOutput->end(stateToken);
}
// Then fill bucket_info (ValueBucketInfo).
for (const auto& bucket : pair.second) {
uint64_t bucketInfoToken = protoOutput->start(
@@ -300,7 +344,7 @@ void ValueMetricProducer::onDumpReportLocked(const int64_t dumpTimeNs,
protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_CONDITION_TRUE_NS,
(long long)bucket.mConditionTrueNs);
}
for (int i = 0; i < (int)bucket.valueIndex.size(); i ++) {
for (int i = 0; i < (int)bucket.valueIndex.size(); i++) {
int index = bucket.valueIndex[i];
const Value& value = bucket.values[i];
uint64_t valueToken = protoOutput->start(
@@ -358,9 +402,10 @@ void ValueMetricProducer::invalidateCurrentBucket(const int64_t dropTimeNs,
}
void ValueMetricProducer::resetBase() {
for (auto& slice : mCurrentSlicedBucket) {
for (auto& interval : slice.second) {
interval.hasBase = false;
for (auto& slice : mCurrentBaseInfo) {
for (auto& baseInfo : slice.second) {
baseInfo.hasBase = false;
baseInfo.hasCurrentState = false;
}
}
mHasGlobalBase = false;
@@ -558,14 +603,20 @@ void ValueMetricProducer::accumulateEvents(const std::vector<std::shared_ptr<Log
onMatchedLogEventLocked(mWhatMatcherIndex, localCopy);
}
}
// If the new pulled data does not contains some keys we track in our intervals, we need to
// reset the base.
// If a key that is:
// 1. Tracked in mCurrentSlicedBucket and
// 2. A superset of the current mStateChangePrimaryKey
// was not found in the new pulled data (i.e. not in mMatchedDimensionInWhatKeys)
// then we need to reset the base.
for (auto& slice : mCurrentSlicedBucket) {
bool presentInPulledData = mMatchedMetricDimensionKeys.find(slice.first)
!= mMatchedMetricDimensionKeys.end();
if (!presentInPulledData) {
for (auto& interval : slice.second) {
interval.hasBase = false;
const auto& whatKey = slice.first.getDimensionKeyInWhat();
bool presentInPulledData =
mMatchedMetricDimensionKeys.find(whatKey) != mMatchedMetricDimensionKeys.end();
if (!presentInPulledData && whatKey.contains(mStateChangePrimaryKey.second)) {
auto it = mCurrentBaseInfo.find(whatKey);
for (auto& baseInfo : it->second) {
baseInfo.hasBase = false;
baseInfo.hasCurrentState = false;
}
}
}
@@ -674,17 +725,30 @@ bool getDoubleOrLong(const LogEvent& event, const Matcher& matcher, Value& ret)
return false;
}
void ValueMetricProducer::onMatchedLogEventInternalLocked(const size_t matcherIndex,
const MetricDimensionKey& eventKey,
const ConditionKey& conditionKey,
bool condition, const LogEvent& event) {
void ValueMetricProducer::onMatchedLogEventInternalLocked(
const size_t matcherIndex, const MetricDimensionKey& eventKey,
const ConditionKey& conditionKey, bool condition, const LogEvent& event,
const map<int, HashableDimensionKey>& statePrimaryKeys) {
auto whatKey = eventKey.getDimensionKeyInWhat();
auto stateKey = eventKey.getStateValuesKey();
// Skip this event if a state changed occurred for a different primary key.
auto it = statePrimaryKeys.find(mStateChangePrimaryKey.first);
// Check that both the atom id and the primary key are equal.
if (it != statePrimaryKeys.end() && it->second != mStateChangePrimaryKey.second) {
VLOG("ValueMetric skip event with primary key %s because state change primary key "
"is %s",
it->second.toString().c_str(), mStateChangePrimaryKey.second.toString().c_str());
return;
}
int64_t eventTimeNs = event.GetElapsedTimestampNs();
if (eventTimeNs < mCurrentBucketStartTimeNs) {
VLOG("Skip event due to late arrival: %lld vs %lld", (long long)eventTimeNs,
(long long)mCurrentBucketStartTimeNs);
return;
}
mMatchedMetricDimensionKeys.insert(eventKey);
mMatchedMetricDimensionKeys.insert(whatKey);
if (!mIsPulled) {
// We cannot flush without doing a pull first.
@@ -709,10 +773,26 @@ void ValueMetricProducer::onMatchedLogEventInternalLocked(const size_t matcherIn
if (hitGuardRailLocked(eventKey)) {
return;
}
vector<Interval>& multiIntervals = mCurrentSlicedBucket[eventKey];
if (multiIntervals.size() < mFieldMatchers.size()) {
vector<BaseInfo>& baseInfos = mCurrentBaseInfo[whatKey];
if (baseInfos.size() < mFieldMatchers.size()) {
VLOG("Resizing number of intervals to %d", (int)mFieldMatchers.size());
multiIntervals.resize(mFieldMatchers.size());
baseInfos.resize(mFieldMatchers.size());
}
for (auto baseInfo : baseInfos) {
if (!baseInfo.hasCurrentState) {
baseInfo.currentState = DEFAULT_DIMENSION_KEY;
baseInfo.hasCurrentState = true;
}
}
// We need to get the intervals stored with the previous state key so we can
// close these value intervals.
const auto oldStateKey = baseInfos[0].currentState;
vector<Interval>& intervals = mCurrentSlicedBucket[MetricDimensionKey(whatKey, oldStateKey)];
if (intervals.size() < mFieldMatchers.size()) {
VLOG("Resizing number of intervals to %d", (int)mFieldMatchers.size());
intervals.resize(mFieldMatchers.size());
}
// We only use anomaly detection under certain cases.
@@ -725,7 +805,8 @@ void ValueMetricProducer::onMatchedLogEventInternalLocked(const size_t matcherIn
for (int i = 0; i < (int)mFieldMatchers.size(); i++) {
const Matcher& matcher = mFieldMatchers[i];
Interval& interval = multiIntervals[i];
BaseInfo& baseInfo = baseInfos[i];
Interval& interval = intervals[i];
interval.valueIndex = i;
Value value;
if (!getDoubleOrLong(event, matcher, value)) {
@@ -736,60 +817,61 @@ void ValueMetricProducer::onMatchedLogEventInternalLocked(const size_t matcherIn
interval.seenNewData = true;
if (mUseDiff) {
if (!interval.hasBase) {
if (!baseInfo.hasBase) {
if (mHasGlobalBase && mUseZeroDefaultBase) {
// The bucket has global base. This key does not.
// Optionally use zero as base.
interval.base = (value.type == LONG ? ZERO_LONG : ZERO_DOUBLE);
interval.hasBase = true;
baseInfo.base = (value.type == LONG ? ZERO_LONG : ZERO_DOUBLE);
baseInfo.hasBase = true;
} else {
// no base. just update base and return.
interval.base = value;
interval.hasBase = true;
baseInfo.base = value;
baseInfo.hasBase = true;
// If we're missing a base, do not use anomaly detection on incomplete data
useAnomalyDetection = false;
// Continue (instead of return) here in order to set interval.base and
// interval.hasBase for other intervals
// Continue (instead of return) here in order to set baseInfo.base and
// baseInfo.hasBase for other baseInfos
continue;
}
}
Value diff;
switch (mValueDirection) {
case ValueMetric::INCREASING:
if (value >= interval.base) {
diff = value - interval.base;
if (value >= baseInfo.base) {
diff = value - baseInfo.base;
} else if (mUseAbsoluteValueOnReset) {
diff = value;
} else {
VLOG("Unexpected decreasing value");
StatsdStats::getInstance().notePullDataError(mPullTagId);
interval.base = value;
baseInfo.base = value;
// If we've got bad data, do not use anomaly detection
useAnomalyDetection = false;
continue;
}
break;
case ValueMetric::DECREASING:
if (interval.base >= value) {
diff = interval.base - value;
if (baseInfo.base >= value) {
diff = baseInfo.base - value;
} else if (mUseAbsoluteValueOnReset) {
diff = value;
} else {
VLOG("Unexpected increasing value");
StatsdStats::getInstance().notePullDataError(mPullTagId);
interval.base = value;
baseInfo.base = value;
// If we've got bad data, do not use anomaly detection
useAnomalyDetection = false;
continue;
}
break;
case ValueMetric::ANY:
diff = value - interval.base;
diff = value - baseInfo.base;
break;
default:
break;
}
interval.base = value;
baseInfo.base = value;
value = diff;
}
@@ -814,12 +896,13 @@ void ValueMetricProducer::onMatchedLogEventInternalLocked(const size_t matcherIn
interval.hasValue = true;
}
interval.sampleSize += 1;
baseInfo.currentState = stateKey;
}
// Only trigger the tracker if all intervals are correct
if (useAnomalyDetection) {
// TODO: propgate proper values down stream when anomaly support doubles
long wholeBucketVal = multiIntervals[0].value.long_value;
long wholeBucketVal = intervals[0].value.long_value;
auto prev = mCurrentFullBucket.find(eventKey);
if (prev != mCurrentFullBucket.end()) {
wholeBucketVal += prev->second;
@@ -953,6 +1036,7 @@ void ValueMetricProducer::initCurrentSlicedBucket(int64_t nextBucketStartTimeNs)
} else {
it++;
}
// TODO: remove mCurrentBaseInfo entries when obsolete
}
mCurrentBucketIsInvalid = false;

32
cmds/statsd/src/metrics/ValueMetricProducer.h
View File

@@ -83,11 +83,14 @@ public:
flushCurrentBucketLocked(eventTimeNs, eventTimeNs);
};
void onStateChanged(int64_t eventTimeNs, int32_t atomId, const HashableDimensionKey& primaryKey,
int oldState, int newState) override;
protected:
void onMatchedLogEventInternalLocked(
const size_t matcherIndex, const MetricDimensionKey& eventKey,
const ConditionKey& conditionKey, bool condition,
const LogEvent& event) override;
const ConditionKey& conditionKey, bool condition, const LogEvent& event,
const std::map<int, HashableDimensionKey>& statePrimaryKeys) override;
private:
void onDumpReportLocked(const int64_t dumpTimeNs,
@@ -144,7 +147,10 @@ private:
std::vector<Matcher> mFieldMatchers;
// Value fields for matching.
std::set<MetricDimensionKey> mMatchedMetricDimensionKeys;
std::set<HashableDimensionKey> mMatchedMetricDimensionKeys;
// Holds the atom id, primary key pair from a state change.
pair<int32_t, HashableDimensionKey> mStateChangePrimaryKey;
// tagId for pulled data. -1 if this is not pulled
const int mPullTagId;
@@ -156,10 +162,6 @@ private:
typedef struct {
// Index in multi value aggregation.
int valueIndex;
// Holds current base value of the dimension. Take diff and update if necessary.
Value base;
// Whether there is a base to diff to.
bool hasBase;
// Current value, depending on the aggregation type.
Value value;
// Number of samples collected.
@@ -171,8 +173,21 @@ private:
bool seenNewData = false;
} Interval;
typedef struct {
// Holds current base value of the dimension. Take diff and update if necessary.
Value base;
// Whether there is a base to diff to.
bool hasBase;
// Last seen state value(s).
HashableDimensionKey currentState;
// Whether this dimensions in what key has a current state key.
bool hasCurrentState;
} BaseInfo;
std::unordered_map<MetricDimensionKey, std::vector<Interval>> mCurrentSlicedBucket;
std::unordered_map<HashableDimensionKey, std::vector<BaseInfo>> mCurrentBaseInfo;
std::unordered_map<MetricDimensionKey, int64_t> mCurrentFullBucket;
// Save the past buckets and we can clear when the StatsLogReport is dumped.
@@ -285,6 +300,9 @@ private:
FRIEND_TEST(ValueMetricProducerTest, TestResetBaseOnPullTooLate);
FRIEND_TEST(ValueMetricProducerTest, TestSkipZeroDiffOutput);
FRIEND_TEST(ValueMetricProducerTest, TestSkipZeroDiffOutputMultiValue);
FRIEND_TEST(ValueMetricProducerTest, TestSlicedState);
FRIEND_TEST(ValueMetricProducerTest, TestSlicedStateWithMap);
FRIEND_TEST(ValueMetricProducerTest, TestSlicedStateWithPrimaryField_WithDimensions);
FRIEND_TEST(ValueMetricProducerTest, TestTrimUnusedDimensionKey);
FRIEND_TEST(ValueMetricProducerTest, TestUseZeroDefaultBase);
FRIEND_TEST(ValueMetricProducerTest, TestUseZeroDefaultBaseWithPullFailures);

44
cmds/statsd/src/metrics/metrics_manager_util.cpp
View File

@@ -18,11 +18,12 @@
#include "Log.h"
#include "metrics_manager_util.h"
#include "MetricProducer.h"
#include <inttypes.h>
#include "atoms_info.h"
#include "FieldValue.h"
#include "MetricProducer.h"
#include "condition/CombinationConditionTracker.h"
#include "condition/SimpleConditionTracker.h"
#include "condition/StateConditionTracker.h"
@@ -173,6 +174,14 @@ bool handleMetricWithStates(
return true;
}
bool handleMetricWithStateLink(const FieldMatcher& stateMatcher,
const vector<Matcher>& dimensionsInWhat) {
vector<Matcher> stateMatchers;
translateFieldMatcher(stateMatcher, &stateMatchers);
return subsetDimensions(stateMatchers, dimensionsInWhat);
}
// Validates a metricActivation and populates state.
// EventActivationMap and EventDeactivationMap are supplied to a MetricProducer
// to provide the producer with state about its activators and deactivators.
@@ -669,18 +678,41 @@ bool initMetrics(const ConfigKey& key, const StatsdConfig& config, const int64_t
}
}
std::vector<int> slicedStateAtoms;
unordered_map<int, unordered_map<int, int64_t>> stateGroupMap;
if (metric.slice_by_state_size() > 0) {
if (!handleMetricWithStates(config, metric.slice_by_state(), stateAtomIdMap,
allStateGroupMaps, slicedStateAtoms, stateGroupMap)) {
return false;
}
} else {
if (metric.state_link_size() > 0) {
ALOGW("ValueMetric has a MetricStateLink but doesn't have a sliced state");
return false;
}
}
// Check that all metric state links are a subset of dimensions_in_what fields.
std::vector<Matcher> dimensionsInWhat;
translateFieldMatcher(metric.dimensions_in_what(), &dimensionsInWhat);
for (const auto& stateLink : metric.state_link()) {
if (!handleMetricWithStateLink(stateLink.fields_in_what(), dimensionsInWhat)) {
return false;
}
}
unordered_map<int, shared_ptr<Activation>> eventActivationMap;
unordered_map<int, vector<shared_ptr<Activation>>> eventDeactivationMap;
bool success = handleMetricActivation(config, metric.id(), metricIndex,
metricToActivationMap, logTrackerMap, activationAtomTrackerToMetricMap,
deactivationAtomTrackerToMetricMap, metricsWithActivation, eventActivationMap,
eventDeactivationMap);
bool success = handleMetricActivation(
config, metric.id(), metricIndex, metricToActivationMap, logTrackerMap,
activationAtomTrackerToMetricMap, deactivationAtomTrackerToMetricMap,
metricsWithActivation, eventActivationMap, eventDeactivationMap);
if (!success) return false;
sp<MetricProducer> valueProducer = new ValueMetricProducer(
key, metric, conditionIndex, wizard, trackerIndex, matcherWizard, pullTagId,
timeBaseTimeNs, currentTimeNs, pullerManager, eventActivationMap,
eventDeactivationMap);
eventDeactivationMap, slicedStateAtoms, stateGroupMap);
allMetricProducers.push_back(valueProducer);
}

10
cmds/statsd/src/state/StateManager.cpp
View File

@@ -28,13 +28,17 @@ StateManager& StateManager::getInstance() {
return sStateManager;
}
void StateManager::clear() {
mStateTrackers.clear();
}
void StateManager::onLogEvent(const LogEvent& event) {
if (mStateTrackers.find(event.GetTagId()) != mStateTrackers.end()) {
mStateTrackers[event.GetTagId()]->onLogEvent(event);
}
}
bool StateManager::registerListener(int32_t atomId, wp<StateListener> listener) {
bool StateManager::registerListener(const int32_t atomId, wp<StateListener> listener) {
// Check if state tracker already exists.
if (mStateTrackers.find(atomId) == mStateTrackers.end()) {
// Create a new state tracker iff atom is a state atom.
@@ -50,7 +54,7 @@ bool StateManager::registerListener(int32_t atomId, wp<StateListener> listener)
return true;
}
void StateManager::unregisterListener(int32_t atomId, wp<StateListener> listener) {
void StateManager::unregisterListener(const int32_t atomId, wp<StateListener> listener) {
std::unique_lock<std::mutex> lock(mMutex);
// Hold the sp<> until the lock is released so that ~StateTracker() is
@@ -74,7 +78,7 @@ void StateManager::unregisterListener(int32_t atomId, wp<StateListener> listener
lock.unlock();
}
bool StateManager::getStateValue(int32_t atomId, const HashableDimensionKey& key,
bool StateManager::getStateValue(const int32_t atomId, const HashableDimensionKey& key,
FieldValue* output) const {
auto it = mStateTrackers.find(atomId);
if (it != mStateTrackers.end()) {

11
cmds/statsd/src/state/StateManager.h
View File

@@ -40,30 +40,33 @@ public:
// Returns a pointer to the single, shared StateManager object.
static StateManager& getInstance();
// Unregisters all listeners and removes all trackers from StateManager.
void clear();
// Notifies the correct StateTracker of an event.
void onLogEvent(const LogEvent& event);
// Returns true if atomId is being tracked and is associated with a state
// atom. StateManager notifies the correct StateTracker to register listener.
// If the correct StateTracker does not exist, a new StateTracker is created.
bool registerListener(int32_t atomId, wp<StateListener> listener);
bool registerListener(const int32_t atomId, wp<StateListener> listener);
// Notifies the correct StateTracker to unregister a listener
// and removes the tracker if it no longer has any listeners.
void unregisterListener(int32_t atomId, wp<StateListener> listener);
void unregisterListener(const int32_t atomId, wp<StateListener> listener);
// Returns true if the StateTracker exists and queries for the
// original state value mapped to the given query key. The state value is
// stored and output in a FieldValue class.
// Returns false if the StateTracker doesn't exist.
bool getStateValue(int32_t atomId, const HashableDimensionKey& queryKey,
bool getStateValue(const int32_t atomId, const HashableDimensionKey& queryKey,
FieldValue* output) const;
inline int getStateTrackersCount() const {
return mStateTrackers.size();
}
inline int getListenersCount(int32_t atomId) const {
inline int getListenersCount(const int32_t atomId) const {
auto it = mStateTrackers.find(atomId);
if (it != mStateTrackers.end()) {
return it->second->getListenersCount();

4
cmds/statsd/src/stats_log.proto
View File

@@ -147,12 +147,14 @@ message ValueBucketInfo {
message ValueMetricData {
optional DimensionsValue dimensions_in_what = 1;
optional DimensionsValue dimensions_in_condition = 2 [deprecated = true];
repeated StateValue slice_by_state = 6;
repeated ValueBucketInfo bucket_info = 3;
repeated DimensionsValue dimension_leaf_values_in_what = 4;
optional DimensionsValue dimensions_in_condition = 2 [deprecated = true];
repeated DimensionsValue dimension_leaf_values_in_condition = 5 [deprecated = true];
}

6
cmds/statsd/src/statsd_config.proto
View File

@@ -290,12 +290,14 @@ message ValueMetric {
optional FieldMatcher dimensions_in_what = 5;
optional FieldMatcher dimensions_in_condition = 9 [deprecated = true];
repeated int64 slice_by_state = 18;
optional TimeUnit bucket = 6;
repeated MetricConditionLink links = 7;
repeated MetricStateLink state_link = 19;
enum AggregationType {
SUM = 1;
MIN = 2;
@@ -325,6 +327,8 @@ message ValueMetric {
optional int32 max_pull_delay_sec = 16 [default = 10];
optional bool split_bucket_for_app_upgrade = 17 [default = true];
optional FieldMatcher dimensions_in_condition = 9 [deprecated = true];
}
message Alert {

3
cmds/statsd/tests/e2e/CountMetric_e2e_test.cpp
View File

@@ -27,9 +27,6 @@ namespace statsd {
#ifdef __ANDROID__
const int SCREEN_STATE_ATOM_ID = android::util::SCREEN_STATE_CHANGED;
const int UID_PROCESS_STATE_ATOM_ID = android::util::UID_PROCESS_STATE_CHANGED;
/**
* Test a count metric that has one slice_by_state with no primary fields.
*

162
cmds/statsd/tests/e2e/ValueMetric_pull_e2e_test.cpp
View File

@@ -369,6 +369,168 @@ TEST(ValueMetricE2eTest, TestPulledEvents_WithActivation) {
EXPECT_EQ(1, bucketInfo.values_size());
}
/**
* Test initialization of a simple value metric that is sliced by a state.
*
* ValueCpuUserTimePerScreenState
*/
TEST(ValueMetricE2eTest, TestInitWithSlicedState) {
// Create config.
StatsdConfig config;
config.add_allowed_log_source("AID_ROOT"); // LogEvent defaults to UID of root.
auto pulledAtomMatcher =
CreateSimpleAtomMatcher("TestMatcher", android::util::SUBSYSTEM_SLEEP_STATE);
*config.add_atom_matcher() = pulledAtomMatcher;
auto screenState = CreateScreenState();
*config.add_state() = screenState;
// Create value metric that slices by screen state without a map.
int64_t metricId = 123456;
auto valueMetric = config.add_value_metric();
valueMetric->set_id(metricId);
valueMetric->set_bucket(TimeUnit::FIVE_MINUTES);
valueMetric->set_what(pulledAtomMatcher.id());
*valueMetric->mutable_value_field() =
CreateDimensions(android::util::CPU_TIME_PER_UID, {2 /* user_time_micros */});
valueMetric->add_slice_by_state(screenState.id());
valueMetric->set_max_pull_delay_sec(INT_MAX);
// Initialize StatsLogProcessor.
const uint64_t bucketStartTimeNs = 10000000000; // 0:10
const uint64_t bucketSizeNs =
TimeUnitToBucketSizeInMillis(config.value_metric(0).bucket()) * 1000000LL;
int uid = 12345;
int64_t cfgId = 98765;
ConfigKey cfgKey(uid, cfgId);
auto processor = CreateStatsLogProcessor(bucketStartTimeNs, bucketStartTimeNs, config, cfgKey);
// Check that StateTrackers were initialized correctly.
EXPECT_EQ(1, StateManager::getInstance().getStateTrackersCount());
EXPECT_EQ(1, StateManager::getInstance().getListenersCount(SCREEN_STATE_ATOM_ID));
// Check that ValueMetricProducer was initialized correctly.
EXPECT_EQ(1U, processor->mMetricsManagers.size());
sp<MetricsManager> metricsManager = processor->mMetricsManagers.begin()->second;
EXPECT_TRUE(metricsManager->isConfigValid());
EXPECT_EQ(1, metricsManager->mAllMetricProducers.size());
sp<MetricProducer> metricProducer = metricsManager->mAllMetricProducers[0];
EXPECT_EQ(1, metricProducer->mSlicedStateAtoms.size());
EXPECT_EQ(SCREEN_STATE_ATOM_ID, metricProducer->mSlicedStateAtoms.at(0));
EXPECT_EQ(0, metricProducer->mStateGroupMap.size());
}
/**
* Test initialization of a value metric that is sliced by state and has
* dimensions_in_what.
*
* ValueCpuUserTimePerUidPerUidProcessState
*/
TEST(ValueMetricE2eTest, TestInitWithSlicedState_WithDimensions) {
// Create config.
StatsdConfig config;
config.add_allowed_log_source("AID_ROOT"); // LogEvent defaults to UID of root.
auto cpuTimePerUidMatcher =
CreateSimpleAtomMatcher("CpuTimePerUidMatcher", android::util::CPU_TIME_PER_UID);
*config.add_atom_matcher() = cpuTimePerUidMatcher;
auto uidProcessState = CreateUidProcessState();
*config.add_state() = uidProcessState;
// Create value metric that slices by screen state with a complete map.
int64_t metricId = 123456;
auto valueMetric = config.add_value_metric();
valueMetric->set_id(metricId);
valueMetric->set_bucket(TimeUnit::FIVE_MINUTES);
valueMetric->set_what(cpuTimePerUidMatcher.id());
*valueMetric->mutable_value_field() =
CreateDimensions(android::util::CPU_TIME_PER_UID, {2 /* user_time_micros */});
*valueMetric->mutable_dimensions_in_what() =
CreateDimensions(android::util::CPU_TIME_PER_UID, {1 /* uid */});
valueMetric->add_slice_by_state(uidProcessState.id());
MetricStateLink* stateLink = valueMetric->add_state_link();
stateLink->set_state_atom_id(UID_PROCESS_STATE_ATOM_ID);
auto fieldsInWhat = stateLink->mutable_fields_in_what();
*fieldsInWhat = CreateDimensions(android::util::CPU_TIME_PER_UID, {1 /* uid */});
auto fieldsInState = stateLink->mutable_fields_in_state();
*fieldsInState = CreateDimensions(UID_PROCESS_STATE_ATOM_ID, {1 /* uid */});
valueMetric->set_max_pull_delay_sec(INT_MAX);
// Initialize StatsLogProcessor.
const uint64_t bucketStartTimeNs = 10000000000; // 0:10
int uid = 12345;
int64_t cfgId = 98765;
ConfigKey cfgKey(uid, cfgId);
auto processor = CreateStatsLogProcessor(bucketStartTimeNs, bucketStartTimeNs, config, cfgKey);
// Check that StateTrackers were initialized correctly.
EXPECT_EQ(1, StateManager::getInstance().getStateTrackersCount());
EXPECT_EQ(1, StateManager::getInstance().getListenersCount(UID_PROCESS_STATE_ATOM_ID));
// Check that ValueMetricProducer was initialized correctly.
EXPECT_EQ(1U, processor->mMetricsManagers.size());
sp<MetricsManager> metricsManager = processor->mMetricsManagers.begin()->second;
EXPECT_TRUE(metricsManager->isConfigValid());
EXPECT_EQ(1, metricsManager->mAllMetricProducers.size());
sp<MetricProducer> metricProducer = metricsManager->mAllMetricProducers[0];
EXPECT_EQ(1, metricProducer->mSlicedStateAtoms.size());
EXPECT_EQ(UID_PROCESS_STATE_ATOM_ID, metricProducer->mSlicedStateAtoms.at(0));
EXPECT_EQ(0, metricProducer->mStateGroupMap.size());
}
/**
* Test initialization of a value metric that is sliced by state and has
* dimensions_in_what.
*
* ValueCpuUserTimePerUidPerUidProcessState
*/
TEST(ValueMetricE2eTest, TestInitWithSlicedState_WithIncorrectDimensions) {
// Create config.
StatsdConfig config;
config.add_allowed_log_source("AID_ROOT"); // LogEvent defaults to UID of root.
auto cpuTimePerUidMatcher =
CreateSimpleAtomMatcher("CpuTimePerUidMatcher", android::util::CPU_TIME_PER_UID);
*config.add_atom_matcher() = cpuTimePerUidMatcher;
auto uidProcessState = CreateUidProcessState();
*config.add_state() = uidProcessState;
// Create value metric that slices by screen state with a complete map.
int64_t metricId = 123456;
auto valueMetric = config.add_value_metric();
valueMetric->set_id(metricId);
valueMetric->set_bucket(TimeUnit::FIVE_MINUTES);
valueMetric->set_what(cpuTimePerUidMatcher.id());
*valueMetric->mutable_value_field() =
CreateDimensions(android::util::CPU_TIME_PER_UID, {2 /* user_time_micros */});
valueMetric->add_slice_by_state(uidProcessState.id());
MetricStateLink* stateLink = valueMetric->add_state_link();
stateLink->set_state_atom_id(UID_PROCESS_STATE_ATOM_ID);
auto fieldsInWhat = stateLink->mutable_fields_in_what();
*fieldsInWhat = CreateDimensions(android::util::CPU_TIME_PER_UID, {1 /* uid */});
auto fieldsInState = stateLink->mutable_fields_in_state();
*fieldsInState = CreateDimensions(UID_PROCESS_STATE_ATOM_ID, {1 /* uid */});
valueMetric->set_max_pull_delay_sec(INT_MAX);
// Initialize StatsLogProcessor.
const uint64_t bucketStartTimeNs = 10000000000; // 0:10
int uid = 12345;
int64_t cfgId = 98765;
ConfigKey cfgKey(uid, cfgId);
auto processor = CreateStatsLogProcessor(bucketStartTimeNs, bucketStartTimeNs, config, cfgKey);
// No StateTrackers are initialized.
EXPECT_EQ(0, StateManager::getInstance().getStateTrackersCount());
// Config initialization fails.
EXPECT_EQ(0, processor->mMetricsManagers.size());
}
#else
GTEST_LOG_(INFO) << "This test does nothing.\n";
#endif

1190
cmds/statsd/tests/metrics/ValueMetricProducer_test.cpp
View File

File diff suppressed because it is too large Load Diff

1
cmds/statsd/tests/state/StateTracker_test.cpp
View File

@@ -146,6 +146,7 @@ TEST(StateListenerTest, TestStateListenerWeakPointer) {
TEST(StateManagerTest, TestStateManagerGetInstance) {
sp<TestStateListener> listener1 = new TestStateListener();
StateManager& mgr = StateManager::getInstance();
mgr.clear();
mgr.registerListener(android::util::SCREEN_STATE_CHANGED, listener1);
EXPECT_EQ(1, mgr.getStateTrackersCount());

3
cmds/statsd/tests/statsd_test_util.h
View File

@@ -30,6 +30,9 @@ namespace statsd {
using android::util::ProtoReader;
using google::protobuf::RepeatedPtrField;
const int SCREEN_STATE_ATOM_ID = android::util::SCREEN_STATE_CHANGED;
const int UID_PROCESS_STATE_ATOM_ID = android::util::UID_PROCESS_STATE_CHANGED;
// Converts a ProtoOutputStream to a StatsLogReport proto.
StatsLogReport outputStreamToProto(ProtoOutputStream* proto);