diff --git a/cmds/statsd/src/logd/LogEvent.h b/cmds/statsd/src/logd/LogEvent.h index 7e8a96b2114ef..176e16e3113a0 100644 --- a/cmds/statsd/src/logd/LogEvent.h +++ b/cmds/statsd/src/logd/LogEvent.h @@ -105,6 +105,11 @@ public: */ void init(); + /** + * Set timestamp if the original timestamp is missing. + */ + void setTimestampNs(uint64_t timestampNs) {mTimestampNs = timestampNs;} + private: /** * Don't copy, it's slower. If we really need this we can add it but let's try to diff --git a/cmds/statsd/src/metrics/CountMetricProducer.cpp b/cmds/statsd/src/metrics/CountMetricProducer.cpp index fc120134ac231..ae297d9eca0d3 100644 --- a/cmds/statsd/src/metrics/CountMetricProducer.cpp +++ b/cmds/statsd/src/metrics/CountMetricProducer.cpp @@ -188,7 +188,7 @@ bool CountMetricProducer::hitGuardRailLocked(const HashableDimensionKey& newKey) void CountMetricProducer::onMatchedLogEventInternalLocked( const size_t matcherIndex, const HashableDimensionKey& eventKey, const map& conditionKey, bool condition, - const LogEvent& event, bool scheduledPull) { + const LogEvent& event) { uint64_t eventTimeNs = event.GetTimestampNs(); flushIfNeededLocked(eventTimeNs); diff --git a/cmds/statsd/src/metrics/CountMetricProducer.h b/cmds/statsd/src/metrics/CountMetricProducer.h index 21bd9d6bf03d3..8a17169724cac 100644 --- a/cmds/statsd/src/metrics/CountMetricProducer.h +++ b/cmds/statsd/src/metrics/CountMetricProducer.h @@ -58,7 +58,7 @@ protected: void onMatchedLogEventInternalLocked( const size_t matcherIndex, const HashableDimensionKey& eventKey, const std::map& conditionKey, bool condition, - const LogEvent& event, bool scheduledPull) override; + const LogEvent& event) override; private: void onDumpReportLocked(const uint64_t dumpTimeNs, diff --git a/cmds/statsd/src/metrics/DurationMetricProducer.cpp b/cmds/statsd/src/metrics/DurationMetricProducer.cpp index 9920f65d962ad..c26879812f634 100644 --- a/cmds/statsd/src/metrics/DurationMetricProducer.cpp +++ b/cmds/statsd/src/metrics/DurationMetricProducer.cpp @@ -251,7 +251,7 @@ bool DurationMetricProducer::hitGuardRailLocked(const HashableDimensionKey& newK void DurationMetricProducer::onMatchedLogEventInternalLocked( const size_t matcherIndex, const HashableDimensionKey& eventKey, const map& conditionKeys, bool condition, - const LogEvent& event, bool scheduledPull) { + const LogEvent& event) { flushIfNeededLocked(event.GetTimestampNs()); if (matcherIndex == mStopAllIndex) { diff --git a/cmds/statsd/src/metrics/DurationMetricProducer.h b/cmds/statsd/src/metrics/DurationMetricProducer.h index e509af47a813d..14504c1f2effa 100644 --- a/cmds/statsd/src/metrics/DurationMetricProducer.h +++ b/cmds/statsd/src/metrics/DurationMetricProducer.h @@ -57,7 +57,7 @@ protected: void onMatchedLogEventInternalLocked( const size_t matcherIndex, const HashableDimensionKey& eventKey, const std::map& conditionKeys, bool condition, - const LogEvent& event, bool scheduledPull) override; + const LogEvent& event) override; private: void onDumpReportLocked(const uint64_t dumpTimeNs, diff --git a/cmds/statsd/src/metrics/EventMetricProducer.cpp b/cmds/statsd/src/metrics/EventMetricProducer.cpp index 217aff00a4143..bcecf163e243a 100644 --- a/cmds/statsd/src/metrics/EventMetricProducer.cpp +++ b/cmds/statsd/src/metrics/EventMetricProducer.cpp @@ -122,7 +122,7 @@ void EventMetricProducer::onConditionChangedLocked(const bool conditionMet, void EventMetricProducer::onMatchedLogEventInternalLocked( const size_t matcherIndex, const HashableDimensionKey& eventKey, const std::map& conditionKey, bool condition, - const LogEvent& event, bool scheduledPull) { + const LogEvent& event) { if (!condition) { return; } diff --git a/cmds/statsd/src/metrics/EventMetricProducer.h b/cmds/statsd/src/metrics/EventMetricProducer.h index 75ccf472e047d..49ba9d863d209 100644 --- a/cmds/statsd/src/metrics/EventMetricProducer.h +++ b/cmds/statsd/src/metrics/EventMetricProducer.h @@ -53,7 +53,7 @@ private: void onMatchedLogEventInternalLocked( const size_t matcherIndex, const HashableDimensionKey& eventKey, const std::map& conditionKey, bool condition, - const LogEvent& event, bool scheduledPull) override; + const LogEvent& event) override; void onDumpReportLocked(const uint64_t dumpTimeNs, android::util::ProtoOutputStream* protoOutput) override; diff --git a/cmds/statsd/src/metrics/GaugeMetricProducer.cpp b/cmds/statsd/src/metrics/GaugeMetricProducer.cpp index 55d84e0bd5269..fffb2bfd70312 100644 --- a/cmds/statsd/src/metrics/GaugeMetricProducer.cpp +++ b/cmds/statsd/src/metrics/GaugeMetricProducer.cpp @@ -187,7 +187,7 @@ void GaugeMetricProducer::onConditionChangedLocked(const bool conditionMet, return; } for (const auto& data : allData) { - onMatchedLogEventLocked(0, *data, false /*scheduledPull*/); + onMatchedLogEventLocked(0, *data); } flushIfNeededLocked(eventTime); } @@ -211,7 +211,7 @@ void GaugeMetricProducer::onDataPulled(const std::vector lock(mMutex); for (const auto& data : allData) { - onMatchedLogEventLocked(0, *data, true /*scheduledPull*/); + onMatchedLogEventLocked(0, *data); } } @@ -238,7 +238,7 @@ bool GaugeMetricProducer::hitGuardRailLocked(const HashableDimensionKey& newKey) void GaugeMetricProducer::onMatchedLogEventInternalLocked( const size_t matcherIndex, const HashableDimensionKey& eventKey, const map& conditionKey, bool condition, - const LogEvent& event, bool scheduledPull) { + const LogEvent& event) { if (condition == false) { return; } diff --git a/cmds/statsd/src/metrics/GaugeMetricProducer.h b/cmds/statsd/src/metrics/GaugeMetricProducer.h index e4bda029bcbbd..ee4f40c97d585 100644 --- a/cmds/statsd/src/metrics/GaugeMetricProducer.h +++ b/cmds/statsd/src/metrics/GaugeMetricProducer.h @@ -66,7 +66,7 @@ protected: void onMatchedLogEventInternalLocked( const size_t matcherIndex, const HashableDimensionKey& eventKey, const std::map& conditionKey, bool condition, - const LogEvent& event, bool scheduledPull) override; + const LogEvent& event) override; private: void onDumpReportLocked(const uint64_t dumpTimeNs, diff --git a/cmds/statsd/src/metrics/MetricProducer.cpp b/cmds/statsd/src/metrics/MetricProducer.cpp index 5a0a7c7fc5889..f38f3df13d9b0 100644 --- a/cmds/statsd/src/metrics/MetricProducer.cpp +++ b/cmds/statsd/src/metrics/MetricProducer.cpp @@ -21,8 +21,7 @@ namespace statsd { using std::map; -void MetricProducer::onMatchedLogEventLocked(const size_t matcherIndex, const LogEvent& event, - bool scheduledPull) { +void MetricProducer::onMatchedLogEventLocked(const size_t matcherIndex, const LogEvent& event) { uint64_t eventTimeNs = event.GetTimestampNs(); // this is old event, maybe statsd restarted? if (eventTimeNs < mStartTimeNs) { @@ -60,8 +59,7 @@ void MetricProducer::onMatchedLogEventLocked(const size_t matcherIndex, const Lo condition = mCondition; } - onMatchedLogEventInternalLocked(matcherIndex, eventKey, conditionKeys, condition, event, - scheduledPull); + onMatchedLogEventInternalLocked(matcherIndex, eventKey, conditionKeys, condition, event); } } // namespace statsd diff --git a/cmds/statsd/src/metrics/MetricProducer.h b/cmds/statsd/src/metrics/MetricProducer.h index ef2ef2965fc16..269bd435e32db 100644 --- a/cmds/statsd/src/metrics/MetricProducer.h +++ b/cmds/statsd/src/metrics/MetricProducer.h @@ -54,9 +54,9 @@ public: virtual ~MetricProducer(){}; // Consume the parsed stats log entry that already matched the "what" of the metric. - void onMatchedLogEvent(const size_t matcherIndex, const LogEvent& event, bool scheduledPull) { + void onMatchedLogEvent(const size_t matcherIndex, const LogEvent& event) { std::lock_guard lock(mMutex); - onMatchedLogEventLocked(matcherIndex, event, scheduledPull); + onMatchedLogEventLocked(matcherIndex, event); } void onConditionChanged(const bool condition, const uint64_t eventTime) { @@ -155,11 +155,10 @@ protected: virtual void onMatchedLogEventInternalLocked( const size_t matcherIndex, const HashableDimensionKey& eventKey, const std::map& conditionKey, bool condition, - const LogEvent& event, bool scheduledPull) = 0; + const LogEvent& event) = 0; // Consume the parsed stats log entry that already matched the "what" of the metric. - void onMatchedLogEventLocked(const size_t matcherIndex, const LogEvent& event, - bool scheduledPull); + void onMatchedLogEventLocked(const size_t matcherIndex, const LogEvent& event); mutable std::mutex mMutex; }; diff --git a/cmds/statsd/src/metrics/MetricsManager.cpp b/cmds/statsd/src/metrics/MetricsManager.cpp index 0510fff92b5cf..b0f0135f4121e 100644 --- a/cmds/statsd/src/metrics/MetricsManager.cpp +++ b/cmds/statsd/src/metrics/MetricsManager.cpp @@ -162,8 +162,7 @@ void MetricsManager::onLogEvent(const LogEvent& event) { auto& metricList = pair->second; for (const int metricIndex : metricList) { // pushed metrics are never scheduled pulls - mAllMetricProducers[metricIndex]->onMatchedLogEvent(i, event, - false /* schedulePull */); + mAllMetricProducers[metricIndex]->onMatchedLogEvent(i, event); } } } diff --git a/cmds/statsd/src/metrics/ValueMetricProducer.cpp b/cmds/statsd/src/metrics/ValueMetricProducer.cpp index c20c302942974..aabe5af698ec5 100644 --- a/cmds/statsd/src/metrics/ValueMetricProducer.cpp +++ b/cmds/statsd/src/metrics/ValueMetricProducer.cpp @@ -185,9 +185,13 @@ void ValueMetricProducer::onConditionChangedLocked(const bool condition, const u mCondition = condition; if (eventTime < mCurrentBucketStartTimeNs) { + VLOG("Skip event due to late arrival: %lld vs %lld", (long long)eventTime, + (long long)mCurrentBucketStartTimeNs); return; } + flushIfNeededLocked(eventTime); + if (mPullTagId != -1) { if (mCondition == true) { mStatsPullerManager->RegisterReceiver(mPullTagId, this, @@ -202,9 +206,8 @@ void ValueMetricProducer::onConditionChangedLocked(const bool condition, const u return; } for (const auto& data : allData) { - onMatchedLogEventLocked(0, *data, false); + onMatchedLogEventLocked(0, *data); } - flushIfNeededLocked(eventTime); } return; } @@ -217,15 +220,22 @@ void ValueMetricProducer::onDataPulled(const std::vectorGetTimestampNs(); - // alarm is not accurate and might drift. - if (eventTime > mCurrentBucketStartTimeNs + mBucketSizeNs * 3 / 2) { - flushIfNeededLocked(eventTime); - } + // For scheduled pulled data, the effective event time is snap to the nearest + // bucket boundary to make bucket finalize. + uint64_t realEventTime = allData.at(0)->GetTimestampNs(); + uint64_t eventTime = mStartTimeNs + ((realEventTime - mStartTimeNs)/mBucketSizeNs) * mBucketSizeNs; + + mCondition = false; for (const auto& data : allData) { - onMatchedLogEventLocked(0, *data, true); + data->setTimestampNs(eventTime-1); + onMatchedLogEventLocked(0, *data); + } + + mCondition = true; + for (const auto& data : allData) { + data->setTimestampNs(eventTime); + onMatchedLogEventLocked(0, *data); } - flushIfNeededLocked(eventTime); } } @@ -253,7 +263,7 @@ bool ValueMetricProducer::hitGuardRailLocked(const HashableDimensionKey& newKey) void ValueMetricProducer::onMatchedLogEventInternalLocked( const size_t matcherIndex, const HashableDimensionKey& eventKey, const map& conditionKey, bool condition, - const LogEvent& event, bool scheduledPull) { + const LogEvent& event) { uint64_t eventTimeNs = event.GetTimestampNs(); if (eventTimeNs < mCurrentBucketStartTimeNs) { VLOG("Skip event due to late arrival: %lld vs %lld", (long long)eventTimeNs, @@ -261,6 +271,8 @@ void ValueMetricProducer::onMatchedLogEventInternalLocked( return; } + flushIfNeededLocked(eventTimeNs); + if (hitGuardRailLocked(eventKey)) { return; } @@ -268,36 +280,21 @@ void ValueMetricProducer::onMatchedLogEventInternalLocked( long value = get_value(event); - if (mPullTagId != -1) { - if (scheduledPull) { - // scheduled pull always sets beginning of current bucket and end - // of next bucket - if (interval.raw.size() > 0) { - interval.raw.back().second = value; - } else { - interval.raw.push_back(make_pair(value, value)); - } - Interval& nextInterval = mNextSlicedBucket[eventKey]; - if (nextInterval.raw.size() == 0) { - nextInterval.raw.push_back(make_pair(value, 0)); - } else { - nextInterval.raw.front().first = value; - } + if (mPullTagId != -1) { // for pulled events + if (mCondition == true) { + interval.start = value; + interval.startUpdated = true; } else { - if (mCondition == true) { - interval.raw.push_back(make_pair(value, 0)); + if (interval.startUpdated) { + interval.sum += (value - interval.start); + interval.startUpdated = false; } else { - if (interval.raw.size() != 0) { - interval.raw.back().second = value; - } else { - interval.tainted = true; - VLOG("Data on condition true missing!"); - } + VLOG("No start for matching end %ld", value); + interval.tainted += 1; } } - } else { - flushIfNeededLocked(eventTimeNs); - interval.raw.push_back(make_pair(value, 0)); + } else { // for pushed events + interval.sum += value; } } @@ -327,27 +324,16 @@ void ValueMetricProducer::flushIfNeededLocked(const uint64_t& eventTimeNs) { int tainted = 0; for (const auto& slice : mCurrentSlicedBucket) { - long value = 0; - if (mPullTagId != -1) { - for (const auto& pair : slice.second.raw) { - value += (pair.second - pair.first); - } - } else { - for (const auto& pair : slice.second.raw) { - value += pair.first; - } - } tainted += slice.second.tainted; - info.mValue = value; - VLOG(" %s, %ld, %d", slice.first.c_str(), value, tainted); + info.mValue = slice.second.sum; // it will auto create new vector of ValuebucketInfo if the key is not found. auto& bucketList = mPastBuckets[slice.first]; bucketList.push_back(info); } + VLOG("%d tainted pairs in the bucket", tainted); // Reset counters - mCurrentSlicedBucket.swap(mNextSlicedBucket); - mNextSlicedBucket.clear(); + mCurrentSlicedBucket.clear(); int64_t numBucketsForward = (eventTimeNs - mCurrentBucketStartTimeNs) / mBucketSizeNs; mCurrentBucketStartTimeNs = mCurrentBucketStartTimeNs + numBucketsForward * mBucketSizeNs; diff --git a/cmds/statsd/src/metrics/ValueMetricProducer.h b/cmds/statsd/src/metrics/ValueMetricProducer.h index 8d60ff6a4e46a..4c49927d9203c 100644 --- a/cmds/statsd/src/metrics/ValueMetricProducer.h +++ b/cmds/statsd/src/metrics/ValueMetricProducer.h @@ -56,7 +56,7 @@ protected: void onMatchedLogEventInternalLocked( const size_t matcherIndex, const HashableDimensionKey& eventKey, const std::map& conditionKey, bool condition, - const LogEvent& event, bool scheduledPull) override; + const LogEvent& event) override; private: void onDumpReportLocked(const uint64_t dumpTimeNs, @@ -89,14 +89,19 @@ private: // internal state of a bucket. typedef struct { - std::vector> raw; - bool tainted; + // Pulled data always come in pair of . This holds the value + // for start. The diff (end - start) is added to sum. + long start; + // Whether the start data point is updated + bool startUpdated; + // If end data point comes before the start, record this pair as tainted + // and the value is not added to the running sum. + int tainted; + // Running sum of known pairs in this bucket + long sum; } Interval; std::unordered_map mCurrentSlicedBucket; - // If condition is true and pulling on schedule, the previous bucket value needs to be carried - // over to the next bucket. - std::unordered_map mNextSlicedBucket; // Save the past buckets and we can clear when the StatsLogReport is dumped. // TODO: Add a lock to mPastBuckets. diff --git a/cmds/statsd/tests/metrics/CountMetricProducer_test.cpp b/cmds/statsd/tests/metrics/CountMetricProducer_test.cpp index d973ba1676a9d..51eabd5387c17 100644 --- a/cmds/statsd/tests/metrics/CountMetricProducer_test.cpp +++ b/cmds/statsd/tests/metrics/CountMetricProducer_test.cpp @@ -54,8 +54,8 @@ TEST(CountMetricProducerTest, TestNonDimensionalEvents) { bucketStartTimeNs); // 2 events in bucket 1. - countProducer.onMatchedLogEvent(1 /*log matcher index*/, event1, false); - countProducer.onMatchedLogEvent(1 /*log matcher index*/, event2, false); + countProducer.onMatchedLogEvent(1 /*log matcher index*/, event1); + countProducer.onMatchedLogEvent(1 /*log matcher index*/, event2); // Flushes at event #2. countProducer.flushIfNeededLocked(bucketStartTimeNs + 2); @@ -74,7 +74,7 @@ TEST(CountMetricProducerTest, TestNonDimensionalEvents) { // 1 matched event happens in bucket 2. LogEvent event3(tagId, bucketStartTimeNs + bucketSizeNs + 2); - countProducer.onMatchedLogEvent(1 /*log matcher index*/, event3, false); + countProducer.onMatchedLogEvent(1 /*log matcher index*/, event3); countProducer.flushIfNeededLocked(bucketStartTimeNs + 2 * bucketSizeNs + 1); EXPECT_EQ(1UL, countProducer.mPastBuckets.size()); EXPECT_TRUE(countProducer.mPastBuckets.find(DEFAULT_DIMENSION_KEY) != @@ -111,12 +111,12 @@ TEST(CountMetricProducerTest, TestEventsWithNonSlicedCondition) { CountMetricProducer countProducer(kConfigKey, metric, 1, wizard, bucketStartTimeNs); countProducer.onConditionChanged(true, bucketStartTimeNs); - countProducer.onMatchedLogEvent(1 /*matcher index*/, event1, false /*pulled*/); + countProducer.onMatchedLogEvent(1 /*matcher index*/, event1); EXPECT_EQ(0UL, countProducer.mPastBuckets.size()); countProducer.onConditionChanged(false /*new condition*/, bucketStartTimeNs + 2); // Upon this match event, the matched event1 is flushed. - countProducer.onMatchedLogEvent(1 /*matcher index*/, event2, false /*pulled*/); + countProducer.onMatchedLogEvent(1 /*matcher index*/, event2); EXPECT_EQ(0UL, countProducer.mPastBuckets.size()); countProducer.flushIfNeededLocked(bucketStartTimeNs + bucketSizeNs + 1); @@ -166,11 +166,11 @@ TEST(CountMetricProducerTest, TestEventsWithSlicedCondition) { CountMetricProducer countProducer(kConfigKey, metric, 1 /*condition tracker index*/, wizard, bucketStartTimeNs); - countProducer.onMatchedLogEvent(1 /*log matcher index*/, event1, false); + countProducer.onMatchedLogEvent(1 /*log matcher index*/, event1); countProducer.flushIfNeededLocked(bucketStartTimeNs + 1); EXPECT_EQ(0UL, countProducer.mPastBuckets.size()); - countProducer.onMatchedLogEvent(1 /*log matcher index*/, event2, false); + countProducer.onMatchedLogEvent(1 /*log matcher index*/, event2); countProducer.flushIfNeededLocked(bucketStartTimeNs + bucketSizeNs + 1); EXPECT_EQ(1UL, countProducer.mPastBuckets.size()); EXPECT_TRUE(countProducer.mPastBuckets.find(DEFAULT_DIMENSION_KEY) != @@ -217,29 +217,29 @@ TEST(CountMetricProducerTest, TestAnomalyDetection) { LogEvent event7(tagId, bucketStartTimeNs + 3 * bucketSizeNs + 3 + NS_PER_SEC); // Two events in bucket #0. - countProducer.onMatchedLogEvent(1 /*log matcher index*/, event1, false); - countProducer.onMatchedLogEvent(1 /*log matcher index*/, event2, false); + countProducer.onMatchedLogEvent(1 /*log matcher index*/, event1); + countProducer.onMatchedLogEvent(1 /*log matcher index*/, event2); EXPECT_EQ(1UL, countProducer.mCurrentSlicedCounter->size()); EXPECT_EQ(2L, countProducer.mCurrentSlicedCounter->begin()->second); EXPECT_EQ(anomalyTracker->getLastAlarmTimestampNs(), -1LL); // One event in bucket #2. No alarm as bucket #0 is trashed out. - countProducer.onMatchedLogEvent(1 /*log matcher index*/, event3, false); + countProducer.onMatchedLogEvent(1 /*log matcher index*/, event3); EXPECT_EQ(1UL, countProducer.mCurrentSlicedCounter->size()); EXPECT_EQ(1L, countProducer.mCurrentSlicedCounter->begin()->second); EXPECT_EQ(anomalyTracker->getLastAlarmTimestampNs(), -1LL); // Two events in bucket #3. - countProducer.onMatchedLogEvent(1 /*log matcher index*/, event4, false); - countProducer.onMatchedLogEvent(1 /*log matcher index*/, event5, false); - countProducer.onMatchedLogEvent(1 /*log matcher index*/, event6, false); + countProducer.onMatchedLogEvent(1 /*log matcher index*/, event4); + countProducer.onMatchedLogEvent(1 /*log matcher index*/, event5); + countProducer.onMatchedLogEvent(1 /*log matcher index*/, event6); EXPECT_EQ(1UL, countProducer.mCurrentSlicedCounter->size()); EXPECT_EQ(3L, countProducer.mCurrentSlicedCounter->begin()->second); // Anomaly at event 6 is within refractory period. The alarm is at event 5 timestamp not event 6 EXPECT_EQ(anomalyTracker->getLastAlarmTimestampNs(), (long long)event5.GetTimestampNs()); - countProducer.onMatchedLogEvent(1 /*log matcher index*/, event7, false); + countProducer.onMatchedLogEvent(1 /*log matcher index*/, event7); EXPECT_EQ(1UL, countProducer.mCurrentSlicedCounter->size()); EXPECT_EQ(4L, countProducer.mCurrentSlicedCounter->begin()->second); EXPECT_EQ(anomalyTracker->getLastAlarmTimestampNs(), (long long)event7.GetTimestampNs()); diff --git a/cmds/statsd/tests/metrics/DurationMetricProducer_test.cpp b/cmds/statsd/tests/metrics/DurationMetricProducer_test.cpp index 3158c27572636..58a4ac6afd51e 100644 --- a/cmds/statsd/tests/metrics/DurationMetricProducer_test.cpp +++ b/cmds/statsd/tests/metrics/DurationMetricProducer_test.cpp @@ -56,8 +56,8 @@ TEST(DurationMetricTrackerTest, TestNoCondition) { kConfigKey, metric, -1 /*no condition*/, 1 /* start index */, 2 /* stop index */, 3 /* stop_all index */, false /*nesting*/, wizard, {}, bucketStartTimeNs); - durationProducer.onMatchedLogEvent(1 /* start index*/, event1, false /* scheduledPull */); - durationProducer.onMatchedLogEvent(2 /* stop index*/, event2, false /* scheduledPull */); + durationProducer.onMatchedLogEvent(1 /* start index*/, event1); + durationProducer.onMatchedLogEvent(2 /* stop index*/, event2); durationProducer.flushIfNeededLocked(bucketStartTimeNs + 2 * bucketSizeNs + 1); EXPECT_EQ(1UL, durationProducer.mPastBuckets.size()); EXPECT_TRUE(durationProducer.mPastBuckets.find(DEFAULT_DIMENSION_KEY) != @@ -94,14 +94,14 @@ TEST(DurationMetricTrackerTest, TestNonSlicedCondition) { EXPECT_FALSE(durationProducer.mCondition); EXPECT_FALSE(durationProducer.isConditionSliced()); - durationProducer.onMatchedLogEvent(1 /* start index*/, event1, false /* scheduledPull */); - durationProducer.onMatchedLogEvent(2 /* stop index*/, event2, false /* scheduledPull */); + durationProducer.onMatchedLogEvent(1 /* start index*/, event1); + durationProducer.onMatchedLogEvent(2 /* stop index*/, event2); durationProducer.flushIfNeededLocked(bucketStartTimeNs + bucketSizeNs + 1); EXPECT_EQ(0UL, durationProducer.mPastBuckets.size()); - durationProducer.onMatchedLogEvent(1 /* start index*/, event3, false /* scheduledPull */); + durationProducer.onMatchedLogEvent(1 /* start index*/, event3); durationProducer.onConditionChanged(true /* condition */, bucketStartTimeNs + bucketSizeNs + 2); - durationProducer.onMatchedLogEvent(2 /* stop index*/, event4, false /* scheduledPull */); + durationProducer.onMatchedLogEvent(2 /* stop index*/, event4); durationProducer.flushIfNeededLocked(bucketStartTimeNs + 2 * bucketSizeNs + 1); EXPECT_EQ(1UL, durationProducer.mPastBuckets.size()); EXPECT_TRUE(durationProducer.mPastBuckets.find(DEFAULT_DIMENSION_KEY) != diff --git a/cmds/statsd/tests/metrics/EventMetricProducer_test.cpp b/cmds/statsd/tests/metrics/EventMetricProducer_test.cpp index f3302fd5b8e96..e4fc67f95ec0b 100644 --- a/cmds/statsd/tests/metrics/EventMetricProducer_test.cpp +++ b/cmds/statsd/tests/metrics/EventMetricProducer_test.cpp @@ -50,8 +50,8 @@ TEST(EventMetricProducerTest, TestNoCondition) { EventMetricProducer eventProducer(kConfigKey, metric, -1 /*-1 meaning no condition*/, wizard, bucketStartTimeNs); - eventProducer.onMatchedLogEvent(1 /*matcher index*/, event1, false /*pulled*/); - eventProducer.onMatchedLogEvent(1 /*matcher index*/, event2, false /*pulled*/); + eventProducer.onMatchedLogEvent(1 /*matcher index*/, event1); + eventProducer.onMatchedLogEvent(1 /*matcher index*/, event2); // TODO: get the report and check the content after the ProtoOutputStream change is done. // eventProducer.onDumpReport(); @@ -74,11 +74,11 @@ TEST(EventMetricProducerTest, TestEventsWithNonSlicedCondition) { EventMetricProducer eventProducer(kConfigKey, metric, 1, wizard, bucketStartTimeNs); eventProducer.onConditionChanged(true /*condition*/, bucketStartTimeNs); - eventProducer.onMatchedLogEvent(1 /*matcher index*/, event1, false /*pulled*/); + eventProducer.onMatchedLogEvent(1 /*matcher index*/, event1); eventProducer.onConditionChanged(false /*condition*/, bucketStartTimeNs + 2); - eventProducer.onMatchedLogEvent(1 /*matcher index*/, event2, false /*pulled*/); + eventProducer.onMatchedLogEvent(1 /*matcher index*/, event2); // TODO: get the report and check the content after the ProtoOutputStream change is done. // eventProducer.onDumpReport(); @@ -115,8 +115,8 @@ TEST(EventMetricProducerTest, TestEventsWithSlicedCondition) { EventMetricProducer eventProducer(kConfigKey, metric, 1, wizard, bucketStartTimeNs); - eventProducer.onMatchedLogEvent(1 /*matcher index*/, event1, false /*pulled*/); - eventProducer.onMatchedLogEvent(1 /*matcher index*/, event2, false /*pulled*/); + eventProducer.onMatchedLogEvent(1 /*matcher index*/, event1); + eventProducer.onMatchedLogEvent(1 /*matcher index*/, event2); // TODO: get the report and check the content after the ProtoOutputStream change is done. // eventProducer.onDumpReport(); diff --git a/cmds/statsd/tests/metrics/ValueMetricProducer_test.cpp b/cmds/statsd/tests/metrics/ValueMetricProducer_test.cpp index d3206977a89d3..146a19d8cbfb8 100644 --- a/cmds/statsd/tests/metrics/ValueMetricProducer_test.cpp +++ b/cmds/statsd/tests/metrics/ValueMetricProducer_test.cpp @@ -35,23 +35,23 @@ namespace os { namespace statsd { const ConfigKey kConfigKey(0, "test"); +const int tagId = 1; +const string metricName = "test_metric"; +const int64_t bucketStartTimeNs = 10000000000; +const int64_t bucketSizeNs = 60 * 1000 * 1000 * 1000LL; +const int64_t bucket2StartTimeNs = bucketStartTimeNs + bucketSizeNs; +const int64_t bucket3StartTimeNs = bucketStartTimeNs + 2 * bucketSizeNs; +const int64_t bucket4StartTimeNs = bucketStartTimeNs + 3 * bucketSizeNs; + /* * Tests pulled atoms with no conditions */ TEST(ValueMetricProducerTest, TestNonDimensionalEvents) { - int64_t bucketStartTimeNs = 10000000000; - int64_t bucketSizeNs = 60 * 1000 * 1000 * 1000LL; - - int64_t bucket2StartTimeNs = bucketStartTimeNs + bucketSizeNs; - int64_t bucket3StartTimeNs = bucketStartTimeNs + 2 * bucketSizeNs; - ValueMetric metric; - metric.set_name("1"); + metric.set_name(metricName); metric.mutable_bucket()->set_bucket_size_millis(bucketSizeNs / 1000000); metric.set_value_field(2); - int tagId = 1; - sp wizard = new NaggyMock(); // TODO: pending refactor of StatsPullerManager // For now we still need this so that it doesn't do real pulling. @@ -65,8 +65,8 @@ TEST(ValueMetricProducerTest, TestNonDimensionalEvents) { vector> allData; allData.clear(); - shared_ptr event = make_shared(tagId, bucketStartTimeNs + 1); - event->write(1); + shared_ptr event = make_shared(tagId, bucket2StartTimeNs + 1); + event->write(tagId); event->write(11); event->init(); allData.push_back(event); @@ -75,76 +75,60 @@ TEST(ValueMetricProducerTest, TestNonDimensionalEvents) { // has one slice EXPECT_EQ(1UL, valueProducer.mCurrentSlicedBucket.size()); ValueMetricProducer::Interval curInterval = valueProducer.mCurrentSlicedBucket.begin()->second; - // has one raw pair - EXPECT_EQ(1UL, curInterval.raw.size()); - // value is 11, 11 - EXPECT_EQ(11, curInterval.raw.front().first); - EXPECT_EQ(11, curInterval.raw.front().second); - ValueMetricProducer::Interval nextInterval = valueProducer.mNextSlicedBucket.begin()->second; - // has one raw pair - EXPECT_EQ(1UL, nextInterval.raw.size()); - // value is 11, 0 - EXPECT_EQ(11, nextInterval.raw.front().first); - EXPECT_EQ(0, nextInterval.raw.front().second); - EXPECT_EQ(0UL, valueProducer.mPastBuckets.size()); + // startUpdated:true tainted:0 sum:0 start:11 + EXPECT_EQ(true, curInterval.startUpdated); + EXPECT_EQ(0, curInterval.tainted); + EXPECT_EQ(0, curInterval.sum); + EXPECT_EQ(11, curInterval.start); + EXPECT_EQ(1UL, valueProducer.mPastBuckets.size()); + EXPECT_EQ(0, valueProducer.mPastBuckets.begin()->second.back().mValue); allData.clear(); - event = make_shared(tagId, bucket2StartTimeNs + 1); - event->write(1); - event->write(22); + event = make_shared(tagId, bucket3StartTimeNs + 1); + event->write(tagId); + event->write(23); event->init(); allData.push_back(event); valueProducer.onDataPulled(allData); // has one slice EXPECT_EQ(1UL, valueProducer.mCurrentSlicedBucket.size()); curInterval = valueProducer.mCurrentSlicedBucket.begin()->second; - // has one raw pair - EXPECT_EQ(1UL, curInterval.raw.size()); - // value is 22, 0 - EXPECT_EQ(22, curInterval.raw.front().first); - EXPECT_EQ(0, curInterval.raw.front().second); - EXPECT_EQ(0UL, valueProducer.mNextSlicedBucket.size()); + // tartUpdated:false tainted:0 sum:12 + EXPECT_EQ(true, curInterval.startUpdated); + EXPECT_EQ(0, curInterval.tainted); + EXPECT_EQ(0, curInterval.sum); EXPECT_EQ(1UL, valueProducer.mPastBuckets.size()); - EXPECT_EQ(1UL, valueProducer.mPastBuckets.begin()->second.size()); - EXPECT_EQ(11, valueProducer.mPastBuckets.begin()->second.back().mValue); + EXPECT_EQ(2UL, valueProducer.mPastBuckets.begin()->second.size()); + EXPECT_EQ(12, valueProducer.mPastBuckets.begin()->second.back().mValue); allData.clear(); - event = make_shared(tagId, bucket3StartTimeNs + 1); - event->write(1); - event->write(33); + event = make_shared(tagId, bucket4StartTimeNs + 1); + event->write(tagId); + event->write(36); event->init(); allData.push_back(event); valueProducer.onDataPulled(allData); EXPECT_EQ(1UL, valueProducer.mCurrentSlicedBucket.size()); curInterval = valueProducer.mCurrentSlicedBucket.begin()->second; - EXPECT_EQ(1UL, curInterval.raw.size()); - // value is 33, 0 - EXPECT_EQ(33, curInterval.raw.front().first); - EXPECT_EQ(0, curInterval.raw.front().second); - EXPECT_EQ(0UL, valueProducer.mNextSlicedBucket.size()); + // startUpdated:false tainted:0 sum:12 + EXPECT_EQ(true, curInterval.startUpdated); + EXPECT_EQ(0, curInterval.tainted); + EXPECT_EQ(0, curInterval.sum); EXPECT_EQ(1UL, valueProducer.mPastBuckets.size()); - EXPECT_EQ(2UL, valueProducer.mPastBuckets.begin()->second.size()); - EXPECT_EQ(11, valueProducer.mPastBuckets.begin()->second.back().mValue); + EXPECT_EQ(3UL, valueProducer.mPastBuckets.begin()->second.size()); + EXPECT_EQ(13, valueProducer.mPastBuckets.begin()->second.back().mValue); } /* * Test pulled event with non sliced condition. */ TEST(ValueMetricProducerTest, TestEventsWithNonSlicedCondition) { - int64_t bucketStartTimeNs = 10000000000; - int64_t bucketSizeNs = 60 * 1000 * 1000 * 1000LL; - - int64_t bucket2StartTimeNs = bucketStartTimeNs + bucketSizeNs; - int64_t bucket3StartTimeNs = bucketStartTimeNs + 2 * bucketSizeNs; - ValueMetric metric; - metric.set_name("1"); + metric.set_name(metricName); metric.mutable_bucket()->set_bucket_size_millis(bucketSizeNs / 1000000); metric.set_value_field(2); metric.set_condition("SCREEN_ON"); - int tagId = 1; - sp wizard = new NaggyMock(); shared_ptr pullerManager = make_shared>(); @@ -153,28 +137,18 @@ TEST(ValueMetricProducerTest, TestEventsWithNonSlicedCondition) { EXPECT_CALL(*pullerManager, Pull(tagId, _)) .WillOnce(Invoke([](int tagId, vector>* data) { - int64_t bucketStartTimeNs = 10000000000; - int64_t bucketSizeNs = 60 * 1000 * 1000 * 1000LL; - - int64_t bucket2StartTimeNs = bucketStartTimeNs + bucketSizeNs; - int64_t bucket3StartTimeNs = bucketStartTimeNs + 2 * bucketSizeNs; data->clear(); shared_ptr event = make_shared(tagId, bucketStartTimeNs + 10); - event->write(1); + event->write(tagId); event->write(100); event->init(); data->push_back(event); return true; })) .WillOnce(Invoke([](int tagId, vector>* data) { - int64_t bucketStartTimeNs = 10000000000; - int64_t bucketSizeNs = 60 * 1000 * 1000 * 1000LL; - - int64_t bucket2StartTimeNs = bucketStartTimeNs + bucketSizeNs; - int64_t bucket3StartTimeNs = bucketStartTimeNs + 2 * bucketSizeNs; data->clear(); shared_ptr event = make_shared(tagId, bucket2StartTimeNs + 10); - event->write(1); + event->write(tagId); event->write(120); event->init(); data->push_back(event); @@ -184,17 +158,16 @@ TEST(ValueMetricProducerTest, TestEventsWithNonSlicedCondition) { ValueMetricProducer valueProducer(kConfigKey, metric, 1, wizard, tagId, bucketStartTimeNs, pullerManager); - valueProducer.onConditionChanged(true, bucketStartTimeNs + 10); + valueProducer.onConditionChanged(true, bucketStartTimeNs + 8); // has one slice EXPECT_EQ(1UL, valueProducer.mCurrentSlicedBucket.size()); ValueMetricProducer::Interval curInterval = valueProducer.mCurrentSlicedBucket.begin()->second; - // has one raw pair - EXPECT_EQ(1UL, curInterval.raw.size()); - // value is 100, 0 - EXPECT_EQ(100, curInterval.raw.front().first); - EXPECT_EQ(0, curInterval.raw.front().second); - EXPECT_EQ(0UL, valueProducer.mNextSlicedBucket.size()); + // startUpdated:false tainted:0 sum:0 start:100 + EXPECT_EQ(100, curInterval.start); + EXPECT_EQ(true, curInterval.startUpdated); + EXPECT_EQ(0, curInterval.tainted); + EXPECT_EQ(0, curInterval.sum); EXPECT_EQ(0UL, valueProducer.mPastBuckets.size()); vector> allData; @@ -209,11 +182,8 @@ TEST(ValueMetricProducerTest, TestEventsWithNonSlicedCondition) { // has one slice EXPECT_EQ(1UL, valueProducer.mCurrentSlicedBucket.size()); curInterval = valueProducer.mCurrentSlicedBucket.begin()->second; - // has one raw pair - EXPECT_EQ(1UL, curInterval.raw.size()); - // value is 110, 0 - EXPECT_EQ(110, curInterval.raw.front().first); - EXPECT_EQ(0, curInterval.raw.front().second); + // startUpdated:false tainted:0 sum:0 start:110 + EXPECT_EQ(110, curInterval.start); EXPECT_EQ(1UL, valueProducer.mPastBuckets.size()); EXPECT_EQ(1UL, valueProducer.mPastBuckets.begin()->second.size()); EXPECT_EQ(10, valueProducer.mPastBuckets.begin()->second.back().mValue); @@ -223,27 +193,17 @@ TEST(ValueMetricProducerTest, TestEventsWithNonSlicedCondition) { // has one slice EXPECT_EQ(1UL, valueProducer.mCurrentSlicedBucket.size()); curInterval = valueProducer.mCurrentSlicedBucket.begin()->second; - // has one raw pair - EXPECT_EQ(1UL, curInterval.raw.size()); - // value is 110, 120 - EXPECT_EQ(110, curInterval.raw.front().first); - EXPECT_EQ(120, curInterval.raw.front().second); + // startUpdated:false tainted:0 sum:0 start:110 + EXPECT_EQ(10, curInterval.sum); + EXPECT_EQ(false, curInterval.startUpdated); } TEST(ValueMetricProducerTest, TestPushedEventsWithoutCondition) { - int64_t bucketStartTimeNs = 10000000000; - int64_t bucketSizeNs = 60 * 1000 * 1000 * 1000LL; - - int64_t bucket2StartTimeNs = bucketStartTimeNs + bucketSizeNs; - int64_t bucket3StartTimeNs = bucketStartTimeNs + 2 * bucketSizeNs; - ValueMetric metric; - metric.set_name("1"); + metric.set_name(metricName); metric.mutable_bucket()->set_bucket_size_millis(bucketSizeNs / 1000000); metric.set_value_field(2); - int tagId = 1; - sp wizard = new NaggyMock(); shared_ptr pullerManager = make_shared>(); @@ -255,32 +215,22 @@ TEST(ValueMetricProducerTest, TestPushedEventsWithoutCondition) { event1->write(1); event1->write(10); event1->init(); - shared_ptr event2 = make_shared(tagId, bucketStartTimeNs + 10); + shared_ptr event2 = make_shared(tagId, bucketStartTimeNs + 20); event2->write(1); event2->write(20); event2->init(); - valueProducer.onMatchedLogEvent(1 /*log matcher index*/, *event1, false); + valueProducer.onMatchedLogEvent(1 /*log matcher index*/, *event1); // has one slice EXPECT_EQ(1UL, valueProducer.mCurrentSlicedBucket.size()); ValueMetricProducer::Interval curInterval = valueProducer.mCurrentSlicedBucket.begin()->second; - // has one raw pair - EXPECT_EQ(1UL, curInterval.raw.size()); - // value is 10, 0 - EXPECT_EQ(10, curInterval.raw.front().first); - EXPECT_EQ(0, curInterval.raw.front().second); - EXPECT_EQ(0UL, valueProducer.mNextSlicedBucket.size()); + EXPECT_EQ(10, curInterval.sum); - valueProducer.onMatchedLogEvent(1 /*log matcher index*/, *event2, false); + valueProducer.onMatchedLogEvent(1 /*log matcher index*/, *event2); // has one slice EXPECT_EQ(1UL, valueProducer.mCurrentSlicedBucket.size()); curInterval = valueProducer.mCurrentSlicedBucket.begin()->second; - // has one raw pair - EXPECT_EQ(2UL, curInterval.raw.size()); - // value is 10, 20 - EXPECT_EQ(10, curInterval.raw.front().first); - EXPECT_EQ(20, curInterval.raw.back().first); - EXPECT_EQ(0UL, valueProducer.mNextSlicedBucket.size()); + EXPECT_EQ(30, curInterval.sum); valueProducer.flushIfNeededLocked(bucket3StartTimeNs); EXPECT_EQ(1UL, valueProducer.mPastBuckets.size());