From 8a8d16ceea1e5b7a2f8c41e17b5d993035f50f5d Mon Sep 17 00:00:00 2001 From: Yao Chen Date: Thu, 8 Feb 2018 14:50:40 -0800 Subject: [PATCH] Statsd CPU optimization. The key change is to revamp how we parse/store/match a log event, especially how we match repeated field and attribution nodes, and how we construct dimensions and compare them. + We use a integer to encode the field of a log element. And also encode the FieldMatcher into an integer and a bit mask. The log matching becomes 2 integer operations. + Dimension is stored as encoded field and value pair. Checking if 2 dimensions are equal is then becoming checking if the underlying integers are equal. The integers are stored contiguously in memory, so it's much faster than previous tree structure. Start review from FieldValue.h Test: statsd_test + new unit tests Bug: 72659059 Change-Id: Iec8daeacdd3f39ab297c10ab9cd7b710a9c42e86 --- cmds/statsd/Android.mk | 6 +- cmds/statsd/src/FieldValue.cpp | 186 +++++ cmds/statsd/src/FieldValue.h | 338 ++++++++++ cmds/statsd/src/HashableDimensionKey.cpp | 348 ++++++---- cmds/statsd/src/HashableDimensionKey.h | 93 ++- cmds/statsd/src/StatsLogProcessor.cpp | 72 +- cmds/statsd/src/StatsLogProcessor.h | 18 +- cmds/statsd/src/StatsService.cpp | 5 +- .../condition/CombinationConditionTracker.cpp | 12 +- .../condition/CombinationConditionTracker.h | 16 +- cmds/statsd/src/condition/ConditionTracker.h | 9 +- cmds/statsd/src/condition/ConditionWizard.cpp | 13 +- cmds/statsd/src/condition/ConditionWizard.h | 13 +- .../src/condition/SimpleConditionTracker.cpp | 135 ++-- .../src/condition/SimpleConditionTracker.h | 18 +- cmds/statsd/src/condition/condition_util.cpp | 104 --- cmds/statsd/src/condition/condition_util.h | 6 - cmds/statsd/src/dimension.cpp | 400 ----------- cmds/statsd/src/dimension.h | 71 -- cmds/statsd/src/external/puller_util.cpp | 226 ++++--- cmds/statsd/src/external/puller_util.h | 1 - cmds/statsd/src/field_util.cpp | 318 --------- cmds/statsd/src/field_util.h | 86 --- cmds/statsd/src/logd/LogEvent.cpp | 361 +++++----- cmds/statsd/src/logd/LogEvent.h | 36 +- cmds/statsd/src/matchers/matcher_util.cpp | 384 ++++++----- cmds/statsd/src/matchers/matcher_util.h | 3 - .../src/metrics/CountMetricProducer.cpp | 54 +- cmds/statsd/src/metrics/CountMetricProducer.h | 1 - .../src/metrics/DurationMetricProducer.cpp | 71 +- .../src/metrics/DurationMetricProducer.h | 3 +- .../src/metrics/EventMetricProducer.cpp | 13 +- cmds/statsd/src/metrics/EventMetricProducer.h | 1 - .../src/metrics/GaugeMetricProducer.cpp | 81 +-- cmds/statsd/src/metrics/GaugeMetricProducer.h | 12 +- cmds/statsd/src/metrics/MetricProducer.cpp | 32 +- cmds/statsd/src/metrics/MetricProducer.h | 13 +- cmds/statsd/src/metrics/MetricsManager.cpp | 13 +- cmds/statsd/src/metrics/MetricsManager.h | 4 +- .../src/metrics/ValueMetricProducer.cpp | 64 +- cmds/statsd/src/metrics/ValueMetricProducer.h | 5 +- .../metrics/duration_helper/DurationTracker.h | 4 +- .../duration_helper/MaxDurationTracker.cpp | 15 +- .../duration_helper/MaxDurationTracker.h | 2 +- .../duration_helper/OringDurationTracker.cpp | 16 +- .../duration_helper/OringDurationTracker.h | 4 +- .../src/metrics/metrics_manager_util.cpp | 16 + cmds/statsd/src/packages/UidMap.cpp | 2 +- cmds/statsd/src/stats_log_util.cpp | 319 +++++---- cmds/statsd/src/stats_log_util.h | 15 +- cmds/statsd/src/statsd_internal.proto | 29 - .../src/subscriber/SubscriberReporter.cpp | 92 +-- .../src/subscriber/SubscriberReporter.h | 10 +- cmds/statsd/tests/FieldValue_test.cpp | 350 ++++++++++ cmds/statsd/tests/LogEntryMatcher_test.cpp | 4 +- cmds/statsd/tests/LogEvent_test.cpp | 634 ++++-------------- cmds/statsd/tests/StatsLogProcessor_test.cpp | 18 +- .../tests/anomaly/AnomalyTracker_test.cpp | 8 +- .../condition/SimpleConditionTracker_test.cpp | 25 +- cmds/statsd/tests/dimension_test.cpp | 149 ---- .../statsd/tests/e2e/Attribution_e2e_test.cpp | 6 +- .../e2e/DimensionInCondition_e2e_test.cpp | 304 +++++---- .../statsd/tests/e2e/GaugeMetric_e2e_test.cpp | 5 +- .../e2e/MetricConditionLink_e2e_test.cpp | 116 +++- .../tests/e2e/WakelockDuration_e2e_test.cpp | 105 ++- .../metrics/CountMetricProducer_test.cpp | 1 - .../metrics/EventMetricProducer_test.cpp | 1 - .../metrics/GaugeMetricProducer_test.cpp | 78 ++- .../tests/metrics/MaxDurationTracker_test.cpp | 11 +- .../metrics/OringDurationTracker_test.cpp | 20 +- .../tests/metrics/metrics_test_helper.cpp | 26 +- .../tests/metrics/metrics_test_helper.h | 14 +- cmds/statsd/tests/statsd_test_util.cpp | 91 ++- cmds/statsd/tests/statsd_test_util.h | 26 +- 74 files changed, 2885 insertions(+), 3276 deletions(-) create mode 100644 cmds/statsd/src/FieldValue.cpp create mode 100644 cmds/statsd/src/FieldValue.h delete mode 100644 cmds/statsd/src/dimension.cpp delete mode 100644 cmds/statsd/src/dimension.h delete mode 100644 cmds/statsd/src/field_util.cpp delete mode 100644 cmds/statsd/src/field_util.h delete mode 100644 cmds/statsd/src/statsd_internal.proto create mode 100644 cmds/statsd/tests/FieldValue_test.cpp delete mode 100644 cmds/statsd/tests/dimension_test.cpp diff --git a/cmds/statsd/Android.mk b/cmds/statsd/Android.mk index 90158a00f8ec4..a3be13296c115 100644 --- a/cmds/statsd/Android.mk +++ b/cmds/statsd/Android.mk @@ -19,11 +19,9 @@ statsd_common_src := \ ../../core/java/android/os/IStatsManager.aidl \ src/stats_log.proto \ src/statsd_config.proto \ - src/statsd_internal.proto \ src/atoms.proto \ - src/field_util.cpp \ + src/FieldValue.cpp \ src/stats_log_util.cpp \ - src/dimension.cpp \ src/anomaly/AnomalyMonitor.cpp \ src/anomaly/AnomalyTracker.cpp \ src/anomaly/DurationAnomalyTracker.cpp \ @@ -172,7 +170,6 @@ LOCAL_CFLAGS += \ LOCAL_SRC_FILES := \ $(statsd_common_src) \ - tests/dimension_test.cpp \ tests/AnomalyMonitor_test.cpp \ tests/anomaly/AnomalyTracker_test.cpp \ tests/ConfigManager_test.cpp \ @@ -184,6 +181,7 @@ LOCAL_SRC_FILES := \ tests/MetricsManager_test.cpp \ tests/StatsLogProcessor_test.cpp \ tests/UidMap_test.cpp \ + tests/FieldValue_test.cpp \ tests/condition/CombinationConditionTracker_test.cpp \ tests/condition/SimpleConditionTracker_test.cpp \ tests/metrics/OringDurationTracker_test.cpp \ diff --git a/cmds/statsd/src/FieldValue.cpp b/cmds/statsd/src/FieldValue.cpp new file mode 100644 index 0000000000000..7b0b69a4bcae9 --- /dev/null +++ b/cmds/statsd/src/FieldValue.cpp @@ -0,0 +1,186 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define DEBUG false +#include "Log.h" +#include "FieldValue.h" +#include "HashableDimensionKey.h" + +namespace android { +namespace os { +namespace statsd { + +bool Field::matches(const Matcher& matcher) const { + if (mTag != matcher.mMatcher.getTag()) { + return false; + } + if ((mField & matcher.mMask) == matcher.mMatcher.getField()) { + return true; + } + + return false; +}; + +void translateFieldMatcher(int tag, const FieldMatcher& matcher, int depth, int* pos, int* mask, + std::vector* output) { + if (depth > kMaxLogDepth) { + ALOGE("depth > 2"); + return; + } + + pos[depth] = matcher.field(); + mask[depth] = 0x7f; + + if (matcher.has_position()) { + depth++; + if (depth > 2) { + return; + } + switch (matcher.position()) { + case Position::ANY: + pos[depth] = 0; + mask[depth] = 0; + break; + case Position::FIRST: + pos[depth] = 1; + mask[depth] = 0x7f; + break; + case Position::LAST: + pos[depth] = 0x80; + mask[depth] = 0x80; + break; + case Position::POSITION_UNKNOWN: + pos[depth] = 0; + mask[depth] = 0; + break; + } + } + + if (matcher.child_size() == 0) { + output->push_back(Matcher(Field(tag, pos, depth), encodeMatcherMask(mask, depth))); + Matcher matcher = Matcher(Field(tag, pos, depth), encodeMatcherMask(mask, depth)); + } else { + for (const auto& child : matcher.child()) { + translateFieldMatcher(tag, child, depth + 1, pos, mask, output); + } + } +} + +void translateFieldMatcher(const FieldMatcher& matcher, std::vector* output) { + int pos[] = {1, 1, 1}; + int mask[] = {0x7f, 0x7f, 0x7f}; + int tag = matcher.field(); + for (const auto& child : matcher.child()) { + translateFieldMatcher(tag, child, 0, pos, mask, output); + } +} + +bool isAttributionUidField(const FieldValue& value) { + int field = value.mField.getField() & 0xff007f; + if (field == 0x10001 && value.mValue.getType() == INT) { + return true; + } + return false; +} + +bool isAttributionUidField(const Field& field, const Value& value) { + int f = field.getField() & 0xff007f; + if (f == 0x10001 && value.getType() == INT) { + return true; + } + return false; +} + +Value::Value(const Value& from) { + type = from.getType(); + switch (type) { + case INT: + int_value = from.int_value; + break; + case LONG: + long_value = from.long_value; + break; + case FLOAT: + float_value = from.float_value; + break; + case STRING: + str_value = from.str_value; + break; + } +} + +std::string Value::toString() const { + switch (type) { + case INT: + return std::to_string(int_value) + "[I]"; + case LONG: + return std::to_string(long_value) + "[L]"; + case FLOAT: + return std::to_string(float_value) + "[F]"; + case STRING: + return str_value + "[S]"; + } +} + +bool Value::operator==(const Value& that) const { + if (type != that.getType()) return false; + + switch (type) { + case INT: + return int_value == that.int_value; + case LONG: + return long_value == that.long_value; + case FLOAT: + return float_value == that.float_value; + case STRING: + return str_value == that.str_value; + } +} + +bool Value::operator!=(const Value& that) const { + if (type != that.getType()) return true; + switch (type) { + case INT: + return int_value != that.int_value; + case LONG: + return long_value != that.long_value; + case FLOAT: + return float_value != that.float_value; + case STRING: + return str_value != that.str_value; + } +} + +bool Value::operator<(const Value& that) const { + if (type != that.getType()) return type < that.getType(); + + switch (type) { + case INT: + return int_value < that.int_value; + case LONG: + return long_value < that.long_value; + case FLOAT: + return float_value < that.float_value; + case STRING: + return str_value < that.str_value; + default: + return false; + } +} + +} // namespace statsd +} // namespace os +} // namespace android \ No newline at end of file diff --git a/cmds/statsd/src/FieldValue.h b/cmds/statsd/src/FieldValue.h new file mode 100644 index 0000000000000..7484108d9e1a5 --- /dev/null +++ b/cmds/statsd/src/FieldValue.h @@ -0,0 +1,338 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "frameworks/base/cmds/statsd/src/statsd_config.pb.h" + +namespace android { +namespace os { +namespace statsd { + +class HashableDimensionKey; +struct Matcher; +struct Field; +struct FieldValue; + +const int32_t kAttributionField = 1; +const int32_t kMaxLogDepth = 2; +const int32_t kLastBitMask = 0x80; +const int32_t kClearLastBitDeco = 0x7f; + +enum Type { INT, LONG, FLOAT, STRING }; + + +static int32_t getEncodedField(int32_t pos[], int32_t depth, bool includeDepth) { + int32_t field = 0; + for (int32_t i = 0; i <= depth; i++) { + int32_t shiftBits = 8 * (kMaxLogDepth - i); + field |= (pos[i] << shiftBits); + } + + if (includeDepth) { + field |= (depth << 24); + } + return field; +} + +static int32_t encodeMatcherMask(int32_t mask[], int32_t depth) { + return getEncodedField(mask, depth, false) | 0xff000000; +} + +// Get the encoded field for a leaf with a [field] number at depth 0; +static int32_t getSimpleField(size_t field) { + return ((int32_t)field << 8 * 2); +} + +/** + * Field is a wrapper class for 2 integers that represents the field of a log element in its Atom + * proto. + * [mTag]: the atom id. + * [mField]: encoded path from the root (atom) to leaf. + * + * For example: + * WakeLockStateChanged { + * repeated AttributionNode = 1; + * int state = 2; + * string tag = 3; + * } + * Read from logd, the items are structured as below: + * [[[1000, "tag"], [2000, "tag2"],], 2,"hello"] + * + * When we read through the list, we will encode each field in a 32bit integer. + * 8bit segments |--------|--------|--------|--------| + * Depth field0 [L]field1 [L]field1 + * + * The first 8 bits are the depth of the field. for example, the uid 1000 has depth 2. + * The following 3 8-bit are for the item's position at each level. + * The first bit of each 8bits field is reserved to mark if the item is the last item at that level + * this is to make matching easier later. + * + * The above wakelock event is translated into FieldValue pairs. + * 0x02010101->1000 + * 0x02010182->tag + * 0x02018201->2000 + * 0x02018282->tag2 + * 0x00020000->2 + * 0x00030000->"hello" + * + * This encoding is the building block for the later operations. + * Please see the definition for Matcher below to see how the matching is done. + */ +struct Field { +private: + int32_t mTag; + int32_t mField; + +public: + Field(int32_t tag, int32_t pos[], int32_t depth) : mTag(tag) { + mField = getEncodedField(pos, depth, true); + } + + Field(const Field& from) : mTag(from.getTag()), mField(from.getField()) { + } + + Field(int32_t tag, int32_t field) : mTag(tag), mField(field){}; + + inline void setField(int32_t field) { + mField = field; + } + + inline void setTag(int32_t tag) { + mTag = tag; + } + + inline void decorateLastPos(int32_t depth) { + int32_t mask = kLastBitMask << 8 * (kMaxLogDepth - depth); + mField |= mask; + } + + inline int32_t getTag() const { + return mTag; + } + + inline int32_t getDepth() const { + return (mField >> 24); + } + + inline int32_t getPath(int32_t depth) const { + if (depth > 2 || depth < 0) return 0; + + int32_t field = (mField & 0x00ffffff); + int32_t mask = 0xffffffff; + return (field & (mask << 8 * (kMaxLogDepth - depth))); + } + + inline int32_t getPrefix(int32_t depth) const { + if (depth == 0) return 0; + return getPath(depth - 1); + } + + inline int32_t getField() const { + return mField; + } + + inline int32_t getRawPosAtDepth(int32_t depth) const { + int32_t field = (mField & 0x00ffffff); + int32_t shift = 8 * (kMaxLogDepth - depth); + int32_t mask = 0xff << shift; + + return (field & mask) >> shift; + } + + inline int32_t getPosAtDepth(int32_t depth) const { + return getRawPosAtDepth(depth) & kClearLastBitDeco; + } + + // Check if the first bit of the 8-bit segment for depth is 1 + inline bool isLastPos(int32_t depth) const { + int32_t field = (mField & 0x00ffffff); + int32_t mask = kLastBitMask << 8 * (kMaxLogDepth - depth); + return (field & mask) != 0; + } + + // if the 8-bit segment is all 0's + inline bool isAnyPosMatcher(int32_t depth) const { + return getDepth() >= depth && getRawPosAtDepth(depth) == 0; + } + // if the 8bit is 0x80 (1000 0000) + inline bool isLastPosMatcher(int32_t depth) const { + return getDepth() >= depth && getRawPosAtDepth(depth) == kLastBitMask; + } + + inline bool operator==(const Field& that) const { + return mTag == that.getTag() && mField == that.getField(); + }; + + inline bool operator!=(const Field& that) const { + return mTag != that.getTag() || mField != that.getField(); + }; + + bool operator<(const Field& that) const { + if (mTag != that.getTag()) { + return mTag < that.getTag(); + } + + if (mField != that.getField()) { + return mField < that.getField(); + } + + return false; + } + bool matches(const Matcher& that) const; +}; + +/** + * Matcher represents a leaf matcher in the FieldMatcher in statsd_config. + * + * It contains all information needed to match one or more leaf node. + * All information is encoded in a Field(2 ints) and a bit mask(1 int). + * + * For example, to match the first/any/last uid field in attribution chain in Atom 10, + * we have the following FieldMatcher in statsd_config + * FieldMatcher { + * field:10 + * FieldMatcher { + * field:1 + * position: any/last/first + * FieldMatcher { + * field:1 + * } + * } + * } + * + * We translate the FieldMatcher into a Field, and mask + * First: [Matcher Field] 0x02010101 [Mask]0xffff7fff + * Last: [Matcher Field] 0x02018001 [Mask]0xffff80ff + * Any: [Matcher Field] 0x02010001 [Mask]0xffff00ff + * + * [To match a log Field with a Matcher] we apply the bit mask to the log Field and check if + * the result is equal to the Matcher Field. That's a bit wise AND operation + check if 2 ints are + * equal. Nothing can beat the performance of this matching algorithm. + * + * TODO: ADD EXAMPLE HERE. + */ +struct Matcher { + Matcher(const Field& matcher, int32_t mask) : mMatcher(matcher), mMask(mask){}; + + const Field mMatcher; + const int32_t mMask; + + bool hasAnyPositionMatcher(int* prefix) const { + if (mMatcher.getDepth() == 2 && mMatcher.getRawPosAtDepth(2) == 0) { + (*prefix) = mMatcher.getPrefix(2); + return true; + } + return false; + } +}; + +/** + * A wrapper for a union type to contain multiple types of values. + * + */ +struct Value { + Value(int32_t v) { + int_value = v; + type = INT; + } + + Value(int64_t v) { + long_value = v; + type = LONG; + } + + Value(float v) { + float_value = v; + type = FLOAT; + } + + Value(const std::string& v) { + str_value = v; + type = STRING; + } + + void setInt(int32_t v) { + int_value = v; + type = INT; + } + + void setLong(int64_t v) { + long_value = v; + type = LONG; + } + + union { + int32_t int_value; + int64_t long_value; + float float_value; + }; + std::string str_value; + + Type type; + + std::string toString() const; + + Type getType() const { + return type; + } + + Value(const Value& from); + + bool operator==(const Value& that) const; + bool operator!=(const Value& that) const; + + bool operator<(const Value& that) const; + +private: + Value(){}; +}; + +/** + * Represents a log item, or a dimension item (They are essentially the same). + */ +struct FieldValue { + FieldValue(const Field& field, const Value& value) : mField(field), mValue(value) { + } + bool operator==(const FieldValue& that) const { + return mField == that.mField && mValue == that.mValue; + } + bool operator!=(const FieldValue& that) const { + return mField != that.mField || mValue != that.mValue; + } + bool operator<(const FieldValue& that) const { + if (mField != that.mField) { + return mField < that.mField; + } + + if (mValue != that.mValue) { + return mValue < that.mValue; + } + + return false; + } + + Field mField; + Value mValue; +}; + +bool isAttributionUidField(const FieldValue& value); + +void translateFieldMatcher(const FieldMatcher& matcher, std::vector* output); + +bool isAttributionUidField(const Field& field, const Value& value); +} // namespace statsd +} // namespace os +} // namespace android diff --git a/cmds/statsd/src/HashableDimensionKey.cpp b/cmds/statsd/src/HashableDimensionKey.cpp index 8483b024aa55b..68e2176c2e6de 100644 --- a/cmds/statsd/src/HashableDimensionKey.cpp +++ b/cmds/statsd/src/HashableDimensionKey.cpp @@ -13,177 +13,253 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +#define DEBUG false // STOPSHIP if true +#include "Log.h" #include "HashableDimensionKey.h" -#include "dimension.h" +#include "FieldValue.h" namespace android { namespace os { namespace statsd { +using std::vector; -android::hash_t hashDimensionsValue(int64_t seed, const DimensionsValue& value) { - android::hash_t hash = seed; - hash = android::JenkinsHashMix(hash, android::hash_type(value.field())); - - hash = android::JenkinsHashMix(hash, android::hash_type((int)value.value_case())); - switch (value.value_case()) { - case DimensionsValue::ValueCase::kValueStr: - hash = android::JenkinsHashMix( - hash, - static_cast(std::hash()(value.value_str()))); - break; - case DimensionsValue::ValueCase::kValueInt: - hash = android::JenkinsHashMix(hash, android::hash_type(value.value_int())); - break; - case DimensionsValue::ValueCase::kValueLong: - hash = android::JenkinsHashMix( - hash, android::hash_type(static_cast(value.value_long()))); - break; - case DimensionsValue::ValueCase::kValueBool: - hash = android::JenkinsHashMix(hash, android::hash_type(value.value_bool())); - break; - case DimensionsValue::ValueCase::kValueFloat: { - float floatVal = value.value_float(); - hash = android::JenkinsHashMixBytes(hash, (uint8_t*)&floatVal, sizeof(float)); - break; - } - case DimensionsValue::ValueCase::kValueTuple: { - hash = android::JenkinsHashMix(hash, android::hash_type( - value.value_tuple().dimensions_value_size())); - for (int i = 0; i < value.value_tuple().dimensions_value_size(); ++i) { - hash = android::JenkinsHashMix( - hash, - hashDimensionsValue(value.value_tuple().dimensions_value(i))); +android::hash_t hashDimension(const HashableDimensionKey& value) { + android::hash_t hash = 0; + for (const auto& fieldValue : value.getValues()) { + hash = android::JenkinsHashMix(hash, android::hash_type((int)fieldValue.mField.getField())); + hash = android::JenkinsHashMix(hash, android::hash_type((int)fieldValue.mField.getTag())); + hash = android::JenkinsHashMix(hash, android::hash_type((int)fieldValue.mValue.getType())); + switch (fieldValue.mValue.getType()) { + case INT: + hash = android::JenkinsHashMix(hash, + android::hash_type(fieldValue.mValue.int_value)); + break; + case LONG: + hash = android::JenkinsHashMix(hash, + android::hash_type(fieldValue.mValue.long_value)); + break; + case STRING: + hash = android::JenkinsHashMix(hash, static_cast(std::hash()( + fieldValue.mValue.str_value))); + break; + case FLOAT: { + float floatVal = fieldValue.mValue.float_value; + hash = android::JenkinsHashMixBytes(hash, (uint8_t*)&floatVal, sizeof(float)); + break; } - break; } - case DimensionsValue::ValueCase::VALUE_NOT_SET: - break; } return JenkinsHashWhiten(hash); } -android::hash_t hashDimensionsValue(const DimensionsValue& value) { - return hashDimensionsValue(0, value); -} - -android::hash_t hashMetricDimensionKey(int64_t seed, const MetricDimensionKey& dimensionKey) { - android::hash_t hash = seed; - hash = android::JenkinsHashMix(hash, std::hash{}(dimensionKey)); - return JenkinsHashWhiten(hash); -} - -using std::string; - -string HashableDimensionKey::toString() const { - return DimensionsValueToString(getDimensionsValue()); -} - -bool EqualsTo(const DimensionsValue& s1, const DimensionsValue& s2) { - if (s1.field() != s2.field()) { - return false; - } - if (s1.value_case() != s2.value_case()) { - return false; - } - switch (s1.value_case()) { - case DimensionsValue::ValueCase::kValueStr: - return (s1.value_str() == s2.value_str()); - case DimensionsValue::ValueCase::kValueInt: - return s1.value_int() == s2.value_int(); - case DimensionsValue::ValueCase::kValueLong: - return s1.value_long() == s2.value_long(); - case DimensionsValue::ValueCase::kValueBool: - return s1.value_bool() == s2.value_bool(); - case DimensionsValue::ValueCase::kValueFloat: - return s1.value_float() == s2.value_float(); - case DimensionsValue::ValueCase::kValueTuple: - { - if (s1.value_tuple().dimensions_value_size() != - s2.value_tuple().dimensions_value_size()) { - return false; - } - bool allMatched = true; - for (int i = 0; allMatched && i < s1.value_tuple().dimensions_value_size(); ++i) { - allMatched &= EqualsTo(s1.value_tuple().dimensions_value(i), - s2.value_tuple().dimensions_value(i)); - } - return allMatched; +// Filter fields using the matchers and output the results as a HashableDimensionKey. +// Note: HashableDimensionKey is just a wrapper for vector +bool filterValues(const vector& matcherFields, const vector& values, + vector* output) { + output->push_back(HashableDimensionKey()); + // Top level is only tag id. Now take the real child matchers + int prevAnyMatcherPrefix = 0; + size_t prevPrevFanout = 0; + size_t prevFanout = 0; + // For each matcher get matched results. + for (const auto& matcher : matcherFields) { + vector matchedResults; + for (const auto& value : values) { + // TODO: potential optimization here to break early because all fields are naturally + // sorted. + int32_t filteredField; + if (value.mField.matches(matcher)) { + matchedResults.push_back(FieldValue( + Field(value.mField.getTag(), (value.mField.getField() & matcher.mMask)), + value.mValue)); } - case DimensionsValue::ValueCase::VALUE_NOT_SET: - default: - return true; - } -} + } -bool LessThan(const DimensionsValue& s1, const DimensionsValue& s2) { - if (s1.field() != s2.field()) { - return s1.field() < s2.field(); - } - if (s1.value_case() != s2.value_case()) { - return s1.value_case() < s2.value_case(); - } - switch (s1.value_case()) { - case DimensionsValue::ValueCase::kValueStr: - return s1.value_str() < s2.value_str(); - case DimensionsValue::ValueCase::kValueInt: - return s1.value_int() < s2.value_int(); - case DimensionsValue::ValueCase::kValueLong: - return s1.value_long() < s2.value_long(); - case DimensionsValue::ValueCase::kValueBool: - return (int)s1.value_bool() < (int)s2.value_bool(); - case DimensionsValue::ValueCase::kValueFloat: - return s1.value_float() < s2.value_float(); - case DimensionsValue::ValueCase::kValueTuple: - { - if (s1.value_tuple().dimensions_value_size() != - s2.value_tuple().dimensions_value_size()) { - return s1.value_tuple().dimensions_value_size() < - s2.value_tuple().dimensions_value_size(); - } - for (int i = 0; i < s1.value_tuple().dimensions_value_size(); ++i) { - if (EqualsTo(s1.value_tuple().dimensions_value(i), - s2.value_tuple().dimensions_value(i))) { - continue; - } else { - return LessThan(s1.value_tuple().dimensions_value(i), - s2.value_tuple().dimensions_value(i)); - } - } + if (matchedResults.size() == 0) { + VLOG("We can't find a dimension value for matcher (%d)%#x.", matcher.mMatcher.getTag(), + matcher.mMatcher.getField()); + continue; + } + + if (matchedResults.size() == 1) { + for (auto& dimension : *output) { + dimension.addValue(matchedResults[0]); + } + prevAnyMatcherPrefix = 0; + prevFanout = 0; + continue; + } + + // All the complexity below is because we support ANY in dimension. + bool createFanout = true; + // createFanout is true when the matcher doesn't need to follow the prev matcher's + // order. + // e.g., get (uid, tag) from any position in attribution. because we have translated + // it as 2 matchers, they need to follow the same ordering, we can't create a cross + // product of all uid and tags. + // However, if the 2 matchers have different prefix, they will create a cross product + // e.g., [any uid] [any some other repeated field], we will create a cross product for them + if (prevAnyMatcherPrefix != 0) { + int anyMatcherPrefix = 0; + bool isAnyMatcher = matcher.hasAnyPositionMatcher(&anyMatcherPrefix); + if (isAnyMatcher && anyMatcherPrefix == prevAnyMatcherPrefix) { + createFanout = false; + } else { + prevAnyMatcherPrefix = anyMatcherPrefix; + } + } + + // Each matcher should match exact one field, unless position is ANY + // When x number of fields matches a matcher, the returned dimension + // size is multiplied by x. + int oldSize; + if (createFanout) { + // First create fanout (fanout size is matchedResults.Size which could be one, + // which means we do nothing here) + oldSize = output->size(); + for (size_t i = 1; i < matchedResults.size(); i++) { + output->insert(output->end(), output->begin(), output->begin() + oldSize); + } + prevPrevFanout = oldSize; + prevFanout = matchedResults.size(); + } else { + // If we should not create fanout, e.g., uid tag from same position should be remain + // together. + oldSize = prevPrevFanout; + if (prevFanout != matchedResults.size()) { + // sanity check. + ALOGE("2 Any matcher result in different output"); return false; } - case DimensionsValue::ValueCase::VALUE_NOT_SET: - default: - return false; + } + // now add the matched field value to output + for (size_t i = 0; i < matchedResults.size(); i++) { + for (int j = 0; j < oldSize; j++) { + (*output)[i * oldSize + j].addValue(matchedResults[i]); + } + } } + + return output->size() > 0 && (*output)[0].getValues().size() > 0; +} + +void filterGaugeValues(const std::vector& matcherFields, + const std::vector& values, std::vector* output) { + for (const auto& field : matcherFields) { + for (const auto& value : values) { + int filteredField; + if (value.mField.matches(field)) { + output->push_back(value); + } + } + } +} + +void getDimensionForCondition(const LogEvent& event, Metric2Condition links, + vector* conditionDimension) { + // Get the dimension first by using dimension from what. + filterValues(links.metricFields, event.getValues(), conditionDimension); + + // Then replace the field with the dimension from condition. + for (auto& dim : *conditionDimension) { + size_t count = dim.getValues().size(); + if (count != links.conditionFields.size()) { + // ALOGE("WTF condition link is bad"); + return; + } + + for (size_t i = 0; i < count; i++) { + dim.mutableValue(i)->mField.setField(links.conditionFields[i].mMatcher.getField()); + dim.mutableValue(i)->mField.setTag(links.conditionFields[i].mMatcher.getTag()); + } + } +} + +bool LessThan(const vector& s1, const vector& s2) { + if (s1.size() != s2.size()) { + return s1.size() < s2.size(); + } + + size_t count = s1.size(); + for (size_t i = 0; i < count; i++) { + if (s1[i] != s2[i]) { + return s1[i] < s2[i]; + } + } + return false; } bool HashableDimensionKey::operator==(const HashableDimensionKey& that) const { - return EqualsTo(getDimensionsValue(), that.getDimensionsValue()); + if (mValues.size() != that.getValues().size()) { + return false; + } + size_t count = mValues.size(); + for (size_t i = 0; i < count; i++) { + if (mValues[i] != (that.getValues())[i]) { + return false; + } + } + return true; }; bool HashableDimensionKey::operator<(const HashableDimensionKey& that) const { - return LessThan(getDimensionsValue(), that.getDimensionsValue()); + return LessThan(getValues(), that.getValues()); }; -string MetricDimensionKey::toString() const { - string flattened = mDimensionKeyInWhat.toString(); - flattened += mDimensionKeyInCondition.toString(); - return flattened; +bool HashableDimensionKey::contains(const HashableDimensionKey& that) const { + if (mValues.size() < that.getValues().size()) { + return false; + } + + if (mValues.size() == that.getValues().size()) { + return (*this) == that; + } + + for (const auto& value : that.getValues()) { + bool found = false; + for (const auto& myValue : mValues) { + if (value.mField == myValue.mField && value.mValue == myValue.mValue) { + found = true; + break; + } + } + if (!found) { + return false; + } + } + + return true; +} + +string HashableDimensionKey::toString() const { + std::string output; + for (const auto& value : mValues) { + output += StringPrintf("(%d)%#x->%s ", value.mField.getTag(), value.mField.getField(), + value.mValue.toString().c_str()); + } + return output; } bool MetricDimensionKey::operator==(const MetricDimensionKey& that) const { return mDimensionKeyInWhat == that.getDimensionKeyInWhat() && - mDimensionKeyInCondition == that.getDimensionKeyInCondition(); + mDimensionKeyInCondition == that.getDimensionKeyInCondition(); }; +string MetricDimensionKey::toString() const { + return mDimensionKeyInWhat.toString() + mDimensionKeyInCondition.toString(); +} + bool MetricDimensionKey::operator<(const MetricDimensionKey& that) const { - return toString().compare(that.toString()) < 0; -}; + if (mDimensionKeyInWhat < that.getDimensionKeyInWhat()) { + return true; + } else if (that.getDimensionKeyInWhat() < mDimensionKeyInWhat) { + return false; + } -bool compareDimensionsValue(const DimensionsValue& s1, const DimensionsValue& s2) { - return EqualsTo(s1, s2); + return mDimensionKeyInCondition < that.getDimensionKeyInCondition(); } + } // namespace statsd } // namespace os } // namespace android \ No newline at end of file diff --git a/cmds/statsd/src/HashableDimensionKey.h b/cmds/statsd/src/HashableDimensionKey.h index a31d7a6d85c6c..89fe317834d81 100644 --- a/cmds/statsd/src/HashableDimensionKey.h +++ b/cmds/statsd/src/HashableDimensionKey.h @@ -17,44 +17,66 @@ #pragma once #include -#include "frameworks/base/cmds/statsd/src/stats_log.pb.h" +#include +#include "FieldValue.h" +#include "android-base/stringprintf.h" +#include "logd/LogEvent.h" namespace android { namespace os { namespace statsd { +using android::base::StringPrintf; + +struct Metric2Condition { + int64_t conditionId; + std::vector metricFields; + std::vector conditionFields; +}; + class HashableDimensionKey { public: - explicit HashableDimensionKey(const DimensionsValue& dimensionsValue) - : mDimensionsValue(dimensionsValue){}; + explicit HashableDimensionKey(const std::vector& values) { + mValues = values; + } HashableDimensionKey(){}; - HashableDimensionKey(const HashableDimensionKey& that) - : mDimensionsValue(that.getDimensionsValue()){}; + HashableDimensionKey(const HashableDimensionKey& that) : mValues(that.getValues()){}; - HashableDimensionKey& operator=(const HashableDimensionKey& from) = default; + inline void addValue(const FieldValue& value) { + mValues.push_back(value); + } + + inline const std::vector& getValues() const { + return mValues; + } + + inline std::vector* mutableValues() { + return &mValues; + } + + inline FieldValue* mutableValue(size_t i) { + if (i >= 0 && i < mValues.size()) { + return &(mValues[i]); + } + return nullptr; + } std::string toString() const; - inline const DimensionsValue& getDimensionsValue() const { - return mDimensionsValue; - } - - inline DimensionsValue* getMutableDimensionsValue() { - return &mDimensionsValue; + inline const char* c_str() const { + return toString().c_str(); } bool operator==(const HashableDimensionKey& that) const; bool operator<(const HashableDimensionKey& that) const; - inline const char* c_str() const { - return toString().c_str(); - } + bool contains(const HashableDimensionKey& that) const; private: - DimensionsValue mDimensionsValue; + std::vector mValues; }; class MetricDimensionKey { @@ -83,7 +105,7 @@ class MetricDimensionKey { } bool hasDimensionKeyInCondition() const { - return mDimensionKeyInCondition.getDimensionsValue().has_field(); + return mDimensionKeyInCondition.getValues().size() > 0; } bool operator==(const MetricDimensionKey& that) const; @@ -98,11 +120,32 @@ class MetricDimensionKey { HashableDimensionKey mDimensionKeyInCondition; }; -bool compareDimensionsValue(const DimensionsValue& s1, const DimensionsValue& s2); +android::hash_t hashDimension(const HashableDimensionKey& key); -android::hash_t hashDimensionsValue(int64_t seed, const DimensionsValue& value); -android::hash_t hashDimensionsValue(const DimensionsValue& value); -android::hash_t hashMetricDimensionKey(int64_t see, const MetricDimensionKey& dimensionKey); +/** + * Creating HashableDimensionKeys from FieldValues using matcher. + * + * This function may make modifications to the Field if the matcher has Position=LAST or ANY in + * it. This is because: for example, when we create dimension from last uid in attribution chain, + * In one event, uid 1000 is at position 5 and it's the last + * In another event, uid 1000 is at position 6, and it's the last + * these 2 events should be mapped to the same dimension. So we will remove the original position + * from the dimension key for the uid field (by applying 0x80 bit mask). + */ +bool filterValues(const std::vector& matcherFields, const std::vector& values, + std::vector* output); + +/** + * Filter the values from FieldValues using the matchers. + * + * In contrast to the above function, this function will not do any modification to the original + * data. Considering it as taking a snapshot on the atom event. + */ +void filterGaugeValues(const std::vector& matchers, const std::vector& values, + std::vector* output); + +void getDimensionForCondition(const LogEvent& event, Metric2Condition links, + std::vector* conditionDimension); } // namespace statsd } // namespace os @@ -116,17 +159,15 @@ using android::os::statsd::MetricDimensionKey; template <> struct hash { std::size_t operator()(const HashableDimensionKey& key) const { - return hashDimensionsValue(key.getDimensionsValue()); + return hashDimension(key); } }; template <> struct hash { std::size_t operator()(const MetricDimensionKey& key) const { - android::hash_t hash = hashDimensionsValue( - key.getDimensionKeyInWhat().getDimensionsValue()); - hash = android::JenkinsHashMix(hash, - hashDimensionsValue(key.getDimensionKeyInCondition().getDimensionsValue())); + android::hash_t hash = hashDimension(key.getDimensionKeyInWhat()); + hash = android::JenkinsHashMix(hash, hashDimension(key.getDimensionKeyInCondition())); return android::JenkinsHashWhiten(hash); } }; diff --git a/cmds/statsd/src/StatsLogProcessor.cpp b/cmds/statsd/src/StatsLogProcessor.cpp index 7662c4025c42e..4fac5aa7c8e77 100644 --- a/cmds/statsd/src/StatsLogProcessor.cpp +++ b/cmds/statsd/src/StatsLogProcessor.cpp @@ -14,7 +14,7 @@ * limitations under the License. */ -#define DEBUG false // STOPSHIP if true +#define DEBUG true // STOPSHIP if true #include "Log.h" #include "statslog.h" @@ -25,8 +25,6 @@ #include "guardrail/StatsdStats.h" #include "metrics/CountMetricProducer.h" #include "external/StatsPullerManager.h" -#include "dimension.h" -#include "field_util.h" #include "stats_util.h" #include "storage/StorageManager.h" @@ -93,27 +91,31 @@ void StatsLogProcessor::onAnomalyAlarmFired( } } +void updateUid(Value* value, int hostUid) { + int uid = value->int_value; + if (uid != hostUid) { + value->setInt(hostUid); + } +} + void StatsLogProcessor::mapIsolatedUidToHostUidIfNecessaryLocked(LogEvent* event) const { - std::set uidFields; if (android::util::kAtomsWithAttributionChain.find(event->GetTagId()) != android::util::kAtomsWithAttributionChain.end()) { - FieldMatcher matcher; - buildAttributionUidFieldMatcher(event->GetTagId(), Position::ANY, &matcher); - findFields(event->getFieldValueMap(), matcher, &uidFields); - } else if (android::util::kAtomsWithUidField.find(event->GetTagId()) != - android::util::kAtomsWithUidField.end()) { - FieldMatcher matcher; - buildSimpleAtomFieldMatcher( - event->GetTagId(), 1 /* uid is always the 1st field. */, &matcher); - findFields(event->getFieldValueMap(), matcher, &uidFields); - } - - for (const auto& uidField : uidFields) { - DimensionsValue* value = event->findFieldValueOrNull(uidField); - if (value != nullptr && value->value_case() == DimensionsValue::ValueCase::kValueInt) { - const int uid = mUidMap->getHostUidOrSelf(value->value_int()); - value->set_value_int(uid); + for (auto& value : *(event->getMutableValues())) { + if (value.mField.getPosAtDepth(0) > kAttributionField) { + break; + } + if (isAttributionUidField(value)) { + const int hostUid = mUidMap->getHostUidOrSelf(value.mValue.int_value); + updateUid(&value.mValue, hostUid); + } } + } else if (android::util::kAtomsWithUidField.find(event->GetTagId()) != + android::util::kAtomsWithUidField.end() && + event->getValues().size() > 0 && (event->getValues())[0].mValue.getType() == INT) { + Value& value = (*event->getMutableValues())[0].mValue; + const int hostUid = mUidMap->getHostUidOrSelf(value.int_value); + updateUid(&value, hostUid); } } @@ -212,27 +214,14 @@ void StatsLogProcessor::dumpStates(FILE* out, bool verbose) { } } -void StatsLogProcessor::onDumpReport(const ConfigKey& key, const uint64_t& dumpTimeStampNs, - ConfigMetricsReportList* report) { +void StatsLogProcessor::onDumpReport(const ConfigKey& key, const uint64_t dumpTimeStampNs, + vector* outData) { std::lock_guard lock(mMetricsMutex); - auto it = mMetricsManagers.find(key); - if (it == mMetricsManagers.end()) { - ALOGW("Config source %s does not exist", key.ToString().c_str()); - return; - } - report->mutable_config_key()->set_uid(key.GetUid()); - report->mutable_config_key()->set_id(key.GetId()); - ConfigMetricsReport* configMetricsReport = report->add_reports(); - it->second->onDumpReport(dumpTimeStampNs, configMetricsReport); - // TODO: dump uid mapping. + onDumpReportLocked(key, dumpTimeStampNs, outData); } -void StatsLogProcessor::onDumpReport(const ConfigKey& key, vector* outData) { - std::lock_guard lock(mMetricsMutex); - onDumpReportLocked(key, outData); -} - -void StatsLogProcessor::onDumpReportLocked(const ConfigKey& key, vector* outData) { +void StatsLogProcessor::onDumpReportLocked(const ConfigKey& key, const uint64_t dumpTimeStampNs, + vector* outData) { auto it = mMetricsManagers.find(key); if (it == mMetricsManagers.end()) { ALOGW("Config source %s does not exist", key.ToString().c_str()); @@ -258,7 +247,7 @@ void StatsLogProcessor::onDumpReportLocked(const ConfigKey& key, vector // First, fill in ConfigMetricsReport using current data on memory, which // starts from filling in StatsLogReport's. - it->second->onDumpReport(&proto); + it->second->onDumpReport(dumpTimeStampNs, &proto); // Fill in UidMap. auto uidMap = mUidMap->getOutput(key); @@ -292,6 +281,7 @@ void StatsLogProcessor::onDumpReportLocked(const ConfigKey& key, vector iter.rp()->move(toRead); } } + StatsdStats::getInstance().noteMetricsReportSent(key); } @@ -327,7 +317,7 @@ void StatsLogProcessor::flushIfNecessaryLocked( StatsdStats::kMaxMetricsBytesPerConfig) { // Too late. We need to start clearing data. // TODO(b/70571383): By 12/15/2017 add API to drop data directly ProtoOutputStream proto; - metricsManager.onDumpReport(&proto); + metricsManager.onDumpReport(time(nullptr) * NS_PER_SEC, &proto); StatsdStats::getInstance().noteDataDropped(key); VLOG("StatsD had to toss out metrics for %s", key.ToString().c_str()); } else if (totalBytes > .9 * StatsdStats::kMaxMetricsBytesPerConfig) { @@ -351,7 +341,7 @@ void StatsLogProcessor::WriteDataToDisk() { for (auto& pair : mMetricsManagers) { const ConfigKey& key = pair.first; vector data; - onDumpReportLocked(key, &data); + onDumpReportLocked(key, time(nullptr) * NS_PER_SEC, &data); // TODO: Add a guardrail to prevent accumulation of file on disk. string file_name = StringPrintf("%s/%ld_%d_%lld", STATS_DATA_DIR, time(nullptr), key.GetUid(), (long long)key.GetId()); diff --git a/cmds/statsd/src/StatsLogProcessor.h b/cmds/statsd/src/StatsLogProcessor.h index 8bbcd751252a6..144430639d9f4 100644 --- a/cmds/statsd/src/StatsLogProcessor.h +++ b/cmds/statsd/src/StatsLogProcessor.h @@ -46,9 +46,7 @@ public: size_t GetMetricsSize(const ConfigKey& key) const; - void onDumpReport(const ConfigKey& key, vector* outData); - void onDumpReport(const ConfigKey& key, const uint64_t& dumpTimeStampNs, - ConfigMetricsReportList* report); + void onDumpReport(const ConfigKey& key, const uint64_t dumpTimeNs, vector* outData); /* Tells MetricsManager that the alarms in anomalySet have fired. Modifies anomalySet. */ void onAnomalyAlarmFired( @@ -80,7 +78,8 @@ private: sp mAnomalyMonitor; - void onDumpReportLocked(const ConfigKey& key, vector* outData); + void onDumpReportLocked(const ConfigKey& key, const uint64_t dumpTimeNs, + vector* outData); /* Check if we should send a broadcast if approaching memory limits and if we're over, we * actually delete the data. */ @@ -105,9 +104,14 @@ private: FRIEND_TEST(StatsLogProcessorTest, TestRateLimitBroadcast); FRIEND_TEST(StatsLogProcessorTest, TestDropWhenByteSizeTooLarge); FRIEND_TEST(StatsLogProcessorTest, TestDropWhenByteSizeTooLarge); - FRIEND_TEST(WakelockDurationE2eTest, TestAggregatedPredicateDimensionsForSumDuration); - FRIEND_TEST(WakelockDurationE2eTest, TestAggregatedPredicateDimensionsForMaxDuration); - FRIEND_TEST(MetricConditionLinkE2eTest, TestMultiplePredicatesAndLinks); + FRIEND_TEST(WakelockDurationE2eTest, TestAggregatedPredicateDimensionsForSumDuration1); + FRIEND_TEST(WakelockDurationE2eTest, TestAggregatedPredicateDimensionsForSumDuration2); + FRIEND_TEST(WakelockDurationE2eTest, TestAggregatedPredicateDimensionsForSumDuration3); + FRIEND_TEST(WakelockDurationE2eTest, TestAggregatedPredicateDimensionsForMaxDuration1); + FRIEND_TEST(WakelockDurationE2eTest, TestAggregatedPredicateDimensionsForMaxDuration2); + FRIEND_TEST(WakelockDurationE2eTest, TestAggregatedPredicateDimensionsForMaxDuration3); + FRIEND_TEST(MetricConditionLinkE2eTest, TestMultiplePredicatesAndLinks1); + FRIEND_TEST(MetricConditionLinkE2eTest, TestMultiplePredicatesAndLinks2); FRIEND_TEST(AttributionE2eTest, TestAttributionMatchAndSlice); FRIEND_TEST(GaugeMetricE2eTest, TestMultipleFieldsForPushedEvent); FRIEND_TEST(DimensionInConditionE2eTest, TestCountMetricNoLink); diff --git a/cmds/statsd/src/StatsService.cpp b/cmds/statsd/src/StatsService.cpp index c24efec4c4fc0..e111f5854a349 100644 --- a/cmds/statsd/src/StatsService.cpp +++ b/cmds/statsd/src/StatsService.cpp @@ -492,7 +492,8 @@ status_t StatsService::cmd_dump_report(FILE* out, FILE* err, const Vector data; - mProcessor->onDumpReport(ConfigKey(uid, StrToInt64(name)), &data); + mProcessor->onDumpReport(ConfigKey(uid, StrToInt64(name)), time(nullptr) * NS_PER_SEC, + &data); // TODO: print the returned StatsLogReport to file instead of printing to logcat. if (proto) { for (size_t i = 0; i < data.size(); i ++) { @@ -780,7 +781,7 @@ Status StatsService::getData(int64_t key, vector* output) { VLOG("StatsService::getData with Pid %i, Uid %i", ipc->getCallingPid(), ipc->getCallingUid()); if (checkCallingPermission(String16(kPermissionDump))) { ConfigKey configKey(ipc->getCallingUid(), key); - mProcessor->onDumpReport(configKey, output); + mProcessor->onDumpReport(configKey, time(nullptr) * NS_PER_SEC, output); return Status::ok(); } else { return Status::fromExceptionCode(binder::Status::EX_SECURITY); diff --git a/cmds/statsd/src/condition/CombinationConditionTracker.cpp b/cmds/statsd/src/condition/CombinationConditionTracker.cpp index 4c20ccb61afe5..13a2b7be9152b 100644 --- a/cmds/statsd/src/condition/CombinationConditionTracker.cpp +++ b/cmds/statsd/src/condition/CombinationConditionTracker.cpp @@ -106,11 +106,9 @@ bool CombinationConditionTracker::init(const vector& allConditionConf } void CombinationConditionTracker::isConditionMet( - const ConditionKey& conditionParameters, - const vector>& allConditions, - const FieldMatcher& dimensionFields, - vector& conditionCache, - std::unordered_set &dimensionsKeySet) const { + const ConditionKey& conditionParameters, const vector>& allConditions, + const std::vector& dimensionFields, vector& conditionCache, + std::unordered_set& dimensionsKeySet) const { // So far, this is fine as there is at most one child having sliced output. for (const int childIndex : mChildren) { if (conditionCache[childIndex] == ConditionState::kNotEvaluated) { @@ -169,8 +167,8 @@ void CombinationConditionTracker::evaluateCondition( ConditionState CombinationConditionTracker::getMetConditionDimension( const std::vector>& allConditions, - const FieldMatcher& dimensionFields, - std::unordered_set &dimensionsKeySet) const { + const std::vector& dimensionFields, + std::unordered_set& dimensionsKeySet) const { vector conditionCache(allConditions.size(), ConditionState::kNotEvaluated); // So far, this is fine as there is at most one child having sliced output. for (const int childIndex : mChildren) { diff --git a/cmds/statsd/src/condition/CombinationConditionTracker.h b/cmds/statsd/src/condition/CombinationConditionTracker.h index 0b7f9492d628d..ba185f6661ecc 100644 --- a/cmds/statsd/src/condition/CombinationConditionTracker.h +++ b/cmds/statsd/src/condition/CombinationConditionTracker.h @@ -41,17 +41,17 @@ public: std::vector& conditionCache, std::vector& changedCache) override; - void isConditionMet( - const ConditionKey& conditionParameters, - const std::vector>& allConditions, - const FieldMatcher& dimensionFields, - std::vector& conditionCache, - std::unordered_set &dimensionsKeySet) const override; + void isConditionMet(const ConditionKey& conditionParameters, + const std::vector>& allConditions, + const vector& dimensionFields, + std::vector& conditionCache, + std::unordered_set& dimensionsKeySet) const override; ConditionState getMetConditionDimension( const std::vector>& allConditions, - const FieldMatcher& dimensionFields, - std::unordered_set &dimensionsKeySet) const override; + const vector& dimensionFields, + std::unordered_set& dimensionsKeySet) const override; + private: LogicalOperation mLogicalOperation; diff --git a/cmds/statsd/src/condition/ConditionTracker.h b/cmds/statsd/src/condition/ConditionTracker.h index 81abbdb36ee4a..2612a9a06da1b 100644 --- a/cmds/statsd/src/condition/ConditionTracker.h +++ b/cmds/statsd/src/condition/ConditionTracker.h @@ -90,14 +90,13 @@ public: virtual void isConditionMet( const ConditionKey& conditionParameters, const std::vector>& allConditions, - const FieldMatcher& dimensionFields, - std::vector& conditionCache, - std::unordered_set &dimensionsKeySet) const = 0; + const vector& dimensionFields, std::vector& conditionCache, + std::unordered_set& dimensionsKeySet) const = 0; virtual ConditionState getMetConditionDimension( const std::vector>& allConditions, - const FieldMatcher& dimensionFields, - std::unordered_set &dimensionsKeySet) const = 0; + const vector& dimensionFields, + std::unordered_set& dimensionsKeySet) const = 0; // return the list of LogMatchingTracker index that this ConditionTracker uses. virtual const std::set& getLogTrackerIndex() const { diff --git a/cmds/statsd/src/condition/ConditionWizard.cpp b/cmds/statsd/src/condition/ConditionWizard.cpp index 0427700fec91f..c8722c362fe69 100644 --- a/cmds/statsd/src/condition/ConditionWizard.cpp +++ b/cmds/statsd/src/condition/ConditionWizard.cpp @@ -24,11 +24,9 @@ using std::map; using std::string; using std::vector; -ConditionState ConditionWizard::query( - const int index, const ConditionKey& parameters, - const FieldMatcher& dimensionFields, - std::unordered_set *dimensionKeySet) { - +ConditionState ConditionWizard::query(const int index, const ConditionKey& parameters, + const vector& dimensionFields, + std::unordered_set* dimensionKeySet) { vector cache(mAllConditions.size(), ConditionState::kNotEvaluated); mAllConditions[index]->isConditionMet( @@ -37,9 +35,8 @@ ConditionState ConditionWizard::query( } ConditionState ConditionWizard::getMetConditionDimension( - const int index, const FieldMatcher& dimensionFields, - std::unordered_set *dimensionsKeySet) const { - + const int index, const vector& dimensionFields, + std::unordered_set* dimensionsKeySet) const { return mAllConditions[index]->getMetConditionDimension(mAllConditions, dimensionFields, *dimensionsKeySet); } diff --git a/cmds/statsd/src/condition/ConditionWizard.h b/cmds/statsd/src/condition/ConditionWizard.h index b38b59ff4cd0b..4831d5622a3db 100644 --- a/cmds/statsd/src/condition/ConditionWizard.h +++ b/cmds/statsd/src/condition/ConditionWizard.h @@ -39,16 +39,13 @@ public: // condition. // The ConditionTracker at [conditionIndex] can be a CombinationConditionTracker. In this case, // the conditionParameters contains the parameters for it's children SimpleConditionTrackers. - virtual ConditionState query( - const int conditionIndex, - const ConditionKey& conditionParameters, - const FieldMatcher& dimensionFields, - std::unordered_set *dimensionKeySet); + virtual ConditionState query(const int conditionIndex, const ConditionKey& conditionParameters, + const vector& dimensionFields, + std::unordered_set* dimensionKeySet); virtual ConditionState getMetConditionDimension( - const int index, - const FieldMatcher& dimensionFields, - std::unordered_set *dimensionsKeySet) const; + const int index, const vector& dimensionFields, + std::unordered_set* dimensionsKeySet) const; private: std::vector> mAllConditions; diff --git a/cmds/statsd/src/condition/SimpleConditionTracker.cpp b/cmds/statsd/src/condition/SimpleConditionTracker.cpp index 25265d5dabd74..624119f3ad045 100644 --- a/cmds/statsd/src/condition/SimpleConditionTracker.cpp +++ b/cmds/statsd/src/condition/SimpleConditionTracker.cpp @@ -19,7 +19,6 @@ #include "SimpleConditionTracker.h" #include "guardrail/StatsdStats.h" -#include "dimension.h" #include @@ -77,10 +76,12 @@ SimpleConditionTracker::SimpleConditionTracker( mStopAllLogMatcherIndex = -1; } - mOutputDimensions = simplePredicate.dimensions(); - - if (mOutputDimensions.child_size() > 0) { - mSliced = true; + if (simplePredicate.has_dimensions()) { + translateFieldMatcher(simplePredicate.dimensions(), &mOutputDimensions); + if (mOutputDimensions.size() > 0) { + mSliced = true; + mDimensionTag = mOutputDimensions[0].mMatcher.getTag(); + } } if (simplePredicate.initial_value() == SimplePredicate_InitialValue_FALSE) { @@ -104,13 +105,10 @@ bool SimpleConditionTracker::init(const vector& allConditionConfig, vector& stack) { // SimpleConditionTracker does not have dependency on other conditions, thus we just return // if the initialization was successful. - if (mOutputDimensions.has_field() || mOutputDimensions.child_size() > 0) { - setSliced(true); - } return mInitialized; } -void print(map& conditions, const int64_t& id) { +void print(const map& conditions, const int64_t& id) { VLOG("%lld DUMP:", (long long)id); for (const auto& pair : conditions) { VLOG("\t%s : %d", pair.first.c_str(), pair.second); @@ -151,24 +149,15 @@ bool SimpleConditionTracker::hitGuardRail(const HashableDimensionKey& newKey) { } void SimpleConditionTracker::handleConditionEvent(const HashableDimensionKey& outputKey, - bool matchStart, - std::vector& conditionCache, - std::vector& conditionChangedCache) { - if ((int)conditionChangedCache.size() <= mIndex) { - ALOGE("handleConditionEvent: param conditionChangedCache not initialized."); - return; - } - if ((int)conditionCache.size() <= mIndex) { - ALOGE("handleConditionEvent: param conditionCache not initialized."); - return; - } + bool matchStart, ConditionState* conditionCache, + bool* conditionChangedCache) { bool changed = false; auto outputIt = mSlicedConditionState.find(outputKey); ConditionState newCondition; if (hitGuardRail(outputKey)) { - conditionChangedCache[mIndex] = false; + (*conditionChangedCache) = false; // Tells the caller it's evaluated. - conditionCache[mIndex] = ConditionState::kUnknown; + (*conditionCache) = ConditionState::kUnknown; return; } if (outputIt == mSlicedConditionState.end()) { @@ -230,9 +219,8 @@ void SimpleConditionTracker::handleConditionEvent(const HashableDimensionKey& ou print(mSlicedConditionState, mConditionId); } - conditionChangedCache[mIndex] = changed; - conditionCache[mIndex] = newCondition; - + (*conditionChangedCache) = changed; + (*conditionCache) = newCondition; VLOG("SimplePredicate %lld nonSlicedChange? %d", (long long)mConditionId, conditionChangedCache[mIndex] == true); } @@ -292,42 +280,42 @@ void SimpleConditionTracker::evaluateCondition( return; } - // outputKey is the output values. e.g, uid:1234 - std::vector outputValues; - getDimensionKeys(event, mOutputDimensions, &outputValues); - if (outputValues.size() == 0) { - // The original implementation would generate an empty string dimension hash when condition - // is not sliced. - handleConditionEvent( - DEFAULT_DIMENSION_KEY, matchedState == 1, conditionCache, conditionChangedCache); - } else if (outputValues.size() == 1) { - handleConditionEvent(HashableDimensionKey(outputValues[0]), matchedState == 1, - conditionCache, conditionChangedCache); + ConditionState overallState = mInitialValue; + bool overallChanged = false; + + if (mOutputDimensions.size() == 0) { + handleConditionEvent(DEFAULT_DIMENSION_KEY, matchedState == 1, &overallState, + &overallChanged); } else { + std::vector outputValues; + filterValues(mOutputDimensions, event.getValues(), &outputValues); + // If this event has multiple nodes in the attribution chain, this log event probably will // generate multiple dimensions. If so, we will find if the condition changes for any // dimension and ask the corresponding metric producer to verify whether the actual sliced // condition has changed or not. // A high level assumption is that a predicate is either sliced or unsliced. We will never // have both sliced and unsliced version of a predicate. - for (const DimensionsValue& outputValue : outputValues) { - vector dimensionalConditionCache(conditionCache.size(), - ConditionState::kNotEvaluated); - vector dimensionalConditionChangedCache(conditionChangedCache.size(), false); - handleConditionEvent(HashableDimensionKey(outputValue), matchedState == 1, - dimensionalConditionCache, dimensionalConditionChangedCache); - OrConditionState(dimensionalConditionCache, &conditionCache); - OrBooleanVector(dimensionalConditionChangedCache, &conditionChangedCache); + for (const HashableDimensionKey& outputValue : outputValues) { + // For sliced conditions, the value in the cache is not used. We don't need to update + // the overall condition state. + ConditionState tempState = ConditionState::kUnknown; + bool tempChanged = false; + handleConditionEvent(outputValue, matchedState == 1, &tempState, &tempChanged); + if (tempChanged) { + overallChanged = true; + } } } + conditionCache[mIndex] = overallState; + conditionChangedCache[mIndex] = overallChanged; } void SimpleConditionTracker::isConditionMet( - const ConditionKey& conditionParameters, - const vector>& allConditions, - const FieldMatcher& dimensionFields, - vector& conditionCache, - std::unordered_set &dimensionsKeySet) const { + const ConditionKey& conditionParameters, const vector>& allConditions, + const vector& dimensionFields, vector& conditionCache, + std::unordered_set& dimensionsKeySet) const { + if (conditionCache[mIndex] != ConditionState::kNotEvaluated) { // it has been evaluated. VLOG("Yes, already evaluated, %lld %d", @@ -338,8 +326,7 @@ void SimpleConditionTracker::isConditionMet( if (pair == conditionParameters.end()) { ConditionState conditionState = ConditionState::kNotEvaluated; - if (dimensionFields.has_field() && dimensionFields.child_size() > 0 && - dimensionFields.field() == mOutputDimensions.field()) { + if (dimensionFields.size() > 0 && dimensionFields[0].mMatcher.getTag() == mDimensionTag) { conditionState = conditionState | getMetConditionDimension( allConditions, dimensionFields, dimensionsKeySet); } else { @@ -368,12 +355,10 @@ void SimpleConditionTracker::isConditionMet( ConditionState sliceState = startedCountIt->second > 0 ? ConditionState::kTrue : ConditionState::kFalse; conditionState = conditionState | sliceState; - if (sliceState == ConditionState::kTrue && dimensionFields.has_field()) { - HashableDimensionKey dimensionKey; - if (getSubDimension(startedCountIt->first.getDimensionsValue(), dimensionFields, - dimensionKey.getMutableDimensionsValue())) { - dimensionsKeySet.insert(dimensionKey); - } + if (sliceState == ConditionState::kTrue && dimensionFields.size() > 0) { + vector dimensionKeys; + filterValues(dimensionFields, startedCountIt->first.getValues(), &dimensionKeys); + dimensionsKeySet.insert(dimensionKeys.begin(), dimensionKeys.end()); } } else { // For unseen key, check whether the require dimensions are subset of sliced condition @@ -382,31 +367,29 @@ void SimpleConditionTracker::isConditionMet( for (const auto& slice : mSlicedConditionState) { ConditionState sliceState = slice.second > 0 ? ConditionState::kTrue : ConditionState::kFalse; - if (IsSubDimension(slice.first.getDimensionsValue(), key.getDimensionsValue())) { + if (slice.first.contains(key)) { conditionState = conditionState | sliceState; - if (sliceState == ConditionState::kTrue && dimensionFields.has_field()) { - HashableDimensionKey dimensionKey; - if (getSubDimension(slice.first.getDimensionsValue(), - dimensionFields, dimensionKey.getMutableDimensionsValue())) { - dimensionsKeySet.insert(dimensionKey); - } + if (sliceState == ConditionState::kTrue && dimensionFields.size() > 0) { + vector dimensionKeys; + filterValues(dimensionFields, slice.first.getValues(), &dimensionKeys); + + dimensionsKeySet.insert(dimensionKeys.begin(), dimensionKeys.end()); + } } } } } - } conditionCache[mIndex] = conditionState; VLOG("Predicate %lld return %d", (long long)mConditionId, conditionCache[mIndex]); } ConditionState SimpleConditionTracker::getMetConditionDimension( const std::vector>& allConditions, - const FieldMatcher& dimensionFields, - std::unordered_set &dimensionsKeySet) const { + const vector& dimensionFields, + std::unordered_set& dimensionsKeySet) const { ConditionState conditionState = mInitialValue; - if (!dimensionFields.has_field() || - !mOutputDimensions.has_field() || - dimensionFields.field() != mOutputDimensions.field()) { + if (dimensionFields.size() == 0 || mOutputDimensions.size() == 0 || + dimensionFields[0].mMatcher.getTag() != mOutputDimensions[0].mMatcher.getTag()) { const auto& itr = mSlicedConditionState.find(DEFAULT_DIMENSION_KEY); if (itr != mSlicedConditionState.end()) { ConditionState sliceState = @@ -419,13 +402,13 @@ ConditionState SimpleConditionTracker::getMetConditionDimension( for (const auto& slice : mSlicedConditionState) { ConditionState sliceState = slice.second > 0 ? ConditionState::kTrue : ConditionState::kFalse; - DimensionsValue dimensionsValue; conditionState = conditionState | sliceState; - HashableDimensionKey dimensionKey; - if (sliceState == ConditionState::kTrue && - getSubDimension(slice.first.getDimensionsValue(), dimensionFields, - dimensionKey.getMutableDimensionsValue())) { - dimensionsKeySet.insert(dimensionKey); + + if (sliceState == ConditionState::kTrue && dimensionFields.size() > 0) { + vector dimensionKeys; + filterValues(dimensionFields, slice.first.getValues(), &dimensionKeys); + + dimensionsKeySet.insert(dimensionKeys.begin(), dimensionKeys.end()); } } return conditionState; diff --git a/cmds/statsd/src/condition/SimpleConditionTracker.h b/cmds/statsd/src/condition/SimpleConditionTracker.h index ce9a02d4a7953..c565129352973 100644 --- a/cmds/statsd/src/condition/SimpleConditionTracker.h +++ b/cmds/statsd/src/condition/SimpleConditionTracker.h @@ -48,14 +48,14 @@ public: void isConditionMet(const ConditionKey& conditionParameters, const std::vector>& allConditions, - const FieldMatcher& dimensionFields, + const vector& dimensionFields, std::vector& conditionCache, - std::unordered_set &dimensionsKeySet) const override; + std::unordered_set& dimensionsKeySet) const override; ConditionState getMetConditionDimension( const std::vector>& allConditions, - const FieldMatcher& dimensionFields, - std::unordered_set &dimensionsKeySet) const override; + const vector& dimensionFields, + std::unordered_set& dimensionsKeySet) const override; private: const ConfigKey mConfigKey; @@ -73,17 +73,17 @@ private: ConditionState mInitialValue; - FieldMatcher mOutputDimensions; + std::vector mOutputDimensions; + + int mDimensionTag; std::map mSlicedConditionState; void handleStopAll(std::vector& conditionCache, std::vector& changedCache); - void handleConditionEvent(const HashableDimensionKey& outputKey, - bool matchStart, - std::vector& conditionCache, - std::vector& changedCache); + void handleConditionEvent(const HashableDimensionKey& outputKey, bool matchStart, + ConditionState* conditionCache, bool* changedCache); bool hitGuardRail(const HashableDimensionKey& newKey); diff --git a/cmds/statsd/src/condition/condition_util.cpp b/cmds/statsd/src/condition/condition_util.cpp index 0ab33cfbaea11..691356b5edc62 100644 --- a/cmds/statsd/src/condition/condition_util.cpp +++ b/cmds/statsd/src/condition/condition_util.cpp @@ -27,7 +27,6 @@ #include "ConditionTracker.h" #include "frameworks/base/cmds/statsd/src/statsd_config.pb.h" #include "stats_util.h" -#include "dimension.h" namespace android { namespace os { @@ -97,109 +96,6 @@ ConditionState evaluateCombinationCondition(const std::vector& children, ConditionState operator|(ConditionState l, ConditionState r) { return l >= r ? l : r; } - -void OrConditionState(const std::vector& ref, vector * ored) { - if (ref.size() != ored->size()) { - return; - } - for (size_t i = 0; i < ored->size(); ++i) { - ored->at(i) = ored->at(i) | ref.at(i); - } -} - -void OrBooleanVector(const std::vector& ref, vector * ored) { - if (ref.size() != ored->size()) { - return; - } - for (size_t i = 0; i < ored->size(); ++i) { - ored->at(i) = ored->at(i) | ref.at(i); - } -} - -void getFieldsFromFieldMatcher(const FieldMatcher& matcher, Field* rootField, Field* leafField, - std::vector *allFields) { - if (matcher.has_position()) { - leafField->set_position_index(0); - } - if (matcher.child_size() == 0) { - allFields->push_back(*rootField); - return; - } - for (int i = 0; i < matcher.child_size(); ++i) { - Field* newLeafField = leafField->add_child(); - newLeafField->set_field(matcher.child(i).field()); - getFieldsFromFieldMatcher(matcher.child(i), rootField, newLeafField, allFields); - } -} - -void getFieldsFromFieldMatcher(const FieldMatcher& matcher, std::vector *allFields) { - if (!matcher.has_field()) { - return; - } - Field rootField; - rootField.set_field(matcher.field()); - getFieldsFromFieldMatcher(matcher, &rootField, &rootField, allFields); -} - -void flattenValueLeaves(const DimensionsValue& value, - std::vector *allLaves) { - switch (value.value_case()) { - case DimensionsValue::ValueCase::kValueStr: - case DimensionsValue::ValueCase::kValueInt: - case DimensionsValue::ValueCase::kValueLong: - case DimensionsValue::ValueCase::kValueBool: - case DimensionsValue::ValueCase::kValueFloat: - case DimensionsValue::ValueCase::VALUE_NOT_SET: - allLaves->push_back(&value); - break; - case DimensionsValue::ValueCase::kValueTuple: - for (int i = 0; i < value.value_tuple().dimensions_value_size(); ++i) { - flattenValueLeaves(value.value_tuple().dimensions_value(i), allLaves); - } - break; - } -} - -void getDimensionKeysForCondition( - const LogEvent& event, const MetricConditionLink& link, - std::vector *hashableDimensionKeys) { - std::vector whatFields; - getFieldsFromFieldMatcher(link.fields_in_what(), &whatFields); - std::vector conditionFields; - getFieldsFromFieldMatcher(link.fields_in_condition(), &conditionFields); - - // TODO(yanglu): here we could simplify the logic to get the leaf value node in what and - // directly construct the full condition value tree. - std::vector whatValues; - getDimensionKeys(event, link.fields_in_what(), &whatValues); - - for (size_t i = 0; i < whatValues.size(); ++i) { - std::vector whatLeaves; - flattenValueLeaves(whatValues[i], &whatLeaves); - if (whatLeaves.size() != whatFields.size() || - whatLeaves.size() != conditionFields.size()) { - ALOGE("Dimensions between what and condition not equal."); - return; - } - FieldValueMap conditionValueMap; - for (size_t j = 0; j < whatLeaves.size(); ++j) { - DimensionsValue* conditionValue = &conditionValueMap[conditionFields[j]]; - *conditionValue = *whatLeaves[i]; - if (!setFieldInLeafValueProto(conditionFields[j], conditionValue)) { - ALOGE("Not able to reset the field for condition leaf value."); - return; - } - } - std::vector conditionValueTrees; - findDimensionsValues(conditionValueMap, link.fields_in_condition(), &conditionValueTrees); - if (conditionValueTrees.size() != 1) { - ALOGE("Not able to find unambiguous field value in condition atom."); - continue; - } - hashableDimensionKeys->push_back(HashableDimensionKey(conditionValueTrees[0])); - } -} - } // namespace statsd } // namespace os } // namespace android diff --git a/cmds/statsd/src/condition/condition_util.h b/cmds/statsd/src/condition/condition_util.h index a7288beb69ca4..fed90ec3da37f 100644 --- a/cmds/statsd/src/condition/condition_util.h +++ b/cmds/statsd/src/condition/condition_util.h @@ -33,16 +33,10 @@ enum ConditionState { }; ConditionState operator|(ConditionState l, ConditionState r); -void OrConditionState(const std::vector& ref, vector * ored); -void OrBooleanVector(const std::vector& ref, vector * ored); ConditionState evaluateCombinationCondition(const std::vector& children, const LogicalOperation& operation, const std::vector& conditionCache); - -void getDimensionKeysForCondition( - const LogEvent& event, const MetricConditionLink& link, - std::vector *dimensionKeys); } // namespace statsd } // namespace os } // namespace android diff --git a/cmds/statsd/src/dimension.cpp b/cmds/statsd/src/dimension.cpp deleted file mode 100644 index 8a2e87128319c..0000000000000 --- a/cmds/statsd/src/dimension.cpp +++ /dev/null @@ -1,400 +0,0 @@ -/* - * Copyright (C) 2017 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "Log.h" - -#include "frameworks/base/cmds/statsd/src/statsd_config.pb.h" -#include "frameworks/base/cmds/statsd/src/statsd_internal.pb.h" -#include "dimension.h" - - -namespace android { -namespace os { -namespace statsd { - -const DimensionsValue* getSingleLeafValue(const DimensionsValue* value) { - if (value->value_case() == DimensionsValue::ValueCase::kValueTuple) { - return getSingleLeafValue(&value->value_tuple().dimensions_value(0)); - } else { - return value; - } -} - -DimensionsValue getSingleLeafValue(const DimensionsValue& value) { - const DimensionsValue* leafValue = getSingleLeafValue(&value); - return *leafValue; -} - -void appendLeafNodeToTree(const Field& field, - const DimensionsValue& value, - DimensionsValue* parentValue) { - if (field.child_size() <= 0) { - *parentValue = value; - parentValue->set_field(field.field()); - return; - } - parentValue->set_field(field.field()); - int idx = -1; - for (int i = 0; i < parentValue->mutable_value_tuple()->dimensions_value_size(); ++i) { - if (parentValue->mutable_value_tuple()->dimensions_value(i).field() == - field.child(0).field()) { - idx = i; - } - } - if (idx < 0) { - parentValue->mutable_value_tuple()->add_dimensions_value(); - idx = parentValue->mutable_value_tuple()->dimensions_value_size() - 1; - } - appendLeafNodeToTree( - field.child(0), value, - parentValue->mutable_value_tuple()->mutable_dimensions_value(idx)); -} - -void appendLeafNodeToTrees(const Field& field, - const DimensionsValue& node, - std::vector* rootTrees) { - if (rootTrees == nullptr) { - return; - } - if (rootTrees->empty()) { - DimensionsValue tree; - appendLeafNodeToTree(field, node, &tree); - rootTrees->push_back(tree); - } else { - for (size_t i = 0; i < rootTrees->size(); ++i) { - appendLeafNodeToTree(field, node, &rootTrees->at(i)); - } - } -} - -namespace { - -void findDimensionsValues( - const FieldValueMap& fieldValueMap, - const FieldMatcher& matcher, - Field* rootField, - Field* leafField, - std::vector* rootDimensionsValues); - -void findNonRepeatedDimensionsValues( - const FieldValueMap& fieldValueMap, - const FieldMatcher& matcher, - Field* rootField, - Field* leafField, - std::vector* rootValues) { - if (matcher.child_size() > 0) { - Field* newLeafField = leafField->add_child(); - for (const auto& childMatcher : matcher.child()) { - newLeafField->set_field(childMatcher.field()); - findDimensionsValues(fieldValueMap, childMatcher, rootField, newLeafField, rootValues); - } - leafField->clear_child(); - } else { - auto ret = fieldValueMap.equal_range(*rootField); - int found = 0; - for (auto it = ret.first; it != ret.second; ++it) { - found++; - } - // Not found. - if (found <= 0) { - return; - } - if (found > 1) { - ALOGE("Found multiple values for optional field."); - return; - } - appendLeafNodeToTrees(*rootField, ret.first->second, rootValues); - } -} - -void findRepeatedDimensionsValues(const FieldValueMap& fieldValueMap, - const FieldMatcher& matcher, - Field* rootField, - Field* leafField, - std::vector* rootValues) { - if (matcher.position() == Position::FIRST) { - leafField->set_position_index(0); - findNonRepeatedDimensionsValues(fieldValueMap, matcher, rootField, leafField, rootValues); - leafField->clear_position_index(); - } else { - auto itLower = fieldValueMap.lower_bound(*rootField); - if (itLower == fieldValueMap.end()) { - return; - } - const int leafFieldNum = leafField->field(); - leafField->set_field(leafFieldNum + 1); - auto itUpper = fieldValueMap.lower_bound(*rootField); - // Resets the field number. - leafField->set_field(leafFieldNum); - - switch (matcher.position()) { - case Position::LAST: - { - itUpper--; - if (itUpper != fieldValueMap.end()) { - int last_index = getPositionByReferenceField(*rootField, itUpper->first); - if (last_index < 0) { - return; - } - leafField->set_position_index(last_index); - findNonRepeatedDimensionsValues( - fieldValueMap, matcher, rootField, leafField, rootValues); - leafField->clear_position_index(); - } - } - break; - case Position::ANY: - { - std::set indexes; - for (auto it = itLower; it != itUpper; ++it) { - int index = getPositionByReferenceField(*rootField, it->first); - if (index >= 0) { - indexes.insert(index); - } - } - if (!indexes.empty()) { - std::vector allValues; - for (const int index : indexes) { - leafField->set_position_index(index); - std::vector newValues = *rootValues; - findNonRepeatedDimensionsValues( - fieldValueMap, matcher, rootField, leafField, &newValues); - allValues.insert(allValues.end(), newValues.begin(), newValues.end()); - leafField->clear_position_index(); - } - rootValues->clear(); - rootValues->insert(rootValues->end(), allValues.begin(), allValues.end()); - } - } - break; - default: - break; - } - } -} - -void findDimensionsValues( - const FieldValueMap& fieldValueMap, - const FieldMatcher& matcher, - Field* rootField, - Field* leafField, - std::vector* rootDimensionsValues) { - if (!matcher.has_position()) { - findNonRepeatedDimensionsValues(fieldValueMap, matcher, rootField, leafField, - rootDimensionsValues); - } else { - findRepeatedDimensionsValues(fieldValueMap, matcher, rootField, leafField, - rootDimensionsValues); - } -} - -} // namespace - -void findDimensionsValues( - const FieldValueMap& fieldValueMap, - const FieldMatcher& matcher, - std::vector* rootDimensionsValues) { - Field rootField; - buildSimpleAtomField(matcher.field(), &rootField); - findDimensionsValues(fieldValueMap, matcher, &rootField, &rootField, rootDimensionsValues); -} - -void buildSimpleAtomFieldMatcher(const int tagId, FieldMatcher* matcher) { - matcher->set_field(tagId); -} - -void buildSimpleAtomFieldMatcher(const int tagId, const int fieldNum, FieldMatcher* matcher) { - matcher->set_field(tagId); - matcher->add_child()->set_field(fieldNum); -} - -constexpr int ATTRIBUTION_FIELD_NUM_IN_ATOM_PROTO = 1; -constexpr int UID_FIELD_NUM_IN_ATTRIBUTION_NODE_PROTO = 1; -constexpr int TAG_FIELD_NUM_IN_ATTRIBUTION_NODE_PROTO = 2; - -void buildAttributionUidFieldMatcher(const int tagId, const Position position, - FieldMatcher* matcher) { - matcher->set_field(tagId); - auto child = matcher->add_child(); - child->set_field(ATTRIBUTION_FIELD_NUM_IN_ATOM_PROTO); - child->set_position(position); - child->add_child()->set_field(UID_FIELD_NUM_IN_ATTRIBUTION_NODE_PROTO); -} - -void buildAttributionTagFieldMatcher(const int tagId, const Position position, - FieldMatcher* matcher) { - matcher->set_field(tagId); - FieldMatcher* child = matcher->add_child(); - child->set_field(ATTRIBUTION_FIELD_NUM_IN_ATOM_PROTO); - child->set_position(position); - child->add_child()->set_field(TAG_FIELD_NUM_IN_ATTRIBUTION_NODE_PROTO); -} - -void buildAttributionFieldMatcher(const int tagId, const Position position, FieldMatcher* matcher) { - matcher->set_field(tagId); - FieldMatcher* child = matcher->add_child(); - child->set_field(ATTRIBUTION_FIELD_NUM_IN_ATOM_PROTO); - child->set_position(position); - child->add_child()->set_field(UID_FIELD_NUM_IN_ATTRIBUTION_NODE_PROTO); - child->add_child()->set_field(TAG_FIELD_NUM_IN_ATTRIBUTION_NODE_PROTO); -} - -void DimensionsValueToString(const DimensionsValue& value, std::string *flattened) { - if (!value.has_field()) { - return; - } - *flattened += std::to_string(value.field()); - *flattened += ":"; - switch (value.value_case()) { - case DimensionsValue::ValueCase::kValueStr: - *flattened += value.value_str(); - break; - case DimensionsValue::ValueCase::kValueInt: - *flattened += std::to_string(value.value_int()); - break; - case DimensionsValue::ValueCase::kValueLong: - *flattened += std::to_string(value.value_long()); - break; - case DimensionsValue::ValueCase::kValueBool: - *flattened += std::to_string(value.value_bool()); - break; - case DimensionsValue::ValueCase::kValueFloat: - *flattened += std::to_string(value.value_float()); - break; - case DimensionsValue::ValueCase::kValueTuple: - { - *flattened += "{"; - for (int i = 0; i < value.value_tuple().dimensions_value_size(); ++i) { - DimensionsValueToString(value.value_tuple().dimensions_value(i), flattened); - *flattened += "|"; - } - *flattened += "}"; - } - break; - case DimensionsValue::ValueCase::VALUE_NOT_SET: - break; - } -} - -std::string DimensionsValueToString(const DimensionsValue& value) { - std::string flatten; - DimensionsValueToString(value, &flatten); - return flatten; -} - -bool IsSubDimension(const DimensionsValue& dimension, const DimensionsValue& sub) { - if (dimension.field() != sub.field()) { - return false; - } - if (dimension.value_case() != sub.value_case()) { - return false; - } - switch (dimension.value_case()) { - case DimensionsValue::ValueCase::kValueStr: - return dimension.value_str() == sub.value_str(); - case DimensionsValue::ValueCase::kValueInt: - return dimension.value_int() == sub.value_int(); - case DimensionsValue::ValueCase::kValueLong: - return dimension.value_long() == sub.value_long(); - case DimensionsValue::ValueCase::kValueBool: - return dimension.value_bool() == sub.value_bool(); - case DimensionsValue::ValueCase::kValueFloat: - return dimension.value_float() == sub.value_float(); - case DimensionsValue::ValueCase::kValueTuple: { - if (dimension.value_tuple().dimensions_value_size() < - sub.value_tuple().dimensions_value_size()) { - return false; - } - bool allSub = true; - for (int i = 0; allSub && i < sub.value_tuple().dimensions_value_size(); ++i) { - bool isSub = false; - for (int j = 0; !isSub && - j < dimension.value_tuple().dimensions_value_size(); ++j) { - isSub |= IsSubDimension(dimension.value_tuple().dimensions_value(j), - sub.value_tuple().dimensions_value(i)); - } - allSub &= isSub; - } - return allSub; - } - break; - case DimensionsValue::ValueCase::VALUE_NOT_SET: - return false; - default: - return false; - } -} - -long getLongFromDimenValue(const DimensionsValue& dimensionValue) { - switch (dimensionValue.value_case()) { - case DimensionsValue::ValueCase::kValueInt: - return dimensionValue.value_int(); - case DimensionsValue::ValueCase::kValueLong: - return dimensionValue.value_long(); - case DimensionsValue::ValueCase::kValueBool: - return dimensionValue.value_bool() ? 1 : 0; - case DimensionsValue::ValueCase::kValueFloat: - return (int64_t)dimensionValue.value_float(); - case DimensionsValue::ValueCase::kValueTuple: - case DimensionsValue::ValueCase::kValueStr: - case DimensionsValue::ValueCase::VALUE_NOT_SET: - return 0; - } -} - -bool getSubDimension(const DimensionsValue& dimension, const FieldMatcher& matcher, - DimensionsValue* subDimension) { - if (!matcher.has_field()) { - return false; - } - if (matcher.field() != dimension.field()) { - return false; - } - if (matcher.child_size() <= 0) { - if (dimension.value_case() == DimensionsValue::ValueCase::kValueTuple || - dimension.value_case() == DimensionsValue::ValueCase::VALUE_NOT_SET) { - return false; - } - *subDimension = dimension; - return true; - } else { - if (dimension.value_case() != DimensionsValue::ValueCase::kValueTuple) { - return false; - } - bool found_value = true; - auto value_tuple = dimension.value_tuple(); - subDimension->set_field(dimension.field()); - for (int i = 0; found_value && i < matcher.child_size(); ++i) { - int j = 0; - for (; j < value_tuple.dimensions_value_size(); ++j) { - if (value_tuple.dimensions_value(j).field() == matcher.child(i).field()) { - break; - } - } - if (j < value_tuple.dimensions_value_size()) { - found_value &= getSubDimension(value_tuple.dimensions_value(j), matcher.child(i), - subDimension->mutable_value_tuple()->add_dimensions_value()); - } else { - found_value = false; - } - } - return found_value; - } -} - -} // namespace statsd -} // namespace os -} // namespace android diff --git a/cmds/statsd/src/dimension.h b/cmds/statsd/src/dimension.h deleted file mode 100644 index 138c6e9b0160c..0000000000000 --- a/cmds/statsd/src/dimension.h +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (C) 2017 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include -#include -#include -#include "frameworks/base/cmds/statsd/src/stats_log.pb.h" -#include "frameworks/base/cmds/statsd/src/statsd_config.pb.h" -#include "field_util.h" - -namespace android { -namespace os { -namespace statsd { - -// Returns the leaf node from the DimensionsValue proto. It assume that the input has only one -// leaf node at most. -const DimensionsValue* getSingleLeafValue(const DimensionsValue* value); -DimensionsValue getSingleLeafValue(const DimensionsValue& value); - -// Appends the leaf node to the parent tree. -void appendLeafNodeToTree(const Field& field, const DimensionsValue& value, DimensionsValue* tree); - -// Constructs the DimensionsValue protos from the FieldMatcher. Each DimensionsValue proto -// represents a tree. When the input proto has repeated fields and the input "dimensions" wants -// "ANY" locations, it will return multiple trees. -void findDimensionsValues( - const FieldValueMap& fieldValueMap, - const FieldMatcher& matcher, - std::vector* rootDimensionsValues); - -// Utils to build FieldMatcher proto for simple one-depth atoms. -void buildSimpleAtomFieldMatcher(const int tagId, const int atomFieldNum, FieldMatcher* matcher); -void buildSimpleAtomFieldMatcher(const int tagId, FieldMatcher* matcher); - -// Utils to build FieldMatcher proto for attribution nodes. -void buildAttributionUidFieldMatcher(const int tagId, const Position position, - FieldMatcher* matcher); -void buildAttributionTagFieldMatcher(const int tagId, const Position position, - FieldMatcher* matcher); -void buildAttributionFieldMatcher(const int tagId, const Position position, - FieldMatcher* matcher); - -// Utils to print pretty string for DimensionsValue proto. -std::string DimensionsValueToString(const DimensionsValue& value); -void DimensionsValueToString(const DimensionsValue& value, std::string *flattened); - -bool IsSubDimension(const DimensionsValue& dimension, const DimensionsValue& sub); - -// Helper function to get long value from the DimensionsValue proto. -long getLongFromDimenValue(const DimensionsValue& dimensionValue); - -bool getSubDimension(const DimensionsValue& dimension, const FieldMatcher& matcher, - DimensionsValue* subDimension); -} // namespace statsd -} // namespace os -} // namespace android diff --git a/cmds/statsd/src/external/puller_util.cpp b/cmds/statsd/src/external/puller_util.cpp index 7cfc1d487dfbd..0b0c5c41bf9dc 100644 --- a/cmds/statsd/src/external/puller_util.cpp +++ b/cmds/statsd/src/external/puller_util.cpp @@ -18,7 +18,6 @@ #include "Log.h" #include "StatsPullerManagerImpl.h" -#include "field_util.h" #include "puller_util.h" #include "statslog.h" @@ -30,125 +29,136 @@ using std::map; using std::shared_ptr; using std::vector; -DimensionsValue* getFieldValue(shared_ptr event, int tagId, int fieldNum) { - Field field; - buildSimpleAtomField(tagId, fieldNum, &field); - return event->findFieldValueOrNull(field); -} - +namespace { bool shouldMerge(shared_ptr& lhs, shared_ptr& rhs, - const vector& nonAdditiveFields, int tagId) { - for (int f : nonAdditiveFields) { - DimensionsValue* lValue = getFieldValue(lhs, tagId, f); - DimensionsValue* rValue = getFieldValue(rhs, tagId, f); - if (!compareDimensionsValue(*lValue, *rValue)) { - return false; + const vector& nonAdditiveFields) { + const auto& l_values = lhs->getValues(); + const auto& r_values = rhs->getValues(); + + for (size_t i : nonAdditiveFields) { + // We store everything starting from index 0, so we need to use i-1 + if (!(l_values.size() > i - 1 && r_values.size() > i - 1 && + l_values[i - 1].mValue == r_values[i - 1].mValue)) { + return false; + } } - } - return true; + return true; } // merge rhs to lhs -void mergeEvent(shared_ptr& lhs, shared_ptr& rhs, - const vector& additiveFields, int tagId) { - for (int f : additiveFields) { - DimensionsValue* lValue = getFieldValue(lhs, tagId, f); - DimensionsValue* rValue = getFieldValue(rhs, tagId, f); - if (lValue->has_value_int()) { - lValue->set_value_int(lValue->value_int() + rValue->value_int()); - } else if (lValue->has_value_long()) { - lValue->set_value_long(lValue->value_long() + rValue->value_long()); +// when calling this function, all sanity check should be done already. +// e.g., index boundary, nonAdditiveFields matching etc. +bool mergeEvent(shared_ptr& lhs, shared_ptr& rhs, + const vector& additiveFields) { + vector* host_values = lhs->getMutableValues(); + const auto& child_values = rhs->getValues(); + for (int i : additiveFields) { + Value& host = (*host_values)[i - 1].mValue; + const Value& child = (child_values[i - 1]).mValue; + if (child.getType() != host.getType()) { + return false; + } + switch (child.getType()) { + case INT: + host.setInt(host.int_value + child.int_value); + break; + case LONG: + host.setLong(host.long_value + child.long_value); + break; + default: + ALOGE("Tried to merge 2 fields with unsupported type"); + return false; + } } - } + return true; } -// process all data and merge isolated with host if necessary -void mergeIsolatedUidsToHostUid(vector>& data, - const sp& uidMap, int tagId) { - if (StatsPullerManagerImpl::kAllPullAtomInfo.find(tagId) == - StatsPullerManagerImpl::kAllPullAtomInfo.end()) { - VLOG("Unknown pull atom id %d", tagId); - return; - } - if (android::util::kAtomsWithUidField.find(tagId) == - android::util::kAtomsWithUidField.end()) { - VLOG("No uid to merge for atom %d", tagId); - return; - } - const vector& additiveFields = - StatsPullerManagerImpl::kAllPullAtomInfo.find(tagId) - ->second.additiveFields; - const vector& nonAdditiveFields = - StatsPullerManagerImpl::kAllPullAtomInfo.find(tagId) - ->second.nonAdditiveFields; - - // map of host uid to isolated uid data index in the original vector. - // because of non additive fields, there could be multiple of them that can't - // be merged into one - map> hostToIsolated; - // map of host uid to their position in the original vector - map> hostPosition; - vector isolatedUidPos; - // all uids in the original vector - vector allUids; - for (size_t i = 0; i < data.size(); i++) { - // uid field is always first primitive filed, if present - DimensionsValue* uidField = getFieldValue(data[i], tagId, 1); - if (!uidField) { - VLOG("Bad data for %d, %s", tagId, data[i]->ToString().c_str()); - return; - } - int uid = uidField->value_int(); - allUids.push_back(uid); - const int hostUid = uidMap->getHostUidOrSelf(uid); - if (hostUid != uid) { - uidField->set_value_int(hostUid); - hostToIsolated[hostUid].push_back(i); - isolatedUidPos.push_back(i); - } - } - vector> mergedData; - for (size_t i = 0; i < allUids.size(); i++) { - if (hostToIsolated.find(allUids[i]) != hostToIsolated.end()) { - hostPosition[allUids[i]].push_back(i); - } else if (std::find(isolatedUidPos.begin(), isolatedUidPos.end(), i) != isolatedUidPos.end()) { - continue; - } else { - mergedData.push_back(data[i]); - } - } - for (auto iter = hostToIsolated.begin(); iter != hostToIsolated.end(); - iter++) { - int uid = iter->first; - vector& isolated = hostToIsolated[uid]; - vector toBeMerged; - toBeMerged.insert(toBeMerged.begin(), isolated.begin(), isolated.end()); - if (hostPosition.find(uid) != hostPosition.end()) { - vector& host = hostPosition[uid]; - toBeMerged.insert(toBeMerged.end(), host.begin(), host.end()); - } - vector used(toBeMerged.size()); - for (size_t i = 0; i < toBeMerged.size(); i++) { - if (used[i] == true) { - continue; - } - for (size_t j = i + 1; j < toBeMerged.size(); j++) { - shared_ptr& lhs = data[toBeMerged[i]]; - shared_ptr& rhs = data[toBeMerged[j]]; - if (shouldMerge(lhs, rhs, nonAdditiveFields, tagId)) { - mergeEvent(lhs, rhs, additiveFields, tagId); - used[j] = true; +bool tryMerge(vector>& data, int child_pos, const vector& host_pos, + const vector& nonAdditiveFields, const vector& additiveFields) { + for (const auto& pos : host_pos) { + if (shouldMerge(data[pos], data[child_pos], nonAdditiveFields) && + mergeEvent(data[pos], data[child_pos], additiveFields)) { + return true; } - } } - for (size_t i = 0; i < toBeMerged.size(); i++) { - if (used[i] == false) { - mergedData.push_back(data[i]); + return false; +} + +} // namespace + +/** + * Process all data and merge isolated with host if necessary. + * For example: + * NetworkBytesAtom { + * int uid = 1; + * State process_state = 2; + * int byte_send = 3; + * int byte_recv = 4; + * } + * additive fields are {3, 4}, non-additive field is {2} + * If we pulled the following events (uid1_child is an isolated uid which maps to uid1): + * [uid1, fg, 100, 200] + * [uid1_child, fg, 100, 200] + * [uid1, bg, 100, 200] + * + * We want to merge them and results should be: + * [uid1, fg, 200, 400] + * [uid1, bg, 100, 200] + */ +void mergeIsolatedUidsToHostUid(vector>& data, const sp& uidMap, + int tagId) { + if (StatsPullerManagerImpl::kAllPullAtomInfo.find(tagId) == + StatsPullerManagerImpl::kAllPullAtomInfo.end()) { + VLOG("Unknown pull atom id %d", tagId); + return; } + if (android::util::kAtomsWithUidField.find(tagId) == android::util::kAtomsWithUidField.end()) { + VLOG("No uid to merge for atom %d", tagId); + return; } - } - data.clear(); - data = mergedData; + const vector& additiveFields = + StatsPullerManagerImpl::kAllPullAtomInfo.find(tagId)->second.additiveFields; + const vector& nonAdditiveFields = + StatsPullerManagerImpl::kAllPullAtomInfo.find(tagId)->second.nonAdditiveFields; + + // map of host uid to their position in the original vector + map> hostPosition; + vector toRemove = vector(data.size(), false); + + for (size_t i = 0; i < data.size(); i++) { + vector* valueList = data[i]->getMutableValues(); + + int err = 0; + int uid = data[i]->GetInt(1, &err); + if (err != 0) { + VLOG("Bad uid field for %s", data[i]->ToString().c_str()); + return; + } + + const int hostUid = uidMap->getHostUidOrSelf(uid); + + if (hostUid != uid) { + (*valueList)[0].mValue.setInt(hostUid); + } + if (hostPosition.find(hostUid) == hostPosition.end()) { + hostPosition[hostUid].push_back(i); + } else { + if (tryMerge(data, i, hostPosition[hostUid], nonAdditiveFields, additiveFields)) { + toRemove[i] = true; + } else { + hostPosition[hostUid].push_back(i); + } + } + } + + vector> mergedData; + for (size_t i = 0; i < toRemove.size(); i++) { + if (!toRemove[i]) { + mergedData.push_back(data[i]); + } + } + data.clear(); + data = mergedData; } } // namespace statsd diff --git a/cmds/statsd/src/external/puller_util.h b/cmds/statsd/src/external/puller_util.h index 70d5321ce29a3..fd4a4a2a57555 100644 --- a/cmds/statsd/src/external/puller_util.h +++ b/cmds/statsd/src/external/puller_util.h @@ -17,7 +17,6 @@ #pragma once #include -#include "HashableDimensionKey.h" #include "StatsPuller.h" #include "logd/LogEvent.h" #include "packages/UidMap.h" diff --git a/cmds/statsd/src/field_util.cpp b/cmds/statsd/src/field_util.cpp deleted file mode 100644 index acf64fe12e6db..0000000000000 --- a/cmds/statsd/src/field_util.cpp +++ /dev/null @@ -1,318 +0,0 @@ -/* - * Copyright (C) 2017 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "Log.h" -#include "field_util.h" - -#include -#include - -namespace android { -namespace os { -namespace statsd { - -// This function is to compare two Field trees where each node has at most one child. -bool CompareField(const Field& a, const Field& b) { - if (a.field() < b.field()) { - return true; - } - if (a.field() > b.field()) { - return false; - } - if (a.position_index() < b.position_index()) { - return true; - } - if (a.position_index() > b.position_index()) { - return false; - } - if (a.child_size() < b.child_size()) { - return true; - } - if (a.child_size() > b.child_size()) { - return false; - } - if (a.child_size() == 0 && b.child_size() == 0) { - return false; - } - return CompareField(a.child(0), b.child(0)); -} - -const Field* getSingleLeaf(const Field* field) { - if (field->child_size() <= 0) { - return field; - } else { - return getSingleLeaf(&field->child(0)); - } -} - -Field* getSingleLeaf(Field* field) { - if (field->child_size() <= 0) { - return field; - } else { - return getSingleLeaf(field->mutable_child(0)); - } -} - -void FieldToString(const Field& field, std::string *flattened) { - *flattened += std::to_string(field.field()); - if (field.has_position_index()) { - *flattened += "["; - *flattened += std::to_string(field.position_index()); - *flattened += "]"; - } - if (field.child_size() <= 0) { - return; - } - *flattened += "."; - *flattened += "{"; - for (int i = 0 ; i < field.child_size(); ++i) { - *flattened += FieldToString(field.child(i)); - } - *flattened += "},"; -} - -std::string FieldToString(const Field& field) { - std::string flatten; - FieldToString(field, &flatten); - return flatten; -} - -bool setFieldInLeafValueProto(const Field &field, DimensionsValue* leafValue) { - if (field.child_size() <= 0) { - leafValue->set_field(field.field()); - return true; - } else if (field.child_size() == 1) { - return setFieldInLeafValueProto(field.child(0), leafValue); - } else { - ALOGE("Not able to set the 'field' in leaf value for multiple children."); - return false; - } -} - -void buildSimpleAtomField(const int tagId, const int atomFieldNum, Field *field) { - field->set_field(tagId); - field->add_child()->set_field(atomFieldNum); -} - -void buildSimpleAtomField(const int tagId, Field *field) { - field->set_field(tagId); -} - -void appendLeaf(Field *parent, int node_field_num) { - if (!parent->has_field()) { - parent->set_field(node_field_num); - } else if (parent->child_size() <= 0) { - parent->add_child()->set_field(node_field_num); - } else { - appendLeaf(parent->mutable_child(0), node_field_num); - } -} - -void appendLeaf(Field *parent, int node_field_num, int position) { - if (!parent->has_field()) { - parent->set_field(node_field_num); - parent->set_position_index(position); - } else if (parent->child_size() <= 0) { - auto child = parent->add_child(); - child->set_field(node_field_num); - child->set_position_index(position); - } else { - appendLeaf(parent->mutable_child(0), node_field_num, position); - } -} - -void increasePosition(Field *field) { - if (!field->has_position_index()) { - field->set_position_index(0); - } else { - field->set_position_index(field->position_index() + 1); - } -} - -int getPositionByReferenceField(const Field& ref, const Field& field_with_index) { - if (ref.child_size() <= 0) { - return field_with_index.position_index(); - } - if (ref.child_size() != 1 || - field_with_index.child_size() != 1) { - return -1; - } - return getPositionByReferenceField(ref.child(0), field_with_index.child(0)); -} - -namespace { - -void findFields( - const FieldValueMap& fieldValueMap, - const FieldMatcher& matcher, - Field* rootField, - Field* leafField, - std::set* rootFields); - -void findNonRepeatedFields( - const FieldValueMap& fieldValueMap, - const FieldMatcher& matcher, - Field* rootField, - Field* leafField, - std::set* rootFields) { - if (matcher.child_size() > 0) { - Field* newLeafField = leafField->add_child(); - for (const auto& childMatcher : matcher.child()) { - newLeafField->set_field(childMatcher.field()); - findFields(fieldValueMap, childMatcher, rootField, newLeafField, rootFields); - } - leafField->clear_child(); - } else { - auto ret = fieldValueMap.equal_range(*rootField); - int found = 0; - for (auto it = ret.first; it != ret.second; ++it) { - found++; - } - // Not found. - if (found <= 0) { - return; - } - if (found > 1) { - ALOGE("Found multiple values for optional field."); - return; - } - rootFields->insert(ret.first->first); - } -} - -void findRepeatedFields(const FieldValueMap& fieldValueMap, const FieldMatcher& matcher, - Field* rootField, Field* leafField, - std::set* rootFields) { - if (matcher.position() == Position::FIRST) { - leafField->set_position_index(0); - findNonRepeatedFields(fieldValueMap, matcher, rootField, leafField, rootFields); - leafField->clear_position_index(); - } else { - auto itLower = fieldValueMap.lower_bound(*rootField); - if (itLower == fieldValueMap.end()) { - return; - } - - const int leafFieldNum = leafField->field(); - leafField->set_field(leafFieldNum + 1); - auto itUpper = fieldValueMap.lower_bound(*rootField); - // Resets the field number. - leafField->set_field(leafFieldNum); - - switch (matcher.position()) { - case Position::LAST: - { - itUpper--; - if (itUpper != fieldValueMap.end()) { - int last_index = getPositionByReferenceField(*rootField, itUpper->first); - if (last_index < 0) { - return; - } - leafField->set_position_index(last_index); - findNonRepeatedFields( - fieldValueMap, matcher, rootField, leafField, rootFields); - leafField->clear_position_index(); - } - } - break; - case Position::ANY: - { - std::set indexes; - for (auto it = itLower; it != itUpper; ++it) { - int index = getPositionByReferenceField(*rootField, it->first); - if (index >= 0) { - indexes.insert(index); - } - } - if (!indexes.empty()) { - for (const int index : indexes) { - leafField->set_position_index(index); - findNonRepeatedFields( - fieldValueMap, matcher, rootField, leafField, rootFields); - leafField->clear_position_index(); - } - } - } - break; - default: - break; - } - } -} - -void findFields( - const FieldValueMap& fieldValueMap, - const FieldMatcher& matcher, - Field* rootField, - Field* leafField, - std::set* rootFields) { - if (!matcher.has_position()) { - findNonRepeatedFields(fieldValueMap, matcher, rootField, leafField, rootFields); - } else { - findRepeatedFields(fieldValueMap, matcher, rootField, leafField, rootFields); - } -} - -} // namespace - -void findFields( - const FieldValueMap& fieldValueMap, - const FieldMatcher& matcher, - std::set* rootFields) { - if (!matcher.has_field() || fieldValueMap.empty()) { - return; - } - Field rootField; - buildSimpleAtomField(matcher.field(), &rootField); - return findFields(fieldValueMap, matcher, &rootField, &rootField, rootFields); -} - -void filterFields(const FieldMatcher& matcher, FieldValueMap* fieldValueMap) { - if (!matcher.has_field()) { - return; - } - std::set rootFields; - findFields(*fieldValueMap, matcher, &rootFields); - auto it = fieldValueMap->begin(); - while (it != fieldValueMap->end()) { - if (rootFields.find(it->first) == rootFields.end()) { - it = fieldValueMap->erase(it); - } else { - it++; - } - } -} - -bool hasLeafNode(const FieldMatcher& matcher) { - if (!matcher.has_field()) { - return false; - } - for (int i = 0; i < matcher.child_size(); ++i) { - if (hasLeafNode(matcher.child(i))) { - return true; - } - } - return true; -} - -bool IsAttributionUidField(const Field& field) { - return field.child_size() == 1 && field.child(0).field() == 1 - && field.child(0).child_size() == 1 && field.child(0).child(0).field() == 1; -} - -} // namespace statsd -} // namespace os -} // namespace android diff --git a/cmds/statsd/src/field_util.h b/cmds/statsd/src/field_util.h deleted file mode 100644 index b04465dc98625..0000000000000 --- a/cmds/statsd/src/field_util.h +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright (C) 2017 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "frameworks/base/cmds/statsd/src/statsd_config.pb.h" -#include "frameworks/base/cmds/statsd/src/statsd_internal.pb.h" -#include "frameworks/base/cmds/statsd/src/stats_log.pb.h" - -#include -#include - -namespace android { -namespace os { -namespace statsd { - -// Function to sort the Field protos. -bool CompareField(const Field& a, const Field& b); -struct FieldCmp { - bool operator()(const Field& a, const Field& b) const { - return CompareField(a, b); - } -}; - -// Flattened dimensions value map. To save space, usually the key contains the tree structure info -// and value field is only leaf node. -typedef std::map FieldValueMap; - -// Util function to print the Field proto. -std::string FieldToString(const Field& field); - -// Util function to find the leaf node from the input Field proto and set it in the corresponding -// value proto. -bool setFieldInLeafValueProto(const Field &field, DimensionsValue* leafValue); - -// Returns the leaf node from the Field proto. It assume that the input has only one -// leaf node at most. -const Field* getSingleLeaf(const Field* field); -Field* getSingleLeaf(Field* field); - -// Append a node to the current leaf. It assumes that the input "parent" has one leaf node at most. -void appendLeaf(Field *parent, int node_field_num); -void appendLeaf(Field *parent, int node_field_num, int position); - -// Increase the position index for the node. If the "position_index" is not set, set it as 0. -void increasePosition(Field *field); - -// Returns true if the matcher has specified at least one leaf node. -bool hasLeafNode(const FieldMatcher& matcher); - -// The two input Field proto are describing the same tree structure. Both contain one leaf node at -// most. This is find the position index info for the leaf node at "reference" stored in the -// "field_with_index" tree. -int getPositionByReferenceField(const Field& reference, const Field& field_with_index); - -// Utils to build the Field proto for simple atom fields. -void buildSimpleAtomField(const int tagId, const int atomFieldNum, Field* field); -void buildSimpleAtomField(const int tagId, Field* field); - -// Find out all the fields specified by the matcher. -void findFields( - const FieldValueMap& fieldValueMap, const FieldMatcher& matcher, - std::set* rootFields); - -// Filter out the fields not in the field matcher. -void filterFields(const FieldMatcher& matcher, FieldValueMap* fieldValueMap); - -// Returns if the field is attribution node uid field. -bool IsAttributionUidField(const Field& field); - -} // namespace statsd -} // namespace os -} // namespace android diff --git a/cmds/statsd/src/logd/LogEvent.cpp b/cmds/statsd/src/logd/LogEvent.cpp index e1ab5d529370c..909b74f305083 100644 --- a/cmds/statsd/src/logd/LogEvent.cpp +++ b/cmds/statsd/src/logd/LogEvent.cpp @@ -17,13 +17,8 @@ #define DEBUG false // STOPSHIP if true #include "logd/LogEvent.h" -#include "frameworks/base/cmds/statsd/src/statsd_config.pb.h" - -#include #include -#include "field_util.h" -#include "dimension.h" #include "stats_log_util.h" namespace android { @@ -152,9 +147,6 @@ bool LogEvent::write(const AttributionNode& node) { if (android_log_write_string8(mContext, node.tag().c_str()) < 0) { return false; } - if (android_log_write_int32(mContext, node.uid()) < 0) { - return false; - } if (android_log_write_list_end(mContext) < 0) { return false; } @@ -163,47 +155,23 @@ bool LogEvent::write(const AttributionNode& node) { return false; } -namespace { - -void increaseField(Field *field, bool is_child) { - if (is_child) { - if (field->child_size() <= 0) { - field->add_child(); - } - } else { - field->clear_child(); - } - Field* curr = is_child ? field->mutable_child(0) : field; - if (!curr->has_field()) { - curr->set_field(1); - } else { - curr->set_field(curr->field() + 1); - } -} - -} // namespace - /** * The elements of each log event are stored as a vector of android_log_list_elements. * The goal is to do as little preprocessing as possible, because we read a tiny fraction * of the elements that are written to the log. + * + * The idea here is to read through the log items once, we get as much information we need for + * matching as possible. Because this log will be matched against lots of matchers. */ void LogEvent::init(android_log_context context) { - if (!context) { - return; - } android_log_list_element elem; - // TODO: The log is actually structured inside one list. This is convenient - // because we'll be able to use it to put the attribution (WorkSource) block first - // without doing our own tagging scheme. Until that change is in, just drop the - // list-related log elements and the order we get there is our index-keyed data - // structure. int i = 0; int seenListStart = 0; - Field fieldTree; - Field* atomField = fieldTree.add_child(); + int32_t field = 0; + int depth = -1; + int pos[] = {1, 1, 1}; do { elem = android_log_read_next(context); switch ((int)elem.type) { @@ -211,55 +179,81 @@ void LogEvent::init(android_log_context context) { // elem at [0] is EVENT_TYPE_LIST, [1] is the tag id. if (i == 1) { mTagId = elem.data.int32; - fieldTree.set_field(mTagId); } else { - increaseField(atomField, seenListStart > 0/* is_child */); - mFieldValueMap[fieldTree].set_value_int(elem.data.int32); - } - break; - case EVENT_TYPE_FLOAT: - { - increaseField(atomField, seenListStart > 0/* is_child */); - mFieldValueMap[fieldTree].set_value_float(elem.data.float32); - } - break; - case EVENT_TYPE_STRING: - { - increaseField(atomField, seenListStart > 0/* is_child */); - mFieldValueMap[fieldTree].set_value_str( - string(elem.data.string, elem.len).c_str()); - } - break; - case EVENT_TYPE_LONG: - { - increaseField(atomField, seenListStart > 0 /* is_child */); - mFieldValueMap[fieldTree].set_value_long(elem.data.int64); - } - break; - case EVENT_TYPE_LIST: - if (i >= 1) { - if (seenListStart > 0) { - increasePosition(atomField); - } else { - increaseField(atomField, false /* is_child */); - } - seenListStart++; - if (seenListStart >= 3) { - ALOGE("Depth > 2. Not supported!"); + if (depth < 0 || depth > 2) { return; } + + mValues.push_back( + FieldValue(Field(mTagId, pos, depth), Value((int32_t)elem.data.int32))); + + pos[depth]++; } break; - case EVENT_TYPE_LIST_STOP: - seenListStart--; - if (seenListStart == 0) { - atomField->clear_position_index(); - } else { - if (atomField->child_size() > 0) { - atomField->mutable_child(0)->clear_field(); + case EVENT_TYPE_FLOAT: { + if (depth < 0 || depth > 2) { + ALOGE("Depth > 2. Not supported!"); + return; + } + + mValues.push_back(FieldValue(Field(mTagId, pos, depth), Value(elem.data.float32))); + + pos[depth]++; + + } break; + case EVENT_TYPE_STRING: { + if (depth < 0 || depth > 2) { + ALOGE("Depth > 2. Not supported!"); + return; + } + + mValues.push_back(FieldValue(Field(mTagId, pos, depth), + Value(string(elem.data.string, elem.len)))); + + pos[depth]++; + + } break; + case EVENT_TYPE_LONG: { + if (depth < 0 || depth > 2) { + ALOGE("Depth > 2. Not supported!"); + return; + } + mValues.push_back( + FieldValue(Field(mTagId, pos, depth), Value((int64_t)elem.data.int64))); + + pos[depth]++; + + } break; + case EVENT_TYPE_LIST: + depth++; + if (depth > 2) { + ALOGE("Depth > 2. Not supported!"); + return; + } + pos[depth] = 1; + + break; + case EVENT_TYPE_LIST_STOP: { + int prevDepth = depth; + depth--; + if (depth >= 0 && depth < 2) { + // Now go back to decorate the previous items that are last at prevDepth. + // So that we can later easily match them with Position=Last matchers. + pos[prevDepth]--; + int path = getEncodedField(pos, prevDepth, false); + for (size_t j = mValues.size() - 1; j >= 0; j--) { + if (mValues[j].mField.getDepth() >= prevDepth && + mValues[j].mField.getPath(prevDepth) == path) { + mValues[j].mField.decorateLastPos(prevDepth); + } else { + // Safe to break, because the items are in DFS order. + break; + } } + pos[depth]++; } break; + } case EVENT_TYPE_UNKNOWN: break; default: @@ -270,162 +264,115 @@ void LogEvent::init(android_log_context context) { } int64_t LogEvent::GetLong(size_t key, status_t* err) const { - DimensionsValue value; - if (!GetSimpleAtomDimensionsValueProto(key, &value)) { - *err = BAD_INDEX; - return 0; - } - const DimensionsValue* leafValue = getSingleLeafValue(&value); - switch (leafValue->value_case()) { - case DimensionsValue::ValueCase::kValueInt: - return (int64_t)leafValue->value_int(); - case DimensionsValue::ValueCase::kValueLong: - return leafValue->value_long(); - case DimensionsValue::ValueCase::kValueBool: - return leafValue->value_bool() ? 1 : 0; - case DimensionsValue::ValueCase::kValueFloat: - return (int64_t)leafValue->value_float(); - case DimensionsValue::ValueCase::kValueTuple: - case DimensionsValue::ValueCase::kValueStr: - case DimensionsValue::ValueCase::VALUE_NOT_SET: { - *err = BAD_TYPE; - return 0; + // TODO: encapsulate the magical operations all in Field struct as a static function. + int field = getSimpleField(key); + for (const auto& value : mValues) { + if (value.mField.getField() == field) { + if (value.mValue.getType() == INT) { + return value.mValue.int_value; + } else { + *err = BAD_TYPE; + return 0; + } + } + if ((size_t)value.mField.getPosAtDepth(0) > key) { + break; } } + + *err = BAD_INDEX; + return 0; } int LogEvent::GetInt(size_t key, status_t* err) const { - DimensionsValue value; - if (!GetSimpleAtomDimensionsValueProto(key, &value)) { + int field = getSimpleField(key); + for (const auto& value : mValues) { + if (value.mField.getField() == field) { + if (value.mValue.getType() == INT) { + return value.mValue.int_value; + } else { + *err = BAD_TYPE; + return 0; + } + } + if ((size_t)value.mField.getPosAtDepth(0) > key) { + break; + } + } + *err = BAD_INDEX; return 0; - } - const DimensionsValue* leafValue = getSingleLeafValue(&value); - switch (leafValue->value_case()) { - case DimensionsValue::ValueCase::kValueInt: - return leafValue->value_int(); - case DimensionsValue::ValueCase::kValueLong: - case DimensionsValue::ValueCase::kValueBool: - case DimensionsValue::ValueCase::kValueFloat: - case DimensionsValue::ValueCase::kValueTuple: - case DimensionsValue::ValueCase::kValueStr: - case DimensionsValue::ValueCase::VALUE_NOT_SET: { - *err = BAD_TYPE; - return 0; - } - } } const char* LogEvent::GetString(size_t key, status_t* err) const { - DimensionsValue value; - if (!GetSimpleAtomDimensionsValueProto(key, &value)) { - *err = BAD_INDEX; - return 0; - } - const DimensionsValue* leafValue = getSingleLeafValue(&value); - switch (leafValue->value_case()) { - case DimensionsValue::ValueCase::kValueStr: - return leafValue->value_str().c_str(); - case DimensionsValue::ValueCase::kValueInt: - case DimensionsValue::ValueCase::kValueLong: - case DimensionsValue::ValueCase::kValueBool: - case DimensionsValue::ValueCase::kValueFloat: - case DimensionsValue::ValueCase::kValueTuple: - case DimensionsValue::ValueCase::VALUE_NOT_SET: { - *err = BAD_TYPE; - return 0; + int field = getSimpleField(key); + for (const auto& value : mValues) { + if (value.mField.getField() == field) { + if (value.mValue.getType() == STRING) { + return value.mValue.str_value.c_str(); + } else { + *err = BAD_TYPE; + return 0; + } + } + if ((size_t)value.mField.getPosAtDepth(0) > key) { + break; } } + + *err = BAD_INDEX; + return NULL; } bool LogEvent::GetBool(size_t key, status_t* err) const { - DimensionsValue value; - if (!GetSimpleAtomDimensionsValueProto(key, &value)) { - *err = BAD_INDEX; - return 0; - } - const DimensionsValue* leafValue = getSingleLeafValue(&value); - switch (leafValue->value_case()) { - case DimensionsValue::ValueCase::kValueInt: - return leafValue->value_int() != 0; - case DimensionsValue::ValueCase::kValueLong: - return leafValue->value_long() != 0; - case DimensionsValue::ValueCase::kValueBool: - return leafValue->value_bool(); - case DimensionsValue::ValueCase::kValueFloat: - return leafValue->value_float() != 0; - case DimensionsValue::ValueCase::kValueTuple: - case DimensionsValue::ValueCase::kValueStr: - case DimensionsValue::ValueCase::VALUE_NOT_SET: { - *err = BAD_TYPE; - return 0; + int field = getSimpleField(key); + for (const auto& value : mValues) { + if (value.mField.getField() == field) { + if (value.mValue.getType() == INT) { + return value.mValue.int_value != 0; + } else if (value.mValue.getType() == LONG) { + return value.mValue.long_value != 0; + } else { + *err = BAD_TYPE; + return false; + } + } + if ((size_t)value.mField.getPosAtDepth(0) > key) { + break; } } + + *err = BAD_INDEX; + return false; } float LogEvent::GetFloat(size_t key, status_t* err) const { - DimensionsValue value; - if (!GetSimpleAtomDimensionsValueProto(key, &value)) { - *err = BAD_INDEX; - return 0; - } - const DimensionsValue* leafValue = getSingleLeafValue(&value); - switch (leafValue->value_case()) { - case DimensionsValue::ValueCase::kValueInt: - return (float)leafValue->value_int(); - case DimensionsValue::ValueCase::kValueLong: - return (float)leafValue->value_long(); - case DimensionsValue::ValueCase::kValueBool: - return leafValue->value_bool() ? 1.0f : 0.0f; - case DimensionsValue::ValueCase::kValueFloat: - return leafValue->value_float(); - case DimensionsValue::ValueCase::kValueTuple: - case DimensionsValue::ValueCase::kValueStr: - case DimensionsValue::ValueCase::VALUE_NOT_SET: { - *err = BAD_TYPE; - return 0; + int field = getSimpleField(key); + for (const auto& value : mValues) { + if (value.mField.getField() == field) { + if (value.mValue.getType() == FLOAT) { + return value.mValue.float_value; + } else { + *err = BAD_TYPE; + return 0.0; + } + } + if ((size_t)value.mField.getPosAtDepth(0) > key) { + break; } } -} -void LogEvent::GetAtomDimensionsValueProtos(const FieldMatcher& matcher, - std::vector *dimensionsValues) const { - findDimensionsValues(mFieldValueMap, matcher, dimensionsValues); -} - -bool LogEvent::GetAtomDimensionsValueProto(const FieldMatcher& matcher, - DimensionsValue* dimensionsValue) const { - std::vector rootDimensionsValues; - findDimensionsValues(mFieldValueMap, matcher, &rootDimensionsValues); - if (rootDimensionsValues.size() != 1) { - return false; - } - *dimensionsValue = rootDimensionsValues.front(); - return true; -} - -bool LogEvent::GetSimpleAtomDimensionsValueProto(size_t atomField, - DimensionsValue* dimensionsValue) const { - FieldMatcher matcher; - buildSimpleAtomFieldMatcher(mTagId, atomField, &matcher); - return GetAtomDimensionsValueProto(matcher, dimensionsValue); -} - -DimensionsValue* LogEvent::findFieldValueOrNull(const Field& field) { - auto it = mFieldValueMap.find(field); - if (it == mFieldValueMap.end()) { - return nullptr; - } - return &it->second; + *err = BAD_INDEX; + return 0.0; } string LogEvent::ToString() const { ostringstream result; result << "{ " << mTimestampNs << " (" << mTagId << ")"; - for (const auto& itr : mFieldValueMap) { - result << FieldToString(itr.first); + for (const auto& value : mValues) { + result << StringPrintf("%#x", value.mField.getField()); result << "->"; - result << DimensionsValueToString(itr.second); + result << value.mValue.toString(); result << " "; } result << " }"; @@ -433,7 +380,7 @@ string LogEvent::ToString() const { } void LogEvent::ToProto(ProtoOutputStream& protoOutput) const { - writeFieldValueTreeToStream(getFieldValueMap(), &protoOutput); + writeFieldValueTreeToStream(mTagId, getValues(), &protoOutput); } } // namespace statsd diff --git a/cmds/statsd/src/logd/LogEvent.h b/cmds/statsd/src/logd/LogEvent.h index d521e09569b61..0895daa49ad38 100644 --- a/cmds/statsd/src/logd/LogEvent.h +++ b/cmds/statsd/src/logd/LogEvent.h @@ -16,7 +16,7 @@ #pragma once -#include "field_util.h" +#include "FieldValue.h" #include "frameworks/base/cmds/statsd/src/stats_log.pb.h" #include @@ -24,11 +24,8 @@ #include #include #include -#include -#include #include -#include #include namespace android { @@ -37,7 +34,6 @@ namespace statsd { using std::string; using std::vector; - /** * Wrapper for the log_msg structure. */ @@ -81,19 +77,6 @@ public: bool GetBool(size_t key, status_t* err) const; float GetFloat(size_t key, status_t* err) const; - /* - * Get DimensionsValue proto objects from FieldMatcher. - */ - void GetAtomDimensionsValueProtos( - const FieldMatcher& matcher, std::vector *dimensionsValues) const; - bool GetAtomDimensionsValueProto( - const FieldMatcher& matcher, DimensionsValue* dimensionsValue) const; - - /* - * Get a DimensionsValue proto objects from Field. - */ - bool GetSimpleAtomDimensionsValueProto(size_t field, DimensionsValue* dimensionsValue) const; - /** * Write test data to the LogEvent. This can only be used when the LogEvent is constructed * using LogEvent(tagId, timestampNs). You need to call init() before you can read from it. @@ -129,15 +112,16 @@ public: void setTimestampNs(uint64_t timestampNs) {mTimestampNs = timestampNs;} inline int size() const { - return mFieldValueMap.size(); + return mValues.size(); } - /** - * Returns the mutable DimensionsValue proto for the specific the field. - */ - DimensionsValue* findFieldValueOrNull(const Field& field); + const std::vector& getValues() const { + return mValues; + } - inline const FieldValueMap& getFieldValueMap() const { return mFieldValueMap; } + std::vector* getMutableValues() { + return &mValues; + } private: /** @@ -151,7 +135,9 @@ private: */ void init(android_log_context context); - FieldValueMap mFieldValueMap; + // The items are naturally sorted in DFS order as we read them. this allows us to do fast + // matching. + std::vector mValues; // This field is used when statsD wants to create log event object and write fields to it. After // calling init() function, this object would be destroyed to save memory usage. diff --git a/cmds/statsd/src/matchers/matcher_util.cpp b/cmds/statsd/src/matchers/matcher_util.cpp index fae91729fe4ff..944764bece8ad 100644 --- a/cmds/statsd/src/matchers/matcher_util.cpp +++ b/cmds/statsd/src/matchers/matcher_util.cpp @@ -13,23 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - +#define DEBUG false // STOPSHIP if true #include "Log.h" #include "frameworks/base/cmds/statsd/src/statsd_config.pb.h" #include "matchers/LogMatchingTracker.h" #include "matchers/matcher_util.h" -#include "dimension.h" #include "stats_util.h" -#include "field_util.h" - -#include -#include -#include -#include - -#include -#include using std::ostringstream; using std::set; @@ -93,198 +83,224 @@ bool combinationMatch(const vector& children, const LogicalOperation& opera return matched; } -namespace { - -bool matchFieldSimple(const UidMap& uidMap, const FieldValueMap& fieldMap, - const FieldValueMatcher&matcher, Field* rootField, Field* leafField); - -bool matchesNonRepeatedField(const UidMap& uidMap, const FieldValueMap& fieldMap, - const FieldValueMatcher&matcher, Field* rootField, Field* leafField) { - if (matcher.value_matcher_case() == - FieldValueMatcher::ValueMatcherCase::VALUE_MATCHER_NOT_SET) { - return !fieldMap.empty() && fieldMap.begin()->first.field() == matcher.field(); - } else if (matcher.value_matcher_case() == FieldValueMatcher::ValueMatcherCase::kMatchesTuple) { - bool allMatched = true; - Field* newLeafField = leafField->add_child(); - for (int i = 0; allMatched && i < matcher.matches_tuple().field_value_matcher_size(); ++i) { - const auto& childMatcher = matcher.matches_tuple().field_value_matcher(i); - newLeafField->set_field(childMatcher.field()); - allMatched &= matchFieldSimple(uidMap, fieldMap, childMatcher, rootField, newLeafField); - } - leafField->clear_child(); - return allMatched; - } else { - auto ret = fieldMap.equal_range(*rootField); - int found = 0; - for (auto it = ret.first; it != ret.second; ++it) { - found++; - } - // Not found. - if (found <= 0) { - return false; - } - if (found > 1) { - ALOGE("Found multiple values for optional field."); - return false; - } - bool matched = false; - switch (matcher.value_matcher_case()) { - case FieldValueMatcher::ValueMatcherCase::kEqBool: { - // Logd does not support bool, it is int instead. - matched = ((ret.first->second.value_int() > 0) == matcher.eq_bool()); - break; - } - case FieldValueMatcher::ValueMatcherCase::kEqString: { - if (IsAttributionUidField(*rootField)) { - const int uid = ret.first->second.value_int(); - std::set packageNames = - uidMap.getAppNamesFromUid(uid, true /* normalize*/); - matched = packageNames.find(matcher.eq_string()) != packageNames.end(); - } else { - matched = (ret.first->second.value_str() == matcher.eq_string()); - } - break; - } - case FieldValueMatcher::ValueMatcherCase::kEqInt: { - int64_t val = ret.first->second.has_value_int() ? - ret.first->second.value_int() : ret.first->second.value_long(); - matched = (val == matcher.eq_int()); - break; - } - case FieldValueMatcher::ValueMatcherCase::kLtInt: { - int64_t val = ret.first->second.has_value_int() ? - ret.first->second.value_int() : ret.first->second.value_long(); - matched = (val < matcher.lt_int()); - break; - } - case FieldValueMatcher::ValueMatcherCase::kGtInt: { - int64_t val = ret.first->second.has_value_int() ? - ret.first->second.value_int() : ret.first->second.value_long(); - matched = (val > matcher.gt_int()); - break; - } - case FieldValueMatcher::ValueMatcherCase::kLtFloat: { - matched = (ret.first->second.value_float() < matcher.lt_float()); - break; - } - case FieldValueMatcher::ValueMatcherCase::kGtFloat: { - matched = (ret.first->second.value_float() > matcher.gt_float()); - break; - } - case FieldValueMatcher::ValueMatcherCase::kLteInt: { - int64_t val = ret.first->second.has_value_int() ? - ret.first->second.value_int() : ret.first->second.value_long(); - matched = (val <= matcher.lte_int()); - break; - } - case FieldValueMatcher::ValueMatcherCase::kGteInt: { - int64_t val = ret.first->second.has_value_int() ? - ret.first->second.value_int() : ret.first->second.value_long(); - matched = (val >= matcher.gte_int()); - break; - } - default: - break; - } - return matched; +bool tryMatchString(const UidMap& uidMap, const Field& field, const Value& value, + const string& str_match) { + if (isAttributionUidField(field, value)) { + int uid = value.int_value; + std::set packageNames = uidMap.getAppNamesFromUid(uid, true /* normalize*/); + return packageNames.find(str_match) != packageNames.end(); + } else if (value.getType() == STRING) { + return value.str_value == str_match; } + return false; } -bool matchesRepeatedField(const UidMap& uidMap, const FieldValueMap& fieldMap, - const FieldValueMatcher&matcher, - Field* rootField, Field* leafField) { - if (matcher.position() == Position::FIRST) { - leafField->set_position_index(0); - bool res = matchesNonRepeatedField(uidMap, fieldMap, matcher, rootField, leafField); - leafField->clear_position_index(); - return res; - } else { - auto itLower = fieldMap.lower_bound(*rootField); - if (itLower == fieldMap.end()) { +bool matchesSimple(const UidMap& uidMap, const FieldValueMatcher& matcher, + const vector& values, int start, int end, int depth) { + if (depth > 2) { + ALOGE("Depth > 3 not supported"); + return false; + } + + if (start >= end) { + return false; + } + + // Filter by entry field first + int newStart = -1; + int newEnd = end; + // because the fields are naturally sorted in the DFS order. we can safely + // break when pos is larger than the one we are searching for. + for (int i = start; i < end; i++) { + int pos = values[i].mField.getPosAtDepth(depth); + if (pos == matcher.field()) { + if (newStart == -1) { + newStart = i; + } + newEnd = i + 1; + } else if (pos > matcher.field()) { + break; + } + } + + // Now we have zoomed in to a new range + start = newStart; + end = newEnd; + + if (start == -1) { + // No such field found. + return false; + } + + vector> ranges; // the ranges are for matching ANY position + if (matcher.has_position()) { + // Repeated fields position is stored as a node in the path. + depth++; + if (depth > 2) { return false; } - - const int leafFieldNum = leafField->field(); - leafField->set_field(leafFieldNum + 1); - auto itUpper = fieldMap.lower_bound(*rootField); - // Resets the field number. - leafField->set_field(leafFieldNum); - switch (matcher.position()) { - case Position::LAST: - { - itUpper--; - if (itUpper == fieldMap.end()) { - return false; - } else { - int last_index = getPositionByReferenceField(*rootField, itUpper->first); - if (last_index < 0) { - return false; - } - leafField->set_position_index(last_index); - bool res = matchesNonRepeatedField(uidMap, fieldMap, matcher, rootField, leafField); - leafField->clear_position_index(); - return res; - } - } - break; - case Position::ANY: - { - bool matched = false; - for (auto it = itLower; it != itUpper; ++it) { - int index = getPositionByReferenceField(*rootField, it->first); - if (index >= 0) { - leafField->set_position_index(index); - matched |= matchesNonRepeatedField(uidMap, fieldMap, matcher, rootField, leafField); - leafField->clear_position_index(); - if (matched) { - break; - } - } + case Position::FIRST: { + for (int i = start; i < end; i++) { + int pos = values[i].mField.getPosAtDepth(depth); + if (pos != 1) { + // Again, the log elements are stored in sorted order. so + // once the position is > 1, we break; + end = i; + break; } - return matched; - } - default: - return false; - } - } - -} - -bool matchFieldSimple(const UidMap& uidMap, const FieldValueMap& fieldMap, - const FieldValueMatcher&matcher, Field* rootField, Field* leafField) { - if (!matcher.has_position()) { - return matchesNonRepeatedField(uidMap, fieldMap, matcher, rootField, leafField); + } + ranges.push_back(std::make_pair(start, end)); + break; + } + case Position::LAST: { + // move the starting index to the first LAST field at the depth. + for (int i = start; i < end; i++) { + if (values[i].mField.isLastPos(depth)) { + start = i; + break; + } + } + ranges.push_back(std::make_pair(start, end)); + break; + } + case Position::ANY: { + // ANY means all the children matchers match in any of the sub trees, it's a match + newStart = start; + newEnd = end; + // Here start is guaranteed to be a valid index. + int currentPos = values[start].mField.getPosAtDepth(depth); + // Now find all sub trees ranges. + for (int i = start; i < end; i++) { + int newPos = values[i].mField.getPosAtDepth(depth); + if (newPos != currentPos) { + ranges.push_back(std::make_pair(newStart, i)); + newStart = i; + currentPos = newPos; + } + } + ranges.push_back(std::make_pair(newStart, end)); + break; + } + case Position::POSITION_UNKNOWN: + break; + } } else { - return matchesRepeatedField(uidMap, fieldMap, matcher, rootField, leafField); + // No position + ranges.push_back(std::make_pair(start, end)); + } + // start and end are still pointing to the matched range. + switch (matcher.value_matcher_case()) { + case FieldValueMatcher::kMatchesTuple: { + ++depth; + // If any range matches all matchers, good. + for (const auto& range : ranges) { + bool matched = true; + for (const auto& subMatcher : matcher.matches_tuple().field_value_matcher()) { + if (!matchesSimple(uidMap, subMatcher, values, range.first, range.second, + depth)) { + matched = false; + break; + } + } + if (matched) return true; + } + return false; + } + case FieldValueMatcher::ValueMatcherCase::kEqBool: { + for (int i = start; i < end; i++) { + if ((values[i].mValue.getType() == INT && + (values[i].mValue.int_value != 0) == matcher.eq_bool()) || + (values[i].mValue.getType() == LONG && + (values[i].mValue.long_value != 0) == matcher.eq_bool())) { + return true; + } + } + return false; + } + case FieldValueMatcher::ValueMatcherCase::kEqString: { + for (int i = start; i < end; i++) { + if (tryMatchString(uidMap, values[i].mField, values[i].mValue, + matcher.eq_string())) { + return true; + } + } + } + return false; + case FieldValueMatcher::ValueMatcherCase::kEqInt: + for (int i = start; i < end; i++) { + if (values[i].mValue.getType() == INT && + (matcher.eq_int() == values[i].mValue.int_value)) { + return true; + } + } + return false; + case FieldValueMatcher::ValueMatcherCase::kLtInt: + for (int i = start; i < end; i++) { + if (values[i].mValue.getType() == INT && + (values[i].mValue.int_value < matcher.lt_int())) { + return true; + } + } + return false; + case FieldValueMatcher::ValueMatcherCase::kGtInt: + for (int i = start; i < end; i++) { + if (values[i].mValue.getType() == INT && + (values[i].mValue.int_value > matcher.gt_int())) { + return true; + } + } + return false; + case FieldValueMatcher::ValueMatcherCase::kLtFloat: + for (int i = start; i < end; i++) { + if (values[i].mValue.getType() == FLOAT && + (values[i].mValue.float_value < matcher.lt_float())) { + return true; + } + } + return false; + case FieldValueMatcher::ValueMatcherCase::kGtFloat: + for (int i = start; i < end; i++) { + if (values[i].mValue.getType() == FLOAT && + (values[i].mValue.float_value > matcher.gt_float())) { + return true; + } + } + return false; + case FieldValueMatcher::ValueMatcherCase::kLteInt: + for (int i = start; i < end; i++) { + if (values[i].mValue.getType() == INT && + (values[i].mValue.int_value <= matcher.lte_int())) { + return true; + } + } + return false; + case FieldValueMatcher::ValueMatcherCase::kGteInt: + for (int i = start; i < end; i++) { + if (values[i].mValue.getType() == INT && + (values[i].mValue.int_value >= matcher.gte_int())) { + return true; + } + } + return false; + default: + return false; } } -} // namespace - bool matchesSimple(const UidMap& uidMap, const SimpleAtomMatcher& simpleMatcher, const LogEvent& event) { if (simpleMatcher.field_value_matcher_size() <= 0) { return event.GetTagId() == simpleMatcher.atom_id(); } - Field root_field; - root_field.set_field(simpleMatcher.atom_id()); - FieldValueMatcher root_field_matcher; - root_field_matcher.set_field(simpleMatcher.atom_id()); - for (int i = 0; i < simpleMatcher.field_value_matcher_size(); i++) { - *root_field_matcher.mutable_matches_tuple()->add_field_value_matcher() = - simpleMatcher.field_value_matcher(i); + for (const auto& matcher : simpleMatcher.field_value_matcher()) { + if (!matchesSimple(uidMap, matcher, event.getValues(), 0, event.getValues().size(), 0)) { + return false; + } } - return matchFieldSimple( - uidMap, event.getFieldValueMap(), root_field_matcher, &root_field, &root_field); + return true; } -void getDimensionKeys(const LogEvent& event, const FieldMatcher& matcher, - std::vector *dimensionKeys) { - if (matcher.has_field()) { - findDimensionsValues(event.getFieldValueMap(), matcher, dimensionKeys); - } -} } // namespace statsd } // namespace os } // namespace android diff --git a/cmds/statsd/src/matchers/matcher_util.h b/cmds/statsd/src/matchers/matcher_util.h index a45a9fb26a135..872cd8e64575c 100644 --- a/cmds/statsd/src/matchers/matcher_util.h +++ b/cmds/statsd/src/matchers/matcher_util.h @@ -45,9 +45,6 @@ bool combinationMatch(const std::vector& children, const LogicalOperation& bool matchesSimple(const UidMap& uidMap, const SimpleAtomMatcher& simpleMatcher, const LogEvent& wrapper); -void getDimensionKeys(const LogEvent& event, const FieldMatcher& matcher, - std::vector *dimensionKeys); - } // namespace statsd } // namespace os } // namespace android diff --git a/cmds/statsd/src/metrics/CountMetricProducer.cpp b/cmds/statsd/src/metrics/CountMetricProducer.cpp index 5a042b63c675d..bd2674b86a469 100644 --- a/cmds/statsd/src/metrics/CountMetricProducer.cpp +++ b/cmds/statsd/src/metrics/CountMetricProducer.cpp @@ -21,7 +21,6 @@ #include "guardrail/StatsdStats.h" #include "stats_util.h" #include "stats_log_util.h" -#include "dimension.h" #include #include @@ -69,16 +68,26 @@ CountMetricProducer::CountMetricProducer(const ConfigKey& key, const CountMetric mBucketSizeNs = LLONG_MAX; } - // TODO: use UidMap if uid->pkg_name is required - mDimensionsInWhat = metric.dimensions_in_what(); - mDimensionsInCondition = metric.dimensions_in_condition(); + if (metric.has_dimensions_in_what()) { + translateFieldMatcher(metric.dimensions_in_what(), &mDimensionsInWhat); + } + + if (metric.has_dimensions_in_condition()) { + translateFieldMatcher(metric.dimensions_in_condition(), &mDimensionsInCondition); + } if (metric.links().size() > 0) { - mConditionLinks.insert(mConditionLinks.begin(), metric.links().begin(), - metric.links().end()); + for (const auto& link : metric.links()) { + Metric2Condition mc; + mc.conditionId = link.condition(); + translateFieldMatcher(link.fields_in_what(), &mc.metricFields); + translateFieldMatcher(link.fields_in_condition(), &mc.conditionFields); + mMetric2ConditionLinks.push_back(mc); + } + mConditionSliced = true; } - mConditionSliced = (metric.links().size() > 0)|| - (mDimensionsInCondition.has_field() && mDimensionsInCondition.child_size() > 0); + + mConditionSliced = (metric.links().size() > 0) || (mDimensionsInCondition.size() > 0); VLOG("metric %lld created. bucket size %lld start_time: %lld", (long long)metric.id(), (long long)mBucketSizeNs, (long long)mStartTimeNs); @@ -92,26 +101,6 @@ void CountMetricProducer::onSlicedConditionMayChangeLocked(const uint64_t eventT VLOG("Metric %lld onSlicedConditionMayChange", (long long)mMetricId); } -void CountMetricProducer::onDumpReportLocked(const uint64_t dumpTimeNs, StatsLogReport* report) { - flushIfNeededLocked(dumpTimeNs); - report->set_metric_id(mMetricId); - - auto count_metrics = report->mutable_count_metrics(); - for (const auto& counter : mPastBuckets) { - CountMetricData* metricData = count_metrics->add_data(); - *metricData->mutable_dimensions_in_what() = - counter.first.getDimensionKeyInWhat().getDimensionsValue(); - *metricData->mutable_dimensions_in_condition() = - counter.first.getDimensionKeyInCondition().getDimensionsValue(); - for (const auto& bucket : counter.second) { - CountBucketInfo* bucketInfo = metricData->add_bucket_info(); - bucketInfo->set_start_bucket_nanos(bucket.mBucketStartNs); - bucketInfo->set_end_bucket_nanos(bucket.mBucketEndNs); - bucketInfo->set_count(bucket.mCount); - } - } -} - void CountMetricProducer::onDumpReportLocked(const uint64_t dumpTimeNs, ProtoOutputStream* protoOutput) { flushIfNeededLocked(dumpTimeNs); @@ -122,8 +111,6 @@ void CountMetricProducer::onDumpReportLocked(const uint64_t dumpTimeNs, protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_ID, (long long)mMetricId); long long protoToken = protoOutput->start(FIELD_TYPE_MESSAGE | FIELD_ID_COUNT_METRICS); - VLOG("metric %lld dump report now...",(long long)mMetricId); - for (const auto& counter : mPastBuckets) { const MetricDimensionKey& dimensionKey = counter.first; VLOG(" dimension key %s", dimensionKey.c_str()); @@ -134,15 +121,13 @@ void CountMetricProducer::onDumpReportLocked(const uint64_t dumpTimeNs, // First fill dimension. long long dimensionInWhatToken = protoOutput->start( FIELD_TYPE_MESSAGE | FIELD_ID_DIMENSION_IN_WHAT); - writeDimensionsValueProtoToStream( - dimensionKey.getDimensionKeyInWhat().getDimensionsValue(), protoOutput); + writeDimensionToProto(dimensionKey.getDimensionKeyInWhat(), protoOutput); protoOutput->end(dimensionInWhatToken); if (dimensionKey.hasDimensionKeyInCondition()) { long long dimensionInConditionToken = protoOutput->start( FIELD_TYPE_MESSAGE | FIELD_ID_DIMENSION_IN_CONDITION); - writeDimensionsValueProtoToStream( - dimensionKey.getDimensionKeyInCondition().getDimensionsValue(), protoOutput); + writeDimensionToProto(dimensionKey.getDimensionKeyInCondition(), protoOutput); protoOutput->end(dimensionInConditionToken); } @@ -200,7 +185,6 @@ void CountMetricProducer::onMatchedLogEventInternalLocked( const ConditionKey& conditionKey, bool condition, const LogEvent& event) { uint64_t eventTimeNs = event.GetTimestampNs(); - flushIfNeededLocked(eventTimeNs); if (condition == false) { diff --git a/cmds/statsd/src/metrics/CountMetricProducer.h b/cmds/statsd/src/metrics/CountMetricProducer.h index b06c77b27faf0..0c4291d5dfc2a 100644 --- a/cmds/statsd/src/metrics/CountMetricProducer.h +++ b/cmds/statsd/src/metrics/CountMetricProducer.h @@ -57,7 +57,6 @@ protected: private: void onDumpReportLocked(const uint64_t dumpTimeNs, android::util::ProtoOutputStream* protoOutput) override; - void onDumpReportLocked(const uint64_t dumpTimeNs, StatsLogReport* report) override; // Internal interface to handle condition change. void onConditionChangedLocked(const bool conditionMet, const uint64_t eventTime) override; diff --git a/cmds/statsd/src/metrics/DurationMetricProducer.cpp b/cmds/statsd/src/metrics/DurationMetricProducer.cpp index 65cbc4afdad4c..6b321e11edcf0 100644 --- a/cmds/statsd/src/metrics/DurationMetricProducer.cpp +++ b/cmds/statsd/src/metrics/DurationMetricProducer.cpp @@ -21,7 +21,6 @@ #include "guardrail/StatsdStats.h" #include "stats_util.h" #include "stats_log_util.h" -#include "dimension.h" #include #include @@ -68,8 +67,7 @@ DurationMetricProducer::DurationMetricProducer(const ConfigKey& key, const Durat mStartIndex(startIndex), mStopIndex(stopIndex), mStopAllIndex(stopAllIndex), - mNested(nesting), - mInternalDimensions(internalDimensions) { + mNested(nesting) { // TODO: The following boiler plate code appears in all MetricProducers, but we can't abstract // them in the base class, because the proto generated CountMetric, and DurationMetric are // not related. Maybe we should add a template in the future?? @@ -79,16 +77,28 @@ DurationMetricProducer::DurationMetricProducer(const ConfigKey& key, const Durat mBucketSizeNs = LLONG_MAX; } - // TODO: use UidMap if uid->pkg_name is required - mDimensionsInWhat = metric.dimensions_in_what(); - mDimensionsInCondition = metric.dimensions_in_condition(); + if (metric.has_dimensions_in_what()) { + translateFieldMatcher(metric.dimensions_in_what(), &mDimensionsInWhat); + } + + if (internalDimensions.has_field()) { + translateFieldMatcher(internalDimensions, &mInternalDimensions); + } + + if (metric.has_dimensions_in_condition()) { + translateFieldMatcher(metric.dimensions_in_condition(), &mDimensionsInCondition); + } if (metric.links().size() > 0) { - mConditionLinks.insert(mConditionLinks.begin(), metric.links().begin(), - metric.links().end()); + for (const auto& link : metric.links()) { + Metric2Condition mc; + mc.conditionId = link.condition(); + translateFieldMatcher(link.fields_in_what(), &mc.metricFields); + translateFieldMatcher(link.fields_in_condition(), &mc.conditionFields); + mMetric2ConditionLinks.push_back(mc); + } } - mConditionSliced = (metric.links().size() > 0)|| - (mDimensionsInCondition.has_field() && mDimensionsInCondition.child_size() > 0); + mConditionSliced = (metric.links().size() > 0) || (mDimensionsInCondition.size() > 0); VLOG("metric %lld created. bucket size %lld start_time: %lld", (long long)metric.id(), (long long)mBucketSizeNs, (long long)mStartTimeNs); @@ -174,37 +184,18 @@ void DurationMetricProducer::onConditionChangedLocked(const bool conditionMet, } } -void DurationMetricProducer::onDumpReportLocked(const uint64_t dumpTimeNs, StatsLogReport* report) { - flushIfNeededLocked(dumpTimeNs); - report->set_metric_id(mMetricId); - - auto duration_metrics = report->mutable_duration_metrics(); - for (const auto& pair : mPastBuckets) { - DurationMetricData* metricData = duration_metrics->add_data(); - *metricData->mutable_dimensions_in_what() = - pair.first.getDimensionKeyInWhat().getDimensionsValue(); - *metricData->mutable_dimensions_in_condition() = - pair.first.getDimensionKeyInCondition().getDimensionsValue(); - for (const auto& bucket : pair.second) { - auto bucketInfo = metricData->add_bucket_info(); - bucketInfo->set_start_bucket_nanos(bucket.mBucketStartNs); - bucketInfo->set_end_bucket_nanos(bucket.mBucketEndNs); - bucketInfo->set_duration_nanos(bucket.mDuration); - } - } -} - void DurationMetricProducer::onDumpReportLocked(const uint64_t dumpTimeNs, ProtoOutputStream* protoOutput) { flushIfNeededLocked(dumpTimeNs); if (mPastBuckets.empty()) { + VLOG(" Duration metric, empty return"); return; } protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_ID, (long long)mMetricId); long long protoToken = protoOutput->start(FIELD_TYPE_MESSAGE | FIELD_ID_DURATION_METRICS); - VLOG("metric %lld dump report now...", (long long)mMetricId); + VLOG("Duration metric %lld dump report now...", (long long)mMetricId); for (const auto& pair : mPastBuckets) { const MetricDimensionKey& dimensionKey = pair.first; @@ -216,15 +207,13 @@ void DurationMetricProducer::onDumpReportLocked(const uint64_t dumpTimeNs, // First fill dimension. long long dimensionToken = protoOutput->start( FIELD_TYPE_MESSAGE | FIELD_ID_DIMENSION_IN_WHAT); - writeDimensionsValueProtoToStream( - dimensionKey.getDimensionKeyInWhat().getDimensionsValue(), protoOutput); + writeDimensionToProto(dimensionKey.getDimensionKeyInWhat(), protoOutput); protoOutput->end(dimensionToken); if (dimensionKey.hasDimensionKeyInCondition()) { long long dimensionInConditionToken = protoOutput->start( FIELD_TYPE_MESSAGE | FIELD_ID_DIMENSION_IN_CONDITION); - writeDimensionsValueProtoToStream( - dimensionKey.getDimensionKeyInCondition().getDimensionsValue(), protoOutput); + writeDimensionToProto(dimensionKey.getDimensionKeyInCondition(), protoOutput); protoOutput->end(dimensionInConditionToken); } @@ -339,8 +328,8 @@ void DurationMetricProducer::onMatchedLogEventInternalLocked( auto it = mCurrentSlicedDurationTrackerMap.find(eventKey); - std::vector values; - getDimensionKeys(event, mInternalDimensions, &values); + std::vector values; + filterValues(mInternalDimensions, event.getValues(), &values); if (values.empty()) { if (matcherIndex == mStartIndex) { it->second->noteStart(DEFAULT_DIMENSION_KEY, condition, @@ -349,13 +338,11 @@ void DurationMetricProducer::onMatchedLogEventInternalLocked( it->second->noteStop(DEFAULT_DIMENSION_KEY, event.GetTimestampNs(), false); } } else { - for (const DimensionsValue& value : values) { + for (const auto& value : values) { if (matcherIndex == mStartIndex) { - it->second->noteStart( - HashableDimensionKey(value), condition, event.GetTimestampNs(), conditionKeys); + it->second->noteStart(value, condition, event.GetTimestampNs(), conditionKeys); } else if (matcherIndex == mStopIndex) { - it->second->noteStop( - HashableDimensionKey(value), event.GetTimestampNs(), false); + it->second->noteStop(value, event.GetTimestampNs(), false); } } } diff --git a/cmds/statsd/src/metrics/DurationMetricProducer.h b/cmds/statsd/src/metrics/DurationMetricProducer.h index a49601655e5ea..5f29281a8a3c1 100644 --- a/cmds/statsd/src/metrics/DurationMetricProducer.h +++ b/cmds/statsd/src/metrics/DurationMetricProducer.h @@ -57,7 +57,6 @@ protected: private: void onDumpReportLocked(const uint64_t dumpTimeNs, android::util::ProtoOutputStream* protoOutput) override; - void onDumpReportLocked(const uint64_t dumpTimeNs, StatsLogReport* report) override; // Internal interface to handle condition change. void onConditionChangedLocked(const bool conditionMet, const uint64_t eventTime) override; @@ -90,7 +89,7 @@ private: const bool mNested; // The dimension from the atom predicate. e.g., uid, wakelock name. - const FieldMatcher mInternalDimensions; + vector mInternalDimensions; // Save the past buckets and we can clear when the StatsLogReport is dumped. // TODO: Add a lock to mPastBuckets. diff --git a/cmds/statsd/src/metrics/EventMetricProducer.cpp b/cmds/statsd/src/metrics/EventMetricProducer.cpp index 0578e0682687f..eed178b08016f 100644 --- a/cmds/statsd/src/metrics/EventMetricProducer.cpp +++ b/cmds/statsd/src/metrics/EventMetricProducer.cpp @@ -55,8 +55,13 @@ EventMetricProducer::EventMetricProducer(const ConfigKey& key, const EventMetric const uint64_t startTimeNs) : MetricProducer(metric.id(), key, startTimeNs, conditionIndex, wizard) { if (metric.links().size() > 0) { - mConditionLinks.insert(mConditionLinks.begin(), metric.links().begin(), - metric.links().end()); + for (const auto& link : metric.links()) { + Metric2Condition mc; + mc.conditionId = link.condition(); + translateFieldMatcher(link.fields_in_what(), &mc.metricFields); + translateFieldMatcher(link.fields_in_condition(), &mc.conditionFields); + mMetric2ConditionLinks.push_back(mc); + } mConditionSliced = true; } @@ -94,10 +99,6 @@ std::unique_ptr> serializeProtoLocked(ProtoOutputStream& pr return buffer; } -void EventMetricProducer::onDumpReportLocked(const uint64_t dumpTimeNs, StatsLogReport* report) { - -} - void EventMetricProducer::onDumpReportLocked(const uint64_t dumpTimeNs, ProtoOutputStream* protoOutput) { if (mProto->size() <= 0) { diff --git a/cmds/statsd/src/metrics/EventMetricProducer.h b/cmds/statsd/src/metrics/EventMetricProducer.h index 935f206017fa2..4cf3094f7d64d 100644 --- a/cmds/statsd/src/metrics/EventMetricProducer.h +++ b/cmds/statsd/src/metrics/EventMetricProducer.h @@ -51,7 +51,6 @@ private: void onDumpReportLocked(const uint64_t dumpTimeNs, android::util::ProtoOutputStream* protoOutput) override; - void onDumpReportLocked(const uint64_t dumpTimeNs, StatsLogReport* report) override; // Internal interface to handle condition change. void onConditionChangedLocked(const bool conditionMet, const uint64_t eventTime) override; diff --git a/cmds/statsd/src/metrics/GaugeMetricProducer.cpp b/cmds/statsd/src/metrics/GaugeMetricProducer.cpp index 62ee6ef17d5ca..da0cafeca427f 100644 --- a/cmds/statsd/src/metrics/GaugeMetricProducer.cpp +++ b/cmds/statsd/src/metrics/GaugeMetricProducer.cpp @@ -19,7 +19,6 @@ #include "GaugeMetricProducer.h" #include "guardrail/StatsdStats.h" -#include "dimension.h" #include "stats_log_util.h" #include @@ -77,18 +76,29 @@ GaugeMetricProducer::GaugeMetricProducer(const ConfigKey& key, const GaugeMetric mBucketSizeNs = bucketSizeMills * 1000000; mSamplingType = metric.sampling_type(); - mFieldFilter = metric.gauge_fields_filter(); + if (!metric.gauge_fields_filter().include_all()) { + translateFieldMatcher(metric.gauge_fields_filter().fields(), &mFieldMatchers); + } // TODO: use UidMap if uid->pkg_name is required - mDimensionsInWhat = metric.dimensions_in_what(); - mDimensionsInCondition = metric.dimensions_in_condition(); + if (metric.has_dimensions_in_what()) { + translateFieldMatcher(metric.dimensions_in_what(), &mDimensionsInWhat); + } + + if (metric.has_dimensions_in_condition()) { + translateFieldMatcher(metric.dimensions_in_condition(), &mDimensionsInCondition); + } if (metric.links().size() > 0) { - mConditionLinks.insert(mConditionLinks.begin(), metric.links().begin(), - metric.links().end()); + for (const auto& link : metric.links()) { + Metric2Condition mc; + mc.conditionId = link.condition(); + translateFieldMatcher(link.fields_in_what(), &mc.metricFields); + translateFieldMatcher(link.fields_in_condition(), &mc.conditionFields); + mMetric2ConditionLinks.push_back(mc); + } } - mConditionSliced = (metric.links().size() > 0)|| - (mDimensionsInCondition.has_field() && mDimensionsInCondition.child_size() > 0); + mConditionSliced = (metric.links().size() > 0) || (mDimensionsInCondition.size() > 0); // Kicks off the puller immediately. if (mPullTagId != -1 && mSamplingType == GaugeMetric::RANDOM_ONE_SAMPLE) { @@ -115,13 +125,6 @@ GaugeMetricProducer::~GaugeMetricProducer() { } } -void GaugeMetricProducer::onDumpReportLocked(const uint64_t dumpTimeNs, StatsLogReport* report) { - flushIfNeededLocked(dumpTimeNs); - ProtoOutputStream pbOutput; - onDumpReportLocked(dumpTimeNs, &pbOutput); - parseProtoOutputStream(pbOutput, report); -} - void GaugeMetricProducer::onDumpReportLocked(const uint64_t dumpTimeNs, ProtoOutputStream* protoOutput) { VLOG("gauge metric %lld report now...", (long long)mMetricId); @@ -144,15 +147,13 @@ void GaugeMetricProducer::onDumpReportLocked(const uint64_t dumpTimeNs, // First fill dimension. long long dimensionToken = protoOutput->start( FIELD_TYPE_MESSAGE | FIELD_ID_DIMENSION_IN_WHAT); - writeDimensionsValueProtoToStream( - dimensionKey.getDimensionKeyInWhat().getDimensionsValue(), protoOutput); + writeDimensionToProto(dimensionKey.getDimensionKeyInWhat(), protoOutput); protoOutput->end(dimensionToken); if (dimensionKey.hasDimensionKeyInCondition()) { long long dimensionInConditionToken = protoOutput->start( FIELD_TYPE_MESSAGE | FIELD_ID_DIMENSION_IN_CONDITION); - writeDimensionsValueProtoToStream( - dimensionKey.getDimensionKeyInCondition().getDimensionsValue(), protoOutput); + writeDimensionToProto(dimensionKey.getDimensionKeyInCondition(), protoOutput); protoOutput->end(dimensionInConditionToken); } @@ -169,7 +170,7 @@ void GaugeMetricProducer::onDumpReportLocked(const uint64_t dumpTimeNs, long long atomsToken = protoOutput->start(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_ATOM); for (const auto& atom : bucket.mGaugeAtoms) { - writeFieldValueTreeToStream(*atom.mFields, protoOutput); + writeFieldValueTreeToStream(mTagId, *(atom.mFields), protoOutput); } protoOutput->end(atomsToken); @@ -246,13 +247,14 @@ void GaugeMetricProducer::onSlicedConditionMayChangeLocked(const uint64_t eventT VLOG("Metric %lld onSlicedConditionMayChange", (long long)mMetricId); } -std::shared_ptr GaugeMetricProducer::getGaugeFields(const LogEvent& event) { - std::shared_ptr gaugeFields = - std::make_shared(event.getFieldValueMap()); - if (!mFieldFilter.include_all()) { - filterFields(mFieldFilter.fields(), gaugeFields.get()); +std::shared_ptr> GaugeMetricProducer::getGaugeFields(const LogEvent& event) { + if (mFieldMatchers.size() > 0) { + std::shared_ptr> gaugeFields = std::make_shared>(); + filterGaugeValues(mFieldMatchers, event.getValues(), gaugeFields.get()); + return gaugeFields; + } else { + return std::make_shared>(event.getValues()); } - return gaugeFields; } void GaugeMetricProducer::onDataPulled(const std::vector>& allData) { @@ -292,6 +294,7 @@ void GaugeMetricProducer::onMatchedLogEventInternalLocked( return; } uint64_t eventTimeNs = event.GetTimestampNs(); + mTagId = event.GetTagId(); if (eventTimeNs < mCurrentBucketStartTimeNs) { VLOG("Skip event due to late arrival: %lld vs %lld", (long long)eventTimeNs, (long long)mCurrentBucketStartTimeNs); @@ -308,20 +311,18 @@ void GaugeMetricProducer::onMatchedLogEventInternalLocked( if (hitGuardRailLocked(eventKey)) { return; } - GaugeAtom gaugeAtom; - gaugeAtom.mFields = getGaugeFields(event); - gaugeAtom.mTimestamps = eventTimeNs; + GaugeAtom gaugeAtom(getGaugeFields(event), eventTimeNs); (*mCurrentSlicedBucket)[eventKey].push_back(gaugeAtom); // Anomaly detection on gauge metric only works when there is one numeric // field specified. if (mAnomalyTrackers.size() > 0) { if (gaugeAtom.mFields->size() == 1) { - const DimensionsValue& dimensionsValue = gaugeAtom.mFields->begin()->second; + const Value& value = gaugeAtom.mFields->begin()->mValue; long gaugeVal = 0; - if (dimensionsValue.has_value_int()) { - gaugeVal = (long)dimensionsValue.value_int(); - } else if (dimensionsValue.has_value_long()) { - gaugeVal = dimensionsValue.value_long(); + if (value.getType() == INT) { + gaugeVal = (long)value.int_value; + } else if (value.getType() == LONG) { + gaugeVal = value.long_value; } for (auto& tracker : mAnomalyTrackers) { tracker->detectAndDeclareAnomaly(eventTimeNs, mCurrentBucketNum, eventKey, @@ -334,15 +335,15 @@ void GaugeMetricProducer::onMatchedLogEventInternalLocked( void GaugeMetricProducer::updateCurrentSlicedBucketForAnomaly() { status_t err = NO_ERROR; for (const auto& slice : *mCurrentSlicedBucket) { - if (slice.second.empty() || slice.second.front().mFields->empty()) { + if (slice.second.empty()) { continue; } - const DimensionsValue& dimensionsValue = slice.second.front().mFields->begin()->second; + const Value& value = slice.second.front().mFields->front().mValue; long gaugeVal = 0; - if (dimensionsValue.has_value_int()) { - gaugeVal = (long)dimensionsValue.value_int(); - } else if (dimensionsValue.has_value_long()) { - gaugeVal = dimensionsValue.value_long(); + if (value.getType() == INT) { + gaugeVal = (long)value.int_value; + } else if (value.getType() == LONG) { + gaugeVal = value.long_value; } (*mCurrentSlicedBucketForAnomaly)[slice.first] = gaugeVal; } diff --git a/cmds/statsd/src/metrics/GaugeMetricProducer.h b/cmds/statsd/src/metrics/GaugeMetricProducer.h index d5d34be045625..c3ae6ce921d26 100644 --- a/cmds/statsd/src/metrics/GaugeMetricProducer.h +++ b/cmds/statsd/src/metrics/GaugeMetricProducer.h @@ -33,7 +33,10 @@ namespace os { namespace statsd { struct GaugeAtom { - std::shared_ptr mFields; + GaugeAtom(std::shared_ptr> fields, int64_t timeNs) + : mFields(fields), mTimestamps(timeNs) { + } + std::shared_ptr> mFields; int64_t mTimestamps; }; @@ -87,7 +90,6 @@ protected: private: void onDumpReportLocked(const uint64_t dumpTimeNs, android::util::ProtoOutputStream* protoOutput) override; - void onDumpReportLocked(const uint64_t dumpTimeNs, StatsLogReport* report) override; // for testing GaugeMetricProducer(const ConfigKey& key, const GaugeMetric& gaugeMetric, @@ -113,6 +115,8 @@ private: void pullLocked(); + int mTagId; + std::shared_ptr mStatsPullerManager; // tagId for pulled data. -1 if this is not pulled const int mPullTagId; @@ -133,12 +137,12 @@ private: void updateCurrentSlicedBucketForAnomaly(); // Whitelist of fields to report. Empty means all are reported. - FieldFilter mFieldFilter; + std::vector mFieldMatchers; GaugeMetric::SamplingType mSamplingType; // apply a whitelist on the original input - std::shared_ptr getGaugeFields(const LogEvent& event); + std::shared_ptr> getGaugeFields(const LogEvent& event); // Util function to check whether the specified dimension hits the guardrail. bool hitGuardRailLocked(const MetricDimensionKey& newKey); diff --git a/cmds/statsd/src/metrics/MetricProducer.cpp b/cmds/statsd/src/metrics/MetricProducer.cpp index 85e655b08f4d9..beb90155f1830 100644 --- a/cmds/statsd/src/metrics/MetricProducer.cpp +++ b/cmds/statsd/src/metrics/MetricProducer.cpp @@ -13,9 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "MetricProducer.h" -#include "dimension.h" +#define DEBUG true // STOPSHIP if true +#include "Log.h" +#include "MetricProducer.h" namespace android { namespace os { @@ -35,9 +36,10 @@ void MetricProducer::onMatchedLogEventLocked(const size_t matcherIndex, const Lo std::unordered_set dimensionKeysInCondition; if (mConditionSliced) { - for (const auto& link : mConditionLinks) { - getDimensionKeysForCondition(event, link, &conditionKey[link.condition()]); + for (const auto& link : mMetric2ConditionLinks) { + getDimensionForCondition(event, link, &conditionKey[link.conditionId]); } + auto conditionState = mWizard->query(mConditionTrackerIndex, conditionKey, mDimensionsInCondition, &dimensionKeysInCondition); @@ -46,20 +48,19 @@ void MetricProducer::onMatchedLogEventLocked(const size_t matcherIndex, const Lo condition = mCondition; } - vector dimensionInWhatValues; - if (mDimensionsInWhat.has_field() && mDimensionsInWhat.child_size() > 0) { - getDimensionKeys(event, mDimensionsInWhat, &dimensionInWhatValues); + vector dimensionInWhatValues; + if (mDimensionsInWhat.size() > 0) { + filterValues(mDimensionsInWhat, event.getValues(), &dimensionInWhatValues); } if (dimensionInWhatValues.empty() && dimensionKeysInCondition.empty()) { onMatchedLogEventInternalLocked( matcherIndex, DEFAULT_METRIC_DIMENSION_KEY, conditionKey, condition, event); } else if (dimensionKeysInCondition.empty()) { - for (const DimensionsValue& whatValue : dimensionInWhatValues) { - onMatchedLogEventInternalLocked( - matcherIndex, - MetricDimensionKey(HashableDimensionKey(whatValue), DEFAULT_DIMENSION_KEY), - conditionKey, condition, event); + for (const HashableDimensionKey& whatValue : dimensionInWhatValues) { + onMatchedLogEventInternalLocked(matcherIndex, + MetricDimensionKey(whatValue, DEFAULT_DIMENSION_KEY), + conditionKey, condition, event); } } else if (dimensionInWhatValues.empty()) { for (const auto& conditionDimensionKey : dimensionKeysInCondition) { @@ -69,12 +70,11 @@ void MetricProducer::onMatchedLogEventLocked(const size_t matcherIndex, const Lo conditionKey, condition, event); } } else { - for (const DimensionsValue& whatValue : dimensionInWhatValues) { + for (const auto& whatValue : dimensionInWhatValues) { for (const auto& conditionDimensionKey : dimensionKeysInCondition) { onMatchedLogEventInternalLocked( - matcherIndex, - MetricDimensionKey(HashableDimensionKey(whatValue), conditionDimensionKey), - conditionKey, condition, event); + matcherIndex, MetricDimensionKey(whatValue, conditionDimensionKey), + conditionKey, condition, event); } } } diff --git a/cmds/statsd/src/metrics/MetricProducer.h b/cmds/statsd/src/metrics/MetricProducer.h index 542dd8ab019df..e8f8299abd89c 100644 --- a/cmds/statsd/src/metrics/MetricProducer.h +++ b/cmds/statsd/src/metrics/MetricProducer.h @@ -19,6 +19,7 @@ #include +#include "HashableDimensionKey.h" #include "anomaly/AnomalyTracker.h" #include "condition/ConditionWizard.h" #include "config/ConfigKey.h" @@ -109,11 +110,6 @@ public: std::lock_guard lock(mMutex); return onDumpReportLocked(dumpTimeNs, protoOutput); } - // This method does not clear the past buckets. - void onDumpReport(const uint64_t dumpTimeNs, StatsLogReport* report) { - std::lock_guard lock(mMutex); - return onDumpReportLocked(dumpTimeNs, report); - } void dumpStates(FILE* out, bool verbose) const { std::lock_guard lock(mMutex); @@ -150,7 +146,6 @@ protected: virtual void onSlicedConditionMayChangeLocked(const uint64_t eventTime) = 0; virtual void onDumpReportLocked(const uint64_t dumpTimeNs, android::util::ProtoOutputStream* protoOutput) = 0; - virtual void onDumpReportLocked(const uint64_t dumpTimeNs, StatsLogReport* report) = 0; virtual size_t byteSizeLocked() const = 0; virtual void dumpStatesLocked(FILE* out, bool verbose) const = 0; @@ -203,10 +198,10 @@ protected: int mConditionTrackerIndex; - FieldMatcher mDimensionsInWhat; // The dimensions_in_what defined in statsd_config - FieldMatcher mDimensionsInCondition; // The dimensions_in_condition defined in statsd_config + vector mDimensionsInWhat; // The dimensions_in_what defined in statsd_config + vector mDimensionsInCondition; // The dimensions_in_condition defined in statsd_config - std::vector mConditionLinks; + std::vector mMetric2ConditionLinks; std::vector> mAnomalyTrackers; diff --git a/cmds/statsd/src/metrics/MetricsManager.cpp b/cmds/statsd/src/metrics/MetricsManager.cpp index 6c21b052e8074..dd6735b6faca6 100644 --- a/cmds/statsd/src/metrics/MetricsManager.cpp +++ b/cmds/statsd/src/metrics/MetricsManager.cpp @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#define DEBUG false // STOPSHIP if true +#define DEBUG true // STOPSHIP if true #include "Log.h" #include "MetricsManager.h" #include "statslog.h" @@ -151,14 +151,6 @@ void MetricsManager::onUidMapReceived(const uint64_t& eventTimeNs) { initLogSourceWhiteList(); } -void MetricsManager::onDumpReport(const uint64_t& dumpTimeStampNs, ConfigMetricsReport* report) { - for (const auto& producer : mAllMetricProducers) { - if (mNoReportMetricIds.find(producer->getMetricId()) == mNoReportMetricIds.end()) { - producer->onDumpReport(dumpTimeStampNs, report->add_metrics()); - } - } -} - void MetricsManager::dumpStates(FILE* out, bool verbose) { fprintf(out, "ConfigKey %s, allowed source:", mConfigKey.ToString().c_str()); { @@ -173,9 +165,8 @@ void MetricsManager::dumpStates(FILE* out, bool verbose) { } } -void MetricsManager::onDumpReport(ProtoOutputStream* protoOutput) { +void MetricsManager::onDumpReport(const uint64_t dumpTimeStampNs, ProtoOutputStream* protoOutput) { VLOG("=========================Metric Reports Start=========================="); - uint64_t dumpTimeStampNs = time(nullptr) * NS_PER_SEC; // one StatsLogReport per MetricProduer for (const auto& producer : mAllMetricProducers) { if (mNoReportMetricIds.find(producer->getMetricId()) == mNoReportMetricIds.end()) { diff --git a/cmds/statsd/src/metrics/MetricsManager.h b/cmds/statsd/src/metrics/MetricsManager.h index 2b30f44d345af..d4f844fe386c2 100644 --- a/cmds/statsd/src/metrics/MetricsManager.h +++ b/cmds/statsd/src/metrics/MetricsManager.h @@ -70,8 +70,8 @@ public: }; // Config source owner can call onDumpReport() to get all the metrics collected. - virtual void onDumpReport(android::util::ProtoOutputStream* protoOutput); - virtual void onDumpReport(const uint64_t& dumpTimeStampNs, ConfigMetricsReport* report); + virtual void onDumpReport(const uint64_t dumpTimeNs, + android::util::ProtoOutputStream* protoOutput); // Computes the total byte size of all metrics managed by a single config source. // Does not change the state. diff --git a/cmds/statsd/src/metrics/ValueMetricProducer.cpp b/cmds/statsd/src/metrics/ValueMetricProducer.cpp index 7b1944c595e5a..45b4ac0cd43ad 100644 --- a/cmds/statsd/src/metrics/ValueMetricProducer.cpp +++ b/cmds/statsd/src/metrics/ValueMetricProducer.cpp @@ -17,7 +17,6 @@ #define DEBUG false // STOPSHIP if true #include "Log.h" -#include "dimension.h" #include "ValueMetricProducer.h" #include "guardrail/StatsdStats.h" #include "stats_log_util.h" @@ -79,15 +78,28 @@ ValueMetricProducer::ValueMetricProducer(const ConfigKey& key, const ValueMetric } mBucketSizeNs = bucketSizeMills * 1000000; - mDimensionsInWhat = metric.dimensions_in_what(); - mDimensionsInCondition = metric.dimensions_in_condition(); + if (metric.has_dimensions_in_what()) { + translateFieldMatcher(metric.dimensions_in_what(), &mDimensionsInWhat); + } + + if (metric.has_dimensions_in_condition()) { + translateFieldMatcher(metric.dimensions_in_condition(), &mDimensionsInCondition); + } if (metric.links().size() > 0) { - mConditionLinks.insert(mConditionLinks.begin(), metric.links().begin(), - metric.links().end()); + for (const auto& link : metric.links()) { + Metric2Condition mc; + mc.conditionId = link.condition(); + translateFieldMatcher(link.fields_in_what(), &mc.metricFields); + translateFieldMatcher(link.fields_in_condition(), &mc.conditionFields); + mMetric2ConditionLinks.push_back(mc); + } } - mConditionSliced = (metric.links().size() > 0)|| - (mDimensionsInCondition.has_field() && mDimensionsInCondition.child_size() > 0); + + if (mValueField.child_size()) { + mField = mValueField.child(0).field(); + } + mConditionSliced = (metric.links().size() > 0) || (mDimensionsInCondition.size() > 0); if (!metric.has_condition() && mPullTagId != -1) { VLOG("Setting up periodic pulling for %d", mPullTagId); @@ -117,25 +129,6 @@ void ValueMetricProducer::onSlicedConditionMayChangeLocked(const uint64_t eventT VLOG("Metric %lld onSlicedConditionMayChange", (long long)mMetricId); } -void ValueMetricProducer::onDumpReportLocked(const uint64_t dumpTimeNs, StatsLogReport* report) { - flushIfNeededLocked(dumpTimeNs); - report->set_metric_id(mMetricId); - auto value_metrics = report->mutable_value_metrics(); - for (const auto& pair : mPastBuckets) { - ValueMetricData* metricData = value_metrics->add_data(); - *metricData->mutable_dimensions_in_what() = - pair.first.getDimensionKeyInWhat().getDimensionsValue(); - *metricData->mutable_dimensions_in_condition() = - pair.first.getDimensionKeyInCondition().getDimensionsValue(); - for (const auto& bucket : pair.second) { - ValueBucketInfo* bucketInfo = metricData->add_bucket_info(); - bucketInfo->set_start_bucket_nanos(bucket.mBucketStartNs); - bucketInfo->set_end_bucket_nanos(bucket.mBucketEndNs); - bucketInfo->set_value(bucket.mValue); - } - } -} - void ValueMetricProducer::onDumpReportLocked(const uint64_t dumpTimeNs, ProtoOutputStream* protoOutput) { VLOG("metric %lld dump report now...", (long long)mMetricId); @@ -155,14 +148,12 @@ void ValueMetricProducer::onDumpReportLocked(const uint64_t dumpTimeNs, // First fill dimension. long long dimensionToken = protoOutput->start( FIELD_TYPE_MESSAGE | FIELD_ID_DIMENSION_IN_WHAT); - writeDimensionsValueProtoToStream( - dimensionKey.getDimensionKeyInWhat().getDimensionsValue(), protoOutput); + writeDimensionToProto(dimensionKey.getDimensionKeyInWhat(), protoOutput); protoOutput->end(dimensionToken); if (dimensionKey.hasDimensionKeyInCondition()) { long long dimensionInConditionToken = protoOutput->start( FIELD_TYPE_MESSAGE | FIELD_ID_DIMENSION_IN_CONDITION); - writeDimensionsValueProtoToStream( - dimensionKey.getDimensionKeyInCondition().getDimensionsValue(), protoOutput); + writeDimensionToProto(dimensionKey.getDimensionKeyInCondition(), protoOutput); protoOutput->end(dimensionInConditionToken); } @@ -284,11 +275,11 @@ void ValueMetricProducer::onMatchedLogEventInternalLocked( } Interval& interval = mCurrentSlicedBucket[eventKey]; - std::shared_ptr valueFieldMap = getValueFields(event); - if (valueFieldMap->empty() || valueFieldMap->size() > 1) { + int error = 0; + const long value = event.GetLong(mField, &error); + if (error < 0) { return; } - const long value = getLongFromDimenValue(valueFieldMap->begin()->second); if (mPullTagId != -1) { // for pulled events if (mCondition == true) { @@ -324,13 +315,6 @@ void ValueMetricProducer::onMatchedLogEventInternalLocked( } } -std::shared_ptr ValueMetricProducer::getValueFields(const LogEvent& event) { - std::shared_ptr valueFields = - std::make_shared(event.getFieldValueMap()); - filterFields(mValueField, valueFields.get()); - return valueFields; -} - void ValueMetricProducer::flushIfNeededLocked(const uint64_t& eventTimeNs) { uint64_t currentBucketEndTimeNs = getCurrentBucketEndTimeNs(); diff --git a/cmds/statsd/src/metrics/ValueMetricProducer.h b/cmds/statsd/src/metrics/ValueMetricProducer.h index bf5b7dfa3ca66..6701a46acde1a 100644 --- a/cmds/statsd/src/metrics/ValueMetricProducer.h +++ b/cmds/statsd/src/metrics/ValueMetricProducer.h @@ -89,7 +89,6 @@ protected: private: void onDumpReportLocked(const uint64_t dumpTimeNs, android::util::ProtoOutputStream* protoOutput) override; - void onDumpReportLocked(const uint64_t dumpTimeNs, StatsLogReport* report) override; // Internal interface to handle condition change. void onConditionChangedLocked(const bool conditionMet, const uint64_t eventTime) override; @@ -120,6 +119,8 @@ private: // tagId for pulled data. -1 if this is not pulled const int mPullTagId; + int mField; + // internal state of a bucket. typedef struct { // Pulled data always come in pair of . This holds the value @@ -142,8 +143,6 @@ private: // TODO: Add a lock to mPastBuckets. std::unordered_map> mPastBuckets; - std::shared_ptr getValueFields(const LogEvent& event); - // Util function to check whether the specified dimension hits the guardrail. bool hitGuardRailLocked(const MetricDimensionKey& newKey); diff --git a/cmds/statsd/src/metrics/duration_helper/DurationTracker.h b/cmds/statsd/src/metrics/duration_helper/DurationTracker.h index 356a81c3e88dd..8f236fa8bab62 100644 --- a/cmds/statsd/src/metrics/duration_helper/DurationTracker.h +++ b/cmds/statsd/src/metrics/duration_helper/DurationTracker.h @@ -62,7 +62,7 @@ class DurationTracker { public: DurationTracker(const ConfigKey& key, const int64_t& id, const MetricDimensionKey& eventKey, sp wizard, int conditionIndex, - const FieldMatcher& dimensionInCondition, bool nesting, + const std::vector& dimensionInCondition, bool nesting, uint64_t currentBucketStartNs, uint64_t currentBucketNum, uint64_t startTimeNs, uint64_t bucketSizeNs, bool conditionSliced, const std::vector>& anomalyTrackers) @@ -182,7 +182,7 @@ protected: const int64_t mBucketSizeNs; - const FieldMatcher mDimensionInCondition; + const std::vector& mDimensionInCondition; const bool mNested; diff --git a/cmds/statsd/src/metrics/duration_helper/MaxDurationTracker.cpp b/cmds/statsd/src/metrics/duration_helper/MaxDurationTracker.cpp index c3bafc67adbf7..c29876b5eae0b 100644 --- a/cmds/statsd/src/metrics/duration_helper/MaxDurationTracker.cpp +++ b/cmds/statsd/src/metrics/duration_helper/MaxDurationTracker.cpp @@ -27,7 +27,7 @@ namespace statsd { MaxDurationTracker::MaxDurationTracker(const ConfigKey& key, const int64_t& id, const MetricDimensionKey& eventKey, sp wizard, int conditionIndex, - const FieldMatcher& dimensionInCondition, bool nesting, + const vector& dimensionInCondition, bool nesting, uint64_t currentBucketStartNs, uint64_t currentBucketNum, uint64_t startTimeNs, uint64_t bucketSizeNs, bool conditionSliced, @@ -55,9 +55,7 @@ bool MaxDurationTracker::hitGuardRail(const HashableDimensionKey& newKey) { // 1. Report the tuple count if the tuple count > soft limit if (mInfos.size() > StatsdStats::kDimensionKeySizeSoftLimit - 1) { size_t newTupleCount = mInfos.size() + 1; - StatsdStats::getInstance().noteMetricDimensionSize( - mConfigKey, hashMetricDimensionKey(mTrackerId, mEventKey), - newTupleCount); + StatsdStats::getInstance().noteMetricDimensionSize(mConfigKey, mTrackerId, newTupleCount); // 2. Don't add more tuples, we are above the allowed threshold. Drop the data. if (newTupleCount > StatsdStats::kDimensionKeySizeHardLimit) { ALOGE("MaxDurTracker %lld dropping data for dimension key %s", @@ -229,10 +227,11 @@ void MaxDurationTracker::onSlicedConditionMayChange(const uint64_t timestamp) { ConditionState conditionState = mWizard->query( mConditionTrackerIndex, pair.second.conditionKeys, mDimensionInCondition, &conditionDimensionKeySet); - bool conditionMet = (conditionState == ConditionState::kTrue) && - (!mDimensionInCondition.has_field() || - conditionDimensionKeySet.find(mEventKey.getDimensionKeyInCondition()) != - conditionDimensionKeySet.end()); + bool conditionMet = + (conditionState == ConditionState::kTrue) && + (mDimensionInCondition.size() == 0 || + conditionDimensionKeySet.find(mEventKey.getDimensionKeyInCondition()) != + conditionDimensionKeySet.end()); VLOG("key: %s, condition: %d", pair.first.c_str(), conditionMet); noteConditionChanged(pair.first, conditionMet, timestamp); } diff --git a/cmds/statsd/src/metrics/duration_helper/MaxDurationTracker.h b/cmds/statsd/src/metrics/duration_helper/MaxDurationTracker.h index fba4119656bd0..95863b60a192c 100644 --- a/cmds/statsd/src/metrics/duration_helper/MaxDurationTracker.h +++ b/cmds/statsd/src/metrics/duration_helper/MaxDurationTracker.h @@ -30,7 +30,7 @@ class MaxDurationTracker : public DurationTracker { public: MaxDurationTracker(const ConfigKey& key, const int64_t& id, const MetricDimensionKey& eventKey, sp wizard, int conditionIndex, - const FieldMatcher& dimensionInCondition, bool nesting, + const std::vector& dimensionInCondition, bool nesting, uint64_t currentBucketStartNs, uint64_t currentBucketNum, uint64_t startTimeNs, uint64_t bucketSizeNs, bool conditionSliced, const std::vector>& anomalyTrackers); diff --git a/cmds/statsd/src/metrics/duration_helper/OringDurationTracker.cpp b/cmds/statsd/src/metrics/duration_helper/OringDurationTracker.cpp index 85f7b7c4afc19..f583f91608b8c 100644 --- a/cmds/statsd/src/metrics/duration_helper/OringDurationTracker.cpp +++ b/cmds/statsd/src/metrics/duration_helper/OringDurationTracker.cpp @@ -26,7 +26,7 @@ using std::pair; OringDurationTracker::OringDurationTracker( const ConfigKey& key, const int64_t& id, const MetricDimensionKey& eventKey, - sp wizard, int conditionIndex, const FieldMatcher& dimensionInCondition, + sp wizard, int conditionIndex, const vector& dimensionInCondition, bool nesting, uint64_t currentBucketStartNs, uint64_t currentBucketNum, uint64_t startTimeNs, uint64_t bucketSizeNs, bool conditionSliced, const vector>& anomalyTrackers) @@ -53,9 +53,7 @@ bool OringDurationTracker::hitGuardRail(const HashableDimensionKey& newKey) { } if (mConditionKeyMap.size() > StatsdStats::kDimensionKeySizeSoftLimit - 1) { size_t newTupleCount = mConditionKeyMap.size() + 1; - StatsdStats::getInstance().noteMetricDimensionSize( - mConfigKey, hashMetricDimensionKey(mTrackerId, mEventKey), - newTupleCount); + StatsdStats::getInstance().noteMetricDimensionSize(mConfigKey, mTrackerId, newTupleCount); // 2. Don't add more tuples, we are above the allowed threshold. Drop the data. if (newTupleCount > StatsdStats::kDimensionKeySizeHardLimit) { ALOGE("OringDurTracker %lld dropping data for dimension key %s", @@ -229,9 +227,9 @@ void OringDurationTracker::onSlicedConditionMayChange(const uint64_t timestamp) mWizard->query(mConditionTrackerIndex, mConditionKeyMap[key], mDimensionInCondition, &conditionDimensionKeySet); if (conditionState != ConditionState::kTrue || - (mDimensionInCondition.has_field() && + (mDimensionInCondition.size() != 0 && conditionDimensionKeySet.find(mEventKey.getDimensionKeyInCondition()) == - conditionDimensionKeySet.end())) { + conditionDimensionKeySet.end())) { startedToPaused.push_back(*it); it = mStarted.erase(it); VLOG("Key %s started -> paused", key.c_str()); @@ -261,9 +259,9 @@ void OringDurationTracker::onSlicedConditionMayChange(const uint64_t timestamp) mWizard->query(mConditionTrackerIndex, mConditionKeyMap[key], mDimensionInCondition, &conditionDimensionKeySet); if (conditionState == ConditionState::kTrue && - (!mDimensionInCondition.has_field() || - conditionDimensionKeySet.find(mEventKey.getDimensionKeyInCondition()) - != conditionDimensionKeySet.end())) { + (mDimensionInCondition.size() == 0 || + conditionDimensionKeySet.find(mEventKey.getDimensionKeyInCondition()) != + conditionDimensionKeySet.end())) { pausedToStarted.push_back(*it); it = mPaused.erase(it); VLOG("Key %s paused -> started", key.c_str()); diff --git a/cmds/statsd/src/metrics/duration_helper/OringDurationTracker.h b/cmds/statsd/src/metrics/duration_helper/OringDurationTracker.h index 73e50e0569cbf..07c13294fa449 100644 --- a/cmds/statsd/src/metrics/duration_helper/OringDurationTracker.h +++ b/cmds/statsd/src/metrics/duration_helper/OringDurationTracker.h @@ -29,8 +29,8 @@ class OringDurationTracker : public DurationTracker { public: OringDurationTracker(const ConfigKey& key, const int64_t& id, const MetricDimensionKey& eventKey, sp wizard, - int conditionIndex, const FieldMatcher& dimensionInCondition, bool nesting, - uint64_t currentBucketStartNs, uint64_t currentBucketNum, + int conditionIndex, const std::vector& dimensionInCondition, + bool nesting, uint64_t currentBucketStartNs, uint64_t currentBucketNum, uint64_t startTimeNs, uint64_t bucketSizeNs, bool conditionSliced, const std::vector>& anomalyTrackers); diff --git a/cmds/statsd/src/metrics/metrics_manager_util.cpp b/cmds/statsd/src/metrics/metrics_manager_util.cpp index 205c8e4b22cd9..769f46dbd7e65 100644 --- a/cmds/statsd/src/metrics/metrics_manager_util.cpp +++ b/cmds/statsd/src/metrics/metrics_manager_util.cpp @@ -38,6 +38,22 @@ namespace android { namespace os { namespace statsd { +namespace { + +bool hasLeafNode(const FieldMatcher& matcher) { + if (!matcher.has_field()) { + return false; + } + for (int i = 0; i < matcher.child_size(); ++i) { + if (hasLeafNode(matcher.child(i))) { + return true; + } + } + return true; +} + +} // namespace + bool handleMetricWithLogTrackers(const int64_t what, const int metricIndex, const bool usedForDimension, const vector>& allAtomMatchers, diff --git a/cmds/statsd/src/packages/UidMap.cpp b/cmds/statsd/src/packages/UidMap.cpp index 0d7b722c40bc1..691423e054b2d 100644 --- a/cmds/statsd/src/packages/UidMap.cpp +++ b/cmds/statsd/src/packages/UidMap.cpp @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#define DEBUG false // STOPSHIP if true +#define DEBUG true // STOPSHIP if true #include "Log.h" #include "guardrail/StatsdStats.h" diff --git a/cmds/statsd/src/stats_log_util.cpp b/cmds/statsd/src/stats_log_util.cpp index 6c6140081bd81..86c258bd3dfd1 100644 --- a/cmds/statsd/src/stats_log_util.cpp +++ b/cmds/statsd/src/stats_log_util.cpp @@ -16,9 +16,10 @@ #include "stats_log_util.h" +#include +#include #include #include -#include using android::util::FIELD_COUNT_REPEATED; using android::util::FIELD_TYPE_BOOL; @@ -42,6 +43,8 @@ const int DIMENSIONS_VALUE_VALUE_BOOL = 5; const int DIMENSIONS_VALUE_VALUE_FLOAT = 6; const int DIMENSIONS_VALUE_VALUE_TUPLE = 7; +const int DIMENSIONS_VALUE_TUPLE_VALUE = 1; + // for MessageValue Proto const int FIELD_ID_FIELD_VALUE_IN_MESSAGE_VALUE_PROTO = 1; @@ -51,52 +54,79 @@ const int FIELD_ID_PULL_ATOM_ID = 1; const int FIELD_ID_TOTAL_PULL = 2; const int FIELD_ID_TOTAL_PULL_FROM_CACHE = 3; const int FIELD_ID_MIN_PULL_INTERVAL_SEC = 4; +namespace { -void writeDimensionsValueProtoToStream(const DimensionsValue& dimensionsValue, - ProtoOutputStream* protoOutput) { - if (!dimensionsValue.has_field()) { +void writeDimensionToProtoHelper(const std::vector& dims, size_t* index, int depth, + int prefix, ProtoOutputStream* protoOutput) { + size_t count = dims.size(); + while (*index < count) { + const auto& dim = dims[*index]; + const int valueDepth = dim.mField.getDepth(); + const int valuePrefix = dim.mField.getPrefix(depth); + const int fieldNum = dim.mField.getPosAtDepth(depth); + if (valueDepth > 2) { + ALOGE("Depth > 2 not supported"); + return; + } + + if (depth == valueDepth && valuePrefix == prefix) { + long long token = protoOutput->start(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | + DIMENSIONS_VALUE_TUPLE_VALUE); + protoOutput->write(FIELD_TYPE_INT32 | DIMENSIONS_VALUE_FIELD, fieldNum); + switch (dim.mValue.getType()) { + case INT: + protoOutput->write(FIELD_TYPE_INT32 | DIMENSIONS_VALUE_VALUE_INT, + dim.mValue.int_value); + break; + case LONG: + protoOutput->write(FIELD_TYPE_INT64 | DIMENSIONS_VALUE_VALUE_LONG, + (long long)dim.mValue.long_value); + break; + case FLOAT: + protoOutput->write(FIELD_TYPE_FLOAT | DIMENSIONS_VALUE_VALUE_FLOAT, + dim.mValue.float_value); + break; + case STRING: + protoOutput->write(FIELD_TYPE_STRING | DIMENSIONS_VALUE_VALUE_STR, + dim.mValue.str_value); + break; + default: + break; + } + if (token != 0) { + protoOutput->end(token); + } + (*index)++; + } else if (valueDepth > depth && valuePrefix == prefix) { + // Writing the sub tree + long long dimensionToken = protoOutput->start( + FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | DIMENSIONS_VALUE_TUPLE_VALUE); + protoOutput->write(FIELD_TYPE_INT32 | DIMENSIONS_VALUE_FIELD, fieldNum); + long long tupleToken = + protoOutput->start(FIELD_TYPE_MESSAGE | DIMENSIONS_VALUE_VALUE_TUPLE); + writeDimensionToProtoHelper(dims, index, valueDepth, dim.mField.getPrefix(valueDepth), + protoOutput); + protoOutput->end(tupleToken); + protoOutput->end(dimensionToken); + } else { + // Done with the prev sub tree + return; + } + } +} + +} // namespace + +void writeDimensionToProto(const HashableDimensionKey& dimension, ProtoOutputStream* protoOutput) { + if (dimension.getValues().size() == 0) { return; } - protoOutput->write(FIELD_TYPE_INT32 | DIMENSIONS_VALUE_FIELD, dimensionsValue.field()); - switch (dimensionsValue.value_case()) { - case DimensionsValue::ValueCase::kValueStr: - protoOutput->write(FIELD_TYPE_STRING | DIMENSIONS_VALUE_VALUE_STR, - dimensionsValue.value_str()); - break; - case DimensionsValue::ValueCase::kValueInt: - protoOutput->write(FIELD_TYPE_INT32 | DIMENSIONS_VALUE_VALUE_INT, - dimensionsValue.value_int()); - break; - case DimensionsValue::ValueCase::kValueLong: - protoOutput->write(FIELD_TYPE_INT64 | DIMENSIONS_VALUE_VALUE_LONG, - dimensionsValue.value_long()); - break; - case DimensionsValue::ValueCase::kValueBool: - protoOutput->write(FIELD_TYPE_BOOL | DIMENSIONS_VALUE_VALUE_BOOL, - dimensionsValue.value_bool()); - break; - case DimensionsValue::ValueCase::kValueFloat: - protoOutput->write(FIELD_TYPE_FLOAT | DIMENSIONS_VALUE_VALUE_FLOAT, - dimensionsValue.value_float()); - break; - case DimensionsValue::ValueCase::kValueTuple: - { - long long tupleToken = protoOutput->start( - FIELD_TYPE_MESSAGE | DIMENSIONS_VALUE_VALUE_TUPLE); - for (int i = 0; i < dimensionsValue.value_tuple().dimensions_value_size(); ++i) { - long long token = protoOutput->start( - FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED - | FIELD_ID_FIELD_VALUE_IN_MESSAGE_VALUE_PROTO); - writeDimensionsValueProtoToStream( - dimensionsValue.value_tuple().dimensions_value(i), protoOutput); - protoOutput->end(token); - } - protoOutput->end(tupleToken); - } - break; - default: - break; - } + protoOutput->write(FIELD_TYPE_INT32 | DIMENSIONS_VALUE_FIELD, + dimension.getValues()[0].mField.getTag()); + long long topToken = protoOutput->start(FIELD_TYPE_MESSAGE | DIMENSIONS_VALUE_VALUE_TUPLE); + size_t index = 0; + writeDimensionToProtoHelper(dimension.getValues(), &index, 0, 0, protoOutput); + protoOutput->end(topToken); } // for Field Proto @@ -104,133 +134,92 @@ const int FIELD_FIELD = 1; const int FIELD_POSITION_INDEX = 2; const int FIELD_CHILD = 3; -void writeFieldProtoToStream( - const Field& field, util::ProtoOutputStream* protoOutput) { - if (!field.has_field()) { - return; - } - protoOutput->write(FIELD_TYPE_INT32 | FIELD_FIELD, field.field()); - if (field.has_position_index()) { - protoOutput->write(FIELD_TYPE_INT32 | FIELD_POSITION_INDEX, field.position_index()); - } - for (int i = 0; i < field.child_size(); ++i) { - long long childToken = protoOutput->start( - FIELD_TYPE_MESSAGE| FIELD_COUNT_REPEATED | FIELD_CHILD); - writeFieldProtoToStream(field.child(i), protoOutput); - protoOutput->end(childToken); - } -} - -namespace { - -void addOrUpdateChildrenMap( - const Field& root, - const Field& node, - std::map, FieldCmp> *childrenMap) { - Field parentNode = root; - if (node.has_position_index()) { - appendLeaf(&parentNode, node.field(), node.position_index()); - } else { - appendLeaf(&parentNode, node.field()); - } - if (childrenMap->find(parentNode) == childrenMap->end()) { - childrenMap->insert(std::make_pair(parentNode, std::set{})); - } - auto it = childrenMap->find(parentNode); - for (int i = 0; i < node.child_size(); ++i) { - auto child = node.child(i); - Field childNode = parentNode; - if (child.has_position_index()) { - appendLeaf(&childNode, child.field(), child.position_index()); - } else { - appendLeaf(&childNode, child.field()); +// Supported Atoms format +// XYZ_Atom { +// repeated SubMsg field_1 = 1; +// SubMsg2 field_2 = 2; +// int32/float/string/int63 field_3 = 3; +// } +// logd's msg format, doesn't allow us to distinguish between the 2 cases below +// Case (1): +// Atom { +// SubMsg { +// int i = 1; +// int j = 2; +// } +// repeated SubMsg +// } +// +// and case (2): +// Atom { +// SubMsg { +// repeated int i = 1; +// repeated int j = 2; +// } +// optional SubMsg = 1; +// } +// +// +void writeFieldValueTreeToStreamHelper(const std::vector& dims, size_t* index, + int depth, int prefix, ProtoOutputStream* protoOutput) { + size_t count = dims.size(); + while (*index < count) { + const auto& dim = dims[*index]; + const int valueDepth = dim.mField.getDepth(); + const int valuePrefix = dim.mField.getPrefix(depth); + const int fieldNum = dim.mField.getPosAtDepth(depth); + if (valueDepth > 2) { + ALOGE("Depth > 2 not supported"); + return; + } + + if (depth == valueDepth && valuePrefix == prefix) { + switch (dim.mValue.getType()) { + case INT: + protoOutput->write(FIELD_TYPE_INT32 | fieldNum, dim.mValue.int_value); + break; + case LONG: + protoOutput->write(FIELD_TYPE_INT64 | fieldNum, + (long long)dim.mValue.long_value); + break; + case FLOAT: + protoOutput->write(FIELD_TYPE_FLOAT | fieldNum, dim.mValue.float_value); + break; + case STRING: + protoOutput->write(FIELD_TYPE_STRING | fieldNum, dim.mValue.str_value); + break; + } + (*index)++; + } else if (valueDepth > depth && valuePrefix == prefix) { + // Writing the sub tree + long long msg_token = 0; + if (valueDepth == depth + 2) { + msg_token = + protoOutput->start(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | fieldNum); + } else if (valueDepth == depth + 1) { + msg_token = protoOutput->start(FIELD_TYPE_MESSAGE | fieldNum); + } + // Directly jump to the leaf value because the repeated position field is implied + // by the position of the sub msg in the parent field. + writeFieldValueTreeToStreamHelper(dims, index, valueDepth, + dim.mField.getPrefix(valueDepth), protoOutput); + if (msg_token != 0) { + protoOutput->end(msg_token); + } + } else { + // Done with the prev sub tree + return; } - it->second.insert(childNode); - addOrUpdateChildrenMap(parentNode, child, childrenMap); } } -void addOrUpdateChildrenMap( - const Field& field, - std::map, FieldCmp> *childrenMap) { - Field root; - addOrUpdateChildrenMap(root, field, childrenMap); -} - -} // namespace - -void writeFieldValueTreeToStream(const FieldValueMap &fieldValueMap, +void writeFieldValueTreeToStream(int tagId, const std::vector& values, util::ProtoOutputStream* protoOutput) { - std::map, FieldCmp> childrenMap; - // Rebuild the field tree. - for (auto it = fieldValueMap.begin(); it != fieldValueMap.end(); ++it) { - addOrUpdateChildrenMap(it->first, &childrenMap); - } - std::stack> tokenStack; - // Iterate over the node tree to fill the Atom proto. - for (auto it = childrenMap.begin(); it != childrenMap.end(); ++it) { - const Field* nodeLeaf = getSingleLeaf(&it->first); - const int fieldNum = nodeLeaf->field(); - while (!tokenStack.empty()) { - auto currentMsgNode = tokenStack.top().second; - auto currentMsgNodeChildrenIt = childrenMap.find(currentMsgNode); - if (currentMsgNodeChildrenIt->second.find(it->first) == - currentMsgNodeChildrenIt->second.end()) { - protoOutput->end(tokenStack.top().first); - tokenStack.pop(); - } else { - break; - } - } - if (it->second.size() == 0) { - auto itValue = fieldValueMap.find(it->first); - if (itValue != fieldValueMap.end()) { - const DimensionsValue& value = itValue->second; - switch (value.value_case()) { - case DimensionsValue::ValueCase::kValueStr: - protoOutput->write(FIELD_TYPE_STRING | fieldNum, - value.value_str()); - break; - case DimensionsValue::ValueCase::kValueInt: - protoOutput->write(FIELD_TYPE_INT32 | fieldNum, - value.value_int()); - break; - case DimensionsValue::ValueCase::kValueLong: - protoOutput->write(FIELD_TYPE_INT64 | fieldNum, - value.value_long()); - break; - case DimensionsValue::ValueCase::kValueBool: - protoOutput->write(FIELD_TYPE_BOOL | fieldNum, - value.value_bool()); - break; - case DimensionsValue::ValueCase::kValueFloat: - protoOutput->write(FIELD_TYPE_FLOAT | fieldNum, - value.value_float()); - break; - // This would not happen as the node has no child. - case DimensionsValue::ValueCase::kValueTuple: - break; - default: - break; - } - } else { - ALOGE("Leaf node value not found. This should never happen."); - } - } else { - long long token; - if (nodeLeaf->has_position_index()) { - token = protoOutput->start(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | fieldNum); - } else { - token = protoOutput->start(FIELD_TYPE_MESSAGE | fieldNum); - } - tokenStack.push(std::make_pair(token, it->first)); - } - } + long long atomToken = protoOutput->start(FIELD_TYPE_MESSAGE | tagId); - while (!tokenStack.empty()) { - protoOutput->end(tokenStack.top().first); - tokenStack.pop(); - } + size_t index = 0; + writeFieldValueTreeToStreamHelper(values, &index, 0, 0, protoOutput); + protoOutput->end(atomToken); } int64_t TimeUnitToBucketSizeInMillis(TimeUnit unit) { diff --git a/cmds/statsd/src/stats_log_util.h b/cmds/statsd/src/stats_log_util.h index cee920038eefc..6583f579648b0 100644 --- a/cmds/statsd/src/stats_log_util.h +++ b/cmds/statsd/src/stats_log_util.h @@ -17,7 +17,8 @@ #pragma once #include -#include "field_util.h" +#include "FieldValue.h" +#include "HashableDimensionKey.h" #include "frameworks/base/cmds/statsd/src/stats_log.pb.h" #include "frameworks/base/cmds/statsd/src/statsd_config.pb.h" #include "guardrail/StatsdStats.h" @@ -26,16 +27,10 @@ namespace android { namespace os { namespace statsd { -// Helper function to write DimensionsValue proto to ProtoOutputStream. -void writeDimensionsValueProtoToStream(const DimensionsValue& fieldValue, - util::ProtoOutputStream* protoOutput); - -// Helper function to write Field proto to ProtoOutputStream. -void writeFieldProtoToStream(const Field& field, util::ProtoOutputStream* protoOutput); - -// Helper function to construct the field value tree and write to ProtoOutputStream -void writeFieldValueTreeToStream(const FieldValueMap& fieldValueMap, +void writeFieldValueTreeToStream(int tagId, const std::vector& values, util::ProtoOutputStream* protoOutput); +void writeDimensionToProto(const HashableDimensionKey& dimension, + util::ProtoOutputStream* protoOutput); // Convert the TimeUnit enum to the bucket size in millis. int64_t TimeUnitToBucketSizeInMillis(TimeUnit unit); diff --git a/cmds/statsd/src/statsd_internal.proto b/cmds/statsd/src/statsd_internal.proto deleted file mode 100644 index 25aacee2770a1..0000000000000 --- a/cmds/statsd/src/statsd_internal.proto +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (C) 2017 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -syntax = "proto2"; -option optimize_for = LITE_RUNTIME; - -package android.os.statsd; - -option java_package = "com.android.os"; -option java_outer_classname = "StatsdInternalProto"; - -message Field { - optional int32 field = 1; - optional int32 position_index = 2 [default = -1]; - repeated Field child = 3; -} diff --git a/cmds/statsd/src/subscriber/SubscriberReporter.cpp b/cmds/statsd/src/subscriber/SubscriberReporter.cpp index 3af684fa60697..9f68fc4c20be1 100644 --- a/cmds/statsd/src/subscriber/SubscriberReporter.cpp +++ b/cmds/statsd/src/subscriber/SubscriberReporter.cpp @@ -98,50 +98,66 @@ void SubscriberReporter::sendBroadcastLocked(const sp& intentSender, ALOGW("Failed to send subscriber broadcast: could not access StatsCompanionService."); return; } - mStatsCompanionService->sendSubscriberBroadcast(intentSender, - configKey.GetUid(), - configKey.GetId(), - subscription.id(), - subscription.rule_id(), - protoToStatsDimensionsValue(dimKey)); + mStatsCompanionService->sendSubscriberBroadcast( + intentSender, configKey.GetUid(), configKey.GetId(), subscription.id(), + subscription.rule_id(), getStatsDimensionsValue(dimKey.getDimensionKeyInWhat())); } -StatsDimensionsValue SubscriberReporter::protoToStatsDimensionsValue( - const MetricDimensionKey& dimKey) { - return protoToStatsDimensionsValue(dimKey.getDimensionKeyInWhat().getDimensionsValue()); -} - -StatsDimensionsValue SubscriberReporter::protoToStatsDimensionsValue( - const DimensionsValue& protoDimsVal) { - int32_t field = protoDimsVal.field(); - - switch (protoDimsVal.value_case()) { - case DimensionsValue::ValueCase::kValueStr: - return StatsDimensionsValue(field, String16(protoDimsVal.value_str().c_str())); - case DimensionsValue::ValueCase::kValueInt: - return StatsDimensionsValue(field, static_cast(protoDimsVal.value_int())); - case DimensionsValue::ValueCase::kValueLong: - return StatsDimensionsValue(field, static_cast(protoDimsVal.value_long())); - case DimensionsValue::ValueCase::kValueBool: - return StatsDimensionsValue(field, static_cast(protoDimsVal.value_bool())); - case DimensionsValue::ValueCase::kValueFloat: - return StatsDimensionsValue(field, static_cast(protoDimsVal.value_float())); - case DimensionsValue::ValueCase::kValueTuple: - { - int sz = protoDimsVal.value_tuple().dimensions_value_size(); - std::vector sdvVec(sz); - for (int i = 0; i < sz; i++) { - sdvVec[i] = protoToStatsDimensionsValue( - protoDimsVal.value_tuple().dimensions_value(i)); - } - return StatsDimensionsValue(field, sdvVec); +void getStatsDimensionsValueHelper(const std::vector& dims, size_t* index, int depth, + int prefix, vector* output) { + size_t count = dims.size(); + while (*index < count) { + const auto& dim = dims[*index]; + const int valueDepth = dim.mField.getDepth(); + const int valuePrefix = dim.mField.getPrefix(depth); + if (valueDepth > 2) { + ALOGE("Depth > 2 not supported"); + return; + } + if (depth == valueDepth && valuePrefix == prefix) { + switch (dim.mValue.getType()) { + case INT: + output->push_back(StatsDimensionsValue(dim.mField.getPosAtDepth(depth), + dim.mValue.int_value)); + break; + case LONG: + output->push_back(StatsDimensionsValue(dim.mField.getPosAtDepth(depth), + dim.mValue.long_value)); + break; + case FLOAT: + output->push_back(StatsDimensionsValue(dim.mField.getPosAtDepth(depth), + dim.mValue.float_value)); + break; + case STRING: + output->push_back(StatsDimensionsValue(dim.mField.getPosAtDepth(depth), + String16(dim.mValue.str_value.c_str()))); + break; + default: + break; } - default: - ALOGW("protoToStatsDimensionsValue failed: illegal type."); - return StatsDimensionsValue(); + (*index)++; + } else if (valueDepth > depth && valuePrefix == prefix) { + vector childOutput; + getStatsDimensionsValueHelper(dims, index, depth + 1, dim.mField.getPrefix(depth + 1), + &childOutput); + output->push_back(StatsDimensionsValue(dim.mField.getPosAtDepth(depth), childOutput)); + } else { + return; + } } } +StatsDimensionsValue SubscriberReporter::getStatsDimensionsValue(const HashableDimensionKey& dim) { + if (dim.getValues().size() == 0) { + return StatsDimensionsValue(); + } + + vector fields; + size_t index = 0; + getStatsDimensionsValueHelper(dim.getValues(), &index, 0, 0, &fields); + return StatsDimensionsValue(dim.getValues()[0].mField.getTag(), fields); +} + } // namespace statsd } // namespace os } // namespace android diff --git a/cmds/statsd/src/subscriber/SubscriberReporter.h b/cmds/statsd/src/subscriber/SubscriberReporter.h index 13fc7fd06279c..c7d1a5ba9425b 100644 --- a/cmds/statsd/src/subscriber/SubscriberReporter.h +++ b/cmds/statsd/src/subscriber/SubscriberReporter.h @@ -82,6 +82,8 @@ public: const Subscription& subscription, const MetricDimensionKey& dimKey) const; + static StatsDimensionsValue getStatsDimensionsValue(const HashableDimensionKey& dim); + private: SubscriberReporter() {}; @@ -102,14 +104,6 @@ private: const ConfigKey& configKey, const Subscription& subscription, const MetricDimensionKey& dimKey) const; - - /** Converts a stats_log.proto DimensionsValue to a StatsDimensionsValue. */ - static StatsDimensionsValue protoToStatsDimensionsValue( - const DimensionsValue& protoDimsVal); - - /** Converts a HashableDimensionKey to a StatsDimensionsValue. */ - static StatsDimensionsValue protoToStatsDimensionsValue( - const MetricDimensionKey& dimKey); }; } // namespace statsd diff --git a/cmds/statsd/tests/FieldValue_test.cpp b/cmds/statsd/tests/FieldValue_test.cpp new file mode 100644 index 0000000000000..f1ad0c88b2422 --- /dev/null +++ b/cmds/statsd/tests/FieldValue_test.cpp @@ -0,0 +1,350 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include "src/logd/LogEvent.h" +#include "frameworks/base/cmds/statsd/src/statsd_config.pb.h" +#include "matchers/matcher_util.h" +#include "stats_log_util.h" +#include "stats_util.h" +#include "subscriber/SubscriberReporter.h" + +#ifdef __ANDROID__ + +namespace android { +namespace os { +namespace statsd { + +TEST(AtomMatcherTest, TestFieldTranslation) { + FieldMatcher matcher1; + matcher1.set_field(10); + FieldMatcher* child = matcher1.add_child(); + child->set_field(1); + child->set_position(Position::ANY); + + child = child->add_child(); + child->set_field(1); + + vector output; + translateFieldMatcher(matcher1, &output); + + EXPECT_EQ((size_t)1, output.size()); + + const auto& matcher12 = output[0]; + EXPECT_EQ((int32_t)10, matcher12.mMatcher.getTag()); + EXPECT_EQ((int32_t)0x2010001, matcher12.mMatcher.getField()); + EXPECT_EQ((int32_t)0xff7f007f, matcher12.mMask); +} + +TEST(AtomMatcherTest, TestFilter) { + FieldMatcher matcher1; + matcher1.set_field(10); + FieldMatcher* child = matcher1.add_child(); + child->set_field(1); + child->set_position(Position::ANY); + + child = child->add_child(); + child->set_field(1); + + child = matcher1.add_child(); + child->set_field(2); + + vector matchers; + translateFieldMatcher(matcher1, &matchers); + + AttributionNode attribution_node1; + attribution_node1.set_uid(1111); + attribution_node1.set_tag("location1"); + + AttributionNode attribution_node2; + attribution_node2.set_uid(2222); + attribution_node2.set_tag("location2"); + + AttributionNode attribution_node3; + attribution_node3.set_uid(3333); + attribution_node3.set_tag("location3"); + std::vector attribution_nodes = {attribution_node1, attribution_node2, + attribution_node3}; + + // Set up the event + LogEvent event(10, 12345); + event.write(attribution_nodes); + event.write("some value"); + // Convert to a LogEvent + event.init(); + vector output; + + filterValues(matchers, event.getValues(), &output); + + EXPECT_EQ((size_t)(3), output.size()); + + const auto& key1 = output[0]; + EXPECT_EQ((size_t)2, key1.getValues().size()); + EXPECT_EQ((int32_t)0x02010001, key1.getValues()[0].mField.getField()); + EXPECT_EQ((int32_t)1111, key1.getValues()[0].mValue.int_value); + EXPECT_EQ((int32_t)0x00020000, key1.getValues()[1].mField.getField()); + EXPECT_EQ("some value", key1.getValues()[1].mValue.str_value); + + const auto& key2 = output[1]; + EXPECT_EQ((size_t)2, key2.getValues().size()); + EXPECT_EQ((int32_t)0x02010001, key2.getValues()[0].mField.getField()); + EXPECT_EQ((int32_t)2222, key2.getValues()[0].mValue.int_value); + EXPECT_EQ((int32_t)0x00020000, key2.getValues()[1].mField.getField()); + EXPECT_EQ("some value", key2.getValues()[1].mValue.str_value); + + const auto& key3 = output[2]; + EXPECT_EQ((size_t)2, key3.getValues().size()); + EXPECT_EQ((int32_t)0x02010001, key3.getValues()[0].mField.getField()); + EXPECT_EQ((int32_t)3333, key3.getValues()[0].mValue.int_value); + EXPECT_EQ((int32_t)0x00020000, key3.getValues()[1].mField.getField()); + EXPECT_EQ("some value", key3.getValues()[1].mValue.str_value); +} + +TEST(AtomMatcherTest, TestSubDimension) { + HashableDimensionKey dim; + + int pos1[] = {1, 1, 1}; + int pos2[] = {1, 1, 2}; + int pos3[] = {1, 1, 3}; + int pos4[] = {2, 0, 0}; + Field field1(10, pos1, 2); + Field field2(10, pos2, 2); + + Field field3(10, pos3, 2); + Field field4(10, pos4, 0); + + Value value1((int32_t)10025); + Value value2("tag"); + + Value value11((int32_t)10026); + Value value22("tag2"); + + dim.addValue(FieldValue(field1, value1)); + dim.addValue(FieldValue(field2, value2)); + + HashableDimensionKey subDim1; + subDim1.addValue(FieldValue(field1, value1)); + + HashableDimensionKey subDim2; + subDim1.addValue(FieldValue(field2, value2)); + + EXPECT_TRUE(dim.contains(dim)); + EXPECT_TRUE(dim.contains(subDim1)); + EXPECT_TRUE(dim.contains(subDim2)); + + HashableDimensionKey subDim3; + subDim3.addValue(FieldValue(field1, value11)); + EXPECT_FALSE(dim.contains(subDim3)); + + HashableDimensionKey subDim4; + // Empty dimension is always a sub dimension of other dimensions + EXPECT_TRUE(dim.contains(subDim4)); +} + +TEST(AtomMatcherTest, TestMetric2ConditionLink) { + AttributionNode attribution_node1; + attribution_node1.set_uid(1111); + attribution_node1.set_tag("location1"); + + AttributionNode attribution_node2; + attribution_node2.set_uid(2222); + attribution_node2.set_tag("location2"); + + AttributionNode attribution_node3; + attribution_node3.set_uid(3333); + attribution_node3.set_tag("location3"); + std::vector attribution_nodes = {attribution_node1, attribution_node2, + attribution_node3}; + + // Set up the event + LogEvent event(10, 12345); + event.write(attribution_nodes); + event.write("some value"); + // Convert to a LogEvent + event.init(); + + FieldMatcher whatMatcher; + whatMatcher.set_field(10); + FieldMatcher* child11 = whatMatcher.add_child(); + child11->set_field(1); + child11->set_position(Position::ANY); + child11 = child11->add_child(); + child11->set_field(1); + + FieldMatcher conditionMatcher; + conditionMatcher.set_field(27); + FieldMatcher* child2 = conditionMatcher.add_child(); + child2->set_field(2); + child2->set_position(Position::LAST); + + child2 = child2->add_child(); + child2->set_field(2); + + Metric2Condition link; + + translateFieldMatcher(whatMatcher, &link.metricFields); + translateFieldMatcher(conditionMatcher, &link.conditionFields); + + EXPECT_EQ((size_t)1, link.metricFields.size()); + EXPECT_EQ((int32_t)0x02010001, link.metricFields[0].mMatcher.getField()); + EXPECT_EQ((int32_t)0xff7f007f, link.metricFields[0].mMask); + EXPECT_EQ((int32_t)10, link.metricFields[0].mMatcher.getTag()); + + EXPECT_EQ((size_t)1, link.conditionFields.size()); + EXPECT_EQ((int32_t)0x02028002, link.conditionFields[0].mMatcher.getField()); + EXPECT_EQ((int32_t)0xff7f807f, link.conditionFields[0].mMask); + EXPECT_EQ((int32_t)27, link.conditionFields[0].mMatcher.getTag()); +} + +TEST(AtomMatcherTest, TestSubscriberDimensionWrite) { + HashableDimensionKey dim; + + int pos1[] = {1, 1, 1}; + int pos2[] = {1, 1, 2}; + int pos3[] = {1, 1, 3}; + int pos4[] = {2, 0, 0}; + + Field field1(10, pos1, 2); + Field field2(10, pos2, 2); + Field field3(10, pos3, 2); + Field field4(10, pos4, 0); + + Value value1((int32_t)10025); + Value value2("tag"); + Value value3((int32_t)987654); + Value value4((int32_t)99999); + + dim.addValue(FieldValue(field1, value1)); + dim.addValue(FieldValue(field2, value2)); + dim.addValue(FieldValue(field3, value3)); + dim.addValue(FieldValue(field4, value4)); + + SubscriberReporter::getStatsDimensionsValue(dim); + // TODO: can't test anything here because SubscriberReport class doesn't have any read api. +} + +TEST(AtomMatcherTest, TestWriteDimensionToProto) { + HashableDimensionKey dim; + int pos1[] = {1, 1, 1}; + int pos2[] = {1, 1, 2}; + int pos3[] = {1, 1, 3}; + int pos4[] = {2, 0, 0}; + Field field1(10, pos1, 2); + Field field2(10, pos2, 2); + Field field3(10, pos3, 2); + Field field4(10, pos4, 0); + + Value value1((int32_t)10025); + Value value2("tag"); + Value value3((int32_t)987654); + Value value4((int32_t)99999); + + dim.addValue(FieldValue(field1, value1)); + dim.addValue(FieldValue(field2, value2)); + dim.addValue(FieldValue(field3, value3)); + dim.addValue(FieldValue(field4, value4)); + + android::util::ProtoOutputStream protoOut; + writeDimensionToProto(dim, &protoOut); + + vector outData; + outData.resize(protoOut.size()); + size_t pos = 0; + auto iter = protoOut.data(); + while (iter.readBuffer() != NULL) { + size_t toRead = iter.currentToRead(); + std::memcpy(&(outData[pos]), iter.readBuffer(), toRead); + pos += toRead; + iter.rp()->move(toRead); + } + + DimensionsValue result; + EXPECT_EQ(true, result.ParseFromArray(&outData[0], outData.size())); + EXPECT_EQ(10, result.field()); + EXPECT_EQ(DimensionsValue::ValueCase::kValueTuple, result.value_case()); + EXPECT_EQ(2, result.value_tuple().dimensions_value_size()); + + const auto& dim1 = result.value_tuple().dimensions_value(0); + EXPECT_EQ(DimensionsValue::ValueCase::kValueTuple, dim1.value_case()); + EXPECT_EQ(3, dim1.value_tuple().dimensions_value_size()); + + const auto& dim11 = dim1.value_tuple().dimensions_value(0); + EXPECT_EQ(DimensionsValue::ValueCase::kValueInt, dim11.value_case()); + EXPECT_EQ(10025, dim11.value_int()); + + const auto& dim12 = dim1.value_tuple().dimensions_value(1); + EXPECT_EQ(DimensionsValue::ValueCase::kValueStr, dim12.value_case()); + EXPECT_EQ("tag", dim12.value_str()); + + const auto& dim13 = dim1.value_tuple().dimensions_value(2); + EXPECT_EQ(DimensionsValue::ValueCase::kValueInt, dim13.value_case()); + EXPECT_EQ(987654, dim13.value_int()); + + const auto& dim2 = result.value_tuple().dimensions_value(1); + EXPECT_EQ(DimensionsValue::ValueCase::kValueInt, dim2.value_case()); + EXPECT_EQ(99999, dim2.value_int()); +} + +TEST(AtomMatcherTest, TestWriteAtomToProto) { + AttributionNode attribution_node1; + attribution_node1.set_uid(1111); + attribution_node1.set_tag("location1"); + + AttributionNode attribution_node2; + attribution_node2.set_uid(2222); + attribution_node2.set_tag("location2"); + + std::vector attribution_nodes = {attribution_node1, attribution_node2}; + + // Set up the event + LogEvent event(4, 12345); + event.write(attribution_nodes); + event.write((int32_t)999); + // Convert to a LogEvent + event.init(); + + android::util::ProtoOutputStream protoOutput; + writeFieldValueTreeToStream(event.GetTagId(), event.getValues(), &protoOutput); + + vector outData; + outData.resize(protoOutput.size()); + size_t pos = 0; + auto iter = protoOutput.data(); + while (iter.readBuffer() != NULL) { + size_t toRead = iter.currentToRead(); + std::memcpy(&(outData[pos]), iter.readBuffer(), toRead); + pos += toRead; + iter.rp()->move(toRead); + } + + Atom result; + EXPECT_EQ(true, result.ParseFromArray(&outData[0], outData.size())); + EXPECT_EQ(Atom::PushedCase::kBleScanResultReceived, result.pushed_case()); + const auto& atom = result.ble_scan_result_received(); + EXPECT_EQ(2, atom.attribution_node_size()); + EXPECT_EQ(1111, atom.attribution_node(0).uid()); + EXPECT_EQ("location1", atom.attribution_node(0).tag()); + EXPECT_EQ(2222, atom.attribution_node(1).uid()); + EXPECT_EQ("location2", atom.attribution_node(1).tag()); + EXPECT_EQ(999, atom.num_of_results()); +} + + +} // namespace statsd +} // namespace os +} // namespace android +#else +GTEST_LOG_(INFO) << "This test does nothing.\n"; +#endif \ No newline at end of file diff --git a/cmds/statsd/tests/LogEntryMatcher_test.cpp b/cmds/statsd/tests/LogEntryMatcher_test.cpp index 111b4ba7a32c3..1023ea40aa55b 100644 --- a/cmds/statsd/tests/LogEntryMatcher_test.cpp +++ b/cmds/statsd/tests/LogEntryMatcher_test.cpp @@ -14,6 +14,7 @@ #include "frameworks/base/cmds/statsd/src/statsd_config.pb.h" #include "matchers/matcher_util.h" +#include "stats_log_util.h" #include "stats_util.h" #include @@ -35,8 +36,6 @@ const int FIELD_ID_3 = 2; const int ATTRIBUTION_UID_FIELD_ID = 1; const int ATTRIBUTION_TAG_FIELD_ID = 2; -// Private API from liblog. -extern "C" void android_log_rewind(android_log_context ctx); #ifdef __ANDROID__ TEST(AtomMatcherTest, TestSimpleMatcher) { @@ -597,7 +596,6 @@ TEST(AtomMatcherTest, TestNorMatcher) { matcherResults.push_back(MatchingState::kMatched); EXPECT_FALSE(combinationMatch(children, operation, matcherResults)); } - #else GTEST_LOG_(INFO) << "This test does nothing.\n"; #endif diff --git a/cmds/statsd/tests/LogEvent_test.cpp b/cmds/statsd/tests/LogEvent_test.cpp index fd28460e8e015..b649215012756 100644 --- a/cmds/statsd/tests/LogEvent_test.cpp +++ b/cmds/statsd/tests/LogEvent_test.cpp @@ -22,548 +22,142 @@ namespace android { namespace os { namespace statsd { -TEST(LogEventTest, testEmptyEvent) { - const int32_t TAG_ID = 123; - LogEvent event(TAG_ID, 0); - event.init(); +TEST(LogEventTest, TestLogParsing) { + LogEvent event1(1, 2000); - DimensionsValue dimensionsValue; - EXPECT_FALSE(event.GetSimpleAtomDimensionsValueProto(234, &dimensionsValue)); - FieldMatcher dimensions; - dimensions.set_field(event.GetTagId()); - EXPECT_FALSE(event.GetAtomDimensionsValueProto(dimensions, &dimensionsValue)); + std::vector nodes; - dimensions.add_child()->set_field(3); - dimensions.mutable_child(0)->set_position(Position::FIRST); - EXPECT_FALSE(event.GetAtomDimensionsValueProto(dimensions, &dimensionsValue)); + AttributionNode node1; + node1.set_uid(1000); + node1.set_tag("tag1"); + nodes.push_back(node1); - dimensions.mutable_child(0)->set_position(Position::ANY); - EXPECT_FALSE(event.GetAtomDimensionsValueProto(dimensions, &dimensionsValue)); + AttributionNode node2; + node2.set_uid(2000); + node2.set_tag("tag2"); + nodes.push_back(node2); - dimensions.mutable_child(0)->set_position(Position::LAST); - EXPECT_FALSE(event.GetAtomDimensionsValueProto(dimensions, &dimensionsValue)); + event1.write(nodes); + event1.write("hello"); + event1.write((int32_t)10); + event1.write((int64_t)20); + event1.write((float)1.1); + event1.init(); + + const auto& items = event1.getValues(); + EXPECT_EQ((size_t)8, items.size()); + EXPECT_EQ(1, event1.GetTagId()); + + const FieldValue& item0 = event1.getValues()[0]; + EXPECT_EQ(0x2010101, item0.mField.getField()); + EXPECT_EQ(Type::INT, item0.mValue.getType()); + EXPECT_EQ(1000, item0.mValue.int_value); + + const FieldValue& item1 = event1.getValues()[1]; + EXPECT_EQ(0x2010182, item1.mField.getField()); + EXPECT_EQ(Type::STRING, item1.mValue.getType()); + EXPECT_EQ("tag1", item1.mValue.str_value); + + const FieldValue& item2 = event1.getValues()[2]; + EXPECT_EQ(0x2018201, item2.mField.getField()); + EXPECT_EQ(Type::INT, item2.mValue.getType()); + EXPECT_EQ(2000, item2.mValue.int_value); + + const FieldValue& item3 = event1.getValues()[3]; + EXPECT_EQ(0x2018282, item3.mField.getField()); + EXPECT_EQ(Type::STRING, item3.mValue.getType()); + EXPECT_EQ("tag2", item3.mValue.str_value); + + const FieldValue& item4 = event1.getValues()[4]; + EXPECT_EQ(0x20000, item4.mField.getField()); + EXPECT_EQ(Type::STRING, item4.mValue.getType()); + EXPECT_EQ("hello", item4.mValue.str_value); + + const FieldValue& item5 = event1.getValues()[5]; + EXPECT_EQ(0x30000, item5.mField.getField()); + EXPECT_EQ(Type::INT, item5.mValue.getType()); + EXPECT_EQ(10, item5.mValue.int_value); + + const FieldValue& item6 = event1.getValues()[6]; + EXPECT_EQ(0x40000, item6.mField.getField()); + EXPECT_EQ(Type::LONG, item6.mValue.getType()); + EXPECT_EQ((int64_t)20, item6.mValue.long_value); + + const FieldValue& item7 = event1.getValues()[7]; + EXPECT_EQ(0x50000, item7.mField.getField()); + EXPECT_EQ(Type::FLOAT, item7.mValue.getType()); + EXPECT_EQ((float)1.1, item7.mValue.float_value); } -TEST(LogEventTest, testRepeatedAttributionNode) { - const int32_t TAG_ID = 123; - LogEvent event(TAG_ID, 0); - AttributionNode attribution_node1; - attribution_node1.set_uid(1111); - attribution_node1.set_tag("locationService"); +TEST(LogEventTest, TestLogParsing2) { + LogEvent event1(1, 2000); - AttributionNode attribution_node2; - attribution_node2.set_uid(2222); - attribution_node2.set_tag("locationService2"); + std::vector nodes; - AttributionNode attribution_node3; - attribution_node3.set_uid(3333); - attribution_node3.set_tag("locationService3"); - std::vector attribution_nodes = - {attribution_node1, attribution_node2, attribution_node3}; + event1.write("hello"); - // 1nd field: int32. - EXPECT_TRUE(event.write(int32_t(11))); - // 2rd field: float. - EXPECT_TRUE(event.write(3.45f)); - // Here it assume that the atom proto contains a repeated AttributionNode field. - // 3rd field: attribution node. This is repeated field. - EXPECT_TRUE(event.write(attribution_nodes)); - // 4th field: bool. - EXPECT_TRUE(event.write(true)); - // 5th field: long. - EXPECT_TRUE(event.write(uint64_t(1234))); + // repeated msg can be in the middle + AttributionNode node1; + node1.set_uid(1000); + node1.set_tag("tag1"); + nodes.push_back(node1); - event.init(); + AttributionNode node2; + node2.set_uid(2000); + node2.set_tag("tag2"); + nodes.push_back(node2); + event1.write(nodes); - DimensionsValue dimensionsValue; - // Query single primitive fields. - EXPECT_TRUE(event.GetSimpleAtomDimensionsValueProto(1, &dimensionsValue)); - EXPECT_EQ(dimensionsValue.field(), event.GetTagId()); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value_size(), 1); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).field(), 1); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).value_int(), 11); + event1.write((int32_t)10); + event1.write((int64_t)20); + event1.write((float)1.1); + event1.init(); - EXPECT_TRUE(event.GetSimpleAtomDimensionsValueProto(2, &dimensionsValue)); - EXPECT_EQ(dimensionsValue.field(), event.GetTagId()); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value_size(), 1); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).field(), 2); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).value_float(), 3.45f); + const auto& items = event1.getValues(); + EXPECT_EQ((size_t)8, items.size()); + EXPECT_EQ(1, event1.GetTagId()); - EXPECT_TRUE(event.GetSimpleAtomDimensionsValueProto(4, &dimensionsValue)); - EXPECT_EQ(dimensionsValue.field(), event.GetTagId()); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value_size(), 1); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).field(), 4); - // The bool value is stored in value_int field as logD does not support bool. - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).value_int(), true); + const FieldValue& item = event1.getValues()[0]; + EXPECT_EQ(0x00010000, item.mField.getField()); + EXPECT_EQ(Type::STRING, item.mValue.getType()); + EXPECT_EQ("hello", item.mValue.str_value); - EXPECT_TRUE(event.GetSimpleAtomDimensionsValueProto(5, &dimensionsValue)); - EXPECT_EQ(dimensionsValue.field(), event.GetTagId()); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value_size(), 1); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).field(), 5); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).value_long(), long(1234)); + const FieldValue& item0 = event1.getValues()[1]; + EXPECT_EQ(0x2020101, item0.mField.getField()); + EXPECT_EQ(Type::INT, item0.mValue.getType()); + EXPECT_EQ(1000, item0.mValue.int_value); - // First attribution. - FieldMatcher first_uid_dimensions; - first_uid_dimensions.set_field(event.GetTagId()); - first_uid_dimensions.add_child()->set_field(3); - first_uid_dimensions.mutable_child(0)->set_position(Position::FIRST); - first_uid_dimensions.mutable_child(0)->add_child()->set_field(1); - EXPECT_TRUE(event.GetAtomDimensionsValueProto(first_uid_dimensions, &dimensionsValue)); - EXPECT_EQ(dimensionsValue.field(), event.GetTagId()); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value_size(), 1); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).field(), 3); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).value_tuple() - .dimensions_value_size(), 1); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).value_tuple() - .dimensions_value(0).field(), 1); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).value_tuple() - .dimensions_value(0).value_int(), 1111); + const FieldValue& item1 = event1.getValues()[2]; + EXPECT_EQ(0x2020182, item1.mField.getField()); + EXPECT_EQ(Type::STRING, item1.mValue.getType()); + EXPECT_EQ("tag1", item1.mValue.str_value); - FieldMatcher first_tag_dimensions = first_uid_dimensions; - first_tag_dimensions.mutable_child(0)->mutable_child(0)->set_field(2); - EXPECT_TRUE(event.GetAtomDimensionsValueProto(first_tag_dimensions, &dimensionsValue)); - EXPECT_EQ(dimensionsValue.field(), event.GetTagId()); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value_size(), 1); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).field(), 3); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).value_tuple() - .dimensions_value_size(), 1); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).value_tuple() - .dimensions_value(0).field(), 2); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).value_tuple() - .dimensions_value(0).value_str(), "locationService"); + const FieldValue& item2 = event1.getValues()[3]; + EXPECT_EQ(0x2028201, item2.mField.getField()); + EXPECT_EQ(Type::INT, item2.mValue.getType()); + EXPECT_EQ(2000, item2.mValue.int_value); - FieldMatcher first_attribution_dimensions = first_uid_dimensions; - first_attribution_dimensions.mutable_child(0)->add_child()->set_field(2); - EXPECT_TRUE(event.GetAtomDimensionsValueProto(first_attribution_dimensions, &dimensionsValue)); - EXPECT_EQ(dimensionsValue.field(), event.GetTagId()); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value_size(), 1); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).field(), 3); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).value_tuple() - .dimensions_value_size(), 2); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).value_tuple() - .dimensions_value(0).field(), 1); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).value_tuple() - .dimensions_value(0).value_int(), 1111); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).value_tuple() - .dimensions_value(1).field(), 2); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).value_tuple() - .dimensions_value(1).value_str(), "locationService"); + const FieldValue& item3 = event1.getValues()[4]; + EXPECT_EQ(0x2028282, item3.mField.getField()); + EXPECT_EQ(Type::STRING, item3.mValue.getType()); + EXPECT_EQ("tag2", item3.mValue.str_value); - FieldMatcher last_attribution_dimensions = first_attribution_dimensions; - last_attribution_dimensions.mutable_child(0)->set_position(Position::LAST); - EXPECT_TRUE(event.GetAtomDimensionsValueProto(last_attribution_dimensions, &dimensionsValue)); - EXPECT_EQ(dimensionsValue.field(), event.GetTagId()); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value_size(), 1); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).field(), 3); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).value_tuple() - .dimensions_value_size(), 2); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).value_tuple() - .dimensions_value(0).field(), 1); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).value_tuple() - .dimensions_value(0).value_int(), 3333); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).value_tuple() - .dimensions_value(1).field(), 2); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).value_tuple() - .dimensions_value(1).value_str(), "locationService3"); + const FieldValue& item5 = event1.getValues()[5]; + EXPECT_EQ(0x30000, item5.mField.getField()); + EXPECT_EQ(Type::INT, item5.mValue.getType()); + EXPECT_EQ(10, item5.mValue.int_value); - FieldMatcher any_attribution_dimensions = first_attribution_dimensions; - any_attribution_dimensions.mutable_child(0)->set_position(Position::ANY); - std::vector dimensionsValues; - event.GetAtomDimensionsValueProtos(any_attribution_dimensions, &dimensionsValues); - EXPECT_EQ(dimensionsValues.size(), 3u); - EXPECT_EQ(dimensionsValues[0].field(), event.GetTagId()); - EXPECT_EQ(dimensionsValues[0].value_tuple().dimensions_value_size(), 1); - EXPECT_EQ(dimensionsValues[0].value_tuple().dimensions_value(0).field(), 3); - EXPECT_EQ(dimensionsValues[0].value_tuple().dimensions_value(0).value_tuple() - .dimensions_value_size(), 2); - EXPECT_EQ(dimensionsValues[0].value_tuple().dimensions_value(0).value_tuple() - .dimensions_value(0).field(), 1); - EXPECT_EQ(dimensionsValues[0].value_tuple().dimensions_value(0).value_tuple() - .dimensions_value(0).value_int(), 1111); - EXPECT_EQ(dimensionsValues[0].value_tuple().dimensions_value(0).value_tuple() - .dimensions_value(1).field(), 2); - EXPECT_EQ(dimensionsValues[0].value_tuple().dimensions_value(0).value_tuple() - .dimensions_value(1).value_str(), "locationService"); - EXPECT_EQ(dimensionsValues[1].field(), event.GetTagId()); - EXPECT_EQ(dimensionsValues[1].value_tuple().dimensions_value_size(), 1); - EXPECT_EQ(dimensionsValues[1].value_tuple().dimensions_value(0).field(), 3); - EXPECT_EQ(dimensionsValues[1].value_tuple().dimensions_value(0).value_tuple() - .dimensions_value_size(), 2); - EXPECT_EQ(dimensionsValues[1].value_tuple().dimensions_value(0).value_tuple() - .dimensions_value(0).field(), 1); - EXPECT_EQ(dimensionsValues[1].value_tuple().dimensions_value(0).value_tuple() - .dimensions_value(0).value_int(), 2222); - EXPECT_EQ(dimensionsValues[1].value_tuple().dimensions_value(0).value_tuple() - .dimensions_value(1).field(), 2); - EXPECT_EQ(dimensionsValues[1].value_tuple().dimensions_value(0).value_tuple() - .dimensions_value(1).value_str(), "locationService2"); - EXPECT_EQ(dimensionsValues[2].field(), event.GetTagId()); - EXPECT_EQ(dimensionsValues[2].value_tuple().dimensions_value_size(), 1); - EXPECT_EQ(dimensionsValues[2].value_tuple().dimensions_value(0).field(), 3); - EXPECT_EQ(dimensionsValues[2].value_tuple().dimensions_value(0).value_tuple() - .dimensions_value_size(), 2); - EXPECT_EQ(dimensionsValues[2].value_tuple().dimensions_value(0).value_tuple() - .dimensions_value(0).field(), 1); - EXPECT_EQ(dimensionsValues[2].value_tuple().dimensions_value(0).value_tuple() - .dimensions_value(0).value_int(), 3333); - EXPECT_EQ(dimensionsValues[2].value_tuple().dimensions_value(0).value_tuple() - .dimensions_value(1).field(), 2); - EXPECT_EQ(dimensionsValues[2].value_tuple().dimensions_value(0).value_tuple() - .dimensions_value(1).value_str(), "locationService3"); + const FieldValue& item6 = event1.getValues()[6]; + EXPECT_EQ(0x40000, item6.mField.getField()); + EXPECT_EQ(Type::LONG, item6.mValue.getType()); + EXPECT_EQ((int64_t)20, item6.mValue.long_value); - FieldMatcher mixed_dimensions = any_attribution_dimensions; - mixed_dimensions.add_child()->set_field(1000); - mixed_dimensions.add_child()->set_field(6); // missing field. - mixed_dimensions.add_child()->set_field(3); // position not set. - mixed_dimensions.add_child()->set_field(5); - mixed_dimensions.add_child()->set_field(1); - dimensionsValues.clear(); - event.GetAtomDimensionsValueProtos(mixed_dimensions, &dimensionsValues); - EXPECT_EQ(dimensionsValues.size(), 3u); - EXPECT_EQ(dimensionsValues[0].field(), event.GetTagId()); - EXPECT_EQ(dimensionsValues[0].value_tuple().dimensions_value_size(), 3); - EXPECT_EQ(dimensionsValues[0].value_tuple().dimensions_value(0).field(), 3); - EXPECT_EQ(dimensionsValues[0].value_tuple().dimensions_value(0).value_tuple().dimensions_value_size(), 2); - EXPECT_EQ(dimensionsValues[0].value_tuple().dimensions_value(0).value_tuple().dimensions_value(0).field(), 1); - EXPECT_EQ(dimensionsValues[0].value_tuple().dimensions_value(0).value_tuple().dimensions_value(0).value_int(), - 1111); - EXPECT_EQ(dimensionsValues[0].value_tuple().dimensions_value(0).value_tuple().dimensions_value(1).field(), 2); - EXPECT_EQ(dimensionsValues[0].value_tuple().dimensions_value(0).value_tuple().dimensions_value(1).value_str(), - "locationService"); - EXPECT_EQ(dimensionsValues[0].value_tuple().dimensions_value(1).field(), 5); - EXPECT_EQ(dimensionsValues[0].value_tuple().dimensions_value(1).value_long(), long(1234)); - EXPECT_EQ(dimensionsValues[0].value_tuple().dimensions_value(2).field(), 1); - EXPECT_EQ(dimensionsValues[0].value_tuple().dimensions_value(2).value_int(), 11); - - EXPECT_EQ(dimensionsValues[1].field(), event.GetTagId()); - EXPECT_EQ(dimensionsValues[1].value_tuple().dimensions_value_size(), 3); - EXPECT_EQ(dimensionsValues[1].value_tuple().dimensions_value(0).field(), 3); - EXPECT_EQ(dimensionsValues[1].value_tuple().dimensions_value(0).value_tuple().dimensions_value_size(), 2); - EXPECT_EQ(dimensionsValues[1].value_tuple().dimensions_value(0).value_tuple().dimensions_value(0).field(), 1); - EXPECT_EQ(dimensionsValues[1].value_tuple().dimensions_value(0).value_tuple().dimensions_value(0).value_int(), - 2222); - EXPECT_EQ(dimensionsValues[1].value_tuple().dimensions_value(0).value_tuple().dimensions_value(1).field(), 2); - EXPECT_EQ(dimensionsValues[1].value_tuple().dimensions_value(0).value_tuple().dimensions_value(1).value_str(), - "locationService2"); - EXPECT_EQ(dimensionsValues[1].value_tuple().dimensions_value(1).field(), 5); - EXPECT_EQ(dimensionsValues[1].value_tuple().dimensions_value(1).value_long(), long(1234)); - EXPECT_EQ(dimensionsValues[1].value_tuple().dimensions_value(2).field(), 1); - EXPECT_EQ(dimensionsValues[1].value_tuple().dimensions_value(2).value_int(), 11); - - EXPECT_EQ(dimensionsValues[2].field(), event.GetTagId()); - EXPECT_EQ(dimensionsValues[2].value_tuple().dimensions_value_size(), 3); - EXPECT_EQ(dimensionsValues[2].value_tuple().dimensions_value(0).field(), 3); - EXPECT_EQ(dimensionsValues[2].value_tuple().dimensions_value(0).value_tuple().dimensions_value_size(), 2); - EXPECT_EQ(dimensionsValues[2].value_tuple().dimensions_value(0).value_tuple().dimensions_value(0).field(), 1); - EXPECT_EQ(dimensionsValues[2].value_tuple().dimensions_value(0).value_tuple().dimensions_value(0).value_int(), - 3333); - EXPECT_EQ(dimensionsValues[2].value_tuple().dimensions_value(0).value_tuple().dimensions_value(1).field(), 2); - EXPECT_EQ(dimensionsValues[2].value_tuple().dimensions_value(0).value_tuple().dimensions_value(1).value_str(), - "locationService3"); - EXPECT_EQ(dimensionsValues[2].value_tuple().dimensions_value(1).field(), 5); - EXPECT_EQ(dimensionsValues[2].value_tuple().dimensions_value(1).value_long(), long(1234)); - EXPECT_EQ(dimensionsValues[2].value_tuple().dimensions_value(2).field(), 1); - EXPECT_EQ(dimensionsValues[2].value_tuple().dimensions_value(2).value_int(), 11); - - FieldMatcher wrong_dimensions = mixed_dimensions; - // Wrong tagId. - wrong_dimensions.set_field(event.GetTagId() + 100); - dimensionsValues.clear(); - event.GetAtomDimensionsValueProtos(wrong_dimensions, &dimensionsValues); - EXPECT_TRUE(dimensionsValues.empty()); + const FieldValue& item7 = event1.getValues()[7]; + EXPECT_EQ(0x50000, item7.mField.getField()); + EXPECT_EQ(Type::FLOAT, item7.mValue.getType()); + EXPECT_EQ((float)1.1, item7.mValue.float_value); } -TEST(LogEventTest, testMessageField) { - const int32_t TAG_ID = 123; - LogEvent event(TAG_ID, 0); - AttributionNode attribution_node1; - attribution_node1.set_uid(1111); - attribution_node1.set_tag("locationService"); - - AttributionNode attribution_node2; - attribution_node2.set_uid(2222); - attribution_node2.set_tag("locationService2"); - - // 1nd field: int32. - EXPECT_TRUE(event.write(int32_t(11))); - // 2rd field: float. - EXPECT_TRUE(event.write(3.45f)); - // Here it assume that the atom proto contains two optional AttributionNode fields. - // 3rd field: attribution node. This is not repeated field. - EXPECT_TRUE(event.write(attribution_node1)); - // 4th field: another attribution field. This is not repeated field. - EXPECT_TRUE(event.write(attribution_node2)); - // 5th field: bool. - EXPECT_TRUE(event.write(true)); - // 6th field: long. - EXPECT_TRUE(event.write(uint64_t(1234))); - - event.init(); - - FieldMatcher uid_dimensions1; - uid_dimensions1.set_field(event.GetTagId()); - uid_dimensions1.add_child()->set_field(3); - uid_dimensions1.mutable_child(0)->add_child()->set_field(1); - - FieldMatcher tag_dimensions1; - tag_dimensions1.set_field(event.GetTagId()); - tag_dimensions1.add_child()->set_field(3); - tag_dimensions1.mutable_child(0)->add_child()->set_field(2); - - FieldMatcher attribution_dimensions1; - attribution_dimensions1.set_field(event.GetTagId()); - attribution_dimensions1.add_child()->set_field(3); - attribution_dimensions1.mutable_child(0)->add_child()->set_field(1); - attribution_dimensions1.mutable_child(0)->add_child()->set_field(2); - - FieldMatcher uid_dimensions2 = uid_dimensions1; - uid_dimensions2.mutable_child(0)->set_field(4); - - FieldMatcher tag_dimensions2 = tag_dimensions1; - tag_dimensions2.mutable_child(0)->set_field(4); - - FieldMatcher attribution_dimensions2 = attribution_dimensions1; - attribution_dimensions2.mutable_child(0)->set_field(4); - - FieldMatcher mixed_dimensions = attribution_dimensions1; - mixed_dimensions.add_child()->set_field(4); - mixed_dimensions.mutable_child(1)->add_child()->set_field(1); - mixed_dimensions.add_child()->set_field(1000); - mixed_dimensions.add_child()->set_field(5); - mixed_dimensions.add_child()->set_field(1); - - DimensionsValue dimensionsValue; - - // Query single primitive fields. - EXPECT_TRUE(event.GetSimpleAtomDimensionsValueProto(1, &dimensionsValue)); - EXPECT_EQ(dimensionsValue.field(), event.GetTagId()); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value_size(), 1); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).field(), 1); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).value_int(), 11); - - EXPECT_TRUE(event.GetSimpleAtomDimensionsValueProto(2, &dimensionsValue)); - EXPECT_EQ(dimensionsValue.field(), event.GetTagId()); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value_size(), 1); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).field(), 2); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).value_float(), 3.45f); - - EXPECT_TRUE(event.GetSimpleAtomDimensionsValueProto(5, &dimensionsValue)); - EXPECT_EQ(dimensionsValue.field(), event.GetTagId()); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value_size(), 1); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).field(), 5); - // The bool value is stored in value_int field as logD does not support bool. - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).value_int(), true); - - EXPECT_TRUE(event.GetSimpleAtomDimensionsValueProto(6, &dimensionsValue)); - EXPECT_EQ(dimensionsValue.field(), event.GetTagId()); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value_size(), 1); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).field(), 6); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).value_long(), long(1234)); - - // Query atom field 3: attribution node uid field only. - EXPECT_TRUE(event.GetAtomDimensionsValueProto(uid_dimensions1, &dimensionsValue)); - EXPECT_EQ(dimensionsValue.field(), event.GetTagId()); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value_size(), 1); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).field(), 3); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).value_tuple() - .dimensions_value_size(), 1); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).value_tuple() - .dimensions_value(0).field(), 1); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).value_tuple() - .dimensions_value(0).value_int(), 1111); - - // Query atom field 3: attribution node tag field only. - EXPECT_TRUE(event.GetAtomDimensionsValueProto(tag_dimensions1, &dimensionsValue)); - EXPECT_EQ(dimensionsValue.field(), event.GetTagId()); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value_size(), 1); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).field(), 3); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).value_tuple() - .dimensions_value_size(), 1); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).value_tuple() - .dimensions_value(0).field(), 2); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).value_tuple() - .dimensions_value(0).value_str(), "locationService"); - - // Query atom field 3: attribution node uid + tag fields. - EXPECT_TRUE(event.GetAtomDimensionsValueProto(attribution_dimensions1, &dimensionsValue)); - EXPECT_EQ(dimensionsValue.field(), event.GetTagId()); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value_size(), 1); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).field(), 3); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).value_tuple() - .dimensions_value_size(), 2); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).value_tuple() - .dimensions_value(0).field(), 1); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).value_tuple() - .dimensions_value(0).value_int(), 1111); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).value_tuple() - .dimensions_value(1).field(), 2); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).value_tuple() - .dimensions_value(1).value_str(), "locationService"); - - // Query atom field 4: attribution node uid field only. - EXPECT_TRUE(event.GetAtomDimensionsValueProto(uid_dimensions2, &dimensionsValue)); - EXPECT_EQ(dimensionsValue.field(), event.GetTagId()); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value_size(), 1); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).field(), 4); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).value_tuple() - .dimensions_value_size(), 1); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).value_tuple() - .dimensions_value(0).field(), 1); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).value_tuple() - .dimensions_value(0).value_int(), 2222); - - // Query atom field 4: attribution node tag field only. - EXPECT_TRUE(event.GetAtomDimensionsValueProto(tag_dimensions2, &dimensionsValue)); - EXPECT_EQ(dimensionsValue.field(), event.GetTagId()); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value_size(), 1); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).field(), 4); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).value_tuple() - .dimensions_value_size(), 1); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).value_tuple() - .dimensions_value(0).field(), 2); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).value_tuple() - .dimensions_value(0).value_str(), "locationService2"); - - // Query atom field 4: attribution node uid + tag fields. - EXPECT_TRUE(event.GetAtomDimensionsValueProto(attribution_dimensions2, &dimensionsValue)); - EXPECT_EQ(dimensionsValue.field(), event.GetTagId()); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value_size(), 1); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).field(), 4); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).value_tuple() - .dimensions_value_size(), 2); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).value_tuple() - .dimensions_value(0).field(), 1); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).value_tuple() - .dimensions_value(0).value_int(), 2222); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).value_tuple() - .dimensions_value(1).field(), 2); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).value_tuple() - .dimensions_value(1).value_str(), "locationService2"); - - // Query multiple fields: - // 1/ Field 3: attribution uid + tag. - // 2/ Field 4: attribution uid only. - // 3/ Field not exist. - // 4/ Primitive fields #5 - // 5/ Primitive fields #1 - EXPECT_TRUE(event.GetAtomDimensionsValueProto(mixed_dimensions, &dimensionsValue)); - EXPECT_EQ(dimensionsValue.field(), event.GetTagId()); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value_size(), 4); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).field(), 3); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).value_tuple() - .dimensions_value_size(), 2); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).value_tuple() - .dimensions_value(0).field(), 1); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).value_tuple() - .dimensions_value(0).value_int(), 1111); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).value_tuple() - .dimensions_value(1).field(), 2); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).value_tuple() - .dimensions_value(1).value_str(), "locationService"); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(1).field(), 4); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(1).value_tuple() - .dimensions_value_size(), 1); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(1).value_tuple() - .dimensions_value(0).field(), 1); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(1).value_tuple() - .dimensions_value(0).value_int(), 2222); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(2).field(), 5); - // The bool value is stored in value_int field as logD does not support bool. - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(2).value_int(), true); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(3).field(), 1); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(3).value_int(), 11); -} - -TEST(LogEventTest, testAllPrimitiveFields) { - const int32_t TAG_ID = 123; - LogEvent event(TAG_ID, 0); - - // 1nd field: int32. - EXPECT_TRUE(event.write(int32_t(11))); - // 2rd field: float. - EXPECT_TRUE(event.write(3.45f)); - // 3th field: string. - EXPECT_TRUE(event.write("test")); - // 4th field: bool. - EXPECT_TRUE(event.write(true)); - // 5th field: bool. - EXPECT_TRUE(event.write(false)); - // 6th field: long. - EXPECT_TRUE(event.write(uint64_t(1234))); - - event.init(); - - DimensionsValue dimensionsValue; - EXPECT_TRUE(event.GetSimpleAtomDimensionsValueProto(1, &dimensionsValue)); - EXPECT_EQ(dimensionsValue.field(), event.GetTagId()); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value_size(), 1); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).field(), 1); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).value_int(), 11); - - EXPECT_TRUE(event.GetSimpleAtomDimensionsValueProto(2, &dimensionsValue)); - EXPECT_EQ(dimensionsValue.field(), event.GetTagId()); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value_size(), 1); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).field(), 2); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).value_float(), 3.45f); - - EXPECT_TRUE(event.GetSimpleAtomDimensionsValueProto(3, &dimensionsValue)); - EXPECT_EQ(dimensionsValue.field(), event.GetTagId()); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value_size(), 1); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).field(), 3); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).value_str(), "test"); - - EXPECT_TRUE(event.GetSimpleAtomDimensionsValueProto(4, &dimensionsValue)); - EXPECT_EQ(dimensionsValue.field(), event.GetTagId()); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value_size(), 1); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).field(), 4); - // The bool value is stored in value_int field as logD does not support bool. - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).value_int(), true); - - EXPECT_TRUE(event.GetSimpleAtomDimensionsValueProto(5, &dimensionsValue)); - EXPECT_EQ(dimensionsValue.field(), event.GetTagId()); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value_size(), 1); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).field(), 5); - // The bool value is stored in value_int field as logD does not support bool. - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).value_int(), false); - - EXPECT_TRUE(event.GetSimpleAtomDimensionsValueProto(6, &dimensionsValue)); - EXPECT_EQ(dimensionsValue.field(), event.GetTagId()); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value_size(), 1); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).field(), 6); - EXPECT_EQ(dimensionsValue.value_tuple().dimensions_value(0).value_long(), long(1234)); - - // Field not exist. - EXPECT_FALSE(event.GetSimpleAtomDimensionsValueProto(7, &dimensionsValue)); -} - -TEST(LogEventTest, testWriteAtomProtoToStream) { - AttributionNode attribution_node1; - attribution_node1.set_uid(1111); - attribution_node1.set_tag("locationService"); - - AttributionNode attribution_node2; - attribution_node2.set_uid(2222); - attribution_node2.set_tag("locationService2"); - - AttributionNode attribution_node3; - attribution_node3.set_uid(3333); - attribution_node3.set_tag("locationService3"); - std::vector attribution_nodes = - {attribution_node1, attribution_node2, attribution_node3}; - - LogEvent event(1, 0); - EXPECT_TRUE(event.write("222")); - EXPECT_TRUE(event.write(attribution_nodes)); - EXPECT_TRUE(event.write(345)); - EXPECT_TRUE(event.write(attribution_node3)); - EXPECT_TRUE(event.write("hello")); - event.init(); - - util::ProtoOutputStream protoOutput; - // For now only see whether it will crash. - // TODO(yanglu): test parsing from stream. - event.ToProto(protoOutput); -} } // namespace statsd } // namespace os diff --git a/cmds/statsd/tests/StatsLogProcessor_test.cpp b/cmds/statsd/tests/StatsLogProcessor_test.cpp index aab5bedb3cbeb..cb72697941e0c 100644 --- a/cmds/statsd/tests/StatsLogProcessor_test.cpp +++ b/cmds/statsd/tests/StatsLogProcessor_test.cpp @@ -45,7 +45,7 @@ public: } MOCK_METHOD0(byteSize, size_t()); - MOCK_METHOD1(onDumpReport, void(ProtoOutputStream* output)); + MOCK_METHOD2(onDumpReport, void(const uint64_t timeNs, ProtoOutputStream* output)); }; TEST(StatsLogProcessorTest, TestRateLimitByteSize) { @@ -69,24 +69,26 @@ TEST(StatsLogProcessorTest, TestRateLimitBroadcast) { sp m = new UidMap(); sp anomalyMonitor; int broadcastCount = 0; - StatsLogProcessor p(m, anomalyMonitor, 0, - [&broadcastCount](const ConfigKey& key) { broadcastCount++; }); + StatsLogProcessor p(m, anomalyMonitor, 0, [&broadcastCount](const ConfigKey& key) { + broadcastCount++; + }); MockMetricsManager mockMetricsManager; ConfigKey key(100, 12345); EXPECT_CALL(mockMetricsManager, byteSize()) - .Times(2) + .Times(1) .WillRepeatedly(Return(int(StatsdStats::kMaxMetricsBytesPerConfig * .95))); // Expect only one broadcast despite always returning a size that should trigger broadcast. p.flushIfNecessaryLocked(1, key, mockMetricsManager); EXPECT_EQ(1, broadcastCount); + // b/73089712 // This next call to flush should not trigger a broadcast. - p.mLastByteSizeTimes.clear(); // Force another check for byte size. - p.flushIfNecessaryLocked(2, key, mockMetricsManager); - EXPECT_EQ(1, broadcastCount); + // p.mLastByteSizeTimes.clear(); // Force another check for byte size. + // p.flushIfNecessaryLocked(2, key, mockMetricsManager); + // EXPECT_EQ(1, broadcastCount); } TEST(StatsLogProcessorTest, TestDropWhenByteSizeTooLarge) { @@ -103,7 +105,7 @@ TEST(StatsLogProcessorTest, TestDropWhenByteSizeTooLarge) { .Times(1) .WillRepeatedly(Return(int(StatsdStats::kMaxMetricsBytesPerConfig * 1.2))); - EXPECT_CALL(mockMetricsManager, onDumpReport(_)).Times(1); + EXPECT_CALL(mockMetricsManager, onDumpReport(_, _)).Times(1); // Expect to call the onDumpReport and skip the broadcast. p.flushIfNecessaryLocked(1, key, mockMetricsManager); diff --git a/cmds/statsd/tests/anomaly/AnomalyTracker_test.cpp b/cmds/statsd/tests/anomaly/AnomalyTracker_test.cpp index a415ea1a3069a..b4a7bb7060294 100644 --- a/cmds/statsd/tests/anomaly/AnomalyTracker_test.cpp +++ b/cmds/statsd/tests/anomaly/AnomalyTracker_test.cpp @@ -34,10 +34,10 @@ namespace statsd { const ConfigKey kConfigKey(0, 12345); MetricDimensionKey getMockMetricDimensionKey(int key, string value) { - DimensionsValue dimensionsValue; - dimensionsValue.set_field(key); - dimensionsValue.set_value_str(value); - return MetricDimensionKey(HashableDimensionKey(dimensionsValue), DEFAULT_DIMENSION_KEY); + int pos[] = {key, 0, 0}; + HashableDimensionKey dim; + dim.addValue(FieldValue(Field(1, pos, 0), Value(value))); + return MetricDimensionKey(dim, DEFAULT_DIMENSION_KEY); } void AddValueToBucket(const std::vector>& key_value_pair_list, diff --git a/cmds/statsd/tests/condition/SimpleConditionTracker_test.cpp b/cmds/statsd/tests/condition/SimpleConditionTracker_test.cpp index d1b7b2842fd2a..038d449368383 100644 --- a/cmds/statsd/tests/condition/SimpleConditionTracker_test.cpp +++ b/cmds/statsd/tests/condition/SimpleConditionTracker_test.cpp @@ -80,31 +80,31 @@ std::map> getWakeLockQueryKey( const std::vector &uids, const string& conditionName) { std::map> outputKeyMap; std::vector uid_indexes; + int pos[] = {1, 1, 1}; + int depth = 2; + Field field(1, pos, depth); switch(position) { case Position::FIRST: uid_indexes.push_back(0); break; case Position::LAST: uid_indexes.push_back(uids.size() - 1); + field.setField(0x02018001); break; case Position::ANY: uid_indexes.resize(uids.size()); std::iota(uid_indexes.begin(), uid_indexes.end(), 0); + field.setField(0x02010001); break; default: break; } for (const int idx : uid_indexes) { - DimensionsValue dimensionsValue; - dimensionsValue.set_field(TAG_ID); - dimensionsValue.mutable_value_tuple()->add_dimensions_value()->set_field( - ATTRIBUTION_NODE_FIELD_ID); - dimensionsValue.mutable_value_tuple()->mutable_dimensions_value(0) - ->mutable_value_tuple()->add_dimensions_value()->set_field(ATTRIBUTION_NODE_FIELD_ID); - dimensionsValue.mutable_value_tuple()->mutable_dimensions_value(0) - ->mutable_value_tuple()->mutable_dimensions_value(0)->set_value_int(uids[idx]); - outputKeyMap[StringToId(conditionName)].push_back(HashableDimensionKey(dimensionsValue)); + Value value((int32_t)uids[idx]); + HashableDimensionKey dim; + dim.addValue(FieldValue(field, value)); + outputKeyMap[StringToId(conditionName)].push_back(dim); } return outputKeyMap; } @@ -265,7 +265,7 @@ TEST(SimpleConditionTrackerTest, TestNonSlicedConditionNestCounting) { TEST(SimpleConditionTrackerTest, TestSlicedCondition) { for (Position position : { Position::ANY, Position::FIRST, Position::LAST}) { - FieldMatcher dimensionInCondition; + vector dimensionInCondition; std::unordered_set dimensionKeys; SimplePredicate simplePredicate = getWakeLockHeldCondition( @@ -374,7 +374,7 @@ TEST(SimpleConditionTrackerTest, TestSlicedCondition) { } TEST(SimpleConditionTrackerTest, TestSlicedWithNoOutputDim) { - FieldMatcher dimensionInCondition; + vector dimensionInCondition; std::unordered_set dimensionKeys; SimplePredicate simplePredicate = getWakeLockHeldCondition( @@ -470,7 +470,7 @@ TEST(SimpleConditionTrackerTest, TestSlicedWithNoOutputDim) { TEST(SimpleConditionTrackerTest, TestStopAll) { for (Position position : {Position::ANY, Position::FIRST, Position::LAST}) { - FieldMatcher dimensionInCondition; + vector dimensionInCondition; std::unordered_set dimensionKeys; SimplePredicate simplePredicate = getWakeLockHeldCondition( true /*nesting*/, true /*default to false*/, true /*output slice by uid*/, @@ -576,7 +576,6 @@ TEST(SimpleConditionTrackerTest, TestStopAll) { conditionCache, dimensionKeys); EXPECT_EQ(ConditionState::kFalse, conditionCache[0]); } - } } // namespace statsd diff --git a/cmds/statsd/tests/dimension_test.cpp b/cmds/statsd/tests/dimension_test.cpp deleted file mode 100644 index 678abaed5fd85..0000000000000 --- a/cmds/statsd/tests/dimension_test.cpp +++ /dev/null @@ -1,149 +0,0 @@ -// Copyright (C) 2017 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "dimension.h" - -#include - -using namespace android::os::statsd; - -#ifdef __ANDROID__ - -TEST(DimensionTest, subLeafNodes) { - DimensionsValue dimension; - int tagId = 100; - dimension.set_field(tagId); - auto child = dimension.mutable_value_tuple()->add_dimensions_value(); - child->set_field(1); - child->set_value_int(2000); - - child = dimension.mutable_value_tuple()->add_dimensions_value(); - child->set_field(3); - child->set_value_str("test"); - - child = dimension.mutable_value_tuple()->add_dimensions_value(); - child->set_field(4); - auto grandChild = child->mutable_value_tuple()->add_dimensions_value(); - grandChild->set_field(1); - grandChild->set_value_float(1.3f); - grandChild = child->mutable_value_tuple()->add_dimensions_value(); - grandChild->set_field(3); - grandChild->set_value_str("tag"); - - child = dimension.mutable_value_tuple()->add_dimensions_value(); - child->set_field(6); - child->set_value_bool(false); - - DimensionsValue sub_dimension; - FieldMatcher matcher; - - // Tag id not matched. - matcher.set_field(tagId + 1); - EXPECT_FALSE(getSubDimension(dimension, matcher, &sub_dimension)); - - // Field not exist. - matcher.Clear(); - matcher.set_field(tagId); - matcher.add_child()->set_field(5); - EXPECT_FALSE(getSubDimension(dimension, matcher, &sub_dimension)); - - // Field exists. - matcher.Clear(); - matcher.set_field(tagId); - matcher.add_child()->set_field(3); - EXPECT_TRUE(getSubDimension(dimension, matcher, &sub_dimension)); - - // Field exists. - matcher.Clear(); - sub_dimension.Clear(); - matcher.set_field(tagId); - matcher.add_child()->set_field(6); - EXPECT_TRUE(getSubDimension(dimension, matcher, &sub_dimension)); - - // Field exists. - matcher.Clear(); - sub_dimension.Clear(); - matcher.set_field(tagId); - matcher.add_child()->set_field(1); - EXPECT_TRUE(getSubDimension(dimension, matcher, &sub_dimension)); - - // Not leaf field. - matcher.Clear(); - sub_dimension.Clear(); - matcher.set_field(tagId); - matcher.add_child()->set_field(4); - EXPECT_FALSE(getSubDimension(dimension, matcher, &sub_dimension)); - - // Grand-child leaf field not exist. - matcher.Clear(); - sub_dimension.Clear(); - matcher.set_field(tagId); - auto childMatcher = matcher.add_child(); - childMatcher->set_field(4); - childMatcher->add_child()->set_field(2); - EXPECT_FALSE(getSubDimension(dimension, matcher, &sub_dimension)); - - // Grand-child leaf field. - matcher.Clear(); - sub_dimension.Clear(); - matcher.set_field(tagId); - childMatcher = matcher.add_child(); - childMatcher->set_field(4); - childMatcher->add_child()->set_field(1); - EXPECT_TRUE(getSubDimension(dimension, matcher, &sub_dimension)); - - matcher.Clear(); - sub_dimension.Clear(); - matcher.set_field(tagId); - childMatcher = matcher.add_child(); - childMatcher->set_field(4); - childMatcher->add_child()->set_field(3); - EXPECT_TRUE(getSubDimension(dimension, matcher, &sub_dimension)); - - // Multiple grand-child fields. - matcher.Clear(); - sub_dimension.Clear(); - matcher.set_field(tagId); - childMatcher = matcher.add_child(); - childMatcher->set_field(4); - childMatcher->add_child()->set_field(3); - childMatcher->add_child()->set_field(1); - EXPECT_TRUE(getSubDimension(dimension, matcher, &sub_dimension)); - - // Multiple fields. - matcher.Clear(); - sub_dimension.Clear(); - matcher.set_field(tagId); - childMatcher = matcher.add_child(); - childMatcher->set_field(4); - childMatcher->add_child()->set_field(3); - childMatcher->add_child()->set_field(1); - matcher.add_child()->set_field(3); - EXPECT_TRUE(getSubDimension(dimension, matcher, &sub_dimension)); - - // Subset of the fields not exist. - matcher.Clear(); - sub_dimension.Clear(); - matcher.set_field(tagId); - childMatcher = matcher.add_child(); - childMatcher->set_field(4); - childMatcher->add_child()->set_field(3); - childMatcher->add_child()->set_field(1); - matcher.add_child()->set_field(2); - EXPECT_FALSE(getSubDimension(dimension, matcher, &sub_dimension)); -} - -#else -GTEST_LOG_(INFO) << "This test does nothing.\n"; -#endif diff --git a/cmds/statsd/tests/e2e/Attribution_e2e_test.cpp b/cmds/statsd/tests/e2e/Attribution_e2e_test.cpp index a56db28e64b39..01743ef1a45c6 100644 --- a/cmds/statsd/tests/e2e/Attribution_e2e_test.cpp +++ b/cmds/statsd/tests/e2e/Attribution_e2e_test.cpp @@ -146,9 +146,13 @@ TEST(AttributionE2eTest, TestAttributionMatchAndSlice) { processor->OnLogEvent(event.get()); } ConfigMetricsReportList reports; - processor->onDumpReport(cfgKey, bucketStartTimeNs + 4 * bucketSizeNs + 1, &reports); + vector buffer; + processor->onDumpReport(cfgKey, bucketStartTimeNs + 4 * bucketSizeNs + 1, &buffer); + EXPECT_TRUE(buffer.size() > 0); + EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size())); EXPECT_EQ(reports.reports_size(), 1); EXPECT_EQ(reports.reports(0).metrics_size(), 1); + StatsLogReport::CountMetricDataWrapper countMetrics; sortMetricDataByDimensionsValue(reports.reports(0).metrics(0).count_metrics(), &countMetrics); EXPECT_EQ(countMetrics.data_size(), 4); diff --git a/cmds/statsd/tests/e2e/DimensionInCondition_e2e_test.cpp b/cmds/statsd/tests/e2e/DimensionInCondition_e2e_test.cpp index b5d48efb55f69..275b58244e0f8 100644 --- a/cmds/statsd/tests/e2e/DimensionInCondition_e2e_test.cpp +++ b/cmds/statsd/tests/e2e/DimensionInCondition_e2e_test.cpp @@ -43,8 +43,8 @@ StatsdConfig CreateCountMetricWithNoLinkConfig() { auto holdingWakelockPredicate = CreateHoldingWakelockPredicate(); // The predicate is dimensioning by any attribution node and both by uid and tag. *holdingWakelockPredicate.mutable_simple_predicate()->mutable_dimensions() = - CreateAttributionUidAndTagDimensions( - android::util::WAKELOCK_STATE_CHANGED, {Position::FIRST}); + CreateAttributionUidAndTagDimensions(android::util::WAKELOCK_STATE_CHANGED, + {Position::FIRST}); *config.add_predicate() = holdingWakelockPredicate; auto combinationPredicate = config.add_predicate(); @@ -57,8 +57,8 @@ StatsdConfig CreateCountMetricWithNoLinkConfig() { metric->set_id(StringToId("ScreenBrightnessChangeMetric")); metric->set_what(screenBrightnessChangeAtomMatcher.id()); metric->set_condition(combinationPredicate->id()); - *metric->mutable_dimensions_in_what() = CreateDimensions( - android::util::SCREEN_BRIGHTNESS_CHANGED, {1 /* level */}); + *metric->mutable_dimensions_in_what() = + CreateDimensions(android::util::SCREEN_BRIGHTNESS_CHANGED, {1 /* level */}); *metric->mutable_dimensions_in_condition() = CreateAttributionUidDimensions( android::util::WAKELOCK_STATE_CHANGED, {Position::FIRST}); metric->set_bucket(ONE_MINUTE); @@ -72,62 +72,54 @@ TEST(DimensionInConditionE2eTest, TestCountMetricNoLink) { auto config = CreateCountMetricWithNoLinkConfig(); int64_t bucketStartTimeNs = 10000000000; int64_t bucketSizeNs = - TimeUnitToBucketSizeInMillis(config.count_metric(0).bucket()) * 1000000LL; + TimeUnitToBucketSizeInMillis(config.count_metric(0).bucket()) * 1000000LL; auto processor = CreateStatsLogProcessor(bucketStartTimeNs / NS_PER_SEC, config, cfgKey); EXPECT_EQ(processor->mMetricsManagers.size(), 1u); EXPECT_TRUE(processor->mMetricsManagers.begin()->second->isConfigValid()); - std::vector attributions1 = - {CreateAttribution(111, "App1"), CreateAttribution(222, "GMSCoreModule1"), - CreateAttribution(222, "GMSCoreModule2")}; + std::vector attributions1 = {CreateAttribution(111, "App1"), + CreateAttribution(222, "GMSCoreModule1"), + CreateAttribution(222, "GMSCoreModule2")}; - std::vector attributions2 = - {CreateAttribution(333, "App2"), CreateAttribution(222, "GMSCoreModule1"), - CreateAttribution(555, "GMSCoreModule2")}; + std::vector attributions2 = {CreateAttribution(333, "App2"), + CreateAttribution(222, "GMSCoreModule1"), + CreateAttribution(555, "GMSCoreModule2")}; std::vector> events; - events.push_back(CreateScreenStateChangedEvent( - android::view::DISPLAY_STATE_ON, bucketStartTimeNs + 10)); - events.push_back(CreateScreenStateChangedEvent( - android::view::DISPLAY_STATE_OFF, bucketStartTimeNs + 100)); - events.push_back(CreateScreenStateChangedEvent( - android::view::DISPLAY_STATE_ON, bucketStartTimeNs + bucketSizeNs + 1)); - events.push_back(CreateScreenStateChangedEvent( - android::view::DISPLAY_STATE_OFF, bucketStartTimeNs + 2 * bucketSizeNs - 10)); + events.push_back( + CreateScreenStateChangedEvent(android::view::DISPLAY_STATE_ON, bucketStartTimeNs + 10)); + events.push_back(CreateScreenStateChangedEvent(android::view::DISPLAY_STATE_OFF, + bucketStartTimeNs + 100)); + events.push_back(CreateScreenStateChangedEvent(android::view::DISPLAY_STATE_ON, + bucketStartTimeNs + bucketSizeNs + 1)); + events.push_back(CreateScreenStateChangedEvent(android::view::DISPLAY_STATE_OFF, + bucketStartTimeNs + 2 * bucketSizeNs - 10)); - events.push_back(CreateAcquireWakelockEvent( - attributions1, "wl1", bucketStartTimeNs + 200)); - events.push_back(CreateReleaseWakelockEvent( - attributions1, "wl1", bucketStartTimeNs + bucketSizeNs + 1)); + events.push_back(CreateAcquireWakelockEvent(attributions1, "wl1", bucketStartTimeNs + 200)); + events.push_back( + CreateReleaseWakelockEvent(attributions1, "wl1", bucketStartTimeNs + bucketSizeNs + 1)); - events.push_back(CreateAcquireWakelockEvent( - attributions2, "wl2", bucketStartTimeNs + bucketSizeNs - 100)); - events.push_back(CreateReleaseWakelockEvent( - attributions2, "wl2", bucketStartTimeNs + 2 * bucketSizeNs - 50)); + events.push_back(CreateAcquireWakelockEvent(attributions2, "wl2", + bucketStartTimeNs + bucketSizeNs - 100)); + events.push_back(CreateReleaseWakelockEvent(attributions2, "wl2", + bucketStartTimeNs + 2 * bucketSizeNs - 50)); - events.push_back(CreateScreenBrightnessChangedEvent( - 123, bucketStartTimeNs + 11)); - events.push_back(CreateScreenBrightnessChangedEvent( - 123, bucketStartTimeNs + 101)); - events.push_back(CreateScreenBrightnessChangedEvent( - 123, bucketStartTimeNs + 201)); - events.push_back(CreateScreenBrightnessChangedEvent( - 456, bucketStartTimeNs + 203)); - events.push_back(CreateScreenBrightnessChangedEvent( - 456, bucketStartTimeNs + bucketSizeNs - 99)); - events.push_back(CreateScreenBrightnessChangedEvent( - 456, bucketStartTimeNs + bucketSizeNs - 2)); - events.push_back(CreateScreenBrightnessChangedEvent( - 789, bucketStartTimeNs + bucketSizeNs - 1)); - events.push_back(CreateScreenBrightnessChangedEvent( - 456, bucketStartTimeNs + bucketSizeNs + 2)); - events.push_back(CreateScreenBrightnessChangedEvent( - 789, bucketStartTimeNs + 2 * bucketSizeNs - 11)); - events.push_back(CreateScreenBrightnessChangedEvent( - 789, bucketStartTimeNs + 2 * bucketSizeNs - 9)); - events.push_back(CreateScreenBrightnessChangedEvent( - 789, bucketStartTimeNs + 2 * bucketSizeNs - 1)); + events.push_back(CreateScreenBrightnessChangedEvent(123, bucketStartTimeNs + 11)); + events.push_back(CreateScreenBrightnessChangedEvent(123, bucketStartTimeNs + 101)); + events.push_back(CreateScreenBrightnessChangedEvent(123, bucketStartTimeNs + 201)); + events.push_back(CreateScreenBrightnessChangedEvent(456, bucketStartTimeNs + 203)); + events.push_back( + CreateScreenBrightnessChangedEvent(456, bucketStartTimeNs + bucketSizeNs - 99)); + events.push_back(CreateScreenBrightnessChangedEvent(456, bucketStartTimeNs + bucketSizeNs - 2)); + events.push_back(CreateScreenBrightnessChangedEvent(789, bucketStartTimeNs + bucketSizeNs - 1)); + events.push_back(CreateScreenBrightnessChangedEvent(456, bucketStartTimeNs + bucketSizeNs + 2)); + events.push_back( + CreateScreenBrightnessChangedEvent(789, bucketStartTimeNs + 2 * bucketSizeNs - 11)); + events.push_back( + CreateScreenBrightnessChangedEvent(789, bucketStartTimeNs + 2 * bucketSizeNs - 9)); + events.push_back( + CreateScreenBrightnessChangedEvent(789, bucketStartTimeNs + 2 * bucketSizeNs - 1)); sortLogEventsByTimestamp(&events); @@ -136,7 +128,10 @@ TEST(DimensionInConditionE2eTest, TestCountMetricNoLink) { } ConfigMetricsReportList reports; - processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, &reports); + vector buffer; + processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, &buffer); + EXPECT_TRUE(buffer.size() > 0); + EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size())); EXPECT_EQ(reports.reports_size(), 1); EXPECT_EQ(reports.reports(0).metrics_size(), 1); @@ -147,7 +142,7 @@ TEST(DimensionInConditionE2eTest, TestCountMetricNoLink) { auto data = countMetrics.data(0); EXPECT_EQ(data.bucket_info_size(), 1); EXPECT_EQ(data.bucket_info(0).count(), 1); - EXPECT_EQ(data.bucket_info(0).start_bucket_nanos(), bucketStartTimeNs ); + EXPECT_EQ(data.bucket_info(0).start_bucket_nanos(), bucketStartTimeNs); EXPECT_EQ(data.bucket_info(0).end_bucket_nanos(), bucketStartTimeNs + bucketSizeNs); EXPECT_EQ(data.dimensions_in_what().field(), android::util::SCREEN_BRIGHTNESS_CHANGED); EXPECT_EQ(data.dimensions_in_what().value_tuple().dimensions_value_size(), 1); @@ -164,7 +159,8 @@ TEST(DimensionInConditionE2eTest, TestCountMetricNoLink) { EXPECT_EQ(data.dimensions_in_what().value_tuple().dimensions_value_size(), 1); EXPECT_EQ(data.dimensions_in_what().value_tuple().dimensions_value(0).field(), 1); EXPECT_EQ(data.dimensions_in_what().value_tuple().dimensions_value(0).value_int(), 123); - ValidateAttributionUidDimension(data.dimensions_in_condition(), android::util::WAKELOCK_STATE_CHANGED, 111); + ValidateAttributionUidDimension(data.dimensions_in_condition(), + android::util::WAKELOCK_STATE_CHANGED, 111); data = countMetrics.data(2); EXPECT_EQ(data.bucket_info_size(), 1); @@ -175,7 +171,8 @@ TEST(DimensionInConditionE2eTest, TestCountMetricNoLink) { EXPECT_EQ(data.dimensions_in_what().value_tuple().dimensions_value_size(), 1); EXPECT_EQ(data.dimensions_in_what().value_tuple().dimensions_value(0).field(), 1); EXPECT_EQ(data.dimensions_in_what().value_tuple().dimensions_value(0).value_int(), 456); - ValidateAttributionUidDimension(data.dimensions_in_condition(), android::util::WAKELOCK_STATE_CHANGED, 111); + ValidateAttributionUidDimension(data.dimensions_in_condition(), + android::util::WAKELOCK_STATE_CHANGED, 111); data = countMetrics.data(3); EXPECT_EQ(data.bucket_info_size(), 2); @@ -189,7 +186,8 @@ TEST(DimensionInConditionE2eTest, TestCountMetricNoLink) { EXPECT_EQ(data.dimensions_in_what().value_tuple().dimensions_value_size(), 1); EXPECT_EQ(data.dimensions_in_what().value_tuple().dimensions_value(0).field(), 1); EXPECT_EQ(data.dimensions_in_what().value_tuple().dimensions_value(0).value_int(), 456); - ValidateAttributionUidDimension(data.dimensions_in_condition(), android::util::WAKELOCK_STATE_CHANGED, 333); + ValidateAttributionUidDimension(data.dimensions_in_condition(), + android::util::WAKELOCK_STATE_CHANGED, 333); data = countMetrics.data(4); EXPECT_EQ(data.bucket_info_size(), 1); @@ -211,7 +209,8 @@ TEST(DimensionInConditionE2eTest, TestCountMetricNoLink) { EXPECT_EQ(data.dimensions_in_what().value_tuple().dimensions_value_size(), 1); EXPECT_EQ(data.dimensions_in_what().value_tuple().dimensions_value(0).field(), 1); EXPECT_EQ(data.dimensions_in_what().value_tuple().dimensions_value(0).value_int(), 789); - ValidateAttributionUidDimension(data.dimensions_in_condition(), android::util::WAKELOCK_STATE_CHANGED, 111); + ValidateAttributionUidDimension(data.dimensions_in_condition(), + android::util::WAKELOCK_STATE_CHANGED, 111); data = countMetrics.data(6); EXPECT_EQ(data.bucket_info_size(), 1); @@ -222,7 +221,8 @@ TEST(DimensionInConditionE2eTest, TestCountMetricNoLink) { EXPECT_EQ(data.dimensions_in_what().value_tuple().dimensions_value_size(), 1); EXPECT_EQ(data.dimensions_in_what().value_tuple().dimensions_value(0).field(), 1); EXPECT_EQ(data.dimensions_in_what().value_tuple().dimensions_value(0).value_int(), 789); - ValidateAttributionUidDimension(data.dimensions_in_condition(), android::util::WAKELOCK_STATE_CHANGED, 333); + ValidateAttributionUidDimension(data.dimensions_in_condition(), + android::util::WAKELOCK_STATE_CHANGED, 333); } namespace { @@ -239,8 +239,8 @@ StatsdConfig CreateCountMetricWithLinkConfig() { auto screenIsOffPredicate = CreateScreenIsOffPredicate(); auto isSyncingPredicate = CreateIsSyncingPredicate(); auto syncDimension = isSyncingPredicate.mutable_simple_predicate()->mutable_dimensions(); - *syncDimension = CreateAttributionUidAndTagDimensions( - android::util::SYNC_STATE_CHANGED, {Position::FIRST}); + *syncDimension = CreateAttributionUidAndTagDimensions(android::util::SYNC_STATE_CHANGED, + {Position::FIRST}); syncDimension->add_child()->set_field(2 /* name field*/); *config.add_predicate() = screenIsOffPredicate; @@ -256,8 +256,8 @@ StatsdConfig CreateCountMetricWithLinkConfig() { metric->set_id(StringToId("AppCrashMetric")); metric->set_what(appCrashMatcher.id()); metric->set_condition(combinationPredicate->id()); - *metric->mutable_dimensions_in_what() = CreateDimensions( - android::util::PROCESS_LIFE_CYCLE_STATE_CHANGED, {1 /* uid */}); + *metric->mutable_dimensions_in_what() = + CreateDimensions(android::util::PROCESS_LIFE_CYCLE_STATE_CHANGED, {1 /* uid */}); *metric->mutable_dimensions_in_condition() = CreateAttributionUidAndTagDimensions( android::util::SYNC_STATE_CHANGED, {Position::FIRST}); @@ -267,8 +267,8 @@ StatsdConfig CreateCountMetricWithLinkConfig() { auto dimensionWhat = links->mutable_fields_in_what(); dimensionWhat->set_field(android::util::PROCESS_LIFE_CYCLE_STATE_CHANGED); dimensionWhat->add_child()->set_field(1); // uid field. - *links->mutable_fields_in_condition() = CreateAttributionUidDimensions( - android::util::SYNC_STATE_CHANGED, {Position::FIRST}); + *links->mutable_fields_in_condition() = + CreateAttributionUidDimensions(android::util::SYNC_STATE_CHANGED, {Position::FIRST}); return config; } @@ -279,18 +279,18 @@ TEST(DimensionInConditionE2eTest, TestCountMetricWithLink) { auto config = CreateCountMetricWithLinkConfig(); int64_t bucketStartTimeNs = 10000000000; int64_t bucketSizeNs = - TimeUnitToBucketSizeInMillis(config.count_metric(0).bucket()) * 1000000LL; + TimeUnitToBucketSizeInMillis(config.count_metric(0).bucket()) * 1000000LL; auto processor = CreateStatsLogProcessor(bucketStartTimeNs / NS_PER_SEC, config, cfgKey); EXPECT_EQ(processor->mMetricsManagers.size(), 1u); EXPECT_TRUE(processor->mMetricsManagers.begin()->second->isConfigValid()); - std::vector attributions1 = - {CreateAttribution(111, "App1"), CreateAttribution(222, "GMSCoreModule1"), - CreateAttribution(222, "GMSCoreModule2")}; + std::vector attributions1 = {CreateAttribution(111, "App1"), + CreateAttribution(222, "GMSCoreModule1"), + CreateAttribution(222, "GMSCoreModule2")}; - std::vector attributions2 = - {CreateAttribution(333, "App2"), CreateAttribution(222, "GMSCoreModule1"), - CreateAttribution(555, "GMSCoreModule2")}; + std::vector attributions2 = {CreateAttribution(333, "App2"), + CreateAttribution(222, "GMSCoreModule1"), + CreateAttribution(555, "GMSCoreModule2")}; std::vector> events; @@ -311,26 +311,26 @@ TEST(DimensionInConditionE2eTest, TestCountMetricWithLink) { events.push_back(CreateAppCrashEvent(777, bucketStartTimeNs + bucketSizeNs + 701)); - events.push_back(CreateScreenStateChangedEvent( - android::view::DISPLAY_STATE_ON, bucketStartTimeNs + 10)); - events.push_back(CreateScreenStateChangedEvent( - android::view::DISPLAY_STATE_OFF, bucketStartTimeNs + 100)); - events.push_back(CreateScreenStateChangedEvent( - android::view::DISPLAY_STATE_ON, bucketStartTimeNs + 202)); - events.push_back(CreateScreenStateChangedEvent( - android::view::DISPLAY_STATE_OFF, bucketStartTimeNs + bucketSizeNs + 700)); + events.push_back( + CreateScreenStateChangedEvent(android::view::DISPLAY_STATE_ON, bucketStartTimeNs + 10)); + events.push_back(CreateScreenStateChangedEvent(android::view::DISPLAY_STATE_OFF, + bucketStartTimeNs + 100)); + events.push_back(CreateScreenStateChangedEvent(android::view::DISPLAY_STATE_ON, + bucketStartTimeNs + 202)); + events.push_back(CreateScreenStateChangedEvent(android::view::DISPLAY_STATE_OFF, + bucketStartTimeNs + bucketSizeNs + 700)); events.push_back(CreateSyncStartEvent(attributions1, "ReadEmail", bucketStartTimeNs + 200)); - events.push_back(CreateSyncEndEvent(attributions1, "ReadEmail", - bucketStartTimeNs + bucketSizeNs + 300)); + events.push_back( + CreateSyncEndEvent(attributions1, "ReadEmail", bucketStartTimeNs + bucketSizeNs + 300)); events.push_back(CreateSyncStartEvent(attributions1, "ReadDoc", bucketStartTimeNs + 400)); - events.push_back(CreateSyncEndEvent(attributions1, "ReadDoc", - bucketStartTimeNs + bucketSizeNs - 1)); + events.push_back( + CreateSyncEndEvent(attributions1, "ReadDoc", bucketStartTimeNs + bucketSizeNs - 1)); events.push_back(CreateSyncStartEvent(attributions2, "ReadEmail", bucketStartTimeNs + 400)); - events.push_back(CreateSyncEndEvent(attributions2, "ReadEmail", - bucketStartTimeNs + bucketSizeNs + 600)); + events.push_back( + CreateSyncEndEvent(attributions2, "ReadEmail", bucketStartTimeNs + bucketSizeNs + 600)); sortLogEventsByTimestamp(&events); @@ -339,7 +339,10 @@ TEST(DimensionInConditionE2eTest, TestCountMetricWithLink) { } ConfigMetricsReportList reports; - processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, &reports); + vector buffer; + processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, &buffer); + EXPECT_TRUE(buffer.size() > 0); + EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size())); EXPECT_EQ(reports.reports_size(), 1); EXPECT_EQ(reports.reports(0).metrics_size(), 1); @@ -363,8 +366,8 @@ TEST(DimensionInConditionE2eTest, TestCountMetricWithLink) { EXPECT_EQ(data.dimensions_in_what().value_tuple().dimensions_value_size(), 1); EXPECT_EQ(data.dimensions_in_what().value_tuple().dimensions_value(0).field(), 1); EXPECT_EQ(data.dimensions_in_what().value_tuple().dimensions_value(0).value_int(), 111); - ValidateAttributionUidAndTagDimension( - data.dimensions_in_condition(), android::util::SYNC_STATE_CHANGED, 111, "App1"); + ValidateAttributionUidAndTagDimension(data.dimensions_in_condition(), + android::util::SYNC_STATE_CHANGED, 111, "App1"); EXPECT_EQ(data.bucket_info_size(), 1); EXPECT_EQ(data.bucket_info(0).count(), 2); EXPECT_EQ(data.bucket_info(0).start_bucket_nanos(), bucketStartTimeNs); @@ -386,8 +389,8 @@ TEST(DimensionInConditionE2eTest, TestCountMetricWithLink) { EXPECT_EQ(data.dimensions_in_what().value_tuple().dimensions_value_size(), 1); EXPECT_EQ(data.dimensions_in_what().value_tuple().dimensions_value(0).field(), 1); EXPECT_EQ(data.dimensions_in_what().value_tuple().dimensions_value(0).value_int(), 333); - ValidateAttributionUidAndTagDimension( - data.dimensions_in_condition(), android::util::SYNC_STATE_CHANGED, 333, "App2"); + ValidateAttributionUidAndTagDimension(data.dimensions_in_condition(), + android::util::SYNC_STATE_CHANGED, 333, "App2"); EXPECT_EQ(data.bucket_info_size(), 2); EXPECT_EQ(data.bucket_info(0).count(), 1); EXPECT_EQ(data.bucket_info(0).start_bucket_nanos(), bucketStartTimeNs); @@ -424,8 +427,8 @@ StatsdConfig CreateDurationMetricConfigNoLink(DurationMetric::AggregationType ag auto screenIsOffPredicate = CreateScreenIsOffPredicate(); auto isSyncingPredicate = CreateIsSyncingPredicate(); auto syncDimension = isSyncingPredicate.mutable_simple_predicate()->mutable_dimensions(); - *syncDimension = CreateAttributionUidAndTagDimensions( - android::util::SYNC_STATE_CHANGED, {Position::FIRST}); + *syncDimension = CreateAttributionUidAndTagDimensions(android::util::SYNC_STATE_CHANGED, + {Position::FIRST}); syncDimension->add_child()->set_field(2 /* name field */); *config.add_predicate() = inBatterySaverModePredicate; @@ -449,26 +452,25 @@ StatsdConfig CreateDurationMetricConfigNoLink(DurationMetric::AggregationType ag } // namespace - TEST(DimensionInConditionE2eTest, TestDurationMetricNoLink) { - for (auto aggregationType : { DurationMetric::SUM, DurationMetric::MAX_SPARSE}) { + for (auto aggregationType : {DurationMetric::SUM, DurationMetric::MAX_SPARSE}) { ConfigKey cfgKey; auto config = CreateDurationMetricConfigNoLink(aggregationType); int64_t bucketStartTimeNs = 10000000000; int64_t bucketSizeNs = - TimeUnitToBucketSizeInMillis(config.duration_metric(0).bucket()) * 1000000LL; + TimeUnitToBucketSizeInMillis(config.duration_metric(0).bucket()) * 1000000LL; auto processor = CreateStatsLogProcessor(bucketStartTimeNs / NS_PER_SEC, config, cfgKey); EXPECT_EQ(processor->mMetricsManagers.size(), 1u); EXPECT_TRUE(processor->mMetricsManagers.begin()->second->isConfigValid()); - std::vector attributions1 = - {CreateAttribution(111, "App1"), CreateAttribution(222, "GMSCoreModule1"), - CreateAttribution(222, "GMSCoreModule2")}; + std::vector attributions1 = {CreateAttribution(111, "App1"), + CreateAttribution(222, "GMSCoreModule1"), + CreateAttribution(222, "GMSCoreModule2")}; - std::vector attributions2 = - {CreateAttribution(333, "App2"), CreateAttribution(222, "GMSCoreModule1"), - CreateAttribution(555, "GMSCoreModule2")}; + std::vector attributions2 = {CreateAttribution(333, "App2"), + CreateAttribution(222, "GMSCoreModule1"), + CreateAttribution(555, "GMSCoreModule2")}; std::vector> events; @@ -485,26 +487,26 @@ TEST(DimensionInConditionE2eTest, TestDurationMetricNoLink) { events.push_back(CreateBatterySaverOnEvent(bucketStartTimeNs + bucketSizeNs + 870)); events.push_back(CreateBatterySaverOffEvent(bucketStartTimeNs + bucketSizeNs + 900)); - events.push_back(CreateScreenStateChangedEvent( - android::view::DISPLAY_STATE_ON, bucketStartTimeNs + 10)); - events.push_back(CreateScreenStateChangedEvent( - android::view::DISPLAY_STATE_OFF, bucketStartTimeNs + 100)); - events.push_back(CreateScreenStateChangedEvent( - android::view::DISPLAY_STATE_ON, bucketStartTimeNs + 202)); - events.push_back(CreateScreenStateChangedEvent( - android::view::DISPLAY_STATE_OFF, bucketStartTimeNs + bucketSizeNs + 800)); + events.push_back(CreateScreenStateChangedEvent(android::view::DISPLAY_STATE_ON, + bucketStartTimeNs + 10)); + events.push_back(CreateScreenStateChangedEvent(android::view::DISPLAY_STATE_OFF, + bucketStartTimeNs + 100)); + events.push_back(CreateScreenStateChangedEvent(android::view::DISPLAY_STATE_ON, + bucketStartTimeNs + 202)); + events.push_back(CreateScreenStateChangedEvent(android::view::DISPLAY_STATE_OFF, + bucketStartTimeNs + bucketSizeNs + 800)); events.push_back(CreateSyncStartEvent(attributions1, "ReadEmail", bucketStartTimeNs + 200)); events.push_back(CreateSyncEndEvent(attributions1, "ReadEmail", - bucketStartTimeNs + bucketSizeNs + 300)); + bucketStartTimeNs + bucketSizeNs + 300)); events.push_back(CreateSyncStartEvent(attributions1, "ReadDoc", bucketStartTimeNs + 400)); - events.push_back(CreateSyncEndEvent(attributions1, "ReadDoc", - bucketStartTimeNs + bucketSizeNs - 1)); + events.push_back( + CreateSyncEndEvent(attributions1, "ReadDoc", bucketStartTimeNs + bucketSizeNs - 1)); events.push_back(CreateSyncStartEvent(attributions2, "ReadEmail", bucketStartTimeNs + 401)); events.push_back(CreateSyncEndEvent(attributions2, "ReadEmail", - bucketStartTimeNs + bucketSizeNs + 700)); + bucketStartTimeNs + bucketSizeNs + 700)); sortLogEventsByTimestamp(&events); @@ -513,7 +515,10 @@ TEST(DimensionInConditionE2eTest, TestDurationMetricNoLink) { } ConfigMetricsReportList reports; - processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, &reports); + vector buffer; + processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, &buffer); + EXPECT_TRUE(buffer.size() > 0); + EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size())); EXPECT_EQ(reports.reports_size(), 1); EXPECT_EQ(reports.reports(0).metrics_size(), 1); @@ -534,8 +539,8 @@ TEST(DimensionInConditionE2eTest, TestDurationMetricNoLink) { data = metrics.data(1); EXPECT_FALSE(data.dimensions_in_what().has_field()); - ValidateAttributionUidAndTagDimension( - data.dimensions_in_condition(), android::util::SYNC_STATE_CHANGED, 111, "App1"); + ValidateAttributionUidAndTagDimension(data.dimensions_in_condition(), + android::util::SYNC_STATE_CHANGED, 111, "App1"); EXPECT_EQ(data.bucket_info_size(), 2); EXPECT_EQ(data.bucket_info(0).duration_nanos(), 500 - 201 + bucketSizeNs - 600); EXPECT_EQ(data.bucket_info(0).start_bucket_nanos(), bucketStartTimeNs); @@ -546,8 +551,8 @@ TEST(DimensionInConditionE2eTest, TestDurationMetricNoLink) { data = metrics.data(2); EXPECT_FALSE(data.dimensions_in_what().has_field()); - ValidateAttributionUidAndTagDimension( - data.dimensions_in_condition(), android::util::SYNC_STATE_CHANGED, 333, "App2"); + ValidateAttributionUidAndTagDimension(data.dimensions_in_condition(), + android::util::SYNC_STATE_CHANGED, 333, "App2"); EXPECT_EQ(data.bucket_info_size(), 2); EXPECT_EQ(data.bucket_info(0).duration_nanos(), 500 - 401 + bucketSizeNs - 600); EXPECT_EQ(data.bucket_info(0).start_bucket_nanos(), bucketStartTimeNs); @@ -572,13 +577,13 @@ StatsdConfig CreateDurationMetricConfigWithLink(DurationMetric::AggregationType auto screenIsOffPredicate = CreateScreenIsOffPredicate(); auto isSyncingPredicate = CreateIsSyncingPredicate(); auto syncDimension = isSyncingPredicate.mutable_simple_predicate()->mutable_dimensions(); - *syncDimension = CreateAttributionUidAndTagDimensions( - android::util::SYNC_STATE_CHANGED, {Position::FIRST}); + *syncDimension = CreateAttributionUidAndTagDimensions(android::util::SYNC_STATE_CHANGED, + {Position::FIRST}); syncDimension->add_child()->set_field(2 /* name field */); auto isInBackgroundPredicate = CreateIsInBackgroundPredicate(); *isInBackgroundPredicate.mutable_simple_predicate()->mutable_dimensions() = - CreateDimensions(android::util::ACTIVITY_FOREGROUND_STATE_CHANGED, {1 /* uid field */ }); + CreateDimensions(android::util::ACTIVITY_FOREGROUND_STATE_CHANGED, {1 /* uid field */}); *config.add_predicate() = screenIsOffPredicate; *config.add_predicate() = isSyncingPredicate; @@ -594,8 +599,8 @@ StatsdConfig CreateDurationMetricConfigWithLink(DurationMetric::AggregationType metric->set_id(StringToId("AppInBackgroundMetric")); metric->set_what(isInBackgroundPredicate.id()); metric->set_condition(combinationPredicate->id()); - *metric->mutable_dimensions_in_what() = CreateDimensions( - android::util::ACTIVITY_FOREGROUND_STATE_CHANGED, {1 /* uid field */ }); + *metric->mutable_dimensions_in_what() = + CreateDimensions(android::util::ACTIVITY_FOREGROUND_STATE_CHANGED, {1 /* uid field */}); *metric->mutable_dimensions_in_condition() = CreateAttributionUidAndTagDimensions( android::util::SYNC_STATE_CHANGED, {Position::FIRST}); @@ -605,32 +610,32 @@ StatsdConfig CreateDurationMetricConfigWithLink(DurationMetric::AggregationType auto dimensionWhat = links->mutable_fields_in_what(); dimensionWhat->set_field(android::util::ACTIVITY_FOREGROUND_STATE_CHANGED); dimensionWhat->add_child()->set_field(1); // uid field. - *links->mutable_fields_in_condition() = CreateAttributionUidDimensions( - android::util::SYNC_STATE_CHANGED, {Position::FIRST}); + *links->mutable_fields_in_condition() = + CreateAttributionUidDimensions(android::util::SYNC_STATE_CHANGED, {Position::FIRST}); return config; } } // namespace TEST(DimensionInConditionE2eTest, TestDurationMetricWithLink) { - for (auto aggregationType : { DurationMetric::SUM, DurationMetric::MAX_SPARSE}) { + for (auto aggregationType : {DurationMetric::SUM, DurationMetric::MAX_SPARSE}) { ConfigKey cfgKey; auto config = CreateDurationMetricConfigWithLink(aggregationType); int64_t bucketStartTimeNs = 10000000000; int64_t bucketSizeNs = - TimeUnitToBucketSizeInMillis(config.duration_metric(0).bucket()) * 1000000LL; + TimeUnitToBucketSizeInMillis(config.duration_metric(0).bucket()) * 1000000LL; auto processor = CreateStatsLogProcessor(bucketStartTimeNs / NS_PER_SEC, config, cfgKey); EXPECT_EQ(processor->mMetricsManagers.size(), 1u); EXPECT_TRUE(processor->mMetricsManagers.begin()->second->isConfigValid()); - std::vector attributions1 = - {CreateAttribution(111, "App1"), CreateAttribution(222, "GMSCoreModule1"), - CreateAttribution(222, "GMSCoreModule2")}; + std::vector attributions1 = {CreateAttribution(111, "App1"), + CreateAttribution(222, "GMSCoreModule1"), + CreateAttribution(222, "GMSCoreModule2")}; - std::vector attributions2 = - {CreateAttribution(333, "App2"), CreateAttribution(222, "GMSCoreModule1"), - CreateAttribution(555, "GMSCoreModule2")}; + std::vector attributions2 = {CreateAttribution(333, "App2"), + CreateAttribution(222, "GMSCoreModule1"), + CreateAttribution(555, "GMSCoreModule2")}; std::vector> events; @@ -643,26 +648,26 @@ TEST(DimensionInConditionE2eTest, TestDurationMetricWithLink) { events.push_back(CreateMoveToBackgroundEvent(333, bucketStartTimeNs + 399)); events.push_back(CreateMoveToForegroundEvent(333, bucketStartTimeNs + bucketSizeNs + 800)); - events.push_back(CreateScreenStateChangedEvent( - android::view::DISPLAY_STATE_ON, bucketStartTimeNs + 10)); - events.push_back(CreateScreenStateChangedEvent( - android::view::DISPLAY_STATE_OFF, bucketStartTimeNs + 100)); - events.push_back(CreateScreenStateChangedEvent( - android::view::DISPLAY_STATE_ON, bucketStartTimeNs + 202)); - events.push_back(CreateScreenStateChangedEvent( - android::view::DISPLAY_STATE_OFF, bucketStartTimeNs + bucketSizeNs + 801)); + events.push_back(CreateScreenStateChangedEvent(android::view::DISPLAY_STATE_ON, + bucketStartTimeNs + 10)); + events.push_back(CreateScreenStateChangedEvent(android::view::DISPLAY_STATE_OFF, + bucketStartTimeNs + 100)); + events.push_back(CreateScreenStateChangedEvent(android::view::DISPLAY_STATE_ON, + bucketStartTimeNs + 202)); + events.push_back(CreateScreenStateChangedEvent(android::view::DISPLAY_STATE_OFF, + bucketStartTimeNs + bucketSizeNs + 801)); events.push_back(CreateSyncStartEvent(attributions1, "ReadEmail", bucketStartTimeNs + 200)); events.push_back(CreateSyncEndEvent(attributions1, "ReadEmail", - bucketStartTimeNs + bucketSizeNs + 300)); + bucketStartTimeNs + bucketSizeNs + 300)); events.push_back(CreateSyncStartEvent(attributions1, "ReadDoc", bucketStartTimeNs + 400)); - events.push_back(CreateSyncEndEvent(attributions1, "ReadDoc", - bucketStartTimeNs + bucketSizeNs - 1)); + events.push_back( + CreateSyncEndEvent(attributions1, "ReadDoc", bucketStartTimeNs + bucketSizeNs - 1)); events.push_back(CreateSyncStartEvent(attributions2, "ReadEmail", bucketStartTimeNs + 401)); events.push_back(CreateSyncEndEvent(attributions2, "ReadEmail", - bucketStartTimeNs + bucketSizeNs + 700)); + bucketStartTimeNs + bucketSizeNs + 700)); sortLogEventsByTimestamp(&events); @@ -671,7 +676,10 @@ TEST(DimensionInConditionE2eTest, TestDurationMetricWithLink) { } ConfigMetricsReportList reports; - processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, &reports); + vector buffer; + processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, &buffer); + EXPECT_TRUE(buffer.size() > 0); + EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size())); EXPECT_EQ(reports.reports_size(), 1); EXPECT_EQ(reports.reports(0).metrics_size(), 1); @@ -691,8 +699,8 @@ TEST(DimensionInConditionE2eTest, TestDurationMetricWithLink) { data = metrics.data(1); EXPECT_EQ(data.dimensions_in_what().value_tuple().dimensions_value(0).field(), 1); EXPECT_EQ(data.dimensions_in_what().value_tuple().dimensions_value(0).value_int(), 111); - ValidateAttributionUidAndTagDimension( - data.dimensions_in_condition(), android::util::SYNC_STATE_CHANGED, 111, "App1"); + ValidateAttributionUidAndTagDimension(data.dimensions_in_condition(), + android::util::SYNC_STATE_CHANGED, 111, "App1"); EXPECT_EQ(data.bucket_info_size(), 2); EXPECT_EQ(data.bucket_info(0).duration_nanos(), bucketSizeNs - 201); EXPECT_EQ(data.bucket_info(0).start_bucket_nanos(), bucketStartTimeNs); @@ -704,8 +712,8 @@ TEST(DimensionInConditionE2eTest, TestDurationMetricWithLink) { data = metrics.data(2); EXPECT_EQ(data.dimensions_in_what().value_tuple().dimensions_value(0).field(), 1); EXPECT_EQ(data.dimensions_in_what().value_tuple().dimensions_value(0).value_int(), 333); - ValidateAttributionUidAndTagDimension( - data.dimensions_in_condition(), android::util::SYNC_STATE_CHANGED, 333, "App2"); + ValidateAttributionUidAndTagDimension(data.dimensions_in_condition(), + android::util::SYNC_STATE_CHANGED, 333, "App2"); EXPECT_EQ(data.bucket_info_size(), 2); EXPECT_EQ(data.bucket_info(0).duration_nanos(), bucketSizeNs - 401); EXPECT_EQ(data.bucket_info(0).start_bucket_nanos(), bucketStartTimeNs); diff --git a/cmds/statsd/tests/e2e/GaugeMetric_e2e_test.cpp b/cmds/statsd/tests/e2e/GaugeMetric_e2e_test.cpp index a80fdc5606b75..674d810a9ea9a 100644 --- a/cmds/statsd/tests/e2e/GaugeMetric_e2e_test.cpp +++ b/cmds/statsd/tests/e2e/GaugeMetric_e2e_test.cpp @@ -140,7 +140,10 @@ TEST(GaugeMetricE2eTest, TestMultipleFieldsForPushedEvent) { processor->OnLogEvent(event.get()); } ConfigMetricsReportList reports; - processor->onDumpReport(cfgKey, bucketStartTimeNs + 3 * bucketSizeNs, &reports); + vector buffer; + processor->onDumpReport(cfgKey, bucketStartTimeNs + 3 * bucketSizeNs, &buffer); + EXPECT_TRUE(buffer.size() > 0); + EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size())); EXPECT_EQ(reports.reports_size(), 1); EXPECT_EQ(reports.reports(0).metrics_size(), 1); StatsLogReport::GaugeMetricDataWrapper gaugeMetrics; diff --git a/cmds/statsd/tests/e2e/MetricConditionLink_e2e_test.cpp b/cmds/statsd/tests/e2e/MetricConditionLink_e2e_test.cpp index 233031c5c8da4..d00518147bfed 100644 --- a/cmds/statsd/tests/e2e/MetricConditionLink_e2e_test.cpp +++ b/cmds/statsd/tests/e2e/MetricConditionLink_e2e_test.cpp @@ -95,8 +95,10 @@ StatsdConfig CreateStatsdConfig() { } } // namespace - -TEST(MetricConditionLinkE2eTest, TestMultiplePredicatesAndLinks) { +// If we want to test multiple dump data, we must do it in separate tests, because in the e2e tests, +// we should use the real API which will clear the data after dump data is called. +// TODO: better refactor the code so that the tests are not so verbose. +TEST(MetricConditionLinkE2eTest, TestMultiplePredicatesAndLinks1) { auto config = CreateStatsdConfig(); uint64_t bucketStartTimeNs = 10000000000; uint64_t bucketSizeNs = @@ -195,7 +197,10 @@ TEST(MetricConditionLinkE2eTest, TestMultiplePredicatesAndLinks) { processor->OnLogEvent(event.get()); } ConfigMetricsReportList reports; - processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs - 1, &reports); + vector buffer; + processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs - 1, &buffer); + EXPECT_TRUE(buffer.size() > 0); + EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size())); EXPECT_EQ(reports.reports_size(), 1); EXPECT_EQ(reports.reports(0).metrics_size(), 1); EXPECT_EQ(reports.reports(0).metrics(0).count_metrics().data_size(), 1); @@ -208,16 +213,115 @@ TEST(MetricConditionLinkE2eTest, TestMultiplePredicatesAndLinks) { // Uid field. EXPECT_EQ(data.dimensions_in_what().value_tuple().dimensions_value(0).field(), 1); EXPECT_EQ(data.dimensions_in_what().value_tuple().dimensions_value(0).value_int(), appUid); +} - reports.Clear(); - processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, &reports); +TEST(MetricConditionLinkE2eTest, TestMultiplePredicatesAndLinks2) { + auto config = CreateStatsdConfig(); + uint64_t bucketStartTimeNs = 10000000000; + uint64_t bucketSizeNs = + TimeUnitToBucketSizeInMillis(config.count_metric(0).bucket()) * 1000000LL; + + ConfigKey cfgKey; + auto processor = CreateStatsLogProcessor(bucketStartTimeNs / NS_PER_SEC, config, cfgKey); + EXPECT_EQ(processor->mMetricsManagers.size(), 1u); + EXPECT_TRUE(processor->mMetricsManagers.begin()->second->isConfigValid()); + + int appUid = 123; + auto crashEvent1 = CreateAppCrashEvent(appUid, bucketStartTimeNs + 1); + auto crashEvent2 = CreateAppCrashEvent(appUid, bucketStartTimeNs + 201); + auto crashEvent3 = CreateAppCrashEvent(appUid, bucketStartTimeNs + 2 * bucketSizeNs - 101); + + auto crashEvent4 = CreateAppCrashEvent(appUid, bucketStartTimeNs + 51); + auto crashEvent5 = CreateAppCrashEvent(appUid, bucketStartTimeNs + bucketSizeNs + 299); + auto crashEvent6 = CreateAppCrashEvent(appUid, bucketStartTimeNs + bucketSizeNs + 2001); + + auto crashEvent7 = CreateAppCrashEvent(appUid, bucketStartTimeNs + 16); + auto crashEvent8 = CreateAppCrashEvent(appUid, bucketStartTimeNs + bucketSizeNs + 249); + + auto crashEvent9 = CreateAppCrashEvent(appUid, bucketStartTimeNs + bucketSizeNs + 351); + auto crashEvent10 = CreateAppCrashEvent(appUid, bucketStartTimeNs + 2 * bucketSizeNs - 2); + + auto screenTurnedOnEvent = CreateScreenStateChangedEvent( + android::view::DisplayStateEnum::DISPLAY_STATE_ON, bucketStartTimeNs + 2); + auto screenTurnedOffEvent = CreateScreenStateChangedEvent( + android::view::DisplayStateEnum::DISPLAY_STATE_OFF, bucketStartTimeNs + 200); + auto screenTurnedOnEvent2 = + CreateScreenStateChangedEvent(android::view::DisplayStateEnum::DISPLAY_STATE_ON, + bucketStartTimeNs + 2 * bucketSizeNs - 100); + + std::vector attributions = {CreateAttribution(appUid, "App1"), + CreateAttribution(appUid + 1, "GMSCoreModule1")}; + auto syncOnEvent1 = CreateSyncStartEvent(attributions, "ReadEmail", bucketStartTimeNs + 50); + auto syncOffEvent1 = + CreateSyncEndEvent(attributions, "ReadEmail", bucketStartTimeNs + bucketSizeNs + 300); + auto syncOnEvent2 = + CreateSyncStartEvent(attributions, "ReadDoc", bucketStartTimeNs + bucketSizeNs + 2000); + + auto moveToBackgroundEvent1 = CreateMoveToBackgroundEvent(appUid, bucketStartTimeNs + 15); + auto moveToForegroundEvent1 = + CreateMoveToForegroundEvent(appUid, bucketStartTimeNs + bucketSizeNs + 250); + + auto moveToBackgroundEvent2 = + CreateMoveToBackgroundEvent(appUid, bucketStartTimeNs + bucketSizeNs + 350); + auto moveToForegroundEvent2 = + CreateMoveToForegroundEvent(appUid, bucketStartTimeNs + 2 * bucketSizeNs - 1); + + /* + bucket #1 bucket #2 + + + | | | | | | | | | | (crashEvents) + |-------------------------------------|-----------------------------------|--------- + + | | (MoveToBkground) + + | | (MoveToForeground) + + | | (SyncIsOn) + | (SyncIsOff) + | | (ScreenIsOn) + | (ScreenIsOff) + */ + std::vector> events; + events.push_back(std::move(crashEvent1)); + events.push_back(std::move(crashEvent2)); + events.push_back(std::move(crashEvent3)); + events.push_back(std::move(crashEvent4)); + events.push_back(std::move(crashEvent5)); + events.push_back(std::move(crashEvent6)); + events.push_back(std::move(crashEvent7)); + events.push_back(std::move(crashEvent8)); + events.push_back(std::move(crashEvent9)); + events.push_back(std::move(crashEvent10)); + events.push_back(std::move(screenTurnedOnEvent)); + events.push_back(std::move(screenTurnedOffEvent)); + events.push_back(std::move(screenTurnedOnEvent2)); + events.push_back(std::move(syncOnEvent1)); + events.push_back(std::move(syncOffEvent1)); + events.push_back(std::move(syncOnEvent2)); + events.push_back(std::move(moveToBackgroundEvent1)); + events.push_back(std::move(moveToForegroundEvent1)); + events.push_back(std::move(moveToBackgroundEvent2)); + events.push_back(std::move(moveToForegroundEvent2)); + + sortLogEventsByTimestamp(&events); + + for (const auto& event : events) { + processor->OnLogEvent(event.get()); + } + ConfigMetricsReportList reports; + vector buffer; + + processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, &buffer); + EXPECT_TRUE(buffer.size() > 0); + EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size())); EXPECT_EQ(reports.reports_size(), 1); EXPECT_EQ(reports.reports(0).metrics_size(), 1); EXPECT_EQ(reports.reports(0).metrics(0).count_metrics().data_size(), 1); EXPECT_EQ(reports.reports(0).metrics(0).count_metrics().data(0).bucket_info_size(), 2); EXPECT_EQ(reports.reports(0).metrics(0).count_metrics().data(0).bucket_info(0).count(), 1); EXPECT_EQ(reports.reports(0).metrics(0).count_metrics().data(0).bucket_info(1).count(), 3); - data = reports.reports(0).metrics(0).count_metrics().data(0); + auto data = reports.reports(0).metrics(0).count_metrics().data(0); // Validate dimension value. EXPECT_EQ(data.dimensions_in_what().field(), android::util::PROCESS_LIFE_CYCLE_STATE_CHANGED); diff --git a/cmds/statsd/tests/e2e/WakelockDuration_e2e_test.cpp b/cmds/statsd/tests/e2e/WakelockDuration_e2e_test.cpp index a99dbe85e7a50..3b25694b65174 100644 --- a/cmds/statsd/tests/e2e/WakelockDuration_e2e_test.cpp +++ b/cmds/statsd/tests/e2e/WakelockDuration_e2e_test.cpp @@ -114,7 +114,7 @@ void FeedEvents(StatsdConfig config, sp processor) { } // namespace -TEST(WakelockDurationE2eTest, TestAggregatedPredicateDimensionsForSumDuration) { +TEST(WakelockDurationE2eTest, TestAggregatedPredicateDimensionsForSumDuration1) { ConfigKey cfgKey; auto config = CreateStatsdConfig(DurationMetric::SUM); uint64_t bucketStartTimeNs = 10000000000; @@ -124,8 +124,11 @@ TEST(WakelockDurationE2eTest, TestAggregatedPredicateDimensionsForSumDuration) { EXPECT_EQ(processor->mMetricsManagers.size(), 1u); EXPECT_TRUE(processor->mMetricsManagers.begin()->second->isConfigValid()); FeedEvents(config, processor); + vector buffer; ConfigMetricsReportList reports; - processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs - 1, &reports); + processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs - 1, &buffer); + EXPECT_TRUE(buffer.size() > 0); + EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size())); EXPECT_EQ(reports.reports_size(), 1); EXPECT_EQ(reports.reports(0).metrics_size(), 1); @@ -142,15 +145,30 @@ TEST(WakelockDurationE2eTest, TestAggregatedPredicateDimensionsForSumDuration) { // The wakelock holding interval starts from the screen off event and to the end of the 1st // bucket. EXPECT_EQ((unsigned long long)data.bucket_info(0).duration_nanos(), bucketSizeNs - 200); +} - reports.Clear(); - processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, &reports); +TEST(WakelockDurationE2eTest, TestAggregatedPredicateDimensionsForSumDuration2) { + ConfigKey cfgKey; + auto config = CreateStatsdConfig(DurationMetric::SUM); + uint64_t bucketStartTimeNs = 10000000000; + uint64_t bucketSizeNs = + TimeUnitToBucketSizeInMillis(config.duration_metric(0).bucket()) * 1000000LL; + auto processor = CreateStatsLogProcessor(bucketStartTimeNs / NS_PER_SEC, config, cfgKey); + EXPECT_EQ(processor->mMetricsManagers.size(), 1u); + EXPECT_TRUE(processor->mMetricsManagers.begin()->second->isConfigValid()); + FeedEvents(config, processor); + vector buffer; + ConfigMetricsReportList reports; + + processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, &buffer); + EXPECT_TRUE(buffer.size() > 0); + EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size())); EXPECT_EQ(reports.reports_size(), 1); EXPECT_EQ(reports.reports(0).metrics_size(), 1); EXPECT_EQ(reports.reports(0).metrics(0).duration_metrics().data_size(), 1); // Dump the report after the end of 2nd bucket. EXPECT_EQ(reports.reports(0).metrics(0).duration_metrics().data(0).bucket_info_size(), 2); - data = reports.reports(0).metrics(0).duration_metrics().data(0); + auto data = reports.reports(0).metrics(0).duration_metrics().data(0); // Validate dimension value. ValidateAttributionUidDimension(data.dimensions_in_what(), android::util::WAKELOCK_STATE_CHANGED, 111); @@ -162,6 +180,19 @@ TEST(WakelockDurationE2eTest, TestAggregatedPredicateDimensionsForSumDuration) { // The wakelock holding interval in the 2nd bucket starts at the beginning of the bucket and // ends at the second screen on event. EXPECT_EQ((unsigned long long)data.bucket_info(1).duration_nanos(), 500UL); +} +TEST(WakelockDurationE2eTest, TestAggregatedPredicateDimensionsForSumDuration3) { + ConfigKey cfgKey; + auto config = CreateStatsdConfig(DurationMetric::SUM); + uint64_t bucketStartTimeNs = 10000000000; + uint64_t bucketSizeNs = + TimeUnitToBucketSizeInMillis(config.duration_metric(0).bucket()) * 1000000LL; + auto processor = CreateStatsLogProcessor(bucketStartTimeNs / NS_PER_SEC, config, cfgKey); + EXPECT_EQ(processor->mMetricsManagers.size(), 1u); + EXPECT_TRUE(processor->mMetricsManagers.begin()->second->isConfigValid()); + FeedEvents(config, processor); + vector buffer; + ConfigMetricsReportList reports; std::vector> events; events.push_back( @@ -175,13 +206,15 @@ TEST(WakelockDurationE2eTest, TestAggregatedPredicateDimensionsForSumDuration) { for (const auto& event : events) { processor->OnLogEvent(event.get()); } - reports.Clear(); - processor->onDumpReport(cfgKey, bucketStartTimeNs + 6 * bucketSizeNs + 1, &reports); + + processor->onDumpReport(cfgKey, bucketStartTimeNs + 6 * bucketSizeNs + 1, &buffer); + EXPECT_TRUE(buffer.size() > 0); + EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size())); EXPECT_EQ(reports.reports_size(), 1); EXPECT_EQ(reports.reports(0).metrics_size(), 1); EXPECT_EQ(reports.reports(0).metrics(0).duration_metrics().data_size(), 1); EXPECT_EQ(reports.reports(0).metrics(0).duration_metrics().data(0).bucket_info_size(), 6); - data = reports.reports(0).metrics(0).duration_metrics().data(0); + auto data = reports.reports(0).metrics(0).duration_metrics().data(0); ValidateAttributionUidDimension(data.dimensions_in_what(), android::util::WAKELOCK_STATE_CHANGED, 111); // The last wakelock holding spans 4 buckets. @@ -191,7 +224,7 @@ TEST(WakelockDurationE2eTest, TestAggregatedPredicateDimensionsForSumDuration) { EXPECT_EQ((unsigned long long)data.bucket_info(5).duration_nanos(), 100UL); } -TEST(WakelockDurationE2eTest, TestAggregatedPredicateDimensionsForMaxDuration) { +TEST(WakelockDurationE2eTest, TestAggregatedPredicateDimensionsForMaxDuration1) { ConfigKey cfgKey; auto config = CreateStatsdConfig(DurationMetric::MAX_SPARSE); uint64_t bucketStartTimeNs = 10000000000; @@ -202,15 +235,35 @@ TEST(WakelockDurationE2eTest, TestAggregatedPredicateDimensionsForMaxDuration) { EXPECT_TRUE(processor->mMetricsManagers.begin()->second->isConfigValid()); FeedEvents(config, processor); ConfigMetricsReportList reports; - processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs - 1, &reports); + vector buffer; + processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs - 1, &buffer); + EXPECT_TRUE(buffer.size() > 0); + + EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size())); EXPECT_EQ(reports.reports_size(), 1); - EXPECT_EQ(reports.reports(0).metrics_size(), 1); - // Nothing has ended in the first bucket. - EXPECT_EQ(reports.reports(0).metrics(0).duration_metrics().data_size(), 0); - reports.Clear(); - processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, &reports); + // When using ProtoOutputStream, if nothing written to a sub msg, it won't be treated as + // one. It was previsouly 1 because we had a fake onDumpReport which calls add_metric() by + // itself. + EXPECT_EQ(0, reports.reports(0).metrics_size()); +} + +TEST(WakelockDurationE2eTest, TestAggregatedPredicateDimensionsForMaxDuration2) { + ConfigKey cfgKey; + auto config = CreateStatsdConfig(DurationMetric::MAX_SPARSE); + uint64_t bucketStartTimeNs = 10000000000; + uint64_t bucketSizeNs = + TimeUnitToBucketSizeInMillis(config.duration_metric(0).bucket()) * 1000000LL; + auto processor = CreateStatsLogProcessor(bucketStartTimeNs / NS_PER_SEC, config, cfgKey); + EXPECT_EQ(processor->mMetricsManagers.size(), 1u); + EXPECT_TRUE(processor->mMetricsManagers.begin()->second->isConfigValid()); + FeedEvents(config, processor); + ConfigMetricsReportList reports; + vector buffer; + processor->onDumpReport(cfgKey, bucketStartTimeNs + 2 * bucketSizeNs + 1, &buffer); + EXPECT_TRUE(buffer.size() > 0); + EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size())); EXPECT_EQ(reports.reports_size(), 1); EXPECT_EQ(reports.reports(0).metrics_size(), 1); EXPECT_EQ(reports.reports(0).metrics(0).duration_metrics().data_size(), 1); @@ -222,6 +275,20 @@ TEST(WakelockDurationE2eTest, TestAggregatedPredicateDimensionsForMaxDuration) { android::util::WAKELOCK_STATE_CHANGED, 111); // The max is acquire event for wl1 to screen off start. EXPECT_EQ((unsigned long long)data.bucket_info(0).duration_nanos(), bucketSizeNs + 2 - 200); +} + +TEST(WakelockDurationE2eTest, TestAggregatedPredicateDimensionsForMaxDuration3) { + ConfigKey cfgKey; + auto config = CreateStatsdConfig(DurationMetric::MAX_SPARSE); + uint64_t bucketStartTimeNs = 10000000000; + uint64_t bucketSizeNs = + TimeUnitToBucketSizeInMillis(config.duration_metric(0).bucket()) * 1000000LL; + auto processor = CreateStatsLogProcessor(bucketStartTimeNs / NS_PER_SEC, config, cfgKey); + EXPECT_EQ(processor->mMetricsManagers.size(), 1u); + EXPECT_TRUE(processor->mMetricsManagers.begin()->second->isConfigValid()); + FeedEvents(config, processor); + ConfigMetricsReportList reports; + vector buffer; std::vector> events; events.push_back( @@ -235,13 +302,15 @@ TEST(WakelockDurationE2eTest, TestAggregatedPredicateDimensionsForMaxDuration) { for (const auto& event : events) { processor->OnLogEvent(event.get()); } - reports.Clear(); - processor->onDumpReport(cfgKey, bucketStartTimeNs + 6 * bucketSizeNs + 1, &reports); + + processor->onDumpReport(cfgKey, bucketStartTimeNs + 6 * bucketSizeNs + 1, &buffer); + EXPECT_TRUE(buffer.size() > 0); + EXPECT_TRUE(reports.ParseFromArray(&buffer[0], buffer.size())); EXPECT_EQ(reports.reports_size(), 1); EXPECT_EQ(reports.reports(0).metrics_size(), 1); EXPECT_EQ(reports.reports(0).metrics(0).duration_metrics().data_size(), 1); EXPECT_EQ(reports.reports(0).metrics(0).duration_metrics().data(0).bucket_info_size(), 2); - data = reports.reports(0).metrics(0).duration_metrics().data(0); + auto data = reports.reports(0).metrics(0).duration_metrics().data(0); ValidateAttributionUidDimension(data.dimensions_in_what(), android::util::WAKELOCK_STATE_CHANGED, 111); // The last wakelock holding spans 4 buckets. diff --git a/cmds/statsd/tests/metrics/CountMetricProducer_test.cpp b/cmds/statsd/tests/metrics/CountMetricProducer_test.cpp index 87a1079aa7223..1e71b73ca9fa9 100644 --- a/cmds/statsd/tests/metrics/CountMetricProducer_test.cpp +++ b/cmds/statsd/tests/metrics/CountMetricProducer_test.cpp @@ -13,7 +13,6 @@ // limitations under the License. #include "src/metrics/CountMetricProducer.h" -#include "src/dimension.h" #include "src/stats_log_util.h" #include "metrics_test_helper.h" #include "tests/statsd_test_util.h" diff --git a/cmds/statsd/tests/metrics/EventMetricProducer_test.cpp b/cmds/statsd/tests/metrics/EventMetricProducer_test.cpp index 3deab3710dd10..8246268d88e78 100644 --- a/cmds/statsd/tests/metrics/EventMetricProducer_test.cpp +++ b/cmds/statsd/tests/metrics/EventMetricProducer_test.cpp @@ -13,7 +13,6 @@ // limitations under the License. #include "src/metrics/EventMetricProducer.h" -#include "src/dimension.h" #include "metrics_test_helper.h" #include "tests/statsd_test_util.h" diff --git a/cmds/statsd/tests/metrics/GaugeMetricProducer_test.cpp b/cmds/statsd/tests/metrics/GaugeMetricProducer_test.cpp index 470d4d0415b20..26f7c2669f151 100644 --- a/cmds/statsd/tests/metrics/GaugeMetricProducer_test.cpp +++ b/cmds/statsd/tests/metrics/GaugeMetricProducer_test.cpp @@ -80,9 +80,10 @@ TEST(GaugeMetricProducerTest, TestNoCondition) { gaugeProducer.onDataPulled(allData); EXPECT_EQ(1UL, gaugeProducer.mCurrentSlicedBucket->size()); auto it = gaugeProducer.mCurrentSlicedBucket->begin()->second.front().mFields->begin(); - EXPECT_EQ(10, it->second.value_int()); + EXPECT_EQ(INT, it->mValue.getType()); + EXPECT_EQ(10, it->mValue.int_value); it++; - EXPECT_EQ(11, it->second.value_int()); + EXPECT_EQ(11, it->mValue.int_value); EXPECT_EQ(0UL, gaugeProducer.mPastBuckets.size()); allData.clear(); @@ -96,16 +97,20 @@ TEST(GaugeMetricProducerTest, TestNoCondition) { gaugeProducer.onDataPulled(allData); EXPECT_EQ(1UL, gaugeProducer.mCurrentSlicedBucket->size()); it = gaugeProducer.mCurrentSlicedBucket->begin()->second.front().mFields->begin(); - EXPECT_EQ(24, it->second.value_int()); + EXPECT_EQ(INT, it->mValue.getType()); + EXPECT_EQ(24, it->mValue.int_value); it++; - EXPECT_EQ(25, it->second.value_int()); + EXPECT_EQ(INT, it->mValue.getType()); + EXPECT_EQ(25, it->mValue.int_value); // One dimension. EXPECT_EQ(1UL, gaugeProducer.mPastBuckets.size()); EXPECT_EQ(1UL, gaugeProducer.mPastBuckets.begin()->second.size()); it = gaugeProducer.mPastBuckets.begin()->second.back().mGaugeAtoms.front().mFields->begin(); - EXPECT_EQ(10L, it->second.value_int()); + EXPECT_EQ(INT, it->mValue.getType()); + EXPECT_EQ(10L, it->mValue.int_value); it++; - EXPECT_EQ(11L, it->second.value_int()); + EXPECT_EQ(INT, it->mValue.getType()); + EXPECT_EQ(11L, it->mValue.int_value); EXPECT_EQ(1UL, gaugeProducer.mPastBuckets.begin()->second.back().mBucketNum); gaugeProducer.flushIfNeededLocked(bucket4StartTimeNs); @@ -114,9 +119,11 @@ TEST(GaugeMetricProducerTest, TestNoCondition) { EXPECT_EQ(1UL, gaugeProducer.mPastBuckets.size()); EXPECT_EQ(2UL, gaugeProducer.mPastBuckets.begin()->second.size()); it = gaugeProducer.mPastBuckets.begin()->second.back().mGaugeAtoms.front().mFields->begin(); - EXPECT_EQ(24L, it->second.value_int()); + EXPECT_EQ(INT, it->mValue.getType()); + EXPECT_EQ(24L, it->mValue.int_value); it++; - EXPECT_EQ(25L, it->second.value_int()); + EXPECT_EQ(INT, it->mValue.getType()); + EXPECT_EQ(25L, it->mValue.int_value); EXPECT_EQ(2UL, gaugeProducer.mPastBuckets.begin()->second.back().mBucketNum); } @@ -230,7 +237,7 @@ TEST(GaugeMetricProducerTest, TestPulledWithUpgrade) { EXPECT_EQ(1, gaugeProducer.mCurrentSlicedBucket->begin() ->second.front() .mFields->begin() - ->second.value_int()); + ->mValue.int_value); gaugeProducer.notifyAppUpgrade(eventUpgradeTimeNs, "ANY.APP", 1, 1); EXPECT_EQ(1UL, gaugeProducer.mPastBuckets[DEFAULT_METRIC_DIMENSION_KEY].size()); @@ -240,7 +247,7 @@ TEST(GaugeMetricProducerTest, TestPulledWithUpgrade) { EXPECT_EQ(2, gaugeProducer.mCurrentSlicedBucket->begin() ->second.front() .mFields->begin() - ->second.value_int()); + ->mValue.int_value); allData.clear(); event = make_shared(tagId, bucketStartTimeNs + bucketSizeNs + 1); @@ -254,7 +261,7 @@ TEST(GaugeMetricProducerTest, TestPulledWithUpgrade) { EXPECT_EQ(3, gaugeProducer.mCurrentSlicedBucket->begin() ->second.front() .mFields->begin() - ->second.value_int()); + ->mValue.int_value); } TEST(GaugeMetricProducerTest, TestWithCondition) { @@ -288,9 +295,10 @@ TEST(GaugeMetricProducerTest, TestWithCondition) { gaugeProducer.onConditionChanged(true, bucketStartTimeNs + 8); EXPECT_EQ(1UL, gaugeProducer.mCurrentSlicedBucket->size()); - EXPECT_EQ(100, - gaugeProducer.mCurrentSlicedBucket->begin()-> - second.front().mFields->begin()->second.value_int()); + EXPECT_EQ(100, gaugeProducer.mCurrentSlicedBucket->begin() + ->second.front() + .mFields->begin() + ->mValue.int_value); EXPECT_EQ(0UL, gaugeProducer.mPastBuckets.size()); vector> allData; @@ -303,19 +311,26 @@ TEST(GaugeMetricProducerTest, TestWithCondition) { gaugeProducer.onDataPulled(allData); EXPECT_EQ(1UL, gaugeProducer.mCurrentSlicedBucket->size()); - EXPECT_EQ(110, - gaugeProducer.mCurrentSlicedBucket->begin()-> - second.front().mFields->begin()->second.value_int()); + EXPECT_EQ(110, gaugeProducer.mCurrentSlicedBucket->begin() + ->second.front() + .mFields->begin() + ->mValue.int_value); EXPECT_EQ(1UL, gaugeProducer.mPastBuckets.size()); - EXPECT_EQ(100, gaugeProducer.mPastBuckets.begin()->second.back() - .mGaugeAtoms.front().mFields->begin()->second.value_int()); + EXPECT_EQ(100, gaugeProducer.mPastBuckets.begin() + ->second.back() + .mGaugeAtoms.front() + .mFields->begin() + ->mValue.int_value); gaugeProducer.onConditionChanged(false, bucket2StartTimeNs + 10); gaugeProducer.flushIfNeededLocked(bucket3StartTimeNs + 10); EXPECT_EQ(1UL, gaugeProducer.mPastBuckets.size()); EXPECT_EQ(2UL, gaugeProducer.mPastBuckets.begin()->second.size()); - EXPECT_EQ(110L, gaugeProducer.mPastBuckets.begin()->second.back() - .mGaugeAtoms.front().mFields->begin()->second.value_int()); + EXPECT_EQ(110L, gaugeProducer.mPastBuckets.begin() + ->second.back() + .mGaugeAtoms.front() + .mFields->begin() + ->mValue.int_value); EXPECT_EQ(1UL, gaugeProducer.mPastBuckets.begin()->second.back().mBucketNum); } @@ -353,9 +368,10 @@ TEST(GaugeMetricProducerTest, TestAnomalyDetection) { gaugeProducer.onDataPulled({event1}); EXPECT_EQ(1UL, gaugeProducer.mCurrentSlicedBucket->size()); - EXPECT_EQ(13L, - gaugeProducer.mCurrentSlicedBucket->begin()-> - second.front().mFields->begin()->second.value_int()); + EXPECT_EQ(13L, gaugeProducer.mCurrentSlicedBucket->begin() + ->second.front() + .mFields->begin() + ->mValue.int_value); EXPECT_EQ(anomalyTracker->getRefractoryPeriodEndsSec(DEFAULT_METRIC_DIMENSION_KEY), 0U); std::shared_ptr event2 = @@ -366,9 +382,10 @@ TEST(GaugeMetricProducerTest, TestAnomalyDetection) { gaugeProducer.onDataPulled({event2}); EXPECT_EQ(1UL, gaugeProducer.mCurrentSlicedBucket->size()); - EXPECT_EQ(15L, - gaugeProducer.mCurrentSlicedBucket->begin()-> - second.front().mFields->begin()->second.value_int()); + EXPECT_EQ(15L, gaugeProducer.mCurrentSlicedBucket->begin() + ->second.front() + .mFields->begin() + ->mValue.int_value); EXPECT_EQ(anomalyTracker->getRefractoryPeriodEndsSec(DEFAULT_METRIC_DIMENSION_KEY), event2->GetTimestampNs() / NS_PER_SEC + refPeriodSec); @@ -380,9 +397,10 @@ TEST(GaugeMetricProducerTest, TestAnomalyDetection) { gaugeProducer.onDataPulled({event3}); EXPECT_EQ(1UL, gaugeProducer.mCurrentSlicedBucket->size()); - EXPECT_EQ(26L, - gaugeProducer.mCurrentSlicedBucket->begin()-> - second.front().mFields->begin()->second.value_int()); + EXPECT_EQ(26L, gaugeProducer.mCurrentSlicedBucket->begin() + ->second.front() + .mFields->begin() + ->mValue.int_value); EXPECT_EQ(anomalyTracker->getRefractoryPeriodEndsSec(DEFAULT_METRIC_DIMENSION_KEY), event2->GetTimestampNs() / NS_PER_SEC + refPeriodSec); diff --git a/cmds/statsd/tests/metrics/MaxDurationTracker_test.cpp b/cmds/statsd/tests/metrics/MaxDurationTracker_test.cpp index 2658e4ecc53c9..3397f144cfbcc 100644 --- a/cmds/statsd/tests/metrics/MaxDurationTracker_test.cpp +++ b/cmds/statsd/tests/metrics/MaxDurationTracker_test.cpp @@ -53,7 +53,8 @@ TEST(MaxDurationTrackerTest, TestSimpleMaxDuration) { const HashableDimensionKey key1 = getMockedDimensionKey(TagId, 1, "1"); const HashableDimensionKey key2 = getMockedDimensionKey(TagId, 1, "2"); - FieldMatcher dimensionInCondition; + vector dimensionInCondition; + sp wizard = new NaggyMock(); unordered_map> buckets; @@ -89,7 +90,7 @@ TEST(MaxDurationTrackerTest, TestStopAll) { const HashableDimensionKey key1 = getMockedDimensionKey(TagId, 1, "1"); const HashableDimensionKey key2 = getMockedDimensionKey(TagId, 1, "2"); - FieldMatcher dimensionInCondition; + vector dimensionInCondition; sp wizard = new NaggyMock(); unordered_map> buckets; @@ -126,7 +127,7 @@ TEST(MaxDurationTrackerTest, TestCrossBucketBoundary) { const std::vector conditionKey = {getMockedDimensionKey(TagId, 4, "1")}; const HashableDimensionKey key1 = getMockedDimensionKey(TagId, 1, "1"); const HashableDimensionKey key2 = getMockedDimensionKey(TagId, 1, "2"); - FieldMatcher dimensionInCondition; + vector dimensionInCondition; sp wizard = new NaggyMock(); unordered_map> buckets; @@ -168,7 +169,7 @@ TEST(MaxDurationTrackerTest, TestCrossBucketBoundary_nested) { const std::vector conditionKey = {getMockedDimensionKey(TagId, 4, "1")}; const HashableDimensionKey key1 = getMockedDimensionKey(TagId, 1, "1"); const HashableDimensionKey key2 = getMockedDimensionKey(TagId, 1, "2"); - FieldMatcher dimensionInCondition; + vector dimensionInCondition; sp wizard = new NaggyMock(); unordered_map> buckets; @@ -206,7 +207,7 @@ TEST(MaxDurationTrackerTest, TestMaxDurationWithCondition) { const std::vector conditionKey = {getMockedDimensionKey(TagId, 4, "1")}; const HashableDimensionKey key1 = getMockedDimensionKey(TagId, 1, "1"); - FieldMatcher dimensionInCondition; + vector dimensionInCondition; sp wizard = new NaggyMock(); ConditionKey conditionKey1; diff --git a/cmds/statsd/tests/metrics/OringDurationTracker_test.cpp b/cmds/statsd/tests/metrics/OringDurationTracker_test.cpp index 4b579b15b16ea..293b1a872a5af 100644 --- a/cmds/statsd/tests/metrics/OringDurationTracker_test.cpp +++ b/cmds/statsd/tests/metrics/OringDurationTracker_test.cpp @@ -52,7 +52,7 @@ TEST(OringDurationTrackerTest, TestDurationOverlap) { {getMockedDimensionKey(TagId, 1, "maps")}; const HashableDimensionKey kEventKey1 = getMockedDimensionKey(TagId, 2, "maps"); const HashableDimensionKey kEventKey2 = getMockedDimensionKey(TagId, 3, "maps"); - FieldMatcher dimensionInCondition; + vector dimensionInCondition; sp wizard = new NaggyMock(); unordered_map> buckets; @@ -88,7 +88,7 @@ TEST(OringDurationTrackerTest, TestDurationNested) { {getMockedDimensionKey(TagId, 1, "maps")}; const HashableDimensionKey kEventKey1 = getMockedDimensionKey(TagId, 2, "maps"); const HashableDimensionKey kEventKey2 = getMockedDimensionKey(TagId, 3, "maps"); - FieldMatcher dimensionInCondition; + vector dimensionInCondition; sp wizard = new NaggyMock(); unordered_map> buckets; @@ -122,7 +122,7 @@ TEST(OringDurationTrackerTest, TestStopAll) { {getMockedDimensionKey(TagId, 1, "maps")}; const HashableDimensionKey kEventKey1 = getMockedDimensionKey(TagId, 2, "maps"); const HashableDimensionKey kEventKey2 = getMockedDimensionKey(TagId, 3, "maps"); - FieldMatcher dimensionInCondition; + vector dimensionInCondition; sp wizard = new NaggyMock(); unordered_map> buckets; @@ -155,7 +155,7 @@ TEST(OringDurationTrackerTest, TestCrossBucketBoundary) { {getMockedDimensionKey(TagId, 1, "maps")}; const HashableDimensionKey kEventKey1 = getMockedDimensionKey(TagId, 2, "maps"); const HashableDimensionKey kEventKey2 = getMockedDimensionKey(TagId, 3, "maps"); - FieldMatcher dimensionInCondition; + vector dimensionInCondition; sp wizard = new NaggyMock(); unordered_map> buckets; @@ -197,7 +197,7 @@ TEST(OringDurationTrackerTest, TestDurationConditionChange) { {getMockedDimensionKey(TagId, 1, "maps")}; const HashableDimensionKey kEventKey1 = getMockedDimensionKey(TagId, 2, "maps"); const HashableDimensionKey kEventKey2 = getMockedDimensionKey(TagId, 3, "maps"); - FieldMatcher dimensionInCondition; + vector dimensionInCondition; sp wizard = new NaggyMock(); ConditionKey key1; @@ -238,7 +238,7 @@ TEST(OringDurationTrackerTest, TestDurationConditionChange2) { {getMockedDimensionKey(TagId, 1, "maps")}; const HashableDimensionKey kEventKey1 = getMockedDimensionKey(TagId, 2, "maps"); const HashableDimensionKey kEventKey2 = getMockedDimensionKey(TagId, 3, "maps"); - FieldMatcher dimensionInCondition; + vector dimensionInCondition; sp wizard = new NaggyMock(); ConditionKey key1; @@ -283,7 +283,7 @@ TEST(OringDurationTrackerTest, TestDurationConditionChangeNested) { {getMockedDimensionKey(TagId, 1, "maps")}; const HashableDimensionKey kEventKey1 = getMockedDimensionKey(TagId, 2, "maps"); const HashableDimensionKey kEventKey2 = getMockedDimensionKey(TagId, 3, "maps"); - FieldMatcher dimensionInCondition; + vector dimensionInCondition; sp wizard = new NaggyMock(); ConditionKey key1; @@ -326,7 +326,7 @@ TEST(OringDurationTrackerTest, TestPredictAnomalyTimestamp) { {getMockedDimensionKey(TagId, 1, "maps")}; const HashableDimensionKey kEventKey1 = getMockedDimensionKey(TagId, 2, "maps"); const HashableDimensionKey kEventKey2 = getMockedDimensionKey(TagId, 3, "maps"); - FieldMatcher dimensionInCondition; + vector dimensionInCondition; Alert alert; alert.set_id(101); alert.set_metric_id(1); @@ -395,7 +395,7 @@ TEST(OringDurationTrackerTest, TestAnomalyDetectionExpiredAlarm) { const std::vector kConditionKey1 = {getMockedDimensionKey(TagId, 1, "maps")}; const HashableDimensionKey kEventKey1 = getMockedDimensionKey(TagId, 2, "maps"); const HashableDimensionKey kEventKey2 = getMockedDimensionKey(TagId, 3, "maps"); - FieldMatcher dimensionInCondition; + vector dimensionInCondition; Alert alert; alert.set_id(101); alert.set_metric_id(1); @@ -446,7 +446,7 @@ TEST(OringDurationTrackerTest, TestAnomalyDetectionFiredAlarm) { {getMockedDimensionKey(TagId, 1, "maps")}; const HashableDimensionKey kEventKey1 = getMockedDimensionKey(TagId, 2, "maps"); const HashableDimensionKey kEventKey2 = getMockedDimensionKey(TagId, 3, "maps"); - FieldMatcher dimensionInCondition; + vector dimensionInCondition; Alert alert; alert.set_id(101); alert.set_metric_id(1); diff --git a/cmds/statsd/tests/metrics/metrics_test_helper.cpp b/cmds/statsd/tests/metrics/metrics_test_helper.cpp index ab9345af172e9..7b9c0d6ab28e8 100644 --- a/cmds/statsd/tests/metrics/metrics_test_helper.cpp +++ b/cmds/statsd/tests/metrics/metrics_test_helper.cpp @@ -19,20 +19,26 @@ namespace os { namespace statsd { HashableDimensionKey getMockedDimensionKey(int tagId, int key, string value) { - DimensionsValue dimensionsValue; - dimensionsValue.set_field(tagId); - dimensionsValue.mutable_value_tuple()->add_dimensions_value()->set_field(key); - dimensionsValue.mutable_value_tuple()->mutable_dimensions_value(0)->set_value_str(value); - return HashableDimensionKey(dimensionsValue); + HashableDimensionKey dimension; + int pos[] = {key, 0, 0}; + dimension.addValue(FieldValue(Field(tagId, pos, 0), Value(value))); + + return dimension; } MetricDimensionKey getMockedMetricDimensionKey(int tagId, int key, string value) { - DimensionsValue dimensionsValue; - dimensionsValue.set_field(tagId); - dimensionsValue.mutable_value_tuple()->add_dimensions_value()->set_field(key); - dimensionsValue.mutable_value_tuple()->mutable_dimensions_value(0)->set_value_str(value); - return MetricDimensionKey(HashableDimensionKey(dimensionsValue), DEFAULT_DIMENSION_KEY); + return MetricDimensionKey(getMockedDimensionKey(tagId, key, value), DEFAULT_DIMENSION_KEY); } + +void buildSimpleAtomFieldMatcher(const int tagId, FieldMatcher* matcher) { + matcher->set_field(tagId); +} + +void buildSimpleAtomFieldMatcher(const int tagId, const int fieldNum, FieldMatcher* matcher) { + matcher->set_field(tagId); + matcher->add_child()->set_field(fieldNum); +} + } // namespace statsd } // namespace os } // namespace android \ No newline at end of file diff --git a/cmds/statsd/tests/metrics/metrics_test_helper.h b/cmds/statsd/tests/metrics/metrics_test_helper.h index b48de540ac092..a01de6334ab4f 100644 --- a/cmds/statsd/tests/metrics/metrics_test_helper.h +++ b/cmds/statsd/tests/metrics/metrics_test_helper.h @@ -26,12 +26,10 @@ namespace statsd { class MockConditionWizard : public ConditionWizard { public: - MOCK_METHOD4( - query, - ConditionState(const int conditionIndex, - const ConditionKey& conditionParameters, - const FieldMatcher& dimensionFields, - std::unordered_set *dimensionKeySet)); + MOCK_METHOD4(query, + ConditionState(const int conditionIndex, const ConditionKey& conditionParameters, + const vector& dimensionFields, + std::unordered_set* dimensionKeySet)); }; class MockStatsPullerManager : public StatsPullerManager { @@ -49,6 +47,10 @@ class MockUidMap : public UidMap { HashableDimensionKey getMockedDimensionKey(int tagId, int key, std::string value); MetricDimensionKey getMockedMetricDimensionKey(int tagId, int key, std::string value); +// Utils to build FieldMatcher proto for simple one-depth atoms. +void buildSimpleAtomFieldMatcher(const int tagId, const int atomFieldNum, FieldMatcher* matcher); +void buildSimpleAtomFieldMatcher(const int tagId, FieldMatcher* matcher); + } // namespace statsd } // namespace os } // namespace android \ No newline at end of file diff --git a/cmds/statsd/tests/statsd_test_util.cpp b/cmds/statsd/tests/statsd_test_util.cpp index 13055cb9e7a30..d3a89617e9216 100644 --- a/cmds/statsd/tests/statsd_test_util.cpp +++ b/cmds/statsd/tests/statsd_test_util.cpp @@ -186,7 +186,7 @@ Predicate CreateScreenIsOnPredicate() { Predicate CreateScreenIsOffPredicate() { Predicate predicate; - predicate.set_id(StringToId("ScreenIsOff")); + predicate.set_id(1111123); predicate.mutable_simple_predicate()->set_start(StringToId("ScreenTurnedOff")); predicate.mutable_simple_predicate()->set_stop(StringToId("ScreenTurnedOn")); return predicate; @@ -202,7 +202,7 @@ Predicate CreateHoldingWakelockPredicate() { Predicate CreateIsSyncingPredicate() { Predicate predicate; - predicate.set_id(StringToId("IsSyncing")); + predicate.set_id(33333333333333); predicate.mutable_simple_predicate()->set_start(StringToId("SyncStart")); predicate.mutable_simple_predicate()->set_stop(StringToId("SyncEnd")); return predicate; @@ -461,6 +461,93 @@ void ValidateAttributionUidAndTagDimension( .value_tuple().dimensions_value(1).value_str(), tag); } +bool EqualsTo(const DimensionsValue& s1, const DimensionsValue& s2) { + if (s1.field() != s2.field()) { + return false; + } + if (s1.value_case() != s2.value_case()) { + return false; + } + switch (s1.value_case()) { + case DimensionsValue::ValueCase::kValueStr: + return (s1.value_str() == s2.value_str()); + case DimensionsValue::ValueCase::kValueInt: + return s1.value_int() == s2.value_int(); + case DimensionsValue::ValueCase::kValueLong: + return s1.value_long() == s2.value_long(); + case DimensionsValue::ValueCase::kValueBool: + return s1.value_bool() == s2.value_bool(); + case DimensionsValue::ValueCase::kValueFloat: + return s1.value_float() == s2.value_float(); + case DimensionsValue::ValueCase::kValueTuple: { + if (s1.value_tuple().dimensions_value_size() != + s2.value_tuple().dimensions_value_size()) { + return false; + } + bool allMatched = true; + for (int i = 0; allMatched && i < s1.value_tuple().dimensions_value_size(); ++i) { + allMatched &= EqualsTo(s1.value_tuple().dimensions_value(i), + s2.value_tuple().dimensions_value(i)); + } + return allMatched; + } + case DimensionsValue::ValueCase::VALUE_NOT_SET: + default: + return true; + } +} + +bool LessThan(const DimensionsValue& s1, const DimensionsValue& s2) { + if (s1.field() != s2.field()) { + return s1.field() < s2.field(); + } + if (s1.value_case() != s2.value_case()) { + return s1.value_case() < s2.value_case(); + } + switch (s1.value_case()) { + case DimensionsValue::ValueCase::kValueStr: + return s1.value_str() < s2.value_str(); + case DimensionsValue::ValueCase::kValueInt: + return s1.value_int() < s2.value_int(); + case DimensionsValue::ValueCase::kValueLong: + return s1.value_long() < s2.value_long(); + case DimensionsValue::ValueCase::kValueBool: + return (int)s1.value_bool() < (int)s2.value_bool(); + case DimensionsValue::ValueCase::kValueFloat: + return s1.value_float() < s2.value_float(); + case DimensionsValue::ValueCase::kValueTuple: { + if (s1.value_tuple().dimensions_value_size() != + s2.value_tuple().dimensions_value_size()) { + return s1.value_tuple().dimensions_value_size() < + s2.value_tuple().dimensions_value_size(); + } + for (int i = 0; i < s1.value_tuple().dimensions_value_size(); ++i) { + if (EqualsTo(s1.value_tuple().dimensions_value(i), + s2.value_tuple().dimensions_value(i))) { + continue; + } else { + return LessThan(s1.value_tuple().dimensions_value(i), + s2.value_tuple().dimensions_value(i)); + } + } + return false; + } + case DimensionsValue::ValueCase::VALUE_NOT_SET: + default: + return false; + } +} + +bool LessThan(const DimensionsPair& s1, const DimensionsPair& s2) { + if (LessThan(s1.dimInWhat, s2.dimInWhat)) { + return true; + } else if (LessThan(s2.dimInWhat, s1.dimInWhat)) { + return false; + } + + return LessThan(s1.dimInCondition, s2.dimInCondition); +} + } // namespace statsd } // namespace os } // namespace android \ No newline at end of file diff --git a/cmds/statsd/tests/statsd_test_util.h b/cmds/statsd/tests/statsd_test_util.h index 6638893f6aebc..5d83ed7c16660 100644 --- a/cmds/statsd/tests/statsd_test_util.h +++ b/cmds/statsd/tests/statsd_test_util.h @@ -159,14 +159,30 @@ void ValidateAttributionUidDimension(const DimensionsValue& value, int atomId, i void ValidateAttributionUidAndTagDimension( const DimensionsValue& value, int atomId, int uid, const std::string& tag); +struct DimensionsPair { + DimensionsPair(DimensionsValue m1, DimensionsValue m2) : dimInWhat(m1), dimInCondition(m2){}; + + DimensionsValue dimInWhat; + DimensionsValue dimInCondition; +}; + +bool LessThan(const DimensionsValue& s1, const DimensionsValue& s2); +bool LessThan(const DimensionsPair& s1, const DimensionsPair& s2); + +struct DimensionCompare { + bool operator()(const DimensionsPair& s1, const DimensionsPair& s2) const { + return LessThan(s1, s2); + } +}; + template void sortMetricDataByDimensionsValue(const T& metricData, T* sortedMetricData) { - std::map dimensionIndexMap; + std::map dimensionIndexMap; for (int i = 0; i < metricData.data_size(); ++i) { - dimensionIndexMap.insert(std::make_pair( - MetricDimensionKey(HashableDimensionKey(metricData.data(i).dimensions_in_what()), - HashableDimensionKey(metricData.data(i).dimensions_in_condition())), - i)); + dimensionIndexMap.insert( + std::make_pair(DimensionsPair(metricData.data(i).dimensions_in_what(), + metricData.data(i).dimensions_in_condition()), + i)); } for (const auto& itr : dimensionIndexMap) { *sortedMetricData->add_data() = metricData.data(itr.second);