From b038b709566c191164bd23539175df8aadeb2f94 Mon Sep 17 00:00:00 2001 From: Chenjie Yu Date: Mon, 18 Dec 2017 15:15:34 -0800 Subject: [PATCH] pullers now cache data to throttle frequent pull requests. all pullers have a default 1s cool down before next pull. We can adjust these later. Also add puller stats in StatsdStats Test: unit test Change-Id: I71894a24c41e059d841591312dbb852f54387b7d --- cmds/statsd/Android.mk | 1 + .../src/external/CpuTimePerUidFreqPuller.cpp | 75 +++++---- .../src/external/CpuTimePerUidFreqPuller.h | 3 +- .../src/external/CpuTimePerUidPuller.cpp | 64 ++++---- .../statsd/src/external/CpuTimePerUidPuller.h | 3 +- .../external/ResourcePowerManagerPuller.cpp | 148 ++++++++++-------- .../src/external/ResourcePowerManagerPuller.h | 3 +- .../external/StatsCompanionServicePuller.cpp | 12 +- .../external/StatsCompanionServicePuller.h | 3 +- cmds/statsd/src/external/StatsPuller.cpp | 64 ++++++++ cmds/statsd/src/external/StatsPuller.h | 32 +++- .../src/external/StatsPullerManagerImpl.cpp | 36 ++--- cmds/statsd/src/guardrail/StatsdStats.cpp | 38 ++++- cmds/statsd/src/guardrail/StatsdStats.h | 25 +++ .../src/metrics/GaugeMetricProducer.cpp | 1 + cmds/statsd/src/stats_log.proto | 8 + cmds/statsd/src/stats_log_util.cpp | 20 +++ cmds/statsd/src/stats_log_util.h | 19 ++- 18 files changed, 386 insertions(+), 169 deletions(-) create mode 100644 cmds/statsd/src/external/StatsPuller.cpp diff --git a/cmds/statsd/Android.mk b/cmds/statsd/Android.mk index d829243619d9f..735efe391eb1b 100644 --- a/cmds/statsd/Android.mk +++ b/cmds/statsd/Android.mk @@ -34,6 +34,7 @@ statsd_common_src := \ src/config/ConfigKey.cpp \ src/config/ConfigListener.cpp \ src/config/ConfigManager.cpp \ + src/external/StatsPuller.cpp \ src/external/StatsCompanionServicePuller.cpp \ src/external/ResourcePowerManagerPuller.cpp \ src/external/CpuTimePerUidPuller.cpp \ diff --git a/cmds/statsd/src/external/CpuTimePerUidFreqPuller.cpp b/cmds/statsd/src/external/CpuTimePerUidFreqPuller.cpp index 97387602627bb..a61afb429f54f 100644 --- a/cmds/statsd/src/external/CpuTimePerUidFreqPuller.cpp +++ b/cmds/statsd/src/external/CpuTimePerUidFreqPuller.cpp @@ -20,6 +20,9 @@ #include #include "external/CpuTimePerUidFreqPuller.h" +#include "../guardrail/StatsdStats.h" +#include "CpuTimePerUidFreqPuller.h" +#include "guardrail/StatsdStats.h" #include "logd/LogEvent.h" #include "statslog.h" @@ -45,43 +48,47 @@ static const int kLineBufferSize = 1024; * This provides the times a UID's processes spent executing at each different cpu frequency. * The file contains a monotonically increasing count of time for a single boot. */ -bool CpuTimePerUidFreqPuller::Pull(const int tagId, vector>* data) { - data->clear(); +CpuTimePerUidFreqPuller::CpuTimePerUidFreqPuller() + : StatsPuller(android::util::CPU_TIME_PER_UID_FREQ) { +} - ifstream fin; - fin.open(sProcFile); - if (!fin.good()) { - VLOG("Failed to read pseudo file %s", sProcFile.c_str()); - return false; - } +bool CpuTimePerUidFreqPuller::PullInternal(vector>* data) { + data->clear(); - uint64_t timestamp = time(nullptr) * NS_PER_SEC; - char buf[kLineBufferSize]; - // first line prints the format and frequencies - fin.getline(buf, kLineBufferSize); - char * pch; - while(!fin.eof()){ + ifstream fin; + fin.open(sProcFile); + if (!fin.good()) { + VLOG("Failed to read pseudo file %s", sProcFile.c_str()); + return false; + } + + uint64_t timestamp = time(nullptr) * NS_PER_SEC; + char buf[kLineBufferSize]; + // first line prints the format and frequencies fin.getline(buf, kLineBufferSize); - pch = strtok (buf, " :"); - if (pch == NULL) break; - uint64_t uid = std::stoull(pch); - pch = strtok(NULL, " "); - uint64_t timeMs; - int idx = 0; - do { - timeMs = std::stoull(pch); - auto ptr = make_shared(android::util::CPU_TIME_PER_UID_FREQ, timestamp); - ptr->write(uid); - ptr->write(idx); - ptr->write(timeMs); - ptr->init(); - data->push_back(ptr); - VLOG("uid %lld, freq idx %d, sys time %lld", (long long)uid, idx, (long long)timeMs); - idx ++; - pch = strtok(NULL, " "); - } while (pch != NULL); - } - return true; + char* pch; + while (!fin.eof()) { + fin.getline(buf, kLineBufferSize); + pch = strtok(buf, " :"); + if (pch == NULL) break; + uint64_t uid = std::stoull(pch); + pch = strtok(NULL, " "); + uint64_t timeMs; + int idx = 0; + do { + timeMs = std::stoull(pch); + auto ptr = make_shared(android::util::CPU_TIME_PER_UID_FREQ, timestamp); + ptr->write(uid); + ptr->write(idx); + ptr->write(timeMs); + ptr->init(); + data->push_back(ptr); + VLOG("uid %lld, freq idx %d, sys time %lld", (long long)uid, idx, (long long)timeMs); + idx++; + pch = strtok(NULL, " "); + } while (pch != NULL); + } + return true; } } // namespace statsd diff --git a/cmds/statsd/src/external/CpuTimePerUidFreqPuller.h b/cmds/statsd/src/external/CpuTimePerUidFreqPuller.h index 839e5aab3dc2a..6f6c669fbea90 100644 --- a/cmds/statsd/src/external/CpuTimePerUidFreqPuller.h +++ b/cmds/statsd/src/external/CpuTimePerUidFreqPuller.h @@ -33,7 +33,8 @@ namespace statsd { */ class CpuTimePerUidFreqPuller : public StatsPuller { public: - bool Pull(const int tagId, vector>* data) override; + CpuTimePerUidFreqPuller(); + bool PullInternal(vector>* data) override; }; } // namespace statsd diff --git a/cmds/statsd/src/external/CpuTimePerUidPuller.cpp b/cmds/statsd/src/external/CpuTimePerUidPuller.cpp index f69b9b5379439..e7ea4b9abf0cd 100644 --- a/cmds/statsd/src/external/CpuTimePerUidPuller.cpp +++ b/cmds/statsd/src/external/CpuTimePerUidPuller.cpp @@ -20,6 +20,8 @@ #include #include "external/CpuTimePerUidPuller.h" +#include "CpuTimePerUidPuller.h" +#include "guardrail/StatsdStats.h" #include "logd/LogEvent.h" #include "statslog.h" @@ -42,38 +44,42 @@ static const int kLineBufferSize = 1024; * This provides the time a UID's processes spent executing in user-space and kernel-space. * The file contains a monotonically increasing count of time for a single boot. */ -bool CpuTimePerUidPuller::Pull(const int tagId, vector>* data) { - data->clear(); +CpuTimePerUidPuller::CpuTimePerUidPuller() : StatsPuller(android::util::CPU_TIME_PER_UID) { +} - ifstream fin; - fin.open(sProcFile); - if (!fin.good()) { - VLOG("Failed to read pseudo file %s", sProcFile.c_str()); - return false; - } +bool CpuTimePerUidPuller::PullInternal(vector>* data) { + data->clear(); - uint64_t timestamp = time(nullptr) * NS_PER_SEC; - char buf[kLineBufferSize]; - char * pch; - while(!fin.eof()){ - fin.getline(buf, kLineBufferSize); - pch = strtok(buf, " :"); - if (pch == NULL) break; - uint64_t uid = std::stoull(pch); - pch = strtok(buf, " "); - uint64_t userTimeMs = std::stoull(pch); - pch = strtok(buf, " "); - uint64_t sysTimeMs = std::stoull(pch); + ifstream fin; + fin.open(sProcFile); + if (!fin.good()) { + VLOG("Failed to read pseudo file %s", sProcFile.c_str()); + return false; + } - auto ptr = make_shared(android::util::CPU_TIME_PER_UID, timestamp); - ptr->write(uid); - ptr->write(userTimeMs); - ptr->write(sysTimeMs); - ptr->init(); - data->push_back(ptr); - VLOG("uid %lld, user time %lld, sys time %lld", (long long)uid, (long long)userTimeMs, (long long)sysTimeMs); - } - return true; + uint64_t timestamp = time(nullptr) * NS_PER_SEC; + char buf[kLineBufferSize]; + char* pch; + while (!fin.eof()) { + fin.getline(buf, kLineBufferSize); + pch = strtok(buf, " :"); + if (pch == NULL) break; + uint64_t uid = std::stoull(pch); + pch = strtok(buf, " "); + uint64_t userTimeMs = std::stoull(pch); + pch = strtok(buf, " "); + uint64_t sysTimeMs = std::stoull(pch); + + auto ptr = make_shared(android::util::CPU_TIME_PER_UID, timestamp); + ptr->write(uid); + ptr->write(userTimeMs); + ptr->write(sysTimeMs); + ptr->init(); + data->push_back(ptr); + VLOG("uid %lld, user time %lld, sys time %lld", (long long)uid, (long long)userTimeMs, + (long long)sysTimeMs); + } + return true; } } // namespace statsd diff --git a/cmds/statsd/src/external/CpuTimePerUidPuller.h b/cmds/statsd/src/external/CpuTimePerUidPuller.h index 9bb89463725ec..d0d39d03aa5d0 100644 --- a/cmds/statsd/src/external/CpuTimePerUidPuller.h +++ b/cmds/statsd/src/external/CpuTimePerUidPuller.h @@ -33,7 +33,8 @@ namespace statsd { */ class CpuTimePerUidPuller : public StatsPuller { public: - bool Pull(const int tagId, vector>* data) override; + CpuTimePerUidPuller(); + bool PullInternal(vector>* data) override; }; } // namespace statsd diff --git a/cmds/statsd/src/external/ResourcePowerManagerPuller.cpp b/cmds/statsd/src/external/ResourcePowerManagerPuller.cpp index cb9f1cc15ce96..2e29fb04abfaf 100644 --- a/cmds/statsd/src/external/ResourcePowerManagerPuller.cpp +++ b/cmds/statsd/src/external/ResourcePowerManagerPuller.cpp @@ -33,6 +33,7 @@ #include "external/ResourcePowerManagerPuller.h" #include "external/StatsPuller.h" +#include "ResourcePowerManagerPuller.h" #include "logd/LogEvent.h" #include "statslog.h" @@ -72,7 +73,10 @@ bool getPowerHal() { return gPowerHalV1_0 != nullptr; } -bool ResourcePowerManagerPuller::Pull(const int tagId, vector>* data) { +ResourcePowerManagerPuller::ResourcePowerManagerPuller(int tagId) : StatsPuller(tagId) { +} + +bool ResourcePowerManagerPuller::PullInternal(vector>* data) { std::lock_guard lock(gPowerHalMutex); if (!getPowerHal()) { @@ -83,79 +87,87 @@ bool ResourcePowerManagerPuller::Pull(const int tagId, vectorclear(); - Return ret = gPowerHalV1_0->getPlatformLowPowerStats( - [&data, timestamp](hidl_vec states, Status status) { - - if (status != Status::SUCCESS) return; - - for (size_t i = 0; i < states.size(); i++) { - const PowerStatePlatformSleepState& state = states[i]; - - auto statePtr = make_shared( - android::util::PLATFORM_SLEEP_STATE, timestamp); - statePtr->write(state.name); - statePtr->write(state.residencyInMsecSinceBoot); - statePtr->write(state.totalTransitions); - statePtr->write(state.supportedOnlyInSuspend); - statePtr->init(); - data->push_back(statePtr); - VLOG("powerstate: %s, %lld, %lld, %d", state.name.c_str(), - (long long)state.residencyInMsecSinceBoot, - (long long)state.totalTransitions, state.supportedOnlyInSuspend ? 1 : 0); - for (auto voter : state.voters) { - auto voterPtr = - make_shared(android::util::SLEEP_STATE_VOTER, timestamp); - voterPtr->write(state.name); - voterPtr->write(voter.name); - voterPtr->write(voter.totalTimeInMsecVotedForSinceBoot); - voterPtr->write(voter.totalNumberOfTimesVotedSinceBoot); - voterPtr->init(); - data->push_back(voterPtr); - VLOG("powerstatevoter: %s, %s, %lld, %lld", state.name.c_str(), - voter.name.c_str(), (long long)voter.totalTimeInMsecVotedForSinceBoot, - (long long)voter.totalNumberOfTimesVotedSinceBoot); - } - } - }); - if (!ret.isOk()) { - ALOGE("getLowPowerStats() failed: power HAL service not available"); - gPowerHalV1_0 = nullptr; - return false; - } - - // Trying to cast to IPower 1.1, this will succeed only for devices supporting 1.1 - sp gPowerHal_1_1 = - android::hardware::power::V1_1::IPower::castFrom(gPowerHalV1_0); - if (gPowerHal_1_1 != nullptr) { - ret = gPowerHal_1_1->getSubsystemLowPowerStats( - [&data, timestamp](hidl_vec subsystems, Status status) { + Return ret; + if (mTagId == android::util::PLATFORM_SLEEP_STATE || + mTagId == android::util::SLEEP_STATE_VOTER) { + ret = gPowerHalV1_0->getPlatformLowPowerStats( + [&data, timestamp](hidl_vec states, Status status) { if (status != Status::SUCCESS) return; - if (subsystems.size() > 0) { - for (size_t i = 0; i < subsystems.size(); i++) { - const PowerStateSubsystem& subsystem = subsystems[i]; - for (size_t j = 0; j < subsystem.states.size(); j++) { - const PowerStateSubsystemSleepState& state = subsystem.states[j]; - auto subsystemStatePtr = make_shared( - android::util::SUBSYSTEM_SLEEP_STATE, timestamp); - subsystemStatePtr->write(subsystem.name); - subsystemStatePtr->write(state.name); - subsystemStatePtr->write(state.residencyInMsecSinceBoot); - subsystemStatePtr->write(state.totalTransitions); - subsystemStatePtr->write(state.lastEntryTimestampMs); - subsystemStatePtr->write(state.supportedOnlyInSuspend); - subsystemStatePtr->init(); - data->push_back(subsystemStatePtr); - VLOG("subsystemstate: %s, %s, %lld, %lld, %lld", - subsystem.name.c_str(), state.name.c_str(), - (long long)state.residencyInMsecSinceBoot, - (long long)state.totalTransitions, - (long long)state.lastEntryTimestampMs); - } + for (size_t i = 0; i < states.size(); i++) { + const PowerStatePlatformSleepState& state = states[i]; + + auto statePtr = make_shared(android::util::PLATFORM_SLEEP_STATE, + timestamp); + statePtr->write(state.name); + statePtr->write(state.residencyInMsecSinceBoot); + statePtr->write(state.totalTransitions); + statePtr->write(state.supportedOnlyInSuspend); + statePtr->init(); + data->push_back(statePtr); + VLOG("powerstate: %s, %lld, %lld, %d", state.name.c_str(), + (long long)state.residencyInMsecSinceBoot, + (long long)state.totalTransitions, + state.supportedOnlyInSuspend ? 1 : 0); + for (auto voter : state.voters) { + auto voterPtr = make_shared(android::util::SLEEP_STATE_VOTER, + timestamp); + voterPtr->write(state.name); + voterPtr->write(voter.name); + voterPtr->write(voter.totalTimeInMsecVotedForSinceBoot); + voterPtr->write(voter.totalNumberOfTimesVotedSinceBoot); + voterPtr->init(); + data->push_back(voterPtr); + VLOG("powerstatevoter: %s, %s, %lld, %lld", state.name.c_str(), + voter.name.c_str(), + (long long)voter.totalTimeInMsecVotedForSinceBoot, + (long long)voter.totalNumberOfTimesVotedSinceBoot); } } }); + if (!ret.isOk()) { + ALOGE("getLowPowerStats() failed: power HAL service not available"); + gPowerHalV1_0 = nullptr; + return false; + } + } + + if (mTagId == android::util::SUBSYSTEM_SLEEP_STATE) { + // Trying to cast to IPower 1.1, this will succeed only for devices supporting 1.1 + sp gPowerHal_1_1 = + android::hardware::power::V1_1::IPower::castFrom(gPowerHalV1_0); + if (gPowerHal_1_1 != nullptr) { + ret = gPowerHal_1_1->getSubsystemLowPowerStats( + [&data, timestamp](hidl_vec subsystems, Status status) { + if (status != Status::SUCCESS) return; + + if (subsystems.size() > 0) { + for (size_t i = 0; i < subsystems.size(); i++) { + const PowerStateSubsystem& subsystem = subsystems[i]; + for (size_t j = 0; j < subsystem.states.size(); j++) { + const PowerStateSubsystemSleepState& state = + subsystem.states[j]; + auto subsystemStatePtr = make_shared( + android::util::SUBSYSTEM_SLEEP_STATE, timestamp); + subsystemStatePtr->write(subsystem.name); + subsystemStatePtr->write(state.name); + subsystemStatePtr->write(state.residencyInMsecSinceBoot); + subsystemStatePtr->write(state.totalTransitions); + subsystemStatePtr->write(state.lastEntryTimestampMs); + subsystemStatePtr->write(state.supportedOnlyInSuspend); + subsystemStatePtr->init(); + data->push_back(subsystemStatePtr); + VLOG("subsystemstate: %s, %s, %lld, %lld, %lld", + subsystem.name.c_str(), state.name.c_str(), + (long long)state.residencyInMsecSinceBoot, + (long long)state.totalTransitions, + (long long)state.lastEntryTimestampMs); + } + } + } + }); + } } return true; } diff --git a/cmds/statsd/src/external/ResourcePowerManagerPuller.h b/cmds/statsd/src/external/ResourcePowerManagerPuller.h index c396c12de0d7a..339940839c4de 100644 --- a/cmds/statsd/src/external/ResourcePowerManagerPuller.h +++ b/cmds/statsd/src/external/ResourcePowerManagerPuller.h @@ -28,7 +28,8 @@ namespace statsd { */ class ResourcePowerManagerPuller : public StatsPuller { public: - bool Pull(const int tagId, vector>* data) override; + ResourcePowerManagerPuller(int tagId); + bool PullInternal(vector>* data) override; }; } // namespace statsd diff --git a/cmds/statsd/src/external/StatsCompanionServicePuller.cpp b/cmds/statsd/src/external/StatsCompanionServicePuller.cpp index ffe1be9ca2e03..b955f1cdaf5a4 100644 --- a/cmds/statsd/src/external/StatsCompanionServicePuller.cpp +++ b/cmds/statsd/src/external/StatsCompanionServicePuller.cpp @@ -22,6 +22,7 @@ #include #include "StatsCompanionServicePuller.h" #include "StatsService.h" +#include "guardrail/StatsdStats.h" using namespace android; using namespace android::base; @@ -39,13 +40,16 @@ const int kLogMsgHeaderSize = 28; // The reading and parsing are implemented in Java. It is not difficult to port over. But for now // let StatsCompanionService handle that and send the data back. -bool StatsCompanionServicePuller::Pull(const int tagId, vector >* data) { +StatsCompanionServicePuller::StatsCompanionServicePuller(int tagId) : StatsPuller(tagId) { +} + +bool StatsCompanionServicePuller::PullInternal(vector >* data) { sp statsCompanion = StatsService::getStatsCompanionService(); vector returned_value; if (statsCompanion != NULL) { - Status status = statsCompanion->pullData(tagId, &returned_value); + Status status = statsCompanion->pullData(mTagId, &returned_value); if (!status.isOk()) { - ALOGW("error pulling for %d", tagId); + ALOGW("error pulling for %d", mTagId); return false; } data->clear(); @@ -60,7 +64,7 @@ bool StatsCompanionServicePuller::Pull(const int tagId, vectorpush_back(make_shared(tmp)); } - ALOGD("StatsCompanionServicePuller::pull succeeded for %d", tagId); + ALOGD("StatsCompanionServicePuller::pull succeeded for %d", mTagId); return true; } else { ALOGW("statsCompanion not found!"); diff --git a/cmds/statsd/src/external/StatsCompanionServicePuller.h b/cmds/statsd/src/external/StatsCompanionServicePuller.h index 3ff2274f1d651..4c91f31c9e2a9 100644 --- a/cmds/statsd/src/external/StatsCompanionServicePuller.h +++ b/cmds/statsd/src/external/StatsCompanionServicePuller.h @@ -25,7 +25,8 @@ namespace statsd { class StatsCompanionServicePuller : public StatsPuller { public: - bool Pull(const int tagId, vector >* data) override; + StatsCompanionServicePuller(int tagId); + bool PullInternal(vector >* data) override; }; } // namespace statsd diff --git a/cmds/statsd/src/external/StatsPuller.cpp b/cmds/statsd/src/external/StatsPuller.cpp new file mode 100644 index 0000000000000..cadc535f2469e --- /dev/null +++ b/cmds/statsd/src/external/StatsPuller.cpp @@ -0,0 +1,64 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define DEBUG false // STOPSHIP if true +#include "Log.h" + +#include "StatsPuller.h" +#include "guardrail/StatsdStats.h" + +namespace android { +namespace os { +namespace statsd { + +using std::lock_guard; + +// ValueMetric has a minimum bucket size of 10min so that we don't pull too frequently +StatsPuller::StatsPuller(const int tagId) + : mTagId(tagId) { + if (StatsdStats::kPullerCooldownMap.find(tagId) == StatsdStats::kPullerCooldownMap.end()) { + mCoolDownSec = StatsdStats::kDefaultPullerCooldown; + } else { + mCoolDownSec = StatsdStats::kPullerCooldownMap[tagId]; + } + VLOG("Puller for tag %d created. Cooldown set to %ld", mTagId, mCoolDownSec); +} + +bool StatsPuller::Pull(std::vector>* data) { + lock_guard lock(mLock); + StatsdStats::getInstance().notePull(mTagId); + long curTime = time(nullptr); + if (curTime - mLastPullTimeSec < mCoolDownSec) { + (*data) = mCachedData; + StatsdStats::getInstance().notePullFromCache(mTagId); + return true; + } + if (mMinPullIntervalSec > curTime - mLastPullTimeSec) { + mMinPullIntervalSec = curTime - mLastPullTimeSec; + StatsdStats::getInstance().updateMinPullIntervalSec(mTagId, mMinPullIntervalSec); + } + mCachedData.clear(); + mLastPullTimeSec = curTime; + bool ret = PullInternal(&mCachedData); + if (ret) { + (*data) = mCachedData; + } + return ret; +} + +} // namespace statsd +} // namespace os +} // namespace android diff --git a/cmds/statsd/src/external/StatsPuller.h b/cmds/statsd/src/external/StatsPuller.h index 940ad9c788ef5..47cc9f01e0547 100644 --- a/cmds/statsd/src/external/StatsPuller.h +++ b/cmds/statsd/src/external/StatsPuller.h @@ -18,11 +18,13 @@ #include #include +#include #include + #include "logd/LogEvent.h" +#include "guardrail/StatsdStats.h" using android::os::StatsLogEventWrapper; -using std::vector; namespace android { namespace os { @@ -30,9 +32,33 @@ namespace statsd { class StatsPuller { public: - virtual ~StatsPuller(){}; + StatsPuller(const int tagId); - virtual bool Pull(const int tagId, vector>* data) = 0; + virtual ~StatsPuller() {} + + bool Pull(std::vector>* data); + +protected: + // The atom tag id this puller pulls + const int mTagId; + +private: + mutable std::mutex mLock; + // Minimum time before this puller does actual pull again. + // If a pull request comes before cooldown, a cached version from purevious pull + // will be returned. + // The actual value should be determined by individual pullers. + long mCoolDownSec; + // For puller stats + long mMinPullIntervalSec = LONG_MAX; + + virtual bool PullInternal(std::vector>* data) = 0; + + // Cache of data from last pull. If next request comes before cool down finishes, + // cached data will be returned. + std::vector> mCachedData; + + long mLastPullTimeSec; }; } // namespace statsd diff --git a/cmds/statsd/src/external/StatsPullerManagerImpl.cpp b/cmds/statsd/src/external/StatsPullerManagerImpl.cpp index d707f8561b215..58c7b126a5a5b 100644 --- a/cmds/statsd/src/external/StatsPullerManagerImpl.cpp +++ b/cmds/statsd/src/external/StatsPullerManagerImpl.cpp @@ -45,29 +45,29 @@ namespace statsd { StatsPullerManagerImpl::StatsPullerManagerImpl() : mCurrentPullingInterval(LONG_MAX) { - shared_ptr statsCompanionServicePuller = make_shared(); - shared_ptr resourcePowerManagerPuller = make_shared(); - shared_ptr cpuTimePerUidPuller = make_shared(); - shared_ptr cpuTimePerUidFreqPuller = make_shared(); - mPullers.insert({android::util::KERNEL_WAKELOCK, - statsCompanionServicePuller}); + make_shared(android::util::KERNEL_WAKELOCK)}); mPullers.insert({android::util::WIFI_BYTES_TRANSFER, - statsCompanionServicePuller}); - mPullers.insert({android::util::MOBILE_BYTES_TRANSFER, - statsCompanionServicePuller}); + make_shared(android::util::WIFI_BYTES_TRANSFER)}); + mPullers.insert( + {android::util::MOBILE_BYTES_TRANSFER, + make_shared(android::util::MOBILE_BYTES_TRANSFER)}); mPullers.insert({android::util::WIFI_BYTES_TRANSFER_BY_FG_BG, - statsCompanionServicePuller}); + make_shared( + android::util::WIFI_BYTES_TRANSFER_BY_FG_BG)}); mPullers.insert({android::util::MOBILE_BYTES_TRANSFER_BY_FG_BG, - statsCompanionServicePuller}); + make_shared( + android::util::MOBILE_BYTES_TRANSFER_BY_FG_BG)}); mPullers.insert({android::util::PLATFORM_SLEEP_STATE, - resourcePowerManagerPuller}); + make_shared(android::util::PLATFORM_SLEEP_STATE)}); mPullers.insert({android::util::SLEEP_STATE_VOTER, - resourcePowerManagerPuller}); - mPullers.insert({android::util::SUBSYSTEM_SLEEP_STATE, - resourcePowerManagerPuller}); - mPullers.insert({android::util::CPU_TIME_PER_UID, cpuTimePerUidPuller}); - mPullers.insert({android::util::CPU_TIME_PER_UID_FREQ, cpuTimePerUidFreqPuller}); + make_shared(android::util::SLEEP_STATE_VOTER)}); + mPullers.insert( + {android::util::SUBSYSTEM_SLEEP_STATE, + make_shared(android::util::SUBSYSTEM_SLEEP_STATE)}); + mPullers.insert({android::util::CPU_TIME_PER_FREQ, make_shared(android::util::CPU_TIME_PER_FREQ)}); + mPullers.insert({android::util::CPU_TIME_PER_UID, make_shared()}); + mPullers.insert({android::util::CPU_TIME_PER_UID_FREQ, make_shared()}); mStatsCompanionService = StatsService::getStatsCompanionService(); } @@ -76,7 +76,7 @@ bool StatsPullerManagerImpl::Pull(int tagId, vector>* data) if (DEBUG) ALOGD("Initiating pulling %d", tagId); if (mPullers.find(tagId) != mPullers.end()) { - bool ret = mPullers.find(tagId)->second->Pull(tagId, data); + bool ret = mPullers.find(tagId)->second->Pull(data); ALOGD("pulled %d items", (int)data->size()); return ret; } else { diff --git a/cmds/statsd/src/guardrail/StatsdStats.cpp b/cmds/statsd/src/guardrail/StatsdStats.cpp index 33927aa9b44c2..5842f3ccbaf9c 100644 --- a/cmds/statsd/src/guardrail/StatsdStats.cpp +++ b/cmds/statsd/src/guardrail/StatsdStats.cpp @@ -19,6 +19,7 @@ #include "StatsdStats.h" #include +#include "../stats_log_util.h" #include "statslog.h" namespace android { @@ -59,6 +60,20 @@ const int FIELD_ID_ATOM_STATS_COUNT = 2; const int FIELD_ID_ANOMALY_ALARMS_REGISTERED = 1; +std::map StatsdStats::kPullerCooldownMap = { + {android::util::KERNEL_WAKELOCK, 1}, + {android::util::WIFI_BYTES_TRANSFER, 1}, + {android::util::MOBILE_BYTES_TRANSFER, 1}, + {android::util::WIFI_BYTES_TRANSFER_BY_FG_BG, 1}, + {android::util::MOBILE_BYTES_TRANSFER_BY_FG_BG, 1}, + {android::util::PLATFORM_SLEEP_STATE, 1}, + {android::util::SLEEP_STATE_VOTER, 1}, + {android::util::SUBSYSTEM_SLEEP_STATE, 1}, + {android::util::CPU_TIME_PER_FREQ, 1}, + {android::util::CPU_TIME_PER_UID, 1}, + {android::util::CPU_TIME_PER_UID_FREQ, 1}, +}; + // TODO: add stats for pulled atoms. StatsdStats::StatsdStats() { mPushedAtomStats.resize(android::util::kMaxPushedAtomId + 1); @@ -231,6 +246,21 @@ void StatsdStats::noteRegisteredAnomalyAlarmChanged() { mAnomalyAlarmRegisteredStats++; } +void StatsdStats::updateMinPullIntervalSec(int pullAtomId, long intervalSec) { + lock_guard lock(mLock); + mPulledAtomStats[pullAtomId].minPullIntervalSec = intervalSec; +} + +void StatsdStats::notePull(int pullAtomId) { + lock_guard lock(mLock); + mPulledAtomStats[pullAtomId].totalPull++; +} + +void StatsdStats::notePullFromCache(int pullAtomId) { + lock_guard lock(mLock); + mPulledAtomStats[pullAtomId].totalPullFromCache++; +} + void StatsdStats::noteAtomLogged(int atomId, int32_t timeSec) { lock_guard lock(mLock); @@ -401,7 +431,7 @@ void StatsdStats::dumpStats(std::vector* output, bool reset) { configStats.clear_alert_stats(); } - VLOG("********Atom stats***********"); + VLOG("********Pushed Atom stats***********"); const size_t atomCounts = mPushedAtomStats.size(); for (size_t i = 2; i < atomCounts; i++) { if (mPushedAtomStats[i] > 0) { @@ -415,6 +445,12 @@ void StatsdStats::dumpStats(std::vector* output, bool reset) { } } + VLOG("********Pulled Atom stats***********"); + for (const auto& pair : mPulledAtomStats) { + android::os::statsd::writePullerStatsToStream(pair, &proto); + VLOG("Atom %d->%ld, %ld, %ld\n", (int) pair.first, (long) pair.second.totalPull, (long) pair.second.totalPullFromCache, (long) pair.second.minPullIntervalSec); + } + if (mAnomalyAlarmRegisteredStats > 0) { VLOG("********AnomalyAlarmStats stats***********"); long long token = proto.start(FIELD_TYPE_MESSAGE | FIELD_ID_ANOMALY_ALARM_STATS); diff --git a/cmds/statsd/src/guardrail/StatsdStats.h b/cmds/statsd/src/guardrail/StatsdStats.h index 45aa192318a4a..14f4132053103 100644 --- a/cmds/statsd/src/guardrail/StatsdStats.h +++ b/cmds/statsd/src/guardrail/StatsdStats.h @@ -17,6 +17,7 @@ #include "config/ConfigKey.h" #include "frameworks/base/cmds/statsd/src/stats_log.pb.h" +#include "statslog.h" #include #include @@ -62,6 +63,13 @@ public: /* Min period between two checks of byte size per config key in nanoseconds. */ static const unsigned long long kMinByteSizeCheckPeriodNs = 10 * NS_PER_SEC; + // Default minimum interval between pulls for an atom. Pullers can return cached values if + // another pull request happens within this interval. + static std::map kPullerCooldownMap; + + // Default cooldown time for a puller + static const long kDefaultPullerCooldown = 1; + /** * Report a new config has been received and report the static stats about the config. * @@ -154,6 +162,15 @@ public: void setUidMapChanges(int changes); void setCurrentUidMapMemory(int bytes); + // Update minimum interval between pulls for an pulled atom + void updateMinPullIntervalSec(int pullAtomId, long intervalSec); + + // Notify pull request for an atom + void notePull(int pullAtomId); + + // Notify pull request for an atom served from cached data + void notePullFromCache(int pullAtomId); + /** * Reset the historical stats. Including all stats in icebox, and the tracked stats about * metrics, matchers, and atoms. The active configs will be kept and StatsdStats will continue @@ -168,6 +185,12 @@ public: */ void dumpStats(std::vector* buffer, bool reset); + typedef struct { + long totalPull; + long totalPullFromCache; + long minPullIntervalSec; + } PulledAtomStats; + private: StatsdStats(); @@ -200,6 +223,8 @@ private: // This is a vector, not a map because it will be accessed A LOT -- for each stats log. std::vector mPushedAtomStats; + std::map mPulledAtomStats; + // Stores the number of times statsd modified the anomaly alarm registered with // StatsCompanionService. int mAnomalyAlarmRegisteredStats = 0; diff --git a/cmds/statsd/src/metrics/GaugeMetricProducer.cpp b/cmds/statsd/src/metrics/GaugeMetricProducer.cpp index e6f311b0a7fd6..1a4888c31ffff 100644 --- a/cmds/statsd/src/metrics/GaugeMetricProducer.cpp +++ b/cmds/statsd/src/metrics/GaugeMetricProducer.cpp @@ -326,6 +326,7 @@ void GaugeMetricProducer::flushIfNeededLocked(const uint64_t& eventTimeNs) { } } + mCurrentSlicedBucketForAnomaly = std::make_shared(); mCurrentSlicedBucket = std::make_shared(); // Adjusts the bucket start time diff --git a/cmds/statsd/src/stats_log.proto b/cmds/statsd/src/stats_log.proto index ae69a5070f5fc..2596a5f4fdc46 100644 --- a/cmds/statsd/src/stats_log.proto +++ b/cmds/statsd/src/stats_log.proto @@ -246,4 +246,12 @@ message StatsdStatsReport { optional int32 alarms_registered = 1; } optional AnomalyAlarmStats anomaly_alarm_stats = 9; + + message PulledAtomStats { + optional int32 atom_id = 1; + optional int64 total_pull = 2; + optional int64 total_pull_from_cache = 3; + optional int64 min_pull_interval_sec = 4; + } + repeated PulledAtomStats pulled_atom_stats = 10; } \ No newline at end of file diff --git a/cmds/statsd/src/stats_log_util.cpp b/cmds/statsd/src/stats_log_util.cpp index b335b580183ce..a41f30c2bece0 100644 --- a/cmds/statsd/src/stats_log_util.cpp +++ b/cmds/statsd/src/stats_log_util.cpp @@ -45,6 +45,13 @@ const int DIMENSIONS_VALUE_VALUE_TUPLE = 7; // for MessageValue Proto const int FIELD_ID_FIELD_VALUE_IN_MESSAGE_VALUE_PROTO = 1; +// for PulledAtomStats proto +const int FIELD_ID_PULLED_ATOM_STATS = 10; +const int FIELD_ID_PULL_ATOM_ID = 1; +const int FIELD_ID_TOTAL_PULL = 2; +const int FIELD_ID_TOTAL_PULL_FROM_CACHE = 3; +const int FIELD_ID_MIN_PULL_INTERVAL_SEC = 4; + void writeDimensionsValueProtoToStream(const DimensionsValue& dimensionsValue, ProtoOutputStream* protoOutput) { protoOutput->write(FIELD_TYPE_INT32 | DIMENSIONS_VALUE_FIELD, dimensionsValue.field()); @@ -248,6 +255,19 @@ int64_t TimeUnitToBucketSizeInMillis(TimeUnit unit) { } } +void writePullerStatsToStream(const std::pair& pair, + util::ProtoOutputStream* protoOutput) { + long long token = protoOutput->start(FIELD_TYPE_MESSAGE | FIELD_ID_PULLED_ATOM_STATS | + FIELD_COUNT_REPEATED); + protoOutput->write(FIELD_TYPE_INT32 | FIELD_ID_PULL_ATOM_ID, (int32_t)pair.first); + protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_TOTAL_PULL, (long long)pair.second.totalPull); + protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_TOTAL_PULL_FROM_CACHE, + (long long)pair.second.totalPullFromCache); + protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_MIN_PULL_INTERVAL_SEC, + (long long)pair.second.minPullIntervalSec); + protoOutput->end(token); +} + } // namespace statsd } // namespace os } // namespace android \ No newline at end of file diff --git a/cmds/statsd/src/stats_log_util.h b/cmds/statsd/src/stats_log_util.h index 33303dc951adb..09a43f5c881f5 100644 --- a/cmds/statsd/src/stats_log_util.h +++ b/cmds/statsd/src/stats_log_util.h @@ -17,29 +17,32 @@ #pragma once #include -#include "frameworks/base/cmds/statsd/src/statsd_config.pb.h" -#include "frameworks/base/cmds/statsd/src/stats_log.pb.h" #include "field_util.h" +#include "frameworks/base/cmds/statsd/src/stats_log.pb.h" +#include "frameworks/base/cmds/statsd/src/statsd_config.pb.h" +#include "guardrail/StatsdStats.h" namespace android { namespace os { namespace statsd { // Helper function to write DimensionsValue proto to ProtoOutputStream. -void writeDimensionsValueProtoToStream( - const DimensionsValue& fieldValue, util::ProtoOutputStream* protoOutput); +void writeDimensionsValueProtoToStream(const DimensionsValue& fieldValue, + util::ProtoOutputStream* protoOutput); // Helper function to write Field proto to ProtoOutputStream. -void writeFieldProtoToStream( - const Field& field, util::ProtoOutputStream* protoOutput); +void writeFieldProtoToStream(const Field& field, util::ProtoOutputStream* protoOutput); // Helper function to construct the field value tree and write to ProtoOutputStream -void writeFieldValueTreeToStream(const FieldValueMap &fieldValueMap, - util::ProtoOutputStream* protoOutput); +void writeFieldValueTreeToStream(const FieldValueMap& fieldValueMap, + util::ProtoOutputStream* protoOutput); // Convert the TimeUnit enum to the bucket size in millis. int64_t TimeUnitToBucketSizeInMillis(TimeUnit unit); +// Helper function to write PulledAtomStats to ProtoOutputStream +void writePullerStatsToStream(const std::pair& pair, + util::ProtoOutputStream* protoOutput); } // namespace statsd } // namespace os } // namespace android \ No newline at end of file