From 9a43b4f02763872a7843d23b2fe050976669f021 Mon Sep 17 00:00:00 2001 From: Yao Chen Date: Wed, 10 Apr 2019 10:43:20 -0700 Subject: [PATCH] Add a config option to save the metrics data locally. Usually after config owner calls statsd to get data, statsd will remove that data forever. This cl adds an option to save historical data locally so that when a bugreport is taken, it contains historical statsd metrics for debugging. + All existing guardrail on storage still apply, and local history data has lower priority when the guardrail is hit. On top of that, statsd will remove history files that are more than 2 days old. + Also fixed issues inside StorageManager (e.g., using string::compare to sort by timestamp is problematic, redundant file size read, etc) Test: statsd_test and manually Fix: 126562025 Change-Id: I5238b9fd3069cbef5bcccabd2212bc4d6c734702 --- cmds/statsd/src/StatsLogProcessor.cpp | 123 +++++----- cmds/statsd/src/StatsLogProcessor.h | 13 +- cmds/statsd/src/guardrail/StatsdStats.h | 3 + cmds/statsd/src/metrics/MetricsManager.cpp | 3 +- cmds/statsd/src/metrics/MetricsManager.h | 6 + cmds/statsd/src/statsd_config.proto | 2 + cmds/statsd/src/storage/StorageManager.cpp | 214 ++++++++++-------- cmds/statsd/src/storage/StorageManager.h | 34 ++- .../tests/storage/StorageManager_test.cpp | 15 ++ 9 files changed, 257 insertions(+), 156 deletions(-) diff --git a/cmds/statsd/src/StatsLogProcessor.cpp b/cmds/statsd/src/StatsLogProcessor.cpp index df84b6a4dc341..a9f5208ef812d 100644 --- a/cmds/statsd/src/StatsLogProcessor.cpp +++ b/cmds/statsd/src/StatsLogProcessor.cpp @@ -77,7 +77,6 @@ const int FIELD_ID_TIME_TO_LIVE_NANOS = 2; #define NS_PER_HOUR 3600 * NS_PER_SEC -#define STATS_DATA_DIR "/data/misc/stats-data" #define STATS_ACTIVE_METRIC_DIR "/data/misc/stats-active-metric" // Cool down period for writing data to disk to avoid overwriting files. @@ -106,6 +105,19 @@ StatsLogProcessor::StatsLogProcessor(const sp& uidMap, StatsLogProcessor::~StatsLogProcessor() { } +static void flushProtoToBuffer(ProtoOutputStream& proto, vector* outData) { + outData->clear(); + outData->resize(proto.size()); + size_t pos = 0; + sp reader = proto.data(); + while (reader->readBuffer() != NULL) { + size_t toRead = reader->currentToRead(); + std::memcpy(&((*outData)[pos]), reader->readBuffer(), toRead); + pos += toRead; + reader->move(toRead); + } +} + void StatsLogProcessor::onAnomalyAlarmFired( const int64_t& timestampNs, unordered_set, SpHash> alarmSet) { @@ -366,25 +378,29 @@ void StatsLogProcessor::onDumpReport(const ConfigKey& key, const int64_t dumpTim proto->end(configKeyToken); // End of ConfigKey. + bool keepFile = false; + auto it = mMetricsManagers.find(key); + if (it != mMetricsManagers.end() && it->second->shouldPersistLocalHistory()) { + keepFile = true; + } + // Then, check stats-data directory to see there's any file containing // ConfigMetricsReport from previous shutdowns to concatenate to reports. - StorageManager::appendConfigMetricsReport(key, proto, erase_data); + StorageManager::appendConfigMetricsReport( + key, proto, erase_data && !keepFile /* should remove file after appending it */, + dumpReportReason == ADB_DUMP /*if caller is adb*/); - auto it = mMetricsManagers.find(key); if (it != mMetricsManagers.end()) { // This allows another broadcast to be sent within the rate-limit period if we get close to // filling the buffer again soon. mLastBroadcastTimes.erase(key); - // Start of ConfigMetricsReport (reports). - uint64_t reportsToken = - proto->start(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_REPORTS); - onConfigMetricsReportLocked(key, dumpTimeStampNs, - include_current_partial_bucket, - erase_data, dumpReportReason, - dumpLatency, proto); - proto->end(reportsToken); - // End of ConfigMetricsReport (reports). + vector buffer; + onConfigMetricsReportLocked(key, dumpTimeStampNs, include_current_partial_bucket, + erase_data, dumpReportReason, dumpLatency, + false /* is this data going to be saved on disk */, &buffer); + proto->write(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_REPORTS, + reinterpret_cast(buffer.data()), buffer.size()); } else { ALOGW("Config source %s does not exist", key.ToString().c_str()); } @@ -404,16 +420,8 @@ void StatsLogProcessor::onDumpReport(const ConfigKey& key, const int64_t dumpTim dumpReportReason, dumpLatency, &proto); if (outData != nullptr) { - outData->clear(); - outData->resize(proto.size()); - size_t pos = 0; - sp reader = proto.data(); - while (reader->readBuffer() != NULL) { - size_t toRead = reader->currentToRead(); - std::memcpy(&((*outData)[pos]), reader->readBuffer(), toRead); - pos += toRead; - reader->move(toRead); - } + flushProtoToBuffer(proto, outData); + VLOG("output data size %zu", outData->size()); } StatsdStats::getInstance().noteMetricsReportSent(key, proto.size()); @@ -422,13 +430,11 @@ void StatsLogProcessor::onDumpReport(const ConfigKey& key, const int64_t dumpTim /* * onConfigMetricsReportLocked dumps serialized ConfigMetricsReport into outData. */ -void StatsLogProcessor::onConfigMetricsReportLocked(const ConfigKey& key, - const int64_t dumpTimeStampNs, - const bool include_current_partial_bucket, - const bool erase_data, - const DumpReportReason dumpReportReason, - const DumpLatency dumpLatency, - ProtoOutputStream* proto) { +void StatsLogProcessor::onConfigMetricsReportLocked( + const ConfigKey& key, const int64_t dumpTimeStampNs, + const bool include_current_partial_bucket, const bool erase_data, + const DumpReportReason dumpReportReason, const DumpLatency dumpLatency, + const bool dataSavedOnDisk, vector* buffer) { // We already checked whether key exists in mMetricsManagers in // WriteDataToDisk. auto it = mMetricsManagers.find(key); @@ -440,35 +446,46 @@ void StatsLogProcessor::onConfigMetricsReportLocked(const ConfigKey& key, std::set str_set; + ProtoOutputStream tempProto; // First, fill in ConfigMetricsReport using current data on memory, which // starts from filling in StatsLogReport's. - it->second->onDumpReport(dumpTimeStampNs, include_current_partial_bucket, - erase_data, dumpLatency, &str_set, proto); + it->second->onDumpReport(dumpTimeStampNs, include_current_partial_bucket, erase_data, + dumpLatency, &str_set, &tempProto); // Fill in UidMap if there is at least one metric to report. // This skips the uid map if it's an empty config. if (it->second->getNumMetrics() > 0) { - uint64_t uidMapToken = proto->start(FIELD_TYPE_MESSAGE | FIELD_ID_UID_MAP); + uint64_t uidMapToken = tempProto.start(FIELD_TYPE_MESSAGE | FIELD_ID_UID_MAP); mUidMap->appendUidMap( dumpTimeStampNs, key, it->second->hashStringInReport() ? &str_set : nullptr, - it->second->versionStringsInReport(), it->second->installerInReport(), proto); - proto->end(uidMapToken); + it->second->versionStringsInReport(), it->second->installerInReport(), &tempProto); + tempProto.end(uidMapToken); } // Fill in the timestamps. - proto->write(FIELD_TYPE_INT64 | FIELD_ID_LAST_REPORT_ELAPSED_NANOS, - (long long)lastReportTimeNs); - proto->write(FIELD_TYPE_INT64 | FIELD_ID_CURRENT_REPORT_ELAPSED_NANOS, - (long long)dumpTimeStampNs); - proto->write(FIELD_TYPE_INT64 | FIELD_ID_LAST_REPORT_WALL_CLOCK_NANOS, - (long long)lastReportWallClockNs); - proto->write(FIELD_TYPE_INT64 | FIELD_ID_CURRENT_REPORT_WALL_CLOCK_NANOS, - (long long)getWallClockNs()); + tempProto.write(FIELD_TYPE_INT64 | FIELD_ID_LAST_REPORT_ELAPSED_NANOS, + (long long)lastReportTimeNs); + tempProto.write(FIELD_TYPE_INT64 | FIELD_ID_CURRENT_REPORT_ELAPSED_NANOS, + (long long)dumpTimeStampNs); + tempProto.write(FIELD_TYPE_INT64 | FIELD_ID_LAST_REPORT_WALL_CLOCK_NANOS, + (long long)lastReportWallClockNs); + tempProto.write(FIELD_TYPE_INT64 | FIELD_ID_CURRENT_REPORT_WALL_CLOCK_NANOS, + (long long)getWallClockNs()); // Dump report reason - proto->write(FIELD_TYPE_INT32 | FIELD_ID_DUMP_REPORT_REASON, dumpReportReason); + tempProto.write(FIELD_TYPE_INT32 | FIELD_ID_DUMP_REPORT_REASON, dumpReportReason); for (const auto& str : str_set) { - proto->write(FIELD_TYPE_STRING | FIELD_COUNT_REPEATED | FIELD_ID_STRINGS, str); + tempProto.write(FIELD_TYPE_STRING | FIELD_COUNT_REPEATED | FIELD_ID_STRINGS, str); + } + + flushProtoToBuffer(tempProto, buffer); + + // save buffer to disk if needed + if (erase_data && !dataSavedOnDisk && it->second->shouldPersistLocalHistory()) { + VLOG("save history to disk"); + string file_name = StorageManager::getDataHistoryFileName((long)getWallClockSec(), + key.GetUid(), key.GetId()); + StorageManager::writeFile(file_name.c_str(), buffer->data(), buffer->size()); } } @@ -584,18 +601,14 @@ void StatsLogProcessor::WriteDataToDiskLocked(const ConfigKey& key, !mMetricsManagers.find(key)->second->shouldWriteToDisk()) { return; } - ProtoOutputStream proto; + vector buffer; onConfigMetricsReportLocked(key, timestampNs, true /* include_current_partial_bucket*/, - true /* erase_data */, dumpReportReason, dumpLatency, &proto); - string file_name = StringPrintf("%s/%ld_%d_%lld", STATS_DATA_DIR, - (long)getWallClockSec(), key.GetUid(), (long long)key.GetId()); - android::base::unique_fd fd(open(file_name.c_str(), - O_WRONLY | O_CREAT | O_CLOEXEC, S_IRUSR | S_IWUSR)); - if (fd == -1) { - ALOGE("Attempt to write %s but failed", file_name.c_str()); - return; - } - proto.flush(fd.get()); + true /* erase_data */, dumpReportReason, dumpLatency, true, + &buffer); + string file_name = + StorageManager::getDataFileName((long)getWallClockSec(), key.GetUid(), key.GetId()); + StorageManager::writeFile(file_name.c_str(), buffer.data(), buffer.size()); + // We were able to write the ConfigMetricsReport to disk, so we should trigger collection ASAP. mOnDiskDataConfigs.insert(key); } diff --git a/cmds/statsd/src/StatsLogProcessor.h b/cmds/statsd/src/StatsLogProcessor.h index 305a4ce24b498..f4db0af29cefe 100644 --- a/cmds/statsd/src/StatsLogProcessor.h +++ b/cmds/statsd/src/StatsLogProcessor.h @@ -164,12 +164,13 @@ private: const DumpReportReason dumpReportReason, const DumpLatency dumpLatency); - void onConfigMetricsReportLocked(const ConfigKey& key, const int64_t dumpTimeStampNs, - const bool include_current_partial_bucket, - const bool erase_data, - const DumpReportReason dumpReportReason, - const DumpLatency dumpLatency, - util::ProtoOutputStream* proto); + void onConfigMetricsReportLocked( + const ConfigKey& key, const int64_t dumpTimeStampNs, + const bool include_current_partial_bucket, const bool erase_data, + const DumpReportReason dumpReportReason, const DumpLatency dumpLatency, + /*if dataSavedToDisk is true, it indicates the caller will write the data to disk + (e.g., before reboot). So no need to further persist local history.*/ + const bool dataSavedToDisk, vector* proto); /* Check if we should send a broadcast if approaching memory limits and if we're over, we * actually delete the data. */ diff --git a/cmds/statsd/src/guardrail/StatsdStats.h b/cmds/statsd/src/guardrail/StatsdStats.h index 88ecccc32407a..194001a1ee39e 100644 --- a/cmds/statsd/src/guardrail/StatsdStats.h +++ b/cmds/statsd/src/guardrail/StatsdStats.h @@ -145,6 +145,9 @@ public: // Maximum age (30 days) that files on disk can exist in seconds. static const int kMaxAgeSecond = 60 * 60 * 24 * 30; + // Maximum age (2 days) that local history files on disk can exist in seconds. + static const int kMaxLocalHistoryAgeSecond = 60 * 60 * 24 * 2; + // Maximum number of files (1000) that can be in stats directory on disk. static const int kMaxFileNumber = 1000; diff --git a/cmds/statsd/src/metrics/MetricsManager.cpp b/cmds/statsd/src/metrics/MetricsManager.cpp index 095f9dde61294..6a55289bc8a25 100644 --- a/cmds/statsd/src/metrics/MetricsManager.cpp +++ b/cmds/statsd/src/metrics/MetricsManager.cpp @@ -65,7 +65,8 @@ MetricsManager::MetricsManager(const ConfigKey& key, const StatsdConfig& config, mTtlNs(config.has_ttl_in_seconds() ? config.ttl_in_seconds() * NS_PER_SEC : -1), mTtlEndNs(-1), mLastReportTimeNs(currentTimeNs), - mLastReportWallClockNs(getWallClockNs()) { + mLastReportWallClockNs(getWallClockNs()), + mShouldPersistHistory(config.persist_locally()) { // Init the ttl end timestamp. refreshTtl(timeBaseNs); diff --git a/cmds/statsd/src/metrics/MetricsManager.h b/cmds/statsd/src/metrics/MetricsManager.h index d317f8e00d820..00ae3b7028d93 100644 --- a/cmds/statsd/src/metrics/MetricsManager.h +++ b/cmds/statsd/src/metrics/MetricsManager.h @@ -78,6 +78,10 @@ public: return mNoReportMetricIds.size() != mAllMetricProducers.size(); } + bool shouldPersistLocalHistory() const { + return mShouldPersistHistory; + } + void dumpStates(FILE* out, bool verbose); inline bool isInTtl(const int64_t timestampNs) const { @@ -184,6 +188,8 @@ private: // Contains the annotations passed in with StatsdConfig. std::list> mAnnotations; + const bool mShouldPersistHistory; + // To guard access to mAllowedLogSources mutable std::mutex mAllowedLogSourcesMutex; diff --git a/cmds/statsd/src/statsd_config.proto b/cmds/statsd/src/statsd_config.proto index 257e65ee423d4..2260b9b56d0b0 100644 --- a/cmds/statsd/src/statsd_config.proto +++ b/cmds/statsd/src/statsd_config.proto @@ -439,6 +439,8 @@ message StatsdConfig { optional bool installer_in_metric_report = 19; + optional bool persist_locally = 20 [default = false]; + // Field number 1000 is reserved for later use. reserved 1000; } diff --git a/cmds/statsd/src/storage/StorageManager.cpp b/cmds/statsd/src/storage/StorageManager.cpp index cf8b97494a067..0a9161d51cfe7 100644 --- a/cmds/statsd/src/storage/StorageManager.cpp +++ b/cmds/statsd/src/storage/StorageManager.cpp @@ -56,9 +56,31 @@ std::mutex StorageManager::sTrainInfoMutex; using android::base::StringPrintf; using std::unique_ptr; -// Returns array of int64_t which contains timestamp in seconds, uid, and -// configID. -static void parseFileName(char* name, int64_t* result) { +struct FileName { + int64_t mTimestampSec; + int mUid; + int64_t mConfigId; + bool mIsHistory; + string getFullFileName(const char* path) { + return StringPrintf("%s/%lld_%d_%lld%s", path, (long long)mTimestampSec, (int)mUid, + (long long)mConfigId, (mIsHistory ? "_history" : "")); + }; +}; + +string StorageManager::getDataFileName(long wallClockSec, int uid, int64_t id) { + return StringPrintf("%s/%ld_%d_%lld", STATS_DATA_DIR, wallClockSec, uid, + (long long)id); +} + +string StorageManager::getDataHistoryFileName(long wallClockSec, int uid, int64_t id) { + return StringPrintf("%s/%ld_%d_%lld_history", STATS_DATA_DIR, wallClockSec, uid, + (long long)id); +} + +// Returns array of int64_t which contains timestamp in seconds, uid, +// configID and whether the file is a local history file. +static void parseFileName(char* name, FileName* output) { + int64_t result[3]; int index = 0; char* substr = strtok(name, "_"); while (substr != nullptr && index < 3) { @@ -72,11 +94,12 @@ static void parseFileName(char* name, int64_t* result) { if (index < 3) { result[0] = -1; } -} -static string getFilePath(const char* path, int64_t timestamp, int64_t uid, int64_t configID) { - return StringPrintf("%s/%lld_%d_%lld", path, (long long)timestamp, (int)uid, - (long long)configID); + output->mTimestampSec = result[0]; + output->mUid = result[1]; + output->mConfigId = result[2]; + // check if the file is a local history. + output->mIsHistory = (substr != nullptr && strcmp("history", substr) == 0); } void StorageManager::writeFile(const char* file, const void* buffer, int numBytes) { @@ -88,14 +111,13 @@ void StorageManager::writeFile(const char* file, const void* buffer, int numByte trimToFit(STATS_SERVICE_DIR); trimToFit(STATS_DATA_DIR); - int result = write(fd, buffer, numBytes); - if (result == numBytes) { + if (android::base::WriteFully(fd, buffer, numBytes)) { VLOG("Successfully wrote %s", file); } else { - VLOG("Failed to write %s", file); + ALOGE("Failed to write %s", file); } - result = fchown(fd, AID_STATSD, AID_STATSD); + int result = fchown(fd, AID_STATSD, AID_STATSD); if (result) { VLOG("Failed to chown %s to statsd", file); } @@ -349,13 +371,10 @@ void StorageManager::sendBroadcast(const char* path, if (name[0] == '.') continue; VLOG("file %s", name); - int64_t result[3]; - parseFileName(name, result); - if (result[0] == -1) continue; - int64_t uid = result[1]; - int64_t configID = result[2]; - - sendBroadcast(ConfigKey((int)uid, configID)); + FileName output; + parseFileName(name, &output); + if (output.mTimestampSec == -1 || output.mIsHistory) continue; + sendBroadcast(ConfigKey((int)output.mUid, output.mConfigId)); } } @@ -378,55 +397,58 @@ bool StorageManager::hasConfigMetricsReport(const ConfigKey& key) { if (suffixLen <= nameLen && strncmp(name + nameLen - suffixLen, suffix.c_str(), suffixLen) == 0) { // Check again that the file name is parseable. - int64_t result[3]; - parseFileName(name, result); - if (result[0] == -1) continue; + FileName output; + parseFileName(name, &output); + if (output.mTimestampSec == -1 || output.mIsHistory) continue; return true; } } return false; } -void StorageManager::appendConfigMetricsReport(const ConfigKey& key, - ProtoOutputStream* proto, - bool erasa_data) { +void StorageManager::appendConfigMetricsReport(const ConfigKey& key, ProtoOutputStream* proto, + bool erase_data, bool isAdb) { unique_ptr dir(opendir(STATS_DATA_DIR), closedir); if (dir == NULL) { VLOG("Path %s does not exist", STATS_DATA_DIR); return; } - string suffix = StringPrintf("%d_%lld", key.GetUid(), (long long)key.GetId()); - dirent* de; while ((de = readdir(dir.get()))) { char* name = de->d_name; + string fileName(name); if (name[0] == '.') continue; + FileName output; + parseFileName(name, &output); - size_t nameLen = strlen(name); - size_t suffixLen = suffix.length(); - if (suffixLen <= nameLen && - strncmp(name + nameLen - suffixLen, suffix.c_str(), suffixLen) == 0) { - int64_t result[3]; - parseFileName(name, result); - if (result[0] == -1) continue; - int64_t timestamp = result[0]; - int64_t uid = result[1]; - int64_t configID = result[2]; + if (output.mTimestampSec == -1 || (output.mIsHistory && !isAdb) || + output.mUid != key.GetUid() || output.mConfigId != key.GetId()) { + continue; + } - string file_name = getFilePath(STATS_DATA_DIR, timestamp, uid, configID); - int fd = open(file_name.c_str(), O_RDONLY | O_CLOEXEC); - if (fd != -1) { - string content; - if (android::base::ReadFdToString(fd, &content)) { - proto->write(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_REPORTS, - content.c_str(), content.size()); - } - close(fd); + auto fullPathName = StringPrintf("%s/%s", STATS_DATA_DIR, fileName.c_str()); + int fd = open(fullPathName.c_str(), O_RDONLY | O_CLOEXEC); + if (fd != -1) { + string content; + if (android::base::ReadFdToString(fd, &content)) { + proto->write(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_REPORTS, + content.c_str(), content.size()); } + close(fd); + } else { + ALOGE("file cannot be opened"); + } - if (erasa_data) { - remove(file_name.c_str()); + if (erase_data) { + remove(fullPathName.c_str()); + } else if (output.mIsHistory && !isAdb) { + // This means a real data owner has called to get this data. But the config says it + // wants to keep a local history. So now this file must be renamed as a history file. + // So that next time, when owner calls getData() again, this data won't be uploaded + // again. rename returns 0 on success + if (rename(fullPathName.c_str(), (fullPathName + "_history").c_str())) { + ALOGE("Failed to rename file %s", fullPathName.c_str()); } } } @@ -458,23 +480,20 @@ void StorageManager::readConfigFromDisk(map& configsMap while ((de = readdir(dir.get()))) { char* name = de->d_name; if (name[0] == '.') continue; - VLOG("file %s", name); - int64_t result[3]; - parseFileName(name, result); - if (result[0] == -1) continue; - int64_t timestamp = result[0]; - int64_t uid = result[1]; - int64_t configID = result[2]; - string file_name = getFilePath(STATS_SERVICE_DIR, timestamp, uid, configID); + FileName output; + parseFileName(name, &output); + if (output.mTimestampSec == -1) continue; + string file_name = output.getFullFileName(STATS_SERVICE_DIR); int fd = open(file_name.c_str(), O_RDONLY | O_CLOEXEC); if (fd != -1) { string content; if (android::base::ReadFdToString(fd, &content)) { StatsdConfig config; if (config.ParseFromString(content)) { - configsMap[ConfigKey(uid, configID)] = config; - VLOG("map key uid=%lld|configID=%lld", (long long)uid, (long long)configID); + configsMap[ConfigKey(output.mUid, output.mConfigId)] = config; + VLOG("map key uid=%lld|configID=%lld", (long long)output.mUid, + (long long)output.mConfigId); } } close(fd); @@ -533,6 +552,30 @@ bool StorageManager::hasIdenticalConfig(const ConfigKey& key, return false; } +void StorageManager::sortFiles(vector* fileNames) { + // Reverse sort to effectively remove from the back (oldest entries). + // This will sort files in reverse-chronological order. Local history files have lower + // priority than regular data files. + sort(fileNames->begin(), fileNames->end(), [](FileInfo& lhs, FileInfo& rhs) { + // first consider if the file is a local history + if (lhs.mIsHistory && !rhs.mIsHistory) { + return false; + } else if (rhs.mIsHistory && !lhs.mIsHistory) { + return true; + } + + // then consider the age. + if (lhs.mFileAgeSec < rhs.mFileAgeSec) { + return true; + } else if (lhs.mFileAgeSec > rhs.mFileAgeSec) { + return false; + } + + // then good luck.... use string::compare + return lhs.mFileName.compare(rhs.mFileName) > 0; + }); +} + void StorageManager::trimToFit(const char* path) { unique_ptr dir(opendir(path), closedir); if (dir == NULL) { @@ -541,55 +584,46 @@ void StorageManager::trimToFit(const char* path) { } dirent* de; int totalFileSize = 0; - vector fileNames; + vector fileNames; + auto nowSec = getWallClockSec(); while ((de = readdir(dir.get()))) { char* name = de->d_name; if (name[0] == '.') continue; - int64_t result[3]; - parseFileName(name, result); - if (result[0] == -1) continue; - int64_t timestamp = result[0]; - int64_t uid = result[1]; - int64_t configID = result[2]; - string file_name = getFilePath(path, timestamp, uid, configID); + FileName output; + parseFileName(name, &output); + if (output.mTimestampSec == -1) continue; + string file_name = output.getFullFileName(path); // Check for timestamp and delete if it's too old. - long fileAge = getWallClockSec() - timestamp; - if (fileAge > StatsdStats::kMaxAgeSecond) { + long fileAge = nowSec - output.mTimestampSec; + if (fileAge > StatsdStats::kMaxAgeSecond || + (output.mIsHistory && fileAge > StatsdStats::kMaxLocalHistoryAgeSecond)) { deleteFile(file_name.c_str()); + continue; } - fileNames.push_back(file_name); ifstream file(file_name.c_str(), ifstream::in | ifstream::binary); + int fileSize = 0; if (file.is_open()) { file.seekg(0, ios::end); - int fileSize = file.tellg(); + fileSize = file.tellg(); file.close(); totalFileSize += fileSize; } + fileNames.emplace_back(file_name, output.mIsHistory, fileSize, fileAge); } if (fileNames.size() > StatsdStats::kMaxFileNumber || totalFileSize > StatsdStats::kMaxFileSize) { - // Reverse sort to effectively remove from the back (oldest entries). - // This will sort files in reverse-chronological order. - sort(fileNames.begin(), fileNames.end(), std::greater()); + sortFiles(&fileNames); } // Start removing files from oldest to be under the limit. while (fileNames.size() > 0 && (fileNames.size() > StatsdStats::kMaxFileNumber || totalFileSize > StatsdStats::kMaxFileSize)) { - string file_name = fileNames.at(fileNames.size() - 1); - ifstream file(file_name.c_str(), ifstream::in | ifstream::binary); - if (file.is_open()) { - file.seekg(0, ios::end); - int fileSize = file.tellg(); - file.close(); - totalFileSize -= fileSize; - } - - deleteFile(file_name.c_str()); + totalFileSize -= fileNames.at(fileNames.size() - 1).mFileSizeBytes; + deleteFile(fileNames.at(fileNames.size() - 1).mFileName.c_str()); fileNames.pop_back(); } } @@ -614,15 +648,13 @@ void StorageManager::printDirStats(int outFd, const char* path) { if (name[0] == '.') { continue; } - int64_t result[3]; - parseFileName(name, result); - if (result[0] == -1) continue; - int64_t timestamp = result[0]; - int64_t uid = result[1]; - int64_t configID = result[2]; - dprintf(outFd, "\t #%d, Last updated: %lld, UID: %d, Config ID: %lld", fileCount + 1, - (long long)timestamp, (int)uid, (long long)configID); - string file_name = getFilePath(path, timestamp, uid, configID); + FileName output; + parseFileName(name, &output); + if (output.mTimestampSec == -1) continue; + dprintf(outFd, "\t #%d, Last updated: %lld, UID: %d, Config ID: %lld, %s", fileCount + 1, + (long long)output.mTimestampSec, output.mUid, (long long)output.mConfigId, + (output.mIsHistory ? "local history" : "")); + string file_name = output.getFullFileName(path); ifstream file(file_name.c_str(), ifstream::in | ifstream::binary); if (file.is_open()) { file.seekg(0, ios::end); diff --git a/cmds/statsd/src/storage/StorageManager.h b/cmds/statsd/src/storage/StorageManager.h index dfcea65b0872a..69b41c2cb9747 100644 --- a/cmds/statsd/src/storage/StorageManager.h +++ b/cmds/statsd/src/storage/StorageManager.h @@ -31,6 +31,19 @@ using android::util::ProtoOutputStream; class StorageManager : public virtual RefBase { public: + struct FileInfo { + FileInfo(std::string name, bool isHistory, int fileSize, long fileAge) + : mFileName(name), + mIsHistory(isHistory), + mFileSizeBytes(fileSize), + mFileAgeSec(fileAge) { + } + std::string mFileName; + bool mIsHistory; + int mFileSizeBytes; + long mFileAgeSec; + }; + /** * Writes a given byte array as a file to the specified file path. */ @@ -81,10 +94,19 @@ public: /** * Appends the ConfigMetricsReport found on disk to the specifid proto * and, if erase_data, deletes it from disk. + * + * [isAdb]: if the caller is adb dump. This includes local adb dump or dumpsys by + * bugreport or incidentd. When true, we will append any local history data too. + * + * When + * erase_data=true, isAdb=true: append history data to output, remove all data after read + * erase_data=false, isAdb=true: append history data to output, keep data after read + * erase_data=true, isAdb=false: do not append history data, and remove data after read + * erase_data=false, isAdb=false: do not append history data and *rename* all data files to + * history files. */ - static void appendConfigMetricsReport(const ConfigKey& key, - ProtoOutputStream* proto, - bool erase_data); + static void appendConfigMetricsReport(const ConfigKey& key, ProtoOutputStream* proto, + bool erase_data, bool isAdb); /** * Call to load the saved configs from disk. @@ -115,6 +137,12 @@ public: */ static void printStats(int out); + static string getDataFileName(long wallClockSec, int uid, int64_t id); + + static string getDataHistoryFileName(long wallClockSec, int uid, int64_t id); + + static void sortFiles(vector* fileNames); + private: /** * Prints disk usage statistics about a directory related to statsd. diff --git a/cmds/statsd/tests/storage/StorageManager_test.cpp b/cmds/statsd/tests/storage/StorageManager_test.cpp index 4564a5d058a3b..cae2f30698556 100644 --- a/cmds/statsd/tests/storage/StorageManager_test.cpp +++ b/cmds/statsd/tests/storage/StorageManager_test.cpp @@ -110,6 +110,21 @@ TEST(StorageManagerTest, TrainInfoReadWriteTrainNameSizeOneTest) { EXPECT_EQ(trainInfo.experimentIds, trainInfoResult.experimentIds); } +TEST(StorageManagerTest, SortFileTest) { + vector list; + // assume now sec is 500 + list.emplace_back("200_5000_123454", false, 20, 300); + list.emplace_back("300_2000_123454_history", true, 30, 200); + list.emplace_back("400_100009_123454_history", true, 40, 100); + list.emplace_back("100_2000_123454", false, 50, 400); + + StorageManager::sortFiles(&list); + EXPECT_EQ("200_5000_123454", list[0].mFileName); + EXPECT_EQ("100_2000_123454", list[1].mFileName); + EXPECT_EQ("400_100009_123454_history", list[2].mFileName); + EXPECT_EQ("300_2000_123454_history", list[3].mFileName); +} + } // namespace statsd } // namespace os } // namespace android