From dd1dabaef7dfae54b20f225ee407f182e86411ac Mon Sep 17 00:00:00 2001 From: Gavin Corkery Date: Wed, 27 Nov 2019 19:11:13 +0000 Subject: [PATCH] Log watchdog-initiated rollback reason To help with monitoring Mainline releases, log the reason for a watchdog-initiated rollback. This may be due to native crashes, app crashes, ANRs or explicit health check failures. Add a mapping from PackageWatchdog failure reason to the new metrics. Bug: 138782888 Test: atest PackageWatchdogTest Test: atest StatsdHostTestCases Change-Id: Ia3e73d955508297004591eac762555665c557b8a --- .../statsd/aidl/android/os/IStatsManager.aidl | 2 +- cmds/statsd/src/StatsService.cpp | 8 ++- cmds/statsd/src/StatsService.h | 4 +- cmds/statsd/src/atoms.proto | 13 +++++ core/java/android/util/StatsLog.java | 6 ++- .../RollbackPackageHealthObserver.java | 52 +++++++++++++++---- 6 files changed, 70 insertions(+), 15 deletions(-) diff --git a/apex/statsd/aidl/android/os/IStatsManager.aidl b/apex/statsd/aidl/android/os/IStatsManager.aidl index 5ebb9f2e4e90f..cc62f07a37500 100644 --- a/apex/statsd/aidl/android/os/IStatsManager.aidl +++ b/apex/statsd/aidl/android/os/IStatsManager.aidl @@ -240,7 +240,7 @@ interface IStatsManager { * Logs an event for watchdog rollbacks. */ oneway void sendWatchdogRollbackOccurredAtom(in int rollbackType, in String packageName, - in long packageVersionCode); + in long packageVersionCode, in int rollbackReason, in String failingPackageName); /** * Returns the most recently registered experiment IDs. diff --git a/cmds/statsd/src/StatsService.cpp b/cmds/statsd/src/StatsService.cpp index d10a661c6d832..3c5ad42311332 100644 --- a/cmds/statsd/src/StatsService.cpp +++ b/cmds/statsd/src/StatsService.cpp @@ -1417,7 +1417,10 @@ Status StatsService::sendBinaryPushStateChangedAtom(const android::String16& tra Status StatsService::sendWatchdogRollbackOccurredAtom(const int32_t rollbackTypeIn, const android::String16& packageNameIn, - const int64_t packageVersionCodeIn) { + const int64_t packageVersionCodeIn, + const int32_t rollbackReasonIn, + const android::String16& + failingPackageNameIn) { // Note: We skip the usage stats op check here since we do not have a package name. // This is ok since we are overloading the usage_stats permission. // This method only sends data, it does not receive it. @@ -1439,7 +1442,8 @@ Status StatsService::sendWatchdogRollbackOccurredAtom(const int32_t rollbackType } android::util::stats_write(android::util::WATCHDOG_ROLLBACK_OCCURRED, - rollbackTypeIn, String8(packageNameIn).string(), packageVersionCodeIn); + rollbackTypeIn, String8(packageNameIn).string(), packageVersionCodeIn, + rollbackReasonIn, String8(failingPackageNameIn).string()); // Fast return to save disk read. if (rollbackTypeIn != android::util::WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_TYPE__ROLLBACK_SUCCESS diff --git a/cmds/statsd/src/StatsService.h b/cmds/statsd/src/StatsService.h index 8c98e7b969364..50b1014f4e8a1 100644 --- a/cmds/statsd/src/StatsService.h +++ b/cmds/statsd/src/StatsService.h @@ -214,7 +214,9 @@ public: virtual Status sendWatchdogRollbackOccurredAtom( const int32_t rollbackTypeIn, const android::String16& packageNameIn, - const int64_t packageVersionCodeIn) override; + const int64_t packageVersionCodeIn, + const int32_t rollbackReasonIn, + const android::String16& failingPackageNameIn) override; /** * Binder call to get registered experiment IDs. diff --git a/cmds/statsd/src/atoms.proto b/cmds/statsd/src/atoms.proto index 2efb789438121..2d7bf593fda23 100644 --- a/cmds/statsd/src/atoms.proto +++ b/cmds/statsd/src/atoms.proto @@ -1742,6 +1742,19 @@ message WatchdogRollbackOccurred { optional string package_name = 2; optional int32 package_version_code = 3; + + enum RollbackReasonType { + REASON_UNKNOWN = 0; + REASON_NATIVE_CRASH = 1; + REASON_EXPLICIT_HEALTH_CHECK = 2; + REASON_APP_CRASH = 3; + REASON_APP_NOT_RESPONDING = 4; + } + optional RollbackReasonType rollback_reason = 4; + + // Set by RollbackPackageHealthObserver to be the package that is failing when a rollback + // is initiated. Empty if the package is unknown. + optional string failing_package_name = 5; } /** diff --git a/core/java/android/util/StatsLog.java b/core/java/android/util/StatsLog.java index 64e15cfb7948d..8cb5b05df6850 100644 --- a/core/java/android/util/StatsLog.java +++ b/core/java/android/util/StatsLog.java @@ -179,6 +179,8 @@ public final class StatsLog extends StatsLogInternal { * @param rollbackType state of the rollback. * @param packageName package name being rolled back. * @param packageVersionCode version of the package being rolled back. + * @param rollbackReason reason the package is being rolled back. + * @param failingPackageName the package name causing the failure. * * @return True if the log request was sent to statsd. * @@ -186,7 +188,7 @@ public final class StatsLog extends StatsLogInternal { */ @RequiresPermission(allOf = {DUMP, PACKAGE_USAGE_STATS}) public static boolean logWatchdogRollbackOccurred(int rollbackType, String packageName, - long packageVersionCode) { + long packageVersionCode, int rollbackReason, String failingPackageName) { synchronized (sLogLock) { try { IStatsManager service = getIStatsManagerLocked(); @@ -198,7 +200,7 @@ public final class StatsLog extends StatsLogInternal { } service.sendWatchdogRollbackOccurredAtom(rollbackType, packageName, - packageVersionCode); + packageVersionCode, rollbackReason, failingPackageName); return true; } catch (RemoteException e) { sService = null; diff --git a/services/core/java/com/android/server/rollback/RollbackPackageHealthObserver.java b/services/core/java/com/android/server/rollback/RollbackPackageHealthObserver.java index bd29d6a5450ab..a62616623cb58 100644 --- a/services/core/java/com/android/server/rollback/RollbackPackageHealthObserver.java +++ b/services/core/java/com/android/server/rollback/RollbackPackageHealthObserver.java @@ -16,6 +16,13 @@ package com.android.server.rollback; +import static android.util.StatsLog.WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_REASON__REASON_APP_CRASH; +import static android.util.StatsLog.WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_REASON__REASON_APP_NOT_RESPONDING; +import static android.util.StatsLog.WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_REASON__REASON_EXPLICIT_HEALTH_CHECK; +import static android.util.StatsLog.WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_REASON__REASON_NATIVE_CRASH; +import static android.util.StatsLog.WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_REASON__REASON_UNKNOWN; + +import android.annotation.NonNull; import android.annotation.Nullable; import android.content.BroadcastReceiver; import android.content.Context; @@ -111,6 +118,7 @@ public final class RollbackPackageHealthObserver implements PackageHealthObserve RollbackManager rollbackManager = mContext.getSystemService(RollbackManager.class); VersionedPackage moduleMetadataPackage = getModuleMetadataPackage(); RollbackInfo rollback = getAvailableRollback(rollbackManager, failedPackage); + int reasonToLog = mapFailureReasonToMetric(rollbackReason); if (rollback == null) { Slog.w(TAG, "Expected rollback but no valid rollback found for package: [ " @@ -120,7 +128,8 @@ public final class RollbackPackageHealthObserver implements PackageHealthObserve } logEvent(moduleMetadataPackage, - StatsLog.WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_TYPE__ROLLBACK_INITIATE); + StatsLog.WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_TYPE__ROLLBACK_INITIATE, + reasonToLog, failedPackage.getPackageName()); LocalIntentReceiver rollbackReceiver = new LocalIntentReceiver((Intent result) -> { int status = result.getIntExtra(RollbackManager.EXTRA_STATUS, RollbackManager.STATUS_FAILURE); @@ -137,11 +146,13 @@ public final class RollbackPackageHealthObserver implements PackageHealthObserve moduleMetadataPackage); } else { logEvent(moduleMetadataPackage, - StatsLog.WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_TYPE__ROLLBACK_SUCCESS); + StatsLog.WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_TYPE__ROLLBACK_SUCCESS, + reasonToLog, failedPackage.getPackageName()); } } else { logEvent(moduleMetadataPackage, - StatsLog.WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_TYPE__ROLLBACK_FAILURE); + StatsLog.WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_TYPE__ROLLBACK_FAILURE, + reasonToLog, failedPackage.getPackageName()); } }); @@ -220,12 +231,14 @@ public final class RollbackPackageHealthObserver implements PackageHealthObserve } if (sessionInfo.isStagedSessionApplied()) { logEvent(oldModuleMetadataPackage, - StatsLog.WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_TYPE__ROLLBACK_SUCCESS); + StatsLog.WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_TYPE__ROLLBACK_SUCCESS, + WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_REASON__REASON_UNKNOWN, ""); } else if (sessionInfo.isStagedSessionReady()) { // TODO: What do for staged session ready but not applied } else { logEvent(oldModuleMetadataPackage, - StatsLog.WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_TYPE__ROLLBACK_FAILURE); + StatsLog.WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_TYPE__ROLLBACK_FAILURE, + WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_REASON__REASON_UNKNOWN, ""); } } @@ -304,12 +317,16 @@ public final class RollbackPackageHealthObserver implements PackageHealthObserve saveLastStagedRollbackId(rollbackId); logEvent(moduleMetadataPackage, StatsLog - .WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_TYPE__ROLLBACK_BOOT_TRIGGERED); + .WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_TYPE__ROLLBACK_BOOT_TRIGGERED, + WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_REASON__REASON_UNKNOWN, + ""); mContext.getSystemService(PowerManager.class).reboot("Rollback staged install"); } else if (sessionInfo.isStagedSessionFailed() && markStagedSessionHandled(rollbackId)) { logEvent(moduleMetadataPackage, - StatsLog.WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_TYPE__ROLLBACK_FAILURE); + StatsLog.WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_TYPE__ROLLBACK_FAILURE, + WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_REASON__REASON_UNKNOWN, + ""); mContext.unregisterReceiver(listener); } } @@ -356,11 +373,12 @@ public final class RollbackPackageHealthObserver implements PackageHealthObserve return rollbackId; } - private static void logEvent(@Nullable VersionedPackage moduleMetadataPackage, int type) { + private static void logEvent(@Nullable VersionedPackage moduleMetadataPackage, int type, + int rollbackReason, @NonNull String failingPackageName) { Slog.i(TAG, "Watchdog event occurred of type: " + type); if (moduleMetadataPackage != null) { StatsLog.logWatchdogRollbackOccurred(type, moduleMetadataPackage.getPackageName(), - moduleMetadataPackage.getVersionCode()); + moduleMetadataPackage.getVersionCode(), rollbackReason, failingPackageName); } } @@ -393,4 +411,20 @@ public final class RollbackPackageHealthObserver implements PackageHealthObserve + "and mitigate native crashes"); mHandler.post(()->checkAndMitigateNativeCrashes()); } + + private int mapFailureReasonToMetric(@FailureReasons int failureReason) { + switch (failureReason) { + case PackageWatchdog.FAILURE_REASON_NATIVE_CRASH: + return WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_REASON__REASON_NATIVE_CRASH; + case PackageWatchdog.FAILURE_REASON_EXPLICIT_HEALTH_CHECK: + return WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_REASON__REASON_EXPLICIT_HEALTH_CHECK; + case PackageWatchdog.FAILURE_REASON_APP_CRASH: + return WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_REASON__REASON_APP_CRASH; + case PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING: + return WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_REASON__REASON_APP_NOT_RESPONDING; + default: + return WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_REASON__REASON_UNKNOWN; + } + } + }