diff --git a/services/core/java/com/android/server/PackageWatchdog.java b/services/core/java/com/android/server/PackageWatchdog.java index ff83663e8c8f3..f079546096031 100644 --- a/services/core/java/com/android/server/PackageWatchdog.java +++ b/services/core/java/com/android/server/PackageWatchdog.java @@ -68,6 +68,7 @@ import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.NoSuchElementException; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -117,6 +118,9 @@ public class PackageWatchdog { static final int DEFAULT_TRIGGER_FAILURE_COUNT = 5; @VisibleForTesting static final long DEFAULT_OBSERVING_DURATION_MS = TimeUnit.DAYS.toMillis(2); + // Sliding window for tracking how many mitigation calls were made for a package. + @VisibleForTesting + static final long DEFAULT_DEESCALATION_WINDOW_MS = TimeUnit.HOURS.toMillis(1); // Whether explicit health checks are enabled or not private static final boolean DEFAULT_EXPLICIT_HEALTH_CHECK_ENABLED = true; @@ -388,6 +392,7 @@ public class PackageWatchdog { // Observer that will receive failure for versionedPackage PackageHealthObserver currentObserverToNotify = null; int currentObserverImpact = Integer.MAX_VALUE; + MonitoredPackage currentMonitoredPackage = null; // Find observer with least user impact for (int oIndex = 0; oIndex < mAllObservers.size(); oIndex++) { @@ -396,19 +401,33 @@ public class PackageWatchdog { if (registeredObserver != null && observer.onPackageFailureLocked( versionedPackage.getPackageName())) { + MonitoredPackage p = observer.getMonitoredPackage( + versionedPackage.getPackageName()); + int mitigationCount = 1; + if (p != null) { + mitigationCount = p.getMitigationCountLocked() + 1; + } int impact = registeredObserver.onHealthCheckFailed( - versionedPackage, failureReason); + versionedPackage, failureReason, mitigationCount); if (impact != PackageHealthObserverImpact.USER_IMPACT_NONE && impact < currentObserverImpact) { currentObserverToNotify = registeredObserver; currentObserverImpact = impact; + currentMonitoredPackage = p; } } } // Execute action with least user impact if (currentObserverToNotify != null) { - currentObserverToNotify.execute(versionedPackage, failureReason); + int mitigationCount = 1; + if (currentMonitoredPackage != null) { + currentMonitoredPackage.noteMitigationCallLocked(); + mitigationCount = + currentMonitoredPackage.getMitigationCountLocked(); + } + currentObserverToNotify.execute(versionedPackage, + failureReason, mitigationCount); } } } @@ -429,7 +448,7 @@ public class PackageWatchdog { PackageHealthObserver registeredObserver = observer.registeredObserver; if (registeredObserver != null) { int impact = registeredObserver.onHealthCheckFailed( - failingPackage, failureReason); + failingPackage, failureReason, 1); if (impact != PackageHealthObserverImpact.USER_IMPACT_NONE && impact < currentObserverImpact) { currentObserverToNotify = registeredObserver; @@ -438,7 +457,7 @@ public class PackageWatchdog { } } if (currentObserverToNotify != null) { - currentObserverToNotify.execute(failingPackage, failureReason); + currentObserverToNotify.execute(failingPackage, failureReason, 1); } } @@ -559,6 +578,8 @@ public class PackageWatchdog { * @param versionedPackage the package that is failing. This may be null if a native * service is crashing. * @param failureReason the type of failure that is occurring. + * @param mitigationCount the number of times mitigation has been called for this package + * (including this time). * * * @return any one of {@link PackageHealthObserverImpact} to express the impact @@ -566,7 +587,8 @@ public class PackageWatchdog { */ @PackageHealthObserverImpact int onHealthCheckFailed( @Nullable VersionedPackage versionedPackage, - @FailureReasons int failureReason); + @FailureReasons int failureReason, + int mitigationCount); /** * Executes mitigation for {@link #onHealthCheckFailed}. @@ -574,10 +596,12 @@ public class PackageWatchdog { * @param versionedPackage the package that is failing. This may be null if a native * service is crashing. * @param failureReason the type of failure that is occurring. + * @param mitigationCount the number of times mitigation has been called for this package + * (including this time). * @return {@code true} if action was executed successfully, {@code false} otherwise */ boolean execute(@Nullable VersionedPackage versionedPackage, - @FailureReasons int failureReason); + @FailureReasons int failureReason, int mitigationCount); /** @@ -859,7 +883,7 @@ public class PackageWatchdog { VersionedPackage versionedPkg = it.next().mPackage; Slog.i(TAG, "Explicit health check failed for package " + versionedPkg); registeredObserver.execute(versionedPkg, - PackageWatchdog.FAILURE_REASON_EXPLICIT_HEALTH_CHECK); + PackageWatchdog.FAILURE_REASON_EXPLICIT_HEALTH_CHECK, 1); } } } @@ -1293,6 +1317,10 @@ public class PackageWatchdog { // Times when package failures happen sorted in ascending order @GuardedBy("mLock") private final LongArrayQueue mFailureHistory = new LongArrayQueue(); + // Times when an observer was called to mitigate this package's failure. Sorted in + // ascending order. + @GuardedBy("mLock") + private final LongArrayQueue mMitigationCalls = new LongArrayQueue(); // One of STATE_[ACTIVE|INACTIVE|PASSED|FAILED]. Updated on construction and after // methods that could change the health check state: handleElapsedTimeLocked and // tryPassHealthCheckLocked @@ -1357,6 +1385,33 @@ public class PackageWatchdog { return failed; } + /** + * Notes the timestamp of a mitigation call into the observer. + */ + @GuardedBy("mLock") + public void noteMitigationCallLocked() { + mMitigationCalls.addLast(mSystemClock.uptimeMillis()); + } + + /** + * Prunes any mitigation calls outside of the de-escalation window, and returns the + * number of calls that are in the window afterwards. + * + * @return the number of mitigation calls made in the de-escalation window. + */ + @GuardedBy("mLock") + public int getMitigationCountLocked() { + try { + final long now = mSystemClock.uptimeMillis(); + while (now - mMitigationCalls.peekFirst() > DEFAULT_DEESCALATION_WINDOW_MS) { + mMitigationCalls.removeFirst(); + } + } catch (NoSuchElementException ignore) { + } + + return mMitigationCalls.size(); + } + /** * Sets the initial health check duration. * diff --git a/services/core/java/com/android/server/RescueParty.java b/services/core/java/com/android/server/RescueParty.java index ef6dab5a45fbb..e2e4d72217176 100644 --- a/services/core/java/com/android/server/RescueParty.java +++ b/services/core/java/com/android/server/RescueParty.java @@ -452,7 +452,7 @@ public class RescueParty { @Override public int onHealthCheckFailed(@Nullable VersionedPackage failedPackage, - @FailureReasons int failureReason) { + @FailureReasons int failureReason, int mitigationCount) { if (!isDisabled() && (failureReason == PackageWatchdog.FAILURE_REASON_APP_CRASH || failureReason == PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING)) { return mapRescueLevelToUserImpact(getNextRescueLevel()); @@ -463,7 +463,7 @@ public class RescueParty { @Override public boolean execute(@Nullable VersionedPackage failedPackage, - @FailureReasons int failureReason) { + @FailureReasons int failureReason, int mitigationCount) { if (isDisabled()) { return false; } diff --git a/services/core/java/com/android/server/rollback/RollbackPackageHealthObserver.java b/services/core/java/com/android/server/rollback/RollbackPackageHealthObserver.java index ba1401d7469e2..1295b7008e67c 100644 --- a/services/core/java/com/android/server/rollback/RollbackPackageHealthObserver.java +++ b/services/core/java/com/android/server/rollback/RollbackPackageHealthObserver.java @@ -92,7 +92,7 @@ final class RollbackPackageHealthObserver implements PackageHealthObserver { @Override public int onHealthCheckFailed(@Nullable VersionedPackage failedPackage, - @FailureReasons int failureReason) { + @FailureReasons int failureReason, int mitigationCount) { // For native crashes, we will roll back any available rollbacks if (failureReason == PackageWatchdog.FAILURE_REASON_NATIVE_CRASH && !mContext.getSystemService(RollbackManager.class) @@ -110,7 +110,7 @@ final class RollbackPackageHealthObserver implements PackageHealthObserver { @Override public boolean execute(@Nullable VersionedPackage failedPackage, - @FailureReasons int rollbackReason) { + @FailureReasons int rollbackReason, int mitigationCount) { if (rollbackReason == PackageWatchdog.FAILURE_REASON_NATIVE_CRASH) { mHandler.post(() -> rollbackAll()); return true; diff --git a/tests/PackageWatchdog/src/com/android/server/PackageWatchdogTest.java b/tests/PackageWatchdog/src/com/android/server/PackageWatchdogTest.java index ae93a81f274ea..fa0574a503f10 100644 --- a/tests/PackageWatchdog/src/com/android/server/PackageWatchdogTest.java +++ b/tests/PackageWatchdog/src/com/android/server/PackageWatchdogTest.java @@ -376,7 +376,7 @@ public class PackageWatchdogTest { TestObserver observer = new TestObserver(OBSERVER_NAME_1) { @Override public int onHealthCheckFailed(VersionedPackage versionedPackage, - int failureReason) { + int failureReason, int mitigationCount) { if (versionedPackage.getVersionCode() == VERSION_CODE) { // Only rollback for specific versionCode return PackageHealthObserverImpact.USER_IMPACT_MEDIUM; @@ -1146,6 +1146,45 @@ public class PackageWatchdogTest { assertThat(observer.mMitigatedPackages).isEqualTo(List.of(APP_A)); } + /** + * Ensure that the sliding window logic results in the correct mitigation count being sent to + * an observer. + */ + @Test + public void testMitigationSlidingWindow() { + PackageWatchdog watchdog = createWatchdog(); + TestObserver observer = new TestObserver(OBSERVER_NAME_1); + watchdog.startObservingHealth(observer, List.of(APP_A), + PackageWatchdog.DEFAULT_OBSERVING_DURATION_MS * 2); + + + raiseFatalFailureAndDispatch(watchdog, Arrays.asList(new VersionedPackage(APP_A, + VERSION_CODE)), PackageWatchdog.FAILURE_REASON_UNKNOWN); + + moveTimeForwardAndDispatch(TimeUnit.MINUTES.toMillis(10)); + + raiseFatalFailureAndDispatch(watchdog, Arrays.asList(new VersionedPackage(APP_A, + VERSION_CODE)), PackageWatchdog.FAILURE_REASON_UNKNOWN); + raiseFatalFailureAndDispatch(watchdog, Arrays.asList(new VersionedPackage(APP_A, + VERSION_CODE)), PackageWatchdog.FAILURE_REASON_UNKNOWN); + + moveTimeForwardAndDispatch(PackageWatchdog.DEFAULT_DEESCALATION_WINDOW_MS); + + // The first failure will be outside the threshold. + raiseFatalFailureAndDispatch(watchdog, Arrays.asList(new VersionedPackage(APP_A, + VERSION_CODE)), PackageWatchdog.FAILURE_REASON_UNKNOWN); + + moveTimeForwardAndDispatch(TimeUnit.MINUTES.toMillis(20)); + + // The next 2 failures will also be outside the threshold. + raiseFatalFailureAndDispatch(watchdog, Arrays.asList(new VersionedPackage(APP_A, + VERSION_CODE)), PackageWatchdog.FAILURE_REASON_UNKNOWN); + raiseFatalFailureAndDispatch(watchdog, Arrays.asList(new VersionedPackage(APP_A, + VERSION_CODE)), PackageWatchdog.FAILURE_REASON_UNKNOWN); + + assertThat(observer.mMitigationCounts).isEqualTo(List.of(1, 2, 3, 3, 2, 3)); + } + private void adoptShellPermissions(String... permissions) { InstrumentationRegistry .getInstrumentation() @@ -1227,6 +1266,7 @@ public class PackageWatchdogTest { private boolean mMitigatedBootLoop = false; final List mHealthCheckFailedPackages = new ArrayList<>(); final List mMitigatedPackages = new ArrayList<>(); + final List mMitigationCounts = new ArrayList<>(); TestObserver(String name) { mName = name; @@ -1238,13 +1278,16 @@ public class PackageWatchdogTest { mImpact = impact; } - public int onHealthCheckFailed(VersionedPackage versionedPackage, int failureReason) { + public int onHealthCheckFailed(VersionedPackage versionedPackage, int failureReason, + int mitigationCount) { mHealthCheckFailedPackages.add(versionedPackage.getPackageName()); return mImpact; } - public boolean execute(VersionedPackage versionedPackage, int failureReason) { + public boolean execute(VersionedPackage versionedPackage, int failureReason, + int mitigationCount) { mMitigatedPackages.add(versionedPackage.getPackageName()); + mMitigationCounts.add(mitigationCount); mLastFailureReason = failureReason; return true; }