From cde1c58d01c6e98c813e8b29ee0781d898bf2bdf Mon Sep 17 00:00:00 2001 From: Alexei Nicoara Date: Tue, 21 Feb 2023 15:04:12 +0000 Subject: [PATCH] Adding reboot throttling For crashes during boot, Rescue Party will start rebooting and offering factory resets as soon it detects those crashes. If the user continuously rejects the factory reset prompts then the device will keep rebooting and will never finish boot. There are some cases when the device could have been saved if it would have finished the boot (by server-side mainline rollbacks, GMS auto-updates, flag changes). Adding throttling mechanism for reboots and factory resets. Rescue Party won't be allowed to do new reboots and factory resets 10 minutes after a factory reset was offered. Test: atest RescuePartyTest Bug: 264885897 Change-Id: I2dae2801b46977d25609a012270351ab0335531c (cherry picked from https://googleplex-android-review.googlesource.com/q/commit:70e8cc604da81bfb10a16dc1b1d0e3f55ed307bb) Merged-In: I2dae2801b46977d25609a012270351ab0335531c --- .../java/com/android/server/RescueParty.java | 42 ++++++++++----- .../com/android/server/RescuePartyTest.java | 51 +++++++++++++++++++ 2 files changed, 81 insertions(+), 12 deletions(-) diff --git a/services/core/java/com/android/server/RescueParty.java b/services/core/java/com/android/server/RescueParty.java index d1e0f16bd0ae4..3de65f94decf2 100644 --- a/services/core/java/com/android/server/RescueParty.java +++ b/services/core/java/com/android/server/RescueParty.java @@ -79,6 +79,7 @@ public class RescueParty { static final String PROP_ATTEMPTING_FACTORY_RESET = "sys.attempting_factory_reset"; static final String PROP_ATTEMPTING_REBOOT = "sys.attempting_reboot"; static final String PROP_MAX_RESCUE_LEVEL_ATTEMPTED = "sys.max_rescue_level_attempted"; + static final String PROP_LAST_FACTORY_RESET_TIME_MS = "persist.sys.last_factory_reset"; @VisibleForTesting static final int LEVEL_NONE = 0; @VisibleForTesting @@ -105,10 +106,11 @@ public class RescueParty { @VisibleForTesting static final String NAMESPACE_TO_PACKAGE_MAPPING_FLAG = "namespace_to_package_mapping"; + @VisibleForTesting + static final long FACTORY_RESET_THROTTLE_DURATION_MS = TimeUnit.MINUTES.toMillis(10); private static final String NAME = "rescue-party-observer"; - private static final String PROP_DISABLE_RESCUE = "persist.sys.disable_rescue"; private static final String PROP_VIRTUAL_DEVICE = "ro.hardware.virtual_device"; private static final String PROP_DEVICE_CONFIG_DISABLE_FLAG = @@ -327,8 +329,8 @@ public class RescueParty { } } - private static int getMaxRescueLevel(boolean mayPerformFactoryReset) { - if (!mayPerformFactoryReset + private static int getMaxRescueLevel(boolean mayPerformReboot) { + if (!mayPerformReboot || SystemProperties.getBoolean(PROP_DISABLE_FACTORY_RESET_FLAG, false)) { return LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS; } @@ -339,11 +341,11 @@ public class RescueParty { * Get the rescue level to perform if this is the n-th attempt at mitigating failure. * * @param mitigationCount: the mitigation attempt number (1 = first attempt etc.) - * @param mayPerformFactoryReset: whether or not a factory reset may be performed for the given - * failure. + * @param mayPerformReboot: whether or not a reboot and factory reset may be performed + * for the given failure. * @return the rescue level for the n-th mitigation attempt. */ - private static int getRescueLevel(int mitigationCount, boolean mayPerformFactoryReset) { + private static int getRescueLevel(int mitigationCount, boolean mayPerformReboot) { if (mitigationCount == 1) { return LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS; } else if (mitigationCount == 2) { @@ -351,9 +353,9 @@ public class RescueParty { } else if (mitigationCount == 3) { return LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS; } else if (mitigationCount == 4) { - return Math.min(getMaxRescueLevel(mayPerformFactoryReset), LEVEL_WARM_REBOOT); + return Math.min(getMaxRescueLevel(mayPerformReboot), LEVEL_WARM_REBOOT); } else if (mitigationCount >= 5) { - return Math.min(getMaxRescueLevel(mayPerformFactoryReset), LEVEL_FACTORY_RESET); + return Math.min(getMaxRescueLevel(mayPerformReboot), LEVEL_FACTORY_RESET); } else { Slog.w(TAG, "Expected positive mitigation count, was " + mitigationCount); return LEVEL_NONE; @@ -450,6 +452,8 @@ public class RescueParty { break; } SystemProperties.set(PROP_ATTEMPTING_FACTORY_RESET, "true"); + long now = System.currentTimeMillis(); + SystemProperties.set(PROP_LAST_FACTORY_RESET_TIME_MS, Long.toString(now)); runnable = new Runnable() { @Override public void run() { @@ -627,7 +631,7 @@ public class RescueParty { if (!isDisabled() && (failureReason == PackageWatchdog.FAILURE_REASON_APP_CRASH || failureReason == PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING)) { return mapRescueLevelToUserImpact(getRescueLevel(mitigationCount, - mayPerformFactoryReset(failedPackage))); + mayPerformReboot(failedPackage))); } else { return PackageHealthObserverImpact.USER_IMPACT_NONE; } @@ -642,7 +646,7 @@ public class RescueParty { if (failureReason == PackageWatchdog.FAILURE_REASON_APP_CRASH || failureReason == PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING) { final int level = getRescueLevel(mitigationCount, - mayPerformFactoryReset(failedPackage)); + mayPerformReboot(failedPackage)); executeRescueLevel(mContext, failedPackage == null ? null : failedPackage.getPackageName(), level); return true; @@ -683,8 +687,9 @@ public class RescueParty { if (isDisabled()) { return false; } + boolean mayPerformReboot = !shouldThrottleReboot(); executeRescueLevel(mContext, /*failedPackage=*/ null, - getRescueLevel(mitigationCount, true)); + getRescueLevel(mitigationCount, mayPerformReboot)); return true; } @@ -698,14 +703,27 @@ public class RescueParty { * prompting a factory reset is an acceptable mitigation strategy for the package's * failure, {@code false} otherwise. */ - private boolean mayPerformFactoryReset(@Nullable VersionedPackage failingPackage) { + private boolean mayPerformReboot(@Nullable VersionedPackage failingPackage) { if (failingPackage == null) { return false; } + if (shouldThrottleReboot()) { + return false; + } return isPersistentSystemApp(failingPackage.getPackageName()); } + /** + * Returns {@code true} if Rescue Party is allowed to attempt a reboot or factory reset. + * Will return {@code false} if a factory reset was already offered recently. + */ + private boolean shouldThrottleReboot() { + Long lastResetTime = SystemProperties.getLong(PROP_LAST_FACTORY_RESET_TIME_MS, 0); + long now = System.currentTimeMillis(); + return now < lastResetTime + FACTORY_RESET_THROTTLE_DURATION_MS; + } + private boolean isPersistentSystemApp(@NonNull String packageName) { PackageManager pm = mContext.getPackageManager(); try { diff --git a/services/tests/mockingservicestests/src/com/android/server/RescuePartyTest.java b/services/tests/mockingservicestests/src/com/android/server/RescuePartyTest.java index 83441bfb8d1e0..1a7517098d18f 100644 --- a/services/tests/mockingservicestests/src/com/android/server/RescuePartyTest.java +++ b/services/tests/mockingservicestests/src/com/android/server/RescuePartyTest.java @@ -68,6 +68,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.concurrent.Executor; +import java.util.concurrent.TimeUnit; /** * Test RescueParty. @@ -94,6 +95,9 @@ public class RescuePartyTest { "persist.device_config.configuration.disable_rescue_party"; private static final String PROP_DISABLE_FACTORY_RESET_FLAG = "persist.device_config.configuration.disable_rescue_party_factory_reset"; + private static final String PROP_LAST_FACTORY_RESET_TIME_MS = "persist.sys.last_factory_reset"; + + private static final int THROTTLING_DURATION_MIN = 10; private MockitoSession mSession; private HashMap mSystemSettingsMap; @@ -458,6 +462,53 @@ public class RescuePartyTest { assertTrue(RescueParty.isFactoryResetPropertySet()); } + @Test + public void testThrottlingOnBootFailures() { + SystemProperties.set(RescueParty.PROP_ATTEMPTING_REBOOT, Boolean.toString(false)); + long now = System.currentTimeMillis(); + long beforeTimeout = now - TimeUnit.MINUTES.toMillis(THROTTLING_DURATION_MIN - 1); + SystemProperties.set(PROP_LAST_FACTORY_RESET_TIME_MS, Long.toString(beforeTimeout)); + for (int i = 1; i <= LEVEL_FACTORY_RESET; i++) { + noteBoot(i); + } + assertFalse(RescueParty.isAttemptingFactoryReset()); + } + + @Test + public void testThrottlingOnAppCrash() { + SystemProperties.set(RescueParty.PROP_ATTEMPTING_REBOOT, Boolean.toString(false)); + long now = System.currentTimeMillis(); + long beforeTimeout = now - TimeUnit.MINUTES.toMillis(THROTTLING_DURATION_MIN - 1); + SystemProperties.set(PROP_LAST_FACTORY_RESET_TIME_MS, Long.toString(beforeTimeout)); + for (int i = 0; i <= LEVEL_FACTORY_RESET; i++) { + noteAppCrash(i + 1, true); + } + assertFalse(RescueParty.isAttemptingFactoryReset()); + } + + @Test + public void testNotThrottlingAfterTimeoutOnBootFailures() { + SystemProperties.set(RescueParty.PROP_ATTEMPTING_REBOOT, Boolean.toString(false)); + long now = System.currentTimeMillis(); + long afterTimeout = now - TimeUnit.MINUTES.toMillis(THROTTLING_DURATION_MIN + 1); + SystemProperties.set(PROP_LAST_FACTORY_RESET_TIME_MS, Long.toString(afterTimeout)); + for (int i = 1; i <= LEVEL_FACTORY_RESET; i++) { + noteBoot(i); + } + assertTrue(RescueParty.isAttemptingFactoryReset()); + } + @Test + public void testNotThrottlingAfterTimeoutOnAppCrash() { + SystemProperties.set(RescueParty.PROP_ATTEMPTING_REBOOT, Boolean.toString(false)); + long now = System.currentTimeMillis(); + long afterTimeout = now - TimeUnit.MINUTES.toMillis(THROTTLING_DURATION_MIN + 1); + SystemProperties.set(PROP_LAST_FACTORY_RESET_TIME_MS, Long.toString(afterTimeout)); + for (int i = 0; i <= LEVEL_FACTORY_RESET; i++) { + noteAppCrash(i + 1, true); + } + assertTrue(RescueParty.isAttemptingFactoryReset()); + } + @Test public void testNativeRescuePartyResets() { doReturn(true).when(() -> SettingsToPropertiesMapper.isNativeFlagsResetPerformed());