diff --git a/services/core/java/com/android/server/rollback/RollbackManagerServiceImpl.java b/services/core/java/com/android/server/rollback/RollbackManagerServiceImpl.java index d0a96b14495ae..02ed0005136fc 100644 --- a/services/core/java/com/android/server/rollback/RollbackManagerServiceImpl.java +++ b/services/core/java/com/android/server/rollback/RollbackManagerServiceImpl.java @@ -1162,6 +1162,7 @@ class RollbackManagerServiceImpl extends IRollbackManager.Stub implements Rollba assertInWorkerThread(); Slog.i(TAG, "makeRollbackAvailable id=" + rollback.info.getRollbackId()); rollback.makeAvailable(); + mPackageHealthObserver.notifyRollbackAvailable(rollback.info); // TODO(zezeozue): Provide API to explicitly start observing instead // of doing this for all rollbacks. If we do this for all rollbacks, diff --git a/services/core/java/com/android/server/rollback/RollbackPackageHealthObserver.java b/services/core/java/com/android/server/rollback/RollbackPackageHealthObserver.java index 1295b7008e67c..1856786e9143e 100644 --- a/services/core/java/com/android/server/rollback/RollbackPackageHealthObserver.java +++ b/services/core/java/com/android/server/rollback/RollbackPackageHealthObserver.java @@ -51,6 +51,7 @@ import com.android.server.pm.ApexManager; import java.io.BufferedReader; import java.io.File; +import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.FileReader; import java.io.IOException; @@ -75,8 +76,11 @@ final class RollbackPackageHealthObserver implements PackageHealthObserver { private final Handler mHandler; private final ApexManager mApexManager; private final File mLastStagedRollbackIdsFile; + private final File mTwoPhaseRollbackEnabledFile; // Staged rollback ids that have been committed but their session is not yet ready private final Set mPendingStagedRollbackIds = new ArraySet<>(); + // True if needing to roll back only rebootless apexes when native crash happens + private boolean mTwoPhaseRollbackEnabled; RollbackPackageHealthObserver(Context context) { mContext = context; @@ -86,8 +90,19 @@ final class RollbackPackageHealthObserver implements PackageHealthObserver { File dataDir = new File(Environment.getDataDirectory(), "rollback-observer"); dataDir.mkdirs(); mLastStagedRollbackIdsFile = new File(dataDir, "last-staged-rollback-ids"); + mTwoPhaseRollbackEnabledFile = new File(dataDir, "two-phase-rollback-enabled"); PackageWatchdog.getInstance(mContext).registerHealthObserver(this); mApexManager = ApexManager.getInstance(); + + if (SystemProperties.getBoolean("sys.boot_completed", false)) { + // Load the value from the file if system server has crashed and restarted + mTwoPhaseRollbackEnabled = readBoolean(mTwoPhaseRollbackEnabledFile); + } else { + // Disable two-phase rollback for a normal reboot. We assume the rebootless apex + // installed before reboot is stable if native crash didn't happen. + mTwoPhaseRollbackEnabled = false; + writeBoolean(mTwoPhaseRollbackEnabledFile, false); + } } @Override @@ -144,6 +159,31 @@ final class RollbackPackageHealthObserver implements PackageHealthObserver { PackageWatchdog.getInstance(mContext).startObservingHealth(this, packages, durationMs); } + @AnyThread + void notifyRollbackAvailable(RollbackInfo rollback) { + mHandler.post(() -> { + // Enable two-phase rollback when a rebootless apex rollback is made available. + // We assume the rebootless apex is stable and is less likely to be the cause + // if native crash doesn't happen before reboot. So we will clear the flag and disable + // two-phase rollback after reboot. + if (isRebootlessApex(rollback)) { + mTwoPhaseRollbackEnabled = true; + writeBoolean(mTwoPhaseRollbackEnabledFile, true); + } + }); + } + + private static boolean isRebootlessApex(RollbackInfo rollback) { + if (!rollback.isStaged()) { + for (PackageRollbackInfo info : rollback.getPackages()) { + if (info.isApex()) { + return true; + } + } + } + return false; + } + /** Verifies the rollback state after a reboot and schedules polling for sometime after reboot * to check for native crashes and mitigate them if needed. */ @@ -155,6 +195,7 @@ final class RollbackPackageHealthObserver implements PackageHealthObserver { @WorkerThread private void onBootCompleted() { assertInWorkerThread(); + RollbackManager rollbackManager = mContext.getSystemService(RollbackManager.class); if (!rollbackManager.getAvailableRollbacks().isEmpty()) { // TODO(gavincorkery): Call into Package Watchdog from outside the observer @@ -277,6 +318,23 @@ final class RollbackPackageHealthObserver implements PackageHealthObserver { return mPendingStagedRollbackIds.isEmpty(); } + private static boolean readBoolean(File file) { + try (FileInputStream fis = new FileInputStream(file)) { + return fis.read() == 1; + } catch (IOException ignore) { + return false; + } + } + + private static void writeBoolean(File file, boolean value) { + try (FileOutputStream fos = new FileOutputStream(file)) { + fos.write(value ? 1 : 0); + fos.flush(); + FileUtils.sync(fos); + } catch (IOException ignore) { + } + } + @WorkerThread private void saveStagedRollbackId(int stagedRollbackId, @Nullable VersionedPackage logPackage) { assertInWorkerThread(); @@ -420,13 +478,44 @@ final class RollbackPackageHealthObserver implements PackageHealthObserver { Collections.singletonList(failedPackage), rollbackReceiver.getIntentSender()); } + /** + * Two-phase rollback: + * 1. roll back rebootless apexes first + * 2. roll back all remaining rollbacks if native crash doesn't stop after (1) is done + * + * This approach gives us a better chance to correctly attribute native crash to rebootless + * apex update without rolling back Mainline updates which might contains critical security + * fixes. + */ + @WorkerThread + private boolean useTwoPhaseRollback(List rollbacks) { + assertInWorkerThread(); + if (!mTwoPhaseRollbackEnabled) { + return false; + } + + Slog.i(TAG, "Rolling back all rebootless APEX rollbacks"); + boolean found = false; + for (RollbackInfo rollback : rollbacks) { + if (isRebootlessApex(rollback)) { + VersionedPackage sample = rollback.getPackages().get(0).getVersionRolledBackFrom(); + rollbackPackage(rollback, sample, PackageWatchdog.FAILURE_REASON_NATIVE_CRASH); + found = true; + } + } + return found; + } + @WorkerThread private void rollbackAll() { assertInWorkerThread(); - Slog.i(TAG, "Rolling back all available rollbacks"); RollbackManager rollbackManager = mContext.getSystemService(RollbackManager.class); List rollbacks = rollbackManager.getAvailableRollbacks(); + if (useTwoPhaseRollback(rollbacks)) { + return; + } + Slog.i(TAG, "Rolling back all available rollbacks"); // Add all rollback ids to mPendingStagedRollbackIds, so that we do not reboot before all // pending staged rollbacks are handled. for (RollbackInfo rollback : rollbacks) { diff --git a/tests/RollbackTest/RollbackTest/src/com/android/tests/rollback/StagedRollbackTest.java b/tests/RollbackTest/RollbackTest/src/com/android/tests/rollback/StagedRollbackTest.java index 4512876a8b00c..ffc8f47ecce96 100644 --- a/tests/RollbackTest/RollbackTest/src/com/android/tests/rollback/StagedRollbackTest.java +++ b/tests/RollbackTest/RollbackTest/src/com/android/tests/rollback/StagedRollbackTest.java @@ -58,6 +58,7 @@ import java.util.concurrent.TimeUnit; public class StagedRollbackTest { private static final String PROPERTY_WATCHDOG_TRIGGER_FAILURE_COUNT = "watchdog_trigger_failure_count"; + private static final String REBOOTLESS_APEX_NAME = "test.apex.rebootless"; /** * Adopts common shell permissions needed for rollback tests. @@ -242,7 +243,7 @@ public class StagedRollbackTest { @Test public void testRollbackRebootlessApex() throws Exception { - final String packageName = "test.apex.rebootless"; + final String packageName = REBOOTLESS_APEX_NAME; assertThat(InstallUtils.getInstalledVersion(packageName)).isEqualTo(1); // install @@ -267,6 +268,28 @@ public class StagedRollbackTest { assertThat(InstallUtils.getInstalledVersion(packageName)).isEqualTo(1); } + @Test + public void testNativeWatchdogTriggersRebootlessApexRollback_Phase1_Install() throws Exception { + assertThat(InstallUtils.getInstalledVersion(REBOOTLESS_APEX_NAME)).isEqualTo(1); + + TestApp apex2 = new TestApp("TestRebootlessApexV2", REBOOTLESS_APEX_NAME, 2, + /* isApex= */ true, "test.rebootless_apex_v2.apex"); + Install.single(apex2).setEnableRollback(PackageManager.ROLLBACK_DATA_POLICY_RETAIN) + .commit(); + Install.single(TestApp.A1).commit(); + Install.single(TestApp.A2).setEnableRollback().commit(); + + RollbackUtils.waitForAvailableRollback(TestApp.A); + RollbackUtils.waitForAvailableRollback(REBOOTLESS_APEX_NAME); + } + + @Test + public void testNativeWatchdogTriggersRebootlessApexRollback_Phase2_Verify() throws Exception { + // Check only rebootless apex is rolled back. Other rollbacks should remain unchanged. + assertThat(RollbackUtils.getCommittedRollback(REBOOTLESS_APEX_NAME)).isNotNull(); + assertThat(RollbackUtils.getAvailableRollback(TestApp.A)).isNotNull(); + } + @Test public void hasMainlineModule() throws Exception { String pkgName = getModuleMetadataPackageName(); diff --git a/tests/RollbackTest/StagedRollbackTest/src/com/android/tests/rollback/host/StagedRollbackTest.java b/tests/RollbackTest/StagedRollbackTest/src/com/android/tests/rollback/host/StagedRollbackTest.java index 3576a78048270..1ab59a8c643d1 100644 --- a/tests/RollbackTest/StagedRollbackTest/src/com/android/tests/rollback/host/StagedRollbackTest.java +++ b/tests/RollbackTest/StagedRollbackTest/src/com/android/tests/rollback/host/StagedRollbackTest.java @@ -191,6 +191,18 @@ public class StagedRollbackTest extends BaseHostJUnit4Test { runPhase("testRollbackRebootlessApex"); } + /** + * Tests only rebootless apex (if any) is rolled back when native crash happens + */ + @Test + public void testNativeWatchdogTriggersRebootlessApexRollback() throws Exception { + pushTestApex("test.rebootless_apex_v1.apex"); + runPhase("testNativeWatchdogTriggersRebootlessApexRollback_Phase1_Install"); + crashProcess("system_server", NATIVE_CRASHES_THRESHOLD); + getDevice().waitForDeviceAvailable(); + runPhase("testNativeWatchdogTriggersRebootlessApexRollback_Phase2_Verify"); + } + /** * Tests that packages are monitored across multiple reboots. */ @@ -253,4 +265,18 @@ public class StagedRollbackTest extends BaseHostJUnit4Test { return false; } } + + private void crashProcess(String processName, int numberOfCrashes) throws Exception { + String pid = ""; + String lastPid = "invalid"; + for (int i = 0; i < numberOfCrashes; ++i) { + // This condition makes sure before we kill the process, the process is running AND + // the last crash was finished. + while ("".equals(pid) || lastPid.equals(pid)) { + pid = getDevice().executeShellCommand("pidof " + processName); + } + getDevice().executeShellCommand("kill " + pid); + lastPid = pid; + } + } }