From b070225f75f48211a0e80181e66280fe0ba074d8 Mon Sep 17 00:00:00 2001 From: JW Wang Date: Thu, 30 Sep 2021 14:53:13 +0800 Subject: [PATCH] Implement two-phase rollback (4/n) Reduce the chance of rolling back Mainline udpates which might contain critical security fixes. It works as: 1. roll back only rebootless apexes if they are available 2. if native crash stops, we are good again 3. if not, roll back all remaining rollbacks which is the best we can do Bug: 195517333 Test: atest StagedRollbackTest Change-Id: If938d7f9f17957ab30d71d2cc87a148c07dd3da7 --- .../rollback/RollbackManagerServiceImpl.java | 1 + .../RollbackPackageHealthObserver.java | 91 ++++++++++++++++++- .../tests/rollback/StagedRollbackTest.java | 25 ++++- .../rollback/host/StagedRollbackTest.java | 26 ++++++ 4 files changed, 141 insertions(+), 2 deletions(-) diff --git a/services/core/java/com/android/server/rollback/RollbackManagerServiceImpl.java b/services/core/java/com/android/server/rollback/RollbackManagerServiceImpl.java index d0a96b14495ae..02ed0005136fc 100644 --- a/services/core/java/com/android/server/rollback/RollbackManagerServiceImpl.java +++ b/services/core/java/com/android/server/rollback/RollbackManagerServiceImpl.java @@ -1162,6 +1162,7 @@ class RollbackManagerServiceImpl extends IRollbackManager.Stub implements Rollba assertInWorkerThread(); Slog.i(TAG, "makeRollbackAvailable id=" + rollback.info.getRollbackId()); rollback.makeAvailable(); + mPackageHealthObserver.notifyRollbackAvailable(rollback.info); // TODO(zezeozue): Provide API to explicitly start observing instead // of doing this for all rollbacks. If we do this for all rollbacks, diff --git a/services/core/java/com/android/server/rollback/RollbackPackageHealthObserver.java b/services/core/java/com/android/server/rollback/RollbackPackageHealthObserver.java index 1295b7008e67c..1856786e9143e 100644 --- a/services/core/java/com/android/server/rollback/RollbackPackageHealthObserver.java +++ b/services/core/java/com/android/server/rollback/RollbackPackageHealthObserver.java @@ -51,6 +51,7 @@ import com.android.server.pm.ApexManager; import java.io.BufferedReader; import java.io.File; +import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.FileReader; import java.io.IOException; @@ -75,8 +76,11 @@ final class RollbackPackageHealthObserver implements PackageHealthObserver { private final Handler mHandler; private final ApexManager mApexManager; private final File mLastStagedRollbackIdsFile; + private final File mTwoPhaseRollbackEnabledFile; // Staged rollback ids that have been committed but their session is not yet ready private final Set mPendingStagedRollbackIds = new ArraySet<>(); + // True if needing to roll back only rebootless apexes when native crash happens + private boolean mTwoPhaseRollbackEnabled; RollbackPackageHealthObserver(Context context) { mContext = context; @@ -86,8 +90,19 @@ final class RollbackPackageHealthObserver implements PackageHealthObserver { File dataDir = new File(Environment.getDataDirectory(), "rollback-observer"); dataDir.mkdirs(); mLastStagedRollbackIdsFile = new File(dataDir, "last-staged-rollback-ids"); + mTwoPhaseRollbackEnabledFile = new File(dataDir, "two-phase-rollback-enabled"); PackageWatchdog.getInstance(mContext).registerHealthObserver(this); mApexManager = ApexManager.getInstance(); + + if (SystemProperties.getBoolean("sys.boot_completed", false)) { + // Load the value from the file if system server has crashed and restarted + mTwoPhaseRollbackEnabled = readBoolean(mTwoPhaseRollbackEnabledFile); + } else { + // Disable two-phase rollback for a normal reboot. We assume the rebootless apex + // installed before reboot is stable if native crash didn't happen. + mTwoPhaseRollbackEnabled = false; + writeBoolean(mTwoPhaseRollbackEnabledFile, false); + } } @Override @@ -144,6 +159,31 @@ final class RollbackPackageHealthObserver implements PackageHealthObserver { PackageWatchdog.getInstance(mContext).startObservingHealth(this, packages, durationMs); } + @AnyThread + void notifyRollbackAvailable(RollbackInfo rollback) { + mHandler.post(() -> { + // Enable two-phase rollback when a rebootless apex rollback is made available. + // We assume the rebootless apex is stable and is less likely to be the cause + // if native crash doesn't happen before reboot. So we will clear the flag and disable + // two-phase rollback after reboot. + if (isRebootlessApex(rollback)) { + mTwoPhaseRollbackEnabled = true; + writeBoolean(mTwoPhaseRollbackEnabledFile, true); + } + }); + } + + private static boolean isRebootlessApex(RollbackInfo rollback) { + if (!rollback.isStaged()) { + for (PackageRollbackInfo info : rollback.getPackages()) { + if (info.isApex()) { + return true; + } + } + } + return false; + } + /** Verifies the rollback state after a reboot and schedules polling for sometime after reboot * to check for native crashes and mitigate them if needed. */ @@ -155,6 +195,7 @@ final class RollbackPackageHealthObserver implements PackageHealthObserver { @WorkerThread private void onBootCompleted() { assertInWorkerThread(); + RollbackManager rollbackManager = mContext.getSystemService(RollbackManager.class); if (!rollbackManager.getAvailableRollbacks().isEmpty()) { // TODO(gavincorkery): Call into Package Watchdog from outside the observer @@ -277,6 +318,23 @@ final class RollbackPackageHealthObserver implements PackageHealthObserver { return mPendingStagedRollbackIds.isEmpty(); } + private static boolean readBoolean(File file) { + try (FileInputStream fis = new FileInputStream(file)) { + return fis.read() == 1; + } catch (IOException ignore) { + return false; + } + } + + private static void writeBoolean(File file, boolean value) { + try (FileOutputStream fos = new FileOutputStream(file)) { + fos.write(value ? 1 : 0); + fos.flush(); + FileUtils.sync(fos); + } catch (IOException ignore) { + } + } + @WorkerThread private void saveStagedRollbackId(int stagedRollbackId, @Nullable VersionedPackage logPackage) { assertInWorkerThread(); @@ -420,13 +478,44 @@ final class RollbackPackageHealthObserver implements PackageHealthObserver { Collections.singletonList(failedPackage), rollbackReceiver.getIntentSender()); } + /** + * Two-phase rollback: + * 1. roll back rebootless apexes first + * 2. roll back all remaining rollbacks if native crash doesn't stop after (1) is done + * + * This approach gives us a better chance to correctly attribute native crash to rebootless + * apex update without rolling back Mainline updates which might contains critical security + * fixes. + */ + @WorkerThread + private boolean useTwoPhaseRollback(List rollbacks) { + assertInWorkerThread(); + if (!mTwoPhaseRollbackEnabled) { + return false; + } + + Slog.i(TAG, "Rolling back all rebootless APEX rollbacks"); + boolean found = false; + for (RollbackInfo rollback : rollbacks) { + if (isRebootlessApex(rollback)) { + VersionedPackage sample = rollback.getPackages().get(0).getVersionRolledBackFrom(); + rollbackPackage(rollback, sample, PackageWatchdog.FAILURE_REASON_NATIVE_CRASH); + found = true; + } + } + return found; + } + @WorkerThread private void rollbackAll() { assertInWorkerThread(); - Slog.i(TAG, "Rolling back all available rollbacks"); RollbackManager rollbackManager = mContext.getSystemService(RollbackManager.class); List rollbacks = rollbackManager.getAvailableRollbacks(); + if (useTwoPhaseRollback(rollbacks)) { + return; + } + Slog.i(TAG, "Rolling back all available rollbacks"); // Add all rollback ids to mPendingStagedRollbackIds, so that we do not reboot before all // pending staged rollbacks are handled. for (RollbackInfo rollback : rollbacks) { diff --git a/tests/RollbackTest/RollbackTest/src/com/android/tests/rollback/StagedRollbackTest.java b/tests/RollbackTest/RollbackTest/src/com/android/tests/rollback/StagedRollbackTest.java index 4512876a8b00c..ffc8f47ecce96 100644 --- a/tests/RollbackTest/RollbackTest/src/com/android/tests/rollback/StagedRollbackTest.java +++ b/tests/RollbackTest/RollbackTest/src/com/android/tests/rollback/StagedRollbackTest.java @@ -58,6 +58,7 @@ import java.util.concurrent.TimeUnit; public class StagedRollbackTest { private static final String PROPERTY_WATCHDOG_TRIGGER_FAILURE_COUNT = "watchdog_trigger_failure_count"; + private static final String REBOOTLESS_APEX_NAME = "test.apex.rebootless"; /** * Adopts common shell permissions needed for rollback tests. @@ -242,7 +243,7 @@ public class StagedRollbackTest { @Test public void testRollbackRebootlessApex() throws Exception { - final String packageName = "test.apex.rebootless"; + final String packageName = REBOOTLESS_APEX_NAME; assertThat(InstallUtils.getInstalledVersion(packageName)).isEqualTo(1); // install @@ -267,6 +268,28 @@ public class StagedRollbackTest { assertThat(InstallUtils.getInstalledVersion(packageName)).isEqualTo(1); } + @Test + public void testNativeWatchdogTriggersRebootlessApexRollback_Phase1_Install() throws Exception { + assertThat(InstallUtils.getInstalledVersion(REBOOTLESS_APEX_NAME)).isEqualTo(1); + + TestApp apex2 = new TestApp("TestRebootlessApexV2", REBOOTLESS_APEX_NAME, 2, + /* isApex= */ true, "test.rebootless_apex_v2.apex"); + Install.single(apex2).setEnableRollback(PackageManager.ROLLBACK_DATA_POLICY_RETAIN) + .commit(); + Install.single(TestApp.A1).commit(); + Install.single(TestApp.A2).setEnableRollback().commit(); + + RollbackUtils.waitForAvailableRollback(TestApp.A); + RollbackUtils.waitForAvailableRollback(REBOOTLESS_APEX_NAME); + } + + @Test + public void testNativeWatchdogTriggersRebootlessApexRollback_Phase2_Verify() throws Exception { + // Check only rebootless apex is rolled back. Other rollbacks should remain unchanged. + assertThat(RollbackUtils.getCommittedRollback(REBOOTLESS_APEX_NAME)).isNotNull(); + assertThat(RollbackUtils.getAvailableRollback(TestApp.A)).isNotNull(); + } + @Test public void hasMainlineModule() throws Exception { String pkgName = getModuleMetadataPackageName(); diff --git a/tests/RollbackTest/StagedRollbackTest/src/com/android/tests/rollback/host/StagedRollbackTest.java b/tests/RollbackTest/StagedRollbackTest/src/com/android/tests/rollback/host/StagedRollbackTest.java index 3576a78048270..1ab59a8c643d1 100644 --- a/tests/RollbackTest/StagedRollbackTest/src/com/android/tests/rollback/host/StagedRollbackTest.java +++ b/tests/RollbackTest/StagedRollbackTest/src/com/android/tests/rollback/host/StagedRollbackTest.java @@ -191,6 +191,18 @@ public class StagedRollbackTest extends BaseHostJUnit4Test { runPhase("testRollbackRebootlessApex"); } + /** + * Tests only rebootless apex (if any) is rolled back when native crash happens + */ + @Test + public void testNativeWatchdogTriggersRebootlessApexRollback() throws Exception { + pushTestApex("test.rebootless_apex_v1.apex"); + runPhase("testNativeWatchdogTriggersRebootlessApexRollback_Phase1_Install"); + crashProcess("system_server", NATIVE_CRASHES_THRESHOLD); + getDevice().waitForDeviceAvailable(); + runPhase("testNativeWatchdogTriggersRebootlessApexRollback_Phase2_Verify"); + } + /** * Tests that packages are monitored across multiple reboots. */ @@ -253,4 +265,18 @@ public class StagedRollbackTest extends BaseHostJUnit4Test { return false; } } + + private void crashProcess(String processName, int numberOfCrashes) throws Exception { + String pid = ""; + String lastPid = "invalid"; + for (int i = 0; i < numberOfCrashes; ++i) { + // This condition makes sure before we kill the process, the process is running AND + // the last crash was finished. + while ("".equals(pid) || lastPid.equals(pid)) { + pid = getDevice().executeShellCommand("pidof " + processName); + } + getDevice().executeShellCommand("kill " + pid); + lastPid = pid; + } + } }