From 9481762932f18968c1963cdd000952f66e5a52cf Mon Sep 17 00:00:00 2001 From: Mohammad Samiul Islam Date: Mon, 13 Jan 2020 11:07:43 +0000 Subject: [PATCH 1/2] Add test for verifying all available rollbacks are triggered during native crash Bug: 135977463 Test: atest StagedRollbackTest#testNativeWatchdogTriggersRollbackForAll Change-Id: Iaff10b39207753434c216de5d462211c9826e69c --- .../tests/rollback/StagedRollbackTest.java | 65 +++++++++++++++++-- .../rollback/host/StagedRollbackTest.java | 33 +++++++++- 2 files changed, 91 insertions(+), 7 deletions(-) diff --git a/tests/RollbackTest/RollbackTest/src/com/android/tests/rollback/StagedRollbackTest.java b/tests/RollbackTest/RollbackTest/src/com/android/tests/rollback/StagedRollbackTest.java index 9e490f765eab0..efbd512e76901 100644 --- a/tests/RollbackTest/RollbackTest/src/com/android/tests/rollback/StagedRollbackTest.java +++ b/tests/RollbackTest/RollbackTest/src/com/android/tests/rollback/StagedRollbackTest.java @@ -185,12 +185,6 @@ public class StagedRollbackTest { */ @Test public void testNativeWatchdogTriggersRollback_Phase1() throws Exception { - // When multiple staged sessions are installed on a device which doesn't support checkpoint, - // only the 1st one will prevail. We have to check no other rollbacks available to ensure - // TestApp.A is always the 1st and the only one to commit so rollback can work as intended. - // If there are leftover rollbacks from previous tests, this assertion will fail. - assertThat(RollbackUtils.getRollbackManager().getAvailableRollbacks()).isEmpty(); - Uninstall.packages(TestApp.A); Install.single(TestApp.A1).commit(); assertThat(InstallUtils.getInstalledVersion(TestApp.A)).isEqualTo(1); @@ -220,6 +214,64 @@ public class StagedRollbackTest { TestApp.A)).isNotNull(); } + /** + * Stage install an apk with rollback that will be later triggered by unattributable crash. + */ + @Test + public void testNativeWatchdogTriggersRollbackForAll_Phase1() throws Exception { + Uninstall.packages(TestApp.A); + Install.single(TestApp.A1).commit(); + assertThat(InstallUtils.getInstalledVersion(TestApp.A)).isEqualTo(1); + + Install.single(TestApp.A2).setEnableRollback().setStaged().commit(); + } + + /** + * Verify the rollback is available and then install another package with rollback. + */ + @Test + public void testNativeWatchdogTriggersRollbackForAll_Phase2() throws Exception { + assertThat(InstallUtils.getInstalledVersion(TestApp.A)).isEqualTo(2); + RollbackManager rm = RollbackUtils.getRollbackManager(); + assertThat(getUniqueRollbackInfoForPackage(rm.getAvailableRollbacks(), + TestApp.A)).isNotNull(); + + // Install another package with rollback + Uninstall.packages(TestApp.B); + Install.single(TestApp.B1).commit(); + assertThat(InstallUtils.getInstalledVersion(TestApp.B)).isEqualTo(1); + + Install.single(TestApp.B2).setEnableRollback().setStaged().commit(); + } + + /** + * Verify the rollbacks are available. + */ + @Test + public void testNativeWatchdogTriggersRollbackForAll_Phase3() throws Exception { + assertThat(InstallUtils.getInstalledVersion(TestApp.A)).isEqualTo(2); + assertThat(InstallUtils.getInstalledVersion(TestApp.B)).isEqualTo(2); + RollbackManager rm = RollbackUtils.getRollbackManager(); + assertThat(getUniqueRollbackInfoForPackage(rm.getAvailableRollbacks(), + TestApp.A)).isNotNull(); + assertThat(getUniqueRollbackInfoForPackage(rm.getAvailableRollbacks(), + TestApp.B)).isNotNull(); + } + + /** + * Verify the rollbacks are committed after crashing. + */ + @Test + public void testNativeWatchdogTriggersRollbackForAll_Phase4() throws Exception { + assertThat(InstallUtils.getInstalledVersion(TestApp.A)).isEqualTo(1); + assertThat(InstallUtils.getInstalledVersion(TestApp.B)).isEqualTo(1); + RollbackManager rm = RollbackUtils.getRollbackManager(); + assertThat(getUniqueRollbackInfoForPackage(rm.getRecentlyCommittedRollbacks(), + TestApp.A)).isNotNull(); + assertThat(getUniqueRollbackInfoForPackage(rm.getRecentlyCommittedRollbacks(), + TestApp.B)).isNotNull(); + } + @Test public void testNetworkFailedRollback_Phase1() throws Exception { // Remove available rollbacks and uninstall NetworkStack on /data/ @@ -438,6 +490,7 @@ public class StagedRollbackTest { RollbackManager rm = RollbackUtils.getRollbackManager(); rm.getAvailableRollbacks().stream().flatMap(info -> info.getPackages().stream()) .map(info -> info.getPackageName()).forEach(rm::expireRollbackForPackage); + assertThat(RollbackUtils.getRollbackManager().getAvailableRollbacks()).isEmpty(); } private static void runShellCommand(String cmd) { diff --git a/tests/RollbackTest/StagedRollbackTest/src/com/android/tests/rollback/host/StagedRollbackTest.java b/tests/RollbackTest/StagedRollbackTest/src/com/android/tests/rollback/host/StagedRollbackTest.java index 91577c202df9d..76c45429c7460 100644 --- a/tests/RollbackTest/StagedRollbackTest/src/com/android/tests/rollback/host/StagedRollbackTest.java +++ b/tests/RollbackTest/StagedRollbackTest/src/com/android/tests/rollback/host/StagedRollbackTest.java @@ -51,6 +51,7 @@ public class StagedRollbackTest extends BaseHostJUnit4Test { @Before public void setUp() throws Exception { getDevice().reboot(); + runPhase("testCleanUp"); } @After @@ -75,7 +76,6 @@ public class StagedRollbackTest extends BaseHostJUnit4Test { @Test public void testNativeWatchdogTriggersRollback() throws Exception { - //Stage install ModuleMetadata package - this simulates a Mainline module update runPhase("testNativeWatchdogTriggersRollback_Phase1"); // Reboot device to activate staged package @@ -101,6 +101,37 @@ public class StagedRollbackTest extends BaseHostJUnit4Test { runPhase("testNativeWatchdogTriggersRollback_Phase3"); } + @Test + public void testNativeWatchdogTriggersRollbackForAll() throws Exception { + // Install a package with rollback enabled. + runPhase("testNativeWatchdogTriggersRollbackForAll_Phase1"); + getDevice().reboot(); + + // Once previous staged install is applied, install another package + runPhase("testNativeWatchdogTriggersRollbackForAll_Phase2"); + getDevice().reboot(); + + // Verify the new staged install has also been applied successfully. + runPhase("testNativeWatchdogTriggersRollbackForAll_Phase3"); + + // crash system_server enough times to trigger a rollback + crashProcess("system_server", NATIVE_CRASHES_THRESHOLD); + + // Rollback should be committed automatically now. + // Give time for rollback to be committed. This could take a while, + // because we need all of the following to happen: + // 1. system_server comes back up and boot completes. + // 2. Rollback health observer detects updatable crashing signal. + // 3. Staged rollback session becomes ready. + // 4. Device actually reboots. + // So we give a generous timeout here. + assertTrue(getDevice().waitForDeviceNotAvailable(TimeUnit.MINUTES.toMillis(5))); + getDevice().waitForDeviceAvailable(); + + // verify all available rollbacks have been committed + runPhase("testNativeWatchdogTriggersRollbackForAll_Phase4"); + } + /** * Tests failed network health check triggers watchdog staged rollbacks. */ From 2f9fa5c9dad373aea442f585c5f38bdab67a3aaa Mon Sep 17 00:00:00 2001 From: Mohammad Samiul Islam Date: Mon, 13 Jan 2020 12:13:48 +0000 Subject: [PATCH 2/2] Allow handling of all pending staged rollback sessions before rebooting When there is an unattributable native service crash during boot, we rollback all packages available for rollback. In the current implementation, we reboot immediately when a staged rollback session becomes ready, even though there can be other staged rollback sessions which are still being handled. In this CL, the logic is changed to allow the health observer wait for all staged sessions to be handled before rebooting. Bug: 141843321 Test: atest StagedRollbackTest#testNativeWatchdogTriggersRollbackForAll Test: atest StagedRollbackTest Change-Id: Ib2aaee22868e5502b68847aa7f64e428070f7e3c --- .../RollbackPackageHealthObserver.java | 29 ++++++++++++++++++- .../tests/rollback/StagedRollbackTest.java | 9 ++++++ .../rollback/host/StagedRollbackTest.java | 13 +++++++++ 3 files changed, 50 insertions(+), 1 deletion(-) diff --git a/services/core/java/com/android/server/rollback/RollbackPackageHealthObserver.java b/services/core/java/com/android/server/rollback/RollbackPackageHealthObserver.java index a0ef8cfec80f2..b9ef7b39c4517 100644 --- a/services/core/java/com/android/server/rollback/RollbackPackageHealthObserver.java +++ b/services/core/java/com/android/server/rollback/RollbackPackageHealthObserver.java @@ -280,7 +280,6 @@ public final class RollbackPackageHealthObserver implements PackageHealthObserve .WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_TYPE__ROLLBACK_BOOT_TRIGGERED, WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_REASON__REASON_UNKNOWN, ""); - mContext.getSystemService(PowerManager.class).reboot("Rollback staged install"); } else if (sessionInfo.isStagedSessionFailed() && markStagedSessionHandled(rollbackId)) { logEvent(moduleMetadataPackage, @@ -291,6 +290,11 @@ public final class RollbackPackageHealthObserver implements PackageHealthObserve } } } + + // Wait for all pending staged sessions to get handled before rebooting. + if (isPendingStagedSessionsEmpty()) { + mContext.getSystemService(PowerManager.class).reboot("Rollback staged install"); + } } /** @@ -303,6 +307,16 @@ public final class RollbackPackageHealthObserver implements PackageHealthObserve } } + /** + * Returns {@code true} if all pending staged rollback sessions were marked as handled, + * {@code false} if there is any left. + */ + private boolean isPendingStagedSessionsEmpty() { + synchronized (mPendingStagedRollbackIds) { + return mPendingStagedRollbackIds.isEmpty(); + } + } + private void saveLastStagedRollbackId(int stagedRollbackId) { try { FileOutputStream fos = new FileOutputStream(mLastStagedRollbackIdFile); @@ -414,6 +428,9 @@ public final class RollbackPackageHealthObserver implements PackageHealthObserve reasonToLog, failedPackageToLog); } } else { + if (rollback.isStaged()) { + markStagedSessionHandled(rollback.getRollbackId()); + } logEvent(logPackage, StatsLog.WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_TYPE__ROLLBACK_FAILURE, reasonToLog, failedPackageToLog); @@ -431,6 +448,16 @@ public final class RollbackPackageHealthObserver implements PackageHealthObserve RollbackManager rollbackManager = mContext.getSystemService(RollbackManager.class); List rollbacks = rollbackManager.getAvailableRollbacks(); + // Add all rollback ids to mPendingStagedRollbackIds, so that we do not reboot before all + // pending staged rollbacks are handled. + synchronized (mPendingStagedRollbackIds) { + for (RollbackInfo rollback : rollbacks) { + if (rollback.isStaged()) { + mPendingStagedRollbackIds.add(rollback.getRollbackId()); + } + } + } + for (RollbackInfo rollback : rollbacks) { VersionedPackage sample = rollback.getPackages().get(0).getVersionRolledBackFrom(); rollbackPackage(rollback, sample, PackageWatchdog.FAILURE_REASON_NATIVE_CRASH); diff --git a/tests/RollbackTest/RollbackTest/src/com/android/tests/rollback/StagedRollbackTest.java b/tests/RollbackTest/RollbackTest/src/com/android/tests/rollback/StagedRollbackTest.java index efbd512e76901..7fb9f348e7c2f 100644 --- a/tests/RollbackTest/RollbackTest/src/com/android/tests/rollback/StagedRollbackTest.java +++ b/tests/RollbackTest/RollbackTest/src/com/android/tests/rollback/StagedRollbackTest.java @@ -23,12 +23,14 @@ import static com.google.common.truth.Truth.assertThat; import android.Manifest; import android.content.ComponentName; +import android.content.Context; import android.content.Intent; import android.content.pm.PackageInstaller; import android.content.pm.PackageManager; import android.content.rollback.RollbackInfo; import android.content.rollback.RollbackManager; import android.os.ParcelFileDescriptor; +import android.os.storage.StorageManager; import android.provider.DeviceConfig; import androidx.test.platform.app.InstrumentationRegistry; @@ -498,4 +500,11 @@ public class StagedRollbackTest { .executeShellCommand(cmd); IoUtils.closeQuietly(pfd); } + + @Test + public void isCheckpointSupported() { + Context context = InstrumentationRegistry.getInstrumentation().getContext(); + StorageManager sm = (StorageManager) context.getSystemService(Context.STORAGE_SERVICE); + assertThat(sm.isCheckpointSupported()).isTrue(); + } } diff --git a/tests/RollbackTest/StagedRollbackTest/src/com/android/tests/rollback/host/StagedRollbackTest.java b/tests/RollbackTest/StagedRollbackTest/src/com/android/tests/rollback/host/StagedRollbackTest.java index 76c45429c7460..76d7a44d33aba 100644 --- a/tests/RollbackTest/StagedRollbackTest/src/com/android/tests/rollback/host/StagedRollbackTest.java +++ b/tests/RollbackTest/StagedRollbackTest/src/com/android/tests/rollback/host/StagedRollbackTest.java @@ -17,6 +17,7 @@ package com.android.tests.rollback.host; import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeTrue; import static org.testng.Assert.assertThrows; import com.android.tradefed.testtype.DeviceJUnit4ClassRunner; @@ -103,6 +104,9 @@ public class StagedRollbackTest extends BaseHostJUnit4Test { @Test public void testNativeWatchdogTriggersRollbackForAll() throws Exception { + // This test requires committing multiple staged rollbacks + assumeTrue(isCheckpointSupported()); + // Install a package with rollback enabled. runPhase("testNativeWatchdogTriggersRollbackForAll_Phase1"); getDevice().reboot(); @@ -233,4 +237,13 @@ public class StagedRollbackTest extends BaseHostJUnit4Test { // Find the NetworkStack path (can be NetworkStack.apk or NetworkStackNext.apk) return getDevice().executeShellCommand("ls /system/priv-app/NetworkStack*/*.apk"); } + + private boolean isCheckpointSupported() throws Exception { + try { + runPhase("isCheckpointSupported"); + return true; + } catch (AssertionError ignore) { + return false; + } + } }