Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge "Implement two-phase rollback (4/n)"

Browse Source
This commit is contained in:
JW Wang
2021-10-19 03:29:59 +00:00
committed by Android (Google) Code Review
parent b0bc55cd83 b070225f75
commit 2ea1d2d121
4 changed files with 141 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
services/core/java/com/android/server/rollback/RollbackManagerServiceImpl.java
View File

@@ -1162,6 +1162,7 @@ class RollbackManagerServiceImpl extends IRollbackManager.Stub implements Rollba
assertInWorkerThread();
Slog.i(TAG, "makeRollbackAvailable id=" + rollback.info.getRollbackId());
rollback.makeAvailable();
mPackageHealthObserver.notifyRollbackAvailable(rollback.info);
// TODO(zezeozue): Provide API to explicitly start observing instead
// of doing this for all rollbacks. If we do this for all rollbacks,

91
services/core/java/com/android/server/rollback/RollbackPackageHealthObserver.java
View File

@@ -51,6 +51,7 @@ import com.android.server.pm.ApexManager;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
@@ -75,8 +76,11 @@ final class RollbackPackageHealthObserver implements PackageHealthObserver {
private final Handler mHandler;
private final ApexManager mApexManager;
private final File mLastStagedRollbackIdsFile;
private final File mTwoPhaseRollbackEnabledFile;
// Staged rollback ids that have been committed but their session is not yet ready
private final Set<Integer> mPendingStagedRollbackIds = new ArraySet<>();
// True if needing to roll back only rebootless apexes when native crash happens
private boolean mTwoPhaseRollbackEnabled;
RollbackPackageHealthObserver(Context context) {
mContext = context;
@@ -86,8 +90,19 @@ final class RollbackPackageHealthObserver implements PackageHealthObserver {
File dataDir = new File(Environment.getDataDirectory(), "rollback-observer");
dataDir.mkdirs();
mLastStagedRollbackIdsFile = new File(dataDir, "last-staged-rollback-ids");
mTwoPhaseRollbackEnabledFile = new File(dataDir, "two-phase-rollback-enabled");
PackageWatchdog.getInstance(mContext).registerHealthObserver(this);
mApexManager = ApexManager.getInstance();
if (SystemProperties.getBoolean("sys.boot_completed", false)) {
// Load the value from the file if system server has crashed and restarted
mTwoPhaseRollbackEnabled = readBoolean(mTwoPhaseRollbackEnabledFile);
} else {
// Disable two-phase rollback for a normal reboot. We assume the rebootless apex
// installed before reboot is stable if native crash didn't happen.
mTwoPhaseRollbackEnabled = false;
writeBoolean(mTwoPhaseRollbackEnabledFile, false);
}
}
@Override
@@ -144,6 +159,31 @@ final class RollbackPackageHealthObserver implements PackageHealthObserver {
PackageWatchdog.getInstance(mContext).startObservingHealth(this, packages, durationMs);
}
@AnyThread
void notifyRollbackAvailable(RollbackInfo rollback) {
mHandler.post(() -> {
// Enable two-phase rollback when a rebootless apex rollback is made available.
// We assume the rebootless apex is stable and is less likely to be the cause
// if native crash doesn't happen before reboot. So we will clear the flag and disable
// two-phase rollback after reboot.
if (isRebootlessApex(rollback)) {
mTwoPhaseRollbackEnabled = true;
writeBoolean(mTwoPhaseRollbackEnabledFile, true);
}
});
}
private static boolean isRebootlessApex(RollbackInfo rollback) {
if (!rollback.isStaged()) {
for (PackageRollbackInfo info : rollback.getPackages()) {
if (info.isApex()) {
return true;
}
}
}
return false;
}
/** Verifies the rollback state after a reboot and schedules polling for sometime after reboot
* to check for native crashes and mitigate them if needed.
*/
@@ -155,6 +195,7 @@ final class RollbackPackageHealthObserver implements PackageHealthObserver {
@WorkerThread
private void onBootCompleted() {
assertInWorkerThread();
RollbackManager rollbackManager = mContext.getSystemService(RollbackManager.class);
if (!rollbackManager.getAvailableRollbacks().isEmpty()) {
// TODO(gavincorkery): Call into Package Watchdog from outside the observer
@@ -277,6 +318,23 @@ final class RollbackPackageHealthObserver implements PackageHealthObserver {
return mPendingStagedRollbackIds.isEmpty();
}
private static boolean readBoolean(File file) {
try (FileInputStream fis = new FileInputStream(file)) {
return fis.read() == 1;
} catch (IOException ignore) {
return false;
}
}
private static void writeBoolean(File file, boolean value) {
try (FileOutputStream fos = new FileOutputStream(file)) {
fos.write(value ? 1 : 0);
fos.flush();
FileUtils.sync(fos);
} catch (IOException ignore) {
}
}
@WorkerThread
private void saveStagedRollbackId(int stagedRollbackId, @Nullable VersionedPackage logPackage) {
assertInWorkerThread();
@@ -420,13 +478,44 @@ final class RollbackPackageHealthObserver implements PackageHealthObserver {
Collections.singletonList(failedPackage), rollbackReceiver.getIntentSender());
}
/**
* Two-phase rollback:
* 1. roll back rebootless apexes first
* 2. roll back all remaining rollbacks if native crash doesn't stop after (1) is done
*
* This approach gives us a better chance to correctly attribute native crash to rebootless
* apex update without rolling back Mainline updates which might contains critical security
* fixes.
*/
@WorkerThread
private boolean useTwoPhaseRollback(List<RollbackInfo> rollbacks) {
assertInWorkerThread();
if (!mTwoPhaseRollbackEnabled) {
return false;
}
Slog.i(TAG, "Rolling back all rebootless APEX rollbacks");
boolean found = false;
for (RollbackInfo rollback : rollbacks) {
if (isRebootlessApex(rollback)) {
VersionedPackage sample = rollback.getPackages().get(0).getVersionRolledBackFrom();
rollbackPackage(rollback, sample, PackageWatchdog.FAILURE_REASON_NATIVE_CRASH);
found = true;
}
}
return found;
}
@WorkerThread
private void rollbackAll() {
assertInWorkerThread();
Slog.i(TAG, "Rolling back all available rollbacks");
RollbackManager rollbackManager = mContext.getSystemService(RollbackManager.class);
List<RollbackInfo> rollbacks = rollbackManager.getAvailableRollbacks();
if (useTwoPhaseRollback(rollbacks)) {
return;
}
Slog.i(TAG, "Rolling back all available rollbacks");
// Add all rollback ids to mPendingStagedRollbackIds, so that we do not reboot before all
// pending staged rollbacks are handled.
for (RollbackInfo rollback : rollbacks) {

25
tests/RollbackTest/RollbackTest/src/com/android/tests/rollback/StagedRollbackTest.java
View File

@@ -58,6 +58,7 @@ import java.util.concurrent.TimeUnit;
public class StagedRollbackTest {
private static final String PROPERTY_WATCHDOG_TRIGGER_FAILURE_COUNT =
"watchdog_trigger_failure_count";
private static final String REBOOTLESS_APEX_NAME = "test.apex.rebootless";
/**
* Adopts common shell permissions needed for rollback tests.
@@ -242,7 +243,7 @@ public class StagedRollbackTest {
@Test
public void testRollbackRebootlessApex() throws Exception {
final String packageName = "test.apex.rebootless";
final String packageName = REBOOTLESS_APEX_NAME;
assertThat(InstallUtils.getInstalledVersion(packageName)).isEqualTo(1);
// install
@@ -267,6 +268,28 @@ public class StagedRollbackTest {
assertThat(InstallUtils.getInstalledVersion(packageName)).isEqualTo(1);
}
@Test
public void testNativeWatchdogTriggersRebootlessApexRollback_Phase1_Install() throws Exception {
assertThat(InstallUtils.getInstalledVersion(REBOOTLESS_APEX_NAME)).isEqualTo(1);
TestApp apex2 = new TestApp("TestRebootlessApexV2", REBOOTLESS_APEX_NAME, 2,
/* isApex= */ true, "test.rebootless_apex_v2.apex");
Install.single(apex2).setEnableRollback(PackageManager.ROLLBACK_DATA_POLICY_RETAIN)
.commit();
Install.single(TestApp.A1).commit();
Install.single(TestApp.A2).setEnableRollback().commit();
RollbackUtils.waitForAvailableRollback(TestApp.A);
RollbackUtils.waitForAvailableRollback(REBOOTLESS_APEX_NAME);
}
@Test
public void testNativeWatchdogTriggersRebootlessApexRollback_Phase2_Verify() throws Exception {
// Check only rebootless apex is rolled back. Other rollbacks should remain unchanged.
assertThat(RollbackUtils.getCommittedRollback(REBOOTLESS_APEX_NAME)).isNotNull();
assertThat(RollbackUtils.getAvailableRollback(TestApp.A)).isNotNull();
}
@Test
public void hasMainlineModule() throws Exception {
String pkgName = getModuleMetadataPackageName();

26
tests/RollbackTest/StagedRollbackTest/src/com/android/tests/rollback/host/StagedRollbackTest.java
View File

@@ -191,6 +191,18 @@ public class StagedRollbackTest extends BaseHostJUnit4Test {
runPhase("testRollbackRebootlessApex");
}
/**
* Tests only rebootless apex (if any) is rolled back when native crash happens
*/
@Test
public void testNativeWatchdogTriggersRebootlessApexRollback() throws Exception {
pushTestApex("test.rebootless_apex_v1.apex");
runPhase("testNativeWatchdogTriggersRebootlessApexRollback_Phase1_Install");
crashProcess("system_server", NATIVE_CRASHES_THRESHOLD);
getDevice().waitForDeviceAvailable();
runPhase("testNativeWatchdogTriggersRebootlessApexRollback_Phase2_Verify");
}
/**
* Tests that packages are monitored across multiple reboots.
*/
@@ -253,4 +265,18 @@ public class StagedRollbackTest extends BaseHostJUnit4Test {
return false;
}
}
private void crashProcess(String processName, int numberOfCrashes) throws Exception {
String pid = "";
String lastPid = "invalid";
for (int i = 0; i < numberOfCrashes; ++i) {
// This condition makes sure before we kill the process, the process is running AND
// the last crash was finished.
while ("".equals(pid) || lastPid.equals(pid)) {
pid = getDevice().executeShellCommand("pidof " + processName);
}
getDevice().executeShellCommand("kill " + pid);
lastPid = pid;
}
}
}