From 7eb1893c08130163852a7df2a53cb321dc493acc Mon Sep 17 00:00:00 2001 From: Mohammad Samiul Islam Date: Thu, 15 Oct 2020 00:59:32 +0100 Subject: [PATCH 1/5] Prevent exceptions in pre-reboot verification from crashing system server An unhandled exception during pre-reboot verification will cause system server to crash and restart. Upon restart, pre-reboot verification will be retried and it will cause system server to crash again. Thus creating a loop. Instead of allowing the exception to crash the system server, we now catch it and fail the corresponding staged session with appropriate message. Bug: 170784748 Test: manual Test: verified that without this fix, any unhandled exception sends system server into a crash loop Change-Id: Ie3343681601c072add0b2d50cbaa2e088a27d94b Merged-In: If100796687906be67bc02ea63a78e1b6e291ce7a (cherry picked from commit 5a2548589560404f9fce4914216564b0333895d4) --- .../com/android/server/pm/StagingManager.java | 133 +++++++++--------- 1 file changed, 70 insertions(+), 63 deletions(-) diff --git a/services/core/java/com/android/server/pm/StagingManager.java b/services/core/java/com/android/server/pm/StagingManager.java index dd7133121b217..96dd001008786 100644 --- a/services/core/java/com/android/server/pm/StagingManager.java +++ b/services/core/java/com/android/server/pm/StagingManager.java @@ -202,85 +202,92 @@ public class StagingManager { } private void preRebootVerification(@NonNull PackageInstallerSession session) { - boolean success = true; + try { + boolean success = true; - final ApexInfoList apexInfoList = new ApexInfoList(); - // APEX checks. For single-package sessions, check if they contain an APEX. For - // multi-package sessions, find all the child sessions that contain an APEX. - if (!session.isMultiPackage() - && isApexSession(session)) { - success = submitSessionToApexService(session, null, apexInfoList); + final ApexInfoList apexInfoList = new ApexInfoList(); + // APEX checks. For single-package sessions, check if they contain an APEX. For + // multi-package sessions, find all the child sessions that contain an APEX. + if (!session.isMultiPackage() + && isApexSession(session)) { + success = submitSessionToApexService(session, null, apexInfoList); - } else if (session.isMultiPackage()) { - List childSessions = - Arrays.stream(session.getChildSessionIds()) - // Retrieve cached sessions matching ids. - .mapToObj(i -> mStagedSessions.get(i)) - // Filter only the ones containing APEX. - .filter(childSession -> isApexSession(childSession)) - .collect(Collectors.toList()); - if (!childSessions.isEmpty()) { - success = submitSessionToApexService(session, childSessions, apexInfoList); - } // else this is a staged multi-package session with no APEX files. - } + } else if (session.isMultiPackage()) { + List childSessions = + Arrays.stream(session.getChildSessionIds()) + // Retrieve cached sessions matching ids. + .mapToObj(i -> mStagedSessions.get(i)) + // Filter only the ones containing APEX. + .filter(childSession -> isApexSession(childSession)) + .collect(Collectors.toList()); + if (!childSessions.isEmpty()) { + success = submitSessionToApexService(session, childSessions, apexInfoList); + } // else this is a staged multi-package session with no APEX files. + } - if (!success) { - // submitSessionToApexService will populate error. - return; - } - - if (sessionContainsApk(session)) { - if (!installApksInSession(session, /* preReboot */ true)) { - session.setStagedSessionFailed(SessionInfo.STAGED_SESSION_VERIFICATION_FAILED, - "APK verification failed. Check logcat messages for " - + "more information."); - // TODO(b/118865310): abort the session on apexd. + if (!success) { + // submitSessionToApexService will populate error. return; } - } - if (apexInfoList.apexInfos != null && apexInfoList.apexInfos.length > 0) { - // For APEXes, we validate the signature here before we mark the session as ready, - // so we fail the session early if there is a signature mismatch. For APKs, the - // signature verification will be done by the package manager at the point at which - // it applies the staged install. - for (ApexInfo apexPackage : apexInfoList.apexInfos) { - if (!validateApexSignature(apexPackage.packagePath, - apexPackage.packageName)) { + if (sessionContainsApk(session)) { + if (!installApksInSession(session, /* preReboot */ true)) { session.setStagedSessionFailed(SessionInfo.STAGED_SESSION_VERIFICATION_FAILED, - "APK-container signature verification failed for package " - + apexPackage.packageName + ". Signature of file " - + apexPackage.packagePath + " does not match the signature of " - + " the package already installed."); + "APK verification failed. Check logcat messages for " + + "more information."); // TODO(b/118865310): abort the session on apexd. return; } } - } - if ((session.params.installFlags & PackageManager.INSTALL_ENABLE_ROLLBACK) != 0) { - // If rollback is enabled for this session, we call through to the RollbackManager - // with the list of sessions it must enable rollback for. Note that notifyStagedSession - // is a synchronous operation. - final IRollbackManager rm = IRollbackManager.Stub.asInterface( - ServiceManager.getService(Context.ROLLBACK_SERVICE)); - try { - // NOTE: To stay consistent with the non-staged install flow, we don't fail the - // entire install if rollbacks can't be enabled. - if (!rm.notifyStagedSession(session.sessionId)) { - Slog.e(TAG, "Unable to enable rollback for session: " + session.sessionId); + if (apexInfoList.apexInfos != null && apexInfoList.apexInfos.length > 0) { + // For APEXes, we validate the signature here before we mark the session as ready, + // so we fail the session early if there is a signature mismatch. For APKs, the + // signature verification will be done by the package manager at the point at which + // it applies the staged install. + for (ApexInfo apexPackage : apexInfoList.apexInfos) { + if (!validateApexSignature(apexPackage.packagePath, + apexPackage.packageName)) { + session.setStagedSessionFailed( + SessionInfo.STAGED_SESSION_VERIFICATION_FAILED, + "APK-container signature verification failed for package " + + apexPackage.packageName + ". Signature of file " + + apexPackage.packagePath + " does not match the signature" + + " of the package already installed."); + // TODO(b/118865310): abort the session on apexd. + return; + } } - } catch (RemoteException re) { - // Cannot happen, the rollback manager is in the same process. } - } - session.setStagedSessionReady(); - if (sessionContainsApex(session) - && !mApexManager.markStagedSessionReady(session.sessionId)) { + if ((session.params.installFlags & PackageManager.INSTALL_ENABLE_ROLLBACK) != 0) { + // If rollback is enabled for this session, we call through to the RollbackManager + // with the list of sessions it must enable rollback for. Note that + // notifyStagedSession is a synchronous operation. + final IRollbackManager rm = IRollbackManager.Stub.asInterface( + ServiceManager.getService(Context.ROLLBACK_SERVICE)); + try { + // NOTE: To stay consistent with the non-staged install flow, we don't fail the + // entire install if rollbacks can't be enabled. + if (!rm.notifyStagedSession(session.sessionId)) { + Slog.e(TAG, "Unable to enable rollback for session: " + session.sessionId); + } + } catch (RemoteException re) { + // Cannot happen, the rollback manager is in the same process. + } + } + + session.setStagedSessionReady(); + if (sessionContainsApex(session) + && !mApexManager.markStagedSessionReady(session.sessionId)) { + session.setStagedSessionFailed(SessionInfo.STAGED_SESSION_VERIFICATION_FAILED, + "APEX staging failed, check logcat messages from apexd for more " + + "details."); + } + } catch (Exception e) { + Slog.e(TAG, "Pre-reboot verification failed due to unhandled exception", e); session.setStagedSessionFailed(SessionInfo.STAGED_SESSION_VERIFICATION_FAILED, - "APEX staging failed, check logcat messages from apexd for more " - + "details."); + "Pre-reboot verification failed due to unhandled exception: " + e); } } From 2cc603b00a87e02a81cbf9d79645bd1e3efc277c Mon Sep 17 00:00:00 2001 From: Mohammad Samiul Islam Date: Fri, 16 Oct 2020 19:18:43 +0100 Subject: [PATCH 2/5] Prevent exceptions during staged install from crashing system server An unhandled exception during staged install at boot time will cause system server to crash and restart. Upon restart, staged install will resume again and it will cause system server to crash again. Thus creating a loop. Instead of allowing the exception to crash the system server, we now catch it and fail the corresponding staged session with appropriate message. Bug: 170784748 Test: manual Test: verified that without this fix, any unhandled exception sends system server into a crash loop Change-Id: If79073a9a40c70e8223acbc47a3da394eb54fb43 Merged-In: I9abec5d2401af95ecb095fa3c45960d2f15d4e74 (cherry picked from commit 1c3e99b6c5a13ec2d7a8d3e618ce459fec0de7b7) --- .../com/android/server/pm/StagingManager.java | 44 ++++++++++++------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/services/core/java/com/android/server/pm/StagingManager.java b/services/core/java/com/android/server/pm/StagingManager.java index 96dd001008786..5adab3e3f56d5 100644 --- a/services/core/java/com/android/server/pm/StagingManager.java +++ b/services/core/java/com/android/server/pm/StagingManager.java @@ -353,22 +353,9 @@ public class StagingManager { } // The APEX part of the session is activated, proceed with the installation of APKs. if (!installApksInSession(session, /* preReboot */ false)) { - session.setStagedSessionFailed(SessionInfo.STAGED_SESSION_ACTIVATION_FAILED, - "Staged installation of APKs failed. Check logcat messages for" - + "more information."); - - if (!hasApex) { - return; - } - - if (!mApexManager.abortActiveSession()) { - Slog.e(TAG, "Failed to abort APEXd session"); - } else { - Slog.e(TAG, - "Successfully aborted apexd session. Rebooting device in order to revert " - + "to the previous state of APEXd."); - mPowerManager.reboot(null); - } + onInstallationFailure(session, new PackageManagerException( + SessionInfo.STAGED_SESSION_ACTIVATION_FAILED, "Staged installation of APKs " + + "failed. Check logcat messages for more information.")); return; } @@ -378,6 +365,21 @@ public class StagingManager { } } + void onInstallationFailure(PackageInstallerSession session, PackageManagerException e) { + session.setStagedSessionFailed(e.error, e.getMessage()); + if (!sessionContainsApex(session)) { + return; + } + if (!mApexManager.abortActiveSession()) { + Slog.e(TAG, "Failed to abort APEXd session"); + } else { + Slog.e(TAG, + "Successfully aborted apexd session. Rebooting device in order to revert " + + "to the previous state of APEXd."); + mPowerManager.reboot(null); + } + } + private List findAPKsInDir(File stageDir) { List ret = new ArrayList<>(); if (stageDir != null && stageDir.exists()) { @@ -671,7 +673,15 @@ public class StagingManager { } else { // Session had already being marked ready. Start the checks to verify if there is any // follow-up work. - resumeSession(session); + try { + resumeSession(session); + } catch (Exception e) { + Slog.e(TAG, "Staged install failed due to unhandled exception", e); + onInstallationFailure(session, new PackageManagerException( + SessionInfo.STAGED_SESSION_ACTIVATION_FAILED, + "Staged install failed due to unhandled exception: " + e)); + + } } } From 533c3deb7ba80fd9271748b788d59bfe4631c52f Mon Sep 17 00:00:00 2001 From: Mohammad Samiul Islam Date: Thu, 15 Oct 2020 01:05:13 +0100 Subject: [PATCH 3/5] Prevent extra sessions owned by staged install from living across restarts If system server crashes due to external reasons (not due to exception in pre-reboot verifications), then the staged install will be retried. But, any extra sessions that have not been cleaned up will linger on until removed due to old age. And if the crash keeps on happening every time at the worst possible time, then we can have 1024 extra sessions created all with copies of apks in it. It will waste a lot of user's space. With this CL, we cleanup any leftover extra sessions whenever system server is restarted. Bug: 170784748 Test: manual Test: verified that without this CL if system server crashses after extra sessions are created but before they are cleaned up, then folders start to pile up in /data/app folder with copy of the apks inside. Change-Id: I5c19093853e28c668f882d37691959b1ac2464cc Merged-In: I5c19093853e28c668f882d37691959b1ac2464cc --- .../com/android/server/pm/PackageInstallerService.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/services/core/java/com/android/server/pm/PackageInstallerService.java b/services/core/java/com/android/server/pm/PackageInstallerService.java index 1c9d56683dba3..47dc5092c31d3 100644 --- a/services/core/java/com/android/server/pm/PackageInstallerService.java +++ b/services/core/java/com/android/server/pm/PackageInstallerService.java @@ -380,6 +380,8 @@ public class PackageInstallerService extends IPackageInstaller.Stub implements Slog.w(TAG, "Abandoning old session created at " + session.createdMillis); valid = false; + } else if (isExtraSessionForStagedInstall(session)) { + valid = false; } else { valid = true; } @@ -410,6 +412,13 @@ public class PackageInstallerService extends IPackageInstaller.Stub implements } } + // Extra sessions are created during staged install on temporary basis. They should not be + // allowed to live across system server restart. + private boolean isExtraSessionForStagedInstall(PackageInstallerSession session) { + return (session.params.installFlags & PackageManager.INSTALL_DRY_RUN) != 0 + || (session.params.installFlags & PackageManager.INSTALL_DISABLE_VERIFICATION) != 0; + } + @GuardedBy("mSessions") private void addHistoricalSessionLocked(PackageInstallerSession session) { CharArrayWriter writer = new CharArrayWriter(); From 739300762b2174932a814afed874ed4f9e923e93 Mon Sep 17 00:00:00 2001 From: Mohammad Samiul Islam Date: Wed, 10 Jun 2020 18:00:24 +0100 Subject: [PATCH 4/5] Prevent sessions from resuming once boot is completed This will prevent sessions from resuming when system server crashes or restarted. Bug: 158283778 Test: staged a session and crashed system server to verify session stays unchanged Test: atest StagedInstallInternalTest Test: atest StagedInstallTest Change-Id: I99337ea2898cfdf2cc515819b4f5b5db4b038f31 Merged-In: I99337ea2898cfdf2cc515819b4f5b5db4b038f31 --- .../core/java/com/android/server/pm/StagingManager.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/services/core/java/com/android/server/pm/StagingManager.java b/services/core/java/com/android/server/pm/StagingManager.java index 5adab3e3f56d5..ff578a665458d 100644 --- a/services/core/java/com/android/server/pm/StagingManager.java +++ b/services/core/java/com/android/server/pm/StagingManager.java @@ -44,6 +44,7 @@ import android.os.ParcelableException; import android.os.PowerManager; import android.os.RemoteException; import android.os.ServiceManager; +import android.os.SystemProperties; import android.util.Slog; import android.util.SparseArray; import android.util.apk.ApkSignatureVerifier; @@ -657,6 +658,11 @@ public class StagingManager { } private void checkStateAndResume(@NonNull PackageInstallerSession session) { + // Do not resume session if boot completed already + if (SystemProperties.getBoolean("sys.boot_completed", false)) { + return; + } + if (!session.isCommitted()) { // Session hasn't been committed yet, ignore. return; From 7347288f9193a3cd0c06ef859a5d1a6b7c707efd Mon Sep 17 00:00:00 2001 From: Rhed Jao Date: Wed, 30 Oct 2019 20:54:37 +0800 Subject: [PATCH 5/5] Reschedule the pre-reboot verification after boot completed. Bug: 143528612 Test: atest CtsStagedInstallHostTestCases Change-Id: I34d7ee3b84bc351d58ebf024a7ed10dd7db1f55b Merged-In: I34d7ee3b84bc351d58ebf024a7ed10dd7db1f55b (cherry picked from commit b793c2174ec133b6f7d3b1d439d30423fffcf59f) --- .../server/pm/PackageInstallerService.java | 1 + .../com/android/server/pm/StagingManager.java | 33 +++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/services/core/java/com/android/server/pm/PackageInstallerService.java b/services/core/java/com/android/server/pm/PackageInstallerService.java index 47dc5092c31d3..6a47c4c544e84 100644 --- a/services/core/java/com/android/server/pm/PackageInstallerService.java +++ b/services/core/java/com/android/server/pm/PackageInstallerService.java @@ -218,6 +218,7 @@ public class PackageInstallerService extends IPackageInstaller.Stub implements public void systemReady() { mAppOps = mContext.getSystemService(AppOpsManager.class); + mStagingManager.systemReady(); synchronized (mSessions) { readSessionsLocked(); diff --git a/services/core/java/com/android/server/pm/StagingManager.java b/services/core/java/com/android/server/pm/StagingManager.java index ff578a665458d..2210ff86133c6 100644 --- a/services/core/java/com/android/server/pm/StagingManager.java +++ b/services/core/java/com/android/server/pm/StagingManager.java @@ -21,10 +21,12 @@ import android.annotation.Nullable; import android.apex.ApexInfo; import android.apex.ApexInfoList; import android.apex.ApexSessionInfo; +import android.content.BroadcastReceiver; import android.content.Context; import android.content.IIntentReceiver; import android.content.IIntentSender; import android.content.Intent; +import android.content.IntentFilter; import android.content.IntentSender; import android.content.pm.PackageInfo; import android.content.pm.PackageInstaller; @@ -73,12 +75,16 @@ public class StagingManager { private final PackageInstallerService mPi; private final ApexManager mApexManager; private final PowerManager mPowerManager; + private final Context mContext; private final Handler mBgHandler; + private PackageInstallerSession mPendingSession; + private boolean mIsReady; @GuardedBy("mStagedSessions") private final SparseArray mStagedSessions = new SparseArray<>(); StagingManager(PackageInstallerService pi, ApexManager am, Context context) { + mContext = context; mPi = pi; mApexManager = am; mPowerManager = (PowerManager) context.getSystemService(Context.POWER_SERVICE); @@ -204,6 +210,11 @@ public class StagingManager { private void preRebootVerification(@NonNull PackageInstallerSession session) { try { + if (!mIsReady) { + mPendingSession = session; + return; + } + boolean success = true; final ApexInfoList apexInfoList = new ApexInfoList(); @@ -691,6 +702,28 @@ public class StagingManager { } } + void systemReady() { + // Register the receiver of boot completed intent for staging manager. + mContext.registerReceiver(new BroadcastReceiver() { + @Override + public void onReceive(Context ctx, Intent intent) { + readyToStart(); + ctx.unregisterReceiver(this); + } + }, new IntentFilter(Intent.ACTION_BOOT_COMPLETED)); + } + + // Notify the handler that system is ready, and reschedule the pre-reboot verifications. + private synchronized void readyToStart() { + mIsReady = true; + if (mPendingSession != null) { + mBgHandler.post(() -> { + preRebootVerification(mPendingSession); + mPendingSession = null; + }); + } + } + private static class LocalIntentReceiver { private final LinkedBlockingQueue mResult = new LinkedBlockingQueue<>();