Merge changes from topic 'cherry_ota_dexopt'

* changes:
  OtaDexopt: Fix after-ota space measure
  OtaDexoptService: Add tron logging
  OtaDexopt: Downgrade apps when low on space
  Installer: Support delete_odex command
  Package Manager: Sort list of packages to dexopt

services/core/java/com/android/server/pm/Installer.java

@@ -230,6 +230,11 @@ public final class Installer extends SystemService {
         mInstaller.execute("move_ab", apkPath, instructionSet, outputPath);
     }
 
+    public void deleteOdex(String apkPath, String instructionSet, String outputPath)
+            throws InstallerException {
+        mInstaller.execute("delete_odex", apkPath, instructionSet, outputPath);
+    }
+
     private static void assertValidInstructionSet(String instructionSet)
             throws InstallerException {
         for (String abi : Build.SUPPORTED_ABIS) {

services/core/java/com/android/server/pm/OtaDexoptService.java

@@ -31,7 +31,7 @@ import android.os.ServiceManager;
 import android.os.storage.StorageManager;
 import android.util.Log;
 import android.util.Slog;
-
+import com.android.internal.logging.MetricsLogger;
 import com.android.internal.os.InstallerConnection;
 import com.android.internal.os.InstallerConnection.InstallerException;
 
@@ -40,6 +40,7 @@ import java.io.FileDescriptor;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
+import java.util.concurrent.TimeUnit;
 
 /**
  * A service for A/B OTA dexopting.
@@ -53,6 +54,10 @@ public class OtaDexoptService extends IOtaDexopt.Stub {
     // The synthetic library dependencies denoting "no checks."
     private final static String[] NO_LIBRARIES = new String[] { "&" };
 
+    // The amount of "available" (free - low threshold) space necessary at the start of an OTA to
+    // not bulk-delete unused apps' odex files.
+    private final static long BULK_DELETE_THRESHOLD = 1024 * 1024 * 1024;  // 1GB.
+
     private final Context mContext;
     private final PackageManagerService mPackageManagerService;
 
@@ -65,6 +70,25 @@ public class OtaDexoptService extends IOtaDexopt.Stub {
 
     private int completeSize;
 
+    // MetricsLogger properties.
+
+    // Space before and after.
+    private long availableSpaceBefore;
+    private long availableSpaceAfterBulkDelete;
+    private long availableSpaceAfterDexopt;
+
+    // Packages.
+    private int importantPackageCount;
+    private int otherPackageCount;
+
+    // Number of dexopt commands. This may be different from the count of packages.
+    private int dexoptCommandCountTotal;
+    private int dexoptCommandCountExecuted;
+
+    // For spent time.
+    private long otaDexoptTimeStart;
+
+
     public OtaDexoptService(Context context, PackageManagerService packageManagerService) {
         this.mContext = context;
         this.mPackageManagerService = packageManagerService;
@@ -128,6 +152,18 @@ public class OtaDexoptService extends IOtaDexopt.Stub {
                     generatePackageDexopts(p, PackageManagerService.REASON_FIRST_BOOT));
         }
         completeSize = mDexoptCommands.size();
+
+        long spaceAvailable = getAvailableSpace();
+        if (spaceAvailable < BULK_DELETE_THRESHOLD) {
+            Log.i(TAG, "Low on space, deleting oat files in an attempt to free up space: "
+                    + PackageManagerServiceUtils.packagesToString(others));
+            for (PackageParser.Package pkg : others) {
+                deleteOatArtifactsOfPackage(pkg);
+            }
+        }
+        long spaceAvailableNow = getAvailableSpace();
+
+        prepareMetricsLogging(important.size(), others.size(), spaceAvailable, spaceAvailableNow);
     }
 
     @Override
@@ -136,6 +172,9 @@ public class OtaDexoptService extends IOtaDexopt.Stub {
             Log.i(TAG, "Cleaning up OTA Dexopt state.");
         }
         mDexoptCommands = null;
+        availableSpaceAfterDexopt = getAvailableSpace();
+
+        performMetricsLogging();
     }
 
     @Override
@@ -169,28 +208,67 @@ public class OtaDexoptService extends IOtaDexopt.Stub {
 
         String next = mDexoptCommands.remove(0);
 
-        if (IsFreeSpaceAvailable()) {
+        if (getAvailableSpace() > 0) {
+            dexoptCommandCountExecuted++;
+
             return next;
         } else {
+            if (DEBUG_DEXOPT) {
+                Log.w(TAG, "Not enough space for OTA dexopt, stopping with "
+                        + (mDexoptCommands.size() + 1) + " commands left.");
+            }
             mDexoptCommands.clear();
             return "(no free space)";
         }
     }
 
-    /**
+    private long getMainLowSpaceThreshold() {
-     * Check for low space. Returns true if there's space left.
-     */
-    private boolean IsFreeSpaceAvailable() {
-        // TODO: If apps are not installed in the internal /data partition, we should compare
-        //       against that storage's free capacity.
         File dataDir = Environment.getDataDirectory();
         @SuppressWarnings("deprecation")
         long lowThreshold = StorageManager.from(mContext).getStorageLowBytes(dataDir);
         if (lowThreshold == 0) {
             throw new IllegalStateException("Invalid low memory threshold");
         }
+        return lowThreshold;
+    }
+
+    /**
+     * Returns the difference of free space to the low-storage-space threshold. Positive values
+     * indicate free bytes.
+     */
+    private long getAvailableSpace() {
+        // TODO: If apps are not installed in the internal /data partition, we should compare
+        //       against that storage's free capacity.
+        long lowThreshold = getMainLowSpaceThreshold();
+
+        File dataDir = Environment.getDataDirectory();
         long usableSpace = dataDir.getUsableSpace();
-        return (usableSpace >= lowThreshold);
+
+        return usableSpace - lowThreshold;
+    }
+
+    private static String getOatDir(PackageParser.Package pkg) {
+        if (!pkg.canHaveOatDir()) {
+            return null;
+        }
+        File codePath = new File(pkg.codePath);
+        if (codePath.isDirectory()) {
+            return PackageDexOptimizer.getOatDir(codePath).getAbsolutePath();
+        }
+        return null;
+    }
+
+    private void deleteOatArtifactsOfPackage(PackageParser.Package pkg) {
+        String[] instructionSets = getAppDexInstructionSets(pkg.applicationInfo);
+        for (String codePath : pkg.getAllCodePaths()) {
+            for (String isa : instructionSets) {
+                try {
+                    mPackageManagerService.mInstaller.deleteOdex(codePath, isa, getOatDir(pkg));
+                } catch (InstallerException e) {
+                    Log.e(TAG, "Failed deleting oat files for " + codePath, e);
+                }
+            }
+        }
     }
 
     /**
@@ -271,6 +349,55 @@ public class OtaDexoptService extends IOtaDexopt.Stub {
         }
     }
 
+    /**
+     * Initialize logging fields.
+     */
+    private void prepareMetricsLogging(int important, int others, long spaceBegin, long spaceBulk) {
+        availableSpaceBefore = spaceBegin;
+        availableSpaceAfterBulkDelete = spaceBulk;
+        availableSpaceAfterDexopt = 0;
+
+        importantPackageCount = important;
+        otherPackageCount = others;
+
+        dexoptCommandCountTotal = mDexoptCommands.size();
+        dexoptCommandCountExecuted = 0;
+
+        otaDexoptTimeStart = System.nanoTime();
+    }
+
+    private static int inMegabytes(long value) {
+        long in_mega_bytes = value / (1024 * 1024);
+        if (in_mega_bytes > Integer.MAX_VALUE) {
+            Log.w(TAG, "Recording " + in_mega_bytes + "MB of free space, overflowing range");
+            return Integer.MAX_VALUE;
+        }
+        return (int)in_mega_bytes;
+    }
+
+    private void performMetricsLogging() {
+        long finalTime = System.nanoTime();
+
+        MetricsLogger.histogram(mContext, "ota_dexopt_available_space_before_mb",
+                inMegabytes(availableSpaceBefore));
+        MetricsLogger.histogram(mContext, "ota_dexopt_available_space_after_bulk_delete_mb",
+                inMegabytes(availableSpaceAfterBulkDelete));
+        MetricsLogger.histogram(mContext, "ota_dexopt_available_space_after_dexopt_mb",
+                inMegabytes(availableSpaceAfterDexopt));
+
+        MetricsLogger.histogram(mContext, "ota_dexopt_num_important_packages",
+                importantPackageCount);
+        MetricsLogger.histogram(mContext, "ota_dexopt_num_other_packages", otherPackageCount);
+
+        MetricsLogger.histogram(mContext, "ota_dexopt_num_commands", dexoptCommandCountTotal);
+        MetricsLogger.histogram(mContext, "ota_dexopt_num_commands_executed",
+                dexoptCommandCountExecuted);
+
+        final int elapsedTimeSeconds =
+                (int) TimeUnit.NANOSECONDS.toSeconds(finalTime - otaDexoptTimeStart);
+        MetricsLogger.histogram(mContext, "ota_dexopt_time_s", elapsedTimeSeconds);
+    }
+
     private static class OTADexoptPackageDexOptimizer extends
             PackageDexOptimizer.ForcedUpdatePackageDexOptimizer {
 

services/core/java/com/android/server/pm/PackageManagerServiceUtils.java

@@ -19,6 +19,7 @@ package com.android.server.pm;
 import static com.android.server.pm.PackageManagerService.DEBUG_DEXOPT;
 import static com.android.server.pm.PackageManagerService.TAG;
 
+import android.annotation.NonNull;
 import android.app.AppGlobals;
 import android.content.Intent;
 import android.content.pm.PackageParser;
@@ -35,13 +36,9 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.Comparator;
-import java.util.Date;
-import java.util.HashSet;
-import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
-import java.util.Set;
+import java.util.function.Predicate;
 
 /**
  * Class containing helper methods for the PackageManagerService.
@@ -67,34 +64,51 @@ public class PackageManagerServiceUtils {
         return pkgNames;
     }
 
-    private static void filterRecentlyUsedApps(Collection<PackageParser.Package> pkgs,
+    // Sort a list of apps by their last usage, most recently used apps first. The order of
-            long estimatedPreviousSystemUseTime,
+    // packages without usage data is undefined (but they will be sorted after the packages
-            long dexOptLRUThresholdInMills) {
+    // that do have usage data).
-        // Filter out packages that aren't recently used.
+    public static void sortPackagesByUsageDate(List<PackageParser.Package> pkgs,
-        int total = pkgs.size();
+            PackageManagerService packageManagerService) {
-        int skipped = 0;
+        if (!packageManagerService.isHistoricalPackageUsageAvailable()) {
-        for (Iterator<PackageParser.Package> i = pkgs.iterator(); i.hasNext();) {
+            return;
-            PackageParser.Package pkg = i.next();
+        }
-            long then = pkg.getLatestForegroundPackageUseTimeInMills();
+
-            if (then < estimatedPreviousSystemUseTime - dexOptLRUThresholdInMills) {
+        Collections.sort(pkgs, (pkg1, pkg2) ->
-                if (DEBUG_DEXOPT) {
+                Long.compare(pkg2.getLatestForegroundPackageUseTimeInMills(),
-                    Log.i(TAG, "Skipping dexopt of " + pkg.packageName +
+                        pkg1.getLatestForegroundPackageUseTimeInMills()));
-                            " last used in foreground: " +
+    }
-                            ((then == 0) ? "never" : new Date(then)));
+
-                }
+    // Apply the given {@code filter} to all packages in {@code packages}. If tested positive, the
-                i.remove();
+    // package will be removed from {@code packages} and added to {@code result} with its
-                skipped++;
+    // dependencies. If usage data is available, the positive packages will be sorted by usage
-            } else {
+    // data (with {@code sortTemp} as temporary storage).
-                if (DEBUG_DEXOPT) {
+    private static void applyPackageFilter(Predicate<PackageParser.Package> filter,
-                    Log.i(TAG, "Will dexopt " + pkg.packageName +
+            Collection<PackageParser.Package> result,
-                            " last used in foreground: " +
+            Collection<PackageParser.Package> packages,
-                            ((then == 0) ? "never" : new Date(then)));
+            @NonNull List<PackageParser.Package> sortTemp,
-                }
+            PackageManagerService packageManagerService) {
+        for (PackageParser.Package pkg : packages) {
+            if (filter.test(pkg)) {
+                sortTemp.add(pkg);
             }
         }
-        if (DEBUG_DEXOPT) {
+
-            Log.i(TAG, "Skipped dexopt " + skipped + " of " + total);
+        sortPackagesByUsageDate(sortTemp, packageManagerService);
+        packages.removeAll(sortTemp);
+
+        for (PackageParser.Package pkg : sortTemp) {
+            result.add(pkg);
+
+            Collection<PackageParser.Package> deps =
+                    packageManagerService.findSharedNonSystemLibraries(pkg);
+            if (!deps.isEmpty()) {
+                deps.removeAll(result);
+                result.addAll(deps);
+                packages.removeAll(deps);
+            }
         }
+
+        sortTemp.clear();
     }
 
     // Sort apps by importance for dexopt ordering. Important apps are given
@@ -104,46 +118,25 @@ public class PackageManagerServiceUtils {
             PackageManagerService packageManagerService) {
         ArrayList<PackageParser.Package> remainingPkgs = new ArrayList<>(packages);
         LinkedList<PackageParser.Package> result = new LinkedList<>();
+        ArrayList<PackageParser.Package> sortTemp = new ArrayList<>(remainingPkgs.size());
 
         // Give priority to core apps.
-        for (PackageParser.Package pkg : remainingPkgs) {
+        applyPackageFilter((pkg) -> pkg.coreApp, result, remainingPkgs, sortTemp,
-            if (pkg.coreApp) {
+                packageManagerService);
-                if (DEBUG_DEXOPT) {
-                    Log.i(TAG, "Adding core app " + result.size() + ": " + pkg.packageName);
-                }
-                result.add(pkg);
-            }
-        }
-        remainingPkgs.removeAll(result);
 
         // Give priority to system apps that listen for pre boot complete.
         Intent intent = new Intent(Intent.ACTION_PRE_BOOT_COMPLETED);
-        ArraySet<String> pkgNames = getPackageNamesForIntent(intent, UserHandle.USER_SYSTEM);
+        final ArraySet<String> pkgNames = getPackageNamesForIntent(intent, UserHandle.USER_SYSTEM);
-        for (PackageParser.Package pkg : remainingPkgs) {
+        applyPackageFilter((pkg) -> pkgNames.contains(pkg.packageName), result, remainingPkgs,
-            if (pkgNames.contains(pkg.packageName)) {
+                sortTemp, packageManagerService);
-                if (DEBUG_DEXOPT) {
-                    Log.i(TAG, "Adding pre boot system app " + result.size() + ": " +
-                            pkg.packageName);
-                }
-                result.add(pkg);
-            }
-        }
-        remainingPkgs.removeAll(result);
 
         // Give priority to apps used by other apps.
-        for (PackageParser.Package pkg : remainingPkgs) {
+        applyPackageFilter((pkg) -> PackageDexOptimizer.isUsedByOtherApps(pkg), result,
-            if (PackageDexOptimizer.isUsedByOtherApps(pkg)) {
+                remainingPkgs, sortTemp, packageManagerService);
-                if (DEBUG_DEXOPT) {
-                    Log.i(TAG, "Adding app used by other apps " + result.size() + ": " +
-                            pkg.packageName);
-                }
-                result.add(pkg);
-            }
-        }
-        remainingPkgs.removeAll(result);
 
         // Filter out packages that aren't recently used, add all remaining apps.
         // TODO: add a property to control this?
+        Predicate<PackageParser.Package> remainingPredicate;
         if (!remainingPkgs.isEmpty() && packageManagerService.isHistoricalPackageUsageAvailable()) {
             if (DEBUG_DEXOPT) {
                 Log.i(TAG, "Looking at historical package use");
@@ -159,34 +152,24 @@ public class PackageManagerServiceUtils {
                     lastUsed.getLatestForegroundPackageUseTimeInMills();
             // Be defensive if for some reason package usage has bogus data.
             if (estimatedPreviousSystemUseTime != 0) {
-                filterRecentlyUsedApps(remainingPkgs, estimatedPreviousSystemUseTime,
+                final long cutoffTime = estimatedPreviousSystemUseTime - SEVEN_DAYS_IN_MILLISECONDS;
-                        SEVEN_DAYS_IN_MILLISECONDS);
+                remainingPredicate =
+                        (pkg) -> pkg.getLatestForegroundPackageUseTimeInMills() >= cutoffTime;
+            } else {
+                // No meaningful historical info. Take all.
+                remainingPredicate = (pkg) -> true;
             }
+            sortPackagesByUsageDate(remainingPkgs, packageManagerService);
+        } else {
+            // No historical info. Take all.
+            remainingPredicate = (pkg) -> true;
         }
-        result.addAll(remainingPkgs);
+        applyPackageFilter(remainingPredicate, result, remainingPkgs, sortTemp,
-
+                packageManagerService);
-        // Now go ahead and also add the libraries required for these packages.
-        // TODO: Think about interleaving things.
-        Set<PackageParser.Package> dependencies = new HashSet<>();
-        for (PackageParser.Package p : result) {
-            dependencies.addAll(packageManagerService.findSharedNonSystemLibraries(p));
-        }
-        if (!dependencies.isEmpty()) {
-            // We might have packages already in `result` that are dependencies
-            // of other packages. Make sure we don't add those to the list twice.
-            dependencies.removeAll(result);
-        }
-        result.addAll(dependencies);
 
         if (DEBUG_DEXOPT) {
-            StringBuilder sb = new StringBuilder();
+            Log.i(TAG, "Packages to be dexopted: " + packagesToString(result));
-            for (PackageParser.Package pkg : result) {
+            Log.i(TAG, "Packages skipped from dexopt: " + packagesToString(remainingPkgs));
-                if (sb.length() > 0) {
-                    sb.append(", ");
-                }
-                sb.append(pkg.packageName);
-            }
-            Log.i(TAG, "Packages to be dexopted: " + sb.toString());
         }
 
         return result;
@@ -203,4 +186,15 @@ public class PackageManagerServiceUtils {
             throw ee.rethrowAsIOException();
         }
     }
+
+    public static String packagesToString(Collection<PackageParser.Package> c) {
+        StringBuilder sb = new StringBuilder();
+        for (PackageParser.Package pkg : c) {
+            if (sb.length() > 0) {
+                sb.append(", ");
+            }
+            sb.append(pkg.packageName);
+        }
+        return sb.toString();
+    }
 }