Merge "Migrate to use process_madvise syscall instead of procfs interface for memory compaction" into sc-dev

services/core/java/com/android/server/am/CachedAppOptimizer.java

@@ -80,20 +80,22 @@ public final class CachedAppOptimizer {
 
     // Phenotype sends int configurations and we map them to the strings we'll use on device,
     // preventing a weird string value entering the kernel.
+    private static final int COMPACT_ACTION_NONE = 0;
+    private static final int COMPACT_ACTION_FILE = 1;
+    private static final int COMPACT_ACTION_ANON = 2;
+    private static final int COMPACT_ACTION_FULL = 3;
+
+    private static final String COMPACT_ACTION_STRING[] = {"", "file", "anon", "all"};
+
+    // Keeps these flags in sync with services/core/jni/com_android_server_am_CachedAppOptimizer.cpp
     private static final int COMPACT_ACTION_FILE_FLAG = 1;
     private static final int COMPACT_ACTION_ANON_FLAG = 2;
-    private static final int COMPACT_ACTION_FULL_FLAG = 3;
-    private static final int COMPACT_ACTION_NONE_FLAG = 4;
-    private static final String COMPACT_ACTION_NONE = "";
-    private static final String COMPACT_ACTION_FILE = "file";
-    private static final String COMPACT_ACTION_ANON = "anon";
-    private static final String COMPACT_ACTION_FULL = "all";
 
     // Defaults for phenotype flags.
     @VisibleForTesting static final Boolean DEFAULT_USE_COMPACTION = false;
     @VisibleForTesting static final Boolean DEFAULT_USE_FREEZER = false;
-    @VisibleForTesting static final int DEFAULT_COMPACT_ACTION_1 = COMPACT_ACTION_FILE_FLAG;
+    @VisibleForTesting static final int DEFAULT_COMPACT_ACTION_1 = COMPACT_ACTION_FILE;
-    @VisibleForTesting static final int DEFAULT_COMPACT_ACTION_2 = COMPACT_ACTION_FULL_FLAG;
+    @VisibleForTesting static final int DEFAULT_COMPACT_ACTION_2 = COMPACT_ACTION_FULL;
     @VisibleForTesting static final long DEFAULT_COMPACT_THROTTLE_1 = 5_000;
     @VisibleForTesting static final long DEFAULT_COMPACT_THROTTLE_2 = 10_000;
     @VisibleForTesting static final long DEFAULT_COMPACT_THROTTLE_3 = 500;
@@ -405,6 +407,14 @@ public final class CachedAppOptimizer {
 
     private native void compactSystem();
 
+    /**
+     * Compacts a process or app
+     * @param pid pid of process to compact
+     * @param compactionFlags selects the compaction type as defined by COMPACT_ACTION_{TYPE}_FLAG
+     *         constants
+     */
+    static private native void compactProcess(int pid, int compactionFlags);
+
     /**
      * Reads the flag value from DeviceConfig to determine whether app compaction
      * should be enabled, and starts the freeze/compaction thread if needed.
@@ -706,18 +716,11 @@ public final class CachedAppOptimizer {
 
     @VisibleForTesting
     static String compactActionIntToString(int action) {
-        switch(action) {
+        if (action < 0 || action >= COMPACT_ACTION_STRING.length) {
-            case COMPACT_ACTION_NONE_FLAG:
+            return "";
-                return COMPACT_ACTION_NONE;
-            case COMPACT_ACTION_FILE_FLAG:
-                return COMPACT_ACTION_FILE;
-            case COMPACT_ACTION_ANON_FLAG:
-                return COMPACT_ACTION_ANON;
-            case COMPACT_ACTION_FULL_FLAG:
-                return COMPACT_ACTION_FULL;
-            default:
-                return COMPACT_ACTION_NONE;
         }
+
+        return COMPACT_ACTION_STRING[action];
     }
 
     // This will ensure app will be out of the freezer for at least FREEZE_TIMEOUT_MS
@@ -950,11 +953,11 @@ public final class CachedAppOptimizer {
                             action = mCompactActionFull;
                             break;
                         default:
-                            action = COMPACT_ACTION_NONE;
+                            action = COMPACT_ACTION_STRING[COMPACT_ACTION_NONE];
                             break;
                     }
 
-                    if (COMPACT_ACTION_NONE.equals(action)) {
+                    if (COMPACT_ACTION_STRING[COMPACT_ACTION_NONE].equals(action)) {
                         return;
                     }
 
@@ -978,7 +981,8 @@ public final class CachedAppOptimizer {
                         return;
                     }
 
-                    if (action.equals(COMPACT_ACTION_FULL) || action.equals(COMPACT_ACTION_ANON)) {
+                    if (action.equals(COMPACT_ACTION_STRING[COMPACT_ACTION_FULL])
+                            || action.equals(COMPACT_ACTION_STRING[COMPACT_ACTION_ANON])) {
                         if (mFullAnonRssThrottleKb > 0L
                                 && anonRssBefore < mFullAnonRssThrottleKb) {
                             if (DEBUG_COMPACTION) {
@@ -1054,8 +1058,8 @@ public final class CachedAppOptimizer {
                             proc.lastCompactTime = end;
                             proc.lastCompactAction = pendingAction;
                         }
-                        if (action.equals(COMPACT_ACTION_FULL)
+                        if (action.equals(COMPACT_ACTION_STRING[COMPACT_ACTION_FULL])
-                                || action.equals(COMPACT_ACTION_ANON)) {
+                                || action.equals(COMPACT_ACTION_STRING[COMPACT_ACTION_ANON])) {
                             // Remove entry and insert again to update insertion order.
                             mLastCompactionStats.remove(pid);
                             mLastCompactionStats.put(pid, new LastCompactionStats(rssAfter));
@@ -1229,8 +1233,12 @@ public final class CachedAppOptimizer {
         // Compact process.
         @Override
         public void performCompaction(String action, int pid) throws IOException {
-            try (FileOutputStream fos = new FileOutputStream("/proc/" + pid + "/reclaim")) {
+            if (action.equals(COMPACT_ACTION_STRING[COMPACT_ACTION_FULL])) {
-                fos.write(action.getBytes());
+                compactProcess(pid, COMPACT_ACTION_FILE_FLAG | COMPACT_ACTION_ANON_FLAG);
+            } else if (action.equals(COMPACT_ACTION_STRING[COMPACT_ACTION_FILE])) {
+                compactProcess(pid, COMPACT_ACTION_FILE_FLAG);
+            } else if (action.equals(COMPACT_ACTION_STRING[COMPACT_ACTION_ANON])) {
+                compactProcess(pid, COMPACT_ACTION_ANON_FLAG);
             }
         }
     }

services/core/jni/Android.bp

@@ -70,6 +70,7 @@ cc_library_static {
         "frameworks/base/libs",
         "frameworks/native/services",
         "system/gatekeeper/include",
+        "system/memory/libmeminfo/include",
     ],
 
     header_libs: [
@@ -97,6 +98,7 @@ cc_defaults {
         "libhardware_legacy",
         "libhidlbase",
         "libkeystore_binder",
+        "libmeminfo",
         "libmtp",
         "libnativehelper",
         "libnativewindow",

services/core/jni/com_android_server_am_CachedAppOptimizer.cpp

@@ -17,15 +17,25 @@
 #define LOG_TAG "CachedAppOptimizer"
 //#define LOG_NDEBUG 0
 
+#include <android-base/file.h>
+#include <android-base/stringprintf.h>
+#include <android_runtime/AndroidRuntime.h>
+#include <cutils/compiler.h>
 #include <dirent.h>
+#include <jni.h>
+#include <linux/errno.h>
+#include <log/log.h>
+#include <meminfo/procmeminfo.h>
+#include <nativehelper/JNIHelp.h>
 #include <stddef.h>
 #include <stdio.h>
+#include <sys/mman.h>
 #include <sys/stat.h>
+#include <sys/syscall.h>
 #include <sys/types.h>
 #include <unistd.h>
 
-#include <android-base/stringprintf.h>
+#include <algorithm>
-#include <android-base/file.h>
 
 #include <nativehelper/JNIHelp.h>
 #include <android_runtime/AndroidRuntime.h>
@@ -35,12 +45,149 @@
 
 using android::base::StringPrintf;
 using android::base::WriteStringToFile;
+using android::meminfo::ProcMemInfo;
+using namespace android::meminfo;
+
+// This is temporarily hard-coded and should be removed once
+// bionic/libc/kernel/uapi/asm-generic/unistd.h are updated with process_madvise syscall header
+#ifndef __NR_process_madvise
+#define __NR_process_madvise 440
+#define MADV_COLD 20 /* deactivate these pages */
+#define MADV_PAGEOUT 21
+#endif
+
+#define COMPACT_ACTION_FILE_FLAG 1
+#define COMPACT_ACTION_ANON_FLAG 2
+
+using VmaToAdviseFunc = std::function<int(const Vma&)>;
 
 #define SYNC_RECEIVED_WHILE_FROZEN (1)
 #define ASYNC_RECEIVED_WHILE_FROZEN (2)
 
 namespace android {
 
+// Legacy method for compacting processes, any new code should
+// use compactProcess instead.
+static inline void compactProcessProcfs(int pid, const std::string& compactionType) {
+    std::string reclaim_path = StringPrintf("/proc/%d/reclaim", pid);
+    WriteStringToFile(compactionType, reclaim_path);
+}
+
+static int compactMemory(const std::vector<Vma>& vmas, int pid, int madviseType) {
+    // UIO_MAXIOV is currently a small value and we might have more addresses
+    // we do multiple syscalls if we exceed its maximum
+    static struct iovec vmasToKernel[UIO_MAXIOV];
+
+    int err = 0;
+
+    if (vmas.empty()) {
+        return err;
+    }
+
+    int pidfd = syscall(__NR_pidfd_open, pid, 0);
+    err = -errno;
+    if (err < 0) {
+        // Skip compaction if failed to open pidfd with any error
+        return err;
+    }
+
+    for (int iBase = 0; iBase < vmas.size(); iBase += UIO_MAXIOV) {
+        int totalVmasToKernel = std::min(UIO_MAXIOV, (int)(vmas.size() - iBase));
+        for (int iVec = 0, iVma = iBase; iVec < totalVmasToKernel; ++iVec, ++iVma) {
+            vmasToKernel[iVec].iov_base = (void*)vmas[iVma].start;
+            vmasToKernel[iVec].iov_len = vmas[iVma].end - vmas[iVma].start;
+        }
+
+        process_madvise(pidfd, vmasToKernel, totalVmasToKernel, madviseType, 0);
+        err = -errno;
+        if (CC_UNLIKELY(err == -ENOSYS)) {
+            // Syscall does not exist, skip trying more calls process_madvise
+            break;
+        }
+    }
+
+    close(pidfd);
+
+    return err;
+}
+
+static int getFilePageAdvice(const Vma& vma) {
+    if (vma.inode > 0 && !vma.is_shared) {
+        return MADV_COLD;
+    }
+    return -1;
+}
+static int getAnonPageAdvice(const Vma& vma) {
+    if (vma.inode == 0 && !vma.is_shared) {
+        return MADV_PAGEOUT;
+    }
+    return -1;
+}
+static bool getAnyPageAdvice(const Vma& vma) {
+    if (vma.inode == 0 && !vma.is_shared) {
+        return MADV_PAGEOUT;
+    }
+    return MADV_COLD;
+}
+
+// Perform a full process compaction using process_madvise syscall
+// reading all filtering VMAs and filtering pages as specified by pageFilter
+static int compactProcess(int pid, VmaToAdviseFunc vmaToAdviseFunc) {
+    ProcMemInfo meminfo(pid);
+    std::vector<Vma> pageoutVmas, coldVmas;
+    auto vmaCollectorCb = [&](Vma vma) {
+        int advice = vmaToAdviseFunc(vma);
+        switch (advice) {
+            case MADV_COLD:
+                coldVmas.push_back(vma);
+                break;
+            case MADV_PAGEOUT:
+                pageoutVmas.push_back(vma);
+                break;
+        }
+    };
+    meminfo.ForEachVma(vmaCollectorCb);
+
+    int err = compactMemory(pageoutVmas, pid, MADV_PAGEOUT);
+    if (!err) {
+        err = compactMemory(coldVmas, pid, MADV_COLD);
+    }
+    return err;
+}
+
+// Compact process using process_madvise syscall or fallback to procfs in
+// case syscall does not exist.
+static void compactProcessOrFallback(int pid, int compactionFlags) {
+    if ((compactionFlags & (COMPACT_ACTION_ANON_FLAG | COMPACT_ACTION_FILE_FLAG)) == 0) return;
+
+    bool compactAnon = compactionFlags & COMPACT_ACTION_ANON_FLAG;
+    bool compactFile = compactionFlags & COMPACT_ACTION_FILE_FLAG;
+
+    // Set when the system does not support process_madvise syscall to avoid
+    // gathering VMAs in subsequent calls prior to falling back to procfs
+    static bool shouldForceProcFs = false;
+    std::string compactionType;
+    VmaToAdviseFunc vmaToAdviseFunc;
+
+    if (compactAnon) {
+        if (compactFile) {
+            compactionType = "all";
+            vmaToAdviseFunc = getAnyPageAdvice;
+        } else {
+            compactionType = "anon";
+            vmaToAdviseFunc = getAnonPageAdvice;
+        }
+    } else {
+        compactionType = "file";
+        vmaToAdviseFunc = getFilePageAdvice;
+    }
+
+    if (shouldForceProcFs || compactProcess(pid, vmaToAdviseFunc) == -ENOSYS) {
+        shouldForceProcFs = true;
+        compactProcessProcfs(pid, compactionType);
+    }
+}
+
 // This performs per-process reclaim on all processes belonging to non-app UIDs.
 // For the most part, these are non-zygote processes like Treble HALs, but it
 // also includes zygote-derived processes that run in system UIDs, like bluetooth
@@ -74,11 +221,17 @@ static void com_android_server_am_CachedAppOptimizer_compactSystem(JNIEnv *, job
             continue;
         }
 
-        std::string reclaim_path = StringPrintf("/proc/%s/reclaim", current->d_name);
+        int pid = atoi(current->d_name);
-        WriteStringToFile(std::string("all"), reclaim_path);
+
+        compactProcessOrFallback(pid, COMPACT_ACTION_ANON_FLAG | COMPACT_ACTION_FILE_FLAG);
     }
 }
 
+static void com_android_server_am_CachedAppOptimizer_compactProcess(JNIEnv*, jobject, jint pid,
+                                                                    jint compactionFlags) {
+    compactProcessOrFallback(pid, compactionFlags);
+}
+
 static void com_android_server_am_CachedAppOptimizer_enableFreezerInternal(
         JNIEnv *env, jobject clazz, jboolean enable) {
     bool success = true;
@@ -126,14 +279,14 @@ static jint com_android_server_am_CachedAppOptimizer_getBinderFreezeInfo(JNIEnv
 }
 
 static const JNINativeMethod sMethods[] = {
-    /* name, signature, funcPtr */
+        /* name, signature, funcPtr */
-    {"compactSystem", "()V", (void*)com_android_server_am_CachedAppOptimizer_compactSystem},
+        {"compactSystem", "()V", (void*)com_android_server_am_CachedAppOptimizer_compactSystem},
-    {"enableFreezerInternal", "(Z)V",
+        {"compactProcess", "(II)V", (void*)com_android_server_am_CachedAppOptimizer_compactProcess},
-        (void*)com_android_server_am_CachedAppOptimizer_enableFreezerInternal},
+        {"enableFreezerInternal", "(Z)V",
-    {"freezeBinder", "(IZ)V", (void*)com_android_server_am_CachedAppOptimizer_freezeBinder},
+         (void*)com_android_server_am_CachedAppOptimizer_enableFreezerInternal},
-    {"getBinderFreezeInfo", "(I)I",
+        {"freezeBinder", "(IZ)V", (void*)com_android_server_am_CachedAppOptimizer_freezeBinder},
-        (void*)com_android_server_am_CachedAppOptimizer_getBinderFreezeInfo}
+        {"getBinderFreezeInfo", "(I)I",
-};
+         (void*)com_android_server_am_CachedAppOptimizer_getBinderFreezeInfo}};
 
 int register_android_server_am_CachedAppOptimizer(JNIEnv* env)
 {