Cleanup: remove old memory atoms

Memory stats has been migrated to ProcessMemorySnapshot. This CL cleans
up the old atoms:
- remove telemetry-specific additions to ProcessMemoryState
- remove NativeProcessMemoryState atom
- move reading process cmdline under stats

Bug: 140986627
Test: atest MemoryStatUtilTest
Test: atest ProcfsMemoryUtilTest
Test: atest UidAtomTests#testProcessMemoryState
Change-Id: I5339577184d17e5cef6d657cf9b08a2fb8e75fca
This commit is contained in:
Rafal Slawik
2019-09-25 19:53:01 +01:00
parent 64247a1661
commit 203c3dbde9
7 changed files with 91 additions and 264 deletions

View File

@@ -378,7 +378,6 @@ message Atom {
PowerProfile power_profile = 10033;
ProcStatsPkgProc proc_stats_pkg_proc = 10034;
ProcessCpuTime process_cpu_time = 10035;
NativeProcessMemoryState native_process_memory_state = 10036;
CpuTimePerThreadFreq cpu_time_per_thread_freq = 10037;
OnDevicePowerMeasurement on_device_power_measurement = 10038;
DeviceCalculatedPowerUse device_calculated_power_use = 10039;
@@ -412,6 +411,8 @@ message Atom {
// DO NOT USE field numbers above 100,000 in AOSP.
// Field numbers 100,000 - 199,999 are reserved for non-AOSP (e.g. OEMs) to use.
// Field numbers 200,000 and above are reserved for future use; do not use them at all.
reserved 10036;
}
/**
@@ -4019,8 +4020,8 @@ message ProcessMemoryState {
optional int64 page_major_fault = 5;
// RSS
// Value is read from /proc/PID/status. Or from memory.stat, field
// total_rss if per-app memory cgroups are enabled.
// Value is read from memory.stat, field total_rss if per-app memory
// cgroups are enabled. Otherwise, value from /proc/pid/stat.
optional int64 rss_in_bytes = 6;
// CACHE
@@ -4030,56 +4031,17 @@ message ProcessMemoryState {
// SWAP
// Value is read from memory.stat, field total_swap if per-app memory
// cgroups are enabled. Otherwise, VmSwap from /proc/PID/status.
// cgroups are enabled. Otherwise, 0.
optional int64 swap_in_bytes = 8;
// Deprecated: use ProcessMemoryHighWaterMark atom instead. Always 0.
// Deprecated: use ProcessMemoryHighWaterMark atom instead. Always -1.
optional int64 rss_high_watermark_in_bytes = 9 [deprecated = true];
// Elapsed real time when the process started.
// Value is read from /proc/PID/stat, field 22. 0 if read from per-app memory cgroups.
optional int64 start_time_nanos = 10;
// Deprecated: use ProcessMemorySnapshot atom instead. Always -1.
optional int64 start_time_nanos = 10 [deprecated = true];
// Anonymous page size plus swap size. Values are read from /proc/PID/status.
optional int32 anon_rss_and_swap_in_kilobytes = 11;
}
/*
* Logs the memory stats for a native process (from procfs).
*
* Pulled from StatsCompanionService for selected native processes.
*/
message NativeProcessMemoryState {
// The uid if available. -1 means not available.
optional int32 uid = 1 [(is_uid) = true];
// The process name.
// Value read from /proc/PID/cmdline.
optional string process_name = 2;
// # of page-faults
optional int64 page_fault = 3;
// # of major page-faults
optional int64 page_major_fault = 4;
// RSS
// Value read from /proc/PID/status.
optional int64 rss_in_bytes = 5;
// Deprecated: use ProcessMemoryHighWaterMark atom instead. Always 0.
optional int64 rss_high_watermark_in_bytes = 6 [deprecated = true];
// Elapsed real time when the process started.
// Value is read from /proc/PID/stat, field 22.
optional int64 start_time_nanos = 7;
// SWAP
// Value read from /proc/PID/status, field VmSwap.
optional int64 swap_in_bytes = 8;
// Anonymous page size plus swap size. Values are read from /proc/PID/status.
optional int32 anon_rss_and_swap_in_kilobytes = 9;
// Deprecated: use ProcessMemorySnapshot atom instead. Always -1.
optional int32 anon_rss_and_swap_in_kilobytes = 11 [deprecated = true];
}
/*

View File

@@ -145,16 +145,11 @@ std::map<int, PullAtomInfo> StatsPullerManager::kAllPullAtomInfo = {
{.puller = new ResourceHealthManagerPuller(android::util::BATTERY_CYCLE_COUNT)}},
// process_memory_state
{android::util::PROCESS_MEMORY_STATE,
{.additiveFields = {4, 5, 6, 7, 8, 9},
{.additiveFields = {4, 5, 6, 7, 8},
.puller = new StatsCompanionServicePuller(android::util::PROCESS_MEMORY_STATE)}},
// native_process_memory_state
{android::util::NATIVE_PROCESS_MEMORY_STATE,
{.additiveFields = {3, 4, 5, 6, 8},
.puller = new StatsCompanionServicePuller(android::util::NATIVE_PROCESS_MEMORY_STATE)}},
// process_memory_high_water_mark
{android::util::PROCESS_MEMORY_HIGH_WATER_MARK,
{.additiveFields = {3},
.puller =
{.puller =
new StatsCompanionServicePuller(android::util::PROCESS_MEMORY_HIGH_WATER_MARK)}},
// process_memory_snapshot
{android::util::PROCESS_MEMORY_SNAPSHOT,

View File

@@ -39,8 +39,7 @@ import java.util.regex.Pattern;
* Static utility methods related to {@link MemoryStat}.
*/
public final class MemoryStatUtil {
static final int BYTES_IN_KILOBYTE = 1024;
static final long JIFFY_NANOS = 1_000_000_000 / Os.sysconf(OsConstants._SC_CLK_TCK);
static final int PAGE_SIZE = (int) Os.sysconf(OsConstants._SC_PAGESIZE);
private static final String TAG = TAG_WITH_CLASS_NAME ? "MemoryStatUtil" : TAG_AM;
@@ -52,10 +51,6 @@ public final class MemoryStatUtil {
private static final String MEMORY_STAT_FILE_FMT = "/dev/memcg/apps/uid_%d/pid_%d/memory.stat";
/** Path to procfs stat file for logging app start memory state */
private static final String PROC_STAT_FILE_FMT = "/proc/%d/stat";
/** Path to procfs status file for logging app memory state */
private static final String PROC_STATUS_FILE_FMT = "/proc/%d/status";
/** Path to procfs cmdline file. Used with pid: /proc/pid/cmdline. */
private static final String PROC_CMDLINE_FILE_FMT = "/proc/%d/cmdline";
private static final Pattern PGFAULT = Pattern.compile("total_pgfault (\\d+)");
private static final Pattern PGMAJFAULT = Pattern.compile("total_pgmajfault (\\d+)");
@@ -63,16 +58,9 @@ public final class MemoryStatUtil {
private static final Pattern CACHE_IN_BYTES = Pattern.compile("total_cache (\\d+)");
private static final Pattern SWAP_IN_BYTES = Pattern.compile("total_swap (\\d+)");
private static final Pattern PROCFS_RSS_IN_KILOBYTES =
Pattern.compile("VmRSS:\\s*(\\d+)\\s*kB");
private static final Pattern PROCFS_ANON_RSS_IN_KILOBYTES =
Pattern.compile("RssAnon:\\s*(\\d+)\\s*kB");
private static final Pattern PROCFS_SWAP_IN_KILOBYTES =
Pattern.compile("VmSwap:\\s*(\\d+)\\s*kB");
private static final int PGFAULT_INDEX = 9;
private static final int PGMAJFAULT_INDEX = 11;
private static final int START_TIME_INDEX = 21;
private static final int RSS_IN_PAGES_INDEX = 23;
private MemoryStatUtil() {}
@@ -106,19 +94,7 @@ public final class MemoryStatUtil {
@Nullable
public static MemoryStat readMemoryStatFromProcfs(int pid) {
final String statPath = String.format(Locale.US, PROC_STAT_FILE_FMT, pid);
final String statusPath = String.format(Locale.US, PROC_STATUS_FILE_FMT, pid);
return parseMemoryStatFromProcfs(readFileContents(statPath), readFileContents(statusPath));
}
/**
* Reads cmdline of a process from procfs.
*
* Returns content of /proc/pid/cmdline (e.g. /system/bin/statsd) or an empty string
* if the file is not available.
*/
public static String readCmdlineFromProcfs(int pid) {
final String path = String.format(Locale.US, PROC_CMDLINE_FILE_FMT, pid);
return parseCmdlineFromProcfs(readFileContents(path));
return parseMemoryStatFromProcfs(readFileContents(statPath));
}
private static String readFileContents(String path) {
@@ -160,31 +136,19 @@ public final class MemoryStatUtil {
*/
@VisibleForTesting
@Nullable
static MemoryStat parseMemoryStatFromProcfs(
String procStatContents, String procStatusContents) {
static MemoryStat parseMemoryStatFromProcfs(String procStatContents) {
if (procStatContents == null || procStatContents.isEmpty()) {
return null;
}
if (procStatusContents == null || procStatusContents.isEmpty()) {
return null;
}
final String[] splits = procStatContents.split(" ");
if (splits.length < 24) {
return null;
}
try {
final MemoryStat memoryStat = new MemoryStat();
memoryStat.pgfault = Long.parseLong(splits[PGFAULT_INDEX]);
memoryStat.pgmajfault = Long.parseLong(splits[PGMAJFAULT_INDEX]);
memoryStat.rssInBytes =
tryParseLong(PROCFS_RSS_IN_KILOBYTES, procStatusContents) * BYTES_IN_KILOBYTE;
memoryStat.anonRssInBytes =
tryParseLong(PROCFS_ANON_RSS_IN_KILOBYTES, procStatusContents) * BYTES_IN_KILOBYTE;
memoryStat.swapInBytes =
tryParseLong(PROCFS_SWAP_IN_KILOBYTES, procStatusContents) * BYTES_IN_KILOBYTE;
memoryStat.startTimeNanos = Long.parseLong(splits[START_TIME_INDEX]) * JIFFY_NANOS;
memoryStat.rssInBytes = Long.parseLong(splits[RSS_IN_PAGES_INDEX]) * PAGE_SIZE;
return memoryStat;
} catch (NumberFormatException e) {
Slog.e(TAG, "Failed to parse value", e);
@@ -192,23 +156,6 @@ public final class MemoryStatUtil {
}
}
/**
* Parses cmdline out of the contents of the /proc/pid/cmdline file in procfs.
*
* Parsing is required to strip anything after first null byte.
*/
@VisibleForTesting
static String parseCmdlineFromProcfs(String cmdline) {
if (cmdline == null) {
return "";
}
int firstNullByte = cmdline.indexOf("\0");
if (firstNullByte == -1) {
return cmdline;
}
return cmdline.substring(0, firstNullByte);
}
/**
* Returns whether per-app memcg is available on device.
*/
@@ -237,13 +184,9 @@ public final class MemoryStatUtil {
public long pgmajfault;
/** For memcg stats, the anon rss + swap cache size. Otherwise total RSS. */
public long rssInBytes;
/** Number of bytes of the anonymous RSS. Only present for non-memcg stats. */
public long anonRssInBytes;
/** Number of bytes of page cache memory. Only present for memcg stats. */
public long cacheInBytes;
/** Number of bytes of swap usage */
public long swapInBytes;
/** Device time when the processes started. */
public long startTimeNanos;
}
}

View File

@@ -72,6 +72,30 @@ final class ProcfsMemoryUtil {
return null;
}
/**
* Reads cmdline of a process from procfs.
*
* Returns content of /proc/pid/cmdline (e.g. /system/bin/statsd) or an empty string
* if the file is not available.
*/
public static String readCmdlineFromProcfs(int pid) {
return parseCmdline(readFile("/proc/" + pid + "/cmdline"));
}
/**
* Parses cmdline out of the contents of the /proc/pid/cmdline file in procfs.
*
* Parsing is required to strip anything after the first null byte.
*/
@VisibleForTesting
static String parseCmdline(String contents) {
int firstNullByte = contents.indexOf("\0");
if (firstNullByte == -1) {
return contents;
}
return contents.substring(0, firstNullByte);
}
private static String readFile(String path) {
try {
final File file = new File(path);

View File

@@ -24,11 +24,10 @@ import static android.os.storage.VolumeInfo.TYPE_PRIVATE;
import static android.os.storage.VolumeInfo.TYPE_PUBLIC;
import static com.android.internal.util.Preconditions.checkNotNull;
import static com.android.server.am.MemoryStatUtil.readCmdlineFromProcfs;
import static com.android.server.am.MemoryStatUtil.readMemoryStatFromFilesystem;
import static com.android.server.am.MemoryStatUtil.readMemoryStatFromProcfs;
import static com.android.server.stats.IonMemoryUtil.readProcessSystemIonHeapSizesFromDebugfs;
import static com.android.server.stats.IonMemoryUtil.readSystemIonHeapSizeFromDebugfs;
import static com.android.server.stats.ProcfsMemoryUtil.readCmdlineFromProcfs;
import static com.android.server.stats.ProcfsMemoryUtil.readMemorySnapshotFromProcfs;
import android.annotation.NonNull;
@@ -1193,48 +1192,13 @@ public class StatsCompanionService extends IStatsCompanionService.Stub {
e.writeLong(memoryStat.rssInBytes);
e.writeLong(memoryStat.cacheInBytes);
e.writeLong(memoryStat.swapInBytes);
e.writeLong(0); // unused
e.writeLong(memoryStat.startTimeNanos);
e.writeInt(anonAndSwapInKilobytes(memoryStat));
e.writeLong(-1); // unused
e.writeLong(-1); // unused
e.writeInt(-1); // unsed
pulledData.add(e);
}
}
private void pullNativeProcessMemoryState(
int tagId, long elapsedNanos, long wallClockNanos,
List<StatsLogEventWrapper> pulledData) {
int[] pids = getPidsForCommands(MEMORY_INTERESTING_NATIVE_PROCESSES);
for (int pid : pids) {
String processName = readCmdlineFromProcfs(pid);
MemoryStat memoryStat = readMemoryStatFromProcfs(pid);
if (memoryStat == null) {
continue;
}
int uid = getUidForPid(pid);
// Sometimes we get here a process that is not included in the whitelist. It comes
// from forking the zygote for an app. We can ignore that sample because this process
// is collected by ProcessMemoryState.
if (isAppUid(uid)) {
continue;
}
StatsLogEventWrapper e = new StatsLogEventWrapper(tagId, elapsedNanos, wallClockNanos);
e.writeInt(uid);
e.writeString(processName);
e.writeLong(memoryStat.pgfault);
e.writeLong(memoryStat.pgmajfault);
e.writeLong(memoryStat.rssInBytes);
e.writeLong(0); // unused
e.writeLong(memoryStat.startTimeNanos);
e.writeLong(memoryStat.swapInBytes);
e.writeInt(anonAndSwapInKilobytes(memoryStat));
pulledData.add(e);
}
}
private static int anonAndSwapInKilobytes(MemoryStat memoryStat) {
return (int) ((memoryStat.anonRssInBytes + memoryStat.swapInBytes) / 1024);
}
private void pullProcessMemoryHighWaterMark(
int tagId, long elapsedNanos, long wallClockNanos,
List<StatsLogEventWrapper> pulledData) {
@@ -2405,10 +2369,6 @@ public class StatsCompanionService extends IStatsCompanionService.Stub {
pullProcessMemoryState(tagId, elapsedNanos, wallClockNanos, ret);
break;
}
case StatsLog.NATIVE_PROCESS_MEMORY_STATE: {
pullNativeProcessMemoryState(tagId, elapsedNanos, wallClockNanos, ret);
break;
}
case StatsLog.PROCESS_MEMORY_HIGH_WATER_MARK: {
pullProcessMemoryHighWaterMark(tagId, elapsedNanos, wallClockNanos, ret);
break;

View File

@@ -16,10 +16,8 @@
package com.android.server.am;
import static com.android.server.am.MemoryStatUtil.BYTES_IN_KILOBYTE;
import static com.android.server.am.MemoryStatUtil.JIFFY_NANOS;
import static com.android.server.am.MemoryStatUtil.MemoryStat;
import static com.android.server.am.MemoryStatUtil.parseCmdlineFromProcfs;
import static com.android.server.am.MemoryStatUtil.PAGE_SIZE;
import static com.android.server.am.MemoryStatUtil.parseMemoryStatFromMemcg;
import static com.android.server.am.MemoryStatUtil.parseMemoryStatFromProcfs;
@@ -30,7 +28,6 @@ import androidx.test.filters.SmallTest;
import org.junit.Test;
import java.io.ByteArrayOutputStream;
import java.util.Collections;
/**
@@ -99,7 +96,7 @@ public class MemoryStatUtilTest {
"0",
"2222", // this in start time (in ticks per second)
"1257177088",
"3",
"3", // this is RSS in pages
"4294967295",
"2936971264",
"2936991289",
@@ -129,53 +126,6 @@ public class MemoryStatUtilTest {
"3198889956",
"0");
private static final String PROC_STATUS_CONTENTS = "Name:\tandroid.youtube\n"
+ "State:\tS (sleeping)\n"
+ "Tgid:\t12088\n"
+ "Pid:\t12088\n"
+ "PPid:\t723\n"
+ "TracerPid:\t0\n"
+ "Uid:\t10083\t10083\t10083\t10083\n"
+ "Gid:\t10083\t10083\t10083\t10083\n"
+ "Ngid:\t0\n"
+ "FDSize:\t128\n"
+ "Groups:\t3003 9997 20083 50083 \n"
+ "VmPeak:\t 4546844 kB\n"
+ "VmSize:\t 4542636 kB\n"
+ "VmLck:\t 0 kB\n"
+ "VmPin:\t 0 kB\n"
+ "VmHWM:\t 137668 kB\n" // RSS high-water mark
+ "VmRSS:\t 126776 kB\n" // RSS
+ "RssAnon:\t 37860 kB\n"
+ "RssFile:\t 88764 kB\n"
+ "RssShmem:\t 152 kB\n"
+ "VmData:\t 4125112 kB\n"
+ "VmStk:\t 8192 kB\n"
+ "VmExe:\t 24 kB\n"
+ "VmLib:\t 102432 kB\n"
+ "VmPTE:\t 1300 kB\n"
+ "VmPMD:\t 36 kB\n"
+ "VmSwap:\t 22 kB\n" // Swap
+ "Threads:\t95\n"
+ "SigQ:\t0/13641\n"
+ "SigPnd:\t0000000000000000\n"
+ "ShdPnd:\t0000000000000000\n"
+ "SigBlk:\t0000000000001204\n"
+ "SigIgn:\t0000000000000001\n"
+ "SigCgt:\t00000006400084f8\n"
+ "CapInh:\t0000000000000000\n"
+ "CapPrm:\t0000000000000000\n"
+ "CapEff:\t0000000000000000\n"
+ "CapBnd:\t0000000000000000\n"
+ "CapAmb:\t0000000000000000\n"
+ "Seccomp:\t2\n"
+ "Cpus_allowed:\tff\n"
+ "Cpus_allowed_list:\t0-7\n"
+ "Mems_allowed:\t1\n"
+ "Mems_allowed_list:\t0\n"
+ "voluntary_ctxt_switches:\t903\n"
+ "nonvoluntary_ctxt_switches:\t104\n";
@Test
public void testParseMemoryStatFromMemcg_parsesCorrectValues() {
MemoryStat stat = parseMemoryStatFromMemcg(MEMORY_STAT_CONTENTS);
@@ -197,71 +147,26 @@ public class MemoryStatUtilTest {
@Test
public void testParseMemoryStatFromProcfs_parsesCorrectValues() {
MemoryStat stat = parseMemoryStatFromProcfs(PROC_STAT_CONTENTS, PROC_STATUS_CONTENTS);
MemoryStat stat = parseMemoryStatFromProcfs(PROC_STAT_CONTENTS);
assertEquals(1, stat.pgfault);
assertEquals(2, stat.pgmajfault);
assertEquals(126776 * BYTES_IN_KILOBYTE, stat.rssInBytes);
assertEquals(3 * PAGE_SIZE, stat.rssInBytes);
assertEquals(0, stat.cacheInBytes);
assertEquals(22 * BYTES_IN_KILOBYTE, stat.swapInBytes);
assertEquals(2222 * JIFFY_NANOS, stat.startTimeNanos);
assertEquals(37860 * BYTES_IN_KILOBYTE, stat.anonRssInBytes);
assertEquals(0, stat.swapInBytes);
}
@Test
public void testParseMemoryStatFromProcfs_emptyContents() {
MemoryStat stat = parseMemoryStatFromProcfs("", PROC_STATUS_CONTENTS);
MemoryStat stat = parseMemoryStatFromProcfs("");
assertNull(stat);
stat = parseMemoryStatFromProcfs(null, PROC_STATUS_CONTENTS);
assertNull(stat);
stat = parseMemoryStatFromProcfs(PROC_STAT_CONTENTS, "");
assertNull(stat);
stat = parseMemoryStatFromProcfs(PROC_STAT_CONTENTS, null);
stat = parseMemoryStatFromProcfs(null);
assertNull(stat);
}
@Test
public void testParseMemoryStatFromProcfs_invalidValue() {
String contents = String.join(" ", Collections.nCopies(24, "memory"));
assertNull(parseMemoryStatFromProcfs(contents, PROC_STATUS_CONTENTS));
}
@Test
public void testParseCmdlineFromProcfs_invalidValue() {
byte[] nothing = new byte[] {0x00, 0x74, 0x65, 0x73, 0x74}; // \0test
assertEquals("", parseCmdlineFromProcfs(bytesToString(nothing)));
}
@Test
public void testParseCmdlineFromProcfs_correctValue_noNullBytes() {
assertEquals("com.google.app", parseCmdlineFromProcfs("com.google.app"));
}
@Test
public void testParseCmdlineFromProcfs_correctValue_withNullBytes() {
byte[] trailing = new byte[] {0x74, 0x65, 0x73, 0x74, 0x00, 0x00, 0x00}; // test\0\0\0
assertEquals("test", parseCmdlineFromProcfs(bytesToString(trailing)));
// test\0\0test
byte[] inside = new byte[] {0x74, 0x65, 0x73, 0x74, 0x00, 0x00, 0x74, 0x65, 0x73, 0x74};
assertEquals("test", parseCmdlineFromProcfs(bytesToString(trailing)));
}
@Test
public void testParseCmdlineFromProcfs_emptyContents() {
assertEquals("", parseCmdlineFromProcfs(""));
assertEquals("", parseCmdlineFromProcfs(null));
}
private static String bytesToString(byte[] bytes) {
ByteArrayOutputStream output = new ByteArrayOutputStream();
output.write(bytes, 0, bytes.length);
return output.toString();
assertNull(parseMemoryStatFromProcfs(contents));
}
}

View File

@@ -15,6 +15,7 @@
*/
package com.android.server.stats;
import static com.android.server.stats.ProcfsMemoryUtil.parseCmdline;
import static com.android.server.stats.ProcfsMemoryUtil.parseMemorySnapshotFromStatus;
import static com.google.common.truth.Truth.assertThat;
@@ -25,6 +26,8 @@ import com.android.server.stats.ProcfsMemoryUtil.MemorySnapshot;
import org.junit.Test;
import java.io.ByteArrayOutputStream;
/**
* Build/Install/Run:
* atest FrameworksServicesTests:ProcfsMemoryUtilTest
@@ -100,4 +103,39 @@ public class ProcfsMemoryUtilTest {
MemorySnapshot snapshot = parseMemorySnapshotFromStatus("");
assertThat(snapshot).isNull();
}
@Test
public void testParseCmdline_invalidValue() {
byte[] nothing = new byte[] {0x00, 0x74, 0x65, 0x73, 0x74}; // \0test
assertThat(parseCmdline(bytesToString(nothing))).isEmpty();
}
@Test
public void testParseCmdline_correctValue_noNullBytes() {
assertThat(parseCmdline("com.google.app")).isEqualTo("com.google.app");
}
@Test
public void testParseCmdline_correctValue_withNullBytes() {
byte[] trailing = new byte[] {0x74, 0x65, 0x73, 0x74, 0x00, 0x00, 0x00}; // test\0\0\0
assertThat(parseCmdline(bytesToString(trailing))).isEqualTo("test");
// test\0\0test
byte[] inside = new byte[] {0x74, 0x65, 0x73, 0x74, 0x00, 0x00, 0x74, 0x65, 0x73, 0x74};
assertThat(parseCmdline(bytesToString(trailing))).isEqualTo("test");
}
@Test
public void testParseCmdline_emptyContents() {
assertThat(parseCmdline("")).isEmpty();
}
private static String bytesToString(byte[] bytes) {
ByteArrayOutputStream output = new ByteArrayOutputStream();
output.write(bytes, 0, bytes.length);
return output.toString();
}
}