Rate limit dropbox entries in ActivityManagerService per process and add

a test for the rate limiting.

Design doc: go/apc-rate-limit-dropbox

Bug: 225173288
Test: atest DropboxRateLimiterTest
Change-Id: I08ccc151a3af867360ec8376725679bcfb92f4f6

services/core/java/com/android/server/am/ActivityManagerService.java

@@ -309,7 +309,6 @@ import android.sysprop.InitProperties;
 import android.sysprop.VoldProperties;
 import android.telephony.TelephonyManager;
 import android.text.TextUtils;
-import android.text.format.DateUtils;
 import android.text.style.SuggestionSpan;
 import android.util.ArrayMap;
 import android.util.ArraySet;
@@ -8618,7 +8617,7 @@ public class ActivityManagerService extends IActivityManager.Stub
         }
     }
 
-    private final ArrayMap<String, long[]> mErrorClusterRecords = new ArrayMap<>();
+    private final DropboxRateLimiter mDropboxRateLimiter = new DropboxRateLimiter();
 
     /**
      * Write a description of an error (crash, WTF, ANR) to the drop box.
@@ -8653,22 +8652,8 @@ public class ActivityManagerService extends IActivityManager.Stub
         final String dropboxTag = processClass(process) + "_" + eventType;
         if (dbox == null || !dbox.isTagEnabled(dropboxTag)) return;
 
-        // Rate-limit how often we're willing to do the heavy lifting below to
+        // Check if we should rate limit and abort early if needed.
-        // collect and record logs; currently 5 logs per 10 second period per eventType.
+        if (mDropboxRateLimiter.shouldRateLimit(eventType, processName)) return;
-        final long now = SystemClock.elapsedRealtime();
-        synchronized (mErrorClusterRecords) {
-            long[] errRecord = mErrorClusterRecords.get(eventType);
-            if (errRecord == null) {
-                errRecord = new long[2]; // [0]: startTime, [1]: count
-                mErrorClusterRecords.put(eventType, errRecord);
-            }
-            if (now - errRecord[0] > 10 * DateUtils.SECOND_IN_MILLIS) {
-                errRecord[0] = now;
-                errRecord[1] = 1L;
-            } else {
-                if (errRecord[1]++ >= 5) return;
-            }
-        }
 
         final StringBuilder sb = new StringBuilder(1024);
         appendDropBoxProcessHeaders(process, processName, sb);

services/core/java/com/android/server/am/DropboxRateLimiter.java

@@ -0,0 +1,125 @@
+/*
+ * Copyright (C) 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.server.am;
+
+import android.os.SystemClock;
+import android.text.format.DateUtils;
+import android.util.ArrayMap;
+
+import com.android.internal.annotations.GuardedBy;
+
+/** Rate limiter for adding errors into dropbox. */
+public class DropboxRateLimiter {
+    private static final long RATE_LIMIT_BUFFER_EXPIRY = 15 * DateUtils.SECOND_IN_MILLIS;
+    private static final long RATE_LIMIT_BUFFER_DURATION = 10 * DateUtils.SECOND_IN_MILLIS;
+    private static final int RATE_LIMIT_ALLOWED_ENTRIES = 5;
+
+    @GuardedBy("mErrorClusterRecords")
+    private final ArrayMap<String, ErrorRecord> mErrorClusterRecords = new ArrayMap<>();
+    private final Clock mClock;
+
+    private long mLastMapCleanUp = 0L;
+
+    public DropboxRateLimiter() {
+        this(new DefaultClock());
+    }
+
+    public DropboxRateLimiter(Clock clock) {
+        mClock = clock;
+    }
+
+    /** The interface clock to use for tracking the time elapsed. */
+    public interface Clock {
+        /** How long in millis has passed since the device came online. */
+        long uptimeMillis();
+    }
+
+    /** Determines whether dropbox entries of a specific tag and process should be rate limited. */
+    public boolean shouldRateLimit(String eventType, String processName) {
+        // Rate-limit how often we're willing to do the heavy lifting to collect and record logs.
+        final long now = mClock.uptimeMillis();
+        synchronized (mErrorClusterRecords) {
+            // Remove expired records if enough time has passed since the last cleanup.
+            maybeRemoveExpiredRecords(now);
+
+            ErrorRecord errRecord = mErrorClusterRecords.get(errorKey(eventType, processName));
+            if (errRecord == null) {
+                errRecord = new ErrorRecord(now, 1);
+                mErrorClusterRecords.put(errorKey(eventType, processName), errRecord);
+            } else if (now - errRecord.getStartTime() > RATE_LIMIT_BUFFER_DURATION) {
+                errRecord.setStartTime(now);
+                errRecord.setCount(1);
+            } else {
+                errRecord.incrementCount();
+                if (errRecord.getCount() > RATE_LIMIT_ALLOWED_ENTRIES) return true;
+            }
+        }
+        return false;
+    }
+
+    private void maybeRemoveExpiredRecords(long now) {
+        if (now - mLastMapCleanUp <= RATE_LIMIT_BUFFER_EXPIRY) return;
+
+        for (int i = mErrorClusterRecords.size() - 1; i >= 0; i--) {
+            if (now - mErrorClusterRecords.valueAt(i).getStartTime() > RATE_LIMIT_BUFFER_EXPIRY) {
+                mErrorClusterRecords.removeAt(i);
+            }
+        }
+
+        mLastMapCleanUp = now;
+    }
+
+    String errorKey(String eventType, String processName) {
+        return eventType + processName;
+    }
+
+    private class ErrorRecord {
+        long mStartTime;
+        int mCount;
+
+        ErrorRecord(long startTime, int count) {
+            mStartTime = startTime;
+            mCount = count;
+        }
+
+        public void setStartTime(long startTime) {
+            mStartTime = startTime;
+        }
+
+        public void setCount(int count) {
+            mCount = count;
+        }
+
+        public void incrementCount() {
+            mCount++;
+        }
+
+        public long getStartTime() {
+            return mStartTime;
+        }
+
+        public int getCount() {
+            return mCount;
+        }
+    }
+
+    private static class DefaultClock implements Clock {
+        public long uptimeMillis() {
+            return SystemClock.uptimeMillis();
+        }
+    }
+}

services/tests/servicestests/src/com/android/server/am/DropboxRateLimiterTest.java

@@ -0,0 +1,87 @@
+/*
+ * Copyright (C) 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.server.am;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import android.os.SystemClock;
+
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test class for {@link DropboxRateLimiter}.
+ *
+ * Build/Install/Run:
+ *  atest DropboxRateLimiterTest
+ */
+public class DropboxRateLimiterTest {
+    private DropboxRateLimiter mRateLimiter;
+    private TestClock mClock;
+
+    @Before
+    public void setUp() {
+        mClock = new TestClock();
+        mRateLimiter = new DropboxRateLimiter(mClock);
+    }
+
+    @Test
+    public void testMultipleProcesses() {
+        // The first 5 entries should not be rate limited.
+        assertFalse(mRateLimiter.shouldRateLimit("tag", "process"));
+        assertFalse(mRateLimiter.shouldRateLimit("tag", "process"));
+        assertFalse(mRateLimiter.shouldRateLimit("tag", "process"));
+        assertFalse(mRateLimiter.shouldRateLimit("tag", "process"));
+        assertFalse(mRateLimiter.shouldRateLimit("tag", "process"));
+        // Different processes and tags should not get rate limited either.
+        assertFalse(mRateLimiter.shouldRateLimit("tag", "process2"));
+        assertFalse(mRateLimiter.shouldRateLimit("tag2", "process"));
+        // The 6th entry of the same process should be rate limited.
+        assertTrue(mRateLimiter.shouldRateLimit("tag", "process"));
+    }
+
+    @Test
+    public void testBufferClearing() throws Exception {
+        // The first 5 entries should not be rate limited.
+        assertFalse(mRateLimiter.shouldRateLimit("tag", "process"));
+        assertFalse(mRateLimiter.shouldRateLimit("tag", "process"));
+        assertFalse(mRateLimiter.shouldRateLimit("tag", "process"));
+        assertFalse(mRateLimiter.shouldRateLimit("tag", "process"));
+        assertFalse(mRateLimiter.shouldRateLimit("tag", "process"));
+        // The 6th entry of the same process should be rate limited.
+        assertTrue(mRateLimiter.shouldRateLimit("tag", "process"));
+
+        // After 11 seconds there should be nothing left in the buffer and the same type of entry
+        // should not get rate limited anymore.
+        mClock.setOffsetMillis(11000);
+
+        assertFalse(mRateLimiter.shouldRateLimit("tag", "process"));
+    }
+
+    private static class TestClock implements DropboxRateLimiter.Clock {
+        long mOffsetMillis = 0L;
+
+        public long uptimeMillis() {
+            return mOffsetMillis + SystemClock.uptimeMillis();
+        }
+
+        public void setOffsetMillis(long millis) {
+            mOffsetMillis = millis;
+        }
+    }
+}