BroadcastQueue: max pending, misc fixes.

To avoid paused or delayed queues from getting backed up too far,
add a "MAX_PENDING_BROADCASTS" release valve.

Ignore ANR timeouts while the system is booting.

Ignore broadcast state changes once it's already reached a terminal
state to handle racing timeout messages.

Fix isEmpty() bug where caller wanted to check for empty-and-idle.

Bug: 245771249
Test: atest FrameworksMockingServicesTests:BroadcastQueueTest
Change-Id: If5a5898bbfc0005e1c33a31aaaf05c8eab7f90fb

services/core/java/com/android/server/am/BroadcastConstants.java

@@ -136,6 +136,14 @@ public class BroadcastConstants {
     public int MAX_RUNNING_ACTIVE_BROADCASTS = DEFAULT_MAX_RUNNING_ACTIVE_BROADCASTS;
     private static final int DEFAULT_MAX_RUNNING_ACTIVE_BROADCASTS = 16;
 
+    /**
+     * For {@link BroadcastQueueModernImpl}: Maximum number of pending
+     * broadcasts to hold for a process before we ignore any delays that policy
+     * might have applied to that process.
+     */
+    public int MAX_PENDING_BROADCASTS = DEFAULT_MAX_PENDING_BROADCASTS;
+    private static final int DEFAULT_MAX_PENDING_BROADCASTS = 256;
+
     /**
      * For {@link BroadcastQueueModernImpl}: Default delay to apply to normal
      * broadcasts, giving a chance for debouncing of rapidly changing events.
@@ -217,6 +225,8 @@ public class BroadcastConstants {
                 DEFAULT_MAX_RUNNING_PROCESS_QUEUES);
         MAX_RUNNING_ACTIVE_BROADCASTS = properties.getInt("bcast_max_running_active_broadcasts",
                 DEFAULT_MAX_RUNNING_ACTIVE_BROADCASTS);
+        MAX_PENDING_BROADCASTS = properties.getInt("bcast_max_pending_broadcasts",
+                DEFAULT_MAX_PENDING_BROADCASTS);
         DELAY_NORMAL_MILLIS = properties.getLong("bcast_delay_normal_millis",
                 DEFAULT_DELAY_NORMAL_MILLIS);
         DELAY_CACHED_MILLIS = properties.getLong("bcast_delay_cached_millis",

services/core/java/com/android/server/am/BroadcastProcessQueue.java

@@ -330,7 +330,7 @@ class BroadcastProcessQueue {
     }
 
     public boolean isEmpty() {
-        return (mActive != null) && mPending.isEmpty();
+        return mPending.isEmpty();
     }
 
     public boolean isActive() {
@@ -388,6 +388,12 @@ class BroadcastProcessQueue {
             } else {
                 mRunnableAt = runnableAt + constants.DELAY_NORMAL_MILLIS;
             }
+
+            // If we have too many broadcasts pending, bypass any delays that
+            // might have been applied above to aid draining
+            if (mPending.size() >= constants.MAX_PENDING_BROADCASTS) {
+                mRunnableAt = runnableAt;
+            }
         } else {
             mRunnableAt = Long.MAX_VALUE;
         }

services/core/java/com/android/server/am/BroadcastQueueModernImpl.java

@@ -290,7 +290,7 @@ class BroadcastQueueModernImpl extends BroadcastQueue {
         }
 
         // If app isn't running, and there's nothing in the queue, clean up
-        if (queue.isEmpty() && !queue.isProcessWarm()) {
+        if (queue.isEmpty() && !queue.isActive() && !queue.isProcessWarm()) {
             removeProcessQueue(queue.processName, queue.uid);
         }
     }
@@ -578,7 +578,7 @@ class BroadcastQueueModernImpl extends BroadcastQueue {
             return;
         }
 
-        if (!r.timeoutExempt) {
+        if (mService.mProcessesReady && !r.timeoutExempt) {
             final long timeout = r.isForeground() ? mFgConstants.TIMEOUT : mBgConstants.TIMEOUT;
             mLocalHandler.sendMessageDelayed(
                     Message.obtain(mLocalHandler, MSG_DELIVERY_TIMEOUT, queue), timeout);
@@ -698,6 +698,7 @@ class BroadcastQueueModernImpl extends BroadcastQueue {
         setDeliveryState(queue, app, r, index, receiver, deliveryState);
 
         if (deliveryState == BroadcastRecord.DELIVERY_TIMEOUT) {
+            r.anrCount++;
             if (app != null && !app.isDebugging()) {
                 mService.appNotResponding(queue.app, TimeoutRecord
                         .forBroadcastReceiver("Broadcast of " + r.toShortString()));
@@ -709,7 +710,7 @@ class BroadcastQueueModernImpl extends BroadcastQueue {
         // Even if we have more broadcasts, if we've made reasonable progress
         // and someone else is waiting, retire ourselves to avoid starvation
         final boolean shouldRetire = (mRunnableHead != null)
-                && (queue.getActiveCountSinceIdle() > mConstants.MAX_RUNNING_ACTIVE_BROADCASTS);
+                && (queue.getActiveCountSinceIdle() >= mConstants.MAX_RUNNING_ACTIVE_BROADCASTS);
 
         if (queue.isRunnable() && queue.isProcessWarm() && !shouldRetire) {
             // We're on a roll; move onto the next broadcast for this process
@@ -749,7 +750,11 @@ class BroadcastQueueModernImpl extends BroadcastQueue {
                     + deliveryStateToString(newDeliveryState));
         }
 
-        r.setDeliveryState(index, newDeliveryState);
+        // Only apply state when we haven't already reached a terminal state;
+        // this is how we ignore racing timeout messages
+        if (!isDeliveryStateTerminal(oldDeliveryState)) {
+            r.setDeliveryState(index, newDeliveryState);
+        }
 
         // Emit any relevant tracing results when we're changing the delivery
         // state as part of running from a queue