diff --git a/src/com/android/settings/datausage/DataSaverBackend.java b/src/com/android/settings/datausage/DataSaverBackend.java
index 513576521e8..4f731c71cde 100644
--- a/src/com/android/settings/datausage/DataSaverBackend.java
+++ b/src/com/android/settings/datausage/DataSaverBackend.java
@@ -82,8 +82,7 @@ public class DataSaverBackend {
     }
 
     public void setIsAllowlisted(int uid, String packageName, boolean allowlisted) {
-        final int policy = allowlisted ? POLICY_ALLOW_METERED_BACKGROUND : POLICY_NONE;
-        mUidPolicies.put(uid, policy);
+        setUidPolicyFlag(uid, POLICY_ALLOW_METERED_BACKGROUND, allowlisted);
         if (allowlisted) {
             mPolicyManager.addUidPolicy(uid, POLICY_ALLOW_METERED_BACKGROUND);
             mMetricsFeatureProvider.action(
@@ -96,17 +95,14 @@ public class DataSaverBackend {
 
     public boolean isAllowlisted(int uid) {
         loadAllowlist();
-        return mUidPolicies.get(uid, POLICY_NONE) == POLICY_ALLOW_METERED_BACKGROUND;
+        return isUidPolicyFlagSet(uid, POLICY_ALLOW_METERED_BACKGROUND);
     }
 
     private void loadAllowlist() {
         if (mAllowlistInitialized) {
             return;
         }
-
-        for (int uid : mPolicyManager.getUidsWithPolicy(POLICY_ALLOW_METERED_BACKGROUND)) {
-            mUidPolicies.put(uid, POLICY_ALLOW_METERED_BACKGROUND);
-        }
+        loadUidPolicies(POLICY_ALLOW_METERED_BACKGROUND);
         mAllowlistInitialized = true;
     }
 
@@ -115,8 +111,7 @@ public class DataSaverBackend {
     }
 
     public void setIsDenylisted(int uid, String packageName, boolean denylisted) {
-        final int policy = denylisted ? POLICY_REJECT_METERED_BACKGROUND : POLICY_NONE;
-        mUidPolicies.put(uid, policy);
+        setUidPolicyFlag(uid, POLICY_REJECT_METERED_BACKGROUND, denylisted);
         if (denylisted) {
             mPolicyManager.addUidPolicy(uid, POLICY_REJECT_METERED_BACKGROUND);
             mMetricsFeatureProvider.action(
@@ -127,18 +122,69 @@ public class DataSaverBackend {
         mPolicyManager.removeUidPolicy(uid, POLICY_ALLOW_METERED_BACKGROUND);
     }
 
+    private void loadUidPolicies(int policy) {
+        final int[] uidsWithPolicyArray = mPolicyManager.getUidsWithPolicy(policy);
+        final ArrayList<Integer> uidsWithPolicy = new ArrayList<>(uidsWithPolicyArray.length);
+        for (final int uid : uidsWithPolicyArray) {
+            uidsWithPolicy.add(uid);
+        }
+        // Update existing cached UID policies.
+        for (int i = 0; i < mUidPolicies.size(); i++) {
+            final Integer cachedEntryUid = mUidPolicies.keyAt(i);
+            if (uidsWithPolicy.remove(cachedEntryUid)) {
+                // UID had the policy. It was removed so we don't have to process it twice.
+                setCachedUidPolicyFlagAt(i, policy, true);
+            } else {
+                // UID does not have the policy.
+                setCachedUidPolicyFlagAt(i, policy, false);
+            }
+        }
+        // Add policies for remaining UIDs, which did not have cached policies, so we're it.
+        for (final int uid : uidsWithPolicy) {
+            mUidPolicies.put(uid, policy);
+        }
+    }
+
+    private void setCachedUidPolicyFlag(int uid, int policy, boolean add) {
+        final int index = mUidPolicies.indexOfKey(uid);
+        if (index < 0) {
+            if (add) {
+                mUidPolicies.put(uid, policy);
+            }
+            return;
+        }
+        setCachedUidPolicyFlagAt(index, policy, add);
+    }
+
+    private void setCachedUidPolicyFlagAt(int index, int policy, boolean add) {
+        final int currentPolicy = mUidPolicies.valueAt(index);
+        final int newPolicy = add ? (currentPolicy | policy) : (currentPolicy & ~policy);
+        mUidPolicies.setValueAt(index, newPolicy);
+    }
+
+    private void setUidPolicyFlag(int uid, int policy, boolean add) {
+        if (add) {
+            mPolicyManager.addUidPolicy(uid, policy);
+        } else {
+            mPolicyManager.removeUidPolicy(uid, policy);
+        }
+        setCachedUidPolicyFlag(uid, policy, add);
+    }
+
+    private boolean isUidPolicyFlagSet(int uid, int policy) {
+        return (mUidPolicies.get(uid, POLICY_NONE) & policy) == policy;
+    }
+
     public boolean isDenylisted(int uid) {
         loadDenylist();
-        return mUidPolicies.get(uid, POLICY_NONE) == POLICY_REJECT_METERED_BACKGROUND;
+        return isUidPolicyFlagSet(uid, POLICY_REJECT_METERED_BACKGROUND);
     }
 
     private void loadDenylist() {
         if (mDenylistInitialized) {
             return;
         }
-        for (int uid : mPolicyManager.getUidsWithPolicy(POLICY_REJECT_METERED_BACKGROUND)) {
-            mUidPolicies.put(uid, POLICY_REJECT_METERED_BACKGROUND);
-        }
+        loadUidPolicies(POLICY_REJECT_METERED_BACKGROUND);
         mDenylistInitialized = true;
     }