diff --git a/packages/SettingsLib/src/com/android/settingslib/bluetooth/CachedBluetoothDevice.java b/packages/SettingsLib/src/com/android/settingslib/bluetooth/CachedBluetoothDevice.java
index 7534b8e3389c4..b0429ef43cf92 100644
--- a/packages/SettingsLib/src/com/android/settingslib/bluetooth/CachedBluetoothDevice.java
+++ b/packages/SettingsLib/src/com/android/settingslib/bluetooth/CachedBluetoothDevice.java
@@ -807,7 +807,9 @@ public final class CachedBluetoothDevice implements Comparable<CachedBluetoothDe
         if (BluetoothUuid.containsAnyUuid(uuids, PbapServerProfile.PBAB_CLIENT_UUIDS)) {
             // The pairing dialog now warns of phone-book access for paired devices.
             // No separate prompt is displayed after pairing.
-            setPhonebookPermissionChoice(CachedBluetoothDevice.ACCESS_ALLOWED);
+            if (getPhonebookPermissionChoice() == CachedBluetoothDevice.ACCESS_UNKNOWN) {
+                setPhonebookPermissionChoice(CachedBluetoothDevice.ACCESS_ALLOWED);
+            }
         }
     }