diff --git a/packages/CompanionDeviceManager/src/com/android/companiondevicemanager/PermissionListAdapter.java b/packages/CompanionDeviceManager/src/com/android/companiondevicemanager/PermissionListAdapter.java
index f594bf270d294..7ed18163dd83f 100644
--- a/packages/CompanionDeviceManager/src/com/android/companiondevicemanager/PermissionListAdapter.java
+++ b/packages/CompanionDeviceManager/src/com/android/companiondevicemanager/PermissionListAdapter.java
@@ -123,12 +123,11 @@ class PermissionListAdapter extends RecyclerView.Adapter<PermissionListAdapter.V
             viewHolder.mExpandButton.setTag(R.drawable.btn_expand_more);
         }
 
-        setAccessibility(view, viewType,
-                AccessibilityNodeInfo.ACTION_CLICK, R.string.permission_expand, 0);
-
         // Add expand buttons if the permissions are more than PERMISSION_SIZE in this list also
         // make the summary invisible by default.
         if (mPermissions.size() > PERMISSION_SIZE) {
+            setAccessibility(view, viewType,
+                    AccessibilityNodeInfo.ACTION_CLICK, R.string.permission_expand, 0);
 
             viewHolder.mPermissionSummary.setVisibility(View.GONE);