Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Keep a separate accessibility focus for proxy-ed displays.

Browse Source 
Proxy display a11y focus changes will not clear non-proxy displays,
and vice-versa.
The logic for a proxy display generally follows the logic for a
non-proxy display when deciding when to update
windows#isAccessibilityFocus/send WINDOWS_CHANGE_ACCESSIBILITY_FOCUSED.

Bug: 254545943
Test: manual check with TalkBack on and a proxy-ed Exo
VirtualDisplay. Both can retain and set their own a11y focus.
atest AccessibilityDisplayProxyTest,
atest AccessibilityManagerServiceTest

Change-Id: I76f72acf4c72ae3ce23cd2240430de3e844d2e4b
This commit is contained in:
sallyyuen
2023-02-13 13:59:53 -08:00
committed by Sally
parent ca9e2463e9
commit 8c43e818ae
6 changed files with 188 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
services/accessibility/java/com/android/server/accessibility/AbstractAccessibilityServiceConnection.java
View File

@@ -142,7 +142,7 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ
protected final SystemSupport mSystemSupport;
protected final WindowManagerInternal mWindowManagerService;
private final SystemActionPerformer mSystemActionPerformer;
private final AccessibilityWindowManager mA11yWindowManager;
final AccessibilityWindowManager mA11yWindowManager;
private final DisplayManager mDisplayManager;
private final PowerManager mPowerManager;
private final IPlatformCompat mIPlatformCompat;
@@ -2006,15 +2006,14 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ
return accessibilityWindowId;
}
private int resolveAccessibilityWindowIdForFindFocusLocked(int windowId, int focusType) {
int resolveAccessibilityWindowIdForFindFocusLocked(int windowId, int focusType) {
if (windowId == AccessibilityWindowInfo.ANY_WINDOW_ID) {
final int focusedWindowId = mA11yWindowManager.getFocusedWindowId(focusType);
// If the caller is a proxy and the found window doesn't belong to a proxy display
// (or vice versa), then return null. This doesn't work if there are multiple active
// proxys, but in the future this code shouldn't be needed if input and a11y focus are
// proxys, but in the future this code shouldn't be needed if input focus are
// properly split. (so we will deal with the issues if we see them).
//TODO(254545943): Remove this when there is user and proxy separation of input and a11y
// focus
//TODO(254545943): Remove this when there is user and proxy separation of input focus
if (!mA11yWindowManager.windowIdBelongsToDisplayType(focusedWindowId, mDisplayTypes)) {
return AccessibilityWindowInfo.UNDEFINED_WINDOW_ID;
}

2
services/accessibility/java/com/android/server/accessibility/AccessibilityManagerService.java
View File

@@ -3823,7 +3823,7 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub
try {
mProxyManager.registerProxy(client, displayId, mContext,
sIdCounter++, mMainHandler, mSecurityPolicy, this, getTraceManager(),
mWindowManagerService, mA11yWindowManager);
mWindowManagerService);
synchronized (mLock) {
notifyClearAccessibilityCacheLocked();

168
services/accessibility/java/com/android/server/accessibility/AccessibilityWindowManager.java
View File

@@ -19,6 +19,7 @@ package com.android.server.accessibility;
import static android.accessibilityservice.AccessibilityTrace.FLAGS_ACCESSIBILITY_INTERACTION_CONNECTION;
import static android.accessibilityservice.AccessibilityTrace.FLAGS_WINDOW_MANAGER_INTERNAL;
import static android.view.WindowManager.LayoutParams.TYPE_ACCESSIBILITY_OVERLAY;
import static android.view.accessibility.AccessibilityEvent.WINDOWS_CHANGE_ACCESSIBILITY_FOCUSED;
import static com.android.internal.util.function.pooled.PooledLambda.obtainMessage;
import static com.android.server.accessibility.AbstractAccessibilityServiceConnection.DISPLAY_TYPE_DEFAULT;
@@ -104,6 +105,8 @@ public class AccessibilityWindowManager {
private boolean mTouchInteractionInProgress;
private boolean mHasProxy;
/** List of Display Windows Observer, mapping from displayId -> DisplayWindowsObserver. */
private final SparseArray<DisplayWindowsObserver> mDisplayWindowsObservers =
new SparseArray<>();
@@ -159,6 +162,9 @@ public class AccessibilityWindowManager {
* Returns {@code true} if the window belongs to a display of {@code displayTypes}.
*/
public boolean windowIdBelongsToDisplayType(int focusedWindowId, int displayTypes) {
if (!mHasProxy) {
return true;
}
// UIAutomation wants focus from any display type.
final int displayTypeMask = DISPLAY_TYPE_PROXY | DISPLAY_TYPE_DEFAULT;
if ((displayTypes & displayTypeMask) == displayTypeMask) {
@@ -195,6 +201,7 @@ public class AccessibilityWindowManager {
private List<AccessibilityWindowInfo> mWindows;
private boolean mTrackingWindows = false;
private boolean mHasWatchOutsideTouchWindow;
private int mProxyDisplayAccessibilityFocusedWindow;
private boolean mIsProxy;
/**
@@ -608,8 +615,11 @@ public class AccessibilityWindowManager {
final int windowCount = windows.size();
final boolean isTopFocusedDisplay = mDisplayId == mTopFocusedDisplayId;
// A proxy with an a11y-focused window is a11y-focused should use the proxy focus id.
final boolean isAccessibilityFocusedDisplay =
mDisplayId == mAccessibilityFocusedDisplayId;
mDisplayId == mAccessibilityFocusedDisplayId
|| (mIsProxy && mProxyDisplayAccessibilityFocusedWindow
!= AccessibilityWindowInfo.UNDEFINED_WINDOW_ID);
// Modifies the value of top focused window, active window and a11y focused window
// only if this display is top focused display which has the top focused window.
if (isTopFocusedDisplay) {
@@ -635,9 +645,12 @@ public class AccessibilityWindowManager {
// We'll clear accessibility focus if the window with focus is no longer visible to
// accessibility services.
int a11yFocusedWindowId = mIsProxy
? mProxyDisplayAccessibilityFocusedWindow
: mAccessibilityFocusedWindowId;
if (isAccessibilityFocusedDisplay) {
shouldClearAccessibilityFocus = mAccessibilityFocusedWindowId
!= AccessibilityWindowInfo.UNDEFINED_WINDOW_ID;
shouldClearAccessibilityFocus = a11yFocusedWindowId
!= AccessibilityWindowInfo.UNDEFINED_WINDOW_ID;
}
boolean hasWindowIgnore = false;
@@ -701,7 +714,7 @@ public class AccessibilityWindowManager {
if (isAccessibilityFocusedDisplay) {
for (int i = 0; i < accessibilityWindowCount; i++) {
final AccessibilityWindowInfo window = mWindows.get(i);
if (window.getId() == mAccessibilityFocusedWindowId) {
if (window.getId() == a11yFocusedWindowId) {
window.setAccessibilityFocused(true);
shouldClearAccessibilityFocus = false;
break;
@@ -718,7 +731,7 @@ public class AccessibilityWindowManager {
}
if (shouldClearAccessibilityFocus) {
clearAccessibilityFocusLocked(mAccessibilityFocusedWindowId);
clearAccessibilityFocusLocked(a11yFocusedWindowId);
}
}
@@ -1022,6 +1035,7 @@ public class AccessibilityWindowManager {
}
if (proxyed && !observer.mIsProxy) {
observer.mIsProxy = true;
mHasProxy = true;
}
if (observer.isTrackingWindowsLocked()) {
return;
@@ -1044,6 +1058,7 @@ public class AccessibilityWindowManager {
observer.stopTrackingWindowsLocked();
mDisplayWindowsObservers.remove(displayId);
}
resetHasProxyIfNeededLocked();
}
}
@@ -1053,11 +1068,26 @@ public class AccessibilityWindowManager {
*/
public void stopTrackingDisplayProxy(int displayId) {
synchronized (mLock) {
DisplayWindowsObserver observer = mDisplayWindowsObservers.get(displayId);
final DisplayWindowsObserver proxyObserver = mDisplayWindowsObservers.get(displayId);
if (proxyObserver != null) {
proxyObserver.mIsProxy = false;
}
resetHasProxyIfNeededLocked();
}
}
private void resetHasProxyIfNeededLocked() {
boolean hasProxy = false;
final int count = mDisplayWindowsObservers.size();
for (int i = 0; i < count; i++) {
final DisplayWindowsObserver observer = mDisplayWindowsObservers.valueAt(i);
if (observer != null) {
observer.mIsProxy = false;
if (observer.mIsProxy) {
hasProxy = true;
}
}
}
mHasProxy = hasProxy;
}
/**
@@ -1490,6 +1520,11 @@ public class AccessibilityWindowManager {
case AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED: {
synchronized (mLock) {
// If window id belongs to a proxy display, then find the display, update the
// observer focus and send WINDOWS_CHANGE_ACCESSIBILITY_FOCUSED events.
if (mHasProxy && setProxyFocusLocked(windowId)) {
return;
}
if (mAccessibilityFocusedWindowId != windowId) {
clearAccessibilityFocusLocked(mAccessibilityFocusedWindowId);
setAccessibilityFocusedWindowLocked(windowId);
@@ -1500,6 +1535,10 @@ public class AccessibilityWindowManager {
case AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUS_CLEARED: {
synchronized (mLock) {
// If cleared happened on the proxy display, then clear the tracked focus.
if (mHasProxy && clearProxyFocusLocked(windowId, eventAction)) {
return;
}
if (mAccessibilityFocusNodeId == nodeId) {
mAccessibilityFocusNodeId = AccessibilityNodeInfo.UNDEFINED_ITEM_ID;
}
@@ -1599,9 +1638,10 @@ public class AccessibilityWindowManager {
if (mAccessibilityFocusedDisplayId != Display.INVALID_DISPLAY
&& mAccessibilityFocusedWindowId
!= AccessibilityWindowInfo.UNDEFINED_WINDOW_ID) {
// Previously focused window -> send a focused event for losing focus
events.add(AccessibilityEvent.obtainWindowsChangedEvent(
mAccessibilityFocusedDisplayId, mAccessibilityFocusedWindowId,
AccessibilityEvent.WINDOWS_CHANGE_ACCESSIBILITY_FOCUSED));
WINDOWS_CHANGE_ACCESSIBILITY_FOCUSED));
}
mAccessibilityFocusedWindowId = windowId;
@@ -1611,8 +1651,9 @@ public class AccessibilityWindowManager {
final DisplayWindowsObserver observer = mDisplayWindowsObservers.valueAt(i);
if (observer != null && observer.setAccessibilityFocusedWindowLocked(windowId)) {
mAccessibilityFocusedDisplayId = observer.mDisplayId;
// Newly focused window -> send a focused event for gaining focus
events.add(AccessibilityEvent.obtainWindowsChangedEvent(observer.mDisplayId,
windowId, AccessibilityEvent.WINDOWS_CHANGE_ACCESSIBILITY_FOCUSED));
windowId, WINDOWS_CHANGE_ACCESSIBILITY_FOCUSED));
}
}
@@ -1662,6 +1703,33 @@ public class AccessibilityWindowManager {
* @return The focused windowId
*/
public int getFocusedWindowId(int focusType) {
return getFocusedWindowId(focusType, Display.INVALID_DISPLAY);
}
/**
* Returns focused windowId or accessibility focused windowId according to given focusType and
* display id.
* @param focusType {@link AccessibilityNodeInfo#FOCUS_INPUT} or
* {@link AccessibilityNodeInfo#FOCUS_ACCESSIBILITY}
* @param displayId the display id to check. If this display is proxy-ed, the proxy's a11y focus
* will be returned.
* @return The focused windowId
*/
public int getFocusedWindowId(int focusType, int displayId) {
if (displayId == Display.INVALID_DISPLAY || displayId == Display.DEFAULT_DISPLAY
|| !mHasProxy) {
return getDefaultFocus(focusType);
}
final DisplayWindowsObserver observer = mDisplayWindowsObservers.get(displayId);
if (observer != null && observer.mIsProxy) {
return getProxyFocus(focusType, observer);
} else {
return getDefaultFocus(focusType);
}
}
private int getDefaultFocus(int focusType) {
if (focusType == AccessibilityNodeInfo.FOCUS_INPUT) {
return mTopFocusedWindowId;
} else if (focusType == AccessibilityNodeInfo.FOCUS_ACCESSIBILITY) {
@@ -1670,6 +1738,16 @@ public class AccessibilityWindowManager {
return AccessibilityWindowInfo.UNDEFINED_WINDOW_ID;
}
private int getProxyFocus(int focusType, DisplayWindowsObserver observer) {
if (focusType == AccessibilityNodeInfo.FOCUS_INPUT) {
return mTopFocusedWindowId;
} else if (focusType == AccessibilityNodeInfo.FOCUS_ACCESSIBILITY) {
return observer.mProxyDisplayAccessibilityFocusedWindow;
} else {
return AccessibilityWindowInfo.UNDEFINED_WINDOW_ID;
}
}
/**
* Returns {@link AccessibilityWindowInfo} of PIP window.
*
@@ -1987,6 +2065,78 @@ public class AccessibilityWindowManager {
return mHostEmbeddedMap.get(token);
}
/**
* Checks if the window belongs to a proxy display and if so clears the focused window id.
* @param focusClearedWindowId the cleared window id.
* @return true if an observer is proxy-ed and has cleared its focused window id.
*/
private boolean clearProxyFocusLocked(int focusClearedWindowId, int eventAction) {
// If we are just moving focus from one view to the other in the same window, do nothing.
if (eventAction == AccessibilityNodeInfo.ACTION_ACCESSIBILITY_FOCUS) {
return false;
}
for (int i = 0; i < mDisplayWindowsObservers.size(); i++) {
final DisplayWindowsObserver observer = mDisplayWindowsObservers.get(i);
if (observer != null && observer.mWindows != null && observer.mIsProxy) {
final int windowCount = observer.mWindows.size();
for (int j = 0; j < windowCount; j++) {
AccessibilityWindowInfo window = observer.mWindows.get(j);
if (window.getId() == focusClearedWindowId) {
observer.mProxyDisplayAccessibilityFocusedWindow =
AccessibilityWindowInfo.UNDEFINED_WINDOW_ID;
// TODO(268754409): Look into sending a WINDOW_FOCUS_CHANGED event since
// window no longer has focus (default window logic doesn't), and
// whether the node id needs to be cached (default window logic does).
return true;
}
}
}
}
return false;
}
/**
* Checks if the window belongs to a proxy display and if so sends
* WINDOWS_CHANGE_ACCESSIBILITY_FOCUSED for that window and the previously focused window.
* @param focusedWindowId the focused window id.
* @return true if an observer is proxy-ed and contains the focused window.
*/
private boolean setProxyFocusLocked(int focusedWindowId) {
for (int i = 0; i < mDisplayWindowsObservers.size(); i++) {
final DisplayWindowsObserver observer = mDisplayWindowsObservers.valueAt(i);
if (observer != null && observer.mIsProxy
&& observer.setAccessibilityFocusedWindowLocked(focusedWindowId)) {
final int previouslyFocusedWindowId =
observer.mProxyDisplayAccessibilityFocusedWindow;
if (previouslyFocusedWindowId == focusedWindowId) {
// Don't send a focus event if the window is already focused.
return true;
}
// Previously focused window -> Clear focus on UI thread and send a focused event
// for losing focus
if (previouslyFocusedWindowId != AccessibilityWindowInfo.UNDEFINED_WINDOW_ID) {
clearAccessibilityFocusLocked(previouslyFocusedWindowId);
mAccessibilityEventSender.sendAccessibilityEventForCurrentUserLocked(
AccessibilityEvent.obtainWindowsChangedEvent(
observer.mDisplayId, previouslyFocusedWindowId,
WINDOWS_CHANGE_ACCESSIBILITY_FOCUSED));
}
observer.mProxyDisplayAccessibilityFocusedWindow = focusedWindowId;
// Newly focused window -> send a focused event for it gaining focus
mAccessibilityEventSender.sendAccessibilityEventForCurrentUserLocked(
AccessibilityEvent.obtainWindowsChangedEvent(
observer.mDisplayId,
observer.mProxyDisplayAccessibilityFocusedWindow,
WINDOWS_CHANGE_ACCESSIBILITY_FOCUSED));
return true;
}
}
return false;
}
/**
* Dumps all {@link AccessibilityWindowInfo}s here.
*/

18
services/accessibility/java/com/android/server/accessibility/ProxyAccessibilityServiceConnection.java
View File

@@ -254,6 +254,24 @@ public class ProxyAccessibilityServiceConnection extends AccessibilityServiceCon
return mFocusColor;
}
@Override
int resolveAccessibilityWindowIdForFindFocusLocked(int windowId, int focusType) {
if (windowId == AccessibilityWindowInfo.ANY_WINDOW_ID) {
final int focusedWindowId = mA11yWindowManager.getFocusedWindowId(focusType,
mDisplayId);
// If the caller is a proxy and the found window doesn't belong to a proxy display
// (or vice versa), then return null. This doesn't work if there are multiple active
// proxys, but in the future this code shouldn't be needed if input focus
// properly split. (so we will deal with the issues if we see them).
//TODO(254545943): Remove this when there is user and proxy separation of input
if (!mA11yWindowManager.windowIdBelongsToDisplayType(focusedWindowId, mDisplayTypes)) {
return AccessibilityWindowInfo.UNDEFINED_WINDOW_ID;
}
return focusedWindowId;
}
return windowId;
}
@Override
public void binderDied() {
}

5
services/accessibility/java/com/android/server/accessibility/ProxyManager.java
View File

@@ -78,8 +78,7 @@ public class ProxyManager {
AccessibilitySecurityPolicy securityPolicy,
AbstractAccessibilityServiceConnection.SystemSupport systemSupport,
AccessibilityTrace trace,
WindowManagerInternal windowManagerInternal,
AccessibilityWindowManager awm) throws RemoteException {
WindowManagerInternal windowManagerInternal) throws RemoteException {
// Set a default AccessibilityServiceInfo that is used before the proxy's info is
// populated. A proxy has the touch exploration and window capabilities.
@@ -93,7 +92,7 @@ public class ProxyManager {
new ProxyAccessibilityServiceConnection(context, info.getComponentName(), info,
id, mainHandler, mLock, securityPolicy, systemSupport, trace,
windowManagerInternal,
awm, displayId);
mA11yWindowManager, displayId);
synchronized (mLock) {
mProxyA11yServiceConnections.put(displayId, connection);

8
services/tests/servicestests/src/com/android/server/accessibility/AccessibilityManagerServiceTest.java
View File

@@ -284,7 +284,7 @@ public class AccessibilityManagerServiceTest {
verify(mProxyManager).registerProxy(eq(mMockServiceClient), eq(TEST_DISPLAY),
eq(mTestableContext), anyInt(), any(), eq(mMockSecurityPolicy),
eq(mA11yms), eq(mA11yms.getTraceManager()),
eq(mMockWindowManagerService), eq(mMockA11yWindowManager));
eq(mMockWindowManagerService));
}
@SmallTest
@@ -296,7 +296,7 @@ public class AccessibilityManagerServiceTest {
assertThrows(SecurityException.class,
() -> mA11yms.registerProxyForDisplay(mMockServiceClient, TEST_DISPLAY));
verify(mProxyManager, never()).registerProxy(any(), anyInt(), any(), anyInt(), any(), any(),
any(), any(), any(), any());
any(), any(), any());
}
@SmallTest
@@ -305,7 +305,7 @@ public class AccessibilityManagerServiceTest {
assertThrows(IllegalArgumentException.class,
() -> mA11yms.registerProxyForDisplay(mMockServiceClient, Display.DEFAULT_DISPLAY));
verify(mProxyManager, never()).registerProxy(any(), anyInt(), any(), anyInt(), any(), any(),
any(), any(), any(), any());
any(), any(), any());
}
@SmallTest
@@ -314,7 +314,7 @@ public class AccessibilityManagerServiceTest {
assertThrows(IllegalArgumentException.class,
() -> mA11yms.registerProxyForDisplay(mMockServiceClient, Display.INVALID_DISPLAY));
verify(mProxyManager, never()).registerProxy(any(), anyInt(), any(), anyInt(), any(), any(),
any(), any(), any(), any());
any(), any(), any());
}
@SmallTest