Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Allow app-process tooling to navigate the AccessibilityNodeInfo tree.

Browse Source 
This change bypasses the AccessibilityService and directly
connects AccessibilityInteractionClient to ViewRootImpl's
AccessibilityInteractionController.

Bug: 238391032
Test: Use for Espresso a11y testing
Test: atest AccessibilityEndToEndTest
Change-Id: I0881067a9e5290e208cf3065c149f2a3b433b80c
This commit is contained in:
Daniel Norman
2022-06-28 16:25:08 -07:00
parent 1c31a34b22
commit 57bb237ac7
5 changed files with 265 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
core/api/current.txt
View File

@@ -51972,6 +51972,7 @@ package android.view.accessibility {
method public boolean isTextEntryKey();
method public boolean isTextSelectable();
method public boolean isVisibleToUser();
method public void makeQueryableFromAppProcess(@NonNull android.view.View);
method @Deprecated public static android.view.accessibility.AccessibilityNodeInfo obtain(android.view.View);
method @Deprecated public static android.view.accessibility.AccessibilityNodeInfo obtain(android.view.View, int);
method @Deprecated public static android.view.accessibility.AccessibilityNodeInfo obtain();

27
core/java/android/view/ViewRootImpl.java
View File

@@ -172,6 +172,7 @@ import android.view.WindowInsets.Type;
import android.view.WindowInsets.Type.InsetsType;
import android.view.WindowManager.LayoutParams.SoftInputModeFlags;
import android.view.accessibility.AccessibilityEvent;
import android.view.accessibility.AccessibilityInteractionClient;
import android.view.accessibility.AccessibilityManager;
import android.view.accessibility.AccessibilityManager.AccessibilityStateChangeListener;
import android.view.accessibility.AccessibilityManager.HighTextContrastChangeListener;
@@ -5323,6 +5324,7 @@ public final class ViewRootImpl implements ViewParent,
}
mAccessibilityInteractionConnectionManager.ensureNoConnection();
mAccessibilityInteractionConnectionManager.ensureNoDirectConnection();
removeSendWindowContentChangedCallback();
destroyHardwareRenderer();
@@ -9524,6 +9526,14 @@ public final class ViewRootImpl implements ViewParent,
}
}
/**
* Return the connection ID for the {@link AccessibilityInteractionController} of this instance.
* @see AccessibilityNodeInfo#makeQueryableFromAppProcess(View)
*/
public int getDirectAccessibilityConnectionId() {
return mAccessibilityInteractionConnectionManager.ensureDirectConnection();
}
@Override
public boolean showContextMenuForChild(View originalView) {
return false;
@@ -10399,6 +10409,8 @@ public final class ViewRootImpl implements ViewParent,
*/
final class AccessibilityInteractionConnectionManager
implements AccessibilityStateChangeListener {
private int mDirectConnectionId = AccessibilityNodeInfo.UNDEFINED_CONNECTION_ID;
@Override
public void onAccessibilityStateChanged(boolean enabled) {
if (enabled) {
@@ -10442,6 +10454,21 @@ public final class ViewRootImpl implements ViewParent,
mAccessibilityManager.removeAccessibilityInteractionConnection(mWindow);
}
}
public int ensureDirectConnection() {
if (mDirectConnectionId == AccessibilityNodeInfo.UNDEFINED_CONNECTION_ID) {
mDirectConnectionId = AccessibilityInteractionClient.addDirectConnection(
new AccessibilityInteractionConnection(ViewRootImpl.this));
}
return mDirectConnectionId;
}
public void ensureNoDirectConnection() {
if (mDirectConnectionId != AccessibilityNodeInfo.UNDEFINED_CONNECTION_ID) {
AccessibilityInteractionClient.removeConnection(mDirectConnectionId);
mDirectConnectionId = AccessibilityNodeInfo.UNDEFINED_CONNECTION_ID;
}
}
}
final class HighContrastTextManager implements HighTextContrastChangeListener {

37
core/java/android/view/accessibility/AccessibilityInteractionClient.java
View File

@@ -114,6 +114,10 @@ public final class AccessibilityInteractionClient
private static final SparseArray<IAccessibilityServiceConnection> sConnectionCache =
new SparseArray<>();
// Used to generate connection ids for direct app-process connections. Start sufficiently far
// enough from the connection ids generated by AccessibilityManagerService.
private static int sDirectConnectionIdCounter = 1 << 30;
/** List of timestamps which indicate the latest time an a11y service receives a scroll event
from a window, mapping from windowId -> timestamp. */
private static final SparseLongArray sScrollingWindows = new SparseLongArray();
@@ -232,6 +236,12 @@ public final class AccessibilityInteractionClient
return;
}
synchronized (sConnectionCache) {
IAccessibilityServiceConnection existingConnection = getConnection(connectionId);
if (existingConnection instanceof DirectAccessibilityConnection) {
throw new IllegalArgumentException(
"Cannot add service connection with id " + connectionId
+ " which conflicts with existing direct connection.");
}
sConnectionCache.put(connectionId, connection);
if (!initializeCache) {
return;
@@ -241,6 +251,33 @@ public final class AccessibilityInteractionClient
}
}
/**
* Adds a new {@link DirectAccessibilityConnection} using the provided
* {@link IAccessibilityInteractionConnection} to create a direct connection between
* this client and the {@link android.view.ViewRootImpl} for queries inside the app process.
*
* <p>
* See {@link DirectAccessibilityConnection} for supported methods.
* </p>
*
* @param connection The ViewRootImpl's {@link IAccessibilityInteractionConnection}.
*/
public static int addDirectConnection(IAccessibilityInteractionConnection connection) {
synchronized (sConnectionCache) {
int connectionId = sDirectConnectionIdCounter++;
if (getConnection(connectionId) != null) {
throw new IllegalArgumentException(
"Cannot add direct connection with existing id " + connectionId);
}
DirectAccessibilityConnection directAccessibilityConnection =
new DirectAccessibilityConnection(connection);
sConnectionCache.put(connectionId, directAccessibilityConnection);
// Do not use AccessibilityCache for this connection, since there is no corresponding
// AccessibilityService to handle cache invalidation events.
return connectionId;
}
}
/**
* Gets a cached associated with the connection id if available.
*

72
core/java/android/view/accessibility/AccessibilityNodeInfo.java
View File

@@ -58,6 +58,7 @@ import android.view.SurfaceView;
import android.view.TouchDelegate;
import android.view.View;
import android.view.ViewGroup;
import android.view.ViewRootImpl;
import android.widget.TextView;
import com.android.internal.R;
@@ -82,7 +83,9 @@ import java.util.Objects;
* </p>
* <p>
* Once an accessibility node info is delivered to an accessibility service it is
* made immutable and calling a state mutation method generates an error.
* made immutable and calling a state mutation method generates an error. See
* {@link #makeQueryableFromAppProcess(View)} if you would like to inspect the
* node tree from the app process for testing or debugging tools.
* </p>
* <p>
* Please refer to {@link android.accessibilityservice.AccessibilityService} for
@@ -1171,8 +1174,8 @@ public class AccessibilityNodeInfo implements Parcelable {
* @param index The child index.
* @return The child node.
*
* @throws IllegalStateException If called outside of an AccessibilityService.
*
* @throws IllegalStateException If called outside of an {@link AccessibilityService} and before
* calling {@link #makeQueryableFromAppProcess(View)}.
*/
public AccessibilityNodeInfo getChild(int index) {
return getChild(index, FLAG_PREFETCH_DESCENDANTS_HYBRID);
@@ -1186,7 +1189,8 @@ public class AccessibilityNodeInfo implements Parcelable {
* @param prefetchingStrategy the prefetching strategy.
* @return The child node.
*
* @throws IllegalStateException If called outside of an AccessibilityService.
* @throws IllegalStateException If called outside of an {@link AccessibilityService} and before
* calling {@link #makeQueryableFromAppProcess(View)}.
*
* @see AccessibilityNodeInfo#getParent(int) for a description of prefetching.
*/
@@ -1908,6 +1912,9 @@ public class AccessibilityNodeInfo implements Parcelable {
* Gets the parent.
*
* @return The parent.
*
* @throws IllegalStateException If called outside of an {@link AccessibilityService} and before
* calling {@link #makeQueryableFromAppProcess(View)}.
*/
public AccessibilityNodeInfo getParent() {
enforceSealed();
@@ -1935,7 +1942,8 @@ public class AccessibilityNodeInfo implements Parcelable {
* @param prefetchingStrategy the prefetching strategy.
* @return The parent.
*
* @throws IllegalStateException If called outside of an AccessibilityService.
* @throws IllegalStateException If called outside of an {@link AccessibilityService} and before
* calling {@link #makeQueryableFromAppProcess(View)}.
*
* @see #FLAG_PREFETCH_ANCESTORS
* @see #FLAG_PREFETCH_DESCENDANTS_BREADTH_FIRST
@@ -3656,6 +3664,47 @@ public class AccessibilityNodeInfo implements Parcelable {
return mLeashedParentNodeId;
}
/**
* Connects this node to the View's root so that operations on this node can query the entire
* {@link AccessibilityNodeInfo} tree and perform accessibility actions on nodes.
*
* <p>
* This is intended for short-lived inspections from testing or debugging tools in the app
* process. After calling this method, all nodes linked to this node (children, ancestors, etc.)
* are also queryable. Operations on this node tree will only succeed as long as the associated
* view hierarchy remains attached to a window.
* </p>
*
* <p>
* Calling this method more than once on the same node is a no-op; if you wish to inspect a
* different view hierarchy then create a new node from any view in that hierarchy and call this
* method on that node.
* </p>
*
* <p>
* Testing or debugging tools should create this {@link AccessibilityNodeInfo} node using
* {@link View#createAccessibilityNodeInfo()} or {@link AccessibilityNodeProvider} and call this
* method, then navigate and interact with the node tree by calling methods on the node.
* </p>
*
* @param view The view that generated this node, or any view in the same view-root hierarchy.
* @throws IllegalStateException If called from an {@link AccessibilityService}, or if provided
* a {@link View} that is not attached to a window.
*/
public void makeQueryableFromAppProcess(@NonNull View view) {
enforceNotSealed();
if (mConnectionId != UNDEFINED_CONNECTION_ID) {
return;
}
ViewRootImpl viewRootImpl = view.getViewRootImpl();
if (viewRootImpl == null) {
throw new IllegalStateException(
"Cannot link a node to a view that is not attached to a window.");
}
setConnectionId(viewRootImpl.getDirectAccessibilityConnectionId());
}
/**
* Sets if this instance is sealed.
*
@@ -3680,15 +3729,21 @@ public class AccessibilityNodeInfo implements Parcelable {
return mSealed;
}
private static boolean usingDirectConnection(int connectionId) {
return AccessibilityInteractionClient.getConnection(
connectionId) instanceof DirectAccessibilityConnection;
}
/**
* Enforces that this instance is sealed.
* Enforces that this instance is sealed, unless using a {@link DirectAccessibilityConnection}
* which allows queries while the node is not sealed.
*
* @throws IllegalStateException If this instance is not sealed.
*
* @hide
*/
protected void enforceSealed() {
if (!isSealed()) {
if (!usingDirectConnection(mConnectionId) && !isSealed()) {
throw new IllegalStateException("Cannot perform this "
+ "action on a not sealed instance.");
}
@@ -4514,7 +4569,8 @@ public class AccessibilityNodeInfo implements Parcelable {
private static boolean canPerformRequestOverConnection(int connectionId,
int windowId, long accessibilityNodeId) {
return ((windowId != AccessibilityWindowInfo.UNDEFINED_WINDOW_ID)
final boolean hasWindowId = windowId != AccessibilityWindowInfo.UNDEFINED_WINDOW_ID;
return ((usingDirectConnection(connectionId) || hasWindowId)
&& (getAccessibilityViewId(accessibilityNodeId) != UNDEFINED_ITEM_ID)
&& (connectionId != UNDEFINED_CONNECTION_ID));
}

136
core/java/android/view/accessibility/DirectAccessibilityConnection.java Normal file
View File

@@ -0,0 +1,136 @@
/*
* Copyright (C) 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package android.view.accessibility;
import android.accessibilityservice.IAccessibilityServiceConnection;
import android.graphics.Matrix;
import android.graphics.Region;
import android.os.Bundle;
import android.os.Process;
import android.os.RemoteException;
import android.view.MagnificationSpec;
/**
* Minimal {@link IAccessibilityServiceConnection} implementation that interacts
* with the {@link android.view.AccessibilityInteractionController} of a
* {@link android.view.ViewRootImpl}.
*
* <p>
* Uses {@link android.view.ViewRootImpl}'s {@link IAccessibilityServiceConnection} that wraps
* {@link android.view.AccessibilityInteractionController} within the app process, so that no
* interprocess communication is performed.
* </p>
*
* <p>
* Only the following methods are supported:
* <li>{@link #findAccessibilityNodeInfoByAccessibilityId}</li>
* <li>{@link #findAccessibilityNodeInfosByText}</li>
* <li>{@link #findAccessibilityNodeInfosByViewId}</li>
* <li>{@link #findFocus}</li>
* <li>{@link #focusSearch}</li>
* <li>{@link #performAccessibilityAction}</li>
* </p>
*
* <p>
* Other methods are no-ops and return default values.
* </p>
*/
class DirectAccessibilityConnection extends IAccessibilityServiceConnection.Default {
private final IAccessibilityInteractionConnection mAccessibilityInteractionConnection;
// Fetch all views, but do not use prefetching/cache since this "connection" does not
// receive cache invalidation events (as it is not linked to an AccessibilityService).
private static final int FETCH_FLAGS =
AccessibilityNodeInfo.FLAG_REPORT_VIEW_IDS
| AccessibilityNodeInfo.FLAG_INCLUDE_NOT_IMPORTANT_VIEWS;
private static final MagnificationSpec MAGNIFICATION_SPEC = new MagnificationSpec();
private static final int PID = Process.myPid();
private static final Region INTERACTIVE_REGION = null;
private static final float[] TRANSFORM_MATRIX = new float[9];
static {
Matrix.IDENTITY_MATRIX.getValues(TRANSFORM_MATRIX);
}
DirectAccessibilityConnection(
IAccessibilityInteractionConnection accessibilityInteractionConnection) {
mAccessibilityInteractionConnection = accessibilityInteractionConnection;
}
@Override
public String[] findAccessibilityNodeInfoByAccessibilityId(int accessibilityWindowId,
long accessibilityNodeId, int interactionId,
IAccessibilityInteractionConnectionCallback callback, int flags, long threadId,
Bundle arguments) throws RemoteException {
mAccessibilityInteractionConnection.findAccessibilityNodeInfoByAccessibilityId(
accessibilityNodeId, INTERACTIVE_REGION, interactionId, callback, FETCH_FLAGS, PID,
threadId, MAGNIFICATION_SPEC, TRANSFORM_MATRIX, arguments);
return new String[0];
}
@Override
public String[] findAccessibilityNodeInfosByText(int accessibilityWindowId,
long accessibilityNodeId, String text, int interactionId,
IAccessibilityInteractionConnectionCallback callback, long threadId)
throws RemoteException {
mAccessibilityInteractionConnection.findAccessibilityNodeInfosByText(accessibilityNodeId,
text, INTERACTIVE_REGION, interactionId, callback, FETCH_FLAGS, PID, threadId,
MAGNIFICATION_SPEC, TRANSFORM_MATRIX);
return new String[0];
}
@Override
public String[] findAccessibilityNodeInfosByViewId(int accessibilityWindowId,
long accessibilityNodeId, String viewId, int interactionId,
IAccessibilityInteractionConnectionCallback callback, long threadId)
throws RemoteException {
mAccessibilityInteractionConnection.findAccessibilityNodeInfosByViewId(accessibilityNodeId,
viewId, INTERACTIVE_REGION, interactionId, callback, FETCH_FLAGS, PID, threadId,
MAGNIFICATION_SPEC, TRANSFORM_MATRIX);
return new String[0];
}
@Override
public String[] findFocus(int accessibilityWindowId, long accessibilityNodeId, int focusType,
int interactionId, IAccessibilityInteractionConnectionCallback callback, long threadId)
throws RemoteException {
mAccessibilityInteractionConnection.findFocus(accessibilityNodeId, focusType,
INTERACTIVE_REGION, interactionId, callback, FETCH_FLAGS, PID, threadId,
MAGNIFICATION_SPEC, TRANSFORM_MATRIX);
return new String[0];
}
@Override
public String[] focusSearch(int accessibilityWindowId, long accessibilityNodeId, int direction,
int interactionId, IAccessibilityInteractionConnectionCallback callback, long threadId)
throws RemoteException {
mAccessibilityInteractionConnection.focusSearch(accessibilityNodeId, direction,
INTERACTIVE_REGION, interactionId, callback, FETCH_FLAGS, PID, threadId,
MAGNIFICATION_SPEC, TRANSFORM_MATRIX);
return new String[0];
}
@Override
public boolean performAccessibilityAction(int accessibilityWindowId, long accessibilityNodeId,
int action, Bundle arguments, int interactionId,
IAccessibilityInteractionConnectionCallback callback, long threadId)
throws RemoteException {
mAccessibilityInteractionConnection.performAccessibilityAction(accessibilityNodeId, action,
arguments, interactionId, callback, FETCH_FLAGS, PID, threadId);
return true;
}
}