From 57bb237ac740d0b60de42a937d9148660f9c8284 Mon Sep 17 00:00:00 2001 From: Daniel Norman Date: Tue, 28 Jun 2022 16:25:08 -0700 Subject: [PATCH] Allow app-process tooling to navigate the AccessibilityNodeInfo tree. This change bypasses the AccessibilityService and directly connects AccessibilityInteractionClient to ViewRootImpl's AccessibilityInteractionController. Bug: 238391032 Test: Use for Espresso a11y testing Test: atest AccessibilityEndToEndTest Change-Id: I0881067a9e5290e208cf3065c149f2a3b433b80c --- core/api/current.txt | 1 + core/java/android/view/ViewRootImpl.java | 27 ++++ .../AccessibilityInteractionClient.java | 37 +++++ .../accessibility/AccessibilityNodeInfo.java | 72 ++++++++-- .../DirectAccessibilityConnection.java | 136 ++++++++++++++++++ 5 files changed, 265 insertions(+), 8 deletions(-) create mode 100644 core/java/android/view/accessibility/DirectAccessibilityConnection.java diff --git a/core/api/current.txt b/core/api/current.txt index ea2ae0c9e15f4..c196789c85d5d 100644 --- a/core/api/current.txt +++ b/core/api/current.txt @@ -51972,6 +51972,7 @@ package android.view.accessibility { method public boolean isTextEntryKey(); method public boolean isTextSelectable(); method public boolean isVisibleToUser(); + method public void makeQueryableFromAppProcess(@NonNull android.view.View); method @Deprecated public static android.view.accessibility.AccessibilityNodeInfo obtain(android.view.View); method @Deprecated public static android.view.accessibility.AccessibilityNodeInfo obtain(android.view.View, int); method @Deprecated public static android.view.accessibility.AccessibilityNodeInfo obtain(); diff --git a/core/java/android/view/ViewRootImpl.java b/core/java/android/view/ViewRootImpl.java index f79f0d49959d3..360caf3952914 100644 --- a/core/java/android/view/ViewRootImpl.java +++ b/core/java/android/view/ViewRootImpl.java @@ -172,6 +172,7 @@ import android.view.WindowInsets.Type; import android.view.WindowInsets.Type.InsetsType; import android.view.WindowManager.LayoutParams.SoftInputModeFlags; import android.view.accessibility.AccessibilityEvent; +import android.view.accessibility.AccessibilityInteractionClient; import android.view.accessibility.AccessibilityManager; import android.view.accessibility.AccessibilityManager.AccessibilityStateChangeListener; import android.view.accessibility.AccessibilityManager.HighTextContrastChangeListener; @@ -5323,6 +5324,7 @@ public final class ViewRootImpl implements ViewParent, } mAccessibilityInteractionConnectionManager.ensureNoConnection(); + mAccessibilityInteractionConnectionManager.ensureNoDirectConnection(); removeSendWindowContentChangedCallback(); destroyHardwareRenderer(); @@ -9524,6 +9526,14 @@ public final class ViewRootImpl implements ViewParent, } } + /** + * Return the connection ID for the {@link AccessibilityInteractionController} of this instance. + * @see AccessibilityNodeInfo#makeQueryableFromAppProcess(View) + */ + public int getDirectAccessibilityConnectionId() { + return mAccessibilityInteractionConnectionManager.ensureDirectConnection(); + } + @Override public boolean showContextMenuForChild(View originalView) { return false; @@ -10399,6 +10409,8 @@ public final class ViewRootImpl implements ViewParent, */ final class AccessibilityInteractionConnectionManager implements AccessibilityStateChangeListener { + private int mDirectConnectionId = AccessibilityNodeInfo.UNDEFINED_CONNECTION_ID; + @Override public void onAccessibilityStateChanged(boolean enabled) { if (enabled) { @@ -10442,6 +10454,21 @@ public final class ViewRootImpl implements ViewParent, mAccessibilityManager.removeAccessibilityInteractionConnection(mWindow); } } + + public int ensureDirectConnection() { + if (mDirectConnectionId == AccessibilityNodeInfo.UNDEFINED_CONNECTION_ID) { + mDirectConnectionId = AccessibilityInteractionClient.addDirectConnection( + new AccessibilityInteractionConnection(ViewRootImpl.this)); + } + return mDirectConnectionId; + } + + public void ensureNoDirectConnection() { + if (mDirectConnectionId != AccessibilityNodeInfo.UNDEFINED_CONNECTION_ID) { + AccessibilityInteractionClient.removeConnection(mDirectConnectionId); + mDirectConnectionId = AccessibilityNodeInfo.UNDEFINED_CONNECTION_ID; + } + } } final class HighContrastTextManager implements HighTextContrastChangeListener { diff --git a/core/java/android/view/accessibility/AccessibilityInteractionClient.java b/core/java/android/view/accessibility/AccessibilityInteractionClient.java index 68532781f08b8..227a8ef6fd604 100644 --- a/core/java/android/view/accessibility/AccessibilityInteractionClient.java +++ b/core/java/android/view/accessibility/AccessibilityInteractionClient.java @@ -114,6 +114,10 @@ public final class AccessibilityInteractionClient private static final SparseArray sConnectionCache = new SparseArray<>(); + // Used to generate connection ids for direct app-process connections. Start sufficiently far + // enough from the connection ids generated by AccessibilityManagerService. + private static int sDirectConnectionIdCounter = 1 << 30; + /** List of timestamps which indicate the latest time an a11y service receives a scroll event from a window, mapping from windowId -> timestamp. */ private static final SparseLongArray sScrollingWindows = new SparseLongArray(); @@ -232,6 +236,12 @@ public final class AccessibilityInteractionClient return; } synchronized (sConnectionCache) { + IAccessibilityServiceConnection existingConnection = getConnection(connectionId); + if (existingConnection instanceof DirectAccessibilityConnection) { + throw new IllegalArgumentException( + "Cannot add service connection with id " + connectionId + + " which conflicts with existing direct connection."); + } sConnectionCache.put(connectionId, connection); if (!initializeCache) { return; @@ -241,6 +251,33 @@ public final class AccessibilityInteractionClient } } + /** + * Adds a new {@link DirectAccessibilityConnection} using the provided + * {@link IAccessibilityInteractionConnection} to create a direct connection between + * this client and the {@link android.view.ViewRootImpl} for queries inside the app process. + * + *

+ * See {@link DirectAccessibilityConnection} for supported methods. + *

+ * + * @param connection The ViewRootImpl's {@link IAccessibilityInteractionConnection}. + */ + public static int addDirectConnection(IAccessibilityInteractionConnection connection) { + synchronized (sConnectionCache) { + int connectionId = sDirectConnectionIdCounter++; + if (getConnection(connectionId) != null) { + throw new IllegalArgumentException( + "Cannot add direct connection with existing id " + connectionId); + } + DirectAccessibilityConnection directAccessibilityConnection = + new DirectAccessibilityConnection(connection); + sConnectionCache.put(connectionId, directAccessibilityConnection); + // Do not use AccessibilityCache for this connection, since there is no corresponding + // AccessibilityService to handle cache invalidation events. + return connectionId; + } + } + /** * Gets a cached associated with the connection id if available. * diff --git a/core/java/android/view/accessibility/AccessibilityNodeInfo.java b/core/java/android/view/accessibility/AccessibilityNodeInfo.java index 15718c4af26f1..6ae59bf0c8fc9 100644 --- a/core/java/android/view/accessibility/AccessibilityNodeInfo.java +++ b/core/java/android/view/accessibility/AccessibilityNodeInfo.java @@ -58,6 +58,7 @@ import android.view.SurfaceView; import android.view.TouchDelegate; import android.view.View; import android.view.ViewGroup; +import android.view.ViewRootImpl; import android.widget.TextView; import com.android.internal.R; @@ -82,7 +83,9 @@ import java.util.Objects; *

*

* Once an accessibility node info is delivered to an accessibility service it is - * made immutable and calling a state mutation method generates an error. + * made immutable and calling a state mutation method generates an error. See + * {@link #makeQueryableFromAppProcess(View)} if you would like to inspect the + * node tree from the app process for testing or debugging tools. *

*

* Please refer to {@link android.accessibilityservice.AccessibilityService} for @@ -1171,8 +1174,8 @@ public class AccessibilityNodeInfo implements Parcelable { * @param index The child index. * @return The child node. * - * @throws IllegalStateException If called outside of an AccessibilityService. - * + * @throws IllegalStateException If called outside of an {@link AccessibilityService} and before + * calling {@link #makeQueryableFromAppProcess(View)}. */ public AccessibilityNodeInfo getChild(int index) { return getChild(index, FLAG_PREFETCH_DESCENDANTS_HYBRID); @@ -1186,7 +1189,8 @@ public class AccessibilityNodeInfo implements Parcelable { * @param prefetchingStrategy the prefetching strategy. * @return The child node. * - * @throws IllegalStateException If called outside of an AccessibilityService. + * @throws IllegalStateException If called outside of an {@link AccessibilityService} and before + * calling {@link #makeQueryableFromAppProcess(View)}. * * @see AccessibilityNodeInfo#getParent(int) for a description of prefetching. */ @@ -1908,6 +1912,9 @@ public class AccessibilityNodeInfo implements Parcelable { * Gets the parent. * * @return The parent. + * + * @throws IllegalStateException If called outside of an {@link AccessibilityService} and before + * calling {@link #makeQueryableFromAppProcess(View)}. */ public AccessibilityNodeInfo getParent() { enforceSealed(); @@ -1935,7 +1942,8 @@ public class AccessibilityNodeInfo implements Parcelable { * @param prefetchingStrategy the prefetching strategy. * @return The parent. * - * @throws IllegalStateException If called outside of an AccessibilityService. + * @throws IllegalStateException If called outside of an {@link AccessibilityService} and before + * calling {@link #makeQueryableFromAppProcess(View)}. * * @see #FLAG_PREFETCH_ANCESTORS * @see #FLAG_PREFETCH_DESCENDANTS_BREADTH_FIRST @@ -3656,6 +3664,47 @@ public class AccessibilityNodeInfo implements Parcelable { return mLeashedParentNodeId; } + /** + * Connects this node to the View's root so that operations on this node can query the entire + * {@link AccessibilityNodeInfo} tree and perform accessibility actions on nodes. + * + *

+ * This is intended for short-lived inspections from testing or debugging tools in the app + * process. After calling this method, all nodes linked to this node (children, ancestors, etc.) + * are also queryable. Operations on this node tree will only succeed as long as the associated + * view hierarchy remains attached to a window. + *

+ * + *

+ * Calling this method more than once on the same node is a no-op; if you wish to inspect a + * different view hierarchy then create a new node from any view in that hierarchy and call this + * method on that node. + *

+ * + *

+ * Testing or debugging tools should create this {@link AccessibilityNodeInfo} node using + * {@link View#createAccessibilityNodeInfo()} or {@link AccessibilityNodeProvider} and call this + * method, then navigate and interact with the node tree by calling methods on the node. + *

+ * + * @param view The view that generated this node, or any view in the same view-root hierarchy. + * @throws IllegalStateException If called from an {@link AccessibilityService}, or if provided + * a {@link View} that is not attached to a window. + */ + public void makeQueryableFromAppProcess(@NonNull View view) { + enforceNotSealed(); + if (mConnectionId != UNDEFINED_CONNECTION_ID) { + return; + } + + ViewRootImpl viewRootImpl = view.getViewRootImpl(); + if (viewRootImpl == null) { + throw new IllegalStateException( + "Cannot link a node to a view that is not attached to a window."); + } + setConnectionId(viewRootImpl.getDirectAccessibilityConnectionId()); + } + /** * Sets if this instance is sealed. * @@ -3680,15 +3729,21 @@ public class AccessibilityNodeInfo implements Parcelable { return mSealed; } + private static boolean usingDirectConnection(int connectionId) { + return AccessibilityInteractionClient.getConnection( + connectionId) instanceof DirectAccessibilityConnection; + } + /** - * Enforces that this instance is sealed. + * Enforces that this instance is sealed, unless using a {@link DirectAccessibilityConnection} + * which allows queries while the node is not sealed. * * @throws IllegalStateException If this instance is not sealed. * * @hide */ protected void enforceSealed() { - if (!isSealed()) { + if (!usingDirectConnection(mConnectionId) && !isSealed()) { throw new IllegalStateException("Cannot perform this " + "action on a not sealed instance."); } @@ -4514,7 +4569,8 @@ public class AccessibilityNodeInfo implements Parcelable { private static boolean canPerformRequestOverConnection(int connectionId, int windowId, long accessibilityNodeId) { - return ((windowId != AccessibilityWindowInfo.UNDEFINED_WINDOW_ID) + final boolean hasWindowId = windowId != AccessibilityWindowInfo.UNDEFINED_WINDOW_ID; + return ((usingDirectConnection(connectionId) || hasWindowId) && (getAccessibilityViewId(accessibilityNodeId) != UNDEFINED_ITEM_ID) && (connectionId != UNDEFINED_CONNECTION_ID)); } diff --git a/core/java/android/view/accessibility/DirectAccessibilityConnection.java b/core/java/android/view/accessibility/DirectAccessibilityConnection.java new file mode 100644 index 0000000000000..71746ee5dbab7 --- /dev/null +++ b/core/java/android/view/accessibility/DirectAccessibilityConnection.java @@ -0,0 +1,136 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.view.accessibility; + +import android.accessibilityservice.IAccessibilityServiceConnection; +import android.graphics.Matrix; +import android.graphics.Region; +import android.os.Bundle; +import android.os.Process; +import android.os.RemoteException; +import android.view.MagnificationSpec; + +/** + * Minimal {@link IAccessibilityServiceConnection} implementation that interacts + * with the {@link android.view.AccessibilityInteractionController} of a + * {@link android.view.ViewRootImpl}. + * + *

+ * Uses {@link android.view.ViewRootImpl}'s {@link IAccessibilityServiceConnection} that wraps + * {@link android.view.AccessibilityInteractionController} within the app process, so that no + * interprocess communication is performed. + *

+ * + *

+ * Only the following methods are supported: + *

  • {@link #findAccessibilityNodeInfoByAccessibilityId}
    • + *
  • {@link #findAccessibilityNodeInfosByText}
    • + *
  • {@link #findAccessibilityNodeInfosByViewId}
    • + *
  • {@link #findFocus}
    • + *
  • {@link #focusSearch}
    • + *
  • {@link #performAccessibilityAction}
    • + *

    + * + *

    + * Other methods are no-ops and return default values. + *

    + */ +class DirectAccessibilityConnection extends IAccessibilityServiceConnection.Default { + private final IAccessibilityInteractionConnection mAccessibilityInteractionConnection; + + // Fetch all views, but do not use prefetching/cache since this "connection" does not + // receive cache invalidation events (as it is not linked to an AccessibilityService). + private static final int FETCH_FLAGS = + AccessibilityNodeInfo.FLAG_REPORT_VIEW_IDS + | AccessibilityNodeInfo.FLAG_INCLUDE_NOT_IMPORTANT_VIEWS; + private static final MagnificationSpec MAGNIFICATION_SPEC = new MagnificationSpec(); + private static final int PID = Process.myPid(); + private static final Region INTERACTIVE_REGION = null; + private static final float[] TRANSFORM_MATRIX = new float[9]; + + static { + Matrix.IDENTITY_MATRIX.getValues(TRANSFORM_MATRIX); + } + + DirectAccessibilityConnection( + IAccessibilityInteractionConnection accessibilityInteractionConnection) { + mAccessibilityInteractionConnection = accessibilityInteractionConnection; + } + + @Override + public String[] findAccessibilityNodeInfoByAccessibilityId(int accessibilityWindowId, + long accessibilityNodeId, int interactionId, + IAccessibilityInteractionConnectionCallback callback, int flags, long threadId, + Bundle arguments) throws RemoteException { + mAccessibilityInteractionConnection.findAccessibilityNodeInfoByAccessibilityId( + accessibilityNodeId, INTERACTIVE_REGION, interactionId, callback, FETCH_FLAGS, PID, + threadId, MAGNIFICATION_SPEC, TRANSFORM_MATRIX, arguments); + return new String[0]; + } + + @Override + public String[] findAccessibilityNodeInfosByText(int accessibilityWindowId, + long accessibilityNodeId, String text, int interactionId, + IAccessibilityInteractionConnectionCallback callback, long threadId) + throws RemoteException { + mAccessibilityInteractionConnection.findAccessibilityNodeInfosByText(accessibilityNodeId, + text, INTERACTIVE_REGION, interactionId, callback, FETCH_FLAGS, PID, threadId, + MAGNIFICATION_SPEC, TRANSFORM_MATRIX); + return new String[0]; + } + + @Override + public String[] findAccessibilityNodeInfosByViewId(int accessibilityWindowId, + long accessibilityNodeId, String viewId, int interactionId, + IAccessibilityInteractionConnectionCallback callback, long threadId) + throws RemoteException { + mAccessibilityInteractionConnection.findAccessibilityNodeInfosByViewId(accessibilityNodeId, + viewId, INTERACTIVE_REGION, interactionId, callback, FETCH_FLAGS, PID, threadId, + MAGNIFICATION_SPEC, TRANSFORM_MATRIX); + return new String[0]; + } + + @Override + public String[] findFocus(int accessibilityWindowId, long accessibilityNodeId, int focusType, + int interactionId, IAccessibilityInteractionConnectionCallback callback, long threadId) + throws RemoteException { + mAccessibilityInteractionConnection.findFocus(accessibilityNodeId, focusType, + INTERACTIVE_REGION, interactionId, callback, FETCH_FLAGS, PID, threadId, + MAGNIFICATION_SPEC, TRANSFORM_MATRIX); + return new String[0]; + } + + @Override + public String[] focusSearch(int accessibilityWindowId, long accessibilityNodeId, int direction, + int interactionId, IAccessibilityInteractionConnectionCallback callback, long threadId) + throws RemoteException { + mAccessibilityInteractionConnection.focusSearch(accessibilityNodeId, direction, + INTERACTIVE_REGION, interactionId, callback, FETCH_FLAGS, PID, threadId, + MAGNIFICATION_SPEC, TRANSFORM_MATRIX); + return new String[0]; + } + + @Override + public boolean performAccessibilityAction(int accessibilityWindowId, long accessibilityNodeId, + int action, Bundle arguments, int interactionId, + IAccessibilityInteractionConnectionCallback callback, long threadId) + throws RemoteException { + mAccessibilityInteractionConnection.performAccessibilityAction(accessibilityNodeId, action, + arguments, interactionId, callback, FETCH_FLAGS, PID, threadId); + return true; + } +}