From d86f7e729ba031acbfcb81c4d1e011ffdc596537 Mon Sep 17 00:00:00 2001 From: sallyyuen Date: Mon, 17 Oct 2022 14:45:35 -0700 Subject: [PATCH] Add AccessibilityDisplayProxy and register/unregister methods An A11yDisplayProxy will be used by Exo to listen to UI changes and perform actions on the UI. For a Display that is shown on a ChromeBook/client device, a privileged app like Exo, or an app with SystemAPI privileges and the MANAGE_ACCESSIBILITY permission, should call A11yManager.registerA11yProxy and A11yManager.unRegisterA11yProxy to enable/disable a11y. The app will only have visibility in the specified proxy display. Currently only few A11yProxy methods are exposed. When the proxy is plugged into the system via ProxyAccessibilityServiceConnection other methods will be exposed. (See link in comments) Bug: 241429275 Test: atest A11yProxyTest(cts), atest A11yManagerTest(unit) Change-Id: I019732e50e6beba0af7f5898a07b44d25e03416b --- core/api/system-current.txt | 7 + .../AccessibilityDisplayProxy.java | 181 ++++++++++++++++++ .../accessibility/AccessibilityManager.java | 61 ++++++ .../accessibility/IAccessibilityManager.aidl | 4 +- .../AccessibilityManagerTest.java | 45 +++++ .../AccessibilityManagerService.java | 4 + .../ProxyAccessibilityServiceConnection.java | 3 +- .../server/accessibility/ProxyManager.java | 12 ++ 8 files changed, 314 insertions(+), 3 deletions(-) create mode 100644 core/java/android/view/accessibility/AccessibilityDisplayProxy.java diff --git a/core/api/system-current.txt b/core/api/system-current.txt index 752b524378a96..936d57f950059 100644 --- a/core/api/system-current.txt +++ b/core/api/system-current.txt @@ -15842,10 +15842,17 @@ package android.view { package android.view.accessibility { + public abstract class AccessibilityDisplayProxy { + ctor public AccessibilityDisplayProxy(int, @NonNull java.util.concurrent.Executor, @NonNull java.util.List); + method public int getDisplayId(); + } + public final class AccessibilityManager { method public int getAccessibilityWindowId(@Nullable android.os.IBinder); method @RequiresPermission(android.Manifest.permission.MANAGE_ACCESSIBILITY) public void performAccessibilityShortcut(); + method @RequiresPermission(android.Manifest.permission.MANAGE_ACCESSIBILITY) public boolean registerDisplayProxy(@NonNull android.view.accessibility.AccessibilityDisplayProxy); method @RequiresPermission(android.Manifest.permission.MANAGE_ACCESSIBILITY) public void registerSystemAction(@NonNull android.app.RemoteAction, int); + method @RequiresPermission(android.Manifest.permission.MANAGE_ACCESSIBILITY) public boolean unregisterDisplayProxy(@NonNull android.view.accessibility.AccessibilityDisplayProxy); method @RequiresPermission(android.Manifest.permission.MANAGE_ACCESSIBILITY) public void unregisterSystemAction(int); } diff --git a/core/java/android/view/accessibility/AccessibilityDisplayProxy.java b/core/java/android/view/accessibility/AccessibilityDisplayProxy.java new file mode 100644 index 0000000000000..85f5056e41160 --- /dev/null +++ b/core/java/android/view/accessibility/AccessibilityDisplayProxy.java @@ -0,0 +1,181 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.view.accessibility; + +import android.accessibilityservice.AccessibilityGestureEvent; +import android.accessibilityservice.AccessibilityService; +import android.accessibilityservice.AccessibilityServiceInfo; +import android.accessibilityservice.IAccessibilityServiceClient; +import android.accessibilityservice.MagnificationConfig; +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.annotation.SystemApi; +import android.content.Context; +import android.graphics.Region; +import android.os.IBinder; +import android.view.KeyEvent; +import android.view.MotionEvent; +import android.view.inputmethod.EditorInfo; + +import com.android.internal.inputmethod.IAccessibilityInputMethodSessionCallback; +import com.android.internal.inputmethod.RemoteAccessibilityInputConnection; + +import java.util.List; +import java.util.concurrent.Executor; + +/** + * Allows a privileged app - an app with MANAGE_ACCESSIBILITY permission and SystemAPI access - to + * interact with the windows in the display that this proxy represents. Proxying the default display + * or a display that is not tracked will throw an exception. Only the real user has access to global + * clients like SystemUI. + * + *

+ * To register and unregister a proxy, use + * {@link AccessibilityManager#registerDisplayProxy(AccessibilityDisplayProxy)} + * and {@link AccessibilityManager#unregisterDisplayProxy(AccessibilityDisplayProxy)}. If the app + * that has registered the proxy dies, the system will remove the proxy. + * + * TODO(241429275): Complete proxy impl and add additional support (if necessary) like cache methods + * @hide + */ +@SystemApi +public abstract class AccessibilityDisplayProxy { + private static final String LOG_TAG = "AccessibilityDisplayProxy"; + private static final int INVALID_CONNECTION_ID = -1; + + private List mInstalledAndEnabledServices; + private Executor mExecutor; + private int mConnectionId = INVALID_CONNECTION_ID; + private int mDisplayId; + IAccessibilityServiceClient mServiceClient; + + /** + * Constructs an AccessibilityDisplayProxy instance. + * @param displayId the id of the display to proxy. + * @param executor the executor used to execute proxy callbacks. + * @param installedAndEnabledServices the list of infos representing the installed and + * enabled a11y services. + */ + public AccessibilityDisplayProxy(int displayId, @NonNull Executor executor, + @NonNull List installedAndEnabledServices) { + mDisplayId = displayId; + mExecutor = executor; + // Typically, the context is the Service context of an accessibility service. + // Context is used for ResolveInfo check, which a proxy won't have, IME input + // (FLAG_INPUT_METHOD_EDITOR), which the proxy doesn't need, and tracing + // A11yInteractionClient methods. + // TODO(254097475): Enable tracing, potentially without exposing Context. + mServiceClient = new IAccessibilityServiceClientImpl(null, mExecutor); + mInstalledAndEnabledServices = installedAndEnabledServices; + } + + /** + * Returns the id of the display being proxy-ed. + */ + public int getDisplayId() { + return mDisplayId; + } + + /** + * An IAccessibilityServiceClient that handles interrupts and accessibility events. + */ + private class IAccessibilityServiceClientImpl extends + AccessibilityService.IAccessibilityServiceClientWrapper { + + IAccessibilityServiceClientImpl(Context context, Executor executor) { + super(context, executor, new AccessibilityService.Callbacks() { + @Override + public void onAccessibilityEvent(AccessibilityEvent event) { + // TODO: call AccessiiblityProxy.onAccessibilityEvent + } + + @Override + public void onInterrupt() { + // TODO: call AccessiiblityProxy.onInterrupt + } + @Override + public void onServiceConnected() { + // TODO: send service infos and call AccessiiblityProxy.onProxyConnected + } + @Override + public void init(int connectionId, IBinder windowToken) { + mConnectionId = connectionId; + } + + @Override + public boolean onGesture(AccessibilityGestureEvent gestureInfo) { + return false; + } + + @Override + public boolean onKeyEvent(KeyEvent event) { + return false; + } + + @Override + public void onMagnificationChanged(int displayId, @NonNull Region region, + MagnificationConfig config) { + } + + @Override + public void onMotionEvent(MotionEvent event) { + } + + @Override + public void onTouchStateChanged(int displayId, int state) { + } + + @Override + public void onSoftKeyboardShowModeChanged(int showMode) { + } + + @Override + public void onPerformGestureResult(int sequence, boolean completedSuccessfully) { + } + + @Override + public void onFingerprintCapturingGesturesChanged(boolean active) { + } + + @Override + public void onFingerprintGesture(int gesture) { + } + + @Override + public void onAccessibilityButtonClicked(int displayId) { + } + + @Override + public void onAccessibilityButtonAvailabilityChanged(boolean available) { + } + + @Override + public void onSystemActionsChanged() { + } + + @Override + public void createImeSession(IAccessibilityInputMethodSessionCallback callback) { + } + + @Override + public void startInput(@Nullable RemoteAccessibilityInputConnection inputConnection, + @NonNull EditorInfo editorInfo, boolean restarting) { + } + }); + } + } +} diff --git a/core/java/android/view/accessibility/AccessibilityManager.java b/core/java/android/view/accessibility/AccessibilityManager.java index 5433fa08ac18a..423c560d5c572 100644 --- a/core/java/android/view/accessibility/AccessibilityManager.java +++ b/core/java/android/view/accessibility/AccessibilityManager.java @@ -1921,6 +1921,67 @@ public final class AccessibilityManager { } } + /** + * Registers an {@link AccessibilityDisplayProxy}, so this proxy can access UI content specific + * to its display. + * + * @param proxy the {@link AccessibilityDisplayProxy} to register. + * @return {@code true} if the proxy is successfully registered. + * + * @throws IllegalArgumentException if the proxy's display is not currently tracked by a11y, is + * {@link android.view.Display#DEFAULT_DISPLAY}, is or lower than + * {@link android.view.Display#INVALID_DISPLAY}, or is already being proxy-ed. + * + * @throws SecurityException if the app does not hold the + * {@link Manifest.permission#MANAGE_ACCESSIBILITY} permission. + * + * @hide + */ + @SystemApi + @RequiresPermission(Manifest.permission.MANAGE_ACCESSIBILITY) + public boolean registerDisplayProxy(@NonNull AccessibilityDisplayProxy proxy) { + final IAccessibilityManager service; + synchronized (mLock) { + service = getServiceLocked(); + if (service == null) { + return false; + } + } + + try { + return service.registerProxyForDisplay(proxy.mServiceClient, proxy.getDisplayId()); + } catch (RemoteException re) { + throw re.rethrowFromSystemServer(); + } + } + + /** + * Unregisters an {@link AccessibilityDisplayProxy}. + * + * @return {@code true} if the proxy is successfully unregistered. + * + * @throws SecurityException if the app does not hold the + * {@link Manifest.permission#MANAGE_ACCESSIBILITY} permission. + * + * @hide + */ + @SystemApi + @RequiresPermission(Manifest.permission.MANAGE_ACCESSIBILITY) + public boolean unregisterDisplayProxy(@NonNull AccessibilityDisplayProxy proxy) { + final IAccessibilityManager service; + synchronized (mLock) { + service = getServiceLocked(); + if (service == null) { + return false; + } + } + try { + return service.unregisterProxyForDisplay(proxy.getDisplayId()); + } catch (RemoteException re) { + throw re.rethrowFromSystemServer(); + } + } + private IAccessibilityManager getServiceLocked() { if (mService == null) { tryConnectToServiceLocked(null); diff --git a/core/java/android/view/accessibility/IAccessibilityManager.aidl b/core/java/android/view/accessibility/IAccessibilityManager.aidl index 36fdcce4e1f25..a25194804987d 100644 --- a/core/java/android/view/accessibility/IAccessibilityManager.aidl +++ b/core/java/android/view/accessibility/IAccessibilityManager.aidl @@ -109,9 +109,9 @@ interface IAccessibilityManager { oneway void setAccessibilityWindowAttributes(int displayId, int windowId, int userId, in AccessibilityWindowAttributes attributes); - // Requires Manifest.permission.MANAGE_ACCESSIBILITY + @JavaPassthrough(annotation="@android.annotation.RequiresPermission(android.Manifest.permission.MANAGE_ACCESSIBILITY)") boolean registerProxyForDisplay(IAccessibilityServiceClient proxy, int displayId); - // Requires Manifest.permission.MANAGE_ACCESSIBILITY + @JavaPassthrough(annotation="@android.annotation.RequiresPermission(android.Manifest.permission.MANAGE_ACCESSIBILITY)") boolean unregisterProxyForDisplay(int displayId); } diff --git a/core/tests/coretests/src/android/view/accessibility/AccessibilityManagerTest.java b/core/tests/coretests/src/android/view/accessibility/AccessibilityManagerTest.java index bb1a3b182f91d..ee1e10f9009ea 100644 --- a/core/tests/coretests/src/android/view/accessibility/AccessibilityManagerTest.java +++ b/core/tests/coretests/src/android/view/accessibility/AccessibilityManagerTest.java @@ -27,6 +27,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import android.accessibilityservice.AccessibilityServiceInfo; +import android.accessibilityservice.IAccessibilityServiceClient; import android.app.Instrumentation; import android.app.PendingIntent; import android.app.RemoteAction; @@ -34,6 +35,7 @@ import android.content.Intent; import android.graphics.drawable.Icon; import android.os.UserHandle; +import androidx.annotation.NonNull; import androidx.test.InstrumentationRegistry; import androidx.test.runner.AndroidJUnit4; @@ -51,6 +53,7 @@ import org.mockito.MockitoAnnotations; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.Executors; /** * Tests for the AccessibilityManager by mocking the backing service. @@ -70,6 +73,7 @@ public class AccessibilityManagerTest { LABEL, DESCRIPTION, TEST_PENDING_INTENT); + private static final int DISPLAY_ID = 22; @Mock private IAccessibilityManager mMockService; private MessageCapturingHandler mHandler; @@ -224,4 +228,45 @@ public class AccessibilityManagerTest { assertEquals(mFocusColorDefaultValue, manager.getAccessibilityFocusColor()); } + + @Test + public void testRegisterAccessibilityProxy() throws Exception { + // Accessibility does not need to be enabled for a proxy to be registered. + AccessibilityManager manager = + new AccessibilityManager(mInstrumentation.getContext(), mHandler, mMockService, + UserHandle.USER_CURRENT, true); + + + ArrayList infos = new ArrayList<>(); + infos.add(new AccessibilityServiceInfo()); + AccessibilityDisplayProxy proxy = new MyAccessibilityProxy(DISPLAY_ID, infos); + manager.registerDisplayProxy(proxy); + // Cannot access proxy.mServiceClient directly due to visibility. + verify(mMockService).registerProxyForDisplay(any(IAccessibilityServiceClient.class), + any(Integer.class)); + } + + @Test + public void testUnregisterAccessibilityProxy() throws Exception { + // Accessibility does not need to be enabled for a proxy to be registered. + final AccessibilityManager manager = + new AccessibilityManager(mInstrumentation.getContext(), mHandler, mMockService, + UserHandle.USER_CURRENT, true); + + final ArrayList infos = new ArrayList<>(); + infos.add(new AccessibilityServiceInfo()); + + final AccessibilityDisplayProxy proxy = new MyAccessibilityProxy(DISPLAY_ID, infos); + manager.registerDisplayProxy(proxy); + manager.unregisterDisplayProxy(proxy); + verify(mMockService).unregisterProxyForDisplay(proxy.getDisplayId()); + } + + private class MyAccessibilityProxy extends AccessibilityDisplayProxy { + // TODO(241429275): Will override A11yProxy methods in the future. + MyAccessibilityProxy(int displayId, + @NonNull List serviceInfos) { + super(displayId, Executors.newSingleThreadExecutor(), serviceInfos); + } + } } diff --git a/services/accessibility/java/com/android/server/accessibility/AccessibilityManagerService.java b/services/accessibility/java/com/android/server/accessibility/AccessibilityManagerService.java index 365068d22171e..1c1f84b96bc3e 100644 --- a/services/accessibility/java/com/android/server/accessibility/AccessibilityManagerService.java +++ b/services/accessibility/java/com/android/server/accessibility/AccessibilityManagerService.java @@ -3622,6 +3622,10 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub throw new IllegalArgumentException("The display " + displayId + " does not exist or is" + " not tracked by accessibility."); } + if (mProxyManager.isProxyed(displayId)) { + throw new IllegalArgumentException("The display " + displayId + " is already being" + + "proxy-ed"); + } mProxyManager.registerProxy(client, displayId); return true; diff --git a/services/accessibility/java/com/android/server/accessibility/ProxyAccessibilityServiceConnection.java b/services/accessibility/java/com/android/server/accessibility/ProxyAccessibilityServiceConnection.java index 934b665d7dae0..247f320adecd9 100644 --- a/services/accessibility/java/com/android/server/accessibility/ProxyAccessibilityServiceConnection.java +++ b/services/accessibility/java/com/android/server/accessibility/ProxyAccessibilityServiceConnection.java @@ -32,6 +32,7 @@ import android.os.Handler; import android.os.IBinder; import android.os.RemoteCallback; import android.view.KeyEvent; +import android.view.accessibility.AccessibilityDisplayProxy; import android.view.accessibility.AccessibilityNodeInfo; import androidx.annotation.Nullable; @@ -44,7 +45,7 @@ import java.util.List; import java.util.Set; /** - * Represents the system connection to an {@link android.view.accessibility.AccessibilityProxy}. + * Represents the system connection to an {@link AccessibilityDisplayProxy}. * *

Most methods are no-ops since this connection does not need to capture input or listen to * hardware-related changes. diff --git a/services/accessibility/java/com/android/server/accessibility/ProxyManager.java b/services/accessibility/java/com/android/server/accessibility/ProxyManager.java index fb0b8f3b17b17..a2ce61063aaf3 100644 --- a/services/accessibility/java/com/android/server/accessibility/ProxyManager.java +++ b/services/accessibility/java/com/android/server/accessibility/ProxyManager.java @@ -16,6 +16,8 @@ package com.android.server.accessibility; import android.accessibilityservice.IAccessibilityServiceClient; +import java.util.HashSet; + /** * Manages proxy connections. * @@ -26,6 +28,7 @@ import android.accessibilityservice.IAccessibilityServiceClient; */ public class ProxyManager { private final Object mLock; + private final HashSet mDisplayIds = new HashSet<>(); ProxyManager(Object lock) { mLock = lock; @@ -35,12 +38,21 @@ public class ProxyManager { * TODO: Create the proxy service connection. */ public void registerProxy(IAccessibilityServiceClient client, int displayId) { + mDisplayIds.add(displayId); } /** * TODO: Unregister the proxy service connection based on display id. */ public boolean unregisterProxy(int displayId) { + mDisplayIds.remove(displayId); return true; } + + /** + * Checks if a display id is being proxy-ed. + */ + public boolean isProxyed(int displayId) { + return mDisplayIds.contains(displayId); + } }