Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add AccessibilityDisplayProxy and register/unregister methods

Browse Source 
An A11yDisplayProxy will be used by Exo to listen to UI changes and
perform actions on the UI.

For a Display that is shown on a ChromeBook/client device,
a privileged app like Exo, or an app with SystemAPI privileges and
the MANAGE_ACCESSIBILITY permission, should call
A11yManager.registerA11yProxy and A11yManager.unRegisterA11yProxy to
enable/disable a11y.

The app will only have visibility in the specified proxy display.

Currently only few A11yProxy methods are exposed. When the proxy is
plugged into the system via ProxyAccessibilityServiceConnection
other methods will be exposed. (See link in comments)

Bug: 241429275
Test: atest A11yProxyTest(cts), atest A11yManagerTest(unit)
Change-Id: I019732e50e6beba0af7f5898a07b44d25e03416b
This commit is contained in:
sallyyuen
2022-10-17 14:45:35 -07:00
parent c81b5c685e
commit d86f7e729b
8 changed files with 314 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
core/api/system-current.txt
View File

@@ -15842,10 +15842,17 @@ package android.view {
package android.view.accessibility {
public abstract class AccessibilityDisplayProxy {
ctor public AccessibilityDisplayProxy(int, @NonNull java.util.concurrent.Executor, @NonNull java.util.List<android.accessibilityservice.AccessibilityServiceInfo>);
method public int getDisplayId();
}
public final class AccessibilityManager {
method public int getAccessibilityWindowId(@Nullable android.os.IBinder);
method @RequiresPermission(android.Manifest.permission.MANAGE_ACCESSIBILITY) public void performAccessibilityShortcut();
method @RequiresPermission(android.Manifest.permission.MANAGE_ACCESSIBILITY) public boolean registerDisplayProxy(@NonNull android.view.accessibility.AccessibilityDisplayProxy);
method @RequiresPermission(android.Manifest.permission.MANAGE_ACCESSIBILITY) public void registerSystemAction(@NonNull android.app.RemoteAction, int);
method @RequiresPermission(android.Manifest.permission.MANAGE_ACCESSIBILITY) public boolean unregisterDisplayProxy(@NonNull android.view.accessibility.AccessibilityDisplayProxy);
method @RequiresPermission(android.Manifest.permission.MANAGE_ACCESSIBILITY) public void unregisterSystemAction(int);
}

181
core/java/android/view/accessibility/AccessibilityDisplayProxy.java Normal file
View File

@@ -0,0 +1,181 @@
/*
* Copyright (C) 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package android.view.accessibility;
import android.accessibilityservice.AccessibilityGestureEvent;
import android.accessibilityservice.AccessibilityService;
import android.accessibilityservice.AccessibilityServiceInfo;
import android.accessibilityservice.IAccessibilityServiceClient;
import android.accessibilityservice.MagnificationConfig;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.annotation.SystemApi;
import android.content.Context;
import android.graphics.Region;
import android.os.IBinder;
import android.view.KeyEvent;
import android.view.MotionEvent;
import android.view.inputmethod.EditorInfo;
import com.android.internal.inputmethod.IAccessibilityInputMethodSessionCallback;
import com.android.internal.inputmethod.RemoteAccessibilityInputConnection;
import java.util.List;
import java.util.concurrent.Executor;
/**
* Allows a privileged app - an app with MANAGE_ACCESSIBILITY permission and SystemAPI access - to
* interact with the windows in the display that this proxy represents. Proxying the default display
* or a display that is not tracked will throw an exception. Only the real user has access to global
* clients like SystemUI.
*
* <p>
* To register and unregister a proxy, use
* {@link AccessibilityManager#registerDisplayProxy(AccessibilityDisplayProxy)}
* and {@link AccessibilityManager#unregisterDisplayProxy(AccessibilityDisplayProxy)}. If the app
* that has registered the proxy dies, the system will remove the proxy.
*
* TODO(241429275): Complete proxy impl and add additional support (if necessary) like cache methods
* @hide
*/
@SystemApi
public abstract class AccessibilityDisplayProxy {
private static final String LOG_TAG = "AccessibilityDisplayProxy";
private static final int INVALID_CONNECTION_ID = -1;
private List<AccessibilityServiceInfo> mInstalledAndEnabledServices;
private Executor mExecutor;
private int mConnectionId = INVALID_CONNECTION_ID;
private int mDisplayId;
IAccessibilityServiceClient mServiceClient;
/**
* Constructs an AccessibilityDisplayProxy instance.
* @param displayId the id of the display to proxy.
* @param executor the executor used to execute proxy callbacks.
* @param installedAndEnabledServices the list of infos representing the installed and
* enabled a11y services.
*/
public AccessibilityDisplayProxy(int displayId, @NonNull Executor executor,
@NonNull List<AccessibilityServiceInfo> installedAndEnabledServices) {
mDisplayId = displayId;
mExecutor = executor;
// Typically, the context is the Service context of an accessibility service.
// Context is used for ResolveInfo check, which a proxy won't have, IME input
// (FLAG_INPUT_METHOD_EDITOR), which the proxy doesn't need, and tracing
// A11yInteractionClient methods.
// TODO(254097475): Enable tracing, potentially without exposing Context.
mServiceClient = new IAccessibilityServiceClientImpl(null, mExecutor);
mInstalledAndEnabledServices = installedAndEnabledServices;
}
/**
* Returns the id of the display being proxy-ed.
*/
public int getDisplayId() {
return mDisplayId;
}
/**
* An IAccessibilityServiceClient that handles interrupts and accessibility events.
*/
private class IAccessibilityServiceClientImpl extends
AccessibilityService.IAccessibilityServiceClientWrapper {
IAccessibilityServiceClientImpl(Context context, Executor executor) {
super(context, executor, new AccessibilityService.Callbacks() {
@Override
public void onAccessibilityEvent(AccessibilityEvent event) {
// TODO: call AccessiiblityProxy.onAccessibilityEvent
}
@Override
public void onInterrupt() {
// TODO: call AccessiiblityProxy.onInterrupt
}
@Override
public void onServiceConnected() {
// TODO: send service infos and call AccessiiblityProxy.onProxyConnected
}
@Override
public void init(int connectionId, IBinder windowToken) {
mConnectionId = connectionId;
}
@Override
public boolean onGesture(AccessibilityGestureEvent gestureInfo) {
return false;
}
@Override
public boolean onKeyEvent(KeyEvent event) {
return false;
}
@Override
public void onMagnificationChanged(int displayId, @NonNull Region region,
MagnificationConfig config) {
}
@Override
public void onMotionEvent(MotionEvent event) {
}
@Override
public void onTouchStateChanged(int displayId, int state) {
}
@Override
public void onSoftKeyboardShowModeChanged(int showMode) {
}
@Override
public void onPerformGestureResult(int sequence, boolean completedSuccessfully) {
}
@Override
public void onFingerprintCapturingGesturesChanged(boolean active) {
}
@Override
public void onFingerprintGesture(int gesture) {
}
@Override
public void onAccessibilityButtonClicked(int displayId) {
}
@Override
public void onAccessibilityButtonAvailabilityChanged(boolean available) {
}
@Override
public void onSystemActionsChanged() {
}
@Override
public void createImeSession(IAccessibilityInputMethodSessionCallback callback) {
}
@Override
public void startInput(@Nullable RemoteAccessibilityInputConnection inputConnection,
@NonNull EditorInfo editorInfo, boolean restarting) {
}
});
}
}
}

61
core/java/android/view/accessibility/AccessibilityManager.java
View File

@@ -1921,6 +1921,67 @@ public final class AccessibilityManager {
}
}
/**
* Registers an {@link AccessibilityDisplayProxy}, so this proxy can access UI content specific
* to its display.
*
* @param proxy the {@link AccessibilityDisplayProxy} to register.
* @return {@code true} if the proxy is successfully registered.
*
* @throws IllegalArgumentException if the proxy's display is not currently tracked by a11y, is
* {@link android.view.Display#DEFAULT_DISPLAY}, is or lower than
* {@link android.view.Display#INVALID_DISPLAY}, or is already being proxy-ed.
*
* @throws SecurityException if the app does not hold the
* {@link Manifest.permission#MANAGE_ACCESSIBILITY} permission.
*
* @hide
*/
@SystemApi
@RequiresPermission(Manifest.permission.MANAGE_ACCESSIBILITY)
public boolean registerDisplayProxy(@NonNull AccessibilityDisplayProxy proxy) {
final IAccessibilityManager service;
synchronized (mLock) {
service = getServiceLocked();
if (service == null) {
return false;
}
}
try {
return service.registerProxyForDisplay(proxy.mServiceClient, proxy.getDisplayId());
} catch (RemoteException re) {
throw re.rethrowFromSystemServer();
}
}
/**
* Unregisters an {@link AccessibilityDisplayProxy}.
*
* @return {@code true} if the proxy is successfully unregistered.
*
* @throws SecurityException if the app does not hold the
* {@link Manifest.permission#MANAGE_ACCESSIBILITY} permission.
*
* @hide
*/
@SystemApi
@RequiresPermission(Manifest.permission.MANAGE_ACCESSIBILITY)
public boolean unregisterDisplayProxy(@NonNull AccessibilityDisplayProxy proxy) {
final IAccessibilityManager service;
synchronized (mLock) {
service = getServiceLocked();
if (service == null) {
return false;
}
}
try {
return service.unregisterProxyForDisplay(proxy.getDisplayId());
} catch (RemoteException re) {
throw re.rethrowFromSystemServer();
}
}
private IAccessibilityManager getServiceLocked() {
if (mService == null) {
tryConnectToServiceLocked(null);

4
core/java/android/view/accessibility/IAccessibilityManager.aidl
View File

@@ -109,9 +109,9 @@ interface IAccessibilityManager {
oneway void setAccessibilityWindowAttributes(int displayId, int windowId, int userId, in AccessibilityWindowAttributes attributes);
// Requires Manifest.permission.MANAGE_ACCESSIBILITY
@JavaPassthrough(annotation="@android.annotation.RequiresPermission(android.Manifest.permission.MANAGE_ACCESSIBILITY)")
boolean registerProxyForDisplay(IAccessibilityServiceClient proxy, int displayId);
// Requires Manifest.permission.MANAGE_ACCESSIBILITY
@JavaPassthrough(annotation="@android.annotation.RequiresPermission(android.Manifest.permission.MANAGE_ACCESSIBILITY)")
boolean unregisterProxyForDisplay(int displayId);
}

45
core/tests/coretests/src/android/view/accessibility/AccessibilityManagerTest.java
View File

@@ -27,6 +27,7 @@ import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import android.accessibilityservice.AccessibilityServiceInfo;
import android.accessibilityservice.IAccessibilityServiceClient;
import android.app.Instrumentation;
import android.app.PendingIntent;
import android.app.RemoteAction;
@@ -34,6 +35,7 @@ import android.content.Intent;
import android.graphics.drawable.Icon;
import android.os.UserHandle;
import androidx.annotation.NonNull;
import androidx.test.InstrumentationRegistry;
import androidx.test.runner.AndroidJUnit4;
@@ -51,6 +53,7 @@ import org.mockito.MockitoAnnotations;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Executors;
/**
* Tests for the AccessibilityManager by mocking the backing service.
@@ -70,6 +73,7 @@ public class AccessibilityManagerTest {
LABEL,
DESCRIPTION,
TEST_PENDING_INTENT);
private static final int DISPLAY_ID = 22;
@Mock private IAccessibilityManager mMockService;
private MessageCapturingHandler mHandler;
@@ -224,4 +228,45 @@ public class AccessibilityManagerTest {
assertEquals(mFocusColorDefaultValue,
manager.getAccessibilityFocusColor());
}
@Test
public void testRegisterAccessibilityProxy() throws Exception {
// Accessibility does not need to be enabled for a proxy to be registered.
AccessibilityManager manager =
new AccessibilityManager(mInstrumentation.getContext(), mHandler, mMockService,
UserHandle.USER_CURRENT, true);
ArrayList<AccessibilityServiceInfo> infos = new ArrayList<>();
infos.add(new AccessibilityServiceInfo());
AccessibilityDisplayProxy proxy = new MyAccessibilityProxy(DISPLAY_ID, infos);
manager.registerDisplayProxy(proxy);
// Cannot access proxy.mServiceClient directly due to visibility.
verify(mMockService).registerProxyForDisplay(any(IAccessibilityServiceClient.class),
any(Integer.class));
}
@Test
public void testUnregisterAccessibilityProxy() throws Exception {
// Accessibility does not need to be enabled for a proxy to be registered.
final AccessibilityManager manager =
new AccessibilityManager(mInstrumentation.getContext(), mHandler, mMockService,
UserHandle.USER_CURRENT, true);
final ArrayList<AccessibilityServiceInfo> infos = new ArrayList<>();
infos.add(new AccessibilityServiceInfo());
final AccessibilityDisplayProxy proxy = new MyAccessibilityProxy(DISPLAY_ID, infos);
manager.registerDisplayProxy(proxy);
manager.unregisterDisplayProxy(proxy);
verify(mMockService).unregisterProxyForDisplay(proxy.getDisplayId());
}
private class MyAccessibilityProxy extends AccessibilityDisplayProxy {
// TODO(241429275): Will override A11yProxy methods in the future.
MyAccessibilityProxy(int displayId,
@NonNull List<AccessibilityServiceInfo> serviceInfos) {
super(displayId, Executors.newSingleThreadExecutor(), serviceInfos);
}
}
}

4
services/accessibility/java/com/android/server/accessibility/AccessibilityManagerService.java
View File

@@ -3622,6 +3622,10 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub
throw new IllegalArgumentException("The display " + displayId + " does not exist or is"
+ " not tracked by accessibility.");
}
if (mProxyManager.isProxyed(displayId)) {
throw new IllegalArgumentException("The display " + displayId + " is already being"
+ "proxy-ed");
}
mProxyManager.registerProxy(client, displayId);
return true;

3
services/accessibility/java/com/android/server/accessibility/ProxyAccessibilityServiceConnection.java
View File

@@ -32,6 +32,7 @@ import android.os.Handler;
import android.os.IBinder;
import android.os.RemoteCallback;
import android.view.KeyEvent;
import android.view.accessibility.AccessibilityDisplayProxy;
import android.view.accessibility.AccessibilityNodeInfo;
import androidx.annotation.Nullable;
@@ -44,7 +45,7 @@ import java.util.List;
import java.util.Set;
/**
* Represents the system connection to an {@link android.view.accessibility.AccessibilityProxy}.
* Represents the system connection to an {@link AccessibilityDisplayProxy}.
*
* <p>Most methods are no-ops since this connection does not need to capture input or listen to
* hardware-related changes.

12
services/accessibility/java/com/android/server/accessibility/ProxyManager.java
View File

@@ -16,6 +16,8 @@
package com.android.server.accessibility;
import android.accessibilityservice.IAccessibilityServiceClient;
import java.util.HashSet;
/**
* Manages proxy connections.
*
@@ -26,6 +28,7 @@ import android.accessibilityservice.IAccessibilityServiceClient;
*/
public class ProxyManager {
private final Object mLock;
private final HashSet<Integer> mDisplayIds = new HashSet<>();
ProxyManager(Object lock) {
mLock = lock;
@@ -35,12 +38,21 @@ public class ProxyManager {
* TODO: Create the proxy service connection.
*/
public void registerProxy(IAccessibilityServiceClient client, int displayId) {
mDisplayIds.add(displayId);
}
/**
* TODO: Unregister the proxy service connection based on display id.
*/
public boolean unregisterProxy(int displayId) {
mDisplayIds.remove(displayId);
return true;
}
/**
* Checks if a display id is being proxy-ed.
*/
public boolean isProxyed(int displayId) {
return mDisplayIds.contains(displayId);
}
}