Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge "Add TouchInteractionController API to allow gesture detection from the service."

Browse Source
This commit is contained in:
Ameer Armaly
2021-11-05 22:23:39 +00:00
committed by Android (Google) Code Review
parent 8d53f84f35 a558ed6808
commit 927b5744d1
16 changed files with 1156 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
core/api/current.txt
View File

@@ -2994,6 +2994,7 @@ package android.accessibilityservice {
}
public final class AccessibilityGestureEvent implements android.os.Parcelable {
ctor public AccessibilityGestureEvent(int, int, @NonNull java.util.List<android.view.MotionEvent>);
method public int describeContents();
method @NonNull public static String gestureIdToString(int);
method public int getDisplayId();
@@ -3016,6 +3017,7 @@ package android.accessibilityservice {
method public final android.accessibilityservice.AccessibilityServiceInfo getServiceInfo();
method @NonNull public final android.accessibilityservice.AccessibilityService.SoftKeyboardController getSoftKeyboardController();
method @NonNull public final java.util.List<android.view.accessibility.AccessibilityNodeInfo.AccessibilityAction> getSystemActions();
method @NonNull public final android.accessibilityservice.TouchInteractionController getTouchInteractionController(int);
method public java.util.List<android.view.accessibility.AccessibilityWindowInfo> getWindows();
method @NonNull public final android.util.SparseArray<java.util.List<android.view.accessibility.AccessibilityWindowInfo>> getWindowsOnAllDisplays();
method public abstract void onAccessibilityEvent(android.view.accessibility.AccessibilityEvent);
@@ -3250,6 +3252,31 @@ package android.accessibilityservice {
method public boolean willContinue();
}
public final class TouchInteractionController {
method public void addListener(@Nullable java.util.concurrent.Executor, @NonNull android.accessibilityservice.TouchInteractionController.Listener);
method public int getDisplayId();
method public int getMaxPointerCount();
method public int getState();
method public void performClick();
method public void performLongClickAndStartDrag();
method public void removeAllListeners();
method public boolean removeListener(@NonNull android.accessibilityservice.TouchInteractionController.Listener);
method public void requestDelegating();
method public void requestDragging(int);
method public void requestTouchExploration();
method @NonNull public static String stateToString(int);
field public static final int STATE_CLEAR = 0; // 0x0
field public static final int STATE_DELEGATING = 4; // 0x4
field public static final int STATE_DRAGGING = 3; // 0x3
field public static final int STATE_TOUCH_EXPLORING = 2; // 0x2
field public static final int STATE_TOUCH_INTERACTING = 1; // 0x1
}
public static interface TouchInteractionController.Listener {
method public void onMotionEvent(@NonNull android.view.MotionEvent);
method public void onStateChanged(int);
}
}
package android.accounts {

7
core/java/android/accessibilityservice/AccessibilityGestureEvent.java
View File

@@ -150,7 +150,12 @@ public final class AccessibilityGestureEvent implements Parcelable {
private final int mDisplayId;
private List<MotionEvent> mMotionEvents = new ArrayList<>();
/** @hide */
/**
* Constructs an AccessibilityGestureEvent to be dispatched to an accessibility service.
* @param gestureId the id number of the gesture.
* @param displayId the display on which this gesture was performed.
* @param motionEvents the motion events that lead to this gesture.
*/
public AccessibilityGestureEvent(
int gestureId, int displayId, @NonNull List<MotionEvent> motionEvents) {
mGestureId = gestureId;

86
core/java/android/accessibilityservice/AccessibilityService.java
View File

@@ -53,6 +53,7 @@ import android.util.Slog;
import android.util.SparseArray;
import android.view.Display;
import android.view.KeyEvent;
import android.view.MotionEvent;
import android.view.SurfaceView;
import android.view.WindowManager;
import android.view.WindowManagerImpl;
@@ -65,6 +66,7 @@ import android.view.accessibility.AccessibilityWindowInfo;
import com.android.internal.os.HandlerCaller;
import com.android.internal.os.SomeArgs;
import com.android.internal.util.Preconditions;
import com.android.internal.util.function.pooled.PooledLambda;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
@@ -582,6 +584,10 @@ public abstract class AccessibilityService extends Service {
/** Magnification changed callbacks for different displays */
void onMagnificationChanged(int displayId, @NonNull Region region,
float scale, float centerX, float centerY);
/** Callbacks for receiving motion events. */
void onMotionEvent(MotionEvent event);
/** Callback for tuch state changes. */
void onTouchStateChanged(int displayId, int state);
void onSoftKeyboardShowModeChanged(int showMode);
void onPerformGestureResult(int sequence, boolean completedSuccessfully);
void onFingerprintCapturingGesturesChanged(boolean active);
@@ -720,6 +726,12 @@ public abstract class AccessibilityService extends Service {
/** List of magnification controllers, mapping from displayId -> MagnificationController. */
private final SparseArray<MagnificationController> mMagnificationControllers =
new SparseArray<>(0);
/**
* List of touch interaction controllers, mapping from displayId -> TouchInteractionController.
*/
private final SparseArray<TouchInteractionController> mTouchInteractionControllers =
new SparseArray<>(0);
private SoftKeyboardController mSoftKeyboardController;
private final SparseArray<AccessibilityButtonController> mAccessibilityButtonControllers =
new SparseArray<>(0);
@@ -1194,6 +1206,10 @@ public abstract class AccessibilityService extends Service {
getFingerprintGestureController().onGesture(gesture);
}
int getConnectionId() {
return mConnectionId;
}
/**
* Used to control and query the state of display magnification.
*/
@@ -2209,6 +2225,16 @@ public abstract class AccessibilityService extends Service {
centerX, centerY);
}
@Override
public void onMotionEvent(MotionEvent event) {
AccessibilityService.this.onMotionEvent(event);
}
@Override
public void onTouchStateChanged(int displayId, int state) {
AccessibilityService.this.onTouchStateChanged(displayId, state);
}
@Override
public void onSoftKeyboardShowModeChanged(int showMode) {
AccessibilityService.this.onSoftKeyboardShowModeChanged(showMode);
@@ -2371,6 +2397,21 @@ public abstract class AccessibilityService extends Service {
mCaller.sendMessage(mCaller.obtainMessage(DO_ON_SYSTEM_ACTIONS_CHANGED));
}
@Override
public void onMotionEvent(MotionEvent event) {
final Message message = PooledLambda.obtainMessage(
Callbacks::onMotionEvent, mCallback, event);
mCaller.sendMessage(message);
}
@Override
public void onTouchStateChanged(int displayId, int state) {
final Message message = PooledLambda.obtainMessage(Callbacks::onTouchStateChanged,
mCallback,
displayId, state);
mCaller.sendMessage(message);
}
@Override
public void executeMessage(Message message) {
switch (message.what) {
@@ -2523,7 +2564,7 @@ public abstract class AccessibilityService extends Service {
}
return;
}
default :
default:
Log.w(LOG_TAG, "Unknown message type " + message.what);
}
}
@@ -2748,4 +2789,47 @@ public abstract class AccessibilityService extends Service {
}
}
}
/**
* Returns the touch interaction controller for the specified logical display, which may be used
* to detect gestures and otherwise control touch interactions. If
* {@link AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE} is disabled the
* controller's methods will have no effect.
*
* @param displayId The logical display id, use {@link Display#DEFAULT_DISPLAY} for default
* display.
* @return the TouchExploration controller
*/
@NonNull
public final TouchInteractionController getTouchInteractionController(int displayId) {
synchronized (mLock) {
TouchInteractionController controller = mTouchInteractionControllers.get(displayId);
if (controller == null) {
controller = new TouchInteractionController(this, mLock, displayId);
mTouchInteractionControllers.put(displayId, controller);
}
return controller;
}
}
void onMotionEvent(MotionEvent event) {
TouchInteractionController controller;
synchronized (mLock) {
int displayId = event.getDisplayId();
controller = mTouchInteractionControllers.get(displayId);
}
if (controller != null) {
controller.onMotionEvent(event);
}
}
void onTouchStateChanged(int displayId, int state) {
TouchInteractionController controller;
synchronized (mLock) {
controller = mTouchInteractionControllers.get(displayId);
}
if (controller != null) {
controller.onStateChanged(state);
}
}
}

5
core/java/android/accessibilityservice/IAccessibilityServiceClient.aidl
View File

@@ -22,6 +22,7 @@ import android.view.accessibility.AccessibilityEvent;
import android.view.accessibility.AccessibilityWindowInfo;
import android.accessibilityservice.AccessibilityGestureEvent;
import android.view.KeyEvent;
import android.view.MotionEvent;
/**
* Top-level interface to an accessibility service component.
@@ -44,6 +45,10 @@ import android.view.KeyEvent;
void onMagnificationChanged(int displayId, in Region region, float scale, float centerX, float centerY);
void onMotionEvent(in MotionEvent event);
void onTouchStateChanged(int displayId, int state);
void onSoftKeyboardShowModeChanged(int showMode);
void onPerformGestureResult(int sequence, boolean completedSuccessfully);

12
core/java/android/accessibilityservice/IAccessibilityServiceConnection.aidl
View File

@@ -122,4 +122,16 @@ interface IAccessibilityServiceConnection {
oneway void logTrace(long timestamp, String where, long loggingTypes, String callingParams,
int processId, long threadId, int callingUid, in Bundle serializedCallingStackInBundle);
void setServiceDetectsGesturesEnabled(int displayId, boolean mode);
void requestTouchExploration(int displayId);
void requestDragging(int displayId, int pointerId);
void requestDelegating(int displayId);
void onDoubleTap(int displayId);
void onDoubleTapAndHold(int displayId);
}

425
core/java/android/accessibilityservice/TouchInteractionController.java Normal file
View File

@@ -0,0 +1,425 @@
/*
* Copyright (C) 2021 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package android.accessibilityservice;
import android.annotation.IntDef;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.os.RemoteException;
import android.util.ArrayMap;
import android.view.MotionEvent;
import android.view.accessibility.AccessibilityInteractionClient;
import java.util.concurrent.Executor;
/**
* This class allows a service to handle touch exploration and the detection of specialized
* accessibility gestures. The service receives motion events and can match those motion events
* against the gestures it supports. The service can also request the framework enter three other
* states of operation for the duration of this interaction. Upon entering any of these states the
* framework will take over and the service will not receive motion events until the start of a new
* interaction. The states are as follows:
*
* <ul>
* <li>The service can tell the framework that this interaction is touch exploration. The user is
* trying to explore the screen rather than manipulate it. The framework will then convert the
* motion events to hover events to support touch exploration.
* <li>The service can tell the framework that this interaction is a dragging interaction where
* two fingers are used to execute a one-finger gesture such as scrolling the screen. The
* service must specify which of the two fingers should be passed through to rest of the input
* pipeline.
* <li>Finally, the service can request that the framework delegate this interaction, meaning pass
* it through to the rest of the input pipeline as-is.
* </ul>
*
* When {@link AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE } is enabled, this
* controller will receive all motion events received by the framework for the specified display
* when not touch-exploring or delegating. If the service classifies this interaction as touch
* exploration or delegating the framework will stop sending motion events to the service for the
* duration of this interaction. If the service classifies this interaction as a dragging
* interaction the framework will send motion events to the service to allow the service to
* determine if the interaction still qualifies as dragging or if it has become a delegating
* interaction. If {@link AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE } is disabled
* this controller will not receive any motion events because touch interactions are being passed
* through to the input pipeline unaltered.
* Note that {@link AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE }
* requires setting {@link android.R.attr#canRequestTouchExplorationMode} as well.
*/
public final class TouchInteractionController {
/** The state where the user is not touching the screen. */
public static final int STATE_CLEAR = 0;
/**
* The state where the user is touching the screen and the service is receiving motion events.
*/
public static final int STATE_TOUCH_INTERACTING = 1;
/**
* The state where the user is explicitly exploring the screen. The service is not receiving
* motion events.
*/
public static final int STATE_TOUCH_EXPLORING = 2;
/**
* The state where the user is dragging with two fingers. The service is not receiving motion
* events. The selected finger is being dispatched to the rest of the input pipeline to execute
* the drag.
*/
public static final int STATE_DRAGGING = 3;
/**
* The user is performing a gesture which is being passed through to the input pipeline as-is.
* The service is not receiving motion events.
*/
public static final int STATE_DELEGATING = 4;
@IntDef({
STATE_CLEAR,
STATE_TOUCH_INTERACTING,
STATE_TOUCH_EXPLORING,
STATE_DRAGGING,
STATE_DELEGATING
})
private @interface State {}
// The maximum number of pointers that can be touching the screen at once. (See MAX_POINTER_ID
// in frameworks/native/include/input/Input.h)
private static final int MAX_POINTER_COUNT = 32;
private final AccessibilityService mService;
private final Object mLock;
private final int mDisplayId;
private boolean mServiceDetectsGestures;
/** Map of listeners to executors. Lazily created when adding the first listener. */
private ArrayMap<Listener, Executor> mListeners;
// The current state of the display.
private int mState = STATE_CLEAR;
TouchInteractionController(
@NonNull AccessibilityService service, @NonNull Object lock, int displayId) {
mDisplayId = displayId;
mLock = lock;
mService = service;
}
/**
* Adds the specified change listener to the list of motion event listeners. The callback will
* run using on the specified {@link Executor}', or on the service's main thread if the
* Executor is {@code null}.
* @param listener the listener to add, must be non-null
* @param executor the executor for this callback, or {@code null} to execute on the service's
* main thread
*/
public void addListener(@Nullable Executor executor, @NonNull Listener listener) {
synchronized (mLock) {
if (mListeners == null) {
mListeners = new ArrayMap<>();
}
mListeners.put(listener, executor);
if (mListeners.size() == 1) {
setServiceDetectsGestures(true);
}
}
}
/**
* Removes the specified listener from the list of motion event listeners.
*
* @param listener the listener to remove, must be non-null
* @return {@code true} if the listener was removed, {@code false} otherwise
*/
public boolean removeListener(@NonNull Listener listener) {
if (mListeners == null) {
return false;
}
synchronized (mLock) {
boolean result = mListeners.remove(listener) != null;
if (result && mListeners.size() == 0) {
setServiceDetectsGestures(false);
}
return result;
}
}
/**
* Removes all listeners and returns control of touch interactions to the framework.
*/
public void removeAllListeners() {
if (mListeners != null) {
synchronized (mLock) {
mListeners.clear();
setServiceDetectsGestures(false);
}
}
}
/**
* Dispatches motion events to any registered listeners. This should be called on the service's
* main thread.
*/
void onMotionEvent(MotionEvent event) {
final ArrayMap<Listener, Executor> entries;
synchronized (mLock) {
// Listeners may remove themselves. Perform a shallow copy to avoid concurrent
// modification.
entries = new ArrayMap<>(mListeners);
}
for (int i = 0, count = entries.size(); i < count; i++) {
final Listener listener = entries.keyAt(i);
final Executor executor = entries.valueAt(i);
if (executor != null) {
executor.execute(() -> listener.onMotionEvent(event));
} else {
// We're already on the main thread, just run the listener.
listener.onMotionEvent(event);
}
}
}
/**
* Dispatches motion events to any registered listeners. This should be called on the service's
* main thread.
*/
void onStateChanged(@State int state) {
mState = state;
final ArrayMap<Listener, Executor> entries;
synchronized (mLock) {
// Listeners may remove themselves. Perform a shallow copy to avoid concurrent
// modification.
entries = new ArrayMap<>(mListeners);
}
for (int i = 0, count = entries.size(); i < count; i++) {
final Listener listener = entries.keyAt(i);
final Executor executor = entries.valueAt(i);
if (executor != null) {
executor.execute(() -> listener.onStateChanged(state));
} else {
// We're already on the main thread, just run the listener.
listener.onStateChanged(state);
}
}
}
/**
* When {@link AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE} is enabled, this
* controller will receive all motion events received by the framework for the specified display
* when not touch-exploring, delegating, or dragging. This allows the service to detect its own
* gestures, and use its own logic to judge when the framework should start touch-exploring,
* delegating, or dragging. If {@link
* AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE } is disabled this flag has no
* effect.
*
* @see AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE
*/
private void setServiceDetectsGestures(boolean mode) {
final IAccessibilityServiceConnection connection =
AccessibilityInteractionClient.getInstance()
.getConnection(mService.getConnectionId());
if (connection != null) {
try {
connection.setServiceDetectsGesturesEnabled(mDisplayId, mode);
mServiceDetectsGestures = mode;
} catch (RemoteException re) {
throw new RuntimeException(re);
}
}
}
/**
* If {@link AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE} is enabled and at
* least one listener has been added for this display this function tells the framework to
* initiate touch exploration. Touch exploration will continue for the duration of this
* interaction.
*/
public void requestTouchExploration() {
checkState();
final IAccessibilityServiceConnection connection =
AccessibilityInteractionClient.getInstance()
.getConnection(mService.getConnectionId());
if (connection != null) {
try {
connection.requestTouchExploration(mDisplayId);
} catch (RemoteException re) {
throw new RuntimeException(re);
}
}
}
/**
* If {@link AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE} and {@link If
* {@link AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE} is enabled and at least
* one listener has been added, this function tells the framework to initiate a dragging
* interaction using the specified pointer. The pointer's movements will be passed through to
* the rest of the input pipeline. Dragging is often used to perform two-finger scrolling.
*
* @param pointerId the pointer to be passed through to the rest of the input pipeline. If the
* pointer id is valid but not actually present on the screen it will be ignored.
* @throws IllegalArgumentException if the pointer id is outside of the allowed range.
*/
public void requestDragging(int pointerId) {
checkState();
if (pointerId < 0 || pointerId > MAX_POINTER_COUNT) {
throw new IllegalArgumentException("Invalid pointer id: " + pointerId);
}
final IAccessibilityServiceConnection connection =
AccessibilityInteractionClient.getInstance()
.getConnection(mService.getConnectionId());
if (connection != null) {
try {
connection.requestDragging(mDisplayId, pointerId);
} catch (RemoteException re) {
throw new RuntimeException(re);
}
}
}
/**
* If {@link AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE} and {@link If
* {@link AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE} is enabled and at least
* one listener has been added, this function tells the framework to initiate a delegating
* interaction. Motion events will be passed through as-is to the rest of the input pipeline for
* the duration of this interaction.
*/
public void requestDelegating() {
checkState();
final IAccessibilityServiceConnection connection =
AccessibilityInteractionClient.getInstance()
.getConnection(mService.getConnectionId());
if (connection != null) {
try {
connection.requestDelegating(mDisplayId);
} catch (RemoteException re) {
throw new RuntimeException(re);
}
}
}
/**
* If {@link AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE} and {@link If
* {@link AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE} is enabled and at least
* one listener has been added, this function tells the framework to perform a click.
* The framework will first try to perform
* {@link AccessibilityNodeInfo.AccessibilityAction#ACTION_CLICK} on the item with
* accessibility focus. If that fails, the framework will simulate a click using motion events
* on the last location to have accessibility focus.
*/
public void performClick() {
final IAccessibilityServiceConnection connection =
AccessibilityInteractionClient.getInstance()
.getConnection(mService.getConnectionId());
if (connection != null) {
try {
connection.onDoubleTap(mDisplayId);
} catch (RemoteException re) {
throw new RuntimeException(re);
}
}
}
/**
* If {@link AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE} and {@link If
* {@link AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE} is enabled and at least
* one listener has been added, this function tells the framework to perform a long click.
* The framework will simulate a long click using motion events on the last location with
* accessibility focus and will delegate any movements to the rest of the input pipeline. This
* allows a user to double-tap and hold to trigger a drag and then execute that drag by moving
* their finger.
*/
public void performLongClickAndStartDrag() {
final IAccessibilityServiceConnection connection =
AccessibilityInteractionClient.getInstance()
.getConnection(mService.getConnectionId());
if (connection != null) {
try {
connection.onDoubleTapAndHold(mDisplayId);
} catch (RemoteException re) {
throw new RuntimeException(re);
}
}
}
private void checkState() {
if (!mServiceDetectsGestures || mListeners.size() == 0) {
throw new IllegalStateException(
"State transitions are not allowed without first adding a listener.");
}
if (mState != STATE_TOUCH_INTERACTING) {
throw new IllegalStateException(
"State transitions are not allowed in " + stateToString(mState));
}
}
/** @return the maximum number of pointers that this display will accept. */
public int getMaxPointerCount() {
return MAX_POINTER_COUNT;
}
/** @return the display id associated with this controller. */
public int getDisplayId() {
return mDisplayId;
}
/**
* @return the current state of this controller.
* @see TouchInteractionController#STATE_CLEAR
* @see TouchInteractionController#STATE_DELEGATING
* @see TouchInteractionController#STATE_DRAGGING
* @see TouchInteractionController#STATE_TOUCH_EXPLORING
*/
public int getState() {
synchronized (mLock) {
return mState;
}
}
/** Returns a string representation of the specified state. */
@NonNull
public static String stateToString(int state) {
switch (state) {
case STATE_CLEAR:
return "STATE_CLEAR";
case STATE_TOUCH_INTERACTING:
return "STATE_TOUCH_INTERACTING";
case STATE_TOUCH_EXPLORING:
return "STATE_TOUCH_EXPLORING";
case STATE_DRAGGING:
return "STATE_DRAGGING";
case STATE_DELEGATING:
return "STATE_DELEGATING";
default:
return "Unknown state: " + state;
}
}
/** Listeners allow services to receive motion events and state change updates. */
public interface Listener {
/**
* Called when the framework has sent a motion event to the service.
*
* @param event the event being passed to the service.
*/
void onMotionEvent(@NonNull MotionEvent event);
/**
* Called when the state of motion event dispatch for this display has changed.
*
* @param state the new state of motion event dispatch.
* @see TouchInteractionController#STATE_CLEAR
* @see TouchInteractionController#STATE_DELEGATING
* @see TouchInteractionController#STATE_DRAGGING
* @see TouchInteractionController#STATE_TOUCH_EXPLORING
*/
void onStateChanged(@State int state);
}
}

10
core/java/android/app/UiAutomation.java
View File

@@ -49,6 +49,7 @@ import android.util.SparseArray;
import android.view.Display;
import android.view.InputEvent;
import android.view.KeyEvent;
import android.view.MotionEvent;
import android.view.Surface;
import android.view.SurfaceControl;
import android.view.View;
@@ -1542,6 +1543,15 @@ public final class UiAutomation {
return false;
}
public void onMotionEvent(MotionEvent event) {
/* do nothing */
}
@Override
public void onTouchStateChanged(int displayId, int state) {
/* do nothing */
}
@Override
public void onAccessibilityEvent(AccessibilityEvent event) {
final OnAccessibilityEventListener listener;

20
core/tests/coretests/src/android/view/accessibility/AccessibilityServiceConnectionImpl.java
View File

@@ -160,15 +160,27 @@ public class AccessibilityServiceConnectionImpl extends IAccessibilityServiceCon
public void takeScreenshot(int displayId, RemoteCallback callback) {}
public void setTouchExplorationPassthroughRegion(int displayId, Region region) {}
public void setGestureDetectionPassthroughRegion(int displayId, Region region) {}
public void setFocusAppearance(int strokeWidth, int color) {}
public void logTrace(long timestamp, String where, String callingParams, int processId,
long threadId, int callingUid, Bundle callingStack) {}
public void setGestureDetectionPassthroughRegion(int displayId, Region region) {}
public void setTouchExplorationPassthroughRegion(int displayId, Region region) {}
public void setServiceDetectsGesturesEnabled(int displayId, boolean mode) {}
public void requestTouchExploration(int displayId) {}
public void requestDragging(int displayId, int pointerId) {}
public void requestDelegating(int displayId) {}
public void onDoubleTap(int displayId) {}
public void onDoubleTapAndHold(int displayId) {}
public void logTrace(long timestamp, String where, long loggingTypes, String callingParams,
int processId, long threadId, int callingUid, Bundle serializedCallingStackInBundle) {}
}

38
services/accessibility/java/com/android/server/accessibility/AbstractAccessibilityServiceConnection.java
View File

@@ -256,6 +256,18 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ
void setGestureDetectionPassthroughRegion(int displayId, Region region);
void setTouchExplorationPassthroughRegion(int displayId, Region region);
void setServiceDetectsGesturesEnabled(int displayId, boolean mode);
void requestTouchExploration(int displayId);
void requestDragging(int displayId, int pointerId);
void requestDelegating(int displayId);
void onDoubleTap(int displayId);
void onDoubleTapAndHold(int displayId);
}
public AbstractAccessibilityServiceConnection(Context context, ComponentName componentName,
@@ -1650,7 +1662,7 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ
}
}
private IAccessibilityServiceClient getServiceInterfaceSafely() {
protected IAccessibilityServiceClient getServiceInterfaceSafely() {
synchronized (mLock) {
return mServiceInterface;
}
@@ -2048,4 +2060,28 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ
protected void logTraceWM(String methodName, String params) {
mTrace.logTrace(TRACE_WM + "." + methodName, FLAGS_WINDOW_MANAGER_INTERNAL, params);
}
public void setServiceDetectsGesturesEnabled(int displayId, boolean mode) {
mSystemSupport.setServiceDetectsGesturesEnabled(displayId, mode);
}
public void requestTouchExploration(int displayId) {
mSystemSupport.requestTouchExploration(displayId);
}
public void requestDragging(int displayId, int pointerId) {
mSystemSupport.requestDragging(displayId, pointerId);
}
public void requestDelegating(int displayId) {
mSystemSupport.requestDelegating(displayId);
}
public void onDoubleTap(int displayId) {
mSystemSupport.onDoubleTap(displayId);
}
public void onDoubleTapAndHold(int displayId) {
mSystemSupport.onDoubleTapAndHold(displayId);
}
}

36
services/accessibility/java/com/android/server/accessibility/AccessibilityInputFilter.java
View File

@@ -893,6 +893,42 @@ class AccessibilityInputFilter extends InputFilter implements EventStreamTransfo
}
}
public void setServiceDetectsGesturesEnabled(int displayId, boolean mode) {
if (mTouchExplorer.contains(displayId)) {
mTouchExplorer.get(displayId).setServiceDetectsGestures(mode);
}
}
public void requestTouchExploration(int displayId) {
if (mTouchExplorer.contains(displayId)) {
mTouchExplorer.get(displayId).requestTouchExploration();
}
}
public void requestDragging(int displayId, int pointerId) {
if (mTouchExplorer.contains(displayId)) {
mTouchExplorer.get(displayId).requestDragging(pointerId);
}
}
public void requestDelegating(int displayId) {
if (mTouchExplorer.contains(displayId)) {
mTouchExplorer.get(displayId).requestDelegating();
}
}
public void onDoubleTap(int displayId) {
if (mTouchExplorer.contains(displayId)) {
mTouchExplorer.get(displayId).onDoubleTap();
}
}
public void onDoubleTapAndHold(int displayId) {
if (mTouchExplorer.contains(displayId)) {
mTouchExplorer.get(displayId).onDoubleTapAndHold();
}
}
/**
* Dumps all {@link AccessibilityInputFilter}s here.
*/

139
services/accessibility/java/com/android/server/accessibility/AccessibilityManagerService.java
View File

@@ -44,6 +44,7 @@ import android.accessibilityservice.AccessibilityService;
import android.accessibilityservice.AccessibilityServiceInfo;
import android.accessibilityservice.AccessibilityShortcutInfo;
import android.accessibilityservice.IAccessibilityServiceClient;
import android.accessibilityservice.TouchInteractionController;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.app.ActivityOptions;
@@ -101,6 +102,7 @@ import android.view.Display;
import android.view.IWindow;
import android.view.KeyEvent;
import android.view.MagnificationSpec;
import android.view.MotionEvent;
import android.view.WindowManager;
import android.view.accessibility.AccessibilityEvent;
import android.view.accessibility.AccessibilityInteractionClient;
@@ -1198,7 +1200,7 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub
}
/**
* Called when a gesture is detected on a display.
* Called when a gesture is detected on a display by the framework.
*
* @param gestureEvent the detail of the gesture.
* @return true if the event is handled.
@@ -1213,6 +1215,29 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub
}
}
/** Send a motion event to the service to allow it to perform gesture detection. */
public boolean sendMotionEventToListeningServices(MotionEvent event) {
synchronized (mLock) {
if (DEBUG) {
Slog.d(LOG_TAG, "Sending event to service: " + event);
}
return notifyMotionEvent(event);
}
}
/**
* Notifies services that the touch state on a given display has changed.
*/
public boolean onTouchStateChanged(int displayId, int state) {
synchronized (mLock) {
if (DEBUG) {
Slog.d(LOG_TAG, "Notifying touch state:"
+ TouchInteractionController.stateToString(state));
}
return notifyTouchState(displayId, state);
}
}
/**
* Called when the system action list is changed.
*/
@@ -1527,6 +1552,30 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub
return false;
}
private boolean notifyMotionEvent(MotionEvent event) {
AccessibilityUserState state = getCurrentUserStateLocked();
for (int i = state.mBoundServices.size() - 1; i >= 0; i--) {
AccessibilityServiceConnection service = state.mBoundServices.get(i);
if (service.mRequestTouchExplorationMode) {
service.notifyMotionEvent(event);
return true;
}
}
return false;
}
private boolean notifyTouchState(int displayId, int touchState) {
AccessibilityUserState state = getCurrentUserStateLocked();
for (int i = state.mBoundServices.size() - 1; i >= 0; i--) {
AccessibilityServiceConnection service = state.mBoundServices.get(i);
if (service.mRequestTouchExplorationMode) {
service.notifyTouchState(displayId, touchState);
return true;
}
}
return false;
}
private void notifyClearAccessibilityCacheLocked() {
AccessibilityUserState state = getCurrentUserStateLocked();
for (int i = state.mBoundServices.size() - 1; i >= 0; i--) {
@@ -2083,7 +2132,6 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub
if (userState.isSendMotionEventsEnabled()) {
flags |= AccessibilityInputFilter.FLAG_SEND_MOTION_EVENTS;
}
if (userState.isAutoclickEnabledLocked()) {
flags |= AccessibilityInputFilter.FLAG_FEATURE_AUTOCLICK;
}
@@ -3960,6 +4008,93 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub
}
}
@Override
public void setServiceDetectsGesturesEnabled(int displayId, boolean mode) {
mMainHandler.sendMessage(
obtainMessage(AccessibilityManagerService::setServiceDetectsGesturesInternal, this,
displayId, mode));
}
private void setServiceDetectsGesturesInternal(int displayId, boolean mode) {
synchronized (mLock) {
if (mHasInputFilter && mInputFilter != null) {
mInputFilter.setServiceDetectsGesturesEnabled(displayId, mode);
}
}
}
@Override
public void requestTouchExploration(int displayId) {
mMainHandler.sendMessage(obtainMessage(
AccessibilityManagerService::requestTouchExplorationInternal, this, displayId));
}
private void requestTouchExplorationInternal(int displayId) {
synchronized (mLock) {
if (mHasInputFilter && mInputFilter != null) {
mInputFilter.requestTouchExploration(displayId);
}
}
}
@Override
public void requestDragging(int displayId, int pointerId) {
mMainHandler.sendMessage(obtainMessage(AccessibilityManagerService::requestDraggingInternal,
this, displayId, pointerId));
}
private void requestDraggingInternal(int displayId, int pointerId) {
synchronized (mLock) {
if (mHasInputFilter && mInputFilter != null) {
mInputFilter.requestDragging(displayId, pointerId);
}
}
}
@Override
public void requestDelegating(int displayId) {
mMainHandler.sendMessage(
obtainMessage(
AccessibilityManagerService::requestDelegatingInternal, this, displayId));
}
private void requestDelegatingInternal(int displayId) {
synchronized (mLock) {
if (mHasInputFilter && mInputFilter != null) {
mInputFilter.requestDelegating(displayId);
}
}
}
@Override
public void onDoubleTap(int displayId) {
mMainHandler.sendMessage(obtainMessage(AccessibilityManagerService::onDoubleTapInternal,
this, displayId));
}
private void onDoubleTapInternal(int displayId) {
synchronized (mLock) {
if (mHasInputFilter && mInputFilter != null) {
mInputFilter.onDoubleTap(displayId);
}
}
}
@Override
public void onDoubleTapAndHold(int displayId) {
mMainHandler
.sendMessage(obtainMessage(AccessibilityManagerService::onDoubleTapAndHoldInternal,
this, displayId));
}
private void onDoubleTapAndHoldInternal(int displayId) {
synchronized (mLock) {
if (mHasInputFilter && mInputFilter != null) {
mInputFilter.onDoubleTapAndHold(displayId);
}
}
}
private void updateFocusAppearanceDataLocked(AccessibilityUserState userState) {
if (userState.mUserId != mCurrentUserId) {
return;

48
services/accessibility/java/com/android/server/accessibility/AccessibilityServiceConnection.java
View File

@@ -22,6 +22,7 @@ import android.Manifest;
import android.accessibilityservice.AccessibilityServiceInfo;
import android.accessibilityservice.AccessibilityTrace;
import android.accessibilityservice.IAccessibilityServiceClient;
import android.accessibilityservice.TouchInteractionController;
import android.app.PendingIntent;
import android.content.ComponentName;
import android.content.Context;
@@ -30,12 +31,15 @@ import android.content.pm.ParceledListSlice;
import android.os.Binder;
import android.os.Handler;
import android.os.IBinder;
import android.os.Message;
import android.os.Process;
import android.os.RemoteException;
import android.os.UserHandle;
import android.provider.Settings;
import android.util.Slog;
import android.view.Display;
import android.view.MotionEvent;
import com.android.server.inputmethod.InputMethodManagerInternal;
import com.android.server.wm.ActivityTaskManagerInternal;
@@ -454,4 +458,48 @@ class AccessibilityServiceConnection extends AbstractAccessibilityServiceConnect
mSystemSupport.onClientChangeLocked(false);
}
}
public void notifyMotionEvent(MotionEvent event) {
final Message msg = obtainMessage(
AccessibilityServiceConnection::notifyMotionEventInternal,
AccessibilityServiceConnection.this, event);
mMainHandler.sendMessage(msg);
}
public void notifyTouchState(int displayId, int state) {
final Message msg = obtainMessage(
AccessibilityServiceConnection::notifyTouchStateInternal,
AccessibilityServiceConnection.this, displayId, state);
mMainHandler.sendMessage(msg);
}
private void notifyMotionEventInternal(MotionEvent event) {
final IAccessibilityServiceClient listener = getServiceInterfaceSafely();
if (listener != null) {
try {
if (mTrace.isA11yTracingEnabled()) {
logTraceSvcClient(".onMotionEvent ",
event.toString());
}
listener.onMotionEvent(event);
} catch (RemoteException re) {
Slog.e(LOG_TAG, "Error sending motion event to" + mService, re);
}
}
}
private void notifyTouchStateInternal(int displayId, int state) {
final IAccessibilityServiceClient listener = getServiceInterfaceSafely();
if (listener != null) {
try {
if (mTrace.isA11yTracingEnabled()) {
logTraceSvcClient(".onTouchStateChanged ",
TouchInteractionController.stateToString(state));
}
listener.onTouchStateChanged(displayId, state);
} catch (RemoteException re) {
Slog.e(LOG_TAG, "Error sending motion event to" + mService, re);
}
}
}
}

17
services/accessibility/java/com/android/server/accessibility/gestures/EventDispatcher.java
View File

@@ -382,18 +382,21 @@ class EventDispatcher {
}
public boolean longPressWithTouchEvents(MotionEvent event, int policyFlags) {
final int pointerIndex = event.getActionIndex();
final int pointerId = event.getPointerId(pointerIndex);
Point clickLocation = mTempPoint;
final int result = computeClickLocation(clickLocation);
if (result == CLICK_LOCATION_NONE) {
return false;
}
mLongPressingPointerId = pointerId;
mLongPressingPointerDeltaX = (int) event.getX(pointerIndex) - clickLocation.x;
mLongPressingPointerDeltaY = (int) event.getY(pointerIndex) - clickLocation.y;
sendDownForAllNotInjectedPointers(event, policyFlags);
return true;
if (event != null) {
final int pointerIndex = event.getActionIndex();
final int pointerId = event.getPointerId(pointerIndex);
mLongPressingPointerId = pointerId;
mLongPressingPointerDeltaX = (int) event.getX(pointerIndex) - clickLocation.x;
mLongPressingPointerDeltaY = (int) event.getY(pointerIndex) - clickLocation.y;
sendDownForAllNotInjectedPointers(event, policyFlags);
return true;
}
return false;
}
void clear() {

273
services/accessibility/java/com/android/server/accessibility/gestures/TouchExplorer.java
View File

@@ -46,6 +46,7 @@ import android.graphics.Region;
import android.os.Handler;
import android.util.DisplayMetrics;
import android.util.Slog;
import android.view.Display;
import android.view.InputDevice;
import android.view.MotionEvent;
import android.view.ViewConfiguration;
@@ -152,6 +153,7 @@ public class TouchExplorer extends BaseEventStreamTransformation
private Region mGestureDetectionPassthroughRegion;
private Region mTouchExplorationPassthroughRegion;
private int mDisplayId = Display.INVALID_DISPLAY;
/**
* Creates a new instance.
@@ -182,8 +184,9 @@ public class TouchExplorer extends BaseEventStreamTransformation
TouchExplorer(Context context, AccessibilityManagerService service, GestureManifold detector,
@NonNull Handler mainHandler) {
mContext = context;
mDisplayId = context.getDisplayId();
mAms = service;
mState = new TouchState();
mState = new TouchState(mDisplayId, mAms);
mReceivedPointerTracker = mState.getReceivedPointerTracker();
mDispatcher = new EventDispatcher(context, mAms, super.getNext(), mState);
mDetermineUserIntentTimeout = ViewConfiguration.getDoubleTapTimeout();
@@ -267,6 +270,7 @@ public class TouchExplorer extends BaseEventStreamTransformation
}
try {
checkForMalformedEvent(event);
checkForMalformedEvent(rawEvent);
} catch (IllegalArgumentException e) {
Slog.e(LOG_TAG, "Ignoring malformed event: " + event.toString(), e);
return;
@@ -277,7 +281,7 @@ public class TouchExplorer extends BaseEventStreamTransformation
Slog.d(LOG_TAG, mState.toString());
}
mState.onReceivedMotionEvent(rawEvent);
mState.onReceivedMotionEvent(event, rawEvent, policyFlags);
if (shouldPerformGestureDetection(event)) {
if (mGestureDetector.onMotionEvent(event, rawEvent, policyFlags)) {
// Event was handled by the gesture detector.
@@ -307,9 +311,12 @@ public class TouchExplorer extends BaseEventStreamTransformation
// It will be delivered on gesture completion or cancelation.
// Note that the delay for sending GESTURE_DETECTION_END remains in place.
mSendTouchInteractionEndDelayed.cancel();
if (mState.isServiceDetectingGestures()) {
mAms.sendMotionEventToListeningServices(rawEvent);
}
} else {
Slog.e(LOG_TAG, "Illegal state: " + mState);
clear(event, policyFlags);
clear(event, policyFlags);
}
}
@@ -367,7 +374,7 @@ public class TouchExplorer extends BaseEventStreamTransformation
AccessibilityGestureEvent gestureEvent =
new AccessibilityGestureEvent(
AccessibilityService.GESTURE_DOUBLE_TAP_AND_HOLD,
event.getDisplayId(),
mDisplayId,
mGestureDetector.getMotionEvents());
dispatchGesture(gestureEvent);
}
@@ -389,7 +396,7 @@ public class TouchExplorer extends BaseEventStreamTransformation
AccessibilityGestureEvent gestureEvent =
new AccessibilityGestureEvent(
AccessibilityService.GESTURE_DOUBLE_TAP,
event.getDisplayId(),
mDisplayId,
mGestureDetector.getMotionEvents());
dispatchGesture(gestureEvent);
}
@@ -404,13 +411,37 @@ public class TouchExplorer extends BaseEventStreamTransformation
if (!mAms.performActionOnAccessibilityFocusedItem(
AccessibilityNodeInfo.AccessibilityAction.ACTION_CLICK)) {
Slog.e(LOG_TAG, "ACTION_CLICK failed. Dispatching motion events to simulate click.");
mDispatcher.clickWithTouchEvents(event, rawEvent, policyFlags);
if (event != null && rawEvent != null) {
mDispatcher.clickWithTouchEvents(event, rawEvent, policyFlags);
}
return true;
}
return true;
}
/**
* Executes a double-tap. The framework will first attempt to execute the appropriate
* accessibility action and if that fails, the framework will deliver touch events to the last
* touch-explored location.
*/
public void onDoubleTap() {
MotionEvent event = mState.getLastReceivedEvent();
MotionEvent rawEvent = mState.getLastReceivedRawEvent();
int policyFlags = mState.getLastReceivedPolicyFlags();
onDoubleTap(event, rawEvent, policyFlags);
}
/**
* Executes a double-tap and hold gesture using touch events. The user can continue to move
* their finger around the screen to execute a drag.
*/
public void onDoubleTapAndHold() {
MotionEvent event = mState.getLastReceivedEvent();
MotionEvent rawEvent = mState.getLastReceivedRawEvent();
int policyFlags = mState.getLastReceivedPolicyFlags();
onDoubleTapAndHold(event, rawEvent, policyFlags);
}
@Override
public boolean onGestureStarted() {
if (mAms.getTraceManager().isA11yTracingEnabledForTypes(LOGGING_FLAGS)) {
@@ -473,7 +504,7 @@ public class TouchExplorer extends BaseEventStreamTransformation
AccessibilityGestureEvent gestureEvent =
new AccessibilityGestureEvent(
AccessibilityService.GESTURE_UNKNOWN,
event.getDisplayId(),
mDisplayId,
mGestureDetector.getMotionEvents());
dispatchGesture(gestureEvent);
}
@@ -502,7 +533,6 @@ public class TouchExplorer extends BaseEventStreamTransformation
*/
private void handleActionDown(MotionEvent event, MotionEvent rawEvent, int policyFlags) {
mAms.onTouchInteractionStart();
// If we still have not notified the user for the last
// touch, we figure out what to do. If were waiting
// we resent the delayed callback and wait again.
@@ -512,7 +542,6 @@ public class TouchExplorer extends BaseEventStreamTransformation
if (mState.isTouchExploring()) {
sendHoverExitAndTouchExplorationGestureEndIfNeeded(policyFlags);
}
if (mState.isClear()) {
if (!mSendHoverEnterAndMoveDelayed.isPending()) {
// Queue a delayed transition to STATE_TOUCH_EXPLORING.
@@ -521,7 +550,14 @@ public class TouchExplorer extends BaseEventStreamTransformation
// The idea is to avoid getting stuck in STATE_TOUCH_INTERACTING
final int pointerId = mReceivedPointerTracker.getPrimaryPointerId();
final int pointerIdBits = (1 << pointerId);
mSendHoverEnterAndMoveDelayed.post(event, rawEvent, pointerIdBits, policyFlags);
if (mState.isServiceDetectingGestures()) {
// This transition will be triggered manually by the service.
mSendHoverEnterAndMoveDelayed.setPointerIdBits(pointerIdBits);
mSendHoverEnterAndMoveDelayed.setPolicyFlags(policyFlags);
mSendHoverEnterAndMoveDelayed.addEvent(event, rawEvent);
} else {
mSendHoverEnterAndMoveDelayed.post(event, rawEvent, pointerIdBits, policyFlags);
}
} else {
// Cache the event until we discern exploration from gesturing.
mSendHoverEnterAndMoveDelayed.addEvent(event, rawEvent);
@@ -548,6 +584,9 @@ public class TouchExplorer extends BaseEventStreamTransformation
// Avoid duplicated TYPE_TOUCH_INTERACTION_START event when 2nd tap of double tap.
mSendTouchInteractionEndDelayed.cancel();
}
if (mState.isServiceDetectingGestures()) {
mAms.sendMotionEventToListeningServices(rawEvent);
}
}
/**
@@ -571,6 +610,11 @@ public class TouchExplorer extends BaseEventStreamTransformation
case ACTION_MOVE:
handleActionMoveStateTouchInteracting(event, rawEvent, policyFlags);
break;
case ACTION_POINTER_UP:
if (mState.isServiceDetectingGestures()) {
mAms.sendMotionEventToListeningServices(rawEvent);
}
break;
case ACTION_UP:
handleActionUp(event, rawEvent, policyFlags);
break;
@@ -612,15 +656,16 @@ public class TouchExplorer extends BaseEventStreamTransformation
// Another finger down means that if we have not started to deliver
// hover events, we will not have to. The code for ACTION_MOVE will
// decide what we will actually do next.
if (mSendHoverEnterAndMoveDelayed.isPending()) {
mSendHoverEnterAndMoveDelayed.cancel();
mSendHoverExitDelayed.cancel();
} else {
// We have already delivered at least one hover event, so send hover exit to keep the
// stream consistent.
// We have already delivered at least one hover event, so send hover exit to keep
sendHoverExitAndTouchExplorationGestureEndIfNeeded(policyFlags);
}
if (mState.isServiceDetectingGestures()) {
mAms.sendMotionEventToListeningServices(rawEvent);
}
}
/**
@@ -632,12 +677,19 @@ public class TouchExplorer extends BaseEventStreamTransformation
final int pointerId = mReceivedPointerTracker.getPrimaryPointerId();
final int pointerIndex = event.findPointerIndex(pointerId);
int pointerIdBits = (1 << pointerId);
if (mState.isServiceDetectingGestures()) {
mAms.sendMotionEventToListeningServices(rawEvent);
mSendHoverEnterAndMoveDelayed.addEvent(event, rawEvent);
return;
}
switch (event.getPointerCount()) {
case 1:
// We have not started sending events since we try to
// figure out what the user is doing.
if (mSendHoverEnterAndMoveDelayed.isPending()) {
// Cache the event until we discern exploration from gesturing.
// When the service is detecting gestures we rely on it to fire touch
// exploration.
mSendHoverEnterAndMoveDelayed.addEvent(event, rawEvent);
}
break;
@@ -684,7 +736,7 @@ public class TouchExplorer extends BaseEventStreamTransformation
AccessibilityGestureEvent gestureEvent =
new AccessibilityGestureEvent(
AccessibilityService.GESTURE_PASSTHROUGH,
event.getDisplayId(),
mDisplayId,
mGestureDetector.getMotionEvents());
dispatchGesture(gestureEvent);
}
@@ -708,7 +760,7 @@ public class TouchExplorer extends BaseEventStreamTransformation
AccessibilityGestureEvent gestureEvent =
new AccessibilityGestureEvent(
AccessibilityService.GESTURE_PASSTHROUGH,
event.getDisplayId(),
mDisplayId,
mGestureDetector.getMotionEvents());
dispatchGesture(gestureEvent);
}
@@ -730,7 +782,7 @@ public class TouchExplorer extends BaseEventStreamTransformation
AccessibilityGestureEvent gestureEvent =
new AccessibilityGestureEvent(
AccessibilityService.GESTURE_PASSTHROUGH,
event.getDisplayId(),
mDisplayId,
mGestureDetector.getMotionEvents());
dispatchGesture(gestureEvent);
}
@@ -751,7 +803,7 @@ public class TouchExplorer extends BaseEventStreamTransformation
AccessibilityGestureEvent gestureEvent =
new AccessibilityGestureEvent(
AccessibilityService.GESTURE_PASSTHROUGH,
event.getDisplayId(),
mDisplayId,
mGestureDetector.getMotionEvents());
dispatchGesture(gestureEvent);
}
@@ -767,7 +819,10 @@ public class TouchExplorer extends BaseEventStreamTransformation
* Handles ACTION_UP while in the touch interacting state. This event represents all fingers
* being lifted from the screen.
*/
private void handleActionUp(MotionEvent event, MotionEvent rawEvent, int policyFlags) {
private void handleActionUp(MotionEvent event, MotionEvent rawEvent, int policyFlags) {
if (mState.isServiceDetectingGestures() && mState.isTouchInteracting()) {
mAms.sendMotionEventToListeningServices(rawEvent);
}
mAms.onTouchInteractionEnd();
final int pointerId = event.getPointerId(event.getActionIndex());
final int pointerIdBits = (1 << pointerId);
@@ -883,6 +938,10 @@ public class TouchExplorer extends BaseEventStreamTransformation
clear(event, policyFlags);
return;
case ACTION_POINTER_DOWN:
if (mState.isServiceDetectingGestures()) {
mAms.sendMotionEventToListeningServices(rawEvent);
return;
}
// We are in dragging state so we have two pointers and another one
// goes down => delegate the three pointers to the view hierarchy
mState.startDelegating();
@@ -896,6 +955,15 @@ public class TouchExplorer extends BaseEventStreamTransformation
if (mDraggingPointerId == INVALID_POINTER_ID) {
break;
}
if (mState.isServiceDetectingGestures()) {
// Allow the service to judge whether this is dragging or delegation
mAms.sendMotionEventToListeningServices(rawEvent);
computeDraggingPointerIdIfNeeded(event);
mDispatcher.sendMotionEvent(
event, ACTION_MOVE, rawEvent, pointerIdBits, policyFlags);
return;
}
switch (event.getPointerCount()) {
case 1:
// do nothing
@@ -921,6 +989,10 @@ public class TouchExplorer extends BaseEventStreamTransformation
}
break;
default:
if (mState.isServiceDetectingGestures()) {
mAms.sendMotionEventToListeningServices(rawEvent);
return;
}
mState.startDelegating();
mDraggingPointerId = INVALID_POINTER_ID;
event = MotionEvent.obtainNoHistory(event);
@@ -1210,9 +1282,7 @@ public class TouchExplorer extends BaseEventStreamTransformation
mTouchExplorationPassthroughRegion = region;
}
/**
* Whether to send the motion events that make up each gesture to the accessibility service.
*/
/** Whether to send the motion events that make up each gesture to the accessibility service. */
public void setSendMotionEventsEnabled(boolean mode) {
mGestureDetector.setSendMotionEventsEnabled(mode);
}
@@ -1221,7 +1291,15 @@ public class TouchExplorer extends BaseEventStreamTransformation
return mGestureDetector.isSendMotionEventsEnabled();
}
/** Sets whether or not motion events should be passed to the service to detect gestures. */
public void setServiceDetectsGestures(boolean mode) {
mState.setServiceDetectsGestures(mode);
}
private boolean shouldPerformGestureDetection(MotionEvent event) {
if (mState.isServiceDetectingGestures()) {
return false;
}
if (mState.isDelegating() || mState.isDragging()) {
return false;
}
@@ -1237,8 +1315,105 @@ public class TouchExplorer extends BaseEventStreamTransformation
}
/**
* Class for delayed exiting from gesture detecting mode.
* This method allows the service to request that TouchExplorer enter the touch exploration
* state.
*/
public void requestTouchExploration() {
if (DEBUG) {
Slog.d(LOG_TAG, "Starting touch explorer from service.");
}
if (mState.isServiceDetectingGestures() && mState.isTouchInteracting()) {
// Cancel without deleting events.
mHandler.removeCallbacks(mSendHoverEnterAndMoveDelayed);
mSendHoverEnterAndMoveDelayed.run();
mSendHoverEnterAndMoveDelayed.clear();
final MotionEvent prototype = mState.getLastReceivedEvent();
final MotionEvent rawEvent = mState.getLastReceivedRawEvent();
final int pointerId = mReceivedPointerTracker.getPrimaryPointerId();
final int pointerIdBits = (1 << pointerId);
final int policyFlags = mState.getLastReceivedPolicyFlags();
mSendHoverExitDelayed.post(prototype, rawEvent, pointerIdBits, policyFlags);
}
}
/** This method allows the service to request that TouchExplorer enter the dragging state. */
public void requestDragging(int pointerId) {
if (mState.isServiceDetectingGestures()) {
if (pointerId < 0 || pointerId > TouchState.MAX_POINTER_COUNT
|| !mReceivedPointerTracker.isReceivedPointerDown(pointerId)) {
Slog.e(LOG_TAG, "Trying to drag with invalid pointer: " + pointerId);
return;
}
if (mState.isTouchExploring()) {
if (mSendHoverExitDelayed.isPending()) {
mSendHoverExitDelayed.forceSendAndRemove();
}
if (mSendTouchExplorationEndDelayed.isPending()) {
mSendTouchExplorationEndDelayed.forceSendAndRemove();
}
}
if (!mState.isTouchInteracting()) {
// It makes no sense to drag.
Slog.e(LOG_TAG, "Error: Trying to drag from "
+ mState.getStateSymbolicName(mState.getState()));
return;
}
mDraggingPointerId = pointerId;
if (DEBUG) {
Slog.d(LOG_TAG, "Drag requested on pointer " + mDraggingPointerId);
}
MotionEvent event = mState.getLastReceivedEvent();
MotionEvent rawEvent = mState.getLastReceivedRawEvent();
if (event == null || rawEvent == null) {
Slog.e(LOG_TAG, "Unable to start dragging: unable to get last event.");
return;
}
int policyFlags = mState.getLastReceivedPolicyFlags();
int pointerIdBits = 1 << mDraggingPointerId;
event.setEdgeFlags(mReceivedPointerTracker.getLastReceivedDownEdgeFlags());
MotionEvent downEvent = computeDownEventForDrag(event);
mState.startDragging();
if (downEvent != null) {
mDispatcher.sendMotionEvent(
downEvent, ACTION_DOWN, rawEvent, pointerIdBits, policyFlags);
mDispatcher.sendMotionEvent(event, ACTION_MOVE, rawEvent, pointerIdBits,
policyFlags);
} else {
mDispatcher.sendMotionEvent(event, ACTION_DOWN, rawEvent, pointerIdBits,
policyFlags);
}
}
}
/** This method allows the service to request that TouchExplorer enter the delegating state. */
public void requestDelegating() {
if (mState.isServiceDetectingGestures()) {
if (mState.isTouchExploring()) {
if (mSendHoverExitDelayed.isPending()) {
mSendHoverExitDelayed.forceSendAndRemove();
}
if (mSendTouchExplorationEndDelayed.isPending()) {
mSendTouchExplorationEndDelayed.forceSendAndRemove();
}
}
if (!mState.isTouchInteracting()) {
// It makes no sense to delegate.
Slog.e(LOG_TAG, "Error: Trying to delegate from "
+ mState.getStateSymbolicName(mState.getState()));
return;
}
mState.startDelegating();
MotionEvent prototype = mState.getLastReceivedEvent();
if (prototype == null) {
Slog.d(LOG_TAG, "Unable to start delegating: unable to get last received event.");
return;
}
int policyFlags = mState.getLastReceivedPolicyFlags();
mDispatcher.sendDownForAllNotInjectedPointers(prototype, policyFlags);
}
}
/** Class for delayed exiting from gesture detecting mode. */
private final class ExitGestureDetectionModeDelayed implements Runnable {
public void post() {
@@ -1283,9 +1458,7 @@ public class TouchExplorer extends BaseEventStreamTransformation
}
}
/**
* Class for delayed sending of hover enter and move events.
*/
/** Class for delayed sending of hover enter and move events. */
class SendHoverEnterAndMoveDelayed implements Runnable {
private final String LOG_TAG_SEND_HOVER_DELAYED = "SendHoverEnterAndMoveDelayed";
@@ -1362,31 +1535,46 @@ public class TouchExplorer extends BaseEventStreamTransformation
}
if (!mEvents.isEmpty() && !mRawEvents.isEmpty()) {
// Deliver a down event.
mDispatcher.sendMotionEvent(mEvents.get(0), ACTION_HOVER_ENTER,
mRawEvents.get(0), mPointerIdBits, mPolicyFlags);
mDispatcher.sendMotionEvent(
mEvents.get(0),
ACTION_HOVER_ENTER,
mRawEvents.get(0),
mPointerIdBits,
mPolicyFlags);
if (DEBUG) {
Slog.d(LOG_TAG_SEND_HOVER_DELAYED,
Slog.d(
LOG_TAG_SEND_HOVER_DELAYED,
"Injecting motion event: ACTION_HOVER_ENTER");
}
// Deliver move events.
final int eventCount = mEvents.size();
for (int i = 1; i < eventCount; i++) {
mDispatcher.sendMotionEvent(mEvents.get(i), ACTION_HOVER_MOVE,
mRawEvents.get(i), mPointerIdBits, mPolicyFlags);
mDispatcher.sendMotionEvent(
mEvents.get(i),
ACTION_HOVER_MOVE,
mRawEvents.get(i),
mPointerIdBits,
mPolicyFlags);
if (DEBUG) {
Slog.d(LOG_TAG_SEND_HOVER_DELAYED,
Slog.d(
LOG_TAG_SEND_HOVER_DELAYED,
"Injecting motion event: ACTION_HOVER_MOVE");
}
}
}
clear();
}
}
/**
* Class for delayed sending of hover exit events.
*/
public void setPointerIdBits(int pointerIdBits) {
mPointerIdBits = pointerIdBits;
}
public void setPolicyFlags(int policyFlags) {
mPolicyFlags = policyFlags;
}
}
/** Class for delayed sending of hover exit events. */
class SendHoverExitDelayed implements Runnable {
private final String LOG_TAG_SEND_HOVER_DELAYED = "SendHoverExitDelayed";
@@ -1438,21 +1626,18 @@ public class TouchExplorer extends BaseEventStreamTransformation
public void run() {
if (DEBUG) {
Slog.d(LOG_TAG_SEND_HOVER_DELAYED, "Injecting motion event:"
+ " ACTION_HOVER_EXIT");
Slog.d(
LOG_TAG_SEND_HOVER_DELAYED,
"Injecting motion event:" + " ACTION_HOVER_EXIT");
}
mDispatcher.sendMotionEvent(
mPrototype,
ACTION_HOVER_EXIT,
mRawEvent,
mPointerIdBits,
mPolicyFlags);
mPrototype, ACTION_HOVER_EXIT, mRawEvent, mPointerIdBits, mPolicyFlags);
if (!mSendTouchExplorationEndDelayed.isPending()) {
mSendTouchExplorationEndDelayed.cancel();
mSendTouchExplorationEndDelayed.post();
}
if (mSendTouchInteractionEndDelayed.isPending()) {
mSendTouchInteractionEndDelayed.cancel();
mSendTouchInteractionEndDelayed.cancel();
mSendTouchInteractionEndDelayed.post();
}
clear();

62
services/accessibility/java/com/android/server/accessibility/gestures/TouchState.java
View File

@@ -22,9 +22,12 @@ import static com.android.server.accessibility.gestures.TouchExplorer.DEBUG;
import android.annotation.IntDef;
import android.util.Slog;
import android.view.Display;
import android.view.MotionEvent;
import android.view.accessibility.AccessibilityEvent;
import com.android.server.accessibility.AccessibilityManagerService;
/**
* This class describes the state of the touch explorer as well as the state of received and
* injected pointers. This data is accessed both for purposes of touch exploration and gesture
@@ -36,7 +39,7 @@ public class TouchState {
// This constant captures the current implementation detail that
// pointer IDs are between 0 and 31 inclusive (subject to change).
// (See MAX_POINTER_ID in frameworks/base/include/ui/Input.h)
static final int MAX_POINTER_COUNT = 32;
public static final int MAX_POINTER_COUNT = 32;
// Constant referring to the ids bits of all pointers.
public static final int ALL_POINTER_ID_BITS = 0xFFFFFFFF;
@@ -50,8 +53,8 @@ public class TouchState {
public static final int STATE_TOUCH_EXPLORING = 2;
// the user is dragging with two fingers.
public static final int STATE_DRAGGING = 3;
// The user is performing some other two finger gesture which we pass through to the view
// hierarchy as a one-finger gesture e.g. two-finger scrolling.
// The user is performing some other two finger gesture which we pass through to the
// input pipeline as a one-finger gesture e.g. two-finger pinch.
public static final int STATE_DELEGATING = 4;
// The user is performing something that might be a gesture.
public static final int STATE_GESTURE_DETECTING = 5;
@@ -75,6 +78,8 @@ public class TouchState {
private MotionEvent mLastReceivedEvent;
// The accompanying raw event without any transformations.
private MotionEvent mLastReceivedRawEvent;
// The policy flags of the last received event.
int mLastReceivedPolicyFlags;
// The id of the last touch explored window.
private int mLastTouchedWindowId;
// The last injected hover event.
@@ -85,14 +90,23 @@ public class TouchState {
private long mLastInjectedDownEventTime;
// Keep track of which pointers sent to the system are down.
private int mInjectedPointersDown;
private boolean mServiceDetectsGestures = false;
// The requested mode for mServiceDetectsGestures. This will take effect on the next touch
// interaction.
private boolean mServiceDetectsGesturesRequested = false;
private AccessibilityManagerService mAms;
private int mDisplayId = Display.INVALID_DISPLAY;
public TouchState() {
public TouchState(int displayId, AccessibilityManagerService ams) {
mDisplayId = displayId;
mAms = ams;
mReceivedPointerTracker = new ReceivedPointerTracker();
}
/** Clears the internal shared state. */
public void clear() {
setState(STATE_CLEAR);
mServiceDetectsGestures = mServiceDetectsGesturesRequested;
// Reset the pointer trackers.
if (mLastReceivedEvent != null) {
mLastReceivedEvent.recycle();
@@ -107,14 +121,19 @@ public class TouchState {
*
* @param rawEvent The raw touch event.
*/
public void onReceivedMotionEvent(MotionEvent rawEvent) {
public void onReceivedMotionEvent(MotionEvent event, MotionEvent rawEvent, int policyFlags) {
if (isClear() && event.getActionMasked() == MotionEvent.ACTION_DOWN) {
clear();
}
if (mLastReceivedEvent != null) {
mLastReceivedEvent.recycle();
}
if (mLastReceivedRawEvent != null) {
mLastReceivedRawEvent.recycle();
}
mLastReceivedEvent = MotionEvent.obtain(rawEvent);
mLastReceivedEvent = MotionEvent.obtain(event);
mLastReceivedRawEvent = MotionEvent.obtain(rawEvent);
mLastReceivedPolicyFlags = policyFlags;
mReceivedPointerTracker.onMotionEvent(rawEvent);
}
@@ -123,7 +142,7 @@ public class TouchState {
*
* @param event The event to process.
*/
void onInjectedMotionEvent(MotionEvent event) {
public void onInjectedMotionEvent(MotionEvent event) {
final int action = event.getActionMasked();
final int pointerId = event.getPointerId(event.getActionIndex());
final int pointerFlag = (1 << pointerId);
@@ -183,6 +202,7 @@ public class TouchState {
}
}
/** Updates the state in response to an injected accessibility event. */
public void onInjectedAccessibilityEvent(int type) {
// The below state transitions go here because the related events are often sent on a
// delay.
@@ -195,7 +215,8 @@ public class TouchState {
startTouchInteracting();
break;
case AccessibilityEvent.TYPE_TOUCH_INTERACTION_END:
clear();
setState(STATE_CLEAR);
// We will clear when we actually handle the next ACTION_DOWN.
break;
case AccessibilityEvent.TYPE_TOUCH_EXPLORATION_GESTURE_START:
startTouchExploring();
@@ -228,6 +249,9 @@ public class TouchState {
Slog.i(LOG_TAG, getStateSymbolicName(mState) + "->" + getStateSymbolicName(state));
}
mState = state;
if (mServiceDetectsGestures) {
mAms.onTouchStateChanged(mDisplayId, state);
}
}
public boolean isTouchExploring() {
@@ -314,6 +338,16 @@ public class TouchState {
return mLastReceivedEvent;
}
/** Gets the most recently received policy flags. */
public int getLastReceivedPolicyFlags() {
return mLastReceivedPolicyFlags;
}
/** Gets the most recently received raw event. */
public MotionEvent getLastReceivedRawEvent() {
return mLastReceivedRawEvent;
}
/** @return The the last injected hover event. */
public MotionEvent getLastInjectedHoverEvent() {
return mLastInjectedHoverEvent;
@@ -354,6 +388,18 @@ public class TouchState {
return mLastInjectedHoverEventForClick;
}
public boolean isServiceDetectingGestures() {
return mServiceDetectsGestures;
}
/** Whether the service is handling gesture detection. */
public void setServiceDetectsGestures(boolean mode) {
if (DEBUG) {
Slog.d(LOG_TAG, "serviceDetectsGestures: " + mode);
}
mServiceDetectsGesturesRequested = mode;
}
/** This class tracks where and when a pointer went down. It does not track its movement. */
class ReceivedPointerTracker {
private static final String LOG_TAG_RECEIVED_POINTER_TRACKER = "ReceivedPointerTracker";

35
services/tests/servicestests/src/com/android/server/accessibility/gestures/GestureManifoldTest.java
View File

@@ -27,18 +27,20 @@ import android.content.Context;
import android.graphics.Point;
import android.graphics.PointF;
import android.os.Handler;
import android.view.Display;
import android.view.MotionEvent;
import androidx.test.InstrumentationRegistry;
import com.android.server.accessibility.AccessibilityManagerService;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mock;
import java.util.ArrayList;
/**
* Tests for GestureManifold
*/
/** Tests for GestureManifold */
public class GestureManifoldTest {
// Constants for testRecognizeGesturePath()
@@ -50,25 +52,23 @@ public class GestureManifoldTest {
private GestureManifold mManifold;
private TouchState mState;
private GestureManifold.Listener mResultListener;
@Mock private AccessibilityManagerService mMockAms;
@Before
public void setUp() {
Context context = InstrumentationRegistry.getContext();
// Construct a testable GestureManifold.
mResultListener = mock(GestureManifold.Listener.class);
mState = new TouchState();
mState = new TouchState(Display.DEFAULT_DISPLAY, mMockAms);
Handler handler = new Handler(context.getMainLooper());
mManifold = new GestureManifold(context, mResultListener, mState, handler);
// Play the role of touch explorer in updating the shared state.
when(mResultListener.onGestureStarted()).thenReturn(onGestureStarted());
}
@Test
public void testRecognizeGesturePath() {
final int d = 1000; // Length of each segment in the test gesture, in pixels.
final int d = 1000; // Length of each segment in the test gesture, in pixels.
testPath(p(-d, +0), AccessibilityService.GESTURE_SWIPE_LEFT);
testPath(p(+d, +0), AccessibilityService.GESTURE_SWIPE_RIGHT);
@@ -138,9 +138,10 @@ public class GestureManifoldTest {
// For each path step from start (non-inclusive) to end ... add a motion point.
for (int step = 1; step < numSteps; ++step) {
path.add(new PointF(
(start.x + (stepX * (float) step)),
(start.y + (stepY * (float) step))));
path.add(
new PointF(
(start.x + (stepX * (float) step)),
(start.y + (stepY * (float) step))));
}
}
@@ -164,11 +165,12 @@ public class GestureManifoldTest {
} else if (pointIndex == path.size() - 1) {
action = MotionEvent.ACTION_UP;
}
MotionEvent event = MotionEvent.obtain(eventDownTimeMs, eventTimeMs, action,
point.x, point.y, 0);
MotionEvent event =
MotionEvent.obtain(eventDownTimeMs, eventTimeMs, action, point.x, point.y, 0);
// Send event.
mState.onReceivedMotionEvent(event);
// In this case the event and raw event values are the same.
mState.onReceivedMotionEvent(event, event, policyFlags);
mManifold.onMotionEvent(event, event, policyFlags);
eventTimeMs += PATH_STEP_MILLISEC;
if (mState.isClear()) {
@@ -178,8 +180,9 @@ public class GestureManifoldTest {
mState.clear();
// Check that correct gesture was recognized.
verify(mResultListener).onGestureCompleted(
argThat(gestureEvent -> gestureEvent.getGestureId() == gestureId));
verify(mResultListener)
.onGestureCompleted(
argThat(gestureEvent -> gestureEvent.getGestureId() == gestureId));
}
private boolean onGestureStarted() {