diff --git a/core/api/current.txt b/core/api/current.txt index ec929ee9d1e43..966a449b72e46 100644 --- a/core/api/current.txt +++ b/core/api/current.txt @@ -2994,6 +2994,7 @@ package android.accessibilityservice { } public final class AccessibilityGestureEvent implements android.os.Parcelable { + ctor public AccessibilityGestureEvent(int, int, @NonNull java.util.List); method public int describeContents(); method @NonNull public static String gestureIdToString(int); method public int getDisplayId(); @@ -3016,6 +3017,7 @@ package android.accessibilityservice { method public final android.accessibilityservice.AccessibilityServiceInfo getServiceInfo(); method @NonNull public final android.accessibilityservice.AccessibilityService.SoftKeyboardController getSoftKeyboardController(); method @NonNull public final java.util.List getSystemActions(); + method @NonNull public final android.accessibilityservice.TouchInteractionController getTouchInteractionController(int); method public java.util.List getWindows(); method @NonNull public final android.util.SparseArray> getWindowsOnAllDisplays(); method public abstract void onAccessibilityEvent(android.view.accessibility.AccessibilityEvent); @@ -3250,6 +3252,31 @@ package android.accessibilityservice { method public boolean willContinue(); } + public final class TouchInteractionController { + method public void addListener(@Nullable java.util.concurrent.Executor, @NonNull android.accessibilityservice.TouchInteractionController.Listener); + method public int getDisplayId(); + method public int getMaxPointerCount(); + method public int getState(); + method public void performClick(); + method public void performLongClickAndStartDrag(); + method public void removeAllListeners(); + method public boolean removeListener(@NonNull android.accessibilityservice.TouchInteractionController.Listener); + method public void requestDelegating(); + method public void requestDragging(int); + method public void requestTouchExploration(); + method @NonNull public static String stateToString(int); + field public static final int STATE_CLEAR = 0; // 0x0 + field public static final int STATE_DELEGATING = 4; // 0x4 + field public static final int STATE_DRAGGING = 3; // 0x3 + field public static final int STATE_TOUCH_EXPLORING = 2; // 0x2 + field public static final int STATE_TOUCH_INTERACTING = 1; // 0x1 + } + + public static interface TouchInteractionController.Listener { + method public void onMotionEvent(@NonNull android.view.MotionEvent); + method public void onStateChanged(int); + } + } package android.accounts { diff --git a/core/java/android/accessibilityservice/AccessibilityGestureEvent.java b/core/java/android/accessibilityservice/AccessibilityGestureEvent.java index 768ec38519915..3c9b232511915 100644 --- a/core/java/android/accessibilityservice/AccessibilityGestureEvent.java +++ b/core/java/android/accessibilityservice/AccessibilityGestureEvent.java @@ -150,7 +150,12 @@ public final class AccessibilityGestureEvent implements Parcelable { private final int mDisplayId; private List mMotionEvents = new ArrayList<>(); - /** @hide */ +/** + * Constructs an AccessibilityGestureEvent to be dispatched to an accessibility service. + * @param gestureId the id number of the gesture. + * @param displayId the display on which this gesture was performed. + * @param motionEvents the motion events that lead to this gesture. + */ public AccessibilityGestureEvent( int gestureId, int displayId, @NonNull List motionEvents) { mGestureId = gestureId; diff --git a/core/java/android/accessibilityservice/AccessibilityService.java b/core/java/android/accessibilityservice/AccessibilityService.java index 61d2b4b22e014..0f852b404440b 100644 --- a/core/java/android/accessibilityservice/AccessibilityService.java +++ b/core/java/android/accessibilityservice/AccessibilityService.java @@ -53,6 +53,7 @@ import android.util.Slog; import android.util.SparseArray; import android.view.Display; import android.view.KeyEvent; +import android.view.MotionEvent; import android.view.SurfaceView; import android.view.WindowManager; import android.view.WindowManagerImpl; @@ -65,6 +66,7 @@ import android.view.accessibility.AccessibilityWindowInfo; import com.android.internal.os.HandlerCaller; import com.android.internal.os.SomeArgs; import com.android.internal.util.Preconditions; +import com.android.internal.util.function.pooled.PooledLambda; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; @@ -582,6 +584,10 @@ public abstract class AccessibilityService extends Service { /** Magnification changed callbacks for different displays */ void onMagnificationChanged(int displayId, @NonNull Region region, float scale, float centerX, float centerY); + /** Callbacks for receiving motion events. */ + void onMotionEvent(MotionEvent event); + /** Callback for tuch state changes. */ + void onTouchStateChanged(int displayId, int state); void onSoftKeyboardShowModeChanged(int showMode); void onPerformGestureResult(int sequence, boolean completedSuccessfully); void onFingerprintCapturingGesturesChanged(boolean active); @@ -720,6 +726,12 @@ public abstract class AccessibilityService extends Service { /** List of magnification controllers, mapping from displayId -> MagnificationController. */ private final SparseArray mMagnificationControllers = new SparseArray<>(0); + /** + * List of touch interaction controllers, mapping from displayId -> TouchInteractionController. + */ + private final SparseArray mTouchInteractionControllers = + new SparseArray<>(0); + private SoftKeyboardController mSoftKeyboardController; private final SparseArray mAccessibilityButtonControllers = new SparseArray<>(0); @@ -1194,6 +1206,10 @@ public abstract class AccessibilityService extends Service { getFingerprintGestureController().onGesture(gesture); } + int getConnectionId() { + return mConnectionId; + } + /** * Used to control and query the state of display magnification. */ @@ -2209,6 +2225,16 @@ public abstract class AccessibilityService extends Service { centerX, centerY); } + @Override + public void onMotionEvent(MotionEvent event) { + AccessibilityService.this.onMotionEvent(event); + } + + @Override + public void onTouchStateChanged(int displayId, int state) { + AccessibilityService.this.onTouchStateChanged(displayId, state); + } + @Override public void onSoftKeyboardShowModeChanged(int showMode) { AccessibilityService.this.onSoftKeyboardShowModeChanged(showMode); @@ -2371,6 +2397,21 @@ public abstract class AccessibilityService extends Service { mCaller.sendMessage(mCaller.obtainMessage(DO_ON_SYSTEM_ACTIONS_CHANGED)); } + @Override + public void onMotionEvent(MotionEvent event) { + final Message message = PooledLambda.obtainMessage( + Callbacks::onMotionEvent, mCallback, event); + mCaller.sendMessage(message); + } + + @Override + public void onTouchStateChanged(int displayId, int state) { + final Message message = PooledLambda.obtainMessage(Callbacks::onTouchStateChanged, + mCallback, + displayId, state); + mCaller.sendMessage(message); + } + @Override public void executeMessage(Message message) { switch (message.what) { @@ -2523,7 +2564,7 @@ public abstract class AccessibilityService extends Service { } return; } - default : + default: Log.w(LOG_TAG, "Unknown message type " + message.what); } } @@ -2748,4 +2789,47 @@ public abstract class AccessibilityService extends Service { } } } + + /** + * Returns the touch interaction controller for the specified logical display, which may be used + * to detect gestures and otherwise control touch interactions. If + * {@link AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE} is disabled the + * controller's methods will have no effect. + * + * @param displayId The logical display id, use {@link Display#DEFAULT_DISPLAY} for default + * display. + * @return the TouchExploration controller + */ + @NonNull + public final TouchInteractionController getTouchInteractionController(int displayId) { + synchronized (mLock) { + TouchInteractionController controller = mTouchInteractionControllers.get(displayId); + if (controller == null) { + controller = new TouchInteractionController(this, mLock, displayId); + mTouchInteractionControllers.put(displayId, controller); + } + return controller; + } + } + + void onMotionEvent(MotionEvent event) { + TouchInteractionController controller; + synchronized (mLock) { + int displayId = event.getDisplayId(); + controller = mTouchInteractionControllers.get(displayId); + } + if (controller != null) { + controller.onMotionEvent(event); + } + } + + void onTouchStateChanged(int displayId, int state) { + TouchInteractionController controller; + synchronized (mLock) { + controller = mTouchInteractionControllers.get(displayId); + } + if (controller != null) { + controller.onStateChanged(state); + } + } } diff --git a/core/java/android/accessibilityservice/IAccessibilityServiceClient.aidl b/core/java/android/accessibilityservice/IAccessibilityServiceClient.aidl index 58b0d19f605a6..651c50f475c68 100644 --- a/core/java/android/accessibilityservice/IAccessibilityServiceClient.aidl +++ b/core/java/android/accessibilityservice/IAccessibilityServiceClient.aidl @@ -22,6 +22,7 @@ import android.view.accessibility.AccessibilityEvent; import android.view.accessibility.AccessibilityWindowInfo; import android.accessibilityservice.AccessibilityGestureEvent; import android.view.KeyEvent; +import android.view.MotionEvent; /** * Top-level interface to an accessibility service component. @@ -44,6 +45,10 @@ import android.view.KeyEvent; void onMagnificationChanged(int displayId, in Region region, float scale, float centerX, float centerY); + void onMotionEvent(in MotionEvent event); + + void onTouchStateChanged(int displayId, int state); + void onSoftKeyboardShowModeChanged(int showMode); void onPerformGestureResult(int sequence, boolean completedSuccessfully); diff --git a/core/java/android/accessibilityservice/IAccessibilityServiceConnection.aidl b/core/java/android/accessibilityservice/IAccessibilityServiceConnection.aidl index 6c360e50a7bde..81457ebf2a514 100644 --- a/core/java/android/accessibilityservice/IAccessibilityServiceConnection.aidl +++ b/core/java/android/accessibilityservice/IAccessibilityServiceConnection.aidl @@ -122,4 +122,16 @@ interface IAccessibilityServiceConnection { oneway void logTrace(long timestamp, String where, long loggingTypes, String callingParams, int processId, long threadId, int callingUid, in Bundle serializedCallingStackInBundle); + + void setServiceDetectsGesturesEnabled(int displayId, boolean mode); + + void requestTouchExploration(int displayId); + + void requestDragging(int displayId, int pointerId); + + void requestDelegating(int displayId); + + void onDoubleTap(int displayId); + + void onDoubleTapAndHold(int displayId); } diff --git a/core/java/android/accessibilityservice/TouchInteractionController.java b/core/java/android/accessibilityservice/TouchInteractionController.java new file mode 100644 index 0000000000000..d9be49a4e7dbf --- /dev/null +++ b/core/java/android/accessibilityservice/TouchInteractionController.java @@ -0,0 +1,425 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.accessibilityservice; + +import android.annotation.IntDef; +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.os.RemoteException; +import android.util.ArrayMap; +import android.view.MotionEvent; +import android.view.accessibility.AccessibilityInteractionClient; + +import java.util.concurrent.Executor; + +/** + * This class allows a service to handle touch exploration and the detection of specialized + * accessibility gestures. The service receives motion events and can match those motion events + * against the gestures it supports. The service can also request the framework enter three other + * states of operation for the duration of this interaction. Upon entering any of these states the + * framework will take over and the service will not receive motion events until the start of a new + * interaction. The states are as follows: + * + *
    + *
  • The service can tell the framework that this interaction is touch exploration. The user is + * trying to explore the screen rather than manipulate it. The framework will then convert the + * motion events to hover events to support touch exploration. + *
  • The service can tell the framework that this interaction is a dragging interaction where + * two fingers are used to execute a one-finger gesture such as scrolling the screen. The + * service must specify which of the two fingers should be passed through to rest of the input + * pipeline. + *
  • Finally, the service can request that the framework delegate this interaction, meaning pass + * it through to the rest of the input pipeline as-is. + *
+ * + * When {@link AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE } is enabled, this + * controller will receive all motion events received by the framework for the specified display + * when not touch-exploring or delegating. If the service classifies this interaction as touch + * exploration or delegating the framework will stop sending motion events to the service for the + * duration of this interaction. If the service classifies this interaction as a dragging + * interaction the framework will send motion events to the service to allow the service to + * determine if the interaction still qualifies as dragging or if it has become a delegating + * interaction. If {@link AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE } is disabled + * this controller will not receive any motion events because touch interactions are being passed + * through to the input pipeline unaltered. + * Note that {@link AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE } + * requires setting {@link android.R.attr#canRequestTouchExplorationMode} as well. + */ +public final class TouchInteractionController { + /** The state where the user is not touching the screen. */ + public static final int STATE_CLEAR = 0; + /** + * The state where the user is touching the screen and the service is receiving motion events. + */ + public static final int STATE_TOUCH_INTERACTING = 1; + /** + * The state where the user is explicitly exploring the screen. The service is not receiving + * motion events. + */ + public static final int STATE_TOUCH_EXPLORING = 2; + /** + * The state where the user is dragging with two fingers. The service is not receiving motion + * events. The selected finger is being dispatched to the rest of the input pipeline to execute + * the drag. + */ + public static final int STATE_DRAGGING = 3; + /** + * The user is performing a gesture which is being passed through to the input pipeline as-is. + * The service is not receiving motion events. + */ + public static final int STATE_DELEGATING = 4; + + @IntDef({ + STATE_CLEAR, + STATE_TOUCH_INTERACTING, + STATE_TOUCH_EXPLORING, + STATE_DRAGGING, + STATE_DELEGATING + }) + private @interface State {} + + // The maximum number of pointers that can be touching the screen at once. (See MAX_POINTER_ID + // in frameworks/native/include/input/Input.h) + private static final int MAX_POINTER_COUNT = 32; + + private final AccessibilityService mService; + private final Object mLock; + private final int mDisplayId; + private boolean mServiceDetectsGestures; + /** Map of listeners to executors. Lazily created when adding the first listener. */ + private ArrayMap mListeners; + + // The current state of the display. + private int mState = STATE_CLEAR; + + TouchInteractionController( + @NonNull AccessibilityService service, @NonNull Object lock, int displayId) { + mDisplayId = displayId; + mLock = lock; + mService = service; + } + + /** + * Adds the specified change listener to the list of motion event listeners. The callback will + * run using on the specified {@link Executor}', or on the service's main thread if the + * Executor is {@code null}. + * @param listener the listener to add, must be non-null + * @param executor the executor for this callback, or {@code null} to execute on the service's + * main thread + */ + public void addListener(@Nullable Executor executor, @NonNull Listener listener) { + synchronized (mLock) { + if (mListeners == null) { + mListeners = new ArrayMap<>(); + } + mListeners.put(listener, executor); + if (mListeners.size() == 1) { + setServiceDetectsGestures(true); + } + } + } + + /** + * Removes the specified listener from the list of motion event listeners. + * + * @param listener the listener to remove, must be non-null + * @return {@code true} if the listener was removed, {@code false} otherwise + */ + public boolean removeListener(@NonNull Listener listener) { + if (mListeners == null) { + return false; + } + synchronized (mLock) { + boolean result = mListeners.remove(listener) != null; + if (result && mListeners.size() == 0) { + setServiceDetectsGestures(false); + } + return result; + } + } + + /** + * Removes all listeners and returns control of touch interactions to the framework. + */ + public void removeAllListeners() { + if (mListeners != null) { + synchronized (mLock) { + mListeners.clear(); + setServiceDetectsGestures(false); + } + } + } + + /** + * Dispatches motion events to any registered listeners. This should be called on the service's + * main thread. + */ + void onMotionEvent(MotionEvent event) { + final ArrayMap entries; + synchronized (mLock) { + // Listeners may remove themselves. Perform a shallow copy to avoid concurrent + // modification. + entries = new ArrayMap<>(mListeners); + } + for (int i = 0, count = entries.size(); i < count; i++) { + final Listener listener = entries.keyAt(i); + final Executor executor = entries.valueAt(i); + if (executor != null) { + executor.execute(() -> listener.onMotionEvent(event)); + } else { + // We're already on the main thread, just run the listener. + listener.onMotionEvent(event); + } + } + } + + /** + * Dispatches motion events to any registered listeners. This should be called on the service's + * main thread. + */ + void onStateChanged(@State int state) { + mState = state; + final ArrayMap entries; + synchronized (mLock) { + // Listeners may remove themselves. Perform a shallow copy to avoid concurrent + // modification. + entries = new ArrayMap<>(mListeners); + } + for (int i = 0, count = entries.size(); i < count; i++) { + final Listener listener = entries.keyAt(i); + final Executor executor = entries.valueAt(i); + if (executor != null) { + executor.execute(() -> listener.onStateChanged(state)); + } else { + // We're already on the main thread, just run the listener. + listener.onStateChanged(state); + } + } + } + + /** + * When {@link AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE} is enabled, this + * controller will receive all motion events received by the framework for the specified display + * when not touch-exploring, delegating, or dragging. This allows the service to detect its own + * gestures, and use its own logic to judge when the framework should start touch-exploring, + * delegating, or dragging. If {@link + * AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE } is disabled this flag has no + * effect. + * + * @see AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE + */ + private void setServiceDetectsGestures(boolean mode) { + final IAccessibilityServiceConnection connection = + AccessibilityInteractionClient.getInstance() + .getConnection(mService.getConnectionId()); + if (connection != null) { + try { + connection.setServiceDetectsGesturesEnabled(mDisplayId, mode); + mServiceDetectsGestures = mode; + } catch (RemoteException re) { + throw new RuntimeException(re); + } + } + } + + /** + * If {@link AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE} is enabled and at + * least one listener has been added for this display this function tells the framework to + * initiate touch exploration. Touch exploration will continue for the duration of this + * interaction. + */ + public void requestTouchExploration() { + checkState(); + final IAccessibilityServiceConnection connection = + AccessibilityInteractionClient.getInstance() + .getConnection(mService.getConnectionId()); + if (connection != null) { + try { + connection.requestTouchExploration(mDisplayId); + } catch (RemoteException re) { + throw new RuntimeException(re); + } + } + } + + /** + * If {@link AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE} and {@link If + * {@link AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE} is enabled and at least + * one listener has been added, this function tells the framework to initiate a dragging + * interaction using the specified pointer. The pointer's movements will be passed through to + * the rest of the input pipeline. Dragging is often used to perform two-finger scrolling. + * + * @param pointerId the pointer to be passed through to the rest of the input pipeline. If the + * pointer id is valid but not actually present on the screen it will be ignored. + * @throws IllegalArgumentException if the pointer id is outside of the allowed range. + */ + public void requestDragging(int pointerId) { + checkState(); + if (pointerId < 0 || pointerId > MAX_POINTER_COUNT) { + throw new IllegalArgumentException("Invalid pointer id: " + pointerId); + } + final IAccessibilityServiceConnection connection = + AccessibilityInteractionClient.getInstance() + .getConnection(mService.getConnectionId()); + if (connection != null) { + try { + connection.requestDragging(mDisplayId, pointerId); + } catch (RemoteException re) { + throw new RuntimeException(re); + } + } + } + + /** + * If {@link AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE} and {@link If + * {@link AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE} is enabled and at least + * one listener has been added, this function tells the framework to initiate a delegating + * interaction. Motion events will be passed through as-is to the rest of the input pipeline for + * the duration of this interaction. + */ + public void requestDelegating() { + checkState(); + final IAccessibilityServiceConnection connection = + AccessibilityInteractionClient.getInstance() + .getConnection(mService.getConnectionId()); + if (connection != null) { + try { + connection.requestDelegating(mDisplayId); + } catch (RemoteException re) { + throw new RuntimeException(re); + } + } + } + + /** + * If {@link AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE} and {@link If + * {@link AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE} is enabled and at least + * one listener has been added, this function tells the framework to perform a click. + * The framework will first try to perform + * {@link AccessibilityNodeInfo.AccessibilityAction#ACTION_CLICK} on the item with + * accessibility focus. If that fails, the framework will simulate a click using motion events + * on the last location to have accessibility focus. + */ + public void performClick() { + final IAccessibilityServiceConnection connection = + AccessibilityInteractionClient.getInstance() + .getConnection(mService.getConnectionId()); + if (connection != null) { + try { + connection.onDoubleTap(mDisplayId); + } catch (RemoteException re) { + throw new RuntimeException(re); + } + } + } + + /** + * If {@link AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE} and {@link If + * {@link AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE} is enabled and at least + * one listener has been added, this function tells the framework to perform a long click. + * The framework will simulate a long click using motion events on the last location with + * accessibility focus and will delegate any movements to the rest of the input pipeline. This + * allows a user to double-tap and hold to trigger a drag and then execute that drag by moving + * their finger. + */ + public void performLongClickAndStartDrag() { + final IAccessibilityServiceConnection connection = + AccessibilityInteractionClient.getInstance() + .getConnection(mService.getConnectionId()); + if (connection != null) { + try { + connection.onDoubleTapAndHold(mDisplayId); + } catch (RemoteException re) { + throw new RuntimeException(re); + } + } + } + + private void checkState() { + if (!mServiceDetectsGestures || mListeners.size() == 0) { + throw new IllegalStateException( + "State transitions are not allowed without first adding a listener."); + } + if (mState != STATE_TOUCH_INTERACTING) { + throw new IllegalStateException( + "State transitions are not allowed in " + stateToString(mState)); + } + } + + /** @return the maximum number of pointers that this display will accept. */ + public int getMaxPointerCount() { + return MAX_POINTER_COUNT; + } + + /** @return the display id associated with this controller. */ + public int getDisplayId() { + return mDisplayId; + } + + /** + * @return the current state of this controller. + * @see TouchInteractionController#STATE_CLEAR + * @see TouchInteractionController#STATE_DELEGATING + * @see TouchInteractionController#STATE_DRAGGING + * @see TouchInteractionController#STATE_TOUCH_EXPLORING + */ + public int getState() { + synchronized (mLock) { + return mState; + } + } + + /** Returns a string representation of the specified state. */ + @NonNull + public static String stateToString(int state) { + switch (state) { + case STATE_CLEAR: + return "STATE_CLEAR"; + case STATE_TOUCH_INTERACTING: + return "STATE_TOUCH_INTERACTING"; + case STATE_TOUCH_EXPLORING: + return "STATE_TOUCH_EXPLORING"; + case STATE_DRAGGING: + return "STATE_DRAGGING"; + case STATE_DELEGATING: + return "STATE_DELEGATING"; + default: + return "Unknown state: " + state; + } + } + + /** Listeners allow services to receive motion events and state change updates. */ + public interface Listener { + /** + * Called when the framework has sent a motion event to the service. + * + * @param event the event being passed to the service. + */ + void onMotionEvent(@NonNull MotionEvent event); + + /** + * Called when the state of motion event dispatch for this display has changed. + * + * @param state the new state of motion event dispatch. + * @see TouchInteractionController#STATE_CLEAR + * @see TouchInteractionController#STATE_DELEGATING + * @see TouchInteractionController#STATE_DRAGGING + * @see TouchInteractionController#STATE_TOUCH_EXPLORING + */ + void onStateChanged(@State int state); + } +} diff --git a/core/java/android/app/UiAutomation.java b/core/java/android/app/UiAutomation.java index 65f71d0483103..828b17156536e 100644 --- a/core/java/android/app/UiAutomation.java +++ b/core/java/android/app/UiAutomation.java @@ -49,6 +49,7 @@ import android.util.SparseArray; import android.view.Display; import android.view.InputEvent; import android.view.KeyEvent; +import android.view.MotionEvent; import android.view.Surface; import android.view.SurfaceControl; import android.view.View; @@ -1542,6 +1543,15 @@ public final class UiAutomation { return false; } + public void onMotionEvent(MotionEvent event) { + /* do nothing */ + } + + @Override + public void onTouchStateChanged(int displayId, int state) { + /* do nothing */ + } + @Override public void onAccessibilityEvent(AccessibilityEvent event) { final OnAccessibilityEventListener listener; diff --git a/core/tests/coretests/src/android/view/accessibility/AccessibilityServiceConnectionImpl.java b/core/tests/coretests/src/android/view/accessibility/AccessibilityServiceConnectionImpl.java index 3045d7d575d9f..c241e36dceb73 100644 --- a/core/tests/coretests/src/android/view/accessibility/AccessibilityServiceConnectionImpl.java +++ b/core/tests/coretests/src/android/view/accessibility/AccessibilityServiceConnectionImpl.java @@ -160,15 +160,27 @@ public class AccessibilityServiceConnectionImpl extends IAccessibilityServiceCon public void takeScreenshot(int displayId, RemoteCallback callback) {} - public void setTouchExplorationPassthroughRegion(int displayId, Region region) {} - - public void setGestureDetectionPassthroughRegion(int displayId, Region region) {} - public void setFocusAppearance(int strokeWidth, int color) {} public void logTrace(long timestamp, String where, String callingParams, int processId, long threadId, int callingUid, Bundle callingStack) {} + public void setGestureDetectionPassthroughRegion(int displayId, Region region) {} + + public void setTouchExplorationPassthroughRegion(int displayId, Region region) {} + + public void setServiceDetectsGesturesEnabled(int displayId, boolean mode) {} + + public void requestTouchExploration(int displayId) {} + + public void requestDragging(int displayId, int pointerId) {} + + public void requestDelegating(int displayId) {} + + public void onDoubleTap(int displayId) {} + + public void onDoubleTapAndHold(int displayId) {} + public void logTrace(long timestamp, String where, long loggingTypes, String callingParams, int processId, long threadId, int callingUid, Bundle serializedCallingStackInBundle) {} } diff --git a/services/accessibility/java/com/android/server/accessibility/AbstractAccessibilityServiceConnection.java b/services/accessibility/java/com/android/server/accessibility/AbstractAccessibilityServiceConnection.java index acdbcb584a8f7..67bb7262b6723 100644 --- a/services/accessibility/java/com/android/server/accessibility/AbstractAccessibilityServiceConnection.java +++ b/services/accessibility/java/com/android/server/accessibility/AbstractAccessibilityServiceConnection.java @@ -256,6 +256,18 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ void setGestureDetectionPassthroughRegion(int displayId, Region region); void setTouchExplorationPassthroughRegion(int displayId, Region region); + + void setServiceDetectsGesturesEnabled(int displayId, boolean mode); + + void requestTouchExploration(int displayId); + + void requestDragging(int displayId, int pointerId); + + void requestDelegating(int displayId); + + void onDoubleTap(int displayId); + + void onDoubleTapAndHold(int displayId); } public AbstractAccessibilityServiceConnection(Context context, ComponentName componentName, @@ -1650,7 +1662,7 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ } } - private IAccessibilityServiceClient getServiceInterfaceSafely() { + protected IAccessibilityServiceClient getServiceInterfaceSafely() { synchronized (mLock) { return mServiceInterface; } @@ -2048,4 +2060,28 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ protected void logTraceWM(String methodName, String params) { mTrace.logTrace(TRACE_WM + "." + methodName, FLAGS_WINDOW_MANAGER_INTERNAL, params); } + + public void setServiceDetectsGesturesEnabled(int displayId, boolean mode) { + mSystemSupport.setServiceDetectsGesturesEnabled(displayId, mode); + } + + public void requestTouchExploration(int displayId) { + mSystemSupport.requestTouchExploration(displayId); + } + + public void requestDragging(int displayId, int pointerId) { + mSystemSupport.requestDragging(displayId, pointerId); + } + + public void requestDelegating(int displayId) { + mSystemSupport.requestDelegating(displayId); + } + + public void onDoubleTap(int displayId) { + mSystemSupport.onDoubleTap(displayId); + } + + public void onDoubleTapAndHold(int displayId) { + mSystemSupport.onDoubleTapAndHold(displayId); + } } diff --git a/services/accessibility/java/com/android/server/accessibility/AccessibilityInputFilter.java b/services/accessibility/java/com/android/server/accessibility/AccessibilityInputFilter.java index 10cfd04ebca6f..75724bffabf80 100644 --- a/services/accessibility/java/com/android/server/accessibility/AccessibilityInputFilter.java +++ b/services/accessibility/java/com/android/server/accessibility/AccessibilityInputFilter.java @@ -893,6 +893,42 @@ class AccessibilityInputFilter extends InputFilter implements EventStreamTransfo } } + public void setServiceDetectsGesturesEnabled(int displayId, boolean mode) { + if (mTouchExplorer.contains(displayId)) { + mTouchExplorer.get(displayId).setServiceDetectsGestures(mode); + } + } + + public void requestTouchExploration(int displayId) { + if (mTouchExplorer.contains(displayId)) { + mTouchExplorer.get(displayId).requestTouchExploration(); + } + } + + public void requestDragging(int displayId, int pointerId) { + if (mTouchExplorer.contains(displayId)) { + mTouchExplorer.get(displayId).requestDragging(pointerId); + } + } + + public void requestDelegating(int displayId) { + if (mTouchExplorer.contains(displayId)) { + mTouchExplorer.get(displayId).requestDelegating(); + } + } + + public void onDoubleTap(int displayId) { + if (mTouchExplorer.contains(displayId)) { + mTouchExplorer.get(displayId).onDoubleTap(); + } + } + + public void onDoubleTapAndHold(int displayId) { + if (mTouchExplorer.contains(displayId)) { + mTouchExplorer.get(displayId).onDoubleTapAndHold(); + } + } + /** * Dumps all {@link AccessibilityInputFilter}s here. */ diff --git a/services/accessibility/java/com/android/server/accessibility/AccessibilityManagerService.java b/services/accessibility/java/com/android/server/accessibility/AccessibilityManagerService.java index e6f91a25f1682..214769bbd9c98 100644 --- a/services/accessibility/java/com/android/server/accessibility/AccessibilityManagerService.java +++ b/services/accessibility/java/com/android/server/accessibility/AccessibilityManagerService.java @@ -44,6 +44,7 @@ import android.accessibilityservice.AccessibilityService; import android.accessibilityservice.AccessibilityServiceInfo; import android.accessibilityservice.AccessibilityShortcutInfo; import android.accessibilityservice.IAccessibilityServiceClient; +import android.accessibilityservice.TouchInteractionController; import android.annotation.NonNull; import android.annotation.Nullable; import android.app.ActivityOptions; @@ -101,6 +102,7 @@ import android.view.Display; import android.view.IWindow; import android.view.KeyEvent; import android.view.MagnificationSpec; +import android.view.MotionEvent; import android.view.WindowManager; import android.view.accessibility.AccessibilityEvent; import android.view.accessibility.AccessibilityInteractionClient; @@ -1198,7 +1200,7 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub } /** - * Called when a gesture is detected on a display. + * Called when a gesture is detected on a display by the framework. * * @param gestureEvent the detail of the gesture. * @return true if the event is handled. @@ -1213,6 +1215,29 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub } } + /** Send a motion event to the service to allow it to perform gesture detection. */ + public boolean sendMotionEventToListeningServices(MotionEvent event) { + synchronized (mLock) { + if (DEBUG) { + Slog.d(LOG_TAG, "Sending event to service: " + event); + } + return notifyMotionEvent(event); + } + } + + /** + * Notifies services that the touch state on a given display has changed. + */ + public boolean onTouchStateChanged(int displayId, int state) { + synchronized (mLock) { + if (DEBUG) { + Slog.d(LOG_TAG, "Notifying touch state:" + + TouchInteractionController.stateToString(state)); + } + return notifyTouchState(displayId, state); + } + } + /** * Called when the system action list is changed. */ @@ -1527,6 +1552,30 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub return false; } + private boolean notifyMotionEvent(MotionEvent event) { + AccessibilityUserState state = getCurrentUserStateLocked(); + for (int i = state.mBoundServices.size() - 1; i >= 0; i--) { + AccessibilityServiceConnection service = state.mBoundServices.get(i); + if (service.mRequestTouchExplorationMode) { + service.notifyMotionEvent(event); + return true; + } + } + return false; + } + + private boolean notifyTouchState(int displayId, int touchState) { + AccessibilityUserState state = getCurrentUserStateLocked(); + for (int i = state.mBoundServices.size() - 1; i >= 0; i--) { + AccessibilityServiceConnection service = state.mBoundServices.get(i); + if (service.mRequestTouchExplorationMode) { + service.notifyTouchState(displayId, touchState); + return true; + } + } + return false; + } + private void notifyClearAccessibilityCacheLocked() { AccessibilityUserState state = getCurrentUserStateLocked(); for (int i = state.mBoundServices.size() - 1; i >= 0; i--) { @@ -2083,7 +2132,6 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub if (userState.isSendMotionEventsEnabled()) { flags |= AccessibilityInputFilter.FLAG_SEND_MOTION_EVENTS; } - if (userState.isAutoclickEnabledLocked()) { flags |= AccessibilityInputFilter.FLAG_FEATURE_AUTOCLICK; } @@ -3960,6 +4008,93 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub } } + @Override + public void setServiceDetectsGesturesEnabled(int displayId, boolean mode) { + mMainHandler.sendMessage( + obtainMessage(AccessibilityManagerService::setServiceDetectsGesturesInternal, this, + displayId, mode)); + } + + private void setServiceDetectsGesturesInternal(int displayId, boolean mode) { + synchronized (mLock) { + if (mHasInputFilter && mInputFilter != null) { + mInputFilter.setServiceDetectsGesturesEnabled(displayId, mode); + } + } + } + + @Override + public void requestTouchExploration(int displayId) { + mMainHandler.sendMessage(obtainMessage( + AccessibilityManagerService::requestTouchExplorationInternal, this, displayId)); + } + + private void requestTouchExplorationInternal(int displayId) { + synchronized (mLock) { + if (mHasInputFilter && mInputFilter != null) { + mInputFilter.requestTouchExploration(displayId); + } + } + } + + @Override + public void requestDragging(int displayId, int pointerId) { + mMainHandler.sendMessage(obtainMessage(AccessibilityManagerService::requestDraggingInternal, + this, displayId, pointerId)); + } + + private void requestDraggingInternal(int displayId, int pointerId) { + synchronized (mLock) { + if (mHasInputFilter && mInputFilter != null) { + mInputFilter.requestDragging(displayId, pointerId); + } + } + } + + @Override + public void requestDelegating(int displayId) { + mMainHandler.sendMessage( + obtainMessage( + AccessibilityManagerService::requestDelegatingInternal, this, displayId)); + } + + private void requestDelegatingInternal(int displayId) { + synchronized (mLock) { + if (mHasInputFilter && mInputFilter != null) { + mInputFilter.requestDelegating(displayId); + } + } + } + + @Override + public void onDoubleTap(int displayId) { + mMainHandler.sendMessage(obtainMessage(AccessibilityManagerService::onDoubleTapInternal, + this, displayId)); + } + + private void onDoubleTapInternal(int displayId) { + synchronized (mLock) { + if (mHasInputFilter && mInputFilter != null) { + mInputFilter.onDoubleTap(displayId); + } + } + } + + @Override + public void onDoubleTapAndHold(int displayId) { + mMainHandler + .sendMessage(obtainMessage(AccessibilityManagerService::onDoubleTapAndHoldInternal, + this, displayId)); + } + + private void onDoubleTapAndHoldInternal(int displayId) { + synchronized (mLock) { + if (mHasInputFilter && mInputFilter != null) { + mInputFilter.onDoubleTapAndHold(displayId); + } + } + } + private void updateFocusAppearanceDataLocked(AccessibilityUserState userState) { if (userState.mUserId != mCurrentUserId) { return; diff --git a/services/accessibility/java/com/android/server/accessibility/AccessibilityServiceConnection.java b/services/accessibility/java/com/android/server/accessibility/AccessibilityServiceConnection.java index 4bf48a25477fc..e9f58708eb2c3 100644 --- a/services/accessibility/java/com/android/server/accessibility/AccessibilityServiceConnection.java +++ b/services/accessibility/java/com/android/server/accessibility/AccessibilityServiceConnection.java @@ -22,6 +22,7 @@ import android.Manifest; import android.accessibilityservice.AccessibilityServiceInfo; import android.accessibilityservice.AccessibilityTrace; import android.accessibilityservice.IAccessibilityServiceClient; +import android.accessibilityservice.TouchInteractionController; import android.app.PendingIntent; import android.content.ComponentName; import android.content.Context; @@ -30,12 +31,15 @@ import android.content.pm.ParceledListSlice; import android.os.Binder; import android.os.Handler; import android.os.IBinder; +import android.os.Message; import android.os.Process; import android.os.RemoteException; import android.os.UserHandle; import android.provider.Settings; import android.util.Slog; import android.view.Display; +import android.view.MotionEvent; + import com.android.server.inputmethod.InputMethodManagerInternal; import com.android.server.wm.ActivityTaskManagerInternal; @@ -454,4 +458,48 @@ class AccessibilityServiceConnection extends AbstractAccessibilityServiceConnect mSystemSupport.onClientChangeLocked(false); } } + + public void notifyMotionEvent(MotionEvent event) { + final Message msg = obtainMessage( + AccessibilityServiceConnection::notifyMotionEventInternal, + AccessibilityServiceConnection.this, event); + mMainHandler.sendMessage(msg); + } + + public void notifyTouchState(int displayId, int state) { + final Message msg = obtainMessage( + AccessibilityServiceConnection::notifyTouchStateInternal, + AccessibilityServiceConnection.this, displayId, state); + mMainHandler.sendMessage(msg); + } + + private void notifyMotionEventInternal(MotionEvent event) { + final IAccessibilityServiceClient listener = getServiceInterfaceSafely(); + if (listener != null) { + try { + if (mTrace.isA11yTracingEnabled()) { + logTraceSvcClient(".onMotionEvent ", + event.toString()); + } + listener.onMotionEvent(event); + } catch (RemoteException re) { + Slog.e(LOG_TAG, "Error sending motion event to" + mService, re); + } + } + } + + private void notifyTouchStateInternal(int displayId, int state) { + final IAccessibilityServiceClient listener = getServiceInterfaceSafely(); + if (listener != null) { + try { + if (mTrace.isA11yTracingEnabled()) { + logTraceSvcClient(".onTouchStateChanged ", + TouchInteractionController.stateToString(state)); + } + listener.onTouchStateChanged(displayId, state); + } catch (RemoteException re) { + Slog.e(LOG_TAG, "Error sending motion event to" + mService, re); + } + } + } } diff --git a/services/accessibility/java/com/android/server/accessibility/gestures/EventDispatcher.java b/services/accessibility/java/com/android/server/accessibility/gestures/EventDispatcher.java index df4a52e308b64..b6223c7b4b35f 100644 --- a/services/accessibility/java/com/android/server/accessibility/gestures/EventDispatcher.java +++ b/services/accessibility/java/com/android/server/accessibility/gestures/EventDispatcher.java @@ -382,18 +382,21 @@ class EventDispatcher { } public boolean longPressWithTouchEvents(MotionEvent event, int policyFlags) { - final int pointerIndex = event.getActionIndex(); - final int pointerId = event.getPointerId(pointerIndex); Point clickLocation = mTempPoint; final int result = computeClickLocation(clickLocation); if (result == CLICK_LOCATION_NONE) { return false; } - mLongPressingPointerId = pointerId; - mLongPressingPointerDeltaX = (int) event.getX(pointerIndex) - clickLocation.x; - mLongPressingPointerDeltaY = (int) event.getY(pointerIndex) - clickLocation.y; - sendDownForAllNotInjectedPointers(event, policyFlags); - return true; + if (event != null) { + final int pointerIndex = event.getActionIndex(); + final int pointerId = event.getPointerId(pointerIndex); + mLongPressingPointerId = pointerId; + mLongPressingPointerDeltaX = (int) event.getX(pointerIndex) - clickLocation.x; + mLongPressingPointerDeltaY = (int) event.getY(pointerIndex) - clickLocation.y; + sendDownForAllNotInjectedPointers(event, policyFlags); + return true; + } + return false; } void clear() { diff --git a/services/accessibility/java/com/android/server/accessibility/gestures/TouchExplorer.java b/services/accessibility/java/com/android/server/accessibility/gestures/TouchExplorer.java index 74f0bcb4245cd..946d22e465890 100644 --- a/services/accessibility/java/com/android/server/accessibility/gestures/TouchExplorer.java +++ b/services/accessibility/java/com/android/server/accessibility/gestures/TouchExplorer.java @@ -46,6 +46,7 @@ import android.graphics.Region; import android.os.Handler; import android.util.DisplayMetrics; import android.util.Slog; +import android.view.Display; import android.view.InputDevice; import android.view.MotionEvent; import android.view.ViewConfiguration; @@ -152,6 +153,7 @@ public class TouchExplorer extends BaseEventStreamTransformation private Region mGestureDetectionPassthroughRegion; private Region mTouchExplorationPassthroughRegion; + private int mDisplayId = Display.INVALID_DISPLAY; /** * Creates a new instance. @@ -182,8 +184,9 @@ public class TouchExplorer extends BaseEventStreamTransformation TouchExplorer(Context context, AccessibilityManagerService service, GestureManifold detector, @NonNull Handler mainHandler) { mContext = context; + mDisplayId = context.getDisplayId(); mAms = service; - mState = new TouchState(); + mState = new TouchState(mDisplayId, mAms); mReceivedPointerTracker = mState.getReceivedPointerTracker(); mDispatcher = new EventDispatcher(context, mAms, super.getNext(), mState); mDetermineUserIntentTimeout = ViewConfiguration.getDoubleTapTimeout(); @@ -267,6 +270,7 @@ public class TouchExplorer extends BaseEventStreamTransformation } try { checkForMalformedEvent(event); + checkForMalformedEvent(rawEvent); } catch (IllegalArgumentException e) { Slog.e(LOG_TAG, "Ignoring malformed event: " + event.toString(), e); return; @@ -277,7 +281,7 @@ public class TouchExplorer extends BaseEventStreamTransformation Slog.d(LOG_TAG, mState.toString()); } - mState.onReceivedMotionEvent(rawEvent); + mState.onReceivedMotionEvent(event, rawEvent, policyFlags); if (shouldPerformGestureDetection(event)) { if (mGestureDetector.onMotionEvent(event, rawEvent, policyFlags)) { // Event was handled by the gesture detector. @@ -307,9 +311,12 @@ public class TouchExplorer extends BaseEventStreamTransformation // It will be delivered on gesture completion or cancelation. // Note that the delay for sending GESTURE_DETECTION_END remains in place. mSendTouchInteractionEndDelayed.cancel(); + if (mState.isServiceDetectingGestures()) { + mAms.sendMotionEventToListeningServices(rawEvent); + } } else { Slog.e(LOG_TAG, "Illegal state: " + mState); - clear(event, policyFlags); + clear(event, policyFlags); } } @@ -367,7 +374,7 @@ public class TouchExplorer extends BaseEventStreamTransformation AccessibilityGestureEvent gestureEvent = new AccessibilityGestureEvent( AccessibilityService.GESTURE_DOUBLE_TAP_AND_HOLD, - event.getDisplayId(), + mDisplayId, mGestureDetector.getMotionEvents()); dispatchGesture(gestureEvent); } @@ -389,7 +396,7 @@ public class TouchExplorer extends BaseEventStreamTransformation AccessibilityGestureEvent gestureEvent = new AccessibilityGestureEvent( AccessibilityService.GESTURE_DOUBLE_TAP, - event.getDisplayId(), + mDisplayId, mGestureDetector.getMotionEvents()); dispatchGesture(gestureEvent); } @@ -404,13 +411,37 @@ public class TouchExplorer extends BaseEventStreamTransformation if (!mAms.performActionOnAccessibilityFocusedItem( AccessibilityNodeInfo.AccessibilityAction.ACTION_CLICK)) { Slog.e(LOG_TAG, "ACTION_CLICK failed. Dispatching motion events to simulate click."); - - mDispatcher.clickWithTouchEvents(event, rawEvent, policyFlags); + if (event != null && rawEvent != null) { + mDispatcher.clickWithTouchEvents(event, rawEvent, policyFlags); + } return true; } return true; } + /** + * Executes a double-tap. The framework will first attempt to execute the appropriate + * accessibility action and if that fails, the framework will deliver touch events to the last + * touch-explored location. + */ + public void onDoubleTap() { + MotionEvent event = mState.getLastReceivedEvent(); + MotionEvent rawEvent = mState.getLastReceivedRawEvent(); + int policyFlags = mState.getLastReceivedPolicyFlags(); + onDoubleTap(event, rawEvent, policyFlags); + } + + /** + * Executes a double-tap and hold gesture using touch events. The user can continue to move + * their finger around the screen to execute a drag. + */ + public void onDoubleTapAndHold() { + MotionEvent event = mState.getLastReceivedEvent(); + MotionEvent rawEvent = mState.getLastReceivedRawEvent(); + int policyFlags = mState.getLastReceivedPolicyFlags(); + onDoubleTapAndHold(event, rawEvent, policyFlags); + } + @Override public boolean onGestureStarted() { if (mAms.getTraceManager().isA11yTracingEnabledForTypes(LOGGING_FLAGS)) { @@ -473,7 +504,7 @@ public class TouchExplorer extends BaseEventStreamTransformation AccessibilityGestureEvent gestureEvent = new AccessibilityGestureEvent( AccessibilityService.GESTURE_UNKNOWN, - event.getDisplayId(), + mDisplayId, mGestureDetector.getMotionEvents()); dispatchGesture(gestureEvent); } @@ -502,7 +533,6 @@ public class TouchExplorer extends BaseEventStreamTransformation */ private void handleActionDown(MotionEvent event, MotionEvent rawEvent, int policyFlags) { mAms.onTouchInteractionStart(); - // If we still have not notified the user for the last // touch, we figure out what to do. If were waiting // we resent the delayed callback and wait again. @@ -512,7 +542,6 @@ public class TouchExplorer extends BaseEventStreamTransformation if (mState.isTouchExploring()) { sendHoverExitAndTouchExplorationGestureEndIfNeeded(policyFlags); } - if (mState.isClear()) { if (!mSendHoverEnterAndMoveDelayed.isPending()) { // Queue a delayed transition to STATE_TOUCH_EXPLORING. @@ -521,7 +550,14 @@ public class TouchExplorer extends BaseEventStreamTransformation // The idea is to avoid getting stuck in STATE_TOUCH_INTERACTING final int pointerId = mReceivedPointerTracker.getPrimaryPointerId(); final int pointerIdBits = (1 << pointerId); - mSendHoverEnterAndMoveDelayed.post(event, rawEvent, pointerIdBits, policyFlags); + if (mState.isServiceDetectingGestures()) { + // This transition will be triggered manually by the service. + mSendHoverEnterAndMoveDelayed.setPointerIdBits(pointerIdBits); + mSendHoverEnterAndMoveDelayed.setPolicyFlags(policyFlags); + mSendHoverEnterAndMoveDelayed.addEvent(event, rawEvent); + } else { + mSendHoverEnterAndMoveDelayed.post(event, rawEvent, pointerIdBits, policyFlags); + } } else { // Cache the event until we discern exploration from gesturing. mSendHoverEnterAndMoveDelayed.addEvent(event, rawEvent); @@ -548,6 +584,9 @@ public class TouchExplorer extends BaseEventStreamTransformation // Avoid duplicated TYPE_TOUCH_INTERACTION_START event when 2nd tap of double tap. mSendTouchInteractionEndDelayed.cancel(); } + if (mState.isServiceDetectingGestures()) { + mAms.sendMotionEventToListeningServices(rawEvent); + } } /** @@ -571,6 +610,11 @@ public class TouchExplorer extends BaseEventStreamTransformation case ACTION_MOVE: handleActionMoveStateTouchInteracting(event, rawEvent, policyFlags); break; + case ACTION_POINTER_UP: + if (mState.isServiceDetectingGestures()) { + mAms.sendMotionEventToListeningServices(rawEvent); + } + break; case ACTION_UP: handleActionUp(event, rawEvent, policyFlags); break; @@ -612,15 +656,16 @@ public class TouchExplorer extends BaseEventStreamTransformation // Another finger down means that if we have not started to deliver // hover events, we will not have to. The code for ACTION_MOVE will // decide what we will actually do next. - if (mSendHoverEnterAndMoveDelayed.isPending()) { mSendHoverEnterAndMoveDelayed.cancel(); mSendHoverExitDelayed.cancel(); } else { - // We have already delivered at least one hover event, so send hover exit to keep the - // stream consistent. + // We have already delivered at least one hover event, so send hover exit to keep sendHoverExitAndTouchExplorationGestureEndIfNeeded(policyFlags); } + if (mState.isServiceDetectingGestures()) { + mAms.sendMotionEventToListeningServices(rawEvent); + } } /** @@ -632,12 +677,19 @@ public class TouchExplorer extends BaseEventStreamTransformation final int pointerId = mReceivedPointerTracker.getPrimaryPointerId(); final int pointerIndex = event.findPointerIndex(pointerId); int pointerIdBits = (1 << pointerId); + if (mState.isServiceDetectingGestures()) { + mAms.sendMotionEventToListeningServices(rawEvent); + mSendHoverEnterAndMoveDelayed.addEvent(event, rawEvent); + return; + } switch (event.getPointerCount()) { case 1: // We have not started sending events since we try to // figure out what the user is doing. if (mSendHoverEnterAndMoveDelayed.isPending()) { // Cache the event until we discern exploration from gesturing. + // When the service is detecting gestures we rely on it to fire touch + // exploration. mSendHoverEnterAndMoveDelayed.addEvent(event, rawEvent); } break; @@ -684,7 +736,7 @@ public class TouchExplorer extends BaseEventStreamTransformation AccessibilityGestureEvent gestureEvent = new AccessibilityGestureEvent( AccessibilityService.GESTURE_PASSTHROUGH, - event.getDisplayId(), + mDisplayId, mGestureDetector.getMotionEvents()); dispatchGesture(gestureEvent); } @@ -708,7 +760,7 @@ public class TouchExplorer extends BaseEventStreamTransformation AccessibilityGestureEvent gestureEvent = new AccessibilityGestureEvent( AccessibilityService.GESTURE_PASSTHROUGH, - event.getDisplayId(), + mDisplayId, mGestureDetector.getMotionEvents()); dispatchGesture(gestureEvent); } @@ -730,7 +782,7 @@ public class TouchExplorer extends BaseEventStreamTransformation AccessibilityGestureEvent gestureEvent = new AccessibilityGestureEvent( AccessibilityService.GESTURE_PASSTHROUGH, - event.getDisplayId(), + mDisplayId, mGestureDetector.getMotionEvents()); dispatchGesture(gestureEvent); } @@ -751,7 +803,7 @@ public class TouchExplorer extends BaseEventStreamTransformation AccessibilityGestureEvent gestureEvent = new AccessibilityGestureEvent( AccessibilityService.GESTURE_PASSTHROUGH, - event.getDisplayId(), + mDisplayId, mGestureDetector.getMotionEvents()); dispatchGesture(gestureEvent); } @@ -767,7 +819,10 @@ public class TouchExplorer extends BaseEventStreamTransformation * Handles ACTION_UP while in the touch interacting state. This event represents all fingers * being lifted from the screen. */ - private void handleActionUp(MotionEvent event, MotionEvent rawEvent, int policyFlags) { + private void handleActionUp(MotionEvent event, MotionEvent rawEvent, int policyFlags) { + if (mState.isServiceDetectingGestures() && mState.isTouchInteracting()) { + mAms.sendMotionEventToListeningServices(rawEvent); + } mAms.onTouchInteractionEnd(); final int pointerId = event.getPointerId(event.getActionIndex()); final int pointerIdBits = (1 << pointerId); @@ -883,6 +938,10 @@ public class TouchExplorer extends BaseEventStreamTransformation clear(event, policyFlags); return; case ACTION_POINTER_DOWN: + if (mState.isServiceDetectingGestures()) { + mAms.sendMotionEventToListeningServices(rawEvent); + return; + } // We are in dragging state so we have two pointers and another one // goes down => delegate the three pointers to the view hierarchy mState.startDelegating(); @@ -896,6 +955,15 @@ public class TouchExplorer extends BaseEventStreamTransformation if (mDraggingPointerId == INVALID_POINTER_ID) { break; } + if (mState.isServiceDetectingGestures()) { + // Allow the service to judge whether this is dragging or delegation + mAms.sendMotionEventToListeningServices(rawEvent); + computeDraggingPointerIdIfNeeded(event); + mDispatcher.sendMotionEvent( + event, ACTION_MOVE, rawEvent, pointerIdBits, policyFlags); + return; + + } switch (event.getPointerCount()) { case 1: // do nothing @@ -921,6 +989,10 @@ public class TouchExplorer extends BaseEventStreamTransformation } break; default: + if (mState.isServiceDetectingGestures()) { + mAms.sendMotionEventToListeningServices(rawEvent); + return; + } mState.startDelegating(); mDraggingPointerId = INVALID_POINTER_ID; event = MotionEvent.obtainNoHistory(event); @@ -1210,9 +1282,7 @@ public class TouchExplorer extends BaseEventStreamTransformation mTouchExplorationPassthroughRegion = region; } - /** - * Whether to send the motion events that make up each gesture to the accessibility service. - */ + /** Whether to send the motion events that make up each gesture to the accessibility service. */ public void setSendMotionEventsEnabled(boolean mode) { mGestureDetector.setSendMotionEventsEnabled(mode); } @@ -1221,7 +1291,15 @@ public class TouchExplorer extends BaseEventStreamTransformation return mGestureDetector.isSendMotionEventsEnabled(); } + /** Sets whether or not motion events should be passed to the service to detect gestures. */ + public void setServiceDetectsGestures(boolean mode) { + mState.setServiceDetectsGestures(mode); + } + private boolean shouldPerformGestureDetection(MotionEvent event) { + if (mState.isServiceDetectingGestures()) { + return false; + } if (mState.isDelegating() || mState.isDragging()) { return false; } @@ -1237,8 +1315,105 @@ public class TouchExplorer extends BaseEventStreamTransformation } /** - * Class for delayed exiting from gesture detecting mode. + * This method allows the service to request that TouchExplorer enter the touch exploration + * state. */ + public void requestTouchExploration() { + if (DEBUG) { + Slog.d(LOG_TAG, "Starting touch explorer from service."); + } + if (mState.isServiceDetectingGestures() && mState.isTouchInteracting()) { + // Cancel without deleting events. + mHandler.removeCallbacks(mSendHoverEnterAndMoveDelayed); + mSendHoverEnterAndMoveDelayed.run(); + mSendHoverEnterAndMoveDelayed.clear(); + final MotionEvent prototype = mState.getLastReceivedEvent(); + final MotionEvent rawEvent = mState.getLastReceivedRawEvent(); + final int pointerId = mReceivedPointerTracker.getPrimaryPointerId(); + final int pointerIdBits = (1 << pointerId); + final int policyFlags = mState.getLastReceivedPolicyFlags(); + mSendHoverExitDelayed.post(prototype, rawEvent, pointerIdBits, policyFlags); + } + } + + /** This method allows the service to request that TouchExplorer enter the dragging state. */ + public void requestDragging(int pointerId) { + if (mState.isServiceDetectingGestures()) { + if (pointerId < 0 || pointerId > TouchState.MAX_POINTER_COUNT + || !mReceivedPointerTracker.isReceivedPointerDown(pointerId)) { + Slog.e(LOG_TAG, "Trying to drag with invalid pointer: " + pointerId); + return; + } + if (mState.isTouchExploring()) { + if (mSendHoverExitDelayed.isPending()) { + mSendHoverExitDelayed.forceSendAndRemove(); + } + if (mSendTouchExplorationEndDelayed.isPending()) { + mSendTouchExplorationEndDelayed.forceSendAndRemove(); + } + } + if (!mState.isTouchInteracting()) { + // It makes no sense to drag. + Slog.e(LOG_TAG, "Error: Trying to drag from " + + mState.getStateSymbolicName(mState.getState())); + return; + } + mDraggingPointerId = pointerId; + if (DEBUG) { + Slog.d(LOG_TAG, "Drag requested on pointer " + mDraggingPointerId); + } + MotionEvent event = mState.getLastReceivedEvent(); + MotionEvent rawEvent = mState.getLastReceivedRawEvent(); + if (event == null || rawEvent == null) { + Slog.e(LOG_TAG, "Unable to start dragging: unable to get last event."); + return; + } + int policyFlags = mState.getLastReceivedPolicyFlags(); + int pointerIdBits = 1 << mDraggingPointerId; + event.setEdgeFlags(mReceivedPointerTracker.getLastReceivedDownEdgeFlags()); + MotionEvent downEvent = computeDownEventForDrag(event); + mState.startDragging(); + if (downEvent != null) { + mDispatcher.sendMotionEvent( + downEvent, ACTION_DOWN, rawEvent, pointerIdBits, policyFlags); + mDispatcher.sendMotionEvent(event, ACTION_MOVE, rawEvent, pointerIdBits, + policyFlags); + } else { + mDispatcher.sendMotionEvent(event, ACTION_DOWN, rawEvent, pointerIdBits, + policyFlags); + } + } + } + + /** This method allows the service to request that TouchExplorer enter the delegating state. */ + public void requestDelegating() { + if (mState.isServiceDetectingGestures()) { + if (mState.isTouchExploring()) { + if (mSendHoverExitDelayed.isPending()) { + mSendHoverExitDelayed.forceSendAndRemove(); + } + if (mSendTouchExplorationEndDelayed.isPending()) { + mSendTouchExplorationEndDelayed.forceSendAndRemove(); + } + } + if (!mState.isTouchInteracting()) { + // It makes no sense to delegate. + Slog.e(LOG_TAG, "Error: Trying to delegate from " + + mState.getStateSymbolicName(mState.getState())); + return; + } + mState.startDelegating(); + MotionEvent prototype = mState.getLastReceivedEvent(); + if (prototype == null) { + Slog.d(LOG_TAG, "Unable to start delegating: unable to get last received event."); + return; + } + int policyFlags = mState.getLastReceivedPolicyFlags(); + mDispatcher.sendDownForAllNotInjectedPointers(prototype, policyFlags); + } + } + + /** Class for delayed exiting from gesture detecting mode. */ private final class ExitGestureDetectionModeDelayed implements Runnable { public void post() { @@ -1283,9 +1458,7 @@ public class TouchExplorer extends BaseEventStreamTransformation } } - /** - * Class for delayed sending of hover enter and move events. - */ + /** Class for delayed sending of hover enter and move events. */ class SendHoverEnterAndMoveDelayed implements Runnable { private final String LOG_TAG_SEND_HOVER_DELAYED = "SendHoverEnterAndMoveDelayed"; @@ -1362,31 +1535,46 @@ public class TouchExplorer extends BaseEventStreamTransformation } if (!mEvents.isEmpty() && !mRawEvents.isEmpty()) { // Deliver a down event. - mDispatcher.sendMotionEvent(mEvents.get(0), ACTION_HOVER_ENTER, - mRawEvents.get(0), mPointerIdBits, mPolicyFlags); + mDispatcher.sendMotionEvent( + mEvents.get(0), + ACTION_HOVER_ENTER, + mRawEvents.get(0), + mPointerIdBits, + mPolicyFlags); if (DEBUG) { - Slog.d(LOG_TAG_SEND_HOVER_DELAYED, + Slog.d( + LOG_TAG_SEND_HOVER_DELAYED, "Injecting motion event: ACTION_HOVER_ENTER"); } // Deliver move events. final int eventCount = mEvents.size(); for (int i = 1; i < eventCount; i++) { - mDispatcher.sendMotionEvent(mEvents.get(i), ACTION_HOVER_MOVE, - mRawEvents.get(i), mPointerIdBits, mPolicyFlags); + mDispatcher.sendMotionEvent( + mEvents.get(i), + ACTION_HOVER_MOVE, + mRawEvents.get(i), + mPointerIdBits, + mPolicyFlags); if (DEBUG) { - Slog.d(LOG_TAG_SEND_HOVER_DELAYED, + Slog.d( + LOG_TAG_SEND_HOVER_DELAYED, "Injecting motion event: ACTION_HOVER_MOVE"); } } } clear(); } - } - /** - * Class for delayed sending of hover exit events. - */ + public void setPointerIdBits(int pointerIdBits) { + mPointerIdBits = pointerIdBits; + } + + public void setPolicyFlags(int policyFlags) { + mPolicyFlags = policyFlags; + } + } + /** Class for delayed sending of hover exit events. */ class SendHoverExitDelayed implements Runnable { private final String LOG_TAG_SEND_HOVER_DELAYED = "SendHoverExitDelayed"; @@ -1438,21 +1626,18 @@ public class TouchExplorer extends BaseEventStreamTransformation public void run() { if (DEBUG) { - Slog.d(LOG_TAG_SEND_HOVER_DELAYED, "Injecting motion event:" - + " ACTION_HOVER_EXIT"); + Slog.d( + LOG_TAG_SEND_HOVER_DELAYED, + "Injecting motion event:" + " ACTION_HOVER_EXIT"); } mDispatcher.sendMotionEvent( - mPrototype, - ACTION_HOVER_EXIT, - mRawEvent, - mPointerIdBits, - mPolicyFlags); + mPrototype, ACTION_HOVER_EXIT, mRawEvent, mPointerIdBits, mPolicyFlags); if (!mSendTouchExplorationEndDelayed.isPending()) { mSendTouchExplorationEndDelayed.cancel(); mSendTouchExplorationEndDelayed.post(); } if (mSendTouchInteractionEndDelayed.isPending()) { - mSendTouchInteractionEndDelayed.cancel(); + mSendTouchInteractionEndDelayed.cancel(); mSendTouchInteractionEndDelayed.post(); } clear(); diff --git a/services/accessibility/java/com/android/server/accessibility/gestures/TouchState.java b/services/accessibility/java/com/android/server/accessibility/gestures/TouchState.java index 6ff082673a153..eb718853a6ce5 100644 --- a/services/accessibility/java/com/android/server/accessibility/gestures/TouchState.java +++ b/services/accessibility/java/com/android/server/accessibility/gestures/TouchState.java @@ -22,9 +22,12 @@ import static com.android.server.accessibility.gestures.TouchExplorer.DEBUG; import android.annotation.IntDef; import android.util.Slog; +import android.view.Display; import android.view.MotionEvent; import android.view.accessibility.AccessibilityEvent; +import com.android.server.accessibility.AccessibilityManagerService; + /** * This class describes the state of the touch explorer as well as the state of received and * injected pointers. This data is accessed both for purposes of touch exploration and gesture @@ -36,7 +39,7 @@ public class TouchState { // This constant captures the current implementation detail that // pointer IDs are between 0 and 31 inclusive (subject to change). // (See MAX_POINTER_ID in frameworks/base/include/ui/Input.h) - static final int MAX_POINTER_COUNT = 32; + public static final int MAX_POINTER_COUNT = 32; // Constant referring to the ids bits of all pointers. public static final int ALL_POINTER_ID_BITS = 0xFFFFFFFF; @@ -50,8 +53,8 @@ public class TouchState { public static final int STATE_TOUCH_EXPLORING = 2; // the user is dragging with two fingers. public static final int STATE_DRAGGING = 3; - // The user is performing some other two finger gesture which we pass through to the view - // hierarchy as a one-finger gesture e.g. two-finger scrolling. + // The user is performing some other two finger gesture which we pass through to the + // input pipeline as a one-finger gesture e.g. two-finger pinch. public static final int STATE_DELEGATING = 4; // The user is performing something that might be a gesture. public static final int STATE_GESTURE_DETECTING = 5; @@ -75,6 +78,8 @@ public class TouchState { private MotionEvent mLastReceivedEvent; // The accompanying raw event without any transformations. private MotionEvent mLastReceivedRawEvent; + // The policy flags of the last received event. + int mLastReceivedPolicyFlags; // The id of the last touch explored window. private int mLastTouchedWindowId; // The last injected hover event. @@ -85,14 +90,23 @@ public class TouchState { private long mLastInjectedDownEventTime; // Keep track of which pointers sent to the system are down. private int mInjectedPointersDown; + private boolean mServiceDetectsGestures = false; + // The requested mode for mServiceDetectsGestures. This will take effect on the next touch + // interaction. + private boolean mServiceDetectsGesturesRequested = false; + private AccessibilityManagerService mAms; + private int mDisplayId = Display.INVALID_DISPLAY; - public TouchState() { + public TouchState(int displayId, AccessibilityManagerService ams) { + mDisplayId = displayId; + mAms = ams; mReceivedPointerTracker = new ReceivedPointerTracker(); } /** Clears the internal shared state. */ public void clear() { setState(STATE_CLEAR); + mServiceDetectsGestures = mServiceDetectsGesturesRequested; // Reset the pointer trackers. if (mLastReceivedEvent != null) { mLastReceivedEvent.recycle(); @@ -107,14 +121,19 @@ public class TouchState { * * @param rawEvent The raw touch event. */ - public void onReceivedMotionEvent(MotionEvent rawEvent) { + public void onReceivedMotionEvent(MotionEvent event, MotionEvent rawEvent, int policyFlags) { + if (isClear() && event.getActionMasked() == MotionEvent.ACTION_DOWN) { + clear(); + } if (mLastReceivedEvent != null) { mLastReceivedEvent.recycle(); } if (mLastReceivedRawEvent != null) { mLastReceivedRawEvent.recycle(); } - mLastReceivedEvent = MotionEvent.obtain(rawEvent); + mLastReceivedEvent = MotionEvent.obtain(event); + mLastReceivedRawEvent = MotionEvent.obtain(rawEvent); + mLastReceivedPolicyFlags = policyFlags; mReceivedPointerTracker.onMotionEvent(rawEvent); } @@ -123,7 +142,7 @@ public class TouchState { * * @param event The event to process. */ - void onInjectedMotionEvent(MotionEvent event) { + public void onInjectedMotionEvent(MotionEvent event) { final int action = event.getActionMasked(); final int pointerId = event.getPointerId(event.getActionIndex()); final int pointerFlag = (1 << pointerId); @@ -183,6 +202,7 @@ public class TouchState { } } + /** Updates the state in response to an injected accessibility event. */ public void onInjectedAccessibilityEvent(int type) { // The below state transitions go here because the related events are often sent on a // delay. @@ -195,7 +215,8 @@ public class TouchState { startTouchInteracting(); break; case AccessibilityEvent.TYPE_TOUCH_INTERACTION_END: - clear(); + setState(STATE_CLEAR); + // We will clear when we actually handle the next ACTION_DOWN. break; case AccessibilityEvent.TYPE_TOUCH_EXPLORATION_GESTURE_START: startTouchExploring(); @@ -228,6 +249,9 @@ public class TouchState { Slog.i(LOG_TAG, getStateSymbolicName(mState) + "->" + getStateSymbolicName(state)); } mState = state; + if (mServiceDetectsGestures) { + mAms.onTouchStateChanged(mDisplayId, state); + } } public boolean isTouchExploring() { @@ -314,6 +338,16 @@ public class TouchState { return mLastReceivedEvent; } + /** Gets the most recently received policy flags. */ + public int getLastReceivedPolicyFlags() { + return mLastReceivedPolicyFlags; + } + + /** Gets the most recently received raw event. */ + public MotionEvent getLastReceivedRawEvent() { + return mLastReceivedRawEvent; + } + /** @return The the last injected hover event. */ public MotionEvent getLastInjectedHoverEvent() { return mLastInjectedHoverEvent; @@ -354,6 +388,18 @@ public class TouchState { return mLastInjectedHoverEventForClick; } + public boolean isServiceDetectingGestures() { + return mServiceDetectsGestures; + } + + /** Whether the service is handling gesture detection. */ + public void setServiceDetectsGestures(boolean mode) { + if (DEBUG) { + Slog.d(LOG_TAG, "serviceDetectsGestures: " + mode); + } + mServiceDetectsGesturesRequested = mode; + } + /** This class tracks where and when a pointer went down. It does not track its movement. */ class ReceivedPointerTracker { private static final String LOG_TAG_RECEIVED_POINTER_TRACKER = "ReceivedPointerTracker"; diff --git a/services/tests/servicestests/src/com/android/server/accessibility/gestures/GestureManifoldTest.java b/services/tests/servicestests/src/com/android/server/accessibility/gestures/GestureManifoldTest.java index 0e787853f617b..45f0e672c1049 100644 --- a/services/tests/servicestests/src/com/android/server/accessibility/gestures/GestureManifoldTest.java +++ b/services/tests/servicestests/src/com/android/server/accessibility/gestures/GestureManifoldTest.java @@ -27,18 +27,20 @@ import android.content.Context; import android.graphics.Point; import android.graphics.PointF; import android.os.Handler; +import android.view.Display; import android.view.MotionEvent; import androidx.test.InstrumentationRegistry; +import com.android.server.accessibility.AccessibilityManagerService; + import org.junit.Before; import org.junit.Test; +import org.mockito.Mock; import java.util.ArrayList; -/** - * Tests for GestureManifold - */ +/** Tests for GestureManifold */ public class GestureManifoldTest { // Constants for testRecognizeGesturePath() @@ -50,25 +52,23 @@ public class GestureManifoldTest { private GestureManifold mManifold; private TouchState mState; private GestureManifold.Listener mResultListener; + @Mock private AccessibilityManagerService mMockAms; @Before public void setUp() { Context context = InstrumentationRegistry.getContext(); // Construct a testable GestureManifold. mResultListener = mock(GestureManifold.Listener.class); - mState = new TouchState(); + mState = new TouchState(Display.DEFAULT_DISPLAY, mMockAms); Handler handler = new Handler(context.getMainLooper()); mManifold = new GestureManifold(context, mResultListener, mState, handler); // Play the role of touch explorer in updating the shared state. when(mResultListener.onGestureStarted()).thenReturn(onGestureStarted()); - - } - @Test public void testRecognizeGesturePath() { - final int d = 1000; // Length of each segment in the test gesture, in pixels. + final int d = 1000; // Length of each segment in the test gesture, in pixels. testPath(p(-d, +0), AccessibilityService.GESTURE_SWIPE_LEFT); testPath(p(+d, +0), AccessibilityService.GESTURE_SWIPE_RIGHT); @@ -138,9 +138,10 @@ public class GestureManifoldTest { // For each path step from start (non-inclusive) to end ... add a motion point. for (int step = 1; step < numSteps; ++step) { - path.add(new PointF( - (start.x + (stepX * (float) step)), - (start.y + (stepY * (float) step)))); + path.add( + new PointF( + (start.x + (stepX * (float) step)), + (start.y + (stepY * (float) step)))); } } @@ -164,11 +165,12 @@ public class GestureManifoldTest { } else if (pointIndex == path.size() - 1) { action = MotionEvent.ACTION_UP; } - MotionEvent event = MotionEvent.obtain(eventDownTimeMs, eventTimeMs, action, - point.x, point.y, 0); + MotionEvent event = + MotionEvent.obtain(eventDownTimeMs, eventTimeMs, action, point.x, point.y, 0); // Send event. - mState.onReceivedMotionEvent(event); + // In this case the event and raw event values are the same. + mState.onReceivedMotionEvent(event, event, policyFlags); mManifold.onMotionEvent(event, event, policyFlags); eventTimeMs += PATH_STEP_MILLISEC; if (mState.isClear()) { @@ -178,8 +180,9 @@ public class GestureManifoldTest { mState.clear(); // Check that correct gesture was recognized. - verify(mResultListener).onGestureCompleted( - argThat(gestureEvent -> gestureEvent.getGestureId() == gestureId)); + verify(mResultListener) + .onGestureCompleted( + argThat(gestureEvent -> gestureEvent.getGestureId() == gestureId)); } private boolean onGestureStarted() {