diff --git a/api/current.txt b/api/current.txt index 6c4904166ea67..d2f80c7448fa7 100644 --- a/api/current.txt +++ b/api/current.txt @@ -334,6 +334,7 @@ package android { field public static final int calendarViewShown = 16843596; // 0x101034c field public static final int calendarViewStyle = 16843613; // 0x101035d field public static final int canControlMagnification = 16844040; // 0x1010508 + field public static final int canPerformGestures = 16844046; // 0x101050e field public static final int canRequestEnhancedWebAccessibility = 16843736; // 0x10103d8 field public static final int canRequestFilterKeyEvents = 16843737; // 0x10103d9 field public static final int canRequestTouchExplorationMode = 16843735; // 0x10103d7 @@ -2610,6 +2611,7 @@ package android.accessibilityservice { public abstract class AccessibilityService extends android.app.Service { ctor public AccessibilityService(); + method public final boolean dispatchGesture(android.accessibilityservice.GestureDescription, android.accessibilityservice.AccessibilityService.GestureResultCallback, android.os.Handler); method public android.view.accessibility.AccessibilityNodeInfo findFocus(int); method public final android.accessibilityservice.AccessibilityService.MagnificationController getMagnificationController(); method public android.view.accessibility.AccessibilityNodeInfo getRootInActiveWindow(); @@ -2649,6 +2651,12 @@ package android.accessibilityservice { field public static final java.lang.String SERVICE_META_DATA = "android.accessibilityservice"; } + public static abstract class AccessibilityService.GestureResultCallback { + ctor public AccessibilityService.GestureResultCallback(); + method public void onCancelled(android.accessibilityservice.GestureDescription); + method public void onCompleted(android.accessibilityservice.GestureDescription); + } + public static final class AccessibilityService.MagnificationController { method public void addListener(android.accessibilityservice.AccessibilityService.MagnificationController.OnMagnificationChangedListener); method public void addListener(android.accessibilityservice.AccessibilityService.MagnificationController.OnMagnificationChangedListener, android.os.Handler); @@ -2681,6 +2689,7 @@ package android.accessibilityservice { method public java.lang.String loadDescription(android.content.pm.PackageManager); method public void writeToParcel(android.os.Parcel, int); field public static final int CAPABILITY_CAN_CONTROL_MAGNIFICATION = 16; // 0x10 + field public static final int CAPABILITY_CAN_PERFORM_GESTURES = 32; // 0x20 field public static final int CAPABILITY_CAN_REQUEST_ENHANCED_WEB_ACCESSIBILITY = 4; // 0x4 field public static final int CAPABILITY_CAN_REQUEST_FILTER_KEY_EVENTS = 8; // 0x8 field public static final int CAPABILITY_CAN_REQUEST_TOUCH_EXPLORATION = 2; // 0x2 @@ -2707,6 +2716,30 @@ package android.accessibilityservice { field public java.lang.String[] packageNames; } + public final class GestureDescription { + method public static android.accessibilityservice.GestureDescription createClick(int, int); + method public static android.accessibilityservice.GestureDescription createLongClick(int, int); + method public static android.accessibilityservice.GestureDescription createPinch(int, int, int, int, float, long); + method public static android.accessibilityservice.GestureDescription createSwipe(int, int, int, int, long); + method public android.accessibilityservice.GestureDescription.StrokeDescription getStroke(int); + method public int getStrokeCount(); + field public static final long MAX_GESTURE_DURATION_MS = 60000L; // 0xea60L + field public static final int MAX_STROKE_COUNT = 10; // 0xa + } + + public static class GestureDescription.Builder { + ctor public GestureDescription.Builder(); + method public android.accessibilityservice.GestureDescription.Builder addStroke(android.accessibilityservice.GestureDescription.StrokeDescription); + method public android.accessibilityservice.GestureDescription build(); + } + + public static class GestureDescription.StrokeDescription { + ctor public GestureDescription.StrokeDescription(android.graphics.Path, long, long); + method public long getDuration(); + method public android.graphics.Path getPath(); + method public long getStartTime(); + } + } package android.accounts { diff --git a/api/system-current.txt b/api/system-current.txt index 3ede80ac0d053..e83c90fc1edd4 100644 --- a/api/system-current.txt +++ b/api/system-current.txt @@ -428,6 +428,7 @@ package android { field public static final int calendarViewShown = 16843596; // 0x101034c field public static final int calendarViewStyle = 16843613; // 0x101035d field public static final int canControlMagnification = 16844040; // 0x1010508 + field public static final int canPerformGestures = 16844046; // 0x101050e field public static final int canRequestEnhancedWebAccessibility = 16843736; // 0x10103d8 field public static final int canRequestFilterKeyEvents = 16843737; // 0x10103d9 field public static final int canRequestTouchExplorationMode = 16843735; // 0x10103d7 @@ -2711,6 +2712,7 @@ package android.accessibilityservice { public abstract class AccessibilityService extends android.app.Service { ctor public AccessibilityService(); + method public final boolean dispatchGesture(android.accessibilityservice.GestureDescription, android.accessibilityservice.AccessibilityService.GestureResultCallback, android.os.Handler); method public android.view.accessibility.AccessibilityNodeInfo findFocus(int); method public final android.accessibilityservice.AccessibilityService.MagnificationController getMagnificationController(); method public android.view.accessibility.AccessibilityNodeInfo getRootInActiveWindow(); @@ -2750,6 +2752,12 @@ package android.accessibilityservice { field public static final java.lang.String SERVICE_META_DATA = "android.accessibilityservice"; } + public static abstract class AccessibilityService.GestureResultCallback { + ctor public AccessibilityService.GestureResultCallback(); + method public void onCancelled(android.accessibilityservice.GestureDescription); + method public void onCompleted(android.accessibilityservice.GestureDescription); + } + public static final class AccessibilityService.MagnificationController { method public void addListener(android.accessibilityservice.AccessibilityService.MagnificationController.OnMagnificationChangedListener); method public void addListener(android.accessibilityservice.AccessibilityService.MagnificationController.OnMagnificationChangedListener, android.os.Handler); @@ -2782,6 +2790,7 @@ package android.accessibilityservice { method public java.lang.String loadDescription(android.content.pm.PackageManager); method public void writeToParcel(android.os.Parcel, int); field public static final int CAPABILITY_CAN_CONTROL_MAGNIFICATION = 16; // 0x10 + field public static final int CAPABILITY_CAN_PERFORM_GESTURES = 32; // 0x20 field public static final int CAPABILITY_CAN_REQUEST_ENHANCED_WEB_ACCESSIBILITY = 4; // 0x4 field public static final int CAPABILITY_CAN_REQUEST_FILTER_KEY_EVENTS = 8; // 0x8 field public static final int CAPABILITY_CAN_REQUEST_TOUCH_EXPLORATION = 2; // 0x2 @@ -2808,6 +2817,30 @@ package android.accessibilityservice { field public java.lang.String[] packageNames; } + public final class GestureDescription { + method public static android.accessibilityservice.GestureDescription createClick(int, int); + method public static android.accessibilityservice.GestureDescription createLongClick(int, int); + method public static android.accessibilityservice.GestureDescription createPinch(int, int, int, int, float, long); + method public static android.accessibilityservice.GestureDescription createSwipe(int, int, int, int, long); + method public android.accessibilityservice.GestureDescription.StrokeDescription getStroke(int); + method public int getStrokeCount(); + field public static final long MAX_GESTURE_DURATION_MS = 60000L; // 0xea60L + field public static final int MAX_STROKE_COUNT = 10; // 0xa + } + + public static class GestureDescription.Builder { + ctor public GestureDescription.Builder(); + method public android.accessibilityservice.GestureDescription.Builder addStroke(android.accessibilityservice.GestureDescription.StrokeDescription); + method public android.accessibilityservice.GestureDescription build(); + } + + public static class GestureDescription.StrokeDescription { + ctor public GestureDescription.StrokeDescription(android.graphics.Path, long, long); + method public long getDuration(); + method public android.graphics.Path getPath(); + method public long getStartTime(); + } + } package android.accounts { diff --git a/api/test-current.txt b/api/test-current.txt index 306cf7362204f..b3a69df6b6de0 100644 --- a/api/test-current.txt +++ b/api/test-current.txt @@ -334,6 +334,7 @@ package android { field public static final int calendarViewShown = 16843596; // 0x101034c field public static final int calendarViewStyle = 16843613; // 0x101035d field public static final int canControlMagnification = 16844040; // 0x1010508 + field public static final int canPerformGestures = 16844046; // 0x101050e field public static final int canRequestEnhancedWebAccessibility = 16843736; // 0x10103d8 field public static final int canRequestFilterKeyEvents = 16843737; // 0x10103d9 field public static final int canRequestTouchExplorationMode = 16843735; // 0x10103d7 @@ -2610,6 +2611,7 @@ package android.accessibilityservice { public abstract class AccessibilityService extends android.app.Service { ctor public AccessibilityService(); + method public final boolean dispatchGesture(android.accessibilityservice.GestureDescription, android.accessibilityservice.AccessibilityService.GestureResultCallback, android.os.Handler); method public android.view.accessibility.AccessibilityNodeInfo findFocus(int); method public final android.accessibilityservice.AccessibilityService.MagnificationController getMagnificationController(); method public android.view.accessibility.AccessibilityNodeInfo getRootInActiveWindow(); @@ -2649,6 +2651,12 @@ package android.accessibilityservice { field public static final java.lang.String SERVICE_META_DATA = "android.accessibilityservice"; } + public static abstract class AccessibilityService.GestureResultCallback { + ctor public AccessibilityService.GestureResultCallback(); + method public void onCancelled(android.accessibilityservice.GestureDescription); + method public void onCompleted(android.accessibilityservice.GestureDescription); + } + public static final class AccessibilityService.MagnificationController { method public void addListener(android.accessibilityservice.AccessibilityService.MagnificationController.OnMagnificationChangedListener); method public void addListener(android.accessibilityservice.AccessibilityService.MagnificationController.OnMagnificationChangedListener, android.os.Handler); @@ -2681,6 +2689,7 @@ package android.accessibilityservice { method public java.lang.String loadDescription(android.content.pm.PackageManager); method public void writeToParcel(android.os.Parcel, int); field public static final int CAPABILITY_CAN_CONTROL_MAGNIFICATION = 16; // 0x10 + field public static final int CAPABILITY_CAN_PERFORM_GESTURES = 32; // 0x20 field public static final int CAPABILITY_CAN_REQUEST_ENHANCED_WEB_ACCESSIBILITY = 4; // 0x4 field public static final int CAPABILITY_CAN_REQUEST_FILTER_KEY_EVENTS = 8; // 0x8 field public static final int CAPABILITY_CAN_REQUEST_TOUCH_EXPLORATION = 2; // 0x2 @@ -2707,6 +2716,30 @@ package android.accessibilityservice { field public java.lang.String[] packageNames; } + public final class GestureDescription { + method public static android.accessibilityservice.GestureDescription createClick(int, int); + method public static android.accessibilityservice.GestureDescription createLongClick(int, int); + method public static android.accessibilityservice.GestureDescription createPinch(int, int, int, int, float, long); + method public static android.accessibilityservice.GestureDescription createSwipe(int, int, int, int, long); + method public android.accessibilityservice.GestureDescription.StrokeDescription getStroke(int); + method public int getStrokeCount(); + field public static final long MAX_GESTURE_DURATION_MS = 60000L; // 0xea60L + field public static final int MAX_STROKE_COUNT = 10; // 0xa + } + + public static class GestureDescription.Builder { + ctor public GestureDescription.Builder(); + method public android.accessibilityservice.GestureDescription.Builder addStroke(android.accessibilityservice.GestureDescription.StrokeDescription); + method public android.accessibilityservice.GestureDescription build(); + } + + public static class GestureDescription.StrokeDescription { + ctor public GestureDescription.StrokeDescription(android.graphics.Path, long, long); + method public long getDuration(); + method public android.graphics.Path getPath(); + method public long getStartTime(); + } + } package android.accounts { diff --git a/core/java/android/accessibilityservice/AccessibilityService.java b/core/java/android/accessibilityservice/AccessibilityService.java index 468c145a698b0..3293c264974a4 100644 --- a/core/java/android/accessibilityservice/AccessibilityService.java +++ b/core/java/android/accessibilityservice/AccessibilityService.java @@ -16,11 +16,13 @@ package android.accessibilityservice; +import android.accessibilityservice.GestureDescription.MotionEventGenerator; import android.annotation.NonNull; import android.annotation.Nullable; import android.app.Service; import android.content.Context; import android.content.Intent; +import android.content.pm.ParceledListSlice; import android.graphics.Region; import android.os.Handler; import android.os.IBinder; @@ -29,8 +31,11 @@ import android.os.Message; import android.os.RemoteException; import android.util.ArrayMap; import android.util.Log; +import android.util.Pair; import android.util.Slog; +import android.util.SparseArray; import android.view.KeyEvent; +import android.view.MotionEvent; import android.view.WindowManager; import android.view.WindowManagerImpl; import android.view.accessibility.AccessibilityEvent; @@ -41,10 +46,7 @@ import android.view.accessibility.AccessibilityWindowInfo; import com.android.internal.os.HandlerCaller; import com.android.internal.os.SomeArgs; -import java.util.ArrayList; import java.util.List; -import java.util.Map.Entry; -import java.util.Set; /** * An accessibility service runs in the background and receives callbacks by the system @@ -376,6 +378,7 @@ public abstract class AccessibilityService extends Service { public boolean onKeyEvent(KeyEvent event); public void onMagnificationChanged(@NonNull Region region, float scale, float centerX, float centerY); + public void onPerformGestureResult(int sequence, boolean completedSuccessfully); } private int mConnectionId; @@ -388,6 +391,12 @@ public abstract class AccessibilityService extends Service { private MagnificationController mMagnificationController; + private int mGestureStatusCallbackSequence; + + private SparseArray mGestureStatusCallbackInfos; + + private final Object mLock = new Object(); + /** * Callback for {@link android.view.accessibility.AccessibilityEvent}s. * @@ -551,6 +560,88 @@ public abstract class AccessibilityService extends Service { return mMagnificationController; } + /** + * Dispatch a gesture to the touch screen. Any gestures currently in progress, whether from + * the user, this service, or another service, will be cancelled. + *

+ * Note: In order to dispatch gestures, your service + * must declare the capability by setting the + * {@link android.R.styleable#AccessibilityService_canPerformGestures} + * property in its meta-data. For more information, see + * {@link #SERVICE_META_DATA}. + * + * @param gesture The gesture to dispatch + * @param callback The object to call back when the status of the gesture is known. If + * {@code null}, no status is reported. + * @param handler The handler on which to call back the {@code callback} object. If + * {@code null}, the object is called back on the service's main thread. + * + * @return {@code true} if the gesture is dispatched, {@code false} if not. + */ + public final boolean dispatchGesture(@NonNull GestureDescription gesture, + @Nullable GestureResultCallback callback, + @Nullable Handler handler) { + final IAccessibilityServiceConnection connection = + AccessibilityInteractionClient.getInstance().getConnection( + mConnectionId); + if (connection == null) { + return false; + } + List events = MotionEventGenerator.getMotionEventsFromGestureDescription( + gesture, 100); + try { + synchronized (mLock) { + connection.sendMotionEvents(++mGestureStatusCallbackSequence, + new ParceledListSlice<>(events)); + if (callback != null) { + if (mGestureStatusCallbackInfos == null) { + mGestureStatusCallbackInfos = new SparseArray<>(); + } + GestureResultCallbackInfo callbackInfo = new GestureResultCallbackInfo(gesture, + callback, handler); + mGestureStatusCallbackInfos.put(mGestureStatusCallbackSequence, callbackInfo); + } + } + } catch (RemoteException re) { + throw new RuntimeException(re); + } + return true; + } + + void onPerformGestureResult(int sequence, final boolean completedSuccessfully) { + if (mGestureStatusCallbackInfos == null) { + return; + } + GestureResultCallbackInfo callbackInfo; + synchronized (mLock) { + callbackInfo = mGestureStatusCallbackInfos.get(sequence); + } + final GestureResultCallbackInfo finalCallbackInfo = callbackInfo; + if ((callbackInfo != null) && (callbackInfo.gestureDescription != null) + && (callbackInfo.callback != null)) { + if (callbackInfo.handler != null) { + callbackInfo.handler.post(new Runnable() { + @Override + public void run() { + if (completedSuccessfully) { + finalCallbackInfo.callback + .onCompleted(finalCallbackInfo.gestureDescription); + } else { + finalCallbackInfo.callback + .onCancelled(finalCallbackInfo.gestureDescription); + } + } + }); + return; + } + if (completedSuccessfully) { + callbackInfo.callback.onCompleted(callbackInfo.gestureDescription); + } else { + callbackInfo.callback.onCancelled(callbackInfo.gestureDescription); + } + } + } + private void onMagnificationChanged(@NonNull Region region, float scale, float centerX, float centerY) { if (mMagnificationController != null) { @@ -1082,6 +1173,11 @@ public abstract class AccessibilityService extends Service { float scale, float centerX, float centerY) { AccessibilityService.this.onMagnificationChanged(region, scale, centerX, centerY); } + + @Override + public void onPerformGestureResult(int sequence, boolean completedSuccessfully) { + AccessibilityService.this.onPerformGestureResult(sequence, completedSuccessfully); + } }); } @@ -1100,6 +1196,7 @@ public abstract class AccessibilityService extends Service { private static final int DO_CLEAR_ACCESSIBILITY_CACHE = 5; private static final int DO_ON_KEY_EVENT = 6; private static final int DO_ON_MAGNIFICATION_CHANGED = 7; + private static final int DO_GESTURE_COMPLETE = 8; private final HandlerCaller mCaller; @@ -1158,6 +1255,12 @@ public abstract class AccessibilityService extends Service { mCaller.sendMessage(message); } + public void onPerformGestureResult(int sequence, boolean successfully) { + Message message = mCaller.obtainMessageII(DO_GESTURE_COMPLETE, sequence, + successfully ? 1 : 0); + mCaller.sendMessage(message); + } + @Override public void executeMessage(Message message) { switch (message.what) { @@ -1242,9 +1345,47 @@ public abstract class AccessibilityService extends Service { mCallback.onMagnificationChanged(region, scale, centerX, centerY); } return; + case DO_GESTURE_COMPLETE: { + final boolean successfully = message.arg2 == 1; + mCallback.onPerformGestureResult(message.arg1, successfully); + } return; + default : Log.w(LOG_TAG, "Unknown message type " + message.what); } } } + + /** + * Class used to report status of dispatched gestures + */ + public static abstract class GestureResultCallback { + /** Called when the gesture has completed successfully + * + * @param gestureDescription The description of the gesture that completed. + */ + public void onCompleted(GestureDescription gestureDescription) { + } + + /** Called when the gesture was cancelled + * + * @param gestureDescription The description of the gesture that was cancelled. + */ + public void onCancelled(GestureDescription gestureDescription) { + } + } + + /* Object to keep track of gesture result callbacks */ + private static class GestureResultCallbackInfo { + GestureDescription gestureDescription; + GestureResultCallback callback; + Handler handler; + + GestureResultCallbackInfo(GestureDescription gestureDescription, + GestureResultCallback callback, Handler handler) { + this.gestureDescription = gestureDescription; + this.callback = callback; + this.handler = handler; + } + } } diff --git a/core/java/android/accessibilityservice/AccessibilityServiceInfo.java b/core/java/android/accessibilityservice/AccessibilityServiceInfo.java index 2c98fef20cddd..079bdfcb7b905 100644 --- a/core/java/android/accessibilityservice/AccessibilityServiceInfo.java +++ b/core/java/android/accessibilityservice/AccessibilityServiceInfo.java @@ -110,6 +110,12 @@ public class AccessibilityServiceInfo implements Parcelable { */ public static final int CAPABILITY_CAN_CONTROL_MAGNIFICATION = 0x00000010; + /** + * Capability: This accessibility service can perform gestures. + * @see android.R.styleable#AccessibilityService_canPerformGestures + */ + public static final int CAPABILITY_CAN_PERFORM_GESTURES = 0x00000020; + private static final SparseArray sAvailableCapabilityInfos = new SparseArray(); static { @@ -133,6 +139,10 @@ public class AccessibilityServiceInfo implements Parcelable { new CapabilityInfo(CAPABILITY_CAN_CONTROL_MAGNIFICATION, R.string.capability_title_canControlMagnification, R.string.capability_desc_canControlMagnification)); + sAvailableCapabilityInfos.put(CAPABILITY_CAN_PERFORM_GESTURES, + new CapabilityInfo(CAPABILITY_CAN_PERFORM_GESTURES, + R.string.capability_title_canPerformGestures, + R.string.capability_desc_canPerformGestures)); } /** @@ -276,12 +286,7 @@ public class AccessibilityServiceInfo implements Parcelable { /** * This flag requests from the system to filter key events. If this flag * is set the accessibility service will receive the key events before - * applications allowing it implement global shortcuts. Setting this flag - * does not guarantee that this service will filter key events since only - * one service can do so at any given time. This avoids user confusion due - * to behavior change in case different key filtering services are enabled. - * If there is already another key filtering service enabled, this one will - * not receive key events. + * applications allowing it implement global shortcuts. *

* Services that want to set this flag have to declare this capability * in their meta-data by setting the attribute {@link android.R.attr @@ -516,6 +521,10 @@ public class AccessibilityServiceInfo implements Parcelable { .AccessibilityService_canControlMagnification, false)) { mCapabilities |= CAPABILITY_CAN_CONTROL_MAGNIFICATION; } + if (asAttributes.getBoolean(com.android.internal.R.styleable + .AccessibilityService_canPerformGestures, false)) { + mCapabilities |= CAPABILITY_CAN_PERFORM_GESTURES; + } TypedValue peekedValue = asAttributes.peekValue( com.android.internal.R.styleable.AccessibilityService_description); if (peekedValue != null) { @@ -616,6 +625,8 @@ public class AccessibilityServiceInfo implements Parcelable { * @see #CAPABILITY_CAN_REQUEST_TOUCH_EXPLORATION * @see #CAPABILITY_CAN_REQUEST_ENHANCED_WEB_ACCESSIBILITY * @see #CAPABILITY_FILTER_KEY_EVENTS + * @see #CAPABILITY_CAN_CONTROL_MAGNIFICATION + * @see #CAPABILITY_CAN_PERFORM_GESTURES */ public int getCapabilities() { return mCapabilities; @@ -631,6 +642,8 @@ public class AccessibilityServiceInfo implements Parcelable { * @see #CAPABILITY_CAN_REQUEST_TOUCH_EXPLORATION * @see #CAPABILITY_CAN_REQUEST_ENHANCED_WEB_ACCESSIBILITY * @see #CAPABILITY_FILTER_KEY_EVENTS + * @see #CAPABILITY_CAN_CONTROL_MAGNIFICATION + * @see #CAPABILITY_CAN_PERFORM_GESTURES * * @hide */ @@ -933,6 +946,8 @@ public class AccessibilityServiceInfo implements Parcelable { return "CAPABILITY_CAN_FILTER_KEY_EVENTS"; case CAPABILITY_CAN_CONTROL_MAGNIFICATION: return "CAPABILITY_CAN_CONTROL_MAGNIFICATION"; + case CAPABILITY_CAN_PERFORM_GESTURES: + return "CAPABILITY_CAN_PERFORM_GESTURES"; default: return "UNKNOWN"; } diff --git a/core/java/android/accessibilityservice/GestureDescription.java b/core/java/android/accessibilityservice/GestureDescription.java new file mode 100644 index 0000000000000..14aabcf69a6cc --- /dev/null +++ b/core/java/android/accessibilityservice/GestureDescription.java @@ -0,0 +1,612 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.accessibilityservice; + +import android.annotation.IntRange; +import android.annotation.NonNull; +import android.graphics.Matrix; +import android.graphics.Path; +import android.graphics.PathMeasure; +import android.graphics.RectF; +import android.view.InputDevice; +import android.view.MotionEvent; +import android.view.MotionEvent.PointerCoords; +import android.view.MotionEvent.PointerProperties; +import android.view.ViewConfiguration; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * Accessibility services with the + * {@link android.R.styleable#AccessibilityService_canPerformGestures} property can dispatch + * gestures. This class describes those gestures. Gestures are made up of one or more strokes. + * Gestures are immutable; use the {@code create} methods to get common gesture, or a + * {@code Builder} to create a new one. + *

+ * Spatial dimensions throughout are in screen pixels. Time is measured in milliseconds. + */ +public final class GestureDescription { + /** Gestures may contain no more than this many strokes */ + public static final int MAX_STROKE_COUNT = 10; + + /** + * Upper bound on total gesture duration. Nearly all gestures will be much shorter. + */ + public static final long MAX_GESTURE_DURATION_MS = 60 * 1000; + + private final List mStrokes = new ArrayList<>(); + private final float[] mTempPos = new float[2]; + + /** + * Create a description of a click gesture + * + * @param x The x coordinate to click. Must not be negative. + * @param y The y coordinate to click. Must not be negative. + * + * @return A description of a click at (x, y) + */ + public static GestureDescription createClick(@IntRange(from = 0) int x, + @IntRange(from = 0) int y) { + Path clickPath = new Path(); + clickPath.moveTo(x, y); + clickPath.lineTo(x + 1, y); + return new GestureDescription( + new StrokeDescription(clickPath, 0, ViewConfiguration.getTapTimeout())); + } + + /** + * Create a description of a long click gesture + * + * @param x The x coordinate to click. Must not be negative. + * @param y The y coordinate to click. Must not be negative. + * + * @return A description of a click at (x, y) + */ + public static GestureDescription createLongClick(@IntRange(from = 0) int x, + @IntRange(from = 0) int y) { + Path clickPath = new Path(); + clickPath.moveTo(x, y); + clickPath.lineTo(x + 1, y); + int longPressTime = ViewConfiguration.getLongPressTimeout(); + return new GestureDescription( + new StrokeDescription(clickPath, 0, longPressTime + (longPressTime / 2))); + } + + /** + * Create a description of a swipe gesture + * + * @param startX The x coordinate of the starting point. Must not be negative. + * @param startY The y coordinate of the starting point. Must not be negative. + * @param endX The x coordinate of the ending point. Must not be negative. + * @param endY The y coordinate of the ending point. Must not be negative. + * @param duration The time, in milliseconds, to complete the gesture. Must not be negative. + * + * @return A description of a swipe from ({@code startX}, {@code startY}) to + * ({@code endX}, {@code endY}) that takes {@code duration} milliseconds. Returns {@code null} + * if the path specified for the swipe is invalid. + */ + public static GestureDescription createSwipe(@IntRange(from = 0) int startX, + @IntRange(from = 0) int startY, + @IntRange(from = 0) int endX, + @IntRange(from = 0) int endY, + @IntRange(from = 0, to = MAX_GESTURE_DURATION_MS) long duration) { + Path swipePath = new Path(); + swipePath.moveTo(startX, startY); + swipePath.lineTo(endX, endY); + return new GestureDescription(new StrokeDescription(swipePath, 0, duration)); + } + + /** + * Create a description for a pinch (or zoom) gesture. + * + * @param centerX The x coordinate of the center of the pinch. Must not be negative. + * @param centerY The y coordinate of the center of the pinch. Must not be negative. + * @param startSpacing The spacing of the touch points at the beginning of the gesture. Must not + * be negative. + * @param endSpacing The spacing of the touch points at the end of the gesture. Must not be + * negative. + * @param orientation The angle, in degrees, of the gesture. 0 represents a horizontal pinch + * @param duration The time, in milliseconds, to complete the gesture. Must not be negative. + * + * @return A description of a pinch centered at ({code centerX}, {@code centerY}) that starts + * with the touch points spaced by {@code startSpacing} and ends with them spaced by + * {@code endSpacing} that lasts {@code duration} ms. Returns {@code null} if either path + * specified for the pinch is invalid. + */ + public static GestureDescription createPinch(@IntRange(from = 0) int centerX, + @IntRange(from = 0) int centerY, + @IntRange(from = 0) int startSpacing, + @IntRange(from = 0) int endSpacing, + float orientation, + @IntRange(from = 0, to = MAX_GESTURE_DURATION_MS) long duration) { + if ((startSpacing < 0) || (endSpacing < 0)) { + throw new IllegalArgumentException("Pinch spacing cannot be negative"); + } + float[] startPoint1 = new float[2]; + float[] endPoint1 = new float[2]; + float[] startPoint2 = new float[2]; + float[] endPoint2 = new float[2]; + + /* Build points for a horizontal gesture centered at the origin */ + startPoint1[0] = startSpacing / 2; + startPoint1[1] = 0; + endPoint1[0] = endSpacing / 2; + endPoint1[1] = 0; + startPoint2[0] = -startSpacing / 2; + startPoint2[1] = 0; + endPoint2[0] = -endSpacing / 2; + endPoint2[1] = 0; + + /* Rotate and translate the points */ + Matrix matrix = new Matrix(); + matrix.setRotate(orientation); + matrix.postTranslate(centerX, centerY); + matrix.mapPoints(startPoint1); + matrix.mapPoints(endPoint1); + matrix.mapPoints(startPoint2); + matrix.mapPoints(endPoint2); + + Path path1 = new Path(); + path1.moveTo(startPoint1[0], startPoint1[1]); + path1.lineTo(endPoint1[0], endPoint1[1]); + Path path2 = new Path(); + path2.moveTo(startPoint2[0], startPoint2[1]); + path2.lineTo(endPoint2[0], endPoint2[1]); + + return new GestureDescription(Arrays.asList( + new StrokeDescription(path1, 0, duration), + new StrokeDescription(path2, 0, duration))); + } + + private GestureDescription() {} + + private GestureDescription(List strokes) { + mStrokes.addAll(strokes); + } + + private GestureDescription(StrokeDescription stroke) { + mStrokes.add(stroke); + } + + /** + * Get the number of stroke in the gesture. + * + * @return the number of strokes in this gesture + */ + public int getStrokeCount() { + return mStrokes.size(); + } + + /** + * Read a stroke from the gesture + * + * @param index the index of the stroke + * + * @return A description of the stroke. + */ + public StrokeDescription getStroke(@IntRange(from = 0) int index) { + return mStrokes.get(index); + } + + /** + * Return the smallest key point (where a path starts or ends) that is at least a specified + * offset + * @param offset the minimum start time + * @return The next key time that is at least the offset or -1 if one can't be found + */ + private long getNextKeyPointAtLeast(long offset) { + long nextKeyPoint = Long.MAX_VALUE; + for (int i = 0; i < mStrokes.size(); i++) { + long thisStartTime = mStrokes.get(i).mStartTime; + if ((thisStartTime < nextKeyPoint) && (thisStartTime >= offset)) { + nextKeyPoint = thisStartTime; + } + long thisEndTime = mStrokes.get(i).mEndTime; + if ((thisEndTime < nextKeyPoint) && (thisEndTime >= offset)) { + nextKeyPoint = thisEndTime; + } + } + return (nextKeyPoint == Long.MAX_VALUE) ? -1L : nextKeyPoint; + } + + /** + * Get the points that correspond to a particular moment in time. + * @param time The time of interest + * @param touchPoints An array to hold the current touch points. Must be preallocated to at + * least the number of paths in the gesture to prevent going out of bounds + * @return The number of points found, and thus the number of elements set in each array + */ + private int getPointsForTime(long time, TouchPoint[] touchPoints) { + int numPointsFound = 0; + for (int i = 0; i < mStrokes.size(); i++) { + StrokeDescription strokeDescription = mStrokes.get(i); + if (strokeDescription.hasPointForTime(time)) { + touchPoints[numPointsFound].mPathIndex = i; + touchPoints[numPointsFound].mIsStartOfPath = (time == strokeDescription.mStartTime); + touchPoints[numPointsFound].mIsEndOfPath = (time == strokeDescription.mEndTime); + strokeDescription.getPosForTime(time, mTempPos); + touchPoints[numPointsFound].mX = Math.round(mTempPos[0]); + touchPoints[numPointsFound].mY = Math.round(mTempPos[1]); + numPointsFound++; + } + } + return numPointsFound; + } + + // Total duration assumes that the gesture starts at 0; waiting around to start a gesture + // counts against total duration + private static long getTotalDuration(List paths) { + long latestEnd = Long.MIN_VALUE; + for (int i = 0; i < paths.size(); i++) { + StrokeDescription path = paths.get(i); + latestEnd = Math.max(latestEnd, path.mEndTime); + } + return Math.max(latestEnd, 0); + } + + /** + * Builder for a {@code GestureDescription} + */ + public static class Builder { + + private final List mStrokes = new ArrayList<>(); + + /** + * Add a stroke to the gesture description. Up to {@code MAX_STROKE_COUNT} paths may be + * added to a gesture, and the total gesture duration (earliest path start time to latest path + * end time) may not exceed {@code MAX_GESTURE_DURATION_MS}. + * + * @param strokeDescription the stroke to add. + * + * @return this + */ + public Builder addStroke(@NonNull StrokeDescription strokeDescription) { + if (mStrokes.size() >= MAX_STROKE_COUNT) { + throw new RuntimeException("Attempting to add too many strokes to a gesture"); + } + + mStrokes.add(strokeDescription); + + if (getTotalDuration(mStrokes) > MAX_GESTURE_DURATION_MS) { + mStrokes.remove(strokeDescription); + throw new RuntimeException("Gesture would exceed maximum duration with new stroke"); + } + return this; + } + + public GestureDescription build() { + if (mStrokes.size() == 0) { + throw new RuntimeException("Gestures must have at least one stroke"); + } + return new GestureDescription(mStrokes); + } + } + + /** + * Immutable description of stroke that can be part of a gesture. + */ + public static class StrokeDescription { + Path mPath; + long mStartTime; + long mEndTime; + private float mTimeToLengthConversion; + private PathMeasure mPathMeasure; + + /** + * @param path The path to follow. Must have exactly one contour, and that contour must + * have nonzero length. The bounds of the path must not be negative. + * @param startTime The time, in milliseconds, from the time the gesture starts to the + * time the stroke should start. Must not be negative. + * @param duration The duration, in milliseconds, the stroke takes to traverse the path. + * Must not be negative. + */ + public StrokeDescription(@NonNull Path path, + @IntRange(from = 0, to = MAX_GESTURE_DURATION_MS) long startTime, + @IntRange(from = 0, to = MAX_GESTURE_DURATION_MS) long duration) { + if (duration <= 0) { + throw new IllegalArgumentException("Duration must be positive"); + } + if (startTime < 0) { + throw new IllegalArgumentException("Start time must not be negative"); + } + RectF bounds = new RectF(); + path.computeBounds(bounds, false /* unused */); + if ((bounds.bottom < 0) || (bounds.top < 0) || (bounds.right < 0) + || (bounds.left < 0)) { + throw new IllegalArgumentException("Path bounds must not be negative"); + } + mPath = new Path(path); + mPathMeasure = new PathMeasure(path, false); + if (mPathMeasure.getLength() == 0) { + throw new IllegalArgumentException("Path has zero length"); + } + if (mPathMeasure.nextContour()) { + throw new IllegalArgumentException("Path has more than one contour"); + } + /* + * Calling nextContour has moved mPathMeasure off the first contour, which is the only + * one we care about. Set the path again to go back to the first contour. + */ + mPathMeasure.setPath(path, false); + mStartTime = startTime; + mEndTime = startTime + duration; + if (duration > 0) { + mTimeToLengthConversion = getLength() / duration; + } + } + + /** + * Retrieve a copy of the path for this stroke + * + * @return A copy of the path + */ + public Path getPath() { + return new Path(mPath); + } + + /** + * Get the stroke's start time + * + * @return the start time for this stroke. + */ + public long getStartTime() { + return mStartTime; + } + + /** + * Get the stroke's duration + * + * @return the duration for this stroke + */ + public long getDuration() { + return mEndTime - mStartTime; + } + + float getLength() { + return mPathMeasure.getLength(); + } + + /* Assumes hasPointForTime returns true */ + boolean getPosForTime(long time, float[] pos) { + if (time == mEndTime) { + // Close to the end time, roundoff can be a problem + return mPathMeasure.getPosTan(getLength(), pos, null); + } + float length = mTimeToLengthConversion * ((float) (time - mStartTime)); + return mPathMeasure.getPosTan(length, pos, null); + } + + boolean hasPointForTime(long time) { + return ((time >= mStartTime) && (time <= mEndTime)); + } + } + + private static class TouchPoint { + int mPathIndex; + boolean mIsStartOfPath; + boolean mIsEndOfPath; + float mX; + float mY; + + void copyFrom(TouchPoint other) { + mPathIndex = other.mPathIndex; + mIsStartOfPath = other.mIsStartOfPath; + mIsEndOfPath = other.mIsEndOfPath; + mX = other.mX; + mY = other.mY; + } + } + + /** + * Class to convert a GestureDescription to a series of MotionEvents. + */ + static class MotionEventGenerator { + /** + * Constants used to initialize all MotionEvents + */ + private static final int EVENT_META_STATE = 0; + private static final int EVENT_BUTTON_STATE = 0; + private static final int EVENT_DEVICE_ID = 0; + private static final int EVENT_EDGE_FLAGS = 0; + private static final int EVENT_SOURCE = InputDevice.SOURCE_TOUCHSCREEN; + private static final int EVENT_FLAGS = 0; + private static final float EVENT_X_PRECISION = 1; + private static final float EVENT_Y_PRECISION = 1; + + /* Lazily-created scratch memory for processing touches */ + private static TouchPoint[] sCurrentTouchPoints; + private static TouchPoint[] sLastTouchPoints; + private static PointerCoords[] sPointerCoords; + private static PointerProperties[] sPointerProps; + + static List getMotionEventsFromGestureDescription( + GestureDescription description, int sampleTimeMs) { + final List motionEvents = new ArrayList<>(); + + // Point data at each time we generate an event for + final TouchPoint[] currentTouchPoints = + getCurrentTouchPoints(description.getStrokeCount()); + // Point data sent in last touch event + int lastTouchPointSize = 0; + final TouchPoint[] lastTouchPoints = + getLastTouchPoints(description.getStrokeCount()); + + /* Loop through each time slice where there are touch points */ + long timeSinceGestureStart = 0; + long nextKeyPointTime = description.getNextKeyPointAtLeast(timeSinceGestureStart); + while (nextKeyPointTime >= 0) { + timeSinceGestureStart = (lastTouchPointSize == 0) ? nextKeyPointTime + : Math.min(nextKeyPointTime, timeSinceGestureStart + sampleTimeMs); + int currentTouchPointSize = description.getPointsForTime(timeSinceGestureStart, + currentTouchPoints); + + appendMoveEventIfNeeded(motionEvents, lastTouchPoints, lastTouchPointSize, + currentTouchPoints, currentTouchPointSize, timeSinceGestureStart); + lastTouchPointSize = appendUpEvents(motionEvents, lastTouchPoints, + lastTouchPointSize, currentTouchPoints, currentTouchPointSize, + timeSinceGestureStart); + lastTouchPointSize = appendDownEvents(motionEvents, lastTouchPoints, + lastTouchPointSize, currentTouchPoints, currentTouchPointSize, + timeSinceGestureStart); + + /* Move to next time slice */ + nextKeyPointTime = description.getNextKeyPointAtLeast(timeSinceGestureStart + 1); + } + return motionEvents; + } + + private static TouchPoint[] getCurrentTouchPoints(int requiredCapacity) { + if ((sCurrentTouchPoints == null) || (sCurrentTouchPoints.length < requiredCapacity)) { + sCurrentTouchPoints = new TouchPoint[requiredCapacity]; + for (int i = 0; i < requiredCapacity; i++) { + sCurrentTouchPoints[i] = new TouchPoint(); + } + } + return sCurrentTouchPoints; + } + + private static TouchPoint[] getLastTouchPoints(int requiredCapacity) { + if ((sLastTouchPoints == null) || (sLastTouchPoints.length < requiredCapacity)) { + sLastTouchPoints = new TouchPoint[requiredCapacity]; + for (int i = 0; i < requiredCapacity; i++) { + sLastTouchPoints[i] = new TouchPoint(); + } + } + return sLastTouchPoints; + } + + private static PointerCoords[] getPointerCoords(int requiredCapacity) { + if ((sPointerCoords == null) || (sPointerCoords.length < requiredCapacity)) { + sPointerCoords = new PointerCoords[requiredCapacity]; + for (int i = 0; i < requiredCapacity; i++) { + sPointerCoords[i] = new PointerCoords(); + } + } + return sPointerCoords; + } + + private static PointerProperties[] getPointerProps(int requiredCapacity) { + if ((sPointerProps == null) || (sPointerProps.length < requiredCapacity)) { + sPointerProps = new PointerProperties[requiredCapacity]; + for (int i = 0; i < requiredCapacity; i++) { + sPointerProps[i] = new PointerProperties(); + } + } + return sPointerProps; + } + + private static void appendMoveEventIfNeeded(List motionEvents, + TouchPoint[] lastTouchPoints, int lastTouchPointsSize, + TouchPoint[] currentTouchPoints, int currentTouchPointsSize, long currentTime) { + /* Look for pointers that have moved */ + boolean moveFound = false; + for (int i = 0; i < currentTouchPointsSize; i++) { + int lastPointsIndex = findPointByPathIndex(lastTouchPoints, lastTouchPointsSize, + currentTouchPoints[i].mPathIndex); + if (lastPointsIndex >= 0) { + moveFound |= (lastTouchPoints[lastPointsIndex].mX != currentTouchPoints[i].mX) + || (lastTouchPoints[lastPointsIndex].mY != currentTouchPoints[i].mY); + lastTouchPoints[lastPointsIndex].copyFrom(currentTouchPoints[i]); + } + } + + if (moveFound) { + long downTime = motionEvents.get(motionEvents.size() - 1).getDownTime(); + motionEvents.add(obtainMotionEvent(downTime, currentTime, MotionEvent.ACTION_MOVE, + lastTouchPoints, lastTouchPointsSize)); + } + } + + private static int appendUpEvents(List motionEvents, + TouchPoint[] lastTouchPoints, int lastTouchPointsSize, + TouchPoint[] currentTouchPoints, int currentTouchPointsSize, long currentTime) { + /* Look for a pointer at the end of its path */ + for (int i = 0; i < currentTouchPointsSize; i++) { + if (currentTouchPoints[i].mIsEndOfPath) { + int indexOfUpEvent = findPointByPathIndex(lastTouchPoints, lastTouchPointsSize, + currentTouchPoints[i].mPathIndex); + if (indexOfUpEvent < 0) { + continue; // Should not happen + } + long downTime = motionEvents.get(motionEvents.size() - 1).getDownTime(); + int action = (lastTouchPointsSize == 1) ? MotionEvent.ACTION_UP + : MotionEvent.ACTION_POINTER_UP; + action |= indexOfUpEvent << MotionEvent.ACTION_POINTER_INDEX_SHIFT; + motionEvents.add(obtainMotionEvent(downTime, currentTime, action, + lastTouchPoints, lastTouchPointsSize)); + /* Remove this point from lastTouchPoints */ + for (int j = indexOfUpEvent; j < lastTouchPointsSize - 1; j++) { + lastTouchPoints[j].copyFrom(lastTouchPoints[j+1]); + } + lastTouchPointsSize--; + } + } + return lastTouchPointsSize; + } + + private static int appendDownEvents(List motionEvents, + TouchPoint[] lastTouchPoints, int lastTouchPointsSize, + TouchPoint[] currentTouchPoints, int currentTouchPointsSize, long currentTime) { + /* Look for a pointer that is just starting */ + for (int i = 0; i < currentTouchPointsSize; i++) { + if (currentTouchPoints[i].mIsStartOfPath) { + /* Add the point to last coords and use the new array to generate the event */ + lastTouchPoints[lastTouchPointsSize++].copyFrom(currentTouchPoints[i]); + int action = (lastTouchPointsSize == 1) ? MotionEvent.ACTION_DOWN + : MotionEvent.ACTION_POINTER_DOWN; + long downTime = (action == MotionEvent.ACTION_DOWN) ? currentTime : + motionEvents.get(motionEvents.size() - 1).getDownTime(); + action |= i << MotionEvent.ACTION_POINTER_INDEX_SHIFT; + motionEvents.add(obtainMotionEvent(downTime, currentTime, action, + lastTouchPoints, lastTouchPointsSize)); + } + } + return lastTouchPointsSize; + } + + private static MotionEvent obtainMotionEvent(long downTime, long eventTime, int action, + TouchPoint[] touchPoints, int touchPointsSize) { + PointerCoords[] pointerCoords = getPointerCoords(touchPointsSize); + PointerProperties[] pointerProperties = getPointerProps(touchPointsSize); + for (int i = 0; i < touchPointsSize; i++) { + pointerProperties[i].id = touchPoints[i].mPathIndex; + pointerProperties[i].toolType = MotionEvent.TOOL_TYPE_UNKNOWN; + pointerCoords[i].clear(); + pointerCoords[i].pressure = 1.0f; + pointerCoords[i].size = 1.0f; + pointerCoords[i].x = touchPoints[i].mX; + pointerCoords[i].y = touchPoints[i].mY; + } + return MotionEvent.obtain(downTime, eventTime, action, touchPointsSize, + pointerProperties, pointerCoords, EVENT_META_STATE, EVENT_BUTTON_STATE, + EVENT_X_PRECISION, EVENT_Y_PRECISION, EVENT_DEVICE_ID, EVENT_EDGE_FLAGS, + EVENT_SOURCE, EVENT_FLAGS); + } + + private static int findPointByPathIndex(TouchPoint[] touchPoints, int touchPointsSize, + int pathIndex) { + for (int i = 0; i < touchPointsSize; i++) { + if (touchPoints[i].mPathIndex == pathIndex) { + return i; + } + } + return -1; + } + } +} diff --git a/core/java/android/accessibilityservice/IAccessibilityServiceClient.aidl b/core/java/android/accessibilityservice/IAccessibilityServiceClient.aidl index 15666bf03ae82..6280542e6ce46 100644 --- a/core/java/android/accessibilityservice/IAccessibilityServiceClient.aidl +++ b/core/java/android/accessibilityservice/IAccessibilityServiceClient.aidl @@ -42,4 +42,6 @@ import android.view.KeyEvent; void onKeyEvent(in KeyEvent event, int sequence); void onMagnificationChanged(in Region region, float scale, float centerX, float centerY); + + void onPerformGestureResult(int sequence, boolean completedSuccessfully); } diff --git a/core/java/android/accessibilityservice/IAccessibilityServiceConnection.aidl b/core/java/android/accessibilityservice/IAccessibilityServiceConnection.aidl index 6ac50bd6efbce..a65b87b7b0c86 100644 --- a/core/java/android/accessibilityservice/IAccessibilityServiceConnection.aidl +++ b/core/java/android/accessibilityservice/IAccessibilityServiceConnection.aidl @@ -16,10 +16,12 @@ package android.accessibilityservice; -import android.os.Bundle; import android.accessibilityservice.AccessibilityServiceInfo; +import android.content.pm.ParceledListSlice; import android.graphics.Region; +import android.os.Bundle; import android.view.MagnificationSpec; +import android.view.MotionEvent; import android.view.accessibility.AccessibilityNodeInfo; import android.view.accessibility.IAccessibilityInteractionConnectionCallback; import android.view.accessibility.AccessibilityWindowInfo; @@ -79,4 +81,6 @@ interface IAccessibilityServiceConnection { boolean animate); void setMagnificationCallbackEnabled(boolean enabled); + + void sendMotionEvents(int sequence, in ParceledListSlice events); } diff --git a/core/java/android/app/UiAutomation.java b/core/java/android/app/UiAutomation.java index 8475840bbae22..dce2e518d1e04 100644 --- a/core/java/android/app/UiAutomation.java +++ b/core/java/android/app/UiAutomation.java @@ -1031,6 +1031,11 @@ public final class UiAutomation { float scale, float centerX, float centerY) { /* do nothing */ } + + @Override + public void onPerformGestureResult(int sequence, boolean completedSuccessfully) { + /* do nothing */ + } }); } } diff --git a/core/res/res/values/attrs.xml b/core/res/res/values/attrs.xml index 6d505a58ab808..e0f9eca0077d9 100644 --- a/core/res/res/values/attrs.xml +++ b/core/res/res/values/attrs.xml @@ -3316,7 +3316,14 @@ i

--> - + + + diff --git a/core/res/res/values/public.xml b/core/res/res/values/public.xml index ebb1b372e7178..acea461f44a1c 100644 --- a/core/res/res/values/public.xml +++ b/core/res/res/values/public.xml @@ -2688,6 +2688,7 @@ + diff --git a/core/res/res/values/strings.xml b/core/res/res/values/strings.xml index be9ba62baa3a6..997371eebf2bb 100644 --- a/core/res/res/values/strings.xml +++ b/core/res/res/values/strings.xml @@ -652,6 +652,12 @@ Control the display\'s zoom level and positioning. + + Perform gestures + + Can tap, swipe, pinch, and perform other + gestures. + diff --git a/core/res/res/values/symbols.xml b/core/res/res/values/symbols.xml index a4654e8a710f5..706dd206fcb2a 100644 --- a/core/res/res/values/symbols.xml +++ b/core/res/res/values/symbols.xml @@ -561,6 +561,8 @@ + + diff --git a/services/accessibility/java/com/android/server/accessibility/AccessibilityInputFilter.java b/services/accessibility/java/com/android/server/accessibility/AccessibilityInputFilter.java index 5f6cbf9f0025e..232c080266c0f 100644 --- a/services/accessibility/java/com/android/server/accessibility/AccessibilityInputFilter.java +++ b/services/accessibility/java/com/android/server/accessibility/AccessibilityInputFilter.java @@ -69,6 +69,13 @@ class AccessibilityInputFilter extends InputFilter implements EventStreamTransfo */ static final int FLAG_FEATURE_AUTOCLICK = 0x00000008; + /** + * Flag for enabling motion event injectsion + * + * @see #setUserAndEnabledFeatures(int, int) + */ + static final int FLAG_FEATURE_INJECT_MOTION_EVENTS = 0x00000010; + private final Runnable mProcessBatchedEventsRunnable = new Runnable() { @Override public void run() { @@ -104,6 +111,8 @@ class AccessibilityInputFilter extends InputFilter implements EventStreamTransfo private MagnificationGestureHandler mMagnificationGestureHandler; + private MotionEventInjector mMotionEventInjector; + private AutoclickController mAutoclickController; private KeyboardInterceptor mKeyboardInterceptor; @@ -191,6 +200,10 @@ class AccessibilityInputFilter extends InputFilter implements EventStreamTransfo } } + public MotionEventInjector getMotionEventInjector() { + return mMotionEventInjector; + } + /** * Gets current event stream state associated with an input event. * @return The event stream state that should be used for the event. Null if the event should @@ -351,6 +364,12 @@ class AccessibilityInputFilter extends InputFilter implements EventStreamTransfo private void enableFeatures() { resetStreamState(); + if ((mEnabledFeatures & FLAG_FEATURE_INJECT_MOTION_EVENTS) != 0) { + mMotionEventInjector = new MotionEventInjector(mContext.getMainLooper()); + addFirstEventHandler(mMotionEventInjector); + mAms.setMotionEventInjector(mMotionEventInjector); + } + if ((mEnabledFeatures & FLAG_FEATURE_AUTOCLICK) != 0) { mAutoclickController = new AutoclickController(mContext, mUserId); addFirstEventHandler(mAutoclickController); @@ -388,6 +407,11 @@ class AccessibilityInputFilter extends InputFilter implements EventStreamTransfo } private void disableFeatures() { + if (mMotionEventInjector != null) { + mAms.setMotionEventInjector(null); + mMotionEventInjector.onDestroy(); + mMotionEventInjector = null; + } if (mAutoclickController != null) { mAutoclickController.onDestroy(); mAutoclickController = null; diff --git a/services/accessibility/java/com/android/server/accessibility/AccessibilityManagerService.java b/services/accessibility/java/com/android/server/accessibility/AccessibilityManagerService.java index 8cf25b3bbd991..39d59525b5228 100644 --- a/services/accessibility/java/com/android/server/accessibility/AccessibilityManagerService.java +++ b/services/accessibility/java/com/android/server/accessibility/AccessibilityManagerService.java @@ -37,6 +37,7 @@ import android.content.Intent; import android.content.IntentFilter; import android.content.ServiceConnection; import android.content.pm.PackageManager; +import android.content.pm.ParceledListSlice; import android.content.pm.ResolveInfo; import android.content.pm.ServiceInfo; import android.content.pm.UserInfo; @@ -72,6 +73,7 @@ import android.view.InputDevice; import android.view.KeyCharacterMap; import android.view.KeyEvent; import android.view.MagnificationSpec; +import android.view.MotionEvent; import android.view.WindowInfo; import android.view.WindowManager; import android.view.WindowManagerInternal; @@ -186,6 +188,8 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub { private KeyEventDispatcher mKeyEventDispatcher; + private MotionEventInjector mMotionEventInjector; + private final Set mTempComponentNameSet = new HashSet<>(); private final List mTempAccessibilityServiceInfoList = @@ -778,6 +782,18 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub { } } + /** + * Called by AccessibilityInputFilter when it creates or destroys the motionEventInjector. + * Not using a getter because the AccessibilityInputFilter isn't thread-safe + * + * @param motionEventInjector The new value of the motionEventInjector. May be null. + */ + void setMotionEventInjector(MotionEventInjector motionEventInjector) { + synchronized (mLock) { + mMotionEventInjector = motionEventInjector; + } + } + /** * Gets a point within the accessibility focused node where we can send down * and up events to perform a click. @@ -1273,6 +1289,9 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub { if (userState.mIsAutoclickEnabled) { flags |= AccessibilityInputFilter.FLAG_FEATURE_AUTOCLICK; } + if (userState.mIsPerformGesturesEnabled) { + flags |= AccessibilityInputFilter.FLAG_FEATURE_INJECT_MOTION_EVENTS; + } if (flags != 0) { if (!mHasInputFilter) { mHasInputFilter = true; @@ -1363,6 +1382,7 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub { updateAccessibilityFocusBehaviorLocked(userState); updateFilterKeyEventsLocked(userState); updateTouchExplorationLocked(userState); + updatePerformGesturesLocked(userState); updateEnhancedWebAccessibilityLocked(userState); updateDisplayColorAdjustmentSettingsLocked(userState); updateMagnificationLocked(userState); @@ -1451,6 +1471,19 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub { } } + private void updatePerformGesturesLocked(UserState userState) { + final int serviceCount = userState.mBoundServices.size(); + for (int i = 0; i < serviceCount; i++) { + Service service = userState.mBoundServices.get(i); + if ((service.mAccessibilityServiceInfo.getCapabilities() + & AccessibilityServiceInfo.CAPABILITY_CAN_PERFORM_GESTURES) != 0) { + userState.mIsPerformGesturesEnabled = true; + return; + } + } + userState.mIsPerformGesturesEnabled = false; + } + private void updateFilterKeyEventsLocked(UserState userState) { final int serviceCount = userState.mBoundServices.size(); for (int i = 0; i < serviceCount; i++) { @@ -2563,6 +2596,23 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub { return false; } + @Override + public void sendMotionEvents(int sequence, ParceledListSlice events) { + synchronized (mLock) { + if (mSecurityPolicy.canPerformGestures(this) && (mMotionEventInjector != null)) { + mMotionEventInjector.injectEvents((List) events.getList(), + mServiceInterface, sequence); + return; + } + } + try { + mServiceInterface.onPerformGestureResult(sequence, false); + } catch (RemoteException re) { + Slog.e(LOG_TAG, "Error sending motion event injection failure to " + + mServiceInterface, re); + } + } + @Override public boolean performAccessibilityAction(int accessibilityWindowId, long accessibilityNodeId, int action, Bundle arguments, int interactionId, @@ -3734,6 +3784,11 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub { & AccessibilityServiceInfo.CAPABILITY_CAN_CONTROL_MAGNIFICATION) != 0; } + public boolean canPerformGestures(Service service) { + return (service.mAccessibilityServiceInfo.getCapabilities() + & AccessibilityServiceInfo.CAPABILITY_CAN_PERFORM_GESTURES) != 0; + } + private int resolveProfileParentLocked(int userId) { if (userId != mCurrentUserId) { final long identity = Binder.clearCallingIdentity(); @@ -3878,6 +3933,7 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub { public boolean mIsEnhancedWebAccessibilityEnabled; public boolean mIsDisplayMagnificationEnabled; public boolean mIsAutoclickEnabled; + public boolean mIsPerformGesturesEnabled; public boolean mIsFilterKeyEventsEnabled; public boolean mHasDisplayColorAdjustment; public boolean mAccessibilityFocusOnlyInActiveWindow; diff --git a/services/accessibility/java/com/android/server/accessibility/MotionEventInjector.java b/services/accessibility/java/com/android/server/accessibility/MotionEventInjector.java new file mode 100644 index 0000000000000..800f0e10cc5e6 --- /dev/null +++ b/services/accessibility/java/com/android/server/accessibility/MotionEventInjector.java @@ -0,0 +1,249 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.accessibility; + +import android.accessibilityservice.IAccessibilityServiceClient; +import android.os.Handler; +import android.os.Looper; +import android.os.Message; +import android.os.RemoteException; +import android.os.SystemClock; +import android.util.Slog; +import android.util.SparseArray; +import android.view.InputDevice; +import android.view.KeyEvent; +import android.view.MotionEvent; +import android.view.WindowManagerPolicy; +import android.view.accessibility.AccessibilityEvent; +import com.android.internal.os.SomeArgs; +import com.android.server.accessibility.AccessibilityManagerService.Service; + +import java.util.List; + +/** + * Injects MotionEvents to permit {@code AccessibilityService}s to touch the screen on behalf of + * users. + * + * All methods except {@code injectEvents} must be called only from the main thread. + */ +public class MotionEventInjector implements EventStreamTransformation { + private static final String LOG_TAG = "MotionEventInjector"; + private static final int MESSAGE_SEND_MOTION_EVENT = 1; + private static final int MESSAGE_INJECT_EVENTS = 2; + private static final int MAX_POINTERS = 11; // Non-binding maximum + + private final Handler mHandler; + private final SparseArray mOpenGesturesInProgress = new SparseArray<>(); + + // These two arrays must be the same length + private MotionEvent.PointerProperties[] mPointerProperties = + new MotionEvent.PointerProperties[MAX_POINTERS]; + private MotionEvent.PointerCoords[] mPointerCoords = + new MotionEvent.PointerCoords[MAX_POINTERS]; + private EventStreamTransformation mNext; + private IAccessibilityServiceClient mServiceInterfaceForCurrentGesture; + private int mSequenceForCurrentGesture; + private int mSourceOfInjectedGesture = InputDevice.SOURCE_UNKNOWN; + private boolean mIsDestroyed = false; + + /** + * @param looper A looper on the main thread to use for dispatching new events + */ + public MotionEventInjector(Looper looper) { + mHandler = new Handler(looper, new Callback()); + } + + /** + * Schedule a series of events for injection. These events must comprise a complete, valid + * sequence. All gestures currently in progress will be cancelled, and all {@code downTime} + * and {@code eventTime} fields will be offset by the current time. + * + * @param events The events to inject. Must all be from the same source. + * @param serviceInterface The interface to call back with a result when the gesture is + * either complete or cancelled. + */ + public void injectEvents(List events, + IAccessibilityServiceClient serviceInterface, int sequence) { + SomeArgs args = SomeArgs.obtain(); + args.arg1 = events; + args.arg2 = serviceInterface; + args.argi1 = sequence; + mHandler.sendMessage(mHandler.obtainMessage(MESSAGE_INJECT_EVENTS, args)); + } + + @Override + public void onMotionEvent(MotionEvent event, MotionEvent rawEvent, int policyFlags) { + cancelAnyPendingInjectedEvents(); + sendMotionEventToNext(event, rawEvent, policyFlags); + } + + @Override + public void onKeyEvent(KeyEvent event, int policyFlags) { + if (mNext != null) { + mNext.onKeyEvent(event, policyFlags); + } + } + + @Override + public void onAccessibilityEvent(AccessibilityEvent event) { + if (mNext != null) { + mNext.onAccessibilityEvent(event); + } + } + + @Override + public void setNext(EventStreamTransformation next) { + mNext = next; + } + + @Override + public void clearEvents(int inputSource) { + /* + * Reset state for motion events passing through so we won't send a cancel event for + * them. + */ + if (!mHandler.hasMessages(MESSAGE_SEND_MOTION_EVENT)) { + mOpenGesturesInProgress.put(inputSource, false); + } + } + + @Override + public void onDestroy() { + cancelAnyPendingInjectedEvents(); + mIsDestroyed = true; + } + + private void injectEventsMainThread(List events, + IAccessibilityServiceClient serviceInterface, int sequence) { + if (mIsDestroyed) { + try { + serviceInterface.onPerformGestureResult(sequence, false); + } catch (RemoteException re) { + Slog.e(LOG_TAG, "Error sending status with mIsDestroyed to " + serviceInterface, + re); + } + return; + } + cancelAnyPendingInjectedEvents(); + mSourceOfInjectedGesture = events.get(0).getSource(); + cancelAnyGestureInProgress(mSourceOfInjectedGesture); + mServiceInterfaceForCurrentGesture = serviceInterface; + mSequenceForCurrentGesture = sequence; + if (mNext == null) { + notifyService(false); + return; + } + + long startTime = SystemClock.uptimeMillis(); + for (int i = 0; i < events.size(); i++) { + MotionEvent event = events.get(i); + int numPointers = event.getPointerCount(); + if (numPointers > mPointerCoords.length) { + mPointerCoords = new MotionEvent.PointerCoords[numPointers]; + mPointerProperties = new MotionEvent.PointerProperties[numPointers]; + } + for (int j = 0; j < numPointers; j++) { + if (mPointerCoords[j] == null) { + mPointerCoords[j] = new MotionEvent.PointerCoords(); + mPointerProperties[j] = new MotionEvent.PointerProperties(); + } + event.getPointerCoords(j, mPointerCoords[j]); + event.getPointerProperties(j, mPointerProperties[j]); + } + + /* + * MotionEvent doesn't have a setEventTime() method (it carries around history data, + * which could become inconsistent), so we need to obtain a new one. + */ + MotionEvent offsetEvent = MotionEvent.obtain(startTime + event.getDownTime(), + startTime + event.getEventTime(), event.getAction(), numPointers, + mPointerProperties, mPointerCoords, event.getMetaState(), + event.getButtonState(), event.getXPrecision(), event.getYPrecision(), + event.getDeviceId(), event.getEdgeFlags(), event.getSource(), + event.getFlags()); + Message message = mHandler.obtainMessage(MESSAGE_SEND_MOTION_EVENT, offsetEvent); + mHandler.sendMessageDelayed(message, event.getEventTime()); + } + } + + private void sendMotionEventToNext(MotionEvent event, MotionEvent rawEvent, + int policyFlags) { + if (mNext != null) { + mNext.onMotionEvent(event, rawEvent, policyFlags); + if (event.getActionMasked() == MotionEvent.ACTION_DOWN) { + mOpenGesturesInProgress.put(event.getSource(), true); + } + if ((event.getActionMasked() == MotionEvent.ACTION_UP) + || (event.getActionMasked() == MotionEvent.ACTION_CANCEL)) { + mOpenGesturesInProgress.put(event.getSource(), false); + } + } + } + + private void cancelAnyGestureInProgress(int source) { + if ((mNext != null) && mOpenGesturesInProgress.get(source, false)) { + long now = SystemClock.uptimeMillis(); + MotionEvent cancelEvent = + MotionEvent.obtain(now, now, MotionEvent.ACTION_CANCEL, 0.0f, 0.0f, 0); + sendMotionEventToNext(cancelEvent, cancelEvent, + WindowManagerPolicy.FLAG_PASS_TO_USER); + } + } + + private void cancelAnyPendingInjectedEvents() { + if (mHandler.hasMessages(MESSAGE_SEND_MOTION_EVENT)) { + cancelAnyGestureInProgress(mSourceOfInjectedGesture); + mHandler.removeMessages(MESSAGE_SEND_MOTION_EVENT); + notifyService(false); + } + + } + + private void notifyService(boolean success) { + try { + mServiceInterfaceForCurrentGesture.onPerformGestureResult( + mSequenceForCurrentGesture, success); + } catch (RemoteException re) { + Slog.e(LOG_TAG, "Error sending motion event injection status to " + + mServiceInterfaceForCurrentGesture, re); + } + } + + private class Callback implements Handler.Callback { + @Override + public boolean handleMessage(Message message) { + if (message.what == MESSAGE_INJECT_EVENTS) { + SomeArgs args = (SomeArgs) message.obj; + injectEventsMainThread((List) args.arg1, + (IAccessibilityServiceClient) args.arg2, args.argi1); + args.recycle(); + return true; + } + if (message.what != MESSAGE_SEND_MOTION_EVENT) { + throw new IllegalArgumentException("Unknown message: " + message.what); + } + MotionEvent motionEvent = (MotionEvent) message.obj; + sendMotionEventToNext(motionEvent, motionEvent, + WindowManagerPolicy.FLAG_PASS_TO_USER); + // If the message queue is now empty, then this gesture is complete + if (!mHandler.hasMessages(MESSAGE_SEND_MOTION_EVENT)) { + notifyService(true); + } + return true; + } + } +}