From 2f165944ce6109134e7285a71da32d1a1647960b Mon Sep 17 00:00:00 2001 From: Phil Weaver Date: Wed, 21 Sep 2016 11:18:05 -0700 Subject: [PATCH] Support continuing dispatched a11y gestures. Test: Ran the tests in this CL, as well as new CTS tests for the new API. Bug: 29477207 Change-Id: Ie5aba553286e954f7afe76ccfa97a7e8be9d75af --- api/current.txt | 5 + api/system-current.txt | 5 + api/test-current.txt | 5 + .../GestureDescription.java | 271 ++---- .../AccessibilityManagerService.java | 11 +- .../accessibility/MotionEventInjector.java | 372 ++++++-- .../accessibility/GestureDescriptionTest.java | 420 ++++++++++ .../MotionEventInjectorTest.java | 793 +++++++++++++----- 8 files changed, 1428 insertions(+), 454 deletions(-) create mode 100644 services/tests/servicestests/src/com/android/server/accessibility/GestureDescriptionTest.java diff --git a/api/current.txt b/api/current.txt index f037f920e296b..de32d15be52db 100644 --- a/api/current.txt +++ b/api/current.txt @@ -2799,9 +2799,14 @@ package android.accessibilityservice { public static class GestureDescription.StrokeDescription { ctor public GestureDescription.StrokeDescription(android.graphics.Path, long, long); + ctor public GestureDescription.StrokeDescription(android.graphics.Path, long, long, int, boolean); + method public int getContinuedStrokeId(); method public long getDuration(); + method public int getId(); method public android.graphics.Path getPath(); method public long getStartTime(); + method public boolean isContinued(); + field public static final int INVALID_STROKE_ID = -1; // 0xffffffff } } diff --git a/api/system-current.txt b/api/system-current.txt index 5e54c7f942792..0211a29de4146 100644 --- a/api/system-current.txt +++ b/api/system-current.txt @@ -2913,9 +2913,14 @@ package android.accessibilityservice { public static class GestureDescription.StrokeDescription { ctor public GestureDescription.StrokeDescription(android.graphics.Path, long, long); + ctor public GestureDescription.StrokeDescription(android.graphics.Path, long, long, int, boolean); + method public int getContinuedStrokeId(); method public long getDuration(); + method public int getId(); method public android.graphics.Path getPath(); method public long getStartTime(); + method public boolean isContinued(); + field public static final int INVALID_STROKE_ID = -1; // 0xffffffff } } diff --git a/api/test-current.txt b/api/test-current.txt index 2cc218a555ca2..6254325277975 100644 --- a/api/test-current.txt +++ b/api/test-current.txt @@ -2799,9 +2799,14 @@ package android.accessibilityservice { public static class GestureDescription.StrokeDescription { ctor public GestureDescription.StrokeDescription(android.graphics.Path, long, long); + ctor public GestureDescription.StrokeDescription(android.graphics.Path, long, long, int, boolean); + method public int getContinuedStrokeId(); method public long getDuration(); + method public int getId(); method public android.graphics.Path getPath(); method public long getStartTime(); + method public boolean isContinued(); + field public static final int INVALID_STROKE_ID = -1; // 0xffffffff } } diff --git a/core/java/android/accessibilityservice/GestureDescription.java b/core/java/android/accessibilityservice/GestureDescription.java index d9b03faa42faa..c9da1526a4fa6 100644 --- a/core/java/android/accessibilityservice/GestureDescription.java +++ b/core/java/android/accessibilityservice/GestureDescription.java @@ -23,10 +23,6 @@ import android.graphics.PathMeasure; import android.graphics.RectF; import android.os.Parcel; import android.os.Parcelable; -import android.view.InputDevice; -import android.view.MotionEvent; -import android.view.MotionEvent.PointerCoords; -import android.view.MotionEvent.PointerProperties; import java.util.ArrayList; import java.util.List; @@ -128,9 +124,14 @@ public final class GestureDescription { for (int i = 0; i < mStrokes.size(); i++) { StrokeDescription strokeDescription = mStrokes.get(i); if (strokeDescription.hasPointForTime(time)) { - touchPoints[numPointsFound].mPathIndex = i; - touchPoints[numPointsFound].mIsStartOfPath = (time == strokeDescription.mStartTime); - touchPoints[numPointsFound].mIsEndOfPath = (time == strokeDescription.mEndTime); + touchPoints[numPointsFound].mStrokeId = strokeDescription.getId(); + touchPoints[numPointsFound].mContinuedStrokeId = + strokeDescription.getContinuedStrokeId(); + touchPoints[numPointsFound].mIsStartOfPath = + (strokeDescription.getContinuedStrokeId() < 0) + && (time == strokeDescription.mStartTime); + touchPoints[numPointsFound].mIsEndOfPath = !strokeDescription.isContinued() + && (time == strokeDescription.mEndTime); strokeDescription.getPosForTime(time, mTempPos); touchPoints[numPointsFound].mX = Math.round(mTempPos[0]); touchPoints[numPointsFound].mY = Math.round(mTempPos[1]); @@ -196,6 +197,10 @@ public final class GestureDescription { * Immutable description of stroke that can be part of a gesture. */ public static class StrokeDescription { + public static final int INVALID_STROKE_ID = -1; + + static int sIdCounter; + Path mPath; long mStartTime; long mEndTime; @@ -203,6 +208,9 @@ public final class GestureDescription { private PathMeasure mPathMeasure; // The tap location is only set for zero-length paths float[] mTapLocation; + int mId; + boolean mContinued; + int mContinuedStrokeId; /** * @param path The path to follow. Must have exactly one contour. The bounds of the path @@ -216,6 +224,32 @@ public final class GestureDescription { public StrokeDescription(@NonNull Path path, @IntRange(from = 0) long startTime, @IntRange(from = 0) long duration) { + this(path, startTime, duration, INVALID_STROKE_ID, false); + } + + /** + * @param path The path to follow. Must have exactly one contour. The bounds of the path + * must not be negative. The path must not be empty. If the path has zero length + * (for example, a single {@code moveTo()}), the stroke is a touch that doesn't move. + * @param startTime The time, in milliseconds, from the time the gesture starts to the + * time the stroke should start. Must not be negative. + * @param duration The duration, in milliseconds, the stroke takes to traverse the path. + * Must be positive. + * @param continuedStrokeId The ID of the stroke that this stroke continues, or + * {@link #INVALID_STROKE_ID} if it continues no stroke. The stroke it + * continues must have its isContinued flag set to {@code true} and must be in the + * gesture dispatched immediately before the one containing this stroke. + * @param isContinued {@code true} if this stroke will be continued by one in the + * next gesture {@code false} otherwise. Continued strokes keep their pointers down when + * the gesture completes. + */ + public StrokeDescription(@NonNull Path path, + @IntRange(from = 0) long startTime, + @IntRange(from = 0) long duration, + @IntRange(from = 0) int continuedStrokeId, + boolean isContinued) { + mContinued = isContinued; + mContinuedStrokeId = continuedStrokeId; if (duration <= 0) { throw new IllegalArgumentException("Duration must be positive"); } @@ -252,6 +286,7 @@ public final class GestureDescription { mStartTime = startTime; mEndTime = startTime + duration; mTimeToLengthConversion = getLength() / duration; + mId = sIdCounter++; } /** @@ -281,6 +316,34 @@ public final class GestureDescription { return mEndTime - mStartTime; } + /** + * Get the stroke's ID. The ID is used when a stroke is to be continued by another + * stroke in a future gesture. + * + * @return the ID of this stroke + */ + public int getId() { + return mId; + } + + /** + * Check if this stroke is marked to continue in the next gesture. + * + * @return {@code true} if the stroke is to be continued. + */ + public boolean isContinued() { + return mContinued; + } + + /** + * Get the ID of the stroke that this one will continue. + * + * @return The ID of the stroke that this stroke continues, or 0 if no such stroke exists. + */ + public int getContinuedStrokeId() { + return mContinuedStrokeId; + } + float getLength() { return mPathMeasure.getLength(); } @@ -314,11 +377,12 @@ public final class GestureDescription { private static final int FLAG_IS_START_OF_PATH = 0x01; private static final int FLAG_IS_END_OF_PATH = 0x02; - int mPathIndex; - boolean mIsStartOfPath; - boolean mIsEndOfPath; - float mX; - float mY; + public int mStrokeId; + public int mContinuedStrokeId; + public boolean mIsStartOfPath; + public boolean mIsEndOfPath; + public float mX; + public float mY; public TouchPoint() { } @@ -328,7 +392,8 @@ public final class GestureDescription { } public TouchPoint(Parcel parcel) { - mPathIndex = parcel.readInt(); + mStrokeId = parcel.readInt(); + mContinuedStrokeId = parcel.readInt(); int startEnd = parcel.readInt(); mIsStartOfPath = (startEnd & FLAG_IS_START_OF_PATH) != 0; mIsEndOfPath = (startEnd & FLAG_IS_END_OF_PATH) != 0; @@ -336,8 +401,9 @@ public final class GestureDescription { mY = parcel.readFloat(); } - void copyFrom(TouchPoint other) { - mPathIndex = other.mPathIndex; + public void copyFrom(TouchPoint other) { + mStrokeId = other.mStrokeId; + mContinuedStrokeId = other.mContinuedStrokeId; mIsStartOfPath = other.mIsStartOfPath; mIsEndOfPath = other.mIsEndOfPath; mX = other.mX; @@ -351,7 +417,8 @@ public final class GestureDescription { @Override public void writeToParcel(Parcel dest, int flags) { - dest.writeInt(mPathIndex); + dest.writeInt(mStrokeId); + dest.writeInt(mContinuedStrokeId); int startEnd = mIsStartOfPath ? FLAG_IS_START_OF_PATH : 0; startEnd |= mIsEndOfPath ? FLAG_IS_END_OF_PATH : 0; dest.writeInt(startEnd); @@ -426,30 +493,15 @@ public final class GestureDescription { } /** - * Class to convert a GestureDescription to a series of MotionEvents. + * Class to convert a GestureDescription to a series of GestureSteps. * * @hide */ public static class MotionEventGenerator { - /** - * Constants used to initialize all MotionEvents - */ - private static final int EVENT_META_STATE = 0; - private static final int EVENT_BUTTON_STATE = 0; - private static final int EVENT_DEVICE_ID = 0; - private static final int EVENT_EDGE_FLAGS = 0; - private static final int EVENT_SOURCE = InputDevice.SOURCE_TOUCHSCREEN; - private static final int EVENT_FLAGS = 0; - private static final float EVENT_X_PRECISION = 1; - private static final float EVENT_Y_PRECISION = 1; - /* Lazily-created scratch memory for processing touches */ private static TouchPoint[] sCurrentTouchPoints; - private static TouchPoint[] sLastTouchPoints; - private static PointerCoords[] sPointerCoords; - private static PointerProperties[] sPointerProps; - static List getGestureStepsFromGestureDescription( + public static List getGestureStepsFromGestureDescription( GestureDescription description, int sampleTimeMs) { final List gestureSteps = new ArrayList<>(); @@ -474,31 +526,6 @@ public final class GestureDescription { return gestureSteps; } - public static List getMotionEventsFromGestureSteps(List steps) { - final List motionEvents = new ArrayList<>(); - - // Number of points in last touch event - int lastTouchPointSize = 0; - TouchPoint[] lastTouchPoints; - - for (int i = 0; i < steps.size(); i++) { - GestureStep step = steps.get(i); - int currentTouchPointSize = step.numTouchPoints; - lastTouchPoints = getLastTouchPoints( - Math.max(lastTouchPointSize, currentTouchPointSize)); - - appendMoveEventIfNeeded(motionEvents, lastTouchPoints, lastTouchPointSize, - step.touchPoints, currentTouchPointSize, step.timeSinceGestureStart); - lastTouchPointSize = appendUpEvents(motionEvents, lastTouchPoints, - lastTouchPointSize, step.touchPoints, currentTouchPointSize, - step.timeSinceGestureStart); - lastTouchPointSize = appendDownEvents(motionEvents, lastTouchPoints, - lastTouchPointSize, step.touchPoints, currentTouchPointSize, - step.timeSinceGestureStart); - } - return motionEvents; - } - private static TouchPoint[] getCurrentTouchPoints(int requiredCapacity) { if ((sCurrentTouchPoints == null) || (sCurrentTouchPoints.length < requiredCapacity)) { sCurrentTouchPoints = new TouchPoint[requiredCapacity]; @@ -508,133 +535,5 @@ public final class GestureDescription { } return sCurrentTouchPoints; } - - private static TouchPoint[] getLastTouchPoints(int requiredCapacity) { - if ((sLastTouchPoints == null) || (sLastTouchPoints.length < requiredCapacity)) { - sLastTouchPoints = new TouchPoint[requiredCapacity]; - for (int i = 0; i < requiredCapacity; i++) { - sLastTouchPoints[i] = new TouchPoint(); - } - } - return sLastTouchPoints; - } - - private static PointerCoords[] getPointerCoords(int requiredCapacity) { - if ((sPointerCoords == null) || (sPointerCoords.length < requiredCapacity)) { - sPointerCoords = new PointerCoords[requiredCapacity]; - for (int i = 0; i < requiredCapacity; i++) { - sPointerCoords[i] = new PointerCoords(); - } - } - return sPointerCoords; - } - - private static PointerProperties[] getPointerProps(int requiredCapacity) { - if ((sPointerProps == null) || (sPointerProps.length < requiredCapacity)) { - sPointerProps = new PointerProperties[requiredCapacity]; - for (int i = 0; i < requiredCapacity; i++) { - sPointerProps[i] = new PointerProperties(); - } - } - return sPointerProps; - } - - private static void appendMoveEventIfNeeded(List motionEvents, - TouchPoint[] lastTouchPoints, int lastTouchPointsSize, - TouchPoint[] currentTouchPoints, int currentTouchPointsSize, long currentTime) { - /* Look for pointers that have moved */ - boolean moveFound = false; - for (int i = 0; i < currentTouchPointsSize; i++) { - int lastPointsIndex = findPointByPathIndex(lastTouchPoints, lastTouchPointsSize, - currentTouchPoints[i].mPathIndex); - if (lastPointsIndex >= 0) { - moveFound |= (lastTouchPoints[lastPointsIndex].mX != currentTouchPoints[i].mX) - || (lastTouchPoints[lastPointsIndex].mY != currentTouchPoints[i].mY); - lastTouchPoints[lastPointsIndex].copyFrom(currentTouchPoints[i]); - } - } - - if (moveFound) { - long downTime = motionEvents.get(motionEvents.size() - 1).getDownTime(); - motionEvents.add(obtainMotionEvent(downTime, currentTime, MotionEvent.ACTION_MOVE, - lastTouchPoints, lastTouchPointsSize)); - } - } - - private static int appendUpEvents(List motionEvents, - TouchPoint[] lastTouchPoints, int lastTouchPointsSize, - TouchPoint[] currentTouchPoints, int currentTouchPointsSize, long currentTime) { - /* Look for a pointer at the end of its path */ - for (int i = 0; i < currentTouchPointsSize; i++) { - if (currentTouchPoints[i].mIsEndOfPath) { - int indexOfUpEvent = findPointByPathIndex(lastTouchPoints, lastTouchPointsSize, - currentTouchPoints[i].mPathIndex); - if (indexOfUpEvent < 0) { - continue; // Should not happen - } - long downTime = motionEvents.get(motionEvents.size() - 1).getDownTime(); - int action = (lastTouchPointsSize == 1) ? MotionEvent.ACTION_UP - : MotionEvent.ACTION_POINTER_UP; - action |= indexOfUpEvent << MotionEvent.ACTION_POINTER_INDEX_SHIFT; - motionEvents.add(obtainMotionEvent(downTime, currentTime, action, - lastTouchPoints, lastTouchPointsSize)); - /* Remove this point from lastTouchPoints */ - for (int j = indexOfUpEvent; j < lastTouchPointsSize - 1; j++) { - lastTouchPoints[j].copyFrom(lastTouchPoints[j+1]); - } - lastTouchPointsSize--; - } - } - return lastTouchPointsSize; - } - - private static int appendDownEvents(List motionEvents, - TouchPoint[] lastTouchPoints, int lastTouchPointsSize, - TouchPoint[] currentTouchPoints, int currentTouchPointsSize, long currentTime) { - /* Look for a pointer that is just starting */ - for (int i = 0; i < currentTouchPointsSize; i++) { - if (currentTouchPoints[i].mIsStartOfPath) { - /* Add the point to last coords and use the new array to generate the event */ - lastTouchPoints[lastTouchPointsSize++].copyFrom(currentTouchPoints[i]); - int action = (lastTouchPointsSize == 1) ? MotionEvent.ACTION_DOWN - : MotionEvent.ACTION_POINTER_DOWN; - long downTime = (action == MotionEvent.ACTION_DOWN) ? currentTime : - motionEvents.get(motionEvents.size() - 1).getDownTime(); - action |= i << MotionEvent.ACTION_POINTER_INDEX_SHIFT; - motionEvents.add(obtainMotionEvent(downTime, currentTime, action, - lastTouchPoints, lastTouchPointsSize)); - } - } - return lastTouchPointsSize; - } - - private static MotionEvent obtainMotionEvent(long downTime, long eventTime, int action, - TouchPoint[] touchPoints, int touchPointsSize) { - PointerCoords[] pointerCoords = getPointerCoords(touchPointsSize); - PointerProperties[] pointerProperties = getPointerProps(touchPointsSize); - for (int i = 0; i < touchPointsSize; i++) { - pointerProperties[i].id = touchPoints[i].mPathIndex; - pointerProperties[i].toolType = MotionEvent.TOOL_TYPE_UNKNOWN; - pointerCoords[i].clear(); - pointerCoords[i].pressure = 1.0f; - pointerCoords[i].size = 1.0f; - pointerCoords[i].x = touchPoints[i].mX; - pointerCoords[i].y = touchPoints[i].mY; - } - return MotionEvent.obtain(downTime, eventTime, action, touchPointsSize, - pointerProperties, pointerCoords, EVENT_META_STATE, EVENT_BUTTON_STATE, - EVENT_X_PRECISION, EVENT_Y_PRECISION, EVENT_DEVICE_ID, EVENT_EDGE_FLAGS, - EVENT_SOURCE, EVENT_FLAGS); - } - - private static int findPointByPathIndex(TouchPoint[] touchPoints, int touchPointsSize, - int pathIndex) { - for (int i = 0; i < touchPointsSize; i++) { - if (touchPoints[i].mPathIndex == pathIndex) { - return i; - } - } - return -1; - } } } diff --git a/services/accessibility/java/com/android/server/accessibility/AccessibilityManagerService.java b/services/accessibility/java/com/android/server/accessibility/AccessibilityManagerService.java index ab111a0b8eb31..df71cedb0fef9 100644 --- a/services/accessibility/java/com/android/server/accessibility/AccessibilityManagerService.java +++ b/services/accessibility/java/com/android/server/accessibility/AccessibilityManagerService.java @@ -2843,15 +2843,8 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub { } if (mMotionEventInjector != null) { List steps = gestureSteps.getList(); - List events = GestureDescription.MotionEventGenerator - .getMotionEventsFromGestureSteps(steps); - // Confirm that the motion events end with an UP event. - if (events.get(events.size() - 1).getAction() == MotionEvent.ACTION_UP) { - mMotionEventInjector.injectEvents(events, mServiceInterface, sequence); - return; - } else { - Slog.e(LOG_TAG, "Gesture is not well-formed"); - } + mMotionEventInjector.injectEvents(steps, mServiceInterface, sequence); + return; } else { Slog.e(LOG_TAG, "MotionEventInjector installation timed out"); } diff --git a/services/accessibility/java/com/android/server/accessibility/MotionEventInjector.java b/services/accessibility/java/com/android/server/accessibility/MotionEventInjector.java index 8042ddb56ead6..48041adbe1edd 100644 --- a/services/accessibility/java/com/android/server/accessibility/MotionEventInjector.java +++ b/services/accessibility/java/com/android/server/accessibility/MotionEventInjector.java @@ -16,49 +16,67 @@ package com.android.server.accessibility; +import android.accessibilityservice.GestureDescription; +import android.accessibilityservice.GestureDescription.GestureStep; +import android.accessibilityservice.GestureDescription.TouchPoint; import android.accessibilityservice.IAccessibilityServiceClient; import android.os.Handler; import android.os.Looper; import android.os.Message; import android.os.RemoteException; import android.os.SystemClock; +import android.util.IntArray; import android.util.Slog; import android.util.SparseArray; +import android.util.SparseIntArray; import android.view.InputDevice; import android.view.KeyEvent; import android.view.MotionEvent; import android.view.WindowManagerPolicy; import android.view.accessibility.AccessibilityEvent; import com.android.internal.os.SomeArgs; -import com.android.server.accessibility.AccessibilityManagerService.Service; +import java.util.ArrayList; import java.util.List; /** * Injects MotionEvents to permit {@code AccessibilityService}s to touch the screen on behalf of * users. - * + *

* All methods except {@code injectEvents} must be called only from the main thread. */ public class MotionEventInjector implements EventStreamTransformation, Handler.Callback { private static final String LOG_TAG = "MotionEventInjector"; private static final int MESSAGE_SEND_MOTION_EVENT = 1; private static final int MESSAGE_INJECT_EVENTS = 2; - private static final int MAX_POINTERS = 11; // Non-binding maximum + + /** + * Constants used to initialize all MotionEvents + */ + private static final int EVENT_META_STATE = 0; + private static final int EVENT_BUTTON_STATE = 0; + private static final int EVENT_DEVICE_ID = 0; + private static final int EVENT_EDGE_FLAGS = 0; + private static final int EVENT_SOURCE = InputDevice.SOURCE_TOUCHSCREEN; + private static final int EVENT_FLAGS = 0; + private static final float EVENT_X_PRECISION = 1; + private static final float EVENT_Y_PRECISION = 1; + + private static MotionEvent.PointerCoords[] sPointerCoords; + private static MotionEvent.PointerProperties[] sPointerProps; private final Handler mHandler; private final SparseArray mOpenGesturesInProgress = new SparseArray<>(); - // These two arrays must be the same length - private MotionEvent.PointerProperties[] mPointerProperties = - new MotionEvent.PointerProperties[MAX_POINTERS]; - private MotionEvent.PointerCoords[] mPointerCoords = - new MotionEvent.PointerCoords[MAX_POINTERS]; private EventStreamTransformation mNext; private IAccessibilityServiceClient mServiceInterfaceForCurrentGesture; - private int mSequenceForCurrentGesture; - private int mSourceOfInjectedGesture = InputDevice.SOURCE_UNKNOWN; + private IntArray mSequencesInProgress = new IntArray(5); private boolean mIsDestroyed = false; + private TouchPoint[] mLastTouchPoints; + private int mNumLastTouchPoints; + private long mDownTime; + private long mLastScheduledEventTime; + private SparseIntArray mStrokeIdToPointerId = new SparseIntArray(5); /** * @param looper A looper on the main thread to use for dispatching new events @@ -75,18 +93,18 @@ public class MotionEventInjector implements EventStreamTransformation, Handler.C } /** - * Schedule a series of events for injection. These events must comprise a complete, valid - * sequence. All gestures currently in progress will be cancelled, and all {@code downTime} - * and {@code eventTime} fields will be offset by the current time. + * Schedule a gesture for injection. The gesture is defined by a set of {@code GestureStep}s, + * from which {@code MotionEvent}s will be derived. All gestures currently in progress will be + * cancelled. * - * @param events The events to inject. Must all be from the same source. + * @param gestureSteps The gesture steps to inject. * @param serviceInterface The interface to call back with a result when the gesture is * either complete or cancelled. */ - public void injectEvents(List events, + public void injectEvents(List gestureSteps, IAccessibilityServiceClient serviceInterface, int sequence) { SomeArgs args = SomeArgs.obtain(); - args.arg1 = events; + args.arg1 = gestureSteps; args.arg2 = serviceInterface; args.argi1 = sequence; mHandler.sendMessage(mHandler.obtainMessage(MESSAGE_INJECT_EVENTS, args)); @@ -138,7 +156,7 @@ public class MotionEventInjector implements EventStreamTransformation, Handler.C public boolean handleMessage(Message message) { if (message.what == MESSAGE_INJECT_EVENTS) { SomeArgs args = (SomeArgs) message.obj; - injectEventsMainThread((List) args.arg1, + injectEventsMainThread((List) args.arg1, (IAccessibilityServiceClient) args.arg2, args.argi1); args.recycle(); return true; @@ -148,16 +166,16 @@ public class MotionEventInjector implements EventStreamTransformation, Handler.C return false; } MotionEvent motionEvent = (MotionEvent) message.obj; - sendMotionEventToNext(motionEvent, motionEvent, - WindowManagerPolicy.FLAG_PASS_TO_USER); - // If the message queue is now empty, then this gesture is complete - if (!mHandler.hasMessages(MESSAGE_SEND_MOTION_EVENT)) { - notifyService(true); + sendMotionEventToNext(motionEvent, motionEvent, WindowManagerPolicy.FLAG_PASS_TO_USER); + boolean isEndOfSequence = message.arg1 != 0; + if (isEndOfSequence) { + notifyService(mServiceInterfaceForCurrentGesture, mSequencesInProgress.get(0), true); + mSequencesInProgress.remove(0); } return true; } - private void injectEventsMainThread(List events, + private void injectEventsMainThread(List gestureSteps, IAccessibilityServiceClient serviceInterface, int sequence) { if (mIsDestroyed) { try { @@ -168,48 +186,110 @@ public class MotionEventInjector implements EventStreamTransformation, Handler.C } return; } - cancelAnyPendingInjectedEvents(); - mSourceOfInjectedGesture = events.get(0).getSource(); - cancelAnyGestureInProgress(mSourceOfInjectedGesture); - mServiceInterfaceForCurrentGesture = serviceInterface; - mSequenceForCurrentGesture = sequence; + if (mNext == null) { - notifyService(false); + notifyService(serviceInterface, sequence, false); return; } - long startTime = SystemClock.uptimeMillis(); + boolean continuingGesture = newGestureTriesToContinueOldOne(gestureSteps); + + if (continuingGesture) { + if ((serviceInterface != mServiceInterfaceForCurrentGesture) + || !prepareToContinueOldGesture(gestureSteps)) { + cancelAnyPendingInjectedEvents(); + notifyService(serviceInterface, sequence, false); + return; + } + } + if (!continuingGesture) { + cancelAnyPendingInjectedEvents(); + // Injected gestures have been canceled, but real gestures still need cancelling + cancelAnyGestureInProgress(EVENT_SOURCE); + } + mServiceInterfaceForCurrentGesture = serviceInterface; + + long currentTime = SystemClock.uptimeMillis(); + List events = getMotionEventsFromGestureSteps(gestureSteps, + (mSequencesInProgress.size() == 0) ? currentTime : mLastScheduledEventTime); + if (events.isEmpty()) { + notifyService(serviceInterface, sequence, false); + return; + } + mSequencesInProgress.add(sequence); + for (int i = 0; i < events.size(); i++) { MotionEvent event = events.get(i); - int numPointers = event.getPointerCount(); - if (numPointers > mPointerCoords.length) { - mPointerCoords = new MotionEvent.PointerCoords[numPointers]; - mPointerProperties = new MotionEvent.PointerProperties[numPointers]; - } - for (int j = 0; j < numPointers; j++) { - if (mPointerCoords[j] == null) { - mPointerCoords[j] = new MotionEvent.PointerCoords(); - mPointerProperties[j] = new MotionEvent.PointerProperties(); - } - event.getPointerCoords(j, mPointerCoords[j]); - event.getPointerProperties(j, mPointerProperties[j]); - } - - /* - * MotionEvent doesn't have a setEventTime() method (it carries around history data, - * which could become inconsistent), so we need to obtain a new one. - */ - MotionEvent offsetEvent = MotionEvent.obtain(startTime + event.getDownTime(), - startTime + event.getEventTime(), event.getAction(), numPointers, - mPointerProperties, mPointerCoords, event.getMetaState(), - event.getButtonState(), event.getXPrecision(), event.getYPrecision(), - event.getDeviceId(), event.getEdgeFlags(), event.getSource(), - event.getFlags()); - Message message = mHandler.obtainMessage(MESSAGE_SEND_MOTION_EVENT, offsetEvent); - mHandler.sendMessageDelayed(message, event.getEventTime()); + int isEndOfSequence = (i == events.size() - 1) ? 1 : 0; + Message message = mHandler.obtainMessage( + MESSAGE_SEND_MOTION_EVENT, isEndOfSequence, 0, event); + mLastScheduledEventTime = event.getEventTime(); + mHandler.sendMessageDelayed(message, Math.max(0, event.getEventTime() - currentTime)); } } + private boolean newGestureTriesToContinueOldOne(List gestureSteps) { + if (gestureSteps.isEmpty()) { + return false; + } + GestureStep firstStep = gestureSteps.get(0); + for (int i = 0; i < firstStep.numTouchPoints; i++) { + if (!firstStep.touchPoints[i].mIsStartOfPath) { + return true; + } + } + return false; + } + + /** + * A gesture can only continue a gesture if it contains intermediate points that continue + * each continued stroke of the last gesture, and no extra points. + * + * @param gestureSteps The steps of the new gesture + * @return {@code true} if the new gesture could continue the last one dispatched. {@code false} + * otherwise. + */ + private boolean prepareToContinueOldGesture(List gestureSteps) { + if (gestureSteps.isEmpty() || (mLastTouchPoints == null) || (mNumLastTouchPoints == 0)) { + return false; + } + GestureStep firstStep = gestureSteps.get(0); + // Make sure all of the continuing paths match up + int numContinuedStrokes = 0; + for (int i = 0; i < firstStep.numTouchPoints; i++) { + TouchPoint touchPoint = firstStep.touchPoints[i]; + if (!touchPoint.mIsStartOfPath) { + int continuedPointerId = mStrokeIdToPointerId + .get(touchPoint.mContinuedStrokeId, -1); + if (continuedPointerId == -1) { + return false; + } + mStrokeIdToPointerId.put(touchPoint.mStrokeId, continuedPointerId); + int lastPointIndex = findPointByStrokeId( + mLastTouchPoints, mNumLastTouchPoints, touchPoint.mContinuedStrokeId); + if (lastPointIndex < 0) { + return false; + } + if (mLastTouchPoints[lastPointIndex].mIsEndOfPath + || (mLastTouchPoints[lastPointIndex].mX != touchPoint.mX) + || (mLastTouchPoints[lastPointIndex].mY != touchPoint.mY)) { + return false; + } + // Update the last touch point to match the continuation, so the gestures will + // line up + mLastTouchPoints[lastPointIndex].mStrokeId = touchPoint.mStrokeId; + } + numContinuedStrokes++; + } + // Make sure we didn't miss any paths + for (int i = 0; i < mNumLastTouchPoints; i++) { + if (!mLastTouchPoints[i].mIsEndOfPath) { + numContinuedStrokes--; + } + } + return numContinuedStrokes == 0; + } + private void sendMotionEventToNext(MotionEvent event, MotionEvent rawEvent, int policyFlags) { if (mNext != null) { @@ -228,7 +308,7 @@ public class MotionEventInjector implements EventStreamTransformation, Handler.C if ((mNext != null) && mOpenGesturesInProgress.get(source, false)) { long now = SystemClock.uptimeMillis(); MotionEvent cancelEvent = - MotionEvent.obtain(now, now, MotionEvent.ACTION_CANCEL, 0.0f, 0.0f, 0); + obtainMotionEvent(now, now, MotionEvent.ACTION_CANCEL, getLastTouchPoints(), 1); sendMotionEventToNext(cancelEvent, cancelEvent, WindowManagerPolicy.FLAG_PASS_TO_USER); mOpenGesturesInProgress.put(source, false); @@ -237,19 +317,187 @@ public class MotionEventInjector implements EventStreamTransformation, Handler.C private void cancelAnyPendingInjectedEvents() { if (mHandler.hasMessages(MESSAGE_SEND_MOTION_EVENT)) { - cancelAnyGestureInProgress(mSourceOfInjectedGesture); mHandler.removeMessages(MESSAGE_SEND_MOTION_EVENT); - notifyService(false); + cancelAnyGestureInProgress(EVENT_SOURCE); + for (int i = mSequencesInProgress.size() - 1; i >= 0; i--) { + notifyService(mServiceInterfaceForCurrentGesture, + mSequencesInProgress.get(i), false); + mSequencesInProgress.remove(i); + } + } else if (mNumLastTouchPoints != 0) { + // An injected gesture is in progress and waiting for a continuation. Cancel it. + cancelAnyGestureInProgress(EVENT_SOURCE); } + mNumLastTouchPoints = 0; + mStrokeIdToPointerId.clear(); } - private void notifyService(boolean success) { + private void notifyService(IAccessibilityServiceClient service, int sequence, boolean success) { try { - mServiceInterfaceForCurrentGesture.onPerformGestureResult( - mSequenceForCurrentGesture, success); + service.onPerformGestureResult(sequence, success); } catch (RemoteException re) { Slog.e(LOG_TAG, "Error sending motion event injection status to " + mServiceInterfaceForCurrentGesture, re); } } + + private List getMotionEventsFromGestureSteps( + List steps, long startTime) { + final List motionEvents = new ArrayList<>(); + + TouchPoint[] lastTouchPoints = getLastTouchPoints(); + + for (int i = 0; i < steps.size(); i++) { + GestureDescription.GestureStep step = steps.get(i); + int currentTouchPointSize = step.numTouchPoints; + if (currentTouchPointSize > lastTouchPoints.length) { + mNumLastTouchPoints = 0; + motionEvents.clear(); + return motionEvents; + } + + appendMoveEventIfNeeded(motionEvents, step.touchPoints, currentTouchPointSize, + startTime + step.timeSinceGestureStart); + appendUpEvents(motionEvents, step.touchPoints, currentTouchPointSize, + startTime + step.timeSinceGestureStart); + appendDownEvents(motionEvents, step.touchPoints, currentTouchPointSize, + startTime + step.timeSinceGestureStart); + } + return motionEvents; + } + + private TouchPoint[] getLastTouchPoints() { + if (mLastTouchPoints == null) { + int capacity = GestureDescription.getMaxStrokeCount(); + mLastTouchPoints = new TouchPoint[capacity]; + for (int i = 0; i < capacity; i++) { + mLastTouchPoints[i] = new GestureDescription.TouchPoint(); + } + } + return mLastTouchPoints; + } + + private void appendMoveEventIfNeeded(List motionEvents, + TouchPoint[] currentTouchPoints, int currentTouchPointsSize, long currentTime) { + /* Look for pointers that have moved */ + boolean moveFound = false; + TouchPoint[] lastTouchPoints = getLastTouchPoints(); + for (int i = 0; i < currentTouchPointsSize; i++) { + int lastPointsIndex = findPointByStrokeId(lastTouchPoints, mNumLastTouchPoints, + currentTouchPoints[i].mStrokeId); + if (lastPointsIndex >= 0) { + moveFound |= (lastTouchPoints[lastPointsIndex].mX != currentTouchPoints[i].mX) + || (lastTouchPoints[lastPointsIndex].mY != currentTouchPoints[i].mY); + lastTouchPoints[lastPointsIndex].copyFrom(currentTouchPoints[i]); + } + } + + if (moveFound) { + motionEvents.add(obtainMotionEvent(mDownTime, currentTime, MotionEvent.ACTION_MOVE, + lastTouchPoints, mNumLastTouchPoints)); + } + } + + private void appendUpEvents(List motionEvents, + TouchPoint[] currentTouchPoints, int currentTouchPointsSize, long currentTime) { + /* Look for a pointer at the end of its path */ + TouchPoint[] lastTouchPoints = getLastTouchPoints(); + for (int i = 0; i < currentTouchPointsSize; i++) { + if (currentTouchPoints[i].mIsEndOfPath) { + int indexOfUpEvent = findPointByStrokeId(lastTouchPoints, mNumLastTouchPoints, + currentTouchPoints[i].mStrokeId); + if (indexOfUpEvent < 0) { + continue; // Should not happen + } + int action = (mNumLastTouchPoints == 1) ? MotionEvent.ACTION_UP + : MotionEvent.ACTION_POINTER_UP; + action |= indexOfUpEvent << MotionEvent.ACTION_POINTER_INDEX_SHIFT; + motionEvents.add(obtainMotionEvent(mDownTime, currentTime, action, + lastTouchPoints, mNumLastTouchPoints)); + /* Remove this point from lastTouchPoints */ + for (int j = indexOfUpEvent; j < mNumLastTouchPoints - 1; j++) { + lastTouchPoints[j].copyFrom(mLastTouchPoints[j + 1]); + } + mNumLastTouchPoints--; + if (mNumLastTouchPoints == 0) { + mStrokeIdToPointerId.clear(); + } + } + } + } + + private void appendDownEvents(List motionEvents, + TouchPoint[] currentTouchPoints, int currentTouchPointsSize, long currentTime) { + /* Look for a pointer that is just starting */ + TouchPoint[] lastTouchPoints = getLastTouchPoints(); + for (int i = 0; i < currentTouchPointsSize; i++) { + if (currentTouchPoints[i].mIsStartOfPath) { + /* Add the point to last coords and use the new array to generate the event */ + lastTouchPoints[mNumLastTouchPoints++].copyFrom(currentTouchPoints[i]); + int action = (mNumLastTouchPoints == 1) ? MotionEvent.ACTION_DOWN + : MotionEvent.ACTION_POINTER_DOWN; + if (action == MotionEvent.ACTION_DOWN) { + mDownTime = currentTime; + } + action |= i << MotionEvent.ACTION_POINTER_INDEX_SHIFT; + motionEvents.add(obtainMotionEvent(mDownTime, currentTime, action, + lastTouchPoints, mNumLastTouchPoints)); + } + } + } + + private MotionEvent obtainMotionEvent(long downTime, long eventTime, int action, + TouchPoint[] touchPoints, int touchPointsSize) { + if ((sPointerCoords == null) || (sPointerCoords.length < touchPointsSize)) { + sPointerCoords = new MotionEvent.PointerCoords[touchPointsSize]; + for (int i = 0; i < touchPointsSize; i++) { + sPointerCoords[i] = new MotionEvent.PointerCoords(); + } + } + if ((sPointerProps == null) || (sPointerProps.length < touchPointsSize)) { + sPointerProps = new MotionEvent.PointerProperties[touchPointsSize]; + for (int i = 0; i < touchPointsSize; i++) { + sPointerProps[i] = new MotionEvent.PointerProperties(); + } + } + for (int i = 0; i < touchPointsSize; i++) { + int pointerId = mStrokeIdToPointerId.get(touchPoints[i].mStrokeId, -1); + if (pointerId == -1) { + pointerId = getUnusedPointerId(); + mStrokeIdToPointerId.put(touchPoints[i].mStrokeId, pointerId); + } + sPointerProps[i].id = pointerId; + sPointerProps[i].toolType = MotionEvent.TOOL_TYPE_UNKNOWN; + sPointerCoords[i].clear(); + sPointerCoords[i].pressure = 1.0f; + sPointerCoords[i].size = 1.0f; + sPointerCoords[i].x = touchPoints[i].mX; + sPointerCoords[i].y = touchPoints[i].mY; + } + return MotionEvent.obtain(downTime, eventTime, action, touchPointsSize, + sPointerProps, sPointerCoords, EVENT_META_STATE, EVENT_BUTTON_STATE, + EVENT_X_PRECISION, EVENT_Y_PRECISION, EVENT_DEVICE_ID, EVENT_EDGE_FLAGS, + EVENT_SOURCE, EVENT_FLAGS); + } + + private static int findPointByStrokeId(TouchPoint[] touchPoints, int touchPointsSize, + int strokeId) { + for (int i = 0; i < touchPointsSize; i++) { + if (touchPoints[i].mStrokeId == strokeId) { + return i; + } + } + return -1; + } + private int getUnusedPointerId() { + int MAX_POINTER_ID = 10; + int pointerId = 0; + while (mStrokeIdToPointerId.indexOfValue(pointerId) >= 0) { + pointerId++; + if (pointerId >= MAX_POINTER_ID) { + return MAX_POINTER_ID; + } + } + return pointerId; + } } diff --git a/services/tests/servicestests/src/com/android/server/accessibility/GestureDescriptionTest.java b/services/tests/servicestests/src/com/android/server/accessibility/GestureDescriptionTest.java new file mode 100644 index 0000000000000..b876a5fde6435 --- /dev/null +++ b/services/tests/servicestests/src/com/android/server/accessibility/GestureDescriptionTest.java @@ -0,0 +1,420 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.accessibility; + +import static android.accessibilityservice.GestureDescription.StrokeDescription.INVALID_STROKE_ID; +import static org.hamcrest.CoreMatchers.allOf; +import static org.hamcrest.CoreMatchers.everyItem; +import static org.hamcrest.MatcherAssert.assertThat; + +import android.accessibilityservice.GestureDescription; +import android.accessibilityservice.GestureDescription.GestureStep; +import android.accessibilityservice.GestureDescription.MotionEventGenerator; +import android.accessibilityservice.GestureDescription.StrokeDescription; +import android.graphics.Path; +import android.graphics.PointF; +import org.hamcrest.Description; +import org.hamcrest.Matcher; +import org.hamcrest.TypeSafeMatcher; +import org.junit.Test; + +import java.util.List; + +import static junit.framework.TestCase.assertEquals; + +/** + * Tests for GestureDescription + */ +public class GestureDescriptionTest { + @Test + public void testGestureShorterThanSampleRate_producesStartAndEnd() { + PointF click = new PointF(10, 20); + Path clickPath = new Path(); + clickPath.moveTo(click.x, click.y); + StrokeDescription clickStroke = new StrokeDescription(clickPath, 0, 10); + GestureDescription.Builder clickBuilder = new GestureDescription.Builder(); + clickBuilder.addStroke(clickStroke); + GestureDescription clickGesture = clickBuilder.build(); + + List clickGestureSteps = MotionEventGenerator + .getGestureStepsFromGestureDescription(clickGesture, 100); + + assertEquals(2, clickGestureSteps.size()); + assertThat(clickGestureSteps.get(0), allOf(numTouchPointsIs(1), numStartsOfStroke(1), + numEndsOfStroke(0), hasPoint(click))); + assertThat(clickGestureSteps.get(1), allOf(numTouchPointsIs(1), numStartsOfStroke(0), + numEndsOfStroke(1), hasPoint(click))); + } + + @Test + public void testSwipe_shouldContainEvenlySpacedPoints() { + int samplePeriod = 10; + int numSamples = 5; + float stepX = 2; + float stepY = 3; + PointF start = new PointF(10, 20); + PointF end = new PointF(10 + numSamples * stepX, 20 + numSamples * stepY); + + GestureDescription swipe = + createSwipe(start.x, start.y, end.x, end.y, numSamples * samplePeriod); + List swipeGestureSteps = MotionEventGenerator + .getGestureStepsFromGestureDescription(swipe, samplePeriod); + assertEquals(numSamples + 1, swipeGestureSteps.size()); + + assertThat(swipeGestureSteps.get(0), allOf(numTouchPointsIs(1), numStartsOfStroke(1), + numEndsOfStroke(0), hasPoint(start))); + assertThat(swipeGestureSteps.get(numSamples), allOf(numTouchPointsIs(1), + numStartsOfStroke(0), numEndsOfStroke(1), hasPoint(end))); + + for (int i = 1; i < numSamples; ++i) { + PointF interpPoint = new PointF(start.x + stepX * i, start.y + stepY * i); + assertThat(swipeGestureSteps.get(i), allOf(numTouchPointsIs(1), + numStartsOfStroke(0), numEndsOfStroke(0), hasPoint(interpPoint))); + } + } + + @Test + public void testSwipeWithNonIntegerValues_shouldRound() { + int strokeTime = 10; + + GestureDescription swipe = createSwipe(10.1f, 20.6f, 11.9f, 22.1f, strokeTime); + List swipeGestureSteps = MotionEventGenerator + .getGestureStepsFromGestureDescription(swipe, strokeTime); + assertEquals(2, swipeGestureSteps.size()); + assertThat(swipeGestureSteps.get(0), hasPoint(new PointF(10, 21))); + assertThat(swipeGestureSteps.get(1), hasPoint(new PointF(12, 22))); + } + + @Test + public void testPathsWithOverlappingTiming_produceCorrectSteps() { + // There are 4 paths + // 0: an L-shaped path that starts first + // 1: a swipe that starts in the middle of the L-shaped path and ends when the L ends + // 2: a swipe that starts at the same time as #1 but extends past the end of the L + // 3: a swipe that starts when #3 ends + PointF path0Start = new PointF(100, 150); + PointF path0Turn = new PointF(100, 200); + PointF path0End = new PointF(250, 200); + int path0StartTime = 0; + int path0EndTime = 100; + int path0Duration = path0EndTime - path0StartTime; + Path path0 = new Path(); + path0.moveTo(path0Start.x, path0Start.y); + path0.lineTo(path0Turn.x, path0Turn.y); + path0.lineTo(path0End.x, path0End.y); + StrokeDescription path0Stroke = new StrokeDescription(path0, path0StartTime, path0Duration); + + PointF path1Start = new PointF(300, 350); + PointF path1End = new PointF(300, 400); + int path1StartTime = 50; + int path1EndTime = path0EndTime; + StrokeDescription path1Stroke = createSwipeStroke( + path1Start.x, path1Start.y, path1End.x, path1End.y, path1StartTime, path1EndTime); + + PointF path2Start = new PointF(400, 450); + PointF path2End = new PointF(400, 500); + int path2StartTime = 50; + int path2EndTime = 150; + StrokeDescription path2Stroke = createSwipeStroke( + path2Start.x, path2Start.y, path2End.x, path2End.y, path2StartTime, path2EndTime); + + PointF path3Start = new PointF(500, 550); + PointF path3End = new PointF(500, 600); + int path3StartTime = path2EndTime; + int path3EndTime = 200; + StrokeDescription path3Stroke = createSwipeStroke( + path3Start.x, path3Start.y, path3End.x, path3End.y, path3StartTime, path3EndTime); + + int deltaT = 12; // Force samples to happen on extra boundaries + GestureDescription.Builder builder = new GestureDescription.Builder(); + builder.addStroke(path0Stroke); + builder.addStroke(path1Stroke); + builder.addStroke(path2Stroke); + builder.addStroke(path3Stroke); + List steps = MotionEventGenerator + .getGestureStepsFromGestureDescription(builder.build(), deltaT); + + long start = 0; + assertThat(steps.get(0), allOf(numStartsOfStroke(1), numEndsOfStroke(0), isAtTime(start), + numTouchPointsIs(1), hasPoint(path0Start))); + assertThat(steps.get(1), allOf(numTouchPointsIs(1), noStartsOrEnds(), + isAtTime(start + deltaT))); + assertThat(steps.get(2), allOf(numTouchPointsIs(1), isAtTime(start + deltaT * 2))); + assertThat(steps.get(3), allOf(numTouchPointsIs(1), isAtTime(start + deltaT * 3))); + assertThat(steps.get(4), allOf(numTouchPointsIs(1), isAtTime(start + deltaT * 4))); + + assertThat(steps.get(5), allOf(numTouchPointsIs(3), numStartsOfStroke(2), + numEndsOfStroke(0), isAtTime(path1StartTime), hasPoint(path1Start), + hasPoint(path2Start))); + + start = path1StartTime; + assertThat(steps.get(6), allOf(numTouchPointsIs(3), isAtTime(start + deltaT * 1))); + assertThat(steps.get(7), allOf(noStartsOrEnds(), isAtTime(start + deltaT * 2))); + assertThat(steps.get(8), allOf(numTouchPointsIs(3), isAtTime(start + deltaT * 3))); + assertThat(steps.get(9), allOf(noStartsOrEnds(), isAtTime(start + deltaT * 4))); + + assertThat(steps.get(10), allOf(numTouchPointsIs(3), numStartsOfStroke(0), + numEndsOfStroke(2), isAtTime(path0EndTime), hasPoint(path0End), + hasPoint(path1End))); + + start = path0EndTime; + assertThat(steps.get(11), allOf(numTouchPointsIs(1), isAtTime(start + deltaT * 1))); + assertThat(steps.get(12), allOf(noStartsOrEnds(), isAtTime(start + deltaT * 2))); + assertThat(steps.get(13), allOf(numTouchPointsIs(1), isAtTime(start + deltaT * 3))); + assertThat(steps.get(14), allOf(noStartsOrEnds(), isAtTime(start + deltaT * 4))); + + assertThat(steps.get(15), allOf(numTouchPointsIs(2), numStartsOfStroke(1), + numEndsOfStroke(1), isAtTime(path2EndTime), hasPoint(path2End), + hasPoint(path3Start))); + + start = path2EndTime; + assertThat(steps.get(16), allOf(numTouchPointsIs(1), isAtTime(start + deltaT * 1))); + assertThat(steps.get(17), allOf(noStartsOrEnds(), isAtTime(start + deltaT * 2))); + assertThat(steps.get(18), allOf(numTouchPointsIs(1), isAtTime(start + deltaT * 3))); + assertThat(steps.get(19), allOf(noStartsOrEnds(), isAtTime(start + deltaT * 4))); + + assertThat(steps.get(20), allOf(numTouchPointsIs(1), numStartsOfStroke(0), + numEndsOfStroke(1), isAtTime(path3EndTime), hasPoint(path3End))); + } + + @Test + public void testMaxTouchpoints_shouldHaveValidCoords() { + GestureDescription.Builder maxPointBuilder = new GestureDescription.Builder(); + PointF baseStartPoint = new PointF(100, 100); + PointF baseEndPoint = new PointF(100, 200); + int xStep = 10; + int samplePeriod = 15; + int numSamples = 2; + int numPoints = GestureDescription.getMaxStrokeCount(); + for (int i = 0; i < numPoints; i++) { + Path path = new Path(); + path.moveTo(baseStartPoint.x + i * xStep, baseStartPoint.y); + path.lineTo(baseEndPoint.x + i * xStep, baseEndPoint.y); + maxPointBuilder.addStroke(new StrokeDescription(path, 0, samplePeriod * numSamples)); + } + + List steps = MotionEventGenerator + .getGestureStepsFromGestureDescription(maxPointBuilder.build(), samplePeriod); + assertEquals(3, steps.size()); + + assertThat(steps.get(0), allOf(numTouchPointsIs(numPoints), numStartsOfStroke(numPoints), + numEndsOfStroke(0), isAtTime(0))); + assertThat(steps.get(1), allOf(numTouchPointsIs(numPoints), numStartsOfStroke(0), + numEndsOfStroke(0), isAtTime(samplePeriod))); + assertThat(steps.get(2), allOf(numTouchPointsIs(numPoints), numStartsOfStroke(0), + numEndsOfStroke(numPoints), isAtTime(samplePeriod * 2))); + + PointF baseMidPoint = new PointF((baseStartPoint.x + baseEndPoint.x) / 2, + (baseStartPoint.y + baseEndPoint.y) / 2); + for (int i = 0; i < numPoints; i++) { + assertThat(steps.get(0), + hasPoint(new PointF(baseStartPoint.x + i * xStep, baseStartPoint.y))); + assertThat(steps.get(1), + hasPoint(new PointF(baseMidPoint.x + i * xStep, baseMidPoint.y))); + assertThat(steps.get(2), + hasPoint(new PointF(baseEndPoint.x + i * xStep, baseEndPoint.y))); + } + } + + @Test + public void testGetGestureSteps_touchPointsHaveStrokeId() { + StrokeDescription swipeStroke = createSwipeStroke(10, 20, 30, 40, 0, 100); + GestureDescription swipe = new GestureDescription.Builder().addStroke(swipeStroke).build(); + List swipeGestureSteps = MotionEventGenerator + .getGestureStepsFromGestureDescription(swipe, 10); + + assertThat(swipeGestureSteps, everyItem(hasStrokeId(swipeStroke.getId()))); + } + + @Test + public void testGetGestureSteps_continuedStroke_hasNoEndPoint() { + Path swipePath = new Path(); + swipePath.moveTo(10, 20); + swipePath.lineTo(30, 40); + StrokeDescription stroke1 = + new StrokeDescription(swipePath, 0, 100, 0, true); + GestureDescription gesture = new GestureDescription.Builder().addStroke(stroke1).build(); + List steps = MotionEventGenerator + .getGestureStepsFromGestureDescription(gesture, 10); + + assertThat(steps, everyItem(numEndsOfStroke(0))); + } + + @Test + public void testGetGestureSteps_continuingStroke_hasNoStartPointAndHasContinuedId() { + Path swipePath = new Path(); + swipePath.moveTo(10, 20); + swipePath.lineTo(30, 40); + StrokeDescription stroke1 = + new StrokeDescription(swipePath, 0, 100, INVALID_STROKE_ID, true); + StrokeDescription stroke2 = + new StrokeDescription(swipePath, 0, 100, stroke1.getId(), false); + GestureDescription gesture = new GestureDescription.Builder().addStroke(stroke2).build(); + List steps = MotionEventGenerator + .getGestureStepsFromGestureDescription(gesture, 10); + + assertThat(steps, everyItem( + allOf(continuesStrokeId(stroke1.getId()), numStartsOfStroke(0)))); + } + + private GestureDescription createSwipe( + float startX, float startY, float endX, float endY, long duration) { + GestureDescription.Builder swipeBuilder = new GestureDescription.Builder(); + swipeBuilder.addStroke(createSwipeStroke(startX, startY, endX, endY, 0, duration)); + return swipeBuilder.build(); + } + + private StrokeDescription createSwipeStroke( + float startX, float startY, float endX, float endY, long startTime, long endTime) { + Path swipePath = new Path(); + swipePath.moveTo(startX, startY); + swipePath.lineTo(endX, endY); + StrokeDescription swipeStroke = + new StrokeDescription(swipePath, startTime, endTime - startTime); + return swipeStroke; + } + + Matcher numTouchPointsIs(final int numTouchPoints) { + return new TypeSafeMatcher() { + @Override + protected boolean matchesSafely(GestureStep gestureStep) { + return gestureStep.numTouchPoints == numTouchPoints; + } + + @Override + public void describeTo(Description description) { + description.appendText("Has " + numTouchPoints + " touch point(s)"); + } + }; + } + + Matcher numStartsOfStroke(final int numStarts) { + return new TypeSafeMatcher() { + @Override + protected boolean matchesSafely(GestureStep gestureStep) { + int numStartsFound = 0; + for (int i = 0; i < gestureStep.numTouchPoints; i++) { + if (gestureStep.touchPoints[i].mIsStartOfPath) { + numStartsFound++; + } + } + return numStartsFound == numStarts; + } + + @Override + public void describeTo(Description description) { + description.appendText("Starts " + numStarts + " stroke(s)"); + } + }; + } + + Matcher numEndsOfStroke(final int numEnds) { + return new TypeSafeMatcher() { + @Override + protected boolean matchesSafely(GestureStep gestureStep) { + int numEndsFound = 0; + for (int i = 0; i < gestureStep.numTouchPoints; i++) { + if (gestureStep.touchPoints[i].mIsEndOfPath) { + numEndsFound++; + } + } + return numEndsFound == numEnds; + } + + @Override + public void describeTo(Description description) { + description.appendText("Ends " + numEnds + " stroke(s)"); + } + }; + } + + Matcher hasPoint(final PointF point) { + return new TypeSafeMatcher() { + @Override + protected boolean matchesSafely(GestureStep gestureStep) { + for (int i = 0; i < gestureStep.numTouchPoints; i++) { + if ((gestureStep.touchPoints[i].mX == point.x) + && (gestureStep.touchPoints[i].mY == point.y)) { + return true; + } + } + return false; + } + + @Override + public void describeTo(Description description) { + description.appendText("Has at least one point at " + point); + } + }; + } + + Matcher hasStrokeId(final int strokeId) { + return new TypeSafeMatcher() { + @Override + protected boolean matchesSafely(GestureStep gestureStep) { + for (int i = 0; i < gestureStep.numTouchPoints; i++) { + if (gestureStep.touchPoints[i].mStrokeId == strokeId) { + return true; + } + } + return false; + } + + @Override + public void describeTo(Description description) { + description.appendText("Has at least one point with stroke id " + strokeId); + } + }; + } + + Matcher continuesStrokeId(final int strokeId) { + return new TypeSafeMatcher() { + @Override + protected boolean matchesSafely(GestureStep gestureStep) { + for (int i = 0; i < gestureStep.numTouchPoints; i++) { + if (gestureStep.touchPoints[i].mContinuedStrokeId == strokeId) { + return true; + } + } + return false; + } + + @Override + public void describeTo(Description description) { + description.appendText("Continues stroke id " + strokeId); + } + }; + } + + Matcher isAtTime(final long time) { + return new TypeSafeMatcher() { + @Override + protected boolean matchesSafely(GestureStep gestureStep) { + return gestureStep.timeSinceGestureStart == time; + } + + @Override + public void describeTo(Description description) { + description.appendText("Is at time " + time); + } + }; + } + + Matcher noStartsOrEnds() { + return allOf(numStartsOfStroke(0), numEndsOfStroke(0)); + } +} diff --git a/services/tests/servicestests/src/com/android/server/accessibility/MotionEventInjectorTest.java b/services/tests/servicestests/src/com/android/server/accessibility/MotionEventInjectorTest.java index d5305d96725db..73344e037cdc8 100644 --- a/services/tests/servicestests/src/com/android/server/accessibility/MotionEventInjectorTest.java +++ b/services/tests/servicestests/src/com/android/server/accessibility/MotionEventInjectorTest.java @@ -16,9 +16,17 @@ package com.android.server.accessibility; +import static android.view.MotionEvent.ACTION_DOWN; +import static android.view.MotionEvent.ACTION_UP; +import static android.view.WindowManagerPolicy.FLAG_PASS_TO_USER; +import static org.hamcrest.CoreMatchers.allOf; +import static org.hamcrest.CoreMatchers.anyOf; +import static org.hamcrest.CoreMatchers.everyItem; +import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.anyBoolean; import static org.mockito.Matchers.anyInt; import static org.mockito.Matchers.argThat; import static org.mockito.Matchers.eq; @@ -29,7 +37,10 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.verifyZeroInteractions; +import android.accessibilityservice.GestureDescription.GestureStep; +import android.accessibilityservice.GestureDescription.TouchPoint; import android.accessibilityservice.IAccessibilityServiceClient; +import android.graphics.Point; import android.os.Handler; import android.os.Looper; import android.os.Message; @@ -40,11 +51,15 @@ import android.util.Pair; import android.view.InputDevice; import android.view.KeyEvent; import android.view.MotionEvent; -import android.view.WindowManagerPolicy; + import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import android.view.accessibility.AccessibilityEvent; +import org.hamcrest.Description; +import org.hamcrest.Matcher; +import org.hamcrest.TypeSafeMatcher; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -58,30 +73,59 @@ import org.mockito.ArgumentMatcher; @RunWith(AndroidJUnit4.class) public class MotionEventInjectorTest { private static final String LOG_TAG = "MotionEventInjectorTest"; - private static final int CLICK_X = 100; - private static final int CLICK_Y_START = 200; - private static final int CLICK_Y_END = 201; - private static final int CLICK_DURATION = 10; - private static final int SEQUENCE = 50; + private static final Matcher IS_ACTION_DOWN = + new MotionEventActionMatcher(ACTION_DOWN); + private static final Matcher IS_ACTION_POINTER_DOWN = + new MotionEventActionMatcher(MotionEvent.ACTION_POINTER_DOWN); + private static final Matcher IS_ACTION_UP = + new MotionEventActionMatcher(ACTION_UP); + private static final Matcher IS_ACTION_POINTER_UP = + new MotionEventActionMatcher(MotionEvent.ACTION_POINTER_UP); + private static final Matcher IS_ACTION_CANCEL = + new MotionEventActionMatcher(MotionEvent.ACTION_CANCEL); + private static final Matcher IS_ACTION_MOVE = + new MotionEventActionMatcher(MotionEvent.ACTION_MOVE); - private static final int SECOND_CLICK_X = 1000; - private static final int SECOND_CLICK_Y = 2000; - private static final int SECOND_SEQUENCE = 51; + private static final Point LINE_START = new Point(100, 200); + private static final Point LINE_END = new Point(100, 300); + private static final int LINE_DURATION = 100; + private static final int LINE_SEQUENCE = 50; + + private static final Point CLICK_POINT = new Point(1000, 2000); + private static final int CLICK_DURATION = 10; + private static final int CLICK_SEQUENCE = 51; private static final int MOTION_EVENT_SOURCE = InputDevice.SOURCE_TOUCHSCREEN; private static final int OTHER_EVENT_SOURCE = InputDevice.SOURCE_MOUSE; + private static final Point CONTINUED_LINE_START = new Point(500, 300); + private static final Point CONTINUED_LINE_MID1 = new Point(500, 400); + private static final Point CONTINUED_LINE_MID2 = new Point(600, 300); + private static final Point CONTINUED_LINE_END = new Point(600, 400); + private static final int CONTINUED_LINE_STROKE_ID_1 = 100; + private static final int CONTINUED_LINE_STROKE_ID_2 = 101; + private static final int CONTINUED_LINE_INTERVAL = 100; + private static final int CONTINUED_LINE_SEQUENCE_1 = 52; + private static final int CONTINUED_LINE_SEQUENCE_2 = 53; + MotionEventInjector mMotionEventInjector; IAccessibilityServiceClient mServiceInterface; - List mClickList = new ArrayList<>(); - List mSecondClickList = new ArrayList<>(); + List mLineList = new ArrayList<>(); + List mClickList = new ArrayList<>(); + List mContinuedLineList1 = new ArrayList<>(); + List mContinuedLineList2 = new ArrayList<>(); + + MotionEvent mClickDownEvent; + MotionEvent mClickUpEvent; + ArgumentCaptor mCaptor1 = ArgumentCaptor.forClass(MotionEvent.class); ArgumentCaptor mCaptor2 = ArgumentCaptor.forClass(MotionEvent.class); MessageCapturingHandler mMessageCapturingHandler; - MotionEventMatcher mClickEvent0Matcher; - MotionEventMatcher mClickEvent1Matcher; - MotionEventMatcher mClickEvent2Matcher; - MotionEventMatcher mSecondClickEvent0Matcher; + Matcher mIsLineStart; + Matcher mIsLineMiddle; + Matcher mIsLineEnd; + Matcher mIsClickDown; + Matcher mIsClickUp; @BeforeClass public static void oneTimeInitialization() { @@ -99,226 +143,191 @@ public class MotionEventInjectorTest { } }); mMotionEventInjector = new MotionEventInjector(mMessageCapturingHandler); - mClickList.add( - MotionEvent.obtain(0, 0, MotionEvent.ACTION_DOWN, CLICK_X, CLICK_Y_START, 0)); - mClickList.add(MotionEvent.obtain( - 0, CLICK_DURATION, MotionEvent.ACTION_MOVE, CLICK_X, CLICK_Y_END, 0)); - mClickList.add(MotionEvent.obtain( - 0, CLICK_DURATION, MotionEvent.ACTION_UP, CLICK_X, CLICK_Y_END, 0)); - for (int i = 0; i < mClickList.size(); i++) { - mClickList.get(i).setSource(MOTION_EVENT_SOURCE); - } - - mClickEvent0Matcher = new MotionEventMatcher(mClickList.get(0)); - mClickEvent1Matcher = new MotionEventMatcher(mClickList.get(1)); - mClickEvent2Matcher = new MotionEventMatcher(mClickList.get(2)); - - mSecondClickList.add(MotionEvent.obtain( - 0, 0, MotionEvent.ACTION_DOWN, SECOND_CLICK_X, SECOND_CLICK_Y, 0)); - mSecondClickList.add(MotionEvent.obtain( - 0, CLICK_DURATION, MotionEvent.ACTION_MOVE, SECOND_CLICK_X, CLICK_Y_END, 0)); - mSecondClickList.add(MotionEvent.obtain( - 0, CLICK_DURATION, MotionEvent.ACTION_UP, SECOND_CLICK_X, CLICK_Y_END, 0)); - for (int i = 0; i < mSecondClickList.size(); i++) { - mSecondClickList.get(i).setSource(MOTION_EVENT_SOURCE); - } - - mSecondClickEvent0Matcher = new MotionEventMatcher(mSecondClickList.get(0)); - mServiceInterface = mock(IAccessibilityServiceClient.class); + + mLineList = createSimpleGestureFromPoints(0, 0, false, LINE_DURATION, LINE_START, LINE_END); + mClickList = createSimpleGestureFromPoints( + 0, 0, false, CLICK_DURATION, CLICK_POINT, CLICK_POINT); + mContinuedLineList1 = createSimpleGestureFromPoints(CONTINUED_LINE_STROKE_ID_1, 0, true, + CONTINUED_LINE_INTERVAL, CONTINUED_LINE_START, CONTINUED_LINE_MID1); + mContinuedLineList2 = createSimpleGestureFromPoints(CONTINUED_LINE_STROKE_ID_2, + CONTINUED_LINE_STROKE_ID_1, false, CONTINUED_LINE_INTERVAL, CONTINUED_LINE_MID1, + CONTINUED_LINE_MID2, CONTINUED_LINE_END); + + mClickDownEvent = MotionEvent.obtain(0, 0, ACTION_DOWN, CLICK_POINT.x, CLICK_POINT.y, 0); + mClickDownEvent.setSource(InputDevice.SOURCE_TOUCHSCREEN); + mClickUpEvent = MotionEvent.obtain(0, CLICK_DURATION, ACTION_UP, CLICK_POINT.x, + CLICK_POINT.y, 0); + mClickUpEvent.setSource(InputDevice.SOURCE_TOUCHSCREEN); + + mIsLineStart = allOf(IS_ACTION_DOWN, isAtPoint(LINE_START), hasStandardInitialization(), + hasTimeFromDown(0)); + mIsLineMiddle = allOf(IS_ACTION_MOVE, isAtPoint(LINE_END), hasStandardInitialization(), + hasTimeFromDown(LINE_DURATION)); + mIsLineEnd = allOf(IS_ACTION_UP, isAtPoint(LINE_END), hasStandardInitialization(), + hasTimeFromDown(LINE_DURATION)); + mIsClickDown = allOf(IS_ACTION_DOWN, isAtPoint(CLICK_POINT), hasStandardInitialization(), + hasTimeFromDown(0)); + mIsClickUp = allOf(IS_ACTION_UP, isAtPoint(CLICK_POINT), hasStandardInitialization(), + hasTimeFromDown(CLICK_DURATION)); } @Test public void testInjectEvents_shouldEmergeInOrderWithCorrectTiming() throws RemoteException { EventStreamTransformation next = attachMockNext(mMotionEventInjector); - mMotionEventInjector.injectEvents(mClickList, mServiceInterface, SEQUENCE); - mMessageCapturingHandler.sendOneMessage(); // Process the event injection + injectEventsSync(mLineList, mServiceInterface, LINE_SEQUENCE); verifyNoMoreInteractions(next); mMessageCapturingHandler.sendOneMessage(); // Send a motion event - verify(next).onMotionEvent(mCaptor1.capture(), mCaptor2.capture(), - eq(WindowManagerPolicy.FLAG_PASS_TO_USER)); - long gestureStart = mCaptor1.getValue().getDownTime(); - mClickEvent0Matcher.offsetTimesBy(gestureStart); - mClickEvent1Matcher.offsetTimesBy(gestureStart); - mClickEvent2Matcher.offsetTimesBy(gestureStart); - - verify(next).onMotionEvent(argThat(mClickEvent0Matcher), argThat(mClickEvent0Matcher), - eq(WindowManagerPolicy.FLAG_PASS_TO_USER)); + verify(next).onMotionEvent(mCaptor1.capture(), mCaptor2.capture(), eq(FLAG_PASS_TO_USER)); + verify(next).onMotionEvent(argThat(mIsLineStart), argThat(mIsLineStart), + eq(FLAG_PASS_TO_USER)); verifyNoMoreInteractions(next); reset(next); + Matcher hasRightDownTime = hasDownTime(mCaptor1.getValue().getDownTime()); + mMessageCapturingHandler.sendOneMessage(); // Send a motion event - verify(next).onMotionEvent(argThat(mClickEvent1Matcher), argThat(mClickEvent1Matcher), - eq(WindowManagerPolicy.FLAG_PASS_TO_USER)); + verify(next).onMotionEvent(argThat(allOf(mIsLineMiddle, hasRightDownTime)), + argThat(allOf(mIsLineMiddle, hasRightDownTime)), eq(FLAG_PASS_TO_USER)); verifyNoMoreInteractions(next); reset(next); verifyZeroInteractions(mServiceInterface); mMessageCapturingHandler.sendOneMessage(); // Send a motion event - verify(next).onMotionEvent(argThat(mClickEvent2Matcher), argThat(mClickEvent2Matcher), - eq(WindowManagerPolicy.FLAG_PASS_TO_USER)); + verify(next).onMotionEvent(argThat(allOf(mIsLineEnd, hasRightDownTime)), + argThat(allOf(mIsLineEnd, hasRightDownTime)), eq(FLAG_PASS_TO_USER)); verifyNoMoreInteractions(next); - reset(next); - verify(mServiceInterface).onPerformGestureResult(SEQUENCE, true); + verify(mServiceInterface).onPerformGestureResult(LINE_SEQUENCE, true); verifyNoMoreInteractions(mServiceInterface); } @Test - public void testInjectEvents_eventWithManyPointers_shouldNotCrash() { - int manyPointersCount = 20; - MotionEvent.PointerCoords[] pointerCoords = - new MotionEvent.PointerCoords[manyPointersCount]; - MotionEvent.PointerProperties[] pointerProperties = - new MotionEvent.PointerProperties[manyPointersCount]; - for (int i = 0; i < manyPointersCount; i++) { - pointerProperties[i] = new MotionEvent.PointerProperties(); - pointerProperties[i].id = i; - pointerProperties[i].toolType = MotionEvent.TOOL_TYPE_UNKNOWN; - pointerCoords[i] = new MotionEvent.PointerCoords(); - pointerCoords[i].clear(); - pointerCoords[i].pressure = 1.0f; - pointerCoords[i].size = 1.0f; - pointerCoords[i].x = i; - pointerCoords[i].y = i; + public void testInjectEvents_gestureWithTooManyPoints_shouldNotCrash() throws Exception { + int tooManyPointsCount = 20; + TouchPoint[] startTouchPoints = new TouchPoint[tooManyPointsCount]; + TouchPoint[] endTouchPoints = new TouchPoint[tooManyPointsCount]; + for (int i = 0; i < tooManyPointsCount; i++) { + startTouchPoints[i] = new TouchPoint(); + startTouchPoints[i].mIsStartOfPath = true; + startTouchPoints[i].mX = i; + startTouchPoints[i].mY = i; + endTouchPoints[i] = new TouchPoint(); + endTouchPoints[i].mIsEndOfPath = true; + endTouchPoints[i].mX = i; + endTouchPoints[i].mY = i; } - List events = new ArrayList<>(); - events.add(MotionEvent.obtain(0, 0, MotionEvent.ACTION_DOWN, manyPointersCount, - pointerProperties, pointerCoords, 0, 0, - 1.0f, 1.0f, 0, 0, InputDevice.SOURCE_TOUCHSCREEN, 0)); - events.add(MotionEvent.obtain(0, 0, MotionEvent.ACTION_UP, manyPointersCount, - pointerProperties, pointerCoords, 0, 0, - 1.0f, 1.0f, 0, 0, InputDevice.SOURCE_TOUCHSCREEN, 0)); - EventStreamTransformation next = attachMockNext(mMotionEventInjector); - mMotionEventInjector.injectEvents(events, mServiceInterface, SEQUENCE); + List events = Arrays.asList( + new GestureStep(0, tooManyPointsCount, startTouchPoints), + new GestureStep(CLICK_DURATION, tooManyPointsCount, endTouchPoints)); + attachMockNext(mMotionEventInjector); + injectEventsSync(events, mServiceInterface, CLICK_SEQUENCE); mMessageCapturingHandler.sendAllMessages(); - verify(next, times(2)).onMotionEvent(mCaptor1.capture(), mCaptor2.capture(), anyInt()); - assertEquals(MotionEvent.ACTION_DOWN, mCaptor1.getAllValues().get(0).getActionMasked()); - assertEquals(MotionEvent.ACTION_UP, mCaptor1.getAllValues().get(1).getActionMasked()); + verify(mServiceInterface).onPerformGestureResult(eq(CLICK_SEQUENCE), anyBoolean()); } @Test public void testRegularEvent_afterGestureComplete_shouldPassToNext() { EventStreamTransformation next = attachMockNext(mMotionEventInjector); - mMotionEventInjector.injectEvents(mClickList, mServiceInterface, SEQUENCE); - mMessageCapturingHandler.sendOneMessage(); // Process the event injection + injectEventsSync(mLineList, mServiceInterface, LINE_SEQUENCE); mMessageCapturingHandler.sendAllMessages(); // Send all motion events reset(next); - mMotionEventInjector.onMotionEvent(mSecondClickList.get(0), mClickList.get(0), 0); - verify(next).onMotionEvent(argThat(mSecondClickEvent0Matcher), - argThat(mClickEvent0Matcher), eq(0)); + mMotionEventInjector.onMotionEvent(mClickDownEvent, mClickDownEvent, 0); + verify(next).onMotionEvent(argThat(mIsClickDown), argThat(mIsClickDown), eq(0)); } @Test public void testInjectEvents_withRealGestureUnderway_shouldCancelRealAndPassInjected() { EventStreamTransformation next = attachMockNext(mMotionEventInjector); - mMotionEventInjector.onMotionEvent(mClickList.get(0), mClickList.get(0), 0); - mMotionEventInjector.injectEvents(mSecondClickList, mServiceInterface, SEQUENCE); - mMessageCapturingHandler.sendOneMessage(); // Process the event injection + mMotionEventInjector.onMotionEvent(mClickDownEvent, mClickDownEvent, 0); + injectEventsSync(mLineList, mServiceInterface, LINE_SEQUENCE); verify(next, times(2)).onMotionEvent(mCaptor1.capture(), mCaptor2.capture(), anyInt()); - assertTrue(mClickEvent0Matcher.matches(mCaptor1.getAllValues().get(0))); - assertEquals(MotionEvent.ACTION_CANCEL, mCaptor1.getAllValues().get(1).getActionMasked()); + assertThat(mCaptor1.getAllValues().get(0), mIsClickDown); + assertThat(mCaptor1.getAllValues().get(1), IS_ACTION_CANCEL); reset(next); mMessageCapturingHandler.sendOneMessage(); // Send a motion event - verify(next).onMotionEvent(mCaptor1.capture(), mCaptor2.capture(), - eq(WindowManagerPolicy.FLAG_PASS_TO_USER)); - long gestureStart = mCaptor1.getValue().getDownTime(); - mSecondClickEvent0Matcher.offsetTimesBy(gestureStart); - - verify(next).onMotionEvent(argThat(mSecondClickEvent0Matcher), - argThat(mSecondClickEvent0Matcher), eq(WindowManagerPolicy.FLAG_PASS_TO_USER)); + verify(next).onMotionEvent( + argThat(mIsLineStart), argThat(mIsLineStart), eq(FLAG_PASS_TO_USER)); } @Test public void testInjectEvents_withRealMouseGestureUnderway_shouldContinueRealAndPassInjected() { EventStreamTransformation next = attachMockNext(mMotionEventInjector); - MotionEvent mouseEvent = MotionEvent.obtain(mClickList.get(0)); + MotionEvent mouseEvent = MotionEvent.obtain(mClickDownEvent); mouseEvent.setSource(InputDevice.SOURCE_MOUSE); - MotionEventMatcher mouseEventMatcher = new MotionEventMatcher(mouseEvent); + MotionEventMatcher isMouseEvent = new MotionEventMatcher(mouseEvent); mMotionEventInjector.onMotionEvent(mouseEvent, mouseEvent, 0); - mMotionEventInjector.injectEvents(mSecondClickList, mServiceInterface, SEQUENCE); - mMessageCapturingHandler.sendOneMessage(); // Process the event injection + injectEventsSync(mLineList, mServiceInterface, LINE_SEQUENCE); mMessageCapturingHandler.sendOneMessage(); // Send a motion event verify(next, times(2)).onMotionEvent(mCaptor1.capture(), mCaptor2.capture(), anyInt()); - assertTrue(mouseEventMatcher.matches(mCaptor1.getAllValues().get(0))); - mSecondClickEvent0Matcher.offsetTimesBy(mCaptor1.getAllValues().get(1).getDownTime()); - assertTrue(mSecondClickEvent0Matcher.matches(mCaptor1.getAllValues().get(1))); + assertThat(mCaptor1.getAllValues().get(0), isMouseEvent); + assertThat(mCaptor1.getAllValues().get(1), mIsLineStart); } @Test public void testInjectEvents_withRealGestureFinished_shouldJustPassInjected() { EventStreamTransformation next = attachMockNext(mMotionEventInjector); - mMotionEventInjector.onMotionEvent(mClickList.get(0), mClickList.get(0), 0); - mMotionEventInjector.onMotionEvent(mClickList.get(1), mClickList.get(1), 0); - mMotionEventInjector.onMotionEvent(mClickList.get(2), mClickList.get(2), 0); + mMotionEventInjector.onMotionEvent(mClickDownEvent, mClickDownEvent, 0); + mMotionEventInjector.onMotionEvent(mClickUpEvent, mClickUpEvent, 0); - mMotionEventInjector.injectEvents(mSecondClickList, mServiceInterface, SEQUENCE); - mMessageCapturingHandler.sendOneMessage(); // Process the event injection - verify(next, times(3)).onMotionEvent(mCaptor1.capture(), mCaptor2.capture(), anyInt()); - - assertTrue(mClickEvent0Matcher.matches(mCaptor1.getAllValues().get(0))); - assertTrue(mClickEvent1Matcher.matches(mCaptor1.getAllValues().get(1))); - assertTrue(mClickEvent2Matcher.matches(mCaptor1.getAllValues().get(2))); + injectEventsSync(mLineList, mServiceInterface, LINE_SEQUENCE); + verify(next, times(2)).onMotionEvent(mCaptor1.capture(), mCaptor2.capture(), anyInt()); + assertThat(mCaptor1.getAllValues().get(0), mIsClickDown); + assertThat(mCaptor1.getAllValues().get(1), mIsClickUp); reset(next); mMessageCapturingHandler.sendOneMessage(); // Send a motion event - verify(next).onMotionEvent(mCaptor1.capture(), mCaptor2.capture(), - eq(WindowManagerPolicy.FLAG_PASS_TO_USER)); - mSecondClickEvent0Matcher.offsetTimesBy(mCaptor1.getValue().getDownTime()); - verify(next).onMotionEvent(argThat(mSecondClickEvent0Matcher), - argThat(mSecondClickEvent0Matcher), eq(WindowManagerPolicy.FLAG_PASS_TO_USER)); + verify(next).onMotionEvent(mCaptor1.capture(), mCaptor2.capture(), eq(FLAG_PASS_TO_USER)); + verify(next).onMotionEvent( + argThat(mIsLineStart), argThat(mIsLineStart), eq(FLAG_PASS_TO_USER)); } @Test public void testOnMotionEvents_openInjectedGestureInProgress_shouldCancelAndNotifyAndPassReal() throws RemoteException { EventStreamTransformation next = attachMockNext(mMotionEventInjector); - mMotionEventInjector.injectEvents(mClickList, mServiceInterface, SEQUENCE); - mMessageCapturingHandler.sendOneMessage(); // Process the event injection - + injectEventsSync(mLineList, mServiceInterface, LINE_SEQUENCE); mMessageCapturingHandler.sendOneMessage(); // Send a motion event - mMotionEventInjector.onMotionEvent(mSecondClickList.get(0), mSecondClickList.get(0), 0); + mMotionEventInjector.onMotionEvent(mClickDownEvent, mClickDownEvent, 0); verify(next, times(3)).onMotionEvent(mCaptor1.capture(), mCaptor2.capture(), anyInt()); - mClickEvent0Matcher.offsetTimesBy(mCaptor1.getAllValues().get(0).getDownTime()); - assertTrue(mClickEvent0Matcher.matches(mCaptor1.getAllValues().get(0))); - assertEquals(MotionEvent.ACTION_CANCEL, mCaptor1.getAllValues().get(1).getActionMasked()); - assertTrue(mSecondClickEvent0Matcher.matches(mCaptor1.getAllValues().get(2))); - verify(mServiceInterface).onPerformGestureResult(SEQUENCE, false); + assertThat(mCaptor1.getAllValues().get(0), mIsLineStart); + assertThat(mCaptor1.getAllValues().get(1), IS_ACTION_CANCEL); + assertThat(mCaptor1.getAllValues().get(2), mIsClickDown); + verify(mServiceInterface).onPerformGestureResult(LINE_SEQUENCE, false); } @Test public void testOnMotionEvents_closedInjectedGestureInProgress_shouldOnlyNotifyAndPassReal() throws RemoteException { EventStreamTransformation next = attachMockNext(mMotionEventInjector); - mClickList.add(MotionEvent.obtain(2 * CLICK_DURATION, 2 * CLICK_DURATION, - MotionEvent.ACTION_DOWN, CLICK_X, CLICK_Y_START, 0)); - mMotionEventInjector.injectEvents(mClickList, mServiceInterface, SEQUENCE); - mMessageCapturingHandler.sendOneMessage(); // Process the event injection + // Tack a click down to the end of the line + TouchPoint clickTouchPoint = new TouchPoint(); + clickTouchPoint.mIsStartOfPath = true; + clickTouchPoint.mX = CLICK_POINT.x; + clickTouchPoint.mY = CLICK_POINT.y; + mLineList.add(new GestureStep(0, 1, new TouchPoint[] {clickTouchPoint})); + + injectEventsSync(mLineList, mServiceInterface, LINE_SEQUENCE); // Send 3 motion events, leaving the extra down in the queue mMessageCapturingHandler.sendOneMessage(); mMessageCapturingHandler.sendOneMessage(); mMessageCapturingHandler.sendOneMessage(); - mMotionEventInjector.onMotionEvent(mSecondClickList.get(0), mClickList.get(0), 0); + mMotionEventInjector.onMotionEvent(mClickDownEvent, mClickDownEvent, 0); verify(next, times(4)).onMotionEvent(mCaptor1.capture(), mCaptor2.capture(), anyInt()); - long gestureStart = mCaptor1.getAllValues().get(0).getDownTime(); - mClickEvent0Matcher.offsetTimesBy(gestureStart); - mClickEvent1Matcher.offsetTimesBy(gestureStart); - mClickEvent2Matcher.offsetTimesBy(gestureStart); - assertTrue(mClickEvent0Matcher.matches(mCaptor1.getAllValues().get(0))); - assertTrue(mClickEvent1Matcher.matches(mCaptor1.getAllValues().get(1))); - assertTrue(mClickEvent2Matcher.matches(mCaptor1.getAllValues().get(2))); - assertTrue(mSecondClickEvent0Matcher.matches(mCaptor1.getAllValues().get(3))); - - verify(mServiceInterface).onPerformGestureResult(SEQUENCE, false); + assertThat(mCaptor1.getAllValues().get(0), mIsLineStart); + assertThat(mCaptor1.getAllValues().get(1), mIsLineMiddle); + assertThat(mCaptor1.getAllValues().get(2), mIsLineEnd); + assertThat(mCaptor1.getAllValues().get(3), mIsClickDown); + verify(mServiceInterface).onPerformGestureResult(LINE_SEQUENCE, false); assertFalse(mMessageCapturingHandler.hasMessages()); } @@ -326,105 +335,327 @@ public class MotionEventInjectorTest { public void testInjectEvents_openInjectedGestureInProgress_shouldCancelAndNotifyAndPassNew() throws RemoteException { EventStreamTransformation next = attachMockNext(mMotionEventInjector); - mMotionEventInjector.injectEvents(mClickList, mServiceInterface, SEQUENCE); - mMessageCapturingHandler.sendOneMessage(); // Process the event injection + injectEventsSync(mLineList, mServiceInterface, LINE_SEQUENCE); mMessageCapturingHandler.sendOneMessage(); // Send a motion event - mMotionEventInjector.injectEvents(mSecondClickList, mServiceInterface, SECOND_SEQUENCE); - mMessageCapturingHandler.sendLastMessage(); // Process the second event injection + injectEventsSync(mClickList, mServiceInterface, CLICK_SEQUENCE); mMessageCapturingHandler.sendOneMessage(); // Send a motion event - verify(mServiceInterface, times(1)).onPerformGestureResult(SEQUENCE, false); + verify(mServiceInterface, times(1)).onPerformGestureResult(LINE_SEQUENCE, false); verify(next, times(3)).onMotionEvent(mCaptor1.capture(), mCaptor2.capture(), anyInt()); - mClickEvent0Matcher.offsetTimesBy(mCaptor1.getAllValues().get(0).getDownTime()); - assertTrue(mClickEvent0Matcher.matches(mCaptor1.getAllValues().get(0))); - assertEquals(MotionEvent.ACTION_CANCEL, mCaptor1.getAllValues().get(1).getActionMasked()); - mSecondClickEvent0Matcher.offsetTimesBy(mCaptor1.getAllValues().get(2).getDownTime()); - assertTrue(mSecondClickEvent0Matcher.matches(mCaptor1.getAllValues().get(2))); + assertThat(mCaptor1.getAllValues().get(0), mIsLineStart); + assertThat(mCaptor1.getAllValues().get(1), IS_ACTION_CANCEL); + assertThat(mCaptor1.getAllValues().get(2), mIsClickDown); } @Test public void testInjectEvents_closedInjectedGestureInProgress_shouldOnlyNotifyAndPassNew() throws RemoteException { EventStreamTransformation next = attachMockNext(mMotionEventInjector); - MotionEvent newEvent = MotionEvent.obtain(2 * CLICK_DURATION, 2 * CLICK_DURATION, - MotionEvent.ACTION_DOWN, CLICK_X, CLICK_Y_START, 0); - newEvent.setSource(mClickList.get(0).getSource()); - mClickList.add(newEvent); - mMotionEventInjector.injectEvents(mClickList, mServiceInterface, SEQUENCE); - mMessageCapturingHandler.sendOneMessage(); // Process the event injection + // Tack a click down to the end of the line + TouchPoint clickTouchPoint = new TouchPoint(); + clickTouchPoint.mIsStartOfPath = true; + clickTouchPoint.mX = CLICK_POINT.x; + clickTouchPoint.mY = CLICK_POINT.y; + mLineList.add(new GestureStep(0, 1, new TouchPoint[] {clickTouchPoint})); + injectEventsSync(mLineList, mServiceInterface, LINE_SEQUENCE); // Send 3 motion events, leaving newEvent in the queue mMessageCapturingHandler.sendOneMessage(); mMessageCapturingHandler.sendOneMessage(); mMessageCapturingHandler.sendOneMessage(); - mMotionEventInjector.injectEvents(mSecondClickList, mServiceInterface, SECOND_SEQUENCE); - mMessageCapturingHandler.sendLastMessage(); // Process the event injection + injectEventsSync(mClickList, mServiceInterface, CLICK_SEQUENCE); mMessageCapturingHandler.sendOneMessage(); // Send a motion event - verify(mServiceInterface).onPerformGestureResult(SEQUENCE, false); + verify(mServiceInterface).onPerformGestureResult(LINE_SEQUENCE, false); verify(next, times(4)).onMotionEvent(mCaptor1.capture(), mCaptor2.capture(), anyInt()); - long gestureStart = mCaptor1.getAllValues().get(0).getDownTime(); - mClickEvent0Matcher.offsetTimesBy(gestureStart); - mClickEvent1Matcher.offsetTimesBy(gestureStart); - mClickEvent2Matcher.offsetTimesBy(gestureStart); - assertTrue(mClickEvent0Matcher.matches(mCaptor1.getAllValues().get(0))); - assertTrue(mClickEvent1Matcher.matches(mCaptor1.getAllValues().get(1))); - assertTrue(mClickEvent2Matcher.matches(mCaptor1.getAllValues().get(2))); - mSecondClickEvent0Matcher.offsetTimesBy(mCaptor1.getAllValues().get(3).getDownTime()); - assertTrue(mSecondClickEvent0Matcher.matches(mCaptor1.getAllValues().get(3))); + assertThat(mCaptor1.getAllValues().get(0), mIsLineStart); + assertThat(mCaptor1.getAllValues().get(1), mIsLineMiddle); + assertThat(mCaptor1.getAllValues().get(2), mIsLineEnd); + assertThat(mCaptor1.getAllValues().get(3), mIsClickDown); + } + + @Test + public void testContinuedGesture_continuationArrivesAfterDispatched_gestureCompletes() + throws Exception { + EventStreamTransformation next = attachMockNext(mMotionEventInjector); + injectEventsSync(mContinuedLineList1, mServiceInterface, CONTINUED_LINE_SEQUENCE_1); + mMessageCapturingHandler.sendAllMessages(); // Send all motion events + verify(mServiceInterface).onPerformGestureResult(CONTINUED_LINE_SEQUENCE_1, true); + injectEventsSync(mContinuedLineList2, mServiceInterface, CONTINUED_LINE_SEQUENCE_2); + mMessageCapturingHandler.sendAllMessages(); // Send all motion events + verify(mServiceInterface).onPerformGestureResult(CONTINUED_LINE_SEQUENCE_2, true); + verify(next, times(5)).onMotionEvent(mCaptor1.capture(), mCaptor2.capture(), anyInt()); + List events = mCaptor1.getAllValues(); + long downTime = events.get(0).getDownTime(); + assertThat(events.get(0), allOf(isAtPoint(CONTINUED_LINE_START), IS_ACTION_DOWN, + hasEventTime(downTime))); + assertThat(events, everyItem(hasDownTime(downTime))); + assertThat(events.get(1), allOf(isAtPoint(CONTINUED_LINE_MID1), IS_ACTION_MOVE, + hasEventTime(downTime + CONTINUED_LINE_INTERVAL))); + // Timing will restart when the gesture continues + long secondSequenceStart = events.get(2).getEventTime(); + assertTrue(secondSequenceStart > events.get(1).getEventTime()); + assertThat(events.get(2), allOf(isAtPoint(CONTINUED_LINE_MID2), IS_ACTION_MOVE)); + assertThat(events.get(3), allOf(isAtPoint(CONTINUED_LINE_END), IS_ACTION_MOVE, + hasEventTime(secondSequenceStart + CONTINUED_LINE_INTERVAL))); + assertThat(events.get(4), allOf(isAtPoint(CONTINUED_LINE_END), IS_ACTION_UP, + hasEventTime(secondSequenceStart + CONTINUED_LINE_INTERVAL))); + } + + @Test + public void testContinuedGesture_withTwoTouchPoints_gestureCompletes() + throws Exception { + // Run one point through the continued line backwards + int backLineId1 = 30; + int backLineId2 = 30; + List continuedBackLineList1 = createSimpleGestureFromPoints(backLineId1, 0, + true, CONTINUED_LINE_INTERVAL, CONTINUED_LINE_END, CONTINUED_LINE_MID2); + List continuedBackLineList2 = createSimpleGestureFromPoints(backLineId2, + backLineId1, false, CONTINUED_LINE_INTERVAL, CONTINUED_LINE_MID2, + CONTINUED_LINE_MID1, CONTINUED_LINE_START); + List combinedLines1 = combineGestureSteps( + mContinuedLineList1, continuedBackLineList1); + List combinedLines2 = combineGestureSteps( + mContinuedLineList2, continuedBackLineList2); + + EventStreamTransformation next = attachMockNext(mMotionEventInjector); + injectEventsSync(combinedLines1, mServiceInterface, CONTINUED_LINE_SEQUENCE_1); + injectEventsSync(combinedLines2, mServiceInterface, CONTINUED_LINE_SEQUENCE_2); + mMessageCapturingHandler.sendAllMessages(); // Send all motion events + verify(mServiceInterface).onPerformGestureResult(CONTINUED_LINE_SEQUENCE_1, true); + verify(mServiceInterface).onPerformGestureResult(CONTINUED_LINE_SEQUENCE_2, true); + verify(next, times(7)).onMotionEvent(mCaptor1.capture(), mCaptor2.capture(), anyInt()); + List events = mCaptor1.getAllValues(); + long downTime = events.get(0).getDownTime(); + assertThat(events.get(0), allOf( + anyOf(isAtPoint(CONTINUED_LINE_END), isAtPoint(CONTINUED_LINE_START)), + IS_ACTION_DOWN, hasEventTime(downTime))); + assertThat(events, everyItem(hasDownTime(downTime))); + assertThat(events.get(1), allOf(containsPoints(CONTINUED_LINE_START, CONTINUED_LINE_END), + IS_ACTION_POINTER_DOWN, hasEventTime(downTime))); + assertThat(events.get(2), allOf(containsPoints(CONTINUED_LINE_MID1, CONTINUED_LINE_MID2), + IS_ACTION_MOVE, hasEventTime(downTime + CONTINUED_LINE_INTERVAL))); + assertThat(events.get(3), allOf(containsPoints(CONTINUED_LINE_MID1, CONTINUED_LINE_MID2), + IS_ACTION_MOVE, hasEventTime(downTime + CONTINUED_LINE_INTERVAL * 2))); + assertThat(events.get(4), allOf(containsPoints(CONTINUED_LINE_START, CONTINUED_LINE_END), + IS_ACTION_MOVE, hasEventTime(downTime + CONTINUED_LINE_INTERVAL * 3))); + assertThat(events.get(5), allOf(containsPoints(CONTINUED_LINE_START, CONTINUED_LINE_END), + IS_ACTION_POINTER_UP, hasEventTime(downTime + CONTINUED_LINE_INTERVAL * 3))); + assertThat(events.get(6), allOf( + anyOf(isAtPoint(CONTINUED_LINE_END), isAtPoint(CONTINUED_LINE_START)), + IS_ACTION_UP, hasEventTime(downTime + CONTINUED_LINE_INTERVAL * 3))); + } + + + @Test + public void testContinuedGesture_continuationArrivesWhileDispatching_gestureCompletes() + throws Exception { + EventStreamTransformation next = attachMockNext(mMotionEventInjector); + injectEventsSync(mContinuedLineList1, mServiceInterface, CONTINUED_LINE_SEQUENCE_1); + mMessageCapturingHandler.sendOneMessage(); // Send a motion event + injectEventsSync(mContinuedLineList2, mServiceInterface, CONTINUED_LINE_SEQUENCE_2); + mMessageCapturingHandler.sendAllMessages(); // Send all motion events + verify(mServiceInterface).onPerformGestureResult(CONTINUED_LINE_SEQUENCE_1, true); + verify(mServiceInterface).onPerformGestureResult(CONTINUED_LINE_SEQUENCE_2, true); + verify(next, times(5)).onMotionEvent(mCaptor1.capture(), mCaptor2.capture(), anyInt()); + List events = mCaptor1.getAllValues(); + long downTime = events.get(0).getDownTime(); + assertThat(events.get(0), allOf(isAtPoint(CONTINUED_LINE_START), IS_ACTION_DOWN, + hasEventTime(downTime))); + assertThat(events, everyItem(hasDownTime(downTime))); + assertThat(events.get(1), allOf(isAtPoint(CONTINUED_LINE_MID1), IS_ACTION_MOVE, + hasEventTime(downTime + CONTINUED_LINE_INTERVAL))); + assertThat(events.get(2), allOf(isAtPoint(CONTINUED_LINE_MID2), IS_ACTION_MOVE, + hasEventTime(downTime + CONTINUED_LINE_INTERVAL * 2))); + assertThat(events.get(3), allOf(isAtPoint(CONTINUED_LINE_END), IS_ACTION_MOVE, + hasEventTime(downTime + CONTINUED_LINE_INTERVAL * 3))); + assertThat(events.get(4), allOf(isAtPoint(CONTINUED_LINE_END), IS_ACTION_UP, + hasEventTime(downTime + CONTINUED_LINE_INTERVAL * 3))); + } + + @Test + public void testContinuedGesture_twoContinuationsArriveWhileDispatching_gestureCompletes() + throws Exception { + EventStreamTransformation next = attachMockNext(mMotionEventInjector); + injectEventsSync(mContinuedLineList1, mServiceInterface, CONTINUED_LINE_SEQUENCE_1); + // Continue line again + List continuedLineList2 = createSimpleGestureFromPoints( + CONTINUED_LINE_STROKE_ID_2, CONTINUED_LINE_STROKE_ID_1, true, + CONTINUED_LINE_INTERVAL, CONTINUED_LINE_MID1, + CONTINUED_LINE_MID2, CONTINUED_LINE_END); + // Finish line by backtracking + int strokeId3 = CONTINUED_LINE_STROKE_ID_2 + 1; + int sequence3 = CONTINUED_LINE_SEQUENCE_2 + 1; + List continuedLineList3 = createSimpleGestureFromPoints(strokeId3, + CONTINUED_LINE_STROKE_ID_2, false, CONTINUED_LINE_INTERVAL, CONTINUED_LINE_END, + CONTINUED_LINE_MID2); + + mMessageCapturingHandler.sendOneMessage(); // Send a motion event + injectEventsSync(continuedLineList2, mServiceInterface, CONTINUED_LINE_SEQUENCE_2); + injectEventsSync(continuedLineList3, mServiceInterface, sequence3); + mMessageCapturingHandler.sendAllMessages(); // Send all motion events + verify(mServiceInterface).onPerformGestureResult(CONTINUED_LINE_SEQUENCE_1, true); + verify(mServiceInterface).onPerformGestureResult(CONTINUED_LINE_SEQUENCE_2, true); + verify(mServiceInterface).onPerformGestureResult(sequence3, true); + verify(next, times(6)).onMotionEvent(mCaptor1.capture(), mCaptor2.capture(), anyInt()); + List events = mCaptor1.getAllValues(); + long downTime = events.get(0).getDownTime(); + assertThat(events.get(0), allOf(isAtPoint(CONTINUED_LINE_START), IS_ACTION_DOWN, + hasEventTime(downTime))); + assertThat(events, everyItem(hasDownTime(downTime))); + assertThat(events.get(1), allOf(isAtPoint(CONTINUED_LINE_MID1), IS_ACTION_MOVE, + hasEventTime(downTime + CONTINUED_LINE_INTERVAL))); + assertThat(events.get(2), allOf(isAtPoint(CONTINUED_LINE_MID2), IS_ACTION_MOVE, + hasEventTime(downTime + CONTINUED_LINE_INTERVAL * 2))); + assertThat(events.get(3), allOf(isAtPoint(CONTINUED_LINE_END), IS_ACTION_MOVE, + hasEventTime(downTime + CONTINUED_LINE_INTERVAL * 3))); + assertThat(events.get(4), allOf(isAtPoint(CONTINUED_LINE_MID2), IS_ACTION_MOVE, + hasEventTime(downTime + CONTINUED_LINE_INTERVAL * 4))); + assertThat(events.get(5), allOf(isAtPoint(CONTINUED_LINE_MID2), IS_ACTION_UP, + hasEventTime(downTime + CONTINUED_LINE_INTERVAL * 4))); + } + + @Test + public void testContinuedGesture_nonContinuingGestureArrivesDuringDispatch_gestureCanceled() + throws Exception { + EventStreamTransformation next = attachMockNext(mMotionEventInjector); + injectEventsSync(mContinuedLineList1, mServiceInterface, CONTINUED_LINE_SEQUENCE_1); + mMessageCapturingHandler.sendOneMessage(); // Send a motion event + injectEventsSync(mLineList, mServiceInterface, LINE_SEQUENCE); + mMessageCapturingHandler.sendAllMessages(); // Send all motion events + verify(mServiceInterface).onPerformGestureResult(CONTINUED_LINE_SEQUENCE_1, false); + verify(mServiceInterface).onPerformGestureResult(LINE_SEQUENCE, true); + verify(next, times(5)).onMotionEvent(mCaptor1.capture(), mCaptor2.capture(), anyInt()); + List events = mCaptor1.getAllValues(); + assertThat(events.get(0), allOf(isAtPoint(CONTINUED_LINE_START), IS_ACTION_DOWN)); + assertThat(events.get(1), IS_ACTION_CANCEL); + assertThat(events.get(2), allOf(isAtPoint(LINE_START), IS_ACTION_DOWN)); + assertThat(events.get(3), allOf(isAtPoint(LINE_END), IS_ACTION_MOVE)); + assertThat(events.get(4), allOf(isAtPoint(LINE_END), IS_ACTION_UP)); + } + + @Test + public void testContinuedGesture_nonContinuingGestureArrivesAfterDispatch_gestureCanceled() + throws Exception { + EventStreamTransformation next = attachMockNext(mMotionEventInjector); + injectEventsSync(mContinuedLineList1, mServiceInterface, CONTINUED_LINE_SEQUENCE_1); + mMessageCapturingHandler.sendAllMessages(); // Send all motion events + injectEventsSync(mLineList, mServiceInterface, LINE_SEQUENCE); + mMessageCapturingHandler.sendAllMessages(); // Send all motion events + verify(mServiceInterface).onPerformGestureResult(CONTINUED_LINE_SEQUENCE_1, true); + verify(mServiceInterface).onPerformGestureResult(LINE_SEQUENCE, true); + verify(next, times(6)).onMotionEvent(mCaptor1.capture(), mCaptor2.capture(), anyInt()); + List events = mCaptor1.getAllValues(); + assertThat(events.get(0), allOf(isAtPoint(CONTINUED_LINE_START), IS_ACTION_DOWN)); + assertThat(events.get(1), allOf(isAtPoint(CONTINUED_LINE_MID1), IS_ACTION_MOVE)); + assertThat(events.get(2), IS_ACTION_CANCEL); + assertThat(events.get(3), allOf(isAtPoint(LINE_START), IS_ACTION_DOWN)); + assertThat(events.get(4), allOf(isAtPoint(LINE_END), IS_ACTION_MOVE)); + assertThat(events.get(5), allOf(isAtPoint(LINE_END), IS_ACTION_UP)); + } + + @Test + public void testContinuedGesture_misMatchedContinuationArrives_bothGesturesCanceled() + throws Exception { + EventStreamTransformation next = attachMockNext(mMotionEventInjector); + injectEventsSync(mContinuedLineList1, mServiceInterface, CONTINUED_LINE_SEQUENCE_1); + mMessageCapturingHandler.sendAllMessages(); // Send all motion events + verify(mServiceInterface).onPerformGestureResult(CONTINUED_LINE_SEQUENCE_1, true); + List discontinuousGesture = mContinuedLineList2 + .subList(1, mContinuedLineList2.size()); + injectEventsSync(discontinuousGesture, mServiceInterface, CONTINUED_LINE_SEQUENCE_2); + mMessageCapturingHandler.sendAllMessages(); // Send all motion events + verify(mServiceInterface).onPerformGestureResult(CONTINUED_LINE_SEQUENCE_2, false); + verify(next, times(3)).onMotionEvent(mCaptor1.capture(), mCaptor2.capture(), anyInt()); + List events = mCaptor1.getAllValues(); + assertThat(events.get(0), allOf(isAtPoint(CONTINUED_LINE_START), IS_ACTION_DOWN)); + assertThat(events.get(1), allOf(isAtPoint(CONTINUED_LINE_MID1), IS_ACTION_MOVE)); + assertThat(events.get(2), allOf(isAtPoint(CONTINUED_LINE_MID1), IS_ACTION_CANCEL)); + } + + @Test + public void testContinuedGesture_continuationArrivesFromOtherService_bothGesturesCanceled() + throws Exception { + EventStreamTransformation next = attachMockNext(mMotionEventInjector); + IAccessibilityServiceClient otherService = mock(IAccessibilityServiceClient.class); + injectEventsSync(mContinuedLineList1, mServiceInterface, CONTINUED_LINE_SEQUENCE_1); + mMessageCapturingHandler.sendOneMessage(); // Send a motion events + injectEventsSync(mContinuedLineList2, otherService, CONTINUED_LINE_SEQUENCE_2); + mMessageCapturingHandler.sendAllMessages(); // Send all motion events + verify(mServiceInterface).onPerformGestureResult(CONTINUED_LINE_SEQUENCE_1, false); + verify(otherService).onPerformGestureResult(CONTINUED_LINE_SEQUENCE_2, false); + verify(next, times(2)).onMotionEvent(mCaptor1.capture(), mCaptor2.capture(), anyInt()); + List events = mCaptor1.getAllValues(); + assertThat(events.get(0), allOf(isAtPoint(CONTINUED_LINE_START), IS_ACTION_DOWN)); + assertThat(events.get(1), IS_ACTION_CANCEL); + } + + @Test + public void testContinuedGesture_realGestureArrivesInBetween_getsCanceled() + throws Exception { + EventStreamTransformation next = attachMockNext(mMotionEventInjector); + injectEventsSync(mContinuedLineList1, mServiceInterface, CONTINUED_LINE_SEQUENCE_1); + mMessageCapturingHandler.sendAllMessages(); // Send all motion events + verify(mServiceInterface).onPerformGestureResult(CONTINUED_LINE_SEQUENCE_1, true); + + mMotionEventInjector.onMotionEvent(mClickDownEvent, mClickDownEvent, 0); + + injectEventsSync(mContinuedLineList2, mServiceInterface, CONTINUED_LINE_SEQUENCE_2); + mMessageCapturingHandler.sendAllMessages(); // Send all motion events + verify(mServiceInterface).onPerformGestureResult(CONTINUED_LINE_SEQUENCE_2, false); + verify(next, times(4)).onMotionEvent(mCaptor1.capture(), mCaptor2.capture(), anyInt()); + List events = mCaptor1.getAllValues(); + assertThat(events.get(0), allOf(isAtPoint(CONTINUED_LINE_START), IS_ACTION_DOWN)); + assertThat(events.get(1), allOf(isAtPoint(CONTINUED_LINE_MID1), IS_ACTION_MOVE)); + assertThat(events.get(2), IS_ACTION_CANCEL); + assertThat(events.get(3), allOf(isAtPoint(CLICK_POINT), IS_ACTION_DOWN)); } @Test public void testClearEvents_realGestureInProgress_shouldForgetAboutGesture() { EventStreamTransformation next = attachMockNext(mMotionEventInjector); - mMotionEventInjector.onMotionEvent(mClickList.get(0), mClickList.get(0), 0); + mMotionEventInjector.onMotionEvent(mClickDownEvent, mClickDownEvent, 0); mMotionEventInjector.clearEvents(MOTION_EVENT_SOURCE); - mMotionEventInjector.injectEvents(mSecondClickList, mServiceInterface, SEQUENCE); - mMessageCapturingHandler.sendOneMessage(); // Process the event injection + injectEventsSync(mLineList, mServiceInterface, LINE_SEQUENCE); mMessageCapturingHandler.sendOneMessage(); // Send a motion event verify(next, times(2)).onMotionEvent(mCaptor1.capture(), mCaptor2.capture(), anyInt()); - assertTrue(mClickEvent0Matcher.matches(mCaptor1.getAllValues().get(0))); - mSecondClickEvent0Matcher.offsetTimesBy(mCaptor1.getAllValues().get(1).getDownTime()); - assertTrue(mSecondClickEvent0Matcher.matches(mCaptor1.getAllValues().get(1))); + assertThat(mCaptor1.getAllValues().get(0), mIsClickDown); + assertThat(mCaptor1.getAllValues().get(1), mIsLineStart); } @Test public void testClearEventsOnOtherSource_realGestureInProgress_shouldNotForgetAboutGesture() { EventStreamTransformation next = attachMockNext(mMotionEventInjector); - mMotionEventInjector.onMotionEvent(mClickList.get(0), mClickList.get(0), 0); + mMotionEventInjector.onMotionEvent(mClickDownEvent, mClickDownEvent, 0); mMotionEventInjector.clearEvents(OTHER_EVENT_SOURCE); - mMotionEventInjector.injectEvents(mSecondClickList, mServiceInterface, SECOND_SEQUENCE); - mMessageCapturingHandler.sendOneMessage(); // Process the event injection + injectEventsSync(mLineList, mServiceInterface, LINE_SEQUENCE); mMessageCapturingHandler.sendOneMessage(); // Send a motion event verify(next, times(3)).onMotionEvent(mCaptor1.capture(), mCaptor2.capture(), anyInt()); - assertTrue(mClickEvent0Matcher.matches(mCaptor1.getAllValues().get(0))); - assertEquals(MotionEvent.ACTION_CANCEL, mCaptor1.getAllValues().get(1).getActionMasked()); - mSecondClickEvent0Matcher.offsetTimesBy(mCaptor1.getAllValues().get(2).getDownTime()); - assertTrue(mSecondClickEvent0Matcher.matches(mCaptor1.getAllValues().get(2))); + assertThat(mCaptor1.getAllValues().get(0), mIsClickDown); + assertThat(mCaptor1.getAllValues().get(1), IS_ACTION_CANCEL); + assertThat(mCaptor1.getAllValues().get(2), mIsLineStart); } @Test public void testOnDestroy_shouldCancelGestures() throws RemoteException { mMotionEventInjector.onDestroy(); - mMotionEventInjector.injectEvents(mClickList, mServiceInterface, SEQUENCE); - mMessageCapturingHandler.sendOneMessage(); // Process the event injection - verify(mServiceInterface).onPerformGestureResult(SEQUENCE, false); + injectEventsSync(mLineList, mServiceInterface, LINE_SEQUENCE); + verify(mServiceInterface).onPerformGestureResult(LINE_SEQUENCE, false); } @Test public void testInjectEvents_withNoNext_shouldCancel() throws RemoteException { - mMotionEventInjector.injectEvents(mClickList, mServiceInterface, SEQUENCE); - mMessageCapturingHandler.sendOneMessage(); // Process the event injection - verify(mServiceInterface).onPerformGestureResult(SEQUENCE, false); + injectEventsSync(mLineList, mServiceInterface, LINE_SEQUENCE); + verify(mServiceInterface).onPerformGestureResult(LINE_SEQUENCE, false); } @Test public void testOnMotionEvent_withNoNext_shouldNotCrash() { - mMotionEventInjector.onMotionEvent(mClickList.get(0), mClickList.get(0), 0); + mMotionEventInjector.onMotionEvent(mClickDownEvent, mClickDownEvent, 0); } @Test @@ -455,6 +686,52 @@ public class MotionEventInjectorTest { mMotionEventInjector.onAccessibilityEvent(event); } + private void injectEventsSync(List gestureSteps, + IAccessibilityServiceClient serviceInterface, int sequence) { + mMotionEventInjector.injectEvents(gestureSteps, serviceInterface, sequence); + // Dispatch the message sent by the injector. Our simple handler doesn't guarantee stuff + // happens in order. + mMessageCapturingHandler.sendLastMessage(); + } + + private List createSimpleGestureFromPoints(int strokeId, int continuedStrokeId, + boolean continued, long interval, Point... points) { + List gesture = new ArrayList<>(points.length); + TouchPoint[] touchPoints = new TouchPoint[1]; + touchPoints[0] = new TouchPoint(); + for (int i = 0; i < points.length; i++) { + touchPoints[0].mX = points[i].x; + touchPoints[0].mY = points[i].y; + touchPoints[0].mIsStartOfPath = ((i == 0) && (continuedStrokeId <= 0)); + touchPoints[0].mContinuedStrokeId = continuedStrokeId; + touchPoints[0].mStrokeId = strokeId; + touchPoints[0].mIsEndOfPath = ((i == points.length - 1) && !continued); + gesture.add(new GestureStep(interval * i, 1, touchPoints)); + } + return gesture; + } + + List combineGestureSteps(List list1, List list2) { + assertEquals(list1.size(), list2.size()); + List gesture = new ArrayList<>(list1.size()); + for (int i = 0; i < list1.size(); i++) { + int numPoints1 = list1.get(i).numTouchPoints; + int numPoints2 = list2.get(i).numTouchPoints; + TouchPoint[] touchPoints = new TouchPoint[numPoints1 + numPoints2]; + for (int j = 0; j < numPoints1; j++) { + touchPoints[j] = new TouchPoint(); + touchPoints[j].copyFrom(list1.get(i).touchPoints[j]); + } + for (int j = 0; j < numPoints2; j++) { + touchPoints[numPoints1 + j] = new TouchPoint(); + touchPoints[numPoints1 + j].copyFrom(list2.get(i).touchPoints[j]); + } + gesture.add(new GestureStep(list1.get(i).timeSinceGestureStart, + numPoints1 + numPoints2, touchPoints)); + } + return gesture; + } + private EventStreamTransformation attachMockNext(MotionEventInjector motionEventInjector) { EventStreamTransformation next = mock(EventStreamTransformation.class); motionEventInjector.setNext(next); @@ -506,4 +783,126 @@ public class MotionEventInjectorTest { return false; } } + + private static class MotionEventActionMatcher extends TypeSafeMatcher { + int mAction; + + MotionEventActionMatcher(int action) { + super(); + mAction = action; + } + + @Override + protected boolean matchesSafely(MotionEvent motionEvent) { + return motionEvent.getActionMasked() == mAction; + } + + @Override + public void describeTo(Description description) { + description.appendText("Matching to action " + mAction); + } + } + + private static TypeSafeMatcher isAtPoint(final Point point) { + return new TypeSafeMatcher() { + @Override + protected boolean matchesSafely(MotionEvent event) { + return ((event.getX() == point.x) && (event.getY() == point.y)); + } + + @Override + public void describeTo(Description description) { + description.appendText("Is at point " + point); + } + }; + } + + private static TypeSafeMatcher containsPoints(final Point... points) { + return new TypeSafeMatcher() { + @Override + protected boolean matchesSafely(MotionEvent event) { + MotionEvent.PointerCoords coords = new MotionEvent.PointerCoords(); + for (int i = 0; i < points.length; i++) { + boolean havePoint = false; + for (int j = 0; j < points.length; j++) { + event.getPointerCoords(j, coords); + if ((points[i].x == coords.x) && (points[i].y == coords.y)) { + havePoint = true; + } + } + if (!havePoint) { + return false; + } + } + return true; + } + + @Override + public void describeTo(Description description) { + description.appendText("Contains points " + points); + } + }; + } + + private static TypeSafeMatcher hasDownTime(final long downTime) { + return new TypeSafeMatcher() { + @Override + protected boolean matchesSafely(MotionEvent event) { + return event.getDownTime() == downTime; + } + + @Override + public void describeTo(Description description) { + description.appendText("Down time = " + downTime); + } + }; + } + + private static TypeSafeMatcher hasEventTime(final long eventTime) { + return new TypeSafeMatcher() { + @Override + protected boolean matchesSafely(MotionEvent event) { + return event.getEventTime() == eventTime; + } + + @Override + public void describeTo(Description description) { + description.appendText("Event time = " + eventTime); + } + }; + } + + private static TypeSafeMatcher hasTimeFromDown(final long timeFromDown) { + return new TypeSafeMatcher() { + @Override + protected boolean matchesSafely(MotionEvent event) { + return (event.getEventTime() - event.getDownTime()) == timeFromDown; + } + + @Override + public void describeTo(Description description) { + description.appendText("Time from down to event times = " + timeFromDown); + } + }; + } + + private static TypeSafeMatcher hasStandardInitialization() { + return new TypeSafeMatcher() { + @Override + protected boolean matchesSafely(MotionEvent event) { + return (0 == event.getActionIndex()) && (0 == event.getDeviceId()) + && (0 == event.getEdgeFlags()) && (0 == event.getFlags()) + && (0 == event.getMetaState()) && (0F == event.getOrientation()) + && (0F == event.getTouchMajor()) && (0F == event.getTouchMinor()) + && (1F == event.getXPrecision()) && (1F == event.getYPrecision()) + && (1 == event.getPointerCount()) && (1F == event.getPressure()) + && (InputDevice.SOURCE_TOUCHSCREEN == event.getSource()); + } + + @Override + public void describeTo(Description description) { + description.appendText("Has standard values for all parameters"); + } + }; + } }