diff --git a/services/accessibility/java/com/android/server/accessibility/AccessibilityGestureDetector.java b/services/accessibility/java/com/android/server/accessibility/AccessibilityGestureDetector.java index 7dff1a8072117..28121b41c84fb 100644 --- a/services/accessibility/java/com/android/server/accessibility/AccessibilityGestureDetector.java +++ b/services/accessibility/java/com/android/server/accessibility/AccessibilityGestureDetector.java @@ -17,53 +17,146 @@ package com.android.server.accessibility; import android.content.Context; +import android.gesture.Gesture; +import android.gesture.GestureLibraries; +import android.gesture.GestureLibrary; +import android.gesture.GesturePoint; +import android.gesture.GestureStore; +import android.gesture.GestureStroke; +import android.gesture.Prediction; +import android.util.Slog; import android.view.GestureDetector; import android.view.MotionEvent; +import com.android.internal.R; + +import java.util.ArrayList; + /** * This class handles gesture detection for the Touch Explorer. It collects * touch events, and sends events to mListener as gestures are recognized. */ class AccessibilityGestureDetector extends GestureDetector.SimpleOnGestureListener { - private final GestureDetector mGestureDetector; + + private static final boolean DEBUG = false; + + // Tag for logging received events. + private static final String LOG_TAG = "AccessibilityGestureDetector"; + + public interface Listener { + public void onDoubleTapAndHold(MotionEvent event, int policyFlags); + public boolean onDoubleTap(MotionEvent event, int policyFlags); + public boolean onGesture(int gestureId); + } + private final Listener mListener; + private final GestureDetector mGestureDetector; + + // The library for gesture detection. + private final GestureLibrary mGestureLibrary; + + // Indicates that a single tap has occurred. private boolean mFirstTapDetected; + + // Indicates that the down event of a double tap has occured. private boolean mDoubleTapDetected; + + // Indicates that motion events are being collected to match a gesture. + private boolean mRecognizingGesture; + + // Policy flags of the previous event. private int mPolicyFlags; + // The X of the previous event. + private float mPreviousX; + + // The Y of the previous event. + private float mPreviousY; + + // Buffer for storing points for gesture detection. + private final ArrayList mStrokeBuffer = new ArrayList(100); + + // The minimal delta between moves to add a gesture point. + private static final int TOUCH_TOLERANCE = 3; + + // The minimal score for accepting a predicted gesture. + private static final float MIN_PREDICTION_SCORE = 2.0f; + AccessibilityGestureDetector(Context context, Listener listener) { mListener = listener; mGestureDetector = new GestureDetector(context, this); mGestureDetector.setOnDoubleTapListener(this); + + mGestureLibrary = GestureLibraries.fromRawResource(context, R.raw.accessibility_gestures); + mGestureLibrary.setOrientationStyle(8 /* GestureStore.ORIENTATION_SENSITIVE_8 */); + mGestureLibrary.setSequenceType(GestureStore.SEQUENCE_SENSITIVE); + mGestureLibrary.load(); } - public void onMotionEvent(MotionEvent event, int policyFlags) { + public boolean onMotionEvent(MotionEvent event, int policyFlags) { + final float x = event.getX(); + final float y = event.getY(); + mPolicyFlags = policyFlags; switch (event.getActionMasked()) { case MotionEvent.ACTION_DOWN: mDoubleTapDetected = false; + mRecognizingGesture = true; + mPreviousX = x; + mPreviousY = y; + mStrokeBuffer.clear(); + mStrokeBuffer.add(new GesturePoint(x, y, event.getEventTime())); + break; + + case MotionEvent.ACTION_MOVE: + if (mRecognizingGesture) { + final float dX = Math.abs(x - mPreviousX); + final float dY = Math.abs(y - mPreviousY); + if (dX >= TOUCH_TOLERANCE || dY >= TOUCH_TOLERANCE) { + mPreviousX = x; + mPreviousY = y; + mStrokeBuffer.add(new GesturePoint(x, y, event.getEventTime())); + } + } break; case MotionEvent.ACTION_UP: - maybeFinishDoubleTap(event, policyFlags); + if (maybeFinishDoubleTap(event, policyFlags)) { + return true; + } + if (mRecognizingGesture) { + mStrokeBuffer.add(new GesturePoint(x, y, event.getEventTime())); + + if (recognizeGesture()) { + return true; + } + } break; } - mGestureDetector.onTouchEvent(event); + + if (!mRecognizingGesture) { + return false; + } + + // Pass the event on to the standard gesture detector. + return mGestureDetector.onTouchEvent(event); } public void clear() { mFirstTapDetected = false; mDoubleTapDetected = false; + cancelGesture(); + mStrokeBuffer.clear(); } public boolean firstTapDetected() { return mFirstTapDetected; } - @Override - public boolean onDown(MotionEvent event) { - return true; + public void cancelGesture() { + mRecognizingGesture = false; + mStrokeBuffer.clear(); } @Override @@ -101,18 +194,39 @@ class AccessibilityGestureDetector extends GestureDetector.SimpleOnGestureListen mListener.onDoubleTapAndHold(event, policyFlags); } - private void maybeFinishDoubleTap(MotionEvent event, int policyFlags) { + private boolean maybeFinishDoubleTap(MotionEvent event, int policyFlags) { if (!mDoubleTapDetected) { - return; + return false; } clear(); - mListener.onDoubleTap(event, policyFlags); + return mListener.onDoubleTap(event, policyFlags); } - public interface Listener { - public void onDoubleTapAndHold(MotionEvent event, int policyFlags); - public void onDoubleTap(MotionEvent event, int policyFlags); + private boolean recognizeGesture() { + Gesture gesture = new Gesture(); + gesture.addStroke(new GestureStroke(mStrokeBuffer)); + + ArrayList predictions = mGestureLibrary.recognize(gesture); + if (!predictions.isEmpty()) { + Prediction bestPrediction = predictions.get(0); + if (bestPrediction.score >= MIN_PREDICTION_SCORE) { + if (DEBUG) { + Slog.i(LOG_TAG, "gesture: " + bestPrediction.name + " score: " + + bestPrediction.score); + } + try { + final int gestureId = Integer.parseInt(bestPrediction.name); + if (mListener.onGesture(gestureId)) { + return true; + } + } catch (NumberFormatException nfe) { + Slog.w(LOG_TAG, "Non numeric gesture id:" + bestPrediction.name); + } + } + } + + return false; } } diff --git a/services/accessibility/java/com/android/server/accessibility/TouchExplorer.java b/services/accessibility/java/com/android/server/accessibility/TouchExplorer.java index 954536ff674a9..ca30349867dea 100644 --- a/services/accessibility/java/com/android/server/accessibility/TouchExplorer.java +++ b/services/accessibility/java/com/android/server/accessibility/TouchExplorer.java @@ -17,13 +17,6 @@ package com.android.server.accessibility; import android.content.Context; -import android.gesture.Gesture; -import android.gesture.GestureLibraries; -import android.gesture.GestureLibrary; -import android.gesture.GesturePoint; -import android.gesture.GestureStore; -import android.gesture.GestureStroke; -import android.gesture.Prediction; import android.graphics.Point; import android.os.Handler; import android.util.Slog; @@ -38,8 +31,6 @@ import android.view.WindowManagerPolicy; import android.view.accessibility.AccessibilityEvent; import android.view.accessibility.AccessibilityManager; -import com.android.internal.R; - import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -167,24 +158,6 @@ class TouchExplorer implements EventStreamTransformation, AccessibilityGestureDe // Context in which this explorer operates. private final Context mContext; - // The X of the previous event. - private float mPreviousX; - - // The Y of the previous event. - private float mPreviousY; - - // Buffer for storing points for gesture detection. - private final ArrayList mStrokeBuffer = new ArrayList(100); - - // The minimal delta between moves to add a gesture point. - private static final int TOUCH_TOLERANCE = 3; - - // The minimal score for accepting a predicted gesture. - private static final float MIN_PREDICTION_SCORE = 2.0f; - - // The library for gesture detection. - private GestureLibrary mGestureLibrary; - // The long pressing pointer id if coordinate remapping is needed. private int mLongPressingPointerId = -1; @@ -215,10 +188,6 @@ class TouchExplorer implements EventStreamTransformation, AccessibilityGestureDe mDoubleTapSlop = ViewConfiguration.get(context).getScaledDoubleTapSlop(); mHandler = new Handler(context.getMainLooper()); mExitGestureDetectionModeDelayed = new ExitGestureDetectionModeDelayed(); - mGestureLibrary = GestureLibraries.fromRawResource(context, R.raw.accessibility_gestures); - mGestureLibrary.setOrientationStyle(8); - mGestureLibrary.setSequenceType(GestureStore.SEQUENCE_SENSITIVE); - mGestureLibrary.load(); mSendHoverEnterAndMoveDelayed = new SendHoverEnterAndMoveDelayed(); mSendHoverExitDelayed = new SendHoverExitDelayed(); mSendTouchExplorationEndDelayed = new SendAccessibilityEventDelayed( @@ -274,8 +243,7 @@ class TouchExplorer implements EventStreamTransformation, AccessibilityGestureDe sendUpForInjectedDownPointers(event, policyFlags); } break; case STATE_GESTURE_DETECTING: { - // Clear the current stroke. - mStrokeBuffer.clear(); + // No state specific cleanup required. } break; } // Remove all pending callbacks. @@ -321,6 +289,11 @@ class TouchExplorer implements EventStreamTransformation, AccessibilityGestureDe mReceivedPointerTracker.onMotionEvent(rawEvent); + if (mGestureDetector.onMotionEvent(event, policyFlags)) { + // Event was handled by the gesture detector. + return; + } + switch(mCurrentState) { case STATE_TOUCH_EXPLORING: { handleMotionEventStateTouchExploring(event, rawEvent, policyFlags); @@ -389,6 +362,11 @@ class TouchExplorer implements EventStreamTransformation, AccessibilityGestureDe @Override public void onDoubleTapAndHold(MotionEvent event, int policyFlags) { + // Ignore the event if we aren't touch exploring. + if (mCurrentState != STATE_TOUCH_EXPLORING) { + return; + } + // Pointers should not be zero when running this command. if (mReceivedPointerTracker.getLastReceivedEvent().getPointerCount() == 0) { return; @@ -415,10 +393,10 @@ class TouchExplorer implements EventStreamTransformation, AccessibilityGestureDe } @Override - public void onDoubleTap(MotionEvent event, int policyFlags) { - // This should never be called when more than two pointers are down. - if (event.getPointerCount() > 2) { - return; + public boolean onDoubleTap(MotionEvent event, int policyFlags) { + // Ignore the event if we aren't touch exploring. + if (mCurrentState != STATE_TOUCH_EXPLORING) { + return false; } // Remove pending event deliveries. @@ -438,7 +416,9 @@ class TouchExplorer implements EventStreamTransformation, AccessibilityGestureDe Point clickLocation = mTempPoint; final int result = computeClickLocation(clickLocation); if (result == CLICK_LOCATION_NONE) { - return; + // We can't send a click to no location, but the gesture was still + // consumed. + return true; } // Do the click. @@ -456,6 +436,28 @@ class TouchExplorer implements EventStreamTransformation, AccessibilityGestureDe final boolean targetAccessibilityFocus = (result == CLICK_LOCATION_ACCESSIBILITY_FOCUS); sendActionDownAndUp(click_event, policyFlags, targetAccessibilityFocus); click_event.recycle(); + return true; + } + + @Override + public boolean onGesture(int gestureId) { + if (mCurrentState != STATE_GESTURE_DETECTING) { + return false; + } + + mAms.onTouchInteractionEnd(); + + // Announce the end of the gesture recognition. + sendAccessibilityEvent(AccessibilityEvent.TYPE_GESTURE_DETECTION_END); + // Announce the end of a the touch interaction. + sendAccessibilityEvent(AccessibilityEvent.TYPE_TOUCH_INTERACTION_END); + + mAms.onGesture(gestureId); + + mExitGestureDetectionModeDelayed.cancel(); + mCurrentState = STATE_TOUCH_EXPLORING; + + return true; } /** @@ -471,17 +473,10 @@ class TouchExplorer implements EventStreamTransformation, AccessibilityGestureDe mVelocityTracker.addMovement(rawEvent); - mGestureDetector.onMotionEvent(event, policyFlags); - switch (event.getActionMasked()) { case MotionEvent.ACTION_DOWN: { mAms.onTouchInteractionStart(); - // Pre-feed the motion events to the gesture detector since we - // have a distance slop before getting into gesture detection - // mode and not using the points within this slop significantly - // decreases the quality of gesture recognition. - handleMotionEventGestureDetecting(rawEvent, policyFlags); sendAccessibilityEvent(AccessibilityEvent.TYPE_TOUCH_INTERACTION_START); // If we still have not notified the user for the last @@ -528,12 +523,6 @@ class TouchExplorer implements EventStreamTransformation, AccessibilityGestureDe // We have not started sending events since we try to // figure out what the user is doing. if (mSendHoverEnterAndMoveDelayed.isPending()) { - // Pre-feed the motion events to the gesture detector since we - // have a distance slop before getting into gesture detection - // mode and not using the points within this slop significantly - // decreases the quality of gesture recognition. - handleMotionEventGestureDetecting(rawEvent, policyFlags); - // Cache the event until we discern exploration from gesturing. mSendHoverEnterAndMoveDelayed.addEvent(event); @@ -568,6 +557,7 @@ class TouchExplorer implements EventStreamTransformation, AccessibilityGestureDe } else { // We have just decided that the user is touch, // exploring so start sending events. + mGestureDetector.cancelGesture(); mSendHoverEnterAndMoveDelayed.forceSendAndRemove(); mSendHoverExitDelayed.cancel(); sendMotionEvent(event, MotionEvent.ACTION_HOVER_MOVE, @@ -613,9 +603,9 @@ class TouchExplorer implements EventStreamTransformation, AccessibilityGestureDe } // We know that a new state transition is to happen and the - // new state will not be gesture recognition, so clear the - // stashed gesture strokes. - mStrokeBuffer.clear(); + // new state will not be gesture recognition, so cancel + // the gesture. + mGestureDetector.cancelGesture(); if (isDraggingGesture(event)) { // Two pointers moving in the same direction within @@ -655,9 +645,6 @@ class TouchExplorer implements EventStreamTransformation, AccessibilityGestureDe } break; case MotionEvent.ACTION_UP: { mAms.onTouchInteractionEnd(); - // We know that we do not need the pre-fed gesture points are not - // needed anymore since the last pointer just went up. - mStrokeBuffer.clear(); final int pointerId = event.getPointerId(event.getActionIndex()); final int pointerIdBits = (1 << pointerId); @@ -820,24 +807,6 @@ class TouchExplorer implements EventStreamTransformation, AccessibilityGestureDe private void handleMotionEventGestureDetecting(MotionEvent event, int policyFlags) { switch (event.getActionMasked()) { - case MotionEvent.ACTION_DOWN: { - final float x = event.getX(); - final float y = event.getY(); - mPreviousX = x; - mPreviousY = y; - mStrokeBuffer.add(new GesturePoint(x, y, event.getEventTime())); - } break; - case MotionEvent.ACTION_MOVE: { - final float x = event.getX(); - final float y = event.getY(); - final float dX = Math.abs(x - mPreviousX); - final float dY = Math.abs(y - mPreviousY); - if (dX >= TOUCH_TOLERANCE || dY >= TOUCH_TOLERANCE) { - mPreviousX = x; - mPreviousY = y; - mStrokeBuffer.add(new GesturePoint(x, y, event.getEventTime())); - } - } break; case MotionEvent.ACTION_UP: { mAms.onTouchInteractionEnd(); // Announce the end of the gesture recognition. @@ -845,31 +814,6 @@ class TouchExplorer implements EventStreamTransformation, AccessibilityGestureDe // Announce the end of a the touch interaction. sendAccessibilityEvent(AccessibilityEvent.TYPE_TOUCH_INTERACTION_END); - float x = event.getX(); - float y = event.getY(); - mStrokeBuffer.add(new GesturePoint(x, y, event.getEventTime())); - - Gesture gesture = new Gesture(); - gesture.addStroke(new GestureStroke(mStrokeBuffer)); - - ArrayList predictions = mGestureLibrary.recognize(gesture); - if (!predictions.isEmpty()) { - Prediction bestPrediction = predictions.get(0); - if (bestPrediction.score >= MIN_PREDICTION_SCORE) { - if (DEBUG) { - Slog.i(LOG_TAG, "gesture: " + bestPrediction.name + " score: " - + bestPrediction.score); - } - try { - final int gestureId = Integer.parseInt(bestPrediction.name); - mAms.onGesture(gestureId); - } catch (NumberFormatException nfe) { - Slog.w(LOG_TAG, "Non numeric gesture id:" + bestPrediction.name); - } - } - } - - mStrokeBuffer.clear(); mExitGestureDetectionModeDelayed.cancel(); mCurrentState = STATE_TOUCH_EXPLORING; } break;