From 77276b60851a158ad3e142cb3b091d57ae5ceffb Mon Sep 17 00:00:00 2001 From: Svetoslav Ganov Date: Fri, 14 Sep 2012 10:23:00 -0700 Subject: [PATCH] Adding accessibility events for touch and gesture detection states. 1. Currently the system fires accessibility events to announce the start and end of a touch exploration gesture. However, such a gesture starts after we have decided that the user is not performing a gesture which is achieved by measuring speed of movement during a threshold distance. This allows an accessibility service to provide some feedback to the user so he knows that he is touch exploring. This change adds event types for the first and last touches of the user. Note that the first touch does not conincide with the start of a touch exploration gesture since we need a time or distance to pass before we know whether the user explores or gestures. However, it is very useful for an accessibility service to know when the user starts to interact with the touch screen so it can turn the speech off, to name one compelling use case. This change also provides event types for the start and end of gesture detection. If the user has moved over the threshold with a speed greater than X, then the system detects gestures. It is useful for an accessibility service to know the begin and end of gesture detection so it can provide given feedback type for such a gesture, say it may produce haptic feedback or sound that differs for the one for touch exploration. The main benefit of announcing these new events is that an accessibility service can provide feedback for each touch state allowing the user to always know what he is doing. bug:7166935 Change-Id: I26270d774cc059cb921d6a4254bc0aab0530c1dd --- api/current.txt | 4 + .../accessibility/AccessibilityEvent.java | 98 ++++++++++--- .../AccessibilityManagerService.java | 36 ----- .../server/accessibility/TouchExplorer.java | 135 +++++++++++++++--- 4 files changed, 199 insertions(+), 74 deletions(-) diff --git a/api/current.txt b/api/current.txt index cda6be188adc9..2b512c9b15f38 100644 --- a/api/current.txt +++ b/api/current.txt @@ -26054,9 +26054,13 @@ package android.view.accessibility { field public static final deprecated int MAX_TEXT_LENGTH = 500; // 0x1f4 field public static final int TYPES_ALL_MASK = -1; // 0xffffffff field public static final int TYPE_ANNOUNCEMENT = 16384; // 0x4000 + field public static final int TYPE_GESTURE_DETECTION_END = 524288; // 0x80000 + field public static final int TYPE_GESTURE_DETECTION_START = 262144; // 0x40000 field public static final int TYPE_NOTIFICATION_STATE_CHANGED = 64; // 0x40 field public static final int TYPE_TOUCH_EXPLORATION_GESTURE_END = 1024; // 0x400 field public static final int TYPE_TOUCH_EXPLORATION_GESTURE_START = 512; // 0x200 + field public static final int TYPE_TOUCH_INTERACTION_END = 2097152; // 0x200000 + field public static final int TYPE_TOUCH_INTERACTION_START = 1048576; // 0x100000 field public static final int TYPE_VIEW_ACCESSIBILITY_FOCUSED = 32768; // 0x8000 field public static final int TYPE_VIEW_ACCESSIBILITY_FOCUS_CLEARED = 65536; // 0x10000 field public static final int TYPE_VIEW_CLICKED = 1; // 0x1 diff --git a/core/java/android/view/accessibility/AccessibilityEvent.java b/core/java/android/view/accessibility/AccessibilityEvent.java index 1a2a194f8211d..15009053571fd 100644 --- a/core/java/android/view/accessibility/AccessibilityEvent.java +++ b/core/java/android/view/accessibility/AccessibilityEvent.java @@ -424,6 +424,28 @@ import java.util.List; * *

*

+ * Touch interaction start - represents the event of starting a touch + * interaction, which is the user starts touching the screen.
+ * Type: {@link #TYPE_TOUCH_INTERACTION_START}
+ * Properties:
+ *

+ * Note: This event is fired only by the system and is not passed to the + * view tree to be populated.
+ *

+ *

+ * Touch interaction end - represents the event of ending a touch + * interaction, which is the user stops touching the screen.
+ * Type: {@link #TYPE_TOUCH_INTERACTION_END}
+ * Properties:
+ *

+ * Note: This event is fired only by the system and is not passed to the + * view tree to be populated.
+ *

+ *

* Touch exploration gesture start - represents the event of starting a touch * exploring gesture.
* Type: {@link #TYPE_TOUCH_EXPLORATION_GESTURE_START}
@@ -431,15 +453,8 @@ import java.util.List; *

- * Note: This event type is not dispatched to descendants though - * {@link android.view.View#dispatchPopulateAccessibilityEvent(AccessibilityEvent) - * View.dispatchPopulateAccessibilityEvent(AccessibilityEvent)}, hence the event - * source {@link android.view.View} and the sub-tree rooted at it will not receive - * calls to {@link android.view.View#onPopulateAccessibilityEvent(AccessibilityEvent) - * View.onPopulateAccessibilityEvent(AccessibilityEvent)}. The preferred way to add - * text content to such events is by setting the - * {@link android.R.styleable#View_contentDescription contentDescription} of the source - * view.
+ * Note: This event is fired only by the system and is not passed to the + * view tree to be populated.
*

*

* Touch exploration gesture end - represents the event of ending a touch @@ -449,15 +464,30 @@ import java.util.List; *

- * Note: This event type is not dispatched to descendants though - * {@link android.view.View#dispatchPopulateAccessibilityEvent(AccessibilityEvent) - * View.dispatchPopulateAccessibilityEvent(AccessibilityEvent)}, hence the event - * source {@link android.view.View} and the sub-tree rooted at it will not receive - * calls to {@link android.view.View#onPopulateAccessibilityEvent(AccessibilityEvent) - * View.onPopulateAccessibilityEvent(AccessibilityEvent)}. The preferred way to add - * text content to such events is by setting the - * {@link android.R.styleable#View_contentDescription contentDescription} of the source - * view.
+ * Note: This event is fired only by the system and is not passed to the + * view tree to be populated.
+ *

+ *

+ * Touch gesture detection start - represents the event of starting a user + * gesture detection.
+ * Type: {@link #TYPE_GESTURE_DETECTION_START}
+ * Properties:
+ *

+ * Note: This event is fired only by the system and is not passed to the + * view tree to be populated.
+ *

+ *

+ * Touch gesture detection end - represents the event of ending a user + * gesture detection.
+ * Type: {@link #TYPE_GESTURE_DETECTION_END}
+ * Properties:
+ *

+ * Note: This event is fired only by the system and is not passed to the + * view tree to be populated.
*

*

* MISCELLANEOUS TYPES
@@ -609,6 +639,26 @@ public final class AccessibilityEvent extends AccessibilityRecord implements Par */ public static final int TYPE_VIEW_TEXT_TRAVERSED_AT_MOVEMENT_GRANULARITY = 0x00020000; + /** + * Represents the event of beginning gesture detection. + */ + public static final int TYPE_GESTURE_DETECTION_START = 0x00040000; + + /** + * Represents the event of ending gesture detection. + */ + public static final int TYPE_GESTURE_DETECTION_END = 0x00080000; + + /** + * Represents the event of the user starting to touch the screen. + */ + public static final int TYPE_TOUCH_INTERACTION_START = 0x00100000; + + /** + * Represents the event of the user ending to touch the screen. + */ + public static final int TYPE_TOUCH_INTERACTION_END = 0x00200000; + /** * Mask for {@link AccessibilityEvent} all types. * @@ -628,6 +678,10 @@ public final class AccessibilityEvent extends AccessibilityRecord implements Par * @see #TYPE_VIEW_TEXT_SELECTION_CHANGED * @see #TYPE_ANNOUNCEMENT * @see #TYPE_VIEW_TEXT_TRAVERSED_AT_MOVEMENT_GRANULARITY + * @see #TYPE_GESTURE_DETECTION_START + * @see #TYPE_GESTURE_DETECTION_END + * @see #TYPE_TOUCH_INTERACTION_START + * @see #TYPE_TOUCH_INTERACTION_END */ public static final int TYPES_ALL_MASK = 0xFFFFFFFF; @@ -1120,6 +1174,14 @@ public final class AccessibilityEvent extends AccessibilityRecord implements Par return "TYPE_VIEW_ACCESSIBILITY_FOCUS_CLEARED"; case TYPE_VIEW_TEXT_TRAVERSED_AT_MOVEMENT_GRANULARITY: return "TYPE_CURRENT_AT_GRANULARITY_MOVEMENT_CHANGED"; + case TYPE_GESTURE_DETECTION_START: + return "TYPE_GESTURE_DETECTION_START"; + case TYPE_GESTURE_DETECTION_END: + return "TYPE_GESTURE_DETECTION_END"; + case TYPE_TOUCH_INTERACTION_START: + return "TYPE_TOUCH_INTERACTION_START"; + case TYPE_TOUCH_INTERACTION_END: + return "TYPE_TOUCH_INTERACTION_END"; default: return null; } diff --git a/services/java/com/android/server/accessibility/AccessibilityManagerService.java b/services/java/com/android/server/accessibility/AccessibilityManagerService.java index 99ec1d2f3060b..e7f3599a56ca2 100644 --- a/services/java/com/android/server/accessibility/AccessibilityManagerService.java +++ b/services/java/com/android/server/accessibility/AccessibilityManagerService.java @@ -173,10 +173,6 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub { private Service mQueryBridge; - private boolean mTouchExplorationGestureEnded; - - private boolean mTouchExplorationGestureStarted; - private AlertDialog mEnableTouchExplorationDialog; /** @@ -400,18 +396,6 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub { } public boolean sendAccessibilityEvent(AccessibilityEvent event) { - final int eventType = event.getEventType(); - - // The event for gesture start should be strictly before the - // first hover enter event for the gesture. - if (eventType == AccessibilityEvent.TYPE_VIEW_HOVER_ENTER - && mTouchExplorationGestureStarted) { - mTouchExplorationGestureStarted = false; - AccessibilityEvent gestureStartEvent = AccessibilityEvent.obtain( - AccessibilityEvent.TYPE_TOUCH_EXPLORATION_GESTURE_START); - sendAccessibilityEvent(gestureStartEvent); - } - synchronized (mLock) { if (mSecurityPolicy.canDispatchAccessibilityEvent(event)) { mSecurityPolicy.updateActiveWindowAndEventSourceLocked(event); @@ -421,22 +405,10 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub { if (mHasInputFilter && mInputFilter != null) { mMainHandler.obtainMessage(MSG_SEND_ACCESSIBILITY_EVENT_TO_INPUT_FILTER, AccessibilityEvent.obtain(event)).sendToTarget(); - } event.recycle(); mHandledFeedbackTypes = 0; } - - // The event for gesture end should be strictly after the - // last hover exit event for the gesture. - if (eventType == AccessibilityEvent.TYPE_VIEW_HOVER_EXIT - && mTouchExplorationGestureEnded) { - mTouchExplorationGestureEnded = false; - AccessibilityEvent gestureEndEvent = AccessibilityEvent.obtain( - AccessibilityEvent.TYPE_TOUCH_EXPLORATION_GESTURE_END); - sendAccessibilityEvent(gestureEndEvent); - } - return (OWN_PROCESS_ID != Binder.getCallingPid()); } @@ -628,14 +600,6 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub { return mQueryBridge; } - public void touchExplorationGestureEnded() { - mTouchExplorationGestureEnded = true; - } - - public void touchExplorationGestureStarted() { - mTouchExplorationGestureStarted = true; - } - private boolean notifyGestureLocked(int gestureId, boolean isDefault) { // TODO: Now we are giving the gestures to the last enabled // service that can handle them which is the last one diff --git a/services/java/com/android/server/accessibility/TouchExplorer.java b/services/java/com/android/server/accessibility/TouchExplorer.java index 9e4f33ea81b4b..d620624c7d713 100644 --- a/services/java/com/android/server/accessibility/TouchExplorer.java +++ b/services/java/com/android/server/accessibility/TouchExplorer.java @@ -25,6 +25,7 @@ import android.gesture.GestureStore; import android.gesture.GestureStroke; import android.gesture.Prediction; import android.graphics.Rect; +import android.os.Build; import android.os.Handler; import android.os.SystemClock; import android.util.Slog; @@ -35,6 +36,7 @@ import android.view.VelocityTracker; import android.view.ViewConfiguration; import android.view.WindowManagerPolicy; import android.view.accessibility.AccessibilityEvent; +import android.view.accessibility.AccessibilityManager; import com.android.internal.R; @@ -168,6 +170,9 @@ class TouchExplorer implements EventStreamTransformation { // Temporary rectangle to avoid instantiation. private final Rect mTempRect = new Rect(); + // Context in which this explorer operates. + private final Context mContext; + // The X of the previous event. private float mPreviousX; @@ -198,6 +203,12 @@ class TouchExplorer implements EventStreamTransformation { // The id of the last touch explored window. private int mLastTouchedWindowId; + // Whether touch exploration gesture has ended. + private boolean mTouchExplorationGestureEnded; + + // Whether touch interaction has ended. + private boolean mTouchInteractionEnded; + /** * Creates a new instance. * @@ -205,11 +216,12 @@ class TouchExplorer implements EventStreamTransformation { * @param context A context handle for accessing resources. */ public TouchExplorer(Context context, AccessibilityManagerService service) { + mContext = context; mAms = service; mReceivedPointerTracker = new ReceivedPointerTracker(context); mInjectedPointerTracker = new InjectedPointerTracker(); mTapTimeout = ViewConfiguration.getTapTimeout(); - mDetermineUserIntentTimeout = (int) (mTapTimeout * 1.5f); + mDetermineUserIntentTimeout = ViewConfiguration.getDoubleTapTimeout(); mDoubleTapTimeout = ViewConfiguration.getDoubleTapTimeout(); mTouchSlop = ViewConfiguration.get(context).getScaledTouchSlop(); mDoubleTapSlop = ViewConfiguration.get(context).getScaledDoubleTapSlop(); @@ -317,9 +329,32 @@ class TouchExplorer implements EventStreamTransformation { } public void onAccessibilityEvent(AccessibilityEvent event) { + final int eventType = event.getEventType(); + + // The event for gesture end should be strictly after the + // last hover exit event. + if (mTouchExplorationGestureEnded) { + switch (eventType) { + case AccessibilityEvent.TYPE_VIEW_HOVER_EXIT: { + mTouchExplorationGestureEnded = false; + sendAccessibilityEvent(AccessibilityEvent.TYPE_TOUCH_EXPLORATION_GESTURE_END); + } break; + } + } + + // The event for touch interaction end should be strictly after the + // last hover exit and the touch exploration gesture end events. + if (mTouchInteractionEnded) { + switch (eventType) { + case AccessibilityEvent.TYPE_VIEW_HOVER_EXIT: { + mTouchInteractionEnded = false; + sendAccessibilityEvent(AccessibilityEvent.TYPE_TOUCH_INTERACTION_END); + } break; + } + } + // If a new window opens or the accessibility focus moves we no longer // want to click/long press on the last touch explored location. - final int eventType = event.getEventType(); switch (eventType) { case AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED: case AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED: { @@ -358,6 +393,15 @@ class TouchExplorer implements EventStreamTransformation { switch (event.getActionMasked()) { case MotionEvent.ACTION_DOWN: + // The delayed enter not delivered implies that we have delivered + // TYPE_TOUCH_INTERACTION_START and not TYPE_TOUCH_INTERACTION_END, + // therefore we need to deliver the interaction end event here. + if (mSendHoverEnterDelayed.isPending()) { + sendAccessibilityEvent(AccessibilityEvent.TYPE_TOUCH_INTERACTION_END); + } + // Announce the start of a new touch interaction. + sendAccessibilityEvent( + AccessibilityEvent.TYPE_TOUCH_INTERACTION_START); // Pre-feed the motion events to the gesture detector since we // have a distance slop before getting into gesture detection // mode and not using the points within this slop significantly @@ -396,7 +440,7 @@ class TouchExplorer implements EventStreamTransformation { // to detect what the user is trying to do. final int pointerId = receivedTracker.getPrimaryActivePointerId(); final int pointerIdBits = (1 << pointerId); - mSendHoverEnterDelayed.post(event, pointerIdBits, policyFlags); + mSendHoverEnterDelayed.post(event, true, pointerIdBits, policyFlags); } break; default: { /* do nothing - let the code for ACTION_MOVE decide what to do */ @@ -443,6 +487,10 @@ class TouchExplorer implements EventStreamTransformation { mSendHoverExitDelayed.remove(); mPerformLongPressDelayed.remove(); mExitGestureDetectionModeDelayed.post(); + // Send accessibility event to announce the start + // of gesture recognition. + sendAccessibilityEvent( + AccessibilityEvent.TYPE_GESTURE_DETECTION_START); } else { // We have just decided that the user is touch, // exploring so start sending events. @@ -551,7 +599,8 @@ class TouchExplorer implements EventStreamTransformation { // If we have not delivered the enter schedule exit. if (mSendHoverEnterDelayed.isPending()) { - mSendHoverExitDelayed.post(event, pointerIdBits, policyFlags); + mSendHoverEnterDelayed.mTouchExplorationInProgress = false; + mSendHoverExitDelayed.post(event, false, pointerIdBits, policyFlags); } else { // The user is touch exploring so we send events for end. sendExitEventsIfNeeded(policyFlags); @@ -656,6 +705,9 @@ class TouchExplorer implements EventStreamTransformation { } } break; case MotionEvent.ACTION_UP: { + // Announce the end of a new touch interaction. + sendAccessibilityEvent( + AccessibilityEvent.TYPE_TOUCH_INTERACTION_END); mCurrentState = STATE_TOUCH_EXPLORING; } break; case MotionEvent.ACTION_CANCEL: { @@ -687,6 +739,10 @@ class TouchExplorer implements EventStreamTransformation { } } break; case MotionEvent.ACTION_UP: + // Announce the end of a new touch interaction. + sendAccessibilityEvent( + AccessibilityEvent.TYPE_TOUCH_INTERACTION_END); + //$FALL-THROUGH$ case MotionEvent.ACTION_POINTER_UP: { mLongPressingPointerId = -1; mLongPressingPointerDeltaX = 0; @@ -725,6 +781,13 @@ class TouchExplorer implements EventStreamTransformation { } } break; case MotionEvent.ACTION_UP: { + // Announce the end of gesture recognition. + sendAccessibilityEvent( + AccessibilityEvent.TYPE_GESTURE_DETECTION_END); + // Announce the end of a new touch interaction. + sendAccessibilityEvent( + AccessibilityEvent.TYPE_TOUCH_INTERACTION_END); + float x = event.getX(); float y = event.getY(); mStrokeBuffer.add(new GesturePoint(x, y, event.getEventTime())); @@ -759,6 +822,19 @@ class TouchExplorer implements EventStreamTransformation { } } + /** + * Sends an accessibility event of the given type. + * + * @param type The event type. + */ + private void sendAccessibilityEvent(int type) { + AccessibilityManager accessibilityManager = AccessibilityManager.getInstance(mContext); + if (accessibilityManager.isEnabled()) { + AccessibilityEvent event = AccessibilityEvent.obtain(type); + accessibilityManager.sendAccessibilityEvent(event); + } + } + /** * Sends down events to the view hierarchy for all active pointers which are * not already being delivered i.e. pointers that are not yet injected. @@ -807,7 +883,8 @@ class TouchExplorer implements EventStreamTransformation { MotionEvent event = mInjectedPointerTracker.getLastInjectedHoverEvent(); if (event != null && event.getActionMasked() != MotionEvent.ACTION_HOVER_EXIT) { final int pointerIdBits = event.getPointerIdBits(); - mAms.touchExplorationGestureEnded(); + mTouchExplorationGestureEnded = true; + mTouchInteractionEnded = true; sendMotionEvent(event, MotionEvent.ACTION_HOVER_EXIT, pointerIdBits, policyFlags); } } @@ -822,7 +899,6 @@ class TouchExplorer implements EventStreamTransformation { MotionEvent event = mInjectedPointerTracker.getLastInjectedHoverEvent(); if (event != null && event.getActionMasked() == MotionEvent.ACTION_HOVER_EXIT) { final int pointerIdBits = event.getPointerIdBits(); - mAms.touchExplorationGestureStarted(); sendMotionEvent(event, MotionEvent.ACTION_HOVER_ENTER, pointerIdBits, policyFlags); } } @@ -1080,16 +1156,24 @@ class TouchExplorer implements EventStreamTransformation { return; } + if (Build.IS_DEBUGGABLE) { + if (mSendHoverEnterDelayed.isPending()) { + throw new IllegalStateException("mSendHoverEnterDelayed must not be pending."); + } + if (mSendHoverExitDelayed.isPending()) { + throw new IllegalStateException("mSendHoverExitDelayed must not be pending."); + } + if (!mPerformLongPressDelayed.isPending()) { + throw new IllegalStateException( + "mPerformLongPressDelayed must not be pending."); + } + } + // Remove pending event deliveries. - mSendHoverEnterDelayed.remove(); - mSendHoverExitDelayed.remove(); mPerformLongPressDelayed.remove(); - // This is a tap so do not send hover events since - // this events will result in firing the corresponding - // accessibility events confusing the user about what - // is actually clicked. - sendExitEventsIfNeeded(policyFlags); + // The touch interaction has ended since we will send a click. + sendAccessibilityEvent(AccessibilityEvent.TYPE_TOUCH_INTERACTION_END); int clickLocationX; int clickLocationY; @@ -1257,13 +1341,13 @@ class TouchExplorer implements EventStreamTransformation { } public void remove() { - if (isPenidng()) { + if (isPending()) { mHandler.removeCallbacks(this); clear(); } } - private boolean isPenidng() { + public boolean isPending() { return (mEvent != null); } @@ -1326,7 +1410,7 @@ class TouchExplorer implements EventStreamTransformation { } private void clear() { - if (!isPenidng()) { + if (!isPending()) { return; } mEvent.recycle(); @@ -1347,15 +1431,18 @@ class TouchExplorer implements EventStreamTransformation { private MotionEvent mPrototype; private int mPointerIdBits; private int mPolicyFlags; + private boolean mTouchExplorationInProgress; public SendHoverDelayed(int hoverAction, boolean gestureStarted) { mHoverAction = hoverAction; mGestureStarted = gestureStarted; } - public void post(MotionEvent prototype, int pointerIdBits, int policyFlags) { + public void post(MotionEvent prototype, boolean touchExplorationInProgress, + int pointerIdBits, int policyFlags) { remove(); mPrototype = MotionEvent.obtain(prototype); + mTouchExplorationInProgress = touchExplorationInProgress; mPointerIdBits = pointerIdBits; mPolicyFlags = policyFlags; mHandler.postDelayed(this, mDetermineUserIntentTimeout); @@ -1392,6 +1479,7 @@ class TouchExplorer implements EventStreamTransformation { mPrototype = null; mPointerIdBits = -1; mPolicyFlags = 0; + mTouchExplorationInProgress = false; } public void forceSendAndRemove() { @@ -1408,10 +1496,17 @@ class TouchExplorer implements EventStreamTransformation { Slog.d(LOG_TAG_SEND_HOVER_DELAYED, mGestureStarted ? "touchExplorationGestureStarted" : "touchExplorationGestureEnded"); } - if (mGestureStarted) { - mAms.touchExplorationGestureStarted(); + if (mTouchExplorationInProgress) { + if (mGestureStarted) { + sendAccessibilityEvent(AccessibilityEvent.TYPE_TOUCH_EXPLORATION_GESTURE_START); + } else { + mTouchExplorationGestureEnded = true; + mTouchInteractionEnded = true; + } } else { - mAms.touchExplorationGestureEnded(); + if (!mGestureStarted) { + mTouchInteractionEnded = true; + } } sendMotionEvent(mPrototype, mHoverAction, mPointerIdBits, mPolicyFlags); clear();