Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Scribe: Prevent scrolling when starting handwriting

Browse Source 
After startStylusHandwriting is called, there is a short period before an ACTION_CANCEL motion event is dispatched. During this period, ACTION_MOVE motion events may continue to be received by the view root. This may cause a short scroll to occur before handwriting begins.

To prevent this, HandwritingInitiator will suppress ACTION_MOVE events
from a stylus gesture after startStylusHandwriting is called.

Fixes: 223805216
Fixes: 228210945
Bug: 222375667
Test: atest FrameworksCoreTests:android.view.stylus.HandwritingInitiatorTest
Test: atest CtsInputMethodTestCases:android.view.inputmethod.cts.StylusHandwritingTest
Change-Id: If7ed76f55c6a5f010631762733e851b5314334de
This commit is contained in:
Justin Ghan
2022-05-10 12:47:36 -07:00
parent c204e92906
commit 02311ce91c
6 changed files with 206 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

102
core/java/android/view/HandwritingInitiator.java
View File

@@ -60,7 +60,7 @@ public class HandwritingInitiator {
*/
private final long mHandwritingTimeoutInMillis;
private final State mState = new State();
private State mState;
private final HandwritingAreaTracker mHandwritingAreasTracker = new HandwritingAreaTracker();
/** The reference to the View that currently has the input connection. */
@@ -86,17 +86,27 @@ public class HandwritingInitiator {
/**
* Notify the HandwritingInitiator that a new MotionEvent has arrived.
* This method is non-block, and the event passed to this method should be dispatched to the
* View tree as usual. If HandwritingInitiator triggers the handwriting mode, an fabricated
* ACTION_CANCEL event will be sent to the ViewRootImpl.
* @param motionEvent the stylus MotionEvent.
*
* <p>The return value indicates whether the event has been fully handled by the
* HandwritingInitiator and should not be dispatched to the view tree. This will be true for
* ACTION_MOVE events from a stylus gesture after handwriting mode has been initiated, in order
* to suppress other actions such as scrolling.
*
* <p>If HandwritingInitiator triggers the handwriting mode, a fabricated ACTION_CANCEL event
* will be sent to the ViewRootImpl.
*
* @param motionEvent the stylus {@link MotionEvent}
* @return true if the event has been fully handled by the {@link HandwritingInitiator} and
* should not be dispatched to the {@link View} tree, or false if the event should be dispatched
* to the {@link View} tree as usual
*/
@VisibleForTesting
public void onTouchEvent(@NonNull MotionEvent motionEvent) {
public boolean onTouchEvent(@NonNull MotionEvent motionEvent) {
final int maskedAction = motionEvent.getActionMasked();
switch (maskedAction) {
case MotionEvent.ACTION_DOWN:
case MotionEvent.ACTION_POINTER_DOWN:
mState = null;
final int actionIndex = motionEvent.getActionIndex();
final int toolType = motionEvent.getToolType(actionIndex);
// TOOL_TYPE_ERASER is also from stylus. This indicates that the user is holding
@@ -104,42 +114,44 @@ public class HandwritingInitiator {
if (toolType != MotionEvent.TOOL_TYPE_STYLUS
&& toolType != MotionEvent.TOOL_TYPE_ERASER) {
// The motion event is not from a stylus event, ignore it.
return;
return false;
}
mState.mStylusPointerId = motionEvent.getPointerId(actionIndex);
mState.mStylusDownTimeInMillis = motionEvent.getEventTime();
mState.mStylusDownX = motionEvent.getX(actionIndex);
mState.mStylusDownY = motionEvent.getY(actionIndex);
mState.mStylusDownCandidateView = new WeakReference<>(
findBestCandidateView(mState.mStylusDownX, mState.mStylusDownY));
mState.mShouldInitHandwriting = true;
mState.mExceedHandwritingSlop = false;
if (!mImm.isStylusHandwritingAvailable()) {
return false;
}
mState = new State(motionEvent);
break;
case MotionEvent.ACTION_POINTER_UP:
final int pointerId = motionEvent.getPointerId(motionEvent.getActionIndex());
if (pointerId != mState.mStylusPointerId) {
if (mState == null || pointerId != mState.mStylusPointerId) {
// ACTION_POINTER_UP is from another stylus pointer, ignore the event.
return;
return false;
}
// Deliberately fall through.
case MotionEvent.ACTION_CANCEL:
case MotionEvent.ACTION_UP:
// If it's ACTION_CANCEL or ACTION_UP, all the pointers go up. There is no need to
// check whether the stylus we are tracking goes up.
mState.mShouldInitHandwriting = false;
break;
if (mState != null) {
mState.mShouldInitHandwriting = false;
}
return false;
case MotionEvent.ACTION_MOVE:
if (mState == null) {
return false;
}
// Either we've already tried to initiate handwriting, or the ongoing MotionEvent
// sequence is considered to be tap, long-click or other gestures.
if (!mState.mShouldInitHandwriting || mState.mExceedHandwritingSlop) {
return;
return mState.mHasInitiatedHandwriting;
}
final long timeElapsed =
motionEvent.getEventTime() - mState.mStylusDownTimeInMillis;
if (timeElapsed > mHandwritingTimeoutInMillis) {
mState.mShouldInitHandwriting = false;
return;
return mState.mHasInitiatedHandwriting;
}
final int pointerIndex = motionEvent.findPointerIndex(mState.mStylusPointerId);
@@ -147,13 +159,8 @@ public class HandwritingInitiator {
final float y = motionEvent.getY(pointerIndex);
if (largerThanTouchSlop(x, y, mState.mStylusDownX, mState.mStylusDownY)) {
mState.mExceedHandwritingSlop = true;
View candidateView = mState.mStylusDownCandidateView.get();
if (candidateView == null || !candidateView.isAttachedToWindow()) {
// If there was no candidate view found in the stylus down event, or if that
// candidate view is no longer attached, search again for a candidate view.
candidateView = findBestCandidateView(mState.mStylusDownX,
mState.mStylusDownY);
}
View candidateView =
findBestCandidateView(mState.mStylusDownX, mState.mStylusDownY);
if (candidateView != null) {
if (candidateView == getConnectedView()) {
startHandwriting(candidateView);
@@ -162,7 +169,9 @@ public class HandwritingInitiator {
}
}
}
return mState.mHasInitiatedHandwriting;
}
return false;
}
@Nullable
@@ -195,7 +204,7 @@ public class HandwritingInitiator {
} else {
mConnectedView = new WeakReference<>(view);
mConnectionCount = 1;
if (mState.mShouldInitHandwriting) {
if (mState != null && mState.mShouldInitHandwriting) {
tryStartHandwriting();
}
}
@@ -259,6 +268,7 @@ public class HandwritingInitiator {
@VisibleForTesting
public void startHandwriting(@NonNull View view) {
mImm.startStylusHandwriting(view);
mState.mHasInitiatedHandwriting = true;
mState.mShouldInitHandwriting = false;
}
@@ -438,28 +448,38 @@ public class HandwritingInitiator {
* b) If the MotionEvent sequence is considered to be tap, long-click or other gestures.
* This boolean will be set to false, and it won't request to start handwriting.
*/
private boolean mShouldInitHandwriting = false;
private boolean mShouldInitHandwriting;
/**
* Whether handwriting mode has already been initiated for the current MotionEvent sequence.
*/
private boolean mHasInitiatedHandwriting;
/**
* Whether the current ongoing stylus MotionEvent sequence already exceeds the
* handwriting slop.
* It's used for the case where the stylus exceeds handwriting slop before the target View
* built InputConnection.
*/
private boolean mExceedHandwritingSlop = false;
private boolean mExceedHandwritingSlop;
/** The pointer id of the stylus pointer that is being tracked. */
private int mStylusPointerId = -1;
private final int mStylusPointerId;
/** The time stamp when the stylus pointer goes down. */
private long mStylusDownTimeInMillis = -1;
private final long mStylusDownTimeInMillis;
/** The initial location where the stylus pointer goes down. */
private float mStylusDownX = Float.NaN;
private float mStylusDownY = Float.NaN;
/**
* The best candidate view to initialize handwriting mode based on the initial location
* where the stylus pointer goes down, or null if the location was not within any candidate
* view's handwriting area.
*/
private WeakReference<View> mStylusDownCandidateView = new WeakReference<>(null);
private final float mStylusDownX;
private final float mStylusDownY;
private State(MotionEvent motionEvent) {
final int actionIndex = motionEvent.getActionIndex();
mStylusPointerId = motionEvent.getPointerId(actionIndex);
mStylusDownTimeInMillis = motionEvent.getEventTime();
mStylusDownX = motionEvent.getX(actionIndex);
mStylusDownY = motionEvent.getY(actionIndex);
mShouldInitHandwriting = true;
mHasInitiatedHandwriting = false;
mExceedHandwritingSlop = false;
}
}
/** The helper method to check if the given view is still active for handwriting. */

6
core/java/android/view/ViewRootImpl.java
View File

@@ -6545,11 +6545,13 @@ public final class ViewRootImpl implements ViewParent,
private int processPointerEvent(QueuedInputEvent q) {
final MotionEvent event = (MotionEvent)q.mEvent;
mHandwritingInitiator.onTouchEvent(event);
boolean handled = mHandwritingInitiator.onTouchEvent(event);
mAttachInfo.mUnbufferedDispatchRequested = false;
mAttachInfo.mHandlingPointerEvent = true;
boolean handled = mView.dispatchPointerEvent(event);
// If the event was fully handled by the handwriting initiator, then don't dispatch it
// to the view tree.
handled = handled || mView.dispatchPointerEvent(event);
maybeUpdatePointerIcon(event);
maybeUpdateTooltip(event);
mAttachInfo.mHandlingPointerEvent = false;

11
core/tests/coretests/AndroidManifest.xml
View File

@@ -1677,6 +1677,17 @@
android:resizeableActivity="true"
android:exported="true">
</activity>
<service android:name="android.view.stylus.HandwritingImeService"
android:label="Handwriting IME"
android:permission="android.permission.BIND_INPUT_METHOD"
android:exported="true">
<intent-filter>
<action android:name="android.view.InputMethod"/>
</intent-filter>
<meta-data android:name="android.view.im"
android:resource="@xml/ime_meta_handwriting"/>
</service>
</application>
<instrumentation android:name="androidx.test.runner.AndroidJUnitRunner"

21
core/tests/coretests/res/xml/ime_meta_handwriting.xml Normal file
View File

@@ -0,0 +1,21 @@
<?xml version="1.0" encoding="utf-8"?>
<!--
~ Copyright (C) 2022 The Android Open Source Project
~
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
~ You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<input-method
xmlns:android="http://schemas.android.com/apk/res/android"
android:settingsActivity="com.android.inputmethod.latin.settings.SettingsActivity"
android:supportsStylusHandwriting="true"/>

32
core/tests/coretests/src/android/view/stylus/HandwritingImeService.java Normal file
View File

@@ -0,0 +1,32 @@
/*
* Copyright (C) 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package android.view.stylus;
import android.content.ComponentName;
import android.inputmethodservice.InputMethodService;
public class HandwritingImeService extends InputMethodService {
private static final String PACKAGE_NAME = "com.android.frameworks.coretests";
private static ComponentName getComponentName() {
return new ComponentName(PACKAGE_NAME, HandwritingImeService.class.getName());
}
static String getImeId() {
return getComponentName().flattenToShortString();
}
}

91
core/tests/coretests/src/android/view/stylus/HandwritingInitiatorTest.java
View File

@@ -43,10 +43,15 @@ import androidx.test.ext.junit.runners.AndroidJUnit4;
import androidx.test.filters.SmallTest;
import androidx.test.platform.app.InstrumentationRegistry;
import com.android.compatibility.common.util.PollingCheck;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import java.util.concurrent.TimeUnit;
/**
* Tests for {@link HandwritingInitiator}
*
@@ -60,7 +65,7 @@ public class HandwritingInitiatorTest {
private static final long TIMEOUT = ViewConfiguration.getLongPressTimeout();
private static final int HW_BOUNDS_OFFSETS_LEFT_PX = 10;
private static final int HW_BOUNDS_OFFSETS_TOP_PX = 20;
private static final int HW_BOUNDS_OFFSETS_RIGHT_PX = 30;
private static final int HW_BOUNDS_OFFSETS_RIGHT_PX = 30;
private static final int HW_BOUNDS_OFFSETS_BOTTOM_PX = 40;
private int mHandwritingSlop = 4;
@@ -71,9 +76,17 @@ public class HandwritingInitiatorTest {
private Context mContext;
@Before
public void setup() {
final Instrumentation mInstrumentation = InstrumentationRegistry.getInstrumentation();
mContext = mInstrumentation.getTargetContext();
public void setup() throws Exception {
final Instrumentation instrumentation = InstrumentationRegistry.getInstrumentation();
mContext = instrumentation.getTargetContext();
String imeId = HandwritingImeService.getImeId();
instrumentation.getUiAutomation().executeShellCommand("ime enable " + imeId);
instrumentation.getUiAutomation().executeShellCommand("ime set " + imeId);
PollingCheck.check("Check that stylus handwriting is available",
TimeUnit.SECONDS.toMillis(10),
() -> mContext.getSystemService(InputMethodManager.class)
.isStylusHandwritingAvailable());
final ViewConfiguration viewConfiguration = ViewConfiguration.get(mContext);
mHandwritingSlop = viewConfiguration.getScaledHandwritingSlop();
@@ -90,22 +103,32 @@ public class HandwritingInitiatorTest {
mHandwritingInitiator.updateHandwritingAreasForView(mTestView);
}
@After
public void tearDown() throws Exception {
InstrumentationRegistry.getInstrumentation().getUiAutomation()
.executeShellCommand("ime reset");
}
@Test
public void onTouchEvent_startHandwriting_when_stylusMoveOnce_withinHWArea() {
mHandwritingInitiator.onInputConnectionCreated(mTestView);
final int x1 = (sHwArea.left + sHwArea.right) / 2;
final int y1 = (sHwArea.top + sHwArea.bottom) / 2;
MotionEvent stylusEvent1 = createStylusEvent(ACTION_DOWN, x1, y1, 0);
mHandwritingInitiator.onTouchEvent(stylusEvent1);
boolean onTouchEventResult1 = mHandwritingInitiator.onTouchEvent(stylusEvent1);
final int x2 = x1 + mHandwritingSlop * 2;
final int y2 = y1;
MotionEvent stylusEvent2 = createStylusEvent(ACTION_MOVE, x2, y2, 0);
mHandwritingInitiator.onTouchEvent(stylusEvent2);
boolean onTouchEventResult2 = mHandwritingInitiator.onTouchEvent(stylusEvent2);
// Stylus movement within HandwritingArea should trigger IMM.startHandwriting once.
verify(mHandwritingInitiator, times(1)).startHandwriting(mTestView);
assertThat(onTouchEventResult1).isFalse();
// After IMM.startHandwriting is triggered, onTouchEvent should return true for ACTION_MOVE
// events so that the events are not dispatched to the view tree.
assertThat(onTouchEventResult2).isTrue();
}
@Test
@@ -114,24 +137,38 @@ public class HandwritingInitiatorTest {
final int x1 = (sHwArea.left + sHwArea.right) / 2;
final int y1 = (sHwArea.top + sHwArea.bottom) / 2;
MotionEvent stylusEvent1 = createStylusEvent(ACTION_DOWN, x1, y1, 0);
mHandwritingInitiator.onTouchEvent(stylusEvent1);
boolean onTouchEventResult1 = mHandwritingInitiator.onTouchEvent(stylusEvent1);
final int x2 = x1 + mHandwritingSlop * 2;
final int x2 = x1 + mHandwritingSlop / 2;
final int y2 = y1;
MotionEvent stylusEvent2 = createStylusEvent(ACTION_MOVE, x2, y2, 0);
mHandwritingInitiator.onTouchEvent(stylusEvent2);
boolean onTouchEventResult2 = mHandwritingInitiator.onTouchEvent(stylusEvent2);
final int x3 = x2 + mHandwritingSlop * 2;
final int y3 = y2;
final int y3 = y1;
MotionEvent stylusEvent3 = createStylusEvent(ACTION_MOVE, x3, y3, 0);
mHandwritingInitiator.onTouchEvent(stylusEvent3);
boolean onTouchEventResult3 = mHandwritingInitiator.onTouchEvent(stylusEvent3);
MotionEvent stylusEvent4 = createStylusEvent(ACTION_UP, x2, y2, 0);
mHandwritingInitiator.onTouchEvent(stylusEvent4);
final int x4 = x3 + mHandwritingSlop * 2;
final int y4 = y1;
MotionEvent stylusEvent4 = createStylusEvent(ACTION_MOVE, x4, y4, 0);
boolean onTouchEventResult4 = mHandwritingInitiator.onTouchEvent(stylusEvent4);
MotionEvent stylusEvent5 = createStylusEvent(ACTION_UP, x4, y4, 0);
boolean onTouchEventResult5 = mHandwritingInitiator.onTouchEvent(stylusEvent5);
// It only calls startHandwriting once for each ACTION_DOWN.
verify(mHandwritingInitiator, times(1)).startHandwriting(mTestView);
assertThat(onTouchEventResult1).isFalse();
// stylusEvent2 does not trigger IMM.startHandwriting since the touch slop distance has not
// been exceeded. onTouchEvent should return false so that the event is dispatched to the
// view tree.
assertThat(onTouchEventResult2).isFalse();
// After IMM.startHandwriting is triggered by stylusEvent3, onTouchEvent should return true
// for ACTION_MOVE events so that the events are not dispatched to the view tree.
assertThat(onTouchEventResult3).isTrue();
assertThat(onTouchEventResult4).isTrue();
assertThat(onTouchEventResult5).isFalse();
}
@Test
@@ -188,6 +225,32 @@ public class HandwritingInitiatorTest {
verify(mHandwritingInitiator, times(1)).startHandwriting(mTestView);
}
@Test
public void onTouchEvent_notStartHandwriting_whenHandwritingNotAvailable() throws Exception {
InstrumentationRegistry.getInstrumentation().getUiAutomation()
.executeShellCommand("ime reset");
PollingCheck.check("Check that stylus handwriting is unavailable",
TimeUnit.SECONDS.toMillis(10),
() -> !mContext.getSystemService(InputMethodManager.class)
.isStylusHandwritingAvailable());
mHandwritingInitiator.onInputConnectionCreated(mTestView);
final int x1 = (sHwArea.left + sHwArea.right) / 2;
final int y1 = (sHwArea.top + sHwArea.bottom) / 2;
MotionEvent stylusEvent1 = createStylusEvent(ACTION_DOWN, x1, y1, 0);
mHandwritingInitiator.onTouchEvent(stylusEvent1);
final int x2 = x1 + mHandwritingSlop * 2;
final int y2 = y1;
MotionEvent stylusEvent2 = createStylusEvent(ACTION_MOVE, x2, y2, 0);
mHandwritingInitiator.onTouchEvent(stylusEvent2);
// Stylus movement within HandwritingArea should not trigger IMM.startHandwriting since
// the current IME doesn't support handwriting.
verify(mHandwritingInitiator, never()).startHandwriting(mTestView);
}
@Test
public void onTouchEvent_notStartHandwriting_when_stylusTap_withinHWArea() {
mHandwritingInitiator.onInputConnectionCreated(mTestView);