From e5f499176d316d4f7e0563497e37efb585110f23 Mon Sep 17 00:00:00 2001 From: Taran Singh Date: Wed, 20 Jul 2022 00:35:32 -0700 Subject: [PATCH] Introduce API to support Stylus Companion widget Introduce APIs necessary for IMEs to be able to distinguish when to show virtual keyboard vs companion widget based on toolType. IME receives click toolType in: 1. IMS#onUpdateEditorToolType(int toolType) for every showSoftInput() called from IMM. 2. Initial value of toolType in EditorInfo#getInitialToolType(). Fix: 233108064 Bug: 210039666 Test: atest StylusHandwritingTest Change-Id: I91ea9204ef62697b8942b0f350169382f18b3278 --- core/api/current.txt | 3 ++ .../IInputMethodWrapper.java | 13 +++++++ .../InputMethodService.java | 25 +++++++++++++ core/java/android/view/ViewRootImpl.java | 17 ++++++++- .../android/view/inputmethod/EditorInfo.java | 37 +++++++++++++++++++ .../IInputMethodManagerInvoker.java | 5 ++- .../android/view/inputmethod/InputMethod.java | 9 +++++ .../view/inputmethod/InputMethodManager.java | 5 +++ .../internal/inputmethod/IInputMethod.aidl | 2 + .../internal/view/IInputMethodManager.aidl | 2 +- .../view/inputmethod/EditorInfoTest.java | 11 ++++-- .../inputmethod/IInputMethodInvoker.java | 11 ++++++ .../InputMethodManagerService.java | 22 +++++++++-- 13 files changed, 151 insertions(+), 11 deletions(-) diff --git a/core/api/current.txt b/core/api/current.txt index 7fbf3ece45508..64d6a442f4d69 100644 --- a/core/api/current.txt +++ b/core/api/current.txt @@ -18903,6 +18903,7 @@ package android.inputmethodservice { method public void onUnbindInput(); method @Deprecated public void onUpdateCursor(android.graphics.Rect); method public void onUpdateCursorAnchorInfo(android.view.inputmethod.CursorAnchorInfo); + method public void onUpdateEditorToolType(int); method public void onUpdateExtractedText(int, android.view.inputmethod.ExtractedText); method public void onUpdateExtractingViews(android.view.inputmethod.EditorInfo); method public void onUpdateExtractingVisibility(android.view.inputmethod.EditorInfo); @@ -52966,9 +52967,11 @@ package android.view.inputmethod { method @Nullable public android.view.inputmethod.SurroundingText getInitialSurroundingText(@IntRange(from=0) int, @IntRange(from=0) int, int); method @Nullable public CharSequence getInitialTextAfterCursor(@IntRange(from=0) int, int); method @Nullable public CharSequence getInitialTextBeforeCursor(@IntRange(from=0) int, int); + method public int getInitialToolType(); method public final void makeCompatible(int); method public void setInitialSurroundingSubText(@NonNull CharSequence, int); method public void setInitialSurroundingText(@NonNull CharSequence); + method public void setInitialToolType(int); method public void writeToParcel(android.os.Parcel, int); field @NonNull public static final android.os.Parcelable.Creator CREATOR; field public static final int IME_ACTION_DONE = 6; // 0x6 diff --git a/core/java/android/inputmethodservice/IInputMethodWrapper.java b/core/java/android/inputmethodservice/IInputMethodWrapper.java index e960df112ae3a..3d593872757d1 100644 --- a/core/java/android/inputmethodservice/IInputMethodWrapper.java +++ b/core/java/android/inputmethodservice/IInputMethodWrapper.java @@ -80,6 +80,7 @@ class IInputMethodWrapper extends IInputMethod.Stub private static final int DO_START_STYLUS_HANDWRITING = 110; private static final int DO_INIT_INK_WINDOW = 120; private static final int DO_FINISH_STYLUS_HANDWRITING = 130; + private static final int DO_UPDATE_TOOL_TYPE = 140; final WeakReference mTarget; final Context mContext; @@ -234,6 +235,10 @@ class IInputMethodWrapper extends IInputMethod.Stub inputMethod.canStartStylusHandwriting(msg.arg1); return; } + case DO_UPDATE_TOOL_TYPE: { + inputMethod.updateEditorToolType(msg.arg1); + return; + } case DO_START_STYLUS_HANDWRITING: { final SomeArgs args = (SomeArgs) msg.obj; inputMethod.startStylusHandwriting(msg.arg1, (InputChannel) args.arg1, @@ -400,6 +405,14 @@ class IInputMethodWrapper extends IInputMethod.Stub mCaller.obtainMessageI(DO_CAN_START_STYLUS_HANDWRITING, requestId)); } + @BinderThread + @Override + public void updateEditorToolType(int toolType) + throws RemoteException { + mCaller.executeOrSendMessage( + mCaller.obtainMessageI(DO_UPDATE_TOOL_TYPE, toolType)); + } + @BinderThread @Override public void startStylusHandwriting(int requestId, @NonNull InputChannel channel, diff --git a/core/java/android/inputmethodservice/InputMethodService.java b/core/java/android/inputmethodservice/InputMethodService.java index b108490e54e98..c2027b136f0e4 100644 --- a/core/java/android/inputmethodservice/InputMethodService.java +++ b/core/java/android/inputmethodservice/InputMethodService.java @@ -948,6 +948,15 @@ public class InputMethodService extends AbstractInputMethodService { Trace.traceEnd(TRACE_TAG_WINDOW_MANAGER); } + /** + * {@inheritDoc} + * @hide + */ + @Override + public void updateEditorToolType(int toolType) { + onUpdateEditorToolType(toolType); + } + /** * {@inheritDoc} * @hide @@ -3012,17 +3021,33 @@ public class InputMethodService extends AbstractInputMethodService { * not call to inform the IME of this interaction. * @param focusChanged true if the user changed the focused view by this click. * @see InputMethodManager#viewClicked(View) + * @see #onUpdateEditorToolType(int) * @deprecated The method may not be called for composite {@link View} that works as a giant * "Canvas", which can host its own UI hierarchy and sub focus state. * {@link android.webkit.WebView} is a good example. Application / IME developers * should not rely on this method. If your goal is just being notified when an * on-going input is interrupted, simply monitor {@link #onFinishInput()}. + * If your goal is to know what {@link MotionEvent#getToolType(int)} clicked on + * editor, use {@link #onUpdateEditorToolType(int)} instead. */ @Deprecated public void onViewClicked(boolean focusChanged) { // Intentionally empty } + /** + * Called when the user tapped or clicked an {@link android.widget.Editor}. + * This can be useful when IME makes a decision of showing Virtual keyboard based on what + * {@link MotionEvent#getToolType(int)} was used to click the editor. + * e.g. when toolType is {@link MotionEvent#TOOL_TYPE_STYLUS}, IME may choose to show a + * companion widget instead of normal virtual keyboard. + *

Default implementation does nothing.

+ * @param toolType what {@link MotionEvent#getToolType(int)} was used to click on editor. + */ + public void onUpdateEditorToolType(int toolType) { + // Intentionally empty + } + /** * Called when the application has reported a new location of its text * cursor. This is only called if explicitly requested by the input method. diff --git a/core/java/android/view/ViewRootImpl.java b/core/java/android/view/ViewRootImpl.java index c1a5a3a256806..a05b883217d80 100644 --- a/core/java/android/view/ViewRootImpl.java +++ b/core/java/android/view/ViewRootImpl.java @@ -732,6 +732,8 @@ public final class ViewRootImpl implements ViewParent, final PointF mDragPoint = new PointF(); final PointF mLastTouchPoint = new PointF(); int mLastTouchSource; + /** Tracks last {@link MotionEvent#getToolType(int)} with {@link MotionEvent#ACTION_UP}. **/ + private int mLastClickToolType; private boolean mProfileRendering; private Choreographer.FrameCallback mRenderProfiler; @@ -6392,11 +6394,16 @@ public final class ViewRootImpl implements ViewParent, event.offsetLocation(0, mCurScrollY); } - // Remember the touch position for possible drag-initiation. if (event.isTouchEvent()) { + // Remember the touch position for possible drag-initiation. mLastTouchPoint.x = event.getRawX(); mLastTouchPoint.y = event.getRawY(); mLastTouchSource = event.getSource(); + + // Register last ACTION_UP. This will be propagated to IME. + if (event.getActionMasked() == MotionEvent.ACTION_UP) { + mLastClickToolType = event.getToolType(event.getActionIndex()); + } } return FORWARD; } @@ -7992,6 +7999,14 @@ public final class ViewRootImpl implements ViewParent, return mLastTouchSource; } + /** + * Used by InputMethodManager. + * @hide + */ + public int getLastClickToolType() { + return mLastClickToolType; + } + public void setDragFocus(View newDragTarget, DragEvent event) { if (mCurrentDragView != newDragTarget && !View.sCascadedDragDrop) { // Send EXITED and ENTERED notifications to the old and new drag focus views. diff --git a/core/java/android/view/inputmethod/EditorInfo.java b/core/java/android/view/inputmethod/EditorInfo.java index fdff7a30a6e94..55a2d22d51b73 100644 --- a/core/java/android/view/inputmethod/EditorInfo.java +++ b/core/java/android/view/inputmethod/EditorInfo.java @@ -30,6 +30,7 @@ import android.annotation.NonNull; import android.annotation.Nullable; import android.annotation.RequiresPermission; import android.content.res.Configuration; +import android.inputmethodservice.InputMethodService; import android.os.Build.VERSION_CODES; import android.os.Bundle; import android.os.LocaleList; @@ -40,6 +41,7 @@ import android.text.InputType; import android.text.TextUtils; import android.util.Printer; import android.util.proto.ProtoOutputStream; +import android.view.MotionEvent; import android.view.View; import android.view.autofill.AutofillId; @@ -564,6 +566,12 @@ public class EditorInfo implements InputType, Parcelable { @Nullable private SurroundingText mInitialSurroundingText = null; + /** + * Initial {@link MotionEvent#ACTION_UP} tool type {@link MotionEvent#getToolType(int)} that + * was used to focus this editor. + */ + private int mInitialToolType = MotionEvent.TOOL_TYPE_UNKNOWN; + /** * Editors may use this method to provide initial input text to IMEs. As the surrounding text @@ -936,6 +944,31 @@ public class EditorInfo implements InputType, Parcelable { } } + /** + * Returns the initial {@link MotionEvent#ACTION_UP} tool type + * {@link MotionEvent#getToolType(int)} responsible for focus on the current editor. + * + * @see #setInitialToolType(int) + * @see MotionEvent#getToolType(int) + * @see InputMethodService#onUpdateEditorToolType(int) + * @return toolType {@link MotionEvent#getToolType(int)}. + */ + public int getInitialToolType() { + return mInitialToolType; + } + + /** + * Set the initial {@link MotionEvent#ACTION_UP} tool type {@link MotionEvent#getToolType(int)}. + * that brought focus to the view. + * + * @see #getInitialToolType() + * @see MotionEvent#getToolType(int) + * @see InputMethodService#onUpdateEditorToolType(int) + */ + public void setInitialToolType(int toolType) { + mInitialToolType = toolType; + } + /** * Export the state of {@link EditorInfo} into a protocol buffer output stream. * @@ -972,6 +1005,7 @@ public class EditorInfo implements InputType, Parcelable { + " actionId=" + actionId); pw.println(prefix + "initialSelStart=" + initialSelStart + " initialSelEnd=" + initialSelEnd + + " initialToolType=" + mInitialToolType + " initialCapsMode=0x" + Integer.toHexString(initialCapsMode)); pw.println(prefix + "hintText=" + hintText @@ -1006,6 +1040,7 @@ public class EditorInfo implements InputType, Parcelable { newEditorInfo.initialSelStart = initialSelStart; newEditorInfo.initialSelEnd = initialSelEnd; newEditorInfo.initialCapsMode = initialCapsMode; + newEditorInfo.mInitialToolType = mInitialToolType; newEditorInfo.hintText = TextUtils.stringOrSpannedString(hintText); newEditorInfo.label = TextUtils.stringOrSpannedString(label); newEditorInfo.packageName = packageName; @@ -1036,6 +1071,7 @@ public class EditorInfo implements InputType, Parcelable { dest.writeInt(initialSelStart); dest.writeInt(initialSelEnd); dest.writeInt(initialCapsMode); + dest.writeInt(mInitialToolType); TextUtils.writeToParcel(hintText, dest, flags); TextUtils.writeToParcel(label, dest, flags); dest.writeString(packageName); @@ -1072,6 +1108,7 @@ public class EditorInfo implements InputType, Parcelable { res.initialSelStart = source.readInt(); res.initialSelEnd = source.readInt(); res.initialCapsMode = source.readInt(); + res.mInitialToolType = source.readInt(); res.hintText = TextUtils.CHAR_SEQUENCE_CREATOR.createFromParcel(source); res.label = TextUtils.CHAR_SEQUENCE_CREATOR.createFromParcel(source); res.packageName = source.readString(); diff --git a/core/java/android/view/inputmethod/IInputMethodManagerInvoker.java b/core/java/android/view/inputmethod/IInputMethodManagerInvoker.java index d21ceb2ec79e2..7c2a6994b0e0d 100644 --- a/core/java/android/view/inputmethod/IInputMethodManagerInvoker.java +++ b/core/java/android/view/inputmethod/IInputMethodManagerInvoker.java @@ -111,10 +111,11 @@ final class IInputMethodManagerInvoker { @AnyThread boolean showSoftInput(@NonNull IInputMethodClient client, @Nullable IBinder windowToken, - int flags, @Nullable ResultReceiver resultReceiver, + int flags, int lastClickToolType, @Nullable ResultReceiver resultReceiver, @SoftInputShowHideReason int reason) { try { - return mTarget.showSoftInput(client, windowToken, flags, resultReceiver, reason); + return mTarget.showSoftInput( + client, windowToken, flags, lastClickToolType, resultReceiver, reason); } catch (RemoteException e) { throw e.rethrowFromSystemServer(); } diff --git a/core/java/android/view/inputmethod/InputMethod.java b/core/java/android/view/inputmethod/InputMethod.java index 95add29788799..bfe6ae6447d0e 100644 --- a/core/java/android/view/inputmethod/InputMethod.java +++ b/core/java/android/view/inputmethod/InputMethod.java @@ -376,6 +376,15 @@ public interface InputMethod { // intentionally empty } + /** + * This method is called when the user tapped or clicked an {@link android.widget.Editor}. + * @param toolType {@link android.view.MotionEvent#getToolType(int)} used for clicking editor. + * @hide + */ + default void updateEditorToolType(int toolType) { + // intentionally empty + } + /** * Start stylus handwriting session. * @hide diff --git a/core/java/android/view/inputmethod/InputMethodManager.java b/core/java/android/view/inputmethod/InputMethodManager.java index e7abbbcee7cad..8a04f4da35482 100644 --- a/core/java/android/view/inputmethod/InputMethodManager.java +++ b/core/java/android/view/inputmethod/InputMethodManager.java @@ -1929,6 +1929,7 @@ public final class InputMethodManager { mClient, view.getWindowToken(), flags, + mCurRootView.getLastClickToolType(), resultReceiver, reason); } @@ -1960,6 +1961,7 @@ public final class InputMethodManager { mClient, mCurRootView.getView().getWindowToken(), flags, + mCurRootView.getLastClickToolType(), resultReceiver, SoftInputShowHideReason.SHOW_SOFT_INPUT); } @@ -2339,6 +2341,9 @@ public final class InputMethodManager { editorInfo.packageName = view.getContext().getOpPackageName(); editorInfo.autofillId = view.getAutofillId(); editorInfo.fieldId = view.getId(); + synchronized (mH) { + editorInfo.setInitialToolType(mCurRootView.getLastClickToolType()); + } InputConnection ic = view.onCreateInputConnection(editorInfo); if (DEBUG) Log.v(TAG, "Starting input: editorInfo=" + editorInfo + " ic=" + ic); diff --git a/core/java/com/android/internal/inputmethod/IInputMethod.aidl b/core/java/com/android/internal/inputmethod/IInputMethod.aidl index 8bab5c31217d8..5db2e84845f50 100644 --- a/core/java/com/android/internal/inputmethod/IInputMethod.aidl +++ b/core/java/com/android/internal/inputmethod/IInputMethod.aidl @@ -74,6 +74,8 @@ oneway interface IInputMethod { void hideSoftInput(in IBinder hideInputToken, int flags, in ResultReceiver resultReceiver); + void updateEditorToolType(int toolType); + void changeInputMethodSubtype(in InputMethodSubtype subtype); void canStartStylusHandwriting(int requestId); diff --git a/core/java/com/android/internal/view/IInputMethodManager.aidl b/core/java/com/android/internal/view/IInputMethodManager.aidl index d5f835295ee78..059cfca5bc027 100644 --- a/core/java/com/android/internal/view/IInputMethodManager.aidl +++ b/core/java/com/android/internal/view/IInputMethodManager.aidl @@ -55,7 +55,7 @@ interface IInputMethodManager { InputMethodSubtype getLastInputMethodSubtype(int userId); boolean showSoftInput(in IInputMethodClient client, @nullable IBinder windowToken, int flags, - in @nullable ResultReceiver resultReceiver, int reason); + int lastClickToolType, in @nullable ResultReceiver resultReceiver, int reason); boolean hideSoftInput(in IInputMethodClient client, @nullable IBinder windowToken, int flags, in @nullable ResultReceiver resultReceiver, int reason); diff --git a/core/tests/coretests/src/android/view/inputmethod/EditorInfoTest.java b/core/tests/coretests/src/android/view/inputmethod/EditorInfoTest.java index b867e44174392..9637de8f31267 100644 --- a/core/tests/coretests/src/android/view/inputmethod/EditorInfoTest.java +++ b/core/tests/coretests/src/android/view/inputmethod/EditorInfoTest.java @@ -42,6 +42,7 @@ import android.text.TextUtils; import android.text.style.MaskFilterSpan; import android.text.style.UnderlineSpan; import android.util.StringBuilderPrinter; +import android.view.MotionEvent; import android.view.autofill.AutofillId; import androidx.test.filters.SmallTest; @@ -490,7 +491,8 @@ public class EditorInfoTest { assertThat(sb.toString()).isEqualTo( "prefix: inputType=0x0 imeOptions=0x0 privateImeOptions=null\n" + "prefix: actionLabel=null actionId=0\n" - + "prefix: initialSelStart=-1 initialSelEnd=-1 initialCapsMode=0x0\n" + + "prefix: initialSelStart=-1 initialSelEnd=-1 initialToolType=0" + + " initialCapsMode=0x0\n" + "prefix: hintText=null label=null\n" + "prefix: packageName=null autofillId=null fieldId=0 fieldName=null\n" + "prefix: extras=null\n" @@ -509,6 +511,7 @@ public class EditorInfoTest { info.initialCapsMode = TextUtils.CAP_MODE_CHARACTERS; // 0x1000 info.hintText = "testHintText"; info.label = "testLabel"; + info.setInitialToolType(MotionEvent.TOOL_TYPE_STYLUS); info.packageName = "android.view.inputmethod"; info.autofillId = new AutofillId(123); info.fieldId = 456; @@ -523,7 +526,8 @@ public class EditorInfoTest { assertThat(sb.toString()).isEqualTo( "prefix2: inputType=0x1 imeOptions=0x2 privateImeOptions=testOptions\n" + "prefix2: actionLabel=null actionId=0\n" - + "prefix2: initialSelStart=0 initialSelEnd=1 initialCapsMode=0x1000\n" + + "prefix2: initialSelStart=0 initialSelEnd=1 initialToolType=2" + + " initialCapsMode=0x1000\n" + "prefix2: hintText=testHintText label=testLabel\n" + "prefix2: packageName=android.view.inputmethod autofillId=123" + " fieldId=456 fieldName=testField\n" @@ -543,7 +547,8 @@ public class EditorInfoTest { assertThat(sb.toString()).isEqualTo( "prefix: inputType=0x0 imeOptions=0x0 privateImeOptions=null\n" + "prefix: actionLabel=null actionId=0\n" - + "prefix: initialSelStart=-1 initialSelEnd=-1 initialCapsMode=0x0\n" + + "prefix: initialSelStart=-1 initialSelEnd=-1 initialToolType=0" + + " initialCapsMode=0x0\n" + "prefix: hintText=null label=null\n" + "prefix: packageName=null autofillId=null fieldId=0 fieldName=null\n" + "prefix: hintLocales=null\n" diff --git a/services/core/java/com/android/server/inputmethod/IInputMethodInvoker.java b/services/core/java/com/android/server/inputmethod/IInputMethodInvoker.java index 6c75dbf033d68..eb73234549c92 100644 --- a/services/core/java/com/android/server/inputmethod/IInputMethodInvoker.java +++ b/services/core/java/com/android/server/inputmethod/IInputMethodInvoker.java @@ -222,6 +222,17 @@ final class IInputMethodInvoker { return true; } + @AnyThread + boolean updateEditorToolType(int toolType) { + try { + mTarget.updateEditorToolType(toolType); + } catch (RemoteException e) { + logRemoteException(e); + return false; + } + return true; + } + @AnyThread void changeInputMethodSubtype(InputMethodSubtype subtype) { try { diff --git a/services/core/java/com/android/server/inputmethod/InputMethodManagerService.java b/services/core/java/com/android/server/inputmethod/InputMethodManagerService.java index af95b4f54eb42..d8cf39926a9a3 100644 --- a/services/core/java/com/android/server/inputmethod/InputMethodManagerService.java +++ b/services/core/java/com/android/server/inputmethod/InputMethodManagerService.java @@ -132,6 +132,7 @@ import android.view.DisplayInfo; import android.view.IWindowManager; import android.view.InputChannel; import android.view.InputDevice; +import android.view.MotionEvent; import android.view.View; import android.view.WindowManager; import android.view.WindowManager.DisplayImePolicy; @@ -3359,7 +3360,8 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub @Override public boolean showSoftInput(IInputMethodClient client, IBinder windowToken, int flags, - ResultReceiver resultReceiver, @SoftInputShowHideReason int reason) { + int lastClickTooType, ResultReceiver resultReceiver, + @SoftInputShowHideReason int reason) { Trace.traceBegin(TRACE_TAG_WINDOW_MANAGER, "IMMS.showSoftInput"); int uid = Binder.getCallingUid(); ImeTracing.getInstance().triggerManagerServiceDump( @@ -3371,7 +3373,8 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub final long ident = Binder.clearCallingIdentity(); try { if (DEBUG) Slog.v(TAG, "Client requesting input be shown"); - return showCurrentInputLocked(windowToken, flags, resultReceiver, reason); + return showCurrentInputLocked( + windowToken, flags, lastClickTooType, resultReceiver, reason); } finally { Binder.restoreCallingIdentity(ident); Trace.traceEnd(TRACE_TAG_WINDOW_MANAGER); @@ -3443,8 +3446,15 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub } @GuardedBy("ImfLock.class") - boolean showCurrentInputLocked(IBinder windowToken, int flags, ResultReceiver resultReceiver, - @SoftInputShowHideReason int reason) { + boolean showCurrentInputLocked(IBinder windowToken, int flags, + ResultReceiver resultReceiver, @SoftInputShowHideReason int reason) { + return showCurrentInputLocked( + windowToken, flags, MotionEvent.TOOL_TYPE_UNKNOWN, resultReceiver, reason); + } + + @GuardedBy("ImfLock.class") + private boolean showCurrentInputLocked(IBinder windowToken, int flags, int lastClickToolType, + ResultReceiver resultReceiver, @SoftInputShowHideReason int reason) { mShowRequested = true; if (mAccessibilityRequestingNoSoftKeyboard || mImeHiddenByDisplayPolicy) { return false; @@ -3473,6 +3483,10 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub + ", " + showFlags + ", " + resultReceiver + ") for reason: " + InputMethodDebug.softInputDisplayReasonToString(reason)); } + + if (lastClickToolType != MotionEvent.TOOL_TYPE_UNKNOWN) { + curMethod.updateEditorToolType(lastClickToolType); + } // TODO(b/192412909): Check if we can always call onShowHideSoftInputRequested() or not. if (curMethod.showSoftInput(showInputToken, showFlags, resultReceiver)) { onShowHideSoftInputRequested(true /* show */, windowToken, reason);