From e35da24d50a724f1ed9e1cc62ed534946a1cb8a8 Mon Sep 17 00:00:00 2001 From: Taran Singh Date: Fri, 10 Feb 2023 23:33:16 +0000 Subject: [PATCH] Fix handwriting support on fake editors using delegation We originally added support for Scribe on fake editors in bug 228100684. However, it doesn't address special cases like when real editor is a different application package. This CL introduces IMF APIs to suport handwriting delegation across different view trees (with or within the same package). Bug: 266834695 Bug: 228100684 Test: atest StylusHandwritingTest Change-Id: I3a5d5dc574c5f78697f2a43e11729febf21cc4e2 --- core/api/current.txt | 4 + .../IInputMethodManagerGlobalInvoker.java | 34 ++++ .../view/inputmethod/InputMethodManager.java | 173 ++++++++++++++++-- .../internal/view/IInputMethodManager.aidl | 9 + .../HandwritingModeController.java | 51 +++++- .../InputMethodManagerService.java | 87 +++++++++ 6 files changed, 338 insertions(+), 20 deletions(-) diff --git a/core/api/current.txt b/core/api/current.txt index 2968b59e16c91..f770e272098f5 100644 --- a/core/api/current.txt +++ b/core/api/current.txt @@ -55616,6 +55616,8 @@ package android.view.inputmethod { } public final class InputMethodManager { + method public boolean acceptStylusHandwritingDelegation(@NonNull android.view.View); + method public boolean acceptStylusHandwritingDelegation(@NonNull android.view.View, @NonNull String); method public void dispatchKeyEventFromInputMethod(@Nullable android.view.View, @NonNull android.view.KeyEvent); method public void displayCompletions(android.view.View, android.view.inputmethod.CompletionInfo[]); method @Nullable public android.view.inputmethod.InputMethodInfo getCurrentInputMethodInfo(); @@ -55637,6 +55639,8 @@ package android.view.inputmethod { method public boolean isInputMethodSuppressingSpellChecker(); method public boolean isStylusHandwritingAvailable(); method @Deprecated public boolean isWatchingCursor(android.view.View); + method public void prepareStylusHandwritingDelegation(@NonNull android.view.View); + method public void prepareStylusHandwritingDelegation(@NonNull android.view.View, @NonNull String); method public void restartInput(android.view.View); method public void sendAppPrivateCommand(android.view.View, String, android.os.Bundle); method @Deprecated public void setAdditionalInputMethodSubtypes(@NonNull String, @NonNull android.view.inputmethod.InputMethodSubtype[]); diff --git a/core/java/android/view/inputmethod/IInputMethodManagerGlobalInvoker.java b/core/java/android/view/inputmethod/IInputMethodManagerGlobalInvoker.java index 966026198a211..db17a533c303e 100644 --- a/core/java/android/view/inputmethod/IInputMethodManagerGlobalInvoker.java +++ b/core/java/android/view/inputmethod/IInputMethodManagerGlobalInvoker.java @@ -504,6 +504,40 @@ final class IInputMethodManagerGlobalInvoker { } } + @AnyThread + static void prepareStylusHandwritingDelegation( + @NonNull IInputMethodClient client, + @NonNull String delegatePackageName, + @NonNull String delegatorPackageName) { + final IInputMethodManager service = getService(); + if (service == null) { + return; + } + try { + service.prepareStylusHandwritingDelegation( + client, delegatePackageName, delegatorPackageName); + } catch (RemoteException e) { + throw e.rethrowFromSystemServer(); + } + } + + @AnyThread + static boolean acceptStylusHandwritingDelegation( + @NonNull IInputMethodClient client, + @NonNull String delegatePackageName, + @NonNull String delegatorPackageName) { + final IInputMethodManager service = getService(); + if (service == null) { + return false; + } + try { + return service.acceptStylusHandwritingDelegation( + client, delegatePackageName, delegatorPackageName); + } catch (RemoteException e) { + throw e.rethrowFromSystemServer(); + } + } + @AnyThread @RequiresPermission(value = Manifest.permission.INTERACT_ACROSS_USERS_FULL, conditional = true) static boolean isStylusHandwritingAvailableAsUser(@UserIdInt int userId) { diff --git a/core/java/android/view/inputmethod/InputMethodManager.java b/core/java/android/view/inputmethod/InputMethodManager.java index 642182b5ddfde..36d2b8a897793 100644 --- a/core/java/android/view/inputmethod/InputMethodManager.java +++ b/core/java/android/view/inputmethod/InputMethodManager.java @@ -99,6 +99,7 @@ import android.view.WindowManager; import android.view.WindowManager.LayoutParams.SoftInputModeFlags; import android.view.autofill.AutofillId; import android.view.autofill.AutofillManager; +import android.widget.Editor; import android.window.ImeOnBackInvokedDispatcher; import android.window.WindowOnBackInvokedDispatcher; @@ -1552,11 +1553,7 @@ public final class InputMethodManager { if (fallbackContext == null) { return false; } - if (Settings.Global.getInt(fallbackContext.getContentResolver(), - Settings.Global.STYLUS_HANDWRITING_ENABLED, 0) == 0) { - if (DEBUG) { - Log.d(TAG, "Stylus handwriting is not enabled in settings."); - } + if (!isStylusHandwritingEnabled(fallbackContext)) { return false; } return IInputMethodManagerGlobalInvoker.isStylusHandwritingAvailableAsUser(userId); @@ -2233,37 +2230,175 @@ public final class InputMethodManager { * @see #isStylusHandwritingAvailable() */ public void startStylusHandwriting(@NonNull View view) { + startStylusHandwritingInternal(view, null /* delegatorPackageName */); + } + + private boolean startStylusHandwritingInternal( + @NonNull View view, @Nullable String delegatorPackageName) { + Objects.requireNonNull(view); + // Re-dispatch if there is a context mismatch. final InputMethodManager fallbackImm = getFallbackInputMethodManagerIfNecessary(view); if (fallbackImm != null) { - fallbackImm.startStylusHandwriting(view); + fallbackImm.startStylusHandwritingInternal(view, delegatorPackageName); } - Objects.requireNonNull(view); - if (Settings.Global.getInt(view.getContext().getContentResolver(), - Settings.Global.STYLUS_HANDWRITING_ENABLED, 0) == 0) { - Log.d(TAG, "Ignoring startStylusHandwriting(view) as stylus handwriting is disabled"); - return; + boolean useDelegation = !TextUtils.isEmpty(delegatorPackageName); + if (!isStylusHandwritingEnabled(view.getContext())) { + Log.w(TAG, "Stylus handwriting pref is disabled. " + + "Ignoring calls to start stylus handwriting."); + return false; } checkFocus(); synchronized (mH) { if (!hasServedByInputMethodLocked(view)) { Log.w(TAG, - "Ignoring startStylusHandwriting() as view=" + view + " is not served."); - return; + "Ignoring startStylusHandwriting as view=" + view + " is not served."); + return false; } if (view.getViewRootImpl() != mCurRootView) { - Log.w(TAG, "Ignoring startStylusHandwriting: View's window does not have focus."); - return; + Log.w(TAG, + "Ignoring startStylusHandwriting: View's window does not have focus."); + return false; } - - IInputMethodManagerGlobalInvoker.startStylusHandwriting(mClient); - // TODO(b/210039666): do we need any extra work for supporting non-native - // UI toolkits? + if (useDelegation) { + return IInputMethodManagerGlobalInvoker.acceptStylusHandwritingDelegation( + mClient, view.getContext().getOpPackageName(), delegatorPackageName); + } else { + IInputMethodManagerGlobalInvoker.startStylusHandwriting(mClient); + } + return false; } } + private boolean isStylusHandwritingEnabled(@NonNull Context context) { + if (Settings.Global.getInt(context.getContentResolver(), + Settings.Global.STYLUS_HANDWRITING_ENABLED, 0) == 0) { + Log.d(TAG, "Stylus handwriting pref is disabled."); + return false; + } + return true; + } + + /** + * Prepares delegation of starting stylus handwriting session to a different editor in same + * or different window than the view on which initial handwriting stroke was detected. + * + * Delegation can be used to start stylus handwriting session before the {@link Editor} view or + * its {@link InputConnection} is started. Calling this method starts buffering of stylus + * motion events until {@link #acceptStylusHandwritingDelegation(View)} is called, at which + * point the handwriting session can be started and the buffered stylus motion events will be + * delivered to the IME. + * e.g. Delegation can be used when initial handwriting stroke is + * on a pseudo {@link Editor} like widget (with no {@link InputConnection}) but actual + * {@link Editor} is on a different window. + * + *

Note: If an actual {@link Editor} capable of {@link InputConnection} is being scribbled + * upon using stylus, use {@link #startStylusHandwriting(View)} instead.

+ * + * @param delegatorView the view that receives initial stylus stroke and delegates it to the + * actual editor. Its window must {@link View#hasWindowFocus have focus}. + * @see #prepareStylusHandwritingDelegation(View, String) + * @see #acceptStylusHandwritingDelegation(View) + * @see #startStylusHandwriting(View) + */ + public void prepareStylusHandwritingDelegation(@NonNull View delegatorView) { + prepareStylusHandwritingDelegation( + delegatorView, delegatorView.getContext().getOpPackageName()); + } + + /** + * Prepares delegation of starting stylus handwriting session to a different editor in same or a + * different window in a different package than the view on which initial handwriting stroke + * was detected. + * + * Delegation can be used to start stylus handwriting session before the {@link Editor} view or + * its {@link InputConnection} is started. Calling this method starts buffering of stylus + * motion events until {@link #acceptStylusHandwritingDelegation(View, String)} is called, at + * which point the handwriting session can be started and the buffered stylus motion events will + * be delivered to the IME. + * e.g. Delegation can be used when initial handwriting stroke is + * on a pseudo {@link Editor} like widget (with no {@link InputConnection}) but actual + * {@link Editor} is on a different window in the given package. + * + *

Note: If delegator and delegate are in same package use + * {@link #prepareStylusHandwritingDelegation(View)} instead.

+ * + * @param delegatorView the view that receives initial stylus stroke and delegates it to the + * actual editor. Its window must {@link View#hasWindowFocus have focus}. + * @param delegatePackageName package name that contains actual {@link Editor} which should + * start stylus handwriting session by calling {@link #acceptStylusHandwritingDelegation}. + * @see #prepareStylusHandwritingDelegation(View) + * @see #acceptStylusHandwritingDelegation(View, String) + */ + public void prepareStylusHandwritingDelegation( + @NonNull View delegatorView, @NonNull String delegatePackageName) { + Objects.requireNonNull(delegatorView); + Objects.requireNonNull(delegatePackageName); + + // Re-dispatch if there is a context mismatch. + final InputMethodManager fallbackImm = + getFallbackInputMethodManagerIfNecessary(delegatorView); + if (fallbackImm != null) { + fallbackImm.prepareStylusHandwritingDelegation(delegatorView, delegatePackageName); + } + + if (!isStylusHandwritingEnabled(delegatorView.getContext())) { + Log.w(TAG, "Stylus handwriting pref is disabled. " + + "Ignoring prepareStylusHandwritingDelegation()."); + return; + } + IInputMethodManagerGlobalInvoker.prepareStylusHandwritingDelegation( + mClient, + delegatePackageName, + delegatorView.getContext().getOpPackageName()); + } + + /** + * Accepts and starts a stylus handwriting session on the delegate view, if handwriting + * initiation delegation was previously requested using + * {@link #prepareStylusHandwritingDelegation(View)} from the delegator. + * + *

Note: If delegator and delegate are in different application packages, use + * {@link #acceptStylusHandwritingDelegation(View, String)} instead.

+ * + * @param delegateView delegate view capable of receiving input via {@link InputConnection} + * on which {@link #startStylusHandwriting(View)} will be called. + * @return {@code true} if view belongs to same application package as used in + * {@link #prepareStylusHandwritingDelegation(View)} and handwriting session can start. + * @see #acceptStylusHandwritingDelegation(View, String) + * @see #prepareStylusHandwritingDelegation(View) + */ + public boolean acceptStylusHandwritingDelegation(@NonNull View delegateView) { + return startStylusHandwritingInternal( + delegateView, delegateView.getContext().getOpPackageName()); + } + + /** + * Accepts and starts a stylus handwriting session on the delegate view, if handwriting + * initiation delegation was previously requested using + * {@link #prepareStylusHandwritingDelegation(View, String)} from te delegator and the view + * belongs to a specified delegate package. + * + *

Note: If delegator and delegate are in same application package use + * {@link #acceptStylusHandwritingDelegation(View)} instead.

+ * + * @param delegateView delegate view capable of receiving input via {@link InputConnection} + * on which {@link #startStylusHandwriting(View)} will be called. + * @param delegatorPackageName package name of the delegator that handled initial stylus stroke. + * @return {@code true} if view belongs to allowed delegate package declared in + * {@link #prepareStylusHandwritingDelegation(View, String)} and handwriting session can start. + * @see #prepareStylusHandwritingDelegation(View, String) + * @see #acceptStylusHandwritingDelegation(View) + */ + public boolean acceptStylusHandwritingDelegation( + @NonNull View delegateView, @NonNull String delegatorPackageName) { + Objects.requireNonNull(delegatorPackageName); + + return startStylusHandwritingInternal(delegateView, delegatorPackageName); + } + /** * This method toggles the input method window display. * If the input window is already displayed, it gets hidden. diff --git a/core/java/com/android/internal/view/IInputMethodManager.aidl b/core/java/com/android/internal/view/IInputMethodManager.aidl index 9116cb3ec9fa2..5805d0e050d16 100644 --- a/core/java/com/android/internal/view/IInputMethodManager.aidl +++ b/core/java/com/android/internal/view/IInputMethodManager.aidl @@ -148,6 +148,15 @@ interface IInputMethodManager { /** Start Stylus handwriting session **/ void startStylusHandwriting(in IInputMethodClient client); + /** Prepares delegation of starting stylus handwriting session to a different editor **/ + void prepareStylusHandwritingDelegation(in IInputMethodClient client, + in String delegatePackageName, + in String delegatorPackageName); + + /** Accepts and starts a stylus handwriting session for the delegate view **/ + boolean acceptStylusHandwritingDelegation(in IInputMethodClient client, + in String delegatePackageName, in String delegatorPackageName); + /** Returns {@code true} if currently selected IME supports Stylus handwriting. */ @JavaPassthrough(annotation="@android.annotation.RequiresPermission(value = " + "android.Manifest.permission.INTERACT_ACROSS_USERS_FULL, conditional = true)") diff --git a/services/core/java/com/android/server/inputmethod/HandwritingModeController.java b/services/core/java/com/android/server/inputmethod/HandwritingModeController.java index 1c7294f8a9d5c..eb4dba63dea9b 100644 --- a/services/core/java/com/android/server/inputmethod/HandwritingModeController.java +++ b/services/core/java/com/android/server/inputmethod/HandwritingModeController.java @@ -20,12 +20,14 @@ import static android.view.InputDevice.SOURCE_STYLUS; import android.Manifest; import android.annotation.AnyThread; +import android.annotation.NonNull; import android.annotation.Nullable; import android.annotation.RequiresPermission; import android.annotation.UiThread; import android.hardware.input.InputManager; import android.os.IBinder; import android.os.Looper; +import android.text.TextUtils; import android.util.Slog; import android.view.BatchedInputEventReceiver; import android.view.Choreographer; @@ -35,6 +37,8 @@ import android.view.InputEvent; import android.view.InputEventReceiver; import android.view.MotionEvent; import android.view.SurfaceControl; +import android.view.View; +import android.view.inputmethod.InputMethodManager; import com.android.server.LocalServices; import com.android.server.input.InputManagerInternal; @@ -52,6 +56,9 @@ final class HandwritingModeController { // TODO(b/210039666): flip the flag. static final boolean DEBUG = true; private static final int EVENT_BUFFER_SIZE = 100; + // A longer event buffer used for handwriting delegation + // TODO(b/210039666): make this device touch sampling rate dependent. + private static final int LONG_EVENT_BUFFER = EVENT_BUFFER_SIZE * 20; // This must be the looper for the UiThread. private final Looper mLooper; @@ -63,6 +70,9 @@ final class HandwritingModeController { private Runnable mInkWindowInitRunnable; private boolean mRecordingGesture; private int mCurrentDisplayId; + // when set, package names are used for handwriting delegation. + private @Nullable String mDelegatePackageName; + private @Nullable String mDelegatorPackageName; private HandwritingEventReceiverSurface mHandwritingSurface; @@ -137,6 +147,41 @@ final class HandwritingModeController { return mRecordingGesture; } + boolean hasOngoingStylusHandwritingSession() { + return mHandwritingSurface != null && mHandwritingSurface.isIntercepting(); + } + + /** + * Prepare delegation of stylus handwriting to a different editor + * @see InputMethodManager#prepareStylusHandwritingDelegation(View, String) + */ + void prepareStylusHandwritingDelegation( + @NonNull String delegatePackageName, @NonNull String delegatorPackageName) { + mDelegatePackageName = delegatePackageName; + mDelegatorPackageName = delegatorPackageName; + ((ArrayList) mHandwritingBuffer).ensureCapacity(LONG_EVENT_BUFFER); + // TODO(b/210039666): cancel delegation after a timeout or next input method client binding. + } + + @Nullable String getDelegatePackageName() { + return mDelegatePackageName; + } + + @Nullable String getDelegatorPackageName() { + return mDelegatorPackageName; + } + + /** + * Clear any pending handwriting delegation info. + */ + void clearPendingHandwritingDelegation() { + if (DEBUG) { + Slog.d(TAG, "clearPendingHandwritingDelegation"); + } + mDelegatorPackageName = null; + mDelegatePackageName = null; + } + /** * Starts a {@link HandwritingSession} to transfer to the IME. * @@ -223,6 +268,7 @@ final class HandwritingModeController { } } + clearPendingHandwritingDelegation(); mRecordingGesture = false; } @@ -259,7 +305,10 @@ final class HandwritingModeController { mInkWindowInitRunnable = null; } - if (action == MotionEvent.ACTION_UP || action == MotionEvent.ACTION_CANCEL) { + // If handwriting delegation is ongoing, don't clear the buffer so that multiple strokes + // can be buffered across windows. + if (TextUtils.isEmpty(mDelegatePackageName) + && (action == MotionEvent.ACTION_UP || action == MotionEvent.ACTION_CANCEL)) { mRecordingGesture = false; mHandwritingBuffer.clear(); return; diff --git a/services/core/java/com/android/server/inputmethod/InputMethodManagerService.java b/services/core/java/com/android/server/inputmethod/InputMethodManagerService.java index c7e4cd22fc08a..de20299c4537a 100644 --- a/services/core/java/com/android/server/inputmethod/InputMethodManagerService.java +++ b/services/core/java/com/android/server/inputmethod/InputMethodManagerService.java @@ -236,6 +236,8 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub private static final int MSG_FINISH_HANDWRITING = 1110; private static final int MSG_REMOVE_HANDWRITING_WINDOW = 1120; + private static final int MSG_PREPARE_HANDWRITING_DELEGATION = 1130; + private static final int MSG_SET_INTERACTIVE = 3030; private static final int MSG_HARD_KEYBOARD_SWITCH_CHANGED = 4000; @@ -2688,6 +2690,14 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub mHandler.obtainMessage(MSG_RESET_HANDWRITING).sendToTarget(); } + @AnyThread + void schedulePrepareStylusHandwritingDelegation( + @NonNull String delegatePackageName, @NonNull String delegatorPackageName) { + mHandler.obtainMessage( + MSG_PREPARE_HANDWRITING_DELEGATION, + new Pair<>(delegatePackageName, delegatorPackageName)).sendToTarget(); + } + @AnyThread void scheduleRemoveStylusHandwritingWindow() { mHandler.obtainMessage(MSG_REMOVE_HANDWRITING_WINDOW).sendToTarget(); @@ -3304,6 +3314,7 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub "InputMethodManagerService#startStylusHandwriting"); int uid = Binder.getCallingUid(); synchronized (ImfLock.class) { + mHwController.clearPendingHandwritingDelegation(); if (!canInteractWithImeLocked(uid, client, "startStylusHandwriting", null /* statsToken */)) { return; @@ -3331,6 +3342,13 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub "There is no ongoing stylus gesture to start stylus handwriting."); return; } + if (mHwController.hasOngoingStylusHandwritingSession()) { + // prevent duplicate calls to startStylusHandwriting(). + Slog.e(TAG, + "Stylus handwriting session is already ongoing." + + " Ignoring startStylusHandwriting()."); + return; + } if (DEBUG) Slog.v(TAG, "Client requesting Stylus Handwriting to be started"); final IInputMethodInvoker curMethod = getCurMethodLocked(); if (curMethod != null) { @@ -3345,6 +3363,68 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub } } + @Override + public void prepareStylusHandwritingDelegation( + @NonNull IInputMethodClient client, + @NonNull String delegatePackageName, + @NonNull String delegatorPackageName) { + if (!verifyClientAndPackageMatch(client, delegatorPackageName)) { + Slog.w(TAG, "prepareStylusHandwritingDelegation() fail"); + throw new IllegalArgumentException("Delegator doesn't match Uid"); + } + schedulePrepareStylusHandwritingDelegation(delegatePackageName, delegatorPackageName); + } + + @Override + public boolean acceptStylusHandwritingDelegation( + @NonNull IInputMethodClient client, + @NonNull String delegatePackageName, + @NonNull String delegatorPackageName) { + if (!verifyDelegator(client, delegatePackageName, delegatorPackageName)) { + return false; + } + + startStylusHandwriting(client); + return true; + } + + private boolean verifyClientAndPackageMatch( + @NonNull IInputMethodClient client, @NonNull String packageName) { + ClientState cs; + synchronized (ImfLock.class) { + cs = mClients.get(client.asBinder()); + } + if (cs == null) { + throw new IllegalArgumentException("unknown client " + client.asBinder()); + } + return InputMethodUtils.checkIfPackageBelongsToUid( + mPackageManagerInternal, cs.mUid, packageName); + } + + private boolean verifyDelegator( + @NonNull IInputMethodClient client, + @NonNull String delegatePackageName, + @NonNull String delegatorPackageName) { + if (!verifyClientAndPackageMatch(client, delegatePackageName)) { + Slog.w(TAG, "Delegate package does not belong to the same user. Ignoring" + + " startStylusHandwriting"); + return false; + } + synchronized (ImfLock.class) { + if (!delegatorPackageName.equals(mHwController.getDelegatorPackageName())) { + Slog.w(TAG, + "Delegator package does not match. Ignoring startStylusHandwriting"); + return false; + } + if (!delegatePackageName.equals(mHwController.getDelegatePackageName())) { + Slog.w(TAG, + "Delegate package does not match. Ignoring startStylusHandwriting"); + return false; + } + } + return true; + } + @BinderThread @Override public void reportPerceptibleAsync(IBinder windowToken, boolean perceptible) { @@ -4842,6 +4922,13 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub } return true; } + case MSG_PREPARE_HANDWRITING_DELEGATION: + synchronized (ImfLock.class) { + String delegate = (String) ((Pair) msg.obj).first; + String delegator = (String) ((Pair) msg.obj).second; + mHwController.prepareStylusHandwritingDelegation(delegate, delegator); + } + return true; case MSG_START_HANDWRITING: synchronized (ImfLock.class) { IInputMethodInvoker curMethod = getCurMethodLocked();