Note: If an actual {@link Editor} capable of {@link InputConnection} is being scribbled + * upon using stylus, use {@link #startStylusHandwriting(View)} instead.
+ * + * @param delegatorView the view that receives initial stylus stroke and delegates it to the + * actual editor. Its window must {@link View#hasWindowFocus have focus}. + * @see #prepareStylusHandwritingDelegation(View, String) + * @see #acceptStylusHandwritingDelegation(View) + * @see #startStylusHandwriting(View) + */ + public void prepareStylusHandwritingDelegation(@NonNull View delegatorView) { + prepareStylusHandwritingDelegation( + delegatorView, delegatorView.getContext().getOpPackageName()); + } + + /** + * Prepares delegation of starting stylus handwriting session to a different editor in same or a + * different window in a different package than the view on which initial handwriting stroke + * was detected. + * + * Delegation can be used to start stylus handwriting session before the {@link Editor} view or + * its {@link InputConnection} is started. Calling this method starts buffering of stylus + * motion events until {@link #acceptStylusHandwritingDelegation(View, String)} is called, at + * which point the handwriting session can be started and the buffered stylus motion events will + * be delivered to the IME. + * e.g. Delegation can be used when initial handwriting stroke is + * on a pseudo {@link Editor} like widget (with no {@link InputConnection}) but actual + * {@link Editor} is on a different window in the given package. + * + *Note: If delegator and delegate are in same package use + * {@link #prepareStylusHandwritingDelegation(View)} instead.
+ * + * @param delegatorView the view that receives initial stylus stroke and delegates it to the + * actual editor. Its window must {@link View#hasWindowFocus have focus}. + * @param delegatePackageName package name that contains actual {@link Editor} which should + * start stylus handwriting session by calling {@link #acceptStylusHandwritingDelegation}. + * @see #prepareStylusHandwritingDelegation(View) + * @see #acceptStylusHandwritingDelegation(View, String) + */ + public void prepareStylusHandwritingDelegation( + @NonNull View delegatorView, @NonNull String delegatePackageName) { + Objects.requireNonNull(delegatorView); + Objects.requireNonNull(delegatePackageName); + + // Re-dispatch if there is a context mismatch. + final InputMethodManager fallbackImm = + getFallbackInputMethodManagerIfNecessary(delegatorView); + if (fallbackImm != null) { + fallbackImm.prepareStylusHandwritingDelegation(delegatorView, delegatePackageName); + } + + if (!isStylusHandwritingEnabled(delegatorView.getContext())) { + Log.w(TAG, "Stylus handwriting pref is disabled. " + + "Ignoring prepareStylusHandwritingDelegation()."); + return; + } + IInputMethodManagerGlobalInvoker.prepareStylusHandwritingDelegation( + mClient, + delegatePackageName, + delegatorView.getContext().getOpPackageName()); + } + + /** + * Accepts and starts a stylus handwriting session on the delegate view, if handwriting + * initiation delegation was previously requested using + * {@link #prepareStylusHandwritingDelegation(View)} from the delegator. + * + *Note: If delegator and delegate are in different application packages, use + * {@link #acceptStylusHandwritingDelegation(View, String)} instead.
+ * + * @param delegateView delegate view capable of receiving input via {@link InputConnection} + * on which {@link #startStylusHandwriting(View)} will be called. + * @return {@code true} if view belongs to same application package as used in + * {@link #prepareStylusHandwritingDelegation(View)} and handwriting session can start. + * @see #acceptStylusHandwritingDelegation(View, String) + * @see #prepareStylusHandwritingDelegation(View) + */ + public boolean acceptStylusHandwritingDelegation(@NonNull View delegateView) { + return startStylusHandwritingInternal( + delegateView, delegateView.getContext().getOpPackageName()); + } + + /** + * Accepts and starts a stylus handwriting session on the delegate view, if handwriting + * initiation delegation was previously requested using + * {@link #prepareStylusHandwritingDelegation(View, String)} from te delegator and the view + * belongs to a specified delegate package. + * + *Note: If delegator and delegate are in same application package use + * {@link #acceptStylusHandwritingDelegation(View)} instead.
+ * + * @param delegateView delegate view capable of receiving input via {@link InputConnection} + * on which {@link #startStylusHandwriting(View)} will be called. + * @param delegatorPackageName package name of the delegator that handled initial stylus stroke. + * @return {@code true} if view belongs to allowed delegate package declared in + * {@link #prepareStylusHandwritingDelegation(View, String)} and handwriting session can start. + * @see #prepareStylusHandwritingDelegation(View, String) + * @see #acceptStylusHandwritingDelegation(View) + */ + public boolean acceptStylusHandwritingDelegation( + @NonNull View delegateView, @NonNull String delegatorPackageName) { + Objects.requireNonNull(delegatorPackageName); + + return startStylusHandwritingInternal(delegateView, delegatorPackageName); + } + /** * This method toggles the input method window display. * If the input window is already displayed, it gets hidden. diff --git a/core/java/com/android/internal/view/IInputMethodManager.aidl b/core/java/com/android/internal/view/IInputMethodManager.aidl index 9116cb3ec9fa2..5805d0e050d16 100644 --- a/core/java/com/android/internal/view/IInputMethodManager.aidl +++ b/core/java/com/android/internal/view/IInputMethodManager.aidl @@ -148,6 +148,15 @@ interface IInputMethodManager { /** Start Stylus handwriting session **/ void startStylusHandwriting(in IInputMethodClient client); + /** Prepares delegation of starting stylus handwriting session to a different editor **/ + void prepareStylusHandwritingDelegation(in IInputMethodClient client, + in String delegatePackageName, + in String delegatorPackageName); + + /** Accepts and starts a stylus handwriting session for the delegate view **/ + boolean acceptStylusHandwritingDelegation(in IInputMethodClient client, + in String delegatePackageName, in String delegatorPackageName); + /** Returns {@code true} if currently selected IME supports Stylus handwriting. */ @JavaPassthrough(annotation="@android.annotation.RequiresPermission(value = " + "android.Manifest.permission.INTERACT_ACROSS_USERS_FULL, conditional = true)") diff --git a/services/core/java/com/android/server/inputmethod/HandwritingModeController.java b/services/core/java/com/android/server/inputmethod/HandwritingModeController.java index 1c7294f8a9d5c..eb4dba63dea9b 100644 --- a/services/core/java/com/android/server/inputmethod/HandwritingModeController.java +++ b/services/core/java/com/android/server/inputmethod/HandwritingModeController.java @@ -20,12 +20,14 @@ import static android.view.InputDevice.SOURCE_STYLUS; import android.Manifest; import android.annotation.AnyThread; +import android.annotation.NonNull; import android.annotation.Nullable; import android.annotation.RequiresPermission; import android.annotation.UiThread; import android.hardware.input.InputManager; import android.os.IBinder; import android.os.Looper; +import android.text.TextUtils; import android.util.Slog; import android.view.BatchedInputEventReceiver; import android.view.Choreographer; @@ -35,6 +37,8 @@ import android.view.InputEvent; import android.view.InputEventReceiver; import android.view.MotionEvent; import android.view.SurfaceControl; +import android.view.View; +import android.view.inputmethod.InputMethodManager; import com.android.server.LocalServices; import com.android.server.input.InputManagerInternal; @@ -52,6 +56,9 @@ final class HandwritingModeController { // TODO(b/210039666): flip the flag. static final boolean DEBUG = true; private static final int EVENT_BUFFER_SIZE = 100; + // A longer event buffer used for handwriting delegation + // TODO(b/210039666): make this device touch sampling rate dependent. + private static final int LONG_EVENT_BUFFER = EVENT_BUFFER_SIZE * 20; // This must be the looper for the UiThread. private final Looper mLooper; @@ -63,6 +70,9 @@ final class HandwritingModeController { private Runnable mInkWindowInitRunnable; private boolean mRecordingGesture; private int mCurrentDisplayId; + // when set, package names are used for handwriting delegation. + private @Nullable String mDelegatePackageName; + private @Nullable String mDelegatorPackageName; private HandwritingEventReceiverSurface mHandwritingSurface; @@ -137,6 +147,41 @@ final class HandwritingModeController { return mRecordingGesture; } + boolean hasOngoingStylusHandwritingSession() { + return mHandwritingSurface != null && mHandwritingSurface.isIntercepting(); + } + + /** + * Prepare delegation of stylus handwriting to a different editor + * @see InputMethodManager#prepareStylusHandwritingDelegation(View, String) + */ + void prepareStylusHandwritingDelegation( + @NonNull String delegatePackageName, @NonNull String delegatorPackageName) { + mDelegatePackageName = delegatePackageName; + mDelegatorPackageName = delegatorPackageName; + ((ArrayList) mHandwritingBuffer).ensureCapacity(LONG_EVENT_BUFFER); + // TODO(b/210039666): cancel delegation after a timeout or next input method client binding. + } + + @Nullable String getDelegatePackageName() { + return mDelegatePackageName; + } + + @Nullable String getDelegatorPackageName() { + return mDelegatorPackageName; + } + + /** + * Clear any pending handwriting delegation info. + */ + void clearPendingHandwritingDelegation() { + if (DEBUG) { + Slog.d(TAG, "clearPendingHandwritingDelegation"); + } + mDelegatorPackageName = null; + mDelegatePackageName = null; + } + /** * Starts a {@link HandwritingSession} to transfer to the IME. * @@ -223,6 +268,7 @@ final class HandwritingModeController { } } + clearPendingHandwritingDelegation(); mRecordingGesture = false; } @@ -259,7 +305,10 @@ final class HandwritingModeController { mInkWindowInitRunnable = null; } - if (action == MotionEvent.ACTION_UP || action == MotionEvent.ACTION_CANCEL) { + // If handwriting delegation is ongoing, don't clear the buffer so that multiple strokes + // can be buffered across windows. + if (TextUtils.isEmpty(mDelegatePackageName) + && (action == MotionEvent.ACTION_UP || action == MotionEvent.ACTION_CANCEL)) { mRecordingGesture = false; mHandwritingBuffer.clear(); return; diff --git a/services/core/java/com/android/server/inputmethod/InputMethodManagerService.java b/services/core/java/com/android/server/inputmethod/InputMethodManagerService.java index c7e4cd22fc08a..de20299c4537a 100644 --- a/services/core/java/com/android/server/inputmethod/InputMethodManagerService.java +++ b/services/core/java/com/android/server/inputmethod/InputMethodManagerService.java @@ -236,6 +236,8 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub private static final int MSG_FINISH_HANDWRITING = 1110; private static final int MSG_REMOVE_HANDWRITING_WINDOW = 1120; + private static final int MSG_PREPARE_HANDWRITING_DELEGATION = 1130; + private static final int MSG_SET_INTERACTIVE = 3030; private static final int MSG_HARD_KEYBOARD_SWITCH_CHANGED = 4000; @@ -2688,6 +2690,14 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub mHandler.obtainMessage(MSG_RESET_HANDWRITING).sendToTarget(); } + @AnyThread + void schedulePrepareStylusHandwritingDelegation( + @NonNull String delegatePackageName, @NonNull String delegatorPackageName) { + mHandler.obtainMessage( + MSG_PREPARE_HANDWRITING_DELEGATION, + new Pair<>(delegatePackageName, delegatorPackageName)).sendToTarget(); + } + @AnyThread void scheduleRemoveStylusHandwritingWindow() { mHandler.obtainMessage(MSG_REMOVE_HANDWRITING_WINDOW).sendToTarget(); @@ -3304,6 +3314,7 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub "InputMethodManagerService#startStylusHandwriting"); int uid = Binder.getCallingUid(); synchronized (ImfLock.class) { + mHwController.clearPendingHandwritingDelegation(); if (!canInteractWithImeLocked(uid, client, "startStylusHandwriting", null /* statsToken */)) { return; @@ -3331,6 +3342,13 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub "There is no ongoing stylus gesture to start stylus handwriting."); return; } + if (mHwController.hasOngoingStylusHandwritingSession()) { + // prevent duplicate calls to startStylusHandwriting(). + Slog.e(TAG, + "Stylus handwriting session is already ongoing." + + " Ignoring startStylusHandwriting()."); + return; + } if (DEBUG) Slog.v(TAG, "Client requesting Stylus Handwriting to be started"); final IInputMethodInvoker curMethod = getCurMethodLocked(); if (curMethod != null) { @@ -3345,6 +3363,68 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub } } + @Override + public void prepareStylusHandwritingDelegation( + @NonNull IInputMethodClient client, + @NonNull String delegatePackageName, + @NonNull String delegatorPackageName) { + if (!verifyClientAndPackageMatch(client, delegatorPackageName)) { + Slog.w(TAG, "prepareStylusHandwritingDelegation() fail"); + throw new IllegalArgumentException("Delegator doesn't match Uid"); + } + schedulePrepareStylusHandwritingDelegation(delegatePackageName, delegatorPackageName); + } + + @Override + public boolean acceptStylusHandwritingDelegation( + @NonNull IInputMethodClient client, + @NonNull String delegatePackageName, + @NonNull String delegatorPackageName) { + if (!verifyDelegator(client, delegatePackageName, delegatorPackageName)) { + return false; + } + + startStylusHandwriting(client); + return true; + } + + private boolean verifyClientAndPackageMatch( + @NonNull IInputMethodClient client, @NonNull String packageName) { + ClientState cs; + synchronized (ImfLock.class) { + cs = mClients.get(client.asBinder()); + } + if (cs == null) { + throw new IllegalArgumentException("unknown client " + client.asBinder()); + } + return InputMethodUtils.checkIfPackageBelongsToUid( + mPackageManagerInternal, cs.mUid, packageName); + } + + private boolean verifyDelegator( + @NonNull IInputMethodClient client, + @NonNull String delegatePackageName, + @NonNull String delegatorPackageName) { + if (!verifyClientAndPackageMatch(client, delegatePackageName)) { + Slog.w(TAG, "Delegate package does not belong to the same user. Ignoring" + + " startStylusHandwriting"); + return false; + } + synchronized (ImfLock.class) { + if (!delegatorPackageName.equals(mHwController.getDelegatorPackageName())) { + Slog.w(TAG, + "Delegator package does not match. Ignoring startStylusHandwriting"); + return false; + } + if (!delegatePackageName.equals(mHwController.getDelegatePackageName())) { + Slog.w(TAG, + "Delegate package does not match. Ignoring startStylusHandwriting"); + return false; + } + } + return true; + } + @BinderThread @Override public void reportPerceptibleAsync(IBinder windowToken, boolean perceptible) { @@ -4842,6 +4922,13 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub } return true; } + case MSG_PREPARE_HANDWRITING_DELEGATION: + synchronized (ImfLock.class) { + String delegate = (String) ((Pair) msg.obj).first; + String delegator = (String) ((Pair) msg.obj).second; + mHwController.prepareStylusHandwritingDelegation(delegate, delegator); + } + return true; case MSG_START_HANDWRITING: synchronized (ImfLock.class) { IInputMethodInvoker curMethod = getCurMethodLocked();