Merge "Fix handwriting support on fake editors using delegation" into udc-dev am: 2cbf9414be

Original change: https://googleplex-android-review.googlesource.com/c/platform/frameworks/base/+/21469658

Change-Id: Iae080491845660280a50ddd153b454f626baf126
Signed-off-by: Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>

core/api/current.txt

@@ -55618,6 +55618,8 @@ package android.view.inputmethod {
   }
 
   public final class InputMethodManager {
+    method public boolean acceptStylusHandwritingDelegation(@NonNull android.view.View);
+    method public boolean acceptStylusHandwritingDelegation(@NonNull android.view.View, @NonNull String);
     method public void dispatchKeyEventFromInputMethod(@Nullable android.view.View, @NonNull android.view.KeyEvent);
     method public void displayCompletions(android.view.View, android.view.inputmethod.CompletionInfo[]);
     method @Nullable public android.view.inputmethod.InputMethodInfo getCurrentInputMethodInfo();
@@ -55639,6 +55641,8 @@ package android.view.inputmethod {
     method public boolean isInputMethodSuppressingSpellChecker();
     method public boolean isStylusHandwritingAvailable();
     method @Deprecated public boolean isWatchingCursor(android.view.View);
+    method public void prepareStylusHandwritingDelegation(@NonNull android.view.View);
+    method public void prepareStylusHandwritingDelegation(@NonNull android.view.View, @NonNull String);
     method public void restartInput(android.view.View);
     method public void sendAppPrivateCommand(android.view.View, String, android.os.Bundle);
     method @Deprecated public void setAdditionalInputMethodSubtypes(@NonNull String, @NonNull android.view.inputmethod.InputMethodSubtype[]);

core/java/android/view/inputmethod/IInputMethodManagerGlobalInvoker.java

@@ -504,6 +504,40 @@ final class IInputMethodManagerGlobalInvoker {
         }
     }
 
+    @AnyThread
+    static void prepareStylusHandwritingDelegation(
+            @NonNull IInputMethodClient client,
+            @NonNull String delegatePackageName,
+            @NonNull String delegatorPackageName) {
+        final IInputMethodManager service = getService();
+        if (service == null) {
+            return;
+        }
+        try {
+            service.prepareStylusHandwritingDelegation(
+                    client, delegatePackageName, delegatorPackageName);
+        } catch (RemoteException e) {
+            throw e.rethrowFromSystemServer();
+        }
+    }
+
+    @AnyThread
+    static boolean acceptStylusHandwritingDelegation(
+            @NonNull IInputMethodClient client,
+            @NonNull String delegatePackageName,
+            @NonNull String delegatorPackageName) {
+        final IInputMethodManager service = getService();
+        if (service == null) {
+            return false;
+        }
+        try {
+            return service.acceptStylusHandwritingDelegation(
+                    client, delegatePackageName, delegatorPackageName);
+        } catch (RemoteException e) {
+            throw e.rethrowFromSystemServer();
+        }
+    }
+
     @AnyThread
     @RequiresPermission(value = Manifest.permission.INTERACT_ACROSS_USERS_FULL, conditional = true)
     static boolean isStylusHandwritingAvailableAsUser(@UserIdInt int userId) {

core/java/android/view/inputmethod/InputMethodManager.java

@@ -99,6 +99,7 @@ import android.view.WindowManager;
 import android.view.WindowManager.LayoutParams.SoftInputModeFlags;
 import android.view.autofill.AutofillId;
 import android.view.autofill.AutofillManager;
+import android.widget.Editor;
 import android.window.ImeOnBackInvokedDispatcher;
 import android.window.WindowOnBackInvokedDispatcher;
 
@@ -1552,11 +1553,7 @@ public final class InputMethodManager {
         if (fallbackContext == null) {
             return false;
         }
-        if (Settings.Global.getInt(fallbackContext.getContentResolver(),
+        if (!isStylusHandwritingEnabled(fallbackContext)) {
-                Settings.Global.STYLUS_HANDWRITING_ENABLED, 0) == 0) {
-            if (DEBUG) {
-                Log.d(TAG, "Stylus handwriting is not enabled in settings.");
-            }
             return false;
         }
         return IInputMethodManagerGlobalInvoker.isStylusHandwritingAvailableAsUser(userId);
@@ -2233,37 +2230,175 @@ public final class InputMethodManager {
      * @see #isStylusHandwritingAvailable()
      */
     public void startStylusHandwriting(@NonNull View view) {
+        startStylusHandwritingInternal(view, null /* delegatorPackageName */);
+    }
+
+    private boolean startStylusHandwritingInternal(
+            @NonNull View view, @Nullable String delegatorPackageName) {
+        Objects.requireNonNull(view);
+
         // Re-dispatch if there is a context mismatch.
         final InputMethodManager fallbackImm = getFallbackInputMethodManagerIfNecessary(view);
         if (fallbackImm != null) {
-            fallbackImm.startStylusHandwriting(view);
+            fallbackImm.startStylusHandwritingInternal(view, delegatorPackageName);
         }
-        Objects.requireNonNull(view);
 
-        if (Settings.Global.getInt(view.getContext().getContentResolver(),
+        boolean useDelegation = !TextUtils.isEmpty(delegatorPackageName);
-                Settings.Global.STYLUS_HANDWRITING_ENABLED, 0) == 0) {
+        if (!isStylusHandwritingEnabled(view.getContext())) {
-            Log.d(TAG, "Ignoring startStylusHandwriting(view) as stylus handwriting is disabled");
+            Log.w(TAG, "Stylus handwriting pref is disabled. "
-            return;
+                    + "Ignoring calls to start stylus handwriting.");
+            return false;
         }
 
         checkFocus();
         synchronized (mH) {
             if (!hasServedByInputMethodLocked(view)) {
                 Log.w(TAG,
-                        "Ignoring startStylusHandwriting() as view=" + view + " is not served.");
+                        "Ignoring startStylusHandwriting as view=" + view + " is not served.");
-                return;
+                return false;
             }
             if (view.getViewRootImpl() != mCurRootView) {
-                Log.w(TAG, "Ignoring startStylusHandwriting: View's window does not have focus.");
+                Log.w(TAG,
-                return;
+                        "Ignoring startStylusHandwriting: View's window does not have focus.");
+                return false;
             }
-
+            if (useDelegation) {
-            IInputMethodManagerGlobalInvoker.startStylusHandwriting(mClient);
+                return IInputMethodManagerGlobalInvoker.acceptStylusHandwritingDelegation(
-            // TODO(b/210039666): do we need any extra work for supporting non-native
+                        mClient, view.getContext().getOpPackageName(), delegatorPackageName);
-            //   UI toolkits?
+            } else {
+                IInputMethodManagerGlobalInvoker.startStylusHandwriting(mClient);
+            }
+            return false;
         }
     }
 
+    private boolean isStylusHandwritingEnabled(@NonNull Context context) {
+        if (Settings.Global.getInt(context.getContentResolver(),
+                Settings.Global.STYLUS_HANDWRITING_ENABLED, 0) == 0) {
+            Log.d(TAG, "Stylus handwriting pref is disabled.");
+            return false;
+        }
+        return true;
+    }
+
+    /**
+     * Prepares delegation of starting stylus handwriting session to a different editor in same
+     * or different window than the view on which initial handwriting stroke was detected.
+     *
+     * Delegation can be used to start stylus handwriting session before the {@link Editor} view or
+     * its {@link InputConnection} is started. Calling this method starts buffering of stylus
+     * motion events until {@link #acceptStylusHandwritingDelegation(View)} is called, at which
+     * point the handwriting session can be started and the buffered stylus motion events will be
+     * delivered to the IME.
+     * e.g. Delegation can be used when initial handwriting stroke is
+     * on a pseudo {@link Editor} like widget (with no {@link InputConnection}) but actual
+     * {@link Editor} is on a different window.
+     *
+     * <p> Note: If an actual {@link Editor} capable of {@link InputConnection} is being scribbled
+     * upon using stylus, use {@link #startStylusHandwriting(View)} instead.</p>
+     *
+     * @param delegatorView the view that receives initial stylus stroke and delegates it to the
+     *  actual editor. Its window must {@link View#hasWindowFocus have focus}.
+     * @see #prepareStylusHandwritingDelegation(View, String)
+     * @see #acceptStylusHandwritingDelegation(View)
+     * @see #startStylusHandwriting(View)
+     */
+    public void prepareStylusHandwritingDelegation(@NonNull View delegatorView) {
+        prepareStylusHandwritingDelegation(
+                delegatorView, delegatorView.getContext().getOpPackageName());
+    }
+
+    /**
+     * Prepares delegation of starting stylus handwriting session to a different editor in same or a
+     * different window in a different package than the view on which initial handwriting stroke
+     * was detected.
+     *
+     * Delegation can be used to start stylus handwriting session before the {@link Editor} view or
+     * its {@link InputConnection} is started. Calling this method starts buffering of stylus
+     * motion events until {@link #acceptStylusHandwritingDelegation(View, String)} is called, at
+     * which point the handwriting session can be started and the buffered stylus motion events will
+     * be delivered to the IME.
+     * e.g. Delegation can be used when initial handwriting stroke is
+     * on a pseudo {@link Editor} like widget (with no {@link InputConnection}) but actual
+     * {@link Editor} is on a different window in the given package.
+     *
+     * <p>Note: If delegator and delegate are in same package use
+     * {@link #prepareStylusHandwritingDelegation(View)} instead.</p>
+     *
+     * @param delegatorView  the view that receives initial stylus stroke and delegates it to the
+     * actual editor. Its window must {@link View#hasWindowFocus have focus}.
+     * @param delegatePackageName package name that contains actual {@link Editor} which should
+     *  start stylus handwriting session by calling {@link #acceptStylusHandwritingDelegation}.
+     * @see #prepareStylusHandwritingDelegation(View)
+     * @see #acceptStylusHandwritingDelegation(View, String)
+     */
+    public void prepareStylusHandwritingDelegation(
+            @NonNull View delegatorView, @NonNull String delegatePackageName) {
+        Objects.requireNonNull(delegatorView);
+        Objects.requireNonNull(delegatePackageName);
+
+        // Re-dispatch if there is a context mismatch.
+        final InputMethodManager fallbackImm =
+                getFallbackInputMethodManagerIfNecessary(delegatorView);
+        if (fallbackImm != null) {
+            fallbackImm.prepareStylusHandwritingDelegation(delegatorView, delegatePackageName);
+        }
+
+        if (!isStylusHandwritingEnabled(delegatorView.getContext())) {
+            Log.w(TAG, "Stylus handwriting pref is disabled. "
+                    + "Ignoring prepareStylusHandwritingDelegation().");
+            return;
+        }
+        IInputMethodManagerGlobalInvoker.prepareStylusHandwritingDelegation(
+                mClient,
+                delegatePackageName,
+                delegatorView.getContext().getOpPackageName());
+    }
+
+    /**
+     * Accepts and starts a stylus handwriting session on the delegate view, if handwriting
+     * initiation delegation was previously requested using
+     * {@link #prepareStylusHandwritingDelegation(View)} from the delegator.
+     *
+     * <p>Note: If delegator and delegate are in different application packages, use
+     * {@link #acceptStylusHandwritingDelegation(View, String)} instead.</p>
+     *
+     * @param delegateView delegate view capable of receiving input via {@link InputConnection}
+     *  on which {@link #startStylusHandwriting(View)} will be called.
+     * @return {@code true} if view belongs to same application package as used in
+     *  {@link #prepareStylusHandwritingDelegation(View)} and handwriting session can start.
+     * @see #acceptStylusHandwritingDelegation(View, String)
+     * @see #prepareStylusHandwritingDelegation(View)
+     */
+    public boolean acceptStylusHandwritingDelegation(@NonNull View delegateView) {
+        return startStylusHandwritingInternal(
+                delegateView, delegateView.getContext().getOpPackageName());
+    }
+
+    /**
+     * Accepts and starts a stylus handwriting session on the delegate view, if handwriting
+     * initiation delegation was previously requested using
+     * {@link #prepareStylusHandwritingDelegation(View, String)} from te delegator and the view
+     * belongs to a specified delegate package.
+     *
+     * <p>Note: If delegator and delegate are in same application package use
+     * {@link #acceptStylusHandwritingDelegation(View)} instead.</p>
+     *
+     * @param delegateView delegate view capable of receiving input via {@link InputConnection}
+     *  on which {@link #startStylusHandwriting(View)} will be called.
+     * @param delegatorPackageName package name of the delegator that handled initial stylus stroke.
+     * @return {@code true} if view belongs to allowed delegate package declared in
+     *  {@link #prepareStylusHandwritingDelegation(View, String)} and handwriting session can start.
+     * @see #prepareStylusHandwritingDelegation(View, String)
+     * @see #acceptStylusHandwritingDelegation(View)
+     */
+    public boolean acceptStylusHandwritingDelegation(
+            @NonNull View delegateView, @NonNull String delegatorPackageName) {
+        Objects.requireNonNull(delegatorPackageName);
+
+        return startStylusHandwritingInternal(delegateView, delegatorPackageName);
+    }
+
     /**
      * This method toggles the input method window display.
      * If the input window is already displayed, it gets hidden.

core/java/com/android/internal/view/IInputMethodManager.aidl

@@ -148,6 +148,15 @@ interface IInputMethodManager {
     /** Start Stylus handwriting session **/
     void startStylusHandwriting(in IInputMethodClient client);
 
+    /** Prepares delegation of starting stylus handwriting session to a different editor **/
+    void prepareStylusHandwritingDelegation(in IInputMethodClient client,
+                in String delegatePackageName,
+                in String delegatorPackageName);
+
+    /** Accepts and starts a stylus handwriting session for the delegate view **/
+    boolean acceptStylusHandwritingDelegation(in IInputMethodClient client,
+                in String delegatePackageName, in String delegatorPackageName);
+
     /** Returns {@code true} if currently selected IME supports Stylus handwriting. */
     @JavaPassthrough(annotation="@android.annotation.RequiresPermission(value = "
             + "android.Manifest.permission.INTERACT_ACROSS_USERS_FULL, conditional = true)")

services/core/java/com/android/server/inputmethod/HandwritingModeController.java

@@ -20,12 +20,14 @@ import static android.view.InputDevice.SOURCE_STYLUS;
 
 import android.Manifest;
 import android.annotation.AnyThread;
+import android.annotation.NonNull;
 import android.annotation.Nullable;
 import android.annotation.RequiresPermission;
 import android.annotation.UiThread;
 import android.hardware.input.InputManager;
 import android.os.IBinder;
 import android.os.Looper;
+import android.text.TextUtils;
 import android.util.Slog;
 import android.view.BatchedInputEventReceiver;
 import android.view.Choreographer;
@@ -35,6 +37,8 @@ import android.view.InputEvent;
 import android.view.InputEventReceiver;
 import android.view.MotionEvent;
 import android.view.SurfaceControl;
+import android.view.View;
+import android.view.inputmethod.InputMethodManager;
 
 import com.android.server.LocalServices;
 import com.android.server.input.InputManagerInternal;
@@ -52,6 +56,9 @@ final class HandwritingModeController {
     // TODO(b/210039666): flip the flag.
     static final boolean DEBUG = true;
     private static final int EVENT_BUFFER_SIZE = 100;
+    // A longer event buffer used for handwriting delegation
+    // TODO(b/210039666): make this device touch sampling rate dependent.
+    private static final int LONG_EVENT_BUFFER = EVENT_BUFFER_SIZE * 20;
 
     // This must be the looper for the UiThread.
     private final Looper mLooper;
@@ -63,6 +70,9 @@ final class HandwritingModeController {
     private Runnable mInkWindowInitRunnable;
     private boolean mRecordingGesture;
     private int mCurrentDisplayId;
+    // when set, package names are used for handwriting delegation.
+    private @Nullable String mDelegatePackageName;
+    private @Nullable String mDelegatorPackageName;
 
     private HandwritingEventReceiverSurface mHandwritingSurface;
 
@@ -137,6 +147,41 @@ final class HandwritingModeController {
         return mRecordingGesture;
     }
 
+    boolean hasOngoingStylusHandwritingSession() {
+        return mHandwritingSurface != null && mHandwritingSurface.isIntercepting();
+    }
+
+    /**
+     * Prepare delegation of stylus handwriting to a different editor
+     * @see InputMethodManager#prepareStylusHandwritingDelegation(View, String)
+     */
+    void prepareStylusHandwritingDelegation(
+            @NonNull String delegatePackageName, @NonNull String delegatorPackageName) {
+        mDelegatePackageName = delegatePackageName;
+        mDelegatorPackageName = delegatorPackageName;
+        ((ArrayList) mHandwritingBuffer).ensureCapacity(LONG_EVENT_BUFFER);
+        // TODO(b/210039666): cancel delegation after a timeout or next input method client binding.
+    }
+
+    @Nullable String getDelegatePackageName() {
+        return mDelegatePackageName;
+    }
+
+    @Nullable String getDelegatorPackageName() {
+        return mDelegatorPackageName;
+    }
+
+    /**
+     * Clear any pending handwriting delegation info.
+     */
+    void clearPendingHandwritingDelegation() {
+        if (DEBUG) {
+            Slog.d(TAG, "clearPendingHandwritingDelegation");
+        }
+        mDelegatorPackageName = null;
+        mDelegatePackageName = null;
+    }
+
     /**
      * Starts a {@link HandwritingSession} to transfer to the IME.
      *
@@ -223,6 +268,7 @@ final class HandwritingModeController {
             }
         }
 
+        clearPendingHandwritingDelegation();
         mRecordingGesture = false;
     }
 
@@ -259,7 +305,10 @@ final class HandwritingModeController {
             mInkWindowInitRunnable = null;
         }
 
-        if (action == MotionEvent.ACTION_UP || action == MotionEvent.ACTION_CANCEL) {
+        // If handwriting delegation is ongoing, don't clear the buffer so that multiple strokes
+        // can be buffered across windows.
+        if (TextUtils.isEmpty(mDelegatePackageName)
+                && (action == MotionEvent.ACTION_UP || action == MotionEvent.ACTION_CANCEL)) {
             mRecordingGesture = false;
             mHandwritingBuffer.clear();
             return;

services/core/java/com/android/server/inputmethod/InputMethodManagerService.java

@@ -236,6 +236,8 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub
     private static final int MSG_FINISH_HANDWRITING = 1110;
     private static final int MSG_REMOVE_HANDWRITING_WINDOW = 1120;
 
+    private static final int MSG_PREPARE_HANDWRITING_DELEGATION = 1130;
+
     private static final int MSG_SET_INTERACTIVE = 3030;
 
     private static final int MSG_HARD_KEYBOARD_SWITCH_CHANGED = 4000;
@@ -2688,6 +2690,14 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub
         mHandler.obtainMessage(MSG_RESET_HANDWRITING).sendToTarget();
     }
 
+    @AnyThread
+    void schedulePrepareStylusHandwritingDelegation(
+            @NonNull String delegatePackageName, @NonNull String delegatorPackageName) {
+        mHandler.obtainMessage(
+                MSG_PREPARE_HANDWRITING_DELEGATION,
+                new Pair<>(delegatePackageName, delegatorPackageName)).sendToTarget();
+    }
+
     @AnyThread
     void scheduleRemoveStylusHandwritingWindow() {
         mHandler.obtainMessage(MSG_REMOVE_HANDWRITING_WINDOW).sendToTarget();
@@ -3304,6 +3314,7 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub
                     "InputMethodManagerService#startStylusHandwriting");
             int uid = Binder.getCallingUid();
             synchronized (ImfLock.class) {
+                mHwController.clearPendingHandwritingDelegation();
                 if (!canInteractWithImeLocked(uid, client, "startStylusHandwriting",
                         null /* statsToken */)) {
                     return;
@@ -3331,6 +3342,13 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub
                                 "There is no ongoing stylus gesture to start stylus handwriting.");
                         return;
                     }
+                    if (mHwController.hasOngoingStylusHandwritingSession()) {
+                        // prevent duplicate calls to startStylusHandwriting().
+                        Slog.e(TAG,
+                                "Stylus handwriting session is already ongoing."
+                                        + " Ignoring startStylusHandwriting().");
+                        return;
+                    }
                     if (DEBUG) Slog.v(TAG, "Client requesting Stylus Handwriting to be started");
                     final IInputMethodInvoker curMethod = getCurMethodLocked();
                     if (curMethod != null) {
@@ -3345,6 +3363,68 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub
         }
     }
 
+    @Override
+    public void prepareStylusHandwritingDelegation(
+            @NonNull IInputMethodClient client,
+            @NonNull String delegatePackageName,
+            @NonNull String delegatorPackageName) {
+        if (!verifyClientAndPackageMatch(client, delegatorPackageName)) {
+            Slog.w(TAG, "prepareStylusHandwritingDelegation() fail");
+            throw new IllegalArgumentException("Delegator doesn't match Uid");
+        }
+        schedulePrepareStylusHandwritingDelegation(delegatePackageName, delegatorPackageName);
+    }
+
+    @Override
+    public boolean acceptStylusHandwritingDelegation(
+            @NonNull IInputMethodClient client,
+            @NonNull String delegatePackageName,
+            @NonNull String delegatorPackageName) {
+        if (!verifyDelegator(client, delegatePackageName, delegatorPackageName)) {
+            return false;
+        }
+
+        startStylusHandwriting(client);
+        return true;
+    }
+
+    private boolean verifyClientAndPackageMatch(
+            @NonNull IInputMethodClient client, @NonNull String packageName) {
+        ClientState cs;
+        synchronized (ImfLock.class) {
+            cs = mClients.get(client.asBinder());
+        }
+        if (cs == null) {
+            throw new IllegalArgumentException("unknown client " + client.asBinder());
+        }
+        return InputMethodUtils.checkIfPackageBelongsToUid(
+                mPackageManagerInternal, cs.mUid, packageName);
+    }
+
+    private boolean verifyDelegator(
+            @NonNull IInputMethodClient client,
+            @NonNull String delegatePackageName,
+            @NonNull String delegatorPackageName) {
+        if (!verifyClientAndPackageMatch(client, delegatePackageName)) {
+            Slog.w(TAG, "Delegate package does not belong to the same user. Ignoring"
+                    + " startStylusHandwriting");
+            return false;
+        }
+        synchronized (ImfLock.class) {
+            if (!delegatorPackageName.equals(mHwController.getDelegatorPackageName())) {
+                Slog.w(TAG,
+                        "Delegator package does not match. Ignoring startStylusHandwriting");
+                return false;
+            }
+            if (!delegatePackageName.equals(mHwController.getDelegatePackageName())) {
+                Slog.w(TAG,
+                        "Delegate package does not match. Ignoring startStylusHandwriting");
+                return false;
+            }
+        }
+        return true;
+    }
+
     @BinderThread
     @Override
     public void reportPerceptibleAsync(IBinder windowToken, boolean perceptible) {
@@ -4852,6 +4932,13 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub
                 }
                 return true;
             }
+            case MSG_PREPARE_HANDWRITING_DELEGATION:
+                synchronized (ImfLock.class) {
+                    String delegate = (String) ((Pair) msg.obj).first;
+                    String delegator = (String) ((Pair) msg.obj).second;
+                    mHwController.prepareStylusHandwritingDelegation(delegate, delegator);
+                }
+                return true;
             case MSG_START_HANDWRITING:
                 synchronized (ImfLock.class) {
                     IInputMethodInvoker curMethod = getCurMethodLocked();