Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix handwriting support on fake editors using delegation

Browse Source 
We originally added support for Scribe on fake editors in bug 228100684.
However, it doesn't address special cases like when real editor is a
different application package.

This CL introduces IMF APIs to suport handwriting delegation across different
view trees (with or within the same package).

Bug: 266834695
Bug: 228100684
Test: atest StylusHandwritingTest

Change-Id: I3a5d5dc574c5f78697f2a43e11729febf21cc4e2
This commit is contained in:
Taran Singh
2023-02-10 23:33:16 +00:00
parent dd76f4b3fc
commit e35da24d50
6 changed files with 338 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
core/api/current.txt
View File

@@ -55616,6 +55616,8 @@ package android.view.inputmethod {
}
public final class InputMethodManager {
method public boolean acceptStylusHandwritingDelegation(@NonNull android.view.View);
method public boolean acceptStylusHandwritingDelegation(@NonNull android.view.View, @NonNull String);
method public void dispatchKeyEventFromInputMethod(@Nullable android.view.View, @NonNull android.view.KeyEvent);
method public void displayCompletions(android.view.View, android.view.inputmethod.CompletionInfo[]);
method @Nullable public android.view.inputmethod.InputMethodInfo getCurrentInputMethodInfo();
@@ -55637,6 +55639,8 @@ package android.view.inputmethod {
method public boolean isInputMethodSuppressingSpellChecker();
method public boolean isStylusHandwritingAvailable();
method @Deprecated public boolean isWatchingCursor(android.view.View);
method public void prepareStylusHandwritingDelegation(@NonNull android.view.View);
method public void prepareStylusHandwritingDelegation(@NonNull android.view.View, @NonNull String);
method public void restartInput(android.view.View);
method public void sendAppPrivateCommand(android.view.View, String, android.os.Bundle);
method @Deprecated public void setAdditionalInputMethodSubtypes(@NonNull String, @NonNull android.view.inputmethod.InputMethodSubtype[]);

34
core/java/android/view/inputmethod/IInputMethodManagerGlobalInvoker.java
View File

@@ -504,6 +504,40 @@ final class IInputMethodManagerGlobalInvoker {
}
}
@AnyThread
static void prepareStylusHandwritingDelegation(
@NonNull IInputMethodClient client,
@NonNull String delegatePackageName,
@NonNull String delegatorPackageName) {
final IInputMethodManager service = getService();
if (service == null) {
return;
}
try {
service.prepareStylusHandwritingDelegation(
client, delegatePackageName, delegatorPackageName);
} catch (RemoteException e) {
throw e.rethrowFromSystemServer();
}
}
@AnyThread
static boolean acceptStylusHandwritingDelegation(
@NonNull IInputMethodClient client,
@NonNull String delegatePackageName,
@NonNull String delegatorPackageName) {
final IInputMethodManager service = getService();
if (service == null) {
return false;
}
try {
return service.acceptStylusHandwritingDelegation(
client, delegatePackageName, delegatorPackageName);
} catch (RemoteException e) {
throw e.rethrowFromSystemServer();
}
}
@AnyThread
@RequiresPermission(value = Manifest.permission.INTERACT_ACROSS_USERS_FULL, conditional = true)
static boolean isStylusHandwritingAvailableAsUser(@UserIdInt int userId) {

173
core/java/android/view/inputmethod/InputMethodManager.java
View File

@@ -99,6 +99,7 @@ import android.view.WindowManager;
import android.view.WindowManager.LayoutParams.SoftInputModeFlags;
import android.view.autofill.AutofillId;
import android.view.autofill.AutofillManager;
import android.widget.Editor;
import android.window.ImeOnBackInvokedDispatcher;
import android.window.WindowOnBackInvokedDispatcher;
@@ -1552,11 +1553,7 @@ public final class InputMethodManager {
if (fallbackContext == null) {
return false;
}
if (Settings.Global.getInt(fallbackContext.getContentResolver(),
Settings.Global.STYLUS_HANDWRITING_ENABLED, 0) == 0) {
if (DEBUG) {
Log.d(TAG, "Stylus handwriting is not enabled in settings.");
}
if (!isStylusHandwritingEnabled(fallbackContext)) {
return false;
}
return IInputMethodManagerGlobalInvoker.isStylusHandwritingAvailableAsUser(userId);
@@ -2233,37 +2230,175 @@ public final class InputMethodManager {
* @see #isStylusHandwritingAvailable()
*/
public void startStylusHandwriting(@NonNull View view) {
startStylusHandwritingInternal(view, null /* delegatorPackageName */);
}
private boolean startStylusHandwritingInternal(
@NonNull View view, @Nullable String delegatorPackageName) {
Objects.requireNonNull(view);
// Re-dispatch if there is a context mismatch.
final InputMethodManager fallbackImm = getFallbackInputMethodManagerIfNecessary(view);
if (fallbackImm != null) {
fallbackImm.startStylusHandwriting(view);
fallbackImm.startStylusHandwritingInternal(view, delegatorPackageName);
}
Objects.requireNonNull(view);
if (Settings.Global.getInt(view.getContext().getContentResolver(),
Settings.Global.STYLUS_HANDWRITING_ENABLED, 0) == 0) {
Log.d(TAG, "Ignoring startStylusHandwriting(view) as stylus handwriting is disabled");
return;
boolean useDelegation = !TextUtils.isEmpty(delegatorPackageName);
if (!isStylusHandwritingEnabled(view.getContext())) {
Log.w(TAG, "Stylus handwriting pref is disabled. "
+ "Ignoring calls to start stylus handwriting.");
return false;
}
checkFocus();
synchronized (mH) {
if (!hasServedByInputMethodLocked(view)) {
Log.w(TAG,
"Ignoring startStylusHandwriting() as view=" + view + " is not served.");
return;
"Ignoring startStylusHandwriting as view=" + view + " is not served.");
return false;
}
if (view.getViewRootImpl() != mCurRootView) {
Log.w(TAG, "Ignoring startStylusHandwriting: View's window does not have focus.");
return;
Log.w(TAG,
"Ignoring startStylusHandwriting: View's window does not have focus.");
return false;
}
IInputMethodManagerGlobalInvoker.startStylusHandwriting(mClient);
// TODO(b/210039666): do we need any extra work for supporting non-native
// UI toolkits?
if (useDelegation) {
return IInputMethodManagerGlobalInvoker.acceptStylusHandwritingDelegation(
mClient, view.getContext().getOpPackageName(), delegatorPackageName);
} else {
IInputMethodManagerGlobalInvoker.startStylusHandwriting(mClient);
}
return false;
}
}
private boolean isStylusHandwritingEnabled(@NonNull Context context) {
if (Settings.Global.getInt(context.getContentResolver(),
Settings.Global.STYLUS_HANDWRITING_ENABLED, 0) == 0) {
Log.d(TAG, "Stylus handwriting pref is disabled.");
return false;
}
return true;
}
/**
* Prepares delegation of starting stylus handwriting session to a different editor in same
* or different window than the view on which initial handwriting stroke was detected.
*
* Delegation can be used to start stylus handwriting session before the {@link Editor} view or
* its {@link InputConnection} is started. Calling this method starts buffering of stylus
* motion events until {@link #acceptStylusHandwritingDelegation(View)} is called, at which
* point the handwriting session can be started and the buffered stylus motion events will be
* delivered to the IME.
* e.g. Delegation can be used when initial handwriting stroke is
* on a pseudo {@link Editor} like widget (with no {@link InputConnection}) but actual
* {@link Editor} is on a different window.
*
* <p> Note: If an actual {@link Editor} capable of {@link InputConnection} is being scribbled
* upon using stylus, use {@link #startStylusHandwriting(View)} instead.</p>
*
* @param delegatorView the view that receives initial stylus stroke and delegates it to the
* actual editor. Its window must {@link View#hasWindowFocus have focus}.
* @see #prepareStylusHandwritingDelegation(View, String)
* @see #acceptStylusHandwritingDelegation(View)
* @see #startStylusHandwriting(View)
*/
public void prepareStylusHandwritingDelegation(@NonNull View delegatorView) {
prepareStylusHandwritingDelegation(
delegatorView, delegatorView.getContext().getOpPackageName());
}
/**
* Prepares delegation of starting stylus handwriting session to a different editor in same or a
* different window in a different package than the view on which initial handwriting stroke
* was detected.
*
* Delegation can be used to start stylus handwriting session before the {@link Editor} view or
* its {@link InputConnection} is started. Calling this method starts buffering of stylus
* motion events until {@link #acceptStylusHandwritingDelegation(View, String)} is called, at
* which point the handwriting session can be started and the buffered stylus motion events will
* be delivered to the IME.
* e.g. Delegation can be used when initial handwriting stroke is
* on a pseudo {@link Editor} like widget (with no {@link InputConnection}) but actual
* {@link Editor} is on a different window in the given package.
*
* <p>Note: If delegator and delegate are in same package use
* {@link #prepareStylusHandwritingDelegation(View)} instead.</p>
*
* @param delegatorView the view that receives initial stylus stroke and delegates it to the
* actual editor. Its window must {@link View#hasWindowFocus have focus}.
* @param delegatePackageName package name that contains actual {@link Editor} which should
* start stylus handwriting session by calling {@link #acceptStylusHandwritingDelegation}.
* @see #prepareStylusHandwritingDelegation(View)
* @see #acceptStylusHandwritingDelegation(View, String)
*/
public void prepareStylusHandwritingDelegation(
@NonNull View delegatorView, @NonNull String delegatePackageName) {
Objects.requireNonNull(delegatorView);
Objects.requireNonNull(delegatePackageName);
// Re-dispatch if there is a context mismatch.
final InputMethodManager fallbackImm =
getFallbackInputMethodManagerIfNecessary(delegatorView);
if (fallbackImm != null) {
fallbackImm.prepareStylusHandwritingDelegation(delegatorView, delegatePackageName);
}
if (!isStylusHandwritingEnabled(delegatorView.getContext())) {
Log.w(TAG, "Stylus handwriting pref is disabled. "
+ "Ignoring prepareStylusHandwritingDelegation().");
return;
}
IInputMethodManagerGlobalInvoker.prepareStylusHandwritingDelegation(
mClient,
delegatePackageName,
delegatorView.getContext().getOpPackageName());
}
/**
* Accepts and starts a stylus handwriting session on the delegate view, if handwriting
* initiation delegation was previously requested using
* {@link #prepareStylusHandwritingDelegation(View)} from the delegator.
*
* <p>Note: If delegator and delegate are in different application packages, use
* {@link #acceptStylusHandwritingDelegation(View, String)} instead.</p>
*
* @param delegateView delegate view capable of receiving input via {@link InputConnection}
* on which {@link #startStylusHandwriting(View)} will be called.
* @return {@code true} if view belongs to same application package as used in
* {@link #prepareStylusHandwritingDelegation(View)} and handwriting session can start.
* @see #acceptStylusHandwritingDelegation(View, String)
* @see #prepareStylusHandwritingDelegation(View)
*/
public boolean acceptStylusHandwritingDelegation(@NonNull View delegateView) {
return startStylusHandwritingInternal(
delegateView, delegateView.getContext().getOpPackageName());
}
/**
* Accepts and starts a stylus handwriting session on the delegate view, if handwriting
* initiation delegation was previously requested using
* {@link #prepareStylusHandwritingDelegation(View, String)} from te delegator and the view
* belongs to a specified delegate package.
*
* <p>Note: If delegator and delegate are in same application package use
* {@link #acceptStylusHandwritingDelegation(View)} instead.</p>
*
* @param delegateView delegate view capable of receiving input via {@link InputConnection}
* on which {@link #startStylusHandwriting(View)} will be called.
* @param delegatorPackageName package name of the delegator that handled initial stylus stroke.
* @return {@code true} if view belongs to allowed delegate package declared in
* {@link #prepareStylusHandwritingDelegation(View, String)} and handwriting session can start.
* @see #prepareStylusHandwritingDelegation(View, String)
* @see #acceptStylusHandwritingDelegation(View)
*/
public boolean acceptStylusHandwritingDelegation(
@NonNull View delegateView, @NonNull String delegatorPackageName) {
Objects.requireNonNull(delegatorPackageName);
return startStylusHandwritingInternal(delegateView, delegatorPackageName);
}
/**
* This method toggles the input method window display.
* If the input window is already displayed, it gets hidden.

9
core/java/com/android/internal/view/IInputMethodManager.aidl
View File

@@ -148,6 +148,15 @@ interface IInputMethodManager {
/** Start Stylus handwriting session **/
void startStylusHandwriting(in IInputMethodClient client);
/** Prepares delegation of starting stylus handwriting session to a different editor **/
void prepareStylusHandwritingDelegation(in IInputMethodClient client,
in String delegatePackageName,
in String delegatorPackageName);
/** Accepts and starts a stylus handwriting session for the delegate view **/
boolean acceptStylusHandwritingDelegation(in IInputMethodClient client,
in String delegatePackageName, in String delegatorPackageName);
/** Returns {@code true} if currently selected IME supports Stylus handwriting. */
@JavaPassthrough(annotation="@android.annotation.RequiresPermission(value = "
+ "android.Manifest.permission.INTERACT_ACROSS_USERS_FULL, conditional = true)")

51
services/core/java/com/android/server/inputmethod/HandwritingModeController.java
View File

@@ -20,12 +20,14 @@ import static android.view.InputDevice.SOURCE_STYLUS;
import android.Manifest;
import android.annotation.AnyThread;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.annotation.RequiresPermission;
import android.annotation.UiThread;
import android.hardware.input.InputManager;
import android.os.IBinder;
import android.os.Looper;
import android.text.TextUtils;
import android.util.Slog;
import android.view.BatchedInputEventReceiver;
import android.view.Choreographer;
@@ -35,6 +37,8 @@ import android.view.InputEvent;
import android.view.InputEventReceiver;
import android.view.MotionEvent;
import android.view.SurfaceControl;
import android.view.View;
import android.view.inputmethod.InputMethodManager;
import com.android.server.LocalServices;
import com.android.server.input.InputManagerInternal;
@@ -52,6 +56,9 @@ final class HandwritingModeController {
// TODO(b/210039666): flip the flag.
static final boolean DEBUG = true;
private static final int EVENT_BUFFER_SIZE = 100;
// A longer event buffer used for handwriting delegation
// TODO(b/210039666): make this device touch sampling rate dependent.
private static final int LONG_EVENT_BUFFER = EVENT_BUFFER_SIZE * 20;
// This must be the looper for the UiThread.
private final Looper mLooper;
@@ -63,6 +70,9 @@ final class HandwritingModeController {
private Runnable mInkWindowInitRunnable;
private boolean mRecordingGesture;
private int mCurrentDisplayId;
// when set, package names are used for handwriting delegation.
private @Nullable String mDelegatePackageName;
private @Nullable String mDelegatorPackageName;
private HandwritingEventReceiverSurface mHandwritingSurface;
@@ -137,6 +147,41 @@ final class HandwritingModeController {
return mRecordingGesture;
}
boolean hasOngoingStylusHandwritingSession() {
return mHandwritingSurface != null && mHandwritingSurface.isIntercepting();
}
/**
* Prepare delegation of stylus handwriting to a different editor
* @see InputMethodManager#prepareStylusHandwritingDelegation(View, String)
*/
void prepareStylusHandwritingDelegation(
@NonNull String delegatePackageName, @NonNull String delegatorPackageName) {
mDelegatePackageName = delegatePackageName;
mDelegatorPackageName = delegatorPackageName;
((ArrayList) mHandwritingBuffer).ensureCapacity(LONG_EVENT_BUFFER);
// TODO(b/210039666): cancel delegation after a timeout or next input method client binding.
}
@Nullable String getDelegatePackageName() {
return mDelegatePackageName;
}
@Nullable String getDelegatorPackageName() {
return mDelegatorPackageName;
}
/**
* Clear any pending handwriting delegation info.
*/
void clearPendingHandwritingDelegation() {
if (DEBUG) {
Slog.d(TAG, "clearPendingHandwritingDelegation");
}
mDelegatorPackageName = null;
mDelegatePackageName = null;
}
/**
* Starts a {@link HandwritingSession} to transfer to the IME.
*
@@ -223,6 +268,7 @@ final class HandwritingModeController {
}
}
clearPendingHandwritingDelegation();
mRecordingGesture = false;
}
@@ -259,7 +305,10 @@ final class HandwritingModeController {
mInkWindowInitRunnable = null;
}
if (action == MotionEvent.ACTION_UP || action == MotionEvent.ACTION_CANCEL) {
// If handwriting delegation is ongoing, don't clear the buffer so that multiple strokes
// can be buffered across windows.
if (TextUtils.isEmpty(mDelegatePackageName)
&& (action == MotionEvent.ACTION_UP || action == MotionEvent.ACTION_CANCEL)) {
mRecordingGesture = false;
mHandwritingBuffer.clear();
return;

87
services/core/java/com/android/server/inputmethod/InputMethodManagerService.java
View File

@@ -236,6 +236,8 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub
private static final int MSG_FINISH_HANDWRITING = 1110;
private static final int MSG_REMOVE_HANDWRITING_WINDOW = 1120;
private static final int MSG_PREPARE_HANDWRITING_DELEGATION = 1130;
private static final int MSG_SET_INTERACTIVE = 3030;
private static final int MSG_HARD_KEYBOARD_SWITCH_CHANGED = 4000;
@@ -2688,6 +2690,14 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub
mHandler.obtainMessage(MSG_RESET_HANDWRITING).sendToTarget();
}
@AnyThread
void schedulePrepareStylusHandwritingDelegation(
@NonNull String delegatePackageName, @NonNull String delegatorPackageName) {
mHandler.obtainMessage(
MSG_PREPARE_HANDWRITING_DELEGATION,
new Pair<>(delegatePackageName, delegatorPackageName)).sendToTarget();
}
@AnyThread
void scheduleRemoveStylusHandwritingWindow() {
mHandler.obtainMessage(MSG_REMOVE_HANDWRITING_WINDOW).sendToTarget();
@@ -3304,6 +3314,7 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub
"InputMethodManagerService#startStylusHandwriting");
int uid = Binder.getCallingUid();
synchronized (ImfLock.class) {
mHwController.clearPendingHandwritingDelegation();
if (!canInteractWithImeLocked(uid, client, "startStylusHandwriting",
null /* statsToken */)) {
return;
@@ -3331,6 +3342,13 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub
"There is no ongoing stylus gesture to start stylus handwriting.");
return;
}
if (mHwController.hasOngoingStylusHandwritingSession()) {
// prevent duplicate calls to startStylusHandwriting().
Slog.e(TAG,
"Stylus handwriting session is already ongoing."
+ " Ignoring startStylusHandwriting().");
return;
}
if (DEBUG) Slog.v(TAG, "Client requesting Stylus Handwriting to be started");
final IInputMethodInvoker curMethod = getCurMethodLocked();
if (curMethod != null) {
@@ -3345,6 +3363,68 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub
}
}
@Override
public void prepareStylusHandwritingDelegation(
@NonNull IInputMethodClient client,
@NonNull String delegatePackageName,
@NonNull String delegatorPackageName) {
if (!verifyClientAndPackageMatch(client, delegatorPackageName)) {
Slog.w(TAG, "prepareStylusHandwritingDelegation() fail");
throw new IllegalArgumentException("Delegator doesn't match Uid");
}
schedulePrepareStylusHandwritingDelegation(delegatePackageName, delegatorPackageName);
}
@Override
public boolean acceptStylusHandwritingDelegation(
@NonNull IInputMethodClient client,
@NonNull String delegatePackageName,
@NonNull String delegatorPackageName) {
if (!verifyDelegator(client, delegatePackageName, delegatorPackageName)) {
return false;
}
startStylusHandwriting(client);
return true;
}
private boolean verifyClientAndPackageMatch(
@NonNull IInputMethodClient client, @NonNull String packageName) {
ClientState cs;
synchronized (ImfLock.class) {
cs = mClients.get(client.asBinder());
}
if (cs == null) {
throw new IllegalArgumentException("unknown client " + client.asBinder());
}
return InputMethodUtils.checkIfPackageBelongsToUid(
mPackageManagerInternal, cs.mUid, packageName);
}
private boolean verifyDelegator(
@NonNull IInputMethodClient client,
@NonNull String delegatePackageName,
@NonNull String delegatorPackageName) {
if (!verifyClientAndPackageMatch(client, delegatePackageName)) {
Slog.w(TAG, "Delegate package does not belong to the same user. Ignoring"
+ " startStylusHandwriting");
return false;
}
synchronized (ImfLock.class) {
if (!delegatorPackageName.equals(mHwController.getDelegatorPackageName())) {
Slog.w(TAG,
"Delegator package does not match. Ignoring startStylusHandwriting");
return false;
}
if (!delegatePackageName.equals(mHwController.getDelegatePackageName())) {
Slog.w(TAG,
"Delegate package does not match. Ignoring startStylusHandwriting");
return false;
}
}
return true;
}
@BinderThread
@Override
public void reportPerceptibleAsync(IBinder windowToken, boolean perceptible) {
@@ -4842,6 +4922,13 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub
}
return true;
}
case MSG_PREPARE_HANDWRITING_DELEGATION:
synchronized (ImfLock.class) {
String delegate = (String) ((Pair) msg.obj).first;
String delegator = (String) ((Pair) msg.obj).second;
mHwController.prepareStylusHandwritingDelegation(delegate, delegator);
}
return true;
case MSG_START_HANDWRITING:
synchronized (ImfLock.class) {
IInputMethodInvoker curMethod = getCurMethodLocked();