From 44dc2805b5e1c432b45043f19f5eea7f3f087158 Mon Sep 17 00:00:00 2001 From: Taran Singh Date: Tue, 16 Nov 2021 01:35:47 +0000 Subject: [PATCH] Scribe in IMF: startStylusHandwriting & lifecycle 2/N Introduce InputMethodManager.startStylusHandwriting(View) API and IME lifecycle. Bug: 203086136 Test: atest StylusHandwritingTest Change-Id: I7b066c2841b713e7a00ae2ea4ca0ce04aad751c6 --- core/api/current.txt | 5 + .../IInputMethodWrapper.java | 44 +++++- .../android/inputmethodservice/InkWindow.java | 88 ++++++++++++ .../InputMethodService.java | 133 +++++++++++++++++- .../android/view/inputmethod/InputMethod.java | 22 ++- .../view/inputmethod/InputMethodManager.java | 44 ++++++ .../IInputMethodPrivilegedOperations.aidl | 1 + .../InputMethodPrivilegedOperations.java | 16 +++ .../android/internal/view/IInputMethod.aidl | 7 +- .../internal/view/IInputMethodManager.aidl | 3 + core/res/res/values/attrs.xml | 17 ++- .../InputMethodBindingController.java | 17 ++- .../InputMethodManagerService.java | 112 ++++++++++++--- .../server/wm/WindowManagerService.java | 8 +- 14 files changed, 487 insertions(+), 30 deletions(-) create mode 100644 core/java/android/inputmethodservice/InkWindow.java diff --git a/core/api/current.txt b/core/api/current.txt index 06f82e0c50a53..61511d423ccab 100644 --- a/core/api/current.txt +++ b/core/api/current.txt @@ -19740,6 +19740,7 @@ package android.inputmethodservice { @UiContext public class InputMethodService extends android.inputmethodservice.AbstractInputMethodService { ctor public InputMethodService(); method @Deprecated public boolean enableHardwareAcceleration(); + method public final void finishStylusHandwriting(); method public int getBackDisposition(); method public int getCandidatesHiddenVisibility(); method public android.view.inputmethod.InputBinding getCurrentInputBinding(); @@ -19749,6 +19750,7 @@ package android.inputmethodservice { method @Deprecated public int getInputMethodWindowRecommendedHeight(); method public android.view.LayoutInflater getLayoutInflater(); method public int getMaxWidth(); + method @Nullable public final android.view.Window getStylusHandwritingWindow(); method public CharSequence getTextForImeAction(int); method public android.app.Dialog getWindow(); method public void hideStatusIcon(); @@ -19779,6 +19781,7 @@ package android.inputmethodservice { method public void onFinishCandidatesView(boolean); method public void onFinishInput(); method public void onFinishInputView(boolean); + method public void onFinishStylusHandwriting(); method public void onInitializeInterface(); method public boolean onInlineSuggestionsResponse(@NonNull android.view.inputmethod.InlineSuggestionsResponse); method public boolean onKeyDown(int, android.view.KeyEvent); @@ -19789,6 +19792,7 @@ package android.inputmethodservice { method public void onStartCandidatesView(android.view.inputmethod.EditorInfo, boolean); method public void onStartInput(android.view.inputmethod.EditorInfo, boolean); method public void onStartInputView(android.view.inputmethod.EditorInfo, boolean); + method public boolean onStartStylusHandwriting(); method public void onUnbindInput(); method @Deprecated public void onUpdateCursor(android.graphics.Rect); method public void onUpdateCursorAnchorInfo(android.view.inputmethod.CursorAnchorInfo); @@ -52885,6 +52889,7 @@ package android.view.inputmethod { method public boolean showSoftInput(android.view.View, int, android.os.ResultReceiver); method @Deprecated public void showSoftInputFromInputMethod(android.os.IBinder, int); method @Deprecated public void showStatusIcon(android.os.IBinder, String, @DrawableRes int); + method public void startStylusHandwriting(@NonNull android.view.View); method @Deprecated public boolean switchToLastInputMethod(android.os.IBinder); method @Deprecated public boolean switchToNextInputMethod(android.os.IBinder, boolean); method @Deprecated public void toggleSoftInput(int, int); diff --git a/core/java/android/inputmethodservice/IInputMethodWrapper.java b/core/java/android/inputmethodservice/IInputMethodWrapper.java index e30594fb9da7b..6f15588c07247 100644 --- a/core/java/android/inputmethodservice/IInputMethodWrapper.java +++ b/core/java/android/inputmethodservice/IInputMethodWrapper.java @@ -18,6 +18,7 @@ package android.inputmethodservice; import android.annotation.BinderThread; import android.annotation.MainThread; +import android.annotation.NonNull; import android.annotation.Nullable; import android.compat.annotation.UnsupportedAppUsage; import android.content.Context; @@ -29,6 +30,7 @@ import android.os.RemoteException; import android.os.ResultReceiver; import android.util.Log; import android.view.InputChannel; +import android.view.MotionEvent; import android.view.inputmethod.EditorInfo; import android.view.inputmethod.InputBinding; import android.view.inputmethod.InputConnection; @@ -50,6 +52,7 @@ import com.android.internal.view.InlineSuggestionsRequestInfo; import java.io.FileDescriptor; import java.io.PrintWriter; import java.lang.ref.WeakReference; +import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -74,6 +77,8 @@ class IInputMethodWrapper extends IInputMethod.Stub private static final int DO_HIDE_SOFT_INPUT = 70; private static final int DO_CHANGE_INPUTMETHOD_SUBTYPE = 80; private static final int DO_CREATE_INLINE_SUGGESTIONS_REQUEST = 90; + private static final int DO_CAN_START_STYLUS_HANDWRITING = 100; + private static final int DO_START_STYLUS_HANDWRITING = 110; final WeakReference mTarget; final Context mContext; @@ -169,7 +174,8 @@ class IInputMethodWrapper extends IInputMethod.Stub SomeArgs args = (SomeArgs) msg.obj; try { inputMethod.initializeInternal((IBinder) args.arg1, - (IInputMethodPrivilegedOperations) args.arg2, msg.arg1); + (IInputMethodPrivilegedOperations) args.arg2, msg.arg1, + (boolean) args.arg3); } finally { args.recycle(); } @@ -229,13 +235,25 @@ class IInputMethodWrapper extends IInputMethod.Stub case DO_CHANGE_INPUTMETHOD_SUBTYPE: inputMethod.changeInputMethodSubtype((InputMethodSubtype)msg.obj); return; - case DO_CREATE_INLINE_SUGGESTIONS_REQUEST: + case DO_CREATE_INLINE_SUGGESTIONS_REQUEST: { final SomeArgs args = (SomeArgs) msg.obj; inputMethod.onCreateInlineSuggestionsRequest( (InlineSuggestionsRequestInfo) args.arg1, (IInlineSuggestionsRequestCallback) args.arg2); args.recycle(); return; + } + case DO_CAN_START_STYLUS_HANDWRITING: { + inputMethod.canStartStylusHandwriting(msg.arg1); + return; + } + case DO_START_STYLUS_HANDWRITING: { + final SomeArgs args = (SomeArgs) msg.obj; + inputMethod.startStylusHandwriting((InputChannel) args.arg1, + (List) args.arg2); + args.recycle(); + return; + } } Log.w(TAG, "Unhandled message code: " + msg.what); @@ -272,9 +290,10 @@ class IInputMethodWrapper extends IInputMethod.Stub @BinderThread @Override public void initializeInternal(IBinder token, IInputMethodPrivilegedOperations privOps, - int configChanges) { + int configChanges, boolean stylusHwSupported) { mCaller.executeOrSendMessage( - mCaller.obtainMessageIOO(DO_INITIALIZE_INTERNAL, configChanges, token, privOps)); + mCaller.obtainMessageIOOO( + DO_INITIALIZE_INTERNAL, configChanges, token, privOps, stylusHwSupported)); } @BinderThread @@ -383,4 +402,21 @@ class IInputMethodWrapper extends IInputMethod.Stub mCaller.executeOrSendMessage(mCaller.obtainMessageO(DO_CHANGE_INPUTMETHOD_SUBTYPE, subtype)); } + + @BinderThread + @Override + public void canStartStylusHandwriting(int requestId) + throws RemoteException { + mCaller.executeOrSendMessage( + mCaller.obtainMessageI(DO_CAN_START_STYLUS_HANDWRITING, requestId)); + } + + @BinderThread + @Override + public void startStylusHandwriting(@NonNull InputChannel channel, + @Nullable List stylusEvents) + throws RemoteException { + mCaller.executeOrSendMessage( + mCaller.obtainMessageOO(DO_START_STYLUS_HANDWRITING, channel, stylusEvents)); + } } diff --git a/core/java/android/inputmethodservice/InkWindow.java b/core/java/android/inputmethodservice/InkWindow.java new file mode 100644 index 0000000000000..e11d63562ce33 --- /dev/null +++ b/core/java/android/inputmethodservice/InkWindow.java @@ -0,0 +1,88 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.inputmethodservice; + +import static android.view.WindowManager.LayoutParams; +import static android.view.WindowManager.LayoutParams.FLAG_LAYOUT_IN_SCREEN; +import static android.view.WindowManager.LayoutParams.FLAG_LAYOUT_NO_LIMITS; +import static android.view.WindowManager.LayoutParams.FLAG_NOT_FOCUSABLE; +import static android.view.WindowManager.LayoutParams.FLAG_NOT_TOUCHABLE; + +import android.annotation.NonNull; +import android.content.Context; +import android.os.IBinder; +import android.util.Slog; +import android.view.View; +import android.view.WindowManager; + +import com.android.internal.policy.PhoneWindow; + +/** + * Window of type {@code LayoutParams.TYPE_INPUT_METHOD_DIALOG} for drawing + * Handwriting Ink on screen. + * @hide + */ +final class InkWindow extends PhoneWindow { + + private final WindowManager mWindowManager; + + public InkWindow(@NonNull Context context) { + super(context); + + setType(LayoutParams.TYPE_INPUT_METHOD); + final LayoutParams attrs = getAttributes(); + attrs.layoutInDisplayCutoutMode = LayoutParams.LAYOUT_IN_DISPLAY_CUTOUT_MODE_ALWAYS; + attrs.setFitInsetsTypes(0); + setAttributes(attrs); + // Ink window is not touchable with finger. + addFlags(FLAG_LAYOUT_IN_SCREEN | FLAG_LAYOUT_NO_LIMITS | FLAG_NOT_TOUCHABLE + | FLAG_NOT_FOCUSABLE); + setBackgroundDrawableResource(android.R.color.transparent); + setLayout(LayoutParams.MATCH_PARENT, LayoutParams.MATCH_PARENT); + mWindowManager = context.getSystemService(WindowManager.class); + } + + /** + * Method to show InkWindow on screen. + * Emulates internal behavior similar to Dialog.show(). + */ + void show() { + if (getDecorView() == null) { + Slog.i(InputMethodService.TAG, "DecorView is not set for InkWindow. show() failed."); + return; + } + getDecorView().setVisibility(View.VISIBLE); + mWindowManager.addView(getDecorView(), getAttributes()); + } + + /** + * Method to hide InkWindow from screen. + * Emulates internal behavior similar to Dialog.hide(). + * @param remove set {@code true} to remove InkWindow surface completely. + */ + void hide(boolean remove) { + if (getDecorView() != null) { + getDecorView().setVisibility(remove ? View.GONE : View.INVISIBLE); + } + } + + void setToken(@NonNull IBinder token) { + WindowManager.LayoutParams lp = getAttributes(); + lp.token = token; + setAttributes(lp); + } +} diff --git a/core/java/android/inputmethodservice/InputMethodService.java b/core/java/android/inputmethodservice/InputMethodService.java index afaa085c7cbdd..588f7cdde555f 100644 --- a/core/java/android/inputmethodservice/InputMethodService.java +++ b/core/java/android/inputmethodservice/InputMethodService.java @@ -94,6 +94,7 @@ import android.util.PrintWriterPrinter; import android.util.Printer; import android.util.proto.ProtoOutputStream; import android.view.Gravity; +import android.view.InputChannel; import android.view.KeyCharacterMap; import android.view.KeyEvent; import android.view.LayoutInflater; @@ -119,6 +120,7 @@ import android.view.inputmethod.InputConnection; import android.view.inputmethod.InputContentInfo; import android.view.inputmethod.InputMethod; import android.view.inputmethod.InputMethodEditorTraceProto.InputMethodServiceTraceProto; +import android.view.inputmethod.InputMethodInfo; import android.view.inputmethod.InputMethodManager; import android.view.inputmethod.InputMethodSubtype; import android.widget.FrameLayout; @@ -142,6 +144,7 @@ import java.io.PrintWriter; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.util.ArrayList; +import java.util.List; import java.util.Objects; /** @@ -520,10 +523,14 @@ public class InputMethodService extends AbstractInputMethodService { private boolean mAutomotiveHideNavBarForKeyboard; private boolean mIsAutomotive; + private boolean mHandwritingStarted; private Handler mHandler; private boolean mImeSurfaceScheduledForRemoval; private ImsConfigurationTracker mConfigTracker = new ImsConfigurationTracker(); + /** Stylus handwriting Ink window. */ + private InkWindow mInkWindow; + /** * An opaque {@link Binder} token of window requesting {@link InputMethodImpl#showSoftInput} * The original app window token is passed from client app window. @@ -598,11 +605,15 @@ public class InputMethodService extends AbstractInputMethodService { @MainThread @Override public final void initializeInternal(@NonNull IBinder token, - IInputMethodPrivilegedOperations privilegedOperations, int configChanges) { + IInputMethodPrivilegedOperations privilegedOperations, int configChanges, + boolean stylusHwSupported) { Trace.traceBegin(TRACE_TAG_WINDOW_MANAGER, "IMS.initializeInternal"); mConfigTracker.onInitialize(configChanges); mPrivOps.set(privilegedOperations); InputMethodPrivilegedOperationsRegistry.put(token, mPrivOps); + if (stylusHwSupported) { + mInkWindow = new InkWindow(mWindow.getContext()); + } attachToken(token); Trace.traceEnd(TRACE_TAG_WINDOW_MANAGER); } @@ -635,6 +646,9 @@ public class InputMethodService extends AbstractInputMethodService { attachToWindowToken(token); mToken = token; mWindow.setToken(token); + if (mInkWindow != null) { + mInkWindow.setToken(token); + } } /** @@ -816,6 +830,49 @@ public class InputMethodService extends AbstractInputMethodService { Trace.traceEnd(TRACE_TAG_WINDOW_MANAGER); } + /** + * {@inheritDoc} + * @hide + */ + @Override + public void canStartStylusHandwriting(int requestId) { + if (DEBUG) Log.v(TAG, "canStartStylusHandwriting()"); + if (mHandwritingStarted) { + Log.d(TAG, "There is an ongoing Handwriting session. ignoring."); + return; + } + if (!mInputStarted) { + Log.d(TAG, "Input should have started before starting Stylus handwriting."); + return; + } + if (onStartStylusHandwriting()) { + mPrivOps.onStylusHandwritingReady(requestId); + } else { + Log.i(TAG, "IME is not ready. Can't start Stylus Handwriting"); + } + } + + /** + * {@inheritDoc} + * @hide + */ + @MainThread + @Override + public void startStylusHandwriting( + @NonNull InputChannel channel, @Nullable List stylusEvents) { + if (DEBUG) Log.v(TAG, "startStylusHandwriting()"); + if (mHandwritingStarted) { + return; + } + + mHandwritingStarted = true; + mShowInputRequested = false; + + mInkWindow.show(); + // TODO: deliver previous @param stylusEvents + // TODO: create spy receiver for @param channel + } + /** * {@inheritDoc} */ @@ -2181,6 +2238,77 @@ public class InputMethodService extends AbstractInputMethodService { } } + /** + * Called when an app requests stylus handwriting + * {@link InputMethodManager#startStylusHandwriting(View)}. + * + * This will always be preceded by {@link #onStartInput(EditorInfo, boolean)} for the + * {@link EditorInfo} and {@link InputConnection} for which stylus handwriting is being + * requested. + * + * If the IME supports handwriting for the current input, it should return {@code true}, + * ensure its inking views are attached to the {@link #getStylusHandwritingWindow()}, and handle + * stylus input received on the ink window via {@link #getCurrentInputConnection()}. + * @return {@code true} if IME can honor the request, {@code false} if IME cannot at this time. + */ + public boolean onStartStylusHandwriting() { + // Intentionally empty + return false; + } + + /** + * Called when the current stylus handwriting session was finished (either by the system or + * via {@link #finishStylusHandwriting()}. + * + * When this is called, the ink window has been made invisible, and the IME no longer + * intercepts handwriting-related {@code MotionEvent}s. + */ + public void onFinishStylusHandwriting() { + // Intentionally empty + } + + /** + * Returns the stylus handwriting inking window. + * IMEs supporting stylus input are expected to attach their inking views to this + * window (e.g. with {@link Window#setContentView(View)} )). Handwriting-related + * {@link MotionEvent}s are dispatched to the attached view hierarchy. + * + * Note: This returns {@code null} if IME doesn't support stylus handwriting + * i.e. if {@link InputMethodInfo#supportsStylusHandwriting()} is false. + * This method should be called after {@link #onStartStylusHandwriting()}. + * @see #onStartStylusHandwriting() + */ + @Nullable + public final Window getStylusHandwritingWindow() { + return mInkWindow; + } + + /** + * Finish the current stylus handwriting session. + * + * This dismisses the {@link #getStylusHandwritingWindow ink window} and stops intercepting + * stylus {@code MotionEvent}s. + * + * Note for IME developers: Call this method at any time to finish current handwriting session. + * Generally, this should be invoked after a short timeout, giving the user enough time + * to start the next stylus stroke, if any. + * + * Handwriting session will be finished by framework on next {@link #onFinishInput()}. + */ + public final void finishStylusHandwriting() { + if (DEBUG) Log.v(TAG, "finishStylusHandwriting()"); + if (mInkWindow == null) { + return; + } + if (!mHandwritingStarted) { + return; + } + + mHandwritingStarted = false; + mInkWindow.hide(false /* remove */); + onFinishStylusHandwriting(); + } + /** * The system has decided that it may be time to show your input method. * This is called due to a corresponding call to your @@ -2450,6 +2578,9 @@ public class InputMethodService extends AbstractInputMethodService { mInputStarted = false; mStartedInputConnection = null; mCurCompletions = null; + if (mInkWindow != null) { + finishStylusHandwriting(); + } } void doStartInput(InputConnection ic, EditorInfo attribute, boolean restarting) { diff --git a/core/java/android/view/inputmethod/InputMethod.java b/core/java/android/view/inputmethod/InputMethod.java index 5b2068ff16cdc..71e7d5778250b 100644 --- a/core/java/android/view/inputmethod/InputMethod.java +++ b/core/java/android/view/inputmethod/InputMethod.java @@ -26,12 +26,16 @@ import android.os.IBinder; import android.os.RemoteException; import android.os.ResultReceiver; import android.util.Log; +import android.view.InputChannel; +import android.view.MotionEvent; import android.view.View; import com.android.internal.inputmethod.IInputMethodPrivilegedOperations; import com.android.internal.view.IInlineSuggestionsRequestCallback; import com.android.internal.view.InlineSuggestionsRequestInfo; +import java.util.List; + /** * The InputMethod interface represents an input method which can generate key * events and text, such as digital, email addresses, CJK characters, other @@ -100,11 +104,13 @@ public interface InputMethod { * operations that are allowed only to the * current IME. * @param configChanges {@link InputMethodInfo#getConfigChanges()} declared by IME. + * @param stylusHwSupported {@link InputMethodInfo#supportsStylusHandwriting()} declared by IME. * @hide */ @MainThread default void initializeInternal(IBinder token, - IInputMethodPrivilegedOperations privilegedOperations, int configChanges) { + IInputMethodPrivilegedOperations privilegedOperations, int configChanges, + boolean stylusHwSupported) { attachToken(token); } @@ -384,4 +390,18 @@ public interface InputMethod { */ public void setCurrentHideInputToken(IBinder hideInputToken); + /** + * Checks if IME is ready to start stylus handwriting session. + * If yes, {@link #startStylusHandwriting(InputChannel, List)} is called. + * @param requestId + * @hide + */ + void canStartStylusHandwriting(int requestId); + + /** + * Start stylus handwriting session. + * @hide + */ + void startStylusHandwriting(@NonNull InputChannel channel, @Nullable List events); + } diff --git a/core/java/android/view/inputmethod/InputMethodManager.java b/core/java/android/view/inputmethod/InputMethodManager.java index 3583cd4c6d8b2..6fc246eb2514b 100644 --- a/core/java/android/view/inputmethod/InputMethodManager.java +++ b/core/java/android/view/inputmethod/InputMethodManager.java @@ -1760,6 +1760,50 @@ public final class InputMethodManager { } } + /** + * Start stylus handwriting session. + * + * If supported by the current input method, a stylus handwriting session is started on the + * given View, capturing all stylus input and converting it to InputConnection commands. + * + * If handwriting mode is started successfully by the IME, any currently dispatched stylus + * pointers will be {@code android.view.MotionEvent#FLAG_CANCELED} cancelled. + * + * If Stylus handwriting mode is not supported or cannot be fulfilled for any reason by IME, + * request will be ignored and Stylus touch will continue as normal touch input. + * + * @param view the View for which stylus handwriting is requested. It and + * {@link View#hasWindowFocus its window} must be {@link View#hasFocus focused}. + */ + public void startStylusHandwriting(@NonNull View view) { + // Re-dispatch if there is a context mismatch. + final InputMethodManager fallbackImm = getFallbackInputMethodManagerIfNecessary(view); + if (fallbackImm != null) { + fallbackImm.startStylusHandwriting(view); + } + + checkFocus(); + synchronized (mH) { + if (view == null || !hasServedByInputMethodLocked(view)) { + Log.w(TAG, + "Ignoring startStylusHandwriting() as view=" + view + " is not served."); + return; + } + if (view.getViewRootImpl() != mCurRootView) { + Log.w(TAG, "Ignoring startStylusHandwriting: View's window does not have focus."); + return; + } + + try { + mService.startStylusHandwriting(mClient); + // TODO(b/210039666): do we need any extra work for supporting non-native + // UI toolkits? + } catch (RemoteException e) { + throw e.rethrowFromSystemServer(); + } + } + } + /** * This method toggles the input method window display. * If the input window is already displayed, it gets hidden. diff --git a/core/java/com/android/internal/inputmethod/IInputMethodPrivilegedOperations.aidl b/core/java/com/android/internal/inputmethod/IInputMethodPrivilegedOperations.aidl index 9d0f209d8b2dd..08bc8c7fa339a 100644 --- a/core/java/com/android/internal/inputmethod/IInputMethodPrivilegedOperations.aidl +++ b/core/java/com/android/internal/inputmethod/IInputMethodPrivilegedOperations.aidl @@ -42,4 +42,5 @@ oneway interface IInputMethodPrivilegedOperations { void shouldOfferSwitchingToNextInputMethod(in AndroidFuture future /* T=Boolean */); void notifyUserActionAsync(); void applyImeVisibilityAsync(IBinder showOrHideInputToken, boolean setVisible); + void onStylusHandwritingReady(int requestId); } diff --git a/core/java/com/android/internal/inputmethod/InputMethodPrivilegedOperations.java b/core/java/com/android/internal/inputmethod/InputMethodPrivilegedOperations.java index d4cc376385b88..7ebcc88b593b4 100644 --- a/core/java/com/android/internal/inputmethod/InputMethodPrivilegedOperations.java +++ b/core/java/com/android/internal/inputmethod/InputMethodPrivilegedOperations.java @@ -394,4 +394,20 @@ public final class InputMethodPrivilegedOperations { throw e.rethrowFromSystemServer(); } } + + /** + * Calls {@link IInputMethodPrivilegedOperations#onStylusHandwritingReady()} + */ + @AnyThread + public void onStylusHandwritingReady(int requestId) { + final IInputMethodPrivilegedOperations ops = mOps.getAndWarnIfNull(); + if (ops == null) { + return; + } + try { + ops.onStylusHandwritingReady(requestId); + } catch (RemoteException e) { + throw e.rethrowFromSystemServer(); + } + } } diff --git a/core/java/com/android/internal/view/IInputMethod.aidl b/core/java/com/android/internal/view/IInputMethod.aidl index 139660a29ede4..402fa64036a05 100644 --- a/core/java/com/android/internal/view/IInputMethod.aidl +++ b/core/java/com/android/internal/view/IInputMethod.aidl @@ -19,6 +19,7 @@ package com.android.internal.view; import android.os.IBinder; import android.os.ResultReceiver; import android.view.InputChannel; +import android.view.MotionEvent; import android.view.inputmethod.EditorInfo; import android.view.inputmethod.InputBinding; import android.view.inputmethod.InputMethodSubtype; @@ -36,7 +37,7 @@ import com.android.internal.view.InlineSuggestionsRequestInfo; */ oneway interface IInputMethod { void initializeInternal(IBinder token, IInputMethodPrivilegedOperations privOps, - int configChanges); + int configChanges, boolean stylusHwSupported); void onCreateInlineSuggestionsRequest(in InlineSuggestionsRequestInfo requestInfo, in IInlineSuggestionsRequestCallback cb); @@ -59,4 +60,8 @@ oneway interface IInputMethod { void hideSoftInput(in IBinder hideInputToken, int flags, in ResultReceiver resultReceiver); void changeInputMethodSubtype(in InputMethodSubtype subtype); + + void canStartStylusHandwriting(int requestId); + + void startStylusHandwriting(in InputChannel channel, in List events); } diff --git a/core/java/com/android/internal/view/IInputMethodManager.aidl b/core/java/com/android/internal/view/IInputMethodManager.aidl index 2dc7c42c95a9b..0df3e870b80cf 100644 --- a/core/java/com/android/internal/view/IInputMethodManager.aidl +++ b/core/java/com/android/internal/view/IInputMethodManager.aidl @@ -81,4 +81,7 @@ interface IInputMethodManager { void startImeTrace(); // Stops an ime trace. void stopImeTrace(); + + /** Start Stylus handwriting session **/ + void startStylusHandwriting(in IInputMethodClient client); } diff --git a/core/res/res/values/attrs.xml b/core/res/res/values/attrs.xml index 1705371a7f4a9..c25b7e8f368b7 100644 --- a/core/res/res/values/attrs.xml +++ b/core/res/res/values/attrs.xml @@ -3638,7 +3638,22 @@ to re-retrieve all resources (including view layouts, drawables, etc) to correctly handle any configuration change.--> - + diff --git a/services/core/java/com/android/server/inputmethod/InputMethodBindingController.java b/services/core/java/com/android/server/inputmethod/InputMethodBindingController.java index 220d790d12086..3af51f4ba55f4 100644 --- a/services/core/java/com/android/server/inputmethod/InputMethodBindingController.java +++ b/services/core/java/com/android/server/inputmethod/InputMethodBindingController.java @@ -263,6 +263,13 @@ final class InputMethodBindingController { return mVisibleBound; } + /** + * Returns {@code true} if current IME supports Stylus Handwriting. + */ + boolean supportsStylusHandwriting() { + return mSupportsStylusHw; + } + /** * Used to bring IME service up to visible adjustment while it is being shown. */ @@ -302,15 +309,17 @@ final class InputMethodBindingController { return; } if (DEBUG) Slog.v(TAG, "Initiating attach with token: " + mCurToken); + final InputMethodInfo info = mMethodMap.get(mSelectedMethodId); + mSupportsStylusHw = info.supportsStylusHandwriting(); // Dispatch display id for InputMethodService to update context display. mService.executeOrSendMessage(mCurMethod, - mService.mCaller.obtainMessageIOO(MSG_INITIALIZE_IME, - mMethodMap.get(mSelectedMethodId).getConfigChanges(), - mCurMethod, mCurToken)); + mService.mCaller.obtainMessageIOOO(MSG_INITIALIZE_IME, + info.getConfigChanges(), mCurMethod, mCurToken, + mSupportsStylusHw)); mService.scheduleNotifyImeUidToAudioService(mCurMethodUid); mService.reRequestCurrentClientSessionLocked(); } - mSupportsStylusHw = mMethodMap.get(mSelectedMethodId).supportsStylusHandwriting(); + if (mSupportsStylusHw) { // TODO init Handwriting spy. } diff --git a/services/core/java/com/android/server/inputmethod/InputMethodManagerService.java b/services/core/java/com/android/server/inputmethod/InputMethodManagerService.java index c87dc8987b256..f1e8d0d599578 100644 --- a/services/core/java/com/android/server/inputmethod/InputMethodManagerService.java +++ b/services/core/java/com/android/server/inputmethod/InputMethodManagerService.java @@ -233,6 +233,7 @@ public class InputMethodManagerService extends IInputMethodManager.Stub static final int MSG_REMOVE_IME_SURFACE = 1060; static final int MSG_REMOVE_IME_SURFACE_FROM_WINDOW = 1061; static final int MSG_UPDATE_IME_WINDOW_STATUS = 1070; + static final int MSG_START_HANDWRITING = 1100; static final int MSG_START_INPUT = 2000; @@ -314,6 +315,12 @@ public class InputMethodManagerService extends IInputMethodManager.Stub @GuardedBy("ImfLock.class") private int mMethodMapUpdateCount = 0; + /** + * Tracks requestIds for Stylus handwriting mode. + */ + @GuardedBy("ImfLock.class") + private int mHwRequestId = 0; + /** * The display id for which the latest startInput was called. */ @@ -3019,21 +3026,8 @@ public class InputMethodManagerService extends IInputMethodManager.Stub } final long ident = Binder.clearCallingIdentity(); try { - if (mCurClient == null || client == null - || mCurClient.client.asBinder() != client.asBinder()) { - // We need to check if this is the current client with - // focus in the window manager, to allow this call to - // be made before input is started in it. - final ClientState cs = mClients.get(client.asBinder()); - if (cs == null) { - throw new IllegalArgumentException( - "unknown client " + client.asBinder()); - } - if (!mWindowManagerInternal.isInputMethodClientFocus(cs.uid, cs.pid, - cs.selfReportedDisplayId)) { - Slog.w(TAG, "Ignoring showSoftInput of uid " + uid + ": " + client); - return false; - } + if (!canInteractWithImeLocked(uid, client, "showSoftInput")) { + return false; } if (DEBUG) Slog.v(TAG, "Client requesting input be shown"); return showCurrentInputLocked(windowToken, flags, resultReceiver, reason); @@ -3044,6 +3038,40 @@ public class InputMethodManagerService extends IInputMethodManager.Stub } } + @Override + public void startStylusHandwriting(IInputMethodClient client) { + Trace.traceBegin(TRACE_TAG_WINDOW_MANAGER, "IMMS.startStylusHandwriting"); + ImeTracing.getInstance().triggerManagerServiceDump( + "InputMethodManagerService#startStylusHandwriting"); + int uid = Binder.getCallingUid(); + synchronized (ImfLock.class) { + if (!calledFromValidUserLocked()) { + Trace.traceEnd(TRACE_TAG_WINDOW_MANAGER); + return; + } + final long ident = Binder.clearCallingIdentity(); + try { + if (!canInteractWithImeLocked(uid, client, "startStylusHandwriting")) { + Trace.traceEnd(TRACE_TAG_WINDOW_MANAGER); + return; + } + if (!mBindingController.supportsStylusHandwriting()) { + Slog.w(TAG, "Stylus HW unsupported by IME. Ignoring startStylusHandwriting()"); + Trace.traceEnd(TRACE_TAG_WINDOW_MANAGER); + return; + } + if (DEBUG) Slog.v(TAG, "Client requesting Stylus Handwriting to be started"); + if (getCurMethodLocked() != null) { + executeOrSendMessage(getCurMethodLocked(), mCaller.obtainMessageIO( + MSG_START_HANDWRITING, ++mHwRequestId, getCurMethodLocked())); + } + } finally { + Binder.restoreCallingIdentity(ident); + Trace.traceEnd(TRACE_TAG_WINDOW_MANAGER); + } + } + } + @BinderThread @Override public void reportPerceptibleAsync(IBinder windowToken, boolean perceptible) { @@ -3523,6 +3551,27 @@ public class InputMethodManagerService extends IInputMethodManager.Stub return res; } + @GuardedBy("ImfLock.class") + private boolean canInteractWithImeLocked( + int uid, IInputMethodClient client, String methodName) { + if (mCurClient == null || client == null + || mCurClient.client.asBinder() != client.asBinder()) { + // We need to check if this is the current client with + // focus in the window manager, to allow this call to + // be made before input is started in it. + final ClientState cs = mClients.get(client.asBinder()); + if (cs == null) { + throw new IllegalArgumentException("unknown client " + client.asBinder()); + } + if (!mWindowManagerInternal.isInputMethodClientFocus(cs.uid, cs.pid, + cs.selfReportedDisplayId)) { + Slog.w(TAG, String.format("Ignoring %s of uid %d : %s", methodName, uid, client)); + return false; + } + } + return true; + } + private boolean shouldRestoreImeVisibility(IBinder windowToken, @SoftInputModeFlags int softInputMode) { switch (softInputMode & LayoutParams.SOFT_INPUT_MASK_STATE) { @@ -4237,7 +4286,8 @@ public class InputMethodManagerService extends IInputMethodManager.Stub } final IBinder token = (IBinder) args.arg2; ((IInputMethod) args.arg1).initializeInternal(token, - new InputMethodPrivilegedOperationsImpl(this, token), msg.arg1); + new InputMethodPrivilegedOperationsImpl(this, token), + msg.arg1, (boolean) args.arg3); } catch (RemoteException e) { } args.recycle(); @@ -4410,10 +4460,34 @@ public class InputMethodManagerService extends IInputMethodManager.Stub } return true; } + case MSG_START_HANDWRITING: + try { + (((IInputMethod) msg.obj)).canStartStylusHandwriting(msg.arg1); + } catch (RemoteException e) { + Slog.w(TAG, "RemoteException calling canStartStylusHandwriting(): ", e); + } + return true; } return false; } + @BinderThread + private void onStylusHandwritingReady(int requestId) { + synchronized (ImfLock.class) { + if (mHwRequestId != requestId) { + // obsolete request + return; + } + + try { + // TODO: replace null with actual Channel, MotionEvents + getCurMethodLocked().startStylusHandwriting(null, null); + } catch (RemoteException e) { + Slog.w(TAG, "RemoteException calling startStylusHandwriting(): ", e); + } + } + } + private void handleSetInteractive(final boolean interactive) { synchronized (ImfLock.class) { mIsInteractive = interactive; @@ -5958,5 +6032,11 @@ public class InputMethodManagerService extends IInputMethodManager.Stub public void applyImeVisibilityAsync(IBinder windowToken, boolean setVisible) { mImms.applyImeVisibility(mToken, windowToken, setVisible); } + + @BinderThread + @Override + public void onStylusHandwritingReady(int requestId) { + mImms.onStylusHandwritingReady(requestId); + } } } diff --git a/services/core/java/com/android/server/wm/WindowManagerService.java b/services/core/java/com/android/server/wm/WindowManagerService.java index 7be128b53f8cf..39a291032e581 100644 --- a/services/core/java/com/android/server/wm/WindowManagerService.java +++ b/services/core/java/com/android/server/wm/WindowManagerService.java @@ -55,6 +55,7 @@ import static android.view.WindowManager.LayoutParams.FLAG_ALT_FOCUSABLE_IM; import static android.view.WindowManager.LayoutParams.FLAG_DISMISS_KEYGUARD; import static android.view.WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON; import static android.view.WindowManager.LayoutParams.FLAG_NOT_FOCUSABLE; +import static android.view.WindowManager.LayoutParams.FLAG_NOT_TOUCHABLE; import static android.view.WindowManager.LayoutParams.FLAG_SECURE; import static android.view.WindowManager.LayoutParams.FLAG_SHOW_WALLPAPER; import static android.view.WindowManager.LayoutParams.FLAG_SHOW_WHEN_LOCKED; @@ -1761,7 +1762,10 @@ public class WindowManagerService extends IWindowManager.Stub activity.attachStartingWindow(win); ProtoLog.v(WM_DEBUG_STARTING_WINDOW, "addWindow: %s startingWindow=%s", activity, win); - } else if (type == TYPE_INPUT_METHOD) { + } else if (type == TYPE_INPUT_METHOD + // IME window is always touchable. + // Ignore non-touchable windows e.g. Stylus InkWindow.java. + && (win.getAttrs().flags & FLAG_NOT_TOUCHABLE) == 0) { displayContent.setInputMethodWindowLocked(win); imMayMove = false; } else if (type == TYPE_INPUT_METHOD_DIALOG) { @@ -8288,7 +8292,7 @@ public class WindowManagerService extends IWindowManager.Stub flags = sanitizeFlagSlippery(flags, name, callingUid, callingPid); - final int sanitizedFlags = flags & (LayoutParams.FLAG_NOT_TOUCHABLE + final int sanitizedFlags = flags & (FLAG_NOT_TOUCHABLE | FLAG_SLIPPERY | LayoutParams.FLAG_NOT_FOCUSABLE); h.layoutParamsFlags = WindowManager.LayoutParams.FLAG_NOT_TOUCH_MODAL | sanitizedFlags; h.layoutParamsType = type;