From 467fd168d5480e8c93b4f947fc9ce3216f887b2e Mon Sep 17 00:00:00 2001 From: Taran Singh Date: Wed, 2 Mar 2022 10:43:55 +0000 Subject: [PATCH] Introduce new IMS method for Stylus events and dispatch As of today we show the InkWindow and dispatch Stylus events without waiting. At the time of dispatch, InkWindow is not added and first stylus gesture is skipped. In order to deliver the first gesture to IME, we should wait until InkWindow is added and IME's Inking View is attached. However, waiting for it will introduce latency in the system. To counter that, we introduce a new API method to listen for Stylus MotionEvent. IME can override the API method to listen to MotionEvents sooner. By default, the stylus MotionEvents are buffered until Ink view is visible. Change-Id: I3a18067da45fd15bf0f6e377da0a82cdd6df1d41 Fix: 222081673 Test: atest StylusHandwriting --- core/api/current.txt | 1 + .../android/inputmethodservice/InkWindow.java | 78 +++++++++++++++++++ .../InputMethodService.java | 49 +++++++++++- 3 files changed, 124 insertions(+), 4 deletions(-) diff --git a/core/api/current.txt b/core/api/current.txt index 542221c3fcc96..ba3028c978326 100644 --- a/core/api/current.txt +++ b/core/api/current.txt @@ -18818,6 +18818,7 @@ package android.inputmethodservice { method public void onStartInput(android.view.inputmethod.EditorInfo, boolean); method public void onStartInputView(android.view.inputmethod.EditorInfo, boolean); method public boolean onStartStylusHandwriting(); + method public void onStylusHandwritingMotionEvent(@NonNull android.view.MotionEvent); method public void onUnbindInput(); method @Deprecated public void onUpdateCursor(android.graphics.Rect); method public void onUpdateCursorAnchorInfo(android.view.inputmethod.CursorAnchorInfo); diff --git a/core/java/android/inputmethodservice/InkWindow.java b/core/java/android/inputmethodservice/InkWindow.java index 499634ae5bec9..8289c268d4c63 100644 --- a/core/java/android/inputmethodservice/InkWindow.java +++ b/core/java/android/inputmethodservice/InkWindow.java @@ -27,6 +27,8 @@ import android.content.Context; import android.os.IBinder; import android.util.Slog; import android.view.View; +import android.view.ViewGroup; +import android.view.ViewTreeObserver; import android.view.WindowManager; import com.android.internal.policy.PhoneWindow; @@ -40,6 +42,9 @@ final class InkWindow extends PhoneWindow { private final WindowManager mWindowManager; private boolean mIsViewAdded; + private View mInkView; + private InkVisibilityListener mInkViewVisibilityListener; + private ViewTreeObserver.OnGlobalLayoutListener mGlobalLayoutListener; public InkWindow(@NonNull Context context) { super(context); @@ -102,4 +107,77 @@ final class InkWindow extends PhoneWindow { lp.token = token; setAttributes(lp); } + + @Override + public void addContentView(View view, ViewGroup.LayoutParams params) { + if (mInkView == null) { + mInkView = view; + } else if (mInkView != view) { + throw new IllegalStateException("Only one Child Inking view is permitted."); + } + super.addContentView(view, params); + initInkViewVisibilityListener(); + } + + @Override + public void setContentView(View view, ViewGroup.LayoutParams params) { + mInkView = view; + super.setContentView(view, params); + initInkViewVisibilityListener(); + } + + @Override + public void setContentView(View view) { + mInkView = view; + super.setContentView(view); + initInkViewVisibilityListener(); + } + + @Override + public void clearContentView() { + if (mGlobalLayoutListener != null && mInkView != null) { + mInkView.getViewTreeObserver().removeOnGlobalLayoutListener(mGlobalLayoutListener); + } + mGlobalLayoutListener = null; + mInkView = null; + super.clearContentView(); + } + + /** + * Listener used by InkWindow to time the dispatching of {@link MotionEvent}s to Ink view, once + * it is visible to user. + */ + interface InkVisibilityListener { + void onInkViewVisible(); + } + + void setInkViewVisibilityListener(InkVisibilityListener listener) { + mInkViewVisibilityListener = listener; + initInkViewVisibilityListener(); + } + + void initInkViewVisibilityListener() { + if (mInkView == null || mInkViewVisibilityListener == null + || mGlobalLayoutListener != null) { + return; + } + mGlobalLayoutListener = new ViewTreeObserver.OnGlobalLayoutListener() { + @Override + public void onGlobalLayout() { + if (mInkView.isVisibleToUser()) { + if (mInkViewVisibilityListener != null) { + mInkViewVisibilityListener.onInkViewVisible(); + } + mInkView.getViewTreeObserver().removeOnGlobalLayoutListener(this); + mGlobalLayoutListener = null; + } + } + }; + mInkView.getViewTreeObserver().addOnGlobalLayoutListener(mGlobalLayoutListener); + } + + boolean isInkViewVisible() { + return getDecorView().getVisibility() == View.VISIBLE + && mInkView != null && mInkView.isVisibleToUser(); + } } diff --git a/core/java/android/inputmethodservice/InputMethodService.java b/core/java/android/inputmethodservice/InputMethodService.java index fbc0732affe1c..4970d445b34de 100644 --- a/core/java/android/inputmethodservice/InputMethodService.java +++ b/core/java/android/inputmethodservice/InputMethodService.java @@ -142,6 +142,7 @@ import com.android.internal.inputmethod.IInputMethodPrivilegedOperations; import com.android.internal.inputmethod.ImeTracing; import com.android.internal.inputmethod.InputMethodPrivilegedOperations; import com.android.internal.inputmethod.InputMethodPrivilegedOperationsRegistry; +import com.android.internal.util.RingBuffer; import com.android.internal.view.IInlineSuggestionsRequestCallback; import com.android.internal.view.IInputContext; import com.android.internal.view.InlineSuggestionsRequestInfo; @@ -332,6 +333,17 @@ public class InputMethodService extends AbstractInputMethodService { private static final String PROP_CAN_RENDER_GESTURAL_NAV_BUTTONS = "persist.sys.ime.can_render_gestural_nav_buttons"; + /** + * Number of {@link MotionEvent} to buffer if IME is not ready with Ink view. + * This number may be configured eventually based on device's touch sampling frequency. + */ + private static final int MAX_EVENTS_BUFFER = 500; + + /** + * A circular buffer of size MAX_EVENTS_BUFFER in case IME is taking too long to add ink view. + **/ + private RingBuffer mPendingEvents; + /** * Returns whether {@link InputMethodService} is responsible for rendering the back button and * the IME switcher button or not when the gestural navigation is enabled. @@ -957,7 +969,8 @@ public class InputMethodService extends AbstractInputMethodService { mInkWindow.show(); // deliver previous @param stylusEvents - stylusEvents.forEach(mInkWindow.getDecorView()::dispatchTouchEvent); + stylusEvents.forEach(InputMethodService.this::onStylusHandwritingMotionEvent); + // create receiver for channel mHandwritingEventReceiver = new SimpleBatchedInputEventReceiver( channel, @@ -966,11 +979,11 @@ public class InputMethodService extends AbstractInputMethodService { if (!(event instanceof MotionEvent)) { return false; } - return mInkWindow.getDecorView().dispatchTouchEvent((MotionEvent) event); + onStylusHandwritingMotionEvent((MotionEvent) event); + return true; }); } - /** * {@inheritDoc} * @hide @@ -2360,7 +2373,8 @@ public class InputMethodService extends AbstractInputMethodService { * * If the IME supports handwriting for the current input, it should return {@code true}, * ensure its inking views are attached to the {@link #getStylusHandwritingWindow()}, and handle - * stylus input received on the ink window via {@link #getCurrentInputConnection()}. + * stylus input received from {@link #onStylusHandwritingMotionEvent(MotionEvent)} on the + * {@link #getStylusHandwritingWindow()} via {@link #getCurrentInputConnection()}. * @return {@code true} if IME can honor the request, {@code false} if IME cannot at this time. */ public boolean onStartStylusHandwriting() { @@ -2368,6 +2382,33 @@ public class InputMethodService extends AbstractInputMethodService { return false; } + /** + * Called after {@link #onStartStylusHandwriting()} returns {@code true} for every Stylus + * {@link MotionEvent}. + * By default, this method forwards all {@link MotionEvent}s to the + * {@link #getStylusHandwritingWindow()} once its visible, however IME can override it to + * receive them sooner. + * @param motionEvent {@link MotionEvent} from stylus. + */ + public void onStylusHandwritingMotionEvent(@NonNull MotionEvent motionEvent) { + if (mInkWindow.isInkViewVisible()) { + mInkWindow.getDecorView().dispatchTouchEvent(motionEvent); + } else { + if (mPendingEvents == null) { + mPendingEvents = new RingBuffer(MotionEvent.class, MAX_EVENTS_BUFFER); + } + mPendingEvents.append(motionEvent); + mInkWindow.setInkViewVisibilityListener(() -> { + if (mPendingEvents != null && !mPendingEvents.isEmpty()) { + for (MotionEvent event : mPendingEvents.toArray()) { + mInkWindow.getDecorView().dispatchTouchEvent(event); + } + mPendingEvents.clear(); + } + }); + } + } + /** * Called when the current stylus handwriting session was finished (either by the system or * via {@link #finishStylusHandwriting()}.