diff --git a/core/java/android/hardware/input/InputManagerInternal.java b/core/java/android/hardware/input/InputManagerInternal.java index ad6c12be69ba2..cc9aeab7de556 100644 --- a/core/java/android/hardware/input/InputManagerInternal.java +++ b/core/java/android/hardware/input/InputManagerInternal.java @@ -20,6 +20,7 @@ import android.annotation.NonNull; import android.graphics.PointF; import android.hardware.display.DisplayViewport; import android.os.IBinder; +import android.view.InputChannel; import android.view.InputEvent; import java.util.List; @@ -123,4 +124,13 @@ public abstract class InputManagerInternal { */ void notifyLidSwitchChanged(long whenNanos, boolean lidOpen); } + + /** Create an {@link InputChannel} that is registered to InputDispatcher. */ + public abstract InputChannel createInputChannel(String inputChannelName); + + /** + * Pilfer pointers from the input channel with the given token so that ongoing gestures are + * canceled for all other channels. + */ + public abstract void pilferPointers(IBinder token); } diff --git a/core/java/android/inputmethodservice/IInputMethodWrapper.java b/core/java/android/inputmethodservice/IInputMethodWrapper.java index cc325cde1f419..4432cafd1cf77 100644 --- a/core/java/android/inputmethodservice/IInputMethodWrapper.java +++ b/core/java/android/inputmethodservice/IInputMethodWrapper.java @@ -245,7 +245,7 @@ class IInputMethodWrapper extends IInputMethod.Stub } case DO_START_STYLUS_HANDWRITING: { final SomeArgs args = (SomeArgs) msg.obj; - inputMethod.startStylusHandwriting((InputChannel) args.arg1, + inputMethod.startStylusHandwriting(msg.arg1, (InputChannel) args.arg1, (List) args.arg2); args.recycle(); return; @@ -393,10 +393,11 @@ class IInputMethodWrapper extends IInputMethod.Stub @BinderThread @Override - public void startStylusHandwriting(@NonNull InputChannel channel, + public void startStylusHandwriting(int requestId, @NonNull InputChannel channel, @Nullable List stylusEvents) throws RemoteException { mCaller.executeOrSendMessage( - mCaller.obtainMessageOO(DO_START_STYLUS_HANDWRITING, channel, stylusEvents)); + mCaller.obtainMessageIOO(DO_START_STYLUS_HANDWRITING, requestId, channel, + stylusEvents)); } } diff --git a/core/java/android/inputmethodservice/InkWindow.java b/core/java/android/inputmethodservice/InkWindow.java index e11d63562ce33..f8d2fe2f21840 100644 --- a/core/java/android/inputmethodservice/InkWindow.java +++ b/core/java/android/inputmethodservice/InkWindow.java @@ -39,6 +39,7 @@ import com.android.internal.policy.PhoneWindow; final class InkWindow extends PhoneWindow { private final WindowManager mWindowManager; + private boolean mIsViewAdded; public InkWindow(@NonNull Context context) { super(context); @@ -47,6 +48,7 @@ final class InkWindow extends PhoneWindow { final LayoutParams attrs = getAttributes(); attrs.layoutInDisplayCutoutMode = LayoutParams.LAYOUT_IN_DISPLAY_CUTOUT_MODE_ALWAYS; attrs.setFitInsetsTypes(0); + // TODO(b/210039666): use INPUT_FEATURE_NO_INPUT_CHANNEL once b/216179339 is fixed. setAttributes(attrs); // Ink window is not touchable with finger. addFlags(FLAG_LAYOUT_IN_SCREEN | FLAG_LAYOUT_NO_LIMITS | FLAG_NOT_TOUCHABLE @@ -66,7 +68,10 @@ final class InkWindow extends PhoneWindow { return; } getDecorView().setVisibility(View.VISIBLE); - mWindowManager.addView(getDecorView(), getAttributes()); + if (!mIsViewAdded) { + mWindowManager.addView(getDecorView(), getAttributes()); + mIsViewAdded = true; + } } /** @@ -78,6 +83,7 @@ final class InkWindow extends PhoneWindow { if (getDecorView() != null) { getDecorView().setVisibility(remove ? View.GONE : View.INVISIBLE); } + //TODO(b/210039666): remove window from WM after a delay. Delay amount TBD. } void setToken(@NonNull IBinder token) { diff --git a/core/java/android/inputmethodservice/InputMethodService.java b/core/java/android/inputmethodservice/InputMethodService.java index 5d2d8eafb3a8f..80da3617d4a41 100644 --- a/core/java/android/inputmethodservice/InputMethodService.java +++ b/core/java/android/inputmethodservice/InputMethodService.java @@ -82,6 +82,7 @@ import android.os.Build; import android.os.Bundle; import android.os.Handler; import android.os.IBinder; +import android.os.Looper; import android.os.ResultReceiver; import android.os.SystemClock; import android.os.SystemProperties; @@ -95,8 +96,11 @@ import android.util.Log; import android.util.PrintWriterPrinter; import android.util.Printer; import android.util.proto.ProtoOutputStream; +import android.view.BatchedInputEventReceiver.SimpleBatchedInputEventReceiver; +import android.view.Choreographer; import android.view.Gravity; import android.view.InputChannel; +import android.view.InputEventReceiver; import android.view.KeyCharacterMap; import android.view.KeyEvent; import android.view.LayoutInflater; @@ -148,6 +152,7 @@ import java.lang.annotation.RetentionPolicy; import java.util.ArrayList; import java.util.List; import java.util.Objects; +import java.util.OptionalInt; /** * InputMethodService provides a standard implementation of an InputMethod, @@ -567,7 +572,8 @@ public class InputMethodService extends AbstractInputMethodService { private boolean mAutomotiveHideNavBarForKeyboard; private boolean mIsAutomotive; - private boolean mHandwritingStarted; + private @NonNull OptionalInt mHandwritingRequestId = OptionalInt.empty(); + private InputEventReceiver mHandwritingEventReceiver; private Handler mHandler; private boolean mImeSurfaceScheduledForRemoval; private ImsConfigurationTracker mConfigTracker = new ImsConfigurationTracker(); @@ -888,7 +894,7 @@ public class InputMethodService extends AbstractInputMethodService { @Override public void canStartStylusHandwriting(int requestId) { if (DEBUG) Log.v(TAG, "canStartStylusHandwriting()"); - if (mHandwritingStarted) { + if (mHandwritingRequestId.isPresent()) { Log.d(TAG, "There is an ongoing Handwriting session. ignoring."); return; } @@ -900,6 +906,7 @@ public class InputMethodService extends AbstractInputMethodService { mPrivOps.onStylusHandwritingReady(requestId); } else { Log.i(TAG, "IME is not ready. Can't start Stylus Handwriting"); + // TODO(b/210039666): see if it's valuable to propagate this back to IMM. } } @@ -910,20 +917,36 @@ public class InputMethodService extends AbstractInputMethodService { @MainThread @Override public void startStylusHandwriting( - @NonNull InputChannel channel, @Nullable List stylusEvents) { + int requestId, @NonNull InputChannel channel, + @NonNull List stylusEvents) { if (DEBUG) Log.v(TAG, "startStylusHandwriting()"); - if (mHandwritingStarted) { + Objects.requireNonNull(channel); + Objects.requireNonNull(stylusEvents); + + if (mHandwritingRequestId.isPresent()) { return; } - mHandwritingStarted = true; + mHandwritingRequestId = OptionalInt.of(requestId); mShowInputRequested = false; mInkWindow.show(); - // TODO: deliver previous @param stylusEvents - // TODO: create spy receiver for @param channel + + // deliver previous @param stylusEvents + stylusEvents.forEach(mInkWindow.getDecorView()::dispatchTouchEvent); + // create receiver for channel + mHandwritingEventReceiver = new SimpleBatchedInputEventReceiver( + channel, + Looper.getMainLooper(), Choreographer.getInstance(), + event -> { + if (!(event instanceof MotionEvent)) { + return false; + } + return mInkWindow.getDecorView().dispatchTouchEvent((MotionEvent) event); + }); } + /** * {@inheritDoc} */ @@ -2358,12 +2381,18 @@ public class InputMethodService extends AbstractInputMethodService { if (mInkWindow == null) { return; } - if (!mHandwritingStarted) { + if (!mHandwritingRequestId.isPresent()) { return; } - mHandwritingStarted = false; + final int requestId = mHandwritingRequestId.getAsInt(); + mHandwritingRequestId = OptionalInt.empty(); + + mHandwritingEventReceiver.dispose(); + mHandwritingEventReceiver = null; mInkWindow.hide(false /* remove */); + + mPrivOps.finishStylusHandwriting(requestId); onFinishStylusHandwriting(); } diff --git a/core/java/android/view/inputmethod/InputMethod.java b/core/java/android/view/inputmethod/InputMethod.java index fda72d5ba9669..a1d10f8853a40 100644 --- a/core/java/android/view/inputmethod/InputMethod.java +++ b/core/java/android/view/inputmethod/InputMethod.java @@ -405,7 +405,7 @@ public interface InputMethod { * @hide */ default void startStylusHandwriting( - @NonNull InputChannel channel, @Nullable List events) { + int requestId, @NonNull InputChannel channel, @Nullable List events) { // intentionally empty } diff --git a/core/java/com/android/internal/inputmethod/IInputMethodPrivilegedOperations.aidl b/core/java/com/android/internal/inputmethod/IInputMethodPrivilegedOperations.aidl index 08bc8c7fa339a..30853bc2ecc54 100644 --- a/core/java/com/android/internal/inputmethod/IInputMethodPrivilegedOperations.aidl +++ b/core/java/com/android/internal/inputmethod/IInputMethodPrivilegedOperations.aidl @@ -43,4 +43,5 @@ oneway interface IInputMethodPrivilegedOperations { void notifyUserActionAsync(); void applyImeVisibilityAsync(IBinder showOrHideInputToken, boolean setVisible); void onStylusHandwritingReady(int requestId); + void finishStylusHandwriting(int requestId); } diff --git a/core/java/com/android/internal/inputmethod/InputMethodPrivilegedOperations.java b/core/java/com/android/internal/inputmethod/InputMethodPrivilegedOperations.java index 7ebcc88b593b4..2a7e1dcedd436 100644 --- a/core/java/com/android/internal/inputmethod/InputMethodPrivilegedOperations.java +++ b/core/java/com/android/internal/inputmethod/InputMethodPrivilegedOperations.java @@ -410,4 +410,21 @@ public final class InputMethodPrivilegedOperations { throw e.rethrowFromSystemServer(); } } + + /** + * IME notifies that the current handwriting session should be closed. + * @param requestId + */ + @AnyThread + public void finishStylusHandwriting(int requestId) { + final IInputMethodPrivilegedOperations ops = mOps.getAndWarnIfNull(); + if (ops == null) { + return; + } + try { + ops.finishStylusHandwriting(requestId); + } catch (RemoteException e) { + throw e.rethrowFromSystemServer(); + } + } } diff --git a/core/java/com/android/internal/view/IInputMethod.aidl b/core/java/com/android/internal/view/IInputMethod.aidl index 6a626ee6eb8f9..fbf0f835b24f2 100644 --- a/core/java/com/android/internal/view/IInputMethod.aidl +++ b/core/java/com/android/internal/view/IInputMethod.aidl @@ -61,5 +61,6 @@ oneway interface IInputMethod { void canStartStylusHandwriting(int requestId); - void startStylusHandwriting(in InputChannel channel, in List events); + void startStylusHandwriting(int requestId, in InputChannel channel, + in List events); } diff --git a/services/core/java/com/android/server/input/InputManagerService.java b/services/core/java/com/android/server/input/InputManagerService.java index b9c7123e10105..de933cc470058 100644 --- a/services/core/java/com/android/server/input/InputManagerService.java +++ b/services/core/java/com/android/server/input/InputManagerService.java @@ -3510,6 +3510,16 @@ public class InputManagerService extends IInputManager.Stub public void unregisterLidSwitchCallback(LidSwitchCallback callbacks) { unregisterLidSwitchCallbackInternal(callbacks); } + + @Override + public InputChannel createInputChannel(String inputChannelName) { + return InputManagerService.this.createInputChannel(inputChannelName); + } + + @Override + public void pilferPointers(IBinder token) { + nativePilferPointers(mPtr, token); + } } @Override diff --git a/services/core/java/com/android/server/inputmethod/HandwritingEventReceiverSurface.java b/services/core/java/com/android/server/inputmethod/HandwritingEventReceiverSurface.java new file mode 100644 index 0000000000000..4c2616667a023 --- /dev/null +++ b/services/core/java/com/android/server/inputmethod/HandwritingEventReceiverSurface.java @@ -0,0 +1,116 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.inputmethod; + +import static android.os.InputConstants.DEFAULT_DISPATCHING_TIMEOUT_MILLIS; + +import android.annotation.NonNull; +import android.graphics.Rect; +import android.os.Process; +import android.view.InputApplicationHandle; +import android.view.InputChannel; +import android.view.InputWindowHandle; +import android.view.SurfaceControl; +import android.view.WindowManager; + + +final class HandwritingEventReceiverSurface { + + public static final String TAG = HandwritingEventReceiverSurface.class.getSimpleName(); + static final boolean DEBUG = HandwritingModeController.DEBUG; + + private final int mClientPid; + private final int mClientUid; + + private final InputApplicationHandle mApplicationHandle; + private final InputWindowHandle mWindowHandle; + private final InputChannel mClientChannel; + private final SurfaceControl mInputSurface; + private boolean mIsIntercepting; + + HandwritingEventReceiverSurface(String name, int displayId, @NonNull SurfaceControl sc, + @NonNull InputChannel inputChannel) { + // Initialized the window as being owned by the system. + mClientPid = Process.myPid(); + mClientUid = Process.myUid(); + mApplicationHandle = new InputApplicationHandle(null, name, + DEFAULT_DISPATCHING_TIMEOUT_MILLIS); + + mClientChannel = inputChannel; + mInputSurface = sc; + + mWindowHandle = new InputWindowHandle(mApplicationHandle, displayId); + mWindowHandle.name = name; + mWindowHandle.token = mClientChannel.getToken(); + mWindowHandle.layoutParamsType = WindowManager.LayoutParams.TYPE_SECURE_SYSTEM_OVERLAY; + mWindowHandle.layoutParamsFlags = WindowManager.LayoutParams.FLAG_NOT_TOUCH_MODAL; + mWindowHandle.dispatchingTimeoutMillis = DEFAULT_DISPATCHING_TIMEOUT_MILLIS; + mWindowHandle.visible = true; + mWindowHandle.focusable = false; + mWindowHandle.hasWallpaper = false; + mWindowHandle.paused = false; + mWindowHandle.ownerPid = mClientPid; + mWindowHandle.ownerUid = mClientUid; + mWindowHandle.inputFeatures = WindowManager.LayoutParams.INPUT_FEATURE_SPY + | WindowManager.LayoutParams.INPUT_FEATURE_INTERCEPTS_STYLUS; + mWindowHandle.scaleFactor = 1.0f; + mWindowHandle.trustedOverlay = true; + mWindowHandle.replaceTouchableRegionWithCrop(null /* use this surface as crop */); + + final SurfaceControl.Transaction t = new SurfaceControl.Transaction(); + t.setInputWindowInfo(mInputSurface, mWindowHandle); + t.setLayer(mInputSurface, Integer.MAX_VALUE); + t.setPosition(mInputSurface, 0, 0); + // Use an arbitrarily large crop that is positioned at the origin. The crop determines the + // bounds and the coordinate space of the input events, so it must start at the origin to + // receive input in display space. + // TODO(b/210039666): fix this in SurfaceFlinger and avoid the hack. + t.setCrop(mInputSurface, new Rect(0, 0, 10000, 10000)); + t.show(mInputSurface); + t.apply(); + + mIsIntercepting = false; + } + + void startIntercepting() { + // TODO(b/210978621): Update the spy window's PID and UID to be associated with the IME so + // that ANRs are correctly attributed to the IME. + final SurfaceControl.Transaction t = new SurfaceControl.Transaction(); + mWindowHandle.inputFeatures &= ~WindowManager.LayoutParams.INPUT_FEATURE_SPY; + t.setInputWindowInfo(mInputSurface, mWindowHandle); + t.apply(); + mIsIntercepting = true; + } + + boolean isIntercepting() { + return mIsIntercepting; + } + + void remove() { + final SurfaceControl.Transaction t = new SurfaceControl.Transaction(); + t.remove(mInputSurface); + t.apply(); + } + + InputChannel getInputChannel() { + return mClientChannel; + } + + SurfaceControl getSurface() { + return mInputSurface; + } +} diff --git a/services/core/java/com/android/server/inputmethod/HandwritingModeController.java b/services/core/java/com/android/server/inputmethod/HandwritingModeController.java new file mode 100644 index 0000000000000..c5210ade57bae --- /dev/null +++ b/services/core/java/com/android/server/inputmethod/HandwritingModeController.java @@ -0,0 +1,274 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.inputmethod; + +import static android.view.InputDevice.SOURCE_STYLUS; + +import android.annotation.AnyThread; +import android.annotation.Nullable; +import android.annotation.UiThread; +import android.hardware.input.InputManagerInternal; +import android.os.Looper; +import android.util.Slog; +import android.view.BatchedInputEventReceiver; +import android.view.Choreographer; +import android.view.Display; +import android.view.InputChannel; +import android.view.InputEvent; +import android.view.InputEventReceiver; +import android.view.MotionEvent; +import android.view.SurfaceControl; + +import com.android.server.LocalServices; +import com.android.server.wm.WindowManagerInternal; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.OptionalInt; + +// TODO(b/210039666): See if we can make this class thread-safe. +final class HandwritingModeController { + + public static final String TAG = HandwritingModeController.class.getSimpleName(); + // TODO(b/210039666): flip the flag. + static final boolean DEBUG = true; + private static final int EVENT_BUFFER_SIZE = 100; + + // This must be the looper for the UiThread. + private final Looper mLooper; + private final InputManagerInternal mInputManagerInternal; + private final WindowManagerInternal mWindowManagerInternal; + + private List mHandwritingBuffer; + private InputEventReceiver mHandwritingEventReceiver; + private boolean mRecordingGesture; + private int mCurrentDisplayId; + + private HandwritingEventReceiverSurface mHandwritingSurface; + + private int mCurrentRequestId; + + @AnyThread + HandwritingModeController(Looper uiThreadLooper) { + mLooper = uiThreadLooper; + mCurrentDisplayId = Display.INVALID_DISPLAY; + mInputManagerInternal = LocalServices.getService(InputManagerInternal.class); + mWindowManagerInternal = LocalServices.getService(WindowManagerInternal.class); + mCurrentRequestId = 0; + } + + // TODO(b/210039666): Consider moving this to MotionEvent + private static boolean isStylusEvent(MotionEvent event) { + if (!event.isFromSource(SOURCE_STYLUS)) { + return false; + } + final int tool = event.getToolType(0); + return tool == MotionEvent.TOOL_TYPE_STYLUS || tool == MotionEvent.TOOL_TYPE_ERASER; + } + + /** + * Initializes the handwriting spy on the given displayId. + * + * This must be called from the UI Thread because it will start processing events using an + * InputEventReceiver that batches events according to the current thread's Choreographer. + */ + @UiThread + void initializeHandwritingSpy(int displayId) { + // When resetting, reuse resources if we are reinitializing on the same display. + reset(displayId == mCurrentDisplayId); + mCurrentDisplayId = displayId; + + if (mHandwritingBuffer == null) { + mHandwritingBuffer = new ArrayList<>(EVENT_BUFFER_SIZE); + } + + if (DEBUG) Slog.d(TAG, "Initializing handwriting spy monitor for display: " + displayId); + final String name = "stylus-handwriting-event-receiver-" + displayId; + final InputChannel channel = mInputManagerInternal.createInputChannel(name); + Objects.requireNonNull(channel, "Failed to create input channel"); + final SurfaceControl surface = + mHandwritingSurface != null ? mHandwritingSurface.getSurface() + : mWindowManagerInternal.getHandwritingSurfaceForDisplay(displayId); + if (surface == null) { + Slog.e(TAG, "Failed to create input surface"); + return; + } + mHandwritingSurface = + new HandwritingEventReceiverSurface(name, displayId, surface, channel); + // Use a dup of the input channel so that event processing can be paused by disposing the + // event receiver without causing a fd hangup. + mHandwritingEventReceiver = new BatchedInputEventReceiver.SimpleBatchedInputEventReceiver( + channel.dup(), mLooper, Choreographer.getInstance(), this::onInputEvent); + mCurrentRequestId++; + } + + OptionalInt getCurrentRequestId() { + if (mHandwritingSurface == null) { + Slog.e(TAG, "Cannot get requestId: Handwriting was not initialized."); + return OptionalInt.empty(); + } + return OptionalInt.of(mCurrentRequestId); + } + + /** + * Starts a {@link HandwritingSession} to transfer to the IME. + * + * This must be called from the UI Thread to avoid race conditions between processing more + * input events and disposing the input event receiver. + * @return the handwriting session to send to the IME, or null if the request was invalid. + */ + @UiThread + @Nullable + HandwritingSession startHandwritingSession(int requestId) { + if (mHandwritingSurface == null) { + Slog.e(TAG, "Cannot start handwriting session: Handwriting was not initialized."); + return null; + } + if (requestId != mCurrentRequestId) { + Slog.e(TAG, "Cannot start handwriting session: Invalid request id: " + requestId); + return null; + } + Objects.requireNonNull(mHandwritingEventReceiver, + "Handwriting session was already transferred to IME."); + if (DEBUG) Slog.d(TAG, "Starting handwriting session in display: " + mCurrentDisplayId); + + mInputManagerInternal.pilferPointers(mHandwritingSurface.getInputChannel().getToken()); + + // Stop processing more events. + mHandwritingEventReceiver.dispose(); + mHandwritingEventReceiver = null; + mRecordingGesture = false; + + if (mHandwritingSurface.isIntercepting()) { + throw new IllegalStateException( + "Handwriting surface should not be already intercepting."); + } + mHandwritingSurface.startIntercepting(); + + return new HandwritingSession(mCurrentRequestId, mHandwritingSurface.getInputChannel(), + mHandwritingBuffer); + } + + /** + * Reset the current handwriting session without initializing another session. + * + * This must be called from UI Thread to avoid race conditions between processing more input + * events and disposing the input event receiver. + */ + @UiThread + void reset() { + reset(false /* reinitializing */); + } + + private void reset(boolean reinitializing) { + if (mHandwritingEventReceiver != null) { + mHandwritingEventReceiver.dispose(); + mHandwritingEventReceiver = null; + } + + if (mHandwritingBuffer != null) { + mHandwritingBuffer.forEach(MotionEvent::recycle); + mHandwritingBuffer.clear(); + if (!reinitializing) { + mHandwritingBuffer = null; + } + } + + if (mHandwritingSurface != null) { + mHandwritingSurface.getInputChannel().dispose(); + if (!reinitializing) { + mHandwritingSurface.remove(); + mHandwritingSurface = null; + } + } + + mRecordingGesture = false; + } + + private boolean onInputEvent(InputEvent ev) { + if (mHandwritingEventReceiver == null) { + throw new IllegalStateException( + "Input Event should not be processed when IME has the spy channel."); + } + + if (!(ev instanceof MotionEvent)) { + Slog.e("Stylus", "Received non-motion event in stylus monitor."); + return false; + } + final MotionEvent event = (MotionEvent) ev; + if (!isStylusEvent(event)) { + return false; + } + + onStylusEvent(event); + return true; + } + + private void onStylusEvent(MotionEvent event) { + final int action = event.getActionMasked(); + if (action == MotionEvent.ACTION_UP) { + mRecordingGesture = false; + mHandwritingBuffer.clear(); + return; + } + + if (action == MotionEvent.ACTION_DOWN) { + mRecordingGesture = true; + } + + if (!mRecordingGesture) { + return; + } + + if (mHandwritingBuffer.size() >= EVENT_BUFFER_SIZE) { + if (DEBUG) { + Slog.w(TAG, "Current gesture exceeds the buffer capacity." + + " The rest of the gesture will not be recorded."); + } + mRecordingGesture = false; + return; + } + + mHandwritingBuffer.add(MotionEvent.obtain(event)); + } + + static final class HandwritingSession { + private final int mRequestId; + private final InputChannel mHandwritingChannel; + private final List mRecordedEvents; + + private HandwritingSession(int requestId, InputChannel handwritingChannel, + List recordedEvents) { + mRequestId = requestId; + mHandwritingChannel = handwritingChannel; + mRecordedEvents = recordedEvents; + } + + int getRequestId() { + return mRequestId; + } + + InputChannel getHandwritingChannel() { + return mHandwritingChannel; + } + + List getRecordedEvents() { + return mRecordedEvents; + } + } +} diff --git a/services/core/java/com/android/server/inputmethod/IInputMethodInvoker.java b/services/core/java/com/android/server/inputmethod/IInputMethodInvoker.java index c86ebd26d8713..f8ceb5f3c5ade 100644 --- a/services/core/java/com/android/server/inputmethod/IInputMethodInvoker.java +++ b/services/core/java/com/android/server/inputmethod/IInputMethodInvoker.java @@ -214,11 +214,13 @@ final class IInputMethodInvoker { } @AnyThread - void startStylusHandwriting(InputChannel channel, List events) { + boolean startStylusHandwriting(int requestId, InputChannel channel, List events) { try { - mTarget.startStylusHandwriting(channel, events); + mTarget.startStylusHandwriting(requestId, channel, events); } catch (RemoteException e) { logRemoteException(e); + return false; } + return true; } } diff --git a/services/core/java/com/android/server/inputmethod/InputMethodBindingController.java b/services/core/java/com/android/server/inputmethod/InputMethodBindingController.java index 2230dcde08691..b81478285a627 100644 --- a/services/core/java/com/android/server/inputmethod/InputMethodBindingController.java +++ b/services/core/java/com/android/server/inputmethod/InputMethodBindingController.java @@ -315,9 +315,10 @@ final class InputMethodBindingController { mService.reRequestCurrentClientSessionLocked(); } - if (mSupportsStylusHw) { - // TODO init Handwriting spy. - } + // reset Handwriting event receiver. + // always call this as it handles changes in mSupportsStylusHw. It is a noop + // if unchanged. + mService.scheduleResetStylusHandwriting(); } Trace.traceEnd(TRACE_TAG_WINDOW_MANAGER); } diff --git a/services/core/java/com/android/server/inputmethod/InputMethodManagerService.java b/services/core/java/com/android/server/inputmethod/InputMethodManagerService.java index feb0d138c3080..a6fa78bf927b5 100644 --- a/services/core/java/com/android/server/inputmethod/InputMethodManagerService.java +++ b/services/core/java/com/android/server/inputmethod/InputMethodManagerService.java @@ -196,6 +196,7 @@ import java.util.Date; import java.util.List; import java.util.Locale; import java.util.Objects; +import java.util.OptionalInt; import java.util.WeakHashMap; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.atomic.AtomicInteger; @@ -223,6 +224,9 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub private static final int MSG_REMOVE_IME_SURFACE_FROM_WINDOW = 1061; private static final int MSG_UPDATE_IME_WINDOW_STATUS = 1070; + private static final int MSG_RESET_HANDWRITING = 1090; + private static final int MSG_START_HANDWRITING = 1100; + private static final int MSG_UNBIND_CLIENT = 3000; private static final int MSG_BIND_CLIENT = 3010; private static final int MSG_SET_ACTIVE = 3020; @@ -298,12 +302,6 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub @GuardedBy("ImfLock.class") private int mMethodMapUpdateCount = 0; - /** - * Tracks requestIds for Stylus handwriting mode. - */ - @GuardedBy("ImfLock.class") - private int mHwRequestId = 0; - /** * The display id for which the latest startInput was called. */ @@ -323,6 +321,8 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub private final PendingIntent mImeSwitchPendingIntent; private boolean mShowOngoingImeSwitcherForPhones; private boolean mNotificationShown; + @GuardedBy("ImfLock.class") + private final HandwritingModeController mHwController; static class SessionState { final ClientState client; @@ -1636,6 +1636,7 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub mBindingController = new InputMethodBindingController(this); mPreventImeStartupUnlessTextEditor = mRes.getBoolean( com.android.internal.R.bool.config_preventImeStartupUnlessTextEditor); + mHwController = new HandwritingModeController(thread.getLooper()); } @GuardedBy("ImfLock.class") @@ -2518,6 +2519,11 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub configChanges, supportStylusHw); } + @AnyThread + void scheduleResetStylusHandwriting() { + mHandler.obtainMessage(MSG_RESET_HANDWRITING).sendToTarget(); + } + @AnyThread void scheduleNotifyImeUidToAudioService(int uid) { mHandler.removeMessages(MSG_NOTIFY_IME_UID_TO_AUDIO_SERVICE); @@ -3060,6 +3066,7 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub } } + @BinderThread @Override public void startStylusHandwriting(IInputMethodClient client) { Trace.traceBegin(TRACE_TAG_WINDOW_MANAGER, "IMMS.startStylusHandwriting"); @@ -3074,18 +3081,21 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub final long ident = Binder.clearCallingIdentity(); try { if (!canInteractWithImeLocked(uid, client, "startStylusHandwriting")) { - Trace.traceEnd(TRACE_TAG_WINDOW_MANAGER); return; } if (!mBindingController.supportsStylusHandwriting()) { Slog.w(TAG, "Stylus HW unsupported by IME. Ignoring startStylusHandwriting()"); - Trace.traceEnd(TRACE_TAG_WINDOW_MANAGER); + return; + } + final OptionalInt requestId = mHwController.getCurrentRequestId(); + if (!requestId.isPresent()) { + Slog.e(TAG, "Stylus handwriting was not initialized."); return; } if (DEBUG) Slog.v(TAG, "Client requesting Stylus Handwriting to be started"); final IInputMethodInvoker curMethod = getCurMethodLocked(); if (curMethod != null) { - curMethod.canStartStylusHandwriting(++mHwRequestId); + curMethod.canStartStylusHandwriting(requestId.getAsInt()); } } finally { Binder.restoreCallingIdentity(ident); @@ -4118,6 +4128,18 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub Trace.traceEnd(TRACE_TAG_WINDOW_MANAGER); } + @BinderThread + private void finishStylusHandwriting(int requestId) { + synchronized (ImfLock.class) { + final OptionalInt curRequest = mHwController.getCurrentRequestId(); + if (!curRequest.isPresent() || curRequest.getAsInt() != requestId) { + Slog.w(TAG, "IME requested to finish handwriting with a mismatched requestId: " + + requestId); + } + scheduleResetStylusHandwriting(); + } + } + @GuardedBy("ImfLock.class") private void setInputMethodWithSubtypeIdLocked(IBinder token, String id, int subtypeId) { if (token == null) { @@ -4358,21 +4380,47 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub } return true; } + + case MSG_RESET_HANDWRITING: { + synchronized (ImfLock.class) { + if (mBindingController.supportsStylusHandwriting() + && getCurMethodLocked() != null) { + mHwController.initializeHandwritingSpy(mCurTokenDisplayId); + } else { + mHwController.reset(); + } + } + return true; + } + case MSG_START_HANDWRITING: + synchronized (ImfLock.class) { + IInputMethodInvoker curMethod = getCurMethodLocked(); + if (curMethod == null) { + return true; + } + final HandwritingModeController.HandwritingSession session = + mHwController.startHandwritingSession(msg.arg1); + if (session == null) { + Slog.e(TAG, + "Failed to start handwriting session for requestId: " + msg.arg1); + return true; + } + + if (!curMethod.startStylusHandwriting(session.getRequestId(), + session.getHandwritingChannel(), session.getRecordedEvents())) { + // When failed to issue IPCs, re-initialize handwriting state. + Slog.w(TAG, "Resetting handwriting mode."); + mHwController.initializeHandwritingSpy(mCurTokenDisplayId); + } + } + return true; } return false; } @BinderThread private void onStylusHandwritingReady(int requestId) { - synchronized (ImfLock.class) { - if (mHwRequestId != requestId) { - // obsolete request - return; - } - - // TODO: replace null with actual Channel, MotionEvents - getCurMethodLocked().startStylusHandwriting(null, null); - } + mHandler.obtainMessage(MSG_START_HANDWRITING, requestId, 0 /* unused */).sendToTarget(); } private void handleSetInteractive(final boolean interactive) { @@ -5917,5 +5965,11 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub public void onStylusHandwritingReady(int requestId) { mImms.onStylusHandwritingReady(requestId); } + + @BinderThread + @Override + public void finishStylusHandwriting(int requestId) { + mImms.finishStylusHandwriting(requestId); + } } } diff --git a/services/core/java/com/android/server/wm/WindowManagerInternal.java b/services/core/java/com/android/server/wm/WindowManagerInternal.java index b9fa29733aa61..ee0af9d786eba 100644 --- a/services/core/java/com/android/server/wm/WindowManagerInternal.java +++ b/services/core/java/com/android/server/wm/WindowManagerInternal.java @@ -35,6 +35,7 @@ import android.view.IWindow; import android.view.InputChannel; import android.view.MagnificationSpec; import android.view.RemoteAnimationTarget; +import android.view.SurfaceControl; import android.view.SurfaceControlViewHost; import android.view.WindowInfo; import android.view.WindowManager.DisplayImePolicy; @@ -82,7 +83,7 @@ public abstract class WindowManagerInternal { * through the tracing file. * @param loggingTypeFlags The flags for the logging types this log entry belongs to. * @param callingParams The parameters for the method to be logged. - * @param a11yDump The proto byte array for a11y state when the entry is generated. + * @param a11yDump The proto byte array for a11y state when the entry is generated * @param callingUid The calling uid. * @param stackTrace The stack trace, null if not needed. * @param ignoreStackEntries The stack entries can be removed @@ -800,4 +801,10 @@ public abstract class WindowManagerInternal { */ public abstract void addTaskOverlay(int taskId, SurfaceControlViewHost.SurfacePackage overlay); public abstract void removeTaskOverlay(int taskId, SurfaceControlViewHost.SurfacePackage overlay); + + /** + * Get a SurfaceControl that is the container layer that should be used to receive input to + * support handwriting (Scribe) by the IME. + */ + public abstract SurfaceControl getHandwritingSurfaceForDisplay(int displayId); } diff --git a/services/core/java/com/android/server/wm/WindowManagerService.java b/services/core/java/com/android/server/wm/WindowManagerService.java index 5397c48689f03..056b0ed4f504a 100644 --- a/services/core/java/com/android/server/wm/WindowManagerService.java +++ b/services/core/java/com/android/server/wm/WindowManagerService.java @@ -7957,6 +7957,25 @@ public class WindowManagerService extends IWindowManager.Stub task.removeOverlay(overlay); } } + + @Override + public SurfaceControl getHandwritingSurfaceForDisplay(int displayId) { + synchronized (mGlobalLock) { + final DisplayContent dc = mRoot.getDisplayContent(displayId); + if (dc == null) { + Slog.e(TAG, "Failed to create a handwriting surface on display: " + + displayId + " - DisplayContent not found."); + return null; + } + //TODO (b/210039666): Use a method like add/removeDisplayOverlay if available. + return makeSurfaceBuilder(dc.getSession()) + .setContainerLayer() + .setName("IME Handwriting Surface") + .setCallsite("getHandwritingSurfaceForDisplay") + .setParent(dc.getSurfaceControl()) + .build(); + } + } } void registerAppFreezeListener(AppFreezeListener listener) {