From 4f4eece2c428bf87ea26a110067ee4a94048838e Mon Sep 17 00:00:00 2001 From: Prabir Pradhan Date: Thu, 3 Mar 2022 20:44:20 +0000 Subject: [PATCH] Ensure handwriting session can only be started from inside app bounds Initialize the touchable region of the handwriting window to be cropped to the focused window bounds. This means the handwriting window will only spy over the app bounds. When attempting to start a handwriting session, ensure that we are currently recording a gesture in the handwriting window. This means we can only start handwriting if the gesture originated within the app bounds. When a handwriting session starts, change the touchable region of the handwriting window to be over the entire display, so that scribbles outside the app window can sent to the IME. Bug: 226081811 Test: manual: using test apps in split screen Change-Id: I95b67ec7be40dc80ed67c91d0ee0b32296e532ca --- core/java/android/view/InputWindowHandle.java | 32 +++++++++++++++++++ .../HandwritingEventReceiverSurface.java | 18 +++++++---- .../HandwritingModeController.java | 25 ++++++++++++--- .../InputMethodManagerService.java | 11 +++++-- .../server/wm/WindowManagerInternal.java | 23 +++++++++++++ .../server/wm/WindowManagerService.java | 21 ++++++++++++ 6 files changed, 116 insertions(+), 14 deletions(-) diff --git a/core/java/android/view/InputWindowHandle.java b/core/java/android/view/InputWindowHandle.java index ef0c19c8dc02b..12f30f1f05c32 100644 --- a/core/java/android/view/InputWindowHandle.java +++ b/core/java/android/view/InputWindowHandle.java @@ -164,6 +164,38 @@ public final class InputWindowHandle { this.displayId = displayId; } + public InputWindowHandle(InputWindowHandle other) { + // Do not copy ptr to prevent this copy from sharing the same native object. + ptr = 0; + inputApplicationHandle = new InputApplicationHandle(other.inputApplicationHandle); + token = other.token; + windowToken = other.windowToken; + window = other.window; + name = other.name; + layoutParamsFlags = other.layoutParamsFlags; + layoutParamsType = other.layoutParamsType; + dispatchingTimeoutMillis = other.dispatchingTimeoutMillis; + frameLeft = other.frameLeft; + frameTop = other.frameTop; + frameRight = other.frameRight; + frameBottom = other.frameBottom; + surfaceInset = other.surfaceInset; + scaleFactor = other.scaleFactor; + touchableRegion.set(other.touchableRegion); + inputConfig = other.inputConfig; + touchOcclusionMode = other.touchOcclusionMode; + ownerPid = other.ownerPid; + ownerUid = other.ownerUid; + packageName = other.packageName; + displayId = other.displayId; + touchableRegionSurfaceControl = other.touchableRegionSurfaceControl; + replaceTouchableRegionWithCrop = other.replaceTouchableRegionWithCrop; + if (other.transform != null) { + transform = new Matrix(); + transform.set(other.transform); + } + } + @Override public String toString() { return new StringBuilder(name != null ? name : "") diff --git a/services/core/java/com/android/server/inputmethod/HandwritingEventReceiverSurface.java b/services/core/java/com/android/server/inputmethod/HandwritingEventReceiverSurface.java index 1a19385e71c52..db17c1056e392 100644 --- a/services/core/java/com/android/server/inputmethod/HandwritingEventReceiverSurface.java +++ b/services/core/java/com/android/server/inputmethod/HandwritingEventReceiverSurface.java @@ -27,7 +27,6 @@ import android.view.InputWindowHandle; import android.view.SurfaceControl; import android.view.WindowManager; - final class HandwritingEventReceiverSurface { public static final String TAG = HandwritingEventReceiverSurface.class.getSimpleName(); @@ -39,7 +38,6 @@ final class HandwritingEventReceiverSurface { // TODO(b/217538817): Specify the ordering in WM by usage. private static final int HANDWRITING_SURFACE_LAYER = Integer.MAX_VALUE - 1; - private final InputApplicationHandle mApplicationHandle; private final InputWindowHandle mWindowHandle; private final InputChannel mClientChannel; private final SurfaceControl mInputSurface; @@ -47,13 +45,11 @@ final class HandwritingEventReceiverSurface { HandwritingEventReceiverSurface(String name, int displayId, @NonNull SurfaceControl sc, @NonNull InputChannel inputChannel) { - mApplicationHandle = new InputApplicationHandle(null, name, - DEFAULT_DISPATCHING_TIMEOUT_MILLIS); - mClientChannel = inputChannel; mInputSurface = sc; - mWindowHandle = new InputWindowHandle(mApplicationHandle, displayId); + mWindowHandle = new InputWindowHandle(new InputApplicationHandle(null, name, + DEFAULT_DISPATCHING_TIMEOUT_MILLIS), displayId); mWindowHandle.name = name; mWindowHandle.token = mClientChannel.getToken(); mWindowHandle.layoutParamsType = WindowManager.LayoutParams.TYPE_SECURE_SYSTEM_OVERLAY; @@ -61,7 +57,6 @@ final class HandwritingEventReceiverSurface { mWindowHandle.ownerPid = Process.myPid(); mWindowHandle.ownerUid = Process.myUid(); mWindowHandle.scaleFactor = 1.0f; - mWindowHandle.replaceTouchableRegionWithCrop(null /* use this surface's bounds */); mWindowHandle.inputConfig = InputConfig.NOT_FOCUSABLE | InputConfig.NOT_TOUCHABLE @@ -69,6 +64,7 @@ final class HandwritingEventReceiverSurface { | InputConfig.INTERCEPTS_STYLUS | InputConfig.TRUSTED_OVERLAY; + // The touchable region of this input surface is not initially configured. final SurfaceControl.Transaction t = new SurfaceControl.Transaction(); t.setInputWindowInfo(mInputSurface, mWindowHandle); t.setLayer(mInputSurface, HANDWRITING_SURFACE_LAYER); @@ -85,6 +81,10 @@ final class HandwritingEventReceiverSurface { mWindowHandle.ownerUid = imeUid; mWindowHandle.inputConfig &= ~InputConfig.SPY; + // Update the touchable region so that the IME can intercept stylus events + // across the entire display. + mWindowHandle.replaceTouchableRegionWithCrop(null /* use this surface's bounds */); + new SurfaceControl.Transaction() .setInputWindowInfo(mInputSurface, mWindowHandle) .apply(); @@ -108,4 +108,8 @@ final class HandwritingEventReceiverSurface { SurfaceControl getSurface() { return mInputSurface; } + + InputWindowHandle getInputWindowHandle() { + return mWindowHandle; + } } diff --git a/services/core/java/com/android/server/inputmethod/HandwritingModeController.java b/services/core/java/com/android/server/inputmethod/HandwritingModeController.java index b301d99b00de5..a706772225063 100644 --- a/services/core/java/com/android/server/inputmethod/HandwritingModeController.java +++ b/services/core/java/com/android/server/inputmethod/HandwritingModeController.java @@ -22,6 +22,7 @@ import android.annotation.AnyThread; import android.annotation.Nullable; import android.annotation.UiThread; import android.hardware.input.InputManagerInternal; +import android.os.IBinder; import android.os.Looper; import android.util.Slog; import android.view.BatchedInputEventReceiver; @@ -90,7 +91,7 @@ final class HandwritingModeController { * InputEventReceiver that batches events according to the current thread's Choreographer. */ @UiThread - void initializeHandwritingSpy(int displayId) { + void initializeHandwritingSpy(int displayId, IBinder focusedWindowToken) { // When resetting, reuse resources if we are reinitializing on the same display. reset(displayId == mCurrentDisplayId); mCurrentDisplayId = displayId; @@ -110,8 +111,16 @@ final class HandwritingModeController { Slog.e(TAG, "Failed to create input surface"); return; } - mHandwritingSurface = - new HandwritingEventReceiverSurface(name, displayId, surface, channel); + + mHandwritingSurface = new HandwritingEventReceiverSurface( + name, displayId, surface, channel); + + // Configure the handwriting window to receive events over the focused window's bounds. + mWindowManagerInternal.replaceInputSurfaceTouchableRegionWithWindowCrop( + mHandwritingSurface.getSurface(), + mHandwritingSurface.getInputWindowHandle(), + focusedWindowToken); + // Use a dup of the input channel so that event processing can be paused by disposing the // event receiver without causing a fd hangup. mHandwritingEventReceiver = new BatchedInputEventReceiver.SimpleBatchedInputEventReceiver( @@ -127,6 +136,10 @@ final class HandwritingModeController { return OptionalInt.of(mCurrentRequestId); } + boolean isStylusGestureOngoing() { + return mRecordingGesture; + } + /** * Starts a {@link HandwritingSession} to transfer to the IME. * @@ -145,6 +158,10 @@ final class HandwritingModeController { Slog.e(TAG, "Cannot start handwriting session: Invalid request id: " + requestId); return null; } + if (!mRecordingGesture) { + Slog.e(TAG, "Cannot start handwriting session: No stylus gesture is being recorded."); + return null; + } Objects.requireNonNull(mHandwritingEventReceiver, "Handwriting session was already transferred to IME."); if (DEBUG) Slog.d(TAG, "Starting handwriting session in display: " + mCurrentDisplayId); @@ -231,7 +248,7 @@ final class HandwritingModeController { mInkWindowInitRunnable = null; } - if (action == MotionEvent.ACTION_UP) { + if (action == MotionEvent.ACTION_UP || action == MotionEvent.ACTION_CANCEL) { mRecordingGesture = false; mHandwritingBuffer.clear(); return; diff --git a/services/core/java/com/android/server/inputmethod/InputMethodManagerService.java b/services/core/java/com/android/server/inputmethod/InputMethodManagerService.java index 77dcbd3e92773..fca9f56b2407b 100644 --- a/services/core/java/com/android/server/inputmethod/InputMethodManagerService.java +++ b/services/core/java/com/android/server/inputmethod/InputMethodManagerService.java @@ -3372,6 +3372,10 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub Slog.e(TAG, "Stylus handwriting was not initialized."); return; } + if (!mHwController.isStylusGestureOngoing()) { + Slog.e(TAG, "There is no ongoing stylus gesture to start stylus handwriting."); + return; + } if (DEBUG) Slog.v(TAG, "Client requesting Stylus Handwriting to be started"); final IInputMethodInvoker curMethod = getCurMethodLocked(); if (curMethod != null) { @@ -4764,8 +4768,9 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub case MSG_RESET_HANDWRITING: { synchronized (ImfLock.class) { if (mBindingController.supportsStylusHandwriting() - && getCurMethodLocked() != null) { - mHwController.initializeHandwritingSpy(mCurTokenDisplayId); + && getCurMethodLocked() != null && mCurFocusedWindow != null) { + mHwController.initializeHandwritingSpy( + mCurTokenDisplayId, mCurFocusedWindow); } else { mHwController.reset(); } @@ -4793,7 +4798,7 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub session.getHandwritingChannel(), session.getRecordedEvents())) { // When failed to issue IPCs, re-initialize handwriting state. Slog.w(TAG, "Resetting handwriting mode."); - mHwController.initializeHandwritingSpy(mCurTokenDisplayId); + scheduleResetStylusHandwriting(); } } return true; diff --git a/services/core/java/com/android/server/wm/WindowManagerInternal.java b/services/core/java/com/android/server/wm/WindowManagerInternal.java index 0a3c3f049f43e..be745966ce306 100644 --- a/services/core/java/com/android/server/wm/WindowManagerInternal.java +++ b/services/core/java/com/android/server/wm/WindowManagerInternal.java @@ -33,6 +33,7 @@ import android.view.IInputFilter; import android.view.IRemoteAnimationFinishedCallback; import android.view.IWindow; import android.view.InputChannel; +import android.view.InputWindowHandle; import android.view.MagnificationSpec; import android.view.RemoteAnimationTarget; import android.view.SurfaceControl; @@ -853,4 +854,26 @@ public abstract class WindowManagerInternal { * support handwriting (Scribe) by the IME. */ public abstract SurfaceControl getHandwritingSurfaceForDisplay(int displayId); + + /** + * Replaces the touchable region of the provided input surface with the crop of the window with + * the provided token. This method will associate the inputSurface with a copy of + * the given inputWindowHandle, where the copy is configured using + * {@link InputWindowHandle#replaceTouchableRegionWithCrop(SurfaceControl)} with the surface + * of the provided windowToken. + * + * This is a no-op if windowToken is not valid or the window is not found. + * + * This does not change any other properties of the inputSurface. + * + * This method exists to avoid leaking the window's SurfaceControl outside WindowManagerService. + * + * @param inputSurface The surface for which the touchable region should be set. + * @param inputWindowHandle The {@link InputWindowHandle} for the input surface. + * @param windowToken The window whose bounds should be used as the touchable region for the + * inputSurface. + */ + public abstract void replaceInputSurfaceTouchableRegionWithWindowCrop( + @NonNull SurfaceControl inputSurface, @NonNull InputWindowHandle inputWindowHandle, + @NonNull IBinder windowToken); } diff --git a/services/core/java/com/android/server/wm/WindowManagerService.java b/services/core/java/com/android/server/wm/WindowManagerService.java index c50888b5a1724..48cf971631fe5 100644 --- a/services/core/java/com/android/server/wm/WindowManagerService.java +++ b/services/core/java/com/android/server/wm/WindowManagerService.java @@ -8121,6 +8121,27 @@ public class WindowManagerService extends IWindowManager.Stub .build(); } } + + @Override + public void replaceInputSurfaceTouchableRegionWithWindowCrop( + @NonNull SurfaceControl inputSurface, + @NonNull InputWindowHandle inputWindowHandle, + @NonNull IBinder windowToken) { + synchronized (mGlobalLock) { + final WindowState w = mWindowMap.get(windowToken); + if (w == null) { + return; + } + // Make a copy of the InputWindowHandle to avoid leaking the window's + // SurfaceControl. + final InputWindowHandle localHandle = new InputWindowHandle(inputWindowHandle); + localHandle.replaceTouchableRegionWithCrop(w.getSurfaceControl()); + final SurfaceControl.Transaction t = mTransactionFactory.get(); + t.setInputWindowInfo(inputSurface, localHandle); + t.apply(); + t.close(); + } + } } void registerAppFreezeListener(AppFreezeListener listener) {