From 04dc4cc64bbc9902643780901d22ccf025efb4b7 Mon Sep 17 00:00:00 2001 From: Tiger Huang Date: Thu, 17 Jan 2019 18:41:41 +0800 Subject: [PATCH] Let the activity embedded in ActivityView can be directly touched The goal is to eliminate the latency caused by forwarding the event. In this CL, we put an input portal window behind an ActivityView, and subtract the touchable region of the the ActivityView, so that the user can touch through the portal window, and let the touch arrive on the embedded activity. Bug: 120675821 Test: Manual test with ActivityViewTest Test: atest CtsActivityManagerDeviceTestCases:ActivityViewTest Change-Id: I6776387b65cf6a7d2ea60cb00ffdd38be22081ac --- core/java/android/app/ActivityView.java | 53 ++------------ core/java/android/view/InputWindowHandle.java | 8 ++- ...droid_hardware_input_InputWindowHandle.cpp | 6 ++ .../com/android/server/wm/DisplayContent.java | 70 ++++++++++++++++--- .../server/wm/TapExcludeRegionHolder.java | 4 +- .../wm/TaskTapPointerEventListener.java | 15 +++- .../server/wm/WindowManagerService.java | 9 ++- .../com/android/server/wm/WindowState.java | 55 ++++++++++++++- 8 files changed, 153 insertions(+), 67 deletions(-) diff --git a/core/java/android/app/ActivityView.java b/core/java/android/app/ActivityView.java index 4d3711ae7ff5a..a122fcef42d87 100644 --- a/core/java/android/app/ActivityView.java +++ b/core/java/android/app/ActivityView.java @@ -27,15 +27,12 @@ import android.content.Context; import android.content.Intent; import android.hardware.display.DisplayManager; import android.hardware.display.VirtualDisplay; -import android.hardware.input.InputManager; import android.os.RemoteException; import android.os.UserHandle; import android.util.AttributeSet; import android.util.DisplayMetrics; import android.util.Log; import android.view.IWindowManager; -import android.view.InputDevice; -import android.view.MotionEvent; import android.view.SurfaceControl; import android.view.SurfaceHolder; import android.view.SurfaceSession; @@ -50,9 +47,7 @@ import dalvik.system.CloseGuard; import java.util.List; /** - * Activity container that allows launching activities into itself and does input forwarding. - *

Creation of this view is only allowed to callers who have - * {@link android.Manifest.permission#INJECT_EVENTS} permission. + * Activity container that allows launching activities into itself. *

Activity launching into this container is restricted by the same rules that apply to launching * on VirtualDisplays. * @hide @@ -75,9 +70,8 @@ public class ActivityView extends ViewGroup { private StateCallback mActivityViewCallback; private IActivityTaskManager mActivityTaskManager; - private IInputForwarder mInputForwarder; - // Temp container to store view coordinates on screen. - private final int[] mLocationOnScreen = new int[2]; + // Temp container to store view coordinates in window. + private final int[] mLocationInWindow = new int[2]; private TaskStackListener mTaskStackListener; @@ -264,7 +258,7 @@ public class ActivityView extends ViewGroup { } /** - * Triggers an update of {@link ActivityView}'s location on screen to properly set touch exclude + * Triggers an update of {@link ActivityView}'s location in window to properly set touch exclude * regions and avoid focus switches by touches on this view. */ public void onLocationChanged() { @@ -279,45 +273,14 @@ public class ActivityView extends ViewGroup { /** Send current location and size to the WM to set tap exclude region for this view. */ private void updateLocation() { try { - getLocationOnScreen(mLocationOnScreen); + getLocationInWindow(mLocationInWindow); WindowManagerGlobal.getWindowSession().updateTapExcludeRegion(getWindow(), hashCode(), - mLocationOnScreen[0], mLocationOnScreen[1], getWidth(), getHeight()); + mLocationInWindow[0], mLocationInWindow[1], getWidth(), getHeight()); } catch (RemoteException e) { e.rethrowAsRuntimeException(); } } - @Override - public boolean onTouchEvent(MotionEvent event) { - return injectInputEvent(event) || super.onTouchEvent(event); - } - - @Override - public boolean onGenericMotionEvent(MotionEvent event) { - if (event.isFromSource(InputDevice.SOURCE_CLASS_POINTER)) { - if (injectInputEvent(event)) { - return true; - } - } - return super.onGenericMotionEvent(event); - } - - private boolean injectInputEvent(MotionEvent event) { - if (mInputForwarder != null) { - try { - // The touch event that the ActivityView gets is in View space, but the event needs - // to get forwarded in screen space. This offsets the touch event by the location - // the ActivityView is on screen and sends it to the input forwarder. - getLocationOnScreen(mLocationOnScreen); - event.offsetLocation(mLocationOnScreen[0], mLocationOnScreen[1]); - return mInputForwarder.forwardEvent(event); - } catch (RemoteException e) { - e.rethrowAsRuntimeException(); - } - } - return false; - } - private class SurfaceCallback implements SurfaceHolder.Callback { @Override public void surfaceCreated(SurfaceHolder surfaceHolder) { @@ -400,7 +363,6 @@ public class ActivityView extends ViewGroup { } mTmpTransaction.show(mRootSurfaceControl).apply(); - mInputForwarder = InputManager.getInstance().createInputForwarder(displayId); mTaskStackListener = new TaskStackListenerImpl(); try { mActivityTaskManager.registerTaskStackListener(mTaskStackListener); @@ -416,9 +378,6 @@ public class ActivityView extends ViewGroup { mSurfaceView.getHolder().removeCallback(mSurfaceCallback); - if (mInputForwarder != null) { - mInputForwarder = null; - } cleanTapExcludeRegion(); if (mTaskStackListener != null) { diff --git a/core/java/android/view/InputWindowHandle.java b/core/java/android/view/InputWindowHandle.java index 92e0009bc21a7..ec79eea45ee6a 100644 --- a/core/java/android/view/InputWindowHandle.java +++ b/core/java/android/view/InputWindowHandle.java @@ -16,10 +16,10 @@ package android.view; +import static android.view.Display.INVALID_DISPLAY; + import android.graphics.Region; import android.os.IBinder; -import android.view.IWindow; -import android.view.InputChannel; /** * Functions as a handle for a window that can receive input. @@ -94,6 +94,10 @@ public final class InputWindowHandle { // Display this input is on. public int displayId; + // If this value is set to a valid display ID, it indicates this window is a portal which + // transports the touch of this window to the display indicated by portalToDisplayId. + public int portalToDisplayId = INVALID_DISPLAY; + private native void nativeDispose(); public InputWindowHandle(InputApplicationHandle inputApplicationHandle, diff --git a/core/jni/android_hardware_input_InputWindowHandle.cpp b/core/jni/android_hardware_input_InputWindowHandle.cpp index 76920f57ed08f..8ee4f42cdaa18 100644 --- a/core/jni/android_hardware_input_InputWindowHandle.cpp +++ b/core/jni/android_hardware_input_InputWindowHandle.cpp @@ -55,6 +55,7 @@ static struct { jfieldID ownerUid; jfieldID inputFeatures; jfieldID displayId; + jfieldID portalToDisplayId; } gInputWindowHandleClassInfo; static Mutex gHandleMutex; @@ -154,6 +155,8 @@ bool NativeInputWindowHandle::updateInfo() { gInputWindowHandleClassInfo.inputFeatures); mInfo.displayId = env->GetIntField(obj, gInputWindowHandleClassInfo.displayId); + mInfo.portalToDisplayId = env->GetIntField(obj, + gInputWindowHandleClassInfo.portalToDisplayId); jobject inputApplicationHandleObj = env->GetObjectField(obj, gInputWindowHandleClassInfo.inputApplicationHandle); @@ -307,6 +310,9 @@ int register_android_server_InputWindowHandle(JNIEnv* env) { GET_FIELD_ID(gInputWindowHandleClassInfo.displayId, clazz, "displayId", "I"); + + GET_FIELD_ID(gInputWindowHandleClassInfo.portalToDisplayId, clazz, + "portalToDisplayId", "I"); return 0; } diff --git a/services/core/java/com/android/server/wm/DisplayContent.java b/services/core/java/com/android/server/wm/DisplayContent.java index 8fefd352e027d..a9d76889ddbb6 100644 --- a/services/core/java/com/android/server/wm/DisplayContent.java +++ b/services/core/java/com/android/server/wm/DisplayContent.java @@ -29,6 +29,7 @@ import static android.content.res.Configuration.ORIENTATION_LANDSCAPE; import static android.content.res.Configuration.ORIENTATION_PORTRAIT; import static android.view.Display.DEFAULT_DISPLAY; import static android.view.Display.FLAG_PRIVATE; +import static android.view.Display.INVALID_DISPLAY; import static android.view.InsetsState.TYPE_IME; import static android.view.InsetsState.TYPE_NAVIGATION_BAR; import static android.view.InsetsState.TYPE_TOP_BAR; @@ -46,6 +47,7 @@ import static android.view.WindowManager.LayoutParams.FLAG_NOT_TOUCHABLE; import static android.view.WindowManager.LayoutParams.FLAG_NOT_TOUCH_MODAL; import static android.view.WindowManager.LayoutParams.FLAG_SECURE; import static android.view.WindowManager.LayoutParams.FLAG_SHOW_WALLPAPER; +import static android.view.WindowManager.LayoutParams.FLAG_SPLIT_TOUCH; import static android.view.WindowManager.LayoutParams.NEEDS_MENU_SET_TRUE; import static android.view.WindowManager.LayoutParams.NEEDS_MENU_UNSET; import static android.view.WindowManager.LayoutParams.PRIVATE_FLAG_KEYGUARD; @@ -143,9 +145,11 @@ import android.graphics.RectF; import android.graphics.Region; import android.graphics.Region.Op; import android.hardware.display.DisplayManagerInternal; +import android.os.Binder; import android.os.Debug; import android.os.Handler; import android.os.IBinder; +import android.os.Process; import android.os.RemoteException; import android.os.SystemClock; import android.os.Trace; @@ -160,6 +164,7 @@ import android.view.DisplayInfo; import android.view.Gravity; import android.view.InputChannel; import android.view.InputDevice; +import android.view.InputWindowHandle; import android.view.InsetsState.InternalInsetType; import android.view.MagnificationSpec; import android.view.Surface; @@ -517,6 +522,9 @@ class DisplayContent extends WindowContainer= 0; i--) { - final WindowState win = mTapExcludeProvidingWindows.valueAt(i); - win.amendTapExcludeRegion(mTouchExcludeRegion); - } + amendWindowTapExcludeRegion(mTouchExcludeRegion); // TODO(multi-display): Support docked stacks on secondary displays. if (mDisplayId == DEFAULT_DISPLAY && getSplitScreenPrimaryStack() != null) { mDividerControllerLocked.getTouchRegion(mTmpRect); @@ -2426,6 +2431,18 @@ class DisplayContent extends WindowContainer= 0; i--) { + final WindowState win = mTapExcludeProvidingWindows.valueAt(i); + win.amendTapExcludeRegion(inOutRegion); + } + } + @Override void switchUser() { super.switchUser(); @@ -3583,6 +3600,13 @@ class DisplayContent extends WindowContainer= 0 ; --i) { final Rect rect = mTapExcludeRects.valueAt(i); - rect.intersect(boundingRegion); + if (boundingRegion != null) { + rect.intersect(boundingRegion); + } region.union(rect); } } diff --git a/services/core/java/com/android/server/wm/TaskTapPointerEventListener.java b/services/core/java/com/android/server/wm/TaskTapPointerEventListener.java index 2e5df45f90808..dd94af657039f 100644 --- a/services/core/java/com/android/server/wm/TaskTapPointerEventListener.java +++ b/services/core/java/com/android/server/wm/TaskTapPointerEventListener.java @@ -70,6 +70,18 @@ public class TaskTapPointerEventListener implements PointerEventListener { // method target window will lose the focus. return; } + final Region windowTapExcludeRegion = Region.obtain(); + mDisplayContent.amendWindowTapExcludeRegion(windowTapExcludeRegion); + if (windowTapExcludeRegion.contains(x, y)) { + windowTapExcludeRegion.recycle(); + // The user is tapping on the window tap exclude region. We don't move this + // display to top. A window tap exclude region, for example, may be set by an + // ActivityView, and the region would match the bounds of both the ActivityView + // and the virtual display in it. In this case, we would take the tap that is on + // the embedded virtual display instead of this display. + return; + } + windowTapExcludeRegion.recycle(); WindowContainer parent = mDisplayContent.getParent(); if (parent != null && parent.getTopChild() != mDisplayContent) { parent.positionChildAt(WindowContainer.POSITION_TOP, mDisplayContent, @@ -81,9 +93,6 @@ public class TaskTapPointerEventListener implements PointerEventListener { @Override public void onPointerEvent(MotionEvent motionEvent) { - if (motionEvent.getDisplayId() != getDisplayId()) { - return; - } switch (motionEvent.getActionMasked()) { case MotionEvent.ACTION_DOWN: { final int x = (int) motionEvent.getX(); diff --git a/services/core/java/com/android/server/wm/WindowManagerService.java b/services/core/java/com/android/server/wm/WindowManagerService.java index c6679a9ad0d76..9ebca456749eb 100644 --- a/services/core/java/com/android/server/wm/WindowManagerService.java +++ b/services/core/java/com/android/server/wm/WindowManagerService.java @@ -6551,8 +6551,13 @@ public class WindowManagerService extends IWindowManager.Stub /** * Update a tap exclude region with a rectangular area in the window identified by the provided - * id. Touches on this region will not switch focus to this window. Passing an empty rect will - * remove the area from the exclude region of this window. + * id. Touches down on this region will not: + *

    + *
  1. Switch focus to this window.
    2. + *
  2. Move the display of this window to top.
    3. + *
  3. Send the touch events to this window.
    4. + *
+ * Passing an empty rect will remove the area from the exclude region of this window. */ void updateTapExcludeRegion(IWindow client, int regionId, int left, int top, int width, int height) { diff --git a/services/core/java/com/android/server/wm/WindowState.java b/services/core/java/com/android/server/wm/WindowState.java index ce5eb8476764a..01839c15bc738 100644 --- a/services/core/java/com/android/server/wm/WindowState.java +++ b/services/core/java/com/android/server/wm/WindowState.java @@ -536,7 +536,7 @@ class WindowState extends WindowContainer implements WindowManagerP private final Point mSurfacePosition = new Point(); /** - * A region inside of this window to be excluded from touch-related focus switches. + * A region inside of this window to be excluded from touch. */ private TapExcludeRegionHolder mTapExcludeRegionHolder; @@ -2168,6 +2168,24 @@ class WindowState extends WindowContainer implements WindowManagerP } region.set(mTmpRect); cropRegionToStackBoundsIfNeeded(region); + subtractTouchExcludeRegionIfNeeded(region); + } else if (modal && mTapExcludeRegionHolder != null) { + final Region touchExcludeRegion = Region.obtain(); + amendTapExcludeRegion(touchExcludeRegion); + if (!touchExcludeRegion.isEmpty()) { + // Remove touch modal because there are some areas that cannot be touched. + flags |= FLAG_NOT_TOUCH_MODAL; + // Give it a large touchable region at first because it was touch modal. The window + // might be moved on the display, so the touchable region should be large enough to + // ensure it covers the whole display, no matter where it is moved. + getDisplayContent().getBounds(mTmpRect); + final int dw = mTmpRect.width(); + final int dh = mTmpRect.height(); + region.set(-dw, -dh, dw + dw, dh + dh); + // Subtract the area that cannot be touched. + region.op(touchExcludeRegion, Region.Op.DIFFERENCE); + } + touchExcludeRegion.recycle(); } else { // Not modal or full screen modal getTouchableRegion(region); @@ -2837,6 +2855,7 @@ class WindowState extends WindowContainer implements WindowManagerP } } cropRegionToStackBoundsIfNeeded(outRegion); + subtractTouchExcludeRegionIfNeeded(outRegion); } private void cropRegionToStackBoundsIfNeeded(Region region) { @@ -2854,6 +2873,22 @@ class WindowState extends WindowContainer implements WindowManagerP region.op(mTmpRect, Region.Op.INTERSECT); } + /** + * If this window has areas that cannot be touched, we subtract those areas from its touchable + * region. + */ + private void subtractTouchExcludeRegionIfNeeded(Region touchableRegion) { + if (mTapExcludeRegionHolder == null) { + return; + } + final Region touchExcludeRegion = Region.obtain(); + amendTapExcludeRegion(touchExcludeRegion); + if (!touchExcludeRegion.isEmpty()) { + touchableRegion.op(touchExcludeRegion, Region.Op.DIFFERENCE); + } + touchExcludeRegion.recycle(); + } + /** * Report a focus change. Must be called with no locks held, and consistently * from the same serialized thread (such as dispatched from a handler). @@ -4716,11 +4751,25 @@ class WindowState extends WindowContainer implements WindowManagerP mTapExcludeRegionHolder.updateRegion(regionId, left, top, width, height); // Trigger touch exclude region update on current display. currentDisplay.updateTouchExcludeRegion(); + // Trigger touchable region update for this window. + currentDisplay.getInputMonitor().updateInputWindowsLw(true /* force */); } - /** Union the region with current tap exclude region that this window provides. */ + /** + * Union the region with current tap exclude region that this window provides. + * + * @param region The region to be amended. It is on the screen coordinates. + */ void amendTapExcludeRegion(Region region) { - mTapExcludeRegionHolder.amendRegion(region, getBounds()); + final Region tempRegion = Region.obtain(); + mTmpRect.set(mWindowFrames.mFrame); + mTmpRect.offsetTo(0, 0); + mTapExcludeRegionHolder.amendRegion(tempRegion, mTmpRect); + // The region held by the holder is on the window coordinates. We need to translate it to + // the screen coordinates. + tempRegion.translate(mWindowFrames.mFrame.left, mWindowFrames.mFrame.top); + region.op(tempRegion, Region.Op.UNION); + tempRegion.recycle(); } @Override