From 4964839ec3b3648ef64782a7005f6ec18cd85548 Mon Sep 17 00:00:00 2001 From: Ming-Shin Lu Date: Fri, 16 Oct 2020 01:23:55 +0800 Subject: [PATCH] Better IME transition while switching app with recents (4/N) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This CL introduces TaskSnapshotController#snapshotImeFromAttachedTask to attach IME screenshot when performing closing transition. This can improve app transition without jank or flickering while the IME insets control transits from closing task to the next task, we keep the IME surface visibility by placing the IME screenshot with calling DC#showImeScreenshot to be a part of task while animating app transition, and then remove it with DC#removeImeScreenshotIfNeeded when the transition animation finished or no longer used gracefully.   Bug: 166736352 Bug: 153145997 Bug: 172815805 Bug: 174222049 Bug: 167604724 Test: manual as below steps: 1) Launch an app with focusing an editor (e.g. Dialer) 2) Swipe down status bar and tap Settings icon. 3) Verify that when doing task transition animation, app activity with IME keeps visible. Test: manual as below steps: 1) With 2-button or 3-button gesture, launch an app with focusing an editor to show soft-keyboard. 2) Pressing home key 3) Verify the IME screenshot keeps visible and animates with closing transition smoothly. Change-Id: I6bef36c779a28777408576f57e5d1c67d5d48e3f --- .../com/android/server/wm/DisplayContent.java | 127 ++++++++++++++++++ .../server/wm/TaskSnapshotController.java | 117 +++++++++++----- .../android/server/wm/WindowContainer.java | 7 + .../com/android/server/wm/WindowState.java | 2 + .../server/wm/DisplayContentTests.java | 62 +++++++++ 5 files changed, 284 insertions(+), 31 deletions(-) diff --git a/services/core/java/com/android/server/wm/DisplayContent.java b/services/core/java/com/android/server/wm/DisplayContent.java index b7970cdf20d10..7e09e63781815 100644 --- a/services/core/java/com/android/server/wm/DisplayContent.java +++ b/services/core/java/com/android/server/wm/DisplayContent.java @@ -113,6 +113,8 @@ import static com.android.server.wm.DisplayContentProto.OPENING_APPS; import static com.android.server.wm.DisplayContentProto.RESUMED_ACTIVITY; import static com.android.server.wm.DisplayContentProto.ROOT_DISPLAY_AREA; import static com.android.server.wm.DisplayContentProto.SCREEN_ROTATION_ANIMATION; +import static com.android.server.wm.SurfaceAnimator.ANIMATION_TYPE_APP_TRANSITION; +import static com.android.server.wm.SurfaceAnimator.ANIMATION_TYPE_RECENTS; import static com.android.server.wm.Task.ActivityState.RESUMED; import static com.android.server.wm.WindowContainer.AnimationFlags.PARENTS; import static com.android.server.wm.WindowContainer.AnimationFlags.TRANSITION; @@ -163,6 +165,7 @@ import android.graphics.Rect; import android.graphics.RectF; import android.graphics.Region; import android.graphics.Region.Op; +import android.hardware.HardwareBuffer; import android.hardware.display.DisplayManagerInternal; import android.metrics.LogMaker; import android.os.Binder; @@ -606,6 +609,9 @@ class DisplayContent extends RootDisplayArea implements WindowManagerPolicy.Disp /** If {@code true} hold off on modifying the animation layer of {@link #mImeLayeringTarget} */ boolean mImeLayeringTargetWaitingAnim; + /** The screenshot IME surface to place on the task while transitioning to the next task. */ + SurfaceControl mImeScreenshot; + private final PointerEventDispatcher mPointerEventDispatcher; private final InsetsStateController mInsetsStateController; @@ -3734,6 +3740,14 @@ class DisplayContent extends RootDisplayArea implements WindowManagerPolicy.Disp if (target == mImeLayeringTarget && mImeLayeringTargetWaitingAnim == targetWaitingAnim) { return; } + // Prepare the IME screenshot for the last IME target when its task is applying app + // transition. This is for the better IME transition to keep IME visibility when + // transitioning to the next task. + if (mImeLayeringTarget != null && mImeLayeringTarget.isAnimating(PARENTS | TRANSITION, + ANIMATION_TYPE_APP_TRANSITION | ANIMATION_TYPE_RECENTS)) { + attachAndShowImeScreenshotOnTarget(); + } + ProtoLog.i(WM_DEBUG_IME, "setInputMethodTarget %s", target); mImeLayeringTarget = target; mImeLayeringTargetWaitingAnim = targetWaitingAnim; @@ -3768,6 +3782,109 @@ class DisplayContent extends RootDisplayArea implements WindowManagerPolicy.Disp mImeControlTarget = target; } + @VisibleForTesting + void attachAndShowImeScreenshotOnTarget() { + // No need to attach screenshot if the IME target not exists or screen is off. + if (!isImeAttachedToApp() || !mWmService.mPolicy.isScreenOn()) { + return; + } + + final SurfaceControl.Transaction t = getPendingTransaction(); + // Prepare IME screenshot for the target if it allows to attach into. + if (mInputMethodWindow != null && mInputMethodWindow.isVisible()) { + final Task task = mImeLayeringTarget.getTask(); + // Re-new the IME screenshot when it does not exist or the size changed. + final boolean renewImeSurface = mImeScreenshot == null + || mImeScreenshot.getWidth() != mInputMethodWindow.getFrame().width() + || mImeScreenshot.getHeight() != mInputMethodWindow.getFrame().height(); + if (task != null && !task.isHomeOrRecentsRootTask()) { + SurfaceControl.ScreenshotHardwareBuffer imeBuffer = renewImeSurface + ? mWmService.mTaskSnapshotController.snapshotImeFromAttachedTask(task) + : null; + if (imeBuffer != null) { + // Remove the last IME surface when the surface needs to renew. + removeImeSurfaceImmediately(); + mImeScreenshot = createImeSurface(imeBuffer, t); + } + } + } + + final boolean isValidSnapshot = mImeScreenshot != null && mImeScreenshot.isValid(); + // Showing the IME screenshot if the target has already in app transition stage. + // Note that if the current IME insets is not showing, no need to show IME screenshot + // to reflect the true IME insets visibility and the app task layout as possible. + if (isValidSnapshot && getInsetsStateController().getImeSourceProvider().isImeShowing()) { + if (DEBUG_INPUT_METHOD) { + Slog.d(TAG, "show IME snapshot, ime target=" + mImeLayeringTarget); + } + t.show(mImeScreenshot); + } else if (!isValidSnapshot) { + removeImeSurfaceImmediately(); + } + } + + @VisibleForTesting + SurfaceControl createImeSurface(SurfaceControl.ScreenshotHardwareBuffer imeBuffer, + Transaction t) { + final HardwareBuffer buffer = imeBuffer.getHardwareBuffer(); + if (DEBUG_INPUT_METHOD) Slog.d(TAG, "create IME snapshot for " + + mImeLayeringTarget + ", buff width=" + buffer.getWidth() + + ", height=" + buffer.getHeight()); + final ActivityRecord activity = mImeLayeringTarget.mActivityRecord; + final SurfaceControl imeSurface = mWmService.mSurfaceControlFactory.apply(null) + .setName("IME-snapshot-surface") + .setBufferSize(buffer.getWidth(), buffer.getHeight()) + .setFormat(buffer.getFormat()) + .setParent(activity.getSurfaceControl()) + .setCallsite("DisplayContent.attachAndShowImeScreenshotOnTarget") + .build(); + // Make IME snapshot as trusted overlay + InputMonitor.setTrustedOverlayInputInfo(imeSurface, t, getDisplayId(), + "IME-snapshot-surface"); + Surface surface = mWmService.mSurfaceFactory.get(); + surface.copyFrom(imeSurface); + surface.attachAndQueueBufferWithColorSpace(buffer, null); + surface.release(); + t.setRelativeLayer(imeSurface, activity.getSurfaceControl(), 1); + t.setPosition(imeSurface, mInputMethodWindow.getDisplayFrame().left, + mInputMethodWindow.getDisplayFrame().top); + return imeSurface; + } + + /** + * Shows the IME screenshot and attach to the IME target window. + * + * Used when the IME target window with IME visible is transitioning to the next target. + * e.g. App transitioning or swiping this the task of the IME target window to recents app. + */ + void showImeScreenshot() { + attachAndShowImeScreenshotOnTarget(); + } + + /** + * Removes the IME screenshot when necessary. + * + * Used when app transition animation finished or obsoleted screenshot surface like size + * changed by rotation. + */ + void removeImeScreenshotIfPossible() { + if (mImeLayeringTarget == null + || mImeLayeringTarget.mAttrs.type != TYPE_APPLICATION_STARTING + && !mImeLayeringTarget.isAnimating(PARENTS | TRANSITION, + ANIMATION_TYPE_APP_TRANSITION | ANIMATION_TYPE_RECENTS)) { + removeImeSurfaceImmediately(); + } + } + + /** Removes the IME screenshot immediately. */ + void removeImeSurfaceImmediately() { + if (mImeScreenshot != null) { + if (DEBUG_INPUT_METHOD) Slog.d(TAG, "remove IME snapshot"); + getSyncTransaction().remove(mImeScreenshot); + mImeScreenshot = null; + } + } + /** * The IME input target is the window which receives input from IME. It is also a candidate * which controls the visibility and animation of the input method window. @@ -4037,6 +4154,16 @@ class DisplayContent extends RootDisplayArea implements WindowManagerPolicy.Disp mWmService.mWindowPlacerLocked.performSurfacePlacement(); } + /** + * Callbacks when the given type of {@link WindowContainer} animation finished running in the + * hierarchy. + */ + void onWindowAnimationFinished(int type) { + if (type == ANIMATION_TYPE_APP_TRANSITION || type == ANIMATION_TYPE_RECENTS) { + removeImeSurfaceImmediately(); + } + } + // TODO: Super unexpected long method that should be broken down... void applySurfaceChangesTransaction() { final WindowSurfacePlacer surfacePlacer = mWmService.mWindowPlacerLocked; diff --git a/services/core/java/com/android/server/wm/TaskSnapshotController.java b/services/core/java/com/android/server/wm/TaskSnapshotController.java index 513fa70b19654..b810de99ee104 100644 --- a/services/core/java/com/android/server/wm/TaskSnapshotController.java +++ b/services/core/java/com/android/server/wm/TaskSnapshotController.java @@ -35,6 +35,7 @@ import android.hardware.HardwareBuffer; import android.os.Environment; import android.os.Handler; import android.util.ArraySet; +import android.util.Pair; import android.util.Slog; import android.view.InsetsState; import android.view.SurfaceControl; @@ -275,39 +276,12 @@ class TaskSnapshotController { */ @VisibleForTesting boolean prepareTaskSnapshot(Task task, int pixelFormat, TaskSnapshot.Builder builder) { - if (!mService.mPolicy.isScreenOn()) { - if (DEBUG_SCREENSHOT) { - Slog.i(TAG_WM, "Attempted to take screenshot while display was off."); - } + final Pair result = checkIfReadyToSnapshot(task); + if (result == null) { return false; } - final ActivityRecord activity = findAppTokenForSnapshot(task); - if (activity == null) { - if (DEBUG_SCREENSHOT) { - Slog.w(TAG_WM, "Failed to take screenshot. No visible windows for " + task); - } - return false; - } - if (activity.hasCommittedReparentToAnimationLeash()) { - if (DEBUG_SCREENSHOT) { - Slog.w(TAG_WM, "Failed to take screenshot. App is animating " + activity); - } - return false; - } - - final WindowState mainWindow = activity.findMainWindow(); - if (mainWindow == null) { - Slog.w(TAG_WM, "Failed to take screenshot. No main window for " + task); - return false; - } - if (activity.hasFixedRotationTransform()) { - if (DEBUG_SCREENSHOT) { - Slog.i(TAG_WM, "Skip taking screenshot. App has fixed rotation " + activity); - } - // The activity is in a temporal state that it has different rotation than the task. - return false; - } - + final ActivityRecord activity = result.first; + final WindowState mainWindow = result.second; final Rect contentInsets = getSystemBarInsets(task.getBounds(), mainWindow.getInsetsStateWithVisibilityOverride()); InsetUtils.addInsets(contentInsets, activity.getLetterboxInsets()); @@ -339,6 +313,50 @@ class TaskSnapshotController { return true; } + /** + * Check if the state of the Task is appropriate to capture a snapshot, such like the task + * snapshot or the associated IME surface snapshot. + * + * @param task the target task to capture the snapshot + * @return Pair of (the top activity of the task, the main window of the task) if passed the + * state checking. Returns {@code null} if the task state isn't ready to snapshot. + */ + Pair checkIfReadyToSnapshot(Task task) { + if (!mService.mPolicy.isScreenOn()) { + if (DEBUG_SCREENSHOT) { + Slog.i(TAG_WM, "Attempted to take screenshot while display was off."); + } + return null; + } + final ActivityRecord activity = findAppTokenForSnapshot(task); + if (activity == null) { + if (DEBUG_SCREENSHOT) { + Slog.w(TAG_WM, "Failed to take screenshot. No visible windows for " + task); + } + return null; + } + if (activity.hasCommittedReparentToAnimationLeash()) { + if (DEBUG_SCREENSHOT) { + Slog.w(TAG_WM, "Failed to take screenshot. App is animating " + activity); + } + return null; + } + + final WindowState mainWindow = activity.findMainWindow(); + if (mainWindow == null) { + Slog.w(TAG_WM, "Failed to take screenshot. No main window for " + task); + return null; + } + if (activity.hasFixedRotationTransform()) { + if (DEBUG_SCREENSHOT) { + Slog.i(TAG_WM, "Skip taking screenshot. App has fixed rotation " + activity); + } + // The activity is in a temporal state that it has different rotation than the task. + return null; + } + return new Pair<>(activity, mainWindow); + } + @Nullable SurfaceControl.ScreenshotHardwareBuffer createTaskSnapshot(@NonNull Task task, TaskSnapshot.Builder builder) { @@ -355,6 +373,43 @@ class TaskSnapshotController { return createTaskSnapshot(task, scaleFraction, PixelFormat.RGBA_8888, null, builder); } + @Nullable + private SurfaceControl.ScreenshotHardwareBuffer createImeSnapshot(@NonNull Task task, + int pixelFormat) { + if (task.getSurfaceControl() == null) { + if (DEBUG_SCREENSHOT) { + Slog.w(TAG_WM, "Failed to take screenshot. No surface control for " + task); + } + return null; + } + final WindowState imeWindow = task.getDisplayContent().mInputMethodWindow; + SurfaceControl.ScreenshotHardwareBuffer imeBuffer = null; + if (imeWindow != null && imeWindow.isWinVisibleLw()) { + final Rect bounds = imeWindow.getContainingFrame(); + bounds.offsetTo(0, 0); + imeBuffer = SurfaceControl.captureLayersExcluding(imeWindow.getSurfaceControl(), + bounds, 1.0f, pixelFormat, null); + } + return imeBuffer; + } + + /** + * Create the snapshot of the IME surface on the task which used for placing on the closing + * task to keep IME visibility while app transitioning. + */ + @Nullable + SurfaceControl.ScreenshotHardwareBuffer snapshotImeFromAttachedTask(@NonNull Task task) { + // Check if the IME targets task ready to take the corresponding IME snapshot, if not, + // means the task is not yet visible for some reasons and no need to snapshot IME surface. + if (checkIfReadyToSnapshot(task) == null) { + return null; + } + final int pixelFormat = mPersister.use16BitFormat() + ? PixelFormat.RGB_565 + : PixelFormat.RGBA_8888; + return createImeSnapshot(task, pixelFormat); + } + @Nullable SurfaceControl.ScreenshotHardwareBuffer createTaskSnapshot(@NonNull Task task, float scaleFraction, int pixelFormat, Point outTaskSize, TaskSnapshot.Builder builder) { diff --git a/services/core/java/com/android/server/wm/WindowContainer.java b/services/core/java/com/android/server/wm/WindowContainer.java index 19a750abe1a75..2b1e7834cd0b6 100644 --- a/services/core/java/com/android/server/wm/WindowContainer.java +++ b/services/core/java/com/android/server/wm/WindowContainer.java @@ -2682,6 +2682,10 @@ class WindowContainer extends ConfigurationContainer< protected void applyAnimationUnchecked(WindowManager.LayoutParams lp, boolean enter, @TransitionOldType int transit, boolean isVoiceInteraction, @Nullable ArrayList sources) { + final Task task = asTask(); + if (task != null && !enter && !task.isHomeOrRecentsRootTask()) { + mDisplayContent.showImeScreenshot(); + } final Pair adapters = getAnimationAdapter(lp, transit, enter, isVoiceInteraction); AnimationAdapter adapter = adapters.first; @@ -2826,6 +2830,9 @@ class WindowContainer extends ConfigurationContainer< mSurfaceAnimationSources.valueAt(i).onAnimationFinished(type, anim); } mSurfaceAnimationSources.clear(); + if (mDisplayContent != null) { + mDisplayContent.onWindowAnimationFinished(type); + } } /** diff --git a/services/core/java/com/android/server/wm/WindowState.java b/services/core/java/com/android/server/wm/WindowState.java index 621b9719cd399..571946a7ace6d 100644 --- a/services/core/java/com/android/server/wm/WindowState.java +++ b/services/core/java/com/android/server/wm/WindowState.java @@ -2166,6 +2166,8 @@ class WindowState extends WindowContainer implements WindowManagerP final DisplayContent dc = getDisplayContent(); if (isImeLayeringTarget()) { + // Remove the IME screenshot surface if the layering target is not animating. + dc.removeImeScreenshotIfPossible(); // Make sure to set mImeLayeringTarget as null when the removed window is the // IME target, in case computeImeTarget may use the outdated target. dc.setImeLayeringTarget(null); diff --git a/services/tests/wmtests/src/com/android/server/wm/DisplayContentTests.java b/services/tests/wmtests/src/com/android/server/wm/DisplayContentTests.java index 30c1008010023..b27d016beb8d9 100644 --- a/services/tests/wmtests/src/com/android/server/wm/DisplayContentTests.java +++ b/services/tests/wmtests/src/com/android/server/wm/DisplayContentTests.java @@ -65,7 +65,11 @@ import static com.android.dx.mockito.inline.extended.ExtendedMockito.times; import static com.android.dx.mockito.inline.extended.ExtendedMockito.verify; import static com.android.server.wm.DisplayContent.IME_TARGET_INPUT; import static com.android.server.wm.DisplayContent.IME_TARGET_LAYERING; +import static com.android.server.wm.SurfaceAnimator.ANIMATION_TYPE_APP_TRANSITION; import static com.android.server.wm.SurfaceAnimator.ANIMATION_TYPE_FIXED_TRANSFORM; +import static com.android.server.wm.SurfaceAnimator.ANIMATION_TYPE_RECENTS; +import static com.android.server.wm.WindowContainer.AnimationFlags.PARENTS; +import static com.android.server.wm.WindowContainer.AnimationFlags.TRANSITION; import static com.android.server.wm.WindowContainer.POSITION_TOP; import static com.android.server.wm.WindowManagerService.UPDATE_FOCUS_NORMAL; @@ -81,6 +85,7 @@ import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.atLeast; import static org.mockito.Mockito.atLeastOnce; import static org.mockito.Mockito.clearInvocations; import static org.mockito.Mockito.doAnswer; @@ -1732,6 +1737,63 @@ public class DisplayContentTests extends WindowTestsBase { verify(child1, never()).needsRelativeLayeringToIme(); } + @UseTestDisplay(addWindows = {W_INPUT_METHOD}, addAllCommonWindows = true) + @Test + public void testAttachAndShowImeScreenshotOnTarget() { + // Preparation: Simulate screen state is on. + spyOn(mWm.mPolicy); + doReturn(true).when(mWm.mPolicy).isScreenOn(); + + // Preparation: Simulate snapshot IME surface. + spyOn(mWm.mTaskSnapshotController); + doReturn(mock(SurfaceControl.ScreenshotHardwareBuffer.class)).when( + mWm.mTaskSnapshotController).snapshotImeFromAttachedTask(any()); + final SurfaceControl imeSurface = mock(SurfaceControl.class); + spyOn(imeSurface); + doReturn(true).when(imeSurface).isValid(); + doReturn(imeSurface).when(mDisplayContent).createImeSurface(any(), any()); + + // Preparation: Simulate snapshot Task. + ActivityRecord act1 = createActivityRecord(mDisplayContent); + final WindowState appWin1 = createWindow(null, TYPE_BASE_APPLICATION, act1, "appWin1"); + spyOn(appWin1); + spyOn(appWin1.mWinAnimator); + appWin1.setHasSurface(true); + assertTrue(appWin1.canBeImeTarget()); + doReturn(true).when(appWin1.mWinAnimator).getShown(); + doReturn(true).when(appWin1.mActivityRecord).isSurfaceShowing(); + appWin1.mWinAnimator.mLastAlpha = 1f; + + // Test step 1: appWin1 is the current IME target and soft-keyboard is visible. + mDisplayContent.computeImeTarget(true); + assertEquals(appWin1, mDisplayContent.getImeTarget(IME_TARGET_LAYERING)); + spyOn(mDisplayContent.mInputMethodWindow); + doReturn(true).when(mDisplayContent.mInputMethodWindow).isVisible(); + mDisplayContent.getInsetsStateController().getImeSourceProvider().setImeShowing(true); + + // Test step 2: Simulate launching appWin2 and appWin1 is in app transition. + ActivityRecord act2 = createActivityRecord(mDisplayContent); + final WindowState appWin2 = createWindow(null, TYPE_BASE_APPLICATION, act2, "appWin2"); + appWin2.setHasSurface(true); + assertTrue(appWin2.canBeImeTarget()); + doReturn(true).when(appWin1).isAnimating(PARENTS | TRANSITION, + ANIMATION_TYPE_APP_TRANSITION | ANIMATION_TYPE_RECENTS); + + // Test step 3: Verify appWin2 will be the next IME target and the IME snapshot surface will + // be shown at this time. + final Transaction t = mDisplayContent.getPendingTransaction(); + spyOn(t); + mDisplayContent.setImeInputTarget(appWin2); + mDisplayContent.computeImeTarget(true); + assertEquals(appWin2, mDisplayContent.getImeTarget(IME_TARGET_LAYERING)); + assertTrue(mDisplayContent.isImeAttachedToApp()); + + verify(mDisplayContent, atLeast(1)).attachAndShowImeScreenshotOnTarget(); + verify(mWm.mTaskSnapshotController).snapshotImeFromAttachedTask(appWin1.getTask()); + assertNotNull(mDisplayContent.mImeScreenshot); + verify(t).show(mDisplayContent.mImeScreenshot); + } + private boolean isOptionsPanelAtRight(int displayId) { return (mWm.getPreferredOptionsPanelGravity(displayId) & Gravity.RIGHT) == Gravity.RIGHT; }