diff --git a/services/core/java/com/android/server/wm/DisplayContent.java b/services/core/java/com/android/server/wm/DisplayContent.java index b7970cdf20d10..7e09e63781815 100644 --- a/services/core/java/com/android/server/wm/DisplayContent.java +++ b/services/core/java/com/android/server/wm/DisplayContent.java @@ -113,6 +113,8 @@ import static com.android.server.wm.DisplayContentProto.OPENING_APPS; import static com.android.server.wm.DisplayContentProto.RESUMED_ACTIVITY; import static com.android.server.wm.DisplayContentProto.ROOT_DISPLAY_AREA; import static com.android.server.wm.DisplayContentProto.SCREEN_ROTATION_ANIMATION; +import static com.android.server.wm.SurfaceAnimator.ANIMATION_TYPE_APP_TRANSITION; +import static com.android.server.wm.SurfaceAnimator.ANIMATION_TYPE_RECENTS; import static com.android.server.wm.Task.ActivityState.RESUMED; import static com.android.server.wm.WindowContainer.AnimationFlags.PARENTS; import static com.android.server.wm.WindowContainer.AnimationFlags.TRANSITION; @@ -163,6 +165,7 @@ import android.graphics.Rect; import android.graphics.RectF; import android.graphics.Region; import android.graphics.Region.Op; +import android.hardware.HardwareBuffer; import android.hardware.display.DisplayManagerInternal; import android.metrics.LogMaker; import android.os.Binder; @@ -606,6 +609,9 @@ class DisplayContent extends RootDisplayArea implements WindowManagerPolicy.Disp /** If {@code true} hold off on modifying the animation layer of {@link #mImeLayeringTarget} */ boolean mImeLayeringTargetWaitingAnim; + /** The screenshot IME surface to place on the task while transitioning to the next task. */ + SurfaceControl mImeScreenshot; + private final PointerEventDispatcher mPointerEventDispatcher; private final InsetsStateController mInsetsStateController; @@ -3734,6 +3740,14 @@ class DisplayContent extends RootDisplayArea implements WindowManagerPolicy.Disp if (target == mImeLayeringTarget && mImeLayeringTargetWaitingAnim == targetWaitingAnim) { return; } + // Prepare the IME screenshot for the last IME target when its task is applying app + // transition. This is for the better IME transition to keep IME visibility when + // transitioning to the next task. + if (mImeLayeringTarget != null && mImeLayeringTarget.isAnimating(PARENTS | TRANSITION, + ANIMATION_TYPE_APP_TRANSITION | ANIMATION_TYPE_RECENTS)) { + attachAndShowImeScreenshotOnTarget(); + } + ProtoLog.i(WM_DEBUG_IME, "setInputMethodTarget %s", target); mImeLayeringTarget = target; mImeLayeringTargetWaitingAnim = targetWaitingAnim; @@ -3768,6 +3782,109 @@ class DisplayContent extends RootDisplayArea implements WindowManagerPolicy.Disp mImeControlTarget = target; } + @VisibleForTesting + void attachAndShowImeScreenshotOnTarget() { + // No need to attach screenshot if the IME target not exists or screen is off. + if (!isImeAttachedToApp() || !mWmService.mPolicy.isScreenOn()) { + return; + } + + final SurfaceControl.Transaction t = getPendingTransaction(); + // Prepare IME screenshot for the target if it allows to attach into. + if (mInputMethodWindow != null && mInputMethodWindow.isVisible()) { + final Task task = mImeLayeringTarget.getTask(); + // Re-new the IME screenshot when it does not exist or the size changed. + final boolean renewImeSurface = mImeScreenshot == null + || mImeScreenshot.getWidth() != mInputMethodWindow.getFrame().width() + || mImeScreenshot.getHeight() != mInputMethodWindow.getFrame().height(); + if (task != null && !task.isHomeOrRecentsRootTask()) { + SurfaceControl.ScreenshotHardwareBuffer imeBuffer = renewImeSurface + ? mWmService.mTaskSnapshotController.snapshotImeFromAttachedTask(task) + : null; + if (imeBuffer != null) { + // Remove the last IME surface when the surface needs to renew. + removeImeSurfaceImmediately(); + mImeScreenshot = createImeSurface(imeBuffer, t); + } + } + } + + final boolean isValidSnapshot = mImeScreenshot != null && mImeScreenshot.isValid(); + // Showing the IME screenshot if the target has already in app transition stage. + // Note that if the current IME insets is not showing, no need to show IME screenshot + // to reflect the true IME insets visibility and the app task layout as possible. + if (isValidSnapshot && getInsetsStateController().getImeSourceProvider().isImeShowing()) { + if (DEBUG_INPUT_METHOD) { + Slog.d(TAG, "show IME snapshot, ime target=" + mImeLayeringTarget); + } + t.show(mImeScreenshot); + } else if (!isValidSnapshot) { + removeImeSurfaceImmediately(); + } + } + + @VisibleForTesting + SurfaceControl createImeSurface(SurfaceControl.ScreenshotHardwareBuffer imeBuffer, + Transaction t) { + final HardwareBuffer buffer = imeBuffer.getHardwareBuffer(); + if (DEBUG_INPUT_METHOD) Slog.d(TAG, "create IME snapshot for " + + mImeLayeringTarget + ", buff width=" + buffer.getWidth() + + ", height=" + buffer.getHeight()); + final ActivityRecord activity = mImeLayeringTarget.mActivityRecord; + final SurfaceControl imeSurface = mWmService.mSurfaceControlFactory.apply(null) + .setName("IME-snapshot-surface") + .setBufferSize(buffer.getWidth(), buffer.getHeight()) + .setFormat(buffer.getFormat()) + .setParent(activity.getSurfaceControl()) + .setCallsite("DisplayContent.attachAndShowImeScreenshotOnTarget") + .build(); + // Make IME snapshot as trusted overlay + InputMonitor.setTrustedOverlayInputInfo(imeSurface, t, getDisplayId(), + "IME-snapshot-surface"); + Surface surface = mWmService.mSurfaceFactory.get(); + surface.copyFrom(imeSurface); + surface.attachAndQueueBufferWithColorSpace(buffer, null); + surface.release(); + t.setRelativeLayer(imeSurface, activity.getSurfaceControl(), 1); + t.setPosition(imeSurface, mInputMethodWindow.getDisplayFrame().left, + mInputMethodWindow.getDisplayFrame().top); + return imeSurface; + } + + /** + * Shows the IME screenshot and attach to the IME target window. + * + * Used when the IME target window with IME visible is transitioning to the next target. + * e.g. App transitioning or swiping this the task of the IME target window to recents app. + */ + void showImeScreenshot() { + attachAndShowImeScreenshotOnTarget(); + } + + /** + * Removes the IME screenshot when necessary. + * + * Used when app transition animation finished or obsoleted screenshot surface like size + * changed by rotation. + */ + void removeImeScreenshotIfPossible() { + if (mImeLayeringTarget == null + || mImeLayeringTarget.mAttrs.type != TYPE_APPLICATION_STARTING + && !mImeLayeringTarget.isAnimating(PARENTS | TRANSITION, + ANIMATION_TYPE_APP_TRANSITION | ANIMATION_TYPE_RECENTS)) { + removeImeSurfaceImmediately(); + } + } + + /** Removes the IME screenshot immediately. */ + void removeImeSurfaceImmediately() { + if (mImeScreenshot != null) { + if (DEBUG_INPUT_METHOD) Slog.d(TAG, "remove IME snapshot"); + getSyncTransaction().remove(mImeScreenshot); + mImeScreenshot = null; + } + } + /** * The IME input target is the window which receives input from IME. It is also a candidate * which controls the visibility and animation of the input method window. @@ -4037,6 +4154,16 @@ class DisplayContent extends RootDisplayArea implements WindowManagerPolicy.Disp mWmService.mWindowPlacerLocked.performSurfacePlacement(); } + /** + * Callbacks when the given type of {@link WindowContainer} animation finished running in the + * hierarchy. + */ + void onWindowAnimationFinished(int type) { + if (type == ANIMATION_TYPE_APP_TRANSITION || type == ANIMATION_TYPE_RECENTS) { + removeImeSurfaceImmediately(); + } + } + // TODO: Super unexpected long method that should be broken down... void applySurfaceChangesTransaction() { final WindowSurfacePlacer surfacePlacer = mWmService.mWindowPlacerLocked; diff --git a/services/core/java/com/android/server/wm/TaskSnapshotController.java b/services/core/java/com/android/server/wm/TaskSnapshotController.java index 513fa70b19654..b810de99ee104 100644 --- a/services/core/java/com/android/server/wm/TaskSnapshotController.java +++ b/services/core/java/com/android/server/wm/TaskSnapshotController.java @@ -35,6 +35,7 @@ import android.hardware.HardwareBuffer; import android.os.Environment; import android.os.Handler; import android.util.ArraySet; +import android.util.Pair; import android.util.Slog; import android.view.InsetsState; import android.view.SurfaceControl; @@ -275,39 +276,12 @@ class TaskSnapshotController { */ @VisibleForTesting boolean prepareTaskSnapshot(Task task, int pixelFormat, TaskSnapshot.Builder builder) { - if (!mService.mPolicy.isScreenOn()) { - if (DEBUG_SCREENSHOT) { - Slog.i(TAG_WM, "Attempted to take screenshot while display was off."); - } + final Pair result = checkIfReadyToSnapshot(task); + if (result == null) { return false; } - final ActivityRecord activity = findAppTokenForSnapshot(task); - if (activity == null) { - if (DEBUG_SCREENSHOT) { - Slog.w(TAG_WM, "Failed to take screenshot. No visible windows for " + task); - } - return false; - } - if (activity.hasCommittedReparentToAnimationLeash()) { - if (DEBUG_SCREENSHOT) { - Slog.w(TAG_WM, "Failed to take screenshot. App is animating " + activity); - } - return false; - } - - final WindowState mainWindow = activity.findMainWindow(); - if (mainWindow == null) { - Slog.w(TAG_WM, "Failed to take screenshot. No main window for " + task); - return false; - } - if (activity.hasFixedRotationTransform()) { - if (DEBUG_SCREENSHOT) { - Slog.i(TAG_WM, "Skip taking screenshot. App has fixed rotation " + activity); - } - // The activity is in a temporal state that it has different rotation than the task. - return false; - } - + final ActivityRecord activity = result.first; + final WindowState mainWindow = result.second; final Rect contentInsets = getSystemBarInsets(task.getBounds(), mainWindow.getInsetsStateWithVisibilityOverride()); InsetUtils.addInsets(contentInsets, activity.getLetterboxInsets()); @@ -339,6 +313,50 @@ class TaskSnapshotController { return true; } + /** + * Check if the state of the Task is appropriate to capture a snapshot, such like the task + * snapshot or the associated IME surface snapshot. + * + * @param task the target task to capture the snapshot + * @return Pair of (the top activity of the task, the main window of the task) if passed the + * state checking. Returns {@code null} if the task state isn't ready to snapshot. + */ + Pair checkIfReadyToSnapshot(Task task) { + if (!mService.mPolicy.isScreenOn()) { + if (DEBUG_SCREENSHOT) { + Slog.i(TAG_WM, "Attempted to take screenshot while display was off."); + } + return null; + } + final ActivityRecord activity = findAppTokenForSnapshot(task); + if (activity == null) { + if (DEBUG_SCREENSHOT) { + Slog.w(TAG_WM, "Failed to take screenshot. No visible windows for " + task); + } + return null; + } + if (activity.hasCommittedReparentToAnimationLeash()) { + if (DEBUG_SCREENSHOT) { + Slog.w(TAG_WM, "Failed to take screenshot. App is animating " + activity); + } + return null; + } + + final WindowState mainWindow = activity.findMainWindow(); + if (mainWindow == null) { + Slog.w(TAG_WM, "Failed to take screenshot. No main window for " + task); + return null; + } + if (activity.hasFixedRotationTransform()) { + if (DEBUG_SCREENSHOT) { + Slog.i(TAG_WM, "Skip taking screenshot. App has fixed rotation " + activity); + } + // The activity is in a temporal state that it has different rotation than the task. + return null; + } + return new Pair<>(activity, mainWindow); + } + @Nullable SurfaceControl.ScreenshotHardwareBuffer createTaskSnapshot(@NonNull Task task, TaskSnapshot.Builder builder) { @@ -355,6 +373,43 @@ class TaskSnapshotController { return createTaskSnapshot(task, scaleFraction, PixelFormat.RGBA_8888, null, builder); } + @Nullable + private SurfaceControl.ScreenshotHardwareBuffer createImeSnapshot(@NonNull Task task, + int pixelFormat) { + if (task.getSurfaceControl() == null) { + if (DEBUG_SCREENSHOT) { + Slog.w(TAG_WM, "Failed to take screenshot. No surface control for " + task); + } + return null; + } + final WindowState imeWindow = task.getDisplayContent().mInputMethodWindow; + SurfaceControl.ScreenshotHardwareBuffer imeBuffer = null; + if (imeWindow != null && imeWindow.isWinVisibleLw()) { + final Rect bounds = imeWindow.getContainingFrame(); + bounds.offsetTo(0, 0); + imeBuffer = SurfaceControl.captureLayersExcluding(imeWindow.getSurfaceControl(), + bounds, 1.0f, pixelFormat, null); + } + return imeBuffer; + } + + /** + * Create the snapshot of the IME surface on the task which used for placing on the closing + * task to keep IME visibility while app transitioning. + */ + @Nullable + SurfaceControl.ScreenshotHardwareBuffer snapshotImeFromAttachedTask(@NonNull Task task) { + // Check if the IME targets task ready to take the corresponding IME snapshot, if not, + // means the task is not yet visible for some reasons and no need to snapshot IME surface. + if (checkIfReadyToSnapshot(task) == null) { + return null; + } + final int pixelFormat = mPersister.use16BitFormat() + ? PixelFormat.RGB_565 + : PixelFormat.RGBA_8888; + return createImeSnapshot(task, pixelFormat); + } + @Nullable SurfaceControl.ScreenshotHardwareBuffer createTaskSnapshot(@NonNull Task task, float scaleFraction, int pixelFormat, Point outTaskSize, TaskSnapshot.Builder builder) { diff --git a/services/core/java/com/android/server/wm/WindowContainer.java b/services/core/java/com/android/server/wm/WindowContainer.java index 19a750abe1a75..2b1e7834cd0b6 100644 --- a/services/core/java/com/android/server/wm/WindowContainer.java +++ b/services/core/java/com/android/server/wm/WindowContainer.java @@ -2682,6 +2682,10 @@ class WindowContainer extends ConfigurationContainer< protected void applyAnimationUnchecked(WindowManager.LayoutParams lp, boolean enter, @TransitionOldType int transit, boolean isVoiceInteraction, @Nullable ArrayList sources) { + final Task task = asTask(); + if (task != null && !enter && !task.isHomeOrRecentsRootTask()) { + mDisplayContent.showImeScreenshot(); + } final Pair adapters = getAnimationAdapter(lp, transit, enter, isVoiceInteraction); AnimationAdapter adapter = adapters.first; @@ -2826,6 +2830,9 @@ class WindowContainer extends ConfigurationContainer< mSurfaceAnimationSources.valueAt(i).onAnimationFinished(type, anim); } mSurfaceAnimationSources.clear(); + if (mDisplayContent != null) { + mDisplayContent.onWindowAnimationFinished(type); + } } /** diff --git a/services/core/java/com/android/server/wm/WindowState.java b/services/core/java/com/android/server/wm/WindowState.java index 621b9719cd399..571946a7ace6d 100644 --- a/services/core/java/com/android/server/wm/WindowState.java +++ b/services/core/java/com/android/server/wm/WindowState.java @@ -2166,6 +2166,8 @@ class WindowState extends WindowContainer implements WindowManagerP final DisplayContent dc = getDisplayContent(); if (isImeLayeringTarget()) { + // Remove the IME screenshot surface if the layering target is not animating. + dc.removeImeScreenshotIfPossible(); // Make sure to set mImeLayeringTarget as null when the removed window is the // IME target, in case computeImeTarget may use the outdated target. dc.setImeLayeringTarget(null); diff --git a/services/tests/wmtests/src/com/android/server/wm/DisplayContentTests.java b/services/tests/wmtests/src/com/android/server/wm/DisplayContentTests.java index 30c1008010023..b27d016beb8d9 100644 --- a/services/tests/wmtests/src/com/android/server/wm/DisplayContentTests.java +++ b/services/tests/wmtests/src/com/android/server/wm/DisplayContentTests.java @@ -65,7 +65,11 @@ import static com.android.dx.mockito.inline.extended.ExtendedMockito.times; import static com.android.dx.mockito.inline.extended.ExtendedMockito.verify; import static com.android.server.wm.DisplayContent.IME_TARGET_INPUT; import static com.android.server.wm.DisplayContent.IME_TARGET_LAYERING; +import static com.android.server.wm.SurfaceAnimator.ANIMATION_TYPE_APP_TRANSITION; import static com.android.server.wm.SurfaceAnimator.ANIMATION_TYPE_FIXED_TRANSFORM; +import static com.android.server.wm.SurfaceAnimator.ANIMATION_TYPE_RECENTS; +import static com.android.server.wm.WindowContainer.AnimationFlags.PARENTS; +import static com.android.server.wm.WindowContainer.AnimationFlags.TRANSITION; import static com.android.server.wm.WindowContainer.POSITION_TOP; import static com.android.server.wm.WindowManagerService.UPDATE_FOCUS_NORMAL; @@ -81,6 +85,7 @@ import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.atLeast; import static org.mockito.Mockito.atLeastOnce; import static org.mockito.Mockito.clearInvocations; import static org.mockito.Mockito.doAnswer; @@ -1732,6 +1737,63 @@ public class DisplayContentTests extends WindowTestsBase { verify(child1, never()).needsRelativeLayeringToIme(); } + @UseTestDisplay(addWindows = {W_INPUT_METHOD}, addAllCommonWindows = true) + @Test + public void testAttachAndShowImeScreenshotOnTarget() { + // Preparation: Simulate screen state is on. + spyOn(mWm.mPolicy); + doReturn(true).when(mWm.mPolicy).isScreenOn(); + + // Preparation: Simulate snapshot IME surface. + spyOn(mWm.mTaskSnapshotController); + doReturn(mock(SurfaceControl.ScreenshotHardwareBuffer.class)).when( + mWm.mTaskSnapshotController).snapshotImeFromAttachedTask(any()); + final SurfaceControl imeSurface = mock(SurfaceControl.class); + spyOn(imeSurface); + doReturn(true).when(imeSurface).isValid(); + doReturn(imeSurface).when(mDisplayContent).createImeSurface(any(), any()); + + // Preparation: Simulate snapshot Task. + ActivityRecord act1 = createActivityRecord(mDisplayContent); + final WindowState appWin1 = createWindow(null, TYPE_BASE_APPLICATION, act1, "appWin1"); + spyOn(appWin1); + spyOn(appWin1.mWinAnimator); + appWin1.setHasSurface(true); + assertTrue(appWin1.canBeImeTarget()); + doReturn(true).when(appWin1.mWinAnimator).getShown(); + doReturn(true).when(appWin1.mActivityRecord).isSurfaceShowing(); + appWin1.mWinAnimator.mLastAlpha = 1f; + + // Test step 1: appWin1 is the current IME target and soft-keyboard is visible. + mDisplayContent.computeImeTarget(true); + assertEquals(appWin1, mDisplayContent.getImeTarget(IME_TARGET_LAYERING)); + spyOn(mDisplayContent.mInputMethodWindow); + doReturn(true).when(mDisplayContent.mInputMethodWindow).isVisible(); + mDisplayContent.getInsetsStateController().getImeSourceProvider().setImeShowing(true); + + // Test step 2: Simulate launching appWin2 and appWin1 is in app transition. + ActivityRecord act2 = createActivityRecord(mDisplayContent); + final WindowState appWin2 = createWindow(null, TYPE_BASE_APPLICATION, act2, "appWin2"); + appWin2.setHasSurface(true); + assertTrue(appWin2.canBeImeTarget()); + doReturn(true).when(appWin1).isAnimating(PARENTS | TRANSITION, + ANIMATION_TYPE_APP_TRANSITION | ANIMATION_TYPE_RECENTS); + + // Test step 3: Verify appWin2 will be the next IME target and the IME snapshot surface will + // be shown at this time. + final Transaction t = mDisplayContent.getPendingTransaction(); + spyOn(t); + mDisplayContent.setImeInputTarget(appWin2); + mDisplayContent.computeImeTarget(true); + assertEquals(appWin2, mDisplayContent.getImeTarget(IME_TARGET_LAYERING)); + assertTrue(mDisplayContent.isImeAttachedToApp()); + + verify(mDisplayContent, atLeast(1)).attachAndShowImeScreenshotOnTarget(); + verify(mWm.mTaskSnapshotController).snapshotImeFromAttachedTask(appWin1.getTask()); + assertNotNull(mDisplayContent.mImeScreenshot); + verify(t).show(mDisplayContent.mImeScreenshot); + } + private boolean isOptionsPanelAtRight(int displayId) { return (mWm.getPreferredOptionsPanelGravity(displayId) & Gravity.RIGHT) == Gravity.RIGHT; }