Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactoring IME snapshot with adopting shell-transition

Browse Source 
Extracts logic into the new ImeScreenshot class with using
TransitionController#{inTranstion, inRecentsTransition}
checks to attach/detach IME screenshot.

Bug: 201139553
Bug: 212570341
Test: atest DisplayContentTests#\
       testAttachAndShowImeScreenshotOnTarget
Test: manual as steps
   0) Settings -> Gestures -> enabling 3-buttons gesture mode
   1) Launch google search bar from the launcher to show IME
   2) Press home key and ensure IME screenshot is still visible
      when the google search activity is leaving w/w.o. enabling
      shell-transition.

Change-Id: I9228740cbf060a01fe985eb37c94c6abae6f79f8
This commit is contained in:
Ming-Shin Lu
2022-02-20 22:13:02 +08:00
parent 2773ac9c72
commit 5351be9ec5
5 changed files with 121 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files

163
services/core/java/com/android/server/wm/DisplayContent.java
View File

@@ -633,9 +633,6 @@ class DisplayContent extends RootDisplayArea implements WindowManagerPolicy.Disp
@VisibleForTesting
SurfaceControl mInputMethodSurfaceParent;
/** The screenshot IME surface to place on the task while transitioning to the next task. */
SurfaceControl mImeScreenshot;
private final PointerEventDispatcher mPointerEventDispatcher;
private final InsetsStateController mInsetsStateController;
@@ -4017,10 +4014,9 @@ class DisplayContent extends RootDisplayArea implements WindowManagerPolicy.Disp
// If the IME target is the input target, before it changes, prepare the IME screenshot
// for the last IME target when its task is applying app transition. This is for the
// better IME transition to keep IME visibility when transitioning to the next task.
if (mImeLayeringTarget != null && mImeLayeringTarget.isAnimating(PARENTS | TRANSITION,
ANIMATION_TYPE_APP_TRANSITION | ANIMATION_TYPE_RECENTS)
if (mImeLayeringTarget != null && mImeLayeringTarget.inAppOrRecentsTransition()
&& mImeLayeringTarget == mImeInputTarget) {
attachAndShowImeScreenshotOnTarget();
showImeScreenshot();
}
ProtoLog.i(WM_DEBUG_IME, "setInputMethodTarget %s", target);
@@ -4068,8 +4064,96 @@ class DisplayContent extends RootDisplayArea implements WindowManagerPolicy.Disp
mImeControlTarget = target;
}
@VisibleForTesting
void attachAndShowImeScreenshotOnTarget() {
// ========== Begin of ImeScreenshot stuff ==========
/** The screenshot IME surface to place on the task while transitioning to the next task. */
ImeScreenshot mImeScreenshot;
static final class ImeScreenshot {
private WindowState mImeTarget;
private SurfaceControl.Builder mSurfaceBuilder;
private SurfaceControl mImeSurface;
ImeScreenshot(SurfaceControl.Builder surfaceBuilder, @NonNull WindowState imeTarget) {
mSurfaceBuilder = surfaceBuilder;
mImeTarget = imeTarget;
}
private SurfaceControl createImeSurface(SurfaceControl.ScreenshotHardwareBuffer b,
Transaction t) {
final HardwareBuffer buffer = b.getHardwareBuffer();
if (DEBUG_INPUT_METHOD) {
Slog.d(TAG, "create IME snapshot for "
+ mImeTarget + ", buff width=" + buffer.getWidth()
+ ", height=" + buffer.getHeight());
}
final WindowState imeWindow = mImeTarget.getDisplayContent().mInputMethodWindow;
final ActivityRecord activity = mImeTarget.mActivityRecord;
final SurfaceControl imeSurface = mSurfaceBuilder
.setName("IME-snapshot-surface")
.setBLASTLayer()
.setFormat(buffer.getFormat())
.setParent(activity.getSurfaceControl())
.setCallsite("DisplayContent.attachAndShowImeScreenshotOnTarget")
.build();
// Make IME snapshot as trusted overlay
InputMonitor.setTrustedOverlayInputInfo(imeSurface, t, imeWindow.getDisplayId(),
"IME-snapshot-surface");
t.setBuffer(imeSurface, buffer);
t.setColorSpace(activity.mSurfaceControl, ColorSpace.get(ColorSpace.Named.SRGB));
t.setLayer(imeSurface, 1);
t.setPosition(imeSurface, imeWindow.getFrame().left - mImeTarget.getFrame().left,
imeWindow.getFrame().top - mImeTarget.getFrame().top);
return imeSurface;
}
private void removeImeSurface(Transaction t) {
if (mImeSurface != null) {
if (DEBUG_INPUT_METHOD) Slog.d(TAG, "remove IME snapshot");
t.remove(mImeSurface);
mImeSurface = null;
}
}
void attachAndShow(Transaction t) {
final DisplayContent dc = mImeTarget.getDisplayContent();
// Prepare IME screenshot for the target if it allows to attach into.
final Task task = mImeTarget.getTask();
// Re-new the IME screenshot when it does not exist or the size changed.
final boolean renewImeSurface = mImeSurface == null
|| mImeSurface.getWidth() != dc.mInputMethodWindow.getFrame().width()
|| mImeSurface.getHeight() != dc.mInputMethodWindow.getFrame().height();
if (task != null && !task.isActivityTypeHomeOrRecents()) {
SurfaceControl.ScreenshotHardwareBuffer imeBuffer = renewImeSurface
? dc.mWmService.mTaskSnapshotController.snapshotImeFromAttachedTask(task)
: null;
if (imeBuffer != null) {
// Remove the last IME surface when the surface needs to renew.
removeImeSurface(t);
mImeSurface = createImeSurface(imeBuffer, t);
}
}
final boolean isValidSnapshot = mImeSurface != null && mImeSurface.isValid();
// Showing the IME screenshot if the target has already in app transition stage.
// Note that if the current IME insets is not showing, no need to show IME screenshot
// to reflect the true IME insets visibility and the app task layout as possible.
if (isValidSnapshot
&& dc.getInsetsStateController().getImeSourceProvider().isImeShowing()) {
if (DEBUG_INPUT_METHOD) {
Slog.d(TAG, "show IME snapshot, ime target=" + mImeTarget);
}
t.show(mImeSurface);
} else if (!isValidSnapshot) {
removeImeSurface(t);
}
}
void detach(Transaction t) {
removeImeSurface(t);
}
}
private void attachAndShowImeScreenshotOnTarget() {
// No need to attach screenshot if the IME target not exists or screen is off.
if (!shouldImeAttachedToApp() || !mWmService.mPolicy.isScreenOn()) {
return;
@@ -4078,61 +4162,10 @@ class DisplayContent extends RootDisplayArea implements WindowManagerPolicy.Disp
final SurfaceControl.Transaction t = getPendingTransaction();
// Prepare IME screenshot for the target if it allows to attach into.
if (mInputMethodWindow != null && mInputMethodWindow.isVisible()) {
final Task task = mImeLayeringTarget.getTask();
// Re-new the IME screenshot when it does not exist or the size changed.
final boolean renewImeSurface = mImeScreenshot == null
|| mImeScreenshot.getWidth() != mInputMethodWindow.getFrame().width()
|| mImeScreenshot.getHeight() != mInputMethodWindow.getFrame().height();
if (task != null && !task.isActivityTypeHomeOrRecents()) {
SurfaceControl.ScreenshotHardwareBuffer imeBuffer = renewImeSurface
? mWmService.mTaskSnapshotController.snapshotImeFromAttachedTask(task)
: null;
if (imeBuffer != null) {
// Remove the last IME surface when the surface needs to renew.
removeImeSurfaceImmediately();
mImeScreenshot = createImeSurface(imeBuffer, t);
}
}
mImeScreenshot = new ImeScreenshot(
mWmService.mSurfaceControlFactory.apply(null), mImeLayeringTarget);
mImeScreenshot.attachAndShow(t);
}
final boolean isValidSnapshot = mImeScreenshot != null && mImeScreenshot.isValid();
// Showing the IME screenshot if the target has already in app transition stage.
// Note that if the current IME insets is not showing, no need to show IME screenshot
// to reflect the true IME insets visibility and the app task layout as possible.
if (isValidSnapshot && getInsetsStateController().getImeSourceProvider().isImeShowing()) {
if (DEBUG_INPUT_METHOD) {
Slog.d(TAG, "show IME snapshot, ime target=" + mImeLayeringTarget);
}
t.show(mImeScreenshot);
} else if (!isValidSnapshot) {
removeImeSurfaceImmediately();
}
}
@VisibleForTesting
SurfaceControl createImeSurface(SurfaceControl.ScreenshotHardwareBuffer imeBuffer,
Transaction t) {
final HardwareBuffer buffer = imeBuffer.getHardwareBuffer();
if (DEBUG_INPUT_METHOD) Slog.d(TAG, "create IME snapshot for "
+ mImeLayeringTarget + ", buff width=" + buffer.getWidth()
+ ", height=" + buffer.getHeight());
final ActivityRecord activity = mImeLayeringTarget.mActivityRecord;
final SurfaceControl imeSurface = mWmService.mSurfaceControlFactory.apply(null)
.setName("IME-snapshot-surface")
.setBLASTLayer()
.setFormat(buffer.getFormat())
.setParent(activity.getSurfaceControl())
.setCallsite("DisplayContent.attachAndShowImeScreenshotOnTarget")
.build();
// Make IME snapshot as trusted overlay
InputMonitor.setTrustedOverlayInputInfo(imeSurface, t, getDisplayId(),
"IME-snapshot-surface");
t.setBuffer(imeSurface, buffer);
t.setColorSpace(mSurfaceControl, ColorSpace.get(ColorSpace.Named.SRGB));
t.setLayer(imeSurface, 1);
t.setPosition(imeSurface, mInputMethodWindow.getDisplayFrame().left,
mInputMethodWindow.getDisplayFrame().top);
return imeSurface;
}
/**
@@ -4154,8 +4187,7 @@ class DisplayContent extends RootDisplayArea implements WindowManagerPolicy.Disp
void removeImeScreenshotIfPossible() {
if (mImeLayeringTarget == null
|| mImeLayeringTarget.mAttrs.type != TYPE_APPLICATION_STARTING
&& !mImeLayeringTarget.isAnimating(PARENTS | TRANSITION,
ANIMATION_TYPE_APP_TRANSITION | ANIMATION_TYPE_RECENTS)) {
&& !mImeLayeringTarget.inAppOrRecentsTransition()) {
removeImeSurfaceImmediately();
}
}
@@ -4164,10 +4196,11 @@ class DisplayContent extends RootDisplayArea implements WindowManagerPolicy.Disp
void removeImeSurfaceImmediately() {
if (mImeScreenshot != null) {
if (DEBUG_INPUT_METHOD) Slog.d(TAG, "remove IME snapshot");
getSyncTransaction().remove(mImeScreenshot);
mImeScreenshot.detach(getSyncTransaction());
mImeScreenshot = null;
}
}
// ========== End of ImeScreenshot stuff ==========
/**
* The IME input target is the window which receives input from IME. It is also a candidate

1
services/core/java/com/android/server/wm/Transition.java
View File

@@ -574,6 +574,7 @@ class Transition extends Binder implements BLASTSyncEngine.TransactionReadyListe
false /* disableImeIcon */);
}
}
dc.removeImeSurfaceImmediately();
dc.handleCompleteDeferredRemoval();
}
}

22
services/tests/wmtests/src/com/android/server/wm/DisplayContentTests.java
View File

@@ -78,7 +78,6 @@ import static com.android.server.wm.ActivityTaskSupervisor.ON_TOP;
import static com.android.server.wm.DisplayContent.IME_TARGET_LAYERING;
import static com.android.server.wm.SurfaceAnimator.ANIMATION_TYPE_APP_TRANSITION;
import static com.android.server.wm.SurfaceAnimator.ANIMATION_TYPE_FIXED_TRANSFORM;
import static com.android.server.wm.SurfaceAnimator.ANIMATION_TYPE_RECENTS;
import static com.android.server.wm.WindowContainer.AnimationFlags.PARENTS;
import static com.android.server.wm.WindowContainer.AnimationFlags.TRANSITION;
import static com.android.server.wm.WindowContainer.POSITION_TOP;
@@ -110,6 +109,7 @@ import android.graphics.Insets;
import android.graphics.Point;
import android.graphics.Rect;
import android.graphics.Region;
import android.hardware.HardwareBuffer;
import android.hardware.display.VirtualDisplay;
import android.metrics.LogMaker;
import android.os.Binder;
@@ -1939,12 +1939,10 @@ public class DisplayContentTests extends WindowTestsBase {
// Preparation: Simulate snapshot IME surface.
spyOn(mWm.mTaskSnapshotController);
doReturn(mock(SurfaceControl.ScreenshotHardwareBuffer.class)).when(
mWm.mTaskSnapshotController).snapshotImeFromAttachedTask(any());
final SurfaceControl imeSurface = mock(SurfaceControl.class);
spyOn(imeSurface);
doReturn(true).when(imeSurface).isValid();
doReturn(imeSurface).when(mDisplayContent).createImeSurface(any(), any());
SurfaceControl.ScreenshotHardwareBuffer mockHwBuffer = mock(
SurfaceControl.ScreenshotHardwareBuffer.class);
doReturn(mock(HardwareBuffer.class)).when(mockHwBuffer).getHardwareBuffer();
doReturn(mockHwBuffer).when(mWm.mTaskSnapshotController).snapshotImeFromAttachedTask(any());
// Preparation: Simulate snapshot Task.
ActivityRecord act1 = createActivityRecord(mDisplayContent);
@@ -1970,21 +1968,17 @@ public class DisplayContentTests extends WindowTestsBase {
final WindowState appWin2 = createWindow(null, TYPE_BASE_APPLICATION, act2, "appWin2");
appWin2.setHasSurface(true);
assertTrue(appWin2.canBeImeTarget());
doReturn(true).when(appWin1).isAnimating(PARENTS | TRANSITION,
ANIMATION_TYPE_APP_TRANSITION | ANIMATION_TYPE_RECENTS);
doReturn(true).when(appWin1).inAppOrRecentsTransition();
// Test step 3: Verify appWin2 will be the next IME target and the IME snapshot surface will
// be shown at this time.
final Transaction t = mDisplayContent.getPendingTransaction();
spyOn(t);
// be attached and shown on the display at this time.
mDisplayContent.computeImeTarget(true);
assertEquals(appWin2, mDisplayContent.getImeTarget(IME_TARGET_LAYERING));
assertTrue(mDisplayContent.shouldImeAttachedToApp());
verify(mDisplayContent, atLeast(1)).attachAndShowImeScreenshotOnTarget();
verify(mDisplayContent, atLeast(1)).showImeScreenshot();
verify(mWm.mTaskSnapshotController).snapshotImeFromAttachedTask(appWin1.getTask());
assertNotNull(mDisplayContent.mImeScreenshot);
verify(t).show(mDisplayContent.mImeScreenshot);
}
@UseTestDisplay(addWindows = {W_INPUT_METHOD}, addAllCommonWindows = true)

6
services/tests/wmtests/src/com/android/server/wm/MockSurfaceControlBuilder.java
View File

@@ -20,6 +20,8 @@ import static org.mockito.Mockito.mock;
import android.view.SurfaceControl;
import org.mockito.Mockito;
/**
* Stubbed {@link SurfaceControl.Builder} class that returns a mocked SurfaceControl instance
* that can be used for unit testing.
@@ -32,6 +34,8 @@ class MockSurfaceControlBuilder extends SurfaceControl.Builder {
@Override
public SurfaceControl build() {
return mock(SurfaceControl.class);
SurfaceControl mockSurfaceControl = mock(SurfaceControl.class);
Mockito.doReturn(true).when(mockSurfaceControl).isValid();
return mockSurfaceControl;
}
}

9
services/tests/wmtests/src/com/android/server/wm/StubTransaction.java
View File

@@ -17,11 +17,13 @@
package com.android.server.wm;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.graphics.ColorSpace;
import android.graphics.GraphicBuffer;
import android.graphics.Matrix;
import android.graphics.Rect;
import android.graphics.Region;
import android.hardware.HardwareBuffer;
import android.os.IBinder;
import android.os.Parcel;
import android.view.InputWindowHandle;
@@ -266,6 +268,13 @@ public class StubTransaction extends SurfaceControl.Transaction {
return this;
}
@Override
@NonNull
public SurfaceControl.Transaction setBuffer(@NonNull SurfaceControl sc,
@Nullable HardwareBuffer buffer) {
return this;
}
@Override
public SurfaceControl.Transaction setColorSpace(SurfaceControl sc, ColorSpace colorSpace) {
return this;