diff --git a/core/java/android/hardware/display/DisplayManagerInternal.java b/core/java/android/hardware/display/DisplayManagerInternal.java index abcc33c43c743..4f205530ef0d7 100644 --- a/core/java/android/hardware/display/DisplayManagerInternal.java +++ b/core/java/android/hardware/display/DisplayManagerInternal.java @@ -21,6 +21,7 @@ import android.annotation.Nullable; import android.graphics.Point; import android.hardware.SensorManager; import android.os.Handler; +import android.os.IBinder; import android.os.PowerManager; import android.util.IntArray; import android.util.Slog; @@ -339,6 +340,28 @@ public abstract class DisplayManagerInternal { */ public abstract List getRefreshRateLimitations(int displayId); + /** + * Returns the window token of the level of the WindowManager hierarchy to mirror. Returns null + * if layer mirroring by SurfaceFlinger should not be performed for the given displayId. + * For now, only used for mirroring started from MediaProjection. + */ + public abstract IBinder getWindowTokenClientToMirror(int displayId); + + /** + * For the given displayId, updates the window token of the level of the WindowManager hierarchy + * to mirror. If windowToken is null, then SurfaceFlinger performs no layer mirroring to the + * given display. + * For now, only used for mirroring started from MediaProjection. + */ + public abstract void setWindowTokenClientToMirror(int displayId, IBinder windowToken); + + /** + * Returns the default size of the surface associated with the display, or null if the surface + * is not provided for layer mirroring by SurfaceFlinger. + * For now, only used for mirroring started from MediaProjection. + */ + public abstract Point getDisplaySurfaceDefaultSize(int displayId); + /** * Describes the requested power state of the display. * diff --git a/core/java/android/hardware/display/VirtualDisplayConfig.java b/core/java/android/hardware/display/VirtualDisplayConfig.java index 71688c7cc7e85..0e86f43207aae 100644 --- a/core/java/android/hardware/display/VirtualDisplayConfig.java +++ b/core/java/android/hardware/display/VirtualDisplayConfig.java @@ -23,6 +23,7 @@ import android.annotation.NonNull; import android.annotation.Nullable; import android.media.projection.MediaProjection; import android.os.Handler; +import android.os.IBinder; import android.os.Parcel; import android.os.Parcelable; import android.view.Surface; @@ -91,9 +92,16 @@ public final class VirtualDisplayConfig implements Parcelable { */ private int mDisplayIdToMirror = DEFAULT_DISPLAY; + /** + * The window token of the level of the WindowManager hierarchy to mirror, or null if mirroring + * should not be performed. + */ + @Nullable + private IBinder mWindowTokenClientToMirror = null; - // Code below generated by codegen v1.0.20. + + // Code below generated by codegen v1.0.23. // // DO NOT MODIFY! // CHECKSTYLE:OFF Generated code @@ -115,7 +123,8 @@ public final class VirtualDisplayConfig implements Parcelable { int flags, @Nullable Surface surface, @Nullable String uniqueId, - int displayIdToMirror) { + int displayIdToMirror, + @Nullable IBinder windowTokenClientToMirror) { this.mName = name; com.android.internal.util.AnnotationValidations.validate( NonNull.class, null, mName); @@ -135,6 +144,7 @@ public final class VirtualDisplayConfig implements Parcelable { this.mSurface = surface; this.mUniqueId = uniqueId; this.mDisplayIdToMirror = displayIdToMirror; + this.mWindowTokenClientToMirror = windowTokenClientToMirror; // onConstructed(); // You can define this method to get a callback } @@ -212,6 +222,15 @@ public final class VirtualDisplayConfig implements Parcelable { return mDisplayIdToMirror; } + /** + * The window token of the level of the WindowManager hierarchy to mirror, or null if mirroring + * should not be performed. + */ + @DataClass.Generated.Member + public @Nullable IBinder getWindowTokenClientToMirror() { + return mWindowTokenClientToMirror; + } + @Override @DataClass.Generated.Member public void writeToParcel(@NonNull Parcel dest, int flags) { @@ -221,6 +240,7 @@ public final class VirtualDisplayConfig implements Parcelable { int flg = 0; if (mSurface != null) flg |= 0x20; if (mUniqueId != null) flg |= 0x40; + if (mWindowTokenClientToMirror != null) flg |= 0x100; dest.writeInt(flg); dest.writeString(mName); dest.writeInt(mWidth); @@ -230,6 +250,7 @@ public final class VirtualDisplayConfig implements Parcelable { if (mSurface != null) dest.writeTypedObject(mSurface, flags); if (mUniqueId != null) dest.writeString(mUniqueId); dest.writeInt(mDisplayIdToMirror); + if (mWindowTokenClientToMirror != null) dest.writeStrongBinder(mWindowTokenClientToMirror); } @Override @@ -252,6 +273,7 @@ public final class VirtualDisplayConfig implements Parcelable { Surface surface = (flg & 0x20) == 0 ? null : (Surface) in.readTypedObject(Surface.CREATOR); String uniqueId = (flg & 0x40) == 0 ? null : in.readString(); int displayIdToMirror = in.readInt(); + IBinder windowTokenClientToMirror = (flg & 0x100) == 0 ? null : (IBinder) in.readStrongBinder(); this.mName = name; com.android.internal.util.AnnotationValidations.validate( @@ -272,6 +294,7 @@ public final class VirtualDisplayConfig implements Parcelable { this.mSurface = surface; this.mUniqueId = uniqueId; this.mDisplayIdToMirror = displayIdToMirror; + this.mWindowTokenClientToMirror = windowTokenClientToMirror; // onConstructed(); // You can define this method to get a callback } @@ -305,6 +328,7 @@ public final class VirtualDisplayConfig implements Parcelable { private @Nullable Surface mSurface; private @Nullable String mUniqueId; private int mDisplayIdToMirror; + private @Nullable IBinder mWindowTokenClientToMirror; private long mBuilderFieldsSet = 0L; @@ -439,10 +463,22 @@ public final class VirtualDisplayConfig implements Parcelable { return this; } + /** + * The window token of the level of the WindowManager hierarchy to mirror, or null if mirroring + * should not be performed. + */ + @DataClass.Generated.Member + public @NonNull Builder setWindowTokenClientToMirror(@NonNull IBinder value) { + checkNotUsed(); + mBuilderFieldsSet |= 0x100; + mWindowTokenClientToMirror = value; + return this; + } + /** Builds the instance. This builder should not be touched after calling this! */ public @NonNull VirtualDisplayConfig build() { checkNotUsed(); - mBuilderFieldsSet |= 0x100; // Mark builder used + mBuilderFieldsSet |= 0x200; // Mark builder used if ((mBuilderFieldsSet & 0x10) == 0) { mFlags = 0; @@ -456,6 +492,9 @@ public final class VirtualDisplayConfig implements Parcelable { if ((mBuilderFieldsSet & 0x80) == 0) { mDisplayIdToMirror = DEFAULT_DISPLAY; } + if ((mBuilderFieldsSet & 0x100) == 0) { + mWindowTokenClientToMirror = null; + } VirtualDisplayConfig o = new VirtualDisplayConfig( mName, mWidth, @@ -464,12 +503,13 @@ public final class VirtualDisplayConfig implements Parcelable { mFlags, mSurface, mUniqueId, - mDisplayIdToMirror); + mDisplayIdToMirror, + mWindowTokenClientToMirror); return o; } private void checkNotUsed() { - if ((mBuilderFieldsSet & 0x100) != 0) { + if ((mBuilderFieldsSet & 0x200) != 0) { throw new IllegalStateException( "This Builder should not be reused. Use a new Builder instance instead"); } @@ -477,10 +517,10 @@ public final class VirtualDisplayConfig implements Parcelable { } @DataClass.Generated( - time = 1604456298440L, - codegenVersion = "1.0.20", + time = 1620657851981L, + codegenVersion = "1.0.23", sourceFile = "frameworks/base/core/java/android/hardware/display/VirtualDisplayConfig.java", - inputSignatures = "private @android.annotation.NonNull java.lang.String mName\nprivate @android.annotation.IntRange int mWidth\nprivate @android.annotation.IntRange int mHeight\nprivate @android.annotation.IntRange int mDensityDpi\nprivate int mFlags\nprivate @android.annotation.Nullable android.view.Surface mSurface\nprivate @android.annotation.Nullable java.lang.String mUniqueId\nprivate int mDisplayIdToMirror\nclass VirtualDisplayConfig extends java.lang.Object implements [android.os.Parcelable]\n@com.android.internal.util.DataClass(genParcelable=true, genAidl=true, genBuilder=true)") + inputSignatures = "private @android.annotation.NonNull java.lang.String mName\nprivate @android.annotation.IntRange int mWidth\nprivate @android.annotation.IntRange int mHeight\nprivate @android.annotation.IntRange int mDensityDpi\nprivate int mFlags\nprivate @android.annotation.Nullable android.view.Surface mSurface\nprivate @android.annotation.Nullable java.lang.String mUniqueId\nprivate int mDisplayIdToMirror\nprivate @android.annotation.Nullable android.os.IBinder mWindowTokenClientToMirror\nclass VirtualDisplayConfig extends java.lang.Object implements [android.os.Parcelable]\n@com.android.internal.util.DataClass(genParcelable=true, genAidl=true, genBuilder=true)") @Deprecated private void __metadata() {} diff --git a/core/java/android/view/Surface.java b/core/java/android/view/Surface.java index ff2d2eb3d3345..aaf53ee1cfc10 100644 --- a/core/java/android/view/Surface.java +++ b/core/java/android/view/Surface.java @@ -29,6 +29,7 @@ import android.graphics.Canvas; import android.graphics.ColorSpace; import android.graphics.HardwareRenderer; import android.graphics.Matrix; +import android.graphics.Point; import android.graphics.RecordingCanvas; import android.graphics.Rect; import android.graphics.RenderNode; @@ -407,6 +408,20 @@ public class Surface implements Parcelable { } } + /** + * Returns the default size of this Surface provided by the consumer of the surface. + * Should only be used by the producer of the surface. + * + * @hide + */ + @NonNull + public Point getDefaultSize() { + synchronized (mLock) { + checkNotReleasedLocked(); + return new Point(nativeGetWidth(mNativeObject), nativeGetHeight(mNativeObject)); + } + } + /** * Gets a {@link Canvas} for drawing into this surface. * diff --git a/media/java/android/media/projection/MediaProjection.java b/media/java/android/media/projection/MediaProjection.java index 37e141537c794..72cddc91f436a 100644 --- a/media/java/android/media/projection/MediaProjection.java +++ b/media/java/android/media/projection/MediaProjection.java @@ -16,14 +16,14 @@ package android.media.projection; +import static android.view.WindowManager.LayoutParams.TYPE_APPLICATION; + import android.annotation.NonNull; import android.annotation.Nullable; import android.content.Context; import android.hardware.display.DisplayManager; import android.hardware.display.VirtualDisplay; import android.hardware.display.VirtualDisplayConfig; -import android.media.projection.IMediaProjection; -import android.media.projection.IMediaProjectionCallback; import android.os.Handler; import android.os.RemoteException; import android.util.ArrayMap; @@ -106,7 +106,7 @@ public final class MediaProjection { if (isSecure) { flags |= DisplayManager.VIRTUAL_DISPLAY_FLAG_SECURE; } - final VirtualDisplayConfig.Builder builder = new VirtualDisplayConfig.Builder(name, width, + final VirtualDisplayConfig.Builder builder = buildMirroredVirtualDisplay(name, width, height, dpi); builder.setFlags(flags); if (surface != null) { @@ -141,7 +141,7 @@ public final class MediaProjection { public VirtualDisplay createVirtualDisplay(@NonNull String name, int width, int height, int dpi, int flags, @Nullable Surface surface, @Nullable VirtualDisplay.Callback callback, @Nullable Handler handler) { - final VirtualDisplayConfig.Builder builder = new VirtualDisplayConfig.Builder(name, width, + final VirtualDisplayConfig.Builder builder = buildMirroredVirtualDisplay(name, width, height, dpi); builder.setFlags(flags); if (surface != null) { @@ -150,6 +150,26 @@ public final class MediaProjection { return createVirtualDisplay(builder.build(), callback, handler); } + /** + * Constructs a {@link VirtualDisplayConfig.Builder}, which will mirror the contents of a + * DisplayArea. The DisplayArea to mirror is from the DisplayArea the caller is launched on. + * + * @param name The name of the virtual display, must be non-empty. + * @param width The width of the virtual display in pixels. Must be greater than 0. + * @param height The height of the virtual display in pixels. Must be greater than 0. + * @param dpi The density of the virtual display in dpi. Must be greater than 0. + * @return a config representing a VirtualDisplay + */ + private VirtualDisplayConfig.Builder buildMirroredVirtualDisplay(@NonNull String name, + int width, int height, int dpi) { + Context windowContext = mContext.createWindowContext(mContext.getDisplayNoVerify(), + TYPE_APPLICATION, null /* options */); + final VirtualDisplayConfig.Builder builder = new VirtualDisplayConfig.Builder(name, width, + height, dpi); + builder.setWindowTokenClientToMirror(windowContext.getWindowContextToken()); + return builder; + } + /** * Creates a {@link android.hardware.display.VirtualDisplay} to capture the * contents of the screen. diff --git a/services/core/java/com/android/server/display/DisplayDevice.java b/services/core/java/com/android/server/display/DisplayDevice.java index 9f806af5b444f..806bcc29f3057 100644 --- a/services/core/java/com/android/server/display/DisplayDevice.java +++ b/services/core/java/com/android/server/display/DisplayDevice.java @@ -16,7 +16,9 @@ package com.android.server.display; +import android.annotation.Nullable; import android.content.Context; +import android.graphics.Point; import android.graphics.Rect; import android.hardware.display.DisplayViewport; import android.os.IBinder; @@ -104,6 +106,34 @@ abstract class DisplayDevice { return Display.DEFAULT_DISPLAY; } + /** + * Returns the window token of the level of the WindowManager hierarchy to mirror, or null + * if layer mirroring by SurfaceFlinger should not be performed. + * For now, only used for mirroring started from MediaProjection. + */ + @Nullable + public IBinder getWindowTokenClientToMirrorLocked() { + return null; + } + + /** + * Updates the window token of the level of the level of the WindowManager hierarchy to mirror. + * If windowToken is null, then no layer mirroring by SurfaceFlinger to should be performed. + * For now, only used for mirroring started from MediaProjection. + */ + public void setWindowTokenClientToMirrorLocked(IBinder windowToken) { + } + + /** + * Returns the default size of the surface associated with the display, or null if the surface + * is not provided for layer mirroring by SurfaceFlinger. + * For now, only used for mirroring started from MediaProjection. + */ + @Nullable + public Point getDisplaySurfaceDefaultSize() { + return null; + } + /** * Gets the name of the display device. * diff --git a/services/core/java/com/android/server/display/DisplayManagerService.java b/services/core/java/com/android/server/display/DisplayManagerService.java index afd18894f5312..39fd962369bfd 100644 --- a/services/core/java/com/android/server/display/DisplayManagerService.java +++ b/services/core/java/com/android/server/display/DisplayManagerService.java @@ -63,8 +63,6 @@ import android.hardware.display.DisplayManagerGlobal; import android.hardware.display.DisplayManagerInternal; import android.hardware.display.DisplayManagerInternal.DisplayGroupListener; import android.hardware.display.DisplayManagerInternal.DisplayTransactionListener; -import android.hardware.display.DisplayManagerInternal.RefreshRateLimitation; -import android.hardware.display.DisplayManagerInternal.RefreshRateRange; import android.hardware.display.DisplayViewport; import android.hardware.display.DisplayedContentSample; import android.hardware.display.DisplayedContentSamplingAttributes; @@ -1749,10 +1747,13 @@ public final class DisplayManagerService extends SystemService { final DisplayDeviceInfo info = device.getDisplayDeviceInfoLocked(); final boolean ownContent = (info.flags & DisplayDeviceInfo.FLAG_OWN_CONTENT_ONLY) != 0; + // Mirror the part of WM hierarchy that corresponds to the provided window token. + IBinder windowTokenClientToMirror = device.getWindowTokenClientToMirrorLocked(); + // Find the logical display that the display device is showing. // Certain displays only ever show their own content. LogicalDisplay display = mLogicalDisplayMapper.getDisplayLocked(device); - if (!ownContent) { + if (!ownContent && windowTokenClientToMirror == null) { if (display != null && !display.hasContentLocked()) { // If the display does not have any content of its own, then // automatically mirror the requested logical display contents if possible. @@ -3314,6 +3315,40 @@ public final class DisplayManagerService extends SystemService { } return config.getRefreshRateLimitations(); } + + @Override + public IBinder getWindowTokenClientToMirror(int displayId) { + final DisplayDevice device; + synchronized (mSyncRoot) { + device = getDeviceForDisplayLocked(displayId); + if (device == null) { + return null; + } + } + return device.getWindowTokenClientToMirrorLocked(); + } + + @Override + public void setWindowTokenClientToMirror(int displayId, IBinder windowToken) { + synchronized (mSyncRoot) { + final DisplayDevice device = getDeviceForDisplayLocked(displayId); + if (device != null) { + device.setWindowTokenClientToMirrorLocked(windowToken); + } + } + } + + @Override + public Point getDisplaySurfaceDefaultSize(int displayId) { + final DisplayDevice device; + synchronized (mSyncRoot) { + device = getDeviceForDisplayLocked(displayId); + if (device == null) { + return null; + } + } + return device.getDisplaySurfaceDefaultSize(); + } } class DesiredDisplayModeSpecsObserver diff --git a/services/core/java/com/android/server/display/VirtualDisplayAdapter.java b/services/core/java/com/android/server/display/VirtualDisplayAdapter.java index b7931c8a8424f..34d2b0160c3c7 100644 --- a/services/core/java/com/android/server/display/VirtualDisplayAdapter.java +++ b/services/core/java/com/android/server/display/VirtualDisplayAdapter.java @@ -31,7 +31,9 @@ import static android.hardware.display.DisplayManager.VIRTUAL_DISPLAY_FLAG_TRUST import static com.android.server.display.DisplayDeviceInfo.FLAG_OWN_DISPLAY_GROUP; import static com.android.server.display.DisplayDeviceInfo.FLAG_TRUSTED; +import android.annotation.Nullable; import android.content.Context; +import android.graphics.Point; import android.hardware.display.IVirtualDisplayCallback; import android.hardware.display.VirtualDisplayConfig; import android.media.projection.IMediaProjection; @@ -231,6 +233,7 @@ public class VirtualDisplayAdapter extends DisplayAdapter { private Display.Mode mMode; private boolean mIsDisplayOn; private int mDisplayIdToMirror; + private IBinder mWindowTokenClientToMirror; public VirtualDisplayDevice(IBinder displayToken, IBinder appToken, int ownerUid, String ownerPackageName, Surface surface, int flags, @@ -253,6 +256,7 @@ public class VirtualDisplayAdapter extends DisplayAdapter { mUniqueIndex = uniqueIndex; mIsDisplayOn = surface != null; mDisplayIdToMirror = virtualDisplayConfig.getDisplayIdToMirror(); + mWindowTokenClientToMirror = virtualDisplayConfig.getWindowTokenClientToMirror(); } @Override @@ -282,6 +286,26 @@ public class VirtualDisplayAdapter extends DisplayAdapter { return mDisplayIdToMirror; } + @Override + @Nullable + public IBinder getWindowTokenClientToMirrorLocked() { + return mWindowTokenClientToMirror; + } + + @Override + public void setWindowTokenClientToMirrorLocked(IBinder windowToken) { + if (mWindowTokenClientToMirror != windowToken) { + mWindowTokenClientToMirror = windowToken; + sendDisplayDeviceEventLocked(this, DISPLAY_DEVICE_EVENT_CHANGED); + sendTraversalRequestLocked(); + } + } + + @Override + public Point getDisplaySurfaceDefaultSize() { + return mSurface.getDefaultSize(); + } + @VisibleForTesting Surface getSurfaceLocked() { return mSurface; @@ -362,6 +386,7 @@ public class VirtualDisplayAdapter extends DisplayAdapter { pw.println("mDisplayState=" + Display.stateToString(mDisplayState)); pw.println("mStopped=" + mStopped); pw.println("mDisplayIdToMirror=" + mDisplayIdToMirror); + pw.println("mWindowTokenClientToMirror=" + mWindowTokenClientToMirror); } diff --git a/services/core/java/com/android/server/wm/DisplayContent.java b/services/core/java/com/android/server/wm/DisplayContent.java index 3f25350ced482..0f7ee0c4d25db 100644 --- a/services/core/java/com/android/server/wm/DisplayContent.java +++ b/services/core/java/com/android/server/wm/DisplayContent.java @@ -299,6 +299,22 @@ class DisplayContent extends RootDisplayArea implements WindowManagerPolicy.Disp */ private final SurfaceControl mWindowingLayer; + /** + * The window token of the layer of the hierarchy to mirror, or null if this DisplayContent + * is not being used for layer mirroring. + */ + @VisibleForTesting IBinder mTokenToMirror = null; + + /** + * The surface for mirroring the contents of this hierarchy. + */ + private SurfaceControl mMirroredSurface = null; + + /** + * The last bounds of the DisplayArea to mirror. + */ + private Rect mLastMirroredDisplayAreaBounds = null; + // Contains all IME window containers. Note that the z-ordering of the IME windows will depend // on the IME target. We mainly have this container grouping so we can keep track of all the IME // window containers together and move them in-sync if/when needed. We use a subclass of @@ -1117,6 +1133,10 @@ class DisplayContent extends RootDisplayArea implements WindowManagerPolicy.Disp if (DEBUG_DISPLAY) Slog.v(TAG_WM, "Creating display=" + display); mWmService.mDisplayWindowSettings.applySettingsToDisplayLocked(this); + + // Check if this DisplayContent is for a new VirtualDisplay, that should use layer mirroring + // to capture the contents of a DisplayArea. + startMirrorIfNeeded(); } boolean isReady() { @@ -2471,6 +2491,25 @@ class DisplayContent extends RootDisplayArea implements WindowManagerPolicy.Disp // Update IME parent if needed. updateImeParent(); + // Update mirroring surface for MediaProjection, if this DisplayContent is being used + // for layer mirroring. + if (mMirroredSurface != null) { + // Retrieve the size of the DisplayArea to mirror, and continue with the update if the + // bounds have changed. + final WindowContainer wc = mWmService.mWindowContextListenerController.getContainer( + mTokenToMirror); + if (wc != null && mLastMirroredDisplayAreaBounds != null) { + // Retrieve the size of the DisplayArea to mirror, and continue with the update + // if the bounds or orientation has changed. + final Rect displayAreaBounds = wc.getDisplayContent().getBounds(); + int displayAreaOrientation = wc.getDisplayContent().getOrientation(); + if (!mLastMirroredDisplayAreaBounds.equals(displayAreaBounds) + || lastOrientation != displayAreaOrientation) { + updateMirroredSurface(mWmService.mTransactionFactory.get(), displayAreaBounds); + } + } + } + if (lastOrientation != getConfiguration().orientation) { getMetricsLogger().write( new LogMaker(MetricsEvent.ACTION_PHONE_ORIENTATION_CHANGED) @@ -4340,6 +4379,7 @@ class DisplayContent extends RootDisplayArea implements WindowManagerPolicy.Disp mTmpApplySurfaceChangesTransactionState.preferMinimalPostProcessing, true /* inTraversal, must call performTraversalInTrans... below */); } + updateMirroring(); final boolean wallpaperVisible = mWallpaperController.isWallpaperVisible(); if (wallpaperVisible != mLastWallpaperVisible) { @@ -5892,6 +5932,133 @@ class DisplayContent extends RootDisplayArea implements WindowManagerPolicy.Disp return mSandboxDisplayApis; } + /** + * Start mirroring to this DisplayContent if it does not have its own content. Captures the + * content of a WindowContainer indicated by a WindowToken. If unable to start mirroring, falls + * back to original MediaProjection approach. + */ + private void startMirrorIfNeeded() { + // Only mirror if this display does not have its own content. + if (mLastHasContent) { + return; + } + // Given the WindowToken of the DisplayArea to mirror, retrieve the associated + // SurfaceControl. + IBinder tokenToMirror = mWmService.mDisplayManagerInternal.getWindowTokenClientToMirror( + mDisplayId); + + if (tokenToMirror == null) { + // This DisplayContent instance is not involved in layer mirroring. If the display + // has been created for capturing, fall back to prior MediaProjection approach. + return; + } + final WindowContainer wc = mWmService.mWindowContextListenerController.getContainer( + tokenToMirror); + if (wc == null) { + // Un-set the window token to mirror for this VirtualDisplay, to fall back to the + // original MediaProjection approach. + mWmService.mDisplayManagerInternal.setWindowTokenClientToMirror(mDisplayId, null); + return; + } + SurfaceControl sc = wc.getDisplayContent().getSurfaceControl(); + + // Create a mirrored hierarchy for the SurfaceControl of the DisplayArea to capture. + mMirroredSurface = SurfaceControl.mirrorSurface(sc); + SurfaceControl.Transaction transaction = mWmService.mTransactionFactory.get() + // Set the mMirroredSurface's parent to the root SurfaceControl for this + // DisplayContent. This brings the new mirrored hierarchy under this DisplayContent, + // so SurfaceControl will write the layers of this hierarchy to the output surface + // provided by the app. + .reparent(mMirroredSurface, mSurfaceControl) + // Reparent the SurfaceControl of this DisplayContent to null, to prevent content + // being added to it. This ensures that no app launched explicitly on the + // VirtualDisplay will show up as part of the mirrored content. + .reparent(mWindowingLayer, null); + // Retrieve the size of the DisplayArea to mirror. + updateMirroredSurface(transaction, wc.getDisplayContent().getBounds()); + mTokenToMirror = tokenToMirror; + + // No need to clean up. In SurfaceFlinger, parents hold references to their children. The + // mirrored SurfaceControl is alive since the parent DisplayContent SurfaceControl is + // holding a reference to it. Therefore, the mirrored SurfaceControl will be cleaned up + // when the VirtualDisplay is destroyed - which will clean up this DisplayContent. + } + + /** + * Start or stop mirroring if this DisplayContent now has content, or no longer has content. + */ + private void updateMirroring() { + if (mLastHasContent && mMirroredSurface != null) { + // Display now has content, so stop mirroring to it. + mWmService.mTransactionFactory.get() + // Remove the reference to mMirroredSurface, to clean up associated memory. + .remove(mMirroredSurface) + // Reparent the SurfaceControl of this DisplayContent back to mSurfaceControl, + // to allow content to be added to it. This allows this DisplayContent to stop + // mirroring and show content normally. + .reparent(mWindowingLayer, mSurfaceControl).apply(); + // Stop mirroring by destroying the reference to the mirrored layer. + mMirroredSurface = null; + // Do not un-set the token, in case content is removed and mirroring should begin again. + } else if (!mLastHasContent && mMirroredSurface == null) { + // Display no longer has content, so start mirroring to it. + startMirrorIfNeeded(); + } + } + + /** + * Apply transformations to the mirrored surface to ensure the captured contents are scaled to + * fit and centred in the output surface. + * + * @param transaction the transaction to include transformations of mMirroredSurface + * to. Transaction is not applied before returning. + * @param displayAreaBounds bounds of the DisplayArea to mirror to the surface provided by + * the app. + */ + @VisibleForTesting + void updateMirroredSurface(SurfaceControl.Transaction transaction, + Rect displayAreaBounds) { + // Retrieve the default size of the surface the app provided to + // MediaProjection#createVirtualDisplay. Note the app is the consumer of the surface, + // since it reads out buffers from the surface, and SurfaceFlinger is the producer since + // it writes the mirrored layers to the buffers. + final Point surfaceSize = mWmService.mDisplayManagerInternal.getDisplaySurfaceDefaultSize( + mDisplayId); + + // Calculate the scale to apply to the root mirror SurfaceControl to fit the size of the + // output surface. + float scaleX = surfaceSize.x / (float) displayAreaBounds.width(); + float scaleY = surfaceSize.y / (float) displayAreaBounds.height(); + float scale = Math.min(scaleX, scaleY); + int scaledWidth = Math.round(scale * (float) displayAreaBounds.width()); + int scaledHeight = Math.round(scale * (float) displayAreaBounds.height()); + + // Calculate the shift to apply to the root mirror SurfaceControl to centre the mirrored + // contents in the output surface. + int shiftedX = 0; + if (scaledWidth != surfaceSize.x) { + shiftedX = (surfaceSize.x - scaledWidth) / 2; + } + int shiftedY = 0; + if (scaledHeight != surfaceSize.y) { + shiftedY = (surfaceSize.y - scaledHeight) / 2; + } + + transaction + // Crop the area to capture to exclude the 'extra' wallpaper that is used + // for parallax (b/189930234). + .setWindowCrop(mMirroredSurface, displayAreaBounds.width(), + displayAreaBounds.height()) + // Scale the root mirror SurfaceControl, based upon the size difference between the + // source (DisplayArea to capture) and output (surface the app reads images from). + .setMatrix(mMirroredSurface, scale, 0 /* dtdx */, 0 /* dtdy */, scale) + // Position needs to be updated when the mirrored DisplayArea has changed, since + // the content will no longer be centered in the output surface. + .setPosition(mMirroredSurface, shiftedX /* x */, shiftedY /* y */) + .apply(); + mLastMirroredDisplayAreaBounds = new Rect(displayAreaBounds); + } + /** The entry for proceeding to handle {@link #mFixedRotationLaunchingApp}. */ class FixedRotationTransitionListener extends WindowManagerInternal.AppTransitionListener { diff --git a/services/tests/wmtests/src/com/android/server/wm/DisplayContentTests.java b/services/tests/wmtests/src/com/android/server/wm/DisplayContentTests.java index ea203c3a82e3f..597ad24d743cf 100644 --- a/services/tests/wmtests/src/com/android/server/wm/DisplayContentTests.java +++ b/services/tests/wmtests/src/com/android/server/wm/DisplayContentTests.java @@ -67,6 +67,7 @@ import static com.android.dx.mockito.inline.extended.ExtendedMockito.anyBoolean; import static com.android.dx.mockito.inline.extended.ExtendedMockito.doNothing; import static com.android.dx.mockito.inline.extended.ExtendedMockito.doReturn; import static com.android.dx.mockito.inline.extended.ExtendedMockito.mock; +import static com.android.dx.mockito.inline.extended.ExtendedMockito.mockitoSession; import static com.android.dx.mockito.inline.extended.ExtendedMockito.never; import static com.android.dx.mockito.inline.extended.ExtendedMockito.reset; import static com.android.dx.mockito.inline.extended.ExtendedMockito.same; @@ -107,9 +108,12 @@ import android.app.WindowConfiguration; import android.app.servertransaction.FixedRotationAdjustmentsItem; import android.content.res.Configuration; import android.graphics.Insets; +import android.graphics.Point; import android.graphics.Rect; import android.graphics.Region; import android.metrics.LogMaker; +import android.os.Binder; +import android.os.IBinder; import android.os.RemoteException; import android.os.SystemClock; import android.platform.test.annotations.Presubmit; @@ -142,6 +146,8 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.ArgumentCaptor; import org.mockito.Mockito; +import org.mockito.MockitoSession; +import org.mockito.quality.Strictness; import java.util.ArrayList; import java.util.Arrays; @@ -2217,6 +2223,113 @@ public class DisplayContentTests extends WindowTestsBase { assertNotEquals(imeMenuDialog, mDisplayContent.findFocusedWindow()); } + @Test + public void testVirtualDisplayContent() { + MockitoSession mockSession = mockitoSession() + .initMocks(this) + .spyStatic(SurfaceControl.class) + .strictness(Strictness.LENIENT) + .startMocking(); + + // GIVEN MediaProjection has already initialized the WindowToken of the DisplayArea to + // mirror. + final IBinder tokenToMirror = setUpDefaultTaskDisplayAreaWindowToken(); + + // GIVEN SurfaceControl can successfully mirror the provided surface. + Point surfaceSize = new Point( + mDefaultDisplay.getDefaultTaskDisplayArea().getBounds().width(), + mDefaultDisplay.getDefaultTaskDisplayArea().getBounds().height()); + surfaceControlMirrors(surfaceSize); + + // WHEN creating the DisplayContent for a new virtual display. + final DisplayContent virtualDisplay = new TestDisplayContent.Builder(mAtm, + mDisplayInfo).build(); + + // THEN mirroring is initiated for the default display's DisplayArea. + assertThat(virtualDisplay.mTokenToMirror).isEqualTo(tokenToMirror); + + mockSession.finishMocking(); + } + + @Test + public void testVirtualDisplayContent_capturedAreaResized() { + MockitoSession mockSession = mockitoSession() + .initMocks(this) + .spyStatic(SurfaceControl.class) + .strictness(Strictness.LENIENT) + .startMocking(); + + // GIVEN MediaProjection has already initialized the WindowToken of the DisplayArea to + // mirror. + final IBinder tokenToMirror = setUpDefaultTaskDisplayAreaWindowToken(); + + // GIVEN SurfaceControl can successfully mirror the provided surface. + Point surfaceSize = new Point( + mDefaultDisplay.getDefaultTaskDisplayArea().getBounds().width(), + mDefaultDisplay.getDefaultTaskDisplayArea().getBounds().height()); + SurfaceControl mirroredSurface = surfaceControlMirrors(surfaceSize); + + // WHEN creating the DisplayContent for a new virtual display. + final DisplayContent virtualDisplay = new TestDisplayContent.Builder(mAtm, + mDisplayInfo).build(); + + // THEN mirroring is initiated for the default display's DisplayArea. + assertThat(virtualDisplay.mTokenToMirror).isEqualTo(tokenToMirror); + + float xScale = 0.7f; + float yScale = 2f; + Rect displayAreaBounds = new Rect(0, 0, Math.round(surfaceSize.x * xScale), + Math.round(surfaceSize.y * yScale)); + virtualDisplay.updateMirroredSurface(mTransaction, displayAreaBounds); + + // THEN content in the captured DisplayArea is scaled to fit the surface size. + verify(mTransaction, atLeastOnce()).setMatrix(mirroredSurface, 1.0f / yScale, 0, 0, + 1.0f / yScale); + // THEN captured content is positioned in the centre of the output surface. + float scaledWidth = displayAreaBounds.width() / xScale; + float xInset = (surfaceSize.x - scaledWidth) / 2; + verify(mTransaction, atLeastOnce()).setPosition(mirroredSurface, xInset, 0); + + mockSession.finishMocking(); + } + + private class TestToken extends Binder { + } + + /** + * Creates a WindowToken associated with the default task DisplayArea, in order for that + * DisplayArea to be mirrored. + */ + private IBinder setUpDefaultTaskDisplayAreaWindowToken() { + // GIVEN MediaProjection has already initialized the WindowToken of the DisplayArea to + // mirror. + final IBinder tokenToMirror = new TestToken(); + doReturn(tokenToMirror).when(mWm.mDisplayManagerInternal).getWindowTokenClientToMirror( + anyInt()); + + // GIVEN the default task display area is represented by the WindowToken. + spyOn(mWm.mWindowContextListenerController); + doReturn(mDefaultDisplay.getDefaultTaskDisplayArea()).when( + mWm.mWindowContextListenerController).getContainer(any()); + return tokenToMirror; + } + + /** + * SurfaceControl successfully creates a mirrored surface of the given size. + */ + private SurfaceControl surfaceControlMirrors(Point surfaceSize) { + // Do not set the parent, since the mirrored surface is the root of a new surface hierarchy. + SurfaceControl mirroredSurface = new SurfaceControl.Builder() + .setName("mirroredSurface") + .setBufferSize(surfaceSize.x, surfaceSize.y) + .setCallsite("mirrorSurface") + .build(); + doReturn(mirroredSurface).when(() -> SurfaceControl.mirrorSurface(any())); + doReturn(surfaceSize).when(mWm.mDisplayManagerInternal).getDisplaySurfaceDefaultSize( + anyInt()); + return mirroredSurface; + } + private void removeRootTaskTests(Runnable runnable) { final TaskDisplayArea taskDisplayArea = mRootWindowContainer.getDefaultTaskDisplayArea(); final Task rootTask1 = taskDisplayArea.createRootTask(WINDOWING_MODE_FULLSCREEN,