Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MediaProjection captures DisplayArea

Browse Source 
MediaProjection is used by apps to capture the contents of
a display. Apps consume the content from a surface passed
in to MediaProjection#createVirtualDisplay. The images
written out to the surface can be used for saving
screenshots of the current display, screen recording,
or casting to a different display.

MediaProjection is currently tied to capturing a physical
display. This change instead captures the DisplayArea
the app is launched on. This is consistent with Display API
sandboxing, which returns the DisplayArea bounds rather
than the physical display bounds.

Note that apps use the Display APIs to size the
VirtualDisplay and output surface.

Verified: screenshots, screen recording, and casting

Bug: 180624777
Bug: 195280234
Test: atest WmTests:DisplayContentTests

Change-Id: I086fa1ae668dbdf0f25252c13614445af04d86f3
This commit is contained in:
Naomi Musgrave
2021-08-05 12:42:18 +00:00
parent da38617955
commit fd5715bb31
9 changed files with 483 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
core/java/android/hardware/display/DisplayManagerInternal.java
View File

@@ -21,6 +21,7 @@ import android.annotation.Nullable;
import android.graphics.Point;
import android.hardware.SensorManager;
import android.os.Handler;
import android.os.IBinder;
import android.os.PowerManager;
import android.util.IntArray;
import android.util.Slog;
@@ -339,6 +340,28 @@ public abstract class DisplayManagerInternal {
*/
public abstract List<RefreshRateLimitation> getRefreshRateLimitations(int displayId);
/**
* Returns the window token of the level of the WindowManager hierarchy to mirror. Returns null
* if layer mirroring by SurfaceFlinger should not be performed for the given displayId.
* For now, only used for mirroring started from MediaProjection.
*/
public abstract IBinder getWindowTokenClientToMirror(int displayId);
/**
* For the given displayId, updates the window token of the level of the WindowManager hierarchy
* to mirror. If windowToken is null, then SurfaceFlinger performs no layer mirroring to the
* given display.
* For now, only used for mirroring started from MediaProjection.
*/
public abstract void setWindowTokenClientToMirror(int displayId, IBinder windowToken);
/**
* Returns the default size of the surface associated with the display, or null if the surface
* is not provided for layer mirroring by SurfaceFlinger.
* For now, only used for mirroring started from MediaProjection.
*/
public abstract Point getDisplaySurfaceDefaultSize(int displayId);
/**
* Describes the requested power state of the display.
*

56
core/java/android/hardware/display/VirtualDisplayConfig.java
View File

@@ -23,6 +23,7 @@ import android.annotation.NonNull;
import android.annotation.Nullable;
import android.media.projection.MediaProjection;
import android.os.Handler;
import android.os.IBinder;
import android.os.Parcel;
import android.os.Parcelable;
import android.view.Surface;
@@ -91,9 +92,16 @@ public final class VirtualDisplayConfig implements Parcelable {
*/
private int mDisplayIdToMirror = DEFAULT_DISPLAY;
/**
* The window token of the level of the WindowManager hierarchy to mirror, or null if mirroring
* should not be performed.
*/
@Nullable
private IBinder mWindowTokenClientToMirror = null;
// Code below generated by codegen v1.0.20.
// Code below generated by codegen v1.0.23.
//
// DO NOT MODIFY!
// CHECKSTYLE:OFF Generated code
@@ -115,7 +123,8 @@ public final class VirtualDisplayConfig implements Parcelable {
int flags,
@Nullable Surface surface,
@Nullable String uniqueId,
int displayIdToMirror) {
int displayIdToMirror,
@Nullable IBinder windowTokenClientToMirror) {
this.mName = name;
com.android.internal.util.AnnotationValidations.validate(
NonNull.class, null, mName);
@@ -135,6 +144,7 @@ public final class VirtualDisplayConfig implements Parcelable {
this.mSurface = surface;
this.mUniqueId = uniqueId;
this.mDisplayIdToMirror = displayIdToMirror;
this.mWindowTokenClientToMirror = windowTokenClientToMirror;
// onConstructed(); // You can define this method to get a callback
}
@@ -212,6 +222,15 @@ public final class VirtualDisplayConfig implements Parcelable {
return mDisplayIdToMirror;
}
/**
* The window token of the level of the WindowManager hierarchy to mirror, or null if mirroring
* should not be performed.
*/
@DataClass.Generated.Member
public @Nullable IBinder getWindowTokenClientToMirror() {
return mWindowTokenClientToMirror;
}
@Override
@DataClass.Generated.Member
public void writeToParcel(@NonNull Parcel dest, int flags) {
@@ -221,6 +240,7 @@ public final class VirtualDisplayConfig implements Parcelable {
int flg = 0;
if (mSurface != null) flg |= 0x20;
if (mUniqueId != null) flg |= 0x40;
if (mWindowTokenClientToMirror != null) flg |= 0x100;
dest.writeInt(flg);
dest.writeString(mName);
dest.writeInt(mWidth);
@@ -230,6 +250,7 @@ public final class VirtualDisplayConfig implements Parcelable {
if (mSurface != null) dest.writeTypedObject(mSurface, flags);
if (mUniqueId != null) dest.writeString(mUniqueId);
dest.writeInt(mDisplayIdToMirror);
if (mWindowTokenClientToMirror != null) dest.writeStrongBinder(mWindowTokenClientToMirror);
}
@Override
@@ -252,6 +273,7 @@ public final class VirtualDisplayConfig implements Parcelable {
Surface surface = (flg & 0x20) == 0 ? null : (Surface) in.readTypedObject(Surface.CREATOR);
String uniqueId = (flg & 0x40) == 0 ? null : in.readString();
int displayIdToMirror = in.readInt();
IBinder windowTokenClientToMirror = (flg & 0x100) == 0 ? null : (IBinder) in.readStrongBinder();
this.mName = name;
com.android.internal.util.AnnotationValidations.validate(
@@ -272,6 +294,7 @@ public final class VirtualDisplayConfig implements Parcelable {
this.mSurface = surface;
this.mUniqueId = uniqueId;
this.mDisplayIdToMirror = displayIdToMirror;
this.mWindowTokenClientToMirror = windowTokenClientToMirror;
// onConstructed(); // You can define this method to get a callback
}
@@ -305,6 +328,7 @@ public final class VirtualDisplayConfig implements Parcelable {
private @Nullable Surface mSurface;
private @Nullable String mUniqueId;
private int mDisplayIdToMirror;
private @Nullable IBinder mWindowTokenClientToMirror;
private long mBuilderFieldsSet = 0L;
@@ -439,10 +463,22 @@ public final class VirtualDisplayConfig implements Parcelable {
return this;
}
/**
* The window token of the level of the WindowManager hierarchy to mirror, or null if mirroring
* should not be performed.
*/
@DataClass.Generated.Member
public @NonNull Builder setWindowTokenClientToMirror(@NonNull IBinder value) {
checkNotUsed();
mBuilderFieldsSet |= 0x100;
mWindowTokenClientToMirror = value;
return this;
}
/** Builds the instance. This builder should not be touched after calling this! */
public @NonNull VirtualDisplayConfig build() {
checkNotUsed();
mBuilderFieldsSet |= 0x100; // Mark builder used
mBuilderFieldsSet |= 0x200; // Mark builder used
if ((mBuilderFieldsSet & 0x10) == 0) {
mFlags = 0;
@@ -456,6 +492,9 @@ public final class VirtualDisplayConfig implements Parcelable {
if ((mBuilderFieldsSet & 0x80) == 0) {
mDisplayIdToMirror = DEFAULT_DISPLAY;
}
if ((mBuilderFieldsSet & 0x100) == 0) {
mWindowTokenClientToMirror = null;
}
VirtualDisplayConfig o = new VirtualDisplayConfig(
mName,
mWidth,
@@ -464,12 +503,13 @@ public final class VirtualDisplayConfig implements Parcelable {
mFlags,
mSurface,
mUniqueId,
mDisplayIdToMirror);
mDisplayIdToMirror,
mWindowTokenClientToMirror);
return o;
}
private void checkNotUsed() {
if ((mBuilderFieldsSet & 0x100) != 0) {
if ((mBuilderFieldsSet & 0x200) != 0) {
throw new IllegalStateException(
"This Builder should not be reused. Use a new Builder instance instead");
}
@@ -477,10 +517,10 @@ public final class VirtualDisplayConfig implements Parcelable {
}
@DataClass.Generated(
time = 1604456298440L,
codegenVersion = "1.0.20",
time = 1620657851981L,
codegenVersion = "1.0.23",
sourceFile = "frameworks/base/core/java/android/hardware/display/VirtualDisplayConfig.java",
inputSignatures = "private @android.annotation.NonNull java.lang.String mName\nprivate @android.annotation.IntRange int mWidth\nprivate @android.annotation.IntRange int mHeight\nprivate @android.annotation.IntRange int mDensityDpi\nprivate int mFlags\nprivate @android.annotation.Nullable android.view.Surface mSurface\nprivate @android.annotation.Nullable java.lang.String mUniqueId\nprivate int mDisplayIdToMirror\nclass VirtualDisplayConfig extends java.lang.Object implements [android.os.Parcelable]\n@com.android.internal.util.DataClass(genParcelable=true, genAidl=true, genBuilder=true)")
inputSignatures = "private @android.annotation.NonNull java.lang.String mName\nprivate @android.annotation.IntRange int mWidth\nprivate @android.annotation.IntRange int mHeight\nprivate @android.annotation.IntRange int mDensityDpi\nprivate int mFlags\nprivate @android.annotation.Nullable android.view.Surface mSurface\nprivate @android.annotation.Nullable java.lang.String mUniqueId\nprivate int mDisplayIdToMirror\nprivate @android.annotation.Nullable android.os.IBinder mWindowTokenClientToMirror\nclass VirtualDisplayConfig extends java.lang.Object implements [android.os.Parcelable]\n@com.android.internal.util.DataClass(genParcelable=true, genAidl=true, genBuilder=true)")
@Deprecated
private void __metadata() {}

15
core/java/android/view/Surface.java
View File

@@ -29,6 +29,7 @@ import android.graphics.Canvas;
import android.graphics.ColorSpace;
import android.graphics.HardwareRenderer;
import android.graphics.Matrix;
import android.graphics.Point;
import android.graphics.RecordingCanvas;
import android.graphics.Rect;
import android.graphics.RenderNode;
@@ -407,6 +408,20 @@ public class Surface implements Parcelable {
}
}
/**
* Returns the default size of this Surface provided by the consumer of the surface.
* Should only be used by the producer of the surface.
*
* @hide
*/
@NonNull
public Point getDefaultSize() {
synchronized (mLock) {
checkNotReleasedLocked();
return new Point(nativeGetWidth(mNativeObject), nativeGetHeight(mNativeObject));
}
}
/**
* Gets a {@link Canvas} for drawing into this surface.
*

28
media/java/android/media/projection/MediaProjection.java
View File

@@ -16,14 +16,14 @@
package android.media.projection;
import static android.view.WindowManager.LayoutParams.TYPE_APPLICATION;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.content.Context;
import android.hardware.display.DisplayManager;
import android.hardware.display.VirtualDisplay;
import android.hardware.display.VirtualDisplayConfig;
import android.media.projection.IMediaProjection;
import android.media.projection.IMediaProjectionCallback;
import android.os.Handler;
import android.os.RemoteException;
import android.util.ArrayMap;
@@ -106,7 +106,7 @@ public final class MediaProjection {
if (isSecure) {
flags |= DisplayManager.VIRTUAL_DISPLAY_FLAG_SECURE;
}
final VirtualDisplayConfig.Builder builder = new VirtualDisplayConfig.Builder(name, width,
final VirtualDisplayConfig.Builder builder = buildMirroredVirtualDisplay(name, width,
height, dpi);
builder.setFlags(flags);
if (surface != null) {
@@ -141,7 +141,7 @@ public final class MediaProjection {
public VirtualDisplay createVirtualDisplay(@NonNull String name,
int width, int height, int dpi, int flags, @Nullable Surface surface,
@Nullable VirtualDisplay.Callback callback, @Nullable Handler handler) {
final VirtualDisplayConfig.Builder builder = new VirtualDisplayConfig.Builder(name, width,
final VirtualDisplayConfig.Builder builder = buildMirroredVirtualDisplay(name, width,
height, dpi);
builder.setFlags(flags);
if (surface != null) {
@@ -150,6 +150,26 @@ public final class MediaProjection {
return createVirtualDisplay(builder.build(), callback, handler);
}
/**
* Constructs a {@link VirtualDisplayConfig.Builder}, which will mirror the contents of a
* DisplayArea. The DisplayArea to mirror is from the DisplayArea the caller is launched on.
*
* @param name The name of the virtual display, must be non-empty.
* @param width The width of the virtual display in pixels. Must be greater than 0.
* @param height The height of the virtual display in pixels. Must be greater than 0.
* @param dpi The density of the virtual display in dpi. Must be greater than 0.
* @return a config representing a VirtualDisplay
*/
private VirtualDisplayConfig.Builder buildMirroredVirtualDisplay(@NonNull String name,
int width, int height, int dpi) {
Context windowContext = mContext.createWindowContext(mContext.getDisplayNoVerify(),
TYPE_APPLICATION, null /* options */);
final VirtualDisplayConfig.Builder builder = new VirtualDisplayConfig.Builder(name, width,
height, dpi);
builder.setWindowTokenClientToMirror(windowContext.getWindowContextToken());
return builder;
}
/**
* Creates a {@link android.hardware.display.VirtualDisplay} to capture the
* contents of the screen.

30
services/core/java/com/android/server/display/DisplayDevice.java
View File

@@ -16,7 +16,9 @@
package com.android.server.display;
import android.annotation.Nullable;
import android.content.Context;
import android.graphics.Point;
import android.graphics.Rect;
import android.hardware.display.DisplayViewport;
import android.os.IBinder;
@@ -104,6 +106,34 @@ abstract class DisplayDevice {
return Display.DEFAULT_DISPLAY;
}
/**
* Returns the window token of the level of the WindowManager hierarchy to mirror, or null
* if layer mirroring by SurfaceFlinger should not be performed.
* For now, only used for mirroring started from MediaProjection.
*/
@Nullable
public IBinder getWindowTokenClientToMirrorLocked() {
return null;
}
/**
* Updates the window token of the level of the level of the WindowManager hierarchy to mirror.
* If windowToken is null, then no layer mirroring by SurfaceFlinger to should be performed.
* For now, only used for mirroring started from MediaProjection.
*/
public void setWindowTokenClientToMirrorLocked(IBinder windowToken) {
}
/**
* Returns the default size of the surface associated with the display, or null if the surface
* is not provided for layer mirroring by SurfaceFlinger.
* For now, only used for mirroring started from MediaProjection.
*/
@Nullable
public Point getDisplaySurfaceDefaultSize() {
return null;
}
/**
* Gets the name of the display device.
*

41
services/core/java/com/android/server/display/DisplayManagerService.java
View File

@@ -63,8 +63,6 @@ import android.hardware.display.DisplayManagerGlobal;
import android.hardware.display.DisplayManagerInternal;
import android.hardware.display.DisplayManagerInternal.DisplayGroupListener;
import android.hardware.display.DisplayManagerInternal.DisplayTransactionListener;
import android.hardware.display.DisplayManagerInternal.RefreshRateLimitation;
import android.hardware.display.DisplayManagerInternal.RefreshRateRange;
import android.hardware.display.DisplayViewport;
import android.hardware.display.DisplayedContentSample;
import android.hardware.display.DisplayedContentSamplingAttributes;
@@ -1749,10 +1747,13 @@ public final class DisplayManagerService extends SystemService {
final DisplayDeviceInfo info = device.getDisplayDeviceInfoLocked();
final boolean ownContent = (info.flags & DisplayDeviceInfo.FLAG_OWN_CONTENT_ONLY) != 0;
// Mirror the part of WM hierarchy that corresponds to the provided window token.
IBinder windowTokenClientToMirror = device.getWindowTokenClientToMirrorLocked();
// Find the logical display that the display device is showing.
// Certain displays only ever show their own content.
LogicalDisplay display = mLogicalDisplayMapper.getDisplayLocked(device);
if (!ownContent) {
if (!ownContent && windowTokenClientToMirror == null) {
if (display != null && !display.hasContentLocked()) {
// If the display does not have any content of its own, then
// automatically mirror the requested logical display contents if possible.
@@ -3314,6 +3315,40 @@ public final class DisplayManagerService extends SystemService {
}
return config.getRefreshRateLimitations();
}
@Override
public IBinder getWindowTokenClientToMirror(int displayId) {
final DisplayDevice device;
synchronized (mSyncRoot) {
device = getDeviceForDisplayLocked(displayId);
if (device == null) {
return null;
}
}
return device.getWindowTokenClientToMirrorLocked();
}
@Override
public void setWindowTokenClientToMirror(int displayId, IBinder windowToken) {
synchronized (mSyncRoot) {
final DisplayDevice device = getDeviceForDisplayLocked(displayId);
if (device != null) {
device.setWindowTokenClientToMirrorLocked(windowToken);
}
}
}
@Override
public Point getDisplaySurfaceDefaultSize(int displayId) {
final DisplayDevice device;
synchronized (mSyncRoot) {
device = getDeviceForDisplayLocked(displayId);
if (device == null) {
return null;
}
}
return device.getDisplaySurfaceDefaultSize();
}
}
class DesiredDisplayModeSpecsObserver

25
services/core/java/com/android/server/display/VirtualDisplayAdapter.java
View File

@@ -31,7 +31,9 @@ import static android.hardware.display.DisplayManager.VIRTUAL_DISPLAY_FLAG_TRUST
import static com.android.server.display.DisplayDeviceInfo.FLAG_OWN_DISPLAY_GROUP;
import static com.android.server.display.DisplayDeviceInfo.FLAG_TRUSTED;
import android.annotation.Nullable;
import android.content.Context;
import android.graphics.Point;
import android.hardware.display.IVirtualDisplayCallback;
import android.hardware.display.VirtualDisplayConfig;
import android.media.projection.IMediaProjection;
@@ -231,6 +233,7 @@ public class VirtualDisplayAdapter extends DisplayAdapter {
private Display.Mode mMode;
private boolean mIsDisplayOn;
private int mDisplayIdToMirror;
private IBinder mWindowTokenClientToMirror;
public VirtualDisplayDevice(IBinder displayToken, IBinder appToken,
int ownerUid, String ownerPackageName, Surface surface, int flags,
@@ -253,6 +256,7 @@ public class VirtualDisplayAdapter extends DisplayAdapter {
mUniqueIndex = uniqueIndex;
mIsDisplayOn = surface != null;
mDisplayIdToMirror = virtualDisplayConfig.getDisplayIdToMirror();
mWindowTokenClientToMirror = virtualDisplayConfig.getWindowTokenClientToMirror();
}
@Override
@@ -282,6 +286,26 @@ public class VirtualDisplayAdapter extends DisplayAdapter {
return mDisplayIdToMirror;
}
@Override
@Nullable
public IBinder getWindowTokenClientToMirrorLocked() {
return mWindowTokenClientToMirror;
}
@Override
public void setWindowTokenClientToMirrorLocked(IBinder windowToken) {
if (mWindowTokenClientToMirror != windowToken) {
mWindowTokenClientToMirror = windowToken;
sendDisplayDeviceEventLocked(this, DISPLAY_DEVICE_EVENT_CHANGED);
sendTraversalRequestLocked();
}
}
@Override
public Point getDisplaySurfaceDefaultSize() {
return mSurface.getDefaultSize();
}
@VisibleForTesting
Surface getSurfaceLocked() {
return mSurface;
@@ -362,6 +386,7 @@ public class VirtualDisplayAdapter extends DisplayAdapter {
pw.println("mDisplayState=" + Display.stateToString(mDisplayState));
pw.println("mStopped=" + mStopped);
pw.println("mDisplayIdToMirror=" + mDisplayIdToMirror);
pw.println("mWindowTokenClientToMirror=" + mWindowTokenClientToMirror);
}

167
services/core/java/com/android/server/wm/DisplayContent.java
View File

@@ -299,6 +299,22 @@ class DisplayContent extends RootDisplayArea implements WindowManagerPolicy.Disp
*/
private final SurfaceControl mWindowingLayer;
/**
* The window token of the layer of the hierarchy to mirror, or null if this DisplayContent
* is not being used for layer mirroring.
*/
@VisibleForTesting IBinder mTokenToMirror = null;
/**
* The surface for mirroring the contents of this hierarchy.
*/
private SurfaceControl mMirroredSurface = null;
/**
* The last bounds of the DisplayArea to mirror.
*/
private Rect mLastMirroredDisplayAreaBounds = null;
// Contains all IME window containers. Note that the z-ordering of the IME windows will depend
// on the IME target. We mainly have this container grouping so we can keep track of all the IME
// window containers together and move them in-sync if/when needed. We use a subclass of
@@ -1117,6 +1133,10 @@ class DisplayContent extends RootDisplayArea implements WindowManagerPolicy.Disp
if (DEBUG_DISPLAY) Slog.v(TAG_WM, "Creating display=" + display);
mWmService.mDisplayWindowSettings.applySettingsToDisplayLocked(this);
// Check if this DisplayContent is for a new VirtualDisplay, that should use layer mirroring
// to capture the contents of a DisplayArea.
startMirrorIfNeeded();
}
boolean isReady() {
@@ -2471,6 +2491,25 @@ class DisplayContent extends RootDisplayArea implements WindowManagerPolicy.Disp
// Update IME parent if needed.
updateImeParent();
// Update mirroring surface for MediaProjection, if this DisplayContent is being used
// for layer mirroring.
if (mMirroredSurface != null) {
// Retrieve the size of the DisplayArea to mirror, and continue with the update if the
// bounds have changed.
final WindowContainer wc = mWmService.mWindowContextListenerController.getContainer(
mTokenToMirror);
if (wc != null && mLastMirroredDisplayAreaBounds != null) {
// Retrieve the size of the DisplayArea to mirror, and continue with the update
// if the bounds or orientation has changed.
final Rect displayAreaBounds = wc.getDisplayContent().getBounds();
int displayAreaOrientation = wc.getDisplayContent().getOrientation();
if (!mLastMirroredDisplayAreaBounds.equals(displayAreaBounds)
|| lastOrientation != displayAreaOrientation) {
updateMirroredSurface(mWmService.mTransactionFactory.get(), displayAreaBounds);
}
}
}
if (lastOrientation != getConfiguration().orientation) {
getMetricsLogger().write(
new LogMaker(MetricsEvent.ACTION_PHONE_ORIENTATION_CHANGED)
@@ -4340,6 +4379,7 @@ class DisplayContent extends RootDisplayArea implements WindowManagerPolicy.Disp
mTmpApplySurfaceChangesTransactionState.preferMinimalPostProcessing,
true /* inTraversal, must call performTraversalInTrans... below */);
}
updateMirroring();
final boolean wallpaperVisible = mWallpaperController.isWallpaperVisible();
if (wallpaperVisible != mLastWallpaperVisible) {
@@ -5892,6 +5932,133 @@ class DisplayContent extends RootDisplayArea implements WindowManagerPolicy.Disp
return mSandboxDisplayApis;
}
/**
* Start mirroring to this DisplayContent if it does not have its own content. Captures the
* content of a WindowContainer indicated by a WindowToken. If unable to start mirroring, falls
* back to original MediaProjection approach.
*/
private void startMirrorIfNeeded() {
// Only mirror if this display does not have its own content.
if (mLastHasContent) {
return;
}
// Given the WindowToken of the DisplayArea to mirror, retrieve the associated
// SurfaceControl.
IBinder tokenToMirror = mWmService.mDisplayManagerInternal.getWindowTokenClientToMirror(
mDisplayId);
if (tokenToMirror == null) {
// This DisplayContent instance is not involved in layer mirroring. If the display
// has been created for capturing, fall back to prior MediaProjection approach.
return;
}
final WindowContainer wc = mWmService.mWindowContextListenerController.getContainer(
tokenToMirror);
if (wc == null) {
// Un-set the window token to mirror for this VirtualDisplay, to fall back to the
// original MediaProjection approach.
mWmService.mDisplayManagerInternal.setWindowTokenClientToMirror(mDisplayId, null);
return;
}
SurfaceControl sc = wc.getDisplayContent().getSurfaceControl();
// Create a mirrored hierarchy for the SurfaceControl of the DisplayArea to capture.
mMirroredSurface = SurfaceControl.mirrorSurface(sc);
SurfaceControl.Transaction transaction = mWmService.mTransactionFactory.get()
// Set the mMirroredSurface's parent to the root SurfaceControl for this
// DisplayContent. This brings the new mirrored hierarchy under this DisplayContent,
// so SurfaceControl will write the layers of this hierarchy to the output surface
// provided by the app.
.reparent(mMirroredSurface, mSurfaceControl)
// Reparent the SurfaceControl of this DisplayContent to null, to prevent content
// being added to it. This ensures that no app launched explicitly on the
// VirtualDisplay will show up as part of the mirrored content.
.reparent(mWindowingLayer, null);
// Retrieve the size of the DisplayArea to mirror.
updateMirroredSurface(transaction, wc.getDisplayContent().getBounds());
mTokenToMirror = tokenToMirror;
// No need to clean up. In SurfaceFlinger, parents hold references to their children. The
// mirrored SurfaceControl is alive since the parent DisplayContent SurfaceControl is
// holding a reference to it. Therefore, the mirrored SurfaceControl will be cleaned up
// when the VirtualDisplay is destroyed - which will clean up this DisplayContent.
}
/**
* Start or stop mirroring if this DisplayContent now has content, or no longer has content.
*/
private void updateMirroring() {
if (mLastHasContent && mMirroredSurface != null) {
// Display now has content, so stop mirroring to it.
mWmService.mTransactionFactory.get()
// Remove the reference to mMirroredSurface, to clean up associated memory.
.remove(mMirroredSurface)
// Reparent the SurfaceControl of this DisplayContent back to mSurfaceControl,
// to allow content to be added to it. This allows this DisplayContent to stop
// mirroring and show content normally.
.reparent(mWindowingLayer, mSurfaceControl).apply();
// Stop mirroring by destroying the reference to the mirrored layer.
mMirroredSurface = null;
// Do not un-set the token, in case content is removed and mirroring should begin again.
} else if (!mLastHasContent && mMirroredSurface == null) {
// Display no longer has content, so start mirroring to it.
startMirrorIfNeeded();
}
}
/**
* Apply transformations to the mirrored surface to ensure the captured contents are scaled to
* fit and centred in the output surface.
*
* @param transaction the transaction to include transformations of mMirroredSurface
* to. Transaction is not applied before returning.
* @param displayAreaBounds bounds of the DisplayArea to mirror to the surface provided by
* the app.
*/
@VisibleForTesting
void updateMirroredSurface(SurfaceControl.Transaction transaction,
Rect displayAreaBounds) {
// Retrieve the default size of the surface the app provided to
// MediaProjection#createVirtualDisplay. Note the app is the consumer of the surface,
// since it reads out buffers from the surface, and SurfaceFlinger is the producer since
// it writes the mirrored layers to the buffers.
final Point surfaceSize = mWmService.mDisplayManagerInternal.getDisplaySurfaceDefaultSize(
mDisplayId);
// Calculate the scale to apply to the root mirror SurfaceControl to fit the size of the
// output surface.
float scaleX = surfaceSize.x / (float) displayAreaBounds.width();
float scaleY = surfaceSize.y / (float) displayAreaBounds.height();
float scale = Math.min(scaleX, scaleY);
int scaledWidth = Math.round(scale * (float) displayAreaBounds.width());
int scaledHeight = Math.round(scale * (float) displayAreaBounds.height());
// Calculate the shift to apply to the root mirror SurfaceControl to centre the mirrored
// contents in the output surface.
int shiftedX = 0;
if (scaledWidth != surfaceSize.x) {
shiftedX = (surfaceSize.x - scaledWidth) / 2;
}
int shiftedY = 0;
if (scaledHeight != surfaceSize.y) {
shiftedY = (surfaceSize.y - scaledHeight) / 2;
}
transaction
// Crop the area to capture to exclude the 'extra' wallpaper that is used
// for parallax (b/189930234).
.setWindowCrop(mMirroredSurface, displayAreaBounds.width(),
displayAreaBounds.height())
// Scale the root mirror SurfaceControl, based upon the size difference between the
// source (DisplayArea to capture) and output (surface the app reads images from).
.setMatrix(mMirroredSurface, scale, 0 /* dtdx */, 0 /* dtdy */, scale)
// Position needs to be updated when the mirrored DisplayArea has changed, since
// the content will no longer be centered in the output surface.
.setPosition(mMirroredSurface, shiftedX /* x */, shiftedY /* y */)
.apply();
mLastMirroredDisplayAreaBounds = new Rect(displayAreaBounds);
}
/** The entry for proceeding to handle {@link #mFixedRotationLaunchingApp}. */
class FixedRotationTransitionListener extends WindowManagerInternal.AppTransitionListener {

113
services/tests/wmtests/src/com/android/server/wm/DisplayContentTests.java
View File

@@ -67,6 +67,7 @@ import static com.android.dx.mockito.inline.extended.ExtendedMockito.anyBoolean;
import static com.android.dx.mockito.inline.extended.ExtendedMockito.doNothing;
import static com.android.dx.mockito.inline.extended.ExtendedMockito.doReturn;
import static com.android.dx.mockito.inline.extended.ExtendedMockito.mock;
import static com.android.dx.mockito.inline.extended.ExtendedMockito.mockitoSession;
import static com.android.dx.mockito.inline.extended.ExtendedMockito.never;
import static com.android.dx.mockito.inline.extended.ExtendedMockito.reset;
import static com.android.dx.mockito.inline.extended.ExtendedMockito.same;
@@ -107,9 +108,12 @@ import android.app.WindowConfiguration;
import android.app.servertransaction.FixedRotationAdjustmentsItem;
import android.content.res.Configuration;
import android.graphics.Insets;
import android.graphics.Point;
import android.graphics.Rect;
import android.graphics.Region;
import android.metrics.LogMaker;
import android.os.Binder;
import android.os.IBinder;
import android.os.RemoteException;
import android.os.SystemClock;
import android.platform.test.annotations.Presubmit;
@@ -142,6 +146,8 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.ArgumentCaptor;
import org.mockito.Mockito;
import org.mockito.MockitoSession;
import org.mockito.quality.Strictness;
import java.util.ArrayList;
import java.util.Arrays;
@@ -2217,6 +2223,113 @@ public class DisplayContentTests extends WindowTestsBase {
assertNotEquals(imeMenuDialog, mDisplayContent.findFocusedWindow());
}
@Test
public void testVirtualDisplayContent() {
MockitoSession mockSession = mockitoSession()
.initMocks(this)
.spyStatic(SurfaceControl.class)
.strictness(Strictness.LENIENT)
.startMocking();
// GIVEN MediaProjection has already initialized the WindowToken of the DisplayArea to
// mirror.
final IBinder tokenToMirror = setUpDefaultTaskDisplayAreaWindowToken();
// GIVEN SurfaceControl can successfully mirror the provided surface.
Point surfaceSize = new Point(
mDefaultDisplay.getDefaultTaskDisplayArea().getBounds().width(),
mDefaultDisplay.getDefaultTaskDisplayArea().getBounds().height());
surfaceControlMirrors(surfaceSize);
// WHEN creating the DisplayContent for a new virtual display.
final DisplayContent virtualDisplay = new TestDisplayContent.Builder(mAtm,
mDisplayInfo).build();
// THEN mirroring is initiated for the default display's DisplayArea.
assertThat(virtualDisplay.mTokenToMirror).isEqualTo(tokenToMirror);
mockSession.finishMocking();
}
@Test
public void testVirtualDisplayContent_capturedAreaResized() {
MockitoSession mockSession = mockitoSession()
.initMocks(this)
.spyStatic(SurfaceControl.class)
.strictness(Strictness.LENIENT)
.startMocking();
// GIVEN MediaProjection has already initialized the WindowToken of the DisplayArea to
// mirror.
final IBinder tokenToMirror = setUpDefaultTaskDisplayAreaWindowToken();
// GIVEN SurfaceControl can successfully mirror the provided surface.
Point surfaceSize = new Point(
mDefaultDisplay.getDefaultTaskDisplayArea().getBounds().width(),
mDefaultDisplay.getDefaultTaskDisplayArea().getBounds().height());
SurfaceControl mirroredSurface = surfaceControlMirrors(surfaceSize);
// WHEN creating the DisplayContent for a new virtual display.
final DisplayContent virtualDisplay = new TestDisplayContent.Builder(mAtm,
mDisplayInfo).build();
// THEN mirroring is initiated for the default display's DisplayArea.
assertThat(virtualDisplay.mTokenToMirror).isEqualTo(tokenToMirror);
float xScale = 0.7f;
float yScale = 2f;
Rect displayAreaBounds = new Rect(0, 0, Math.round(surfaceSize.x * xScale),
Math.round(surfaceSize.y * yScale));
virtualDisplay.updateMirroredSurface(mTransaction, displayAreaBounds);
// THEN content in the captured DisplayArea is scaled to fit the surface size.
verify(mTransaction, atLeastOnce()).setMatrix(mirroredSurface, 1.0f / yScale, 0, 0,
1.0f / yScale);
// THEN captured content is positioned in the centre of the output surface.
float scaledWidth = displayAreaBounds.width() / xScale;
float xInset = (surfaceSize.x - scaledWidth) / 2;
verify(mTransaction, atLeastOnce()).setPosition(mirroredSurface, xInset, 0);
mockSession.finishMocking();
}
private class TestToken extends Binder {
}
/**
* Creates a WindowToken associated with the default task DisplayArea, in order for that
* DisplayArea to be mirrored.
*/
private IBinder setUpDefaultTaskDisplayAreaWindowToken() {
// GIVEN MediaProjection has already initialized the WindowToken of the DisplayArea to
// mirror.
final IBinder tokenToMirror = new TestToken();
doReturn(tokenToMirror).when(mWm.mDisplayManagerInternal).getWindowTokenClientToMirror(
anyInt());
// GIVEN the default task display area is represented by the WindowToken.
spyOn(mWm.mWindowContextListenerController);
doReturn(mDefaultDisplay.getDefaultTaskDisplayArea()).when(
mWm.mWindowContextListenerController).getContainer(any());
return tokenToMirror;
}
/**
* SurfaceControl successfully creates a mirrored surface of the given size.
*/
private SurfaceControl surfaceControlMirrors(Point surfaceSize) {
// Do not set the parent, since the mirrored surface is the root of a new surface hierarchy.
SurfaceControl mirroredSurface = new SurfaceControl.Builder()
.setName("mirroredSurface")
.setBufferSize(surfaceSize.x, surfaceSize.y)
.setCallsite("mirrorSurface")
.build();
doReturn(mirroredSurface).when(() -> SurfaceControl.mirrorSurface(any()));
doReturn(surfaceSize).when(mWm.mDisplayManagerInternal).getDisplaySurfaceDefaultSize(
anyInt());
return mirroredSurface;
}
private void removeRootTaskTests(Runnable runnable) {
final TaskDisplayArea taskDisplayArea = mRootWindowContainer.getDefaultTaskDisplayArea();
final Task rootTask1 = taskDisplayArea.createRootTask(WINDOWING_MODE_FULLSCREEN,