Merge "[MediaProjection] Introduce hidden/visible callback."

This commit is contained in:
Mariia Sandrikova
2022-12-22 02:39:46 +00:00
committed by Android (Google) Code Review
14 changed files with 366 additions and 39 deletions

View File

@@ -25552,6 +25552,7 @@ package android.media.projection {
public abstract static class MediaProjection.Callback {
ctor public MediaProjection.Callback();
method public void onCapturedContentResize(int, int);
method public void onCapturedContentVisibilityChanged(boolean);
method public void onStop();
}

View File

@@ -1633,12 +1633,6 @@
"group": "WM_ERROR",
"at": "com\/android\/server\/wm\/WindowManagerService.java"
},
"-576580969": {
"message": "viewServerWindowCommand: bootFinished() failed.",
"level": "WARN",
"group": "WM_ERROR",
"at": "com\/android\/server\/wm\/WindowManagerService.java"
},
"-576070986": {
"message": "Performing post-rotate rotation after seamless rotation",
"level": "INFO",
@@ -1981,6 +1975,12 @@
"group": "WM_DEBUG_STATES",
"at": "com\/android\/server\/wm\/ActivityRecord.java"
},
"-254406860": {
"message": "Unable to tell MediaProjectionManagerService about visibility change on the active projection: %s",
"level": "ERROR",
"group": "WM_DEBUG_CONTENT_RECORDING",
"at": "com\/android\/server\/wm\/ContentRecorder.java"
},
"-251259736": {
"message": "No longer freezing: %s",
"level": "VERBOSE",
@@ -4177,12 +4177,6 @@
"group": "WM_DEBUG_REMOTE_ANIMATIONS",
"at": "com\/android\/server\/wm\/RemoteAnimationController.java"
},
"1903353011": {
"message": "notifyAppStopped: %s",
"level": "VERBOSE",
"group": "WM_DEBUG_ADD_REMOVE",
"at": "com\/android\/server\/wm\/ActivityRecord.java"
},
"1912291550": {
"message": "Sleep still waiting to pause %s",
"level": "VERBOSE",

View File

@@ -20,4 +20,5 @@ package android.media.projection;
oneway interface IMediaProjectionCallback {
void onStop();
void onCapturedContentResize(int width, int height);
void onCapturedContentVisibilityChanged(boolean isVisible);
}

View File

@@ -48,7 +48,11 @@ interface IMediaProjectionManager {
void notifyActiveProjectionCapturedContentResized(int width, int height);
@JavaPassthrough(annotation = "@android.annotation.RequiresPermission(android.Manifest"
+ ".permission.MANAGE_MEDIA_PROJECTION)")
+ ".permission.MANAGE_MEDIA_PROJECTION)")
void notifyActiveProjectionCapturedContentVisibilityChanged(boolean isVisible);
@JavaPassthrough(annotation = "@android.annotation.RequiresPermission(android.Manifest"
+ ".permission.MANAGE_MEDIA_PROJECTION)")
void addCallback(IMediaProjectionWatcherCallback callback);
@JavaPassthrough(annotation = "@android.annotation.RequiresPermission(android.Manifest"

View File

@@ -66,13 +66,20 @@ public final class MediaProjection {
}
}
/** Register a listener to receive notifications about when the {@link
* MediaProjection} changes state.
/**
* Register a listener to receive notifications about when the {@link MediaProjection} or
* captured content changes state.
* <p>
* The callback should be registered before invoking
* {@link #createVirtualDisplay(String, int, int, int, int, Surface, VirtualDisplay.Callback,
* Handler)}
* to ensure that any notifications on the callback are not missed.
* </p>
*
* @param callback The callback to call.
* @param handler The handler on which the callback should be invoked, or
* null if the callback should be invoked on the calling thread's looper.
*
* @param handler The handler on which the callback should be invoked, or
* null if the callback should be invoked on the calling thread's looper.
* @throws IllegalArgumentException If the given callback is null.
* @see #unregisterCallback
*/
public void registerCallback(Callback callback, Handler handler) {
@@ -85,10 +92,11 @@ public final class MediaProjection {
mCallbacks.put(callback, new CallbackRecord(callback, handler));
}
/** Unregister a MediaProjection listener.
/**
* Unregister a {@link MediaProjection} listener.
*
* @param callback The callback to unregister.
*
* @throws IllegalArgumentException If the given callback is null.
* @see #registerCallback
*/
public void unregisterCallback(Callback callback) {
@@ -283,6 +291,34 @@ public final class MediaProjection {
* }</pre>
*/
public void onCapturedContentResize(int width, int height) { }
/**
* Indicates the visibility of the captured region has changed. Called immediately after
* capture begins with the initial visibility state, and when visibility changes. Provides
* the app with accurate state for presenting its own UI. The application can take advantage
* of this by showing or hiding the captured content, based on if the captured region is
* currently visible to the user.
* <p>
* For example, if the user elected to capture a single app (from the activity shown from
* {@link MediaProjectionManager#createScreenCaptureIntent()}), the callback may be
* triggered for the following reasons:
* <ul>
* <li>
* The captured region may become visible ({@code isVisible} with value
* {@code true}), because the captured app is at least partially visible. This may
* happen if the captured app was previously covered by another app. The other app
* moves to show at least some portion of the captured app.
* </li>
* <li>
* The captured region may become invisible ({@code isVisible} with value
* {@code false}) if it is entirely hidden. This may happen if the captured app is
* entirely covered by another app, or the user navigates away from the captured
* app.
* </li>
* </ul>
* </p>
*/
public void onCapturedContentVisibilityChanged(boolean isVisible) { }
}
private final class MediaProjectionCallback extends IMediaProjectionCallback.Stub {
@@ -299,6 +335,13 @@ public final class MediaProjection {
cbr.onCapturedContentResize(width, height);
}
}
@Override
public void onCapturedContentVisibilityChanged(boolean isVisible) {
for (CallbackRecord cbr : mCallbacks.values()) {
cbr.onCapturedContentVisibilityChanged(isVisible);
}
}
}
private final static class CallbackRecord {
@@ -322,5 +365,9 @@ public final class MediaProjection {
public void onCapturedContentResize(int width, int height) {
mHandler.post(() -> mCallback.onCapturedContentResize(width, height));
}
public void onCapturedContentVisibilityChanged(boolean isVisible) {
mHandler.post(() -> mCallback.onCapturedContentVisibilityChanged(isVisible));
}
}
}

View File

@@ -11510,6 +11510,11 @@ public class AudioService extends IAudioService.Stub
public void onCapturedContentResize(int width, int height) {
// Ignore resize of the captured content.
}
@Override
public void onCapturedContentVisibilityChanged(boolean isVisible) {
// Ignore visibility changes of the captured content.
}
};
UnregisterOnStopCallback mProjectionCallback;

View File

@@ -621,6 +621,13 @@ public class VirtualDisplayAdapter extends DisplayAdapter {
// expect), and there will still be letterboxing on the output content since the
// Surface and VirtualDisplay would then have different aspect ratios.
}
@Override
public void onCapturedContentVisibilityChanged(boolean isVisible) {
// Do nothing when we tell the client that the content has a visibility change - it is
// up to them to decide to pause recording, and update their own UI, depending on their
// use case.
}
}
@VisibleForTesting

View File

@@ -370,6 +370,26 @@ public final class MediaProjectionManagerService extends SystemService
}
}
@Override
public void notifyActiveProjectionCapturedContentVisibilityChanged(boolean isVisible) {
if (mContext.checkCallingOrSelfPermission(Manifest.permission.MANAGE_MEDIA_PROJECTION)
!= PackageManager.PERMISSION_GRANTED) {
throw new SecurityException("Requires MANAGE_MEDIA_PROJECTION in order to notify "
+ "on captured content resize");
}
if (!isValidMediaProjection(mProjectionGrant)) {
return;
}
final long token = Binder.clearCallingIdentity();
try {
if (mProjectionGrant != null && mCallbackDelegate != null) {
mCallbackDelegate.dispatchVisibilityChanged(mProjectionGrant, isVisible);
}
} finally {
Binder.restoreCallingIdentity(token);
}
}
@Override //Binder call
public void addCallback(final IMediaProjectionWatcherCallback callback) {
if (mContext.checkCallingPermission(Manifest.permission.MANAGE_MEDIA_PROJECTION)
@@ -750,8 +770,9 @@ public final class MediaProjectionManagerService extends SystemService
public void dispatchResize(MediaProjection projection, int width, int height) {
if (projection == null) {
Slog.e(TAG, "Tried to dispatch stop notification for a null media projection."
+ " Ignoring!");
Slog.e(TAG,
"Tried to dispatch resize notification for a null media projection. "
+ "Ignoring!");
return;
}
synchronized (mLock) {
@@ -774,6 +795,36 @@ public final class MediaProjectionManagerService extends SystemService
// is for passing along if recording is still ongoing or not.
}
}
public void dispatchVisibilityChanged(MediaProjection projection, boolean isVisible) {
if (projection == null) {
Slog.e(TAG,
"Tried to dispatch visibility changed notification for a null media "
+ "projection. Ignoring!");
return;
}
synchronized (mLock) {
// TODO(b/249827847) Currently the service assumes there is only one projection
// at once - need to find the callback for the given projection, when there are
// multiple sessions.
for (IMediaProjectionCallback callback : mClientCallbacks.values()) {
mHandler.post(() -> {
try {
// Notify every callback the client has registered for a particular
// MediaProjection instance.
callback.onCapturedContentVisibilityChanged(isVisible);
} catch (RemoteException e) {
Slog.w(TAG,
"Failed to notify media projection has captured content "
+ "visibility change to "
+ isVisible, e);
}
});
}
// Do not need to notify watcher callback about visibility changes, since watcher
// callback is for passing along if recording is still ongoing or not.
}
}
}
private static final class WatcherStartCallback implements Runnable {

View File

@@ -211,6 +211,7 @@ final class ContentRecorder implements WindowContainerListener {
* Stops recording on this DisplayContent, and updates the session details.
*/
void stopRecording() {
unregisterListener();
if (mRecordedSurface != null) {
// Do not wait for the mirrored surface to be garbage collected, but clean up
// immediately.
@@ -227,7 +228,7 @@ final class ContentRecorder implements WindowContainerListener {
* Ensure recording does not fall back to the display stack; ensure the recording is stopped
* and the client notified by tearing down the virtual display.
*/
void stopMediaProjection() {
private void stopMediaProjection() {
ProtoLog.v(WM_DEBUG_CONTENT_RECORDING,
"Stop MediaProjection on virtual display %d", mDisplayContent.getDisplayId());
if (mMediaProjectionManager != null) {
@@ -247,6 +248,16 @@ final class ContentRecorder implements WindowContainerListener {
null, mDisplayContent.mWmService);
}
private void unregisterListener() {
Task recordedTask = mRecordedWindowContainer != null
? mRecordedWindowContainer.asTask() : null;
if (recordedTask == null || !isRecordingContentTask()) {
return;
}
recordedTask.unregisterWindowContainerListener(this);
mRecordedWindowContainer = null;
}
/**
* Start recording to this DisplayContent if it does not have its own content. Captures the
* content of a WindowContainer indicated by a WindowToken. If unable to start recording, falls
@@ -301,6 +312,13 @@ final class ContentRecorder implements WindowContainerListener {
// Retrieve the size of the DisplayArea to mirror.
updateMirroredSurface(transaction, mRecordedWindowContainer.getBounds(), surfaceSize);
// Notify the client about the visibility of the mirrored region, now that we have begun
// capture.
if (mContentRecordingSession.getContentToRecord() == RECORD_CONTENT_TASK) {
mMediaProjectionManager.notifyActiveProjectionCapturedContentVisibilityChanged(
mRecordedWindowContainer.asTask().isVisibleRequested());
}
// No need to clean up. In SurfaceFlinger, parents hold references to their children. The
// mirrored SurfaceControl is alive since the parent DisplayContent SurfaceControl is
// holding a reference to it. Therefore, the mirrored SurfaceControl will be cleaned up
@@ -389,6 +407,7 @@ final class ContentRecorder implements WindowContainerListener {
*/
private void handleStartRecordingFailed() {
final boolean shouldExitTaskRecording = isRecordingContentTask();
unregisterListener();
clearContentRecordingSession();
if (shouldExitTaskRecording) {
// Clean up the cached session first to ensure recording doesn't re-start, since
@@ -478,12 +497,7 @@ final class ContentRecorder implements WindowContainerListener {
"Recorded task is removed, so stop recording on display %d",
mDisplayContent.getDisplayId());
Task recordedTask = mRecordedWindowContainer != null
? mRecordedWindowContainer.asTask() : null;
if (recordedTask == null || !isRecordingContentTask()) {
return;
}
recordedTask.unregisterWindowContainerListener(this);
unregisterListener();
// Stop mirroring and teardown.
clearContentRecordingSession();
// Clean up the cached session first to ensure recording doesn't re-start, since
@@ -501,9 +515,20 @@ final class ContentRecorder implements WindowContainerListener {
mLastOrientation = mergedOverrideConfiguration.orientation;
}
// WindowContainerListener
@Override
public void onVisibleRequestedChanged(boolean isVisibleRequested) {
// Check still recording just to be safe.
if (isCurrentlyRecording() && mLastRecordedBounds != null) {
mMediaProjectionManager.notifyActiveProjectionCapturedContentVisibilityChanged(
isVisibleRequested);
}
}
@VisibleForTesting interface MediaProjectionManagerWrapper {
void stopActiveProjection();
void notifyActiveProjectionCapturedContentResized(int width, int height);
void notifyActiveProjectionCapturedContentVisibilityChanged(boolean isVisible);
}
private static final class RemoteMediaProjectionManagerWrapper implements
@@ -543,6 +568,23 @@ final class ContentRecorder implements WindowContainerListener {
}
}
@Override
public void notifyActiveProjectionCapturedContentVisibilityChanged(boolean isVisible) {
fetchMediaProjectionManager();
if (mIMediaProjectionManager == null) {
return;
}
try {
mIMediaProjectionManager.notifyActiveProjectionCapturedContentVisibilityChanged(
isVisible);
} catch (RemoteException e) {
ProtoLog.e(WM_DEBUG_CONTENT_RECORDING,
"Unable to tell MediaProjectionManagerService about visibility change on "
+ "the active projection: %s",
e);
}
}
private void fetchMediaProjectionManager() {
if (mIMediaProjectionManager != null) {
return;

View File

@@ -6504,15 +6504,6 @@ class DisplayContent extends RootDisplayArea implements WindowManagerPolicy.Disp
}
}
/**
* The MediaProjection instance is torn down.
*/
@VisibleForTesting void stopMediaProjection() {
if (mContentRecorder != null) {
mContentRecorder.stopMediaProjection();
}
}
/**
* Sets the incoming recording session. Should only be used when starting to record on
* this display; stopping recording is handled separately when the display is destroyed.

View File

@@ -1305,6 +1305,11 @@ class WindowContainer<E extends WindowContainer> extends ConfigurationContainer<
if (parent != null) {
parent.onChildVisibleRequestedChanged(this);
}
// Notify listeners about visibility change.
for (int i = mListeners.size() - 1; i >= 0; --i) {
mListeners.get(i).onVisibleRequestedChanged(mVisibleRequested);
}
return true;
}

View File

@@ -27,4 +27,13 @@ interface WindowContainerListener extends ConfigurationContainerListener {
/** Called when {@link WindowContainer#removeImmediately()} is invoked. */
default void onRemoved() {}
/**
* Only invoked if the child successfully requested a visibility change.
*
* @param isVisibleRequested The current {@link WindowContainer#isVisibleRequested()} of this
* {@link WindowContainer} (not of the child).
* @see WindowContainer#onChildVisibleRequestedChanged(WindowContainer)
*/
default void onVisibleRequestedChanged(boolean isVisibleRequested) { }
}

View File

@@ -16,7 +16,6 @@
package com.android.server.wm;
import static android.content.res.Configuration.ORIENTATION_PORTRAIT;
import static android.hardware.display.DisplayManager.VIRTUAL_DISPLAY_FLAG_AUTO_MIRROR;
import static android.view.Display.INVALID_DISPLAY;
@@ -280,6 +279,64 @@ public class ContentRecorderTests extends WindowTestsBase {
recordedWidth, recordedHeight);
}
@Test
public void testStartRecording_notifiesCallback() {
// WHEN a recording is ongoing.
mContentRecorder.setContentRecordingSession(mTaskSession);
mContentRecorder.updateRecording();
assertThat(mContentRecorder.isCurrentlyRecording()).isTrue();
// THEN the visibility change callback is notified.
verify(mMediaProjectionManagerWrapper)
.notifyActiveProjectionCapturedContentVisibilityChanged(true);
}
@Test
public void testOnVisibleRequestedChanged_notifiesCallback() {
// WHEN a recording is ongoing.
mContentRecorder.setContentRecordingSession(mTaskSession);
mContentRecorder.updateRecording();
assertThat(mContentRecorder.isCurrentlyRecording()).isTrue();
// WHEN the child requests a visibility change.
boolean isVisibleRequested = true;
mContentRecorder.onVisibleRequestedChanged(isVisibleRequested);
// THEN the visibility change callback is notified.
verify(mMediaProjectionManagerWrapper, atLeastOnce())
.notifyActiveProjectionCapturedContentVisibilityChanged(isVisibleRequested);
// WHEN the child requests a visibility change.
isVisibleRequested = false;
mContentRecorder.onVisibleRequestedChanged(isVisibleRequested);
// THEN the visibility change callback is notified.
verify(mMediaProjectionManagerWrapper)
.notifyActiveProjectionCapturedContentVisibilityChanged(isVisibleRequested);
}
@Test
public void testOnVisibleRequestedChanged_noRecording_doesNotNotifyCallback() {
// WHEN a recording is not ongoing.
assertThat(mContentRecorder.isCurrentlyRecording()).isFalse();
// WHEN the child requests a visibility change.
boolean isVisibleRequested = true;
mContentRecorder.onVisibleRequestedChanged(isVisibleRequested);
// THEN the visibility change callback is not notified.
verify(mMediaProjectionManagerWrapper, never())
.notifyActiveProjectionCapturedContentVisibilityChanged(isVisibleRequested);
// WHEN the child requests a visibility change.
isVisibleRequested = false;
mContentRecorder.onVisibleRequestedChanged(isVisibleRequested);
// THEN the visibility change callback is not notified.
verify(mMediaProjectionManagerWrapper, never())
.notifyActiveProjectionCapturedContentVisibilityChanged(isVisibleRequested);
}
@Test
public void testPauseRecording_pausesRecording() {
mContentRecorder.setContentRecordingSession(mDisplaySession);

View File

@@ -56,6 +56,8 @@ import static com.android.server.wm.WindowContainer.AnimationFlags.TRANSITION;
import static com.android.server.wm.WindowContainer.POSITION_BOTTOM;
import static com.android.server.wm.WindowContainer.POSITION_TOP;
import static com.google.common.truth.Truth.assertThat;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;
@@ -659,6 +661,111 @@ public class WindowContainerTests extends WindowTestsBase {
assertEquals(SCREEN_ORIENTATION_PORTRAIT, root.getOrientation());
}
@Test
public void testSetVisibleRequested() {
final TestWindowContainer root = spy(new TestWindowContainerBuilder(mWm).setLayer(
0).build());
assertThat(root.isVisibleRequested()).isFalse();
final TestWindowContainerListener listener = new TestWindowContainerListener();
root.registerWindowContainerListener(listener);
assertThat(root.setVisibleRequested(/* isVisible= */ false)).isFalse();
assertThat(root.isVisibleRequested()).isFalse();
assertThat(root.setVisibleRequested(/* isVisible= */ true)).isTrue();
assertThat(root.isVisibleRequested()).isTrue();
assertThat(listener.mIsVisibleRequested).isTrue();
}
@Test
public void testSetVisibleRequested_childRequestsVisible() {
final TestWindowContainer root = spy(new TestWindowContainerBuilder(mWm).setLayer(
0).build());
final TestWindowContainer child1 = root.addChildWindow();
assertThat(child1.isVisibleRequested()).isFalse();
final TestWindowContainerListener listener = new TestWindowContainerListener();
root.registerWindowContainerListener(listener);
// Hidden root and child request hidden.
assertThat(root.setVisibleRequested(/* isVisible= */ false)).isFalse();
assertThat(listener.mIsVisibleRequested).isFalse();
assertThat(child1.isVisibleRequested()).isFalse();
// Child requests to be visible, so child and root request visible.
assertThat(child1.setVisibleRequested(/* isVisible= */ true)).isTrue();
assertThat(root.isVisibleRequested()).isTrue();
assertThat(listener.mIsVisibleRequested).isTrue();
assertThat(child1.isVisibleRequested()).isTrue();
// Visible request didn't change.
assertThat(child1.setVisibleRequested(/* isVisible= */ true)).isFalse();
verify(root, times(2)).onChildVisibleRequestedChanged(child1);
}
@Test
public void testSetVisibleRequested_childRequestsHidden() {
final TestWindowContainer root = spy(new TestWindowContainerBuilder(mWm).setLayer(
0).build());
final TestWindowContainer child1 = root.addChildWindow();
assertThat(child1.isVisibleRequested()).isFalse();
final TestWindowContainerListener listener = new TestWindowContainerListener();
root.registerWindowContainerListener(listener);
// Root and child requests visible.
assertThat(root.setVisibleRequested(/* isVisible= */ true)).isTrue();
assertThat(listener.mIsVisibleRequested).isTrue();
assertThat(child1.setVisibleRequested(/* isVisible= */ true)).isTrue();
assertThat(child1.isVisibleRequested()).isTrue();
// Child requests hidden, so child and root request hidden.
assertThat(child1.setVisibleRequested(/* isVisible= */ false)).isTrue();
assertThat(root.isVisibleRequested()).isFalse();
assertThat(listener.mIsVisibleRequested).isFalse();
assertThat(child1.isVisibleRequested()).isFalse();
// Visible request didn't change.
assertThat(child1.setVisibleRequested(/* isVisible= */ false)).isFalse();
verify(root, times(3)).onChildVisibleRequestedChanged(child1);
}
@Test
public void testOnChildVisibleRequestedChanged_bothVisible() {
final TestWindowContainer root = spy(new TestWindowContainerBuilder(mWm).setLayer(
0).build());
final TestWindowContainer child1 = root.addChildWindow();
// Child and root request visible.
assertThat(root.setVisibleRequested(/* isVisible= */ true)).isTrue();
assertThat(child1.setVisibleRequested(/* isVisible= */ true)).isTrue();
// Visible request already updated on root when child requested.
assertThat(root.onChildVisibleRequestedChanged(child1)).isFalse();
}
@Test
public void testOnChildVisibleRequestedChanged_childVisible() {
final TestWindowContainer root = spy(new TestWindowContainerBuilder(mWm).setLayer(
0).build());
final TestWindowContainer child1 = root.addChildWindow();
assertThat(root.setVisibleRequested(/* isVisible= */ false)).isFalse();
assertThat(child1.setVisibleRequested(/* isVisible= */ true)).isTrue();
// Visible request already updated on root when child requested.
assertThat(root.onChildVisibleRequestedChanged(child1)).isFalse();
}
@Test
public void testOnChildVisibleRequestedChanged_childHidden() {
final TestWindowContainer root = spy(new TestWindowContainerBuilder(mWm).setLayer(
0).build());
final TestWindowContainer child1 = root.addChildWindow();
assertThat(root.setVisibleRequested(/* isVisible= */ false)).isFalse();
assertThat(child1.setVisibleRequested(/* isVisible= */ false)).isFalse();
// Visible request did not change.
assertThat(root.onChildVisibleRequestedChanged(child1)).isFalse();
}
@Test
public void testSetOrientation() {
final TestWindowContainer root = spy(new TestWindowContainerBuilder(mWm).build());
@@ -1656,6 +1763,7 @@ public class WindowContainerTests extends WindowTestsBase {
private static class TestWindowContainerListener implements WindowContainerListener {
private Configuration mConfiguration = new Configuration();
private DisplayContent mDisplayContent;
private boolean mIsVisibleRequested;
@Override
public void onRequestedOverrideConfigurationChanged(Configuration overrideConfiguration) {
@@ -1666,5 +1774,10 @@ public class WindowContainerTests extends WindowTestsBase {
public void onDisplayChanged(DisplayContent dc) {
mDisplayContent = dc;
}
@Override
public void onVisibleRequestedChanged(boolean isVisibleRequested) {
mIsVisibleRequested = isVisibleRequested;
}
}
}