Adds a model & view layer for long screenshot tiles

An ImageTile wraps an Image with an attached HardwareBuffer,
as received from an ImageReader as used by scroll capture.

ImageTileSet provides a drawable to present the result on-screen,
TiledImageDrawable maintains a display list which is updated when
notified of a change. This object can also produce a Bitmap as
output for export to a compressed image.

This change also tightens up the ScrollCaptureClient interface
to internally define tile size and limits. Instead of accepted
arbitrary rectangles, accept only a single y-coordinate to
specify the verical position of the tile to capture.

Bug: 174571735
Bug: 148519342
Test: manually, take long screenshot
Change-Id: I81032766c33396f51cb117f87a4d65288990f66d
This commit is contained in:
Mark Renouf
2020-12-09 17:33:54 -05:00
parent f167275f60
commit 3277394936
6 changed files with 479 additions and 94 deletions

View File

@@ -0,0 +1,113 @@
/*
* Copyright (C) 2020 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.systemui.screenshot;
import static android.graphics.ColorSpace.Named.SRGB;
import static java.util.Objects.requireNonNull;
import android.graphics.Bitmap;
import android.graphics.ColorSpace;
import android.graphics.RecordingCanvas;
import android.graphics.Rect;
import android.graphics.RenderNode;
import android.media.Image;
/**
* Holds a hardware image, coordinates and render node to draw the tile. The tile manages clipping
* and dimensions. The tile must be drawn translated to the correct target position:
* <pre>
* ImageTile tile = getTile();
* canvas.save();
* canvas.translate(tile.getLeft(), tile.getTop());
* canvas.drawRenderNode(tile.getDisplayList());
* canvas.restore();
* </pre>
*/
class ImageTile implements AutoCloseable {
private final Image mImage;
private final Rect mLocation;
private RenderNode mNode;
private static final ColorSpace COLOR_SPACE = ColorSpace.get(SRGB);
/**
* Create an image tile from the given image.
*
* @param image an image containing a hardware buffer
* @param location the captured area represented by image tile (virtual coordinates)
*/
ImageTile(Image image, Rect location) {
mImage = requireNonNull(image, "image");
mLocation = location;
requireNonNull(mImage.getHardwareBuffer(), "image must be a hardware image");
}
RenderNode getDisplayList() {
if (mNode == null) {
mNode = new RenderNode("Tile{" + Integer.toHexString(mImage.hashCode()) + "}");
}
if (mNode.hasDisplayList()) {
return mNode;
}
final int w = Math.min(mImage.getWidth(), mLocation.width());
final int h = Math.min(mImage.getHeight(), mLocation.height());
mNode.setPosition(0, 0, w, h);
RecordingCanvas canvas = mNode.beginRecording(w, h);
Rect rect = new Rect(0, 0, w, h);
canvas.save();
canvas.clipRect(0, 0, mLocation.right, mLocation.bottom);
canvas.drawBitmap(Bitmap.wrapHardwareBuffer(mImage.getHardwareBuffer(), COLOR_SPACE),
0, 0, null);
canvas.restore();
mNode.endRecording();
return mNode;
}
Rect getLocation() {
return mLocation;
}
int getLeft() {
return mLocation.left;
}
int getTop() {
return mLocation.top;
}
int getRight() {
return mLocation.right;
}
int getBottom() {
return mLocation.bottom;
}
@Override
public void close() {
mImage.close();
mNode.discardDisplayList();
}
@Override
public String toString() {
return "{location=" + mLocation + ", source=" + mImage
+ ", buffer=" + mImage.getHardwareBuffer() + "}";
}
}

View File

@@ -0,0 +1,163 @@
/*
* Copyright (C) 2020 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.systemui.screenshot;
import android.graphics.Bitmap;
import android.graphics.HardwareRenderer;
import android.graphics.RecordingCanvas;
import android.graphics.Rect;
import android.graphics.RenderNode;
import android.graphics.drawable.Drawable;
import androidx.annotation.UiThread;
import java.util.ArrayList;
import java.util.List;
/**
* Owns a series of partial screen captures (tiles).
* <p>
* To display on-screen, use {@link #getDrawable()}.
*/
@UiThread
class ImageTileSet {
private static final String TAG = "ImageTileSet";
interface OnBoundsChangedListener {
/**
* Reports an update to the bounding box that contains all active tiles. These are virtual
* (capture) coordinates which can be either negative or positive.
*/
void onBoundsChanged(int left, int top, int right, int bottom);
}
interface OnContentChangedListener {
/**
* Mark as dirty and rebuild display list.
*/
void onContentChanged();
}
private final List<ImageTile> mTiles = new ArrayList<>();
private final Rect mBounds = new Rect();
private OnContentChangedListener mOnContentChangedListener;
private OnBoundsChangedListener mOnBoundsChangedListener;
void setOnBoundsChangedListener(OnBoundsChangedListener listener) {
mOnBoundsChangedListener = listener;
}
void setOnContentChangedListener(OnContentChangedListener listener) {
mOnContentChangedListener = listener;
}
void addTile(ImageTile tile) {
final Rect newBounds = new Rect(mBounds);
final Rect newRect = tile.getLocation();
mTiles.add(tile);
newBounds.union(newRect);
if (!newBounds.equals(mBounds)) {
mBounds.set(newBounds);
if (mOnBoundsChangedListener != null) {
mOnBoundsChangedListener.onBoundsChanged(
newBounds.left, newBounds.top, newBounds.right, newBounds.bottom);
}
}
if (mOnContentChangedListener != null) {
mOnContentChangedListener.onContentChanged();
}
}
/**
* Returns a drawable to paint the combined contents of the tiles. Drawable dimensions are
* zero-based and map directly to {@link #getLeft()}, {@link #getTop()}, {@link #getRight()},
* and {@link #getBottom()} which are dimensions relative to the capture start position
* (positive or negative).
*
* @return a drawable to display the image content
*/
Drawable getDrawable() {
return new TiledImageDrawable(this);
}
boolean isEmpty() {
return mTiles.isEmpty();
}
int size() {
return mTiles.size();
}
ImageTile get(int i) {
return mTiles.get(i);
}
Bitmap toBitmap() {
if (mTiles.isEmpty()) {
return null;
}
final RenderNode output = new RenderNode("Bitmap Export");
output.setPosition(0, 0, getWidth(), getHeight());
RecordingCanvas canvas = output.beginRecording();
canvas.translate(-getLeft(), -getTop());
for (ImageTile tile : mTiles) {
canvas.save();
canvas.translate(tile.getLeft(), tile.getTop());
canvas.drawRenderNode(tile.getDisplayList());
canvas.restore();
}
output.endRecording();
return HardwareRenderer.createHardwareBitmap(output, getWidth(), getHeight());
}
int getLeft() {
return mBounds.left;
}
int getTop() {
return mBounds.top;
}
int getRight() {
return mBounds.right;
}
int getBottom() {
return mBounds.bottom;
}
int getWidth() {
return mBounds.width();
}
int getHeight() {
return mBounds.height();
}
void clear() {
mBounds.set(0, 0, 0, 0);
mTiles.forEach(ImageTile::close);
mTiles.clear();
if (mOnBoundsChangedListener != null) {
mOnBoundsChangedListener.onBoundsChanged(0, 0, 0, 0);
}
if (mOnContentChangedListener != null) {
mOnContentChangedListener.onContentChanged();
}
}
}

View File

@@ -18,6 +18,7 @@ package com.android.systemui.screenshot;
import static com.android.systemui.screenshot.LogConfig.DEBUG_SCROLL;
import static java.lang.Math.min;
import static java.util.Objects.requireNonNull;
import android.annotation.UiContext;
@@ -44,15 +45,20 @@ import java.util.function.Consumer;
import javax.inject.Inject;
/**
* High level interface to scroll capture API.
* High(er) level interface to scroll capture API.
*/
public class ScrollCaptureClient {
private static final int TILE_SIZE_PX_MAX = 4 * (1024 * 1024);
private static final int TILES_PER_PAGE = 2; // increase once b/174571735 is addressed
private static final int MAX_PAGES = 5;
private static final int MAX_IMAGE_COUNT = MAX_PAGES * TILES_PER_PAGE;
@VisibleForTesting
static final int MATCH_ANY_TASK = ActivityTaskManager.INVALID_TASK_ID;
private static final String TAG = LogConfig.logTag(ScrollCaptureClient.class);
/**
* A connection to a remote window. Starts a capture session.
*/
@@ -60,13 +66,10 @@ public class ScrollCaptureClient {
/**
* Session start should be deferred until UI is active because of resource allocation and
* potential visible side effects in the target window.
*
* @param maxBuffers the maximum number of buffers (tiles) that may be in use at one
* time, tiles are not cached anywhere so set this to a large enough
* number to retain offscreen content until it is no longer needed
* @param sessionConsumer listener to receive the session once active
*/
void start(int maxBuffers, Consumer<Session> sessionConsumer);
void start(Consumer<Session> sessionConsumer);
/**
* Close the connection.
@@ -100,26 +103,33 @@ public class ScrollCaptureClient {
*/
interface Session {
/**
* Request the given horizontal strip. Values are y-coordinates in captured space, relative
* to start position.
* Request an image tile at the given position, from top, to top + {@link #getTileHeight()},
* and from left 0, to {@link #getPageWidth()}
*
* @param contentRect the area to capture, in content rect space, relative to scroll-bounds
* @param top the top (y) position of the tile to capture, in content rect space
* @param consumer listener to be informed of the result
*/
void requestTile(Rect contentRect, Consumer<CaptureResult> consumer);
void requestTile(int top, Consumer<CaptureResult> consumer);
/**
* End the capture session, return the target app to original state. The returned
* stage must be waited for to complete to allow the target app a chance to restore to
* original state before becoming visible.
* Returns the maximum number of tiles which may be requested and retained without
* being {@link Image#close() closed}.
*
* @return a stage presenting the session shutdown
* @return the maximum number of open tiles allowed
*/
int getMaxTiles();
int getTileHeight();
int getPageHeight();
int getPageWidth();
/**
* End the capture session, return the target app to original state. The listener
* will be called when the target app is ready to before visible and interactive.
*/
void end(Runnable listener);
int getMaxTileHeight();
int getMaxTileWidth();
}
private final IWindowManager mWindowManagerService;
@@ -131,6 +141,12 @@ public class ScrollCaptureClient {
mWindowManagerService = windowManagerService;
}
/**
* Set the window token for the screenshot window/ This is required to avoid targeting our
* window or any above it.
*
* @param token the windowToken of the screenshot window
*/
public void setHostWindowToken(IBinder token) {
mHostWindowToken = token;
}
@@ -176,6 +192,8 @@ public class ScrollCaptureClient {
private ImageReader mReader;
private Rect mScrollBounds;
private int mTileHeight;
private int mTileWidth;
private Rect mRequestRect;
private boolean mStarted;
@@ -197,6 +215,15 @@ public class ScrollCaptureClient {
mScrollBounds = scrollBounds;
mConnectionConsumer.accept(this);
mConnectionConsumer = null;
int pxPerPage = mScrollBounds.width() * mScrollBounds.height();
int pxPerTile = min(TILE_SIZE_PX_MAX, (pxPerPage / TILES_PER_PAGE));
mTileWidth = mScrollBounds.width();
mTileHeight = pxPerTile / mScrollBounds.width();
if (DEBUG_SCROLL) {
Log.d(TAG, "scrollBounds: " + mScrollBounds);
Log.d(TAG, "tile dimen: " + mTileWidth + "x" + mTileHeight);
}
}
@Override
@@ -257,24 +284,19 @@ public class ScrollCaptureClient {
// ScrollCaptureController.Connection
// -> Error handling: BiConsumer<Session, Throwable> ?
@Override
public void start(int maxBufferCount, Consumer<Session> sessionConsumer) {
public void start(Consumer<Session> sessionConsumer) {
if (DEBUG_SCROLL) {
Log.d(TAG, "start(maxBufferCount=" + maxBufferCount
+ ", sessionConsumer=" + sessionConsumer + ")");
Log.d(TAG, "start(sessionConsumer=" + sessionConsumer + ")");
}
mReader = ImageReader.newInstance(mScrollBounds.width(), mScrollBounds.height(),
PixelFormat.RGBA_8888, maxBufferCount, HardwareBuffer.USAGE_GPU_SAMPLED_IMAGE);
mReader = ImageReader.newInstance(mTileWidth, mTileHeight, PixelFormat.RGBA_8888,
MAX_IMAGE_COUNT, HardwareBuffer.USAGE_GPU_SAMPLED_IMAGE);
mSessionConsumer = sessionConsumer;
try {
mConnection.startCapture(mReader.getSurface());
mStarted = true;
} catch (RemoteException e) {
Log.w(TAG, "should not be happening :-(");
// ?
//mSessionListener.onError(e);
//mSessionListener = null;
Log.w(TAG, "Failed to start", e);
}
}
@@ -307,27 +329,36 @@ public class ScrollCaptureClient {
}
@Override
public int getMaxTileHeight() {
public int getPageHeight() {
return mScrollBounds.height();
}
@Override
public int getMaxTileWidth() {
public int getPageWidth() {
return mScrollBounds.width();
}
@Override
public void requestTile(Rect contentRect, Consumer<CaptureResult> consumer) {
public int getTileHeight() {
return mTileHeight;
}
@Override
public int getMaxTiles() {
return MAX_IMAGE_COUNT;
}
@Override
public void requestTile(int top, Consumer<CaptureResult> consumer) {
if (DEBUG_SCROLL) {
Log.d(TAG, "requestTile(contentRect=" + contentRect + "consumer=" + consumer + ")");
Log.d(TAG, "requestTile(top=" + top + ", consumer=" + consumer + ")");
}
mRequestRect = new Rect(contentRect);
mRequestRect = new Rect(0, top, mTileWidth, top + mTileHeight);
mResultConsumer = consumer;
try {
mConnection.requestImage(mRequestRect);
} catch (RemoteException e) {
Log.e(TAG, "Caught remote exception from requestImage", e);
// ?
}
}

View File

@@ -16,16 +16,9 @@
package com.android.systemui.screenshot;
import static android.graphics.ColorSpace.Named.SRGB;
import android.content.Context;
import android.content.Intent;
import android.graphics.Bitmap;
import android.graphics.Canvas;
import android.graphics.ColorSpace;
import android.graphics.Picture;
import android.graphics.Rect;
import android.media.Image;
import android.net.Uri;
import android.os.UserHandle;
import android.util.Log;
@@ -46,6 +39,8 @@ import java.util.function.Consumer;
public class ScrollCaptureController {
private static final String TAG = "ScrollCaptureController";
private static final boolean USE_TILED_IMAGE = false;
public static final int MAX_PAGES = 5;
public static final int MAX_HEIGHT = 12000;
@@ -55,7 +50,7 @@ public class ScrollCaptureController {
private final Executor mUiExecutor;
private final Executor mBgExecutor;
private final ImageExporter mImageExporter;
private Picture mPicture;
private final ImageTileSet mImageTileSet;
public ScrollCaptureController(Context context, Connection connection, Executor uiExecutor,
Executor bgExecutor, ImageExporter exporter) {
@@ -64,6 +59,7 @@ public class ScrollCaptureController {
mUiExecutor = uiExecutor;
mBgExecutor = bgExecutor;
mImageExporter = exporter;
mImageTileSet = new ImageTileSet();
}
/**
@@ -72,50 +68,50 @@ public class ScrollCaptureController {
* @param after action to take after the flow is complete
*/
public void run(final Runnable after) {
mConnection.start(MAX_PAGES, (session) -> startCapture(session, after));
mConnection.start((session) -> startCapture(session, after));
}
private void startCapture(Session session, final Runnable onDismiss) {
Rect requestRect = new Rect(0, 0,
session.getMaxTileWidth(), session.getMaxTileHeight());
Consumer<ScrollCaptureClient.CaptureResult> consumer =
new Consumer<ScrollCaptureClient.CaptureResult>() {
int mFrameCount = 0;
int mTop = 0;
@Override
public void accept(ScrollCaptureClient.CaptureResult result) {
mFrameCount++;
boolean emptyFrame = result.captured.height() == 0;
if (!emptyFrame) {
mPicture = stackBelow(mPicture, result.image, result.captured.width(),
result.captured.height());
mImageTileSet.addTile(new ImageTile(result.image, result.captured));
}
if (emptyFrame || mFrameCount >= MAX_PAGES
|| requestRect.bottom > MAX_HEIGHT) {
if (mPicture != null) {
exportToFile(mPicture, session, onDismiss);
|| mTop + session.getTileHeight() > MAX_HEIGHT) {
if (!mImageTileSet.isEmpty()) {
exportToFile(mImageTileSet.toBitmap(), session, onDismiss);
mImageTileSet.clear();
} else {
session.end(onDismiss);
}
return;
}
requestRect.offset(0, session.getMaxTileHeight());
session.requestTile(requestRect, /* consumer */ this);
mTop += result.captured.height();
session.requestTile(mTop, /* consumer */ this);
}
};
// fire it up!
session.requestTile(requestRect, consumer);
session.requestTile(0, consumer);
};
void exportToFile(Picture picture, Session session, Runnable afterEnd) {
void exportToFile(Bitmap bitmap, Session session, Runnable afterEnd) {
mImageExporter.setFormat(Bitmap.CompressFormat.PNG);
mImageExporter.setQuality(6);
ListenableFuture<Uri> future =
mImageExporter.export(mBgExecutor, Bitmap.createBitmap(picture));
mImageExporter.export(mBgExecutor, bitmap);
future.addListener(() -> {
picture.close(); // release resources
try {
launchViewer(future.get());
} catch (InterruptedException | ExecutionException e) {
@@ -126,41 +122,6 @@ public class ScrollCaptureController {
}, mUiExecutor);
}
/**
* Combine the top {@link Picture} with an {@link Image} by appending the image directly
* below, creating a result that is the combined height of both.
* <p>
* Note: no pixel data is transferred here, only a record of drawing commands. Backing
* hardware buffers must not be modified/recycled until the picture is
* {@link Picture#close closed}.
*
* @param top the existing picture
* @param below the image to append below
* @param cropWidth the width of the pixel data to use from the image
* @param cropHeight the height of the pixel data to use from the image
*
* @return a new Picture which draws the previous picture with the image below it
*/
private static Picture stackBelow(Picture top, Image below, int cropWidth, int cropHeight) {
int width = cropWidth;
int height = cropHeight;
if (top != null) {
height += top.getHeight();
width = Math.max(width, top.getWidth());
}
Picture combined = new Picture();
Canvas canvas = combined.beginRecording(width, height);
int y = 0;
if (top != null) {
canvas.drawPicture(top, new Rect(0, 0, top.getWidth(), top.getHeight()));
y += top.getHeight();
}
canvas.drawBitmap(Bitmap.wrapHardwareBuffer(
below.getHardwareBuffer(), ColorSpace.get(SRGB)), 0, y, null);
combined.endRecording();
return combined;
}
void launchViewer(Uri uri) {
Intent editIntent = new Intent(Intent.ACTION_VIEW);
editIntent.setType("image/png");

View File

@@ -0,0 +1,117 @@
/*
* Copyright (C) 2020 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.systemui.screenshot;
import android.annotation.Nullable;
import android.graphics.Canvas;
import android.graphics.ColorFilter;
import android.graphics.PixelFormat;
import android.graphics.Rect;
import android.graphics.RenderNode;
import android.graphics.drawable.Drawable;
import android.util.Log;
/**
* Draws a set of hardware image tiles from a display list. The tiles exist in virtual coordinate
* space that may extend into positive or negative values. The origin is the upper-left-most corner
* of bounding box, which is drawn at 0,0 to the drawable (output) bounds.
*/
public class TiledImageDrawable extends Drawable {
private static final String TAG = "TiledImageDrawable";
private final ImageTileSet mTiles;
private RenderNode mNode;
public TiledImageDrawable(ImageTileSet tiles) {
mTiles = tiles;
mTiles.setOnContentChangedListener(this::onContentChanged);
}
private void onContentChanged() {
if (mNode != null && mNode.hasDisplayList()) {
mNode.discardDisplayList();
}
invalidateSelf();
}
private void rebuildDisplayListIfNeeded() {
if (mNode != null && mNode.hasDisplayList()) {
return;
}
if (mNode == null) {
mNode = new RenderNode("TiledImageDrawable");
}
mNode.setPosition(0, 0, mTiles.getWidth(), mTiles.getHeight());
Canvas canvas = mNode.beginRecording(mTiles.getWidth(), mTiles.getHeight());
// Align content (virtual) top/left with 0,0, within the render node
canvas.translate(-mTiles.getLeft(), -mTiles.getTop());
for (int i = 0; i < mTiles.size(); i++) {
ImageTile tile = mTiles.get(i);
canvas.save();
canvas.translate(tile.getLeft(), tile.getTop());
canvas.drawRenderNode(tile.getDisplayList());
canvas.restore();
}
mNode.endRecording();
}
/**
* Draws the tiled image to the canvas, with the top/left (virtual) coordinate aligned to 0,0
* placed at left/top of the drawable's bounds.
*/
@Override
public void draw(Canvas canvas) {
rebuildDisplayListIfNeeded();
if (canvas.isHardwareAccelerated()) {
Rect bounds = getBounds();
canvas.save();
canvas.clipRect(bounds);
canvas.translate(bounds.left, bounds.top);
canvas.drawRenderNode(mNode);
canvas.restore();
} else {
Log.d(TAG, "Canvas is not hardware accelerated. Skipping draw!");
}
}
@Override
public int getIntrinsicWidth() {
return mTiles.getWidth();
}
@Override
public int getIntrinsicHeight() {
return mTiles.getHeight();
}
@Override
public void setAlpha(int alpha) {
if (mNode.setAlpha(alpha / 255f)) {
invalidateSelf();
}
}
@Override
public void setColorFilter(@Nullable ColorFilter colorFilter) {
throw new IllegalArgumentException("not implemented");
}
@Override
public int getOpacity() {
return PixelFormat.OPAQUE;
}
}

View File

@@ -96,13 +96,13 @@ public class ScrollCaptureClientTest extends SysuiTestCase {
Connection conn = mConnectionConsumer.getValue();
conn.start(5, mSessionConsumer);
conn.start(mSessionConsumer);
verify(mSessionConsumer, timeout(100)).accept(any(Session.class));
Session session = mSessionConsumer.getValue();
Rect request = new Rect(0, 0, session.getMaxTileWidth(), session.getMaxTileHeight());
Rect request = new Rect(0, 0, session.getPageWidth(), session.getTileHeight());
session.requestTile(request, mResultConsumer);
session.requestTile(0, mResultConsumer);
verify(mResultConsumer, timeout(100)).accept(any(CaptureResult.class));
CaptureResult result = mResultConsumer.getValue();