Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ImageReader/Writer: Add ImageWriter and Opaque ImageReader

Browse Source 
ImageWriter/Reader API change, including below changes
* Interface for Opaque ImageReader
* ImageWriter Interface and implementation
* Image class minor update to support ImageWriter opaque ImageReader

detach/attach interface are defined but yet to be implemented.

Change-Id: Ic7c0d2df73c80b1a81a7316d8c4556bf7703c309
This commit is contained in:
Zhijun He
2015-02-24 18:12:23 -08:00
parent 9b58c85524
commit f6a09e5106
10 changed files with 2245 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
api/current.txt
View File

@@ -14746,7 +14746,9 @@ package android.media {
method public abstract android.media.Image.Plane[] getPlanes();
method public abstract long getTimestamp();
method public abstract int getWidth();
method public boolean isOpaque();
method public void setCropRect(android.graphics.Rect);
method public void setTimestamp(long);
}
public static abstract class Image.Plane {
@@ -14764,7 +14766,9 @@ package android.media {
method public int getMaxImages();
method public android.view.Surface getSurface();
method public int getWidth();
method public boolean isOpaque();
method public static android.media.ImageReader newInstance(int, int, int, int);
method public static android.media.ImageReader newOpaqueInstance(int, int, int);
method public void setOnImageAvailableListener(android.media.ImageReader.OnImageAvailableListener, android.os.Handler);
}
@@ -14772,6 +14776,19 @@ package android.media {
method public abstract void onImageAvailable(android.media.ImageReader);
}
public class ImageWriter implements java.lang.AutoCloseable {
method public void close();
method public android.media.Image dequeueInputImage();
method public int getMaxImages();
method public static android.media.ImageWriter newInstance(android.view.Surface, int);
method public void queueInputImage(android.media.Image);
method public void setImageListener(android.media.ImageWriter.ImageListener, android.os.Handler);
}
public static abstract interface ImageWriter.ImageListener {
method public abstract void onInputImageReleased(android.media.ImageWriter);
}
public class JetPlayer {
method public boolean clearQueue();
method public java.lang.Object clone() throws java.lang.CloneNotSupportedException;

17
api/system-current.txt
View File

@@ -15934,7 +15934,9 @@ package android.media {
method public abstract android.media.Image.Plane[] getPlanes();
method public abstract long getTimestamp();
method public abstract int getWidth();
method public boolean isOpaque();
method public void setCropRect(android.graphics.Rect);
method public void setTimestamp(long);
}
public static abstract class Image.Plane {
@@ -15952,7 +15954,9 @@ package android.media {
method public int getMaxImages();
method public android.view.Surface getSurface();
method public int getWidth();
method public boolean isOpaque();
method public static android.media.ImageReader newInstance(int, int, int, int);
method public static android.media.ImageReader newOpaqueInstance(int, int, int);
method public void setOnImageAvailableListener(android.media.ImageReader.OnImageAvailableListener, android.os.Handler);
}
@@ -15960,6 +15964,19 @@ package android.media {
method public abstract void onImageAvailable(android.media.ImageReader);
}
public class ImageWriter implements java.lang.AutoCloseable {
method public void close();
method public android.media.Image dequeueInputImage();
method public int getMaxImages();
method public static android.media.ImageWriter newInstance(android.view.Surface, int);
method public void queueInputImage(android.media.Image);
method public void setImageListener(android.media.ImageWriter.ImageListener, android.os.Handler);
}
public static abstract interface ImageWriter.ImageListener {
method public abstract void onInputImageReleased(android.media.ImageWriter);
}
public class JetPlayer {
method public boolean clearQueue();
method public java.lang.Object clone() throws java.lang.CloneNotSupportedException;

92
media/java/android/media/Image.java
View File

@@ -115,14 +115,49 @@ public abstract class Image implements AutoCloseable {
/**
* Get the timestamp associated with this frame.
* <p>
* The timestamp is measured in nanoseconds, and is monotonically
* increasing. However, the zero point and whether the timestamp can be
* compared against other sources of time or images depend on the source of
* this image.
* The timestamp is measured in nanoseconds, and is normally monotonically
* increasing. However, the behavior of the timestamp depends on the source
* of this image. See {@link android.hardware.Camera Camera},
* {@link android.hardware.camera2.CameraDevice CameraDevice}, {@link MediaPlayer} and
* {@link MediaCodec} for more details.
* </p>
*/
public abstract long getTimestamp();
/**
* Set the timestamp associated with this frame.
* <p>
* The timestamp is measured in nanoseconds, and is normally monotonically
* increasing. However, However, the behavior of the timestamp depends on
* the destination of this image. See {@link android.hardware.Camera Camera}
* , {@link android.hardware.camera2.CameraDevice CameraDevice},
* {@link MediaPlayer} and {@link MediaCodec} for more details.
* </p>
* <p>
* For images dequeued from {@link ImageWriter} via
* {@link ImageWriter#dequeueInputImage()}, it's up to the application to
* set the timestamps correctly before sending them back to the
* {@link ImageWriter}, or the timestamp will be generated automatically when
* {@link ImageWriter#queueInputImage queueInputImage()} is called.
* </p>
*
* @param timestamp The timestamp to be set for this image.
*/
public void setTimestamp(long timestamp) {
return;
}
/**
* <p>Check if the image is opaque.</p>
*
* <p>The pixel data of opaque images are not accessible to the application,
* and therefore {@link #getPlanes} will return an empty array for an opaque image.
* </p>
*/
public boolean isOpaque() {
return false;
}
private Rect mCropRect;
/**
@@ -155,7 +190,10 @@ public abstract class Image implements AutoCloseable {
/**
* Get the array of pixel planes for this Image. The number of planes is
* determined by the format of the Image.
* determined by the format of the Image. The application will get an
* empty array if the image is opaque because the opaque image pixel data
* is not directly accessible. The application can check if an image is
* opaque by calling {@link Image#isOpaque}.
*/
public abstract Plane[] getPlanes();
@@ -164,13 +202,53 @@ public abstract class Image implements AutoCloseable {
* <p>
* After calling this method, calling any methods on this {@code Image} will
* result in an {@link IllegalStateException}, and attempting to read from
* {@link ByteBuffer ByteBuffers} returned by an earlier
* {@link Plane#getBuffer} call will have undefined behavior.
* or write to {@link ByteBuffer ByteBuffers} returned by an earlier
* {@link Plane#getBuffer} call will have undefined behavior. If the image
* was obtained from {@link ImageWriter} via
* {@link ImageWriter#dequeueInputImage()}, after calling this method, any
* image data filled by the application will be lost and the image will be
* returned to {@link ImageWriter} for reuse. Images given to
* {@link ImageWriter#queueInputImage queueInputImage()} are automatically
* closed.
* </p>
*/
@Override
public abstract void close();
/**
* <p>
* Check if the image can be attached to a new owner (e.g. {@link ImageWriter}).
* </p>
* <p>
* This is a package private method that is only used internally.
* </p>
*
* @return true if the image is attachable to a new owner, false if the image is still attached
* to its current owner, or the image is a stand-alone image and is not attachable to
* a new owner.
*/
boolean isAttachable() {
return false;
}
/**
* <p>
* Get the owner of the {@link Image}.
* </p>
* <p>
* The owner of an {@link Image} could be {@link ImageReader}, {@link ImageWriter},
* {@link MediaCodec} etc. This method returns the owner that produces this image, or null
* if the image is stand-alone image or the owner is unknown.
* </p>
* <p>
* This is a package private method that is only used internally.
* </p>
*
* @return The owner of the Image.
*/
Object getOwner() {
return null;
}
/**
* <p>A single color plane of image data.</p>
*

169
media/java/android/media/ImageReader.java
View File

@@ -27,6 +27,7 @@ import java.lang.ref.WeakReference;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.NioUtils;
import java.util.concurrent.atomic.AtomicBoolean;
/**
* <p>The ImageReader class allows direct application access to image data
@@ -34,7 +35,7 @@ import java.nio.NioUtils;
*
* <p>Several Android media API classes accept Surface objects as targets to
* render to, including {@link MediaPlayer}, {@link MediaCodec},
* {@link android.hardware.camera2.CameraDevice}, and
* {@link android.hardware.camera2.CameraDevice}, {@link ImageWriter} and
* {@link android.renderscript.Allocation RenderScript Allocations}. The image
* sizes and formats that can be used with each source vary, and should be
* checked in the documentation for the specific API.</p>
@@ -97,9 +98,59 @@ public class ImageReader implements AutoCloseable {
* @see Image
*/
public static ImageReader newInstance(int width, int height, int format, int maxImages) {
if (format == PixelFormat.OPAQUE) {
throw new IllegalArgumentException("To obtain an opaque ImageReader, please use"
+ " newOpaqueInstance rather than newInstance");
}
return new ImageReader(width, height, format, maxImages);
}
/**
* <p>
* Create a new opaque reader for images of the desired size.
* </p>
* <p>
* An opaque {@link ImageReader} produces images that are not directly
* accessible by the application. The application can still acquire images
* from an opaque image reader, and send them to the
* {@link android.hardware.camera2.CameraDevice camera} for reprocessing via
* {@link ImageWriter} interface. However, the {@link Image#getPlanes()
* getPlanes()} will return an empty array for opaque images. The
* application can check if an existing reader is an opaque reader by
* calling {@link #isOpaque()}.
* </p>
* <p>
* The {@code maxImages} parameter determines the maximum number of
* {@link Image} objects that can be be acquired from the
* {@code ImageReader} simultaneously. Requesting more buffers will use up
* more memory, so it is important to use only the minimum number necessary.
* </p>
* <p>
* The valid sizes and formats depend on the source of the image data.
* </p>
* <p>
* Opaque ImageReaders are more efficient to use when application access to
* image data is not necessary, comparing to ImageReaders using a non-opaque
* format such as {@link ImageFormat#YUV_420_888 YUV_420_888}.
* </p>
*
* @param width The default width in pixels of the Images that this reader
* will produce.
* @param height The default height in pixels of the Images that this reader
* will produce.
* @param maxImages The maximum number of images the user will want to
* access simultaneously. This should be as small as possible to
* limit memory use. Once maxImages Images are obtained by the
* user, one of them has to be released before a new Image will
* become available for access through
* {@link #acquireLatestImage()} or {@link #acquireNextImage()}.
* Must be greater than 0.
* @see Image
*/
public static ImageReader newOpaqueInstance(int width, int height, int maxImages) {
return new ImageReader(width, height, PixelFormat.OPAQUE, maxImages);
}
/**
* @hide
*/
@@ -196,6 +247,23 @@ public class ImageReader implements AutoCloseable {
return mMaxImages;
}
/**
* <p>
* Check if the {@link ImageReader} is an opaque reader.
* </p>
* <p>
* An opaque image reader produces opaque images, see {@link Image#isOpaque}
* for more details.
* </p>
*
* @return true if the ImageReader is opaque.
* @see Image#isOpaque
* @see ImageReader#newOpaqueInstance
*/
public boolean isOpaque() {
return mFormat == PixelFormat.OPAQUE;
}
/**
* <p>Get a {@link Surface} that can be used to produce {@link Image Images} for this
* {@code ImageReader}.</p>
@@ -456,6 +524,58 @@ public class ImageReader implements AutoCloseable {
}
}
/**
* <p>
* Remove the ownership of this image from the ImageReader.
* </p>
* <p>
* After this call, the ImageReader no longer owns this image, and the image
* ownership can be transfered to another entity like {@link ImageWriter}
* via {@link ImageWriter#queueInputImage}. It's up to the new owner to
* release the resources held by this image. For example, if the ownership
* of this image is transfered to an {@link ImageWriter}, the image will be
* freed by the ImageWriter after the image data consumption is done.
* </p>
* <p>
* This method can be used to achieve zero buffer copy for use cases like
* {@link android.hardware.camera2.CameraDevice Camera2 API} OPAQUE and YUV
* reprocessing, where the application can select an output image from
* {@link ImageReader} and transfer this image directly to
* {@link ImageWriter}, where this image can be consumed by camera directly.
* For OPAQUE reprocessing, this is the only way to send input buffers to
* the {@link android.hardware.camera2.CameraDevice camera} for
* reprocessing.
* </p>
* <p>
* This is a package private method that is only used internally.
* </p>
*
* @param image The image to be detached from this ImageReader.
* @throws IllegalStateException If the ImageReader or image have been
* closed, or the has been detached, or has not yet been
* acquired.
*/
void detachImage(Image image) {
if (image == null) {
throw new IllegalArgumentException("input image must not be null");
}
if (!isImageOwnedbyMe(image)) {
throw new IllegalArgumentException("Trying to detach an image that is not owned by"
+ " this ImageReader");
}
SurfaceImage si = (SurfaceImage) image;
if (!si.isImageValid()) {
throw new IllegalStateException("Image is no longer valid");
}
if (si.isAttachable()) {
throw new IllegalStateException("Image was already detached from this ImageReader");
}
nativeDetachImage(image);
si.setDetached(true);
}
/**
* Only a subset of the formats defined in
* {@link android.graphics.ImageFormat ImageFormat} and
@@ -487,12 +607,22 @@ public class ImageReader implements AutoCloseable {
case ImageFormat.DEPTH16:
case ImageFormat.DEPTH_POINT_CLOUD:
return 1;
case PixelFormat.OPAQUE:
return 0;
default:
throw new UnsupportedOperationException(
String.format("Invalid format specified %d", mFormat));
}
}
private boolean isImageOwnedbyMe(Image image) {
if (!(image instanceof SurfaceImage)) {
return false;
}
SurfaceImage si = (SurfaceImage) image;
return si.getReader() == this;
}
/**
* Called from Native code when an Event happens.
*
@@ -561,7 +691,11 @@ public class ImageReader implements AutoCloseable {
@Override
public void close() {
if (mIsImageValid) {
ImageReader.this.releaseImage(this);
if (!mIsDetached.get()) {
// For detached images, the new owner is responsible for
// releasing the resources
ImageReader.this.releaseImage(this);
}
}
}
@@ -613,6 +747,15 @@ public class ImageReader implements AutoCloseable {
}
}
@Override
public void setTimestamp(long timestampNs) {
if (mIsImageValid) {
mTimestamp = timestampNs;
} else {
throw new IllegalStateException("Image is already released");
}
}
@Override
public Plane[] getPlanes() {
if (mIsImageValid) {
@@ -623,6 +766,11 @@ public class ImageReader implements AutoCloseable {
}
}
@Override
public boolean isOpaque() {
return mFormat == PixelFormat.OPAQUE;
}
@Override
protected final void finalize() throws Throwable {
try {
@@ -632,6 +780,20 @@ public class ImageReader implements AutoCloseable {
}
}
@Override
boolean isAttachable() {
return mIsDetached.get();
}
@Override
ImageReader getOwner() {
return ImageReader.this;
}
private void setDetached(boolean detached) {
mIsDetached.getAndSet(detached);
}
private void setImageValid(boolean isValid) {
mIsImageValid = isValid;
}
@@ -734,6 +896,8 @@ public class ImageReader implements AutoCloseable {
private boolean mIsImageValid;
private int mHeight = -1;
private int mWidth = -1;
// If this image is detached from the ImageReader.
private AtomicBoolean mIsDetached = new AtomicBoolean(false);
private synchronized native ByteBuffer nativeImageGetBuffer(int idx, int readerFormat);
private synchronized native SurfacePlane nativeCreatePlane(int idx, int readerFormat);
@@ -746,6 +910,7 @@ public class ImageReader implements AutoCloseable {
private synchronized native void nativeClose();
private synchronized native void nativeReleaseImage(Image i);
private synchronized native Surface nativeGetSurface();
private synchronized native void nativeDetachImage(Image i);
/**
* @return A return code {@code ACQUIRE_*}

120
media/java/android/media/ImageUtils.java Normal file
View File

@@ -0,0 +1,120 @@
/*
* Copyright 2015 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package android.media;
import android.graphics.ImageFormat;
import android.graphics.PixelFormat;
import android.media.Image.Plane;
import android.util.Size;
import java.nio.ByteBuffer;
/**
* Package private utility class for hosting commonly used Image related methods.
*/
class ImageUtils {
/**
* Only a subset of the formats defined in
* {@link android.graphics.ImageFormat ImageFormat} and
* {@link android.graphics.PixelFormat PixelFormat} are supported by
* ImageReader. When reading RGB data from a surface, the formats defined in
* {@link android.graphics.PixelFormat PixelFormat} can be used; when
* reading YUV, JPEG or raw sensor data (for example, from the camera or video
* decoder), formats from {@link android.graphics.ImageFormat ImageFormat}
* are used.
*/
public static int getNumPlanesForFormat(int format) {
switch (format) {
case ImageFormat.YV12:
case ImageFormat.YUV_420_888:
case ImageFormat.NV21:
return 3;
case ImageFormat.NV16:
return 2;
case PixelFormat.RGB_565:
case PixelFormat.RGBA_8888:
case PixelFormat.RGBX_8888:
case PixelFormat.RGB_888:
case ImageFormat.JPEG:
case ImageFormat.YUY2:
case ImageFormat.Y8:
case ImageFormat.Y16:
case ImageFormat.RAW_SENSOR:
case ImageFormat.RAW10:
return 1;
case PixelFormat.OPAQUE:
return 0;
default:
throw new UnsupportedOperationException(
String.format("Invalid format specified %d", format));
}
}
/**
* <p>
* Copy source image data to destination Image.
* </p>
* <p>
* Only support the copy between two non-opaque images with same properties
* (format, size, etc.). The data from the source image will be copied to
* the byteBuffers from the destination Image starting from position zero,
* and the destination image will be rewound to zero after copy is done.
* </p>
*
* @param src The source image to be copied from.
* @param dst The destination image to be copied to.
* @throws IllegalArgumentException If the source and destination images
* have different format, or one of the images is not copyable.
*/
public static void imageCopy(Image src, Image dst) {
if (src == null || dst == null) {
throw new IllegalArgumentException("Images should be non-null");
}
if (src.getFormat() != dst.getFormat()) {
throw new IllegalArgumentException("Src and dst images should have the same format");
}
if (src.isOpaque() || dst.isOpaque()) {
throw new IllegalArgumentException("Opaque image is not copyable");
}
if (!(dst.getOwner() instanceof ImageWriter)) {
throw new IllegalArgumentException("Destination image is not from ImageWriter. Only"
+ " the images from ImageWriter are writable");
}
Size srcSize = new Size(src.getWidth(), src.getHeight());
Size dstSize = new Size(dst.getWidth(), dst.getHeight());
if (!srcSize.equals(dstSize)) {
throw new IllegalArgumentException("source image size " + srcSize + " is different"
+ " with " + "destination image size " + dstSize);
}
Plane[] srcPlanes = src.getPlanes();
Plane[] dstPlanes = dst.getPlanes();
ByteBuffer srcBuffer = null;
ByteBuffer dstBuffer = null;
for (int i = 0; i < srcPlanes.length; i++) {
srcBuffer = srcPlanes[i].getBuffer();
int srcPos = srcBuffer.position();
srcBuffer.rewind();
dstBuffer = dstPlanes[i].getBuffer();
dstBuffer.rewind();
dstBuffer.put(srcBuffer);
srcBuffer.position(srcPos);
dstBuffer.rewind();
}
}
}

798
media/java/android/media/ImageWriter.java Normal file
View File

@@ -0,0 +1,798 @@
/*
* Copyright 2015 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package android.media;
import android.graphics.PixelFormat;
import android.graphics.Rect;
import android.os.Handler;
import android.os.Looper;
import android.os.Message;
import android.view.Surface;
import java.lang.ref.WeakReference;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.NioUtils;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
/**
* <p>
* The ImageWriter class allows an application to produce Image data into a
* {@link android.view.Surface}, and have it be consumed by another component like
* {@link android.hardware.camera2.CameraDevice CameraDevice}.
* </p>
* <p>
* Several Android API classes can provide input {@link android.view.Surface
* Surface} objects for ImageWriter to produce data into, including
* {@link MediaCodec MediaCodec} (encoder),
* {@link android.hardware.camera2.CameraDevice CameraDevice} (reprocessing
* input), {@link ImageReader}, etc.
* </p>
* <p>
* The input Image data is encapsulated in {@link Image} objects. To produce
* Image data into a destination {@link android.view.Surface Surface}, the
* application can get an input Image via {@link #dequeueInputImage} then write
* Image data into it. Multiple such {@link Image} objects can be dequeued at
* the same time and queued back in any order, up to the number specified by the
* {@code maxImages} constructor parameter.
* </p>
* <p>
* If the application already has an Image from {@link ImageReader}, the
* application can directly queue this Image into ImageWriter (via
* {@link #queueInputImage}), potentially with zero buffer copies. For the opaque
* Images produced by an opaque ImageReader (created by
* {@link ImageReader#newOpaqueInstance}), this is the only way to send Image
* data to ImageWriter, as the Image data aren't accessible by the application.
* </p>
* Once new input Images are queued into an ImageWriter, it's up to the downstream
* components (e.g. {@link ImageReader} or
* {@link android.hardware.camera2.CameraDevice}) to consume the Images. If the
* downstream components cannot consume the Images at least as fast as the
* ImageWriter production rate, the {@link #dequeueInputImage} call will eventually
* block and the application will have to drop input frames. </p>
*/
public class ImageWriter implements AutoCloseable {
private final Object mListenerLock = new Object();
private ImageListener mListener;
private ListenerHandler mListenerHandler;
private long mNativeContext;
// Field below is used by native code, do not access or modify.
private int mWriterFormat;
private final int mMaxImages;
// Keep track of the currently attached Image; or an attached Image that is
// released will be removed from this list.
private List<Image> mAttachedImages = new ArrayList<Image>();
private List<Image> mDequeuedImages = new ArrayList<Image>();
/**
* <p>
* Create a new ImageWriter.
* </p>
* <p>
* The {@code maxImages} parameter determines the maximum number of
* {@link Image} objects that can be be dequeued from the
* {@code ImageWriter} simultaneously. Requesting more buffers will use up
* more memory, so it is important to use only the minimum number necessary.
* </p>
* <p>
* The input Image size and format depend on the Surface that is provided by
* the downstream consumer end-point.
* </p>
*
* @param surface The destination Surface this writer produces Image data
* into.
* @param maxImages The maximum number of Images the user will want to
* access simultaneously for producing Image data. This should be
* as small as possible to limit memory use. Once maxImages
* Images are dequeued by the user, one of them has to be queued
* back before a new Image can be dequeued for access via
* {@link #dequeueInputImage()}.
* @return a new ImageWriter instance.
*/
public static ImageWriter newInstance(Surface surface, int maxImages) {
return new ImageWriter(surface, maxImages);
}
/**
* @hide
*/
protected ImageWriter(Surface surface, int maxImages) {
if (surface == null || maxImages < 1) {
throw new IllegalArgumentException("Illegal input argument: surface " + surface
+ ", maxImages: " + maxImages);
}
mMaxImages = maxImages;
// Note that the underlying BufferQueue is working in synchronous mode
// to avoid dropping any buffers.
mNativeContext = nativeInit(new WeakReference<ImageWriter>(this), surface, maxImages);
}
/**
* <p>
* Maximum number of Images that can be dequeued from the ImageWriter
* simultaneously (for example, with {@link #dequeueInputImage()}).
* </p>
* <p>
* An Image is considered dequeued after it's returned by
* {@link #dequeueInputImage()} from ImageWriter, and until the Image is
* sent back to ImageWriter via {@link #queueInputImage}, or
* {@link Image#close()}.
* </p>
* <p>
* Attempting to dequeue more than {@code maxImages} concurrently will
* result in the {@link #dequeueInputImage()} function throwing an
* {@link IllegalStateException}.
* </p>
*
* @return Maximum number of Images that can be dequeued from this
* ImageWriter.
* @see #dequeueInputImage
* @see #queueInputImage
* @see Image#close
*/
public int getMaxImages() {
return mMaxImages;
}
/**
* <p>
* Dequeue the next available input Image for the application to produce
* data into.
* </p>
* <p>
* This method requests a new input Image from ImageWriter. The application
* owns this Image after this call. Once the application fills the Image
* data, it is expected to return this Image back to ImageWriter for
* downstream consumer components (e.g.
* {@link android.hardware.camera2.CameraDevice}) to consume. The Image can
* be returned to ImageWriter via {@link #queueInputImage} or
* {@link Image#close()}.
* </p>
* <p>
* This call will block if all available input images have been filled by
* the application and the downstream consumer has not yet consumed any.
* When an Image is consumed by the downstream consumer, an
* {@link ImageListener#onInputImageReleased} callback will be fired, which
* indicates that there is one input Image available. It is recommended to
* dequeue next Image only after this callback is fired, in the steady state.
* </p>
*
* @return The next available input Image from this ImageWriter.
* @throws IllegalStateException if {@code maxImages} Images are currently
* dequeued.
* @see #queueInputImage
* @see Image#close
*/
public Image dequeueInputImage() {
if (mDequeuedImages.size() >= mMaxImages) {
throw new IllegalStateException("Already dequeued max number of Images " + mMaxImages);
}
WriterSurfaceImage newImage = new WriterSurfaceImage(this);
nativeDequeueInputImage(mNativeContext, newImage);
mDequeuedImages.add(newImage);
newImage.setImageValid(true);
return newImage;
}
/**
* <p>
* Queue an input {@link Image} back to ImageWriter for the downstream
* consumer to access.
* </p>
* <p>
* The input {@link Image} could be from ImageReader (acquired via
* {@link ImageReader#acquireNextImage} or
* {@link ImageReader#acquireLatestImage}), or from this ImageWriter
* (acquired via {@link #dequeueInputImage}). In the former case, the Image
* data will be moved to this ImageWriter. Note that the Image properties
* (size, format, strides, etc.) must be the same as the properties of the
* images dequeued from this ImageWriter, or this method will throw an
* {@link IllegalArgumentException}. In the latter case, the application has
* filled the input image with data. This method then passes the filled
* buffer to the downstream consumer. In both cases, it's up to the caller
* to ensure that the Image timestamp (in nanoseconds) is correctly set, as
* the downstream component may want to use it to indicate the Image data
* capture time.
* </p>
* <p>
* Passing in a non-opaque Image may result in a memory copy, which also
* requires a free input Image from this ImageWriter as the destination. In
* this case, this call will block, as {@link #dequeueInputImage} does, if
* there are no free Images available. To be safe, the application should ensure
* that there is at least one free Image available in this ImageWriter before calling
* this method.
* </p>
* <p>
* After this call, the input Image is no longer valid for further access,
* as if the Image is {@link Image#close closed}. Attempting to access the
* {@link ByteBuffer ByteBuffers} returned by an earlier
* {@link Image.Plane#getBuffer Plane#getBuffer} call will result in an
* {@link IllegalStateException}.
* </p>
*
* @param image The Image to be queued back to ImageWriter for future
* consumption.
* @see #dequeueInputImage()
*/
public void queueInputImage(Image image) {
if (image == null) {
throw new IllegalArgumentException("image shouldn't be null");
}
boolean ownedByMe = isImageOwnedByMe(image);
if (ownedByMe && !(((WriterSurfaceImage) image).isImageValid())) {
throw new IllegalStateException("Image from ImageWriter is invalid");
}
// For images from other components, need to detach first, then attach.
if (!ownedByMe) {
if (!(image.getOwner() instanceof ImageReader)) {
throw new IllegalArgumentException("Only images from ImageReader can be queued to"
+ " ImageWriter, other image source is not supported yet!");
}
ImageReader prevOwner = (ImageReader) image.getOwner();
// Only do the image attach for opaque images for now. Do the image
// copy for other formats. TODO: use attach for other formats to
// improve the performance, and fall back to copy when attach/detach fails.
if (image.isOpaque()) {
prevOwner.detachImage(image);
attachInputImage(image);
} else {
Image inputImage = dequeueInputImage();
inputImage.setTimestamp(image.getTimestamp());
inputImage.setCropRect(image.getCropRect());
ImageUtils.imageCopy(image, inputImage);
image.close();
image = inputImage;
ownedByMe = true;
}
}
Rect crop = image.getCropRect();
nativeQueueInputImage(mNativeContext, image, image.getTimestamp(), crop.left, crop.top,
crop.right, crop.bottom);
/**
* Only remove and cleanup the Images that are owned by this
* ImageWriter. Images detached from other owners are only
* temporarily owned by this ImageWriter and will be detached immediately
* after they are released by downstream consumers, so there is no need to
* keep track of them in mDequeuedImages.
*/
if (ownedByMe) {
mDequeuedImages.remove(image);
WriterSurfaceImage wi = (WriterSurfaceImage) image;
wi.clearSurfacePlanes();
wi.setImageValid(false);
} else {
// This clears the native reference held by the original owner. When
// this Image is detached later by this ImageWriter, the native
// memory won't be leaked.
image.close();
}
}
/**
* ImageWriter callback interface, used to to asynchronously notify the
* application of various ImageWriter events.
*/
public interface ImageListener {
/**
* <p>
* Callback that is called when an input Image is released back to
* ImageWriter after the data consumption.
* </p>
* <p>
* The client can use this callback to indicate either an input Image is
* available to fill data into, or the input Image is returned and freed
* if it was attached from other components (e.g. an
* {@link ImageReader}). For the latter case, the ownership of the Image
* will be automatically removed by ImageWriter right before this
* callback is fired.
* </p>
*
* @param writer the ImageWriter the callback is associated with.
* @see ImageWriter
* @see Image
*/
// TODO: the semantics is confusion, does't tell which buffer is
// released if an application is doing queueInputImage with a mix of
// buffers from dequeueInputImage and from an ImageReader. see b/19872821
void onInputImageReleased(ImageWriter writer);
}
/**
* Register a listener to be invoked when an input Image is returned to
* the ImageWriter.
*
* @param listener The listener that will be run.
* @param handler The handler on which the listener should be invoked, or
* null if the listener should be invoked on the calling thread's
* looper.
* @throws IllegalArgumentException If no handler specified and the calling
* thread has no looper.
*/
public void setImageListener(ImageListener listener, Handler handler) {
synchronized (mListenerLock) {
if (listener != null) {
Looper looper = handler != null ? handler.getLooper() : Looper.myLooper();
if (looper == null) {
throw new IllegalArgumentException(
"handler is null but the current thread is not a looper");
}
if (mListenerHandler == null || mListenerHandler.getLooper() != looper) {
mListenerHandler = new ListenerHandler(looper);
}
mListener = listener;
} else {
mListener = null;
mListenerHandler = null;
}
}
}
/**
* Free up all the resources associated with this ImageWriter.
* <p>
* After calling this method, this ImageWriter cannot be used. Calling any
* methods on this ImageWriter and Images previously provided by
* {@link #dequeueInputImage()} will result in an
* {@link IllegalStateException}, and attempting to write into
* {@link ByteBuffer ByteBuffers} returned by an earlier
* {@link Image.Plane#getBuffer Plane#getBuffer} call will have undefined
* behavior.
* </p>
*/
@Override
public void close() {
setImageListener(null, null);
for (Image image : mDequeuedImages) {
image.close();
}
mDequeuedImages.clear();
nativeClose(mNativeContext);
mNativeContext = 0;
}
@Override
protected void finalize() throws Throwable {
try {
close();
} finally {
super.finalize();
}
}
/**
* Get the ImageWriter format.
* <p>
* This format may be different than the Image format returned by
* {@link Image#getFormat()}
* </p>
*
* @return The ImageWriter format.
*/
int getFormat() {
return mWriterFormat;
}
/**
* <p>
* Attach input Image to this ImageWriter.
* </p>
* <p>
* When an Image is from an opaque source (e.g. an opaque ImageReader created
* by {@link ImageReader#newOpaqueInstance}), or the source Image is so large
* that copying its data is too expensive, this method can be used to
* migrate the source Image into ImageWriter without a data copy. The source
* Image must be detached from its previous owner already, or this call will
* throw an {@link IllegalStateException}.
* </p>
* <p>
* After this call, the ImageWriter takes ownership of this Image.
* This ownership will be automatically removed from this writer after the
* consumer releases this Image, that is, after
* {@link ImageListener#onInputImageReleased}. The caller is
* responsible for closing this Image through {@link Image#close()} to free up
* the resources held by this Image.
* </p>
*
* @param image The source Image to be attached and queued into this
* ImageWriter for downstream consumer to use.
* @throws IllegalStateException if the Image is not detached from its
* previous owner, or the Image is already attached to this
* ImageWriter, or the source Image is invalid.
*/
private void attachInputImage(Image image) {
if (image == null) {
throw new IllegalArgumentException("image shouldn't be null");
}
if (isImageOwnedByMe(image)) {
throw new IllegalArgumentException(
"Can not attach an image that is owned ImageWriter already");
}
/**
* Throw ISE if the image is not attachable, which means that it is
* either owned by other entity now, or completely non-attachable (some
* stand-alone images are not backed by native gralloc buffer, thus not
* attachable).
*/
if (!image.isAttachable()) {
throw new IllegalStateException("Image was not detached from last owner, or image "
+ " is not detachable");
}
if (mAttachedImages.contains(image)) {
throw new IllegalStateException("Image was already attached to ImageWritter");
}
// TODO: what if attach failed, throw RTE or detach a slot then attach?
// need do some cleanup to make sure no orphaned
// buffer caused leak.
nativeAttachImage(mNativeContext, image);
mAttachedImages.add(image);
}
/**
* This custom handler runs asynchronously so callbacks don't get queued
* behind UI messages.
*/
private final class ListenerHandler extends Handler {
public ListenerHandler(Looper looper) {
super(looper, null, true /* async */);
}
@Override
public void handleMessage(Message msg) {
ImageListener listener;
synchronized (mListenerLock) {
listener = mListener;
}
// TODO: detach Image from ImageWriter and remove the Image from
// mAttachedImage list.
if (listener != null) {
listener.onInputImageReleased(ImageWriter.this);
}
}
}
/**
* Called from Native code when an Event happens. This may be called from an
* arbitrary Binder thread, so access to the ImageWriter must be
* synchronized appropriately.
*/
private static void postEventFromNative(Object selfRef) {
@SuppressWarnings("unchecked")
WeakReference<ImageWriter> weakSelf = (WeakReference<ImageWriter>) selfRef;
final ImageWriter iw = weakSelf.get();
if (iw == null) {
return;
}
final Handler handler;
synchronized (iw.mListenerLock) {
handler = iw.mListenerHandler;
}
if (handler != null) {
handler.sendEmptyMessage(0);
}
}
/**
* <p>
* Abort the Images that were dequeued from this ImageWriter, and return
* them to this writer for reuse.
* </p>
* <p>
* This method is used for the cases where the application dequeued the
* Image, may have filled the data, but does not want the downstream
* component to consume it. The Image will be returned to this ImageWriter
* for reuse after this call, and the ImageWriter will immediately have an
* Image available to be dequeued. This aborted Image will be invisible to
* the downstream consumer, as if nothing happened.
* </p>
*
* @param image The Image to be aborted.
* @see #dequeueInputImage()
* @see Image#close()
*/
private void abortImage(Image image) {
if (image == null) {
throw new IllegalArgumentException("image shouldn't be null");
}
if (!mDequeuedImages.contains(image)) {
throw new IllegalStateException("It is illegal to abort some image that is not"
+ " dequeued yet");
}
WriterSurfaceImage wi = (WriterSurfaceImage) image;
if (!wi.isImageValid()) {
throw new IllegalStateException("Image is invalid");
}
/**
* We only need abort Images that are owned and dequeued by ImageWriter.
* For attached Images, no need to abort, as there are only two cases:
* attached + queued successfully, and attach failed. Neither of the
* cases need abort.
*/
cancelImage(mNativeContext,image);
mDequeuedImages.remove(image);
wi.clearSurfacePlanes();
wi.setImageValid(false);
}
private boolean isImageOwnedByMe(Image image) {
if (!(image instanceof WriterSurfaceImage)) {
return false;
}
WriterSurfaceImage wi = (WriterSurfaceImage) image;
if (wi.getOwner() != this) {
return false;
}
return true;
}
private static class WriterSurfaceImage extends android.media.Image {
private ImageWriter mOwner;
private AtomicBoolean mIsImageValid = new AtomicBoolean(false);
// This field is used by native code, do not access or modify.
private long mNativeBuffer;
private int mNativeFenceFd = -1;
private SurfacePlane[] mPlanes;
private int mHeight = -1;
private int mWidth = -1;
private int mFormat = -1;
// When this default timestamp is used, timestamp for the input Image
// will be generated automatically when queueInputBuffer is called.
private final long DEFAULT_TIMESTAMP = Long.MIN_VALUE;
private long mTimestamp = DEFAULT_TIMESTAMP;
public WriterSurfaceImage(ImageWriter writer) {
mOwner = writer;
}
@Override
public int getFormat() {
if (!mIsImageValid.get()) {
throw new IllegalStateException("Image is already released");
}
if (mFormat == -1) {
mFormat = nativeGetFormat();
}
return mFormat;
}
@Override
public int getWidth() {
if (!mIsImageValid.get()) {
throw new IllegalStateException("Image is already released");
}
if (mWidth == -1) {
mWidth = nativeGetWidth();
}
return mWidth;
}
@Override
public int getHeight() {
if (!mIsImageValid.get()) {
throw new IllegalStateException("Image is already released");
}
if (mHeight == -1) {
mHeight = nativeGetHeight();
}
return mHeight;
}
@Override
public long getTimestamp() {
if (!mIsImageValid.get()) {
throw new IllegalStateException("Image is already released");
}
return mTimestamp;
}
@Override
public void setTimestamp(long timestamp) {
if (!mIsImageValid.get()) {
throw new IllegalStateException("Image is already released");
}
mTimestamp = timestamp;
}
@Override
public boolean isOpaque() {
if (!mIsImageValid.get()) {
throw new IllegalStateException("Image is already released");
}
return getFormat() == PixelFormat.OPAQUE;
}
@Override
public Plane[] getPlanes() {
if (!mIsImageValid.get()) {
throw new IllegalStateException("Image is already released");
}
if (mPlanes == null) {
int numPlanes = ImageUtils.getNumPlanesForFormat(getFormat());
mPlanes = nativeCreatePlanes(numPlanes, getOwner().getFormat());
}
return mPlanes.clone();
}
@Override
boolean isAttachable() {
if (!mIsImageValid.get()) {
throw new IllegalStateException("Image is already released");
}
// Don't allow Image to be detached from ImageWriter for now, as no
// detach API is exposed.
return false;
}
@Override
ImageWriter getOwner() {
return mOwner;
}
@Override
public void close() {
if (mIsImageValid.get()) {
getOwner().abortImage(this);
}
}
@Override
protected final void finalize() throws Throwable {
try {
close();
} finally {
super.finalize();
}
}
private boolean isImageValid() {
return mIsImageValid.get();
}
private void setImageValid(boolean isValid) {
mIsImageValid.getAndSet(isValid);
}
private void clearSurfacePlanes() {
if (mIsImageValid.get()) {
for (int i = 0; i < mPlanes.length; i++) {
if (mPlanes[i] != null) {
mPlanes[i].clearBuffer();
mPlanes[i] = null;
}
}
}
}
private class SurfacePlane extends android.media.Image.Plane {
private ByteBuffer mBuffer;
final private int mPixelStride;
final private int mRowStride;
// SurfacePlane instance is created by native code when a new
// SurfaceImage is created
private SurfacePlane(int rowStride, int pixelStride, ByteBuffer buffer) {
mRowStride = rowStride;
mPixelStride = pixelStride;
mBuffer = buffer;
/**
* Set the byteBuffer order according to host endianness (native
* order), otherwise, the byteBuffer order defaults to
* ByteOrder.BIG_ENDIAN.
*/
mBuffer.order(ByteOrder.nativeOrder());
}
@Override
public int getRowStride() {
if (WriterSurfaceImage.this.isImageValid() == false) {
throw new IllegalStateException("Image is already released");
}
return mRowStride;
}
@Override
public int getPixelStride() {
if (WriterSurfaceImage.this.isImageValid() == false) {
throw new IllegalStateException("Image is already released");
}
return mPixelStride;
}
@Override
public ByteBuffer getBuffer() {
if (WriterSurfaceImage.this.isImageValid() == false) {
throw new IllegalStateException("Image is already released");
}
return mBuffer;
}
private void clearBuffer() {
// Need null check first, as the getBuffer() may not be called
// before an Image is closed.
if (mBuffer == null) {
return;
}
if (mBuffer.isDirect()) {
NioUtils.freeDirectBuffer(mBuffer);
}
mBuffer = null;
}
}
// this will create the SurfacePlane object and fill the information
private synchronized native SurfacePlane[] nativeCreatePlanes(int numPlanes, int writerFmt);
private synchronized native int nativeGetWidth();
private synchronized native int nativeGetHeight();
private synchronized native int nativeGetFormat();
}
// Native implemented ImageWriter methods.
private synchronized native long nativeInit(Object weakSelf, Surface surface, int maxImgs);
private synchronized native void nativeClose(long nativeCtx);
private synchronized native void nativeAttachImage(long nativeCtx, Image image);
private synchronized native void nativeDequeueInputImage(long nativeCtx, Image wi);
private synchronized native void nativeQueueInputImage(long nativeCtx, Image image,
long timestampNs, int left, int top, int right, int bottom);
private synchronized native void cancelImage(long nativeCtx, Image image);
/**
* We use a class initializer to allow the native code to cache some field
* offsets.
*/
private static native void nativeClassInit();
static {
System.loadLibrary("media_jni");
nativeClassInit();
}
}

1
media/jni/Android.mk
View File

@@ -2,6 +2,7 @@ LOCAL_PATH:= $(call my-dir)
include $(CLEAR_VARS)
LOCAL_SRC_FILES:= \
android_media_ImageWriter.cpp \
android_media_ImageReader.cpp \
android_media_MediaCrypto.cpp \
android_media_MediaCodec.cpp \

20
media/jni/android_media_ImageReader.cpp
View File

@@ -860,6 +860,25 @@ static jint ImageReader_imageSetup(JNIEnv* env, jobject thiz,
return ACQUIRE_SUCCESS;
}
static void ImageReader_detachImage(JNIEnv* env, jobject thiz, jobject image) {
ALOGV("%s:", __FUNCTION__);
JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz);
if (ctx == NULL) {
jniThrowException(env, "java/lang/IllegalStateException", "ImageReader was already closed");
return;
}
// CpuConsumer* consumer = ctx->getCpuConsumer();
CpuConsumer::LockedBuffer* buffer = Image_getLockedBuffer(env, image);
if (!buffer) {
ALOGW("Image already released!!!");
return;
}
// TODO: need implement
jniThrowRuntimeException(env, "nativeDetachImage is not implemented yet!!!");
}
static jobject ImageReader_getSurface(JNIEnv* env, jobject thiz)
{
ALOGV("%s: ", __FUNCTION__);
@@ -961,6 +980,7 @@ static JNINativeMethod gImageReaderMethods[] = {
{"nativeReleaseImage", "(Landroid/media/Image;)V", (void*)ImageReader_imageRelease },
{"nativeImageSetup", "(Landroid/media/Image;)I", (void*)ImageReader_imageSetup },
{"nativeGetSurface", "()Landroid/view/Surface;", (void*)ImageReader_getSurface },
{"nativeDetachImage", "(Landroid/media/Image;)V", (void*)ImageReader_detachImage },
};
static JNINativeMethod gImageMethods[] = {

1014
media/jni/android_media_ImageWriter.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

7
media/jni/android_media_MediaPlayer.cpp
View File

@@ -914,8 +914,8 @@ static int register_android_media_MediaPlayer(JNIEnv *env)
return AndroidRuntime::registerNativeMethods(env,
"android/media/MediaPlayer", gMethods, NELEM(gMethods));
}
extern int register_android_media_ImageReader(JNIEnv *env);
extern int register_android_media_ImageWriter(JNIEnv *env);
extern int register_android_media_Crypto(JNIEnv *env);
extern int register_android_media_Drm(JNIEnv *env);
extern int register_android_media_MediaCodec(JNIEnv *env);
@@ -944,6 +944,11 @@ jint JNI_OnLoad(JavaVM* vm, void* /* reserved */)
}
assert(env != NULL);
if (register_android_media_ImageWriter(env) != JNI_OK) {
ALOGE("ERROR: ImageWriter native registration failed");
goto bail;
}
if (register_android_media_ImageReader(env) < 0) {
ALOGE("ERROR: ImageReader native registration failed");
goto bail;