Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge "ImageReader/Writer: add usage flag support" into oc-dev

Browse Source
This commit is contained in:
TreeHugger Robot
2017-04-11 16:17:11 +00:00
committed by Android (Google) Code Review
parent 3151a09dac 916d8ac650
commit 96d4ac0cc6
5 changed files with 227 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

133
media/java/android/media/ImageReader.java
View File

@@ -17,6 +17,8 @@
package android.media;
import android.graphics.ImageFormat;
import android.graphics.PixelFormat;
import android.hardware.HardwareBuffer;
import android.os.Handler;
import android.os.Looper;
import android.os.Message;
@@ -71,6 +73,12 @@ public class ImageReader implements AutoCloseable {
*/
private static final int ACQUIRE_MAX_IMAGES = 2;
/**
* Invalid consumer buffer usage flag. This usage flag will be ignored
* by the {@code ImageReader} instance is constructed with this value.
*/
private static final long BUFFER_USAGE_UNKNOWN = 0;
/**
* <p>
* Create a new reader for images of the desired size and format.
@@ -121,13 +129,104 @@ public class ImageReader implements AutoCloseable {
* @see Image
*/
public static ImageReader newInstance(int width, int height, int format, int maxImages) {
return new ImageReader(width, height, format, maxImages);
return new ImageReader(width, height, format, maxImages, BUFFER_USAGE_UNKNOWN);
}
/**
* <p>
* Create a new reader for images of the desired size, format and consumer usage flag.
* </p>
* <p>
* The {@code maxImages} parameter determines the maximum number of {@link Image} objects that
* can be be acquired from the {@code ImageReader} simultaneously. Requesting more buffers will
* use up more memory, so it is important to use only the minimum number necessary for the use
* case.
* </p>
* <p>
* The valid sizes and formats depend on the source of the image data.
* </p>
* <p>
* The format and usage flag combination describes how the buffer will be used by
* consumer end-points. For example, if the application intends to send the images to
* {@link android.media.MediaCodec} or {@link android.media.MediaRecorder} for hardware video
* encoding, the format and usage flag combination needs to be
* {@link ImageFormat#PRIVATE PRIVATE} and {@link HardwareBuffer#USAGE0_VIDEO_ENCODE}. When an
* {@link ImageReader} object is created with a valid size and such format/usage flag
* combination, the application can send the {@link Image images} to an {@link ImageWriter} that
* is created with the input {@link android.view.Surface} provided by the
* {@link android.media.MediaCodec} or {@link android.media.MediaRecorder}.
* </p>
* <p>
* If the {@code format} is {@link ImageFormat#PRIVATE PRIVATE}, the created {@link ImageReader}
* will produce images that are not directly accessible by the application. The application can
* still acquire images from this {@link ImageReader}, and send them to the
* {@link android.hardware.camera2.CameraDevice camera} for reprocessing, or to the
* {@link android.media.MediaCodec} / {@link android.media.MediaRecorder} for hardware video
* encoding via {@link ImageWriter} interface. However, the {@link Image#getPlanes()
* getPlanes()} will return an empty array for {@link ImageFormat#PRIVATE PRIVATE} format
* images. The application can check if an existing reader's format by calling
* {@link #getImageFormat()}.
* </p>
* <p>
* {@link ImageFormat#PRIVATE PRIVATE} format {@link ImageReader ImageReaders} are more
* efficient to use when application access to image data is not necessary, compared to
* ImageReaders using other format such as {@link ImageFormat#YUV_420_888 YUV_420_888}.
* </p>
* <p>
* Note that not all format and usage flag combination is supported by the
* {@link ImageReader}. Below are the supported combinations by the {@link ImageReader}
* (assuming the consumer end-points support the such image consumption, e.g., hardware video
* encoding).
* <table>
* <tr>
* <th>Format</th>
* <th>Compatible usage flags</th>
* </tr>
* <tr>
* <td>non-{@link android.graphics.ImageFormat#PRIVATE PRIVATE} formats defined by
* {@link android.graphics.ImageFormat ImageFormat} or
* {@link android.graphics.PixelFormat PixelFormat}</td>
* <td>{@link HardwareBuffer#USAGE0_CPU_READ} or
* {@link HardwareBuffer#USAGE0_CPU_READ_OFTEN}</td>
* </tr>
* <tr>
* <td>{@link android.graphics.ImageFormat#PRIVATE}</td>
* <td>{@link HardwareBuffer#USAGE0_VIDEO_ENCODE} or
* {@link HardwareBuffer#USAGE0_GPU_SAMPLED_IMAGE}, or combined</td>
* </tr>
* </table>
* Using other combinations may result in {@link IllegalArgumentException}.
* </p>
* @param width The default width in pixels of the Images that this reader will produce.
* @param height The default height in pixels of the Images that this reader will produce.
* @param format The format of the Image that this reader will produce. This must be one of the
* {@link android.graphics.ImageFormat} or {@link android.graphics.PixelFormat}
* constants. Note that not all formats are supported, like ImageFormat.NV21.
* @param maxImages The maximum number of images the user will want to access simultaneously.
* This should be as small as possible to limit memory use. Once maxImages Images are
* obtained by the user, one of them has to be released before a new Image will
* become available for access through {@link #acquireLatestImage()} or
* {@link #acquireNextImage()}. Must be greater than 0.
* @param usage The intended usage of the images produced by this ImageReader. It needs
* to be one of the Usage0 defined by {@link HardwareBuffer}, or an
* {@link IllegalArgumentException} will be thrown.
* @see Image
* @see HardwareBuffer
* @hide
*/
public static ImageReader newInstance(int width, int height, int format, int maxImages,
long usage) {
if (!isFormatUsageCombinationAllowed(format, usage)) {
throw new IllegalArgumentException("Format usage combination is not supported:"
+ " format = " + format + ", usage = " + usage);
}
return new ImageReader(width, height, format, maxImages, usage);
}
/**
* @hide
*/
protected ImageReader(int width, int height, int format, int maxImages) {
protected ImageReader(int width, int height, int format, int maxImages, long usage) {
mWidth = width;
mHeight = height;
mFormat = format;
@@ -149,7 +248,7 @@ public class ImageReader implements AutoCloseable {
mNumPlanes = ImageUtils.getNumPlanesForFormat(mFormat);
nativeInit(new WeakReference<ImageReader>(this), width, height, format, maxImages);
nativeInit(new WeakReference<>(this), width, height, format, maxImages, usage);
mSurface = nativeGetSurface();
@@ -617,6 +716,30 @@ public class ImageReader implements AutoCloseable {
return si.getReader() == this;
}
private static boolean isFormatUsageCombinationAllowed(int format, long usage) {
if (!ImageFormat.isPublicFormat(format) && !PixelFormat.isPublicFormat(format)) {
return false;
}
// Valid usage needs to be provided.
if (usage == BUFFER_USAGE_UNKNOWN) {
return false;
}
if (format == ImageFormat.PRIVATE) {
// Usage need to be either USAGE0_GPU_SAMPLED_IMAGE or USAGE0_VIDEO_ENCODE or combined.
boolean isAllowed = (usage == HardwareBuffer.USAGE0_GPU_SAMPLED_IMAGE);
isAllowed = isAllowed || (usage == HardwareBuffer.USAGE0_VIDEO_ENCODE);
isAllowed = isAllowed || (usage ==
(HardwareBuffer.USAGE0_VIDEO_ENCODE | HardwareBuffer.USAGE0_GPU_SAMPLED_IMAGE));
return isAllowed;
} else {
// Usage need to make the buffer CPU readable for explicit format.
return ((usage == HardwareBuffer.USAGE0_CPU_READ) ||
(usage == HardwareBuffer.USAGE0_CPU_READ_OFTEN));
}
}
/**
* Called from Native code when an Event happens.
*
@@ -655,7 +778,7 @@ public class ImageReader implements AutoCloseable {
private ListenerHandler mListenerHandler;
// Keep track of the successfully acquired Images. This need to be thread safe as the images
// could be closed by different threads (e.g., application thread and GC thread).
private List<Image> mAcquiredImages = new CopyOnWriteArrayList<Image>();
private List<Image> mAcquiredImages = new CopyOnWriteArrayList<>();
/**
* This field is used by native code, do not access or modify.
@@ -896,7 +1019,7 @@ public class ImageReader implements AutoCloseable {
}
private synchronized native void nativeInit(Object weakSelf, int w, int h,
int fmt, int maxImgs);
int fmt, int maxImgs, long consumerUsage);
private synchronized native void nativeClose();
private synchronized native void nativeReleaseImage(Image i);
private synchronized native Surface nativeGetSurface();

66
media/java/android/media/ImageWriter.java
View File

@@ -17,6 +17,7 @@
package android.media;
import android.graphics.ImageFormat;
import android.graphics.PixelFormat;
import android.graphics.Rect;
import android.hardware.camera2.utils.SurfaceUtils;
import android.os.Handler;
@@ -89,7 +90,7 @@ public class ImageWriter implements AutoCloseable {
private final int mMaxImages;
// Keep track of the currently dequeued Image. This need to be thread safe as the images
// could be closed by different threads (e.g., application thread and GC thread).
private List<Image> mDequeuedImages = new CopyOnWriteArrayList<Image>();
private List<Image> mDequeuedImages = new CopyOnWriteArrayList<>();
private int mEstimatedNativeAllocBytes;
/**
@@ -118,22 +119,75 @@ public class ImageWriter implements AutoCloseable {
* @return a new ImageWriter instance.
*/
public static ImageWriter newInstance(Surface surface, int maxImages) {
return new ImageWriter(surface, maxImages);
return new ImageWriter(surface, maxImages, ImageFormat.UNKNOWN);
}
/**
* <p>
* Create a new ImageWriter with given number of max Images and format.
* </p>
* <p>
* The {@code maxImages} parameter determines the maximum number of
* {@link Image} objects that can be be dequeued from the
* {@code ImageWriter} simultaneously. Requesting more buffers will use up
* more memory, so it is important to use only the minimum number necessary.
* </p>
* <p>
* The format specifies the image format of this ImageWriter. The format
* from the {@code surface} will be overridden with this format. For example,
* if the surface is obtained from a {@link android.graphics.SurfaceTexture}, the default
* format may be {@link PixelFormat#RGBA_8888}. If the application creates an ImageWriter
* with this surface and {@link ImageFormat#PRIVATE}, this ImageWriter will be able to operate
* with {@link ImageFormat#PRIVATE} Images.
* </p>
* <p>
* Note that the consumer end-point may or may not be able to support Images with different
* format, for such case, the application should only use this method if the consumer is able
* to consume such images.
* </p>
* <p>
* The input Image size depends on the Surface that is provided by
* the downstream consumer end-point.
* </p>
*
* @param surface The destination Surface this writer produces Image data
* into.
* @param maxImages The maximum number of Images the user will want to
* access simultaneously for producing Image data. This should be
* as small as possible to limit memory use. Once maxImages
* Images are dequeued by the user, one of them has to be queued
* back before a new Image can be dequeued for access via
* {@link #dequeueInputImage()}.
* @param format The format of this ImageWriter. It can be any valid format specified by
* {@link ImageFormat} or {@link PixelFormat}.
*
* @return a new ImageWriter instance.
* @hide
*/
public static ImageWriter newInstance(Surface surface, int maxImages, int format) {
if (!ImageFormat.isPublicFormat(format) && !PixelFormat.isPublicFormat(format)) {
throw new IllegalArgumentException("Invalid format is specified: " + format);
}
return new ImageWriter(surface, maxImages, format);
}
/**
* @hide
*/
protected ImageWriter(Surface surface, int maxImages) {
protected ImageWriter(Surface surface, int maxImages, int format) {
if (surface == null || maxImages < 1) {
throw new IllegalArgumentException("Illegal input argument: surface " + surface
+ ", maxImages: " + maxImages);
}
mMaxImages = maxImages;
if (format == ImageFormat.UNKNOWN) {
format = SurfaceUtils.getSurfaceFormat(surface);
}
// Note that the underlying BufferQueue is working in synchronous mode
// to avoid dropping any buffers.
mNativeContext = nativeInit(new WeakReference<ImageWriter>(this), surface, maxImages);
mNativeContext = nativeInit(new WeakReference<>(this), surface, maxImages, format);
// Estimate the native buffer allocation size and register it so it gets accounted for
// during GC. Note that this doesn't include the buffers required by the buffer queue
@@ -142,7 +196,6 @@ public class ImageWriter implements AutoCloseable {
// complex, and 1 buffer is enough for the VM to treat the ImageWriter as being of some
// size.
Size surfSize = SurfaceUtils.getSurfaceSize(surface);
int format = SurfaceUtils.getSurfaceFormat(surface);
mEstimatedNativeAllocBytes =
ImageUtils.getEstimatedNativeAllocBytes(surfSize.getWidth(),surfSize.getHeight(),
format, /*buffer count*/ 1);
@@ -809,7 +862,8 @@ public class ImageWriter implements AutoCloseable {
}
// Native implemented ImageWriter methods.
private synchronized native long nativeInit(Object weakSelf, Surface surface, int maxImgs);
private synchronized native long nativeInit(Object weakSelf, Surface surface, int maxImgs,
int format);
private synchronized native void nativeClose(long nativeCtx);

1
media/jni/Android.mk
View File

@@ -51,6 +51,7 @@ LOCAL_SHARED_LIBRARIES := \
libandroidfw
LOCAL_STATIC_LIBRARIES := \
libgrallocusage \
LOCAL_C_INCLUDES += \
external/libexif/ \

23
media/jni/android_media_ImageReader.cpp
View File

@@ -31,6 +31,8 @@
#include <android_runtime/AndroidRuntime.h>
#include <android_runtime/android_view_Surface.h>
#include <android_runtime/android_hardware_HardwareBuffer.h>
#include <grallocusage/GrallocUsageConversion.h>
#include <jni.h>
#include <JNIHelp.h>
@@ -42,6 +44,7 @@
#define ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID "mNativeBuffer"
#define ANDROID_MEDIA_SURFACEIMAGE_TS_JNI_ID "mTimestamp"
#define CONSUMER_BUFFER_USAGE_UNKNOWN 0;
// ----------------------------------------------------------------------------
using namespace android;
@@ -327,8 +330,8 @@ static void ImageReader_classInit(JNIEnv* env, jclass clazz)
"Can not find SurfacePlane constructor");
}
static void ImageReader_init(JNIEnv* env, jobject thiz, jobject weakThiz,
jint width, jint height, jint format, jint maxImages)
static void ImageReader_init(JNIEnv* env, jobject thiz, jobject weakThiz, jint width, jint height,
jint format, jint maxImages, jlong ndkUsage)
{
status_t res;
int nativeFormat;
@@ -358,17 +361,29 @@ static void ImageReader_init(JNIEnv* env, jobject thiz, jobject weakThiz,
width, height, format, maxImages, getpid(),
createProcessUniqueId());
uint32_t consumerUsage = GRALLOC_USAGE_SW_READ_OFTEN;
bool needUsageOverride = ndkUsage != CONSUMER_BUFFER_USAGE_UNKNOWN;
uint64_t outProducerUsage = 0;
uint64_t outConsumerUsage = 0;
android_hardware_HardwareBuffer_convertToGrallocUsageBits(&outProducerUsage, &outConsumerUsage,
ndkUsage, 0);
if (isFormatOpaque(nativeFormat)) {
// Use the SW_READ_NEVER usage to tell producer that this format is not for preview or video
// encoding. The only possibility will be ZSL output.
consumerUsage = GRALLOC_USAGE_SW_READ_NEVER;
if (needUsageOverride) {
consumerUsage = android_convertGralloc1To0Usage(0, outConsumerUsage);
}
} else if (needUsageOverride) {
ALOGW("Consumer usage override for non-opaque format is not implemented yet, "
"ignore the provided usage from the application");
}
bufferConsumer = new BufferItemConsumer(gbConsumer, consumerUsage, maxImages,
/*controlledByApp*/true);
if (bufferConsumer == nullptr) {
jniThrowExceptionFmt(env, "java/lang/RuntimeException",
"Failed to allocate native buffer consumer for format 0x%x", nativeFormat);
"Failed to allocate native buffer consumer for format 0x%x and usage 0x%x",
nativeFormat, consumerUsage);
return;
}
ctx->setBufferConsumer(bufferConsumer);
@@ -788,7 +803,7 @@ static jint Image_getFormat(JNIEnv* env, jobject thiz, jint readerFormat)
static const JNINativeMethod gImageReaderMethods[] = {
{"nativeClassInit", "()V", (void*)ImageReader_classInit },
{"nativeInit", "(Ljava/lang/Object;IIII)V", (void*)ImageReader_init },
{"nativeInit", "(Ljava/lang/Object;IIIIJ)V", (void*)ImageReader_init },
{"nativeClose", "()V", (void*)ImageReader_close },
{"nativeReleaseImage", "(Landroid/media/Image;)V", (void*)ImageReader_imageRelease },
{"nativeImageSetup", "(Landroid/media/Image;)I", (void*)ImageReader_imageSetup },

32
media/jni/android_media_ImageWriter.cpp
View File

@@ -33,7 +33,7 @@
#include <inttypes.h>
#define IMAGE_BUFFER_JNI_ID "mNativeBuffer"
#define IMAGE_FORMAT_UNKNOWN 0 // This is the same value as ImageFormat#UNKNOWN.
// ----------------------------------------------------------------------------
using namespace android;
@@ -222,7 +222,7 @@ static void ImageWriter_classInit(JNIEnv* env, jclass clazz) {
}
static jlong ImageWriter_init(JNIEnv* env, jobject thiz, jobject weakThiz, jobject jsurface,
jint maxImages) {
jint maxImages, jint userFormat) {
status_t res;
ALOGV("%s: maxImages:%d", __FUNCTION__, maxImages);
@@ -255,7 +255,7 @@ static jlong ImageWriter_init(JNIEnv* env, jobject thiz, jobject weakThiz, jobje
// Get the dimension and format of the producer.
sp<ANativeWindow> anw = producer;
int32_t width, height, format;
int32_t width, height, surfaceFormat;
if ((res = anw->query(anw.get(), NATIVE_WINDOW_WIDTH, &width)) != OK) {
ALOGE("%s: Query Surface width failed: %s (%d)", __FUNCTION__, strerror(-res), res);
jniThrowRuntimeException(env, "Failed to query Surface width");
@@ -270,21 +270,27 @@ static jlong ImageWriter_init(JNIEnv* env, jobject thiz, jobject weakThiz, jobje
}
ctx->setBufferHeight(height);
if ((res = anw->query(anw.get(), NATIVE_WINDOW_FORMAT, &format)) != OK) {
ALOGE("%s: Query Surface format failed: %s (%d)", __FUNCTION__, strerror(-res), res);
jniThrowRuntimeException(env, "Failed to query Surface format");
return 0;
// Query surface format if no valid user format is specified, otherwise, override surface format
// with user format.
if (userFormat == IMAGE_FORMAT_UNKNOWN) {
if ((res = anw->query(anw.get(), NATIVE_WINDOW_FORMAT, &surfaceFormat)) != OK) {
ALOGE("%s: Query Surface format failed: %s (%d)", __FUNCTION__, strerror(-res), res);
jniThrowRuntimeException(env, "Failed to query Surface format");
return 0;
}
} else {
surfaceFormat = userFormat;
}
ctx->setBufferFormat(format);
env->SetIntField(thiz, gImageWriterClassInfo.mWriterFormat, reinterpret_cast<jint>(format));
ctx->setBufferFormat(surfaceFormat);
env->SetIntField(thiz,
gImageWriterClassInfo.mWriterFormat, reinterpret_cast<jint>(surfaceFormat));
if (!isFormatOpaque(format)) {
if (!isFormatOpaque(surfaceFormat)) {
res = native_window_set_usage(anw.get(), GRALLOC_USAGE_SW_WRITE_OFTEN);
if (res != OK) {
ALOGE("%s: Configure usage %08x for format %08x failed: %s (%d)",
__FUNCTION__, static_cast<unsigned int>(GRALLOC_USAGE_SW_WRITE_OFTEN),
format, strerror(-res), res);
surfaceFormat, strerror(-res), res);
jniThrowRuntimeException(env, "Failed to SW_WRITE_OFTEN configure usage");
return 0;
}
@@ -784,7 +790,7 @@ static jobjectArray Image_createSurfacePlanes(JNIEnv* env, jobject thiz,
static JNINativeMethod gImageWriterMethods[] = {
{"nativeClassInit", "()V", (void*)ImageWriter_classInit },
{"nativeInit", "(Ljava/lang/Object;Landroid/view/Surface;I)J",
{"nativeInit", "(Ljava/lang/Object;Landroid/view/Surface;II)J",
(void*)ImageWriter_init },
{"nativeClose", "(J)V", (void*)ImageWriter_close },
{"nativeAttachAndQueueImage", "(JJIJIIII)I", (void*)ImageWriter_attachAndQueueImage },