From 916d8ac650865cd05808d48bad68b69bebbc95ab Mon Sep 17 00:00:00 2001 From: Zhijun He Date: Wed, 22 Mar 2017 17:33:47 -0700 Subject: [PATCH] ImageReader/Writer: add usage flag support Also add an ImageWriter ctor to take additional arugment (format) Test: ImageReader and Writer CTS tests Bug: 32766711 Change-Id: I99e3862dd5b9a85c9df7879c14c84b68a35718ec --- media/java/android/media/ImageReader.java | 133 +++++++++++++++++++++- media/java/android/media/ImageWriter.java | 66 ++++++++++- media/jni/Android.mk | 1 + media/jni/android_media_ImageReader.cpp | 23 +++- media/jni/android_media_ImageWriter.cpp | 32 +++--- 5 files changed, 227 insertions(+), 28 deletions(-) diff --git a/media/java/android/media/ImageReader.java b/media/java/android/media/ImageReader.java index ed718493bd789..06555c1fa90b6 100644 --- a/media/java/android/media/ImageReader.java +++ b/media/java/android/media/ImageReader.java @@ -17,6 +17,8 @@ package android.media; import android.graphics.ImageFormat; +import android.graphics.PixelFormat; +import android.hardware.HardwareBuffer; import android.os.Handler; import android.os.Looper; import android.os.Message; @@ -71,6 +73,12 @@ public class ImageReader implements AutoCloseable { */ private static final int ACQUIRE_MAX_IMAGES = 2; + /** + * Invalid consumer buffer usage flag. This usage flag will be ignored + * by the {@code ImageReader} instance is constructed with this value. + */ + private static final long BUFFER_USAGE_UNKNOWN = 0; + /** *

* Create a new reader for images of the desired size and format. @@ -121,13 +129,104 @@ public class ImageReader implements AutoCloseable { * @see Image */ public static ImageReader newInstance(int width, int height, int format, int maxImages) { - return new ImageReader(width, height, format, maxImages); + return new ImageReader(width, height, format, maxImages, BUFFER_USAGE_UNKNOWN); + } + + /** + *

+ * Create a new reader for images of the desired size, format and consumer usage flag. + *

+ *

+ * The {@code maxImages} parameter determines the maximum number of {@link Image} objects that + * can be be acquired from the {@code ImageReader} simultaneously. Requesting more buffers will + * use up more memory, so it is important to use only the minimum number necessary for the use + * case. + *

+ *

+ * The valid sizes and formats depend on the source of the image data. + *

+ *

+ * The format and usage flag combination describes how the buffer will be used by + * consumer end-points. For example, if the application intends to send the images to + * {@link android.media.MediaCodec} or {@link android.media.MediaRecorder} for hardware video + * encoding, the format and usage flag combination needs to be + * {@link ImageFormat#PRIVATE PRIVATE} and {@link HardwareBuffer#USAGE0_VIDEO_ENCODE}. When an + * {@link ImageReader} object is created with a valid size and such format/usage flag + * combination, the application can send the {@link Image images} to an {@link ImageWriter} that + * is created with the input {@link android.view.Surface} provided by the + * {@link android.media.MediaCodec} or {@link android.media.MediaRecorder}. + *

+ *

+ * If the {@code format} is {@link ImageFormat#PRIVATE PRIVATE}, the created {@link ImageReader} + * will produce images that are not directly accessible by the application. The application can + * still acquire images from this {@link ImageReader}, and send them to the + * {@link android.hardware.camera2.CameraDevice camera} for reprocessing, or to the + * {@link android.media.MediaCodec} / {@link android.media.MediaRecorder} for hardware video + * encoding via {@link ImageWriter} interface. However, the {@link Image#getPlanes() + * getPlanes()} will return an empty array for {@link ImageFormat#PRIVATE PRIVATE} format + * images. The application can check if an existing reader's format by calling + * {@link #getImageFormat()}. + *

+ *

+ * {@link ImageFormat#PRIVATE PRIVATE} format {@link ImageReader ImageReaders} are more + * efficient to use when application access to image data is not necessary, compared to + * ImageReaders using other format such as {@link ImageFormat#YUV_420_888 YUV_420_888}. + *

+ *

+ * Note that not all format and usage flag combination is supported by the + * {@link ImageReader}. Below are the supported combinations by the {@link ImageReader} + * (assuming the consumer end-points support the such image consumption, e.g., hardware video + * encoding). + * + * + * + * + * + * + * + * + * + * + * + * + * + *
FormatCompatible usage flags
non-{@link android.graphics.ImageFormat#PRIVATE PRIVATE} formats defined by + * {@link android.graphics.ImageFormat ImageFormat} or + * {@link android.graphics.PixelFormat PixelFormat}{@link HardwareBuffer#USAGE0_CPU_READ} or + * {@link HardwareBuffer#USAGE0_CPU_READ_OFTEN}
{@link android.graphics.ImageFormat#PRIVATE}{@link HardwareBuffer#USAGE0_VIDEO_ENCODE} or + * {@link HardwareBuffer#USAGE0_GPU_SAMPLED_IMAGE}, or combined
+ * Using other combinations may result in {@link IllegalArgumentException}. + *

+ * @param width The default width in pixels of the Images that this reader will produce. + * @param height The default height in pixels of the Images that this reader will produce. + * @param format The format of the Image that this reader will produce. This must be one of the + * {@link android.graphics.ImageFormat} or {@link android.graphics.PixelFormat} + * constants. Note that not all formats are supported, like ImageFormat.NV21. + * @param maxImages The maximum number of images the user will want to access simultaneously. + * This should be as small as possible to limit memory use. Once maxImages Images are + * obtained by the user, one of them has to be released before a new Image will + * become available for access through {@link #acquireLatestImage()} or + * {@link #acquireNextImage()}. Must be greater than 0. + * @param usage The intended usage of the images produced by this ImageReader. It needs + * to be one of the Usage0 defined by {@link HardwareBuffer}, or an + * {@link IllegalArgumentException} will be thrown. + * @see Image + * @see HardwareBuffer + * @hide + */ + public static ImageReader newInstance(int width, int height, int format, int maxImages, + long usage) { + if (!isFormatUsageCombinationAllowed(format, usage)) { + throw new IllegalArgumentException("Format usage combination is not supported:" + + " format = " + format + ", usage = " + usage); + } + return new ImageReader(width, height, format, maxImages, usage); } /** * @hide */ - protected ImageReader(int width, int height, int format, int maxImages) { + protected ImageReader(int width, int height, int format, int maxImages, long usage) { mWidth = width; mHeight = height; mFormat = format; @@ -149,7 +248,7 @@ public class ImageReader implements AutoCloseable { mNumPlanes = ImageUtils.getNumPlanesForFormat(mFormat); - nativeInit(new WeakReference(this), width, height, format, maxImages); + nativeInit(new WeakReference<>(this), width, height, format, maxImages, usage); mSurface = nativeGetSurface(); @@ -617,6 +716,30 @@ public class ImageReader implements AutoCloseable { return si.getReader() == this; } + private static boolean isFormatUsageCombinationAllowed(int format, long usage) { + if (!ImageFormat.isPublicFormat(format) && !PixelFormat.isPublicFormat(format)) { + return false; + } + + // Valid usage needs to be provided. + if (usage == BUFFER_USAGE_UNKNOWN) { + return false; + } + + if (format == ImageFormat.PRIVATE) { + // Usage need to be either USAGE0_GPU_SAMPLED_IMAGE or USAGE0_VIDEO_ENCODE or combined. + boolean isAllowed = (usage == HardwareBuffer.USAGE0_GPU_SAMPLED_IMAGE); + isAllowed = isAllowed || (usage == HardwareBuffer.USAGE0_VIDEO_ENCODE); + isAllowed = isAllowed || (usage == + (HardwareBuffer.USAGE0_VIDEO_ENCODE | HardwareBuffer.USAGE0_GPU_SAMPLED_IMAGE)); + return isAllowed; + } else { + // Usage need to make the buffer CPU readable for explicit format. + return ((usage == HardwareBuffer.USAGE0_CPU_READ) || + (usage == HardwareBuffer.USAGE0_CPU_READ_OFTEN)); + } + } + /** * Called from Native code when an Event happens. * @@ -655,7 +778,7 @@ public class ImageReader implements AutoCloseable { private ListenerHandler mListenerHandler; // Keep track of the successfully acquired Images. This need to be thread safe as the images // could be closed by different threads (e.g., application thread and GC thread). - private List mAcquiredImages = new CopyOnWriteArrayList(); + private List mAcquiredImages = new CopyOnWriteArrayList<>(); /** * This field is used by native code, do not access or modify. @@ -896,7 +1019,7 @@ public class ImageReader implements AutoCloseable { } private synchronized native void nativeInit(Object weakSelf, int w, int h, - int fmt, int maxImgs); + int fmt, int maxImgs, long consumerUsage); private synchronized native void nativeClose(); private synchronized native void nativeReleaseImage(Image i); private synchronized native Surface nativeGetSurface(); diff --git a/media/java/android/media/ImageWriter.java b/media/java/android/media/ImageWriter.java index b142ddd9fbff5..349c9cb948602 100644 --- a/media/java/android/media/ImageWriter.java +++ b/media/java/android/media/ImageWriter.java @@ -17,6 +17,7 @@ package android.media; import android.graphics.ImageFormat; +import android.graphics.PixelFormat; import android.graphics.Rect; import android.hardware.camera2.utils.SurfaceUtils; import android.os.Handler; @@ -89,7 +90,7 @@ public class ImageWriter implements AutoCloseable { private final int mMaxImages; // Keep track of the currently dequeued Image. This need to be thread safe as the images // could be closed by different threads (e.g., application thread and GC thread). - private List mDequeuedImages = new CopyOnWriteArrayList(); + private List mDequeuedImages = new CopyOnWriteArrayList<>(); private int mEstimatedNativeAllocBytes; /** @@ -118,22 +119,75 @@ public class ImageWriter implements AutoCloseable { * @return a new ImageWriter instance. */ public static ImageWriter newInstance(Surface surface, int maxImages) { - return new ImageWriter(surface, maxImages); + return new ImageWriter(surface, maxImages, ImageFormat.UNKNOWN); + } + + /** + *

+ * Create a new ImageWriter with given number of max Images and format. + *

+ *

+ * The {@code maxImages} parameter determines the maximum number of + * {@link Image} objects that can be be dequeued from the + * {@code ImageWriter} simultaneously. Requesting more buffers will use up + * more memory, so it is important to use only the minimum number necessary. + *

+ *

+ * The format specifies the image format of this ImageWriter. The format + * from the {@code surface} will be overridden with this format. For example, + * if the surface is obtained from a {@link android.graphics.SurfaceTexture}, the default + * format may be {@link PixelFormat#RGBA_8888}. If the application creates an ImageWriter + * with this surface and {@link ImageFormat#PRIVATE}, this ImageWriter will be able to operate + * with {@link ImageFormat#PRIVATE} Images. + *

+ *

+ * Note that the consumer end-point may or may not be able to support Images with different + * format, for such case, the application should only use this method if the consumer is able + * to consume such images. + *

+ *

+ * The input Image size depends on the Surface that is provided by + * the downstream consumer end-point. + *

+ * + * @param surface The destination Surface this writer produces Image data + * into. + * @param maxImages The maximum number of Images the user will want to + * access simultaneously for producing Image data. This should be + * as small as possible to limit memory use. Once maxImages + * Images are dequeued by the user, one of them has to be queued + * back before a new Image can be dequeued for access via + * {@link #dequeueInputImage()}. + * @param format The format of this ImageWriter. It can be any valid format specified by + * {@link ImageFormat} or {@link PixelFormat}. + * + * @return a new ImageWriter instance. + * @hide + */ + public static ImageWriter newInstance(Surface surface, int maxImages, int format) { + if (!ImageFormat.isPublicFormat(format) && !PixelFormat.isPublicFormat(format)) { + throw new IllegalArgumentException("Invalid format is specified: " + format); + } + return new ImageWriter(surface, maxImages, format); } /** * @hide */ - protected ImageWriter(Surface surface, int maxImages) { + protected ImageWriter(Surface surface, int maxImages, int format) { if (surface == null || maxImages < 1) { throw new IllegalArgumentException("Illegal input argument: surface " + surface + ", maxImages: " + maxImages); } mMaxImages = maxImages; + + if (format == ImageFormat.UNKNOWN) { + format = SurfaceUtils.getSurfaceFormat(surface); + } // Note that the underlying BufferQueue is working in synchronous mode // to avoid dropping any buffers. - mNativeContext = nativeInit(new WeakReference(this), surface, maxImages); + mNativeContext = nativeInit(new WeakReference<>(this), surface, maxImages, format); // Estimate the native buffer allocation size and register it so it gets accounted for // during GC. Note that this doesn't include the buffers required by the buffer queue @@ -142,7 +196,6 @@ public class ImageWriter implements AutoCloseable { // complex, and 1 buffer is enough for the VM to treat the ImageWriter as being of some // size. Size surfSize = SurfaceUtils.getSurfaceSize(surface); - int format = SurfaceUtils.getSurfaceFormat(surface); mEstimatedNativeAllocBytes = ImageUtils.getEstimatedNativeAllocBytes(surfSize.getWidth(),surfSize.getHeight(), format, /*buffer count*/ 1); @@ -809,7 +862,8 @@ public class ImageWriter implements AutoCloseable { } // Native implemented ImageWriter methods. - private synchronized native long nativeInit(Object weakSelf, Surface surface, int maxImgs); + private synchronized native long nativeInit(Object weakSelf, Surface surface, int maxImgs, + int format); private synchronized native void nativeClose(long nativeCtx); diff --git a/media/jni/Android.mk b/media/jni/Android.mk index 431d5d83a4bb6..c2ed8cf15c1ab 100644 --- a/media/jni/Android.mk +++ b/media/jni/Android.mk @@ -51,6 +51,7 @@ LOCAL_SHARED_LIBRARIES := \ libandroidfw LOCAL_STATIC_LIBRARIES := \ + libgrallocusage \ LOCAL_C_INCLUDES += \ external/libexif/ \ diff --git a/media/jni/android_media_ImageReader.cpp b/media/jni/android_media_ImageReader.cpp index f5e19f908d719..163c4b0122626 100644 --- a/media/jni/android_media_ImageReader.cpp +++ b/media/jni/android_media_ImageReader.cpp @@ -31,6 +31,8 @@ #include #include +#include +#include #include #include @@ -42,6 +44,7 @@ #define ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID "mNativeBuffer" #define ANDROID_MEDIA_SURFACEIMAGE_TS_JNI_ID "mTimestamp" +#define CONSUMER_BUFFER_USAGE_UNKNOWN 0; // ---------------------------------------------------------------------------- using namespace android; @@ -327,8 +330,8 @@ static void ImageReader_classInit(JNIEnv* env, jclass clazz) "Can not find SurfacePlane constructor"); } -static void ImageReader_init(JNIEnv* env, jobject thiz, jobject weakThiz, - jint width, jint height, jint format, jint maxImages) +static void ImageReader_init(JNIEnv* env, jobject thiz, jobject weakThiz, jint width, jint height, + jint format, jint maxImages, jlong ndkUsage) { status_t res; int nativeFormat; @@ -358,17 +361,29 @@ static void ImageReader_init(JNIEnv* env, jobject thiz, jobject weakThiz, width, height, format, maxImages, getpid(), createProcessUniqueId()); uint32_t consumerUsage = GRALLOC_USAGE_SW_READ_OFTEN; + bool needUsageOverride = ndkUsage != CONSUMER_BUFFER_USAGE_UNKNOWN; + uint64_t outProducerUsage = 0; + uint64_t outConsumerUsage = 0; + android_hardware_HardwareBuffer_convertToGrallocUsageBits(&outProducerUsage, &outConsumerUsage, + ndkUsage, 0); if (isFormatOpaque(nativeFormat)) { // Use the SW_READ_NEVER usage to tell producer that this format is not for preview or video // encoding. The only possibility will be ZSL output. consumerUsage = GRALLOC_USAGE_SW_READ_NEVER; + if (needUsageOverride) { + consumerUsage = android_convertGralloc1To0Usage(0, outConsumerUsage); + } + } else if (needUsageOverride) { + ALOGW("Consumer usage override for non-opaque format is not implemented yet, " + "ignore the provided usage from the application"); } bufferConsumer = new BufferItemConsumer(gbConsumer, consumerUsage, maxImages, /*controlledByApp*/true); if (bufferConsumer == nullptr) { jniThrowExceptionFmt(env, "java/lang/RuntimeException", - "Failed to allocate native buffer consumer for format 0x%x", nativeFormat); + "Failed to allocate native buffer consumer for format 0x%x and usage 0x%x", + nativeFormat, consumerUsage); return; } ctx->setBufferConsumer(bufferConsumer); @@ -788,7 +803,7 @@ static jint Image_getFormat(JNIEnv* env, jobject thiz, jint readerFormat) static const JNINativeMethod gImageReaderMethods[] = { {"nativeClassInit", "()V", (void*)ImageReader_classInit }, - {"nativeInit", "(Ljava/lang/Object;IIII)V", (void*)ImageReader_init }, + {"nativeInit", "(Ljava/lang/Object;IIIIJ)V", (void*)ImageReader_init }, {"nativeClose", "()V", (void*)ImageReader_close }, {"nativeReleaseImage", "(Landroid/media/Image;)V", (void*)ImageReader_imageRelease }, {"nativeImageSetup", "(Landroid/media/Image;)I", (void*)ImageReader_imageSetup }, diff --git a/media/jni/android_media_ImageWriter.cpp b/media/jni/android_media_ImageWriter.cpp index 0149d76236605..ed5fbcfcd5b30 100644 --- a/media/jni/android_media_ImageWriter.cpp +++ b/media/jni/android_media_ImageWriter.cpp @@ -33,7 +33,7 @@ #include #define IMAGE_BUFFER_JNI_ID "mNativeBuffer" - +#define IMAGE_FORMAT_UNKNOWN 0 // This is the same value as ImageFormat#UNKNOWN. // ---------------------------------------------------------------------------- using namespace android; @@ -222,7 +222,7 @@ static void ImageWriter_classInit(JNIEnv* env, jclass clazz) { } static jlong ImageWriter_init(JNIEnv* env, jobject thiz, jobject weakThiz, jobject jsurface, - jint maxImages) { + jint maxImages, jint userFormat) { status_t res; ALOGV("%s: maxImages:%d", __FUNCTION__, maxImages); @@ -255,7 +255,7 @@ static jlong ImageWriter_init(JNIEnv* env, jobject thiz, jobject weakThiz, jobje // Get the dimension and format of the producer. sp anw = producer; - int32_t width, height, format; + int32_t width, height, surfaceFormat; if ((res = anw->query(anw.get(), NATIVE_WINDOW_WIDTH, &width)) != OK) { ALOGE("%s: Query Surface width failed: %s (%d)", __FUNCTION__, strerror(-res), res); jniThrowRuntimeException(env, "Failed to query Surface width"); @@ -270,21 +270,27 @@ static jlong ImageWriter_init(JNIEnv* env, jobject thiz, jobject weakThiz, jobje } ctx->setBufferHeight(height); - if ((res = anw->query(anw.get(), NATIVE_WINDOW_FORMAT, &format)) != OK) { - ALOGE("%s: Query Surface format failed: %s (%d)", __FUNCTION__, strerror(-res), res); - jniThrowRuntimeException(env, "Failed to query Surface format"); - return 0; + // Query surface format if no valid user format is specified, otherwise, override surface format + // with user format. + if (userFormat == IMAGE_FORMAT_UNKNOWN) { + if ((res = anw->query(anw.get(), NATIVE_WINDOW_FORMAT, &surfaceFormat)) != OK) { + ALOGE("%s: Query Surface format failed: %s (%d)", __FUNCTION__, strerror(-res), res); + jniThrowRuntimeException(env, "Failed to query Surface format"); + return 0; + } + } else { + surfaceFormat = userFormat; } - ctx->setBufferFormat(format); - env->SetIntField(thiz, gImageWriterClassInfo.mWriterFormat, reinterpret_cast(format)); + ctx->setBufferFormat(surfaceFormat); + env->SetIntField(thiz, + gImageWriterClassInfo.mWriterFormat, reinterpret_cast(surfaceFormat)); - - if (!isFormatOpaque(format)) { + if (!isFormatOpaque(surfaceFormat)) { res = native_window_set_usage(anw.get(), GRALLOC_USAGE_SW_WRITE_OFTEN); if (res != OK) { ALOGE("%s: Configure usage %08x for format %08x failed: %s (%d)", __FUNCTION__, static_cast(GRALLOC_USAGE_SW_WRITE_OFTEN), - format, strerror(-res), res); + surfaceFormat, strerror(-res), res); jniThrowRuntimeException(env, "Failed to SW_WRITE_OFTEN configure usage"); return 0; } @@ -784,7 +790,7 @@ static jobjectArray Image_createSurfacePlanes(JNIEnv* env, jobject thiz, static JNINativeMethod gImageWriterMethods[] = { {"nativeClassInit", "()V", (void*)ImageWriter_classInit }, - {"nativeInit", "(Ljava/lang/Object;Landroid/view/Surface;I)J", + {"nativeInit", "(Ljava/lang/Object;Landroid/view/Surface;II)J", (void*)ImageWriter_init }, {"nativeClose", "(J)V", (void*)ImageWriter_close }, {"nativeAttachAndQueueImage", "(JJIJIIII)I", (void*)ImageWriter_attachAndQueueImage },