Merge "ImageReader/Writer: add usage flag support" into oc-dev

am: 96d4ac0cc6

Change-Id: Ia98cc7f1caf6d5628e2aae1d78d159dc3ac6d261

media/java/android/media/ImageReader.java

@@ -17,6 +17,8 @@
 package android.media;
 
 import android.graphics.ImageFormat;
+import android.graphics.PixelFormat;
+import android.hardware.HardwareBuffer;
 import android.os.Handler;
 import android.os.Looper;
 import android.os.Message;
@@ -71,6 +73,12 @@ public class ImageReader implements AutoCloseable {
      */
     private static final int ACQUIRE_MAX_IMAGES = 2;
 
+    /**
+     * Invalid consumer buffer usage flag. This usage flag will be ignored
+     * by the {@code ImageReader} instance is constructed with this value.
+     */
+    private static final long BUFFER_USAGE_UNKNOWN = 0;
+
     /**
      * <p>
      * Create a new reader for images of the desired size and format.
@@ -121,13 +129,104 @@ public class ImageReader implements AutoCloseable {
      * @see Image
      */
     public static ImageReader newInstance(int width, int height, int format, int maxImages) {
-        return new ImageReader(width, height, format, maxImages);
+        return new ImageReader(width, height, format, maxImages, BUFFER_USAGE_UNKNOWN);
+    }
+
+    /**
+     * <p>
+     * Create a new reader for images of the desired size, format and consumer usage flag.
+     * </p>
+     * <p>
+     * The {@code maxImages} parameter determines the maximum number of {@link Image} objects that
+     * can be be acquired from the {@code ImageReader} simultaneously. Requesting more buffers will
+     * use up more memory, so it is important to use only the minimum number necessary for the use
+     * case.
+     * </p>
+     * <p>
+     * The valid sizes and formats depend on the source of the image data.
+     * </p>
+     * <p>
+     * The format and usage flag combination describes how the buffer will be used by
+     * consumer end-points. For example, if the application intends to send the images to
+     * {@link android.media.MediaCodec} or {@link android.media.MediaRecorder} for hardware video
+     * encoding, the format and usage flag combination needs to be
+     * {@link ImageFormat#PRIVATE PRIVATE} and {@link HardwareBuffer#USAGE0_VIDEO_ENCODE}. When an
+     * {@link ImageReader} object is created with a valid size and such format/usage flag
+     * combination, the application can send the {@link Image images} to an {@link ImageWriter} that
+     * is created with the input {@link android.view.Surface} provided by the
+     * {@link android.media.MediaCodec} or {@link android.media.MediaRecorder}.
+     * </p>
+     * <p>
+     * If the {@code format} is {@link ImageFormat#PRIVATE PRIVATE}, the created {@link ImageReader}
+     * will produce images that are not directly accessible by the application. The application can
+     * still acquire images from this {@link ImageReader}, and send them to the
+     * {@link android.hardware.camera2.CameraDevice camera} for reprocessing, or to the
+     * {@link android.media.MediaCodec} / {@link android.media.MediaRecorder} for hardware video
+     * encoding via {@link ImageWriter} interface. However, the {@link Image#getPlanes()
+     * getPlanes()} will return an empty array for {@link ImageFormat#PRIVATE PRIVATE} format
+     * images. The application can check if an existing reader's format by calling
+     * {@link #getImageFormat()}.
+     * </p>
+     * <p>
+     * {@link ImageFormat#PRIVATE PRIVATE} format {@link ImageReader ImageReaders} are more
+     * efficient to use when application access to image data is not necessary, compared to
+     * ImageReaders using other format such as {@link ImageFormat#YUV_420_888 YUV_420_888}.
+     * </p>
+     * <p>
+     * Note that not all format and usage flag combination is supported by the
+     * {@link ImageReader}. Below are the supported combinations by the {@link ImageReader}
+     * (assuming the consumer end-points support the such image consumption, e.g., hardware video
+     * encoding).
+     * <table>
+     * <tr>
+     *   <th>Format</th>
+     *   <th>Compatible usage flags</th>
+     * </tr>
+     * <tr>
+     *   <td>non-{@link android.graphics.ImageFormat#PRIVATE PRIVATE} formats defined by
+     *   {@link android.graphics.ImageFormat ImageFormat} or
+     *   {@link android.graphics.PixelFormat PixelFormat}</td>
+     *   <td>{@link HardwareBuffer#USAGE0_CPU_READ} or
+     *   {@link HardwareBuffer#USAGE0_CPU_READ_OFTEN}</td>
+     * </tr>
+     * <tr>
+     *   <td>{@link android.graphics.ImageFormat#PRIVATE}</td>
+     *   <td>{@link HardwareBuffer#USAGE0_VIDEO_ENCODE} or
+     *   {@link HardwareBuffer#USAGE0_GPU_SAMPLED_IMAGE}, or combined</td>
+     * </tr>
+     * </table>
+     * Using other combinations may result in {@link IllegalArgumentException}.
+     * </p>
+     * @param width The default width in pixels of the Images that this reader will produce.
+     * @param height The default height in pixels of the Images that this reader will produce.
+     * @param format The format of the Image that this reader will produce. This must be one of the
+     *            {@link android.graphics.ImageFormat} or {@link android.graphics.PixelFormat}
+     *            constants. Note that not all formats are supported, like ImageFormat.NV21.
+     * @param maxImages The maximum number of images the user will want to access simultaneously.
+     *            This should be as small as possible to limit memory use. Once maxImages Images are
+     *            obtained by the user, one of them has to be released before a new Image will
+     *            become available for access through {@link #acquireLatestImage()} or
+     *            {@link #acquireNextImage()}. Must be greater than 0.
+     * @param usage The intended usage of the images produced by this ImageReader. It needs
+     *            to be one of the Usage0 defined by {@link HardwareBuffer}, or an
+     *            {@link IllegalArgumentException} will be thrown.
+     * @see Image
+     * @see HardwareBuffer
+     * @hide
+     */
+    public static ImageReader newInstance(int width, int height, int format, int maxImages,
+            long usage) {
+        if (!isFormatUsageCombinationAllowed(format, usage)) {
+            throw new IllegalArgumentException("Format usage combination is not supported:"
+                    + " format = " + format + ", usage = " + usage);
+        }
+        return new ImageReader(width, height, format, maxImages, usage);
     }
 
     /**
      * @hide
      */
-    protected ImageReader(int width, int height, int format, int maxImages) {
+    protected ImageReader(int width, int height, int format, int maxImages, long usage) {
         mWidth = width;
         mHeight = height;
         mFormat = format;
@@ -149,7 +248,7 @@ public class ImageReader implements AutoCloseable {
 
         mNumPlanes = ImageUtils.getNumPlanesForFormat(mFormat);
 
-        nativeInit(new WeakReference<ImageReader>(this), width, height, format, maxImages);
+        nativeInit(new WeakReference<>(this), width, height, format, maxImages, usage);
 
         mSurface = nativeGetSurface();
 
@@ -617,6 +716,30 @@ public class ImageReader implements AutoCloseable {
         return si.getReader() == this;
     }
 
+    private static boolean isFormatUsageCombinationAllowed(int format, long usage) {
+        if (!ImageFormat.isPublicFormat(format) && !PixelFormat.isPublicFormat(format)) {
+            return false;
+        }
+
+        // Valid usage needs to be provided.
+        if (usage == BUFFER_USAGE_UNKNOWN) {
+            return false;
+        }
+
+        if (format == ImageFormat.PRIVATE) {
+            // Usage need to be either USAGE0_GPU_SAMPLED_IMAGE or USAGE0_VIDEO_ENCODE or combined.
+            boolean isAllowed = (usage == HardwareBuffer.USAGE0_GPU_SAMPLED_IMAGE);
+            isAllowed = isAllowed || (usage == HardwareBuffer.USAGE0_VIDEO_ENCODE);
+            isAllowed = isAllowed || (usage ==
+                    (HardwareBuffer.USAGE0_VIDEO_ENCODE | HardwareBuffer.USAGE0_GPU_SAMPLED_IMAGE));
+            return isAllowed;
+        } else {
+            // Usage need to make the buffer CPU readable for explicit format.
+            return ((usage == HardwareBuffer.USAGE0_CPU_READ) ||
+                    (usage == HardwareBuffer.USAGE0_CPU_READ_OFTEN));
+        }
+    }
+
     /**
      * Called from Native code when an Event happens.
      *
@@ -655,7 +778,7 @@ public class ImageReader implements AutoCloseable {
     private ListenerHandler mListenerHandler;
     // Keep track of the successfully acquired Images. This need to be thread safe as the images
     // could be closed by different threads (e.g., application thread and GC thread).
-    private List<Image> mAcquiredImages = new CopyOnWriteArrayList<Image>();
+    private List<Image> mAcquiredImages = new CopyOnWriteArrayList<>();
 
     /**
      * This field is used by native code, do not access or modify.
@@ -896,7 +1019,7 @@ public class ImageReader implements AutoCloseable {
     }
 
     private synchronized native void nativeInit(Object weakSelf, int w, int h,
-                                                    int fmt, int maxImgs);
+                                                    int fmt, int maxImgs, long consumerUsage);
     private synchronized native void nativeClose();
     private synchronized native void nativeReleaseImage(Image i);
     private synchronized native Surface nativeGetSurface();

media/java/android/media/ImageWriter.java

@@ -17,6 +17,7 @@
 package android.media;
 
 import android.graphics.ImageFormat;
+import android.graphics.PixelFormat;
 import android.graphics.Rect;
 import android.hardware.camera2.utils.SurfaceUtils;
 import android.os.Handler;
@@ -89,7 +90,7 @@ public class ImageWriter implements AutoCloseable {
     private final int mMaxImages;
     // Keep track of the currently dequeued Image. This need to be thread safe as the images
     // could be closed by different threads (e.g., application thread and GC thread).
-    private List<Image> mDequeuedImages = new CopyOnWriteArrayList<Image>();
+    private List<Image> mDequeuedImages = new CopyOnWriteArrayList<>();
     private int mEstimatedNativeAllocBytes;
 
     /**
@@ -118,22 +119,75 @@ public class ImageWriter implements AutoCloseable {
      * @return a new ImageWriter instance.
      */
     public static ImageWriter newInstance(Surface surface, int maxImages) {
-        return new ImageWriter(surface, maxImages);
+        return new ImageWriter(surface, maxImages, ImageFormat.UNKNOWN);
+    }
+
+    /**
+     * <p>
+     * Create a new ImageWriter with given number of max Images and format.
+     * </p>
+     * <p>
+     * The {@code maxImages} parameter determines the maximum number of
+     * {@link Image} objects that can be be dequeued from the
+     * {@code ImageWriter} simultaneously. Requesting more buffers will use up
+     * more memory, so it is important to use only the minimum number necessary.
+     * </p>
+     * <p>
+     * The format specifies the image format of this ImageWriter. The format
+     * from the {@code surface} will be overridden with this format. For example,
+     * if the surface is obtained from a {@link android.graphics.SurfaceTexture}, the default
+     * format may be {@link PixelFormat#RGBA_8888}. If the application creates an ImageWriter
+     * with this surface and {@link ImageFormat#PRIVATE}, this ImageWriter will be able to operate
+     * with {@link ImageFormat#PRIVATE} Images.
+     * </p>
+     * <p>
+     * Note that the consumer end-point may or may not be able to support Images with different
+     * format, for such case, the application should only use this method if the consumer is able
+     * to consume such images.
+     * </p>
+     * <p>
+     * The input Image size depends on the Surface that is provided by
+     * the downstream consumer end-point.
+     * </p>
+     *
+     * @param surface The destination Surface this writer produces Image data
+     *            into.
+     * @param maxImages The maximum number of Images the user will want to
+     *            access simultaneously for producing Image data. This should be
+     *            as small as possible to limit memory use. Once maxImages
+     *            Images are dequeued by the user, one of them has to be queued
+     *            back before a new Image can be dequeued for access via
+     *            {@link #dequeueInputImage()}.
+     * @param format The format of this ImageWriter. It can be any valid format specified by
+     *            {@link ImageFormat} or {@link PixelFormat}.
+     *
+     * @return a new ImageWriter instance.
+     * @hide
+     */
+    public static ImageWriter newInstance(Surface surface, int maxImages, int format) {
+        if (!ImageFormat.isPublicFormat(format) && !PixelFormat.isPublicFormat(format)) {
+            throw new IllegalArgumentException("Invalid format is specified: " + format);
+        }
+        return new ImageWriter(surface, maxImages, format);
     }
 
     /**
      * @hide
      */
-    protected ImageWriter(Surface surface, int maxImages) {
+    protected ImageWriter(Surface surface, int maxImages, int format) {
         if (surface == null || maxImages < 1) {
             throw new IllegalArgumentException("Illegal input argument: surface " + surface
                     + ", maxImages: " + maxImages);
         }
 
         mMaxImages = maxImages;
+
+        if (format == ImageFormat.UNKNOWN) {
+            format = SurfaceUtils.getSurfaceFormat(surface);
+        }
         // Note that the underlying BufferQueue is working in synchronous mode
         // to avoid dropping any buffers.
-        mNativeContext = nativeInit(new WeakReference<ImageWriter>(this), surface, maxImages);
+        mNativeContext = nativeInit(new WeakReference<>(this), surface, maxImages, format);
 
         // Estimate the native buffer allocation size and register it so it gets accounted for
         // during GC. Note that this doesn't include the buffers required by the buffer queue
@@ -142,7 +196,6 @@ public class ImageWriter implements AutoCloseable {
         // complex, and 1 buffer is enough for the VM to treat the ImageWriter as being of some
         // size.
         Size surfSize = SurfaceUtils.getSurfaceSize(surface);
-        int format = SurfaceUtils.getSurfaceFormat(surface);
         mEstimatedNativeAllocBytes =
                 ImageUtils.getEstimatedNativeAllocBytes(surfSize.getWidth(),surfSize.getHeight(),
                         format, /*buffer count*/ 1);
@@ -809,7 +862,8 @@ public class ImageWriter implements AutoCloseable {
     }
 
     // Native implemented ImageWriter methods.
-    private synchronized native long nativeInit(Object weakSelf, Surface surface, int maxImgs);
+    private synchronized native long nativeInit(Object weakSelf, Surface surface, int maxImgs,
+            int format);
 
     private synchronized native void nativeClose(long nativeCtx);
 

media/jni/Android.mk

@@ -51,6 +51,7 @@ LOCAL_SHARED_LIBRARIES := \
     libandroidfw
 
 LOCAL_STATIC_LIBRARIES := \
+    libgrallocusage \
 
 LOCAL_C_INCLUDES += \
     external/libexif/ \

media/jni/android_media_ImageReader.cpp

@@ -31,6 +31,8 @@
 
 #include <android_runtime/AndroidRuntime.h>
 #include <android_runtime/android_view_Surface.h>
+#include <android_runtime/android_hardware_HardwareBuffer.h>
+#include <grallocusage/GrallocUsageConversion.h>
 
 #include <jni.h>
 #include <JNIHelp.h>
@@ -42,6 +44,7 @@
 #define ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID   "mNativeBuffer"
 #define ANDROID_MEDIA_SURFACEIMAGE_TS_JNI_ID       "mTimestamp"
 
+#define CONSUMER_BUFFER_USAGE_UNKNOWN              0;
 // ----------------------------------------------------------------------------
 
 using namespace android;
@@ -327,8 +330,8 @@ static void ImageReader_classInit(JNIEnv* env, jclass clazz)
             "Can not find SurfacePlane constructor");
 }
 
-static void ImageReader_init(JNIEnv* env, jobject thiz, jobject weakThiz,
+static void ImageReader_init(JNIEnv* env, jobject thiz, jobject weakThiz, jint width, jint height,
-                             jint width, jint height, jint format, jint maxImages)
+                             jint format, jint maxImages, jlong ndkUsage)
 {
     status_t res;
     int nativeFormat;
@@ -358,17 +361,29 @@ static void ImageReader_init(JNIEnv* env, jobject thiz, jobject weakThiz,
             width, height, format, maxImages, getpid(),
             createProcessUniqueId());
     uint32_t consumerUsage = GRALLOC_USAGE_SW_READ_OFTEN;
+    bool needUsageOverride = ndkUsage != CONSUMER_BUFFER_USAGE_UNKNOWN;
+    uint64_t outProducerUsage = 0;
+    uint64_t outConsumerUsage = 0;
+    android_hardware_HardwareBuffer_convertToGrallocUsageBits(&outProducerUsage, &outConsumerUsage,
+            ndkUsage, 0);
 
     if (isFormatOpaque(nativeFormat)) {
         // Use the SW_READ_NEVER usage to tell producer that this format is not for preview or video
         // encoding. The only possibility will be ZSL output.
         consumerUsage = GRALLOC_USAGE_SW_READ_NEVER;
+        if (needUsageOverride) {
+            consumerUsage = android_convertGralloc1To0Usage(0, outConsumerUsage);
+        }
+    } else if (needUsageOverride) {
+        ALOGW("Consumer usage override for non-opaque format is not implemented yet, "
+                "ignore the provided usage from the application");
     }
     bufferConsumer = new BufferItemConsumer(gbConsumer, consumerUsage, maxImages,
             /*controlledByApp*/true);
     if (bufferConsumer == nullptr) {
         jniThrowExceptionFmt(env, "java/lang/RuntimeException",
-                "Failed to allocate native buffer consumer for format 0x%x", nativeFormat);
+                "Failed to allocate native buffer consumer for format 0x%x and usage 0x%x",
+                nativeFormat, consumerUsage);
         return;
     }
     ctx->setBufferConsumer(bufferConsumer);
@@ -788,7 +803,7 @@ static jint Image_getFormat(JNIEnv* env, jobject thiz, jint readerFormat)
 
 static const JNINativeMethod gImageReaderMethods[] = {
     {"nativeClassInit",        "()V",                        (void*)ImageReader_classInit },
-    {"nativeInit",             "(Ljava/lang/Object;IIII)V",  (void*)ImageReader_init },
+    {"nativeInit",             "(Ljava/lang/Object;IIIIJ)V",  (void*)ImageReader_init },
     {"nativeClose",            "()V",                        (void*)ImageReader_close },
     {"nativeReleaseImage",     "(Landroid/media/Image;)V",   (void*)ImageReader_imageRelease },
     {"nativeImageSetup",       "(Landroid/media/Image;)I",   (void*)ImageReader_imageSetup },

media/jni/android_media_ImageWriter.cpp

@@ -33,7 +33,7 @@
 #include <inttypes.h>
 
 #define IMAGE_BUFFER_JNI_ID           "mNativeBuffer"
-
+#define IMAGE_FORMAT_UNKNOWN          0 // This is the same value as ImageFormat#UNKNOWN.
 // ----------------------------------------------------------------------------
 
 using namespace android;
@@ -222,7 +222,7 @@ static void ImageWriter_classInit(JNIEnv* env, jclass clazz) {
 }
 
 static jlong ImageWriter_init(JNIEnv* env, jobject thiz, jobject weakThiz, jobject jsurface,
-        jint maxImages) {
+        jint maxImages, jint userFormat) {
     status_t res;
 
     ALOGV("%s: maxImages:%d", __FUNCTION__, maxImages);
@@ -255,7 +255,7 @@ static jlong ImageWriter_init(JNIEnv* env, jobject thiz, jobject weakThiz, jobje
 
     // Get the dimension and format of the producer.
     sp<ANativeWindow> anw = producer;
-    int32_t width, height, format;
+    int32_t width, height, surfaceFormat;
     if ((res = anw->query(anw.get(), NATIVE_WINDOW_WIDTH, &width)) != OK) {
         ALOGE("%s: Query Surface width failed: %s (%d)", __FUNCTION__, strerror(-res), res);
         jniThrowRuntimeException(env, "Failed to query Surface width");
@@ -270,21 +270,27 @@ static jlong ImageWriter_init(JNIEnv* env, jobject thiz, jobject weakThiz, jobje
     }
     ctx->setBufferHeight(height);
 
-    if ((res = anw->query(anw.get(), NATIVE_WINDOW_FORMAT, &format)) != OK) {
+    // Query surface format if no valid user format is specified, otherwise, override surface format
-        ALOGE("%s: Query Surface format failed: %s (%d)", __FUNCTION__, strerror(-res), res);
+    // with user format.
-        jniThrowRuntimeException(env, "Failed to query Surface format");
+    if (userFormat == IMAGE_FORMAT_UNKNOWN) {
-        return 0;
+        if ((res = anw->query(anw.get(), NATIVE_WINDOW_FORMAT, &surfaceFormat)) != OK) {
+            ALOGE("%s: Query Surface format failed: %s (%d)", __FUNCTION__, strerror(-res), res);
+            jniThrowRuntimeException(env, "Failed to query Surface format");
+            return 0;
+        }
+    } else {
+        surfaceFormat = userFormat;
     }
-    ctx->setBufferFormat(format);
+    ctx->setBufferFormat(surfaceFormat);
-    env->SetIntField(thiz, gImageWriterClassInfo.mWriterFormat, reinterpret_cast<jint>(format));
+    env->SetIntField(thiz,
+            gImageWriterClassInfo.mWriterFormat, reinterpret_cast<jint>(surfaceFormat));
 
-
+    if (!isFormatOpaque(surfaceFormat)) {
-    if (!isFormatOpaque(format)) {
         res = native_window_set_usage(anw.get(), GRALLOC_USAGE_SW_WRITE_OFTEN);
         if (res != OK) {
             ALOGE("%s: Configure usage %08x for format %08x failed: %s (%d)",
                   __FUNCTION__, static_cast<unsigned int>(GRALLOC_USAGE_SW_WRITE_OFTEN),
-                  format, strerror(-res), res);
+                  surfaceFormat, strerror(-res), res);
             jniThrowRuntimeException(env, "Failed to SW_WRITE_OFTEN configure usage");
             return 0;
         }
@@ -784,7 +790,7 @@ static jobjectArray Image_createSurfacePlanes(JNIEnv* env, jobject thiz,
 
 static JNINativeMethod gImageWriterMethods[] = {
     {"nativeClassInit",         "()V",                        (void*)ImageWriter_classInit },
-    {"nativeInit",              "(Ljava/lang/Object;Landroid/view/Surface;I)J",
+    {"nativeInit",              "(Ljava/lang/Object;Landroid/view/Surface;II)J",
                                                               (void*)ImageWriter_init },
     {"nativeClose",              "(J)V",                      (void*)ImageWriter_close },
     {"nativeAttachAndQueueImage", "(JJIJIIII)I",          (void*)ImageWriter_attachAndQueueImage },