From be09219f61213f9fb600220e487120bd0a612ff4 Mon Sep 17 00:00:00 2001 From: renn Date: Mon, 7 May 2018 10:18:05 -0700 Subject: [PATCH] Adds support for hardware buffers in ImageWriter Images. This allows dequeuing private image buffers from ImageWriters, and accessing their hardware buffers. Bug: 79422702 Test: verified cts locally Change-Id: Ia003ffa0af305fad8ca9407f5379a3c9c1ab07b5 --- media/java/android/media/Image.java | 5 ++- media/java/android/media/ImageWriter.java | 48 ++++++++++++++--------- media/jni/android_media_ImageWriter.cpp | 27 +++++++++++-- 3 files changed, 56 insertions(+), 24 deletions(-) diff --git a/media/java/android/media/Image.java b/media/java/android/media/Image.java index 9828275e4a9fe..a768dd3faf7bb 100644 --- a/media/java/android/media/Image.java +++ b/media/java/android/media/Image.java @@ -207,8 +207,9 @@ public abstract class Image implements AutoCloseable { * after {@link Image#close Image.close()} has been called. *

* @return the HardwareBuffer associated with this Image or null if this Image doesn't support - * this feature (e.g. {@link android.media.ImageWriter ImageWriter} or - * {@link android.media.MediaCodec MediaCodec} don't). + * this feature. (Unsupported use cases include Image instances obtained through + * {@link android.media.MediaCodec MediaCodec}, and on versions prior to Android P, + * {@link android.media.ImageWriter ImageWriter}). */ @Nullable public HardwareBuffer getHardwareBuffer() { diff --git a/media/java/android/media/ImageWriter.java b/media/java/android/media/ImageWriter.java index 397768af84d14..4c0153f701920 100644 --- a/media/java/android/media/ImageWriter.java +++ b/media/java/android/media/ImageWriter.java @@ -20,6 +20,7 @@ import android.graphics.ImageFormat; import android.graphics.PixelFormat; import android.graphics.Rect; import android.hardware.camera2.utils.SurfaceUtils; +import android.hardware.HardwareBuffer; import android.os.Handler; import android.os.Looper; import android.os.Message; @@ -58,12 +59,17 @@ import java.util.concurrent.CopyOnWriteArrayList; *

*

* If the application already has an Image from {@link ImageReader}, the - * application can directly queue this Image into ImageWriter (via - * {@link #queueInputImage}), potentially with zero buffer copies. For the - * {@link ImageFormat#PRIVATE PRIVATE} format Images produced by - * {@link ImageReader}, this is the only way to send Image data to ImageWriter, - * as the Image data aren't accessible by the application. + * application can directly queue this Image into the ImageWriter (via + * {@link #queueInputImage}), potentially with zero buffer copies. This + * even works if the image format of the ImageWriter is + * {@link ImageFormat#PRIVATE PRIVATE}, and prior to Android P is the only + * way to enqueue images into such an ImageWriter. Starting in Android P + * private images may also be accessed through their hardware buffers + * (when available) through the {@link Image#getHardwareBuffer()} method. + * Attempting to access the planes of a private image, will return an + * empty array. *

+ *

* Once new input Images are queued into an ImageWriter, it's up to the * downstream components (e.g. {@link ImageReader} or * {@link android.hardware.camera2.CameraDevice}) to consume the Images. If the @@ -257,29 +263,26 @@ public class ImageWriter implements AutoCloseable { *

* If the format of ImageWriter is {@link ImageFormat#PRIVATE PRIVATE} ( * {@link ImageWriter#getFormat()} == {@link ImageFormat#PRIVATE}), the - * image buffer is inaccessible to the application, and calling this method - * will result in an {@link IllegalStateException}. Instead, the application - * should acquire images from some other component (e.g. an + * image buffer is accessible to the application only through the hardware + * buffer obtained through {@link Image#getHardwareBuffer()}. (On Android + * versions prior to P, dequeueing private buffers will cause an + * {@link IllegalStateException} to be thrown). Alternatively, + * the application can acquire images from some other component (e.g. an * {@link ImageReader}), and queue them directly to this ImageWriter via the * {@link ImageWriter#queueInputImage queueInputImage()} method. *

* * @return The next available input Image from this ImageWriter. * @throws IllegalStateException if {@code maxImages} Images are currently - * dequeued, or the ImageWriter format is - * {@link ImageFormat#PRIVATE PRIVATE}, or the input - * {@link android.view.Surface Surface} has been abandoned by the - * consumer component that provided the {@link android.view.Surface Surface}. + * dequeued, or the input {@link android.view.Surface Surface} + * has been abandoned by the consumer component that provided + * the {@link android.view.Surface Surface}. Prior to Android + * P, throws if the ImageWriter format is + * {@link ImageFormat#PRIVATE PRIVATE}. * @see #queueInputImage * @see Image#close */ public Image dequeueInputImage() { - if (mWriterFormat == ImageFormat.PRIVATE) { - throw new IllegalStateException( - "PRIVATE format ImageWriter doesn't support this operation since the images are" - + " inaccessible to the application!"); - } - if (mDequeuedImages.size() >= mMaxImages) { throw new IllegalStateException("Already dequeued max number of Images " + mMaxImages); } @@ -742,6 +745,13 @@ public class ImageWriter implements AutoCloseable { mTimestamp = timestamp; } + @Override + public HardwareBuffer getHardwareBuffer() { + throwISEIfImageIsInvalid(); + + return nativeGetHardwareBuffer(); + } + @Override public Plane[] getPlanes() { throwISEIfImageIsInvalid(); @@ -863,6 +873,8 @@ public class ImageWriter implements AutoCloseable { private synchronized native int nativeGetHeight(); private synchronized native int nativeGetFormat(); + + private synchronized native HardwareBuffer nativeGetHardwareBuffer(); } // Native implemented ImageWriter methods. diff --git a/media/jni/android_media_ImageWriter.cpp b/media/jni/android_media_ImageWriter.cpp index 11659e4826c54..f8f7a90b39371 100644 --- a/media/jni/android_media_ImageWriter.cpp +++ b/media/jni/android_media_ImageWriter.cpp @@ -25,11 +25,14 @@ #include #include #include +#include +#include #include #include #include #include +#include #define IMAGE_BUFFER_JNI_ID "mNativeBuffer" #define IMAGE_FORMAT_UNKNOWN 0 // This is the same value as ImageFormat#UNKNOWN. @@ -701,6 +704,20 @@ static jint Image_getFormat(JNIEnv* env, jobject thiz) { return static_cast(publicFmt); } +static jobject Image_getHardwareBuffer(JNIEnv* env, jobject thiz) { + GraphicBuffer* buffer; + Image_getNativeContext(env, thiz, &buffer, NULL); + if (buffer == NULL) { + jniThrowException(env, "java/lang/IllegalStateException", + "Image is not initialized"); + return NULL; + } + AHardwareBuffer* b = AHardwareBuffer_from_GraphicBuffer(buffer); + // don't user the public AHardwareBuffer_toHardwareBuffer() because this would force us + // to link against libandroid.so + return android_hardware_HardwareBuffer_createFromAHardwareBuffer(env, b); +} + static void Image_setFenceFd(JNIEnv* env, jobject thiz, int fenceFd) { ALOGV("%s:", __FUNCTION__); env->SetIntField(thiz, gSurfaceImageClassInfo.mNativeFenceFd, reinterpret_cast(fenceFd)); @@ -818,10 +835,12 @@ static JNINativeMethod gImageWriterMethods[] = { static JNINativeMethod gImageMethods[] = { {"nativeCreatePlanes", "(II)[Landroid/media/ImageWriter$WriterSurfaceImage$SurfacePlane;", - (void*)Image_createSurfacePlanes }, - {"nativeGetWidth", "()I", (void*)Image_getWidth }, - {"nativeGetHeight", "()I", (void*)Image_getHeight }, - {"nativeGetFormat", "()I", (void*)Image_getFormat }, + (void*)Image_createSurfacePlanes }, + {"nativeGetWidth", "()I", (void*)Image_getWidth }, + {"nativeGetHeight", "()I", (void*)Image_getHeight }, + {"nativeGetFormat", "()I", (void*)Image_getFormat }, + {"nativeGetHardwareBuffer", "()Landroid/hardware/HardwareBuffer;", + (void*)Image_getHardwareBuffer }, }; int register_android_media_ImageWriter(JNIEnv *env) {