Use this function if the ImageReader instance is created by factory method + * {@code newInstance} function or by builder pattern {@code ImageReader.Builder} and using + * {@link Builder#setImageFormat}.
+ * * @return the expected format of an Image * * @see ImageFormat @@ -362,6 +392,32 @@ public class ImageReader implements AutoCloseable { return mFormat; } + /** + * The default {@link HardwareBuffer} format of {@link Image Images}. + * + *Use this function if the ImageReader instance is created by builder pattern + * {@code ImageReader.Builder} and using {@link Builder#setDefaultHardwareBufferFormat} and + * {@link Builder#setDefaultDataSpace}.
+ * + * @return the expected {@link HardwareBuffer} format of an Image. + */ + public @HardwareBuffer.Format int getHardwareBufferFormat() { + return mHardwareBufferFormat; + } + + /** + * The default dataspace of {@link Image Images}. + * + *Use this function if the ImageReader instance is created by builder pattern + * {@code ImageReader.Builder} and {@link Builder#setDefaultDataSpace}.
+ * + * @return the expected dataspace of an Image. + */ + @SuppressLint("MethodNameUnits") + public @NamedDataSpace long getDataSpace() { + return mDataSpace; + } + /** * Maximum number of images that can be acquired from the ImageReader by any time (for example, * with {@link #acquireNextImage}). @@ -383,6 +439,15 @@ public class ImageReader implements AutoCloseable { return mMaxImages; } + /** + * The usage flag of images that can be produced by the ImageReader. + * + * @return The usage flag of the images for this ImageReader. + */ + public @Usage long getUsage() { + return mUsage; + } + /** *Get a {@link Surface} that can be used to produce {@link Image Images} for this * {@code ImageReader}.
@@ -469,7 +534,12 @@ public class ImageReader implements AutoCloseable { * @hide */ public Image acquireNextImageNoThrowISE() { - SurfaceImage si = new SurfaceImage(mFormat); + SurfaceImage si; + if (mUseLegacyImageFormat) { + si = new SurfaceImage(mFormat); + } else { + si = new SurfaceImage(mHardwareBufferFormat, mDataSpace); + } return acquireNextSurfaceImage(si) == ACQUIRE_SUCCESS ? si : null; } @@ -492,7 +562,7 @@ public class ImageReader implements AutoCloseable { // A null image will eventually be returned if ImageReader is already closed. int status = ACQUIRE_NO_BUFS; if (mIsReaderValid) { - status = nativeImageSetup(si); + status = nativeImageSetup(si, mUseLegacyImageFormat); } switch (status) { @@ -545,7 +615,12 @@ public class ImageReader implements AutoCloseable { public Image acquireNextImage() { // Initialize with reader format, but can be overwritten by native if the image // format is different from the reader format. - SurfaceImage si = new SurfaceImage(mFormat); + SurfaceImage si; + if (mUseLegacyImageFormat) { + si = new SurfaceImage(mFormat); + } else { + si = new SurfaceImage(mHardwareBufferFormat, mDataSpace); + } int status = acquireNextSurfaceImage(si); switch (status) { @@ -838,13 +913,161 @@ public class ImageReader implements AutoCloseable { } } + /** + * Builder class for {@link ImageReader} objects. + */ + public static final class Builder { + private int mWidth; + private int mHeight; + private int mMaxImages = 1; + private int mImageFormat = ImageFormat.UNKNOWN; + private int mHardwareBufferFormat = HardwareBuffer.RGBA_8888; + private long mDataSpace = DataSpace.DATASPACE_UNKNOWN; + private long mUsage = HardwareBuffer.USAGE_CPU_READ_OFTEN; + private boolean mUseLegacyImageFormat = false; + + /** + * Constructs a new builder for {@link ImageReader}. + * + * @param width The default width in pixels that will be passed to the producer. + * May be overridden by the producer. + * @param height The default height in pixels that will be passed to the producer. + * May be overridden by the producer. + * @see Image + */ + public Builder(@IntRange(from = 1) int width, @IntRange(from = 1) int height) { + mWidth = width; + mHeight = height; + } + + /** + * Set the maximal number of images. + * + * @param maxImages The maximum number of images the user will want to + * access simultaneously. This should be as small as possible to + * limit memory use. Default value is 1. + * @return the Builder instance with customized usage value. + */ + public @NonNull Builder setMaxImages(int maxImages) { + mMaxImages = maxImages; + return this; + } + + /** + * Set the consumer usage flag. + * + * @param usage The intended usage of the images consumed by this ImageReader. + * See the usages on {@link HardwareBuffer} for a list of valid usage bits. + * Default value is {@link HardwareBuffer#USAGE_CPU_READ_OFTEN}. + * @return the Builder instance with customized usage value. + * + * @see HardwareBuffer + */ + public @NonNull Builder setUsage(long usage) { + mUsage = usage; + return this; + } + + /** + * Set the default image format passed by the producer. May be overridden by the producer. + * + *{@link #setImageFormat} function replaces the combination of + * {@link #setDefaultHardwareBufferFormat} and {@link #setDefaultDataSpace} functions. + * Either this or these two functions must be called to initialize an {@code ImageReader} + * instance.
+ * + * @param imageFormat The format of the image that this reader will produce. This + * must be one of the {@link android.graphics.ImageFormat} or + * {@link android.graphics.PixelFormat} constants. Note that not + * all formats are supported, like ImageFormat.NV21. The default value is + * {@link ImageFormat#UNKNOWN}. + * @return the builder instance with customized image format value. + * + * @see #setDefaultHardwareBufferFormat + * @see #setDefaultDataSpace + */ + public @NonNull Builder setImageFormat(@Format int imageFormat) { + mImageFormat = imageFormat; + mUseLegacyImageFormat = true; + mHardwareBufferFormat = HardwareBuffer.RGBA_8888; + mDataSpace = DataSpace.DATASPACE_UNKNOWN; + return this; + } + + /** + * Set the default hardwareBuffer format passed by the producer. + * May be overridden by the producer. + * + *This function works together with {@link #setDefaultDataSpace} for an + * {@link ImageReader} instance. Setting at least one of these two replaces + * {@link #setImageFormat} function.
+ * + *The format of the Image can be overridden after {@link #setImageFormat} by calling + * this function and then {@link #setDefaultDataSpace} functions. + * Warning: Missing one of callings for initializing or overriding the format may + * involve undefined behaviors.
+ * + * @param hardwareBufferFormat The HardwareBuffer format of the image that this reader + * will produce. The default value is + * {@link HardwareBuffer#RGBA_8888 HardwareBuffer.RGBA_8888}. + * @return the builder instance with customized hardwareBuffer value. + * + * @see #setDefaultDataSpace + * @see #setImageFormat + */ + @SuppressLint("MissingGetterMatchingBuilder") + public @NonNull Builder setDefaultHardwareBufferFormat( + @HardwareBuffer.Format int hardwareBufferFormat) { + mHardwareBufferFormat = hardwareBufferFormat; + mUseLegacyImageFormat = false; + mImageFormat = ImageFormat.UNKNOWN; + return this; + } + + /** + * Set the default dataspace passed by the producer. + * May be overridden by the producer. + * + *This function works together with {@link #setDefaultHardwareBufferFormat} for an + * {@link ImageReader} instance. Setting at least one of these two replaces + * {@link #setImageFormat} function.
+ * + * @param dataSpace The dataspace of the image that this reader will produce. + * The default value is {@link DataSpace#DATASPACE_UNKNOWN}. + * @return the builder instance with customized dataspace value. + * + * @see #setDefaultHardwareBufferFormat + */ + @SuppressLint("MissingGetterMatchingBuilder") + public @NonNull Builder setDefaultDataSpace(@NamedDataSpace long dataSpace) { + mDataSpace = dataSpace; + mUseLegacyImageFormat = false; + mImageFormat = ImageFormat.UNKNOWN; + return this; + } + + /** + * Builds a new ImageReader object. + * + * @return The new ImageReader object. + */ + public @NonNull ImageReader build() { + if (mUseLegacyImageFormat) { + return new ImageReader(mWidth, mHeight, mImageFormat, mMaxImages, mUsage, null); + } else { + return new ImageReader(mWidth, mHeight, mMaxImages, mUsage, null, + mHardwareBufferFormat, mDataSpace); + } + } + } + private final int mWidth; private final int mHeight; private final int mFormat; private final long mUsage; private final int mMaxImages; private final int mNumPlanes; - private final Surface mSurface; + private Surface mSurface; private int mEstimatedNativeAllocBytes; private final Object mListenerLock = new Object(); @@ -861,6 +1084,12 @@ public class ImageReader implements AutoCloseable { // MultiResolutionImageReader. private final MultiResolutionImageReader mParent; + private final int mHardwareBufferFormat; + + private final long mDataSpace; + + private final boolean mUseLegacyImageFormat; + /** * This field is used by native code, do not access or modify. */ @@ -897,6 +1126,12 @@ public class ImageReader implements AutoCloseable { mFormat = format; } + SurfaceImage(int hardwareBufferFormat, long dataSpace) { + mHardwareBufferFormat = hardwareBufferFormat; + mDataSpace = dataSpace; + mFormat = PublicFormatUtils.getPublicFormat(mHardwareBufferFormat, mDataSpace); + } + @Override public void close() { ImageReader.this.releaseImage(this); @@ -909,10 +1144,15 @@ public class ImageReader implements AutoCloseable { @Override public int getFormat() { throwISEIfImageIsInvalid(); - int readerFormat = ImageReader.this.getImageFormat(); - // Assume opaque reader always produce opaque images. - mFormat = (readerFormat == ImageFormat.PRIVATE) ? readerFormat : - nativeGetFormat(readerFormat); + // update mFormat only if ImageReader is initialized by factory pattern. + // if using builder pattern, mFormat has been updated upon initialization. + // no need update here. + if (ImageReader.this.mUseLegacyImageFormat) { + int readerFormat = ImageReader.this.getImageFormat(); + // Assume opaque reader always produce opaque images. + mFormat = (readerFormat == ImageFormat.PRIVATE) ? readerFormat : + nativeGetFormat(readerFormat); + } return mFormat; } @@ -1125,6 +1365,8 @@ public class ImageReader implements AutoCloseable { private SurfacePlane[] mPlanes; private int mFormat = ImageFormat.UNKNOWN; + private int mHardwareBufferFormat = HardwareBuffer.RGBA_8888; + private long mDataSpace = DataSpace.DATASPACE_UNKNOWN; // If this image is detached from the ImageReader. private AtomicBoolean mIsDetached = new AtomicBoolean(false); @@ -1137,8 +1379,8 @@ public class ImageReader implements AutoCloseable { private synchronized native HardwareBuffer nativeGetHardwareBuffer(); } - private synchronized native void nativeInit(Object weakSelf, int w, int h, - int fmt, int maxImgs, long consumerUsage); + private synchronized native void nativeInit(Object weakSelf, int w, int h, int maxImgs, + long consumerUsage, int hardwareBufferFormat, long dataSpace); private synchronized native void nativeClose(); private synchronized native void nativeReleaseImage(Image i); private synchronized native Surface nativeGetSurface(); @@ -1152,7 +1394,7 @@ public class ImageReader implements AutoCloseable { * @see #ACQUIRE_NO_BUFS * @see #ACQUIRE_MAX_IMAGES */ - private synchronized native int nativeImageSetup(Image i); + private synchronized native int nativeImageSetup(Image i, boolean legacyValidateImageFormat); /** * @hide diff --git a/media/java/android/media/ImageUtils.java b/media/java/android/media/ImageUtils.java index 7837d7e395997..2f1a36cba9d0a 100644 --- a/media/java/android/media/ImageUtils.java +++ b/media/java/android/media/ImageUtils.java @@ -18,6 +18,7 @@ package android.media; import android.graphics.ImageFormat; import android.graphics.PixelFormat; +import android.hardware.HardwareBuffer; import android.media.Image.Plane; import android.util.Size; @@ -76,6 +77,34 @@ class ImageUtils { } } + /** + * Only a subset of the formats defined in + * {@link android.graphics.HardwareBuffer.Format} constants are supported by ImageReader. + */ + public static int getNumPlanesForHardwareBufferFormat(int hardwareBufferFormat) { + switch(hardwareBufferFormat) { + case HardwareBuffer.YCBCR_420_888: + return 3; + case HardwareBuffer.RGBA_8888: + case HardwareBuffer.RGBX_8888: + case HardwareBuffer.RGB_888: + case HardwareBuffer.RGB_565: + case HardwareBuffer.RGBA_FP16: + case HardwareBuffer.RGBA_1010102: + case HardwareBuffer.BLOB: + case HardwareBuffer.D_16: + case HardwareBuffer.D_24: + case HardwareBuffer.DS_24UI8: + case HardwareBuffer.D_FP32: + case HardwareBuffer.DS_FP32UI8: + case HardwareBuffer.S_UI8: + return 1; + default: + throw new UnsupportedOperationException( + String.format("Invalid hardwareBuffer format specified %d", + hardwareBufferFormat)); + } + } /** *
* Copy source image data to destination Image.
diff --git a/media/java/android/media/PublicFormatUtils.java b/media/java/android/media/PublicFormatUtils.java
new file mode 100644
index 0000000000000..6268804128c65
--- /dev/null
+++ b/media/java/android/media/PublicFormatUtils.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package android.media;
+
+/**
+ * Package private utility class for PublicFormat related methods.
+ */
+class PublicFormatUtils {
+ public static int getHalFormat(int imageFormat) {
+ return nativeGetHalFormat(imageFormat);
+ }
+ public static long getHalDataspace(int imageFormat) {
+ return nativeGetHalDataspace(imageFormat);
+ }
+ public static int getPublicFormat(int imageFormat, long dataspace) {
+ return nativeGetPublicFormat(imageFormat, dataspace);
+ }
+ private static native int nativeGetHalFormat(int imageFormat);
+ private static native long nativeGetHalDataspace(int imageFormat);
+ private static native int nativeGetPublicFormat(int imageFormat, long dataspace);
+}
diff --git a/media/jni/Android.bp b/media/jni/Android.bp
index e817f2dc9e1d2..feae6065c6801 100644
--- a/media/jni/Android.bp
+++ b/media/jni/Android.bp
@@ -39,6 +39,7 @@ cc_library_shared {
"android_media_MediaProfiles.cpp",
"android_media_MediaRecorder.cpp",
"android_media_MediaSync.cpp",
+ "android_media_PublicFormatUtils.cpp",
"android_media_ResampleInputStream.cpp",
"android_media_Streams.cpp",
"android_media_SyncParams.cpp",
diff --git a/media/jni/android_media_ImageReader.cpp b/media/jni/android_media_ImageReader.cpp
index 021507c8631c1..6002e2884db8a 100644
--- a/media/jni/android_media_ImageReader.cpp
+++ b/media/jni/android_media_ImageReader.cpp
@@ -375,18 +375,13 @@ static void ImageReader_classInit(JNIEnv* env, jclass clazz)
}
static void ImageReader_init(JNIEnv* env, jobject thiz, jobject weakThiz, jint width, jint height,
- jint format, jint maxImages, jlong ndkUsage)
-{
+ jint maxImages, jlong ndkUsage, jint nativeFormat, jlong dataSpace) {
status_t res;
- int nativeFormat;
- android_dataspace nativeDataspace;
- ALOGV("%s: width:%d, height: %d, format: 0x%x, maxImages:%d",
- __FUNCTION__, width, height, format, maxImages);
+ ALOGV("%s: width:%d, height: %d, nativeFormat: %d, maxImages:%d",
+ __FUNCTION__, width, height, nativeFormat, maxImages);
- PublicFormat publicFormat = static_cast