Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Introduce builder pattern in ImageWriter class

Browse Source 
- allow ImageWriter to set usage flag on the buffer
- allow app to provide HardwareBuffer.Format which is 1:1 mapping with
the HAL PixelFormat
- invovle dataspace setter into ImageWriter build pattern
- allow app to override image width and height

Bug: 213331412
Test: android.hardware.cts.DataSpaceTest, android.hardware.camera2.cts.ImageReaderTest, android.hardware.camera2.cts.ImageWriterTest
Change-Id: Ie8e39a5daadf68b3809dd0035a902959dc18c9c3
This commit is contained in:
Sally Qi
2022-01-05 11:53:44 -08:00
parent 20ba147a11
commit db57de3f85
3 changed files with 337 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
core/api/current.txt
View File

@@ -22040,14 +22040,30 @@ package android.media {
public class ImageWriter implements java.lang.AutoCloseable {
method public void close();
method public android.media.Image dequeueInputImage();
method public long getDataSpace();
method public int getFormat();
method public int getHardwareBufferFormat();
method public int getHeight();
method public int getMaxImages();
method public long getUsage();
method public int getWidth();
method @NonNull public static android.media.ImageWriter newInstance(@NonNull android.view.Surface, @IntRange(from=1) int);
method @NonNull public static android.media.ImageWriter newInstance(@NonNull android.view.Surface, @IntRange(from=1) int, int);
method public void queueInputImage(android.media.Image);
method public void setOnImageReleasedListener(android.media.ImageWriter.OnImageReleasedListener, android.os.Handler);
}
public static final class ImageWriter.Builder {
ctor public ImageWriter.Builder(@NonNull android.view.Surface);
method @NonNull public android.media.ImageWriter build();
method @NonNull public android.media.ImageWriter.Builder setDataSpace(long);
method @NonNull public android.media.ImageWriter.Builder setHardwareBufferFormat(int);
method @NonNull public android.media.ImageWriter.Builder setImageFormat(int);
method @NonNull public android.media.ImageWriter.Builder setMaxImages(@IntRange(from=1) int);
method @NonNull public android.media.ImageWriter.Builder setUsage(long);
method @NonNull public android.media.ImageWriter.Builder setWidthAndHeight(@IntRange(from=1) int, @IntRange(from=1) int);
}
public static interface ImageWriter.OnImageReleasedListener {
method public void onImageReleased(android.media.ImageWriter);
}

344
media/java/android/media/ImageWriter.java
View File

@@ -18,12 +18,16 @@ package android.media;
import android.annotation.IntRange;
import android.annotation.NonNull;
import android.annotation.SuppressLint;
import android.graphics.GraphicBuffer;
import android.graphics.ImageFormat;
import android.graphics.ImageFormat.Format;
import android.graphics.PixelFormat;
import android.graphics.Rect;
import android.hardware.DataSpace;
import android.hardware.DataSpace.NamedDataSpace;
import android.hardware.HardwareBuffer;
import android.hardware.HardwareBuffer.Usage;
import android.hardware.camera2.params.StreamConfigurationMap;
import android.hardware.camera2.utils.SurfaceUtils;
import android.os.Handler;
@@ -95,10 +99,18 @@ public class ImageWriter implements AutoCloseable {
private ListenerHandler mListenerHandler;
private long mNativeContext;
private int mWidth;
private int mHeight;
private final int mMaxImages;
private @Usage long mUsage = HardwareBuffer.USAGE_CPU_WRITE_OFTEN;
private @HardwareBuffer.Format int mHardwareBufferFormat;
private @NamedDataSpace long mDataSpace;
private boolean mUseLegacyImageFormat;
private boolean mUseSurfaceImageFormatInfo;
// Field below is used by native code, do not access or modify.
private int mWriterFormat;
private final int mMaxImages;
// Keep track of the currently dequeued Image. This need to be thread safe as the images
// could be closed by different threads (e.g., application thread and GC thread).
private List<Image> mDequeuedImages = new CopyOnWriteArrayList<>();
@@ -131,7 +143,7 @@ public class ImageWriter implements AutoCloseable {
*/
public static @NonNull ImageWriter newInstance(@NonNull Surface surface,
@IntRange(from = 1) int maxImages) {
return new ImageWriter(surface, maxImages, ImageFormat.UNKNOWN, -1 /*width*/,
return new ImageWriter(surface, maxImages, true, ImageFormat.UNKNOWN, -1 /*width*/,
-1 /*height*/);
}
@@ -183,7 +195,7 @@ public class ImageWriter implements AutoCloseable {
if (!ImageFormat.isPublicFormat(format) && !PixelFormat.isPublicFormat(format)) {
throw new IllegalArgumentException("Invalid format is specified: " + format);
}
return new ImageWriter(surface, maxImages, format, width, height);
return new ImageWriter(surface, maxImages, false, format, width, height);
}
/**
@@ -232,48 +244,49 @@ public class ImageWriter implements AutoCloseable {
if (!ImageFormat.isPublicFormat(format) && !PixelFormat.isPublicFormat(format)) {
throw new IllegalArgumentException("Invalid format is specified: " + format);
}
return new ImageWriter(surface, maxImages, format, -1 /*width*/, -1 /*height*/);
return new ImageWriter(surface, maxImages, false, format, -1 /*width*/, -1 /*height*/);
}
/**
* @hide
*/
protected ImageWriter(Surface surface, int maxImages, int format, int width, int height) {
private void initializeImageWriter(Surface surface, int maxImages,
boolean useSurfaceImageFormatInfo, boolean useLegacyImageFormat, int imageFormat,
int hardwareBufferFormat, long dataSpace, int width, int height, long usage) {
if (surface == null || maxImages < 1) {
throw new IllegalArgumentException("Illegal input argument: surface " + surface
+ ", maxImages: " + maxImages);
+ ", maxImages: " + maxImages);
}
mMaxImages = maxImages;
mUseSurfaceImageFormatInfo = useSurfaceImageFormatInfo;
mUseLegacyImageFormat = useLegacyImageFormat;
// Note that the underlying BufferQueue is working in synchronous mode
// to avoid dropping any buffers.
mNativeContext = nativeInit(new WeakReference<>(this), surface, maxImages, format, width,
height);
mNativeContext = nativeInit(new WeakReference<>(this), surface, maxImages, width, height,
useSurfaceImageFormatInfo, hardwareBufferFormat, dataSpace, usage);
// nativeInit internally overrides UNKNOWN format. So does surface format query after
// nativeInit and before getEstimatedNativeAllocBytes().
if (format == ImageFormat.UNKNOWN) {
format = SurfaceUtils.getSurfaceFormat(surface);
}
// Several public formats use the same native HAL_PIXEL_FORMAT_BLOB. The native
// allocation estimation sequence depends on the public formats values. To avoid
// possible errors, convert where necessary.
if (format == StreamConfigurationMap.HAL_PIXEL_FORMAT_BLOB) {
int surfaceDataspace = SurfaceUtils.getSurfaceDataspace(surface);
switch (surfaceDataspace) {
case StreamConfigurationMap.HAL_DATASPACE_DEPTH:
format = ImageFormat.DEPTH_POINT_CLOUD;
break;
case StreamConfigurationMap.HAL_DATASPACE_DYNAMIC_DEPTH:
format = ImageFormat.DEPTH_JPEG;
break;
case StreamConfigurationMap.HAL_DATASPACE_HEIF:
format = ImageFormat.HEIC;
break;
default:
format = ImageFormat.JPEG;
if (useSurfaceImageFormatInfo) {
// nativeInit internally overrides UNKNOWN format. So does surface format query after
// nativeInit and before getEstimatedNativeAllocBytes().
imageFormat = SurfaceUtils.getSurfaceFormat(surface);
// Several public formats use the same native HAL_PIXEL_FORMAT_BLOB. The native
// allocation estimation sequence depends on the public formats values. To avoid
// possible errors, convert where necessary.
if (imageFormat == StreamConfigurationMap.HAL_PIXEL_FORMAT_BLOB) {
int surfaceDataspace = SurfaceUtils.getSurfaceDataspace(surface);
switch (surfaceDataspace) {
case StreamConfigurationMap.HAL_DATASPACE_DEPTH:
imageFormat = ImageFormat.DEPTH_POINT_CLOUD;
break;
case StreamConfigurationMap.HAL_DATASPACE_DYNAMIC_DEPTH:
imageFormat = ImageFormat.DEPTH_JPEG;
break;
case StreamConfigurationMap.HAL_DATASPACE_HEIF:
imageFormat = ImageFormat.HEIC;
break;
default:
imageFormat = ImageFormat.JPEG;
}
}
mHardwareBufferFormat = PublicFormatUtils.getHalFormat(imageFormat);
mDataSpace = PublicFormatUtils.getHalDataspace(imageFormat);
}
// Estimate the native buffer allocation size and register it so it gets accounted for
// during GC. Note that this doesn't include the buffers required by the buffer queue
@@ -282,12 +295,49 @@ public class ImageWriter implements AutoCloseable {
// complex, and 1 buffer is enough for the VM to treat the ImageWriter as being of some
// size.
Size surfSize = SurfaceUtils.getSurfaceSize(surface);
mWidth = width == -1 ? surfSize.getWidth() : width;
mHeight = height == -1 ? surfSize.getHeight() : height;
mEstimatedNativeAllocBytes =
ImageUtils.getEstimatedNativeAllocBytes(surfSize.getWidth(),surfSize.getHeight(),
format, /*buffer count*/ 1);
ImageUtils.getEstimatedNativeAllocBytes(mWidth, mHeight,
useLegacyImageFormat ? imageFormat : hardwareBufferFormat, /*buffer count*/ 1);
VMRuntime.getRuntime().registerNativeAllocation(mEstimatedNativeAllocBytes);
}
private ImageWriter(Surface surface, int maxImages, boolean useSurfaceImageFormatInfo,
int imageFormat, int width, int height) {
mMaxImages = maxImages;
// update hal format and dataspace only if image format is overridden by producer.
mHardwareBufferFormat = PublicFormatUtils.getHalFormat(imageFormat);
mDataSpace = PublicFormatUtils.getHalDataspace(imageFormat);
initializeImageWriter(surface, maxImages, useSurfaceImageFormatInfo, true,
imageFormat, mHardwareBufferFormat, mDataSpace, width, height, mUsage);
}
private ImageWriter(Surface surface, int maxImages, boolean useSurfaceImageFormatInfo,
int imageFormat, int width, int height, long usage) {
mMaxImages = maxImages;
mUsage = usage;
mHardwareBufferFormat = PublicFormatUtils.getHalFormat(imageFormat);
mDataSpace = PublicFormatUtils.getHalDataspace(imageFormat);
initializeImageWriter(surface, maxImages, useSurfaceImageFormatInfo, true,
imageFormat, mHardwareBufferFormat, mDataSpace, width, height, usage);
}
private ImageWriter(Surface surface, int maxImages, boolean useSurfaceImageFormatInfo,
int hardwareBufferFormat, long dataSpace, int width, int height, long usage) {
mMaxImages = maxImages;
mUsage = usage;
mHardwareBufferFormat = hardwareBufferFormat;
mDataSpace = dataSpace;
int publicFormat = PublicFormatUtils.getPublicFormat(hardwareBufferFormat, dataSpace);
initializeImageWriter(surface, maxImages, useSurfaceImageFormatInfo, false,
publicFormat, hardwareBufferFormat, dataSpace, width, height, usage);
}
/**
* <p>
* Maximum number of Images that can be dequeued from the ImageWriter
@@ -315,6 +365,30 @@ public class ImageWriter implements AutoCloseable {
return mMaxImages;
}
/**
* The width of {@link Image Images}, in pixels.
*
* <p>If {@link Builder#setWidthAndHeight} is not called, the default width of the Image
* depends on the Surface provided by customer end-point.</p>
*
* @return the expected actual width of an Image.
*/
public int getWidth() {
return mWidth;
}
/**
* The height of {@link Image Images}, in pixels.
*
* <p>If {@link Builder#setWidthAndHeight} is not called, the default height of the Image
* depends on the Surface provided by customer end-point.</p>
*
* @return the expected height of an Image.
*/
public int getHeight() {
return mHeight;
}
/**
* <p>
* Dequeue the next available input Image for the application to produce
@@ -489,6 +563,41 @@ public class ImageWriter implements AutoCloseable {
return mWriterFormat;
}
/**
* Get the ImageWriter usage flag.
*
* @return The ImageWriter usage flag.
*/
public @Usage long getUsage() {
return mUsage;
}
/**
* Get the ImageWriter hardwareBuffer format.
*
* <p>Use this function if the ImageWriter instance is created by builder pattern
* {@code ImageWriter.Builder} and using {@link Builder#setHardwareBufferFormat} and
* {@link Builder#setDataSpace}.</p>
*
* @return The ImageWriter hardwareBuffer format.
*/
public @HardwareBuffer.Format int getHardwareBufferFormat() {
return mHardwareBufferFormat;
}
/**
* Get the ImageWriter dataspace.
*
* <p>Use this function if the ImageWriter instance is created by builder pattern
* {@code ImageWriter.Builder} and {@link Builder#setDataSpace}.</p>
*
* @return The ImageWriter dataspace.
*/
@SuppressLint("MethodNameUnits")
public @NamedDataSpace long getDataSpace() {
return mDataSpace;
}
/**
* ImageWriter callback interface, used to to asynchronously notify the
* application of various ImageWriter events.
@@ -755,6 +864,155 @@ public class ImageWriter implements AutoCloseable {
return true;
}
/**
* Builder class for {@link ImageWriter} objects.
*/
public static final class Builder {
private Surface mSurface;
private int mWidth = -1;
private int mHeight = -1;
private int mMaxImages = 1;
private int mImageFormat = ImageFormat.UNKNOWN;
private @Usage long mUsage = HardwareBuffer.USAGE_CPU_WRITE_OFTEN;
private @HardwareBuffer.Format int mHardwareBufferFormat = HardwareBuffer.RGBA_8888;
private @NamedDataSpace long mDataSpace = DataSpace.DATASPACE_UNKNOWN;
private boolean mUseSurfaceImageFormatInfo = true;
// set this as true temporarily now as a workaround to get correct format
// when using surface format by default without overriding the image format
// in the builder pattern
private boolean mUseLegacyImageFormat = true;
/**
* Constructs a new builder for {@link ImageWriter}.
*
* @param surface The destination Surface this writer produces Image data into.
*/
public Builder(@NonNull Surface surface) {
mSurface = surface;
}
/**
* Set the width and height of images. Default size is dependent on the Surface that is
* provided by the downstream end-point.
*
* @param width The width in pixels that will be passed to the producer.
* @param height The height in pixels that will be passed to the producer.
* @return the Builder instance with customized width and height.
*/
@SuppressLint("MissingGetterMatchingBuilder")
public @NonNull Builder setWidthAndHeight(@IntRange(from = 1) int width,
@IntRange(from = 1) int height) {
mWidth = width;
mHeight = height;
return this;
}
/**
* Set the maximum number of images. Default value is 1.
*
* @param maxImages The maximum number of Images the user will want to access simultaneously
* for producing Image data.
* @return the Builder instance with customized usage value.
*/
public @NonNull Builder setMaxImages(@IntRange(from = 1) int maxImages) {
mMaxImages = maxImages;
return this;
}
/**
* Set the image format of this ImageWriter.
* Default format depends on the Surface provided.
*
* @param imageFormat The format of the {@link ImageWriter}. It can be any valid specified
* by {@link ImageFormat} or {@link PixelFormat}.
* @return the Builder instance with customized image format.
*/
@SuppressLint("MissingGetterMatchingBuilder")
public @NonNull Builder setImageFormat(@Format int imageFormat) {
if (!ImageFormat.isPublicFormat(imageFormat)
&& !PixelFormat.isPublicFormat(imageFormat)) {
throw new IllegalArgumentException(
"Invalid imageFormat is specified: " + imageFormat);
}
mImageFormat = imageFormat;
mUseLegacyImageFormat = true;
mHardwareBufferFormat = HardwareBuffer.RGBA_8888;
mDataSpace = DataSpace.DATASPACE_UNKNOWN;
mUseSurfaceImageFormatInfo = false;
return this;
}
/**
* Set the hardwareBuffer format of this ImageWriter. The default value is
* {@link HardwareBuffer#RGBA_8888 HardwareBuffer.RGBA_8888}.
*
* <p>This function works together with {@link #setDataSpace} for an
* {@link ImageWriter} instance. Setting at least one of these two replaces
* {@link #setImageFormat} function.</p>
*
* @param hardwareBufferFormat The HardwareBuffer format of the image that this writer
* will produce.
* @return the Builder instance with customized buffer format.
*
* @see #setDataSpace
* @see #setImageFormat
*/
public @NonNull Builder setHardwareBufferFormat(
@HardwareBuffer.Format int hardwareBufferFormat) {
mHardwareBufferFormat = hardwareBufferFormat;
mImageFormat = ImageFormat.UNKNOWN;
mUseLegacyImageFormat = false;
mUseSurfaceImageFormatInfo = false;
return this;
}
/**
* Set the dataspace of this ImageWriter.
* The default value is {@link DataSpace#DATASPACE_UNKNOWN}.
*
* @param dataSpace The dataspace of the image that this writer will produce.
* @return the builder instance with customized dataspace value.
*
* @see #setHardwareBufferFormat
*/
public @NonNull Builder setDataSpace(@NamedDataSpace long dataSpace) {
mDataSpace = dataSpace;
mImageFormat = ImageFormat.UNKNOWN;
mUseLegacyImageFormat = false;
mUseSurfaceImageFormatInfo = false;
return this;
}
/**
* Set the usage flag of this ImageWriter.
* Default value is {@link HardwareBuffer#USAGE_CPU_WRITE_OFTEN}.
*
* @param usage The intended usage of the images produced by this ImageWriter.
* @return the Builder instance with customized usage flag.
*
* @see HardwareBuffer
*/
public @NonNull Builder setUsage(@Usage long usage) {
mUsage = usage;
return this;
}
/**
* Builds a new ImageWriter object.
*
* @return The new ImageWriter object.
*/
public @NonNull ImageWriter build() {
if (mUseLegacyImageFormat) {
return new ImageWriter(mSurface, mMaxImages, mUseSurfaceImageFormatInfo,
mImageFormat, mWidth, mHeight, mUsage);
} else {
return new ImageWriter(mSurface, mMaxImages, mUseSurfaceImageFormatInfo,
mHardwareBufferFormat, mDataSpace, mWidth, mHeight, mUsage);
}
}
}
private static class WriterSurfaceImage extends android.media.Image {
private ImageWriter mOwner;
// This field is used by native code, do not access or modify.
@@ -774,6 +1032,13 @@ public class ImageWriter implements AutoCloseable {
public WriterSurfaceImage(ImageWriter writer) {
mOwner = writer;
mWidth = writer.mWidth;
mHeight = writer.mHeight;
if (!writer.mUseLegacyImageFormat) {
mFormat = PublicFormatUtils.getPublicFormat(
writer.mHardwareBufferFormat, writer.mDataSpace);
}
}
@Override
@@ -969,8 +1234,9 @@ public class ImageWriter implements AutoCloseable {
}
// Native implemented ImageWriter methods.
private synchronized native long nativeInit(Object weakSelf, Surface surface, int maxImgs,
int format, int width, int height);
private synchronized native long nativeInit(Object weakSelf, Surface surface, int maxImages,
int width, int height, boolean useSurfaceImageFormatInfo, int hardwareBufferFormat,
long dataSpace, long usage);
private synchronized native void nativeClose(long nativeCtx);

33
media/jni/android_media_ImageWriter.cpp
View File

@@ -375,7 +375,8 @@ static void ImageWriter_classInit(JNIEnv* env, jclass clazz) {
}
static jlong ImageWriter_init(JNIEnv* env, jobject thiz, jobject weakThiz, jobject jsurface,
jint maxImages, jint userFormat, jint userWidth, jint userHeight) {
jint maxImages, jint userWidth, jint userHeight, jboolean useSurfaceImageFormatInfo,
jint hardwareBufferFormat, jlong dataSpace, jlong ndkUsage) {
status_t res;
ALOGV("%s: maxImages:%d", __FUNCTION__, maxImages);
@@ -450,7 +451,7 @@ static jlong ImageWriter_init(JNIEnv* env, jobject thiz, jobject weakThiz, jobje
// Query surface format if no valid user format is specified, otherwise, override surface format
// with user format.
if (userFormat == IMAGE_FORMAT_UNKNOWN) {
if (useSurfaceImageFormatInfo) {
if ((res = anw->query(anw.get(), NATIVE_WINDOW_FORMAT, &surfaceFormat)) != OK) {
ALOGE("%s: Query Surface format failed: %s (%d)", __FUNCTION__, strerror(-res), res);
jniThrowRuntimeException(env, "Failed to query Surface format");
@@ -458,13 +459,13 @@ static jlong ImageWriter_init(JNIEnv* env, jobject thiz, jobject weakThiz, jobje
}
} else {
// Set consumer buffer format to user specified format
PublicFormat publicFormat = static_cast<PublicFormat>(userFormat);
int nativeFormat = mapPublicFormatToHalFormat(publicFormat);
android_dataspace nativeDataspace = mapPublicFormatToHalDataspace(publicFormat);
res = native_window_set_buffers_format(anw.get(), nativeFormat);
android_dataspace nativeDataspace = static_cast<android_dataspace>(dataSpace);
int userFormat = static_cast<int>(mapHalFormatDataspaceToPublicFormat(
hardwareBufferFormat, nativeDataspace));
res = native_window_set_buffers_format(anw.get(), hardwareBufferFormat);
if (res != OK) {
ALOGE("%s: Unable to configure consumer native buffer format to %#x",
__FUNCTION__, nativeFormat);
__FUNCTION__, hardwareBufferFormat);
jniThrowRuntimeException(env, "Failed to set Surface format");
return 0;
}
@@ -484,15 +485,13 @@ static jlong ImageWriter_init(JNIEnv* env, jobject thiz, jobject weakThiz, jobje
env->SetIntField(thiz,
gImageWriterClassInfo.mWriterFormat, reinterpret_cast<jint>(surfaceFormat));
if (!isFormatOpaque(surfaceFormat)) {
res = native_window_set_usage(anw.get(), GRALLOC_USAGE_SW_WRITE_OFTEN);
if (res != OK) {
ALOGE("%s: Configure usage %08x for format %08x failed: %s (%d)",
__FUNCTION__, static_cast<unsigned int>(GRALLOC_USAGE_SW_WRITE_OFTEN),
surfaceFormat, strerror(-res), res);
jniThrowRuntimeException(env, "Failed to SW_WRITE_OFTEN configure usage");
return 0;
}
res = native_window_set_usage(anw.get(), ndkUsage);
if (res != OK) {
ALOGE("%s: Configure usage %08x for format %08x failed: %s (%d)",
__FUNCTION__, static_cast<unsigned int>(ndkUsage),
surfaceFormat, strerror(-res), res);
jniThrowRuntimeException(env, "Failed to SW_WRITE_OFTEN configure usage");
return 0;
}
int minUndequeuedBufferCount = 0;
@@ -1093,7 +1092,7 @@ static jobjectArray Image_createSurfacePlanes(JNIEnv* env, jobject thiz,
static JNINativeMethod gImageWriterMethods[] = {
{"nativeClassInit", "()V", (void*)ImageWriter_classInit },
{"nativeInit", "(Ljava/lang/Object;Landroid/view/Surface;IIII)J",
{"nativeInit", "(Ljava/lang/Object;Landroid/view/Surface;IIIZIJJ)J",
(void*)ImageWriter_init },
{"nativeClose", "(J)V", (void*)ImageWriter_close },
{"nativeAttachAndQueueImage",