From d2087e77813d55cc6083b6332550230291512f11 Mon Sep 17 00:00:00 2001 From: Sally Qi Date: Thu, 13 Jan 2022 14:05:16 -0800 Subject: [PATCH] Add fence support in ImageWriter class. - Add SyncFence class that include FileDescriptor and related functions - expose fence file descriptor setter/getter in the Image class Bug: 210919185 Test: android.hardware.camera2.cts.ImageWriterTest Change-Id: I7428db26e3e6ca84675d2ce2ef4def46ed0b1397 --- core/api/current.txt | 10 ++ core/java/android/hardware/SyncFence.java | 141 ++++++++++++++++++ .../CameraAdvancedExtensionSessionImpl.java | 13 +- .../impl/CameraExtensionSessionImpl.java | 21 +-- media/java/android/media/Image.java | 29 +++- media/java/android/media/ImageReader.java | 13 +- media/java/android/media/ImageWriter.java | 22 +++ media/jni/android_media_ImageWriter.cpp | 7 +- 8 files changed, 232 insertions(+), 24 deletions(-) create mode 100644 core/java/android/hardware/SyncFence.java diff --git a/core/api/current.txt b/core/api/current.txt index 0316875332237..b9cdc2f886518 100644 --- a/core/api/current.txt +++ b/core/api/current.txt @@ -17061,6 +17061,14 @@ package android.hardware { field public static final int MICROPHONE = 1; // 0x1 } + public final class SyncFence implements java.io.Closeable android.os.Parcelable { + method public void close() throws java.io.IOException; + method public int describeContents(); + method public boolean isValid(); + method public void writeToParcel(@NonNull android.os.Parcel, int); + field @NonNull public static final android.os.Parcelable.Creator CREATOR; + } + public final class TriggerEvent { field public android.hardware.Sensor sensor; field public long timestamp; @@ -20773,6 +20781,7 @@ package android.media { method public abstract void close(); method public android.graphics.Rect getCropRect(); method public long getDataSpace(); + method @NonNull public android.hardware.SyncFence getFence() throws java.io.IOException; method public abstract int getFormat(); method @Nullable public android.hardware.HardwareBuffer getHardwareBuffer(); method public abstract int getHeight(); @@ -20781,6 +20790,7 @@ package android.media { method public abstract int getWidth(); method public void setCropRect(android.graphics.Rect); method public void setDataSpace(long); + method public void setFence(@NonNull android.hardware.SyncFence) throws java.io.IOException; method public void setTimestamp(long); } diff --git a/core/java/android/hardware/SyncFence.java b/core/java/android/hardware/SyncFence.java new file mode 100644 index 0000000000000..b0a6f51c2a39e --- /dev/null +++ b/core/java/android/hardware/SyncFence.java @@ -0,0 +1,141 @@ +/** + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.hardware; + +import android.annotation.NonNull; +import android.os.Parcel; +import android.os.ParcelFileDescriptor; +import android.os.Parcelable; + +import java.io.Closeable; +import java.io.IOException; + +/** + * A SyncFence represents a synchronization primitive which signals when hardware buffers have + * completed work on a particular resource. + * + *

For example, a GPU rendering to a framebuffer may generate a synchronization fence, + * e.g., a VkFence, which signals when rendering has completed. + * + * Once the fence signals, then the backing storage for the framebuffer may be safely read from, + * such as for display or for media encoding.

+ * + * @see + * VkFence + */ +public final class SyncFence implements Closeable, Parcelable { + private static final String TAG = "SyncFence"; + + /** + * Wrapped {@link android.os.ParcelFileDescriptor}. + */ + private ParcelFileDescriptor mWrapped; + + private SyncFence(@NonNull ParcelFileDescriptor wrapped) { + mWrapped = wrapped; + } + + /*** + * Create an empty SyncFence + * + * @return a SyncFence with invalid fence + * @hide + */ + public static @NonNull SyncFence createEmpty() { + return new SyncFence(ParcelFileDescriptor.adoptFd(-1)); + } + + /** + * Create a new SyncFence wrapped around another descriptor. By default, all method calls are + * delegated to the wrapped descriptor. + * + * @param wrapped The descriptor to be wrapped. + * @hide + */ + public static @NonNull SyncFence create(@NonNull ParcelFileDescriptor wrapped) { + return new SyncFence(wrapped); + } + + /** + * Return a dup'd ParcelFileDescriptor from the SyncFence ParcelFileDescriptor. + * @hide + */ + public @NonNull ParcelFileDescriptor getFdDup() throws IOException { + return mWrapped.dup(); + } + + /** + * Checks if the SyncFile object is valid. + * + * @return {@code true} if the file descriptor represents a valid, open file; + * {@code false} otherwise. + */ + public boolean isValid() { + return mWrapped.getFileDescriptor().valid(); + } + + /** + * Close the SyncFence. This implementation closes the underlying OS resources allocated + * this stream. + * + * @throws IOException If an error occurs attempting to close this SyncFence. + */ + @Override + public void close() throws IOException { + if (mWrapped != null) { + try { + mWrapped.close(); + } finally { + // success + } + } + } + + @Override + public int describeContents() { + return mWrapped.describeContents(); + } + + /** + * Flatten this object into a Parcel. + * + * @param out The Parcel in which the object should be written. + * @param flags Additional flags about how the object should be written. + * May be {@code 0} or {@link #PARCELABLE_WRITE_RETURN_VALUE} + */ + @Override + public void writeToParcel(@NonNull Parcel out, int flags) { + try { + mWrapped.writeToParcel(out, flags); + } finally { + // success + } + } + + public static final @NonNull Parcelable.Creator CREATOR = + new Parcelable.Creator() { + @Override + public SyncFence createFromParcel(Parcel in) { + return new SyncFence(ParcelFileDescriptor.CREATOR.createFromParcel(in)); + } + + @Override + public SyncFence[] newArray(int size) { + return new SyncFence[size]; + } + }; +} diff --git a/core/java/android/hardware/camera2/impl/CameraAdvancedExtensionSessionImpl.java b/core/java/android/hardware/camera2/impl/CameraAdvancedExtensionSessionImpl.java index 9d2c901ed049f..4a7ef555c065b 100644 --- a/core/java/android/hardware/camera2/impl/CameraAdvancedExtensionSessionImpl.java +++ b/core/java/android/hardware/camera2/impl/CameraAdvancedExtensionSessionImpl.java @@ -22,6 +22,7 @@ import android.annotation.RequiresPermission; import android.content.Context; import android.graphics.ImageFormat; import android.graphics.SurfaceTexture; +import android.hardware.SyncFence; import android.hardware.camera2.CameraAccessException; import android.hardware.camera2.CameraCaptureSession; import android.hardware.camera2.CameraCharacteristics; @@ -58,7 +59,6 @@ import android.media.ImageReader; import android.os.Binder; import android.os.Handler; import android.os.HandlerThread; -import android.os.ParcelFileDescriptor; import android.os.RemoteException; import android.util.Log; import android.util.Size; @@ -817,12 +817,13 @@ public final class CameraAdvancedExtensionSessionImpl extends CameraExtensionSes ParcelImage parcelImage = new ParcelImage(); parcelImage.buffer = img.getHardwareBuffer(); - if (img.getFenceFd() >= 0) { - try { - parcelImage.fence = ParcelFileDescriptor.fromFd(img.getFenceFd()); - } catch (IOException e) { - Log.e(TAG,"Failed to parcel buffer fence!"); + try { + SyncFence fd = img.getFence(); + if (fd.isValid()) { + parcelImage.fence = fd.getFdDup(); } + } catch (IOException e) { + Log.e(TAG, "Failed to parcel buffer fence!"); } parcelImage.width = img.getWidth(); parcelImage.height = img.getHeight(); diff --git a/core/java/android/hardware/camera2/impl/CameraExtensionSessionImpl.java b/core/java/android/hardware/camera2/impl/CameraExtensionSessionImpl.java index c8ecfd0bdea9f..289ae8ad9198b 100644 --- a/core/java/android/hardware/camera2/impl/CameraExtensionSessionImpl.java +++ b/core/java/android/hardware/camera2/impl/CameraExtensionSessionImpl.java @@ -16,10 +16,14 @@ package android.hardware.camera2.impl; +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.annotation.RequiresPermission; import android.content.Context; import android.graphics.ImageFormat; import android.graphics.SurfaceTexture; import android.hardware.HardwareBuffer; +import android.hardware.SyncFence; import android.hardware.camera2.CameraAccessException; import android.hardware.camera2.CameraCaptureSession; import android.hardware.camera2.CameraCharacteristics; @@ -30,6 +34,7 @@ import android.hardware.camera2.CameraManager; import android.hardware.camera2.CaptureFailure; import android.hardware.camera2.CaptureRequest; import android.hardware.camera2.CaptureResult; +import android.hardware.camera2.TotalCaptureResult; import android.hardware.camera2.extension.CaptureBundle; import android.hardware.camera2.extension.CaptureStageImpl; import android.hardware.camera2.extension.ICaptureProcessorImpl; @@ -38,7 +43,6 @@ import android.hardware.camera2.extension.IInitializeSessionCallback; import android.hardware.camera2.extension.IPreviewExtenderImpl; import android.hardware.camera2.extension.IRequestUpdateProcessorImpl; import android.hardware.camera2.extension.ParcelImage; -import android.hardware.camera2.TotalCaptureResult; import android.hardware.camera2.params.DynamicRangeProfiles; import android.hardware.camera2.params.ExtensionSessionConfiguration; import android.hardware.camera2.params.OutputConfiguration; @@ -50,11 +54,7 @@ import android.media.ImageWriter; import android.os.Binder; import android.os.Handler; import android.os.HandlerThread; -import android.os.ParcelFileDescriptor; import android.os.RemoteException; -import android.annotation.NonNull; -import android.annotation.Nullable; -import android.annotation.RequiresPermission; import android.util.Log; import android.util.LongSparseArray; import android.util.Pair; @@ -1656,12 +1656,13 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession { private static ParcelImage initializeParcelImage(Image img) { ParcelImage parcelImage = new ParcelImage(); parcelImage.buffer = img.getHardwareBuffer(); - if (img.getFenceFd() >= 0) { - try { - parcelImage.fence = ParcelFileDescriptor.fromFd(img.getFenceFd()); - } catch (IOException e) { - Log.e(TAG,"Failed to parcel buffer fence!"); + try { + SyncFence fd = img.getFence(); + if (fd.isValid()) { + parcelImage.fence = fd.getFdDup(); } + } catch (IOException e) { + Log.e(TAG, "Failed to parcel buffer fence!"); } parcelImage.width = img.getWidth(); parcelImage.height = img.getHeight(); diff --git a/media/java/android/media/Image.java b/media/java/android/media/Image.java index 5261555d2fa89..cf1aac0e4d4c5 100644 --- a/media/java/android/media/Image.java +++ b/media/java/android/media/Image.java @@ -16,6 +16,7 @@ package android.media; +import android.annotation.NonNull; import android.annotation.Nullable; import android.annotation.SuppressLint; import android.annotation.TestApi; @@ -24,7 +25,9 @@ import android.graphics.Rect; import android.hardware.DataSpace; import android.hardware.DataSpace.NamedDataSpace; import android.hardware.HardwareBuffer; +import android.hardware.SyncFence; +import java.io.IOException; import java.nio.ByteBuffer; /** @@ -223,12 +226,17 @@ public abstract class Image implements AutoCloseable { public abstract int getScalingMode(); /** - * Get the fence file descriptor associated with this frame. - * @return The fence file descriptor for this frame. - * @hide + * Get the SyncFence object associated with this frame. + * + *

This function returns an invalid SyncFence after {@link #getPlanes()} on the image + * dequeued from {@link ImageWriter} via {@link ImageWriter#dequeueInputImage()}.

+ * + * @return The SyncFence for this frame. + * @throws IOException if there is an error when a SyncFence object returns. + * @see android.hardware.SyncFence */ - public int getFenceFd() { - return -1; + public @NonNull SyncFence getFence() throws IOException { + return SyncFence.createEmpty(); } /** @@ -283,6 +291,17 @@ public abstract class Image implements AutoCloseable { return; } + /** + * Set the fence file descriptor with this frame. + * @param fence The fence file descriptor to be set for this frame. + * @throws IOException if there is an error when setting a SyncFence. + * @see android.hardware.SyncFence + */ + public void setFence(@NonNull SyncFence fence) throws IOException { + throwISEIfImageIsInvalid(); + return; + } + private @NamedDataSpace long mDataSpace = DataSpace.DATASPACE_UNKNOWN; /** diff --git a/media/java/android/media/ImageReader.java b/media/java/android/media/ImageReader.java index e2e48d35a6725..c9bda48ec27c4 100644 --- a/media/java/android/media/ImageReader.java +++ b/media/java/android/media/ImageReader.java @@ -27,13 +27,16 @@ import android.hardware.DataSpace; import android.hardware.DataSpace.NamedDataSpace; import android.hardware.HardwareBuffer; import android.hardware.HardwareBuffer.Usage; +import android.hardware.SyncFence; import android.hardware.camera2.MultiResolutionImageReader; import android.os.Handler; import android.os.Looper; +import android.os.ParcelFileDescriptor; import android.view.Surface; import dalvik.system.VMRuntime; +import java.io.IOException; import java.lang.ref.WeakReference; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -1219,9 +1222,15 @@ public class ImageReader implements AutoCloseable { } @Override - public int getFenceFd() { + public SyncFence getFence() throws IOException { throwISEIfImageIsInvalid(); - return nativeGetFenceFd(); + // duplicate ParcelFileDescriptor because native still retains the fence ownership. + int fence = nativeGetFenceFd(); + if (fence != -1) { + return SyncFence.create(ParcelFileDescriptor.fromFd(nativeGetFenceFd())); + } else { + return SyncFence.createEmpty(); + } } @Override diff --git a/media/java/android/media/ImageWriter.java b/media/java/android/media/ImageWriter.java index a1aedf17e55ad..5c3f918dd98df 100644 --- a/media/java/android/media/ImageWriter.java +++ b/media/java/android/media/ImageWriter.java @@ -28,16 +28,19 @@ import android.hardware.DataSpace; import android.hardware.DataSpace.NamedDataSpace; import android.hardware.HardwareBuffer; import android.hardware.HardwareBuffer.Usage; +import android.hardware.SyncFence; import android.hardware.camera2.params.StreamConfigurationMap; import android.hardware.camera2.utils.SurfaceUtils; import android.os.Handler; import android.os.Looper; import android.os.Message; +import android.os.ParcelFileDescriptor; import android.util.Size; import android.view.Surface; import dalvik.system.VMRuntime; +import java.io.IOException; import java.lang.ref.WeakReference; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -1143,6 +1146,23 @@ public class ImageWriter implements AutoCloseable { return nativeGetHardwareBuffer(); } + @Override + public SyncFence getFence() throws IOException { + throwISEIfImageIsInvalid(); + // if mNativeFenceFd is -1, the fence is closed + if (mNativeFenceFd != -1) { + return SyncFence.create(ParcelFileDescriptor.fromFd(mNativeFenceFd)); + } else { + return SyncFence.createEmpty(); + } + } + + @Override + public void setFence(@NonNull SyncFence fence) throws IOException { + throwISEIfImageIsInvalid(); + nativeSetFenceFd(fence.getFdDup().detachFd()); + } + @Override public Plane[] getPlanes() { throwISEIfImageIsInvalid(); @@ -1268,6 +1288,8 @@ public class ImageWriter implements AutoCloseable { private synchronized native int nativeGetFormat(long dataSpace); private synchronized native HardwareBuffer nativeGetHardwareBuffer(); + + private synchronized native void nativeSetFenceFd(int fenceFd); } // Native implemented ImageWriter methods. diff --git a/media/jni/android_media_ImageWriter.cpp b/media/jni/android_media_ImageWriter.cpp index eca26dc14fa9a..8f5c9da4fbe1e 100644 --- a/media/jni/android_media_ImageWriter.cpp +++ b/media/jni/android_media_ImageWriter.cpp @@ -42,7 +42,6 @@ #include #define IMAGE_BUFFER_JNI_ID "mNativeBuffer" -#define IMAGE_FORMAT_UNKNOWN 0 // This is the same value as ImageFormat#UNKNOWN. // ---------------------------------------------------------------------------- using namespace android; @@ -991,6 +990,11 @@ static jobject Image_getHardwareBuffer(JNIEnv* env, jobject thiz) { static void Image_setFenceFd(JNIEnv* env, jobject thiz, int fenceFd) { ALOGV("%s:", __FUNCTION__); + int curtFenceFd = reinterpret_cast( + env->GetIntField(thiz,gSurfaceImageClassInfo.mNativeFenceFd)); + if (curtFenceFd != -1) { + close(curtFenceFd); + } env->SetIntField(thiz, gSurfaceImageClassInfo.mNativeFenceFd, reinterpret_cast(fenceFd)); } @@ -1120,6 +1124,7 @@ static JNINativeMethod gImageMethods[] = { {"nativeGetWidth", "()I", (void*)Image_getWidth }, {"nativeGetHeight", "()I", (void*)Image_getHeight }, {"nativeGetFormat", "(J)I", (void*)Image_getFormat }, + {"nativeSetFenceFd", "(I)V", (void*)Image_setFenceFd }, {"nativeGetHardwareBuffer", "()Landroid/hardware/HardwareBuffer;", (void*)Image_getHardwareBuffer }, };