From cbd1a8fa1de58eb5d564f6e967d80480cd249b30 Mon Sep 17 00:00:00 2001 From: Shuzhen Wang Date: Wed, 25 May 2022 23:46:38 +0000 Subject: [PATCH] Camera: Unhide readout timestamp APIs Move the readout timestamp to a separate control in OutputConfiguration so that it's orthogonal to timestamp base. Test: Camera CTS Bug: 232437315 Change-Id: I1c0f8ba2b1db05f766c2ce2011dda2d54a459d43 --- core/api/current.txt | 3 + .../camera2/CameraCaptureSession.java | 28 ++++-- .../camera2/CameraCharacteristics.java | 30 +++--- .../hardware/camera2/CameraMetadata.java | 5 +- .../camera2/params/OutputConfiguration.java | 92 ++++++++++++++----- 5 files changed, 115 insertions(+), 43 deletions(-) diff --git a/core/api/current.txt b/core/api/current.txt index 6fb45078e2086..c18b681677b79 100644 --- a/core/api/current.txt +++ b/core/api/current.txt @@ -17753,6 +17753,7 @@ package android.hardware.camera2 { method public void onCaptureSequenceAborted(@NonNull android.hardware.camera2.CameraCaptureSession, int); method public void onCaptureSequenceCompleted(@NonNull android.hardware.camera2.CameraCaptureSession, int, long); method public void onCaptureStarted(@NonNull android.hardware.camera2.CameraCaptureSession, @NonNull android.hardware.camera2.CaptureRequest, long, long); + method public void onReadoutStarted(@NonNull android.hardware.camera2.CameraCaptureSession, @NonNull android.hardware.camera2.CaptureRequest, long, long); } public abstract static class CameraCaptureSession.StateCallback { @@ -18742,6 +18743,7 @@ package android.hardware.camera2.params { method public int getSurfaceGroupId(); method @NonNull public java.util.List getSurfaces(); method public int getTimestampBase(); + method public boolean isReadoutTimestampUsed(); method public void removeSensorPixelModeUsed(int); method public void removeSurface(@NonNull android.view.Surface); method public void setDynamicRangeProfile(long); @@ -18749,6 +18751,7 @@ package android.hardware.camera2.params { method public void setPhysicalCameraId(@Nullable String); method public void setStreamUseCase(long); method public void setTimestampBase(int); + method public void useReadoutTimestamp(boolean); method public void writeToParcel(android.os.Parcel, int); field @NonNull public static final android.os.Parcelable.Creator CREATOR; field public static final int MIRROR_MODE_AUTO = 0; // 0x0 diff --git a/core/java/android/hardware/camera2/CameraCaptureSession.java b/core/java/android/hardware/camera2/CameraCaptureSession.java index 8e4a10879ce51..ff6e8975f8daf 100644 --- a/core/java/android/hardware/camera2/CameraCaptureSession.java +++ b/core/java/android/hardware/camera2/CameraCaptureSession.java @@ -1245,10 +1245,28 @@ public abstract class CameraCaptureSession implements AutoCloseable { * the start of exposure, particularly when autoexposure is changing exposure duration * between frames.

* - *

This timestamp may not match {@link CaptureResult#SENSOR_TIMESTAMP the result - * timestamp field}. It will, however, match the timestamp of buffers sent to the - * output surfaces with {@link OutputConfiguration#TIMESTAMP_BASE_READOUT_SENSOR} - * timestamp base.

+ *

The timestamps match the timestamps of the output surfaces with readout timestamp + * enabled (via {@link OutputConfiguration#useReadoutTimestamp}) if:

+ *
    + *
  • Timestamp base is {@link OutputConfiguration#TIMESTAMP_BASE_DEFAULT} and the + * output + *
      + *
    • is not a SurfaceView surface, and
      • + *
    • is not a MediaRecoder, MediaCodec, or ImageReader surface with {@link + * android.hardware.HardwareBuffer#USAGE_VIDEO_ENCODE} usage flag or the device's {@link + * CameraCharacteristics#SENSOR_INFO_TIMESTAMP_SOURCE} is {@code UNKNOWN}
      • + *
    + *
    • + *
  • Timestamp base is {@link OutputConfiguration#TIMESTAMP_BASE_SENSOR},
    • + *
  • Timestamp base is {@link OutputConfiguration#TIMESTAMP_BASE_MONOTONIC} and the + * device's {@link CameraCharacteristics#SENSOR_INFO_TIMESTAMP_SOURCE} is {@code + * UNKNOWN},
    • + *
  • Timestamp base is {@link OutputConfiguration#TIMESTAMP_BASE_REALTIME} and the + * device's {@link CameraCharacteristics#SENSOR_INFO_TIMESTAMP_SOURCE} is {@code REALTIME} + *
    • + *
+ *

Otherwise, the timestamps won't match the timestamp of the output surfaces. See + * the possible parameters for {@link OutputConfiguration#setTimestampBase} for details.

* *

This callback will be called only if {@link * CameraCharacteristics#SENSOR_READOUT_TIMESTAMP} is @@ -1261,8 +1279,6 @@ public abstract class CameraCaptureSession implements AutoCloseable { * the timestamp at the input image's start of readout for a * reprocess request, in nanoseconds. * @param frameNumber the frame number for this capture - * - * @hide */ public void onReadoutStarted(@NonNull CameraCaptureSession session, @NonNull CaptureRequest request, long timestamp, long frameNumber) { diff --git a/core/java/android/hardware/camera2/CameraCharacteristics.java b/core/java/android/hardware/camera2/CameraCharacteristics.java index a6f7e945bef80..320af06f34a33 100644 --- a/core/java/android/hardware/camera2/CameraCharacteristics.java +++ b/core/java/android/hardware/camera2/CameraCharacteristics.java @@ -4588,22 +4588,26 @@ public final class CameraCharacteristics extends CameraMetadataWhether or not the camera device supports readout timestamp and - * onReadoutStarted callback.

- *

If this tag is HARDWARE, the camera device calls onReadoutStarted in addition to the - * onCaptureStarted callback for each capture. The timestamp passed into the callback - * is the start of camera image readout rather than the start of the exposure. In - * addition, the application can configure an - * {@link android.hardware.camera2.params.OutputConfiguration } with - * TIMESTAMP_BASE_READOUT_SENSOR timestamp base, in which case, the timestamp of the - * output surface matches the timestamp from the corresponding onReadoutStarted callback.

+ * {@code onReadoutStarted} callback.

+ *

If this tag is {@code HARDWARE}, the camera device calls + * {@link CameraCaptureSession.CaptureCallback#onReadoutStarted } in addition to the + * {@link CameraCaptureSession.CaptureCallback#onCaptureStarted } callback for each capture. + * The timestamp passed into the callback is the start of camera image readout rather than + * the start of the exposure. The timestamp source of + * {@link CameraCaptureSession.CaptureCallback#onReadoutStarted } is the same as that of + * {@link CameraCaptureSession.CaptureCallback#onCaptureStarted }.

+ *

In addition, the application can switch an output surface's timestamp from start of + * exposure to start of readout by calling + * {@link android.hardware.camera2.params.OutputConfiguration#useReadoutTimestamp }.

*

The readout timestamp is beneficial for video recording, because the encoder favors * uniform timestamps, and the readout timestamps better reflect the cadence camera sensors * output data.

- *

If this tag is HARDWARE, the camera device produces the start-of-exposure and - * start-of-readout together. As a result, the onReadoutStarted is called right after - * onCaptureStarted. The difference in start-of-readout and start-of-exposure is the sensor - * exposure time, plus certain constant offset. The offset is usually due to camera sensor - * level crop, and it remains constant for a given camera sensor mode.

+ *

Note that the camera device produces the start-of-exposure and start-of-readout callbacks + * together. As a result, the {@link CameraCaptureSession.CaptureCallback#onReadoutStarted } + * is called right after {@link CameraCaptureSession.CaptureCallback#onCaptureStarted }. The + * difference in start-of-readout and start-of-exposure is the sensor exposure time, plus + * certain constant offset. The offset is usually due to camera sensor level crop, and it is + * generally constant over time for the same set of output resolutions and capture settings.

*

Possible values:

*
    *
  • {@link #SENSOR_READOUT_TIMESTAMP_NOT_SUPPORTED NOT_SUPPORTED}
    • diff --git a/core/java/android/hardware/camera2/CameraMetadata.java b/core/java/android/hardware/camera2/CameraMetadata.java index b2428b1d80a38..67dcd93757e13 100644 --- a/core/java/android/hardware/camera2/CameraMetadata.java +++ b/core/java/android/hardware/camera2/CameraMetadata.java @@ -1700,9 +1700,8 @@ public abstract class CameraMetadata { /** *

    This camera device supports the onReadoutStarted callback as well as outputting - * readout timestamp for streams with TIMESTAMP_BASE_READOUT_SENSOR timestamp base. The - * readout timestamp is generated by the camera hardware and it has the same accuracy - * and timing characteristics of the start-of-exposure time.

    + * readout timestamps. The readout timestamp is generated by the camera hardware and it + * has the same accuracy and timing characteristics of the start-of-exposure time.

    * @see CameraCharacteristics#SENSOR_READOUT_TIMESTAMP */ public static final int SENSOR_READOUT_TIMESTAMP_HARDWARE = 1; diff --git a/core/java/android/hardware/camera2/params/OutputConfiguration.java b/core/java/android/hardware/camera2/params/OutputConfiguration.java index f4b87b9f5ae91..65df5d1d43597 100644 --- a/core/java/android/hardware/camera2/params/OutputConfiguration.java +++ b/core/java/android/hardware/camera2/params/OutputConfiguration.java @@ -191,7 +191,9 @@ public final class OutputConfiguration implements Parcelable { * *

    The timestamps of the output images are in the time base as specified by {@link * CameraCharacteristics#SENSOR_INFO_TIMESTAMP_SOURCE}. The application can look up the - * corresponding result metadata for a particular output image using this timestamp.

    + * corresponding result metadata by matching the timestamp with a {@link + * CameraCaptureSession.CaptureCallback#onCaptureStarted}, or with a {@link + * CameraCaptureSession.CaptureCallback#onReadoutStarted} if readout timestamp is used.

    */ public static final int TIMESTAMP_BASE_SENSOR = 1; @@ -204,7 +206,8 @@ public final class OutputConfiguration implements Parcelable { * *

    If the camera device's {@link CameraCharacteristics#SENSOR_INFO_TIMESTAMP_SOURCE} is * REALTIME, timestamps with this time base cannot directly match the timestamps in - * {@link CameraCaptureSession.CaptureCallback#onCaptureStarted} or the sensor timestamps in + * {@link CameraCaptureSession.CaptureCallback#onCaptureStarted}, {@link + * CameraCaptureSession.CaptureCallback#onReadoutStarted}, or the sensor timestamps in * {@link android.hardware.camera2.CaptureResult}.

    */ public static final int TIMESTAMP_BASE_MONOTONIC = 2; @@ -218,7 +221,8 @@ public final class OutputConfiguration implements Parcelable { * *

    If the camera device's {@link CameraCharacteristics#SENSOR_INFO_TIMESTAMP_SOURCE} is * UNKNOWN, timestamps with this time base cannot directly match the timestamps in - * {@link CameraCaptureSession.CaptureCallback#onCaptureStarted} or the sensor timestamps in + * {@link CameraCaptureSession.CaptureCallback#onCaptureStarted}, {@link + * CameraCaptureSession.CaptureCallback#onReadoutStarted}, or the sensor timestamps in * {@link android.hardware.camera2.CaptureResult}.

    * *

    If using a REALTIME timestamp base on a device that supports only @@ -243,7 +247,8 @@ public final class OutputConfiguration implements Parcelable { * displayed right away.

    * *

    Timestamps with this time base cannot directly match the timestamps in - * {@link CameraCaptureSession.CaptureCallback#onCaptureStarted} or the sensor timestamps in + * {@link CameraCaptureSession.CaptureCallback#onCaptureStarted}, {@link + * CameraCaptureSession.CaptureCallback#onReadoutStarted}, or the sensor timestamps in * {@link android.hardware.camera2.CaptureResult}. This timestamp base shouldn't be used if the * timestamp needs to be used for audio-video synchronization.

    */ @@ -252,18 +257,7 @@ public final class OutputConfiguration implements Parcelable { /** * Timestamp is the start of readout in the same time domain as TIMESTAMP_BASE_SENSOR. * - *

    The start of the camera sensor readout after exposure. For a rolling shutter camera - * sensor, the timestamp is typically equal to the start of exposure time + - * exposure time + certain fixed offset. The fixed offset could be due to camera sensor - * level crop. The benefit of using readout time is that when camera runs in a fixed - * frame rate, the timestamp intervals between frames are constant.

    - * - *

    This timestamp is in the same time domain as in TIMESTAMP_BASE_SENSOR, with the exception - * that one is start of exposure, and the other is start of readout.

    - * - *

    This timestamp base is supported only if {@link - * CameraCharacteristics#SENSOR_READOUT_TIMESTAMP} is - * {@link CameraMetadata#SENSOR_READOUT_TIMESTAMP_HARDWARE}.

    + *

    NOTE: do not use! Use useReadoutTimestamp instead.

    * * @hide */ @@ -579,6 +573,8 @@ public final class OutputConfiguration implements Parcelable { mStreamUseCase = CameraMetadata.SCALER_AVAILABLE_STREAM_USE_CASES_DEFAULT; mTimestampBase = TIMESTAMP_BASE_DEFAULT; mMirrorMode = MIRROR_MODE_AUTO; + mUseReadoutTimestamp = false; + mIsReadoutSensorTimestampBase = false; } /** @@ -679,6 +675,8 @@ public final class OutputConfiguration implements Parcelable { mDynamicRangeProfile = DynamicRangeProfiles.STANDARD; mColorSpace = ColorSpaceProfiles.UNSPECIFIED; mStreamUseCase = CameraMetadata.SCALER_AVAILABLE_STREAM_USE_CASES_DEFAULT; + mUseReadoutTimestamp = false; + mIsReadoutSensorTimestampBase = false; } /** @@ -1048,7 +1046,15 @@ public final class OutputConfiguration implements Parcelable { throw new IllegalArgumentException("Not a valid timestamp base value " + timestampBase); } - mTimestampBase = timestampBase; + + if (timestampBase == TIMESTAMP_BASE_READOUT_SENSOR) { + mTimestampBase = TIMESTAMP_BASE_SENSOR; + mUseReadoutTimestamp = true; + mIsReadoutSensorTimestampBase = true; + } else { + mTimestampBase = timestampBase; + mIsReadoutSensorTimestampBase = false; + } } /** @@ -1060,7 +1066,11 @@ public final class OutputConfiguration implements Parcelable { * @return The currently set timestamp base */ public @TimestampBase int getTimestampBase() { - return mTimestampBase; + if (mIsReadoutSensorTimestampBase) { + return TIMESTAMP_BASE_READOUT_SENSOR; + } else { + return mTimestampBase; + } } /** @@ -1099,6 +1109,37 @@ public final class OutputConfiguration implements Parcelable { return mMirrorMode; } + /** + * Use the camera sensor's readout time for the image timestamp. + * + *

    The start of the camera sensor readout after exposure. For a rolling shutter camera + * sensor, the timestamp is typically equal to {@code (the start of exposure time) + + * (exposure time) + (certain fixed offset)}. The fixed offset can vary per session, depending + * on the underlying sensor configuration. The benefit of using readout time is that when + * camera runs in a fixed frame rate, the timestamp intervals between frames are constant.

    + * + *

    Readout timestamp is supported only if {@link + * CameraCharacteristics#SENSOR_READOUT_TIMESTAMP} is + * {@link CameraMetadata#SENSOR_READOUT_TIMESTAMP_HARDWARE}.

    + * + *

    As long as readout timestamp is supported, if the timestamp base isi + * {@link #TIMESTAMP_BASE_CHOREOGRAPHER_SYNCED}, or if the timestamp base is DEFAULT for a + * SurfaceView output, the image timestamps for the output are always readout time regardless + * of whether this function is called.

    + * + * @param on The output image timestamp is the start of exposure time if false, and + * the start of readout time if true. + */ + public void useReadoutTimestamp(boolean on) { + mUseReadoutTimestamp = on; + } + + /** Whether readout timestamp is used for this OutputConfiguration. + */ + public boolean isReadoutTimestampUsed() { + return mUseReadoutTimestamp; + } + /** * Create a new {@link OutputConfiguration} instance with another {@link OutputConfiguration} * instance. @@ -1130,6 +1171,7 @@ public final class OutputConfiguration implements Parcelable { this.mStreamUseCase = other.mStreamUseCase; this.mTimestampBase = other.mTimestampBase; this.mMirrorMode = other.mMirrorMode; + this.mUseReadoutTimestamp = other.mUseReadoutTimestamp; } /** @@ -1157,6 +1199,7 @@ public final class OutputConfiguration implements Parcelable { int timestampBase = source.readInt(); int mirrorMode = source.readInt(); + boolean useReadoutTimestamp = source.readInt() == 1; mSurfaceGroupId = surfaceSetId; mRotation = rotation; @@ -1185,6 +1228,7 @@ public final class OutputConfiguration implements Parcelable { mStreamUseCase = streamUseCase; mTimestampBase = timestampBase; mMirrorMode = mirrorMode; + mUseReadoutTimestamp = useReadoutTimestamp; } /** @@ -1305,6 +1349,7 @@ public final class OutputConfiguration implements Parcelable { dest.writeLong(mStreamUseCase); dest.writeInt(mTimestampBase); dest.writeInt(mMirrorMode); + dest.writeInt(mUseReadoutTimestamp ? 1 : 0); } /** @@ -1338,7 +1383,8 @@ public final class OutputConfiguration implements Parcelable { mIsMultiResolution != other.mIsMultiResolution || mStreamUseCase != other.mStreamUseCase || mTimestampBase != other.mTimestampBase || - mMirrorMode != other.mMirrorMode) + mMirrorMode != other.mMirrorMode || + mUseReadoutTimestamp != other.mUseReadoutTimestamp) return false; if (mSensorPixelModesUsed.size() != other.mSensorPixelModesUsed.size()) { return false; @@ -1381,7 +1427,7 @@ public final class OutputConfiguration implements Parcelable { mPhysicalCameraId == null ? 0 : mPhysicalCameraId.hashCode(), mIsMultiResolution ? 1 : 0, mSensorPixelModesUsed.hashCode(), mDynamicRangeProfile, mColorSpace, mStreamUseCase, - mTimestampBase, mMirrorMode); + mTimestampBase, mMirrorMode, mUseReadoutTimestamp ? 1 : 0); } return HashCodeHelpers.hashCode( @@ -1391,7 +1437,7 @@ public final class OutputConfiguration implements Parcelable { mPhysicalCameraId == null ? 0 : mPhysicalCameraId.hashCode(), mIsMultiResolution ? 1 : 0, mSensorPixelModesUsed.hashCode(), mDynamicRangeProfile, mColorSpace, mStreamUseCase, mTimestampBase, - mMirrorMode); + mMirrorMode, mUseReadoutTimestamp ? 1 : 0); } private static final String TAG = "OutputConfiguration"; @@ -1433,4 +1479,8 @@ public final class OutputConfiguration implements Parcelable { private int mTimestampBase; // Mirroring mode private int mMirrorMode; + // Use readout timestamp + private boolean mUseReadoutTimestamp; + // Whether the timestamp base is set to READOUT_SENSOR + private boolean mIsReadoutSensorTimestampBase; }