diff --git a/core/java/android/hardware/camera2/CameraCaptureSession.java b/core/java/android/hardware/camera2/CameraCaptureSession.java index 691690c09e0e0..5dbe9bff328e1 100644 --- a/core/java/android/hardware/camera2/CameraCaptureSession.java +++ b/core/java/android/hardware/camera2/CameraCaptureSession.java @@ -1233,6 +1233,41 @@ public abstract class CameraCaptureSession implements AutoCloseable { // default empty implementation } + /** + * This method is called when the camera device has started reading out the output + * image for the request, at the beginning of the sensor image readout. + * + *

For a capture request, this callback is invoked right after + * {@link #onCaptureStarted}. Unlike {@link #onCaptureStarted}, instead of passing + * a timestamp of start of exposure, this callback passes a timestamp of start of + * camera data readout. This is useful because for a camera running at fixed frame + * rate, the start of readout is at fixed interval, but not necessary for the start + * of exposure.

+ * + *

This timestamp may not match {@link CaptureResult#SENSOR_TIMESTAMP the result + * timestamp field}. It will, however, match the timestamp of buffers sent to the + * output surfaces with {@link OutputConfiguration#TIMESTAMP_BASE_READOUT_SENSOR} + * timestamp base.

+ * + *

This callback will be called only if {@link + * CameraCharacteristics#SENSOR_READOUT_TIMESTAMP} is + * {@link CameraMetadata#SENSOR_READOUT_TIMESTAMP_HARDWARE}, and it's called + * right after {@link #onCaptureStarted}.

+ * + * @param session the session returned by {@link CameraDevice#createCaptureSession} + * @param request the request for the readout that just began + * @param timestamp the timestamp at start of readout for a regular request, or + * the timestamp at the input image's start of readout for a + * reprocess request, in nanoseconds. + * @param frameNumber the frame number for this capture + * + * @hide + */ + public void onReadoutStarted(@NonNull CameraCaptureSession session, + @NonNull CaptureRequest request, long timestamp, long frameNumber) { + // default empty implementation + } + /** * This method is called when some results from an image capture are * available. diff --git a/core/java/android/hardware/camera2/CameraCharacteristics.java b/core/java/android/hardware/camera2/CameraCharacteristics.java index b05e6d1319577..fe4e69e6067fa 100644 --- a/core/java/android/hardware/camera2/CameraCharacteristics.java +++ b/core/java/android/hardware/camera2/CameraCharacteristics.java @@ -4381,6 +4381,40 @@ public final class CameraCharacteristics extends CameraMetadata SENSOR_OPTICAL_BLACK_REGIONS = new Key("android.sensor.opticalBlackRegions", android.graphics.Rect[].class); + /** + *

Whether or not the camera device supports readout timestamp and + * onReadoutStarted callback.

+ *

If this tag is HARDWARE, the camera device calls onReadoutStarted in addition to the + * onCaptureStarted callback for each capture. The timestamp passed into the callback + * is the start of camera image readout rather than the start of the exposure. In + * addition, the application can configure an + * {@link android.hardware.camera2.params.OutputConfiguration } with + * TIMESTAMP_BASE_READOUT_SENSOR timestamp base, in which case, the timestamp of the + * output surface matches the timestamp from the corresponding onReadoutStarted callback.

+ *

The readout timestamp is beneficial for video recording, because the encoder favors + * uniform timestamps, and the readout timestamps better reflect the cadence camera sensors + * output data.

+ *

If this tag is HARDWARE, the camera device produces the start-of-exposure and + * start-of-readout together. As a result, the onReadoutStarted is called right after + * onCaptureStarted. The difference in start-of-readout and start-of-exposure is the sensor + * exposure time, plus certain constant offset. The offset is usually due to camera sensor + * level crop, and it remains constant for a given camera sensor mode.

+ *

Possible values:

+ * + * + *

This key is available on all devices.

+ * @see #SENSOR_READOUT_TIMESTAMP_NOT_SUPPORTED + * @see #SENSOR_READOUT_TIMESTAMP_HARDWARE + * @hide + */ + @PublicKey + @NonNull + public static final Key SENSOR_READOUT_TIMESTAMP = + new Key("android.sensor.readoutTimestamp", int.class); + /** *

List of lens shading modes for {@link CaptureRequest#SHADING_MODE android.shading.mode} that are supported by this camera device.

*

This list contains lens shading modes that can be set for the camera device. diff --git a/core/java/android/hardware/camera2/CameraMetadata.java b/core/java/android/hardware/camera2/CameraMetadata.java index 40565b06a8f2c..5c272660b9626 100644 --- a/core/java/android/hardware/camera2/CameraMetadata.java +++ b/core/java/android/hardware/camera2/CameraMetadata.java @@ -1649,6 +1649,28 @@ public abstract class CameraMetadata { */ public static final int SENSOR_REFERENCE_ILLUMINANT1_ISO_STUDIO_TUNGSTEN = 24; + // + // Enumeration values for CameraCharacteristics#SENSOR_READOUT_TIMESTAMP + // + + /** + *

This camera device doesn't support readout timestamp and onReadoutStarted + * callback.

+ * @see CameraCharacteristics#SENSOR_READOUT_TIMESTAMP + * @hide + */ + public static final int SENSOR_READOUT_TIMESTAMP_NOT_SUPPORTED = 0; + + /** + *

This camera device supports the onReadoutStarted callback as well as outputting + * readout timestamp for streams with TIMESTAMP_BASE_READOUT_SENSOR timestamp base. The + * readout timestamp is generated by the camera hardware and it has the same accuracy + * and timing characteristics of the start-of-exposure time.

+ * @see CameraCharacteristics#SENSOR_READOUT_TIMESTAMP + * @hide + */ + public static final int SENSOR_READOUT_TIMESTAMP_HARDWARE = 1; + // // Enumeration values for CameraCharacteristics#LED_AVAILABLE_LEDS // diff --git a/core/java/android/hardware/camera2/impl/CameraCaptureSessionImpl.java b/core/java/android/hardware/camera2/impl/CameraCaptureSessionImpl.java index 9a9163c724ff0..b9eba9c1d5415 100644 --- a/core/java/android/hardware/camera2/impl/CameraCaptureSessionImpl.java +++ b/core/java/android/hardware/camera2/impl/CameraCaptureSessionImpl.java @@ -656,6 +656,21 @@ public class CameraCaptureSessionImpl extends CameraCaptureSession } } + @Override + public void onReadoutStarted(CameraDevice camera, + CaptureRequest request, long timestamp, long frameNumber) { + if ((callback != null) && (executor != null)) { + final long ident = Binder.clearCallingIdentity(); + try { + executor.execute(() -> callback.onReadoutStarted( + CameraCaptureSessionImpl.this, request, timestamp, + frameNumber)); + } finally { + Binder.restoreCallingIdentity(ident); + } + } + } + @Override public void onCapturePartial(CameraDevice camera, CaptureRequest request, android.hardware.camera2.CaptureResult result) { diff --git a/core/java/android/hardware/camera2/impl/CameraDeviceImpl.java b/core/java/android/hardware/camera2/impl/CameraDeviceImpl.java index 3cb0c93d84090..a6c79b3a289f8 100644 --- a/core/java/android/hardware/camera2/impl/CameraDeviceImpl.java +++ b/core/java/android/hardware/camera2/impl/CameraDeviceImpl.java @@ -2025,12 +2025,16 @@ public class CameraDeviceImpl extends CameraDevice resultExtras.getLastCompletedReprocessFrameNumber(); final long lastCompletedZslFrameNumber = resultExtras.getLastCompletedZslFrameNumber(); + final boolean hasReadoutTimestamp = resultExtras.hasReadoutTimestamp(); + final long readoutTimestamp = resultExtras.getReadoutTimestamp(); if (DEBUG) { Log.d(TAG, "Capture started for id " + requestId + " frame number " + frameNumber + ": completedRegularFrameNumber " + lastCompletedRegularFrameNumber + ", completedReprocessFrameNUmber " + lastCompletedReprocessFrameNumber - + ", completedZslFrameNumber " + lastCompletedZslFrameNumber); + + ", completedZslFrameNumber " + lastCompletedZslFrameNumber + + ", hasReadoutTimestamp " + hasReadoutTimestamp + + (hasReadoutTimestamp ? ", readoutTimestamp " + readoutTimestamp : "")) ; } final CaptureCallbackHolder holder; @@ -2082,14 +2086,26 @@ public class CameraDeviceImpl extends CameraDevice CameraDeviceImpl.this, holder.getRequest(i), timestamp - (subsequenceId - i) * - NANO_PER_SECOND/fpsRange.getUpper(), + NANO_PER_SECOND / fpsRange.getUpper(), frameNumber - (subsequenceId - i)); + if (hasReadoutTimestamp) { + holder.getCallback().onReadoutStarted( + CameraDeviceImpl.this, + holder.getRequest(i), + readoutTimestamp - (subsequenceId - i) * + NANO_PER_SECOND / fpsRange.getUpper(), + frameNumber - (subsequenceId - i)); + } } } else { holder.getCallback().onCaptureStarted( - CameraDeviceImpl.this, - holder.getRequest(resultExtras.getSubsequenceId()), + CameraDeviceImpl.this, request, timestamp, frameNumber); + if (hasReadoutTimestamp) { + holder.getCallback().onReadoutStarted( + CameraDeviceImpl.this, request, + readoutTimestamp, frameNumber); + } } } } diff --git a/core/java/android/hardware/camera2/impl/CaptureCallback.java b/core/java/android/hardware/camera2/impl/CaptureCallback.java index 6defe63b1766a..b064e6a1f9752 100644 --- a/core/java/android/hardware/camera2/impl/CaptureCallback.java +++ b/core/java/android/hardware/camera2/impl/CaptureCallback.java @@ -65,6 +65,13 @@ public abstract class CaptureCallback { public abstract void onCaptureStarted(CameraDevice camera, CaptureRequest request, long timestamp, long frameNumber); + /** + * This method is called when the camera device has started reading out the output + * image for the request, at the beginning of the sensor image readout. + */ + public abstract void onReadoutStarted(CameraDevice camera, + CaptureRequest request, long timestamp, long frameNumber); + /** * This method is called when some results from an image capture are * available. diff --git a/core/java/android/hardware/camera2/impl/CaptureResultExtras.java b/core/java/android/hardware/camera2/impl/CaptureResultExtras.java index 5d9da73fd5c06..8bf94986a4905 100644 --- a/core/java/android/hardware/camera2/impl/CaptureResultExtras.java +++ b/core/java/android/hardware/camera2/impl/CaptureResultExtras.java @@ -33,6 +33,8 @@ public class CaptureResultExtras implements Parcelable { private long lastCompletedRegularFrameNumber; private long lastCompletedReprocessFrameNumber; private long lastCompletedZslFrameNumber; + private boolean hasReadoutTimestamp; + private long readoutTimestamp; public static final @android.annotation.NonNull Parcelable.Creator CREATOR = new Parcelable.Creator() { @@ -56,7 +58,8 @@ public class CaptureResultExtras implements Parcelable { int partialResultCount, int errorStreamId, String errorPhysicalCameraId, long lastCompletedRegularFrameNumber, long lastCompletedReprocessFrameNumber, - long lastCompletedZslFrameNumber) { + long lastCompletedZslFrameNumber, boolean hasReadoutTimestamp, + long readoutTimestamp) { this.requestId = requestId; this.subsequenceId = subsequenceId; this.afTriggerId = afTriggerId; @@ -68,6 +71,8 @@ public class CaptureResultExtras implements Parcelable { this.lastCompletedRegularFrameNumber = lastCompletedRegularFrameNumber; this.lastCompletedReprocessFrameNumber = lastCompletedReprocessFrameNumber; this.lastCompletedZslFrameNumber = lastCompletedZslFrameNumber; + this.hasReadoutTimestamp = hasReadoutTimestamp; + this.readoutTimestamp = readoutTimestamp; } @Override @@ -93,6 +98,10 @@ public class CaptureResultExtras implements Parcelable { dest.writeLong(lastCompletedRegularFrameNumber); dest.writeLong(lastCompletedReprocessFrameNumber); dest.writeLong(lastCompletedZslFrameNumber); + dest.writeBoolean(hasReadoutTimestamp); + if (hasReadoutTimestamp) { + dest.writeLong(readoutTimestamp); + } } public void readFromParcel(Parcel in) { @@ -110,6 +119,10 @@ public class CaptureResultExtras implements Parcelable { lastCompletedRegularFrameNumber = in.readLong(); lastCompletedReprocessFrameNumber = in.readLong(); lastCompletedZslFrameNumber = in.readLong(); + hasReadoutTimestamp = in.readBoolean(); + if (hasReadoutTimestamp) { + readoutTimestamp = in.readLong(); + } } public String getErrorPhysicalCameraId() { @@ -155,4 +168,12 @@ public class CaptureResultExtras implements Parcelable { public long getLastCompletedZslFrameNumber() { return lastCompletedZslFrameNumber; } + + public boolean hasReadoutTimestamp() { + return hasReadoutTimestamp; + } + + public long getReadoutTimestamp() { + return readoutTimestamp; + } } diff --git a/core/java/android/hardware/camera2/params/OutputConfiguration.java b/core/java/android/hardware/camera2/params/OutputConfiguration.java index bb0caa787e275..204e6b60d3dc4 100644 --- a/core/java/android/hardware/camera2/params/OutputConfiguration.java +++ b/core/java/android/hardware/camera2/params/OutputConfiguration.java @@ -236,6 +236,26 @@ public final class OutputConfiguration implements Parcelable { */ public static final int TIMESTAMP_BASE_CHOREOGRAPHER_SYNCED = 4; + /** + * Timestamp is the start of readout in the same time domain as TIMESTAMP_BASE_SENSOR. + * + *

The start of the camera sensor readout after exposure. For a rolling shutter camera + * sensor, the timestamp is typically equal to the start of exposure time + + * exposure time + certain fixed offset. The fixed offset could be due to camera sensor + * level crop. The benefit of using readout time is that when camera runs in a fixed + * frame rate, the timestamp intervals between frames are constant.

+ * + *

This timestamp is in the same time domain as in TIMESTAMP_BASE_SENSOR, with the exception + * that one is start of exposure, and the other is start of readout.

+ * + *

This timestamp base is supported only if {@link + * CameraCharacteristics#SENSOR_READOUT_TIMESTAMP} is + * {@link CameraMetadata#SENSOR_READOUT_TIMESTAMP_HARDWARE}.

+ * + * @hide + */ + public static final int TIMESTAMP_BASE_READOUT_SENSOR = 5; + /** @hide */ @Retention(RetentionPolicy.SOURCE) @IntDef(prefix = {"TIMESTAMP_BASE_"}, value = @@ -243,7 +263,8 @@ public final class OutputConfiguration implements Parcelable { TIMESTAMP_BASE_SENSOR, TIMESTAMP_BASE_MONOTONIC, TIMESTAMP_BASE_REALTIME, - TIMESTAMP_BASE_CHOREOGRAPHER_SYNCED}) + TIMESTAMP_BASE_CHOREOGRAPHER_SYNCED, + TIMESTAMP_BASE_READOUT_SENSOR}) public @interface TimestampBase {}; /** @hide */ @@ -965,7 +986,7 @@ public final class OutputConfiguration implements Parcelable { public void setTimestampBase(@TimestampBase int timestampBase) { // Verify that the value is in range if (timestampBase < TIMESTAMP_BASE_DEFAULT || - timestampBase > TIMESTAMP_BASE_CHOREOGRAPHER_SYNCED) { + timestampBase > TIMESTAMP_BASE_READOUT_SENSOR) { throw new IllegalArgumentException("Not a valid timestamp base value " + timestampBase); }