Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Camera: Add support for readout timestamp

Browse Source 
- Add new CameraCharacteristics key to indicate readout timestamp
  support.
- Add onReadoutStarted() in CaptureCallback
- Add READOUT_SENSOR time base for OutputConfiguration.

Test: atest Experimental2022SystemTest
Bug: 229773853
Bug: 229097448
Change-Id: Icf831367d3ac92dfd0ae439724d1433968b36dd6
This commit is contained in:
Shuzhen Wang
2022-04-19 10:24:08 -07:00
parent 5673c705d0
commit 91e7522cb2
8 changed files with 178 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
core/java/android/hardware/camera2/CameraCaptureSession.java
View File

@@ -1233,6 +1233,41 @@ public abstract class CameraCaptureSession implements AutoCloseable {
// default empty implementation
}
/**
* This method is called when the camera device has started reading out the output
* image for the request, at the beginning of the sensor image readout.
*
* <p>For a capture request, this callback is invoked right after
* {@link #onCaptureStarted}. Unlike {@link #onCaptureStarted}, instead of passing
* a timestamp of start of exposure, this callback passes a timestamp of start of
* camera data readout. This is useful because for a camera running at fixed frame
* rate, the start of readout is at fixed interval, but not necessary for the start
* of exposure.</p>
*
* <p>This timestamp may not match {@link CaptureResult#SENSOR_TIMESTAMP the result
* timestamp field}. It will, however, match the timestamp of buffers sent to the
* output surfaces with {@link OutputConfiguration#TIMESTAMP_BASE_READOUT_SENSOR}
* timestamp base.</p>
*
* <p>This callback will be called only if {@link
* CameraCharacteristics#SENSOR_READOUT_TIMESTAMP} is
* {@link CameraMetadata#SENSOR_READOUT_TIMESTAMP_HARDWARE}, and it's called
* right after {@link #onCaptureStarted}.</p>
*
* @param session the session returned by {@link CameraDevice#createCaptureSession}
* @param request the request for the readout that just began
* @param timestamp the timestamp at start of readout for a regular request, or
* the timestamp at the input image's start of readout for a
* reprocess request, in nanoseconds.
* @param frameNumber the frame number for this capture
*
* @hide
*/
public void onReadoutStarted(@NonNull CameraCaptureSession session,
@NonNull CaptureRequest request, long timestamp, long frameNumber) {
// default empty implementation
}
/**
* This method is called when some results from an image capture are
* available.

34
core/java/android/hardware/camera2/CameraCharacteristics.java
View File

@@ -4381,6 +4381,40 @@ public final class CameraCharacteristics extends CameraMetadata<CameraCharacteri
public static final Key<android.graphics.Rect[]> SENSOR_OPTICAL_BLACK_REGIONS =
new Key<android.graphics.Rect[]>("android.sensor.opticalBlackRegions", android.graphics.Rect[].class);
/**
* <p>Whether or not the camera device supports readout timestamp and
* onReadoutStarted callback.</p>
* <p>If this tag is HARDWARE, the camera device calls onReadoutStarted in addition to the
* onCaptureStarted callback for each capture. The timestamp passed into the callback
* is the start of camera image readout rather than the start of the exposure. In
* addition, the application can configure an
* {@link android.hardware.camera2.params.OutputConfiguration } with
* TIMESTAMP_BASE_READOUT_SENSOR timestamp base, in which case, the timestamp of the
* output surface matches the timestamp from the corresponding onReadoutStarted callback.</p>
* <p>The readout timestamp is beneficial for video recording, because the encoder favors
* uniform timestamps, and the readout timestamps better reflect the cadence camera sensors
* output data.</p>
* <p>If this tag is HARDWARE, the camera device produces the start-of-exposure and
* start-of-readout together. As a result, the onReadoutStarted is called right after
* onCaptureStarted. The difference in start-of-readout and start-of-exposure is the sensor
* exposure time, plus certain constant offset. The offset is usually due to camera sensor
* level crop, and it remains constant for a given camera sensor mode.</p>
* <p><b>Possible values:</b></p>
* <ul>
* <li>{@link #SENSOR_READOUT_TIMESTAMP_NOT_SUPPORTED NOT_SUPPORTED}</li>
* <li>{@link #SENSOR_READOUT_TIMESTAMP_HARDWARE HARDWARE}</li>
* </ul>
*
* <p>This key is available on all devices.</p>
* @see #SENSOR_READOUT_TIMESTAMP_NOT_SUPPORTED
* @see #SENSOR_READOUT_TIMESTAMP_HARDWARE
* @hide
*/
@PublicKey
@NonNull
public static final Key<Integer> SENSOR_READOUT_TIMESTAMP =
new Key<Integer>("android.sensor.readoutTimestamp", int.class);
/**
* <p>List of lens shading modes for {@link CaptureRequest#SHADING_MODE android.shading.mode} that are supported by this camera device.</p>
* <p>This list contains lens shading modes that can be set for the camera device.

22
core/java/android/hardware/camera2/CameraMetadata.java
View File

@@ -1649,6 +1649,28 @@ public abstract class CameraMetadata<TKey> {
*/
public static final int SENSOR_REFERENCE_ILLUMINANT1_ISO_STUDIO_TUNGSTEN = 24;
//
// Enumeration values for CameraCharacteristics#SENSOR_READOUT_TIMESTAMP
//
/**
* <p>This camera device doesn't support readout timestamp and onReadoutStarted
* callback.</p>
* @see CameraCharacteristics#SENSOR_READOUT_TIMESTAMP
* @hide
*/
public static final int SENSOR_READOUT_TIMESTAMP_NOT_SUPPORTED = 0;
/**
* <p>This camera device supports the onReadoutStarted callback as well as outputting
* readout timestamp for streams with TIMESTAMP_BASE_READOUT_SENSOR timestamp base. The
* readout timestamp is generated by the camera hardware and it has the same accuracy
* and timing characteristics of the start-of-exposure time.</p>
* @see CameraCharacteristics#SENSOR_READOUT_TIMESTAMP
* @hide
*/
public static final int SENSOR_READOUT_TIMESTAMP_HARDWARE = 1;
//
// Enumeration values for CameraCharacteristics#LED_AVAILABLE_LEDS
//

15
core/java/android/hardware/camera2/impl/CameraCaptureSessionImpl.java
View File

@@ -656,6 +656,21 @@ public class CameraCaptureSessionImpl extends CameraCaptureSession
}
}
@Override
public void onReadoutStarted(CameraDevice camera,
CaptureRequest request, long timestamp, long frameNumber) {
if ((callback != null) && (executor != null)) {
final long ident = Binder.clearCallingIdentity();
try {
executor.execute(() -> callback.onReadoutStarted(
CameraCaptureSessionImpl.this, request, timestamp,
frameNumber));
} finally {
Binder.restoreCallingIdentity(ident);
}
}
}
@Override
public void onCapturePartial(CameraDevice camera,
CaptureRequest request, android.hardware.camera2.CaptureResult result) {

24
core/java/android/hardware/camera2/impl/CameraDeviceImpl.java
View File

@@ -2025,12 +2025,16 @@ public class CameraDeviceImpl extends CameraDevice
resultExtras.getLastCompletedReprocessFrameNumber();
final long lastCompletedZslFrameNumber =
resultExtras.getLastCompletedZslFrameNumber();
final boolean hasReadoutTimestamp = resultExtras.hasReadoutTimestamp();
final long readoutTimestamp = resultExtras.getReadoutTimestamp();
if (DEBUG) {
Log.d(TAG, "Capture started for id " + requestId + " frame number " + frameNumber
+ ": completedRegularFrameNumber " + lastCompletedRegularFrameNumber
+ ", completedReprocessFrameNUmber " + lastCompletedReprocessFrameNumber
+ ", completedZslFrameNumber " + lastCompletedZslFrameNumber);
+ ", completedZslFrameNumber " + lastCompletedZslFrameNumber
+ ", hasReadoutTimestamp " + hasReadoutTimestamp
+ (hasReadoutTimestamp ? ", readoutTimestamp " + readoutTimestamp : "")) ;
}
final CaptureCallbackHolder holder;
@@ -2082,14 +2086,26 @@ public class CameraDeviceImpl extends CameraDevice
CameraDeviceImpl.this,
holder.getRequest(i),
timestamp - (subsequenceId - i) *
NANO_PER_SECOND/fpsRange.getUpper(),
NANO_PER_SECOND / fpsRange.getUpper(),
frameNumber - (subsequenceId - i));
if (hasReadoutTimestamp) {
holder.getCallback().onReadoutStarted(
CameraDeviceImpl.this,
holder.getRequest(i),
readoutTimestamp - (subsequenceId - i) *
NANO_PER_SECOND / fpsRange.getUpper(),
frameNumber - (subsequenceId - i));
}
}
} else {
holder.getCallback().onCaptureStarted(
CameraDeviceImpl.this,
holder.getRequest(resultExtras.getSubsequenceId()),
CameraDeviceImpl.this, request,
timestamp, frameNumber);
if (hasReadoutTimestamp) {
holder.getCallback().onReadoutStarted(
CameraDeviceImpl.this, request,
readoutTimestamp, frameNumber);
}
}
}
}

7
core/java/android/hardware/camera2/impl/CaptureCallback.java
View File

@@ -65,6 +65,13 @@ public abstract class CaptureCallback {
public abstract void onCaptureStarted(CameraDevice camera,
CaptureRequest request, long timestamp, long frameNumber);
/**
* This method is called when the camera device has started reading out the output
* image for the request, at the beginning of the sensor image readout.
*/
public abstract void onReadoutStarted(CameraDevice camera,
CaptureRequest request, long timestamp, long frameNumber);
/**
* This method is called when some results from an image capture are
* available.

23
core/java/android/hardware/camera2/impl/CaptureResultExtras.java
View File

@@ -33,6 +33,8 @@ public class CaptureResultExtras implements Parcelable {
private long lastCompletedRegularFrameNumber;
private long lastCompletedReprocessFrameNumber;
private long lastCompletedZslFrameNumber;
private boolean hasReadoutTimestamp;
private long readoutTimestamp;
public static final @android.annotation.NonNull Parcelable.Creator<CaptureResultExtras> CREATOR =
new Parcelable.Creator<CaptureResultExtras>() {
@@ -56,7 +58,8 @@ public class CaptureResultExtras implements Parcelable {
int partialResultCount, int errorStreamId,
String errorPhysicalCameraId, long lastCompletedRegularFrameNumber,
long lastCompletedReprocessFrameNumber,
long lastCompletedZslFrameNumber) {
long lastCompletedZslFrameNumber, boolean hasReadoutTimestamp,
long readoutTimestamp) {
this.requestId = requestId;
this.subsequenceId = subsequenceId;
this.afTriggerId = afTriggerId;
@@ -68,6 +71,8 @@ public class CaptureResultExtras implements Parcelable {
this.lastCompletedRegularFrameNumber = lastCompletedRegularFrameNumber;
this.lastCompletedReprocessFrameNumber = lastCompletedReprocessFrameNumber;
this.lastCompletedZslFrameNumber = lastCompletedZslFrameNumber;
this.hasReadoutTimestamp = hasReadoutTimestamp;
this.readoutTimestamp = readoutTimestamp;
}
@Override
@@ -93,6 +98,10 @@ public class CaptureResultExtras implements Parcelable {
dest.writeLong(lastCompletedRegularFrameNumber);
dest.writeLong(lastCompletedReprocessFrameNumber);
dest.writeLong(lastCompletedZslFrameNumber);
dest.writeBoolean(hasReadoutTimestamp);
if (hasReadoutTimestamp) {
dest.writeLong(readoutTimestamp);
}
}
public void readFromParcel(Parcel in) {
@@ -110,6 +119,10 @@ public class CaptureResultExtras implements Parcelable {
lastCompletedRegularFrameNumber = in.readLong();
lastCompletedReprocessFrameNumber = in.readLong();
lastCompletedZslFrameNumber = in.readLong();
hasReadoutTimestamp = in.readBoolean();
if (hasReadoutTimestamp) {
readoutTimestamp = in.readLong();
}
}
public String getErrorPhysicalCameraId() {
@@ -155,4 +168,12 @@ public class CaptureResultExtras implements Parcelable {
public long getLastCompletedZslFrameNumber() {
return lastCompletedZslFrameNumber;
}
public boolean hasReadoutTimestamp() {
return hasReadoutTimestamp;
}
public long getReadoutTimestamp() {
return readoutTimestamp;
}
}

25
core/java/android/hardware/camera2/params/OutputConfiguration.java
View File

@@ -236,6 +236,26 @@ public final class OutputConfiguration implements Parcelable {
*/
public static final int TIMESTAMP_BASE_CHOREOGRAPHER_SYNCED = 4;
/**
* Timestamp is the start of readout in the same time domain as TIMESTAMP_BASE_SENSOR.
*
* <p>The start of the camera sensor readout after exposure. For a rolling shutter camera
* sensor, the timestamp is typically equal to the start of exposure time +
* exposure time + certain fixed offset. The fixed offset could be due to camera sensor
* level crop. The benefit of using readout time is that when camera runs in a fixed
* frame rate, the timestamp intervals between frames are constant.</p>
*
* <p>This timestamp is in the same time domain as in TIMESTAMP_BASE_SENSOR, with the exception
* that one is start of exposure, and the other is start of readout.</p>
*
* <p>This timestamp base is supported only if {@link
* CameraCharacteristics#SENSOR_READOUT_TIMESTAMP} is
* {@link CameraMetadata#SENSOR_READOUT_TIMESTAMP_HARDWARE}.</p>
*
* @hide
*/
public static final int TIMESTAMP_BASE_READOUT_SENSOR = 5;
/** @hide */
@Retention(RetentionPolicy.SOURCE)
@IntDef(prefix = {"TIMESTAMP_BASE_"}, value =
@@ -243,7 +263,8 @@ public final class OutputConfiguration implements Parcelable {
TIMESTAMP_BASE_SENSOR,
TIMESTAMP_BASE_MONOTONIC,
TIMESTAMP_BASE_REALTIME,
TIMESTAMP_BASE_CHOREOGRAPHER_SYNCED})
TIMESTAMP_BASE_CHOREOGRAPHER_SYNCED,
TIMESTAMP_BASE_READOUT_SENSOR})
public @interface TimestampBase {};
/** @hide */
@@ -965,7 +986,7 @@ public final class OutputConfiguration implements Parcelable {
public void setTimestampBase(@TimestampBase int timestampBase) {
// Verify that the value is in range
if (timestampBase < TIMESTAMP_BASE_DEFAULT ||
timestampBase > TIMESTAMP_BASE_CHOREOGRAPHER_SYNCED) {
timestampBase > TIMESTAMP_BASE_READOUT_SENSOR) {
throw new IllegalArgumentException("Not a valid timestamp base value " +
timestampBase);
}