From db075afc85b6b50a5d3a988a17ed0d4e09ef0823 Mon Sep 17 00:00:00 2001 From: Igor Murashkin Date: Wed, 21 May 2014 10:07:08 -0700 Subject: [PATCH] camera2: Add partial results to the capture listener (API change) * Also adds capture sequence aborted callback Bug: 12492876 Change-Id: I56ec8edbcda8b64c2262ff7472961ac78042c7f4 --- api/current.txt | 18 ++- .../camera2/CameraCaptureSession.java | 99 +++++++++++++++-- .../hardware/camera2/CameraDevice.java | 103 ++++++++++++++++-- .../hardware/camera2/CaptureResult.java | 25 +++-- .../hardware/camera2/TotalCaptureResult.java | 84 ++++++++++++++ .../hardware/camera2/impl/CameraDevice.java | 13 ++- .../samples/simplecamera/Camera2Source.java | 3 +- 7 files changed, 309 insertions(+), 36 deletions(-) create mode 100644 core/java/android/hardware/camera2/TotalCaptureResult.java diff --git a/api/current.txt b/api/current.txt index 812dfc4976bd1..4a538c0f4a669 100644 --- a/api/current.txt +++ b/api/current.txt @@ -12140,9 +12140,11 @@ package android.hardware.camera2 { public static abstract class CameraCaptureSession.CaptureListener { ctor public CameraCaptureSession.CaptureListener(); - method public void onCaptureCompleted(android.hardware.camera2.CameraDevice, android.hardware.camera2.CaptureRequest, android.hardware.camera2.CaptureResult); + method public void onCaptureCompleted(android.hardware.camera2.CameraDevice, android.hardware.camera2.CaptureRequest, android.hardware.camera2.TotalCaptureResult); method public void onCaptureFailed(android.hardware.camera2.CameraDevice, android.hardware.camera2.CaptureRequest, android.hardware.camera2.CaptureFailure); - method public void onCaptureSequenceCompleted(android.hardware.camera2.CameraDevice, int, int); + method public void onCaptureProgressed(android.hardware.camera2.CameraDevice, android.hardware.camera2.CaptureRequest, android.hardware.camera2.CaptureResult); + method public void onCaptureSequenceAborted(android.hardware.camera2.CameraDevice, int); + method public void onCaptureSequenceCompleted(android.hardware.camera2.CameraDevice, int, long); method public void onCaptureStarted(android.hardware.camera2.CameraDevice, android.hardware.camera2.CaptureRequest, long); } @@ -12249,9 +12251,11 @@ package android.hardware.camera2 { public static abstract deprecated class CameraDevice.CaptureListener { ctor public CameraDevice.CaptureListener(); - method public void onCaptureCompleted(android.hardware.camera2.CameraDevice, android.hardware.camera2.CaptureRequest, android.hardware.camera2.CaptureResult); + method public void onCaptureCompleted(android.hardware.camera2.CameraDevice, android.hardware.camera2.CaptureRequest, android.hardware.camera2.TotalCaptureResult); method public void onCaptureFailed(android.hardware.camera2.CameraDevice, android.hardware.camera2.CaptureRequest, android.hardware.camera2.CaptureFailure); - method public void onCaptureSequenceCompleted(android.hardware.camera2.CameraDevice, int, int); + method public void onCaptureProgressed(android.hardware.camera2.CameraDevice, android.hardware.camera2.CaptureRequest, android.hardware.camera2.CaptureResult); + method public void onCaptureSequenceAborted(android.hardware.camera2.CameraDevice, int); + method public void onCaptureSequenceCompleted(android.hardware.camera2.CameraDevice, int, long); method public void onCaptureStarted(android.hardware.camera2.CameraDevice, android.hardware.camera2.CaptureRequest, long); } @@ -12542,7 +12546,7 @@ package android.hardware.camera2 { method public final int hashCode(); } - public final class CaptureResult extends android.hardware.camera2.CameraMetadata { + public class CaptureResult extends android.hardware.camera2.CameraMetadata { method public T get(android.hardware.camera2.CaptureResult.Key); method public int getFrameNumber(); method public android.hardware.camera2.CaptureRequest getRequest(); @@ -12635,6 +12639,10 @@ package android.hardware.camera2 { method public void writeInputStream(java.io.OutputStream, android.util.Size, java.io.InputStream, long) throws java.io.IOException; } + public final class TotalCaptureResult extends android.hardware.camera2.CaptureResult { + method public java.util.List getPartialResults(); + } + } package android.hardware.camera2.params { diff --git a/core/java/android/hardware/camera2/CameraCaptureSession.java b/core/java/android/hardware/camera2/CameraCaptureSession.java index 8391209687fa5..7738d2d498aef 100644 --- a/core/java/android/hardware/camera2/CameraCaptureSession.java +++ b/core/java/android/hardware/camera2/CameraCaptureSession.java @@ -508,14 +508,62 @@ public abstract class CameraCaptureSession implements AutoCloseable { } /** - * This method is called when an image capture has completed and the - * result metadata is available. + * This method is called when an image capture makes partial forward progress; some + * (but not all) results from an image capture are available. + * + *

The result provided here will contain some subset of the fields of + * a full result. Multiple {@link #onCaptureProgressed} calls may happen per + * capture; a given result field will only be present in one partial + * capture at most. The final {@link #onCaptureCompleted} call will always + * contain all the fields (in particular, the union of all the fields of all + * the partial results composing the total result).

+ * + *

For each request, some result data might be available earlier than others. The typical + * delay between each partial result (per request) is a single frame interval. + * For performance-oriented use-cases, applications should query the metadata they need + * to make forward progress from the partial results and avoid waiting for the completed + * result.

+ * + *

Each request will generate at least {@code 1} partial results, and at most + * {@link CameraCharacteristics#REQUEST_PARTIAL_RESULT_COUNT} partial results.

+ * + *

Depending on the request settings, the number of partial results per request + * will vary, although typically the partial count could be the same as long as the + * camera device subsystems enabled stay the same.

* *

The default implementation of this method does nothing.

* * @param camera The CameraDevice sending the callback. * @param request The request that was given to the CameraDevice - * @param result The output metadata from the capture, including the + * @param partialResult The partial output metadata from the capture, which + * includes a subset of the {@link TotalCaptureResult} fields. + * + * @see #capture + * @see #captureBurst + * @see #setRepeatingRequest + * @see #setRepeatingBurst + */ + public void onCaptureProgressed(CameraDevice camera, + CaptureRequest request, CaptureResult partialResult) { + // default empty implementation + } + + /** + * This method is called when an image capture has fully completed and all the + * result metadata is available. + * + *

This callback will always fire after the last {@link #onCaptureProgressed}; + * in other words, no more partial results will be delivered once the completed result + * is available.

+ * + *

For performance-intensive use-cases where latency is a factor, consider + * using {@link #onCaptureProgressed} instead.

+ * + *

The default implementation of this method does nothing.

+ * + * @param camera The CameraDevice sending the callback. + * @param request The request that was given to the CameraDevice + * @param result The total output metadata from the capture, including the * final capture parameters and the state of the camera system during * capture. * @@ -525,7 +573,7 @@ public abstract class CameraCaptureSession implements AutoCloseable { * @see #setRepeatingBurst */ public void onCaptureCompleted(CameraDevice camera, - CaptureRequest request, CaptureResult result) { + CaptureRequest request, TotalCaptureResult result) { // default empty implementation } @@ -563,24 +611,57 @@ public abstract class CameraCaptureSession implements AutoCloseable { * when a capture sequence finishes and all {@link CaptureResult} * or {@link CaptureFailure} for it have been returned via this listener. * + *

In total, there will be at least one result/failure returned by this listener + * before this callback is invoked. If the capture sequence is aborted before any + * requests have been processed, {@link #onCaptureSequenceAborted} is invoked instead.

+ * + *

The default implementation does nothing.

+ * * @param camera * The CameraDevice sending the callback. * @param sequenceId * A sequence ID returned by the {@link #capture} family of functions. - * @param lastFrameNumber + * @param frameNumber * The last frame number (returned by {@link CaptureResult#getFrameNumber} * or {@link CaptureFailure#getFrameNumber}) in the capture sequence. - * The last frame number may be equal to NO_FRAMES_CAPTURED if no images - * were captured for this sequence. This can happen, for example, when a - * repeating request or burst is cleared right after being set. * * @see CaptureResult#getFrameNumber() * @see CaptureFailure#getFrameNumber() * @see CaptureResult#getSequenceId() * @see CaptureFailure#getSequenceId() + * @see #onCaptureSequenceAborted */ public void onCaptureSequenceCompleted(CameraDevice camera, - int sequenceId, int lastFrameNumber) { + int sequenceId, long frameNumber) { + // default empty implementation + } + + /** + * This method is called independently of the others in CaptureListener, + * when a capture sequence aborts before any {@link CaptureResult} + * or {@link CaptureFailure} for it have been returned via this listener. + * + *

Due to the asynchronous nature of the camera device, not all submitted captures + * are immediately processed. It is possible to clear out the pending requests + * by a variety of operations such as {@link CameraDevice#stopRepeating} or + * {@link CameraDevice#flush}. When such an event happens, + * {@link #onCaptureSequenceCompleted} will not be called.

+ * + *

The default implementation does nothing.

+ * + * @param camera + * The CameraDevice sending the callback. + * @param sequenceId + * A sequence ID returned by the {@link #capture} family of functions. + * + * @see CaptureResult#getFrameNumber() + * @see CaptureFailure#getFrameNumber() + * @see CaptureResult#getSequenceId() + * @see CaptureFailure#getSequenceId() + * @see #onCaptureSequenceCompleted + */ + public void onCaptureSequenceAborted(CameraDevice camera, + int sequenceId) { // default empty implementation } } diff --git a/core/java/android/hardware/camera2/CameraDevice.java b/core/java/android/hardware/camera2/CameraDevice.java index 77640d1d338db..6f5099b5b8cac 100644 --- a/core/java/android/hardware/camera2/CameraDevice.java +++ b/core/java/android/hardware/camera2/CameraDevice.java @@ -766,14 +766,62 @@ public interface CameraDevice extends AutoCloseable { } /** - * This method is called when an image capture has completed and the - * result metadata is available. + * This method is called when an image capture makes partial forward progress; some + * (but not all) results from an image capture are available. + * + *

The result provided here will contain some subset of the fields of + * a full result. Multiple {@link #onCaptureProgressed} calls may happen per + * capture; a given result field will only be present in one partial + * capture at most. The final {@link #onCaptureCompleted} call will always + * contain all the fields (in particular, the union of all the fields of all + * the partial results composing the total result).

+ * + *

For each request, some result data might be available earlier than others. The typical + * delay between each partial result (per request) is a single frame interval. + * For performance-oriented use-cases, applications should query the metadata they need + * to make forward progress from the partial results and avoid waiting for the completed + * result.

+ * + *

Each request will generate at least {@code 1} partial results, and at most + * {@link CameraCharacteristics#REQUEST_PARTIAL_RESULT_COUNT} partial results.

+ * + *

Depending on the request settings, the number of partial results per request + * will vary, although typically the partial count could be the same as long as the + * camera device subsystems enabled stay the same.

* *

The default implementation of this method does nothing.

* * @param camera The CameraDevice sending the callback. * @param request The request that was given to the CameraDevice - * @param result The output metadata from the capture, including the + * @param partialResult The partial output metadata from the capture, which + * includes a subset of the {@link TotalCaptureResult} fields. + * + * @see #capture + * @see #captureBurst + * @see #setRepeatingRequest + * @see #setRepeatingBurst + */ + public void onCaptureProgressed(CameraDevice camera, + CaptureRequest request, CaptureResult partialResult) { + // default empty implementation + } + + /** + * This method is called when an image capture has fully completed and all the + * result metadata is available. + * + *

This callback will always fire after the last {@link #onCaptureProgressed}; + * in other words, no more partial results will be delivered once the completed result + * is available.

+ * + *

For performance-intensive use-cases where latency is a factor, consider + * using {@link #onCaptureProgressed} instead.

+ * + *

The default implementation of this method does nothing.

+ * + * @param camera The CameraDevice sending the callback. + * @param request The request that was given to the CameraDevice + * @param result The total output metadata from the capture, including the * final capture parameters and the state of the camera system during * capture. * @@ -783,7 +831,7 @@ public interface CameraDevice extends AutoCloseable { * @see #setRepeatingBurst */ public void onCaptureCompleted(CameraDevice camera, - CaptureRequest request, CaptureResult result) { + CaptureRequest request, TotalCaptureResult result) { // default empty implementation } @@ -796,6 +844,10 @@ public interface CameraDevice extends AutoCloseable { * the capture may have been pushed to their respective output * streams.

* + *

Some partial results may have been delivered before the capture fails; + * however after this callback fires, no more partial results will be delivered by + * {@link #onCaptureProgressed}.

+ * *

The default implementation of this method does nothing.

* * @param camera @@ -821,24 +873,57 @@ public interface CameraDevice extends AutoCloseable { * when a capture sequence finishes and all {@link CaptureResult} * or {@link CaptureFailure} for it have been returned via this listener. * + *

In total, there will be at least one result/failure returned by this listener + * before this callback is invoked. If the capture sequence is aborted before any + * requests have been processed, {@link #onCaptureSequenceAborted} is invoked instead.

+ * + *

The default implementation does nothing.

+ * * @param camera * The CameraDevice sending the callback. * @param sequenceId * A sequence ID returned by the {@link #capture} family of functions. - * @param lastFrameNumber + * @param frameNumber * The last frame number (returned by {@link CaptureResult#getFrameNumber} * or {@link CaptureFailure#getFrameNumber}) in the capture sequence. - * The last frame number may be equal to NO_FRAMES_CAPTURED if no images - * were captured for this sequence. This can happen, for example, when a - * repeating request or burst is cleared right after being set. * * @see CaptureResult#getFrameNumber() * @see CaptureFailure#getFrameNumber() * @see CaptureResult#getSequenceId() * @see CaptureFailure#getSequenceId() + * @see #onCaptureSequenceAborted */ public void onCaptureSequenceCompleted(CameraDevice camera, - int sequenceId, int lastFrameNumber) { + int sequenceId, long frameNumber) { + // default empty implementation + } + + /** + * This method is called independently of the others in CaptureListener, + * when a capture sequence aborts before any {@link CaptureResult} + * or {@link CaptureFailure} for it have been returned via this listener. + * + *

Due to the asynchronous nature of the camera device, not all submitted captures + * are immediately processed. It is possible to clear out the pending requests + * by a variety of operations such as {@link CameraDevice#stopRepeating} or + * {@link CameraDevice#flush}. When such an event happens, + * {@link #onCaptureSequenceCompleted} will not be called.

+ * + *

The default implementation does nothing.

+ * + * @param camera + * The CameraDevice sending the callback. + * @param sequenceId + * A sequence ID returned by the {@link #capture} family of functions. + * + * @see CaptureResult#getFrameNumber() + * @see CaptureFailure#getFrameNumber() + * @see CaptureResult#getSequenceId() + * @see CaptureFailure#getSequenceId() + * @see #onCaptureSequenceCompleted + */ + public void onCaptureSequenceAborted(CameraDevice camera, + int sequenceId) { // default empty implementation } } diff --git a/core/java/android/hardware/camera2/CaptureResult.java b/core/java/android/hardware/camera2/CaptureResult.java index 61d491b593e9f..9aa56cf6636e4 100644 --- a/core/java/android/hardware/camera2/CaptureResult.java +++ b/core/java/android/hardware/camera2/CaptureResult.java @@ -24,9 +24,9 @@ import android.util.Rational; import java.util.List; /** - *

The results of a single image capture from the image sensor.

+ *

The subset of the results of a single image capture from the image sensor.

* - *

Contains the final configuration for the capture hardware (sensor, lens, + *

Contains a subset of the final configuration for the capture hardware (sensor, lens, * flash), the processing pipeline, the control algorithms, and the output * buffers.

* @@ -36,10 +36,15 @@ import java.util.List; * capture. The result also includes additional metadata about the state of the * camera device during the capture.

* - *

{@link CameraCharacteristics} objects are immutable.

+ *

Not all properties returned by {@link CameraCharacteristics#getAvailableCaptureResultKeys()} + * are necessarily available. Some results are {@link CaptureResult partial} and will + * not have every key set. Only {@link TotalCaptureResult total} results are guaranteed to have + * every key available that was enabled by the request.

+ * + *

{@link CaptureResult} objects are immutable.

* */ -public final class CaptureResult extends CameraMetadata> { +public class CaptureResult extends CameraMetadata> { private static final String TAG = "CaptureResult"; private static final boolean VERBOSE = false; @@ -249,18 +254,19 @@ public final class CaptureResult extends CameraMetadata> { /** * Get the request associated with this result. * - *

Whenever a request is successfully captured, with - * {@link CameraDevice.CaptureListener#onCaptureCompleted}, - * the {@code result}'s {@code getRequest()} will return that {@code request}. + *

Whenever a request has been fully or partially captured, with + * {@link CameraDevice.CaptureListener#onCaptureCompleted} or + * {@link CameraDevice.CaptureListener#onCaptureProgressed}, the {@code result}'s + * {@code getRequest()} will return that {@code request}. *

* - *

In particular, + *

For example, * 

cameraDevice.capture(someRequest, new CaptureListener() {
      *     {@literal @}Override
      *     void onCaptureCompleted(CaptureRequest myRequest, CaptureResult myResult) {
      *         assert(myResult.getRequest.equals(myRequest) == true);
      *     }
-     * };
+     * }, null);
      *
*

* @@ -297,6 +303,7 @@ public final class CaptureResult extends CameraMetadata> { * @return int The ID for the sequence of requests that this capture result is a part of * * @see CameraDevice.CaptureListener#onCaptureSequenceCompleted + * @see CameraDevice.CaptureListener#onCaptureSequenceAborted */ public int getSequenceId() { return mSequenceId; diff --git a/core/java/android/hardware/camera2/TotalCaptureResult.java b/core/java/android/hardware/camera2/TotalCaptureResult.java new file mode 100644 index 0000000000000..2647959730371 --- /dev/null +++ b/core/java/android/hardware/camera2/TotalCaptureResult.java @@ -0,0 +1,84 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.hardware.camera2; + +import android.hardware.camera2.impl.CameraMetadataNative; + +import java.util.Collections; +import java.util.List; + +/** + *

The total assembled results of a single image capture from the image sensor.

+ * + *

Contains the final configuration for the capture hardware (sensor, lens, + * flash), the processing pipeline, the control algorithms, and the output + * buffers.

+ * + *

A {@code TotalCaptureResult} is produced by a {@link CameraDevice} after processing a + * {@link CaptureRequest}. All properties listed for capture requests can also + * be queried on the capture result, to determine the final values used for + * capture. The result also includes additional metadata about the state of the + * camera device during the capture.

+ * + *

All properties returned by {@link CameraCharacteristics#getAvailableCaptureResultKeys()} + * are available (that is {@link CaptureResult#get} will return non-{@code null}, if and only if + * that key that was enabled by the request. A few keys such as + * {@link CaptureResult#STATISTICS_FACES} are disabled by default unless enabled with a switch (such + * as {@link CaptureRequest#STATISTICS_FACE_DETECT_MODE}). Refer to each key documentation on + * a case-by-case basis.

+ * + *

{@link TotalCaptureResult} objects are immutable.

+ * + * @see CameraDevice.CaptureListener#onCaptureCompleted + */ +public final class TotalCaptureResult extends CaptureResult { + + /** + * Takes ownership of the passed-in properties object + * @hide + */ + public TotalCaptureResult(CameraMetadataNative results, CaptureRequest parent, int sequenceId) { + super(results, parent, sequenceId); + } + + /** + * Creates a request-less result. + * + *

For testing only.

+ * @hide + */ + public TotalCaptureResult(CameraMetadataNative results, int sequenceId) { + super(results, sequenceId); + } + + /** + * Get the read-only list of partial results that compose this total result. + * + *

The list is returned is unmodifiable; attempting to modify it will result in a + * {@code UnsupportedOperationException} being thrown.

+ * + *

The list size will be inclusive between {@code 1} and + * {@link CameraCharacteristics#REQUEST_PARTIAL_RESULT_COUNT}, in ascending order + * of when {@link CameraDevice.CaptureListener#onCaptureProgressed} was invoked.

+ * + * @return unmodifiable list of partial results + */ + public List getPartialResults() { + // TODO + return Collections.unmodifiableList(null); + } +} diff --git a/core/java/android/hardware/camera2/impl/CameraDevice.java b/core/java/android/hardware/camera2/impl/CameraDevice.java index b082a70764527..7b249766a9773 100644 --- a/core/java/android/hardware/camera2/impl/CameraDevice.java +++ b/core/java/android/hardware/camera2/impl/CameraDevice.java @@ -24,6 +24,7 @@ import android.hardware.camera2.CaptureRequest; import android.hardware.camera2.CaptureResult; import android.hardware.camera2.ICameraDeviceCallbacks; import android.hardware.camera2.ICameraDeviceUser; +import android.hardware.camera2.TotalCaptureResult; import android.hardware.camera2.utils.CameraBinderDecorator; import android.hardware.camera2.utils.CameraRuntimeException; import android.hardware.camera2.utils.LongParcelable; @@ -359,7 +360,7 @@ public class CameraDevice implements android.hardware.camera2.CameraDevice { holder.getListener().onCaptureSequenceCompleted( CameraDevice.this, requestId, - (int)lastFrameNumber); + lastFrameNumber); } } }; @@ -717,7 +718,7 @@ public class CameraDevice implements android.hardware.camera2.CameraDevice { holder.getListener().onCaptureSequenceCompleted( CameraDevice.this, requestId, - (int)lastFrameNumber); + lastFrameNumber); } } }; @@ -888,12 +889,15 @@ public class CameraDevice implements android.hardware.camera2.CameraDevice { } final CaptureRequest request = holder.getRequest(resultExtras.getSubsequenceId()); - final CaptureResult resultAsCapture = new CaptureResult(result, request, requestId); + Runnable resultDispatch = null; // Either send a partial result or the final capture completed result if (quirkIsPartialResult) { + final CaptureResult resultAsCapture = + new CaptureResult(result, request, requestId); + // Partial result resultDispatch = new Runnable() { @Override @@ -907,6 +911,9 @@ public class CameraDevice implements android.hardware.camera2.CameraDevice { } }; } else { + final TotalCaptureResult resultAsCapture = + new TotalCaptureResult(result, request, requestId); + // Final capture result resultDispatch = new Runnable() { @Override diff --git a/tests/Camera2Tests/SmartCamera/SimpleCamera/src/androidx/media/filterfw/samples/simplecamera/Camera2Source.java b/tests/Camera2Tests/SmartCamera/SimpleCamera/src/androidx/media/filterfw/samples/simplecamera/Camera2Source.java index fa0f995174eaa..6876f5afa8460 100644 --- a/tests/Camera2Tests/SmartCamera/SimpleCamera/src/androidx/media/filterfw/samples/simplecamera/Camera2Source.java +++ b/tests/Camera2Tests/SmartCamera/SimpleCamera/src/androidx/media/filterfw/samples/simplecamera/Camera2Source.java @@ -26,6 +26,7 @@ import android.hardware.camera2.CameraManager; import android.hardware.camera2.CaptureFailure; import android.hardware.camera2.CaptureRequest; import android.hardware.camera2.CaptureResult; +import android.hardware.camera2.TotalCaptureResult; import android.os.Handler; import android.renderscript.Allocation; import android.renderscript.Element; @@ -86,7 +87,7 @@ public class Camera2Source extends Filter implements Allocation.OnBufferAvailabl @Override public void onCaptureCompleted(CameraDevice camera, CaptureRequest request, - CaptureResult result) { + TotalCaptureResult result) { // TODO Auto-generated method stub Log.v(TAG, "in onCaptureComplete");