From ceb6591e53c6bf5297a5dac93613e18d32ce6278 Mon Sep 17 00:00:00 2001 From: Emilian Peev Date: Fri, 29 Apr 2022 17:58:18 -0700 Subject: [PATCH] Camera: Propagate extension results in case of dropped frames Capture results can be present and can be safely passed back to client in case of frame drops within the source image reader. Detect cases where results will get discarded, filter the tags and notify clients accordingly. Bug: 230790669 Test: Camera CTS Change-Id: Ia3f77d9398165663073c01bdf8d3421fd86a67c4 --- .../impl/CameraExtensionSessionImpl.java | 129 +++++++++++++----- 1 file changed, 96 insertions(+), 33 deletions(-) diff --git a/core/java/android/hardware/camera2/impl/CameraExtensionSessionImpl.java b/core/java/android/hardware/camera2/impl/CameraExtensionSessionImpl.java index 1263da6e4884c..4d0ba63d7759e 100644 --- a/core/java/android/hardware/camera2/impl/CameraExtensionSessionImpl.java +++ b/core/java/android/hardware/camera2/impl/CameraExtensionSessionImpl.java @@ -72,7 +72,7 @@ import java.util.Set; import java.util.concurrent.Executor; public final class CameraExtensionSessionImpl extends CameraExtensionSession { - private static final int PREVIEW_QUEUE_SIZE = 3; + private static final int PREVIEW_QUEUE_SIZE = 10; private static final String TAG = "CameraExtensionSessionImpl"; private final Executor mExecutor; @@ -1057,15 +1057,8 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession { mClientRequest)); if (mCaptureResultHandler != null) { - CameraMetadataNative captureResults = new CameraMetadataNative(); - for (CaptureResult.Key key : mSupportedResultKeys) { - Object value = result.get(key); - if (value != null) { - captureResults.set(key, value); - } - } mCaptureResultHandler.onCaptureCompleted(timestamp, - captureResults); + initializeFilteredResults(result)); } } finally { Binder.restoreCallingIdentity(ident); @@ -1126,6 +1119,11 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession { } private class ImageCallback implements OnImageAvailableListener { + @Override + public void onImageDropped(long timestamp) { + notifyCaptureFailed(); + } + @Override public void onImageAvailable(ImageReader reader, Image img) { if (mCaptureFailed) { @@ -1159,6 +1157,9 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession { } private class ImageLoopbackCallback implements OnImageAvailableListener { + @Override + public void onImageDropped(long timestamp) { } + @Override public void onImageAvailable(ImageReader reader, Image img) { img.close(); @@ -1221,7 +1222,8 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession { } private interface OnImageAvailableListener { - public void onImageAvailable (ImageReader reader, Image img); + void onImageDropped(long timestamp); + void onImageAvailable (ImageReader reader, Image img); } private class CameraOutputImageCallback implements ImageReader.OnImageAvailableListener, @@ -1263,6 +1265,29 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession { } else { mImageListenerMap.put(img.getTimestamp(), new Pair<>(img, null)); } + + notifyDroppedImages(timestamp); + } + + private void notifyDroppedImages(long timestamp) { + Set timestamps = mImageListenerMap.keySet(); + ArrayList removedTs = new ArrayList<>(); + for (long ts : timestamps) { + if (ts < timestamp) { + Log.e(TAG, "Dropped image with ts: " + ts); + Pair entry = mImageListenerMap.get(ts); + if (entry.second != null) { + entry.second.onImageDropped(ts); + } + if (entry.first != null) { + entry.first.close(); + } + removedTs.add(ts); + } + } + for (long ts : removedTs) { + mImageListenerMap.remove(ts); + } } public void registerListener(Long timestamp, OnImageAvailableListener listener) { @@ -1291,6 +1316,12 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession { entry.first.close(); } } + for (long timestamp : mImageListenerMap.keySet()) { + Pair entry = mImageListenerMap.get(timestamp); + if (entry.second != null) { + entry.second.onImageDropped(timestamp); + } + } mImageListenerMap.clear(); } } @@ -1447,7 +1478,6 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession { } else { notifyConfigurationFailure(); } - } @Override @@ -1536,7 +1566,16 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession { } else if (mPreviewProcessorType == IPreviewExtenderImpl.PROCESSOR_TYPE_IMAGE_PROCESSOR) { int idx = mPendingResultMap.indexOfKey(timestamp); - if (idx >= 0) { + + if ((idx >= 0) && (mPendingResultMap.get(timestamp).first == null)) { + // Image was dropped before we can receive the capture results + if ((mCaptureResultHandler != null)) { + mCaptureResultHandler.onCaptureCompleted(timestamp, + initializeFilteredResults(result)); + } + discardPendingRepeatingResults(idx, mPendingResultMap, false); + } else if (idx >= 0) { + // Image came before the capture results ParcelImage parcelImage = initializeParcelImage( mPendingResultMap.get(timestamp).first); try { @@ -1563,6 +1602,7 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession { } discardPendingRepeatingResults(idx, mPendingResultMap, false); } else { + // Image not yet available notifyClient = false; mPendingResultMap.put(timestamp, new Pair<>(null, @@ -1581,16 +1621,8 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession { mClientRequest)); if ((mCaptureResultHandler != null) && (mPreviewProcessorType != IPreviewExtenderImpl.PROCESSOR_TYPE_IMAGE_PROCESSOR)) { - CameraMetadataNative captureResults = - new CameraMetadataNative(); - for (CaptureResult.Key key : mSupportedResultKeys) { - Object value = result.get(key); - if (value != null) { - captureResults.set(key, value); - } - } mCaptureResultHandler.onCaptureCompleted(timestamp, - captureResults); + initializeFilteredResults(result)); } } else { mExecutor.execute( @@ -1657,19 +1689,24 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession { for (int i = idx; i >= 0; i--) { if (previewMap.valueAt(i).first != null) { previewMap.valueAt(i).first.close(); - } else { - if (mClientNotificationsEnabled && ((i != idx) || notifyCurrentIndex)) { - Log.w(TAG, "Preview frame drop with timestamp: " + previewMap.keyAt(i)); - final long ident = Binder.clearCallingIdentity(); - try { - mExecutor.execute( - () -> mCallbacks - .onCaptureFailed(CameraExtensionSessionImpl.this, - mClientRequest)); - } finally { - Binder.restoreCallingIdentity(ident); - } + } else if (mClientNotificationsEnabled && (previewMap.valueAt(i).second != null) && + ((i != idx) || notifyCurrentIndex)) { + TotalCaptureResult result = previewMap.valueAt(i).second; + Long timestamp = result.get(CaptureResult.SENSOR_TIMESTAMP); + mCaptureResultHandler.onCaptureCompleted(timestamp, + initializeFilteredResults(result)); + + Log.w(TAG, "Preview frame drop with timestamp: " + previewMap.keyAt(i)); + final long ident = Binder.clearCallingIdentity(); + try { + mExecutor.execute( + () -> mCallbacks + .onCaptureFailed(CameraExtensionSessionImpl.this, + mClientRequest)); + } finally { + Binder.restoreCallingIdentity(ident); } + } previewMap.removeAt(i); } @@ -1682,6 +1719,12 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession { mOutputWriter = imageWriter; } + @Override + public void onImageDropped(long timestamp) { + discardPendingRepeatingResults(mPendingResultMap.indexOfKey(timestamp), + mPendingResultMap, true); + } + @Override public void onImageAvailable(ImageReader reader, Image img) { if (img == null) { @@ -1702,6 +1745,15 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession { private class ImageProcessCallback implements OnImageAvailableListener { + @Override + public void onImageDropped(long timestamp) { + discardPendingRepeatingResults(mPendingResultMap.indexOfKey(timestamp), + mPendingResultMap, true); + // Add an empty frame&results entry to flag that we dropped a frame + // and valid capture results can immediately return to client. + mPendingResultMap.put(timestamp, new Pair<>(null, null)); + } + @Override public void onImageAvailable(ImageReader reader, Image img) { if (mPendingResultMap.size() + 1 >= PREVIEW_QUEUE_SIZE) { @@ -1768,6 +1820,17 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession { } } + private CameraMetadataNative initializeFilteredResults(TotalCaptureResult result) { + CameraMetadataNative captureResults = new CameraMetadataNative(); + for (CaptureResult.Key key : mSupportedResultKeys) { + Object value = result.get(key); + if (value != null) { + captureResults.set(key, value); + } + } + return captureResults; + } + private static Size findSmallestAspectMatchedSize(@NonNull List sizes, @NonNull Size arSize) { final float TOLL = .01f;