Merge "Camera: Propagate extension results in case of dropped frames" into tm-dev

core/java/android/hardware/camera2/impl/CameraExtensionSessionImpl.java

@@ -72,7 +72,7 @@ import java.util.Set;
 import java.util.concurrent.Executor;
 
 public final class CameraExtensionSessionImpl extends CameraExtensionSession {
-    private static final int PREVIEW_QUEUE_SIZE = 3;
+    private static final int PREVIEW_QUEUE_SIZE = 10;
     private static final String TAG = "CameraExtensionSessionImpl";
 
     private final Executor mExecutor;
@@ -1057,15 +1057,8 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession {
                                                 mClientRequest));
 
                         if (mCaptureResultHandler != null) {
-                            CameraMetadataNative captureResults = new CameraMetadataNative();
-                            for (CaptureResult.Key key : mSupportedResultKeys) {
-                                Object value = result.get(key);
-                                if (value != null) {
-                                    captureResults.set(key, value);
-                                }
-                            }
                             mCaptureResultHandler.onCaptureCompleted(timestamp,
-                                    captureResults);
+                                    initializeFilteredResults(result));
                         }
                     } finally {
                         Binder.restoreCallingIdentity(ident);
@@ -1126,6 +1119,11 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession {
         }
 
         private class ImageCallback implements OnImageAvailableListener {
+            @Override
+            public void onImageDropped(long timestamp) {
+                notifyCaptureFailed();
+            }
+
             @Override
             public void onImageAvailable(ImageReader reader, Image img) {
                 if (mCaptureFailed) {
@@ -1159,6 +1157,9 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession {
     }
 
     private class ImageLoopbackCallback implements OnImageAvailableListener {
+        @Override
+        public void onImageDropped(long timestamp) { }
+
         @Override
         public void onImageAvailable(ImageReader reader, Image img) {
             img.close();
@@ -1221,7 +1222,8 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession {
     }
 
     private interface OnImageAvailableListener {
-        public void onImageAvailable (ImageReader reader, Image img);
+        void onImageDropped(long timestamp);
+        void onImageAvailable (ImageReader reader, Image img);
     }
 
     private class CameraOutputImageCallback implements ImageReader.OnImageAvailableListener,
@@ -1263,6 +1265,29 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession {
             } else {
                 mImageListenerMap.put(img.getTimestamp(), new Pair<>(img, null));
             }
+
+            notifyDroppedImages(timestamp);
+        }
+
+        private void notifyDroppedImages(long timestamp) {
+            Set<Long> timestamps = mImageListenerMap.keySet();
+            ArrayList<Long> removedTs = new ArrayList<>();
+            for (long ts : timestamps) {
+                if (ts < timestamp) {
+                    Log.e(TAG, "Dropped image with ts: " + ts);
+                    Pair<Image, OnImageAvailableListener> entry = mImageListenerMap.get(ts);
+                    if (entry.second != null) {
+                        entry.second.onImageDropped(ts);
+                    }
+                    if (entry.first != null) {
+                        entry.first.close();
+                    }
+                    removedTs.add(ts);
+                }
+            }
+            for (long ts : removedTs) {
+                mImageListenerMap.remove(ts);
+            }
         }
 
         public void registerListener(Long timestamp, OnImageAvailableListener listener) {
@@ -1291,6 +1316,12 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession {
                     entry.first.close();
                 }
             }
+            for (long timestamp : mImageListenerMap.keySet()) {
+                Pair<Image, OnImageAvailableListener> entry = mImageListenerMap.get(timestamp);
+                if (entry.second != null) {
+                    entry.second.onImageDropped(timestamp);
+                }
+            }
             mImageListenerMap.clear();
         }
     }
@@ -1447,7 +1478,6 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession {
             } else {
                 notifyConfigurationFailure();
             }
-
         }
 
         @Override
@@ -1536,7 +1566,16 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession {
                     } else if (mPreviewProcessorType ==
                             IPreviewExtenderImpl.PROCESSOR_TYPE_IMAGE_PROCESSOR) {
                         int idx = mPendingResultMap.indexOfKey(timestamp);
-                        if (idx >= 0) {
+
+                        if ((idx >= 0) && (mPendingResultMap.get(timestamp).first == null)) {
+                            // Image was dropped before we can receive the capture results
+                            if ((mCaptureResultHandler != null)) {
+                                mCaptureResultHandler.onCaptureCompleted(timestamp,
+                                        initializeFilteredResults(result));
+                            }
+                            discardPendingRepeatingResults(idx, mPendingResultMap, false);
+                        } else  if (idx >= 0) {
+                            // Image came before the capture results
                             ParcelImage parcelImage = initializeParcelImage(
                                     mPendingResultMap.get(timestamp).first);
                             try {
@@ -1563,6 +1602,7 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession {
                             }
                             discardPendingRepeatingResults(idx, mPendingResultMap, false);
                         } else {
+                            // Image not yet available
                             notifyClient = false;
                             mPendingResultMap.put(timestamp,
                                     new Pair<>(null,
@@ -1581,16 +1621,8 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession {
                                                 mClientRequest));
                                 if ((mCaptureResultHandler != null) && (mPreviewProcessorType !=
                                         IPreviewExtenderImpl.PROCESSOR_TYPE_IMAGE_PROCESSOR)) {
-                                    CameraMetadataNative captureResults =
-                                            new CameraMetadataNative();
-                                    for (CaptureResult.Key key : mSupportedResultKeys) {
-                                        Object value = result.get(key);
-                                        if (value != null) {
-                                            captureResults.set(key, value);
-                                        }
-                                    }
                                     mCaptureResultHandler.onCaptureCompleted(timestamp,
-                                            captureResults);
+                                            initializeFilteredResults(result));
                                 }
                             } else {
                                 mExecutor.execute(
@@ -1657,8 +1689,13 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession {
             for (int i = idx; i >= 0; i--) {
                 if (previewMap.valueAt(i).first != null) {
                     previewMap.valueAt(i).first.close();
-                } else {
+                } else if (mClientNotificationsEnabled && (previewMap.valueAt(i).second != null) &&
-                    if (mClientNotificationsEnabled && ((i != idx) || notifyCurrentIndex)) {
+                        ((i != idx) || notifyCurrentIndex)) {
+                    TotalCaptureResult result = previewMap.valueAt(i).second;
+                    Long timestamp = result.get(CaptureResult.SENSOR_TIMESTAMP);
+                    mCaptureResultHandler.onCaptureCompleted(timestamp,
+                            initializeFilteredResults(result));
+
                     Log.w(TAG, "Preview frame drop with timestamp: " + previewMap.keyAt(i));
                     final long ident = Binder.clearCallingIdentity();
                     try {
@@ -1669,7 +1706,7 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession {
                     } finally {
                         Binder.restoreCallingIdentity(ident);
                     }
-                    }
+
                 }
                 previewMap.removeAt(i);
             }
@@ -1682,6 +1719,12 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession {
                 mOutputWriter = imageWriter;
             }
 
+            @Override
+            public void onImageDropped(long timestamp) {
+                discardPendingRepeatingResults(mPendingResultMap.indexOfKey(timestamp),
+                        mPendingResultMap, true);
+            }
+
             @Override
             public void onImageAvailable(ImageReader reader, Image img) {
                 if (img == null) {
@@ -1702,6 +1745,15 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession {
 
         private class ImageProcessCallback implements OnImageAvailableListener {
 
+            @Override
+            public void onImageDropped(long timestamp) {
+                discardPendingRepeatingResults(mPendingResultMap.indexOfKey(timestamp),
+                        mPendingResultMap, true);
+                // Add an empty frame&results entry to flag that we dropped a frame
+                // and valid capture results can immediately return to client.
+                mPendingResultMap.put(timestamp, new Pair<>(null, null));
+            }
+
             @Override
             public void onImageAvailable(ImageReader reader, Image img) {
                 if (mPendingResultMap.size() + 1 >= PREVIEW_QUEUE_SIZE) {
@@ -1768,6 +1820,17 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession {
         }
     }
 
+    private CameraMetadataNative initializeFilteredResults(TotalCaptureResult result) {
+        CameraMetadataNative captureResults = new CameraMetadataNative();
+        for (CaptureResult.Key key : mSupportedResultKeys) {
+            Object value = result.get(key);
+            if (value != null) {
+                captureResults.set(key, value);
+            }
+        }
+        return captureResults;
+    }
+
     private static Size findSmallestAspectMatchedSize(@NonNull List<Size> sizes,
             @NonNull Size arSize) {
         final float TOLL = .01f;