From 5b3d7a7c3c04096deb6e4de650c9c4e12ebff7a9 Mon Sep 17 00:00:00 2001 From: Emilian Peev Date: Thu, 23 Dec 2021 15:23:05 -0800 Subject: [PATCH] Camera: Add support for client requests&results when using extension sessions Allow camera clients to query for any orthogonal camera parameters that can be applied during extension capture sessions. Enable capture result callbacks in the same scenario. Bug: 198447410 Test: Camera CTS Change-Id: Ifb472cd051121d83d230bcc152c6942b8fb82a2d --- core/api/current.txt | 3 + .../camera2/CameraCharacteristics.java | 2 +- .../CameraExtensionCharacteristics.java | 144 +++++++++++++- .../camera2/CameraExtensionSession.java | 26 +++ .../extension/ICaptureProcessorImpl.aidl | 3 +- .../extension/IImageCaptureExtenderImpl.aidl | 2 + .../extension/IPreviewImageProcessorImpl.aidl | 4 +- .../camera2/extension/IProcessResultImpl.aidl | 24 +++ .../CameraAdvancedExtensionSessionImpl.java | 2 +- .../camera2/impl/CameraDeviceImpl.java | 4 +- .../impl/CameraExtensionForwardProcessor.java | 7 +- .../impl/CameraExtensionJpegProcessor.java | 7 +- .../impl/CameraExtensionSessionImpl.java | 177 ++++++++++++++---- .../CameraExtensionsProxyService.java | 128 +++++++++++-- 14 files changed, 474 insertions(+), 59 deletions(-) create mode 100644 core/java/android/hardware/camera2/extension/IProcessResultImpl.aidl diff --git a/core/api/current.txt b/core/api/current.txt index 40f589d6f1690..96f813625af0c 100644 --- a/core/api/current.txt +++ b/core/api/current.txt @@ -17434,6 +17434,8 @@ package android.hardware.camera2 { } public final class CameraExtensionCharacteristics { + method @NonNull public java.util.Set getAvailableCaptureRequestKeys(int); + method @NonNull public java.util.Set getAvailableCaptureResultKeys(int); method @Nullable public android.util.Range getEstimatedCaptureLatencyRangeMillis(int, @NonNull android.util.Size, int); method @NonNull public java.util.List getExtensionSupportedSizes(int, @NonNull Class); method @NonNull public java.util.List getExtensionSupportedSizes(int, int); @@ -17458,6 +17460,7 @@ package android.hardware.camera2 { ctor public CameraExtensionSession.ExtensionCaptureCallback(); method public void onCaptureFailed(@NonNull android.hardware.camera2.CameraExtensionSession, @NonNull android.hardware.camera2.CaptureRequest); method public void onCaptureProcessStarted(@NonNull android.hardware.camera2.CameraExtensionSession, @NonNull android.hardware.camera2.CaptureRequest); + method public void onCaptureResultAvailable(@NonNull android.hardware.camera2.CameraExtensionSession, @NonNull android.hardware.camera2.CaptureRequest, @NonNull android.hardware.camera2.TotalCaptureResult); method public void onCaptureSequenceAborted(@NonNull android.hardware.camera2.CameraExtensionSession, int); method public void onCaptureSequenceCompleted(@NonNull android.hardware.camera2.CameraExtensionSession, int); method public void onCaptureStarted(@NonNull android.hardware.camera2.CameraExtensionSession, @NonNull android.hardware.camera2.CaptureRequest, long); diff --git a/core/java/android/hardware/camera2/CameraCharacteristics.java b/core/java/android/hardware/camera2/CameraCharacteristics.java index d2dc314585d6c..6a809adc04330 100644 --- a/core/java/android/hardware/camera2/CameraCharacteristics.java +++ b/core/java/android/hardware/camera2/CameraCharacteristics.java @@ -657,7 +657,7 @@ public final class CameraCharacteristics extends CameraMetadata List + List getAvailableKeyList(Class metadataClass, Class keyClass, int[] filterTags, boolean includeSynthetic) { diff --git a/core/java/android/hardware/camera2/CameraExtensionCharacteristics.java b/core/java/android/hardware/camera2/CameraExtensionCharacteristics.java index 5c636c7fe2f09..aa98f1fdf9bcb 100644 --- a/core/java/android/hardware/camera2/CameraExtensionCharacteristics.java +++ b/core/java/android/hardware/camera2/CameraExtensionCharacteristics.java @@ -30,6 +30,7 @@ import android.hardware.camera2.extension.IInitializeSessionCallback; import android.hardware.camera2.extension.IPreviewExtenderImpl; import android.hardware.camera2.extension.LatencyRange; import android.hardware.camera2.extension.SizeList; +import android.hardware.camera2.impl.CameraMetadataNative; import android.hardware.camera2.params.ExtensionSessionConfiguration; import android.hardware.camera2.params.StreamConfigurationMap; import android.os.ConditionVariable; @@ -49,6 +50,7 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Objects; +import java.util.Set; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; @@ -785,8 +787,8 @@ public final class CameraExtensionCharacteristics { if (latencyRange != null) { return new Range(latencyRange.min, latencyRange.max); } - } - } catch (RemoteException e) { + } + } catch (RemoteException e) { Log.e(TAG, "Failed to query the extension capture latency! Extension service does" + " not respond!"); } finally { @@ -795,4 +797,142 @@ public final class CameraExtensionCharacteristics { return null; } + + /** + * Returns the set of keys supported by a {@link CaptureRequest} submitted in a + * {@link CameraExtensionSession} with a given extension type. + * + *

The set returned is not modifiable, so any attempts to modify it will throw + * a {@code UnsupportedOperationException}.

+ * + * @param extension the extension type + * + * @return non-modifiable set of capture keys supported by camera extension session initialized + * with the given extension type. + * @throws IllegalArgumentException in case of unsupported extension. + */ + @NonNull + public Set getAvailableCaptureRequestKeys(@Extension int extension) { + long clientId = registerClient(mContext); + if (clientId < 0) { + throw new IllegalArgumentException("Unsupported extensions"); + } + + HashSet ret = new HashSet<>(); + + try { + if (!isExtensionSupported(mCameraId, extension, mChars)) { + throw new IllegalArgumentException("Unsupported extension"); + } + Pair extenders = + initializeExtension(extension); + extenders.second.onInit(mCameraId, mChars.getNativeMetadata()); + extenders.second.init(mCameraId, mChars.getNativeMetadata()); + CameraMetadataNative captureRequestMeta = + extenders.second.getAvailableCaptureRequestKeys(); + + if (captureRequestMeta != null) { + int[] requestKeys = captureRequestMeta.get( + CameraCharacteristics.REQUEST_AVAILABLE_REQUEST_KEYS); + if (requestKeys == null) { + throw new AssertionError("android.request.availableRequestKeys must be non-null" + + " in the characteristics"); + } + CameraCharacteristics requestChars = new CameraCharacteristics(captureRequestMeta); + + Object crKey = CaptureRequest.Key.class; + Class> crKeyTyped = (Class>)crKey; + + ret.addAll(requestChars.getAvailableKeyList(CaptureRequest.class, crKeyTyped, + requestKeys, /*includeSynthetic*/ false)); + } + + // Jpeg quality and orientation must always be supported + if (!ret.contains(CaptureRequest.JPEG_QUALITY)) { + ret.add(CaptureRequest.JPEG_QUALITY); + } + if (!ret.contains(CaptureRequest.JPEG_ORIENTATION)) { + ret.add(CaptureRequest.JPEG_ORIENTATION); + } + extenders.second.onDeInit(); + } catch (RemoteException e) { + throw new IllegalStateException("Failed to query the available capture request keys!"); + } finally { + unregisterClient(clientId); + } + + return Collections.unmodifiableSet(ret); + } + + /** + * Returns the set of keys supported by a {@link CaptureResult} passed as an argument to + * {@link CameraExtensionSession.ExtensionCaptureCallback#onCaptureResultAvailable}. + * + *

The set returned is not modifiable, so any attempts to modify it will throw + * a {@code UnsupportedOperationException}.

+ * + *

In case the set is empty, then the extension is not able to support any capture results + * and the {@link CameraExtensionSession.ExtensionCaptureCallback#onCaptureResultAvailable} + * callback will not be fired.

+ * + * @param extension the extension type + * + * @return non-modifiable set of capture result keys supported by camera extension session + * initialized with the given extension type. + * @throws IllegalArgumentException in case of unsupported extension. + */ + @NonNull + public Set getAvailableCaptureResultKeys(@Extension int extension) { + long clientId = registerClient(mContext); + if (clientId < 0) { + throw new IllegalArgumentException("Unsupported extensions"); + } + + HashSet ret = new HashSet<>(); + try { + if (!isExtensionSupported(mCameraId, extension, mChars)) { + throw new IllegalArgumentException("Unsupported extension"); + } + + Pair extenders = + initializeExtension(extension); + extenders.second.onInit(mCameraId, mChars.getNativeMetadata()); + extenders.second.init(mCameraId, mChars.getNativeMetadata()); + CameraMetadataNative captureResultMeta = + extenders.second.getAvailableCaptureResultKeys(); + + if (captureResultMeta != null) { + int[] resultKeys = captureResultMeta.get( + CameraCharacteristics.REQUEST_AVAILABLE_RESULT_KEYS); + if (resultKeys == null) { + throw new AssertionError("android.request.availableResultKeys must be non-null " + + "in the characteristics"); + } + CameraCharacteristics resultChars = new CameraCharacteristics(captureResultMeta); + Object crKey = CaptureResult.Key.class; + Class> crKeyTyped = (Class>)crKey; + + ret.addAll(resultChars.getAvailableKeyList(CaptureResult.class, crKeyTyped, + resultKeys, /*includeSynthetic*/ false)); + + // Jpeg quality, orientation and sensor timestamp must always be supported + if (!ret.contains(CaptureResult.JPEG_QUALITY)) { + ret.add(CaptureResult.JPEG_QUALITY); + } + if (!ret.contains(CaptureResult.JPEG_ORIENTATION)) { + ret.add(CaptureResult.JPEG_ORIENTATION); + } + if (!ret.contains(CaptureResult.SENSOR_TIMESTAMP)) { + ret.add(CaptureResult.SENSOR_TIMESTAMP); + } + } + extenders.second.onDeInit(); + } catch (RemoteException e) { + throw new IllegalStateException("Failed to query the available capture result keys!"); + } finally { + unregisterClient(clientId); + } + + return Collections.unmodifiableSet(ret); + } } diff --git a/core/java/android/hardware/camera2/CameraExtensionSession.java b/core/java/android/hardware/camera2/CameraExtensionSession.java index 5892f682dd494..ee3441fc16f1f 100644 --- a/core/java/android/hardware/camera2/CameraExtensionSession.java +++ b/core/java/android/hardware/camera2/CameraExtensionSession.java @@ -172,6 +172,32 @@ public abstract class CameraExtensionSession implements AutoCloseable { int sequenceId) { // default empty implementation } + + /** + * This method is called when an image capture has fully completed and all the + * result metadata is available. + * + *

This callback will only be called in case + * {@link CameraExtensionCharacteristics#getAvailableCaptureResultKeys} returns a valid + * non-empty list.

+ * + *

The default implementation of this method does nothing.

+ * + * @param session The session received during + * {@link StateCallback#onConfigured(CameraExtensionSession)} + * @param request The request that was given to the CameraDevice + * @param result The total output metadata from the capture, which only includes the + * capture result keys advertised as supported in + * {@link CameraExtensionCharacteristics#getAvailableCaptureResultKeys}. + * + * @see #capture + * @see #setRepeatingRequest + * @see CameraExtensionCharacteristics#getAvailableCaptureResultKeys + */ + public void onCaptureResultAvailable(@NonNull CameraExtensionSession session, + @NonNull CaptureRequest request, @NonNull TotalCaptureResult result) { + // default empty implementation + } } /** diff --git a/core/java/android/hardware/camera2/extension/ICaptureProcessorImpl.aidl b/core/java/android/hardware/camera2/extension/ICaptureProcessorImpl.aidl index 022b084f613b3..3c5f5ff63a348 100644 --- a/core/java/android/hardware/camera2/extension/ICaptureProcessorImpl.aidl +++ b/core/java/android/hardware/camera2/extension/ICaptureProcessorImpl.aidl @@ -17,6 +17,7 @@ package android.hardware.camera2.extension; import android.view.Surface; import android.hardware.camera2.extension.CaptureBundle; +import android.hardware.camera2.extension.IProcessResultImpl; import android.hardware.camera2.extension.Size; /** @hide */ @@ -25,5 +26,5 @@ interface ICaptureProcessorImpl void onOutputSurface(in Surface surface, int imageFormat); void onResolutionUpdate(in Size size); void onImageFormatUpdate(int imageFormat); - void process(in List capturelist); + void process(in List capturelist, in IProcessResultImpl resultCallback); } diff --git a/core/java/android/hardware/camera2/extension/IImageCaptureExtenderImpl.aidl b/core/java/android/hardware/camera2/extension/IImageCaptureExtenderImpl.aidl index 3ebf63793b798..a8a7866e5ca4d 100644 --- a/core/java/android/hardware/camera2/extension/IImageCaptureExtenderImpl.aidl +++ b/core/java/android/hardware/camera2/extension/IImageCaptureExtenderImpl.aidl @@ -39,4 +39,6 @@ interface IImageCaptureExtenderImpl int getMaxCaptureStage(); @nullable List getSupportedResolutions(); LatencyRange getEstimatedCaptureLatencyRange(in Size outputSize); + CameraMetadataNative getAvailableCaptureRequestKeys(); + CameraMetadataNative getAvailableCaptureResultKeys(); } diff --git a/core/java/android/hardware/camera2/extension/IPreviewImageProcessorImpl.aidl b/core/java/android/hardware/camera2/extension/IPreviewImageProcessorImpl.aidl index f7e4023c50676..ecd098be9d3b7 100644 --- a/core/java/android/hardware/camera2/extension/IPreviewImageProcessorImpl.aidl +++ b/core/java/android/hardware/camera2/extension/IPreviewImageProcessorImpl.aidl @@ -17,6 +17,7 @@ package android.hardware.camera2.extension; import android.hardware.camera2.impl.CameraMetadataNative; import android.view.Surface; +import android.hardware.camera2.extension.IProcessResultImpl; import android.hardware.camera2.extension.ParcelImage; import android.hardware.camera2.extension.Size; @@ -26,5 +27,6 @@ interface IPreviewImageProcessorImpl void onOutputSurface(in Surface surface, int imageFormat); void onResolutionUpdate(in Size size); void onImageFormatUpdate(int imageFormat); - void process(in ParcelImage image, in CameraMetadataNative result, int sequenceId); + void process(in ParcelImage image, in CameraMetadataNative result, int sequenceId, + in IProcessResultImpl resultCallback); } diff --git a/core/java/android/hardware/camera2/extension/IProcessResultImpl.aidl b/core/java/android/hardware/camera2/extension/IProcessResultImpl.aidl new file mode 100644 index 0000000000000..4114edb37a8cd --- /dev/null +++ b/core/java/android/hardware/camera2/extension/IProcessResultImpl.aidl @@ -0,0 +1,24 @@ +/** + * Copyright (c) 2022, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package android.hardware.camera2.extension; + +import android.hardware.camera2.impl.CameraMetadataNative; + +/** @hide */ +interface IProcessResultImpl +{ + void onCaptureCompleted(long shutterTimestamp, in CameraMetadataNative results); +} diff --git a/core/java/android/hardware/camera2/impl/CameraAdvancedExtensionSessionImpl.java b/core/java/android/hardware/camera2/impl/CameraAdvancedExtensionSessionImpl.java index 9d2c901ed049f..0116117cc47a9 100644 --- a/core/java/android/hardware/camera2/impl/CameraAdvancedExtensionSessionImpl.java +++ b/core/java/android/hardware/camera2/impl/CameraAdvancedExtensionSessionImpl.java @@ -105,7 +105,7 @@ public final class CameraAdvancedExtensionSessionImpl extends CameraExtensionSes @RequiresPermission(android.Manifest.permission.CAMERA) public static CameraAdvancedExtensionSessionImpl createCameraAdvancedExtensionSession( @NonNull CameraDevice cameraDevice, @NonNull Context ctx, - @NonNull ExtensionSessionConfiguration config) + @NonNull ExtensionSessionConfiguration config, int sessionId) throws CameraAccessException, RemoteException { long clientId = CameraExtensionCharacteristics.registerClient(ctx); if (clientId < 0) { diff --git a/core/java/android/hardware/camera2/impl/CameraDeviceImpl.java b/core/java/android/hardware/camera2/impl/CameraDeviceImpl.java index 9b19fc4d3ef2a..3cb0c93d84090 100644 --- a/core/java/android/hardware/camera2/impl/CameraDeviceImpl.java +++ b/core/java/android/hardware/camera2/impl/CameraDeviceImpl.java @@ -2495,10 +2495,10 @@ public class CameraDeviceImpl extends CameraDevice if (CameraExtensionCharacteristics.areAdvancedExtensionsSupported()) { mCurrentAdvancedExtensionSession = CameraAdvancedExtensionSessionImpl.createCameraAdvancedExtensionSession( - this, mContext, extensionConfiguration); + this, mContext, extensionConfiguration, mNextSessionId++); } else { mCurrentExtensionSession = CameraExtensionSessionImpl.createCameraExtensionSession( - this, mContext, extensionConfiguration); + this, mContext, extensionConfiguration, mNextSessionId++); } } catch (RemoteException e) { throw new CameraAccessException(CameraAccessException.CAMERA_ERROR); diff --git a/core/java/android/hardware/camera2/impl/CameraExtensionForwardProcessor.java b/core/java/android/hardware/camera2/impl/CameraExtensionForwardProcessor.java index bf4593260a701..d148d87fc9d86 100644 --- a/core/java/android/hardware/camera2/impl/CameraExtensionForwardProcessor.java +++ b/core/java/android/hardware/camera2/impl/CameraExtensionForwardProcessor.java @@ -19,6 +19,7 @@ package android.hardware.camera2.impl; import android.annotation.SuppressLint; import android.hardware.camera2.CameraExtensionCharacteristics; import android.hardware.camera2.extension.IPreviewImageProcessorImpl; +import android.hardware.camera2.extension.IProcessResultImpl; import android.hardware.camera2.extension.ParcelImage; import android.hardware.camera2.TotalCaptureResult; import android.media.Image; @@ -114,12 +115,12 @@ public class CameraExtensionForwardProcessor { } } - public void process(ParcelImage image, TotalCaptureResult totalCaptureResult) - throws RemoteException { + public void process(ParcelImage image, TotalCaptureResult totalCaptureResult, + IProcessResultImpl resultCallback) throws RemoteException { if ((mIntermediateSurface != null) && (mIntermediateSurface.isValid()) && !mOutputAbandoned) { mProcessor.process(image, totalCaptureResult.getNativeMetadata(), - totalCaptureResult.getSequenceId()); + totalCaptureResult.getSequenceId(), resultCallback); } } diff --git a/core/java/android/hardware/camera2/impl/CameraExtensionJpegProcessor.java b/core/java/android/hardware/camera2/impl/CameraExtensionJpegProcessor.java index 425f22c313066..1514a2be5de8a 100644 --- a/core/java/android/hardware/camera2/impl/CameraExtensionJpegProcessor.java +++ b/core/java/android/hardware/camera2/impl/CameraExtensionJpegProcessor.java @@ -24,6 +24,7 @@ import android.graphics.ImageFormat; import android.hardware.camera2.CaptureResult; import android.hardware.camera2.extension.CaptureBundle; import android.hardware.camera2.extension.ICaptureProcessorImpl; +import android.hardware.camera2.extension.IProcessResultImpl; import android.media.Image; import android.media.Image.Plane; import android.media.ImageReader; @@ -183,11 +184,13 @@ public class CameraExtensionJpegProcessor implements ICaptureProcessorImpl { int cropLeft, int cropTop, int cropRight, int cropBottom, int rot90); - public void process(List captureBundle) throws RemoteException { + @Override + public void process(List captureBundle, IProcessResultImpl captureCallback) + throws RemoteException { JpegParameters jpegParams = getJpegParameters(captureBundle); try { mJpegParameters.add(jpegParams); - mProcessor.process(captureBundle); + mProcessor.process(captureBundle, captureCallback); } catch (Exception e) { mJpegParameters.remove(jpegParams); throw e; diff --git a/core/java/android/hardware/camera2/impl/CameraExtensionSessionImpl.java b/core/java/android/hardware/camera2/impl/CameraExtensionSessionImpl.java index c8ecfd0bdea9f..922caa3c20ea6 100644 --- a/core/java/android/hardware/camera2/impl/CameraExtensionSessionImpl.java +++ b/core/java/android/hardware/camera2/impl/CameraExtensionSessionImpl.java @@ -36,6 +36,7 @@ import android.hardware.camera2.extension.ICaptureProcessorImpl; import android.hardware.camera2.extension.IImageCaptureExtenderImpl; import android.hardware.camera2.extension.IInitializeSessionCallback; import android.hardware.camera2.extension.IPreviewExtenderImpl; +import android.hardware.camera2.extension.IProcessResultImpl; import android.hardware.camera2.extension.IRequestUpdateProcessorImpl; import android.hardware.camera2.extension.ParcelImage; import android.hardware.camera2.TotalCaptureResult; @@ -67,6 +68,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.Executor; public final class CameraExtensionSessionImpl extends CameraExtensionSession { @@ -83,6 +85,10 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession { private final StateCallback mCallbacks; private final List mSupportedPreviewSizes; private final InitializeSessionHandler mInitializeHandler; + private final int mSessionId; + private final Set mSupportedRequestKeys; + private final Set mSupportedResultKeys; + private boolean mCaptureResultsSupported; private CameraCaptureSession mCaptureSession = null; private Surface mCameraRepeatingSurface, mClientRepeatingRequestSurface; @@ -121,7 +127,8 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession { public static CameraExtensionSessionImpl createCameraExtensionSession( @NonNull CameraDevice cameraDevice, @NonNull Context ctx, - @NonNull ExtensionSessionConfiguration config) + @NonNull ExtensionSessionConfiguration config, + int sessionId) throws CameraAccessException, RemoteException { long clientId = CameraExtensionCharacteristics.registerClient(ctx); if (clientId < 0) { @@ -197,7 +204,10 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession { repeatingRequestSurface, burstCaptureSurface, config.getStateCallback(), - config.getExecutor()); + config.getExecutor(), + sessionId, + extensionChars.getAvailableCaptureRequestKeys(config.getExtension()), + extensionChars.getAvailableCaptureResultKeys(config.getExtension())); session.initialize(); @@ -212,7 +222,10 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession { @Nullable Surface repeatingRequestSurface, @Nullable Surface burstCaptureSurface, @NonNull StateCallback callback, - @NonNull Executor executor) { + @NonNull Executor executor, + int sessionId, + @NonNull Set requestKeys, + @Nullable Set resultKeys) { mExtensionClientId = extensionClientId; mImageExtender = imageExtender; mPreviewExtender = previewExtender; @@ -227,6 +240,10 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession { mHandler = new Handler(mHandlerThread.getLooper()); mInitialized = false; mInitializeHandler = new InitializeSessionHandler(); + mSessionId = sessionId; + mSupportedRequestKeys = requestKeys; + mSupportedResultKeys = resultKeys; + mCaptureResultsSupported = !resultKeys.isEmpty(); } private void initializeRepeatingRequestPipeline() throws RemoteException { @@ -483,7 +500,7 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession { return captureStageList; } - private static List createBurstRequest(CameraDevice cameraDevice, + private List createBurstRequest(CameraDevice cameraDevice, List captureStageList, CaptureRequest clientRequest, Surface target, int captureTemplate, Map captureMap) { CaptureRequest.Builder requestBuilder; @@ -495,16 +512,13 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession { return null; } - // This will override the extension capture stage jpeg parameters with the user set - // jpeg quality and rotation. This will guarantee that client configured jpeg - // parameters always have highest priority. - Integer jpegRotation = clientRequest.get(CaptureRequest.JPEG_ORIENTATION); - if (jpegRotation != null) { - captureStage.parameters.set(CaptureRequest.JPEG_ORIENTATION, jpegRotation); - } - Byte jpegQuality = clientRequest.get(CaptureRequest.JPEG_QUALITY); - if (jpegQuality != null) { - captureStage.parameters.set(CaptureRequest.JPEG_QUALITY, jpegQuality); + // This will guarantee that client configured + // parameters always have the highest priority. + for (CaptureRequest.Key requestKey : mSupportedRequestKeys){ + Object value = clientRequest.get(requestKey); + if (value != null) { + captureStage.parameters.set(requestKey, value); + } } requestBuilder.addTarget(target); @@ -517,10 +531,9 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession { return ret; } - private static CaptureRequest createRequest(CameraDevice cameraDevice, - List captureStageList, - Surface target, - int captureTemplate) throws CameraAccessException { + private CaptureRequest createRequest(CameraDevice cameraDevice, + List captureStageList, Surface target, int captureTemplate, + CaptureRequest clientRequest) throws CameraAccessException { CaptureRequest.Builder requestBuilder; requestBuilder = cameraDevice.createCaptureRequest(captureTemplate); if (target != null) { @@ -528,14 +541,35 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession { } CaptureRequest ret = requestBuilder.build(); + CameraMetadataNative nativeMeta = ret.getNativeMetadata(); for (CaptureStageImpl captureStage : captureStageList) { if (captureStage != null) { - CameraMetadataNative.update(ret.getNativeMetadata(), captureStage.parameters); + CameraMetadataNative.update(nativeMeta, captureStage.parameters); } } + + if (clientRequest != null) { + // This will guarantee that client configured + // parameters always have the highest priority. + for (CaptureRequest.Key requestKey : mSupportedRequestKeys) { + Object value = clientRequest.get(requestKey); + if (value != null) { + nativeMeta.set(requestKey, value); + } + } + } + return ret; } + private CaptureRequest createRequest(CameraDevice cameraDevice, + List captureStageList, + Surface target, + int captureTemplate) throws CameraAccessException { + return createRequest(cameraDevice, captureStageList, target, captureTemplate, + /*clientRequest*/ null); + } + @Override public int capture(@NonNull CaptureRequest request, @NonNull Executor executor, @@ -629,12 +663,17 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession { } private int setRepeatingRequest(CaptureStageImpl captureStage, - CameraCaptureSession.CaptureCallback requestHandler) + CameraCaptureSession.CaptureCallback requestHandler) throws CameraAccessException { + return setRepeatingRequest(captureStage, requestHandler, /*clientRequest*/ null); + } + + private int setRepeatingRequest(CaptureStageImpl captureStage, + CameraCaptureSession.CaptureCallback requestHandler, CaptureRequest clientRequest) throws CameraAccessException { ArrayList captureStageList = new ArrayList<>(); captureStageList.add(captureStage); - CaptureRequest repeatingRequest = createRequest(mCameraDevice, - captureStageList, mCameraRepeatingSurface, CameraDevice.TEMPLATE_PREVIEW); + CaptureRequest repeatingRequest = createRequest(mCameraDevice, captureStageList, + mCameraRepeatingSurface, CameraDevice.TEMPLATE_PREVIEW, clientRequest); return mCaptureSession.setSingleRepeatingRequest(repeatingRequest, new CameraExtensionUtils.HandlerExecutor(mHandler), requestHandler); } @@ -843,6 +882,7 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession { private ImageCallback mImageCallback = null; private boolean mCaptureFailed = false; + private CaptureResultHandler mCaptureResultHandler = null; public BurstRequestHandler(@NonNull CaptureRequest request, @NonNull Executor executor, @NonNull ExtensionCaptureCallback callbacks, @@ -963,20 +1003,18 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession { Long timestamp = result.get(CaptureResult.SENSOR_TIMESTAMP); if (timestamp != null) { + if (mCaptureResultsSupported && (mCaptureResultHandler == null)) { + mCaptureResultHandler = new CaptureResultHandler(mClientRequest, mExecutor, + mCallbacks, result.getSessionId()); + } if (mImageProcessor != null) { if (mCapturePendingMap.indexOfKey(timestamp) >= 0) { Image img = mCapturePendingMap.get(timestamp).first; - mCaptureStageMap.put(stageId, - new Pair<>(img, - result)); + mCaptureStageMap.put(stageId, new Pair<>(img, result)); checkAndFireBurstProcessing(); } else { - mCapturePendingMap.put(timestamp, - new Pair<>(null, - stageId)); - mCaptureStageMap.put(stageId, - new Pair<>(null, - result)); + mCapturePendingMap.put(timestamp, new Pair<>(null, stageId)); + mCaptureStageMap.put(stageId, new Pair<>(null, result)); } } else { mCaptureRequestMap.clear(); @@ -986,6 +1024,18 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession { () -> mCallbacks .onCaptureProcessStarted(CameraExtensionSessionImpl.this, mClientRequest)); + + if (mCaptureResultHandler != null) { + CameraMetadataNative captureResults = new CameraMetadataNative(); + for (CaptureResult.Key key : mSupportedResultKeys) { + Object value = result.get(key); + if (value != null) { + captureResults.set(key, value); + } + } + mCaptureResultHandler.onCaptureCompleted(timestamp, + captureResults); + } } finally { Binder.restoreCallingIdentity(ident); } @@ -1013,7 +1063,7 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession { List captureList = initializeParcelable(mCaptureStageMap, jpegOrientation, jpegQuality); try { - mImageProcessor.process(captureList); + mImageProcessor.process(captureList, mCaptureResultHandler); } catch (RemoteException e) { Log.e(TAG, "Failed to process multi-frame request! Extension service " + "does not respond!"); @@ -1228,6 +1278,43 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession { } } + private class CaptureResultHandler extends IProcessResultImpl.Stub { + private final Executor mExecutor; + private final ExtensionCaptureCallback mCallbacks; + private final CaptureRequest mClientRequest; + private final int mRequestId; + + public CaptureResultHandler(@NonNull CaptureRequest clientRequest, + @NonNull Executor executor, @NonNull ExtensionCaptureCallback listener, + int requestId) { + mClientRequest = clientRequest; + mExecutor = executor; + mCallbacks = listener; + mRequestId = requestId; + } + + @Override + public void onCaptureCompleted(long shutterTimestamp, CameraMetadataNative result) { + if (result == null) { + Log.e(TAG,"Invalid capture result!"); + return; + } + + result.set(CaptureResult.SENSOR_TIMESTAMP, shutterTimestamp); + TotalCaptureResult totalResult = new TotalCaptureResult(mCameraDevice.getId(), result, + mClientRequest, mRequestId, shutterTimestamp, new ArrayList(), + mSessionId, new PhysicalCaptureResultInfo[0]); + final long ident = Binder.clearCallingIdentity(); + try { + mExecutor.execute( + () -> mCallbacks.onCaptureResultAvailable(CameraExtensionSessionImpl.this, + mClientRequest, totalResult)); + } finally { + Binder.restoreCallingIdentity(ident); + } + } + } + // This handler can operate in two modes: // 1) Using valid client callbacks, which means camera buffers will be propagated the // registered output surfaces and clients will be notified accordingly. @@ -1242,6 +1329,7 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession { private OnImageAvailableListener mImageCallback = null; private LongSparseArray> mPendingResultMap = new LongSparseArray<>(); + private CaptureResultHandler mCaptureResultHandler = null; private boolean mRequestUpdatedNeeded = false; @@ -1375,6 +1463,11 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession { synchronized (mInterfaceLock) { final Long timestamp = result.get(CaptureResult.SENSOR_TIMESTAMP); if (timestamp != null) { + if (mCaptureResultsSupported && mClientNotificationsEnabled && + (mCaptureResultHandler == null)) { + mCaptureResultHandler = new CaptureResultHandler(mClientRequest, mExecutor, + mCallbacks, result.getSessionId()); + } if (mPreviewProcessorType == IPreviewExtenderImpl.PROCESSOR_TYPE_REQUEST_UPDATE_ONLY) { CaptureStageImpl captureStage = null; @@ -1387,7 +1480,7 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession { } if (captureStage != null) { try { - setRepeatingRequest(captureStage, this); + setRepeatingRequest(captureStage, this, request); mRequestUpdatedNeeded = true; } catch (IllegalStateException e) { // This is possible in case the camera device closes and the @@ -1406,7 +1499,8 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession { ParcelImage parcelImage = initializeParcelImage( mPendingResultMap.get(timestamp).first); try { - mPreviewImageProcessor.process(parcelImage, result); + mPreviewImageProcessor.process(parcelImage, result, + mCaptureResultHandler); } catch (RemoteException e) { processStatus = false; Log.e(TAG, "Extension service does not respond during " + @@ -1444,6 +1538,19 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession { .onCaptureProcessStarted( CameraExtensionSessionImpl.this, mClientRequest)); + if ((mCaptureResultHandler != null) && (mPreviewProcessorType != + IPreviewExtenderImpl.PROCESSOR_TYPE_IMAGE_PROCESSOR)) { + CameraMetadataNative captureResults = + new CameraMetadataNative(); + for (CaptureResult.Key key : mSupportedResultKeys) { + Object value = result.get(key); + if (value != null) { + captureResults.set(key, value); + } + } + mCaptureResultHandler.onCaptureCompleted(timestamp, + captureResults); + } } else { mExecutor.execute( () -> mCallbacks @@ -1587,7 +1694,7 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession { ParcelImage parcelImage = initializeParcelImage(img); try { mPreviewImageProcessor.process(parcelImage, - mPendingResultMap.get(timestamp).second); + mPendingResultMap.get(timestamp).second, mCaptureResultHandler); } catch (RemoteException e) { processStatus = false; Log.e(TAG, "Extension service does not respond during " + diff --git a/packages/services/CameraExtensionsProxy/src/com/android/cameraextensions/CameraExtensionsProxyService.java b/packages/services/CameraExtensionsProxy/src/com/android/cameraextensions/CameraExtensionsProxyService.java index 6938e25ea9afe..c9903ea198686 100644 --- a/packages/services/CameraExtensionsProxy/src/com/android/cameraextensions/CameraExtensionsProxyService.java +++ b/packages/services/CameraExtensionsProxy/src/com/android/cameraextensions/CameraExtensionsProxyService.java @@ -43,6 +43,7 @@ import android.hardware.camera2.extension.IImageProcessorImpl; import android.hardware.camera2.extension.IInitializeSessionCallback; import android.hardware.camera2.extension.IPreviewExtenderImpl; import android.hardware.camera2.extension.IPreviewImageProcessorImpl; +import android.hardware.camera2.extension.IProcessResultImpl; import android.hardware.camera2.extension.IRequestCallback; import android.hardware.camera2.extension.IRequestProcessorImpl; import android.hardware.camera2.extension.IRequestUpdateProcessorImpl; @@ -90,6 +91,7 @@ import androidx.camera.extensions.impl.NightPreviewExtenderImpl; import androidx.camera.extensions.impl.PreviewExtenderImpl; import androidx.camera.extensions.impl.PreviewExtenderImpl.ProcessorType; import androidx.camera.extensions.impl.PreviewImageProcessorImpl; +import androidx.camera.extensions.impl.ProcessResultImpl; import androidx.camera.extensions.impl.RequestUpdateProcessorImpl; import androidx.camera.extensions.impl.advanced.AdvancedExtenderImpl; import androidx.camera.extensions.impl.advanced.AutoAdvancedExtenderImpl; @@ -124,14 +126,17 @@ public class CameraExtensionsProxyService extends Service { private static final String LATEST_VERSION = "1.2.0"; private static final String NON_INIT_VERSION_PREFIX = "1.0"; private static final String ADVANCED_VERSION_PREFIX = "1.2"; - private static final String[] SUPPORTED_VERSION_PREFIXES = {ADVANCED_VERSION_PREFIX, - "1.1", NON_INIT_VERSION_PREFIX}; + private static final String RESULTS_VERSION_PREFIX = "1.3"; + private static final String[] SUPPORTED_VERSION_PREFIXES = {RESULTS_VERSION_PREFIX, + ADVANCED_VERSION_PREFIX, "1.1", NON_INIT_VERSION_PREFIX}; private static final boolean EXTENSIONS_PRESENT = checkForExtensions(); private static final String EXTENSIONS_VERSION = EXTENSIONS_PRESENT ? (new ExtensionVersionImpl()).checkApiVersion(LATEST_VERSION) : null; private static final boolean ADVANCED_API_SUPPORTED = checkForAdvancedAPI(); private static final boolean INIT_API_SUPPORTED = EXTENSIONS_PRESENT && (!EXTENSIONS_VERSION.startsWith(NON_INIT_VERSION_PREFIX)); + private static final boolean RESULT_API_SUPPORTED = EXTENSIONS_PRESENT && + (EXTENSIONS_VERSION.startsWith(RESULTS_VERSION_PREFIX)); private HashMap mCharacteristicsHashMap = new HashMap<>(); private HashMap mMetadataVendorIdMap = new HashMap<>(); @@ -1242,7 +1247,7 @@ public class CameraExtensionsProxyService extends Service { } if (processor != null) { - return new PreviewImageProcessorImplStub(processor); + return new PreviewImageProcessorImplStub(processor, mCameraId); } return null; @@ -1332,7 +1337,7 @@ public class CameraExtensionsProxyService extends Service { public ICaptureProcessorImpl getCaptureProcessor() { CaptureProcessorImpl captureProcessor = mImageExtender.getCaptureProcessor(); if (captureProcessor != null) { - return new CaptureProcessorImplStub(captureProcessor); + return new CaptureProcessorImplStub(captureProcessor, mCameraId); } return null; @@ -1390,13 +1395,97 @@ public class CameraExtensionsProxyService extends Service { return null; } + + @Override + public CameraMetadataNative getAvailableCaptureRequestKeys() { + if (RESULT_API_SUPPORTED) { + List supportedCaptureKeys = + mImageExtender.getAvailableCaptureRequestKeys(); + + if ((supportedCaptureKeys != null) && !supportedCaptureKeys.isEmpty()) { + CameraMetadataNative ret = new CameraMetadataNative(); + long vendorId = mMetadataVendorIdMap.containsKey(mCameraId) ? + mMetadataVendorIdMap.get(mCameraId) : Long.MAX_VALUE; + ret.setVendorId(vendorId); + int requestKeyTags [] = new int[supportedCaptureKeys.size()]; + int i = 0; + for (CaptureRequest.Key key : supportedCaptureKeys) { + requestKeyTags[i++] = CameraMetadataNative.getTag(key.getName(), vendorId); + } + ret.set(CameraCharacteristics.REQUEST_AVAILABLE_REQUEST_KEYS, requestKeyTags); + + return ret; + } + } + + return null; + } + + @Override + public CameraMetadataNative getAvailableCaptureResultKeys() { + if (RESULT_API_SUPPORTED) { + List supportedResultKeys = + mImageExtender.getAvailableCaptureResultKeys(); + + if ((supportedResultKeys != null) && !supportedResultKeys.isEmpty()) { + CameraMetadataNative ret = new CameraMetadataNative(); + long vendorId = mMetadataVendorIdMap.containsKey(mCameraId) ? + mMetadataVendorIdMap.get(mCameraId) : Long.MAX_VALUE; + ret.setVendorId(vendorId); + int resultKeyTags [] = new int[supportedResultKeys.size()]; + int i = 0; + for (CaptureResult.Key key : supportedResultKeys) { + resultKeyTags[i++] = CameraMetadataNative.getTag(key.getName(), vendorId); + } + ret.set(CameraCharacteristics.REQUEST_AVAILABLE_RESULT_KEYS, resultKeyTags); + + return ret; + } + } + + return null; + } + } + + private class ProcessResultCallback implements ProcessResultImpl { + private final IProcessResultImpl mProcessResult; + private final String mCameraId; + + private ProcessResultCallback(IProcessResultImpl processResult, String cameraId) { + mProcessResult = processResult; + mCameraId = cameraId; + } + + @Override + public void onCaptureCompleted(long shutterTimestamp, + List> result) { + if (result == null) { + Log.e(TAG, "Invalid capture result received!"); + } + + CameraMetadataNative captureResults = new CameraMetadataNative(); + if (mMetadataVendorIdMap.containsKey(mCameraId)) { + captureResults.setVendorId(mMetadataVendorIdMap.get(mCameraId)); + } + for (Pair pair : result) { + captureResults.set(pair.first, pair.second); + } + + try { + mProcessResult.onCaptureCompleted(shutterTimestamp, captureResults); + } catch (RemoteException e) { + Log.e(TAG, "Remote client doesn't respond to capture results!"); + } + } } private class CaptureProcessorImplStub extends ICaptureProcessorImpl.Stub { private final CaptureProcessorImpl mCaptureProcessor; + private final String mCameraId; - public CaptureProcessorImplStub(CaptureProcessorImpl captureProcessor) { + public CaptureProcessorImplStub(CaptureProcessorImpl captureProcessor, String cameraId) { mCaptureProcessor = captureProcessor; + mCameraId = cameraId; } @Override @@ -1415,7 +1504,7 @@ public class CameraExtensionsProxyService extends Service { } @Override - public void process(List captureList) { + public void process(List captureList, IProcessResultImpl resultCallback) { HashMap> captureMap = new HashMap<>(); for (CaptureBundle captureBundle : captureList) { captureMap.put(captureBundle.stage, new Pair<> ( @@ -1424,7 +1513,14 @@ public class CameraExtensionsProxyService extends Service { captureBundle.sequenceId))); } if (!captureMap.isEmpty()) { - mCaptureProcessor.process(captureMap); + if ((resultCallback != null) && (RESULT_API_SUPPORTED)) { + mCaptureProcessor.process(captureMap, new ProcessResultCallback(resultCallback, + mCameraId), null /*executor*/); + } else if (resultCallback == null) { + mCaptureProcessor.process(captureMap); + } else { + Log.e(TAG, "Process requests with capture results are not supported!"); + } } else { Log.e(TAG, "Process request with absent capture stages!"); } @@ -1433,9 +1529,11 @@ public class CameraExtensionsProxyService extends Service { private class PreviewImageProcessorImplStub extends IPreviewImageProcessorImpl.Stub { private final PreviewImageProcessorImpl mProcessor; + private final String mCameraId; - public PreviewImageProcessorImplStub(PreviewImageProcessorImpl processor) { + public PreviewImageProcessorImplStub(PreviewImageProcessorImpl processor, String cameraId) { mProcessor = processor; + mCameraId = cameraId; } @Override @@ -1455,9 +1553,17 @@ public class CameraExtensionsProxyService extends Service { @Override public void process(android.hardware.camera2.extension.ParcelImage image, - CameraMetadataNative result, int sequenceId) { - mProcessor.process(new ExtensionImage(image), - new TotalCaptureResult(result, sequenceId)); + CameraMetadataNative result, int sequenceId, IProcessResultImpl resultCallback) { + if ((resultCallback != null) && RESULT_API_SUPPORTED) { + mProcessor.process(new ExtensionImage(image), + new TotalCaptureResult(result, sequenceId), + new ProcessResultCallback(resultCallback, mCameraId), null /*executor*/); + } else if (resultCallback == null) { + mProcessor.process(new ExtensionImage(image), + new TotalCaptureResult(result, sequenceId)); + } else { + + } } }