From 9c778e19e12f10f511ffbde54506bfb8c68998a7 Mon Sep 17 00:00:00 2001 From: Emilian Peev Date: Thu, 29 Oct 2020 17:36:22 -0700 Subject: [PATCH] Camera: Initial support for Camera2 extension Add the necessary APIs that will allow Camera2 clients to access and work with existing CameraX extensions. Bug: 170481080 Test: Camera CTS Change-Id: Iaa22fea69ebe01599b3f2e66c1b46ca92015cdc5 --- core/api/current.txt | 45 + .../hardware/camera2/CameraDevice.java | 61 + .../CameraExtensionCharacteristics.java | 546 ++++++ .../camera2/CameraExtensionSession.java | 383 ++++ .../hardware/camera2/CameraManager.java | 26 +- .../hardware/camera2/CameraMetadata.java | 9 + .../camera2/extension/CaptureBundle.aidl | 28 + .../camera2/extension/CaptureStageImpl.aidl | 25 + .../ICameraExtensionsProxyService.aidl | 28 + .../extension/ICaptureProcessorImpl.aidl | 29 + .../extension/IImageCaptureExtenderImpl.aidl | 39 + .../extension/IPreviewExtenderImpl.aidl | 46 + .../extension/IPreviewImageProcessorImpl.aidl | 30 + .../IRequestUpdateProcessorImpl.aidl | 30 + .../camera2/extension/ParcelImage.aidl | 35 + .../hardware/camera2/extension/Size.aidl | 23 + .../hardware/camera2/extension/SizeList.aidl | 25 + .../camera2/impl/CameraDeviceImpl.java | 25 +- .../impl/CameraExtensionForwardProcessor.java | 150 ++ .../impl/CameraExtensionSessionImpl.java | 1649 +++++++++++++++++ .../camera2/impl/CameraMetadataNative.java | 16 + .../params/ExtensionSessionConfiguration.java | 97 + .../params/StreamConfigurationMap.java | 6 +- core/jni/android_hardware_HardwareBuffer.cpp | 11 +- ...ndroid_hardware_camera2_CameraMetadata.cpp | 29 + .../android_hardware_HardwareBuffer.h | 5 + media/java/android/media/Image.java | 20 +- media/java/android/media/ImageReader.java | 81 +- media/java/android/media/ImageWriter.java | 35 +- media/jni/android_media_ImageReader.cpp | 116 ++ media/jni/android_media_ImageWriter.cpp | 97 +- .../services/CameraExtensionsProxy/Android.bp | 29 + .../CameraExtensionsProxy/AndroidManifest.xml | 18 + .../res/values/strings.xml | 6 + .../camera/CameraExtensionsProxyService.java | 857 +++++++++ 35 files changed, 4610 insertions(+), 45 deletions(-) create mode 100644 core/java/android/hardware/camera2/CameraExtensionCharacteristics.java create mode 100644 core/java/android/hardware/camera2/CameraExtensionSession.java create mode 100644 core/java/android/hardware/camera2/extension/CaptureBundle.aidl create mode 100644 core/java/android/hardware/camera2/extension/CaptureStageImpl.aidl create mode 100644 core/java/android/hardware/camera2/extension/ICameraExtensionsProxyService.aidl create mode 100644 core/java/android/hardware/camera2/extension/ICaptureProcessorImpl.aidl create mode 100644 core/java/android/hardware/camera2/extension/IImageCaptureExtenderImpl.aidl create mode 100644 core/java/android/hardware/camera2/extension/IPreviewExtenderImpl.aidl create mode 100644 core/java/android/hardware/camera2/extension/IPreviewImageProcessorImpl.aidl create mode 100644 core/java/android/hardware/camera2/extension/IRequestUpdateProcessorImpl.aidl create mode 100644 core/java/android/hardware/camera2/extension/ParcelImage.aidl create mode 100644 core/java/android/hardware/camera2/extension/Size.aidl create mode 100644 core/java/android/hardware/camera2/extension/SizeList.aidl create mode 100644 core/java/android/hardware/camera2/impl/CameraExtensionForwardProcessor.java create mode 100644 core/java/android/hardware/camera2/impl/CameraExtensionSessionImpl.java create mode 100644 core/java/android/hardware/camera2/params/ExtensionSessionConfiguration.java create mode 100644 packages/services/CameraExtensionsProxy/Android.bp create mode 100644 packages/services/CameraExtensionsProxy/AndroidManifest.xml create mode 100644 packages/services/CameraExtensionsProxy/res/values/strings.xml create mode 100644 packages/services/CameraExtensionsProxy/src/com/android/camera/CameraExtensionsProxyService.java diff --git a/core/api/current.txt b/core/api/current.txt index 892ddf4a08f84..55521d3a77927 100644 --- a/core/api/current.txt +++ b/core/api/current.txt @@ -17645,6 +17645,7 @@ package android.hardware.camera2 { method public void createCaptureSession(android.hardware.camera2.params.SessionConfiguration) throws android.hardware.camera2.CameraAccessException; method @Deprecated public abstract void createCaptureSessionByOutputConfigurations(java.util.List, android.hardware.camera2.CameraCaptureSession.StateCallback, @Nullable android.os.Handler) throws android.hardware.camera2.CameraAccessException; method @Deprecated public abstract void createConstrainedHighSpeedCaptureSession(@NonNull java.util.List, @NonNull android.hardware.camera2.CameraCaptureSession.StateCallback, @Nullable android.os.Handler) throws android.hardware.camera2.CameraAccessException; + method public void createExtensionSession(@NonNull android.hardware.camera2.params.ExtensionSessionConfiguration) throws android.hardware.camera2.CameraAccessException; method @NonNull public abstract android.hardware.camera2.CaptureRequest.Builder createReprocessCaptureRequest(@NonNull android.hardware.camera2.TotalCaptureResult) throws android.hardware.camera2.CameraAccessException; method @Deprecated public abstract void createReprocessableCaptureSession(@NonNull android.hardware.camera2.params.InputConfiguration, @NonNull java.util.List, @NonNull android.hardware.camera2.CameraCaptureSession.StateCallback, @Nullable android.os.Handler) throws android.hardware.camera2.CameraAccessException; method @Deprecated public abstract void createReprocessableCaptureSessionByConfigurations(@NonNull android.hardware.camera2.params.InputConfiguration, @NonNull java.util.List, @NonNull android.hardware.camera2.CameraCaptureSession.StateCallback, @Nullable android.os.Handler) throws android.hardware.camera2.CameraAccessException; @@ -17676,8 +17677,44 @@ package android.hardware.camera2 { field public static final int ERROR_MAX_CAMERAS_IN_USE = 2; // 0x2 } + public final class CameraExtensionCharacteristics { + method @NonNull public java.util.List getExtensionSupportedSizes(int, @NonNull Class); + method @NonNull public java.util.List getExtensionSupportedSizes(int, int); + method @NonNull public java.util.List getSupportedExtensions(); + field public static final int EXTENSION_AUTOMATIC = 0; // 0x0 + field public static final int EXTENSION_BEAUTY = 1; // 0x1 + field public static final int EXTENSION_BOKEH = 2; // 0x2 + field public static final int EXTENSION_HDR = 3; // 0x3 + field public static final int EXTENSION_NIGHT = 4; // 0x4 + } + + public abstract class CameraExtensionSession implements java.lang.AutoCloseable { + method public int capture(@NonNull android.hardware.camera2.CaptureRequest, @NonNull java.util.concurrent.Executor, @NonNull android.hardware.camera2.CameraExtensionSession.ExtensionCaptureCallback) throws android.hardware.camera2.CameraAccessException; + method public void close() throws android.hardware.camera2.CameraAccessException; + method @NonNull public android.hardware.camera2.CameraDevice getDevice(); + method public int setRepeatingRequest(@NonNull android.hardware.camera2.CaptureRequest, @NonNull java.util.concurrent.Executor, @NonNull android.hardware.camera2.CameraExtensionSession.ExtensionCaptureCallback) throws android.hardware.camera2.CameraAccessException; + method public void stopRepeating() throws android.hardware.camera2.CameraAccessException; + } + + public abstract static class CameraExtensionSession.ExtensionCaptureCallback { + ctor public CameraExtensionSession.ExtensionCaptureCallback(); + method public void onCaptureFailed(@NonNull android.hardware.camera2.CameraExtensionSession, @NonNull android.hardware.camera2.CaptureRequest); + method public void onCaptureProcessStarted(@NonNull android.hardware.camera2.CameraExtensionSession, @NonNull android.hardware.camera2.CaptureRequest); + method public void onCaptureSequenceAborted(@NonNull android.hardware.camera2.CameraExtensionSession, int); + method public void onCaptureSequenceCompleted(@NonNull android.hardware.camera2.CameraExtensionSession, int); + method public void onCaptureStarted(@NonNull android.hardware.camera2.CameraExtensionSession, @NonNull android.hardware.camera2.CaptureRequest, long); + } + + public abstract static class CameraExtensionSession.StateCallback { + ctor public CameraExtensionSession.StateCallback(); + method public void onClosed(@NonNull android.hardware.camera2.CameraExtensionSession); + method public abstract void onConfigureFailed(@NonNull android.hardware.camera2.CameraExtensionSession); + method public abstract void onConfigured(@NonNull android.hardware.camera2.CameraExtensionSession); + } + public final class CameraManager { method @NonNull public android.hardware.camera2.CameraCharacteristics getCameraCharacteristics(@NonNull String) throws android.hardware.camera2.CameraAccessException; + method @NonNull public android.hardware.camera2.CameraExtensionCharacteristics getCameraExtensionCharacteristics(@NonNull String) throws android.hardware.camera2.CameraAccessException; method @NonNull public String[] getCameraIdList() throws android.hardware.camera2.CameraAccessException; method @NonNull public java.util.Set> getConcurrentCameraIds() throws android.hardware.camera2.CameraAccessException; method @RequiresPermission(android.Manifest.permission.CAMERA) public boolean isConcurrentSessionConfigurationSupported(@NonNull java.util.Map) throws android.hardware.camera2.CameraAccessException; @@ -18187,6 +18224,14 @@ package android.hardware.camera2.params { method public android.util.Rational getElement(int, int); } + public final class ExtensionSessionConfiguration { + ctor public ExtensionSessionConfiguration(int, @NonNull java.util.List, @NonNull java.util.concurrent.Executor, @NonNull android.hardware.camera2.CameraExtensionSession.StateCallback); + method @NonNull public java.util.concurrent.Executor getExecutor(); + method public int getExtension(); + method @NonNull public java.util.List getOutputConfigurations(); + method @NonNull public android.hardware.camera2.CameraExtensionSession.StateCallback getStateCallback(); + } + public final class Face { method public android.graphics.Rect getBounds(); method public int getId(); diff --git a/core/java/android/hardware/camera2/CameraDevice.java b/core/java/android/hardware/camera2/CameraDevice.java index 37e12809467d7..f9eecaec5bb2a 100644 --- a/core/java/android/hardware/camera2/CameraDevice.java +++ b/core/java/android/hardware/camera2/CameraDevice.java @@ -20,6 +20,7 @@ import android.annotation.IntDef; import android.annotation.NonNull; import android.annotation.Nullable; import android.annotation.SystemApi; +import android.hardware.camera2.params.ExtensionSessionConfiguration; import android.hardware.camera2.params.InputConfiguration; import android.hardware.camera2.params.OutputConfiguration; import android.hardware.camera2.params.SessionConfiguration; @@ -350,6 +351,66 @@ public abstract class CameraDevice implements AutoCloseable { @Nullable Handler handler) throws CameraAccessException; + /** + * Initialize a specific device-specific extension augmented camera capture + * session. + * + *

Extension sessions can be used to enable device-specific operation modes like + * {@link CameraExtensionCharacteristics#EXTENSION_NIGHT} or + * {@link CameraExtensionCharacteristics#EXTENSION_HDR}. These modes are less flexible than the + * full camera API, but enable access to more sophisticated processing algorithms that can + * capture multi-frame bursts to generate single output images. To query for available + * extensions on this device call + * {@link CameraExtensionCharacteristics#getSupportedExtensions()}.

+ * + *

This method will also trigger the setup of the internal + * processing pipeline for extension augmented preview and multi-frame + * still capture.

+ * + *

If a prior CameraCaptureSession already exists when this method is called, the previous + * session will no longer be able to accept new capture requests and will be closed. Any + * in-progress capture requests made on the prior session will be completed before it's closed. + *

+ * + *

The CameraExtensionSession will be active until the client + * either calls CameraExtensionSession.close() or creates a new camera + * capture session. In both cases all internal resources will be + * released, continuous repeating requests stopped and any pending + * multi-frame capture requests flushed.

+ * + *

Note that the CameraExtensionSession currently supports at most two + * multi frame capture surface formats: ImageFormat.YUV_420_888 and + * ImageFormat.JPEG. Clients must query the multi-frame capture format support using + * {@link CameraExtensionCharacteristics#getExtensionSupportedSizes(int, int)}. + * For repeating requests CameraExtensionSession supports only + * {@link android.graphics.SurfaceTexture} as output. Clients can query the supported resolution + * for the repeating request output using + * {@link CameraExtensionCharacteristics#getExtensionSupportedSizes(int, Class) + * getExtensionSupportedSizes(..., Class)}.

+ * + *

At the very minimum the initialization expects either one valid output + * surface for repeating or one valid output for high-quality single requests registered in the + * outputs argument of the extension configuration argument. At the maximum the initialization + * will accept two valid output surfaces, one for repeating and the other for single requests. + * Additional unsupported surfaces passed to ExtensionSessionConfiguration will cause an + * {@link IllegalArgumentException} to be thrown.

+ * + * @param extensionConfiguration extension configuration + * @throws IllegalArgumentException If both the preview and still + * capture surfaces are not set or invalid, or if any of the + * registered surfaces do not meet the device-specific + * extension requirements such as dimensions and/or + * (output format)/(surface type), or if the extension is not + * supported. + * @see CameraExtensionCharacteristics#getSupportedExtensions + * @see CameraExtensionCharacteristics#getExtensionSupportedSizes + */ + public void createExtensionSession( + @NonNull ExtensionSessionConfiguration extensionConfiguration) + throws CameraAccessException { + throw new UnsupportedOperationException("No default implementation"); + } + /** * Standard camera operation mode. * diff --git a/core/java/android/hardware/camera2/CameraExtensionCharacteristics.java b/core/java/android/hardware/camera2/CameraExtensionCharacteristics.java new file mode 100644 index 0000000000000..d3eb3779189bf --- /dev/null +++ b/core/java/android/hardware/camera2/CameraExtensionCharacteristics.java @@ -0,0 +1,546 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package android.hardware.camera2; + +import android.content.ComponentName; +import android.content.Context; +import android.content.Intent; +import android.content.ServiceConnection; +import android.graphics.ImageFormat; +import android.hardware.camera2.extension.ICameraExtensionsProxyService; +import android.hardware.camera2.extension.IImageCaptureExtenderImpl; +import android.hardware.camera2.extension.IPreviewExtenderImpl; +import android.hardware.camera2.extension.SizeList; +import android.hardware.camera2.params.ExtensionSessionConfiguration; +import android.hardware.camera2.params.StreamConfigurationMap; +import android.os.ConditionVariable; +import android.os.IBinder; +import android.os.RemoteException; +import android.annotation.IntDef; +import android.annotation.NonNull; +import android.util.Log; +import android.util.Pair; +import android.util.Size; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.List; +import java.util.Objects; + +/** + *

Allows clients to query availability and supported resolutions of camera extensions.

+ * + *

Camera extensions give camera clients access to device-specific algorithms and sequences that + * can improve the overall image quality of snapshots in various cases such as low light, selfies, + * portraits, and scenes that can benefit from enhanced dynamic range. Often such sophisticated + * processing sequences will rely on multiple camera frames as input and will produce a single + * output.

+ * + *

Camera extensions are not guaranteed to be present on all devices so camera clients must + * query for their availability via {@link CameraExtensionCharacteristics#getSupportedExtensions()}. + *

+ * + *

In order to use any available camera extension, camera clients must create a corresponding + * {@link CameraExtensionSession} via + * {@link CameraDevice#createExtensionSession(ExtensionSessionConfiguration)}

+ * + *

Camera clients must be aware that device-specific camera extensions may support only a + * subset of the available camera resolutions and must first query + * {@link CameraExtensionCharacteristics#getExtensionSupportedSizes(int, int)} for supported + * single high-quality request output sizes and + * {@link CameraExtensionCharacteristics#getExtensionSupportedSizes(int, Class)} for supported + * repeating request output sizes.

+ * + * @see CameraManager#getCameraExtensionCharacteristics(String) + */ +public final class CameraExtensionCharacteristics { + private static final String TAG = "CameraExtensionCharacteristics"; + + /** + * Device-specific extension implementation for automatic selection of particular extension + * such as HDR or NIGHT depending on the current lighting and environment conditions. + */ + public static final int EXTENSION_AUTOMATIC = 0; + + /** + * Device-specific extension implementation which tends to smooth the skin and apply other + * cosmetic effects to people's faces. + */ + public static final int EXTENSION_BEAUTY = 1; + + /** + * Device-specific extension implementation which can blur certain regions of the final image + * thereby "enhancing" focus for all remaining non-blurred parts. + */ + public static final int EXTENSION_BOKEH = 2; + + /** + * Device-specific extension implementation for enhancing the dynamic range of the + * final image. + */ + public static final int EXTENSION_HDR = 3; + + /** + * Device-specific extension implementation that aims to suppress noise and improve the + * overall image quality under low light conditions. + */ + public static final int EXTENSION_NIGHT = 4; + + /** + * @hide + */ + @Retention(RetentionPolicy.SOURCE) + @IntDef(flag = true, value = {EXTENSION_AUTOMATIC, + EXTENSION_BEAUTY, + EXTENSION_BOKEH, + EXTENSION_HDR, + EXTENSION_NIGHT}) + public @interface Extension { + } + + /** + * Default camera output in case additional processing from CameraX extensions is not needed + * + * @hide + */ + public static final int NON_PROCESSING_INPUT_FORMAT = ImageFormat.PRIVATE; + + /** + * CameraX extensions require YUV_420_888 as default input for processing at the moment + * + * @hide + */ + public static final int PROCESSING_INPUT_FORMAT = ImageFormat.YUV_420_888; + + private static final @Extension + int[] EXTENSION_LIST = new int[]{ + EXTENSION_AUTOMATIC, + EXTENSION_BEAUTY, + EXTENSION_BOKEH, + EXTENSION_HDR, + EXTENSION_NIGHT}; + + private final Context mContext; + private final String mCameraId; + private final CameraCharacteristics mChars; + + /** + * @hide + */ + public CameraExtensionCharacteristics(Context context, String cameraId, + CameraCharacteristics chars) { + mContext = context; + mCameraId = cameraId; + mChars = chars; + } + + private static List generateSupportedSizes(List sizesList, + Integer format, + StreamConfigurationMap streamMap) { + // Per API contract it is assumed that the extension is able to support all + // camera advertised sizes for a given format in case it doesn't return + // a valid non-empty size list. + ArrayList ret = new ArrayList<>(); + if ((sizesList != null) && (!sizesList.isEmpty())) { + for (SizeList entry : sizesList) { + if ((entry.format == format) && !entry.sizes.isEmpty()) { + for (android.hardware.camera2.extension.Size sz : entry.sizes) { + ret.add(new Size(sz.width, sz.height)); + } + return ret; + } + } + } + Size[] supportedSizes = streamMap.getOutputSizes(format); + if (supportedSizes != null) { + ret.addAll(Arrays.asList(supportedSizes)); + } + return ret; + } + + /** + * A per-process global camera extension manager instance, to track and + * initialize/release extensions depending on client activity. + */ + private static final class CameraExtensionManagerGlobal { + private static final String TAG = "CameraExtensionManagerGlobal"; + private static final String PROXY_PACKAGE_NAME = "com.android.camera"; + private static final String PROXY_SERVICE_NAME = + "com.android.camera.CameraExtensionsProxyService"; + + // Singleton instance + private static final CameraExtensionManagerGlobal GLOBAL_CAMERA_MANAGER = + new CameraExtensionManagerGlobal(); + private final Object mLock = new Object(); + private final int PROXY_SERVICE_DELAY_MS = 1000; + private InitializerFuture mInitFuture = null; + private ServiceConnection mConnection = null; + private ICameraExtensionsProxyService mProxy = null; + + // Singleton, don't allow construction + private CameraExtensionManagerGlobal() {} + + public static CameraExtensionManagerGlobal get() { + return GLOBAL_CAMERA_MANAGER; + } + + private void connectToProxyLocked(Context ctx) { + if (mConnection == null) { + Intent intent = new Intent(); + intent.setClassName(PROXY_PACKAGE_NAME, PROXY_SERVICE_NAME); + mInitFuture = new InitializerFuture(); + mConnection = new ServiceConnection() { + @Override + public void onServiceDisconnected(ComponentName component) { + mInitFuture.setStatus(false); + mConnection = null; + mProxy = null; + } + + @Override + public void onServiceConnected(ComponentName component, IBinder binder) { + mProxy = ICameraExtensionsProxyService.Stub.asInterface(binder); + mInitFuture.setStatus(true); + } + }; + ctx.bindService(intent, mConnection, Context.BIND_AUTO_CREATE | + Context.BIND_IMPORTANT | Context.BIND_ABOVE_CLIENT | + Context.BIND_NOT_VISIBLE); + + try { + mInitFuture.get(PROXY_SERVICE_DELAY_MS, TimeUnit.MILLISECONDS); + } catch (TimeoutException e) { + Log.e(TAG, "Timed out while initializing proxy service!"); + } + } + } + + private static class InitializerFuture implements Future { + private volatile Boolean mStatus; + ConditionVariable mCondVar = new ConditionVariable(/*opened*/false); + + public void setStatus(boolean status) { + mStatus = status; + mCondVar.open(); + } + + @Override + public boolean cancel(boolean mayInterruptIfRunning) { + return false; // don't allow canceling this task + } + + @Override + public boolean isCancelled() { + return false; // can never cancel this task + } + + @Override + public boolean isDone() { + return mStatus != null; + } + + @Override + public Boolean get() { + mCondVar.block(); + return mStatus; + } + + @Override + public Boolean get(long timeout, TimeUnit unit) throws TimeoutException { + long timeoutMs = unit.convert(timeout, TimeUnit.MILLISECONDS); + if (!mCondVar.block(timeoutMs)) { + throw new TimeoutException( + "Failed to receive status after " + timeout + " " + unit); + } + + if (mStatus == null) { + throw new AssertionError(); + } + return mStatus; + } + } + + public long registerClient(Context ctx) { + synchronized (mLock) { + connectToProxyLocked(ctx); + if (mProxy != null) { + try { + return mProxy.registerClient(); + } catch (RemoteException e) { + Log.e(TAG, "Failed to initialize extension! Extension service does " + + " not respond!"); + return -1; + } + } else { + return -1; + } + } + } + + public void unregisterClient(long clientId) { + synchronized (mLock) { + if (mProxy != null) { + try { + mProxy.unregisterClient(clientId); + } catch (RemoteException e) { + Log.e(TAG, "Failed to de-initialize extension! Extension service does" + + " not respond!"); + } + } + } + } + + public IPreviewExtenderImpl initializePreviewExtension(int extensionType) + throws RemoteException { + synchronized (mLock) { + if (mProxy != null) { + return mProxy.initializePreviewExtension(extensionType); + } else { + return null; + } + } + } + + public IImageCaptureExtenderImpl initializeImageExtension(int extensionType) + throws RemoteException { + synchronized (mLock) { + if (mProxy != null) { + return mProxy.initializeImageExtension(extensionType); + } else { + return null; + } + } + } + } + + /** + * @hide + */ + public static long registerClient(Context ctx) { + return CameraExtensionManagerGlobal.get().registerClient(ctx); + } + + /** + * @hide + */ + public static void unregisterClient(long clientId) { + CameraExtensionManagerGlobal.get().unregisterClient(clientId); + } + + /** + * @hide + */ + public static boolean isExtensionSupported(String cameraId, int extensionType, + CameraCharacteristics chars) { + Pair extenders; + try { + extenders = initializeExtension(extensionType); + } catch (IllegalArgumentException e) { + return false; + } + + try { + return extenders.first.isExtensionAvailable(cameraId, chars.getNativeMetadata()) && + extenders.second.isExtensionAvailable(cameraId, chars.getNativeMetadata()); + } catch (RemoteException e) { + Log.e(TAG, "Failed to query extension availability! Extension service does not" + + " respond!"); + return false; + } + } + + /** + * @hide + */ + public static Pair initializeExtension( + @Extension int extensionType) { + IPreviewExtenderImpl previewExtender; + IImageCaptureExtenderImpl imageExtender; + try { + previewExtender = + CameraExtensionManagerGlobal.get().initializePreviewExtension(extensionType); + imageExtender = + CameraExtensionManagerGlobal.get().initializeImageExtension(extensionType); + } catch (RemoteException e) { + throw new IllegalStateException("Failed to initialize extension: " + extensionType); + } + if ((imageExtender == null) || (previewExtender == null)) { + throw new IllegalArgumentException("Unknown extension: " + extensionType); + } + + return new Pair<>(previewExtender, imageExtender); + } + + private static boolean isOutputSupportedFor(Class klass) { + Objects.requireNonNull(klass, "klass must not be null"); + + if (klass == android.graphics.SurfaceTexture.class) { + return true; + } + + return false; + } + + /** + * Return a list of supported device-specific extensions for a given camera device. + * + * @return non-modifiable list of available extensions + */ + public @NonNull List getSupportedExtensions() { + ArrayList ret = new ArrayList<>(); + long clientId = registerClient(mContext); + if (clientId < 0) { + return Collections.unmodifiableList(ret); + } + + try { + for (int extensionType : EXTENSION_LIST) { + if (isExtensionSupported(mCameraId, extensionType, mChars)) { + ret.add(extensionType); + } + } + } finally { + unregisterClient(clientId); + } + + return Collections.unmodifiableList(ret); + } + + /** + * Get a list of sizes compatible with {@code klass} to use as an output for the + * repeating request + * {@link CameraExtensionSession#setRepeatingRequest}. + * + *

Note that device-specific extensions are allowed to support only a subset + * of the camera output surfaces and resolutions. + * The {@link android.graphics.SurfaceTexture} class is guaranteed at least one size for + * backward compatible cameras whereas other output classes are not guaranteed to be supported. + *

+ * + * @param extension the extension type + * @param klass a non-{@code null} {@link Class} object reference + * @return non-modifiable list of available sizes or an empty list if the Surface output is not + * supported + * @throws NullPointerException if {@code klass} was {@code null} + * @throws IllegalArgumentException in case of unsupported extension. + */ + @NonNull + public List getExtensionSupportedSizes(@Extension int extension, + @NonNull Class klass) { + if (!isOutputSupportedFor(klass)) { + return new ArrayList<>(); + } + // TODO: Revisit this code once the Extension preview processor output format + // ambiguity is resolved in b/169799538. + + long clientId = registerClient(mContext); + if (clientId < 0) { + throw new IllegalArgumentException("Unsupported extensions"); + } + + try { + if (!isExtensionSupported(mCameraId, extension, mChars)) { + throw new IllegalArgumentException("Unsupported extension"); + } + + Pair extenders = + initializeExtension(extension); + StreamConfigurationMap streamMap = mChars.get( + CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP); + extenders.first.init(mCameraId, mChars.getNativeMetadata()); + return generateSupportedSizes(extenders.first.getSupportedResolutions(), + ImageFormat.PRIVATE, streamMap); + } catch (RemoteException e) { + Log.e(TAG, "Failed to query the extension supported sizes! Extension service does" + + " not respond!"); + return new ArrayList<>(); + } finally { + unregisterClient(clientId); + } + } + + /** + * Check whether a given extension is available and return the + * supported output surface resolutions that can be used for high-quality capture + * requests via {@link CameraExtensionSession#capture}. + * + *

Note that device-specific extensions are allowed to support only a subset + * of the camera resolutions advertised by + * {@link StreamConfigurationMap#getOutputSizes}.

+ * + *

Device-specific extensions currently support at most two + * multi-frame capture surface formats, ImageFormat.YUV_420_888 or + * ImageFormat.JPEG.

+ * + * @param extension the extension type + * @param format device-specific extension output format + * @return non-modifiable list of available sizes or an empty list if the format is not + * supported. + * @throws IllegalArgumentException in case of format different from ImageFormat.JPEG / + * ImageFormat.YUV_420_888; or unsupported extension. + */ + public @NonNull + List getExtensionSupportedSizes(@Extension int extension, int format) { + try { + long clientId = registerClient(mContext); + if (clientId < 0) { + throw new IllegalArgumentException("Unsupported extensions"); + } + + try { + if (!isExtensionSupported(mCameraId, extension, mChars)) { + throw new IllegalArgumentException("Unsupported extension"); + } + + Pair extenders = + initializeExtension(extension); + StreamConfigurationMap streamMap = mChars.get( + CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP); + if (format == ImageFormat.YUV_420_888) { + extenders.second.init(mCameraId, mChars.getNativeMetadata()); + if (extenders.second.getCaptureProcessor() == null) { + // Extensions that don't implement any capture processor are limited to + // JPEG only! + return new ArrayList<>(); + } + return generateSupportedSizes(extenders.second.getSupportedResolutions(), + format, streamMap); + } else if (format == ImageFormat.JPEG) { + extenders.second.init(mCameraId, mChars.getNativeMetadata()); + if (extenders.second.getCaptureProcessor() == null) { + return generateSupportedSizes(null, format, streamMap); + } + + return new ArrayList<>(); + } + + throw new IllegalArgumentException("Unsupported format: " + format); + } finally { + unregisterClient(clientId); + } + } catch (RemoteException e) { + Log.e(TAG, "Failed to query the extension supported sizes! Extension service does" + + " not respond!"); + return new ArrayList<>(); + } + } +} diff --git a/core/java/android/hardware/camera2/CameraExtensionSession.java b/core/java/android/hardware/camera2/CameraExtensionSession.java new file mode 100644 index 0000000000000..877dfbc49df2b --- /dev/null +++ b/core/java/android/hardware/camera2/CameraExtensionSession.java @@ -0,0 +1,383 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.hardware.camera2; + +import android.annotation.NonNull; + +import java.util.concurrent.Executor; + +/** + * A camera capture session that enables access to device-specific camera extensions, which + * often use multi-frame bursts and sophisticated post-process algorithms for image capture. + * + *

The capture session will be returned after a successful call to + * {@link CameraDevice#createExtensionSession} as part of the argument + * in the registered state callback {@link StateCallback#onConfigured} + * method.

+ * + *

Note that CameraExtensionSession is currently limited to a maximum of two output + * surfaces for continuous repeating and multi-frame processing respectively. Some + * features such as capture settings will not be supported as the device-specific + * Extension is allowed to override all capture parameters.

+ * + *

Information about support for specific device-specific extensions can be queried + * from {@link CameraExtensionCharacteristics}.

+ */ +public abstract class CameraExtensionSession implements AutoCloseable { + /** @hide */ + public CameraExtensionSession () {} + + /** + * A callback object for tracking the progress of a + * {@link CaptureRequest} submitted to the camera device. + * + *

This callback is invoked when a request triggers a capture to start, + * and when the device-specific Extension post processing begins. In case of an + * error capturing an image, the error method is triggered instead of + * the completion method.

+ * + * @see #capture + * @see #setRepeatingRequest + */ + public static abstract class ExtensionCaptureCallback { + + /** + * This method is called when the camera device has started + * capturing the initial input image of the device-specific extension + * post-process request. + * + *

This callback is invoked right as the capture of a frame begins, + * so it is the most appropriate time for playing a shutter sound, + * or triggering UI indicators of capture.

+ * + *

The request that is being used for this capture is provided, + * along with the actual timestamp for the start of exposure.

+ * + *

The default implementation of this method does nothing.

+ * + * @param session the session received during + * {@link StateCallback#onConfigured(CameraExtensionSession)} + * @param request the request for the capture that just begun + * @param timestamp the timestamp at start of capture for repeating + * request or the timestamp at start of capture of the + * first frame in a multi-frame capture. + */ + public void onCaptureStarted(@NonNull CameraExtensionSession session, + @NonNull CaptureRequest request, long timestamp) { + // default empty implementation + } + + /** + * This method is called when an image (or images in case of multi-frame + * capture) is captured and device-specific extension + * processing is triggered. + * + *

Each request will generate at most {@code 1} + * {@link #onCaptureProcessStarted}.

+ * + *

The default implementation of this method does nothing.

+ * + * @param session the session received during + * {@link StateCallback#onConfigured(CameraExtensionSession)} + * @param request The request that was given to the CameraExtensionSession + * + * @see #capture + * @see #setRepeatingRequest + */ + public void onCaptureProcessStarted(@NonNull CameraExtensionSession session, + @NonNull CaptureRequest request) { + // default empty implementation + } + + /** + * This method is called instead of + * {@link #onCaptureProcessStarted} when the camera device failed + * to produce the required input for the device-specific extension. The + * cause could be a failed camera capture request, a failed + * capture result or dropped camera frame. + * + *

Other requests are unaffected, and some or all image buffers + * from the capture may have been pushed to their respective output + * streams.

+ * + *

The default implementation of this method does nothing.

+ * + * @param session the session received during + * {@link StateCallback#onConfigured(CameraExtensionSession)} + * @param request The request that was given to the CameraDevice + * + * @see #capture + * @see #setRepeatingRequest + */ + public void onCaptureFailed(@NonNull CameraExtensionSession session, + @NonNull CaptureRequest request) { + // default empty implementation + } + + /** + * This method is called independently of the others in + * ExtensionCaptureCallback, when a capture sequence finishes. + * + *

In total, there will be at least one + * {@link #onCaptureProcessStarted}/{@link #onCaptureFailed} + * invocation before this callback is triggered. If the capture + * sequence is aborted before any requests have begun processing, + * {@link #onCaptureSequenceAborted} is invoked instead.

+ * + *

The default implementation does nothing.

+ * + * @param session the session received during + * {@link StateCallback#onConfigured(CameraExtensionSession)} + * @param sequenceId A sequence ID returned by the {@link #capture} + * family of functions. + * @see #onCaptureSequenceAborted + */ + public void onCaptureSequenceCompleted(@NonNull CameraExtensionSession session, + int sequenceId) { + // default empty implementation + } + + /** + * This method is called when a capture sequence aborts. + * + *

Due to the asynchronous nature of the camera device, not all + * submitted captures are immediately processed. It is possible to + * clear out the pending requests by a variety of operations such + * as {@link CameraExtensionSession#stopRepeating}. When such an event + * happens, {@link #onCaptureProcessStarted} will not be called.

+ * + *

The default implementation does nothing.

+ * + * @param session the session received during + * {@link StateCallback#onConfigured(CameraExtensionSession)} + * @param sequenceId A sequence ID returned by the {@link #capture} + * family of functions. + * @see #onCaptureProcessStarted + */ + public void onCaptureSequenceAborted(@NonNull CameraExtensionSession session, + int sequenceId) { + // default empty implementation + } + } + + /** + * A callback object for receiving updates about the state of a camera extension session. + * + */ + public static abstract class StateCallback { + + /** + * This method is called when the camera device has finished configuring itself, and the + * session can start processing capture requests. + * + *

If the camera device configuration fails, then {@link #onConfigureFailed} will + * be invoked instead of this callback.

+ * + * @param session A valid extension session + */ + public abstract void onConfigured(@NonNull CameraExtensionSession session); + + /** + * This method is called if the session cannot be configured as requested. + * + *

This can happen if the set of requested outputs contains unsupported sizes, + * too many outputs are requested at once or the camera device encounters an + * unrecoverable error during configuration.

+ * + *

The session is considered to be closed, and all methods called on it after this + * callback is invoked will throw an IllegalStateException.

+ * + * @param session the session instance that failed to configure + */ + public abstract void onConfigureFailed(@NonNull CameraExtensionSession session); + + /** + * This method is called when the session is closed. + * + *

A session is closed when a new session is created by the parent camera device, + * or when the parent camera device is closed (either by the user closing the device, + * or due to a camera device disconnection or fatal error).

+ * + *

Once a session is closed, all methods on it will throw an IllegalStateException, and + * any repeating requests are stopped (as if {@link #stopRepeating()} was called). + * However, any in-progress capture requests submitted to the session will be completed + * as normal.

+ * + * @param session the session received during + * {@link StateCallback#onConfigured(CameraExtensionSession)} + */ + public void onClosed(@NonNull CameraExtensionSession session) { + // default empty implementation + } + } + + /** + * Get the camera device that this session is created for. + */ + @NonNull + public android.hardware.camera2.CameraDevice getDevice() { + throw new UnsupportedOperationException("Subclasses must override this method"); + } + + /** + * Submit a request for device-specific processing using input + * from the camera device, to produce a single high-quality output result. + * + *

Note that single capture requests currently do not support + * client parameters. Settings included in the request will + * be entirely overridden by the device-specific extension.

+ * + *

The {@link CaptureRequest.Builder#addTarget} supports only one + * ImageFormat.YUV_420_888 or ImageFormat.JPEG target surface. {@link CaptureRequest} + * arguments that include further targets will cause + * IllegalArgumentException to be thrown.

+ * + *

Each request will produce one new frame for one target Surface, set + * with the CaptureRequest builder's + * {@link CaptureRequest.Builder#addTarget} method.

+ * + *

Multiple requests can be in progress at once. Requests are + * processed in first-in, first-out order.

+ * + *

Requests submitted through this method have higher priority than + * those submitted through {@link #setRepeatingRequest}, and will be + * processed as soon as the current repeat processing completes.

+ * + * @param request the settings for this capture + * @param executor the executor which will be used for invoking the + * listener. + * @param listener The callback object to notify once this request has + * been processed. + * @return int A unique capture sequence ID used by + * {@link ExtensionCaptureCallback#onCaptureSequenceCompleted}. + * @throws CameraAccessException if the camera device is no longer + * connected or has encountered a fatal error + * @throws IllegalStateException if this session is no longer active, + * either because the session was explicitly closed, a new + * session has been created or the camera device has been + * closed. + * @throws IllegalArgumentException if the request targets no Surfaces + * or Surfaces that are not configured as outputs for this + * session; or the request targets a set of Surfaces that + * cannot be submitted simultaneously. + */ + public int capture(@NonNull CaptureRequest request, + @NonNull Executor executor, + @NonNull ExtensionCaptureCallback listener) throws CameraAccessException { + throw new UnsupportedOperationException("Subclasses must override this method"); + } + + /** + * Request endlessly repeating device-specific extension processing of + * camera images. + * + *

With this method, the camera device will continually capture images + * and process them using the device-specific extension at the maximum + * rate possible.

+ * + *

Note that repeating capture requests currently do not support + * client parameters. Settings included in the request will + * be completely overridden by the device-specific extension.

+ * + *

The {@link CaptureRequest.Builder#addTarget} supports only one + * target surface. {@link CaptureRequest} arguments that include further + * targets will cause IllegalArgumentException to be thrown.

+ * + *

Repeating requests are a simple way for an application to maintain a + * preview or other continuous stream of frames.

+ * + *

Repeat requests have lower priority than those submitted + * through {@link #capture}, so if {@link #capture} is called when a + * repeating request is active, the capture request will be processed + * before any further repeating requests are processed.

+ * + *

To stop the repeating capture, call {@link #stopRepeating}.

+ * + *

Calling this method will replace any earlier repeating request.

+ * + * @param request the request to repeat indefinitely + * @param executor the executor which will be used for invoking the + * listener. + * @param listener The callback object to notify every time the + * request finishes processing. + * @return int A unique capture sequence ID used by + * {@link ExtensionCaptureCallback#onCaptureSequenceCompleted}. + * @throws CameraAccessException if the camera device is no longer + * connected or has encountered a fatal error + * @throws IllegalStateException if this session is no longer active, + * either because the session was explicitly closed, a new + * session has been created or the camera device has been + * closed. + * @throws IllegalArgumentException If the request references no + * Surfaces or references Surfaces that are not currently + * configured as outputs. + * @see #capture + */ + public int setRepeatingRequest(@NonNull CaptureRequest request, + @NonNull Executor executor, + @NonNull ExtensionCaptureCallback listener) throws CameraAccessException { + throw new UnsupportedOperationException("Subclasses must override this method"); + } + + /** + * Cancel any ongoing repeating capture set by + * {@link #setRepeatingRequest setRepeatingRequest}. Has no effect on + * requests submitted through {@link #capture capture}. + * + *

Any currently in-flight captures will still complete.

+ * + * @throws CameraAccessException if the camera device is no longer + * connected or has encountered a fatal error + * @throws IllegalStateException if this session is no longer active, + * either because the session was explicitly closed, a new + * session has been created or the camera device has been closed. + * @see #setRepeatingRequest + */ + public void stopRepeating() throws CameraAccessException { + throw new UnsupportedOperationException("Subclasses must override this method"); + } + + /** + * Close this capture session asynchronously. + * + *

Closing a session frees up the target output Surfaces of the session + * for reuse with either a new session, or to other APIs that can draw + * to Surfaces.

+ * + *

Note that creating a new capture session with + * {@link android.hardware.camera2.CameraDevice#createCaptureSession} or + * {@link android.hardware.camera2.CameraDevice#createExtensionSession} + * will close any existing capture session automatically, and call the + * older session listener's {@link StateCallback#onClosed} callback. + * Using + * {@link android.hardware.camera2.CameraDevice#createCaptureSession} or + * {@link android.hardware.camera2.CameraDevice#createExtensionSession} + * directly without closing is the recommended approach for quickly + * switching to a new session, since unchanged target outputs can be + * reused more efficiently.

+ * + *

Once a session is closed, all methods on it will throw an + * IllegalStateException, and any repeating requests are + * stopped (as if {@link #stopRepeating()} was called).

+ * + *

Closing a session is idempotent; closing more than once has no + * effect.

+ */ + public void close() throws CameraAccessException { + throw new UnsupportedOperationException("Subclasses must override this method"); + } +} diff --git a/core/java/android/hardware/camera2/CameraManager.java b/core/java/android/hardware/camera2/CameraManager.java index cdfdc1fdb2c39..3e0e3f62574fb 100644 --- a/core/java/android/hardware/camera2/CameraManager.java +++ b/core/java/android/hardware/camera2/CameraManager.java @@ -29,6 +29,7 @@ import android.hardware.ICameraService; import android.hardware.ICameraServiceListener; import android.hardware.camera2.impl.CameraDeviceImpl; import android.hardware.camera2.impl.CameraMetadataNative; +import android.hardware.camera2.params.ExtensionSessionConfiguration; import android.hardware.camera2.params.SessionConfiguration; import android.hardware.camera2.utils.CameraIdAndSessionConfiguration; import android.hardware.camera2.utils.ConcurrentCameraIdCombination; @@ -434,6 +435,28 @@ public final class CameraManager { return characteristics; } + /** + *

Query the camera extension capabilities of a camera device.

+ * + * @param cameraId The id of the camera device to query. This must be a standalone + * camera ID which can be directly opened by {@link #openCamera}. + * @return The properties of the given camera + * + * @throws IllegalArgumentException if the cameraId does not match any + * known camera device. + * @throws CameraAccessException if the camera device has been disconnected. + * + * @see CameraExtensionCharacteristics + * @see CameraDevice#createExtensionSession(ExtensionSessionConfiguration) + * @see CameraExtensionSession + */ + @NonNull + public CameraExtensionCharacteristics getCameraExtensionCharacteristics( + @NonNull String cameraId) throws CameraAccessException { + CameraCharacteristics chars = getCameraCharacteristics(cameraId); + return new CameraExtensionCharacteristics(mContext, cameraId, chars); + } + /** * Helper for opening a connection to a camera with the given ID. * @@ -473,7 +496,8 @@ public final class CameraManager { callback, executor, characteristics, - mContext.getApplicationInfo().targetSdkVersion); + mContext.getApplicationInfo().targetSdkVersion, + mContext); ICameraDeviceCallbacks callbacks = deviceImpl.getCallbacks(); diff --git a/core/java/android/hardware/camera2/CameraMetadata.java b/core/java/android/hardware/camera2/CameraMetadata.java index 41cc12f3b2e3f..924dcee31a33d 100644 --- a/core/java/android/hardware/camera2/CameraMetadata.java +++ b/core/java/android/hardware/camera2/CameraMetadata.java @@ -111,6 +111,15 @@ public abstract class CameraMetadata { } } + /** + * Retrieves the CameraMetadataNative instance. + * + * @hide + */ + public CameraMetadataNative getNativeMetadata() { + return mNativeInstance; + } + /** * @hide */ diff --git a/core/java/android/hardware/camera2/extension/CaptureBundle.aidl b/core/java/android/hardware/camera2/extension/CaptureBundle.aidl new file mode 100644 index 0000000000000..24a0d2f2b0e91 --- /dev/null +++ b/core/java/android/hardware/camera2/extension/CaptureBundle.aidl @@ -0,0 +1,28 @@ +/** + * Copyright (c) 2020, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package android.hardware.camera2.extension; + +import android.hardware.camera2.impl.CameraMetadataNative; +import android.hardware.camera2.extension.ParcelImage; + +/** @hide */ +parcelable CaptureBundle +{ + int stage; + int sequenceId; + CameraMetadataNative captureResult; + ParcelImage captureImage; +} diff --git a/core/java/android/hardware/camera2/extension/CaptureStageImpl.aidl b/core/java/android/hardware/camera2/extension/CaptureStageImpl.aidl new file mode 100644 index 0000000000000..f2187a3ea900b --- /dev/null +++ b/core/java/android/hardware/camera2/extension/CaptureStageImpl.aidl @@ -0,0 +1,25 @@ +/** + * Copyright (c) 2020, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package android.hardware.camera2.extension; + +import android.hardware.camera2.impl.CameraMetadataNative; + +/** @hide */ +parcelable CaptureStageImpl +{ + int id; + CameraMetadataNative parameters; +} diff --git a/core/java/android/hardware/camera2/extension/ICameraExtensionsProxyService.aidl b/core/java/android/hardware/camera2/extension/ICameraExtensionsProxyService.aidl new file mode 100644 index 0000000000000..2a6d22cd807cf --- /dev/null +++ b/core/java/android/hardware/camera2/extension/ICameraExtensionsProxyService.aidl @@ -0,0 +1,28 @@ +/** + * Copyright (c) 2020, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package android.hardware.camera2.extension; + +import android.hardware.camera2.extension.IPreviewExtenderImpl; +import android.hardware.camera2.extension.IImageCaptureExtenderImpl; + +/** @hide */ +interface ICameraExtensionsProxyService +{ + long registerClient(); + void unregisterClient(long clientId); + @nullable IPreviewExtenderImpl initializePreviewExtension(int extensionType); + @nullable IImageCaptureExtenderImpl initializeImageExtension(int extensionType); +} diff --git a/core/java/android/hardware/camera2/extension/ICaptureProcessorImpl.aidl b/core/java/android/hardware/camera2/extension/ICaptureProcessorImpl.aidl new file mode 100644 index 0000000000000..022b084f613b3 --- /dev/null +++ b/core/java/android/hardware/camera2/extension/ICaptureProcessorImpl.aidl @@ -0,0 +1,29 @@ +/** + * Copyright (c) 2020, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package android.hardware.camera2.extension; + +import android.view.Surface; +import android.hardware.camera2.extension.CaptureBundle; +import android.hardware.camera2.extension.Size; + +/** @hide */ +interface ICaptureProcessorImpl +{ + void onOutputSurface(in Surface surface, int imageFormat); + void onResolutionUpdate(in Size size); + void onImageFormatUpdate(int imageFormat); + void process(in List capturelist); +} diff --git a/core/java/android/hardware/camera2/extension/IImageCaptureExtenderImpl.aidl b/core/java/android/hardware/camera2/extension/IImageCaptureExtenderImpl.aidl new file mode 100644 index 0000000000000..c04e75ea67583 --- /dev/null +++ b/core/java/android/hardware/camera2/extension/IImageCaptureExtenderImpl.aidl @@ -0,0 +1,39 @@ +/** + * Copyright (c) 2020, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package android.hardware.camera2.extension; + +import android.hardware.camera2.impl.CameraMetadataNative; + +import android.hardware.camera2.extension.CaptureStageImpl; +import android.hardware.camera2.extension.ICaptureProcessorImpl; +import android.hardware.camera2.extension.SizeList; + +/** @hide */ +interface IImageCaptureExtenderImpl +{ + void onInit(in String cameraId, in CameraMetadataNative cameraCharacteristics); + void onDeInit(); + @nullable CaptureStageImpl onPresetSession(); + @nullable CaptureStageImpl onEnableSession(); + @nullable CaptureStageImpl onDisableSession(); + + boolean isExtensionAvailable(in String cameraId, in CameraMetadataNative chars); + void init(in String cameraId, in CameraMetadataNative chars); + @nullable ICaptureProcessorImpl getCaptureProcessor(); + @nullable List getCaptureStages(); + int getMaxCaptureStage(); + @nullable List getSupportedResolutions(); +} diff --git a/core/java/android/hardware/camera2/extension/IPreviewExtenderImpl.aidl b/core/java/android/hardware/camera2/extension/IPreviewExtenderImpl.aidl new file mode 100644 index 0000000000000..2d673448fe27a --- /dev/null +++ b/core/java/android/hardware/camera2/extension/IPreviewExtenderImpl.aidl @@ -0,0 +1,46 @@ +/** + * Copyright (c) 2020, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package android.hardware.camera2.extension; + +import android.hardware.camera2.impl.CameraMetadataNative; + +import android.hardware.camera2.extension.CaptureStageImpl; +import android.hardware.camera2.extension.IPreviewImageProcessorImpl; +import android.hardware.camera2.extension.IRequestUpdateProcessorImpl; +import android.hardware.camera2.extension.SizeList; + +/** @hide */ +interface IPreviewExtenderImpl +{ + void onInit(in String cameraId, in CameraMetadataNative cameraCharacteristics); + void onDeInit(); + @nullable CaptureStageImpl onPresetSession(); + @nullable CaptureStageImpl onEnableSession(); + @nullable CaptureStageImpl onDisableSession(); + + void init(in String cameraId, in CameraMetadataNative chars); + boolean isExtensionAvailable(in String cameraId, in CameraMetadataNative chars); + @nullable CaptureStageImpl getCaptureStage(); + + const int PROCESSOR_TYPE_REQUEST_UPDATE_ONLY = 0; + const int PROCESSOR_TYPE_IMAGE_PROCESSOR = 1; + const int PROCESSOR_TYPE_NONE = 2; + int getProcessorType(); + @nullable IPreviewImageProcessorImpl getPreviewImageProcessor(); + @nullable IRequestUpdateProcessorImpl getRequestUpdateProcessor(); + + @nullable List getSupportedResolutions(); +} diff --git a/core/java/android/hardware/camera2/extension/IPreviewImageProcessorImpl.aidl b/core/java/android/hardware/camera2/extension/IPreviewImageProcessorImpl.aidl new file mode 100644 index 0000000000000..f7e4023c50676 --- /dev/null +++ b/core/java/android/hardware/camera2/extension/IPreviewImageProcessorImpl.aidl @@ -0,0 +1,30 @@ +/** + * Copyright (c) 2020, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package android.hardware.camera2.extension; + +import android.hardware.camera2.impl.CameraMetadataNative; +import android.view.Surface; +import android.hardware.camera2.extension.ParcelImage; +import android.hardware.camera2.extension.Size; + +/** @hide */ +interface IPreviewImageProcessorImpl +{ + void onOutputSurface(in Surface surface, int imageFormat); + void onResolutionUpdate(in Size size); + void onImageFormatUpdate(int imageFormat); + void process(in ParcelImage image, in CameraMetadataNative result, int sequenceId); +} diff --git a/core/java/android/hardware/camera2/extension/IRequestUpdateProcessorImpl.aidl b/core/java/android/hardware/camera2/extension/IRequestUpdateProcessorImpl.aidl new file mode 100644 index 0000000000000..f07702416d63e --- /dev/null +++ b/core/java/android/hardware/camera2/extension/IRequestUpdateProcessorImpl.aidl @@ -0,0 +1,30 @@ +/** + * Copyright (c) 2020, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package android.hardware.camera2.extension; + +import android.hardware.camera2.impl.CameraMetadataNative; +import android.view.Surface; +import android.hardware.camera2.extension.CaptureStageImpl; +import android.hardware.camera2.extension.Size; + +/** @hide */ +interface IRequestUpdateProcessorImpl +{ + void onOutputSurface(in Surface surface, int imageFormat); + void onResolutionUpdate(in Size size); + void onImageFormatUpdate(int imageFormat); + @nullable CaptureStageImpl process(in CameraMetadataNative result, int sequenceId); +} diff --git a/core/java/android/hardware/camera2/extension/ParcelImage.aidl b/core/java/android/hardware/camera2/extension/ParcelImage.aidl new file mode 100644 index 0000000000000..dd5584f357771 --- /dev/null +++ b/core/java/android/hardware/camera2/extension/ParcelImage.aidl @@ -0,0 +1,35 @@ +/** + * Copyright (c) 2020, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package android.hardware.camera2.extension; + +import android.graphics.Rect; +import android.hardware.HardwareBuffer; +import android.os.ParcelFileDescriptor; + +/** @hide */ +parcelable ParcelImage +{ + int format; + int width; + int height; + int transform; + int scalingMode; + long timestamp; + int planeCount; + Rect crop; + HardwareBuffer buffer; + ParcelFileDescriptor fence; +} diff --git a/core/java/android/hardware/camera2/extension/Size.aidl b/core/java/android/hardware/camera2/extension/Size.aidl new file mode 100644 index 0000000000000..ec222d3d039f2 --- /dev/null +++ b/core/java/android/hardware/camera2/extension/Size.aidl @@ -0,0 +1,23 @@ +/** + * Copyright (c) 2020, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package android.hardware.camera2.extension; + +/** @hide */ +parcelable Size +{ + int width; + int height; +} diff --git a/core/java/android/hardware/camera2/extension/SizeList.aidl b/core/java/android/hardware/camera2/extension/SizeList.aidl new file mode 100644 index 0000000000000..946cf98fe075c --- /dev/null +++ b/core/java/android/hardware/camera2/extension/SizeList.aidl @@ -0,0 +1,25 @@ +/** + * Copyright (c) 2020, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package android.hardware.camera2.extension; + +import android.hardware.camera2.extension.Size; + +/** @hide */ +parcelable SizeList +{ + int format; + List sizes; +} diff --git a/core/java/android/hardware/camera2/impl/CameraDeviceImpl.java b/core/java/android/hardware/camera2/impl/CameraDeviceImpl.java index f564ad7436dea..ce3c81a2bfd62 100644 --- a/core/java/android/hardware/camera2/impl/CameraDeviceImpl.java +++ b/core/java/android/hardware/camera2/impl/CameraDeviceImpl.java @@ -19,10 +19,12 @@ package android.hardware.camera2.impl; import static com.android.internal.util.function.pooled.PooledLambda.obtainRunnable; import android.annotation.NonNull; +import android.content.Context; import android.hardware.ICameraService; import android.hardware.camera2.CameraAccessException; import android.hardware.camera2.CameraCaptureSession; import android.hardware.camera2.CameraCharacteristics; +import android.hardware.camera2.CameraExtensionSession; import android.hardware.camera2.CameraOfflineSession; import android.hardware.camera2.CameraDevice; import android.hardware.camera2.CaptureFailure; @@ -32,6 +34,7 @@ import android.hardware.camera2.ICameraDeviceCallbacks; import android.hardware.camera2.ICameraDeviceUser; import android.hardware.camera2.ICameraOfflineSession; import android.hardware.camera2.TotalCaptureResult; +import android.hardware.camera2.params.ExtensionSessionConfiguration; import android.hardware.camera2.params.InputConfiguration; import android.hardware.camera2.params.OutputConfiguration; import android.hardware.camera2.params.SessionConfiguration; @@ -110,6 +113,7 @@ public class CameraDeviceImpl extends CameraDevice private final String mCameraId; private final CameraCharacteristics mCharacteristics; private final int mTotalPartialCount; + private final Context mContext; private static final long NANO_PER_SECOND = 1000000000; //ns @@ -127,6 +131,7 @@ public class CameraDeviceImpl extends CameraDevice private FrameNumberTracker mFrameNumberTracker = new FrameNumberTracker(); private CameraCaptureSessionCore mCurrentSession; + private CameraExtensionSessionImpl mCurrentExtensionSession; private int mNextSessionId = 0; private final int mAppTargetSdkVersion; @@ -250,7 +255,8 @@ public class CameraDeviceImpl extends CameraDevice }; public CameraDeviceImpl(String cameraId, StateCallback callback, Executor executor, - CameraCharacteristics characteristics, int appTargetSdkVersion) { + CameraCharacteristics characteristics, int appTargetSdkVersion, + Context ctx) { if (cameraId == null || callback == null || executor == null || characteristics == null) { throw new IllegalArgumentException("Null argument given"); } @@ -259,6 +265,7 @@ public class CameraDeviceImpl extends CameraDevice mDeviceExecutor = executor; mCharacteristics = characteristics; mAppTargetSdkVersion = appTargetSdkVersion; + mContext = ctx; final int MAX_TAG_LEN = 23; String tag = String.format("CameraDevice-JV-%s", mCameraId); @@ -1322,6 +1329,10 @@ public class CameraDeviceImpl extends CameraDevice mRemoteDevice.unlinkToDeath(this, /*flags*/0); } + if (mCurrentExtensionSession != null) { + mCurrentExtensionSession.release(); + mCurrentExtensionSession = null; + } // Only want to fire the onClosed callback once; // either a normal close where the remote device is valid // or a close after a startup error (no remote device but in error state) @@ -2297,4 +2308,16 @@ public class CameraDeviceImpl extends CameraDevice return mRemoteDevice.getGlobalAudioRestriction(); } } + + @Override + public void createExtensionSession(ExtensionSessionConfiguration extensionConfiguration) + throws CameraAccessException { + try { + mCurrentExtensionSession = CameraExtensionSessionImpl.createCameraExtensionSession(this, + mContext, + extensionConfiguration); + } catch (RemoteException e) { + throw new CameraAccessException(CameraAccessException.CAMERA_ERROR); + } + } } diff --git a/core/java/android/hardware/camera2/impl/CameraExtensionForwardProcessor.java b/core/java/android/hardware/camera2/impl/CameraExtensionForwardProcessor.java new file mode 100644 index 0000000000000..8eb1dccdf65ec --- /dev/null +++ b/core/java/android/hardware/camera2/impl/CameraExtensionForwardProcessor.java @@ -0,0 +1,150 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.hardware.camera2.impl; + +import android.annotation.SuppressLint; +import android.hardware.camera2.CameraExtensionCharacteristics; +import android.hardware.camera2.extension.IPreviewImageProcessorImpl; +import android.hardware.camera2.extension.ParcelImage; +import android.hardware.camera2.TotalCaptureResult; +import android.media.Image; +import android.media.ImageReader; +import android.media.ImageWriter; +import android.annotation.NonNull; +import android.os.RemoteException; +import android.util.Log; +import android.util.Size; +import android.view.Surface; + +// Wrap a given 'PreviewImageProcessorImpl' so that the processed output can +// be redirected to a given surface or looped back in the internal intermediate surface. +public class CameraExtensionForwardProcessor { + public final static String TAG = "CameraExtensionForward"; + private final static int FORWARD_QUEUE_SIZE = 3; + + private final IPreviewImageProcessorImpl mProcessor; + private final long mOutputSurfaceUsage; + private final int mOutputSurfaceFormat; + + private ImageReader mIntermediateReader = null; + private Surface mIntermediateSurface = null; + private Size mResolution = null; + private Surface mOutputSurface = null; + private ImageWriter mOutputWriter = null; + private boolean mOutputAbandoned = false; + + public CameraExtensionForwardProcessor(@NonNull IPreviewImageProcessorImpl processor, + int format, long surfaceUsage) { + mProcessor = processor; + mOutputSurfaceUsage = surfaceUsage; + mOutputSurfaceFormat = format; + } + + public void close() { + if (mOutputWriter != null) { + mOutputWriter.close(); + mOutputWriter = null; + } + + if (mIntermediateReader != null) { + mIntermediateReader.close(); + mIntermediateReader = null; + } + } + + public void onOutputSurface(Surface surface, int format) { + mOutputSurface = surface; + try { + initializePipeline(); + } catch (RemoteException e) { + Log.e(TAG, "Failed to initialize forward processor, extension service does not" + + " respond!"); + } + } + + public void onResolutionUpdate(Size size) { + mResolution = size; + } + + public void onImageFormatUpdate(int format) { + if (format != CameraExtensionCharacteristics.PROCESSING_INPUT_FORMAT) { + Log.e(TAG, "Unsupported input format: " + format); + } + } + + @SuppressLint("WrongConstant") + private void initializePipeline() throws RemoteException { + if (mOutputWriter != null) { + mOutputWriter.close(); + mOutputWriter = null; + } + + if (mIntermediateReader == null) { + mIntermediateReader = ImageReader.newInstance(mResolution.getWidth(), + mResolution.getHeight(), CameraExtensionCharacteristics.PROCESSING_INPUT_FORMAT, + FORWARD_QUEUE_SIZE, mOutputSurfaceUsage); + mIntermediateSurface = mIntermediateReader.getSurface(); + mIntermediateReader.setOnImageAvailableListener(new ForwardCallback(), null); + + mProcessor.onOutputSurface(mIntermediateSurface, mOutputSurfaceFormat); + // PreviewImageProcessorImpl always expect the extension processing format as input + mProcessor.onImageFormatUpdate(CameraExtensionCharacteristics.PROCESSING_INPUT_FORMAT); + android.hardware.camera2.extension.Size sz = + new android.hardware.camera2.extension.Size(); + sz.width = mResolution.getWidth(); + sz.height = mResolution.getHeight(); + mProcessor.onResolutionUpdate(sz); + } + } + + public void process(ParcelImage image, TotalCaptureResult totalCaptureResult) + throws RemoteException { + if ((mIntermediateSurface != null) && (mIntermediateSurface.isValid()) && + !mOutputAbandoned) { + mProcessor.process(image, totalCaptureResult.getNativeMetadata(), + totalCaptureResult.getSequenceId()); + } + } + + private class ForwardCallback implements ImageReader.OnImageAvailableListener { + @Override public void onImageAvailable(ImageReader reader) { + Image processedImage = null; + try { + processedImage = mIntermediateReader.acquireNextImage(); + } catch (IllegalStateException e) { + Log.e(TAG, "Failed to acquire processed image!"); + return; + } + + if (mOutputSurface != null && mOutputSurface.isValid() && !mOutputAbandoned) { + if (mOutputWriter == null) { + mOutputWriter = ImageWriter.newInstance(mOutputSurface, FORWARD_QUEUE_SIZE, + processedImage.getFormat()); + } + try { + mOutputWriter.queueInputImage(processedImage); + } catch (IllegalStateException e) { + Log.e(TAG, "Failed to queue processed buffer!"); + processedImage.close(); + mOutputAbandoned = true; + } + } else { + processedImage.close(); + } + } + } +} diff --git a/core/java/android/hardware/camera2/impl/CameraExtensionSessionImpl.java b/core/java/android/hardware/camera2/impl/CameraExtensionSessionImpl.java new file mode 100644 index 0000000000000..8fe71583912ca --- /dev/null +++ b/core/java/android/hardware/camera2/impl/CameraExtensionSessionImpl.java @@ -0,0 +1,1649 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.hardware.camera2.impl; + +import android.content.Context; +import android.graphics.ImageFormat; +import android.graphics.PixelFormat; +import android.graphics.SurfaceTexture; +import android.hardware.HardwareBuffer; +import android.hardware.camera2.CameraAccessException; +import android.hardware.camera2.CameraCaptureSession; +import android.hardware.camera2.CameraCharacteristics; +import android.hardware.camera2.CameraDevice; +import android.hardware.camera2.CameraExtensionCharacteristics; +import android.hardware.camera2.CameraExtensionSession; +import android.hardware.camera2.CameraManager; +import android.hardware.camera2.CaptureFailure; +import android.hardware.camera2.CaptureRequest; +import android.hardware.camera2.CaptureResult; +import android.hardware.camera2.extension.CaptureBundle; +import android.hardware.camera2.extension.CaptureStageImpl; +import android.hardware.camera2.extension.ICaptureProcessorImpl; +import android.hardware.camera2.extension.IImageCaptureExtenderImpl; +import android.hardware.camera2.extension.IPreviewExtenderImpl; +import android.hardware.camera2.extension.IRequestUpdateProcessorImpl; +import android.hardware.camera2.extension.ParcelImage; +import android.hardware.camera2.TotalCaptureResult; +import android.hardware.camera2.params.ExtensionSessionConfiguration; +import android.hardware.camera2.params.OutputConfiguration; +import android.hardware.camera2.params.SessionConfiguration; +import android.hardware.camera2.params.StreamConfigurationMap; +import android.hardware.camera2.utils.SurfaceUtils; +import android.media.Image; +import android.media.ImageReader; +import android.media.ImageWriter; +import android.os.Binder; +import android.os.Handler; +import android.os.HandlerThread; +import android.os.ParcelFileDescriptor; +import android.os.RemoteException; +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.annotation.RequiresPermission; +import android.util.Log; +import android.util.LongSparseArray; +import android.util.Pair; +import android.util.Size; +import android.view.Surface; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Executor; +import java.util.concurrent.RejectedExecutionException; + +public final class CameraExtensionSessionImpl extends CameraExtensionSession { + private static final int PREVIEW_QUEUE_SIZE = 3; + private static final String TAG = "CameraExtensionSessionImpl"; + + private final Executor mExecutor; + private final CameraDevice mCameraDevice; + private final long mExtensionClientId; + private final IImageCaptureExtenderImpl mImageExtender; + private final IPreviewExtenderImpl mPreviewExtender; + private final Handler mHandler; + private final HandlerThread mHandlerThread; + private final StateCallback mCallbacks; + private final List mSupportedPreviewSizes; + + private CameraCaptureSession mCaptureSession = null; + private Surface mCameraRepeatingSurface, mClientRepeatingRequestSurface; + private Surface mCameraBurstSurface, mClientCaptureSurface; + private ImageReader mRepeatingRequestImageReader = null; + private ImageReader mBurstCaptureImageReader = null; + private ImageReader mStubCaptureImageReader = null; + private ImageWriter mRepeatingRequestImageWriter = null; + + private ICaptureProcessorImpl mImageProcessor = null; + private CameraExtensionForwardProcessor mPreviewImageProcessor = null; + private IRequestUpdateProcessorImpl mPreviewRequestUpdateProcessor = null; + private int mPreviewProcessorType = IPreviewExtenderImpl.PROCESSOR_TYPE_NONE; + + private boolean mInitialized; + // Enable/Disable internal preview/(repeating request). Extensions expect + // that preview/(repeating request) is enabled and active at any point in time. + // In case the client doesn't explicitly enable repeating requests, the framework + // will do so internally. + private boolean mInternalRepeatingRequestEnabled = true; + + // Lock to synchronize cross-thread access to device public interface + final Object mInterfaceLock = new Object(); // access from this class and Session only! + + private static class SurfaceInfo { + public int mWidth = 0; + public int mHeight = 0; + public int mFormat = PixelFormat.RGBA_8888; + public long mUsage = 0; + } + + private static final int SUPPORTED_CAPTURE_OUTPUT_FORMATS[] = { + CameraExtensionCharacteristics.PROCESSING_INPUT_FORMAT, + ImageFormat.JPEG + }; + + private static int nativeGetSurfaceWidth(Surface surface) { + return SurfaceUtils.getSurfaceSize(surface).getWidth(); + } + + private static int nativeGetSurfaceHeight(Surface surface) { + return SurfaceUtils.getSurfaceSize(surface).getHeight(); + } + + private static int nativeGetSurfaceFormat(Surface surface) { + return SurfaceUtils.getSurfaceFormat(surface); + } + + private static Surface getBurstCaptureSurface( + @NonNull List outputConfigs, + @NonNull HashMap> supportedCaptureSizes) { + for (OutputConfiguration config : outputConfigs) { + SurfaceInfo surfaceInfo = querySurface(config.getSurface()); + for (int supportedFormat : SUPPORTED_CAPTURE_OUTPUT_FORMATS) { + if (surfaceInfo.mFormat == supportedFormat) { + Size captureSize = new Size(surfaceInfo.mWidth, surfaceInfo.mHeight); + if (supportedCaptureSizes.containsKey(supportedFormat)) { + if (supportedCaptureSizes.get(surfaceInfo.mFormat).contains(captureSize)) { + return config.getSurface(); + } else { + throw new IllegalArgumentException("Capture size not supported!"); + } + } + return config.getSurface(); + } + } + } + + return null; + } + + private static @Nullable Surface getRepeatingRequestSurface( + @NonNull List outputConfigs, + @Nullable List supportedPreviewSizes) { + for (OutputConfiguration config : outputConfigs) { + SurfaceInfo surfaceInfo = querySurface(config.getSurface()); + if ((surfaceInfo.mFormat == + CameraExtensionCharacteristics.NON_PROCESSING_INPUT_FORMAT) || + // The default RGBA_8888 is also implicitly supported because camera will + // internally override it to + // 'CameraExtensionCharacteristics.NON_PROCESSING_INPUT_FORMAT' + (surfaceInfo.mFormat == PixelFormat.RGBA_8888)) { + Size repeatingRequestSurfaceSize = new Size(surfaceInfo.mWidth, + surfaceInfo.mHeight); + if ((supportedPreviewSizes == null) || + (!supportedPreviewSizes.contains(repeatingRequestSurfaceSize))) { + throw new IllegalArgumentException("Repeating request surface size " + + repeatingRequestSurfaceSize + " not supported!"); + } + + return config.getSurface(); + } + } + + return null; + } + + /** + * @hide + */ + @RequiresPermission(android.Manifest.permission.CAMERA) + public static CameraExtensionSessionImpl createCameraExtensionSession( + @NonNull CameraDevice cameraDevice, + @NonNull Context ctx, + @NonNull ExtensionSessionConfiguration config) + throws CameraAccessException, RemoteException { + long clientId = CameraExtensionCharacteristics.registerClient(ctx); + if (clientId < 0) { + throw new UnsupportedOperationException("Unsupported extension!"); + } + + String cameraId = cameraDevice.getId(); + CameraManager manager = ctx.getSystemService(CameraManager.class); + CameraCharacteristics chars = manager.getCameraCharacteristics(cameraId); + CameraExtensionCharacteristics extensionChars = new CameraExtensionCharacteristics(ctx, + cameraId, chars); + + if (!CameraExtensionCharacteristics.isExtensionSupported(cameraDevice.getId(), + config.getExtension(), chars)) { + throw new UnsupportedOperationException("Unsupported extension type: " + + config.getExtension()); + } + + if (config.getOutputConfigurations().isEmpty() || + config.getOutputConfigurations().size() > 2) { + throw new IllegalArgumentException("Unexpected amount of output surfaces, received: " + + config.getOutputConfigurations().size() + " expected <= 2"); + } + + Pair extenders = + CameraExtensionCharacteristics.initializeExtension(config.getExtension()); + + int suitableSurfaceCount = 0; + List supportedPreviewSizes = extensionChars.getExtensionSupportedSizes( + config.getExtension(), SurfaceTexture.class); + Surface repeatingRequestSurface = getRepeatingRequestSurface( + config.getOutputConfigurations(), supportedPreviewSizes); + if (repeatingRequestSurface != null) { + suitableSurfaceCount++; + } + + HashMap> supportedCaptureSizes = new HashMap<>(); + for (int format : SUPPORTED_CAPTURE_OUTPUT_FORMATS) { + List supportedSizes = extensionChars.getExtensionSupportedSizes( + config.getExtension(), format); + if (supportedSizes != null) { + supportedCaptureSizes.put(format, supportedSizes); + } + } + Surface burstCaptureSurface = getBurstCaptureSurface(config.getOutputConfigurations(), + supportedCaptureSizes); + if (burstCaptureSurface != null) { + suitableSurfaceCount++; + } + + if (suitableSurfaceCount != config.getOutputConfigurations().size()) { + throw new IllegalArgumentException("One or more unsupported output surfaces found!"); + } + + extenders.first.init(cameraId, chars.getNativeMetadata()); + extenders.first.onInit(cameraId, chars.getNativeMetadata()); + extenders.second.init(cameraId, chars.getNativeMetadata()); + extenders.second.onInit(cameraId, chars.getNativeMetadata()); + + CameraExtensionSessionImpl session = new CameraExtensionSessionImpl( + extenders.second, + extenders.first, + supportedPreviewSizes, + clientId, + cameraDevice, + repeatingRequestSurface, + burstCaptureSurface, + config.getStateCallback(), + config.getExecutor()); + + session.initialize(); + + return session; + } + + private CameraExtensionSessionImpl(@NonNull IImageCaptureExtenderImpl imageExtender, + @NonNull IPreviewExtenderImpl previewExtender, + @NonNull List previewSizes, + long extensionClientId, + @NonNull CameraDevice cameraDevice, + @Nullable Surface repeatingRequestSurface, + @Nullable Surface burstCaptureSurface, + @NonNull StateCallback callback, + @NonNull Executor executor) { + mExtensionClientId = extensionClientId; + mImageExtender = imageExtender; + mPreviewExtender = previewExtender; + mCameraDevice = cameraDevice; + mCallbacks = callback; + mExecutor = executor; + mClientRepeatingRequestSurface = repeatingRequestSurface; + mClientCaptureSurface = burstCaptureSurface; + mSupportedPreviewSizes = previewSizes; + mHandlerThread = new HandlerThread(TAG); + mHandlerThread.start(); + mHandler = new Handler(mHandlerThread.getLooper()); + mInitialized = false; + } + + private static @NonNull SurfaceInfo querySurface(@NonNull Surface s) { + ImageWriter writer = null; + Image img = null; + SurfaceInfo surfaceInfo = new SurfaceInfo(); + int nativeFormat = nativeGetSurfaceFormat(s); + int dataspace = SurfaceUtils.getSurfaceDataspace(s); + // Jpeg surfaces cannot be queried for their usage and other parameters + // in the usual way below. A buffer can only be de-queued after the + // producer overrides the surface dimensions to (width*height) x 1. + if ((nativeFormat == StreamConfigurationMap.HAL_PIXEL_FORMAT_BLOB) && + (dataspace == StreamConfigurationMap.HAL_DATASPACE_V0_JFIF)) { + surfaceInfo.mFormat = ImageFormat.JPEG; + surfaceInfo.mWidth = nativeGetSurfaceWidth(s); + surfaceInfo.mHeight = nativeGetSurfaceHeight(s); + return surfaceInfo; + } + + HardwareBuffer buffer = null; + try { + writer = ImageWriter.newInstance(s, 1); + img = writer.dequeueInputImage(); + buffer = img.getHardwareBuffer(); + surfaceInfo.mFormat = buffer.getFormat(); + surfaceInfo.mWidth = buffer.getWidth(); + surfaceInfo.mHeight = buffer.getHeight(); + surfaceInfo.mUsage = buffer.getUsage(); + } catch (Exception e) { + Log.e(TAG, "Failed to query surface, returning defaults!"); + } finally { + if (buffer != null) { + buffer.close(); + } + if (img != null) { + img.close(); + } + if (writer != null) { + writer.close(); + } + } + + return surfaceInfo; + } + + private void initializeRepeatingRequestPipeline() throws RemoteException { + SurfaceInfo repeatingSurfaceInfo = new SurfaceInfo(); + mPreviewProcessorType = mPreviewExtender.getProcessorType(); + if (mClientRepeatingRequestSurface != null) { + repeatingSurfaceInfo = querySurface(mClientRepeatingRequestSurface); + } else { + // Make the intermediate surface behave as any regular 'SurfaceTexture' + SurfaceInfo captureSurfaceInfo = querySurface(mClientCaptureSurface); + Size captureSize = new Size(captureSurfaceInfo.mWidth, captureSurfaceInfo.mHeight); + Size previewSize = findSmallestAspectMatchedSize(mSupportedPreviewSizes, captureSize); + repeatingSurfaceInfo.mWidth = previewSize.getWidth(); + repeatingSurfaceInfo.mHeight = previewSize.getHeight(); + repeatingSurfaceInfo.mUsage = HardwareBuffer.USAGE_GPU_SAMPLED_IMAGE; + } + + if (mPreviewProcessorType == IPreviewExtenderImpl.PROCESSOR_TYPE_IMAGE_PROCESSOR) { + try { + mPreviewImageProcessor = new CameraExtensionForwardProcessor( + mPreviewExtender.getPreviewImageProcessor(), repeatingSurfaceInfo.mFormat, + repeatingSurfaceInfo.mUsage); + } catch (ClassCastException e) { + throw new UnsupportedOperationException("Failed casting preview processor!"); + } + mPreviewImageProcessor.onImageFormatUpdate( + CameraExtensionCharacteristics.PROCESSING_INPUT_FORMAT); + mPreviewImageProcessor.onResolutionUpdate(new Size(repeatingSurfaceInfo.mWidth, + repeatingSurfaceInfo.mHeight)); + mRepeatingRequestImageReader = ImageReader.newInstance(repeatingSurfaceInfo.mWidth, + repeatingSurfaceInfo.mHeight, + CameraExtensionCharacteristics.PROCESSING_INPUT_FORMAT, PREVIEW_QUEUE_SIZE, + repeatingSurfaceInfo.mUsage); + mCameraRepeatingSurface = mRepeatingRequestImageReader.getSurface(); + } else if (mPreviewProcessorType == + IPreviewExtenderImpl.PROCESSOR_TYPE_REQUEST_UPDATE_ONLY) { + try { + mPreviewRequestUpdateProcessor = mPreviewExtender.getRequestUpdateProcessor(); + } catch (ClassCastException e) { + throw new UnsupportedOperationException("Failed casting preview processor!"); + } + if (mClientRepeatingRequestSurface != null) { + mPreviewRequestUpdateProcessor.onOutputSurface(mClientRepeatingRequestSurface, + nativeGetSurfaceFormat(mClientRepeatingRequestSurface)); + mRepeatingRequestImageWriter = ImageWriter.newInstance( + mClientRepeatingRequestSurface, PREVIEW_QUEUE_SIZE, + CameraExtensionCharacteristics.NON_PROCESSING_INPUT_FORMAT); + } + mRepeatingRequestImageReader = ImageReader.newInstance(repeatingSurfaceInfo.mWidth, + repeatingSurfaceInfo.mHeight, + CameraExtensionCharacteristics.NON_PROCESSING_INPUT_FORMAT, + PREVIEW_QUEUE_SIZE, repeatingSurfaceInfo.mUsage); + mCameraRepeatingSurface = mRepeatingRequestImageReader.getSurface(); + android.hardware.camera2.extension.Size sz = + new android.hardware.camera2.extension.Size(); + sz.width = repeatingSurfaceInfo.mWidth; + sz.height = repeatingSurfaceInfo.mHeight; + mPreviewRequestUpdateProcessor.onResolutionUpdate(sz); + mPreviewRequestUpdateProcessor.onImageFormatUpdate( + CameraExtensionCharacteristics.NON_PROCESSING_INPUT_FORMAT); + } else { + if (mClientRepeatingRequestSurface != null) { + mRepeatingRequestImageWriter = ImageWriter.newInstance( + mClientRepeatingRequestSurface, PREVIEW_QUEUE_SIZE, + CameraExtensionCharacteristics.NON_PROCESSING_INPUT_FORMAT); + } + mRepeatingRequestImageReader = ImageReader.newInstance(repeatingSurfaceInfo.mWidth, + repeatingSurfaceInfo.mHeight, + CameraExtensionCharacteristics.NON_PROCESSING_INPUT_FORMAT, + PREVIEW_QUEUE_SIZE, repeatingSurfaceInfo.mUsage); + mCameraRepeatingSurface = mRepeatingRequestImageReader.getSurface(); + } + mRepeatingRequestImageReader + .setOnImageAvailableListener(new ImageLoopbackCallback(), mHandler); + } + + private void initializeBurstCapturePipeline() throws RemoteException { + mImageProcessor = mImageExtender.getCaptureProcessor(); + if ((mImageProcessor == null) && (mImageExtender.getMaxCaptureStage() != 1)) { + throw new UnsupportedOperationException("Multiple stages expected without" + + " a valid capture processor!"); + } + + if (mImageProcessor != null) { + if (mClientCaptureSurface != null) { + SurfaceInfo surfaceInfo = querySurface(mClientCaptureSurface); + mBurstCaptureImageReader = ImageReader.newInstance(surfaceInfo.mWidth, + surfaceInfo.mHeight, CameraExtensionCharacteristics.PROCESSING_INPUT_FORMAT, + mImageExtender.getMaxCaptureStage()); + mImageProcessor.onOutputSurface(mClientCaptureSurface, surfaceInfo.mFormat); + } else { + // The client doesn't intend to trigger multi-frame capture, however the + // image extender still needs to get initialized and the camera still capture + // stream configured for the repeating request processing to work. + mBurstCaptureImageReader = ImageReader.newInstance( + mRepeatingRequestImageReader.getWidth(), + mRepeatingRequestImageReader.getHeight(), + CameraExtensionCharacteristics.PROCESSING_INPUT_FORMAT, 1); + // The still capture output is not going to be used but we still need a valid + // surface to register. + mStubCaptureImageReader = ImageReader.newInstance( + mRepeatingRequestImageReader.getWidth(), + mRepeatingRequestImageReader.getHeight(), + CameraExtensionCharacteristics.PROCESSING_INPUT_FORMAT, 1); + mImageProcessor.onOutputSurface(mStubCaptureImageReader.getSurface(), + CameraExtensionCharacteristics.PROCESSING_INPUT_FORMAT); + } + + mCameraBurstSurface = mBurstCaptureImageReader.getSurface(); + android.hardware.camera2.extension.Size sz = + new android.hardware.camera2.extension.Size(); + sz.width = mBurstCaptureImageReader.getWidth(); + sz.height = mBurstCaptureImageReader.getHeight(); + mImageProcessor.onResolutionUpdate(sz); + mImageProcessor.onImageFormatUpdate(mBurstCaptureImageReader.getImageFormat()); + } else { + if (mClientCaptureSurface != null) { + // Redirect camera output directly in to client output surface + mCameraBurstSurface = mClientCaptureSurface; + } else { + // The client doesn't intend to trigger multi-frame capture, however the + // image extender still needs to get initialized and the camera still capture + // stream configured for the repeating request processing to work. + mBurstCaptureImageReader = ImageReader.newInstance( + mRepeatingRequestImageReader.getWidth(), + mRepeatingRequestImageReader.getHeight(), + // Camera devices accept only Jpeg output if the image processor is null + ImageFormat.JPEG, 1); + mCameraBurstSurface = mBurstCaptureImageReader.getSurface(); + } + } + } + + /** + * @hide + */ + public synchronized void initialize() throws CameraAccessException, RemoteException { + if (mInitialized) { + Log.d(TAG, + "Session already initialized"); + return; + } + + ArrayList sessionParamsList = new ArrayList<>(); + ArrayList outputList = new ArrayList<>(); + initializeRepeatingRequestPipeline(); + outputList.add(new OutputConfiguration(mCameraRepeatingSurface)); + CaptureStageImpl previewSessionParams = mPreviewExtender.onPresetSession(); + if (previewSessionParams != null) { + sessionParamsList.add(previewSessionParams); + } + initializeBurstCapturePipeline(); + outputList.add(new OutputConfiguration(mCameraBurstSurface)); + CaptureStageImpl stillCaptureSessionParams = mImageExtender.onPresetSession(); + if (stillCaptureSessionParams != null) { + sessionParamsList.add(stillCaptureSessionParams); + } + + SessionConfiguration sessionConfig = new SessionConfiguration( + SessionConfiguration.SESSION_REGULAR, + outputList, + new HandlerExecutor(mHandler), + new SessionStateHandler()); + + if (!sessionParamsList.isEmpty()) { + CaptureRequest sessionParamRequest = createRequest(mCameraDevice, sessionParamsList, + null, CameraDevice.TEMPLATE_PREVIEW); + sessionConfig.setSessionParameters(sessionParamRequest); + } + + mCameraDevice.createCaptureSession(sessionConfig); + } + + @Override + public @NonNull CameraDevice getDevice() { + synchronized (mInterfaceLock) { + return mCameraDevice; + } + } + + @Override + public int setRepeatingRequest(@NonNull CaptureRequest request, + @NonNull Executor executor, + @NonNull ExtensionCaptureCallback listener) + throws CameraAccessException { + synchronized (mInterfaceLock) { + if (!mInitialized) { + throw new IllegalStateException("Uninitialized component"); + } + + if (mClientRepeatingRequestSurface == null) { + throw new IllegalArgumentException("No registered preview surface"); + } + + if (!request.containsTarget(mClientRepeatingRequestSurface) || + (request.getTargets().size() != 1)) { + throw new IllegalArgumentException("Invalid repeating request output target!"); + } + + mInternalRepeatingRequestEnabled = false; + try { + return setRepeatingRequest(mPreviewExtender.getCaptureStage(), + new RepeatingRequestHandler(request, executor, listener)); + } catch (RemoteException e) { + Log.e(TAG, "Failed to set repeating request! Extension service does not " + + "respond"); + throw new CameraAccessException(CameraAccessException.CAMERA_ERROR); + } + } + } + + private ArrayList compileInitialRequestList() { + ArrayList captureStageList = new ArrayList<>(); + try { + CaptureStageImpl initialPreviewParams = mPreviewExtender.onEnableSession(); + if (initialPreviewParams != null) { + captureStageList.add(initialPreviewParams); + } + + CaptureStageImpl initialStillCaptureParams = mImageExtender.onEnableSession(); + if (initialStillCaptureParams != null) { + captureStageList.add(initialStillCaptureParams); + } + } catch (RemoteException e) { + Log.e(TAG, "Failed to initialize session parameters! Extension service does not" + + " respond!"); + } + + return captureStageList; + } + + private static List createBurstRequest(CameraDevice cameraDevice, + List captureStageList, CaptureRequest clientRequest, + Surface target, int captureTemplate, Map captureMap) { + CaptureRequest.Builder requestBuilder; + ArrayList ret = new ArrayList<>(); + for (CaptureStageImpl captureStage : captureStageList) { + try { + requestBuilder = cameraDevice.createCaptureRequest(captureTemplate); + } catch (CameraAccessException e) { + return null; + } + + // Set user supported jpeg quality and rotation parameters + Integer jpegRotation = clientRequest.get(CaptureRequest.JPEG_ORIENTATION); + if (jpegRotation != null) { + requestBuilder.set(CaptureRequest.JPEG_ORIENTATION, jpegRotation); + } + Byte jpegQuality = clientRequest.get(CaptureRequest.JPEG_QUALITY); + if (jpegQuality != null) { + requestBuilder.set(CaptureRequest.JPEG_QUALITY, jpegQuality); + } + + requestBuilder.addTarget(target); + CaptureRequest request = requestBuilder.build(); + CameraMetadataNative.update(request.getNativeMetadata(), captureStage.parameters); + ret.add(request); + captureMap.put(request, captureStage.id); + } + + return ret; + } + + private static CaptureRequest createRequest(CameraDevice cameraDevice, + List captureStageList, + Surface target, + int captureTemplate) throws CameraAccessException { + CaptureRequest.Builder requestBuilder; + requestBuilder = cameraDevice.createCaptureRequest(captureTemplate); + if (target != null) { + requestBuilder.addTarget(target); + } + + CaptureRequest ret = requestBuilder.build(); + for (CaptureStageImpl captureStage : captureStageList) { + if (captureStage != null) { + CameraMetadataNative.update(ret.getNativeMetadata(), captureStage.parameters); + } + } + return ret; + } + + @Override + public int capture(@NonNull CaptureRequest request, + @NonNull Executor executor, + @NonNull ExtensionCaptureCallback listener) throws CameraAccessException { + if (!mInitialized) { + throw new IllegalStateException("Uninitialized component"); + } + + if (mClientCaptureSurface == null) { + throw new IllegalArgumentException("No output surface registered for single requests!"); + } + + if (!request.containsTarget(mClientCaptureSurface) || (request.getTargets().size() != 1)) { + throw new IllegalArgumentException("Invalid single capture output target!"); + } + + HashMap requestMap = new HashMap<>(); + List burstRequest; + try { + burstRequest = createBurstRequest(mCameraDevice, + mImageExtender.getCaptureStages(), request, mCameraBurstSurface, + CameraDevice.TEMPLATE_STILL_CAPTURE, requestMap); + } catch (RemoteException e) { + Log.e(TAG, "Failed to initialize internal burst request! Extension service does" + + " not respond!"); + throw new CameraAccessException(CameraAccessException.CAMERA_ERROR); + } + if (burstRequest == null) { + throw new UnsupportedOperationException("Failed to create still capture burst request"); + } + + return mCaptureSession.captureBurstRequests(burstRequest, new HandlerExecutor(mHandler), + new BurstRequestHandler(request, executor, listener, requestMap)); + } + + @Override + public void stopRepeating() throws CameraAccessException { + synchronized (mInterfaceLock) { + if (!mInitialized) { + throw new IllegalStateException("Uninitialized component"); + } + + mInternalRepeatingRequestEnabled = true; + mCaptureSession.stopRepeating(); + } + } + + @Override + public void close() throws CameraAccessException { + synchronized (mInterfaceLock) { + if (mInitialized) { + mInternalRepeatingRequestEnabled = false; + mCaptureSession.stopRepeating(); + + ArrayList captureStageList = new ArrayList<>(); + try { + CaptureStageImpl disableParams = mPreviewExtender.onDisableSession(); + if (disableParams != null) { + captureStageList.add(disableParams); + } + + CaptureStageImpl disableStillCaptureParams = + mImageExtender.onDisableSession(); + if (disableStillCaptureParams != null) { + captureStageList.add(disableStillCaptureParams); + } + } catch (RemoteException e) { + Log.e(TAG, "Failed to disable extension! Extension service does not " + + "respond!"); + } + if (!captureStageList.isEmpty()) { + CaptureRequest disableRequest = createRequest(mCameraDevice, captureStageList, + mCameraRepeatingSurface, CameraDevice.TEMPLATE_PREVIEW); + mCaptureSession.capture(disableRequest, new CloseRequestHandler(), mHandler); + } + + mCaptureSession.close(); + } + } + } + + private void setInitialCaptureRequest(List captureStageList, + InitialRequestHandler requestHandler) + throws CameraAccessException { + CaptureRequest initialRequest = createRequest(mCameraDevice, + captureStageList, mCameraRepeatingSurface, CameraDevice.TEMPLATE_PREVIEW); + mCaptureSession.capture(initialRequest, requestHandler, mHandler); + } + + private int setRepeatingRequest(CaptureStageImpl captureStage, + CameraCaptureSession.CaptureCallback requestHandler) + throws CameraAccessException { + ArrayList captureStageList = new ArrayList<>(); + captureStageList.add(captureStage); + CaptureRequest repeatingRequest = createRequest(mCameraDevice, + captureStageList, mCameraRepeatingSurface, CameraDevice.TEMPLATE_PREVIEW); + return mCaptureSession.setSingleRepeatingRequest(repeatingRequest, + new HandlerExecutor(mHandler), requestHandler); + } + + /** @hide */ + public void release() { + synchronized (mInterfaceLock) { + mInternalRepeatingRequestEnabled = false; + mInitialized = false; + mHandlerThread.quitSafely(); + + try { + mPreviewExtender.onDeInit(); + mImageExtender.onDeInit(); + } catch (RemoteException e) { + Log.e(TAG, "Failed to release extensions! Extension service does not" + + " respond!"); + } + + if (mExtensionClientId >= 0) { + CameraExtensionCharacteristics.unregisterClient(mExtensionClientId); + } + + if (mRepeatingRequestImageReader != null) { + mRepeatingRequestImageReader.close(); + mRepeatingRequestImageReader = null; + } + + if (mBurstCaptureImageReader != null) { + mBurstCaptureImageReader.close(); + mBurstCaptureImageReader = null; + } + + if (mStubCaptureImageReader != null) { + mStubCaptureImageReader.close(); + mStubCaptureImageReader = null; + } + + if (mRepeatingRequestImageWriter != null) { + mRepeatingRequestImageWriter.close(); + mRepeatingRequestImageWriter = null; + } + + if (mPreviewImageProcessor != null) { + mPreviewImageProcessor.close(); + mPreviewImageProcessor = null; + } + + mCaptureSession = null; + mImageProcessor = null; + mCameraRepeatingSurface = mClientRepeatingRequestSurface = null; + mCameraBurstSurface = mClientCaptureSurface = null; + } + } + + private void notifyConfigurationFailure() { + synchronized (mInterfaceLock) { + if (mInitialized) { + return; + } + } + + release(); + + final long ident = Binder.clearCallingIdentity(); + try { + mExecutor.execute( + () -> mCallbacks.onConfigureFailed(CameraExtensionSessionImpl.this)); + } finally { + Binder.restoreCallingIdentity(ident); + } + } + + private void notifyConfigurationSuccess() { + synchronized (mInterfaceLock) { + if (mInitialized) { + return; + } else { + mInitialized = true; + } + } + + final long ident = Binder.clearCallingIdentity(); + try { + mExecutor.execute(() -> mCallbacks.onConfigured(CameraExtensionSessionImpl.this)); + } finally { + Binder.restoreCallingIdentity(ident); + } + } + + private class SessionStateHandler extends + android.hardware.camera2.CameraCaptureSession.StateCallback { + @Override + public void onClosed(@NonNull CameraCaptureSession session) { + release(); + + final long ident = Binder.clearCallingIdentity(); + try { + mExecutor.execute(() -> mCallbacks.onClosed(CameraExtensionSessionImpl.this)); + } finally { + Binder.restoreCallingIdentity(ident); + } + } + + @Override + public void onConfigureFailed(@NonNull CameraCaptureSession session) { + notifyConfigurationFailure(); + } + + @Override + public void onConfigured(@NonNull CameraCaptureSession session) { + boolean status = true; + synchronized (mInterfaceLock) { + mCaptureSession = session; + + ArrayList initialRequestList = compileInitialRequestList(); + if (!initialRequestList.isEmpty()) { + try { + setInitialCaptureRequest(initialRequestList, new InitialRequestHandler()); + } catch (CameraAccessException e) { + Log.e(TAG, "Failed to initialize the initial capture request!"); + status = false; + } + } else { + try { + setRepeatingRequest(mPreviewExtender.getCaptureStage(), + new RepeatingRequestHandler(null, null, null)); + } catch (CameraAccessException | RemoteException e) { + Log.e(TAG, "Failed to initialize internal repeating request!"); + status = false; + } + + } + } + + if (!status) { + notifyConfigurationFailure(); + } + } + } + + private class BurstRequestHandler extends CameraCaptureSession.CaptureCallback { + private final Executor mExecutor; + private final ExtensionCaptureCallback mCallbacks; + private final CaptureRequest mClientRequest; + private final HashMap mCaptureRequestMap; + + private HashMap> mCaptureStageMap = + new HashMap<>(); + private LongSparseArray> mCapturePendingMap = + new LongSparseArray<>(); + + private ImageCallback mImageCallback = null; + private boolean mCaptureFailed = false; + + public BurstRequestHandler(@NonNull CaptureRequest request, @NonNull Executor executor, + @NonNull ExtensionCaptureCallback callbacks, + @NonNull HashMap requestMap) { + mClientRequest = request; + mExecutor = executor; + mCallbacks = callbacks; + mCaptureRequestMap = requestMap; + } + + private void notifyCaptureFailed() { + if (!mCaptureFailed) { + mCaptureFailed = true; + + final long ident = Binder.clearCallingIdentity(); + try { + mExecutor.execute( + () -> mCallbacks.onCaptureFailed(CameraExtensionSessionImpl.this, + mClientRequest)); + } finally { + Binder.restoreCallingIdentity(ident); + } + } + } + + @Override + public void onCaptureStarted(@NonNull CameraCaptureSession session, + @NonNull CaptureRequest request, + long timestamp, + long frameNumber) { + // Trigger the client callback only once in case of burst request + boolean initialCallback = false; + synchronized (mInterfaceLock) { + if ((mImageProcessor != null) && (mImageCallback == null)) { + mImageCallback = new ImageCallback(mBurstCaptureImageReader); + mBurstCaptureImageReader.setOnImageAvailableListener(mImageCallback, mHandler); + initialCallback = true; + } else if (mImageProcessor == null) { + // No burst expected in this case + initialCallback = true; + } + } + + if (initialCallback) { + final long ident = Binder.clearCallingIdentity(); + try { + mExecutor.execute( + () -> mCallbacks.onCaptureStarted(CameraExtensionSessionImpl.this, + mClientRequest, timestamp)); + } finally { + Binder.restoreCallingIdentity(ident); + } + } + } + + @Override + public void onCaptureBufferLost(@NonNull CameraCaptureSession session, + @NonNull CaptureRequest request, + @NonNull Surface target, long frameNumber) { + notifyCaptureFailed(); + } + + @Override + public void onCaptureFailed(@NonNull CameraCaptureSession session, + @NonNull CaptureRequest request, + @NonNull CaptureFailure failure) { + notifyCaptureFailed(); + } + + @Override + public void onCaptureSequenceAborted(@NonNull CameraCaptureSession session, + int sequenceId) { + final long ident = Binder.clearCallingIdentity(); + try { + mExecutor.execute( + () -> mCallbacks.onCaptureSequenceAborted(CameraExtensionSessionImpl.this, + sequenceId)); + } finally { + Binder.restoreCallingIdentity(ident); + } + } + + @Override + public void onCaptureSequenceCompleted(@NonNull CameraCaptureSession session, + int sequenceId, + long frameNumber) { + final long ident = Binder.clearCallingIdentity(); + try { + mExecutor.execute( + () -> mCallbacks + .onCaptureSequenceCompleted(CameraExtensionSessionImpl.this, + sequenceId)); + } finally { + Binder.restoreCallingIdentity(ident); + } + } + + @Override + public void onCaptureCompleted(@NonNull CameraCaptureSession session, + @NonNull CaptureRequest request, + @NonNull TotalCaptureResult result) { + if (!mCaptureRequestMap.containsKey(request)) { + Log.e(TAG, + "Unexpected still capture request received!"); + return; + } + Integer stageId = mCaptureRequestMap.get(request); + + Long timestamp = result.get(CaptureResult.SENSOR_TIMESTAMP); + if (timestamp != null) { + if (mImageProcessor != null) { + if (mCapturePendingMap.indexOfKey(timestamp) >= 0) { + Image img = mCapturePendingMap.get(timestamp).first; + mCaptureStageMap.put(stageId, + new Pair<>(img, + result)); + checkAndFireBurstProcessing(); + } else { + mCapturePendingMap.put(timestamp, + new Pair<>(null, + stageId)); + mCaptureStageMap.put(stageId, + new Pair<>(null, + result)); + } + } else { + mCaptureRequestMap.clear(); + final long ident = Binder.clearCallingIdentity(); + try { + mExecutor.execute( + () -> mCallbacks + .onCaptureProcessStarted(CameraExtensionSessionImpl.this, + mClientRequest)); + } finally { + Binder.restoreCallingIdentity(ident); + } + } + } else { + Log.e(TAG, + "Capture result without valid sensor timestamp!"); + } + } + + private void checkAndFireBurstProcessing() { + if (mCaptureRequestMap.size() == mCaptureStageMap.size()) { + for (Pair captureStage : mCaptureStageMap + .values()) { + if ((captureStage.first == null) || (captureStage.second == null)) { + return; + } + } + + mCaptureRequestMap.clear(); + mCapturePendingMap.clear(); + boolean processStatus = true; + List captureList = initializeParcelable(mCaptureStageMap); + try { + mImageProcessor.process(captureList); + } catch (RemoteException e) { + Log.e(TAG, "Failed to process multi-frame request! Extension service " + + "does not respond!"); + processStatus = false; + } + + for (CaptureBundle bundle : captureList) { + bundle.captureImage.buffer.close(); + } + captureList.clear(); + for (Pair captureStage : mCaptureStageMap.values()) { + captureStage.first.close(); + } + mCaptureStageMap.clear(); + + final long ident = Binder.clearCallingIdentity(); + try { + if (processStatus) { + mExecutor.execute(() -> mCallbacks.onCaptureProcessStarted( + CameraExtensionSessionImpl.this, mClientRequest)); + } else { + mExecutor.execute(() -> mCallbacks.onCaptureFailed( + CameraExtensionSessionImpl.this, mClientRequest)); + } + } finally { + Binder.restoreCallingIdentity(ident); + } + } + } + + private class ImageCallback implements ImageReader.OnImageAvailableListener { + public ImageCallback(@NonNull ImageReader reader) { + //Check for any pending buffers + onImageAvailable(reader); + } + + @Override + public void onImageAvailable(ImageReader reader) { + Image img; + try { + while ((!mCaptureRequestMap.isEmpty()) && + (img = reader.acquireNextImage()) != null) { + long timestamp = img.getTimestamp(); + reader.detachImage(img); + if (mCapturePendingMap.indexOfKey(timestamp) >= 0) { + Integer stageId = mCapturePendingMap.get(timestamp).second; + Pair captureStage = + mCaptureStageMap.get(stageId); + if (captureStage != null) { + mCaptureStageMap.put(stageId, + new Pair<>(img, + captureStage.second)); + checkAndFireBurstProcessing(); + } else { + Log.e(TAG, + "Capture stage: " + + mCapturePendingMap.get(timestamp).second + + " is absent!"); + } + } else { + mCapturePendingMap.put(timestamp, + new Pair<>(img, + -1)); + } + } + } catch (IllegalStateException e) { + // This is possible in case the maximum number of images is acquired. + } + } + } + } + + private class ImageLoopbackCallback implements ImageReader.OnImageAvailableListener { + @Override public void onImageAvailable(ImageReader reader) { + Image img; + try { + img = reader.acquireNextImage(); + } catch (IllegalStateException e) { + Log.e(TAG, "Failed to acquire and loopback image!"); + return; + } + if (img == null) { + Log.e(TAG, + "Invalid image!"); + return; + } + img.close(); + } + } + + private class InitialRequestHandler extends CameraCaptureSession.CaptureCallback { + @Override + public void onCaptureSequenceAborted(@NonNull CameraCaptureSession session, + int sequenceId) { + Log.e(TAG, "Initial capture request aborted!"); + notifyConfigurationFailure(); + } + + @Override + public void onCaptureFailed(@NonNull CameraCaptureSession session, + @NonNull CaptureRequest request, + @NonNull CaptureFailure failure) { + Log.e(TAG, "Initial capture request failed!"); + notifyConfigurationFailure(); + } + + @Override + public void onCaptureSequenceCompleted(@NonNull CameraCaptureSession session, + int sequenceId, + long frameNumber) { + boolean status = true; + synchronized (mInterfaceLock) { + /** + * Initialize and set the initial repeating request which will execute in the + * absence of client repeating requests. + */ + try { + setRepeatingRequest(mPreviewExtender.getCaptureStage(), + new RepeatingRequestHandler(null, null, null)); + } catch (CameraAccessException | RemoteException e) { + Log.e(TAG, "Failed to start the internal repeating request!"); + status = false; + } + + } + + if (!status) { + notifyConfigurationFailure(); + } + } + } + + private class CloseRequestHandler extends CameraCaptureSession.CaptureCallback { + @Override + public void onCaptureStarted(@NonNull CameraCaptureSession session, + @NonNull CaptureRequest request, + long timestamp, + long frameNumber) { + synchronized (mInterfaceLock) { + mRepeatingRequestImageReader + .setOnImageAvailableListener(new ImageLoopbackCallback(), mHandler); + } + } + } + + // This handler can operate in two modes: + // 1) Using valid client callbacks, which means camera buffers will be propagated the + // registered output surfaces and clients will be notified accordingly. + // 2) Without any client callbacks where an internal repeating request is kept active + // to satisfy the extensions continuous preview/(repeating request) requirement. + private class RepeatingRequestHandler extends CameraCaptureSession.CaptureCallback { + private final Executor mExecutor; + private final ExtensionCaptureCallback mCallbacks; + private final CaptureRequest mClientRequest; + private final boolean mClientNotificationsEnabled; + private ImageReader.OnImageAvailableListener mImageCallback = null; + private LongSparseArray> mPendingResultMap = + new LongSparseArray<>(); + + private boolean mRequestUpdatedNeeded = false; + + public RepeatingRequestHandler(@Nullable CaptureRequest clientRequest, + @Nullable Executor executor, + @Nullable ExtensionCaptureCallback listener) { + mClientRequest = clientRequest; + mExecutor = executor; + mCallbacks = listener; + mClientNotificationsEnabled = + (mClientRequest != null) && (mExecutor != null) && (mCallbacks != null); + } + + @Override + public void onCaptureStarted(@NonNull CameraCaptureSession session, + @NonNull CaptureRequest request, + long timestamp, + long frameNumber) { + synchronized (mInterfaceLock) { + // Setup the image callback handler for this repeating request just once + // after streaming resumes. + if (mImageCallback == null) { + if (mPreviewProcessorType == + IPreviewExtenderImpl.PROCESSOR_TYPE_IMAGE_PROCESSOR) { + if (mClientNotificationsEnabled) { + mPreviewImageProcessor.onOutputSurface(mClientRepeatingRequestSurface, + nativeGetSurfaceFormat(mClientRepeatingRequestSurface)); + } else { + mPreviewImageProcessor.onOutputSurface(null, -1); + } + mImageCallback = new ImageProcessCallback(); + } else { + mImageCallback = mClientNotificationsEnabled ? + new ImageForwardCallback(mRepeatingRequestImageWriter) : + new ImageLoopbackCallback(); + } + mRepeatingRequestImageReader + .setOnImageAvailableListener(mImageCallback, mHandler); + } + } + + if (mClientNotificationsEnabled) { + final long ident = Binder.clearCallingIdentity(); + try { + mExecutor.execute( + () -> mCallbacks.onCaptureStarted(CameraExtensionSessionImpl.this, + mClientRequest, timestamp)); + } finally { + Binder.restoreCallingIdentity(ident); + } + } + } + + @Override + public void onCaptureSequenceAborted(@NonNull CameraCaptureSession session, + int sequenceId) { + synchronized (mInterfaceLock) { + if (mInternalRepeatingRequestEnabled) { + resumeInternalRepeatingRequest(true); + } + } + + if (mClientNotificationsEnabled) { + final long ident = Binder.clearCallingIdentity(); + try { + mExecutor.execute( + () -> mCallbacks + .onCaptureSequenceAborted(CameraExtensionSessionImpl.this, + sequenceId)); + } finally { + Binder.restoreCallingIdentity(ident); + } + } else { + notifyConfigurationFailure(); + } + + } + + @Override + public void onCaptureSequenceCompleted(@NonNull CameraCaptureSession session, + int sequenceId, + long frameNumber) { + + synchronized (mInterfaceLock) { + if (mRequestUpdatedNeeded) { + mRequestUpdatedNeeded = false; + resumeInternalRepeatingRequest(false); + } else if (mInternalRepeatingRequestEnabled) { + resumeInternalRepeatingRequest(true); + } + } + + if (mClientNotificationsEnabled) { + final long ident = Binder.clearCallingIdentity(); + try { + mExecutor.execute( + () -> mCallbacks + .onCaptureSequenceCompleted(CameraExtensionSessionImpl.this, + sequenceId)); + } finally { + Binder.restoreCallingIdentity(ident); + } + } + } + + @Override + public void onCaptureFailed(@NonNull CameraCaptureSession session, + @NonNull CaptureRequest request, + @NonNull CaptureFailure failure) { + + if (mClientNotificationsEnabled) { + final long ident = Binder.clearCallingIdentity(); + try { + mExecutor.execute( + () -> mCallbacks.onCaptureFailed(CameraExtensionSessionImpl.this, + mClientRequest)); + } finally { + Binder.restoreCallingIdentity(ident); + } + } + } + + @Override + public void onCaptureCompleted(@NonNull CameraCaptureSession session, + @NonNull CaptureRequest request, + @NonNull TotalCaptureResult result) { + boolean notifyClient = mClientNotificationsEnabled; + boolean processStatus = true; + + synchronized (mInterfaceLock) { + final Long timestamp = result.get(CaptureResult.SENSOR_TIMESTAMP); + if (timestamp != null) { + if (mPreviewProcessorType == + IPreviewExtenderImpl.PROCESSOR_TYPE_REQUEST_UPDATE_ONLY) { + CaptureStageImpl captureStage = null; + try { + captureStage = mPreviewRequestUpdateProcessor.process( + result.getNativeMetadata(), result.getSequenceId()); + } catch (RemoteException e) { + Log.e(TAG, "Extension service does not respond during " + + "processing!"); + } + if (captureStage != null) { + try { + setRepeatingRequest(captureStage, this); + mRequestUpdatedNeeded = true; + } catch (IllegalStateException e) { + // This is possible in case the camera device closes and the + // and the callback here is executed before the onClosed + // notification. + } catch (CameraAccessException e) { + Log.e(TAG, "Failed to update repeating request settings!"); + } + } else { + mRequestUpdatedNeeded = false; + } + } else if (mPreviewProcessorType == + IPreviewExtenderImpl.PROCESSOR_TYPE_IMAGE_PROCESSOR) { + int idx = mPendingResultMap.indexOfKey(timestamp); + if (idx >= 0) { + ParcelImage parcelImage = initializeParcelImage( + mPendingResultMap.get(timestamp).first); + try { + mPreviewImageProcessor.process(parcelImage, result); + } catch (RemoteException e) { + processStatus = false; + Log.e(TAG, "Extension service does not respond during " + + "processing, dropping frame!"); + } catch (RuntimeException e) { + // Runtime exceptions can happen in a few obscure cases where the + // client tries to initialize a new capture session while this + // session is still ongoing. In such scenario, the camera will + // disconnect from the intermediate output surface, which will + // invalidate the images that we acquired previously. This can + // happen before we get notified via "onClosed" so there aren't + // many options to avoid the exception. + processStatus = false; + Log.e(TAG, "Runtime exception encountered during buffer " + + "processing, dropping frame!"); + } finally { + parcelImage.buffer.close(); + mPendingResultMap.get(timestamp).first.close(); + } + discardPendingRepeatingResults(idx, mPendingResultMap, false); + } else { + notifyClient = false; + mPendingResultMap.put(timestamp, + new Pair<>(null, + result)); + } + } else { + // No special handling for PROCESSOR_TYPE_NONE + } + if (notifyClient) { + final long ident = Binder.clearCallingIdentity(); + try { + if (processStatus) { + mExecutor.execute(() -> mCallbacks + .onCaptureProcessStarted(CameraExtensionSessionImpl.this, + mClientRequest)); + } else { + mExecutor.execute( + () -> mCallbacks + .onCaptureFailed(CameraExtensionSessionImpl.this, + mClientRequest)); + } + } finally { + Binder.restoreCallingIdentity(ident); + } + } + } else { + Log.e(TAG, + "Result without valid sensor timestamp!"); + } + } + + if (!notifyClient) { + notifyConfigurationSuccess(); + } + } + + private void resumeInternalRepeatingRequest(boolean internal) { + try { + if (internal) { + setRepeatingRequest(mPreviewExtender.getCaptureStage(), + new RepeatingRequestHandler(null, null, null)); + } else { + setRepeatingRequest(mPreviewExtender.getCaptureStage(), this); + } + } catch (RemoteException e) { + Log.e(TAG, "Failed to resume internal repeating request, extension service" + + " fails to respond!"); + } catch (IllegalStateException e) { + // This is possible in case we try to resume before the state "onClosed" + // notification is able to reach us. + Log.w(TAG, "Failed to resume internal repeating request!"); + } catch (CameraAccessException e) { + Log.e(TAG, "Failed to resume internal repeating request!"); + } + } + + // Find the timestamp of the oldest pending buffer + private Long calculatePruneThreshold( + LongSparseArray> previewMap) { + long oldestTimestamp = Long.MAX_VALUE; + for (int idx = 0; idx < previewMap.size(); idx++) { + Pair entry = previewMap.valueAt(idx); + long timestamp = previewMap.keyAt(idx); + if ((entry.first != null) && (timestamp < oldestTimestamp)) { + oldestTimestamp = timestamp; + } + } + return (oldestTimestamp == Long.MAX_VALUE) ? 0 : oldestTimestamp; + } + + private void discardPendingRepeatingResults(int idx, LongSparseArray> previewMap, boolean notifyCurrentIndex) { + if (idx < 0) { + return; + } + for (int i = idx; i >= 0; i--) { + if (previewMap.valueAt(i).first != null) { + Log.w(TAG, "Discard pending buffer with timestamp: " + previewMap.keyAt(i)); + previewMap.valueAt(i).first.close(); + } else { + Log.w(TAG, "Discard pending result with timestamp: " + previewMap.keyAt(i)); + if (mClientNotificationsEnabled && ((i != idx) || notifyCurrentIndex)) { + Log.w(TAG, "Preview frame drop with timestamp: " + previewMap.keyAt(i)); + final long ident = Binder.clearCallingIdentity(); + try { + mExecutor.execute( + () -> mCallbacks + .onCaptureFailed(CameraExtensionSessionImpl.this, + mClientRequest)); + } finally { + Binder.restoreCallingIdentity(ident); + } + } + } + previewMap.removeAt(i); + } + } + + private class ImageForwardCallback implements ImageReader.OnImageAvailableListener { + private final ImageWriter mOutputWriter; + + public ImageForwardCallback(@NonNull ImageWriter imageWriter) { + mOutputWriter = imageWriter; + } + + @Override public void onImageAvailable(ImageReader reader) { + Image img; + try { + img = reader.acquireNextImage(); + } catch (IllegalStateException e) { + Log.e(TAG, "Failed to acquire and propagate repeating request image!"); + return; + } + if (img == null) { + Log.e(TAG, + "Invalid image!"); + return; + } + try { + mOutputWriter.queueInputImage(img); + } catch (IllegalStateException e) { + // This is possible in case the client disconnects from the output surface + // abruptly. + Log.w(TAG, "Output surface likely abandoned, dropping buffer!"); + img.close(); + } + } + } + + private class ImageProcessCallback implements ImageReader.OnImageAvailableListener { + @Override + public void onImageAvailable(ImageReader reader) { + Image img; + try { + img = reader.acquireNextImage(); + } catch (IllegalStateException e) { + // We reached the maximum acquired images limit. This is possible in case we + // have capture failures that result in absent or missing capture results. In + // such scenario we can prune the oldest pending buffer. + discardPendingRepeatingResults( + mPendingResultMap + .indexOfKey(calculatePruneThreshold(mPendingResultMap)), + mPendingResultMap, true); + + img = reader.acquireNextImage(); + } + if (img == null) { + Log.e(TAG, + "Invalid preview buffer!"); + return; + } + + try { + reader.detachImage(img); + } catch (Exception e) { + Log.e(TAG, + "Failed to detach image!"); + img.close(); + return; + } + + long timestamp = img.getTimestamp(); + int idx = mPendingResultMap.indexOfKey(timestamp); + if (idx >= 0) { + boolean processStatus = true; + ParcelImage parcelImage = initializeParcelImage(img); + try { + mPreviewImageProcessor.process(parcelImage, + mPendingResultMap.get(timestamp).second); + } catch (RemoteException e) { + processStatus = false; + Log.e(TAG, "Extension service does not respond during " + + "processing, dropping frame!"); + } finally { + parcelImage.buffer.close(); + img.close(); + } + discardPendingRepeatingResults(idx, mPendingResultMap, false); + if (mClientNotificationsEnabled) { + final long ident = Binder.clearCallingIdentity(); + try { + if (processStatus) { + mExecutor.execute(() -> mCallbacks.onCaptureProcessStarted( + CameraExtensionSessionImpl.this, + mClientRequest)); + } else { + mExecutor.execute(() -> mCallbacks.onCaptureFailed( + CameraExtensionSessionImpl.this, + mClientRequest)); + } + } finally { + Binder.restoreCallingIdentity(ident); + } + } + } else { + mPendingResultMap.put(timestamp, new Pair<>(img, null)); + } + } + } + } + + private static Size findSmallestAspectMatchedSize(@NonNull List sizes, + @NonNull Size arSize) { + final float TOLL = .01f; + + if (arSize.getHeight() == 0) { + throw new IllegalArgumentException("Invalid input aspect ratio"); + } + + float targetAR = ((float) arSize.getWidth()) / arSize.getHeight(); + Size ret = null; + Size fallbackSize = null; + for (Size sz : sizes) { + if (fallbackSize == null) { + fallbackSize = sz; + } + if ((sz.getHeight() > 0) && + ((ret == null) || + (ret.getWidth() * ret.getHeight()) < + (sz.getWidth() * sz.getHeight()))) { + float currentAR = ((float) sz.getWidth()) / sz.getHeight(); + if (Math.abs(currentAR - targetAR) <= TOLL) { + ret = sz; + } + } + } + if (ret == null) { + Log.e(TAG, "AR matched size not found returning first size in list"); + ret = fallbackSize; + } + + return ret; + } + + private final class HandlerExecutor implements Executor { + private final Handler mHandler; + + public HandlerExecutor(Handler handler) { + mHandler = handler; + } + + @Override + public void execute(Runnable runCmd) { + try { + mHandler.post(runCmd); + } catch (RejectedExecutionException e) { + Log.w(TAG, "Handler thread unavailable, skipping message!"); + } + } + } + + private static ParcelImage initializeParcelImage(Image img) { + ParcelImage parcelImage = new ParcelImage(); + parcelImage.buffer = img.getHardwareBuffer(); + if (img.getFenceFd() >= 0) { + try { + parcelImage.fence = ParcelFileDescriptor.fromFd(img.getFenceFd()); + } catch (IOException e) { + Log.e(TAG,"Failed to parcel buffer fence!"); + } + } + parcelImage.format = img.getFormat(); + parcelImage.timestamp = img.getTimestamp(); + parcelImage.transform = img.getTransform(); + parcelImage.scalingMode = img.getScalingMode(); + parcelImage.planeCount = img.getPlaneCount(); + parcelImage.crop = img.getCropRect(); + + return parcelImage; + } + + private static List initializeParcelable( + HashMap> captureMap) { + ArrayList ret = new ArrayList<>(); + for (Integer stagetId : captureMap.keySet()) { + Pair entry = captureMap.get(stagetId); + CaptureBundle bundle = new CaptureBundle(); + bundle.stage = stagetId; + bundle.captureImage = initializeParcelImage(entry.first); + bundle.sequenceId = entry.second.getSequenceId(); + bundle.captureResult = entry.second.getNativeMetadata(); + ret.add(bundle); + } + + return ret; + } +} diff --git a/core/java/android/hardware/camera2/impl/CameraMetadataNative.java b/core/java/android/hardware/camera2/impl/CameraMetadataNative.java index 5cc7bf8cefed8..e9bae0b621ce1 100644 --- a/core/java/android/hardware/camera2/impl/CameraMetadataNative.java +++ b/core/java/android/hardware/camera2/impl/CameraMetadataNative.java @@ -382,6 +382,19 @@ public class CameraMetadataNative implements Parcelable { return newObject; } + /** + * Set all metadata values in the destination argument by using the corresponding + * values from the source. Metadata tags present in the destination and absent + * from the source will remain unmodified. + * + * @param dst Destination metadata + * @param src Source metadata + * @hide + */ + public static void update(CameraMetadataNative dst, CameraMetadataNative src) { + nativeUpdate(dst.mMetadataPtr, src.mMetadataPtr); + } + public static final @android.annotation.NonNull Parcelable.Creator CREATOR = new Parcelable.Creator() { @Override @@ -1719,6 +1732,9 @@ public class CameraMetadataNative implements Parcelable { private static native long nativeAllocateCopy(long ptr) throws NullPointerException; + + @FastNative + private static native void nativeUpdate(long dst, long src); private static synchronized native void nativeWriteToParcel(Parcel dest, long ptr); private static synchronized native void nativeReadFromParcel(Parcel source, long ptr); private static synchronized native void nativeSwap(long ptr, long otherPtr) diff --git a/core/java/android/hardware/camera2/params/ExtensionSessionConfiguration.java b/core/java/android/hardware/camera2/params/ExtensionSessionConfiguration.java new file mode 100644 index 0000000000000..c81d339578cf6 --- /dev/null +++ b/core/java/android/hardware/camera2/params/ExtensionSessionConfiguration.java @@ -0,0 +1,97 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package android.hardware.camera2.params; + +import android.annotation.NonNull; + +import android.hardware.camera2.CameraExtensionCharacteristics.Extension; +import android.hardware.camera2.CameraExtensionSession; + +import java.util.List; +import java.util.concurrent.Executor; + +/** + * A class that aggregates all supported arguments for + * {@link CameraExtensionSession} initialization. + */ +public final class ExtensionSessionConfiguration { + private static final String TAG = "ExtensionSessionConfiguration"; + + private int mExtensionType; + private List mOutputs; + private Executor mExecutor = null; + private CameraExtensionSession.StateCallback mCallback = null; + + /** + * Create a new ExtensionSessionConfiguration + * + * @param extension to be used for processing + * @param outputs a list of output configurations for the capture session + * @param executor the executor which will be used for invoking the callbacks + * @param listener callbacks to be invoked when the state of the + * CameraExtensionSession changes + */ + public ExtensionSessionConfiguration(@Extension int extension, + @NonNull List outputs, @NonNull Executor executor, + @NonNull CameraExtensionSession.StateCallback listener) { + mExtensionType = extension; + mOutputs = outputs; + mExecutor = executor; + mCallback = listener; + } + + /** + * Retrieve the extension type. + * + * @return the extension type. + */ + public @Extension + int getExtension() { + return mExtensionType; + } + + /** + * Retrieve the {@link OutputConfiguration} list for the capture + * session. + * + * @return A list of output configurations for the capture session. + */ + public @NonNull + List getOutputConfigurations() { + return mOutputs; + } + + /** + * Retrieve the CameraCaptureSession.StateCallback + * listener. + * + * @return A state callback interface implementation. + */ + public @NonNull + CameraExtensionSession.StateCallback getStateCallback() { + return mCallback; + } + + /** + * Retrieve the Executor for the CameraExtensionSession instance. + * + * @return The Executor on which the callback will be invoked. + */ + public @NonNull + Executor getExecutor() { + return mExecutor; + } +} diff --git a/core/java/android/hardware/camera2/params/StreamConfigurationMap.java b/core/java/android/hardware/camera2/params/StreamConfigurationMap.java index 9d32a809fe0df..8bb0b184ce024 100644 --- a/core/java/android/hardware/camera2/params/StreamConfigurationMap.java +++ b/core/java/android/hardware/camera2/params/StreamConfigurationMap.java @@ -1896,7 +1896,8 @@ public final class StreamConfigurationMap { // from system/core/include/system/graphics.h private static final int HAL_PIXEL_FORMAT_RAW16 = 0x20; - private static final int HAL_PIXEL_FORMAT_BLOB = 0x21; + /** @hide */ + public static final int HAL_PIXEL_FORMAT_BLOB = 0x21; private static final int HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED = 0x22; private static final int HAL_PIXEL_FORMAT_YCbCr_420_888 = 0x23; private static final int HAL_PIXEL_FORMAT_RAW_OPAQUE = 0x24; @@ -1910,7 +1911,8 @@ public final class StreamConfigurationMap { private static final int HAL_DATASPACE_RANGE_SHIFT = 27; private static final int HAL_DATASPACE_UNKNOWN = 0x0; - private static final int HAL_DATASPACE_V0_JFIF = + /** @hide */ + public static final int HAL_DATASPACE_V0_JFIF = (2 << HAL_DATASPACE_STANDARD_SHIFT) | (3 << HAL_DATASPACE_TRANSFER_SHIFT) | (1 << HAL_DATASPACE_RANGE_SHIFT); diff --git a/core/jni/android_hardware_HardwareBuffer.cpp b/core/jni/android_hardware_HardwareBuffer.cpp index 2944f72f7f4d4..b25b4f766b89c 100644 --- a/core/jni/android_hardware_HardwareBuffer.cpp +++ b/core/jni/android_hardware_HardwareBuffer.cpp @@ -31,7 +31,6 @@ #include #include -#include #include #include @@ -224,6 +223,16 @@ AHardwareBuffer* android_hardware_HardwareBuffer_getNativeHardwareBuffer( } } +GraphicBuffer* android_hardware_HardwareBuffer_getNativeGraphicBuffer( + JNIEnv* env, jobject hardwareBufferObj) { + if (env->IsInstanceOf(hardwareBufferObj, gHardwareBufferClassInfo.clazz)) { + return GraphicBufferWrapper_to_GraphicBuffer( + env->GetLongField(hardwareBufferObj, gHardwareBufferClassInfo.mNativeObject)); + } else { + return nullptr; + } +} + jobject android_hardware_HardwareBuffer_createFromAHardwareBuffer( JNIEnv* env, AHardwareBuffer* hardwareBuffer) { GraphicBuffer* buffer = AHardwareBuffer_to_GraphicBuffer(hardwareBuffer); diff --git a/core/jni/android_hardware_camera2_CameraMetadata.cpp b/core/jni/android_hardware_camera2_CameraMetadata.cpp index 919e3513e061e..315ca2ed5b476 100644 --- a/core/jni/android_hardware_camera2_CameraMetadata.cpp +++ b/core/jni/android_hardware_camera2_CameraMetadata.cpp @@ -249,6 +249,32 @@ static jint CameraMetadata_getEntryCount(JNIEnv *env, jclass thiz, jlong ptr) { return metadata->entryCount(); } +static void CameraMetadata_update(JNIEnv *env, jclass thiz, jlong dst, jlong src) { + ALOGV("%s", __FUNCTION__); + + CameraMetadata* metadataDst = CameraMetadata_getPointerThrow(env, dst); + CameraMetadata* metadataSrc = CameraMetadata_getPointerThrow(env, src); + + if (((metadataDst == NULL) || (metadataDst->isEmpty())) || + ((metadataSrc == NULL) || metadataSrc->isEmpty())) { + return; + } + + auto metaBuffer = metadataSrc->getAndLock(); + camera_metadata_ro_entry_t entry; + auto entryCount = get_camera_metadata_entry_count(metaBuffer); + for (size_t i = 0; i < entryCount; i++) { + auto stat = get_camera_metadata_ro_entry(metaBuffer, i, &entry); + if (stat != NO_ERROR) { + ALOGE("%s: Failed to retrieve source metadata!", __func__); + metadataSrc->unlock(metaBuffer); + return; + } + metadataDst->update(entry.tag, entry.data.u8, entry.count); + } + metadataSrc->unlock(metaBuffer); +} + static jlong CameraMetadata_getBufferSize(JNIEnv *env, jclass thiz, jlong ptr) { ALOGV("%s", __FUNCTION__); @@ -565,6 +591,9 @@ static const JNINativeMethod gCameraMetadataMethods[] = { { "nativeAllocateCopy", "(J)J", (void *)CameraMetadata_allocateCopy }, + { "nativeUpdate", + "(JJ)V", + (void*)CameraMetadata_update }, { "nativeIsEmpty", "(J)Z", (void*)CameraMetadata_isEmpty }, diff --git a/core/jni/include/android_runtime/android_hardware_HardwareBuffer.h b/core/jni/include/android_runtime/android_hardware_HardwareBuffer.h index c452b74c802bc..dfd80359e1698 100644 --- a/core/jni/include/android_runtime/android_hardware_HardwareBuffer.h +++ b/core/jni/include/android_runtime/android_hardware_HardwareBuffer.h @@ -18,6 +18,7 @@ #define _ANDROID_HARDWARE_HARDWAREBUFFER_H #include +#include #include "jni.h" @@ -27,6 +28,10 @@ namespace android { extern AHardwareBuffer* android_hardware_HardwareBuffer_getNativeHardwareBuffer( JNIEnv* env, jobject hardwareBufferObj); +/* Gets the underlying GraphicBuffer for a HardwareBuffer. */ +extern GraphicBuffer* android_hardware_HardwareBuffer_getNativeGraphicBuffer( + JNIEnv* env, jobject hardwareBufferObj); + /* Returns a HardwareBuffer wrapper for the underlying AHardwareBuffer. */ extern jobject android_hardware_HardwareBuffer_createFromAHardwareBuffer( JNIEnv* env, AHardwareBuffer* hardwareBuffer); diff --git a/media/java/android/media/Image.java b/media/java/android/media/Image.java index 0ef4b94f6e372..1616c03112a8f 100644 --- a/media/java/android/media/Image.java +++ b/media/java/android/media/Image.java @@ -211,6 +211,23 @@ public abstract class Image implements AutoCloseable { @SuppressWarnings("HiddenAbstractMethod") public abstract int getScalingMode(); + /** + * Get the fence file descriptor associated with this frame. + * @return The fence file descriptor for this frame. + * @hide + */ + public int getFenceFd() { + return -1; + } + + /** + * Get the number of planes. + * @return The number of expected planes. + * @hide + */ + public int getPlaneCount() { + return -1; + } /** * Get the {@link android.hardware.HardwareBuffer HardwareBuffer} handle of the input image * intended for GPU and/or hardware access. @@ -330,8 +347,9 @@ public abstract class Image implements AutoCloseable { * @return true if the image is attachable to a new owner, false if the image is still attached * to its current owner, or the image is a stand-alone image and is not attachable to * a new owner. + * @hide */ - boolean isAttachable() { + public boolean isAttachable() { throwISEIfImageIsInvalid(); return false; diff --git a/media/java/android/media/ImageReader.java b/media/java/android/media/ImageReader.java index 5dc6f755d0664..f5b204a4a9082 100644 --- a/media/java/android/media/ImageReader.java +++ b/media/java/android/media/ImageReader.java @@ -18,8 +18,10 @@ package android.media; import android.annotation.IntRange; import android.annotation.NonNull; +import android.graphics.GraphicBuffer; import android.graphics.ImageFormat; import android.graphics.ImageFormat.Format; +import android.graphics.Rect; import android.hardware.HardwareBuffer; import android.hardware.HardwareBuffer.Usage; import android.os.Handler; @@ -715,8 +717,9 @@ public class ImageReader implements AutoCloseable { * @throws IllegalStateException If the ImageReader or image have been * closed, or the has been detached, or has not yet been * acquired. + * @hide */ - void detachImage(Image image) { + public void detachImage(Image image) { if (image == null) { throw new IllegalArgumentException("input image must not be null"); } @@ -897,6 +900,18 @@ public class ImageReader implements AutoCloseable { return mScalingMode; } + @Override + public int getPlaneCount() { + throwISEIfImageIsInvalid(); + return ImageReader.this.mNumPlanes; + } + + @Override + public int getFenceFd() { + throwISEIfImageIsInvalid(); + return nativeGetFenceFd(); + } + @Override public HardwareBuffer getHardwareBuffer() { throwISEIfImageIsInvalid(); @@ -931,7 +946,7 @@ public class ImageReader implements AutoCloseable { } @Override - boolean isAttachable() { + public boolean isAttachable() { throwISEIfImageIsInvalid(); return mIsDetached.get(); } @@ -1048,6 +1063,7 @@ public class ImageReader implements AutoCloseable { private synchronized native int nativeGetWidth(); private synchronized native int nativeGetHeight(); private synchronized native int nativeGetFormat(int readerFormat); + private synchronized native int nativeGetFenceFd(); private synchronized native HardwareBuffer nativeGetHardwareBuffer(); } @@ -1068,6 +1084,67 @@ public class ImageReader implements AutoCloseable { */ private synchronized native int nativeImageSetup(Image i); + /** + * @hide + */ + public static class ImagePlane extends android.media.Image.Plane { + private ImagePlane(int rowStride, int pixelStride, ByteBuffer buffer) { + mRowStride = rowStride; + mPixelStride = pixelStride; + mBuffer = buffer; + /** + * Set the byteBuffer order according to host endianness (native + * order), otherwise, the byteBuffer order defaults to + * ByteOrder.BIG_ENDIAN. + */ + mBuffer.order(ByteOrder.nativeOrder()); + } + + @Override + public ByteBuffer getBuffer() { + return mBuffer; + } + + @Override + public int getPixelStride() { + return mPixelStride; + } + + @Override + public int getRowStride() { + return mRowStride; + } + + final private int mPixelStride; + final private int mRowStride; + + private ByteBuffer mBuffer; + } + + /** + * @hide + */ + public static ImagePlane[] initializeImagePlanes(int numPlanes, + GraphicBuffer buffer, int fenceFd, int format, long timestamp, int transform, + int scalingMode, Rect crop) { + + return nativeCreateImagePlanes(numPlanes, buffer, fenceFd, format, crop.left, crop.top, + crop.right, crop.bottom); + } + + private synchronized static native ImagePlane[] nativeCreateImagePlanes(int numPlanes, + GraphicBuffer buffer, int fenceFd, int format, int cropLeft, int cropTop, + int cropRight, int cropBottom); + + /** + * @hide + */ + public static void unlockGraphicBuffer(GraphicBuffer buffer) { + nativeUnlockGraphicBuffer(buffer); + } + + private synchronized static native void nativeUnlockGraphicBuffer(GraphicBuffer buffer); + /** * We use a class initializer to allow the native code to cache some * field offsets. diff --git a/media/java/android/media/ImageWriter.java b/media/java/android/media/ImageWriter.java index 7bc2b31b16fae..92db946ab5bad 100644 --- a/media/java/android/media/ImageWriter.java +++ b/media/java/android/media/ImageWriter.java @@ -18,6 +18,7 @@ package android.media; import android.annotation.IntRange; import android.annotation.NonNull; +import android.graphics.GraphicBuffer; import android.graphics.ImageFormat; import android.graphics.ImageFormat.Format; import android.graphics.PixelFormat; @@ -360,16 +361,20 @@ public class ImageWriter implements AutoCloseable { throw new IllegalStateException("Image from ImageWriter is invalid"); } - // For images from other components, need to detach first, then attach. + // For images from other components that have non-null owner, need to detach first, + // then attach. Images without owners must already be attachable. if (!ownedByMe) { - if (!(image.getOwner() instanceof ImageReader)) { + if (image.getOwner() == null) { + + } else if ((image.getOwner() instanceof ImageReader)) { + ImageReader prevOwner = (ImageReader) image.getOwner(); + + prevOwner.detachImage(image); + } else { throw new IllegalArgumentException("Only images from ImageReader can be queued to" + " ImageWriter, other image source is not supported yet!"); } - ImageReader prevOwner = (ImageReader) image.getOwner(); - - prevOwner.detachImage(image); attachAndQueueInputImage(image); // This clears the native reference held by the original owner. // When this Image is detached later by this ImageWriter, the @@ -565,9 +570,18 @@ public class ImageWriter implements AutoCloseable { // need do some cleanup to make sure no orphaned // buffer caused leak. Rect crop = image.getCropRect(); - nativeAttachAndQueueImage(mNativeContext, image.getNativeContext(), image.getFormat(), - image.getTimestamp(), crop.left, crop.top, crop.right, crop.bottom, - image.getTransform(), image.getScalingMode()); + if (image.getNativeContext() != 0) { + nativeAttachAndQueueImage(mNativeContext, image.getNativeContext(), image.getFormat(), + image.getTimestamp(), crop.left, crop.top, crop.right, crop.bottom, + image.getTransform(), image.getScalingMode()); + } else { + GraphicBuffer gb = GraphicBuffer.createFromHardwareBuffer(image.getHardwareBuffer()); + nativeAttachAndQueueGraphicBuffer(mNativeContext, gb, image.getFormat(), + image.getTimestamp(), crop.left, crop.top, crop.right, crop.bottom, + image.getTransform(), image.getScalingMode()); + gb.destroy(); + image.close(); + } } /** @@ -771,7 +785,7 @@ public class ImageWriter implements AutoCloseable { } @Override - boolean isAttachable() { + public boolean isAttachable() { throwISEIfImageIsInvalid(); // Don't allow Image to be detached from ImageWriter for now, as no // detach API is exposed. @@ -898,6 +912,9 @@ public class ImageWriter implements AutoCloseable { private synchronized native int nativeAttachAndQueueImage(long nativeCtx, long imageNativeBuffer, int imageFormat, long timestampNs, int left, int top, int right, int bottom, int transform, int scalingMode); + private synchronized native int nativeAttachAndQueueGraphicBuffer(long nativeCtx, + GraphicBuffer graphicBuffer, int imageFormat, long timestampNs, int left, + int top, int right, int bottom, int transform, int scalingMode); private synchronized native void cancelImage(long nativeCtx, Image image); diff --git a/media/jni/android_media_ImageReader.cpp b/media/jni/android_media_ImageReader.cpp index bd2a0eaa75c99..5174c0c04dbf2 100644 --- a/media/jni/android_media_ImageReader.cpp +++ b/media/jni/android_media_ImageReader.cpp @@ -30,6 +30,7 @@ #include #include +#include #include #include @@ -42,6 +43,8 @@ #include #include +#include + #define ANDROID_MEDIA_IMAGEREADER_CTX_JNI_ID "mNativeContext" #define ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID "mNativeBuffer" #define ANDROID_MEDIA_SURFACEIMAGE_TS_JNI_ID "mTimestamp" @@ -78,6 +81,11 @@ static struct { jmethodID ctor; } gSurfacePlaneClassInfo; +static struct { + jclass clazz; + jmethodID ctor; +} gImagePlaneClassInfo; + // Get an ID that's unique within this process. static int32_t createProcessUniqueId() { static volatile int32_t globalCounter = 0; @@ -347,6 +355,15 @@ static void ImageReader_classInit(JNIEnv* env, jclass clazz) "(Landroid/media/ImageReader$SurfaceImage;IILjava/nio/ByteBuffer;)V"); LOG_ALWAYS_FATAL_IF(gSurfacePlaneClassInfo.ctor == NULL, "Can not find SurfacePlane constructor"); + + planeClazz = env->FindClass("android/media/ImageReader$ImagePlane"); + LOG_ALWAYS_FATAL_IF(planeClazz == NULL, "Can not find ImagePlane class"); + // FindClass only gives a local reference of jclass object. + gImagePlaneClassInfo.clazz = (jclass) env->NewGlobalRef(planeClazz); + gImagePlaneClassInfo.ctor = env->GetMethodID(gImagePlaneClassInfo.clazz, "", + "(IILjava/nio/ByteBuffer;)V"); + LOG_ALWAYS_FATAL_IF(gImagePlaneClassInfo.ctor == NULL, + "Can not find ImagePlane constructor"); } static void ImageReader_init(JNIEnv* env, jobject thiz, jobject weakThiz, jint width, jint height, @@ -729,6 +746,89 @@ static bool Image_getLockedImageInfo(JNIEnv* env, LockedImage* buffer, int idx, return true; } +static void ImageReader_unlockGraphicBuffer(JNIEnv* env, jobject /*thiz*/, + jobject buffer) { + sp graphicBuffer = + android_graphics_GraphicBuffer_getNativeGraphicsBuffer(env, buffer); + if (graphicBuffer.get() == NULL) { + jniThrowRuntimeException(env, "Invalid graphic buffer!"); + } + + status_t res = graphicBuffer->unlock(); + if (res != OK) { + jniThrowRuntimeException(env, "unlock buffer failed"); + } +} + +static jobjectArray ImageReader_createImagePlanes(JNIEnv* env, jobject /*thiz*/, + int numPlanes, jobject buffer, int fenceFd, int format, int cropLeft, int cropTop, + int cropRight, int cropBottom) +{ + ALOGV("%s: create ImagePlane array with size %d", __FUNCTION__, numPlanes); + int rowStride = 0; + int pixelStride = 0; + uint8_t *pData = NULL; + uint32_t dataSize = 0; + jobject byteBuffer = NULL; + + PublicFormat publicReaderFormat = static_cast(format); + int halReaderFormat = mapPublicFormatToHalFormat(publicReaderFormat); + + if (isFormatOpaque(halReaderFormat) && numPlanes > 0) { + String8 msg; + msg.appendFormat("Format 0x%x is opaque, thus not writable, the number of planes (%d)" + " must be 0", halReaderFormat, numPlanes); + jniThrowException(env, "java/lang/IllegalArgumentException", msg.string()); + return NULL; + } + + jobjectArray imagePlanes = env->NewObjectArray(numPlanes, gImagePlaneClassInfo.clazz, + /*initial_element*/NULL); + if (imagePlanes == NULL) { + jniThrowRuntimeException(env, "Failed to create ImagePlane arrays," + " probably out of memory"); + return NULL; + } + if (isFormatOpaque(halReaderFormat)) { + // Return 0 element surface array. + return imagePlanes; + } + + LockedImage lockedImg = LockedImage(); + uint32_t lockUsage = GRALLOC_USAGE_SW_READ_OFTEN; + + Rect cropRect(cropLeft, cropTop, cropRight, cropBottom); + status_t res = lockImageFromBuffer( + android_graphics_GraphicBuffer_getNativeGraphicsBuffer(env, buffer), lockUsage, + cropRect, fenceFd, &lockedImg); + if (res != OK) { + jniThrowExceptionFmt(env, "java/lang/RuntimeException", + "lock buffer failed for format 0x%x", format); + return nullptr; + } + + // Create all ImagePlanes + for (int i = 0; i < numPlanes; i++) { + if (!Image_getLockedImageInfo(env, &lockedImg, i, halReaderFormat, + &pData, &dataSize, &pixelStride, &rowStride)) { + return NULL; + } + byteBuffer = env->NewDirectByteBuffer(pData, dataSize); + if ((byteBuffer == NULL) && (env->ExceptionCheck() == false)) { + jniThrowException(env, "java/lang/IllegalStateException", + "Failed to allocate ByteBuffer"); + return NULL; + } + + // Finally, create this ImagePlane. + jobject imagePlane = env->NewObject(gImagePlaneClassInfo.clazz, + gImagePlaneClassInfo.ctor, rowStride, pixelStride, byteBuffer); + env->SetObjectArrayElement(imagePlanes, i, imagePlane); + } + + return imagePlanes; +} + static jobjectArray Image_createSurfacePlanes(JNIEnv* env, jobject thiz, int numPlanes, int readerFormat, uint64_t ndkReaderUsage) { @@ -801,6 +901,16 @@ static jint Image_getHeight(JNIEnv* env, jobject thiz) return getBufferHeight(buffer); } +static jint Image_getFenceFd(JNIEnv* env, jobject thiz) +{ + BufferItem* buffer = Image_getBufferItem(env, thiz); + if ((buffer != NULL) && (buffer->mFence.get() != NULL)){ + return buffer->mFence->get(); + } + + return -1; +} + static jint Image_getFormat(JNIEnv* env, jobject thiz, jint readerFormat) { if (isFormatOpaque(readerFormat)) { @@ -844,6 +954,11 @@ static const JNINativeMethod gImageReaderMethods[] = { {"nativeImageSetup", "(Landroid/media/Image;)I", (void*)ImageReader_imageSetup }, {"nativeGetSurface", "()Landroid/view/Surface;", (void*)ImageReader_getSurface }, {"nativeDetachImage", "(Landroid/media/Image;)I", (void*)ImageReader_detachImage }, + {"nativeCreateImagePlanes", + "(ILandroid/graphics/GraphicBuffer;IIIIII)[Landroid/media/ImageReader$ImagePlane;", + (void*)ImageReader_createImagePlanes }, + {"nativeUnlockGraphicBuffer", + "(Landroid/graphics/GraphicBuffer;)V", (void*)ImageReader_unlockGraphicBuffer }, {"nativeDiscardFreeBuffers", "()V", (void*)ImageReader_discardFreeBuffers } }; @@ -853,6 +968,7 @@ static const JNINativeMethod gImageMethods[] = { {"nativeGetWidth", "()I", (void*)Image_getWidth }, {"nativeGetHeight", "()I", (void*)Image_getHeight }, {"nativeGetFormat", "(I)I", (void*)Image_getFormat }, + {"nativeGetFenceFd", "()I", (void*)Image_getFenceFd }, {"nativeGetHardwareBuffer", "()Landroid/hardware/HardwareBuffer;", (void*)Image_getHardwareBuffer }, }; diff --git a/media/jni/android_media_ImageWriter.cpp b/media/jni/android_media_ImageWriter.cpp index 936edb3fb0058..5d959a3930f47 100644 --- a/media/jni/android_media_ImageWriter.cpp +++ b/media/jni/android_media_ImageWriter.cpp @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -673,36 +674,12 @@ static void ImageWriter_queueImage(JNIEnv* env, jobject thiz, jlong nativeCtx, j Image_setNativeContext(env, image, NULL, -1); } -static jint ImageWriter_attachAndQueueImage(JNIEnv* env, jobject thiz, jlong nativeCtx, - jlong nativeBuffer, jint imageFormat, jlong timestampNs, jint left, jint top, +static status_t attachAndQeueuGraphicBuffer(JNIEnv* env, JNIImageWriterContext *ctx, + sp surface, sp gb, jlong timestampNs, jint left, jint top, jint right, jint bottom, jint transform, jint scalingMode) { - ALOGV("%s", __FUNCTION__); - JNIImageWriterContext* const ctx = reinterpret_cast(nativeCtx); - if (ctx == NULL || thiz == NULL) { - jniThrowException(env, "java/lang/IllegalStateException", - "ImageWriterContext is not initialized"); - return -1; - } - - sp surface = ctx->getProducer(); status_t res = OK; - if (isFormatOpaque(imageFormat) != isFormatOpaque(ctx->getBufferFormat())) { - jniThrowException(env, "java/lang/IllegalStateException", - "Trying to attach an opaque image into a non-opaque ImageWriter, or vice versa"); - return -1; - } - - // Image is guaranteed to be from ImageReader at this point, so it is safe to - // cast to BufferItem pointer. - BufferItem* buffer = reinterpret_cast(nativeBuffer); - if (buffer == NULL) { - jniThrowException(env, "java/lang/IllegalStateException", - "Image is not initialized or already closed"); - return -1; - } - // Step 1. Attach Image - res = surface->attachBuffer(buffer->mGraphicBuffer.get()); + res = surface->attachBuffer(gb.get()); if (res != OK) { ALOGE("Attach image failed: %s (%d)", strerror(-res), res); switch (res) { @@ -756,7 +733,7 @@ static jint ImageWriter_attachAndQueueImage(JNIEnv* env, jobject thiz, jlong nat // queue the "attached" flag before calling queueBuffer. In case // queueBuffer() fails, remove it from the queue. ctx->queueAttachedFlag(true); - res = anw->queueBuffer(anw.get(), buffer->mGraphicBuffer.get(), /*fenceFd*/ + res = anw->queueBuffer(anw.get(), gb.get(), /*fenceFd*/ -1); if (res != OK) { ALOGE("%s: Queue buffer failed: %s (%d)", __FUNCTION__, strerror(-res), res); @@ -779,6 +756,67 @@ static jint ImageWriter_attachAndQueueImage(JNIEnv* env, jobject thiz, jlong nat return res; } +static jint ImageWriter_attachAndQueueImage(JNIEnv* env, jobject thiz, jlong nativeCtx, + jlong nativeBuffer, jint imageFormat, jlong timestampNs, jint left, jint top, + jint right, jint bottom, jint transform, jint scalingMode) { + ALOGV("%s", __FUNCTION__); + JNIImageWriterContext* const ctx = reinterpret_cast(nativeCtx); + if (ctx == NULL || thiz == NULL) { + jniThrowException(env, "java/lang/IllegalStateException", + "ImageWriterContext is not initialized"); + return -1; + } + + sp surface = ctx->getProducer(); + if (isFormatOpaque(imageFormat) != isFormatOpaque(ctx->getBufferFormat())) { + jniThrowException(env, "java/lang/IllegalStateException", + "Trying to attach an opaque image into a non-opaque ImageWriter, or vice versa"); + return -1; + } + + // Image is guaranteed to be from ImageReader at this point, so it is safe to + // cast to BufferItem pointer. + BufferItem* buffer = reinterpret_cast(nativeBuffer); + if (buffer == NULL) { + jniThrowException(env, "java/lang/IllegalStateException", + "Image is not initialized or already closed"); + return -1; + } + + return attachAndQeueuGraphicBuffer(env, ctx, surface, buffer->mGraphicBuffer, timestampNs, left, + top, right, bottom, transform, scalingMode); +} + +static jint ImageWriter_attachAndQueueGraphicBuffer(JNIEnv* env, jobject thiz, jlong nativeCtx, + jobject buffer, jint format, jlong timestampNs, jint left, jint top, + jint right, jint bottom, jint transform, jint scalingMode) { + ALOGV("%s", __FUNCTION__); + JNIImageWriterContext* const ctx = reinterpret_cast(nativeCtx); + if (ctx == NULL || thiz == NULL) { + jniThrowException(env, "java/lang/IllegalStateException", + "ImageWriterContext is not initialized"); + return -1; + } + + sp surface = ctx->getProducer(); + if (isFormatOpaque(format) != isFormatOpaque(ctx->getBufferFormat())) { + jniThrowException(env, "java/lang/IllegalStateException", + "Trying to attach an opaque image into a non-opaque ImageWriter, or vice versa"); + return -1; + } + + sp graphicBuffer = android_graphics_GraphicBuffer_getNativeGraphicsBuffer(env, + buffer); + if (graphicBuffer.get() == NULL) { + jniThrowException(env, "java/lang/IllegalArgumentException", + "Trying to attach an invalid graphic buffer"); + return -1; + } + + return attachAndQeueuGraphicBuffer(env, ctx, surface, graphicBuffer, timestampNs, left, + top, right, bottom, transform, scalingMode); +} + // --------------------------Image methods--------------------------------------- static void Image_getNativeContext(JNIEnv* env, jobject thiz, @@ -1011,6 +1049,9 @@ static JNINativeMethod gImageWriterMethods[] = { (void*)ImageWriter_init }, {"nativeClose", "(J)V", (void*)ImageWriter_close }, {"nativeAttachAndQueueImage", "(JJIJIIIIII)I", (void*)ImageWriter_attachAndQueueImage }, + {"nativeAttachAndQueueGraphicBuffer", + "(JLandroid/graphics/GraphicBuffer;IJIIIIII)I", + (void*)ImageWriter_attachAndQueueGraphicBuffer }, {"nativeDequeueInputImage", "(JLandroid/media/Image;)V", (void*)ImageWriter_dequeueImage }, {"nativeQueueInputImage", "(JLandroid/media/Image;JIIIIII)V", (void*)ImageWriter_queueImage }, {"cancelImage", "(JLandroid/media/Image;)V", (void*)ImageWriter_cancelImage }, diff --git a/packages/services/CameraExtensionsProxy/Android.bp b/packages/services/CameraExtensionsProxy/Android.bp new file mode 100644 index 0000000000000..54b7453da5852 --- /dev/null +++ b/packages/services/CameraExtensionsProxy/Android.bp @@ -0,0 +1,29 @@ +// +// Copyright (C) 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +android_app { + name: "CameraExtensionsProxy", + srcs: ["src/**/*.java"], + libs: ["androidx.camera.extensions.stub"], + platform_apis: true, + certificate: "platform", +} + +filegroup { + name: "CameraExtensionsProxy-aidl-sources", + srcs: ["src/**/*.aidl"], + path: "src", +} diff --git a/packages/services/CameraExtensionsProxy/AndroidManifest.xml b/packages/services/CameraExtensionsProxy/AndroidManifest.xml new file mode 100644 index 0000000000000..7416cbacc3252 --- /dev/null +++ b/packages/services/CameraExtensionsProxy/AndroidManifest.xml @@ -0,0 +1,18 @@ + + + + + + + + + + + + + diff --git a/packages/services/CameraExtensionsProxy/res/values/strings.xml b/packages/services/CameraExtensionsProxy/res/values/strings.xml new file mode 100644 index 0000000000000..5062b3db4c8ce --- /dev/null +++ b/packages/services/CameraExtensionsProxy/res/values/strings.xml @@ -0,0 +1,6 @@ + + + + CameraExtensionsProxy + + diff --git a/packages/services/CameraExtensionsProxy/src/com/android/camera/CameraExtensionsProxyService.java b/packages/services/CameraExtensionsProxy/src/com/android/camera/CameraExtensionsProxyService.java new file mode 100644 index 0000000000000..0f0501972d3ce --- /dev/null +++ b/packages/services/CameraExtensionsProxy/src/com/android/camera/CameraExtensionsProxyService.java @@ -0,0 +1,857 @@ +/** + * Copyright (c) 2020, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.android.camera; + +import android.app.Service; +import android.content.Context; +import android.content.Intent; +import android.graphics.GraphicBuffer; +import android.graphics.Rect; +import android.hardware.camera2.CameraAccessException; +import android.hardware.camera2.CameraCharacteristics; +import android.hardware.camera2.CameraExtensionCharacteristics; +import android.hardware.camera2.CameraManager; +import android.hardware.camera2.CaptureRequest; +import android.hardware.camera2.extension.CaptureBundle; +import android.hardware.camera2.extension.CaptureStageImpl; +import android.hardware.camera2.extension.ICameraExtensionsProxyService; +import android.hardware.camera2.extension.ICaptureProcessorImpl; +import android.hardware.camera2.extension.IPreviewExtenderImpl; +import android.hardware.camera2.extension.IPreviewImageProcessorImpl; +import android.hardware.camera2.extension.IRequestUpdateProcessorImpl; +import android.hardware.camera2.extension.ParcelImage; +import android.hardware.camera2.extension.IImageCaptureExtenderImpl; +import android.hardware.camera2.extension.SizeList; +import android.hardware.camera2.impl.CameraMetadataNative; +import android.hardware.camera2.TotalCaptureResult; +import android.hardware.HardwareBuffer; +import android.media.Image; +import android.media.ImageReader; +import android.os.ConditionVariable; +import android.os.Handler; +import android.os.HandlerExecutor; +import android.os.HandlerThread; +import android.os.IBinder; +import android.util.ArraySet; +import android.util.Log; +import android.util.Pair; +import android.util.Size; +import android.view.Surface; + +import androidx.camera.extensions.impl.AutoImageCaptureExtenderImpl; +import androidx.camera.extensions.impl.AutoPreviewExtenderImpl; +import androidx.camera.extensions.impl.BeautyImageCaptureExtenderImpl; +import androidx.camera.extensions.impl.BeautyPreviewExtenderImpl; +import androidx.camera.extensions.impl.BokehImageCaptureExtenderImpl; +import androidx.camera.extensions.impl.BokehPreviewExtenderImpl; +import androidx.camera.extensions.impl.CaptureProcessorImpl; +import androidx.camera.extensions.impl.ExtensionVersionImpl; +import androidx.camera.extensions.impl.HdrImageCaptureExtenderImpl; +import androidx.camera.extensions.impl.HdrPreviewExtenderImpl; +import androidx.camera.extensions.impl.ImageCaptureExtenderImpl; +import androidx.camera.extensions.impl.InitializerImpl; +import androidx.camera.extensions.impl.NightImageCaptureExtenderImpl; +import androidx.camera.extensions.impl.NightPreviewExtenderImpl; +import androidx.camera.extensions.impl.PreviewExtenderImpl; +import androidx.camera.extensions.impl.PreviewExtenderImpl.ProcessorType; +import androidx.camera.extensions.impl.PreviewImageProcessorImpl; +import androidx.camera.extensions.impl.RequestUpdateProcessorImpl; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.List; + +public class CameraExtensionsProxyService extends Service { + private static final String TAG = "CameraExtensionsProxyService"; + + private static final String CAMERA_EXTENSION_VERSION_NAME = + "androidx.camera.extensions.impl.ExtensionVersionImpl"; + private static final String LATEST_VERSION = "1.1.0"; + private static final String[] SUPPORTED_VERSION_PREFIXES = {"1.1.", "1.0."}; + private static final boolean EXTENSIONS_PRESENT = checkForExtensions(); + private static final String EXTENSIONS_VERSION = EXTENSIONS_PRESENT ? + (new ExtensionVersionImpl()).checkApiVersion(LATEST_VERSION) : null; + private static final boolean LATEST_VERSION_SUPPORTED = + EXTENSIONS_PRESENT && EXTENSIONS_VERSION.startsWith(SUPPORTED_VERSION_PREFIXES[0]); + + private static boolean checkForExtensions() { + try { + Class.forName(CAMERA_EXTENSION_VERSION_NAME); + } catch (ClassNotFoundException e) { + return false; + } + + String extensionVersion = (new ExtensionVersionImpl()).checkApiVersion(LATEST_VERSION); + for (String supportedVersion : SUPPORTED_VERSION_PREFIXES) { + if (extensionVersion.startsWith(supportedVersion)) { + return true; + } + } + + return false; + } + + /** + * A per-process global camera extension manager instance, to track and + * initialize/release extensions depending on client activity. + */ + private static final class CameraExtensionManagerGlobal { + private static final String TAG = "CameraExtensionManagerGlobal"; + private final int EXTENSION_DELAY_MS = 1000; + + private final Handler mHandler; + private final HandlerThread mHandlerThread; + private final Object mLock = new Object(); + + private long mCurrentClientCount = 0; + private ArraySet mActiveClients = new ArraySet<>(); + + // Singleton instance + private static final CameraExtensionManagerGlobal GLOBAL_CAMERA_MANAGER = + new CameraExtensionManagerGlobal(); + + // Singleton, don't allow construction + private CameraExtensionManagerGlobal() { + mHandlerThread = new HandlerThread(TAG); + mHandlerThread.start(); + mHandler = new Handler(mHandlerThread.getLooper()); + } + + private final static class InitializeHandler + implements InitializerImpl.OnExtensionsInitializedCallback { + private final InitializerFuture mStatusFuture; + + public InitializeHandler(InitializerFuture statusFuture) { + mStatusFuture = statusFuture; + } + + @Override + public void onSuccess() { + mStatusFuture.setStatus(true); + } + + @Override + public void onFailure(int error) { + mStatusFuture.setStatus(false); + } + } + + private final static class ReleaseHandler + implements InitializerImpl.OnExtensionsDeinitializedCallback { + private final InitializerFuture mStatusFuture; + + public ReleaseHandler(InitializerFuture statusFuture) { + mStatusFuture = statusFuture; + } + + @Override public void onSuccess() { + mStatusFuture.setStatus(true); + } + + @Override + public void onFailure(int i) { + mStatusFuture.setStatus(false); + } + } + + private static class InitializerFuture implements Future { + private volatile Boolean mStatus; + ConditionVariable mCondVar = new ConditionVariable(/*opened*/false); + + public void setStatus(boolean status) { + mStatus = status; + mCondVar.open(); + } + + @Override + public boolean cancel(boolean mayInterruptIfRunning) { + return false; // don't allow canceling this task + } + + @Override + public boolean isCancelled() { + return false; // can never cancel this task + } + + @Override + public boolean isDone() { + return mStatus != null; + } + + @Override + public Boolean get() { + mCondVar.block(); + return mStatus; + } + + @Override + public Boolean get(long timeout, TimeUnit unit) throws TimeoutException { + long timeoutMs = unit.convert(timeout, TimeUnit.MILLISECONDS); + if (!mCondVar.block(timeoutMs)) { + throw new TimeoutException( + "Failed to receive status after " + timeout + " " + unit); + } + + if (mStatus == null) { + throw new AssertionError(); + } + return mStatus; + } + + } + + public static CameraExtensionManagerGlobal get() { + return GLOBAL_CAMERA_MANAGER; + } + + public long registerClient(Context ctx) { + synchronized (mLock) { + if (LATEST_VERSION_SUPPORTED) { + if (mActiveClients.isEmpty()) { + InitializerFuture status = new InitializerFuture(); + InitializerImpl.init(EXTENSIONS_VERSION, ctx, new InitializeHandler(status), + new HandlerExecutor(mHandler)); + boolean initSuccess; + try { + initSuccess = status.get(EXTENSION_DELAY_MS, + TimeUnit.MILLISECONDS); + } catch (TimeoutException e) { + Log.e(TAG, "Timed out while initializing camera extensions!"); + return -1; + } + if (!initSuccess) { + Log.e(TAG, "Failed while initializing camera extensions!"); + return -1; + } + } + } + + long ret = mCurrentClientCount; + mCurrentClientCount++; + if (mCurrentClientCount < 0) { + mCurrentClientCount = 0; + } + mActiveClients.add(ret); + + return ret; + } + } + + public void unregisterClient(long clientId) { + synchronized (mLock) { + if (mActiveClients.remove(clientId) && mActiveClients.isEmpty() && + LATEST_VERSION_SUPPORTED) { + InitializerFuture status = new InitializerFuture(); + InitializerImpl.deinit(new ReleaseHandler(status), + new HandlerExecutor(mHandler)); + boolean releaseSuccess; + try { + releaseSuccess = status.get(EXTENSION_DELAY_MS, TimeUnit.MILLISECONDS); + } catch (TimeoutException e) { + Log.e(TAG, "Timed out while releasing camera extensions!"); + return; + } + if (!releaseSuccess) { + Log.e(TAG, "Failed while releasing camera extensions!"); + } + } + } + } + } + + /** + * @hide + */ + private static long registerClient(Context ctx) { + if (!EXTENSIONS_PRESENT) { + return -1; + } + return CameraExtensionManagerGlobal.get().registerClient(ctx); + } + + /** + * @hide + */ + public static void unregisterClient(long clientId) { + if (!EXTENSIONS_PRESENT) { + return; + } + CameraExtensionManagerGlobal.get().unregisterClient(clientId); + } + + /** + * @hide + */ + public static Pair initializeExtension( + int extensionType) { + switch (extensionType) { + case CameraExtensionCharacteristics.EXTENSION_AUTOMATIC: + return new Pair<>(new AutoPreviewExtenderImpl(), + new AutoImageCaptureExtenderImpl()); + case CameraExtensionCharacteristics.EXTENSION_BEAUTY: + return new Pair<>(new BeautyPreviewExtenderImpl(), + new BeautyImageCaptureExtenderImpl()); + case CameraExtensionCharacteristics.EXTENSION_BOKEH: + return new Pair<>(new BokehPreviewExtenderImpl(), + new BokehImageCaptureExtenderImpl()); + case CameraExtensionCharacteristics.EXTENSION_HDR: + return new Pair<>(new HdrPreviewExtenderImpl(), new HdrImageCaptureExtenderImpl()); + case CameraExtensionCharacteristics.EXTENSION_NIGHT: + return new Pair<>(new NightPreviewExtenderImpl(), + new NightImageCaptureExtenderImpl()); + default: + throw new IllegalArgumentException("Unknown extension: " + extensionType); + } + } + + @Override + public void onCreate() { + super.onCreate(); + // This will setup the camera vendor tag descriptor in the service process + try { + CameraManager manager = getSystemService(CameraManager.class); + + String [] cameraIds = manager.getCameraIdListNoLazy(); + if (cameraIds != null) { + for (String cameraId : cameraIds) { + CameraCharacteristics ch = manager.getCameraCharacteristics(cameraId); + } + } + } catch (CameraAccessException e) { + Log.e(TAG, "Failed to query camera characteristics!"); + } + } + + @Override + public void onDestroy() { + super.onDestroy(); + } + + + @Override + public IBinder onBind(Intent intent) { + return new CameraExtensionsProxyServiceStub(); + } + + private static List initializeParcelable( + List> sizes) { + if (sizes == null) { + return null; + } + ArrayList ret = new ArrayList<>(sizes.size()); + for (Pair entry : sizes) { + SizeList sizeList = new SizeList(); + sizeList.format = entry.first; + sizeList.sizes = new ArrayList<>(); + for (android.util.Size size : entry.second) { + android.hardware.camera2.extension.Size sz = + new android.hardware.camera2.extension.Size(); + sz.width = size.getWidth(); + sz.height = size.getHeight(); + sizeList.sizes.add(sz); + } + } + + return ret; + } + + private static CameraMetadataNative initializeParcelableMetadata( + List> paramList) { + if (paramList == null) { + return null; + } + + CameraMetadataNative ret = new CameraMetadataNative(); + for (Pair param : paramList) { + ret.set(param.first, param.second); + } + + return ret; + } + + private static android.hardware.camera2.extension.CaptureStageImpl initializeParcelable( + androidx.camera.extensions.impl.CaptureStageImpl captureStage) { + if (captureStage == null) { + return null; + } + + android.hardware.camera2.extension.CaptureStageImpl ret = + new android.hardware.camera2.extension.CaptureStageImpl(); + ret.id = captureStage.getId(); + ret.parameters = initializeParcelableMetadata(captureStage.getParameters()); + + return ret; + } + + private class CameraExtensionsProxyServiceStub extends ICameraExtensionsProxyService.Stub { + @Override + public long registerClient() { + return CameraExtensionsProxyService.registerClient(CameraExtensionsProxyService.this); + } + + @Override + public void unregisterClient(long clientId) { + CameraExtensionsProxyService.unregisterClient(clientId); + } + + @Override + public IPreviewExtenderImpl initializePreviewExtension(int extensionType) { + Pair extension; + try { + extension = initializeExtension(extensionType); + } catch (IllegalArgumentException e) { + return null; + } + + return new PreviewExtenderImplStub(extension.first); + } + + @Override + public IImageCaptureExtenderImpl initializeImageExtension(int extensionType) { + Pair extension; + try { + extension = initializeExtension(extensionType); + } catch (IllegalArgumentException e) { + return null; + } + + return new ImageCaptureExtenderImplStub(extension.second); + } + } + + private class PreviewExtenderImplStub extends IPreviewExtenderImpl.Stub { + private final PreviewExtenderImpl mPreviewExtender; + + public PreviewExtenderImplStub(PreviewExtenderImpl previewExtender) { + mPreviewExtender = previewExtender; + } + + @Override + public void onInit(String cameraId, CameraMetadataNative cameraCharacteristics) { + mPreviewExtender.onInit(cameraId, new CameraCharacteristics(cameraCharacteristics), + CameraExtensionsProxyService.this); + } + + @Override + public void onDeInit() { + mPreviewExtender.onDeInit(); + } + + @Override + public CaptureStageImpl onPresetSession() { + return initializeParcelable(mPreviewExtender.onPresetSession()); + } + + @Override + public CaptureStageImpl onEnableSession() { + return initializeParcelable(mPreviewExtender.onEnableSession()); + } + + @Override + public CaptureStageImpl onDisableSession() { + return initializeParcelable(mPreviewExtender.onDisableSession()); + } + + @Override + public void init(String cameraId, CameraMetadataNative chars) { + if (LATEST_VERSION_SUPPORTED) { + mPreviewExtender.init(cameraId, new CameraCharacteristics(chars)); + } + } + + @Override + public boolean isExtensionAvailable(String cameraId, CameraMetadataNative chars) { + return mPreviewExtender.isExtensionAvailable(cameraId, + new CameraCharacteristics(chars)); + } + + @Override + public CaptureStageImpl getCaptureStage() { + return initializeParcelable(mPreviewExtender.getCaptureStage()); + } + + @Override + public int getProcessorType() { + ProcessorType processorType = mPreviewExtender.getProcessorType(); + if (processorType == ProcessorType.PROCESSOR_TYPE_REQUEST_UPDATE_ONLY) { + return IPreviewExtenderImpl.PROCESSOR_TYPE_REQUEST_UPDATE_ONLY; + } else if (processorType == ProcessorType.PROCESSOR_TYPE_IMAGE_PROCESSOR) { + return IPreviewExtenderImpl.PROCESSOR_TYPE_IMAGE_PROCESSOR; + } else { + return IPreviewExtenderImpl.PROCESSOR_TYPE_NONE; + } + } + + @Override + public IPreviewImageProcessorImpl getPreviewImageProcessor() { + PreviewImageProcessorImpl processor; + try { + processor = (PreviewImageProcessorImpl) mPreviewExtender.getProcessor(); + } catch (ClassCastException e) { + Log.e(TAG, "Failed casting preview processor!"); + return null; + } + + if (processor != null) { + return new PreviewImageProcessorImplStub(processor); + } + + return null; + } + + @Override + public IRequestUpdateProcessorImpl getRequestUpdateProcessor() { + RequestUpdateProcessorImpl processor; + try { + processor = (RequestUpdateProcessorImpl) mPreviewExtender.getProcessor(); + } catch (ClassCastException e) { + Log.e(TAG, "Failed casting preview processor!"); + return null; + } + + if (processor != null) { + return new RequestUpdateProcessorImplStub(processor); + } + + return null; + } + + @Override + public List getSupportedResolutions() { + if (LATEST_VERSION_SUPPORTED) { + List> sizes = + mPreviewExtender.getSupportedResolutions(); + if ((sizes != null) && !sizes.isEmpty()) { + return initializeParcelable(sizes); + } + } + return null; + } + } + + private class ImageCaptureExtenderImplStub extends IImageCaptureExtenderImpl.Stub { + private final ImageCaptureExtenderImpl mImageExtender; + + public ImageCaptureExtenderImplStub(ImageCaptureExtenderImpl imageExtender) { + mImageExtender = imageExtender; + } + + @Override + public void onInit(String cameraId, CameraMetadataNative cameraCharacteristics) { + mImageExtender.onInit(cameraId, new CameraCharacteristics(cameraCharacteristics), + CameraExtensionsProxyService.this); + } + + @Override + public void onDeInit() { + mImageExtender.onDeInit(); + } + + @Override + public CaptureStageImpl onPresetSession() { + return initializeParcelable(mImageExtender.onPresetSession()); + } + + @Override + public CaptureStageImpl onEnableSession() { + return initializeParcelable(mImageExtender.onEnableSession()); + } + + @Override + public CaptureStageImpl onDisableSession() { + return initializeParcelable(mImageExtender.onDisableSession()); + } + + @Override + public void init(String cameraId, CameraMetadataNative chars) { + if (LATEST_VERSION_SUPPORTED) { + mImageExtender.init(cameraId, new CameraCharacteristics(chars)); + } + } + + @Override + public boolean isExtensionAvailable(String cameraId, CameraMetadataNative chars) { + return mImageExtender.isExtensionAvailable(cameraId, + new CameraCharacteristics(chars)); + } + + @Override + public ICaptureProcessorImpl getCaptureProcessor() { + CaptureProcessorImpl captureProcessor = mImageExtender.getCaptureProcessor(); + if (captureProcessor != null) { + return new CaptureProcessorImplStub(captureProcessor); + } + + return null; + } + + + @Override + public List getCaptureStages() { + List captureStages = + mImageExtender.getCaptureStages(); + if (captureStages != null) { + ArrayList ret = + new ArrayList<>(); + for (androidx.camera.extensions.impl.CaptureStageImpl stage : captureStages) { + ret.add(initializeParcelable(stage)); + } + + return ret; + } + + return null; + } + + @Override + public int getMaxCaptureStage() { + return mImageExtender.getMaxCaptureStage(); + } + + @Override + public List getSupportedResolutions() { + if (LATEST_VERSION_SUPPORTED) { + List> sizes = + mImageExtender.getSupportedResolutions(); + if ((sizes != null) && !sizes.isEmpty()) { + return initializeParcelable(sizes); + } + } + + return null; + } + } + + private class CaptureProcessorImplStub extends ICaptureProcessorImpl.Stub { + private final CaptureProcessorImpl mCaptureProcessor; + + public CaptureProcessorImplStub(CaptureProcessorImpl captureProcessor) { + mCaptureProcessor = captureProcessor; + } + + @Override + public void onOutputSurface(Surface surface, int imageFormat) { + mCaptureProcessor.onOutputSurface(surface, imageFormat); + } + + @Override + public void onResolutionUpdate(android.hardware.camera2.extension.Size size) { + mCaptureProcessor.onResolutionUpdate(new android.util.Size(size.width, size.height)); + } + + @Override + public void onImageFormatUpdate(int imageFormat) { + mCaptureProcessor.onImageFormatUpdate(imageFormat); + } + + @Override + public void process(List captureList) { + HashMap> captureMap = new HashMap<>(); + for (CaptureBundle captureBundle : captureList) { + captureMap.put(captureBundle.stage, new Pair<> ( + new ExtensionImage(captureBundle.captureImage), + new TotalCaptureResult(captureBundle.captureResult, + captureBundle.sequenceId))); + } + if (!captureMap.isEmpty()) { + mCaptureProcessor.process(captureMap); + } else { + Log.e(TAG, "Process request with absent capture stages!"); + } + } + } + + private class PreviewImageProcessorImplStub extends IPreviewImageProcessorImpl.Stub { + private final PreviewImageProcessorImpl mProcessor; + + public PreviewImageProcessorImplStub(PreviewImageProcessorImpl processor) { + mProcessor = processor; + } + + @Override + public void onOutputSurface(Surface surface, int imageFormat) { + mProcessor.onOutputSurface(surface, imageFormat); + } + + @Override + public void onResolutionUpdate(android.hardware.camera2.extension.Size size) { + mProcessor.onResolutionUpdate(new android.util.Size(size.width, size.height)); + } + + @Override + public void onImageFormatUpdate(int imageFormat) { + mProcessor.onImageFormatUpdate(imageFormat); + } + + @Override + public void process(android.hardware.camera2.extension.ParcelImage image, + CameraMetadataNative result, int sequenceId) { + mProcessor.process(new ExtensionImage(image), + new TotalCaptureResult(result, sequenceId)); + } + } + + private class RequestUpdateProcessorImplStub extends IRequestUpdateProcessorImpl.Stub { + private final RequestUpdateProcessorImpl mProcessor; + + public RequestUpdateProcessorImplStub(RequestUpdateProcessorImpl processor) { + mProcessor = processor; + } + + @Override + public void onOutputSurface(Surface surface, int imageFormat) { + mProcessor.onOutputSurface(surface, imageFormat); + } + + @Override + public void onResolutionUpdate(android.hardware.camera2.extension.Size size) { + mProcessor.onResolutionUpdate(new android.util.Size(size.width, size.height)); + } + + @Override + public void onImageFormatUpdate(int imageFormat) { + mProcessor.onImageFormatUpdate(imageFormat); + } + + @Override + public CaptureStageImpl process(CameraMetadataNative result, int sequenceId) { + return initializeParcelable( + mProcessor.process(new TotalCaptureResult(result, sequenceId))); + } + } + + private class ExtensionImage extends android.media.Image { + private final android.hardware.camera2.extension.ParcelImage mParcelImage; + private GraphicBuffer mGraphicBuffer; + private ImageReader.ImagePlane[] mPlanes; + + private ExtensionImage(android.hardware.camera2.extension.ParcelImage parcelImage) { + mParcelImage = parcelImage; + mIsImageValid = true; + } + + @Override + public int getFormat() { + throwISEIfImageIsInvalid(); + return mParcelImage.format; + } + + @Override + public int getWidth() { + throwISEIfImageIsInvalid(); + return mParcelImage.width; + } + + @Override + public HardwareBuffer getHardwareBuffer() { + throwISEIfImageIsInvalid(); + return mParcelImage.buffer; + } + + @Override + public int getHeight() { + throwISEIfImageIsInvalid(); + return mParcelImage.height; + } + + @Override + public long getTimestamp() { + throwISEIfImageIsInvalid(); + return mParcelImage.timestamp; + } + + @Override + public int getTransform() { + throwISEIfImageIsInvalid(); + return mParcelImage.transform; + } + + @Override + public int getScalingMode() { + throwISEIfImageIsInvalid(); + return mParcelImage.scalingMode; + } + + @Override + public Plane[] getPlanes() { + throwISEIfImageIsInvalid(); + if (mPlanes == null) { + int fenceFd = mParcelImage.fence != null ? mParcelImage.fence.getFd() : -1; + mGraphicBuffer = GraphicBuffer.createFromHardwareBuffer(mParcelImage.buffer); + mPlanes = ImageReader.initializeImagePlanes(mParcelImage.planeCount, mGraphicBuffer, + fenceFd, mParcelImage.format, mParcelImage.timestamp, + mParcelImage.transform, mParcelImage.scalingMode, mParcelImage.crop); + } + // Shallow copy is fine. + return mPlanes.clone(); + } + + @Override + protected final void finalize() throws Throwable { + try { + close(); + } finally { + super.finalize(); + } + } + + @Override + public boolean isAttachable() { + throwISEIfImageIsInvalid(); + // Clients must always detach parcelable images + return true; + } + + @Override + public Rect getCropRect() { + throwISEIfImageIsInvalid(); + return mParcelImage.crop; + } + + @Override + public void close() { + mIsImageValid = false; + + if (mGraphicBuffer != null) { + ImageReader.unlockGraphicBuffer(mGraphicBuffer); + mGraphicBuffer.destroy(); + mGraphicBuffer = null; + } + + if (mPlanes != null) { + mPlanes = null; + } + + if (mParcelImage.buffer != null) { + mParcelImage.buffer.close(); + mParcelImage.buffer = null; + } + + if (mParcelImage.fence != null) { + try { + mParcelImage.fence.close(); + } catch (IOException e) { + e.printStackTrace(); + } + mParcelImage.fence = null; + } + } + } +}