From c8b181e95d17c00f9fe4a8338c4cdd0eeac3a0dc Mon Sep 17 00:00:00 2001 From: Zhijun He Date: Mon, 30 May 2016 14:54:39 -0700 Subject: [PATCH] Camera2: add deferred output config support Allow surfaces to be deferred during session creation. Once the surfaces are ready, the application can finish the deferred output configuration to be able to submit requests with these surface targets. Bug: 28323863 Change-Id: Id6634c3ef2ecc84422a88f63de0a19a0cb496e96 --- .../camera2/CameraCaptureSession.java | 48 +++++ .../impl/CameraCaptureSessionImpl.java | 7 + ...onstrainedHighSpeedCaptureSessionImpl.java | 13 ++ .../camera2/impl/CameraDeviceImpl.java | 37 +++- .../impl/ICameraDeviceUserWrapper.java | 9 + .../camera2/legacy/CameraDeviceUserShim.java | 7 + .../camera2/params/OutputConfiguration.java | 178 ++++++++++++++++-- 7 files changed, 287 insertions(+), 12 deletions(-) diff --git a/core/java/android/hardware/camera2/CameraCaptureSession.java b/core/java/android/hardware/camera2/CameraCaptureSession.java index 38279a47f2a9b..4b21187359836 100644 --- a/core/java/android/hardware/camera2/CameraCaptureSession.java +++ b/core/java/android/hardware/camera2/CameraCaptureSession.java @@ -18,6 +18,7 @@ package android.hardware.camera2; import android.annotation.NonNull; import android.annotation.Nullable; +import android.hardware.camera2.params.OutputConfiguration; import android.os.Handler; import android.view.Surface; @@ -219,6 +220,53 @@ public abstract class CameraCaptureSession implements AutoCloseable { */ public abstract void tearDown(@NonNull Surface surface) throws CameraAccessException; + /** + *

+ * Finish the deferred output configurations where the output Surface was not configured before. + *

+ *

+ * For camera use cases where a preview and other output configurations need to be configured, + * it can take some time for the preview Surface to be ready (e.g., if the preview Surface is + * obtained from {@link android.view.SurfaceView}, the SurfaceView is ready after the UI layout + * is done, then it takes some time to get the preview Surface). + *

+ *

+ * To speed up camera startup time, the application can configure the + * {@link CameraCaptureSession} with the desired preview size, and defer the preview output + * configuration until the Surface is ready. After the {@link CameraCaptureSession} is created + * successfully with this deferred configuration and other normal configurations, the + * application can submit requests that don't include deferred output Surfaces. Once the + * deferred Surface is ready, the application can set the Surface to the same deferred output + * configuration with the {@link OutputConfiguration#setDeferredSurface} method, and then finish + * the deferred output configuration via this method, before it can submit requests with this + * output target. + *

+ *

+ * The output Surfaces included by this list of deferred {@link OutputConfiguration + * OutputConfigurations} can be used as {@link CaptureRequest} targets as soon as this call + * returns; + *

+ *

+ * This method is not supported by Legacy devices. + *

+ * + * @param deferredOutputConfigs a list of {@link OutputConfiguration OutputConfigurations} that + * have had {@link OutputConfiguration#setDeferredSurface setDeferredSurface} invoked + * with a valid output Surface. + * @throws CameraAccessException if the camera device is no longer connected or has encountered + * a fatal error. + * @throws IllegalStateException if this session is no longer active, either because the session + * was explicitly closed, a new session has been created or the camera device has + * been closed. Or if this output configuration was already finished with the + * included surface before. + * @throws IllegalArgumentException for invalid output configurations, including ones where the + * source of the Surface is no longer valid or the Surface is from a unsupported + * source. + * @hide + */ + public abstract void finishDeferredConfiguration( + List deferredOutputConfigs) throws CameraAccessException; + /** *

Submit a request for an image to be captured by the camera device.

* diff --git a/core/java/android/hardware/camera2/impl/CameraCaptureSessionImpl.java b/core/java/android/hardware/camera2/impl/CameraCaptureSessionImpl.java index 6736d3466e2cd..c23bd5bcee472 100644 --- a/core/java/android/hardware/camera2/impl/CameraCaptureSessionImpl.java +++ b/core/java/android/hardware/camera2/impl/CameraCaptureSessionImpl.java @@ -24,6 +24,7 @@ import android.hardware.camera2.dispatch.BroadcastDispatcher; import android.hardware.camera2.dispatch.DuckTypingDispatcher; import android.hardware.camera2.dispatch.HandlerDispatcher; import android.hardware.camera2.dispatch.InvokeDispatcher; +import android.hardware.camera2.params.OutputConfiguration; import android.hardware.camera2.utils.TaskDrainer; import android.hardware.camera2.utils.TaskSingleDrainer; import android.os.Handler; @@ -155,6 +156,12 @@ public class CameraCaptureSessionImpl extends CameraCaptureSession mDeviceImpl.tearDown(surface); } + @Override + public void finishDeferredConfiguration( + List deferredOutputConfigs) throws CameraAccessException { + mDeviceImpl.finishDeferredConfig(deferredOutputConfigs); + } + @Override public synchronized int capture(CaptureRequest request, CaptureCallback callback, Handler handler) throws CameraAccessException { diff --git a/core/java/android/hardware/camera2/impl/CameraConstrainedHighSpeedCaptureSessionImpl.java b/core/java/android/hardware/camera2/impl/CameraConstrainedHighSpeedCaptureSessionImpl.java index 8cd1da50c58a3..1c8e124331124 100644 --- a/core/java/android/hardware/camera2/impl/CameraConstrainedHighSpeedCaptureSessionImpl.java +++ b/core/java/android/hardware/camera2/impl/CameraConstrainedHighSpeedCaptureSessionImpl.java @@ -21,6 +21,7 @@ import android.hardware.camera2.CameraCharacteristics; import android.hardware.camera2.CameraConstrainedHighSpeedCaptureSession; import android.hardware.camera2.CameraDevice; import android.hardware.camera2.CaptureRequest; +import android.hardware.camera2.params.OutputConfiguration; import android.hardware.camera2.params.StreamConfigurationMap; import android.hardware.camera2.utils.SurfaceUtils; import android.os.Handler; @@ -256,6 +257,12 @@ public class CameraConstrainedHighSpeedCaptureSessionImpl return mSessionImpl.isAborting(); } + @Override + public void finishDeferredConfiguration(List deferredOutputConfigs) + throws CameraAccessException { + mSessionImpl.finishDeferredConfiguration(deferredOutputConfigs); + } + private class WrapperCallback extends StateCallback { private final StateCallback mCallback; @@ -263,26 +270,32 @@ public class CameraConstrainedHighSpeedCaptureSessionImpl mCallback = callback; } + @Override public void onConfigured(CameraCaptureSession session) { mCallback.onConfigured(CameraConstrainedHighSpeedCaptureSessionImpl.this); } + @Override public void onConfigureFailed(CameraCaptureSession session) { mCallback.onConfigureFailed(CameraConstrainedHighSpeedCaptureSessionImpl.this); } + @Override public void onReady(CameraCaptureSession session) { mCallback.onReady(CameraConstrainedHighSpeedCaptureSessionImpl.this); } + @Override public void onActive(CameraCaptureSession session) { mCallback.onActive(CameraConstrainedHighSpeedCaptureSessionImpl.this); } + @Override public void onClosed(CameraCaptureSession session) { mCallback.onClosed(CameraConstrainedHighSpeedCaptureSessionImpl.this); } + @Override public void onSurfacePrepared(CameraCaptureSession session, Surface surface) { mCallback.onSurfacePrepared(CameraConstrainedHighSpeedCaptureSessionImpl.this, surface); diff --git a/core/java/android/hardware/camera2/impl/CameraDeviceImpl.java b/core/java/android/hardware/camera2/impl/CameraDeviceImpl.java index 5743b4d42a373..e184d6d076b06 100644 --- a/core/java/android/hardware/camera2/impl/CameraDeviceImpl.java +++ b/core/java/android/hardware/camera2/impl/CameraDeviceImpl.java @@ -407,7 +407,10 @@ public class CameraDeviceImpl extends CameraDevice int streamId = mConfiguredOutputs.keyAt(i); OutputConfiguration outConfig = mConfiguredOutputs.valueAt(i); - if (!outputs.contains(outConfig)) { + if (!outputs.contains(outConfig) || outConfig.isDeferredConfiguration()) { + // Always delete the deferred output configuration when the session + // is created, as the deferred output configuration doesn't have unique surface + // related identifies. deleteList.add(streamId); } else { addSet.remove(outConfig); // Don't create a stream previously created @@ -744,6 +747,37 @@ public class CameraDeviceImpl extends CameraDevice } } + public void finishDeferredConfig(List deferredConfigs) + throws CameraAccessException { + if (deferredConfigs == null || deferredConfigs.size() == 0) { + throw new IllegalArgumentException("deferred config is null or empty"); + } + + synchronized(mInterfaceLock) { + for (OutputConfiguration config : deferredConfigs) { + int streamId = -1; + for (int i = 0; i < mConfiguredOutputs.size(); i++) { + // Have to use equal here, as createCaptureSessionByOutputConfigurations() and + // createReprocessableCaptureSessionByConfigurations() do a copy of the configs. + if (config.equals(mConfiguredOutputs.valueAt(i))) { + streamId = mConfiguredOutputs.keyAt(i); + break; + } + } + if (streamId == -1) { + throw new IllegalArgumentException("Deferred config is not part of this " + + "session"); + } + + if (config.getSurface() == null) { + throw new IllegalArgumentException("The deferred config for stream " + streamId + + " must have a non-null surface"); + } + mRemoteDevice.setDeferredConfiguration(streamId, config); + } + } + } + public int capture(CaptureRequest request, CaptureCallback callback, Handler handler) throws CameraAccessException { if (DEBUG) { @@ -2037,6 +2071,7 @@ public class CameraDeviceImpl extends CameraDevice * *

Handle binder death for ICameraDeviceUser. Trigger onError.

*/ + @Override public void binderDied() { Log.w(TAG, "CameraDevice " + mCameraId + " died unexpectedly"); diff --git a/core/java/android/hardware/camera2/impl/ICameraDeviceUserWrapper.java b/core/java/android/hardware/camera2/impl/ICameraDeviceUserWrapper.java index ef5f6d71b8b6e..d77f60bf0953d 100644 --- a/core/java/android/hardware/camera2/impl/ICameraDeviceUserWrapper.java +++ b/core/java/android/hardware/camera2/impl/ICameraDeviceUserWrapper.java @@ -215,5 +215,14 @@ public class ICameraDeviceUserWrapper { } } + public void setDeferredConfiguration(int streamId, OutputConfiguration deferredConfig) + throws CameraAccessException { + try { + mRemoteDevice.setDeferredConfiguration(streamId, deferredConfig); + } catch (Throwable t) { + CameraManager.throwAsPublicException(t); + throw new UnsupportedOperationException("Unexpected exception", t); + } + } } diff --git a/core/java/android/hardware/camera2/legacy/CameraDeviceUserShim.java b/core/java/android/hardware/camera2/legacy/CameraDeviceUserShim.java index acbf214942c52..b9e75eec1a34c 100644 --- a/core/java/android/hardware/camera2/legacy/CameraDeviceUserShim.java +++ b/core/java/android/hardware/camera2/legacy/CameraDeviceUserShim.java @@ -566,6 +566,13 @@ public class CameraDeviceUserShim implements ICameraDeviceUser { } } + @Override + public void setDeferredConfiguration(int steamId, OutputConfiguration config) { + String err = "Set deferred configuration is not supported on legacy devices"; + Log.e(TAG, err); + throw new ServiceSpecificException(ICameraService.ERROR_INVALID_OPERATION, err); + } + @Override public int createInputStream(int width, int height, int format) { String err = "Creating input stream is not supported on legacy devices"; diff --git a/core/java/android/hardware/camera2/params/OutputConfiguration.java b/core/java/android/hardware/camera2/params/OutputConfiguration.java index 61b534bf22deb..69c00e9873021 100644 --- a/core/java/android/hardware/camera2/params/OutputConfiguration.java +++ b/core/java/android/hardware/camera2/params/OutputConfiguration.java @@ -20,6 +20,8 @@ package android.hardware.camera2.params; import android.annotation.NonNull; import android.annotation.Nullable; import android.annotation.SystemApi; +import android.graphics.ImageFormat; +import android.hardware.camera2.CameraCaptureSession; import android.hardware.camera2.CameraDevice; import android.hardware.camera2.utils.HashCodeHelpers; import android.hardware.camera2.utils.SurfaceUtils; @@ -94,6 +96,21 @@ public final class OutputConfiguration implements Parcelable { this(SURFACE_GROUP_ID_NONE, surface, ROTATION_0); } + /** + * Unknown surface source type. + */ + private final int SURFACE_TYPE_UNKNOWN = -1; + + /** + * The surface is obtained from {@link android.view.SurfaceView}. + */ + private final int SURFACE_TYPE_SURFACE_VIEW = 0; + + /** + * The surface is obtained from {@link android.graphics.SurfaceTexture}. + */ + private final int SURFACE_TYPE_SURFACE_TEXTURE = 1; + /** * Create a new {@link OutputConfiguration} instance with a {@link Surface}, * with a surface group ID. @@ -179,12 +196,110 @@ public final class OutputConfiguration implements Parcelable { checkNotNull(surface, "Surface must not be null"); checkArgumentInRange(rotation, ROTATION_0, ROTATION_270, "Rotation constant"); mSurfaceGroupId = surfaceGroupId; + mSurfaceType = SURFACE_TYPE_UNKNOWN; mSurface = surface; mRotation = rotation; mConfiguredSize = SurfaceUtils.getSurfaceSize(surface); mConfiguredFormat = SurfaceUtils.getSurfaceFormat(surface); mConfiguredDataspace = SurfaceUtils.getSurfaceDataspace(surface); mConfiguredGenerationId = surface.getGenerationId(); + mIsDeferredConfig = false; + } + + /** + * Create a new {@link OutputConfiguration} instance, with desired Surface size and Surface + * source class. + *

+ * This constructor takes an argument for desired Surface size and the Surface source class + * without providing the actual output Surface. This is used to setup a output configuration + * with a deferred Surface. The application can use this output configuration to create a + * session. + *

+ *

+ * However, the actual output Surface must be set via {@link #setDeferredSurface} and finish the + * deferred Surface configuration via {@link CameraCaptureSession#finishDeferredConfiguration} + * before submitting a request with this Surface target. The deferred Surface can only be + * obtained from either from {@link android.view.SurfaceView} by calling + * {@link android.view.SurfaceHolder#getSurface}, or from + * {@link android.graphics.SurfaceTexture} via + * {@link android.view.Surface#Surface(android.graphics.SurfaceTexture)}). + *

+ * + * @param surfaceSize Size for the deferred surface. + * @param klass a non-{@code null} {@link Class} object reference that indicates the source of + * this surface. Only {@link android.view.SurfaceHolder SurfaceHolder.class} and + * {@link android.graphics.SurfaceTexture SurfaceTexture.class} are supported. + * @hide + */ + public OutputConfiguration(@NonNull Size surfaceSize, @NonNull Class klass) { + checkNotNull(klass, "surfaceSize must not be null"); + checkNotNull(klass, "klass must not be null"); + if (klass == android.view.SurfaceHolder.class) { + mSurfaceType = SURFACE_TYPE_SURFACE_VIEW; + } else if (klass == android.graphics.SurfaceTexture.class) { + mSurfaceType = SURFACE_TYPE_SURFACE_TEXTURE; + } else { + mSurfaceType = SURFACE_TYPE_UNKNOWN; + throw new IllegalArgumentException("Unknow surface source class type"); + } + + mSurfaceGroupId = SURFACE_GROUP_ID_NONE; + mSurface = null; + mRotation = ROTATION_0; + mConfiguredSize = surfaceSize; + mConfiguredFormat = StreamConfigurationMap.imageFormatToInternal(ImageFormat.PRIVATE); + mConfiguredDataspace = StreamConfigurationMap.imageFormatToDataspace(ImageFormat.PRIVATE); + mConfiguredGenerationId = 0; + mIsDeferredConfig = true; + } + + /** + * Check if this configuration has deferred configuration. + * + *

This will return true if the output configuration was constructed with surface deferred. + * It will return true even after the deferred surface is set later.

+ * + * @return true if this configuration has deferred surface. + * @hide + */ + public boolean isDeferredConfiguration() { + return mIsDeferredConfig; + } + + /** + * Set the deferred surface to this OutputConfiguration. + * + *

+ * The deferred surface must be obtained from either from {@link android.view.SurfaceView} by + * calling {@link android.view.SurfaceHolder#getSurface}, or from + * {@link android.graphics.SurfaceTexture} via + * {@link android.view.Surface#Surface(android.graphics.SurfaceTexture)}). After the deferred + * surface is set, the application must finish the deferred surface configuration via + * {@link CameraCaptureSession#finishDeferredConfiguration} before submitting a request with + * this surface target. + *

+ * + * @param surface The deferred surface to be set. + * @throws IllegalArgumentException if the Surface is invalid. + * @throws IllegalStateException if a Surface was already set to this deferred + * OutputConfiguration. + * @hide + */ + public void setDeferredSurface(@NonNull Surface surface) { + checkNotNull(surface, "Surface must not be null"); + if (mSurface != null) { + throw new IllegalStateException("Deferred surface is already set!"); + } + + // This will throw IAE is the surface was abandoned. + Size surfaceSize = SurfaceUtils.getSurfaceSize(surface); + if (!surfaceSize.equals(mConfiguredSize)) { + Log.w(TAG, "Deferred surface size " + surfaceSize + + " is different with pre-configured size " + mConfiguredSize + + ", the pre-configured size will be used."); + } + + mSurface = surface; } /** @@ -203,10 +318,12 @@ public final class OutputConfiguration implements Parcelable { this.mSurface = other.mSurface; this.mRotation = other.mRotation; this.mSurfaceGroupId = other.mSurfaceGroupId; + this.mSurfaceType = other.mSurfaceType; this.mConfiguredDataspace = other.mConfiguredDataspace; this.mConfiguredFormat = other.mConfiguredFormat; this.mConfiguredSize = other.mConfiguredSize; this.mConfiguredGenerationId = other.mConfiguredGenerationId; + this.mIsDeferredConfig = other.mIsDeferredConfig; } /** @@ -215,16 +332,30 @@ public final class OutputConfiguration implements Parcelable { private OutputConfiguration(@NonNull Parcel source) { int rotation = source.readInt(); int surfaceSetId = source.readInt(); + int surfaceType = source.readInt(); + int width = source.readInt(); + int height = source.readInt(); Surface surface = Surface.CREATOR.createFromParcel(source); - checkNotNull(surface, "Surface must not be null"); checkArgumentInRange(rotation, ROTATION_0, ROTATION_270, "Rotation constant"); mSurfaceGroupId = surfaceSetId; mSurface = surface; mRotation = rotation; - mConfiguredSize = SurfaceUtils.getSurfaceSize(mSurface); - mConfiguredFormat = SurfaceUtils.getSurfaceFormat(mSurface); - mConfiguredDataspace = SurfaceUtils.getSurfaceDataspace(mSurface); - mConfiguredGenerationId = mSurface.getGenerationId(); + if (surface != null) { + mSurfaceType = SURFACE_TYPE_UNKNOWN; + mConfiguredSize = SurfaceUtils.getSurfaceSize(mSurface); + mConfiguredFormat = SurfaceUtils.getSurfaceFormat(mSurface); + mConfiguredDataspace = SurfaceUtils.getSurfaceDataspace(mSurface); + mConfiguredGenerationId = mSurface.getGenerationId(); + mIsDeferredConfig = true; + } else { + mSurfaceType = surfaceType; + mConfiguredSize = new Size(width, height); + mConfiguredFormat = StreamConfigurationMap.imageFormatToInternal(ImageFormat.PRIVATE); + mConfiguredGenerationId = 0; + mConfiguredDataspace = + StreamConfigurationMap.imageFormatToDataspace(ImageFormat.PRIVATE); + mIsDeferredConfig = false; + } } /** @@ -291,7 +422,12 @@ public final class OutputConfiguration implements Parcelable { } dest.writeInt(mRotation); dest.writeInt(mSurfaceGroupId); - mSurface.writeToParcel(dest, flags); + dest.writeInt(mSurfaceType); + dest.writeInt(mConfiguredSize.getWidth()); + dest.writeInt(mConfiguredSize.getHeight()); + if (mSurface != null) { + mSurface.writeToParcel(dest, flags); + } } /** @@ -311,13 +447,20 @@ public final class OutputConfiguration implements Parcelable { return true; } else if (obj instanceof OutputConfiguration) { final OutputConfiguration other = (OutputConfiguration) obj; + boolean iSSurfaceEqual = mSurface == other.mSurface && + mConfiguredGenerationId == other.mConfiguredGenerationId ; + if (mIsDeferredConfig) { + Log.i(TAG, "deferred config has the same surface"); + iSSurfaceEqual = true; + } return mRotation == other.mRotation && - mSurface == other.mSurface && - mConfiguredGenerationId == other.mConfiguredGenerationId && + iSSurfaceEqual&& mConfiguredSize.equals(other.mConfiguredSize) && mConfiguredFormat == other.mConfiguredFormat && mConfiguredDataspace == other.mConfiguredDataspace && - mSurfaceGroupId == other.mSurfaceGroupId; + mSurfaceGroupId == other.mSurfaceGroupId && + mSurfaceType == other.mSurfaceType && + mIsDeferredConfig == other.mIsDeferredConfig; } return false; } @@ -327,15 +470,26 @@ public final class OutputConfiguration implements Parcelable { */ @Override public int hashCode() { + // Need ensure that the hashcode remains unchanged after set a deferred surface. Otherwise + // The deferred output configuration will be lost in the camera streammap after the deferred + // surface is set. + if (mIsDeferredConfig) { + return HashCodeHelpers.hashCode( + mRotation, mConfiguredSize.hashCode(), mConfiguredFormat, mConfiguredDataspace, + mSurfaceGroupId, mSurfaceType); + } + return HashCodeHelpers.hashCode( mRotation, mSurface.hashCode(), mConfiguredGenerationId, mConfiguredSize.hashCode(), mConfiguredFormat, mConfiguredDataspace, mSurfaceGroupId); } private static final String TAG = "OutputConfiguration"; - private final Surface mSurface; + private Surface mSurface; private final int mRotation; - private int mSurfaceGroupId; + private final int mSurfaceGroupId; + // Surface source type, this is only used by the deferred surface configuration objects. + private final int mSurfaceType; // The size, format, and dataspace of the surface when OutputConfiguration is created. private final Size mConfiguredSize; @@ -343,4 +497,6 @@ public final class OutputConfiguration implements Parcelable { private final int mConfiguredDataspace; // Surface generation ID to distinguish changes to Surface native internals private final int mConfiguredGenerationId; + // Flag indicating if this config has deferred surface. + private final boolean mIsDeferredConfig; }