Merge "Camera2: add deferred output config support" into nyc-mr1-dev

am: e8526a0105

* commit 'e8526a01051fc2b8929870c8bff28f1d6b8b652c':
  Camera2: add deferred output config support

Change-Id: I5bfa1c90e4ab5910ccd3a8727b95200e4b4221ca

core/java/android/hardware/camera2/CameraCaptureSession.java

@@ -18,6 +18,7 @@ package android.hardware.camera2;
 
 import android.annotation.NonNull;
 import android.annotation.Nullable;
+import android.hardware.camera2.params.OutputConfiguration;
 import android.os.Handler;
 import android.view.Surface;
 
@@ -219,6 +220,53 @@ public abstract class CameraCaptureSession implements AutoCloseable {
      */
     public abstract void tearDown(@NonNull Surface surface) throws CameraAccessException;
 
+    /**
+     * <p>
+     * Finish the deferred output configurations where the output Surface was not configured before.
+     * </p>
+     * <p>
+     * For camera use cases where a preview and other output configurations need to be configured,
+     * it can take some time for the preview Surface to be ready (e.g., if the preview Surface is
+     * obtained from {@link android.view.SurfaceView}, the SurfaceView is ready after the UI layout
+     * is done, then it takes some time to get the preview Surface).
+     * </p>
+     * <p>
+     * To speed up camera startup time, the application can configure the
+     * {@link CameraCaptureSession} with the desired preview size, and defer the preview output
+     * configuration until the Surface is ready. After the {@link CameraCaptureSession} is created
+     * successfully with this deferred configuration and other normal configurations, the
+     * application can submit requests that don't include deferred output Surfaces. Once the
+     * deferred Surface is ready, the application can set the Surface to the same deferred output
+     * configuration with the {@link OutputConfiguration#setDeferredSurface} method, and then finish
+     * the deferred output configuration via this method, before it can submit requests with this
+     * output target.
+     * </p>
+     * <p>
+     * The output Surfaces included by this list of deferred {@link OutputConfiguration
+     * OutputConfigurations} can be used as {@link CaptureRequest} targets as soon as this call
+     * returns;
+     * </p>
+     * <p>
+     * This method is not supported by Legacy devices.
+     * </p>
+     *
+     * @param deferredOutputConfigs a list of {@link OutputConfiguration OutputConfigurations} that
+     *            have had {@link OutputConfiguration#setDeferredSurface setDeferredSurface} invoked
+     *            with a valid output Surface.
+     * @throws CameraAccessException if the camera device is no longer connected or has encountered
+     *             a fatal error.
+     * @throws IllegalStateException if this session is no longer active, either because the session
+     *             was explicitly closed, a new session has been created or the camera device has
+     *             been closed. Or if this output configuration was already finished with the
+     *             included surface before.
+     * @throws IllegalArgumentException for invalid output configurations, including ones where the
+     *             source of the Surface is no longer valid or the Surface is from a unsupported
+     *             source.
+     * @hide
+     */
+    public abstract void finishDeferredConfiguration(
+            List<OutputConfiguration> deferredOutputConfigs) throws CameraAccessException;
+
     /**
      * <p>Submit a request for an image to be captured by the camera device.</p>
      *

core/java/android/hardware/camera2/impl/CameraCaptureSessionImpl.java

@@ -24,6 +24,7 @@ import android.hardware.camera2.dispatch.BroadcastDispatcher;
 import android.hardware.camera2.dispatch.DuckTypingDispatcher;
 import android.hardware.camera2.dispatch.HandlerDispatcher;
 import android.hardware.camera2.dispatch.InvokeDispatcher;
+import android.hardware.camera2.params.OutputConfiguration;
 import android.hardware.camera2.utils.TaskDrainer;
 import android.hardware.camera2.utils.TaskSingleDrainer;
 import android.os.Handler;
@@ -155,6 +156,12 @@ public class CameraCaptureSessionImpl extends CameraCaptureSession
         mDeviceImpl.tearDown(surface);
     }
 
+    @Override
+    public void finishDeferredConfiguration(
+            List<OutputConfiguration> deferredOutputConfigs) throws CameraAccessException {
+        mDeviceImpl.finishDeferredConfig(deferredOutputConfigs);
+    }
+
     @Override
     public synchronized int capture(CaptureRequest request, CaptureCallback callback,
             Handler handler) throws CameraAccessException {

core/java/android/hardware/camera2/impl/CameraConstrainedHighSpeedCaptureSessionImpl.java

@@ -21,6 +21,7 @@ import android.hardware.camera2.CameraCharacteristics;
 import android.hardware.camera2.CameraConstrainedHighSpeedCaptureSession;
 import android.hardware.camera2.CameraDevice;
 import android.hardware.camera2.CaptureRequest;
+import android.hardware.camera2.params.OutputConfiguration;
 import android.hardware.camera2.params.StreamConfigurationMap;
 import android.hardware.camera2.utils.SurfaceUtils;
 import android.os.Handler;
@@ -256,6 +257,12 @@ public class CameraConstrainedHighSpeedCaptureSessionImpl
         return mSessionImpl.isAborting();
     }
 
+    @Override
+    public void finishDeferredConfiguration(List<OutputConfiguration> deferredOutputConfigs)
+            throws CameraAccessException {
+        mSessionImpl.finishDeferredConfiguration(deferredOutputConfigs);
+    }
+
     private class WrapperCallback extends StateCallback {
         private final StateCallback mCallback;
 
@@ -263,26 +270,32 @@ public class CameraConstrainedHighSpeedCaptureSessionImpl
             mCallback = callback;
         }
 
+        @Override
         public void onConfigured(CameraCaptureSession session) {
             mCallback.onConfigured(CameraConstrainedHighSpeedCaptureSessionImpl.this);
         }
 
+        @Override
         public void onConfigureFailed(CameraCaptureSession session) {
             mCallback.onConfigureFailed(CameraConstrainedHighSpeedCaptureSessionImpl.this);
         }
 
+        @Override
         public void onReady(CameraCaptureSession session) {
             mCallback.onReady(CameraConstrainedHighSpeedCaptureSessionImpl.this);
         }
 
+        @Override
         public void onActive(CameraCaptureSession session) {
             mCallback.onActive(CameraConstrainedHighSpeedCaptureSessionImpl.this);
         }
 
+        @Override
         public void onClosed(CameraCaptureSession session) {
             mCallback.onClosed(CameraConstrainedHighSpeedCaptureSessionImpl.this);
         }
 
+        @Override
         public void onSurfacePrepared(CameraCaptureSession session, Surface surface) {
             mCallback.onSurfacePrepared(CameraConstrainedHighSpeedCaptureSessionImpl.this,
                     surface);

core/java/android/hardware/camera2/impl/CameraDeviceImpl.java

@@ -407,7 +407,10 @@ public class CameraDeviceImpl extends CameraDevice
                 int streamId = mConfiguredOutputs.keyAt(i);
                 OutputConfiguration outConfig = mConfiguredOutputs.valueAt(i);
 
-                if (!outputs.contains(outConfig)) {
+                if (!outputs.contains(outConfig) || outConfig.isDeferredConfiguration()) {
+                    // Always delete the deferred output configuration when the session
+                    // is created, as the deferred output configuration doesn't have unique surface
+                    // related identifies.
                     deleteList.add(streamId);
                 } else {
                     addSet.remove(outConfig);  // Don't create a stream previously created
@@ -744,6 +747,37 @@ public class CameraDeviceImpl extends CameraDevice
         }
     }
 
+    public void finishDeferredConfig(List<OutputConfiguration> deferredConfigs)
+            throws CameraAccessException {
+        if (deferredConfigs == null || deferredConfigs.size() == 0) {
+            throw new IllegalArgumentException("deferred config is null or empty");
+        }
+
+        synchronized(mInterfaceLock) {
+            for (OutputConfiguration config : deferredConfigs) {
+                int streamId = -1;
+                for (int i = 0; i < mConfiguredOutputs.size(); i++) {
+                    // Have to use equal here, as createCaptureSessionByOutputConfigurations() and
+                    // createReprocessableCaptureSessionByConfigurations() do a copy of the configs.
+                    if (config.equals(mConfiguredOutputs.valueAt(i))) {
+                        streamId = mConfiguredOutputs.keyAt(i);
+                        break;
+                    }
+                }
+                if (streamId == -1) {
+                    throw new IllegalArgumentException("Deferred config is not part of this "
+                            + "session");
+                }
+
+                if (config.getSurface() == null) {
+                    throw new IllegalArgumentException("The deferred config for stream " + streamId
+                            + " must have a non-null surface");
+                }
+                mRemoteDevice.setDeferredConfiguration(streamId, config);
+            }
+        }
+    }
+
     public int capture(CaptureRequest request, CaptureCallback callback, Handler handler)
             throws CameraAccessException {
         if (DEBUG) {
@@ -2037,6 +2071,7 @@ public class CameraDeviceImpl extends CameraDevice
      *
      * <p> Handle binder death for ICameraDeviceUser. Trigger onError.</p>
      */
+    @Override
     public void binderDied() {
         Log.w(TAG, "CameraDevice " + mCameraId + " died unexpectedly");
 

core/java/android/hardware/camera2/impl/ICameraDeviceUserWrapper.java

@@ -215,5 +215,14 @@ public class ICameraDeviceUserWrapper {
         }
     }
 
+    public void setDeferredConfiguration(int streamId, OutputConfiguration deferredConfig)
+            throws CameraAccessException {
+        try {
+            mRemoteDevice.setDeferredConfiguration(streamId, deferredConfig);
+        } catch (Throwable t) {
+            CameraManager.throwAsPublicException(t);
+            throw new UnsupportedOperationException("Unexpected exception", t);
+        }
+    }
 
 }

core/java/android/hardware/camera2/legacy/CameraDeviceUserShim.java

@@ -566,6 +566,13 @@ public class CameraDeviceUserShim implements ICameraDeviceUser {
         }
     }
 
+    @Override
+    public void setDeferredConfiguration(int steamId, OutputConfiguration config) {
+        String err = "Set deferred configuration is not supported on legacy devices";
+        Log.e(TAG, err);
+        throw new ServiceSpecificException(ICameraService.ERROR_INVALID_OPERATION, err);
+    }
+
     @Override
     public int createInputStream(int width, int height, int format) {
         String err = "Creating input stream is not supported on legacy devices";

core/java/android/hardware/camera2/params/OutputConfiguration.java

@@ -20,6 +20,8 @@ package android.hardware.camera2.params;
 import android.annotation.NonNull;
 import android.annotation.Nullable;
 import android.annotation.SystemApi;
+import android.graphics.ImageFormat;
+import android.hardware.camera2.CameraCaptureSession;
 import android.hardware.camera2.CameraDevice;
 import android.hardware.camera2.utils.HashCodeHelpers;
 import android.hardware.camera2.utils.SurfaceUtils;
@@ -94,6 +96,21 @@ public final class OutputConfiguration implements Parcelable {
         this(SURFACE_GROUP_ID_NONE, surface, ROTATION_0);
     }
 
+    /**
+     * Unknown surface source type.
+     */
+    private final int SURFACE_TYPE_UNKNOWN = -1;
+
+    /**
+     * The surface is obtained from {@link android.view.SurfaceView}.
+     */
+    private final int SURFACE_TYPE_SURFACE_VIEW = 0;
+
+    /**
+     * The surface is obtained from {@link android.graphics.SurfaceTexture}.
+     */
+    private final int SURFACE_TYPE_SURFACE_TEXTURE = 1;
+
     /**
      * Create a new {@link OutputConfiguration} instance with a {@link Surface},
      * with a surface group ID.
@@ -179,12 +196,110 @@ public final class OutputConfiguration implements Parcelable {
         checkNotNull(surface, "Surface must not be null");
         checkArgumentInRange(rotation, ROTATION_0, ROTATION_270, "Rotation constant");
         mSurfaceGroupId = surfaceGroupId;
+        mSurfaceType = SURFACE_TYPE_UNKNOWN;
         mSurface = surface;
         mRotation = rotation;
         mConfiguredSize = SurfaceUtils.getSurfaceSize(surface);
         mConfiguredFormat = SurfaceUtils.getSurfaceFormat(surface);
         mConfiguredDataspace = SurfaceUtils.getSurfaceDataspace(surface);
         mConfiguredGenerationId = surface.getGenerationId();
+        mIsDeferredConfig = false;
+    }
+
+    /**
+     * Create a new {@link OutputConfiguration} instance, with desired Surface size and Surface
+     * source class.
+     * <p>
+     * This constructor takes an argument for desired Surface size and the Surface source class
+     * without providing the actual output Surface. This is used to setup a output configuration
+     * with a deferred Surface. The application can use this output configuration to create a
+     * session.
+     * </p>
+     * <p>
+     * However, the actual output Surface must be set via {@link #setDeferredSurface} and finish the
+     * deferred Surface configuration via {@link CameraCaptureSession#finishDeferredConfiguration}
+     * before submitting a request with this Surface target. The deferred Surface can only be
+     * obtained from either from {@link android.view.SurfaceView} by calling
+     * {@link android.view.SurfaceHolder#getSurface}, or from
+     * {@link android.graphics.SurfaceTexture} via
+     * {@link android.view.Surface#Surface(android.graphics.SurfaceTexture)}).
+     * </p>
+     *
+     * @param surfaceSize Size for the deferred surface.
+     * @param klass a non-{@code null} {@link Class} object reference that indicates the source of
+     *            this surface. Only {@link android.view.SurfaceHolder SurfaceHolder.class} and
+     *            {@link android.graphics.SurfaceTexture SurfaceTexture.class} are supported.
+     * @hide
+     */
+    public <T> OutputConfiguration(@NonNull Size surfaceSize, @NonNull Class<T> klass) {
+        checkNotNull(klass, "surfaceSize must not be null");
+        checkNotNull(klass, "klass must not be null");
+        if (klass == android.view.SurfaceHolder.class) {
+            mSurfaceType = SURFACE_TYPE_SURFACE_VIEW;
+        } else if (klass == android.graphics.SurfaceTexture.class) {
+            mSurfaceType = SURFACE_TYPE_SURFACE_TEXTURE;
+        } else {
+            mSurfaceType = SURFACE_TYPE_UNKNOWN;
+            throw new IllegalArgumentException("Unknow surface source class type");
+        }
+
+        mSurfaceGroupId = SURFACE_GROUP_ID_NONE;
+        mSurface = null;
+        mRotation = ROTATION_0;
+        mConfiguredSize = surfaceSize;
+        mConfiguredFormat = StreamConfigurationMap.imageFormatToInternal(ImageFormat.PRIVATE);
+        mConfiguredDataspace = StreamConfigurationMap.imageFormatToDataspace(ImageFormat.PRIVATE);
+        mConfiguredGenerationId = 0;
+        mIsDeferredConfig = true;
+    }
+
+    /**
+     * Check if this configuration has deferred configuration.
+     *
+     * <p>This will return true if the output configuration was constructed with surface deferred.
+     * It will return true even after the deferred surface is set later.</p>
+     *
+     * @return true if this configuration has deferred surface.
+     * @hide
+     */
+    public boolean isDeferredConfiguration() {
+        return mIsDeferredConfig;
+    }
+
+    /**
+     * Set the deferred surface to this OutputConfiguration.
+     *
+     * <p>
+     * The deferred surface must be obtained from either from {@link android.view.SurfaceView} by
+     * calling {@link android.view.SurfaceHolder#getSurface}, or from
+     * {@link android.graphics.SurfaceTexture} via
+     * {@link android.view.Surface#Surface(android.graphics.SurfaceTexture)}). After the deferred
+     * surface is set, the application must finish the deferred surface configuration via
+     * {@link CameraCaptureSession#finishDeferredConfiguration} before submitting a request with
+     * this surface target.
+     * </p>
+     *
+     * @param surface The deferred surface to be set.
+     * @throws IllegalArgumentException if the Surface is invalid.
+     * @throws IllegalStateException if a Surface was already set to this deferred
+     *         OutputConfiguration.
+     * @hide
+     */
+    public void setDeferredSurface(@NonNull Surface surface) {
+        checkNotNull(surface, "Surface must not be null");
+        if (mSurface != null) {
+            throw new IllegalStateException("Deferred surface is already set!");
+        }
+
+        // This will throw IAE is the surface was abandoned.
+        Size surfaceSize = SurfaceUtils.getSurfaceSize(surface);
+        if (!surfaceSize.equals(mConfiguredSize)) {
+            Log.w(TAG, "Deferred surface size " + surfaceSize +
+                    " is different with pre-configured size " + mConfiguredSize +
+                    ", the pre-configured size will be used.");
+        }
+
+        mSurface = surface;
     }
 
     /**
@@ -203,10 +318,12 @@ public final class OutputConfiguration implements Parcelable {
         this.mSurface = other.mSurface;
         this.mRotation = other.mRotation;
         this.mSurfaceGroupId = other.mSurfaceGroupId;
+        this.mSurfaceType = other.mSurfaceType;
         this.mConfiguredDataspace = other.mConfiguredDataspace;
         this.mConfiguredFormat = other.mConfiguredFormat;
         this.mConfiguredSize = other.mConfiguredSize;
         this.mConfiguredGenerationId = other.mConfiguredGenerationId;
+        this.mIsDeferredConfig = other.mIsDeferredConfig;
     }
 
     /**
@@ -215,16 +332,30 @@ public final class OutputConfiguration implements Parcelable {
     private OutputConfiguration(@NonNull Parcel source) {
         int rotation = source.readInt();
         int surfaceSetId = source.readInt();
+        int surfaceType = source.readInt();
+        int width = source.readInt();
+        int height = source.readInt();
         Surface surface = Surface.CREATOR.createFromParcel(source);
-        checkNotNull(surface, "Surface must not be null");
         checkArgumentInRange(rotation, ROTATION_0, ROTATION_270, "Rotation constant");
         mSurfaceGroupId = surfaceSetId;
         mSurface = surface;
         mRotation = rotation;
-        mConfiguredSize = SurfaceUtils.getSurfaceSize(mSurface);
+        if (surface != null) {
-        mConfiguredFormat = SurfaceUtils.getSurfaceFormat(mSurface);
+            mSurfaceType = SURFACE_TYPE_UNKNOWN;
-        mConfiguredDataspace = SurfaceUtils.getSurfaceDataspace(mSurface);
+            mConfiguredSize = SurfaceUtils.getSurfaceSize(mSurface);
-        mConfiguredGenerationId = mSurface.getGenerationId();
+            mConfiguredFormat = SurfaceUtils.getSurfaceFormat(mSurface);
+            mConfiguredDataspace = SurfaceUtils.getSurfaceDataspace(mSurface);
+            mConfiguredGenerationId = mSurface.getGenerationId();
+            mIsDeferredConfig = true;
+        } else {
+            mSurfaceType = surfaceType;
+            mConfiguredSize = new Size(width, height);
+            mConfiguredFormat = StreamConfigurationMap.imageFormatToInternal(ImageFormat.PRIVATE);
+            mConfiguredGenerationId = 0;
+            mConfiguredDataspace =
+                    StreamConfigurationMap.imageFormatToDataspace(ImageFormat.PRIVATE);
+            mIsDeferredConfig = false;
+        }
     }
 
     /**
@@ -291,7 +422,12 @@ public final class OutputConfiguration implements Parcelable {
         }
         dest.writeInt(mRotation);
         dest.writeInt(mSurfaceGroupId);
-        mSurface.writeToParcel(dest, flags);
+        dest.writeInt(mSurfaceType);
+        dest.writeInt(mConfiguredSize.getWidth());
+        dest.writeInt(mConfiguredSize.getHeight());
+        if (mSurface != null) {
+            mSurface.writeToParcel(dest, flags);
+        }
     }
 
     /**
@@ -311,13 +447,20 @@ public final class OutputConfiguration implements Parcelable {
             return true;
         } else if (obj instanceof OutputConfiguration) {
             final OutputConfiguration other = (OutputConfiguration) obj;
+            boolean iSSurfaceEqual = mSurface == other.mSurface &&
+                    mConfiguredGenerationId == other.mConfiguredGenerationId ;
+            if (mIsDeferredConfig) {
+                Log.i(TAG, "deferred config has the same surface");
+                iSSurfaceEqual = true;
+            }
             return mRotation == other.mRotation &&
-                   mSurface == other.mSurface &&
+                   iSSurfaceEqual&&
-                   mConfiguredGenerationId == other.mConfiguredGenerationId &&
                    mConfiguredSize.equals(other.mConfiguredSize) &&
                    mConfiguredFormat == other.mConfiguredFormat &&
                    mConfiguredDataspace == other.mConfiguredDataspace &&
-                   mSurfaceGroupId == other.mSurfaceGroupId;
+                   mSurfaceGroupId == other.mSurfaceGroupId &&
+                   mSurfaceType == other.mSurfaceType &&
+                   mIsDeferredConfig == other.mIsDeferredConfig;
         }
         return false;
     }
@@ -327,15 +470,26 @@ public final class OutputConfiguration implements Parcelable {
      */
     @Override
     public int hashCode() {
+        // Need ensure that the hashcode remains unchanged after set a deferred surface. Otherwise
+        // The deferred output configuration will be lost in the camera streammap after the deferred
+        // surface is set.
+        if (mIsDeferredConfig) {
+            return HashCodeHelpers.hashCode(
+                    mRotation, mConfiguredSize.hashCode(), mConfiguredFormat, mConfiguredDataspace,
+                    mSurfaceGroupId, mSurfaceType);
+        }
+
         return HashCodeHelpers.hashCode(
             mRotation, mSurface.hashCode(), mConfiguredGenerationId,
             mConfiguredSize.hashCode(), mConfiguredFormat, mConfiguredDataspace, mSurfaceGroupId);
     }
 
     private static final String TAG = "OutputConfiguration";
-    private final Surface mSurface;
+    private Surface mSurface;
     private final int mRotation;
-    private int mSurfaceGroupId;
+    private final int mSurfaceGroupId;
+    // Surface source type, this is only used by the deferred surface configuration objects.
+    private final int mSurfaceType;
 
     // The size, format, and dataspace of the surface when OutputConfiguration is created.
     private final Size mConfiguredSize;
@@ -343,4 +497,6 @@ public final class OutputConfiguration implements Parcelable {
     private final int mConfiguredDataspace;
     // Surface generation ID to distinguish changes to Surface native internals
     private final int mConfiguredGenerationId;
+    // Flag indicating if this config has deferred surface.
+    private final boolean mIsDeferredConfig;
 }