diff --git a/core/api/current.txt b/core/api/current.txt index a0bec24e65f6d..96df429f26d01 100644 --- a/core/api/current.txt +++ b/core/api/current.txt @@ -17771,6 +17771,7 @@ package android.hardware.camera2 { field @NonNull public static final android.hardware.camera2.CameraCharacteristics.Key SCALER_DEFAULT_SECURE_IMAGE_SIZE; field @NonNull public static final android.hardware.camera2.CameraCharacteristics.Key SCALER_MANDATORY_CONCURRENT_STREAM_COMBINATIONS; field @NonNull public static final android.hardware.camera2.CameraCharacteristics.Key SCALER_MANDATORY_STREAM_COMBINATIONS; + field @NonNull public static final android.hardware.camera2.CameraCharacteristics.Key SCALER_MULTI_RESOLUTION_STREAM_CONFIGURATION_MAP; field @NonNull public static final android.hardware.camera2.CameraCharacteristics.Key SCALER_STREAM_CONFIGURATION_MAP; field @NonNull public static final android.hardware.camera2.CameraCharacteristics.Key SENSOR_AVAILABLE_TEST_PATTERN_MODES; field @NonNull public static final android.hardware.camera2.CameraCharacteristics.Key SENSOR_BLACK_LEVEL_PATTERN; @@ -18375,9 +18376,20 @@ package android.hardware.camera2 { field public static final int MAX_THUMBNAIL_DIMENSION = 256; // 0x100 } + public class MultiResolutionImageReader implements java.lang.AutoCloseable { + method public void close(); + method protected void finalize(); + method public void flush(); + method @NonNull public android.hardware.camera2.params.MultiResolutionStreamInfo getStreamInfoForImageReader(@NonNull android.media.ImageReader); + method @NonNull public android.view.Surface getSurface(); + method @NonNull public static android.hardware.camera2.MultiResolutionImageReader newInstance(@NonNull java.util.Collection, int, @IntRange(from=1) int); + method public void setOnImageAvailableListener(@Nullable android.media.ImageReader.OnImageAvailableListener, @Nullable java.util.concurrent.Executor); + } + public final class TotalCaptureResult extends android.hardware.camera2.CaptureResult { method @NonNull public java.util.List getPartialResults(); - method public java.util.Map getPhysicalCameraResults(); + method @Deprecated public java.util.Map getPhysicalCameraResults(); + method @NonNull public java.util.Map getPhysicalCameraTotalResults(); } } @@ -18426,9 +18438,11 @@ package android.hardware.camera2.params { public final class InputConfiguration { ctor public InputConfiguration(int, int, int); + ctor public InputConfiguration(@NonNull java.util.Collection, int); method public int getFormat(); method public int getHeight(); method public int getWidth(); + method public boolean isMultiResolution(); } public final class LensShadingMap { @@ -18471,6 +18485,20 @@ package android.hardware.camera2.params { field public static final int METERING_WEIGHT_MIN = 0; // 0x0 } + public final class MultiResolutionStreamConfigurationMap { + method @NonNull public int[] getInputFormats(); + method @NonNull public java.util.Collection getInputInfo(int); + method @NonNull public int[] getOutputFormats(); + method @NonNull public java.util.Collection getOutputInfo(int); + } + + public class MultiResolutionStreamInfo { + ctor public MultiResolutionStreamInfo(int, int, @NonNull String); + method public int getHeight(); + method @NonNull public String getPhysicalCameraId(); + method public int getWidth(); + } + public final class OisSample { ctor public OisSample(long, float, float); method public long getTimestamp(); @@ -18483,6 +18511,7 @@ package android.hardware.camera2.params { ctor public OutputConfiguration(int, @NonNull android.view.Surface); ctor public OutputConfiguration(@NonNull android.util.Size, @NonNull Class); method public void addSurface(@NonNull android.view.Surface); + method @NonNull public static java.util.Collection createInstancesForMultiResolutionOutput(@NonNull android.hardware.camera2.MultiResolutionImageReader); method public int describeContents(); method public void enableSurfaceSharing(); method public int getMaxSharedSurfaceCount(); diff --git a/core/java/android/hardware/camera2/CameraCharacteristics.java b/core/java/android/hardware/camera2/CameraCharacteristics.java index 16ab900dee06c..07ebbaff67eab 100644 --- a/core/java/android/hardware/camera2/CameraCharacteristics.java +++ b/core/java/android/hardware/camera2/CameraCharacteristics.java @@ -24,13 +24,11 @@ import android.hardware.camera2.impl.PublicKey; import android.hardware.camera2.impl.SyntheticKey; import android.hardware.camera2.params.RecommendedStreamConfigurationMap; import android.hardware.camera2.params.SessionConfiguration; -import android.hardware.camera2.utils.ArrayUtils; import android.hardware.camera2.utils.TypeReference; import android.os.Build; import android.util.Rational; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.List; @@ -641,27 +639,7 @@ public final class CameraCharacteristics extends CameraMetadata getPhysicalCameraIds() { - int[] availableCapabilities = get(REQUEST_AVAILABLE_CAPABILITIES); - if (availableCapabilities == null) { - throw new AssertionError("android.request.availableCapabilities must be non-null " - + "in the characteristics"); - } - - if (!ArrayUtils.contains(availableCapabilities, - REQUEST_AVAILABLE_CAPABILITIES_LOGICAL_MULTI_CAMERA)) { - return Collections.emptySet(); - } - byte[] physicalCamIds = get(LOGICAL_MULTI_CAMERA_PHYSICAL_IDS); - - String physicalCamIdString = null; - try { - physicalCamIdString = new String(physicalCamIds, "UTF-8"); - } catch (java.io.UnsupportedEncodingException e) { - throw new AssertionError("android.logicalCam.physicalIds must be UTF-8 string"); - } - String[] physicalCameraIdArray = physicalCamIdString.split("\0"); - - return Collections.unmodifiableSet(new HashSet(Arrays.asList(physicalCameraIdArray))); + return mProperties.getPhysicalCameraIds(); } /*@O~@~@~@~@~@~@~@~@~@~@~@~@~@~@~@~@~@~@~@~@~@~@~@~@~@~@~@~@~@~@~@~@~@~ @@ -2930,6 +2908,74 @@ public final class CameraCharacteristics extends CameraMetadata SCALER_DEFAULT_SECURE_IMAGE_SIZE = new Key("android.scaler.defaultSecureImageSize", android.util.Size.class); + /** + *

The available multi-resolution stream configurations that this + * physical camera device supports + * (i.e. format, width, height, output/input stream).

+ *

This list contains a subset of the parent logical camera's multi-resolution stream + * configurations which belong to this physical camera, and it will advertise and will only + * advertise the maximum supported resolutions for a particular format.

+ *

If this camera device isn't a physical camera device constituting a logical camera, + * but a standalone ULTRA_HIGH_RESOLUTION_SENSOR camera, this field represents the + * multi-resolution input/output stream configurations of default mode and max resolution + * modes. The sizes will be the maximum resolution of a particular format for default mode + * and max resolution mode.

+ *

This field will only be advertised if the device is a physical camera of a + * logical multi-camera device or an ultra high resolution sensor camera. For a logical + * multi-camera, the camera API will derive the logical camera’s multi-resolution stream + * configurations from all physical cameras. For an ultra high resolution sensor camera, this + * is used directly as the camera’s multi-resolution stream configurations.

+ *

Optional - The value for this key may be {@code null} on some devices.

+ *

Limited capability - + * Present on all camera devices that report being at least {@link CameraCharacteristics#INFO_SUPPORTED_HARDWARE_LEVEL_LIMITED HARDWARE_LEVEL_LIMITED} devices in the + * {@link CameraCharacteristics#INFO_SUPPORTED_HARDWARE_LEVEL android.info.supportedHardwareLevel} key

+ * + * @see CameraCharacteristics#INFO_SUPPORTED_HARDWARE_LEVEL + * @hide + */ + public static final Key SCALER_PHYSICAL_CAMERA_MULTI_RESOLUTION_STREAM_CONFIGURATIONS = + new Key("android.scaler.physicalCameraMultiResolutionStreamConfigurations", android.hardware.camera2.params.StreamConfiguration[].class); + + /** + *

The multi-resolution stream configurations supported by this logical camera + * or ultra high resolution sensor camera device.

+ *

Multi-resolution streams can be used by a LOGICAL_MULTI_CAMERA or an + * ULTRA_HIGH_RESOLUTION_SENSOR camera where the images sent or received can vary in + * resolution per frame. This is useful in cases where the camera device's effective full + * resolution changes depending on factors such as the current zoom level, lighting + * condition, focus distance, or pixel mode.

+ *
    + *
  • For a logical multi-camera implementing optical zoom, at different zoom level, a + * different physical camera may be active, resulting in different full-resolution image + * sizes.
    • + *
  • For an ultra high resolution camera, depending on whether the camera operates in default + * mode, or maximum resolution mode, the output full-size images may be of either binned + * resolution or maximum resolution.
    • + *
+ *

To use multi-resolution output streams, the supported formats can be queried by {@link android.hardware.camera2.params.MultiResolutionStreamConfigurationMap#getOutputFormats }. + * A {@link android.hardware.camera2.MultiResolutionImageReader } can then be created for a + * supported format with the MultiResolutionStreamInfo group queried by {@link android.hardware.camera2.params.MultiResolutionStreamConfigurationMap#getOutputInfo }.

+ *

If a camera device supports multi-resolution output streams for a particular format, for + * each of its mandatory stream combinations, the camera device will support using a + * MultiResolutionImageReader for the MAXIMUM stream of supported formats. Refer to + * {@link android.hardware.camera2.CameraDevice#createCaptureSession } for additional details.

+ *

To use multi-resolution input streams, the supported formats can be queried by {@link android.hardware.camera2.params.MultiResolutionStreamConfigurationMap#getInputFormats }. + * A reprocessable CameraCaptureSession can then be created using an {@link android.hardware.camera2.params.InputConfiguration InputConfiguration} constructed with + * the input MultiResolutionStreamInfo group, queried by {@link android.hardware.camera2.params.MultiResolutionStreamConfigurationMap#getInputInfo }.

+ *

If a camera device supports multi-resolution {@code YUV} input and multi-resolution + * {@code YUV} output, or multi-resolution {@code PRIVATE} input and multi-resolution + * {@code PRIVATE} output, {@code JPEG} and {@code YUV} are guaranteed to be supported + * multi-resolution output stream formats. Refer to + * {@link android.hardware.camera2.CameraDevice#createCaptureSession } for + * details about the additional mandatory stream combinations in this case.

+ *

Optional - The value for this key may be {@code null} on some devices.

+ */ + @PublicKey + @NonNull + @SyntheticKey + public static final Key SCALER_MULTI_RESOLUTION_STREAM_CONFIGURATION_MAP = + new Key("android.scaler.multiResolutionStreamConfigurationMap", android.hardware.camera2.params.MultiResolutionStreamConfigurationMap.class); + /** *

The area of the image sensor which corresponds to active pixels after any geometric * distortion correction has been applied.

diff --git a/core/java/android/hardware/camera2/CameraDevice.java b/core/java/android/hardware/camera2/CameraDevice.java index ac6ba0a4ac58d..af48b71f99621 100644 --- a/core/java/android/hardware/camera2/CameraDevice.java +++ b/core/java/android/hardware/camera2/CameraDevice.java @@ -770,6 +770,8 @@ public abstract class CameraDevice implements AutoCloseable { * streams with {@code Y8} in all guaranteed stream combinations for the device's hardware level * and capabilities.

* + *

Clients can access the above mandatory stream combination tables via + * {@link android.hardware.camera2.params.MandatoryStreamCombination}.

* *

Devices capable of outputting HEIC formats ({@link StreamConfigurationMap#getOutputFormats} * contains {@link android.graphics.ImageFormat#HEIC}) will support substituting {@code JPEG} @@ -777,8 +779,33 @@ public abstract class CameraDevice implements AutoCloseable { * level and capabilities. Calling createCaptureSession with both JPEG and HEIC outputs is not * supported.

* - *

Clients can access the above mandatory stream combination tables via - * {@link android.hardware.camera2.params.MandatoryStreamCombination}.

+ *

Devices capable of multi-resolution output for a particular format ( + * {@link android.hardware.camera2.params.MultiResolutionStreamConfigurationMap#getOutputInfo} + * returns a non-empty list) support using {@link MultiResolutionImageReader} for MAXIMUM + * resolution streams of that format for all mandatory stream combinations. For example, + * if a LIMITED camera device supports multi-resolution output streams for both {@code JPEG} and + * {@code PRIVATE}, in addition to the stream configurations + * in the LIMITED and Legacy table above, the camera device supports the following guaranteed + * stream combinations ({@code MULTI_RES} in the Max size column refers to a {@link + * MultiResolutionImageReader} created based on the variable max resolutions supported): + * + * + * + * + * + * + * + * + * + *
LEGACY-level additional guaranteed combinations with MultiResolutionoutputs
Target 1 Target 2 Target 3 Sample use case(s)
TypeMax size TypeMax size TypeMax size
{@code PRIV}{@code MULTI_RES} Simple preview, GPU video processing, or no-preview video recording.
{@code JPEG}{@code MULTI_RES} No-viewfinder still image capture.
{@code PRIV}{@code PREVIEW} {@code JPEG}{@code MULTI_RES} Standard still imaging.
{@code PRIV}{@code PREVIEW} {@code YUV }{@code PREVIEW} {@code JPEG}{@code MULTI_RES} Still capture plus in-app processing.

+ * + * + * + * + * + *
LIMITED-level additional guaranteed configurations with MultiResolutionoutputs
Target 1Target 2Target 3 Sample use case(s)
TypeMax sizeTypeMax sizeTypeMax size
{@code YUV }{@code PREVIEW} {@code YUV }{@code PREVIEW} {@code JPEG}{@code MULTI_RES} Two-input in-app processing with still capture.

+ * The same logic applies to other hardware levels and capabilities. + *

* *

Since the capabilities of camera devices vary greatly, a given camera device may support * target combinations with sizes outside of these guarantees, but this can only be tested for @@ -939,6 +966,32 @@ public abstract class CameraDevice implements AutoCloseable { *
*

* + *

If a camera device supports multi-resolution {@code YUV} input and multi-resolution + * {@code YUV} output or supports multi-resolution {@code PRIVATE} input and multi-resolution + * {@code PRIVATE} output, the additional mandatory stream combinations for LIMITED and FULL devices are listed + * below ({@code MULTI_RES} in the Max size column refers to a + * {@link MultiResolutionImageReader} for output, and a multi-resolution + * {@link InputConfiguration} for input): + * + * + * + * + * + * + * + * + *
LIMITED-level additional guaranteed configurations for creating a reprocessable capture session with multi-resolution input and multi-resolution outputs
({@code PRIV} input is guaranteed only if PRIVATE reprocessing is supported. {@code YUV} input is guaranteed only if YUV reprocessing is supported)
InputTarget 1Target 2Target 3Target 4Sample use case(s)
TypeMax sizeTypeMax sizeTypeMax sizeTypeMax sizeTypeMax size
{@code PRIV}/{@code YUV}{@code MULTI_RES} Same as input{@code MULTI_RES} {@code JPEG}{@code MULTI_RES} No-viewfinder still image reprocessing.
{@code PRIV}/{@code YUV}{@code MULTI_RES} Same as input{@code MULTI_RES} {@code PRIV}{@code PREVIEW} {@code JPEG}{@code MULTI_RES} ZSL(Zero-Shutter-Lag) still imaging.
{@code PRIV}/{@code YUV}{@code MULTI_RES} Same as input{@code MULTI_RES} {@code YUV}{@code PREVIEW} {@code JPEG}{@code MULTI_RES} ZSL still and in-app processing imaging.
{@code PRIV}/{@code YUV}{@code MULTI_RES} Same as input{@code MULTI_RES} {@code YUV}{@code PREVIEW} {@code YUV}{@code PREVIEW} {@code JPEG}{@code MULTI_RES} ZSL in-app processing with still capture.

+ * + * + * + * + * + * + * + *
FULL-level additional guaranteed configurations for creating a reprocessable capture session with multi-resolution input and multi-resolution outputs
({@code PRIV} input is guaranteed only if PRIVATE reprocessing is supported. {@code YUV} input is guaranteed only if YUV reprocessing is supported)
InputTarget 1Target 2Target 3Target 4Sample use case(s)
TypeMax sizeTypeMax sizeTypeMax sizeTypeMax sizeTypeMax size
{@code PRIV}{@code MULTI_RES} {@code PRIV}{@code MULTI_RES} {@code PRIV}{@code PREVIEW} {@code YUV}{@code MULTI_RES} Maximum-resolution ZSL in-app processing with regular preview.
{@code PRIV}{@code MULTI_RES} {@code PRIV}{@code MULTI_RES} {@code YUV}{@code PREVIEW} {@code YUV}{@code MULTI_RES} Maximum-resolution two-input ZSL in-app processing.
{@code PRIV}/{@code YUV}{@code MULTI_RES} Same as input{@code MULTI_RES} {@code PRIV}{@code PREVIEW} {@code YUV}{@code PREVIEW} {@code JPEG}{@code MULTI_RES} ZSL still capture and in-app processing.

+ * No additional mandatory stream combinations for RAW capability and LEVEL-3 hardware level. + *

+ * *

Constrained high-speed recording

* *

The application can use a diff --git a/core/java/android/hardware/camera2/CameraManager.java b/core/java/android/hardware/camera2/CameraManager.java index 3e0e3f62574fb..a3c6f2f1eafd3 100644 --- a/core/java/android/hardware/camera2/CameraManager.java +++ b/core/java/android/hardware/camera2/CameraManager.java @@ -31,6 +31,7 @@ import android.hardware.camera2.impl.CameraDeviceImpl; import android.hardware.camera2.impl.CameraMetadataNative; import android.hardware.camera2.params.ExtensionSessionConfiguration; import android.hardware.camera2.params.SessionConfiguration; +import android.hardware.camera2.params.StreamConfiguration; import android.hardware.camera2.utils.CameraIdAndSessionConfiguration; import android.hardware.camera2.utils.ConcurrentCameraIdCombination; import android.hardware.display.DisplayManager; @@ -51,6 +52,7 @@ import android.view.Display; import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; +import java.util.HashMap; import java.util.Map; import java.util.Set; import java.util.concurrent.Executor; @@ -371,6 +373,47 @@ public final class CameraManager { return ret; } + /** + * Get all physical cameras' multi-resolution stream configuration map + * + *

For a logical multi-camera, query the map between physical camera id and + * the physical camera's multi-resolution stream configuration. This map is in turn + * combined to form the logical camera's multi-resolution stream configuration map.

+ */ + private Map getPhysicalCameraMultiResolutionConfigs( + CameraMetadataNative info, ICameraService cameraService) + throws CameraAccessException { + HashMap multiResolutionStreamConfigurations = + new HashMap(); + + // Query the characteristics of all physical sub-cameras, and combine the multi-resolution + // stream configurations. Note that framework derived formats such as HEIC and DEPTH_JPEG + // aren't supported as multi-resolution input or output formats. + Set physicalCameraIds = info.getPhysicalCameraIds(); + try { + for (String physicalCameraId : physicalCameraIds) { + CameraMetadataNative physicalCameraInfo = + cameraService.getCameraCharacteristics(physicalCameraId); + StreamConfiguration[] configs = physicalCameraInfo.get( + CameraCharacteristics. + SCALER_PHYSICAL_CAMERA_MULTI_RESOLUTION_STREAM_CONFIGURATIONS); + if (configs != null) { + multiResolutionStreamConfigurations.put(physicalCameraId, configs); + } + } + + // TODO: If this is an ultra high resolution sensor camera, combine the multi-resolution + // stream combination from "info" as well. + } catch (RemoteException e) { + ServiceSpecificException sse = new ServiceSpecificException( + ICameraService.ERROR_DISCONNECTED, + "Camera service is currently unavailable"); + throwAsPublicException(sse); + } + + return multiResolutionStreamConfigurations; + } + /** *

Query the capabilities of a camera device. These capabilities are * immutable for a given camera.

@@ -418,12 +461,19 @@ public final class CameraManager { } catch (NumberFormatException e) { Log.v(TAG, "Failed to parse camera Id " + cameraId + " to integer"); } + boolean hasConcurrentStreams = CameraManagerGlobal.get().cameraIdHasConcurrentStreamsLocked(cameraId); info.setHasMandatoryConcurrentStreams(hasConcurrentStreams); info.setDisplaySize(displaySize); - characteristics = new CameraCharacteristics(info); + Map multiResolutionSizeMap = + getPhysicalCameraMultiResolutionConfigs(info, cameraService); + if (multiResolutionSizeMap.size() > 0) { + info.setMultiResolutionStreamConfigurationMap(multiResolutionSizeMap); + } + + characteristics = new CameraCharacteristics(info); } catch (ServiceSpecificException e) { throwAsPublicException(e); } catch (RemoteException e) { diff --git a/core/java/android/hardware/camera2/MultiResolutionImageReader.java b/core/java/android/hardware/camera2/MultiResolutionImageReader.java new file mode 100644 index 0000000000000..c592f19bc45c0 --- /dev/null +++ b/core/java/android/hardware/camera2/MultiResolutionImageReader.java @@ -0,0 +1,309 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.hardware.camera2; + +import android.annotation.CallbackExecutor; +import android.annotation.IntRange; +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.annotation.SuppressLint; +import android.graphics.ImageFormat; +import android.graphics.ImageFormat.Format; +import android.hardware.HardwareBuffer; +import android.hardware.HardwareBuffer.Usage; +import android.media.Image; +import android.media.ImageReader; +import android.hardware.camera2.params.MultiResolutionStreamInfo; +import android.os.Looper; +import android.os.Message; +import android.util.Log; +import android.view.Surface; + + +import java.nio.NioUtils; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.Executor; + +/** + *

The MultiResolutionImageReader class wraps a group of {@link ImageReader ImageReaders} with + * the same format and different sizes, source camera Id, or camera sensor modes.

+ * + *

The main use case of this class is for a + * {@link CameraCharacteristics#REQUEST_AVAILABLE_CAPABILITIES_LOGICAL_MULTI_CAMERA logical + * multi-camera} or an ultra high resolution sensor camera to output variable-size images. For a + * logical multi-camera which implements optical zoom, different physical cameras may have different + * maximum resolutions. As a result, when the camera device switches between physical cameras + * depending on zoom ratio, the maximum resolution for a particular format may change. For an + * ultra high resolution sensor camera, the camera device may deem it better or worse to run in + * maximum resolution mode / default mode depending on lighting conditions. So the application may + * choose to let the camera device decide on its behalf.

+ * + *

MultiResolutionImageReader should be used for a camera device only if the camera device + * supports multi-resolution output stream by advertising the specified output format in {@link + * CameraCharacteristics#SCALER_MULTI_RESOLUTION_STREAM_CONFIGURATION_MAP}.

+ * + *

To acquire images from the MultiResolutionImageReader, the application must use the + * {@link ImageReader} object passed by + * {@link ImageReader.OnImageAvailableListener#onImageAvailable} callback to call + * {@link ImageReader#acquireNextImage} or {@link ImageReader#acquireLatestImage}. The application + * must not use the {@link ImageReader} passed by an {@link + * ImageReader.OnImageAvailableListener#onImageAvailable} callback to acquire future images + * because future images may originate from a different {@link ImageReader} contained within the + * {@code MultiResolutionImageReader}.

+ * + * + * @see ImageReader + * @see android.hardware.camera2.CameraCharacteristics#SCALER_MULTI_RESOLUTION_STREAM_CONFIGURATION_MAP + */ +public class MultiResolutionImageReader implements AutoCloseable { + + private static final String TAG = "MultiResolutionImageReader"; + + /** + *

+ * Create a new multi-resolution reader based on a group of camera stream properties returned + * by a camera device. + *

+ *

+ * The valid size and formats depend on the camera characteristics. + * {@code MultiResolutionImageReader} for an image format is supported by the camera device if + * the format is in the supported multi-resolution output stream formats returned by + * {@link android.hardware.camera2.params.MultiResolutionStreamConfigurationMap#getOutputFormats}. + * If the image format is supported, the {@code MultiResolutionImageReader} object can be + * created with the {@code streams} objects returned by + * {@link android.hardware.camera2.params.MultiResolutionStreamConfigurationMap#getOutputInfo}. + *

+ *

+ * The {@code maxImages} parameter determines the maximum number of + * {@link Image} objects that can be be acquired from each of the {@code ImageReader} + * within the {@code MultiResolutionImageReader}. However, requesting more buffers will + * use up more memory, so it is important to use only the minimum number necessary. The + * application is strongly recommended to acquire no more than {@code maxImages} images + * from all of the internal ImageReader objects combined. By keeping track of the number of + * acquired images for the MultiResolutionImageReader, the application doesn't need to do the + * bookkeeping for each internal ImageReader returned from {@link + * ImageReader.OnImageAvailableListener#onImageAvailable onImageAvailable} callback. + *

+ *

+ * Unlike the normal ImageReader, the MultiResolutionImageReader has a more complex + * configuration sequence. Instead of passing the same surface to OutputConfiguration and + * CaptureRequest, the + * {@link android.hardware.camera2.params.OutputConfiguration#createInstancesForMultiResolutionOutput} + * call needs to be used to create the OutputConfigurations for session creation, and then + * {@link #getSurface} is used to get {@link CaptureRequest.Builder#addTarget the target for + * CaptureRequest}. + *

+ * @param streams The group of multi-resolution stream info, which is used to create + * a multi-resolution reader containing a number of ImageReader objects. Each + * ImageReader object represents a multi-resolution stream in the group. + * @param format The format of the Image that this multi-resolution reader will produce. + * This must be one of the {@link android.graphics.ImageFormat} or + * {@link android.graphics.PixelFormat} constants. Note that not all formats are + * supported, like ImageFormat.NV21. The supported multi-resolution + * reader format can be queried by {@link + * android.hardware.camera2.params.MultiResolutionStreamConfigurationMap#getOutputFormats}. + * @param maxImages The maximum number of images the user will want to + * access simultaneously. This should be as small as possible to + * limit memory use. Once maxImages images are obtained by the + * user from any given internal ImageReader, one of them has to be released before + * a new Image will become available for access through the ImageReader's + * {@link ImageReader#acquireLatestImage()} or + * {@link ImageReader#acquireNextImage()}. Must be greater than 0. + * @see Image + * @see + * android.hardware.camera2.CameraCharacteristics#SCALER_MULTI_RESOLUTION_STREAM_CONFIGURATION_MAP + * @see + * android.hardware.camera2.params.MultiResolutionStreamConfigurationMap + */ + public static @NonNull MultiResolutionImageReader newInstance( + @NonNull Collection streams, + @Format int format, + @IntRange(from = 1) int maxImages) { + return new MultiResolutionImageReader(streams, format, maxImages); + } + + /** + * @hide + */ + protected MultiResolutionImageReader(Collection streams, + int format, int maxImages) { + mFormat = format; + mMaxImages = maxImages; + + if (streams == null || streams.size() <= 1) { + throw new IllegalArgumentException( + "The streams info collection must contain at least 2 entries"); + } + if (mMaxImages < 1) { + throw new IllegalArgumentException( + "Maximum outstanding image count must be at least 1"); + } + + if (format == ImageFormat.NV21) { + throw new IllegalArgumentException( + "NV21 format is not supported"); + } + + int numImageReaders = streams.size(); + mReaders = new ImageReader[numImageReaders]; + mStreamInfo = new MultiResolutionStreamInfo[numImageReaders]; + int index = 0; + for (MultiResolutionStreamInfo streamInfo : streams) { + mReaders[index] = ImageReader.newInstance(streamInfo.getWidth(), + streamInfo.getHeight(), format, maxImages); + mStreamInfo[index] = streamInfo; + index++; + } + } + + /** + * Set onImageAvailableListener callback. + * + *

This function sets the onImageAvailableListener for all the internal + * {@link ImageReader} objects.

+ * + *

For a multi-resolution ImageReader, the timestamps of images acquired in + * onImageAvailable callback from different internal ImageReaders may become + * out-of-order due to the asynchronous callbacks between the different resolution + * image queues.

+ * + * @param listener + * The listener that will be run. + * @param executor + * The executor which will be used when invoking the callback. + */ + @SuppressLint({"ExecutorRegistration", "SamShouldBeLast"}) + public void setOnImageAvailableListener( + @Nullable ImageReader.OnImageAvailableListener listener, + @Nullable @CallbackExecutor Executor executor) { + for (int i = 0; i < mReaders.length; i++) { + mReaders[i].setOnImageAvailableListenerWithExecutor(listener, executor); + } + } + + @Override + public void close() { + flush(); + + for (int i = 0; i < mReaders.length; i++) { + mReaders[i].close(); + } + } + + @Override + protected void finalize() { + close(); + } + + /** + * Flush pending images from all internal ImageReaders + * + *

Acquire and close pending images from all internal ImageReaders. This has the same + * effect as calling acquireLatestImage() on all internal ImageReaders, and closing all + * latest images.

+ */ + public void flush() { + flushOther(null); + } + + /** + * Flush pending images from other internal ImageReaders + * + *

Acquire and close pending images from all internal ImageReaders except for the + * one specified.

+ * + * @param reader The ImageReader object that won't be flushed. + * + * @hide + */ + public void flushOther(ImageReader reader) { + for (int i = 0; i < mReaders.length; i++) { + if (reader != null && reader == mReaders[i]) { + continue; + } + + while (true) { + Image image = mReaders[i].acquireNextImageNoThrowISE(); + if (image == null) { + break; + } else { + image.close(); + } + } + } + } + + /** + * Get the internal ImageReader objects + * + * @hide + */ + public @NonNull ImageReader[] getReaders() { + return mReaders; + } + + /** + * Get the surface that is used as a target for {@link CaptureRequest} + * + *

The application must use the surface returned by this function as a target for + * {@link CaptureRequest}. The camera device makes the decision on which internal + * {@code ImageReader} will receive the output image.

+ * + *

Please note that holding on to the Surface objects returned by this method is not enough + * to keep their parent MultiResolutionImageReaders from being reclaimed. In that sense, a + * Surface acts like a {@link java.lang.ref.WeakReference weak reference} to the + * MultiResolutionImageReader that provides it.

+ * + * @return a {@link Surface} to use as the target for a capture request. + */ + public @NonNull Surface getSurface() { + //TODO: Pick the surface from the reader for default mode stream. + return mReaders[0].getSurface(); + } + + /** + * Get the MultiResolutionStreamInfo describing the ImageReader an image originates from + * + *

An image from a {@code MultiResolutionImageReader} is produced from one of the underlying + *{@code ImageReader}s. This function returns the {@link MultiResolutionStreamInfo} to describe + *the property for that {@code ImageReader}, such as width, height, and physical camera Id.

+ * + * @param reader An internal ImageReader within {@code MultiResolutionImageReader}. + * + * @return The stream info describing the internal {@code ImageReader}. + */ + public @NonNull MultiResolutionStreamInfo getStreamInfoForImageReader( + @NonNull ImageReader reader) { + for (int i = 0; i < mReaders.length; i++) { + if (reader == mReaders[i]) { + return mStreamInfo[i]; + } + } + + throw new IllegalArgumentException("ImageReader doesn't belong to this multi-resolution " + + "imagereader"); + } + + // mReaders and mStreamInfo has the same length, and their entries are 1:1 mapped. + private final ImageReader[] mReaders; + private final MultiResolutionStreamInfo[] mStreamInfo; + + private final int mFormat; + private final int mMaxImages; +} diff --git a/core/java/android/hardware/camera2/TotalCaptureResult.java b/core/java/android/hardware/camera2/TotalCaptureResult.java index da65f71ce02c5..df8eeccbe8007 100644 --- a/core/java/android/hardware/camera2/TotalCaptureResult.java +++ b/core/java/android/hardware/camera2/TotalCaptureResult.java @@ -61,8 +61,8 @@ public final class TotalCaptureResult extends CaptureResult { private final List mPartialResults; private final int mSessionId; - // The map between physical camera id and capture result - private final HashMap mPhysicalCaptureResults; + // The map between physical camera ids and their total capture result + private final HashMap mPhysicalCaptureResults; /** * Takes ownership of the passed-in camera metadata and the partial results @@ -83,10 +83,11 @@ public final class TotalCaptureResult extends CaptureResult { mSessionId = sessionId; - mPhysicalCaptureResults = new HashMap(); + mPhysicalCaptureResults = new HashMap(); for (PhysicalCaptureResultInfo onePhysicalResult : physicalResults) { - CaptureResult physicalResult = new CaptureResult(onePhysicalResult.getCameraId(), - onePhysicalResult.getCameraMetadata(), parent, extras); + TotalCaptureResult physicalResult = new TotalCaptureResult( + onePhysicalResult.getCameraId(), onePhysicalResult.getCameraMetadata(), + parent, extras, /*partials*/null, sessionId, new PhysicalCaptureResultInfo[0]); mPhysicalCaptureResults.put(onePhysicalResult.getCameraId(), physicalResult); } @@ -103,7 +104,7 @@ public final class TotalCaptureResult extends CaptureResult { mPartialResults = new ArrayList<>(); mSessionId = CameraCaptureSession.SESSION_ID_NONE; - mPhysicalCaptureResults = new HashMap(); + mPhysicalCaptureResults = new HashMap(); } /** @@ -146,8 +147,37 @@ public final class TotalCaptureResult extends CaptureResult { * cameras. Otherwise, an empty map is returned.

* @return unmodifiable map between physical camera ids and their capture result metadata + * + * @deprecated + *

Please use {@link #getPhysicalCameraTotalResults() instead to get the + * physical cameras' {@code TotalCaptureResult}.

*/ public Map getPhysicalCameraResults() { return Collections.unmodifiableMap(mPhysicalCaptureResults); } + + /** + * Get the map between physical camera ids and their total capture result metadata + * + *

This function can be called for logical multi-camera devices, which are devices that have + * REQUEST_AVAILABLE_CAPABILITIES_LOGICAL_MULTI_CAMERA capability.

+ * + *

If one or more streams from the underlying physical cameras were requested by the + * corresponding capture request, this function returns the total result metadata for those + * physical cameras. Otherwise, an empty map is returned.

+ * + *

This function replaces the deprecated {@link #getPhysicalCameraResults}, and its return + * value is a map of TotalCaptureResult rather than CaptureResult.

+ * + *

To reprocess an image from a physical camera stream, typically returned from a + * {@link MultiResolutionImageReader}, the application must look up this map to get the {@link + * TotalCaptureResult} from the physical camera and pass it to {@link + * CameraDevice#createReprocessCaptureRequest}.

+ * + * @return unmodifiable map between physical camera ids and their total capture result metadata + */ + @NonNull + public Map getPhysicalCameraTotalResults() { + return Collections.unmodifiableMap(mPhysicalCaptureResults); + } } diff --git a/core/java/android/hardware/camera2/impl/CameraDeviceImpl.java b/core/java/android/hardware/camera2/impl/CameraDeviceImpl.java index ce3c81a2bfd62..4defd23231b8f 100644 --- a/core/java/android/hardware/camera2/impl/CameraDeviceImpl.java +++ b/core/java/android/hardware/camera2/impl/CameraDeviceImpl.java @@ -36,6 +36,8 @@ import android.hardware.camera2.ICameraOfflineSession; import android.hardware.camera2.TotalCaptureResult; import android.hardware.camera2.params.ExtensionSessionConfiguration; import android.hardware.camera2.params.InputConfiguration; +import android.hardware.camera2.params.MultiResolutionStreamConfigurationMap; +import android.hardware.camera2.params.MultiResolutionStreamInfo; import android.hardware.camera2.params.OutputConfiguration; import android.hardware.camera2.params.SessionConfiguration; import android.hardware.camera2.params.StreamConfigurationMap; @@ -468,7 +470,8 @@ public class CameraDeviceImpl extends CameraDevice } if (inputConfig != null) { int streamId = mRemoteDevice.createInputStream(inputConfig.getWidth(), - inputConfig.getHeight(), inputConfig.getFormat()); + inputConfig.getHeight(), inputConfig.getFormat(), + inputConfig.isMultiResolution()); mConfiguredInput = new SimpleEntry( streamId, inputConfig); } @@ -1355,7 +1358,42 @@ public class CameraDeviceImpl extends CameraDevice } private void checkInputConfiguration(InputConfiguration inputConfig) { - if (inputConfig != null) { + if (inputConfig == null) { + return; + } + + if (inputConfig.isMultiResolution()) { + MultiResolutionStreamConfigurationMap configMap = mCharacteristics.get( + CameraCharacteristics.SCALER_MULTI_RESOLUTION_STREAM_CONFIGURATION_MAP); + + int[] inputFormats = configMap.getInputFormats(); + boolean validFormat = false; + for (int format : inputFormats) { + if (format == inputConfig.getFormat()) { + validFormat = true; + } + } + + if (validFormat == false) { + throw new IllegalArgumentException("multi-resolution input format " + + inputConfig.getFormat() + " is not valid"); + } + + boolean validSize = false; + Collection inputStreamInfo = + configMap.getInputInfo(inputConfig.getFormat()); + for (MultiResolutionStreamInfo info : inputStreamInfo) { + if (inputConfig.getWidth() == info.getWidth() && + inputConfig.getHeight() == info.getHeight()) { + validSize = true; + } + } + + if (validSize == false) { + throw new IllegalArgumentException("Multi-resolution input size " + + inputConfig.getWidth() + "x" + inputConfig.getHeight() + " is not valid"); + } + } else { StreamConfigurationMap configMap = mCharacteristics.get( CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP); diff --git a/core/java/android/hardware/camera2/impl/CameraMetadataNative.java b/core/java/android/hardware/camera2/impl/CameraMetadataNative.java index e9bae0b621ce1..0cdf744ecd68f 100644 --- a/core/java/android/hardware/camera2/impl/CameraMetadataNative.java +++ b/core/java/android/hardware/camera2/impl/CameraMetadataNative.java @@ -16,6 +16,7 @@ package android.hardware.camera2.impl; +import android.annotation.NonNull; import android.compat.annotation.UnsupportedAppUsage; import android.graphics.ImageFormat; import android.graphics.Point; @@ -53,6 +54,7 @@ import android.hardware.camera2.params.Face; import android.hardware.camera2.params.HighSpeedVideoConfiguration; import android.hardware.camera2.params.LensShadingMap; import android.hardware.camera2.params.MandatoryStreamCombination; +import android.hardware.camera2.params.MultiResolutionStreamConfigurationMap; import android.hardware.camera2.params.OisSample; import android.hardware.camera2.params.RecommendedStreamConfiguration; import android.hardware.camera2.params.RecommendedStreamConfigurationMap; @@ -61,6 +63,7 @@ import android.hardware.camera2.params.StreamConfiguration; import android.hardware.camera2.params.StreamConfigurationDuration; import android.hardware.camera2.params.StreamConfigurationMap; import android.hardware.camera2.params.TonemapCurve; +import android.hardware.camera2.utils.ArrayUtils; import android.hardware.camera2.utils.TypeReference; import android.location.Location; import android.location.LocationManager; @@ -79,9 +82,14 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; import java.util.List; import java.util.Objects; +import java.util.Set; /** * Implementation of camera metadata marshal/unmarshal across Binder to @@ -747,6 +755,15 @@ public class CameraMetadataNative implements Parcelable { return (T) metadata.getExtendedSceneModeCapabilities(); } }); + sGetCommandMap.put( + CameraCharacteristics.SCALER_MULTI_RESOLUTION_STREAM_CONFIGURATION_MAP.getNativeKey(), + new GetCommand() { + @Override + @SuppressWarnings("unchecked") + public T getValue(CameraMetadataNative metadata, Key key) { + return (T) metadata.getMultiResolutionStreamConfigurationMap(); + } + }); } private int[] getAvailableFormats() { @@ -1688,6 +1705,7 @@ public class CameraMetadataNative implements Parcelable { private boolean mHasMandatoryConcurrentStreams = false; private Size mDisplaySize = new Size(0, 0); private long mBufferSize = 0; + private MultiResolutionStreamConfigurationMap mMultiResolutionStreamConfigurationMap = null; /** * Set the current camera Id. @@ -1723,6 +1741,30 @@ public class CameraMetadataNative implements Parcelable { mDisplaySize = displaySize; } + /** + * Set the multi-resolution stream configuration map. + * + * @param multiResolutionMap The multi-resolution stream configuration map. + * + * @hide + */ + public void setMultiResolutionStreamConfigurationMap( + @NonNull Map multiResolutionMap) { + mMultiResolutionStreamConfigurationMap = + new MultiResolutionStreamConfigurationMap(multiResolutionMap); + } + + /** + * Get the multi-resolution stream configuration map. + * + * @return The multi-resolution stream configuration map. + * + * @hide + */ + public MultiResolutionStreamConfigurationMap getMultiResolutionStreamConfigurationMap() { + return mMultiResolutionStreamConfigurationMap; + } + @UnsupportedAppUsage(maxTargetSdk = Build.VERSION_CODES.R, trackingBug = 170729553) private long mMetadataPtr; // native std::shared_ptr* @@ -1777,6 +1819,7 @@ public class CameraMetadataNative implements Parcelable { mCameraId = other.mCameraId; mHasMandatoryConcurrentStreams = other.mHasMandatoryConcurrentStreams; mDisplaySize = other.mDisplaySize; + mMultiResolutionStreamConfigurationMap = other.mMultiResolutionStreamConfigurationMap; updateNativeAllocation(); other.updateNativeAllocation(); } @@ -1980,6 +2023,39 @@ public class CameraMetadataNative implements Parcelable { return true; } + /** + * Return the set of physical camera ids that this logical {@link CameraDevice} is made + * up of. + * + * If the camera device isn't a logical camera, return an empty set. + * + * @hide + */ + public Set getPhysicalCameraIds() { + int[] availableCapabilities = get(CameraCharacteristics.REQUEST_AVAILABLE_CAPABILITIES); + if (availableCapabilities == null) { + throw new AssertionError("android.request.availableCapabilities must be non-null " + + "in the characteristics"); + } + + if (!ArrayUtils.contains(availableCapabilities, + CameraCharacteristics.REQUEST_AVAILABLE_CAPABILITIES_LOGICAL_MULTI_CAMERA)) { + return Collections.emptySet(); + } + byte[] physicalCamIds = get(CameraCharacteristics.LOGICAL_MULTI_CAMERA_PHYSICAL_IDS); + + String physicalCamIdString = null; + try { + physicalCamIdString = new String(physicalCamIds, "UTF-8"); + } catch (java.io.UnsupportedEncodingException e) { + throw new AssertionError("android.logicalCam.physicalIds must be UTF-8 string"); + } + String[] physicalCameraIdArray = physicalCamIdString.split("\0"); + + return Collections.unmodifiableSet( + new HashSet(Arrays.asList(physicalCameraIdArray))); + } + static { registerAllMarshalers(); } diff --git a/core/java/android/hardware/camera2/impl/ICameraDeviceUserWrapper.java b/core/java/android/hardware/camera2/impl/ICameraDeviceUserWrapper.java index ba4395f702140..b6b1968bfcdd6 100644 --- a/core/java/android/hardware/camera2/impl/ICameraDeviceUserWrapper.java +++ b/core/java/android/hardware/camera2/impl/ICameraDeviceUserWrapper.java @@ -140,9 +140,10 @@ public class ICameraDeviceUserWrapper { } } - public int createInputStream(int width, int height, int format) throws CameraAccessException { + public int createInputStream(int width, int height, int format, boolean isMultiResolution) + throws CameraAccessException { try { - return mRemoteDevice.createInputStream(width, height, format); + return mRemoteDevice.createInputStream(width, height, format, isMultiResolution); } catch (Throwable t) { CameraManager.throwAsPublicException(t); throw new UnsupportedOperationException("Unexpected exception", t); diff --git a/core/java/android/hardware/camera2/params/InputConfiguration.java b/core/java/android/hardware/camera2/params/InputConfiguration.java index 0a50f974aca8b..d63683feed9b0 100644 --- a/core/java/android/hardware/camera2/params/InputConfiguration.java +++ b/core/java/android/hardware/camera2/params/InputConfiguration.java @@ -16,9 +16,17 @@ package android.hardware.camera2.params; +import android.annotation.NonNull; import android.annotation.Nullable; +import android.graphics.ImageFormat.Format; +import android.hardware.camera2.params.MultiResolutionStreamInfo; import android.hardware.camera2.utils.HashCodeHelpers; +import java.util.Collection; +import java.util.List; + +import static com.android.internal.util.Preconditions.*; + /** * Immutable class to store an input configuration that is used to create a reprocessable capture * session. @@ -31,11 +39,12 @@ public final class InputConfiguration { private final int mWidth; private final int mHeight; private final int mFormat; + private final boolean mIsMultiResolution; /** * Create an input configration with the width, height, and user-defined format. * - *

Images of an user-defined format are accessible by applications. Use + *

Images of a user-defined format are accessible by applications. Use * {@link android.hardware.camera2.CameraCharacteristics#SCALER_STREAM_CONFIGURATION_MAP} * to query supported input formats

* @@ -51,6 +60,52 @@ public final class InputConfiguration { mWidth = width; mHeight = height; mFormat = format; + mIsMultiResolution = false; + } + + /** + * Create an input configration with the format and a list of multi-resolution input stream + * info. + * + *

Use {@link + * android.hardware.camera2.CameraCharacteristics#SCALER_MULTI_RESOLUTION_STREAM_CONFIGURATION_MAP} + * to query supported multi-resolution input formats.

+ * + *

To do reprocessing with variable resolution input, the application calls + * {@link android.media.ImageWriter#queueInputImage ImageWriter.queueInputImage} + * using an image from an {@link android.media.ImageReader ImageReader} or {@link + * android.hardware.camera2.MultiResolutionImageReader MultiResolutionImageReader}. See + * {@link android.hardware.camera2.CameraDevice#createReprocessCaptureRequest} for more + * details on camera reprocessing. + *

+ * + * @param multiResolutionInputs A group of multi-resolution input info for the specified format. + * @param format Format of the input buffers. One of ImageFormat or PixelFormat constants. + * + * @see android.graphics.ImageFormat + * @see android.graphics.PixelFormat + * @see + * android.hardware.camera2.CameraCharacteristics#SCALER_MULTI_RESOLUTION_STREAM_CONFIGURATION_MAP + */ + public InputConfiguration(@NonNull Collection multiResolutionInputs, + @Format int format) { + checkCollectionNotEmpty(multiResolutionInputs, "Input multi-resolution stream info"); + //TODO: Pick the default mode stream info for ultra-high resolution sensor camera + MultiResolutionStreamInfo info = multiResolutionInputs.iterator().next(); + mWidth = info.getWidth(); + mHeight = info.getHeight(); + mFormat = format; + mIsMultiResolution = true; + } + + /** + * @hide + */ + public InputConfiguration(int width, int height, int format, boolean isMultiResolution) { + mWidth = width; + mHeight = height; + mFormat = format; + mIsMultiResolution = isMultiResolution; } /** @@ -80,6 +135,18 @@ public final class InputConfiguration { return mFormat; } + /** + * Whether this input configuration is of multi-resolution. + * + *

An multi-resolution InputConfiguration means that the reprocessing session created from it + * allows input images of different sizes.

+ * + * @return this input configuration is multi-resolution or not. + */ + public boolean isMultiResolution() { + return mIsMultiResolution; + } + /** * Check if this InputConfiguration is equal to another InputConfiguration. * @@ -100,7 +167,8 @@ public final class InputConfiguration { if (otherInputConfig.getWidth() == mWidth && otherInputConfig.getHeight() == mHeight && - otherInputConfig.getFormat() == mFormat) { + otherInputConfig.getFormat() == mFormat && + otherInputConfig.isMultiResolution() == mIsMultiResolution) { return true; } return false; @@ -111,19 +179,21 @@ public final class InputConfiguration { */ @Override public int hashCode() { - return HashCodeHelpers.hashCode(mWidth, mHeight, mFormat); + return HashCodeHelpers.hashCode(mWidth, mHeight, mFormat, mIsMultiResolution ? 1 : 0); } /** * Return this {@link InputConfiguration} as a string representation. * - *

{@code "InputConfiguration(w:%d, h:%d, format:%d)"}, where {@code %d} represents - * the width, height, and format, respectively.

+ *

{@code "InputConfiguration(w:%d, h:%d, format:%d, isMultiResolution:%d)"}, + * where {@code %d} represents the width, height, format, and multi-resolution flag + * respectively.

* * @return string representation of {@link InputConfiguration} */ @Override public String toString() { - return String.format("InputConfiguration(w:%d, h:%d, format:%d)", mWidth, mHeight, mFormat); + return String.format("InputConfiguration(w:%d, h:%d, format:%d, isMultiResolution %b)", + mWidth, mHeight, mFormat, mIsMultiResolution); } } diff --git a/core/java/android/hardware/camera2/params/MandatoryStreamCombination.java b/core/java/android/hardware/camera2/params/MandatoryStreamCombination.java index 776d155e5b3e5..8a0172ee80185 100644 --- a/core/java/android/hardware/camera2/params/MandatoryStreamCombination.java +++ b/core/java/android/hardware/camera2/params/MandatoryStreamCombination.java @@ -1296,19 +1296,6 @@ public final class MandatoryStreamCombination { return kPreviewSizeBound; } - /** - * Size comparison method used by size comparators. - */ - private static int compareSizes(int widthA, int heightA, int widthB, int heightB) { - long left = widthA * (long) heightA; - long right = widthB * (long) heightB; - if (left == right) { - left = widthA; - right = widthB; - } - return (left < right) ? -1 : (left > right ? 1 : 0); - } - /** * Size comparator that compares the number of pixels it covers. * @@ -1317,8 +1304,8 @@ public final class MandatoryStreamCombination { public static class SizeComparator implements Comparator { @Override public int compare(@NonNull Size lhs, @NonNull Size rhs) { - return compareSizes(lhs.getWidth(), lhs.getHeight(), rhs.getWidth(), - rhs.getHeight()); + return StreamConfigurationMap.compareSizes(lhs.getWidth(), lhs.getHeight(), + rhs.getWidth(), rhs.getHeight()); } } diff --git a/core/java/android/hardware/camera2/params/MultiResolutionStreamConfigurationMap.java b/core/java/android/hardware/camera2/params/MultiResolutionStreamConfigurationMap.java new file mode 100644 index 0000000000000..1b368fb8c0103 --- /dev/null +++ b/core/java/android/hardware/camera2/params/MultiResolutionStreamConfigurationMap.java @@ -0,0 +1,335 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package android.hardware.camera2.params; + +import android.annotation.NonNull; +import android.annotation.Nullable; + +import android.graphics.ImageFormat; +import android.graphics.ImageFormat.Format; +import android.graphics.PixelFormat; +import android.hardware.camera2.params.MultiResolutionStreamInfo; +import android.hardware.camera2.params.StreamConfigurationMap; +import android.hardware.camera2.utils.HashCodeHelpers; + +import android.util.Size; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Map; +import java.util.List; +import java.util.Set; + +import static com.android.internal.util.Preconditions.*; + +/** + * Immutable class to store the information of the multi-resolution streams supported by + * the camera device. + * + *

For a {@link + * android.hardware.camera2.CameraCharacteristics#REQUEST_AVAILABLE_CAPABILITIES_LOGICAL_MULTI_CAMERA + * logical multi-camera} or an ultra high resolution sensor camera, the maximum resolution of images + * produced by the camera device may be variable. For example, for a logical multi-camera, depending + * on factors such as current zoom ratio, the camera device may be backed by different physical + * cameras. If the physical cameras are of different resolutions, the application may intend to + * consume the variable full resolution images from the physical cameras. For an ultra high + * resolution sensor camera, the same use case exists where depending on lighting conditions, the + * camera device may deem it better to run in default mode and maximum resolution mode. + *

+ * + *

For the use cases described above, multi-resolution output streams can be used by + * {@link android.hardware.camera2.MultiResolutionImageReader} to allow the + * camera device to output variable size maximum-resolution images.

+ * + *

Similarly, multi-resolution input streams can be used for reprocessing of variable size + * images. In order to reprocess input images of different sizes, the {@link InputConfiguration} + * used for creating reprocessable session can be initialized using the group of input stream + * configurations returned by {@link #getInputInfo}.

+ */ +public final class MultiResolutionStreamConfigurationMap { + /** + * Create a new {@link MultiResolutionStreamConfigurationMap}. + * + * @param configurations a non-{@code null} array of multi-resolution stream + * configurations supported by this camera device + * @hide + */ + public MultiResolutionStreamConfigurationMap( + @NonNull Map configurations) { + checkNotNull(configurations, "multi-resolution configurations must not be null"); + if (configurations.size() == 0) { + throw new IllegalArgumentException("multi-resolution configurations must not be empty"); + } + + mConfigurations = configurations; + + // For each multi-resolution stream configuration, track how many formats and sizes there + // are available to configure + for (Map.Entry entry : + mConfigurations.entrySet()) { + String cameraId = entry.getKey(); + StreamConfiguration[] configs = entry.getValue(); + + for (int i = 0; i < configs.length; i++) { + StreamConfiguration config = configs[i]; + int format = config.getFormat(); + + MultiResolutionStreamInfo multiResolutionStreamInfo = new MultiResolutionStreamInfo( + config.getWidth(), config.getHeight(), cameraId); + Map> destMap; + if (config.isInput()) { + destMap = mMultiResolutionInputConfigs; + } else { + destMap = mMultiResolutionOutputConfigs; + } + + if (!destMap.containsKey(format)) { + List multiResolutionStreamInfoList = + new ArrayList(); + destMap.put(format, multiResolutionStreamInfoList); + } + destMap.get(format).add(multiResolutionStreamInfo); + } + } + } + + /** + * Size comparator that compares the number of pixels two MultiResolutionStreamInfo size covers. + * + *

If two the areas of two sizes are same, compare the widths.

+ * + * @hide + */ + public static class SizeComparator implements Comparator { + @Override + public int compare(@NonNull MultiResolutionStreamInfo lhs, + @NonNull MultiResolutionStreamInfo rhs) { + return StreamConfigurationMap.compareSizes( + lhs.getWidth(), lhs.getHeight(), rhs.getWidth(), rhs.getHeight()); + } + } + + /** + * Get the output formats in this multi-resolution stream configuration. + * + *

A logical multi-camera or an ultra high resolution sensor camera may support + * {@link android.hardware.camera2.MultiResolutionImageReader} to dynamically output maximum + * resolutions of different sizes (when switching between physical cameras, or between different + * modes of an ultra high resolution sensor camera). This function returns the formats + * supported for such case.

+ * + *

All image formats returned by this function will be defined in either {@link ImageFormat} + * or in {@link PixelFormat} (and there is no possibility of collision).

+ * + * @return an array of integer format, or empty array if multi-resolution output is not + * supported + * + * @see ImageFormat + * @see PixelFormat + * @see android.hardware.camera2.MultiResolutionImageReader + */ + public @NonNull @Format int[] getOutputFormats() { + return getPublicImageFormats(/*output*/true); + } + + /** + * Get the input formats in this multi-resolution stream configuration. + * + *

A logical multi-camera or ultra high resolution sensor camera may support reprocessing + * images of different resolutions when switching between physical cameras, or between + * different modes of the ultra high resolution sensor camera. This function returns the + * formats supported for such case.

+ * + *

The supported output format for an input format can be queried by calling the camera + * device's {@link StreamConfigurationMap#getValidOutputFormatsForInput}.

+ * + *

All image formats returned by this function will be defined in either {@link ImageFormat} + * or in {@link PixelFormat} (and there is no possibility of collision).

+ * + * @return an array of integer format, or empty array if no multi-resolution reprocessing is + * supported + * + * @see ImageFormat + * @see PixelFormat + */ + public @NonNull @Format int[] getInputFormats() { + return getPublicImageFormats(/*output*/false); + } + + // Get the list of publicly visible multi-resolution input/output stream formats + private int[] getPublicImageFormats(boolean output) { + Map> multiResolutionConfigs = + output ? mMultiResolutionOutputConfigs : mMultiResolutionInputConfigs; + int formatCount = multiResolutionConfigs.size(); + + int[] formats = new int[formatCount]; + int i = 0; + for (Integer format : multiResolutionConfigs.keySet()) { + formats[i++] = StreamConfigurationMap.imageFormatToPublic(format); + } + + return formats; + } + + /** + * Get a group of {@code MultiResolutionStreamInfo} with the requested output image + * {@code format} + * + *

The {@code format} should be a supported format (one of the formats returned by + * {@link #getOutputFormats}).

+ * + * @param format an image format from {@link ImageFormat} or {@link PixelFormat} + * @return + * a group of supported {@link MultiResolutionStreamInfo}. If the {@code format} is not + * a supported multi-resolution output, an empty group is returned. + * + * @see ImageFormat + * @see PixelFormat + * @see #getOutputFormats + */ + public @NonNull Collection getOutputInfo(@Format int format) { + return getInfo(format, /*false*/ true); + } + + /** + * Get a group of {@code MultiResolutionStreamInfo} with the requested input image {@code format} + * + *

The {@code format} should be a supported format (one of the formats returned by + * {@link #getInputFormats}).

+ * + * @param format an image format from {@link ImageFormat} or {@link PixelFormat} + * @return + * a group of supported {@link MultiResolutionStreamInfo}. If the {@code format} is not + * a supported multi-resolution input, an empty group is returned. + * + * @see ImageFormat + * @see PixelFormat + * @see #getInputFormats + */ + public @NonNull Collection getInputInfo(@Format int format) { + return getInfo(format, /*false*/ false); + } + + // Get multi-resolution stream info for a particular format + private @NonNull Collection getInfo(int format, boolean output) { + int internalFormat = StreamConfigurationMap.imageFormatToInternal(format); + Map> multiResolutionConfigs = + output ? mMultiResolutionOutputConfigs : mMultiResolutionInputConfigs; + if (multiResolutionConfigs.containsKey(internalFormat)) { + return Collections.unmodifiableCollection(multiResolutionConfigs.get(internalFormat)); + } else { + return Collections.emptyList(); + } + } + + private void appendConfigurationsString(StringBuilder sb, boolean output) { + sb.append(output ? "Outputs(" : "Inputs("); + int[] formats = getPublicImageFormats(output); + if (formats != null) { + for (int format : formats) { + Collection streamInfoList = + getInfo(format, output); + sb.append("[" + StreamConfigurationMap.formatToString(format) + ":"); + for (MultiResolutionStreamInfo streamInfo : streamInfoList) { + sb.append(String.format("[w:%d, h:%d, id:%s], ", + streamInfo.getWidth(), streamInfo.getHeight(), + streamInfo.getPhysicalCameraId())); + } + // Remove the pending ", " + if (sb.charAt(sb.length() - 1) == ' ') { + sb.delete(sb.length() - 2, sb.length()); + } + sb.append("]"); + } + } + sb.append(")"); + } + + /** + * Check if this {@link MultiResolutionStreamConfigurationMap} is equal to another + * {@link MultiResolutionStreamConfigurationMap}. + * + * @return {@code true} if the objects were equal, {@code false} otherwise + */ + @Override + public boolean equals(final Object obj) { + if (obj == null) { + return false; + } + if (this == obj) { + return true; + } + if (obj instanceof MultiResolutionStreamConfigurationMap) { + final MultiResolutionStreamConfigurationMap other = + (MultiResolutionStreamConfigurationMap) obj; + if (!mConfigurations.keySet().equals(other.mConfigurations.keySet())) { + return false; + } + + for (String id : mConfigurations.keySet()) { + if (!Arrays.equals(mConfigurations.get(id), other.mConfigurations.get(id))) { + return false; + } + } + + return true; + } + return false; + } + + /** + * {@inheritDoc} + */ + @Override + public int hashCode() { + return HashCodeHelpers.hashCodeGeneric( + mConfigurations, mMultiResolutionOutputConfigs, mMultiResolutionInputConfigs); + } + + /** + * Return this {@link MultiResolutionStreamConfigurationMap} as a string representation. + * + *

{@code "MultiResolutionStreamConfigurationMap(Outputs([format1: [w:%d, h:%d, id:%s], ... + * ... [w:%d, h:%d, id:%s]), [format2: [w:%d, h:%d, id:%s], ... [w:%d, h:%d, id:%s]], ...), + * Inputs([format1: [w:%d, h:%d, id:%s], ... [w:%d, h:%d, id:%s], ...).

+ * + * @return string representation of {@link MultiResolutionStreamConfigurationMap} + */ + @Override + public String toString() { + StringBuilder sb = new StringBuilder("MultiResolutionStreamConfigurationMap("); + appendConfigurationsString(sb, /*output*/ true); + sb.append(","); + appendConfigurationsString(sb, /*output*/ false); + sb.append(")"); + + return sb.toString(); + } + + + private final Map mConfigurations; + + /** Format -> list of MultiResolutionStreamInfo used to create MultiResolutionImageReader */ + private final Map> mMultiResolutionOutputConfigs + = new HashMap>(); + /** Format -> list of MultiResolutionStreamInfo used for multi-resolution reprocessing */ + private final Map> mMultiResolutionInputConfigs + = new HashMap>(); +} diff --git a/core/java/android/hardware/camera2/params/MultiResolutionStreamInfo.java b/core/java/android/hardware/camera2/params/MultiResolutionStreamInfo.java new file mode 100644 index 0000000000000..aa1d1d4aaa185 --- /dev/null +++ b/core/java/android/hardware/camera2/params/MultiResolutionStreamInfo.java @@ -0,0 +1,113 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.hardware.camera2.params; + +import android.annotation.NonNull; + +import java.util.Objects; + +/** + * A utility class describing the properties of one stream of fixed-size image buffers + * backing a multi-resolution image stream. + * + *

A group of {@link MultiResolutionStreamInfo} are used to describe the properties of a + * multi-resolution image stream for a particular format. The + * {@link android.hardware.camera2.MultiResolutionImageReader} class represents a + * multi-resolution output stream, and is constructed using a group of + * {@link MultiResolutionStreamInfo}. A group of {@link MultiResolutionStreamInfo} can also be used + * to create a multi-resolution reprocessable camera capture session. See + * {@link android.hardware.camera2.params.InputConfiguration} for details.

+ * + * @see InputConfiguration + * @see android.hardware.camera2.MultiResolutionImageReader + */ +public class MultiResolutionStreamInfo { + private int mStreamWidth; + private int mStreamHeight; + private String mPhysicalCameraId; + + /** + * Create a new {@link MultiResolutionStreamInfo}. + * + *

This class creates a {@link MultiResolutionStreamInfo} using image width, image height, + * and the physical camera Id images originate from.

+ * + *

Normally applications do not need to create these directly. Use {@link + * MultiResolutionStreamConfigurationMap#getOutputInfo} or {@link + * MultiResolutionStreamConfigurationMap#getInputInfo} to obtain them for a particular format + * instead.

+ */ + public MultiResolutionStreamInfo(int streamWidth, int streamHeight, + @NonNull String physicalCameraId) { + mStreamWidth = streamWidth; + mStreamHeight = streamHeight; + mPhysicalCameraId = physicalCameraId; + } + + /** + * The width of this particular image buffer stream in pixels. + */ + public int getWidth() { + return mStreamWidth; + } + + /** + * The height of this particular image buffer stream in pixels. + */ + public int getHeight() { + return mStreamHeight; + } + + /** + * The physical camera Id of this particular image buffer stream. + */ + public @NonNull String getPhysicalCameraId() { + return mPhysicalCameraId; + } + + /** + * Check if this {@link MultiResolutionStreamInfo} is equal to another + * {@link MultiResolutionStreamInfo}. + * + * @return {@code true} if the objects were equal, {@code false} otherwise + */ + @Override + public boolean equals(final Object obj) { + if (obj == null) { + return false; + } + if (this == obj) { + return true; + } + if (obj instanceof MultiResolutionStreamInfo) { + final MultiResolutionStreamInfo other = (MultiResolutionStreamInfo) obj; + return mStreamWidth == other.mStreamWidth && + mStreamHeight == other.mStreamHeight && + mPhysicalCameraId.equals(other.mPhysicalCameraId); + } + return false; + } + + /** + * {@inheritDoc} + */ + @Override + public int hashCode() { + return Objects.hash( + mStreamWidth, mStreamHeight, mPhysicalCameraId); + } +} diff --git a/core/java/android/hardware/camera2/params/OutputConfiguration.java b/core/java/android/hardware/camera2/params/OutputConfiguration.java index a20a1bf194ea6..e31bd601fc03a 100644 --- a/core/java/android/hardware/camera2/params/OutputConfiguration.java +++ b/core/java/android/hardware/camera2/params/OutputConfiguration.java @@ -24,9 +24,13 @@ import android.annotation.Nullable; import android.annotation.SystemApi; import android.graphics.ImageFormat; import android.hardware.camera2.CameraCaptureSession; +import android.hardware.camera2.CameraCharacteristics; import android.hardware.camera2.CameraDevice; +import android.hardware.camera2.MultiResolutionImageReader; +import android.hardware.camera2.params.MultiResolutionStreamInfo; import android.hardware.camera2.utils.HashCodeHelpers; import android.hardware.camera2.utils.SurfaceUtils; +import android.media.ImageReader; import android.os.Parcel; import android.os.Parcelable; import android.util.Log; @@ -34,6 +38,7 @@ import android.util.Size; import android.view.Surface; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Objects; @@ -81,6 +86,13 @@ import java.util.Objects; * {@link CameraCaptureSession#updateOutputConfiguration} can be called after the configuration * finalize method returns without exceptions. * + *
  • If the camera device supports multi-resolution output streams, {@link + * CameraCharacteristics#SCALER_MULTI_RESOLUTION_STREAM_CONFIGURATION_MAP} will contain the + * formats and their corresponding stream info. The application can use an OutputConfiguration + * created with the multi-resolution stream info queried from {@link + * MultiResolutionStreamConfigurationMap#getOutputInfo} and + * {@link android.hardware.camera2.MultiResolutionImageReader} to capture variable size images. + * * * *

    As of {@link android.os.Build.VERSION_CODES#P Android P}, all formats except @@ -88,6 +100,7 @@ import java.util.Objects; * device support. On prior API levels, only {@link ImageFormat#PRIVATE} format may be used.

    * * @see CameraDevice#createCaptureSessionByOutputConfigurations + * @see CameraCharacteristics#SCALER_MULTI_RESOLUTION_STREAM_CONFIGURATION_MAP * */ public final class OutputConfiguration implements Parcelable { @@ -205,6 +218,33 @@ public final class OutputConfiguration implements Parcelable { this(surfaceGroupId, surface, ROTATION_0); } + /** + * Set the multi-resolution output flag. + * + *

    Specify that this OutputConfiguration is part of a multi-resolution output stream group + * used by {@link android.hardware.camera2.MultiResolutionImageReader}.

    + * + *

    This function must only be called for an OutputConfiguration with a non-negative + * group ID. And all OutputConfigurations of a MultiResolutionImageReader will have the same + * group ID and have this flag set.

    + * + * @throws IllegalStateException If surface sharing is enabled via {@link #enableSurfaceSharing} + * call, or no non-negative group ID has been set. + * @hide + */ + void setMultiResolutionOutput() { + if (mIsShared) { + throw new IllegalStateException("Multi-resolution output flag must not be set for " + + "configuration with surface sharing"); + } + if (mSurfaceGroupId == SURFACE_GROUP_ID_NONE) { + throw new IllegalStateException("Multi-resolution output flag should only be set for " + + "surface with non-negative group ID"); + } + + mIsMultiResolution = true; + } + /** * Create a new {@link OutputConfiguration} instance. * @@ -265,6 +305,45 @@ public final class OutputConfiguration implements Parcelable { mIsDeferredConfig = false; mIsShared = false; mPhysicalCameraId = null; + mIsMultiResolution = false; + } + + /** + * Create a list of {@link OutputConfiguration} instances for the outputs used by a + * {@link android.hardware.camera2.MultiResolutionImageReader}. + * + *

    This constructor takes an argument for a + * {@link android.hardware.camera2.MultiResolutionImageReader}.

    + * + * @param multiResolutionImageReader + * The multi-resolution image reader object. + */ + public static @NonNull Collection createInstancesForMultiResolutionOutput( + @NonNull MultiResolutionImageReader multiResolutionImageReader) { + checkNotNull(multiResolutionImageReader, "Multi-resolution image reader must not be null"); + + int groupId = MULTI_RESOLUTION_GROUP_ID_COUNTER; + MULTI_RESOLUTION_GROUP_ID_COUNTER++; + // Skip in case the group id counter overflows to -1, the invalid value. + if (MULTI_RESOLUTION_GROUP_ID_COUNTER == -1) { + MULTI_RESOLUTION_GROUP_ID_COUNTER++; + } + + ImageReader[] imageReaders = multiResolutionImageReader.getReaders(); + ArrayList configs = new ArrayList(); + for (int i = 0; i < imageReaders.length; i++) { + MultiResolutionStreamInfo streamInfo = + multiResolutionImageReader.getStreamInfoForImageReader(imageReaders[i]); + + OutputConfiguration config = new OutputConfiguration( + groupId, imageReaders[i].getSurface()); + config.setPhysicalCameraId(streamInfo.getPhysicalCameraId()); + config.setMultiResolutionOutput(); + configs.add(config); + // TODO: Set sensor pixel mode for ultra high resolution sensor camera. + } + + return configs; } /** @@ -319,6 +398,7 @@ public final class OutputConfiguration implements Parcelable { mIsDeferredConfig = true; mIsShared = false; mPhysicalCameraId = null; + mIsMultiResolution = false; } /** @@ -355,8 +435,18 @@ public final class OutputConfiguration implements Parcelable { *

    Up to {@link #getMaxSharedSurfaceCount} surfaces can be shared for an OutputConfiguration. * The supported surfaces for sharing must be of type SurfaceTexture, SurfaceView, * MediaRecorder, MediaCodec, or implementation defined ImageReader.

    + * + *

    This function must not be called from OuptutConfigurations created by {@link + * #createInstancesForMultiResolutionOutput}.

    + * + * @throws IllegalStateException If this OutputConfiguration is created via {@link + * #createInstancesForMultiResolutionOutput} to back a MultiResolutionImageReader. */ public void enableSurfaceSharing() { + if (mIsMultiResolution) { + throw new IllegalStateException("Cannot enable surface sharing on " + + "multi-resolution output configurations"); + } mIsShared = true; } @@ -368,8 +458,7 @@ public final class OutputConfiguration implements Parcelable { * This call achieves it by mapping the OutputConfiguration to the physical camera id.

    * *

    The valid physical camera ids can be queried by {@link - * android.hardware.camera2.CameraCharacteristics#getPhysicalCameraIds}. - *

    + * CameraCharacteristics#getPhysicalCameraIds}.

    * *

    Passing in a null physicalCameraId means that the OutputConfiguration is for a logical * stream.

    @@ -380,8 +469,16 @@ public final class OutputConfiguration implements Parcelable { * after {@link CameraDevice#createCaptureSessionByOutputConfigurations} or {@link * CameraDevice#createReprocessableCaptureSessionByConfigurations} has no effect.

    * - *

    The surface belonging to a physical camera OutputConfiguration must not be used as input - * or output of a reprocessing request.

    + *

    As of {@link android.os.Build.VERSION_CODES#S Android 12}, an image buffer from a + * physical camera stream can be used for reprocessing to logical camera streams and streams + * from the same physical camera if the camera device supports multi-resolution input and output + * streams. See {@link CameraCharacteristics#SCALER_MULTI_RESOLUTION_STREAM_CONFIGURATION_MAP} + * for details. The behaviors of reprocessing from a non-physical camera stream to a physical + * camera stream, and from a physical camera stream to a physical camera stream of different + * physical camera, are device-specific and not guaranteed to be supported.

    + * + *

    On prior API levels, the surface belonging to a physical camera OutputConfiguration must + * not be used as input or output of a reprocessing request.

    */ public void setPhysicalCameraId(@Nullable String physicalCameraId) { mPhysicalCameraId = physicalCameraId; @@ -527,6 +624,7 @@ public final class OutputConfiguration implements Parcelable { this.mIsDeferredConfig = other.mIsDeferredConfig; this.mIsShared = other.mIsShared; this.mPhysicalCameraId = other.mPhysicalCameraId; + this.mIsMultiResolution = other.mIsMultiResolution; } /** @@ -543,6 +641,7 @@ public final class OutputConfiguration implements Parcelable { ArrayList surfaces = new ArrayList(); source.readTypedList(surfaces, Surface.CREATOR); String physicalCameraId = source.readString(); + boolean isMultiResolutionOutput = source.readInt() == 1; checkArgumentInRange(rotation, ROTATION_0, ROTATION_270, "Rotation constant"); @@ -566,6 +665,7 @@ public final class OutputConfiguration implements Parcelable { mConfiguredGenerationId = 0; } mPhysicalCameraId = physicalCameraId; + mIsMultiResolution = isMultiResolutionOutput; } /** @@ -665,6 +765,7 @@ public final class OutputConfiguration implements Parcelable { dest.writeInt(mIsShared ? 1 : 0); dest.writeTypedList(mSurfaces); dest.writeString(mPhysicalCameraId); + dest.writeInt(mIsMultiResolution ? 1 : 0); } /** @@ -694,7 +795,8 @@ public final class OutputConfiguration implements Parcelable { mConfiguredFormat != other.mConfiguredFormat || mConfiguredDataspace != other.mConfiguredDataspace || mConfiguredGenerationId != other.mConfiguredGenerationId || - !Objects.equals(mPhysicalCameraId, other.mPhysicalCameraId)) + !Objects.equals(mPhysicalCameraId, other.mPhysicalCameraId) || + mIsMultiResolution != other.mIsMultiResolution) return false; int minLen = Math.min(mSurfaces.size(), other.mSurfaces.size()); @@ -720,17 +822,24 @@ public final class OutputConfiguration implements Parcelable { return HashCodeHelpers.hashCode( mRotation, mConfiguredSize.hashCode(), mConfiguredFormat, mConfiguredDataspace, mSurfaceGroupId, mSurfaceType, mIsShared ? 1 : 0, - mPhysicalCameraId == null ? 0 : mPhysicalCameraId.hashCode()); + mPhysicalCameraId == null ? 0 : mPhysicalCameraId.hashCode(), + mIsMultiResolution ? 1 : 0); } return HashCodeHelpers.hashCode( mRotation, mSurfaces.hashCode(), mConfiguredGenerationId, mConfiguredSize.hashCode(), mConfiguredFormat, mConfiguredDataspace, mSurfaceGroupId, mIsShared ? 1 : 0, - mPhysicalCameraId == null ? 0 : mPhysicalCameraId.hashCode()); + mPhysicalCameraId == null ? 0 : mPhysicalCameraId.hashCode(), + mIsMultiResolution ? 1 : 0); } private static final String TAG = "OutputConfiguration"; + + // A surfaceGroupId counter used for MultiResolutionImageReader. Its value is + // incremented everytime {@link createInstancesForMultiResolutionOutput} is called. + private static int MULTI_RESOLUTION_GROUP_ID_COUNTER = 0; + private ArrayList mSurfaces; private final int mRotation; private final int mSurfaceGroupId; @@ -749,4 +858,7 @@ public final class OutputConfiguration implements Parcelable { private boolean mIsShared; // The physical camera id that this output configuration is for. private String mPhysicalCameraId; + // Flag indicating if this config is for a multi-resolution output with a + // MultiResolutionImageReader + private boolean mIsMultiResolution; } diff --git a/core/java/android/hardware/camera2/params/SessionConfiguration.java b/core/java/android/hardware/camera2/params/SessionConfiguration.java index 8fc919f142a21..ea6b92d4f33ec 100644 --- a/core/java/android/hardware/camera2/params/SessionConfiguration.java +++ b/core/java/android/hardware/camera2/params/SessionConfiguration.java @@ -130,11 +130,13 @@ public final class SessionConfiguration implements Parcelable { int inputWidth = source.readInt(); int inputHeight = source.readInt(); int inputFormat = source.readInt(); + boolean isInputMultiResolution = source.readBoolean(); ArrayList outConfigs = new ArrayList(); source.readTypedList(outConfigs, OutputConfiguration.CREATOR); if ((inputWidth > 0) && (inputHeight > 0) && (inputFormat != -1)) { - mInputConfig = new InputConfiguration(inputWidth, inputHeight, inputFormat); + mInputConfig = new InputConfiguration(inputWidth, inputHeight, + inputFormat, isInputMultiResolution); } mSessionType = sessionType; mOutputConfigurations = outConfigs; @@ -169,10 +171,12 @@ public final class SessionConfiguration implements Parcelable { dest.writeInt(mInputConfig.getWidth()); dest.writeInt(mInputConfig.getHeight()); dest.writeInt(mInputConfig.getFormat()); + dest.writeBoolean(mInputConfig.isMultiResolution()); } else { dest.writeInt(/*inputWidth*/ 0); dest.writeInt(/*inputHeight*/ 0); dest.writeInt(/*inputFormat*/ -1); + dest.writeBoolean(/*isMultiResolution*/ false); } dest.writeTypedList(mOutputConfigurations); } diff --git a/core/java/android/hardware/camera2/params/StreamConfigurationMap.java b/core/java/android/hardware/camera2/params/StreamConfigurationMap.java index 10a814acd70b7..a25ae6041d77c 100644 --- a/core/java/android/hardware/camera2/params/StreamConfigurationMap.java +++ b/core/java/android/hardware/camera2/params/StreamConfigurationMap.java @@ -1575,7 +1575,7 @@ public final class StreamConfigurationMap { return sizes; } - /** Get the list of publically visible output formats; does not include IMPL_DEFINED */ + /** Get the list of publicly visible output formats */ private int[] getPublicFormats(boolean output) { int[] formats = new int[getPublicFormatCount(output)]; @@ -1746,6 +1746,21 @@ public final class StreamConfigurationMap { return sb.toString(); } + /** + * Size comparison method used by size comparators. + * + * @hide + */ + public static int compareSizes(int widthA, int heightA, int widthB, int heightB) { + long left = widthA * (long) heightA; + long right = widthB * (long) heightB; + if (left == right) { + left = widthA; + right = widthB; + } + return (left < right) ? -1 : (left > right ? 1 : 0); + } + private void appendOutputsString(StringBuilder sb) { sb.append("Outputs("); int[] formats = getOutputFormats(); @@ -1843,7 +1858,10 @@ public final class StreamConfigurationMap { sb.append(")"); } - private String formatToString(int format) { + /** + * @hide + */ + public static String formatToString(int format) { switch (format) { case ImageFormat.YV12: return "YV12"; diff --git a/media/java/android/media/ImageReader.java b/media/java/android/media/ImageReader.java index f5b204a4a9082..5656dffe1d4ad 100644 --- a/media/java/android/media/ImageReader.java +++ b/media/java/android/media/ImageReader.java @@ -16,6 +16,7 @@ package android.media; +import android.annotation.CallbackExecutor; import android.annotation.IntRange; import android.annotation.NonNull; import android.graphics.GraphicBuffer; @@ -24,6 +25,7 @@ import android.graphics.ImageFormat.Format; import android.graphics.Rect; import android.hardware.HardwareBuffer; import android.hardware.HardwareBuffer.Usage; +import android.hardware.camera2.MultiResolutionImageReader; import android.os.Handler; import android.os.Looper; import android.os.Message; @@ -36,7 +38,9 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.NioUtils; import java.util.List; +import java.util.Objects; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.Executor; import java.util.concurrent.atomic.AtomicBoolean; /** @@ -134,7 +138,8 @@ public class ImageReader implements AutoCloseable { // If the format is private don't default to USAGE_CPU_READ_OFTEN since it may not // work, and is inscrutable anyway return new ImageReader(width, height, format, maxImages, - format == ImageFormat.PRIVATE ? 0 : HardwareBuffer.USAGE_CPU_READ_OFTEN); + format == ImageFormat.PRIVATE ? 0 : HardwareBuffer.USAGE_CPU_READ_OFTEN, + /*parent*/ null); } /** @@ -250,18 +255,37 @@ public class ImageReader implements AutoCloseable { // throw new IllegalArgumentException("The given format=" + Integer.toHexString(format) // + " & usage=" + Long.toHexString(usage) + " is not supported"); // } - return new ImageReader(width, height, format, maxImages, usage); + return new ImageReader(width, height, format, maxImages, usage, /*parent*/ null); } + /** + * @hide + */ + public static @NonNull ImageReader newInstance( + @IntRange(from = 1) int width, + @IntRange(from = 1) int height, + @Format int format, + @IntRange(from = 1) int maxImages, + @NonNull MultiResolutionImageReader parent) { + // If the format is private don't default to USAGE_CPU_READ_OFTEN since it may not + // work, and is inscrutable anyway + return new ImageReader(width, height, format, maxImages, + format == ImageFormat.PRIVATE ? 0 : HardwareBuffer.USAGE_CPU_READ_OFTEN, + parent); + } + + /** * @hide */ - protected ImageReader(int width, int height, int format, int maxImages, long usage) { + protected ImageReader(int width, int height, int format, int maxImages, long usage, + MultiResolutionImageReader parent) { mWidth = width; mHeight = height; mFormat = format; mUsage = usage; mMaxImages = maxImages; + mParent = parent; if (width < 1 || height < 1) { throw new IllegalArgumentException( @@ -433,6 +457,9 @@ public class ImageReader implements AutoCloseable { if (image != null) { image.close(); } + if (mParent != null) { + mParent.flushOther(this); + } } } @@ -584,12 +611,38 @@ public class ImageReader implements AutoCloseable { } if (mListenerHandler == null || mListenerHandler.getLooper() != looper) { mListenerHandler = new ListenerHandler(looper); + mListenerExecutor = new HandlerExecutor(mListenerHandler); } - mListener = listener; } else { - mListener = null; mListenerHandler = null; + mListenerExecutor = null; } + mListener = listener; + } + } + + /** + * Register a listener to be invoked when a new image becomes available + * from the ImageReader. + * + * @param listener + * The listener that will be run. + * @param executor + * The executor which will be used to invoke the listener. + * @throws IllegalArgumentException + * If no handler specified and the calling thread has no looper. + * + * @hide + */ + public void setOnImageAvailableListenerWithExecutor(@NonNull OnImageAvailableListener listener, + @NonNull Executor executor) { + if (executor == null) { + throw new IllegalArgumentException("executor must not be null"); + } + + synchronized (mListenerLock) { + mListenerExecutor = executor; + mListener = listener; } } @@ -763,12 +816,27 @@ public class ImageReader implements AutoCloseable { return; } - final Handler handler; + final Executor executor; + final OnImageAvailableListener listener; synchronized (ir.mListenerLock) { - handler = ir.mListenerHandler; + executor = ir.mListenerExecutor; + listener = ir.mListener; } - if (handler != null) { - handler.sendEmptyMessage(0); + final boolean isReaderValid; + synchronized (ir.mCloseLock) { + isReaderValid = ir.mIsReaderValid; + } + + // It's dangerous to fire onImageAvailable() callback when the ImageReader + // is being closed, as application could acquire next image in the + // onImageAvailable() callback. + if (executor != null && listener != null && isReaderValid) { + executor.execute(new Runnable() { + @Override + public void run() { + listener.onImageAvailable(ir); + } + }); } } @@ -785,11 +853,16 @@ public class ImageReader implements AutoCloseable { private final Object mCloseLock = new Object(); private boolean mIsReaderValid = false; private OnImageAvailableListener mListener; + private Executor mListenerExecutor; private ListenerHandler mListenerHandler; // Keep track of the successfully acquired Images. This need to be thread safe as the images // could be closed by different threads (e.g., application thread and GC thread). private List mAcquiredImages = new CopyOnWriteArrayList<>(); + // Applicable if this isn't a standalone ImageReader, but belongs to a + // MultiResolutionImageReader. + private final MultiResolutionImageReader mParent; + /** * This field is used by native code, do not access or modify. */ @@ -802,23 +875,22 @@ public class ImageReader implements AutoCloseable { public ListenerHandler(Looper looper) { super(looper, null, true /*async*/); } + } + + /** + * An adapter {@link Executor} that posts all executed tasks onto the + * given {@link Handler}. + **/ + private final class HandlerExecutor implements Executor { + private final Handler mHandler; + + public HandlerExecutor(@NonNull Handler handler) { + mHandler = Objects.requireNonNull(handler); + } @Override - public void handleMessage(Message msg) { - OnImageAvailableListener listener; - synchronized (mListenerLock) { - listener = mListener; - } - - // It's dangerous to fire onImageAvailable() callback when the ImageReader is being - // closed, as application could acquire next image in the onImageAvailable() callback. - boolean isReaderValid = false; - synchronized (mCloseLock) { - isReaderValid = mIsReaderValid; - } - if (listener != null && isReaderValid) { - listener.onImageAvailable(ImageReader.this); - } + public void execute(Runnable command) { + mHandler.post(command); } }