From 90ae8577384a4d8dd31b55c5486835f2c6e2592d Mon Sep 17 00:00:00 2001 From: Yushin Cho Date: Fri, 12 Nov 2021 15:34:56 -0800 Subject: [PATCH] enc stat: Add feauture and its keys to MediaCodec Add feature name 'EncodingStatistics' to MediaCodecInfo and its two keys of average_qp and picture_type to MediaFormat. App client chooses the 'Level' of an Encoding Statistics information, where the Level decides the scope of encoding statistics that video encoder provide the App client with. With Android T, there will be one level only and the expected kinds of encoding statistics are average block QP and the picture type of a frame. Bug: 205793764 Bug: 203696003 Test: cts Change-Id: Id8c61de9646ac03926f00a9d223b022bbb9a6925 --- core/api/current.txt | 10 +++ media/java/android/media/MediaCodecInfo.java | 13 ++++ media/java/android/media/MediaFormat.java | 70 ++++++++++++++++++++ 3 files changed, 93 insertions(+) diff --git a/core/api/current.txt b/core/api/current.txt index d696bc7978254..ff3ffdc04a66f 100644 --- a/core/api/current.txt +++ b/core/api/current.txt @@ -21920,6 +21920,7 @@ package android.media { field @Deprecated public static final int COLOR_TI_FormatYUV420PackedSemiPlanar = 2130706688; // 0x7f000100 field public static final String FEATURE_AdaptivePlayback = "adaptive-playback"; field public static final String FEATURE_DynamicTimestamp = "dynamic-timestamp"; + field public static final String FEATURE_EncodingStatistics = "encoding-statistics"; field public static final String FEATURE_FrameParsing = "frame-parsing"; field public static final String FEATURE_IntraRefresh = "intra-refresh"; field public static final String FEATURE_LowLatency = "low-latency"; @@ -22693,6 +22694,7 @@ package android.media { field public static final String KEY_OPERATING_RATE = "operating-rate"; field public static final String KEY_OUTPUT_REORDER_DEPTH = "output-reorder-depth"; field public static final String KEY_PCM_ENCODING = "pcm-encoding"; + field public static final String KEY_PICTURE_TYPE = "picture-type"; field public static final String KEY_PIXEL_ASPECT_RATIO_HEIGHT = "sar-height"; field public static final String KEY_PIXEL_ASPECT_RATIO_WIDTH = "sar-width"; field public static final String KEY_PREPEND_HEADER_TO_SYNC_FRAMES = "prepend-sps-pps-to-idr-frames"; @@ -22710,6 +22712,8 @@ package android.media { field public static final String KEY_TILE_HEIGHT = "tile-height"; field public static final String KEY_TILE_WIDTH = "tile-width"; field public static final String KEY_TRACK_ID = "track-id"; + field public static final String KEY_VIDEO_ENCODING_STATISTICS_LEVEL = "video-encoding-statistics-level"; + field public static final String KEY_VIDEO_QP_AVERAGE = "video-qp-average"; field public static final String KEY_VIDEO_QP_B_MAX = "video-qp-b-max"; field public static final String KEY_VIDEO_QP_B_MIN = "video-qp-b-min"; field public static final String KEY_VIDEO_QP_I_MAX = "video-qp-i-max"; @@ -22754,12 +22758,18 @@ package android.media { field public static final String MIMETYPE_VIDEO_SCRAMBLED = "video/scrambled"; field public static final String MIMETYPE_VIDEO_VP8 = "video/x-vnd.on2.vp8"; field public static final String MIMETYPE_VIDEO_VP9 = "video/x-vnd.on2.vp9"; + field public static final int PICTURE_TYPE_B = 3; // 0x3 + field public static final int PICTURE_TYPE_I = 1; // 0x1 + field public static final int PICTURE_TYPE_P = 2; // 0x2 + field public static final int PICTURE_TYPE_UNKNOWN = 0; // 0x0 field public static final int TYPE_BYTE_BUFFER = 5; // 0x5 field public static final int TYPE_FLOAT = 3; // 0x3 field public static final int TYPE_INTEGER = 1; // 0x1 field public static final int TYPE_LONG = 2; // 0x2 field public static final int TYPE_NULL = 0; // 0x0 field public static final int TYPE_STRING = 4; // 0x4 + field public static final int VIDEO_ENCODING_STATISTICS_LEVEL_1 = 1; // 0x1 + field public static final int VIDEO_ENCODING_STATISTICS_LEVEL_NONE = 0; // 0x0 } public final class MediaMetadata implements android.os.Parcelable { diff --git a/media/java/android/media/MediaCodecInfo.java b/media/java/android/media/MediaCodecInfo.java index 3c152fb68c0af..e75df1d9b6913 100644 --- a/media/java/android/media/MediaCodecInfo.java +++ b/media/java/android/media/MediaCodecInfo.java @@ -602,6 +602,18 @@ public final class MediaCodecInfo { @SuppressLint("AllUpper") public static final String FEATURE_QpBounds = "qp-bounds"; + /** + * video encoder only: codec supports exporting encoding statistics. + * Encoders with this feature can provide the App clients with the encoding statistics + * information about the frame. + * The scope of encoding statistics is controlled by + * {@link MediaFormat#KEY_VIDEO_ENCODING_STATISTICS_LEVEL}. + * + * @see MediaFormat#KEY_VIDEO_ENCODING_STATISTICS_LEVEL + */ + @SuppressLint("AllUpper") // for consistency with other FEATURE_* constants + public static final String FEATURE_EncodingStatistics = "encoding-statistics"; + /** * Query codec feature capabilities. *

@@ -641,6 +653,7 @@ public final class MediaCodecInfo { new Feature(FEATURE_MultipleFrames, (1 << 1), false), new Feature(FEATURE_DynamicTimestamp, (1 << 2), false), new Feature(FEATURE_QpBounds, (1 << 3), false), + new Feature(FEATURE_EncodingStatistics, (1 << 4), false), // feature to exclude codec from REGULAR codec list new Feature(FEATURE_SpecialCodec, (1 << 30), false, true), }; diff --git a/media/java/android/media/MediaFormat.java b/media/java/android/media/MediaFormat.java index 9bf0db52f66d7..aa5c404800e81 100644 --- a/media/java/android/media/MediaFormat.java +++ b/media/java/android/media/MediaFormat.java @@ -1101,6 +1101,76 @@ public final class MediaFormat { */ public static final String KEY_VIDEO_QP_B_MIN = "video-qp-b-min"; + /** + * A key describing the level of encoding statistics information emitted from video encoder. + * + * The associated value is an integer. + */ + public static final String KEY_VIDEO_ENCODING_STATISTICS_LEVEL = + "video-encoding-statistics-level"; + + /** + * Encoding Statistics Level None. + * Encoder generates no information about Encoding statistics. + */ + public static final int VIDEO_ENCODING_STATISTICS_LEVEL_NONE = 0; + + /** + * Encoding Statistics Level 1. + * Encoder generates {@link MediaFormat#KEY_PICTURE_TYPE} and + * {@link MediaFormat#KEY_VIDEO_QP_AVERAGE} for each frame. + */ + public static final int VIDEO_ENCODING_STATISTICS_LEVEL_1 = 1; + + /** @hide */ + @IntDef({ + VIDEO_ENCODING_STATISTICS_LEVEL_NONE, + VIDEO_ENCODING_STATISTICS_LEVEL_1, + }) + @Retention(RetentionPolicy.SOURCE) + public @interface VideoEncodingStatisticsLevel {} + + /** + * A key describing the per-frame average block QP (Quantization Parameter). + * This is a part of a video 'Encoding Statistics' export feature. + * This value is emitted from video encoder for a video frame. + * The average value is rounded down (using floor()) to integer value. + * + * The associated value is an integer. + */ + public static final String KEY_VIDEO_QP_AVERAGE = "video-qp-average"; + + /** + * A key describing the picture type of the encoded frame. + * This is a part of a video 'Encoding Statistics' export feature. + * This value is emitted from video encoder for a video frame. + * + * The associated value is an integer. + */ + public static final String KEY_PICTURE_TYPE = "picture-type"; + + /** Picture Type is unknown. */ + public static final int PICTURE_TYPE_UNKNOWN = 0; + + /** Picture Type is I Frame. */ + public static final int PICTURE_TYPE_I = 1; + + /** Picture Type is P Frame. */ + public static final int PICTURE_TYPE_P = 2; + + /** Picture Type is B Frame. */ + public static final int PICTURE_TYPE_B = 3; + + /** @hide */ + @IntDef({ + PICTURE_TYPE_UNKNOWN, + PICTURE_TYPE_I, + PICTURE_TYPE_P, + PICTURE_TYPE_B, + }) + @Retention(RetentionPolicy.SOURCE) + public @interface PictureType {} + /** * A key describing the audio session ID of the AudioTrack associated * to a tunneled video codec.