Merge "Use heif embedded thumbnail if available" into pi-dev

am: 11d203ea0d

Change-Id: I99ecf1986d921c00f70e7b9dbd2a8c67c5c9a923

media/java/android/media/MediaMetadataRetriever.java

@@ -566,6 +566,25 @@ public class MediaMetadataRetriever
         return getImageAtIndexInternal(imageIndex, params);
     }
 
+    /**
+     * @hide
+     *
+     * This method retrieves the thumbnail image for a still image if it's available.
+     * It should only be called after {@link #setDataSource}.
+     *
+     * @param imageIndex 0-based index of the image, negative value indicates primary image.
+     * @param params BitmapParams that controls the returned bitmap config (such as pixel formats).
+     * @param targetSize intended size of one edge (wdith or height) of the thumbnail,
+     *                   this is a heuristic for the framework to decide whether the embedded
+     *                   thumbnail should be used.
+     * @param maxPixels maximum pixels of thumbnail, this is a heuristic for the frameowrk to
+     *                  decide whehther the embedded thumnbail (or a downscaled version of it)
+     *                  should be used.
+     * @return the retrieved thumbnail, or null if no suitable thumbnail is available.
+     */
+    public native @Nullable Bitmap getThumbnailImageAtIndex(
+            int imageIndex, @NonNull BitmapParams params, int targetSize, int maxPixels);
+
     /**
      * This method is similar to {@link #getImageAtIndex(int, BitmapParams)} except that
      * the default for {@link BitmapParams} will be used.

media/java/android/media/ThumbnailUtils.java

@@ -92,10 +92,14 @@ public class ThumbnailUtils {
         SizedThumbnailBitmap sizedThumbnailBitmap = new SizedThumbnailBitmap();
         Bitmap bitmap = null;
         MediaFileType fileType = MediaFile.getFileType(filePath);
-        if (fileType != null && (fileType.fileType == MediaFile.FILE_TYPE_JPEG
-                || MediaFile.isRawImageFileType(fileType.fileType))) {
-            createThumbnailFromEXIF(filePath, targetSize, maxPixels, sizedThumbnailBitmap);
-            bitmap = sizedThumbnailBitmap.mBitmap;
+        if (fileType != null) {
+            if (fileType.fileType == MediaFile.FILE_TYPE_JPEG
+                    || MediaFile.isRawImageFileType(fileType.fileType)) {
+                createThumbnailFromEXIF(filePath, targetSize, maxPixels, sizedThumbnailBitmap);
+                bitmap = sizedThumbnailBitmap.mBitmap;
+            } else if (fileType.fileType == MediaFile.FILE_TYPE_HEIF) {
+                bitmap = createThumbnailFromMetadataRetriever(filePath, targetSize, maxPixels);
+            }
         }
 
         if (bitmap == null) {
@@ -519,4 +523,26 @@ public class ThumbnailUtils {
             sizedThumbBitmap.mBitmap = BitmapFactory.decodeFile(filePath, fullOptions);
         }
     }
+
+    private static Bitmap createThumbnailFromMetadataRetriever(
+            String filePath, int targetSize, int maxPixels) {
+        if (filePath == null) {
+            return null;
+        }
+        Bitmap thumbnail = null;
+        MediaMetadataRetriever retriever = new MediaMetadataRetriever();
+        try {
+            retriever.setDataSource(filePath);
+            MediaMetadataRetriever.BitmapParams params = new MediaMetadataRetriever.BitmapParams();
+            params.setPreferredConfig(Bitmap.Config.ARGB_8888);
+            thumbnail = retriever.getThumbnailImageAtIndex(-1, params, targetSize, maxPixels);
+        } catch (RuntimeException ex) {
+            // Assume this is a corrupt video file.
+        } finally {
+            if (retriever != null) {
+                retriever.release();
+            }
+        }
+        return thumbnail;
+    }
 }

media/jni/android_media_MediaMetadataRetriever.cpp

@@ -263,7 +263,9 @@ static void rotate(T *dst, const T *src, size_t width, size_t height, int angle)
 static jobject getBitmapFromVideoFrame(
         JNIEnv *env, VideoFrame *videoFrame, jint dst_width, jint dst_height,
         SkColorType outColorType) {
-    ALOGV("getBitmapFromVideoFrame: dimension = %dx%d and bytes = %d",
+    ALOGV("getBitmapFromVideoFrame: dimension = %dx%d, displaySize = %dx%d, bytes = %d",
+            videoFrame->mWidth,
+            videoFrame->mHeight,
             videoFrame->mDisplayWidth,
             videoFrame->mDisplayHeight,
             videoFrame->mSize);
@@ -330,8 +332,7 @@ static jobject getBitmapFromVideoFrame(
         dst_height = std::round(displayHeight * factor);
     }
 
-    if ((uint32_t)dst_width != videoFrame->mWidth ||
-        (uint32_t)dst_height != videoFrame->mHeight) {
+    if ((uint32_t)dst_width != width || (uint32_t)dst_height != height) {
         ALOGV("Bitmap dimension is scaled from %dx%d to %dx%d",
                 width, height, dst_width, dst_height);
         jobject scaledBitmap = env->CallStaticObjectMethod(fields.bitmapClazz,
@@ -433,6 +434,61 @@ static jobject android_media_MediaMetadataRetriever_getImageAtIndex(
     return getBitmapFromVideoFrame(env, videoFrame, -1, -1, outColorType);
 }
 
+static jobject android_media_MediaMetadataRetriever_getThumbnailImageAtIndex(
+        JNIEnv *env, jobject thiz, jint index, jobject params, jint targetSize, jint maxPixels)
+{
+    ALOGV("getThumbnailImageAtIndex: index %d", index);
+
+    sp<MediaMetadataRetriever> retriever = getRetriever(env, thiz);
+    if (retriever == 0) {
+        jniThrowException(env, "java/lang/IllegalStateException", "No retriever available");
+        return NULL;
+    }
+
+    int colorFormat = getColorFormat(env, params);
+    jint dst_width = -1, dst_height = -1;
+
+    // Call native method to retrieve an image
+    VideoFrame *videoFrame = NULL;
+    sp<IMemory> frameMemory = retriever->getImageAtIndex(
+            index, colorFormat, true /*metaOnly*/, true /*thumbnail*/);
+    if (frameMemory != 0) {
+        videoFrame = static_cast<VideoFrame *>(frameMemory->pointer());
+        int32_t thumbWidth = videoFrame->mWidth;
+        int32_t thumbHeight = videoFrame->mHeight;
+        videoFrame = NULL;
+        int64_t thumbPixels = thumbWidth * thumbHeight;
+
+        // Here we try to use the included thumbnail if it's not too shabby.
+        // If this fails ThumbnailUtils would have to decode the full image and
+        // downscale which could take long.
+        if (thumbWidth >= targetSize || thumbHeight >= targetSize
+                || thumbPixels * 6 >= maxPixels) {
+            frameMemory = retriever->getImageAtIndex(
+                    index, colorFormat, false /*metaOnly*/, true /*thumbnail*/);
+            videoFrame = static_cast<VideoFrame *>(frameMemory->pointer());
+
+            if (thumbPixels > maxPixels) {
+                int downscale = ceil(sqrt(thumbPixels / (float)maxPixels));
+                dst_width = thumbWidth / downscale;
+                dst_height = thumbHeight /downscale;
+            }
+        }
+    }
+    if (videoFrame == NULL) {
+        ALOGV("getThumbnailImageAtIndex: no suitable thumbnails available");
+        return NULL;
+    }
+
+    // Ignore rotation for thumbnail extraction to be consistent with
+    // thumbnails extracted by BitmapFactory APIs.
+    videoFrame->mRotationAngle = 0;
+
+    SkColorType outColorType = setOutColorType(env, colorFormat, params);
+
+    return getBitmapFromVideoFrame(env, videoFrame, dst_width, dst_height, outColorType);
+}
+
 static jobject android_media_MediaMetadataRetriever_getFrameAtIndex(
         JNIEnv *env, jobject thiz, jint frameIndex, jint numFrames, jobject params)
 {
@@ -663,6 +719,12 @@ static const JNINativeMethod nativeMethods[] = {
             (void *)android_media_MediaMetadataRetriever_getImageAtIndex
         },
 
+        {
+            "getThumbnailImageAtIndex",
+            "(ILandroid/media/MediaMetadataRetriever$BitmapParams;II)Landroid/graphics/Bitmap;",
+            (void *)android_media_MediaMetadataRetriever_getThumbnailImageAtIndex
+        },
+
         {
             "_getFrameAtIndex",
             "(IILandroid/media/MediaMetadataRetriever$BitmapParams;)Ljava/util/List;",