From b26aebc325fd0e607c59e04d79b3d81d0d8d7edf Mon Sep 17 00:00:00 2001 From: Leon Scroggins III Date: Mon, 21 Dec 2020 14:55:22 -0500 Subject: [PATCH] Implement AImageDecoder _advanceFrame and _rewind Bug: 160984428 Test: Iae7d274b69999c471fd5610c6ef4d148cca81bec Disallow AImageDecoder_set* methods after the first frame, since changing the settings would interfere with blending and caching for kRestorePrevious frames. Add a cache (and a state machine) for handling kRestorePrevious frames. Follow-on to Ib93b0ced09fa3cca4a6681745406355c48158fae - support using a matrix for unpremul + orientation (the orientation was previously handled by a matrix internally in SkAndroidCodec). Change-Id: I7c32ede013fa83f1fe95c35778c33278ca6fe6a3 --- libs/hwui/hwui/ImageDecoder.cpp | 242 +++++++++++++++++---- libs/hwui/hwui/ImageDecoder.h | 50 ++++- native/graphics/jni/imagedecoder.cpp | 70 +++++- native/graphics/jni/libjnigraphics.map.txt | 2 + 4 files changed, 316 insertions(+), 48 deletions(-) diff --git a/libs/hwui/hwui/ImageDecoder.cpp b/libs/hwui/hwui/ImageDecoder.cpp index 6889134719138..b713f88c52c9f 100644 --- a/libs/hwui/hwui/ImageDecoder.cpp +++ b/libs/hwui/hwui/ImageDecoder.cpp @@ -17,11 +17,19 @@ #include "ImageDecoder.h" #include +#include #include +#include +#include #include +#include +#include #include +#undef LOG_TAG +#define LOG_TAG "ImageDecoder" + using namespace android; sk_sp ImageDecoder::getDefaultColorSpace() const { @@ -44,17 +52,29 @@ ImageDecoder::ImageDecoder(std::unique_ptr codec, sk_spcomputeOutputColorType(kN32_SkColorType)) , mUnpremultipliedRequired(false) , mOutColorSpace(getDefaultColorSpace()) - , mSampleSize(1) { mTargetSize = swapWidthHeight() ? SkISize { mDecodeSize.height(), mDecodeSize.width() } : mDecodeSize; + this->rewind(); } +ImageDecoder::~ImageDecoder() = default; + SkAlphaType ImageDecoder::getOutAlphaType() const { return opaque() ? kOpaque_SkAlphaType : mUnpremultipliedRequired ? kUnpremul_SkAlphaType : kPremul_SkAlphaType; } +static SkISize swapped(const SkISize& size) { + return SkISize { size.height(), size.width() }; +} + +static bool requires_matrix_scaling(bool swapWidthHeight, const SkISize& decodeSize, + const SkISize& targetSize) { + return (swapWidthHeight && decodeSize != swapped(targetSize)) + || (!swapWidthHeight && decodeSize != targetSize); +} + bool ImageDecoder::setTargetSize(int width, int height) { if (width <= 0 || height <= 0) { return false; @@ -78,17 +98,21 @@ bool ImageDecoder::setTargetSize(int width, int height) { } } - SkISize targetSize = { width, height }; - SkISize decodeSize = swapWidthHeight() ? SkISize { height, width } : targetSize; + const bool swap = swapWidthHeight(); + const SkISize targetSize = { width, height }; + SkISize decodeSize = swap ? SkISize { height, width } : targetSize; int sampleSize = mCodec->computeSampleSize(&decodeSize); - if (decodeSize != targetSize && mUnpremultipliedRequired && !opaque()) { - return false; + if (mUnpremultipliedRequired && !opaque()) { + // Allow using a matrix to handle orientation, but not scaling. + if (requires_matrix_scaling(swap, decodeSize, targetSize)) { + return false; + } } mTargetSize = targetSize; mDecodeSize = decodeSize; - mSampleSize = sampleSize; + mOptions.fSampleSize = sampleSize; return true; } @@ -137,8 +161,10 @@ bool ImageDecoder::setOutColorType(SkColorType colorType) { } bool ImageDecoder::setUnpremultipliedRequired(bool required) { - if (required && !opaque() && mDecodeSize != mTargetSize) { - return false; + if (required && !opaque()) { + if (requires_matrix_scaling(swapWidthHeight(), mDecodeSize, mTargetSize)) { + return false; + } } mUnpremultipliedRequired = required; return true; @@ -176,51 +202,150 @@ int ImageDecoder::height() const { } bool ImageDecoder::opaque() const { - return mCodec->getInfo().alphaType() == kOpaque_SkAlphaType; + return mCurrentFrameIsOpaque; } bool ImageDecoder::gray() const { return mCodec->getInfo().colorType() == kGray_8_SkColorType; } +bool ImageDecoder::isAnimated() { + return mCodec->codec()->getFrameCount() > 1; +} + +int ImageDecoder::currentFrame() const { + return mOptions.fFrameIndex; +} + +bool ImageDecoder::rewind() { + mOptions.fFrameIndex = 0; + mOptions.fPriorFrame = SkCodec::kNoFrame; + mCurrentFrameIsIndependent = true; + mCurrentFrameIsOpaque = mCodec->getInfo().isOpaque(); + mRestoreState = RestoreState::kDoNothing; + mRestoreFrame = nullptr; + + // TODO: Rewind the input now instead of in the next call to decode, and + // plumb through whether rewind succeeded. + return true; +} + +bool ImageDecoder::advanceFrame() { + const int frameIndex = ++mOptions.fFrameIndex; + const int frameCount = mCodec->codec()->getFrameCount(); + if (frameIndex >= frameCount) { + // Prevent overflow from repeated calls to advanceFrame. + mOptions.fFrameIndex = frameCount; + return false; + } + + SkCodec::FrameInfo frameInfo; + if (!mCodec->codec()->getFrameInfo(frameIndex, &frameInfo) + || !frameInfo.fFullyReceived) { + // Mark the decoder as finished, requiring a rewind. + mOptions.fFrameIndex = frameCount; + return false; + } + + mCurrentFrameIsIndependent = frameInfo.fRequiredFrame == SkCodec::kNoFrame; + mCurrentFrameIsOpaque = frameInfo.fAlphaType == kOpaque_SkAlphaType; + + if (frameInfo.fDisposalMethod == SkCodecAnimation::DisposalMethod::kRestorePrevious) { + switch (mRestoreState) { + case RestoreState::kDoNothing: + case RestoreState::kNeedsRestore: + mRestoreState = RestoreState::kFirstRPFrame; + break; + case RestoreState::kFirstRPFrame: + mRestoreState = RestoreState::kRPFrame; + break; + case RestoreState::kRPFrame: + // Unchanged. + break; + } + } else { // New frame is not restore previous + switch (mRestoreState) { + case RestoreState::kFirstRPFrame: + case RestoreState::kRPFrame: + mRestoreState = RestoreState::kNeedsRestore; + break; + case RestoreState::kNeedsRestore: + mRestoreState = RestoreState::kDoNothing; + mRestoreFrame = nullptr; + [[fallthrough]]; + case RestoreState::kDoNothing: + mOptions.fPriorFrame = frameIndex - 1; + break; + } + } + + return true; +} + +bool ImageDecoder::finished() const { + return mOptions.fFrameIndex >= mCodec->codec()->getFrameCount(); +} + SkCodec::Result ImageDecoder::decode(void* pixels, size_t rowBytes) { + // This was checked inside setTargetSize, but it's possible the first frame + // was opaque, so that method succeeded, but after calling advanceFrame, the + // current frame is not opaque. + if (mUnpremultipliedRequired && !opaque()) { + // Allow using a matrix to handle orientation, but not scaling. + if (requires_matrix_scaling(swapWidthHeight(), mDecodeSize, mTargetSize)) { + return SkCodec::kInvalidScale; + } + } + void* decodePixels = pixels; size_t decodeRowBytes = rowBytes; - auto decodeInfo = SkImageInfo::Make(mDecodeSize, mOutColorType, getOutAlphaType(), - getOutputColorSpace()); + const auto decodeInfo = SkImageInfo::Make(mDecodeSize, mOutColorType, getOutAlphaType(), + getOutputColorSpace()); + const auto outputInfo = getOutputInfo(); + switch (mRestoreState) { + case RestoreState::kFirstRPFrame:{ + // This frame is marked kRestorePrevious. The prior frame should be in + // |pixels|, and it is what we'll restore after each consecutive + // kRestorePrevious frame. Cache it now. + if (!(mRestoreFrame = Bitmap::allocateHeapBitmap(outputInfo))) { + return SkCodec::kInternalError; + } + + const uint8_t* srcRow = static_cast(pixels); + uint8_t* dstRow = static_cast(mRestoreFrame->pixels()); + for (int y = 0; y < outputInfo.height(); y++) { + memcpy(dstRow, srcRow, outputInfo.minRowBytes()); + srcRow += rowBytes; + dstRow += mRestoreFrame->rowBytes(); + } + break; + } + case RestoreState::kRPFrame: + case RestoreState::kNeedsRestore: + // Restore the cached frame. It's possible that the client skipped decoding a frame, so + // we never cached it. + if (mRestoreFrame) { + const uint8_t* srcRow = static_cast(mRestoreFrame->pixels()); + uint8_t* dstRow = static_cast(pixels); + for (int y = 0; y < outputInfo.height(); y++) { + memcpy(dstRow, srcRow, outputInfo.minRowBytes()); + srcRow += mRestoreFrame->rowBytes(); + dstRow += rowBytes; + } + } + break; + case RestoreState::kDoNothing: + break; + } + // Used if we need a temporary before scaling or subsetting. // FIXME: Use scanline decoding on only a couple lines to save memory. b/70709380. SkBitmap tmp; const bool scale = mDecodeSize != mTargetSize; const auto origin = mCodec->codec()->getOrigin(); const bool handleOrigin = origin != kDefault_SkEncodedOrigin; + SkMatrix outputMatrix; if (scale || handleOrigin || mCropRect) { - if (!tmp.setInfo(decodeInfo)) { - return SkCodec::kInternalError; - } - if (!Bitmap::allocateHeapBitmap(&tmp)) { - return SkCodec::kInternalError; - } - decodePixels = tmp.getPixels(); - decodeRowBytes = tmp.rowBytes(); - } - - SkAndroidCodec::AndroidOptions options; - options.fSampleSize = mSampleSize; - auto result = mCodec->getAndroidPixels(decodeInfo, decodePixels, decodeRowBytes, &options); - - if (scale || handleOrigin || mCropRect) { - SkBitmap scaledBm; - if (!scaledBm.installPixels(getOutputInfo(), pixels, rowBytes)) { - return SkCodec::kInternalError; - } - - SkPaint paint; - paint.setBlendMode(SkBlendMode::kSrc); - paint.setFilterQuality(kLow_SkFilterQuality); // bilinear filtering - - SkCanvas canvas(scaledBm, SkCanvas::ColorBehavior::kLegacy); - SkMatrix outputMatrix; if (mCropRect) { outputMatrix.setTranslate(-mCropRect->fLeft, -mCropRect->fTop); } @@ -233,13 +358,54 @@ SkCodec::Result ImageDecoder::decode(void* pixels, size_t rowBytes) { std::swap(targetWidth, targetHeight); } } - if (scale) { float scaleX = (float) targetWidth / mDecodeSize.width(); float scaleY = (float) targetHeight / mDecodeSize.height(); outputMatrix.preScale(scaleX, scaleY); } + // It's possible that this portion *does* have alpha, even if the + // composed frame does not. In that case, the SkBitmap needs to have + // alpha so it blends properly. + if (!tmp.setInfo(decodeInfo.makeAlphaType(mUnpremultipliedRequired ? kUnpremul_SkAlphaType + : kPremul_SkAlphaType))) + { + return SkCodec::kInternalError; + } + if (!Bitmap::allocateHeapBitmap(&tmp)) { + return SkCodec::kInternalError; + } + decodePixels = tmp.getPixels(); + decodeRowBytes = tmp.rowBytes(); + if (!mCurrentFrameIsIndependent) { + SkMatrix inverse; + if (outputMatrix.invert(&inverse)) { + SkCanvas canvas(tmp, SkCanvas::ColorBehavior::kLegacy); + canvas.setMatrix(inverse); + SkPaint paint; + paint.setFilterQuality(kLow_SkFilterQuality); // bilinear + SkBitmap priorFrame; + priorFrame.installPixels(outputInfo, pixels, rowBytes); + canvas.drawBitmap(priorFrame, 0, 0, &paint); + } else { + ALOGE("Failed to invert matrix!"); + } + } + } + + auto result = mCodec->getAndroidPixels(decodeInfo, decodePixels, decodeRowBytes, &mOptions); + + if (scale || handleOrigin || mCropRect) { + SkBitmap scaledBm; + if (!scaledBm.installPixels(outputInfo, pixels, rowBytes)) { + return SkCodec::kInternalError; + } + + SkPaint paint; + paint.setBlendMode(SkBlendMode::kSrc); + paint.setFilterQuality(kLow_SkFilterQuality); // bilinear filtering + + SkCanvas canvas(scaledBm, SkCanvas::ColorBehavior::kLegacy); canvas.setMatrix(outputMatrix); canvas.drawBitmap(tmp, 0.0f, 0.0f, &paint); } diff --git a/libs/hwui/hwui/ImageDecoder.h b/libs/hwui/hwui/ImageDecoder.h index a08e92478fb0a..498593240425e 100644 --- a/libs/hwui/hwui/ImageDecoder.h +++ b/libs/hwui/hwui/ImageDecoder.h @@ -15,6 +15,7 @@ */ #pragma once +#include #include #include #include @@ -24,17 +25,18 @@ #include -class SkAndroidCodec; - namespace android { -class ANDROID_API ImageDecoder { +class Bitmap; + +class ANDROID_API ImageDecoder final { public: std::unique_ptr mCodec; sk_sp mPeeker; ImageDecoder(std::unique_ptr codec, sk_sp peeker = nullptr); + ~ImageDecoder(); bool setTargetSize(int width, int height); bool setCropRect(const SkIRect*); @@ -46,25 +48,63 @@ public: sk_sp getDefaultColorSpace() const; void setOutColorSpace(sk_sp cs); - // The size is the final size after scaling and cropping. + // The size is the final size after scaling, adjusting for the origin, and + // cropping. SkImageInfo getOutputInfo() const; int width() const; int height() const; + // True if the current frame is opaque. bool opaque() const; bool gray() const; SkCodec::Result decode(void* pixels, size_t rowBytes); + // Return true if the decoder has advanced beyond all frames. + bool finished() const; + + bool advanceFrame(); + bool rewind(); + + bool isAnimated(); + int currentFrame() const; + private: + // State machine for keeping track of how to handle RestorePrevious (RP) + // frames in decode(). + enum class RestoreState { + // Neither this frame nor the prior is RP, so there is no need to cache + // or restore. + kDoNothing, + + // This is the first in a sequence of one or more RP frames. decode() + // needs to cache the provided pixels. + kFirstRPFrame, + + // This is the second (or later) in a sequence of multiple RP frames. + // decode() needs to restore the cached frame that preceded the first RP + // frame in the sequence. + kRPFrame, + + // This is the first non-RP frame after a sequence of one or more RP + // frames. decode() still needs to restore the cached frame. Separate + // from kRPFrame because if the following frame is RP the state will + // change to kFirstRPFrame. + kNeedsRestore, + }; + SkISize mTargetSize; SkISize mDecodeSize; SkColorType mOutColorType; bool mUnpremultipliedRequired; sk_sp mOutColorSpace; - int mSampleSize; + SkAndroidCodec::AndroidOptions mOptions; + bool mCurrentFrameIsIndependent; + bool mCurrentFrameIsOpaque; + RestoreState mRestoreState; + sk_sp mRestoreFrame; std::optional mCropRect; ImageDecoder(const ImageDecoder&) = delete; diff --git a/native/graphics/jni/imagedecoder.cpp b/native/graphics/jni/imagedecoder.cpp index 4aeebe47f1aef..971ba374d6418 100644 --- a/native/graphics/jni/imagedecoder.cpp +++ b/native/graphics/jni/imagedecoder.cpp @@ -173,7 +173,13 @@ int AImageDecoder_setAndroidBitmapFormat(AImageDecoder* decoder, int32_t format) || format > ANDROID_BITMAP_FORMAT_RGBA_F16) { return ANDROID_IMAGE_DECODER_BAD_PARAMETER; } - return toDecoder(decoder)->setOutColorType(getColorType((AndroidBitmapFormat) format)) + + auto* imageDecoder = toDecoder(decoder); + if (imageDecoder->currentFrame() != 0) { + return ANDROID_IMAGE_DECODER_INVALID_STATE; + } + + return imageDecoder->setOutColorType(getColorType((AndroidBitmapFormat) format)) ? ANDROID_IMAGE_DECODER_SUCCESS : ANDROID_IMAGE_DECODER_INVALID_CONVERSION; } @@ -185,6 +191,10 @@ int AImageDecoder_setDataSpace(AImageDecoder* decoder, int32_t dataspace) { } ImageDecoder* imageDecoder = toDecoder(decoder); + if (imageDecoder->currentFrame() != 0) { + return ANDROID_IMAGE_DECODER_INVALID_STATE; + } + imageDecoder->setOutColorSpace(std::move(cs)); return ANDROID_IMAGE_DECODER_SUCCESS; } @@ -279,7 +289,12 @@ int AImageDecoder_setUnpremultipliedRequired(AImageDecoder* decoder, bool requir return ANDROID_IMAGE_DECODER_BAD_PARAMETER; } - return toDecoder(decoder)->setUnpremultipliedRequired(required) + auto* imageDecoder = toDecoder(decoder); + if (imageDecoder->currentFrame() != 0) { + return ANDROID_IMAGE_DECODER_INVALID_STATE; + } + + return imageDecoder->setUnpremultipliedRequired(required) ? ANDROID_IMAGE_DECODER_SUCCESS : ANDROID_IMAGE_DECODER_INVALID_CONVERSION; } @@ -288,7 +303,12 @@ int AImageDecoder_setTargetSize(AImageDecoder* decoder, int32_t width, int32_t h return ANDROID_IMAGE_DECODER_BAD_PARAMETER; } - return toDecoder(decoder)->setTargetSize(width, height) + auto* imageDecoder = toDecoder(decoder); + if (imageDecoder->currentFrame() != 0) { + return ANDROID_IMAGE_DECODER_INVALID_STATE; + } + + return imageDecoder->setTargetSize(width, height) ? ANDROID_IMAGE_DECODER_SUCCESS : ANDROID_IMAGE_DECODER_INVALID_SCALE; } @@ -309,10 +329,15 @@ int AImageDecoder_setCrop(AImageDecoder* decoder, ARect crop) { return ANDROID_IMAGE_DECODER_BAD_PARAMETER; } + auto* imageDecoder = toDecoder(decoder); + if (imageDecoder->currentFrame() != 0) { + return ANDROID_IMAGE_DECODER_INVALID_STATE; + } + SkIRect cropIRect; cropIRect.setLTRB(crop.left, crop.top, crop.right, crop.bottom); SkIRect* cropPtr = cropIRect == SkIRect::MakeEmpty() ? nullptr : &cropIRect; - return toDecoder(decoder)->setCropRect(cropPtr) + return imageDecoder->setCropRect(cropPtr) ? ANDROID_IMAGE_DECODER_SUCCESS : ANDROID_IMAGE_DECODER_BAD_PARAMETER; } @@ -341,6 +366,10 @@ int AImageDecoder_decodeImage(AImageDecoder* decoder, return ANDROID_IMAGE_DECODER_BAD_PARAMETER; } + if (imageDecoder->finished()) { + return ANDROID_IMAGE_DECODER_FINISHED; + } + return ResultToErrorCode(imageDecoder->decode(pixels, stride)); } @@ -352,7 +381,7 @@ bool AImageDecoder_isAnimated(AImageDecoder* decoder) { if (!decoder) return false; ImageDecoder* imageDecoder = toDecoder(decoder); - return imageDecoder->mCodec->codec()->getFrameCount() > 1; + return imageDecoder->isAnimated(); } int32_t AImageDecoder_getRepeatCount(AImageDecoder* decoder) { @@ -369,3 +398,34 @@ int32_t AImageDecoder_getRepeatCount(AImageDecoder* decoder) { } return count; } + +int AImageDecoder_advanceFrame(AImageDecoder* decoder) { + if (!decoder) return ANDROID_IMAGE_DECODER_BAD_PARAMETER; + + ImageDecoder* imageDecoder = toDecoder(decoder); + if (!imageDecoder->isAnimated()) { + return ANDROID_IMAGE_DECODER_BAD_PARAMETER; + } + + if (imageDecoder->advanceFrame()) { + return ANDROID_IMAGE_DECODER_SUCCESS; + } + + if (imageDecoder->finished()) { + return ANDROID_IMAGE_DECODER_FINISHED; + } + + return ANDROID_IMAGE_DECODER_INCOMPLETE; +} + +int AImageDecoder_rewind(AImageDecoder* decoder) { + if (!decoder) return ANDROID_IMAGE_DECODER_BAD_PARAMETER; + + ImageDecoder* imageDecoder = toDecoder(decoder); + if (!imageDecoder->isAnimated()) { + return ANDROID_IMAGE_DECODER_BAD_PARAMETER; + } + + return imageDecoder->rewind() ? ANDROID_IMAGE_DECODER_SUCCESS + : ANDROID_IMAGE_DECODER_SEEK_ERROR; +} diff --git a/native/graphics/jni/libjnigraphics.map.txt b/native/graphics/jni/libjnigraphics.map.txt index a184ab9d42e95..bd659e0e9247e 100644 --- a/native/graphics/jni/libjnigraphics.map.txt +++ b/native/graphics/jni/libjnigraphics.map.txt @@ -15,6 +15,8 @@ LIBJNIGRAPHICS { AImageDecoder_setCrop; # introduced=30 AImageDecoder_isAnimated; # introduced=31 AImageDecoder_getRepeatCount; # introduced=31 + AImageDecoder_advanceFrame; # introduced=31 + AImageDecoder_rewind; # introduced=31 AImageDecoderHeaderInfo_getWidth; # introduced=30 AImageDecoderHeaderInfo_getHeight; # introduced=30 AImageDecoderHeaderInfo_getMimeType; # introduced=30