Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement AImageDecoder _advanceFrame and _rewind

Browse Source 
Bug: 160984428
Test: Iae7d274b69999c471fd5610c6ef4d148cca81bec

Disallow AImageDecoder_set* methods after the first frame, since
changing the settings would interfere with blending and caching for
kRestorePrevious frames.

Add a cache (and a state machine) for handling kRestorePrevious frames.

Follow-on to Ib93b0ced09fa3cca4a6681745406355c48158fae - support using
a matrix for unpremul + orientation (the orientation was previously
handled by a matrix internally in SkAndroidCodec).

Change-Id: I7c32ede013fa83f1fe95c35778c33278ca6fe6a3
This commit is contained in:
Leon Scroggins III
2020-12-21 14:55:22 -05:00
parent 139145be80
commit b26aebc325
4 changed files with 316 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

242
libs/hwui/hwui/ImageDecoder.cpp
View File

@@ -17,11 +17,19 @@
#include "ImageDecoder.h"
#include <hwui/Bitmap.h>
#include <log/log.h>
#include <SkAndroidCodec.h>
#include <SkBitmap.h>
#include <SkBlendMode.h>
#include <SkCanvas.h>
#include <SkEncodedOrigin.h>
#include <SkFilterQuality.h>
#include <SkPaint.h>
#undef LOG_TAG
#define LOG_TAG "ImageDecoder"
using namespace android;
sk_sp<SkColorSpace> ImageDecoder::getDefaultColorSpace() const {
@@ -44,17 +52,29 @@ ImageDecoder::ImageDecoder(std::unique_ptr<SkAndroidCodec> codec, sk_sp<SkPngChu
, mOutColorType(mCodec->computeOutputColorType(kN32_SkColorType))
, mUnpremultipliedRequired(false)
, mOutColorSpace(getDefaultColorSpace())
, mSampleSize(1)
{
mTargetSize = swapWidthHeight() ? SkISize { mDecodeSize.height(), mDecodeSize.width() }
: mDecodeSize;
this->rewind();
}
ImageDecoder::~ImageDecoder() = default;
SkAlphaType ImageDecoder::getOutAlphaType() const {
return opaque() ? kOpaque_SkAlphaType
: mUnpremultipliedRequired ? kUnpremul_SkAlphaType : kPremul_SkAlphaType;
}
static SkISize swapped(const SkISize& size) {
return SkISize { size.height(), size.width() };
}
static bool requires_matrix_scaling(bool swapWidthHeight, const SkISize& decodeSize,
const SkISize& targetSize) {
return (swapWidthHeight && decodeSize != swapped(targetSize))
|| (!swapWidthHeight && decodeSize != targetSize);
}
bool ImageDecoder::setTargetSize(int width, int height) {
if (width <= 0 || height <= 0) {
return false;
@@ -78,17 +98,21 @@ bool ImageDecoder::setTargetSize(int width, int height) {
}
}
SkISize targetSize = { width, height };
SkISize decodeSize = swapWidthHeight() ? SkISize { height, width } : targetSize;
const bool swap = swapWidthHeight();
const SkISize targetSize = { width, height };
SkISize decodeSize = swap ? SkISize { height, width } : targetSize;
int sampleSize = mCodec->computeSampleSize(&decodeSize);
if (decodeSize != targetSize && mUnpremultipliedRequired && !opaque()) {
return false;
if (mUnpremultipliedRequired && !opaque()) {
// Allow using a matrix to handle orientation, but not scaling.
if (requires_matrix_scaling(swap, decodeSize, targetSize)) {
return false;
}
}
mTargetSize = targetSize;
mDecodeSize = decodeSize;
mSampleSize = sampleSize;
mOptions.fSampleSize = sampleSize;
return true;
}
@@ -137,8 +161,10 @@ bool ImageDecoder::setOutColorType(SkColorType colorType) {
}
bool ImageDecoder::setUnpremultipliedRequired(bool required) {
if (required && !opaque() && mDecodeSize != mTargetSize) {
return false;
if (required && !opaque()) {
if (requires_matrix_scaling(swapWidthHeight(), mDecodeSize, mTargetSize)) {
return false;
}
}
mUnpremultipliedRequired = required;
return true;
@@ -176,51 +202,150 @@ int ImageDecoder::height() const {
}
bool ImageDecoder::opaque() const {
return mCodec->getInfo().alphaType() == kOpaque_SkAlphaType;
return mCurrentFrameIsOpaque;
}
bool ImageDecoder::gray() const {
return mCodec->getInfo().colorType() == kGray_8_SkColorType;
}
bool ImageDecoder::isAnimated() {
return mCodec->codec()->getFrameCount() > 1;
}
int ImageDecoder::currentFrame() const {
return mOptions.fFrameIndex;
}
bool ImageDecoder::rewind() {
mOptions.fFrameIndex = 0;
mOptions.fPriorFrame = SkCodec::kNoFrame;
mCurrentFrameIsIndependent = true;
mCurrentFrameIsOpaque = mCodec->getInfo().isOpaque();
mRestoreState = RestoreState::kDoNothing;
mRestoreFrame = nullptr;
// TODO: Rewind the input now instead of in the next call to decode, and
// plumb through whether rewind succeeded.
return true;
}
bool ImageDecoder::advanceFrame() {
const int frameIndex = ++mOptions.fFrameIndex;
const int frameCount = mCodec->codec()->getFrameCount();
if (frameIndex >= frameCount) {
// Prevent overflow from repeated calls to advanceFrame.
mOptions.fFrameIndex = frameCount;
return false;
}
SkCodec::FrameInfo frameInfo;
if (!mCodec->codec()->getFrameInfo(frameIndex, &frameInfo)
|| !frameInfo.fFullyReceived) {
// Mark the decoder as finished, requiring a rewind.
mOptions.fFrameIndex = frameCount;
return false;
}
mCurrentFrameIsIndependent = frameInfo.fRequiredFrame == SkCodec::kNoFrame;
mCurrentFrameIsOpaque = frameInfo.fAlphaType == kOpaque_SkAlphaType;
if (frameInfo.fDisposalMethod == SkCodecAnimation::DisposalMethod::kRestorePrevious) {
switch (mRestoreState) {
case RestoreState::kDoNothing:
case RestoreState::kNeedsRestore:
mRestoreState = RestoreState::kFirstRPFrame;
break;
case RestoreState::kFirstRPFrame:
mRestoreState = RestoreState::kRPFrame;
break;
case RestoreState::kRPFrame:
// Unchanged.
break;
}
} else { // New frame is not restore previous
switch (mRestoreState) {
case RestoreState::kFirstRPFrame:
case RestoreState::kRPFrame:
mRestoreState = RestoreState::kNeedsRestore;
break;
case RestoreState::kNeedsRestore:
mRestoreState = RestoreState::kDoNothing;
mRestoreFrame = nullptr;
[[fallthrough]];
case RestoreState::kDoNothing:
mOptions.fPriorFrame = frameIndex - 1;
break;
}
}
return true;
}
bool ImageDecoder::finished() const {
return mOptions.fFrameIndex >= mCodec->codec()->getFrameCount();
}
SkCodec::Result ImageDecoder::decode(void* pixels, size_t rowBytes) {
// This was checked inside setTargetSize, but it's possible the first frame
// was opaque, so that method succeeded, but after calling advanceFrame, the
// current frame is not opaque.
if (mUnpremultipliedRequired && !opaque()) {
// Allow using a matrix to handle orientation, but not scaling.
if (requires_matrix_scaling(swapWidthHeight(), mDecodeSize, mTargetSize)) {
return SkCodec::kInvalidScale;
}
}
void* decodePixels = pixels;
size_t decodeRowBytes = rowBytes;
auto decodeInfo = SkImageInfo::Make(mDecodeSize, mOutColorType, getOutAlphaType(),
getOutputColorSpace());
const auto decodeInfo = SkImageInfo::Make(mDecodeSize, mOutColorType, getOutAlphaType(),
getOutputColorSpace());
const auto outputInfo = getOutputInfo();
switch (mRestoreState) {
case RestoreState::kFirstRPFrame:{
// This frame is marked kRestorePrevious. The prior frame should be in
// |pixels|, and it is what we'll restore after each consecutive
// kRestorePrevious frame. Cache it now.
if (!(mRestoreFrame = Bitmap::allocateHeapBitmap(outputInfo))) {
return SkCodec::kInternalError;
}
const uint8_t* srcRow = static_cast<uint8_t*>(pixels);
uint8_t* dstRow = static_cast<uint8_t*>(mRestoreFrame->pixels());
for (int y = 0; y < outputInfo.height(); y++) {
memcpy(dstRow, srcRow, outputInfo.minRowBytes());
srcRow += rowBytes;
dstRow += mRestoreFrame->rowBytes();
}
break;
}
case RestoreState::kRPFrame:
case RestoreState::kNeedsRestore:
// Restore the cached frame. It's possible that the client skipped decoding a frame, so
// we never cached it.
if (mRestoreFrame) {
const uint8_t* srcRow = static_cast<uint8_t*>(mRestoreFrame->pixels());
uint8_t* dstRow = static_cast<uint8_t*>(pixels);
for (int y = 0; y < outputInfo.height(); y++) {
memcpy(dstRow, srcRow, outputInfo.minRowBytes());
srcRow += mRestoreFrame->rowBytes();
dstRow += rowBytes;
}
}
break;
case RestoreState::kDoNothing:
break;
}
// Used if we need a temporary before scaling or subsetting.
// FIXME: Use scanline decoding on only a couple lines to save memory. b/70709380.
SkBitmap tmp;
const bool scale = mDecodeSize != mTargetSize;
const auto origin = mCodec->codec()->getOrigin();
const bool handleOrigin = origin != kDefault_SkEncodedOrigin;
SkMatrix outputMatrix;
if (scale || handleOrigin || mCropRect) {
if (!tmp.setInfo(decodeInfo)) {
return SkCodec::kInternalError;
}
if (!Bitmap::allocateHeapBitmap(&tmp)) {
return SkCodec::kInternalError;
}
decodePixels = tmp.getPixels();
decodeRowBytes = tmp.rowBytes();
}
SkAndroidCodec::AndroidOptions options;
options.fSampleSize = mSampleSize;
auto result = mCodec->getAndroidPixels(decodeInfo, decodePixels, decodeRowBytes, &options);
if (scale || handleOrigin || mCropRect) {
SkBitmap scaledBm;
if (!scaledBm.installPixels(getOutputInfo(), pixels, rowBytes)) {
return SkCodec::kInternalError;
}
SkPaint paint;
paint.setBlendMode(SkBlendMode::kSrc);
paint.setFilterQuality(kLow_SkFilterQuality); // bilinear filtering
SkCanvas canvas(scaledBm, SkCanvas::ColorBehavior::kLegacy);
SkMatrix outputMatrix;
if (mCropRect) {
outputMatrix.setTranslate(-mCropRect->fLeft, -mCropRect->fTop);
}
@@ -233,13 +358,54 @@ SkCodec::Result ImageDecoder::decode(void* pixels, size_t rowBytes) {
std::swap(targetWidth, targetHeight);
}
}
if (scale) {
float scaleX = (float) targetWidth / mDecodeSize.width();
float scaleY = (float) targetHeight / mDecodeSize.height();
outputMatrix.preScale(scaleX, scaleY);
}
// It's possible that this portion *does* have alpha, even if the
// composed frame does not. In that case, the SkBitmap needs to have
// alpha so it blends properly.
if (!tmp.setInfo(decodeInfo.makeAlphaType(mUnpremultipliedRequired ? kUnpremul_SkAlphaType
: kPremul_SkAlphaType)))
{
return SkCodec::kInternalError;
}
if (!Bitmap::allocateHeapBitmap(&tmp)) {
return SkCodec::kInternalError;
}
decodePixels = tmp.getPixels();
decodeRowBytes = tmp.rowBytes();
if (!mCurrentFrameIsIndependent) {
SkMatrix inverse;
if (outputMatrix.invert(&inverse)) {
SkCanvas canvas(tmp, SkCanvas::ColorBehavior::kLegacy);
canvas.setMatrix(inverse);
SkPaint paint;
paint.setFilterQuality(kLow_SkFilterQuality); // bilinear
SkBitmap priorFrame;
priorFrame.installPixels(outputInfo, pixels, rowBytes);
canvas.drawBitmap(priorFrame, 0, 0, &paint);
} else {
ALOGE("Failed to invert matrix!");
}
}
}
auto result = mCodec->getAndroidPixels(decodeInfo, decodePixels, decodeRowBytes, &mOptions);
if (scale || handleOrigin || mCropRect) {
SkBitmap scaledBm;
if (!scaledBm.installPixels(outputInfo, pixels, rowBytes)) {
return SkCodec::kInternalError;
}
SkPaint paint;
paint.setBlendMode(SkBlendMode::kSrc);
paint.setFilterQuality(kLow_SkFilterQuality); // bilinear filtering
SkCanvas canvas(scaledBm, SkCanvas::ColorBehavior::kLegacy);
canvas.setMatrix(outputMatrix);
canvas.drawBitmap(tmp, 0.0f, 0.0f, &paint);
}

50
libs/hwui/hwui/ImageDecoder.h
View File

@@ -15,6 +15,7 @@
*/
#pragma once
#include <SkAndroidCodec.h>
#include <SkCodec.h>
#include <SkImageInfo.h>
#include <SkPngChunkReader.h>
@@ -24,17 +25,18 @@
#include <optional>
class SkAndroidCodec;
namespace android {
class ANDROID_API ImageDecoder {
class Bitmap;
class ANDROID_API ImageDecoder final {
public:
std::unique_ptr<SkAndroidCodec> mCodec;
sk_sp<SkPngChunkReader> mPeeker;
ImageDecoder(std::unique_ptr<SkAndroidCodec> codec,
sk_sp<SkPngChunkReader> peeker = nullptr);
~ImageDecoder();
bool setTargetSize(int width, int height);
bool setCropRect(const SkIRect*);
@@ -46,25 +48,63 @@ public:
sk_sp<SkColorSpace> getDefaultColorSpace() const;
void setOutColorSpace(sk_sp<SkColorSpace> cs);
// The size is the final size after scaling and cropping.
// The size is the final size after scaling, adjusting for the origin, and
// cropping.
SkImageInfo getOutputInfo() const;
int width() const;
int height() const;
// True if the current frame is opaque.
bool opaque() const;
bool gray() const;
SkCodec::Result decode(void* pixels, size_t rowBytes);
// Return true if the decoder has advanced beyond all frames.
bool finished() const;
bool advanceFrame();
bool rewind();
bool isAnimated();
int currentFrame() const;
private:
// State machine for keeping track of how to handle RestorePrevious (RP)
// frames in decode().
enum class RestoreState {
// Neither this frame nor the prior is RP, so there is no need to cache
// or restore.
kDoNothing,
// This is the first in a sequence of one or more RP frames. decode()
// needs to cache the provided pixels.
kFirstRPFrame,
// This is the second (or later) in a sequence of multiple RP frames.
// decode() needs to restore the cached frame that preceded the first RP
// frame in the sequence.
kRPFrame,
// This is the first non-RP frame after a sequence of one or more RP
// frames. decode() still needs to restore the cached frame. Separate
// from kRPFrame because if the following frame is RP the state will
// change to kFirstRPFrame.
kNeedsRestore,
};
SkISize mTargetSize;
SkISize mDecodeSize;
SkColorType mOutColorType;
bool mUnpremultipliedRequired;
sk_sp<SkColorSpace> mOutColorSpace;
int mSampleSize;
SkAndroidCodec::AndroidOptions mOptions;
bool mCurrentFrameIsIndependent;
bool mCurrentFrameIsOpaque;
RestoreState mRestoreState;
sk_sp<Bitmap> mRestoreFrame;
std::optional<SkIRect> mCropRect;
ImageDecoder(const ImageDecoder&) = delete;

70
native/graphics/jni/imagedecoder.cpp
View File

@@ -173,7 +173,13 @@ int AImageDecoder_setAndroidBitmapFormat(AImageDecoder* decoder, int32_t format)
|| format > ANDROID_BITMAP_FORMAT_RGBA_F16) {
return ANDROID_IMAGE_DECODER_BAD_PARAMETER;
}
return toDecoder(decoder)->setOutColorType(getColorType((AndroidBitmapFormat) format))
auto* imageDecoder = toDecoder(decoder);
if (imageDecoder->currentFrame() != 0) {
return ANDROID_IMAGE_DECODER_INVALID_STATE;
}
return imageDecoder->setOutColorType(getColorType((AndroidBitmapFormat) format))
? ANDROID_IMAGE_DECODER_SUCCESS : ANDROID_IMAGE_DECODER_INVALID_CONVERSION;
}
@@ -185,6 +191,10 @@ int AImageDecoder_setDataSpace(AImageDecoder* decoder, int32_t dataspace) {
}
ImageDecoder* imageDecoder = toDecoder(decoder);
if (imageDecoder->currentFrame() != 0) {
return ANDROID_IMAGE_DECODER_INVALID_STATE;
}
imageDecoder->setOutColorSpace(std::move(cs));
return ANDROID_IMAGE_DECODER_SUCCESS;
}
@@ -279,7 +289,12 @@ int AImageDecoder_setUnpremultipliedRequired(AImageDecoder* decoder, bool requir
return ANDROID_IMAGE_DECODER_BAD_PARAMETER;
}
return toDecoder(decoder)->setUnpremultipliedRequired(required)
auto* imageDecoder = toDecoder(decoder);
if (imageDecoder->currentFrame() != 0) {
return ANDROID_IMAGE_DECODER_INVALID_STATE;
}
return imageDecoder->setUnpremultipliedRequired(required)
? ANDROID_IMAGE_DECODER_SUCCESS : ANDROID_IMAGE_DECODER_INVALID_CONVERSION;
}
@@ -288,7 +303,12 @@ int AImageDecoder_setTargetSize(AImageDecoder* decoder, int32_t width, int32_t h
return ANDROID_IMAGE_DECODER_BAD_PARAMETER;
}
return toDecoder(decoder)->setTargetSize(width, height)
auto* imageDecoder = toDecoder(decoder);
if (imageDecoder->currentFrame() != 0) {
return ANDROID_IMAGE_DECODER_INVALID_STATE;
}
return imageDecoder->setTargetSize(width, height)
? ANDROID_IMAGE_DECODER_SUCCESS : ANDROID_IMAGE_DECODER_INVALID_SCALE;
}
@@ -309,10 +329,15 @@ int AImageDecoder_setCrop(AImageDecoder* decoder, ARect crop) {
return ANDROID_IMAGE_DECODER_BAD_PARAMETER;
}
auto* imageDecoder = toDecoder(decoder);
if (imageDecoder->currentFrame() != 0) {
return ANDROID_IMAGE_DECODER_INVALID_STATE;
}
SkIRect cropIRect;
cropIRect.setLTRB(crop.left, crop.top, crop.right, crop.bottom);
SkIRect* cropPtr = cropIRect == SkIRect::MakeEmpty() ? nullptr : &cropIRect;
return toDecoder(decoder)->setCropRect(cropPtr)
return imageDecoder->setCropRect(cropPtr)
? ANDROID_IMAGE_DECODER_SUCCESS : ANDROID_IMAGE_DECODER_BAD_PARAMETER;
}
@@ -341,6 +366,10 @@ int AImageDecoder_decodeImage(AImageDecoder* decoder,
return ANDROID_IMAGE_DECODER_BAD_PARAMETER;
}
if (imageDecoder->finished()) {
return ANDROID_IMAGE_DECODER_FINISHED;
}
return ResultToErrorCode(imageDecoder->decode(pixels, stride));
}
@@ -352,7 +381,7 @@ bool AImageDecoder_isAnimated(AImageDecoder* decoder) {
if (!decoder) return false;
ImageDecoder* imageDecoder = toDecoder(decoder);
return imageDecoder->mCodec->codec()->getFrameCount() > 1;
return imageDecoder->isAnimated();
}
int32_t AImageDecoder_getRepeatCount(AImageDecoder* decoder) {
@@ -369,3 +398,34 @@ int32_t AImageDecoder_getRepeatCount(AImageDecoder* decoder) {
}
return count;
}
int AImageDecoder_advanceFrame(AImageDecoder* decoder) {
if (!decoder) return ANDROID_IMAGE_DECODER_BAD_PARAMETER;
ImageDecoder* imageDecoder = toDecoder(decoder);
if (!imageDecoder->isAnimated()) {
return ANDROID_IMAGE_DECODER_BAD_PARAMETER;
}
if (imageDecoder->advanceFrame()) {
return ANDROID_IMAGE_DECODER_SUCCESS;
}
if (imageDecoder->finished()) {
return ANDROID_IMAGE_DECODER_FINISHED;
}
return ANDROID_IMAGE_DECODER_INCOMPLETE;
}
int AImageDecoder_rewind(AImageDecoder* decoder) {
if (!decoder) return ANDROID_IMAGE_DECODER_BAD_PARAMETER;
ImageDecoder* imageDecoder = toDecoder(decoder);
if (!imageDecoder->isAnimated()) {
return ANDROID_IMAGE_DECODER_BAD_PARAMETER;
}
return imageDecoder->rewind() ? ANDROID_IMAGE_DECODER_SUCCESS
: ANDROID_IMAGE_DECODER_SEEK_ERROR;
}

2
native/graphics/jni/libjnigraphics.map.txt
View File

@@ -15,6 +15,8 @@ LIBJNIGRAPHICS {
AImageDecoder_setCrop; # introduced=30
AImageDecoder_isAnimated; # introduced=31
AImageDecoder_getRepeatCount; # introduced=31
AImageDecoder_advanceFrame; # introduced=31
AImageDecoder_rewind; # introduced=31
AImageDecoderHeaderInfo_getWidth; # introduced=30
AImageDecoderHeaderInfo_getHeight; # introduced=30
AImageDecoderHeaderInfo_getMimeType; # introduced=30