Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge "In order to recover from video lagging behind audio, drop avc frames"

Browse Source
This commit is contained in:
Andreas Huber
2011-09-19 08:54:59 -07:00
committed by Android (Google) Code Review
parent 5c1c26a837 950fc9d115
commit c7342fbf99
8 changed files with 155 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

117
media/libmediaplayerservice/nuplayer/NuPlayer.cpp
View File

@@ -34,17 +34,21 @@
#include <media/stagefright/foundation/ADebug.h>
#include <media/stagefright/foundation/AMessage.h>
#include <media/stagefright/ACodec.h>
#include <media/stagefright/MediaDefs.h>
#include <media/stagefright/MediaErrors.h>
#include <media/stagefright/MetaData.h>
#include <surfaceflinger/Surface.h>
#include <gui/ISurfaceTexture.h>
#include "avc_utils.h"
namespace android {
////////////////////////////////////////////////////////////////////////////////
NuPlayer::NuPlayer()
: mUIDValid(false),
mVideoIsAVC(false),
mAudioEOS(false),
mVideoEOS(false),
mScanSourcesPending(false),
@@ -52,7 +56,12 @@ NuPlayer::NuPlayer()
mFlushingAudio(NONE),
mFlushingVideo(NONE),
mResetInProgress(false),
mResetPostponed(false) {
mResetPostponed(false),
mSkipRenderingAudioUntilMediaTimeUs(-1ll),
mSkipRenderingVideoUntilMediaTimeUs(-1ll),
mVideoLateByUs(0ll),
mNumFramesTotal(0ll),
mNumFramesDropped(0ll) {
}
NuPlayer::~NuPlayer() {
@@ -185,10 +194,14 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) {
{
LOGV("kWhatStart");
mVideoIsAVC = false;
mAudioEOS = false;
mVideoEOS = false;
mSkipRenderingAudioUntilMediaTimeUs = -1;
mSkipRenderingVideoUntilMediaTimeUs = -1;
mVideoLateByUs = 0;
mNumFramesTotal = 0;
mNumFramesDropped = 0;
mSource->start();
@@ -269,6 +282,8 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) {
} else {
CHECK(IsFlushingState(mFlushingVideo, &needShutdown));
mFlushingVideo = FLUSHED;
mVideoLateByUs = 0;
}
LOGV("decoder %s flush completed", audio ? "audio" : "video");
@@ -397,13 +412,18 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) {
int64_t positionUs;
CHECK(msg->findInt64("positionUs", &positionUs));
CHECK(msg->findInt64("videoLateByUs", &mVideoLateByUs));
if (mDriver != NULL) {
sp<NuPlayerDriver> driver = mDriver.promote();
if (driver != NULL) {
driver->notifyPosition(positionUs);
driver->notifyFrameStats(
mNumFramesTotal, mNumFramesDropped);
}
}
} else {
} else if (what == Renderer::kWhatFlushComplete) {
CHECK_EQ(what, (int32_t)Renderer::kWhatFlushComplete);
int32_t audio;
@@ -565,6 +585,12 @@ status_t NuPlayer::instantiateDecoder(bool audio, sp<Decoder> *decoder) {
return -EWOULDBLOCK;
}
if (!audio) {
const char *mime;
CHECK(meta->findCString(kKeyMIMEType, &mime));
mVideoIsAVC = !strcasecmp(MEDIA_MIMETYPE_VIDEO_AVC, mime);
}
sp<AMessage> notify =
new AMessage(audio ? kWhatAudioNotify : kWhatVideoNotify,
id());
@@ -598,53 +624,70 @@ status_t NuPlayer::feedDecoderInputData(bool audio, const sp<AMessage> &msg) {
}
sp<ABuffer> accessUnit;
status_t err = mSource->dequeueAccessUnit(audio, &accessUnit);
if (err == -EWOULDBLOCK) {
return err;
} else if (err != OK) {
if (err == INFO_DISCONTINUITY) {
int32_t type;
CHECK(accessUnit->meta()->findInt32("discontinuity", &type));
bool dropAccessUnit;
do {
status_t err = mSource->dequeueAccessUnit(audio, &accessUnit);
bool formatChange =
type == ATSParser::DISCONTINUITY_FORMATCHANGE;
if (err == -EWOULDBLOCK) {
return err;
} else if (err != OK) {
if (err == INFO_DISCONTINUITY) {
int32_t type;
CHECK(accessUnit->meta()->findInt32("discontinuity", &type));
LOGV("%s discontinuity (formatChange=%d)",
audio ? "audio" : "video", formatChange);
bool formatChange =
type == ATSParser::DISCONTINUITY_FORMATCHANGE;
if (audio) {
mSkipRenderingAudioUntilMediaTimeUs = -1;
} else {
mSkipRenderingVideoUntilMediaTimeUs = -1;
}
LOGV("%s discontinuity (formatChange=%d)",
audio ? "audio" : "video", formatChange);
sp<AMessage> extra;
if (accessUnit->meta()->findMessage("extra", &extra)
&& extra != NULL) {
int64_t resumeAtMediaTimeUs;
if (extra->findInt64(
"resume-at-mediatimeUs", &resumeAtMediaTimeUs)) {
LOGI("suppressing rendering of %s until %lld us",
audio ? "audio" : "video", resumeAtMediaTimeUs);
if (audio) {
mSkipRenderingAudioUntilMediaTimeUs = -1;
} else {
mSkipRenderingVideoUntilMediaTimeUs = -1;
}
if (audio) {
mSkipRenderingAudioUntilMediaTimeUs =
resumeAtMediaTimeUs;
} else {
mSkipRenderingVideoUntilMediaTimeUs =
resumeAtMediaTimeUs;
sp<AMessage> extra;
if (accessUnit->meta()->findMessage("extra", &extra)
&& extra != NULL) {
int64_t resumeAtMediaTimeUs;
if (extra->findInt64(
"resume-at-mediatimeUs", &resumeAtMediaTimeUs)) {
LOGI("suppressing rendering of %s until %lld us",
audio ? "audio" : "video", resumeAtMediaTimeUs);
if (audio) {
mSkipRenderingAudioUntilMediaTimeUs =
resumeAtMediaTimeUs;
} else {
mSkipRenderingVideoUntilMediaTimeUs =
resumeAtMediaTimeUs;
}
}
}
flushDecoder(audio, formatChange);
}
flushDecoder(audio, formatChange);
reply->setInt32("err", err);
reply->post();
return OK;
}
reply->setInt32("err", err);
reply->post();
return OK;
}
if (!audio) {
++mNumFramesTotal;
}
dropAccessUnit = false;
if (!audio
&& mVideoLateByUs > 100000ll
&& mVideoIsAVC
&& !IsAVCReferenceFrame(accessUnit)) {
dropAccessUnit = true;
++mNumFramesDropped;
}
} while (dropAccessUnit);
// LOGV("returned a valid buffer of %s data", audio ? "audio" : "video");

4
media/libmediaplayerservice/nuplayer/NuPlayer.h
View File

@@ -92,6 +92,7 @@ private:
sp<NativeWindowWrapper> mNativeWindow;
sp<MediaPlayerBase::AudioSink> mAudioSink;
sp<Decoder> mVideoDecoder;
bool mVideoIsAVC;
sp<Decoder> mAudioDecoder;
sp<Renderer> mRenderer;
@@ -119,6 +120,9 @@ private:
int64_t mSkipRenderingAudioUntilMediaTimeUs;
int64_t mSkipRenderingVideoUntilMediaTimeUs;
int64_t mVideoLateByUs;
int64_t mNumFramesTotal, mNumFramesDropped;
status_t instantiateDecoder(bool audio, sp<Decoder> *decoder);
status_t feedDecoderInputData(bool audio, const sp<AMessage> &msg);

28
media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp
View File

@@ -31,6 +31,8 @@ NuPlayerDriver::NuPlayerDriver()
: mResetInProgress(false),
mDurationUs(-1),
mPositionUs(-1),
mNumFramesTotal(0),
mNumFramesDropped(0),
mLooper(new ALooper),
mState(UNINITIALIZED),
mStartupSeekTimeUs(-1) {
@@ -292,4 +294,30 @@ void NuPlayerDriver::notifySeekComplete() {
sendEvent(MEDIA_SEEK_COMPLETE);
}
void NuPlayerDriver::notifyFrameStats(
int64_t numFramesTotal, int64_t numFramesDropped) {
Mutex::Autolock autoLock(mLock);
mNumFramesTotal = numFramesTotal;
mNumFramesDropped = numFramesDropped;
}
status_t NuPlayerDriver::dump(int fd, const Vector<String16> &args) const {
Mutex::Autolock autoLock(mLock);
FILE *out = fdopen(dup(fd), "w");
fprintf(out, " NuPlayer\n");
fprintf(out, " numFramesTotal(%lld), numFramesDropped(%lld), "
"percentageDropped(%.2f)\n",
mNumFramesTotal,
mNumFramesDropped,
mNumFramesTotal == 0
? 0.0 : (double)mNumFramesDropped / mNumFramesTotal);
fclose(out);
out = NULL;
return OK;
}
} // namespace android

7
media/libmediaplayerservice/nuplayer/NuPlayerDriver.h
View File

@@ -60,16 +60,19 @@ struct NuPlayerDriver : public MediaPlayerInterface {
virtual status_t getMetadata(
const media::Metadata::Filter& ids, Parcel *records);
virtual status_t dump(int fd, const Vector<String16> &args) const;
void notifyResetComplete();
void notifyDuration(int64_t durationUs);
void notifyPosition(int64_t positionUs);
void notifySeekComplete();
void notifyFrameStats(int64_t numFramesTotal, int64_t numFramesDropped);
protected:
virtual ~NuPlayerDriver();
private:
Mutex mLock;
mutable Mutex mLock;
Condition mCondition;
// The following are protected through "mLock"
@@ -77,6 +80,8 @@ private:
bool mResetInProgress;
int64_t mDurationUs;
int64_t mPositionUs;
int64_t mNumFramesTotal;
int64_t mNumFramesDropped;
// <<<
sp<ALooper> mLooper;

18
media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp
View File

@@ -47,7 +47,8 @@ NuPlayer::Renderer::Renderer(
mHasVideo(false),
mSyncQueues(false),
mPaused(false),
mLastPositionUpdateUs(-1ll) {
mLastPositionUpdateUs(-1ll),
mVideoLateByUs(0ll) {
}
NuPlayer::Renderer::~Renderer() {
@@ -357,22 +358,26 @@ void NuPlayer::Renderer::onDrainVideoQueue() {
mVideoQueue.erase(mVideoQueue.begin());
entry = NULL;
mVideoLateByUs = 0ll;
notifyPosition();
return;
}
#if 0
int64_t mediaTimeUs;
CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
int64_t realTimeUs = mediaTimeUs - mAnchorTimeMediaUs + mAnchorTimeRealUs;
int64_t lateByUs = ALooper::GetNowUs() - realTimeUs;
mVideoLateByUs = ALooper::GetNowUs() - realTimeUs;
if (lateByUs > 40000) {
LOGI("video late by %lld us (%.2f secs)", lateByUs, lateByUs / 1E6);
bool tooLate = (mVideoLateByUs > 40000);
if (tooLate) {
LOGV("video late by %lld us (%.2f secs)", lateByUs, lateByUs / 1E6);
} else {
LOGV("rendering video at media time %.2f secs", mediaTimeUs / 1E6);
}
#endif
entry->mNotifyConsumed->setInt32("render", true);
entry->mNotifyConsumed->post();
@@ -604,6 +609,7 @@ void NuPlayer::Renderer::notifyPosition() {
sp<AMessage> notify = mNotify->dup();
notify->setInt32("what", kWhatPosition);
notify->setInt64("positionUs", positionUs);
notify->setInt64("videoLateByUs", mVideoLateByUs);
notify->post();
}

2
media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h
View File

@@ -101,6 +101,7 @@ private:
bool mPaused;
int64_t mLastPositionUpdateUs;
int64_t mVideoLateByUs;
bool onDrainAudioQueue();
void postDrainAudioQueue(int64_t delayUs = 0);
@@ -118,6 +119,7 @@ private:
void notifyEOS(bool audio, status_t finalResult);
void notifyFlushComplete(bool audio);
void notifyPosition();
void notifyVideoLateBy(int64_t lateByUs);
void flushQueue(List<QueueEntry> *queue);
bool dropBufferWhileFlushing(bool audio, const sp<AMessage> &msg);

22
media/libstagefright/avc_utils.cpp
View File

@@ -329,6 +329,28 @@ bool IsIDR(const sp<ABuffer> &buffer) {
return foundIDR;
}
bool IsAVCReferenceFrame(const sp<ABuffer> &accessUnit) {
const uint8_t *data = accessUnit->data();
size_t size = accessUnit->size();
const uint8_t *nalStart;
size_t nalSize;
while (getNextNALUnit(&data, &size, &nalStart, &nalSize, true) == OK) {
CHECK_GT(nalSize, 0u);
unsigned nalType = nalStart[0] & 0x1f;
if (nalType == 5) {
return true;
} else if (nalType == 1) {
unsigned nal_ref_idc = (nalStart[0] >> 5) & 3;
return nal_ref_idc != 0;
}
}
return true;
}
sp<MetaData> MakeAACCodecSpecificData(
unsigned profile, unsigned sampling_freq_index,
unsigned channel_configuration) {

1
media/libstagefright/include/avc_utils.h
View File

@@ -50,6 +50,7 @@ struct MetaData;
sp<MetaData> MakeAVCCodecSpecificData(const sp<ABuffer> &accessUnit);
bool IsIDR(const sp<ABuffer> &accessUnit);
bool IsAVCReferenceFrame(const sp<ABuffer> &accessUnit);
const char *AVCProfileToString(uint8_t profile);