From 53d4e0d58e2d5c18f6e026c705af833b9bdd7aba Mon Sep 17 00:00:00 2001 From: James Dong Date: Wed, 21 Jul 2010 14:51:35 -0700 Subject: [PATCH] Allows the authoring engine to skip frame. This is 1st part of the work to allow audio and video resync if we found out that audio and video are out of sync during authoring - also fixed a problem in AACEncoder::read() where the buffer acquired from the buffer group does not release when error out at reading from source. Change-Id: I8a2740097fcfdf85e6178869afeb9f3687a99118 --- include/media/stagefright/MediaSource.h | 13 +++ include/media/stagefright/OMXCodec.h | 1 + media/libstagefright/AudioSource.cpp | 103 +++++++++++------- media/libstagefright/CameraSource.cpp | 51 ++++++--- media/libstagefright/MediaSource.cpp | 16 +++ media/libstagefright/OMXCodec.cpp | 15 ++- .../codecs/aacenc/AACEncoder.cpp | 1 + 7 files changed, 142 insertions(+), 58 deletions(-) diff --git a/include/media/stagefright/MediaSource.h b/include/media/stagefright/MediaSource.h index a31395ed95090..dafc621a991a1 100644 --- a/include/media/stagefright/MediaSource.h +++ b/include/media/stagefright/MediaSource.h @@ -78,18 +78,31 @@ struct MediaSource : public RefBase { void clearSeekTo(); bool getSeekTo(int64_t *time_us, SeekMode *mode) const; + // Option allows encoder to skip some frames until the specified + // time stamp. + // To prevent from being abused, when the skipFrame timestamp is + // found to be more than 1 second later than the current timestamp, + // an error will be returned from read(). + void clearSkipFrame(); + bool getSkipFrame(int64_t *timeUs) const; + void setSkipFrame(int64_t timeUs); + void setLateBy(int64_t lateness_us); int64_t getLateBy() const; private: enum Options { + // Bit map kSeekTo_Option = 1, + kSkipFrame_Option = 2, }; uint32_t mOptions; int64_t mSeekTimeUs; SeekMode mSeekMode; int64_t mLatenessUs; + + int64_t mSkipFrameUntilTimeUs; }; // Causes this source to suspend pulling data from its upstream source diff --git a/include/media/stagefright/OMXCodec.h b/include/media/stagefright/OMXCodec.h index 79e7a2f4fb0b4..6c6949b1d5bd5 100644 --- a/include/media/stagefright/OMXCodec.h +++ b/include/media/stagefright/OMXCodec.h @@ -143,6 +143,7 @@ private: int64_t mSeekTimeUs; ReadOptions::SeekMode mSeekMode; int64_t mTargetTimeUs; + int64_t mSkipTimeUs; MediaBuffer *mLeftOverBuffer; diff --git a/media/libstagefright/AudioSource.cpp b/media/libstagefright/AudioSource.cpp index 6031797d7418f..50c0edc84a7c3 100644 --- a/media/libstagefright/AudioSource.cpp +++ b/media/libstagefright/AudioSource.cpp @@ -137,53 +137,76 @@ status_t AudioSource::read( MediaBuffer *buffer; CHECK_EQ(mGroup->acquire_buffer(&buffer), OK); - uint32_t numFramesRecorded; - mRecord->getPosition(&numFramesRecorded); - int64_t latency = mRecord->latency() * 1000; + while (mStarted) { + uint32_t numFramesRecorded; + mRecord->getPosition(&numFramesRecorded); + int64_t latency = mRecord->latency() * 1000; - int64_t readTime = systemTime() / 1000; - if (numFramesRecorded == 0) { - // Initial delay - if (mStartTimeUs > 0) { - mStartTimeUs = readTime - mStartTimeUs; + int64_t readTime = systemTime() / 1000; + + if (numFramesRecorded == 0) { + // Initial delay + if (mStartTimeUs > 0) { + mStartTimeUs = readTime - mStartTimeUs; + } else { + mStartTimeUs += latency; + } + } + + ssize_t n = 0; + if (mCollectStats) { + n = mRecord->read(buffer->data(), buffer->size()); + int64_t endTime = systemTime() / 1000; + mTotalReadTimeUs += (endTime - readTime); + if (n >= 0) { + mTotalReadBytes += n; + } } else { - mStartTimeUs += latency; + n = mRecord->read(buffer->data(), buffer->size()); } - } - ssize_t n = 0; - if (mCollectStats) { - n = mRecord->read(buffer->data(), buffer->size()); - int64_t endTime = systemTime() / 1000; - mTotalReadTimeUs += (endTime - readTime); - if (n >= 0) { - mTotalReadBytes += n; + if (n < 0) { + buffer->release(); + buffer = NULL; + + return (status_t)n; } - } else { - n = mRecord->read(buffer->data(), buffer->size()); + + uint32_t sampleRate = mRecord->getSampleRate(); + int64_t timestampUs = (1000000LL * numFramesRecorded) / sampleRate + + mStartTimeUs; + int64_t skipFrameUs; + if (!options || !options->getSkipFrame(&skipFrameUs)) { + skipFrameUs = timestampUs; // Don't skip frame + } + + if (skipFrameUs > timestampUs) { + // Safe guard against the abuse of the kSkipFrame_Option. + if (skipFrameUs - timestampUs >= 1E6) { + LOGE("Frame skipping requested is way too long: %lld us", + skipFrameUs - timestampUs); + buffer->release(); + return UNKNOWN_ERROR; + } + LOGV("skipFrame: %lld us > timestamp: %lld us, samples %d", + skipFrameUs, timestampUs, numFramesRecorded); + continue; + } + + if (mTrackMaxAmplitude) { + trackMaxAmplitude((int16_t *) buffer->data(), n >> 1); + } + + buffer->meta_data()->setInt64(kKeyTime, timestampUs); + LOGV("initial delay: %lld, sample rate: %d, timestamp: %lld", + mStartTimeUs, sampleRate, timestampUs); + + buffer->set_range(0, n); + + *out = buffer; + return OK; } - if (n < 0) { - buffer->release(); - buffer = NULL; - - return (status_t)n; - } - - if (mTrackMaxAmplitude) { - trackMaxAmplitude((int16_t *) buffer->data(), n >> 1); - } - - uint32_t sampleRate = mRecord->getSampleRate(); - int64_t timestampUs = (1000000LL * numFramesRecorded) / sampleRate + mStartTimeUs; - buffer->meta_data()->setInt64(kKeyTime, timestampUs); - LOGV("initial delay: %lld, sample rate: %d, timestamp: %lld", - mStartTimeUs, sampleRate, timestampUs); - - buffer->set_range(0, n); - - *out = buffer; - return OK; } diff --git a/media/libstagefright/CameraSource.cpp b/media/libstagefright/CameraSource.cpp index 4cc176a0edc25..9c48daf9348a3 100644 --- a/media/libstagefright/CameraSource.cpp +++ b/media/libstagefright/CameraSource.cpp @@ -277,23 +277,44 @@ status_t CameraSource::read( { Mutex::Autolock autoLock(mLock); - while (mStarted && mFramesReceived.empty()) { - mFrameAvailableCondition.wait(mLock); - } - if (!mStarted) { - return OK; - } - frame = *mFramesReceived.begin(); - mFramesReceived.erase(mFramesReceived.begin()); + while (mStarted) { + while(mFramesReceived.empty()) { + mFrameAvailableCondition.wait(mLock); + } - frameTime = *mFrameTimes.begin(); - mFrameTimes.erase(mFrameTimes.begin()); + if (!mStarted) { + return OK; + } - mFramesBeingEncoded.push_back(frame); - *buffer = new MediaBuffer(frame->pointer(), frame->size()); - (*buffer)->setObserver(this); - (*buffer)->add_ref(); - (*buffer)->meta_data()->setInt64(kKeyTime, frameTime); + frame = *mFramesReceived.begin(); + mFramesReceived.erase(mFramesReceived.begin()); + + frameTime = *mFrameTimes.begin(); + mFrameTimes.erase(mFrameTimes.begin()); + int64_t skipTimeUs; + if (!options || !options->getSkipFrame(&skipTimeUs)) { + skipTimeUs = frameTime; + } + if (skipTimeUs > frameTime) { + LOGV("skipTimeUs: %lld us > frameTime: %lld us", + skipTimeUs, frameTime); + releaseOneRecordingFrame(frame); + ++mNumFramesDropped; + // Safeguard against the abuse of the kSkipFrame_Option. + if (skipTimeUs - frameTime >= 1E6) { + LOGE("Frame skipping requested is way too long: %lld us", + skipTimeUs - frameTime); + return UNKNOWN_ERROR; + } + } else { + mFramesBeingEncoded.push_back(frame); + *buffer = new MediaBuffer(frame->pointer(), frame->size()); + (*buffer)->setObserver(this); + (*buffer)->add_ref(); + (*buffer)->meta_data()->setInt64(kKeyTime, frameTime); + return OK; + } + } } return OK; } diff --git a/media/libstagefright/MediaSource.cpp b/media/libstagefright/MediaSource.cpp index fd0e79c34f0ef..b4ef338fed7e8 100644 --- a/media/libstagefright/MediaSource.cpp +++ b/media/libstagefright/MediaSource.cpp @@ -32,6 +32,7 @@ void MediaSource::ReadOptions::reset() { mOptions = 0; mSeekTimeUs = 0; mLatenessUs = 0; + mSkipFrameUntilTimeUs = 0; } void MediaSource::ReadOptions::setSeekTo(int64_t time_us, SeekMode mode) { @@ -53,6 +54,21 @@ bool MediaSource::ReadOptions::getSeekTo( return (mOptions & kSeekTo_Option) != 0; } +void MediaSource::ReadOptions::clearSkipFrame() { + mOptions &= ~kSkipFrame_Option; + mSkipFrameUntilTimeUs = 0; +} + +void MediaSource::ReadOptions::setSkipFrame(int64_t timeUs) { + mOptions |= kSkipFrame_Option; + mSkipFrameUntilTimeUs = timeUs; +} + +bool MediaSource::ReadOptions::getSkipFrame(int64_t *timeUs) const { + *timeUs = mSkipFrameUntilTimeUs; + return (mOptions & kSkipFrame_Option) != 0; +} + void MediaSource::ReadOptions::setLateBy(int64_t lateness_us) { mLatenessUs = lateness_us; } diff --git a/media/libstagefright/OMXCodec.cpp b/media/libstagefright/OMXCodec.cpp index 1b6308361869c..3fba2d96ba259 100644 --- a/media/libstagefright/OMXCodec.cpp +++ b/media/libstagefright/OMXCodec.cpp @@ -1274,6 +1274,7 @@ OMXCodec::OMXCodec( mSeekTimeUs(-1), mSeekMode(ReadOptions::SEEK_CLOSEST_SYNC), mTargetTimeUs(-1), + mSkipTimeUs(-1), mLeftOverBuffer(NULL), mPaused(false) { mPortStatus[kPortIndexInput] = ENABLED; @@ -2200,13 +2201,15 @@ void OMXCodec::drainInputBuffer(BufferInfo *info) { int32_t n = 0; for (;;) { MediaBuffer *srcBuffer; + MediaSource::ReadOptions options; + if (mSkipTimeUs >= 0) { + options.setSkipFrame(mSkipTimeUs); + } if (mSeekTimeUs >= 0) { if (mLeftOverBuffer) { mLeftOverBuffer->release(); mLeftOverBuffer = NULL; } - - MediaSource::ReadOptions options; options.setSeekTo(mSeekTimeUs, mSeekMode); mSeekTimeUs = -1; @@ -2231,7 +2234,7 @@ void OMXCodec::drainInputBuffer(BufferInfo *info) { err = OK; } else { - err = mSource->read(&srcBuffer); + err = mSource->read(&srcBuffer, &options); } if (err != OK) { @@ -2830,6 +2833,12 @@ status_t OMXCodec::read( if (options && options->getSeekTo(&seekTimeUs, &seekMode)) { seeking = true; } + int64_t skipTimeUs; + if (options && options->getSkipFrame(&skipTimeUs)) { + mSkipTimeUs = skipTimeUs; + } else { + mSkipTimeUs = -1; + } if (mInitialBufferSubmit) { mInitialBufferSubmit = false; diff --git a/media/libstagefright/codecs/aacenc/AACEncoder.cpp b/media/libstagefright/codecs/aacenc/AACEncoder.cpp index e8235c26229c0..052c3542ddd71 100644 --- a/media/libstagefright/codecs/aacenc/AACEncoder.cpp +++ b/media/libstagefright/codecs/aacenc/AACEncoder.cpp @@ -224,6 +224,7 @@ status_t AACEncoder::read( if (mInputBuffer == NULL) { if (mSource->read(&mInputBuffer, options) != OK) { if (mNumInputSamples == 0) { + buffer->release(); return ERROR_END_OF_STREAM; } memset(&mInputFrame[mNumInputSamples],