Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

RELAND MediaCodec: don't cache buffers until requested

Browse Source 
Do not cache input/output buffers until explicitly requested.

Track validity of buffers before getInput/OutputBuffers so that the
dequeued buffers are marked accessible when the array is returned
to the client.

Bug: 189889230
Bug: 236106096
Test: manual
Change-Id: I6892f02853f00c3e355c827ec943fbdf183469f1
This commit is contained in:
Wonsik Kim
2022-07-25 15:14:14 -07:00
parent 5b2d2cf6b9
commit 0e2297b795
1 changed files with 90 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

131
media/java/android/media/MediaCodec.java
View File

@@ -42,6 +42,7 @@ import java.nio.ByteOrder;
import java.nio.ReadOnlyBufferException; import java.nio.ReadOnlyBufferException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.BitSet;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
@@ -1794,7 +1795,7 @@ final public class MediaCodec {
synchronized(mBufferLock) { synchronized(mBufferLock) {
switch (mBufferMode) { switch (mBufferMode) {
case BUFFER_MODE_LEGACY: case BUFFER_MODE_LEGACY:
validateInputByteBuffer(mCachedInputBuffers, index); validateInputByteBufferLocked(mCachedInputBuffers, index);
break; break;
case BUFFER_MODE_BLOCK: case BUFFER_MODE_BLOCK:
while (mQueueRequests.size() <= index) { while (mQueueRequests.size() <= index) {
@@ -1823,7 +1824,7 @@ final public class MediaCodec {
synchronized(mBufferLock) { synchronized(mBufferLock) {
switch (mBufferMode) { switch (mBufferMode) {
case BUFFER_MODE_LEGACY: case BUFFER_MODE_LEGACY:
validateOutputByteBuffer(mCachedOutputBuffers, index, info); validateOutputByteBufferLocked(mCachedOutputBuffers, index, info);
break; break;
case BUFFER_MODE_BLOCK: case BUFFER_MODE_BLOCK:
while (mOutputFrames.size() <= index) { while (mOutputFrames.size() <= index) {
@@ -2282,10 +2283,6 @@ final public class MediaCodec {
*/ */
public final void start() { public final void start() {
native_start(); native_start();
synchronized(mBufferLock) {
cacheBuffers(true /* input */);
cacheBuffers(false /* input */);
}
} }
private native final void native_start(); private native final void native_start();
@@ -2342,8 +2339,10 @@ final public class MediaCodec {
*/ */
public final void flush() { public final void flush() {
synchronized(mBufferLock) { synchronized(mBufferLock) {
invalidateByteBuffers(mCachedInputBuffers); invalidateByteBuffersLocked(mCachedInputBuffers);
invalidateByteBuffers(mCachedOutputBuffers); invalidateByteBuffersLocked(mCachedOutputBuffers);
mValidInputIndices.clear();
mValidOutputIndices.clear();
mDequeuedInputBuffers.clear(); mDequeuedInputBuffers.clear();
mDequeuedOutputBuffers.clear(); mDequeuedOutputBuffers.clear();
} }
@@ -2627,14 +2626,14 @@ final public class MediaCodec {
+ "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. "
+ "Please use getQueueRequest() to queue buffers"); + "Please use getQueueRequest() to queue buffers");
} }
invalidateByteBuffer(mCachedInputBuffers, index); invalidateByteBufferLocked(mCachedInputBuffers, index, true /* input */);
mDequeuedInputBuffers.remove(index); mDequeuedInputBuffers.remove(index);
} }
try { try {
native_queueInputBuffer( native_queueInputBuffer(
index, offset, size, presentationTimeUs, flags); index, offset, size, presentationTimeUs, flags);
} catch (CryptoException | IllegalStateException e) { } catch (CryptoException | IllegalStateException e) {
revalidateByteBuffer(mCachedInputBuffers, index); revalidateByteBuffer(mCachedInputBuffers, index, true /* input */);
throw e; throw e;
} }
} }
@@ -2897,14 +2896,14 @@ final public class MediaCodec {
+ "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. "
+ "Please use getQueueRequest() to queue buffers"); + "Please use getQueueRequest() to queue buffers");
} }
invalidateByteBuffer(mCachedInputBuffers, index); invalidateByteBufferLocked(mCachedInputBuffers, index, true /* input */);
mDequeuedInputBuffers.remove(index); mDequeuedInputBuffers.remove(index);
} }
try { try {
native_queueSecureInputBuffer( native_queueSecureInputBuffer(
index, offset, info, presentationTimeUs, flags); index, offset, info, presentationTimeUs, flags);
} catch (CryptoException | IllegalStateException e) { } catch (CryptoException | IllegalStateException e) {
revalidateByteBuffer(mCachedInputBuffers, index); revalidateByteBuffer(mCachedInputBuffers, index, true /* input */);
throw e; throw e;
} }
} }
@@ -2938,7 +2937,7 @@ final public class MediaCodec {
int res = native_dequeueInputBuffer(timeoutUs); int res = native_dequeueInputBuffer(timeoutUs);
if (res >= 0) { if (res >= 0) {
synchronized(mBufferLock) { synchronized(mBufferLock) {
validateInputByteBuffer(mCachedInputBuffers, res); validateInputByteBufferLocked(mCachedInputBuffers, res);
} }
} }
return res; return res;
@@ -3535,10 +3534,10 @@ final public class MediaCodec {
int res = native_dequeueOutputBuffer(info, timeoutUs); int res = native_dequeueOutputBuffer(info, timeoutUs);
synchronized (mBufferLock) { synchronized (mBufferLock) {
if (res == INFO_OUTPUT_BUFFERS_CHANGED) { if (res == INFO_OUTPUT_BUFFERS_CHANGED) {
cacheBuffers(false /* input */); cacheBuffersLocked(false /* input */);
} else if (res >= 0) { } else if (res >= 0) {
validateOutputByteBuffer(mCachedOutputBuffers, res, info); validateOutputByteBufferLocked(mCachedOutputBuffers, res, info);
if (mHasSurface) { if (mHasSurface || mCachedOutputBuffers == null) {
mDequeuedOutputInfos.put(res, info.dup()); mDequeuedOutputInfos.put(res, info.dup());
} }
} }
@@ -3632,9 +3631,9 @@ final public class MediaCodec {
synchronized(mBufferLock) { synchronized(mBufferLock) {
switch (mBufferMode) { switch (mBufferMode) {
case BUFFER_MODE_LEGACY: case BUFFER_MODE_LEGACY:
invalidateByteBuffer(mCachedOutputBuffers, index); invalidateByteBufferLocked(mCachedOutputBuffers, index, false /* input */);
mDequeuedOutputBuffers.remove(index); mDequeuedOutputBuffers.remove(index);
if (mHasSurface) { if (mHasSurface || mCachedOutputBuffers == null) {
info = mDequeuedOutputInfos.remove(index); info = mDequeuedOutputInfos.remove(index);
} }
break; break;
@@ -3786,15 +3785,24 @@ final public class MediaCodec {
private ByteBuffer[] mCachedInputBuffers; private ByteBuffer[] mCachedInputBuffers;
private ByteBuffer[] mCachedOutputBuffers; private ByteBuffer[] mCachedOutputBuffers;
private BitSet mValidInputIndices = new BitSet();
private BitSet mValidOutputIndices = new BitSet();
private final BufferMap mDequeuedInputBuffers = new BufferMap(); private final BufferMap mDequeuedInputBuffers = new BufferMap();
private final BufferMap mDequeuedOutputBuffers = new BufferMap(); private final BufferMap mDequeuedOutputBuffers = new BufferMap();
private final Map<Integer, BufferInfo> mDequeuedOutputInfos = private final Map<Integer, BufferInfo> mDequeuedOutputInfos =
new HashMap<Integer, BufferInfo>(); new HashMap<Integer, BufferInfo>();
final private Object mBufferLock; final private Object mBufferLock;
private final void invalidateByteBuffer( private void invalidateByteBufferLocked(
@Nullable ByteBuffer[] buffers, int index) { @Nullable ByteBuffer[] buffers, int index, boolean input) {
if (buffers != null && index >= 0 && index < buffers.length) { if (buffers == null) {
if (index < 0) {
throw new IllegalStateException("index is negative (" + index + ")");
}
BitSet indices = input ? mValidInputIndices : mValidOutputIndices;
indices.clear(index);
} else if (index >= 0 && index < buffers.length) {
ByteBuffer buffer = buffers[index]; ByteBuffer buffer = buffers[index];
if (buffer != null) { if (buffer != null) {
buffer.setAccessible(false); buffer.setAccessible(false);
@@ -3802,9 +3810,14 @@ final public class MediaCodec {
} }
} }
private final void validateInputByteBuffer( private void validateInputByteBufferLocked(
@Nullable ByteBuffer[] buffers, int index) { @Nullable ByteBuffer[] buffers, int index) {
if (buffers != null && index >= 0 && index < buffers.length) { if (buffers == null) {
if (index < 0) {
throw new IllegalStateException("index is negative (" + index + ")");
}
mValidInputIndices.set(index);
} else if (index >= 0 && index < buffers.length) {
ByteBuffer buffer = buffers[index]; ByteBuffer buffer = buffers[index];
if (buffer != null) { if (buffer != null) {
buffer.setAccessible(true); buffer.setAccessible(true);
@@ -3813,10 +3826,16 @@ final public class MediaCodec {
} }
} }
private final void revalidateByteBuffer( private void revalidateByteBuffer(
@Nullable ByteBuffer[] buffers, int index) { @Nullable ByteBuffer[] buffers, int index, boolean input) {
synchronized(mBufferLock) { synchronized(mBufferLock) {
if (buffers != null && index >= 0 && index < buffers.length) { if (buffers == null) {
if (index < 0) {
throw new IllegalStateException("index is negative (" + index + ")");
}
BitSet indices = input ? mValidInputIndices : mValidOutputIndices;
indices.set(index);
} else if (index >= 0 && index < buffers.length) {
ByteBuffer buffer = buffers[index]; ByteBuffer buffer = buffers[index];
if (buffer != null) { if (buffer != null) {
buffer.setAccessible(true); buffer.setAccessible(true);
@@ -3825,9 +3844,14 @@ final public class MediaCodec {
} }
} }
private final void validateOutputByteBuffer( private void validateOutputByteBufferLocked(
@Nullable ByteBuffer[] buffers, int index, @NonNull BufferInfo info) { @Nullable ByteBuffer[] buffers, int index, @NonNull BufferInfo info) {
if (buffers != null && index >= 0 && index < buffers.length) { if (buffers == null) {
if (index < 0) {
throw new IllegalStateException("index is negative (" + index + ")");
}
mValidOutputIndices.set(index);
} else if (index >= 0 && index < buffers.length) {
ByteBuffer buffer = buffers[index]; ByteBuffer buffer = buffers[index];
if (buffer != null) { if (buffer != null) {
buffer.setAccessible(true); buffer.setAccessible(true);
@@ -3836,7 +3860,7 @@ final public class MediaCodec {
} }
} }
private final void invalidateByteBuffers(@Nullable ByteBuffer[] buffers) { private void invalidateByteBuffersLocked(@Nullable ByteBuffer[] buffers) {
if (buffers != null) { if (buffers != null) {
for (ByteBuffer buffer: buffers) { for (ByteBuffer buffer: buffers) {
if (buffer != null) { if (buffer != null) {
@@ -3846,27 +3870,29 @@ final public class MediaCodec {
} }
} }
private final void freeByteBuffer(@Nullable ByteBuffer buffer) { private void freeByteBufferLocked(@Nullable ByteBuffer buffer) {
if (buffer != null /* && buffer.isDirect() */) { if (buffer != null /* && buffer.isDirect() */) {
// all of our ByteBuffers are direct // all of our ByteBuffers are direct
java.nio.NioUtils.freeDirectBuffer(buffer); java.nio.NioUtils.freeDirectBuffer(buffer);
} }
} }
private final void freeByteBuffers(@Nullable ByteBuffer[] buffers) { private void freeByteBuffersLocked(@Nullable ByteBuffer[] buffers) {
if (buffers != null) { if (buffers != null) {
for (ByteBuffer buffer: buffers) { for (ByteBuffer buffer: buffers) {
freeByteBuffer(buffer); freeByteBufferLocked(buffer);
} }
} }
} }
private final void freeAllTrackedBuffers() { private void freeAllTrackedBuffers() {
synchronized(mBufferLock) { synchronized(mBufferLock) {
freeByteBuffers(mCachedInputBuffers); freeByteBuffersLocked(mCachedInputBuffers);
freeByteBuffers(mCachedOutputBuffers); freeByteBuffersLocked(mCachedOutputBuffers);
mCachedInputBuffers = null; mCachedInputBuffers = null;
mCachedOutputBuffers = null; mCachedOutputBuffers = null;
mValidInputIndices.clear();
mValidOutputIndices.clear();
mDequeuedInputBuffers.clear(); mDequeuedInputBuffers.clear();
mDequeuedOutputBuffers.clear(); mDequeuedOutputBuffers.clear();
mQueueRequests.clear(); mQueueRequests.clear();
@@ -3874,14 +3900,31 @@ final public class MediaCodec {
} }
} }
private final void cacheBuffers(boolean input) { private void cacheBuffersLocked(boolean input) {
ByteBuffer[] buffers = null; ByteBuffer[] buffers = null;
try { try {
buffers = getBuffers(input); buffers = getBuffers(input);
invalidateByteBuffers(buffers); invalidateByteBuffersLocked(buffers);
} catch (IllegalStateException e) { } catch (IllegalStateException e) {
// we don't get buffers in async mode // we don't get buffers in async mode
} }
if (buffers != null) {
BitSet indices = input ? mValidInputIndices : mValidOutputIndices;
for (int i = 0; i < buffers.length; ++i) {
ByteBuffer buffer = buffers[i];
if (buffer == null || !indices.get(i)) {
continue;
}
buffer.setAccessible(true);
if (!input) {
BufferInfo info = mDequeuedOutputInfos.get(i);
if (info != null) {
buffer.limit(info.offset + info.size).position(info.offset);
}
}
}
indices.clear();
}
if (input) { if (input) {
mCachedInputBuffers = buffers; mCachedInputBuffers = buffers;
} else { } else {
@@ -3916,6 +3959,9 @@ final public class MediaCodec {
+ "Please obtain MediaCodec.LinearBlock or HardwareBuffer " + "Please obtain MediaCodec.LinearBlock or HardwareBuffer "
+ "objects and attach to QueueRequest objects."); + "objects and attach to QueueRequest objects.");
} }
if (mCachedInputBuffers == null) {
cacheBuffersLocked(true /* input */);
}
if (mCachedInputBuffers == null) { if (mCachedInputBuffers == null) {
throw new IllegalStateException(); throw new IllegalStateException();
} }
@@ -3954,6 +4000,9 @@ final public class MediaCodec {
+ "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. "
+ "Please use getOutputFrame to get output frames."); + "Please use getOutputFrame to get output frames.");
} }
if (mCachedOutputBuffers == null) {
cacheBuffersLocked(false /* input */);
}
if (mCachedOutputBuffers == null) { if (mCachedOutputBuffers == null) {
throw new IllegalStateException(); throw new IllegalStateException();
} }
@@ -3992,7 +4041,7 @@ final public class MediaCodec {
} }
ByteBuffer newBuffer = getBuffer(true /* input */, index); ByteBuffer newBuffer = getBuffer(true /* input */, index);
synchronized (mBufferLock) { synchronized (mBufferLock) {
invalidateByteBuffer(mCachedInputBuffers, index); invalidateByteBufferLocked(mCachedInputBuffers, index, true /* input */);
mDequeuedInputBuffers.put(index, newBuffer); mDequeuedInputBuffers.put(index, newBuffer);
} }
return newBuffer; return newBuffer;
@@ -4029,7 +4078,7 @@ final public class MediaCodec {
} }
Image newImage = getImage(true /* input */, index); Image newImage = getImage(true /* input */, index);
synchronized (mBufferLock) { synchronized (mBufferLock) {
invalidateByteBuffer(mCachedInputBuffers, index); invalidateByteBufferLocked(mCachedInputBuffers, index, true /* input */);
mDequeuedInputBuffers.put(index, newImage); mDequeuedInputBuffers.put(index, newImage);
} }
return newImage; return newImage;
@@ -4065,7 +4114,7 @@ final public class MediaCodec {
} }
ByteBuffer newBuffer = getBuffer(false /* input */, index); ByteBuffer newBuffer = getBuffer(false /* input */, index);
synchronized (mBufferLock) { synchronized (mBufferLock) {
invalidateByteBuffer(mCachedOutputBuffers, index); invalidateByteBufferLocked(mCachedOutputBuffers, index, false /* input */);
mDequeuedOutputBuffers.put(index, newBuffer); mDequeuedOutputBuffers.put(index, newBuffer);
} }
return newBuffer; return newBuffer;
@@ -4100,7 +4149,7 @@ final public class MediaCodec {
} }
Image newImage = getImage(false /* input */, index); Image newImage = getImage(false /* input */, index);
synchronized (mBufferLock) { synchronized (mBufferLock) {
invalidateByteBuffer(mCachedOutputBuffers, index); invalidateByteBufferLocked(mCachedOutputBuffers, index, false /* input */);
mDequeuedOutputBuffers.put(index, newImage); mDequeuedOutputBuffers.put(index, newImage);
} }
return newImage; return newImage;