Stagefright: idle OMX after ANW errors

This change fixes an issue in Stagefright where the state of an OMXCodec
object can get out of sync with the state of the OMX component.  In
particular, if one of the ANativeWindow functions failed and put the
OMXCodec into the ERROR state, this would cause Stagefright to skip
doing the Executing -> Idle transition.  Without this transition the
freeBuffersOnPort call would never be made, and the MediaBuffers would
end up being leaked (which would also leak the Gralloc buffers they
reference).

Bug: 5333695
Change-Id: I85ea0cf92d18e7ef6d35c7d1e2a7b4e2c9745d34

include/media/IOMX.h

@@ -78,6 +78,9 @@ public:
             node_id node, OMX_INDEXTYPE index,
             const void *params, size_t size) = 0;
 
+    virtual status_t getState(
+            node_id node, OMX_STATETYPE* state) = 0;
+
     virtual status_t storeMetaDataInBuffers(
             node_id node, OMX_U32 port_index, OMX_BOOL enable) = 0;
 

media/libmedia/IOMX.cpp

@@ -38,6 +38,7 @@ enum {
     SET_PARAMETER,
     GET_CONFIG,
     SET_CONFIG,
+    GET_STATE,
     ENABLE_GRAPHIC_BUFFERS,
     USE_BUFFER,
     USE_GRAPHIC_BUFFER,
@@ -198,6 +199,17 @@ public:
         return reply.readInt32();
     }
 
+    virtual status_t getState(
+            node_id node, OMX_STATETYPE* state) {
+        Parcel data, reply;
+        data.writeInterfaceToken(IOMX::getInterfaceDescriptor());
+        data.writeIntPtr((intptr_t)node);
+        remote()->transact(GET_STATE, data, &reply);
+
+        *state = static_cast<OMX_STATETYPE>(reply.readInt32());
+        return reply.readInt32();
+    }
+
     virtual status_t enableGraphicBuffers(
             node_id node, OMX_U32 port_index, OMX_BOOL enable) {
         Parcel data, reply;
@@ -524,6 +536,20 @@ status_t BnOMX::onTransact(
             return NO_ERROR;
         }
 
+        case GET_STATE:
+        {
+            CHECK_INTERFACE(IOMX, data, reply);
+
+            node_id node = (void*)data.readIntPtr();
+            OMX_STATETYPE state = OMX_StateInvalid;
+
+            status_t err = getState(node, &state);
+            reply->writeInt32(state);
+            reply->writeInt32(err);
+
+            return NO_ERROR;
+        }
+
         case ENABLE_GRAPHIC_BUFFERS:
         {
             CHECK_INTERFACE(IOMX, data, reply);

media/libstagefright/OMXCodec.cpp

@@ -3600,11 +3600,24 @@ status_t OMXCodec::stop() {
         mAsyncCompletion.wait(mLock);
     }
 
+    bool isError = false;
     switch (mState) {
         case LOADED:
-        case ERROR:
             break;
 
+        case ERROR:
+        {
+            OMX_STATETYPE state = OMX_StateInvalid;
+            status_t err = mOMX->getState(mNode, &state);
+            CHECK_EQ(err, (status_t)OK);
+
+            if (state != OMX_StateExecuting) {
+                break;
+            }
+            // else fall through to the idling code
+            isError = true;
+        }
+
         case EXECUTING:
         {
             setState(EXECUTING_TO_IDLE);
@@ -3639,6 +3652,12 @@ status_t OMXCodec::stop() {
                 mAsyncCompletion.wait(mLock);
             }
 
+            if (isError) {
+                // We were in the ERROR state coming in, so restore that now
+                // that we've idled the OMX component.
+                setState(ERROR);
+            }
+
             break;
         }
 

media/libstagefright/include/OMX.h

@@ -59,6 +59,9 @@ public:
             node_id node, OMX_INDEXTYPE index,
             const void *params, size_t size);
 
+    virtual status_t getState(
+            node_id node, OMX_STATETYPE* state);
+
     virtual status_t enableGraphicBuffers(
             node_id node, OMX_U32 port_index, OMX_BOOL enable);
 

media/libstagefright/include/OMXNodeInstance.h

@@ -49,6 +49,8 @@ struct OMXNodeInstance {
     status_t getConfig(OMX_INDEXTYPE index, void *params, size_t size);
     status_t setConfig(OMX_INDEXTYPE index, const void *params, size_t size);
 
+    status_t getState(OMX_STATETYPE* state);
+
     status_t enableGraphicBuffers(OMX_U32 portIndex, OMX_BOOL enable);
 
     status_t getGraphicBufferUsage(OMX_U32 portIndex, OMX_U32* usage);

media/libstagefright/omx/OMX.cpp

@@ -303,6 +303,12 @@ status_t OMX::setConfig(
             index, params, size);
 }
 
+status_t OMX::getState(
+        node_id node, OMX_STATETYPE* state) {
+    return findInstance(node)->getState(
+            state);
+}
+
 status_t OMX::enableGraphicBuffers(
         node_id node, OMX_U32 port_index, OMX_BOOL enable) {
     return findInstance(node)->enableGraphicBuffers(port_index, enable);

media/libstagefright/omx/OMXNodeInstance.cpp

@@ -266,6 +266,14 @@ status_t OMXNodeInstance::setConfig(
     return StatusFromOMXError(err);
 }
 
+status_t OMXNodeInstance::getState(OMX_STATETYPE* state) {
+    Mutex::Autolock autoLock(mLock);
+
+    OMX_ERRORTYPE err = OMX_GetState(mHandle, state);
+
+    return StatusFromOMXError(err);
+}
+
 status_t OMXNodeInstance::enableGraphicBuffers(
         OMX_U32 portIndex, OMX_BOOL enable) {
     Mutex::Autolock autoLock(mLock);