Merge "Send input timeline from app to InputDispatcher" into sc-dev

core/java/android/view/InputEventReceiver.java

@@ -54,6 +54,8 @@ public abstract class InputEventReceiver {
             InputChannel inputChannel, MessageQueue messageQueue);
     private static native void nativeDispose(long receiverPtr);
     private static native void nativeFinishInputEvent(long receiverPtr, int seq, boolean handled);
+    private static native void nativeReportTimeline(long receiverPtr, int inputEventId,
+            long gpuCompletedTime, long presentTime);
     private static native boolean nativeConsumeBatchedInputEvents(long receiverPtr,
             long frameTimeNanos);
     private static native String nativeDump(long receiverPtr, String prefix);
@@ -209,11 +211,11 @@ public abstract class InputEventReceiver {
     }
 
     /**
-     * Report the latency information for a specific input event.
+     * Report the timing / latency information for a specific input event.
      */
-    public final void reportLatencyInfo(int inputEventId, long gpuCompletedTime, long presentTime) {
+    public final void reportTimeline(int inputEventId, long gpuCompletedTime, long presentTime) {
-        Trace.traceBegin(Trace.TRACE_TAG_INPUT, "reportLatencyInfo");
+        Trace.traceBegin(Trace.TRACE_TAG_INPUT, "reportTimeline");
-        // TODO(b/169866723) : send this data to InputDispatcher via InputChannel
+        nativeReportTimeline(mReceiverPtr, inputEventId, gpuCompletedTime, presentTime);
         Trace.traceEnd(Trace.TRACE_TAG_INPUT);
     }
 

core/java/android/view/InputEventSender.java

@@ -111,6 +111,16 @@ public abstract class InputEventSender {
     public void onInputEventFinished(int seq, boolean handled) {
     }
 
+    /**
+     * Called when timeline is sent to the publisher.
+     *
+     * @param inputEventId The id of the input event that caused the frame being reported
+     * @param gpuCompletedTime The time when the frame left the app process
+     * @param presentTime The time when the frame was presented on screen
+     */
+    public void onTimelineReported(int inputEventId, long gpuCompletedTime, long presentTime) {
+    }
+
     /**
      * Sends an input event.
      * Must be called on the same Looper thread to which the sender is attached.
@@ -143,4 +153,11 @@ public abstract class InputEventSender {
     private void dispatchInputEventFinished(int seq, boolean handled) {
         onInputEventFinished(seq, handled);
     }
+
+    // Called from native code.
+    @SuppressWarnings("unused")
+    private void dispatchTimelineReported(
+            int inputEventId, long gpuCompletedTime, long presentTime) {
+        onTimelineReported(inputEventId, gpuCompletedTime, presentTime);
+    }
 }

core/java/android/view/ViewRootImpl.java

@@ -8601,7 +8601,7 @@ public final class ViewRootImpl implements ViewParent,
                 return;
             }
             final long gpuCompletedTime = data[FrameMetrics.Index.GPU_COMPLETED];
-            mReceiver.reportLatencyInfo(inputEventId, gpuCompletedTime, presentTime);
+            mReceiver.reportTimeline(inputEventId, gpuCompletedTime, presentTime);
         }
     }
     HardwareRendererObserver mHardwareRendererObserver;

core/jni/android_view_InputEventReceiver.cpp

@@ -27,6 +27,7 @@
 #include <input/InputTransport.h>
 #include <log/log.h>
 #include <utils/Looper.h>
+#include <variant>
 #include <vector>
 #include "android_os_MessageQueue.h"
 #include "android_view_InputChannel.h"
@@ -80,6 +81,7 @@ public:
     status_t initialize();
     void dispose();
     status_t finishInputEvent(uint32_t seq, bool handled);
+    status_t reportTimeline(int32_t inputEventId, nsecs_t gpuCompletedTime, nsecs_t presentTime);
     status_t consumeEvents(JNIEnv* env, bool consumeBatches, nsecs_t frameTime,
             bool* outConsumedBatch);
     std::string dump(const char* prefix);
@@ -93,13 +95,19 @@ private:
         bool handled;
     };
 
+    struct Timeline {
+        int32_t inputEventId;
+        std::array<nsecs_t, GraphicsTimeline::SIZE> timeline;
+    };
+    typedef std::variant<Finish, Timeline> OutboundEvent;
+
     jobject mReceiverWeakGlobal;
     InputConsumer mInputConsumer;
     sp<MessageQueue> mMessageQueue;
     PreallocatedInputEventFactory mInputEventFactory;
     bool mBatchedInputEventPending;
     int mFdEvents;
-    std::vector<Finish> mFinishQueue;
+    std::vector<OutboundEvent> mOutboundQueue;
 
     void setFdEvents(int events);
 
@@ -152,7 +160,23 @@ status_t NativeInputEventReceiver::finishInputEvent(uint32_t seq, bool handled)
             .seq = seq,
             .handled = handled,
     };
-    mFinishQueue.push_back(finish);
+    mOutboundQueue.push_back(finish);
+    return processOutboundEvents();
+}
+
+status_t NativeInputEventReceiver::reportTimeline(int32_t inputEventId, nsecs_t gpuCompletedTime,
+                                                  nsecs_t presentTime) {
+    if (kDebugDispatchCycle) {
+        ALOGD("channel '%s' ~ %s", getInputChannelName().c_str(), __func__);
+    }
+    std::array<nsecs_t, GraphicsTimeline::SIZE> graphicsTimeline;
+    graphicsTimeline[GraphicsTimeline::GPU_COMPLETED_TIME] = gpuCompletedTime;
+    graphicsTimeline[GraphicsTimeline::PRESENT_TIME] = presentTime;
+    Timeline timeline{
+            .inputEventId = inputEventId,
+            .timeline = graphicsTimeline,
+    };
+    mOutboundQueue.push_back(timeline);
     return processOutboundEvents();
 }
 
@@ -170,7 +194,7 @@ void NativeInputEventReceiver::setFdEvents(int events) {
 
 /**
  * Receiver's primary role is to receive input events, but it has an additional duty of sending
- * 'ack' for events (using the call 'finishInputEvent').
+ * 'ack' for events (using the call 'finishInputEvent') and reporting input event timeline.
  *
  * If we are looking at the communication between InputPublisher and InputConsumer, we can say that
  * from the InputConsumer's perspective, InputMessage's that are sent from publisher to consumer are
@@ -178,19 +202,31 @@ void NativeInputEventReceiver::setFdEvents(int events) {
  * InputPublisher are 'outbound / outgoing' events.
  *
  * NativeInputEventReceiver owns (and acts like) an InputConsumer. So the finish events are outbound
- * from InputEventReceiver (and will be sent to the InputPublisher).
+ * from InputEventReceiver (and will be sent to the InputPublisher). Likewise, timeline events are
+ * outbound events.
  *
- * In this function, send as many events from 'mFinishQueue' as possible across the socket to the
+ * In this function, send as many events from 'mOutboundQueue' as possible across the socket to the
  * InputPublisher. If no events are remaining, let the looper know so that it doesn't wake up
  * unnecessarily.
  */
 status_t NativeInputEventReceiver::processOutboundEvents() {
-    while (!mFinishQueue.empty()) {
+    while (!mOutboundQueue.empty()) {
-        const Finish& finish = *mFinishQueue.begin();
+        OutboundEvent& outbound = *mOutboundQueue.begin();
-        status_t status = mInputConsumer.sendFinishedSignal(finish.seq, finish.handled);
+        status_t status;
+
+        if (std::holds_alternative<Finish>(outbound)) {
+            const Finish& finish = std::get<Finish>(outbound);
+            status = mInputConsumer.sendFinishedSignal(finish.seq, finish.handled);
+        } else if (std::holds_alternative<Timeline>(outbound)) {
+            const Timeline& timeline = std::get<Timeline>(outbound);
+            status = mInputConsumer.sendTimeline(timeline.inputEventId, timeline.timeline);
+        } else {
+            LOG_ALWAYS_FATAL("Unexpected event type in std::variant");
+            status = BAD_VALUE;
+        }
         if (status == OK) {
             // Successful send. Erase the entry and keep trying to send more
-            mFinishQueue.erase(mFinishQueue.begin());
+            mOutboundQueue.erase(mOutboundQueue.begin());
             continue;
         }
 
@@ -198,7 +234,7 @@ status_t NativeInputEventReceiver::processOutboundEvents() {
         if (status == WOULD_BLOCK) {
             if (kDebugDispatchCycle) {
                 ALOGD("channel '%s' ~ Remaining outbound events: %zu.",
-                      getInputChannelName().c_str(), mFinishQueue.size());
+                      getInputChannelName().c_str(), mOutboundQueue.size());
             }
             setFdEvents(ALOOPER_EVENT_INPUT | ALOOPER_EVENT_OUTPUT);
             return WOULD_BLOCK; // try again later
@@ -425,12 +461,23 @@ std::string NativeInputEventReceiver::dump(const char* prefix) {
 
     out += android::base::StringPrintf("mBatchedInputEventPending: %s\n",
                                        toString(mBatchedInputEventPending));
-    out = out + "mFinishQueue:\n";
+    out = out + "mOutboundQueue:\n";
-    for (const Finish& finish : mFinishQueue) {
+    for (const OutboundEvent& outbound : mOutboundQueue) {
-        out += android::base::StringPrintf("  seq=%" PRIu32 " handled=%s\n", finish.seq,
+        if (std::holds_alternative<Finish>(outbound)) {
+            const Finish& finish = std::get<Finish>(outbound);
+            out += android::base::StringPrintf("  Finish: seq=%" PRIu32 " handled=%s\n", finish.seq,
                                                toString(finish.handled));
+        } else if (std::holds_alternative<Timeline>(outbound)) {
+            const Timeline& timeline = std::get<Timeline>(outbound);
+            out += android::base::
+                    StringPrintf("  Timeline: inputEventId=%" PRId32 " gpuCompletedTime=%" PRId64
+                                 ", presentTime=%" PRId64 "\n",
+                                 timeline.inputEventId,
+                                 timeline.timeline[GraphicsTimeline::GPU_COMPLETED_TIME],
+                                 timeline.timeline[GraphicsTimeline::PRESENT_TIME]);
         }
-    if (mFinishQueue.empty()) {
+    }
+    if (mOutboundQueue.empty()) {
         out = out + "  <empty>\n";
     }
     return addPrefix(out, prefix);
@@ -489,6 +536,25 @@ static void nativeFinishInputEvent(JNIEnv* env, jclass clazz, jlong receiverPtr,
     }
 }
 
+static void nativeReportTimeline(JNIEnv* env, jclass clazz, jlong receiverPtr, jint inputEventId,
+                                 jlong gpuCompletedTime, jlong presentTime) {
+    if (IdGenerator::getSource(inputEventId) != IdGenerator::Source::INPUT_READER) {
+        // skip this event, it did not originate from hardware
+        return;
+    }
+    sp<NativeInputEventReceiver> receiver =
+            reinterpret_cast<NativeInputEventReceiver*>(receiverPtr);
+    status_t status = receiver->reportTimeline(inputEventId, gpuCompletedTime, presentTime);
+    if (status == OK || status == WOULD_BLOCK) {
+        return; // normal operation
+    }
+    if (status != DEAD_OBJECT) {
+        std::string message = android::base::StringPrintf("Failed to send timeline.  status=%s(%d)",
+                                                          strerror(-status), status);
+        jniThrowRuntimeException(env, message.c_str());
+    }
+}
+
 static jboolean nativeConsumeBatchedInputEvents(JNIEnv* env, jclass clazz, jlong receiverPtr,
         jlong frameTimeNanos) {
     sp<NativeInputEventReceiver> receiver =
@@ -520,6 +586,7 @@ static const JNINativeMethod gMethods[] = {
          (void*)nativeInit},
         {"nativeDispose", "(J)V", (void*)nativeDispose},
         {"nativeFinishInputEvent", "(JIZ)V", (void*)nativeFinishInputEvent},
+        {"nativeReportTimeline", "(JIJJ)V", (void*)nativeReportTimeline},
         {"nativeConsumeBatchedInputEvents", "(JJ)Z", (void*)nativeConsumeBatchedInputEvents},
         {"nativeDump", "(JLjava/lang/String;)Ljava/lang/String;", (void*)nativeDump},
 };

core/jni/android_view_InputEventSender.cpp

@@ -45,6 +45,7 @@ static struct {
     jclass clazz;
 
     jmethodID dispatchInputEventFinished;
+    jmethodID dispatchTimelineReported;
 } gInputEventSenderClassInfo;
 
 
@@ -75,8 +76,9 @@ private:
     }
 
     int handleEvent(int receiveFd, int events, void* data) override;
-    status_t receiveFinishedSignals(JNIEnv* env);
+    status_t processConsumerResponse(JNIEnv* env);
-    bool notifyFinishedSignal(JNIEnv* env, jobject sender, const InputPublisher::Finished& finished,
+    bool notifyConsumerResponse(JNIEnv* env, jobject sender,
+                                const InputPublisher::ConsumerResponse& response,
                                 bool skipCallbacks);
 };
 
@@ -188,12 +190,12 @@ int NativeInputEventSender::handleEvent(int receiveFd, int events, void* data) {
     }
 
     JNIEnv* env = AndroidRuntime::getJNIEnv();
-    status_t status = receiveFinishedSignals(env);
+    status_t status = processConsumerResponse(env);
     mMessageQueue->raiseAndClearException(env, "handleReceiveCallback");
     return status == OK || status == NO_MEMORY ? 1 : 0;
 }
 
-status_t NativeInputEventSender::receiveFinishedSignals(JNIEnv* env) {
+status_t NativeInputEventSender::processConsumerResponse(JNIEnv* env) {
     if (kDebugDispatchCycle) {
         ALOGD("channel '%s' ~ Receiving finished signals.", getInputChannelName().c_str());
     }
@@ -206,18 +208,18 @@ status_t NativeInputEventSender::receiveFinishedSignals(JNIEnv* env) {
     }
     bool skipCallbacks = false; // stop calling Java functions after an exception occurs
     for (;;) {
-        Result<InputPublisher::Finished> result = mInputPublisher.receiveFinishedSignal();
+        Result<InputPublisher::ConsumerResponse> result = mInputPublisher.receiveConsumerResponse();
         if (!result.ok()) {
             const status_t status = result.error().code();
             if (status == WOULD_BLOCK) {
                 return OK;
             }
-            ALOGE("channel '%s' ~ Failed to consume finished signals.  status=%d",
+            ALOGE("channel '%s' ~ Failed to process consumer response.  status=%d",
                   getInputChannelName().c_str(), status);
             return status;
         }
 
-        const bool notified = notifyFinishedSignal(env, senderObj.get(), *result, skipCallbacks);
+        const bool notified = notifyConsumerResponse(env, senderObj.get(), *result, skipCallbacks);
         if (!notified) {
             skipCallbacks = true;
         }
@@ -225,16 +227,49 @@ status_t NativeInputEventSender::receiveFinishedSignals(JNIEnv* env) {
 }
 
 /**
- * Invoke the Java function dispatchInputEventFinished for the received "Finished" signal.
+ * Invoke the corresponding Java function for the different variants of response.
- * Set the variable 'skipCallbacks' to 'true' if a Java exception occurred.
+ * If the response is a Finished object, invoke dispatchInputEventFinished.
+ * If the response is a Timeline object, invoke dispatchTimelineReported.
+ * Set 'skipCallbacks' to 'true' if a Java exception occurred.
  * Java function will only be called if 'skipCallbacks' is originally 'false'.
  *
  * Return "false" if an exception occurred while calling the Java function
  *        "true" otherwise
  */
-bool NativeInputEventSender::notifyFinishedSignal(JNIEnv* env, jobject sender,
+bool NativeInputEventSender::notifyConsumerResponse(
-                                                  const InputPublisher::Finished& finished,
+        JNIEnv* env, jobject sender, const InputPublisher::ConsumerResponse& response,
         bool skipCallbacks) {
+    if (std::holds_alternative<InputPublisher::Timeline>(response)) {
+        const InputPublisher::Timeline& timeline = std::get<InputPublisher::Timeline>(response);
+
+        if (kDebugDispatchCycle) {
+            ALOGD("channel '%s' ~ Received timeline, inputEventId=%" PRId32
+                  ", gpuCompletedTime=%" PRId64 ", presentTime=%" PRId64,
+                  getInputChannelName().c_str(), timeline.inputEventId,
+                  timeline.graphicsTimeline[GraphicsTimeline::GPU_COMPLETED_TIME],
+                  timeline.graphicsTimeline[GraphicsTimeline::PRESENT_TIME]);
+        }
+
+        if (skipCallbacks) {
+            ALOGW("Java exception occurred. Skipping dispatchTimelineReported for "
+                  "inputEventId=%" PRId32,
+                  timeline.inputEventId);
+            return true;
+        }
+
+        env->CallVoidMethod(sender, gInputEventSenderClassInfo.dispatchTimelineReported,
+                            timeline.inputEventId, timeline.graphicsTimeline);
+        if (env->ExceptionCheck()) {
+            ALOGE("Exception dispatching timeline, inputEventId=%" PRId32, timeline.inputEventId);
+            return false;
+        }
+
+        return true;
+    }
+
+    // Must be a Finished event
+    const InputPublisher::Finished& finished = std::get<InputPublisher::Finished>(response);
+
     auto it = mPublishedSeqMap.find(finished.seq);
     if (it == mPublishedSeqMap.end()) {
         ALOGW("Received 'finished' signal for unknown seq number = %" PRIu32, finished.seq);
@@ -340,6 +375,9 @@ int register_android_view_InputEventSender(JNIEnv* env) {
 
     gInputEventSenderClassInfo.dispatchInputEventFinished = GetMethodIDOrDie(
             env, gInputEventSenderClassInfo.clazz, "dispatchInputEventFinished", "(IZ)V");
+    gInputEventSenderClassInfo.dispatchTimelineReported =
+            GetMethodIDOrDie(env, gInputEventSenderClassInfo.clazz, "dispatchTimelineReported",
+                             "(IJJ)V");
 
     return res;
 }

tests/Input/src/com/android/test/input/InputEventSenderAndReceiverTest.kt

@@ -75,16 +75,26 @@ class TestInputEventReceiver(channel: InputChannel, looper: Looper) :
 class TestInputEventSender(channel: InputChannel, looper: Looper) :
         InputEventSender(channel, looper) {
     data class FinishedSignal(val seq: Int, val handled: Boolean)
+    data class Timeline(val inputEventId: Int, val gpuCompletedTime: Long, val presentTime: Long)
 
     private val mFinishedSignals = LinkedBlockingQueue<FinishedSignal>()
+    private val mTimelines = LinkedBlockingQueue<Timeline>()
 
     override fun onInputEventFinished(seq: Int, handled: Boolean) {
         mFinishedSignals.put(FinishedSignal(seq, handled))
     }
 
+    override fun onTimelineReported(inputEventId: Int, gpuCompletedTime: Long, presentTime: Long) {
+        mTimelines.put(Timeline(inputEventId, gpuCompletedTime, presentTime))
+    }
+
     fun getFinishedSignal(): FinishedSignal {
         return getEvent(mFinishedSignals)
     }
+
+    fun getTimeline(): Timeline {
+        return getEvent(mTimelines)
+    }
 }
 
 class InputEventSenderAndReceiverTest {
@@ -125,4 +135,14 @@ class InputEventSenderAndReceiverTest {
         // Check sender
         assertEquals(TestInputEventSender.FinishedSignal(seq, handled = true), finishedSignal)
     }
+
+    // The timeline case is slightly unusual because it goes from InputConsumer to InputPublisher.
+    @Test
+    fun testSendAndReceiveTimeline() {
+        val sent = TestInputEventSender.Timeline(
+            inputEventId = 1, gpuCompletedTime = 2, presentTime = 3)
+        mReceiver.reportTimeline(sent.inputEventId, sent.gpuCompletedTime, sent.presentTime)
+        val received = mSender.getTimeline()
+        assertEquals(sent, received)
+    }
 }