From 56a7cce3097523eefb8a21b79443874a0b473d2b Mon Sep 17 00:00:00 2001 From: Philip Quinn Date: Sun, 11 Dec 2022 18:29:01 -0800 Subject: [PATCH] Add support for TFLite motion prediction model. Disable motion prediction by default and only enable on tested devices. Bug: 167946763 Test: atest InputTests Test: MotionPrediction test app Change-Id: I187a4aec7ad07e7b4a7d2b500e3ca8803f8bb395 --- core/java/android/view/MotionPredictor.java | 73 +++++++++++-------- core/jni/Android.bp | 3 + core/res/res/values/config.xml | 19 ++--- .../android/test/input/MotionPredictorTest.kt | 19 ++--- 4 files changed, 62 insertions(+), 52 deletions(-) diff --git a/core/java/android/view/MotionPredictor.java b/core/java/android/view/MotionPredictor.java index fa86a4c73fb20..4d32efea9081e 100644 --- a/core/java/android/view/MotionPredictor.java +++ b/core/java/android/view/MotionPredictor.java @@ -22,26 +22,27 @@ import android.content.Context; import libcore.util.NativeAllocationRegistry; import java.util.Arrays; +import java.util.Collections; import java.util.List; /** * Calculate motion predictions. * - * Feed motion events to this class in order to generate the predicted events. The prediction - * functionality may not be available on all devices. Check if a specific source is supported on a - * given input device using #isPredictionAvailable. + * Feed motion events to this class in order to generate predicted future events. The prediction + * functionality may not be available on all devices: check if a specific source is supported on a + * given input device using {@link #isPredictionAvailable}. * - * Send all of the events that were received from the system here in order to generate complete, - * accurate predictions. When processing the returned predictions, make sure to consider all of the - * {@link MotionEvent#getHistoricalAxisValue historical samples}. + * Send all of the events that were received from the system to {@link #record} to generate + * complete, accurate predictions from {@link #predict}. When processing the returned predictions, + * make sure to consider all of the {@link MotionEvent#getHistoricalAxisValue historical samples}. */ -// Acts as a pass-through to the native MotionPredictor object. -// Do not store any state in this Java layer, or add any business logic here. All of the -// implementation details should go into the native MotionPredictor. -// The context / resource access must be here rather than in native layer due to the lack of the -// corresponding native API surface. public final class MotionPredictor { + // This is a pass-through to the native MotionPredictor object (mPtr). Do not store any state or + // add any business logic here -- all of the implementation details should go into the native + // MotionPredictor (except for accessing the context/resources, which have no corresponding + // native API). + private static class RegistryHolder { public static final NativeAllocationRegistry REGISTRY = NativeAllocationRegistry.createMalloced( @@ -67,49 +68,63 @@ public final class MotionPredictor { /** * Record a movement so that in the future, a prediction for the current gesture can be - * generated. Ensure to add all motions from the gesture of interest to generate the correct - * prediction. + * generated. Ensure to add all motions from the gesture of interest to generate correct + * predictions. * @param event The received event */ public void record(@NonNull MotionEvent event) { + if (!isPredictionEnabled()) { + return; + } nativeRecord(mPtr, event); } /** - * Get predicted events for all gestures that have been provided to the 'record' function. + * Get predicted events for all gestures that have been provided to {@link #record}. * If events from multiple devices were sent to 'record', this will produce a separate - * {@link MotionEvent} for each device id. The returned list may be empty if no predictions for - * any of the added events are available. + * {@link MotionEvent} for each device. The returned list may be empty if no predictions for + * any of the added events/devices are available. * Predictions may not reach the requested timestamp if the confidence in the prediction results * is low. * * @param predictionTimeNanos The time that the prediction should target, in the * {@link android.os.SystemClock#uptimeMillis} time base, but in nanoseconds. * - * @return the list of predicted motion events, for each device id. Ensure to check the - * historical data in addition to the latest ({@link MotionEvent#getX getX()}, - * {@link MotionEvent#getY getY()}) coordinates for smoothest prediction curves. Empty list is - * returned if predictions are not supported, or not possible for the current set of gestures. + * @return A list of predicted motion events, with at most one for each device observed by + * {@link #record}. Be sure to check the historical data in addition to the latest + * ({@link MotionEvent#getX getX()}, {@link MotionEvent#getY getY()}) coordinates for smooth + * prediction curves. An empty list is returned if predictions are not supported, or not + * possible for the current set of gestures. */ @NonNull public List predict(long predictionTimeNanos) { + if (!isPredictionEnabled()) { + return Collections.emptyList(); + } return Arrays.asList(nativePredict(mPtr, predictionTimeNanos)); } - /** - * Check whether this device supports motion predictions for the given source type. - * - * @param deviceId The input device id - * @param source The source of input events - * @return True if the current device supports predictions, false otherwise. - */ - public boolean isPredictionAvailable(int deviceId, int source) { + private boolean isPredictionEnabled() { // Device-specific override if (!mContext.getResources().getBoolean( com.android.internal.R.bool.config_enableMotionPrediction)) { return false; } - return nativeIsPredictionAvailable(mPtr, deviceId, source); + return true; + } + + /** + * Check whether a device supports motion predictions for a given source type. + * + * @param deviceId The input device id. + * @param source The source of input events. + * @return True if the current device supports predictions, false otherwise. + * + * @see MotionEvent#getDeviceId + * @see MotionEvent#getSource + */ + public boolean isPredictionAvailable(int deviceId, int source) { + return isPredictionEnabled() && nativeIsPredictionAvailable(mPtr, deviceId, source); } private static native long nativeInitialize(int offsetNanos); diff --git a/core/jni/Android.bp b/core/jni/Android.bp index 91d2ecb593a53..d33f6d3d1cbf1 100644 --- a/core/jni/Android.bp +++ b/core/jni/Android.bp @@ -326,6 +326,7 @@ cc_library_shared { "libdl", "libdl_android", "libtimeinstate", + "libtflite", "server_configurable_flags", "libimage_io", "libjpegdecoder", @@ -343,7 +344,9 @@ cc_library_shared { header_libs: [ "bionic_libc_platform_headers", "dnsproxyd_protocol_headers", + "flatbuffer_headers", "libtextclassifier_hash_headers", + "tensorflow_headers", ], runtime_libs: [ "libidmap2", diff --git a/core/res/res/values/config.xml b/core/res/res/values/config.xml index be22095174edd..15e252039dbe1 100644 --- a/core/res/res/values/config.xml +++ b/core/res/res/values/config.xml @@ -2351,22 +2351,13 @@ display, this value should be true. --> false - - true + + false + that the prediction will take place further in the future and, in general, should represent + some built-in hardware delays that prediction should try to recover. --> 0