Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge "Add support for TFLite motion prediction model."

Browse Source
This commit is contained in:
Philip Quinn
2023-01-28 21:45:47 +00:00
committed by Android (Google) Code Review
parent f9412d0d9c 56a7cce309
commit e5b2c6f41e
4 changed files with 62 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

73
core/java/android/view/MotionPredictor.java
View File

@@ -22,26 +22,27 @@ import android.content.Context;
import libcore.util.NativeAllocationRegistry;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
/**
* Calculate motion predictions.
*
* Feed motion events to this class in order to generate the predicted events. The prediction
* functionality may not be available on all devices. Check if a specific source is supported on a
* given input device using #isPredictionAvailable.
* Feed motion events to this class in order to generate predicted future events. The prediction
* functionality may not be available on all devices: check if a specific source is supported on a
* given input device using {@link #isPredictionAvailable}.
*
* Send all of the events that were received from the system here in order to generate complete,
* accurate predictions. When processing the returned predictions, make sure to consider all of the
* {@link MotionEvent#getHistoricalAxisValue historical samples}.
* Send all of the events that were received from the system to {@link #record} to generate
* complete, accurate predictions from {@link #predict}. When processing the returned predictions,
* make sure to consider all of the {@link MotionEvent#getHistoricalAxisValue historical samples}.
*/
// Acts as a pass-through to the native MotionPredictor object.
// Do not store any state in this Java layer, or add any business logic here. All of the
// implementation details should go into the native MotionPredictor.
// The context / resource access must be here rather than in native layer due to the lack of the
// corresponding native API surface.
public final class MotionPredictor {
// This is a pass-through to the native MotionPredictor object (mPtr). Do not store any state or
// add any business logic here -- all of the implementation details should go into the native
// MotionPredictor (except for accessing the context/resources, which have no corresponding
// native API).
private static class RegistryHolder {
public static final NativeAllocationRegistry REGISTRY =
NativeAllocationRegistry.createMalloced(
@@ -67,49 +68,63 @@ public final class MotionPredictor {
/**
* Record a movement so that in the future, a prediction for the current gesture can be
* generated. Ensure to add all motions from the gesture of interest to generate the correct
* prediction.
* generated. Ensure to add all motions from the gesture of interest to generate correct
* predictions.
* @param event The received event
*/
public void record(@NonNull MotionEvent event) {
if (!isPredictionEnabled()) {
return;
}
nativeRecord(mPtr, event);
}
/**
* Get predicted events for all gestures that have been provided to the 'record' function.
* Get predicted events for all gestures that have been provided to {@link #record}.
* If events from multiple devices were sent to 'record', this will produce a separate
* {@link MotionEvent} for each device id. The returned list may be empty if no predictions for
* any of the added events are available.
* {@link MotionEvent} for each device. The returned list may be empty if no predictions for
* any of the added events/devices are available.
* Predictions may not reach the requested timestamp if the confidence in the prediction results
* is low.
*
* @param predictionTimeNanos The time that the prediction should target, in the
* {@link android.os.SystemClock#uptimeMillis} time base, but in nanoseconds.
*
* @return the list of predicted motion events, for each device id. Ensure to check the
* historical data in addition to the latest ({@link MotionEvent#getX getX()},
* {@link MotionEvent#getY getY()}) coordinates for smoothest prediction curves. Empty list is
* returned if predictions are not supported, or not possible for the current set of gestures.
* @return A list of predicted motion events, with at most one for each device observed by
* {@link #record}. Be sure to check the historical data in addition to the latest
* ({@link MotionEvent#getX getX()}, {@link MotionEvent#getY getY()}) coordinates for smooth
* prediction curves. An empty list is returned if predictions are not supported, or not
* possible for the current set of gestures.
*/
@NonNull
public List<MotionEvent> predict(long predictionTimeNanos) {
if (!isPredictionEnabled()) {
return Collections.emptyList();
}
return Arrays.asList(nativePredict(mPtr, predictionTimeNanos));
}
/**
* Check whether this device supports motion predictions for the given source type.
*
* @param deviceId The input device id
* @param source The source of input events
* @return True if the current device supports predictions, false otherwise.
*/
public boolean isPredictionAvailable(int deviceId, int source) {
private boolean isPredictionEnabled() {
// Device-specific override
if (!mContext.getResources().getBoolean(
com.android.internal.R.bool.config_enableMotionPrediction)) {
return false;
}
return nativeIsPredictionAvailable(mPtr, deviceId, source);
return true;
}
/**
* Check whether a device supports motion predictions for a given source type.
*
* @param deviceId The input device id.
* @param source The source of input events.
* @return True if the current device supports predictions, false otherwise.
*
* @see MotionEvent#getDeviceId
* @see MotionEvent#getSource
*/
public boolean isPredictionAvailable(int deviceId, int source) {
return isPredictionEnabled() && nativeIsPredictionAvailable(mPtr, deviceId, source);
}
private static native long nativeInitialize(int offsetNanos);

3
core/jni/Android.bp
View File

@@ -326,6 +326,7 @@ cc_library_shared {
"libdl",
"libdl_android",
"libtimeinstate",
"libtflite",
"server_configurable_flags",
"libimage_io",
"libjpegdecoder",
@@ -343,7 +344,9 @@ cc_library_shared {
header_libs: [
"bionic_libc_platform_headers",
"dnsproxyd_protocol_headers",
"flatbuffer_headers",
"libtextclassifier_hash_headers",
"tensorflow_headers",
],
runtime_libs: [
"libidmap2",

19
core/res/res/values/config.xml
View File

@@ -2383,22 +2383,13 @@
display, this value should be true. -->
<bool name="config_perDisplayFocusEnabled">false</bool>
<!-- Whether the system enables motion prediction. Only enable this after confirming that the
model works well on your device. To enable system-based prediction, set this value to true.
-->
<bool name="config_enableMotionPrediction">true</bool>
<!-- Whether to use the system motion prediction model. Only set this value to true after
confirming that the model works well on your device. -->
<bool name="config_enableMotionPrediction">false</bool>
<!-- Additional offset to use for motion prediction, in nanoseconds. A positive number indicates
that the prediction will take place further in the future. For example, suppose a
MotionEvent arrives with timestamp t=1, and the current expected presentation time is t=2.
Typically, the prediction will target the presentation time, t=2. If you'd like to make
prediction more aggressive, you could set the offset to a positive number.
Setting the offset to 1 here would mean that the prediction will be done for time t=3.
A negative number may also be provided, to make the prediction less aggressive. In general,
the offset here should represent some built-in hardware delays that may not be accounted
for by the "expected present time". See also:
https://developer.android.com/reference/android/view/
Choreographer.FrameTimeline#getExpectedPresentationTimeNanos() -->
that the prediction will take place further in the future and, in general, should represent
some built-in hardware delays that prediction should try to recover. -->
<integer name="config_motionPredictionOffsetNanos">0</integer>
<!-- Whether a software navigation bar should be shown. NOTE: in the future this may be

19
tests/Input/src/com/android/test/input/MotionPredictorTest.kt
View File

@@ -102,9 +102,9 @@ class MotionPredictorTest {
* a prediction. Here, we send 2 events to the predictor and check the returned event.
* Input:
* t = 0 x = 0 y = 0
* t = 1 x = 1 y = 2
* t = 4 x = 10 y = 20
* Output (expected):
* t = 3 x = 3 y = 6
* t = 12 x = 30 y = 60 ± error
*
* Historical data is ignored for simplicity.
*/
@@ -118,19 +118,20 @@ class MotionPredictorTest {
// ACTION_DOWN t=0 x=0 y=0
predictor.record(downEvent)
eventTime += Duration.ofMillis(1)
val moveEvent = getStylusMotionEvent(eventTime, ACTION_MOVE, /*x=*/1f, /*y=*/2f)
eventTime += Duration.ofMillis(4)
val moveEvent = getStylusMotionEvent(eventTime, ACTION_MOVE, /*x=*/10f, /*y=*/20f)
// ACTION_MOVE t=1 x=1 y=2
predictor.record(moveEvent)
val predicted = predictor.predict(Duration.ofMillis(2).toNanos())
val predicted = predictor.predict(Duration.ofMillis(8).toNanos())
assertEquals(1, predicted.size)
val event = predicted[0]
assertNotNull(event)
// Prediction will happen for t=3 (2 + 1, since offset is 1 and present time is 2)
assertEquals(3, event.eventTime)
assertEquals(3f, event.x, /*delta=*/0.001f)
assertEquals(6f, event.y, /*delta=*/0.001f)
// Prediction will happen for t=12 (since it is the next input interval after the requested
// time, 8, plus the model offset, 1).
assertEquals(12, event.eventTime)
assertEquals(30f, event.x, /*delta=*/5f)
assertEquals(60f, event.y, /*delta=*/15f)
}
}