Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge changes from topic "back_gesture_model_qpr" into rvc-qpr-dev

Browse Source 
* changes:
  Add flag to get the name of the Model file from the flag in case we want to try out different models. In addition, we should not log touches in the middle of the screen for logging so moving the block at the very beginning.
  Add package_name to log for debugging the back gesture model.
  Add log entries for the results for the ML model for the back gesture.
  Use ML model for the Back Gesture in EdgeBackGestureHandler.
This commit is contained in:
Winson Chung
2020-11-17 03:07:30 +00:00
committed by Android (Google) Code Review
parent 9649d433d0 0ed68db06c
commit 898602efc6
5 changed files with 220 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
cmds/statsd/src/atoms.proto
View File

@@ -3038,6 +3038,9 @@ message BackGesture {
optional int32 end_y = 7; // Y coordinate for ACTION_MOVE event.
optional int32 left_boundary = 8; // left edge width + left inset
optional int32 right_boundary = 9; // screen width - (right edge width + right inset)
// The score between 0 and 1 which is the prediction output for the Back Gesture model.
optional float ml_model_score = 10;
optional string package_name = 11; // The name of the top 100 most used package by all users.
enum WindowHorizontalLocation {
DEFAULT_LOCATION = 0;

16
core/java/com/android/internal/config/sysui/SystemUiDeviceConfigFlags.java
View File

@@ -428,6 +428,22 @@ public final class SystemUiDeviceConfigFlags {
*/
public static final String SCREENSHOT_KEYCHORD_DELAY = "screenshot_keychord_delay";
/**
* (boolean) Whether to use an ML model for the Back Gesture.
*/
public static final String USE_BACK_GESTURE_ML_MODEL = "use_back_gesture_ml_model";
/**
* (string) The name of the ML model for Back Gesture.
*/
public static final String BACK_GESTURE_ML_MODEL_NAME = "back_gesture_ml_model_name";
/**
* (float) Threshold for Back Gesture ML model prediction.
*/
public static final String BACK_GESTURE_ML_MODEL_THRESHOLD = "back_gesture_ml_model_threshold";
private SystemUiDeviceConfigFlags() {
}
}

11
packages/SystemUI/src/com/android/systemui/SystemUIFactory.java
View File

@@ -18,6 +18,7 @@ package com.android.systemui;
import android.annotation.NonNull;
import android.content.Context;
import android.content.res.AssetManager;
import android.content.res.Resources;
import android.os.Handler;
import android.os.Looper;
@@ -39,6 +40,7 @@ import com.android.systemui.screenshot.ScreenshotNotificationSmartActionsProvide
import com.android.systemui.statusbar.NotificationListener;
import com.android.systemui.statusbar.NotificationMediaManager;
import com.android.systemui.statusbar.notification.NotificationWakeUpCoordinator;
import com.android.systemui.statusbar.phone.BackGestureTfClassifierProvider;
import com.android.systemui.statusbar.phone.DozeParameters;
import com.android.systemui.statusbar.phone.KeyguardBouncer;
import com.android.systemui.statusbar.phone.KeyguardBypassController;
@@ -182,4 +184,13 @@ public class SystemUIFactory {
return mContext;
}
}
/**
* Creates an instance of BackGestureTfClassifierProvider.
* This method is overridden in vendor specific implementation of Sys UI.
*/
public BackGestureTfClassifierProvider createBackGestureTfClassifierProvider(
AssetManager am, String modelName) {
return new BackGestureTfClassifierProvider();
}
}

66
packages/SystemUI/src/com/android/systemui/statusbar/phone/BackGestureTfClassifierProvider.java Normal file
View File

@@ -0,0 +1,66 @@
/*
* Copyright (C) 2020 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.systemui.statusbar.phone;
import android.content.res.AssetManager;
import java.util.HashMap;
import java.util.Map;
/**
* This class can be overridden by a vendor-specific sys UI implementation,
* in order to provide classification models for the Back Gesture.
*/
public class BackGestureTfClassifierProvider {
private static final String TAG = "BackGestureTfClassifierProvider";
/**
* Default implementation that returns an empty map.
* This method is overridden in vendor-specific Sys UI implementation.
*
* @param am An AssetManager to get the vocab file.
*/
public Map<String, Integer> loadVocab(AssetManager am) {
return new HashMap<String, Integer>();
}
/**
* This method is overridden in vendor-specific Sys UI implementation.
*
* @param featuresVector List of input features.
*
*/
public float predict(Object[] featuresVector) {
return -1;
}
/**
* Interpreter owns resources. This method releases the resources after
* use to avoid memory leak.
* This method is overridden in vendor-specific Sys UI implementation.
*
*/
public void release() {}
/**
* Returns whether to use the ML model for Back Gesture.
* This method is overridden in vendor-specific Sys UI implementation.
*
*/
public boolean isActive() {
return false;
}
}

132
packages/SystemUI/src/com/android/systemui/statusbar/phone/EdgeBackGestureHandler.java
View File

@@ -56,6 +56,7 @@ import com.android.internal.config.sysui.SystemUiDeviceConfigFlags;
import com.android.internal.policy.GestureNavigationSettingsObserver;
import com.android.systemui.Dependency;
import com.android.systemui.R;
import com.android.systemui.SystemUIFactory;
import com.android.systemui.broadcast.BroadcastDispatcher;
import com.android.systemui.bubbles.BubbleController;
import com.android.systemui.model.SysUiState;
@@ -76,6 +77,7 @@ import com.android.systemui.tracing.nano.SystemUiTraceProto;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Executor;
/**
@@ -117,8 +119,33 @@ public class EdgeBackGestureHandler extends CurrentUserTracker implements Displa
public void onTaskStackChanged() {
mGestureBlockingActivityRunning = isGestureBlockingActivityRunning();
}
@Override
public void onTaskCreated(int taskId, ComponentName componentName) {
if (componentName != null) {
mPackageName = componentName.getPackageName();
} else {
mPackageName = "_UNKNOWN";
}
}
};
private DeviceConfig.OnPropertiesChangedListener mOnPropertiesChangedListener =
new DeviceConfig.OnPropertiesChangedListener() {
@Override
public void onPropertiesChanged(DeviceConfig.Properties properties) {
if (DeviceConfig.NAMESPACE_SYSTEMUI.equals(properties.getNamespace())
&& (properties.getKeyset().contains(
SystemUiDeviceConfigFlags.BACK_GESTURE_ML_MODEL_THRESHOLD)
|| properties.getKeyset().contains(
SystemUiDeviceConfigFlags.USE_BACK_GESTURE_ML_MODEL)
|| properties.getKeyset().contains(
SystemUiDeviceConfigFlags.BACK_GESTURE_ML_MODEL_NAME))) {
updateMLModelState();
}
}
};
private final Context mContext;
private final OverviewProxyService mOverviewProxyService;
private final Runnable mStateChangeCallback;
@@ -173,6 +200,14 @@ public class EdgeBackGestureHandler extends CurrentUserTracker implements Displa
private int mRightInset;
private int mSysUiFlags;
// For Tf-Lite model.
private BackGestureTfClassifierProvider mBackGestureTfClassifierProvider;
private Map<String, Integer> mVocab;
private boolean mUseMLModel;
private float mMLModelThreshold;
private String mPackageName;
private float mMLResults;
private final GestureNavigationSettingsObserver mGestureNavigationSettingsObserver;
private final NavigationEdgeBackPlugin.BackCallback mBackCallback =
@@ -230,7 +265,6 @@ public class EdgeBackGestureHandler extends CurrentUserTracker implements Displa
Log.e(TAG, "Failed to add gesture blocking activities", e);
}
}
mLongPressTimeout = Math.min(MAX_LONG_PRESS_TIMEOUT,
ViewConfiguration.getLongPressTimeout());
@@ -344,6 +378,7 @@ public class EdgeBackGestureHandler extends CurrentUserTracker implements Displa
mContext.getSystemService(DisplayManager.class).unregisterDisplayListener(this);
mPluginManager.removePluginListener(this);
ActivityManagerWrapper.getInstance().unregisterTaskStackListener(mTaskStackListener);
DeviceConfig.removeOnPropertiesChangedListener(mOnPropertiesChangedListener);
try {
WindowManagerGlobal.getWindowManagerService()
@@ -359,6 +394,9 @@ public class EdgeBackGestureHandler extends CurrentUserTracker implements Displa
mContext.getSystemService(DisplayManager.class).registerDisplayListener(this,
mContext.getMainThreadHandler());
ActivityManagerWrapper.getInstance().registerTaskStackListener(mTaskStackListener);
DeviceConfig.addOnPropertiesChangedListener(DeviceConfig.NAMESPACE_SYSTEMUI,
runnable -> (mContext.getMainThreadHandler()).post(runnable),
mOnPropertiesChangedListener);
try {
WindowManagerGlobal.getWindowManagerService()
@@ -379,6 +417,8 @@ public class EdgeBackGestureHandler extends CurrentUserTracker implements Displa
mPluginManager.addPluginListener(
this, NavigationEdgeBackPlugin.class, /*allowMultiple=*/ false);
}
// Update the ML model resources.
updateMLModelState();
}
@Override
@@ -431,12 +471,72 @@ public class EdgeBackGestureHandler extends CurrentUserTracker implements Displa
}
}
private void updateMLModelState() {
boolean newState = mIsEnabled && DeviceConfig.getBoolean(DeviceConfig.NAMESPACE_SYSTEMUI,
SystemUiDeviceConfigFlags.USE_BACK_GESTURE_ML_MODEL, false);
if (newState == mUseMLModel) {
return;
}
if (newState) {
String mlModelName = DeviceConfig.getString(DeviceConfig.NAMESPACE_SYSTEMUI,
SystemUiDeviceConfigFlags.BACK_GESTURE_ML_MODEL_NAME, "backgesture");
mBackGestureTfClassifierProvider = SystemUIFactory.getInstance()
.createBackGestureTfClassifierProvider(mContext.getAssets(), mlModelName);
mMLModelThreshold = DeviceConfig.getFloat(DeviceConfig.NAMESPACE_SYSTEMUI,
SystemUiDeviceConfigFlags.BACK_GESTURE_ML_MODEL_THRESHOLD, 0.9f);
if (mBackGestureTfClassifierProvider.isActive()) {
mVocab = mBackGestureTfClassifierProvider.loadVocab(mContext.getAssets());
mUseMLModel = true;
return;
}
}
mUseMLModel = false;
if (mBackGestureTfClassifierProvider != null) {
mBackGestureTfClassifierProvider.release();
mBackGestureTfClassifierProvider = null;
}
}
private float getBackGesturePredictionsCategory(int x, int y) {
if (!mVocab.containsKey(mPackageName)) {
return -1;
}
int distanceFromEdge;
int location;
if (x <= mDisplaySize.x / 2.0) {
location = 1; // left
distanceFromEdge = x;
} else {
location = 2; // right
distanceFromEdge = mDisplaySize.x - x;
}
Object[] featuresVector = {
new long[]{(long) mDisplaySize.x},
new long[]{(long) distanceFromEdge},
new long[]{(long) location},
new long[]{(long) mVocab.get(mPackageName)},
new long[]{(long) y},
};
mMLResults = mBackGestureTfClassifierProvider.predict(featuresVector);
if (mMLResults == -1) return -1;
return mMLResults >= mMLModelThreshold ? 1 : 0;
}
private boolean isWithinTouchRegion(int x, int y) {
boolean withinRange = false;
float results = -1;
// Disallow if we are in the bottom gesture area
if (y >= (mDisplaySize.y - mBottomGestureHeight)) {
return false;
}
// If the point is way too far (twice the margin), it is
// not interesting to us for logging purposes, nor we
// should process it. Simply return false and keep
@@ -446,11 +546,15 @@ public class EdgeBackGestureHandler extends CurrentUserTracker implements Displa
return false;
}
// Denotes whether we should proceed with the gesture.
// Even if it is false, we may want to log it assuming
// it is not invalid due to exclusion.
boolean withinRange = x <= mEdgeWidthLeft + mLeftInset
|| x >= (mDisplaySize.x - mEdgeWidthRight - mRightInset);
if (mUseMLModel && (results = getBackGesturePredictionsCategory(x, y)) != -1) {
withinRange = results == 1 ? true : false;
} else {
// Denotes whether we should proceed with the gesture.
// Even if it is false, we may want to log it assuming
// it is not invalid due to exclusion.
withinRange = x <= mEdgeWidthLeft + mLeftInset
|| x >= (mDisplaySize.x - mEdgeWidthRight - mRightInset);
}
// Always allow if the user is in a transient sticky immersive state
if (mIsNavBarShownTransiently) {
@@ -493,6 +597,11 @@ public class EdgeBackGestureHandler extends CurrentUserTracker implements Displa
return;
}
mLogGesture = false;
String logPackageName = "";
// Due to privacy, only top 100 most used apps by all users can be logged.
if (mUseMLModel && mVocab.containsKey(mPackageName) && mVocab.get(mPackageName) < 100) {
logPackageName = mPackageName;
}
SysUiStatsLog.write(SysUiStatsLog.BACK_GESTURE_REPORTED_REPORTED, backType,
(int) mDownPoint.y, mIsOnLeftEdge
? SysUiStatsLog.BACK_GESTURE__X_LOCATION__LEFT
@@ -500,7 +609,8 @@ public class EdgeBackGestureHandler extends CurrentUserTracker implements Displa
(int) mDownPoint.x, (int) mDownPoint.y,
(int) mEndPoint.x, (int) mEndPoint.y,
mEdgeWidthLeft + mLeftInset,
mDisplaySize.x - (mEdgeWidthRight + mRightInset));
mDisplaySize.x - (mEdgeWidthRight + mRightInset),
mUseMLModel ? mMLResults : -2, logPackageName);
}
private void onMotionEvent(MotionEvent ev) {
@@ -509,6 +619,7 @@ public class EdgeBackGestureHandler extends CurrentUserTracker implements Displa
// Verify if this is in within the touch region and we aren't in immersive mode, and
// either the bouncer is showing or the notification panel is hidden
mIsOnLeftEdge = ev.getX() <= mEdgeWidthLeft + mLeftInset;
mMLResults = 0;
mLogGesture = false;
mInRejectedExclusion = false;
mAllowGesture = !mDisabledForQuickstep && mIsBackGestureAllowed
@@ -648,6 +759,11 @@ public class EdgeBackGestureHandler extends CurrentUserTracker implements Displa
ActivityManager.RunningTaskInfo runningTask =
ActivityManagerWrapper.getInstance().getRunningTask();
ComponentName topActivity = runningTask == null ? null : runningTask.topActivity;
if (topActivity != null) {
mPackageName = topActivity.getPackageName();
} else {
mPackageName = "_UNKNOWN";
}
return topActivity != null && mGestureBlockingActivities.contains(topActivity);
}