From bbc1459cf08b411cd9259b709622d784da1b14a9 Mon Sep 17 00:00:00 2001 From: Nicolo' Mazzucato Date: Mon, 15 Mar 2021 13:45:31 +0000 Subject: [PATCH] Always call onHandleAssist with AssistState containing at least activity id With this change, even if clients are not requesting any assist data (e.g. VoiceInteractionSession.showSession is called with 0 as flags), Activity's `onHandleAssist(AssistState)` is always called with non-null ActivityId. If no other data were requested, AssistContent, AssistData and AssistStructure inside VoiceInteractionSession.AssistState are null. The previous behaviour was that if no assistData were requested, Activity's `onHandleAssist` would not have been called. This new behaviour saves time to clients interested only in ActivityIds, because no activities methods need to be called (to get AssistData and AssistContent). We are not adding a new flag "SHOW_WITH_ACTIVITY_ID" because this ActivityId is always non-null in ActivityState, and it's cheap to get it. Bug: 178020517 Test: atest CtsAssistTestCases Test: atest CtsVoiceInteractionTestCases Change-Id: Icb46a928c8508a4bd355622df78f9371c21fd0e6 --- .../voice/VoiceInteractionSession.java | 25 +++++--- .../wm/ActivityTaskManagerInternal.java | 7 +- .../server/wm/ActivityTaskManagerService.java | 3 +- .../server/wm/RootWindowContainer.java | 16 +++-- .../VoiceInteractionManagerServiceImpl.java | 23 +++++-- .../VoiceInteractionSessionConnection.java | 64 +++++++++++++++---- 6 files changed, 100 insertions(+), 38 deletions(-) diff --git a/core/java/android/service/voice/VoiceInteractionSession.java b/core/java/android/service/voice/VoiceInteractionSession.java index c1b66c7b446ed..ad09a48cf4c39 100644 --- a/core/java/android/service/voice/VoiceInteractionSession.java +++ b/core/java/android/service/voice/VoiceInteractionSession.java @@ -1645,7 +1645,7 @@ public class VoiceInteractionSession implements KeyEvent.Callback, ComponentCall /** * Called to receive data from the application that the user was currently viewing when - * an assist session is started. If the original show request did not specify +- * an assist session is started. If the original show request did not specify * {@link #SHOW_WITH_ASSIST}, this method will not be called. * * @param data Arbitrary data supplied by the app through @@ -1670,8 +1670,9 @@ public class VoiceInteractionSession implements KeyEvent.Callback, ComponentCall /** * Called to receive data from the application that the user was currently viewing when - * an assist session is started. If the original show request did not specify - * {@link #SHOW_WITH_ASSIST}, this method will not be called. + * an assist session is started. If the original show request did not specify + * {@link #SHOW_WITH_ASSIST}, {@link AssistState} parameter will only provide + * {@link ActivityId}. * *

This method is called for all activities along with an index and count that indicates * which activity the data is for. {@code index} will be between 0 and {@code count}-1 and @@ -1685,7 +1686,10 @@ public class VoiceInteractionSession implements KeyEvent.Callback, ComponentCall * @param state The state object capturing the state of an activity. */ public void onHandleAssist(@NonNull AssistState state) { - if (state.getIndex() == 0) { + if (state.getAssistData() == null && state.getAssistStructure() == null + && state.getAssistContent() == null) { + return; + } else if (state.getIndex() == 0) { onHandleAssist(state.getAssistData(), state.getAssistStructure(), state.getAssistContent()); } else { @@ -2028,7 +2032,8 @@ public class VoiceInteractionSession implements KeyEvent.Callback, ComponentCall /** * @return Arbitrary data supplied by the app through * {@link android.app.Activity#onProvideAssistData Activity.onProvideAssistData}. - * May be null if assist data has been disabled by the user or device policy. + * May be null if assist data has been disabled by the user or device policy; will be null + * if the original show request did not specify {@link #SHOW_WITH_ASSIST}. */ public @Nullable Bundle getAssistData() { return mData; @@ -2037,7 +2042,8 @@ public class VoiceInteractionSession implements KeyEvent.Callback, ComponentCall /** * @return If available, the structure definition of all windows currently * displayed by the app. May be null if assist data has been disabled by the user - * or device policy; will be an empty stub if the application has disabled assist + * or device policy; will be null if the original show request did not specify + * {@link #SHOW_WITH_ASSIST}; will be an empty stub if the application has disabled assist * by marking its window as secure. */ public @Nullable AssistStructure getAssistStructure() { @@ -2047,9 +2053,10 @@ public class VoiceInteractionSession implements KeyEvent.Callback, ComponentCall /** * @return Additional content data supplied by the app through * {@link android.app.Activity#onProvideAssistContent Activity.onProvideAssistContent}. - * May be null if assist data has been disabled by the user or device policy; will - * not be automatically filled in with data from the app if the app has marked its - * window as secure. + * May be null if assist data has been disabled by the user or device policy; will be null + * if the original show request did not specify {@link #SHOW_WITH_ASSIST}. Will not be + * automatically filled in with data from the app if the app has marked its window as + * secure. */ public @Nullable AssistContent getAssistContent() { return mContent; diff --git a/services/core/java/com/android/server/wm/ActivityTaskManagerInternal.java b/services/core/java/com/android/server/wm/ActivityTaskManagerInternal.java index e858fe1034b1b..060323c34b40e 100644 --- a/services/core/java/com/android/server/wm/ActivityTaskManagerInternal.java +++ b/services/core/java/com/android/server/wm/ActivityTaskManagerInternal.java @@ -32,6 +32,7 @@ import android.os.Bundle; import android.os.IBinder; import android.os.RemoteException; import android.service.voice.IVoiceInteractionSession; +import android.util.Pair; import android.util.proto.ProtoOutputStream; import android.window.TaskSnapshot; @@ -162,10 +163,10 @@ public abstract class ActivityTaskManagerInternal { IVoiceInteractor mInteractor); /** - * Returns the top activity from each of the currently visible root tasks. The first entry - * will be the focused activity. + * Returns the top activity from each of the currently visible root tasks, and the related task + * id. The first entry will be the focused activity. */ - public abstract List getTopVisibleActivities(); + public abstract List> getTopVisibleActivities(); /** * Returns whether {@code uid} has any resumed activity. diff --git a/services/core/java/com/android/server/wm/ActivityTaskManagerService.java b/services/core/java/com/android/server/wm/ActivityTaskManagerService.java index 52d110c95e36a..09f5c937928c2 100644 --- a/services/core/java/com/android/server/wm/ActivityTaskManagerService.java +++ b/services/core/java/com/android/server/wm/ActivityTaskManagerService.java @@ -215,6 +215,7 @@ import android.text.format.TimeMigrationUtils; import android.util.ArrayMap; import android.util.ArraySet; import android.util.Log; +import android.util.Pair; import android.util.Slog; import android.util.SparseArray; import android.util.TimeUtils; @@ -5074,7 +5075,7 @@ public class ActivityTaskManagerService extends IActivityTaskManager.Stub { } @Override - public List getTopVisibleActivities() { + public List> getTopVisibleActivities() { synchronized (mGlobalLock) { return mRootWindowContainer.getTopVisibleActivities(); } diff --git a/services/core/java/com/android/server/wm/RootWindowContainer.java b/services/core/java/com/android/server/wm/RootWindowContainer.java index 422d4e79e259e..857217fb19e2a 100644 --- a/services/core/java/com/android/server/wm/RootWindowContainer.java +++ b/services/core/java/com/android/server/wm/RootWindowContainer.java @@ -1814,11 +1814,11 @@ class RootWindowContainer extends WindowContainer } /** - * @return a list of activities which are the top ones in each visible root task. The first - * entry will be the focused activity. + * @return a list of pairs, containing activities and their task id which are the top ones in + * each visible root task. The first entry will be the focused activity. */ - List getTopVisibleActivities() { - final ArrayList topActivityTokens = new ArrayList<>(); + List> getTopVisibleActivities() { + final ArrayList> topVisibleActivities = new ArrayList<>(); final Task topFocusedRootTask = getTopDisplayFocusedRootTask(); // Traverse all displays. forAllRootTasks(rootTask -> { @@ -1826,15 +1826,17 @@ class RootWindowContainer extends WindowContainer if (rootTask.shouldBeVisible(null /* starting */)) { final ActivityRecord top = rootTask.getTopNonFinishingActivity(); if (top != null) { + Pair visibleActivity = new Pair<>(top.appToken, + top.getTask().mTaskId); if (rootTask == topFocusedRootTask) { - topActivityTokens.add(0, top.appToken); + topVisibleActivities.add(0, visibleActivity); } else { - topActivityTokens.add(top.appToken); + topVisibleActivities.add(visibleActivity); } } } }); - return topActivityTokens; + return topVisibleActivities; } @Nullable diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerServiceImpl.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerServiceImpl.java index 58616104755d1..05573f1d7d607 100644 --- a/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerServiceImpl.java +++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerServiceImpl.java @@ -50,6 +50,7 @@ import android.service.voice.IVoiceInteractionSession; import android.service.voice.VoiceInteractionService; import android.service.voice.VoiceInteractionServiceInfo; import android.system.OsConstants; +import android.util.Pair; import android.util.PrintWriterPrinter; import android.util.Slog; import android.view.IWindowManager; @@ -186,17 +187,27 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne mSessionComponentName, mUser, mContext, this, mInfo.getServiceInfo().applicationInfo.uid, mHandler); } - List activityTokens = null; + List> allVisibleActivities = + LocalServices.getService(ActivityTaskManagerInternal.class) + .getTopVisibleActivities(); + + List> visibleActivities = null; if (activityToken != null) { - activityTokens = new ArrayList<>(); - activityTokens.add(activityToken); + visibleActivities = new ArrayList(); + int activitiesCount = allVisibleActivities.size(); + for (int i = 0; i < activitiesCount; i++) { + if (allVisibleActivities.get(i).first == activityToken) { + visibleActivities.add( + new Pair<>(activityToken, allVisibleActivities.get(i).second)); + break; + } + } } else { - // Let's get top activities from all visible stacks - activityTokens = LocalServices.getService(ActivityTaskManagerInternal.class) + visibleActivities = LocalServices.getService(ActivityTaskManagerInternal.class) .getTopVisibleActivities(); } return mActiveSession.showLocked(args, flags, mDisabledShowContext, showCallback, - activityTokens); + visibleActivities); } public void getActiveServiceSupportedActions(List commands, diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionSessionConnection.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionSessionConnection.java index 84f4f6a017d1e..428d342a80c95 100644 --- a/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionSessionConnection.java +++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionSessionConnection.java @@ -56,6 +56,7 @@ import android.service.voice.IVoiceInteractionSession; import android.service.voice.IVoiceInteractionSessionService; import android.service.voice.VoiceInteractionService; import android.service.voice.VoiceInteractionSession; +import android.util.Pair; import android.util.Slog; import android.view.IWindowManager; @@ -190,7 +191,8 @@ final class VoiceInteractionSessionConnection implements ServiceConnection, } public boolean showLocked(Bundle args, int flags, int disabledContext, - IVoiceInteractionSessionShowCallback showCallback, List topActivities) { + IVoiceInteractionSessionShowCallback showCallback, + List> topActivities) { if (mBound) { if (!mFullyBound) { mFullyBound = mContext.bindServiceAsUser(mBindIntent, mFullConnection, @@ -205,17 +207,29 @@ final class VoiceInteractionSessionConnection implements ServiceConnection, mShowFlags = flags; disabledContext |= getUserDisabledShowContextLocked(); - mAssistDataRequester.requestAssistData(topActivities, - (flags & VoiceInteractionSession.SHOW_WITH_ASSIST) != 0, - (flags & VoiceInteractionSession.SHOW_WITH_SCREENSHOT) != 0, - (disabledContext & VoiceInteractionSession.SHOW_WITH_ASSIST) == 0, - (disabledContext & VoiceInteractionSession.SHOW_WITH_SCREENSHOT) == 0, - mCallingUid, mSessionComponentName.getPackageName()); - boolean needDisclosure = mAssistDataRequester.getPendingDataCount() > 0 - || mAssistDataRequester.getPendingScreenshotCount() > 0; - if (needDisclosure && AssistUtils.shouldDisclose(mContext, mSessionComponentName)) { - mHandler.post(mShowAssistDisclosureRunnable); + boolean fetchData = (flags & VoiceInteractionSession.SHOW_WITH_ASSIST) != 0; + boolean fetchScreenshot = (flags & VoiceInteractionSession.SHOW_WITH_SCREENSHOT) != 0; + boolean assistDataRequestNeeded = fetchData || fetchScreenshot; + + if (assistDataRequestNeeded) { + int topActivitiesCount = topActivities.size(); + final ArrayList topActivitiesToken = new ArrayList<>(topActivitiesCount); + for (int i = 0; i < topActivitiesCount; i++) { + topActivitiesToken.add(topActivities.get(i).first); + } + mAssistDataRequester.requestAssistData(topActivitiesToken, + fetchData, + fetchScreenshot, + (disabledContext & VoiceInteractionSession.SHOW_WITH_ASSIST) == 0, + (disabledContext & VoiceInteractionSession.SHOW_WITH_SCREENSHOT) == 0, + mCallingUid, mSessionComponentName.getPackageName()); + + boolean needDisclosure = mAssistDataRequester.getPendingDataCount() > 0 + || mAssistDataRequester.getPendingScreenshotCount() > 0; + if (needDisclosure && AssistUtils.shouldDisclose(mContext, mSessionComponentName)) { + mHandler.post(mShowAssistDisclosureRunnable); + } } if (mSession != null) { try { @@ -224,7 +238,11 @@ final class VoiceInteractionSessionConnection implements ServiceConnection, mShowFlags = 0; } catch (RemoteException e) { } - mAssistDataRequester.processPendingAssistData(); + if (assistDataRequestNeeded) { + mAssistDataRequester.processPendingAssistData(); + } else { + doHandleAssistWithoutData(topActivities); + } } else if (showCallback != null) { mPendingShowCallbacks.add(showCallback); } @@ -240,6 +258,28 @@ final class VoiceInteractionSessionConnection implements ServiceConnection, return false; } + private void doHandleAssistWithoutData(List> topActivities) { + final int activityCount = topActivities.size(); + for (int i = 0; i < activityCount; i++) { + final Pair topActivity = topActivities.get(i); + final IBinder activityId = topActivity.first; + final int taskId = topActivity.second; + final int activityIndex = i; + try { + mSession.handleAssist( + taskId, + activityId, + /* assistData = */ null, + /* assistStructure = */ null, + /* assistContent = */ null, + activityIndex, + activityCount); + } catch (RemoteException e) { + // Ignore + } + } + } + @Override public boolean canHandleReceivedAssistDataLocked() { return mSession != null;