From eedc06b4636462dd60fb74f841979566b559698b Mon Sep 17 00:00:00 2001 From: Peter Li Date: Fri, 13 May 2022 21:15:45 +0800 Subject: [PATCH] Inform Assistant visible activity when the stage of activity lifecycle is invoked. By the original design, we register the activity event in the ActivityManagerService. After getting the activity event, we will query current visible activities and report them to Assistant. Assistant will start to get direct actions of those visible activities. But sometimes they can not get the direct actions due to the stage of activity lifecycle before onStart(). We change the logic to make sure that the stage of visible activity that we reported to the Assistant is after onStart(). Bug: 235296082 Test: atest VoiceInteractionSessionVisibleActivityTest Test: atest android.app.activity.ActivityThreadClientTest Test: atest android.view.DisplayTest Test: atest android.window.ConfigurationHelperTest Test: atest android.window.SizeConfigurationBucketsTest Change-Id: Ie439ff3706dab35e875c1eb995a67fa73aec4d76 (cherry picked from commit 29bb3a3428fcd272f8ec236f14354b9ec6c28f3e) Merged-In: Ie439ff3706dab35e875c1eb995a67fa73aec4d76 --- core/java/android/app/Activity.java | 38 +++ .../android/app/ActivityManagerInternal.java | 5 +- .../voice/VoiceInteractionSession.java | 22 ++ .../app/IVoiceInteractionManagerService.aidl | 25 +- .../server/am/ActivityManagerService.java | 11 +- .../VoiceInteractionManagerService.java | 23 +- .../VoiceInteractionManagerServiceImpl.java | 20 +- .../VoiceInteractionSessionConnection.java | 233 +++++++++++++----- 8 files changed, 287 insertions(+), 90 deletions(-) diff --git a/core/java/android/app/Activity.java b/core/java/android/app/Activity.java index 855366abae4ec..81c3e8957cd1e 100644 --- a/core/java/android/app/Activity.java +++ b/core/java/android/app/Activity.java @@ -87,10 +87,12 @@ import android.os.Parcelable; import android.os.PersistableBundle; import android.os.Process; import android.os.RemoteException; +import android.os.ServiceManager; import android.os.ServiceManager.ServiceNotFoundException; import android.os.StrictMode; import android.os.Trace; import android.os.UserHandle; +import android.service.voice.VoiceInteractionSession; import android.text.Selection; import android.text.SpannableStringBuilder; import android.text.TextUtils; @@ -154,6 +156,7 @@ import android.window.WindowOnBackInvokedDispatcher; import com.android.internal.R; import com.android.internal.annotations.GuardedBy; import com.android.internal.annotations.VisibleForTesting; +import com.android.internal.app.IVoiceInteractionManagerService; import com.android.internal.app.IVoiceInteractor; import com.android.internal.app.ToolbarActionBar; import com.android.internal.app.WindowDecorActionBar; @@ -1601,6 +1604,25 @@ public class Activity extends ContextThemeWrapper return callbacks; } + private void notifyVoiceInteractionManagerServiceActivityEvent( + @VoiceInteractionSession.VoiceInteractionActivityEventType int type) { + + final IVoiceInteractionManagerService service = + IVoiceInteractionManagerService.Stub.asInterface( + ServiceManager.getService(Context.VOICE_INTERACTION_MANAGER_SERVICE)); + if (service == null) { + Log.w(TAG, "notifyVoiceInteractionManagerServiceActivityEvent: Can not get " + + "VoiceInteractionManagerService"); + return; + } + + try { + service.notifyActivityEventChanged(mToken, type); + } catch (RemoteException e) { + // Empty + } + } + /** * Called when the activity is starting. This is where most initialization * should go: calling {@link #setContentView(int)} to inflate the @@ -1876,6 +1898,9 @@ public class Activity extends ContextThemeWrapper mCalled = true; notifyContentCaptureManagerIfNeeded(CONTENT_CAPTURE_START); + + notifyVoiceInteractionManagerServiceActivityEvent( + VoiceInteractionSession.VOICE_INTERACTION_ACTIVITY_EVENT_START); } /** @@ -2019,6 +2044,12 @@ public class Activity extends ContextThemeWrapper final Window win = getWindow(); if (win != null) win.makeActive(); if (mActionBar != null) mActionBar.setShowHideAnimationEnabled(true); + + // Because the test case "com.android.launcher3.jank.BinderTests#testPressHome" doesn't + // allow any binder call in onResume, we call this method in onPostResume. + notifyVoiceInteractionManagerServiceActivityEvent( + VoiceInteractionSession.VOICE_INTERACTION_ACTIVITY_EVENT_RESUME); + mCalled = true; } @@ -2394,6 +2425,10 @@ public class Activity extends ContextThemeWrapper getAutofillClientController().onActivityPaused(); notifyContentCaptureManagerIfNeeded(CONTENT_CAPTURE_PAUSE); + + notifyVoiceInteractionManagerServiceActivityEvent( + VoiceInteractionSession.VOICE_INTERACTION_ACTIVITY_EVENT_PAUSE); + mCalled = true; } @@ -2623,6 +2658,9 @@ public class Activity extends ContextThemeWrapper getAutofillClientController().onActivityStopped(mIntent, mChangingConfigurations); mEnterAnimationComplete = false; + + notifyVoiceInteractionManagerServiceActivityEvent( + VoiceInteractionSession.VOICE_INTERACTION_ACTIVITY_EVENT_STOP); } /** diff --git a/core/java/android/app/ActivityManagerInternal.java b/core/java/android/app/ActivityManagerInternal.java index f2ea060b16946..c95a7deba61fa 100644 --- a/core/java/android/app/ActivityManagerInternal.java +++ b/core/java/android/app/ActivityManagerInternal.java @@ -731,10 +731,9 @@ public abstract class ActivityManagerInternal { */ public interface VoiceInteractionManagerProvider { /** - * Notifies the service when a high-level activity event has been changed, for example, - * an activity was resumed or stopped. + * Notifies the service when an activity is destroyed. */ - void notifyActivityEventChanged(); + void notifyActivityDestroyed(IBinder activityToken); } /** diff --git a/core/java/android/service/voice/VoiceInteractionSession.java b/core/java/android/service/voice/VoiceInteractionSession.java index df727e9b7d3dc..48f732ec9f124 100644 --- a/core/java/android/service/voice/VoiceInteractionSession.java +++ b/core/java/android/service/voice/VoiceInteractionSession.java @@ -19,6 +19,7 @@ package android.service.voice; import static android.view.ViewGroup.LayoutParams.MATCH_PARENT; import android.annotation.CallbackExecutor; +import android.annotation.IntDef; import android.annotation.IntRange; import android.annotation.NonNull; import android.annotation.Nullable; @@ -71,6 +72,8 @@ import com.android.internal.util.function.pooled.PooledLambda; import java.io.FileDescriptor; import java.io.PrintWriter; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; import java.lang.ref.WeakReference; import java.util.ArrayList; import java.util.Collections; @@ -143,6 +146,25 @@ public class VoiceInteractionSession implements KeyEvent.Callback, ComponentCall */ public static final int SHOW_SOURCE_AUTOMOTIVE_SYSTEM_UI = 1 << 7; + /** @hide */ + public static final int VOICE_INTERACTION_ACTIVITY_EVENT_START = 1; + /** @hide */ + public static final int VOICE_INTERACTION_ACTIVITY_EVENT_RESUME = 2; + /** @hide */ + public static final int VOICE_INTERACTION_ACTIVITY_EVENT_PAUSE = 3; + /** @hide */ + public static final int VOICE_INTERACTION_ACTIVITY_EVENT_STOP = 4; + + /** @hide */ + @IntDef(prefix = { "VOICE_INTERACTION_ACTIVITY_EVENT_" }, value = { + VOICE_INTERACTION_ACTIVITY_EVENT_START, + VOICE_INTERACTION_ACTIVITY_EVENT_RESUME, + VOICE_INTERACTION_ACTIVITY_EVENT_PAUSE, + VOICE_INTERACTION_ACTIVITY_EVENT_STOP + }) + @Retention(RetentionPolicy.SOURCE) + public @interface VoiceInteractionActivityEventType{} + final Context mContext; final HandlerCaller mHandlerCaller; diff --git a/core/java/com/android/internal/app/IVoiceInteractionManagerService.aidl b/core/java/com/android/internal/app/IVoiceInteractionManagerService.aidl index 681693b1dbade..bd51f12539e83 100644 --- a/core/java/com/android/internal/app/IVoiceInteractionManagerService.aidl +++ b/core/java/com/android/internal/app/IVoiceInteractionManagerService.aidl @@ -18,6 +18,8 @@ package com.android.internal.app; import android.content.ComponentName; import android.content.Intent; +import android.hardware.soundtrigger.KeyphraseMetadata; +import android.hardware.soundtrigger.SoundTrigger; import android.media.AudioFormat; import android.media.permission.Identity; import android.os.Bundle; @@ -25,18 +27,17 @@ import android.os.ParcelFileDescriptor; import android.os.PersistableBundle; import android.os.RemoteCallback; import android.os.SharedMemory; +import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback; +import android.service.voice.IVoiceInteractionService; +import android.service.voice.IVoiceInteractionSession; +import android.service.voice.VisibleActivityInfo; import com.android.internal.app.IHotwordRecognitionStatusCallback; import com.android.internal.app.IVoiceActionCheckCallback; -import com.android.internal.app.IVoiceInteractionSessionShowCallback; -import com.android.internal.app.IVoiceInteractor; import com.android.internal.app.IVoiceInteractionSessionListener; +import com.android.internal.app.IVoiceInteractionSessionShowCallback; import com.android.internal.app.IVoiceInteractionSoundTriggerSession; -import android.hardware.soundtrigger.KeyphraseMetadata; -import android.hardware.soundtrigger.SoundTrigger; -import android.service.voice.IVoiceInteractionService; -import android.service.voice.IVoiceInteractionSession; -import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback; +import com.android.internal.app.IVoiceInteractor; interface IVoiceInteractionManagerService { void showSession(in Bundle sessionArgs, int flags); @@ -289,4 +290,14 @@ interface IVoiceInteractionManagerService { * Notifies when the session window is shown or hidden. */ void setSessionWindowVisible(in IBinder token, boolean visible); + + /** + * Notifies when the Activity lifecycle event changed. + * + * @param activityToken The token of activity. + * @param type The type of lifecycle event of the activity lifecycle. + */ + oneway void notifyActivityEventChanged( + in IBinder activityToken, + int type); } diff --git a/services/core/java/com/android/server/am/ActivityManagerService.java b/services/core/java/com/android/server/am/ActivityManagerService.java index df5113b16f497..85d6f293852ca 100644 --- a/services/core/java/com/android/server/am/ActivityManagerService.java +++ b/services/core/java/com/android/server/am/ActivityManagerService.java @@ -2886,10 +2886,13 @@ public class ActivityManagerService extends IActivityManager.Stub || event == Event.ACTIVITY_DESTROYED)) { contentCaptureService.notifyActivityEvent(userId, activity, event); } - // TODO(b/201234353): Move the logic to client side. - if (mVoiceInteractionManagerProvider != null && (event == Event.ACTIVITY_PAUSED - || event == Event.ACTIVITY_RESUMED || event == Event.ACTIVITY_STOPPED)) { - mVoiceInteractionManagerProvider.notifyActivityEventChanged(); + // Currently we have move most of logic to the client side. When the activity lifecycle + // event changed, the client side will notify the VoiceInteractionManagerService. But + // when the application process died, the VoiceInteractionManagerService will miss the + // activity lifecycle event changed, so we still need ACTIVITY_DESTROYED event here to + // know if the activity has been destroyed. + if (mVoiceInteractionManagerProvider != null && event == Event.ACTIVITY_DESTROYED) { + mVoiceInteractionManagerProvider.notifyActivityDestroyed(appToken); } } diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerService.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerService.java index 352d8d115b6a9..bc5c9ec7a29ca 100644 --- a/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerService.java +++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerService.java @@ -172,11 +172,11 @@ public class VoiceInteractionManagerService extends SystemService { mAmInternal.setVoiceInteractionManagerProvider( new ActivityManagerInternal.VoiceInteractionManagerProvider() { @Override - public void notifyActivityEventChanged() { + public void notifyActivityDestroyed(IBinder activityToken) { if (DEBUG) { - Slog.d(TAG, "call notifyActivityEventChanged"); + Slog.d(TAG, "notifyActivityDestroyed activityToken=" + activityToken); } - mServiceStub.notifyActivityEventChanged(); + mServiceStub.notifyActivityDestroyed(activityToken); } }); } @@ -447,11 +447,12 @@ public class VoiceInteractionManagerService extends SystemService { return mImpl.supportsLocalVoiceInteraction(); } - void notifyActivityEventChanged() { + void notifyActivityDestroyed(@NonNull IBinder activityToken) { synchronized (this) { - if (mImpl == null) return; + if (mImpl == null || activityToken == null) return; - Binder.withCleanCallingIdentity(() -> mImpl.notifyActivityEventChangedLocked()); + Binder.withCleanCallingIdentity( + () -> mImpl.notifyActivityDestroyedLocked(activityToken)); } } @@ -1223,6 +1224,16 @@ public class VoiceInteractionManagerService extends SystemService { } } + @Override + public void notifyActivityEventChanged(@NonNull IBinder activityToken, int type) { + synchronized (this) { + if (mImpl == null || activityToken == null) { + return; + } + Binder.withCleanCallingIdentity( + () -> mImpl.notifyActivityEventChangedLocked(activityToken, type)); + } + } //----------------- Hotword Detection/Validation APIs --------------------------------// @Override diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerServiceImpl.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerServiceImpl.java index b9793cafb6fed..fabab2524e836 100644 --- a/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerServiceImpl.java +++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerServiceImpl.java @@ -520,9 +520,23 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne mActiveSession.stopListeningVisibleActivityChangedLocked(); } - public void notifyActivityEventChangedLocked() { + public void notifyActivityDestroyedLocked(@NonNull IBinder activityToken) { if (DEBUG) { - Slog.d(TAG, "notifyActivityEventChangedLocked"); + Slog.d(TAG, "notifyActivityDestroyedLocked activityToken=" + activityToken); + } + if (mActiveSession == null || !mActiveSession.mShown) { + if (DEBUG) { + Slog.d(TAG, "notifyActivityDestroyedLocked not allowed on no session or" + + " hidden session"); + } + return; + } + mActiveSession.notifyActivityDestroyedLocked(activityToken); + } + + public void notifyActivityEventChangedLocked(@NonNull IBinder activityToken, int type) { + if (DEBUG) { + Slog.d(TAG, "notifyActivityEventChangedLocked type=" + type); } if (mActiveSession == null || !mActiveSession.mShown) { if (DEBUG) { @@ -531,7 +545,7 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne } return; } - mActiveSession.notifyActivityEventChangedLocked(); + mActiveSession.notifyActivityEventChangedLocked(activityToken, type); } public void updateStateLocked( diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionSessionConnection.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionSessionConnection.java index ae9be8c09e605..b24337fd54c93 100644 --- a/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionSessionConnection.java +++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionSessionConnection.java @@ -29,6 +29,7 @@ import static com.android.server.wm.ActivityTaskManagerInternal.ASSIST_KEY_DATA; import static com.android.server.wm.ActivityTaskManagerInternal.ASSIST_KEY_STRUCTURE; import static com.android.server.wm.ActivityTaskManagerInternal.ASSIST_TASK_ID; +import android.annotation.NonNull; import android.app.ActivityManager; import android.app.ActivityTaskManager; import android.app.AppOpsManager; @@ -59,6 +60,7 @@ import android.service.voice.IVoiceInteractionSessionService; import android.service.voice.VisibleActivityInfo; import android.service.voice.VoiceInteractionService; import android.service.voice.VoiceInteractionSession; +import android.util.ArrayMap; import android.util.Slog; import android.view.IWindowManager; @@ -128,7 +130,11 @@ final class VoiceInteractionSessionConnection implements ServiceConnection, private boolean mListeningVisibleActivity; private final ScheduledExecutorService mScheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); - private final List mVisibleActivityInfos = new ArrayList<>(); + // Records the visible activity information the system has already called onVisible, without + // confirming the result of callback. When activity visible state is changed, we use this to + // determine to call onVisible or onInvisible to assistant application. + private final ArrayMap mVisibleActivityInfoForToken = + new ArrayMap<>(); private final PowerManagerInternal mPowerManagerInternal; private final LowPowerStandbyControllerInternal mLowPowerStandbyControllerInternal; private final Runnable mRemoveFromLowPowerStandbyAllowlistRunnable = @@ -530,7 +536,7 @@ final class VoiceInteractionSessionConnection implements ServiceConnection, public void cancelLocked(boolean finishTask) { mListeningVisibleActivity = false; - mVisibleActivityInfos.clear(); + mVisibleActivityInfoForToken.clear(); hideLocked(); mCanceled = true; if (mBound) { @@ -608,17 +614,24 @@ final class VoiceInteractionSessionConnection implements ServiceConnection, if (DEBUG) { Slog.d(TAG, "startListeningVisibleActivityChangedLocked"); } - mListeningVisibleActivity = true; - mVisibleActivityInfos.clear(); - mScheduledExecutorService.execute(() -> { - if (DEBUG) { - Slog.d(TAG, "call handleVisibleActivitiesLocked from enable listening"); - } - synchronized (mLock) { - handleVisibleActivitiesLocked(); - } - }); + if (!mShown || mCanceled || mSession == null) { + return; + } + + mListeningVisibleActivity = true; + mVisibleActivityInfoForToken.clear(); + + // It should only need to report which activities are visible + final ArrayMap newVisibleActivityInfos = + getTopVisibleActivityInfosLocked(); + + if (newVisibleActivityInfos == null || newVisibleActivityInfos.isEmpty()) { + return; + } + notifyVisibleActivitiesChangedLocked(newVisibleActivityInfos, + VisibleActivityInfo.TYPE_ACTIVITY_ADDED); + mVisibleActivityInfoForToken.putAll(newVisibleActivityInfos); } void stopListeningVisibleActivityChangedLocked() { @@ -626,12 +639,13 @@ final class VoiceInteractionSessionConnection implements ServiceConnection, Slog.d(TAG, "stopListeningVisibleActivityChangedLocked"); } mListeningVisibleActivity = false; - mVisibleActivityInfos.clear(); + mVisibleActivityInfoForToken.clear(); } - void notifyActivityEventChangedLocked() { + void notifyActivityEventChangedLocked(@NonNull IBinder activityToken, int type) { if (DEBUG) { - Slog.d(TAG, "notifyActivityEventChangedLocked"); + Slog.d(TAG, "notifyActivityEventChangedLocked activityToken=" + activityToken + + ", type=" + type); } if (!mListeningVisibleActivity) { if (DEBUG) { @@ -640,99 +654,139 @@ final class VoiceInteractionSessionConnection implements ServiceConnection, return; } mScheduledExecutorService.execute(() -> { - if (DEBUG) { - Slog.d(TAG, "call handleVisibleActivitiesLocked from activity event"); - } synchronized (mLock) { - handleVisibleActivitiesLocked(); + handleVisibleActivitiesLocked(activityToken, type); } }); } - private List getVisibleActivityInfosLocked() { + private ArrayMap getTopVisibleActivityInfosLocked() { if (DEBUG) { - Slog.d(TAG, "getVisibleActivityInfosLocked"); + Slog.d(TAG, "getTopVisibleActivityInfosLocked"); } List allVisibleActivities = LocalServices.getService(ActivityTaskManagerInternal.class) .getTopVisibleActivities(); if (DEBUG) { - Slog.d(TAG, - "getVisibleActivityInfosLocked: allVisibleActivities=" + allVisibleActivities); + Slog.d(TAG, "getTopVisibleActivityInfosLocked: allVisibleActivities=" + + allVisibleActivities); } - if (allVisibleActivities == null || allVisibleActivities.isEmpty()) { + if (allVisibleActivities.isEmpty()) { Slog.w(TAG, "no visible activity"); return null; } final int count = allVisibleActivities.size(); - final List visibleActivityInfos = new ArrayList<>(count); + final ArrayMap visibleActivityInfoArrayMap = + new ArrayMap<>(count); for (int i = 0; i < count; i++) { ActivityAssistInfo info = allVisibleActivities.get(i); if (DEBUG) { - Slog.d(TAG, " : activityToken=" + info.getActivityToken() + Slog.d(TAG, "ActivityAssistInfo : activityToken=" + info.getActivityToken() + ", assistToken=" + info.getAssistToken() + ", taskId=" + info.getTaskId()); } - visibleActivityInfos.add( + visibleActivityInfoArrayMap.put(info.getActivityToken(), new VisibleActivityInfo(info.getTaskId(), info.getAssistToken())); } - return visibleActivityInfos; + return visibleActivityInfoArrayMap; } - private void handleVisibleActivitiesLocked() { + // TODO(b/242359988): Split this method up + private void handleVisibleActivitiesLocked(@NonNull IBinder activityToken, int type) { if (DEBUG) { - Slog.d(TAG, "handleVisibleActivitiesLocked"); + Slog.d(TAG, "handleVisibleActivitiesLocked activityToken=" + activityToken + + ", type=" + type); } - if (mSession == null) { - return; - } - if (!mShown || !mListeningVisibleActivity || mCanceled) { - return; - } - final List newVisibleActivityInfos = getVisibleActivityInfosLocked(); - if (newVisibleActivityInfos == null || newVisibleActivityInfos.isEmpty()) { - notifyVisibleActivitiesChangedLocked(mVisibleActivityInfos, - VisibleActivityInfo.TYPE_ACTIVITY_REMOVED); - mVisibleActivityInfos.clear(); + if (!mListeningVisibleActivity) { + if (DEBUG) { + Slog.d(TAG, "not enable listening visible activity"); + } return; } - if (mVisibleActivityInfos.isEmpty()) { - notifyVisibleActivitiesChangedLocked(newVisibleActivityInfos, - VisibleActivityInfo.TYPE_ACTIVITY_ADDED); - mVisibleActivityInfos.addAll(newVisibleActivityInfos); + if (!mShown || mCanceled || mSession == null) { return; } - final List addedActivities = new ArrayList<>(); - final List removedActivities = new ArrayList<>(); + // We use this local variable to determine to call onVisible or onInvisible. + boolean notifyOnVisible = false; + VisibleActivityInfo notifyVisibleActivityInfo = null; - removedActivities.addAll(mVisibleActivityInfos); - for (int i = 0; i < newVisibleActivityInfos.size(); i++) { - final VisibleActivityInfo candidateVisibleActivityInfo = newVisibleActivityInfos.get(i); - if (!removedActivities.isEmpty() && removedActivities.contains( - candidateVisibleActivityInfo)) { - removedActivities.remove(candidateVisibleActivityInfo); - } else { - addedActivities.add(candidateVisibleActivityInfo); + if (type == VoiceInteractionSession.VOICE_INTERACTION_ACTIVITY_EVENT_START + || type == VoiceInteractionSession.VOICE_INTERACTION_ACTIVITY_EVENT_RESUME) { + // It seems that the onStart is unnecessary. But if we have it, the assistant + // application can request the directActions early. Even if we have the onStart, + // we still need the onResume because it is possible that the activity goes to + // onResume from onPause with invisible before the activity goes to onStop from + // onPause. + + // Check if we have reported this activity as visible. If we have reported it as + // visible, do nothing. + if (mVisibleActivityInfoForToken.containsKey(activityToken)) { + return; + } + + // Before reporting this activity as visible, we need to make sure the activity + // is really visible. + notifyVisibleActivityInfo = getVisibleActivityInfoFromTopVisibleActivity( + activityToken); + if (notifyVisibleActivityInfo == null) { + return; + } + notifyOnVisible = true; + } else if (type == VoiceInteractionSession.VOICE_INTERACTION_ACTIVITY_EVENT_PAUSE) { + // For the onPause stage, the Activity is not necessarily invisible now, so we need + // to check its state. + // Note: After syncing with Activity owner, before the onPause is called, the + // visibility state has been updated. + notifyVisibleActivityInfo = getVisibleActivityInfoFromTopVisibleActivity( + activityToken); + if (notifyVisibleActivityInfo != null) { + return; + } + + // Also make sure we previously reported this Activity as visible. + notifyVisibleActivityInfo = mVisibleActivityInfoForToken.get(activityToken); + if (notifyVisibleActivityInfo == null) { + return; + } + } else if (type == VoiceInteractionSession.VOICE_INTERACTION_ACTIVITY_EVENT_STOP) { + // For the onStop stage, the activity is in invisible state. We only need to consider if + // we have reported this activity as visible. If we have reported it as visible, we + // need to report it as invisible. + // Why we still need onStop? Because it is possible that the activity is in a visible + // state during onPause stage, when the activity enters onStop from onPause, we may + // need to notify onInvisible. + // Note: After syncing with Activity owner, before the onStop is called, the + // visibility state has been updated. + notifyVisibleActivityInfo = mVisibleActivityInfoForToken.get(activityToken); + if (notifyVisibleActivityInfo == null) { + return; + } + } else { + Slog.w(TAG, "notifyActivityEventChangedLocked unexpected type=" + type); + return; + } + + try { + mSession.notifyVisibleActivityInfoChanged(notifyVisibleActivityInfo, + notifyOnVisible ? VisibleActivityInfo.TYPE_ACTIVITY_ADDED + : VisibleActivityInfo.TYPE_ACTIVITY_REMOVED); + } catch (RemoteException e) { + if (DEBUG) { + Slog.w(TAG, "handleVisibleActivitiesLocked RemoteException : " + e); } } - if (!addedActivities.isEmpty()) { - notifyVisibleActivitiesChangedLocked(addedActivities, - VisibleActivityInfo.TYPE_ACTIVITY_ADDED); + if (notifyOnVisible) { + mVisibleActivityInfoForToken.put(activityToken, notifyVisibleActivityInfo); + } else { + mVisibleActivityInfoForToken.remove(activityToken); } - if (!removedActivities.isEmpty()) { - notifyVisibleActivitiesChangedLocked(removedActivities, - VisibleActivityInfo.TYPE_ACTIVITY_REMOVED); - } - - mVisibleActivityInfos.clear(); - mVisibleActivityInfos.addAll(newVisibleActivityInfos); } private void notifyVisibleActivitiesChangedLocked( - List visibleActivityInfos, int type) { + ArrayMap visibleActivityInfos, int type) { if (visibleActivityInfos == null || visibleActivityInfos.isEmpty()) { return; } @@ -741,7 +795,7 @@ final class VoiceInteractionSessionConnection implements ServiceConnection, } try { for (int i = 0; i < visibleActivityInfos.size(); i++) { - mSession.notifyVisibleActivityInfoChanged(visibleActivityInfos.get(i), type); + mSession.notifyVisibleActivityInfoChanged(visibleActivityInfos.valueAt(i), type); } } catch (RemoteException e) { if (DEBUG) { @@ -754,6 +808,51 @@ final class VoiceInteractionSessionConnection implements ServiceConnection, } } + private VisibleActivityInfo getVisibleActivityInfoFromTopVisibleActivity( + @NonNull IBinder activityToken) { + final ArrayMap visibleActivityInfos = + getTopVisibleActivityInfosLocked(); + if (visibleActivityInfos == null) { + return null; + } + return visibleActivityInfos.get(activityToken); + } + + void notifyActivityDestroyedLocked(@NonNull IBinder activityToken) { + if (DEBUG) { + Slog.d(TAG, "notifyActivityDestroyedLocked activityToken=" + activityToken); + } + if (!mListeningVisibleActivity) { + if (DEBUG) { + Slog.d(TAG, "not enable listening visible activity"); + } + return; + } + mScheduledExecutorService.execute(() -> { + synchronized (mLock) { + if (!mListeningVisibleActivity) { + return; + } + if (!mShown || mCanceled || mSession == null) { + return; + } + + VisibleActivityInfo visibleActivityInfo = mVisibleActivityInfoForToken.remove( + activityToken); + if (visibleActivityInfo != null) { + try { + mSession.notifyVisibleActivityInfoChanged(visibleActivityInfo, + VisibleActivityInfo.TYPE_ACTIVITY_REMOVED); + } catch (RemoteException e) { + if (DEBUG) { + Slog.w(TAG, "notifyVisibleActivityInfoChanged RemoteException : " + e); + } + } + } + } + }); + } + private void removeFromLowPowerStandbyAllowlist() { synchronized (mLock) { if (mLowPowerStandbyAllowlisted) {