Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Integrate VQDS lifecycle into connection session

Browse Source 
The change focuses on the initiliazation and the life cycle of the
VisualQueryDetectionService. It also adds the VisualQueryDetector
to the service by adding necessary connection sessions to the
existing HotwordDetectionConnection which will now contain two
ServiceConnection instances, for HotwordDetectionService and
VisualQueryDetectionService respectively.

Bug: 261783492
Test: Manual & atest CtsVoiceInteractionTestCases
Change-Id: Ia3c771f3304c3a8849aa7cc209a5fc4ad838db45
This commit is contained in:
Charles Chen
2022-12-19 07:24:32 +00:00
parent 28b528f72d
commit fd9097a758
5 changed files with 445 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files

81
core/java/android/service/voice/VisualQueryDetector.java
View File

@@ -25,12 +25,16 @@ import android.annotation.Nullable;
import android.annotation.RequiresPermission;
import android.annotation.SuppressLint;
import android.annotation.SystemApi;
import android.hardware.soundtrigger.SoundTrigger;
import android.media.AudioFormat;
import android.os.Binder;
import android.os.ParcelFileDescriptor;
import android.os.PersistableBundle;
import android.os.RemoteException;
import android.os.SharedMemory;
import android.util.Slog;
import com.android.internal.app.IHotwordRecognitionStatusCallback;
import com.android.internal.app.IVoiceInteractionManagerService;
import java.io.PrintWriter;
@@ -208,8 +212,9 @@ public class VisualQueryDetector {
@Override
void initialize(@Nullable PersistableBundle options, @Nullable SharedMemory sharedMemory) {
//TODO(261783492): call initAndVerify to create VisualQueryDetectionService
// from the system server.
initAndVerifyDetector(options, sharedMemory,
new InitializationStateListener(mExecutor, mCallback),
DETECTOR_TYPE_VISUAL_QUERY_DETECTOR);
}
@Override
@@ -238,4 +243,76 @@ public class VisualQueryDetector {
return true;
}
}
private static class InitializationStateListener
extends IHotwordRecognitionStatusCallback.Stub {
private final Executor mExecutor;
private final Callback mCallback;
InitializationStateListener(Executor executor, Callback callback) {
this.mExecutor = executor;
this.mCallback = callback;
}
@Override
public void onKeyphraseDetected(
SoundTrigger.KeyphraseRecognitionEvent recognitionEvent,
HotwordDetectedResult result) {
if (DEBUG) {
Slog.i(TAG, "Ignored #onKeyphraseDetected event");
}
}
@Override
public void onGenericSoundTriggerDetected(
SoundTrigger.GenericRecognitionEvent recognitionEvent) throws RemoteException {
if (DEBUG) {
Slog.i(TAG, "Ignored #onGenericSoundTriggerDetected event");
}
}
@Override
public void onRejected(HotwordRejectedResult result) throws RemoteException {
if (DEBUG) {
Slog.i(TAG, "Ignored #onRejected event");
}
}
@Override
public void onRecognitionPaused() throws RemoteException {
if (DEBUG) {
Slog.i(TAG, "Ignored #onRecognitionPaused event");
}
}
@Override
public void onRecognitionResumed() throws RemoteException {
if (DEBUG) {
Slog.i(TAG, "Ignored #onRecognitionResumed event");
}
}
@Override
public void onStatusReported(int status) {
Slog.v(TAG, "onStatusReported" + (DEBUG ? "(" + status + ")" : ""));
//TODO: rename the target callback with a more general term
Binder.withCleanCallingIdentity(() -> mExecutor.execute(
() -> mCallback.onVisualQueryDetectionServiceInitialized(status)));
}
@Override
public void onProcessRestarted() throws RemoteException {
Slog.v(TAG, "onProcessRestarted()");
//TODO: rename the target callback with a more general term
Binder.withCleanCallingIdentity(() -> mExecutor.execute(
() -> mCallback.onVisualQueryDetectionServiceRestarted()));
}
@Override
public void onError(int status) throws RemoteException {
Slog.v(TAG, "Initialization Error: (" + status + ")");
// Do nothing
}
}
}

95
services/voiceinteraction/java/com/android/server/voiceinteraction/DetectorSession.java
View File

@@ -205,9 +205,15 @@ abstract class DetectorSession {
mVoiceInteractionServiceUid = voiceInteractionServiceUid;
mVoiceInteractorIdentity = voiceInteractorIdentity;
mAppOpsManager = mContext.getSystemService(AppOpsManager.class);
mHotwordAudioStreamCopier = new HotwordAudioStreamCopier(mAppOpsManager, getDetectorType(),
mVoiceInteractorIdentity.uid, mVoiceInteractorIdentity.packageName,
mVoiceInteractorIdentity.attributionTag);
if (getDetectorType() != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) {
mHotwordAudioStreamCopier = new HotwordAudioStreamCopier(mAppOpsManager,
getDetectorType(),
mVoiceInteractorIdentity.uid, mVoiceInteractorIdentity.packageName,
mVoiceInteractorIdentity.attributionTag);
} else {
mHotwordAudioStreamCopier = null;
}
mScheduledExecutorService = scheduledExecutorService;
mDebugHotwordLogging = logging;
@@ -236,9 +242,12 @@ abstract class DetectorSession {
future.complete(null);
if (mUpdateStateAfterStartFinished.getAndSet(true)) {
Slog.w(TAG, "call callback after timeout");
HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
if (getDetectorType()
!= HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) {
HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_UPDATE_STATE_AFTER_TIMEOUT,
mVoiceInteractionServiceUid);
}
return;
}
Pair<Integer, Integer> statusResultPair = getInitStatusAndMetricsResult(bundle);
@@ -246,27 +255,37 @@ abstract class DetectorSession {
int initResultMetricsResult = statusResultPair.second;
try {
mCallback.onStatusReported(status);
HotwordMetricsLogger.writeServiceInitResultEvent(getDetectorType(),
initResultMetricsResult, mVoiceInteractionServiceUid);
if (getDetectorType()
!= HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) {
HotwordMetricsLogger.writeServiceInitResultEvent(getDetectorType(),
initResultMetricsResult, mVoiceInteractionServiceUid);
}
} catch (RemoteException e) {
Slog.w(TAG, "Failed to report initialization status: " + e);
HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
METRICS_CALLBACK_ON_STATUS_REPORTED_EXCEPTION,
mVoiceInteractionServiceUid);
if (getDetectorType()
!= HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) {
HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
METRICS_CALLBACK_ON_STATUS_REPORTED_EXCEPTION,
mVoiceInteractionServiceUid);
}
}
}
};
try {
service.updateState(options, sharedMemory, statusCallback);
HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_UPDATE_STATE,
mVoiceInteractionServiceUid);
if (getDetectorType() != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) {
HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_UPDATE_STATE,
mVoiceInteractionServiceUid);
}
} catch (RemoteException e) {
// TODO: (b/181842909) Report an error to voice interactor
Slog.w(TAG, "Failed to updateState for HotwordDetectionService", e);
HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
HOTWORD_DETECTOR_EVENTS__EVENT__CALL_UPDATE_STATE_EXCEPTION,
mVoiceInteractionServiceUid);
if (getDetectorType() != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) {
HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
HOTWORD_DETECTOR_EVENTS__EVENT__CALL_UPDATE_STATE_EXCEPTION,
mVoiceInteractionServiceUid);
}
}
return future.orTimeout(MAX_UPDATE_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS);
}).whenComplete((res, err) -> {
@@ -277,13 +296,17 @@ abstract class DetectorSession {
}
try {
mCallback.onStatusReported(INITIALIZATION_STATUS_UNKNOWN);
HotwordMetricsLogger.writeServiceInitResultEvent(getDetectorType(),
METRICS_INIT_UNKNOWN_TIMEOUT, mVoiceInteractionServiceUid);
if (getDetectorType() != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) {
HotwordMetricsLogger.writeServiceInitResultEvent(getDetectorType(),
METRICS_INIT_UNKNOWN_TIMEOUT, mVoiceInteractionServiceUid);
}
} catch (RemoteException e) {
Slog.w(TAG, "Failed to report initialization status UNKNOWN", e);
HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
METRICS_CALLBACK_ON_STATUS_REPORTED_EXCEPTION,
mVoiceInteractionServiceUid);
if (getDetectorType() != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) {
HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
METRICS_CALLBACK_ON_STATUS_REPORTED_EXCEPTION,
mVoiceInteractionServiceUid);
}
}
} else if (err != null) {
Slog.w(TAG, "Failed to update state: " + err);
@@ -315,9 +338,11 @@ abstract class DetectorSession {
@SuppressWarnings("GuardedBy")
void updateStateLocked(PersistableBundle options, SharedMemory sharedMemory,
Instant lastRestartInstant) {
HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
HOTWORD_DETECTOR_EVENTS__EVENT__APP_REQUEST_UPDATE_STATE,
mVoiceInteractionServiceUid);
if (getDetectorType() != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) {
HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
HOTWORD_DETECTOR_EVENTS__EVENT__APP_REQUEST_UPDATE_STATE,
mVoiceInteractionServiceUid);
}
// Prevent doing the init late, so restart is handled equally to a clean process start.
// TODO(b/191742511): this logic needs a test
if (!mUpdateStateAfterStartFinished.get() && Instant.now().minus(
@@ -399,9 +424,11 @@ abstract class DetectorSession {
callback.onError();
} catch (RemoteException ex) {
Slog.w(TAG, "Failed to report onError status: " + ex);
HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION,
mVoiceInteractionServiceUid);
if (getDetectorType() != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) {
HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION,
mVoiceInteractionServiceUid);
}
}
} finally {
synchronized (mLock) {
@@ -427,7 +454,8 @@ abstract class DetectorSession {
throws RemoteException {
synchronized (mLock) {
mPerformingExternalSourceHotwordDetection = false;
HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
HotwordMetricsLogger.writeDetectorEvent(
getDetectorType(),
METRICS_EXTERNAL_SOURCE_REJECTED,
mVoiceInteractionServiceUid);
mScheduledExecutorService.schedule(
@@ -454,7 +482,8 @@ abstract class DetectorSession {
throws RemoteException {
synchronized (mLock) {
mPerformingExternalSourceHotwordDetection = false;
HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
HotwordMetricsLogger.writeDetectorEvent(
getDetectorType(),
METRICS_EXTERNAL_SOURCE_DETECTED,
mVoiceInteractionServiceUid);
mScheduledExecutorService.schedule(
@@ -540,9 +569,11 @@ abstract class DetectorSession {
mCallback.onError(status);
} catch (RemoteException e) {
Slog.w(TAG, "Failed to report onError status: " + e);
HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION,
mVoiceInteractionServiceUid);
if (getDetectorType() != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) {
HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION,
mVoiceInteractionServiceUid);
}
}
}
@@ -679,6 +710,8 @@ abstract class DetectorSession {
return HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP;
} else if (this instanceof SoftwareTrustedHotwordDetectorSession) {
return HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE;
} else if (this instanceof VisualQueryDetectorSession) {
return HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR;
}
Slog.v(TAG, "Unexpected detector type");
return -1;

206
services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java
View File

@@ -52,6 +52,7 @@ import android.service.voice.HotwordDetectionService;
import android.service.voice.HotwordDetector;
import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback;
import android.service.voice.ISandboxedDetectionService;
import android.service.voice.VisualQueryDetectionService;
import android.service.voice.VoiceInteractionManagerInternal.HotwordDetectionServiceIdentity;
import android.speech.IRecognitionServiceManager;
import android.util.Slog;
@@ -74,7 +75,8 @@ import java.util.function.Consumer;
import java.util.function.Function;
/**
* A class that provides the communication with the HotwordDetectionService.
* A class that provides the communication with the {@link HotwordDetectionService} and
* {@link VisualQueryDetectionService}.
*/
final class HotwordDetectionConnection {
private static final String TAG = "HotwordDetectionConnection";
@@ -90,7 +92,8 @@ final class HotwordDetectionConnection {
@Nullable private final ScheduledFuture<?> mCancellationTaskFuture;
private final IBinder.DeathRecipient mAudioServerDeathRecipient = this::audioServerDied;
@NonNull private final ServiceConnectionFactory mHotwordDetectionServiceConnectionFactory;
private final int mDetectorType;
@NonNull private final ServiceConnectionFactory mVisualQueryDetectionServiceConnectionFactory;
private int mDetectorType;
/**
* Time after which each HotwordDetectionService process is stopped and replaced by a new one.
* 0 indicates no restarts.
@@ -100,9 +103,11 @@ final class HotwordDetectionConnection {
final Object mLock;
final int mVoiceInteractionServiceUid;
final ComponentName mHotwordDetectionComponentName;
final ComponentName mVisualQueryDetectionComponentName;
final int mUser;
final Context mContext;
volatile HotwordDetectionServiceIdentity mIdentity;
//TODO: add similar identity for visual query service for the use of content capturing
private Instant mLastRestartInstant;
private ScheduledFuture<?> mDebugHotwordLoggingTimeoutFuture = null;
@@ -112,6 +117,7 @@ final class HotwordDetectionConnection {
@Nullable
private final Identity mVoiceInteractorIdentity;
@NonNull private ServiceConnection mRemoteHotwordDetectionService;
@NonNull private ServiceConnection mRemoteVisualQueryDetectionService;
private IBinder mAudioFlinger;
@GuardedBy("mLock")
private boolean mDebugHotwordLogging = false;
@@ -126,26 +132,39 @@ final class HotwordDetectionConnection {
new SparseArray<>();
HotwordDetectionConnection(Object lock, Context context, int voiceInteractionServiceUid,
Identity voiceInteractorIdentity, ComponentName hotwordDetectionServiceName, int userId,
Identity voiceInteractorIdentity, ComponentName hotwordDetectionServiceName,
ComponentName visualQueryDetectionServiceName, int userId,
boolean bindInstantServiceAllowed, int detectorType) {
mLock = lock;
mContext = context;
mVoiceInteractionServiceUid = voiceInteractionServiceUid;
mVoiceInteractorIdentity = voiceInteractorIdentity;
mHotwordDetectionComponentName = hotwordDetectionServiceName;
mVisualQueryDetectionComponentName = visualQueryDetectionServiceName;
mUser = userId;
mDetectorType = detectorType;
mReStartPeriodSeconds = DeviceConfig.getInt(DeviceConfig.NAMESPACE_VOICE_INTERACTION,
KEY_RESTART_PERIOD_IN_SECONDS, 0);
final Intent hotwordDetectionServiceIntent =
new Intent(HotwordDetectionService.SERVICE_INTERFACE);
hotwordDetectionServiceIntent.setComponent(mHotwordDetectionComponentName);
final Intent visualQueryDetectionServiceIntent =
new Intent(VisualQueryDetectionService.SERVICE_INTERFACE);
visualQueryDetectionServiceIntent.setComponent(mVisualQueryDetectionComponentName);
initAudioFlingerLocked();
mHotwordDetectionServiceConnectionFactory =
new ServiceConnectionFactory(hotwordDetectionServiceIntent,
bindInstantServiceAllowed);
mRemoteHotwordDetectionService = mHotwordDetectionServiceConnectionFactory.createLocked();
mVisualQueryDetectionServiceConnectionFactory =
new ServiceConnectionFactory(visualQueryDetectionServiceIntent,
bindInstantServiceAllowed);
mLastRestartInstant = Instant.now();
if (mReStartPeriodSeconds <= 0) {
@@ -157,9 +176,11 @@ final class HotwordDetectionConnection {
Slog.v(TAG, "Time to restart the process, TTL has passed");
synchronized (mLock) {
restartProcessLocked();
HotwordMetricsLogger.writeServiceRestartEvent(mDetectorType,
HOTWORD_DETECTION_SERVICE_RESTARTED__REASON__SCHEDULE,
mVoiceInteractionServiceUid);
if (mDetectorType != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) {
HotwordMetricsLogger.writeServiceRestartEvent(mDetectorType,
HOTWORD_DETECTION_SERVICE_RESTARTED__REASON__SCHEDULE,
mVoiceInteractionServiceUid);
}
}
}, mReStartPeriodSeconds, mReStartPeriodSeconds, TimeUnit.SECONDS);
}
@@ -193,9 +214,11 @@ final class HotwordDetectionConnection {
// We restart the process instead of simply sending over the new binder, to avoid race
// conditions with audio reading in the service.
restartProcessLocked();
HotwordMetricsLogger.writeServiceRestartEvent(mDetectorType,
HOTWORD_DETECTION_SERVICE_RESTARTED__REASON__AUDIO_SERVICE_DIED,
mVoiceInteractionServiceUid);
if (mDetectorType != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) {
HotwordMetricsLogger.writeServiceRestartEvent(mDetectorType,
HOTWORD_DETECTION_SERVICE_RESTARTED__REASON__AUDIO_SERVICE_DIED,
mVoiceInteractionServiceUid);
}
}
}
@@ -208,9 +231,8 @@ final class HotwordDetectionConnection {
});
mDetectorSessions.clear();
mDebugHotwordLogging = false;
mRemoteHotwordDetectionService.unbind();
LocalServices.getService(PermissionManagerServiceInternal.class)
.setHotwordDetectionServiceProvider(null);
unbindVisualQueryDetectionService();
unbindHotwordDetectionService();
if (mIdentity != null) {
removeServiceUidForAudioPolicy(mIdentity.getIsolatedUid());
}
@@ -223,6 +245,21 @@ final class HotwordDetectionConnection {
}
}
private void unbindVisualQueryDetectionService() {
if (mRemoteVisualQueryDetectionService != null) {
mRemoteVisualQueryDetectionService.unbind();
//TODO: Set visual query detection service provider to null
}
}
private void unbindHotwordDetectionService() {
if (mRemoteHotwordDetectionService != null) {
mRemoteHotwordDetectionService.unbind();
LocalServices.getService(PermissionManagerServiceInternal.class)
.setHotwordDetectionServiceProvider(null);
}
}
@SuppressWarnings("GuardedBy")
void updateStateLocked(PersistableBundle options, SharedMemory sharedMemory,
@NonNull IBinder token) {
@@ -342,6 +379,10 @@ final class HotwordDetectionConnection {
}
}
void setDetectorType(int detectorType) {
mDetectorType = detectorType;
}
private void clearDebugHotwordLoggingTimeoutLocked() {
if (mDebugHotwordLoggingTimeoutFuture != null) {
mDebugHotwordLoggingTimeoutFuture.cancel(/* mayInterruptIfRunning= */ true);
@@ -353,24 +394,41 @@ final class HotwordDetectionConnection {
private void restartProcessLocked() {
// TODO(b/244598068): Check HotwordAudioStreamManager first
Slog.v(TAG, "Restarting hotword detection process");
ServiceConnection oldHotwordConnection = mRemoteHotwordDetectionService;
ServiceConnection oldVisualQueryDetectionConnection = mRemoteVisualQueryDetectionService;
HotwordDetectionServiceIdentity previousIdentity = mIdentity;
//TODO: Add previousIdentity for visual query detection service
mLastRestartInstant = Instant.now();
// Recreate connection to reset the cache.
mRemoteHotwordDetectionService = mHotwordDetectionServiceConnectionFactory.createLocked();
mRemoteVisualQueryDetectionService =
mVisualQueryDetectionServiceConnectionFactory.createLocked();
Slog.v(TAG, "Started the new process, dispatching processRestarted to detector");
runForEachDetectorSessionLocked((session) -> {
session.updateRemoteSandboxedDetectionServiceLocked(mRemoteHotwordDetectionService);
HotwordDetectionConnection.ServiceConnection newRemoteService =
(session instanceof VisualQueryDetectorSession)
? mRemoteVisualQueryDetectionService : mRemoteHotwordDetectionService;
session.updateRemoteSandboxedDetectionServiceLocked(newRemoteService);
session.informRestartProcessLocked();
});
if (DEBUG) {
Slog.i(TAG, "processRestarted is dispatched done, unbinding from the old process");
}
oldHotwordConnection.ignoreConnectionStatusEvents();
oldHotwordConnection.unbind();
if (oldHotwordConnection != null) {
oldHotwordConnection.ignoreConnectionStatusEvents();
oldHotwordConnection.unbind();
}
if (oldVisualQueryDetectionConnection != null) {
oldVisualQueryDetectionConnection.ignoreConnectionStatusEvents();
oldVisualQueryDetectionConnection.unbind();
}
if (previousIdentity != null) {
removeServiceUidForAudioPolicy(previousIdentity.getIsolatedUid());
}
@@ -438,9 +496,14 @@ final class HotwordDetectionConnection {
synchronized (mLock) {
pw.print(prefix); pw.print("mReStartPeriodSeconds="); pw.println(mReStartPeriodSeconds);
pw.print(prefix); pw.print("mBound=");
pw.println(mRemoteHotwordDetectionService.isBound());
pw.println(mRemoteHotwordDetectionService != null
&& mRemoteHotwordDetectionService.isBound());
pw.println(mRemoteVisualQueryDetectionService != null
&& mRemoteHotwordDetectionService != null
&& mRemoteHotwordDetectionService.isBound());
pw.print(prefix); pw.print("mRestartCount=");
pw.println(mHotwordDetectionServiceConnectionFactory.mRestartCount);
pw.println(mVisualQueryDetectionServiceConnectionFactory.mRestartCount);
pw.print(prefix); pw.print("mLastRestartInstant="); pw.println(mLastRestartInstant);
pw.print(prefix); pw.print("mDetectorType=");
pw.println(HotwordDetector.detectorTypeToString(mDetectorType));
@@ -489,11 +552,11 @@ final class HotwordDetectionConnection {
private boolean mIsLoggedFirstConnect = false;
ServiceConnection(@NonNull Context context,
@NonNull Intent intent, int bindingFlags, int userId,
@NonNull Intent serviceIntent, int bindingFlags, int userId,
@Nullable Function<IBinder, ISandboxedDetectionService> binderAsInterface,
int instanceNumber) {
super(context, intent, bindingFlags, userId, binderAsInterface);
this.mIntent = intent;
super(context, serviceIntent, bindingFlags, userId, binderAsInterface);
this.mIntent = serviceIntent;
this.mBindingFlags = bindingFlags;
this.mInstanceNumber = instanceNumber;
}
@@ -512,14 +575,18 @@ final class HotwordDetectionConnection {
mIsBound = connected;
if (!connected) {
HotwordMetricsLogger.writeDetectorEvent(mDetectorType,
HOTWORD_DETECTOR_EVENTS__EVENT__ON_DISCONNECTED,
mVoiceInteractionServiceUid);
if (mDetectorType != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) {
HotwordMetricsLogger.writeDetectorEvent(mDetectorType,
HOTWORD_DETECTOR_EVENTS__EVENT__ON_DISCONNECTED,
mVoiceInteractionServiceUid);
}
} else if (!mIsLoggedFirstConnect) {
mIsLoggedFirstConnect = true;
HotwordMetricsLogger.writeDetectorEvent(mDetectorType,
HOTWORD_DETECTOR_EVENTS__EVENT__ON_CONNECTED,
mVoiceInteractionServiceUid);
if (mDetectorType != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) {
HotwordMetricsLogger.writeDetectorEvent(mDetectorType,
HOTWORD_DETECTOR_EVENTS__EVENT__ON_CONNECTED,
mVoiceInteractionServiceUid);
}
}
}
}
@@ -546,35 +613,46 @@ final class HotwordDetectionConnection {
});
}
// Can improve to log exit reason if needed
HotwordMetricsLogger.writeKeyphraseTriggerEvent(
mDetectorType,
HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__SERVICE_CRASH,
mVoiceInteractionServiceUid);
if (mDetectorType != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) {
HotwordMetricsLogger.writeKeyphraseTriggerEvent(
mDetectorType,
HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__SERVICE_CRASH,
mVoiceInteractionServiceUid);
}
}
@Override
protected boolean bindService(
@NonNull android.content.ServiceConnection serviceConnection) {
try {
HotwordMetricsLogger.writeDetectorEvent(mDetectorType,
HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_BIND_SERVICE,
mVoiceInteractionServiceUid);
if (mDetectorType != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) {
HotwordMetricsLogger.writeDetectorEvent(mDetectorType,
HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_BIND_SERVICE,
mVoiceInteractionServiceUid);
}
String instancePrefix =
mIntent.getAction().equals(HotwordDetectionService.SERVICE_INTERFACE)
? "hotword_detector_" : "visual_query_detector_";
boolean bindResult = mContext.bindIsolatedService(
mIntent,
Context.BIND_AUTO_CREATE | Context.BIND_FOREGROUND_SERVICE | mBindingFlags,
"hotword_detector_" + mInstanceNumber,
instancePrefix + mInstanceNumber,
mExecutor,
serviceConnection);
if (!bindResult) {
if (mDetectorType != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) {
HotwordMetricsLogger.writeDetectorEvent(mDetectorType,
HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_BIND_SERVICE_FAIL,
mVoiceInteractionServiceUid);
}
}
return bindResult;
} catch (IllegalArgumentException e) {
if (mDetectorType != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) {
HotwordMetricsLogger.writeDetectorEvent(mDetectorType,
HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_BIND_SERVICE_FAIL,
mVoiceInteractionServiceUid);
}
return bindResult;
} catch (IllegalArgumentException e) {
HotwordMetricsLogger.writeDetectorEvent(mDetectorType,
HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_BIND_SERVICE_FAIL,
mVoiceInteractionServiceUid);
Slog.wtf(TAG, "Can't bind to the hotword detection service!", e);
return false;
}
@@ -609,10 +687,27 @@ final class HotwordDetectionConnection {
}
final DetectorSession session;
if (detectorType == HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP) {
if (mRemoteHotwordDetectionService == null) {
mRemoteHotwordDetectionService =
mHotwordDetectionServiceConnectionFactory.createLocked();
}
session = new DspTrustedHotwordDetectorSession(mRemoteHotwordDetectionService,
mLock, mContext, token, callback, mVoiceInteractionServiceUid,
mVoiceInteractorIdentity, mScheduledExecutorService, mDebugHotwordLogging);
} else if (detectorType == HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) {
if (mRemoteVisualQueryDetectionService == null) {
mRemoteVisualQueryDetectionService =
mVisualQueryDetectionServiceConnectionFactory.createLocked();
}
session = new VisualQueryDetectorSession(
mRemoteVisualQueryDetectionService, mLock, mContext, token, callback,
mVoiceInteractionServiceUid, mVoiceInteractorIdentity,
mScheduledExecutorService, mDebugHotwordLogging);
} else {
if (mRemoteHotwordDetectionService == null) {
mRemoteHotwordDetectionService =
mHotwordDetectionServiceConnectionFactory.createLocked();
}
session = new SoftwareTrustedHotwordDetectorSession(
mRemoteHotwordDetectionService, mLock, mContext, token, callback,
mVoiceInteractionServiceUid, mVoiceInteractorIdentity,
@@ -625,13 +720,23 @@ final class HotwordDetectionConnection {
@SuppressWarnings("GuardedBy")
void destroyDetectorLocked(@NonNull IBinder token) {
final DetectorSession session = getDetectorSessionByTokenLocked(token);
if (session != null) {
session.destroyLocked();
final int index = mDetectorSessions.indexOfValue(session);
if (index < 0 || index > mDetectorSessions.size() - 1) {
return;
}
mDetectorSessions.removeAt(index);
if (session == null) {
return;
}
session.destroyLocked();
final int index = mDetectorSessions.indexOfValue(session);
if (index < 0 || index > mDetectorSessions.size() - 1) {
return;
}
mDetectorSessions.removeAt(index);
if (session instanceof VisualQueryDetectorSession) {
unbindVisualQueryDetectionService();
}
// Handle case where all hotword detector sessions are destroyed with only the visual
// detector session left
if (mDetectorSessions.size() == 1
&& mDetectorSessions.get(0) instanceof VisualQueryDetectorSession) {
unbindHotwordDetectionService();
}
}
@@ -672,6 +777,15 @@ final class HotwordDetectionConnection {
}
@SuppressWarnings("GuardedBy")
private VisualQueryDetectorSession getVisualQueryDetectorSessionLocked() {
final DetectorSession session = mDetectorSessions.get(
HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR);
if (session == null || session.isDestroyed()) {
Slog.v(TAG, "Not found the look and talk perceiver");
return null;
}
return (VisualQueryDetectorSession) session;
}
private void runForEachDetectorSessionLocked(
@NonNull Consumer<DetectorSession> action) {
for (int i = 0; i < mDetectorSessions.size(); i++) {

87
services/voiceinteraction/java/com/android/server/voiceinteraction/VisualQueryDetectorSession.java Normal file
View File

@@ -0,0 +1,87 @@
/*
* Copyright (C) 2023 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.server.voiceinteraction;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.content.Context;
import android.media.AudioFormat;
import android.media.permission.Identity;
import android.os.IBinder;
import android.os.ParcelFileDescriptor;
import android.os.PersistableBundle;
import android.os.SharedMemory;
import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback;
import android.util.Slog;
import com.android.internal.app.IHotwordRecognitionStatusCallback;
import java.io.PrintWriter;
import java.util.concurrent.ScheduledExecutorService;
/**
* A class that provides visual query detector to communicate with the {@link
* android.service.voice.VisualQueryDetectionService}.
*
* This class can handle the visual query detection whose detector is created by using
* {@link android.service.voice.VoiceInteractionService#createVisualQueryDetector(PersistableBundle
* ,SharedMemory, HotwordDetector.Callback)}.
*/
final class VisualQueryDetectorSession extends DetectorSession {
private static final String TAG = "VisualQueryDetectorSession";
//TODO(b/261783819): Determines actual functionalities, e.g., startRecognition etc.
VisualQueryDetectorSession(
@NonNull HotwordDetectionConnection.ServiceConnection remoteService,
@NonNull Object lock, @NonNull Context context, @NonNull IBinder token,
@NonNull IHotwordRecognitionStatusCallback callback, int voiceInteractionServiceUid,
Identity voiceInteractorIdentity,
@NonNull ScheduledExecutorService scheduledExecutorService, boolean logging) {
super(remoteService, lock, context, token, callback,
voiceInteractionServiceUid, voiceInteractorIdentity, scheduledExecutorService,
logging);
}
@Override
@SuppressWarnings("GuardedBy")
void informRestartProcessLocked() {
Slog.v(TAG, "informRestartProcessLocked");
mUpdateStateAfterStartFinished.set(false);
//TODO(b/261783819): Starts detection in VisualQueryDetectionService.
}
@Override
void startListeningFromExternalSourceLocked(
ParcelFileDescriptor audioStream,
AudioFormat audioFormat,
@Nullable PersistableBundle options,
IMicrophoneHotwordDetectionVoiceInteractionCallback callback)
throws UnsupportedOperationException {
throw new UnsupportedOperationException("HotwordDetectionService method"
+ " should not be called from VisualQueryDetectorSession.");
}
@SuppressWarnings("GuardedBy")
public void dumpLocked(String prefix, PrintWriter pw) {
super.dumpLocked(prefix, pw);
pw.print(prefix);
}
}

62
services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerServiceImpl.java
View File

@@ -58,6 +58,7 @@ import android.os.RemoteException;
import android.os.ServiceManager;
import android.os.SharedMemory;
import android.os.UserHandle;
import android.service.voice.HotwordDetector;
import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback;
import android.service.voice.IVoiceInteractionService;
import android.service.voice.IVoiceInteractionSession;
@@ -109,6 +110,7 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
final ComponentName mSessionComponentName;
final IWindowManager mIWindowManager;
final ComponentName mHotwordDetectionComponentName;
final ComponentName mVisualQueryDetectionComponentName;
boolean mBound = false;
IVoiceInteractionService mService;
volatile HotwordDetectionConnection mHotwordDetectionConnection;
@@ -211,6 +213,7 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
mInfo = null;
mSessionComponentName = null;
mHotwordDetectionComponentName = null;
mVisualQueryDetectionComponentName = null;
mIWindowManager = null;
mValid = false;
return;
@@ -220,6 +223,7 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
Slog.w(TAG, "Bad voice interaction service: " + mInfo.getParseError());
mSessionComponentName = null;
mHotwordDetectionComponentName = null;
mVisualQueryDetectionComponentName = null;
mIWindowManager = null;
mValid = false;
return;
@@ -230,6 +234,9 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
final String hotwordDetectionServiceName = mInfo.getHotwordDetectionService();
mHotwordDetectionComponentName = hotwordDetectionServiceName != null
? new ComponentName(service.getPackageName(), hotwordDetectionServiceName) : null;
final String visualQueryDetectionServiceName = mInfo.getVisualQueryDetectionService();
mVisualQueryDetectionComponentName = visualQueryDetectionServiceName != null ? new
ComponentName(service.getPackageName(), visualQueryDetectionServiceName) : null;
mIWindowManager = IWindowManager.Stub.asInterface(
ServiceManager.getService(Context.WINDOW_SERVICE));
IntentFilter filter = new IntentFilter();
@@ -573,14 +580,11 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
}
}
public void initAndVerifyDetectorLocked(
@NonNull Identity voiceInteractorIdentity,
@Nullable PersistableBundle options,
private void verifyDetectorForHotwordDetectionLocked(
@Nullable SharedMemory sharedMemory,
@NonNull IBinder token,
IHotwordRecognitionStatusCallback callback,
int detectorType) {
Slog.v(TAG, "initAndVerifyDetectorLocked");
Slog.v(TAG, "verifyDetectorForHotwordDetectionLocked");
int voiceInteractionServiceUid = mInfo.getServiceInfo().applicationInfo.uid;
if (mHotwordDetectionComponentName == null) {
Slog.w(TAG, "Hotword detection service name not found");
@@ -631,11 +635,55 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
logDetectorCreateEventIfNeeded(callback, detectorType, true,
voiceInteractionServiceUid);
}
private void verifyDetectorForVisualQueryDetectionLocked(@Nullable SharedMemory sharedMemory) {
Slog.v(TAG, "verifyDetectorForVisualQueryDetectionLocked");
if (mVisualQueryDetectionComponentName == null) {
Slog.w(TAG, "Visual query detection service name not found");
throw new IllegalStateException("Visual query detection service name not found");
}
ServiceInfo visualQueryDetectionServiceInfo = getServiceInfoLocked(
mVisualQueryDetectionComponentName, mUser);
if (visualQueryDetectionServiceInfo == null) {
Slog.w(TAG, "Visual query detection service info not found");
throw new IllegalStateException("Visual query detection service name not found");
}
if (!isIsolatedProcessLocked(visualQueryDetectionServiceInfo)) {
Slog.w(TAG, "Visual query detection service not in isolated process");
throw new IllegalStateException("Visual query detection not in isolated process");
}
if (sharedMemory != null && !sharedMemory.setProtect(OsConstants.PROT_READ)) {
Slog.w(TAG, "Can't set sharedMemory to be read-only");
throw new IllegalStateException("Can't set sharedMemory to be read-only");
}
}
public void initAndVerifyDetectorLocked(
@NonNull Identity voiceInteractorIdentity,
@Nullable PersistableBundle options,
@Nullable SharedMemory sharedMemory,
@NonNull IBinder token,
IHotwordRecognitionStatusCallback callback,
int detectorType) {
if (detectorType != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) {
verifyDetectorForHotwordDetectionLocked(sharedMemory, callback, detectorType);
} else {
verifyDetectorForVisualQueryDetectionLocked(sharedMemory);
}
if (mHotwordDetectionConnection == null) {
mHotwordDetectionConnection = new HotwordDetectionConnection(mServiceStub, mContext,
mInfo.getServiceInfo().applicationInfo.uid, voiceInteractorIdentity,
mHotwordDetectionComponentName, mUser, /* bindInstantServiceAllowed= */ false,
detectorType);
mHotwordDetectionComponentName, mVisualQueryDetectionComponentName, mUser,
/* bindInstantServiceAllowed= */ false, detectorType);
} else if (detectorType != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) {
// TODO: Logger events should be handled in session instead. Temporary adding the
// checking to prevent confusion so VisualQueryDetection events won't be logged if the
// connection is instantiated by the VisualQueryDetector.
mHotwordDetectionConnection.setDetectorType(detectorType);
}
mHotwordDetectionConnection.createDetectorLocked(options, sharedMemory, token, callback,
detectorType);