Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge "Integrate VQDS lifecycle into connection session"

Browse Source
This commit is contained in:
Charles Chen
2023-01-14 06:06:25 +00:00
committed by Android (Google) Code Review
parent f98fc3d1fd fd9097a758
commit ea43c801b6
5 changed files with 445 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files

81
core/java/android/service/voice/VisualQueryDetector.java
View File

@@ -25,12 +25,16 @@ import android.annotation.Nullable;
import android.annotation.RequiresPermission;
import android.annotation.SuppressLint;
import android.annotation.SystemApi;
import android.hardware.soundtrigger.SoundTrigger;
import android.media.AudioFormat;
import android.os.Binder;
import android.os.ParcelFileDescriptor;
import android.os.PersistableBundle;
import android.os.RemoteException;
import android.os.SharedMemory;
import android.util.Slog;
import com.android.internal.app.IHotwordRecognitionStatusCallback;
import com.android.internal.app.IVoiceInteractionManagerService;
import java.io.PrintWriter;
@@ -208,8 +212,9 @@ public class VisualQueryDetector {
@Override
void initialize(@Nullable PersistableBundle options, @Nullable SharedMemory sharedMemory) {
//TODO(261783492): call initAndVerify to create VisualQueryDetectionService
// from the system server.
initAndVerifyDetector(options, sharedMemory,
new InitializationStateListener(mExecutor, mCallback),
DETECTOR_TYPE_VISUAL_QUERY_DETECTOR);
}
@Override
@@ -238,4 +243,76 @@ public class VisualQueryDetector {
return true;
}
}
private static class InitializationStateListener
extends IHotwordRecognitionStatusCallback.Stub {
private final Executor mExecutor;
private final Callback mCallback;
InitializationStateListener(Executor executor, Callback callback) {
this.mExecutor = executor;
this.mCallback = callback;
}
@Override
public void onKeyphraseDetected(
SoundTrigger.KeyphraseRecognitionEvent recognitionEvent,
HotwordDetectedResult result) {
if (DEBUG) {
Slog.i(TAG, "Ignored #onKeyphraseDetected event");
}
}
@Override
public void onGenericSoundTriggerDetected(
SoundTrigger.GenericRecognitionEvent recognitionEvent) throws RemoteException {
if (DEBUG) {
Slog.i(TAG, "Ignored #onGenericSoundTriggerDetected event");
}
}
@Override
public void onRejected(HotwordRejectedResult result) throws RemoteException {
if (DEBUG) {
Slog.i(TAG, "Ignored #onRejected event");
}
}
@Override
public void onRecognitionPaused() throws RemoteException {
if (DEBUG) {
Slog.i(TAG, "Ignored #onRecognitionPaused event");
}
}
@Override
public void onRecognitionResumed() throws RemoteException {
if (DEBUG) {
Slog.i(TAG, "Ignored #onRecognitionResumed event");
}
}
@Override
public void onStatusReported(int status) {
Slog.v(TAG, "onStatusReported" + (DEBUG ? "(" + status + ")" : ""));
//TODO: rename the target callback with a more general term
Binder.withCleanCallingIdentity(() -> mExecutor.execute(
() -> mCallback.onVisualQueryDetectionServiceInitialized(status)));
}
@Override
public void onProcessRestarted() throws RemoteException {
Slog.v(TAG, "onProcessRestarted()");
//TODO: rename the target callback with a more general term
Binder.withCleanCallingIdentity(() -> mExecutor.execute(
() -> mCallback.onVisualQueryDetectionServiceRestarted()));
}
@Override
public void onError(int status) throws RemoteException {
Slog.v(TAG, "Initialization Error: (" + status + ")");
// Do nothing
}
}
}

95
services/voiceinteraction/java/com/android/server/voiceinteraction/DetectorSession.java
View File

@@ -205,9 +205,15 @@ abstract class DetectorSession {
mVoiceInteractionServiceUid = voiceInteractionServiceUid;
mVoiceInteractorIdentity = voiceInteractorIdentity;
mAppOpsManager = mContext.getSystemService(AppOpsManager.class);
mHotwordAudioStreamCopier = new HotwordAudioStreamCopier(mAppOpsManager, getDetectorType(),
mVoiceInteractorIdentity.uid, mVoiceInteractorIdentity.packageName,
mVoiceInteractorIdentity.attributionTag);
if (getDetectorType() != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) {
mHotwordAudioStreamCopier = new HotwordAudioStreamCopier(mAppOpsManager,
getDetectorType(),
mVoiceInteractorIdentity.uid, mVoiceInteractorIdentity.packageName,
mVoiceInteractorIdentity.attributionTag);
} else {
mHotwordAudioStreamCopier = null;
}
mScheduledExecutorService = scheduledExecutorService;
mDebugHotwordLogging = logging;
@@ -236,9 +242,12 @@ abstract class DetectorSession {
future.complete(null);
if (mUpdateStateAfterStartFinished.getAndSet(true)) {
Slog.w(TAG, "call callback after timeout");
HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
if (getDetectorType()
!= HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) {
HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_UPDATE_STATE_AFTER_TIMEOUT,
mVoiceInteractionServiceUid);
}
return;
}
Pair<Integer, Integer> statusResultPair = getInitStatusAndMetricsResult(bundle);
@@ -246,27 +255,37 @@ abstract class DetectorSession {
int initResultMetricsResult = statusResultPair.second;
try {
mCallback.onStatusReported(status);
HotwordMetricsLogger.writeServiceInitResultEvent(getDetectorType(),
initResultMetricsResult, mVoiceInteractionServiceUid);
if (getDetectorType()
!= HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) {
HotwordMetricsLogger.writeServiceInitResultEvent(getDetectorType(),
initResultMetricsResult, mVoiceInteractionServiceUid);
}
} catch (RemoteException e) {
Slog.w(TAG, "Failed to report initialization status: " + e);
HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
METRICS_CALLBACK_ON_STATUS_REPORTED_EXCEPTION,
mVoiceInteractionServiceUid);
if (getDetectorType()
!= HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) {
HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
METRICS_CALLBACK_ON_STATUS_REPORTED_EXCEPTION,
mVoiceInteractionServiceUid);
}
}
}
};
try {
service.updateState(options, sharedMemory, statusCallback);
HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_UPDATE_STATE,
mVoiceInteractionServiceUid);
if (getDetectorType() != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) {
HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_UPDATE_STATE,
mVoiceInteractionServiceUid);
}
} catch (RemoteException e) {
// TODO: (b/181842909) Report an error to voice interactor
Slog.w(TAG, "Failed to updateState for HotwordDetectionService", e);
HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
HOTWORD_DETECTOR_EVENTS__EVENT__CALL_UPDATE_STATE_EXCEPTION,
mVoiceInteractionServiceUid);
if (getDetectorType() != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) {
HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
HOTWORD_DETECTOR_EVENTS__EVENT__CALL_UPDATE_STATE_EXCEPTION,
mVoiceInteractionServiceUid);
}
}
return future.orTimeout(MAX_UPDATE_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS);
}).whenComplete((res, err) -> {
@@ -277,13 +296,17 @@ abstract class DetectorSession {
}
try {
mCallback.onStatusReported(INITIALIZATION_STATUS_UNKNOWN);
HotwordMetricsLogger.writeServiceInitResultEvent(getDetectorType(),
METRICS_INIT_UNKNOWN_TIMEOUT, mVoiceInteractionServiceUid);
if (getDetectorType() != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) {
HotwordMetricsLogger.writeServiceInitResultEvent(getDetectorType(),
METRICS_INIT_UNKNOWN_TIMEOUT, mVoiceInteractionServiceUid);
}
} catch (RemoteException e) {
Slog.w(TAG, "Failed to report initialization status UNKNOWN", e);
HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
METRICS_CALLBACK_ON_STATUS_REPORTED_EXCEPTION,
mVoiceInteractionServiceUid);
if (getDetectorType() != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) {
HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
METRICS_CALLBACK_ON_STATUS_REPORTED_EXCEPTION,
mVoiceInteractionServiceUid);
}
}
} else if (err != null) {
Slog.w(TAG, "Failed to update state: " + err);
@@ -315,9 +338,11 @@ abstract class DetectorSession {
@SuppressWarnings("GuardedBy")
void updateStateLocked(PersistableBundle options, SharedMemory sharedMemory,
Instant lastRestartInstant) {
HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
HOTWORD_DETECTOR_EVENTS__EVENT__APP_REQUEST_UPDATE_STATE,
mVoiceInteractionServiceUid);
if (getDetectorType() != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) {
HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
HOTWORD_DETECTOR_EVENTS__EVENT__APP_REQUEST_UPDATE_STATE,
mVoiceInteractionServiceUid);
}
// Prevent doing the init late, so restart is handled equally to a clean process start.
// TODO(b/191742511): this logic needs a test
if (!mUpdateStateAfterStartFinished.get() && Instant.now().minus(
@@ -399,9 +424,11 @@ abstract class DetectorSession {
callback.onError();
} catch (RemoteException ex) {
Slog.w(TAG, "Failed to report onError status: " + ex);
HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION,
mVoiceInteractionServiceUid);
if (getDetectorType() != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) {
HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION,
mVoiceInteractionServiceUid);
}
}
} finally {
synchronized (mLock) {
@@ -427,7 +454,8 @@ abstract class DetectorSession {
throws RemoteException {
synchronized (mLock) {
mPerformingExternalSourceHotwordDetection = false;
HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
HotwordMetricsLogger.writeDetectorEvent(
getDetectorType(),
METRICS_EXTERNAL_SOURCE_REJECTED,
mVoiceInteractionServiceUid);
mScheduledExecutorService.schedule(
@@ -454,7 +482,8 @@ abstract class DetectorSession {
throws RemoteException {
synchronized (mLock) {
mPerformingExternalSourceHotwordDetection = false;
HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
HotwordMetricsLogger.writeDetectorEvent(
getDetectorType(),
METRICS_EXTERNAL_SOURCE_DETECTED,
mVoiceInteractionServiceUid);
mScheduledExecutorService.schedule(
@@ -540,9 +569,11 @@ abstract class DetectorSession {
mCallback.onError(status);
} catch (RemoteException e) {
Slog.w(TAG, "Failed to report onError status: " + e);
HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION,
mVoiceInteractionServiceUid);
if (getDetectorType() != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) {
HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION,
mVoiceInteractionServiceUid);
}
}
}
@@ -679,6 +710,8 @@ abstract class DetectorSession {
return HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP;
} else if (this instanceof SoftwareTrustedHotwordDetectorSession) {
return HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE;
} else if (this instanceof VisualQueryDetectorSession) {
return HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR;
}
Slog.v(TAG, "Unexpected detector type");
return -1;

206
services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java
View File

@@ -52,6 +52,7 @@ import android.service.voice.HotwordDetectionService;
import android.service.voice.HotwordDetector;
import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback;
import android.service.voice.ISandboxedDetectionService;
import android.service.voice.VisualQueryDetectionService;
import android.service.voice.VoiceInteractionManagerInternal.HotwordDetectionServiceIdentity;
import android.speech.IRecognitionServiceManager;
import android.util.Slog;
@@ -74,7 +75,8 @@ import java.util.function.Consumer;
import java.util.function.Function;
/**
* A class that provides the communication with the HotwordDetectionService.
* A class that provides the communication with the {@link HotwordDetectionService} and
* {@link VisualQueryDetectionService}.
*/
final class HotwordDetectionConnection {
private static final String TAG = "HotwordDetectionConnection";
@@ -90,7 +92,8 @@ final class HotwordDetectionConnection {
@Nullable private final ScheduledFuture<?> mCancellationTaskFuture;
private final IBinder.DeathRecipient mAudioServerDeathRecipient = this::audioServerDied;
@NonNull private final ServiceConnectionFactory mHotwordDetectionServiceConnectionFactory;
private final int mDetectorType;
@NonNull private final ServiceConnectionFactory mVisualQueryDetectionServiceConnectionFactory;
private int mDetectorType;
/**
* Time after which each HotwordDetectionService process is stopped and replaced by a new one.
* 0 indicates no restarts.
@@ -100,9 +103,11 @@ final class HotwordDetectionConnection {
final Object mLock;
final int mVoiceInteractionServiceUid;
final ComponentName mHotwordDetectionComponentName;
final ComponentName mVisualQueryDetectionComponentName;
final int mUser;
final Context mContext;
volatile HotwordDetectionServiceIdentity mIdentity;
//TODO: add similar identity for visual query service for the use of content capturing
private Instant mLastRestartInstant;
private ScheduledFuture<?> mDebugHotwordLoggingTimeoutFuture = null;
@@ -112,6 +117,7 @@ final class HotwordDetectionConnection {
@Nullable
private final Identity mVoiceInteractorIdentity;
@NonNull private ServiceConnection mRemoteHotwordDetectionService;
@NonNull private ServiceConnection mRemoteVisualQueryDetectionService;
private IBinder mAudioFlinger;
@GuardedBy("mLock")
private boolean mDebugHotwordLogging = false;
@@ -126,26 +132,39 @@ final class HotwordDetectionConnection {
new SparseArray<>();
HotwordDetectionConnection(Object lock, Context context, int voiceInteractionServiceUid,
Identity voiceInteractorIdentity, ComponentName hotwordDetectionServiceName, int userId,
Identity voiceInteractorIdentity, ComponentName hotwordDetectionServiceName,
ComponentName visualQueryDetectionServiceName, int userId,
boolean bindInstantServiceAllowed, int detectorType) {
mLock = lock;
mContext = context;
mVoiceInteractionServiceUid = voiceInteractionServiceUid;
mVoiceInteractorIdentity = voiceInteractorIdentity;
mHotwordDetectionComponentName = hotwordDetectionServiceName;
mVisualQueryDetectionComponentName = visualQueryDetectionServiceName;
mUser = userId;
mDetectorType = detectorType;
mReStartPeriodSeconds = DeviceConfig.getInt(DeviceConfig.NAMESPACE_VOICE_INTERACTION,
KEY_RESTART_PERIOD_IN_SECONDS, 0);
final Intent hotwordDetectionServiceIntent =
new Intent(HotwordDetectionService.SERVICE_INTERFACE);
hotwordDetectionServiceIntent.setComponent(mHotwordDetectionComponentName);
final Intent visualQueryDetectionServiceIntent =
new Intent(VisualQueryDetectionService.SERVICE_INTERFACE);
visualQueryDetectionServiceIntent.setComponent(mVisualQueryDetectionComponentName);
initAudioFlingerLocked();
mHotwordDetectionServiceConnectionFactory =
new ServiceConnectionFactory(hotwordDetectionServiceIntent,
bindInstantServiceAllowed);
mRemoteHotwordDetectionService = mHotwordDetectionServiceConnectionFactory.createLocked();
mVisualQueryDetectionServiceConnectionFactory =
new ServiceConnectionFactory(visualQueryDetectionServiceIntent,
bindInstantServiceAllowed);
mLastRestartInstant = Instant.now();
if (mReStartPeriodSeconds <= 0) {
@@ -157,9 +176,11 @@ final class HotwordDetectionConnection {
Slog.v(TAG, "Time to restart the process, TTL has passed");
synchronized (mLock) {
restartProcessLocked();
HotwordMetricsLogger.writeServiceRestartEvent(mDetectorType,
HOTWORD_DETECTION_SERVICE_RESTARTED__REASON__SCHEDULE,
mVoiceInteractionServiceUid);
if (mDetectorType != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) {
HotwordMetricsLogger.writeServiceRestartEvent(mDetectorType,
HOTWORD_DETECTION_SERVICE_RESTARTED__REASON__SCHEDULE,
mVoiceInteractionServiceUid);
}
}
}, mReStartPeriodSeconds, mReStartPeriodSeconds, TimeUnit.SECONDS);
}
@@ -193,9 +214,11 @@ final class HotwordDetectionConnection {
// We restart the process instead of simply sending over the new binder, to avoid race
// conditions with audio reading in the service.
restartProcessLocked();
HotwordMetricsLogger.writeServiceRestartEvent(mDetectorType,
HOTWORD_DETECTION_SERVICE_RESTARTED__REASON__AUDIO_SERVICE_DIED,
mVoiceInteractionServiceUid);
if (mDetectorType != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) {
HotwordMetricsLogger.writeServiceRestartEvent(mDetectorType,
HOTWORD_DETECTION_SERVICE_RESTARTED__REASON__AUDIO_SERVICE_DIED,
mVoiceInteractionServiceUid);
}
}
}
@@ -208,9 +231,8 @@ final class HotwordDetectionConnection {
});
mDetectorSessions.clear();
mDebugHotwordLogging = false;
mRemoteHotwordDetectionService.unbind();
LocalServices.getService(PermissionManagerServiceInternal.class)
.setHotwordDetectionServiceProvider(null);
unbindVisualQueryDetectionService();
unbindHotwordDetectionService();
if (mIdentity != null) {
removeServiceUidForAudioPolicy(mIdentity.getIsolatedUid());
}
@@ -223,6 +245,21 @@ final class HotwordDetectionConnection {
}
}
private void unbindVisualQueryDetectionService() {
if (mRemoteVisualQueryDetectionService != null) {
mRemoteVisualQueryDetectionService.unbind();
//TODO: Set visual query detection service provider to null
}
}
private void unbindHotwordDetectionService() {
if (mRemoteHotwordDetectionService != null) {
mRemoteHotwordDetectionService.unbind();
LocalServices.getService(PermissionManagerServiceInternal.class)
.setHotwordDetectionServiceProvider(null);
}
}
@SuppressWarnings("GuardedBy")
void updateStateLocked(PersistableBundle options, SharedMemory sharedMemory,
@NonNull IBinder token) {
@@ -342,6 +379,10 @@ final class HotwordDetectionConnection {
}
}
void setDetectorType(int detectorType) {
mDetectorType = detectorType;
}
private void clearDebugHotwordLoggingTimeoutLocked() {
if (mDebugHotwordLoggingTimeoutFuture != null) {
mDebugHotwordLoggingTimeoutFuture.cancel(/* mayInterruptIfRunning= */ true);
@@ -353,24 +394,41 @@ final class HotwordDetectionConnection {
private void restartProcessLocked() {
// TODO(b/244598068): Check HotwordAudioStreamManager first
Slog.v(TAG, "Restarting hotword detection process");
ServiceConnection oldHotwordConnection = mRemoteHotwordDetectionService;
ServiceConnection oldVisualQueryDetectionConnection = mRemoteVisualQueryDetectionService;
HotwordDetectionServiceIdentity previousIdentity = mIdentity;
//TODO: Add previousIdentity for visual query detection service
mLastRestartInstant = Instant.now();
// Recreate connection to reset the cache.
mRemoteHotwordDetectionService = mHotwordDetectionServiceConnectionFactory.createLocked();
mRemoteVisualQueryDetectionService =
mVisualQueryDetectionServiceConnectionFactory.createLocked();
Slog.v(TAG, "Started the new process, dispatching processRestarted to detector");
runForEachDetectorSessionLocked((session) -> {
session.updateRemoteSandboxedDetectionServiceLocked(mRemoteHotwordDetectionService);
HotwordDetectionConnection.ServiceConnection newRemoteService =
(session instanceof VisualQueryDetectorSession)
? mRemoteVisualQueryDetectionService : mRemoteHotwordDetectionService;
session.updateRemoteSandboxedDetectionServiceLocked(newRemoteService);
session.informRestartProcessLocked();
});
if (DEBUG) {
Slog.i(TAG, "processRestarted is dispatched done, unbinding from the old process");
}
oldHotwordConnection.ignoreConnectionStatusEvents();
oldHotwordConnection.unbind();
if (oldHotwordConnection != null) {
oldHotwordConnection.ignoreConnectionStatusEvents();
oldHotwordConnection.unbind();
}
if (oldVisualQueryDetectionConnection != null) {
oldVisualQueryDetectionConnection.ignoreConnectionStatusEvents();
oldVisualQueryDetectionConnection.unbind();
}
if (previousIdentity != null) {
removeServiceUidForAudioPolicy(previousIdentity.getIsolatedUid());
}
@@ -438,9 +496,14 @@ final class HotwordDetectionConnection {
synchronized (mLock) {
pw.print(prefix); pw.print("mReStartPeriodSeconds="); pw.println(mReStartPeriodSeconds);
pw.print(prefix); pw.print("mBound=");
pw.println(mRemoteHotwordDetectionService.isBound());
pw.println(mRemoteHotwordDetectionService != null
&& mRemoteHotwordDetectionService.isBound());
pw.println(mRemoteVisualQueryDetectionService != null
&& mRemoteHotwordDetectionService != null
&& mRemoteHotwordDetectionService.isBound());
pw.print(prefix); pw.print("mRestartCount=");
pw.println(mHotwordDetectionServiceConnectionFactory.mRestartCount);
pw.println(mVisualQueryDetectionServiceConnectionFactory.mRestartCount);
pw.print(prefix); pw.print("mLastRestartInstant="); pw.println(mLastRestartInstant);
pw.print(prefix); pw.print("mDetectorType=");
pw.println(HotwordDetector.detectorTypeToString(mDetectorType));
@@ -489,11 +552,11 @@ final class HotwordDetectionConnection {
private boolean mIsLoggedFirstConnect = false;
ServiceConnection(@NonNull Context context,
@NonNull Intent intent, int bindingFlags, int userId,
@NonNull Intent serviceIntent, int bindingFlags, int userId,
@Nullable Function<IBinder, ISandboxedDetectionService> binderAsInterface,
int instanceNumber) {
super(context, intent, bindingFlags, userId, binderAsInterface);
this.mIntent = intent;
super(context, serviceIntent, bindingFlags, userId, binderAsInterface);
this.mIntent = serviceIntent;
this.mBindingFlags = bindingFlags;
this.mInstanceNumber = instanceNumber;
}
@@ -512,14 +575,18 @@ final class HotwordDetectionConnection {
mIsBound = connected;
if (!connected) {
HotwordMetricsLogger.writeDetectorEvent(mDetectorType,
HOTWORD_DETECTOR_EVENTS__EVENT__ON_DISCONNECTED,
mVoiceInteractionServiceUid);
if (mDetectorType != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) {
HotwordMetricsLogger.writeDetectorEvent(mDetectorType,
HOTWORD_DETECTOR_EVENTS__EVENT__ON_DISCONNECTED,
mVoiceInteractionServiceUid);
}
} else if (!mIsLoggedFirstConnect) {
mIsLoggedFirstConnect = true;
HotwordMetricsLogger.writeDetectorEvent(mDetectorType,
HOTWORD_DETECTOR_EVENTS__EVENT__ON_CONNECTED,
mVoiceInteractionServiceUid);
if (mDetectorType != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) {
HotwordMetricsLogger.writeDetectorEvent(mDetectorType,
HOTWORD_DETECTOR_EVENTS__EVENT__ON_CONNECTED,
mVoiceInteractionServiceUid);
}
}
}
}
@@ -546,35 +613,46 @@ final class HotwordDetectionConnection {
});
}
// Can improve to log exit reason if needed
HotwordMetricsLogger.writeKeyphraseTriggerEvent(
mDetectorType,
HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__SERVICE_CRASH,
mVoiceInteractionServiceUid);
if (mDetectorType != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) {
HotwordMetricsLogger.writeKeyphraseTriggerEvent(
mDetectorType,
HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__SERVICE_CRASH,
mVoiceInteractionServiceUid);
}
}
@Override
protected boolean bindService(
@NonNull android.content.ServiceConnection serviceConnection) {
try {
HotwordMetricsLogger.writeDetectorEvent(mDetectorType,
HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_BIND_SERVICE,
mVoiceInteractionServiceUid);
if (mDetectorType != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) {
HotwordMetricsLogger.writeDetectorEvent(mDetectorType,
HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_BIND_SERVICE,
mVoiceInteractionServiceUid);
}
String instancePrefix =
mIntent.getAction().equals(HotwordDetectionService.SERVICE_INTERFACE)
? "hotword_detector_" : "visual_query_detector_";
boolean bindResult = mContext.bindIsolatedService(
mIntent,
Context.BIND_AUTO_CREATE | Context.BIND_FOREGROUND_SERVICE | mBindingFlags,
"hotword_detector_" + mInstanceNumber,
instancePrefix + mInstanceNumber,
mExecutor,
serviceConnection);
if (!bindResult) {
if (mDetectorType != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) {
HotwordMetricsLogger.writeDetectorEvent(mDetectorType,
HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_BIND_SERVICE_FAIL,
mVoiceInteractionServiceUid);
}
}
return bindResult;
} catch (IllegalArgumentException e) {
if (mDetectorType != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) {
HotwordMetricsLogger.writeDetectorEvent(mDetectorType,
HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_BIND_SERVICE_FAIL,
mVoiceInteractionServiceUid);
}
return bindResult;
} catch (IllegalArgumentException e) {
HotwordMetricsLogger.writeDetectorEvent(mDetectorType,
HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_BIND_SERVICE_FAIL,
mVoiceInteractionServiceUid);
Slog.wtf(TAG, "Can't bind to the hotword detection service!", e);
return false;
}
@@ -609,10 +687,27 @@ final class HotwordDetectionConnection {
}
final DetectorSession session;
if (detectorType == HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP) {
if (mRemoteHotwordDetectionService == null) {
mRemoteHotwordDetectionService =
mHotwordDetectionServiceConnectionFactory.createLocked();
}
session = new DspTrustedHotwordDetectorSession(mRemoteHotwordDetectionService,
mLock, mContext, token, callback, mVoiceInteractionServiceUid,
mVoiceInteractorIdentity, mScheduledExecutorService, mDebugHotwordLogging);
} else if (detectorType == HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) {
if (mRemoteVisualQueryDetectionService == null) {
mRemoteVisualQueryDetectionService =
mVisualQueryDetectionServiceConnectionFactory.createLocked();
}
session = new VisualQueryDetectorSession(
mRemoteVisualQueryDetectionService, mLock, mContext, token, callback,
mVoiceInteractionServiceUid, mVoiceInteractorIdentity,
mScheduledExecutorService, mDebugHotwordLogging);
} else {
if (mRemoteHotwordDetectionService == null) {
mRemoteHotwordDetectionService =
mHotwordDetectionServiceConnectionFactory.createLocked();
}
session = new SoftwareTrustedHotwordDetectorSession(
mRemoteHotwordDetectionService, mLock, mContext, token, callback,
mVoiceInteractionServiceUid, mVoiceInteractorIdentity,
@@ -625,13 +720,23 @@ final class HotwordDetectionConnection {
@SuppressWarnings("GuardedBy")
void destroyDetectorLocked(@NonNull IBinder token) {
final DetectorSession session = getDetectorSessionByTokenLocked(token);
if (session != null) {
session.destroyLocked();
final int index = mDetectorSessions.indexOfValue(session);
if (index < 0 || index > mDetectorSessions.size() - 1) {
return;
}
mDetectorSessions.removeAt(index);
if (session == null) {
return;
}
session.destroyLocked();
final int index = mDetectorSessions.indexOfValue(session);
if (index < 0 || index > mDetectorSessions.size() - 1) {
return;
}
mDetectorSessions.removeAt(index);
if (session instanceof VisualQueryDetectorSession) {
unbindVisualQueryDetectionService();
}
// Handle case where all hotword detector sessions are destroyed with only the visual
// detector session left
if (mDetectorSessions.size() == 1
&& mDetectorSessions.get(0) instanceof VisualQueryDetectorSession) {
unbindHotwordDetectionService();
}
}
@@ -672,6 +777,15 @@ final class HotwordDetectionConnection {
}
@SuppressWarnings("GuardedBy")
private VisualQueryDetectorSession getVisualQueryDetectorSessionLocked() {
final DetectorSession session = mDetectorSessions.get(
HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR);
if (session == null || session.isDestroyed()) {
Slog.v(TAG, "Not found the look and talk perceiver");
return null;
}
return (VisualQueryDetectorSession) session;
}
private void runForEachDetectorSessionLocked(
@NonNull Consumer<DetectorSession> action) {
for (int i = 0; i < mDetectorSessions.size(); i++) {

87
services/voiceinteraction/java/com/android/server/voiceinteraction/VisualQueryDetectorSession.java Normal file
View File

@@ -0,0 +1,87 @@
/*
* Copyright (C) 2023 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.server.voiceinteraction;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.content.Context;
import android.media.AudioFormat;
import android.media.permission.Identity;
import android.os.IBinder;
import android.os.ParcelFileDescriptor;
import android.os.PersistableBundle;
import android.os.SharedMemory;
import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback;
import android.util.Slog;
import com.android.internal.app.IHotwordRecognitionStatusCallback;
import java.io.PrintWriter;
import java.util.concurrent.ScheduledExecutorService;
/**
* A class that provides visual query detector to communicate with the {@link
* android.service.voice.VisualQueryDetectionService}.
*
* This class can handle the visual query detection whose detector is created by using
* {@link android.service.voice.VoiceInteractionService#createVisualQueryDetector(PersistableBundle
* ,SharedMemory, HotwordDetector.Callback)}.
*/
final class VisualQueryDetectorSession extends DetectorSession {
private static final String TAG = "VisualQueryDetectorSession";
//TODO(b/261783819): Determines actual functionalities, e.g., startRecognition etc.
VisualQueryDetectorSession(
@NonNull HotwordDetectionConnection.ServiceConnection remoteService,
@NonNull Object lock, @NonNull Context context, @NonNull IBinder token,
@NonNull IHotwordRecognitionStatusCallback callback, int voiceInteractionServiceUid,
Identity voiceInteractorIdentity,
@NonNull ScheduledExecutorService scheduledExecutorService, boolean logging) {
super(remoteService, lock, context, token, callback,
voiceInteractionServiceUid, voiceInteractorIdentity, scheduledExecutorService,
logging);
}
@Override
@SuppressWarnings("GuardedBy")
void informRestartProcessLocked() {
Slog.v(TAG, "informRestartProcessLocked");
mUpdateStateAfterStartFinished.set(false);
//TODO(b/261783819): Starts detection in VisualQueryDetectionService.
}
@Override
void startListeningFromExternalSourceLocked(
ParcelFileDescriptor audioStream,
AudioFormat audioFormat,
@Nullable PersistableBundle options,
IMicrophoneHotwordDetectionVoiceInteractionCallback callback)
throws UnsupportedOperationException {
throw new UnsupportedOperationException("HotwordDetectionService method"
+ " should not be called from VisualQueryDetectorSession.");
}
@SuppressWarnings("GuardedBy")
public void dumpLocked(String prefix, PrintWriter pw) {
super.dumpLocked(prefix, pw);
pw.print(prefix);
}
}

62
services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerServiceImpl.java
View File

@@ -58,6 +58,7 @@ import android.os.RemoteException;
import android.os.ServiceManager;
import android.os.SharedMemory;
import android.os.UserHandle;
import android.service.voice.HotwordDetector;
import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback;
import android.service.voice.IVoiceInteractionService;
import android.service.voice.IVoiceInteractionSession;
@@ -109,6 +110,7 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
final ComponentName mSessionComponentName;
final IWindowManager mIWindowManager;
final ComponentName mHotwordDetectionComponentName;
final ComponentName mVisualQueryDetectionComponentName;
boolean mBound = false;
IVoiceInteractionService mService;
volatile HotwordDetectionConnection mHotwordDetectionConnection;
@@ -211,6 +213,7 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
mInfo = null;
mSessionComponentName = null;
mHotwordDetectionComponentName = null;
mVisualQueryDetectionComponentName = null;
mIWindowManager = null;
mValid = false;
return;
@@ -220,6 +223,7 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
Slog.w(TAG, "Bad voice interaction service: " + mInfo.getParseError());
mSessionComponentName = null;
mHotwordDetectionComponentName = null;
mVisualQueryDetectionComponentName = null;
mIWindowManager = null;
mValid = false;
return;
@@ -230,6 +234,9 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
final String hotwordDetectionServiceName = mInfo.getHotwordDetectionService();
mHotwordDetectionComponentName = hotwordDetectionServiceName != null
? new ComponentName(service.getPackageName(), hotwordDetectionServiceName) : null;
final String visualQueryDetectionServiceName = mInfo.getVisualQueryDetectionService();
mVisualQueryDetectionComponentName = visualQueryDetectionServiceName != null ? new
ComponentName(service.getPackageName(), visualQueryDetectionServiceName) : null;
mIWindowManager = IWindowManager.Stub.asInterface(
ServiceManager.getService(Context.WINDOW_SERVICE));
IntentFilter filter = new IntentFilter();
@@ -591,14 +598,11 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
}
}
public void initAndVerifyDetectorLocked(
@NonNull Identity voiceInteractorIdentity,
@Nullable PersistableBundle options,
private void verifyDetectorForHotwordDetectionLocked(
@Nullable SharedMemory sharedMemory,
@NonNull IBinder token,
IHotwordRecognitionStatusCallback callback,
int detectorType) {
Slog.v(TAG, "initAndVerifyDetectorLocked");
Slog.v(TAG, "verifyDetectorForHotwordDetectionLocked");
int voiceInteractionServiceUid = mInfo.getServiceInfo().applicationInfo.uid;
if (mHotwordDetectionComponentName == null) {
Slog.w(TAG, "Hotword detection service name not found");
@@ -649,11 +653,55 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
logDetectorCreateEventIfNeeded(callback, detectorType, true,
voiceInteractionServiceUid);
}
private void verifyDetectorForVisualQueryDetectionLocked(@Nullable SharedMemory sharedMemory) {
Slog.v(TAG, "verifyDetectorForVisualQueryDetectionLocked");
if (mVisualQueryDetectionComponentName == null) {
Slog.w(TAG, "Visual query detection service name not found");
throw new IllegalStateException("Visual query detection service name not found");
}
ServiceInfo visualQueryDetectionServiceInfo = getServiceInfoLocked(
mVisualQueryDetectionComponentName, mUser);
if (visualQueryDetectionServiceInfo == null) {
Slog.w(TAG, "Visual query detection service info not found");
throw new IllegalStateException("Visual query detection service name not found");
}
if (!isIsolatedProcessLocked(visualQueryDetectionServiceInfo)) {
Slog.w(TAG, "Visual query detection service not in isolated process");
throw new IllegalStateException("Visual query detection not in isolated process");
}
if (sharedMemory != null && !sharedMemory.setProtect(OsConstants.PROT_READ)) {
Slog.w(TAG, "Can't set sharedMemory to be read-only");
throw new IllegalStateException("Can't set sharedMemory to be read-only");
}
}
public void initAndVerifyDetectorLocked(
@NonNull Identity voiceInteractorIdentity,
@Nullable PersistableBundle options,
@Nullable SharedMemory sharedMemory,
@NonNull IBinder token,
IHotwordRecognitionStatusCallback callback,
int detectorType) {
if (detectorType != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) {
verifyDetectorForHotwordDetectionLocked(sharedMemory, callback, detectorType);
} else {
verifyDetectorForVisualQueryDetectionLocked(sharedMemory);
}
if (mHotwordDetectionConnection == null) {
mHotwordDetectionConnection = new HotwordDetectionConnection(mServiceStub, mContext,
mInfo.getServiceInfo().applicationInfo.uid, voiceInteractorIdentity,
mHotwordDetectionComponentName, mUser, /* bindInstantServiceAllowed= */ false,
detectorType);
mHotwordDetectionComponentName, mVisualQueryDetectionComponentName, mUser,
/* bindInstantServiceAllowed= */ false, detectorType);
} else if (detectorType != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) {
// TODO: Logger events should be handled in session instead. Temporary adding the
// checking to prevent confusion so VisualQueryDetection events won't be logged if the
// connection is instantiated by the VisualQueryDetector.
mHotwordDetectionConnection.setDetectorType(detectorType);
}
mHotwordDetectionConnection.createDetectorLocked(options, sharedMemory, token, callback,
detectorType);