From fdb3dc63f5176e110992087cd19848356d62f3ee Mon Sep 17 00:00:00 2001 From: lpeter Date: Wed, 16 Nov 2022 17:35:41 +0800 Subject: [PATCH 1/3] Refactor the logic of HotwordDetectionConnection The HotwordDetectionConnection controls the communication with the HotwordDetectionService and handles the detection of trusted hotword detector. We separate the detection of trusted hotword detector into HotwordDetectorSession. It will be more clear to use original single detector to implement multiple detectors. Test: atest CtsVoiceInteractionTestCases Bug: 241041976 Change-Id: Ib13b906fc4fc27f07325ec6faf2f4948627f4492 --- .../HotwordDetectionConnection.java | 826 ++---------------- ...ssion.java => HotwordDetectorSession.java} | 786 +++++------------ 2 files changed, 250 insertions(+), 1362 deletions(-) rename services/voiceinteraction/java/com/android/server/voiceinteraction/{TrustedHotwordDetectorSession.java => HotwordDetectorSession.java} (60%) diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java index 2ac25b65a0828..5141dd34a892d 100644 --- a/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java +++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java @@ -16,67 +16,27 @@ package com.android.server.voiceinteraction; -import static android.Manifest.permission.CAPTURE_AUDIO_HOTWORD; -import static android.Manifest.permission.RECORD_AUDIO; -import static android.service.attention.AttentionService.PROXIMITY_UNKNOWN; -import static android.service.voice.HotwordDetectionService.AUDIO_SOURCE_EXTERNAL; -import static android.service.voice.HotwordDetectionService.AUDIO_SOURCE_MICROPHONE; -import static android.service.voice.HotwordDetectionService.ENABLE_PROXIMITY_RESULT; -import static android.service.voice.HotwordDetectionService.INITIALIZATION_STATUS_SUCCESS; -import static android.service.voice.HotwordDetectionService.INITIALIZATION_STATUS_UNKNOWN; -import static android.service.voice.HotwordDetectionService.KEY_INITIALIZATION_STATUS; - -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_ERROR; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_SUCCESS; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_UNKNOWN_NO_VALUE; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_UNKNOWN_OVER_MAX_CUSTOM_VALUE; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_UNKNOWN_TIMEOUT; import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTION_SERVICE_RESTARTED__REASON__AUDIO_SERVICE_DIED; import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTION_SERVICE_RESTARTED__REASON__SCHEDULE; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__APP_REQUEST_UPDATE_STATE; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_PROCESS_RESTARTED_EXCEPTION; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_REJECTED_EXCEPTION; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_STATUS_REPORTED_EXCEPTION; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_UPDATE_STATE_AFTER_TIMEOUT; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALL_UPDATE_STATE_EXCEPTION; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__EXTERNAL_SOURCE_DETECTED; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__EXTERNAL_SOURCE_DETECT_SECURITY_EXCEPTION; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__EXTERNAL_SOURCE_REJECTED; import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__ON_CONNECTED; import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__ON_DISCONNECTED; import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_BIND_SERVICE; import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_BIND_SERVICE_FAIL; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_UPDATE_STATE; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__START_EXTERNAL_SOURCE_DETECTION; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__START_SOFTWARE_DETECTION; import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__DETECTOR_TYPE__NORMAL_DETECTOR; import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__DETECTOR_TYPE__TRUSTED_DETECTOR_DSP; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECTED; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_SECURITY_EXCEPTION; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_TIMEOUT; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_UNEXPECTED_CALLBACK; import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__KEYPHRASE_TRIGGER; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED_FROM_RESTART; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECT_UNEXPECTED_CALLBACK; -import static com.android.server.voiceinteraction.SoundTriggerSessionPermissionsDecorator.enforcePermissionForPreflight; import android.annotation.NonNull; import android.annotation.Nullable; -import android.app.AppOpsManager; -import android.attention.AttentionManagerInternal; import android.content.ComponentName; import android.content.ContentCaptureOptions; import android.content.Context; import android.content.Intent; -import android.content.PermissionChecker; import android.hardware.soundtrigger.IRecognitionStatusCallback; import android.hardware.soundtrigger.SoundTrigger; import android.media.AudioFormat; import android.media.AudioManagerInternal; import android.media.permission.Identity; -import android.media.permission.PermissionUtil; import android.os.Binder; import android.os.Bundle; import android.os.IBinder; @@ -87,41 +47,27 @@ import android.os.RemoteException; import android.os.ServiceManager; import android.os.SharedMemory; import android.provider.DeviceConfig; -import android.service.voice.HotwordDetectedResult; import android.service.voice.HotwordDetectionService; import android.service.voice.HotwordDetector; -import android.service.voice.HotwordRejectedResult; -import android.service.voice.IDspHotwordDetectionCallback; import android.service.voice.IHotwordDetectionService; import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback; import android.service.voice.VoiceInteractionManagerInternal.HotwordDetectionServiceIdentity; import android.speech.IRecognitionServiceManager; -import android.text.TextUtils; -import android.util.Pair; import android.util.Slog; import android.view.contentcapture.IContentCaptureManager; import com.android.internal.annotations.GuardedBy; import com.android.internal.app.IHotwordRecognitionStatusCallback; -import com.android.internal.infra.AndroidFuture; import com.android.internal.infra.ServiceConnector; import com.android.server.LocalServices; import com.android.server.pm.permission.PermissionManagerServiceInternal; -import java.io.Closeable; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; import java.io.PrintWriter; -import java.time.Duration; import java.time.Instant; -import java.util.concurrent.Executor; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Function; /** @@ -132,63 +78,15 @@ final class HotwordDetectionConnection { static final boolean DEBUG = false; private static final String KEY_RESTART_PERIOD_IN_SECONDS = "restart_period_in_seconds"; - // TODO: These constants need to be refined. - // The validation timeout value is 3 seconds for onDetect of DSP trigger event. - private static final long VALIDATION_TIMEOUT_MILLIS = 3000; - // Write the onDetect timeout metric when it takes more time than MAX_VALIDATION_TIMEOUT_MILLIS. - private static final long MAX_VALIDATION_TIMEOUT_MILLIS = 4000; - private static final long MAX_UPDATE_TIMEOUT_MILLIS = 30000; - private static final long EXTERNAL_HOTWORD_CLEANUP_MILLIS = 2000; - private static final Duration MAX_UPDATE_TIMEOUT_DURATION = - Duration.ofMillis(MAX_UPDATE_TIMEOUT_MILLIS); private static final long RESET_DEBUG_HOTWORD_LOGGING_TIMEOUT_MILLIS = 60 * 60 * 1000; // 1 hour private static final int MAX_ISOLATED_PROCESS_NUMBER = 10; - // The error codes are used for onError callback - private static final int HOTWORD_DETECTION_SERVICE_DIED = -1; - private static final int CALLBACK_ONDETECTED_GOT_SECURITY_EXCEPTION = -2; - private static final int CALLBACK_DETECT_TIMEOUT = -3; - private static final int CALLBACK_ONDETECTED_STREAM_COPY_ERROR = -4; - - // Hotword metrics - private static final int METRICS_INIT_UNKNOWN_TIMEOUT = - HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_UNKNOWN_TIMEOUT; - private static final int METRICS_INIT_UNKNOWN_NO_VALUE = - HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_UNKNOWN_NO_VALUE; - private static final int METRICS_INIT_UNKNOWN_OVER_MAX_CUSTOM_VALUE = - HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_UNKNOWN_OVER_MAX_CUSTOM_VALUE; - private static final int METRICS_INIT_CALLBACK_STATE_ERROR = - HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_ERROR; - private static final int METRICS_INIT_CALLBACK_STATE_SUCCESS = - HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_SUCCESS; - - private static final int METRICS_KEYPHRASE_TRIGGERED_DETECT_SECURITY_EXCEPTION = - HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_SECURITY_EXCEPTION; - private static final int METRICS_KEYPHRASE_TRIGGERED_DETECT_UNEXPECTED_CALLBACK = - HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_UNEXPECTED_CALLBACK; - private static final int METRICS_KEYPHRASE_TRIGGERED_REJECT_UNEXPECTED_CALLBACK = - HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECT_UNEXPECTED_CALLBACK; - - private static final int METRICS_EXTERNAL_SOURCE_DETECTED = - HOTWORD_DETECTOR_EVENTS__EVENT__EXTERNAL_SOURCE_DETECTED; - private static final int METRICS_EXTERNAL_SOURCE_REJECTED = - HOTWORD_DETECTOR_EVENTS__EVENT__EXTERNAL_SOURCE_REJECTED; - private static final int METRICS_EXTERNAL_SOURCE_DETECT_SECURITY_EXCEPTION = - HOTWORD_DETECTOR_EVENTS__EVENT__EXTERNAL_SOURCE_DETECT_SECURITY_EXCEPTION; - private static final int METRICS_CALLBACK_ON_STATUS_REPORTED_EXCEPTION = - HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_STATUS_REPORTED_EXCEPTION; - - private final Executor mAudioCopyExecutor = Executors.newCachedThreadPool(); // TODO: This may need to be a Handler(looper) private final ScheduledExecutorService mScheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); - private final AppOpsManager mAppOpsManager; - private final HotwordAudioStreamCopier mHotwordAudioStreamCopier; @Nullable private final ScheduledFuture mCancellationTaskFuture; - private final AtomicBoolean mUpdateStateAfterStartFinished = new AtomicBoolean(false); private final IBinder.DeathRecipient mAudioServerDeathRecipient = this::audioServerDied; - private final @NonNull ServiceConnectionFactory mServiceConnectionFactory; - private final IHotwordRecognitionStatusCallback mCallback; + @NonNull private final ServiceConnectionFactory mServiceConnectionFactory; private final int mDetectorType; /** * Time after which each HotwordDetectionService process is stopped and replaced by a new one. @@ -201,35 +99,22 @@ final class HotwordDetectionConnection { final ComponentName mDetectionComponentName; final int mUser; final Context mContext; - - @Nullable AttentionManagerInternal mAttentionManagerInternal = null; - - final AttentionManagerInternal.ProximityUpdateCallbackInternal mProximityCallbackInternal = - this::setProximityValue; - - volatile HotwordDetectionServiceIdentity mIdentity; - private IMicrophoneHotwordDetectionVoiceInteractionCallback mSoftwareCallback; private Instant mLastRestartInstant; - private ScheduledFuture mCancellationKeyPhraseDetectionFuture; private ScheduledFuture mDebugHotwordLoggingTimeoutFuture = null; /** Identity used for attributing app ops when delivering data to the Interactor. */ @GuardedBy("mLock") @Nullable private final Identity mVoiceInteractorIdentity; - @GuardedBy("mLock") - private ParcelFileDescriptor mCurrentAudioSink; - @GuardedBy("mLock") - private boolean mValidatingDspTrigger = false; - @GuardedBy("mLock") - private boolean mPerformingSoftwareHotwordDetection; - private @NonNull ServiceConnection mRemoteHotwordDetectionService; + @NonNull private ServiceConnection mRemoteHotwordDetectionService; private IBinder mAudioFlinger; - private boolean mDebugHotwordLogging = false; @GuardedBy("mLock") - private double mProximityMeters = PROXIMITY_UNKNOWN; + private boolean mDebugHotwordLogging = false; + + @GuardedBy("mLock") + private final HotwordDetectorSession mHotwordDetectorSession; HotwordDetectionConnection(Object lock, Context context, int voiceInteractionServiceUid, Identity voiceInteractorIdentity, ComponentName serviceName, int userId, @@ -244,13 +129,8 @@ final class HotwordDetectionConnection { mContext = context; mVoiceInteractionServiceUid = voiceInteractionServiceUid; mVoiceInteractorIdentity = voiceInteractorIdentity; - mAppOpsManager = mContext.getSystemService(AppOpsManager.class); - mHotwordAudioStreamCopier = new HotwordAudioStreamCopier(mAppOpsManager, detectorType, - mVoiceInteractorIdentity.uid, mVoiceInteractorIdentity.packageName, - mVoiceInteractorIdentity.attributionTag); mDetectionComponentName = serviceName; mUser = userId; - mCallback = callback; mDetectorType = detectorType; mReStartPeriodSeconds = DeviceConfig.getInt(DeviceConfig.NAMESPACE_VOICE_INTERACTION, KEY_RESTART_PERIOD_IN_SECONDS, 0); @@ -259,17 +139,13 @@ final class HotwordDetectionConnection { initAudioFlingerLocked(); mServiceConnectionFactory = new ServiceConnectionFactory(intent, bindInstantServiceAllowed); - mRemoteHotwordDetectionService = mServiceConnectionFactory.createLocked(); - if (ENABLE_PROXIMITY_RESULT) { - mAttentionManagerInternal = LocalServices.getService(AttentionManagerInternal.class); - if (mAttentionManagerInternal != null) { - mAttentionManagerInternal.onStartProximityUpdates(mProximityCallbackInternal); - } - } - mLastRestartInstant = Instant.now(); - updateStateAfterProcessStart(options, sharedMemory); + + mHotwordDetectorSession = new HotwordDetectorSession(mRemoteHotwordDetectionService, + mLock, mContext, callback, mDetectorType, mVoiceInteractionServiceUid, + mVoiceInteractorIdentity, mScheduledExecutorService, mDebugHotwordLogging); + mHotwordDetectorSession.initialize(options, sharedMemory); if (mReStartPeriodSeconds <= 0) { mCancellationTaskFuture = null; @@ -320,106 +196,11 @@ final class HotwordDetectionConnection { } } - private void updateStateAfterProcessStart( - PersistableBundle options, SharedMemory sharedMemory) { - if (DEBUG) { - Slog.d(TAG, "updateStateAfterProcessStart"); - } - mRemoteHotwordDetectionService.postAsync(service -> { - AndroidFuture future = new AndroidFuture<>(); - IRemoteCallback statusCallback = new IRemoteCallback.Stub() { - @Override - public void sendResult(Bundle bundle) throws RemoteException { - if (DEBUG) { - Slog.d(TAG, "updateState finish"); - } - future.complete(null); - if (mUpdateStateAfterStartFinished.getAndSet(true)) { - Slog.w(TAG, "call callback after timeout"); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_UPDATE_STATE_AFTER_TIMEOUT, - mVoiceInteractionServiceUid); - return; - } - Pair statusResultPair = getInitStatusAndMetricsResult(bundle); - int status = statusResultPair.first; - int initResultMetricsResult = statusResultPair.second; - try { - mCallback.onStatusReported(status); - HotwordMetricsLogger.writeServiceInitResultEvent(mDetectorType, - initResultMetricsResult); - } catch (RemoteException e) { - Slog.w(TAG, "Failed to report initialization status: " + e); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - METRICS_CALLBACK_ON_STATUS_REPORTED_EXCEPTION, - mVoiceInteractionServiceUid); - } - } - }; - try { - service.updateState(options, sharedMemory, statusCallback); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_UPDATE_STATE, - mVoiceInteractionServiceUid); - } catch (RemoteException e) { - // TODO: (b/181842909) Report an error to voice interactor - Slog.w(TAG, "Failed to updateState for HotwordDetectionService", e); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - HOTWORD_DETECTOR_EVENTS__EVENT__CALL_UPDATE_STATE_EXCEPTION, - mVoiceInteractionServiceUid); - } - return future.orTimeout(MAX_UPDATE_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS); - }).whenComplete((res, err) -> { - if (err instanceof TimeoutException) { - Slog.w(TAG, "updateState timed out"); - if (mUpdateStateAfterStartFinished.getAndSet(true)) { - return; - } - try { - mCallback.onStatusReported(INITIALIZATION_STATUS_UNKNOWN); - HotwordMetricsLogger.writeServiceInitResultEvent(mDetectorType, - METRICS_INIT_UNKNOWN_TIMEOUT); - } catch (RemoteException e) { - Slog.w(TAG, "Failed to report initialization status UNKNOWN", e); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - METRICS_CALLBACK_ON_STATUS_REPORTED_EXCEPTION, - mVoiceInteractionServiceUid); - } - } else if (err != null) { - Slog.w(TAG, "Failed to update state: " + err); - } else { - // NOTE: so far we don't need to take any action. - } - }); - } - - private static Pair getInitStatusAndMetricsResult(Bundle bundle) { - if (bundle == null) { - return new Pair<>(INITIALIZATION_STATUS_UNKNOWN, METRICS_INIT_UNKNOWN_NO_VALUE); - } - int status = bundle.getInt(KEY_INITIALIZATION_STATUS, INITIALIZATION_STATUS_UNKNOWN); - if (status > HotwordDetectionService.getMaxCustomInitializationStatus()) { - return new Pair<>(INITIALIZATION_STATUS_UNKNOWN, - status == INITIALIZATION_STATUS_UNKNOWN - ? METRICS_INIT_UNKNOWN_NO_VALUE - : METRICS_INIT_UNKNOWN_OVER_MAX_CUSTOM_VALUE); - } - // TODO: should guard against negative here - int metricsResult = status == INITIALIZATION_STATUS_SUCCESS - ? METRICS_INIT_CALLBACK_STATE_SUCCESS - : METRICS_INIT_CALLBACK_STATE_ERROR; - return new Pair<>(status, metricsResult); - } - - private boolean isBound() { - synchronized (mLock) { - return mRemoteHotwordDetectionService.isBound(); - } - } - + @SuppressWarnings("GuardedBy") void cancelLocked() { Slog.v(TAG, "cancelLocked"); clearDebugHotwordLoggingTimeoutLocked(); + mHotwordDetectorSession.destroyLocked(); mDebugHotwordLogging = false; mRemoteHotwordDetectionService.unbind(); LocalServices.getService(PermissionManagerServiceInternal.class) @@ -434,118 +215,27 @@ final class HotwordDetectionConnection { if (mAudioFlinger != null) { mAudioFlinger.unlinkToDeath(mAudioServerDeathRecipient, /* flags= */ 0); } - if (mAttentionManagerInternal != null) { - mAttentionManagerInternal.onStopProximityUpdates(mProximityCallbackInternal); - } } + @SuppressWarnings("GuardedBy") void updateStateLocked(PersistableBundle options, SharedMemory sharedMemory) { - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - HOTWORD_DETECTOR_EVENTS__EVENT__APP_REQUEST_UPDATE_STATE, - mVoiceInteractionServiceUid); - - // Prevent doing the init late, so restart is handled equally to a clean process start. - // TODO(b/191742511): this logic needs a test - if (!mUpdateStateAfterStartFinished.get() - && Instant.now().minus(MAX_UPDATE_TIMEOUT_DURATION).isBefore(mLastRestartInstant)) { - Slog.v(TAG, "call updateStateAfterProcessStart"); - updateStateAfterProcessStart(options, sharedMemory); - } else { - mRemoteHotwordDetectionService.run( - service -> service.updateState(options, sharedMemory, null /* callback */)); - } + mHotwordDetectorSession.updateStateLocked(options, sharedMemory, mLastRestartInstant); } + /** + * This method is only used by SoftwareHotwordDetector. + */ void startListeningFromMic( AudioFormat audioFormat, IMicrophoneHotwordDetectionVoiceInteractionCallback callback) { if (DEBUG) { Slog.d(TAG, "startListeningFromMic"); } - mSoftwareCallback = callback; - synchronized (mLock) { - if (mPerformingSoftwareHotwordDetection) { - Slog.i(TAG, "Hotword validation is already in progress, ignoring."); - return; - } - mPerformingSoftwareHotwordDetection = true; - - startListeningFromMicLocked(); + mHotwordDetectorSession.startListeningFromMicLocked(audioFormat, callback); } } - private void startListeningFromMicLocked() { - // TODO: consider making this a non-anonymous class. - IDspHotwordDetectionCallback internalCallback = new IDspHotwordDetectionCallback.Stub() { - @Override - public void onDetected(HotwordDetectedResult result) throws RemoteException { - if (DEBUG) { - Slog.d(TAG, "onDetected"); - } - synchronized (mLock) { - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - mDetectorType, - HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECTED); - if (!mPerformingSoftwareHotwordDetection) { - Slog.i(TAG, "Hotword detection has already completed"); - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - mDetectorType, - METRICS_KEYPHRASE_TRIGGERED_DETECT_UNEXPECTED_CALLBACK); - return; - } - mPerformingSoftwareHotwordDetection = false; - try { - enforcePermissionsForDataDelivery(); - } catch (SecurityException e) { - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - mDetectorType, - METRICS_KEYPHRASE_TRIGGERED_DETECT_SECURITY_EXCEPTION); - mSoftwareCallback.onError(); - return; - } - saveProximityValueToBundle(result); - HotwordDetectedResult newResult; - try { - newResult = mHotwordAudioStreamCopier.startCopyingAudioStreams(result); - } catch (IOException e) { - // TODO: Write event - mSoftwareCallback.onError(); - return; - } - mSoftwareCallback.onDetected(newResult, null, null); - Slog.i(TAG, "Egressed " + HotwordDetectedResult.getUsageSize(newResult) - + " bits from hotword trusted process"); - if (mDebugHotwordLogging) { - Slog.i(TAG, "Egressed detected result: " + newResult); - } - } - } - - @Override - public void onRejected(HotwordRejectedResult result) throws RemoteException { - if (DEBUG) { - Slog.wtf(TAG, "onRejected"); - } - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - mDetectorType, - HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED); - // onRejected isn't allowed here, and we are not expecting it. - } - }; - - mRemoteHotwordDetectionService.run( - service -> service.detectFromMicrophoneSource( - null, - AUDIO_SOURCE_MICROPHONE, - null, - null, - internalCallback)); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - HOTWORD_DETECTOR_EVENTS__EVENT__START_SOFTWARE_DETECTION, - mVoiceInteractionServiceUid); - } - public void startListeningFromExternalSource( ParcelFileDescriptor audioStream, AudioFormat audioFormat, @@ -554,39 +244,24 @@ final class HotwordDetectionConnection { if (DEBUG) { Slog.d(TAG, "startListeningFromExternalSource"); } - - handleExternalSourceHotwordDetection( - audioStream, - audioFormat, - options, - callback); + synchronized (mLock) { + mHotwordDetectorSession.startListeningFromExternalSourceLocked(audioStream, audioFormat, + options, callback); + } } + /** + * This method is only used by SoftwareHotwordDetector. + */ void stopListening() { if (DEBUG) { Slog.d(TAG, "stopListening"); } synchronized (mLock) { - stopListeningLocked(); + mHotwordDetectorSession.stopListeningLocked(); } } - private void stopListeningLocked() { - if (!mPerformingSoftwareHotwordDetection) { - Slog.i(TAG, "Hotword detection is not running"); - return; - } - mPerformingSoftwareHotwordDetection = false; - - mRemoteHotwordDetectionService.run(IHotwordDetectionService::stopDetection); - - if (mCurrentAudioSink != null) { - Slog.i(TAG, "Closing audio stream to hotword detector: stopping requested"); - bestEffortClose(mCurrentAudioSink); - } - mCurrentAudioSink = null; - } - void triggerHardwareRecognitionEventForTestLocked( SoundTrigger.KeyphraseRecognitionEvent event, IHotwordRecognitionStatusCallback callback) { @@ -596,130 +271,13 @@ final class HotwordDetectionConnection { detectFromDspSource(event, callback); } - void detectFromDspSource(SoundTrigger.KeyphraseRecognitionEvent recognitionEvent, + private void detectFromDspSource(SoundTrigger.KeyphraseRecognitionEvent recognitionEvent, IHotwordRecognitionStatusCallback externalCallback) { if (DEBUG) { Slog.d(TAG, "detectFromDspSource"); } - - AtomicBoolean timeoutDetected = new AtomicBoolean(false); - // TODO: consider making this a non-anonymous class. - IDspHotwordDetectionCallback internalCallback = new IDspHotwordDetectionCallback.Stub() { - @Override - public void onDetected(HotwordDetectedResult result) throws RemoteException { - if (DEBUG) { - Slog.d(TAG, "onDetected"); - } - synchronized (mLock) { - if (mCancellationKeyPhraseDetectionFuture != null) { - mCancellationKeyPhraseDetectionFuture.cancel(true); - } - if (timeoutDetected.get()) { - return; - } - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - mDetectorType, - HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECTED); - if (!mValidatingDspTrigger) { - Slog.i(TAG, "Ignoring #onDetected due to a process restart"); - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - mDetectorType, - METRICS_KEYPHRASE_TRIGGERED_DETECT_UNEXPECTED_CALLBACK); - return; - } - mValidatingDspTrigger = false; - try { - enforcePermissionsForDataDelivery(); - enforceExtraKeyphraseIdNotLeaked(result, recognitionEvent); - } catch (SecurityException e) { - Slog.i(TAG, "Ignoring #onDetected due to a SecurityException", e); - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - mDetectorType, - METRICS_KEYPHRASE_TRIGGERED_DETECT_SECURITY_EXCEPTION); - externalCallback.onError(CALLBACK_ONDETECTED_GOT_SECURITY_EXCEPTION); - return; - } - saveProximityValueToBundle(result); - HotwordDetectedResult newResult; - try { - newResult = mHotwordAudioStreamCopier.startCopyingAudioStreams(result); - } catch (IOException e) { - // TODO: Write event - externalCallback.onError(CALLBACK_ONDETECTED_STREAM_COPY_ERROR); - return; - } - externalCallback.onKeyphraseDetected(recognitionEvent, newResult); - Slog.i(TAG, "Egressed " + HotwordDetectedResult.getUsageSize(newResult) - + " bits from hotword trusted process"); - if (mDebugHotwordLogging) { - Slog.i(TAG, "Egressed detected result: " + newResult); - } - } - } - - @Override - public void onRejected(HotwordRejectedResult result) throws RemoteException { - if (DEBUG) { - Slog.d(TAG, "onRejected"); - } - synchronized (mLock) { - if (mCancellationKeyPhraseDetectionFuture != null) { - mCancellationKeyPhraseDetectionFuture.cancel(true); - } - if (timeoutDetected.get()) { - return; - } - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - mDetectorType, - HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED); - if (!mValidatingDspTrigger) { - Slog.i(TAG, "Ignoring #onRejected due to a process restart"); - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - mDetectorType, - METRICS_KEYPHRASE_TRIGGERED_REJECT_UNEXPECTED_CALLBACK); - return; - } - mValidatingDspTrigger = false; - externalCallback.onRejected(result); - if (mDebugHotwordLogging && result != null) { - Slog.i(TAG, "Egressed rejected result: " + result); - } - } - } - }; - synchronized (mLock) { - mValidatingDspTrigger = true; - mRemoteHotwordDetectionService.run(service -> { - // We use the VALIDATION_TIMEOUT_MILLIS to inform that the client needs to invoke - // the callback before timeout value. In order to reduce the latency impact between - // server side and client side, we need to use another timeout value - // MAX_VALIDATION_TIMEOUT_MILLIS to monitor it. - mCancellationKeyPhraseDetectionFuture = mScheduledExecutorService.schedule( - () -> { - // TODO: avoid allocate every time - timeoutDetected.set(true); - Slog.w(TAG, "Timed out on #detectFromDspSource"); - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - mDetectorType, - HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_TIMEOUT); - try { - externalCallback.onError(CALLBACK_DETECT_TIMEOUT); - } catch (RemoteException e) { - Slog.w(TAG, "Failed to report onError status: ", e); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION, - mVoiceInteractionServiceUid); - } - }, - MAX_VALIDATION_TIMEOUT_MILLIS, - TimeUnit.MILLISECONDS); - service.detectFromDspSource( - recognitionEvent, - recognitionEvent.getCaptureFormat(), - VALIDATION_TIMEOUT_MILLIS, - internalCallback); - }); + mHotwordDetectorSession.detectFromDspSourceLocked(recognitionEvent, externalCallback); } } @@ -730,10 +288,12 @@ final class HotwordDetectionConnection { } } + @SuppressWarnings("GuardedBy") void setDebugHotwordLoggingLocked(boolean logging) { Slog.v(TAG, "setDebugHotwordLoggingLocked: " + logging); clearDebugHotwordLoggingTimeoutLocked(); mDebugHotwordLogging = logging; + mHotwordDetectorSession.setDebugHotwordLoggingLocked(logging); if (logging) { // Reset mDebugHotwordLogging to false after one hour @@ -741,6 +301,7 @@ final class HotwordDetectionConnection { Slog.v(TAG, "Timeout to reset mDebugHotwordLogging to false"); synchronized (mLock) { mDebugHotwordLogging = false; + mHotwordDetectorSession.setDebugHotwordLoggingLocked(false); } }, RESET_DEBUG_HOTWORD_LOGGING_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS); } @@ -753,60 +314,23 @@ final class HotwordDetectionConnection { } } + @SuppressWarnings("GuardedBy") private void restartProcessLocked() { // TODO(b/244598068): Check HotwordAudioStreamManager first Slog.v(TAG, "Restarting hotword detection process"); ServiceConnection oldConnection = mRemoteHotwordDetectionService; HotwordDetectionServiceIdentity previousIdentity = mIdentity; - // TODO(volnov): this can be done after connect() has been successful. - if (mValidatingDspTrigger) { - // We're restarting the process while it's processing a DSP trigger, so report a - // rejection. This also allows the Interactor to startReco again - try { - mCallback.onRejected(new HotwordRejectedResult.Builder().build()); - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - mDetectorType, - HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED_FROM_RESTART); - } catch (RemoteException e) { - Slog.w(TAG, "Failed to call #rejected"); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_REJECTED_EXCEPTION, - mVoiceInteractionServiceUid); - } - mValidatingDspTrigger = false; - } - - mUpdateStateAfterStartFinished.set(false); mLastRestartInstant = Instant.now(); - // Recreate connection to reset the cache. mRemoteHotwordDetectionService = mServiceConnectionFactory.createLocked(); - Slog.v(TAG, "Started the new process, issuing #onProcessRestarted"); - try { - mCallback.onProcessRestarted(); - } catch (RemoteException e) { - Slog.w(TAG, "Failed to communicate #onProcessRestarted", e); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_PROCESS_RESTARTED_EXCEPTION, - mVoiceInteractionServiceUid); - } - - // Restart listening from microphone if the hotword process has been restarted. - if (mPerformingSoftwareHotwordDetection) { - Slog.i(TAG, "Process restarted: calling startRecognition() again"); - startListeningFromMicLocked(); - } - - if (mCurrentAudioSink != null) { - Slog.i(TAG, "Closing external audio stream to hotword detector: process restarted"); - bestEffortClose(mCurrentAudioSink); - mCurrentAudioSink = null; - } - + Slog.v(TAG, "Started the new process, dispatching processRestarted to detector"); + mHotwordDetectorSession.updateRemoteHotwordDetectionServiceLocked( + mRemoteHotwordDetectionService); + mHotwordDetectorSession.informRestartProcessLocked(); if (DEBUG) { - Slog.i(TAG, "#onProcessRestarted called, unbinding from the old process"); + Slog.i(TAG, "processRestarted is dispatched done, unbinding from the old process"); } oldConnection.ignoreConnectionStatusEvents(); oldConnection.unbind(); @@ -816,7 +340,6 @@ final class HotwordDetectionConnection { } static final class SoundTriggerCallback extends IRecognitionStatusCallback.Stub { - private SoundTrigger.KeyphraseRecognitionEvent mRecognitionEvent; private final HotwordDetectionConnection mHotwordDetectionConnection; private final IHotwordRecognitionStatusCallback mExternalCallback; @@ -837,7 +360,6 @@ final class HotwordDetectionConnection { HotwordMetricsLogger.writeKeyphraseTriggerEvent( HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__DETECTOR_TYPE__TRUSTED_DETECTOR_DSP, HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__KEYPHRASE_TRIGGER); - mRecognitionEvent = recognitionEvent; mHotwordDetectionConnection.detectFromDspSource( recognitionEvent, mExternalCallback); } else { @@ -872,160 +394,18 @@ final class HotwordDetectionConnection { } public void dump(String prefix, PrintWriter pw) { - pw.print(prefix); pw.print("mReStartPeriodSeconds="); pw.println(mReStartPeriodSeconds); - pw.print(prefix); - pw.print("mBound=" + mRemoteHotwordDetectionService.isBound()); - pw.print(", mValidatingDspTrigger=" + mValidatingDspTrigger); - pw.print(", mPerformingSoftwareHotwordDetection=" + mPerformingSoftwareHotwordDetection); - pw.print(", mRestartCount=" + mServiceConnectionFactory.mRestartCount); - pw.print(", mLastRestartInstant=" + mLastRestartInstant); - pw.println(", mDetectorType=" + HotwordDetector.detectorTypeToString(mDetectorType)); - } - - private void handleExternalSourceHotwordDetection( - ParcelFileDescriptor audioStream, - AudioFormat audioFormat, - @Nullable PersistableBundle options, - IMicrophoneHotwordDetectionVoiceInteractionCallback callback) { - if (DEBUG) { - Slog.d(TAG, "#handleExternalSourceHotwordDetection"); - } - InputStream audioSource = new ParcelFileDescriptor.AutoCloseInputStream(audioStream); - - Pair clientPipe = createPipe(); - if (clientPipe == null) { - // TODO: Need to propagate as unknown error or something? - return; - } - ParcelFileDescriptor serviceAudioSink = clientPipe.second; - ParcelFileDescriptor serviceAudioSource = clientPipe.first; - synchronized (mLock) { - mCurrentAudioSink = serviceAudioSink; + pw.print(prefix); pw.print("mReStartPeriodSeconds="); pw.println(mReStartPeriodSeconds); + pw.print(prefix); pw.print("mBound="); + pw.println(mRemoteHotwordDetectionService.isBound()); + pw.print(prefix); pw.print("mRestartCount="); + pw.println(mServiceConnectionFactory.mRestartCount); + pw.print(prefix); pw.print("mLastRestartInstant="); pw.println(mLastRestartInstant); + pw.print(prefix); pw.print("mDetectorType="); + pw.println(HotwordDetector.detectorTypeToString(mDetectorType)); + pw.print(prefix); pw.println("HotwordDetectorSession"); + mHotwordDetectorSession.dumpLocked(prefix, pw); } - - mAudioCopyExecutor.execute(() -> { - try (InputStream source = audioSource; - OutputStream fos = - new ParcelFileDescriptor.AutoCloseOutputStream(serviceAudioSink)) { - - byte[] buffer = new byte[1024]; - while (true) { - int bytesRead = source.read(buffer, 0, 1024); - - if (bytesRead < 0) { - Slog.i(TAG, "Reached end of stream for external hotword"); - break; - } - - // TODO: First write to ring buffer to make sure we don't lose data if the next - // statement fails. - // ringBuffer.append(buffer, bytesRead); - fos.write(buffer, 0, bytesRead); - } - } catch (IOException e) { - Slog.w(TAG, "Failed supplying audio data to validator", e); - - try { - callback.onError(); - } catch (RemoteException ex) { - Slog.w(TAG, "Failed to report onError status: " + ex); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION, - mVoiceInteractionServiceUid); - } - } finally { - synchronized (mLock) { - mCurrentAudioSink = null; - } - } - }); - - // TODO: handle cancellations well - // TODO: what if we cancelled and started a new one? - mRemoteHotwordDetectionService.run( - service -> { - service.detectFromMicrophoneSource( - serviceAudioSource, - // TODO: consider making a proxy callback + copy of audio format - AUDIO_SOURCE_EXTERNAL, - audioFormat, - options, - new IDspHotwordDetectionCallback.Stub() { - @Override - public void onRejected(HotwordRejectedResult result) - throws RemoteException { - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - METRICS_EXTERNAL_SOURCE_REJECTED, - mVoiceInteractionServiceUid); - mScheduledExecutorService.schedule( - () -> { - bestEffortClose(serviceAudioSink, audioSource); - }, - EXTERNAL_HOTWORD_CLEANUP_MILLIS, - TimeUnit.MILLISECONDS); - - callback.onRejected(result); - - if (result != null) { - Slog.i(TAG, "Egressed 'hotword rejected result' " - + "from hotword trusted process"); - if (mDebugHotwordLogging) { - Slog.i(TAG, "Egressed detected result: " + result); - } - } - } - - @Override - public void onDetected(HotwordDetectedResult triggerResult) - throws RemoteException { - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - METRICS_EXTERNAL_SOURCE_DETECTED, - mVoiceInteractionServiceUid); - mScheduledExecutorService.schedule( - () -> { - bestEffortClose(serviceAudioSink, audioSource); - }, - EXTERNAL_HOTWORD_CLEANUP_MILLIS, - TimeUnit.MILLISECONDS); - - try { - enforcePermissionsForDataDelivery(); - } catch (SecurityException e) { - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - METRICS_EXTERNAL_SOURCE_DETECT_SECURITY_EXCEPTION, - mVoiceInteractionServiceUid); - callback.onError(); - return; - } - HotwordDetectedResult newResult; - try { - newResult = - mHotwordAudioStreamCopier.startCopyingAudioStreams( - triggerResult); - } catch (IOException e) { - // TODO: Write event - callback.onError(); - return; - } - callback.onDetected(newResult, null /* audioFormat */, - null /* audioStream */); - Slog.i(TAG, "Egressed " - + HotwordDetectedResult.getUsageSize(newResult) - + " bits from hotword trusted process"); - if (mDebugHotwordLogging) { - Slog.i(TAG, - "Egressed detected result: " + newResult); - } - } - }); - - // A copy of this has been created and passed to the hotword validator - bestEffortClose(serviceAudioSource); - }); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - HOTWORD_DETECTOR_EVENTS__EVENT__START_EXTERNAL_SOURCE_DETECTION, - mVoiceInteractionServiceUid); } private class ServiceConnectionFactory { @@ -1054,7 +434,7 @@ final class HotwordDetectionConnection { } } - private class ServiceConnection extends ServiceConnector.Impl { + class ServiceConnection extends ServiceConnector.Impl { private final Object mLock = new Object(); private final Intent mIntent; @@ -1109,21 +489,16 @@ final class HotwordDetectionConnection { @Override public void binderDied() { super.binderDied(); + Slog.w(TAG, "binderDied"); synchronized (mLock) { if (!mRespectServiceConnectionStatusChanged) { Slog.v(TAG, "Ignored #binderDied event"); return; } - - Slog.w(TAG, "binderDied"); - try { - mCallback.onError(HOTWORD_DETECTION_SERVICE_DIED); - } catch (RemoteException e) { - Slog.w(TAG, "Failed to report onError status: " + e); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION, - mVoiceInteractionServiceUid); - } + } + synchronized (HotwordDetectionConnection.this.mLock) { + mHotwordDetectorSession.reportErrorLocked( + HotwordDetectorSession.HOTWORD_DETECTION_SERVICE_DIED); } } @@ -1168,18 +543,6 @@ final class HotwordDetectionConnection { } } - private static Pair createPipe() { - ParcelFileDescriptor[] fileDescriptors; - try { - fileDescriptors = ParcelFileDescriptor.createPipe(); - } catch (IOException e) { - Slog.e(TAG, "Failed to create audio stream pipe", e); - return null; - } - - return Pair.create(fileDescriptors[0], fileDescriptors[1]); - } - private static void updateAudioFlinger(ServiceConnection connection, IBinder audioFlinger) { // TODO: Consider using a proxy that limits the exposed API surface. connection.run(service -> service.updateAudioFlinger(audioFlinger)); @@ -1241,85 +604,4 @@ final class HotwordDetectionConnection { } }); } - - private void saveProximityValueToBundle(HotwordDetectedResult result) { - synchronized (mLock) { - if (result != null && mProximityMeters != PROXIMITY_UNKNOWN) { - result.setProximity(mProximityMeters); - } - } - } - - private void setProximityValue(double proximityMeters) { - synchronized (mLock) { - mProximityMeters = proximityMeters; - } - } - - private static void bestEffortClose(Closeable... closeables) { - for (Closeable closeable : closeables) { - bestEffortClose(closeable); - } - } - - private static void bestEffortClose(Closeable closeable) { - try { - closeable.close(); - } catch (IOException e) { - if (DEBUG) { - Slog.w(TAG, "Failed closing", e); - } - } - } - - // TODO: Share this code with SoundTriggerMiddlewarePermission. - private void enforcePermissionsForDataDelivery() { - Binder.withCleanCallingIdentity(() -> { - enforcePermissionForPreflight(mContext, mVoiceInteractorIdentity, RECORD_AUDIO); - int hotwordOp = AppOpsManager.strOpToOp(AppOpsManager.OPSTR_RECORD_AUDIO_HOTWORD); - mAppOpsManager.noteOpNoThrow(hotwordOp, - mVoiceInteractorIdentity.uid, mVoiceInteractorIdentity.packageName, - mVoiceInteractorIdentity.attributionTag, OP_MESSAGE); - enforcePermissionForDataDelivery(mContext, mVoiceInteractorIdentity, - CAPTURE_AUDIO_HOTWORD, OP_MESSAGE); - }); - } - - /** - * Throws a {@link SecurityException} iff the given identity has given permission to receive - * data. - * - * @param context A {@link Context}, used for permission checks. - * @param identity The identity to check. - * @param permission The identifier of the permission we want to check. - * @param reason The reason why we're requesting the permission, for auditing purposes. - */ - private static void enforcePermissionForDataDelivery(@NonNull Context context, - @NonNull Identity identity, - @NonNull String permission, @NonNull String reason) { - final int status = PermissionUtil.checkPermissionForDataDelivery(context, identity, - permission, reason); - if (status != PermissionChecker.PERMISSION_GRANTED) { - throw new SecurityException( - TextUtils.formatSimple("Failed to obtain permission %s for identity %s", - permission, - SoundTriggerSessionPermissionsDecorator.toString(identity))); - } - } - - private static void enforceExtraKeyphraseIdNotLeaked(HotwordDetectedResult result, - SoundTrigger.KeyphraseRecognitionEvent recognitionEvent) { - // verify the phrase ID in HotwordDetectedResult is not exposing extra phrases - // the DSP did not detect - for (SoundTrigger.KeyphraseRecognitionExtra keyphrase : recognitionEvent.keyphraseExtras) { - if (keyphrase.getKeyphraseId() == result.getHotwordPhraseId()) { - return; - } - } - throw new SecurityException("Ignoring #onDetected due to trusted service " - + "sharing a keyphrase ID which the DSP did not detect"); - } - - private static final String OP_MESSAGE = - "Providing hotword detection result to VoiceInteractionService"; } diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/TrustedHotwordDetectorSession.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectorSession.java similarity index 60% rename from services/voiceinteraction/java/com/android/server/voiceinteraction/TrustedHotwordDetectorSession.java rename to services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectorSession.java index 02f5889cc5203..87d85ecec7966 100644 --- a/services/voiceinteraction/java/com/android/server/voiceinteraction/TrustedHotwordDetectorSession.java +++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectorSession.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2021 The Android Open Source Project + * Copyright (C) 2022 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -31,8 +31,6 @@ import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTION_SERV import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_UNKNOWN_NO_VALUE; import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_UNKNOWN_OVER_MAX_CUSTOM_VALUE; import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_UNKNOWN_TIMEOUT; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTION_SERVICE_RESTARTED__REASON__AUDIO_SERVICE_DIED; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTION_SERVICE_RESTARTED__REASON__SCHEDULE; import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__APP_REQUEST_UPDATE_STATE; import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION; import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_PROCESS_RESTARTED_EXCEPTION; @@ -43,20 +41,13 @@ import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENT import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__EXTERNAL_SOURCE_DETECTED; import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__EXTERNAL_SOURCE_DETECT_SECURITY_EXCEPTION; import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__EXTERNAL_SOURCE_REJECTED; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__ON_CONNECTED; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__ON_DISCONNECTED; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_BIND_SERVICE; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_BIND_SERVICE_FAIL; import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_UPDATE_STATE; import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__START_EXTERNAL_SOURCE_DETECTION; import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__START_SOFTWARE_DETECTION; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__DETECTOR_TYPE__NORMAL_DETECTOR; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__DETECTOR_TYPE__TRUSTED_DETECTOR_DSP; import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECTED; import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_SECURITY_EXCEPTION; import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_TIMEOUT; import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_UNEXPECTED_CALLBACK; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__KEYPHRASE_TRIGGER; import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED; import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED_FROM_RESTART; import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECT_UNEXPECTED_CALLBACK; @@ -66,27 +57,19 @@ import android.annotation.NonNull; import android.annotation.Nullable; import android.app.AppOpsManager; import android.attention.AttentionManagerInternal; -import android.content.ComponentName; -import android.content.ContentCaptureOptions; import android.content.Context; -import android.content.Intent; import android.content.PermissionChecker; -import android.hardware.soundtrigger.IRecognitionStatusCallback; import android.hardware.soundtrigger.SoundTrigger; import android.media.AudioFormat; -import android.media.AudioManagerInternal; import android.media.permission.Identity; import android.media.permission.PermissionUtil; import android.os.Binder; import android.os.Bundle; -import android.os.IBinder; import android.os.IRemoteCallback; import android.os.ParcelFileDescriptor; import android.os.PersistableBundle; import android.os.RemoteException; -import android.os.ServiceManager; import android.os.SharedMemory; -import android.provider.DeviceConfig; import android.service.voice.HotwordDetectedResult; import android.service.voice.HotwordDetectionService; import android.service.voice.HotwordDetector; @@ -94,19 +77,14 @@ import android.service.voice.HotwordRejectedResult; import android.service.voice.IDspHotwordDetectionCallback; import android.service.voice.IHotwordDetectionService; import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback; -import android.service.voice.VoiceInteractionManagerInternal.HotwordDetectionServiceIdentity; -import android.speech.IRecognitionServiceManager; import android.text.TextUtils; import android.util.Pair; import android.util.Slog; -import android.view.contentcapture.IContentCaptureManager; import com.android.internal.annotations.GuardedBy; import com.android.internal.app.IHotwordRecognitionStatusCallback; import com.android.internal.infra.AndroidFuture; -import com.android.internal.infra.ServiceConnector; import com.android.server.LocalServices; -import com.android.server.pm.permission.PermissionManagerServiceInternal; import java.io.Closeable; import java.io.IOException; @@ -122,16 +100,24 @@ import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.Function; /** - * A class that provides the communication with the HotwordDetectionService. + * A class that provides trusted hotword detector to communicate with the {@link + * HotwordDetectionService}. */ -final class TrustedHotwordDetectorSession { - private static final String TAG = "TrustedHotwordDetectorSession"; +final class HotwordDetectorSession { + private static final String TAG = "HotwordDetectorSession"; static final boolean DEBUG = false; - private static final String KEY_RESTART_PERIOD_IN_SECONDS = "restart_period_in_seconds"; + private static final String OP_MESSAGE = + "Providing hotword detection result to VoiceInteractionService"; + + // The error codes are used for onError callback + static final int HOTWORD_DETECTION_SERVICE_DIED = -1; + static final int CALLBACK_ONDETECTED_GOT_SECURITY_EXCEPTION = -2; + static final int CALLBACK_DETECT_TIMEOUT = -3; + static final int CALLBACK_ONDETECTED_STREAM_COPY_ERROR = -4; + // TODO: These constants need to be refined. // The validation timeout value is 3 seconds for onDetect of DSP trigger event. private static final long VALIDATION_TIMEOUT_MILLIS = 3000; @@ -141,14 +127,6 @@ final class TrustedHotwordDetectorSession { private static final long EXTERNAL_HOTWORD_CLEANUP_MILLIS = 2000; private static final Duration MAX_UPDATE_TIMEOUT_DURATION = Duration.ofMillis(MAX_UPDATE_TIMEOUT_MILLIS); - private static final long RESET_DEBUG_HOTWORD_LOGGING_TIMEOUT_MILLIS = 60 * 60 * 1000; // 1 hour - private static final int MAX_ISOLATED_PROCESS_NUMBER = 10; - - // The error codes are used for onError callback - private static final int HOTWORD_DETECTION_SERVICE_DIED = -1; - private static final int CALLBACK_ONDETECTED_GOT_SECURITY_EXCEPTION = -2; - private static final int CALLBACK_DETECT_TIMEOUT = -3; - private static final int CALLBACK_ONDETECTED_STREAM_COPY_ERROR = -4; // Hotword metrics private static final int METRICS_INIT_UNKNOWN_TIMEOUT = @@ -180,26 +158,15 @@ final class TrustedHotwordDetectorSession { private final Executor mAudioCopyExecutor = Executors.newCachedThreadPool(); // TODO: This may need to be a Handler(looper) - private final ScheduledExecutorService mScheduledExecutorService = - Executors.newSingleThreadScheduledExecutor(); + private final ScheduledExecutorService mScheduledExecutorService; private final AppOpsManager mAppOpsManager; private final HotwordAudioStreamCopier mHotwordAudioStreamCopier; - @Nullable private final ScheduledFuture mCancellationTaskFuture; private final AtomicBoolean mUpdateStateAfterStartFinished = new AtomicBoolean(false); - private final IBinder.DeathRecipient mAudioServerDeathRecipient = this::audioServerDied; - private final @NonNull ServiceConnectionFactory mServiceConnectionFactory; private final IHotwordRecognitionStatusCallback mCallback; private final int mDetectorType; - /** - * Time after which each HotwordDetectionService process is stopped and replaced by a new one. - * 0 indicates no restarts. - */ - private final int mReStartPeriodSeconds; final Object mLock; final int mVoiceInteractionServiceUid; - final ComponentName mDetectionComponentName; - final int mUser; final Context mContext; @Nullable AttentionManagerInternal mAttentionManagerInternal = null; @@ -207,13 +174,9 @@ final class TrustedHotwordDetectorSession { final AttentionManagerInternal.ProximityUpdateCallbackInternal mProximityCallbackInternal = this::setProximityValue; - - volatile HotwordDetectionServiceIdentity mIdentity; private IMicrophoneHotwordDetectionVoiceInteractionCallback mSoftwareCallback; - private Instant mLastRestartInstant; private ScheduledFuture mCancellationKeyPhraseDetectionFuture; - private ScheduledFuture mDebugHotwordLoggingTimeoutFuture = null; /** Identity used for attributing app ops when delivering data to the Interactor. */ @GuardedBy("mLock") @@ -225,107 +188,49 @@ final class TrustedHotwordDetectorSession { private boolean mValidatingDspTrigger = false; @GuardedBy("mLock") private boolean mPerformingSoftwareHotwordDetection; - private @NonNull ServiceConnection mRemoteHotwordDetectionService; - private IBinder mAudioFlinger; + @NonNull private HotwordDetectionConnection.ServiceConnection mRemoteHotwordDetectionService; private boolean mDebugHotwordLogging = false; @GuardedBy("mLock") private double mProximityMeters = PROXIMITY_UNKNOWN; + @GuardedBy("mLock") + private boolean mInitialized = false; + @GuardedBy("mLock") + private boolean mDestroyed = false; - TrustedHotwordDetectorSession(Object lock, Context context, int voiceInteractionServiceUid, - Identity voiceInteractorIdentity, ComponentName serviceName, int userId, - boolean bindInstantServiceAllowed, @Nullable PersistableBundle options, - @Nullable SharedMemory sharedMemory, - @NonNull IHotwordRecognitionStatusCallback callback, int detectorType) { - if (callback == null) { - Slog.w(TAG, "Callback is null while creating connection"); - throw new IllegalArgumentException("Callback is null while creating connection"); - } + HotwordDetectorSession( + @NonNull HotwordDetectionConnection.ServiceConnection remoteHotwordDetectionService, + @NonNull Object lock, @NonNull Context context, + @NonNull IHotwordRecognitionStatusCallback callback, int detectorType, + int voiceInteractionServiceUid, Identity voiceInteractorIdentity, + @NonNull ScheduledExecutorService scheduledExecutorService, boolean logging) { + mRemoteHotwordDetectionService = remoteHotwordDetectionService; mLock = lock; mContext = context; + mCallback = callback; + mDetectorType = detectorType; mVoiceInteractionServiceUid = voiceInteractionServiceUid; mVoiceInteractorIdentity = voiceInteractorIdentity; mAppOpsManager = mContext.getSystemService(AppOpsManager.class); mHotwordAudioStreamCopier = new HotwordAudioStreamCopier(mAppOpsManager, detectorType, mVoiceInteractorIdentity.uid, mVoiceInteractorIdentity.packageName, mVoiceInteractorIdentity.attributionTag); - mDetectionComponentName = serviceName; - mUser = userId; - mCallback = callback; - mDetectorType = detectorType; - mReStartPeriodSeconds = DeviceConfig.getInt(DeviceConfig.NAMESPACE_VOICE_INTERACTION, - KEY_RESTART_PERIOD_IN_SECONDS, 0); - final Intent intent = new Intent(HotwordDetectionService.SERVICE_INTERFACE); - intent.setComponent(mDetectionComponentName); - initAudioFlingerLocked(); + mScheduledExecutorService = scheduledExecutorService; + mDebugHotwordLogging = logging; - mServiceConnectionFactory = new ServiceConnectionFactory(intent, bindInstantServiceAllowed); - - mRemoteHotwordDetectionService = mServiceConnectionFactory.createLocked(); if (ENABLE_PROXIMITY_RESULT) { mAttentionManagerInternal = LocalServices.getService(AttentionManagerInternal.class); if (mAttentionManagerInternal != null) { mAttentionManagerInternal.onStartProximityUpdates(mProximityCallbackInternal); } } - - mLastRestartInstant = Instant.now(); - updateStateAfterProcessStart(options, sharedMemory); - - if (mReStartPeriodSeconds <= 0) { - mCancellationTaskFuture = null; - } else { - // TODO: we need to be smarter here, e.g. schedule it a bit more often, - // but wait until the current session is closed. - mCancellationTaskFuture = mScheduledExecutorService.scheduleAtFixedRate(() -> { - Slog.v(TAG, "Time to restart the process, TTL has passed"); - synchronized (mLock) { - restartProcessLocked(); - HotwordMetricsLogger.writeServiceRestartEvent(mDetectorType, - HOTWORD_DETECTION_SERVICE_RESTARTED__REASON__SCHEDULE); - } - }, mReStartPeriodSeconds, mReStartPeriodSeconds, TimeUnit.SECONDS); - } } - private void initAudioFlingerLocked() { + private void updateStateAfterProcessStartLocked(PersistableBundle options, + SharedMemory sharedMemory) { if (DEBUG) { - Slog.d(TAG, "initAudioFlingerLocked"); + Slog.d(TAG, "updateStateAfterProcessStartLocked"); } - mAudioFlinger = ServiceManager.waitForService("media.audio_flinger"); - if (mAudioFlinger == null) { - throw new IllegalStateException("Service media.audio_flinger wasn't found."); - } - if (DEBUG) { - Slog.d(TAG, "Obtained audio_flinger binder."); - } - try { - mAudioFlinger.linkToDeath(mAudioServerDeathRecipient, /* flags= */ 0); - } catch (RemoteException e) { - Slog.w(TAG, "Audio server died before we registered a DeathRecipient; retrying init.", - e); - initAudioFlingerLocked(); - } - } - - private void audioServerDied() { - Slog.w(TAG, "Audio server died; restarting the HotwordDetectionService."); - synchronized (mLock) { - // TODO: Check if this needs to be scheduled on a different thread. - initAudioFlingerLocked(); - // We restart the process instead of simply sending over the new binder, to avoid race - // conditions with audio reading in the service. - restartProcessLocked(); - HotwordMetricsLogger.writeServiceRestartEvent(mDetectorType, - HOTWORD_DETECTION_SERVICE_RESTARTED__REASON__AUDIO_SERVICE_DIED); - } - } - - private void updateStateAfterProcessStart( - PersistableBundle options, SharedMemory sharedMemory) { - if (DEBUG) { - Slog.d(TAG, "updateStateAfterProcessStart"); - } - mRemoteHotwordDetectionService.postAsync(service -> { + AndroidFuture voidFuture = mRemoteHotwordDetectionService.postAsync(service -> { AndroidFuture future = new AndroidFuture<>(); IRemoteCallback statusCallback = new IRemoteCallback.Stub() { @Override @@ -387,10 +292,11 @@ final class TrustedHotwordDetectorSession { } } else if (err != null) { Slog.w(TAG, "Failed to update state: " + err); - } else { - // NOTE: so far we don't need to take any action. } }); + if (voidFuture == null) { + Slog.w(TAG, "Failed to create AndroidFuture"); + } } private static Pair getInitStatusAndMetricsResult(Bundle bundle) { @@ -401,8 +307,8 @@ final class TrustedHotwordDetectorSession { if (status > HotwordDetectionService.getMaxCustomInitializationStatus()) { return new Pair<>(INITIALIZATION_STATUS_UNKNOWN, status == INITIALIZATION_STATUS_UNKNOWN - ? METRICS_INIT_UNKNOWN_NO_VALUE - : METRICS_INIT_UNKNOWN_OVER_MAX_CUSTOM_VALUE); + ? METRICS_INIT_UNKNOWN_NO_VALUE + : METRICS_INIT_UNKNOWN_OVER_MAX_CUSTOM_VALUE); } // TODO: should guard against negative here int metricsResult = status == INITIALIZATION_STATUS_SUCCESS @@ -411,52 +317,28 @@ final class TrustedHotwordDetectorSession { return new Pair<>(status, metricsResult); } - private boolean isBound() { - synchronized (mLock) { - return mRemoteHotwordDetectionService.isBound(); - } - } - - void cancelLocked() { - Slog.v(TAG, "cancelLocked"); - clearDebugHotwordLoggingTimeoutLocked(); - mDebugHotwordLogging = false; - mRemoteHotwordDetectionService.unbind(); - LocalServices.getService(PermissionManagerServiceInternal.class) - .setHotwordDetectionServiceProvider(null); - if (mIdentity != null) { - removeServiceUidForAudioPolicy(mIdentity.getIsolatedUid()); - } - mIdentity = null; - if (mCancellationTaskFuture != null) { - mCancellationTaskFuture.cancel(/* may interrupt */ true); - } - if (mAudioFlinger != null) { - mAudioFlinger.unlinkToDeath(mAudioServerDeathRecipient, /* flags= */ 0); - } - if (mAttentionManagerInternal != null) { - mAttentionManagerInternal.onStopProximityUpdates(mProximityCallbackInternal); - } - } - - void updateStateLocked(PersistableBundle options, SharedMemory sharedMemory) { + void updateStateLocked(PersistableBundle options, SharedMemory sharedMemory, + Instant lastRestartInstant) { HotwordMetricsLogger.writeDetectorEvent(mDetectorType, HOTWORD_DETECTOR_EVENTS__EVENT__APP_REQUEST_UPDATE_STATE, mVoiceInteractionServiceUid); - // Prevent doing the init late, so restart is handled equally to a clean process start. // TODO(b/191742511): this logic needs a test - if (!mUpdateStateAfterStartFinished.get() - && Instant.now().minus(MAX_UPDATE_TIMEOUT_DURATION).isBefore(mLastRestartInstant)) { - Slog.v(TAG, "call updateStateAfterProcessStart"); - updateStateAfterProcessStart(options, sharedMemory); + if (!mUpdateStateAfterStartFinished.get() && Instant.now().minus( + MAX_UPDATE_TIMEOUT_DURATION).isBefore(lastRestartInstant)) { + Slog.v(TAG, "call updateStateAfterProcessStartLocked"); + updateStateAfterProcessStartLocked(options, sharedMemory); } else { mRemoteHotwordDetectionService.run( - service -> service.updateState(options, sharedMemory, null /* callback */)); + service -> service.updateState(options, sharedMemory, /* callback= */ null)); } } - void startListeningFromMic( + /** + * This method is only used by SoftwareHotwordDetector. + */ + @SuppressWarnings("GuardedBy") + void startListeningFromMicLocked( AudioFormat audioFormat, IMicrophoneHotwordDetectionVoiceInteractionCallback callback) { if (DEBUG) { @@ -464,17 +346,18 @@ final class TrustedHotwordDetectorSession { } mSoftwareCallback = callback; - synchronized (mLock) { - if (mPerformingSoftwareHotwordDetection) { - Slog.i(TAG, "Hotword validation is already in progress, ignoring."); - return; - } - mPerformingSoftwareHotwordDetection = true; - - startListeningFromMicLocked(); + if (mPerformingSoftwareHotwordDetection) { + Slog.i(TAG, "Hotword validation is already in progress, ignoring."); + return; } + mPerformingSoftwareHotwordDetection = true; + + startListeningFromMicLocked(); } + /** + * This method is only used by SoftwareHotwordDetector. + */ private void startListeningFromMicLocked() { // TODO: consider making this a non-anonymous class. IDspHotwordDetectionCallback internalCallback = new IDspHotwordDetectionCallback.Stub() { @@ -546,7 +429,7 @@ final class TrustedHotwordDetectorSession { mVoiceInteractionServiceUid); } - public void startListeningFromExternalSource( + public void startListeningFromExternalSourceLocked( ParcelFileDescriptor audioStream, AudioFormat audioFormat, @Nullable PersistableBundle options, @@ -555,23 +438,21 @@ final class TrustedHotwordDetectorSession { Slog.d(TAG, "startListeningFromExternalSource"); } - handleExternalSourceHotwordDetection( + handleExternalSourceHotwordDetectionLocked( audioStream, audioFormat, options, callback); } - void stopListening() { + /** + * This method is only used by SoftwareHotwordDetector. + */ + @SuppressWarnings("GuardedBy") + void stopListeningLocked() { if (DEBUG) { Slog.d(TAG, "stopListening"); } - synchronized (mLock) { - stopListeningLocked(); - } - } - - private void stopListeningLocked() { if (!mPerformingSoftwareHotwordDetection) { Slog.i(TAG, "Hotword detection is not running"); return; @@ -587,16 +468,8 @@ final class TrustedHotwordDetectorSession { mCurrentAudioSink = null; } - void triggerHardwareRecognitionEventForTestLocked( - SoundTrigger.KeyphraseRecognitionEvent event, - IHotwordRecognitionStatusCallback callback) { - if (DEBUG) { - Slog.d(TAG, "triggerHardwareRecognitionEventForTestLocked"); - } - detectFromDspSource(event, callback); - } - - private void detectFromDspSource(SoundTrigger.KeyphraseRecognitionEvent recognitionEvent, + @SuppressWarnings("GuardedBy") + void detectFromDspSourceLocked(SoundTrigger.KeyphraseRecognitionEvent recognitionEvent, IHotwordRecognitionStatusCallback externalCallback) { if (DEBUG) { Slog.d(TAG, "detectFromDspSource"); @@ -688,207 +561,61 @@ final class TrustedHotwordDetectorSession { } }; - synchronized (mLock) { - mValidatingDspTrigger = true; - mRemoteHotwordDetectionService.run(service -> { - // We use the VALIDATION_TIMEOUT_MILLIS to inform that the client needs to invoke - // the callback before timeout value. In order to reduce the latency impact between - // server side and client side, we need to use another timeout value - // MAX_VALIDATION_TIMEOUT_MILLIS to monitor it. - mCancellationKeyPhraseDetectionFuture = mScheduledExecutorService.schedule( - () -> { - // TODO: avoid allocate every time - timeoutDetected.set(true); - Slog.w(TAG, "Timed out on #detectFromDspSource"); - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - mDetectorType, - HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_TIMEOUT); - try { - externalCallback.onError(CALLBACK_DETECT_TIMEOUT); - } catch (RemoteException e) { - Slog.w(TAG, "Failed to report onError status: ", e); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION, - mVoiceInteractionServiceUid); - } - }, - MAX_VALIDATION_TIMEOUT_MILLIS, - TimeUnit.MILLISECONDS); - service.detectFromDspSource( - recognitionEvent, - recognitionEvent.getCaptureFormat(), - VALIDATION_TIMEOUT_MILLIS, - internalCallback); - }); - } + mValidatingDspTrigger = true; + mRemoteHotwordDetectionService.run(service -> { + // We use the VALIDATION_TIMEOUT_MILLIS to inform that the client needs to invoke + // the callback before timeout value. In order to reduce the latency impact between + // server side and client side, we need to use another timeout value + // MAX_VALIDATION_TIMEOUT_MILLIS to monitor it. + mCancellationKeyPhraseDetectionFuture = mScheduledExecutorService.schedule( + () -> { + // TODO: avoid allocate every time + timeoutDetected.set(true); + Slog.w(TAG, "Timed out on #detectFromDspSource"); + HotwordMetricsLogger.writeKeyphraseTriggerEvent( + mDetectorType, + HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_TIMEOUT); + try { + externalCallback.onError(CALLBACK_DETECT_TIMEOUT); + } catch (RemoteException e) { + Slog.w(TAG, "Failed to report onError status: ", e); + HotwordMetricsLogger.writeDetectorEvent(mDetectorType, + HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION, + mVoiceInteractionServiceUid); + } + }, + MAX_VALIDATION_TIMEOUT_MILLIS, + TimeUnit.MILLISECONDS); + service.detectFromDspSource( + recognitionEvent, + recognitionEvent.getCaptureFormat(), + VALIDATION_TIMEOUT_MILLIS, + internalCallback); + }); } - void forceRestart() { - Slog.v(TAG, "Requested to restart the service internally. Performing the restart"); - synchronized (mLock) { - restartProcessLocked(); - } + @SuppressWarnings("GuardedBy") + public void dumpLocked(String prefix, PrintWriter pw) { + pw.print(prefix); pw.print("mCallback="); pw.println(mCallback); + pw.print(prefix); pw.print("mUpdateStateAfterStartFinished="); + pw.println(mUpdateStateAfterStartFinished); + pw.print(prefix); pw.print("mInitialized="); pw.println(mInitialized); + pw.print(prefix); pw.print("mDestroyed="); pw.println(mDestroyed); + pw.print(prefix); pw.print("mValidatingDspTrigger="); pw.println(mValidatingDspTrigger); + pw.print(prefix); pw.print("mPerformingSoftwareHotwordDetection="); + pw.println(mPerformingSoftwareHotwordDetection); + pw.print(prefix); pw.print("mDetectorType="); + pw.println(HotwordDetector.detectorTypeToString(mDetectorType)); } - void setDebugHotwordLoggingLocked(boolean logging) { - Slog.v(TAG, "setDebugHotwordLoggingLocked: " + logging); - clearDebugHotwordLoggingTimeoutLocked(); - mDebugHotwordLogging = logging; - - if (logging) { - // Reset mDebugHotwordLogging to false after one hour - mDebugHotwordLoggingTimeoutFuture = mScheduledExecutorService.schedule(() -> { - Slog.v(TAG, "Timeout to reset mDebugHotwordLogging to false"); - synchronized (mLock) { - mDebugHotwordLogging = false; - } - }, RESET_DEBUG_HOTWORD_LOGGING_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS); - } - } - - private void clearDebugHotwordLoggingTimeoutLocked() { - if (mDebugHotwordLoggingTimeoutFuture != null) { - mDebugHotwordLoggingTimeoutFuture.cancel(/* mayInterruptIfRunning= */true); - mDebugHotwordLoggingTimeoutFuture = null; - } - } - - private void restartProcessLocked() { - // TODO(b/244598068): Check HotwordAudioStreamManager first - Slog.v(TAG, "Restarting hotword detection process"); - ServiceConnection oldConnection = mRemoteHotwordDetectionService; - HotwordDetectionServiceIdentity previousIdentity = mIdentity; - - // TODO(volnov): this can be done after connect() has been successful. - if (mValidatingDspTrigger) { - // We're restarting the process while it's processing a DSP trigger, so report a - // rejection. This also allows the Interactor to startReco again - try { - mCallback.onRejected(new HotwordRejectedResult.Builder().build()); - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - mDetectorType, - HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED_FROM_RESTART); - } catch (RemoteException e) { - Slog.w(TAG, "Failed to call #rejected"); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_REJECTED_EXCEPTION, - mVoiceInteractionServiceUid); - } - mValidatingDspTrigger = false; - } - - mUpdateStateAfterStartFinished.set(false); - mLastRestartInstant = Instant.now(); - - // Recreate connection to reset the cache. - mRemoteHotwordDetectionService = mServiceConnectionFactory.createLocked(); - - Slog.v(TAG, "Started the new process, issuing #onProcessRestarted"); - try { - mCallback.onProcessRestarted(); - } catch (RemoteException e) { - Slog.w(TAG, "Failed to communicate #onProcessRestarted", e); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_PROCESS_RESTARTED_EXCEPTION, - mVoiceInteractionServiceUid); - } - - // Restart listening from microphone if the hotword process has been restarted. - if (mPerformingSoftwareHotwordDetection) { - Slog.i(TAG, "Process restarted: calling startRecognition() again"); - startListeningFromMicLocked(); - } - - if (mCurrentAudioSink != null) { - Slog.i(TAG, "Closing external audio stream to hotword detector: process restarted"); - bestEffortClose(mCurrentAudioSink); - mCurrentAudioSink = null; - } - - if (DEBUG) { - Slog.i(TAG, "#onProcessRestarted called, unbinding from the old process"); - } - oldConnection.ignoreConnectionStatusEvents(); - oldConnection.unbind(); - if (previousIdentity != null) { - removeServiceUidForAudioPolicy(previousIdentity.getIsolatedUid()); - } - } - - static final class SoundTriggerCallback extends IRecognitionStatusCallback.Stub { - private SoundTrigger.KeyphraseRecognitionEvent mRecognitionEvent; - private final HotwordDetectionConnection mHotwordDetectionConnection; - private final IHotwordRecognitionStatusCallback mExternalCallback; - - SoundTriggerCallback(IHotwordRecognitionStatusCallback callback, - HotwordDetectionConnection connection) { - mHotwordDetectionConnection = connection; - mExternalCallback = callback; - } - - @Override - public void onKeyphraseDetected(SoundTrigger.KeyphraseRecognitionEvent recognitionEvent) - throws RemoteException { - if (DEBUG) { - Slog.d(TAG, "onKeyphraseDetected recognitionEvent : " + recognitionEvent); - } - final boolean useHotwordDetectionService = mHotwordDetectionConnection != null; - if (useHotwordDetectionService) { - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__DETECTOR_TYPE__TRUSTED_DETECTOR_DSP, - HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__KEYPHRASE_TRIGGER); - mRecognitionEvent = recognitionEvent; - mHotwordDetectionConnection.detectFromDspSource( - recognitionEvent, mExternalCallback); - } else { - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__DETECTOR_TYPE__NORMAL_DETECTOR, - HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__KEYPHRASE_TRIGGER); - mExternalCallback.onKeyphraseDetected(recognitionEvent, null); - } - } - - @Override - public void onGenericSoundTriggerDetected( - SoundTrigger.GenericRecognitionEvent recognitionEvent) - throws RemoteException { - mExternalCallback.onGenericSoundTriggerDetected(recognitionEvent); - } - - @Override - public void onError(int status) throws RemoteException { - mExternalCallback.onError(status); - } - - @Override - public void onRecognitionPaused() throws RemoteException { - mExternalCallback.onRecognitionPaused(); - } - - @Override - public void onRecognitionResumed() throws RemoteException { - mExternalCallback.onRecognitionResumed(); - } - } - - public void dump(String prefix, PrintWriter pw) { - pw.print(prefix); pw.print("mReStartPeriodSeconds="); pw.println(mReStartPeriodSeconds); - pw.print(prefix); - pw.print("mBound=" + mRemoteHotwordDetectionService.isBound()); - pw.print(", mValidatingDspTrigger=" + mValidatingDspTrigger); - pw.print(", mPerformingSoftwareHotwordDetection=" + mPerformingSoftwareHotwordDetection); - pw.print(", mRestartCount=" + mServiceConnectionFactory.mRestartCount); - pw.print(", mLastRestartInstant=" + mLastRestartInstant); - pw.println(", mDetectorType=" + HotwordDetector.detectorTypeToString(mDetectorType)); - } - - private void handleExternalSourceHotwordDetection( + @SuppressWarnings("GuardedBy") + private void handleExternalSourceHotwordDetectionLocked( ParcelFileDescriptor audioStream, AudioFormat audioFormat, @Nullable PersistableBundle options, IMicrophoneHotwordDetectionVoiceInteractionCallback callback) { if (DEBUG) { - Slog.d(TAG, "#handleExternalSourceHotwordDetection"); + Slog.d(TAG, "#handleExternalSourceHotwordDetectionLocked"); } InputStream audioSource = new ParcelFileDescriptor.AutoCloseInputStream(audioStream); @@ -900,9 +627,7 @@ final class TrustedHotwordDetectorSession { ParcelFileDescriptor serviceAudioSink = clientPipe.second; ParcelFileDescriptor serviceAudioSource = clientPipe.first; - synchronized (mLock) { - mCurrentAudioSink = serviceAudioSink; - } + mCurrentAudioSink = serviceAudioSink; mAudioCopyExecutor.execute(() -> { try (InputStream source = audioSource; @@ -1028,143 +753,89 @@ final class TrustedHotwordDetectorSession { mVoiceInteractionServiceUid); } - private class ServiceConnectionFactory { - private final Intent mIntent; - private final int mBindingFlags; - - private int mRestartCount = 0; - - ServiceConnectionFactory(@NonNull Intent intent, boolean bindInstantServiceAllowed) { - mIntent = intent; - mBindingFlags = bindInstantServiceAllowed ? Context.BIND_ALLOW_INSTANT : 0; - } - - ServiceConnection createLocked() { - ServiceConnection connection = - new ServiceConnection(mContext, mIntent, mBindingFlags, mUser, - IHotwordDetectionService.Stub::asInterface, - mRestartCount++ % MAX_ISOLATED_PROCESS_NUMBER); - connection.connect(); - - updateAudioFlinger(connection, mAudioFlinger); - updateContentCaptureManager(connection); - updateSpeechService(connection); - updateServiceIdentity(connection); - return connection; + void initialize(@Nullable PersistableBundle options, @Nullable SharedMemory sharedMemory) { + synchronized (mLock) { + if (mInitialized || mDestroyed) { + return; + } + updateStateAfterProcessStartLocked(options, sharedMemory); + mInitialized = true; } } - private class ServiceConnection extends ServiceConnector.Impl { - private final Object mLock = new Object(); - - private final Intent mIntent; - private final int mBindingFlags; - private final int mInstanceNumber; - - private boolean mRespectServiceConnectionStatusChanged = true; - private boolean mIsBound = false; - private boolean mIsLoggedFirstConnect = false; - - ServiceConnection(@NonNull Context context, - @NonNull Intent intent, int bindingFlags, int userId, - @Nullable Function binderAsInterface, - int instanceNumber) { - super(context, intent, bindingFlags, userId, binderAsInterface); - this.mIntent = intent; - this.mBindingFlags = bindingFlags; - this.mInstanceNumber = instanceNumber; + @SuppressWarnings("GuardedBy") + void destroyLocked() { + mDestroyed = true; + mDebugHotwordLogging = false; + mRemoteHotwordDetectionService = null; + if (mAttentionManagerInternal != null) { + mAttentionManagerInternal.onStopProximityUpdates(mProximityCallbackInternal); } + } - @Override // from ServiceConnector.Impl - protected void onServiceConnectionStatusChanged(IHotwordDetectionService service, - boolean connected) { - if (DEBUG) { - Slog.d(TAG, "onServiceConnectionStatusChanged connected = " + connected); - } - synchronized (mLock) { - if (!mRespectServiceConnectionStatusChanged) { - Slog.v(TAG, "Ignored onServiceConnectionStatusChanged event"); - return; - } - mIsBound = connected; + void setDebugHotwordLoggingLocked(boolean logging) { + Slog.v(TAG, "setDebugHotwordLogging: " + logging); + mDebugHotwordLogging = logging; + } - if (!connected) { - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - HOTWORD_DETECTOR_EVENTS__EVENT__ON_DISCONNECTED, - mVoiceInteractionServiceUid); - } else if (!mIsLoggedFirstConnect) { - mIsLoggedFirstConnect = true; - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - HOTWORD_DETECTOR_EVENTS__EVENT__ON_CONNECTED, - mVoiceInteractionServiceUid); - } - } - } + void updateRemoteHotwordDetectionServiceLocked( + @NonNull HotwordDetectionConnection.ServiceConnection remoteHotwordDetectionService) { + mRemoteHotwordDetectionService = remoteHotwordDetectionService; + } - @Override - protected long getAutoDisconnectTimeoutMs() { - return -1; - } - - @Override - public void binderDied() { - super.binderDied(); - synchronized (mLock) { - if (!mRespectServiceConnectionStatusChanged) { - Slog.v(TAG, "Ignored #binderDied event"); - return; - } - - Slog.w(TAG, "binderDied"); - try { - mCallback.onError(HOTWORD_DETECTION_SERVICE_DIED); - } catch (RemoteException e) { - Slog.w(TAG, "Failed to report onError status: " + e); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION, - mVoiceInteractionServiceUid); - } - } - } - - @Override - protected boolean bindService( - @NonNull android.content.ServiceConnection serviceConnection) { + @SuppressWarnings("GuardedBy") + void informRestartProcessLocked() { + // TODO(b/244598068): Check HotwordAudioStreamManager first + Slog.v(TAG, "informRestartProcess"); + if (mValidatingDspTrigger) { + // We're restarting the process while it's processing a DSP trigger, so report a + // rejection. This also allows the Interactor to startRecognition again try { + mCallback.onRejected(new HotwordRejectedResult.Builder().build()); + HotwordMetricsLogger.writeKeyphraseTriggerEvent( + mDetectorType, + HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED_FROM_RESTART); + } catch (RemoteException e) { + Slog.w(TAG, "Failed to call #rejected"); HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_BIND_SERVICE, + HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_REJECTED_EXCEPTION, mVoiceInteractionServiceUid); - boolean bindResult = mContext.bindIsolatedService( - mIntent, - Context.BIND_AUTO_CREATE | Context.BIND_FOREGROUND_SERVICE | mBindingFlags, - "hotword_detector_" + mInstanceNumber, - mExecutor, - serviceConnection); - if (!bindResult) { - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_BIND_SERVICE_FAIL, - mVoiceInteractionServiceUid); - } - return bindResult; - } catch (IllegalArgumentException e) { - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, - HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_BIND_SERVICE_FAIL, - mVoiceInteractionServiceUid); - Slog.wtf(TAG, "Can't bind to the hotword detection service!", e); - return false; } + mValidatingDspTrigger = false; } - boolean isBound() { - synchronized (mLock) { - return mIsBound; - } + mUpdateStateAfterStartFinished.set(false); + + try { + mCallback.onProcessRestarted(); + } catch (RemoteException e) { + Slog.w(TAG, "Failed to communicate #onProcessRestarted", e); + HotwordMetricsLogger.writeDetectorEvent(mDetectorType, + HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_PROCESS_RESTARTED_EXCEPTION, + mVoiceInteractionServiceUid); } - void ignoreConnectionStatusEvents() { - synchronized (mLock) { - mRespectServiceConnectionStatusChanged = false; - } + // Restart listening from microphone if the hotword process has been restarted. + if (mPerformingSoftwareHotwordDetection) { + Slog.i(TAG, "Process restarted: calling startRecognition() again"); + startListeningFromMicLocked(); + } + + if (mCurrentAudioSink != null) { + Slog.i(TAG, "Closing external audio stream to hotword detector: process restarted"); + bestEffortClose(mCurrentAudioSink); + mCurrentAudioSink = null; + } + } + + void reportErrorLocked(int status) { + try { + mCallback.onError(status); + } catch (RemoteException e) { + Slog.w(TAG, "Failed to report onError status: " + e); + HotwordMetricsLogger.writeDetectorEvent(mDetectorType, + HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION, + mVoiceInteractionServiceUid); } } @@ -1180,68 +851,6 @@ final class TrustedHotwordDetectorSession { return Pair.create(fileDescriptors[0], fileDescriptors[1]); } - private static void updateAudioFlinger(ServiceConnection connection, IBinder audioFlinger) { - // TODO: Consider using a proxy that limits the exposed API surface. - connection.run(service -> service.updateAudioFlinger(audioFlinger)); - } - - private static void updateContentCaptureManager(ServiceConnection connection) { - IBinder b = ServiceManager - .getService(Context.CONTENT_CAPTURE_MANAGER_SERVICE); - IContentCaptureManager binderService = IContentCaptureManager.Stub.asInterface(b); - connection.run( - service -> service.updateContentCaptureManager(binderService, - new ContentCaptureOptions(null))); - } - - private static void updateSpeechService(ServiceConnection connection) { - IBinder b = ServiceManager.getService(Context.SPEECH_RECOGNITION_SERVICE); - IRecognitionServiceManager binderService = IRecognitionServiceManager.Stub.asInterface(b); - connection.run(service -> { - service.updateRecognitionServiceManager(binderService); - }); - } - - private void updateServiceIdentity(ServiceConnection connection) { - connection.run(service -> service.ping(new IRemoteCallback.Stub() { - @Override - public void sendResult(Bundle bundle) throws RemoteException { - // TODO: Exit if the service has been unbound already (though there's a very low - // chance this happens). - if (DEBUG) { - Slog.d(TAG, "updating hotword UID " + Binder.getCallingUid()); - } - // TODO: Have the provider point to the current state stored in - // VoiceInteractionManagerServiceImpl. - final int uid = Binder.getCallingUid(); - LocalServices.getService(PermissionManagerServiceInternal.class) - .setHotwordDetectionServiceProvider(() -> uid); - mIdentity = new HotwordDetectionServiceIdentity(uid, mVoiceInteractionServiceUid); - addServiceUidForAudioPolicy(uid); - } - })); - } - - private void addServiceUidForAudioPolicy(int uid) { - mScheduledExecutorService.execute(() -> { - AudioManagerInternal audioManager = - LocalServices.getService(AudioManagerInternal.class); - if (audioManager != null) { - audioManager.addAssistantServiceUid(uid); - } - }); - } - - private void removeServiceUidForAudioPolicy(int uid) { - mScheduledExecutorService.execute(() -> { - AudioManagerInternal audioManager = - LocalServices.getService(AudioManagerInternal.class); - if (audioManager != null) { - audioManager.removeAssistantServiceUid(uid); - } - }); - } - private void saveProximityValueToBundle(HotwordDetectedResult result) { synchronized (mLock) { if (result != null && mProximityMeters != PROXIMITY_UNKNOWN) { @@ -1275,19 +884,20 @@ final class TrustedHotwordDetectorSession { // TODO: Share this code with SoundTriggerMiddlewarePermission. private void enforcePermissionsForDataDelivery() { Binder.withCleanCallingIdentity(() -> { - enforcePermissionForPreflight(mContext, mVoiceInteractorIdentity, RECORD_AUDIO); - int hotwordOp = AppOpsManager.strOpToOp(AppOpsManager.OPSTR_RECORD_AUDIO_HOTWORD); - mAppOpsManager.noteOpNoThrow(hotwordOp, - mVoiceInteractorIdentity.uid, mVoiceInteractorIdentity.packageName, - mVoiceInteractorIdentity.attributionTag, OP_MESSAGE); - enforcePermissionForDataDelivery(mContext, mVoiceInteractorIdentity, - CAPTURE_AUDIO_HOTWORD, OP_MESSAGE); + synchronized (mLock) { + enforcePermissionForPreflight(mContext, mVoiceInteractorIdentity, RECORD_AUDIO); + int hotwordOp = AppOpsManager.strOpToOp(AppOpsManager.OPSTR_RECORD_AUDIO_HOTWORD); + mAppOpsManager.noteOpNoThrow(hotwordOp, + mVoiceInteractorIdentity.uid, mVoiceInteractorIdentity.packageName, + mVoiceInteractorIdentity.attributionTag, OP_MESSAGE); + enforcePermissionForDataDelivery(mContext, mVoiceInteractorIdentity, + CAPTURE_AUDIO_HOTWORD, OP_MESSAGE); + } }); } /** - * Throws a {@link SecurityException} iff the given identity has given permission to receive - * data. + * Throws a {@link SecurityException} if the given identity has no permission to receive data. * * @param context A {@link Context}, used for permission checks. * @param identity The identity to check. @@ -1295,8 +905,7 @@ final class TrustedHotwordDetectorSession { * @param reason The reason why we're requesting the permission, for auditing purposes. */ private static void enforcePermissionForDataDelivery(@NonNull Context context, - @NonNull Identity identity, - @NonNull String permission, @NonNull String reason) { + @NonNull Identity identity, @NonNull String permission, @NonNull String reason) { final int status = PermissionUtil.checkPermissionForDataDelivery(context, identity, permission, reason); if (status != PermissionChecker.PERMISSION_GRANTED) { @@ -1319,7 +928,4 @@ final class TrustedHotwordDetectorSession { throw new SecurityException("Ignoring #onDetected due to trusted service " + "sharing a keyphrase ID which the DSP did not detect"); } - - private static final String OP_MESSAGE = - "Providing hotword detection result to VoiceInteractionService"; } From 1566016119cc3fa5365e380ca3cbc9d127786d86 Mon Sep 17 00:00:00 2001 From: lpeter Date: Fri, 18 Nov 2022 01:28:40 +0000 Subject: [PATCH 2/3] Use Dsp session and Software session in HotwordDetectionConnection We use the HotwordDetectorSession to handle the Dsp and Software trusted hotword detectors, it would be better to separate them. We create the DspTrustedHotwordDetectorSession to handle Dsp trusted hotword detector, SoftwareTrustedHotwordDetectorSession to handle software trusted hotword detector. Currently they are just the simple classes, we will have another change to move the methods of HotwordDetectorSession into DspTrustedHotwordDetectorSession and SoftwareTrustedHotwordDetectorSession. Test: atest CtsVoiceInteractionTestCases Bug: 241041976 Change-Id: I0d569eae319acd8260808aac545babd323f1657e --- .../DspTrustedHotwordDetectorSession.java | 43 ++++++++++ .../HotwordDetectionConnection.java | 26 +++++- .../HotwordDetectorSession.java | 82 ++++++++++--------- ...SoftwareTrustedHotwordDetectorSession.java | 43 ++++++++++ 4 files changed, 154 insertions(+), 40 deletions(-) create mode 100644 services/voiceinteraction/java/com/android/server/voiceinteraction/DspTrustedHotwordDetectorSession.java create mode 100644 services/voiceinteraction/java/com/android/server/voiceinteraction/SoftwareTrustedHotwordDetectorSession.java diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/DspTrustedHotwordDetectorSession.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/DspTrustedHotwordDetectorSession.java new file mode 100644 index 0000000000000..c092d9badcb24 --- /dev/null +++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/DspTrustedHotwordDetectorSession.java @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.voiceinteraction; + +import android.annotation.NonNull; +import android.content.Context; +import android.media.permission.Identity; +import android.service.voice.HotwordDetectionService; + +import com.android.internal.app.IHotwordRecognitionStatusCallback; + +import java.util.concurrent.ScheduledExecutorService; + +/** + * A class that provides Dsp trusted hotword detector to communicate with the {@link + * HotwordDetectionService}. + */ +final class DspTrustedHotwordDetectorSession extends HotwordDetectorSession { + + DspTrustedHotwordDetectorSession( + @NonNull HotwordDetectionConnection.ServiceConnection remoteHotwordDetectionService, + @NonNull Object lock, @NonNull Context context, + @NonNull IHotwordRecognitionStatusCallback callback, int voiceInteractionServiceUid, + Identity voiceInteractorIdentity, + @NonNull ScheduledExecutorService scheduledExecutorService, boolean logging) { + super(remoteHotwordDetectionService, lock, context, callback, voiceInteractionServiceUid, + voiceInteractorIdentity, scheduledExecutorService, logging); + } +} diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java index 5141dd34a892d..7e105c3b18c52 100644 --- a/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java +++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java @@ -142,9 +142,17 @@ final class HotwordDetectionConnection { mRemoteHotwordDetectionService = mServiceConnectionFactory.createLocked(); mLastRestartInstant = Instant.now(); - mHotwordDetectorSession = new HotwordDetectorSession(mRemoteHotwordDetectionService, - mLock, mContext, callback, mDetectorType, mVoiceInteractionServiceUid, - mVoiceInteractorIdentity, mScheduledExecutorService, mDebugHotwordLogging); + if (detectorType == HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP) { + mHotwordDetectorSession = new DspTrustedHotwordDetectorSession( + mRemoteHotwordDetectionService, mLock, mContext, callback, + mVoiceInteractionServiceUid, mVoiceInteractorIdentity, + mScheduledExecutorService, mDebugHotwordLogging); + } else { + mHotwordDetectorSession = new SoftwareTrustedHotwordDetectorSession( + mRemoteHotwordDetectionService, mLock, mContext, callback, + mVoiceInteractionServiceUid, mVoiceInteractorIdentity, + mScheduledExecutorService, mDebugHotwordLogging); + } mHotwordDetectorSession.initialize(options, sharedMemory); if (mReStartPeriodSeconds <= 0) { @@ -232,6 +240,10 @@ final class HotwordDetectionConnection { Slog.d(TAG, "startListeningFromMic"); } synchronized (mLock) { + if (!(mHotwordDetectorSession instanceof SoftwareTrustedHotwordDetectorSession)) { + Slog.d(TAG, "It is not a software detector"); + return; + } mHotwordDetectorSession.startListeningFromMicLocked(audioFormat, callback); } } @@ -258,6 +270,10 @@ final class HotwordDetectionConnection { Slog.d(TAG, "stopListening"); } synchronized (mLock) { + if (!(mHotwordDetectorSession instanceof SoftwareTrustedHotwordDetectorSession)) { + Slog.d(TAG, "It is not a software detector"); + return; + } mHotwordDetectorSession.stopListeningLocked(); } } @@ -277,6 +293,10 @@ final class HotwordDetectionConnection { Slog.d(TAG, "detectFromDspSource"); } synchronized (mLock) { + if (!(mHotwordDetectorSession instanceof DspTrustedHotwordDetectorSession)) { + Slog.d(TAG, "It is not a Dsp detector"); + return; + } mHotwordDetectorSession.detectFromDspSourceLocked(recognitionEvent, externalCallback); } } diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectorSession.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectorSession.java index 87d85ecec7966..9cd4f68222a91 100644 --- a/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectorSession.java +++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectorSession.java @@ -105,7 +105,7 @@ import java.util.concurrent.atomic.AtomicBoolean; * A class that provides trusted hotword detector to communicate with the {@link * HotwordDetectionService}. */ -final class HotwordDetectorSession { +class HotwordDetectorSession { private static final String TAG = "HotwordDetectorSession"; static final boolean DEBUG = false; @@ -163,7 +163,6 @@ final class HotwordDetectorSession { private final HotwordAudioStreamCopier mHotwordAudioStreamCopier; private final AtomicBoolean mUpdateStateAfterStartFinished = new AtomicBoolean(false); private final IHotwordRecognitionStatusCallback mCallback; - private final int mDetectorType; final Object mLock; final int mVoiceInteractionServiceUid; @@ -200,18 +199,17 @@ final class HotwordDetectorSession { HotwordDetectorSession( @NonNull HotwordDetectionConnection.ServiceConnection remoteHotwordDetectionService, @NonNull Object lock, @NonNull Context context, - @NonNull IHotwordRecognitionStatusCallback callback, int detectorType, - int voiceInteractionServiceUid, Identity voiceInteractorIdentity, + @NonNull IHotwordRecognitionStatusCallback callback, int voiceInteractionServiceUid, + Identity voiceInteractorIdentity, @NonNull ScheduledExecutorService scheduledExecutorService, boolean logging) { mRemoteHotwordDetectionService = remoteHotwordDetectionService; mLock = lock; mContext = context; mCallback = callback; - mDetectorType = detectorType; mVoiceInteractionServiceUid = voiceInteractionServiceUid; mVoiceInteractorIdentity = voiceInteractorIdentity; mAppOpsManager = mContext.getSystemService(AppOpsManager.class); - mHotwordAudioStreamCopier = new HotwordAudioStreamCopier(mAppOpsManager, detectorType, + mHotwordAudioStreamCopier = new HotwordAudioStreamCopier(mAppOpsManager, getDetectorType(), mVoiceInteractorIdentity.uid, mVoiceInteractorIdentity.packageName, mVoiceInteractorIdentity.attributionTag); mScheduledExecutorService = scheduledExecutorService; @@ -241,7 +239,7 @@ final class HotwordDetectorSession { future.complete(null); if (mUpdateStateAfterStartFinished.getAndSet(true)) { Slog.w(TAG, "call callback after timeout"); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_UPDATE_STATE_AFTER_TIMEOUT, mVoiceInteractionServiceUid); return; @@ -251,11 +249,11 @@ final class HotwordDetectorSession { int initResultMetricsResult = statusResultPair.second; try { mCallback.onStatusReported(status); - HotwordMetricsLogger.writeServiceInitResultEvent(mDetectorType, + HotwordMetricsLogger.writeServiceInitResultEvent(getDetectorType(), initResultMetricsResult); } catch (RemoteException e) { Slog.w(TAG, "Failed to report initialization status: " + e); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), METRICS_CALLBACK_ON_STATUS_REPORTED_EXCEPTION, mVoiceInteractionServiceUid); } @@ -263,13 +261,13 @@ final class HotwordDetectorSession { }; try { service.updateState(options, sharedMemory, statusCallback); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_UPDATE_STATE, mVoiceInteractionServiceUid); } catch (RemoteException e) { // TODO: (b/181842909) Report an error to voice interactor Slog.w(TAG, "Failed to updateState for HotwordDetectionService", e); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), HOTWORD_DETECTOR_EVENTS__EVENT__CALL_UPDATE_STATE_EXCEPTION, mVoiceInteractionServiceUid); } @@ -282,11 +280,11 @@ final class HotwordDetectorSession { } try { mCallback.onStatusReported(INITIALIZATION_STATUS_UNKNOWN); - HotwordMetricsLogger.writeServiceInitResultEvent(mDetectorType, + HotwordMetricsLogger.writeServiceInitResultEvent(getDetectorType(), METRICS_INIT_UNKNOWN_TIMEOUT); } catch (RemoteException e) { Slog.w(TAG, "Failed to report initialization status UNKNOWN", e); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), METRICS_CALLBACK_ON_STATUS_REPORTED_EXCEPTION, mVoiceInteractionServiceUid); } @@ -319,7 +317,7 @@ final class HotwordDetectorSession { void updateStateLocked(PersistableBundle options, SharedMemory sharedMemory, Instant lastRestartInstant) { - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), HOTWORD_DETECTOR_EVENTS__EVENT__APP_REQUEST_UPDATE_STATE, mVoiceInteractionServiceUid); // Prevent doing the init late, so restart is handled equally to a clean process start. @@ -368,12 +366,12 @@ final class HotwordDetectorSession { } synchronized (mLock) { HotwordMetricsLogger.writeKeyphraseTriggerEvent( - mDetectorType, + getDetectorType(), HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECTED); if (!mPerformingSoftwareHotwordDetection) { Slog.i(TAG, "Hotword detection has already completed"); HotwordMetricsLogger.writeKeyphraseTriggerEvent( - mDetectorType, + getDetectorType(), METRICS_KEYPHRASE_TRIGGERED_DETECT_UNEXPECTED_CALLBACK); return; } @@ -382,7 +380,7 @@ final class HotwordDetectorSession { enforcePermissionsForDataDelivery(); } catch (SecurityException e) { HotwordMetricsLogger.writeKeyphraseTriggerEvent( - mDetectorType, + getDetectorType(), METRICS_KEYPHRASE_TRIGGERED_DETECT_SECURITY_EXCEPTION); mSoftwareCallback.onError(); return; @@ -411,7 +409,7 @@ final class HotwordDetectorSession { Slog.wtf(TAG, "onRejected"); } HotwordMetricsLogger.writeKeyphraseTriggerEvent( - mDetectorType, + getDetectorType(), HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED); // onRejected isn't allowed here, and we are not expecting it. } @@ -424,7 +422,7 @@ final class HotwordDetectorSession { null, null, internalCallback)); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), HOTWORD_DETECTOR_EVENTS__EVENT__START_SOFTWARE_DETECTION, mVoiceInteractionServiceUid); } @@ -491,12 +489,12 @@ final class HotwordDetectorSession { return; } HotwordMetricsLogger.writeKeyphraseTriggerEvent( - mDetectorType, + getDetectorType(), HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECTED); if (!mValidatingDspTrigger) { Slog.i(TAG, "Ignoring #onDetected due to a process restart"); HotwordMetricsLogger.writeKeyphraseTriggerEvent( - mDetectorType, + getDetectorType(), METRICS_KEYPHRASE_TRIGGERED_DETECT_UNEXPECTED_CALLBACK); return; } @@ -507,7 +505,7 @@ final class HotwordDetectorSession { } catch (SecurityException e) { Slog.i(TAG, "Ignoring #onDetected due to a SecurityException", e); HotwordMetricsLogger.writeKeyphraseTriggerEvent( - mDetectorType, + getDetectorType(), METRICS_KEYPHRASE_TRIGGERED_DETECT_SECURITY_EXCEPTION); externalCallback.onError(CALLBACK_ONDETECTED_GOT_SECURITY_EXCEPTION); return; @@ -543,12 +541,12 @@ final class HotwordDetectorSession { return; } HotwordMetricsLogger.writeKeyphraseTriggerEvent( - mDetectorType, + getDetectorType(), HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED); if (!mValidatingDspTrigger) { Slog.i(TAG, "Ignoring #onRejected due to a process restart"); HotwordMetricsLogger.writeKeyphraseTriggerEvent( - mDetectorType, + getDetectorType(), METRICS_KEYPHRASE_TRIGGERED_REJECT_UNEXPECTED_CALLBACK); return; } @@ -573,13 +571,13 @@ final class HotwordDetectorSession { timeoutDetected.set(true); Slog.w(TAG, "Timed out on #detectFromDspSource"); HotwordMetricsLogger.writeKeyphraseTriggerEvent( - mDetectorType, + getDetectorType(), HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_TIMEOUT); try { externalCallback.onError(CALLBACK_DETECT_TIMEOUT); } catch (RemoteException e) { Slog.w(TAG, "Failed to report onError status: ", e); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION, mVoiceInteractionServiceUid); } @@ -604,8 +602,8 @@ final class HotwordDetectorSession { pw.print(prefix); pw.print("mValidatingDspTrigger="); pw.println(mValidatingDspTrigger); pw.print(prefix); pw.print("mPerformingSoftwareHotwordDetection="); pw.println(mPerformingSoftwareHotwordDetection); - pw.print(prefix); pw.print("mDetectorType="); - pw.println(HotwordDetector.detectorTypeToString(mDetectorType)); + pw.print(prefix); pw.print("DetectorType="); + pw.println(HotwordDetector.detectorTypeToString(getDetectorType())); } @SuppressWarnings("GuardedBy") @@ -655,7 +653,7 @@ final class HotwordDetectorSession { callback.onError(); } catch (RemoteException ex) { Slog.w(TAG, "Failed to report onError status: " + ex); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION, mVoiceInteractionServiceUid); } @@ -680,7 +678,7 @@ final class HotwordDetectorSession { @Override public void onRejected(HotwordRejectedResult result) throws RemoteException { - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), METRICS_EXTERNAL_SOURCE_REJECTED, mVoiceInteractionServiceUid); mScheduledExecutorService.schedule( @@ -704,7 +702,7 @@ final class HotwordDetectorSession { @Override public void onDetected(HotwordDetectedResult triggerResult) throws RemoteException { - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), METRICS_EXTERNAL_SOURCE_DETECTED, mVoiceInteractionServiceUid); mScheduledExecutorService.schedule( @@ -717,7 +715,7 @@ final class HotwordDetectorSession { try { enforcePermissionsForDataDelivery(); } catch (SecurityException e) { - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), METRICS_EXTERNAL_SOURCE_DETECT_SECURITY_EXCEPTION, mVoiceInteractionServiceUid); callback.onError(); @@ -748,7 +746,7 @@ final class HotwordDetectorSession { // A copy of this has been created and passed to the hotword validator bestEffortClose(serviceAudioSource); }); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), HOTWORD_DETECTOR_EVENTS__EVENT__START_EXTERNAL_SOURCE_DETECTION, mVoiceInteractionServiceUid); } @@ -793,11 +791,11 @@ final class HotwordDetectorSession { try { mCallback.onRejected(new HotwordRejectedResult.Builder().build()); HotwordMetricsLogger.writeKeyphraseTriggerEvent( - mDetectorType, + getDetectorType(), HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED_FROM_RESTART); } catch (RemoteException e) { Slog.w(TAG, "Failed to call #rejected"); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_REJECTED_EXCEPTION, mVoiceInteractionServiceUid); } @@ -810,7 +808,7 @@ final class HotwordDetectorSession { mCallback.onProcessRestarted(); } catch (RemoteException e) { Slog.w(TAG, "Failed to communicate #onProcessRestarted", e); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_PROCESS_RESTARTED_EXCEPTION, mVoiceInteractionServiceUid); } @@ -833,7 +831,7 @@ final class HotwordDetectorSession { mCallback.onError(status); } catch (RemoteException e) { Slog.w(TAG, "Failed to report onError status: " + e); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION, mVoiceInteractionServiceUid); } @@ -928,4 +926,14 @@ final class HotwordDetectorSession { throw new SecurityException("Ignoring #onDetected due to trusted service " + "sharing a keyphrase ID which the DSP did not detect"); } + + private int getDetectorType() { + if (this instanceof DspTrustedHotwordDetectorSession) { + return HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP; + } else if (this instanceof SoftwareTrustedHotwordDetectorSession) { + return HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE; + } + Slog.v(TAG, "Unexpected detector type"); + return -1; + } } diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/SoftwareTrustedHotwordDetectorSession.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/SoftwareTrustedHotwordDetectorSession.java new file mode 100644 index 0000000000000..b89930074707c --- /dev/null +++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/SoftwareTrustedHotwordDetectorSession.java @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.voiceinteraction; + +import android.annotation.NonNull; +import android.content.Context; +import android.media.permission.Identity; +import android.service.voice.HotwordDetectionService; + +import com.android.internal.app.IHotwordRecognitionStatusCallback; + +import java.util.concurrent.ScheduledExecutorService; + +/** + * A class that provides software trusted hotword detector to communicate with the {@link + * HotwordDetectionService}. + */ +final class SoftwareTrustedHotwordDetectorSession extends HotwordDetectorSession { + + SoftwareTrustedHotwordDetectorSession( + @NonNull HotwordDetectionConnection.ServiceConnection remoteHotwordDetectionService, + @NonNull Object lock, @NonNull Context context, + @NonNull IHotwordRecognitionStatusCallback callback, int voiceInteractionServiceUid, + Identity voiceInteractorIdentity, + @NonNull ScheduledExecutorService scheduledExecutorService, boolean logging) { + super(remoteHotwordDetectionService, lock, context, callback, voiceInteractionServiceUid, + voiceInteractorIdentity, scheduledExecutorService, logging); + } +} From 96e34f053b2fa321f8aa687600e4d3f096676293 Mon Sep 17 00:00:00 2001 From: lpeter Date: Fri, 18 Nov 2022 09:30:08 +0000 Subject: [PATCH 3/3] Move the methods into Dsp session and Software session Move the methods of HotwordDetectorSession into DspTrustedHotwordDetectorSession and SoftwareTrustedHotwordDetectorSession Test: atest CtsVoiceInteractionTestCases Bug: 241041976 Change-Id: I53139507697ba86f04158896ef95095993f23f2c --- .../DspTrustedHotwordDetectorSession.java | 211 +++++++ .../HotwordDetectionConnection.java | 8 +- .../HotwordDetectorSession.java | 515 ++++-------------- ...SoftwareTrustedHotwordDetectorSession.java | 171 ++++++ 4 files changed, 508 insertions(+), 397 deletions(-) diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/DspTrustedHotwordDetectorSession.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/DspTrustedHotwordDetectorSession.java index c092d9badcb24..2812264e989cb 100644 --- a/services/voiceinteraction/java/com/android/server/voiceinteraction/DspTrustedHotwordDetectorSession.java +++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/DspTrustedHotwordDetectorSession.java @@ -16,20 +16,61 @@ package com.android.server.voiceinteraction; +import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION; +import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_PROCESS_RESTARTED_EXCEPTION; +import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_REJECTED_EXCEPTION; +import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECTED; +import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_TIMEOUT; +import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED; +import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED_FROM_RESTART; + import android.annotation.NonNull; import android.content.Context; +import android.hardware.soundtrigger.SoundTrigger; import android.media.permission.Identity; +import android.os.PersistableBundle; +import android.os.RemoteException; +import android.os.SharedMemory; +import android.service.voice.AlwaysOnHotwordDetector; +import android.service.voice.HotwordDetectedResult; import android.service.voice.HotwordDetectionService; +import android.service.voice.HotwordDetector; +import android.service.voice.HotwordRejectedResult; +import android.service.voice.IDspHotwordDetectionCallback; +import android.util.Slog; +import com.android.internal.annotations.GuardedBy; import com.android.internal.app.IHotwordRecognitionStatusCallback; +import java.io.IOException; +import java.io.PrintWriter; +import java.util.Locale; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; /** * A class that provides Dsp trusted hotword detector to communicate with the {@link * HotwordDetectionService}. + * + * This class can handle the hotword detection which detector is created by using + * {@link android.service.voice.VoiceInteractionService#createAlwaysOnHotwordDetector(String, + * Locale, PersistableBundle, SharedMemory, AlwaysOnHotwordDetector.Callback)}. */ final class DspTrustedHotwordDetectorSession extends HotwordDetectorSession { + private static final String TAG = "DspTrustedHotwordDetectorSession"; + + // The validation timeout value is 3 seconds for onDetect of DSP trigger event. + private static final long VALIDATION_TIMEOUT_MILLIS = 3000; + // Write the onDetect timeout metric when it takes more time than MAX_VALIDATION_TIMEOUT_MILLIS. + private static final long MAX_VALIDATION_TIMEOUT_MILLIS = 4000; + + @GuardedBy("mLock") + private ScheduledFuture mCancellationKeyPhraseDetectionFuture; + + @GuardedBy("mLock") + private boolean mValidatingDspTrigger = false; DspTrustedHotwordDetectorSession( @NonNull HotwordDetectionConnection.ServiceConnection remoteHotwordDetectionService, @@ -40,4 +81,174 @@ final class DspTrustedHotwordDetectorSession extends HotwordDetectorSession { super(remoteHotwordDetectionService, lock, context, callback, voiceInteractionServiceUid, voiceInteractorIdentity, scheduledExecutorService, logging); } + + @SuppressWarnings("GuardedBy") + void detectFromDspSourceLocked(SoundTrigger.KeyphraseRecognitionEvent recognitionEvent, + IHotwordRecognitionStatusCallback externalCallback) { + if (DEBUG) { + Slog.d(TAG, "detectFromDspSourceLocked"); + } + + AtomicBoolean timeoutDetected = new AtomicBoolean(false); + // TODO: consider making this a non-anonymous class. + IDspHotwordDetectionCallback internalCallback = new IDspHotwordDetectionCallback.Stub() { + @Override + public void onDetected(HotwordDetectedResult result) throws RemoteException { + if (DEBUG) { + Slog.d(TAG, "onDetected"); + } + synchronized (mLock) { + if (mCancellationKeyPhraseDetectionFuture != null) { + mCancellationKeyPhraseDetectionFuture.cancel(true); + } + if (timeoutDetected.get()) { + return; + } + HotwordMetricsLogger.writeKeyphraseTriggerEvent( + HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP, + HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECTED); + if (!mValidatingDspTrigger) { + Slog.i(TAG, "Ignoring #onDetected due to a process restart"); + HotwordMetricsLogger.writeKeyphraseTriggerEvent( + HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP, + METRICS_KEYPHRASE_TRIGGERED_DETECT_UNEXPECTED_CALLBACK); + return; + } + mValidatingDspTrigger = false; + try { + enforcePermissionsForDataDelivery(); + enforceExtraKeyphraseIdNotLeaked(result, recognitionEvent); + } catch (SecurityException e) { + Slog.i(TAG, "Ignoring #onDetected due to a SecurityException", e); + HotwordMetricsLogger.writeKeyphraseTriggerEvent( + HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP, + METRICS_KEYPHRASE_TRIGGERED_DETECT_SECURITY_EXCEPTION); + externalCallback.onError(CALLBACK_ONDETECTED_GOT_SECURITY_EXCEPTION); + return; + } + saveProximityValueToBundle(result); + HotwordDetectedResult newResult; + try { + newResult = mHotwordAudioStreamCopier.startCopyingAudioStreams(result); + } catch (IOException e) { + externalCallback.onError(CALLBACK_ONDETECTED_STREAM_COPY_ERROR); + return; + } + externalCallback.onKeyphraseDetected(recognitionEvent, newResult); + Slog.i(TAG, "Egressed " + HotwordDetectedResult.getUsageSize(newResult) + + " bits from hotword trusted process"); + if (mDebugHotwordLogging) { + Slog.i(TAG, "Egressed detected result: " + newResult); + } + } + } + + @Override + public void onRejected(HotwordRejectedResult result) throws RemoteException { + if (DEBUG) { + Slog.d(TAG, "onRejected"); + } + synchronized (mLock) { + if (mCancellationKeyPhraseDetectionFuture != null) { + mCancellationKeyPhraseDetectionFuture.cancel(true); + } + if (timeoutDetected.get()) { + return; + } + HotwordMetricsLogger.writeKeyphraseTriggerEvent( + HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP, + HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED); + if (!mValidatingDspTrigger) { + Slog.i(TAG, "Ignoring #onRejected due to a process restart"); + HotwordMetricsLogger.writeKeyphraseTriggerEvent( + HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP, + METRICS_KEYPHRASE_TRIGGERED_REJECT_UNEXPECTED_CALLBACK); + return; + } + mValidatingDspTrigger = false; + externalCallback.onRejected(result); + if (mDebugHotwordLogging && result != null) { + Slog.i(TAG, "Egressed rejected result: " + result); + } + } + } + }; + + mValidatingDspTrigger = true; + mRemoteHotwordDetectionService.run(service -> { + // We use the VALIDATION_TIMEOUT_MILLIS to inform that the client needs to invoke + // the callback before timeout value. In order to reduce the latency impact between + // server side and client side, we need to use another timeout value + // MAX_VALIDATION_TIMEOUT_MILLIS to monitor it. + mCancellationKeyPhraseDetectionFuture = mScheduledExecutorService.schedule( + () -> { + // TODO: avoid allocate every time + timeoutDetected.set(true); + Slog.w(TAG, "Timed out on #detectFromDspSource"); + HotwordMetricsLogger.writeKeyphraseTriggerEvent( + HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP, + HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_TIMEOUT); + try { + externalCallback.onError(CALLBACK_DETECT_TIMEOUT); + } catch (RemoteException e) { + Slog.w(TAG, "Failed to report onError status: ", e); + HotwordMetricsLogger.writeDetectorEvent( + HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP, + HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION, + mVoiceInteractionServiceUid); + } + }, + MAX_VALIDATION_TIMEOUT_MILLIS, + TimeUnit.MILLISECONDS); + service.detectFromDspSource( + recognitionEvent, + recognitionEvent.getCaptureFormat(), + VALIDATION_TIMEOUT_MILLIS, + internalCallback); + }); + } + + @Override + @SuppressWarnings("GuardedBy") + void informRestartProcessLocked() { + // TODO(b/244598068): Check HotwordAudioStreamManager first + Slog.v(TAG, "informRestartProcessLocked"); + if (mValidatingDspTrigger) { + // We're restarting the process while it's processing a DSP trigger, so report a + // rejection. This also allows the Interactor to startRecognition again + try { + mCallback.onRejected(new HotwordRejectedResult.Builder().build()); + HotwordMetricsLogger.writeKeyphraseTriggerEvent( + HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP, + HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED_FROM_RESTART); + } catch (RemoteException e) { + Slog.w(TAG, "Failed to call #rejected"); + HotwordMetricsLogger.writeDetectorEvent( + HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP, + HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_REJECTED_EXCEPTION, + mVoiceInteractionServiceUid); + } + mValidatingDspTrigger = false; + } + mUpdateStateAfterStartFinished.set(false); + + try { + mCallback.onProcessRestarted(); + } catch (RemoteException e) { + Slog.w(TAG, "Failed to communicate #onProcessRestarted", e); + HotwordMetricsLogger.writeDetectorEvent( + HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP, + HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_PROCESS_RESTARTED_EXCEPTION, + mVoiceInteractionServiceUid); + } + + mPerformingExternalSourceHotwordDetection = false; + closeExternalAudioStreamLocked("process restarted"); + } + + @SuppressWarnings("GuardedBy") + public void dumpLocked(String prefix, PrintWriter pw) { + super.dumpLocked(prefix, pw); + pw.print(prefix); pw.print("mValidatingDspTrigger="); pw.println(mValidatingDspTrigger); + } } diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java index 7e105c3b18c52..2a4d4a15afdf0 100644 --- a/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java +++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java @@ -244,7 +244,8 @@ final class HotwordDetectionConnection { Slog.d(TAG, "It is not a software detector"); return; } - mHotwordDetectorSession.startListeningFromMicLocked(audioFormat, callback); + ((SoftwareTrustedHotwordDetectorSession) mHotwordDetectorSession) + .startListeningFromMicLocked(audioFormat, callback); } } @@ -274,7 +275,7 @@ final class HotwordDetectionConnection { Slog.d(TAG, "It is not a software detector"); return; } - mHotwordDetectorSession.stopListeningLocked(); + ((SoftwareTrustedHotwordDetectorSession) mHotwordDetectorSession).stopListeningLocked(); } } @@ -297,7 +298,8 @@ final class HotwordDetectionConnection { Slog.d(TAG, "It is not a Dsp detector"); return; } - mHotwordDetectorSession.detectFromDspSourceLocked(recognitionEvent, externalCallback); + ((DspTrustedHotwordDetectorSession) mHotwordDetectorSession).detectFromDspSourceLocked( + recognitionEvent, externalCallback); } } diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectorSession.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectorSession.java index 9cd4f68222a91..f9f43c9ba49a9 100644 --- a/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectorSession.java +++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectorSession.java @@ -20,7 +20,6 @@ import static android.Manifest.permission.CAPTURE_AUDIO_HOTWORD; import static android.Manifest.permission.RECORD_AUDIO; import static android.service.attention.AttentionService.PROXIMITY_UNKNOWN; import static android.service.voice.HotwordDetectionService.AUDIO_SOURCE_EXTERNAL; -import static android.service.voice.HotwordDetectionService.AUDIO_SOURCE_MICROPHONE; import static android.service.voice.HotwordDetectionService.ENABLE_PROXIMITY_RESULT; import static android.service.voice.HotwordDetectionService.INITIALIZATION_STATUS_SUCCESS; import static android.service.voice.HotwordDetectionService.INITIALIZATION_STATUS_UNKNOWN; @@ -33,8 +32,6 @@ import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTION_SERV import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_UNKNOWN_TIMEOUT; import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__APP_REQUEST_UPDATE_STATE; import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_PROCESS_RESTARTED_EXCEPTION; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_REJECTED_EXCEPTION; import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_STATUS_REPORTED_EXCEPTION; import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_UPDATE_STATE_AFTER_TIMEOUT; import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALL_UPDATE_STATE_EXCEPTION; @@ -43,13 +40,8 @@ import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENT import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__EXTERNAL_SOURCE_REJECTED; import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_UPDATE_STATE; import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__START_EXTERNAL_SOURCE_DETECTION; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__START_SOFTWARE_DETECTION; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECTED; import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_SECURITY_EXCEPTION; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_TIMEOUT; import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_UNEXPECTED_CALLBACK; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED; -import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED_FROM_RESTART; import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECT_UNEXPECTED_CALLBACK; import static com.android.server.voiceinteraction.SoundTriggerSessionPermissionsDecorator.enforcePermissionForPreflight; @@ -75,7 +67,6 @@ import android.service.voice.HotwordDetectionService; import android.service.voice.HotwordDetector; import android.service.voice.HotwordRejectedResult; import android.service.voice.IDspHotwordDetectionCallback; -import android.service.voice.IHotwordDetectionService; import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback; import android.text.TextUtils; import android.util.Pair; @@ -96,7 +87,6 @@ import java.time.Instant; import java.util.concurrent.Executor; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; @@ -104,8 +94,15 @@ import java.util.concurrent.atomic.AtomicBoolean; /** * A class that provides trusted hotword detector to communicate with the {@link * HotwordDetectionService}. + * + * This class provides the methods to do initialization with the {@link HotwordDetectionService} + * and handle external source detection. It also provides the methods to check if we can egress + * the data from the {@link HotwordDetectionService}. + * + * The subclass should override the {@link #informRestartProcessLocked()} to handle the trusted + * process restart. */ -class HotwordDetectorSession { +abstract class HotwordDetectorSession { private static final String TAG = "HotwordDetectorSession"; static final boolean DEBUG = false; @@ -119,10 +116,6 @@ class HotwordDetectorSession { static final int CALLBACK_ONDETECTED_STREAM_COPY_ERROR = -4; // TODO: These constants need to be refined. - // The validation timeout value is 3 seconds for onDetect of DSP trigger event. - private static final long VALIDATION_TIMEOUT_MILLIS = 3000; - // Write the onDetect timeout metric when it takes more time than MAX_VALIDATION_TIMEOUT_MILLIS. - private static final long MAX_VALIDATION_TIMEOUT_MILLIS = 4000; private static final long MAX_UPDATE_TIMEOUT_MILLIS = 30000; private static final long EXTERNAL_HOTWORD_CLEANUP_MILLIS = 2000; private static final Duration MAX_UPDATE_TIMEOUT_DURATION = @@ -140,29 +133,29 @@ class HotwordDetectorSession { private static final int METRICS_INIT_CALLBACK_STATE_SUCCESS = HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_SUCCESS; - private static final int METRICS_KEYPHRASE_TRIGGERED_DETECT_SECURITY_EXCEPTION = + static final int METRICS_KEYPHRASE_TRIGGERED_DETECT_SECURITY_EXCEPTION = HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_SECURITY_EXCEPTION; - private static final int METRICS_KEYPHRASE_TRIGGERED_DETECT_UNEXPECTED_CALLBACK = + static final int METRICS_KEYPHRASE_TRIGGERED_DETECT_UNEXPECTED_CALLBACK = HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_UNEXPECTED_CALLBACK; - private static final int METRICS_KEYPHRASE_TRIGGERED_REJECT_UNEXPECTED_CALLBACK = + static final int METRICS_KEYPHRASE_TRIGGERED_REJECT_UNEXPECTED_CALLBACK = HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECT_UNEXPECTED_CALLBACK; private static final int METRICS_EXTERNAL_SOURCE_DETECTED = HOTWORD_DETECTOR_EVENTS__EVENT__EXTERNAL_SOURCE_DETECTED; private static final int METRICS_EXTERNAL_SOURCE_REJECTED = HOTWORD_DETECTOR_EVENTS__EVENT__EXTERNAL_SOURCE_REJECTED; - private static final int METRICS_EXTERNAL_SOURCE_DETECT_SECURITY_EXCEPTION = + private static final int EXTERNAL_SOURCE_DETECT_SECURITY_EXCEPTION = HOTWORD_DETECTOR_EVENTS__EVENT__EXTERNAL_SOURCE_DETECT_SECURITY_EXCEPTION; private static final int METRICS_CALLBACK_ON_STATUS_REPORTED_EXCEPTION = HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_STATUS_REPORTED_EXCEPTION; private final Executor mAudioCopyExecutor = Executors.newCachedThreadPool(); // TODO: This may need to be a Handler(looper) - private final ScheduledExecutorService mScheduledExecutorService; + final ScheduledExecutorService mScheduledExecutorService; private final AppOpsManager mAppOpsManager; - private final HotwordAudioStreamCopier mHotwordAudioStreamCopier; - private final AtomicBoolean mUpdateStateAfterStartFinished = new AtomicBoolean(false); - private final IHotwordRecognitionStatusCallback mCallback; + final HotwordAudioStreamCopier mHotwordAudioStreamCopier; + final AtomicBoolean mUpdateStateAfterStartFinished = new AtomicBoolean(false); + final IHotwordRecognitionStatusCallback mCallback; final Object mLock; final int mVoiceInteractionServiceUid; @@ -173,28 +166,22 @@ class HotwordDetectorSession { final AttentionManagerInternal.ProximityUpdateCallbackInternal mProximityCallbackInternal = this::setProximityValue; - private IMicrophoneHotwordDetectionVoiceInteractionCallback mSoftwareCallback; - - private ScheduledFuture mCancellationKeyPhraseDetectionFuture; - /** Identity used for attributing app ops when delivering data to the Interactor. */ - @GuardedBy("mLock") @Nullable private final Identity mVoiceInteractorIdentity; @GuardedBy("mLock") - private ParcelFileDescriptor mCurrentAudioSink; + ParcelFileDescriptor mCurrentAudioSink; @GuardedBy("mLock") - private boolean mValidatingDspTrigger = false; - @GuardedBy("mLock") - private boolean mPerformingSoftwareHotwordDetection; - @NonNull private HotwordDetectionConnection.ServiceConnection mRemoteHotwordDetectionService; - private boolean mDebugHotwordLogging = false; + @NonNull HotwordDetectionConnection.ServiceConnection mRemoteHotwordDetectionService; + boolean mDebugHotwordLogging = false; @GuardedBy("mLock") private double mProximityMeters = PROXIMITY_UNKNOWN; @GuardedBy("mLock") private boolean mInitialized = false; @GuardedBy("mLock") private boolean mDestroyed = false; + @GuardedBy("mLock") + boolean mPerformingExternalSourceHotwordDetection; HotwordDetectorSession( @NonNull HotwordDetectionConnection.ServiceConnection remoteHotwordDetectionService, @@ -223,6 +210,7 @@ class HotwordDetectorSession { } } + @SuppressWarnings("GuardedBy") private void updateStateAfterProcessStartLocked(PersistableBundle options, SharedMemory sharedMemory) { if (DEBUG) { @@ -315,6 +303,7 @@ class HotwordDetectorSession { return new Pair<>(status, metricsResult); } + @SuppressWarnings("GuardedBy") void updateStateLocked(PersistableBundle options, SharedMemory sharedMemory, Instant lastRestartInstant) { HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), @@ -332,108 +321,13 @@ class HotwordDetectorSession { } } - /** - * This method is only used by SoftwareHotwordDetector. - */ - @SuppressWarnings("GuardedBy") - void startListeningFromMicLocked( - AudioFormat audioFormat, - IMicrophoneHotwordDetectionVoiceInteractionCallback callback) { - if (DEBUG) { - Slog.d(TAG, "startListeningFromMic"); - } - mSoftwareCallback = callback; - - if (mPerformingSoftwareHotwordDetection) { - Slog.i(TAG, "Hotword validation is already in progress, ignoring."); - return; - } - mPerformingSoftwareHotwordDetection = true; - - startListeningFromMicLocked(); - } - - /** - * This method is only used by SoftwareHotwordDetector. - */ - private void startListeningFromMicLocked() { - // TODO: consider making this a non-anonymous class. - IDspHotwordDetectionCallback internalCallback = new IDspHotwordDetectionCallback.Stub() { - @Override - public void onDetected(HotwordDetectedResult result) throws RemoteException { - if (DEBUG) { - Slog.d(TAG, "onDetected"); - } - synchronized (mLock) { - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - getDetectorType(), - HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECTED); - if (!mPerformingSoftwareHotwordDetection) { - Slog.i(TAG, "Hotword detection has already completed"); - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - getDetectorType(), - METRICS_KEYPHRASE_TRIGGERED_DETECT_UNEXPECTED_CALLBACK); - return; - } - mPerformingSoftwareHotwordDetection = false; - try { - enforcePermissionsForDataDelivery(); - } catch (SecurityException e) { - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - getDetectorType(), - METRICS_KEYPHRASE_TRIGGERED_DETECT_SECURITY_EXCEPTION); - mSoftwareCallback.onError(); - return; - } - saveProximityValueToBundle(result); - HotwordDetectedResult newResult; - try { - newResult = mHotwordAudioStreamCopier.startCopyingAudioStreams(result); - } catch (IOException e) { - // TODO: Write event - mSoftwareCallback.onError(); - return; - } - mSoftwareCallback.onDetected(newResult, null, null); - Slog.i(TAG, "Egressed " + HotwordDetectedResult.getUsageSize(newResult) - + " bits from hotword trusted process"); - if (mDebugHotwordLogging) { - Slog.i(TAG, "Egressed detected result: " + newResult); - } - } - } - - @Override - public void onRejected(HotwordRejectedResult result) throws RemoteException { - if (DEBUG) { - Slog.wtf(TAG, "onRejected"); - } - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - getDetectorType(), - HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED); - // onRejected isn't allowed here, and we are not expecting it. - } - }; - - mRemoteHotwordDetectionService.run( - service -> service.detectFromMicrophoneSource( - null, - AUDIO_SOURCE_MICROPHONE, - null, - null, - internalCallback)); - HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), - HOTWORD_DETECTOR_EVENTS__EVENT__START_SOFTWARE_DETECTION, - mVoiceInteractionServiceUid); - } - - public void startListeningFromExternalSourceLocked( + void startListeningFromExternalSourceLocked( ParcelFileDescriptor audioStream, AudioFormat audioFormat, @Nullable PersistableBundle options, IMicrophoneHotwordDetectionVoiceInteractionCallback callback) { if (DEBUG) { - Slog.d(TAG, "startListeningFromExternalSource"); + Slog.d(TAG, "startListeningFromExternalSourceLocked"); } handleExternalSourceHotwordDetectionLocked( @@ -443,169 +337,6 @@ class HotwordDetectorSession { callback); } - /** - * This method is only used by SoftwareHotwordDetector. - */ - @SuppressWarnings("GuardedBy") - void stopListeningLocked() { - if (DEBUG) { - Slog.d(TAG, "stopListening"); - } - if (!mPerformingSoftwareHotwordDetection) { - Slog.i(TAG, "Hotword detection is not running"); - return; - } - mPerformingSoftwareHotwordDetection = false; - - mRemoteHotwordDetectionService.run(IHotwordDetectionService::stopDetection); - - if (mCurrentAudioSink != null) { - Slog.i(TAG, "Closing audio stream to hotword detector: stopping requested"); - bestEffortClose(mCurrentAudioSink); - } - mCurrentAudioSink = null; - } - - @SuppressWarnings("GuardedBy") - void detectFromDspSourceLocked(SoundTrigger.KeyphraseRecognitionEvent recognitionEvent, - IHotwordRecognitionStatusCallback externalCallback) { - if (DEBUG) { - Slog.d(TAG, "detectFromDspSource"); - } - - AtomicBoolean timeoutDetected = new AtomicBoolean(false); - // TODO: consider making this a non-anonymous class. - IDspHotwordDetectionCallback internalCallback = new IDspHotwordDetectionCallback.Stub() { - @Override - public void onDetected(HotwordDetectedResult result) throws RemoteException { - if (DEBUG) { - Slog.d(TAG, "onDetected"); - } - synchronized (mLock) { - if (mCancellationKeyPhraseDetectionFuture != null) { - mCancellationKeyPhraseDetectionFuture.cancel(true); - } - if (timeoutDetected.get()) { - return; - } - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - getDetectorType(), - HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECTED); - if (!mValidatingDspTrigger) { - Slog.i(TAG, "Ignoring #onDetected due to a process restart"); - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - getDetectorType(), - METRICS_KEYPHRASE_TRIGGERED_DETECT_UNEXPECTED_CALLBACK); - return; - } - mValidatingDspTrigger = false; - try { - enforcePermissionsForDataDelivery(); - enforceExtraKeyphraseIdNotLeaked(result, recognitionEvent); - } catch (SecurityException e) { - Slog.i(TAG, "Ignoring #onDetected due to a SecurityException", e); - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - getDetectorType(), - METRICS_KEYPHRASE_TRIGGERED_DETECT_SECURITY_EXCEPTION); - externalCallback.onError(CALLBACK_ONDETECTED_GOT_SECURITY_EXCEPTION); - return; - } - saveProximityValueToBundle(result); - HotwordDetectedResult newResult; - try { - newResult = mHotwordAudioStreamCopier.startCopyingAudioStreams(result); - } catch (IOException e) { - // TODO: Write event - externalCallback.onError(CALLBACK_ONDETECTED_STREAM_COPY_ERROR); - return; - } - externalCallback.onKeyphraseDetected(recognitionEvent, newResult); - Slog.i(TAG, "Egressed " + HotwordDetectedResult.getUsageSize(newResult) - + " bits from hotword trusted process"); - if (mDebugHotwordLogging) { - Slog.i(TAG, "Egressed detected result: " + newResult); - } - } - } - - @Override - public void onRejected(HotwordRejectedResult result) throws RemoteException { - if (DEBUG) { - Slog.d(TAG, "onRejected"); - } - synchronized (mLock) { - if (mCancellationKeyPhraseDetectionFuture != null) { - mCancellationKeyPhraseDetectionFuture.cancel(true); - } - if (timeoutDetected.get()) { - return; - } - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - getDetectorType(), - HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED); - if (!mValidatingDspTrigger) { - Slog.i(TAG, "Ignoring #onRejected due to a process restart"); - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - getDetectorType(), - METRICS_KEYPHRASE_TRIGGERED_REJECT_UNEXPECTED_CALLBACK); - return; - } - mValidatingDspTrigger = false; - externalCallback.onRejected(result); - if (mDebugHotwordLogging && result != null) { - Slog.i(TAG, "Egressed rejected result: " + result); - } - } - } - }; - - mValidatingDspTrigger = true; - mRemoteHotwordDetectionService.run(service -> { - // We use the VALIDATION_TIMEOUT_MILLIS to inform that the client needs to invoke - // the callback before timeout value. In order to reduce the latency impact between - // server side and client side, we need to use another timeout value - // MAX_VALIDATION_TIMEOUT_MILLIS to monitor it. - mCancellationKeyPhraseDetectionFuture = mScheduledExecutorService.schedule( - () -> { - // TODO: avoid allocate every time - timeoutDetected.set(true); - Slog.w(TAG, "Timed out on #detectFromDspSource"); - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - getDetectorType(), - HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_TIMEOUT); - try { - externalCallback.onError(CALLBACK_DETECT_TIMEOUT); - } catch (RemoteException e) { - Slog.w(TAG, "Failed to report onError status: ", e); - HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), - HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION, - mVoiceInteractionServiceUid); - } - }, - MAX_VALIDATION_TIMEOUT_MILLIS, - TimeUnit.MILLISECONDS); - service.detectFromDspSource( - recognitionEvent, - recognitionEvent.getCaptureFormat(), - VALIDATION_TIMEOUT_MILLIS, - internalCallback); - }); - } - - @SuppressWarnings("GuardedBy") - public void dumpLocked(String prefix, PrintWriter pw) { - pw.print(prefix); pw.print("mCallback="); pw.println(mCallback); - pw.print(prefix); pw.print("mUpdateStateAfterStartFinished="); - pw.println(mUpdateStateAfterStartFinished); - pw.print(prefix); pw.print("mInitialized="); pw.println(mInitialized); - pw.print(prefix); pw.print("mDestroyed="); pw.println(mDestroyed); - pw.print(prefix); pw.print("mValidatingDspTrigger="); pw.println(mValidatingDspTrigger); - pw.print(prefix); pw.print("mPerformingSoftwareHotwordDetection="); - pw.println(mPerformingSoftwareHotwordDetection); - pw.print(prefix); pw.print("DetectorType="); - pw.println(HotwordDetector.detectorTypeToString(getDetectorType())); - } - @SuppressWarnings("GuardedBy") private void handleExternalSourceHotwordDetectionLocked( ParcelFileDescriptor audioStream, @@ -615,6 +346,11 @@ class HotwordDetectorSession { if (DEBUG) { Slog.d(TAG, "#handleExternalSourceHotwordDetectionLocked"); } + if (mPerformingExternalSourceHotwordDetection) { + Slog.i(TAG, "Hotword validation is already in progress for external source."); + return; + } + InputStream audioSource = new ParcelFileDescriptor.AutoCloseInputStream(audioStream); Pair clientPipe = createPipe(); @@ -626,6 +362,7 @@ class HotwordDetectorSession { ParcelFileDescriptor serviceAudioSource = clientPipe.first; mCurrentAudioSink = serviceAudioSink; + mPerformingExternalSourceHotwordDetection = true; mAudioCopyExecutor.execute(() -> { try (InputStream source = audioSource; @@ -659,7 +396,8 @@ class HotwordDetectorSession { } } finally { synchronized (mLock) { - mCurrentAudioSink = null; + mPerformingExternalSourceHotwordDetection = false; + closeExternalAudioStreamLocked("start external source"); } } }); @@ -678,23 +416,26 @@ class HotwordDetectorSession { @Override public void onRejected(HotwordRejectedResult result) throws RemoteException { - HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), - METRICS_EXTERNAL_SOURCE_REJECTED, - mVoiceInteractionServiceUid); - mScheduledExecutorService.schedule( - () -> { - bestEffortClose(serviceAudioSink, audioSource); - }, - EXTERNAL_HOTWORD_CLEANUP_MILLIS, - TimeUnit.MILLISECONDS); + synchronized (mLock) { + mPerformingExternalSourceHotwordDetection = false; + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), + METRICS_EXTERNAL_SOURCE_REJECTED, + mVoiceInteractionServiceUid); + mScheduledExecutorService.schedule( + () -> { + bestEffortClose(serviceAudioSink, audioSource); + }, + EXTERNAL_HOTWORD_CLEANUP_MILLIS, + TimeUnit.MILLISECONDS); - callback.onRejected(result); + callback.onRejected(result); - if (result != null) { - Slog.i(TAG, "Egressed 'hotword rejected result' " - + "from hotword trusted process"); - if (mDebugHotwordLogging) { - Slog.i(TAG, "Egressed detected result: " + result); + if (result != null) { + Slog.i(TAG, "Egressed 'hotword rejected result' " + + "from hotword trusted process"); + if (mDebugHotwordLogging) { + Slog.i(TAG, "Egressed detected result: " + result); + } } } } @@ -702,43 +443,46 @@ class HotwordDetectorSession { @Override public void onDetected(HotwordDetectedResult triggerResult) throws RemoteException { - HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), - METRICS_EXTERNAL_SOURCE_DETECTED, - mVoiceInteractionServiceUid); - mScheduledExecutorService.schedule( - () -> { - bestEffortClose(serviceAudioSink, audioSource); - }, - EXTERNAL_HOTWORD_CLEANUP_MILLIS, - TimeUnit.MILLISECONDS); - - try { - enforcePermissionsForDataDelivery(); - } catch (SecurityException e) { + synchronized (mLock) { + mPerformingExternalSourceHotwordDetection = false; HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), - METRICS_EXTERNAL_SOURCE_DETECT_SECURITY_EXCEPTION, + METRICS_EXTERNAL_SOURCE_DETECTED, mVoiceInteractionServiceUid); - callback.onError(); - return; - } - HotwordDetectedResult newResult; - try { - newResult = - mHotwordAudioStreamCopier.startCopyingAudioStreams( - triggerResult); - } catch (IOException e) { - // TODO: Write event - callback.onError(); - return; - } - callback.onDetected(newResult, null /* audioFormat */, - null /* audioStream */); - Slog.i(TAG, "Egressed " - + HotwordDetectedResult.getUsageSize(newResult) - + " bits from hotword trusted process"); - if (mDebugHotwordLogging) { - Slog.i(TAG, - "Egressed detected result: " + newResult); + mScheduledExecutorService.schedule( + () -> { + bestEffortClose(serviceAudioSink, audioSource); + }, + EXTERNAL_HOTWORD_CLEANUP_MILLIS, + TimeUnit.MILLISECONDS); + + try { + enforcePermissionsForDataDelivery(); + } catch (SecurityException e) { + HotwordMetricsLogger.writeDetectorEvent( + getDetectorType(), + EXTERNAL_SOURCE_DETECT_SECURITY_EXCEPTION, + mVoiceInteractionServiceUid); + callback.onError(); + return; + } + HotwordDetectedResult newResult; + try { + newResult = mHotwordAudioStreamCopier + .startCopyingAudioStreams(triggerResult); + } catch (IOException e) { + // TODO: Write event + callback.onError(); + return; + } + callback.onDetected(newResult, null /* audioFormat */, + null /* audioStream */); + Slog.i(TAG, "Egressed " + + HotwordDetectedResult.getUsageSize(newResult) + + " bits from hotword trusted process"); + if (mDebugHotwordLogging) { + Slog.i(TAG, + "Egressed detected result: " + newResult); + } } } }); @@ -772,60 +516,16 @@ class HotwordDetectorSession { } void setDebugHotwordLoggingLocked(boolean logging) { - Slog.v(TAG, "setDebugHotwordLogging: " + logging); + Slog.v(TAG, "setDebugHotwordLoggingLocked: " + logging); mDebugHotwordLogging = logging; } + @SuppressWarnings("GuardedBy") void updateRemoteHotwordDetectionServiceLocked( @NonNull HotwordDetectionConnection.ServiceConnection remoteHotwordDetectionService) { mRemoteHotwordDetectionService = remoteHotwordDetectionService; } - @SuppressWarnings("GuardedBy") - void informRestartProcessLocked() { - // TODO(b/244598068): Check HotwordAudioStreamManager first - Slog.v(TAG, "informRestartProcess"); - if (mValidatingDspTrigger) { - // We're restarting the process while it's processing a DSP trigger, so report a - // rejection. This also allows the Interactor to startRecognition again - try { - mCallback.onRejected(new HotwordRejectedResult.Builder().build()); - HotwordMetricsLogger.writeKeyphraseTriggerEvent( - getDetectorType(), - HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED_FROM_RESTART); - } catch (RemoteException e) { - Slog.w(TAG, "Failed to call #rejected"); - HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), - HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_REJECTED_EXCEPTION, - mVoiceInteractionServiceUid); - } - mValidatingDspTrigger = false; - } - - mUpdateStateAfterStartFinished.set(false); - - try { - mCallback.onProcessRestarted(); - } catch (RemoteException e) { - Slog.w(TAG, "Failed to communicate #onProcessRestarted", e); - HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), - HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_PROCESS_RESTARTED_EXCEPTION, - mVoiceInteractionServiceUid); - } - - // Restart listening from microphone if the hotword process has been restarted. - if (mPerformingSoftwareHotwordDetection) { - Slog.i(TAG, "Process restarted: calling startRecognition() again"); - startListeningFromMicLocked(); - } - - if (mCurrentAudioSink != null) { - Slog.i(TAG, "Closing external audio stream to hotword detector: process restarted"); - bestEffortClose(mCurrentAudioSink); - mCurrentAudioSink = null; - } - } - void reportErrorLocked(int status) { try { mCallback.onError(status); @@ -837,6 +537,11 @@ class HotwordDetectorSession { } } + /** + * Called when the trusted process is restarted. + */ + abstract void informRestartProcessLocked(); + private static Pair createPipe() { ParcelFileDescriptor[] fileDescriptors; try { @@ -849,7 +554,7 @@ class HotwordDetectorSession { return Pair.create(fileDescriptors[0], fileDescriptors[1]); } - private void saveProximityValueToBundle(HotwordDetectedResult result) { + void saveProximityValueToBundle(HotwordDetectedResult result) { synchronized (mLock) { if (result != null && mProximityMeters != PROXIMITY_UNKNOWN) { result.setProximity(mProximityMeters); @@ -863,6 +568,15 @@ class HotwordDetectorSession { } } + @SuppressWarnings("GuardedBy") + void closeExternalAudioStreamLocked(String reason) { + if (mCurrentAudioSink != null) { + Slog.i(TAG, "Closing external audio stream to hotword detector: " + reason); + bestEffortClose(mCurrentAudioSink); + mCurrentAudioSink = null; + } + } + private static void bestEffortClose(Closeable... closeables) { for (Closeable closeable : closeables) { bestEffortClose(closeable); @@ -880,7 +594,7 @@ class HotwordDetectorSession { } // TODO: Share this code with SoundTriggerMiddlewarePermission. - private void enforcePermissionsForDataDelivery() { + void enforcePermissionsForDataDelivery() { Binder.withCleanCallingIdentity(() -> { synchronized (mLock) { enforcePermissionForPreflight(mContext, mVoiceInteractorIdentity, RECORD_AUDIO); @@ -914,7 +628,7 @@ class HotwordDetectorSession { } } - private static void enforceExtraKeyphraseIdNotLeaked(HotwordDetectedResult result, + static void enforceExtraKeyphraseIdNotLeaked(HotwordDetectedResult result, SoundTrigger.KeyphraseRecognitionEvent recognitionEvent) { // verify the phrase ID in HotwordDetectedResult is not exposing extra phrases // the DSP did not detect @@ -936,4 +650,17 @@ class HotwordDetectorSession { Slog.v(TAG, "Unexpected detector type"); return -1; } + + @SuppressWarnings("GuardedBy") + public void dumpLocked(String prefix, PrintWriter pw) { + pw.print(prefix); pw.print("mCallback="); pw.println(mCallback); + pw.print(prefix); pw.print("mUpdateStateAfterStartFinished="); + pw.println(mUpdateStateAfterStartFinished); + pw.print(prefix); pw.print("mInitialized="); pw.println(mInitialized); + pw.print(prefix); pw.print("mDestroyed="); pw.println(mDestroyed); + pw.print(prefix); pw.print("DetectorType="); + pw.println(HotwordDetector.detectorTypeToString(getDetectorType())); + pw.print(prefix); pw.print("mPerformingExternalSourceHotwordDetection="); + pw.println(mPerformingExternalSourceHotwordDetection); + } } diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/SoftwareTrustedHotwordDetectorSession.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/SoftwareTrustedHotwordDetectorSession.java index b89930074707c..6930689585687 100644 --- a/services/voiceinteraction/java/com/android/server/voiceinteraction/SoftwareTrustedHotwordDetectorSession.java +++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/SoftwareTrustedHotwordDetectorSession.java @@ -16,20 +16,50 @@ package com.android.server.voiceinteraction; +import static android.service.voice.HotwordDetectionService.AUDIO_SOURCE_MICROPHONE; + +import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_PROCESS_RESTARTED_EXCEPTION; +import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__START_SOFTWARE_DETECTION; +import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECTED; +import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED; + import android.annotation.NonNull; import android.content.Context; +import android.media.AudioFormat; import android.media.permission.Identity; +import android.os.PersistableBundle; +import android.os.RemoteException; +import android.os.SharedMemory; +import android.service.voice.HotwordDetectedResult; import android.service.voice.HotwordDetectionService; +import android.service.voice.HotwordDetector; +import android.service.voice.HotwordRejectedResult; +import android.service.voice.IDspHotwordDetectionCallback; +import android.service.voice.IHotwordDetectionService; +import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback; +import android.util.Slog; +import com.android.internal.annotations.GuardedBy; import com.android.internal.app.IHotwordRecognitionStatusCallback; +import java.io.IOException; +import java.io.PrintWriter; import java.util.concurrent.ScheduledExecutorService; /** * A class that provides software trusted hotword detector to communicate with the {@link * HotwordDetectionService}. + * + * This class can handle the hotword detection which detector is created by using + * {@link android.service.voice.VoiceInteractionService#createHotwordDetector(PersistableBundle, + * SharedMemory, HotwordDetector.Callback)}. */ final class SoftwareTrustedHotwordDetectorSession extends HotwordDetectorSession { + private static final String TAG = "SoftwareTrustedHotwordDetectorSession"; + + private IMicrophoneHotwordDetectionVoiceInteractionCallback mSoftwareCallback; + @GuardedBy("mLock") + private boolean mPerformingSoftwareHotwordDetection; SoftwareTrustedHotwordDetectorSession( @NonNull HotwordDetectionConnection.ServiceConnection remoteHotwordDetectionService, @@ -40,4 +70,145 @@ final class SoftwareTrustedHotwordDetectorSession extends HotwordDetectorSession super(remoteHotwordDetectionService, lock, context, callback, voiceInteractionServiceUid, voiceInteractorIdentity, scheduledExecutorService, logging); } + + @SuppressWarnings("GuardedBy") + void startListeningFromMicLocked( + AudioFormat audioFormat, + IMicrophoneHotwordDetectionVoiceInteractionCallback callback) { + if (DEBUG) { + Slog.d(TAG, "startListeningFromMicLocked"); + } + mSoftwareCallback = callback; + + if (mPerformingSoftwareHotwordDetection) { + Slog.i(TAG, "Hotword validation is already in progress, ignoring."); + return; + } + mPerformingSoftwareHotwordDetection = true; + + startListeningFromMicLocked(); + } + + @SuppressWarnings("GuardedBy") + private void startListeningFromMicLocked() { + // TODO: consider making this a non-anonymous class. + IDspHotwordDetectionCallback internalCallback = new IDspHotwordDetectionCallback.Stub() { + @Override + public void onDetected(HotwordDetectedResult result) throws RemoteException { + if (DEBUG) { + Slog.d(TAG, "onDetected"); + } + synchronized (mLock) { + HotwordMetricsLogger.writeKeyphraseTriggerEvent( + HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE, + HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECTED); + if (!mPerformingSoftwareHotwordDetection) { + Slog.i(TAG, "Hotword detection has already completed"); + HotwordMetricsLogger.writeKeyphraseTriggerEvent( + HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE, + METRICS_KEYPHRASE_TRIGGERED_DETECT_UNEXPECTED_CALLBACK); + return; + } + mPerformingSoftwareHotwordDetection = false; + try { + enforcePermissionsForDataDelivery(); + } catch (SecurityException e) { + HotwordMetricsLogger.writeKeyphraseTriggerEvent( + HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE, + METRICS_KEYPHRASE_TRIGGERED_DETECT_SECURITY_EXCEPTION); + mSoftwareCallback.onError(); + return; + } + saveProximityValueToBundle(result); + HotwordDetectedResult newResult; + try { + newResult = mHotwordAudioStreamCopier.startCopyingAudioStreams(result); + } catch (IOException e) { + // TODO: Write event + mSoftwareCallback.onError(); + return; + } + mSoftwareCallback.onDetected(newResult, null, null); + Slog.i(TAG, "Egressed " + HotwordDetectedResult.getUsageSize(newResult) + + " bits from hotword trusted process"); + if (mDebugHotwordLogging) { + Slog.i(TAG, "Egressed detected result: " + newResult); + } + } + } + + @Override + public void onRejected(HotwordRejectedResult result) throws RemoteException { + if (DEBUG) { + Slog.wtf(TAG, "onRejected"); + } + HotwordMetricsLogger.writeKeyphraseTriggerEvent( + HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE, + HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED); + // onRejected isn't allowed here, and we are not expecting it. + } + }; + + mRemoteHotwordDetectionService.run( + service -> service.detectFromMicrophoneSource( + null, + AUDIO_SOURCE_MICROPHONE, + null, + null, + internalCallback)); + HotwordMetricsLogger.writeDetectorEvent( + HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE, + HOTWORD_DETECTOR_EVENTS__EVENT__START_SOFTWARE_DETECTION, + mVoiceInteractionServiceUid); + } + + @SuppressWarnings("GuardedBy") + void stopListeningLocked() { + if (DEBUG) { + Slog.d(TAG, "stopListeningLocked"); + } + if (!mPerformingSoftwareHotwordDetection) { + Slog.i(TAG, "Hotword detection is not running"); + return; + } + mPerformingSoftwareHotwordDetection = false; + + mRemoteHotwordDetectionService.run(IHotwordDetectionService::stopDetection); + + closeExternalAudioStreamLocked("stopping requested"); + } + + @Override + @SuppressWarnings("GuardedBy") + void informRestartProcessLocked() { + // TODO(b/244598068): Check HotwordAudioStreamManager first + Slog.v(TAG, "informRestartProcessLocked"); + mUpdateStateAfterStartFinished.set(false); + + try { + mCallback.onProcessRestarted(); + } catch (RemoteException e) { + Slog.w(TAG, "Failed to communicate #onProcessRestarted", e); + HotwordMetricsLogger.writeDetectorEvent( + HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE, + HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_PROCESS_RESTARTED_EXCEPTION, + mVoiceInteractionServiceUid); + } + + // Restart listening from microphone if the hotword process has been restarted. + if (mPerformingSoftwareHotwordDetection) { + Slog.i(TAG, "Process restarted: calling startRecognition() again"); + startListeningFromMicLocked(); + } + + mPerformingExternalSourceHotwordDetection = false; + closeExternalAudioStreamLocked("process restarted"); + } + + @SuppressWarnings("GuardedBy") + public void dumpLocked(String prefix, PrintWriter pw) { + super.dumpLocked(prefix, pw); + pw.print(prefix); pw.print("mPerformingSoftwareHotwordDetection="); + pw.println(mPerformingSoftwareHotwordDetection); + } }