From 1566016119cc3fa5365e380ca3cbc9d127786d86 Mon Sep 17 00:00:00 2001 From: lpeter Date: Fri, 18 Nov 2022 01:28:40 +0000 Subject: [PATCH] Use Dsp session and Software session in HotwordDetectionConnection We use the HotwordDetectorSession to handle the Dsp and Software trusted hotword detectors, it would be better to separate them. We create the DspTrustedHotwordDetectorSession to handle Dsp trusted hotword detector, SoftwareTrustedHotwordDetectorSession to handle software trusted hotword detector. Currently they are just the simple classes, we will have another change to move the methods of HotwordDetectorSession into DspTrustedHotwordDetectorSession and SoftwareTrustedHotwordDetectorSession. Test: atest CtsVoiceInteractionTestCases Bug: 241041976 Change-Id: I0d569eae319acd8260808aac545babd323f1657e --- .../DspTrustedHotwordDetectorSession.java | 43 ++++++++++ .../HotwordDetectionConnection.java | 26 +++++- .../HotwordDetectorSession.java | 82 ++++++++++--------- ...SoftwareTrustedHotwordDetectorSession.java | 43 ++++++++++ 4 files changed, 154 insertions(+), 40 deletions(-) create mode 100644 services/voiceinteraction/java/com/android/server/voiceinteraction/DspTrustedHotwordDetectorSession.java create mode 100644 services/voiceinteraction/java/com/android/server/voiceinteraction/SoftwareTrustedHotwordDetectorSession.java diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/DspTrustedHotwordDetectorSession.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/DspTrustedHotwordDetectorSession.java new file mode 100644 index 0000000000000..c092d9badcb24 --- /dev/null +++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/DspTrustedHotwordDetectorSession.java @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.voiceinteraction; + +import android.annotation.NonNull; +import android.content.Context; +import android.media.permission.Identity; +import android.service.voice.HotwordDetectionService; + +import com.android.internal.app.IHotwordRecognitionStatusCallback; + +import java.util.concurrent.ScheduledExecutorService; + +/** + * A class that provides Dsp trusted hotword detector to communicate with the {@link + * HotwordDetectionService}. + */ +final class DspTrustedHotwordDetectorSession extends HotwordDetectorSession { + + DspTrustedHotwordDetectorSession( + @NonNull HotwordDetectionConnection.ServiceConnection remoteHotwordDetectionService, + @NonNull Object lock, @NonNull Context context, + @NonNull IHotwordRecognitionStatusCallback callback, int voiceInteractionServiceUid, + Identity voiceInteractorIdentity, + @NonNull ScheduledExecutorService scheduledExecutorService, boolean logging) { + super(remoteHotwordDetectionService, lock, context, callback, voiceInteractionServiceUid, + voiceInteractorIdentity, scheduledExecutorService, logging); + } +} diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java index 5141dd34a892d..7e105c3b18c52 100644 --- a/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java +++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java @@ -142,9 +142,17 @@ final class HotwordDetectionConnection { mRemoteHotwordDetectionService = mServiceConnectionFactory.createLocked(); mLastRestartInstant = Instant.now(); - mHotwordDetectorSession = new HotwordDetectorSession(mRemoteHotwordDetectionService, - mLock, mContext, callback, mDetectorType, mVoiceInteractionServiceUid, - mVoiceInteractorIdentity, mScheduledExecutorService, mDebugHotwordLogging); + if (detectorType == HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP) { + mHotwordDetectorSession = new DspTrustedHotwordDetectorSession( + mRemoteHotwordDetectionService, mLock, mContext, callback, + mVoiceInteractionServiceUid, mVoiceInteractorIdentity, + mScheduledExecutorService, mDebugHotwordLogging); + } else { + mHotwordDetectorSession = new SoftwareTrustedHotwordDetectorSession( + mRemoteHotwordDetectionService, mLock, mContext, callback, + mVoiceInteractionServiceUid, mVoiceInteractorIdentity, + mScheduledExecutorService, mDebugHotwordLogging); + } mHotwordDetectorSession.initialize(options, sharedMemory); if (mReStartPeriodSeconds <= 0) { @@ -232,6 +240,10 @@ final class HotwordDetectionConnection { Slog.d(TAG, "startListeningFromMic"); } synchronized (mLock) { + if (!(mHotwordDetectorSession instanceof SoftwareTrustedHotwordDetectorSession)) { + Slog.d(TAG, "It is not a software detector"); + return; + } mHotwordDetectorSession.startListeningFromMicLocked(audioFormat, callback); } } @@ -258,6 +270,10 @@ final class HotwordDetectionConnection { Slog.d(TAG, "stopListening"); } synchronized (mLock) { + if (!(mHotwordDetectorSession instanceof SoftwareTrustedHotwordDetectorSession)) { + Slog.d(TAG, "It is not a software detector"); + return; + } mHotwordDetectorSession.stopListeningLocked(); } } @@ -277,6 +293,10 @@ final class HotwordDetectionConnection { Slog.d(TAG, "detectFromDspSource"); } synchronized (mLock) { + if (!(mHotwordDetectorSession instanceof DspTrustedHotwordDetectorSession)) { + Slog.d(TAG, "It is not a Dsp detector"); + return; + } mHotwordDetectorSession.detectFromDspSourceLocked(recognitionEvent, externalCallback); } } diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectorSession.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectorSession.java index 87d85ecec7966..9cd4f68222a91 100644 --- a/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectorSession.java +++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectorSession.java @@ -105,7 +105,7 @@ import java.util.concurrent.atomic.AtomicBoolean; * A class that provides trusted hotword detector to communicate with the {@link * HotwordDetectionService}. */ -final class HotwordDetectorSession { +class HotwordDetectorSession { private static final String TAG = "HotwordDetectorSession"; static final boolean DEBUG = false; @@ -163,7 +163,6 @@ final class HotwordDetectorSession { private final HotwordAudioStreamCopier mHotwordAudioStreamCopier; private final AtomicBoolean mUpdateStateAfterStartFinished = new AtomicBoolean(false); private final IHotwordRecognitionStatusCallback mCallback; - private final int mDetectorType; final Object mLock; final int mVoiceInteractionServiceUid; @@ -200,18 +199,17 @@ final class HotwordDetectorSession { HotwordDetectorSession( @NonNull HotwordDetectionConnection.ServiceConnection remoteHotwordDetectionService, @NonNull Object lock, @NonNull Context context, - @NonNull IHotwordRecognitionStatusCallback callback, int detectorType, - int voiceInteractionServiceUid, Identity voiceInteractorIdentity, + @NonNull IHotwordRecognitionStatusCallback callback, int voiceInteractionServiceUid, + Identity voiceInteractorIdentity, @NonNull ScheduledExecutorService scheduledExecutorService, boolean logging) { mRemoteHotwordDetectionService = remoteHotwordDetectionService; mLock = lock; mContext = context; mCallback = callback; - mDetectorType = detectorType; mVoiceInteractionServiceUid = voiceInteractionServiceUid; mVoiceInteractorIdentity = voiceInteractorIdentity; mAppOpsManager = mContext.getSystemService(AppOpsManager.class); - mHotwordAudioStreamCopier = new HotwordAudioStreamCopier(mAppOpsManager, detectorType, + mHotwordAudioStreamCopier = new HotwordAudioStreamCopier(mAppOpsManager, getDetectorType(), mVoiceInteractorIdentity.uid, mVoiceInteractorIdentity.packageName, mVoiceInteractorIdentity.attributionTag); mScheduledExecutorService = scheduledExecutorService; @@ -241,7 +239,7 @@ final class HotwordDetectorSession { future.complete(null); if (mUpdateStateAfterStartFinished.getAndSet(true)) { Slog.w(TAG, "call callback after timeout"); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_UPDATE_STATE_AFTER_TIMEOUT, mVoiceInteractionServiceUid); return; @@ -251,11 +249,11 @@ final class HotwordDetectorSession { int initResultMetricsResult = statusResultPair.second; try { mCallback.onStatusReported(status); - HotwordMetricsLogger.writeServiceInitResultEvent(mDetectorType, + HotwordMetricsLogger.writeServiceInitResultEvent(getDetectorType(), initResultMetricsResult); } catch (RemoteException e) { Slog.w(TAG, "Failed to report initialization status: " + e); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), METRICS_CALLBACK_ON_STATUS_REPORTED_EXCEPTION, mVoiceInteractionServiceUid); } @@ -263,13 +261,13 @@ final class HotwordDetectorSession { }; try { service.updateState(options, sharedMemory, statusCallback); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_UPDATE_STATE, mVoiceInteractionServiceUid); } catch (RemoteException e) { // TODO: (b/181842909) Report an error to voice interactor Slog.w(TAG, "Failed to updateState for HotwordDetectionService", e); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), HOTWORD_DETECTOR_EVENTS__EVENT__CALL_UPDATE_STATE_EXCEPTION, mVoiceInteractionServiceUid); } @@ -282,11 +280,11 @@ final class HotwordDetectorSession { } try { mCallback.onStatusReported(INITIALIZATION_STATUS_UNKNOWN); - HotwordMetricsLogger.writeServiceInitResultEvent(mDetectorType, + HotwordMetricsLogger.writeServiceInitResultEvent(getDetectorType(), METRICS_INIT_UNKNOWN_TIMEOUT); } catch (RemoteException e) { Slog.w(TAG, "Failed to report initialization status UNKNOWN", e); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), METRICS_CALLBACK_ON_STATUS_REPORTED_EXCEPTION, mVoiceInteractionServiceUid); } @@ -319,7 +317,7 @@ final class HotwordDetectorSession { void updateStateLocked(PersistableBundle options, SharedMemory sharedMemory, Instant lastRestartInstant) { - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), HOTWORD_DETECTOR_EVENTS__EVENT__APP_REQUEST_UPDATE_STATE, mVoiceInteractionServiceUid); // Prevent doing the init late, so restart is handled equally to a clean process start. @@ -368,12 +366,12 @@ final class HotwordDetectorSession { } synchronized (mLock) { HotwordMetricsLogger.writeKeyphraseTriggerEvent( - mDetectorType, + getDetectorType(), HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECTED); if (!mPerformingSoftwareHotwordDetection) { Slog.i(TAG, "Hotword detection has already completed"); HotwordMetricsLogger.writeKeyphraseTriggerEvent( - mDetectorType, + getDetectorType(), METRICS_KEYPHRASE_TRIGGERED_DETECT_UNEXPECTED_CALLBACK); return; } @@ -382,7 +380,7 @@ final class HotwordDetectorSession { enforcePermissionsForDataDelivery(); } catch (SecurityException e) { HotwordMetricsLogger.writeKeyphraseTriggerEvent( - mDetectorType, + getDetectorType(), METRICS_KEYPHRASE_TRIGGERED_DETECT_SECURITY_EXCEPTION); mSoftwareCallback.onError(); return; @@ -411,7 +409,7 @@ final class HotwordDetectorSession { Slog.wtf(TAG, "onRejected"); } HotwordMetricsLogger.writeKeyphraseTriggerEvent( - mDetectorType, + getDetectorType(), HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED); // onRejected isn't allowed here, and we are not expecting it. } @@ -424,7 +422,7 @@ final class HotwordDetectorSession { null, null, internalCallback)); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), HOTWORD_DETECTOR_EVENTS__EVENT__START_SOFTWARE_DETECTION, mVoiceInteractionServiceUid); } @@ -491,12 +489,12 @@ final class HotwordDetectorSession { return; } HotwordMetricsLogger.writeKeyphraseTriggerEvent( - mDetectorType, + getDetectorType(), HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECTED); if (!mValidatingDspTrigger) { Slog.i(TAG, "Ignoring #onDetected due to a process restart"); HotwordMetricsLogger.writeKeyphraseTriggerEvent( - mDetectorType, + getDetectorType(), METRICS_KEYPHRASE_TRIGGERED_DETECT_UNEXPECTED_CALLBACK); return; } @@ -507,7 +505,7 @@ final class HotwordDetectorSession { } catch (SecurityException e) { Slog.i(TAG, "Ignoring #onDetected due to a SecurityException", e); HotwordMetricsLogger.writeKeyphraseTriggerEvent( - mDetectorType, + getDetectorType(), METRICS_KEYPHRASE_TRIGGERED_DETECT_SECURITY_EXCEPTION); externalCallback.onError(CALLBACK_ONDETECTED_GOT_SECURITY_EXCEPTION); return; @@ -543,12 +541,12 @@ final class HotwordDetectorSession { return; } HotwordMetricsLogger.writeKeyphraseTriggerEvent( - mDetectorType, + getDetectorType(), HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED); if (!mValidatingDspTrigger) { Slog.i(TAG, "Ignoring #onRejected due to a process restart"); HotwordMetricsLogger.writeKeyphraseTriggerEvent( - mDetectorType, + getDetectorType(), METRICS_KEYPHRASE_TRIGGERED_REJECT_UNEXPECTED_CALLBACK); return; } @@ -573,13 +571,13 @@ final class HotwordDetectorSession { timeoutDetected.set(true); Slog.w(TAG, "Timed out on #detectFromDspSource"); HotwordMetricsLogger.writeKeyphraseTriggerEvent( - mDetectorType, + getDetectorType(), HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_TIMEOUT); try { externalCallback.onError(CALLBACK_DETECT_TIMEOUT); } catch (RemoteException e) { Slog.w(TAG, "Failed to report onError status: ", e); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION, mVoiceInteractionServiceUid); } @@ -604,8 +602,8 @@ final class HotwordDetectorSession { pw.print(prefix); pw.print("mValidatingDspTrigger="); pw.println(mValidatingDspTrigger); pw.print(prefix); pw.print("mPerformingSoftwareHotwordDetection="); pw.println(mPerformingSoftwareHotwordDetection); - pw.print(prefix); pw.print("mDetectorType="); - pw.println(HotwordDetector.detectorTypeToString(mDetectorType)); + pw.print(prefix); pw.print("DetectorType="); + pw.println(HotwordDetector.detectorTypeToString(getDetectorType())); } @SuppressWarnings("GuardedBy") @@ -655,7 +653,7 @@ final class HotwordDetectorSession { callback.onError(); } catch (RemoteException ex) { Slog.w(TAG, "Failed to report onError status: " + ex); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION, mVoiceInteractionServiceUid); } @@ -680,7 +678,7 @@ final class HotwordDetectorSession { @Override public void onRejected(HotwordRejectedResult result) throws RemoteException { - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), METRICS_EXTERNAL_SOURCE_REJECTED, mVoiceInteractionServiceUid); mScheduledExecutorService.schedule( @@ -704,7 +702,7 @@ final class HotwordDetectorSession { @Override public void onDetected(HotwordDetectedResult triggerResult) throws RemoteException { - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), METRICS_EXTERNAL_SOURCE_DETECTED, mVoiceInteractionServiceUid); mScheduledExecutorService.schedule( @@ -717,7 +715,7 @@ final class HotwordDetectorSession { try { enforcePermissionsForDataDelivery(); } catch (SecurityException e) { - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), METRICS_EXTERNAL_SOURCE_DETECT_SECURITY_EXCEPTION, mVoiceInteractionServiceUid); callback.onError(); @@ -748,7 +746,7 @@ final class HotwordDetectorSession { // A copy of this has been created and passed to the hotword validator bestEffortClose(serviceAudioSource); }); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), HOTWORD_DETECTOR_EVENTS__EVENT__START_EXTERNAL_SOURCE_DETECTION, mVoiceInteractionServiceUid); } @@ -793,11 +791,11 @@ final class HotwordDetectorSession { try { mCallback.onRejected(new HotwordRejectedResult.Builder().build()); HotwordMetricsLogger.writeKeyphraseTriggerEvent( - mDetectorType, + getDetectorType(), HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED_FROM_RESTART); } catch (RemoteException e) { Slog.w(TAG, "Failed to call #rejected"); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_REJECTED_EXCEPTION, mVoiceInteractionServiceUid); } @@ -810,7 +808,7 @@ final class HotwordDetectorSession { mCallback.onProcessRestarted(); } catch (RemoteException e) { Slog.w(TAG, "Failed to communicate #onProcessRestarted", e); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_PROCESS_RESTARTED_EXCEPTION, mVoiceInteractionServiceUid); } @@ -833,7 +831,7 @@ final class HotwordDetectorSession { mCallback.onError(status); } catch (RemoteException e) { Slog.w(TAG, "Failed to report onError status: " + e); - HotwordMetricsLogger.writeDetectorEvent(mDetectorType, + HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION, mVoiceInteractionServiceUid); } @@ -928,4 +926,14 @@ final class HotwordDetectorSession { throw new SecurityException("Ignoring #onDetected due to trusted service " + "sharing a keyphrase ID which the DSP did not detect"); } + + private int getDetectorType() { + if (this instanceof DspTrustedHotwordDetectorSession) { + return HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP; + } else if (this instanceof SoftwareTrustedHotwordDetectorSession) { + return HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE; + } + Slog.v(TAG, "Unexpected detector type"); + return -1; + } } diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/SoftwareTrustedHotwordDetectorSession.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/SoftwareTrustedHotwordDetectorSession.java new file mode 100644 index 0000000000000..b89930074707c --- /dev/null +++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/SoftwareTrustedHotwordDetectorSession.java @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.voiceinteraction; + +import android.annotation.NonNull; +import android.content.Context; +import android.media.permission.Identity; +import android.service.voice.HotwordDetectionService; + +import com.android.internal.app.IHotwordRecognitionStatusCallback; + +import java.util.concurrent.ScheduledExecutorService; + +/** + * A class that provides software trusted hotword detector to communicate with the {@link + * HotwordDetectionService}. + */ +final class SoftwareTrustedHotwordDetectorSession extends HotwordDetectorSession { + + SoftwareTrustedHotwordDetectorSession( + @NonNull HotwordDetectionConnection.ServiceConnection remoteHotwordDetectionService, + @NonNull Object lock, @NonNull Context context, + @NonNull IHotwordRecognitionStatusCallback callback, int voiceInteractionServiceUid, + Identity voiceInteractorIdentity, + @NonNull ScheduledExecutorService scheduledExecutorService, boolean logging) { + super(remoteHotwordDetectionService, lock, context, callback, voiceInteractionServiceUid, + voiceInteractorIdentity, scheduledExecutorService, logging); + } +}