Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move the methods into Dsp session and Software session

Browse Source 
Move the methods of HotwordDetectorSession into
DspTrustedHotwordDetectorSession and
SoftwareTrustedHotwordDetectorSession

Test: atest CtsVoiceInteractionTestCases
Bug: 241041976
Change-Id: I53139507697ba86f04158896ef95095993f23f2c
This commit is contained in:
lpeter
2022-11-18 09:30:08 +00:00
parent 1566016119
commit 96e34f053b
4 changed files with 508 additions and 397 deletions
Download Patch File Download Diff File Expand all files Collapse all files

211
services/voiceinteraction/java/com/android/server/voiceinteraction/DspTrustedHotwordDetectorSession.java
View File

@@ -16,20 +16,61 @@
package com.android.server.voiceinteraction;
import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION;
import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_PROCESS_RESTARTED_EXCEPTION;
import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_REJECTED_EXCEPTION;
import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECTED;
import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_TIMEOUT;
import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED;
import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED_FROM_RESTART;
import android.annotation.NonNull;
import android.content.Context;
import android.hardware.soundtrigger.SoundTrigger;
import android.media.permission.Identity;
import android.os.PersistableBundle;
import android.os.RemoteException;
import android.os.SharedMemory;
import android.service.voice.AlwaysOnHotwordDetector;
import android.service.voice.HotwordDetectedResult;
import android.service.voice.HotwordDetectionService;
import android.service.voice.HotwordDetector;
import android.service.voice.HotwordRejectedResult;
import android.service.voice.IDspHotwordDetectionCallback;
import android.util.Slog;
import com.android.internal.annotations.GuardedBy;
import com.android.internal.app.IHotwordRecognitionStatusCallback;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.Locale;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
/**
* A class that provides Dsp trusted hotword detector to communicate with the {@link
* HotwordDetectionService}.
*
* This class can handle the hotword detection which detector is created by using
* {@link android.service.voice.VoiceInteractionService#createAlwaysOnHotwordDetector(String,
* Locale, PersistableBundle, SharedMemory, AlwaysOnHotwordDetector.Callback)}.
*/
final class DspTrustedHotwordDetectorSession extends HotwordDetectorSession {
private static final String TAG = "DspTrustedHotwordDetectorSession";
// The validation timeout value is 3 seconds for onDetect of DSP trigger event.
private static final long VALIDATION_TIMEOUT_MILLIS = 3000;
// Write the onDetect timeout metric when it takes more time than MAX_VALIDATION_TIMEOUT_MILLIS.
private static final long MAX_VALIDATION_TIMEOUT_MILLIS = 4000;
@GuardedBy("mLock")
private ScheduledFuture<?> mCancellationKeyPhraseDetectionFuture;
@GuardedBy("mLock")
private boolean mValidatingDspTrigger = false;
DspTrustedHotwordDetectorSession(
@NonNull HotwordDetectionConnection.ServiceConnection remoteHotwordDetectionService,
@@ -40,4 +81,174 @@ final class DspTrustedHotwordDetectorSession extends HotwordDetectorSession {
super(remoteHotwordDetectionService, lock, context, callback, voiceInteractionServiceUid,
voiceInteractorIdentity, scheduledExecutorService, logging);
}
@SuppressWarnings("GuardedBy")
void detectFromDspSourceLocked(SoundTrigger.KeyphraseRecognitionEvent recognitionEvent,
IHotwordRecognitionStatusCallback externalCallback) {
if (DEBUG) {
Slog.d(TAG, "detectFromDspSourceLocked");
}
AtomicBoolean timeoutDetected = new AtomicBoolean(false);
// TODO: consider making this a non-anonymous class.
IDspHotwordDetectionCallback internalCallback = new IDspHotwordDetectionCallback.Stub() {
@Override
public void onDetected(HotwordDetectedResult result) throws RemoteException {
if (DEBUG) {
Slog.d(TAG, "onDetected");
}
synchronized (mLock) {
if (mCancellationKeyPhraseDetectionFuture != null) {
mCancellationKeyPhraseDetectionFuture.cancel(true);
}
if (timeoutDetected.get()) {
return;
}
HotwordMetricsLogger.writeKeyphraseTriggerEvent(
HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP,
HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECTED);
if (!mValidatingDspTrigger) {
Slog.i(TAG, "Ignoring #onDetected due to a process restart");
HotwordMetricsLogger.writeKeyphraseTriggerEvent(
HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP,
METRICS_KEYPHRASE_TRIGGERED_DETECT_UNEXPECTED_CALLBACK);
return;
}
mValidatingDspTrigger = false;
try {
enforcePermissionsForDataDelivery();
enforceExtraKeyphraseIdNotLeaked(result, recognitionEvent);
} catch (SecurityException e) {
Slog.i(TAG, "Ignoring #onDetected due to a SecurityException", e);
HotwordMetricsLogger.writeKeyphraseTriggerEvent(
HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP,
METRICS_KEYPHRASE_TRIGGERED_DETECT_SECURITY_EXCEPTION);
externalCallback.onError(CALLBACK_ONDETECTED_GOT_SECURITY_EXCEPTION);
return;
}
saveProximityValueToBundle(result);
HotwordDetectedResult newResult;
try {
newResult = mHotwordAudioStreamCopier.startCopyingAudioStreams(result);
} catch (IOException e) {
externalCallback.onError(CALLBACK_ONDETECTED_STREAM_COPY_ERROR);
return;
}
externalCallback.onKeyphraseDetected(recognitionEvent, newResult);
Slog.i(TAG, "Egressed " + HotwordDetectedResult.getUsageSize(newResult)
+ " bits from hotword trusted process");
if (mDebugHotwordLogging) {
Slog.i(TAG, "Egressed detected result: " + newResult);
}
}
}
@Override
public void onRejected(HotwordRejectedResult result) throws RemoteException {
if (DEBUG) {
Slog.d(TAG, "onRejected");
}
synchronized (mLock) {
if (mCancellationKeyPhraseDetectionFuture != null) {
mCancellationKeyPhraseDetectionFuture.cancel(true);
}
if (timeoutDetected.get()) {
return;
}
HotwordMetricsLogger.writeKeyphraseTriggerEvent(
HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP,
HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED);
if (!mValidatingDspTrigger) {
Slog.i(TAG, "Ignoring #onRejected due to a process restart");
HotwordMetricsLogger.writeKeyphraseTriggerEvent(
HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP,
METRICS_KEYPHRASE_TRIGGERED_REJECT_UNEXPECTED_CALLBACK);
return;
}
mValidatingDspTrigger = false;
externalCallback.onRejected(result);
if (mDebugHotwordLogging && result != null) {
Slog.i(TAG, "Egressed rejected result: " + result);
}
}
}
};
mValidatingDspTrigger = true;
mRemoteHotwordDetectionService.run(service -> {
// We use the VALIDATION_TIMEOUT_MILLIS to inform that the client needs to invoke
// the callback before timeout value. In order to reduce the latency impact between
// server side and client side, we need to use another timeout value
// MAX_VALIDATION_TIMEOUT_MILLIS to monitor it.
mCancellationKeyPhraseDetectionFuture = mScheduledExecutorService.schedule(
() -> {
// TODO: avoid allocate every time
timeoutDetected.set(true);
Slog.w(TAG, "Timed out on #detectFromDspSource");
HotwordMetricsLogger.writeKeyphraseTriggerEvent(
HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP,
HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_TIMEOUT);
try {
externalCallback.onError(CALLBACK_DETECT_TIMEOUT);
} catch (RemoteException e) {
Slog.w(TAG, "Failed to report onError status: ", e);
HotwordMetricsLogger.writeDetectorEvent(
HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP,
HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION,
mVoiceInteractionServiceUid);
}
},
MAX_VALIDATION_TIMEOUT_MILLIS,
TimeUnit.MILLISECONDS);
service.detectFromDspSource(
recognitionEvent,
recognitionEvent.getCaptureFormat(),
VALIDATION_TIMEOUT_MILLIS,
internalCallback);
});
}
@Override
@SuppressWarnings("GuardedBy")
void informRestartProcessLocked() {
// TODO(b/244598068): Check HotwordAudioStreamManager first
Slog.v(TAG, "informRestartProcessLocked");
if (mValidatingDspTrigger) {
// We're restarting the process while it's processing a DSP trigger, so report a
// rejection. This also allows the Interactor to startRecognition again
try {
mCallback.onRejected(new HotwordRejectedResult.Builder().build());
HotwordMetricsLogger.writeKeyphraseTriggerEvent(
HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP,
HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED_FROM_RESTART);
} catch (RemoteException e) {
Slog.w(TAG, "Failed to call #rejected");
HotwordMetricsLogger.writeDetectorEvent(
HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP,
HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_REJECTED_EXCEPTION,
mVoiceInteractionServiceUid);
}
mValidatingDspTrigger = false;
}
mUpdateStateAfterStartFinished.set(false);
try {
mCallback.onProcessRestarted();
} catch (RemoteException e) {
Slog.w(TAG, "Failed to communicate #onProcessRestarted", e);
HotwordMetricsLogger.writeDetectorEvent(
HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP,
HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_PROCESS_RESTARTED_EXCEPTION,
mVoiceInteractionServiceUid);
}
mPerformingExternalSourceHotwordDetection = false;
closeExternalAudioStreamLocked("process restarted");
}
@SuppressWarnings("GuardedBy")
public void dumpLocked(String prefix, PrintWriter pw) {
super.dumpLocked(prefix, pw);
pw.print(prefix); pw.print("mValidatingDspTrigger="); pw.println(mValidatingDspTrigger);
}
}

8
services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java
View File

@@ -244,7 +244,8 @@ final class HotwordDetectionConnection {
Slog.d(TAG, "It is not a software detector");
return;
}
mHotwordDetectorSession.startListeningFromMicLocked(audioFormat, callback);
((SoftwareTrustedHotwordDetectorSession) mHotwordDetectorSession)
.startListeningFromMicLocked(audioFormat, callback);
}
}
@@ -274,7 +275,7 @@ final class HotwordDetectionConnection {
Slog.d(TAG, "It is not a software detector");
return;
}
mHotwordDetectorSession.stopListeningLocked();
((SoftwareTrustedHotwordDetectorSession) mHotwordDetectorSession).stopListeningLocked();
}
}
@@ -297,7 +298,8 @@ final class HotwordDetectionConnection {
Slog.d(TAG, "It is not a Dsp detector");
return;
}
mHotwordDetectorSession.detectFromDspSourceLocked(recognitionEvent, externalCallback);
((DspTrustedHotwordDetectorSession) mHotwordDetectorSession).detectFromDspSourceLocked(
recognitionEvent, externalCallback);
}
}

515
services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectorSession.java
View File

@@ -20,7 +20,6 @@ import static android.Manifest.permission.CAPTURE_AUDIO_HOTWORD;
import static android.Manifest.permission.RECORD_AUDIO;
import static android.service.attention.AttentionService.PROXIMITY_UNKNOWN;
import static android.service.voice.HotwordDetectionService.AUDIO_SOURCE_EXTERNAL;
import static android.service.voice.HotwordDetectionService.AUDIO_SOURCE_MICROPHONE;
import static android.service.voice.HotwordDetectionService.ENABLE_PROXIMITY_RESULT;
import static android.service.voice.HotwordDetectionService.INITIALIZATION_STATUS_SUCCESS;
import static android.service.voice.HotwordDetectionService.INITIALIZATION_STATUS_UNKNOWN;
@@ -33,8 +32,6 @@ import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTION_SERV
import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_UNKNOWN_TIMEOUT;
import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__APP_REQUEST_UPDATE_STATE;
import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION;
import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_PROCESS_RESTARTED_EXCEPTION;
import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_REJECTED_EXCEPTION;
import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_STATUS_REPORTED_EXCEPTION;
import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_UPDATE_STATE_AFTER_TIMEOUT;
import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALL_UPDATE_STATE_EXCEPTION;
@@ -43,13 +40,8 @@ import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENT
import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__EXTERNAL_SOURCE_REJECTED;
import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_UPDATE_STATE;
import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__START_EXTERNAL_SOURCE_DETECTION;
import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__START_SOFTWARE_DETECTION;
import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECTED;
import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_SECURITY_EXCEPTION;
import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_TIMEOUT;
import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_UNEXPECTED_CALLBACK;
import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED;
import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED_FROM_RESTART;
import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECT_UNEXPECTED_CALLBACK;
import static com.android.server.voiceinteraction.SoundTriggerSessionPermissionsDecorator.enforcePermissionForPreflight;
@@ -75,7 +67,6 @@ import android.service.voice.HotwordDetectionService;
import android.service.voice.HotwordDetector;
import android.service.voice.HotwordRejectedResult;
import android.service.voice.IDspHotwordDetectionCallback;
import android.service.voice.IHotwordDetectionService;
import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback;
import android.text.TextUtils;
import android.util.Pair;
@@ -96,7 +87,6 @@ import java.time.Instant;
import java.util.concurrent.Executor;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -104,8 +94,15 @@ import java.util.concurrent.atomic.AtomicBoolean;
/**
* A class that provides trusted hotword detector to communicate with the {@link
* HotwordDetectionService}.
*
* This class provides the methods to do initialization with the {@link HotwordDetectionService}
* and handle external source detection. It also provides the methods to check if we can egress
* the data from the {@link HotwordDetectionService}.
*
* The subclass should override the {@link #informRestartProcessLocked()} to handle the trusted
* process restart.
*/
class HotwordDetectorSession {
abstract class HotwordDetectorSession {
private static final String TAG = "HotwordDetectorSession";
static final boolean DEBUG = false;
@@ -119,10 +116,6 @@ class HotwordDetectorSession {
static final int CALLBACK_ONDETECTED_STREAM_COPY_ERROR = -4;
// TODO: These constants need to be refined.
// The validation timeout value is 3 seconds for onDetect of DSP trigger event.
private static final long VALIDATION_TIMEOUT_MILLIS = 3000;
// Write the onDetect timeout metric when it takes more time than MAX_VALIDATION_TIMEOUT_MILLIS.
private static final long MAX_VALIDATION_TIMEOUT_MILLIS = 4000;
private static final long MAX_UPDATE_TIMEOUT_MILLIS = 30000;
private static final long EXTERNAL_HOTWORD_CLEANUP_MILLIS = 2000;
private static final Duration MAX_UPDATE_TIMEOUT_DURATION =
@@ -140,29 +133,29 @@ class HotwordDetectorSession {
private static final int METRICS_INIT_CALLBACK_STATE_SUCCESS =
HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_SUCCESS;
private static final int METRICS_KEYPHRASE_TRIGGERED_DETECT_SECURITY_EXCEPTION =
static final int METRICS_KEYPHRASE_TRIGGERED_DETECT_SECURITY_EXCEPTION =
HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_SECURITY_EXCEPTION;
private static final int METRICS_KEYPHRASE_TRIGGERED_DETECT_UNEXPECTED_CALLBACK =
static final int METRICS_KEYPHRASE_TRIGGERED_DETECT_UNEXPECTED_CALLBACK =
HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_UNEXPECTED_CALLBACK;
private static final int METRICS_KEYPHRASE_TRIGGERED_REJECT_UNEXPECTED_CALLBACK =
static final int METRICS_KEYPHRASE_TRIGGERED_REJECT_UNEXPECTED_CALLBACK =
HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECT_UNEXPECTED_CALLBACK;
private static final int METRICS_EXTERNAL_SOURCE_DETECTED =
HOTWORD_DETECTOR_EVENTS__EVENT__EXTERNAL_SOURCE_DETECTED;
private static final int METRICS_EXTERNAL_SOURCE_REJECTED =
HOTWORD_DETECTOR_EVENTS__EVENT__EXTERNAL_SOURCE_REJECTED;
private static final int METRICS_EXTERNAL_SOURCE_DETECT_SECURITY_EXCEPTION =
private static final int EXTERNAL_SOURCE_DETECT_SECURITY_EXCEPTION =
HOTWORD_DETECTOR_EVENTS__EVENT__EXTERNAL_SOURCE_DETECT_SECURITY_EXCEPTION;
private static final int METRICS_CALLBACK_ON_STATUS_REPORTED_EXCEPTION =
HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_STATUS_REPORTED_EXCEPTION;
private final Executor mAudioCopyExecutor = Executors.newCachedThreadPool();
// TODO: This may need to be a Handler(looper)
private final ScheduledExecutorService mScheduledExecutorService;
final ScheduledExecutorService mScheduledExecutorService;
private final AppOpsManager mAppOpsManager;
private final HotwordAudioStreamCopier mHotwordAudioStreamCopier;
private final AtomicBoolean mUpdateStateAfterStartFinished = new AtomicBoolean(false);
private final IHotwordRecognitionStatusCallback mCallback;
final HotwordAudioStreamCopier mHotwordAudioStreamCopier;
final AtomicBoolean mUpdateStateAfterStartFinished = new AtomicBoolean(false);
final IHotwordRecognitionStatusCallback mCallback;
final Object mLock;
final int mVoiceInteractionServiceUid;
@@ -173,28 +166,22 @@ class HotwordDetectorSession {
final AttentionManagerInternal.ProximityUpdateCallbackInternal mProximityCallbackInternal =
this::setProximityValue;
private IMicrophoneHotwordDetectionVoiceInteractionCallback mSoftwareCallback;
private ScheduledFuture<?> mCancellationKeyPhraseDetectionFuture;
/** Identity used for attributing app ops when delivering data to the Interactor. */
@GuardedBy("mLock")
@Nullable
private final Identity mVoiceInteractorIdentity;
@GuardedBy("mLock")
private ParcelFileDescriptor mCurrentAudioSink;
ParcelFileDescriptor mCurrentAudioSink;
@GuardedBy("mLock")
private boolean mValidatingDspTrigger = false;
@GuardedBy("mLock")
private boolean mPerformingSoftwareHotwordDetection;
@NonNull private HotwordDetectionConnection.ServiceConnection mRemoteHotwordDetectionService;
private boolean mDebugHotwordLogging = false;
@NonNull HotwordDetectionConnection.ServiceConnection mRemoteHotwordDetectionService;
boolean mDebugHotwordLogging = false;
@GuardedBy("mLock")
private double mProximityMeters = PROXIMITY_UNKNOWN;
@GuardedBy("mLock")
private boolean mInitialized = false;
@GuardedBy("mLock")
private boolean mDestroyed = false;
@GuardedBy("mLock")
boolean mPerformingExternalSourceHotwordDetection;
HotwordDetectorSession(
@NonNull HotwordDetectionConnection.ServiceConnection remoteHotwordDetectionService,
@@ -223,6 +210,7 @@ class HotwordDetectorSession {
}
}
@SuppressWarnings("GuardedBy")
private void updateStateAfterProcessStartLocked(PersistableBundle options,
SharedMemory sharedMemory) {
if (DEBUG) {
@@ -315,6 +303,7 @@ class HotwordDetectorSession {
return new Pair<>(status, metricsResult);
}
@SuppressWarnings("GuardedBy")
void updateStateLocked(PersistableBundle options, SharedMemory sharedMemory,
Instant lastRestartInstant) {
HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
@@ -332,108 +321,13 @@ class HotwordDetectorSession {
}
}
/**
* This method is only used by SoftwareHotwordDetector.
*/
@SuppressWarnings("GuardedBy")
void startListeningFromMicLocked(
AudioFormat audioFormat,
IMicrophoneHotwordDetectionVoiceInteractionCallback callback) {
if (DEBUG) {
Slog.d(TAG, "startListeningFromMic");
}
mSoftwareCallback = callback;
if (mPerformingSoftwareHotwordDetection) {
Slog.i(TAG, "Hotword validation is already in progress, ignoring.");
return;
}
mPerformingSoftwareHotwordDetection = true;
startListeningFromMicLocked();
}
/**
* This method is only used by SoftwareHotwordDetector.
*/
private void startListeningFromMicLocked() {
// TODO: consider making this a non-anonymous class.
IDspHotwordDetectionCallback internalCallback = new IDspHotwordDetectionCallback.Stub() {
@Override
public void onDetected(HotwordDetectedResult result) throws RemoteException {
if (DEBUG) {
Slog.d(TAG, "onDetected");
}
synchronized (mLock) {
HotwordMetricsLogger.writeKeyphraseTriggerEvent(
getDetectorType(),
HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECTED);
if (!mPerformingSoftwareHotwordDetection) {
Slog.i(TAG, "Hotword detection has already completed");
HotwordMetricsLogger.writeKeyphraseTriggerEvent(
getDetectorType(),
METRICS_KEYPHRASE_TRIGGERED_DETECT_UNEXPECTED_CALLBACK);
return;
}
mPerformingSoftwareHotwordDetection = false;
try {
enforcePermissionsForDataDelivery();
} catch (SecurityException e) {
HotwordMetricsLogger.writeKeyphraseTriggerEvent(
getDetectorType(),
METRICS_KEYPHRASE_TRIGGERED_DETECT_SECURITY_EXCEPTION);
mSoftwareCallback.onError();
return;
}
saveProximityValueToBundle(result);
HotwordDetectedResult newResult;
try {
newResult = mHotwordAudioStreamCopier.startCopyingAudioStreams(result);
} catch (IOException e) {
// TODO: Write event
mSoftwareCallback.onError();
return;
}
mSoftwareCallback.onDetected(newResult, null, null);
Slog.i(TAG, "Egressed " + HotwordDetectedResult.getUsageSize(newResult)
+ " bits from hotword trusted process");
if (mDebugHotwordLogging) {
Slog.i(TAG, "Egressed detected result: " + newResult);
}
}
}
@Override
public void onRejected(HotwordRejectedResult result) throws RemoteException {
if (DEBUG) {
Slog.wtf(TAG, "onRejected");
}
HotwordMetricsLogger.writeKeyphraseTriggerEvent(
getDetectorType(),
HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED);
// onRejected isn't allowed here, and we are not expecting it.
}
};
mRemoteHotwordDetectionService.run(
service -> service.detectFromMicrophoneSource(
null,
AUDIO_SOURCE_MICROPHONE,
null,
null,
internalCallback));
HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
HOTWORD_DETECTOR_EVENTS__EVENT__START_SOFTWARE_DETECTION,
mVoiceInteractionServiceUid);
}
public void startListeningFromExternalSourceLocked(
void startListeningFromExternalSourceLocked(
ParcelFileDescriptor audioStream,
AudioFormat audioFormat,
@Nullable PersistableBundle options,
IMicrophoneHotwordDetectionVoiceInteractionCallback callback) {
if (DEBUG) {
Slog.d(TAG, "startListeningFromExternalSource");
Slog.d(TAG, "startListeningFromExternalSourceLocked");
}
handleExternalSourceHotwordDetectionLocked(
@@ -443,169 +337,6 @@ class HotwordDetectorSession {
callback);
}
/**
* This method is only used by SoftwareHotwordDetector.
*/
@SuppressWarnings("GuardedBy")
void stopListeningLocked() {
if (DEBUG) {
Slog.d(TAG, "stopListening");
}
if (!mPerformingSoftwareHotwordDetection) {
Slog.i(TAG, "Hotword detection is not running");
return;
}
mPerformingSoftwareHotwordDetection = false;
mRemoteHotwordDetectionService.run(IHotwordDetectionService::stopDetection);
if (mCurrentAudioSink != null) {
Slog.i(TAG, "Closing audio stream to hotword detector: stopping requested");
bestEffortClose(mCurrentAudioSink);
}
mCurrentAudioSink = null;
}
@SuppressWarnings("GuardedBy")
void detectFromDspSourceLocked(SoundTrigger.KeyphraseRecognitionEvent recognitionEvent,
IHotwordRecognitionStatusCallback externalCallback) {
if (DEBUG) {
Slog.d(TAG, "detectFromDspSource");
}
AtomicBoolean timeoutDetected = new AtomicBoolean(false);
// TODO: consider making this a non-anonymous class.
IDspHotwordDetectionCallback internalCallback = new IDspHotwordDetectionCallback.Stub() {
@Override
public void onDetected(HotwordDetectedResult result) throws RemoteException {
if (DEBUG) {
Slog.d(TAG, "onDetected");
}
synchronized (mLock) {
if (mCancellationKeyPhraseDetectionFuture != null) {
mCancellationKeyPhraseDetectionFuture.cancel(true);
}
if (timeoutDetected.get()) {
return;
}
HotwordMetricsLogger.writeKeyphraseTriggerEvent(
getDetectorType(),
HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECTED);
if (!mValidatingDspTrigger) {
Slog.i(TAG, "Ignoring #onDetected due to a process restart");
HotwordMetricsLogger.writeKeyphraseTriggerEvent(
getDetectorType(),
METRICS_KEYPHRASE_TRIGGERED_DETECT_UNEXPECTED_CALLBACK);
return;
}
mValidatingDspTrigger = false;
try {
enforcePermissionsForDataDelivery();
enforceExtraKeyphraseIdNotLeaked(result, recognitionEvent);
} catch (SecurityException e) {
Slog.i(TAG, "Ignoring #onDetected due to a SecurityException", e);
HotwordMetricsLogger.writeKeyphraseTriggerEvent(
getDetectorType(),
METRICS_KEYPHRASE_TRIGGERED_DETECT_SECURITY_EXCEPTION);
externalCallback.onError(CALLBACK_ONDETECTED_GOT_SECURITY_EXCEPTION);
return;
}
saveProximityValueToBundle(result);
HotwordDetectedResult newResult;
try {
newResult = mHotwordAudioStreamCopier.startCopyingAudioStreams(result);
} catch (IOException e) {
// TODO: Write event
externalCallback.onError(CALLBACK_ONDETECTED_STREAM_COPY_ERROR);
return;
}
externalCallback.onKeyphraseDetected(recognitionEvent, newResult);
Slog.i(TAG, "Egressed " + HotwordDetectedResult.getUsageSize(newResult)
+ " bits from hotword trusted process");
if (mDebugHotwordLogging) {
Slog.i(TAG, "Egressed detected result: " + newResult);
}
}
}
@Override
public void onRejected(HotwordRejectedResult result) throws RemoteException {
if (DEBUG) {
Slog.d(TAG, "onRejected");
}
synchronized (mLock) {
if (mCancellationKeyPhraseDetectionFuture != null) {
mCancellationKeyPhraseDetectionFuture.cancel(true);
}
if (timeoutDetected.get()) {
return;
}
HotwordMetricsLogger.writeKeyphraseTriggerEvent(
getDetectorType(),
HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED);
if (!mValidatingDspTrigger) {
Slog.i(TAG, "Ignoring #onRejected due to a process restart");
HotwordMetricsLogger.writeKeyphraseTriggerEvent(
getDetectorType(),
METRICS_KEYPHRASE_TRIGGERED_REJECT_UNEXPECTED_CALLBACK);
return;
}
mValidatingDspTrigger = false;
externalCallback.onRejected(result);
if (mDebugHotwordLogging && result != null) {
Slog.i(TAG, "Egressed rejected result: " + result);
}
}
}
};
mValidatingDspTrigger = true;
mRemoteHotwordDetectionService.run(service -> {
// We use the VALIDATION_TIMEOUT_MILLIS to inform that the client needs to invoke
// the callback before timeout value. In order to reduce the latency impact between
// server side and client side, we need to use another timeout value
// MAX_VALIDATION_TIMEOUT_MILLIS to monitor it.
mCancellationKeyPhraseDetectionFuture = mScheduledExecutorService.schedule(
() -> {
// TODO: avoid allocate every time
timeoutDetected.set(true);
Slog.w(TAG, "Timed out on #detectFromDspSource");
HotwordMetricsLogger.writeKeyphraseTriggerEvent(
getDetectorType(),
HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_TIMEOUT);
try {
externalCallback.onError(CALLBACK_DETECT_TIMEOUT);
} catch (RemoteException e) {
Slog.w(TAG, "Failed to report onError status: ", e);
HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION,
mVoiceInteractionServiceUid);
}
},
MAX_VALIDATION_TIMEOUT_MILLIS,
TimeUnit.MILLISECONDS);
service.detectFromDspSource(
recognitionEvent,
recognitionEvent.getCaptureFormat(),
VALIDATION_TIMEOUT_MILLIS,
internalCallback);
});
}
@SuppressWarnings("GuardedBy")
public void dumpLocked(String prefix, PrintWriter pw) {
pw.print(prefix); pw.print("mCallback="); pw.println(mCallback);
pw.print(prefix); pw.print("mUpdateStateAfterStartFinished=");
pw.println(mUpdateStateAfterStartFinished);
pw.print(prefix); pw.print("mInitialized="); pw.println(mInitialized);
pw.print(prefix); pw.print("mDestroyed="); pw.println(mDestroyed);
pw.print(prefix); pw.print("mValidatingDspTrigger="); pw.println(mValidatingDspTrigger);
pw.print(prefix); pw.print("mPerformingSoftwareHotwordDetection=");
pw.println(mPerformingSoftwareHotwordDetection);
pw.print(prefix); pw.print("DetectorType=");
pw.println(HotwordDetector.detectorTypeToString(getDetectorType()));
}
@SuppressWarnings("GuardedBy")
private void handleExternalSourceHotwordDetectionLocked(
ParcelFileDescriptor audioStream,
@@ -615,6 +346,11 @@ class HotwordDetectorSession {
if (DEBUG) {
Slog.d(TAG, "#handleExternalSourceHotwordDetectionLocked");
}
if (mPerformingExternalSourceHotwordDetection) {
Slog.i(TAG, "Hotword validation is already in progress for external source.");
return;
}
InputStream audioSource = new ParcelFileDescriptor.AutoCloseInputStream(audioStream);
Pair<ParcelFileDescriptor, ParcelFileDescriptor> clientPipe = createPipe();
@@ -626,6 +362,7 @@ class HotwordDetectorSession {
ParcelFileDescriptor serviceAudioSource = clientPipe.first;
mCurrentAudioSink = serviceAudioSink;
mPerformingExternalSourceHotwordDetection = true;
mAudioCopyExecutor.execute(() -> {
try (InputStream source = audioSource;
@@ -659,7 +396,8 @@ class HotwordDetectorSession {
}
} finally {
synchronized (mLock) {
mCurrentAudioSink = null;
mPerformingExternalSourceHotwordDetection = false;
closeExternalAudioStreamLocked("start external source");
}
}
});
@@ -678,23 +416,26 @@ class HotwordDetectorSession {
@Override
public void onRejected(HotwordRejectedResult result)
throws RemoteException {
HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
METRICS_EXTERNAL_SOURCE_REJECTED,
mVoiceInteractionServiceUid);
mScheduledExecutorService.schedule(
() -> {
bestEffortClose(serviceAudioSink, audioSource);
},
EXTERNAL_HOTWORD_CLEANUP_MILLIS,
TimeUnit.MILLISECONDS);
synchronized (mLock) {
mPerformingExternalSourceHotwordDetection = false;
HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
METRICS_EXTERNAL_SOURCE_REJECTED,
mVoiceInteractionServiceUid);
mScheduledExecutorService.schedule(
() -> {
bestEffortClose(serviceAudioSink, audioSource);
},
EXTERNAL_HOTWORD_CLEANUP_MILLIS,
TimeUnit.MILLISECONDS);
callback.onRejected(result);
callback.onRejected(result);
if (result != null) {
Slog.i(TAG, "Egressed 'hotword rejected result' "
+ "from hotword trusted process");
if (mDebugHotwordLogging) {
Slog.i(TAG, "Egressed detected result: " + result);
if (result != null) {
Slog.i(TAG, "Egressed 'hotword rejected result' "
+ "from hotword trusted process");
if (mDebugHotwordLogging) {
Slog.i(TAG, "Egressed detected result: " + result);
}
}
}
}
@@ -702,43 +443,46 @@ class HotwordDetectorSession {
@Override
public void onDetected(HotwordDetectedResult triggerResult)
throws RemoteException {
HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
METRICS_EXTERNAL_SOURCE_DETECTED,
mVoiceInteractionServiceUid);
mScheduledExecutorService.schedule(
() -> {
bestEffortClose(serviceAudioSink, audioSource);
},
EXTERNAL_HOTWORD_CLEANUP_MILLIS,
TimeUnit.MILLISECONDS);
try {
enforcePermissionsForDataDelivery();
} catch (SecurityException e) {
synchronized (mLock) {
mPerformingExternalSourceHotwordDetection = false;
HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
METRICS_EXTERNAL_SOURCE_DETECT_SECURITY_EXCEPTION,
METRICS_EXTERNAL_SOURCE_DETECTED,
mVoiceInteractionServiceUid);
callback.onError();
return;
}
HotwordDetectedResult newResult;
try {
newResult =
mHotwordAudioStreamCopier.startCopyingAudioStreams(
triggerResult);
} catch (IOException e) {
// TODO: Write event
callback.onError();
return;
}
callback.onDetected(newResult, null /* audioFormat */,
null /* audioStream */);
Slog.i(TAG, "Egressed "
+ HotwordDetectedResult.getUsageSize(newResult)
+ " bits from hotword trusted process");
if (mDebugHotwordLogging) {
Slog.i(TAG,
"Egressed detected result: " + newResult);
mScheduledExecutorService.schedule(
() -> {
bestEffortClose(serviceAudioSink, audioSource);
},
EXTERNAL_HOTWORD_CLEANUP_MILLIS,
TimeUnit.MILLISECONDS);
try {
enforcePermissionsForDataDelivery();
} catch (SecurityException e) {
HotwordMetricsLogger.writeDetectorEvent(
getDetectorType(),
EXTERNAL_SOURCE_DETECT_SECURITY_EXCEPTION,
mVoiceInteractionServiceUid);
callback.onError();
return;
}
HotwordDetectedResult newResult;
try {
newResult = mHotwordAudioStreamCopier
.startCopyingAudioStreams(triggerResult);
} catch (IOException e) {
// TODO: Write event
callback.onError();
return;
}
callback.onDetected(newResult, null /* audioFormat */,
null /* audioStream */);
Slog.i(TAG, "Egressed "
+ HotwordDetectedResult.getUsageSize(newResult)
+ " bits from hotword trusted process");
if (mDebugHotwordLogging) {
Slog.i(TAG,
"Egressed detected result: " + newResult);
}
}
}
});
@@ -772,60 +516,16 @@ class HotwordDetectorSession {
}
void setDebugHotwordLoggingLocked(boolean logging) {
Slog.v(TAG, "setDebugHotwordLogging: " + logging);
Slog.v(TAG, "setDebugHotwordLoggingLocked: " + logging);
mDebugHotwordLogging = logging;
}
@SuppressWarnings("GuardedBy")
void updateRemoteHotwordDetectionServiceLocked(
@NonNull HotwordDetectionConnection.ServiceConnection remoteHotwordDetectionService) {
mRemoteHotwordDetectionService = remoteHotwordDetectionService;
}
@SuppressWarnings("GuardedBy")
void informRestartProcessLocked() {
// TODO(b/244598068): Check HotwordAudioStreamManager first
Slog.v(TAG, "informRestartProcess");
if (mValidatingDspTrigger) {
// We're restarting the process while it's processing a DSP trigger, so report a
// rejection. This also allows the Interactor to startRecognition again
try {
mCallback.onRejected(new HotwordRejectedResult.Builder().build());
HotwordMetricsLogger.writeKeyphraseTriggerEvent(
getDetectorType(),
HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED_FROM_RESTART);
} catch (RemoteException e) {
Slog.w(TAG, "Failed to call #rejected");
HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_REJECTED_EXCEPTION,
mVoiceInteractionServiceUid);
}
mValidatingDspTrigger = false;
}
mUpdateStateAfterStartFinished.set(false);
try {
mCallback.onProcessRestarted();
} catch (RemoteException e) {
Slog.w(TAG, "Failed to communicate #onProcessRestarted", e);
HotwordMetricsLogger.writeDetectorEvent(getDetectorType(),
HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_PROCESS_RESTARTED_EXCEPTION,
mVoiceInteractionServiceUid);
}
// Restart listening from microphone if the hotword process has been restarted.
if (mPerformingSoftwareHotwordDetection) {
Slog.i(TAG, "Process restarted: calling startRecognition() again");
startListeningFromMicLocked();
}
if (mCurrentAudioSink != null) {
Slog.i(TAG, "Closing external audio stream to hotword detector: process restarted");
bestEffortClose(mCurrentAudioSink);
mCurrentAudioSink = null;
}
}
void reportErrorLocked(int status) {
try {
mCallback.onError(status);
@@ -837,6 +537,11 @@ class HotwordDetectorSession {
}
}
/**
* Called when the trusted process is restarted.
*/
abstract void informRestartProcessLocked();
private static Pair<ParcelFileDescriptor, ParcelFileDescriptor> createPipe() {
ParcelFileDescriptor[] fileDescriptors;
try {
@@ -849,7 +554,7 @@ class HotwordDetectorSession {
return Pair.create(fileDescriptors[0], fileDescriptors[1]);
}
private void saveProximityValueToBundle(HotwordDetectedResult result) {
void saveProximityValueToBundle(HotwordDetectedResult result) {
synchronized (mLock) {
if (result != null && mProximityMeters != PROXIMITY_UNKNOWN) {
result.setProximity(mProximityMeters);
@@ -863,6 +568,15 @@ class HotwordDetectorSession {
}
}
@SuppressWarnings("GuardedBy")
void closeExternalAudioStreamLocked(String reason) {
if (mCurrentAudioSink != null) {
Slog.i(TAG, "Closing external audio stream to hotword detector: " + reason);
bestEffortClose(mCurrentAudioSink);
mCurrentAudioSink = null;
}
}
private static void bestEffortClose(Closeable... closeables) {
for (Closeable closeable : closeables) {
bestEffortClose(closeable);
@@ -880,7 +594,7 @@ class HotwordDetectorSession {
}
// TODO: Share this code with SoundTriggerMiddlewarePermission.
private void enforcePermissionsForDataDelivery() {
void enforcePermissionsForDataDelivery() {
Binder.withCleanCallingIdentity(() -> {
synchronized (mLock) {
enforcePermissionForPreflight(mContext, mVoiceInteractorIdentity, RECORD_AUDIO);
@@ -914,7 +628,7 @@ class HotwordDetectorSession {
}
}
private static void enforceExtraKeyphraseIdNotLeaked(HotwordDetectedResult result,
static void enforceExtraKeyphraseIdNotLeaked(HotwordDetectedResult result,
SoundTrigger.KeyphraseRecognitionEvent recognitionEvent) {
// verify the phrase ID in HotwordDetectedResult is not exposing extra phrases
// the DSP did not detect
@@ -936,4 +650,17 @@ class HotwordDetectorSession {
Slog.v(TAG, "Unexpected detector type");
return -1;
}
@SuppressWarnings("GuardedBy")
public void dumpLocked(String prefix, PrintWriter pw) {
pw.print(prefix); pw.print("mCallback="); pw.println(mCallback);
pw.print(prefix); pw.print("mUpdateStateAfterStartFinished=");
pw.println(mUpdateStateAfterStartFinished);
pw.print(prefix); pw.print("mInitialized="); pw.println(mInitialized);
pw.print(prefix); pw.print("mDestroyed="); pw.println(mDestroyed);
pw.print(prefix); pw.print("DetectorType=");
pw.println(HotwordDetector.detectorTypeToString(getDetectorType()));
pw.print(prefix); pw.print("mPerformingExternalSourceHotwordDetection=");
pw.println(mPerformingExternalSourceHotwordDetection);
}
}

171
services/voiceinteraction/java/com/android/server/voiceinteraction/SoftwareTrustedHotwordDetectorSession.java
View File

@@ -16,20 +16,50 @@
package com.android.server.voiceinteraction;
import static android.service.voice.HotwordDetectionService.AUDIO_SOURCE_MICROPHONE;
import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_PROCESS_RESTARTED_EXCEPTION;
import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__START_SOFTWARE_DETECTION;
import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECTED;
import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED;
import android.annotation.NonNull;
import android.content.Context;
import android.media.AudioFormat;
import android.media.permission.Identity;
import android.os.PersistableBundle;
import android.os.RemoteException;
import android.os.SharedMemory;
import android.service.voice.HotwordDetectedResult;
import android.service.voice.HotwordDetectionService;
import android.service.voice.HotwordDetector;
import android.service.voice.HotwordRejectedResult;
import android.service.voice.IDspHotwordDetectionCallback;
import android.service.voice.IHotwordDetectionService;
import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback;
import android.util.Slog;
import com.android.internal.annotations.GuardedBy;
import com.android.internal.app.IHotwordRecognitionStatusCallback;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.concurrent.ScheduledExecutorService;
/**
* A class that provides software trusted hotword detector to communicate with the {@link
* HotwordDetectionService}.
*
* This class can handle the hotword detection which detector is created by using
* {@link android.service.voice.VoiceInteractionService#createHotwordDetector(PersistableBundle,
* SharedMemory, HotwordDetector.Callback)}.
*/
final class SoftwareTrustedHotwordDetectorSession extends HotwordDetectorSession {
private static final String TAG = "SoftwareTrustedHotwordDetectorSession";
private IMicrophoneHotwordDetectionVoiceInteractionCallback mSoftwareCallback;
@GuardedBy("mLock")
private boolean mPerformingSoftwareHotwordDetection;
SoftwareTrustedHotwordDetectorSession(
@NonNull HotwordDetectionConnection.ServiceConnection remoteHotwordDetectionService,
@@ -40,4 +70,145 @@ final class SoftwareTrustedHotwordDetectorSession extends HotwordDetectorSession
super(remoteHotwordDetectionService, lock, context, callback, voiceInteractionServiceUid,
voiceInteractorIdentity, scheduledExecutorService, logging);
}
@SuppressWarnings("GuardedBy")
void startListeningFromMicLocked(
AudioFormat audioFormat,
IMicrophoneHotwordDetectionVoiceInteractionCallback callback) {
if (DEBUG) {
Slog.d(TAG, "startListeningFromMicLocked");
}
mSoftwareCallback = callback;
if (mPerformingSoftwareHotwordDetection) {
Slog.i(TAG, "Hotword validation is already in progress, ignoring.");
return;
}
mPerformingSoftwareHotwordDetection = true;
startListeningFromMicLocked();
}
@SuppressWarnings("GuardedBy")
private void startListeningFromMicLocked() {
// TODO: consider making this a non-anonymous class.
IDspHotwordDetectionCallback internalCallback = new IDspHotwordDetectionCallback.Stub() {
@Override
public void onDetected(HotwordDetectedResult result) throws RemoteException {
if (DEBUG) {
Slog.d(TAG, "onDetected");
}
synchronized (mLock) {
HotwordMetricsLogger.writeKeyphraseTriggerEvent(
HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE,
HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECTED);
if (!mPerformingSoftwareHotwordDetection) {
Slog.i(TAG, "Hotword detection has already completed");
HotwordMetricsLogger.writeKeyphraseTriggerEvent(
HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE,
METRICS_KEYPHRASE_TRIGGERED_DETECT_UNEXPECTED_CALLBACK);
return;
}
mPerformingSoftwareHotwordDetection = false;
try {
enforcePermissionsForDataDelivery();
} catch (SecurityException e) {
HotwordMetricsLogger.writeKeyphraseTriggerEvent(
HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE,
METRICS_KEYPHRASE_TRIGGERED_DETECT_SECURITY_EXCEPTION);
mSoftwareCallback.onError();
return;
}
saveProximityValueToBundle(result);
HotwordDetectedResult newResult;
try {
newResult = mHotwordAudioStreamCopier.startCopyingAudioStreams(result);
} catch (IOException e) {
// TODO: Write event
mSoftwareCallback.onError();
return;
}
mSoftwareCallback.onDetected(newResult, null, null);
Slog.i(TAG, "Egressed " + HotwordDetectedResult.getUsageSize(newResult)
+ " bits from hotword trusted process");
if (mDebugHotwordLogging) {
Slog.i(TAG, "Egressed detected result: " + newResult);
}
}
}
@Override
public void onRejected(HotwordRejectedResult result) throws RemoteException {
if (DEBUG) {
Slog.wtf(TAG, "onRejected");
}
HotwordMetricsLogger.writeKeyphraseTriggerEvent(
HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE,
HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED);
// onRejected isn't allowed here, and we are not expecting it.
}
};
mRemoteHotwordDetectionService.run(
service -> service.detectFromMicrophoneSource(
null,
AUDIO_SOURCE_MICROPHONE,
null,
null,
internalCallback));
HotwordMetricsLogger.writeDetectorEvent(
HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE,
HOTWORD_DETECTOR_EVENTS__EVENT__START_SOFTWARE_DETECTION,
mVoiceInteractionServiceUid);
}
@SuppressWarnings("GuardedBy")
void stopListeningLocked() {
if (DEBUG) {
Slog.d(TAG, "stopListeningLocked");
}
if (!mPerformingSoftwareHotwordDetection) {
Slog.i(TAG, "Hotword detection is not running");
return;
}
mPerformingSoftwareHotwordDetection = false;
mRemoteHotwordDetectionService.run(IHotwordDetectionService::stopDetection);
closeExternalAudioStreamLocked("stopping requested");
}
@Override
@SuppressWarnings("GuardedBy")
void informRestartProcessLocked() {
// TODO(b/244598068): Check HotwordAudioStreamManager first
Slog.v(TAG, "informRestartProcessLocked");
mUpdateStateAfterStartFinished.set(false);
try {
mCallback.onProcessRestarted();
} catch (RemoteException e) {
Slog.w(TAG, "Failed to communicate #onProcessRestarted", e);
HotwordMetricsLogger.writeDetectorEvent(
HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE,
HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_PROCESS_RESTARTED_EXCEPTION,
mVoiceInteractionServiceUid);
}
// Restart listening from microphone if the hotword process has been restarted.
if (mPerformingSoftwareHotwordDetection) {
Slog.i(TAG, "Process restarted: calling startRecognition() again");
startListeningFromMicLocked();
}
mPerformingExternalSourceHotwordDetection = false;
closeExternalAudioStreamLocked("process restarted");
}
@SuppressWarnings("GuardedBy")
public void dumpLocked(String prefix, PrintWriter pw) {
super.dumpLocked(prefix, pw);
pw.print(prefix); pw.print("mPerformingSoftwareHotwordDetection=");
pw.println(mPerformingSoftwareHotwordDetection);
}
}