Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve error handling for Trusted Hotword

Browse Source 
Add a new callback with an DetectorFailure parameter to let the
assistant know what went wrong during using the hotword detector.

Bug: 256780491
Bug: 261012843
Test: atest CtsVoiceInteractionTestCases

Change-Id: I0b7d9625abfa10f93ad8564e85faedd1db4fa51d
This commit is contained in:
lpeter
2022-12-28 18:50:58 +00:00
committed by Peter Li
parent 7137d9165d
commit 321950f368
14 changed files with 153 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
core/api/system-current.txt
View File

@@ -13141,7 +13141,8 @@ package android.service.voice {
public static interface HotwordDetector.Callback {
method public void onDetected(@NonNull android.service.voice.AlwaysOnHotwordDetector.EventPayload);
method public void onError();
method @Deprecated public void onError();
method public default void onFailure(@NonNull android.service.voice.DetectorFailure);
method public void onHotwordDetectionServiceInitialized(int);
method public void onHotwordDetectionServiceRestarted();
method public void onRecognitionPaused();
@@ -13202,7 +13203,7 @@ package android.service.voice {
}
public static interface VisualQueryDetector.Callback {
method public void onError();
method public void onFailure(@NonNull android.service.voice.DetectorFailure);
method public void onQueryDetected(@NonNull String);
method public void onQueryFinished();
method public void onQueryRejected();

9
core/java/android/service/voice/AbstractDetector.java
View File

@@ -231,9 +231,12 @@ abstract class AbstractDetector implements HotwordDetector {
/** Called when the detection fails due to an error. */
@Override
public void onError() {
Slog.v(TAG, "BinderCallback#onError");
Binder.withCleanCallingIdentity(() -> mExecutor.execute(() -> mCallback.onError()));
public void onError(DetectorFailure detectorFailure) {
Slog.v(TAG, "BinderCallback#onError detectorFailure: " + detectorFailure);
Binder.withCleanCallingIdentity(() -> mExecutor.execute(() -> {
mCallback.onFailure(detectorFailure != null ? detectorFailure
: new UnknownFailure("Error data is null"));
}));
}
@Override

21
core/java/android/service/voice/AlwaysOnHotwordDetector.java
View File

@@ -732,7 +732,13 @@ public class AlwaysOnHotwordDetector extends AbstractDetector {
*/
public abstract void onDetected(@NonNull EventPayload eventPayload);
/** {@inheritDoc} */
/**
* {@inheritDoc}
*
* @deprecated Use {@link HotwordDetector.Callback#onError(DetectorFailure)} instead.
*/
@Deprecated
@Override
public abstract void onError();
/** {@inheritDoc} */
@@ -1658,9 +1664,18 @@ public class AlwaysOnHotwordDetector extends AbstractDetector {
@Override
public void onError(int status) {
Slog.i(TAG, "onError: " + status);
mHandler.sendEmptyMessage(MSG_DETECTION_ERROR);
// This is a workaround before the sound trigger uses the onDetectionFailure method.
Message.obtain(mHandler, MSG_DETECTION_ERROR,
new SoundTriggerFailure(status, "Sound trigger error")).sendToTarget();
}
@Override
public void onDetectionFailure(DetectorFailure detectorFailure) {
Slog.v(TAG, "onDetectionFailure detectorFailure: " + detectorFailure);
Message.obtain(mHandler, MSG_DETECTION_ERROR,
detectorFailure != null ? detectorFailure
: new UnknownFailure("Error data is null")).sendToTarget();
}
@Override
public void onRecognitionPaused() {
Slog.i(TAG, "onRecognitionPaused");
@@ -1716,7 +1731,7 @@ public class AlwaysOnHotwordDetector extends AbstractDetector {
mExternalCallback.onDetected((EventPayload) message.obj);
break;
case MSG_DETECTION_ERROR:
mExternalCallback.onError();
mExternalCallback.onFailure((DetectorFailure) msg.obj);
break;
case MSG_DETECTION_PAUSE:
mExternalCallback.onRecognitionPaused();

19
core/java/android/service/voice/HotwordDetector.java
View File

@@ -231,9 +231,28 @@ public interface HotwordDetector {
/**
* Called when the detection fails due to an error.
*
* @deprecated On Android 14 and above, implement {@link #onFailure(DetectorFailure)}
* instead.
*/
@Deprecated
void onError();
/**
* Called when the detection fails due to an error, the subclasses of
* {@link DetectorFailure} will be reported to the detector.
*
* @see android.service.voice.HotwordDetectionServiceFailure
* @see android.service.voice.SoundTriggerFailure
* @see android.service.voice.UnknownFailure
* @see android.service.voice.VisualQueryDetectionServiceFailure
*
* @param detectorFailure It provides the error code, error message and suggested action.
*/
default void onFailure(@NonNull DetectorFailure detectorFailure) {
onError();
}
/**
* Called when the recognition is paused temporarily for some reason.
* This is an informational callback, and the clients shouldn't be doing anything here

3
core/java/android/service/voice/IMicrophoneHotwordDetectionVoiceInteractionCallback.aidl
View File

@@ -17,6 +17,7 @@
package android.service.voice;
import android.media.AudioFormat;
import android.service.voice.DetectorFailure;
import android.service.voice.HotwordDetectedResult;
import android.service.voice.HotwordRejectedResult;
@@ -38,7 +39,7 @@ oneway interface IMicrophoneHotwordDetectionVoiceInteractionCallback {
/**
* Called when the detection fails due to an error.
*/
void onError();
void onError(in DetectorFailure detectorFailure);
/**
* Called when the detected result was not detected.

5
core/java/android/service/voice/IVisualQueryDetectionVoiceInteractionCallback.aidl
View File

@@ -16,7 +16,7 @@
package android.service.voice;
import android.media.AudioFormat;
import android.service.voice.DetectorFailure;
/**
* Callback for returning the detected result from the VisualQueryDetectionService.
@@ -43,6 +43,5 @@ oneway interface IVisualQueryDetectionVoiceInteractionCallback {
/**
* Called when the detection fails due to an error.
*/
void onError();
void onDetectionFailure(in DetectorFailure detectorFailure);
}

19
core/java/android/service/voice/SoftwareHotwordDetector.java
View File

@@ -169,9 +169,12 @@ class SoftwareHotwordDetector extends AbstractDetector {
/** Called when the detection fails due to an error. */
@Override
public void onError() {
Slog.v(TAG, "BinderCallback#onError");
Binder.withCleanCallingIdentity(() -> mExecutor.execute(() -> mCallback.onError()));
public void onError(DetectorFailure detectorFailure) {
Slog.v(TAG, "BinderCallback#onError detectorFailure: " + detectorFailure);
Binder.withCleanCallingIdentity(() -> mExecutor.execute(() -> {
mCallback.onFailure(detectorFailure != null ? detectorFailure
: new UnknownFailure("Error data is null"));
}));
}
@Override
@@ -222,6 +225,16 @@ class SoftwareHotwordDetector extends AbstractDetector {
if (DEBUG) {
Slog.i(TAG, "Ignored #onError (" + status + ") event");
}
// TODO: Check if we still need to implement this method with DetectorFailure mechanism.
}
@Override
public void onDetectionFailure(DetectorFailure detectorFailure) throws RemoteException {
Slog.v(TAG, "onDetectionFailure detectorFailure: " + detectorFailure);
Binder.withCleanCallingIdentity(() -> mExecutor.execute(() -> {
mCallback.onFailure(detectorFailure != null ? detectorFailure
: new UnknownFailure("Error data is null"));
}));
}
@Override

14
core/java/android/service/voice/VisualQueryDetector.java
View File

@@ -216,8 +216,7 @@ public class VisualQueryDetector {
/**
* Called when the detection fails due to an error.
*/
//TODO(b/265390855): Replace this callback with the new onError(DetectorError) design.
void onError();
void onFailure(@NonNull DetectorFailure detectorFailure);
}
private class VisualQueryDetectorInitializationDelegate extends AbstractDetector {
@@ -294,12 +293,11 @@ public class VisualQueryDetector {
/** Called when the detection fails due to an error. */
@Override
public void onError() {
Slog.v(TAG, "BinderCallback#onError");
public void onDetectionFailure(DetectorFailure detectorFailure) {
Slog.v(TAG, "BinderCallback#onDetectionFailure");
Binder.withCleanCallingIdentity(() -> mExecutor.execute(
() -> mCallback.onError()));
() -> mCallback.onFailure(detectorFailure)));
}
}
@@ -373,5 +371,9 @@ public class VisualQueryDetector {
Slog.v(TAG, "Initialization Error: (" + status + ")");
// Do nothing
}
@Override
public void onDetectionFailure(DetectorFailure detectorFailure) throws RemoteException {
}
}
}

8
core/java/com/android/internal/app/IHotwordRecognitionStatusCallback.aidl
View File

@@ -17,6 +17,7 @@
package com.android.internal.app;
import android.hardware.soundtrigger.SoundTrigger;
import android.service.voice.DetectorFailure;
import android.service.voice.HotwordDetectedResult;
import android.service.voice.HotwordRejectedResult;
@@ -61,6 +62,13 @@ oneway interface IHotwordRecognitionStatusCallback {
*/
void onError(int status);
/**
* Called when the detection fails due to an error.
*
* @param detectorFailure It provides the error code, error message and suggested action.
*/
void onDetectionFailure(in DetectorFailure detectorFailure);
/**
* Called when the recognition is paused temporarily for some reason.
*/

34
services/voiceinteraction/java/com/android/server/voiceinteraction/DetectorSession.java
View File

@@ -70,6 +70,7 @@ import android.os.RemoteException;
import android.os.SharedMemory;
import android.service.voice.HotwordDetectedResult;
import android.service.voice.HotwordDetectionService;
import android.service.voice.HotwordDetectionServiceFailure;
import android.service.voice.HotwordDetector;
import android.service.voice.HotwordRejectedResult;
import android.service.voice.IDspHotwordDetectionCallback;
@@ -122,10 +123,16 @@ abstract class DetectorSession {
"Providing hotword detection result to VoiceInteractionService";
// The error codes are used for onError callback
static final int HOTWORD_DETECTION_SERVICE_DIED = -1;
static final int CALLBACK_ONDETECTED_GOT_SECURITY_EXCEPTION = -2;
static final int CALLBACK_DETECT_TIMEOUT = -3;
static final int CALLBACK_ONDETECTED_STREAM_COPY_ERROR = -4;
static final int HOTWORD_DETECTION_SERVICE_DIED =
HotwordDetectionServiceFailure.ERROR_CODE_BINDING_DIED;
static final int CALLBACK_ONDETECTED_GOT_SECURITY_EXCEPTION =
HotwordDetectionServiceFailure.ERROR_CODE_ON_DETECTED_SECURITY_EXCEPTION;
static final int CALLBACK_DETECT_TIMEOUT =
HotwordDetectionServiceFailure.ERROR_CODE_DETECT_TIMEOUT;
static final int CALLBACK_ONDETECTED_STREAM_COPY_ERROR =
HotwordDetectionServiceFailure.ERROR_CODE_ON_DETECTED_STREAM_COPY_FAILURE;
static final int CALLBACK_COPY_AUDIO_DATA_FAILURE =
HotwordDetectionServiceFailure.ERROR_CODE_COPY_AUDIO_DATA_FAILURE;
// TODO: These constants need to be refined.
private static final long MAX_UPDATE_TIMEOUT_MILLIS = 30000;
@@ -426,7 +433,9 @@ abstract class DetectorSession {
Slog.w(TAG, "Failed supplying audio data to validator", e);
try {
callback.onError();
callback.onError(
new HotwordDetectionServiceFailure(CALLBACK_COPY_AUDIO_DATA_FAILURE,
"Copy audio data failure for external source detection."));
} catch (RemoteException ex) {
Slog.w(TAG, "Failed to report onError status: " + ex);
if (getDetectorType() != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) {
@@ -505,7 +514,10 @@ abstract class DetectorSession {
getDetectorType(),
EXTERNAL_SOURCE_DETECT_SECURITY_EXCEPTION,
mVoiceInteractionServiceUid);
callback.onError();
callback.onError(new HotwordDetectionServiceFailure(
CALLBACK_ONDETECTED_GOT_SECURITY_EXCEPTION,
"Security exception occurs in #onDetected"
+ " method."));
return;
}
HotwordDetectedResult newResult;
@@ -514,7 +526,9 @@ abstract class DetectorSession {
.startCopyingAudioStreams(triggerResult);
} catch (IOException e) {
// TODO: Write event
callback.onError();
callback.onError(new HotwordDetectionServiceFailure(
CALLBACK_ONDETECTED_STREAM_COPY_ERROR,
"Copy audio stream failure."));
return;
}
callback.onDetected(newResult, /* audioFormat= */ null,
@@ -569,9 +583,11 @@ abstract class DetectorSession {
mRemoteDetectionService = remoteDetectionService;
}
void reportErrorLocked(int status) {
void reportErrorLocked(int errorCode, @NonNull String errorMessage) {
try {
mCallback.onError(status);
// TODO: Use instanceof(this) to get different detector to set the right error source.
mCallback.onDetectionFailure(
new HotwordDetectionServiceFailure(errorCode, errorMessage));
} catch (RemoteException e) {
Slog.w(TAG, "Failed to report onError status: " + e);
if (getDetectorType() != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) {

13
services/voiceinteraction/java/com/android/server/voiceinteraction/DspTrustedHotwordDetectorSession.java
View File

@@ -34,6 +34,7 @@ import android.os.RemoteException;
import android.os.SharedMemory;
import android.service.voice.HotwordDetectedResult;
import android.service.voice.HotwordDetectionService;
import android.service.voice.HotwordDetectionServiceFailure;
import android.service.voice.HotwordDetector;
import android.service.voice.HotwordRejectedResult;
import android.service.voice.IDspHotwordDetectionCallback;
@@ -130,7 +131,9 @@ final class DspTrustedHotwordDetectorSession extends DetectorSession {
HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP,
METRICS_KEYPHRASE_TRIGGERED_DETECT_SECURITY_EXCEPTION,
mVoiceInteractionServiceUid);
externalCallback.onError(CALLBACK_ONDETECTED_GOT_SECURITY_EXCEPTION);
externalCallback.onDetectionFailure(new HotwordDetectionServiceFailure(
CALLBACK_ONDETECTED_GOT_SECURITY_EXCEPTION,
"Security exception occurs in #onDetected method."));
return;
}
saveProximityValueToBundle(result);
@@ -138,7 +141,9 @@ final class DspTrustedHotwordDetectorSession extends DetectorSession {
try {
newResult = mHotwordAudioStreamCopier.startCopyingAudioStreams(result);
} catch (IOException e) {
externalCallback.onError(CALLBACK_ONDETECTED_STREAM_COPY_ERROR);
externalCallback.onDetectionFailure(new HotwordDetectionServiceFailure(
CALLBACK_ONDETECTED_STREAM_COPY_ERROR,
"Copy audio stream failure."));
return;
}
externalCallback.onKeyphraseDetected(recognitionEvent, newResult);
@@ -201,7 +206,9 @@ final class DspTrustedHotwordDetectorSession extends DetectorSession {
HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_TIMEOUT,
mVoiceInteractionServiceUid);
try {
externalCallback.onError(CALLBACK_DETECT_TIMEOUT);
externalCallback.onDetectionFailure(
new HotwordDetectionServiceFailure(CALLBACK_DETECT_TIMEOUT,
"Timeout to response to the detection result."));
} catch (RemoteException e) {
Slog.w(TAG, "Failed to report onError status: ", e);
HotwordMetricsLogger.writeDetectorEvent(

4
services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java
View File

@@ -690,8 +690,8 @@ final class HotwordDetectionConnection {
//TODO(b265535257): report error to either service only.
synchronized (HotwordDetectionConnection.this.mLock) {
runForEachDetectorSessionLocked((session) -> {
session.reportErrorLocked(
DetectorSession.HOTWORD_DETECTION_SERVICE_DIED);
session.reportErrorLocked(DetectorSession.HOTWORD_DETECTION_SERVICE_DIED,
"Detection service is dead.");
});
}
// Can improve to log exit reason if needed

9
services/voiceinteraction/java/com/android/server/voiceinteraction/SoftwareTrustedHotwordDetectorSession.java
View File

@@ -33,6 +33,7 @@ import android.os.RemoteException;
import android.os.SharedMemory;
import android.service.voice.HotwordDetectedResult;
import android.service.voice.HotwordDetectionService;
import android.service.voice.HotwordDetectionServiceFailure;
import android.service.voice.HotwordDetector;
import android.service.voice.HotwordRejectedResult;
import android.service.voice.IDspHotwordDetectionCallback;
@@ -121,7 +122,9 @@ final class SoftwareTrustedHotwordDetectorSession extends DetectorSession {
HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE,
METRICS_KEYPHRASE_TRIGGERED_DETECT_SECURITY_EXCEPTION,
mVoiceInteractionServiceUid);
mSoftwareCallback.onError();
mSoftwareCallback.onError(new HotwordDetectionServiceFailure(
CALLBACK_ONDETECTED_GOT_SECURITY_EXCEPTION,
"Security exception occurs in #onDetected method."));
return;
}
saveProximityValueToBundle(result);
@@ -130,7 +133,9 @@ final class SoftwareTrustedHotwordDetectorSession extends DetectorSession {
newResult = mHotwordAudioStreamCopier.startCopyingAudioStreams(result);
} catch (IOException e) {
// TODO: Write event
mSoftwareCallback.onError();
mSoftwareCallback.onError(new HotwordDetectionServiceFailure(
CALLBACK_ONDETECTED_STREAM_COPY_ERROR,
"Copy audio stream failure."));
return;
}
mSoftwareCallback.onDetected(newResult, null, null);

27
services/voiceinteraction/java/com/android/server/voiceinteraction/VisualQueryDetectorSession.java
View File

@@ -16,6 +16,9 @@
package com.android.server.voiceinteraction;
import static android.service.voice.VisualQueryDetectionServiceFailure.ERROR_CODE_ILLEGAL_ATTENTION_STATE;
import static android.service.voice.VisualQueryDetectionServiceFailure.ERROR_CODE_ILLEGAL_STREAMING_STATE;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.content.Context;
@@ -30,6 +33,7 @@ import android.service.voice.IDetectorSessionVisualQueryDetectionCallback;
import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback;
import android.service.voice.ISandboxedDetectionService;
import android.service.voice.IVisualQueryDetectionVoiceInteractionCallback;
import android.service.voice.VisualQueryDetectionServiceFailure;
import android.util.Slog;
import com.android.internal.app.IHotwordRecognitionStatusCallback;
@@ -102,6 +106,13 @@ final class VisualQueryDetectorSession extends DetectorSession {
mAttentionListener.onAttentionGained();
} catch (RemoteException e) {
Slog.e(TAG, "Error delivering attention gained event.", e);
try {
callback.onDetectionFailure(new VisualQueryDetectionServiceFailure(
ERROR_CODE_ILLEGAL_ATTENTION_STATE,
"Attention listener failed to switch to GAINED state."));
} catch (RemoteException ex) {
Slog.v(TAG, "Fail to call onDetectionFailure");
}
return;
}
}
@@ -117,6 +128,13 @@ final class VisualQueryDetectorSession extends DetectorSession {
mAttentionListener.onAttentionLost();
} catch (RemoteException e) {
Slog.e(TAG, "Error delivering attention lost event.", e);
try {
callback.onDetectionFailure(new VisualQueryDetectionServiceFailure(
ERROR_CODE_ILLEGAL_ATTENTION_STATE,
"Attention listener failed to switch to LOST state."));
} catch (RemoteException ex) {
Slog.v(TAG, "Fail to call onDetectionFailure");
}
return;
}
}
@@ -127,6 +145,9 @@ final class VisualQueryDetectorSession extends DetectorSession {
Slog.v(TAG, "BinderCallback#onQueryDetected");
if (!mEgressingData) {
Slog.v(TAG, "Query should not be egressed within the unattention state.");
callback.onDetectionFailure(new VisualQueryDetectionServiceFailure(
ERROR_CODE_ILLEGAL_STREAMING_STATE,
"Cannot stream queries without attention signals."));
return;
}
mQueryStreaming = true;
@@ -140,6 +161,9 @@ final class VisualQueryDetectorSession extends DetectorSession {
if (!mQueryStreaming) {
Slog.v(TAG, "Query streaming state signal FINISHED is block since there is"
+ " no active query being streamed.");
callback.onDetectionFailure(new VisualQueryDetectionServiceFailure(
ERROR_CODE_ILLEGAL_STREAMING_STATE,
"Cannot send FINISHED signal with no query streamed."));
return;
}
callback.onQueryFinished();
@@ -152,6 +176,9 @@ final class VisualQueryDetectorSession extends DetectorSession {
if (!mQueryStreaming) {
Slog.v(TAG, "Query streaming state signal REJECTED is block since there is"
+ " no active query being streamed.");
callback.onDetectionFailure(new VisualQueryDetectionServiceFailure(
ERROR_CODE_ILLEGAL_STREAMING_STATE,
"Cannot send REJECTED signal with no query streamed."));
return;
}
callback.onQueryRejected();