From 2f7b2d5f9424e0bf5f5f11cf5b24409393d4248c Mon Sep 17 00:00:00 2001 From: Nicholas Ambur Date: Mon, 7 Feb 2022 19:23:49 -0800 Subject: [PATCH] expose HAL keyphrase trigger extras to EventPayload The SoundTrigger HAL can deliver additional context about the kephrase that was triggered along with the trigger audio. This change moves to expose the information which was previously suppressed by the Android framework. Data exposed: - Keyphrase extras providing details on what phrase was interpreted by the voice engine Extra keyphrase information provided by the SoundTrigger HAL which was previously suppressed is also exposed. Test: atest AlwaysOnHotwordDetectorEventPayloadTest CTS-Coverage-Bug: 215375531 Bug: 215066299 Change-Id: I3b20fd3692c3f9c86181f87d66622c77778b9ab2 --- core/api/system-current.txt | 10 + core/api/test-current.txt | 17 +- .../hardware/soundtrigger/SoundTrigger.java | 117 ++++++++-- .../voice/AbstractHotwordDetector.java | 5 +- .../voice/AlwaysOnHotwordDetector.java | 212 +++++++++++++++--- .../voice/HotwordDetectionService.java | 4 +- .../voice/SoftwareHotwordDetector.java | 7 +- 7 files changed, 308 insertions(+), 64 deletions(-) diff --git a/core/api/system-current.txt b/core/api/system-current.txt index 4b3d0a4d23751..452cbe0b8e690 100644 --- a/core/api/system-current.txt +++ b/core/api/system-current.txt @@ -5172,6 +5172,15 @@ package android.hardware.soundtrigger { field @NonNull public static final android.os.Parcelable.Creator CREATOR; } + public static final class SoundTrigger.KeyphraseRecognitionExtra implements android.os.Parcelable { + method public int describeContents(); + method public int getCoarseConfidenceLevel(); + method public int getKeyphraseId(); + method public int getRecognitionModes(); + method public void writeToParcel(@NonNull android.os.Parcel, int); + field @NonNull public static final android.os.Parcelable.Creator CREATOR; + } + public static final class SoundTrigger.KeyphraseSoundModel extends android.hardware.soundtrigger.SoundTrigger.SoundModel implements android.os.Parcelable { ctor public SoundTrigger.KeyphraseSoundModel(@NonNull java.util.UUID, @NonNull java.util.UUID, @Nullable byte[], @Nullable android.hardware.soundtrigger.SoundTrigger.Keyphrase[], int); ctor public SoundTrigger.KeyphraseSoundModel(@NonNull java.util.UUID, @NonNull java.util.UUID, @Nullable byte[], @Nullable android.hardware.soundtrigger.SoundTrigger.Keyphrase[]); @@ -11758,6 +11767,7 @@ package android.service.voice { method @Nullable public byte[] getData(); method public int getDataFormat(); method @Nullable public android.service.voice.HotwordDetectedResult getHotwordDetectedResult(); + method @NonNull public java.util.List getKeyphraseRecognitionExtras(); method @Deprecated @Nullable public byte[] getTriggerAudio(); field public static final int DATA_FORMAT_RAW = 0; // 0x0 field public static final int DATA_FORMAT_TRIGGER_AUDIO = 1; // 0x1 diff --git a/core/api/test-current.txt b/core/api/test-current.txt index d85f20582b496..23f4fe03c2140 100644 --- a/core/api/test-current.txt +++ b/core/api/test-current.txt @@ -1276,6 +1276,10 @@ package android.hardware.soundtrigger { field @NonNull public static final android.os.Parcelable.Creator CREATOR; } + public static final class SoundTrigger.KeyphraseRecognitionExtra implements android.os.Parcelable { + ctor public SoundTrigger.KeyphraseRecognitionExtra(int, int, int); + } + public static final class SoundTrigger.ModelParamRange implements android.os.Parcelable { ctor public SoundTrigger.ModelParamRange(int, int); } @@ -2398,8 +2402,17 @@ package android.service.voice { method @RequiresPermission(allOf={android.Manifest.permission.RECORD_AUDIO, android.Manifest.permission.CAPTURE_AUDIO_HOTWORD}) public void triggerHardwareRecognitionEventForTest(int, int, boolean, int, int, int, boolean, @NonNull android.media.AudioFormat, @Nullable byte[]); } - public static class AlwaysOnHotwordDetector.EventPayload { - ctor public AlwaysOnHotwordDetector.EventPayload(boolean, boolean, @Nullable android.media.AudioFormat, int, @Nullable byte[], @Nullable android.service.voice.HotwordDetectedResult, @Nullable android.os.ParcelFileDescriptor); + public static final class AlwaysOnHotwordDetector.EventPayload.Builder { + ctor public AlwaysOnHotwordDetector.EventPayload.Builder(); + method @NonNull public android.service.voice.AlwaysOnHotwordDetector.EventPayload build(); + method @NonNull public android.service.voice.AlwaysOnHotwordDetector.EventPayload.Builder setAudioStream(@NonNull android.os.ParcelFileDescriptor); + method @NonNull public android.service.voice.AlwaysOnHotwordDetector.EventPayload.Builder setCaptureAudioFormat(@NonNull android.media.AudioFormat); + method @NonNull public android.service.voice.AlwaysOnHotwordDetector.EventPayload.Builder setCaptureAvailable(boolean); + method @NonNull public android.service.voice.AlwaysOnHotwordDetector.EventPayload.Builder setCaptureSession(int); + method @NonNull public android.service.voice.AlwaysOnHotwordDetector.EventPayload.Builder setData(@NonNull byte[]); + method @NonNull public android.service.voice.AlwaysOnHotwordDetector.EventPayload.Builder setDataFormat(int); + method @NonNull public android.service.voice.AlwaysOnHotwordDetector.EventPayload.Builder setHotwordDetectedResult(@NonNull android.service.voice.HotwordDetectedResult); + method @NonNull public android.service.voice.AlwaysOnHotwordDetector.EventPayload.Builder setKeyphraseRecognitionExtras(@NonNull java.util.List); } public final class VisibleActivityInfo implements android.os.Parcelable { diff --git a/core/java/android/hardware/soundtrigger/SoundTrigger.java b/core/java/android/hardware/soundtrigger/SoundTrigger.java index b0439d0e52536..c36390917cf18 100644 --- a/core/java/android/hardware/soundtrigger/SoundTrigger.java +++ b/core/java/android/hardware/soundtrigger/SoundTrigger.java @@ -61,6 +61,7 @@ import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Locale; import java.util.UUID; @@ -1576,31 +1577,58 @@ public class SoundTrigger { } /** - * Additional data conveyed by a {@link KeyphraseRecognitionEvent} - * for a key phrase detection. - * - * @hide + * Additional data conveyed by a {@link KeyphraseRecognitionEvent} + * for a key phrase detection. */ - public static class KeyphraseRecognitionExtra implements Parcelable { - /** The keyphrase ID */ + public static final class KeyphraseRecognitionExtra implements Parcelable { + /** + * The keyphrase ID + * + * @hide + */ @UnsupportedAppUsage public final int id; - /** Recognition modes matched for this event */ + /** + * Recognition modes matched for this event + * + * @hide + */ @UnsupportedAppUsage(maxTargetSdk = Build.VERSION_CODES.R, trackingBug = 170729553) public final int recognitionModes; - /** Confidence level for mode RECOGNITION_MODE_VOICE_TRIGGER when user identification - * is not performed */ + /** + * Confidence level for mode RECOGNITION_MODE_VOICE_TRIGGER when user identification + * is not performed + * + * @hide + */ @UnsupportedAppUsage public final int coarseConfidenceLevel; - /** Confidence levels for all users recognized (KeyphraseRecognitionEvent) or to - * be recognized (RecognitionConfig) */ + /** + * Confidence levels for all users recognized (KeyphraseRecognitionEvent) or to + * be recognized (RecognitionConfig) + * + * @hide + */ @UnsupportedAppUsage @NonNull public final ConfidenceLevel[] confidenceLevels; + + /** + * @hide + */ + @TestApi + public KeyphraseRecognitionExtra(int id, @RecognitionModes int recognitionModes, + int coarseConfidenceLevel) { + this(id, recognitionModes, coarseConfidenceLevel, new ConfidenceLevel[0]); + } + + /** + * @hide + */ @UnsupportedAppUsage public KeyphraseRecognitionExtra(int id, int recognitionModes, int coarseConfidenceLevel, @Nullable ConfidenceLevel[] confidenceLevels) { @@ -1611,7 +1639,47 @@ public class SoundTrigger { confidenceLevels != null ? confidenceLevels : new ConfidenceLevel[0]; } - public static final @android.annotation.NonNull Parcelable.Creator CREATOR + /** + * The keyphrase ID associated with this class' additional data + */ + public int getKeyphraseId() { + return id; + } + + /** + * Recognition modes matched for this event + */ + @RecognitionModes + public int getRecognitionModes() { + return recognitionModes; + } + + /** + * Confidence level for mode RECOGNITION_MODE_VOICE_TRIGGER when user identification + * is not performed + * + *

The confidence level is expressed in percent (0% -100%). + */ + public int getCoarseConfidenceLevel() { + return coarseConfidenceLevel; + } + + /** + * Detected confidence level for users defined in a keyphrase. + * + *

The confidence level is expressed in percent (0% -100%). + * + *

The user ID is derived from the system ID + * {@link android.os.UserHandle#getIdentifier()}. + * + * @hide + */ + @NonNull + public Collection getConfidenceLevels() { + return Arrays.asList(confidenceLevels); + } + + public static final @NonNull Parcelable.Creator CREATOR = new Parcelable.Creator() { public KeyphraseRecognitionExtra createFromParcel(Parcel in) { return KeyphraseRecognitionExtra.fromParcel(in); @@ -1632,7 +1700,7 @@ public class SoundTrigger { } @Override - public void writeToParcel(Parcel dest, int flags) { + public void writeToParcel(@NonNull Parcel dest, int flags) { dest.writeInt(id); dest.writeInt(recognitionModes); dest.writeInt(coarseConfidenceLevel); @@ -1657,21 +1725,28 @@ public class SoundTrigger { @Override public boolean equals(@Nullable Object obj) { - if (this == obj) + if (this == obj) { return true; - if (obj == null) + } + if (obj == null) { return false; - if (getClass() != obj.getClass()) + } + if (getClass() != obj.getClass()) { return false; + } KeyphraseRecognitionExtra other = (KeyphraseRecognitionExtra) obj; - if (!Arrays.equals(confidenceLevels, other.confidenceLevels)) + if (!Arrays.equals(confidenceLevels, other.confidenceLevels)) { return false; - if (id != other.id) + } + if (id != other.id) { return false; - if (recognitionModes != other.recognitionModes) + } + if (recognitionModes != other.recognitionModes) { return false; - if (coarseConfidenceLevel != other.coarseConfidenceLevel) + } + if (coarseConfidenceLevel != other.coarseConfidenceLevel) { return false; + } return true; } @@ -1715,7 +1790,7 @@ public class SoundTrigger { keyphraseExtras != null ? keyphraseExtras : new KeyphraseRecognitionExtra[0]; } - public static final @android.annotation.NonNull Parcelable.Creator CREATOR + public static final @NonNull Parcelable.Creator CREATOR = new Parcelable.Creator() { public KeyphraseRecognitionEvent createFromParcel(Parcel in) { return KeyphraseRecognitionEvent.fromParcelForKeyphrase(in); diff --git a/core/java/android/service/voice/AbstractHotwordDetector.java b/core/java/android/service/voice/AbstractHotwordDetector.java index 192260791a8b3..c3bf5694dbffc 100644 --- a/core/java/android/service/voice/AbstractHotwordDetector.java +++ b/core/java/android/service/voice/AbstractHotwordDetector.java @@ -146,7 +146,10 @@ abstract class AbstractHotwordDetector implements HotwordDetector { mHandler.sendMessage(obtainMessage( HotwordDetector.Callback::onDetected, mCallback, - new AlwaysOnHotwordDetector.EventPayload(audioFormat, hotwordDetectedResult))); + new AlwaysOnHotwordDetector.EventPayload.Builder() + .setCaptureAudioFormat(audioFormat) + .setHotwordDetectedResult(hotwordDetectedResult) + .build())); } } } diff --git a/core/java/android/service/voice/AlwaysOnHotwordDetector.java b/core/java/android/service/voice/AlwaysOnHotwordDetector.java index 69f771bbac4a7..b90f27b986c11 100644 --- a/core/java/android/service/voice/AlwaysOnHotwordDetector.java +++ b/core/java/android/service/voice/AlwaysOnHotwordDetector.java @@ -23,6 +23,7 @@ import android.annotation.IntDef; import android.annotation.NonNull; import android.annotation.Nullable; import android.annotation.RequiresPermission; +import android.annotation.SuppressLint; import android.annotation.SystemApi; import android.annotation.TestApi; import android.app.ActivityThread; @@ -59,6 +60,9 @@ import com.android.internal.app.IVoiceInteractionSoundTriggerSession; import java.io.PrintWriter; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; import java.util.Locale; /** @@ -380,47 +384,24 @@ public class AlwaysOnHotwordDetector extends AbstractHotwordDetector { private final byte[] mData; private final HotwordDetectedResult mHotwordDetectedResult; private final ParcelFileDescriptor mAudioStream; + private final List mKephraseExtras; - EventPayload(boolean triggerInData, boolean captureAvailable, - AudioFormat audioFormat, int captureSession, byte[] data) { - this(triggerInData, captureAvailable, audioFormat, captureSession, data, null, - null); - } - - EventPayload(boolean triggerInData, boolean captureAvailable, - AudioFormat audioFormat, int captureSession, byte[] data, - HotwordDetectedResult hotwordDetectedResult) { - this(triggerInData, captureAvailable, audioFormat, captureSession, data, - hotwordDetectedResult, null); - } - - EventPayload(AudioFormat audioFormat, HotwordDetectedResult hotwordDetectedResult) { - this(false, false, audioFormat, -1, null, hotwordDetectedResult, null); - } - - EventPayload(AudioFormat audioFormat, - HotwordDetectedResult hotwordDetectedResult, - ParcelFileDescriptor audioStream) { - this(false, false, audioFormat, -1, null, hotwordDetectedResult, audioStream); - } - - /** @hide */ - @TestApi - public EventPayload(boolean triggerInData, boolean captureAvailable, - @Nullable AudioFormat audioFormat, int captureSession, @Nullable byte[] data, + private EventPayload(boolean captureAvailable, + @Nullable AudioFormat audioFormat, + int captureSession, + @DataFormat int dataFormat, + @Nullable byte[] data, @Nullable HotwordDetectedResult hotwordDetectedResult, - @Nullable ParcelFileDescriptor audioStream) { + @Nullable ParcelFileDescriptor audioStream, + @NonNull List keyphraseExtras) { mCaptureAvailable = captureAvailable; mCaptureSession = captureSession; mAudioFormat = audioFormat; - if (triggerInData) { - mDataFormat = DATA_FORMAT_TRIGGER_AUDIO; - } else { - mDataFormat = DATA_FORMAT_RAW; - } + mDataFormat = dataFormat; mData = data; mHotwordDetectedResult = hotwordDetectedResult; mAudioStream = audioStream; + mKephraseExtras = keyphraseExtras; } /** @@ -535,6 +516,166 @@ public class AlwaysOnHotwordDetector extends AbstractHotwordDetector { public ParcelFileDescriptor getAudioStream() { return mAudioStream; } + + /** + * Returns the keyphrases recognized by the voice engine with additional confidence + * information + * + * @return List of keyphrase extras describing additional data for each keyphrase the voice + * engine triggered on for this event. The ordering of the list is preserved based on what + * the ordering provided by {@link android.hardware.soundtrigger.SoundTriggerModule}. + */ + @NonNull + public List getKeyphraseRecognitionExtras() { + return mKephraseExtras; + } + + /** + * Builder class for {@link EventPayload} objects + * + * @hide + */ + @TestApi + public static final class Builder { + private boolean mCaptureAvailable = false; + private int mCaptureSession = -1; + private AudioFormat mAudioFormat = null; + @DataFormat + private int mDataFormat = DATA_FORMAT_RAW; + private byte[] mData = null; + private HotwordDetectedResult mHotwordDetectedResult = null; + private ParcelFileDescriptor mAudioStream = null; + private List mKeyphraseExtras = Collections.emptyList(); + + public Builder() {} + + Builder(SoundTrigger.KeyphraseRecognitionEvent keyphraseRecognitionEvent) { + setCaptureAvailable(keyphraseRecognitionEvent.isCaptureAvailable()); + setCaptureSession(keyphraseRecognitionEvent.getCaptureSession()); + if (keyphraseRecognitionEvent.getCaptureFormat() != null) { + setCaptureAudioFormat(keyphraseRecognitionEvent.getCaptureFormat()); + } + setDataFormat((keyphraseRecognitionEvent.triggerInData) ? DATA_FORMAT_TRIGGER_AUDIO + : DATA_FORMAT_RAW); + if (keyphraseRecognitionEvent.getData() != null) { + setData(keyphraseRecognitionEvent.getData()); + } + if (keyphraseRecognitionEvent.keyphraseExtras != null) { + setKeyphraseRecognitionExtras( + Arrays.asList(keyphraseRecognitionEvent.keyphraseExtras)); + } + } + + /** + * Indicates if {@code captureSession} can be used to continue capturing more audio from + * the DSP hardware. + */ + @SuppressLint("MissingGetterMatchingBuilder") + @NonNull + public Builder setCaptureAvailable(boolean captureAvailable) { + mCaptureAvailable = captureAvailable; + return this; + } + + /** + * Sets the session ID to start a capture from the DSP. + */ + @SuppressLint("MissingGetterMatchingBuilder") + @NonNull + public Builder setCaptureSession(int captureSession) { + mCaptureSession = captureSession; + return this; + } + + /** + * Sets the format of the audio obtained using {@link #getTriggerAudio()}. + */ + @NonNull + public Builder setCaptureAudioFormat(@NonNull AudioFormat audioFormat) { + mAudioFormat = audioFormat; + return this; + } + + /** + * Conveys the format of the additional data that is triggered with the keyphrase event. + * + * @see AlwaysOnHotwordDetector#RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO + * @see DataFormat + */ + @NonNull + public Builder setDataFormat(@DataFormat int dataFormat) { + mDataFormat = dataFormat; + return this; + } + + /** + * Sets additional raw data that is triggered with the keyphrase event. + * + *

A {@link android.hardware.soundtrigger.SoundTriggerModule} may populate this + * field with opaque data for use by system applications who know about voice + * engine internals. Data may be null if the field is not populated by the + * {@link android.hardware.soundtrigger.SoundTriggerModule}. + * + *

If {@link #getDataFormat()} is {@link #DATA_FORMAT_TRIGGER_AUDIO}, then the + * entirety of this + * buffer is expected to be of the format from {@link #getCaptureAudioFormat()}. + * + * @see AlwaysOnHotwordDetector#RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO + */ + @NonNull + public Builder setData(@NonNull byte[] data) { + mData = data; + return this; + } + + /** + * Sets {@link HotwordDetectedResult} associated with the hotword event, passed from + * {@link HotwordDetectionService}. + */ + @NonNull + public Builder setHotwordDetectedResult( + @NonNull HotwordDetectedResult hotwordDetectedResult) { + mHotwordDetectedResult = hotwordDetectedResult; + return this; + } + + /** + * Sets a stream with bytes corresponding to the open audio stream with hotword data. + * + *

This data represents an audio stream in the format returned by + * {@link #getCaptureAudioFormat}. + * + *

Clients are expected to start consuming the stream within 1 second of receiving + * the + * event. + */ + @NonNull + public Builder setAudioStream(@NonNull ParcelFileDescriptor audioStream) { + mAudioStream = audioStream; + return this; + } + + /** + * Sets the keyphrases recognized by the voice engine with additional confidence + * information + */ + @NonNull + public Builder setKeyphraseRecognitionExtras( + @NonNull List keyphraseRecognitionExtras) { + mKeyphraseExtras = keyphraseRecognitionExtras; + return this; + } + + /** + * Builds an {@link EventPayload} instance + */ + @NonNull + public EventPayload build() { + return new EventPayload(mCaptureAvailable, mAudioFormat, mCaptureSession, + mDataFormat, mData, mHotwordDetectedResult, mAudioStream, + mKeyphraseExtras); + } + } } /** @@ -1242,8 +1383,9 @@ public class AlwaysOnHotwordDetector extends AbstractHotwordDetector { Slog.i(TAG, "onDetected"); } Message.obtain(mHandler, MSG_HOTWORD_DETECTED, - new EventPayload(event.triggerInData, event.captureAvailable, - event.captureFormat, event.captureSession, event.data, result)) + new EventPayload.Builder(event) + .setHotwordDetectedResult(result) + .build()) .sendToTarget(); } @Override diff --git a/core/java/android/service/voice/HotwordDetectionService.java b/core/java/android/service/voice/HotwordDetectionService.java index e3bb589c9a19d..dfe0f542b3cab 100644 --- a/core/java/android/service/voice/HotwordDetectionService.java +++ b/core/java/android/service/voice/HotwordDetectionService.java @@ -140,9 +140,7 @@ public abstract class HotwordDetectionService extends Service { Log.d(TAG, "#detectFromDspSource"); } HotwordDetectionService.this.onDetect( - new AlwaysOnHotwordDetector.EventPayload( - event.triggerInData, event.captureAvailable, - event.captureFormat, event.captureSession, event.data), + new AlwaysOnHotwordDetector.EventPayload.Builder(event).build(), timeoutMillis, new Callback(callback)); } diff --git a/core/java/android/service/voice/SoftwareHotwordDetector.java b/core/java/android/service/voice/SoftwareHotwordDetector.java index 512a654adbab7..426f312b2e86a 100644 --- a/core/java/android/service/voice/SoftwareHotwordDetector.java +++ b/core/java/android/service/voice/SoftwareHotwordDetector.java @@ -135,8 +135,11 @@ class SoftwareHotwordDetector extends AbstractHotwordDetector { mHandler.sendMessage(obtainMessage( HotwordDetector.Callback::onDetected, mCallback, - new AlwaysOnHotwordDetector.EventPayload( - audioFormat, hotwordDetectedResult, audioStream))); + new AlwaysOnHotwordDetector.EventPayload.Builder() + .setCaptureAudioFormat(audioFormat) + .setAudioStream(audioStream) + .setHotwordDetectedResult(hotwordDetectedResult) + .build())); } }