From 6b6d178bd6ad0eb088154c48fcf289fa53dbc021 Mon Sep 17 00:00:00 2001 From: Nicholas Ambur Date: Mon, 7 Feb 2022 19:23:18 -0800 Subject: [PATCH 1/3] expose HAL keyphrase trigger data to EventPayload The SoundTrigger HAL can deliver additional context about the kephrase that was triggered along with the trigger audio. This change moves to expose the information which was previously suppressed by the Android framework. Data exposed: - Raw data field Previously raw data was only exposed when the data was of a specific audio format. This change allows the raw data to be retrieved regardless of format. Test: atest AlwaysOnHotwordDetectorEventPayloadTest CTS-Coverage-Bug: 215375531 Bug: 215066299 Change-Id: Ie75773853d3b338e092dfbf3bd6ccef61a1420c9 --- core/api/system-current.txt | 6 +- core/api/test-current.txt | 4 + .../voice/AlwaysOnHotwordDetector.java | 91 +++++++++++++++++-- 3 files changed, 90 insertions(+), 11 deletions(-) diff --git a/core/api/system-current.txt b/core/api/system-current.txt index 321ebcf42ea56..4b3d0a4d23751 100644 --- a/core/api/system-current.txt +++ b/core/api/system-current.txt @@ -11755,8 +11755,12 @@ package android.service.voice { public static class AlwaysOnHotwordDetector.EventPayload { method @Nullable public android.os.ParcelFileDescriptor getAudioStream(); method @Nullable public android.media.AudioFormat getCaptureAudioFormat(); + method @Nullable public byte[] getData(); + method public int getDataFormat(); method @Nullable public android.service.voice.HotwordDetectedResult getHotwordDetectedResult(); - method @Nullable public byte[] getTriggerAudio(); + method @Deprecated @Nullable public byte[] getTriggerAudio(); + field public static final int DATA_FORMAT_RAW = 0; // 0x0 + field public static final int DATA_FORMAT_TRIGGER_AUDIO = 1; // 0x1 } public static final class AlwaysOnHotwordDetector.ModelParamRange { diff --git a/core/api/test-current.txt b/core/api/test-current.txt index ff4ec311349cb..d85f20582b496 100644 --- a/core/api/test-current.txt +++ b/core/api/test-current.txt @@ -2398,6 +2398,10 @@ package android.service.voice { method @RequiresPermission(allOf={android.Manifest.permission.RECORD_AUDIO, android.Manifest.permission.CAPTURE_AUDIO_HOTWORD}) public void triggerHardwareRecognitionEventForTest(int, int, boolean, int, int, int, boolean, @NonNull android.media.AudioFormat, @Nullable byte[]); } + public static class AlwaysOnHotwordDetector.EventPayload { + ctor public AlwaysOnHotwordDetector.EventPayload(boolean, boolean, @Nullable android.media.AudioFormat, int, @Nullable byte[], @Nullable android.service.voice.HotwordDetectedResult, @Nullable android.os.ParcelFileDescriptor); + } + public final class VisibleActivityInfo implements android.os.Parcelable { ctor public VisibleActivityInfo(int, @NonNull android.os.IBinder); } diff --git a/core/java/android/service/voice/AlwaysOnHotwordDetector.java b/core/java/android/service/voice/AlwaysOnHotwordDetector.java index c9daf52b56853..69f771bbac4a7 100644 --- a/core/java/android/service/voice/AlwaysOnHotwordDetector.java +++ b/core/java/android/service/voice/AlwaysOnHotwordDetector.java @@ -336,7 +336,39 @@ public class AlwaysOnHotwordDetector extends AbstractHotwordDetector { * Additional payload for {@link Callback#onDetected}. */ public static class EventPayload { - private final boolean mTriggerAvailable; + + /** + * Flags for describing the data format provided in the event payload. + * + * @hide + */ + @Retention(RetentionPolicy.SOURCE) + @IntDef(prefix = {"DATA_FORMAT_"}, value = { + DATA_FORMAT_RAW, + DATA_FORMAT_TRIGGER_AUDIO, + }) + public @interface DataFormat { + } + + /** + * Data format is not strictly defined by the framework, and the + * {@link android.hardware.soundtrigger.SoundTriggerModule} voice engine may populate this + * field in any format. + */ + public static final int DATA_FORMAT_RAW = 0; + + /** + * Data format is defined as trigger audio. + * + *

When this format is used, {@link #getCaptureAudioFormat()} can be used to understand + * further the audio format for reading the data. + * + * @see AlwaysOnHotwordDetector#RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO + */ + public static final int DATA_FORMAT_TRIGGER_AUDIO = 1; + + @DataFormat + private final int mDataFormat; // Indicates if {@code captureSession} can be used to continue capturing more audio // from the DSP hardware. private final boolean mCaptureAvailable; @@ -349,16 +381,16 @@ public class AlwaysOnHotwordDetector extends AbstractHotwordDetector { private final HotwordDetectedResult mHotwordDetectedResult; private final ParcelFileDescriptor mAudioStream; - EventPayload(boolean triggerAvailable, boolean captureAvailable, + EventPayload(boolean triggerInData, boolean captureAvailable, AudioFormat audioFormat, int captureSession, byte[] data) { - this(triggerAvailable, captureAvailable, audioFormat, captureSession, data, null, + this(triggerInData, captureAvailable, audioFormat, captureSession, data, null, null); } - EventPayload(boolean triggerAvailable, boolean captureAvailable, + EventPayload(boolean triggerInData, boolean captureAvailable, AudioFormat audioFormat, int captureSession, byte[] data, HotwordDetectedResult hotwordDetectedResult) { - this(triggerAvailable, captureAvailable, audioFormat, captureSession, data, + this(triggerInData, captureAvailable, audioFormat, captureSession, data, hotwordDetectedResult, null); } @@ -372,13 +404,20 @@ public class AlwaysOnHotwordDetector extends AbstractHotwordDetector { this(false, false, audioFormat, -1, null, hotwordDetectedResult, audioStream); } - private EventPayload(boolean triggerAvailable, boolean captureAvailable, - AudioFormat audioFormat, int captureSession, byte[] data, - HotwordDetectedResult hotwordDetectedResult, ParcelFileDescriptor audioStream) { - mTriggerAvailable = triggerAvailable; + /** @hide */ + @TestApi + public EventPayload(boolean triggerInData, boolean captureAvailable, + @Nullable AudioFormat audioFormat, int captureSession, @Nullable byte[] data, + @Nullable HotwordDetectedResult hotwordDetectedResult, + @Nullable ParcelFileDescriptor audioStream) { mCaptureAvailable = captureAvailable; mCaptureSession = captureSession; mAudioFormat = audioFormat; + if (triggerInData) { + mDataFormat = DATA_FORMAT_TRIGGER_AUDIO; + } else { + mDataFormat = DATA_FORMAT_RAW; + } mData = data; mHotwordDetectedResult = hotwordDetectedResult; mAudioStream = audioStream; @@ -400,16 +439,48 @@ public class AlwaysOnHotwordDetector extends AbstractHotwordDetector { * {@link #getCaptureAudioFormat()}. * * @see AlwaysOnHotwordDetector#RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO + * @deprecated Use {@link #getData()} instead. */ + @Deprecated @Nullable public byte[] getTriggerAudio() { - if (mTriggerAvailable) { + if (mDataFormat == DATA_FORMAT_TRIGGER_AUDIO) { return mData; } else { return null; } } + /** + * Conveys the format of the additional data that is triggered with the keyphrase event. + * + * @see AlwaysOnHotwordDetector#RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO + * @see DataFormat + */ + @DataFormat + public int getDataFormat() { + return mDataFormat; + } + + /** + * Gets additional raw data that is triggered with the keyphrase event. + * + *

A {@link android.hardware.soundtrigger.SoundTriggerModule} may populate this + * field with opaque data for use by system applications who know about voice + * engine internals. Data may be null if the field is not populated by the + * {@link android.hardware.soundtrigger.SoundTriggerModule}. + * + *

If {@link #getDataFormat()} is {@link #DATA_FORMAT_TRIGGER_AUDIO}, then the + * entirety of this buffer is expected to be of the format from + * {@link #getCaptureAudioFormat()}. + * + * @see AlwaysOnHotwordDetector#RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO + */ + @Nullable + public byte[] getData() { + return mData; + } + /** * Gets the session ID to start a capture from the DSP. * This may be null if streaming capture isn't possible. From 2f7b2d5f9424e0bf5f5f11cf5b24409393d4248c Mon Sep 17 00:00:00 2001 From: Nicholas Ambur Date: Mon, 7 Feb 2022 19:23:49 -0800 Subject: [PATCH 2/3] expose HAL keyphrase trigger extras to EventPayload The SoundTrigger HAL can deliver additional context about the kephrase that was triggered along with the trigger audio. This change moves to expose the information which was previously suppressed by the Android framework. Data exposed: - Keyphrase extras providing details on what phrase was interpreted by the voice engine Extra keyphrase information provided by the SoundTrigger HAL which was previously suppressed is also exposed. Test: atest AlwaysOnHotwordDetectorEventPayloadTest CTS-Coverage-Bug: 215375531 Bug: 215066299 Change-Id: I3b20fd3692c3f9c86181f87d66622c77778b9ab2 --- core/api/system-current.txt | 10 + core/api/test-current.txt | 17 +- .../hardware/soundtrigger/SoundTrigger.java | 117 ++++++++-- .../voice/AbstractHotwordDetector.java | 5 +- .../voice/AlwaysOnHotwordDetector.java | 212 +++++++++++++++--- .../voice/HotwordDetectionService.java | 4 +- .../voice/SoftwareHotwordDetector.java | 7 +- 7 files changed, 308 insertions(+), 64 deletions(-) diff --git a/core/api/system-current.txt b/core/api/system-current.txt index 4b3d0a4d23751..452cbe0b8e690 100644 --- a/core/api/system-current.txt +++ b/core/api/system-current.txt @@ -5172,6 +5172,15 @@ package android.hardware.soundtrigger { field @NonNull public static final android.os.Parcelable.Creator CREATOR; } + public static final class SoundTrigger.KeyphraseRecognitionExtra implements android.os.Parcelable { + method public int describeContents(); + method public int getCoarseConfidenceLevel(); + method public int getKeyphraseId(); + method public int getRecognitionModes(); + method public void writeToParcel(@NonNull android.os.Parcel, int); + field @NonNull public static final android.os.Parcelable.Creator CREATOR; + } + public static final class SoundTrigger.KeyphraseSoundModel extends android.hardware.soundtrigger.SoundTrigger.SoundModel implements android.os.Parcelable { ctor public SoundTrigger.KeyphraseSoundModel(@NonNull java.util.UUID, @NonNull java.util.UUID, @Nullable byte[], @Nullable android.hardware.soundtrigger.SoundTrigger.Keyphrase[], int); ctor public SoundTrigger.KeyphraseSoundModel(@NonNull java.util.UUID, @NonNull java.util.UUID, @Nullable byte[], @Nullable android.hardware.soundtrigger.SoundTrigger.Keyphrase[]); @@ -11758,6 +11767,7 @@ package android.service.voice { method @Nullable public byte[] getData(); method public int getDataFormat(); method @Nullable public android.service.voice.HotwordDetectedResult getHotwordDetectedResult(); + method @NonNull public java.util.List getKeyphraseRecognitionExtras(); method @Deprecated @Nullable public byte[] getTriggerAudio(); field public static final int DATA_FORMAT_RAW = 0; // 0x0 field public static final int DATA_FORMAT_TRIGGER_AUDIO = 1; // 0x1 diff --git a/core/api/test-current.txt b/core/api/test-current.txt index d85f20582b496..23f4fe03c2140 100644 --- a/core/api/test-current.txt +++ b/core/api/test-current.txt @@ -1276,6 +1276,10 @@ package android.hardware.soundtrigger { field @NonNull public static final android.os.Parcelable.Creator CREATOR; } + public static final class SoundTrigger.KeyphraseRecognitionExtra implements android.os.Parcelable { + ctor public SoundTrigger.KeyphraseRecognitionExtra(int, int, int); + } + public static final class SoundTrigger.ModelParamRange implements android.os.Parcelable { ctor public SoundTrigger.ModelParamRange(int, int); } @@ -2398,8 +2402,17 @@ package android.service.voice { method @RequiresPermission(allOf={android.Manifest.permission.RECORD_AUDIO, android.Manifest.permission.CAPTURE_AUDIO_HOTWORD}) public void triggerHardwareRecognitionEventForTest(int, int, boolean, int, int, int, boolean, @NonNull android.media.AudioFormat, @Nullable byte[]); } - public static class AlwaysOnHotwordDetector.EventPayload { - ctor public AlwaysOnHotwordDetector.EventPayload(boolean, boolean, @Nullable android.media.AudioFormat, int, @Nullable byte[], @Nullable android.service.voice.HotwordDetectedResult, @Nullable android.os.ParcelFileDescriptor); + public static final class AlwaysOnHotwordDetector.EventPayload.Builder { + ctor public AlwaysOnHotwordDetector.EventPayload.Builder(); + method @NonNull public android.service.voice.AlwaysOnHotwordDetector.EventPayload build(); + method @NonNull public android.service.voice.AlwaysOnHotwordDetector.EventPayload.Builder setAudioStream(@NonNull android.os.ParcelFileDescriptor); + method @NonNull public android.service.voice.AlwaysOnHotwordDetector.EventPayload.Builder setCaptureAudioFormat(@NonNull android.media.AudioFormat); + method @NonNull public android.service.voice.AlwaysOnHotwordDetector.EventPayload.Builder setCaptureAvailable(boolean); + method @NonNull public android.service.voice.AlwaysOnHotwordDetector.EventPayload.Builder setCaptureSession(int); + method @NonNull public android.service.voice.AlwaysOnHotwordDetector.EventPayload.Builder setData(@NonNull byte[]); + method @NonNull public android.service.voice.AlwaysOnHotwordDetector.EventPayload.Builder setDataFormat(int); + method @NonNull public android.service.voice.AlwaysOnHotwordDetector.EventPayload.Builder setHotwordDetectedResult(@NonNull android.service.voice.HotwordDetectedResult); + method @NonNull public android.service.voice.AlwaysOnHotwordDetector.EventPayload.Builder setKeyphraseRecognitionExtras(@NonNull java.util.List); } public final class VisibleActivityInfo implements android.os.Parcelable { diff --git a/core/java/android/hardware/soundtrigger/SoundTrigger.java b/core/java/android/hardware/soundtrigger/SoundTrigger.java index b0439d0e52536..c36390917cf18 100644 --- a/core/java/android/hardware/soundtrigger/SoundTrigger.java +++ b/core/java/android/hardware/soundtrigger/SoundTrigger.java @@ -61,6 +61,7 @@ import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Locale; import java.util.UUID; @@ -1576,31 +1577,58 @@ public class SoundTrigger { } /** - * Additional data conveyed by a {@link KeyphraseRecognitionEvent} - * for a key phrase detection. - * - * @hide + * Additional data conveyed by a {@link KeyphraseRecognitionEvent} + * for a key phrase detection. */ - public static class KeyphraseRecognitionExtra implements Parcelable { - /** The keyphrase ID */ + public static final class KeyphraseRecognitionExtra implements Parcelable { + /** + * The keyphrase ID + * + * @hide + */ @UnsupportedAppUsage public final int id; - /** Recognition modes matched for this event */ + /** + * Recognition modes matched for this event + * + * @hide + */ @UnsupportedAppUsage(maxTargetSdk = Build.VERSION_CODES.R, trackingBug = 170729553) public final int recognitionModes; - /** Confidence level for mode RECOGNITION_MODE_VOICE_TRIGGER when user identification - * is not performed */ + /** + * Confidence level for mode RECOGNITION_MODE_VOICE_TRIGGER when user identification + * is not performed + * + * @hide + */ @UnsupportedAppUsage public final int coarseConfidenceLevel; - /** Confidence levels for all users recognized (KeyphraseRecognitionEvent) or to - * be recognized (RecognitionConfig) */ + /** + * Confidence levels for all users recognized (KeyphraseRecognitionEvent) or to + * be recognized (RecognitionConfig) + * + * @hide + */ @UnsupportedAppUsage @NonNull public final ConfidenceLevel[] confidenceLevels; + + /** + * @hide + */ + @TestApi + public KeyphraseRecognitionExtra(int id, @RecognitionModes int recognitionModes, + int coarseConfidenceLevel) { + this(id, recognitionModes, coarseConfidenceLevel, new ConfidenceLevel[0]); + } + + /** + * @hide + */ @UnsupportedAppUsage public KeyphraseRecognitionExtra(int id, int recognitionModes, int coarseConfidenceLevel, @Nullable ConfidenceLevel[] confidenceLevels) { @@ -1611,7 +1639,47 @@ public class SoundTrigger { confidenceLevels != null ? confidenceLevels : new ConfidenceLevel[0]; } - public static final @android.annotation.NonNull Parcelable.Creator CREATOR + /** + * The keyphrase ID associated with this class' additional data + */ + public int getKeyphraseId() { + return id; + } + + /** + * Recognition modes matched for this event + */ + @RecognitionModes + public int getRecognitionModes() { + return recognitionModes; + } + + /** + * Confidence level for mode RECOGNITION_MODE_VOICE_TRIGGER when user identification + * is not performed + * + *

The confidence level is expressed in percent (0% -100%). + */ + public int getCoarseConfidenceLevel() { + return coarseConfidenceLevel; + } + + /** + * Detected confidence level for users defined in a keyphrase. + * + *

The confidence level is expressed in percent (0% -100%). + * + *

The user ID is derived from the system ID + * {@link android.os.UserHandle#getIdentifier()}. + * + * @hide + */ + @NonNull + public Collection getConfidenceLevels() { + return Arrays.asList(confidenceLevels); + } + + public static final @NonNull Parcelable.Creator CREATOR = new Parcelable.Creator() { public KeyphraseRecognitionExtra createFromParcel(Parcel in) { return KeyphraseRecognitionExtra.fromParcel(in); @@ -1632,7 +1700,7 @@ public class SoundTrigger { } @Override - public void writeToParcel(Parcel dest, int flags) { + public void writeToParcel(@NonNull Parcel dest, int flags) { dest.writeInt(id); dest.writeInt(recognitionModes); dest.writeInt(coarseConfidenceLevel); @@ -1657,21 +1725,28 @@ public class SoundTrigger { @Override public boolean equals(@Nullable Object obj) { - if (this == obj) + if (this == obj) { return true; - if (obj == null) + } + if (obj == null) { return false; - if (getClass() != obj.getClass()) + } + if (getClass() != obj.getClass()) { return false; + } KeyphraseRecognitionExtra other = (KeyphraseRecognitionExtra) obj; - if (!Arrays.equals(confidenceLevels, other.confidenceLevels)) + if (!Arrays.equals(confidenceLevels, other.confidenceLevels)) { return false; - if (id != other.id) + } + if (id != other.id) { return false; - if (recognitionModes != other.recognitionModes) + } + if (recognitionModes != other.recognitionModes) { return false; - if (coarseConfidenceLevel != other.coarseConfidenceLevel) + } + if (coarseConfidenceLevel != other.coarseConfidenceLevel) { return false; + } return true; } @@ -1715,7 +1790,7 @@ public class SoundTrigger { keyphraseExtras != null ? keyphraseExtras : new KeyphraseRecognitionExtra[0]; } - public static final @android.annotation.NonNull Parcelable.Creator CREATOR + public static final @NonNull Parcelable.Creator CREATOR = new Parcelable.Creator() { public KeyphraseRecognitionEvent createFromParcel(Parcel in) { return KeyphraseRecognitionEvent.fromParcelForKeyphrase(in); diff --git a/core/java/android/service/voice/AbstractHotwordDetector.java b/core/java/android/service/voice/AbstractHotwordDetector.java index 192260791a8b3..c3bf5694dbffc 100644 --- a/core/java/android/service/voice/AbstractHotwordDetector.java +++ b/core/java/android/service/voice/AbstractHotwordDetector.java @@ -146,7 +146,10 @@ abstract class AbstractHotwordDetector implements HotwordDetector { mHandler.sendMessage(obtainMessage( HotwordDetector.Callback::onDetected, mCallback, - new AlwaysOnHotwordDetector.EventPayload(audioFormat, hotwordDetectedResult))); + new AlwaysOnHotwordDetector.EventPayload.Builder() + .setCaptureAudioFormat(audioFormat) + .setHotwordDetectedResult(hotwordDetectedResult) + .build())); } } } diff --git a/core/java/android/service/voice/AlwaysOnHotwordDetector.java b/core/java/android/service/voice/AlwaysOnHotwordDetector.java index 69f771bbac4a7..b90f27b986c11 100644 --- a/core/java/android/service/voice/AlwaysOnHotwordDetector.java +++ b/core/java/android/service/voice/AlwaysOnHotwordDetector.java @@ -23,6 +23,7 @@ import android.annotation.IntDef; import android.annotation.NonNull; import android.annotation.Nullable; import android.annotation.RequiresPermission; +import android.annotation.SuppressLint; import android.annotation.SystemApi; import android.annotation.TestApi; import android.app.ActivityThread; @@ -59,6 +60,9 @@ import com.android.internal.app.IVoiceInteractionSoundTriggerSession; import java.io.PrintWriter; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; import java.util.Locale; /** @@ -380,47 +384,24 @@ public class AlwaysOnHotwordDetector extends AbstractHotwordDetector { private final byte[] mData; private final HotwordDetectedResult mHotwordDetectedResult; private final ParcelFileDescriptor mAudioStream; + private final List mKephraseExtras; - EventPayload(boolean triggerInData, boolean captureAvailable, - AudioFormat audioFormat, int captureSession, byte[] data) { - this(triggerInData, captureAvailable, audioFormat, captureSession, data, null, - null); - } - - EventPayload(boolean triggerInData, boolean captureAvailable, - AudioFormat audioFormat, int captureSession, byte[] data, - HotwordDetectedResult hotwordDetectedResult) { - this(triggerInData, captureAvailable, audioFormat, captureSession, data, - hotwordDetectedResult, null); - } - - EventPayload(AudioFormat audioFormat, HotwordDetectedResult hotwordDetectedResult) { - this(false, false, audioFormat, -1, null, hotwordDetectedResult, null); - } - - EventPayload(AudioFormat audioFormat, - HotwordDetectedResult hotwordDetectedResult, - ParcelFileDescriptor audioStream) { - this(false, false, audioFormat, -1, null, hotwordDetectedResult, audioStream); - } - - /** @hide */ - @TestApi - public EventPayload(boolean triggerInData, boolean captureAvailable, - @Nullable AudioFormat audioFormat, int captureSession, @Nullable byte[] data, + private EventPayload(boolean captureAvailable, + @Nullable AudioFormat audioFormat, + int captureSession, + @DataFormat int dataFormat, + @Nullable byte[] data, @Nullable HotwordDetectedResult hotwordDetectedResult, - @Nullable ParcelFileDescriptor audioStream) { + @Nullable ParcelFileDescriptor audioStream, + @NonNull List keyphraseExtras) { mCaptureAvailable = captureAvailable; mCaptureSession = captureSession; mAudioFormat = audioFormat; - if (triggerInData) { - mDataFormat = DATA_FORMAT_TRIGGER_AUDIO; - } else { - mDataFormat = DATA_FORMAT_RAW; - } + mDataFormat = dataFormat; mData = data; mHotwordDetectedResult = hotwordDetectedResult; mAudioStream = audioStream; + mKephraseExtras = keyphraseExtras; } /** @@ -535,6 +516,166 @@ public class AlwaysOnHotwordDetector extends AbstractHotwordDetector { public ParcelFileDescriptor getAudioStream() { return mAudioStream; } + + /** + * Returns the keyphrases recognized by the voice engine with additional confidence + * information + * + * @return List of keyphrase extras describing additional data for each keyphrase the voice + * engine triggered on for this event. The ordering of the list is preserved based on what + * the ordering provided by {@link android.hardware.soundtrigger.SoundTriggerModule}. + */ + @NonNull + public List getKeyphraseRecognitionExtras() { + return mKephraseExtras; + } + + /** + * Builder class for {@link EventPayload} objects + * + * @hide + */ + @TestApi + public static final class Builder { + private boolean mCaptureAvailable = false; + private int mCaptureSession = -1; + private AudioFormat mAudioFormat = null; + @DataFormat + private int mDataFormat = DATA_FORMAT_RAW; + private byte[] mData = null; + private HotwordDetectedResult mHotwordDetectedResult = null; + private ParcelFileDescriptor mAudioStream = null; + private List mKeyphraseExtras = Collections.emptyList(); + + public Builder() {} + + Builder(SoundTrigger.KeyphraseRecognitionEvent keyphraseRecognitionEvent) { + setCaptureAvailable(keyphraseRecognitionEvent.isCaptureAvailable()); + setCaptureSession(keyphraseRecognitionEvent.getCaptureSession()); + if (keyphraseRecognitionEvent.getCaptureFormat() != null) { + setCaptureAudioFormat(keyphraseRecognitionEvent.getCaptureFormat()); + } + setDataFormat((keyphraseRecognitionEvent.triggerInData) ? DATA_FORMAT_TRIGGER_AUDIO + : DATA_FORMAT_RAW); + if (keyphraseRecognitionEvent.getData() != null) { + setData(keyphraseRecognitionEvent.getData()); + } + if (keyphraseRecognitionEvent.keyphraseExtras != null) { + setKeyphraseRecognitionExtras( + Arrays.asList(keyphraseRecognitionEvent.keyphraseExtras)); + } + } + + /** + * Indicates if {@code captureSession} can be used to continue capturing more audio from + * the DSP hardware. + */ + @SuppressLint("MissingGetterMatchingBuilder") + @NonNull + public Builder setCaptureAvailable(boolean captureAvailable) { + mCaptureAvailable = captureAvailable; + return this; + } + + /** + * Sets the session ID to start a capture from the DSP. + */ + @SuppressLint("MissingGetterMatchingBuilder") + @NonNull + public Builder setCaptureSession(int captureSession) { + mCaptureSession = captureSession; + return this; + } + + /** + * Sets the format of the audio obtained using {@link #getTriggerAudio()}. + */ + @NonNull + public Builder setCaptureAudioFormat(@NonNull AudioFormat audioFormat) { + mAudioFormat = audioFormat; + return this; + } + + /** + * Conveys the format of the additional data that is triggered with the keyphrase event. + * + * @see AlwaysOnHotwordDetector#RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO + * @see DataFormat + */ + @NonNull + public Builder setDataFormat(@DataFormat int dataFormat) { + mDataFormat = dataFormat; + return this; + } + + /** + * Sets additional raw data that is triggered with the keyphrase event. + * + *

A {@link android.hardware.soundtrigger.SoundTriggerModule} may populate this + * field with opaque data for use by system applications who know about voice + * engine internals. Data may be null if the field is not populated by the + * {@link android.hardware.soundtrigger.SoundTriggerModule}. + * + *

If {@link #getDataFormat()} is {@link #DATA_FORMAT_TRIGGER_AUDIO}, then the + * entirety of this + * buffer is expected to be of the format from {@link #getCaptureAudioFormat()}. + * + * @see AlwaysOnHotwordDetector#RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO + */ + @NonNull + public Builder setData(@NonNull byte[] data) { + mData = data; + return this; + } + + /** + * Sets {@link HotwordDetectedResult} associated with the hotword event, passed from + * {@link HotwordDetectionService}. + */ + @NonNull + public Builder setHotwordDetectedResult( + @NonNull HotwordDetectedResult hotwordDetectedResult) { + mHotwordDetectedResult = hotwordDetectedResult; + return this; + } + + /** + * Sets a stream with bytes corresponding to the open audio stream with hotword data. + * + *

This data represents an audio stream in the format returned by + * {@link #getCaptureAudioFormat}. + * + *

Clients are expected to start consuming the stream within 1 second of receiving + * the + * event. + */ + @NonNull + public Builder setAudioStream(@NonNull ParcelFileDescriptor audioStream) { + mAudioStream = audioStream; + return this; + } + + /** + * Sets the keyphrases recognized by the voice engine with additional confidence + * information + */ + @NonNull + public Builder setKeyphraseRecognitionExtras( + @NonNull List keyphraseRecognitionExtras) { + mKeyphraseExtras = keyphraseRecognitionExtras; + return this; + } + + /** + * Builds an {@link EventPayload} instance + */ + @NonNull + public EventPayload build() { + return new EventPayload(mCaptureAvailable, mAudioFormat, mCaptureSession, + mDataFormat, mData, mHotwordDetectedResult, mAudioStream, + mKeyphraseExtras); + } + } } /** @@ -1242,8 +1383,9 @@ public class AlwaysOnHotwordDetector extends AbstractHotwordDetector { Slog.i(TAG, "onDetected"); } Message.obtain(mHandler, MSG_HOTWORD_DETECTED, - new EventPayload(event.triggerInData, event.captureAvailable, - event.captureFormat, event.captureSession, event.data, result)) + new EventPayload.Builder(event) + .setHotwordDetectedResult(result) + .build()) .sendToTarget(); } @Override diff --git a/core/java/android/service/voice/HotwordDetectionService.java b/core/java/android/service/voice/HotwordDetectionService.java index e3bb589c9a19d..dfe0f542b3cab 100644 --- a/core/java/android/service/voice/HotwordDetectionService.java +++ b/core/java/android/service/voice/HotwordDetectionService.java @@ -140,9 +140,7 @@ public abstract class HotwordDetectionService extends Service { Log.d(TAG, "#detectFromDspSource"); } HotwordDetectionService.this.onDetect( - new AlwaysOnHotwordDetector.EventPayload( - event.triggerInData, event.captureAvailable, - event.captureFormat, event.captureSession, event.data), + new AlwaysOnHotwordDetector.EventPayload.Builder(event).build(), timeoutMillis, new Callback(callback)); } diff --git a/core/java/android/service/voice/SoftwareHotwordDetector.java b/core/java/android/service/voice/SoftwareHotwordDetector.java index 512a654adbab7..426f312b2e86a 100644 --- a/core/java/android/service/voice/SoftwareHotwordDetector.java +++ b/core/java/android/service/voice/SoftwareHotwordDetector.java @@ -135,8 +135,11 @@ class SoftwareHotwordDetector extends AbstractHotwordDetector { mHandler.sendMessage(obtainMessage( HotwordDetector.Callback::onDetected, mCallback, - new AlwaysOnHotwordDetector.EventPayload( - audioFormat, hotwordDetectedResult, audioStream))); + new AlwaysOnHotwordDetector.EventPayload.Builder() + .setCaptureAudioFormat(audioFormat) + .setAudioStream(audioStream) + .setHotwordDetectedResult(hotwordDetectedResult) + .build())); } } From 935e33f41671847f5bf117dad78d2f6f759819f6 Mon Sep 17 00:00:00 2001 From: Nicholas Ambur Date: Tue, 8 Feb 2022 19:58:14 -0800 Subject: [PATCH 3/3] API to destroy active detectors The active AlwaysOnHotwordDetector and SoftwareHotwordDetector is always maintained in memory regardless if the client is using it or not. This added API allows an active VoiceInteractionService to indicate that they will no longer use the detector, and it can be cleaned up. Test: atest HotwordDetectionServiceBasicTest Bug: 193232191 Change-Id: I47c6c64c5c85c01e75ddc6bc504664883a57730b --- core/api/system-current.txt | 1 + .../voice/AbstractHotwordDetector.java | 38 ++++++++++ .../voice/AlwaysOnHotwordDetector.java | 10 ++- .../service/voice/HotwordDetector.java | 11 +++ .../voice/SoftwareHotwordDetector.java | 16 ++++- .../voice/VoiceInteractionService.java | 70 ++++++++----------- 6 files changed, 102 insertions(+), 44 deletions(-) diff --git a/core/api/system-current.txt b/core/api/system-current.txt index 452cbe0b8e690..38bed01228bcc 100644 --- a/core/api/system-current.txt +++ b/core/api/system-current.txt @@ -11841,6 +11841,7 @@ package android.service.voice { } public interface HotwordDetector { + method public default void destroy(); method @RequiresPermission(allOf={android.Manifest.permission.RECORD_AUDIO, android.Manifest.permission.CAPTURE_AUDIO_HOTWORD}) public boolean startRecognition(); method public boolean startRecognition(@NonNull android.os.ParcelFileDescriptor, @NonNull android.media.AudioFormat, @Nullable android.os.PersistableBundle); method public boolean stopRecognition(); diff --git a/core/java/android/service/voice/AbstractHotwordDetector.java b/core/java/android/service/voice/AbstractHotwordDetector.java index c3bf5694dbffc..01d5638461af8 100644 --- a/core/java/android/service/voice/AbstractHotwordDetector.java +++ b/core/java/android/service/voice/AbstractHotwordDetector.java @@ -18,6 +18,7 @@ package android.service.voice; import static com.android.internal.util.function.pooled.PooledLambda.obtainMessage; +import android.annotation.CallSuper; import android.annotation.NonNull; import android.annotation.Nullable; import android.app.ActivityThread; @@ -34,6 +35,9 @@ import android.util.Slog; import com.android.internal.app.IHotwordRecognitionStatusCallback; import com.android.internal.app.IVoiceInteractionManagerService; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Consumer; + /** Base implementation of {@link HotwordDetector}. */ abstract class AbstractHotwordDetector implements HotwordDetector { private static final String TAG = AbstractHotwordDetector.class.getSimpleName(); @@ -45,6 +49,8 @@ abstract class AbstractHotwordDetector implements HotwordDetector { private final Handler mHandler; private final HotwordDetector.Callback mCallback; private final int mDetectorType; + private Consumer mOnDestroyListener; + private final AtomicBoolean mIsDetectorActive; AbstractHotwordDetector( IVoiceInteractionManagerService managerService, @@ -55,6 +61,7 @@ abstract class AbstractHotwordDetector implements HotwordDetector { mHandler = new Handler(Looper.getMainLooper()); mCallback = callback; mDetectorType = detectorType; + mIsDetectorActive = new AtomicBoolean(true); } /** @@ -70,6 +77,7 @@ abstract class AbstractHotwordDetector implements HotwordDetector { if (DEBUG) { Slog.i(TAG, "#recognizeHotword"); } + throwIfDetectorIsNoLongerActive(); // TODO: consider closing existing session. @@ -106,6 +114,7 @@ abstract class AbstractHotwordDetector implements HotwordDetector { if (DEBUG) { Slog.d(TAG, "updateState()"); } + throwIfDetectorIsNoLongerActive(); synchronized (mLock) { updateStateLocked(options, sharedMemory, null /* callback */, mDetectorType); } @@ -126,6 +135,35 @@ abstract class AbstractHotwordDetector implements HotwordDetector { } } + void registerOnDestroyListener(Consumer onDestroyListener) { + synchronized (mLock) { + if (mOnDestroyListener != null) { + throw new IllegalStateException("only one destroy listener can be registered"); + } + mOnDestroyListener = onDestroyListener; + } + } + + @CallSuper + @Override + public void destroy() { + if (!mIsDetectorActive.get()) { + return; + } + mIsDetectorActive.set(false); + synchronized (mLock) { + mOnDestroyListener.accept(this); + } + } + + protected void throwIfDetectorIsNoLongerActive() { + if (!mIsDetectorActive.get()) { + Slog.e(TAG, "attempting to use a destroyed detector which is no longer active"); + throw new IllegalStateException( + "attempting to use a destroyed detector which is no longer active"); + } + } + private static class BinderCallback extends IMicrophoneHotwordDetectionVoiceInteractionCallback.Stub { private final Handler mHandler; diff --git a/core/java/android/service/voice/AlwaysOnHotwordDetector.java b/core/java/android/service/voice/AlwaysOnHotwordDetector.java index b90f27b986c11..bec5d1be57fd1 100644 --- a/core/java/android/service/voice/AlwaysOnHotwordDetector.java +++ b/core/java/android/service/voice/AlwaysOnHotwordDetector.java @@ -1205,11 +1205,14 @@ public class AlwaysOnHotwordDetector extends AbstractHotwordDetector { /** * Invalidates this hotword detector so that any future calls to this result * in an IllegalStateException. - * - * @hide */ - void invalidate() { + @Override + public void destroy() { synchronized (mLock) { + if (mAvailability == STATE_KEYPHRASE_ENROLLED) { + stopRecognition(); + } + mAvailability = STATE_INVALID; notifyStateChangedLocked(); @@ -1221,6 +1224,7 @@ public class AlwaysOnHotwordDetector extends AbstractHotwordDetector { } } } + super.destroy(); } /** diff --git a/core/java/android/service/voice/HotwordDetector.java b/core/java/android/service/voice/HotwordDetector.java index 969ec22beb978..96fd8bbda0160 100644 --- a/core/java/android/service/voice/HotwordDetector.java +++ b/core/java/android/service/voice/HotwordDetector.java @@ -118,6 +118,17 @@ public interface HotwordDetector { */ void updateState(@Nullable PersistableBundle options, @Nullable SharedMemory sharedMemory); + /** + * Invalidates this hotword detector so that any future calls to this result + * in an {@link IllegalStateException}. + * + *

If there are no other {@link HotwordDetector} instances linked to the + * {@link HotwordDetectionService}, the service will be shutdown. + */ + default void destroy() { + throw new UnsupportedOperationException("Not implemented. Must override in a subclass."); + } + /** * @hide */ diff --git a/core/java/android/service/voice/SoftwareHotwordDetector.java b/core/java/android/service/voice/SoftwareHotwordDetector.java index 426f312b2e86a..2d662eaf0a4f0 100644 --- a/core/java/android/service/voice/SoftwareHotwordDetector.java +++ b/core/java/android/service/voice/SoftwareHotwordDetector.java @@ -77,7 +77,7 @@ class SoftwareHotwordDetector extends AbstractHotwordDetector { if (DEBUG) { Slog.i(TAG, "#startRecognition"); } - + throwIfDetectorIsNoLongerActive(); maybeCloseExistingSession(); try { @@ -100,6 +100,7 @@ class SoftwareHotwordDetector extends AbstractHotwordDetector { if (DEBUG) { Slog.i(TAG, "#stopRecognition"); } + throwIfDetectorIsNoLongerActive(); try { mManagerService.stopListeningFromMic(); @@ -110,6 +111,19 @@ class SoftwareHotwordDetector extends AbstractHotwordDetector { return true; } + @Override + public void destroy() { + stopRecognition(); + maybeCloseExistingSession(); + + try { + mManagerService.shutdownHotwordDetectionService(); + } catch (RemoteException ex) { + ex.rethrowFromSystemServer(); + } + super.destroy(); + } + private void maybeCloseExistingSession() { // TODO: needs to be synchronized. // TODO: implement this diff --git a/core/java/android/service/voice/VoiceInteractionService.java b/core/java/android/service/voice/VoiceInteractionService.java index f52c9ff210d65..bf0cfbe49f31d 100644 --- a/core/java/android/service/voice/VoiceInteractionService.java +++ b/core/java/android/service/voice/VoiceInteractionService.java @@ -271,7 +271,7 @@ public class VoiceInteractionService extends Service { // It's still guaranteed to have been stopped. // This helps with cases where the voice interaction implementation is changed // by the user. - safelyShutdownHotwordDetector(); + safelyShutdownAllHotwordDetectors(); } /** @@ -380,11 +380,13 @@ public class VoiceInteractionService extends Service { } synchronized (mLock) { // Allow only one concurrent recognition via the APIs. - safelyShutdownHotwordDetector(); + safelyShutdownAllHotwordDetectors(); mHotwordDetector = new AlwaysOnHotwordDetector(keyphrase, locale, callback, mKeyphraseEnrollmentInfo, mSystemService, getApplicationContext().getApplicationInfo().targetSdkVersion, supportHotwordDetectionService, options, sharedMemory); + mHotwordDetector.registerOnDestroyListener((detector) -> onDspHotwordDetectorDestroyed( + (AlwaysOnHotwordDetector) detector)); } return mHotwordDetector; } @@ -433,10 +435,13 @@ public class VoiceInteractionService extends Service { } synchronized (mLock) { // Allow only one concurrent recognition via the APIs. - safelyShutdownHotwordDetector(); + safelyShutdownAllHotwordDetectors(); mSoftwareHotwordDetector = new SoftwareHotwordDetector( mSystemService, null, options, sharedMemory, callback); + mSoftwareHotwordDetector.registerOnDestroyListener( + (detector) -> onMicrophoneHotwordDetectorDestroyed( + (SoftwareHotwordDetector) detector)); } return mSoftwareHotwordDetector; } @@ -482,51 +487,36 @@ public class VoiceInteractionService extends Service { return mKeyphraseEnrollmentInfo.getKeyphraseMetadata(keyphrase, locale) != null; } - private void safelyShutdownHotwordDetector() { + private void safelyShutdownAllHotwordDetectors() { synchronized (mLock) { - shutdownDspHotwordDetectorLocked(); - shutdownMicrophoneHotwordDetectorLocked(); + if (mHotwordDetector != null) { + try { + mHotwordDetector.destroy(); + } catch (Exception ex) { + Log.i(TAG, "exception destroying AlwaysOnHotwordDetector", ex); + } + } + + if (mSoftwareHotwordDetector != null) { + try { + mSoftwareHotwordDetector.destroy(); + } catch (Exception ex) { + Log.i(TAG, "exception destroying SoftwareHotwordDetector", ex); + } + } } } - private void shutdownDspHotwordDetectorLocked() { - if (mHotwordDetector == null) { - return; + private void onDspHotwordDetectorDestroyed(@NonNull AlwaysOnHotwordDetector detector) { + synchronized (mLock) { + mHotwordDetector = null; } - - try { - mHotwordDetector.stopRecognition(); - } catch (Exception ex) { - // Ignore. - } - - try { - mHotwordDetector.invalidate(); - } catch (Exception ex) { - // Ignore. - } - - mHotwordDetector = null; } - private void shutdownMicrophoneHotwordDetectorLocked() { - if (mSoftwareHotwordDetector == null) { - return; + private void onMicrophoneHotwordDetectorDestroyed(@NonNull SoftwareHotwordDetector detector) { + synchronized (mLock) { + mSoftwareHotwordDetector = null; } - - try { - mSoftwareHotwordDetector.stopRecognition(); - } catch (Exception ex) { - // Ignore. - } - - try { - mSystemService.shutdownHotwordDetectionService(); - } catch (Exception ex) { - // Ignore. - } - - mSoftwareHotwordDetector = null; } /**