diff --git a/core/api/system-current.txt b/core/api/system-current.txt index 4b3d0a4d23751..452cbe0b8e690 100644 --- a/core/api/system-current.txt +++ b/core/api/system-current.txt @@ -5172,6 +5172,15 @@ package android.hardware.soundtrigger { field @NonNull public static final android.os.Parcelable.Creator CREATOR; } + public static final class SoundTrigger.KeyphraseRecognitionExtra implements android.os.Parcelable { + method public int describeContents(); + method public int getCoarseConfidenceLevel(); + method public int getKeyphraseId(); + method public int getRecognitionModes(); + method public void writeToParcel(@NonNull android.os.Parcel, int); + field @NonNull public static final android.os.Parcelable.Creator CREATOR; + } + public static final class SoundTrigger.KeyphraseSoundModel extends android.hardware.soundtrigger.SoundTrigger.SoundModel implements android.os.Parcelable { ctor public SoundTrigger.KeyphraseSoundModel(@NonNull java.util.UUID, @NonNull java.util.UUID, @Nullable byte[], @Nullable android.hardware.soundtrigger.SoundTrigger.Keyphrase[], int); ctor public SoundTrigger.KeyphraseSoundModel(@NonNull java.util.UUID, @NonNull java.util.UUID, @Nullable byte[], @Nullable android.hardware.soundtrigger.SoundTrigger.Keyphrase[]); @@ -11758,6 +11767,7 @@ package android.service.voice { method @Nullable public byte[] getData(); method public int getDataFormat(); method @Nullable public android.service.voice.HotwordDetectedResult getHotwordDetectedResult(); + method @NonNull public java.util.List getKeyphraseRecognitionExtras(); method @Deprecated @Nullable public byte[] getTriggerAudio(); field public static final int DATA_FORMAT_RAW = 0; // 0x0 field public static final int DATA_FORMAT_TRIGGER_AUDIO = 1; // 0x1 diff --git a/core/api/test-current.txt b/core/api/test-current.txt index d85f20582b496..23f4fe03c2140 100644 --- a/core/api/test-current.txt +++ b/core/api/test-current.txt @@ -1276,6 +1276,10 @@ package android.hardware.soundtrigger { field @NonNull public static final android.os.Parcelable.Creator CREATOR; } + public static final class SoundTrigger.KeyphraseRecognitionExtra implements android.os.Parcelable { + ctor public SoundTrigger.KeyphraseRecognitionExtra(int, int, int); + } + public static final class SoundTrigger.ModelParamRange implements android.os.Parcelable { ctor public SoundTrigger.ModelParamRange(int, int); } @@ -2398,8 +2402,17 @@ package android.service.voice { method @RequiresPermission(allOf={android.Manifest.permission.RECORD_AUDIO, android.Manifest.permission.CAPTURE_AUDIO_HOTWORD}) public void triggerHardwareRecognitionEventForTest(int, int, boolean, int, int, int, boolean, @NonNull android.media.AudioFormat, @Nullable byte[]); } - public static class AlwaysOnHotwordDetector.EventPayload { - ctor public AlwaysOnHotwordDetector.EventPayload(boolean, boolean, @Nullable android.media.AudioFormat, int, @Nullable byte[], @Nullable android.service.voice.HotwordDetectedResult, @Nullable android.os.ParcelFileDescriptor); + public static final class AlwaysOnHotwordDetector.EventPayload.Builder { + ctor public AlwaysOnHotwordDetector.EventPayload.Builder(); + method @NonNull public android.service.voice.AlwaysOnHotwordDetector.EventPayload build(); + method @NonNull public android.service.voice.AlwaysOnHotwordDetector.EventPayload.Builder setAudioStream(@NonNull android.os.ParcelFileDescriptor); + method @NonNull public android.service.voice.AlwaysOnHotwordDetector.EventPayload.Builder setCaptureAudioFormat(@NonNull android.media.AudioFormat); + method @NonNull public android.service.voice.AlwaysOnHotwordDetector.EventPayload.Builder setCaptureAvailable(boolean); + method @NonNull public android.service.voice.AlwaysOnHotwordDetector.EventPayload.Builder setCaptureSession(int); + method @NonNull public android.service.voice.AlwaysOnHotwordDetector.EventPayload.Builder setData(@NonNull byte[]); + method @NonNull public android.service.voice.AlwaysOnHotwordDetector.EventPayload.Builder setDataFormat(int); + method @NonNull public android.service.voice.AlwaysOnHotwordDetector.EventPayload.Builder setHotwordDetectedResult(@NonNull android.service.voice.HotwordDetectedResult); + method @NonNull public android.service.voice.AlwaysOnHotwordDetector.EventPayload.Builder setKeyphraseRecognitionExtras(@NonNull java.util.List); } public final class VisibleActivityInfo implements android.os.Parcelable { diff --git a/core/java/android/hardware/soundtrigger/SoundTrigger.java b/core/java/android/hardware/soundtrigger/SoundTrigger.java index b0439d0e52536..c36390917cf18 100644 --- a/core/java/android/hardware/soundtrigger/SoundTrigger.java +++ b/core/java/android/hardware/soundtrigger/SoundTrigger.java @@ -61,6 +61,7 @@ import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Locale; import java.util.UUID; @@ -1576,31 +1577,58 @@ public class SoundTrigger { } /** - * Additional data conveyed by a {@link KeyphraseRecognitionEvent} - * for a key phrase detection. - * - * @hide + * Additional data conveyed by a {@link KeyphraseRecognitionEvent} + * for a key phrase detection. */ - public static class KeyphraseRecognitionExtra implements Parcelable { - /** The keyphrase ID */ + public static final class KeyphraseRecognitionExtra implements Parcelable { + /** + * The keyphrase ID + * + * @hide + */ @UnsupportedAppUsage public final int id; - /** Recognition modes matched for this event */ + /** + * Recognition modes matched for this event + * + * @hide + */ @UnsupportedAppUsage(maxTargetSdk = Build.VERSION_CODES.R, trackingBug = 170729553) public final int recognitionModes; - /** Confidence level for mode RECOGNITION_MODE_VOICE_TRIGGER when user identification - * is not performed */ + /** + * Confidence level for mode RECOGNITION_MODE_VOICE_TRIGGER when user identification + * is not performed + * + * @hide + */ @UnsupportedAppUsage public final int coarseConfidenceLevel; - /** Confidence levels for all users recognized (KeyphraseRecognitionEvent) or to - * be recognized (RecognitionConfig) */ + /** + * Confidence levels for all users recognized (KeyphraseRecognitionEvent) or to + * be recognized (RecognitionConfig) + * + * @hide + */ @UnsupportedAppUsage @NonNull public final ConfidenceLevel[] confidenceLevels; + + /** + * @hide + */ + @TestApi + public KeyphraseRecognitionExtra(int id, @RecognitionModes int recognitionModes, + int coarseConfidenceLevel) { + this(id, recognitionModes, coarseConfidenceLevel, new ConfidenceLevel[0]); + } + + /** + * @hide + */ @UnsupportedAppUsage public KeyphraseRecognitionExtra(int id, int recognitionModes, int coarseConfidenceLevel, @Nullable ConfidenceLevel[] confidenceLevels) { @@ -1611,7 +1639,47 @@ public class SoundTrigger { confidenceLevels != null ? confidenceLevels : new ConfidenceLevel[0]; } - public static final @android.annotation.NonNull Parcelable.Creator CREATOR + /** + * The keyphrase ID associated with this class' additional data + */ + public int getKeyphraseId() { + return id; + } + + /** + * Recognition modes matched for this event + */ + @RecognitionModes + public int getRecognitionModes() { + return recognitionModes; + } + + /** + * Confidence level for mode RECOGNITION_MODE_VOICE_TRIGGER when user identification + * is not performed + * + *

The confidence level is expressed in percent (0% -100%). + */ + public int getCoarseConfidenceLevel() { + return coarseConfidenceLevel; + } + + /** + * Detected confidence level for users defined in a keyphrase. + * + *

The confidence level is expressed in percent (0% -100%). + * + *

The user ID is derived from the system ID + * {@link android.os.UserHandle#getIdentifier()}. + * + * @hide + */ + @NonNull + public Collection getConfidenceLevels() { + return Arrays.asList(confidenceLevels); + } + + public static final @NonNull Parcelable.Creator CREATOR = new Parcelable.Creator() { public KeyphraseRecognitionExtra createFromParcel(Parcel in) { return KeyphraseRecognitionExtra.fromParcel(in); @@ -1632,7 +1700,7 @@ public class SoundTrigger { } @Override - public void writeToParcel(Parcel dest, int flags) { + public void writeToParcel(@NonNull Parcel dest, int flags) { dest.writeInt(id); dest.writeInt(recognitionModes); dest.writeInt(coarseConfidenceLevel); @@ -1657,21 +1725,28 @@ public class SoundTrigger { @Override public boolean equals(@Nullable Object obj) { - if (this == obj) + if (this == obj) { return true; - if (obj == null) + } + if (obj == null) { return false; - if (getClass() != obj.getClass()) + } + if (getClass() != obj.getClass()) { return false; + } KeyphraseRecognitionExtra other = (KeyphraseRecognitionExtra) obj; - if (!Arrays.equals(confidenceLevels, other.confidenceLevels)) + if (!Arrays.equals(confidenceLevels, other.confidenceLevels)) { return false; - if (id != other.id) + } + if (id != other.id) { return false; - if (recognitionModes != other.recognitionModes) + } + if (recognitionModes != other.recognitionModes) { return false; - if (coarseConfidenceLevel != other.coarseConfidenceLevel) + } + if (coarseConfidenceLevel != other.coarseConfidenceLevel) { return false; + } return true; } @@ -1715,7 +1790,7 @@ public class SoundTrigger { keyphraseExtras != null ? keyphraseExtras : new KeyphraseRecognitionExtra[0]; } - public static final @android.annotation.NonNull Parcelable.Creator CREATOR + public static final @NonNull Parcelable.Creator CREATOR = new Parcelable.Creator() { public KeyphraseRecognitionEvent createFromParcel(Parcel in) { return KeyphraseRecognitionEvent.fromParcelForKeyphrase(in); diff --git a/core/java/android/service/voice/AbstractHotwordDetector.java b/core/java/android/service/voice/AbstractHotwordDetector.java index 192260791a8b3..c3bf5694dbffc 100644 --- a/core/java/android/service/voice/AbstractHotwordDetector.java +++ b/core/java/android/service/voice/AbstractHotwordDetector.java @@ -146,7 +146,10 @@ abstract class AbstractHotwordDetector implements HotwordDetector { mHandler.sendMessage(obtainMessage( HotwordDetector.Callback::onDetected, mCallback, - new AlwaysOnHotwordDetector.EventPayload(audioFormat, hotwordDetectedResult))); + new AlwaysOnHotwordDetector.EventPayload.Builder() + .setCaptureAudioFormat(audioFormat) + .setHotwordDetectedResult(hotwordDetectedResult) + .build())); } } } diff --git a/core/java/android/service/voice/AlwaysOnHotwordDetector.java b/core/java/android/service/voice/AlwaysOnHotwordDetector.java index 69f771bbac4a7..b90f27b986c11 100644 --- a/core/java/android/service/voice/AlwaysOnHotwordDetector.java +++ b/core/java/android/service/voice/AlwaysOnHotwordDetector.java @@ -23,6 +23,7 @@ import android.annotation.IntDef; import android.annotation.NonNull; import android.annotation.Nullable; import android.annotation.RequiresPermission; +import android.annotation.SuppressLint; import android.annotation.SystemApi; import android.annotation.TestApi; import android.app.ActivityThread; @@ -59,6 +60,9 @@ import com.android.internal.app.IVoiceInteractionSoundTriggerSession; import java.io.PrintWriter; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; import java.util.Locale; /** @@ -380,47 +384,24 @@ public class AlwaysOnHotwordDetector extends AbstractHotwordDetector { private final byte[] mData; private final HotwordDetectedResult mHotwordDetectedResult; private final ParcelFileDescriptor mAudioStream; + private final List mKephraseExtras; - EventPayload(boolean triggerInData, boolean captureAvailable, - AudioFormat audioFormat, int captureSession, byte[] data) { - this(triggerInData, captureAvailable, audioFormat, captureSession, data, null, - null); - } - - EventPayload(boolean triggerInData, boolean captureAvailable, - AudioFormat audioFormat, int captureSession, byte[] data, - HotwordDetectedResult hotwordDetectedResult) { - this(triggerInData, captureAvailable, audioFormat, captureSession, data, - hotwordDetectedResult, null); - } - - EventPayload(AudioFormat audioFormat, HotwordDetectedResult hotwordDetectedResult) { - this(false, false, audioFormat, -1, null, hotwordDetectedResult, null); - } - - EventPayload(AudioFormat audioFormat, - HotwordDetectedResult hotwordDetectedResult, - ParcelFileDescriptor audioStream) { - this(false, false, audioFormat, -1, null, hotwordDetectedResult, audioStream); - } - - /** @hide */ - @TestApi - public EventPayload(boolean triggerInData, boolean captureAvailable, - @Nullable AudioFormat audioFormat, int captureSession, @Nullable byte[] data, + private EventPayload(boolean captureAvailable, + @Nullable AudioFormat audioFormat, + int captureSession, + @DataFormat int dataFormat, + @Nullable byte[] data, @Nullable HotwordDetectedResult hotwordDetectedResult, - @Nullable ParcelFileDescriptor audioStream) { + @Nullable ParcelFileDescriptor audioStream, + @NonNull List keyphraseExtras) { mCaptureAvailable = captureAvailable; mCaptureSession = captureSession; mAudioFormat = audioFormat; - if (triggerInData) { - mDataFormat = DATA_FORMAT_TRIGGER_AUDIO; - } else { - mDataFormat = DATA_FORMAT_RAW; - } + mDataFormat = dataFormat; mData = data; mHotwordDetectedResult = hotwordDetectedResult; mAudioStream = audioStream; + mKephraseExtras = keyphraseExtras; } /** @@ -535,6 +516,166 @@ public class AlwaysOnHotwordDetector extends AbstractHotwordDetector { public ParcelFileDescriptor getAudioStream() { return mAudioStream; } + + /** + * Returns the keyphrases recognized by the voice engine with additional confidence + * information + * + * @return List of keyphrase extras describing additional data for each keyphrase the voice + * engine triggered on for this event. The ordering of the list is preserved based on what + * the ordering provided by {@link android.hardware.soundtrigger.SoundTriggerModule}. + */ + @NonNull + public List getKeyphraseRecognitionExtras() { + return mKephraseExtras; + } + + /** + * Builder class for {@link EventPayload} objects + * + * @hide + */ + @TestApi + public static final class Builder { + private boolean mCaptureAvailable = false; + private int mCaptureSession = -1; + private AudioFormat mAudioFormat = null; + @DataFormat + private int mDataFormat = DATA_FORMAT_RAW; + private byte[] mData = null; + private HotwordDetectedResult mHotwordDetectedResult = null; + private ParcelFileDescriptor mAudioStream = null; + private List mKeyphraseExtras = Collections.emptyList(); + + public Builder() {} + + Builder(SoundTrigger.KeyphraseRecognitionEvent keyphraseRecognitionEvent) { + setCaptureAvailable(keyphraseRecognitionEvent.isCaptureAvailable()); + setCaptureSession(keyphraseRecognitionEvent.getCaptureSession()); + if (keyphraseRecognitionEvent.getCaptureFormat() != null) { + setCaptureAudioFormat(keyphraseRecognitionEvent.getCaptureFormat()); + } + setDataFormat((keyphraseRecognitionEvent.triggerInData) ? DATA_FORMAT_TRIGGER_AUDIO + : DATA_FORMAT_RAW); + if (keyphraseRecognitionEvent.getData() != null) { + setData(keyphraseRecognitionEvent.getData()); + } + if (keyphraseRecognitionEvent.keyphraseExtras != null) { + setKeyphraseRecognitionExtras( + Arrays.asList(keyphraseRecognitionEvent.keyphraseExtras)); + } + } + + /** + * Indicates if {@code captureSession} can be used to continue capturing more audio from + * the DSP hardware. + */ + @SuppressLint("MissingGetterMatchingBuilder") + @NonNull + public Builder setCaptureAvailable(boolean captureAvailable) { + mCaptureAvailable = captureAvailable; + return this; + } + + /** + * Sets the session ID to start a capture from the DSP. + */ + @SuppressLint("MissingGetterMatchingBuilder") + @NonNull + public Builder setCaptureSession(int captureSession) { + mCaptureSession = captureSession; + return this; + } + + /** + * Sets the format of the audio obtained using {@link #getTriggerAudio()}. + */ + @NonNull + public Builder setCaptureAudioFormat(@NonNull AudioFormat audioFormat) { + mAudioFormat = audioFormat; + return this; + } + + /** + * Conveys the format of the additional data that is triggered with the keyphrase event. + * + * @see AlwaysOnHotwordDetector#RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO + * @see DataFormat + */ + @NonNull + public Builder setDataFormat(@DataFormat int dataFormat) { + mDataFormat = dataFormat; + return this; + } + + /** + * Sets additional raw data that is triggered with the keyphrase event. + * + *

A {@link android.hardware.soundtrigger.SoundTriggerModule} may populate this + * field with opaque data for use by system applications who know about voice + * engine internals. Data may be null if the field is not populated by the + * {@link android.hardware.soundtrigger.SoundTriggerModule}. + * + *

If {@link #getDataFormat()} is {@link #DATA_FORMAT_TRIGGER_AUDIO}, then the + * entirety of this + * buffer is expected to be of the format from {@link #getCaptureAudioFormat()}. + * + * @see AlwaysOnHotwordDetector#RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO + */ + @NonNull + public Builder setData(@NonNull byte[] data) { + mData = data; + return this; + } + + /** + * Sets {@link HotwordDetectedResult} associated with the hotword event, passed from + * {@link HotwordDetectionService}. + */ + @NonNull + public Builder setHotwordDetectedResult( + @NonNull HotwordDetectedResult hotwordDetectedResult) { + mHotwordDetectedResult = hotwordDetectedResult; + return this; + } + + /** + * Sets a stream with bytes corresponding to the open audio stream with hotword data. + * + *

This data represents an audio stream in the format returned by + * {@link #getCaptureAudioFormat}. + * + *

Clients are expected to start consuming the stream within 1 second of receiving + * the + * event. + */ + @NonNull + public Builder setAudioStream(@NonNull ParcelFileDescriptor audioStream) { + mAudioStream = audioStream; + return this; + } + + /** + * Sets the keyphrases recognized by the voice engine with additional confidence + * information + */ + @NonNull + public Builder setKeyphraseRecognitionExtras( + @NonNull List keyphraseRecognitionExtras) { + mKeyphraseExtras = keyphraseRecognitionExtras; + return this; + } + + /** + * Builds an {@link EventPayload} instance + */ + @NonNull + public EventPayload build() { + return new EventPayload(mCaptureAvailable, mAudioFormat, mCaptureSession, + mDataFormat, mData, mHotwordDetectedResult, mAudioStream, + mKeyphraseExtras); + } + } } /** @@ -1242,8 +1383,9 @@ public class AlwaysOnHotwordDetector extends AbstractHotwordDetector { Slog.i(TAG, "onDetected"); } Message.obtain(mHandler, MSG_HOTWORD_DETECTED, - new EventPayload(event.triggerInData, event.captureAvailable, - event.captureFormat, event.captureSession, event.data, result)) + new EventPayload.Builder(event) + .setHotwordDetectedResult(result) + .build()) .sendToTarget(); } @Override diff --git a/core/java/android/service/voice/HotwordDetectionService.java b/core/java/android/service/voice/HotwordDetectionService.java index e3bb589c9a19d..dfe0f542b3cab 100644 --- a/core/java/android/service/voice/HotwordDetectionService.java +++ b/core/java/android/service/voice/HotwordDetectionService.java @@ -140,9 +140,7 @@ public abstract class HotwordDetectionService extends Service { Log.d(TAG, "#detectFromDspSource"); } HotwordDetectionService.this.onDetect( - new AlwaysOnHotwordDetector.EventPayload( - event.triggerInData, event.captureAvailable, - event.captureFormat, event.captureSession, event.data), + new AlwaysOnHotwordDetector.EventPayload.Builder(event).build(), timeoutMillis, new Callback(callback)); } diff --git a/core/java/android/service/voice/SoftwareHotwordDetector.java b/core/java/android/service/voice/SoftwareHotwordDetector.java index 512a654adbab7..426f312b2e86a 100644 --- a/core/java/android/service/voice/SoftwareHotwordDetector.java +++ b/core/java/android/service/voice/SoftwareHotwordDetector.java @@ -135,8 +135,11 @@ class SoftwareHotwordDetector extends AbstractHotwordDetector { mHandler.sendMessage(obtainMessage( HotwordDetector.Callback::onDetected, mCallback, - new AlwaysOnHotwordDetector.EventPayload( - audioFormat, hotwordDetectedResult, audioStream))); + new AlwaysOnHotwordDetector.EventPayload.Builder() + .setCaptureAudioFormat(audioFormat) + .setAudioStream(audioStream) + .setHotwordDetectedResult(hotwordDetectedResult) + .build())); } }