Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

expose HAL keyphrase trigger extras to EventPayload

Browse Source 
The SoundTrigger HAL can deliver additional context about the kephrase
that was triggered along with the trigger audio. This change moves to
expose the information which was previously suppressed by the Android
framework.

Data exposed:
- Keyphrase extras providing details on what phrase was interpreted by
the voice engine

Extra keyphrase information provided by the SoundTrigger HAL which was
previously suppressed is also exposed.

Test: atest AlwaysOnHotwordDetectorEventPayloadTest
CTS-Coverage-Bug: 215375531
Bug: 215066299
Change-Id: I3b20fd3692c3f9c86181f87d66622c77778b9ab2
This commit is contained in:
Nicholas Ambur
2022-02-07 19:23:49 -08:00
parent 6b6d178bd6
commit 2f7b2d5f94
7 changed files with 308 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
core/api/system-current.txt
View File

@@ -5172,6 +5172,15 @@ package android.hardware.soundtrigger {
field @NonNull public static final android.os.Parcelable.Creator<android.hardware.soundtrigger.SoundTrigger.Keyphrase> CREATOR;
}
public static final class SoundTrigger.KeyphraseRecognitionExtra implements android.os.Parcelable {
method public int describeContents();
method public int getCoarseConfidenceLevel();
method public int getKeyphraseId();
method public int getRecognitionModes();
method public void writeToParcel(@NonNull android.os.Parcel, int);
field @NonNull public static final android.os.Parcelable.Creator<android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionExtra> CREATOR;
}
public static final class SoundTrigger.KeyphraseSoundModel extends android.hardware.soundtrigger.SoundTrigger.SoundModel implements android.os.Parcelable {
ctor public SoundTrigger.KeyphraseSoundModel(@NonNull java.util.UUID, @NonNull java.util.UUID, @Nullable byte[], @Nullable android.hardware.soundtrigger.SoundTrigger.Keyphrase[], int);
ctor public SoundTrigger.KeyphraseSoundModel(@NonNull java.util.UUID, @NonNull java.util.UUID, @Nullable byte[], @Nullable android.hardware.soundtrigger.SoundTrigger.Keyphrase[]);
@@ -11758,6 +11767,7 @@ package android.service.voice {
method @Nullable public byte[] getData();
method public int getDataFormat();
method @Nullable public android.service.voice.HotwordDetectedResult getHotwordDetectedResult();
method @NonNull public java.util.List<android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionExtra> getKeyphraseRecognitionExtras();
method @Deprecated @Nullable public byte[] getTriggerAudio();
field public static final int DATA_FORMAT_RAW = 0; // 0x0
field public static final int DATA_FORMAT_TRIGGER_AUDIO = 1; // 0x1

17
core/api/test-current.txt
View File

@@ -1276,6 +1276,10 @@ package android.hardware.soundtrigger {
field @NonNull public static final android.os.Parcelable.Creator<android.hardware.soundtrigger.KeyphraseMetadata> CREATOR;
}
public static final class SoundTrigger.KeyphraseRecognitionExtra implements android.os.Parcelable {
ctor public SoundTrigger.KeyphraseRecognitionExtra(int, int, int);
}
public static final class SoundTrigger.ModelParamRange implements android.os.Parcelable {
ctor public SoundTrigger.ModelParamRange(int, int);
}
@@ -2398,8 +2402,17 @@ package android.service.voice {
method @RequiresPermission(allOf={android.Manifest.permission.RECORD_AUDIO, android.Manifest.permission.CAPTURE_AUDIO_HOTWORD}) public void triggerHardwareRecognitionEventForTest(int, int, boolean, int, int, int, boolean, @NonNull android.media.AudioFormat, @Nullable byte[]);
}
public static class AlwaysOnHotwordDetector.EventPayload {
ctor public AlwaysOnHotwordDetector.EventPayload(boolean, boolean, @Nullable android.media.AudioFormat, int, @Nullable byte[], @Nullable android.service.voice.HotwordDetectedResult, @Nullable android.os.ParcelFileDescriptor);
public static final class AlwaysOnHotwordDetector.EventPayload.Builder {
ctor public AlwaysOnHotwordDetector.EventPayload.Builder();
method @NonNull public android.service.voice.AlwaysOnHotwordDetector.EventPayload build();
method @NonNull public android.service.voice.AlwaysOnHotwordDetector.EventPayload.Builder setAudioStream(@NonNull android.os.ParcelFileDescriptor);
method @NonNull public android.service.voice.AlwaysOnHotwordDetector.EventPayload.Builder setCaptureAudioFormat(@NonNull android.media.AudioFormat);
method @NonNull public android.service.voice.AlwaysOnHotwordDetector.EventPayload.Builder setCaptureAvailable(boolean);
method @NonNull public android.service.voice.AlwaysOnHotwordDetector.EventPayload.Builder setCaptureSession(int);
method @NonNull public android.service.voice.AlwaysOnHotwordDetector.EventPayload.Builder setData(@NonNull byte[]);
method @NonNull public android.service.voice.AlwaysOnHotwordDetector.EventPayload.Builder setDataFormat(int);
method @NonNull public android.service.voice.AlwaysOnHotwordDetector.EventPayload.Builder setHotwordDetectedResult(@NonNull android.service.voice.HotwordDetectedResult);
method @NonNull public android.service.voice.AlwaysOnHotwordDetector.EventPayload.Builder setKeyphraseRecognitionExtras(@NonNull java.util.List<android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionExtra>);
}
public final class VisibleActivityInfo implements android.os.Parcelable {

117
core/java/android/hardware/soundtrigger/SoundTrigger.java
View File

@@ -61,6 +61,7 @@ import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Locale;
import java.util.UUID;
@@ -1576,31 +1577,58 @@ public class SoundTrigger {
}
/**
* Additional data conveyed by a {@link KeyphraseRecognitionEvent}
* for a key phrase detection.
*
* @hide
* Additional data conveyed by a {@link KeyphraseRecognitionEvent}
* for a key phrase detection.
*/
public static class KeyphraseRecognitionExtra implements Parcelable {
/** The keyphrase ID */
public static final class KeyphraseRecognitionExtra implements Parcelable {
/**
* The keyphrase ID
*
* @hide
*/
@UnsupportedAppUsage
public final int id;
/** Recognition modes matched for this event */
/**
* Recognition modes matched for this event
*
* @hide
*/
@UnsupportedAppUsage(maxTargetSdk = Build.VERSION_CODES.R, trackingBug = 170729553)
public final int recognitionModes;
/** Confidence level for mode RECOGNITION_MODE_VOICE_TRIGGER when user identification
* is not performed */
/**
* Confidence level for mode RECOGNITION_MODE_VOICE_TRIGGER when user identification
* is not performed
*
* @hide
*/
@UnsupportedAppUsage
public final int coarseConfidenceLevel;
/** Confidence levels for all users recognized (KeyphraseRecognitionEvent) or to
* be recognized (RecognitionConfig) */
/**
* Confidence levels for all users recognized (KeyphraseRecognitionEvent) or to
* be recognized (RecognitionConfig)
*
* @hide
*/
@UnsupportedAppUsage
@NonNull
public final ConfidenceLevel[] confidenceLevels;
/**
* @hide
*/
@TestApi
public KeyphraseRecognitionExtra(int id, @RecognitionModes int recognitionModes,
int coarseConfidenceLevel) {
this(id, recognitionModes, coarseConfidenceLevel, new ConfidenceLevel[0]);
}
/**
* @hide
*/
@UnsupportedAppUsage
public KeyphraseRecognitionExtra(int id, int recognitionModes, int coarseConfidenceLevel,
@Nullable ConfidenceLevel[] confidenceLevels) {
@@ -1611,7 +1639,47 @@ public class SoundTrigger {
confidenceLevels != null ? confidenceLevels : new ConfidenceLevel[0];
}
public static final @android.annotation.NonNull Parcelable.Creator<KeyphraseRecognitionExtra> CREATOR
/**
* The keyphrase ID associated with this class' additional data
*/
public int getKeyphraseId() {
return id;
}
/**
* Recognition modes matched for this event
*/
@RecognitionModes
public int getRecognitionModes() {
return recognitionModes;
}
/**
* Confidence level for mode RECOGNITION_MODE_VOICE_TRIGGER when user identification
* is not performed
*
* <p>The confidence level is expressed in percent (0% -100%).
*/
public int getCoarseConfidenceLevel() {
return coarseConfidenceLevel;
}
/**
* Detected confidence level for users defined in a keyphrase.
*
* <p>The confidence level is expressed in percent (0% -100%).
*
* <p>The user ID is derived from the system ID
* {@link android.os.UserHandle#getIdentifier()}.
*
* @hide
*/
@NonNull
public Collection<ConfidenceLevel> getConfidenceLevels() {
return Arrays.asList(confidenceLevels);
}
public static final @NonNull Parcelable.Creator<KeyphraseRecognitionExtra> CREATOR
= new Parcelable.Creator<KeyphraseRecognitionExtra>() {
public KeyphraseRecognitionExtra createFromParcel(Parcel in) {
return KeyphraseRecognitionExtra.fromParcel(in);
@@ -1632,7 +1700,7 @@ public class SoundTrigger {
}
@Override
public void writeToParcel(Parcel dest, int flags) {
public void writeToParcel(@NonNull Parcel dest, int flags) {
dest.writeInt(id);
dest.writeInt(recognitionModes);
dest.writeInt(coarseConfidenceLevel);
@@ -1657,21 +1725,28 @@ public class SoundTrigger {
@Override
public boolean equals(@Nullable Object obj) {
if (this == obj)
if (this == obj) {
return true;
if (obj == null)
}
if (obj == null) {
return false;
if (getClass() != obj.getClass())
}
if (getClass() != obj.getClass()) {
return false;
}
KeyphraseRecognitionExtra other = (KeyphraseRecognitionExtra) obj;
if (!Arrays.equals(confidenceLevels, other.confidenceLevels))
if (!Arrays.equals(confidenceLevels, other.confidenceLevels)) {
return false;
if (id != other.id)
}
if (id != other.id) {
return false;
if (recognitionModes != other.recognitionModes)
}
if (recognitionModes != other.recognitionModes) {
return false;
if (coarseConfidenceLevel != other.coarseConfidenceLevel)
}
if (coarseConfidenceLevel != other.coarseConfidenceLevel) {
return false;
}
return true;
}
@@ -1715,7 +1790,7 @@ public class SoundTrigger {
keyphraseExtras != null ? keyphraseExtras : new KeyphraseRecognitionExtra[0];
}
public static final @android.annotation.NonNull Parcelable.Creator<KeyphraseRecognitionEvent> CREATOR
public static final @NonNull Parcelable.Creator<KeyphraseRecognitionEvent> CREATOR
= new Parcelable.Creator<KeyphraseRecognitionEvent>() {
public KeyphraseRecognitionEvent createFromParcel(Parcel in) {
return KeyphraseRecognitionEvent.fromParcelForKeyphrase(in);

5
core/java/android/service/voice/AbstractHotwordDetector.java
View File

@@ -146,7 +146,10 @@ abstract class AbstractHotwordDetector implements HotwordDetector {
mHandler.sendMessage(obtainMessage(
HotwordDetector.Callback::onDetected,
mCallback,
new AlwaysOnHotwordDetector.EventPayload(audioFormat, hotwordDetectedResult)));
new AlwaysOnHotwordDetector.EventPayload.Builder()
.setCaptureAudioFormat(audioFormat)
.setHotwordDetectedResult(hotwordDetectedResult)
.build()));
}
}
}

212
core/java/android/service/voice/AlwaysOnHotwordDetector.java
View File

@@ -23,6 +23,7 @@ import android.annotation.IntDef;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.annotation.RequiresPermission;
import android.annotation.SuppressLint;
import android.annotation.SystemApi;
import android.annotation.TestApi;
import android.app.ActivityThread;
@@ -59,6 +60,9 @@ import com.android.internal.app.IVoiceInteractionSoundTriggerSession;
import java.io.PrintWriter;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
/**
@@ -380,47 +384,24 @@ public class AlwaysOnHotwordDetector extends AbstractHotwordDetector {
private final byte[] mData;
private final HotwordDetectedResult mHotwordDetectedResult;
private final ParcelFileDescriptor mAudioStream;
private final List<KeyphraseRecognitionExtra> mKephraseExtras;
EventPayload(boolean triggerInData, boolean captureAvailable,
AudioFormat audioFormat, int captureSession, byte[] data) {
this(triggerInData, captureAvailable, audioFormat, captureSession, data, null,
null);
}
EventPayload(boolean triggerInData, boolean captureAvailable,
AudioFormat audioFormat, int captureSession, byte[] data,
HotwordDetectedResult hotwordDetectedResult) {
this(triggerInData, captureAvailable, audioFormat, captureSession, data,
hotwordDetectedResult, null);
}
EventPayload(AudioFormat audioFormat, HotwordDetectedResult hotwordDetectedResult) {
this(false, false, audioFormat, -1, null, hotwordDetectedResult, null);
}
EventPayload(AudioFormat audioFormat,
HotwordDetectedResult hotwordDetectedResult,
ParcelFileDescriptor audioStream) {
this(false, false, audioFormat, -1, null, hotwordDetectedResult, audioStream);
}
/** @hide */
@TestApi
public EventPayload(boolean triggerInData, boolean captureAvailable,
@Nullable AudioFormat audioFormat, int captureSession, @Nullable byte[] data,
private EventPayload(boolean captureAvailable,
@Nullable AudioFormat audioFormat,
int captureSession,
@DataFormat int dataFormat,
@Nullable byte[] data,
@Nullable HotwordDetectedResult hotwordDetectedResult,
@Nullable ParcelFileDescriptor audioStream) {
@Nullable ParcelFileDescriptor audioStream,
@NonNull List<KeyphraseRecognitionExtra> keyphraseExtras) {
mCaptureAvailable = captureAvailable;
mCaptureSession = captureSession;
mAudioFormat = audioFormat;
if (triggerInData) {
mDataFormat = DATA_FORMAT_TRIGGER_AUDIO;
} else {
mDataFormat = DATA_FORMAT_RAW;
}
mDataFormat = dataFormat;
mData = data;
mHotwordDetectedResult = hotwordDetectedResult;
mAudioStream = audioStream;
mKephraseExtras = keyphraseExtras;
}
/**
@@ -535,6 +516,166 @@ public class AlwaysOnHotwordDetector extends AbstractHotwordDetector {
public ParcelFileDescriptor getAudioStream() {
return mAudioStream;
}
/**
* Returns the keyphrases recognized by the voice engine with additional confidence
* information
*
* @return List of keyphrase extras describing additional data for each keyphrase the voice
* engine triggered on for this event. The ordering of the list is preserved based on what
* the ordering provided by {@link android.hardware.soundtrigger.SoundTriggerModule}.
*/
@NonNull
public List<KeyphraseRecognitionExtra> getKeyphraseRecognitionExtras() {
return mKephraseExtras;
}
/**
* Builder class for {@link EventPayload} objects
*
* @hide
*/
@TestApi
public static final class Builder {
private boolean mCaptureAvailable = false;
private int mCaptureSession = -1;
private AudioFormat mAudioFormat = null;
@DataFormat
private int mDataFormat = DATA_FORMAT_RAW;
private byte[] mData = null;
private HotwordDetectedResult mHotwordDetectedResult = null;
private ParcelFileDescriptor mAudioStream = null;
private List<KeyphraseRecognitionExtra> mKeyphraseExtras = Collections.emptyList();
public Builder() {}
Builder(SoundTrigger.KeyphraseRecognitionEvent keyphraseRecognitionEvent) {
setCaptureAvailable(keyphraseRecognitionEvent.isCaptureAvailable());
setCaptureSession(keyphraseRecognitionEvent.getCaptureSession());
if (keyphraseRecognitionEvent.getCaptureFormat() != null) {
setCaptureAudioFormat(keyphraseRecognitionEvent.getCaptureFormat());
}
setDataFormat((keyphraseRecognitionEvent.triggerInData) ? DATA_FORMAT_TRIGGER_AUDIO
: DATA_FORMAT_RAW);
if (keyphraseRecognitionEvent.getData() != null) {
setData(keyphraseRecognitionEvent.getData());
}
if (keyphraseRecognitionEvent.keyphraseExtras != null) {
setKeyphraseRecognitionExtras(
Arrays.asList(keyphraseRecognitionEvent.keyphraseExtras));
}
}
/**
* Indicates if {@code captureSession} can be used to continue capturing more audio from
* the DSP hardware.
*/
@SuppressLint("MissingGetterMatchingBuilder")
@NonNull
public Builder setCaptureAvailable(boolean captureAvailable) {
mCaptureAvailable = captureAvailable;
return this;
}
/**
* Sets the session ID to start a capture from the DSP.
*/
@SuppressLint("MissingGetterMatchingBuilder")
@NonNull
public Builder setCaptureSession(int captureSession) {
mCaptureSession = captureSession;
return this;
}
/**
* Sets the format of the audio obtained using {@link #getTriggerAudio()}.
*/
@NonNull
public Builder setCaptureAudioFormat(@NonNull AudioFormat audioFormat) {
mAudioFormat = audioFormat;
return this;
}
/**
* Conveys the format of the additional data that is triggered with the keyphrase event.
*
* @see AlwaysOnHotwordDetector#RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO
* @see DataFormat
*/
@NonNull
public Builder setDataFormat(@DataFormat int dataFormat) {
mDataFormat = dataFormat;
return this;
}
/**
* Sets additional raw data that is triggered with the keyphrase event.
*
* <p>A {@link android.hardware.soundtrigger.SoundTriggerModule} may populate this
* field with opaque data for use by system applications who know about voice
* engine internals. Data may be null if the field is not populated by the
* {@link android.hardware.soundtrigger.SoundTriggerModule}.
*
* <p>If {@link #getDataFormat()} is {@link #DATA_FORMAT_TRIGGER_AUDIO}, then the
* entirety of this
* buffer is expected to be of the format from {@link #getCaptureAudioFormat()}.
*
* @see AlwaysOnHotwordDetector#RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO
*/
@NonNull
public Builder setData(@NonNull byte[] data) {
mData = data;
return this;
}
/**
* Sets {@link HotwordDetectedResult} associated with the hotword event, passed from
* {@link HotwordDetectionService}.
*/
@NonNull
public Builder setHotwordDetectedResult(
@NonNull HotwordDetectedResult hotwordDetectedResult) {
mHotwordDetectedResult = hotwordDetectedResult;
return this;
}
/**
* Sets a stream with bytes corresponding to the open audio stream with hotword data.
*
* <p>This data represents an audio stream in the format returned by
* {@link #getCaptureAudioFormat}.
*
* <p>Clients are expected to start consuming the stream within 1 second of receiving
* the
* event.
*/
@NonNull
public Builder setAudioStream(@NonNull ParcelFileDescriptor audioStream) {
mAudioStream = audioStream;
return this;
}
/**
* Sets the keyphrases recognized by the voice engine with additional confidence
* information
*/
@NonNull
public Builder setKeyphraseRecognitionExtras(
@NonNull List<KeyphraseRecognitionExtra> keyphraseRecognitionExtras) {
mKeyphraseExtras = keyphraseRecognitionExtras;
return this;
}
/**
* Builds an {@link EventPayload} instance
*/
@NonNull
public EventPayload build() {
return new EventPayload(mCaptureAvailable, mAudioFormat, mCaptureSession,
mDataFormat, mData, mHotwordDetectedResult, mAudioStream,
mKeyphraseExtras);
}
}
}
/**
@@ -1242,8 +1383,9 @@ public class AlwaysOnHotwordDetector extends AbstractHotwordDetector {
Slog.i(TAG, "onDetected");
}
Message.obtain(mHandler, MSG_HOTWORD_DETECTED,
new EventPayload(event.triggerInData, event.captureAvailable,
event.captureFormat, event.captureSession, event.data, result))
new EventPayload.Builder(event)
.setHotwordDetectedResult(result)
.build())
.sendToTarget();
}
@Override

4
core/java/android/service/voice/HotwordDetectionService.java
View File

@@ -140,9 +140,7 @@ public abstract class HotwordDetectionService extends Service {
Log.d(TAG, "#detectFromDspSource");
}
HotwordDetectionService.this.onDetect(
new AlwaysOnHotwordDetector.EventPayload(
event.triggerInData, event.captureAvailable,
event.captureFormat, event.captureSession, event.data),
new AlwaysOnHotwordDetector.EventPayload.Builder(event).build(),
timeoutMillis,
new Callback(callback));
}

7
core/java/android/service/voice/SoftwareHotwordDetector.java
View File

@@ -135,8 +135,11 @@ class SoftwareHotwordDetector extends AbstractHotwordDetector {
mHandler.sendMessage(obtainMessage(
HotwordDetector.Callback::onDetected,
mCallback,
new AlwaysOnHotwordDetector.EventPayload(
audioFormat, hotwordDetectedResult, audioStream)));
new AlwaysOnHotwordDetector.EventPayload.Builder()
.setCaptureAudioFormat(audioFormat)
.setAudioStream(audioStream)
.setHotwordDetectedResult(hotwordDetectedResult)
.build()));
}
}