diff --git a/core/api/system-current.txt b/core/api/system-current.txt index 8aef638411b79..5e5f3871782ca 100644 --- a/core/api/system-current.txt +++ b/core/api/system-current.txt @@ -5176,6 +5176,15 @@ package android.hardware.soundtrigger { field @NonNull public static final android.os.Parcelable.Creator CREATOR; } + public static final class SoundTrigger.KeyphraseRecognitionExtra implements android.os.Parcelable { + method public int describeContents(); + method public int getCoarseConfidenceLevel(); + method public int getKeyphraseId(); + method public int getRecognitionModes(); + method public void writeToParcel(@NonNull android.os.Parcel, int); + field @NonNull public static final android.os.Parcelable.Creator CREATOR; + } + public static final class SoundTrigger.KeyphraseSoundModel extends android.hardware.soundtrigger.SoundTrigger.SoundModel implements android.os.Parcelable { ctor public SoundTrigger.KeyphraseSoundModel(@NonNull java.util.UUID, @NonNull java.util.UUID, @Nullable byte[], @Nullable android.hardware.soundtrigger.SoundTrigger.Keyphrase[], int); ctor public SoundTrigger.KeyphraseSoundModel(@NonNull java.util.UUID, @NonNull java.util.UUID, @Nullable byte[], @Nullable android.hardware.soundtrigger.SoundTrigger.Keyphrase[]); @@ -11760,8 +11769,13 @@ package android.service.voice { public static class AlwaysOnHotwordDetector.EventPayload { method @Nullable public android.os.ParcelFileDescriptor getAudioStream(); method @Nullable public android.media.AudioFormat getCaptureAudioFormat(); + method @Nullable public byte[] getData(); + method public int getDataFormat(); method @Nullable public android.service.voice.HotwordDetectedResult getHotwordDetectedResult(); - method @Nullable public byte[] getTriggerAudio(); + method @NonNull public java.util.List getKeyphraseRecognitionExtras(); + method @Deprecated @Nullable public byte[] getTriggerAudio(); + field public static final int DATA_FORMAT_RAW = 0; // 0x0 + field public static final int DATA_FORMAT_TRIGGER_AUDIO = 1; // 0x1 } public static final class AlwaysOnHotwordDetector.ModelParamRange { @@ -11832,6 +11846,7 @@ package android.service.voice { } public interface HotwordDetector { + method public default void destroy(); method @RequiresPermission(allOf={android.Manifest.permission.RECORD_AUDIO, android.Manifest.permission.CAPTURE_AUDIO_HOTWORD}) public boolean startRecognition(); method public boolean startRecognition(@NonNull android.os.ParcelFileDescriptor, @NonNull android.media.AudioFormat, @Nullable android.os.PersistableBundle); method public boolean stopRecognition(); diff --git a/core/api/test-current.txt b/core/api/test-current.txt index dad7b48d15dd7..d683190f46e2d 100644 --- a/core/api/test-current.txt +++ b/core/api/test-current.txt @@ -1279,6 +1279,10 @@ package android.hardware.soundtrigger { field @NonNull public static final android.os.Parcelable.Creator CREATOR; } + public static final class SoundTrigger.KeyphraseRecognitionExtra implements android.os.Parcelable { + ctor public SoundTrigger.KeyphraseRecognitionExtra(int, int, int); + } + public static final class SoundTrigger.ModelParamRange implements android.os.Parcelable { ctor public SoundTrigger.ModelParamRange(int, int); } @@ -2401,6 +2405,19 @@ package android.service.voice { method @RequiresPermission(allOf={android.Manifest.permission.RECORD_AUDIO, android.Manifest.permission.CAPTURE_AUDIO_HOTWORD}) public void triggerHardwareRecognitionEventForTest(int, int, boolean, int, int, int, boolean, @NonNull android.media.AudioFormat, @Nullable byte[]); } + public static final class AlwaysOnHotwordDetector.EventPayload.Builder { + ctor public AlwaysOnHotwordDetector.EventPayload.Builder(); + method @NonNull public android.service.voice.AlwaysOnHotwordDetector.EventPayload build(); + method @NonNull public android.service.voice.AlwaysOnHotwordDetector.EventPayload.Builder setAudioStream(@NonNull android.os.ParcelFileDescriptor); + method @NonNull public android.service.voice.AlwaysOnHotwordDetector.EventPayload.Builder setCaptureAudioFormat(@NonNull android.media.AudioFormat); + method @NonNull public android.service.voice.AlwaysOnHotwordDetector.EventPayload.Builder setCaptureAvailable(boolean); + method @NonNull public android.service.voice.AlwaysOnHotwordDetector.EventPayload.Builder setCaptureSession(int); + method @NonNull public android.service.voice.AlwaysOnHotwordDetector.EventPayload.Builder setData(@NonNull byte[]); + method @NonNull public android.service.voice.AlwaysOnHotwordDetector.EventPayload.Builder setDataFormat(int); + method @NonNull public android.service.voice.AlwaysOnHotwordDetector.EventPayload.Builder setHotwordDetectedResult(@NonNull android.service.voice.HotwordDetectedResult); + method @NonNull public android.service.voice.AlwaysOnHotwordDetector.EventPayload.Builder setKeyphraseRecognitionExtras(@NonNull java.util.List); + } + public final class VisibleActivityInfo implements android.os.Parcelable { ctor public VisibleActivityInfo(int, @NonNull android.os.IBinder); } diff --git a/core/java/android/hardware/soundtrigger/SoundTrigger.java b/core/java/android/hardware/soundtrigger/SoundTrigger.java index b0439d0e52536..c36390917cf18 100644 --- a/core/java/android/hardware/soundtrigger/SoundTrigger.java +++ b/core/java/android/hardware/soundtrigger/SoundTrigger.java @@ -61,6 +61,7 @@ import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Locale; import java.util.UUID; @@ -1576,31 +1577,58 @@ public class SoundTrigger { } /** - * Additional data conveyed by a {@link KeyphraseRecognitionEvent} - * for a key phrase detection. - * - * @hide + * Additional data conveyed by a {@link KeyphraseRecognitionEvent} + * for a key phrase detection. */ - public static class KeyphraseRecognitionExtra implements Parcelable { - /** The keyphrase ID */ + public static final class KeyphraseRecognitionExtra implements Parcelable { + /** + * The keyphrase ID + * + * @hide + */ @UnsupportedAppUsage public final int id; - /** Recognition modes matched for this event */ + /** + * Recognition modes matched for this event + * + * @hide + */ @UnsupportedAppUsage(maxTargetSdk = Build.VERSION_CODES.R, trackingBug = 170729553) public final int recognitionModes; - /** Confidence level for mode RECOGNITION_MODE_VOICE_TRIGGER when user identification - * is not performed */ + /** + * Confidence level for mode RECOGNITION_MODE_VOICE_TRIGGER when user identification + * is not performed + * + * @hide + */ @UnsupportedAppUsage public final int coarseConfidenceLevel; - /** Confidence levels for all users recognized (KeyphraseRecognitionEvent) or to - * be recognized (RecognitionConfig) */ + /** + * Confidence levels for all users recognized (KeyphraseRecognitionEvent) or to + * be recognized (RecognitionConfig) + * + * @hide + */ @UnsupportedAppUsage @NonNull public final ConfidenceLevel[] confidenceLevels; + + /** + * @hide + */ + @TestApi + public KeyphraseRecognitionExtra(int id, @RecognitionModes int recognitionModes, + int coarseConfidenceLevel) { + this(id, recognitionModes, coarseConfidenceLevel, new ConfidenceLevel[0]); + } + + /** + * @hide + */ @UnsupportedAppUsage public KeyphraseRecognitionExtra(int id, int recognitionModes, int coarseConfidenceLevel, @Nullable ConfidenceLevel[] confidenceLevels) { @@ -1611,7 +1639,47 @@ public class SoundTrigger { confidenceLevels != null ? confidenceLevels : new ConfidenceLevel[0]; } - public static final @android.annotation.NonNull Parcelable.Creator CREATOR + /** + * The keyphrase ID associated with this class' additional data + */ + public int getKeyphraseId() { + return id; + } + + /** + * Recognition modes matched for this event + */ + @RecognitionModes + public int getRecognitionModes() { + return recognitionModes; + } + + /** + * Confidence level for mode RECOGNITION_MODE_VOICE_TRIGGER when user identification + * is not performed + * + *

The confidence level is expressed in percent (0% -100%). + */ + public int getCoarseConfidenceLevel() { + return coarseConfidenceLevel; + } + + /** + * Detected confidence level for users defined in a keyphrase. + * + *

The confidence level is expressed in percent (0% -100%). + * + *

The user ID is derived from the system ID + * {@link android.os.UserHandle#getIdentifier()}. + * + * @hide + */ + @NonNull + public Collection getConfidenceLevels() { + return Arrays.asList(confidenceLevels); + } + + public static final @NonNull Parcelable.Creator CREATOR = new Parcelable.Creator() { public KeyphraseRecognitionExtra createFromParcel(Parcel in) { return KeyphraseRecognitionExtra.fromParcel(in); @@ -1632,7 +1700,7 @@ public class SoundTrigger { } @Override - public void writeToParcel(Parcel dest, int flags) { + public void writeToParcel(@NonNull Parcel dest, int flags) { dest.writeInt(id); dest.writeInt(recognitionModes); dest.writeInt(coarseConfidenceLevel); @@ -1657,21 +1725,28 @@ public class SoundTrigger { @Override public boolean equals(@Nullable Object obj) { - if (this == obj) + if (this == obj) { return true; - if (obj == null) + } + if (obj == null) { return false; - if (getClass() != obj.getClass()) + } + if (getClass() != obj.getClass()) { return false; + } KeyphraseRecognitionExtra other = (KeyphraseRecognitionExtra) obj; - if (!Arrays.equals(confidenceLevels, other.confidenceLevels)) + if (!Arrays.equals(confidenceLevels, other.confidenceLevels)) { return false; - if (id != other.id) + } + if (id != other.id) { return false; - if (recognitionModes != other.recognitionModes) + } + if (recognitionModes != other.recognitionModes) { return false; - if (coarseConfidenceLevel != other.coarseConfidenceLevel) + } + if (coarseConfidenceLevel != other.coarseConfidenceLevel) { return false; + } return true; } @@ -1715,7 +1790,7 @@ public class SoundTrigger { keyphraseExtras != null ? keyphraseExtras : new KeyphraseRecognitionExtra[0]; } - public static final @android.annotation.NonNull Parcelable.Creator CREATOR + public static final @NonNull Parcelable.Creator CREATOR = new Parcelable.Creator() { public KeyphraseRecognitionEvent createFromParcel(Parcel in) { return KeyphraseRecognitionEvent.fromParcelForKeyphrase(in); diff --git a/core/java/android/service/voice/AbstractHotwordDetector.java b/core/java/android/service/voice/AbstractHotwordDetector.java index 192260791a8b3..01d5638461af8 100644 --- a/core/java/android/service/voice/AbstractHotwordDetector.java +++ b/core/java/android/service/voice/AbstractHotwordDetector.java @@ -18,6 +18,7 @@ package android.service.voice; import static com.android.internal.util.function.pooled.PooledLambda.obtainMessage; +import android.annotation.CallSuper; import android.annotation.NonNull; import android.annotation.Nullable; import android.app.ActivityThread; @@ -34,6 +35,9 @@ import android.util.Slog; import com.android.internal.app.IHotwordRecognitionStatusCallback; import com.android.internal.app.IVoiceInteractionManagerService; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Consumer; + /** Base implementation of {@link HotwordDetector}. */ abstract class AbstractHotwordDetector implements HotwordDetector { private static final String TAG = AbstractHotwordDetector.class.getSimpleName(); @@ -45,6 +49,8 @@ abstract class AbstractHotwordDetector implements HotwordDetector { private final Handler mHandler; private final HotwordDetector.Callback mCallback; private final int mDetectorType; + private Consumer mOnDestroyListener; + private final AtomicBoolean mIsDetectorActive; AbstractHotwordDetector( IVoiceInteractionManagerService managerService, @@ -55,6 +61,7 @@ abstract class AbstractHotwordDetector implements HotwordDetector { mHandler = new Handler(Looper.getMainLooper()); mCallback = callback; mDetectorType = detectorType; + mIsDetectorActive = new AtomicBoolean(true); } /** @@ -70,6 +77,7 @@ abstract class AbstractHotwordDetector implements HotwordDetector { if (DEBUG) { Slog.i(TAG, "#recognizeHotword"); } + throwIfDetectorIsNoLongerActive(); // TODO: consider closing existing session. @@ -106,6 +114,7 @@ abstract class AbstractHotwordDetector implements HotwordDetector { if (DEBUG) { Slog.d(TAG, "updateState()"); } + throwIfDetectorIsNoLongerActive(); synchronized (mLock) { updateStateLocked(options, sharedMemory, null /* callback */, mDetectorType); } @@ -126,6 +135,35 @@ abstract class AbstractHotwordDetector implements HotwordDetector { } } + void registerOnDestroyListener(Consumer onDestroyListener) { + synchronized (mLock) { + if (mOnDestroyListener != null) { + throw new IllegalStateException("only one destroy listener can be registered"); + } + mOnDestroyListener = onDestroyListener; + } + } + + @CallSuper + @Override + public void destroy() { + if (!mIsDetectorActive.get()) { + return; + } + mIsDetectorActive.set(false); + synchronized (mLock) { + mOnDestroyListener.accept(this); + } + } + + protected void throwIfDetectorIsNoLongerActive() { + if (!mIsDetectorActive.get()) { + Slog.e(TAG, "attempting to use a destroyed detector which is no longer active"); + throw new IllegalStateException( + "attempting to use a destroyed detector which is no longer active"); + } + } + private static class BinderCallback extends IMicrophoneHotwordDetectionVoiceInteractionCallback.Stub { private final Handler mHandler; @@ -146,7 +184,10 @@ abstract class AbstractHotwordDetector implements HotwordDetector { mHandler.sendMessage(obtainMessage( HotwordDetector.Callback::onDetected, mCallback, - new AlwaysOnHotwordDetector.EventPayload(audioFormat, hotwordDetectedResult))); + new AlwaysOnHotwordDetector.EventPayload.Builder() + .setCaptureAudioFormat(audioFormat) + .setHotwordDetectedResult(hotwordDetectedResult) + .build())); } } } diff --git a/core/java/android/service/voice/AlwaysOnHotwordDetector.java b/core/java/android/service/voice/AlwaysOnHotwordDetector.java index c9daf52b56853..bec5d1be57fd1 100644 --- a/core/java/android/service/voice/AlwaysOnHotwordDetector.java +++ b/core/java/android/service/voice/AlwaysOnHotwordDetector.java @@ -23,6 +23,7 @@ import android.annotation.IntDef; import android.annotation.NonNull; import android.annotation.Nullable; import android.annotation.RequiresPermission; +import android.annotation.SuppressLint; import android.annotation.SystemApi; import android.annotation.TestApi; import android.app.ActivityThread; @@ -59,6 +60,9 @@ import com.android.internal.app.IVoiceInteractionSoundTriggerSession; import java.io.PrintWriter; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; import java.util.Locale; /** @@ -336,7 +340,39 @@ public class AlwaysOnHotwordDetector extends AbstractHotwordDetector { * Additional payload for {@link Callback#onDetected}. */ public static class EventPayload { - private final boolean mTriggerAvailable; + + /** + * Flags for describing the data format provided in the event payload. + * + * @hide + */ + @Retention(RetentionPolicy.SOURCE) + @IntDef(prefix = {"DATA_FORMAT_"}, value = { + DATA_FORMAT_RAW, + DATA_FORMAT_TRIGGER_AUDIO, + }) + public @interface DataFormat { + } + + /** + * Data format is not strictly defined by the framework, and the + * {@link android.hardware.soundtrigger.SoundTriggerModule} voice engine may populate this + * field in any format. + */ + public static final int DATA_FORMAT_RAW = 0; + + /** + * Data format is defined as trigger audio. + * + *

When this format is used, {@link #getCaptureAudioFormat()} can be used to understand + * further the audio format for reading the data. + * + * @see AlwaysOnHotwordDetector#RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO + */ + public static final int DATA_FORMAT_TRIGGER_AUDIO = 1; + + @DataFormat + private final int mDataFormat; // Indicates if {@code captureSession} can be used to continue capturing more audio // from the DSP hardware. private final boolean mCaptureAvailable; @@ -348,40 +384,24 @@ public class AlwaysOnHotwordDetector extends AbstractHotwordDetector { private final byte[] mData; private final HotwordDetectedResult mHotwordDetectedResult; private final ParcelFileDescriptor mAudioStream; + private final List mKephraseExtras; - EventPayload(boolean triggerAvailable, boolean captureAvailable, - AudioFormat audioFormat, int captureSession, byte[] data) { - this(triggerAvailable, captureAvailable, audioFormat, captureSession, data, null, - null); - } - - EventPayload(boolean triggerAvailable, boolean captureAvailable, - AudioFormat audioFormat, int captureSession, byte[] data, - HotwordDetectedResult hotwordDetectedResult) { - this(triggerAvailable, captureAvailable, audioFormat, captureSession, data, - hotwordDetectedResult, null); - } - - EventPayload(AudioFormat audioFormat, HotwordDetectedResult hotwordDetectedResult) { - this(false, false, audioFormat, -1, null, hotwordDetectedResult, null); - } - - EventPayload(AudioFormat audioFormat, - HotwordDetectedResult hotwordDetectedResult, - ParcelFileDescriptor audioStream) { - this(false, false, audioFormat, -1, null, hotwordDetectedResult, audioStream); - } - - private EventPayload(boolean triggerAvailable, boolean captureAvailable, - AudioFormat audioFormat, int captureSession, byte[] data, - HotwordDetectedResult hotwordDetectedResult, ParcelFileDescriptor audioStream) { - mTriggerAvailable = triggerAvailable; + private EventPayload(boolean captureAvailable, + @Nullable AudioFormat audioFormat, + int captureSession, + @DataFormat int dataFormat, + @Nullable byte[] data, + @Nullable HotwordDetectedResult hotwordDetectedResult, + @Nullable ParcelFileDescriptor audioStream, + @NonNull List keyphraseExtras) { mCaptureAvailable = captureAvailable; mCaptureSession = captureSession; mAudioFormat = audioFormat; + mDataFormat = dataFormat; mData = data; mHotwordDetectedResult = hotwordDetectedResult; mAudioStream = audioStream; + mKephraseExtras = keyphraseExtras; } /** @@ -400,16 +420,48 @@ public class AlwaysOnHotwordDetector extends AbstractHotwordDetector { * {@link #getCaptureAudioFormat()}. * * @see AlwaysOnHotwordDetector#RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO + * @deprecated Use {@link #getData()} instead. */ + @Deprecated @Nullable public byte[] getTriggerAudio() { - if (mTriggerAvailable) { + if (mDataFormat == DATA_FORMAT_TRIGGER_AUDIO) { return mData; } else { return null; } } + /** + * Conveys the format of the additional data that is triggered with the keyphrase event. + * + * @see AlwaysOnHotwordDetector#RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO + * @see DataFormat + */ + @DataFormat + public int getDataFormat() { + return mDataFormat; + } + + /** + * Gets additional raw data that is triggered with the keyphrase event. + * + *

A {@link android.hardware.soundtrigger.SoundTriggerModule} may populate this + * field with opaque data for use by system applications who know about voice + * engine internals. Data may be null if the field is not populated by the + * {@link android.hardware.soundtrigger.SoundTriggerModule}. + * + *

If {@link #getDataFormat()} is {@link #DATA_FORMAT_TRIGGER_AUDIO}, then the + * entirety of this buffer is expected to be of the format from + * {@link #getCaptureAudioFormat()}. + * + * @see AlwaysOnHotwordDetector#RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO + */ + @Nullable + public byte[] getData() { + return mData; + } + /** * Gets the session ID to start a capture from the DSP. * This may be null if streaming capture isn't possible. @@ -464,6 +516,166 @@ public class AlwaysOnHotwordDetector extends AbstractHotwordDetector { public ParcelFileDescriptor getAudioStream() { return mAudioStream; } + + /** + * Returns the keyphrases recognized by the voice engine with additional confidence + * information + * + * @return List of keyphrase extras describing additional data for each keyphrase the voice + * engine triggered on for this event. The ordering of the list is preserved based on what + * the ordering provided by {@link android.hardware.soundtrigger.SoundTriggerModule}. + */ + @NonNull + public List getKeyphraseRecognitionExtras() { + return mKephraseExtras; + } + + /** + * Builder class for {@link EventPayload} objects + * + * @hide + */ + @TestApi + public static final class Builder { + private boolean mCaptureAvailable = false; + private int mCaptureSession = -1; + private AudioFormat mAudioFormat = null; + @DataFormat + private int mDataFormat = DATA_FORMAT_RAW; + private byte[] mData = null; + private HotwordDetectedResult mHotwordDetectedResult = null; + private ParcelFileDescriptor mAudioStream = null; + private List mKeyphraseExtras = Collections.emptyList(); + + public Builder() {} + + Builder(SoundTrigger.KeyphraseRecognitionEvent keyphraseRecognitionEvent) { + setCaptureAvailable(keyphraseRecognitionEvent.isCaptureAvailable()); + setCaptureSession(keyphraseRecognitionEvent.getCaptureSession()); + if (keyphraseRecognitionEvent.getCaptureFormat() != null) { + setCaptureAudioFormat(keyphraseRecognitionEvent.getCaptureFormat()); + } + setDataFormat((keyphraseRecognitionEvent.triggerInData) ? DATA_FORMAT_TRIGGER_AUDIO + : DATA_FORMAT_RAW); + if (keyphraseRecognitionEvent.getData() != null) { + setData(keyphraseRecognitionEvent.getData()); + } + if (keyphraseRecognitionEvent.keyphraseExtras != null) { + setKeyphraseRecognitionExtras( + Arrays.asList(keyphraseRecognitionEvent.keyphraseExtras)); + } + } + + /** + * Indicates if {@code captureSession} can be used to continue capturing more audio from + * the DSP hardware. + */ + @SuppressLint("MissingGetterMatchingBuilder") + @NonNull + public Builder setCaptureAvailable(boolean captureAvailable) { + mCaptureAvailable = captureAvailable; + return this; + } + + /** + * Sets the session ID to start a capture from the DSP. + */ + @SuppressLint("MissingGetterMatchingBuilder") + @NonNull + public Builder setCaptureSession(int captureSession) { + mCaptureSession = captureSession; + return this; + } + + /** + * Sets the format of the audio obtained using {@link #getTriggerAudio()}. + */ + @NonNull + public Builder setCaptureAudioFormat(@NonNull AudioFormat audioFormat) { + mAudioFormat = audioFormat; + return this; + } + + /** + * Conveys the format of the additional data that is triggered with the keyphrase event. + * + * @see AlwaysOnHotwordDetector#RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO + * @see DataFormat + */ + @NonNull + public Builder setDataFormat(@DataFormat int dataFormat) { + mDataFormat = dataFormat; + return this; + } + + /** + * Sets additional raw data that is triggered with the keyphrase event. + * + *

A {@link android.hardware.soundtrigger.SoundTriggerModule} may populate this + * field with opaque data for use by system applications who know about voice + * engine internals. Data may be null if the field is not populated by the + * {@link android.hardware.soundtrigger.SoundTriggerModule}. + * + *

If {@link #getDataFormat()} is {@link #DATA_FORMAT_TRIGGER_AUDIO}, then the + * entirety of this + * buffer is expected to be of the format from {@link #getCaptureAudioFormat()}. + * + * @see AlwaysOnHotwordDetector#RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO + */ + @NonNull + public Builder setData(@NonNull byte[] data) { + mData = data; + return this; + } + + /** + * Sets {@link HotwordDetectedResult} associated with the hotword event, passed from + * {@link HotwordDetectionService}. + */ + @NonNull + public Builder setHotwordDetectedResult( + @NonNull HotwordDetectedResult hotwordDetectedResult) { + mHotwordDetectedResult = hotwordDetectedResult; + return this; + } + + /** + * Sets a stream with bytes corresponding to the open audio stream with hotword data. + * + *

This data represents an audio stream in the format returned by + * {@link #getCaptureAudioFormat}. + * + *

Clients are expected to start consuming the stream within 1 second of receiving + * the + * event. + */ + @NonNull + public Builder setAudioStream(@NonNull ParcelFileDescriptor audioStream) { + mAudioStream = audioStream; + return this; + } + + /** + * Sets the keyphrases recognized by the voice engine with additional confidence + * information + */ + @NonNull + public Builder setKeyphraseRecognitionExtras( + @NonNull List keyphraseRecognitionExtras) { + mKeyphraseExtras = keyphraseRecognitionExtras; + return this; + } + + /** + * Builds an {@link EventPayload} instance + */ + @NonNull + public EventPayload build() { + return new EventPayload(mCaptureAvailable, mAudioFormat, mCaptureSession, + mDataFormat, mData, mHotwordDetectedResult, mAudioStream, + mKeyphraseExtras); + } + } } /** @@ -993,11 +1205,14 @@ public class AlwaysOnHotwordDetector extends AbstractHotwordDetector { /** * Invalidates this hotword detector so that any future calls to this result * in an IllegalStateException. - * - * @hide */ - void invalidate() { + @Override + public void destroy() { synchronized (mLock) { + if (mAvailability == STATE_KEYPHRASE_ENROLLED) { + stopRecognition(); + } + mAvailability = STATE_INVALID; notifyStateChangedLocked(); @@ -1009,6 +1224,7 @@ public class AlwaysOnHotwordDetector extends AbstractHotwordDetector { } } } + super.destroy(); } /** @@ -1171,8 +1387,9 @@ public class AlwaysOnHotwordDetector extends AbstractHotwordDetector { Slog.i(TAG, "onDetected"); } Message.obtain(mHandler, MSG_HOTWORD_DETECTED, - new EventPayload(event.triggerInData, event.captureAvailable, - event.captureFormat, event.captureSession, event.data, result)) + new EventPayload.Builder(event) + .setHotwordDetectedResult(result) + .build()) .sendToTarget(); } @Override diff --git a/core/java/android/service/voice/HotwordDetectionService.java b/core/java/android/service/voice/HotwordDetectionService.java index e3bb589c9a19d..dfe0f542b3cab 100644 --- a/core/java/android/service/voice/HotwordDetectionService.java +++ b/core/java/android/service/voice/HotwordDetectionService.java @@ -140,9 +140,7 @@ public abstract class HotwordDetectionService extends Service { Log.d(TAG, "#detectFromDspSource"); } HotwordDetectionService.this.onDetect( - new AlwaysOnHotwordDetector.EventPayload( - event.triggerInData, event.captureAvailable, - event.captureFormat, event.captureSession, event.data), + new AlwaysOnHotwordDetector.EventPayload.Builder(event).build(), timeoutMillis, new Callback(callback)); } diff --git a/core/java/android/service/voice/HotwordDetector.java b/core/java/android/service/voice/HotwordDetector.java index 969ec22beb978..96fd8bbda0160 100644 --- a/core/java/android/service/voice/HotwordDetector.java +++ b/core/java/android/service/voice/HotwordDetector.java @@ -118,6 +118,17 @@ public interface HotwordDetector { */ void updateState(@Nullable PersistableBundle options, @Nullable SharedMemory sharedMemory); + /** + * Invalidates this hotword detector so that any future calls to this result + * in an {@link IllegalStateException}. + * + *

If there are no other {@link HotwordDetector} instances linked to the + * {@link HotwordDetectionService}, the service will be shutdown. + */ + default void destroy() { + throw new UnsupportedOperationException("Not implemented. Must override in a subclass."); + } + /** * @hide */ diff --git a/core/java/android/service/voice/SoftwareHotwordDetector.java b/core/java/android/service/voice/SoftwareHotwordDetector.java index 512a654adbab7..2d662eaf0a4f0 100644 --- a/core/java/android/service/voice/SoftwareHotwordDetector.java +++ b/core/java/android/service/voice/SoftwareHotwordDetector.java @@ -77,7 +77,7 @@ class SoftwareHotwordDetector extends AbstractHotwordDetector { if (DEBUG) { Slog.i(TAG, "#startRecognition"); } - + throwIfDetectorIsNoLongerActive(); maybeCloseExistingSession(); try { @@ -100,6 +100,7 @@ class SoftwareHotwordDetector extends AbstractHotwordDetector { if (DEBUG) { Slog.i(TAG, "#stopRecognition"); } + throwIfDetectorIsNoLongerActive(); try { mManagerService.stopListeningFromMic(); @@ -110,6 +111,19 @@ class SoftwareHotwordDetector extends AbstractHotwordDetector { return true; } + @Override + public void destroy() { + stopRecognition(); + maybeCloseExistingSession(); + + try { + mManagerService.shutdownHotwordDetectionService(); + } catch (RemoteException ex) { + ex.rethrowFromSystemServer(); + } + super.destroy(); + } + private void maybeCloseExistingSession() { // TODO: needs to be synchronized. // TODO: implement this @@ -135,8 +149,11 @@ class SoftwareHotwordDetector extends AbstractHotwordDetector { mHandler.sendMessage(obtainMessage( HotwordDetector.Callback::onDetected, mCallback, - new AlwaysOnHotwordDetector.EventPayload( - audioFormat, hotwordDetectedResult, audioStream))); + new AlwaysOnHotwordDetector.EventPayload.Builder() + .setCaptureAudioFormat(audioFormat) + .setAudioStream(audioStream) + .setHotwordDetectedResult(hotwordDetectedResult) + .build())); } } diff --git a/core/java/android/service/voice/VoiceInteractionService.java b/core/java/android/service/voice/VoiceInteractionService.java index f52c9ff210d65..bf0cfbe49f31d 100644 --- a/core/java/android/service/voice/VoiceInteractionService.java +++ b/core/java/android/service/voice/VoiceInteractionService.java @@ -271,7 +271,7 @@ public class VoiceInteractionService extends Service { // It's still guaranteed to have been stopped. // This helps with cases where the voice interaction implementation is changed // by the user. - safelyShutdownHotwordDetector(); + safelyShutdownAllHotwordDetectors(); } /** @@ -380,11 +380,13 @@ public class VoiceInteractionService extends Service { } synchronized (mLock) { // Allow only one concurrent recognition via the APIs. - safelyShutdownHotwordDetector(); + safelyShutdownAllHotwordDetectors(); mHotwordDetector = new AlwaysOnHotwordDetector(keyphrase, locale, callback, mKeyphraseEnrollmentInfo, mSystemService, getApplicationContext().getApplicationInfo().targetSdkVersion, supportHotwordDetectionService, options, sharedMemory); + mHotwordDetector.registerOnDestroyListener((detector) -> onDspHotwordDetectorDestroyed( + (AlwaysOnHotwordDetector) detector)); } return mHotwordDetector; } @@ -433,10 +435,13 @@ public class VoiceInteractionService extends Service { } synchronized (mLock) { // Allow only one concurrent recognition via the APIs. - safelyShutdownHotwordDetector(); + safelyShutdownAllHotwordDetectors(); mSoftwareHotwordDetector = new SoftwareHotwordDetector( mSystemService, null, options, sharedMemory, callback); + mSoftwareHotwordDetector.registerOnDestroyListener( + (detector) -> onMicrophoneHotwordDetectorDestroyed( + (SoftwareHotwordDetector) detector)); } return mSoftwareHotwordDetector; } @@ -482,51 +487,36 @@ public class VoiceInteractionService extends Service { return mKeyphraseEnrollmentInfo.getKeyphraseMetadata(keyphrase, locale) != null; } - private void safelyShutdownHotwordDetector() { + private void safelyShutdownAllHotwordDetectors() { synchronized (mLock) { - shutdownDspHotwordDetectorLocked(); - shutdownMicrophoneHotwordDetectorLocked(); + if (mHotwordDetector != null) { + try { + mHotwordDetector.destroy(); + } catch (Exception ex) { + Log.i(TAG, "exception destroying AlwaysOnHotwordDetector", ex); + } + } + + if (mSoftwareHotwordDetector != null) { + try { + mSoftwareHotwordDetector.destroy(); + } catch (Exception ex) { + Log.i(TAG, "exception destroying SoftwareHotwordDetector", ex); + } + } } } - private void shutdownDspHotwordDetectorLocked() { - if (mHotwordDetector == null) { - return; + private void onDspHotwordDetectorDestroyed(@NonNull AlwaysOnHotwordDetector detector) { + synchronized (mLock) { + mHotwordDetector = null; } - - try { - mHotwordDetector.stopRecognition(); - } catch (Exception ex) { - // Ignore. - } - - try { - mHotwordDetector.invalidate(); - } catch (Exception ex) { - // Ignore. - } - - mHotwordDetector = null; } - private void shutdownMicrophoneHotwordDetectorLocked() { - if (mSoftwareHotwordDetector == null) { - return; + private void onMicrophoneHotwordDetectorDestroyed(@NonNull SoftwareHotwordDetector detector) { + synchronized (mLock) { + mSoftwareHotwordDetector = null; } - - try { - mSoftwareHotwordDetector.stopRecognition(); - } catch (Exception ex) { - // Ignore. - } - - try { - mSystemService.shutdownHotwordDetectionService(); - } catch (Exception ex) { - // Ignore. - } - - mSoftwareHotwordDetector = null; } /**