+ * This can be {@code null} if reprocessing the hotword trigger isn't required. + */ + @Nullable + private MediaSyncEvent mMediaSyncEvent = null; + /** * Byte offset in the audio stream when the trigger event happened. - * - *
If unset, the most recent bytes in the audio stream will be used. */ private final int mByteOffset; private static int defaultByteOffset() { @@ -84,7 +93,7 @@ public final class HotwordDetectedResult implements Parcelable { /** * Returns the maximum values of {@link #getScore} and {@link #getPersonalizedScore}. - * + *
* The float value should be calculated as {@code getScore() / getMaxScore()}. */ public static int getMaxScore() { @@ -159,6 +168,7 @@ public final class HotwordDetectedResult implements Parcelable { @DataClass.Generated.Member /* package-private */ HotwordDetectedResult( @HotwordDetector.HotwordConfidenceLevelValue int confidenceLevel, + @Nullable MediaSyncEvent mediaSyncEvent, int byteOffset, int score, int personalizedScore, @@ -167,6 +177,7 @@ public final class HotwordDetectedResult implements Parcelable { this.mConfidenceLevel = confidenceLevel; com.android.internal.util.AnnotationValidations.validate( HotwordDetector.HotwordConfidenceLevelValue.class, null, mConfidenceLevel); + this.mMediaSyncEvent = mediaSyncEvent; this.mByteOffset = byteOffset; this.mScore = score; this.mPersonalizedScore = personalizedScore; @@ -186,10 +197,20 @@ public final class HotwordDetectedResult implements Parcelable { return mConfidenceLevel; } + /** + * A {@code MediaSyncEvent} that allows the {@link HotwordDetector} to recapture the audio + * that contains the hotword trigger. This must be obtained using + * {@link android.media.AudioRecord#shareAudioHistory(String, long)}. + *
+ * This can be {@code null} if reprocessing the hotword trigger isn't required. + */ + @DataClass.Generated.Member + public @Nullable MediaSyncEvent getMediaSyncEvent() { + return mMediaSyncEvent; + } + /** * Byte offset in the audio stream when the trigger event happened. - * - *
If unset, the most recent bytes in the audio stream will be used. */ @DataClass.Generated.Member public int getByteOffset() { @@ -237,6 +258,9 @@ public final class HotwordDetectedResult implements Parcelable { * *
The use of this method is discouraged, and support for it will be removed in future * versions of Android. + * + *
This is a PersistableBundle so it doesn't allow any remotable objects or other contents + * that can be used to communicate with other processes. */ @DataClass.Generated.Member public @NonNull PersistableBundle getExtras() { @@ -251,6 +275,7 @@ public final class HotwordDetectedResult implements Parcelable { return "HotwordDetectedResult { " + "confidenceLevel = " + mConfidenceLevel + ", " + + "mediaSyncEvent = " + mMediaSyncEvent + ", " + "byteOffset = " + mByteOffset + ", " + "score = " + mScore + ", " + "personalizedScore = " + mPersonalizedScore + ", " + @@ -273,6 +298,7 @@ public final class HotwordDetectedResult implements Parcelable { //noinspection PointlessBooleanExpression return true && mConfidenceLevel == that.mConfidenceLevel + && java.util.Objects.equals(mMediaSyncEvent, that.mMediaSyncEvent) && mByteOffset == that.mByteOffset && mScore == that.mScore && mPersonalizedScore == that.mPersonalizedScore @@ -288,6 +314,7 @@ public final class HotwordDetectedResult implements Parcelable { int _hash = 1; _hash = 31 * _hash + mConfidenceLevel; + _hash = 31 * _hash + java.util.Objects.hashCode(mMediaSyncEvent); _hash = 31 * _hash + mByteOffset; _hash = 31 * _hash + mScore; _hash = 31 * _hash + mPersonalizedScore; @@ -302,7 +329,11 @@ public final class HotwordDetectedResult implements Parcelable { // You can override field parcelling by defining methods like: // void parcelFieldName(Parcel dest, int flags) { ... } + byte flg = 0; + if (mMediaSyncEvent != null) flg |= 0x2; + dest.writeByte(flg); dest.writeInt(mConfidenceLevel); + if (mMediaSyncEvent != null) dest.writeTypedObject(mMediaSyncEvent, flags); dest.writeInt(mByteOffset); dest.writeInt(mScore); dest.writeInt(mPersonalizedScore); @@ -321,7 +352,9 @@ public final class HotwordDetectedResult implements Parcelable { // You can override field unparcelling by defining methods like: // static FieldType unparcelFieldName(Parcel in) { ... } + byte flg = in.readByte(); int confidenceLevel = in.readInt(); + MediaSyncEvent mediaSyncEvent = (flg & 0x2) == 0 ? null : (MediaSyncEvent) in.readTypedObject(MediaSyncEvent.CREATOR); int byteOffset = in.readInt(); int score = in.readInt(); int personalizedScore = in.readInt(); @@ -331,6 +364,7 @@ public final class HotwordDetectedResult implements Parcelable { this.mConfidenceLevel = confidenceLevel; com.android.internal.util.AnnotationValidations.validate( HotwordDetector.HotwordConfidenceLevelValue.class, null, mConfidenceLevel); + this.mMediaSyncEvent = mediaSyncEvent; this.mByteOffset = byteOffset; this.mScore = score; this.mPersonalizedScore = personalizedScore; @@ -364,6 +398,7 @@ public final class HotwordDetectedResult implements Parcelable { public static final class Builder { private @HotwordDetector.HotwordConfidenceLevelValue int mConfidenceLevel; + private @Nullable MediaSyncEvent mMediaSyncEvent; private int mByteOffset; private int mScore; private int mPersonalizedScore; @@ -386,15 +421,28 @@ public final class HotwordDetectedResult implements Parcelable { return this; } + /** + * A {@code MediaSyncEvent} that allows the {@link HotwordDetector} to recapture the audio + * that contains the hotword trigger. This must be obtained using + * {@link android.media.AudioRecord#shareAudioHistory(String, long)}. + *
+ * This can be {@code null} if reprocessing the hotword trigger isn't required. + */ + @DataClass.Generated.Member + public @NonNull Builder setMediaSyncEvent(@NonNull MediaSyncEvent value) { + checkNotUsed(); + mBuilderFieldsSet |= 0x2; + mMediaSyncEvent = value; + return this; + } + /** * Byte offset in the audio stream when the trigger event happened. - * - *
If unset, the most recent bytes in the audio stream will be used. */ @DataClass.Generated.Member public @NonNull Builder setByteOffset(int value) { checkNotUsed(); - mBuilderFieldsSet |= 0x2; + mBuilderFieldsSet |= 0x4; mByteOffset = value; return this; } @@ -407,7 +455,7 @@ public final class HotwordDetectedResult implements Parcelable { @DataClass.Generated.Member public @NonNull Builder setScore(int value) { checkNotUsed(); - mBuilderFieldsSet |= 0x4; + mBuilderFieldsSet |= 0x8; mScore = value; return this; } @@ -420,7 +468,7 @@ public final class HotwordDetectedResult implements Parcelable { @DataClass.Generated.Member public @NonNull Builder setPersonalizedScore(int value) { checkNotUsed(); - mBuilderFieldsSet |= 0x8; + mBuilderFieldsSet |= 0x10; mPersonalizedScore = value; return this; } @@ -433,7 +481,7 @@ public final class HotwordDetectedResult implements Parcelable { @DataClass.Generated.Member public @NonNull Builder setHotwordPhraseId(int value) { checkNotUsed(); - mBuilderFieldsSet |= 0x10; + mBuilderFieldsSet |= 0x20; mHotwordPhraseId = value; return this; } @@ -449,11 +497,14 @@ public final class HotwordDetectedResult implements Parcelable { * *
The use of this method is discouraged, and support for it will be removed in future * versions of Android. + * + *
This is a PersistableBundle so it doesn't allow any remotable objects or other contents + * that can be used to communicate with other processes. */ @DataClass.Generated.Member public @NonNull Builder setExtras(@NonNull PersistableBundle value) { checkNotUsed(); - mBuilderFieldsSet |= 0x20; + mBuilderFieldsSet |= 0x40; mExtras = value; return this; } @@ -461,28 +512,32 @@ public final class HotwordDetectedResult implements Parcelable { /** Builds the instance. This builder should not be touched after calling this! */ public @NonNull HotwordDetectedResult build() { checkNotUsed(); - mBuilderFieldsSet |= 0x40; // Mark builder used + mBuilderFieldsSet |= 0x80; // Mark builder used if ((mBuilderFieldsSet & 0x1) == 0) { mConfidenceLevel = defaultConfidenceLevel(); } if ((mBuilderFieldsSet & 0x2) == 0) { - mByteOffset = defaultByteOffset(); + mMediaSyncEvent = null; } if ((mBuilderFieldsSet & 0x4) == 0) { - mScore = defaultScore(); + mByteOffset = defaultByteOffset(); } if ((mBuilderFieldsSet & 0x8) == 0) { - mPersonalizedScore = defaultPersonalizedScore(); + mScore = defaultScore(); } if ((mBuilderFieldsSet & 0x10) == 0) { - mHotwordPhraseId = defaultHotwordPhraseId(); + mPersonalizedScore = defaultPersonalizedScore(); } if ((mBuilderFieldsSet & 0x20) == 0) { + mHotwordPhraseId = defaultHotwordPhraseId(); + } + if ((mBuilderFieldsSet & 0x40) == 0) { mExtras = defaultExtras(); } HotwordDetectedResult o = new HotwordDetectedResult( mConfidenceLevel, + mMediaSyncEvent, mByteOffset, mScore, mPersonalizedScore, @@ -492,7 +547,7 @@ public final class HotwordDetectedResult implements Parcelable { } private void checkNotUsed() { - if ((mBuilderFieldsSet & 0x40) != 0) { + if ((mBuilderFieldsSet & 0x80) != 0) { throw new IllegalStateException( "This Builder should not be reused. Use a new Builder instance instead"); } @@ -500,10 +555,10 @@ public final class HotwordDetectedResult implements Parcelable { } @DataClass.Generated( - time = 1616965644404L, + time = 1619059352684L, codegenVersion = "1.0.23", sourceFile = "frameworks/base/core/java/android/service/voice/HotwordDetectedResult.java", - inputSignatures = "public static final int BYTE_OFFSET_UNSET\nprivate final @android.service.voice.HotwordDetector.HotwordConfidenceLevelValue int mConfidenceLevel\nprivate final int mByteOffset\nprivate final int mScore\nprivate final int mPersonalizedScore\nprivate final int mHotwordPhraseId\nprivate final @android.annotation.NonNull android.os.PersistableBundle mExtras\nprivate static int defaultConfidenceLevel()\nprivate static int defaultByteOffset()\nprivate static int defaultScore()\nprivate static int defaultPersonalizedScore()\npublic static int getMaxScore()\nprivate static int defaultHotwordPhraseId()\npublic static int getMaxHotwordPhraseId()\nprivate static android.os.PersistableBundle defaultExtras()\npublic static int getMaxBundleSize()\nclass HotwordDetectedResult extends java.lang.Object implements [android.os.Parcelable]\n@com.android.internal.util.DataClass(genConstructor=false, genBuilder=true, genEqualsHashCode=true, genHiddenConstDefs=true, genParcelable=true, genToString=true)") + inputSignatures = "public static final int BYTE_OFFSET_UNSET\nprivate final @android.service.voice.HotwordDetector.HotwordConfidenceLevelValue int mConfidenceLevel\nprivate @android.annotation.Nullable android.media.MediaSyncEvent mMediaSyncEvent\nprivate final int mByteOffset\nprivate final int mScore\nprivate final int mPersonalizedScore\nprivate final int mHotwordPhraseId\nprivate final @android.annotation.NonNull android.os.PersistableBundle mExtras\nprivate static int defaultConfidenceLevel()\nprivate static int defaultByteOffset()\nprivate static int defaultScore()\nprivate static int defaultPersonalizedScore()\npublic static int getMaxScore()\nprivate static int defaultHotwordPhraseId()\npublic static int getMaxHotwordPhraseId()\nprivate static android.os.PersistableBundle defaultExtras()\npublic static int getMaxBundleSize()\nclass HotwordDetectedResult extends java.lang.Object implements [android.os.Parcelable]\n@com.android.internal.util.DataClass(genConstructor=false, genBuilder=true, genEqualsHashCode=true, genHiddenConstDefs=true, genParcelable=true, genToString=true)") @Deprecated private void __metadata() {} diff --git a/core/java/android/service/voice/HotwordDetectionService.java b/core/java/android/service/voice/HotwordDetectionService.java index ea854e8c92834..7e0117df288e7 100644 --- a/core/java/android/service/voice/HotwordDetectionService.java +++ b/core/java/android/service/voice/HotwordDetectionService.java @@ -30,6 +30,7 @@ import android.app.Service; import android.content.ContentCaptureOptions; import android.content.Context; import android.content.Intent; +import android.hardware.soundtrigger.SoundTrigger; import android.media.AudioFormat; import android.os.Bundle; import android.os.Handler; @@ -133,7 +134,7 @@ public abstract class HotwordDetectionService extends Service { private final IHotwordDetectionService mInterface = new IHotwordDetectionService.Stub() { @Override public void detectFromDspSource( - ParcelFileDescriptor audioStream, + SoundTrigger.KeyphraseRecognitionEvent event, AudioFormat audioFormat, long timeoutMillis, IDspHotwordDetectionCallback callback) @@ -143,8 +144,9 @@ public abstract class HotwordDetectionService extends Service { } mHandler.sendMessage(obtainMessage(HotwordDetectionService::onDetect, HotwordDetectionService.this, - audioStream, - audioFormat, + new AlwaysOnHotwordDetector.EventPayload( + event.triggerInData, event.captureAvailable, + event.captureFormat, event.captureSession, event.data), timeoutMillis, new Callback(callback))); } @@ -178,8 +180,6 @@ public abstract class HotwordDetectionService extends Service { mHandler.sendMessage(obtainMessage( HotwordDetectionService::onDetect, HotwordDetectionService.this, - audioStream, - audioFormat, new Callback(callback))); break; case AUDIO_SOURCE_EXTERNAL: @@ -246,13 +246,42 @@ public abstract class HotwordDetectionService extends Service { * the application fails to abide by the timeout, system will close the * microphone and cancel the operation. * @param callback The callback to use for responding to the detection request. + * @deprecated Implement + * {@link #onDetect(AlwaysOnHotwordDetector.EventPayload, long, Callback)} instead. + * + * @hide + */ + @Deprecated + @SystemApi + public void onDetect( + @NonNull ParcelFileDescriptor audioStream, + @NonNull AudioFormat audioFormat, + @DurationMillisLong long timeoutMillis, + @NonNull Callback callback) { + // TODO: Add a helpful error message. + throw new UnsupportedOperationException(); + } + + /** + * Called when the device hardware (such as a DSP) detected the hotword, to request second stage + * validation before handing over the audio to the {@link AlwaysOnHotwordDetector}. + *
+ * After {@code callback} is invoked or {@code timeoutMillis} has passed, and invokes the + * appropriate {@link AlwaysOnHotwordDetector.Callback callback}. + * + * @param eventPayload Payload data for the hardware detection event. This may contain the + * trigger audio, if requested when calling + * {@link AlwaysOnHotwordDetector#startRecognition(int)}. + * @param timeoutMillis Timeout in milliseconds for the operation to invoke the callback. If + * the application fails to abide by the timeout, system will close the + * microphone and cancel the operation. + * @param callback The callback to use for responding to the detection request. * * @hide */ @SystemApi public void onDetect( - @NonNull ParcelFileDescriptor audioStream, - @NonNull AudioFormat audioFormat, + @NonNull AlwaysOnHotwordDetector.EventPayload eventPayload, @DurationMillisLong long timeoutMillis, @NonNull Callback callback) { // TODO: Add a helpful error message. @@ -305,7 +334,9 @@ public abstract class HotwordDetectionService extends Service { * @param audioFormat Format of the supplied audio * @param callback The callback to use for responding to the detection request. * {@link Callback#onRejected(HotwordRejectedResult) callback.onRejected} cannot be used here. + * @deprecated Implement {@link #onDetect(Callback)} instead. */ + @Deprecated public void onDetect( @NonNull ParcelFileDescriptor audioStream, @NonNull AudioFormat audioFormat, @@ -314,6 +345,22 @@ public abstract class HotwordDetectionService extends Service { throw new UnsupportedOperationException(); } + /** + * Called when the {@link VoiceInteractionService} requests that this service + * {@link HotwordDetector#startRecognition() start} hotword recognition on audio coming directly + * from the device microphone. + *
+ * On successful detection of a hotword, call + * {@link Callback#onDetected(HotwordDetectedResult)}. + * + * @param callback The callback to use for responding to the detection request. + * {@link Callback#onRejected(HotwordRejectedResult) callback.onRejected} cannot be used here. + */ + public void onDetect(@NonNull Callback callback) { + // TODO: Add a helpful error message. + throw new UnsupportedOperationException(); + } + /** * Called when the {@link VoiceInteractionService} requests that this service * {@link HotwordDetector#startRecognition(ParcelFileDescriptor, AudioFormat, diff --git a/core/java/android/service/voice/IHotwordDetectionService.aidl b/core/java/android/service/voice/IHotwordDetectionService.aidl index 2ffe787e895af..7ba00982e6a2e 100644 --- a/core/java/android/service/voice/IHotwordDetectionService.aidl +++ b/core/java/android/service/voice/IHotwordDetectionService.aidl @@ -16,6 +16,8 @@ package android.service.voice; +import android.content.ContentCaptureOptions; +import android.hardware.soundtrigger.SoundTrigger; import android.media.AudioFormat; import android.os.IRemoteCallback; import android.os.ParcelFileDescriptor; @@ -23,7 +25,6 @@ import android.os.PersistableBundle; import android.os.SharedMemory; import android.service.voice.IDspHotwordDetectionCallback; import android.view.contentcapture.IContentCaptureManager; -import android.content.ContentCaptureOptions; /** * Provide the interface to communicate with hotword detection service. @@ -32,7 +33,7 @@ import android.content.ContentCaptureOptions; */ oneway interface IHotwordDetectionService { void detectFromDspSource( - in ParcelFileDescriptor audioStream, + in SoundTrigger.KeyphraseRecognitionEvent event, in AudioFormat audioFormat, long timeoutMillis, in IDspHotwordDetectionCallback callback); diff --git a/core/java/android/service/voice/VoiceInteractionService.java b/core/java/android/service/voice/VoiceInteractionService.java index 2a25227419557..b5c838b1871b1 100644 --- a/core/java/android/service/voice/VoiceInteractionService.java +++ b/core/java/android/service/voice/VoiceInteractionService.java @@ -420,9 +420,13 @@ public class VoiceInteractionService extends Service { * * @see #createAlwaysOnHotwordDetector(String, Locale, PersistableBundle, SharedMemory, * AlwaysOnHotwordDetector.Callback) + * @deprecated Use + * {@link #createHotwordDetector(PersistableBundle, SharedMemory, HotwordDetector.Callback)} + * instead. * * @hide */ + @Deprecated @SystemApi @RequiresPermission(Manifest.permission.MANAGE_HOTWORD_DETECTION) @NonNull @@ -444,6 +448,58 @@ public class VoiceInteractionService extends Service { return mSoftwareHotwordDetector; } + /** + * Creates a {@link HotwordDetector} and initializes the application's + * {@link HotwordDetectionService} using {@code options} and {code sharedMemory}. + * + *
To be able to call this, you need to set android:hotwordDetectionService in the + * android.voice_interaction metadata file to a valid hotword detection service, and set + * android:isolatedProcess="true" in the hotword detection service's declaration. Otherwise, + * this throws an {@link IllegalStateException}. + * + *
This instance must be retained and used by the client. + * Calling this a second time invalidates the previously created hotword detector + * which can no longer be used to manage recognition. + * + *
Using this has a noticeable impact on battery, since the microphone is kept open
+ * for the lifetime of the recognition {@link HotwordDetector#startRecognition() session}. On
+ * devices where hardware filtering is available (such as through a DSP), it's highly
+ * recommended to use {@link #createAlwaysOnHotwordDetector} instead.
+ *
+ * @param options Application configuration data to be provided to the
+ * {@link HotwordDetectionService}. PersistableBundle does not allow any remotable objects or
+ * other contents that can be used to communicate with other processes.
+ * @param sharedMemory The unrestricted data blob to be provided to the
+ * {@link HotwordDetectionService}. Use this to provide hotword models or other such data to the
+ * sandboxed process.
+ * @param callback The callback to notify of detection events.
+ * @return A hotword detector for the given audio format.
+ *
+ * @see #createAlwaysOnHotwordDetector(String, Locale, PersistableBundle, SharedMemory,
+ * AlwaysOnHotwordDetector.Callback)
+ *
+ * @hide
+ */
+ @SystemApi
+ @RequiresPermission(Manifest.permission.MANAGE_HOTWORD_DETECTION)
+ @NonNull
+ public final HotwordDetector createHotwordDetector(
+ @Nullable PersistableBundle options,
+ @Nullable SharedMemory sharedMemory,
+ @NonNull HotwordDetector.Callback callback) {
+ if (mSystemService == null) {
+ throw new IllegalStateException("Not available until onReady() is called");
+ }
+ synchronized (mLock) {
+ // Allow only one concurrent recognition via the APIs.
+ safelyShutdownHotwordDetector();
+ mSoftwareHotwordDetector =
+ new SoftwareHotwordDetector(
+ mSystemService, null, options, sharedMemory, callback);
+ }
+ return mSoftwareHotwordDetector;
+ }
+
/**
* Creates an {@link KeyphraseModelManager} to use for enrolling voice models outside of the
* pre-bundled system voice models.
diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java
index d6ed98f2c3d41..3fbd40f8a060f 100644
--- a/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java
+++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java
@@ -65,7 +65,6 @@ import java.io.PrintWriter;
import java.util.concurrent.Executor;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -235,19 +234,32 @@ final class HotwordDetectionConnection {
Slog.d(TAG, "startListeningFromMic");
}
- AudioRecord audioRecord = createMicAudioRecord(audioFormat);
- if (audioRecord == null) {
- // TODO: Callback.onError();
- return;
- }
+ // TODO: consider making this a non-anonymous class.
+ IDspHotwordDetectionCallback internalCallback = new IDspHotwordDetectionCallback.Stub() {
+ @Override
+ public void onDetected(HotwordDetectedResult result) throws RemoteException {
+ if (DEBUG) {
+ Slog.d(TAG, "onDetected");
+ }
+ callback.onDetected(result, null, null);
+ }
- handleSoftwareHotwordDetection(
- audioFormat,
- AudioReader.createFromAudioRecord(audioRecord),
- AUDIO_SOURCE_MICROPHONE,
- // TODO: handle bundles better.
- new PersistableBundle(),
- callback);
+ @Override
+ public void onRejected(HotwordRejectedResult result) throws RemoteException {
+ if (DEBUG) {
+ Slog.d(TAG, "onRejected");
+ }
+ // onRejected isn't allowed here
+ }
+ };
+
+ mRemoteHotwordDetectionService.run(
+ service -> service.detectFromMicrophoneSource(
+ null,
+ AUDIO_SOURCE_MICROPHONE,
+ null,
+ null,
+ internalCallback));
}
public void startListeningFromExternalSource(
@@ -298,74 +310,12 @@ final class HotwordDetectionConnection {
if (DEBUG) {
Slog.d(TAG, "detectFromDspSourceForTest");
}
-
- AudioRecord record = createFakeAudioRecord();
- if (record == null) {
- Slog.d(TAG, "Failed to create fake audio record");
- return;
- }
-
- Pair