Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Leverage new Audio history sharing for HotwordDetectionService.

Browse Source 
The Audio Framework now supports sharing audio history, so
VoiceInteractionManagerService no longer needs to handle audio streams
itself.
See I5beba6c1e489148a14ba86165b8ef2fdc78c802a.

Bug: 168305377
Test: atest CtsVoiceInteractionTestCases
Test: manual
Change-Id: Ia7c5830aa5af6c015ae26619e0a90967ddff74e5
This commit is contained in:
Ahaan Ugale
2021-04-22 01:05:24 -07:00
parent 463493f57d
commit 35f28d1bb2
7 changed files with 224 additions and 173 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
core/api/system-current.txt
View File

@@ -10520,6 +10520,7 @@ package android.service.voice {
method public static int getMaxBundleSize();
method public static int getMaxHotwordPhraseId();
method public static int getMaxScore();
method @Nullable public android.media.MediaSyncEvent getMediaSyncEvent();
method public int getPersonalizedScore();
method public int getScore();
method public void writeToParcel(@NonNull android.os.Parcel, int);
@@ -10534,6 +10535,7 @@ package android.service.voice {
method @NonNull public android.service.voice.HotwordDetectedResult.Builder setConfidenceLevel(int);
method @NonNull public android.service.voice.HotwordDetectedResult.Builder setExtras(@NonNull android.os.PersistableBundle);
method @NonNull public android.service.voice.HotwordDetectedResult.Builder setHotwordPhraseId(int);
method @NonNull public android.service.voice.HotwordDetectedResult.Builder setMediaSyncEvent(@NonNull android.media.MediaSyncEvent);
method @NonNull public android.service.voice.HotwordDetectedResult.Builder setPersonalizedScore(int);
method @NonNull public android.service.voice.HotwordDetectedResult.Builder setScore(int);
}
@@ -10541,8 +10543,10 @@ package android.service.voice {
public abstract class HotwordDetectionService extends android.app.Service {
ctor public HotwordDetectionService();
method @Nullable public final android.os.IBinder onBind(@NonNull android.content.Intent);
method public void onDetect(@NonNull android.os.ParcelFileDescriptor, @NonNull android.media.AudioFormat, long, @NonNull android.service.voice.HotwordDetectionService.Callback);
method public void onDetect(@NonNull android.os.ParcelFileDescriptor, @NonNull android.media.AudioFormat, @NonNull android.service.voice.HotwordDetectionService.Callback);
method @Deprecated public void onDetect(@NonNull android.os.ParcelFileDescriptor, @NonNull android.media.AudioFormat, long, @NonNull android.service.voice.HotwordDetectionService.Callback);
method public void onDetect(@NonNull android.service.voice.AlwaysOnHotwordDetector.EventPayload, long, @NonNull android.service.voice.HotwordDetectionService.Callback);
method @Deprecated public void onDetect(@NonNull android.os.ParcelFileDescriptor, @NonNull android.media.AudioFormat, @NonNull android.service.voice.HotwordDetectionService.Callback);
method public void onDetect(@NonNull android.service.voice.HotwordDetectionService.Callback);
method public void onDetect(@NonNull android.os.ParcelFileDescriptor, @NonNull android.media.AudioFormat, @Nullable android.os.PersistableBundle, @NonNull android.service.voice.HotwordDetectionService.Callback);
method public void onUpdateState(@Nullable android.os.PersistableBundle, @Nullable android.os.SharedMemory, long, @Nullable java.util.function.IntConsumer);
field public static final int INITIALIZATION_STATUS_CUSTOM_ERROR_1 = 1; // 0x1
@@ -10588,7 +10592,8 @@ package android.service.voice {
public class VoiceInteractionService extends android.app.Service {
method @NonNull public final android.service.voice.AlwaysOnHotwordDetector createAlwaysOnHotwordDetector(String, java.util.Locale, android.service.voice.AlwaysOnHotwordDetector.Callback);
method @NonNull @RequiresPermission(android.Manifest.permission.MANAGE_HOTWORD_DETECTION) public final android.service.voice.AlwaysOnHotwordDetector createAlwaysOnHotwordDetector(String, java.util.Locale, @Nullable android.os.PersistableBundle, @Nullable android.os.SharedMemory, android.service.voice.AlwaysOnHotwordDetector.Callback);
method @NonNull @RequiresPermission(android.Manifest.permission.MANAGE_HOTWORD_DETECTION) public final android.service.voice.HotwordDetector createHotwordDetector(@NonNull android.media.AudioFormat, @Nullable android.os.PersistableBundle, @Nullable android.os.SharedMemory, @NonNull android.service.voice.HotwordDetector.Callback);
method @Deprecated @NonNull @RequiresPermission(android.Manifest.permission.MANAGE_HOTWORD_DETECTION) public final android.service.voice.HotwordDetector createHotwordDetector(@NonNull android.media.AudioFormat, @Nullable android.os.PersistableBundle, @Nullable android.os.SharedMemory, @NonNull android.service.voice.HotwordDetector.Callback);
method @NonNull @RequiresPermission(android.Manifest.permission.MANAGE_HOTWORD_DETECTION) public final android.service.voice.HotwordDetector createHotwordDetector(@Nullable android.os.PersistableBundle, @Nullable android.os.SharedMemory, @NonNull android.service.voice.HotwordDetector.Callback);
method @NonNull @RequiresPermission("android.permission.MANAGE_VOICE_KEYPHRASES") public final android.media.voice.KeyphraseModelManager createKeyphraseModelManager();
}

2
core/java/android/service/voice/AlwaysOnHotwordDetector.java
View File

@@ -349,7 +349,7 @@ public class AlwaysOnHotwordDetector extends AbstractHotwordDetector {
private final HotwordDetectedResult mHotwordDetectedResult;
private final ParcelFileDescriptor mAudioStream;
private EventPayload(boolean triggerAvailable, boolean captureAvailable,
EventPayload(boolean triggerAvailable, boolean captureAvailable,
AudioFormat audioFormat, int captureSession, byte[] data) {
this(triggerAvailable, captureAvailable, audioFormat, captureSession, data, null,
null);

95
core/java/android/service/voice/HotwordDetectedResult.java
View File

@@ -21,6 +21,7 @@ import static android.service.voice.HotwordDetector.CONFIDENCE_LEVEL_NONE;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.annotation.SystemApi;
import android.media.MediaSyncEvent;
import android.os.Parcelable;
import android.os.PersistableBundle;
@@ -52,10 +53,18 @@ public final class HotwordDetectedResult implements Parcelable {
return CONFIDENCE_LEVEL_NONE;
}
/**
* A {@code MediaSyncEvent} that allows the {@link HotwordDetector} to recapture the audio
* that contains the hotword trigger. This must be obtained using
* {@link android.media.AudioRecord#shareAudioHistory(String, long)}.
* <p>
* This can be {@code null} if reprocessing the hotword trigger isn't required.
*/
@Nullable
private MediaSyncEvent mMediaSyncEvent = null;
/**
* Byte offset in the audio stream when the trigger event happened.
*
* <p>If unset, the most recent bytes in the audio stream will be used.
*/
private final int mByteOffset;
private static int defaultByteOffset() {
@@ -84,7 +93,7 @@ public final class HotwordDetectedResult implements Parcelable {
/**
* Returns the maximum values of {@link #getScore} and {@link #getPersonalizedScore}.
*
* <p>
* The float value should be calculated as {@code getScore() / getMaxScore()}.
*/
public static int getMaxScore() {
@@ -159,6 +168,7 @@ public final class HotwordDetectedResult implements Parcelable {
@DataClass.Generated.Member
/* package-private */ HotwordDetectedResult(
@HotwordDetector.HotwordConfidenceLevelValue int confidenceLevel,
@Nullable MediaSyncEvent mediaSyncEvent,
int byteOffset,
int score,
int personalizedScore,
@@ -167,6 +177,7 @@ public final class HotwordDetectedResult implements Parcelable {
this.mConfidenceLevel = confidenceLevel;
com.android.internal.util.AnnotationValidations.validate(
HotwordDetector.HotwordConfidenceLevelValue.class, null, mConfidenceLevel);
this.mMediaSyncEvent = mediaSyncEvent;
this.mByteOffset = byteOffset;
this.mScore = score;
this.mPersonalizedScore = personalizedScore;
@@ -186,10 +197,20 @@ public final class HotwordDetectedResult implements Parcelable {
return mConfidenceLevel;
}
/**
* A {@code MediaSyncEvent} that allows the {@link HotwordDetector} to recapture the audio
* that contains the hotword trigger. This must be obtained using
* {@link android.media.AudioRecord#shareAudioHistory(String, long)}.
* <p>
* This can be {@code null} if reprocessing the hotword trigger isn't required.
*/
@DataClass.Generated.Member
public @Nullable MediaSyncEvent getMediaSyncEvent() {
return mMediaSyncEvent;
}
/**
* Byte offset in the audio stream when the trigger event happened.
*
* <p>If unset, the most recent bytes in the audio stream will be used.
*/
@DataClass.Generated.Member
public int getByteOffset() {
@@ -237,6 +258,9 @@ public final class HotwordDetectedResult implements Parcelable {
*
* <p>The use of this method is discouraged, and support for it will be removed in future
* versions of Android.
*
* <p>This is a PersistableBundle so it doesn't allow any remotable objects or other contents
* that can be used to communicate with other processes.
*/
@DataClass.Generated.Member
public @NonNull PersistableBundle getExtras() {
@@ -251,6 +275,7 @@ public final class HotwordDetectedResult implements Parcelable {
return "HotwordDetectedResult { " +
"confidenceLevel = " + mConfidenceLevel + ", " +
"mediaSyncEvent = " + mMediaSyncEvent + ", " +
"byteOffset = " + mByteOffset + ", " +
"score = " + mScore + ", " +
"personalizedScore = " + mPersonalizedScore + ", " +
@@ -273,6 +298,7 @@ public final class HotwordDetectedResult implements Parcelable {
//noinspection PointlessBooleanExpression
return true
&& mConfidenceLevel == that.mConfidenceLevel
&& java.util.Objects.equals(mMediaSyncEvent, that.mMediaSyncEvent)
&& mByteOffset == that.mByteOffset
&& mScore == that.mScore
&& mPersonalizedScore == that.mPersonalizedScore
@@ -288,6 +314,7 @@ public final class HotwordDetectedResult implements Parcelable {
int _hash = 1;
_hash = 31 * _hash + mConfidenceLevel;
_hash = 31 * _hash + java.util.Objects.hashCode(mMediaSyncEvent);
_hash = 31 * _hash + mByteOffset;
_hash = 31 * _hash + mScore;
_hash = 31 * _hash + mPersonalizedScore;
@@ -302,7 +329,11 @@ public final class HotwordDetectedResult implements Parcelable {
// You can override field parcelling by defining methods like:
// void parcelFieldName(Parcel dest, int flags) { ... }
byte flg = 0;
if (mMediaSyncEvent != null) flg |= 0x2;
dest.writeByte(flg);
dest.writeInt(mConfidenceLevel);
if (mMediaSyncEvent != null) dest.writeTypedObject(mMediaSyncEvent, flags);
dest.writeInt(mByteOffset);
dest.writeInt(mScore);
dest.writeInt(mPersonalizedScore);
@@ -321,7 +352,9 @@ public final class HotwordDetectedResult implements Parcelable {
// You can override field unparcelling by defining methods like:
// static FieldType unparcelFieldName(Parcel in) { ... }
byte flg = in.readByte();
int confidenceLevel = in.readInt();
MediaSyncEvent mediaSyncEvent = (flg & 0x2) == 0 ? null : (MediaSyncEvent) in.readTypedObject(MediaSyncEvent.CREATOR);
int byteOffset = in.readInt();
int score = in.readInt();
int personalizedScore = in.readInt();
@@ -331,6 +364,7 @@ public final class HotwordDetectedResult implements Parcelable {
this.mConfidenceLevel = confidenceLevel;
com.android.internal.util.AnnotationValidations.validate(
HotwordDetector.HotwordConfidenceLevelValue.class, null, mConfidenceLevel);
this.mMediaSyncEvent = mediaSyncEvent;
this.mByteOffset = byteOffset;
this.mScore = score;
this.mPersonalizedScore = personalizedScore;
@@ -364,6 +398,7 @@ public final class HotwordDetectedResult implements Parcelable {
public static final class Builder {
private @HotwordDetector.HotwordConfidenceLevelValue int mConfidenceLevel;
private @Nullable MediaSyncEvent mMediaSyncEvent;
private int mByteOffset;
private int mScore;
private int mPersonalizedScore;
@@ -386,15 +421,28 @@ public final class HotwordDetectedResult implements Parcelable {
return this;
}
/**
* A {@code MediaSyncEvent} that allows the {@link HotwordDetector} to recapture the audio
* that contains the hotword trigger. This must be obtained using
* {@link android.media.AudioRecord#shareAudioHistory(String, long)}.
* <p>
* This can be {@code null} if reprocessing the hotword trigger isn't required.
*/
@DataClass.Generated.Member
public @NonNull Builder setMediaSyncEvent(@NonNull MediaSyncEvent value) {
checkNotUsed();
mBuilderFieldsSet |= 0x2;
mMediaSyncEvent = value;
return this;
}
/**
* Byte offset in the audio stream when the trigger event happened.
*
* <p>If unset, the most recent bytes in the audio stream will be used.
*/
@DataClass.Generated.Member
public @NonNull Builder setByteOffset(int value) {
checkNotUsed();
mBuilderFieldsSet |= 0x2;
mBuilderFieldsSet |= 0x4;
mByteOffset = value;
return this;
}
@@ -407,7 +455,7 @@ public final class HotwordDetectedResult implements Parcelable {
@DataClass.Generated.Member
public @NonNull Builder setScore(int value) {
checkNotUsed();
mBuilderFieldsSet |= 0x4;
mBuilderFieldsSet |= 0x8;
mScore = value;
return this;
}
@@ -420,7 +468,7 @@ public final class HotwordDetectedResult implements Parcelable {
@DataClass.Generated.Member
public @NonNull Builder setPersonalizedScore(int value) {
checkNotUsed();
mBuilderFieldsSet |= 0x8;
mBuilderFieldsSet |= 0x10;
mPersonalizedScore = value;
return this;
}
@@ -433,7 +481,7 @@ public final class HotwordDetectedResult implements Parcelable {
@DataClass.Generated.Member
public @NonNull Builder setHotwordPhraseId(int value) {
checkNotUsed();
mBuilderFieldsSet |= 0x10;
mBuilderFieldsSet |= 0x20;
mHotwordPhraseId = value;
return this;
}
@@ -449,11 +497,14 @@ public final class HotwordDetectedResult implements Parcelable {
*
* <p>The use of this method is discouraged, and support for it will be removed in future
* versions of Android.
*
* <p>This is a PersistableBundle so it doesn't allow any remotable objects or other contents
* that can be used to communicate with other processes.
*/
@DataClass.Generated.Member
public @NonNull Builder setExtras(@NonNull PersistableBundle value) {
checkNotUsed();
mBuilderFieldsSet |= 0x20;
mBuilderFieldsSet |= 0x40;
mExtras = value;
return this;
}
@@ -461,28 +512,32 @@ public final class HotwordDetectedResult implements Parcelable {
/** Builds the instance. This builder should not be touched after calling this! */
public @NonNull HotwordDetectedResult build() {
checkNotUsed();
mBuilderFieldsSet |= 0x40; // Mark builder used
mBuilderFieldsSet |= 0x80; // Mark builder used
if ((mBuilderFieldsSet & 0x1) == 0) {
mConfidenceLevel = defaultConfidenceLevel();
}
if ((mBuilderFieldsSet & 0x2) == 0) {
mByteOffset = defaultByteOffset();
mMediaSyncEvent = null;
}
if ((mBuilderFieldsSet & 0x4) == 0) {
mScore = defaultScore();
mByteOffset = defaultByteOffset();
}
if ((mBuilderFieldsSet & 0x8) == 0) {
mPersonalizedScore = defaultPersonalizedScore();
mScore = defaultScore();
}
if ((mBuilderFieldsSet & 0x10) == 0) {
mHotwordPhraseId = defaultHotwordPhraseId();
mPersonalizedScore = defaultPersonalizedScore();
}
if ((mBuilderFieldsSet & 0x20) == 0) {
mHotwordPhraseId = defaultHotwordPhraseId();
}
if ((mBuilderFieldsSet & 0x40) == 0) {
mExtras = defaultExtras();
}
HotwordDetectedResult o = new HotwordDetectedResult(
mConfidenceLevel,
mMediaSyncEvent,
mByteOffset,
mScore,
mPersonalizedScore,
@@ -492,7 +547,7 @@ public final class HotwordDetectedResult implements Parcelable {
}
private void checkNotUsed() {
if ((mBuilderFieldsSet & 0x40) != 0) {
if ((mBuilderFieldsSet & 0x80) != 0) {
throw new IllegalStateException(
"This Builder should not be reused. Use a new Builder instance instead");
}
@@ -500,10 +555,10 @@ public final class HotwordDetectedResult implements Parcelable {
}
@DataClass.Generated(
time = 1616965644404L,
time = 1619059352684L,
codegenVersion = "1.0.23",
sourceFile = "frameworks/base/core/java/android/service/voice/HotwordDetectedResult.java",
inputSignatures = "public static final int BYTE_OFFSET_UNSET\nprivate final @android.service.voice.HotwordDetector.HotwordConfidenceLevelValue int mConfidenceLevel\nprivate final int mByteOffset\nprivate final int mScore\nprivate final int mPersonalizedScore\nprivate final int mHotwordPhraseId\nprivate final @android.annotation.NonNull android.os.PersistableBundle mExtras\nprivate static int defaultConfidenceLevel()\nprivate static int defaultByteOffset()\nprivate static int defaultScore()\nprivate static int defaultPersonalizedScore()\npublic static int getMaxScore()\nprivate static int defaultHotwordPhraseId()\npublic static int getMaxHotwordPhraseId()\nprivate static android.os.PersistableBundle defaultExtras()\npublic static int getMaxBundleSize()\nclass HotwordDetectedResult extends java.lang.Object implements [android.os.Parcelable]\n@com.android.internal.util.DataClass(genConstructor=false, genBuilder=true, genEqualsHashCode=true, genHiddenConstDefs=true, genParcelable=true, genToString=true)")
inputSignatures = "public static final int BYTE_OFFSET_UNSET\nprivate final @android.service.voice.HotwordDetector.HotwordConfidenceLevelValue int mConfidenceLevel\nprivate @android.annotation.Nullable android.media.MediaSyncEvent mMediaSyncEvent\nprivate final int mByteOffset\nprivate final int mScore\nprivate final int mPersonalizedScore\nprivate final int mHotwordPhraseId\nprivate final @android.annotation.NonNull android.os.PersistableBundle mExtras\nprivate static int defaultConfidenceLevel()\nprivate static int defaultByteOffset()\nprivate static int defaultScore()\nprivate static int defaultPersonalizedScore()\npublic static int getMaxScore()\nprivate static int defaultHotwordPhraseId()\npublic static int getMaxHotwordPhraseId()\nprivate static android.os.PersistableBundle defaultExtras()\npublic static int getMaxBundleSize()\nclass HotwordDetectedResult extends java.lang.Object implements [android.os.Parcelable]\n@com.android.internal.util.DataClass(genConstructor=false, genBuilder=true, genEqualsHashCode=true, genHiddenConstDefs=true, genParcelable=true, genToString=true)")
@Deprecated
private void __metadata() {}

61
core/java/android/service/voice/HotwordDetectionService.java
View File

@@ -30,6 +30,7 @@ import android.app.Service;
import android.content.ContentCaptureOptions;
import android.content.Context;
import android.content.Intent;
import android.hardware.soundtrigger.SoundTrigger;
import android.media.AudioFormat;
import android.os.Bundle;
import android.os.Handler;
@@ -133,7 +134,7 @@ public abstract class HotwordDetectionService extends Service {
private final IHotwordDetectionService mInterface = new IHotwordDetectionService.Stub() {
@Override
public void detectFromDspSource(
ParcelFileDescriptor audioStream,
SoundTrigger.KeyphraseRecognitionEvent event,
AudioFormat audioFormat,
long timeoutMillis,
IDspHotwordDetectionCallback callback)
@@ -143,8 +144,9 @@ public abstract class HotwordDetectionService extends Service {
}
mHandler.sendMessage(obtainMessage(HotwordDetectionService::onDetect,
HotwordDetectionService.this,
audioStream,
audioFormat,
new AlwaysOnHotwordDetector.EventPayload(
event.triggerInData, event.captureAvailable,
event.captureFormat, event.captureSession, event.data),
timeoutMillis,
new Callback(callback)));
}
@@ -178,8 +180,6 @@ public abstract class HotwordDetectionService extends Service {
mHandler.sendMessage(obtainMessage(
HotwordDetectionService::onDetect,
HotwordDetectionService.this,
audioStream,
audioFormat,
new Callback(callback)));
break;
case AUDIO_SOURCE_EXTERNAL:
@@ -246,13 +246,42 @@ public abstract class HotwordDetectionService extends Service {
* the application fails to abide by the timeout, system will close the
* microphone and cancel the operation.
* @param callback The callback to use for responding to the detection request.
* @deprecated Implement
* {@link #onDetect(AlwaysOnHotwordDetector.EventPayload, long, Callback)} instead.
*
* @hide
*/
@Deprecated
@SystemApi
public void onDetect(
@NonNull ParcelFileDescriptor audioStream,
@NonNull AudioFormat audioFormat,
@DurationMillisLong long timeoutMillis,
@NonNull Callback callback) {
// TODO: Add a helpful error message.
throw new UnsupportedOperationException();
}
/**
* Called when the device hardware (such as a DSP) detected the hotword, to request second stage
* validation before handing over the audio to the {@link AlwaysOnHotwordDetector}.
* <p>
* After {@code callback} is invoked or {@code timeoutMillis} has passed, and invokes the
* appropriate {@link AlwaysOnHotwordDetector.Callback callback}.
*
* @param eventPayload Payload data for the hardware detection event. This may contain the
* trigger audio, if requested when calling
* {@link AlwaysOnHotwordDetector#startRecognition(int)}.
* @param timeoutMillis Timeout in milliseconds for the operation to invoke the callback. If
* the application fails to abide by the timeout, system will close the
* microphone and cancel the operation.
* @param callback The callback to use for responding to the detection request.
*
* @hide
*/
@SystemApi
public void onDetect(
@NonNull ParcelFileDescriptor audioStream,
@NonNull AudioFormat audioFormat,
@NonNull AlwaysOnHotwordDetector.EventPayload eventPayload,
@DurationMillisLong long timeoutMillis,
@NonNull Callback callback) {
// TODO: Add a helpful error message.
@@ -305,7 +334,9 @@ public abstract class HotwordDetectionService extends Service {
* @param audioFormat Format of the supplied audio
* @param callback The callback to use for responding to the detection request.
* {@link Callback#onRejected(HotwordRejectedResult) callback.onRejected} cannot be used here.
* @deprecated Implement {@link #onDetect(Callback)} instead.
*/
@Deprecated
public void onDetect(
@NonNull ParcelFileDescriptor audioStream,
@NonNull AudioFormat audioFormat,
@@ -314,6 +345,22 @@ public abstract class HotwordDetectionService extends Service {
throw new UnsupportedOperationException();
}
/**
* Called when the {@link VoiceInteractionService} requests that this service
* {@link HotwordDetector#startRecognition() start} hotword recognition on audio coming directly
* from the device microphone.
* <p>
* On successful detection of a hotword, call
* {@link Callback#onDetected(HotwordDetectedResult)}.
*
* @param callback The callback to use for responding to the detection request.
* {@link Callback#onRejected(HotwordRejectedResult) callback.onRejected} cannot be used here.
*/
public void onDetect(@NonNull Callback callback) {
// TODO: Add a helpful error message.
throw new UnsupportedOperationException();
}
/**
* Called when the {@link VoiceInteractionService} requests that this service
* {@link HotwordDetector#startRecognition(ParcelFileDescriptor, AudioFormat,

5
core/java/android/service/voice/IHotwordDetectionService.aidl
View File

@@ -16,6 +16,8 @@
package android.service.voice;
import android.content.ContentCaptureOptions;
import android.hardware.soundtrigger.SoundTrigger;
import android.media.AudioFormat;
import android.os.IRemoteCallback;
import android.os.ParcelFileDescriptor;
@@ -23,7 +25,6 @@ import android.os.PersistableBundle;
import android.os.SharedMemory;
import android.service.voice.IDspHotwordDetectionCallback;
import android.view.contentcapture.IContentCaptureManager;
import android.content.ContentCaptureOptions;
/**
* Provide the interface to communicate with hotword detection service.
@@ -32,7 +33,7 @@ import android.content.ContentCaptureOptions;
*/
oneway interface IHotwordDetectionService {
void detectFromDspSource(
in ParcelFileDescriptor audioStream,
in SoundTrigger.KeyphraseRecognitionEvent event,
in AudioFormat audioFormat,
long timeoutMillis,
in IDspHotwordDetectionCallback callback);

56
core/java/android/service/voice/VoiceInteractionService.java
View File

@@ -420,9 +420,13 @@ public class VoiceInteractionService extends Service {
*
* @see #createAlwaysOnHotwordDetector(String, Locale, PersistableBundle, SharedMemory,
* AlwaysOnHotwordDetector.Callback)
* @deprecated Use
* {@link #createHotwordDetector(PersistableBundle, SharedMemory, HotwordDetector.Callback)}
* instead.
*
* @hide
*/
@Deprecated
@SystemApi
@RequiresPermission(Manifest.permission.MANAGE_HOTWORD_DETECTION)
@NonNull
@@ -444,6 +448,58 @@ public class VoiceInteractionService extends Service {
return mSoftwareHotwordDetector;
}
/**
* Creates a {@link HotwordDetector} and initializes the application's
* {@link HotwordDetectionService} using {@code options} and {code sharedMemory}.
*
* <p>To be able to call this, you need to set android:hotwordDetectionService in the
* android.voice_interaction metadata file to a valid hotword detection service, and set
* android:isolatedProcess="true" in the hotword detection service's declaration. Otherwise,
* this throws an {@link IllegalStateException}.
*
* <p>This instance must be retained and used by the client.
* Calling this a second time invalidates the previously created hotword detector
* which can no longer be used to manage recognition.
*
* <p>Using this has a noticeable impact on battery, since the microphone is kept open
* for the lifetime of the recognition {@link HotwordDetector#startRecognition() session}. On
* devices where hardware filtering is available (such as through a DSP), it's highly
* recommended to use {@link #createAlwaysOnHotwordDetector} instead.
*
* @param options Application configuration data to be provided to the
* {@link HotwordDetectionService}. PersistableBundle does not allow any remotable objects or
* other contents that can be used to communicate with other processes.
* @param sharedMemory The unrestricted data blob to be provided to the
* {@link HotwordDetectionService}. Use this to provide hotword models or other such data to the
* sandboxed process.
* @param callback The callback to notify of detection events.
* @return A hotword detector for the given audio format.
*
* @see #createAlwaysOnHotwordDetector(String, Locale, PersistableBundle, SharedMemory,
* AlwaysOnHotwordDetector.Callback)
*
* @hide
*/
@SystemApi
@RequiresPermission(Manifest.permission.MANAGE_HOTWORD_DETECTION)
@NonNull
public final HotwordDetector createHotwordDetector(
@Nullable PersistableBundle options,
@Nullable SharedMemory sharedMemory,
@NonNull HotwordDetector.Callback callback) {
if (mSystemService == null) {
throw new IllegalStateException("Not available until onReady() is called");
}
synchronized (mLock) {
// Allow only one concurrent recognition via the APIs.
safelyShutdownHotwordDetector();
mSoftwareHotwordDetector =
new SoftwareHotwordDetector(
mSystemService, null, options, sharedMemory, callback);
}
return mSoftwareHotwordDetector;
}
/**
* Creates an {@link KeyphraseModelManager} to use for enrolling voice models outside of the
* pre-bundled system voice models.

167
services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java
View File

@@ -65,7 +65,6 @@ import java.io.PrintWriter;
import java.util.concurrent.Executor;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -235,19 +234,32 @@ final class HotwordDetectionConnection {
Slog.d(TAG, "startListeningFromMic");
}
AudioRecord audioRecord = createMicAudioRecord(audioFormat);
if (audioRecord == null) {
// TODO: Callback.onError();
return;
}
// TODO: consider making this a non-anonymous class.
IDspHotwordDetectionCallback internalCallback = new IDspHotwordDetectionCallback.Stub() {
@Override
public void onDetected(HotwordDetectedResult result) throws RemoteException {
if (DEBUG) {
Slog.d(TAG, "onDetected");
}
callback.onDetected(result, null, null);
}
handleSoftwareHotwordDetection(
audioFormat,
AudioReader.createFromAudioRecord(audioRecord),
AUDIO_SOURCE_MICROPHONE,
// TODO: handle bundles better.
new PersistableBundle(),
callback);
@Override
public void onRejected(HotwordRejectedResult result) throws RemoteException {
if (DEBUG) {
Slog.d(TAG, "onRejected");
}
// onRejected isn't allowed here
}
};
mRemoteHotwordDetectionService.run(
service -> service.detectFromMicrophoneSource(
null,
AUDIO_SOURCE_MICROPHONE,
null,
null,
internalCallback));
}
public void startListeningFromExternalSource(
@@ -298,74 +310,12 @@ final class HotwordDetectionConnection {
if (DEBUG) {
Slog.d(TAG, "detectFromDspSourceForTest");
}
AudioRecord record = createFakeAudioRecord();
if (record == null) {
Slog.d(TAG, "Failed to create fake audio record");
return;
}
Pair<ParcelFileDescriptor, ParcelFileDescriptor> clientPipe = createPipe();
if (clientPipe == null) {
Slog.d(TAG, "Failed to create pipe");
return;
}
ParcelFileDescriptor audioSink = clientPipe.second;
ParcelFileDescriptor clientRead = clientPipe.first;
record.startRecording();
mAudioCopyExecutor.execute(() -> {
try (OutputStream fos =
new ParcelFileDescriptor.AutoCloseOutputStream(audioSink)) {
int remainToRead = 10240;
byte[] buffer = new byte[1024];
while (remainToRead > 0) {
int bytesRead = record.read(buffer, 0, 1024);
if (DEBUG) {
Slog.d(TAG, "bytesRead = " + bytesRead);
}
if (bytesRead <= 0) {
break;
}
if (bytesRead > 8) {
System.arraycopy(new byte[] {'h', 'o', 't', 'w', 'o', 'r', 'd', '!'}, 0,
buffer, 0, 8);
}
fos.write(buffer, 0, bytesRead);
remainToRead -= bytesRead;
}
} catch (IOException e) {
Slog.w(TAG, "Failed supplying audio data to validator", e);
}
});
Runnable cancellingJob = () -> {
Slog.d(TAG, "Timeout for getting callback from HotwordDetectionService");
record.stop();
record.release();
bestEffortClose(audioSink);
bestEffortClose(clientRead);
};
ScheduledFuture<?> cancelingFuture =
mScheduledExecutorService.schedule(
cancellingJob, VALIDATION_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS);
IDspHotwordDetectionCallback internalCallback = new IDspHotwordDetectionCallback.Stub() {
@Override
public void onDetected(HotwordDetectedResult result) throws RemoteException {
if (DEBUG) {
Slog.d(TAG, "onDetected");
}
cancelingFuture.cancel(true);
record.stop();
record.release();
bestEffortClose(audioSink);
bestEffortClose(clientRead);
externalCallback.onKeyphraseDetected(recognitionEvent);
}
@@ -374,19 +324,13 @@ final class HotwordDetectionConnection {
if (DEBUG) {
Slog.d(TAG, "onRejected");
}
cancelingFuture.cancel(true);
record.stop();
record.release();
bestEffortClose(audioSink);
bestEffortClose(clientRead);
externalCallback.onRejected(result);
}
};
mRemoteHotwordDetectionService.run(
service -> service.detectFromDspSource(
clientRead,
recognitionEvent,
recognitionEvent.getCaptureFormat(),
VALIDATION_TIMEOUT_MILLIS,
internalCallback));
@@ -398,49 +342,6 @@ final class HotwordDetectionConnection {
Slog.d(TAG, "detectFromDspSource");
}
AudioRecord record = createAudioRecord(recognitionEvent);
Pair<ParcelFileDescriptor, ParcelFileDescriptor> clientPipe = createPipe();
if (clientPipe == null) {
// Error.
// Need to propagate as unknown error or something?
return;
}
ParcelFileDescriptor audioSink = clientPipe.second;
ParcelFileDescriptor clientRead = clientPipe.first;
record.startRecording();
mAudioCopyExecutor.execute(() -> {
try (OutputStream fos =
new ParcelFileDescriptor.AutoCloseOutputStream(audioSink)) {
byte[] buffer = new byte[1024];
while (true) {
int bytesRead = record.read(buffer, 0, 1024);
if (bytesRead < 0) {
break;
}
fos.write(buffer, 0, bytesRead);
}
} catch (IOException e) {
Slog.w(TAG, "Failed supplying audio data to validator", e);
}
});
Runnable cancellingJob = () -> {
record.stop();
bestEffortClose(audioSink);
// TODO: consider calling externalCallback.onRejected(ERROR_TIMEOUT).
};
ScheduledFuture<?> cancelingFuture =
mScheduledExecutorService.schedule(
cancellingJob, VALIDATION_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS);
// TODO: consider making this a non-anonymous class.
IDspHotwordDetectionCallback internalCallback = new IDspHotwordDetectionCallback.Stub() {
@Override
@@ -448,18 +349,6 @@ final class HotwordDetectionConnection {
if (DEBUG) {
Slog.d(TAG, "onDetected");
}
bestEffortClose(audioSink);
cancelingFuture.cancel(true);
// Give 2 more seconds for the interactor to start consuming the mic. If it fails to
// do so under the given time, we'll force-close the mic to make sure resources are
// freed up.
// TODO: consider modelling these 2 seconds in the API.
mScheduledExecutorService.schedule(
cancellingJob,
VOICE_INTERACTION_TIMEOUT_TO_OPEN_MIC_MILLIS,
TimeUnit.MILLISECONDS);
// TODO: Propagate the HotwordDetectedResult.
externalCallback.onKeyphraseDetected(recognitionEvent);
}
@@ -469,18 +358,16 @@ final class HotwordDetectionConnection {
if (DEBUG) {
Slog.d(TAG, "onRejected");
}
cancelingFuture.cancel(true);
externalCallback.onRejected(result);
}
};
mRemoteHotwordDetectionService.run(
service -> service.detectFromDspSource(
clientRead,
recognitionEvent,
recognitionEvent.getCaptureFormat(),
VALIDATION_TIMEOUT_MILLIS,
internalCallback));
bestEffortClose(clientRead);
}
static final class SoundTriggerCallback extends IRecognitionStatusCallback.Stub {