Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge "Introduce the concept of software and external hotword." into sc-dev

Browse Source
This commit is contained in:
Sergey Volnov
2021-03-26 12:28:21 +00:00
committed by Android (Google) Code Review
parent c247fad24e baceb11e71
commit 86d7e7d2e0
14 changed files with 1087 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
core/api/system-current.txt
View File

@@ -10338,7 +10338,7 @@ package android.service.trust {
package android.service.voice {
public class AlwaysOnHotwordDetector {
public class AlwaysOnHotwordDetector implements android.service.voice.HotwordDetector {
method @Nullable public android.content.Intent createEnrollIntent();
method @Nullable public android.content.Intent createReEnrollIntent();
method @Nullable public android.content.Intent createUnEnrollIntent();
@@ -10348,6 +10348,8 @@ package android.service.voice {
method @Nullable @RequiresPermission(allOf={android.Manifest.permission.RECORD_AUDIO, android.Manifest.permission.CAPTURE_AUDIO_HOTWORD}) public android.service.voice.AlwaysOnHotwordDetector.ModelParamRange queryParameter(int);
method @RequiresPermission(allOf={android.Manifest.permission.RECORD_AUDIO, android.Manifest.permission.CAPTURE_AUDIO_HOTWORD}) public int setParameter(int, int);
method @RequiresPermission(allOf={android.Manifest.permission.RECORD_AUDIO, android.Manifest.permission.CAPTURE_AUDIO_HOTWORD}) public boolean startRecognition(int);
method @RequiresPermission(allOf={android.Manifest.permission.RECORD_AUDIO, android.Manifest.permission.CAPTURE_AUDIO_HOTWORD}) public boolean startRecognition();
method @Nullable public boolean startRecognition(@NonNull android.os.ParcelFileDescriptor, @NonNull android.media.AudioFormat, @Nullable android.os.PersistableBundle);
method @RequiresPermission(allOf={android.Manifest.permission.RECORD_AUDIO, android.Manifest.permission.CAPTURE_AUDIO_HOTWORD}) public boolean stopRecognition();
method public final void updateState(@Nullable android.os.PersistableBundle, @Nullable android.os.SharedMemory);
field public static final int AUDIO_CAPABILITY_ECHO_CANCELLATION = 1; // 0x1
@@ -10367,13 +10369,9 @@ package android.service.voice {
field @Deprecated public static final int STATE_KEYPHRASE_UNSUPPORTED = -1; // 0xffffffff
}
public abstract static class AlwaysOnHotwordDetector.Callback {
public abstract static class AlwaysOnHotwordDetector.Callback implements android.service.voice.HotwordDetector.Callback {
ctor public AlwaysOnHotwordDetector.Callback();
method public abstract void onAvailabilityChanged(int);
method public abstract void onDetected(@NonNull android.service.voice.AlwaysOnHotwordDetector.EventPayload);
method public abstract void onError();
method public abstract void onRecognitionPaused();
method public abstract void onRecognitionResumed();
method public void onRejected(@Nullable android.service.voice.HotwordRejectedResult);
}
@@ -10418,23 +10416,36 @@ package android.service.voice {
public abstract class HotwordDetectionService extends android.app.Service {
ctor public HotwordDetectionService();
method @Nullable public final android.os.IBinder onBind(@NonNull android.content.Intent);
method public void onDetectFromDspSource(@NonNull android.os.ParcelFileDescriptor, @NonNull android.media.AudioFormat, long, @NonNull android.service.voice.HotwordDetectionService.DspHotwordDetectionCallback);
method public void onDetect(@NonNull android.os.ParcelFileDescriptor, @NonNull android.media.AudioFormat, long, @NonNull android.service.voice.HotwordDetectionService.Callback);
method public void onDetect(@NonNull android.os.ParcelFileDescriptor, @NonNull android.media.AudioFormat, @NonNull android.service.voice.HotwordDetectionService.Callback);
method public void onDetect(@NonNull android.os.ParcelFileDescriptor, @NonNull android.media.AudioFormat, @Nullable android.os.PersistableBundle, @NonNull android.service.voice.HotwordDetectionService.Callback);
method public void onUpdateState(@Nullable android.os.PersistableBundle, @Nullable android.os.SharedMemory);
field public static final String SERVICE_INTERFACE = "android.service.voice.HotwordDetectionService";
}
public static final class HotwordDetectionService.DspHotwordDetectionCallback {
method public void onDetected();
public static final class HotwordDetectionService.Callback {
method public void onDetected(@Nullable android.service.voice.HotwordDetectedResult);
method public void onRejected(@Nullable android.service.voice.HotwordRejectedResult);
}
public interface HotwordDetector {
method @RequiresPermission(allOf={android.Manifest.permission.RECORD_AUDIO, android.Manifest.permission.CAPTURE_AUDIO_HOTWORD}) public boolean startRecognition();
method public boolean startRecognition(@NonNull android.os.ParcelFileDescriptor, @NonNull android.media.AudioFormat, @Nullable android.os.PersistableBundle);
method public boolean stopRecognition();
field public static final int CONFIDENCE_LEVEL_HIGH = 3; // 0x3
field public static final int CONFIDENCE_LEVEL_LOW = 1; // 0x1
field public static final int CONFIDENCE_LEVEL_MEDIUM = 2; // 0x2
field public static final int CONFIDENCE_LEVEL_NONE = 0; // 0x0
}
public static interface HotwordDetector.Callback {
method public void onDetected(@NonNull android.service.voice.AlwaysOnHotwordDetector.EventPayload);
method public void onError();
method public void onRecognitionPaused();
method public void onRecognitionResumed();
method public void onRejected(@Nullable android.service.voice.HotwordRejectedResult);
}
public final class HotwordRejectedResult implements android.os.Parcelable {
method public int describeContents();
method public int getConfidenceLevel();
@@ -10445,6 +10456,7 @@ package android.service.voice {
public class VoiceInteractionService extends android.app.Service {
method @NonNull public final android.service.voice.AlwaysOnHotwordDetector createAlwaysOnHotwordDetector(String, java.util.Locale, android.service.voice.AlwaysOnHotwordDetector.Callback);
method @NonNull @RequiresPermission(android.Manifest.permission.MANAGE_HOTWORD_DETECTION) public final android.service.voice.AlwaysOnHotwordDetector createAlwaysOnHotwordDetector(String, java.util.Locale, @Nullable android.os.PersistableBundle, @Nullable android.os.SharedMemory, android.service.voice.AlwaysOnHotwordDetector.Callback);
method @NonNull @RequiresPermission(android.Manifest.permission.MANAGE_HOTWORD_DETECTION) public final android.service.voice.HotwordDetector createHotwordDetector(@NonNull android.media.AudioFormat, @Nullable android.os.PersistableBundle, @Nullable android.os.SharedMemory, @NonNull android.service.voice.HotwordDetector.Callback);
method @NonNull @RequiresPermission("android.permission.MANAGE_VOICE_KEYPHRASES") public final android.media.voice.KeyphraseModelManager createKeyphraseModelManager();
}

105
core/java/android/service/voice/AbstractHotwordDetector.java Normal file
View File

@@ -0,0 +1,105 @@
/*
* Copyright (C) 2021 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package android.service.voice;
import static com.android.internal.util.function.pooled.PooledLambda.obtainMessage;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.media.AudioFormat;
import android.os.Handler;
import android.os.Looper;
import android.os.ParcelFileDescriptor;
import android.os.PersistableBundle;
import android.os.RemoteException;
import android.util.Slog;
import com.android.internal.app.IVoiceInteractionManagerService;
/** Base implementation of {@link HotwordDetector}. */
abstract class AbstractHotwordDetector implements HotwordDetector {
private static final String TAG = AbstractHotwordDetector.class.getSimpleName();
private static final boolean DEBUG = false;
private final IVoiceInteractionManagerService mManagerService;
private final Handler mHandler;
private final HotwordDetector.Callback mCallback;
AbstractHotwordDetector(
IVoiceInteractionManagerService managerService,
HotwordDetector.Callback callback) {
mManagerService = managerService;
// TODO: this needs to be supplied from above
mHandler = new Handler(Looper.getMainLooper());
mCallback = callback;
}
/**
* Detect hotword from an externally supplied stream of data.
*
* @return a writeable file descriptor that clients can start writing data in the given format.
* In order to stop detection, clients can close the given stream.
*/
@Nullable
@Override
public boolean startRecognition(
@NonNull ParcelFileDescriptor audioStream,
@NonNull AudioFormat audioFormat,
@Nullable PersistableBundle options) {
if (DEBUG) {
Slog.i(TAG, "#recognizeHotword");
}
// TODO: consider closing existing session.
try {
mManagerService.startListeningFromExternalSource(
audioStream,
audioFormat,
options,
new BinderCallback(mHandler, mCallback));
} catch (RemoteException e) {
e.rethrowFromSystemServer();
}
return true;
}
private static class BinderCallback
extends IMicrophoneHotwordDetectionVoiceInteractionCallback.Stub {
private final Handler mHandler;
// TODO: these need to be weak references.
private final HotwordDetector.Callback mCallback;
BinderCallback(Handler handler, HotwordDetector.Callback callback) {
this.mHandler = handler;
this.mCallback = callback;
}
/** TODO: onDetected */
@Override
public void onDetected(
@Nullable HotwordDetectedResult hotwordDetectedResult,
@Nullable AudioFormat audioFormat,
@Nullable ParcelFileDescriptor audioStreamIgnored) {
mHandler.sendMessage(obtainMessage(
HotwordDetector.Callback::onDetected,
mCallback,
new AlwaysOnHotwordDetector.EventPayload(audioFormat, hotwordDetectedResult)));
}
}
}

20
core/java/android/service/voice/AlwaysOnHotwordDetector.java
View File

@@ -68,7 +68,7 @@ import java.util.Locale;
* mark and track it as such.
*/
@SystemApi
public class AlwaysOnHotwordDetector {
public class AlwaysOnHotwordDetector extends AbstractHotwordDetector {
//---- States of Keyphrase availability. Return codes for onAvailabilityChanged() ----//
/**
* Indicates that this hotword detector is no longer valid for any recognition
@@ -459,7 +459,7 @@ public class AlwaysOnHotwordDetector {
/**
* Callbacks for always-on hotword detection.
*/
public static abstract class Callback {
public abstract static class Callback implements HotwordDetector.Callback {
/**
* Updates the availability state of the active keyphrase and locale on every keyphrase
@@ -547,11 +547,13 @@ public class AlwaysOnHotwordDetector {
IVoiceInteractionManagerService modelManagementService, int targetSdkVersion,
boolean supportHotwordDetectionService, @Nullable PersistableBundle options,
@Nullable SharedMemory sharedMemory) {
super(modelManagementService, callback);
mHandler = new MyHandler();
mText = text;
mLocale = locale;
mKeyphraseEnrollmentInfo = keyphraseEnrollmentInfo;
mExternalCallback = callback;
mHandler = new MyHandler();
mInternalCallback = new SoundTriggerListener(mHandler);
mModelManagementService = modelManagementService;
mTargetSdkVersion = targetSdkVersion;
@@ -704,6 +706,17 @@ public class AlwaysOnHotwordDetector {
}
}
/**
* Starts recognition for the associated keyphrase.
*
* @see #startRecognition(int)
*/
@RequiresPermission(allOf = {RECORD_AUDIO, CAPTURE_AUDIO_HOTWORD})
@Override
public boolean startRecognition() {
return startRecognition(0);
}
/**
* Stops recognition for the associated keyphrase.
* Caller must be the active voice interaction service via
@@ -718,6 +731,7 @@ public class AlwaysOnHotwordDetector {
* {@link VoiceInteractionService} hosting this detector has been shut down.
*/
@RequiresPermission(allOf = {RECORD_AUDIO, CAPTURE_AUDIO_HOTWORD})
@Override
public boolean stopRecognition() {
if (DBG) Slog.d(TAG, "stopRecognition()");
synchronized (mLock) {

149
core/java/android/service/voice/HotwordDetectionService.java
View File

@@ -20,6 +20,7 @@ import static com.android.internal.util.function.pooled.PooledLambda.obtainMessa
import android.annotation.CallSuper;
import android.annotation.DurationMillisLong;
import android.annotation.IntDef;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.annotation.SdkConstant;
@@ -36,6 +37,9 @@ import android.os.RemoteException;
import android.os.SharedMemory;
import android.util.Log;
import java.lang.annotation.Documented;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.util.Locale;
/**
@@ -50,6 +54,24 @@ public abstract class HotwordDetectionService extends Service {
// TODO (b/177502877): Set the Debug flag to false before shipping.
private static final boolean DBG = true;
/**
* Source for the given audio stream.
*
* @hide
*/
@Documented
@Retention(RetentionPolicy.SOURCE)
@IntDef({
AUDIO_SOURCE_MICROPHONE,
AUDIO_SOURCE_EXTERNAL
})
@interface AudioSource {}
/** @hide */
public static final int AUDIO_SOURCE_MICROPHONE = 1;
/** @hide */
public static final int AUDIO_SOURCE_EXTERNAL = 2;
/**
* The {@link Intent} that must be declared as handled by the service.
* To be supported, the service must also require the
@@ -73,12 +95,12 @@ public abstract class HotwordDetectionService extends Service {
if (DBG) {
Log.d(TAG, "#detectFromDspSource");
}
mHandler.sendMessage(obtainMessage(HotwordDetectionService::onDetectFromDspSource,
mHandler.sendMessage(obtainMessage(HotwordDetectionService::onDetect,
HotwordDetectionService.this,
audioStream,
audioFormat,
timeoutMillis,
new DspHotwordDetectionCallback(callback)));
new Callback(callback)));
}
@Override
@@ -92,6 +114,40 @@ public abstract class HotwordDetectionService extends Service {
options,
sharedMemory));
}
@Override
public void detectFromMicrophoneSource(
ParcelFileDescriptor audioStream,
@AudioSource int audioSource,
AudioFormat audioFormat,
PersistableBundle options,
IDspHotwordDetectionCallback callback)
throws RemoteException {
if (DBG) {
Log.d(TAG, "#detectFromMicrophoneSource");
}
switch (audioSource) {
case AUDIO_SOURCE_MICROPHONE:
mHandler.sendMessage(obtainMessage(
HotwordDetectionService::onDetect,
HotwordDetectionService.this,
audioStream,
audioFormat,
new Callback(callback)));
break;
case AUDIO_SOURCE_EXTERNAL:
mHandler.sendMessage(obtainMessage(
HotwordDetectionService::onDetect,
HotwordDetectionService.this,
audioStream,
audioFormat,
options,
new Callback(callback)));
break;
default:
Log.i(TAG, "Unsupported audio source " + audioSource);
}
}
};
@CallSuper
@@ -113,27 +169,30 @@ public abstract class HotwordDetectionService extends Service {
}
/**
* Detect the audio data generated from Dsp.
*
* <p>Note: the clients are supposed to call {@code close} on the input stream when they are
* done with the operation in order to free up resources.
* Called when the device hardware (such as a DSP) detected the hotword, to request second stage
* validation before handing over the audio to the {@link AlwaysOnHotwordDetector}.
* <p>
* After {@code callback} is invoked or {@code timeoutMillis} has passed, the system closes
* {@code audioStream} and invokes the appropriate {@link AlwaysOnHotwordDetector.Callback
* callback}.
*
* @param audioStream Stream containing audio bytes returned from DSP
* @param audioFormat Format of the supplied audio
* @param timeoutMillis Timeout in milliseconds for the operation to invoke the callback. If
* the application fails to abide by the timeout, system will close the
* microphone and cancel the operation.
* @param callback Use {@link HotwordDetectionService#DspHotwordDetectionCallback} to return
* the detected result.
* @param callback The callback to use for responding to the detection request.
*
* @hide
*/
@SystemApi
public void onDetectFromDspSource(
public void onDetect(
@NonNull ParcelFileDescriptor audioStream,
@NonNull AudioFormat audioFormat,
@DurationMillisLong long timeoutMillis,
@NonNull DspHotwordDetectionCallback callback) {
@NonNull Callback callback) {
// TODO: Add a helpful error message.
throw new UnsupportedOperationException();
}
/**
@@ -154,38 +213,94 @@ public abstract class HotwordDetectionService extends Service {
@SystemApi
public void onUpdateState(@Nullable PersistableBundle options,
@Nullable SharedMemory sharedMemory) {
// TODO: Handle the unimplemented case by throwing?
}
/**
* Callback for returning the detected result.
* Called when the {@link VoiceInteractionService} requests that this service
* {@link HotwordDetector#startRecognition() start} hotword recognition on audio coming directly
* from the device microphone.
* <p>
* On such a request, the system streams mic audio to this service through {@code audioStream}.
* Audio is streamed until {@link HotwordDetector#stopRecognition()} is called, at which point
* the system closes {code audioStream}.
* <p>
* On successful detection of a hotword within {@code audioStream}, call
* {@link Callback#onDetected(HotwordDetectedResult)}. The system continues to stream audio
* through {@code audioStream}; {@code callback} is reusable.
*
* @param audioStream Stream containing audio bytes returned from a microphone
* @param audioFormat Format of the supplied audio
* @param callback The callback to use for responding to the detection request.
* {@link Callback#onRejected(HotwordRejectedResult) callback.onRejected} cannot be used here.
*/
public void onDetect(
@NonNull ParcelFileDescriptor audioStream,
@NonNull AudioFormat audioFormat,
@NonNull Callback callback) {
// TODO: Add a helpful error message.
throw new UnsupportedOperationException();
}
/**
* Called when the {@link VoiceInteractionService} requests that this service
* {@link HotwordDetector#startRecognition(ParcelFileDescriptor, AudioFormat,
* PersistableBundle)} run} hotword recognition on audio coming from an external connected
* microphone.
* <p>
* Upon invoking the {@code callback}, the system closes {@code audioStream} and sends the
* detection result to the {@link HotwordDetector.Callback hotword detector}.
*
* @param audioStream Stream containing audio bytes returned from a microphone
* @param audioFormat Format of the supplied audio
* @param options Options supporting detection, such as configuration specific to the source of
* the audio, provided through
* {@link HotwordDetector#startRecognition(ParcelFileDescriptor, AudioFormat,
* PersistableBundle)}.
* @param callback The callback to use for responding to the detection request.
*/
public void onDetect(
@NonNull ParcelFileDescriptor audioStream,
@NonNull AudioFormat audioFormat,
@Nullable PersistableBundle options,
@NonNull Callback callback) {
// TODO: Add a helpful error message.
throw new UnsupportedOperationException();
}
/**
* Callback for returning the detection result.
*
* @hide
*/
@SystemApi
public static final class DspHotwordDetectionCallback {
public static final class Callback {
// TODO: need to make sure we don't store remote references, but not a high priority.
private final IDspHotwordDetectionCallback mRemoteCallback;
private DspHotwordDetectionCallback(IDspHotwordDetectionCallback remoteCallback) {
private Callback(IDspHotwordDetectionCallback remoteCallback) {
mRemoteCallback = remoteCallback;
}
/**
* Called when the detected result is valid.
*/
public void onDetected() {
public void onDetected(@Nullable HotwordDetectedResult hotwordDetectedResult) {
try {
mRemoteCallback.onDetected();
mRemoteCallback.onDetected(hotwordDetectedResult);
} catch (RemoteException e) {
throw e.rethrowFromSystemServer();
}
}
/**
* Informs the {@link AlwaysOnHotwordDetector} that the keyphrase was not detected.
* Informs the {@link HotwordDetector} that the keyphrase was not detected.
* <p>
* This cannot not be used when recognition is done through
* {@link #onDetect(ParcelFileDescriptor, AudioFormat, Callback)}.
*
* @param result Info about the second stage detection result. This is provided to
* the {@link AlwaysOnHotwordDetector}.
* the {@link HotwordDetector}.
*/
public void onRejected(@Nullable HotwordRejectedResult result) {
try {

102
core/java/android/service/voice/HotwordDetector.java
View File

@@ -16,8 +16,17 @@
package android.service.voice;
import static android.Manifest.permission.CAPTURE_AUDIO_HOTWORD;
import static android.Manifest.permission.RECORD_AUDIO;
import android.annotation.IntDef;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.annotation.RequiresPermission;
import android.annotation.SystemApi;
import android.media.AudioFormat;
import android.os.ParcelFileDescriptor;
import android.os.PersistableBundle;
/**
* Basic functionality for hotword detectors.
@@ -40,11 +49,100 @@ public interface HotwordDetector {
int CONFIDENCE_LEVEL_HIGH = 3;
/** @hide */
@IntDef(prefix = { "CONFIDENCE_LEVEL_" }, value = {
@IntDef(prefix = {"CONFIDENCE_LEVEL_"}, value = {
CONFIDENCE_LEVEL_NONE,
CONFIDENCE_LEVEL_LOW,
CONFIDENCE_LEVEL_MEDIUM,
CONFIDENCE_LEVEL_HIGH
})
@interface HotwordConfidenceLevelValue {}
@interface HotwordConfidenceLevelValue {
}
/**
* Starts hotword recognition.
* <p>
* On calling this, the system streams audio from the device microphone to this application's
* {@link HotwordDetectionService}. Audio is streamed until {@link #stopRecognition()} is
* called.
* <p>
* On detection of a hotword,
* {@link AlwaysOnHotwordDetector.Callback#onDetected(AlwaysOnHotwordDetector.EventPayload)}
* is called on the callback provided when creating this {@link HotwordDetector}.
* <p>
* There is a noticeable impact on battery while recognition is active, so make sure to call
* {@link #stopRecognition()} when detection isn't needed.
* <p>
* Calling this again while recognition is active does nothing.
*
* @return true if the request to start recognition succeeded
*/
@RequiresPermission(allOf = {RECORD_AUDIO, CAPTURE_AUDIO_HOTWORD})
boolean startRecognition();
/**
* Stops hotword recognition.
*
* @return true if the request to stop recognition succeeded
*/
boolean stopRecognition();
/**
* Starts hotword recognition on audio coming from an external connected microphone.
* <p>
* {@link #stopRecognition()} must be called before {@code audioStream} is closed.
*
* @param audioStream stream containing the audio bytes to run detection on
* @param audioFormat format of the encoded audio
* @param options options supporting detection, such as configuration specific to the
* source of the audio. This will be provided to the {@link HotwordDetectionService}.
* PersistableBundle does not allow any remotable objects or other contents that can be
* used to communicate with other processes.
* @return true if the request to start recognition succeeded
*/
boolean startRecognition(
@NonNull ParcelFileDescriptor audioStream,
@NonNull AudioFormat audioFormat,
@Nullable PersistableBundle options);
/**
* The callback to notify of detection events.
*/
interface Callback {
/**
* Called when the keyphrase is spoken.
*
* @param eventPayload Payload data for the detection event.
*/
// TODO: Consider creating a new EventPayload that the AOHD one subclasses.
void onDetected(@NonNull AlwaysOnHotwordDetector.EventPayload eventPayload);
/**
* Called when the detection fails due to an error.
*/
void onError();
/**
* Called when the recognition is paused temporarily for some reason.
* This is an informational callback, and the clients shouldn't be doing anything here
* except showing an indication on their UI if they have to.
*/
void onRecognitionPaused();
/**
* Called when the recognition is resumed after it was temporarily paused.
* This is an informational callback, and the clients shouldn't be doing anything here
* except showing an indication on their UI if they have to.
*/
void onRecognitionResumed();
/**
* Called when the {@link HotwordDetectionService second stage detection} did not detect the
* keyphrase.
*
* @param result Info about the second stage detection result, provided by the
* {@link HotwordDetectionService}.
*/
void onRejected(@Nullable HotwordRejectedResult result);
}
}

6
core/java/android/service/voice/IDspHotwordDetectionCallback.aidl
View File

@@ -16,6 +16,7 @@
package android.service.voice;
import android.service.voice.HotwordDetectedResult;
import android.service.voice.HotwordRejectedResult;
/**
@@ -23,11 +24,14 @@ import android.service.voice.HotwordRejectedResult;
*
* @hide
*/
// TODO: Rename this.
oneway interface IDspHotwordDetectionCallback {
/**
* Called when the detected result is valid.
*/
void onDetected();
void onDetected(
in HotwordDetectedResult hotwordDetectedResult);
/**
* Sends {@code result} to the HotwordDetector.

15
core/java/android/service/voice/IHotwordDetectionService.aidl
View File

@@ -29,10 +29,17 @@ import android.service.voice.IDspHotwordDetectionCallback;
*/
oneway interface IHotwordDetectionService {
void detectFromDspSource(
in ParcelFileDescriptor audioStream,
in AudioFormat audioFormat,
long timeoutMillis,
in IDspHotwordDetectionCallback callback);
in ParcelFileDescriptor audioStream,
in AudioFormat audioFormat,
long timeoutMillis,
in IDspHotwordDetectionCallback callback);
void detectFromMicrophoneSource(
in ParcelFileDescriptor audioStream,
int audioSource,
in AudioFormat audioFormat,
in PersistableBundle options,
in IDspHotwordDetectionCallback callback);
void updateState(in PersistableBundle options, in SharedMemory sharedMemory);
}

36
core/java/android/service/voice/IMicrophoneHotwordDetectionVoiceInteractionCallback.aidl Normal file
View File

@@ -0,0 +1,36 @@
/*
* Copyright (C) 2021 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package android.service.voice;
import android.media.AudioFormat;
import android.service.voice.HotwordDetectedResult;
/**
* Callback for returning the detected result from the HotwordDetectionService.
*
* @hide
*/
oneway interface IMicrophoneHotwordDetectionVoiceInteractionCallback {
/**
* Called when the detected result is valid.
*/
void onDetected(
in HotwordDetectedResult hotwordDetectedResult,
in AudioFormat audioFormat,
in ParcelFileDescriptor audioStream);
}

146
core/java/android/service/voice/SoftwareHotwordDetector.java Normal file
View File

@@ -0,0 +1,146 @@
/*
* Copyright (C) 2021 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package android.service.voice;
import static android.Manifest.permission.RECORD_AUDIO;
import static com.android.internal.util.function.pooled.PooledLambda.obtainMessage;
import android.annotation.Nullable;
import android.annotation.RequiresPermission;
import android.media.AudioFormat;
import android.os.Handler;
import android.os.Looper;
import android.os.ParcelFileDescriptor;
import android.os.PersistableBundle;
import android.os.RemoteException;
import android.os.SharedMemory;
import android.util.Slog;
import com.android.internal.app.IVoiceInteractionManagerService;
import java.io.PrintWriter;
/**
* Manages hotword detection not relying on a specific hardware.
*
* <p>On devices where DSP is available it's strongly recommended to use
* {@link AlwaysOnHotwordDetector}.
*
* @hide
**/
class SoftwareHotwordDetector extends AbstractHotwordDetector {
private static final String TAG = SoftwareHotwordDetector.class.getSimpleName();
private static final boolean DEBUG = true;
private final IVoiceInteractionManagerService mManagerService;
private final HotwordDetector.Callback mCallback;
private final AudioFormat mAudioFormat;
private final Handler mHandler;
private final Object mLock = new Object();
SoftwareHotwordDetector(
IVoiceInteractionManagerService managerService,
AudioFormat audioFormat,
PersistableBundle options,
SharedMemory sharedMemory,
HotwordDetector.Callback callback) {
super(managerService, callback);
mManagerService = managerService;
mAudioFormat = audioFormat;
mCallback = callback;
mHandler = new Handler(Looper.getMainLooper());
try {
mManagerService.updateState(options, sharedMemory);
} catch (RemoteException e) {
throw e.rethrowFromSystemServer();
}
}
@RequiresPermission(RECORD_AUDIO)
@Override
public boolean startRecognition() {
if (DEBUG) {
Slog.i(TAG, "#startRecognition");
}
maybeCloseExistingSession();
try {
mManagerService.startListeningFromMic(
mAudioFormat, new BinderCallback(mHandler, mCallback));
} catch (RemoteException e) {
e.rethrowFromSystemServer();
}
return true;
}
/** TODO: stopRecognition */
@RequiresPermission(RECORD_AUDIO)
@Override
public boolean stopRecognition() {
if (DEBUG) {
Slog.i(TAG, "#stopRecognition");
}
try {
mManagerService.stopListeningFromMic();
} catch (RemoteException e) {
e.rethrowFromSystemServer();
}
return true;
}
private void maybeCloseExistingSession() {
// TODO: needs to be synchronized.
// TODO: implement this
}
private static class BinderCallback
extends IMicrophoneHotwordDetectionVoiceInteractionCallback.Stub {
private final Handler mHandler;
// TODO: this needs to be a weak reference.
private final HotwordDetector.Callback mCallback;
BinderCallback(Handler handler, HotwordDetector.Callback callback) {
this.mHandler = handler;
this.mCallback = callback;
}
/** TODO: onDetected */
@Override
public void onDetected(
@Nullable HotwordDetectedResult hotwordDetectedResult,
@Nullable AudioFormat audioFormat,
@Nullable ParcelFileDescriptor audioStream) {
mHandler.sendMessage(obtainMessage(
HotwordDetector.Callback::onDetected,
mCallback,
new AlwaysOnHotwordDetector.EventPayload(
audioFormat, hotwordDetectedResult, audioStream)));
}
}
/** @hide */
public void dump(String prefix, PrintWriter pw) {
// TODO: implement this
}
}

116
core/java/android/service/voice/VoiceInteractionService.java
View File

@@ -29,6 +29,7 @@ import android.content.ComponentName;
import android.content.Context;
import android.content.Intent;
import android.hardware.soundtrigger.KeyphraseEnrollmentInfo;
import android.media.AudioFormat;
import android.media.voice.KeyphraseModelManager;
import android.os.Bundle;
import android.os.Handler;
@@ -134,6 +135,7 @@ public class VoiceInteractionService extends Service {
private KeyphraseEnrollmentInfo mKeyphraseEnrollmentInfo;
private AlwaysOnHotwordDetector mHotwordDetector;
private SoftwareHotwordDetector mSoftwareHotwordDetector;
/**
* Called when a user has activated an affordance to launch voice assist from the Keyguard.
@@ -388,6 +390,60 @@ public class VoiceInteractionService extends Service {
return mHotwordDetector;
}
/**
* Creates a {@link HotwordDetector} and initializes the application's
* {@link HotwordDetectionService} using {@code options} and {code sharedMemory}.
*
* <p>To be able to call this, you need to set android:hotwordDetectionService in the
* android.voice_interaction metadata file to a valid hotword detection service, and set
* android:isolatedProcess="true" in the hotword detection service's declaration. Otherwise,
* this throws an {@link IllegalStateException}.
*
* <p>This instance must be retained and used by the client.
* Calling this a second time invalidates the previously created hotword detector
* which can no longer be used to manage recognition.
*
* <p>Using this has a noticeable impact on battery, since the microphone is kept open
* for the lifetime of the recognition {@link HotwordDetector#startRecognition() session}. On
* devices where hardware filtering is available (such as through a DSP), it's highly
* recommended to use {@link #createAlwaysOnHotwordDetector} instead.
*
* @param audioFormat Format of the audio to be passed to {@link HotwordDetectionService}.
* @param options Application configuration data to be provided to the
* {@link HotwordDetectionService}. PersistableBundle does not allow any remotable objects or
* other contents that can be used to communicate with other processes.
* @param sharedMemory The unrestricted data blob to be provided to the
* {@link HotwordDetectionService}. Use this to provide hotword models or other such data to the
* sandboxed process.
* @param callback The callback to notify of detection events.
* @return A hotword detector for the given audio format.
*
* @see #createAlwaysOnHotwordDetector(String, Locale, PersistableBundle, SharedMemory,
* AlwaysOnHotwordDetector.Callback)
*
* @hide
*/
@SystemApi
@RequiresPermission(Manifest.permission.MANAGE_HOTWORD_DETECTION)
@NonNull
public final HotwordDetector createHotwordDetector(
@NonNull AudioFormat audioFormat,
@Nullable PersistableBundle options,
@Nullable SharedMemory sharedMemory,
@NonNull HotwordDetector.Callback callback) {
if (mSystemService == null) {
throw new IllegalStateException("Not available until onReady() is called");
}
synchronized (mLock) {
// Allow only one concurrent recognition via the APIs.
safelyShutdownHotwordDetector();
mSoftwareHotwordDetector =
new SoftwareHotwordDetector(
mSystemService, audioFormat, options, sharedMemory, callback);
}
return mSoftwareHotwordDetector;
}
/**
* Creates an {@link KeyphraseModelManager} to use for enrolling voice models outside of the
* pre-bundled system voice models.
@@ -431,26 +487,45 @@ public class VoiceInteractionService extends Service {
private void safelyShutdownHotwordDetector() {
synchronized (mLock) {
if (mHotwordDetector == null) {
return;
}
try {
mHotwordDetector.stopRecognition();
} catch (Exception ex) {
// Ignore.
}
try {
mHotwordDetector.invalidate();
} catch (Exception ex) {
// Ignore.
}
mHotwordDetector = null;
shutdownDspHotwordDetectorLocked();
shutdownMicrophoneHotwordDetectorLocked();
}
}
private void shutdownDspHotwordDetectorLocked() {
if (mHotwordDetector == null) {
return;
}
try {
mHotwordDetector.stopRecognition();
} catch (Exception ex) {
// Ignore.
}
try {
mHotwordDetector.invalidate();
} catch (Exception ex) {
// Ignore.
}
mHotwordDetector = null;
}
private void shutdownMicrophoneHotwordDetectorLocked() {
if (mSoftwareHotwordDetector == null) {
return;
}
try {
mSoftwareHotwordDetector.stopRecognition();
} catch (Exception ex) {
// Ignore.
}
mSoftwareHotwordDetector = null;
}
/**
* Provide hints to be reflected in the system UI.
*
@@ -478,6 +553,13 @@ public class VoiceInteractionService extends Service {
} else {
mHotwordDetector.dump(" ", pw);
}
pw.println(" MicrophoneHotwordDetector");
if (mSoftwareHotwordDetector == null) {
pw.println(" NULL");
} else {
mSoftwareHotwordDetector.dump(" ", pw);
}
}
}
}

15
core/java/com/android/internal/app/IVoiceInteractionManagerService.aidl
View File

@@ -18,8 +18,10 @@ package com.android.internal.app;
import android.content.ComponentName;
import android.content.Intent;
import android.media.AudioFormat;
import android.media.permission.Identity;
import android.os.Bundle;
import android.os.ParcelFileDescriptor;
import android.os.PersistableBundle;
import android.os.RemoteCallback;
import android.os.SharedMemory;
@@ -33,6 +35,7 @@ import android.hardware.soundtrigger.KeyphraseMetadata;
import android.hardware.soundtrigger.SoundTrigger;
import android.service.voice.IVoiceInteractionService;
import android.service.voice.IVoiceInteractionSession;
import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback;
interface IVoiceInteractionManagerService {
void showSession(in Bundle sessionArgs, int flags);
@@ -242,4 +245,16 @@ interface IVoiceInteractionManagerService {
* Requests to shutdown hotword detection service.
*/
void shutdownHotwordDetectionService();
void startListeningFromMic(
in AudioFormat audioFormat,
in IMicrophoneHotwordDetectionVoiceInteractionCallback callback);
void stopListeningFromMic();
void startListeningFromExternalSource(
in ParcelFileDescriptor audioStream,
in AudioFormat audioFormat,
in PersistableBundle options,
in IMicrophoneHotwordDetectionVoiceInteractionCallback callback);
}

297
services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java
View File

@@ -16,6 +16,9 @@
package com.android.server.voiceinteraction;
import static android.service.voice.HotwordDetectionService.AUDIO_SOURCE_EXTERNAL;
import static android.service.voice.HotwordDetectionService.AUDIO_SOURCE_MICROPHONE;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.content.ComponentName;
@@ -24,23 +27,30 @@ import android.content.Intent;
import android.hardware.soundtrigger.IRecognitionStatusCallback;
import android.hardware.soundtrigger.SoundTrigger;
import android.media.AudioAttributes;
import android.media.AudioFormat;
import android.media.AudioManager;
import android.media.AudioRecord;
import android.media.MediaRecorder;
import android.os.ParcelFileDescriptor;
import android.os.PersistableBundle;
import android.os.RemoteException;
import android.os.SharedMemory;
import android.service.voice.HotwordDetectedResult;
import android.service.voice.HotwordDetectionService;
import android.service.voice.HotwordRejectedResult;
import android.service.voice.IDspHotwordDetectionCallback;
import android.service.voice.IHotwordDetectionService;
import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback;
import android.util.Pair;
import android.util.Slog;
import com.android.internal.annotations.GuardedBy;
import com.android.internal.app.IHotwordRecognitionStatusCallback;
import com.android.internal.infra.ServiceConnector;
import java.io.Closeable;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.PrintWriter;
import java.util.concurrent.Executor;
@@ -63,6 +73,8 @@ final class HotwordDetectionConnection {
private static final long VALIDATION_TIMEOUT_MILLIS = 3000;
private static final long VOICE_INTERACTION_TIMEOUT_TO_OPEN_MIC_MILLIS = 2000;
private static final int MAX_STREAMING_SECONDS = 10;
private static final int MICROPHONE_BUFFER_LENGTH_SECONDS = 8;
private static final int HOTWORD_AUDIO_LENGTH_SECONDS = 3;
private final Executor mAudioCopyExecutor = Executors.newCachedThreadPool();
// TODO: This may need to be a Handler(looper)
@@ -76,6 +88,9 @@ final class HotwordDetectionConnection {
final @NonNull ServiceConnector<IHotwordDetectionService> mRemoteHotwordDetectionService;
boolean mBound;
@GuardedBy("mLock")
private ParcelFileDescriptor mCurrentAudioSink;
HotwordDetectionConnection(Object lock, Context context, ComponentName serviceName,
int userId, boolean bindInstantServiceAllowed, @Nullable PersistableBundle options,
@Nullable SharedMemory sharedMemory) {
@@ -134,6 +149,62 @@ final class HotwordDetectionConnection {
service -> service.updateState(options, sharedMemory));
}
void startListeningFromMic(
AudioFormat audioFormat,
IMicrophoneHotwordDetectionVoiceInteractionCallback callback) {
if (DEBUG) {
Slog.d(TAG, "startListeningFromMic");
}
AudioRecord audioRecord = createMicAudioRecord(audioFormat);
if (audioRecord == null) {
// TODO: Callback.onError();
return;
}
handleSoftwareHotwordDetection(
audioFormat,
AudioReader.createFromAudioRecord(audioRecord),
AUDIO_SOURCE_MICROPHONE,
// TODO: handle bundles better.
new PersistableBundle(),
callback);
}
public void startListeningFromExternalSource(
ParcelFileDescriptor audioStream,
AudioFormat audioFormat,
@Nullable PersistableBundle options,
IMicrophoneHotwordDetectionVoiceInteractionCallback callback) {
if (DEBUG) {
Slog.d(TAG, "startListeningFromExternalSource");
}
ParcelFileDescriptor.AutoCloseInputStream audioReader =
new ParcelFileDescriptor.AutoCloseInputStream(audioStream);
handleSoftwareHotwordDetection(
audioFormat,
AudioReader.createFromInputStream(audioReader),
AUDIO_SOURCE_EXTERNAL,
options,
callback);
}
void stopListening() {
if (DEBUG) {
Slog.d(TAG, "stopListening");
}
synchronized (mLock) {
if (mCurrentAudioSink != null) {
Slog.i(TAG, "Closing audio stream to hotword detector: stopping requested");
bestEffortClose(mCurrentAudioSink);
}
mCurrentAudioSink = null;
}
}
private void detectFromDspSource(SoundTrigger.KeyphraseRecognitionEvent recognitionEvent,
IHotwordRecognitionStatusCallback externalCallback) {
if (DEBUG) {
@@ -175,7 +246,7 @@ final class HotwordDetectionConnection {
Runnable cancellingJob = () -> {
record.stop();
bestEffortCloseFileDescriptor(audioSink);
bestEffortClose(audioSink);
// TODO: consider calling externalCallback.onRejected(ERROR_TIMEOUT).
};
@@ -186,11 +257,11 @@ final class HotwordDetectionConnection {
// TODO: consider making this a non-anonymous class.
IDspHotwordDetectionCallback internalCallback = new IDspHotwordDetectionCallback.Stub() {
@Override
public void onDetected() throws RemoteException {
public void onDetected(HotwordDetectedResult result) throws RemoteException {
if (DEBUG) {
Slog.d(TAG, "onDetected");
}
bestEffortCloseFileDescriptor(audioSink);
bestEffortClose(audioSink);
cancelingFuture.cancel(true);
// Give 2 more seconds for the interactor to start consuming the mic. If it fails to
@@ -202,6 +273,7 @@ final class HotwordDetectionConnection {
VOICE_INTERACTION_TIMEOUT_TO_OPEN_MIC_MILLIS,
TimeUnit.MILLISECONDS);
// TODO: Propagate the HotwordDetectedResult.
externalCallback.onKeyphraseDetected(recognitionEvent);
}
@@ -221,7 +293,7 @@ final class HotwordDetectionConnection {
recognitionEvent.getCaptureFormat(),
VALIDATION_TIMEOUT_MILLIS,
internalCallback));
bestEffortCloseFileDescriptor(clientRead);
bestEffortClose(clientRead);
}
static final class SoundTriggerCallback extends IRecognitionStatusCallback.Stub {
@@ -283,16 +355,48 @@ final class HotwordDetectionConnection {
new AudioAttributes.Builder()
.setInternalCapturePreset(MediaRecorder.AudioSource.HOTWORD).build(),
recognitionEvent.getCaptureFormat(),
getBufferSizeInBytes(sampleRate, MAX_STREAMING_SECONDS),
getBufferSizeInBytes(
sampleRate,
MAX_STREAMING_SECONDS,
recognitionEvent.getCaptureFormat().getChannelCount()),
recognitionEvent.getCaptureSession());
}
@Nullable
private AudioRecord createMicAudioRecord(AudioFormat audioFormat) {
if (DEBUG) {
Slog.i(TAG, "#createAudioRecord");
}
try {
AudioRecord audioRecord = new AudioRecord(
new AudioAttributes.Builder().setHotwordMode().build(),
audioFormat,
getBufferSizeInBytes(
audioFormat.getSampleRate(),
MICROPHONE_BUFFER_LENGTH_SECONDS,
audioFormat.getChannelCount()),
AudioManager.AUDIO_SESSION_ID_GENERATE);
if (audioRecord.getState() != AudioRecord.STATE_INITIALIZED) {
Slog.w(TAG, "Failed to initialize AudioRecord");
audioRecord.release();
return null;
}
return audioRecord;
} catch (IllegalArgumentException e) {
Slog.e(TAG, "Failed to create AudioRecord", e);
return null;
}
}
/**
* Returns the number of bytes required to store {@code bufferLengthSeconds} of audio sampled at
* {@code sampleRate} Hz, using the format returned by DSP audio capture.
*/
private static int getBufferSizeInBytes(int sampleRate, int bufferLengthSeconds) {
return BYTES_PER_SAMPLE * sampleRate * bufferLengthSeconds;
private static int getBufferSizeInBytes(
int sampleRate, int bufferLengthSeconds, int intChannelCount) {
return BYTES_PER_SAMPLE * sampleRate * bufferLengthSeconds * intChannelCount;
}
private static Pair<ParcelFileDescriptor, ParcelFileDescriptor> createPipe() {
@@ -307,17 +411,176 @@ final class HotwordDetectionConnection {
return Pair.create(fileDescriptors[0], fileDescriptors[1]);
}
private static void bestEffortCloseFileDescriptor(ParcelFileDescriptor fd) {
try {
fd.close();
} catch (IOException e) {
if (DEBUG) {
Slog.w(TAG, "Failed closing file descriptor", e);
}
}
}
public void dump(String prefix, PrintWriter pw) {
pw.print(prefix); pw.print("mBound="); pw.println(mBound);
}
private interface AudioReader extends Closeable {
int read(byte[] dest, int offset, int length) throws IOException;
static AudioReader createFromInputStream(InputStream is) {
return new AudioReader() {
@Override
public int read(byte[] dest, int offset, int length) throws IOException {
return is.read(dest, offset, length);
}
@Override
public void close() throws IOException {
is.close();
}
};
}
static AudioReader createFromAudioRecord(AudioRecord record) {
record.startRecording();
return new AudioReader() {
@Override
public int read(byte[] dest, int offset, int length) throws IOException {
return record.read(dest, offset, length);
}
@Override
public void close() throws IOException {
record.stop();
record.release();
}
};
}
}
private void handleSoftwareHotwordDetection(
AudioFormat audioFormat,
AudioReader audioSource,
int audioSourceValue,
@Nullable PersistableBundle options,
IMicrophoneHotwordDetectionVoiceInteractionCallback callback) {
Pair<ParcelFileDescriptor, ParcelFileDescriptor> clientPipe = createPipe();
if (clientPipe == null) {
// TODO: Need to propagate as unknown error or something?
return;
}
ParcelFileDescriptor serviceAudioSink = clientPipe.second;
ParcelFileDescriptor serviceAudioSource = clientPipe.first;
synchronized (mLock) {
mCurrentAudioSink = serviceAudioSink;
}
mAudioCopyExecutor.execute(() -> {
try (AudioReader source = audioSource;
OutputStream fos =
new ParcelFileDescriptor.AutoCloseOutputStream(serviceAudioSink)) {
byte[] buffer = new byte[1024];
while (true) {
int bytesRead = source.read(buffer, 0, 1024);
if (bytesRead < 0) {
break;
}
// TODO: First write to ring buffer to make sure we don't lose data if the next
// statement fails.
// ringBuffer.append(buffer, bytesRead);
fos.write(buffer, 0, bytesRead);
}
} catch (IOException e) {
Slog.w(TAG, "Failed supplying audio data to validator", e);
} finally {
synchronized (mLock) {
mCurrentAudioSink = null;
}
}
});
// TODO: handle cancellations well
// TODO: what if we cancelled and started a new one?
mRemoteHotwordDetectionService.run(
service -> service.detectFromMicrophoneSource(
serviceAudioSource,
// TODO: consider making a proxy callback + copy of audio format
audioSourceValue, audioFormat, options,
new IDspHotwordDetectionCallback.Stub() {
@Override
public void onRejected(@Nullable HotwordRejectedResult result)
throws RemoteException {
// TODO: Propagate the HotwordRejectedResult.
}
@Override
public void onDetected(@Nullable HotwordDetectedResult triggerResult)
throws RemoteException {
// Stop
bestEffortClose(serviceAudioSink);
if (audioSourceValue == AUDIO_SOURCE_EXTERNAL) {
callback.onDetected(triggerResult, null, null);
// TODO: Add a delay before closing.
bestEffortClose(audioSource);
}
Pair<ParcelFileDescriptor, ParcelFileDescriptor> clientPipe =
createPipe();
if (clientPipe == null) {
// Error.
// Need to propagate as unknown error or something?
return;
}
ParcelFileDescriptor callbackAudioSink = clientPipe.second;
ParcelFileDescriptor callbackClientRead = clientPipe.first;
mAudioCopyExecutor.execute(() -> {
try (AudioReader source = audioSource;
OutputStream fos =
new ParcelFileDescriptor.AutoCloseOutputStream(
callbackAudioSink)) {
// TODO: get ring buffer suffix here
// fos.write(lastSeveralSeconds);
byte[] buffer = new byte[1024];
while (true) {
int bytesRead = source.read(buffer, 0, 1024);
if (bytesRead < 0) {
break;
}
fos.write(buffer, 0, bytesRead);
}
} catch (IOException e) {
Slog.w(TAG, "Failed supplying audio data to validator", e);
} finally {
synchronized (mLock) {
mCurrentAudioSink = null;
}
}
});
// TODO: noteOp here.
// Remove hotword offset from trigger result
// TODO: consider propagating only capture session.
callback.onDetected(triggerResult, null, callbackClientRead);
// TODO: Add a delay before closing.
bestEffortClose(callbackClientRead);
}
}));
// TODO: verify this is the right thing to do here.
bestEffortClose(serviceAudioSource);
}
private static void bestEffortClose(Closeable closeable) {
try {
closeable.close();
} catch (IOException e) {
if (DEBUG) {
Slog.w(TAG, "Failed closing", e);
}
}
}
};

71
services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerService.java
View File

@@ -48,6 +48,7 @@ import android.hardware.soundtrigger.SoundTrigger.KeyphraseSoundModel;
import android.hardware.soundtrigger.SoundTrigger.ModelParamRange;
import android.hardware.soundtrigger.SoundTrigger.ModuleProperties;
import android.hardware.soundtrigger.SoundTrigger.RecognitionConfig;
import android.media.AudioFormat;
import android.media.permission.Identity;
import android.media.permission.PermissionUtil;
import android.media.permission.SafeCloseable;
@@ -56,6 +57,7 @@ import android.os.Bundle;
import android.os.Handler;
import android.os.IBinder;
import android.os.Parcel;
import android.os.ParcelFileDescriptor;
import android.os.PersistableBundle;
import android.os.RemoteCallback;
import android.os.RemoteCallbackList;
@@ -66,6 +68,7 @@ import android.os.ShellCallback;
import android.os.Trace;
import android.os.UserHandle;
import android.provider.Settings;
import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback;
import android.service.voice.IVoiceInteractionSession;
import android.service.voice.VoiceInteractionManagerInternal;
import android.service.voice.VoiceInteractionService;
@@ -983,6 +986,8 @@ public class VoiceInteractionManagerService extends SystemService {
}
}
//----------------- Hotword Detection/Validation APIs --------------------------------//
@Override
public void updateState(@Nullable PersistableBundle options,
@Nullable SharedMemory sharedMemory) {
@@ -1023,6 +1028,72 @@ public class VoiceInteractionManagerService extends SystemService {
}
}
@Override
public void startListeningFromMic(
AudioFormat audioFormat,
IMicrophoneHotwordDetectionVoiceInteractionCallback callback)
throws RemoteException {
enforceCallingPermission(Manifest.permission.RECORD_AUDIO);
enforceCallingPermission(Manifest.permission.CAPTURE_AUDIO_HOTWORD);
synchronized (this) {
enforceIsCurrentVoiceInteractionService();
if (mImpl == null) {
Slog.w(TAG, "startListeningFromMic without running voice interaction service");
return;
}
final long caller = Binder.clearCallingIdentity();
try {
mImpl.startListeningFromMicLocked(audioFormat, callback);
} finally {
Binder.restoreCallingIdentity(caller);
}
}
}
@Override
public void startListeningFromExternalSource(
ParcelFileDescriptor audioStream,
AudioFormat audioFormat,
PersistableBundle options,
IMicrophoneHotwordDetectionVoiceInteractionCallback callback)
throws RemoteException {
synchronized (this) {
enforceIsCurrentVoiceInteractionService();
if (mImpl == null) {
Slog.w(TAG, "startListeningFromExternalSource without running voice"
+ " interaction service");
return;
}
final long caller = Binder.clearCallingIdentity();
try {
mImpl.startListeningFromExternalSourceLocked(
audioStream, audioFormat, options, callback);
} finally {
Binder.restoreCallingIdentity(caller);
}
}
}
@Override
public void stopListeningFromMic() throws RemoteException {
synchronized (this) {
enforceIsCurrentVoiceInteractionService();
if (mImpl == null) {
Slog.w(TAG, "stopListeningFromMic without running voice interaction service");
return;
}
final long caller = Binder.clearCallingIdentity();
try {
mImpl.stopListeningFromMicLocked();
} finally {
Binder.restoreCallingIdentity(caller);
}
}
}
//----------------- Model management APIs --------------------------------//
@Override

49
services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerServiceImpl.java
View File

@@ -40,15 +40,18 @@ import android.content.pm.IPackageManager;
import android.content.pm.PackageManager;
import android.content.pm.ServiceInfo;
import android.hardware.soundtrigger.IRecognitionStatusCallback;
import android.media.AudioFormat;
import android.os.Bundle;
import android.os.Handler;
import android.os.IBinder;
import android.os.ParcelFileDescriptor;
import android.os.PersistableBundle;
import android.os.RemoteCallback;
import android.os.RemoteException;
import android.os.ServiceManager;
import android.os.SharedMemory;
import android.os.UserHandle;
import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback;
import android.service.voice.IVoiceInteractionService;
import android.service.voice.IVoiceInteractionSession;
import android.service.voice.VoiceInteractionService;
@@ -444,6 +447,52 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
mHotwordDetectionConnection = null;
}
public void startListeningFromMicLocked(
AudioFormat audioFormat,
IMicrophoneHotwordDetectionVoiceInteractionCallback callback) {
if (DEBUG) {
Slog.d(TAG, "startListeningFromMic");
}
if (mHotwordDetectionConnection == null) {
// TODO: callback.onError();
return;
}
mHotwordDetectionConnection.startListeningFromMic(audioFormat, callback);
}
public void startListeningFromExternalSourceLocked(
ParcelFileDescriptor audioStream,
AudioFormat audioFormat,
@Nullable PersistableBundle options,
IMicrophoneHotwordDetectionVoiceInteractionCallback callback) {
if (DEBUG) {
Slog.d(TAG, "startListeningFromExternalSource");
}
if (mHotwordDetectionConnection == null) {
// TODO: callback.onError();
return;
}
mHotwordDetectionConnection
.startListeningFromExternalSource(audioStream, audioFormat, options, callback);
}
public void stopListeningFromMicLocked() {
if (DEBUG) {
Slog.d(TAG, "stopListeningFromMic");
}
if (mHotwordDetectionConnection == null) {
Slog.w(TAG, "stopListeningFromMic() called but connection isn't established");
return;
}
mHotwordDetectionConnection.stopListening();
}
public IRecognitionStatusCallback createSoundTriggerCallbackLocked(
IHotwordRecognitionStatusCallback callback) {
if (DEBUG) {