Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

AlwaysOnHotwordDetector needs to reflect enrollment changes

Browse Source 
Add a callback for when any sound model change happens. This helps the VIS
to re-check the availability and either enroll the user, or start/stop recognition.

Also shut down any active recognition when VIS dies, or a different hotword detector instance is obtained from VIS.

Change-Id: I03f94e78c6ee307afe822a84aebc7e74c64de7b4
This commit is contained in:
Sandeep Siddhartha
2014-07-21 10:31:34 -07:00
parent 70441467f4
commit 6daae96226
7 changed files with 250 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
api/current.txt
View File

@@ -27324,10 +27324,6 @@ package android.service.voice {
method public int getSupportedRecognitionModes();
method public int startRecognition(int);
method public int stopRecognition();
field public static final int KEYPHRASE_ENROLLED = 2; // 0x2
field public static final int KEYPHRASE_HARDWARE_UNAVAILABLE = -2; // 0xfffffffe
field public static final int KEYPHRASE_UNENROLLED = 1; // 0x1
field public static final int KEYPHRASE_UNSUPPORTED = -1; // 0xffffffff
field public static final int MANAGE_ACTION_ENROLL = 0; // 0x0
field public static final int MANAGE_ACTION_RE_ENROLL = 1; // 0x1
field public static final int MANAGE_ACTION_UN_ENROLL = 2; // 0x2
@@ -27335,6 +27331,11 @@ package android.service.voice {
field public static final int RECOGNITION_FLAG_NONE = 0; // 0x0
field public static final int RECOGNITION_MODE_USER_IDENTIFICATION = 2; // 0x2
field public static final int RECOGNITION_MODE_VOICE_TRIGGER = 1; // 0x1
field public static final int STATE_HARDWARE_UNAVAILABLE = -2; // 0xfffffffe
field public static final int STATE_INVALID = -3; // 0xfffffffd
field public static final int STATE_KEYPHRASE_ENROLLED = 2; // 0x2
field public static final int STATE_KEYPHRASE_UNENROLLED = 1; // 0x1
field public static final int STATE_KEYPHRASE_UNSUPPORTED = -1; // 0xffffffff
field public static final int STATUS_ERROR = -2147483648; // 0x80000000
field public static final int STATUS_OK = 0; // 0x0
}
@@ -27346,10 +27347,12 @@ package android.service.voice {
public class VoiceInteractionService extends android.app.Service {
ctor public VoiceInteractionService();
method public final android.service.voice.AlwaysOnHotwordDetector getAlwaysOnHotwordDetector(java.lang.String, java.lang.String, android.service.voice.AlwaysOnHotwordDetector.Callback);
method public final android.service.voice.AlwaysOnHotwordDetector createAlwaysOnHotwordDetector(java.lang.String, java.lang.String, android.service.voice.AlwaysOnHotwordDetector.Callback);
method public static boolean isActiveService(android.content.Context, android.content.ComponentName);
method public android.os.IBinder onBind(android.content.Intent);
method public void onReady();
method public void onShutdown();
method public void onSoundModelsChanged();
method public void startSession(android.os.Bundle);
field public static final java.lang.String SERVICE_INTERFACE = "android.service.voice.VoiceInteractionService";
field public static final java.lang.String SERVICE_META_DATA = "android.voice_interaction";

122
core/java/android/service/voice/AlwaysOnHotwordDetector.java
View File

@@ -41,26 +41,32 @@ import java.util.List;
* always-on keyphrase detection APIs.
*/
public class AlwaysOnHotwordDetector {
//---- States of Keyphrase availability ----//
//---- States of Keyphrase availability. Return codes for getAvailability() ----//
/**
* Indicates that the given keyphrase is not available on the system because of the
* hardware configuration.
* Indicates that this hotword detector is no longer valid for any recognition
* and should not be used anymore.
*/
public static final int KEYPHRASE_HARDWARE_UNAVAILABLE = -2;
public static final int STATE_INVALID = -3;
/**
* Indicates that the given keyphrase is not supported.
* Indicates that recognition for the given keyphrase is not available on the system
* because of the hardware configuration.
*/
public static final int KEYPHRASE_UNSUPPORTED = -1;
public static final int STATE_HARDWARE_UNAVAILABLE = -2;
/**
* Indicates that recognition for the given keyphrase is not supported.
*/
public static final int STATE_KEYPHRASE_UNSUPPORTED = -1;
/**
* Indicates that the given keyphrase is not enrolled.
*/
public static final int KEYPHRASE_UNENROLLED = 1;
public static final int STATE_KEYPHRASE_UNENROLLED = 1;
/**
* Indicates that the given keyphrase is currently enrolled but not being actively listened for.
* Indicates that the given keyphrase is currently enrolled and it's possible to start
* recognition for it.
*/
public static final int KEYPHRASE_ENROLLED = 2;
public static final int STATE_KEYPHRASE_ENROLLED = 2;
// Keyphrase management actions ----//
// Keyphrase management actions. Used in getManageIntent() ----//
/** Indicates that we need to enroll. */
public static final int MANAGE_ACTION_ENROLL = 0;
/** Indicates that we need to re-enroll. */
@@ -83,7 +89,7 @@ public class AlwaysOnHotwordDetector {
*/
public static final int RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO = 0x1;
//---- Recognition mode flags ----//
//---- Recognition mode flags. Return codes for getSupportedRecognitionModes() ----//
// Must be kept in sync with the related attribute defined as searchKeyphraseRecognitionFlags.
/**
@@ -109,17 +115,20 @@ public class AlwaysOnHotwordDetector {
* This may be null if this keyphrase isn't supported by the enrollment application.
*/
private final KeyphraseMetadata mKeyphraseMetadata;
/**
* The sound model for the keyphrase, derived from the model management service
* (IVoiceInteractionManagerService). May be null if the keyphrase isn't enrolled yet.
*/
private final KeyphraseSoundModel mEnrolledSoundModel;
private final KeyphraseEnrollmentInfo mKeyphraseEnrollmentInfo;
private final IVoiceInteractionService mVoiceInteractionService;
private final IVoiceInteractionManagerService mModelManagementService;
private final SoundTriggerListener mInternalCallback;
private final Callback mExternalCallback;
private final boolean mDisabled;
private final Object mLock = new Object();
/**
* The sound model for the keyphrase, derived from the model management service
* (IVoiceInteractionManagerService). May be null if the keyphrase isn't enrolled yet.
*/
private KeyphraseSoundModel mEnrolledSoundModel;
private boolean mInvalidated;
/**
* Callbacks for always-on hotword detection.
@@ -151,6 +160,7 @@ public class AlwaysOnHotwordDetector {
KeyphraseEnrollmentInfo keyphraseEnrollmentInfo,
IVoiceInteractionService voiceInteractionService,
IVoiceInteractionManagerService modelManagementService) {
mInvalidated = false;
mText = text;
mLocale = locale;
mKeyphraseEnrollmentInfo = keyphraseEnrollmentInfo;
@@ -160,28 +170,34 @@ public class AlwaysOnHotwordDetector {
mVoiceInteractionService = voiceInteractionService;
mModelManagementService = modelManagementService;
if (mKeyphraseMetadata != null) {
mEnrolledSoundModel = internalGetKeyphraseSoundModel(mKeyphraseMetadata.id);
} else {
mEnrolledSoundModel = null;
mEnrolledSoundModel = internalGetKeyphraseSoundModelLocked(mKeyphraseMetadata.id);
}
int initialAvailability = internalGetAvailabilityLocked();
mDisabled = (initialAvailability == KEYPHRASE_HARDWARE_UNAVAILABLE)
|| (initialAvailability == KEYPHRASE_UNSUPPORTED);
mDisabled = (initialAvailability == STATE_HARDWARE_UNAVAILABLE)
|| (initialAvailability == STATE_KEYPHRASE_UNSUPPORTED);
}
/**
* Gets the state of always-on hotword detection for the given keyphrase and locale
* on this system.
* Availability implies that the hardware on this system is capable of listening for
* the given keyphrase or not.
* the given keyphrase or not. <p/>
* If the return code is one of {@link #STATE_HARDWARE_UNAVAILABLE} or
* {@link #STATE_KEYPHRASE_UNSUPPORTED}, no further interaction should be performed with this
* detector. <br/>
* If the state is {@link #STATE_KEYPHRASE_UNENROLLED} the caller may choose to begin
* an enrollment flow for the keyphrase. <br/>
* For {@value #STATE_KEYPHRASE_ENROLLED} a recognition can be started as desired. <br/>
* If the return code is {@link #STATE_INVALID}, this detector is stale and must not be used.
* A new detector should be obtained and used.
*
* @return Indicates if always-on hotword detection is available for the given keyphrase.
* The return code is one of {@link #KEYPHRASE_HARDWARE_UNAVAILABLE},
* {@link #KEYPHRASE_UNSUPPORTED}, {@link #KEYPHRASE_UNENROLLED} or
* {@link #KEYPHRASE_ENROLLED}.
* The return code is one of {@link #STATE_HARDWARE_UNAVAILABLE},
* {@link #STATE_KEYPHRASE_UNSUPPORTED}, {@link #STATE_KEYPHRASE_UNENROLLED},
* {@link #STATE_KEYPHRASE_ENROLLED}, or {@link #STATE_INVALID}.
*/
public int getAvailability() {
synchronized (this) {
synchronized (mLock) {
return internalGetAvailabilityLocked();
}
}
@@ -194,7 +210,7 @@ public class AlwaysOnHotwordDetector {
* before calling this method to avoid this exception.
*/
public int getSupportedRecognitionModes() {
synchronized (this) {
synchronized (mLock) {
return getSupportedRecognitionModesLocked();
}
}
@@ -220,13 +236,13 @@ public class AlwaysOnHotwordDetector {
* before calling this method to avoid this exception.
*/
public int startRecognition(int recognitionFlags) {
synchronized (this) {
synchronized (mLock) {
return startRecognitionLocked(recognitionFlags);
}
}
private int startRecognitionLocked(int recognitionFlags) {
if (internalGetAvailabilityLocked() != KEYPHRASE_ENROLLED) {
if (internalGetAvailabilityLocked() != STATE_KEYPHRASE_ENROLLED) {
throw new UnsupportedOperationException(
"Recognition for the given keyphrase is not supported");
}
@@ -261,13 +277,13 @@ public class AlwaysOnHotwordDetector {
* before calling this method to avoid this exception.
*/
public int stopRecognition() {
synchronized (this) {
synchronized (mLock) {
return stopRecognitionLocked();
}
}
private synchronized int stopRecognitionLocked() {
if (internalGetAvailabilityLocked() != KEYPHRASE_ENROLLED) {
private int stopRecognitionLocked() {
if (internalGetAvailabilityLocked() != STATE_KEYPHRASE_ENROLLED) {
throw new UnsupportedOperationException(
"Recognition for the given keyphrase is not supported");
}
@@ -312,6 +328,10 @@ public class AlwaysOnHotwordDetector {
}
private int internalGetAvailabilityLocked() {
if (mInvalidated) {
return STATE_INVALID;
}
ModuleProperties dspModuleProperties = null;
try {
dspModuleProperties =
@@ -321,23 +341,51 @@ public class AlwaysOnHotwordDetector {
}
// No DSP available
if (dspModuleProperties == null) {
return KEYPHRASE_HARDWARE_UNAVAILABLE;
return STATE_HARDWARE_UNAVAILABLE;
}
// No enrollment application supports this keyphrase/locale
if (mKeyphraseMetadata == null) {
return KEYPHRASE_UNSUPPORTED;
return STATE_KEYPHRASE_UNSUPPORTED;
}
// This keyphrase hasn't been enrolled.
if (mEnrolledSoundModel == null) {
return KEYPHRASE_UNENROLLED;
return STATE_KEYPHRASE_UNENROLLED;
}
return STATE_KEYPHRASE_ENROLLED;
}
/**
* Invalidates this hotword detector so that any future calls to this result
* in an IllegalStateException.
*
* @hide
*/
void invalidate() {
synchronized (mLock) {
mInvalidated = true;
}
}
/**
* Reloads the sound models from the service.
*
* @hide
*/
void onSoundModelsChanged() {
synchronized (mLock) {
// TODO: This should stop the recognition if it was using an enrolled sound model
// that's no longer available.
if (mKeyphraseMetadata != null) {
mEnrolledSoundModel = internalGetKeyphraseSoundModelLocked(mKeyphraseMetadata.id);
}
}
return KEYPHRASE_ENROLLED;
}
/**
* @return The corresponding {@link KeyphraseSoundModel} or null if none is found.
*/
private KeyphraseSoundModel internalGetKeyphraseSoundModel(int keyphraseId) {
private KeyphraseSoundModel internalGetKeyphraseSoundModelLocked(int keyphraseId) {
List<KeyphraseSoundModel> soundModels;
try {
soundModels = mModelManagementService

2
core/java/android/service/voice/IVoiceInteractionService.aidl
View File

@@ -21,4 +21,6 @@ package android.service.voice;
*/
oneway interface IVoiceInteractionService {
void ready();
void soundModelsChanged();
void shutdown();
}

96
core/java/android/service/voice/VoiceInteractionService.java
View File

@@ -28,8 +28,8 @@ import android.os.IBinder;
import android.os.Message;
import android.os.RemoteException;
import android.os.ServiceManager;
import android.provider.Settings;
import com.android.internal.annotations.VisibleForTesting;
import com.android.internal.app.IVoiceInteractionManagerService;
@@ -70,15 +70,27 @@ public class VoiceInteractionService extends Service {
@Override public void ready() {
mHandler.sendEmptyMessage(MSG_READY);
}
@Override public void shutdown() {
mHandler.sendEmptyMessage(MSG_SHUTDOWN);
}
@Override public void soundModelsChanged() {
mHandler.sendEmptyMessage(MSG_SOUND_MODELS_CHANGED);
}
};
MyHandler mHandler;
IVoiceInteractionManagerService mSystemService;
private final Object mLock = new Object();
private KeyphraseEnrollmentInfo mKeyphraseEnrollmentInfo;
private AlwaysOnHotwordDetector mHotwordDetector;
static final int MSG_READY = 1;
static final int MSG_SHUTDOWN = 2;
static final int MSG_SOUND_MODELS_CHANGED = 3;
class MyHandler extends Handler {
@Override
@@ -87,6 +99,12 @@ public class VoiceInteractionService extends Service {
case MSG_READY:
onReady();
break;
case MSG_SHUTDOWN:
onShutdownInternal();
break;
case MSG_SOUND_MODELS_CHANGED:
onSoundModelsChangedInternal();
break;
default:
super.handleMessage(msg);
}
@@ -140,9 +158,10 @@ public class VoiceInteractionService extends Service {
/**
* Called during service initialization to tell you when the system is ready
* to receive interaction from it. You should generally do initialization here
* rather than in {@link #onCreate()}. Methods such as {@link #startSession}
* and {@link #getAlwaysOnHotwordDetector} will not be operational until this point.
* to receive interaction from it. You should generally do initialization here
* rather than in {@link #onCreate()}. Methods such as {@link #startSession(Bundle)} and
* {@link #createAlwaysOnHotwordDetector(String, String, android.service.voice.AlwaysOnHotwordDetector.Callback)}
* will not be operational until this point.
*/
public void onReady() {
mSystemService = IVoiceInteractionManagerService.Stub.asInterface(
@@ -150,22 +169,67 @@ public class VoiceInteractionService extends Service {
mKeyphraseEnrollmentInfo = new KeyphraseEnrollmentInfo(getPackageManager());
}
private void onShutdownInternal() {
onShutdown();
// Stop any active recognitions when shutting down.
// This ensures that if implementations forget to stop any active recognition,
// It's still guaranteed to have been stopped.
// This helps with cases where the voice interaction implementation is changed
// by the user.
safelyShutdownHotwordDetector();
}
/**
* Called during service de-initialization to tell you when the system is shutting the
* service down.
*/
public void onShutdown() {
}
private void onSoundModelsChangedInternal() {
synchronized (this) {
if (mHotwordDetector != null) {
// TODO: Stop recognition if a sound model that was being recognized gets deleted.
mHotwordDetector.onSoundModelsChanged();
}
}
onSoundModelsChanged();
}
/**
* Called when the sound models available for recognition change.
* This may be called if a new sound model is available or
* an existing one is updated or removed.
* Implementations must check the availability of the hotword detector if they own one
* by calling {@link AlwaysOnHotwordDetector#getAvailability()} before calling into it.
*/
public void onSoundModelsChanged() {
}
/**
* Creates an {@link AlwaysOnHotwordDetector} for the given keyphrase and locale.
* This instance must be retained and used by the client.
* Calling this a second time invalidates the previously created hotword detector
* which can no longer be used to manage recognition.
*
* @param keyphrase The keyphrase that's being used, for example "Hello Android".
* @param locale The locale for which the enrollment needs to be performed.
* This is a Java locale, for example "en_US".
* @param callback The callback to notify of detection events.
* @return An always-on hotword detector for the given keyphrase and locale.
*/
public final AlwaysOnHotwordDetector getAlwaysOnHotwordDetector(
public final AlwaysOnHotwordDetector createAlwaysOnHotwordDetector(
String keyphrase, String locale, AlwaysOnHotwordDetector.Callback callback) {
if (mSystemService == null) {
throw new IllegalStateException("Not available until onReady() is called");
}
// TODO: Cache instances and return the same one instead of creating a new interactor
// for the same keyphrase/locale combination.
return new AlwaysOnHotwordDetector(keyphrase, locale, callback,
mKeyphraseEnrollmentInfo, mInterface, mSystemService);
synchronized (mLock) {
// Allow only one concurrent recognition via the APIs.
safelyShutdownHotwordDetector();
mHotwordDetector = new AlwaysOnHotwordDetector(keyphrase, locale, callback,
mKeyphraseEnrollmentInfo, mInterface, mSystemService);
}
return mHotwordDetector;
}
/**
@@ -176,4 +240,18 @@ public class VoiceInteractionService extends Service {
protected final KeyphraseEnrollmentInfo getKeyphraseEnrollmentInfo() {
return mKeyphraseEnrollmentInfo;
}
private void safelyShutdownHotwordDetector() {
try {
synchronized (mLock) {
if (mHotwordDetector != null) {
mHotwordDetector.stopRecognition();
mHotwordDetector.invalidate();
mHotwordDetector = null;
}
}
} catch (Exception ex) {
// Ignore.
}
}
}

23
services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerService.java
View File

@@ -290,19 +290,22 @@ public class VoiceInteractionManagerService extends SystemService {
final long caller = Binder.clearCallingIdentity();
try {
// If the keyphrases are not present in the model, delete the model.
boolean success = false;
if (model.keyphrases == null) {
if (mDbHelper.deleteKeyphraseSoundModel(model.uuid)) {
return SoundTriggerHelper.STATUS_OK;
} else {
return SoundTriggerHelper.STATUS_ERROR;
}
// If the keyphrases are not present in the model, delete the model.
success = mDbHelper.deleteKeyphraseSoundModel(model.uuid);
} else {
if (mDbHelper.addOrUpdateKeyphraseSoundModel(model)) {
return SoundTriggerHelper.STATUS_OK;
} else {
return SoundTriggerHelper.STATUS_ERROR;
// Else update the model.
success = mDbHelper.addOrUpdateKeyphraseSoundModel(model);
}
if (success) {
// Notify the voice interaction service of a change in sound models.
if (mImpl != null && mImpl.mService != null) {
mImpl.notifySoundModelsChangedLocked();
}
return SoundTriggerHelper.STATUS_OK;
} else {
return SoundTriggerHelper.STATUS_ERROR;
}
} finally {
Binder.restoreCallingIdentity(caller);

18
services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerServiceImpl.java
View File

@@ -96,6 +96,13 @@ class VoiceInteractionManagerServiceImpl {
@Override
public void onServiceDisconnected(ComponentName name) {
try {
if (mService != null) {
mService.shutdown();
}
} catch (RemoteException e) {
Slog.w(TAG, "RemoteException in shutdown", e);
}
mService = null;
}
};
@@ -308,4 +315,15 @@ class VoiceInteractionManagerServiceImpl {
mContext.unregisterReceiver(mBroadcastReceiver);
}
}
void notifySoundModelsChangedLocked() {
if (mService == null) {
Slog.w(TAG, "Not bound to voice interaction service " + mComponent);
}
try {
mService.soundModelsChanged();
} catch (RemoteException e) {
Slog.w(TAG, "RemoteException while calling soundModelsChanged", e);
}
}
}

58
tests/VoiceInteraction/src/com/android/test/voiceinteraction/MainInteractionService.java
View File

@@ -50,30 +50,18 @@ public class MainInteractionService extends VoiceInteractionService {
Log.i(TAG, "Keyphrase enrollment meta-data: "
+ Arrays.toString(getKeyphraseEnrollmentInfo().listKeyphraseMetadata()));
mHotwordDetector = getAlwaysOnHotwordDetector("Hello There", "en-US", mHotwordCallback);
mHotwordDetector = createAlwaysOnHotwordDetector("Hello There", "en-US", mHotwordCallback);
testHotwordAvailabilityStates();
}
@Override
public void onSoundModelsChanged() {
int availability = mHotwordDetector.getAvailability();
Log.i(TAG, "Hotword availability = " + availability);
switch (availability) {
case AlwaysOnHotwordDetector.KEYPHRASE_HARDWARE_UNAVAILABLE:
Log.i(TAG, "KEYPHRASE_HARDWARE_UNAVAILABLE");
break;
case AlwaysOnHotwordDetector.KEYPHRASE_UNSUPPORTED:
Log.i(TAG, "KEYPHRASE_UNSUPPORTED");
break;
case AlwaysOnHotwordDetector.KEYPHRASE_UNENROLLED:
Log.i(TAG, "KEYPHRASE_UNENROLLED");
Intent enroll = mHotwordDetector.getManageIntent(
AlwaysOnHotwordDetector.MANAGE_ACTION_ENROLL);
Log.i(TAG, "Need to enroll with " + enroll);
break;
case AlwaysOnHotwordDetector.KEYPHRASE_ENROLLED:
Log.i(TAG, "KEYPHRASE_ENROLLED");
int status = mHotwordDetector.startRecognition(
AlwaysOnHotwordDetector.RECOGNITION_FLAG_NONE);
Log.i(TAG, "startRecognition status = " + status);
break;
if (availability == AlwaysOnHotwordDetector.STATE_INVALID) {
mHotwordDetector = createAlwaysOnHotwordDetector("Hello There", "en-US", mHotwordCallback);
}
testHotwordAvailabilityStates();
}
@Override
@@ -84,4 +72,32 @@ public class MainInteractionService extends VoiceInteractionService {
stopSelf(startId);
return START_NOT_STICKY;
}
private void testHotwordAvailabilityStates() {
int availability = mHotwordDetector.getAvailability();
Log.i(TAG, "Hotword availability = " + availability);
switch (availability) {
case AlwaysOnHotwordDetector.STATE_INVALID:
Log.i(TAG, "STATE_INVALID");
break;
case AlwaysOnHotwordDetector.STATE_HARDWARE_UNAVAILABLE:
Log.i(TAG, "STATE_HARDWARE_UNAVAILABLE");
break;
case AlwaysOnHotwordDetector.STATE_KEYPHRASE_UNSUPPORTED:
Log.i(TAG, "STATE_KEYPHRASE_UNSUPPORTED");
break;
case AlwaysOnHotwordDetector.STATE_KEYPHRASE_UNENROLLED:
Log.i(TAG, "STATE_KEYPHRASE_UNENROLLED");
Intent enroll = mHotwordDetector.getManageIntent(
AlwaysOnHotwordDetector.MANAGE_ACTION_ENROLL);
Log.i(TAG, "Need to enroll with " + enroll);
break;
case AlwaysOnHotwordDetector.STATE_KEYPHRASE_ENROLLED:
Log.i(TAG, "STATE_KEYPHRASE_ENROLLED");
int status = mHotwordDetector.startRecognition(
AlwaysOnHotwordDetector.RECOGNITION_FLAG_NONE);
Log.i(TAG, "startRecognition status = " + status);
break;
}
}
}