Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

am 15f3c08d: am 8a121f48: Merge "Hook in startRecogniton call" into lmp-dev

Browse Source 
* commit '15f3c08d18de3e48c460ead3942f77ef63ca5133':
  Hook in startRecogniton call
This commit is contained in:
Sandeep Siddhartha
2014-07-16 17:08:14 +00:00
committed by Android Git Automerger
parent faee1741db 24e119a24b
commit e2a9caffe6
7 changed files with 217 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
core/java/android/hardware/soundtrigger/Keyphrase.java
View File

@@ -30,7 +30,11 @@ public class Keyphrase implements Parcelable {
/** A hint text to display corresponding to this keyphrase, e.g. "Hello There". */
public final String hintText;
/** The locale of interest when using this Keyphrase. */
public String locale;
public final String locale;
/** The various recognition modes supported by this keyphrase */
public final int recognitionModeFlags;
/** The users associated with this keyphrase */
public final int[] users;
public static final Parcelable.Creator<Keyphrase> CREATOR
= new Parcelable.Creator<Keyphrase>() {
@@ -44,13 +48,26 @@ public class Keyphrase implements Parcelable {
};
private static Keyphrase fromParcel(Parcel in) {
return new Keyphrase(in.readInt(), in.readString(), in.readString());
int id = in.readInt();
String hintText = in.readString();
String locale = in.readString();
int recognitionModeFlags = in.readInt();
int numUsers = in.readInt();
int[] users = null;
if (numUsers > 0) {
users = new int[numUsers];
in.readIntArray(users);
}
return new Keyphrase(id, hintText, locale, recognitionModeFlags, users);
}
public Keyphrase(int id, String hintText, String locale) {
public Keyphrase(int id, String hintText, String locale, int recognitionModeFlags,
int[] users) {
this.id = id;
this.hintText = hintText;
this.locale = locale;
this.recognitionModeFlags = recognitionModeFlags;
this.users = users;
}
@Override
@@ -58,6 +75,13 @@ public class Keyphrase implements Parcelable {
dest.writeInt(id);
dest.writeString(hintText);
dest.writeString(locale);
dest.writeInt(recognitionModeFlags);
if (users != null) {
dest.writeInt(users.length);
dest.writeIntArray(users);
} else {
dest.writeInt(0);
}
}
@Override
@@ -98,4 +122,14 @@ public class Keyphrase implements Parcelable {
return false;
return true;
}
@Override
public String toString() {
return "Keyphrase[id=" + id + ", text=" + hintText + ", locale=" + locale
+ ", recognitionModes=" + recognitionModeFlags + "]";
}
protected SoundTrigger.Keyphrase convertToSoundTriggerKeyphrase() {
return new SoundTrigger.Keyphrase(id, recognitionModeFlags, locale, hintText, users);
}
}

3
core/java/android/hardware/soundtrigger/KeyphraseEnrollmentInfo.java
View File

@@ -239,7 +239,8 @@ public class KeyphraseEnrollmentInfo {
* @param keyphrase The keyphrase that the user needs to be enrolled to.
* @param locale The locale for which the enrollment needs to be performed.
* This is a Java locale, for example "en_US".
* @return true, if an enrollment client supports the given keyphrase and the given locale.
* @return The metadata, if an enrollment client supports the given keyphrase
* and the given locale, null otherwise.
*/
public KeyphraseMetadata getKeyphraseMetadata(String keyphrase, String locale) {
if (mKeyphrases == null || mKeyphrases.length == 0) {

11
core/java/android/hardware/soundtrigger/KeyphraseSoundModel.java
View File

@@ -65,4 +65,15 @@ public class KeyphraseSoundModel implements Parcelable {
}
dest.writeParcelableArray(keyphrases, 0);
}
public SoundTrigger.KeyphraseSoundModel convertToSoundTriggerKeyphraseSoundModel() {
SoundTrigger.Keyphrase[] stKeyphrases = null;
if (keyphrases != null) {
stKeyphrases = new SoundTrigger.Keyphrase[keyphrases.length];
for (int i = 0; i < keyphrases.length; i++) {
stKeyphrases[i] = keyphrases[i].convertToSoundTriggerKeyphrase();
}
}
return new SoundTrigger.KeyphraseSoundModel(uuid, data, stKeyphrases);
}
}

122
core/java/android/hardware/soundtrigger/SoundTriggerHelper.java
View File

@@ -17,6 +17,7 @@
package android.hardware.soundtrigger;
import android.hardware.soundtrigger.SoundTrigger.ModuleProperties;
import android.hardware.soundtrigger.SoundTrigger.RecognitionConfig;
import android.hardware.soundtrigger.SoundTrigger.RecognitionEvent;
import android.util.Slog;
import android.util.SparseArray;
@@ -56,7 +57,7 @@ public class SoundTriggerHelper implements SoundTrigger.StatusListener {
private final ModuleProperties mModuleProperties;
private final SoundTriggerModule mModule;
private final SparseArray<Listener> mListeners;
private final SparseArray<Listener> mActiveListeners;
private int mCurrentSoundModelHandle = INVALID_SOUND_MODEL_HANDLE;
@@ -77,7 +78,7 @@ public class SoundTriggerHelper implements SoundTrigger.StatusListener {
public SoundTriggerHelper() {
ArrayList <ModuleProperties> modules = new ArrayList<>();
int status = SoundTrigger.listModules(modules);
mListeners = new SparseArray<>(1);
mActiveListeners = new SparseArray<>(1);
if (status != SoundTrigger.STATUS_OK || modules.size() == 0) {
// TODO: Figure out how to handle errors in listing the modules here.
dspInfo = null;
@@ -93,28 +94,10 @@ public class SoundTriggerHelper implements SoundTrigger.StatusListener {
}
}
/**
* @return True, if the given {@link Keyphrase} is supported on DSP.
*/
public boolean isKeyphraseSupported(Keyphrase keyphrase) {
// TODO: We also need to look into a SoundTrigger API that let's us
// query this. For now just return true.
return true;
}
/**
* @return True, if the given {@link Keyphrase} has been enrolled.
*/
public boolean isKeyphraseEnrolled(Keyphrase keyphrase) {
// TODO: Query VoiceInteractionManagerService
// to list registered sound models.
return false;
}
/**
* @return True, if a recognition for the given {@link Keyphrase} is active.
*/
public boolean isKeyphraseActive(Keyphrase keyphrase) {
public synchronized boolean isKeyphraseActive(Keyphrase keyphrase) {
// TODO: Check if the recognition for the keyphrase is currently active.
return false;
}
@@ -124,55 +107,98 @@ public class SoundTriggerHelper implements SoundTrigger.StatusListener {
*
* @param keyphraseId The identifier of the keyphrase for which
* the recognition is to be started.
* @param soundModel The sound model to use for recognition.
* @param listener The listener for the recognition events related to the given keyphrase.
* @return One of {@link #STATUS_ERROR} or {@link #STATUS_OK}.
*/
public int startRecognition(int keyphraseId, Listener listener) {
public synchronized int startRecognition(int keyphraseId,
SoundTrigger.KeyphraseSoundModel soundModel,
Listener listener, RecognitionConfig recognitionConfig) {
if (dspInfo == null || mModule == null) {
Slog.w(TAG, "Attempting startRecognition without the capability");
return STATUS_ERROR;
}
if (mListeners.get(keyphraseId) != listener) {
Listener oldListener = mActiveListeners.get(keyphraseId);
if (oldListener != null && oldListener != listener) {
if (mCurrentSoundModelHandle != INVALID_SOUND_MODEL_HANDLE) {
Slog.w(TAG, "Canceling previous recognition");
// TODO: Inspect the return codes here.
mModule.unloadSoundModel(mCurrentSoundModelHandle);
}
mListeners.get(keyphraseId).onListeningStateChanged(STATE_STOPPED);
mActiveListeners.get(keyphraseId).onListeningStateChanged(STATE_STOPPED);
mActiveListeners.remove(keyphraseId);
}
int[] handle = new int[] { INVALID_SOUND_MODEL_HANDLE };
int status = mModule.loadSoundModel(soundModel, handle);
if (status != SoundTrigger.STATUS_OK) {
Slog.w(TAG, "loadSoundModel call failed with " + status);
return STATUS_ERROR;
}
if (handle[0] == INVALID_SOUND_MODEL_HANDLE) {
Slog.w(TAG, "loadSoundModel call returned invalid sound model handle");
return STATUS_ERROR;
}
// Start the recognition.
status = mModule.startRecognition(handle[0], recognitionConfig);
if (status != SoundTrigger.STATUS_OK) {
Slog.w(TAG, "startRecognition failed with " + status);
return STATUS_ERROR;
}
// Everything went well!
mCurrentSoundModelHandle = handle[0];
// Register the new listener. This replaces the old one.
// There can only be a maximum of one active listener for a keyphrase
// at any given time.
mListeners.put(keyphraseId, listener);
// TODO: Get the sound model for the given keyphrase here.
// mModule.loadSoundModel(model, soundModelHandle);
// mModule.startRecognition(soundModelHandle, data);
// mCurrentSoundModelHandle = soundModelHandle;
return STATUS_ERROR;
mActiveListeners.put(keyphraseId, listener);
return STATUS_OK;
}
/**
* Stops recognition for the given {@link Keyphrase} if a recognition is currently active.
*
* @param keyphraseId The identifier of the keyphrase for which
* the recognition is to be stopped.
* @param listener The listener for the recognition events related to the given keyphrase.
*
* @return One of {@link #STATUS_ERROR} or {@link #STATUS_OK}.
*/
public int stopRecognition(int id, Listener listener) {
public synchronized int stopRecognition(int keyphraseId, Listener listener) {
if (dspInfo == null || mModule == null) {
Slog.w(TAG, "Attempting stopRecognition without the capability");
return STATUS_ERROR;
}
if (mListeners.get(id) != listener) {
Listener currentListener = mActiveListeners.get(keyphraseId);
if (currentListener == null) {
// startRecognition hasn't been called or it failed.
Slog.w(TAG, "Attempting stopRecognition without a successful startRecognition");
return STATUS_ERROR;
} else if (currentListener != listener) {
// TODO: Figure out if this should match the listener that was passed in during
// startRecognition, or should we allow a different listener to stop the recognition,
// in which case we don't need to pass in a listener here.
Slog.w(TAG, "Attempting stopRecognition for another recognition");
return STATUS_ERROR;
} else {
// Stop recognition if it's the current one, ignore otherwise.
// TODO: Inspect the return codes here.
mModule.stopRecognition(mCurrentSoundModelHandle);
mModule.unloadSoundModel(mCurrentSoundModelHandle);
int status = mModule.stopRecognition(mCurrentSoundModelHandle);
if (status != SoundTrigger.STATUS_OK) {
Slog.w(TAG, "stopRecognition call failed with " + status);
return STATUS_ERROR;
}
status = mModule.unloadSoundModel(mCurrentSoundModelHandle);
if (status != SoundTrigger.STATUS_OK) {
Slog.w(TAG, "unloadSoundModel call failed with " + status);
return STATUS_ERROR;
}
mCurrentSoundModelHandle = INVALID_SOUND_MODEL_HANDLE;
mActiveListeners.remove(keyphraseId);
return STATUS_OK;
}
}
@@ -184,28 +210,26 @@ public class SoundTriggerHelper implements SoundTrigger.StatusListener {
// TODO: Get the keyphrase out of the event and fire events on it.
// For now, as a nasty workaround, we fire all events to the listener for
// keyphrase with TEMP_KEYPHRASE_ID.
Listener listener = null;
synchronized(this) {
// TODO: The keyphrase should come from the recognition event
// as it may be for a different keyphrase than the current one.
listener = mActiveListeners.get(TEMP_KEYPHRASE_ID);
}
if (listener == null) {
Slog.w(TAG, "received onRecognition event without any listener for it");
return;
}
switch (event.status) {
case SoundTrigger.RECOGNITION_STATUS_SUCCESS:
// TODO: The keyphrase should come from the recognition event
// as it may be for a different keyphrase than the current one.
if (mListeners.get(TEMP_KEYPHRASE_ID) != null) {
mListeners.get(TEMP_KEYPHRASE_ID).onKeyphraseSpoken();
}
listener.onKeyphraseSpoken();
break;
case SoundTrigger.RECOGNITION_STATUS_ABORT:
// TODO: The keyphrase should come from the recognition event
// as it may be for a different keyphrase than the current one.
if (mListeners.get(TEMP_KEYPHRASE_ID) != null) {
mListeners.get(TEMP_KEYPHRASE_ID).onListeningStateChanged(STATE_STOPPED);
}
listener.onListeningStateChanged(STATE_STOPPED);
break;
case SoundTrigger.RECOGNITION_STATUS_FAILURE:
// TODO: The keyphrase should come from the recognition event
// as it may be for a different keyphrase than the current one.
if (mListeners.get(TEMP_KEYPHRASE_ID) != null) {
mListeners.get(TEMP_KEYPHRASE_ID).onListeningStateChanged(STATE_STOPPED);
}
listener.onListeningStateChanged(STATE_STOPPED);
break;
}
}

98
core/java/android/service/voice/AlwaysOnHotwordDetector.java
View File

@@ -20,9 +20,19 @@ import android.content.Intent;
import android.hardware.soundtrigger.Keyphrase;
import android.hardware.soundtrigger.KeyphraseEnrollmentInfo;
import android.hardware.soundtrigger.KeyphraseMetadata;
import android.hardware.soundtrigger.KeyphraseSoundModel;
import android.hardware.soundtrigger.SoundTrigger;
import android.hardware.soundtrigger.SoundTrigger.ConfidenceLevel;
import android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionExtra;
import android.hardware.soundtrigger.SoundTriggerHelper;
import android.hardware.soundtrigger.SoundTrigger.RecognitionConfig;
import android.os.RemoteException;
import android.util.Slog;
import com.android.internal.app.IVoiceInteractionManagerService;
import java.util.List;
/**
* A class that lets a VoiceInteractionService implementation interact with
* always-on keyphrase detection APIs.
@@ -72,11 +82,22 @@ public class AlwaysOnHotwordDetector {
private final String mText;
private final String mLocale;
private final Keyphrase mKeyphrase;
/**
* The metadata of the Keyphrase, derived from the enrollment application.
* This may be null if this keyphrase isn't supported by the enrollment application.
*/
private final KeyphraseMetadata mKeyphraseMetadata;
/**
* The sound model for the keyphrase, derived from the model management service
* (IVoiceInteractionManagerService). May be null if the keyphrase isn't enrolled yet.
*/
private final KeyphraseSoundModel mEnrolledSoundModel;
private final KeyphraseEnrollmentInfo mKeyphraseEnrollmentInfo;
private final SoundTriggerHelper mSoundTriggerHelper;
private final SoundTriggerHelper.Listener mListener;
private final int mAvailability;
private final IVoiceInteractionService mVoiceInteractionService;
private final IVoiceInteractionManagerService mModelManagementService;
private int mRecognitionState;
@@ -103,25 +124,30 @@ public class AlwaysOnHotwordDetector {
* @param text The keyphrase text to get the detector for.
* @param locale The java locale for the detector.
* @param callback A non-null Callback for receiving the recognition events.
* @param voiceInteractionService The current voice interaction service.
* @param modelManagementService A service that allows management of sound models.
*
* @hide
*/
public AlwaysOnHotwordDetector(String text, String locale, Callback callback,
KeyphraseEnrollmentInfo keyphraseEnrollmentInfo,
SoundTriggerHelper soundTriggerHelper) {
SoundTriggerHelper soundTriggerHelper,
IVoiceInteractionService voiceInteractionService,
IVoiceInteractionManagerService modelManagementService) {
mText = text;
mLocale = locale;
mKeyphraseEnrollmentInfo = keyphraseEnrollmentInfo;
KeyphraseMetadata keyphraseMetadata =
mKeyphraseEnrollmentInfo.getKeyphraseMetadata(text, locale);
if (keyphraseMetadata != null) {
mKeyphrase = new Keyphrase(keyphraseMetadata.id, text, locale);
} else {
mKeyphrase = null;
}
mKeyphraseMetadata = mKeyphraseEnrollmentInfo.getKeyphraseMetadata(text, locale);
mListener = new SoundTriggerListener(callback);
mSoundTriggerHelper = soundTriggerHelper;
mAvailability = getAvailabilityInternal();
mVoiceInteractionService = voiceInteractionService;
mModelManagementService = modelManagementService;
if (mKeyphraseMetadata != null) {
mEnrolledSoundModel = internalGetKeyphraseSoundModel(mKeyphraseMetadata.id);
} else {
mEnrolledSoundModel = null;
}
mAvailability = internalGetAvailability();
}
/**
@@ -171,7 +197,16 @@ public class AlwaysOnHotwordDetector {
}
mRecognitionState = RECOGNITION_REQUESTED;
int code = mSoundTriggerHelper.startRecognition(mKeyphrase.id, mListener);
mRecognitionState = RECOGNITION_REQUESTED;
KeyphraseRecognitionExtra[] recognitionExtra = new KeyphraseRecognitionExtra[1];
// TODO: Do we need to do something about the confidence level here?
// TODO: Read the recognition mode flag from the KeyphraseMetadata.
// TODO: Take in captureTriggerAudio as a method param here.
recognitionExtra[0] = new KeyphraseRecognitionExtra(mKeyphraseMetadata.id,
SoundTrigger.RECOGNITION_MODE_VOICE_TRIGGER, new ConfidenceLevel[0]);
int code = mSoundTriggerHelper.startRecognition(mKeyphraseMetadata.id,
mEnrolledSoundModel.convertToSoundTriggerKeyphraseSoundModel(), mListener,
new RecognitionConfig(false, recognitionExtra, null /* additional data */));
if (code != SoundTriggerHelper.STATUS_OK) {
Slog.w(TAG, "startRecognition() failed with error code " + code);
return STATUS_ERROR;
@@ -195,7 +230,8 @@ public class AlwaysOnHotwordDetector {
}
mRecognitionState = RECOGNITION_NOT_REQUESTED;
int code = mSoundTriggerHelper.stopRecognition(mKeyphrase.id, mListener);
int code = mSoundTriggerHelper.stopRecognition(mKeyphraseMetadata.id, mListener);
if (code != SoundTriggerHelper.STATUS_OK) {
Slog.w(TAG, "stopRecognition() failed with error code " + code);
return STATUS_ERROR;
@@ -230,19 +266,51 @@ public class AlwaysOnHotwordDetector {
return mKeyphraseEnrollmentInfo.getManageKeyphraseIntent(action, mText, mLocale);
}
private int getAvailabilityInternal() {
private int internalGetAvailability() {
// No DSP available
if (mSoundTriggerHelper.dspInfo == null) {
return KEYPHRASE_HARDWARE_UNAVAILABLE;
}
if (mKeyphrase == null || !mSoundTriggerHelper.isKeyphraseSupported(mKeyphrase)) {
// No enrollment application supports this keyphrase/locale
if (mKeyphraseMetadata == null) {
return KEYPHRASE_UNSUPPORTED;
}
if (!mSoundTriggerHelper.isKeyphraseEnrolled(mKeyphrase)) {
// This keyphrase hasn't been enrolled.
if (mEnrolledSoundModel == null) {
return KEYPHRASE_UNENROLLED;
}
return KEYPHRASE_ENROLLED;
}
/**
* @return The corresponding {@link KeyphraseSoundModel} or null if none is found.
*/
private KeyphraseSoundModel internalGetKeyphraseSoundModel(int keyphraseId) {
List<KeyphraseSoundModel> soundModels;
try {
soundModels = mModelManagementService
.listRegisteredKeyphraseSoundModels(mVoiceInteractionService);
if (soundModels == null || soundModels.isEmpty()) {
Slog.i(TAG, "No available sound models for keyphrase ID: " + keyphraseId);
return null;
}
for (KeyphraseSoundModel soundModel : soundModels) {
if (soundModel.keyphrases == null) {
continue;
}
for (Keyphrase keyphrase : soundModel.keyphrases) {
// TODO: Check the user handle here to only load a model for the current user.
if (keyphrase.id == keyphraseId) {
return soundModel;
}
}
}
} catch (RemoteException e) {
Slog.w(TAG, "RemoteException in listRegisteredKeyphraseSoundModels!");
}
return null;
}
/** @hide */
static final class SoundTriggerListener implements SoundTriggerHelper.Listener {
private final Callback mCallback;

4
core/java/android/service/voice/VoiceInteractionService.java
View File

@@ -104,10 +104,8 @@ public class VoiceInteractionService extends Service {
*/
public final AlwaysOnHotwordDetector getAlwaysOnHotwordDetector(
String keyphrase, String locale, AlwaysOnHotwordDetector.Callback callback) {
// TODO: Cache instances and return the same one instead of creating a new interactor
// for the same keyphrase/locale combination.
return new AlwaysOnHotwordDetector(keyphrase, locale, callback,
mKeyphraseEnrollmentInfo, mSoundTriggerHelper);
mKeyphraseEnrollmentInfo, mSoundTriggerHelper, mInterface, mSystemService);
}
/**

18
services/voiceinteraction/java/com/android/server/voiceinteraction/DatabaseHelper.java
View File

@@ -23,8 +23,8 @@ import android.database.sqlite.SQLiteDatabase;
import android.database.sqlite.SQLiteDatabase.CursorFactory;
import android.database.sqlite.SQLiteOpenHelper;
import android.hardware.soundtrigger.SoundTrigger;
import android.hardware.soundtrigger.SoundTrigger.Keyphrase;
import android.hardware.soundtrigger.SoundTrigger.KeyphraseSoundModel;
import android.hardware.soundtrigger.Keyphrase;
import android.hardware.soundtrigger.KeyphraseSoundModel;
import android.util.Slog;
import java.util.ArrayList;
@@ -32,6 +32,8 @@ import java.util.List;
import java.util.UUID;
/**
* Helper to manage the database of the sound models that have been registered on the device.
*
* @hide
*/
public class DatabaseHelper extends SQLiteOpenHelper {
@@ -86,14 +88,14 @@ public class DatabaseHelper extends SQLiteOpenHelper {
@Override
public void onUpgrade(SQLiteDatabase db, int oldVersion, int newVersion) {
// TODO(sansid): For now, drop older tables and recreate new ones.
// TODO: For now, drop older tables and recreate new ones.
db.execSQL("DROP TABLE IF EXISTS " + KeyphraseContract.TABLE);
db.execSQL("DROP TABLE IF EXISTS " + SoundModelContract.TABLE);
onCreate(db);
}
/**
* TODO(sansid): Change to addOrUpdate to handle changes here.
* TODO: Change to addOrUpdate to handle changes here.
*/
public void addKeyphraseSoundModel(KeyphraseSoundModel soundModel) {
SQLiteDatabase db = this.getWritableDatabase();
@@ -115,13 +117,13 @@ public class DatabaseHelper extends SQLiteOpenHelper {
/**
* TODO(sansid): Change to addOrUpdate to handle changes here.
*/
private void addKeyphrase(UUID modelId, SoundTrigger.Keyphrase keyphrase) {
private void addKeyphrase(UUID modelId, Keyphrase keyphrase) {
SQLiteDatabase db = this.getWritableDatabase();
ContentValues values = new ContentValues();
values.put(KeyphraseContract.KEY_ID, keyphrase.id);
values.put(KeyphraseContract.KEY_RECOGNITION_MODES, keyphrase.recognitionModes);
values.put(KeyphraseContract.KEY_RECOGNITION_MODES, keyphrase.recognitionModeFlags);
values.put(KeyphraseContract.KEY_SOUND_MODEL_ID, keyphrase.id);
values.put(KeyphraseContract.KEY_HINT_TEXT, keyphrase.text);
values.put(KeyphraseContract.KEY_HINT_TEXT, keyphrase.hintText);
values.put(KeyphraseContract.KEY_LOCALE, keyphrase.locale);
if (db.insert(KeyphraseContract.TABLE, null, values) == -1) {
Slog.w(TAG, "Failed to persist keyphrase to database");
@@ -171,7 +173,7 @@ public class DatabaseHelper extends SQLiteOpenHelper {
String locale = c.getString(c.getColumnIndex(KeyphraseContract.KEY_LOCALE));
String hintText = c.getString(c.getColumnIndex(KeyphraseContract.KEY_HINT_TEXT));
keyphrases.add(new Keyphrase(id, modes, locale, hintText, users));
keyphrases.add(new Keyphrase(id, hintText, locale, modes, users));
} while (c.moveToNext());
}
Keyphrase[] keyphraseArr = new Keyphrase[keyphrases.size()];