Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge "Speech: Concurrent recognition service"

Browse Source
This commit is contained in:
Aleksandar Kiridžić
2022-12-19 12:16:45 +00:00
committed by Android (Google) Code Review
parent 2d20d1c998 ec5a9ecb55
commit c13a065298
4 changed files with 335 additions and 195 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
core/api/current.txt
View File

@@ -40475,6 +40475,7 @@ package android.speech {
public abstract class RecognitionService extends android.app.Service {
ctor public RecognitionService();
method public int getMaxConcurrentSessionsCount();
method public final android.os.IBinder onBind(android.content.Intent);
method protected abstract void onCancel(android.speech.RecognitionService.Callback);
method public void onCheckRecognitionSupport(@NonNull android.content.Intent, @NonNull android.speech.RecognitionService.SupportCallback);

240
core/java/android/speech/RecognitionService.java
View File

@@ -42,6 +42,8 @@ import android.util.Pair;
import com.android.internal.util.function.pooled.PooledLambda;
import java.lang.ref.WeakReference;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
/**
@@ -55,7 +57,7 @@ public abstract class RecognitionService extends Service {
*/
@SdkConstant(SdkConstantType.SERVICE_ACTION)
public static final String SERVICE_INTERFACE = "android.speech.RecognitionService";
/**
* Name under which a RecognitionService component publishes information about itself.
* This meta-data should reference an XML resource containing a
@@ -71,17 +73,12 @@ public abstract class RecognitionService extends Service {
/** Debugging flag */
private static final boolean DBG = false;
private static final int DEFAULT_MAX_CONCURRENT_SESSIONS_COUNT = 1;
private final Map<IBinder, SessionState> mSessions = new HashMap<>();
/** Binder of the recognition service */
private RecognitionServiceBinder mBinder = new RecognitionServiceBinder(this);
/**
* The current callback of an application that invoked the
*
* {@link RecognitionService#onStartListening(Intent, Callback)} method
*/
private Callback mCurrentCallback = null;
private boolean mStartedDataDelivery;
private final RecognitionServiceBinder mBinder = new RecognitionServiceBinder(this);
private static final int MSG_START_LISTENING = 1;
@@ -110,7 +107,7 @@ public abstract class RecognitionService extends Service {
dispatchCancel((IRecognitionListener) msg.obj);
break;
case MSG_RESET:
dispatchClearCallback();
dispatchClearCallback((IRecognitionListener) msg.obj);
break;
case MSG_CHECK_RECOGNITION_SUPPORT:
Pair<Intent, IRecognitionSupportCallback> intentAndListener =
@@ -127,71 +124,90 @@ public abstract class RecognitionService extends Service {
private void dispatchStartListening(Intent intent, final IRecognitionListener listener,
@NonNull AttributionSource attributionSource) {
Callback currentCallback = null;
SessionState sessionState = mSessions.get(listener.asBinder());
try {
if (mCurrentCallback == null) {
boolean preflightPermissionCheckPassed =
intent.hasExtra(RecognizerIntent.EXTRA_AUDIO_SOURCE)
|| checkPermissionForPreflightNotHardDenied(attributionSource);
if (preflightPermissionCheckPassed) {
if (DBG) {
Log.d(TAG, "created new mCurrentCallback, listener = "
+ listener.asBinder());
}
mCurrentCallback = new Callback(listener, attributionSource);
RecognitionService.this.onStartListening(intent, mCurrentCallback);
if (sessionState == null) {
if (mSessions.size() >= getMaxConcurrentSessionsCount()) {
listener.onError(SpeechRecognizer.ERROR_RECOGNIZER_BUSY);
Log.i(TAG, "#startListening received "
+ "when the service's capacity is full - ignoring this call.");
return;
}
if (!preflightPermissionCheckPassed || !checkPermissionAndStartDataDelivery()) {
boolean preflightPermissionCheckPassed =
intent.hasExtra(RecognizerIntent.EXTRA_AUDIO_SOURCE)
|| checkPermissionForPreflightNotHardDenied(attributionSource);
if (preflightPermissionCheckPassed) {
currentCallback = new Callback(listener, attributionSource);
sessionState = new SessionState(currentCallback);
RecognitionService.this.onStartListening(intent, currentCallback);
}
if (!preflightPermissionCheckPassed
|| !checkPermissionAndStartDataDelivery(sessionState)) {
listener.onError(SpeechRecognizer.ERROR_INSUFFICIENT_PERMISSIONS);
if (preflightPermissionCheckPassed) {
// If we attempted to start listening, cancel the callback
RecognitionService.this.onCancel(mCurrentCallback);
dispatchClearCallback();
// If start listening was attempted, cancel the callback.
RecognitionService.this.onCancel(currentCallback);
finishDataDelivery(sessionState);
sessionState.reset();
}
Log.i(TAG, "caller doesn't have permission:"
+ Manifest.permission.RECORD_AUDIO);
Log.i(TAG, "#startListening received from a caller "
+ "without permission " + Manifest.permission.RECORD_AUDIO + ".");
} else {
if (DBG) {
Log.d(TAG, "Added a new session to the map.");
}
mSessions.put(listener.asBinder(), sessionState);
}
} else {
listener.onError(SpeechRecognizer.ERROR_RECOGNIZER_BUSY);
Log.i(TAG, "concurrent startListening received - ignoring this call");
listener.onError(SpeechRecognizer.ERROR_CLIENT);
Log.i(TAG, "#startListening received "
+ "for a listener which is already in session - ignoring this call.");
}
} catch (RemoteException e) {
Log.d(TAG, "onError call from startListening failed");
Log.d(TAG, "#onError call from #startListening failed.");
}
}
private void dispatchStopListening(IRecognitionListener listener) {
try {
if (mCurrentCallback == null) {
SessionState sessionState = mSessions.get(listener.asBinder());
if (sessionState == null) {
try {
listener.onError(SpeechRecognizer.ERROR_CLIENT);
Log.w(TAG, "stopListening called with no preceding startListening - ignoring");
} else if (mCurrentCallback.mListener.asBinder() != listener.asBinder()) {
listener.onError(SpeechRecognizer.ERROR_RECOGNIZER_BUSY);
Log.w(TAG, "stopListening called by other caller than startListening - ignoring");
} else { // the correct state
RecognitionService.this.onStopListening(mCurrentCallback);
} catch (RemoteException e) {
Log.d(TAG, "#onError call from #stopListening failed.");
}
} catch (RemoteException e) { // occurs if onError fails
Log.d(TAG, "onError call from stopListening failed");
Log.w(TAG, "#stopListening received for a listener "
+ "which has not started a session - ignoring this call.");
} else {
RecognitionService.this.onStopListening(sessionState.mCallback);
}
}
private void dispatchCancel(IRecognitionListener listener) {
if (mCurrentCallback == null) {
if (DBG) Log.d(TAG, "cancel called with no preceding startListening - ignoring");
} else if (mCurrentCallback.mListener.asBinder() != listener.asBinder()) {
Log.w(TAG, "cancel called by client who did not call startListening - ignoring");
} else { // the correct state
RecognitionService.this.onCancel(mCurrentCallback);
dispatchClearCallback();
if (DBG) Log.d(TAG, "canceling - setting mCurrentCallback to null");
SessionState sessionState = mSessions.get(listener.asBinder());
if (sessionState == null) {
Log.w(TAG, "#cancel received for a listener which has not started a session "
+ "- ignoring this call.");
} else {
RecognitionService.this.onCancel(sessionState.mCallback);
dispatchClearCallback(listener);
}
}
private void dispatchClearCallback() {
finishDataDelivery();
mCurrentCallback = null;
mStartedDataDelivery = false;
private void dispatchClearCallback(IRecognitionListener listener) {
SessionState sessionState = mSessions.remove(listener.asBinder());
if (sessionState != null) {
if (DBG) {
Log.d(TAG, "Removed session from the map for listener = "
+ listener.asBinder() + ".");
}
finishDataDelivery(sessionState);
sessionState.reset();
}
}
private void dispatchCheckRecognitionSupport(
@@ -203,11 +219,11 @@ public abstract class RecognitionService extends Service {
RecognitionService.this.onTriggerModelDownload(intent);
}
private class StartListeningArgs {
private static class StartListeningArgs {
public final Intent mIntent;
public final IRecognitionListener mListener;
public final @NonNull AttributionSource mAttributionSource;
@NonNull public final AttributionSource mAttributionSource;
public StartListeningArgs(Intent intent, IRecognitionListener listener,
@NonNull AttributionSource attributionSource) {
@@ -306,27 +322,42 @@ public abstract class RecognitionService extends Service {
}
private void handleAttributionContextCreation(@NonNull AttributionSource attributionSource) {
if (mCurrentCallback != null
&& mCurrentCallback.mCallingAttributionSource.equals(attributionSource)) {
mCurrentCallback.mAttributionContextCreated = true;
for (SessionState sessionState : mSessions.values()) {
Callback currentCallback = sessionState.mCallback;
if (currentCallback != null
&& currentCallback.mCallingAttributionSource.equals(attributionSource)) {
currentCallback.mAttributionContextCreated = true;
}
}
}
@Override
public final IBinder onBind(final Intent intent) {
if (DBG) Log.d(TAG, "onBind, intent=" + intent);
if (DBG) Log.d(TAG, "#onBind, intent=" + intent);
return mBinder;
}
@Override
public void onDestroy() {
if (DBG) Log.d(TAG, "onDestroy");
finishDataDelivery();
mCurrentCallback = null;
if (DBG) Log.d(TAG, "#onDestroy");
for (SessionState sessionState : mSessions.values()) {
finishDataDelivery(sessionState);
sessionState.reset();
}
mSessions.clear();
mBinder.clearReference();
super.onDestroy();
}
/**
* Returns the maximal number of recognition sessions ongoing at the same time.
* <p>
* The default value is 1, meaning concurrency should be enabled by overriding this method.
*/
public int getMaxConcurrentSessionsCount() {
return DEFAULT_MAX_CONCURRENT_SESSIONS_COUNT;
}
/**
* This class receives callbacks from the speech recognition service and forwards them to the
* user. An instance of this class is passed to the
@@ -335,8 +366,8 @@ public abstract class RecognitionService extends Service {
*/
public class Callback {
private final IRecognitionListener mListener;
private final @NonNull AttributionSource mCallingAttributionSource;
private @Nullable Context mAttributionContext;
@NonNull private final AttributionSource mCallingAttributionSource;
@Nullable private Context mAttributionContext;
private boolean mAttributionContextCreated;
private Callback(IRecognitionListener listener,
@@ -355,7 +386,7 @@ public abstract class RecognitionService extends Service {
/**
* The service should call this method when sound has been received. The purpose of this
* function is to allow giving feedback to the user regarding the captured audio.
*
*
* @param buffer a buffer containing a sequence of big-endian 16-bit integers representing a
* single channel audio stream. The sample rate is implementation dependent.
*/
@@ -372,11 +403,11 @@ public abstract class RecognitionService extends Service {
/**
* The service should call this method when a network or recognition error occurred.
*
*
* @param error code is defined in {@link SpeechRecognizer}
*/
public void error(@SpeechRecognizer.RecognitionError int error) throws RemoteException {
Message.obtain(mHandler, MSG_RESET).sendToTarget();
Message.obtain(mHandler, MSG_RESET, mListener).sendToTarget();
mListener.onError(error);
}
@@ -386,7 +417,7 @@ public abstract class RecognitionService extends Service {
* {@link #results(Bundle)} when partial results are ready. This method may be called zero,
* one or multiple times for each call to {@link SpeechRecognizer#startListening(Intent)},
* depending on the speech recognition service implementation.
*
*
* @param partialResults the returned results. To retrieve the results in
* ArrayList&lt;String&gt; format use {@link Bundle#getStringArrayList(String)} with
* {@link SpeechRecognizer#RESULTS_RECOGNITION} as a parameter
@@ -398,7 +429,7 @@ public abstract class RecognitionService extends Service {
/**
* The service should call this method when the endpointer is ready for the user to start
* speaking.
*
*
* @param params parameters set by the recognition service. Reserved for future use.
*/
public void readyForSpeech(Bundle params) throws RemoteException {
@@ -407,20 +438,20 @@ public abstract class RecognitionService extends Service {
/**
* The service should call this method when recognition results are ready.
*
*
* @param results the recognition results. To retrieve the results in {@code
* ArrayList<String>} format use {@link Bundle#getStringArrayList(String)} with
* {@link SpeechRecognizer#RESULTS_RECOGNITION} as a parameter
*/
public void results(Bundle results) throws RemoteException {
Message.obtain(mHandler, MSG_RESET).sendToTarget();
Message.obtain(mHandler, MSG_RESET, mListener).sendToTarget();
mListener.onResults(results);
}
/**
* The service should call this method when the sound level in the audio stream has changed.
* There is no guarantee that this method will be called.
*
*
* @param rmsdB the new RMS dB value
*/
public void rmsChanged(float rmsdB) throws RemoteException {
@@ -444,7 +475,7 @@ public abstract class RecognitionService extends Service {
*/
@SuppressLint({"CallbackMethodName", "RethrowRemoteException"})
public void endOfSegmentedSession() throws RemoteException {
Message.obtain(mHandler, MSG_RESET).sendToTarget();
Message.obtain(mHandler, MSG_RESET, mListener).sendToTarget();
mListener.onEndOfSegmentedSession();
}
@@ -469,7 +500,8 @@ public abstract class RecognitionService extends Service {
* AttributionSource)
*/
@SuppressLint("CallbackMethodName")
public @NonNull AttributionSource getCallingAttributionSource() {
@NonNull
public AttributionSource getCallingAttributionSource() {
return mCallingAttributionSource;
}
@@ -490,7 +522,6 @@ public abstract class RecognitionService extends Service {
* these methods on any thread.
*/
public static class SupportCallback {
private final IRecognitionSupportCallback mCallback;
private SupportCallback(IRecognitionSupportCallback callback) {
@@ -521,7 +552,7 @@ public abstract class RecognitionService extends Service {
}
}
/** Binder of the recognition service */
/** Binder of the recognition service. */
private static final class RecognitionServiceBinder extends IRecognitionService.Stub {
private final WeakReference<RecognitionService> mServiceRef;
@@ -538,7 +569,7 @@ public abstract class RecognitionService extends Service {
final RecognitionService service = mServiceRef.get();
if (service != null) {
service.mHandler.sendMessage(Message.obtain(service.mHandler,
MSG_START_LISTENING, service.new StartListeningArgs(
MSG_START_LISTENING, new StartListeningArgs(
recognizerIntent, listener, attributionSource)));
}
}
@@ -589,17 +620,21 @@ public abstract class RecognitionService extends Service {
}
}
private boolean checkPermissionAndStartDataDelivery() {
if (mCurrentCallback.mAttributionContextCreated) {
private boolean checkPermissionAndStartDataDelivery(SessionState sessionState) {
if (sessionState.mCallback.mAttributionContextCreated) {
return true;
}
if (PermissionChecker.checkPermissionAndStartDataDelivery(
RecognitionService.this, Manifest.permission.RECORD_AUDIO,
mCurrentCallback.getAttributionContextForCaller().getAttributionSource(),
/*message*/ null) == PermissionChecker.PERMISSION_GRANTED) {
mStartedDataDelivery = true;
RecognitionService.this,
Manifest.permission.RECORD_AUDIO,
sessionState.mCallback.getAttributionContextForCaller().getAttributionSource(),
/* message */ null)
== PermissionChecker.PERMISSION_GRANTED) {
sessionState.mStartedDataDelivery = true;
}
return mStartedDataDelivery;
return sessionState.mStartedDataDelivery;
}
private boolean checkPermissionForPreflightNotHardDenied(AttributionSource attributionSource) {
@@ -609,12 +644,39 @@ public abstract class RecognitionService extends Service {
|| result == PermissionChecker.PERMISSION_SOFT_DENIED;
}
void finishDataDelivery() {
if (mStartedDataDelivery) {
mStartedDataDelivery = false;
void finishDataDelivery(SessionState sessionState) {
if (sessionState.mStartedDataDelivery) {
sessionState.mStartedDataDelivery = false;
final String op = AppOpsManager.permissionToOp(Manifest.permission.RECORD_AUDIO);
PermissionChecker.finishDataDelivery(RecognitionService.this, op,
mCurrentCallback.getAttributionContextForCaller().getAttributionSource());
sessionState.mCallback.getAttributionContextForCaller().getAttributionSource());
}
}
/**
* Data class containing information about an ongoing session:
* <ul>
* <li> {@link SessionState#mCallback} - callback of the client that invoked the
* {@link RecognitionService#onStartListening(Intent, Callback)} method;
* <li> {@link SessionState#mStartedDataDelivery} - flag denoting if data
* is being delivered to the client.
*/
private static class SessionState {
private Callback mCallback;
private boolean mStartedDataDelivery;
SessionState(Callback callback, boolean startedDataDelivery) {
mCallback = callback;
mStartedDataDelivery = startedDataDelivery;
}
SessionState(Callback currentCallback) {
this(currentCallback, false);
}
void reset() {
mCallback = null;
mStartedDataDelivery = false;
}
}
}

266
services/core/java/com/android/server/speech/RemoteSpeechRecognitionService.java
View File

@@ -19,46 +19,49 @@ package com.android.server.speech;
import static com.android.internal.infra.AbstractRemoteService.PERMANENT_BOUND_TIMEOUT_MS;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.content.AttributionSource;
import android.content.ComponentName;
import android.content.Context;
import android.content.Intent;
import android.os.Bundle;
import android.os.IBinder;
import android.os.RemoteException;
import android.speech.IRecognitionListener;
import android.speech.IRecognitionService;
import android.speech.IRecognitionSupportCallback;
import android.speech.RecognitionService;
import android.speech.SpeechRecognizer;
import android.text.TextUtils;
import android.util.Log;
import android.util.Pair;
import android.util.Slog;
import com.android.internal.annotations.GuardedBy;
import com.android.internal.infra.ServiceConnector;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
final class RemoteSpeechRecognitionService extends ServiceConnector.Impl<IRecognitionService> {
private static final String TAG = RemoteSpeechRecognitionService.class.getSimpleName();
private static final boolean DEBUG = false;
/** Maximum number of clients connected to this object at the same time. */
private static final int MAX_CONCURRENT_CLIENTS = 100;
private final Object mLock = new Object();
private boolean mConnected = false;
@Nullable
private IRecognitionListener mListener;
@Nullable
/** Map containing info about connected clients indexed by the their listeners. */
@GuardedBy("mLock")
private DelegatingListener mDelegatingListener;
private final Map<IBinder, ClientState> mClients = new HashMap<>();
// Makes sure we can block startListening() if session is still in progress.
/** List of pairs associating clients' binder tokens with corresponding listeners. */
@GuardedBy("mLock")
private boolean mSessionInProgress = false;
// Makes sure we call startProxyOp / finishProxyOp at right times and only once per session.
@GuardedBy("mLock")
private boolean mRecordingInProgress = false;
private final List<Pair<IBinder, IRecognitionListener>> mClientListeners = new ArrayList<>();
private final int mCallingUid;
private final ComponentName mComponentName;
@@ -78,7 +81,7 @@ final class RemoteSpeechRecognitionService extends ServiceConnector.Impl<IRecogn
mComponentName = serviceName;
if (DEBUG) {
Slog.i(TAG, "Bound to recognition service at: " + serviceName.flattenToString());
Slog.i(TAG, "Bound to recognition service at: " + serviceName.flattenToString() + ".");
}
}
@@ -89,13 +92,14 @@ final class RemoteSpeechRecognitionService extends ServiceConnector.Impl<IRecogn
void startListening(Intent recognizerIntent, IRecognitionListener listener,
@NonNull AttributionSource attributionSource) {
if (DEBUG) {
Slog.i(TAG, String.format("#startListening for package: %s, feature=%s, callingUid=%d",
Slog.i(TAG, TextUtils.formatSimple("#startListening for package: "
+ "%s, feature=%s, callingUid=%d.",
attributionSource.getPackageName(), attributionSource.getAttributionTag(),
mCallingUid));
}
if (listener == null) {
Log.w(TAG, "#startListening called with no preceding #setListening - ignoring");
Slog.w(TAG, "#startListening called with no preceding #setListening - ignoring.");
return;
}
@@ -105,29 +109,52 @@ final class RemoteSpeechRecognitionService extends ServiceConnector.Impl<IRecogn
}
synchronized (mLock) {
if (mSessionInProgress) {
Slog.i(TAG, "#startListening called while listening is in progress.");
tryRespondWithError(listener, SpeechRecognizer.ERROR_RECOGNIZER_BUSY);
return;
ClientState clientState = mClients.get(listener.asBinder());
if (clientState == null) {
if (mClients.size() >= MAX_CONCURRENT_CLIENTS) {
tryRespondWithError(listener, SpeechRecognizer.ERROR_RECOGNIZER_BUSY);
Log.i(TAG, "#startListening received "
+ "when the recognizer's capacity is full - ignoring this call.");
return;
}
final ClientState newClientState = new ClientState();
newClientState.mDelegatingListener = new DelegatingListener(listener,
() -> {
// To be invoked in terminal calls on success.
if (DEBUG) {
Slog.i(TAG, "Recognition session completed successfully.");
}
synchronized (mLock) {
newClientState.mRecordingInProgress = false;
}
},
() -> {
// To be invoked in terminal calls on failure.
if (DEBUG) {
Slog.i(TAG, "Recognition session failed.");
}
removeClient(listener);
});
if (DEBUG) {
Log.d(TAG, "Added a new client to the map.");
}
mClients.put(listener.asBinder(), newClientState);
clientState = newClientState;
} else {
if (clientState.mRecordingInProgress) {
Slog.i(TAG, "#startListening called "
+ "while listening is in progress for this caller.");
tryRespondWithError(listener, SpeechRecognizer.ERROR_CLIENT);
return;
}
clientState.mRecordingInProgress = true;
}
mSessionInProgress = true;
mRecordingInProgress = true;
mListener = listener;
mDelegatingListener = new DelegatingListener(listener, () -> {
// To be invoked in terminal calls of the callback: results() or error()
if (DEBUG) {
Slog.i(TAG, "Recognition session complete");
}
synchronized (mLock) {
resetStateLocked();
}
});
// Eager local evaluation to avoid reading a different or null value at closure-run-time
final DelegatingListener listenerToStart = this.mDelegatingListener;
// Eager local evaluation to avoid reading a different or null value at closure runtime.
final DelegatingListener listenerToStart = clientState.mDelegatingListener;
run(service ->
service.startListening(
recognizerIntent,
@@ -147,26 +174,22 @@ final class RemoteSpeechRecognitionService extends ServiceConnector.Impl<IRecogn
}
synchronized (mLock) {
if (mListener == null) {
Log.w(TAG, "#stopListening called with no preceding #startListening - ignoring");
ClientState clientState = mClients.get(listener.asBinder());
if (clientState == null) {
Slog.w(TAG, "#stopListening called with no preceding #startListening - ignoring.");
tryRespondWithError(listener, SpeechRecognizer.ERROR_CLIENT);
return;
}
if (mListener.asBinder() != listener.asBinder()) {
Log.w(TAG, "#stopListening called with an unexpected listener");
if (!clientState.mRecordingInProgress) {
tryRespondWithError(listener, SpeechRecognizer.ERROR_CLIENT);
Slog.i(TAG, "#stopListening called while listening isn't in progress - ignoring.");
return;
}
clientState.mRecordingInProgress = false;
if (!mRecordingInProgress) {
Slog.i(TAG, "#stopListening called while listening isn't in progress, ignoring.");
return;
}
mRecordingInProgress = false;
// Eager local evaluation to avoid reading a different or null value at closure-run-time
final DelegatingListener listenerToStop = this.mDelegatingListener;
// Eager local evaluation to avoid reading a different or null value at closure runtime.
final DelegatingListener listenerToStop = clientState.mDelegatingListener;
run(service -> service.stopListening(listenerToStop));
}
}
@@ -181,33 +204,30 @@ final class RemoteSpeechRecognitionService extends ServiceConnector.Impl<IRecogn
}
synchronized (mLock) {
if (mListener == null) {
ClientState clientState = mClients.get(listener.asBinder());
if (clientState == null) {
if (DEBUG) {
Log.w(TAG, "#cancel called with no preceding #startListening - ignoring");
Slog.w(TAG, "#cancel called with no preceding #startListening - ignoring.");
}
return;
}
if (mListener.asBinder() != listener.asBinder()) {
Log.w(TAG, "#cancel called with an unexpected listener");
tryRespondWithError(listener, SpeechRecognizer.ERROR_CLIENT);
return;
}
clientState.mRecordingInProgress = false;
// Temporary reference to allow for resetting the hard link mDelegatingListener to null.
IRecognitionListener delegatingListener = mDelegatingListener;
final IRecognitionListener delegatingListener = clientState.mDelegatingListener;
run(service -> service.cancel(delegatingListener, isShutdown));
mRecordingInProgress = false;
mSessionInProgress = false;
mDelegatingListener = null;
mListener = null;
// Schedule to unbind after cancel is delivered.
// If shutdown, remove the client info from the map. Unbind if that was the last client.
if (isShutdown) {
run(service -> unbind());
removeClient(listener);
if (mClients.isEmpty()) {
if (DEBUG) {
Slog.d(TAG, "Unbinding from the recognition service.");
}
run(service -> unbind());
}
}
}
}
@@ -215,7 +235,6 @@ final class RemoteSpeechRecognitionService extends ServiceConnector.Impl<IRecogn
void checkRecognitionSupport(
Intent recognizerIntent,
IRecognitionSupportCallback callback) {
if (!mConnected) {
try {
callback.onError(SpeechRecognizer.ERROR_SERVER_DISCONNECTED);
@@ -236,18 +255,14 @@ final class RemoteSpeechRecognitionService extends ServiceConnector.Impl<IRecogn
run(service -> service.triggerModelDownload(recognizerIntent));
}
void shutdown() {
void shutdown(IBinder clientToken) {
synchronized (mLock) {
if (this.mListener == null) {
if (DEBUG) {
Slog.i(TAG, "Package died, but session wasn't initialized. "
+ "Not invoking #cancel");
for (Pair<IBinder, IRecognitionListener> clientListener : mClientListeners) {
if (clientListener.first == clientToken) {
cancel(clientListener.second, /* isShutdown */ true);
}
return;
}
}
cancel(mListener, true /* isShutdown */);
}
@Override // from ServiceConnector.Impl
@@ -265,15 +280,18 @@ final class RemoteSpeechRecognitionService extends ServiceConnector.Impl<IRecogn
synchronized (mLock) {
if (!connected) {
if (mListener == null) {
if (mClients.isEmpty()) {
Slog.i(TAG, "Connection to speech recognition service lost, but no "
+ "#startListening has been invoked yet.");
return;
}
tryRespondWithError(mListener, SpeechRecognizer.ERROR_SERVER_DISCONNECTED);
resetStateLocked();
for (ClientState clientState : mClients.values()) {
tryRespondWithError(
clientState.mDelegatingListener.mRemoteListener,
SpeechRecognizer.ERROR_SERVER_DISCONNECTED);
removeClient(clientState.mDelegatingListener.mRemoteListener);
}
}
}
}
@@ -283,11 +301,18 @@ final class RemoteSpeechRecognitionService extends ServiceConnector.Impl<IRecogn
return PERMANENT_BOUND_TIMEOUT_MS;
}
private void resetStateLocked() {
mListener = null;
mDelegatingListener = null;
mSessionInProgress = false;
mRecordingInProgress = false;
private void removeClient(IRecognitionListener listener) {
synchronized (mLock) {
ClientState clientState = mClients.remove(listener.asBinder());
if (clientState != null) {
if (DEBUG) {
Slog.d(TAG, "Removed a client from the map with listener = "
+ listener.asBinder() + ".");
}
clientState.reset();
}
mClientListeners.removeIf(clientListener -> clientListener.second == listener);
}
}
private static void tryRespondWithError(IRecognitionListener listener, int errorCode) {
@@ -301,19 +326,35 @@ final class RemoteSpeechRecognitionService extends ServiceConnector.Impl<IRecogn
}
} catch (RemoteException e) {
Slog.w(TAG,
String.format("Failed to respond with an error %d to the client", errorCode),
e);
TextUtils.formatSimple("Failed to respond with an error %d to the client",
errorCode), e);
}
}
boolean hasActiveSessions() {
synchronized (mLock) {
return !mClients.isEmpty();
}
}
void associateClientWithActiveListener(IBinder clientToken, IRecognitionListener listener) {
synchronized (mLock) {
if (mClients.containsKey(listener.asBinder())) {
mClientListeners.add(new Pair<>(clientToken, listener));
}
}
}
private static class DelegatingListener extends IRecognitionListener.Stub {
private final IRecognitionListener mRemoteListener;
private final Runnable mOnSessionComplete;
private final Runnable mOnSessionSuccess;
private final Runnable mOnSessionFailure;
DelegatingListener(IRecognitionListener listener, Runnable onSessionComplete) {
DelegatingListener(IRecognitionListener listener,
Runnable onSessionSuccess, Runnable onSessionFailure) {
mRemoteListener = listener;
mOnSessionComplete = onSessionComplete;
mOnSessionSuccess = onSessionSuccess;
mOnSessionFailure = onSessionFailure;
}
@Override
@@ -344,18 +385,18 @@ final class RemoteSpeechRecognitionService extends ServiceConnector.Impl<IRecogn
@Override
public void onError(int error) throws RemoteException {
if (DEBUG) {
Slog.i(TAG, String.format("Error %d during recognition session", error));
Slog.i(TAG, TextUtils.formatSimple("Error %d during recognition session.", error));
}
mOnSessionComplete.run();
mOnSessionFailure.run();
mRemoteListener.onError(error);
}
@Override
public void onResults(Bundle results) throws RemoteException {
if (DEBUG) {
Slog.i(TAG, "#onResults invoked for a recognition session");
Slog.i(TAG, "#onResults invoked for a recognition session.");
}
mOnSessionComplete.run();
mOnSessionSuccess.run();
mRemoteListener.onResults(results);
}
@@ -372,9 +413,9 @@ final class RemoteSpeechRecognitionService extends ServiceConnector.Impl<IRecogn
@Override
public void onEndOfSegmentedSession() throws RemoteException {
if (DEBUG) {
Slog.i(TAG, "#onEndOfSegmentedSession invoked for a recognition session");
Slog.i(TAG, "#onEndOfSegmentedSession invoked for a recognition session.");
}
mOnSessionComplete.run();
mOnSessionSuccess.run();
mRemoteListener.onEndOfSegmentedSession();
}
@@ -383,4 +424,35 @@ final class RemoteSpeechRecognitionService extends ServiceConnector.Impl<IRecogn
mRemoteListener.onEvent(eventType, params);
}
}
/**
* Data class holding info about a connected client:
* <ul>
* <li> {@link ClientState#mDelegatingListener}
* - object holding callbacks to be invoked after the session is complete;
* <li> {@link ClientState#mRecordingInProgress}
* - flag denoting if the client is currently recording.
*/
static class ClientState {
DelegatingListener mDelegatingListener;
boolean mRecordingInProgress;
ClientState(DelegatingListener delegatingListener, boolean recordingInProgress) {
mDelegatingListener = delegatingListener;
mRecordingInProgress = recordingInProgress;
}
ClientState(DelegatingListener delegatingListener) {
this(delegatingListener, true);
}
ClientState() {
this(null, true);
}
void reset() {
mDelegatingListener = null;
mRecordingInProgress = false;
}
}
}

23
services/core/java/com/android/server/speech/SpeechRecognitionManagerServiceImpl.java
View File

@@ -51,7 +51,7 @@ import java.util.Set;
final class SpeechRecognitionManagerServiceImpl extends
AbstractPerUserSystemService<SpeechRecognitionManagerServiceImpl,
SpeechRecognitionManagerService> {
SpeechRecognitionManagerService> {
private static final String TAG = SpeechRecognitionManagerServiceImpl.class.getSimpleName();
private static final int MAX_CONCURRENT_CONNECTIONS_BY_CLIENT = 10;
@@ -127,13 +127,14 @@ final class SpeechRecognitionManagerServiceImpl extends
}
IBinder.DeathRecipient deathRecipient =
() -> handleClientDeath(creatorCallingUid, service, true /* invoke #cancel */);
() -> handleClientDeath(
clientToken, creatorCallingUid, service, true /* invoke #cancel */);
try {
clientToken.linkToDeath(deathRecipient, 0);
} catch (RemoteException e) {
// RemoteException == binder already died, schedule disconnect anyway.
handleClientDeath(creatorCallingUid, service, true /* invoke #cancel */);
handleClientDeath(clientToken, creatorCallingUid, service, true /* invoke #cancel */);
return;
}
@@ -146,7 +147,7 @@ final class SpeechRecognitionManagerServiceImpl extends
Intent recognizerIntent,
IRecognitionListener listener,
@NonNull AttributionSource attributionSource)
throws RemoteException {
throws RemoteException {
attributionSource.enforceCallingUid();
if (!attributionSource.isTrusted(mMaster.getContext())) {
attributionSource = mMaster.getContext()
@@ -154,6 +155,7 @@ final class SpeechRecognitionManagerServiceImpl extends
.registerAttributionSource(attributionSource);
}
service.startListening(recognizerIntent, listener, attributionSource);
service.associateClientWithActiveListener(clientToken, listener);
}
@Override
@@ -166,11 +168,11 @@ final class SpeechRecognitionManagerServiceImpl extends
public void cancel(
IRecognitionListener listener,
boolean isShutdown) throws RemoteException {
service.cancel(listener, isShutdown);
if (isShutdown) {
handleClientDeath(
clientToken,
creatorCallingUid,
service,
false /* invoke #cancel */);
@@ -201,12 +203,16 @@ final class SpeechRecognitionManagerServiceImpl extends
}
private void handleClientDeath(
int callingUid,
IBinder clientToken, int callingUid,
RemoteSpeechRecognitionService service, boolean invokeCancel) {
if (invokeCancel) {
service.shutdown();
service.shutdown(clientToken);
}
synchronized (mLock) {
if (!service.hasActiveSessions()) {
removeService(callingUid, service);
}
}
removeService(callingUid, service);
}
@GuardedBy("mLock")
@@ -245,7 +251,6 @@ final class SpeechRecognitionManagerServiceImpl extends
service.getServiceComponentName().equals(serviceComponent))
.findFirst();
if (existingService.isPresent()) {
if (mMaster.debug) {
Slog.i(TAG, "Reused existing connection to " + serviceComponent);
}