Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge "speech: Add progress listener for model downloads"

Browse Source
This commit is contained in:
Andrea Ambu
2023-02-15 08:53:13 +00:00
committed by Android (Google) Code Review
parent bec7c17eec a710049db5
commit 722601a838
8 changed files with 471 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
core/api/current.txt
View File

@@ -41372,6 +41372,13 @@ package android.speech {
field @NonNull public static final android.os.Parcelable.Creator<android.speech.AlternativeSpans> CREATOR;
}
public interface ModelDownloadListener {
method public void onError(int);
method public void onProgress(int);
method public void onScheduled();
method public void onSuccess();
}
public interface RecognitionListener {
method public void onBeginningOfSpeech();
method public void onBufferReceived(byte[]);
@@ -41413,6 +41420,7 @@ package android.speech {
public abstract class RecognitionService extends android.app.Service {
ctor public RecognitionService();
method public void clearModelDownloadListener(@NonNull android.content.Intent, @NonNull android.content.AttributionSource);
method public int getMaxConcurrentSessionsCount();
method public final android.os.IBinder onBind(android.content.Intent);
method protected abstract void onCancel(android.speech.RecognitionService.Callback);
@@ -41422,6 +41430,7 @@ package android.speech {
method protected abstract void onStopListening(android.speech.RecognitionService.Callback);
method public void onTriggerModelDownload(@NonNull android.content.Intent);
method public void onTriggerModelDownload(@NonNull android.content.Intent, @NonNull android.content.AttributionSource);
method public void setModelDownloadListener(@NonNull android.content.Intent, @NonNull android.content.AttributionSource, @NonNull android.speech.ModelDownloadListener);
field public static final String SERVICE_INTERFACE = "android.speech.RecognitionService";
field public static final String SERVICE_META_DATA = "android.speech";
}
@@ -41538,12 +41547,14 @@ package android.speech {
public class SpeechRecognizer {
method @MainThread public void cancel();
method public void checkRecognitionSupport(@NonNull android.content.Intent, @NonNull java.util.concurrent.Executor, @NonNull android.speech.RecognitionSupportCallback);
method public void clearModelDownloadListener(@NonNull android.content.Intent);
method @MainThread @NonNull public static android.speech.SpeechRecognizer createOnDeviceSpeechRecognizer(@NonNull android.content.Context);
method @MainThread public static android.speech.SpeechRecognizer createSpeechRecognizer(android.content.Context);
method @MainThread public static android.speech.SpeechRecognizer createSpeechRecognizer(android.content.Context, android.content.ComponentName);
method public void destroy();
method public static boolean isOnDeviceRecognitionAvailable(@NonNull android.content.Context);
method public static boolean isRecognitionAvailable(@NonNull android.content.Context);
method public void setModelDownloadListener(@NonNull android.content.Intent, @NonNull java.util.concurrent.Executor, @NonNull android.speech.ModelDownloadListener);
method @MainThread public void setRecognitionListener(android.speech.RecognitionListener);
method @MainThread public void startListening(android.content.Intent);
method @MainThread public void stopListening();
@@ -41551,6 +41562,7 @@ package android.speech {
field public static final String CONFIDENCE_SCORES = "confidence_scores";
field public static final int ERROR_AUDIO = 3; // 0x3
field public static final int ERROR_CANNOT_CHECK_SUPPORT = 14; // 0xe
field public static final int ERROR_CANNOT_LISTEN_TO_DOWNLOAD_EVENTS = 15; // 0xf
field public static final int ERROR_CLIENT = 5; // 0x5
field public static final int ERROR_INSUFFICIENT_PERMISSIONS = 9; // 0x9
field public static final int ERROR_LANGUAGE_NOT_SUPPORTED = 12; // 0xc

49
core/java/android/speech/IModelDownloadListener.aidl Normal file
View File

@@ -0,0 +1,49 @@
/*
* Copyright (C) 2023 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package android.speech;
/**
* Listener for model download events. It makes RecognitionService let callers know about the
* progress of model download requests.
*
* {@hide}
*/
oneway interface IModelDownloadListener {
/**
* Called by RecognitionService when there's an update on the download progress.
*/
void onProgress(in int completedPercent);
/**
* Called when RecognitionService completed the download and it can now be used to satisfy
* recognition requests.
*/
void onSuccess();
/**
* Called when RecognitionService scheduled the download but won't satisfy it immediately.
* There will be no further updates on this callback.
*/
void onScheduled();
/**
* A network or scheduling error occurred.
*
* @param error code is defined in {@link SpeechRecognizer}
*/
void onError(in int error);
}

19
core/java/android/speech/IRecognitionService.aidl
View File

@@ -19,6 +19,7 @@ package android.speech;
import android.os.Bundle;
import android.content.AttributionSource;
import android.content.Intent;
import android.speech.IModelDownloadListener;
import android.speech.IRecognitionListener;
import android.speech.IRecognitionSupportCallback;
@@ -76,6 +77,24 @@ oneway interface IRecognitionService {
* Requests RecognitionService to download the support for the given recognizerIntent. For more
* information see {@link #checkRecognitionSupport}, {@link #startListening} and
* {@link RecognizerIntent}.
*
* Progress can be monitord by calling {@link #setModelDownloadListener} before a trigger.
*/
void triggerModelDownload(in Intent recognizerIntent, in AttributionSource attributionSource);
/**
* Sets listener to received download progress updates. Clients still have to call
* {@link #triggerModelDownload} to trigger a model download.
*/
void setModelDownloadListener(
in Intent recognizerIntent,
in AttributionSource attributionSource,
in IModelDownloadListener listener);
/**
* Clears the listener for model download events attached to a recognitionIntent if any.
*/
void clearModelDownloadListener(
in Intent recognizerIntent,
in AttributionSource attributionSource);
}

49
core/java/android/speech/ModelDownloadListener.java Normal file
View File

@@ -0,0 +1,49 @@
/*
* Copyright (C) 2023 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package android.speech;
/**
* Listener for model download events. It makes {@link RecognitionService} let callers know about
* the progress of model download for a single recognition request.
*/
public interface ModelDownloadListener {
/**
* Called by {@link RecognitionService} when there's an update on the download progress.
*
* <p>RecognitionService will call this zero or more times during the download.</p>
*/
void onProgress(int completedPercent);
/**
* Called when {@link RecognitionService} completed the download and it can now be used to
* satisfy recognition requests.
*/
void onSuccess();
/**
* Called when {@link RecognitionService} scheduled the download but won't satisfy it
* immediately. There will be no further updates on this listener.
*/
void onScheduled();
/**
* A network or scheduling error occurred.
*
* @param error code is defined in {@link SpeechRecognizer}
*/
void onError(@SpeechRecognizer.RecognitionError int error);
}

145
core/java/android/speech/RecognitionService.java
View File

@@ -36,6 +36,7 @@ import android.os.IBinder;
import android.os.Looper;
import android.os.Message;
import android.os.RemoteException;
import android.text.TextUtils;
import android.util.Log;
import android.util.Pair;
@@ -92,6 +93,10 @@ public abstract class RecognitionService extends Service {
private static final int MSG_TRIGGER_MODEL_DOWNLOAD = 6;
private static final int MSG_SET_MODEL_DOWNLOAD_LISTENER = 7;
private static final int MSG_CLEAR_MODEL_DOWNLOAD_LISTENER = 8;
private final Handler mHandler = new Handler() {
@Override
public void handleMessage(Message msg) {
@@ -119,6 +124,18 @@ public abstract class RecognitionService extends Service {
(Pair<Intent, AttributionSource>) msg.obj;
dispatchTriggerModelDownload(params.first, params.second);
break;
case MSG_SET_MODEL_DOWNLOAD_LISTENER:
ModelDownloadListenerArgs dListenerArgs = (ModelDownloadListenerArgs) msg.obj;
dispatchSetModelDownloadListener(
dListenerArgs.mIntent,
dListenerArgs.mListener,
dListenerArgs.mAttributionSource);
break;
case MSG_CLEAR_MODEL_DOWNLOAD_LISTENER:
Pair<Intent, AttributionSource> clearDlPair =
(Pair<Intent, AttributionSource>) msg.obj;
dispatchClearModelDownloadListener(clearDlPair.first, clearDlPair.second);
break;
}
}
};
@@ -226,6 +243,57 @@ public abstract class RecognitionService extends Service {
RecognitionService.this.onTriggerModelDownload(intent, attributionSource);
}
private void dispatchSetModelDownloadListener(
Intent intent,
IModelDownloadListener listener,
AttributionSource attributionSource) {
RecognitionService.this.setModelDownloadListener(
intent,
attributionSource,
new ModelDownloadListener() {
@Override
public void onProgress(int completedPercent) {
try {
listener.onProgress(completedPercent);
} catch (RemoteException e) {
throw e.rethrowFromSystemServer();
}
}
@Override
public void onSuccess() {
try {
listener.onSuccess();
} catch (RemoteException e) {
throw e.rethrowFromSystemServer();
}
}
@Override
public void onScheduled() {
try {
listener.onScheduled();
} catch (RemoteException e) {
throw e.rethrowFromSystemServer();
}
}
@Override
public void onError(int error) {
try {
listener.onError(error);
} catch (RemoteException e) {
throw e.rethrowFromSystemServer();
}
}
});
}
private void dispatchClearModelDownloadListener(
Intent intent, AttributionSource attributionSource) {
RecognitionService.this.clearModelDownloadListener(intent, attributionSource);
}
private static class StartListeningArgs {
public final Intent mIntent;
@@ -255,6 +323,20 @@ public abstract class RecognitionService extends Service {
}
}
private static class ModelDownloadListenerArgs {
final Intent mIntent;
final IModelDownloadListener mListener;
final AttributionSource mAttributionSource;
private ModelDownloadListenerArgs(Intent intent,
IModelDownloadListener listener,
AttributionSource attributionSource) {
mIntent = intent;
this.mListener = listener;
mAttributionSource = attributionSource;
}
}
/**
* Notifies the service that it should start listening for speech.
*
@@ -360,6 +442,41 @@ public abstract class RecognitionService extends Service {
onTriggerModelDownload(recognizerIntent);
}
/**
* Sets a {@link ModelDownloadListener} to receive progress updates after
* {@link #onTriggerModelDownload} calls.
*
* @param recognizerIntent the request to monitor model download progress for.
* @param modelDownloadListener the listener to keep updated.
*/
public void setModelDownloadListener(
@NonNull Intent recognizerIntent,
@NonNull AttributionSource attributionSource,
@NonNull ModelDownloadListener modelDownloadListener) {
if (DBG) {
Log.i(TAG, TextUtils.formatSimple(
"#setModelDownloadListener [%s] [%s]",
recognizerIntent,
modelDownloadListener));
}
modelDownloadListener.onError(SpeechRecognizer.ERROR_CANNOT_LISTEN_TO_DOWNLOAD_EVENTS);
}
/**
* Clears the {@link ModelDownloadListener} set to receive progress updates for the given
* {@code recognizerIntent}, if any.
*
* @param recognizerIntent the request to monitor model download progress for.
*/
public void clearModelDownloadListener(
@NonNull Intent recognizerIntent,
@NonNull AttributionSource attributionSource) {
if (DBG) {
Log.i(TAG, TextUtils.formatSimple(
"#clearModelDownloadListener [%s]", recognizerIntent));
}
}
@Override
@SuppressLint("MissingNullability")
public Context createContext(@NonNull ContextParams contextParams) {
@@ -675,6 +792,34 @@ public abstract class RecognitionService extends Service {
}
}
@Override
public void setModelDownloadListener(
Intent recognizerIntent,
AttributionSource attributionSource,
IModelDownloadListener listener) throws RemoteException {
final RecognitionService service = mServiceRef.get();
if (service != null) {
service.mHandler.sendMessage(
Message.obtain(service.mHandler, MSG_SET_MODEL_DOWNLOAD_LISTENER,
new ModelDownloadListenerArgs(
recognizerIntent,
listener,
attributionSource)));
}
}
@Override
public void clearModelDownloadListener(
Intent recognizerIntent,
AttributionSource attributionSource) throws RemoteException {
final RecognitionService service = mServiceRef.get();
if (service != null) {
service.mHandler.sendMessage(
Message.obtain(service.mHandler, MSG_CLEAR_MODEL_DOWNLOAD_LISTENER,
Pair.create(recognizerIntent, attributionSource)));
}
}
public void clearReference() {
mServiceRef.clear();
}

150
core/java/android/speech/SpeechRecognizer.java
View File

@@ -145,6 +145,7 @@ public class SpeechRecognizer {
ERROR_LANGUAGE_NOT_SUPPORTED,
ERROR_LANGUAGE_UNAVAILABLE,
ERROR_CANNOT_CHECK_SUPPORT,
ERROR_CANNOT_LISTEN_TO_DOWNLOAD_EVENTS,
})
public @interface RecognitionError {}
@@ -190,6 +191,9 @@ public class SpeechRecognizer {
/** The service does not allow to check for support. */
public static final int ERROR_CANNOT_CHECK_SUPPORT = 14;
/** The service does not support listening to model downloads events. */
public static final int ERROR_CANNOT_LISTEN_TO_DOWNLOAD_EVENTS = 15;
/** action codes */
private static final int MSG_START = 1;
private static final int MSG_STOP = 2;
@@ -198,6 +202,8 @@ public class SpeechRecognizer {
private static final int MSG_SET_TEMPORARY_ON_DEVICE_COMPONENT = 5;
private static final int MSG_CHECK_RECOGNITION_SUPPORT = 6;
private static final int MSG_TRIGGER_MODEL_DOWNLOAD = 7;
private static final int MSG_SET_MODEL_DOWNLOAD_LISTENER = 8;
private static final int MSG_CLEAR_MODEL_DOWNLOAD_LISTENER = 9;
/** The actual RecognitionService endpoint */
private IRecognitionService mService;
@@ -242,6 +248,17 @@ public class SpeechRecognizer {
case MSG_TRIGGER_MODEL_DOWNLOAD:
handleTriggerModelDownload((Intent) msg.obj);
break;
case MSG_SET_MODEL_DOWNLOAD_LISTENER:
ModelDownloadListenerArgs modelDownloadListenerArgs =
(ModelDownloadListenerArgs) msg.obj;
handleSetModelDownloadListener(
modelDownloadListenerArgs.mIntent,
modelDownloadListenerArgs.mExecutor,
modelDownloadListenerArgs.mModelDownloadListener);
break;
case MSG_CLEAR_MODEL_DOWNLOAD_LISTENER:
handleClearModelDownloadListener((Intent) msg.obj);
break;
}
}
};
@@ -545,6 +562,10 @@ public class SpeechRecognizer {
* user interaction to approve the download. Callers can verify the status of the request via
* {@link #checkRecognitionSupport(Intent, Executor, RecognitionSupportCallback)}.
*
* <p>Listeners set via
* {@link #setModelDownloadListener(Intent, Executor, ModelDownloadListener)} will receive
* updates about this download request.</p>
*
* @param recognizerIntent contains parameters for the recognition to be performed. The intent
* may also contain optional extras, see {@link RecognizerIntent}.
*/
@@ -563,6 +584,54 @@ public class SpeechRecognizer {
putMessage(Message.obtain(mHandler, MSG_TRIGGER_MODEL_DOWNLOAD, recognizerIntent));
}
/**
* Sets a listener to model download updates. Clients will have to call this method before
* {@link #triggerModelDownload(Intent)}.
*
* @param recognizerIntent the request to monitor support for.
* @param listener on which to receive updates about the model download request.
*/
public void setModelDownloadListener(
@NonNull Intent recognizerIntent,
@NonNull @CallbackExecutor Executor executor,
@NonNull ModelDownloadListener listener) {
Objects.requireNonNull(recognizerIntent, "intent must not be null");
if (DBG) {
Slog.i(TAG, "#setModelDownloadListener called");
if (mService == null) {
Slog.i(TAG, "Connection is not established yet");
}
}
if (mService == null) {
// First time connection: first establish a connection, then dispatch.
connectToSystemService();
}
putMessage(Message.obtain(
mHandler, MSG_SET_MODEL_DOWNLOAD_LISTENER,
new ModelDownloadListenerArgs(recognizerIntent, executor, listener)));
}
/**
* Clears the listener for model download updates if any.
*
* @param recognizerIntent the request to monitor support for.
*/
public void clearModelDownloadListener(@NonNull Intent recognizerIntent) {
Objects.requireNonNull(recognizerIntent, "intent must not be null");
if (DBG) {
Slog.i(TAG, "#clearModelDownloadListener called");
if (mService == null) {
Slog.i(TAG, "Connection is not established yet");
}
}
if (mService == null) {
// First time connection: first establish a connection, then dispatch.
connectToSystemService();
}
putMessage(Message.obtain(
mHandler, MSG_CLEAR_MODEL_DOWNLOAD_LISTENER, recognizerIntent));
}
/**
* Sets a temporary component to power on-device speech recognizer.
*
@@ -684,6 +753,42 @@ public class SpeechRecognizer {
}
}
private void handleSetModelDownloadListener(
Intent recognizerIntent,
Executor callbackExecutor,
@Nullable ModelDownloadListener modelDownloadListener) {
if (!maybeInitializeManagerService()) {
return;
}
try {
InternalModelDownloadListener listener =
modelDownloadListener == null
? null
: new InternalModelDownloadListener(
callbackExecutor,
modelDownloadListener);
mService.setModelDownloadListener(
recognizerIntent, mContext.getAttributionSource(), listener);
if (DBG) Log.d(TAG, "setModelDownloadListener()");
} catch (final RemoteException e) {
Log.e(TAG, "setModelDownloadListener() failed", e);
callbackExecutor.execute(() -> modelDownloadListener.onError(ERROR_CLIENT));
}
}
private void handleClearModelDownloadListener(Intent recognizerIntent) {
if (!maybeInitializeManagerService()) {
return;
}
try {
mService.clearModelDownloadListener(
recognizerIntent, mContext.getAttributionSource());
if (DBG) Log.d(TAG, "clearModelDownloadListener()");
} catch (final RemoteException e) {
Log.e(TAG, "clearModelDownloadListener() failed", e);
}
}
private boolean checkOpenConnection() {
if (mService != null) {
return true;
@@ -827,6 +932,19 @@ public class SpeechRecognizer {
}
}
private static class ModelDownloadListenerArgs {
final Intent mIntent;
final Executor mExecutor;
final ModelDownloadListener mModelDownloadListener;
private ModelDownloadListenerArgs(Intent intent, Executor executor,
ModelDownloadListener modelDownloadListener) {
mIntent = intent;
mExecutor = executor;
mModelDownloadListener = modelDownloadListener;
}
}
/**
* Internal wrapper of IRecognitionListener which will propagate the results to
* RecognitionListener
@@ -955,4 +1073,36 @@ public class SpeechRecognizer {
mExecutor.execute(() -> mCallback.onError(errorCode));
}
}
private static class InternalModelDownloadListener extends IModelDownloadListener.Stub {
private final Executor mExecutor;
private final ModelDownloadListener mModelDownloadListener;
private InternalModelDownloadListener(
Executor executor,
@NonNull ModelDownloadListener modelDownloadListener) {
mExecutor = executor;
mModelDownloadListener = modelDownloadListener;
}
@Override
public void onProgress(int completedPercent) throws RemoteException {
mExecutor.execute(() -> mModelDownloadListener.onProgress(completedPercent));
}
@Override
public void onSuccess() throws RemoteException {
mExecutor.execute(() -> mModelDownloadListener.onSuccess());
}
@Override
public void onScheduled() throws RemoteException {
mExecutor.execute(() -> mModelDownloadListener.onScheduled());
}
@Override
public void onError(int error) throws RemoteException {
mExecutor.execute(() -> mModelDownloadListener.onError(error));
}
}
}

30
services/core/java/com/android/server/speech/RemoteSpeechRecognitionService.java
View File

@@ -26,6 +26,7 @@ import android.content.Intent;
import android.os.Bundle;
import android.os.IBinder;
import android.os.RemoteException;
import android.speech.IModelDownloadListener;
import android.speech.IRecognitionListener;
import android.speech.IRecognitionService;
import android.speech.IRecognitionSupportCallback;
@@ -257,6 +258,35 @@ final class RemoteSpeechRecognitionService extends ServiceConnector.Impl<IRecogn
run(service -> service.triggerModelDownload(recognizerIntent, attributionSource));
}
void setModelDownloadListener(
Intent recognizerIntent,
AttributionSource attributionSource,
IModelDownloadListener listener) {
if (!mConnected) {
try {
listener.onError(SpeechRecognizer.ERROR_SERVER_DISCONNECTED);
} catch (RemoteException e) {
Slog.w(TAG, "Failed to report the connection broke to the caller.", e);
e.printStackTrace();
}
return;
}
run(service ->
service.setModelDownloadListener(recognizerIntent, attributionSource, listener));
}
void clearModelDownloadListener(
Intent recognizerIntent,
AttributionSource attributionSource) {
if (!mConnected) {
return;
}
run(service ->
service.clearModelDownloadListener(recognizerIntent, attributionSource));
}
void shutdown(IBinder clientToken) {
synchronized (mLock) {
for (Pair<IBinder, IRecognitionListener> clientListener : mClientListeners) {

17
services/core/java/com/android/server/speech/SpeechRecognitionManagerServiceImpl.java
View File

@@ -31,6 +31,7 @@ import android.os.IBinder;
import android.os.Process;
import android.os.RemoteException;
import android.permission.PermissionManager;
import android.speech.IModelDownloadListener;
import android.speech.IRecognitionListener;
import android.speech.IRecognitionService;
import android.speech.IRecognitionServiceManagerCallback;
@@ -195,6 +196,22 @@ final class SpeechRecognitionManagerServiceImpl extends
AttributionSource attributionSource) {
service.triggerModelDownload(recognizerIntent, attributionSource);
}
@Override
public void setModelDownloadListener(
Intent recognizerIntent,
AttributionSource attributionSource,
IModelDownloadListener listener) throws RemoteException {
service.setModelDownloadListener(
recognizerIntent, attributionSource, listener);
}
@Override
public void clearModelDownloadListener(
Intent recognizerIntent,
AttributionSource attributionSource) throws RemoteException {
service.clearModelDownloadListener(recognizerIntent, attributionSource);
}
});
} catch (RemoteException e) {
Slog.e(TAG, "Error creating a speech recognition session", e);