diff --git a/core/api/current.txt b/core/api/current.txt index d54290b70a728..96bfd7b9207e7 100644 --- a/core/api/current.txt +++ b/core/api/current.txt @@ -41406,6 +41406,13 @@ package android.speech { field @NonNull public static final android.os.Parcelable.Creator CREATOR; } + public interface ModelDownloadListener { + method public void onError(int); + method public void onProgress(int); + method public void onScheduled(); + method public void onSuccess(); + } + public interface RecognitionListener { method public void onBeginningOfSpeech(); method public void onBufferReceived(byte[]); @@ -41447,6 +41454,7 @@ package android.speech { public abstract class RecognitionService extends android.app.Service { ctor public RecognitionService(); + method public void clearModelDownloadListener(@NonNull android.content.Intent, @NonNull android.content.AttributionSource); method public int getMaxConcurrentSessionsCount(); method public final android.os.IBinder onBind(android.content.Intent); method protected abstract void onCancel(android.speech.RecognitionService.Callback); @@ -41456,6 +41464,7 @@ package android.speech { method protected abstract void onStopListening(android.speech.RecognitionService.Callback); method public void onTriggerModelDownload(@NonNull android.content.Intent); method public void onTriggerModelDownload(@NonNull android.content.Intent, @NonNull android.content.AttributionSource); + method public void setModelDownloadListener(@NonNull android.content.Intent, @NonNull android.content.AttributionSource, @NonNull android.speech.ModelDownloadListener); field public static final String SERVICE_INTERFACE = "android.speech.RecognitionService"; field public static final String SERVICE_META_DATA = "android.speech"; } @@ -41572,12 +41581,14 @@ package android.speech { public class SpeechRecognizer { method @MainThread public void cancel(); method public void checkRecognitionSupport(@NonNull android.content.Intent, @NonNull java.util.concurrent.Executor, @NonNull android.speech.RecognitionSupportCallback); + method public void clearModelDownloadListener(@NonNull android.content.Intent); method @MainThread @NonNull public static android.speech.SpeechRecognizer createOnDeviceSpeechRecognizer(@NonNull android.content.Context); method @MainThread public static android.speech.SpeechRecognizer createSpeechRecognizer(android.content.Context); method @MainThread public static android.speech.SpeechRecognizer createSpeechRecognizer(android.content.Context, android.content.ComponentName); method public void destroy(); method public static boolean isOnDeviceRecognitionAvailable(@NonNull android.content.Context); method public static boolean isRecognitionAvailable(@NonNull android.content.Context); + method public void setModelDownloadListener(@NonNull android.content.Intent, @NonNull java.util.concurrent.Executor, @NonNull android.speech.ModelDownloadListener); method @MainThread public void setRecognitionListener(android.speech.RecognitionListener); method @MainThread public void startListening(android.content.Intent); method @MainThread public void stopListening(); @@ -41585,6 +41596,7 @@ package android.speech { field public static final String CONFIDENCE_SCORES = "confidence_scores"; field public static final int ERROR_AUDIO = 3; // 0x3 field public static final int ERROR_CANNOT_CHECK_SUPPORT = 14; // 0xe + field public static final int ERROR_CANNOT_LISTEN_TO_DOWNLOAD_EVENTS = 15; // 0xf field public static final int ERROR_CLIENT = 5; // 0x5 field public static final int ERROR_INSUFFICIENT_PERMISSIONS = 9; // 0x9 field public static final int ERROR_LANGUAGE_NOT_SUPPORTED = 12; // 0xc diff --git a/core/java/android/speech/IModelDownloadListener.aidl b/core/java/android/speech/IModelDownloadListener.aidl new file mode 100644 index 0000000000000..9bbf28b5bfc7f --- /dev/null +++ b/core/java/android/speech/IModelDownloadListener.aidl @@ -0,0 +1,49 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.speech; + +/** + * Listener for model download events. It makes RecognitionService let callers know about the + * progress of model download requests. + * + * {@hide} + */ +oneway interface IModelDownloadListener { + /** + * Called by RecognitionService when there's an update on the download progress. + */ + void onProgress(in int completedPercent); + + /** + * Called when RecognitionService completed the download and it can now be used to satisfy + * recognition requests. + */ + void onSuccess(); + + /** + * Called when RecognitionService scheduled the download but won't satisfy it immediately. + * There will be no further updates on this callback. + */ + void onScheduled(); + + /** + * A network or scheduling error occurred. + * + * @param error code is defined in {@link SpeechRecognizer} + */ + void onError(in int error); +} diff --git a/core/java/android/speech/IRecognitionService.aidl b/core/java/android/speech/IRecognitionService.aidl index cc64c45c7606b..3134dcd508bb1 100644 --- a/core/java/android/speech/IRecognitionService.aidl +++ b/core/java/android/speech/IRecognitionService.aidl @@ -19,6 +19,7 @@ package android.speech; import android.os.Bundle; import android.content.AttributionSource; import android.content.Intent; +import android.speech.IModelDownloadListener; import android.speech.IRecognitionListener; import android.speech.IRecognitionSupportCallback; @@ -76,6 +77,24 @@ oneway interface IRecognitionService { * Requests RecognitionService to download the support for the given recognizerIntent. For more * information see {@link #checkRecognitionSupport}, {@link #startListening} and * {@link RecognizerIntent}. + * + * Progress can be monitord by calling {@link #setModelDownloadListener} before a trigger. */ void triggerModelDownload(in Intent recognizerIntent, in AttributionSource attributionSource); + + /** + * Sets listener to received download progress updates. Clients still have to call + * {@link #triggerModelDownload} to trigger a model download. + */ + void setModelDownloadListener( + in Intent recognizerIntent, + in AttributionSource attributionSource, + in IModelDownloadListener listener); + + /** + * Clears the listener for model download events attached to a recognitionIntent if any. + */ + void clearModelDownloadListener( + in Intent recognizerIntent, + in AttributionSource attributionSource); } diff --git a/core/java/android/speech/ModelDownloadListener.java b/core/java/android/speech/ModelDownloadListener.java new file mode 100644 index 0000000000000..6c24399acb777 --- /dev/null +++ b/core/java/android/speech/ModelDownloadListener.java @@ -0,0 +1,49 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.speech; + +/** + * Listener for model download events. It makes {@link RecognitionService} let callers know about + * the progress of model download for a single recognition request. + */ +public interface ModelDownloadListener { + /** + * Called by {@link RecognitionService} when there's an update on the download progress. + * + *

RecognitionService will call this zero or more times during the download.

+ */ + void onProgress(int completedPercent); + + /** + * Called when {@link RecognitionService} completed the download and it can now be used to + * satisfy recognition requests. + */ + void onSuccess(); + + /** + * Called when {@link RecognitionService} scheduled the download but won't satisfy it + * immediately. There will be no further updates on this listener. + */ + void onScheduled(); + + /** + * A network or scheduling error occurred. + * + * @param error code is defined in {@link SpeechRecognizer} + */ + void onError(@SpeechRecognizer.RecognitionError int error); +} diff --git a/core/java/android/speech/RecognitionService.java b/core/java/android/speech/RecognitionService.java index 0b0b3b56acd35..d1d80a4b96507 100644 --- a/core/java/android/speech/RecognitionService.java +++ b/core/java/android/speech/RecognitionService.java @@ -36,6 +36,7 @@ import android.os.IBinder; import android.os.Looper; import android.os.Message; import android.os.RemoteException; +import android.text.TextUtils; import android.util.Log; import android.util.Pair; @@ -92,6 +93,10 @@ public abstract class RecognitionService extends Service { private static final int MSG_TRIGGER_MODEL_DOWNLOAD = 6; + private static final int MSG_SET_MODEL_DOWNLOAD_LISTENER = 7; + + private static final int MSG_CLEAR_MODEL_DOWNLOAD_LISTENER = 8; + private final Handler mHandler = new Handler() { @Override public void handleMessage(Message msg) { @@ -119,6 +124,18 @@ public abstract class RecognitionService extends Service { (Pair) msg.obj; dispatchTriggerModelDownload(params.first, params.second); break; + case MSG_SET_MODEL_DOWNLOAD_LISTENER: + ModelDownloadListenerArgs dListenerArgs = (ModelDownloadListenerArgs) msg.obj; + dispatchSetModelDownloadListener( + dListenerArgs.mIntent, + dListenerArgs.mListener, + dListenerArgs.mAttributionSource); + break; + case MSG_CLEAR_MODEL_DOWNLOAD_LISTENER: + Pair clearDlPair = + (Pair) msg.obj; + dispatchClearModelDownloadListener(clearDlPair.first, clearDlPair.second); + break; } } }; @@ -226,6 +243,57 @@ public abstract class RecognitionService extends Service { RecognitionService.this.onTriggerModelDownload(intent, attributionSource); } + private void dispatchSetModelDownloadListener( + Intent intent, + IModelDownloadListener listener, + AttributionSource attributionSource) { + RecognitionService.this.setModelDownloadListener( + intent, + attributionSource, + new ModelDownloadListener() { + @Override + public void onProgress(int completedPercent) { + try { + listener.onProgress(completedPercent); + } catch (RemoteException e) { + throw e.rethrowFromSystemServer(); + } + } + + @Override + public void onSuccess() { + try { + listener.onSuccess(); + } catch (RemoteException e) { + throw e.rethrowFromSystemServer(); + } + } + + @Override + public void onScheduled() { + try { + listener.onScheduled(); + } catch (RemoteException e) { + throw e.rethrowFromSystemServer(); + } + } + + @Override + public void onError(int error) { + try { + listener.onError(error); + } catch (RemoteException e) { + throw e.rethrowFromSystemServer(); + } + } + }); + } + + private void dispatchClearModelDownloadListener( + Intent intent, AttributionSource attributionSource) { + RecognitionService.this.clearModelDownloadListener(intent, attributionSource); + } + private static class StartListeningArgs { public final Intent mIntent; @@ -255,6 +323,20 @@ public abstract class RecognitionService extends Service { } } + private static class ModelDownloadListenerArgs { + final Intent mIntent; + final IModelDownloadListener mListener; + final AttributionSource mAttributionSource; + + private ModelDownloadListenerArgs(Intent intent, + IModelDownloadListener listener, + AttributionSource attributionSource) { + mIntent = intent; + this.mListener = listener; + mAttributionSource = attributionSource; + } + } + /** * Notifies the service that it should start listening for speech. * @@ -360,6 +442,41 @@ public abstract class RecognitionService extends Service { onTriggerModelDownload(recognizerIntent); } + /** + * Sets a {@link ModelDownloadListener} to receive progress updates after + * {@link #onTriggerModelDownload} calls. + * + * @param recognizerIntent the request to monitor model download progress for. + * @param modelDownloadListener the listener to keep updated. + */ + public void setModelDownloadListener( + @NonNull Intent recognizerIntent, + @NonNull AttributionSource attributionSource, + @NonNull ModelDownloadListener modelDownloadListener) { + if (DBG) { + Log.i(TAG, TextUtils.formatSimple( + "#setModelDownloadListener [%s] [%s]", + recognizerIntent, + modelDownloadListener)); + } + modelDownloadListener.onError(SpeechRecognizer.ERROR_CANNOT_LISTEN_TO_DOWNLOAD_EVENTS); + } + + /** + * Clears the {@link ModelDownloadListener} set to receive progress updates for the given + * {@code recognizerIntent}, if any. + * + * @param recognizerIntent the request to monitor model download progress for. + */ + public void clearModelDownloadListener( + @NonNull Intent recognizerIntent, + @NonNull AttributionSource attributionSource) { + if (DBG) { + Log.i(TAG, TextUtils.formatSimple( + "#clearModelDownloadListener [%s]", recognizerIntent)); + } + } + @Override @SuppressLint("MissingNullability") public Context createContext(@NonNull ContextParams contextParams) { @@ -675,6 +792,34 @@ public abstract class RecognitionService extends Service { } } + @Override + public void setModelDownloadListener( + Intent recognizerIntent, + AttributionSource attributionSource, + IModelDownloadListener listener) throws RemoteException { + final RecognitionService service = mServiceRef.get(); + if (service != null) { + service.mHandler.sendMessage( + Message.obtain(service.mHandler, MSG_SET_MODEL_DOWNLOAD_LISTENER, + new ModelDownloadListenerArgs( + recognizerIntent, + listener, + attributionSource))); + } + } + + @Override + public void clearModelDownloadListener( + Intent recognizerIntent, + AttributionSource attributionSource) throws RemoteException { + final RecognitionService service = mServiceRef.get(); + if (service != null) { + service.mHandler.sendMessage( + Message.obtain(service.mHandler, MSG_CLEAR_MODEL_DOWNLOAD_LISTENER, + Pair.create(recognizerIntent, attributionSource))); + } + } + public void clearReference() { mServiceRef.clear(); } diff --git a/core/java/android/speech/SpeechRecognizer.java b/core/java/android/speech/SpeechRecognizer.java index 9c46e55d19968..dce72f598cfc5 100644 --- a/core/java/android/speech/SpeechRecognizer.java +++ b/core/java/android/speech/SpeechRecognizer.java @@ -145,6 +145,7 @@ public class SpeechRecognizer { ERROR_LANGUAGE_NOT_SUPPORTED, ERROR_LANGUAGE_UNAVAILABLE, ERROR_CANNOT_CHECK_SUPPORT, + ERROR_CANNOT_LISTEN_TO_DOWNLOAD_EVENTS, }) public @interface RecognitionError {} @@ -190,6 +191,9 @@ public class SpeechRecognizer { /** The service does not allow to check for support. */ public static final int ERROR_CANNOT_CHECK_SUPPORT = 14; + /** The service does not support listening to model downloads events. */ + public static final int ERROR_CANNOT_LISTEN_TO_DOWNLOAD_EVENTS = 15; + /** action codes */ private static final int MSG_START = 1; private static final int MSG_STOP = 2; @@ -198,6 +202,8 @@ public class SpeechRecognizer { private static final int MSG_SET_TEMPORARY_ON_DEVICE_COMPONENT = 5; private static final int MSG_CHECK_RECOGNITION_SUPPORT = 6; private static final int MSG_TRIGGER_MODEL_DOWNLOAD = 7; + private static final int MSG_SET_MODEL_DOWNLOAD_LISTENER = 8; + private static final int MSG_CLEAR_MODEL_DOWNLOAD_LISTENER = 9; /** The actual RecognitionService endpoint */ private IRecognitionService mService; @@ -242,6 +248,17 @@ public class SpeechRecognizer { case MSG_TRIGGER_MODEL_DOWNLOAD: handleTriggerModelDownload((Intent) msg.obj); break; + case MSG_SET_MODEL_DOWNLOAD_LISTENER: + ModelDownloadListenerArgs modelDownloadListenerArgs = + (ModelDownloadListenerArgs) msg.obj; + handleSetModelDownloadListener( + modelDownloadListenerArgs.mIntent, + modelDownloadListenerArgs.mExecutor, + modelDownloadListenerArgs.mModelDownloadListener); + break; + case MSG_CLEAR_MODEL_DOWNLOAD_LISTENER: + handleClearModelDownloadListener((Intent) msg.obj); + break; } } }; @@ -545,6 +562,10 @@ public class SpeechRecognizer { * user interaction to approve the download. Callers can verify the status of the request via * {@link #checkRecognitionSupport(Intent, Executor, RecognitionSupportCallback)}. * + *

Listeners set via + * {@link #setModelDownloadListener(Intent, Executor, ModelDownloadListener)} will receive + * updates about this download request.

+ * * @param recognizerIntent contains parameters for the recognition to be performed. The intent * may also contain optional extras, see {@link RecognizerIntent}. */ @@ -563,6 +584,54 @@ public class SpeechRecognizer { putMessage(Message.obtain(mHandler, MSG_TRIGGER_MODEL_DOWNLOAD, recognizerIntent)); } + /** + * Sets a listener to model download updates. Clients will have to call this method before + * {@link #triggerModelDownload(Intent)}. + * + * @param recognizerIntent the request to monitor support for. + * @param listener on which to receive updates about the model download request. + */ + public void setModelDownloadListener( + @NonNull Intent recognizerIntent, + @NonNull @CallbackExecutor Executor executor, + @NonNull ModelDownloadListener listener) { + Objects.requireNonNull(recognizerIntent, "intent must not be null"); + if (DBG) { + Slog.i(TAG, "#setModelDownloadListener called"); + if (mService == null) { + Slog.i(TAG, "Connection is not established yet"); + } + } + if (mService == null) { + // First time connection: first establish a connection, then dispatch. + connectToSystemService(); + } + putMessage(Message.obtain( + mHandler, MSG_SET_MODEL_DOWNLOAD_LISTENER, + new ModelDownloadListenerArgs(recognizerIntent, executor, listener))); + } + + /** + * Clears the listener for model download updates if any. + * + * @param recognizerIntent the request to monitor support for. + */ + public void clearModelDownloadListener(@NonNull Intent recognizerIntent) { + Objects.requireNonNull(recognizerIntent, "intent must not be null"); + if (DBG) { + Slog.i(TAG, "#clearModelDownloadListener called"); + if (mService == null) { + Slog.i(TAG, "Connection is not established yet"); + } + } + if (mService == null) { + // First time connection: first establish a connection, then dispatch. + connectToSystemService(); + } + putMessage(Message.obtain( + mHandler, MSG_CLEAR_MODEL_DOWNLOAD_LISTENER, recognizerIntent)); + } + /** * Sets a temporary component to power on-device speech recognizer. * @@ -684,6 +753,42 @@ public class SpeechRecognizer { } } + private void handleSetModelDownloadListener( + Intent recognizerIntent, + Executor callbackExecutor, + @Nullable ModelDownloadListener modelDownloadListener) { + if (!maybeInitializeManagerService()) { + return; + } + try { + InternalModelDownloadListener listener = + modelDownloadListener == null + ? null + : new InternalModelDownloadListener( + callbackExecutor, + modelDownloadListener); + mService.setModelDownloadListener( + recognizerIntent, mContext.getAttributionSource(), listener); + if (DBG) Log.d(TAG, "setModelDownloadListener()"); + } catch (final RemoteException e) { + Log.e(TAG, "setModelDownloadListener() failed", e); + callbackExecutor.execute(() -> modelDownloadListener.onError(ERROR_CLIENT)); + } + } + + private void handleClearModelDownloadListener(Intent recognizerIntent) { + if (!maybeInitializeManagerService()) { + return; + } + try { + mService.clearModelDownloadListener( + recognizerIntent, mContext.getAttributionSource()); + if (DBG) Log.d(TAG, "clearModelDownloadListener()"); + } catch (final RemoteException e) { + Log.e(TAG, "clearModelDownloadListener() failed", e); + } + } + private boolean checkOpenConnection() { if (mService != null) { return true; @@ -827,6 +932,19 @@ public class SpeechRecognizer { } } + private static class ModelDownloadListenerArgs { + final Intent mIntent; + final Executor mExecutor; + final ModelDownloadListener mModelDownloadListener; + + private ModelDownloadListenerArgs(Intent intent, Executor executor, + ModelDownloadListener modelDownloadListener) { + mIntent = intent; + mExecutor = executor; + mModelDownloadListener = modelDownloadListener; + } + } + /** * Internal wrapper of IRecognitionListener which will propagate the results to * RecognitionListener @@ -955,4 +1073,36 @@ public class SpeechRecognizer { mExecutor.execute(() -> mCallback.onError(errorCode)); } } + + private static class InternalModelDownloadListener extends IModelDownloadListener.Stub { + private final Executor mExecutor; + private final ModelDownloadListener mModelDownloadListener; + + private InternalModelDownloadListener( + Executor executor, + @NonNull ModelDownloadListener modelDownloadListener) { + mExecutor = executor; + mModelDownloadListener = modelDownloadListener; + } + + @Override + public void onProgress(int completedPercent) throws RemoteException { + mExecutor.execute(() -> mModelDownloadListener.onProgress(completedPercent)); + } + + @Override + public void onSuccess() throws RemoteException { + mExecutor.execute(() -> mModelDownloadListener.onSuccess()); + } + + @Override + public void onScheduled() throws RemoteException { + mExecutor.execute(() -> mModelDownloadListener.onScheduled()); + } + + @Override + public void onError(int error) throws RemoteException { + mExecutor.execute(() -> mModelDownloadListener.onError(error)); + } + } } diff --git a/services/core/java/com/android/server/speech/RemoteSpeechRecognitionService.java b/services/core/java/com/android/server/speech/RemoteSpeechRecognitionService.java index 4839c96a1b056..b04d7df39d036 100644 --- a/services/core/java/com/android/server/speech/RemoteSpeechRecognitionService.java +++ b/services/core/java/com/android/server/speech/RemoteSpeechRecognitionService.java @@ -26,6 +26,7 @@ import android.content.Intent; import android.os.Bundle; import android.os.IBinder; import android.os.RemoteException; +import android.speech.IModelDownloadListener; import android.speech.IRecognitionListener; import android.speech.IRecognitionService; import android.speech.IRecognitionSupportCallback; @@ -257,6 +258,35 @@ final class RemoteSpeechRecognitionService extends ServiceConnector.Impl service.triggerModelDownload(recognizerIntent, attributionSource)); } + void setModelDownloadListener( + Intent recognizerIntent, + AttributionSource attributionSource, + IModelDownloadListener listener) { + if (!mConnected) { + try { + listener.onError(SpeechRecognizer.ERROR_SERVER_DISCONNECTED); + } catch (RemoteException e) { + Slog.w(TAG, "Failed to report the connection broke to the caller.", e); + e.printStackTrace(); + } + return; + } + + run(service -> + service.setModelDownloadListener(recognizerIntent, attributionSource, listener)); + } + + void clearModelDownloadListener( + Intent recognizerIntent, + AttributionSource attributionSource) { + if (!mConnected) { + return; + } + + run(service -> + service.clearModelDownloadListener(recognizerIntent, attributionSource)); + } + void shutdown(IBinder clientToken) { synchronized (mLock) { for (Pair clientListener : mClientListeners) { diff --git a/services/core/java/com/android/server/speech/SpeechRecognitionManagerServiceImpl.java b/services/core/java/com/android/server/speech/SpeechRecognitionManagerServiceImpl.java index 6aa600ad63cdc..bc73db18b379a 100644 --- a/services/core/java/com/android/server/speech/SpeechRecognitionManagerServiceImpl.java +++ b/services/core/java/com/android/server/speech/SpeechRecognitionManagerServiceImpl.java @@ -31,6 +31,7 @@ import android.os.IBinder; import android.os.Process; import android.os.RemoteException; import android.permission.PermissionManager; +import android.speech.IModelDownloadListener; import android.speech.IRecognitionListener; import android.speech.IRecognitionService; import android.speech.IRecognitionServiceManagerCallback; @@ -195,6 +196,22 @@ final class SpeechRecognitionManagerServiceImpl extends AttributionSource attributionSource) { service.triggerModelDownload(recognizerIntent, attributionSource); } + + @Override + public void setModelDownloadListener( + Intent recognizerIntent, + AttributionSource attributionSource, + IModelDownloadListener listener) throws RemoteException { + service.setModelDownloadListener( + recognizerIntent, attributionSource, listener); + } + + @Override + public void clearModelDownloadListener( + Intent recognizerIntent, + AttributionSource attributionSource) throws RemoteException { + service.clearModelDownloadListener(recognizerIntent, attributionSource); + } }); } catch (RemoteException e) { Slog.e(TAG, "Error creating a speech recognition session", e);