From c3c0a1ee7e16630ffae56253d008bfd9597d7c16 Mon Sep 17 00:00:00 2001 From: Eugenio Marchiori Date: Wed, 23 Dec 2020 10:44:55 +0000 Subject: [PATCH] Add System speech recognition service. Bug: 176578753 Test: local demo, CTS will follow Change-Id: Ia7766c506bf43a5db5b9719363da66aab263fd8d --- core/api/current.txt | 1 + core/java/android/content/Context.java | 14 +- .../speech/IRecognitionServiceManager.aidl | 28 +++ .../IRecognitionServiceManagerCallback.aidl | 29 ++++ .../android/speech/RecognitionService.java | 13 +- .../java/android/speech/SpeechRecognizer.java | 137 +++++++++++---- core/res/res/values/attrs.xml | 17 ++ core/res/res/values/config.xml | 6 + core/res/res/values/symbols.xml | 1 + services/Android.bp | 2 + .../java/com/android/server/speech/Android.bp | 13 ++ .../RemoteSpeechRecognitionService.java | 93 ++++++++++ .../SpeechRecognitionManagerService.java | 74 ++++++++ .../SpeechRecognitionManagerServiceImpl.java | 160 ++++++++++++++++++ .../java/com/android/server/SystemServer.java | 13 +- 15 files changed, 566 insertions(+), 35 deletions(-) create mode 100644 core/java/android/speech/IRecognitionServiceManager.aidl create mode 100644 core/java/android/speech/IRecognitionServiceManagerCallback.aidl create mode 100644 services/core/java/com/android/server/speech/Android.bp create mode 100644 services/core/java/com/android/server/speech/RemoteSpeechRecognitionService.java create mode 100644 services/core/java/com/android/server/speech/SpeechRecognitionManagerService.java create mode 100644 services/core/java/com/android/server/speech/SpeechRecognitionManagerServiceImpl.java diff --git a/core/api/current.txt b/core/api/current.txt index f49ce1fd48a5b..fcf0fe6bec913 100644 --- a/core/api/current.txt +++ b/core/api/current.txt @@ -38570,6 +38570,7 @@ package android.speech { public class SpeechRecognizer { method public void cancel(); + method @NonNull public static android.speech.SpeechRecognizer createOnDeviceSpeechRecognizer(@NonNull android.content.Context); method public static android.speech.SpeechRecognizer createSpeechRecognizer(android.content.Context); method public static android.speech.SpeechRecognizer createSpeechRecognizer(android.content.Context, android.content.ComponentName); method public void destroy(); diff --git a/core/java/android/content/Context.java b/core/java/android/content/Context.java index c5206d77ea502..0c50446e0a4e7 100644 --- a/core/java/android/content/Context.java +++ b/core/java/android/content/Context.java @@ -64,7 +64,6 @@ import android.os.HandlerExecutor; import android.os.IBinder; import android.os.Looper; import android.os.StatFs; -import android.os.StrictMode; import android.os.UserHandle; import android.os.UserManager; import android.os.storage.StorageManager; @@ -3575,6 +3574,7 @@ public abstract class Context { LIGHTS_SERVICE, //@hide: PEOPLE_SERVICE, //@hide: DEVICE_STATE_SERVICE, + //@hide: SPEECH_RECOGNITION_SERVICE, UWB_SERVICE, MEDIA_METRICS_SERVICE, }) @@ -5409,6 +5409,14 @@ public abstract class Context { */ public static final String MEDIA_METRICS_SERVICE = "media_metrics"; + /** + * Use with {@link #getSystemService(String)} to access system speech recognition service. + * + * @see #getSystemService(String) + * @hide + */ + public static final String SPEECH_RECOGNITION_SERVICE = "speech_recognition"; + /** * Determine whether the given permission is allowed for a particular * process and user ID running in the system. @@ -6469,7 +6477,7 @@ public abstract class Context { * {@link WindowManager}, {@link android.view.LayoutInflater LayoutInflater} or * {@link android.app.WallpaperManager WallpaperManager}. Accessing UI components from non-UI * contexts throws {@link android.os.strictmode.Violation} if - * {@link StrictMode.VmPolicy.Builder#detectIncorrectContextUse()} is enabled. + * {@link android.os.StrictMode.VmPolicy.Builder#detectIncorrectContextUse()} is enabled. *

* Examples of UI contexts are * an {@link android.app.Activity Activity}, a context created from @@ -6479,7 +6487,7 @@ public abstract class Context { * * @see #getDisplay() * @see #getSystemService(String) - * @see StrictMode.VmPolicy.Builder#detectIncorrectContextUse() + * @see android.os.StrictMode.VmPolicy.Builder#detectIncorrectContextUse() */ public static boolean isUiContext(@NonNull Context context) { return context.isUiContext(); diff --git a/core/java/android/speech/IRecognitionServiceManager.aidl b/core/java/android/speech/IRecognitionServiceManager.aidl new file mode 100644 index 0000000000000..7158ba2f9f638 --- /dev/null +++ b/core/java/android/speech/IRecognitionServiceManager.aidl @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.speech; + +import android.speech.IRecognitionServiceManagerCallback; + +/** + * Binder service allowing speech recognition proxied by the system. + * + * {@hide} + */ +interface IRecognitionServiceManager { + void createSession(in IRecognitionServiceManagerCallback callback); +} diff --git a/core/java/android/speech/IRecognitionServiceManagerCallback.aidl b/core/java/android/speech/IRecognitionServiceManagerCallback.aidl new file mode 100644 index 0000000000000..d760810deda86 --- /dev/null +++ b/core/java/android/speech/IRecognitionServiceManagerCallback.aidl @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.speech; + +import android.speech.IRecognitionService; + +/** + * Callback for the service allowing speech recognition proxied by the system. + * + * {@hide} + */ +oneway interface IRecognitionServiceManagerCallback { + void onSuccess(in IRecognitionService service); + void onError(); +} diff --git a/core/java/android/speech/RecognitionService.java b/core/java/android/speech/RecognitionService.java index 5fd192afed005..c97dbfe38eada 100644 --- a/core/java/android/speech/RecognitionService.java +++ b/core/java/android/speech/RecognitionService.java @@ -28,6 +28,7 @@ import android.os.Bundle; import android.os.Handler; import android.os.IBinder; import android.os.Message; +import android.os.Process; import android.os.RemoteException; import android.util.Log; @@ -50,7 +51,9 @@ public abstract class RecognitionService extends Service { /** * Name under which a RecognitionService component publishes information about itself. * This meta-data should reference an XML resource containing a - * <{@link android.R.styleable#RecognitionService recognition-service}> tag. + * <{@link android.R.styleable#RecognitionService recognition-service}> or + * <{@link android.R.styleable#RecognitionService on-device-recognition-service} + * > tag. */ public static final String SERVICE_META_DATA = "android.speech"; @@ -182,6 +185,13 @@ public abstract class RecognitionService extends Service { private boolean checkPermissions(IRecognitionListener listener, boolean forDataDelivery, @NonNull String packageName, @Nullable String featureId) { if (DBG) Log.d(TAG, "checkPermissions"); + + final int callingUid = Binder.getCallingUid(); + if (callingUid == Process.SYSTEM_UID) { + // Assuming system has verified permissions of the caller. + return true; + } + if (forDataDelivery) { if (PermissionChecker.checkCallingOrSelfPermissionForDataDelivery(this, android.Manifest.permission.RECORD_AUDIO, packageName, featureId, @@ -342,6 +352,7 @@ public abstract class RecognitionService extends Service { * Return the Linux uid assigned to the process that sent you the current transaction that * is being processed. This is obtained from {@link Binder#getCallingUid()}. */ + // TODO(b/176578753): need to make sure this is fixed when proxied through system. public int getCallingUid() { return mCallingUid; } diff --git a/core/java/android/speech/SpeechRecognizer.java b/core/java/android/speech/SpeechRecognizer.java index aea94bfb1bbb6..de879c63a1a64 100644 --- a/core/java/android/speech/SpeechRecognizer.java +++ b/core/java/android/speech/SpeechRecognizer.java @@ -16,6 +16,7 @@ package android.speech; +import android.annotation.NonNull; import android.content.ComponentName; import android.content.Context; import android.content.Intent; @@ -27,6 +28,7 @@ import android.os.IBinder; import android.os.Looper; import android.os.Message; import android.os.RemoteException; +import android.os.ServiceManager; import android.provider.Settings; import android.text.TextUtils; import android.util.Log; @@ -38,8 +40,9 @@ import java.util.Queue; /** * This class provides access to the speech recognition service. This service allows access to the * speech recognizer. Do not instantiate this class directly, instead, call - * {@link SpeechRecognizer#createSpeechRecognizer(Context)}. This class's methods must be - * invoked only from the main application thread. + * {@link SpeechRecognizer#createSpeechRecognizer(Context)}, or + * {@link SpeechRecognizer#createOnDeviceSpeechRecognizer(Context)}. This class's methods must be + * invoked only from the main application thread. * *

The implementation of this API is likely to stream audio to remote servers to perform speech * recognition. As such this API is not intended to be used for continuous recognition, which would @@ -122,8 +125,13 @@ public class SpeechRecognizer { /** Component to direct service intent to */ private final ComponentName mServiceComponent; + /** Whether to use on-device speech recognizer. */ + private final boolean mOnDevice; + + private IRecognitionServiceManager mManagerService; + /** Handler that will execute the main tasks */ - private Handler mHandler = new Handler() { + private Handler mHandler = new Handler(Looper.getMainLooper()) { @Override public void handleMessage(Message msg) { switch (msg.what) { @@ -159,6 +167,17 @@ public class SpeechRecognizer { private SpeechRecognizer(final Context context, final ComponentName serviceComponent) { mContext = context; mServiceComponent = serviceComponent; + mOnDevice = false; + } + + /** + * The right way to create a {@code SpeechRecognizer} is by using + * {@link #createOnDeviceSpeechRecognizer} static factory method + */ + private SpeechRecognizer(final Context context, boolean onDevice) { + mContext = context; + mServiceComponent = null; + mOnDevice = onDevice; } /** @@ -194,6 +213,7 @@ public class SpeechRecognizer { * @return {@code true} if recognition is available, {@code false} otherwise */ public static boolean isRecognitionAvailable(final Context context) { + // TODO(b/176578753): make sure this works well with system speech recognizers. final List list = context.getPackageManager().queryIntentServices( new Intent(RecognitionService.SERVICE_INTERFACE), 0); return list != null && list.size() != 0; @@ -231,12 +251,31 @@ public class SpeechRecognizer { public static SpeechRecognizer createSpeechRecognizer(final Context context, final ComponentName serviceComponent) { if (context == null) { - throw new IllegalArgumentException("Context cannot be null)"); + throw new IllegalArgumentException("Context cannot be null"); } checkIsCalledFromMainThread(); return new SpeechRecognizer(context, serviceComponent); } + /** + * Factory method to create a new {@code SpeechRecognizer}. + * + *

Please note that {@link #setRecognitionListener(RecognitionListener)} should be called + * before dispatching any command to the created {@code SpeechRecognizer}, otherwise no + * notifications will be received. + * + * @param context in which to create {@code SpeechRecognizer} + * @return a new on-device {@code SpeechRecognizer}. + */ + @NonNull + public static SpeechRecognizer createOnDeviceSpeechRecognizer(@NonNull final Context context) { + if (context == null) { + throw new IllegalArgumentException("Context cannot be null"); + } + checkIsCalledFromMainThread(); + return new SpeechRecognizer(context, /* onDevice */ true); + } + /** * Sets the listener that will receive all the callbacks. The previous unfinished commands will * be executed with the old listener, while any following command will be executed with the new @@ -265,36 +304,74 @@ public class SpeechRecognizer { } checkIsCalledFromMainThread(); if (mConnection == null) { // first time connection - mConnection = new Connection(); - - Intent serviceIntent = new Intent(RecognitionService.SERVICE_INTERFACE); - - if (mServiceComponent == null) { - String serviceComponent = Settings.Secure.getString(mContext.getContentResolver(), - Settings.Secure.VOICE_RECOGNITION_SERVICE); - - if (TextUtils.isEmpty(serviceComponent)) { - Log.e(TAG, "no selected voice recognition service"); - mListener.onError(ERROR_CLIENT); - return; - } - - serviceIntent.setComponent(ComponentName.unflattenFromString(serviceComponent)); + // TODO(b/176578753): both flows should go through system service. + if (mOnDevice) { + connectToSystemService(); } else { - serviceIntent.setComponent(mServiceComponent); - } - if (!mContext.bindService(serviceIntent, mConnection, - Context.BIND_AUTO_CREATE | Context.BIND_INCLUDE_CAPABILITIES)) { - Log.e(TAG, "bind to recognition service failed"); - mConnection = null; - mService = null; - mListener.onError(ERROR_CLIENT); - return; + connectToService(); } } putMessage(Message.obtain(mHandler, MSG_START, recognizerIntent)); } + private void connectToSystemService() { + mManagerService = IRecognitionServiceManager.Stub.asInterface( + ServiceManager.getService(Context.SPEECH_RECOGNITION_SERVICE)); + + if (mManagerService == null) { + mListener.onError(ERROR_CLIENT); + return; + } + + try { + // TODO(b/176578753): this has to supply information on whether to use on-device impl. + mManagerService.createSession(new IRecognitionServiceManagerCallback.Stub(){ + @Override + public void onSuccess(IRecognitionService service) throws RemoteException { + mService = service; + } + + @Override + public void onError() throws RemoteException { + Log.e(TAG, "Bind to system recognition service failed"); + mListener.onError(ERROR_CLIENT); + } + }); + } catch (RemoteException e) { + e.rethrowFromSystemServer(); + } + } + + private void connectToService() { + mConnection = new Connection(); + + Intent serviceIntent = new Intent(RecognitionService.SERVICE_INTERFACE); + + if (mServiceComponent == null) { + String serviceComponent = Settings.Secure.getString(mContext.getContentResolver(), + Settings.Secure.VOICE_RECOGNITION_SERVICE); + + if (TextUtils.isEmpty(serviceComponent)) { + Log.e(TAG, "no selected voice recognition service"); + mListener.onError(ERROR_CLIENT); + return; + } + + serviceIntent.setComponent( + ComponentName.unflattenFromString(serviceComponent)); + } else { + serviceIntent.setComponent(mServiceComponent); + } + if (!mContext.bindService(serviceIntent, mConnection, + Context.BIND_AUTO_CREATE | Context.BIND_INCLUDE_CAPABILITIES)) { + Log.e(TAG, "bind to recognition service failed"); + mConnection = null; + mService = null; + mListener.onError(ERROR_CLIENT); + return; + } + } + /** * Stops listening for speech. Speech captured so far will be recognized as if the user had * stopped speaking at this point. Note that in the default case, this does not need to be @@ -378,7 +455,7 @@ public class SpeechRecognizer { mListener.onError(ERROR_CLIENT); } } - + private boolean checkOpenConnection() { if (mService != null) { return true; @@ -433,7 +510,7 @@ public class SpeechRecognizer { private final static int MSG_RMS_CHANGED = 8; private final static int MSG_ON_EVENT = 9; - private final Handler mInternalHandler = new Handler() { + private final Handler mInternalHandler = new Handler(Looper.getMainLooper()) { @Override public void handleMessage(Message msg) { if (mInternalListener == null) { diff --git a/core/res/res/values/attrs.xml b/core/res/res/values/attrs.xml index 243c2448a3b10..415a0a2ed5958 100644 --- a/core/res/res/values/attrs.xml +++ b/core/res/res/values/attrs.xml @@ -8287,6 +8287,23 @@ + + + + + + + + + + + diff --git a/core/res/res/values/config.xml b/core/res/res/values/config.xml index 4e272f110a6ed..a928408f68ed2 100644 --- a/core/res/res/values/config.xml +++ b/core/res/res/values/config.xml @@ -3800,6 +3800,12 @@ --> + + +