Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge "Implement passing the model data to the trusted process" into sc-dev

Browse Source
This commit is contained in:
TreeHugger Robot
2021-03-12 02:32:22 +00:00
committed by Android (Google) Code Review
parent d77d2bf6e4 9a75f1c529
commit 98dfc2a69c
9 changed files with 249 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
core/api/system-current.txt
View File

@@ -10251,6 +10251,7 @@ package android.service.voice {
ctor public HotwordDetectionService();
method @Nullable public final android.os.IBinder onBind(@NonNull android.content.Intent);
method public void onDetectFromDspSource(@NonNull android.os.ParcelFileDescriptor, @NonNull android.media.AudioFormat, long, @NonNull android.service.voice.HotwordDetectionService.DspHotwordDetectionCallback);
method public void onUpdateState(@Nullable android.os.Bundle, @Nullable android.os.SharedMemory);
field public static final String SERVICE_INTERFACE = "android.service.voice.HotwordDetectionService";
}
@@ -10261,10 +10262,8 @@ package android.service.voice {
public class VoiceInteractionService extends android.app.Service {
method @NonNull public final android.service.voice.AlwaysOnHotwordDetector createAlwaysOnHotwordDetector(String, java.util.Locale, android.service.voice.AlwaysOnHotwordDetector.Callback);
method @NonNull public final android.service.voice.AlwaysOnHotwordDetector createAlwaysOnHotwordDetector(String, java.util.Locale, @Nullable android.os.Bundle, @Nullable android.os.SharedMemory, android.service.voice.AlwaysOnHotwordDetector.Callback);
method @NonNull @RequiresPermission("android.permission.MANAGE_VOICE_KEYPHRASES") public final android.media.voice.KeyphraseModelManager createKeyphraseModelManager();
method public final int setHotwordDetectionConfig(@Nullable android.os.Bundle);
field public static final int HOTWORD_CONFIG_FAILURE = 1; // 0x1
field public static final int HOTWORD_CONFIG_SUCCESS = 0; // 0x0
}
}

60
core/java/android/service/voice/AlwaysOnHotwordDetector.java
View File

@@ -41,10 +41,12 @@ import android.media.permission.Identity;
import android.os.AsyncTask;
import android.os.Binder;
import android.os.Build;
import android.os.Bundle;
import android.os.Handler;
import android.os.IBinder;
import android.os.Message;
import android.os.RemoteException;
import android.os.SharedMemory;
import android.util.Slog;
import com.android.internal.app.IHotwordRecognitionStatusCallback;
@@ -287,6 +289,7 @@ public class AlwaysOnHotwordDetector {
private final Handler mHandler;
private final IBinder mBinder = new Binder();
private final int mTargetSdkVersion;
private final boolean mSupportHotwordDetectionService;
private int mAvailability = STATE_NOT_READY;
@@ -488,11 +491,22 @@ public class AlwaysOnHotwordDetector {
* @param callback A non-null Callback for receiving the recognition events.
* @param modelManagementService A service that allows management of sound models.
* @param targetSdkVersion The target SDK version.
* @param supportHotwordDetectionService {@code true} if hotword detection service should be
* triggered, otherwise {@code false}.
* @param options Application configuration data provided by the
* {@link VoiceInteractionService}. The system strips out any remotable objects or other
* contents that can be used to communicate with other processes.
* @param sharedMemory The unrestricted data blob provided by the
* {@link VoiceInteractionService}. Use this to provide the hotword models data or other
* such data to the trusted process.
*
* @hide
*/
public AlwaysOnHotwordDetector(String text, Locale locale, Callback callback,
KeyphraseEnrollmentInfo keyphraseEnrollmentInfo,
IVoiceInteractionManagerService modelManagementService, int targetSdkVersion) {
IVoiceInteractionManagerService modelManagementService, int targetSdkVersion,
boolean supportHotwordDetectionService, @Nullable Bundle options,
@Nullable SharedMemory sharedMemory) {
mText = text;
mLocale = locale;
mKeyphraseEnrollmentInfo = keyphraseEnrollmentInfo;
@@ -501,6 +515,10 @@ public class AlwaysOnHotwordDetector {
mInternalCallback = new SoundTriggerListener(mHandler);
mModelManagementService = modelManagementService;
mTargetSdkVersion = targetSdkVersion;
mSupportHotwordDetectionService = supportHotwordDetectionService;
if (mSupportHotwordDetectionService) {
setHotwordDetectionServiceConfig(options, sharedMemory);
}
try {
Identity identity = new Identity();
identity.packageName = ActivityThread.currentOpPackageName();
@@ -512,6 +530,38 @@ public class AlwaysOnHotwordDetector {
new RefreshAvailabiltyTask().execute();
}
/**
* Set configuration and pass read-only data to hotword detection service.
*
* @param options Application configuration data provided by the
* {@link VoiceInteractionService}. The system strips out any remotable objects or other
* contents that can be used to communicate with other processes.
* @param sharedMemory The unrestricted data blob provided by the
* {@link VoiceInteractionService}. Use this to provide the hotword models data or other
* such data to the trusted process.
*
* @throws IllegalStateException if it doesn't support hotword detection service.
*
* @hide
*/
public final void setHotwordDetectionServiceConfig(@Nullable Bundle options,
@Nullable SharedMemory sharedMemory) {
if (DBG) {
Slog.d(TAG, "setHotwordDetectionServiceConfig()");
}
if (!mSupportHotwordDetectionService) {
throw new IllegalStateException(
"setHotwordDetectionServiceConfig called, but it doesn't support hotword"
+ " detection service");
}
try {
mModelManagementService.setHotwordDetectionServiceConfig(options, sharedMemory);
} catch (RemoteException e) {
throw e.rethrowFromSystemServer();
}
}
/**
* Gets the recognition modes supported by the associated keyphrase.
*
@@ -839,6 +889,14 @@ public class AlwaysOnHotwordDetector {
synchronized (mLock) {
mAvailability = STATE_INVALID;
notifyStateChangedLocked();
if (mSupportHotwordDetectionService) {
try {
mModelManagementService.shutdownHotwordDetectionService();
} catch (RemoteException e) {
throw e.rethrowFromSystemServer();
}
}
}
}

34
core/java/android/service/voice/HotwordDetectionService.java
View File

@@ -27,13 +27,17 @@ import android.annotation.SystemApi;
import android.app.Service;
import android.content.Intent;
import android.media.AudioFormat;
import android.os.Bundle;
import android.os.Handler;
import android.os.IBinder;
import android.os.Looper;
import android.os.ParcelFileDescriptor;
import android.os.RemoteException;
import android.os.SharedMemory;
import android.util.Log;
import java.util.Locale;
/**
* Implemented by an application that wants to offer detection for hotword. The system will
* start the service after calling {@link VoiceInteractionService#setHotwordDetectionConfig}.
@@ -76,6 +80,17 @@ public abstract class HotwordDetectionService extends Service {
timeoutMillis,
new DspHotwordDetectionCallback(callback)));
}
@Override
public void setConfig(Bundle options, SharedMemory sharedMemory) throws RemoteException {
if (DBG) {
Log.d(TAG, "#setConfig");
}
mHandler.sendMessage(obtainMessage(HotwordDetectionService::onUpdateState,
HotwordDetectionService.this,
options,
sharedMemory));
}
};
@CallSuper
@@ -120,6 +135,25 @@ public abstract class HotwordDetectionService extends Service {
@NonNull DspHotwordDetectionCallback callback) {
}
/**
* Called when the {@link VoiceInteractionService#createAlwaysOnHotwordDetector(String, Locale,
* Bundle, SharedMemory, AlwaysOnHotwordDetector.Callback)} or {@link AlwaysOnHotwordDetector#
* setHotwordDetectionServiceConfig(Bundle, SharedMemory)} requests an update of the hotword
* detection parameters.
*
* @param options Application configuration data provided by the
* {@link VoiceInteractionService}. The system strips out any remotable objects or other
* contents that can be used to communicate with other processes.
* @param sharedMemory The unrestricted data blob provided by the
* {@link VoiceInteractionService}. Use this to provide the hotword models data or other
* such data to the trusted process.
*
* @hide
*/
@SystemApi
public void onUpdateState(@Nullable Bundle options, @Nullable SharedMemory sharedMemory) {
}
/**
* Callback for returning the detected result.
*

4
core/java/android/service/voice/IHotwordDetectionService.aidl
View File

@@ -17,7 +17,9 @@
package android.service.voice;
import android.media.AudioFormat;
import android.os.Bundle;
import android.os.ParcelFileDescriptor;
import android.os.SharedMemory;
import android.service.voice.IDspHotwordDetectionCallback;
/**
@@ -31,4 +33,6 @@ oneway interface IHotwordDetectionService {
in AudioFormat audioFormat,
long timeoutMillis,
in IDspHotwordDetectionCallback callback);
void setConfig(in Bundle options, in SharedMemory sharedMemory);
}

117
core/java/android/service/voice/VoiceInteractionService.java
View File

@@ -17,7 +17,6 @@
package android.service.voice;
import android.Manifest;
import android.annotation.IntDef;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.annotation.RequiresPermission;
@@ -36,6 +35,7 @@ import android.os.Handler;
import android.os.IBinder;
import android.os.RemoteException;
import android.os.ServiceManager;
import android.os.SharedMemory;
import android.provider.Settings;
import android.util.ArraySet;
import android.util.Log;
@@ -47,8 +47,6 @@ import com.android.internal.util.function.pooled.PooledLambda;
import java.io.FileDescriptor;
import java.io.PrintWriter;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@@ -72,32 +70,6 @@ import java.util.Set;
public class VoiceInteractionService extends Service {
static final String TAG = VoiceInteractionService.class.getSimpleName();
/**
* Indicates that the given configs have been set successfully after calling
* {@link VoiceInteractionService#setHotwordDetectionConfig}.
*
* @hide
*/
@SystemApi
public static final int HOTWORD_CONFIG_SUCCESS = 0;
/**
* Indicates that the given configs have been set unsuccessfully after calling
* {@link VoiceInteractionService#setHotwordDetectionConfig}.
*
* @hide
*/
@SystemApi
public static final int HOTWORD_CONFIG_FAILURE = 1;
/** @hide */
@Retention(RetentionPolicy.SOURCE)
@IntDef(flag = true, prefix = { "HOTWORD_CONFIG_" }, value = {
HOTWORD_CONFIG_SUCCESS,
HOTWORD_CONFIG_FAILURE,
})
public @interface HotwordConfigResult {}
/**
* The {@link Intent} that must be declared as handled by the service.
* To be supported, the service must also require the
@@ -329,35 +301,6 @@ public class VoiceInteractionService extends Service {
}
}
/**
* Set hotword detection configuration.
*
* Note: Currently it will trigger hotword detection service after calling this function when
* all conditions meet the requirements.
*
* @param options Config data.
* @return {@link VoiceInteractionService#HOTWORD_CONFIG_SUCCESS} in case of success,
* {@link VoiceInteractionService#HOTWORD_CONFIG_FAILURE} in case of failure.
*
* @throws IllegalStateException if the function is called before onReady() is called.
*
* @hide
*/
@SystemApi
@HotwordConfigResult
public final int setHotwordDetectionConfig(
@SuppressLint("NullableCollection") @Nullable Bundle options) {
if (mSystemService == null) {
throw new IllegalStateException("Not available until onReady() is called");
}
try {
return mSystemService.setHotwordDetectionConfig(options);
} catch (RemoteException e) {
throw e.rethrowFromSystemServer();
}
}
/**
* Creates an {@link AlwaysOnHotwordDetector} for the given keyphrase and locale.
* This instance must be retained and used by the client.
@@ -374,9 +317,61 @@ public class VoiceInteractionService extends Service {
@SystemApi
@NonNull
public final AlwaysOnHotwordDetector createAlwaysOnHotwordDetector(
@SuppressLint("MissingNullability") String keyphrase, // TODO: annotate nullability properly
@SuppressLint("MissingNullability") String keyphrase, // TODO: nullability properly
@SuppressLint({"MissingNullability", "UseIcu"}) Locale locale,
@SuppressLint("MissingNullability") AlwaysOnHotwordDetector.Callback callback) {
return createAlwaysOnHotwordDetectorInternal(keyphrase, locale,
/* supportHotwordDetectionService= */ false, /* options= */ null,
/* sharedMemory= */ null, callback);
}
/**
* Create an {@link AlwaysOnHotwordDetector} and trigger a {@link HotwordDetectionService}
* service, then it will also pass the read-only data to hotword detection service.
*
* Like {@see #createAlwaysOnHotwordDetector(String, Locale, AlwaysOnHotwordDetector.Callback)
* }. Before calling this function, you should set a valid hotword detection service with
* android:hotwordDetectionService in an android.voice_interaction metadata file and set
* android:isolatedProcess="true" in the AndroidManifest.xml of hotword detection service.
* Otherwise it will throw IllegalStateException. After calling this function, the system will
* also trigger a hotword detection service and pass the read-only data back to it.
*
* <p>Note: The system will trigger hotword detection service after calling this function when
* all conditions meet the requirements.
*
* @param keyphrase The keyphrase that's being used, for example "Hello Android".
* @param locale The locale for which the enrollment needs to be performed.
* @param options Application configuration data provided by the
* {@link VoiceInteractionService}. The system strips out any remotable objects or other
* contents that can be used to communicate with other processes.
* @param sharedMemory The unrestricted data blob provided by the
* {@link VoiceInteractionService}. Use this to provide the hotword models data or other
* such data to the trusted process.
* @param callback The callback to notify of detection events.
* @return An always-on hotword detector for the given keyphrase and locale.
*
* @hide
*/
@SystemApi
@NonNull
public final AlwaysOnHotwordDetector createAlwaysOnHotwordDetector(
@SuppressLint("MissingNullability") String keyphrase, // TODO: nullability properly
@SuppressLint({"MissingNullability", "UseIcu"}) Locale locale,
@Nullable Bundle options,
@Nullable SharedMemory sharedMemory,
@SuppressLint("MissingNullability") AlwaysOnHotwordDetector.Callback callback) {
return createAlwaysOnHotwordDetectorInternal(keyphrase, locale,
/* supportHotwordDetectionService= */ true, options,
sharedMemory, callback);
}
private AlwaysOnHotwordDetector createAlwaysOnHotwordDetectorInternal(
@SuppressLint("MissingNullability") String keyphrase, // TODO: nullability properly
@SuppressLint({"MissingNullability", "UseIcu"}) Locale locale,
boolean supportHotwordDetectionService,
@Nullable Bundle options,
@Nullable SharedMemory sharedMemory,
@SuppressLint("MissingNullability") AlwaysOnHotwordDetector.Callback callback) {
if (mSystemService == null) {
throw new IllegalStateException("Not available until onReady() is called");
}
@@ -385,7 +380,8 @@ public class VoiceInteractionService extends Service {
safelyShutdownHotwordDetector();
mHotwordDetector = new AlwaysOnHotwordDetector(keyphrase, locale, callback,
mKeyphraseEnrollmentInfo, mSystemService,
getApplicationContext().getApplicationInfo().targetSdkVersion);
getApplicationContext().getApplicationInfo().targetSdkVersion,
supportHotwordDetectionService, options, sharedMemory);
}
return mHotwordDetector;
}
@@ -432,7 +428,6 @@ public class VoiceInteractionService extends Service {
}
private void safelyShutdownHotwordDetector() {
// TODO (b/178171906): Need to check if the HotwordDetectionService should be unbound.
synchronized (mLock) {
if (mHotwordDetector == null) {
return;

22
core/java/com/android/internal/app/IVoiceInteractionManagerService.aidl
View File

@@ -21,6 +21,7 @@ import android.content.Intent;
import android.media.permission.Identity;
import android.os.Bundle;
import android.os.RemoteCallback;
import android.os.SharedMemory;
import com.android.internal.app.IVoiceActionCheckCallback;
import com.android.internal.app.IVoiceInteractionSessionShowCallback;
@@ -225,14 +226,19 @@ interface IVoiceInteractionManagerService {
IBinder client);
/**
* Sets hotword detection configuration.
* Set configuration and pass read-only data to hotword detection service.
*
* Note: Currently it will trigger hotword detection service after calling this function when
* all conditions meet the requirements.
*
* @param options Config data.
* @return {@link VoiceInteractionService#HOTWORD_CONFIG_SUCCESS} in case of success,
* {@link VoiceInteractionService#HOTWORD_CONFIG_FAILURE} in case of failure.
* @param options Application configuration data provided by the
* {@link VoiceInteractionService}. The system strips out any remotable objects or other
* contents that can be used to communicate with other processes.
* @param sharedMemory The unrestricted data blob provided by the
* {@link VoiceInteractionService}. Use this to provide the hotword models data or other
* such data to the trusted process.
*/
int setHotwordDetectionConfig(in Bundle options);
void setHotwordDetectionServiceConfig(in Bundle options, in SharedMemory sharedMemory);
/**
* Requests to shutdown hotword detection service.
*/
void shutdownHotwordDetectionService();
}

19
services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java
View File

@@ -17,6 +17,7 @@
package com.android.server.voiceinteraction;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.content.ComponentName;
import android.content.Context;
import android.content.Intent;
@@ -25,8 +26,10 @@ import android.hardware.soundtrigger.SoundTrigger;
import android.media.AudioAttributes;
import android.media.AudioRecord;
import android.media.MediaRecorder;
import android.os.Bundle;
import android.os.ParcelFileDescriptor;
import android.os.RemoteException;
import android.os.SharedMemory;
import android.service.voice.AlwaysOnHotwordDetector;
import android.service.voice.HotwordDetectionService;
import android.service.voice.IDspHotwordDetectionCallback;
@@ -74,7 +77,8 @@ final class HotwordDetectionConnection {
boolean mBound;
HotwordDetectionConnection(Object lock, Context context, ComponentName serviceName,
int userId, boolean bindInstantServiceAllowed) {
int userId, boolean bindInstantServiceAllowed, @Nullable Bundle options,
@Nullable SharedMemory sharedMemory) {
mLock = lock;
mContext = context;
mDetectionComponentName = serviceName;
@@ -90,6 +94,14 @@ final class HotwordDetectionConnection {
boolean connected) {
synchronized (mLock) {
mBound = connected;
if (connected) {
try {
service.setConfig(options, sharedMemory);
} catch (RemoteException e) {
// TODO: (b/181842909) Report an error to voice interactor
Slog.w(TAG, "Failed to setConfig for HotwordDetectionService", e);
}
}
}
}
@@ -117,6 +129,11 @@ final class HotwordDetectionConnection {
}
}
void setConfigLocked(Bundle options, SharedMemory sharedMemory) {
mRemoteHotwordDetectionService.run(
service -> service.setConfig(options, sharedMemory));
}
private void detectFromDspSource(SoundTrigger.KeyphraseRecognitionEvent recognitionEvent,
IHotwordRecognitionStatusCallback externalCallback) {
if (DEBUG) {

32
services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerService.java
View File

@@ -60,6 +60,7 @@ import android.os.RemoteCallback;
import android.os.RemoteCallbackList;
import android.os.RemoteException;
import android.os.ResultReceiver;
import android.os.SharedMemory;
import android.os.ShellCallback;
import android.os.Trace;
import android.os.UserHandle;
@@ -982,23 +983,46 @@ public class VoiceInteractionManagerService extends SystemService {
}
@Override
public int setHotwordDetectionConfig(Bundle options) {
public void setHotwordDetectionServiceConfig(@Nullable Bundle options,
@Nullable SharedMemory sharedMemory) {
synchronized (this) {
enforceIsCurrentVoiceInteractionService();
if (mImpl == null) {
Slog.w(TAG,
"setHotwordDetectionConfig without running voice interaction service");
return VoiceInteractionService.HOTWORD_CONFIG_FAILURE;
"setHotwordDetectionServiceConfig without running voice"
+ " interaction service");
return;
}
final long caller = Binder.clearCallingIdentity();
try {
return mImpl.setHotwordDetectionConfigLocked(options);
mImpl.setHotwordDetectionServiceConfigLocked(options, sharedMemory);
} finally {
Binder.restoreCallingIdentity(caller);
}
}
}
@Override
public void shutdownHotwordDetectionService() {
synchronized (this) {
enforceIsCurrentVoiceInteractionService();
if (mImpl == null) {
Slog.w(TAG,
"shutdownHotwordDetectionService without running voice"
+ " interaction service");
return;
}
final long caller = Binder.clearCallingIdentity();
try {
mImpl.shutdownHotwordDetectionServiceLocked();
} finally {
Binder.restoreCallingIdentity(caller);
}
}
}
//----------------- Model management APIs --------------------------------//
@Override

43
services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerServiceImpl.java
View File

@@ -43,11 +43,13 @@ import android.os.IBinder;
import android.os.RemoteCallback;
import android.os.RemoteException;
import android.os.ServiceManager;
import android.os.SharedMemory;
import android.os.UserHandle;
import android.service.voice.IVoiceInteractionService;
import android.service.voice.IVoiceInteractionSession;
import android.service.voice.VoiceInteractionService;
import android.service.voice.VoiceInteractionServiceInfo;
import android.system.OsConstants;
import android.util.PrintWriterPrinter;
import android.util.Slog;
import android.view.IWindowManager;
@@ -389,25 +391,48 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
return mInfo.getSupportsLocalInteraction();
}
public int setHotwordDetectionConfigLocked(Bundle options) {
public void setHotwordDetectionServiceConfigLocked(@Nullable Bundle options,
@Nullable SharedMemory sharedMemory) {
if (DEBUG) {
Slog.d(TAG, "setHotwordDetectionConfigLocked");
Slog.d(TAG, "setHotwordDetectionServiceConfigLocked");
}
if (mHotwordDetectionComponentName == null) {
Slog.e(TAG, "Calling setHotwordDetectionConfigLocked, but hotword detection service"
+ " name not found");
return VoiceInteractionService.HOTWORD_CONFIG_FAILURE;
Slog.w(TAG, "Hotword detection service name not found");
throw new IllegalStateException("Hotword detection service name not found");
}
if (!isIsolatedProcessLocked(mHotwordDetectionComponentName)) {
return VoiceInteractionService.HOTWORD_CONFIG_FAILURE;
Slog.w(TAG, "Hotword detection service not in isolated process");
throw new IllegalStateException("Hotword detection service not in isolated process");
}
// TODO : Need to check related permissions for hotword detection service
// TODO : Sanitize for bundle
mHotwordDetectionConnection = new HotwordDetectionConnection(mServiceStub, mContext,
mHotwordDetectionComponentName, mUser, /* bindInstantServiceAllowed= */ false);
if (sharedMemory != null && !sharedMemory.setProtect(OsConstants.PROT_READ)) {
Slog.w(TAG, "Can't set sharedMemory to be read-only");
throw new IllegalStateException("Can't set sharedMemory to be read-only");
}
return VoiceInteractionService.HOTWORD_CONFIG_SUCCESS;
if (mHotwordDetectionConnection == null) {
mHotwordDetectionConnection = new HotwordDetectionConnection(mServiceStub, mContext,
mHotwordDetectionComponentName, mUser, /* bindInstantServiceAllowed= */ false,
options, sharedMemory);
} else {
mHotwordDetectionConnection.setConfigLocked(options, sharedMemory);
}
}
public void shutdownHotwordDetectionServiceLocked() {
if (DEBUG) {
Slog.d(TAG, "shutdownHotwordDetectionServiceLocked");
}
if (mHotwordDetectionConnection == null) {
Slog.w(TAG, "shutdown, but no hotword detection connection");
return;
}
mHotwordDetectionConnection.cancelLocked();
mHotwordDetectionConnection = null;
}
public IRecognitionStatusCallback createSoundTriggerCallbackLocked(