Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge changes from topic "stmockhal_cts" into udc-dev

Browse Source 
* changes:
  Add VIMService model enrollment override
  ST HAL Instrumentation client side
  Add TestAPI for VIService to create AOHD for test
This commit is contained in:
Atneya Nair
2023-04-10 18:32:40 +00:00
committed by Android (Google) Code Review
parent d301a7c86d b32371dde1
commit 5a162ba7be
14 changed files with 1177 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files

78
core/api/test-current.txt
View File

@@ -1651,6 +1651,11 @@ package android.hardware.soundtrigger {
field @NonNull public static final android.os.Parcelable.Creator<android.hardware.soundtrigger.KeyphraseMetadata> CREATOR;
}
public class SoundTrigger {
field public static final int MODEL_PARAM_INVALID = -1; // 0xffffffff
field public static final int MODEL_PARAM_THRESHOLD_FACTOR = 0; // 0x0
}
public static final class SoundTrigger.KeyphraseRecognitionExtra implements android.os.Parcelable {
ctor public SoundTrigger.KeyphraseRecognitionExtra(int, int, int);
}
@@ -1663,6 +1668,19 @@ package android.hardware.soundtrigger {
ctor public SoundTrigger.ModuleProperties(int, @NonNull String, @NonNull String, @NonNull String, int, @NonNull String, int, int, int, int, boolean, int, boolean, int, boolean, int);
}
public static final class SoundTrigger.RecognitionConfig implements android.os.Parcelable {
ctor public SoundTrigger.RecognitionConfig(boolean, boolean, @Nullable android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionExtra[], @Nullable byte[], int);
ctor public SoundTrigger.RecognitionConfig(boolean, boolean, @Nullable android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionExtra[], @Nullable byte[]);
method public int describeContents();
method public void writeToParcel(@NonNull android.os.Parcel, int);
field @NonNull public static final android.os.Parcelable.Creator<android.hardware.soundtrigger.SoundTrigger.RecognitionConfig> CREATOR;
field public final boolean allowMultipleTriggers;
field public final int audioCapabilities;
field public final boolean captureRequested;
field @NonNull public final byte[] data;
field @NonNull public final android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionExtra[] keyphrases;
}
public static class SoundTrigger.RecognitionEvent {
ctor public SoundTrigger.RecognitionEvent(int, int, boolean, int, int, int, boolean, @NonNull android.media.AudioFormat, @Nullable byte[], long);
}
@@ -2000,6 +2018,57 @@ package android.media.metrics {
}
package android.media.soundtrigger {
public final class SoundTriggerInstrumentation {
method public void setResourceContention(boolean);
method public void triggerOnResourcesAvailable();
method public void triggerRestart();
}
public static interface SoundTriggerInstrumentation.GlobalCallback {
method public default void onClientAttached();
method public default void onClientDetached();
method public default void onFrameworkDetached();
method public void onModelLoaded(@NonNull android.media.soundtrigger.SoundTriggerInstrumentation.ModelSession);
method public default void onPreempted();
method public default void onRestarted();
}
public static interface SoundTriggerInstrumentation.ModelCallback {
method public default void onModelUnloaded();
method public default void onParamSet(int, int);
method public void onRecognitionStarted(@NonNull android.media.soundtrigger.SoundTriggerInstrumentation.RecognitionSession);
}
public class SoundTriggerInstrumentation.ModelSession {
method public void clearModelCallback();
method @NonNull public java.util.List<android.hardware.soundtrigger.SoundTrigger.Keyphrase> getPhrases();
method @NonNull public android.media.soundtrigger.SoundTriggerManager.Model getSoundModel();
method public boolean isKeyphrase();
method public void setModelCallback(@NonNull java.util.concurrent.Executor, @NonNull android.media.soundtrigger.SoundTriggerInstrumentation.ModelCallback);
method public void triggerUnloadModel();
}
public static interface SoundTriggerInstrumentation.RecognitionCallback {
method public void onRecognitionStopped();
}
public class SoundTriggerInstrumentation.RecognitionSession {
method public void clearRecognitionCallback();
method public int getAudioSession();
method @NonNull public android.hardware.soundtrigger.SoundTrigger.RecognitionConfig getRecognitionConfig();
method public void setRecognitionCallback(@NonNull java.util.concurrent.Executor, @NonNull android.media.soundtrigger.SoundTriggerInstrumentation.RecognitionCallback);
method public void triggerAbortRecognition();
method public void triggerRecognitionEvent(@NonNull byte[], @Nullable java.util.List<android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionExtra>);
}
public final class SoundTriggerManager {
method @NonNull @RequiresPermission(android.Manifest.permission.MANAGE_SOUND_TRIGGER) public static android.media.soundtrigger.SoundTriggerInstrumentation attachInstrumentation(@NonNull java.util.concurrent.Executor, @NonNull android.media.soundtrigger.SoundTriggerInstrumentation.GlobalCallback);
}
}
package android.media.tv {
public final class TvInputManager {
@@ -2024,6 +2093,14 @@ package android.media.tv.tuner {
}
package android.media.voice {
public final class KeyphraseModelManager {
method @RequiresPermission("android.permission.MANAGE_VOICE_KEYPHRASES") public void setModelDatabaseForTestEnabled(boolean);
}
}
package android.net {
public class NetworkPolicyManager {
@@ -2944,6 +3021,7 @@ package android.service.voice {
method @NonNull @RequiresPermission(android.Manifest.permission.MANAGE_HOTWORD_DETECTION) public final android.service.voice.AlwaysOnHotwordDetector createAlwaysOnHotwordDetectorForTest(@NonNull String, @NonNull java.util.Locale, @NonNull android.hardware.soundtrigger.SoundTrigger.ModuleProperties, @NonNull java.util.concurrent.Executor, @NonNull android.service.voice.AlwaysOnHotwordDetector.Callback);
method @NonNull @RequiresPermission(android.Manifest.permission.MANAGE_HOTWORD_DETECTION) public final android.service.voice.AlwaysOnHotwordDetector createAlwaysOnHotwordDetectorForTest(@NonNull String, @NonNull java.util.Locale, @Nullable android.os.PersistableBundle, @Nullable android.os.SharedMemory, @NonNull android.hardware.soundtrigger.SoundTrigger.ModuleProperties, @NonNull java.util.concurrent.Executor, @NonNull android.service.voice.AlwaysOnHotwordDetector.Callback);
method @NonNull public final java.util.List<android.hardware.soundtrigger.SoundTrigger.ModuleProperties> listModuleProperties();
method public final void setTestModuleForAlwaysOnHotwordDetectorEnabled(boolean);
}
public static class VoiceInteractionSession.ActivityId {

36
core/java/android/hardware/soundtrigger/SoundTrigger.java
View File

@@ -1051,6 +1051,29 @@ public class SoundTrigger {
return "ModelParamRange [start=" + mStart + ", end=" + mEnd + "]";
}
}
/**
* SoundTrigger model parameter types.
* @hide
*/
@Retention(RetentionPolicy.SOURCE)
@IntDef(flag = true, prefix = { "MODEL_PARAM" }, value = {
MODEL_PARAM_INVALID,
MODEL_PARAM_THRESHOLD_FACTOR
})
public @interface ModelParamTypes {}
/**
* See {@link ModelParams.INVALID}
* @hide
*/
@TestApi
public static final int MODEL_PARAM_INVALID = ModelParams.INVALID;
/**
* See {@link ModelParams.THRESHOLD_FACTOR}
* @hide
*/
@TestApi
public static final int MODEL_PARAM_THRESHOLD_FACTOR = ModelParams.THRESHOLD_FACTOR;
/**
* Modes for key phrase recognition
@@ -1450,7 +1473,8 @@ public class SoundTrigger {
*
* @hide
*/
public static class RecognitionConfig implements Parcelable {
@TestApi
public static final class RecognitionConfig implements Parcelable {
/** True if the DSP should capture the trigger sound and make it available for further
* capture. */
@UnsupportedAppUsage(maxTargetSdk = Build.VERSION_CODES.R, trackingBug = 170729553)
@@ -1464,6 +1488,7 @@ public class SoundTrigger {
* options for each keyphrase. */
@UnsupportedAppUsage(maxTargetSdk = Build.VERSION_CODES.R, trackingBug = 170729553)
@NonNull
@SuppressLint("ArrayReturn")
public final KeyphraseRecognitionExtra keyphrases[];
/** Opaque data for use by system applications who know about voice engine internals,
* typically during enrollment. */
@@ -1479,8 +1504,8 @@ public class SoundTrigger {
public final int audioCapabilities;
public RecognitionConfig(boolean captureRequested, boolean allowMultipleTriggers,
@Nullable KeyphraseRecognitionExtra[] keyphrases, @Nullable byte[] data,
int audioCapabilities) {
@SuppressLint("ArrayReturn") @Nullable KeyphraseRecognitionExtra[] keyphrases,
@Nullable byte[] data, int audioCapabilities) {
this.captureRequested = captureRequested;
this.allowMultipleTriggers = allowMultipleTriggers;
this.keyphrases = keyphrases != null ? keyphrases : new KeyphraseRecognitionExtra[0];
@@ -1490,7 +1515,8 @@ public class SoundTrigger {
@UnsupportedAppUsage
public RecognitionConfig(boolean captureRequested, boolean allowMultipleTriggers,
@Nullable KeyphraseRecognitionExtra[] keyphrases, @Nullable byte[] data) {
@SuppressLint("ArrayReturn") @Nullable KeyphraseRecognitionExtra[] keyphrases,
@Nullable byte[] data) {
this(captureRequested, allowMultipleTriggers, keyphrases, data, 0);
}
@@ -1517,7 +1543,7 @@ public class SoundTrigger {
}
@Override
public void writeToParcel(Parcel dest, int flags) {
public void writeToParcel(@NonNull Parcel dest, int flags) {
dest.writeByte((byte) (captureRequested ? 1 : 0));
dest.writeByte((byte) (allowMultipleTriggers ? 1 : 0));
dest.writeTypedArray(keyphrases, flags);

54
core/java/android/service/voice/VoiceInteractionService.java
View File

@@ -50,6 +50,7 @@ import android.provider.Settings;
import android.util.ArraySet;
import android.util.Log;
import com.android.internal.annotations.GuardedBy;
import com.android.internal.annotations.VisibleForTesting;
import com.android.internal.app.IVoiceActionCheckCallback;
import com.android.internal.app.IVoiceInteractionManagerService;
@@ -200,6 +201,9 @@ public class VoiceInteractionService extends Service {
private final Set<HotwordDetector> mActiveDetectors = new ArraySet<>();
// True if any of the createAOHD methods should use the test ST module.
@GuardedBy("mLock")
private boolean mTestModuleForAlwaysOnHotwordDetectorEnabled = false;
private void onDetectorRemoteException(@NonNull IBinder token, int detectorType) {
Log.d(TAG, "onDetectorRemoteException for " + HotwordDetector.detectorTypeToString(
@@ -512,14 +516,14 @@ public class VoiceInteractionService extends Service {
Objects.requireNonNull(callback);
return createAlwaysOnHotwordDetectorInternal(keyphrase, locale,
/* supportHotwordDetectionService= */ false, /* options= */ null,
/* sharedMemory= */ null, /* moduleProperties */ null, executor, callback);
/* sharedMemory= */ null, /* moduleProperties= */ null, executor, callback);
}
/**
* Same as {@link createAlwaysOnHotwordDetector(String, Locale, Executor,
* AlwaysOnHotwordDetector.Callback)}, but allow explicit selection of the underlying ST
* module to attach to.
* Use {@link listModuleProperties} to get available modules to attach to.
* Use {@link #listModuleProperties()} to get available modules to attach to.
* @hide
*/
@TestApi
@@ -645,14 +649,14 @@ public class VoiceInteractionService extends Service {
Objects.requireNonNull(callback);
return createAlwaysOnHotwordDetectorInternal(keyphrase, locale,
/* supportHotwordDetectionService= */ true, options, sharedMemory,
/* moduleProperties */ null, executor, callback);
/* moduleProperties= */ null, executor, callback);
}
/**
* Same as {@link createAlwaysOnHotwordDetector(String, Locale,
* PersistableBundle, SharedMemory, Executor, AlwaysOnHotwordDetector.Callback)},
* but allow explicit selection of the underlying ST module to attach to.
* Use {@link listModuleProperties} to get available modules to attach to.
* Use {@link #listModuleProperties()} to get available modules to attach to.
* @hide
*/
@TestApi
@@ -717,6 +721,10 @@ public class VoiceInteractionService extends Service {
try {
dspDetector.registerOnDestroyListener(this::onHotwordDetectorDestroyed);
// Check if we are currently overridden, and should use the test module.
if (mTestModuleForAlwaysOnHotwordDetectorEnabled) {
moduleProperties = getTestModuleProperties();
}
// If moduleProperties is null, the default STModule is used.
dspDetector.initialize(options, sharedMemory, moduleProperties);
} catch (Exception e) {
@@ -990,6 +998,44 @@ public class VoiceInteractionService extends Service {
return mKeyphraseEnrollmentInfo;
}
/**
* Configure {@link createAlwaysOnHotwordDetector(String, Locale,
* SoundTrigger.ModuleProperties, Executor, AlwaysOnHotwordDetector.Callback)}
* and similar overloads to utilize the test SoundTrigger module instead of the
* actual DSP module.
* @param isEnabled - {@code true} if subsequently created {@link AlwaysOnHotwordDetector}
* objects should attach to a test module. {@code false} if subsequently created
* {@link AlwaysOnHotwordDetector} should attach to the actual DSP module.
* @hide
*/
@TestApi
public final void setTestModuleForAlwaysOnHotwordDetectorEnabled(boolean isEnabled) {
synchronized (mLock) {
mTestModuleForAlwaysOnHotwordDetectorEnabled = isEnabled;
}
}
/**
* Get the {@link SoundTrigger.ModuleProperties} representing the fake
* STHAL to attach to via {@link createAlwaysOnHotwordDetector(String, Locale,
* SoundTrigger.ModuleProperties, Executor, AlwaysOnHotwordDetector.Callback)} and
* similar overloads for test purposes.
* @return ModuleProperties to use for test purposes.
*/
private final @NonNull SoundTrigger.ModuleProperties getTestModuleProperties() {
var moduleProps = listModuleProperties()
.stream()
.filter((SoundTrigger.ModuleProperties prop)
-> prop.getSupportedModelArch().equals(SoundTrigger.FAKE_HAL_ARCH))
.findFirst()
.orElse(null);
if (moduleProps == null) {
throw new IllegalStateException("Fake ST HAL should always be available");
}
return moduleProps;
}
/**
* Checks if a given keyphrase and locale are supported to create an
* {@link AlwaysOnHotwordDetector}.

7
core/java/com/android/internal/app/ISoundTriggerService.aidl
View File

@@ -16,8 +16,9 @@
package com.android.internal.app;
import android.media.permission.Identity;
import android.hardware.soundtrigger.SoundTrigger;
import android.media.permission.Identity;
import android.media.soundtrigger_middleware.ISoundTriggerInjection;
import com.android.internal.app.ISoundTriggerSession;
/**
@@ -74,4 +75,8 @@ interface ISoundTriggerService {
*/
List<SoundTrigger.ModuleProperties> listModuleProperties(in Identity originatorIdentity);
/**
* Attach an HAL injection interface.
*/
void attachInjection(ISoundTriggerInjection injection);
}

16
core/java/com/android/internal/app/IVoiceInteractionManagerService.aidl
View File

@@ -96,6 +96,21 @@ interface IVoiceInteractionManagerService {
* @RequiresPermission Manifest.permission.MANAGE_VOICE_KEYPHRASES
*/
int deleteKeyphraseSoundModel(int keyphraseId, in String bcp47Locale);
/**
* Override the persistent enrolled model database with an in-memory
* fake for testing purposes.
*
* @param enabled - {@code true} to enable the test database. {@code false} to enable
* the real, persistent database.
* @param token - IBinder used to register a death listener to clean-up the override
* if tests do not clean up gracefully.
*/
@EnforcePermission("MANAGE_VOICE_KEYPHRASES")
@JavaPassthrough(annotation= "@android.annotation.RequiresPermission(" +
"android.Manifest.permission.MANAGE_VOICE_KEYPHRASES)")
void setModelDatabaseForTestEnabled(boolean enabled, IBinder token);
/**
* Indicates if there's a keyphrase sound model available for the given keyphrase ID and the
* user ID of the caller.
@@ -106,6 +121,7 @@ interface IVoiceInteractionManagerService {
* @param bcp47Locale The BCP47 language tag for the keyphrase's locale.
*/
boolean isEnrolledForKeyphrase(int keyphraseId, String bcp47Locale);
/**
* Generates KeyphraseMetadata for an enrolled sound model based on keyphrase string, locale,
* and the user ID of the caller.

630
media/java/android/media/soundtrigger/SoundTriggerInstrumentation.java Normal file
View File

@@ -0,0 +1,630 @@
/**
* Copyright (C) 2023 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package android.media.soundtrigger;
import android.annotation.CallbackExecutor;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.annotation.RequiresPermission;
import android.annotation.TestApi;
import android.hardware.soundtrigger.ConversionUtil;
import android.hardware.soundtrigger.SoundTrigger;
import android.media.soundtrigger_middleware.IAcknowledgeEvent;
import android.media.soundtrigger_middleware.IInjectGlobalEvent;
import android.media.soundtrigger_middleware.IInjectModelEvent;
import android.media.soundtrigger_middleware.IInjectRecognitionEvent;
import android.media.soundtrigger_middleware.ISoundTriggerInjection;
import android.os.IBinder;
import android.os.RemoteException;
import com.android.internal.annotations.GuardedBy;
import com.android.internal.app.ISoundTriggerService;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.UUID;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Executor;
import java.util.function.Consumer;
/**
* Used to inject/observe events when using a fake SoundTrigger HAL for test purposes.
* Created by {@link SoundTriggerManager#getInjection(Executor, GlobalCallback)}.
* Only one instance of this class is valid at any given time, old instances will be delivered
* {@link GlobalCallback#onPreempted()}.
* @hide
*/
@TestApi
public final class SoundTriggerInstrumentation {
private final Object mLock = new Object();
@GuardedBy("mLock")
private IInjectGlobalEvent mInjectGlobalEvent = null;
@GuardedBy("mLock")
private Map<IBinder, ModelSession> mModelSessionMap = new HashMap<>();
@GuardedBy("mLock")
private Map<IBinder, RecognitionSession> mRecognitionSessionMap = new HashMap<>();
@GuardedBy("mLock")
private IBinder mClientToken = null;
private final GlobalCallback mClientCallback;
private final Executor mGlobalCallbackExecutor;
/**
* Callback interface for un-sessioned events observed from the fake STHAL.
* Registered upon construction of {@link SoundTriggerInstrumentation}
* @hide
*/
@TestApi
public interface GlobalCallback {
/**
* Called when the created {@link SoundTriggerInstrumentation} object is invalidated
* by another client creating an {@link SoundTriggerInstrumentation} to instrument the
* fake STHAL. Only one client may inject at a time.
* All sessions are invalidated, no further events will be received, and no
* injected events will be delivered.
*/
default void onPreempted() {}
/**
* Called when the STHAL has been restarted by the framework, due to unexpected
* error conditions.
* Not called when {@link SoundTriggerInstrumentation#triggerRestart()} is injected.
*/
default void onRestarted() {}
/**
* Called when the framework detaches from the fake HAL.
* This is not transmitted to real HALs, but it indicates that the
* framework has flushed its global state.
*/
default void onFrameworkDetached() {}
/**
* Called when a client application attaches to the framework.
* This is not transmitted to real HALs, but it represents the state of
* the framework.
*/
default void onClientAttached() {}
/**
* Called when a client application detaches from the framework.
* This is not transmitted to real HALs, but it represents the state of
* the framework.
*/
default void onClientDetached() {}
/**
* Called when the fake HAL receives a model load from the framework.
* @param modelSession - A session which exposes additional injection
* functionality associated with the newly loaded
* model. See {@link ModelSession}.
*/
void onModelLoaded(@NonNull ModelSession modelSession);
}
/**
* Callback for HAL events related to a loaded model. Register with
* {@link ModelSession#setModelCallback(Executor, ModelCallback)}
* Note, callbacks will not be delivered for events triggered by the injection.
* @hide
*/
@TestApi
public interface ModelCallback {
/**
* Called when the model associated with the {@link ModelSession} this callback
* was registered for was unloaded by the framework.
*/
default void onModelUnloaded() {}
/**
* Called when the model associated with the {@link ModelSession} this callback
* was registered for receives a set parameter call from the framework.
* @param param - Parameter being set.
* See {@link SoundTrigger.ModelParamTypes}
* @param value - Value the model parameter was set to.
*/
default void onParamSet(@SoundTrigger.ModelParamTypes int param, int value) {}
/**
* Called when the model associated with the {@link ModelSession} this callback
* was registered for receives a recognition start request.
* @param recognitionSession - A session which exposes additional injection
* functionality associated with the newly started
* recognition. See {@link RecognitionSession}
*/
void onRecognitionStarted(@NonNull RecognitionSession recognitionSession);
}
/**
* Callback for HAL events related to a started recognition. Register with
* {@link RecognitionSession#setRecognitionCallback(Executor, RecognitionCallback)}
* Note, callbacks will not be delivered for events triggered by the injection.
* @hide
*/
@TestApi
public interface RecognitionCallback {
/**
* Called when the recognition associated with the {@link RecognitionSession} this
* callback was registered for was stopped by the framework.
*/
void onRecognitionStopped();
}
/**
* Session associated with a loaded model in the fake STHAL.
* Can be used to query details about the loaded model, register a callback for future
* model events, or trigger HAL events associated with a loaded model.
* This session is invalid once the model is unloaded, caused by a
* {@link ModelSession#triggerUnloadModel()},
* the client unloading recognition, or if a {@link GlobalCallback#onRestarted()} is
* received.
* Further injections on an invalidated session will not be respected, and no future
* callbacks will be delivered.
* @hide
*/
@TestApi
public class ModelSession {
/**
* Trigger the HAL to preemptively unload the model associated with this session.
* Typically occurs when a higher priority model is loaded which utilizes the same
* resources.
*/
public void triggerUnloadModel() {
synchronized (SoundTriggerInstrumentation.this.mLock) {
try {
mInjectModelEvent.triggerUnloadModel();
} catch (RemoteException e) {
throw e.rethrowFromSystemServer();
}
mModelSessionMap.remove(mInjectModelEvent.asBinder());
}
}
/**
* Get the {@link SoundTriggerManager.Model} associated with this session.
* @return - The model associated with this session.
*/
public @NonNull SoundTriggerManager.Model getSoundModel() {
return mModel;
}
/**
* Get the list of {@link SoundTrigger.Keyphrase} associated with this session.
* @return - The keyphrases associated with this session.
*/
public @NonNull List<SoundTrigger.Keyphrase> getPhrases() {
if (mPhrases == null) {
return new ArrayList<>();
} else {
return new ArrayList<>(Arrays.asList(mPhrases));
}
}
/**
* Get whether this model is of keyphrase type.
* @return - true if the model is a keyphrase model, false otherwise
*/
public boolean isKeyphrase() {
return (mPhrases != null);
}
/**
* Registers the model callback associated with this session. Events associated
* with this model session will be reported via this callback.
* See {@link ModelCallback}
* @param executor - Executor which the callback is dispatched on
* @param callback - Model callback for reporting model session events.
*/
public void setModelCallback(@NonNull @CallbackExecutor Executor executor, @NonNull
ModelCallback callback) {
synchronized (SoundTriggerInstrumentation.this.mLock) {
mModelCallback = Objects.requireNonNull(callback);
mModelExecutor = Objects.requireNonNull(executor);
}
}
/**
* Clear the model callback associated with this session, if any has been
* set by {@link #setModelCallback(Executor, ModelCallback)}.
*/
public void clearModelCallback() {
synchronized (SoundTriggerInstrumentation.this.mLock) {
mModelCallback = null;
mModelExecutor = null;
}
}
private ModelSession(SoundModel model, Phrase[] phrases,
IInjectModelEvent injection) {
mModel = SoundTriggerManager.Model.create(UUID.fromString(model.uuid),
UUID.fromString(model.vendorUuid),
ConversionUtil.sharedMemoryToByteArray(model.data, model.dataSize));
if (phrases != null) {
mPhrases = new SoundTrigger.Keyphrase[phrases.length];
int i = 0;
for (var phrase : phrases) {
mPhrases[i++] = ConversionUtil.aidl2apiPhrase(phrase);
}
} else {
mPhrases = null;
}
mInjectModelEvent = injection;
}
private void wrap(Consumer<ModelCallback> consumer) {
synchronized (SoundTriggerInstrumentation.this.mLock) {
if (mModelCallback != null && mModelExecutor != null) {
final ModelCallback callback = mModelCallback;
mModelExecutor.execute(() -> consumer.accept(callback));
}
}
}
private final SoundTriggerManager.Model mModel;
private final SoundTrigger.Keyphrase[] mPhrases;
private final IInjectModelEvent mInjectModelEvent;
@GuardedBy("SoundTriggerInstrumentation.this.mLock")
private ModelCallback mModelCallback = null;
@GuardedBy("SoundTriggerInstrumentation.this.mLock")
private Executor mModelExecutor = null;
}
/**
* Session associated with a recognition start in the fake STHAL.
* Can be used to get information about the started recognition, register a callback
* for future events associated with this recognition, and triggering
* recognition events or aborts.
* This session is invalid once the recognition is stopped, caused by a
* {@link RecognitionSession#triggerAbortRecognition()},
* {@link RecognitionSession#triggerRecognitionEvent(byte[], List)},
* the client stopping recognition, or any operation which invalidates the
* {@link ModelSession} which the session was created from.
* Further injections on an invalidated session will not be respected, and no future
* callbacks will be delivered.
* @hide
*/
@TestApi
public class RecognitionSession {
/**
* Get an integer token representing the audio session associated with this
* recognition in the STHAL.
* @return - The session token.
*/
public int getAudioSession() {
return mAudioSession;
}
/**
* Get the recognition config used to start this recognition.
* @return - The config passed to the HAL for startRecognition.
*/
public @NonNull SoundTrigger.RecognitionConfig getRecognitionConfig() {
return mRecognitionConfig;
}
/**
* Trigger a recognition in the fake STHAL.
* @param data - The opaque data buffer included in the recognition event.
* @param phraseExtras - Keyphrase metadata included in the event. The
* event must include metadata for the keyphrase id
* associated with this model to be received by the
* client application.
*/
public void triggerRecognitionEvent(@NonNull byte[] data, @Nullable
List<SoundTrigger.KeyphraseRecognitionExtra> phraseExtras) {
PhraseRecognitionExtra[] converted = null;
if (phraseExtras != null) {
converted = new PhraseRecognitionExtra[phraseExtras.size()];
int i = 0;
for (var phraseExtra : phraseExtras) {
converted[i++] = ConversionUtil.api2aidlPhraseRecognitionExtra(phraseExtra);
}
}
synchronized (SoundTriggerInstrumentation.this.mLock) {
mRecognitionSessionMap.remove(mInjectRecognitionEvent.asBinder());
try {
mInjectRecognitionEvent.triggerRecognitionEvent(data, converted);
} catch (RemoteException e) {
throw e.rethrowFromSystemServer();
}
}
}
/**
* Trigger an abort recognition event in the fake HAL. This represents a
* preemptive ending of the recognition session by the HAL, despite no
* recognition detection. Typically occurs during contention for microphone
* usage, or if model limits are hit.
* See {@link SoundTriggerInstrumentation#setResourceContention(boolean)} to block
* subsequent downward calls for contention reasons.
*/
public void triggerAbortRecognition() {
synchronized (SoundTriggerInstrumentation.this.mLock) {
mRecognitionSessionMap.remove(mInjectRecognitionEvent.asBinder());
try {
mInjectRecognitionEvent.triggerAbortRecognition();
} catch (RemoteException e) {
throw e.rethrowFromSystemServer();
}
}
}
/**
* Registers the recognition callback associated with this session. Events associated
* with this recognition session will be reported via this callback.
* See {@link RecognitionCallback}
* @param executor - Executor which the callback is dispatched on
* @param callback - Recognition callback for reporting recognition session events.
*/
public void setRecognitionCallback(@NonNull @CallbackExecutor Executor executor,
@NonNull RecognitionCallback callback) {
synchronized (SoundTriggerInstrumentation.this.mLock) {
mRecognitionCallback = callback;
mRecognitionExecutor = executor;
}
}
/**
* Clear the recognition callback associated with this session, if any has been
* set by {@link #setRecognitionCallback(Executor, RecognitionCallback)}.
*/
public void clearRecognitionCallback() {
synchronized (SoundTriggerInstrumentation.this.mLock) {
mRecognitionCallback = null;
mRecognitionExecutor = null;
}
}
private RecognitionSession(int audioSession,
RecognitionConfig recognitionConfig,
IInjectRecognitionEvent injectRecognitionEvent) {
mAudioSession = audioSession;
mRecognitionConfig = ConversionUtil.aidl2apiRecognitionConfig(recognitionConfig);
mInjectRecognitionEvent = injectRecognitionEvent;
}
private void wrap(Consumer<RecognitionCallback> consumer) {
synchronized (SoundTriggerInstrumentation.this.mLock) {
if (mRecognitionCallback != null && mRecognitionExecutor != null) {
final RecognitionCallback callback = mRecognitionCallback;
mRecognitionExecutor.execute(() -> consumer.accept(callback));
}
}
}
private final int mAudioSession;
private final SoundTrigger.RecognitionConfig mRecognitionConfig;
private final IInjectRecognitionEvent mInjectRecognitionEvent;
@GuardedBy("SoundTriggerInstrumentation.this.mLock")
private Executor mRecognitionExecutor = null;
@GuardedBy("SoundTriggerInstrumentation.this.mLock")
private RecognitionCallback mRecognitionCallback = null;
}
// Implementation of injection interface passed to the HAL.
// This class will re-associate events received on this callback interface
// with sessions, to avoid staleness issues.
private class Injection extends ISoundTriggerInjection.Stub {
@Override
public void registerGlobalEventInjection(IInjectGlobalEvent globalInjection) {
synchronized (SoundTriggerInstrumentation.this.mLock) {
mInjectGlobalEvent = globalInjection;
}
}
@Override
public void onSoundModelLoaded(SoundModel model, @Nullable Phrase[] phrases,
IInjectModelEvent modelInjection, IInjectGlobalEvent globalSession) {
synchronized (SoundTriggerInstrumentation.this.mLock) {
if (globalSession.asBinder() != mInjectGlobalEvent.asBinder()) return;
ModelSession modelSession = new ModelSession(model, phrases, modelInjection);
mModelSessionMap.put(modelInjection.asBinder(), modelSession);
mGlobalCallbackExecutor.execute(() -> mClientCallback.onModelLoaded(modelSession));
}
}
@Override
public void onSoundModelUnloaded(IInjectModelEvent modelSession) {
synchronized (SoundTriggerInstrumentation.this.mLock) {
ModelSession clientModelSession = mModelSessionMap.remove(modelSession.asBinder());
if (clientModelSession == null) return;
clientModelSession.wrap((ModelCallback cb) -> cb.onModelUnloaded());
}
}
@Override
public void onRecognitionStarted(int audioSessionHandle, RecognitionConfig config,
IInjectRecognitionEvent recognitionInjection, IInjectModelEvent modelSession) {
synchronized (SoundTriggerInstrumentation.this.mLock) {
ModelSession clientModelSession = mModelSessionMap.get(modelSession.asBinder());
if (clientModelSession == null) return;
RecognitionSession recogSession = new RecognitionSession(
audioSessionHandle, config, recognitionInjection);
mRecognitionSessionMap.put(recognitionInjection.asBinder(), recogSession);
clientModelSession.wrap((ModelCallback cb) ->
cb.onRecognitionStarted(recogSession));
}
}
@Override
public void onRecognitionStopped(IInjectRecognitionEvent recognitionSession) {
synchronized (SoundTriggerInstrumentation.this.mLock) {
RecognitionSession clientRecognitionSession =
mRecognitionSessionMap.remove(recognitionSession.asBinder());
if (clientRecognitionSession == null) return;
clientRecognitionSession.wrap((RecognitionCallback cb)
-> cb.onRecognitionStopped());
}
}
@Override
public void onParamSet(int modelParam, int value, IInjectModelEvent modelSession) {
synchronized (SoundTriggerInstrumentation.this.mLock) {
ModelSession clientModelSession = mModelSessionMap.get(modelSession.asBinder());
if (clientModelSession == null) return;
clientModelSession.wrap((ModelCallback cb) -> cb.onParamSet(modelParam, value));
}
}
@Override
public void onRestarted(IInjectGlobalEvent globalSession) {
synchronized (SoundTriggerInstrumentation.this.mLock) {
if (globalSession.asBinder() != mInjectGlobalEvent.asBinder()) return;
mRecognitionSessionMap.clear();
mModelSessionMap.clear();
mGlobalCallbackExecutor.execute(() -> mClientCallback.onRestarted());
}
}
@Override
public void onFrameworkDetached(IInjectGlobalEvent globalSession) {
synchronized (SoundTriggerInstrumentation.this.mLock) {
if (globalSession.asBinder() != mInjectGlobalEvent.asBinder()) return;
mGlobalCallbackExecutor.execute(() -> mClientCallback.onFrameworkDetached());
}
}
@Override
public void onClientAttached(IBinder token, IInjectGlobalEvent globalSession) {
synchronized (SoundTriggerInstrumentation.this.mLock) {
if (globalSession.asBinder() != mInjectGlobalEvent.asBinder()) return;
mClientToken = token;
mGlobalCallbackExecutor.execute(() -> mClientCallback.onClientAttached());
}
}
@Override
public void onClientDetached(IBinder token) {
synchronized (SoundTriggerInstrumentation.this.mLock) {
if (token != mClientToken) return;
mClientToken = null;
mGlobalCallbackExecutor.execute(() -> mClientCallback.onClientDetached());
}
}
@Override
public void onPreempted() {
// This is always valid, independent of session
mGlobalCallbackExecutor.execute(() -> mClientCallback.onPreempted());
// Callbacks will no longer be delivered, and injection will be silently dropped.
}
}
/**
* @hide
*/
@RequiresPermission(android.Manifest.permission.MANAGE_SOUND_TRIGGER)
public SoundTriggerInstrumentation(ISoundTriggerService service,
@CallbackExecutor @NonNull Executor executor,
@NonNull GlobalCallback callback) {
mClientCallback = Objects.requireNonNull(callback);
mGlobalCallbackExecutor = Objects.requireNonNull(executor);
try {
service.attachInjection(new Injection());
} catch (RemoteException e) {
e.rethrowFromSystemServer();
}
}
/**
* Simulate a HAL restart, typically caused by the framework on an unexpected error,
* or a restart of the core audio HAL.
* Application sessions will be detached, and all state will be cleared. The framework
* will re-attach to the HAL following restart.
* @hide
*/
@TestApi
public void triggerRestart() {
synchronized (mLock) {
if (mInjectGlobalEvent == null) {
throw new IllegalStateException(
"Attempted to trigger HAL restart before registration");
}
try {
mInjectGlobalEvent.triggerRestart();
} catch (RemoteException e) {
throw e.rethrowFromSystemServer();
}
}
}
/**
* Trigger a resource available callback from the fake SoundTrigger HAL to the framework.
* This callback notifies the framework that methods which previously failed due to
* resource contention may now succeed.
* @hide
*/
@TestApi
public void triggerOnResourcesAvailable() {
synchronized (mLock) {
if (mInjectGlobalEvent == null) {
throw new IllegalStateException(
"Attempted to trigger HAL resources available before registration");
}
try {
mInjectGlobalEvent.triggerOnResourcesAvailable();
} catch (RemoteException e) {
throw e.rethrowFromSystemServer();
}
}
}
/**
* Simulate resource contention, similar to when HAL which does not
* support concurrent capture opens a capture stream, or when a HAL
* has reached its maximum number of models.
* Subsequent model loads and recognition starts will gracefully error.
* Since this call does not trigger a callback through the framework, the
* call will block until the fake HAL has acknowledged the state change.
* @param isResourceContended - true to enable contention, false to return
* to normal functioning.
* @hide
*/
@TestApi
public void setResourceContention(boolean isResourceContended) {
synchronized (mLock) {
if (mInjectGlobalEvent == null) {
throw new IllegalStateException("Injection interface not set up");
}
IInjectGlobalEvent current = mInjectGlobalEvent;
final CountDownLatch signal = new CountDownLatch(1);
try {
current.setResourceContention(isResourceContended, new IAcknowledgeEvent.Stub() {
@Override
public void eventReceived() {
signal.countDown();
}
});
} catch (RemoteException e) {
throw e.rethrowFromSystemServer();
}
// Block until we get a callback from the service that our request was serviced.
try {
// Rely on test timeout if we don't get a response.
signal.await();
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
}
}
}

25
media/java/android/media/soundtrigger/SoundTriggerManager.java
View File

@@ -18,11 +18,13 @@ package android.media.soundtrigger;
import static android.hardware.soundtrigger.SoundTrigger.STATUS_ERROR;
import android.annotation.CallbackExecutor;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.annotation.RequiresPermission;
import android.annotation.SystemApi;
import android.annotation.SystemService;
import android.annotation.TestApi;
import android.app.ActivityThread;
import android.compat.annotation.UnsupportedAppUsage;
import android.content.ComponentName;
@@ -45,6 +47,7 @@ import android.os.Handler;
import android.os.IBinder;
import android.os.ParcelUuid;
import android.os.RemoteException;
import android.os.ServiceManager;
import android.provider.Settings;
import android.util.Slog;
@@ -53,9 +56,9 @@ import com.android.internal.app.ISoundTriggerSession;
import com.android.internal.util.Preconditions;
import java.util.HashMap;
import java.util.List;
import java.util.Objects;
import java.util.UUID;
import java.util.concurrent.Executor;
/**
* This class provides management of non-voice (general sound trigger) based sound recognition
@@ -609,4 +612,24 @@ public final class SoundTriggerManager {
throw e.rethrowFromSystemServer();
}
}
/**
* Create a {@link SoundTriggerInstrumentation} for test purposes, which instruments a fake
* STHAL. Clients must attach to the appropriate underlying ST module.
* @param executor - Executor to dispatch global callbacks on
* @param callback - Callback for unsessioned events received by the fake STHAL
* @return - A {@link SoundTriggerInstrumentation} for observation/injection.
* @hide
*/
@TestApi
@RequiresPermission(android.Manifest.permission.MANAGE_SOUND_TRIGGER)
@NonNull
public static SoundTriggerInstrumentation attachInstrumentation(
@CallbackExecutor @NonNull Executor executor,
@NonNull SoundTriggerInstrumentation.GlobalCallback callback) {
ISoundTriggerService service = ISoundTriggerService.Stub.asInterface(
ServiceManager.getService(Context.SOUND_TRIGGER_SERVICE));
return new SoundTriggerInstrumentation(service, executor, callback);
}
}

21
media/java/android/media/voice/KeyphraseModelManager.java
View File

@@ -21,7 +21,9 @@ import android.annotation.NonNull;
import android.annotation.Nullable;
import android.annotation.RequiresPermission;
import android.annotation.SystemApi;
import android.annotation.TestApi;
import android.hardware.soundtrigger.SoundTrigger;
import android.os.Binder;
import android.os.RemoteException;
import android.os.ServiceSpecificException;
import android.util.Slog;
@@ -154,4 +156,23 @@ public final class KeyphraseModelManager {
throw e.rethrowFromSystemServer();
}
}
/**
* Override the persistent enrolled model database with an in-memory
* fake for testing purposes.
*
* @param enabled - {@code true} if the model enrollment database should be overridden with an
* in-memory fake. {@code false} if the real, persistent model enrollment database should be
* used.
* @hide
*/
@RequiresPermission(Manifest.permission.MANAGE_VOICE_KEYPHRASES)
@TestApi
public void setModelDatabaseForTestEnabled(boolean enabled) {
try {
mVoiceInteractionManagerService.setModelDatabaseForTestEnabled(enabled, new Binder());
} catch (RemoteException e) {
throw e.rethrowFromSystemServer();
}
}
}

25
services/voiceinteraction/java/com/android/server/soundtrigger/SoundTriggerService.java
View File

@@ -39,12 +39,13 @@ import android.app.ActivityThread;
import android.content.ComponentName;
import android.content.Context;
import android.content.Intent;
import android.content.PermissionChecker;
import android.content.ServiceConnection;
import android.content.pm.PackageManager;
import android.content.pm.ResolveInfo;
import android.hardware.soundtrigger.ConversionUtil;
import android.hardware.soundtrigger.IRecognitionStatusCallback;
import android.hardware.soundtrigger.ModelParams;
import android.hardware.soundtrigger.ConversionUtil;
import android.hardware.soundtrigger.SoundTrigger;
import android.hardware.soundtrigger.SoundTrigger.GenericSoundModel;
import android.hardware.soundtrigger.SoundTrigger.KeyphraseSoundModel;
@@ -64,6 +65,7 @@ import android.media.permission.SafeCloseable;
import android.media.soundtrigger.ISoundTriggerDetectionService;
import android.media.soundtrigger.ISoundTriggerDetectionServiceClient;
import android.media.soundtrigger.SoundTriggerDetectionService;
import android.media.soundtrigger_middleware.ISoundTriggerInjection;
import android.media.soundtrigger_middleware.ISoundTriggerMiddlewareService;
import android.os.Binder;
import android.os.Bundle;
@@ -74,8 +76,8 @@ import android.os.Parcel;
import android.os.ParcelUuid;
import android.os.PowerManager;
import android.os.RemoteException;
import android.os.ServiceSpecificException;
import android.os.ServiceManager;
import android.os.ServiceSpecificException;
import android.os.SystemClock;
import android.os.UserHandle;
import android.provider.Settings;
@@ -98,8 +100,8 @@ import java.util.Map;
import java.util.Objects;
import java.util.TreeMap;
import java.util.UUID;
import java.util.stream.Collectors;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
/**
* A single SystemService to manage all sound/voice-based sound models on the DSP.
@@ -296,6 +298,23 @@ public class SoundTriggerService extends SystemService {
return listUnderlyingModuleProperties(originatorIdentity);
}
}
@Override
public void attachInjection(@NonNull ISoundTriggerInjection injection) {
if (PermissionChecker.checkCallingPermissionForPreflight(mContext,
android.Manifest.permission.MANAGE_SOUND_TRIGGER, null)
!= PermissionChecker.PERMISSION_GRANTED) {
throw new SecurityException();
}
try {
ISoundTriggerMiddlewareService.Stub
.asInterface(ServiceManager
.waitForService(Context.SOUND_TRIGGER_MIDDLEWARE_SERVICE))
.attachFakeHalInjection(injection);
} catch (RemoteException e) {
throw e.rethrowFromSystemServer();
}
}
}
class SoundTriggerSessionStub extends ISoundTriggerSession.Stub {

26
services/voiceinteraction/java/com/android/server/voiceinteraction/DatabaseHelper.java
View File

@@ -39,7 +39,7 @@ import java.util.UUID;
*
* @hide
*/
public class DatabaseHelper extends SQLiteOpenHelper {
public class DatabaseHelper extends SQLiteOpenHelper implements IEnrolledModelDb {
static final String TAG = "SoundModelDBHelper";
static final boolean DBG = false;
@@ -153,11 +153,7 @@ public class DatabaseHelper extends SQLiteOpenHelper {
}
}
/**
* Updates the given keyphrase model, adds it, if it doesn't already exist.
*
* TODO: We only support one keyphrase currently.
*/
@Override
public boolean updateKeyphraseSoundModel(KeyphraseSoundModel soundModel) {
synchronized(this) {
SQLiteDatabase db = getWritableDatabase();
@@ -193,9 +189,7 @@ public class DatabaseHelper extends SQLiteOpenHelper {
}
}
/**
* Deletes the sound model and associated keyphrases.
*/
@Override
public boolean deleteKeyphraseSoundModel(int keyphraseId, int userHandle, String bcp47Locale) {
// Normalize the locale to guard against SQL injection.
bcp47Locale = Locale.forLanguageTag(bcp47Locale).toLanguageTag();
@@ -218,12 +212,7 @@ public class DatabaseHelper extends SQLiteOpenHelper {
}
}
/**
* Returns a matching {@link KeyphraseSoundModel} for the keyphrase ID.
* Returns null if a match isn't found.
*
* TODO: We only support one keyphrase currently.
*/
@Override
public KeyphraseSoundModel getKeyphraseSoundModel(int keyphraseId, int userHandle,
String bcp47Locale) {
// Sanitize the locale to guard against SQL injection.
@@ -237,12 +226,7 @@ public class DatabaseHelper extends SQLiteOpenHelper {
}
}
/**
* Returns a matching {@link KeyphraseSoundModel} for the keyphrase string.
* Returns null if a match isn't found.
*
* TODO: We only support one keyphrase currently.
*/
@Override
public KeyphraseSoundModel getKeyphraseSoundModel(String keyphrase, int userHandle,
String bcp47Locale) {
// Sanitize the locale to guard against SQL injection.

90
services/voiceinteraction/java/com/android/server/voiceinteraction/IEnrolledModelDb.java Normal file
View File

@@ -0,0 +1,90 @@
/**
* Copyright (C) 2023 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.server.voiceinteraction;
import android.hardware.soundtrigger.SoundTrigger.Keyphrase;
import android.hardware.soundtrigger.SoundTrigger.KeyphraseSoundModel;
import java.io.PrintWriter;
/**
* Interface for registering and querying the enrolled keyphrase model database for
* {@link VoiceInteractionManagerService}.
* This interface only supports one keyphrase per {@link KeyphraseSoundModel}.
* The non-update methods are uniquely keyed on fields of the first keyphrase
* {@link KeyphraseSoundModel#getKeyphrases()}.
* @hide
*/
public interface IEnrolledModelDb {
//TODO(273286174): We only support one keyphrase currently.
/**
* Register the given {@link KeyphraseSoundModel}, or updates it if it already exists.
*
* @param soundModel - The sound model to register in the database.
* Updates the sound model if the keyphrase id, users, locale match an existing entry.
* Must have one and only one associated {@link Keyphrase}.
* @return - {@code true} if successful, {@code false} if unsuccessful
*/
boolean updateKeyphraseSoundModel(KeyphraseSoundModel soundModel);
/**
* Deletes the previously registered keyphrase sound model from the database.
*
* @param keyphraseId - The (first) keyphrase ID of the KeyphraseSoundModel to delete.
* @param userHandle - The user handle making this request. Must be included in the user
* list of the registered sound model.
* @param bcp47Locale - The locale of the (first) keyphrase associated with this model.
* @return - {@code true} if successful, {@code false} if unsuccessful
*/
boolean deleteKeyphraseSoundModel(int keyphraseId, int userHandle, String bcp47Locale);
//TODO(273286174): We only support one keyphrase currently.
/**
* Returns the first matching {@link KeyphraseSoundModel} for the keyphrase ID, locale pair,
* contingent on the userHandle existing in the user list for the model.
* Returns null if a match isn't found.
*
* @param keyphraseId - The (first) keyphrase ID of the KeyphraseSoundModel to query.
* @param userHandle - The user handle making this request. Must be included in the user
* list of the registered sound model.
* @param bcp47Locale - The locale of the (first) keyphrase associated with this model.
* @return - {@code true} if successful, {@code false} if unsuccessful
*/
KeyphraseSoundModel getKeyphraseSoundModel(int keyphraseId, int userHandle,
String bcp47Locale);
//TODO(273286174): We only support one keyphrase currently.
/**
* Returns the first matching {@link KeyphraseSoundModel} for the keyphrase ID, locale pair,
* contingent on the userHandle existing in the user list for the model.
* Returns null if a match isn't found.
*
* @param keyphrase - The text of (the first) keyphrase of the KeyphraseSoundModel to query.
* @param userHandle - The user handle making this request. Must be included in the user
* list of the registered sound model.
* @param bcp47Locale - The locale of the (first) keyphrase associated with this model.
* @return - {@code true} if successful, {@code false} if unsuccessful
*/
KeyphraseSoundModel getKeyphraseSoundModel(String keyphrase, int userHandle,
String bcp47Locale);
/**
* Dumps contents of database for dumpsys
*/
void dump(PrintWriter pw);
}

148
services/voiceinteraction/java/com/android/server/voiceinteraction/TestModelEnrollmentDatabase.java Normal file
View File

@@ -0,0 +1,148 @@
/**
* Copyright (C) 2023 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.server.voiceinteraction;
import android.annotation.NonNull;
import android.hardware.soundtrigger.SoundTrigger.Keyphrase;
import android.hardware.soundtrigger.SoundTrigger.KeyphraseSoundModel;
import java.io.PrintWriter;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.StringJoiner;
/**
* In memory model enrollment database for testing purposes.
* @hide
*/
public class TestModelEnrollmentDatabase implements IEnrolledModelDb {
// Record representing the primary key used in the real model database.
private static final class EnrollmentKey {
private final int mKeyphraseId;
private final List<Integer> mUserIds;
private final String mLocale;
EnrollmentKey(int keyphraseId,
@NonNull List<Integer> userIds, @NonNull String locale) {
mKeyphraseId = keyphraseId;
mUserIds = Objects.requireNonNull(userIds);
mLocale = Objects.requireNonNull(locale);
}
int keyphraseId() {
return mKeyphraseId;
}
List<Integer> userIds() {
return mUserIds;
}
String locale() {
return mLocale;
}
@Override
public String toString() {
StringJoiner sj = new StringJoiner(", ", "{", "}");
sj.add("keyphraseId: " + mKeyphraseId);
sj.add("userIds: " + mUserIds.toString());
sj.add("locale: " + mLocale.toString());
return "EnrollmentKey: " + sj.toString();
}
@Override
public int hashCode() {
final int prime = 31;
int res = 1;
res = prime * res + mKeyphraseId;
res = prime * res + mUserIds.hashCode();
res = prime * res + mLocale.hashCode();
return res;
}
@Override
public boolean equals(Object other) {
if (this == other) return true;
if (other == null) return false;
if (!(other instanceof EnrollmentKey)) return false;
EnrollmentKey that = (EnrollmentKey) other;
if (mKeyphraseId != that.mKeyphraseId) return false;
if (!mUserIds.equals(that.mUserIds)) return false;
if (!mLocale.equals(that.mLocale)) return false;
return true;
}
}
private final Map<EnrollmentKey, KeyphraseSoundModel> mModelMap = new HashMap<>();
@Override
public boolean updateKeyphraseSoundModel(KeyphraseSoundModel soundModel) {
final Keyphrase keyphrase = soundModel.getKeyphrases()[0];
mModelMap.put(new EnrollmentKey(keyphrase.getId(),
Arrays.stream(keyphrase.getUsers()).boxed().toList(),
keyphrase.getLocale().toLanguageTag()),
soundModel);
return true;
}
@Override
public boolean deleteKeyphraseSoundModel(int keyphraseId, int userHandle, String bcp47Locale) {
return mModelMap.keySet().removeIf(key -> (key.keyphraseId() == keyphraseId)
&& key.locale().equals(bcp47Locale)
&& key.userIds().contains(userHandle));
}
@Override
public KeyphraseSoundModel getKeyphraseSoundModel(int keyphraseId, int userHandle,
String bcp47Locale) {
return mModelMap.entrySet()
.stream()
.filter((entry) -> (entry.getKey().keyphraseId() == keyphraseId)
&& entry.getKey().locale().equals(bcp47Locale)
&& entry.getKey().userIds().contains(userHandle))
.findFirst()
.map((entry) -> entry.getValue())
.orElse(null);
}
@Override
public KeyphraseSoundModel getKeyphraseSoundModel(String keyphrase, int userHandle,
String bcp47Locale) {
return mModelMap.entrySet()
.stream()
.filter((entry) -> (entry.getValue().getKeyphrases()[0].getText().equals(keyphrase)
&& entry.getKey().locale().equals(bcp47Locale)
&& entry.getKey().userIds().contains(userHandle)))
.findFirst()
.map((entry) -> entry.getValue())
.orElse(null);
}
/**
* Dumps contents of database for dumpsys
*/
public void dump(PrintWriter pw) {
pw.println("Using test enrollment database, with enrolled models:");
pw.println(mModelMap);
}
}

73
services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerService.java
View File

@@ -125,7 +125,9 @@ public class VoiceInteractionManagerService extends SystemService {
final Context mContext;
final ContentResolver mResolver;
final DatabaseHelper mDbHelper;
// Can be overridden for testing purposes
private IEnrolledModelDb mDbHelper;
private final IEnrolledModelDb mRealDbHelper;
final ActivityManagerInternal mAmInternal;
final ActivityTaskManagerInternal mAtmInternal;
final UserManagerInternal mUserManagerInternal;
@@ -143,7 +145,7 @@ public class VoiceInteractionManagerService extends SystemService {
mResolver = context.getContentResolver();
mUserManagerInternal = Objects.requireNonNull(
LocalServices.getService(UserManagerInternal.class));
mDbHelper = new DatabaseHelper(context);
mDbHelper = mRealDbHelper = new DatabaseHelper(context);
mServiceStub = new VoiceInteractionManagerServiceStub();
mAmInternal = Objects.requireNonNull(
LocalServices.getService(ActivityManagerInternal.class));
@@ -1605,6 +1607,42 @@ public class VoiceInteractionManagerService extends SystemService {
}
}
@Override
@android.annotation.EnforcePermission(android.Manifest.permission.MANAGE_VOICE_KEYPHRASES)
public void setModelDatabaseForTestEnabled(boolean enabled, IBinder token) {
super.setModelDatabaseForTestEnabled_enforcePermission();
enforceCallerAllowedToEnrollVoiceModel();
synchronized (this) {
if (enabled) {
// Replace the dbhelper with a new test db
final var db = new TestModelEnrollmentDatabase();
try {
// Listen to our caller death, and make sure we revert to the real
// db if they left the model in a test state.
token.linkToDeath(() -> {
synchronized (this) {
if (mDbHelper == db) {
mDbHelper = mRealDbHelper;
mImpl.notifySoundModelsChangedLocked();
}
}
}, 0);
} catch (RemoteException e) {
// If the caller is already dead, nothing to do.
return;
}
mDbHelper = db;
mImpl.notifySoundModelsChangedLocked();
} else {
// Nothing to do if the db is already set to the real impl.
if (mDbHelper != mRealDbHelper) {
mDbHelper = mRealDbHelper;
mImpl.notifySoundModelsChangedLocked();
}
}
}
}
//----------------- SoundTrigger APIs --------------------------------//
@Override
public boolean isEnrolledForKeyphrase(int keyphraseId, String bcp47Locale) {
@@ -1712,28 +1750,27 @@ public class VoiceInteractionManagerService extends SystemService {
final long caller = Binder.clearCallingIdentity();
try {
KeyphraseSoundModel soundModel =
mDbHelper.getKeyphraseSoundModel(keyphraseId, callingUserId, bcp47Locale);
mDbHelper.getKeyphraseSoundModel(keyphraseId,
callingUserId, bcp47Locale);
if (soundModel == null
|| soundModel.getUuid() == null
|| soundModel.getKeyphrases() == null) {
Slog.w(TAG, "No matching sound model found in startRecognition");
return SoundTriggerInternal.STATUS_ERROR;
} else {
// Regardless of the status of the start recognition, we need to make sure
// that we unload this model if needed later.
synchronized (VoiceInteractionManagerServiceStub.this) {
mLoadedKeyphraseIds.put(keyphraseId, this);
if (mSessionExternalCallback == null
|| mSessionInternalCallback == null
|| callback.asBinder() != mSessionExternalCallback.asBinder()) {
mSessionInternalCallback = createSoundTriggerCallbackLocked(
callback);
mSessionExternalCallback = callback;
}
}
return mSession.startRecognition(keyphraseId, soundModel,
mSessionInternalCallback, recognitionConfig, runInBatterySaverMode);
}
// Regardless of the status of the start recognition, we need to make sure
// that we unload this model if needed later.
synchronized (VoiceInteractionManagerServiceStub.this) {
mLoadedKeyphraseIds.put(keyphraseId, this);
if (mSessionExternalCallback == null
|| mSessionInternalCallback == null
|| callback.asBinder() != mSessionExternalCallback.asBinder()) {
mSessionInternalCallback = createSoundTriggerCallbackLocked(callback);
mSessionExternalCallback = callback;
}
}
return mSession.startRecognition(keyphraseId, soundModel,
mSessionInternalCallback, recognitionConfig, runInBatterySaverMode);
} finally {
Binder.restoreCallingIdentity(caller);
}

42
services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerServiceImpl.java
View File

@@ -71,7 +71,6 @@ import android.util.PrintWriterPrinter;
import android.util.Slog;
import android.view.IWindowManager;
import com.android.internal.annotations.GuardedBy;
import com.android.internal.app.IHotwordRecognitionStatusCallback;
import com.android.internal.app.IVisualQueryDetectionAttentionListener;
import com.android.internal.app.IVoiceActionCheckCallback;
@@ -248,7 +247,6 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
Context.RECEIVER_EXPORTED);
}
@GuardedBy("this")
public void grantImplicitAccessLocked(int grantRecipientUid, @Nullable Intent intent) {
final int grantRecipientAppId = UserHandle.getAppId(grantRecipientUid);
final int grantRecipientUserId = UserHandle.getUserId(grantRecipientUid);
@@ -258,7 +256,6 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
/* direct= */ true);
}
@GuardedBy("this")
public boolean showSessionLocked(@Nullable Bundle args, int flags,
@Nullable String attributionTag,
@Nullable IVoiceInteractionSessionShowCallback showCallback,
@@ -331,7 +328,6 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
}
}
@GuardedBy("this")
public boolean hideSessionLocked() {
if (mActiveSession != null) {
return mActiveSession.hideLocked();
@@ -339,7 +335,6 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
return false;
}
@GuardedBy("this")
public boolean deliverNewSessionLocked(IBinder token,
IVoiceInteractionSession session, IVoiceInteractor interactor) {
if (mActiveSession == null || token != mActiveSession.mToken) {
@@ -350,7 +345,6 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
return true;
}
@GuardedBy("this")
public int startVoiceActivityLocked(@Nullable String callingFeatureId, int callingPid,
int callingUid, IBinder token, Intent intent, String resolvedType) {
try {
@@ -373,7 +367,6 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
}
}
@GuardedBy("this")
public int startAssistantActivityLocked(@Nullable String callingFeatureId, int callingPid,
int callingUid, IBinder token, Intent intent, String resolvedType,
@NonNull Bundle bundle) {
@@ -397,7 +390,6 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
}
}
@GuardedBy("this")
public void requestDirectActionsLocked(@NonNull IBinder token, int taskId,
@NonNull IBinder assistToken, @Nullable RemoteCallback cancellationCallback,
@NonNull RemoteCallback callback) {
@@ -453,7 +445,6 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
}
}
@GuardedBy("this")
void performDirectActionLocked(@NonNull IBinder token, @NonNull String actionId,
@Nullable Bundle arguments, int taskId, IBinder assistToken,
@Nullable RemoteCallback cancellationCallback,
@@ -480,7 +471,6 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
}
}
@GuardedBy("this")
public void setKeepAwakeLocked(IBinder token, boolean keepAwake) {
try {
if (mActiveSession == null || token != mActiveSession.mToken) {
@@ -493,7 +483,6 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
}
}
@GuardedBy("this")
public void closeSystemDialogsLocked(IBinder token) {
try {
if (mActiveSession == null || token != mActiveSession.mToken) {
@@ -506,7 +495,6 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
}
}
@GuardedBy("this")
public void finishLocked(IBinder token, boolean finishTask) {
if (mActiveSession == null || (!finishTask && token != mActiveSession.mToken)) {
Slog.w(TAG, "finish does not match active session");
@@ -516,7 +504,6 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
mActiveSession = null;
}
@GuardedBy("this")
public void setDisabledShowContextLocked(int callingUid, int flags) {
int activeUid = mInfo.getServiceInfo().applicationInfo.uid;
if (callingUid != activeUid) {
@@ -526,7 +513,6 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
mDisabledShowContext = flags;
}
@GuardedBy("this")
public int getDisabledShowContextLocked(int callingUid) {
int activeUid = mInfo.getServiceInfo().applicationInfo.uid;
if (callingUid != activeUid) {
@@ -536,7 +522,6 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
return mDisabledShowContext;
}
@GuardedBy("this")
public int getUserDisabledShowContextLocked(int callingUid) {
int activeUid = mInfo.getServiceInfo().applicationInfo.uid;
if (callingUid != activeUid) {
@@ -550,7 +535,6 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
return mInfo.getSupportsLocalInteraction();
}
@GuardedBy("this")
public void startListeningVisibleActivityChangedLocked(@NonNull IBinder token) {
if (DEBUG) {
Slog.d(TAG, "startListeningVisibleActivityChangedLocked: token=" + token);
@@ -563,7 +547,6 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
mActiveSession.startListeningVisibleActivityChangedLocked();
}
@GuardedBy("this")
public void stopListeningVisibleActivityChangedLocked(@NonNull IBinder token) {
if (DEBUG) {
Slog.d(TAG, "stopListeningVisibleActivityChangedLocked: token=" + token);
@@ -576,7 +559,6 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
mActiveSession.stopListeningVisibleActivityChangedLocked();
}
@GuardedBy("this")
public void notifyActivityDestroyedLocked(@NonNull IBinder activityToken) {
if (DEBUG) {
Slog.d(TAG, "notifyActivityDestroyedLocked activityToken=" + activityToken);
@@ -591,7 +573,6 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
mActiveSession.notifyActivityDestroyedLocked(activityToken);
}
@GuardedBy("this")
public void notifyActivityEventChangedLocked(@NonNull IBinder activityToken, int type) {
if (DEBUG) {
Slog.d(TAG, "notifyActivityEventChangedLocked type=" + type);
@@ -606,7 +587,6 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
mActiveSession.notifyActivityEventChangedLocked(activityToken, type);
}
@GuardedBy("this")
public void updateStateLocked(
@Nullable PersistableBundle options,
@Nullable SharedMemory sharedMemory,
@@ -627,7 +607,6 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
}
}
@GuardedBy("this")
private void verifyDetectorForHotwordDetectionLocked(
@Nullable SharedMemory sharedMemory,
IHotwordRecognitionStatusCallback callback,
@@ -685,7 +664,6 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
voiceInteractionServiceUid);
}
@GuardedBy("this")
private void verifyDetectorForVisualQueryDetectionLocked(@Nullable SharedMemory sharedMemory) {
Slog.v(TAG, "verifyDetectorForVisualQueryDetectionLocked");
@@ -724,7 +702,6 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
}
}
@GuardedBy("this")
public void initAndVerifyDetectorLocked(
@NonNull Identity voiceInteractorIdentity,
@Nullable PersistableBundle options,
@@ -769,7 +746,6 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
detectorType);
}
@GuardedBy("this")
public void destroyDetectorLocked(IBinder token) {
Slog.v(TAG, "destroyDetectorLocked");
@@ -788,7 +764,6 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
}
}
@GuardedBy("this")
public void shutdownHotwordDetectionServiceLocked() {
if (DEBUG) {
Slog.d(TAG, "shutdownHotwordDetectionServiceLocked");
@@ -801,7 +776,6 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
mHotwordDetectionConnection = null;
}
@GuardedBy("this")
public void setVisualQueryDetectionAttentionListenerLocked(
@Nullable IVisualQueryDetectionAttentionListener listener) {
if (mHotwordDetectionConnection == null) {
@@ -810,7 +784,6 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
mHotwordDetectionConnection.setVisualQueryDetectionAttentionListenerLocked(listener);
}
@GuardedBy("this")
public void startPerceivingLocked(IVisualQueryDetectionVoiceInteractionCallback callback) {
if (DEBUG) {
Slog.d(TAG, "startPerceivingLocked");
@@ -824,7 +797,6 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
mHotwordDetectionConnection.startPerceivingLocked(callback);
}
@GuardedBy("this")
public void stopPerceivingLocked() {
if (DEBUG) {
Slog.d(TAG, "stopPerceivingLocked");
@@ -838,7 +810,6 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
mHotwordDetectionConnection.stopPerceivingLocked();
}
@GuardedBy("this")
public void startListeningFromMicLocked(
AudioFormat audioFormat,
IMicrophoneHotwordDetectionVoiceInteractionCallback callback) {
@@ -854,7 +825,6 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
mHotwordDetectionConnection.startListeningFromMicLocked(audioFormat, callback);
}
@GuardedBy("this")
public void startListeningFromExternalSourceLocked(
ParcelFileDescriptor audioStream,
AudioFormat audioFormat,
@@ -879,7 +849,6 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
options, token, callback);
}
@GuardedBy("this")
public void stopListeningFromMicLocked() {
if (DEBUG) {
Slog.d(TAG, "stopListeningFromMicLocked");
@@ -893,7 +862,6 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
mHotwordDetectionConnection.stopListeningFromMicLocked();
}
@GuardedBy("this")
public void triggerHardwareRecognitionEventForTestLocked(
SoundTrigger.KeyphraseRecognitionEvent event,
IHotwordRecognitionStatusCallback callback) {
@@ -908,7 +876,6 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
mHotwordDetectionConnection.triggerHardwareRecognitionEventForTestLocked(event, callback);
}
@GuardedBy("this")
public IRecognitionStatusCallback createSoundTriggerCallbackLocked(
IHotwordRecognitionStatusCallback callback) {
if (DEBUG) {
@@ -933,12 +900,11 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
return null;
}
@GuardedBy("this")
boolean isIsolatedProcessLocked(@NonNull ServiceInfo serviceInfo) {
return (serviceInfo.flags & ServiceInfo.FLAG_ISOLATED_PROCESS) != 0
&& (serviceInfo.flags & ServiceInfo.FLAG_EXTERNAL_SERVICE) == 0;
}
@GuardedBy("this")
boolean verifyProcessSharingLocked() {
// only check this if both VQDS and HDS are declared in the app
ServiceInfo hotwordInfo = getServiceInfoLocked(mHotwordDetectionComponentName, mUser);
@@ -960,7 +926,6 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
mHotwordDetectionConnection.forceRestart();
}
@GuardedBy("this")
void setDebugHotwordLoggingLocked(boolean logging) {
if (mHotwordDetectionConnection == null) {
Slog.w(TAG, "Failed to set temporary debug logging: no hotword detection active");
@@ -969,7 +934,6 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
mHotwordDetectionConnection.setDebugHotwordLoggingLocked(logging);
}
@GuardedBy("this")
void resetHotwordDetectionConnectionLocked() {
if (DEBUG) {
Slog.d(TAG, "resetHotwordDetectionConnectionLocked");
@@ -984,7 +948,6 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
mHotwordDetectionConnection = null;
}
@GuardedBy("this")
public void dumpLocked(FileDescriptor fd, PrintWriter pw, String[] args) {
if (!mValid) {
pw.print(" NOT VALID: ");
@@ -1023,7 +986,6 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
}
}
@GuardedBy("this")
void startLocked() {
Intent intent = new Intent(VoiceInteractionService.SERVICE_INTERFACE);
intent.setComponent(mComponent);
@@ -1048,7 +1010,6 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
}
}
@GuardedBy("this")
void shutdownLocked() {
// If there is an active session, cancel it to allow it to clean up its window and other
// state.
@@ -1076,7 +1037,6 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
}
}
@GuardedBy("this")
void notifySoundModelsChangedLocked() {
if (mService == null) {
Slog.w(TAG, "Not bound to voice interaction service " + mComponent);