Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge "Add capture timestamp field to AIDL recognition event" into udc-dev

Browse Source
This commit is contained in:
Treehugger Robot
2023-05-03 05:29:30 +00:00
committed by Android (Google) Code Review
parent c9174e2f86 25272aabef
commit 4e419dfea7
19 changed files with 376 additions and 211 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
core/java/android/hardware/soundtrigger/ConversionUtil.java
View File

@@ -32,10 +32,12 @@ import android.media.soundtrigger.RecognitionConfig;
import android.media.soundtrigger.RecognitionEvent;
import android.media.soundtrigger.RecognitionMode;
import android.media.soundtrigger.SoundModel;
import android.media.soundtrigger_middleware.PhraseRecognitionEventSys;
import android.media.soundtrigger_middleware.RecognitionEventSys;
import android.media.soundtrigger_middleware.SoundTriggerModuleDescriptor;
import android.os.ParcelFileDescriptor;
import android.system.ErrnoException;
import android.os.SharedMemory;
import android.system.ErrnoException;
import java.nio.ByteBuffer;
import java.util.Arrays;
@@ -219,36 +221,40 @@ public class ConversionUtil {
return new SoundTrigger.ConfidenceLevel(apiLevel.userId, apiLevel.levelPercent);
}
public static SoundTrigger.RecognitionEvent aidl2apiRecognitionEvent(
int modelHandle, int captureSession, RecognitionEvent aidlEvent) {
public static SoundTrigger.RecognitionEvent aidl2apiRecognitionEvent(int modelHandle,
int captureSession, RecognitionEventSys aidlEvent) {
RecognitionEvent recognitionEvent = aidlEvent.recognitionEvent;
// The API recognition event doesn't allow for a null audio format, even though it doesn't
// always make sense. We thus replace it with a default.
AudioFormat audioFormat = aidl2apiAudioFormatWithDefault(aidlEvent.audioConfig,
AudioFormat audioFormat = aidl2apiAudioFormatWithDefault(recognitionEvent.audioConfig,
true /*isInput*/);
// TODO(b/265852186) propagate a timestamp from aidl interfaces
return new SoundTrigger.GenericRecognitionEvent(aidlEvent.status, modelHandle,
aidlEvent.captureAvailable, captureSession, aidlEvent.captureDelayMs,
aidlEvent.capturePreambleMs, aidlEvent.triggerInData, audioFormat, aidlEvent.data,
aidlEvent.recognitionStillActive, -1 /* halEventReceivedMillis */);
return new SoundTrigger.GenericRecognitionEvent(recognitionEvent.status, modelHandle,
recognitionEvent.captureAvailable, captureSession, recognitionEvent.captureDelayMs,
recognitionEvent.capturePreambleMs, recognitionEvent.triggerInData, audioFormat,
recognitionEvent.data,
recognitionEvent.recognitionStillActive, aidlEvent.halEventReceivedMillis);
}
public static SoundTrigger.RecognitionEvent aidl2apiPhraseRecognitionEvent(
int modelHandle, int captureSession,
PhraseRecognitionEvent aidlEvent) {
int modelHandle, int captureSession, PhraseRecognitionEventSys aidlEvent) {
PhraseRecognitionEvent recognitionEvent = aidlEvent.phraseRecognitionEvent;
SoundTrigger.KeyphraseRecognitionExtra[] apiExtras =
new SoundTrigger.KeyphraseRecognitionExtra[aidlEvent.phraseExtras.length];
for (int i = 0; i < aidlEvent.phraseExtras.length; ++i) {
apiExtras[i] = aidl2apiPhraseRecognitionExtra(aidlEvent.phraseExtras[i]);
new SoundTrigger.KeyphraseRecognitionExtra[recognitionEvent.phraseExtras.length];
for (int i = 0; i < recognitionEvent.phraseExtras.length; ++i) {
apiExtras[i] = aidl2apiPhraseRecognitionExtra(recognitionEvent.phraseExtras[i]);
}
// The API recognition event doesn't allow for a null audio format, even though it doesn't
// always make sense. We thus replace it with a default.
AudioFormat audioFormat = aidl2apiAudioFormatWithDefault(aidlEvent.common.audioConfig,
AudioFormat audioFormat = aidl2apiAudioFormatWithDefault(
recognitionEvent.common.audioConfig,
true /*isInput*/);
// TODO(b/265852186) propagate a timestamp from aidl interfaces
return new SoundTrigger.KeyphraseRecognitionEvent(aidlEvent.common.status, modelHandle,
aidlEvent.common.captureAvailable, captureSession, aidlEvent.common.captureDelayMs,
aidlEvent.common.capturePreambleMs, aidlEvent.common.triggerInData, audioFormat,
aidlEvent.common.data, apiExtras, -1 /* halEventReceivedMillis */);
return new SoundTrigger.KeyphraseRecognitionEvent(recognitionEvent.common.status,
modelHandle,
recognitionEvent.common.captureAvailable, captureSession,
recognitionEvent.common.captureDelayMs,
recognitionEvent.common.capturePreambleMs, recognitionEvent.common.triggerInData,
audioFormat,
recognitionEvent.common.data, apiExtras, aidlEvent.halEventReceivedMillis);
}
// In case of a null input returns a non-null valid output.

8
core/java/android/hardware/soundtrigger/SoundTriggerModule.java
View File

@@ -22,13 +22,13 @@ import android.compat.annotation.UnsupportedAppUsage;
import android.media.permission.ClearCallingIdentityContext;
import android.media.permission.Identity;
import android.media.permission.SafeCloseable;
import android.media.soundtrigger.PhraseRecognitionEvent;
import android.media.soundtrigger.PhraseSoundModel;
import android.media.soundtrigger.RecognitionEvent;
import android.media.soundtrigger.SoundModel;
import android.media.soundtrigger_middleware.ISoundTriggerCallback;
import android.media.soundtrigger_middleware.ISoundTriggerMiddlewareService;
import android.media.soundtrigger_middleware.ISoundTriggerModule;
import android.media.soundtrigger_middleware.PhraseRecognitionEventSys;
import android.media.soundtrigger_middleware.RecognitionEventSys;
import android.os.Build;
import android.os.Handler;
import android.os.IBinder;
@@ -398,7 +398,7 @@ public class SoundTriggerModule {
}
@Override
public synchronized void onRecognition(int handle, RecognitionEvent event,
public synchronized void onRecognition(int handle, RecognitionEventSys event,
int captureSession)
throws RemoteException {
Message m = mHandler.obtainMessage(EVENT_RECOGNITION,
@@ -407,7 +407,7 @@ public class SoundTriggerModule {
}
@Override
public synchronized void onPhraseRecognition(int handle, PhraseRecognitionEvent event,
public synchronized void onPhraseRecognition(int handle, PhraseRecognitionEventSys event,
int captureSession)
throws RemoteException {
Message m = mHandler.obtainMessage(EVENT_RECOGNITION,

8
media/aidl/android/media/soundtrigger_middleware/ISoundTriggerCallback.aidl
View File

@@ -15,8 +15,8 @@
*/
package android.media.soundtrigger_middleware;
import android.media.soundtrigger.RecognitionEvent;
import android.media.soundtrigger.PhraseRecognitionEvent;
import android.media.soundtrigger_middleware.RecognitionEventSys;
import android.media.soundtrigger_middleware.PhraseRecognitionEventSys;
/**
* Main interface for a client to get notifications of events coming from this module.
@@ -31,7 +31,7 @@ oneway interface ISoundTriggerCallback {
* In case of abortion, the caller may retry after the next onRecognitionAvailabilityChange()
* callback.
*/
void onRecognition(int modelHandle, in RecognitionEvent event, int captureSession);
void onRecognition(int modelHandle, in RecognitionEventSys event, int captureSession);
/**
* Invoked whenever a phrase recognition event is triggered (typically, on recognition, but
* also in case of external aborting of a recognition or a forced recognition event - see the
@@ -39,7 +39,7 @@ oneway interface ISoundTriggerCallback {
* In case of abortion, the caller may retry after the next onRecognitionAvailabilityChange()
* callback.
*/
void onPhraseRecognition(int modelHandle, in PhraseRecognitionEvent event, int captureSession);
void onPhraseRecognition(int modelHandle, in PhraseRecognitionEventSys event, int captureSession);
/**
* Notifies the client that some start/load operations that have previously failed for resource
* reasons (threw a ServiceSpecificException(RESOURCE_CONTENTION) or have been preempted) may

36
media/aidl/android/media/soundtrigger_middleware/PhraseRecognitionEventSys.aidl Normal file
View File

@@ -0,0 +1,36 @@
/*
* Copyright (C) 2023 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package android.media.soundtrigger_middleware;
import android.media.soundtrigger.PhraseRecognitionEvent;
/**
* Wrapper to android.media.soundtrigger.RecognitionEvent providing additional fields used by the
* framework.
*/
parcelable PhraseRecognitionEventSys {
PhraseRecognitionEvent phraseRecognitionEvent;
/**
* Timestamp of when the trigger event from SoundTriggerHal was received by the
* framework.
*
* <p>same units and timebase as {@link SystemClock#elapsedRealtime()}.
* The value will be -1 if the event was not generated from the HAL.
*/
// @ElapsedRealtimeLong
long halEventReceivedMillis = -1;
}

36
media/aidl/android/media/soundtrigger_middleware/RecognitionEventSys.aidl Normal file
View File

@@ -0,0 +1,36 @@
/*
* Copyright (C) 2023 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package android.media.soundtrigger_middleware;
import android.media.soundtrigger.RecognitionEvent;
/**
* Wrapper to android.media.soundtrigger.RecognitionEvent providing additional fields used by the
* framework.
*/
parcelable RecognitionEventSys {
RecognitionEvent recognitionEvent;
/**
* Timestamp of when the trigger event from SoundTriggerHal was received by the
* framework.
*
* <p>same units and timebase as {@link SystemClock#elapsedRealtime()}.
* The value will be -1 if the event was not generated from the HAL.
*/
// @ElapsedRealtimeLong
long halEventReceivedMillis = -1;
}

57
services/tests/voiceinteractiontests/src/com/android/server/soundtrigger_middleware/SoundHw2CompatTest.java
View File

@@ -16,6 +16,8 @@
package com.android.server.soundtrigger_middleware;
import static com.google.common.truth.Truth.assertThat;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
@@ -34,12 +36,12 @@ import static org.mockito.Mockito.verifyNoMoreInteractions;
import static org.mockito.Mockito.when;
import android.media.soundtrigger.ModelParameterRange;
import android.media.soundtrigger.PhraseRecognitionEvent;
import android.media.soundtrigger.Properties;
import android.media.soundtrigger.RecognitionConfig;
import android.media.soundtrigger.RecognitionEvent;
import android.media.soundtrigger.RecognitionStatus;
import android.media.soundtrigger.Status;
import android.media.soundtrigger_middleware.PhraseRecognitionEventSys;
import android.media.soundtrigger_middleware.RecognitionEventSys;
import android.os.HwParcel;
import android.os.IBinder;
import android.os.IHwBinder;
@@ -617,13 +619,16 @@ public class SoundHw2CompatTest {
final int handle = 85;
final int status =
android.hardware.soundtrigger.V2_0.ISoundTriggerHwCallback.RecognitionStatus.ABORT;
ArgumentCaptor<RecognitionEvent> eventCaptor = ArgumentCaptor.forClass(
RecognitionEvent.class);
ArgumentCaptor<RecognitionEventSys> eventCaptor = ArgumentCaptor.forClass(
RecognitionEventSys.class);
hwCallback.recognitionCallback(TestUtil.createRecognitionEvent_2_0(handle, status), 99);
mCanonical.flushCallbacks();
verify(canonicalCallback).recognitionCallback(eq(handle), eventCaptor.capture());
TestUtil.validateRecognitionEvent(eventCaptor.getValue(), RecognitionStatus.ABORTED,
RecognitionEventSys lastEvent = eventCaptor.getValue();
assertThat(lastEvent.halEventReceivedMillis).isGreaterThan(0);
TestUtil.validateRecognitionEvent(lastEvent.recognitionEvent,
RecognitionStatus.ABORTED,
false);
}
@@ -631,14 +636,16 @@ public class SoundHw2CompatTest {
final int handle = 92;
final int status =
android.hardware.soundtrigger.V2_0.ISoundTriggerHwCallback.RecognitionStatus.SUCCESS;
ArgumentCaptor<PhraseRecognitionEvent> eventCaptor = ArgumentCaptor.forClass(
PhraseRecognitionEvent.class);
ArgumentCaptor<PhraseRecognitionEventSys> eventCaptor = ArgumentCaptor.forClass(
PhraseRecognitionEventSys.class);
hwCallback.phraseRecognitionCallback(
TestUtil.createPhraseRecognitionEvent_2_0(handle, status), 99);
mCanonical.flushCallbacks();
verify(canonicalCallback).phraseRecognitionCallback(eq(handle), eventCaptor.capture());
TestUtil.validatePhraseRecognitionEvent(eventCaptor.getValue(),
PhraseRecognitionEventSys lastEvent = eventCaptor.getValue();
assertThat(lastEvent.halEventReceivedMillis).isGreaterThan(0);
TestUtil.validatePhraseRecognitionEvent(lastEvent.phraseRecognitionEvent,
RecognitionStatus.SUCCESS, false);
}
verifyNoMoreInteractions(canonicalCallback);
@@ -652,28 +659,34 @@ public class SoundHw2CompatTest {
final int handle = 85;
final int status =
android.hardware.soundtrigger.V2_0.ISoundTriggerHwCallback.RecognitionStatus.ABORT;
ArgumentCaptor<RecognitionEvent> eventCaptor = ArgumentCaptor.forClass(
RecognitionEvent.class);
ArgumentCaptor<RecognitionEventSys> eventCaptor = ArgumentCaptor.forClass(
RecognitionEventSys.class);
hwCallback.recognitionCallback_2_1(TestUtil.createRecognitionEvent_2_1(handle, status),
99);
mCanonical.flushCallbacks();
verify(canonicalCallback).recognitionCallback(eq(handle), eventCaptor.capture());
TestUtil.validateRecognitionEvent(eventCaptor.getValue(), RecognitionStatus.ABORTED,
RecognitionEventSys lastEvent = eventCaptor.getValue();
assertThat(lastEvent.halEventReceivedMillis).isGreaterThan(0);
TestUtil.validateRecognitionEvent(lastEvent.recognitionEvent,
RecognitionStatus.ABORTED,
false);
}
{
final int handle = 87;
final int status = 3; // FORCED;
ArgumentCaptor<RecognitionEvent> eventCaptor = ArgumentCaptor.forClass(
RecognitionEvent.class);
ArgumentCaptor<RecognitionEventSys> eventCaptor = ArgumentCaptor.forClass(
RecognitionEventSys.class);
hwCallback.recognitionCallback_2_1(TestUtil.createRecognitionEvent_2_1(handle, status),
99);
mCanonical.flushCallbacks();
verify(canonicalCallback).recognitionCallback(eq(handle), eventCaptor.capture());
TestUtil.validateRecognitionEvent(eventCaptor.getValue(), RecognitionStatus.FORCED,
RecognitionEventSys lastEvent = eventCaptor.getValue();
assertThat(lastEvent.halEventReceivedMillis).isGreaterThan(0);
TestUtil.validateRecognitionEvent(lastEvent.recognitionEvent,
RecognitionStatus.FORCED,
true);
}
@@ -681,28 +694,32 @@ public class SoundHw2CompatTest {
final int handle = 92;
final int status =
android.hardware.soundtrigger.V2_0.ISoundTriggerHwCallback.RecognitionStatus.SUCCESS;
ArgumentCaptor<PhraseRecognitionEvent> eventCaptor = ArgumentCaptor.forClass(
PhraseRecognitionEvent.class);
ArgumentCaptor<PhraseRecognitionEventSys> eventCaptor = ArgumentCaptor.forClass(
PhraseRecognitionEventSys.class);
hwCallback.phraseRecognitionCallback_2_1(
TestUtil.createPhraseRecognitionEvent_2_1(handle, status), 99);
mCanonical.flushCallbacks();
verify(canonicalCallback).phraseRecognitionCallback(eq(handle), eventCaptor.capture());
TestUtil.validatePhraseRecognitionEvent(eventCaptor.getValue(),
PhraseRecognitionEventSys lastEvent = eventCaptor.getValue();
assertThat(lastEvent.halEventReceivedMillis).isGreaterThan(0);
TestUtil.validatePhraseRecognitionEvent(lastEvent.phraseRecognitionEvent,
RecognitionStatus.SUCCESS, false);
}
{
final int handle = 102;
final int status = 3; // FORCED;
ArgumentCaptor<PhraseRecognitionEvent> eventCaptor = ArgumentCaptor.forClass(
PhraseRecognitionEvent.class);
ArgumentCaptor<PhraseRecognitionEventSys> eventCaptor = ArgumentCaptor.forClass(
PhraseRecognitionEventSys.class);
hwCallback.phraseRecognitionCallback_2_1(
TestUtil.createPhraseRecognitionEvent_2_1(handle, status), 99);
mCanonical.flushCallbacks();
verify(canonicalCallback).phraseRecognitionCallback(eq(handle), eventCaptor.capture());
TestUtil.validatePhraseRecognitionEvent(eventCaptor.getValue(),
PhraseRecognitionEventSys lastEvent = eventCaptor.getValue();
assertThat(lastEvent.halEventReceivedMillis).isGreaterThan(0);
TestUtil.validatePhraseRecognitionEvent(lastEvent.phraseRecognitionEvent,
RecognitionStatus.FORCED, true);
}
verifyNoMoreInteractions(canonicalCallback);

76
services/tests/voiceinteractiontests/src/com/android/server/soundtrigger_middleware/SoundTriggerHalConcurrentCaptureHandlerTest.java
View File

@@ -30,8 +30,8 @@ import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoMoreInteractions;
import static org.mockito.Mockito.verifyZeroInteractions;
import android.media.soundtrigger.RecognitionEvent;
import android.media.soundtrigger.RecognitionStatus;
import android.media.soundtrigger_middleware.RecognitionEventSys;
import androidx.annotation.NonNull;
@@ -68,13 +68,14 @@ public class SoundTriggerHalConcurrentCaptureHandlerTest {
mNotifier.setActive(true);
verify(mUnderlying).stopRecognition(handle);
ArgumentCaptor<RecognitionEvent> eventCaptor = ArgumentCaptor.forClass(
RecognitionEvent.class);
ArgumentCaptor<RecognitionEventSys> eventCaptor = ArgumentCaptor.forClass(
RecognitionEventSys.class);
Thread.sleep(50);
verify(callback).recognitionCallback(eq(handle), eventCaptor.capture());
RecognitionEvent event = eventCaptor.getValue();
assertEquals(event.status, RecognitionStatus.ABORTED);
assertFalse(event.recognitionStillActive);
RecognitionEventSys event = eventCaptor.getValue();
assertEquals(event.halEventReceivedMillis, -1);
assertEquals(event.recognitionEvent.status, RecognitionStatus.ABORTED);
assertFalse(event.recognitionEvent.recognitionStillActive);
verifyZeroInteractions(mGlobalCallback);
clearInvocations(callback, mUnderlying);
@@ -116,8 +117,11 @@ public class SoundTriggerHalConcurrentCaptureHandlerTest {
mNotifier.setActive(true);
verify(mUnderlying, times(1)).stopRecognition(handle);
ArgumentCaptor<RecognitionEventSys> eventCaptor = ArgumentCaptor.forClass(
RecognitionEventSys.class);
mHandler.stopRecognition(handle);
verify(callback, times(1)).recognitionCallback(eq(handle), any());
verify(callback).recognitionCallback(eq(handle), eventCaptor.capture());
assertEquals(eventCaptor.getValue().halEventReceivedMillis, -1);
}
@Test(timeout = 200)
@@ -133,19 +137,21 @@ public class SoundTriggerHalConcurrentCaptureHandlerTest {
verify(mUnderlying).startRecognition(eq(handle), eq(101), eq(102), any());
doAnswer(invocation -> {
RecognitionEvent event = TestUtil.createRecognitionEvent(RecognitionStatus.ABORTED,
RecognitionEventSys recognitionEventSys = new RecognitionEventSys();
recognitionEventSys.recognitionEvent = TestUtil.createRecognitionEvent(
RecognitionStatus.ABORTED,
false);
recognitionEventSys.halEventReceivedMillis = 12345;
// Call the callback from a different thread to detect deadlocks by preventing recursive
// locking from working.
runOnSeparateThread(() -> modelCallback.recognitionCallback(handle, event));
runOnSeparateThread(
() -> modelCallback.recognitionCallback(handle, recognitionEventSys));
return null;
}).when(mUnderlying).stopRecognition(handle);
mHandler.stopRecognition(handle);
verify(mUnderlying, times(1)).stopRecognition(handle);
ArgumentCaptor<RecognitionEvent> eventCaptor = ArgumentCaptor.forClass(
RecognitionEvent.class);
verify(callback, atMost(1)).recognitionCallback(eq(handle), eventCaptor.capture());
verify(callback, atMost(1)).recognitionCallback(eq(handle), any(RecognitionEventSys.class));
}
@Test(timeout = 200)
@@ -162,11 +168,15 @@ public class SoundTriggerHalConcurrentCaptureHandlerTest {
doAnswer(invocation -> {
// The stop request causes a callback to be flushed.
RecognitionEvent event = TestUtil.createRecognitionEvent(RecognitionStatus.FORCED,
RecognitionEventSys recognitionEventSys = new RecognitionEventSys();
recognitionEventSys.recognitionEvent = TestUtil.createRecognitionEvent(
RecognitionStatus.FORCED,
true);
recognitionEventSys.halEventReceivedMillis = 12345;
// Call the callback from a different thread to detect deadlocks by preventing recursive
// locking from working.
runOnSeparateThread(() -> modelCallback.recognitionCallback(handle, event));
runOnSeparateThread(
() -> modelCallback.recognitionCallback(handle, recognitionEventSys));
// While the HAL is processing the stop request, capture state becomes active.
new Thread(() -> mNotifier.setActive(true)).start();
Thread.sleep(50);
@@ -194,11 +204,15 @@ public class SoundTriggerHalConcurrentCaptureHandlerTest {
doAnswer(invocation -> {
// The stop request causes a callback to be flushed.
RecognitionEvent event = TestUtil.createRecognitionEvent(RecognitionStatus.FORCED,
RecognitionEventSys recognitionEventSys = new RecognitionEventSys();
recognitionEventSys.recognitionEvent = TestUtil.createRecognitionEvent(
RecognitionStatus.FORCED,
true);
recognitionEventSys.halEventReceivedMillis = 12345;
// Call the callback from a different thread to detect deadlocks by preventing recursive
// locking from working.
runOnSeparateThread(() -> modelCallback.recognitionCallback(handle, event));
runOnSeparateThread(
() -> modelCallback.recognitionCallback(handle, recognitionEventSys));
// While the HAL is processing the stop request, client requests stop.
new Thread(() -> mHandler.stopRecognition(handle)).start();
Thread.sleep(50);
@@ -223,23 +237,22 @@ public class SoundTriggerHalConcurrentCaptureHandlerTest {
verify(mUnderlying).startRecognition(eq(handle), eq(101), eq(102), any());
doAnswer(invocation -> {
RecognitionEvent event = TestUtil.createRecognitionEvent(RecognitionStatus.SUCCESS,
RecognitionEventSys recognitionEventSys = new RecognitionEventSys();
recognitionEventSys.recognitionEvent = TestUtil.createRecognitionEvent(
RecognitionStatus.SUCCESS,
false);
recognitionEventSys.halEventReceivedMillis = 12345;
// Call the callback from a different thread to detect deadlocks by preventing recursive
// locking from working.
runOnSeparateThread(() -> modelCallback.recognitionCallback(handle, event));
runOnSeparateThread(
() -> modelCallback.recognitionCallback(handle, recognitionEventSys));
return null;
}).when(mUnderlying).stopRecognition(handle);
mNotifier.setActive(true);
verify(mUnderlying, times(1)).stopRecognition(handle);
Thread.sleep(50);
ArgumentCaptor<RecognitionEvent> eventCaptor = ArgumentCaptor.forClass(
RecognitionEvent.class);
verify(callback, atMost(2)).recognitionCallback(eq(handle), eventCaptor.capture());
RecognitionEvent lastEvent = eventCaptor.getValue();
assertEquals(lastEvent.status, RecognitionStatus.ABORTED);
assertFalse(lastEvent.recognitionStillActive);
verify(callback, atMost(2)).recognitionCallback(eq(handle), any());
}
@@ -256,11 +269,15 @@ public class SoundTriggerHalConcurrentCaptureHandlerTest {
verify(mUnderlying).startRecognition(eq(handle), eq(101), eq(102), any());
doAnswer(invocation -> {
RecognitionEvent event = TestUtil.createRecognitionEvent(RecognitionStatus.FORCED,
RecognitionEventSys recognitionEventSys = new RecognitionEventSys();
recognitionEventSys.recognitionEvent = TestUtil.createRecognitionEvent(
RecognitionStatus.FORCED,
true);
recognitionEventSys.halEventReceivedMillis = 12345;
// Call the callback from a different thread to detect deadlocks by preventing recursive
// locking from working.
runOnSeparateThread(() -> modelCallback.recognitionCallback(handle, event));
runOnSeparateThread(
() -> modelCallback.recognitionCallback(handle, recognitionEventSys));
return null;
}).when(mUnderlying).stopRecognition(handle);
@@ -268,12 +285,7 @@ public class SoundTriggerHalConcurrentCaptureHandlerTest {
verify(mUnderlying, times(1)).stopRecognition(handle);
Thread.sleep(50);
ArgumentCaptor<RecognitionEvent> eventCaptor = ArgumentCaptor.forClass(
RecognitionEvent.class);
verify(callback, atMost(2)).recognitionCallback(eq(handle), eventCaptor.capture());
RecognitionEvent lastEvent = eventCaptor.getValue();
assertEquals(lastEvent.status, RecognitionStatus.ABORTED);
assertFalse(lastEvent.recognitionStillActive);
verify(callback, atMost(2)).recognitionCallback(eq(handle), any());
}
private static void runOnSeparateThread(Runnable runnable) {

99
services/tests/voiceinteractiontests/src/com/android/server/soundtrigger_middleware/SoundTriggerMiddlewareImplTest.java
View File

@@ -30,18 +30,19 @@ import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import android.annotation.ElapsedRealtimeLong;
import android.media.soundtrigger.ModelParameter;
import android.media.soundtrigger.ModelParameterRange;
import android.media.soundtrigger.PhraseRecognitionEvent;
import android.media.soundtrigger.PhraseSoundModel;
import android.media.soundtrigger.Properties;
import android.media.soundtrigger.RecognitionConfig;
import android.media.soundtrigger.RecognitionEvent;
import android.media.soundtrigger.RecognitionStatus;
import android.media.soundtrigger.SoundModel;
import android.media.soundtrigger.Status;
import android.media.soundtrigger_middleware.ISoundTriggerCallback;
import android.media.soundtrigger_middleware.ISoundTriggerModule;
import android.media.soundtrigger_middleware.PhraseRecognitionEventSys;
import android.media.soundtrigger_middleware.RecognitionEventSys;
import android.media.soundtrigger_middleware.SoundTriggerModuleDescriptor;
import android.os.RemoteException;
import android.util.Pair;
@@ -224,10 +225,12 @@ public class SoundTriggerMiddlewareImplTest {
// Stop the recognition.
stopRecognition(module, handle, hwHandle);
ArgumentCaptor<RecognitionEvent> eventCaptor = ArgumentCaptor.forClass(
RecognitionEvent.class);
ArgumentCaptor<RecognitionEventSys> eventCaptor = ArgumentCaptor.forClass(
RecognitionEventSys.class);
verify(callback).onRecognition(eq(handle), eventCaptor.capture(), eq(101));
assertEquals(RecognitionStatus.ABORTED, eventCaptor.getValue().status);
RecognitionEventSys lastEvent = eventCaptor.getValue();
assertEquals(-1, lastEvent.halEventReceivedMillis);
assertEquals(RecognitionStatus.ABORTED, lastEvent.recognitionEvent.status);
// Unload the model.
unloadModel(module, handle, hwHandle);
@@ -273,10 +276,12 @@ public class SoundTriggerMiddlewareImplTest {
// Stop the recognition.
stopRecognition(module, handle, hwHandle);
ArgumentCaptor<PhraseRecognitionEvent> eventCaptor = ArgumentCaptor.forClass(
PhraseRecognitionEvent.class);
ArgumentCaptor<PhraseRecognitionEventSys> eventCaptor = ArgumentCaptor.forClass(
PhraseRecognitionEventSys.class);
verify(callback).onPhraseRecognition(eq(handle), eventCaptor.capture(), eq(101));
assertEquals(RecognitionStatus.ABORTED, eventCaptor.getValue().common.status);
PhraseRecognitionEventSys lastEvent = eventCaptor.getValue();
assertEquals(-1, lastEvent.halEventReceivedMillis);
assertEquals(RecognitionStatus.ABORTED, lastEvent.phraseRecognitionEvent.common.status);
// Unload the model.
unloadModel(module, handle, hwHandle);
@@ -299,11 +304,11 @@ public class SoundTriggerMiddlewareImplTest {
{
// Signal a capture from the driver (with "still active").
RecognitionEvent event = hwCallback.sendRecognitionEvent(hwHandle,
RecognitionStatus.SUCCESS, true);
RecognitionEventSys event = hwCallback.sendRecognitionEvent(hwHandle,
RecognitionStatus.SUCCESS, true, 12345);
ArgumentCaptor<RecognitionEvent> eventCaptor = ArgumentCaptor.forClass(
RecognitionEvent.class);
ArgumentCaptor<RecognitionEventSys> eventCaptor = ArgumentCaptor.forClass(
RecognitionEventSys.class);
verify(callback).onRecognition(eq(handle), eventCaptor.capture(), eq(101));
// Validate the event.
@@ -312,11 +317,11 @@ public class SoundTriggerMiddlewareImplTest {
{
// Signal a capture from the driver (without "still active").
RecognitionEvent event = hwCallback.sendRecognitionEvent(hwHandle,
RecognitionStatus.SUCCESS, false);
RecognitionEventSys event = hwCallback.sendRecognitionEvent(hwHandle,
RecognitionStatus.SUCCESS, false, 12345);
ArgumentCaptor<RecognitionEvent> eventCaptor = ArgumentCaptor.forClass(
RecognitionEvent.class);
ArgumentCaptor<RecognitionEventSys> eventCaptor = ArgumentCaptor.forClass(
RecognitionEventSys.class);
verify(callback, times(2)).onRecognition(eq(handle), eventCaptor.capture(), eq(101));
// Validate the event.
@@ -343,11 +348,11 @@ public class SoundTriggerMiddlewareImplTest {
startRecognition(module, handle, hwHandle);
// Signal a capture from the driver.
PhraseRecognitionEvent event = hwCallback.sendPhraseRecognitionEvent(hwHandle,
RecognitionStatus.SUCCESS, false);
PhraseRecognitionEventSys event = hwCallback.sendPhraseRecognitionEvent(hwHandle,
RecognitionStatus.SUCCESS, false, 12345);
ArgumentCaptor<PhraseRecognitionEvent> eventCaptor = ArgumentCaptor.forClass(
PhraseRecognitionEvent.class);
ArgumentCaptor<PhraseRecognitionEventSys> eventCaptor = ArgumentCaptor.forClass(
PhraseRecognitionEventSys.class);
verify(callback).onPhraseRecognition(eq(handle), eventCaptor.capture(), eq(101));
// Validate the event.
@@ -377,11 +382,11 @@ public class SoundTriggerMiddlewareImplTest {
verify(mHalDriver).forceRecognitionEvent(hwHandle);
// Signal a capture from the driver.
RecognitionEvent event = hwCallback.sendRecognitionEvent(hwHandle,
RecognitionStatus.FORCED, true);
RecognitionEventSys event = hwCallback.sendRecognitionEvent(hwHandle,
RecognitionStatus.FORCED, true, 12345);
ArgumentCaptor<RecognitionEvent> eventCaptor = ArgumentCaptor.forClass(
RecognitionEvent.class);
ArgumentCaptor<RecognitionEventSys> eventCaptor = ArgumentCaptor.forClass(
RecognitionEventSys.class);
verify(callback).onRecognition(eq(handle), eventCaptor.capture(), eq(101));
// Validate the event.
@@ -445,11 +450,11 @@ public class SoundTriggerMiddlewareImplTest {
verify(mHalDriver).forceRecognitionEvent(hwHandle);
// Signal a capture from the driver.
PhraseRecognitionEvent event = hwCallback.sendPhraseRecognitionEvent(hwHandle,
RecognitionStatus.FORCED, true);
PhraseRecognitionEventSys event = hwCallback.sendPhraseRecognitionEvent(hwHandle,
RecognitionStatus.FORCED, true, 12345);
ArgumentCaptor<PhraseRecognitionEvent> eventCaptor = ArgumentCaptor.forClass(
PhraseRecognitionEvent.class);
ArgumentCaptor<PhraseRecognitionEventSys> eventCaptor = ArgumentCaptor.forClass(
PhraseRecognitionEventSys.class);
verify(callback).onPhraseRecognition(eq(handle), eventCaptor.capture(), eq(101));
// Validate the event.
@@ -510,14 +515,16 @@ public class SoundTriggerMiddlewareImplTest {
startRecognition(module, handle, hwHandle);
// Abort.
hwCallback.sendRecognitionEvent(hwHandle, RecognitionStatus.ABORTED, false);
hwCallback.sendRecognitionEvent(hwHandle, RecognitionStatus.ABORTED, false, 12345);
ArgumentCaptor<RecognitionEvent> eventCaptor = ArgumentCaptor.forClass(
RecognitionEvent.class);
ArgumentCaptor<RecognitionEventSys> eventCaptor = ArgumentCaptor.forClass(
RecognitionEventSys.class);
verify(callback).onRecognition(eq(handle), eventCaptor.capture(), eq(101));
// Validate the event.
assertEquals(RecognitionStatus.ABORTED, eventCaptor.getValue().status);
RecognitionEventSys lastEvent = eventCaptor.getValue();
assertEquals(12345, lastEvent.halEventReceivedMillis);
assertEquals(RecognitionStatus.ABORTED, lastEvent.recognitionEvent.status);
// Unload the model.
unloadModel(module, handle, hwHandle);
@@ -540,14 +547,16 @@ public class SoundTriggerMiddlewareImplTest {
startRecognition(module, handle, hwHandle);
// Abort.
hwCallback.sendPhraseRecognitionEvent(hwHandle, RecognitionStatus.ABORTED, false);
hwCallback.sendPhraseRecognitionEvent(hwHandle, RecognitionStatus.ABORTED, false, 12345);
ArgumentCaptor<PhraseRecognitionEvent> eventCaptor = ArgumentCaptor.forClass(
PhraseRecognitionEvent.class);
ArgumentCaptor<PhraseRecognitionEventSys> eventCaptor = ArgumentCaptor.forClass(
PhraseRecognitionEventSys.class);
verify(callback).onPhraseRecognition(eq(handle), eventCaptor.capture(), eq(101));
// Validate the event.
assertEquals(RecognitionStatus.ABORTED, eventCaptor.getValue().common.status);
PhraseRecognitionEventSys lastEvent = eventCaptor.getValue();
assertEquals(12345, lastEvent.halEventReceivedMillis);
assertEquals(RecognitionStatus.ABORTED, lastEvent.phraseRecognitionEvent.common.status);
// Unload the model.
unloadModel(module, handle, hwHandle);
@@ -630,18 +639,24 @@ public class SoundTriggerMiddlewareImplTest {
mCallback = callback;
}
private RecognitionEvent sendRecognitionEvent(int hwHandle, @RecognitionStatus int status,
boolean recognitionStillActive) {
RecognitionEvent event = TestUtil.createRecognitionEvent(status,
private RecognitionEventSys sendRecognitionEvent(int hwHandle,
@RecognitionStatus int status,
boolean recognitionStillActive, @ElapsedRealtimeLong long halEventReceivedMillis) {
RecognitionEventSys event = new RecognitionEventSys();
event.recognitionEvent = TestUtil.createRecognitionEvent(status,
recognitionStillActive);
event.halEventReceivedMillis = halEventReceivedMillis;
mCallback.recognitionCallback(hwHandle, event);
return event;
}
private PhraseRecognitionEvent sendPhraseRecognitionEvent(int hwHandle,
@RecognitionStatus int status, boolean recognitionStillActive) {
PhraseRecognitionEvent event = TestUtil.createPhraseRecognitionEvent(status,
private PhraseRecognitionEventSys sendPhraseRecognitionEvent(int hwHandle,
@RecognitionStatus int status, boolean recognitionStillActive,
@ElapsedRealtimeLong long halEventReceivedMillis) {
PhraseRecognitionEventSys event = new PhraseRecognitionEventSys();
event.phraseRecognitionEvent = TestUtil.createPhraseRecognitionEvent(status,
recognitionStillActive);
event.halEventReceivedMillis = halEventReceivedMillis;
mCallback.phraseRecognitionCallback(hwHandle, event);
return event;
}

34
services/tests/voiceinteractiontests/src/com/android/server/soundtrigger_middleware/SoundTriggerMiddlewareLoggingLatencyTest.java
View File

@@ -32,7 +32,7 @@ import android.media.soundtrigger.PhraseRecognitionExtra;
import android.media.soundtrigger.RecognitionEvent;
import android.media.soundtrigger.RecognitionStatus;
import android.media.soundtrigger_middleware.ISoundTriggerCallback;
import android.media.soundtrigger_middleware.ISoundTriggerModule;
import android.media.soundtrigger_middleware.PhraseRecognitionEventSys;
import android.os.BatteryStatsInternal;
import android.os.Process;
import android.os.RemoteException;
@@ -51,8 +51,6 @@ import org.mockito.ArgumentCaptor;
import org.mockito.Mock;
import org.mockito.MockitoAnnotations;
import java.util.Optional;
@RunWith(JUnit4.class)
public class SoundTriggerMiddlewareLoggingLatencyTest {
@@ -63,8 +61,6 @@ public class SoundTriggerMiddlewareLoggingLatencyTest {
private ISoundTriggerMiddlewareInternal mDelegateMiddleware;
@Mock
private ISoundTriggerCallback mISoundTriggerCallback;
@Mock
private ISoundTriggerModule mSoundTriggerModule;
private SoundTriggerMiddlewareLogging mSoundTriggerMiddlewareLogging;
@Before
@@ -109,7 +105,7 @@ public class SoundTriggerMiddlewareLoggingLatencyTest {
verify(mDelegateMiddleware).attach(anyInt(), soundTriggerCallbackCaptor.capture());
triggerPhraseRecognitionEvent(soundTriggerCallbackCaptor.getValue(),
RecognitionStatus.SUCCESS, Optional.of(100) /* keyphraseId */);
RecognitionStatus.SUCCESS, 100 /* keyphraseId */);
assertThat(mLatencyTracker.getActiveActionStartTime(
ACTION_SHOW_VOICE_INTERACTION)).isGreaterThan(-1);
@@ -124,11 +120,11 @@ public class SoundTriggerMiddlewareLoggingLatencyTest {
verify(mDelegateMiddleware).attach(anyInt(), soundTriggerCallbackCaptor.capture());
triggerPhraseRecognitionEvent(soundTriggerCallbackCaptor.getValue(),
RecognitionStatus.SUCCESS, Optional.of(100) /* keyphraseId */);
RecognitionStatus.SUCCESS, 100 /* keyphraseId */);
long firstTriggerSessionStartTime = mLatencyTracker.getActiveActionStartTime(
ACTION_SHOW_VOICE_INTERACTION);
triggerPhraseRecognitionEvent(soundTriggerCallbackCaptor.getValue(),
RecognitionStatus.SUCCESS, Optional.of(100) /* keyphraseId */);
RecognitionStatus.SUCCESS, 100 /* keyphraseId */);
assertThat(mLatencyTracker.getActiveActionStartTime(
ACTION_SHOW_VOICE_INTERACTION)).isGreaterThan(-1);
assertThat(mLatencyTracker.getActiveActionStartTime(
@@ -145,7 +141,7 @@ public class SoundTriggerMiddlewareLoggingLatencyTest {
verify(mDelegateMiddleware).attach(anyInt(), soundTriggerCallbackCaptor.capture());
triggerPhraseRecognitionEvent(soundTriggerCallbackCaptor.getValue(),
RecognitionStatus.ABORTED, Optional.of(100) /* keyphraseId */);
RecognitionStatus.ABORTED, 100 /* keyphraseId */);
assertThat(
mLatencyTracker.getActiveActionStartTime(ACTION_SHOW_VOICE_INTERACTION)).isEqualTo(
@@ -162,7 +158,7 @@ public class SoundTriggerMiddlewareLoggingLatencyTest {
verify(mDelegateMiddleware).attach(anyInt(), soundTriggerCallbackCaptor.capture());
triggerPhraseRecognitionEvent(soundTriggerCallbackCaptor.getValue(),
RecognitionStatus.SUCCESS, Optional.empty() /* keyphraseId */);
RecognitionStatus.SUCCESS);
assertThat(
mLatencyTracker.getActiveActionStartTime(ACTION_SHOW_VOICE_INTERACTION)).isEqualTo(
@@ -170,19 +166,27 @@ public class SoundTriggerMiddlewareLoggingLatencyTest {
}
private void triggerPhraseRecognitionEvent(ISoundTriggerCallback callback,
@RecognitionStatus int triggerEventStatus, Optional<Integer> optionalKeyphraseId)
throws RemoteException {
@RecognitionStatus int triggerEventStatus) throws RemoteException {
triggerPhraseRecognitionEvent(callback, triggerEventStatus, -1 /* keyphraseId */);
}
private void triggerPhraseRecognitionEvent(ISoundTriggerCallback callback,
@RecognitionStatus int triggerEventStatus, int keyphraseId) throws RemoteException {
// trigger a phrase recognition to start a latency tracker session
PhraseRecognitionEvent successEventWithKeyphraseId = new PhraseRecognitionEvent();
successEventWithKeyphraseId.common = new RecognitionEvent();
successEventWithKeyphraseId.common.status = triggerEventStatus;
if (optionalKeyphraseId.isPresent()) {
if (keyphraseId > 0) {
PhraseRecognitionExtra recognitionExtra = new PhraseRecognitionExtra();
recognitionExtra.id = optionalKeyphraseId.get();
recognitionExtra.id = keyphraseId;
successEventWithKeyphraseId.phraseExtras =
new PhraseRecognitionExtra[]{recognitionExtra};
}
callback.onPhraseRecognition(0 /* modelHandle */, successEventWithKeyphraseId,
PhraseRecognitionEventSys phraseRecognitionEventSys = new PhraseRecognitionEventSys();
phraseRecognitionEventSys.phraseRecognitionEvent = successEventWithKeyphraseId;
phraseRecognitionEventSys.halEventReceivedMillis = 12345;
callback.onPhraseRecognition(0 /* modelHandle */, phraseRecognitionEventSys,
0 /* captureSession */);
}
}

28
services/voiceinteraction/java/com/android/server/soundtrigger_middleware/AidlUtil.java
View File

@@ -21,6 +21,8 @@ import android.media.soundtrigger.PhraseRecognitionExtra;
import android.media.soundtrigger.RecognitionEvent;
import android.media.soundtrigger.RecognitionStatus;
import android.media.soundtrigger.SoundModelType;
import android.media.soundtrigger_middleware.PhraseRecognitionEventSys;
import android.media.soundtrigger_middleware.RecognitionEventSys;
/**
* Utilities for working with sound trigger related AIDL generated types.
@@ -49,23 +51,29 @@ public class AidlUtil {
/**
* Creates a new generic abort event.
*
* @return The new event.
*/
static RecognitionEvent newAbortEvent() {
RecognitionEvent event = newEmptyRecognitionEvent();
event.type = SoundModelType.GENERIC;
event.status = RecognitionStatus.ABORTED;
return event;
static RecognitionEventSys newAbortEvent() {
RecognitionEvent recognitionEvent = newEmptyRecognitionEvent();
recognitionEvent.type = SoundModelType.GENERIC;
recognitionEvent.status = RecognitionStatus.ABORTED;
RecognitionEventSys recognitionEventSys = new RecognitionEventSys();
recognitionEventSys.recognitionEvent = recognitionEvent;
return recognitionEventSys;
}
/**
* Creates a new generic phrase event.
*
* @return The new event.
*/
static PhraseRecognitionEvent newAbortPhraseEvent() {
PhraseRecognitionEvent event = newEmptyPhraseRecognitionEvent();
event.common.type = SoundModelType.KEYPHRASE;
event.common.status = RecognitionStatus.ABORTED;
return event;
static PhraseRecognitionEventSys newAbortPhraseEvent() {
PhraseRecognitionEvent recognitionEvent = newEmptyPhraseRecognitionEvent();
recognitionEvent.common.type = SoundModelType.KEYPHRASE;
recognitionEvent.common.status = RecognitionStatus.ABORTED;
PhraseRecognitionEventSys phraseRecognitionEventSys = new PhraseRecognitionEventSys();
phraseRecognitionEventSys.phraseRecognitionEvent = recognitionEvent;
return phraseRecognitionEventSys;
}
}

17
services/voiceinteraction/java/com/android/server/soundtrigger_middleware/ISoundTriggerHal.java
View File

@@ -20,12 +20,12 @@ import android.hardware.soundtrigger3.ISoundTriggerHw;
import android.hardware.soundtrigger3.ISoundTriggerHwCallback;
import android.hardware.soundtrigger3.ISoundTriggerHwGlobalCallback;
import android.media.soundtrigger.ModelParameterRange;
import android.media.soundtrigger.PhraseRecognitionEvent;
import android.media.soundtrigger.PhraseSoundModel;
import android.media.soundtrigger.Properties;
import android.media.soundtrigger.RecognitionConfig;
import android.media.soundtrigger.RecognitionEvent;
import android.media.soundtrigger.SoundModel;
import android.media.soundtrigger_middleware.PhraseRecognitionEventSys;
import android.media.soundtrigger_middleware.RecognitionEventSys;
import android.os.IBinder;
/**
@@ -173,14 +173,19 @@ interface ISoundTriggerHal {
*/
interface ModelCallback {
/**
* @see ISoundTriggerHwCallback#recognitionCallback(int, RecognitionEvent)
* Decorated callback of
* {@link ISoundTriggerHwCallback#recognitionCallback(int, RecognitionEvent)} where
* {@link RecognitionEventSys} is decorating the returned {@link RecognitionEvent}
*/
void recognitionCallback(int modelHandle, RecognitionEvent event);
void recognitionCallback(int modelHandle, RecognitionEventSys event);
/**
* @see ISoundTriggerHwCallback#phraseRecognitionCallback(int, PhraseRecognitionEvent)
* Decorated callback of
* {@link ISoundTriggerHwCallback#phraseRecognitionCallback(int, PhraseRecognitionEvent)}
* where {@link PhraseRecognitionEventSys} is decorating the returned
* {@link PhraseRecognitionEvent}
*/
void phraseRecognitionCallback(int modelHandle, PhraseRecognitionEvent event);
void phraseRecognitionCallback(int modelHandle, PhraseRecognitionEventSys event);
/**
* @see ISoundTriggerHwCallback#modelUnloaded(int)

12
services/voiceinteraction/java/com/android/server/soundtrigger_middleware/SoundTriggerHalConcurrentCaptureHandler.java
View File

@@ -19,14 +19,14 @@ package com.android.server.soundtrigger_middleware;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.media.soundtrigger.ModelParameterRange;
import android.media.soundtrigger.PhraseRecognitionEvent;
import android.media.soundtrigger.PhraseSoundModel;
import android.media.soundtrigger.Properties;
import android.media.soundtrigger.RecognitionConfig;
import android.media.soundtrigger.RecognitionEvent;
import android.media.soundtrigger.SoundModel;
import android.media.soundtrigger.SoundModelType;
import android.media.soundtrigger.Status;
import android.media.soundtrigger_middleware.PhraseRecognitionEventSys;
import android.media.soundtrigger_middleware.RecognitionEventSys;
import android.os.IBinder;
import java.util.HashSet;
@@ -238,13 +238,13 @@ public class SoundTriggerHalConcurrentCaptureHandler implements ISoundTriggerHal
}
@Override
public void recognitionCallback(int modelHandle, RecognitionEvent event) {
public void recognitionCallback(int modelHandle, RecognitionEventSys event) {
synchronized (mActiveModels) {
if (!mActiveModels.contains(modelHandle)) {
// Discard the event.
return;
}
if (!event.recognitionStillActive) {
if (!event.recognitionEvent.recognitionStillActive) {
mActiveModels.remove(modelHandle);
}
// A recognition event must be the last one for its model, unless it indicates that
@@ -255,13 +255,13 @@ public class SoundTriggerHalConcurrentCaptureHandler implements ISoundTriggerHal
}
@Override
public void phraseRecognitionCallback(int modelHandle, PhraseRecognitionEvent event) {
public void phraseRecognitionCallback(int modelHandle, PhraseRecognitionEventSys event) {
synchronized (mActiveModels) {
if (!mActiveModels.contains(modelHandle)) {
// Discard the event.
return;
}
if (!event.common.recognitionStillActive) {
if (!event.phraseRecognitionEvent.common.recognitionStillActive) {
mActiveModels.remove(modelHandle);
}
// A recognition event must be the last one for its model, unless it indicates that

23
services/voiceinteraction/java/com/android/server/soundtrigger_middleware/SoundTriggerHalEnforcer.java
View File

@@ -17,14 +17,14 @@
package com.android.server.soundtrigger_middleware;
import android.media.soundtrigger.ModelParameterRange;
import android.media.soundtrigger.PhraseRecognitionEvent;
import android.media.soundtrigger.PhraseSoundModel;
import android.media.soundtrigger.Properties;
import android.media.soundtrigger.RecognitionConfig;
import android.media.soundtrigger.RecognitionEvent;
import android.media.soundtrigger.RecognitionStatus;
import android.media.soundtrigger.SoundModel;
import android.media.soundtrigger.Status;
import android.media.soundtrigger_middleware.PhraseRecognitionEventSys;
import android.media.soundtrigger_middleware.RecognitionEventSys;
import android.os.DeadObjectException;
import android.os.IBinder;
import android.util.Log;
@@ -253,7 +253,7 @@ public class SoundTriggerHalEnforcer implements ISoundTriggerHal {
}
@Override
public void recognitionCallback(int model, RecognitionEvent event) {
public void recognitionCallback(int model, RecognitionEventSys event) {
synchronized (mModelStates) {
ModelState state = mModelStates.get(model);
if (state == null || state == ModelState.INACTIVE) {
@@ -261,15 +261,16 @@ public class SoundTriggerHalEnforcer implements ISoundTriggerHal {
reboot();
return;
}
if (event.recognitionStillActive && event.status != RecognitionStatus.SUCCESS
&& event.status != RecognitionStatus.FORCED) {
if (event.recognitionEvent.recognitionStillActive
&& event.recognitionEvent.status != RecognitionStatus.SUCCESS
&& event.recognitionEvent.status != RecognitionStatus.FORCED) {
Log.wtfStack(TAG,
"recognitionStillActive is only allowed when the recognition status "
+ "is SUCCESS");
reboot();
return;
}
if (!event.recognitionStillActive) {
if (!event.recognitionEvent.recognitionStillActive) {
mModelStates.replace(model, ModelState.INACTIVE);
}
}
@@ -278,7 +279,7 @@ public class SoundTriggerHalEnforcer implements ISoundTriggerHal {
}
@Override
public void phraseRecognitionCallback(int model, PhraseRecognitionEvent event) {
public void phraseRecognitionCallback(int model, PhraseRecognitionEventSys event) {
synchronized (mModelStates) {
ModelState state = mModelStates.get(model);
if (state == null || state == ModelState.INACTIVE) {
@@ -286,16 +287,16 @@ public class SoundTriggerHalEnforcer implements ISoundTriggerHal {
reboot();
return;
}
if (event.common.recognitionStillActive
&& event.common.status != RecognitionStatus.SUCCESS
&& event.common.status != RecognitionStatus.FORCED) {
if (event.phraseRecognitionEvent.common.recognitionStillActive
&& event.phraseRecognitionEvent.common.status != RecognitionStatus.SUCCESS
&& event.phraseRecognitionEvent.common.status != RecognitionStatus.FORCED) {
Log.wtfStack(TAG,
"recognitionStillActive is only allowed when the recognition status "
+ "is SUCCESS");
reboot();
return;
}
if (!event.common.recognitionStillActive) {
if (!event.phraseRecognitionEvent.common.recognitionStillActive) {
mModelStates.replace(model, ModelState.INACTIVE);
}
}

15
services/voiceinteraction/java/com/android/server/soundtrigger_middleware/SoundTriggerHw2Compat.java
View File

@@ -25,9 +25,12 @@ import android.media.soundtrigger.Properties;
import android.media.soundtrigger.RecognitionConfig;
import android.media.soundtrigger.SoundModel;
import android.media.soundtrigger.Status;
import android.media.soundtrigger_middleware.PhraseRecognitionEventSys;
import android.media.soundtrigger_middleware.RecognitionEventSys;
import android.os.IBinder;
import android.os.IHwBinder;
import android.os.RemoteException;
import android.os.SystemClock;
import android.system.OsConstants;
import android.util.Log;
@@ -570,16 +573,20 @@ final class SoundTriggerHw2Compat implements ISoundTriggerHal {
public void recognitionCallback_2_1(
android.hardware.soundtrigger.V2_1.ISoundTriggerHwCallback.RecognitionEvent event,
int cookie) {
mDelegate.recognitionCallback(event.header.model,
ConversionUtil.hidl2aidlRecognitionEvent(event));
RecognitionEventSys eventSys = new RecognitionEventSys();
eventSys.recognitionEvent = ConversionUtil.hidl2aidlRecognitionEvent(event);
eventSys.halEventReceivedMillis = SystemClock.elapsedRealtime();
mDelegate.recognitionCallback(event.header.model, eventSys);
}
@Override
public void phraseRecognitionCallback_2_1(
android.hardware.soundtrigger.V2_1.ISoundTriggerHwCallback.PhraseRecognitionEvent event,
int cookie) {
mDelegate.phraseRecognitionCallback(event.common.header.model,
ConversionUtil.hidl2aidlPhraseRecognitionEvent(event));
PhraseRecognitionEventSys eventSys = new PhraseRecognitionEventSys();
eventSys.phraseRecognitionEvent = ConversionUtil.hidl2aidlPhraseRecognitionEvent(event);
eventSys.halEventReceivedMillis = SystemClock.elapsedRealtime();
mDelegate.phraseRecognitionCallback(event.common.header.model, eventSys);
}
@Override

13
services/voiceinteraction/java/com/android/server/soundtrigger_middleware/SoundTriggerHw3Compat.java
View File

@@ -29,9 +29,12 @@ import android.media.soundtrigger.RecognitionEvent;
import android.media.soundtrigger.RecognitionStatus;
import android.media.soundtrigger.SoundModel;
import android.media.soundtrigger.Status;
import android.media.soundtrigger_middleware.PhraseRecognitionEventSys;
import android.media.soundtrigger_middleware.RecognitionEventSys;
import android.os.IBinder;
import android.os.RemoteException;
import android.os.ServiceSpecificException;
import android.os.SystemClock;
public class SoundTriggerHw3Compat implements ISoundTriggerHal {
private final @NonNull ISoundTriggerHw mDriver;
@@ -244,14 +247,20 @@ public class SoundTriggerHw3Compat implements ISoundTriggerHal {
public void phraseRecognitionCallback(int model, PhraseRecognitionEvent event) {
// A FORCED status implies that recognition is still active after the event.
event.common.recognitionStillActive |= event.common.status == RecognitionStatus.FORCED;
mDelegate.phraseRecognitionCallback(model, event);
PhraseRecognitionEventSys phraseRecognitionEventSys = new PhraseRecognitionEventSys();
phraseRecognitionEventSys.phraseRecognitionEvent = event;
phraseRecognitionEventSys.halEventReceivedMillis = SystemClock.elapsedRealtimeNanos();
mDelegate.phraseRecognitionCallback(model, phraseRecognitionEventSys);
}
@Override
public void recognitionCallback(int model, RecognitionEvent event) {
// A FORCED status implies that recognition is still active after the event.
event.recognitionStillActive |= event.status == RecognitionStatus.FORCED;
mDelegate.recognitionCallback(model, event);
RecognitionEventSys recognitionEventSys = new RecognitionEventSys();
recognitionEventSys.recognitionEvent = event;
recognitionEventSys.halEventReceivedMillis = SystemClock.elapsedRealtimeNanos();
mDelegate.recognitionCallback(model, recognitionEventSys);
}
@Override

30
services/voiceinteraction/java/com/android/server/soundtrigger_middleware/SoundTriggerMiddlewareLogging.java
View File

@@ -16,12 +16,24 @@
package com.android.server.soundtrigger_middleware;
import static com.android.server.soundtrigger_middleware.SoundTriggerMiddlewareLogging.SessionEvent.Type.*;
import static com.android.server.soundtrigger_middleware.SoundTriggerMiddlewareLogging.SessionEvent.Type.DETACH;
import static com.android.server.soundtrigger_middleware.SoundTriggerMiddlewareLogging.SessionEvent.Type.FORCE_RECOGNITION;
import static com.android.server.soundtrigger_middleware.SoundTriggerMiddlewareLogging.SessionEvent.Type.GET_MODEL_PARAMETER;
import static com.android.server.soundtrigger_middleware.SoundTriggerMiddlewareLogging.SessionEvent.Type.LOAD_MODEL;
import static com.android.server.soundtrigger_middleware.SoundTriggerMiddlewareLogging.SessionEvent.Type.LOAD_PHRASE_MODEL;
import static com.android.server.soundtrigger_middleware.SoundTriggerMiddlewareLogging.SessionEvent.Type.MODEL_UNLOADED;
import static com.android.server.soundtrigger_middleware.SoundTriggerMiddlewareLogging.SessionEvent.Type.MODULE_DIED;
import static com.android.server.soundtrigger_middleware.SoundTriggerMiddlewareLogging.SessionEvent.Type.QUERY_MODEL_PARAMETER;
import static com.android.server.soundtrigger_middleware.SoundTriggerMiddlewareLogging.SessionEvent.Type.RECOGNITION;
import static com.android.server.soundtrigger_middleware.SoundTriggerMiddlewareLogging.SessionEvent.Type.RESOURCES_AVAILABLE;
import static com.android.server.soundtrigger_middleware.SoundTriggerMiddlewareLogging.SessionEvent.Type.SET_MODEL_PARAMETER;
import static com.android.server.soundtrigger_middleware.SoundTriggerMiddlewareLogging.SessionEvent.Type.START_RECOGNITION;
import static com.android.server.soundtrigger_middleware.SoundTriggerMiddlewareLogging.SessionEvent.Type.STOP_RECOGNITION;
import static com.android.server.soundtrigger_middleware.SoundTriggerMiddlewareLogging.SessionEvent.Type.UNLOAD_MODEL;
import static com.android.server.utils.EventLogger.Event.ALOGI;
import static com.android.server.utils.EventLogger.Event.ALOGW;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.content.Context;
import android.media.permission.Identity;
import android.media.permission.IdentityContext;
@@ -29,11 +41,12 @@ import android.media.soundtrigger.ModelParameterRange;
import android.media.soundtrigger.PhraseRecognitionEvent;
import android.media.soundtrigger.PhraseSoundModel;
import android.media.soundtrigger.RecognitionConfig;
import android.media.soundtrigger.RecognitionEvent;
import android.media.soundtrigger.RecognitionStatus;
import android.media.soundtrigger.SoundModel;
import android.media.soundtrigger_middleware.ISoundTriggerCallback;
import android.media.soundtrigger_middleware.ISoundTriggerModule;
import android.media.soundtrigger_middleware.PhraseRecognitionEventSys;
import android.media.soundtrigger_middleware.RecognitionEventSys;
import android.media.soundtrigger_middleware.SoundTriggerModuleDescriptor;
import android.os.BatteryStatsInternal;
import android.os.IBinder;
@@ -45,19 +58,18 @@ import com.android.internal.annotations.VisibleForTesting;
import com.android.internal.util.ArrayUtils;
import com.android.internal.util.LatencyTracker;
import com.android.server.LocalServices;
import com.android.server.utils.EventLogger.Event;
import com.android.server.utils.EventLogger;
import com.android.server.utils.EventLogger.Event;
import java.io.PrintWriter;
import java.util.Arrays;
import java.util.Deque;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Supplier;
import java.util.Deque;
/**
@@ -370,7 +382,7 @@ public class SoundTriggerMiddlewareLogging implements ISoundTriggerMiddlewareInt
}
@Override
public void onRecognition(int modelHandle, RecognitionEvent event, int captureSession)
public void onRecognition(int modelHandle, RecognitionEventSys event, int captureSession)
throws RemoteException {
try {
mBatteryStatsInternalSupplier.get().noteWakingSoundTrigger(
@@ -388,13 +400,13 @@ public class SoundTriggerMiddlewareLogging implements ISoundTriggerMiddlewareInt
}
@Override
public void onPhraseRecognition(int modelHandle, PhraseRecognitionEvent event,
public void onPhraseRecognition(int modelHandle, PhraseRecognitionEventSys event,
int captureSession)
throws RemoteException {
try {
mBatteryStatsInternalSupplier.get().noteWakingSoundTrigger(
SystemClock.elapsedRealtime(), mOriginatorIdentity.uid);
startKeyphraseEventLatencyTracking(event);
startKeyphraseEventLatencyTracking(event.phraseRecognitionEvent);
mCallbackDelegate.onPhraseRecognition(modelHandle, event, captureSession);
mEventLogger.enqueue(SessionEvent.createForVoid(
RECOGNITION, modelHandle, event, captureSession)

13
services/voiceinteraction/java/com/android/server/soundtrigger_middleware/SoundTriggerMiddlewarePermission.java
View File

@@ -27,15 +27,15 @@ import android.media.permission.Identity;
import android.media.permission.IdentityContext;
import android.media.permission.PermissionUtil;
import android.media.soundtrigger.ModelParameterRange;
import android.media.soundtrigger.PhraseRecognitionEvent;
import android.media.soundtrigger.PhraseSoundModel;
import android.media.soundtrigger.RecognitionConfig;
import android.media.soundtrigger.RecognitionEvent;
import android.media.soundtrigger.SoundModel;
import android.media.soundtrigger.Status;
import android.media.soundtrigger_middleware.ISoundTriggerCallback;
import android.media.soundtrigger_middleware.ISoundTriggerMiddlewareService;
import android.media.soundtrigger_middleware.ISoundTriggerModule;
import android.media.soundtrigger_middleware.PhraseRecognitionEventSys;
import android.media.soundtrigger_middleware.RecognitionEventSys;
import android.media.soundtrigger_middleware.SoundTriggerModuleDescriptor;
import android.os.IBinder;
import android.os.RemoteException;
@@ -307,16 +307,15 @@ public class SoundTriggerMiddlewarePermission implements ISoundTriggerMiddleware
}
@Override
public void onRecognition(int modelHandle, RecognitionEvent event, int captureSession)
throws RemoteException {
public void onRecognition(int modelHandle, RecognitionEventSys event,
int captureSession) throws RemoteException {
enforcePermissions("Sound trigger recognition.");
mDelegate.onRecognition(modelHandle, event, captureSession);
}
@Override
public void onPhraseRecognition(int modelHandle, PhraseRecognitionEvent event,
int captureSession)
throws RemoteException {
public void onPhraseRecognition(int modelHandle, PhraseRecognitionEventSys event,
int captureSession) throws RemoteException {
enforcePermissions("Sound trigger phrase recognition.");
mDelegate.onPhraseRecognition(modelHandle, event, captureSession);
}

19
services/voiceinteraction/java/com/android/server/soundtrigger_middleware/SoundTriggerMiddlewareValidation.java
View File

@@ -21,17 +21,17 @@ import android.annotation.Nullable;
import android.media.permission.Identity;
import android.media.permission.IdentityContext;
import android.media.soundtrigger.ModelParameterRange;
import android.media.soundtrigger.PhraseRecognitionEvent;
import android.media.soundtrigger.PhraseSoundModel;
import android.media.soundtrigger.Properties;
import android.media.soundtrigger.RecognitionConfig;
import android.media.soundtrigger.RecognitionEvent;
import android.media.soundtrigger.RecognitionStatus;
import android.media.soundtrigger.SoundModel;
import android.media.soundtrigger.Status;
import android.media.soundtrigger_middleware.ISoundTriggerCallback;
import android.media.soundtrigger_middleware.ISoundTriggerMiddlewareService;
import android.media.soundtrigger_middleware.ISoundTriggerModule;
import android.media.soundtrigger_middleware.PhraseRecognitionEventSys;
import android.media.soundtrigger_middleware.RecognitionEventSys;
import android.media.soundtrigger_middleware.SoundTriggerModuleDescriptor;
import android.os.IBinder;
import android.os.RemoteException;
@@ -710,8 +710,7 @@ public class SoundTriggerMiddlewareValidation implements ISoundTriggerMiddleware
}
}
class CallbackWrapper implements ISoundTriggerCallback,
IBinder.DeathRecipient {
class CallbackWrapper implements ISoundTriggerCallback, IBinder.DeathRecipient {
private final ISoundTriggerCallback mCallback;
CallbackWrapper(ISoundTriggerCallback callback) {
@@ -728,11 +727,11 @@ public class SoundTriggerMiddlewareValidation implements ISoundTriggerMiddleware
}
@Override
public void onRecognition(int modelHandle, @NonNull RecognitionEvent event,
public void onRecognition(int modelHandle, @NonNull RecognitionEventSys event,
int captureSession) {
synchronized (SoundTriggerMiddlewareValidation.this) {
ModelState modelState = mLoadedModels.get(modelHandle);
if (!event.recognitionStillActive) {
if (!event.recognitionEvent.recognitionStillActive) {
modelState.activityState = ModelState.Activity.LOADED;
}
}
@@ -744,7 +743,7 @@ public class SoundTriggerMiddlewareValidation implements ISoundTriggerMiddleware
Log.w(TAG, "Client callback exception.", e);
synchronized (SoundTriggerMiddlewareValidation.this) {
ModelState modelState = mLoadedModels.get(modelHandle);
if (event.status != RecognitionStatus.FORCED) {
if (event.recognitionEvent.status != RecognitionStatus.FORCED) {
modelState.activityState = ModelState.Activity.INTERCEPTED;
// If we failed to deliver an actual event to the client, they would
// never know to restart it whenever circumstances change. Thus, we
@@ -758,10 +757,10 @@ public class SoundTriggerMiddlewareValidation implements ISoundTriggerMiddleware
@Override
public void onPhraseRecognition(int modelHandle,
@NonNull PhraseRecognitionEvent event, int captureSession) {
@NonNull PhraseRecognitionEventSys event, int captureSession) {
synchronized (SoundTriggerMiddlewareValidation.this) {
ModelState modelState = mLoadedModels.get(modelHandle);
if (!event.common.recognitionStillActive) {
if (!event.phraseRecognitionEvent.common.recognitionStillActive) {
modelState.activityState = ModelState.Activity.LOADED;
}
}
@@ -773,7 +772,7 @@ public class SoundTriggerMiddlewareValidation implements ISoundTriggerMiddleware
Log.w(TAG, "Client callback exception.", e);
synchronized (SoundTriggerMiddlewareValidation.this) {
ModelState modelState = mLoadedModels.get(modelHandle);
if (!event.common.recognitionStillActive) {
if (!event.phraseRecognitionEvent.common.recognitionStillActive) {
modelState.activityState = ModelState.Activity.INTERCEPTED;
// If we failed to deliver an actual event to the client, they would
// never know to restart it whenever circumstances change. Thus, we

17
services/voiceinteraction/java/com/android/server/soundtrigger_middleware/SoundTriggerModule.java
View File

@@ -19,16 +19,16 @@ package com.android.server.soundtrigger_middleware;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.media.soundtrigger.ModelParameterRange;
import android.media.soundtrigger.PhraseRecognitionEvent;
import android.media.soundtrigger.PhraseSoundModel;
import android.media.soundtrigger.Properties;
import android.media.soundtrigger.RecognitionConfig;
import android.media.soundtrigger.RecognitionEvent;
import android.media.soundtrigger.SoundModel;
import android.media.soundtrigger.SoundModelType;
import android.media.soundtrigger.Status;
import android.media.soundtrigger_middleware.ISoundTriggerCallback;
import android.media.soundtrigger_middleware.ISoundTriggerModule;
import android.media.soundtrigger_middleware.PhraseRecognitionEventSys;
import android.media.soundtrigger_middleware.RecognitionEventSys;
import android.os.Binder;
import android.os.IBinder;
import android.os.RemoteException;
@@ -499,10 +499,10 @@ class SoundTriggerModule implements IBinder.DeathRecipient, ISoundTriggerHal.Glo
@Override
public void recognitionCallback(int modelHandle,
@NonNull RecognitionEvent recognitionEvent) {
@NonNull RecognitionEventSys event) {
ISoundTriggerCallback callback;
synchronized (SoundTriggerModule.this) {
if (!recognitionEvent.recognitionStillActive) {
if (!event.recognitionEvent.recognitionStillActive) {
setState(ModelState.LOADED);
}
callback = mCallback;
@@ -510,7 +510,7 @@ class SoundTriggerModule implements IBinder.DeathRecipient, ISoundTriggerHal.Glo
// The callback must be invoked outside of the lock.
try {
if (callback != null) {
callback.onRecognition(mHandle, recognitionEvent, mSession.mSessionHandle);
callback.onRecognition(mHandle, event, mSession.mSessionHandle);
}
} catch (RemoteException e) {
// We're not expecting any exceptions here.
@@ -520,10 +520,10 @@ class SoundTriggerModule implements IBinder.DeathRecipient, ISoundTriggerHal.Glo
@Override
public void phraseRecognitionCallback(int modelHandle,
@NonNull PhraseRecognitionEvent phraseRecognitionEvent) {
@NonNull PhraseRecognitionEventSys event) {
ISoundTriggerCallback callback;
synchronized (SoundTriggerModule.this) {
if (!phraseRecognitionEvent.common.recognitionStillActive) {
if (!event.phraseRecognitionEvent.common.recognitionStillActive) {
setState(ModelState.LOADED);
}
callback = mCallback;
@@ -532,8 +532,7 @@ class SoundTriggerModule implements IBinder.DeathRecipient, ISoundTriggerHal.Glo
// The callback must be invoked outside of the lock.
try {
if (callback != null) {
mCallback.onPhraseRecognition(mHandle, phraseRecognitionEvent,
mSession.mSessionHandle);
mCallback.onPhraseRecognition(mHandle, event, mSession.mSessionHandle);
}
} catch (RemoteException e) {
// We're not expecting any exceptions here.