From 78bb22a1c7db2b5d3b100b2d271eeb5c5e411c89 Mon Sep 17 00:00:00 2001 From: Ytai Ben-Tsvi Date: Tue, 12 Oct 2021 16:48:25 -0700 Subject: [PATCH] Support back-to-back soundtrigger recognitions This change allows the HAL driver, at its discretion, indicate that recognition is still active after a success event. This is achieved by an additional flag added to the event. The behavior to support this case has already been in place, for the sake of supporting a FORCED event. This change just generalizes this behavior to be able to cover SUCCESS as well. For b/w compat, when the status is FORCED, we override the new flag with 'true', indicating that recognition is still active. We do not allow the flag to be set for status codes other than FORCED or SUCCESS. Test: atest FrameworksServicesTests:{SoundTriggerMiddlewareImplTest,SoundHw2CompatTest} Test: Manual verification of sound trigger operation by invoking the assistant and now playing multiple times. Bug: 186031938 Change-Id: Ie4edf82607c72ccb0b8d90a828b04c93153ec8f3 --- .../hardware/soundtrigger/ConversionUtil.java | 2 +- .../hardware/soundtrigger/SoundTrigger.java | 72 +++++++++++++++---- .../media/soundtrigger/RecognitionEvent.aidl | 9 +++ .../media/soundtrigger/RecognitionEvent.aidl | 1 + .../ConversionUtil.java | 1 + ...undTriggerHalConcurrentCaptureHandler.java | 6 +- .../SoundTriggerHalEnforcer.java | 30 +++++--- .../SoundTriggerHw3Compat.java | 5 ++ .../SoundTriggerMiddlewareValidation.java | 6 +- .../SoundTriggerModule.java | 5 +- .../SoundHw2CompatTest.java | 38 ++++++++-- .../SoundTriggerMiddlewareImplTest.java | 53 +++++++++----- .../soundtrigger_middleware/TestUtil.java | 17 +++-- .../soundtrigger/SoundTriggerHelper.java | 4 +- 14 files changed, 187 insertions(+), 62 deletions(-) diff --git a/core/java/android/hardware/soundtrigger/ConversionUtil.java b/core/java/android/hardware/soundtrigger/ConversionUtil.java index 35a974b6f57e0..9a2cd0631cc45 100644 --- a/core/java/android/hardware/soundtrigger/ConversionUtil.java +++ b/core/java/android/hardware/soundtrigger/ConversionUtil.java @@ -204,7 +204,7 @@ class ConversionUtil { aidlEvent.status, modelHandle, aidlEvent.captureAvailable, captureSession, aidlEvent.captureDelayMs, aidlEvent.capturePreambleMs, aidlEvent.triggerInData, - audioFormat, aidlEvent.data); + audioFormat, aidlEvent.data, aidlEvent.recognitionStillActive); } public static SoundTrigger.RecognitionEvent aidl2apiPhraseRecognitionEvent( diff --git a/core/java/android/hardware/soundtrigger/SoundTrigger.java b/core/java/android/hardware/soundtrigger/SoundTrigger.java index 163e6f0a01141..b0439d0e52536 100644 --- a/core/java/android/hardware/soundtrigger/SoundTrigger.java +++ b/core/java/android/hardware/soundtrigger/SoundTrigger.java @@ -1175,6 +1175,11 @@ public class SoundTrigger { @UnsupportedAppUsage @NonNull public final byte[] data; + /** + * Is recognition still active after this event. + * @hide + */ + public final boolean recognitionStillActive; /** @hide */ @TestApi @@ -1182,6 +1187,16 @@ public class SoundTrigger { public RecognitionEvent(int status, int soundModelHandle, boolean captureAvailable, int captureSession, int captureDelayMs, int capturePreambleMs, boolean triggerInData, @NonNull AudioFormat captureFormat, @Nullable byte[] data) { + this(status, soundModelHandle, captureAvailable, captureSession, captureDelayMs, + capturePreambleMs, triggerInData, captureFormat, data, + status == RECOGNITION_STATUS_GET_STATE_RESPONSE); + } + + /** @hide */ + public RecognitionEvent(int status, int soundModelHandle, boolean captureAvailable, + int captureSession, int captureDelayMs, int capturePreambleMs, + boolean triggerInData, @NonNull AudioFormat captureFormat, @Nullable byte[] data, + boolean recognitionStillActive) { this.status = status; this.soundModelHandle = soundModelHandle; this.captureAvailable = captureAvailable; @@ -1191,6 +1206,7 @@ public class SoundTrigger { this.triggerInData = triggerInData; this.captureFormat = requireNonNull(captureFormat); this.data = data != null ? data : new byte[0]; + this.recognitionStillActive = recognitionStillActive; } /** @@ -1266,8 +1282,10 @@ public class SoundTrigger { .build(); } byte[] data = in.readBlob(); + boolean recognitionStillActive = in.readBoolean(); return new RecognitionEvent(status, soundModelHandle, captureAvailable, captureSession, - captureDelayMs, capturePreambleMs, triggerInData, captureFormat, data); + captureDelayMs, capturePreambleMs, triggerInData, captureFormat, data, + recognitionStillActive); } /** @hide */ @@ -1293,8 +1311,8 @@ public class SoundTrigger { dest.writeByte((byte)0); } dest.writeBlob(data); + dest.writeBoolean(recognitionStillActive); } - @Override public int hashCode() { final int prime = 31; @@ -1312,6 +1330,7 @@ public class SoundTrigger { result = prime * result + Arrays.hashCode(data); result = prime * result + soundModelHandle; result = prime * result + status; + result = result + (recognitionStillActive ? 1289 : 1291); return result; } @@ -1334,6 +1353,8 @@ public class SoundTrigger { return false; if (!Arrays.equals(data, other.data)) return false; + if (recognitionStillActive != other.recognitionStillActive) + return false; if (soundModelHandle != other.soundModelHandle) return false; if (status != other.status) @@ -1370,7 +1391,9 @@ public class SoundTrigger { (", encoding=" + captureFormat.getEncoding())) + ((captureFormat == null) ? "" : (", channelMask=" + captureFormat.getChannelMask())) - + ", data=" + (data == null ? 0 : data.length) + "]"; + + ", data=" + (data == null ? 0 : data.length) + + ", recognitionStillActive=" + recognitionStillActive + + "]"; } } @@ -1673,11 +1696,21 @@ public class SoundTrigger { @UnsupportedAppUsage(maxTargetSdk = Build.VERSION_CODES.R, trackingBug = 170729553) public KeyphraseRecognitionEvent(int status, int soundModelHandle, boolean captureAvailable, - int captureSession, int captureDelayMs, int capturePreambleMs, - boolean triggerInData, @NonNull AudioFormat captureFormat, @Nullable byte[] data, - @Nullable KeyphraseRecognitionExtra[] keyphraseExtras) { + int captureSession, int captureDelayMs, int capturePreambleMs, + boolean triggerInData, @NonNull AudioFormat captureFormat, @Nullable byte[] data, + @Nullable KeyphraseRecognitionExtra[] keyphraseExtras) { + this(status, soundModelHandle, captureAvailable, captureSession, captureDelayMs, + capturePreambleMs, triggerInData, captureFormat, data, keyphraseExtras, + status == RECOGNITION_STATUS_GET_STATE_RESPONSE); + } + + public KeyphraseRecognitionEvent(int status, int soundModelHandle, boolean captureAvailable, + int captureSession, int captureDelayMs, int capturePreambleMs, + boolean triggerInData, @NonNull AudioFormat captureFormat, @Nullable byte[] data, + @Nullable KeyphraseRecognitionExtra[] keyphraseExtras, + boolean recognitionStillActive) { super(status, soundModelHandle, captureAvailable, captureSession, captureDelayMs, - capturePreambleMs, triggerInData, captureFormat, data); + capturePreambleMs, triggerInData, captureFormat, data, recognitionStillActive); this.keyphraseExtras = keyphraseExtras != null ? keyphraseExtras : new KeyphraseRecognitionExtra[0]; } @@ -1713,11 +1746,12 @@ public class SoundTrigger { .build(); } byte[] data = in.readBlob(); + boolean recognitionStillActive = in.readBoolean(); KeyphraseRecognitionExtra[] keyphraseExtras = in.createTypedArray(KeyphraseRecognitionExtra.CREATOR); return new KeyphraseRecognitionEvent(status, soundModelHandle, captureAvailable, captureSession, captureDelayMs, capturePreambleMs, triggerInData, - captureFormat, data, keyphraseExtras); + captureFormat, data, keyphraseExtras, recognitionStillActive); } @Override @@ -1738,6 +1772,7 @@ public class SoundTrigger { dest.writeByte((byte)0); } dest.writeBlob(data); + dest.writeBoolean(recognitionStillActive); dest.writeTypedArray(keyphraseExtras, flags); } @@ -1782,7 +1817,9 @@ public class SoundTrigger { (", encoding=" + captureFormat.getEncoding())) + ((captureFormat == null) ? "" : (", channelMask=" + captureFormat.getChannelMask())) - + ", data=" + (data == null ? 0 : data.length) + "]"; + + ", data=" + (data == null ? 0 : data.length) + + ", recognitionStillActive=" + recognitionStillActive + + "]"; } } @@ -1798,9 +1835,17 @@ public class SoundTrigger { boolean captureAvailable, int captureSession, int captureDelayMs, int capturePreambleMs, boolean triggerInData, @NonNull AudioFormat captureFormat, @Nullable byte[] data) { - super(status, soundModelHandle, captureAvailable, captureSession, - captureDelayMs, capturePreambleMs, triggerInData, captureFormat, - data); + this(status, soundModelHandle, captureAvailable, captureSession, captureDelayMs, + capturePreambleMs, triggerInData, captureFormat, data, + status == RECOGNITION_STATUS_GET_STATE_RESPONSE); + } + + public GenericRecognitionEvent(int status, int soundModelHandle, + boolean captureAvailable, int captureSession, int captureDelayMs, + int capturePreambleMs, boolean triggerInData, @NonNull AudioFormat captureFormat, + @Nullable byte[] data, boolean recognitionStillActive) { + super(status, soundModelHandle, captureAvailable, captureSession, captureDelayMs, + capturePreambleMs, triggerInData, captureFormat, data, recognitionStillActive); } public static final @android.annotation.NonNull Parcelable.Creator CREATOR @@ -1818,7 +1863,8 @@ public class SoundTrigger { RecognitionEvent event = RecognitionEvent.fromParcel(in); return new GenericRecognitionEvent(event.status, event.soundModelHandle, event.captureAvailable, event.captureSession, event.captureDelayMs, - event.capturePreambleMs, event.triggerInData, event.captureFormat, event.data); + event.capturePreambleMs, event.triggerInData, event.captureFormat, event.data, + event.recognitionStillActive); } @Override diff --git a/media/aidl/android/media/soundtrigger/RecognitionEvent.aidl b/media/aidl/android/media/soundtrigger/RecognitionEvent.aidl index 94668a30606da..6d69038b8dbd1 100644 --- a/media/aidl/android/media/soundtrigger/RecognitionEvent.aidl +++ b/media/aidl/android/media/soundtrigger/RecognitionEvent.aidl @@ -48,4 +48,13 @@ parcelable RecognitionEvent { @nullable AudioConfig audioConfig; /** Additional data. */ byte[] data; + /** + * If true, recognition is still active after this event. + * For compatibility with earlier versions of this data type, when the status field is set to + * RecognitionStatus.FORCED, the value of this field should be treated as 'true', regardless of + * the actual value. + * When the status is RecognitionStatus.ABORTED or RecognitionStatus.FAILURE, this must be set + * to false. + */ + boolean recognitionStillActive; } diff --git a/media/aidl_api/android.media.soundtrigger.types/current/android/media/soundtrigger/RecognitionEvent.aidl b/media/aidl_api/android.media.soundtrigger.types/current/android/media/soundtrigger/RecognitionEvent.aidl index e6cfb6bfc3f6e..0209602a1535b 100644 --- a/media/aidl_api/android.media.soundtrigger.types/current/android/media/soundtrigger/RecognitionEvent.aidl +++ b/media/aidl_api/android.media.soundtrigger.types/current/android/media/soundtrigger/RecognitionEvent.aidl @@ -43,4 +43,5 @@ parcelable RecognitionEvent { boolean triggerInData; @nullable android.media.audio.common.AudioConfig audioConfig; byte[] data; + boolean recognitionStillActive; } diff --git a/services/core/java/com/android/server/soundtrigger_middleware/ConversionUtil.java b/services/core/java/com/android/server/soundtrigger_middleware/ConversionUtil.java index 9ac7e3bd79c18..9f211db5fdc5b 100644 --- a/services/core/java/com/android/server/soundtrigger_middleware/ConversionUtil.java +++ b/services/core/java/com/android/server/soundtrigger_middleware/ConversionUtil.java @@ -310,6 +310,7 @@ class ConversionUtil { for (int i = 0; i < aidlEvent.data.length; ++i) { aidlEvent.data[i] = hidlEvent.data.get(i); } + aidlEvent.recognitionStillActive = aidlEvent.status == RecognitionStatus.FORCED; return aidlEvent; } diff --git a/services/core/java/com/android/server/soundtrigger_middleware/SoundTriggerHalConcurrentCaptureHandler.java b/services/core/java/com/android/server/soundtrigger_middleware/SoundTriggerHalConcurrentCaptureHandler.java index e3ce719627941..990b21c211a3f 100644 --- a/services/core/java/com/android/server/soundtrigger_middleware/SoundTriggerHalConcurrentCaptureHandler.java +++ b/services/core/java/com/android/server/soundtrigger_middleware/SoundTriggerHalConcurrentCaptureHandler.java @@ -226,8 +226,7 @@ public class SoundTriggerHalConcurrentCaptureHandler implements ISoundTriggerHal public void recognitionCallback(int modelHandle, RecognitionEvent event) { // A recognition event must be the last one for its model, unless it is a forced one // (those leave the model active). - mCallbackThread.pushWithDedupe(modelHandle, - event.status != RecognitionStatus.FORCED, + mCallbackThread.pushWithDedupe(modelHandle, !event.recognitionStillActive, () -> mDelegateCallback.recognitionCallback(modelHandle, event)); } @@ -235,8 +234,7 @@ public class SoundTriggerHalConcurrentCaptureHandler implements ISoundTriggerHal public void phraseRecognitionCallback(int modelHandle, PhraseRecognitionEvent event) { // A recognition event must be the last one for its model, unless it is a forced one // (those leave the model active). - mCallbackThread.pushWithDedupe(modelHandle, - event.common.status != RecognitionStatus.FORCED, + mCallbackThread.pushWithDedupe(modelHandle, !event.common.recognitionStillActive, () -> mDelegateCallback.phraseRecognitionCallback(modelHandle, event)); } diff --git a/services/core/java/com/android/server/soundtrigger_middleware/SoundTriggerHalEnforcer.java b/services/core/java/com/android/server/soundtrigger_middleware/SoundTriggerHalEnforcer.java index 6870f4feb3de2..235d10fb5da37 100644 --- a/services/core/java/com/android/server/soundtrigger_middleware/SoundTriggerHalEnforcer.java +++ b/services/core/java/com/android/server/soundtrigger_middleware/SoundTriggerHalEnforcer.java @@ -55,8 +55,7 @@ public class SoundTriggerHalEnforcer implements ISoundTriggerHal { private final ISoundTriggerHal mUnderlying; private final Map mModelStates = new HashMap<>(); - public SoundTriggerHalEnforcer( - ISoundTriggerHal underlying) { + public SoundTriggerHalEnforcer(ISoundTriggerHal underlying) { mUnderlying = underlying; } @@ -239,15 +238,12 @@ public class SoundTriggerHalEnforcer implements ISoundTriggerHal { private class ModelCallbackEnforcer implements ModelCallback { private final ModelCallback mUnderlying; - private ModelCallbackEnforcer( - ModelCallback underlying) { + private ModelCallbackEnforcer(ModelCallback underlying) { mUnderlying = underlying; } @Override public void recognitionCallback(int model, RecognitionEvent event) { - int status = event.status; - synchronized (mModelStates) { ModelState state = mModelStates.get(model); if (state == null || state == ModelState.INACTIVE) { @@ -255,7 +251,15 @@ public class SoundTriggerHalEnforcer implements ISoundTriggerHal { reboot(); return; } - if (status != RecognitionStatus.FORCED) { + if (event.recognitionStillActive && event.status != RecognitionStatus.SUCCESS + && event.status != RecognitionStatus.FORCED) { + Log.wtfStack(TAG, + "recognitionStillActive is only allowed when the recognition status " + + "is SUCCESS"); + reboot(); + return; + } + if (!event.recognitionStillActive) { mModelStates.replace(model, ModelState.INACTIVE); } } @@ -265,7 +269,6 @@ public class SoundTriggerHalEnforcer implements ISoundTriggerHal { @Override public void phraseRecognitionCallback(int model, PhraseRecognitionEvent event) { - int status = event.common.status; synchronized (mModelStates) { ModelState state = mModelStates.get(model); if (state == null || state == ModelState.INACTIVE) { @@ -273,7 +276,16 @@ public class SoundTriggerHalEnforcer implements ISoundTriggerHal { reboot(); return; } - if (status != RecognitionStatus.FORCED) { + if (event.common.recognitionStillActive + && event.common.status != RecognitionStatus.SUCCESS + && event.common.status != RecognitionStatus.FORCED) { + Log.wtfStack(TAG, + "recognitionStillActive is only allowed when the recognition status " + + "is SUCCESS"); + reboot(); + return; + } + if (!event.common.recognitionStillActive) { mModelStates.replace(model, ModelState.INACTIVE); } } diff --git a/services/core/java/com/android/server/soundtrigger_middleware/SoundTriggerHw3Compat.java b/services/core/java/com/android/server/soundtrigger_middleware/SoundTriggerHw3Compat.java index f56475682be0d..0a085ba4265f7 100644 --- a/services/core/java/com/android/server/soundtrigger_middleware/SoundTriggerHw3Compat.java +++ b/services/core/java/com/android/server/soundtrigger_middleware/SoundTriggerHw3Compat.java @@ -26,6 +26,7 @@ import android.media.soundtrigger.PhraseSoundModel; import android.media.soundtrigger.Properties; import android.media.soundtrigger.RecognitionConfig; import android.media.soundtrigger.RecognitionEvent; +import android.media.soundtrigger.RecognitionStatus; import android.media.soundtrigger.SoundModel; import android.media.soundtrigger.Status; import android.os.IBinder; @@ -221,11 +222,15 @@ public class SoundTriggerHw3Compat implements ISoundTriggerHal { @Override public void phraseRecognitionCallback(int model, PhraseRecognitionEvent event) { + // A FORCED status implies that recognition is still active after the event. + event.common.recognitionStillActive |= event.common.status == RecognitionStatus.FORCED; mDelegate.phraseRecognitionCallback(model, event); } @Override public void recognitionCallback(int model, RecognitionEvent event) { + // A FORCED status implies that recognition is still active after the event. + event.recognitionStillActive |= event.status == RecognitionStatus.FORCED; mDelegate.recognitionCallback(model, event); } } diff --git a/services/core/java/com/android/server/soundtrigger_middleware/SoundTriggerMiddlewareValidation.java b/services/core/java/com/android/server/soundtrigger_middleware/SoundTriggerMiddlewareValidation.java index 7b31946e0c9dc..4243fc775ef67 100644 --- a/services/core/java/com/android/server/soundtrigger_middleware/SoundTriggerMiddlewareValidation.java +++ b/services/core/java/com/android/server/soundtrigger_middleware/SoundTriggerMiddlewareValidation.java @@ -731,7 +731,7 @@ public class SoundTriggerMiddlewareValidation implements ISoundTriggerMiddleware int captureSession) { synchronized (SoundTriggerMiddlewareValidation.this) { ModelState modelState = mLoadedModels.get(modelHandle); - if (event.status != RecognitionStatus.FORCED) { + if (!event.recognitionStillActive) { modelState.activityState = ModelState.Activity.LOADED; } } @@ -760,7 +760,7 @@ public class SoundTriggerMiddlewareValidation implements ISoundTriggerMiddleware @NonNull PhraseRecognitionEvent event, int captureSession) { synchronized (SoundTriggerMiddlewareValidation.this) { ModelState modelState = mLoadedModels.get(modelHandle); - if (event.common.status != RecognitionStatus.FORCED) { + if (!event.common.recognitionStillActive) { modelState.activityState = ModelState.Activity.LOADED; } } @@ -772,7 +772,7 @@ public class SoundTriggerMiddlewareValidation implements ISoundTriggerMiddleware Log.w(TAG, "Client callback exception.", e); synchronized (SoundTriggerMiddlewareValidation.this) { ModelState modelState = mLoadedModels.get(modelHandle); - if (event.common.status != RecognitionStatus.FORCED) { + if (!event.common.recognitionStillActive) { modelState.activityState = ModelState.Activity.INTERCEPTED; // If we failed to deliver an actual event to the client, they would // never know to restart it whenever circumstances change. Thus, we diff --git a/services/core/java/com/android/server/soundtrigger_middleware/SoundTriggerModule.java b/services/core/java/com/android/server/soundtrigger_middleware/SoundTriggerModule.java index f2111581d3404..934b0e46ee957 100644 --- a/services/core/java/com/android/server/soundtrigger_middleware/SoundTriggerModule.java +++ b/services/core/java/com/android/server/soundtrigger_middleware/SoundTriggerModule.java @@ -24,7 +24,6 @@ import android.media.soundtrigger.PhraseSoundModel; import android.media.soundtrigger.Properties; import android.media.soundtrigger.RecognitionConfig; import android.media.soundtrigger.RecognitionEvent; -import android.media.soundtrigger.RecognitionStatus; import android.media.soundtrigger.SoundModel; import android.media.soundtrigger.Status; import android.media.soundtrigger_middleware.ISoundTriggerCallback; @@ -461,7 +460,7 @@ class SoundTriggerModule implements IBinder.DeathRecipient, ISoundTriggerHal.Glo @NonNull RecognitionEvent recognitionEvent) { ISoundTriggerCallback callback; synchronized (SoundTriggerModule.this) { - if (recognitionEvent.status != RecognitionStatus.FORCED) { + if (!recognitionEvent.recognitionStillActive) { setState(ModelState.LOADED); } callback = mCallback; @@ -482,7 +481,7 @@ class SoundTriggerModule implements IBinder.DeathRecipient, ISoundTriggerHal.Glo @NonNull PhraseRecognitionEvent phraseRecognitionEvent) { ISoundTriggerCallback callback; synchronized (SoundTriggerModule.this) { - if (phraseRecognitionEvent.common.status != RecognitionStatus.FORCED) { + if (!phraseRecognitionEvent.common.recognitionStillActive) { setState(ModelState.LOADED); } callback = mCallback; diff --git a/services/tests/servicestests/src/com/android/server/soundtrigger_middleware/SoundHw2CompatTest.java b/services/tests/servicestests/src/com/android/server/soundtrigger_middleware/SoundHw2CompatTest.java index 3f8cf9cc416f5..16cfd13b18c22 100644 --- a/services/tests/servicestests/src/com/android/server/soundtrigger_middleware/SoundHw2CompatTest.java +++ b/services/tests/servicestests/src/com/android/server/soundtrigger_middleware/SoundHw2CompatTest.java @@ -717,7 +717,8 @@ public class SoundHw2CompatTest { hwCallback.recognitionCallback(TestUtil.createRecognitionEvent_2_0(handle, status), 99); mCanonical.flushCallbacks(); verify(canonicalCallback).recognitionCallback(eq(handle), eventCaptor.capture()); - TestUtil.validateRecognitionEvent(eventCaptor.getValue(), RecognitionStatus.ABORTED); + TestUtil.validateRecognitionEvent(eventCaptor.getValue(), RecognitionStatus.ABORTED, + false); } { @@ -732,7 +733,7 @@ public class SoundHw2CompatTest { mCanonical.flushCallbacks(); verify(canonicalCallback).phraseRecognitionCallback(eq(handle), eventCaptor.capture()); TestUtil.validatePhraseRecognitionEvent(eventCaptor.getValue(), - RecognitionStatus.SUCCESS); + RecognitionStatus.SUCCESS, false); } verifyNoMoreInteractions(canonicalCallback); clearInvocations(canonicalCallback); @@ -752,7 +753,22 @@ public class SoundHw2CompatTest { 99); mCanonical.flushCallbacks(); verify(canonicalCallback).recognitionCallback(eq(handle), eventCaptor.capture()); - TestUtil.validateRecognitionEvent(eventCaptor.getValue(), RecognitionStatus.ABORTED); + TestUtil.validateRecognitionEvent(eventCaptor.getValue(), RecognitionStatus.ABORTED, + false); + } + + { + final int handle = 87; + final int status = 3; // FORCED; + ArgumentCaptor eventCaptor = ArgumentCaptor.forClass( + RecognitionEvent.class); + + hwCallback.recognitionCallback_2_1(TestUtil.createRecognitionEvent_2_1(handle, status), + 99); + mCanonical.flushCallbacks(); + verify(canonicalCallback).recognitionCallback(eq(handle), eventCaptor.capture()); + TestUtil.validateRecognitionEvent(eventCaptor.getValue(), RecognitionStatus.FORCED, + true); } { @@ -767,7 +783,21 @@ public class SoundHw2CompatTest { mCanonical.flushCallbacks(); verify(canonicalCallback).phraseRecognitionCallback(eq(handle), eventCaptor.capture()); TestUtil.validatePhraseRecognitionEvent(eventCaptor.getValue(), - RecognitionStatus.SUCCESS); + RecognitionStatus.SUCCESS, false); + } + + { + final int handle = 102; + final int status = 3; // FORCED; + ArgumentCaptor eventCaptor = ArgumentCaptor.forClass( + PhraseRecognitionEvent.class); + + hwCallback.phraseRecognitionCallback_2_1( + TestUtil.createPhraseRecognitionEvent_2_1(handle, status), 99); + mCanonical.flushCallbacks(); + verify(canonicalCallback).phraseRecognitionCallback(eq(handle), eventCaptor.capture()); + TestUtil.validatePhraseRecognitionEvent(eventCaptor.getValue(), + RecognitionStatus.FORCED, true); } verifyNoMoreInteractions(canonicalCallback); clearInvocations(canonicalCallback); diff --git a/services/tests/servicestests/src/com/android/server/soundtrigger_middleware/SoundTriggerMiddlewareImplTest.java b/services/tests/servicestests/src/com/android/server/soundtrigger_middleware/SoundTriggerMiddlewareImplTest.java index 1daf8317c5415..0eba6a335d006 100644 --- a/services/tests/servicestests/src/com/android/server/soundtrigger_middleware/SoundTriggerMiddlewareImplTest.java +++ b/services/tests/servicestests/src/com/android/server/soundtrigger_middleware/SoundTriggerMiddlewareImplTest.java @@ -26,6 +26,7 @@ import static org.mockito.Mockito.clearInvocations; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -286,16 +287,31 @@ public class SoundTriggerMiddlewareImplTest { // Initiate a recognition. startRecognition(module, handle, hwHandle); - // Signal a capture from the driver. - RecognitionEvent event = hwCallback.sendRecognitionEvent(hwHandle, - RecognitionStatus.SUCCESS); + { + // Signal a capture from the driver (with "still active"). + RecognitionEvent event = hwCallback.sendRecognitionEvent(hwHandle, + RecognitionStatus.SUCCESS, true); - ArgumentCaptor eventCaptor = ArgumentCaptor.forClass( - RecognitionEvent.class); - verify(callback).onRecognition(eq(handle), eventCaptor.capture(), eq(101)); + ArgumentCaptor eventCaptor = ArgumentCaptor.forClass( + RecognitionEvent.class); + verify(callback).onRecognition(eq(handle), eventCaptor.capture(), eq(101)); - // Validate the event. - assertEquals(event, eventCaptor.getValue()); + // Validate the event. + assertEquals(event, eventCaptor.getValue()); + } + + { + // Signal a capture from the driver (without "still active"). + RecognitionEvent event = hwCallback.sendRecognitionEvent(hwHandle, + RecognitionStatus.SUCCESS, false); + + ArgumentCaptor eventCaptor = ArgumentCaptor.forClass( + RecognitionEvent.class); + verify(callback, times(2)).onRecognition(eq(handle), eventCaptor.capture(), eq(101)); + + // Validate the event. + assertEquals(event, eventCaptor.getValue()); + } // Unload the model. unloadModel(module, handle, hwHandle); @@ -318,7 +334,7 @@ public class SoundTriggerMiddlewareImplTest { // Signal a capture from the driver. PhraseRecognitionEvent event = hwCallback.sendPhraseRecognitionEvent(hwHandle, - RecognitionStatus.SUCCESS); + RecognitionStatus.SUCCESS, false); ArgumentCaptor eventCaptor = ArgumentCaptor.forClass( PhraseRecognitionEvent.class); @@ -352,7 +368,7 @@ public class SoundTriggerMiddlewareImplTest { // Signal a capture from the driver. RecognitionEvent event = hwCallback.sendRecognitionEvent(hwHandle, - RecognitionStatus.FORCED); + RecognitionStatus.FORCED, true); ArgumentCaptor eventCaptor = ArgumentCaptor.forClass( RecognitionEvent.class); @@ -420,7 +436,7 @@ public class SoundTriggerMiddlewareImplTest { // Signal a capture from the driver. PhraseRecognitionEvent event = hwCallback.sendPhraseRecognitionEvent(hwHandle, - RecognitionStatus.FORCED); + RecognitionStatus.FORCED, true); ArgumentCaptor eventCaptor = ArgumentCaptor.forClass( PhraseRecognitionEvent.class); @@ -484,7 +500,7 @@ public class SoundTriggerMiddlewareImplTest { startRecognition(module, handle, hwHandle); // Abort. - hwCallback.sendRecognitionEvent(hwHandle, RecognitionStatus.ABORTED); + hwCallback.sendRecognitionEvent(hwHandle, RecognitionStatus.ABORTED, false); ArgumentCaptor eventCaptor = ArgumentCaptor.forClass( RecognitionEvent.class); @@ -514,7 +530,7 @@ public class SoundTriggerMiddlewareImplTest { startRecognition(module, handle, hwHandle); // Abort. - hwCallback.sendPhraseRecognitionEvent(hwHandle, RecognitionStatus.ABORTED); + hwCallback.sendPhraseRecognitionEvent(hwHandle, RecognitionStatus.ABORTED, false); ArgumentCaptor eventCaptor = ArgumentCaptor.forClass( PhraseRecognitionEvent.class); @@ -604,15 +620,18 @@ public class SoundTriggerMiddlewareImplTest { mCallback = callback; } - private RecognitionEvent sendRecognitionEvent(int hwHandle, @RecognitionStatus int status) { - RecognitionEvent event = TestUtil.createRecognitionEvent(status); + private RecognitionEvent sendRecognitionEvent(int hwHandle, @RecognitionStatus int status, + boolean recognitionStillActive) { + RecognitionEvent event = TestUtil.createRecognitionEvent(status, + recognitionStillActive); mCallback.recognitionCallback(hwHandle, event); return event; } private PhraseRecognitionEvent sendPhraseRecognitionEvent(int hwHandle, - @RecognitionStatus int status) { - PhraseRecognitionEvent event = TestUtil.createPhraseRecognitionEvent(status); + @RecognitionStatus int status, boolean recognitionStillActive) { + PhraseRecognitionEvent event = TestUtil.createPhraseRecognitionEvent(status, + recognitionStillActive); mCallback.phraseRecognitionCallback(hwHandle, event); return event; } diff --git a/services/tests/servicestests/src/com/android/server/soundtrigger_middleware/TestUtil.java b/services/tests/servicestests/src/com/android/server/soundtrigger_middleware/TestUtil.java index 43d646a7f9bd2..e687a2adeb396 100644 --- a/services/tests/servicestests/src/com/android/server/soundtrigger_middleware/TestUtil.java +++ b/services/tests/servicestests/src/com/android/server/soundtrigger_middleware/TestUtil.java @@ -330,7 +330,8 @@ class TestUtil { return format; } - static RecognitionEvent createRecognitionEvent(@RecognitionStatus int status) { + static RecognitionEvent createRecognitionEvent(@RecognitionStatus int status, + boolean recognitionStillActive) { RecognitionEvent event = new RecognitionEvent(); event.status = status; event.type = SoundModelType.GENERIC; @@ -346,6 +347,7 @@ class TestUtil { event.audioConfig.base.format = createAudioFormatMp3(); //event.audioConfig.offloadInfo is irrelevant. event.data = new byte[]{31, 32, 33}; + event.recognitionStillActive = recognitionStillActive; return event; } @@ -360,7 +362,8 @@ class TestUtil { return halEvent; } - static void validateRecognitionEvent(RecognitionEvent event, @RecognitionStatus int status) { + static void validateRecognitionEvent(RecognitionEvent event, @RecognitionStatus int status, + boolean recognitionStillActive) { assertEquals(status, event.status); assertEquals(SoundModelType.GENERIC, event.type); assertTrue(event.captureAvailable); @@ -372,11 +375,13 @@ class TestUtil { event.audioConfig.base.channelMask); assertEquals(createAudioFormatMp3(), event.audioConfig.base.format); assertArrayEquals(new byte[]{31, 32, 33}, event.data); + assertEquals(recognitionStillActive, event.recognitionStillActive); } - static PhraseRecognitionEvent createPhraseRecognitionEvent(@RecognitionStatus int status) { + static PhraseRecognitionEvent createPhraseRecognitionEvent(@RecognitionStatus int status, + boolean recognitionStillActive) { PhraseRecognitionEvent event = new PhraseRecognitionEvent(); - event.common = createRecognitionEvent(status); + event.common = createRecognitionEvent(status, recognitionStillActive); PhraseRecognitionExtra extra = new PhraseRecognitionExtra(); extra.id = 123; @@ -434,8 +439,8 @@ class TestUtil { } static void validatePhraseRecognitionEvent(PhraseRecognitionEvent event, - @RecognitionStatus int status) { - validateRecognitionEvent(event.common, status); + @RecognitionStatus int status, boolean recognitionStillActive) { + validateRecognitionEvent(event.common, status, recognitionStillActive); assertEquals(1, event.phraseExtras.length); assertEquals(123, event.phraseExtras[0].id); diff --git a/services/voiceinteraction/java/com/android/server/soundtrigger/SoundTriggerHelper.java b/services/voiceinteraction/java/com/android/server/soundtrigger/SoundTriggerHelper.java index be37a9139bb4d..24ce7e76a49c6 100644 --- a/services/voiceinteraction/java/com/android/server/soundtrigger/SoundTriggerHelper.java +++ b/services/voiceinteraction/java/com/android/server/soundtrigger/SoundTriggerHelper.java @@ -830,7 +830,7 @@ public class SoundTriggerHelper implements SoundTrigger.StatusListener { return; } - if (event.status != SoundTrigger.RECOGNITION_STATUS_GET_STATE_RESPONSE) { + if (!event.recognitionStillActive) { model.setStopped(); } @@ -971,7 +971,7 @@ public class SoundTriggerHelper implements SoundTrigger.StatusListener { return; } - if (event.status != SoundTrigger.RECOGNITION_STATUS_GET_STATE_RESPONSE) { + if (!event.recognitionStillActive) { modelData.setStopped(); }