Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add system change to enable stop/start recognition

Browse Source 
Add necessary function to enable system server to start VQDS and change
egression states with the callback toggleing.

Bug: 261783496
Test: atest CtsVoiceInteractionTestCases
Change-Id: Id5565eb8138ad48c5141013e725863041b431410
This commit is contained in:
Charles Chen
2023-01-09 06:26:04 +00:00
parent c9e0fd5585
commit 83ed20d223
6 changed files with 230 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
core/java/android/service/voice/IVisualQueryDetectionVoiceInteractionCallback.aidl Normal file
View File

@@ -0,0 +1,48 @@
/*
* Copyright (C) 2023 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package android.service.voice;
import android.media.AudioFormat;
/**
* Callback for returning the detected result from the VisualQueryDetectionService.
*
* @hide
*/
oneway interface IVisualQueryDetectionVoiceInteractionCallback {
/**
* Called when the detected query is streamed
*/
void onQueryDetected(in String partialQuery);
/**
* Called when the detected result is valid.
*/
void onQueryFinished();
/**
* Called when the detected result is invalid.
*/
void onQueryRejected();
/**
* Called when the detection fails due to an error.
*/
void onError();
}

5
core/java/com/android/internal/app/IVoiceInteractionManagerService.aidl
View File

@@ -27,6 +27,7 @@ import android.os.ParcelFileDescriptor;
import android.os.PersistableBundle;
import android.os.RemoteCallback;
import android.os.SharedMemory;
import android.service.voice.IVisualQueryDetectionVoiceInteractionCallback;
import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback;
import android.service.voice.IVoiceInteractionService;
import android.service.voice.IVoiceInteractionSession;
@@ -299,6 +300,10 @@ interface IVoiceInteractionManagerService {
*/
void shutdownHotwordDetectionService();
void startPerceiving(in IVisualQueryDetectionVoiceInteractionCallback callback);
void stopPerceiving();
void startListeningFromMic(
in AudioFormat audioFormat,
in IMicrophoneHotwordDetectionVoiceInteractionCallback callback);

29
services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java
View File

@@ -52,6 +52,7 @@ import android.service.voice.HotwordDetectionService;
import android.service.voice.HotwordDetector;
import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback;
import android.service.voice.ISandboxedDetectionService;
import android.service.voice.IVisualQueryDetectionVoiceInteractionCallback;
import android.service.voice.VisualQueryDetectionService;
import android.service.voice.VoiceInteractionManagerInternal.HotwordDetectionServiceIdentity;
import android.speech.IRecognitionServiceManager;
@@ -290,6 +291,34 @@ final class HotwordDetectionConnection {
session.startListeningFromMicLocked(audioFormat, callback);
}
/**
* This method is only used by VisualQueryDetector.
*/
void startPerceivingLocked(IVisualQueryDetectionVoiceInteractionCallback callback) {
if (DEBUG) {
Slog.d(TAG, "startPerceivingLocked");
}
final VisualQueryDetectorSession session = getVisualQueryDetectorSessionLocked();
if (session == null) {
return;
}
session.startPerceivingLocked(callback);
}
/**
* This method is only used by VisaulQueryDetector.
*/
void stopPerceivingLocked() {
if (DEBUG) {
Slog.d(TAG, "stopPerceivingLocked");
}
final VisualQueryDetectorSession session = getVisualQueryDetectorSessionLocked();
if (session == null) {
return;
}
session.stopPerceivingLocked();
}
public void startListeningFromExternalSourceLocked(
ParcelFileDescriptor audioStream,
AudioFormat audioFormat,

80
services/voiceinteraction/java/com/android/server/voiceinteraction/VisualQueryDetectorSession.java
View File

@@ -24,13 +24,18 @@ import android.media.permission.Identity;
import android.os.IBinder;
import android.os.ParcelFileDescriptor;
import android.os.PersistableBundle;
import android.os.RemoteException;
import android.os.SharedMemory;
import android.service.voice.IDetectorSessionVisualQueryDetectionCallback;
import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback;
import android.service.voice.ISandboxedDetectionService;
import android.service.voice.IVisualQueryDetectionVoiceInteractionCallback;
import android.util.Slog;
import com.android.internal.app.IHotwordRecognitionStatusCallback;
import java.io.PrintWriter;
import java.util.Objects;
import java.util.concurrent.ScheduledExecutorService;
/**
@@ -44,6 +49,8 @@ import java.util.concurrent.ScheduledExecutorService;
final class VisualQueryDetectorSession extends DetectorSession {
private static final String TAG = "VisualQueryDetectorSession";
private boolean mEgressingData;
private boolean mQueryStreaming;
//TODO(b/261783819): Determines actual functionalities, e.g., startRecognition etc.
VisualQueryDetectorSession(
@@ -55,6 +62,8 @@ final class VisualQueryDetectorSession extends DetectorSession {
super(remoteService, lock, context, token, callback,
voiceInteractionServiceUid, voiceInteractorIdentity, scheduledExecutorService,
logging);
mEgressingData = false;
mQueryStreaming = false;
}
@Override
@@ -65,6 +74,77 @@ final class VisualQueryDetectorSession extends DetectorSession {
//TODO(b/261783819): Starts detection in VisualQueryDetectionService.
}
@SuppressWarnings("GuardedBy")
void startPerceivingLocked(IVisualQueryDetectionVoiceInteractionCallback callback) {
if (DEBUG) {
Slog.d(TAG, "startPerceivingLocked");
}
IDetectorSessionVisualQueryDetectionCallback internalCallback =
new IDetectorSessionVisualQueryDetectionCallback.Stub(){
@Override
public void onAttentionGained() {
Slog.v(TAG, "BinderCallback#onAttentionGained");
//TODO check to see if there is an active SysUI listener registered
mEgressingData = true;
}
@Override
public void onAttentionLost() {
Slog.v(TAG, "BinderCallback#onAttentionLost");
//TODO check to see if there is an active SysUI listener registered
mEgressingData = false;
}
@Override
public void onQueryDetected(@NonNull String partialQuery) throws RemoteException {
Objects.requireNonNull(partialQuery);
Slog.v(TAG, "BinderCallback#onQueryDetected");
if (!mEgressingData) {
Slog.v(TAG, "Query should not be egressed within the unattention state.");
return;
}
mQueryStreaming = true;
callback.onQueryDetected(partialQuery);
Slog.i(TAG, "Egressed from visual query detection process.");
}
@Override
public void onQueryFinished() throws RemoteException {
Slog.v(TAG, "BinderCallback#onQueryFinished");
if (!mQueryStreaming) {
Slog.v(TAG, "Query streaming state signal FINISHED is block since there is"
+ " no active query being streamed.");
return;
}
callback.onQueryFinished();
mQueryStreaming = false;
}
@Override
public void onQueryRejected() throws RemoteException {
Slog.v(TAG, "BinderCallback#onQueryRejected");
if (!mQueryStreaming) {
Slog.v(TAG, "Query streaming state signal REJECTED is block since there is"
+ " no active query being streamed.");
return;
}
callback.onQueryRejected();
mQueryStreaming = false;
}
};
mRemoteDetectionService.run(service -> service.detectWithVisualSignals(internalCallback));
}
@SuppressWarnings("GuardedBy")
void stopPerceivingLocked() {
if (DEBUG) {
Slog.d(TAG, "stopPerceivingLocked");
}
mRemoteDetectionService.run(ISandboxedDetectionService::stopDetection);
}
@Override
void startListeningFromExternalSourceLocked(
ParcelFileDescriptor audioStream,

41
services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerService.java
View File

@@ -72,6 +72,7 @@ import android.os.Trace;
import android.os.UserHandle;
import android.provider.Settings;
import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback;
import android.service.voice.IVisualQueryDetectionVoiceInteractionCallback;
import android.service.voice.IVoiceInteractionSession;
import android.service.voice.VoiceInteractionManagerInternal;
import android.service.voice.VoiceInteractionService;
@@ -1313,6 +1314,46 @@ public class VoiceInteractionManagerService extends SystemService {
}
}
@Override
public void startPerceiving(
IVisualQueryDetectionVoiceInteractionCallback callback)
throws RemoteException {
enforceCallingPermission(Manifest.permission.RECORD_AUDIO);
enforceCallingPermission(Manifest.permission.CAMERA);
synchronized (this) {
enforceIsCurrentVoiceInteractionService();
if (mImpl == null) {
Slog.w(TAG, "startPerceiving without running voice interaction service");
return;
}
final long caller = Binder.clearCallingIdentity();
try {
mImpl.startPerceivingLocked(callback);
} finally {
Binder.restoreCallingIdentity(caller);
}
}
}
@Override
public void stopPerceiving() throws RemoteException {
synchronized (this) {
enforceIsCurrentVoiceInteractionService();
if (mImpl == null) {
Slog.w(TAG, "stopPerceiving without running voice interaction service");
return;
}
final long caller = Binder.clearCallingIdentity();
try {
mImpl.stopPerceivingLocked();
} finally {
Binder.restoreCallingIdentity(caller);
}
}
}
@Override
public void startListeningFromMic(
AudioFormat audioFormat,

27
services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerServiceImpl.java
View File

@@ -60,6 +60,7 @@ import android.os.SharedMemory;
import android.os.UserHandle;
import android.service.voice.HotwordDetector;
import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback;
import android.service.voice.IVisualQueryDetectionVoiceInteractionCallback;
import android.service.voice.IVoiceInteractionService;
import android.service.voice.IVoiceInteractionSession;
import android.service.voice.VoiceInteractionService;
@@ -737,6 +738,32 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne
mHotwordDetectionConnection = null;
}
public void startPerceivingLocked(IVisualQueryDetectionVoiceInteractionCallback callback) {
if (DEBUG) {
Slog.d(TAG, "startPerceivingLocked");
}
if (mHotwordDetectionConnection == null) {
// TODO: callback.onError();
return;
}
mHotwordDetectionConnection.startPerceivingLocked(callback);
}
public void stopPerceivingLocked() {
if (DEBUG) {
Slog.d(TAG, "stopPerceivingLocked");
}
if (mHotwordDetectionConnection == null) {
Slog.w(TAG, "stopPerceivingLocked() called but connection isn't established");
return;
}
mHotwordDetectionConnection.stopPerceivingLocked();
}
public void startListeningFromMicLocked(
AudioFormat audioFormat,
IMicrophoneHotwordDetectionVoiceInteractionCallback callback) {