- * On calling this, the system streams audio from the device microphone to this application's + * If a {@link VisualQueryDetector} calls this method, {@link VisualQueryDetectionService + * #onStartDetection(VisualQueryDetectionService.Callback)} will be called to start detection. + *
+ * Otherwise if a {@link AlwaysOnHotwordDetector} or {@link SoftwareHotwordDetector} calls this, + * the system streams audio from the device microphone to this application's * {@link HotwordDetectionService}. Audio is streamed until {@link #stopRecognition()} is * called. *
@@ -192,6 +204,8 @@ public interface HotwordDetector {
return "trusted_hotword_dsp";
case DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE:
return "trusted_hotword_software";
+ case DETECTOR_TYPE_VISUAL_QUERY_DETECTOR:
+ return "visual_query_detector";
default:
return Integer.toString(detectorType);
}
@@ -244,18 +258,21 @@ public interface HotwordDetector {
void onRejected(@NonNull HotwordRejectedResult result);
/**
- * Called when the {@link HotwordDetectionService} is created by the system and given a
- * short amount of time to report its initialization state.
+ * Called when the {@link HotwordDetectionService} or {@link VisualQueryDetectionService} is
+ * created by the system and given a short amount of time to report their initialization
+ * state.
*
- * @param status Info about initialization state of {@link HotwordDetectionService}; the
- * allowed values are {@link SandboxedDetectionServiceBase#INITIALIZATION_STATUS_SUCCESS},
+ * @param status Info about initialization state of {@link HotwordDetectionService} or
+ * {@link VisualQueryDetectionService}; allowed values are
+ * {@link SandboxedDetectionServiceBase#INITIALIZATION_STATUS_SUCCESS},
* 1<->{@link SandboxedDetectionServiceBase#getMaxCustomInitializationStatus()},
* {@link SandboxedDetectionServiceBase#INITIALIZATION_STATUS_UNKNOWN}.
*/
void onHotwordDetectionServiceInitialized(int status);
/**
- * Called with the {@link HotwordDetectionService} is restarted.
+ * Called with the {@link HotwordDetectionService} or {@link VisualQueryDetectionService} is
+ * restarted.
*
* Clients are expected to call {@link HotwordDetector#updateState} to share the state with
* the newly created service.
diff --git a/core/java/android/service/voice/VisualQueryDetector.java b/core/java/android/service/voice/VisualQueryDetector.java
new file mode 100644
index 0000000000000..241f5bac55ff9
--- /dev/null
+++ b/core/java/android/service/voice/VisualQueryDetector.java
@@ -0,0 +1,241 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.service.voice;
+
+import static android.Manifest.permission.CAMERA;
+import static android.Manifest.permission.RECORD_AUDIO;
+
+import android.annotation.CallbackExecutor;
+import android.annotation.NonNull;
+import android.annotation.Nullable;
+import android.annotation.RequiresPermission;
+import android.annotation.SuppressLint;
+import android.annotation.SystemApi;
+import android.media.AudioFormat;
+import android.os.ParcelFileDescriptor;
+import android.os.PersistableBundle;
+import android.os.SharedMemory;
+import android.util.Slog;
+
+import com.android.internal.app.IVoiceInteractionManagerService;
+
+import java.io.PrintWriter;
+import java.util.concurrent.Executor;
+import java.util.function.Consumer;
+
+/**
+ * Manages VisualQueryDetectionService.
+ *
+ * This detector provides necessary functionalities to initialize, start, update and destroy a
+ * {@link VisualQueryDetectionService}.
+ *
+ * @hide
+ **/
+@SystemApi
+@SuppressLint("NotCloseable")
+public class VisualQueryDetector {
+ private static final String TAG = VisualQueryDetector.class.getSimpleName();
+ private static final boolean DEBUG = false;
+
+ private final Callback mCallback;
+ private final Executor mExecutor;
+ private final IVoiceInteractionManagerService mManagerService;
+ private final VisualQueryDetectorInitializationDelegate mInitializationDelegate;
+
+ VisualQueryDetector(
+ IVoiceInteractionManagerService managerService,
+ @NonNull @CallbackExecutor Executor executor,
+ Callback callback) {
+
+ mManagerService = managerService;
+ mCallback = callback;
+ mExecutor = executor;
+ mInitializationDelegate = new VisualQueryDetectorInitializationDelegate();
+ }
+
+ /**
+ * Initialize the {@link VisualQueryDetectionService} by passing configurations and read-only
+ * data.
+ */
+ void initialize(@Nullable PersistableBundle options, @Nullable SharedMemory sharedMemory) {
+ mInitializationDelegate.initialize(options, sharedMemory);
+ }
+
+ /**
+ * Set configuration and pass read-only data to {@link VisualQueryDetectionService}.
+ *
+ * @see HotwordDetector#updateState(PersistableBundle, SharedMemory)
+ */
+ public void updateState(@Nullable PersistableBundle options,
+ @Nullable SharedMemory sharedMemory) throws
+ HotwordDetector.IllegalDetectorStateException {
+ mInitializationDelegate.updateState(options, sharedMemory);
+ }
+
+
+ /**
+ * On calling this method, {@link VisualQueryDetectionService
+ * #onStartDetection(VisualQueryDetectionService.Callback)} will be called to start using
+ * visual signals such as camera frames and microphone audio to perform detection. When user
+ * attention is captured and the {@link VisualQueryDetectionService} streams queries,
+ * {@link VisualQueryDetector.Callback#onQueryDetected(String)} is called to control the
+ * behavior of handling {@code transcribedText}. When the query streaming is finished,
+ * {@link VisualQueryDetector.Callback#onQueryFinished()} is called. If the current streamed
+ * query is invalid, {@link VisualQueryDetector.Callback#onQueryRejected()} is called to abandon
+ * the streamed query.
+ *
+ * @see HotwordDetector#startRecognition()
+ */
+ @RequiresPermission(allOf = {CAMERA, RECORD_AUDIO})
+ public boolean startRecognition() throws HotwordDetector.IllegalDetectorStateException {
+ if (DEBUG) {
+ Slog.i(TAG, "#startRecognition");
+ }
+ // TODO(b/261783819): Call StartDetection on VisualQueryDetectionService with the system.
+ return false;
+ }
+
+ /**
+ * Stops visual query detection recognition.
+ *
+ * @see HotwordDetector#stopRecognition()
+ */
+ @RequiresPermission(allOf = {CAMERA, RECORD_AUDIO})
+ public boolean stopRecognition() throws HotwordDetector.IllegalDetectorStateException {
+ if (DEBUG) {
+ Slog.i(TAG, "#stopRecognition");
+ }
+ // TODO(b/261783819): Call StopDetection on VisualQueryDetectionService with the system.
+ return false;
+ }
+
+ /**
+ * Destroy the current detector.
+ *
+ * @see HotwordDetector#destroy()
+ */
+ public void destroy() {
+ if (DEBUG) {
+ Slog.i(TAG, "#destroy");
+ }
+ mInitializationDelegate.destroy();
+ }
+
+ /** @hide */
+ public void dump(String prefix, PrintWriter pw) {
+ // TODO: implement this
+ }
+
+ /** @hide */
+ public HotwordDetector getInitializationDelegate() {
+ return mInitializationDelegate;
+ }
+
+ /** @hide */
+ void registerOnDestroyListener(Consumer To be able to call this, you need to set android:visualQueryDetectionService in the
+ * android.voice_interaction metadata file to a valid visual query detection service, and set
+ * android:isolatedProcess="true" in the service's declaration. Otherwise, this throws an
+ * {@link IllegalStateException}.
+ *
+ * Using this has a noticeable impact on battery, since the microphone is kept open
+ * for the lifetime of the recognition {@link VisualQueryDetector#startRecognition() session}.
+ *
+ * @param options Application configuration data to be provided to the
+ * {@link VisualQueryDetectionService}. PersistableBundle does not allow any remotable objects
+ * or other contents that can be used to communicate with other processes.
+ * @param sharedMemory The unrestricted data blob to be provided to the
+ * {@link VisualQueryDetectionService}. Use this to provide models or other such data to the
+ * sandboxed process.
+ * @param callback The callback to notify of detection events.
+ * @return An instanece of {@link VisualQueryDetector}.
+ * @throws UnsupportedOperationException if only single detector is supported. Multiple detector
+ * is only available for apps targeting {@link Build.VERSION_CODES#TIRAMISU} and above.
+ * @throws IllegalStateException when there is an existing {@link VisualQueryDetector}, or when
+ * there is a non-trusted hotword detector running.
+ *
+ * @hide
+ */
+ // TODO: add MANAGE_HOTWORD_DETECTION permission to protect this API and update java doc.
+ @SystemApi
+ @NonNull
+ public final VisualQueryDetector createVisualQueryDetector(
+ @Nullable PersistableBundle options,
+ @Nullable SharedMemory sharedMemory,
+ @NonNull @CallbackExecutor Executor executor,
+ @NonNull VisualQueryDetector.Callback callback) {
+ Objects.requireNonNull(executor);
+ Objects.requireNonNull(callback);
+
+ if (mSystemService == null) {
+ throw new IllegalStateException("Not available until onReady() is called");
+ }
+ synchronized (mLock) {
+ if (!CompatChanges.isChangeEnabled(MULTIPLE_ACTIVE_HOTWORD_DETECTORS)) {
+ throw new UnsupportedOperationException("VisualQueryDetector is only available if "
+ + "multiple detectors are allowed");
+ } else {
+ if (mActiveVisualQueryDetector != null) {
+ throw new IllegalStateException(
+ "There is already an active VisualQueryDetector. "
+ + "It must be destroyed to create a new one.");
+ }
+ for (HotwordDetector detector : mActiveDetectors) {
+ if (!detector.isUsingSandboxedDetectionService()) {
+ throw new IllegalStateException(
+ "It disallows to create trusted and non-trusted detectors "
+ + "at the same time.");
+ }
+ }
+ }
+
+ VisualQueryDetector visualQueryDetector =
+ new VisualQueryDetector(mSystemService, executor, callback);
+ HotwordDetector visualQueryDetectorInitializationDelegate =
+ visualQueryDetector.getInitializationDelegate();
+ mActiveDetectors.add(visualQueryDetectorInitializationDelegate);
+
+ try {
+ visualQueryDetector.registerOnDestroyListener(this::onHotwordDetectorDestroyed);
+ visualQueryDetector.initialize(options, sharedMemory);
+ } catch (Exception e) {
+ mActiveDetectors.remove(visualQueryDetectorInitializationDelegate);
+ visualQueryDetector.destroy();
+ throw e;
+ }
+ mActiveVisualQueryDetector = visualQueryDetector;
+ return visualQueryDetector;
+ }
+ }
+
/**
* Creates an {@link KeyphraseModelManager} to use for enrolling voice models outside of the
* pre-bundled system voice models.
@@ -598,6 +680,10 @@ public class VoiceInteractionService extends Service {
private void onHotwordDetectorDestroyed(@NonNull HotwordDetector detector) {
synchronized (mLock) {
+ if (mActiveVisualQueryDetector!= null &&
+ detector == mActiveVisualQueryDetector.getInitializationDelegate()) {
+ mActiveVisualQueryDetector = null;
+ }
mActiveDetectors.remove(detector);
shutdownHotwordDetectionServiceIfRequiredLocked();
}