Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge "Add VisaulQueryDetector API"

Browse Source
This commit is contained in:
Charles Chen
2023-01-14 02:58:32 +00:00
committed by Android (Google) Code Review
parent 8017ab3185 28b528f72d
commit a26357b75d
5 changed files with 389 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
core/api/system-current.txt
View File

@@ -12592,11 +12592,28 @@ package android.service.voice {
method public void onQueryRejected() throws java.lang.IllegalStateException;
}
public class VisualQueryDetector {
method public void destroy();
method @RequiresPermission(allOf={android.Manifest.permission.CAMERA, android.Manifest.permission.RECORD_AUDIO}) public boolean startRecognition() throws android.service.voice.HotwordDetector.IllegalDetectorStateException;
method @RequiresPermission(allOf={android.Manifest.permission.CAMERA, android.Manifest.permission.RECORD_AUDIO}) public boolean stopRecognition() throws android.service.voice.HotwordDetector.IllegalDetectorStateException;
method public void updateState(@Nullable android.os.PersistableBundle, @Nullable android.os.SharedMemory) throws android.service.voice.HotwordDetector.IllegalDetectorStateException;
}
public static interface VisualQueryDetector.Callback {
method public void onError();
method public void onQueryDetected(@NonNull String);
method public void onQueryFinished();
method public void onQueryRejected();
method public void onVisualQueryDetectionServiceInitialized(int);
method public void onVisualQueryDetectionServiceRestarted();
}
public class VoiceInteractionService extends android.app.Service {
method @NonNull public final android.service.voice.AlwaysOnHotwordDetector createAlwaysOnHotwordDetector(String, java.util.Locale, android.service.voice.AlwaysOnHotwordDetector.Callback);
method @NonNull @RequiresPermission(android.Manifest.permission.MANAGE_HOTWORD_DETECTION) public final android.service.voice.AlwaysOnHotwordDetector createAlwaysOnHotwordDetector(String, java.util.Locale, @Nullable android.os.PersistableBundle, @Nullable android.os.SharedMemory, android.service.voice.AlwaysOnHotwordDetector.Callback);
method @NonNull @RequiresPermission(android.Manifest.permission.MANAGE_HOTWORD_DETECTION) public final android.service.voice.HotwordDetector createHotwordDetector(@Nullable android.os.PersistableBundle, @Nullable android.os.SharedMemory, @NonNull android.service.voice.HotwordDetector.Callback);
method @NonNull @RequiresPermission("android.permission.MANAGE_VOICE_KEYPHRASES") public final android.media.voice.KeyphraseModelManager createKeyphraseModelManager();
method @NonNull public final android.service.voice.VisualQueryDetector createVisualQueryDetector(@Nullable android.os.PersistableBundle, @Nullable android.os.SharedMemory, @NonNull java.util.concurrent.Executor, @NonNull android.service.voice.VisualQueryDetector.Callback);
}
}

22
core/java/android/service/voice/AbstractDetector.java
View File

@@ -84,7 +84,7 @@ abstract class AbstractDetector implements HotwordDetector {
@Nullable SharedMemory sharedMemory);
/**
* Detect hotword from an externally supplied stream of data.
* Detect from an externally supplied stream of data.
*
* @return {@code true} if the request to start recognition succeeded
*/
@@ -114,7 +114,25 @@ abstract class AbstractDetector implements HotwordDetector {
return true;
}
/** {@inheritDoc} */
/**
* Set configuration and pass read-only data to trusted detection service.
*
* @param options Application configuration data to provide to the
* {@link VisualQueryDetectionService} and {@link HotwordDetectionService}.
* PersistableBundle does not allow any remotable objects or other contents that can be
* used to communicate with other processes.
* @param sharedMemory The unrestricted data blob to provide to the
* {@link VisualQueryDetectionService} and {@link HotwordDetectionService}. Use this to
* provide the hotword models data or other such data to the trusted process.
* @throws IllegalDetectorStateException Thrown when a caller has a target SDK of
* Android Tiramisu or above and attempts to start a recognition when the detector is
* not able based on the state. Because the caller receives updates via an asynchronous
* callback and the state of the detector can change without caller's knowledge, a
* checked exception is thrown.
* @throws IllegalStateException if this {@link HotwordDetector} wasn't specified to use a
* {@link HotwordDetectionService} or {@link VisualQueryDetectionService} when it was
* created.
*/
@Override
public void updateState(@Nullable PersistableBundle options,
@Nullable SharedMemory sharedMemory) throws IllegalDetectorStateException {

31
core/java/android/service/voice/HotwordDetector.java
View File

@@ -35,7 +35,8 @@ import android.util.AndroidException;
import java.io.PrintWriter;
/**
* Basic functionality for sandboxed detectors.
* Basic functionality for sandboxed detectors. This interface will be used by detectors that
* manages their service lifecycle.
*
* @hide
*/
@@ -80,10 +81,21 @@ public interface HotwordDetector {
*/
int DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE = 2;
/**
* Indicates that it is a visual query detector.
*
* @hide
*/
int DETECTOR_TYPE_VISUAL_QUERY_DETECTOR = 3;
/**
* Starts sandboxed detection recognition.
* <p>
* On calling this, the system streams audio from the device microphone to this application's
* If a {@link VisualQueryDetector} calls this method, {@link VisualQueryDetectionService
* #onStartDetection(VisualQueryDetectionService.Callback)} will be called to start detection.
* <p>
* Otherwise if a {@link AlwaysOnHotwordDetector} or {@link SoftwareHotwordDetector} calls this,
* the system streams audio from the device microphone to this application's
* {@link HotwordDetectionService}. Audio is streamed until {@link #stopRecognition()} is
* called.
* <p>
@@ -192,6 +204,8 @@ public interface HotwordDetector {
return "trusted_hotword_dsp";
case DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE:
return "trusted_hotword_software";
case DETECTOR_TYPE_VISUAL_QUERY_DETECTOR:
return "visual_query_detector";
default:
return Integer.toString(detectorType);
}
@@ -244,18 +258,21 @@ public interface HotwordDetector {
void onRejected(@NonNull HotwordRejectedResult result);
/**
* Called when the {@link HotwordDetectionService} is created by the system and given a
* short amount of time to report its initialization state.
* Called when the {@link HotwordDetectionService} or {@link VisualQueryDetectionService} is
* created by the system and given a short amount of time to report their initialization
* state.
*
* @param status Info about initialization state of {@link HotwordDetectionService}; the
* allowed values are {@link SandboxedDetectionServiceBase#INITIALIZATION_STATUS_SUCCESS},
* @param status Info about initialization state of {@link HotwordDetectionService} or
* {@link VisualQueryDetectionService}; allowed values are
* {@link SandboxedDetectionServiceBase#INITIALIZATION_STATUS_SUCCESS},
* 1<->{@link SandboxedDetectionServiceBase#getMaxCustomInitializationStatus()},
* {@link SandboxedDetectionServiceBase#INITIALIZATION_STATUS_UNKNOWN}.
*/
void onHotwordDetectionServiceInitialized(int status);
/**
* Called with the {@link HotwordDetectionService} is restarted.
* Called with the {@link HotwordDetectionService} or {@link VisualQueryDetectionService} is
* restarted.
*
* Clients are expected to call {@link HotwordDetector#updateState} to share the state with
* the newly created service.

241
core/java/android/service/voice/VisualQueryDetector.java Normal file
View File

@@ -0,0 +1,241 @@
/*
* Copyright (C) 2023 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package android.service.voice;
import static android.Manifest.permission.CAMERA;
import static android.Manifest.permission.RECORD_AUDIO;
import android.annotation.CallbackExecutor;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.annotation.RequiresPermission;
import android.annotation.SuppressLint;
import android.annotation.SystemApi;
import android.media.AudioFormat;
import android.os.ParcelFileDescriptor;
import android.os.PersistableBundle;
import android.os.SharedMemory;
import android.util.Slog;
import com.android.internal.app.IVoiceInteractionManagerService;
import java.io.PrintWriter;
import java.util.concurrent.Executor;
import java.util.function.Consumer;
/**
* Manages VisualQueryDetectionService.
*
* This detector provides necessary functionalities to initialize, start, update and destroy a
* {@link VisualQueryDetectionService}.
*
* @hide
**/
@SystemApi
@SuppressLint("NotCloseable")
public class VisualQueryDetector {
private static final String TAG = VisualQueryDetector.class.getSimpleName();
private static final boolean DEBUG = false;
private final Callback mCallback;
private final Executor mExecutor;
private final IVoiceInteractionManagerService mManagerService;
private final VisualQueryDetectorInitializationDelegate mInitializationDelegate;
VisualQueryDetector(
IVoiceInteractionManagerService managerService,
@NonNull @CallbackExecutor Executor executor,
Callback callback) {
mManagerService = managerService;
mCallback = callback;
mExecutor = executor;
mInitializationDelegate = new VisualQueryDetectorInitializationDelegate();
}
/**
* Initialize the {@link VisualQueryDetectionService} by passing configurations and read-only
* data.
*/
void initialize(@Nullable PersistableBundle options, @Nullable SharedMemory sharedMemory) {
mInitializationDelegate.initialize(options, sharedMemory);
}
/**
* Set configuration and pass read-only data to {@link VisualQueryDetectionService}.
*
* @see HotwordDetector#updateState(PersistableBundle, SharedMemory)
*/
public void updateState(@Nullable PersistableBundle options,
@Nullable SharedMemory sharedMemory) throws
HotwordDetector.IllegalDetectorStateException {
mInitializationDelegate.updateState(options, sharedMemory);
}
/**
* On calling this method, {@link VisualQueryDetectionService
* #onStartDetection(VisualQueryDetectionService.Callback)} will be called to start using
* visual signals such as camera frames and microphone audio to perform detection. When user
* attention is captured and the {@link VisualQueryDetectionService} streams queries,
* {@link VisualQueryDetector.Callback#onQueryDetected(String)} is called to control the
* behavior of handling {@code transcribedText}. When the query streaming is finished,
* {@link VisualQueryDetector.Callback#onQueryFinished()} is called. If the current streamed
* query is invalid, {@link VisualQueryDetector.Callback#onQueryRejected()} is called to abandon
* the streamed query.
*
* @see HotwordDetector#startRecognition()
*/
@RequiresPermission(allOf = {CAMERA, RECORD_AUDIO})
public boolean startRecognition() throws HotwordDetector.IllegalDetectorStateException {
if (DEBUG) {
Slog.i(TAG, "#startRecognition");
}
// TODO(b/261783819): Call StartDetection on VisualQueryDetectionService with the system.
return false;
}
/**
* Stops visual query detection recognition.
*
* @see HotwordDetector#stopRecognition()
*/
@RequiresPermission(allOf = {CAMERA, RECORD_AUDIO})
public boolean stopRecognition() throws HotwordDetector.IllegalDetectorStateException {
if (DEBUG) {
Slog.i(TAG, "#stopRecognition");
}
// TODO(b/261783819): Call StopDetection on VisualQueryDetectionService with the system.
return false;
}
/**
* Destroy the current detector.
*
* @see HotwordDetector#destroy()
*/
public void destroy() {
if (DEBUG) {
Slog.i(TAG, "#destroy");
}
mInitializationDelegate.destroy();
}
/** @hide */
public void dump(String prefix, PrintWriter pw) {
// TODO: implement this
}
/** @hide */
public HotwordDetector getInitializationDelegate() {
return mInitializationDelegate;
}
/** @hide */
void registerOnDestroyListener(Consumer<AbstractDetector> onDestroyListener) {
mInitializationDelegate.registerOnDestroyListener(onDestroyListener);
}
/**
* A class that lets a VoiceInteractionService implementation interact with
* visual query detection APIs.
*/
public interface Callback {
/**
* Called when the {@link VisualQueryDetectionService} starts to stream partial queries.
*
* @param partialQuery The partial query in a text form being streamed.
*/
void onQueryDetected(@NonNull String partialQuery);
/**
* Called when the {@link VisualQueryDetectionService} decides to abandon the streamed
* partial queries.
*/
void onQueryRejected();
/**
* Called when the {@link VisualQueryDetectionService} finishes streaming partial queries.
*/
void onQueryFinished();
/**
* Called when the {@link VisualQueryDetectionService} is created by the system and given a
* short amount of time to report its initialization state.
*
* @param status Info about initialization state of {@link VisualQueryDetectionService}; the
* allowed values are {@link SandboxedDetectionServiceBase#INITIALIZATION_STATUS_SUCCESS},
* 1<->{@link SandboxedDetectionServiceBase#getMaxCustomInitializationStatus()},
* {@link SandboxedDetectionServiceBase#INITIALIZATION_STATUS_UNKNOWN}.
*/
void onVisualQueryDetectionServiceInitialized(int status);
/**
* Called with the {@link VisualQueryDetectionService} is restarted.
*
* Clients are expected to call {@link HotwordDetector#updateState} to share the state with
* the newly created service.
*/
void onVisualQueryDetectionServiceRestarted();
/**
* Called when the detection fails due to an error.
*/
//TODO(b/265390855): Replace this callback with the new onError(DetectorError) design.
void onError();
}
private class VisualQueryDetectorInitializationDelegate extends AbstractDetector {
VisualQueryDetectorInitializationDelegate() {
super(mManagerService, null);
}
@Override
void initialize(@Nullable PersistableBundle options, @Nullable SharedMemory sharedMemory) {
//TODO(261783492): call initAndVerify to create VisualQueryDetectionService
// from the system server.
}
@Override
public boolean stopRecognition() throws IllegalDetectorStateException {
//No-op, we only reuse the initialization methods.
return false;
}
@Override
public boolean startRecognition() throws IllegalDetectorStateException {
//No-op, we only reuse the initialization methods.
return false;
}
@Override
public final boolean startRecognition(
@NonNull ParcelFileDescriptor audioStream,
@NonNull AudioFormat audioFormat,
@Nullable PersistableBundle options) throws IllegalDetectorStateException {
//No-op, not supported by VisualQueryDetector as it should be trusted.
return false;
}
@Override
public boolean isUsingSandboxedDetectionService() {
return true;
}
}
}

88
core/java/android/service/voice/VoiceInteractionService.java
View File

@@ -17,6 +17,7 @@
package android.service.voice;
import android.Manifest;
import android.annotation.CallbackExecutor;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.annotation.RequiresPermission;
@@ -58,7 +59,7 @@ import java.util.List;
import java.util.Locale;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.Executor;
/**
* Top-level service of the current global voice interactor, which is providing
* support for hotwording, the back-end of a {@link android.app.VoiceInteractor}, etc.
@@ -178,6 +179,8 @@ public class VoiceInteractionService extends Service {
IVoiceInteractionManagerService mSystemService;
private VisualQueryDetector mActiveVisualQueryDetector;
private final Object mLock = new Object();
private KeyphraseEnrollmentInfo mKeyphraseEnrollmentInfo;
@@ -582,6 +585,85 @@ public class VoiceInteractionService extends Service {
}
}
/**
* Creates a {@link VisualQueryDetector} and initializes the application's
* {@link VisualQueryDetectionService} using {@code options} and {@code sharedMemory}.
*
* <p>To be able to call this, you need to set android:visualQueryDetectionService in the
* android.voice_interaction metadata file to a valid visual query detection service, and set
* android:isolatedProcess="true" in the service's declaration. Otherwise, this throws an
* {@link IllegalStateException}.
*
* <p>Using this has a noticeable impact on battery, since the microphone is kept open
* for the lifetime of the recognition {@link VisualQueryDetector#startRecognition() session}.
*
* @param options Application configuration data to be provided to the
* {@link VisualQueryDetectionService}. PersistableBundle does not allow any remotable objects
* or other contents that can be used to communicate with other processes.
* @param sharedMemory The unrestricted data blob to be provided to the
* {@link VisualQueryDetectionService}. Use this to provide models or other such data to the
* sandboxed process.
* @param callback The callback to notify of detection events.
* @return An instanece of {@link VisualQueryDetector}.
* @throws UnsupportedOperationException if only single detector is supported. Multiple detector
* is only available for apps targeting {@link Build.VERSION_CODES#TIRAMISU} and above.
* @throws IllegalStateException when there is an existing {@link VisualQueryDetector}, or when
* there is a non-trusted hotword detector running.
*
* @hide
*/
// TODO: add MANAGE_HOTWORD_DETECTION permission to protect this API and update java doc.
@SystemApi
@NonNull
public final VisualQueryDetector createVisualQueryDetector(
@Nullable PersistableBundle options,
@Nullable SharedMemory sharedMemory,
@NonNull @CallbackExecutor Executor executor,
@NonNull VisualQueryDetector.Callback callback) {
Objects.requireNonNull(executor);
Objects.requireNonNull(callback);
if (mSystemService == null) {
throw new IllegalStateException("Not available until onReady() is called");
}
synchronized (mLock) {
if (!CompatChanges.isChangeEnabled(MULTIPLE_ACTIVE_HOTWORD_DETECTORS)) {
throw new UnsupportedOperationException("VisualQueryDetector is only available if "
+ "multiple detectors are allowed");
} else {
if (mActiveVisualQueryDetector != null) {
throw new IllegalStateException(
"There is already an active VisualQueryDetector. "
+ "It must be destroyed to create a new one.");
}
for (HotwordDetector detector : mActiveDetectors) {
if (!detector.isUsingSandboxedDetectionService()) {
throw new IllegalStateException(
"It disallows to create trusted and non-trusted detectors "
+ "at the same time.");
}
}
}
VisualQueryDetector visualQueryDetector =
new VisualQueryDetector(mSystemService, executor, callback);
HotwordDetector visualQueryDetectorInitializationDelegate =
visualQueryDetector.getInitializationDelegate();
mActiveDetectors.add(visualQueryDetectorInitializationDelegate);
try {
visualQueryDetector.registerOnDestroyListener(this::onHotwordDetectorDestroyed);
visualQueryDetector.initialize(options, sharedMemory);
} catch (Exception e) {
mActiveDetectors.remove(visualQueryDetectorInitializationDelegate);
visualQueryDetector.destroy();
throw e;
}
mActiveVisualQueryDetector = visualQueryDetector;
return visualQueryDetector;
}
}
/**
* Creates an {@link KeyphraseModelManager} to use for enrolling voice models outside of the
* pre-bundled system voice models.
@@ -637,6 +719,10 @@ public class VoiceInteractionService extends Service {
private void onHotwordDetectorDestroyed(@NonNull HotwordDetector detector) {
synchronized (mLock) {
if (mActiveVisualQueryDetector!= null &&
detector == mActiveVisualQueryDetector.getInitializationDelegate()) {
mActiveVisualQueryDetector = null;
}
mActiveDetectors.remove(detector);
shutdownHotwordDetectionServiceIfRequiredLocked();
}