Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

API feedback: remove callback for VisualQueryDetectionService

Browse Source 
We may not like the design to send things back to the system server
with callback since VisualQueryDetectionService does not perform
per-session based detection on external signals.

Bug: 266564316
Test: atest CtsVoiceInteractionTestCases
Change-Id: I3ea140902bb83645f5dc5d3bcc0af0b3232f74b1
This commit is contained in:
Charles Chen
2023-02-21 23:42:16 +00:00
parent c949def472
commit 334ef4eef2
3 changed files with 112 additions and 126 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
core/api/system-current.txt
View File

@@ -13255,22 +13255,18 @@ package android.service.voice {
public abstract class VisualQueryDetectionService extends android.app.Service implements android.service.voice.SandboxedDetectionServiceBase {
ctor public VisualQueryDetectionService();
method public final void finishQuery() throws java.lang.IllegalStateException;
method public final void gainedAttention();
method public final void lostAttention();
method @Nullable public android.os.IBinder onBind(@NonNull android.content.Intent);
method public void onStartDetection(@NonNull android.service.voice.VisualQueryDetectionService.Callback);
method public void onStartDetection();
method public void onStopDetection();
method public void onUpdateState(@Nullable android.os.PersistableBundle, @Nullable android.os.SharedMemory, long, @Nullable java.util.function.IntConsumer);
method public final void rejectQuery() throws java.lang.IllegalStateException;
method public final void streamQuery(@NonNull String) throws java.lang.IllegalStateException;
field public static final String SERVICE_INTERFACE = "android.service.voice.VisualQueryDetectionService";
}
public static final class VisualQueryDetectionService.Callback {
ctor public VisualQueryDetectionService.Callback();
method public void onAttentionGained();
method public void onAttentionLost();
method public void onQueryDetected(@NonNull String) throws java.lang.IllegalStateException;
method public void onQueryFinished() throws java.lang.IllegalStateException;
method public void onQueryRejected() throws java.lang.IllegalStateException;
}
public final class VisualQueryDetectionServiceFailure extends android.service.voice.DetectorFailure {
method public int getErrorCode();
method public int getSuggestedAction();

214
core/java/android/service/voice/VisualQueryDetectionService.java
View File

@@ -49,7 +49,7 @@ import java.util.function.IntConsumer;
* {@link VoiceInteractionService#createVisualQueryDetector(PersistableBundle, SharedMemory,
* Executor, VisualQueryDetector.Callback)} is called, the system will bind the application's
* {@link VisualQueryDetectionService}. When requested from {@link VoiceInteractionService}, the
* system calls into the {@link VisualQueryDetectionService#onStartDetection(Callback)} to enable
* system calls into the {@link VisualQueryDetectionService#onStartDetection()} to enable
* detection. This method MUST be implemented to support visual query detection service.
*
* Note: Methods in this class may be called concurrently.
@@ -78,6 +78,8 @@ public abstract class VisualQueryDetectionService extends Service
/** @hide */
public static final String KEY_INITIALIZATION_STATUS = "initialization_status";
private IDetectorSessionVisualQueryDetectionCallback mRemoteCallback = null;
private final ISandboxedDetectionService mInterface = new ISandboxedDetectionService.Stub() {
@@ -85,7 +87,8 @@ public abstract class VisualQueryDetectionService extends Service
public void detectWithVisualSignals(
IDetectorSessionVisualQueryDetectionCallback callback) {
Log.v(TAG, "#detectWithVisualSignals");
VisualQueryDetectionService.this.onStartDetection(new Callback(callback));
mRemoteCallback = callback;
VisualQueryDetectionService.this.onStartDetection();
}
@Override
@@ -178,16 +181,41 @@ public abstract class VisualQueryDetectionService extends Service
/**
* This is called after the service is set up and the client should open the camera and the
* microphone to start recognition.
*
* Called when the {@link VoiceInteractionService} requests that this service
* {@link HotwordDetector#startRecognition()} start recognition on audio coming directly
* microphone to start recognition. When the {@link VoiceInteractionService} requests that this
* service {@link HotwordDetector#startRecognition()} start recognition on audio coming directly
* from the device microphone.
*
* @param callback The callback to use for responding to the detection request.
*
* <p>
* Signal senders that return attention and query results are also expected to be called in this
* method according to the detection outcomes.
* <p>
* On successful user attention, developers should call
* {@link VisualQueryDetectionService#gainedAttention()} to enable the streaming of the query.
* <p>
* On user attention is lost, developers should call
* {@link VisualQueryDetectionService#lostAttention()} to disable the streaming of the query.
* <p>
* On query is detected and ready to stream, developers should call
* {@link VisualQueryDetectionService#streamQuery(String)} to return detected query to the
* {@link VisualQueryDetector}.
* <p>
* On streamed query should be rejected, clients should call
* {@link VisualQueryDetectionService#rejectQuery()} to abandon query streamed to the
* {@link VisualQueryDetector}.
* <p>
* On streamed query is finished, clients should call
* {@link VisualQueryDetectionService#finishQuery()} to complete query streamed to
* {@link VisualQueryDetector}.
* <p>
* Before a call for {@link VisualQueryDetectionService#streamQuery(String)} is triggered,
* {@link VisualQueryDetectionService#gainedAttention()} MUST be called to enable the streaming
* of query. A query streaming is also expected to be finished by calling either
* {@link VisualQueryDetectionService#finishQuery()} or
* {@link VisualQueryDetectionService#rejectQuery()} before a new query should start streaming.
* When the service enters the state where query streaming should be disabled,
* {@link VisualQueryDetectionService#lostAttention()} MUST be called to block unnecessary
* streaming.
*/
public void onStartDetection(@NonNull Callback callback) {
public void onStartDetection() {
throw new UnsupportedOperationException();
}
@@ -199,118 +227,78 @@ public abstract class VisualQueryDetectionService extends Service
}
/**
* Callback for sending out signals and returning query results.
*
* On successful user attention, developers should call {@link Callback#onAttentionGained()}
* to enable the streaming of the query.
* <p>
* On user attention is lost, developers should call {@link Callback#onAttentionLost()} to
* disable the streaming of the query.
* <p>
* On query is detected and ready to stream, developers should call
* {@link Callback#onQueryDetected(String)} to return detected query to the
* {@link VisualQueryDetector}.
* <p>
* On streamed query should be rejected, clients should call {@link Callback#onQueryRejected()}
* to abandon query streamed to the {@link VisualQueryDetector}.
* <p>
* On streamed query is finished, clients should call {@link Callback#onQueryFinished()} to
* complete query streamed to {@link VisualQueryDetector}.
* <p>
* Before a call for {@link Callback#onQueryDetected(String)} is triggered,
* {@link Callback#onAttentionGained()} MUST be called to enable the streaming of query. A query
* streaming is also expected to be finished by calling either
* {@link Callback#onQueryFinished()} or {@link Callback#onQueryRejected()} before a new query
* should start streaming. When the service enters the state where query streaming should be
* disabled, {@link Callback#onAttentionLost()} MUST be called to block unnecessary streaming.
* Informs the system that the user attention is gained so queries can be streamed.
*/
public static final class Callback {
// TODO: consider making the constructor a test api for testing purpose
public Callback() {
mRemoteCallback = null;
public final void gainedAttention() {
try {
mRemoteCallback.onAttentionGained();
} catch (RemoteException e) {
throw e.rethrowFromSystemServer();
}
}
private final IDetectorSessionVisualQueryDetectionCallback mRemoteCallback;
private Callback(IDetectorSessionVisualQueryDetectionCallback remoteCallback) {
mRemoteCallback = remoteCallback;
/**
* Informs the system that the user attention is lost to stop streaming.
*/
public final void lostAttention() {
try {
mRemoteCallback.onAttentionLost();
} catch (RemoteException e) {
throw e.rethrowFromSystemServer();
}
}
/**
* Informs attention listener that the user attention is gained.
*/
public void onAttentionGained() {
try {
mRemoteCallback.onAttentionGained();
} catch (RemoteException e) {
throw e.rethrowFromSystemServer();
}
/**
* Informs the {@link VisualQueryDetector} with the text content being captured about the
* query from the audio source. {@code partialQuery} is provided to the
* {@link VisualQueryDetector}. This method is expected to be only triggered if
* {@link VisualQueryDetectionService#gainedAttention()} is called to put the service into the
* attention gained state.
*
* @param partialQuery Partially detected query in string.
* @throws IllegalStateException if method called without attention gained.
*/
public final void streamQuery(@NonNull String partialQuery) throws IllegalStateException {
Objects.requireNonNull(partialQuery);
try {
mRemoteCallback.onQueryDetected(partialQuery);
} catch (RemoteException e) {
throw new IllegalStateException("#streamQuery must be only be triggered after "
+ "calling #gainedAttention to be in the attention gained state.");
}
}
/**
* Informs attention listener that the user attention is lost.
*/
public void onAttentionLost() {
try {
mRemoteCallback.onAttentionLost();
} catch (RemoteException e) {
throw e.rethrowFromSystemServer();
}
/**
* Informs the {@link VisualQueryDetector} to abandon the streamed partial query that has
* been sent to {@link VisualQueryDetector}.This method is expected to be only triggered if
* {@link VisualQueryDetectionService#streamQuery(String)} is called to put the service into
* the query streaming state.
*
* @throws IllegalStateException if method called without query streamed.
*/
public final void rejectQuery() throws IllegalStateException {
try {
mRemoteCallback.onQueryRejected();
} catch (RemoteException e) {
throw new IllegalStateException("#rejectQuery must be only be triggered after "
+ "calling #streamQuery to be in the query streaming state.");
}
}
/**
* Informs the {@link VisualQueryDetector} with the text content being captured about the
* query from the audio source. {@code partialQuery} is provided to the
* {@link VisualQueryDetector}. This method is expected to be only triggered if
* {@link Callback#onAttentionGained()} is called to put the service into the attention
* gained state.
*
* @param partialQuery Partially detected query in string.
* @throws IllegalStateException if method called without attention gained.
*/
public void onQueryDetected(@NonNull String partialQuery) throws IllegalStateException {
Objects.requireNonNull(partialQuery);
try {
mRemoteCallback.onQueryDetected(partialQuery);
} catch (RemoteException e) {
throw new IllegalStateException("#onQueryDetected must be only be triggered after "
+ "calling #onAttentionGained to be in the attention gained state.");
}
}
/**
* Informs the {@link VisualQueryDetector} to abandon the streamed partial query that has
* been sent to {@link VisualQueryDetector}.This method is expected to be only triggered if
* {@link Callback#onQueryDetected()} is called to put the service into the query streaming
* state.
*
* @throws IllegalStateException if method called without query streamed.
*/
public void onQueryRejected() throws IllegalStateException {
try {
mRemoteCallback.onQueryRejected();
} catch (RemoteException e) {
throw new IllegalStateException("#onQueryRejected must be only be triggered after "
+ "calling #onQueryDetected to be in the query streaming state.");
}
}
/**
* Informs {@link VisualQueryDetector} with the metadata to complete the streamed partial
* query that has been sent to {@link VisualQueryDetector}. This method is expected to be
* only triggered if {@link Callback#onQueryDetected()} is called to put the service into
* the query streaming state.
*
* @throws IllegalStateException if method called without query streamed.
*/
public void onQueryFinished() throws IllegalStateException {
try {
mRemoteCallback.onQueryFinished();
} catch (RemoteException e) {
throw new IllegalStateException("#onQueryFinished must be only be triggered after "
+ "calling #onQueryDetected to be in the query streaming state.");
}
/**
* Informs {@link VisualQueryDetector} with the metadata to complete the streamed partial
* query that has been sent to {@link VisualQueryDetector}. This method is expected to be
* only triggered if {@link VisualQueryDetectionService#streamQuery(String)} is called to put
* the service into the query streaming state.
*
* @throws IllegalStateException if method called without query streamed.
*/
public final void finishQuery() throws IllegalStateException {
try {
mRemoteCallback.onQueryFinished();
} catch (RemoteException e) {
throw new IllegalStateException("#finishQuery must be only be triggered after "
+ "calling #streamQuery to be in the query streaming state.");
}
}

8
core/java/android/service/voice/VisualQueryDetector.java
View File

@@ -177,7 +177,8 @@ public class VisualQueryDetector {
public interface Callback {
/**
* Called when the {@link VisualQueryDetectionService} starts to stream partial queries.
* Called when the {@link VisualQueryDetectionService} starts to stream partial queries
* with {@link VisualQueryDetectionService#streamQuery(String)}.
*
* @param partialQuery The partial query in a text form being streamed.
*/
@@ -185,12 +186,13 @@ public class VisualQueryDetector {
/**
* Called when the {@link VisualQueryDetectionService} decides to abandon the streamed
* partial queries.
* partial queries with {@link VisualQueryDetectionService#rejectQuery()}.
*/
void onQueryRejected();
/**
* Called when the {@link VisualQueryDetectionService} finishes streaming partial queries.
* Called when the {@link VisualQueryDetectionService} finishes streaming partial queries
* with {@link VisualQueryDetectionService#finishQuery()}.
*/
void onQueryFinished();