From 4cff16fda9b7675c0414ce5d26727672905adcd0 Mon Sep 17 00:00:00 2001 From: Matt Casey Date: Mon, 14 Jan 2019 14:56:06 -0500 Subject: [PATCH] Voice state + transcription in VoiceInteractionSvc Voice state as well as voice transcription can be provided by the VoiceInteractionService. These get proxied to the AssistManager which can update the system UI to reflect the state & transcription. Test: TBD Bug: 122740752 Bug: 123080754 Change-Id: I79cac1d89fe0123bf25a05d551cb4ef40ae1368e --- api/current.txt | 7 ++ .../voice/VoiceInteractionService.java | 67 +++++++++++++++++++ .../app/IVoiceInteractionManagerService.aidl | 15 +++++ .../app/IVoiceInteractionSessionListener.aidl | 16 +++++ .../systemui/assist/AssistManager.java | 53 ++++++++++++--- .../VoiceInteractionManagerService.java | 58 ++++++++++++++-- 6 files changed, 201 insertions(+), 15 deletions(-) diff --git a/api/current.txt b/api/current.txt index 529f2918f94ff..9e17352e841e2 100644 --- a/api/current.txt +++ b/api/current.txt @@ -41671,6 +41671,7 @@ package android.service.voice { public class VoiceInteractionService extends android.app.Service { ctor public VoiceInteractionService(); + method public final void clearTranscription(boolean); method public final android.service.voice.AlwaysOnHotwordDetector createAlwaysOnHotwordDetector(String, java.util.Locale, android.service.voice.AlwaysOnHotwordDetector.Callback); method public int getDisabledShowContext(); method public static boolean isActiveService(android.content.Context, android.content.ComponentName); @@ -41680,9 +41681,15 @@ package android.service.voice { method public void onReady(); method public void onShutdown(); method public void setDisabledShowContext(int); + method public final void setTranscription(@NonNull String); + method public final void setVoiceState(int); method public void showSession(android.os.Bundle, int); field public static final String SERVICE_INTERFACE = "android.service.voice.VoiceInteractionService"; field public static final String SERVICE_META_DATA = "android.voice_interaction"; + field public static final int VOICE_STATE_CONDITIONAL_LISTENING = 1; // 0x1 + field public static final int VOICE_STATE_FULFILLING = 3; // 0x3 + field public static final int VOICE_STATE_LISTENING = 2; // 0x2 + field public static final int VOICE_STATE_NONE = 0; // 0x0 } public class VoiceInteractionSession implements android.content.ComponentCallbacks2 android.view.KeyEvent.Callback { diff --git a/core/java/android/service/voice/VoiceInteractionService.java b/core/java/android/service/voice/VoiceInteractionService.java index e105fdf6cfb88..2789651c4eaf7 100644 --- a/core/java/android/service/voice/VoiceInteractionService.java +++ b/core/java/android/service/voice/VoiceInteractionService.java @@ -16,6 +16,7 @@ package android.service.voice; +import android.annotation.IntDef; import android.annotation.NonNull; import android.annotation.Nullable; import android.annotation.SdkConstant; @@ -40,6 +41,8 @@ import com.android.internal.util.function.pooled.PooledLambda; import java.io.FileDescriptor; import java.io.PrintWriter; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; import java.util.ArrayList; import java.util.List; import java.util.Locale; @@ -77,6 +80,33 @@ public class VoiceInteractionService extends Service { */ public static final String SERVICE_META_DATA = "android.voice_interaction"; + /** @hide */ + @Retention(RetentionPolicy.SOURCE) + @IntDef(prefix = {"VOICE_STATE_"}, value = { + VOICE_STATE_NONE, + VOICE_STATE_CONDITIONAL_LISTENING, + VOICE_STATE_LISTENING, + VOICE_STATE_FULFILLING}) + public @interface VoiceState { + } + + /** + * Voice assistant inactive. + */ + public static final int VOICE_STATE_NONE = 0; + /** + * Voice assistant listening, but will only trigger if it hears a request it can fulfill. + */ + public static final int VOICE_STATE_CONDITIONAL_LISTENING = 1; + /** + * Voice assistant is listening to user speech. + */ + public static final int VOICE_STATE_LISTENING = 2; + /** + * Voice assistant is fulfilling an action requested by the user. + */ + public static final int VOICE_STATE_FULFILLING = 3; + IVoiceInteractionService mInterface = new IVoiceInteractionService.Stub() { @Override public void ready() { @@ -341,6 +371,43 @@ public class VoiceInteractionService extends Service { } } + /** + * Requests that the voice state UI indicate the given state. + * + * @param state value indicating whether the assistant is listening, fulfilling, etc. + */ + public final void setVoiceState(@VoiceState int state) { + try { + mSystemService.setVoiceState(state); + } catch (RemoteException e) { + throw e.rethrowFromSystemServer(); + } + } + + /** + * Displays the given voice transcription contents. + */ + public final void setTranscription(@NonNull String transcription) { + try { + mSystemService.setTranscription(transcription); + } catch (RemoteException e) { + throw e.rethrowFromSystemServer(); + } + } + + /** + * Hides transcription. + * + * @param immediate if {@code true}, remove before transcription animation completes. + */ + public final void clearTranscription(boolean immediate) { + try { + mSystemService.clearTranscription(immediate); + } catch (RemoteException e) { + throw e.rethrowFromSystemServer(); + } + } + @Override protected void dump(FileDescriptor fd, PrintWriter pw, String[] args) { pw.println("VOICE INTERACTION"); diff --git a/core/java/com/android/internal/app/IVoiceInteractionManagerService.aidl b/core/java/com/android/internal/app/IVoiceInteractionManagerService.aidl index 5088ccae5c1f7..b85488ffe70c7 100644 --- a/core/java/com/android/internal/app/IVoiceInteractionManagerService.aidl +++ b/core/java/com/android/internal/app/IVoiceInteractionManagerService.aidl @@ -151,4 +151,19 @@ interface IVoiceInteractionManagerService { */ void getActiveServiceSupportedActions(in List voiceActions, in IVoiceActionCheckCallback callback); + + /** + * Sets the transcribed voice to the given string. + */ + void setTranscription(String transcription); + + /** + * Indicates that the transcription session is finished. + */ + void clearTranscription(boolean immediate); + + /** + * Sets the voice state indication based upon the given value. + */ + void setVoiceState(int state); } diff --git a/core/java/com/android/internal/app/IVoiceInteractionSessionListener.aidl b/core/java/com/android/internal/app/IVoiceInteractionSessionListener.aidl index 87749d26e4a0c..674ad5b4ab678 100644 --- a/core/java/com/android/internal/app/IVoiceInteractionSessionListener.aidl +++ b/core/java/com/android/internal/app/IVoiceInteractionSessionListener.aidl @@ -26,4 +26,20 @@ * Called when a voice session is hidden. */ void onVoiceSessionHidden(); + + /** + * Called when voice assistant transcription has been updated to the given string. + */ + void onTranscriptionUpdate(in String transcription); + + /** + * Called when voice transcription is completed. + */ + void onTranscriptionComplete(in boolean immediate); + + /** + * Called when the voice assistant's state has changed. Values are from + * VoiceInteractionService's VOICE_STATE* constants. + */ + void onVoiceStateChange(in int state); } \ No newline at end of file diff --git a/packages/SystemUI/src/com/android/systemui/assist/AssistManager.java b/packages/SystemUI/src/com/android/systemui/assist/AssistManager.java index 53cdee5495365..4d708908975e7 100644 --- a/packages/SystemUI/src/com/android/systemui/assist/AssistManager.java +++ b/packages/SystemUI/src/com/android/systemui/assist/AssistManager.java @@ -47,6 +47,10 @@ import com.android.systemui.statusbar.policy.DeviceProvisionedController; public class AssistManager implements ConfigurationChangedReceiver { private static final String TAG = "AssistManager"; + + // Note that VERBOSE logging may leak PII (e.g. transcription contents). + private static final boolean VERBOSE = false; + private static final String ASSIST_ICON_METADATA_NAME = "com.android.systemui.action_assist_icon"; @@ -103,16 +107,41 @@ public class AssistManager implements ConfigurationChangedReceiver { protected void registerVoiceInteractionSessionListener() { mAssistUtils.registerVoiceInteractionSessionListener( new IVoiceInteractionSessionListener.Stub() { - @Override - public void onVoiceSessionShown() throws RemoteException { - Log.v(TAG, "Voice open"); - } + @Override + public void onVoiceSessionShown() throws RemoteException { + if (VERBOSE) { + Log.v(TAG, "Voice open"); + } + } - @Override - public void onVoiceSessionHidden() throws RemoteException { - Log.v(TAG, "Voice closed"); - } - }); + @Override + public void onVoiceSessionHidden() throws RemoteException { + if (VERBOSE) { + Log.v(TAG, "Voice closed"); + } + } + + @Override + public void onTranscriptionUpdate(String transcription) { + if (VERBOSE) { + Log.v(TAG, "Transcription Updated: \"" + transcription + "\""); + } + } + + @Override + public void onTranscriptionComplete(boolean immediate) { + if (VERBOSE) { + Log.v(TAG, "Transcription complete (immediate=" + immediate + ")"); + } + } + + @Override + public void onVoiceStateChange(int state) { + if (VERBOSE) { + Log.v(TAG, "Voice state is now " + state); + } + } + }); } public void onConfigurationChanged(Configuration newConfiguration) { @@ -291,8 +320,10 @@ public class AssistManager implements ConfigurationChangedReceiver { } } } catch (PackageManager.NameNotFoundException e) { - Log.v(TAG, "Assistant component " - + component.flattenToShortString() + " not found"); + if (VERBOSE) { + Log.v(TAG, "Assistant component " + + component.flattenToShortString() + " not found"); + } } catch (Resources.NotFoundException nfe) { Log.w(TAG, "Failed to swap drawable from " + component.flattenToShortString(), nfe); diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerService.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerService.java index 613c4ffceffc2..bb01f041c5af6 100644 --- a/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerService.java +++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerService.java @@ -19,9 +19,6 @@ package com.android.server.voiceinteraction; import android.Manifest; import android.app.ActivityManager; import android.app.ActivityManagerInternal; - -import com.android.internal.app.IVoiceActionCheckCallback; -import com.android.server.wm.ActivityTaskManagerInternal; import android.app.AppGlobals; import android.content.ComponentName; import android.content.ContentResolver; @@ -62,8 +59,9 @@ import android.util.ArraySet; import android.util.Log; import android.util.Slog; -import com.android.internal.app.IVoiceInteractionSessionListener; +import com.android.internal.app.IVoiceActionCheckCallback; import com.android.internal.app.IVoiceInteractionManagerService; +import com.android.internal.app.IVoiceInteractionSessionListener; import com.android.internal.app.IVoiceInteractionSessionShowCallback; import com.android.internal.app.IVoiceInteractor; import com.android.internal.content.PackageMonitor; @@ -75,6 +73,7 @@ import com.android.server.LocalServices; import com.android.server.SystemService; import com.android.server.UiThread; import com.android.server.soundtrigger.SoundTriggerInternal; +import com.android.server.wm.ActivityTaskManagerInternal; import java.io.FileDescriptor; import java.io.PrintWriter; @@ -1205,6 +1204,57 @@ public class VoiceInteractionManagerService extends SystemService { mSoundTriggerInternal.dump(fd, pw, args); } + @Override + public void setTranscription(String transcription) { + synchronized (this) { + final int size = mVoiceInteractionSessionListeners.beginBroadcast(); + for (int i = 0; i < size; ++i) { + final IVoiceInteractionSessionListener listener = + mVoiceInteractionSessionListeners.getBroadcastItem(i); + try { + listener.onTranscriptionUpdate(transcription); + } catch (RemoteException e) { + Slog.e(TAG, "Error delivering voice transcription.", e); + } + } + mVoiceInteractionSessionListeners.finishBroadcast(); + } + } + + @Override + public void clearTranscription(boolean immediate) { + synchronized (this) { + final int size = mVoiceInteractionSessionListeners.beginBroadcast(); + for (int i = 0; i < size; ++i) { + final IVoiceInteractionSessionListener listener = + mVoiceInteractionSessionListeners.getBroadcastItem(i); + try { + listener.onTranscriptionComplete(immediate); + } catch (RemoteException e) { + Slog.e(TAG, "Error delivering transcription complete event.", e); + } + } + mVoiceInteractionSessionListeners.finishBroadcast(); + } + } + + @Override + public void setVoiceState(int state) { + synchronized (this) { + final int size = mVoiceInteractionSessionListeners.beginBroadcast(); + for (int i = 0; i < size; ++i) { + final IVoiceInteractionSessionListener listener = + mVoiceInteractionSessionListeners.getBroadcastItem(i); + try { + listener.onVoiceStateChange(state); + } catch (RemoteException e) { + Slog.e(TAG, "Error delivering voice state change.", e); + } + } + mVoiceInteractionSessionListeners.finishBroadcast(); + } + } + private void enforceCallingPermission(String permission) { if (mContext.checkCallingOrSelfPermission(permission) != PackageManager.PERMISSION_GRANTED) {