diff --git a/core/java/android/service/voice/AlwaysOnHotwordDetector.java b/core/java/android/service/voice/AlwaysOnHotwordDetector.java index 67b97cee1b514..41374167cc56e 100644 --- a/core/java/android/service/voice/AlwaysOnHotwordDetector.java +++ b/core/java/android/service/voice/AlwaysOnHotwordDetector.java @@ -263,6 +263,7 @@ public class AlwaysOnHotwordDetector extends AbstractHotwordDetector { private static final int MSG_DETECTION_RESUME = 5; private static final int MSG_HOTWORD_REJECTED = 6; private static final int MSG_HOTWORD_STATUS_REPORTED = 7; + private static final int MSG_PROCESS_RESTARTED = 8; private final String mText; private final Locale mLocale; @@ -1212,6 +1213,12 @@ public class AlwaysOnHotwordDetector extends AbstractHotwordDetector { message.arg1 = status; message.sendToTarget(); } + + @Override + public void onProcessRestarted() { + Slog.i(TAG, "onProcessRestarted"); + mHandler.sendEmptyMessage(MSG_PROCESS_RESTARTED); + } } class MyHandler extends Handler { @@ -1246,6 +1253,9 @@ public class AlwaysOnHotwordDetector extends AbstractHotwordDetector { case MSG_HOTWORD_STATUS_REPORTED: mExternalCallback.onHotwordDetectionServiceInitialized(msg.arg1); break; + case MSG_PROCESS_RESTARTED: + mExternalCallback.onHotwordDetectionServiceRestarted(); + break; default: super.handleMessage(msg); } diff --git a/core/java/android/service/voice/HotwordDetectionService.java b/core/java/android/service/voice/HotwordDetectionService.java index b66d93d6316e7..93a7ec793536f 100644 --- a/core/java/android/service/voice/HotwordDetectionService.java +++ b/core/java/android/service/voice/HotwordDetectionService.java @@ -291,9 +291,7 @@ public abstract class HotwordDetectionService extends Service { @Nullable PersistableBundle options, @Nullable SharedMemory sharedMemory, @DurationMillisLong long callbackTimeoutMillis, - @Nullable IntConsumer statusCallback) { - // TODO: Handle the unimplemented case by throwing? - } + @Nullable IntConsumer statusCallback) {} /** * Called when the {@link VoiceInteractionService} requests that this service diff --git a/core/java/android/service/voice/SoftwareHotwordDetector.java b/core/java/android/service/voice/SoftwareHotwordDetector.java index 204e7df89706a..fb540b1622e62 100644 --- a/core/java/android/service/voice/SoftwareHotwordDetector.java +++ b/core/java/android/service/voice/SoftwareHotwordDetector.java @@ -122,7 +122,7 @@ class SoftwareHotwordDetector extends AbstractHotwordDetector { this.mCallback = callback; } - /** TODO: onDetected */ + /** Called when the detected result is valid. */ @Override public void onDetected( @Nullable HotwordDetectedResult hotwordDetectedResult, @@ -150,33 +150,45 @@ class SoftwareHotwordDetector extends AbstractHotwordDetector { public void onKeyphraseDetected( SoundTrigger.KeyphraseRecognitionEvent recognitionEvent, HotwordDetectedResult result) { - + if (DEBUG) { + Slog.i(TAG, "Ignored #onKeyphraseDetected event"); + } } @Override public void onGenericSoundTriggerDetected( SoundTrigger.GenericRecognitionEvent recognitionEvent) throws RemoteException { - + if (DEBUG) { + Slog.i(TAG, "Ignored #onGenericSoundTriggerDetected event"); + } } @Override public void onRejected(HotwordRejectedResult result) throws RemoteException { - + if (DEBUG) { + Slog.i(TAG, "Ignored #onRejected event"); + } } @Override public void onError(int status) throws RemoteException { - + if (DEBUG) { + Slog.i(TAG, "Ignored #onError (" + status + ") event"); + } } @Override public void onRecognitionPaused() throws RemoteException { - + if (DEBUG) { + Slog.i(TAG, "Ignored #onRecognitionPaused event"); + } } @Override public void onRecognitionResumed() throws RemoteException { - + if (DEBUG) { + Slog.i(TAG, "Ignored #onRecognitionResumed event"); + } } @Override @@ -187,6 +199,14 @@ class SoftwareHotwordDetector extends AbstractHotwordDetector { mCallback, status)); } + + @Override + public void onProcessRestarted() throws RemoteException { + Slog.v(TAG, "onProcessRestarted()"); + mHandler.sendMessage(obtainMessage( + HotwordDetector.Callback::onHotwordDetectionServiceRestarted, + mCallback)); + } } /** @hide */ diff --git a/core/java/com/android/internal/app/IHotwordRecognitionStatusCallback.aidl b/core/java/com/android/internal/app/IHotwordRecognitionStatusCallback.aidl index ec99c95a737c5..d0214e6e0082b 100644 --- a/core/java/com/android/internal/app/IHotwordRecognitionStatusCallback.aidl +++ b/core/java/com/android/internal/app/IHotwordRecognitionStatusCallback.aidl @@ -78,4 +78,7 @@ oneway interface IHotwordRecognitionStatusCallback { * @param status The status about the result of requesting update state action. */ void onStatusReported(int status); + + /** Called when the hotword detection process is restarted */ + void onProcessRestarted(); } diff --git a/core/java/com/android/internal/infra/ServiceConnector.java b/core/java/com/android/internal/infra/ServiceConnector.java index af21f8183d3a6..9ced6097804de 100644 --- a/core/java/com/android/internal/infra/ServiceConnector.java +++ b/core/java/com/android/internal/infra/ServiceConnector.java @@ -228,7 +228,7 @@ public interface ServiceConnector { private final int mBindingFlags; private final @Nullable Function mBinderAsInterface; private final @NonNull Handler mHandler; - private final @NonNull Executor mExecutor; + protected final @NonNull Executor mExecutor; private volatile I mService = null; private boolean mBinding = false; diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java index 17303a4aa7e14..beaca68b9a37d 100644 --- a/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java +++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java @@ -29,11 +29,7 @@ import android.content.Context; import android.content.Intent; import android.hardware.soundtrigger.IRecognitionStatusCallback; import android.hardware.soundtrigger.SoundTrigger; -import android.media.AudioAttributes; import android.media.AudioFormat; -import android.media.AudioManager; -import android.media.AudioRecord; -import android.media.MediaRecorder; import android.os.Binder; import android.os.Bundle; import android.os.IBinder; @@ -66,12 +62,16 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.io.PrintWriter; +import java.time.Duration; +import java.time.Instant; import java.util.concurrent.Executor; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Function; /** * A class that provides the communication with the HotwordDetectionService. @@ -81,33 +81,38 @@ final class HotwordDetectionConnection { // TODO (b/177502877): Set the Debug flag to false before shipping. private static final boolean DEBUG = true; - // Number of bytes per sample of audio (which is a short). - private static final int BYTES_PER_SAMPLE = 2; // TODO: These constants need to be refined. private static final long VALIDATION_TIMEOUT_MILLIS = 3000; - private static final long VOICE_INTERACTION_TIMEOUT_TO_OPEN_MIC_MILLIS = 2000; - private static final int MAX_STREAMING_SECONDS = 10; - private static final int MICROPHONE_BUFFER_LENGTH_SECONDS = 8; - private static final int HOTWORD_AUDIO_LENGTH_SECONDS = 3; private static final long MAX_UPDATE_TIMEOUT_MILLIS = 6000; + private static final Duration MAX_UPDATE_TIMEOUT_DURATION = + Duration.ofMillis(MAX_UPDATE_TIMEOUT_MILLIS); private final Executor mAudioCopyExecutor = Executors.newCachedThreadPool(); // TODO: This may need to be a Handler(looper) private final ScheduledExecutorService mScheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); - private final AtomicBoolean mUpdateStateFinish = new AtomicBoolean(false); + private final AtomicBoolean mUpdateStateAfterStartFinished = new AtomicBoolean(false); + private final @NonNull ServiceConnectionFactory mServiceConnectionFactory; final Object mLock; final int mVoiceInteractionServiceUid; final ComponentName mDetectionComponentName; final int mUser; final Context mContext; - final @NonNull ServiceConnector mRemoteHotwordDetectionService; - boolean mBound; volatile HotwordDetectionServiceIdentity mIdentity; + private IHotwordRecognitionStatusCallback mCallback; + private IMicrophoneHotwordDetectionVoiceInteractionCallback mSoftwareCallback; + private Instant mLastRestartInstant; + + private ScheduledFuture mCancellationTaskFuture; @GuardedBy("mLock") private ParcelFileDescriptor mCurrentAudioSink; + @GuardedBy("mLock") + private boolean mValidatingDspTrigger = false; + @GuardedBy("mLock") + private boolean mPerformingSoftwareHotwordDetection; + private @NonNull ServiceConnection mRemoteHotwordDetectionService; HotwordDetectionConnection(Object lock, Context context, int voiceInteractionServiceUid, ComponentName serviceName, int userId, boolean bindInstantServiceAllowed, @@ -121,50 +126,36 @@ final class HotwordDetectionConnection { final Intent intent = new Intent(HotwordDetectionService.SERVICE_INTERFACE); intent.setComponent(mDetectionComponentName); - mRemoteHotwordDetectionService = new ServiceConnector.Impl( - mContext, intent, bindInstantServiceAllowed ? Context.BIND_ALLOW_INSTANT : 0, mUser, - IHotwordDetectionService.Stub::asInterface) { - @Override // from ServiceConnector.Impl - protected void onServiceConnectionStatusChanged(IHotwordDetectionService service, - boolean connected) { - if (DEBUG) { - Slog.d(TAG, "onServiceConnectionStatusChanged connected = " + connected); - } - synchronized (mLock) { - mBound = connected; - } - } + mServiceConnectionFactory = new ServiceConnectionFactory(intent, bindInstantServiceAllowed); - @Override - protected long getAutoDisconnectTimeoutMs() { - return -1; - } + mRemoteHotwordDetectionService = mServiceConnectionFactory.create(); - @Override - public void binderDied() { - super.binderDied(); - Slog.w(TAG, "binderDied"); - try { - callback.onError(-1); - } catch (RemoteException e) { - Slog.w(TAG, "Failed to report onError status: " + e); - } - } - }; - mRemoteHotwordDetectionService.connect(); if (callback == null) { updateStateLocked(options, sharedMemory); return; } - updateAudioFlinger(); - updateContentCaptureManager(); - updateStateWithCallbackLocked(options, sharedMemory, callback); + mCallback = callback; + + mLastRestartInstant = Instant.now(); + updateStateAfterProcessStart(options, sharedMemory); + + // TODO(volnov): we need to be smarter here, e.g. schedule it a bit more often, but wait + // until the current session is closed. + mCancellationTaskFuture = mScheduledExecutorService.scheduleAtFixedRate(() -> { + if (DEBUG) { + Slog.i(TAG, "Time to restart the process, TTL has passed"); + } + + synchronized (mLock) { + restartProcessLocked(); + } + }, 30, 30, TimeUnit.MINUTES); } - private void updateStateWithCallbackLocked(PersistableBundle options, - SharedMemory sharedMemory, IHotwordRecognitionStatusCallback callback) { + private void updateStateAfterProcessStart( + PersistableBundle options, SharedMemory sharedMemory) { if (DEBUG) { - Slog.d(TAG, "updateStateWithCallbackLocked"); + Slog.d(TAG, "updateStateAfterProcessStart"); } mRemoteHotwordDetectionService.postAsync(service -> { AndroidFuture future = new AndroidFuture<>(); @@ -183,21 +174,21 @@ final class HotwordDetectionConnection { mIdentity = new HotwordDetectionServiceIdentity(uid, mVoiceInteractionServiceUid); future.complete(null); + if (mUpdateStateAfterStartFinished.getAndSet(true)) { + Slog.w(TAG, "call callback after timeout"); + return; + } + int status = bundle != null ? bundle.getInt( + KEY_INITIALIZATION_STATUS, + INITIALIZATION_STATUS_UNKNOWN) + : INITIALIZATION_STATUS_UNKNOWN; + // Add the protection to avoid unexpected status + if (status > HotwordDetectionService.getMaxCustomInitializationStatus() + && status != INITIALIZATION_STATUS_UNKNOWN) { + status = INITIALIZATION_STATUS_UNKNOWN; + } try { - if (mUpdateStateFinish.getAndSet(true)) { - Slog.w(TAG, "call callback after timeout"); - return; - } - int status = bundle != null ? bundle.getInt( - KEY_INITIALIZATION_STATUS, - INITIALIZATION_STATUS_UNKNOWN) - : INITIALIZATION_STATUS_UNKNOWN; - // Add the protection to avoid unexpected status - if (status > HotwordDetectionService.getMaxCustomInitializationStatus() - && status != INITIALIZATION_STATUS_UNKNOWN) { - status = INITIALIZATION_STATUS_UNKNOWN; - } - callback.onStatusReported(status); + mCallback.onStatusReported(status); } catch (RemoteException e) { Slog.w(TAG, "Failed to report initialization status: " + e); } @@ -214,13 +205,13 @@ final class HotwordDetectionConnection { .whenComplete((res, err) -> { if (err instanceof TimeoutException) { Slog.w(TAG, "updateState timed out"); + if (mUpdateStateAfterStartFinished.getAndSet(true)) { + return; + } try { - if (mUpdateStateFinish.getAndSet(true)) { - return; - } - callback.onStatusReported(INITIALIZATION_STATUS_UNKNOWN); + mCallback.onStatusReported(INITIALIZATION_STATUS_UNKNOWN); } catch (RemoteException e) { - Slog.w(TAG, "Failed to report initialization status: " + e); + Slog.w(TAG, "Failed to report initialization status UNKNOWN", e); } } else if (err != null) { Slog.w(TAG, "Failed to update state: " + err); @@ -230,27 +221,9 @@ final class HotwordDetectionConnection { }); } - private void updateAudioFlinger() { - // TODO: Consider using a proxy that limits the exposed API surface. - IBinder audioFlinger = ServiceManager.getService("media.audio_flinger"); - if (audioFlinger == null) { - throw new IllegalStateException("Service media.audio_flinger wasn't found."); - } - mRemoteHotwordDetectionService.post(service -> service.updateAudioFlinger(audioFlinger)); - } - - private void updateContentCaptureManager() { - IBinder b = ServiceManager - .getService(Context.CONTENT_CAPTURE_MANAGER_SERVICE); - IContentCaptureManager binderService = IContentCaptureManager.Stub.asInterface(b); - mRemoteHotwordDetectionService.post( - service -> service.updateContentCaptureManager(binderService, - new ContentCaptureOptions(null))); - } - private boolean isBound() { synchronized (mLock) { - return mBound; + return mRemoteHotwordDetectionService.isBound(); } } @@ -258,18 +231,25 @@ final class HotwordDetectionConnection { if (DEBUG) { Slog.d(TAG, "cancelLocked"); } - if (mBound) { + if (mRemoteHotwordDetectionService.isBound()) { mRemoteHotwordDetectionService.unbind(); - mBound = false; LocalServices.getService(PermissionManagerServiceInternal.class) .setHotwordDetectionServiceProvider(null); mIdentity = null; } + mCancellationTaskFuture.cancel(/* may interrupt */ true); } void updateStateLocked(PersistableBundle options, SharedMemory sharedMemory) { - mRemoteHotwordDetectionService.run( - service -> service.updateState(options, sharedMemory, null /* callback */)); + // Prevent doing the init late, so restart is handled equally to a clean process start. + // TODO(b/191742511): this logic needs a test + if (!mUpdateStateAfterStartFinished.get() + && Instant.now().minus(MAX_UPDATE_TIMEOUT_DURATION).isBefore(mLastRestartInstant)) { + updateStateAfterProcessStart(options, sharedMemory); + } else { + mRemoteHotwordDetectionService.run( + service -> service.updateState(options, sharedMemory, null /* callback */)); + } } void startListeningFromMic( @@ -278,7 +258,20 @@ final class HotwordDetectionConnection { if (DEBUG) { Slog.d(TAG, "startListeningFromMic"); } + mSoftwareCallback = callback; + synchronized (mLock) { + if (mPerformingSoftwareHotwordDetection) { + Slog.i(TAG, "Hotword validation is already in progress, ignoring."); + return; + } + mPerformingSoftwareHotwordDetection = true; + + startListeningFromMicLocked(); + } + } + + private void startListeningFromMicLocked() { // TODO: consider making this a non-anonymous class. IDspHotwordDetectionCallback internalCallback = new IDspHotwordDetectionCallback.Stub() { @Override @@ -286,15 +279,22 @@ final class HotwordDetectionConnection { if (DEBUG) { Slog.d(TAG, "onDetected"); } - callback.onDetected(result, null, null); + synchronized (mLock) { + if (mPerformingSoftwareHotwordDetection) { + mSoftwareCallback.onDetected(result, null, null); + mPerformingSoftwareHotwordDetection = false; + } else { + Slog.i(TAG, "Hotword detection has already completed"); + } + } } @Override public void onRejected(HotwordRejectedResult result) throws RemoteException { if (DEBUG) { - Slog.d(TAG, "onRejected"); + Slog.wtf(TAG, "onRejected"); } - // onRejected isn't allowed here + // onRejected isn't allowed here, and we are not expecting it. } }; @@ -315,6 +315,7 @@ final class HotwordDetectionConnection { if (DEBUG) { Slog.d(TAG, "startListeningFromExternalSource"); } + handleExternalSourceHotwordDetection( audioStream, audioFormat, @@ -326,18 +327,27 @@ final class HotwordDetectionConnection { if (DEBUG) { Slog.d(TAG, "stopListening"); } - - mRemoteHotwordDetectionService.run(service -> service.stopDetection()); - synchronized (mLock) { - if (mCurrentAudioSink != null) { - Slog.i(TAG, "Closing audio stream to hotword detector: stopping requested"); - bestEffortClose(mCurrentAudioSink); - } - mCurrentAudioSink = null; + stopListeningLocked(); } } + private void stopListeningLocked() { + if (!mPerformingSoftwareHotwordDetection) { + Slog.i(TAG, "Hotword detection is not running"); + return; + } + mPerformingSoftwareHotwordDetection = false; + + mRemoteHotwordDetectionService.run(IHotwordDetectionService::stopDetection); + + if (mCurrentAudioSink != null) { + Slog.i(TAG, "Closing audio stream to hotword detector: stopping requested"); + bestEffortClose(mCurrentAudioSink); + } + mCurrentAudioSink = null; + } + void triggerHardwareRecognitionEventForTestLocked( SoundTrigger.KeyphraseRecognitionEvent event, IHotwordRecognitionStatusCallback callback) { @@ -358,7 +368,14 @@ final class HotwordDetectionConnection { if (DEBUG) { Slog.d(TAG, "onDetected"); } - externalCallback.onKeyphraseDetected(recognitionEvent, result); + synchronized (mLock) { + if (mValidatingDspTrigger) { + mValidatingDspTrigger = false; + externalCallback.onKeyphraseDetected(recognitionEvent, result); + } else { + Slog.i(TAG, "Ignored hotword detected since trigger has been handled"); + } + } } @Override @@ -366,16 +383,26 @@ final class HotwordDetectionConnection { if (DEBUG) { Slog.d(TAG, "onRejected"); } - externalCallback.onRejected(result); + synchronized (mLock) { + if (mValidatingDspTrigger) { + mValidatingDspTrigger = false; + externalCallback.onRejected(result); + } else { + Slog.i(TAG, "Ignored hotword rejected since trigger has been handled"); + } + } } }; - mRemoteHotwordDetectionService.run( - service -> service.detectFromDspSource( - recognitionEvent, - recognitionEvent.getCaptureFormat(), - VALIDATION_TIMEOUT_MILLIS, - internalCallback)); + synchronized (mLock) { + mValidatingDspTrigger = true; + mRemoteHotwordDetectionService.run( + service -> service.detectFromDspSource( + recognitionEvent, + recognitionEvent.getCaptureFormat(), + VALIDATION_TIMEOUT_MILLIS, + internalCallback)); + } } private void detectFromDspSource(SoundTrigger.KeyphraseRecognitionEvent recognitionEvent, @@ -391,7 +418,14 @@ final class HotwordDetectionConnection { if (DEBUG) { Slog.d(TAG, "onDetected"); } - externalCallback.onKeyphraseDetected(recognitionEvent, result); + synchronized (mLock) { + if (!mValidatingDspTrigger) { + Slog.i(TAG, "Ignoring #onDetected due to a process restart"); + return; + } + mValidatingDspTrigger = false; + externalCallback.onKeyphraseDetected(recognitionEvent, result); + } } @Override @@ -399,16 +433,88 @@ final class HotwordDetectionConnection { if (DEBUG) { Slog.d(TAG, "onRejected"); } - externalCallback.onRejected(result); + synchronized (mLock) { + if (!mValidatingDspTrigger) { + Slog.i(TAG, "Ignoring #onRejected due to a process restart"); + return; + } + mValidatingDspTrigger = false; + externalCallback.onRejected(result); + } } }; - mRemoteHotwordDetectionService.run( - service -> service.detectFromDspSource( - recognitionEvent, - recognitionEvent.getCaptureFormat(), - VALIDATION_TIMEOUT_MILLIS, - internalCallback)); + synchronized (mLock) { + mValidatingDspTrigger = true; + mRemoteHotwordDetectionService.run( + service -> service.detectFromDspSource( + recognitionEvent, + recognitionEvent.getCaptureFormat(), + VALIDATION_TIMEOUT_MILLIS, + internalCallback)); + } + } + + void forceRestart() { + if (DEBUG) { + Slog.i(TAG, "Requested to restart the service internally. Performing the restart"); + } + synchronized (mLock) { + restartProcessLocked(); + } + } + + private void restartProcessLocked() { + if (DEBUG) { + Slog.i(TAG, "Restarting hotword detection process"); + } + + ServiceConnection oldConnection = mRemoteHotwordDetectionService; + + // TODO(volnov): this can be done after connect() has been successful. + if (mValidatingDspTrigger) { + // We're restarting the process while it's processing a DSP trigger, so report a + // rejection. This also allows the Interactor to startReco again + try { + mCallback.onRejected(new HotwordRejectedResult.Builder().build()); + } catch (RemoteException e) { + Slog.w(TAG, "Failed to call #rejected"); + } + mValidatingDspTrigger = false; + } + + mUpdateStateAfterStartFinished.set(false); + mLastRestartInstant = Instant.now(); + + // Recreate connection to reset the cache. + mRemoteHotwordDetectionService = mServiceConnectionFactory.create(); + + if (DEBUG) { + Slog.i(TAG, "Started the new process, issuing #onProcessRestarted"); + } + try { + mCallback.onProcessRestarted(); + } catch (RemoteException e) { + Slog.w(TAG, "Failed to communicate #onProcessRestarted", e); + } + + // Restart listening from microphone if the hotword process has been restarted. + if (mPerformingSoftwareHotwordDetection) { + Slog.i(TAG, "Process restarted: calling startRecognition() again"); + startListeningFromMicLocked(); + } + + if (mCurrentAudioSink != null) { + Slog.i(TAG, "Closing external audio stream to hotword detector: process restarted"); + bestEffortClose(mCurrentAudioSink); + mCurrentAudioSink = null; + } + + if (DEBUG) { + Slog.i(TAG, "#onProcessRestarted called, unbinding from the old process"); + } + oldConnection.ignoreConnectionStatusEvents(); + oldConnection.unbind(); } static final class SoundTriggerCallback extends IRecognitionStatusCallback.Stub { @@ -462,139 +568,13 @@ final class HotwordDetectionConnection { } } - // TODO: figure out if we need to let the client configure some of the parameters. - private static AudioRecord createAudioRecord( - @NonNull SoundTrigger.KeyphraseRecognitionEvent recognitionEvent) { - int sampleRate = recognitionEvent.getCaptureFormat().getSampleRate(); - return new AudioRecord( - new AudioAttributes.Builder() - .setInternalCapturePreset(MediaRecorder.AudioSource.HOTWORD).build(), - recognitionEvent.getCaptureFormat(), - getBufferSizeInBytes( - sampleRate, - MAX_STREAMING_SECONDS, - recognitionEvent.getCaptureFormat().getChannelCount()), - recognitionEvent.getCaptureSession()); - } - - @Nullable - private AudioRecord createMicAudioRecord(AudioFormat audioFormat) { - if (DEBUG) { - Slog.i(TAG, "#createAudioRecord"); - } - try { - AudioRecord audioRecord = new AudioRecord( - new AudioAttributes.Builder() - .setInternalCapturePreset(MediaRecorder.AudioSource.HOTWORD).build(), - audioFormat, - getBufferSizeInBytes( - audioFormat.getSampleRate(), - MICROPHONE_BUFFER_LENGTH_SECONDS, - audioFormat.getChannelCount()), - AudioManager.AUDIO_SESSION_ID_GENERATE); - - if (audioRecord.getState() != AudioRecord.STATE_INITIALIZED) { - Slog.w(TAG, "Failed to initialize AudioRecord"); - audioRecord.release(); - return null; - } - - return audioRecord; - } catch (IllegalArgumentException e) { - Slog.e(TAG, "Failed to create AudioRecord", e); - return null; - } - } - - @Nullable - private AudioRecord createFakeAudioRecord() { - if (DEBUG) { - Slog.i(TAG, "#createFakeAudioRecord"); - } - try { - AudioRecord audioRecord = new AudioRecord.Builder() - .setAudioFormat(new AudioFormat.Builder() - .setSampleRate(32000) - .setEncoding(AudioFormat.ENCODING_PCM_16BIT) - .setChannelMask(AudioFormat.CHANNEL_IN_MONO).build()) - .setAudioAttributes(new AudioAttributes.Builder() - .setInternalCapturePreset(MediaRecorder.AudioSource.HOTWORD).build()) - .setBufferSizeInBytes( - AudioRecord.getMinBufferSize(32000, - AudioFormat.CHANNEL_IN_MONO, - AudioFormat.ENCODING_PCM_16BIT) * 2) - .build(); - - if (audioRecord.getState() != AudioRecord.STATE_INITIALIZED) { - Slog.w(TAG, "Failed to initialize AudioRecord"); - audioRecord.release(); - return null; - } - return audioRecord; - } catch (IllegalArgumentException e) { - Slog.e(TAG, "Failed to create AudioRecord", e); - } - return null; - } - - /** - * Returns the number of bytes required to store {@code bufferLengthSeconds} of audio sampled at - * {@code sampleRate} Hz, using the format returned by DSP audio capture. - */ - private static int getBufferSizeInBytes( - int sampleRate, int bufferLengthSeconds, int intChannelCount) { - return BYTES_PER_SAMPLE * sampleRate * bufferLengthSeconds * intChannelCount; - } - - private static Pair createPipe() { - ParcelFileDescriptor[] fileDescriptors; - try { - fileDescriptors = ParcelFileDescriptor.createPipe(); - } catch (IOException e) { - Slog.e(TAG, "Failed to create audio stream pipe", e); - return null; - } - - return Pair.create(fileDescriptors[0], fileDescriptors[1]); - } - public void dump(String prefix, PrintWriter pw) { - pw.print(prefix); pw.print("mBound="); pw.println(mBound); - } - - private interface AudioReader extends Closeable { - int read(byte[] dest, int offset, int length) throws IOException; - - static AudioReader createFromInputStream(InputStream is) { - return new AudioReader() { - @Override - public int read(byte[] dest, int offset, int length) throws IOException { - return is.read(dest, offset, length); - } - - @Override - public void close() throws IOException { - is.close(); - } - }; - } - - static AudioReader createFromAudioRecord(AudioRecord record) { - record.startRecording(); - - return new AudioReader() { - @Override - public int read(byte[] dest, int offset, int length) throws IOException { - return record.read(dest, offset, length); - } - - @Override - public void close() throws IOException { - record.stop(); - record.release(); - } - }; - } + pw.print(prefix); + pw.print("mBound=" + mRemoteHotwordDetectionService.isBound()); + pw.print(", mValidatingDspTrigger=" + mValidatingDspTrigger); + pw.print(", mPerformingSoftwareHotwordDetection=" + mPerformingSoftwareHotwordDetection); + pw.print(", mRestartCount=" + mServiceConnectionFactory.mRestartCount); + pw.println(", mLastRestartInstant=" + mLastRestartInstant); } private void handleExternalSourceHotwordDetection( @@ -605,8 +585,7 @@ final class HotwordDetectionConnection { if (DEBUG) { Slog.d(TAG, "#handleExternalSourceHotwordDetection"); } - AudioReader audioSource = AudioReader.createFromInputStream( - new ParcelFileDescriptor.AutoCloseInputStream(audioStream)); + InputStream audioSource = new ParcelFileDescriptor.AutoCloseInputStream(audioStream); Pair clientPipe = createPipe(); if (clientPipe == null) { @@ -621,7 +600,7 @@ final class HotwordDetectionConnection { } mAudioCopyExecutor.execute(() -> { - try (AudioReader source = audioSource; + try (InputStream source = audioSource; OutputStream fos = new ParcelFileDescriptor.AutoCloseOutputStream(serviceAudioSink)) { @@ -681,6 +660,150 @@ final class HotwordDetectionConnection { })); } + private class ServiceConnectionFactory { + private final Intent mIntent; + private final int mBindingFlags; + + private int mRestartCount = 0; + + ServiceConnectionFactory(@NonNull Intent intent, boolean bindInstantServiceAllowed) { + mIntent = intent; + mBindingFlags = bindInstantServiceAllowed ? Context.BIND_ALLOW_INSTANT : 0; + } + + ServiceConnection create() { + ServiceConnection connection = + new ServiceConnection(mContext, mIntent, mBindingFlags, mUser, + IHotwordDetectionService.Stub::asInterface, ++mRestartCount); + connection.connect(); + + updateAudioFlinger(connection); + updateContentCaptureManager(connection); + return connection; + } + } + + private class ServiceConnection extends ServiceConnector.Impl { + private final Object mLock = new Object(); + + private final Intent mIntent; + private final int mBindingFlags; + private final int mInstanceNumber; + + private boolean mRespectServiceConnectionStatusChanged = true; + private boolean mIsBound = false; + + ServiceConnection(@NonNull Context context, + @NonNull Intent intent, int bindingFlags, int userId, + @Nullable Function binderAsInterface, + int instanceNumber) { + super(context, intent, bindingFlags, userId, binderAsInterface); + this.mIntent = intent; + this.mBindingFlags = bindingFlags; + this.mInstanceNumber = instanceNumber; + } + + @Override // from ServiceConnector.Impl + protected void onServiceConnectionStatusChanged(IHotwordDetectionService service, + boolean connected) { + if (DEBUG) { + Slog.d(TAG, "onServiceConnectionStatusChanged connected = " + connected); + } + synchronized (mLock) { + if (!mRespectServiceConnectionStatusChanged) { + if (DEBUG) { + Slog.d(TAG, "Ignored onServiceConnectionStatusChanged event"); + } + return; + } + mIsBound = connected; + } + } + + @Override + protected long getAutoDisconnectTimeoutMs() { + return -1; + } + + @Override + public void binderDied() { + super.binderDied(); + synchronized (mLock) { + if (!mRespectServiceConnectionStatusChanged) { + if (DEBUG) { + Slog.d(TAG, "Ignored #binderDied event"); + } + return; + } + + Slog.w(TAG, "binderDied"); + try { + mCallback.onError(-1); + } catch (RemoteException e) { + Slog.w(TAG, "Failed to report onError status: " + e); + } + } + } + + @Override + protected boolean bindService( + @NonNull android.content.ServiceConnection serviceConnection) { + try { + return mContext.bindIsolatedService( + mIntent, + Context.BIND_AUTO_CREATE | mBindingFlags, + "hotword_detector_" + mInstanceNumber, + mExecutor, + serviceConnection); + } catch (IllegalArgumentException e) { + Slog.wtf(TAG, "Can't bind to the hotword detection service!", e); + return false; + } + } + + boolean isBound() { + synchronized (mLock) { + return mIsBound; + } + } + + void ignoreConnectionStatusEvents() { + synchronized (mLock) { + mRespectServiceConnectionStatusChanged = false; + } + } + } + + private static Pair createPipe() { + ParcelFileDescriptor[] fileDescriptors; + try { + fileDescriptors = ParcelFileDescriptor.createPipe(); + } catch (IOException e) { + Slog.e(TAG, "Failed to create audio stream pipe", e); + return null; + } + + return Pair.create(fileDescriptors[0], fileDescriptors[1]); + } + + private static void updateAudioFlinger(ServiceConnection connection) { + // TODO: Consider using a proxy that limits the exposed API surface. + IBinder audioFlinger = ServiceManager.getService("media.audio_flinger"); + if (audioFlinger == null) { + throw new IllegalStateException("Service media.audio_flinger wasn't found."); + } + connection.post(service -> service.updateAudioFlinger(audioFlinger)); + } + + private static void updateContentCaptureManager(ServiceConnection connection) { + IBinder b = ServiceManager + .getService(Context.CONTENT_CAPTURE_MANAGER_SERVICE); + IContentCaptureManager binderService = IContentCaptureManager.Stub.asInterface(b); + connection.post( + service -> service.updateContentCaptureManager(binderService, + new ContentCaptureOptions(null))); + } + private static void bestEffortClose(Closeable closeable) { try { closeable.close(); diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerService.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerService.java index bc812c2ae4a7a..162acba8002db 100644 --- a/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerService.java +++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerService.java @@ -795,6 +795,10 @@ public class VoiceInteractionManagerService extends SystemService { Settings.Secure.ASSISTANT, null, userHandle); } + void forceRestartHotwordDetector() { + mImpl.forceRestartHotwordDetector(); + } + @Override public void showSession(Bundle args, int flags) { synchronized (this) { diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerServiceImpl.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerServiceImpl.java index ca30bc51e0464..89c5a720ee7ee 100644 --- a/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerServiceImpl.java +++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerServiceImpl.java @@ -562,6 +562,14 @@ class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConne && (serviceInfo.flags & ServiceInfo.FLAG_EXTERNAL_SERVICE) == 0; } + void forceRestartHotwordDetector() { + if (mHotwordDetectionConnection == null) { + Slog.w(TAG, "Failed to force-restart hotword detection: no hotword detection active"); + return; + } + mHotwordDetectionConnection.forceRestart(); + } + public void dumpLocked(FileDescriptor fd, PrintWriter pw, String[] args) { if (!mValid) { pw.print(" NOT VALID: "); diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerServiceShellCommand.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerServiceShellCommand.java index 2e3ca0157a3b6..cdd8f7b91d9d8 100644 --- a/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerServiceShellCommand.java +++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerServiceShellCommand.java @@ -54,6 +54,8 @@ final class VoiceInteractionManagerServiceShellCommand extends ShellCommand { return requestHide(pw); case "disable": return requestDisable(pw); + case "restart-detection": + return requestRestartDetection(pw); default: return handleDefaultCommands(cmd); } @@ -74,6 +76,8 @@ final class VoiceInteractionManagerServiceShellCommand extends ShellCommand { pw.println(""); pw.println(" disable [true|false]"); pw.println(" Temporarily disable (when true) service"); + pw.println(" restart-detection"); + pw.println(" Force a restart of a hotword detection service"); pw.println(""); } } @@ -143,6 +147,16 @@ final class VoiceInteractionManagerServiceShellCommand extends ShellCommand { return 0; } + private int requestRestartDetection(PrintWriter pw) { + Slog.i(TAG, "requestRestartDetection()"); + try { + mService.forceRestartHotwordDetector(); + } catch (Exception e) { + return handleError(pw, "requestRestartDetection()", e); + } + return 0; + } + private static int handleError(PrintWriter pw, String message, Exception e) { Slog.e(TAG, "error calling " + message, e); pw.printf("Error calling %s: %s\n", message, e);