diff --git a/core/api/system-current.txt b/core/api/system-current.txt index 7c296ec79f659..3da6add3a04ef 100644 --- a/core/api/system-current.txt +++ b/core/api/system-current.txt @@ -10268,7 +10268,7 @@ package android.service.voice { public abstract class HotwordDetectionService extends android.app.Service { ctor public HotwordDetectionService(); method @Nullable public final android.os.IBinder onBind(@NonNull android.content.Intent); - method public void onDetectFromDspSource(int, @NonNull android.service.voice.HotwordDetectionService.DspHotwordDetectionCallback); + method public void onDetectFromDspSource(@NonNull android.os.ParcelFileDescriptor, @NonNull android.media.AudioFormat, long, @NonNull android.service.voice.HotwordDetectionService.DspHotwordDetectionCallback); field public static final String SERVICE_INTERFACE = "android.service.voice.HotwordDetectionService"; } diff --git a/core/java/android/service/voice/HotwordDetectionService.java b/core/java/android/service/voice/HotwordDetectionService.java index 3c35d2ca78eb3..7f1c5ff966362 100644 --- a/core/java/android/service/voice/HotwordDetectionService.java +++ b/core/java/android/service/voice/HotwordDetectionService.java @@ -19,15 +19,18 @@ package android.service.voice; import static com.android.internal.util.function.pooled.PooledLambda.obtainMessage; import android.annotation.CallSuper; +import android.annotation.DurationMillisLong; import android.annotation.NonNull; import android.annotation.Nullable; import android.annotation.SdkConstant; import android.annotation.SystemApi; import android.app.Service; import android.content.Intent; +import android.media.AudioFormat; import android.os.Handler; import android.os.IBinder; import android.os.Looper; +import android.os.ParcelFileDescriptor; import android.os.RemoteException; import android.util.Log; @@ -57,14 +60,21 @@ public abstract class HotwordDetectionService extends Service { private final IHotwordDetectionService mInterface = new IHotwordDetectionService.Stub() { @Override - public void detectFromDspSource(int sessionId, IDspHotwordDetectionCallback callback) + public void detectFromDspSource( + ParcelFileDescriptor audioStream, + AudioFormat audioFormat, + long timeoutMillis, + IDspHotwordDetectionCallback callback) throws RemoteException { if (DBG) { Log.d(TAG, "#detectFromDspSource"); } mHandler.sendMessage(obtainMessage(HotwordDetectionService::onDetectFromDspSource, HotwordDetectionService.this, - sessionId, new DspHotwordDetectionCallback(callback))); + audioStream, + audioFormat, + timeoutMillis, + new DspHotwordDetectionCallback(callback))); } }; @@ -89,15 +99,24 @@ public abstract class HotwordDetectionService extends Service { /** * Detect the audio data generated from Dsp. * - * @param sessionId The session to use when attempting to capture more audio from the DSP - * hardware. + *

Note: the clients are supposed to call {@code close} on the input stream when they are + * done with the operation in order to free up resources. + * + * @param audioStream Stream containing audio bytes returned from DSP + * @param audioFormat Format of the supplied audio + * @param timeoutMillis Timeout in milliseconds for the operation to invoke the callback. If + * the application fails to abide by the timeout, system will close the + * microphone and cancel the operation. * @param callback Use {@link HotwordDetectionService#DspHotwordDetectionCallback} to return * the detected result. * * @hide */ @SystemApi - public void onDetectFromDspSource(int sessionId, + public void onDetectFromDspSource( + @NonNull ParcelFileDescriptor audioStream, + @NonNull AudioFormat audioFormat, + @DurationMillisLong long timeoutMillis, @NonNull DspHotwordDetectionCallback callback) { } diff --git a/core/java/android/service/voice/IHotwordDetectionService.aidl b/core/java/android/service/voice/IHotwordDetectionService.aidl index 990ad4574788f..cbe76e4bf69fa 100644 --- a/core/java/android/service/voice/IHotwordDetectionService.aidl +++ b/core/java/android/service/voice/IHotwordDetectionService.aidl @@ -16,6 +16,8 @@ package android.service.voice; +import android.media.AudioFormat; +import android.os.ParcelFileDescriptor; import android.service.voice.IDspHotwordDetectionCallback; /** @@ -24,5 +26,9 @@ import android.service.voice.IDspHotwordDetectionCallback; * @hide */ oneway interface IHotwordDetectionService { - void detectFromDspSource(int sessionId, in IDspHotwordDetectionCallback callback); + void detectFromDspSource( + in ParcelFileDescriptor audioStream, + in AudioFormat audioFormat, + long timeoutMillis, + in IDspHotwordDetectionCallback callback); } diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java index 35e65c1050557..ed71d17b0ddea 100644 --- a/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java +++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java @@ -22,25 +22,49 @@ import android.content.Context; import android.content.Intent; import android.hardware.soundtrigger.IRecognitionStatusCallback; import android.hardware.soundtrigger.SoundTrigger; +import android.media.AudioAttributes; +import android.media.AudioRecord; +import android.media.MediaRecorder; +import android.os.ParcelFileDescriptor; import android.os.RemoteException; import android.service.voice.AlwaysOnHotwordDetector; import android.service.voice.HotwordDetectionService; import android.service.voice.IDspHotwordDetectionCallback; import android.service.voice.IHotwordDetectionService; +import android.util.Pair; import android.util.Slog; import com.android.internal.app.IHotwordRecognitionStatusCallback; import com.android.internal.infra.ServiceConnector; +import java.io.IOException; +import java.io.OutputStream; import java.io.PrintWriter; +import java.util.concurrent.Executor; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; /** * A class that provides the communication with the HotwordDetectionService. */ final class HotwordDetectionConnection { - static final String TAG = "HotwordDetectionConnection"; + private static final String TAG = "HotwordDetectionConnection"; // TODO (b/177502877): Set the Debug flag to false before shipping. - static final boolean DEBUG = true; + private static final boolean DEBUG = true; + + // Number of bytes per sample of audio (which is a short). + private static final int BYTES_PER_SAMPLE = 2; + // TODO: These constants need to be refined. + private static final long VALIDATION_TIMEOUT_MILLIS = 3000; + private static final long VOICE_INTERACTION_TIMEOUT_TO_OPEN_MIC_MILLIS = 2000; + private static final int MAX_STREAMING_SECONDS = 10; + + private final Executor mAudioCopyExecutor = Executors.newCachedThreadPool(); + // TODO: This may need to be a Handler(looper) + private final ScheduledExecutorService mScheduledExecutorService = + Executors.newSingleThreadScheduledExecutor(); final Object mLock; final ComponentName mDetectionComponentName; @@ -93,42 +117,106 @@ final class HotwordDetectionConnection { } } - private void detectFromDspSource(int sessionId, IDspHotwordDetectionCallback callback) { + private void detectFromDspSource(SoundTrigger.KeyphraseRecognitionEvent recognitionEvent, + IHotwordRecognitionStatusCallback externalCallback) { if (DEBUG) { Slog.d(TAG, "detectFromDspSource"); } + + AudioRecord record = createAudioRecord(recognitionEvent); + + Pair clientPipe = createPipe(); + + if (clientPipe == null) { + // Error. + // Need to propagate as unknown error or something? + return; + } + ParcelFileDescriptor audioSink = clientPipe.second; + ParcelFileDescriptor clientRead = clientPipe.first; + + record.startRecording(); + + mAudioCopyExecutor.execute(() -> { + try (OutputStream fos = + new ParcelFileDescriptor.AutoCloseOutputStream(audioSink)) { + byte[] buffer = new byte[1024]; + + while (true) { + int bytesRead = record.read(buffer, 0, 1024); + + if (bytesRead < 0) { + break; + } + + fos.write(buffer, 0, bytesRead); + } + } catch (IOException e) { + Slog.w(TAG, "Failed supplying audio data to validator", e); + } + }); + + Runnable cancellingJob = () -> { + record.stop(); + bestEffortCloseFileDescriptor(audioSink); + // TODO: consider calling externalCallback.onRejected(ERROR_TIMEOUT). + }; + + ScheduledFuture cancelingFuture = + mScheduledExecutorService.schedule( + cancellingJob, VALIDATION_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS); + + // TODO: consider making this a non-anonymous class. + IDspHotwordDetectionCallback internalCallback = new IDspHotwordDetectionCallback.Stub() { + @Override + public void onDetected() throws RemoteException { + if (DEBUG) { + Slog.d(TAG, "onDetected"); + } + bestEffortCloseFileDescriptor(audioSink); + cancelingFuture.cancel(true); + + // Give 2 more seconds for the interactor to start consuming the mic. If it fails to + // do so under the given time, we'll force-close the mic to make sure resources are + // freed up. + // TODO: consider modelling these 2 seconds in the API. + mScheduledExecutorService.schedule( + cancellingJob, + VOICE_INTERACTION_TIMEOUT_TO_OPEN_MIC_MILLIS, + TimeUnit.MILLISECONDS); + + externalCallback.onKeyphraseDetected(recognitionEvent); + } + + @Override + public void onRejected() throws RemoteException { + if (DEBUG) { + Slog.d(TAG, "onRejected"); + } + cancelingFuture.cancel(true); + externalCallback.onRejected( + AlwaysOnHotwordDetector.HOTWORD_DETECTION_FALSE_ALERT); + } + }; + mRemoteHotwordDetectionService.run( - service -> service.detectFromDspSource(sessionId, callback)); + service -> service.detectFromDspSource( + clientRead, + recognitionEvent.getCaptureFormat(), + VALIDATION_TIMEOUT_MILLIS, + internalCallback)); + bestEffortCloseFileDescriptor(clientRead); } static final class SoundTriggerCallback extends IRecognitionStatusCallback.Stub { private SoundTrigger.KeyphraseRecognitionEvent mRecognitionEvent; private final HotwordDetectionConnection mHotwordDetectionConnection; private final IHotwordRecognitionStatusCallback mExternalCallback; - private final IDspHotwordDetectionCallback mInternalCallback; SoundTriggerCallback(IHotwordRecognitionStatusCallback callback, HotwordDetectionConnection connection) { mHotwordDetectionConnection = connection; mExternalCallback = callback; - mInternalCallback = new IDspHotwordDetectionCallback.Stub() { - @Override - public void onDetected() throws RemoteException { - if (DEBUG) { - Slog.d(TAG, "onDetected"); - } - mExternalCallback.onKeyphraseDetected(mRecognitionEvent); - } - - @Override - public void onRejected() throws RemoteException { - if (DEBUG) { - Slog.d(TAG, "onRejected"); - } - mExternalCallback.onRejected( - AlwaysOnHotwordDetector.HOTWORD_DETECTION_FALSE_ALERT); - } - }; } @Override @@ -142,7 +230,7 @@ final class HotwordDetectionConnection { if (useHotwordDetectionService) { mRecognitionEvent = recognitionEvent; mHotwordDetectionConnection.detectFromDspSource( - recognitionEvent.getCaptureSession(), mInternalCallback); + recognitionEvent, mExternalCallback); } else { mExternalCallback.onKeyphraseDetected(recognitionEvent); } @@ -171,6 +259,48 @@ final class HotwordDetectionConnection { } } + // TODO: figure out if we need to let the client configure some of the parameters. + private static AudioRecord createAudioRecord( + @NonNull SoundTrigger.KeyphraseRecognitionEvent recognitionEvent) { + int sampleRate = recognitionEvent.getCaptureFormat().getSampleRate(); + return new AudioRecord( + new AudioAttributes.Builder() + .setInternalCapturePreset(MediaRecorder.AudioSource.HOTWORD).build(), + recognitionEvent.getCaptureFormat(), + getBufferSizeInBytes(sampleRate, MAX_STREAMING_SECONDS), + recognitionEvent.getCaptureSession()); + } + + /** + * Returns the number of bytes required to store {@code bufferLengthSeconds} of audio sampled at + * {@code sampleRate} Hz, using the format returned by DSP audio capture. + */ + private static int getBufferSizeInBytes(int sampleRate, int bufferLengthSeconds) { + return BYTES_PER_SAMPLE * sampleRate * bufferLengthSeconds; + } + + private static Pair createPipe() { + ParcelFileDescriptor[] fileDescriptors; + try { + fileDescriptors = ParcelFileDescriptor.createPipe(); + } catch (IOException e) { + Slog.e(TAG, "Failed to create audio stream pipe", e); + return null; + } + + return Pair.create(fileDescriptors[0], fileDescriptors[1]); + } + + private static void bestEffortCloseFileDescriptor(ParcelFileDescriptor fd) { + try { + fd.close(); + } catch (IOException e) { + if (DEBUG) { + Slog.w(TAG, "Failed closing file descriptor", e); + } + } + } + public void dump(String prefix, PrintWriter pw) { pw.print(prefix); pw.print("mBound="); pw.println(mBound); }