Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Copy data in HotwordAudioStream ParcelFileDescriptors

Browse Source 
Bug: 244595867
Test: TODO - Manual E2E test for now, CTS tests later
Change-Id: I02050cc39146e93e9dca1f3db6e0b5b894e85dd6
Merged-In: I02050cc39146e93e9dca1f3db6e0b5b894e85dd6
This commit is contained in:
Mark Punzalan
2022-10-21 11:04:54 +00:00
committed by Ivan Chiang
parent 3b18e55af3
commit 15055a4e41
4 changed files with 316 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
core/java/android/service/voice/HotwordAudioStream.java
View File

@@ -123,6 +123,16 @@ public final class HotwordAudioStream implements Parcelable {
}
}
/**
* Provides an instance of {@link Builder} with state corresponding to this instance.
* @hide
*/
public Builder buildUpon() {
return new Builder(mAudioFormat, mAudioStreamParcelFileDescriptor)
.setTimestamp(mTimestamp)
.setMetadata(mMetadata);
}
/* package-private */
HotwordAudioStream(
@NonNull AudioFormat audioFormat,

23
core/java/android/service/voice/HotwordDetectedResult.java
View File

@@ -397,6 +397,25 @@ public final class HotwordDetectedResult implements Parcelable {
}
}
/**
* Provides an instance of {@link Builder} with state corresponding to this instance.
* @hide
*/
public Builder buildUpon() {
return new Builder()
.setConfidenceLevel(mConfidenceLevel)
.setMediaSyncEvent(mMediaSyncEvent)
.setHotwordOffsetMillis(mHotwordOffsetMillis)
.setHotwordDurationMillis(mHotwordDurationMillis)
.setAudioChannel(mAudioChannel)
.setHotwordDetectionPersonalized(mHotwordDetectionPersonalized)
.setScore(mScore)
.setPersonalizedScore(mPersonalizedScore)
.setHotwordPhraseId(mHotwordPhraseId)
.setAudioStreams(mAudioStreams)
.setExtras(mExtras);
}
// Code below generated by codegen v1.0.23.
@@ -993,10 +1012,10 @@ public final class HotwordDetectedResult implements Parcelable {
}
@DataClass.Generated(
time = 1668405106028L,
time = 1668466781144L,
codegenVersion = "1.0.23",
sourceFile = "frameworks/base/core/java/android/service/voice/HotwordDetectedResult.java",
inputSignatures = "public static final int CONFIDENCE_LEVEL_NONE\npublic static final int CONFIDENCE_LEVEL_LOW\npublic static final int CONFIDENCE_LEVEL_LOW_MEDIUM\npublic static final int CONFIDENCE_LEVEL_MEDIUM\npublic static final int CONFIDENCE_LEVEL_MEDIUM_HIGH\npublic static final int CONFIDENCE_LEVEL_HIGH\npublic static final int CONFIDENCE_LEVEL_VERY_HIGH\npublic static final int HOTWORD_OFFSET_UNSET\npublic static final int AUDIO_CHANNEL_UNSET\nprivate static final int LIMIT_HOTWORD_OFFSET_MAX_VALUE\nprivate static final int LIMIT_AUDIO_CHANNEL_MAX_VALUE\npublic static final java.lang.String EXTRA_PROXIMITY_METERS\nprivate final @android.service.voice.HotwordDetectedResult.HotwordConfidenceLevelValue int mConfidenceLevel\nprivate @android.annotation.Nullable android.media.MediaSyncEvent mMediaSyncEvent\nprivate int mHotwordOffsetMillis\nprivate int mHotwordDurationMillis\nprivate int mAudioChannel\nprivate boolean mHotwordDetectionPersonalized\nprivate final int mScore\nprivate final int mPersonalizedScore\nprivate final int mHotwordPhraseId\nprivate final @android.annotation.NonNull java.util.List<android.service.voice.HotwordAudioStream> mAudioStreams\nprivate final @android.annotation.NonNull android.os.PersistableBundle mExtras\nprivate static int sMaxBundleSize\nprivate static int defaultConfidenceLevel()\nprivate static int defaultScore()\nprivate static int defaultPersonalizedScore()\npublic static int getMaxScore()\nprivate static int defaultHotwordPhraseId()\npublic static int getMaxHotwordPhraseId()\nprivate static java.util.List<android.service.voice.HotwordAudioStream> defaultAudioStreams()\nprivate static android.os.PersistableBundle defaultExtras()\npublic static int getMaxBundleSize()\npublic @android.annotation.Nullable android.media.MediaSyncEvent getMediaSyncEvent()\npublic static int getParcelableSize(android.os.Parcelable)\npublic static int getUsageSize(android.service.voice.HotwordDetectedResult)\nprivate static int bitCount(long)\nprivate void onConstructed()\npublic @android.compat.annotation.UnsupportedAppUsage @android.annotation.NonNull java.util.List<android.service.voice.HotwordAudioStream> getAudioStreams()\nclass HotwordDetectedResult extends java.lang.Object implements [android.os.Parcelable]\npublic @android.compat.annotation.UnsupportedAppUsage @android.annotation.NonNull android.service.voice.HotwordDetectedResult.Builder setAudioStreams(java.util.List<android.service.voice.HotwordAudioStream>)\nclass BaseBuilder extends java.lang.Object implements []\n@com.android.internal.util.DataClass(genConstructor=false, genBuilder=true, genEqualsHashCode=true, genHiddenConstDefs=true, genParcelable=true, genToString=true)\npublic @android.compat.annotation.UnsupportedAppUsage @android.annotation.NonNull android.service.voice.HotwordDetectedResult.Builder setAudioStreams(java.util.List<android.service.voice.HotwordAudioStream>)\nclass BaseBuilder extends java.lang.Object implements []")
inputSignatures = "public static final int CONFIDENCE_LEVEL_NONE\npublic static final int CONFIDENCE_LEVEL_LOW\npublic static final int CONFIDENCE_LEVEL_LOW_MEDIUM\npublic static final int CONFIDENCE_LEVEL_MEDIUM\npublic static final int CONFIDENCE_LEVEL_MEDIUM_HIGH\npublic static final int CONFIDENCE_LEVEL_HIGH\npublic static final int CONFIDENCE_LEVEL_VERY_HIGH\npublic static final int HOTWORD_OFFSET_UNSET\npublic static final int AUDIO_CHANNEL_UNSET\nprivate static final int LIMIT_HOTWORD_OFFSET_MAX_VALUE\nprivate static final int LIMIT_AUDIO_CHANNEL_MAX_VALUE\npublic static final java.lang.String EXTRA_PROXIMITY_METERS\nprivate final @android.service.voice.HotwordDetectedResult.HotwordConfidenceLevelValue int mConfidenceLevel\nprivate @android.annotation.Nullable android.media.MediaSyncEvent mMediaSyncEvent\nprivate int mHotwordOffsetMillis\nprivate int mHotwordDurationMillis\nprivate int mAudioChannel\nprivate boolean mHotwordDetectionPersonalized\nprivate final int mScore\nprivate final int mPersonalizedScore\nprivate final int mHotwordPhraseId\nprivate final @android.annotation.NonNull java.util.List<android.service.voice.HotwordAudioStream> mAudioStreams\nprivate final @android.annotation.NonNull android.os.PersistableBundle mExtras\nprivate static int sMaxBundleSize\nprivate static int defaultConfidenceLevel()\nprivate static int defaultScore()\nprivate static int defaultPersonalizedScore()\npublic static int getMaxScore()\nprivate static int defaultHotwordPhraseId()\npublic static int getMaxHotwordPhraseId()\nprivate static java.util.List<android.service.voice.HotwordAudioStream> defaultAudioStreams()\nprivate static android.os.PersistableBundle defaultExtras()\npublic static int getMaxBundleSize()\npublic @android.annotation.Nullable android.media.MediaSyncEvent getMediaSyncEvent()\npublic static int getParcelableSize(android.os.Parcelable)\npublic static int getUsageSize(android.service.voice.HotwordDetectedResult)\nprivate static int bitCount(long)\nprivate void onConstructed()\npublic @android.compat.annotation.UnsupportedAppUsage @android.annotation.NonNull java.util.List<android.service.voice.HotwordAudioStream> getAudioStreams()\npublic android.service.voice.HotwordDetectedResult.Builder buildUpon()\nclass HotwordDetectedResult extends java.lang.Object implements [android.os.Parcelable]\npublic @android.compat.annotation.UnsupportedAppUsage @android.annotation.NonNull android.service.voice.HotwordDetectedResult.Builder setAudioStreams(java.util.List<android.service.voice.HotwordAudioStream>)\nclass BaseBuilder extends java.lang.Object implements []\n@com.android.internal.util.DataClass(genConstructor=false, genBuilder=true, genEqualsHashCode=true, genHiddenConstDefs=true, genParcelable=true, genToString=true)\npublic @android.compat.annotation.UnsupportedAppUsage @android.annotation.NonNull android.service.voice.HotwordDetectedResult.Builder setAudioStreams(java.util.List<android.service.voice.HotwordAudioStream>)\nclass BaseBuilder extends java.lang.Object implements []")
@Deprecated
private void __metadata() {}

233
services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordAudioStreamManager.java Normal file
View File

@@ -0,0 +1,233 @@
/*
* Copyright (C) 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.server.voiceinteraction;
import static android.app.AppOpsManager.MODE_ALLOWED;
import static com.android.server.voiceinteraction.HotwordDetectionConnection.DEBUG;
import android.annotation.NonNull;
import android.app.AppOpsManager;
import android.media.permission.Identity;
import android.os.ParcelFileDescriptor;
import android.service.voice.HotwordAudioStream;
import android.service.voice.HotwordDetectedResult;
import android.util.Pair;
import android.util.Slog;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
final class HotwordAudioStreamManager {
private static final String TAG = "HotwordAudioStreamManager";
private static final String OP_MESSAGE = "Streaming hotword audio to VoiceInteractionService";
private static final String TASK_ID_PREFIX = "HotwordDetectedResult@";
private static final String THREAD_NAME_PREFIX = "Copy-";
private final AppOpsManager mAppOpsManager;
private final Identity mVoiceInteractorIdentity;
private final ExecutorService mExecutorService = Executors.newCachedThreadPool();
HotwordAudioStreamManager(@NonNull AppOpsManager appOpsManager,
@NonNull Identity voiceInteractorIdentity) {
mAppOpsManager = appOpsManager;
mVoiceInteractorIdentity = voiceInteractorIdentity;
}
/**
* Starts copying the audio streams in the given {@link HotwordDetectedResult}.
* <p>
* The returned {@link HotwordDetectedResult} is identical the one that was passed in, except
* that the {@link ParcelFileDescriptor}s within {@link HotwordDetectedResult#getAudioStreams()}
* are replaced with descriptors from pipes managed by {@link HotwordAudioStreamManager}. The
* returned value should be passed on to the client (i.e., the voice interactor).
* </p>
*
* @throws IOException If there was an error creating the managed pipe.
*/
@NonNull
public HotwordDetectedResult startCopyingAudioStreams(@NonNull HotwordDetectedResult result)
throws IOException {
List<HotwordAudioStream> audioStreams = result.getAudioStreams();
if (audioStreams.isEmpty()) {
return result;
}
List<HotwordAudioStream> newAudioStreams = new ArrayList<>(audioStreams.size());
List<Pair<ParcelFileDescriptor, ParcelFileDescriptor>> sourcesAndSinks = new ArrayList<>(
audioStreams.size());
for (HotwordAudioStream audioStream : audioStreams) {
ParcelFileDescriptor[] clientPipe = ParcelFileDescriptor.createReliablePipe();
ParcelFileDescriptor clientAudioSource = clientPipe[0];
ParcelFileDescriptor clientAudioSink = clientPipe[1];
HotwordAudioStream newAudioStream =
audioStream.buildUpon().setAudioStreamParcelFileDescriptor(
clientAudioSource).build();
newAudioStreams.add(newAudioStream);
ParcelFileDescriptor serviceAudioSource =
audioStream.getAudioStreamParcelFileDescriptor();
sourcesAndSinks.add(new Pair<>(serviceAudioSource, clientAudioSink));
}
String resultTaskId = TASK_ID_PREFIX + System.identityHashCode(result);
mExecutorService.execute(new HotwordDetectedResultCopyTask(resultTaskId, sourcesAndSinks));
return result.buildUpon().setAudioStreams(newAudioStreams).build();
}
private class HotwordDetectedResultCopyTask implements Runnable {
private final String mResultTaskId;
private final List<Pair<ParcelFileDescriptor, ParcelFileDescriptor>> mSourcesAndSinks;
private final ExecutorService mExecutorService = Executors.newCachedThreadPool();
HotwordDetectedResultCopyTask(String resultTaskId,
List<Pair<ParcelFileDescriptor, ParcelFileDescriptor>> sourcesAndSinks) {
mResultTaskId = resultTaskId;
mSourcesAndSinks = sourcesAndSinks;
}
@Override
public void run() {
Thread.currentThread().setName(THREAD_NAME_PREFIX + mResultTaskId);
int size = mSourcesAndSinks.size();
List<SingleAudioStreamCopyTask> tasks = new ArrayList<>(size);
for (int i = 0; i < size; i++) {
Pair<ParcelFileDescriptor, ParcelFileDescriptor> sourceAndSink =
mSourcesAndSinks.get(i);
ParcelFileDescriptor serviceAudioSource = sourceAndSink.first;
ParcelFileDescriptor clientAudioSink = sourceAndSink.second;
String streamTaskId = mResultTaskId + "@" + i;
tasks.add(new SingleAudioStreamCopyTask(streamTaskId, serviceAudioSource,
clientAudioSink));
}
if (mAppOpsManager.startOpNoThrow(AppOpsManager.OPSTR_RECORD_AUDIO_HOTWORD,
mVoiceInteractorIdentity.uid, mVoiceInteractorIdentity.packageName,
mVoiceInteractorIdentity.attributionTag, OP_MESSAGE) == MODE_ALLOWED) {
try {
// TODO(b/244599891): Set timeout, close after inactivity
mExecutorService.invokeAll(tasks);
} catch (InterruptedException e) {
Slog.e(TAG, mResultTaskId + ": Task was interrupted", e);
bestEffortPropagateError(e.getMessage());
} finally {
mAppOpsManager.finishOp(AppOpsManager.OPSTR_RECORD_AUDIO_HOTWORD,
mVoiceInteractorIdentity.uid, mVoiceInteractorIdentity.packageName,
mVoiceInteractorIdentity.attributionTag);
}
} else {
bestEffortPropagateError(
"Failed to obtain RECORD_AUDIO_HOTWORD permission for "
+ SoundTriggerSessionPermissionsDecorator.toString(
mVoiceInteractorIdentity));
}
}
private void bestEffortPropagateError(@NonNull String errorMessage) {
try {
for (Pair<ParcelFileDescriptor, ParcelFileDescriptor> sourceAndSink :
mSourcesAndSinks) {
ParcelFileDescriptor serviceAudioSource = sourceAndSink.first;
ParcelFileDescriptor clientAudioSink = sourceAndSink.second;
serviceAudioSource.closeWithError(errorMessage);
clientAudioSink.closeWithError(errorMessage);
}
} catch (IOException e) {
Slog.e(TAG, mResultTaskId + ": Failed to propagate error", e);
}
}
}
private static class SingleAudioStreamCopyTask implements Callable<Void> {
// TODO: Make this buffer size customizable from updateState()
private static final int COPY_BUFFER_LENGTH = 1_024;
private final String mStreamTaskId;
private final ParcelFileDescriptor mAudioSource;
private final ParcelFileDescriptor mAudioSink;
SingleAudioStreamCopyTask(String streamTaskId, ParcelFileDescriptor audioSource,
ParcelFileDescriptor audioSink) {
mStreamTaskId = streamTaskId;
mAudioSource = audioSource;
mAudioSink = audioSink;
}
@Override
public Void call() throws Exception {
Thread.currentThread().setName(THREAD_NAME_PREFIX + mStreamTaskId);
// Note: We are intentionally NOT using try-with-resources here. If we did,
// the ParcelFileDescriptors will be automatically closed WITHOUT errors before we go
// into the IOException-catch block. We want to propagate the error while closing the
// PFDs.
InputStream fis = null;
OutputStream fos = null;
try {
fis = new ParcelFileDescriptor.AutoCloseInputStream(mAudioSource);
fos = new ParcelFileDescriptor.AutoCloseOutputStream(mAudioSink);
byte[] buffer = new byte[COPY_BUFFER_LENGTH];
while (true) {
if (Thread.interrupted()) {
Slog.e(TAG,
mStreamTaskId + ": SingleAudioStreamCopyTask task was interrupted");
break;
}
int bytesRead = fis.read(buffer);
if (bytesRead < 0) {
Slog.i(TAG, mStreamTaskId + ": Reached end of audio stream");
break;
}
if (bytesRead > 0) {
if (DEBUG) {
// TODO(b/244599440): Add proper logging
Slog.d(TAG, mStreamTaskId + ": Copied " + bytesRead
+ " bytes from audio stream. First 20 bytes=" + Arrays.toString(
Arrays.copyOfRange(buffer, 0, 20)));
}
fos.write(buffer, 0, bytesRead);
}
// TODO(b/244599891): Close PFDs after inactivity
}
} catch (IOException e) {
mAudioSource.closeWithError(e.getMessage());
mAudioSink.closeWithError(e.getMessage());
Slog.e(TAG, mStreamTaskId + ": Failed to copy audio stream", e);
} finally {
if (fis != null) {
fis.close();
}
if (fos != null) {
fos.close();
}
}
return null;
}
}
}

75
services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java
View File

@@ -59,6 +59,7 @@ import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPH
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.app.AppOpsManager;
import android.attention.AttentionManagerInternal;
import android.content.ComponentName;
import android.content.ContentCaptureOptions;
@@ -137,6 +138,7 @@ final class HotwordDetectionConnection {
// The error codes are used for onError callback
private static final int HOTWORD_DETECTION_SERVICE_DIED = -1;
private static final int CALLBACK_ONDETECTED_GOT_SECURITY_EXCEPTION = -2;
private static final int CALLBACK_ONDETECTED_STREAM_COPY_ERROR = -4;
// Hotword metrics
private static final int METRICS_INIT_UNKNOWN_TIMEOUT =
@@ -168,6 +170,8 @@ final class HotwordDetectionConnection {
// TODO: This may need to be a Handler(looper)
private final ScheduledExecutorService mScheduledExecutorService =
Executors.newSingleThreadScheduledExecutor();
private final AppOpsManager mAppOpsManager;
private final HotwordAudioStreamManager mHotwordAudioStreamManager;
@Nullable private final ScheduledFuture<?> mCancellationTaskFuture;
private final AtomicBoolean mUpdateStateAfterStartFinished = new AtomicBoolean(false);
private final IBinder.DeathRecipient mAudioServerDeathRecipient = this::audioServerDied;
@@ -228,6 +232,9 @@ final class HotwordDetectionConnection {
mContext = context;
mVoiceInteractionServiceUid = voiceInteractionServiceUid;
mVoiceInteractorIdentity = voiceInteractorIdentity;
mAppOpsManager = mContext.getSystemService(AppOpsManager.class);
mHotwordAudioStreamManager = new HotwordAudioStreamManager(mAppOpsManager,
mVoiceInteractorIdentity);
mDetectionComponentName = serviceName;
mUser = userId;
mCallback = callback;
@@ -482,13 +489,19 @@ final class HotwordDetectionConnection {
return;
}
saveProximityMetersToBundle(result);
mSoftwareCallback.onDetected(result, null, null);
if (result != null) {
Slog.i(TAG, "Egressed " + HotwordDetectedResult.getUsageSize(result)
+ " bits from hotword trusted process");
if (mDebugHotwordLogging) {
Slog.i(TAG, "Egressed detected result: " + result);
}
HotwordDetectedResult newResult;
try {
newResult = mHotwordAudioStreamManager.startCopyingAudioStreams(result);
} catch (IOException e) {
// TODO: Write event
mSoftwareCallback.onError();
return;
}
mSoftwareCallback.onDetected(newResult, null, null);
Slog.i(TAG, "Egressed " + HotwordDetectedResult.getUsageSize(newResult)
+ " bits from hotword trusted process");
if (mDebugHotwordLogging) {
Slog.i(TAG, "Egressed detected result: " + newResult);
}
}
}
@@ -660,6 +673,7 @@ final class HotwordDetectionConnection {
try {
enforcePermissionsForDataDelivery();
} catch (SecurityException e) {
Slog.i(TAG, "Ignoring #onDetected due to a SecurityException", e);
HotwordMetricsLogger.writeKeyphraseTriggerEvent(
mDetectorType,
METRICS_KEYPHRASE_TRIGGERED_DETECT_SECURITY_EXCEPTION);
@@ -667,13 +681,19 @@ final class HotwordDetectionConnection {
return;
}
saveProximityMetersToBundle(result);
externalCallback.onKeyphraseDetected(recognitionEvent, result);
if (result != null) {
Slog.i(TAG, "Egressed " + HotwordDetectedResult.getUsageSize(result)
+ " bits from hotword trusted process");
if (mDebugHotwordLogging) {
Slog.i(TAG, "Egressed detected result: " + result);
}
HotwordDetectedResult newResult;
try {
newResult = mHotwordAudioStreamManager.startCopyingAudioStreams(result);
} catch (IOException e) {
// TODO: Write event
externalCallback.onError(CALLBACK_ONDETECTED_STREAM_COPY_ERROR);
return;
}
externalCallback.onKeyphraseDetected(recognitionEvent, newResult);
Slog.i(TAG, "Egressed " + HotwordDetectedResult.getUsageSize(newResult)
+ " bits from hotword trusted process");
if (mDebugHotwordLogging) {
Slog.i(TAG, "Egressed detected result: " + newResult);
}
}
}
@@ -757,6 +777,7 @@ final class HotwordDetectionConnection {
}
private void restartProcessLocked() {
// TODO(b/244598068): Check HotwordAudioStreamManager first
Slog.v(TAG, "Restarting hotword detection process");
ServiceConnection oldConnection = mRemoteHotwordDetectionService;
HotwordDetectionServiceIdentity previousIdentity = mIdentity;
@@ -991,16 +1012,24 @@ final class HotwordDetectionConnection {
callback.onError();
return;
}
callback.onDetected(triggerResult, null /* audioFormat */,
HotwordDetectedResult newResult;
try {
newResult =
mHotwordAudioStreamManager.startCopyingAudioStreams(
triggerResult);
} catch (IOException e) {
// TODO: Write event
callback.onError();
return;
}
callback.onDetected(newResult, null /* audioFormat */,
null /* audioStream */);
if (triggerResult != null) {
Slog.i(TAG, "Egressed "
+ HotwordDetectedResult.getUsageSize(triggerResult)
+ " bits from hotword trusted process");
if (mDebugHotwordLogging) {
Slog.i(TAG,
"Egressed detected result: " + triggerResult);
}
Slog.i(TAG, "Egressed "
+ HotwordDetectedResult.getUsageSize(newResult)
+ " bits from hotword trusted process");
if (mDebugHotwordLogging) {
Slog.i(TAG,
"Egressed detected result: " + newResult);
}
}
});