Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge changes I43a24b27,I1c70f95a,I28c7f21b

Browse Source 
* changes:
  Rename HotwordAudioStreamManager and add Javadoc
  Remove dependency on Identity in HotwordAudioStreamManager
  Allow setting buffer length for HotwordAudioStreamManager
This commit is contained in:
Mark Punzalan
2022-12-01 23:29:02 +00:00
committed by Android (Google) Code Review
parent 67709e076d 585c555cc9
commit 7d87534508
5 changed files with 110 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
core/api/system-current.txt
View File

@@ -12277,6 +12277,7 @@ package android.service.voice {
method @Nullable public android.media.AudioTimestamp getTimestamp();
method public void writeToParcel(@NonNull android.os.Parcel, int);
field @NonNull public static final android.os.Parcelable.Creator<android.service.voice.HotwordAudioStream> CREATOR;
field public static final String KEY_AUDIO_STREAM_COPY_BUFFER_LENGTH_BYTES = "android.service.voice.key.AUDIO_STREAM_COPY_BUFFER_LENGTH_BYTES";
}
public static final class HotwordAudioStream.Builder {

19
core/java/android/service/voice/HotwordAudioStream.java
View File

@@ -46,6 +46,21 @@ import java.util.Objects;
@SystemApi
public final class HotwordAudioStream implements Parcelable {
/**
* Key for int value to be read from {@link #getMetadata()}. The value is read by the system and
* is the length (in bytes) of the byte buffers created to copy bytes in the
* {@link #getAudioStreamParcelFileDescriptor()} written by the {@link HotwordDetectionService}.
* The buffer length should be chosen such that no additional latency is introduced. Typically,
* this should be <em>at least</em> the size of byte chunks written by the
* {@link HotwordDetectionService}.
*
* <p>If no value specified in the metadata for the buffer length, or if the value is less than
* 1, or if it is greater than 65,536, or if it is not an int, the default value of 2,560 will
* be used.</p>
*/
public static final String KEY_AUDIO_STREAM_COPY_BUFFER_LENGTH_BYTES =
"android.service.voice.key.AUDIO_STREAM_COPY_BUFFER_LENGTH_BYTES";
/**
* The {@link AudioFormat} of the audio stream.
*/
@@ -414,10 +429,10 @@ public final class HotwordAudioStream implements Parcelable {
}
@DataClass.Generated(
time = 1669184301563L,
time = 1669916341034L,
codegenVersion = "1.0.23",
sourceFile = "frameworks/base/core/java/android/service/voice/HotwordAudioStream.java",
inputSignatures = "private final @android.annotation.NonNull android.media.AudioFormat mAudioFormat\nprivate final @android.annotation.NonNull android.os.ParcelFileDescriptor mAudioStreamParcelFileDescriptor\nprivate final @android.annotation.Nullable android.media.AudioTimestamp mTimestamp\nprivate final @android.annotation.NonNull android.os.PersistableBundle mMetadata\nprivate static android.media.AudioTimestamp defaultTimestamp()\nprivate static android.os.PersistableBundle defaultMetadata()\npublic android.service.voice.HotwordAudioStream.Builder buildUpon()\nclass HotwordAudioStream extends java.lang.Object implements [android.os.Parcelable]\n@com.android.internal.util.DataClass(genConstructor=false, genBuilder=true, genEqualsHashCode=true, genParcelable=true, genToString=true)")
inputSignatures = "public static final java.lang.String KEY_AUDIO_STREAM_COPY_BUFFER_LENGTH_BYTES\nprivate final @android.annotation.NonNull android.media.AudioFormat mAudioFormat\nprivate final @android.annotation.NonNull android.os.ParcelFileDescriptor mAudioStreamParcelFileDescriptor\nprivate final @android.annotation.Nullable android.media.AudioTimestamp mTimestamp\nprivate final @android.annotation.NonNull android.os.PersistableBundle mMetadata\nprivate static android.media.AudioTimestamp defaultTimestamp()\nprivate static android.os.PersistableBundle defaultMetadata()\npublic android.service.voice.HotwordAudioStream.Builder buildUpon()\nclass HotwordAudioStream extends java.lang.Object implements [android.os.Parcelable]\n@com.android.internal.util.DataClass(genConstructor=false, genBuilder=true, genEqualsHashCode=true, genParcelable=true, genToString=true)")
@Deprecated
private void __metadata() {}

120
services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordAudioStreamManager.java → services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordAudioStreamCopier.java
View File

@@ -17,16 +17,16 @@
package com.android.server.voiceinteraction;
import static android.app.AppOpsManager.MODE_ALLOWED;
import static android.service.voice.HotwordAudioStream.KEY_AUDIO_STREAM_COPY_BUFFER_LENGTH_BYTES;
import static com.android.server.voiceinteraction.HotwordDetectionConnection.DEBUG;
import android.annotation.NonNull;
import android.app.AppOpsManager;
import android.media.permission.Identity;
import android.os.ParcelFileDescriptor;
import android.os.PersistableBundle;
import android.service.voice.HotwordAudioStream;
import android.service.voice.HotwordDetectedResult;
import android.util.Pair;
import android.util.Slog;
import java.io.IOException;
@@ -39,21 +39,38 @@ import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
final class HotwordAudioStreamManager {
/**
* Copies the audio streams in {@link HotwordDetectedResult}s. This allows the system to manage the
* lifetime of the {@link ParcelFileDescriptor}s and ensures that the flow of data is in the right
* direction from the {@link android.service.voice.HotwordDetectionService} to the client (i.e., the
* voice interactor).
*
* @hide
*/
final class HotwordAudioStreamCopier {
private static final String TAG = "HotwordAudioStreamManager";
private static final String TAG = "HotwordAudioStreamCopier";
private static final String OP_MESSAGE = "Streaming hotword audio to VoiceInteractionService";
private static final String TASK_ID_PREFIX = "HotwordDetectedResult@";
private static final String THREAD_NAME_PREFIX = "Copy-";
private static final int DEFAULT_COPY_BUFFER_LENGTH_BYTES = 2_560;
// Corresponds to the OS pipe capacity in bytes
private static final int MAX_COPY_BUFFER_LENGTH_BYTES = 65_536;
private final AppOpsManager mAppOpsManager;
private final Identity mVoiceInteractorIdentity;
private final int mVoiceInteractorUid;
private final String mVoiceInteractorPackageName;
private final String mVoiceInteractorAttributionTag;
private final ExecutorService mExecutorService = Executors.newCachedThreadPool();
HotwordAudioStreamManager(@NonNull AppOpsManager appOpsManager,
@NonNull Identity voiceInteractorIdentity) {
HotwordAudioStreamCopier(@NonNull AppOpsManager appOpsManager,
int voiceInteractorUid, @NonNull String voiceInteractorPackageName,
@NonNull String voiceInteractorAttributionTag) {
mAppOpsManager = appOpsManager;
mVoiceInteractorIdentity = voiceInteractorIdentity;
mVoiceInteractorUid = voiceInteractorUid;
mVoiceInteractorPackageName = voiceInteractorPackageName;
mVoiceInteractorAttributionTag = voiceInteractorAttributionTag;
}
/**
@@ -61,7 +78,7 @@ final class HotwordAudioStreamManager {
* <p>
* The returned {@link HotwordDetectedResult} is identical the one that was passed in, except
* that the {@link ParcelFileDescriptor}s within {@link HotwordDetectedResult#getAudioStreams()}
* are replaced with descriptors from pipes managed by {@link HotwordAudioStreamManager}. The
* are replaced with descriptors from pipes managed by {@link HotwordAudioStreamCopier}. The
* returned value should be passed on to the client (i.e., the voice interactor).
* </p>
*
@@ -76,8 +93,7 @@ final class HotwordAudioStreamManager {
}
List<HotwordAudioStream> newAudioStreams = new ArrayList<>(audioStreams.size());
List<Pair<ParcelFileDescriptor, ParcelFileDescriptor>> sourcesAndSinks = new ArrayList<>(
audioStreams.size());
List<CopyTaskInfo> copyTaskInfos = new ArrayList<>(audioStreams.size());
for (HotwordAudioStream audioStream : audioStreams) {
ParcelFileDescriptor[] clientPipe = ParcelFileDescriptor.createReliablePipe();
ParcelFileDescriptor clientAudioSource = clientPipe[0];
@@ -87,46 +103,69 @@ final class HotwordAudioStreamManager {
clientAudioSource).build();
newAudioStreams.add(newAudioStream);
int copyBufferLength = DEFAULT_COPY_BUFFER_LENGTH_BYTES;
PersistableBundle metadata = audioStream.getMetadata();
if (metadata.containsKey(KEY_AUDIO_STREAM_COPY_BUFFER_LENGTH_BYTES)) {
copyBufferLength = metadata.getInt(KEY_AUDIO_STREAM_COPY_BUFFER_LENGTH_BYTES, -1);
if (copyBufferLength < 1 || copyBufferLength > MAX_COPY_BUFFER_LENGTH_BYTES) {
Slog.w(TAG, "Attempted to set an invalid copy buffer length ("
+ copyBufferLength + ") for: " + audioStream);
copyBufferLength = DEFAULT_COPY_BUFFER_LENGTH_BYTES;
} else if (DEBUG) {
Slog.i(TAG, "Copy buffer length set to " + copyBufferLength + " for: "
+ audioStream);
}
}
ParcelFileDescriptor serviceAudioSource =
audioStream.getAudioStreamParcelFileDescriptor();
sourcesAndSinks.add(new Pair<>(serviceAudioSource, clientAudioSink));
copyTaskInfos.add(new CopyTaskInfo(serviceAudioSource, clientAudioSink,
copyBufferLength));
}
String resultTaskId = TASK_ID_PREFIX + System.identityHashCode(result);
mExecutorService.execute(new HotwordDetectedResultCopyTask(resultTaskId, sourcesAndSinks));
mExecutorService.execute(new HotwordDetectedResultCopyTask(resultTaskId, copyTaskInfos));
return result.buildUpon().setAudioStreams(newAudioStreams).build();
}
private static class CopyTaskInfo {
private final ParcelFileDescriptor mSource;
private final ParcelFileDescriptor mSink;
private final int mCopyBufferLength;
CopyTaskInfo(ParcelFileDescriptor source, ParcelFileDescriptor sink, int copyBufferLength) {
mSource = source;
mSink = sink;
mCopyBufferLength = copyBufferLength;
}
}
private class HotwordDetectedResultCopyTask implements Runnable {
private final String mResultTaskId;
private final List<Pair<ParcelFileDescriptor, ParcelFileDescriptor>> mSourcesAndSinks;
private final List<CopyTaskInfo> mCopyTaskInfos;
private final ExecutorService mExecutorService = Executors.newCachedThreadPool();
HotwordDetectedResultCopyTask(String resultTaskId,
List<Pair<ParcelFileDescriptor, ParcelFileDescriptor>> sourcesAndSinks) {
HotwordDetectedResultCopyTask(String resultTaskId, List<CopyTaskInfo> copyTaskInfos) {
mResultTaskId = resultTaskId;
mSourcesAndSinks = sourcesAndSinks;
mCopyTaskInfos = copyTaskInfos;
}
@Override
public void run() {
Thread.currentThread().setName(THREAD_NAME_PREFIX + mResultTaskId);
int size = mSourcesAndSinks.size();
int size = mCopyTaskInfos.size();
List<SingleAudioStreamCopyTask> tasks = new ArrayList<>(size);
for (int i = 0; i < size; i++) {
Pair<ParcelFileDescriptor, ParcelFileDescriptor> sourceAndSink =
mSourcesAndSinks.get(i);
ParcelFileDescriptor serviceAudioSource = sourceAndSink.first;
ParcelFileDescriptor clientAudioSink = sourceAndSink.second;
CopyTaskInfo copyTaskInfo = mCopyTaskInfos.get(i);
String streamTaskId = mResultTaskId + "@" + i;
tasks.add(new SingleAudioStreamCopyTask(streamTaskId, serviceAudioSource,
clientAudioSink));
tasks.add(new SingleAudioStreamCopyTask(streamTaskId, copyTaskInfo.mSource,
copyTaskInfo.mSink, copyTaskInfo.mCopyBufferLength));
}
if (mAppOpsManager.startOpNoThrow(AppOpsManager.OPSTR_RECORD_AUDIO_HOTWORD,
mVoiceInteractorIdentity.uid, mVoiceInteractorIdentity.packageName,
mVoiceInteractorIdentity.attributionTag, OP_MESSAGE) == MODE_ALLOWED) {
mVoiceInteractorUid, mVoiceInteractorPackageName,
mVoiceInteractorAttributionTag, OP_MESSAGE) == MODE_ALLOWED) {
try {
// TODO(b/244599891): Set timeout, close after inactivity
mExecutorService.invokeAll(tasks);
@@ -135,25 +174,23 @@ final class HotwordAudioStreamManager {
bestEffortPropagateError(e.getMessage());
} finally {
mAppOpsManager.finishOp(AppOpsManager.OPSTR_RECORD_AUDIO_HOTWORD,
mVoiceInteractorIdentity.uid, mVoiceInteractorIdentity.packageName,
mVoiceInteractorIdentity.attributionTag);
mVoiceInteractorUid, mVoiceInteractorPackageName,
mVoiceInteractorAttributionTag);
}
} else {
bestEffortPropagateError(
"Failed to obtain RECORD_AUDIO_HOTWORD permission for "
+ SoundTriggerSessionPermissionsDecorator.toString(
mVoiceInteractorIdentity));
"Failed to obtain RECORD_AUDIO_HOTWORD permission for voice interactor with"
+ " uid=" + mVoiceInteractorUid
+ " packageName=" + mVoiceInteractorPackageName
+ " attributionTag=" + mVoiceInteractorAttributionTag);
}
}
private void bestEffortPropagateError(@NonNull String errorMessage) {
try {
for (Pair<ParcelFileDescriptor, ParcelFileDescriptor> sourceAndSink :
mSourcesAndSinks) {
ParcelFileDescriptor serviceAudioSource = sourceAndSink.first;
ParcelFileDescriptor clientAudioSink = sourceAndSink.second;
serviceAudioSource.closeWithError(errorMessage);
clientAudioSink.closeWithError(errorMessage);
for (CopyTaskInfo copyTaskInfo : mCopyTaskInfos) {
copyTaskInfo.mSource.closeWithError(errorMessage);
copyTaskInfo.mSink.closeWithError(errorMessage);
}
} catch (IOException e) {
Slog.e(TAG, mResultTaskId + ": Failed to propagate error", e);
@@ -162,18 +199,17 @@ final class HotwordAudioStreamManager {
}
private static class SingleAudioStreamCopyTask implements Callable<Void> {
// TODO: Make this buffer size customizable from updateState()
private static final int COPY_BUFFER_LENGTH = 2_560;
private final String mStreamTaskId;
private final ParcelFileDescriptor mAudioSource;
private final ParcelFileDescriptor mAudioSink;
private final int mCopyBufferLength;
SingleAudioStreamCopyTask(String streamTaskId, ParcelFileDescriptor audioSource,
ParcelFileDescriptor audioSink) {
ParcelFileDescriptor audioSink, int copyBufferLength) {
mStreamTaskId = streamTaskId;
mAudioSource = audioSource;
mAudioSink = audioSink;
mCopyBufferLength = copyBufferLength;
}
@Override
@@ -189,7 +225,7 @@ final class HotwordAudioStreamManager {
try {
fis = new ParcelFileDescriptor.AutoCloseInputStream(mAudioSource);
fos = new ParcelFileDescriptor.AutoCloseOutputStream(mAudioSink);
byte[] buffer = new byte[COPY_BUFFER_LENGTH];
byte[] buffer = new byte[mCopyBufferLength];
while (true) {
if (Thread.interrupted()) {
Slog.e(TAG,

13
services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordDetectionConnection.java
View File

@@ -183,7 +183,7 @@ final class HotwordDetectionConnection {
private final ScheduledExecutorService mScheduledExecutorService =
Executors.newSingleThreadScheduledExecutor();
private final AppOpsManager mAppOpsManager;
private final HotwordAudioStreamManager mHotwordAudioStreamManager;
private final HotwordAudioStreamCopier mHotwordAudioStreamCopier;
@Nullable private final ScheduledFuture<?> mCancellationTaskFuture;
private final AtomicBoolean mUpdateStateAfterStartFinished = new AtomicBoolean(false);
private final IBinder.DeathRecipient mAudioServerDeathRecipient = this::audioServerDied;
@@ -245,8 +245,9 @@ final class HotwordDetectionConnection {
mVoiceInteractionServiceUid = voiceInteractionServiceUid;
mVoiceInteractorIdentity = voiceInteractorIdentity;
mAppOpsManager = mContext.getSystemService(AppOpsManager.class);
mHotwordAudioStreamManager = new HotwordAudioStreamManager(mAppOpsManager,
mVoiceInteractorIdentity);
mHotwordAudioStreamCopier = new HotwordAudioStreamCopier(mAppOpsManager,
mVoiceInteractorIdentity.uid, mVoiceInteractorIdentity.packageName,
mVoiceInteractorIdentity.attributionTag);
mDetectionComponentName = serviceName;
mUser = userId;
mCallback = callback;
@@ -506,7 +507,7 @@ final class HotwordDetectionConnection {
saveProximityValueToBundle(result);
HotwordDetectedResult newResult;
try {
newResult = mHotwordAudioStreamManager.startCopyingAudioStreams(result);
newResult = mHotwordAudioStreamCopier.startCopyingAudioStreams(result);
} catch (IOException e) {
// TODO: Write event
mSoftwareCallback.onError();
@@ -641,7 +642,7 @@ final class HotwordDetectionConnection {
saveProximityValueToBundle(result);
HotwordDetectedResult newResult;
try {
newResult = mHotwordAudioStreamManager.startCopyingAudioStreams(result);
newResult = mHotwordAudioStreamCopier.startCopyingAudioStreams(result);
} catch (IOException e) {
// TODO: Write event
externalCallback.onError(CALLBACK_ONDETECTED_STREAM_COPY_ERROR);
@@ -1000,7 +1001,7 @@ final class HotwordDetectionConnection {
HotwordDetectedResult newResult;
try {
newResult =
mHotwordAudioStreamManager.startCopyingAudioStreams(
mHotwordAudioStreamCopier.startCopyingAudioStreams(
triggerResult);
} catch (IOException e) {
// TODO: Write event

13
services/voiceinteraction/java/com/android/server/voiceinteraction/TrustedHotwordDetectorSession.java
View File

@@ -183,7 +183,7 @@ final class TrustedHotwordDetectorSession {
private final ScheduledExecutorService mScheduledExecutorService =
Executors.newSingleThreadScheduledExecutor();
private final AppOpsManager mAppOpsManager;
private final HotwordAudioStreamManager mHotwordAudioStreamManager;
private final HotwordAudioStreamCopier mHotwordAudioStreamCopier;
@Nullable private final ScheduledFuture<?> mCancellationTaskFuture;
private final AtomicBoolean mUpdateStateAfterStartFinished = new AtomicBoolean(false);
private final IBinder.DeathRecipient mAudioServerDeathRecipient = this::audioServerDied;
@@ -245,8 +245,9 @@ final class TrustedHotwordDetectorSession {
mVoiceInteractionServiceUid = voiceInteractionServiceUid;
mVoiceInteractorIdentity = voiceInteractorIdentity;
mAppOpsManager = mContext.getSystemService(AppOpsManager.class);
mHotwordAudioStreamManager = new HotwordAudioStreamManager(mAppOpsManager,
mVoiceInteractorIdentity);
mHotwordAudioStreamCopier = new HotwordAudioStreamCopier(mAppOpsManager,
mVoiceInteractorIdentity.uid, mVoiceInteractorIdentity.packageName,
mVoiceInteractorIdentity.attributionTag);
mDetectionComponentName = serviceName;
mUser = userId;
mCallback = callback;
@@ -506,7 +507,7 @@ final class TrustedHotwordDetectorSession {
saveProximityValueToBundle(result);
HotwordDetectedResult newResult;
try {
newResult = mHotwordAudioStreamManager.startCopyingAudioStreams(result);
newResult = mHotwordAudioStreamCopier.startCopyingAudioStreams(result);
} catch (IOException e) {
// TODO: Write event
mSoftwareCallback.onError();
@@ -641,7 +642,7 @@ final class TrustedHotwordDetectorSession {
saveProximityValueToBundle(result);
HotwordDetectedResult newResult;
try {
newResult = mHotwordAudioStreamManager.startCopyingAudioStreams(result);
newResult = mHotwordAudioStreamCopier.startCopyingAudioStreams(result);
} catch (IOException e) {
// TODO: Write event
externalCallback.onError(CALLBACK_ONDETECTED_STREAM_COPY_ERROR);
@@ -1000,7 +1001,7 @@ final class TrustedHotwordDetectorSession {
HotwordDetectedResult newResult;
try {
newResult =
mHotwordAudioStreamManager.startCopyingAudioStreams(
mHotwordAudioStreamCopier.startCopyingAudioStreams(
triggerResult);
} catch (IOException e) {
// TODO: Write event