Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement VirtualAudioDevice API

Browse Source 
Add createVirtualAudioDevice() API in VirtualDevice to return a VirtualAudioDevice object for the caller to capture audio and inject microphone to applications running on the specified virtual display.

The VirtualAudioController in service side will responsible for listening virtual display running applications change, playback and recording config change, then notify the VirtualAudioSession to update AudioRecord/AudioTrack inside the AudioCapture/AudioInjection class internally.

Bug: 201558304
CTS-Coverage-Bug: 218528439
Test: atest FrameworksCoreTests:android.companion.virtual, atest FrameworksServicesTests:com.android.server.companion.virtual, manual testing with Exo by ag/16575248
Change-Id: I74f3da67b76aaa909943fbfc8418bf22d637ca39
This commit is contained in:
yuanjiahsu
2021-12-20 18:54:53 +08:00
committed by Yuanjia Hsu
parent b2134e69f7
commit 69cf28591c
16 changed files with 1753 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
core/api/system-current.txt
View File

@@ -2750,6 +2750,7 @@ package android.companion.virtual {
method public void addActivityListener(@NonNull android.companion.virtual.VirtualDeviceManager.ActivityListener);
method public void addActivityListener(@NonNull android.companion.virtual.VirtualDeviceManager.ActivityListener, @NonNull java.util.concurrent.Executor);
method @RequiresPermission(android.Manifest.permission.CREATE_VIRTUAL_DEVICE) public void close();
method @NonNull @RequiresPermission(android.Manifest.permission.CREATE_VIRTUAL_DEVICE) public android.companion.virtual.audio.VirtualAudioDevice createVirtualAudioDevice(@NonNull android.hardware.display.VirtualDisplay, @Nullable java.util.concurrent.Executor, @Nullable android.companion.virtual.audio.VirtualAudioDevice.AudioConfigurationChangeCallback);
method @Nullable public android.hardware.display.VirtualDisplay createVirtualDisplay(int, int, int, @Nullable android.view.Surface, int, @Nullable android.os.Handler, @Nullable android.hardware.display.VirtualDisplay.Callback);
method @NonNull @RequiresPermission(android.Manifest.permission.CREATE_VIRTUAL_DEVICE) public android.hardware.input.VirtualKeyboard createVirtualKeyboard(@NonNull android.hardware.display.VirtualDisplay, @NonNull String, int, int);
method @NonNull @RequiresPermission(android.Manifest.permission.CREATE_VIRTUAL_DEVICE) public android.hardware.input.VirtualMouse createVirtualMouse(@NonNull android.hardware.display.VirtualDisplay, @NonNull String, int, int);
@@ -2782,6 +2783,39 @@ package android.companion.virtual {
}
package android.companion.virtual.audio {
public final class AudioCapture {
ctor public AudioCapture();
method public int getRecordingState();
method public int read(@NonNull java.nio.ByteBuffer, int);
method public void startRecording();
method public void stop();
}
public final class AudioInjection {
ctor public AudioInjection();
method public int getPlayState();
method public void play();
method public void stop();
method public int write(@NonNull java.nio.ByteBuffer, int, int);
}
public final class VirtualAudioDevice implements java.io.Closeable {
method @RequiresPermission(android.Manifest.permission.MODIFY_AUDIO_ROUTING) public void close();
method @Nullable public android.companion.virtual.audio.AudioCapture getAudioCapture();
method @Nullable public android.companion.virtual.audio.AudioInjection getAudioInjection();
method @NonNull @RequiresPermission(android.Manifest.permission.MODIFY_AUDIO_ROUTING) public android.companion.virtual.audio.AudioCapture startAudioCapture(@NonNull android.media.AudioFormat);
method @NonNull @RequiresPermission(android.Manifest.permission.MODIFY_AUDIO_ROUTING) public android.companion.virtual.audio.AudioInjection startAudioInjection(@NonNull android.media.AudioFormat);
}
public static interface VirtualAudioDevice.AudioConfigurationChangeCallback {
method public void onPlaybackConfigChanged(@NonNull java.util.List<android.media.AudioPlaybackConfiguration>);
method public void onRecordingConfigChanged(@NonNull java.util.List<android.media.AudioRecordingConfiguration>);
}
}
package android.content {
public class ApexEnvironment {

10
core/java/android/companion/virtual/IVirtualDevice.aidl
View File

@@ -17,6 +17,7 @@
package android.companion.virtual;
import android.app.PendingIntent;
import android.companion.virtual.audio.IAudioSessionCallback;
import android.graphics.Point;
import android.graphics.PointF;
import android.hardware.input.VirtualKeyEvent;
@@ -45,6 +46,15 @@ interface IVirtualDevice {
*/
void close();
/**
* Notifies of an audio session being started.
*/
void onAudioSessionStarting(
int displayId,
IAudioSessionCallback callback);
void onAudioSessionEnded();
void createVirtualKeyboard(
int displayId,
String inputDeviceName,

26
core/java/android/companion/virtual/VirtualDeviceManager.java
View File

@@ -25,6 +25,8 @@ import android.annotation.SystemService;
import android.app.Activity;
import android.app.PendingIntent;
import android.companion.AssociationInfo;
import android.companion.virtual.audio.VirtualAudioDevice;
import android.companion.virtual.audio.VirtualAudioDevice.AudioConfigurationChangeCallback;
import android.content.ComponentName;
import android.content.Context;
import android.graphics.Point;
@@ -338,6 +340,30 @@ public final class VirtualDeviceManager {
}
}
/**
* Creates a VirtualAudioDevice, capable of recording audio emanating from this device,
* or injecting audio from another device.
*
* <p>Note: This object does not support capturing privileged playback, such as voice call
* audio.
*
* @param display The target virtual display to capture from and inject into.
* @param executor The {@link Executor} object for the thread on which to execute
* the callback. If <code>null</code>, the {@link Executor} associated with
* the main {@link Looper} will be used.
* @param callback Interface to be notified when playback or recording configuration of
* applications running on virtual display is changed.
* @return A {@link VirtualAudioDevice} instance.
*/
@RequiresPermission(android.Manifest.permission.CREATE_VIRTUAL_DEVICE)
@NonNull
public VirtualAudioDevice createVirtualAudioDevice(
@NonNull VirtualDisplay display,
@Nullable Executor executor,
@Nullable AudioConfigurationChangeCallback callback) {
return new VirtualAudioDevice(mContext, mVirtualDevice, display, executor, callback);
}
/**
* Sets the visibility of the pointer icon for this VirtualDevice's associated displays.
*

124
core/java/android/companion/virtual/audio/AudioCapture.java Normal file
View File

@@ -0,0 +1,124 @@
/*
* Copyright (C) 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package android.companion.virtual.audio;
import static android.media.AudioRecord.RECORDSTATE_RECORDING;
import static android.media.AudioRecord.RECORDSTATE_STOPPED;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.annotation.SuppressLint;
import android.annotation.SystemApi;
import android.media.AudioRecord;
import android.util.Log;
import com.android.internal.annotations.GuardedBy;
import java.nio.ByteBuffer;
/**
* Wrapper around {@link AudioRecord} that allows for the underlying {@link AudioRecord} to
* be swapped out while recording is ongoing.
*
* @hide
*/
// The stop() actually doesn't release resources, so should not force implementing Closeable.
@SuppressLint("NotCloseable")
@SystemApi
public final class AudioCapture {
private static final String TAG = "AudioCapture";
private final Object mLock = new Object();
@GuardedBy("mLock")
@Nullable
private AudioRecord mAudioRecord;
@GuardedBy("mLock")
private int mRecordingState = RECORDSTATE_STOPPED;
/**
* Sets the {@link AudioRecord} to handle audio capturing.
* Callers may call this multiple times with different audio records to change
* the underlying {@link AudioRecord} without stopping and re-starting recording.
*
* @param audioRecord The underlying {@link AudioRecord} to use for capture,
* or null if no audio (i.e. silence) should be captured while still keeping the
* record in a recording state.
*/
void setAudioRecord(@Nullable AudioRecord audioRecord) {
Log.d(TAG, "set AudioRecord with " + audioRecord);
synchronized (mLock) {
// Release old reference.
if (mAudioRecord != null) {
mAudioRecord.release();
}
// Sync recording state for new reference.
if (audioRecord != null) {
if (mRecordingState == RECORDSTATE_RECORDING
&& audioRecord.getRecordingState() != RECORDSTATE_RECORDING) {
audioRecord.startRecording();
}
if (mRecordingState == RECORDSTATE_STOPPED
&& audioRecord.getRecordingState() != RECORDSTATE_STOPPED) {
audioRecord.stop();
}
}
mAudioRecord = audioRecord;
}
}
/** See {@link AudioRecord#read(ByteBuffer, int)}. */
public int read(@NonNull ByteBuffer audioBuffer, int sizeInBytes) {
final int sizeRead;
synchronized (mLock) {
if (mAudioRecord != null) {
sizeRead = mAudioRecord.read(audioBuffer, sizeInBytes);
} else {
sizeRead = 0;
}
}
return sizeRead;
}
/** See {@link AudioRecord#startRecording()}. */
public void startRecording() {
synchronized (mLock) {
mRecordingState = RECORDSTATE_RECORDING;
if (mAudioRecord != null && mAudioRecord.getRecordingState() != RECORDSTATE_RECORDING) {
mAudioRecord.startRecording();
}
}
}
/** See {@link AudioRecord#stop()}. */
public void stop() {
synchronized (mLock) {
mRecordingState = RECORDSTATE_STOPPED;
if (mAudioRecord != null && mAudioRecord.getRecordingState() != RECORDSTATE_STOPPED) {
mAudioRecord.stop();
}
}
}
/** See {@link AudioRecord#getRecordingState()}. */
public int getRecordingState() {
synchronized (mLock) {
return mRecordingState;
}
}
}

124
core/java/android/companion/virtual/audio/AudioInjection.java Normal file
View File

@@ -0,0 +1,124 @@
/*
* Copyright (C) 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package android.companion.virtual.audio;
import static android.media.AudioTrack.PLAYSTATE_PLAYING;
import static android.media.AudioTrack.PLAYSTATE_STOPPED;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.annotation.SuppressLint;
import android.annotation.SystemApi;
import android.media.AudioTrack;
import android.util.Log;
import com.android.internal.annotations.GuardedBy;
import java.nio.ByteBuffer;
/**
* Wrapper around {@link AudioTrack} that allows for the underlying {@link AudioTrack} to
* be swapped out while playout is ongoing.
*
* @hide
*/
// The stop() actually doesn't release resources, so should not force implementing Closeable.
@SuppressLint("NotCloseable")
@SystemApi
public final class AudioInjection {
private static final String TAG = "AudioInjection";
private final Object mLock = new Object();
@GuardedBy("mLock")
@Nullable
private AudioTrack mAudioTrack;
@GuardedBy("mLock")
private int mPlayState = PLAYSTATE_STOPPED;
/**
* Sets the {@link AudioTrack} to handle audio injection.
* Callers may call this multiple times with different audio tracks to change
* the underlying {@link AudioTrack} without stopping and re-starting injection.
*
* @param audioTrack The underlying {@link AudioTrack} to use for injection,
* or null if no audio (i.e. silence) should be injected while still keeping the
* record in a playing state.
*/
void setAudioTrack(@Nullable AudioTrack audioTrack) {
Log.d(TAG, "set AudioTrack with " + audioTrack);
synchronized (mLock) {
// Release old reference.
if (mAudioTrack != null) {
mAudioTrack.release();
}
// Sync play state for new reference.
if (audioTrack != null) {
if (mPlayState == PLAYSTATE_PLAYING
&& audioTrack.getPlayState() != PLAYSTATE_PLAYING) {
audioTrack.play();
}
if (mPlayState == PLAYSTATE_STOPPED
&& audioTrack.getPlayState() != PLAYSTATE_STOPPED) {
audioTrack.stop();
}
}
mAudioTrack = audioTrack;
}
}
/** See {@link AudioTrack#write(ByteBuffer, int, int)}. */
public int write(@NonNull ByteBuffer audioBuffer, int sizeInBytes, int writeMode) {
final int sizeWrite;
synchronized (mLock) {
if (mAudioTrack != null) {
sizeWrite = mAudioTrack.write(audioBuffer, sizeInBytes, writeMode);
} else {
sizeWrite = 0;
}
}
return sizeWrite;
}
/** See {@link AudioTrack#play()}. */
public void play() {
synchronized (mLock) {
mPlayState = PLAYSTATE_PLAYING;
if (mAudioTrack != null && mAudioTrack.getPlayState() != PLAYSTATE_PLAYING) {
mAudioTrack.play();
}
}
}
/** See {@link AudioTrack#stop()}. */
public void stop() {
synchronized (mLock) {
mPlayState = PLAYSTATE_STOPPED;
if (mAudioTrack != null && mAudioTrack.getPlayState() != PLAYSTATE_STOPPED) {
mAudioTrack.stop();
}
}
}
/** See {@link AudioTrack#getPlayState()}. */
public int getPlayState() {
synchronized (mLock) {
return mPlayState;
}
}
}

44
core/java/android/companion/virtual/audio/IAudioSessionCallback.aidl Normal file
View File

@@ -0,0 +1,44 @@
/*
* Copyright (C) 2021 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package android.companion.virtual.audio;
import android.media.AudioPlaybackConfiguration;
import android.media.AudioRecordingConfiguration;
/**
* Callback to control audio rerouting, notify playback and recording state of applications running
* on virtual device.
*
* @hide
*/
oneway interface IAudioSessionCallback {
/** Updates the set of applications that need to have their audio rerouted. */
void onAppsNeedingAudioRoutingChanged(in int[] appUids);
/**
* Called whenever the playback configuration of applications running on virtual device has
* changed.
*/
void onPlaybackConfigChanged(in List<AudioPlaybackConfiguration> configs);
/**
* Called whenever the recording configuration of applications running on virtual device has
* changed.
*/
void onRecordingConfigChanged(in List<AudioRecordingConfiguration> configs);
}

167
core/java/android/companion/virtual/audio/VirtualAudioDevice.java Normal file
View File

@@ -0,0 +1,167 @@
/*
* Copyright (C) 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package android.companion.virtual.audio;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.annotation.RequiresPermission;
import android.annotation.SystemApi;
import android.companion.virtual.IVirtualDevice;
import android.content.Context;
import android.hardware.display.VirtualDisplay;
import android.media.AudioFormat;
import android.media.AudioPlaybackConfiguration;
import android.media.AudioRecordingConfiguration;
import android.os.RemoteException;
import java.io.Closeable;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.Executor;
/**
* The class stores an {@link AudioCapture} for audio capturing and an {@link AudioInjection} for
* audio injection.
*
* @hide
*/
@SystemApi
public final class VirtualAudioDevice implements Closeable {
/**
* Interface to be notified when playback or recording configuration of applications running on
* virtual display was changed.
*
* @hide
*/
@SystemApi
public interface AudioConfigurationChangeCallback {
/**
* Notifies when playback configuration of applications running on virtual display was
* changed.
*/
void onPlaybackConfigChanged(@NonNull List<AudioPlaybackConfiguration> configs);
/**
* Notifies when recording configuration of applications running on virtual display was
* changed.
*/
void onRecordingConfigChanged(@NonNull List<AudioRecordingConfiguration> configs);
}
private final Context mContext;
private final IVirtualDevice mVirtualDevice;
private final VirtualDisplay mVirtualDisplay;
private final AudioConfigurationChangeCallback mCallback;
private final Executor mExecutor;
@Nullable
private VirtualAudioSession mOngoingSession;
/**
* @hide
*/
public VirtualAudioDevice(Context context, IVirtualDevice virtualDevice,
VirtualDisplay virtualDisplay, Executor executor,
AudioConfigurationChangeCallback callback) {
mContext = context;
mVirtualDevice = virtualDevice;
mVirtualDisplay = virtualDisplay;
mExecutor = executor;
mCallback = callback;
}
/**
* Begins injecting audio from a remote device into this device.
*
* @return An {@link AudioInjection} containing the injected audio.
*/
@RequiresPermission(android.Manifest.permission.MODIFY_AUDIO_ROUTING)
@NonNull
public AudioInjection startAudioInjection(@NonNull AudioFormat injectionFormat) {
Objects.requireNonNull(injectionFormat, "injectionFormat must not be null");
if (mOngoingSession != null && mOngoingSession.getAudioInjection() != null) {
throw new IllegalStateException("Cannot start an audio injection while a session is "
+ "ongoing. Call close() on this device first to end the previous injection.");
}
if (mOngoingSession == null) {
mOngoingSession = new VirtualAudioSession(mContext, mCallback, mExecutor);
}
try {
mVirtualDevice.onAudioSessionStarting(mVirtualDisplay.getDisplay().getDisplayId(),
/* callback= */ mOngoingSession);
} catch (RemoteException e) {
throw e.rethrowFromSystemServer();
}
return mOngoingSession.startAudioInjection(injectionFormat);
}
/**
* Begins recording audio emanating from this device.
*
* @return An {@link AudioCapture} containing the recorded audio.
*/
@RequiresPermission(android.Manifest.permission.MODIFY_AUDIO_ROUTING)
@NonNull
public AudioCapture startAudioCapture(@NonNull AudioFormat captureFormat) {
Objects.requireNonNull(captureFormat, "captureFormat must not be null");
if (mOngoingSession != null && mOngoingSession.getAudioCapture() != null) {
throw new IllegalStateException("Cannot start an audio capture while a session is "
+ "ongoing. Call close() on this device first to end the previous session.");
}
if (mOngoingSession == null) {
mOngoingSession = new VirtualAudioSession(mContext, mCallback, mExecutor);
}
try {
mVirtualDevice.onAudioSessionStarting(mVirtualDisplay.getDisplay().getDisplayId(),
/* callback= */ mOngoingSession);
} catch (RemoteException e) {
throw e.rethrowFromSystemServer();
}
return mOngoingSession.startAudioCapture(captureFormat);
}
/** Returns the {@link AudioCapture} instance. */
@Nullable
public AudioCapture getAudioCapture() {
return mOngoingSession != null ? mOngoingSession.getAudioCapture() : null;
}
/** Returns the {@link AudioInjection} instance. */
@Nullable
public AudioInjection getAudioInjection() {
return mOngoingSession != null ? mOngoingSession.getAudioInjection() : null;
}
@RequiresPermission(android.Manifest.permission.MODIFY_AUDIO_ROUTING)
@Override
public void close() {
if (mOngoingSession != null) {
mOngoingSession.close();
mOngoingSession = null;
try {
mVirtualDevice.onAudioSessionEnded();
} catch (RemoteException e) {
throw e.rethrowFromSystemServer();
}
}
}
}

305
core/java/android/companion/virtual/audio/VirtualAudioSession.java Normal file
View File

@@ -0,0 +1,305 @@
/*
* Copyright (C) 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package android.companion.virtual.audio;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.annotation.RequiresPermission;
import android.companion.virtual.audio.VirtualAudioDevice.AudioConfigurationChangeCallback;
import android.content.Context;
import android.media.AudioFormat;
import android.media.AudioManager;
import android.media.AudioPlaybackConfiguration;
import android.media.AudioRecord;
import android.media.AudioRecordingConfiguration;
import android.media.AudioTrack;
import android.media.audiopolicy.AudioMix;
import android.media.audiopolicy.AudioMixingRule;
import android.media.audiopolicy.AudioPolicy;
import android.util.IntArray;
import android.util.Log;
import com.android.internal.annotations.GuardedBy;
import com.android.internal.annotations.VisibleForTesting;
import java.io.Closeable;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.Executor;
/**
* Manages an ongiong audio session in which audio can be captured (recorded) and/or
* injected from a remote device.
*
* @hide
*/
@VisibleForTesting
public final class VirtualAudioSession extends IAudioSessionCallback.Stub implements
Closeable {
private static final String TAG = "VirtualAudioSession";
private final Context mContext;
/** The {@link Executor} for sending {@link AudioConfigurationChangeCallback} to the caller */
private final Executor mExecutor;
@Nullable
private final AudioConfigurationChangeCallback mCallback;
private final Object mLock = new Object();
@GuardedBy("mLock")
private final IntArray mReroutedAppUids = new IntArray();
@Nullable
@GuardedBy("mLock")
private AudioPolicy mAudioPolicy;
@Nullable
@GuardedBy("mLock")
private AudioFormat mCaptureFormat;
@Nullable
@GuardedBy("mLock")
private AudioFormat mInjectionFormat;
@Nullable
@GuardedBy("mLock")
private AudioCapture mAudioCapture;
@Nullable
@GuardedBy("mLock")
private AudioInjection mAudioInjection;
@VisibleForTesting
public VirtualAudioSession(Context context,
@Nullable AudioConfigurationChangeCallback callback, @Nullable Executor executor) {
mContext = context;
mCallback = callback;
mExecutor = executor != null ? executor : context.getMainExecutor();
}
/**
* Begins recording audio emanating from this device.
*
* @return An {@link AudioCapture} containing the recorded audio.
*/
@VisibleForTesting
@NonNull
public AudioCapture startAudioCapture(@NonNull AudioFormat captureFormat) {
Objects.requireNonNull(captureFormat, "captureFormat must not be null");
synchronized (mLock) {
if (mAudioCapture != null) {
throw new IllegalStateException(
"Cannot start capture while another capture is ongoing.");
}
mCaptureFormat = captureFormat;
mAudioCapture = new AudioCapture();
mAudioCapture.startRecording();
return mAudioCapture;
}
}
/**
* Begins injecting audio from a remote device into this device.
*
* @return An {@link AudioInjection} containing the injected audio.
*/
@VisibleForTesting
@NonNull
public AudioInjection startAudioInjection(@NonNull AudioFormat injectionFormat) {
Objects.requireNonNull(injectionFormat, "injectionFormat must not be null");
synchronized (mLock) {
if (mAudioInjection != null) {
throw new IllegalStateException(
"Cannot start injection while injection is already ongoing.");
}
mInjectionFormat = injectionFormat;
mAudioInjection = new AudioInjection();
mAudioInjection.play();
return mAudioInjection;
}
}
/** @hide */
@VisibleForTesting
@Nullable
public AudioCapture getAudioCapture() {
synchronized (mLock) {
return mAudioCapture;
}
}
/** @hide */
@VisibleForTesting
@Nullable
public AudioInjection getAudioInjection() {
synchronized (mLock) {
return mAudioInjection;
}
}
@RequiresPermission(android.Manifest.permission.MODIFY_AUDIO_ROUTING)
@Override
public void onAppsNeedingAudioRoutingChanged(int[] appUids) {
synchronized (mLock) {
if (Arrays.equals(mReroutedAppUids.toArray(), appUids)) {
return;
}
}
releaseAudioStreams();
if (appUids.length == 0) {
return;
}
createAudioStreams(appUids);
}
@RequiresPermission(android.Manifest.permission.MODIFY_AUDIO_ROUTING)
@Override
public void close() {
releaseAudioStreams();
synchronized (mLock) {
mAudioCapture = null;
mAudioInjection = null;
mCaptureFormat = null;
mInjectionFormat = null;
}
}
@RequiresPermission(android.Manifest.permission.MODIFY_AUDIO_ROUTING)
private void createAudioStreams(int[] appUids) {
synchronized (mLock) {
if (mCaptureFormat == null && mInjectionFormat == null) {
throw new IllegalStateException(
"At least one of captureFormat and injectionFormat must be specified.");
}
if (mAudioPolicy != null) {
throw new IllegalStateException(
"Cannot create audio streams while the audio policy is registered. Call "
+ "releaseAudioStreams() first to unregister the previous audio "
+ "policy."
);
}
mReroutedAppUids.clear();
for (int appUid : appUids) {
mReroutedAppUids.add(appUid);
}
AudioMix audioRecordMix = null;
AudioMix audioTrackMix = null;
AudioPolicy.Builder builder = new AudioPolicy.Builder(mContext);
if (mCaptureFormat != null) {
audioRecordMix = createAudioRecordMix(mCaptureFormat, appUids);
builder.addMix(audioRecordMix);
}
if (mInjectionFormat != null) {
audioTrackMix = createAudioTrackMix(mInjectionFormat, appUids);
builder.addMix(audioTrackMix);
}
mAudioPolicy = builder.build();
AudioManager audioManager = mContext.getSystemService(AudioManager.class);
if (audioManager.registerAudioPolicy(mAudioPolicy) == AudioManager.ERROR) {
Log.e(TAG, "Failed to register audio policy!");
}
AudioRecord audioRecord =
audioRecordMix != null ? mAudioPolicy.createAudioRecordSink(audioRecordMix)
: null;
AudioTrack audioTrack =
audioTrackMix != null ? mAudioPolicy.createAudioTrackSource(audioTrackMix)
: null;
if (mAudioCapture != null) {
mAudioCapture.setAudioRecord(audioRecord);
}
if (mAudioInjection != null) {
mAudioInjection.setAudioTrack(audioTrack);
}
}
}
@RequiresPermission(android.Manifest.permission.MODIFY_AUDIO_ROUTING)
private void releaseAudioStreams() {
synchronized (mLock) {
if (mAudioCapture != null) {
mAudioCapture.setAudioRecord(null);
}
if (mAudioInjection != null) {
mAudioInjection.setAudioTrack(null);
}
mReroutedAppUids.clear();
if (mAudioPolicy != null) {
AudioManager audioManager = mContext.getSystemService(AudioManager.class);
audioManager.unregisterAudioPolicy(mAudioPolicy);
mAudioPolicy = null;
Log.i(TAG, "AudioPolicy unregistered");
}
}
}
@Override
public void onPlaybackConfigChanged(List<AudioPlaybackConfiguration> configs) {
if (mCallback != null) {
mExecutor.execute(() -> mCallback.onPlaybackConfigChanged(configs));
}
}
@Override
public void onRecordingConfigChanged(List<AudioRecordingConfiguration> configs) {
if (mCallback != null) {
mExecutor.execute(() -> mCallback.onRecordingConfigChanged(configs));
}
}
/** @hide */
@VisibleForTesting
public IntArray getReroutedAppUids() {
synchronized (mLock) {
return mReroutedAppUids;
}
}
private static AudioMix createAudioRecordMix(@NonNull AudioFormat audioFormat, int[] appUids) {
AudioMixingRule.Builder builder = new AudioMixingRule.Builder();
builder.setTargetMixRole(AudioMixingRule.MIX_ROLE_PLAYERS);
for (int uid : appUids) {
builder.addMixRule(AudioMixingRule.RULE_MATCH_UID, uid);
}
AudioMixingRule audioMixingRule = builder.allowPrivilegedPlaybackCapture(false).build();
AudioMix audioMix =
new AudioMix.Builder(audioMixingRule)
.setFormat(audioFormat)
.setRouteFlags(AudioMix.ROUTE_FLAG_LOOP_BACK)
.build();
return audioMix;
}
private static AudioMix createAudioTrackMix(@NonNull AudioFormat audioFormat, int[] appUids) {
AudioMixingRule.Builder builder = new AudioMixingRule.Builder();
builder.setTargetMixRole(AudioMixingRule.MIX_ROLE_INJECTOR);
for (int uid : appUids) {
builder.addMixRule(AudioMixingRule.RULE_MATCH_UID, uid);
}
AudioMixingRule audioMixingRule = builder.build();
AudioMix audioMix =
new AudioMix.Builder(audioMixingRule)
.setFormat(audioFormat)
.setRouteFlags(AudioMix.ROUTE_FLAG_LOOP_BACK)
.build();
return audioMix;
}
}

182
core/tests/coretests/src/android/companion/virtual/audio/VirtualAudioSessionTest.java Normal file
View File

@@ -0,0 +1,182 @@
/*
* Copyright (C) 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package android.companion.virtual.audio;
import static android.media.AudioFormat.CHANNEL_IN_MONO;
import static android.media.AudioFormat.CHANNEL_OUT_MONO;
import static android.media.AudioFormat.ENCODING_PCM_16BIT;
import static com.google.common.truth.Truth.assertThat;
import static org.mockito.Mockito.verify;
import static org.testng.Assert.assertThrows;
import android.companion.virtual.audio.VirtualAudioDevice.AudioConfigurationChangeCallback;
import android.content.Context;
import android.content.ContextWrapper;
import android.media.AudioFormat;
import android.media.AudioPlaybackConfiguration;
import android.media.AudioRecordingConfiguration;
import android.platform.test.annotations.Presubmit;
import androidx.test.InstrumentationRegistry;
import androidx.test.runner.AndroidJUnit4;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.MockitoAnnotations;
import org.mockito.junit.MockitoJUnit;
import org.mockito.junit.MockitoRule;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@Presubmit
@RunWith(AndroidJUnit4.class)
public class VirtualAudioSessionTest {
@Rule
public final MockitoRule mockito = MockitoJUnit.rule();
@Mock
private AudioConfigurationChangeCallback mCallback;
private static final int APP_UID = 100;
private static final int APP_UID2 = 200;
private static final AudioFormat AUDIO_CAPTURE_FORMAT =
new AudioFormat.Builder()
.setSampleRate(48000)
.setEncoding(ENCODING_PCM_16BIT)
.setChannelMask(CHANNEL_IN_MONO)
.build();
private static final AudioFormat AUDIO_INJECT_FORMAT =
new AudioFormat.Builder()
.setSampleRate(48000)
.setEncoding(ENCODING_PCM_16BIT)
.setChannelMask(CHANNEL_OUT_MONO)
.build();
private Context mContext;
private VirtualAudioSession mVirtualAudioSession;
@Before
public void setUp() {
MockitoAnnotations.initMocks(this);
mContext = Mockito.spy(new ContextWrapper(InstrumentationRegistry.getTargetContext()));
mVirtualAudioSession = new VirtualAudioSession(
mContext, mCallback, /* executor= */ null);
}
@Test
public void startAudioCapture_isSuccessful() {
AudioCapture audioCapture = mVirtualAudioSession.startAudioCapture(AUDIO_CAPTURE_FORMAT);
assertThat(audioCapture).isNotNull();
assertThat(mVirtualAudioSession.getAudioCapture()).isEqualTo(audioCapture);
}
@Test
public void startAudioCapture_audioCaptureAlreadyStarted_throws() {
mVirtualAudioSession.startAudioCapture(AUDIO_CAPTURE_FORMAT);
assertThrows(IllegalStateException.class,
() -> mVirtualAudioSession.startAudioCapture(AUDIO_CAPTURE_FORMAT));
}
@Test
public void startAudioInjection_isSuccessful() {
AudioInjection audioInjection = mVirtualAudioSession.startAudioInjection(
AUDIO_INJECT_FORMAT);
assertThat(audioInjection).isNotNull();
assertThat(mVirtualAudioSession.getAudioInjection()).isEqualTo(audioInjection);
}
@Test
public void startAudioInjection_audioInjectionAlreadyStarted_throws() {
mVirtualAudioSession.startAudioInjection(AUDIO_INJECT_FORMAT);
assertThrows(IllegalStateException.class,
() -> mVirtualAudioSession.startAudioInjection(AUDIO_INJECT_FORMAT));
}
@Test
public void onAppsNeedingAudioRoutingChanged_neverStartAudioCaptureOrInjection_throws() {
int[] uids = new int[]{APP_UID};
assertThrows(IllegalStateException.class,
() -> mVirtualAudioSession.onAppsNeedingAudioRoutingChanged(uids));
}
@Test
public void onAppsNeedingAudioRoutingChanged_cachesReroutedApps() {
mVirtualAudioSession.startAudioCapture(AUDIO_CAPTURE_FORMAT);
mVirtualAudioSession.startAudioInjection(AUDIO_INJECT_FORMAT);
int[] appUids = new int[]{APP_UID};
mVirtualAudioSession.onAppsNeedingAudioRoutingChanged(appUids);
assertThat(Arrays.equals(mVirtualAudioSession.getReroutedAppUids().toArray(),
appUids)).isTrue();
}
@Test
public void onAppsNeedingAudioRoutingChanged_receiveManyTimes_reroutedAppsSizeIsCorrect() {
mVirtualAudioSession.startAudioCapture(AUDIO_CAPTURE_FORMAT);
mVirtualAudioSession.startAudioInjection(AUDIO_INJECT_FORMAT);
int[] appUids = new int[]{APP_UID, APP_UID2};
mVirtualAudioSession.onAppsNeedingAudioRoutingChanged(new int[]{1234});
mVirtualAudioSession.onAppsNeedingAudioRoutingChanged(new int[]{5678});
mVirtualAudioSession.onAppsNeedingAudioRoutingChanged(appUids);
assertThat(Arrays.equals(mVirtualAudioSession.getReroutedAppUids().toArray(),
appUids)).isTrue();
assertThat(mVirtualAudioSession.getReroutedAppUids().size()).isEqualTo(2);
}
@Test
public void close_releasesCaptureAndInjection() {
mVirtualAudioSession.startAudioCapture(AUDIO_CAPTURE_FORMAT);
mVirtualAudioSession.startAudioInjection(AUDIO_INJECT_FORMAT);
mVirtualAudioSession.close();
assertThat(mVirtualAudioSession.getAudioCapture()).isNull();
assertThat(mVirtualAudioSession.getAudioInjection()).isNull();
}
@Test
public void onPlaybackConfigChanged_sendsCallback() {
List<AudioPlaybackConfiguration> configs = new ArrayList<>();
mVirtualAudioSession.onPlaybackConfigChanged(configs);
verify(mCallback).onPlaybackConfigChanged(configs);
}
@Test
public void onRecordingConfigChanged_sendCallback() {
List<AudioRecordingConfiguration> configs = new ArrayList<>();
mVirtualAudioSession.onRecordingConfigChanged(configs);
verify(mCallback).onRecordingConfigChanged(configs);
}
}

25
services/companion/java/com/android/server/companion/virtual/GenericWindowPolicyController.java
View File

@@ -47,9 +47,17 @@ import java.util.function.Consumer;
/**
* A controller to control the policies of the windows that can be displayed on the virtual display.
*/
class GenericWindowPolicyController extends DisplayWindowPolicyController {
public class GenericWindowPolicyController extends DisplayWindowPolicyController {
private static final String TAG = "VirtualDeviceManager";
private static final String TAG = "GenericWindowPolicyController";
/** Interface to listen running applications change on virtual display. */
public interface RunningAppsChangedListener {
/**
* Notifies the running applications change.
*/
void onRunningAppsChanged(ArraySet<Integer> runningUids);
}
private static final ComponentName BLOCKED_APP_STREAMING_COMPONENT =
new ComponentName("android", BlockedAppStreamingActivity.class.getName());
@@ -74,6 +82,9 @@ class GenericWindowPolicyController extends DisplayWindowPolicyController {
@Nullable private final ActivityListener mActivityListener;
private final Handler mHandler = new Handler(Looper.getMainLooper());
@Nullable
private RunningAppsChangedListener mRunningAppsChangedListener;
/**
* Creates a window policy controller that is generic to the different use cases of virtual
* device.
@@ -84,7 +95,7 @@ class GenericWindowPolicyController extends DisplayWindowPolicyController {
* @param activityListener Activity listener to listen for activity changes. The display ID
* is not populated in this callback and is always {@link Display#INVALID_DISPLAY}.
*/
GenericWindowPolicyController(int windowFlags, int systemWindowFlags,
public GenericWindowPolicyController(int windowFlags, int systemWindowFlags,
@NonNull ArraySet<UserHandle> allowedUsers,
@Nullable Set<ComponentName> allowedActivities,
@Nullable Set<ComponentName> blockedActivities,
@@ -98,6 +109,11 @@ class GenericWindowPolicyController extends DisplayWindowPolicyController {
mActivityListener = activityListener;
}
/** Sets listener for running applications change. */
public void setRunningAppsChangedListener(@Nullable RunningAppsChangedListener listener) {
mRunningAppsChangedListener = listener;
}
@Override
public boolean canContainActivities(@NonNull List<ActivityInfo> activities) {
// Can't display all the activities if any of them don't want to be displayed.
@@ -139,6 +155,9 @@ class GenericWindowPolicyController extends DisplayWindowPolicyController {
// Post callback on the main thread so it doesn't block activity launching
mHandler.post(() -> mActivityListener.onDisplayEmpty(Display.INVALID_DISPLAY));
}
if (mRunningAppsChangedListener != null) {
mRunningAppsChangedListener.onRunningAppsChanged(runningUids);
}
}
/**

44
services/companion/java/com/android/server/companion/virtual/VirtualDeviceImpl.java
View File

@@ -23,6 +23,7 @@ import static android.view.WindowManager.LayoutParams.FLAG_SECURE;
import static android.view.WindowManager.LayoutParams.SYSTEM_FLAG_HIDE_NON_SYSTEM_OVERLAY_WINDOWS;
import android.annotation.NonNull;
import android.annotation.RequiresPermission;
import android.app.Activity;
import android.app.ActivityOptions;
import android.app.PendingIntent;
@@ -32,6 +33,7 @@ import android.companion.virtual.IVirtualDevice;
import android.companion.virtual.IVirtualDeviceActivityListener;
import android.companion.virtual.VirtualDeviceManager.ActivityListener;
import android.companion.virtual.VirtualDeviceParams;
import android.companion.virtual.audio.IAudioSessionCallback;
import android.content.ComponentName;
import android.content.Context;
import android.content.Intent;
@@ -60,6 +62,7 @@ import android.window.DisplayWindowPolicyController;
import com.android.internal.annotations.GuardedBy;
import com.android.internal.annotations.VisibleForTesting;
import com.android.internal.app.BlockedAppStreamingActivity;
import com.android.server.companion.virtual.audio.VirtualAudioController;
import java.io.FileDescriptor;
import java.io.PrintWriter;
@@ -78,6 +81,7 @@ final class VirtualDeviceImpl extends IVirtualDevice.Stub
private final PendingTrampolineCallback mPendingTrampolineCallback;
private final int mOwnerUid;
private final InputController mInputController;
private VirtualAudioController mVirtualAudioController;
@VisibleForTesting
final Set<Integer> mVirtualDisplayIds = new ArraySet<>();
private final OnDeviceCloseListener mListener;
@@ -245,6 +249,46 @@ final class VirtualDeviceImpl extends IVirtualDevice.Stub
close();
}
@VisibleForTesting
VirtualAudioController getVirtualAudioControllerForTesting() {
return mVirtualAudioController;
}
@RequiresPermission(android.Manifest.permission.CREATE_VIRTUAL_DEVICE)
@Override // Binder call
public void onAudioSessionStarting(int displayId, IAudioSessionCallback callback) {
mContext.enforceCallingOrSelfPermission(
android.Manifest.permission.CREATE_VIRTUAL_DEVICE,
"Permission required to start audio session");
synchronized (mVirtualDeviceLock) {
if (!mVirtualDisplayIds.contains(displayId)) {
throw new SecurityException(
"Cannot start audio session for a display not associated with this virtual "
+ "device");
}
if (mVirtualAudioController == null) {
mVirtualAudioController = new VirtualAudioController(mContext);
GenericWindowPolicyController gwpc = mWindowPolicyControllers.get(displayId);
mVirtualAudioController.startListening(gwpc, callback);
}
}
}
@RequiresPermission(android.Manifest.permission.CREATE_VIRTUAL_DEVICE)
@Override // Binder call
public void onAudioSessionEnded() {
mContext.enforceCallingOrSelfPermission(
android.Manifest.permission.CREATE_VIRTUAL_DEVICE,
"Permission required to stop audio session");
synchronized (mVirtualDeviceLock) {
if (mVirtualAudioController != null) {
mVirtualAudioController.stopListening();
mVirtualAudioController = null;
}
}
}
@Override // Binder call
public void createVirtualKeyboard(
int displayId,

65
services/companion/java/com/android/server/companion/virtual/audio/AudioPlaybackDetector.java Normal file
View File

@@ -0,0 +1,65 @@
/*
* Copyright (C) 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.server.companion.virtual.audio;
import android.annotation.NonNull;
import android.content.Context;
import android.media.AudioManager;
import android.media.AudioPlaybackConfiguration;
import java.util.List;
/**
* Wrapper class for other classes to listen {@link #onPlaybackConfigChanged(List)} by implementing
* {@link AudioPlaybackCallback} instead of inheriting the
* {@link AudioManager.AudioPlaybackCallback}.
*/
final class AudioPlaybackDetector extends AudioManager.AudioPlaybackCallback {
/**
* Interface to listen {@link #onPlaybackConfigChanged(List)} from
* {@link AudioManager.AudioPlaybackCallback}.
*/
interface AudioPlaybackCallback {
void onPlaybackConfigChanged(List<AudioPlaybackConfiguration> configs);
}
private final AudioManager mAudioManager;
private AudioPlaybackCallback mAudioPlaybackCallback;
AudioPlaybackDetector(Context context) {
mAudioManager = context.getSystemService(AudioManager.class);
}
void register(@NonNull AudioPlaybackCallback callback) {
mAudioPlaybackCallback = callback;
mAudioManager.registerAudioPlaybackCallback(/* cb= */ this, /* handler= */ null);
}
void unregister() {
mAudioPlaybackCallback = null;
mAudioManager.unregisterAudioPlaybackCallback(/* cb= */ this);
}
@Override
public void onPlaybackConfigChanged(List<AudioPlaybackConfiguration> configs) {
super.onPlaybackConfigChanged(configs);
if (mAudioPlaybackCallback != null) {
mAudioPlaybackCallback.onPlaybackConfigChanged(configs);
}
}
}

65
services/companion/java/com/android/server/companion/virtual/audio/AudioRecordingDetector.java Normal file
View File

@@ -0,0 +1,65 @@
/*
* Copyright (C) 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.server.companion.virtual.audio;
import android.annotation.NonNull;
import android.content.Context;
import android.media.AudioManager;
import android.media.AudioRecordingConfiguration;
import java.util.List;
/**
* Wrapper class for other classes to listen {@link #onRecordingConfigChanged(List)} by implementing
* {@link AudioRecordingCallback} instead of inheriting the
* {@link AudioManager.AudioRecordingCallback}.
*/
final class AudioRecordingDetector extends AudioManager.AudioRecordingCallback {
/**
* Interface to listen {@link #onRecordingConfigChanged(List)} from
* {@link AudioManager.AudioRecordingCallback}.
*/
interface AudioRecordingCallback {
void onRecordingConfigChanged(List<AudioRecordingConfiguration> configs);
}
private final AudioManager mAudioManager;
private AudioRecordingCallback mAudioRecordingCallback;
AudioRecordingDetector(Context context) {
mAudioManager = context.getSystemService(AudioManager.class);
}
void register(@NonNull AudioRecordingCallback callback) {
mAudioRecordingCallback = callback;
mAudioManager.registerAudioRecordingCallback(/* cb= */ this, /* handler= */ null);
}
void unregister() {
mAudioRecordingCallback = null;
mAudioManager.unregisterAudioRecordingCallback(/* cb= */ this);
}
@Override
public void onRecordingConfigChanged(List<AudioRecordingConfiguration> configs) {
super.onRecordingConfigChanged(configs);
if (mAudioRecordingCallback != null) {
mAudioRecordingCallback.onRecordingConfigChanged(configs);
}
}
}

294
services/companion/java/com/android/server/companion/virtual/audio/VirtualAudioController.java Normal file
View File

@@ -0,0 +1,294 @@
/*
* Copyright (C) 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.server.companion.virtual.audio;
import static android.media.AudioPlaybackConfiguration.PLAYER_STATE_STARTED;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.annotation.RequiresPermission;
import android.companion.virtual.audio.IAudioSessionCallback;
import android.content.Context;
import android.media.AudioManager;
import android.media.AudioPlaybackConfiguration;
import android.media.AudioRecordingConfiguration;
import android.os.Handler;
import android.os.Looper;
import android.os.RemoteException;
import android.util.ArraySet;
import android.util.Slog;
import com.android.internal.annotations.GuardedBy;
import com.android.internal.annotations.VisibleForTesting;
import com.android.server.companion.virtual.GenericWindowPolicyController;
import com.android.server.companion.virtual.GenericWindowPolicyController.RunningAppsChangedListener;
import com.android.server.companion.virtual.audio.AudioPlaybackDetector.AudioPlaybackCallback;
import com.android.server.companion.virtual.audio.AudioRecordingDetector.AudioRecordingCallback;
import java.util.ArrayList;
import java.util.List;
/**
* Manages audio streams associated with a {@link VirtualAudioDevice}. Responsible for monitoring
* running applications and playback configuration changes in order to correctly re-route audio and
* then notify clients of these changes.
*/
public final class VirtualAudioController implements AudioPlaybackCallback,
AudioRecordingCallback, RunningAppsChangedListener {
private static final String TAG = "VirtualAudioController";
private static final int UPDATE_REROUTING_APPS_DELAY_MS = 2000;
private final Context mContext;
private final Handler mHandler = new Handler(Looper.getMainLooper());
private final Runnable mUpdateAudioRoutingRunnable = this::notifyAppsNeedingAudioRoutingChanged;
private final AudioPlaybackDetector mAudioPlaybackDetector;
private final AudioRecordingDetector mAudioRecordingDetector;
private final Object mLock = new Object();
@GuardedBy("mLock")
private final ArraySet<Integer> mRunningAppUids = new ArraySet<>();
@GuardedBy("mLock")
private ArraySet<Integer> mPlayingAppUids = new ArraySet<>();
private GenericWindowPolicyController mGenericWindowPolicyController;
private final Object mCallbackLock = new Object();
@Nullable
@GuardedBy("mCallbackLock")
private IAudioSessionCallback mCallback;
public VirtualAudioController(Context context) {
mContext = context;
mAudioPlaybackDetector = new AudioPlaybackDetector(context);
mAudioRecordingDetector = new AudioRecordingDetector(context);
}
/**
* Starts to listen to running applications and audio configuration changes on virtual display
* for audio capture and injection.
*/
public void startListening(
@NonNull GenericWindowPolicyController genericWindowPolicyController,
@Nullable IAudioSessionCallback callback) {
mGenericWindowPolicyController = genericWindowPolicyController;
mGenericWindowPolicyController.setRunningAppsChangedListener(/* listener= */ this);
synchronized (mCallbackLock) {
mCallback = callback;
}
synchronized (mLock) {
mRunningAppUids.clear();
mPlayingAppUids.clear();
}
mAudioPlaybackDetector.register(/* callback= */ this);
mAudioRecordingDetector.register(/* callback= */ this);
}
/**
* Stops listening to running applications and audio configuration changes on virtual display
* for audio capture and injection.
*/
public void stopListening() {
if (mHandler.hasCallbacks(mUpdateAudioRoutingRunnable)) {
mHandler.removeCallbacks(mUpdateAudioRoutingRunnable);
}
mAudioPlaybackDetector.unregister();
mAudioRecordingDetector.unregister();
if (mGenericWindowPolicyController != null) {
mGenericWindowPolicyController.setRunningAppsChangedListener(/* listener= */ null);
mGenericWindowPolicyController = null;
}
synchronized (mCallbackLock) {
mCallback = null;
}
}
@Override
public void onRunningAppsChanged(ArraySet<Integer> runningUids) {
synchronized (mLock) {
if (mRunningAppUids.equals(runningUids)) {
// Ignore no-op events.
return;
}
mRunningAppUids.clear();
mRunningAppUids.addAll(runningUids);
ArraySet<Integer> oldPlayingAppUids = mPlayingAppUids;
// Update the list of playing apps after caching the old list, and before checking if
// the list of playing apps is empty. This is a subset of the running apps, so we need
// to update this here as well.
AudioManager audioManager = mContext.getSystemService(AudioManager.class);
List<AudioPlaybackConfiguration> configs =
audioManager.getActivePlaybackConfigurations();
mPlayingAppUids = findPlayingAppUids(configs, mRunningAppUids);
// Do not change rerouted applications while any application is playing, or the sound
// will be leaked from phone during the transition. Delay the change until we detect
// there is no application is playing in onPlaybackConfigChanged().
if (!mPlayingAppUids.isEmpty()) {
Slog.i(TAG, "Audio is playing, do not change rerouted apps");
return;
}
// An application previously playing audio was removed from the display.
if (!oldPlayingAppUids.isEmpty()) {
// Delay changing the rerouted application when the last application playing audio
// was removed from virtual device, or the sound will be leaked from phone side
// during the transition.
Slog.i(TAG, "The last playing app removed, delay change rerouted apps");
if (mHandler.hasCallbacks(mUpdateAudioRoutingRunnable)) {
mHandler.removeCallbacks(mUpdateAudioRoutingRunnable);
}
mHandler.postDelayed(mUpdateAudioRoutingRunnable, UPDATE_REROUTING_APPS_DELAY_MS);
return;
}
}
// Normal case with no application playing, just update routing.
notifyAppsNeedingAudioRoutingChanged();
}
@Override
public void onPlaybackConfigChanged(List<AudioPlaybackConfiguration> configs) {
updatePlayingApplications(configs);
List<AudioPlaybackConfiguration> audioPlaybackConfigurations;
synchronized (mLock) {
// Filter configurations of applications running on virtual device.
audioPlaybackConfigurations = findPlaybackConfigurations(configs, mRunningAppUids);
}
synchronized (mCallbackLock) {
if (mCallback != null) {
try {
mCallback.onPlaybackConfigChanged(audioPlaybackConfigurations);
} catch (RemoteException e) {
Slog.e(TAG, "RemoteException when calling onPlaybackConfigChanged", e);
}
}
}
}
@RequiresPermission(android.Manifest.permission.MODIFY_AUDIO_ROUTING)
@Override
public void onRecordingConfigChanged(List<AudioRecordingConfiguration> configs) {
List<AudioRecordingConfiguration> audioRecordingConfigurations;
synchronized (mLock) {
// Filter configurations of applications running on virtual device.
audioRecordingConfigurations = findRecordingConfigurations(configs, mRunningAppUids);
}
synchronized (mCallbackLock) {
if (mCallback != null) {
try {
mCallback.onRecordingConfigChanged(audioRecordingConfigurations);
} catch (RemoteException e) {
Slog.e(TAG, "RemoteException when calling onRecordingConfigChanged", e);
}
}
}
}
private void updatePlayingApplications(List<AudioPlaybackConfiguration> configs) {
synchronized (mLock) {
ArraySet<Integer> playingAppUids = findPlayingAppUids(configs, mRunningAppUids);
if (mPlayingAppUids.equals(playingAppUids)) {
return;
}
mPlayingAppUids = playingAppUids;
}
// Updated rerouted apps, even if the app is already playing. It originally should be done
// when onRunningAppsChanged() is called, but we don't want to interrupt the audio
// streaming and cause the sound leak from phone when it's playing, so delay until here.
notifyAppsNeedingAudioRoutingChanged();
}
private void notifyAppsNeedingAudioRoutingChanged() {
if (mHandler.hasCallbacks(mUpdateAudioRoutingRunnable)) {
mHandler.removeCallbacks(mUpdateAudioRoutingRunnable);
}
int[] runningUids;
synchronized (mLock) {
runningUids = new int[mRunningAppUids.size()];
for (int i = 0; i < mRunningAppUids.size(); i++) {
runningUids[i] = mRunningAppUids.valueAt(i);
}
}
synchronized (mCallbackLock) {
if (mCallback != null) {
try {
mCallback.onAppsNeedingAudioRoutingChanged(runningUids);
} catch (RemoteException e) {
Slog.e(TAG, "RemoteException when calling updateReroutingApps", e);
}
}
}
}
/**
* Finds uid of playing applications from the given running applications.
*
* @param configs a list of playback configs which get from {@link AudioManager}
*/
private static ArraySet<Integer> findPlayingAppUids(List<AudioPlaybackConfiguration> configs,
ArraySet<Integer> runningAppUids) {
ArraySet<Integer> playingAppUids = new ArraySet<>();
for (AudioPlaybackConfiguration config : configs) {
if (runningAppUids.contains(config.getClientUid())
&& config.getPlayerState() == PLAYER_STATE_STARTED) {
playingAppUids.add(config.getClientUid());
}
}
return playingAppUids;
}
/** Finds a list of {@link AudioPlaybackConfiguration} for the given running applications. */
private static List<AudioPlaybackConfiguration> findPlaybackConfigurations(
List<AudioPlaybackConfiguration> configs,
ArraySet<Integer> runningAppUids) {
List<AudioPlaybackConfiguration> runningConfigs = new ArrayList<>();
for (AudioPlaybackConfiguration config : configs) {
if (runningAppUids.contains(config.getClientUid())) {
runningConfigs.add(config);
}
}
return runningConfigs;
}
/** Finds a list of {@link AudioRecordingConfiguration} for the given running applications. */
@RequiresPermission(android.Manifest.permission.MODIFY_AUDIO_ROUTING)
private static List<AudioRecordingConfiguration> findRecordingConfigurations(
List<AudioRecordingConfiguration> configs, ArraySet<Integer> runningAppUids) {
List<AudioRecordingConfiguration> runningConfigs = new ArrayList<>();
for (AudioRecordingConfiguration config : configs) {
if (runningAppUids.contains(config.getClientUid())) {
runningConfigs.add(config);
}
}
return runningConfigs;
}
@VisibleForTesting
boolean hasPendingRunnable() {
return mHandler.hasCallbacks(mUpdateAudioRoutingRunnable);
}
@VisibleForTesting
void addPlayingAppsForTesting(int appUid) {
synchronized (mLock) {
mPlayingAppUids.add(appUid);
}
}
}

45
services/tests/servicestests/src/com/android/server/companion/virtual/VirtualDeviceManagerServiceTest.java
View File

@@ -16,6 +16,7 @@
package com.android.server.companion.virtual;
import static com.google.common.truth.Truth.assertThat;
import static com.google.common.truth.Truth.assertWithMessage;
import static org.mockito.ArgumentMatchers.any;
@@ -39,6 +40,7 @@ import android.app.admin.DevicePolicyManager;
import android.companion.AssociationInfo;
import android.companion.virtual.IVirtualDeviceActivityListener;
import android.companion.virtual.VirtualDeviceParams;
import android.companion.virtual.audio.IAudioSessionCallback;
import android.content.Context;
import android.content.ContextWrapper;
import android.graphics.Point;
@@ -111,6 +113,8 @@ public class VirtualDeviceManagerServiceTest {
@Mock
IThermalService mIThermalServiceMock;
private PowerManager mPowerManager;
@Mock
private IAudioSessionCallback mCallback;
@Before
public void setUp() {
@@ -249,6 +253,12 @@ public class VirtualDeviceManagerServiceTest {
VENDOR_ID, PRODUCT_ID, BINDER, new Point(WIDTH, HEIGHT)));
}
@Test
public void onAudioSessionStarting_noDisplay_failsSecurityException() {
assertThrows(SecurityException.class,
() -> mDeviceImpl.onAudioSessionStarting(DISPLAY_ID, mCallback));
}
@Test
public void createVirtualKeyboard_noPermission_failsSecurityException() {
mDeviceImpl.mVirtualDisplayIds.add(DISPLAY_ID);
@@ -282,6 +292,22 @@ public class VirtualDeviceManagerServiceTest {
VENDOR_ID, PRODUCT_ID, BINDER, new Point(WIDTH, HEIGHT)));
}
@Test
public void onAudioSessionStarting_noPermission_failsSecurityException() {
mDeviceImpl.mVirtualDisplayIds.add(DISPLAY_ID);
doCallRealMethod().when(mContext).enforceCallingOrSelfPermission(
eq(Manifest.permission.CREATE_VIRTUAL_DEVICE), anyString());
assertThrows(SecurityException.class,
() -> mDeviceImpl.onAudioSessionStarting(DISPLAY_ID, mCallback));
}
@Test
public void onAudioSessionEnded_noPermission_failsSecurityException() {
doCallRealMethod().when(mContext).enforceCallingOrSelfPermission(
eq(Manifest.permission.CREATE_VIRTUAL_DEVICE), anyString());
assertThrows(SecurityException.class, () -> mDeviceImpl.onAudioSessionEnded());
}
@Test
public void createVirtualKeyboard_hasDisplay_obtainFileDescriptor() {
mDeviceImpl.mVirtualDisplayIds.add(DISPLAY_ID);
@@ -315,6 +341,25 @@ public class VirtualDeviceManagerServiceTest {
eq(PRODUCT_ID), anyString(), eq(HEIGHT), eq(WIDTH));
}
@Test
public void onAudioSessionStarting_hasVirtualAudioController() {
mDeviceImpl.onVirtualDisplayCreatedLocked(DISPLAY_ID);
mDeviceImpl.onAudioSessionStarting(DISPLAY_ID, mCallback);
assertThat(mDeviceImpl.getVirtualAudioControllerForTesting()).isNotNull();
}
@Test
public void onAudioSessionEnded_noVirtualAudioController() {
mDeviceImpl.onVirtualDisplayCreatedLocked(DISPLAY_ID);
mDeviceImpl.onAudioSessionStarting(DISPLAY_ID, mCallback);
mDeviceImpl.onAudioSessionEnded();
assertThat(mDeviceImpl.getVirtualAudioControllerForTesting()).isNull();
}
@Test
public void sendKeyEvent_noFd() {
assertThrows(

202
services/tests/servicestests/src/com/android/server/companion/virtual/audio/VirtualAudioControllerTest.java Normal file
View File

@@ -0,0 +1,202 @@
/*
* Copyright (C) 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.server.companion.virtual.audio;
import static android.media.AudioAttributes.FLAG_SECURE;
import static android.media.AudioPlaybackConfiguration.PLAYER_STATE_STARTED;
import static android.view.WindowManager.LayoutParams.SYSTEM_FLAG_HIDE_NON_SYSTEM_OVERLAY_WINDOWS;
import static com.google.common.truth.Truth.assertThat;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import android.companion.virtual.audio.IAudioSessionCallback;
import android.content.Context;
import android.content.ContextWrapper;
import android.media.AudioPlaybackConfiguration;
import android.media.AudioRecordingConfiguration;
import android.media.MediaRecorder;
import android.media.PlayerBase;
import android.os.Parcel;
import android.os.RemoteException;
import android.util.ArraySet;
import androidx.test.InstrumentationRegistry;
import androidx.test.runner.AndroidJUnit4;
import com.android.server.companion.virtual.GenericWindowPolicyController;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.MockitoAnnotations;
import java.util.ArrayList;
import java.util.List;
@RunWith(AndroidJUnit4.class)
public class VirtualAudioControllerTest {
private static final int APP1_UID = 100;
private static final int APP2_UID = 200;
private Context mContext;
private VirtualAudioController mVirtualAudioController;
private GenericWindowPolicyController mGenericWindowPolicyController;
@Mock IAudioSessionCallback mCallback;
@Before
public void setUp() {
MockitoAnnotations.initMocks(this);
mContext = Mockito.spy(new ContextWrapper(InstrumentationRegistry.getTargetContext()));
mVirtualAudioController = new VirtualAudioController(mContext);
mGenericWindowPolicyController = new GenericWindowPolicyController(
FLAG_SECURE,
SYSTEM_FLAG_HIDE_NON_SYSTEM_OVERLAY_WINDOWS,
/* allowedUsers= */ new ArraySet<>(),
/* allowedActivities= */ new ArraySet<>(),
/* blockedActivities= */ new ArraySet<>(),
/* activityListener= */null,
/* activityBlockedCallback= */ null);
}
@Test
public void startListening_receivesCallback() throws RemoteException {
ArraySet<Integer> runningUids = new ArraySet<>();
runningUids.add(APP1_UID);
int[] appUids = new int[] {APP1_UID};
mVirtualAudioController.startListening(mGenericWindowPolicyController, mCallback);
mGenericWindowPolicyController.onRunningAppsChanged(runningUids);
verify(mCallback).onAppsNeedingAudioRoutingChanged(appUids);
}
@Test
public void stopListening_removesCallback() throws RemoteException {
ArraySet<Integer> runningUids = new ArraySet<>();
runningUids.add(APP1_UID);
int[] appUids = new int[] {APP1_UID};
mVirtualAudioController.startListening(mGenericWindowPolicyController, mCallback);
mVirtualAudioController.stopListening();
mGenericWindowPolicyController.onRunningAppsChanged(runningUids);
verify(mCallback, never()).onAppsNeedingAudioRoutingChanged(appUids);
}
@Test
public void onRunningAppsChanged_notifiesAudioRoutingModified() throws RemoteException {
mVirtualAudioController.startListening(mGenericWindowPolicyController, mCallback);
ArraySet<Integer> runningUids = new ArraySet<>();
runningUids.add(APP1_UID);
mVirtualAudioController.onRunningAppsChanged(runningUids);
int[] appUids = new int[] {APP1_UID};
verify(mCallback).onAppsNeedingAudioRoutingChanged(appUids);
}
@Test
public void onRunningAppsChanged_audioIsPlaying_doesNothing() throws RemoteException {
mVirtualAudioController.startListening(mGenericWindowPolicyController, mCallback);
mVirtualAudioController.addPlayingAppsForTesting(APP2_UID);
ArraySet<Integer> runningUids = new ArraySet<>();
runningUids.add(APP1_UID);
mVirtualAudioController.onRunningAppsChanged(runningUids);
int[] appUids = new int[]{APP1_UID};
verify(mCallback, never()).onAppsNeedingAudioRoutingChanged(appUids);
}
@Test
public void onRunningAppsChanged_lastPlayingAppRemoved_delaysReroutingAudio() {
ArraySet<Integer> runningUids = new ArraySet<>();
runningUids.add(APP1_UID);
runningUids.add(APP2_UID);
mVirtualAudioController.onRunningAppsChanged(runningUids);
mVirtualAudioController.addPlayingAppsForTesting(APP2_UID);
ArraySet<Integer> appUids = new ArraySet<>();
appUids.add(APP1_UID);
mVirtualAudioController.onRunningAppsChanged(appUids);
assertThat(mVirtualAudioController.hasPendingRunnable()).isTrue();
}
@Test
public void onPlaybackConfigChanged_sendsCallback() throws RemoteException {
mVirtualAudioController.startListening(mGenericWindowPolicyController, mCallback);
ArraySet<Integer> runningUids = new ArraySet<>();
runningUids.add(APP1_UID);
mVirtualAudioController.onRunningAppsChanged(runningUids);
List<AudioPlaybackConfiguration> configs = createPlaybackConfigurations(runningUids);
mVirtualAudioController.onPlaybackConfigChanged(configs);
verify(mCallback).onPlaybackConfigChanged(configs);
}
@Test
public void onRecordingConfigChanged_sendsCallback() throws RemoteException {
mVirtualAudioController.startListening(mGenericWindowPolicyController, mCallback);
ArraySet<Integer> runningUids = new ArraySet<>();
runningUids.add(APP1_UID);
mVirtualAudioController.onRunningAppsChanged(runningUids);
List<AudioRecordingConfiguration> configs = createRecordingConfigurations(runningUids);
mVirtualAudioController.onRecordingConfigChanged(configs);
verify(mCallback).onRecordingConfigChanged(configs);
}
private List<AudioPlaybackConfiguration> createPlaybackConfigurations(
ArraySet<Integer> appUids) {
List<AudioPlaybackConfiguration> configs = new ArrayList<>();
for (int appUid : appUids) {
PlayerBase.PlayerIdCard playerIdCard =
PlayerBase.PlayerIdCard.CREATOR.createFromParcel(Parcel.obtain());
AudioPlaybackConfiguration audioPlaybackConfiguration =
new AudioPlaybackConfiguration(
playerIdCard, /* piid= */ 1000, appUid, /* pid= */ 1000);
audioPlaybackConfiguration.handleStateEvent(PLAYER_STATE_STARTED, /* deviceId= */1);
configs.add(audioPlaybackConfiguration);
}
return configs;
}
private List<AudioRecordingConfiguration> createRecordingConfigurations(
ArraySet<Integer> appUids) {
List<AudioRecordingConfiguration> configs = new ArrayList<>();
for (int appUid : appUids) {
AudioRecordingConfiguration audioRecordingConfiguration =
new AudioRecordingConfiguration(
/* uid= */ appUid,
/* session= */ 1000,
MediaRecorder.AudioSource.MIC,
/* clientFormat= */ null,
/* devFormat= */ null,
/* patchHandle= */ 1000,
"com.android.example");
configs.add(audioRecordingConfiguration);
}
return configs;
}
}