Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge "Implement VirtualAudioDevice API"

Browse Source
This commit is contained in:
TreeHugger Robot
2022-02-10 07:41:25 +00:00
committed by Android (Google) Code Review
parent 5c393c4ac8 69cf28591c
commit 391e76398d
16 changed files with 1753 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
core/api/system-current.txt
View File

@@ -2750,6 +2750,7 @@ package android.companion.virtual {
method public void addActivityListener(@NonNull android.companion.virtual.VirtualDeviceManager.ActivityListener);
method public void addActivityListener(@NonNull android.companion.virtual.VirtualDeviceManager.ActivityListener, @NonNull java.util.concurrent.Executor);
method @RequiresPermission(android.Manifest.permission.CREATE_VIRTUAL_DEVICE) public void close();
method @NonNull @RequiresPermission(android.Manifest.permission.CREATE_VIRTUAL_DEVICE) public android.companion.virtual.audio.VirtualAudioDevice createVirtualAudioDevice(@NonNull android.hardware.display.VirtualDisplay, @Nullable java.util.concurrent.Executor, @Nullable android.companion.virtual.audio.VirtualAudioDevice.AudioConfigurationChangeCallback);
method @Nullable public android.hardware.display.VirtualDisplay createVirtualDisplay(int, int, int, @Nullable android.view.Surface, int, @Nullable android.os.Handler, @Nullable android.hardware.display.VirtualDisplay.Callback);
method @NonNull @RequiresPermission(android.Manifest.permission.CREATE_VIRTUAL_DEVICE) public android.hardware.input.VirtualKeyboard createVirtualKeyboard(@NonNull android.hardware.display.VirtualDisplay, @NonNull String, int, int);
method @NonNull @RequiresPermission(android.Manifest.permission.CREATE_VIRTUAL_DEVICE) public android.hardware.input.VirtualMouse createVirtualMouse(@NonNull android.hardware.display.VirtualDisplay, @NonNull String, int, int);
@@ -2782,6 +2783,39 @@ package android.companion.virtual {
}
package android.companion.virtual.audio {
public final class AudioCapture {
ctor public AudioCapture();
method public int getRecordingState();
method public int read(@NonNull java.nio.ByteBuffer, int);
method public void startRecording();
method public void stop();
}
public final class AudioInjection {
ctor public AudioInjection();
method public int getPlayState();
method public void play();
method public void stop();
method public int write(@NonNull java.nio.ByteBuffer, int, int);
}
public final class VirtualAudioDevice implements java.io.Closeable {
method @RequiresPermission(android.Manifest.permission.MODIFY_AUDIO_ROUTING) public void close();
method @Nullable public android.companion.virtual.audio.AudioCapture getAudioCapture();
method @Nullable public android.companion.virtual.audio.AudioInjection getAudioInjection();
method @NonNull @RequiresPermission(android.Manifest.permission.MODIFY_AUDIO_ROUTING) public android.companion.virtual.audio.AudioCapture startAudioCapture(@NonNull android.media.AudioFormat);
method @NonNull @RequiresPermission(android.Manifest.permission.MODIFY_AUDIO_ROUTING) public android.companion.virtual.audio.AudioInjection startAudioInjection(@NonNull android.media.AudioFormat);
}
public static interface VirtualAudioDevice.AudioConfigurationChangeCallback {
method public void onPlaybackConfigChanged(@NonNull java.util.List<android.media.AudioPlaybackConfiguration>);
method public void onRecordingConfigChanged(@NonNull java.util.List<android.media.AudioRecordingConfiguration>);
}
}
package android.content {
public class ApexEnvironment {

10
core/java/android/companion/virtual/IVirtualDevice.aidl
View File

@@ -17,6 +17,7 @@
package android.companion.virtual;
import android.app.PendingIntent;
import android.companion.virtual.audio.IAudioSessionCallback;
import android.graphics.Point;
import android.graphics.PointF;
import android.hardware.input.VirtualKeyEvent;
@@ -45,6 +46,15 @@ interface IVirtualDevice {
*/
void close();
/**
* Notifies of an audio session being started.
*/
void onAudioSessionStarting(
int displayId,
IAudioSessionCallback callback);
void onAudioSessionEnded();
void createVirtualKeyboard(
int displayId,
String inputDeviceName,

26
core/java/android/companion/virtual/VirtualDeviceManager.java
View File

@@ -25,6 +25,8 @@ import android.annotation.SystemService;
import android.app.Activity;
import android.app.PendingIntent;
import android.companion.AssociationInfo;
import android.companion.virtual.audio.VirtualAudioDevice;
import android.companion.virtual.audio.VirtualAudioDevice.AudioConfigurationChangeCallback;
import android.content.ComponentName;
import android.content.Context;
import android.graphics.Point;
@@ -338,6 +340,30 @@ public final class VirtualDeviceManager {
}
}
/**
* Creates a VirtualAudioDevice, capable of recording audio emanating from this device,
* or injecting audio from another device.
*
* <p>Note: This object does not support capturing privileged playback, such as voice call
* audio.
*
* @param display The target virtual display to capture from and inject into.
* @param executor The {@link Executor} object for the thread on which to execute
* the callback. If <code>null</code>, the {@link Executor} associated with
* the main {@link Looper} will be used.
* @param callback Interface to be notified when playback or recording configuration of
* applications running on virtual display is changed.
* @return A {@link VirtualAudioDevice} instance.
*/
@RequiresPermission(android.Manifest.permission.CREATE_VIRTUAL_DEVICE)
@NonNull
public VirtualAudioDevice createVirtualAudioDevice(
@NonNull VirtualDisplay display,
@Nullable Executor executor,
@Nullable AudioConfigurationChangeCallback callback) {
return new VirtualAudioDevice(mContext, mVirtualDevice, display, executor, callback);
}
/**
* Sets the visibility of the pointer icon for this VirtualDevice's associated displays.
*

124
core/java/android/companion/virtual/audio/AudioCapture.java Normal file
View File

@@ -0,0 +1,124 @@
/*
* Copyright (C) 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package android.companion.virtual.audio;
import static android.media.AudioRecord.RECORDSTATE_RECORDING;
import static android.media.AudioRecord.RECORDSTATE_STOPPED;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.annotation.SuppressLint;
import android.annotation.SystemApi;
import android.media.AudioRecord;
import android.util.Log;
import com.android.internal.annotations.GuardedBy;
import java.nio.ByteBuffer;
/**
* Wrapper around {@link AudioRecord} that allows for the underlying {@link AudioRecord} to
* be swapped out while recording is ongoing.
*
* @hide
*/
// The stop() actually doesn't release resources, so should not force implementing Closeable.
@SuppressLint("NotCloseable")
@SystemApi
public final class AudioCapture {
private static final String TAG = "AudioCapture";
private final Object mLock = new Object();
@GuardedBy("mLock")
@Nullable
private AudioRecord mAudioRecord;
@GuardedBy("mLock")
private int mRecordingState = RECORDSTATE_STOPPED;
/**
* Sets the {@link AudioRecord} to handle audio capturing.
* Callers may call this multiple times with different audio records to change
* the underlying {@link AudioRecord} without stopping and re-starting recording.
*
* @param audioRecord The underlying {@link AudioRecord} to use for capture,
* or null if no audio (i.e. silence) should be captured while still keeping the
* record in a recording state.
*/
void setAudioRecord(@Nullable AudioRecord audioRecord) {
Log.d(TAG, "set AudioRecord with " + audioRecord);
synchronized (mLock) {
// Release old reference.
if (mAudioRecord != null) {
mAudioRecord.release();
}
// Sync recording state for new reference.
if (audioRecord != null) {
if (mRecordingState == RECORDSTATE_RECORDING
&& audioRecord.getRecordingState() != RECORDSTATE_RECORDING) {
audioRecord.startRecording();
}
if (mRecordingState == RECORDSTATE_STOPPED
&& audioRecord.getRecordingState() != RECORDSTATE_STOPPED) {
audioRecord.stop();
}
}
mAudioRecord = audioRecord;
}
}
/** See {@link AudioRecord#read(ByteBuffer, int)}. */
public int read(@NonNull ByteBuffer audioBuffer, int sizeInBytes) {
final int sizeRead;
synchronized (mLock) {
if (mAudioRecord != null) {
sizeRead = mAudioRecord.read(audioBuffer, sizeInBytes);
} else {
sizeRead = 0;
}
}
return sizeRead;
}
/** See {@link AudioRecord#startRecording()}. */
public void startRecording() {
synchronized (mLock) {
mRecordingState = RECORDSTATE_RECORDING;
if (mAudioRecord != null && mAudioRecord.getRecordingState() != RECORDSTATE_RECORDING) {
mAudioRecord.startRecording();
}
}
}
/** See {@link AudioRecord#stop()}. */
public void stop() {
synchronized (mLock) {
mRecordingState = RECORDSTATE_STOPPED;
if (mAudioRecord != null && mAudioRecord.getRecordingState() != RECORDSTATE_STOPPED) {
mAudioRecord.stop();
}
}
}
/** See {@link AudioRecord#getRecordingState()}. */
public int getRecordingState() {
synchronized (mLock) {
return mRecordingState;
}
}
}

124
core/java/android/companion/virtual/audio/AudioInjection.java Normal file
View File

@@ -0,0 +1,124 @@
/*
* Copyright (C) 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package android.companion.virtual.audio;
import static android.media.AudioTrack.PLAYSTATE_PLAYING;
import static android.media.AudioTrack.PLAYSTATE_STOPPED;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.annotation.SuppressLint;
import android.annotation.SystemApi;
import android.media.AudioTrack;
import android.util.Log;
import com.android.internal.annotations.GuardedBy;
import java.nio.ByteBuffer;
/**
* Wrapper around {@link AudioTrack} that allows for the underlying {@link AudioTrack} to
* be swapped out while playout is ongoing.
*
* @hide
*/
// The stop() actually doesn't release resources, so should not force implementing Closeable.
@SuppressLint("NotCloseable")
@SystemApi
public final class AudioInjection {
private static final String TAG = "AudioInjection";
private final Object mLock = new Object();
@GuardedBy("mLock")
@Nullable
private AudioTrack mAudioTrack;
@GuardedBy("mLock")
private int mPlayState = PLAYSTATE_STOPPED;
/**
* Sets the {@link AudioTrack} to handle audio injection.
* Callers may call this multiple times with different audio tracks to change
* the underlying {@link AudioTrack} without stopping and re-starting injection.
*
* @param audioTrack The underlying {@link AudioTrack} to use for injection,
* or null if no audio (i.e. silence) should be injected while still keeping the
* record in a playing state.
*/
void setAudioTrack(@Nullable AudioTrack audioTrack) {
Log.d(TAG, "set AudioTrack with " + audioTrack);
synchronized (mLock) {
// Release old reference.
if (mAudioTrack != null) {
mAudioTrack.release();
}
// Sync play state for new reference.
if (audioTrack != null) {
if (mPlayState == PLAYSTATE_PLAYING
&& audioTrack.getPlayState() != PLAYSTATE_PLAYING) {
audioTrack.play();
}
if (mPlayState == PLAYSTATE_STOPPED
&& audioTrack.getPlayState() != PLAYSTATE_STOPPED) {
audioTrack.stop();
}
}
mAudioTrack = audioTrack;
}
}
/** See {@link AudioTrack#write(ByteBuffer, int, int)}. */
public int write(@NonNull ByteBuffer audioBuffer, int sizeInBytes, int writeMode) {
final int sizeWrite;
synchronized (mLock) {
if (mAudioTrack != null) {
sizeWrite = mAudioTrack.write(audioBuffer, sizeInBytes, writeMode);
} else {
sizeWrite = 0;
}
}
return sizeWrite;
}
/** See {@link AudioTrack#play()}. */
public void play() {
synchronized (mLock) {
mPlayState = PLAYSTATE_PLAYING;
if (mAudioTrack != null && mAudioTrack.getPlayState() != PLAYSTATE_PLAYING) {
mAudioTrack.play();
}
}
}
/** See {@link AudioTrack#stop()}. */
public void stop() {
synchronized (mLock) {
mPlayState = PLAYSTATE_STOPPED;
if (mAudioTrack != null && mAudioTrack.getPlayState() != PLAYSTATE_STOPPED) {
mAudioTrack.stop();
}
}
}
/** See {@link AudioTrack#getPlayState()}. */
public int getPlayState() {
synchronized (mLock) {
return mPlayState;
}
}
}

44
core/java/android/companion/virtual/audio/IAudioSessionCallback.aidl Normal file
View File

@@ -0,0 +1,44 @@
/*
* Copyright (C) 2021 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package android.companion.virtual.audio;
import android.media.AudioPlaybackConfiguration;
import android.media.AudioRecordingConfiguration;
/**
* Callback to control audio rerouting, notify playback and recording state of applications running
* on virtual device.
*
* @hide
*/
oneway interface IAudioSessionCallback {
/** Updates the set of applications that need to have their audio rerouted. */
void onAppsNeedingAudioRoutingChanged(in int[] appUids);
/**
* Called whenever the playback configuration of applications running on virtual device has
* changed.
*/
void onPlaybackConfigChanged(in List<AudioPlaybackConfiguration> configs);
/**
* Called whenever the recording configuration of applications running on virtual device has
* changed.
*/
void onRecordingConfigChanged(in List<AudioRecordingConfiguration> configs);
}

167
core/java/android/companion/virtual/audio/VirtualAudioDevice.java Normal file
View File

@@ -0,0 +1,167 @@
/*
* Copyright (C) 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package android.companion.virtual.audio;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.annotation.RequiresPermission;
import android.annotation.SystemApi;
import android.companion.virtual.IVirtualDevice;
import android.content.Context;
import android.hardware.display.VirtualDisplay;
import android.media.AudioFormat;
import android.media.AudioPlaybackConfiguration;
import android.media.AudioRecordingConfiguration;
import android.os.RemoteException;
import java.io.Closeable;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.Executor;
/**
* The class stores an {@link AudioCapture} for audio capturing and an {@link AudioInjection} for
* audio injection.
*
* @hide
*/
@SystemApi
public final class VirtualAudioDevice implements Closeable {
/**
* Interface to be notified when playback or recording configuration of applications running on
* virtual display was changed.
*
* @hide
*/
@SystemApi
public interface AudioConfigurationChangeCallback {
/**
* Notifies when playback configuration of applications running on virtual display was
* changed.
*/
void onPlaybackConfigChanged(@NonNull List<AudioPlaybackConfiguration> configs);
/**
* Notifies when recording configuration of applications running on virtual display was
* changed.
*/
void onRecordingConfigChanged(@NonNull List<AudioRecordingConfiguration> configs);
}
private final Context mContext;
private final IVirtualDevice mVirtualDevice;
private final VirtualDisplay mVirtualDisplay;
private final AudioConfigurationChangeCallback mCallback;
private final Executor mExecutor;
@Nullable
private VirtualAudioSession mOngoingSession;
/**
* @hide
*/
public VirtualAudioDevice(Context context, IVirtualDevice virtualDevice,
VirtualDisplay virtualDisplay, Executor executor,
AudioConfigurationChangeCallback callback) {
mContext = context;
mVirtualDevice = virtualDevice;
mVirtualDisplay = virtualDisplay;
mExecutor = executor;
mCallback = callback;
}
/**
* Begins injecting audio from a remote device into this device.
*
* @return An {@link AudioInjection} containing the injected audio.
*/
@RequiresPermission(android.Manifest.permission.MODIFY_AUDIO_ROUTING)
@NonNull
public AudioInjection startAudioInjection(@NonNull AudioFormat injectionFormat) {
Objects.requireNonNull(injectionFormat, "injectionFormat must not be null");
if (mOngoingSession != null && mOngoingSession.getAudioInjection() != null) {
throw new IllegalStateException("Cannot start an audio injection while a session is "
+ "ongoing. Call close() on this device first to end the previous injection.");
}
if (mOngoingSession == null) {
mOngoingSession = new VirtualAudioSession(mContext, mCallback, mExecutor);
}
try {
mVirtualDevice.onAudioSessionStarting(mVirtualDisplay.getDisplay().getDisplayId(),
/* callback= */ mOngoingSession);
} catch (RemoteException e) {
throw e.rethrowFromSystemServer();
}
return mOngoingSession.startAudioInjection(injectionFormat);
}
/**
* Begins recording audio emanating from this device.
*
* @return An {@link AudioCapture} containing the recorded audio.
*/
@RequiresPermission(android.Manifest.permission.MODIFY_AUDIO_ROUTING)
@NonNull
public AudioCapture startAudioCapture(@NonNull AudioFormat captureFormat) {
Objects.requireNonNull(captureFormat, "captureFormat must not be null");
if (mOngoingSession != null && mOngoingSession.getAudioCapture() != null) {
throw new IllegalStateException("Cannot start an audio capture while a session is "
+ "ongoing. Call close() on this device first to end the previous session.");
}
if (mOngoingSession == null) {
mOngoingSession = new VirtualAudioSession(mContext, mCallback, mExecutor);
}
try {
mVirtualDevice.onAudioSessionStarting(mVirtualDisplay.getDisplay().getDisplayId(),
/* callback= */ mOngoingSession);
} catch (RemoteException e) {
throw e.rethrowFromSystemServer();
}
return mOngoingSession.startAudioCapture(captureFormat);
}
/** Returns the {@link AudioCapture} instance. */
@Nullable
public AudioCapture getAudioCapture() {
return mOngoingSession != null ? mOngoingSession.getAudioCapture() : null;
}
/** Returns the {@link AudioInjection} instance. */
@Nullable
public AudioInjection getAudioInjection() {
return mOngoingSession != null ? mOngoingSession.getAudioInjection() : null;
}
@RequiresPermission(android.Manifest.permission.MODIFY_AUDIO_ROUTING)
@Override
public void close() {
if (mOngoingSession != null) {
mOngoingSession.close();
mOngoingSession = null;
try {
mVirtualDevice.onAudioSessionEnded();
} catch (RemoteException e) {
throw e.rethrowFromSystemServer();
}
}
}
}

305
core/java/android/companion/virtual/audio/VirtualAudioSession.java Normal file
View File

@@ -0,0 +1,305 @@
/*
* Copyright (C) 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package android.companion.virtual.audio;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.annotation.RequiresPermission;
import android.companion.virtual.audio.VirtualAudioDevice.AudioConfigurationChangeCallback;
import android.content.Context;
import android.media.AudioFormat;
import android.media.AudioManager;
import android.media.AudioPlaybackConfiguration;
import android.media.AudioRecord;
import android.media.AudioRecordingConfiguration;
import android.media.AudioTrack;
import android.media.audiopolicy.AudioMix;
import android.media.audiopolicy.AudioMixingRule;
import android.media.audiopolicy.AudioPolicy;
import android.util.IntArray;
import android.util.Log;
import com.android.internal.annotations.GuardedBy;
import com.android.internal.annotations.VisibleForTesting;
import java.io.Closeable;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.Executor;
/**
* Manages an ongiong audio session in which audio can be captured (recorded) and/or
* injected from a remote device.
*
* @hide
*/
@VisibleForTesting
public final class VirtualAudioSession extends IAudioSessionCallback.Stub implements
Closeable {
private static final String TAG = "VirtualAudioSession";
private final Context mContext;
/** The {@link Executor} for sending {@link AudioConfigurationChangeCallback} to the caller */
private final Executor mExecutor;
@Nullable
private final AudioConfigurationChangeCallback mCallback;
private final Object mLock = new Object();
@GuardedBy("mLock")
private final IntArray mReroutedAppUids = new IntArray();
@Nullable
@GuardedBy("mLock")
private AudioPolicy mAudioPolicy;
@Nullable
@GuardedBy("mLock")
private AudioFormat mCaptureFormat;
@Nullable
@GuardedBy("mLock")
private AudioFormat mInjectionFormat;
@Nullable
@GuardedBy("mLock")
private AudioCapture mAudioCapture;
@Nullable
@GuardedBy("mLock")
private AudioInjection mAudioInjection;
@VisibleForTesting
public VirtualAudioSession(Context context,
@Nullable AudioConfigurationChangeCallback callback, @Nullable Executor executor) {
mContext = context;
mCallback = callback;
mExecutor = executor != null ? executor : context.getMainExecutor();
}
/**
* Begins recording audio emanating from this device.
*
* @return An {@link AudioCapture} containing the recorded audio.
*/
@VisibleForTesting
@NonNull
public AudioCapture startAudioCapture(@NonNull AudioFormat captureFormat) {
Objects.requireNonNull(captureFormat, "captureFormat must not be null");
synchronized (mLock) {
if (mAudioCapture != null) {
throw new IllegalStateException(
"Cannot start capture while another capture is ongoing.");
}
mCaptureFormat = captureFormat;
mAudioCapture = new AudioCapture();
mAudioCapture.startRecording();
return mAudioCapture;
}
}
/**
* Begins injecting audio from a remote device into this device.
*
* @return An {@link AudioInjection} containing the injected audio.
*/
@VisibleForTesting
@NonNull
public AudioInjection startAudioInjection(@NonNull AudioFormat injectionFormat) {
Objects.requireNonNull(injectionFormat, "injectionFormat must not be null");
synchronized (mLock) {
if (mAudioInjection != null) {
throw new IllegalStateException(
"Cannot start injection while injection is already ongoing.");
}
mInjectionFormat = injectionFormat;
mAudioInjection = new AudioInjection();
mAudioInjection.play();
return mAudioInjection;
}
}
/** @hide */
@VisibleForTesting
@Nullable
public AudioCapture getAudioCapture() {
synchronized (mLock) {
return mAudioCapture;
}
}
/** @hide */
@VisibleForTesting
@Nullable
public AudioInjection getAudioInjection() {
synchronized (mLock) {
return mAudioInjection;
}
}
@RequiresPermission(android.Manifest.permission.MODIFY_AUDIO_ROUTING)
@Override
public void onAppsNeedingAudioRoutingChanged(int[] appUids) {
synchronized (mLock) {
if (Arrays.equals(mReroutedAppUids.toArray(), appUids)) {
return;
}
}
releaseAudioStreams();
if (appUids.length == 0) {
return;
}
createAudioStreams(appUids);
}
@RequiresPermission(android.Manifest.permission.MODIFY_AUDIO_ROUTING)
@Override
public void close() {
releaseAudioStreams();
synchronized (mLock) {
mAudioCapture = null;
mAudioInjection = null;
mCaptureFormat = null;
mInjectionFormat = null;
}
}
@RequiresPermission(android.Manifest.permission.MODIFY_AUDIO_ROUTING)
private void createAudioStreams(int[] appUids) {
synchronized (mLock) {
if (mCaptureFormat == null && mInjectionFormat == null) {
throw new IllegalStateException(
"At least one of captureFormat and injectionFormat must be specified.");
}
if (mAudioPolicy != null) {
throw new IllegalStateException(
"Cannot create audio streams while the audio policy is registered. Call "
+ "releaseAudioStreams() first to unregister the previous audio "
+ "policy."
);
}
mReroutedAppUids.clear();
for (int appUid : appUids) {
mReroutedAppUids.add(appUid);
}
AudioMix audioRecordMix = null;
AudioMix audioTrackMix = null;
AudioPolicy.Builder builder = new AudioPolicy.Builder(mContext);
if (mCaptureFormat != null) {
audioRecordMix = createAudioRecordMix(mCaptureFormat, appUids);
builder.addMix(audioRecordMix);
}
if (mInjectionFormat != null) {
audioTrackMix = createAudioTrackMix(mInjectionFormat, appUids);
builder.addMix(audioTrackMix);
}
mAudioPolicy = builder.build();
AudioManager audioManager = mContext.getSystemService(AudioManager.class);
if (audioManager.registerAudioPolicy(mAudioPolicy) == AudioManager.ERROR) {
Log.e(TAG, "Failed to register audio policy!");
}
AudioRecord audioRecord =
audioRecordMix != null ? mAudioPolicy.createAudioRecordSink(audioRecordMix)
: null;
AudioTrack audioTrack =
audioTrackMix != null ? mAudioPolicy.createAudioTrackSource(audioTrackMix)
: null;
if (mAudioCapture != null) {
mAudioCapture.setAudioRecord(audioRecord);
}
if (mAudioInjection != null) {
mAudioInjection.setAudioTrack(audioTrack);
}
}
}
@RequiresPermission(android.Manifest.permission.MODIFY_AUDIO_ROUTING)
private void releaseAudioStreams() {
synchronized (mLock) {
if (mAudioCapture != null) {
mAudioCapture.setAudioRecord(null);
}
if (mAudioInjection != null) {
mAudioInjection.setAudioTrack(null);
}
mReroutedAppUids.clear();
if (mAudioPolicy != null) {
AudioManager audioManager = mContext.getSystemService(AudioManager.class);
audioManager.unregisterAudioPolicy(mAudioPolicy);
mAudioPolicy = null;
Log.i(TAG, "AudioPolicy unregistered");
}
}
}
@Override
public void onPlaybackConfigChanged(List<AudioPlaybackConfiguration> configs) {
if (mCallback != null) {
mExecutor.execute(() -> mCallback.onPlaybackConfigChanged(configs));
}
}
@Override
public void onRecordingConfigChanged(List<AudioRecordingConfiguration> configs) {
if (mCallback != null) {
mExecutor.execute(() -> mCallback.onRecordingConfigChanged(configs));
}
}
/** @hide */
@VisibleForTesting
public IntArray getReroutedAppUids() {
synchronized (mLock) {
return mReroutedAppUids;
}
}
private static AudioMix createAudioRecordMix(@NonNull AudioFormat audioFormat, int[] appUids) {
AudioMixingRule.Builder builder = new AudioMixingRule.Builder();
builder.setTargetMixRole(AudioMixingRule.MIX_ROLE_PLAYERS);
for (int uid : appUids) {
builder.addMixRule(AudioMixingRule.RULE_MATCH_UID, uid);
}
AudioMixingRule audioMixingRule = builder.allowPrivilegedPlaybackCapture(false).build();
AudioMix audioMix =
new AudioMix.Builder(audioMixingRule)
.setFormat(audioFormat)
.setRouteFlags(AudioMix.ROUTE_FLAG_LOOP_BACK)
.build();
return audioMix;
}
private static AudioMix createAudioTrackMix(@NonNull AudioFormat audioFormat, int[] appUids) {
AudioMixingRule.Builder builder = new AudioMixingRule.Builder();
builder.setTargetMixRole(AudioMixingRule.MIX_ROLE_INJECTOR);
for (int uid : appUids) {
builder.addMixRule(AudioMixingRule.RULE_MATCH_UID, uid);
}
AudioMixingRule audioMixingRule = builder.build();
AudioMix audioMix =
new AudioMix.Builder(audioMixingRule)
.setFormat(audioFormat)
.setRouteFlags(AudioMix.ROUTE_FLAG_LOOP_BACK)
.build();
return audioMix;
}
}

182
core/tests/coretests/src/android/companion/virtual/audio/VirtualAudioSessionTest.java Normal file
View File

@@ -0,0 +1,182 @@
/*
* Copyright (C) 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package android.companion.virtual.audio;
import static android.media.AudioFormat.CHANNEL_IN_MONO;
import static android.media.AudioFormat.CHANNEL_OUT_MONO;
import static android.media.AudioFormat.ENCODING_PCM_16BIT;
import static com.google.common.truth.Truth.assertThat;
import static org.mockito.Mockito.verify;
import static org.testng.Assert.assertThrows;
import android.companion.virtual.audio.VirtualAudioDevice.AudioConfigurationChangeCallback;
import android.content.Context;
import android.content.ContextWrapper;
import android.media.AudioFormat;
import android.media.AudioPlaybackConfiguration;
import android.media.AudioRecordingConfiguration;
import android.platform.test.annotations.Presubmit;
import androidx.test.InstrumentationRegistry;
import androidx.test.runner.AndroidJUnit4;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.MockitoAnnotations;
import org.mockito.junit.MockitoJUnit;
import org.mockito.junit.MockitoRule;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@Presubmit
@RunWith(AndroidJUnit4.class)
public class VirtualAudioSessionTest {
@Rule
public final MockitoRule mockito = MockitoJUnit.rule();
@Mock
private AudioConfigurationChangeCallback mCallback;
private static final int APP_UID = 100;
private static final int APP_UID2 = 200;
private static final AudioFormat AUDIO_CAPTURE_FORMAT =
new AudioFormat.Builder()
.setSampleRate(48000)
.setEncoding(ENCODING_PCM_16BIT)
.setChannelMask(CHANNEL_IN_MONO)
.build();
private static final AudioFormat AUDIO_INJECT_FORMAT =
new AudioFormat.Builder()
.setSampleRate(48000)
.setEncoding(ENCODING_PCM_16BIT)
.setChannelMask(CHANNEL_OUT_MONO)
.build();
private Context mContext;
private VirtualAudioSession mVirtualAudioSession;
@Before
public void setUp() {
MockitoAnnotations.initMocks(this);
mContext = Mockito.spy(new ContextWrapper(InstrumentationRegistry.getTargetContext()));
mVirtualAudioSession = new VirtualAudioSession(
mContext, mCallback, /* executor= */ null);
}
@Test
public void startAudioCapture_isSuccessful() {
AudioCapture audioCapture = mVirtualAudioSession.startAudioCapture(AUDIO_CAPTURE_FORMAT);
assertThat(audioCapture).isNotNull();
assertThat(mVirtualAudioSession.getAudioCapture()).isEqualTo(audioCapture);
}
@Test
public void startAudioCapture_audioCaptureAlreadyStarted_throws() {
mVirtualAudioSession.startAudioCapture(AUDIO_CAPTURE_FORMAT);
assertThrows(IllegalStateException.class,
() -> mVirtualAudioSession.startAudioCapture(AUDIO_CAPTURE_FORMAT));
}
@Test
public void startAudioInjection_isSuccessful() {
AudioInjection audioInjection = mVirtualAudioSession.startAudioInjection(
AUDIO_INJECT_FORMAT);
assertThat(audioInjection).isNotNull();
assertThat(mVirtualAudioSession.getAudioInjection()).isEqualTo(audioInjection);
}
@Test
public void startAudioInjection_audioInjectionAlreadyStarted_throws() {
mVirtualAudioSession.startAudioInjection(AUDIO_INJECT_FORMAT);
assertThrows(IllegalStateException.class,
() -> mVirtualAudioSession.startAudioInjection(AUDIO_INJECT_FORMAT));
}
@Test
public void onAppsNeedingAudioRoutingChanged_neverStartAudioCaptureOrInjection_throws() {
int[] uids = new int[]{APP_UID};
assertThrows(IllegalStateException.class,
() -> mVirtualAudioSession.onAppsNeedingAudioRoutingChanged(uids));
}
@Test
public void onAppsNeedingAudioRoutingChanged_cachesReroutedApps() {
mVirtualAudioSession.startAudioCapture(AUDIO_CAPTURE_FORMAT);
mVirtualAudioSession.startAudioInjection(AUDIO_INJECT_FORMAT);
int[] appUids = new int[]{APP_UID};
mVirtualAudioSession.onAppsNeedingAudioRoutingChanged(appUids);
assertThat(Arrays.equals(mVirtualAudioSession.getReroutedAppUids().toArray(),
appUids)).isTrue();
}
@Test
public void onAppsNeedingAudioRoutingChanged_receiveManyTimes_reroutedAppsSizeIsCorrect() {
mVirtualAudioSession.startAudioCapture(AUDIO_CAPTURE_FORMAT);
mVirtualAudioSession.startAudioInjection(AUDIO_INJECT_FORMAT);
int[] appUids = new int[]{APP_UID, APP_UID2};
mVirtualAudioSession.onAppsNeedingAudioRoutingChanged(new int[]{1234});
mVirtualAudioSession.onAppsNeedingAudioRoutingChanged(new int[]{5678});
mVirtualAudioSession.onAppsNeedingAudioRoutingChanged(appUids);
assertThat(Arrays.equals(mVirtualAudioSession.getReroutedAppUids().toArray(),
appUids)).isTrue();
assertThat(mVirtualAudioSession.getReroutedAppUids().size()).isEqualTo(2);
}
@Test
public void close_releasesCaptureAndInjection() {
mVirtualAudioSession.startAudioCapture(AUDIO_CAPTURE_FORMAT);
mVirtualAudioSession.startAudioInjection(AUDIO_INJECT_FORMAT);
mVirtualAudioSession.close();
assertThat(mVirtualAudioSession.getAudioCapture()).isNull();
assertThat(mVirtualAudioSession.getAudioInjection()).isNull();
}
@Test
public void onPlaybackConfigChanged_sendsCallback() {
List<AudioPlaybackConfiguration> configs = new ArrayList<>();
mVirtualAudioSession.onPlaybackConfigChanged(configs);
verify(mCallback).onPlaybackConfigChanged(configs);
}
@Test
public void onRecordingConfigChanged_sendCallback() {
List<AudioRecordingConfiguration> configs = new ArrayList<>();
mVirtualAudioSession.onRecordingConfigChanged(configs);
verify(mCallback).onRecordingConfigChanged(configs);
}
}

25
services/companion/java/com/android/server/companion/virtual/GenericWindowPolicyController.java
View File

@@ -47,9 +47,17 @@ import java.util.function.Consumer;
/**
* A controller to control the policies of the windows that can be displayed on the virtual display.
*/
class GenericWindowPolicyController extends DisplayWindowPolicyController {
public class GenericWindowPolicyController extends DisplayWindowPolicyController {
private static final String TAG = "VirtualDeviceManager";
private static final String TAG = "GenericWindowPolicyController";
/** Interface to listen running applications change on virtual display. */
public interface RunningAppsChangedListener {
/**
* Notifies the running applications change.
*/
void onRunningAppsChanged(ArraySet<Integer> runningUids);
}
private static final ComponentName BLOCKED_APP_STREAMING_COMPONENT =
new ComponentName("android", BlockedAppStreamingActivity.class.getName());
@@ -74,6 +82,9 @@ class GenericWindowPolicyController extends DisplayWindowPolicyController {
@Nullable private final ActivityListener mActivityListener;
private final Handler mHandler = new Handler(Looper.getMainLooper());
@Nullable
private RunningAppsChangedListener mRunningAppsChangedListener;
/**
* Creates a window policy controller that is generic to the different use cases of virtual
* device.
@@ -84,7 +95,7 @@ class GenericWindowPolicyController extends DisplayWindowPolicyController {
* @param activityListener Activity listener to listen for activity changes. The display ID
* is not populated in this callback and is always {@link Display#INVALID_DISPLAY}.
*/
GenericWindowPolicyController(int windowFlags, int systemWindowFlags,
public GenericWindowPolicyController(int windowFlags, int systemWindowFlags,
@NonNull ArraySet<UserHandle> allowedUsers,
@Nullable Set<ComponentName> allowedActivities,
@Nullable Set<ComponentName> blockedActivities,
@@ -98,6 +109,11 @@ class GenericWindowPolicyController extends DisplayWindowPolicyController {
mActivityListener = activityListener;
}
/** Sets listener for running applications change. */
public void setRunningAppsChangedListener(@Nullable RunningAppsChangedListener listener) {
mRunningAppsChangedListener = listener;
}
@Override
public boolean canContainActivities(@NonNull List<ActivityInfo> activities) {
// Can't display all the activities if any of them don't want to be displayed.
@@ -139,6 +155,9 @@ class GenericWindowPolicyController extends DisplayWindowPolicyController {
// Post callback on the main thread so it doesn't block activity launching
mHandler.post(() -> mActivityListener.onDisplayEmpty(Display.INVALID_DISPLAY));
}
if (mRunningAppsChangedListener != null) {
mRunningAppsChangedListener.onRunningAppsChanged(runningUids);
}
}
/**

44
services/companion/java/com/android/server/companion/virtual/VirtualDeviceImpl.java
View File

@@ -23,6 +23,7 @@ import static android.view.WindowManager.LayoutParams.FLAG_SECURE;
import static android.view.WindowManager.LayoutParams.SYSTEM_FLAG_HIDE_NON_SYSTEM_OVERLAY_WINDOWS;
import android.annotation.NonNull;
import android.annotation.RequiresPermission;
import android.app.Activity;
import android.app.ActivityOptions;
import android.app.PendingIntent;
@@ -32,6 +33,7 @@ import android.companion.virtual.IVirtualDevice;
import android.companion.virtual.IVirtualDeviceActivityListener;
import android.companion.virtual.VirtualDeviceManager.ActivityListener;
import android.companion.virtual.VirtualDeviceParams;
import android.companion.virtual.audio.IAudioSessionCallback;
import android.content.ComponentName;
import android.content.Context;
import android.content.Intent;
@@ -60,6 +62,7 @@ import android.window.DisplayWindowPolicyController;
import com.android.internal.annotations.GuardedBy;
import com.android.internal.annotations.VisibleForTesting;
import com.android.internal.app.BlockedAppStreamingActivity;
import com.android.server.companion.virtual.audio.VirtualAudioController;
import java.io.FileDescriptor;
import java.io.PrintWriter;
@@ -78,6 +81,7 @@ final class VirtualDeviceImpl extends IVirtualDevice.Stub
private final PendingTrampolineCallback mPendingTrampolineCallback;
private final int mOwnerUid;
private final InputController mInputController;
private VirtualAudioController mVirtualAudioController;
@VisibleForTesting
final Set<Integer> mVirtualDisplayIds = new ArraySet<>();
private final OnDeviceCloseListener mListener;
@@ -245,6 +249,46 @@ final class VirtualDeviceImpl extends IVirtualDevice.Stub
close();
}
@VisibleForTesting
VirtualAudioController getVirtualAudioControllerForTesting() {
return mVirtualAudioController;
}
@RequiresPermission(android.Manifest.permission.CREATE_VIRTUAL_DEVICE)
@Override // Binder call
public void onAudioSessionStarting(int displayId, IAudioSessionCallback callback) {
mContext.enforceCallingOrSelfPermission(
android.Manifest.permission.CREATE_VIRTUAL_DEVICE,
"Permission required to start audio session");
synchronized (mVirtualDeviceLock) {
if (!mVirtualDisplayIds.contains(displayId)) {
throw new SecurityException(
"Cannot start audio session for a display not associated with this virtual "
+ "device");
}
if (mVirtualAudioController == null) {
mVirtualAudioController = new VirtualAudioController(mContext);
GenericWindowPolicyController gwpc = mWindowPolicyControllers.get(displayId);
mVirtualAudioController.startListening(gwpc, callback);
}
}
}
@RequiresPermission(android.Manifest.permission.CREATE_VIRTUAL_DEVICE)
@Override // Binder call
public void onAudioSessionEnded() {
mContext.enforceCallingOrSelfPermission(
android.Manifest.permission.CREATE_VIRTUAL_DEVICE,
"Permission required to stop audio session");
synchronized (mVirtualDeviceLock) {
if (mVirtualAudioController != null) {
mVirtualAudioController.stopListening();
mVirtualAudioController = null;
}
}
}
@Override // Binder call
public void createVirtualKeyboard(
int displayId,

65
services/companion/java/com/android/server/companion/virtual/audio/AudioPlaybackDetector.java Normal file
View File

@@ -0,0 +1,65 @@
/*
* Copyright (C) 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.server.companion.virtual.audio;
import android.annotation.NonNull;
import android.content.Context;
import android.media.AudioManager;
import android.media.AudioPlaybackConfiguration;
import java.util.List;
/**
* Wrapper class for other classes to listen {@link #onPlaybackConfigChanged(List)} by implementing
* {@link AudioPlaybackCallback} instead of inheriting the
* {@link AudioManager.AudioPlaybackCallback}.
*/
final class AudioPlaybackDetector extends AudioManager.AudioPlaybackCallback {
/**
* Interface to listen {@link #onPlaybackConfigChanged(List)} from
* {@link AudioManager.AudioPlaybackCallback}.
*/
interface AudioPlaybackCallback {
void onPlaybackConfigChanged(List<AudioPlaybackConfiguration> configs);
}
private final AudioManager mAudioManager;
private AudioPlaybackCallback mAudioPlaybackCallback;
AudioPlaybackDetector(Context context) {
mAudioManager = context.getSystemService(AudioManager.class);
}
void register(@NonNull AudioPlaybackCallback callback) {
mAudioPlaybackCallback = callback;
mAudioManager.registerAudioPlaybackCallback(/* cb= */ this, /* handler= */ null);
}
void unregister() {
mAudioPlaybackCallback = null;
mAudioManager.unregisterAudioPlaybackCallback(/* cb= */ this);
}
@Override
public void onPlaybackConfigChanged(List<AudioPlaybackConfiguration> configs) {
super.onPlaybackConfigChanged(configs);
if (mAudioPlaybackCallback != null) {
mAudioPlaybackCallback.onPlaybackConfigChanged(configs);
}
}
}

65
services/companion/java/com/android/server/companion/virtual/audio/AudioRecordingDetector.java Normal file
View File

@@ -0,0 +1,65 @@
/*
* Copyright (C) 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.server.companion.virtual.audio;
import android.annotation.NonNull;
import android.content.Context;
import android.media.AudioManager;
import android.media.AudioRecordingConfiguration;
import java.util.List;
/**
* Wrapper class for other classes to listen {@link #onRecordingConfigChanged(List)} by implementing
* {@link AudioRecordingCallback} instead of inheriting the
* {@link AudioManager.AudioRecordingCallback}.
*/
final class AudioRecordingDetector extends AudioManager.AudioRecordingCallback {
/**
* Interface to listen {@link #onRecordingConfigChanged(List)} from
* {@link AudioManager.AudioRecordingCallback}.
*/
interface AudioRecordingCallback {
void onRecordingConfigChanged(List<AudioRecordingConfiguration> configs);
}
private final AudioManager mAudioManager;
private AudioRecordingCallback mAudioRecordingCallback;
AudioRecordingDetector(Context context) {
mAudioManager = context.getSystemService(AudioManager.class);
}
void register(@NonNull AudioRecordingCallback callback) {
mAudioRecordingCallback = callback;
mAudioManager.registerAudioRecordingCallback(/* cb= */ this, /* handler= */ null);
}
void unregister() {
mAudioRecordingCallback = null;
mAudioManager.unregisterAudioRecordingCallback(/* cb= */ this);
}
@Override
public void onRecordingConfigChanged(List<AudioRecordingConfiguration> configs) {
super.onRecordingConfigChanged(configs);
if (mAudioRecordingCallback != null) {
mAudioRecordingCallback.onRecordingConfigChanged(configs);
}
}
}

294
services/companion/java/com/android/server/companion/virtual/audio/VirtualAudioController.java Normal file
View File

@@ -0,0 +1,294 @@
/*
* Copyright (C) 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.server.companion.virtual.audio;
import static android.media.AudioPlaybackConfiguration.PLAYER_STATE_STARTED;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.annotation.RequiresPermission;
import android.companion.virtual.audio.IAudioSessionCallback;
import android.content.Context;
import android.media.AudioManager;
import android.media.AudioPlaybackConfiguration;
import android.media.AudioRecordingConfiguration;
import android.os.Handler;
import android.os.Looper;
import android.os.RemoteException;
import android.util.ArraySet;
import android.util.Slog;
import com.android.internal.annotations.GuardedBy;
import com.android.internal.annotations.VisibleForTesting;
import com.android.server.companion.virtual.GenericWindowPolicyController;
import com.android.server.companion.virtual.GenericWindowPolicyController.RunningAppsChangedListener;
import com.android.server.companion.virtual.audio.AudioPlaybackDetector.AudioPlaybackCallback;
import com.android.server.companion.virtual.audio.AudioRecordingDetector.AudioRecordingCallback;
import java.util.ArrayList;
import java.util.List;
/**
* Manages audio streams associated with a {@link VirtualAudioDevice}. Responsible for monitoring
* running applications and playback configuration changes in order to correctly re-route audio and
* then notify clients of these changes.
*/
public final class VirtualAudioController implements AudioPlaybackCallback,
AudioRecordingCallback, RunningAppsChangedListener {
private static final String TAG = "VirtualAudioController";
private static final int UPDATE_REROUTING_APPS_DELAY_MS = 2000;
private final Context mContext;
private final Handler mHandler = new Handler(Looper.getMainLooper());
private final Runnable mUpdateAudioRoutingRunnable = this::notifyAppsNeedingAudioRoutingChanged;
private final AudioPlaybackDetector mAudioPlaybackDetector;
private final AudioRecordingDetector mAudioRecordingDetector;
private final Object mLock = new Object();
@GuardedBy("mLock")
private final ArraySet<Integer> mRunningAppUids = new ArraySet<>();
@GuardedBy("mLock")
private ArraySet<Integer> mPlayingAppUids = new ArraySet<>();
private GenericWindowPolicyController mGenericWindowPolicyController;
private final Object mCallbackLock = new Object();
@Nullable
@GuardedBy("mCallbackLock")
private IAudioSessionCallback mCallback;
public VirtualAudioController(Context context) {
mContext = context;
mAudioPlaybackDetector = new AudioPlaybackDetector(context);
mAudioRecordingDetector = new AudioRecordingDetector(context);
}
/**
* Starts to listen to running applications and audio configuration changes on virtual display
* for audio capture and injection.
*/
public void startListening(
@NonNull GenericWindowPolicyController genericWindowPolicyController,
@Nullable IAudioSessionCallback callback) {
mGenericWindowPolicyController = genericWindowPolicyController;
mGenericWindowPolicyController.setRunningAppsChangedListener(/* listener= */ this);
synchronized (mCallbackLock) {
mCallback = callback;
}
synchronized (mLock) {
mRunningAppUids.clear();
mPlayingAppUids.clear();
}
mAudioPlaybackDetector.register(/* callback= */ this);
mAudioRecordingDetector.register(/* callback= */ this);
}
/**
* Stops listening to running applications and audio configuration changes on virtual display
* for audio capture and injection.
*/
public void stopListening() {
if (mHandler.hasCallbacks(mUpdateAudioRoutingRunnable)) {
mHandler.removeCallbacks(mUpdateAudioRoutingRunnable);
}
mAudioPlaybackDetector.unregister();
mAudioRecordingDetector.unregister();
if (mGenericWindowPolicyController != null) {
mGenericWindowPolicyController.setRunningAppsChangedListener(/* listener= */ null);
mGenericWindowPolicyController = null;
}
synchronized (mCallbackLock) {
mCallback = null;
}
}
@Override
public void onRunningAppsChanged(ArraySet<Integer> runningUids) {
synchronized (mLock) {
if (mRunningAppUids.equals(runningUids)) {
// Ignore no-op events.
return;
}
mRunningAppUids.clear();
mRunningAppUids.addAll(runningUids);
ArraySet<Integer> oldPlayingAppUids = mPlayingAppUids;
// Update the list of playing apps after caching the old list, and before checking if
// the list of playing apps is empty. This is a subset of the running apps, so we need
// to update this here as well.
AudioManager audioManager = mContext.getSystemService(AudioManager.class);
List<AudioPlaybackConfiguration> configs =
audioManager.getActivePlaybackConfigurations();
mPlayingAppUids = findPlayingAppUids(configs, mRunningAppUids);
// Do not change rerouted applications while any application is playing, or the sound
// will be leaked from phone during the transition. Delay the change until we detect
// there is no application is playing in onPlaybackConfigChanged().
if (!mPlayingAppUids.isEmpty()) {
Slog.i(TAG, "Audio is playing, do not change rerouted apps");
return;
}
// An application previously playing audio was removed from the display.
if (!oldPlayingAppUids.isEmpty()) {
// Delay changing the rerouted application when the last application playing audio
// was removed from virtual device, or the sound will be leaked from phone side
// during the transition.
Slog.i(TAG, "The last playing app removed, delay change rerouted apps");
if (mHandler.hasCallbacks(mUpdateAudioRoutingRunnable)) {
mHandler.removeCallbacks(mUpdateAudioRoutingRunnable);
}
mHandler.postDelayed(mUpdateAudioRoutingRunnable, UPDATE_REROUTING_APPS_DELAY_MS);
return;
}
}
// Normal case with no application playing, just update routing.
notifyAppsNeedingAudioRoutingChanged();
}
@Override
public void onPlaybackConfigChanged(List<AudioPlaybackConfiguration> configs) {
updatePlayingApplications(configs);
List<AudioPlaybackConfiguration> audioPlaybackConfigurations;
synchronized (mLock) {
// Filter configurations of applications running on virtual device.
audioPlaybackConfigurations = findPlaybackConfigurations(configs, mRunningAppUids);
}
synchronized (mCallbackLock) {
if (mCallback != null) {
try {
mCallback.onPlaybackConfigChanged(audioPlaybackConfigurations);
} catch (RemoteException e) {
Slog.e(TAG, "RemoteException when calling onPlaybackConfigChanged", e);
}
}
}
}
@RequiresPermission(android.Manifest.permission.MODIFY_AUDIO_ROUTING)
@Override
public void onRecordingConfigChanged(List<AudioRecordingConfiguration> configs) {
List<AudioRecordingConfiguration> audioRecordingConfigurations;
synchronized (mLock) {
// Filter configurations of applications running on virtual device.
audioRecordingConfigurations = findRecordingConfigurations(configs, mRunningAppUids);
}
synchronized (mCallbackLock) {
if (mCallback != null) {
try {
mCallback.onRecordingConfigChanged(audioRecordingConfigurations);
} catch (RemoteException e) {
Slog.e(TAG, "RemoteException when calling onRecordingConfigChanged", e);
}
}
}
}
private void updatePlayingApplications(List<AudioPlaybackConfiguration> configs) {
synchronized (mLock) {
ArraySet<Integer> playingAppUids = findPlayingAppUids(configs, mRunningAppUids);
if (mPlayingAppUids.equals(playingAppUids)) {
return;
}
mPlayingAppUids = playingAppUids;
}
// Updated rerouted apps, even if the app is already playing. It originally should be done
// when onRunningAppsChanged() is called, but we don't want to interrupt the audio
// streaming and cause the sound leak from phone when it's playing, so delay until here.
notifyAppsNeedingAudioRoutingChanged();
}
private void notifyAppsNeedingAudioRoutingChanged() {
if (mHandler.hasCallbacks(mUpdateAudioRoutingRunnable)) {
mHandler.removeCallbacks(mUpdateAudioRoutingRunnable);
}
int[] runningUids;
synchronized (mLock) {
runningUids = new int[mRunningAppUids.size()];
for (int i = 0; i < mRunningAppUids.size(); i++) {
runningUids[i] = mRunningAppUids.valueAt(i);
}
}
synchronized (mCallbackLock) {
if (mCallback != null) {
try {
mCallback.onAppsNeedingAudioRoutingChanged(runningUids);
} catch (RemoteException e) {
Slog.e(TAG, "RemoteException when calling updateReroutingApps", e);
}
}
}
}
/**
* Finds uid of playing applications from the given running applications.
*
* @param configs a list of playback configs which get from {@link AudioManager}
*/
private static ArraySet<Integer> findPlayingAppUids(List<AudioPlaybackConfiguration> configs,
ArraySet<Integer> runningAppUids) {
ArraySet<Integer> playingAppUids = new ArraySet<>();
for (AudioPlaybackConfiguration config : configs) {
if (runningAppUids.contains(config.getClientUid())
&& config.getPlayerState() == PLAYER_STATE_STARTED) {
playingAppUids.add(config.getClientUid());
}
}
return playingAppUids;
}
/** Finds a list of {@link AudioPlaybackConfiguration} for the given running applications. */
private static List<AudioPlaybackConfiguration> findPlaybackConfigurations(
List<AudioPlaybackConfiguration> configs,
ArraySet<Integer> runningAppUids) {
List<AudioPlaybackConfiguration> runningConfigs = new ArrayList<>();
for (AudioPlaybackConfiguration config : configs) {
if (runningAppUids.contains(config.getClientUid())) {
runningConfigs.add(config);
}
}
return runningConfigs;
}
/** Finds a list of {@link AudioRecordingConfiguration} for the given running applications. */
@RequiresPermission(android.Manifest.permission.MODIFY_AUDIO_ROUTING)
private static List<AudioRecordingConfiguration> findRecordingConfigurations(
List<AudioRecordingConfiguration> configs, ArraySet<Integer> runningAppUids) {
List<AudioRecordingConfiguration> runningConfigs = new ArrayList<>();
for (AudioRecordingConfiguration config : configs) {
if (runningAppUids.contains(config.getClientUid())) {
runningConfigs.add(config);
}
}
return runningConfigs;
}
@VisibleForTesting
boolean hasPendingRunnable() {
return mHandler.hasCallbacks(mUpdateAudioRoutingRunnable);
}
@VisibleForTesting
void addPlayingAppsForTesting(int appUid) {
synchronized (mLock) {
mPlayingAppUids.add(appUid);
}
}
}

45
services/tests/servicestests/src/com/android/server/companion/virtual/VirtualDeviceManagerServiceTest.java
View File

@@ -16,6 +16,7 @@
package com.android.server.companion.virtual;
import static com.google.common.truth.Truth.assertThat;
import static com.google.common.truth.Truth.assertWithMessage;
import static org.mockito.ArgumentMatchers.any;
@@ -39,6 +40,7 @@ import android.app.admin.DevicePolicyManager;
import android.companion.AssociationInfo;
import android.companion.virtual.IVirtualDeviceActivityListener;
import android.companion.virtual.VirtualDeviceParams;
import android.companion.virtual.audio.IAudioSessionCallback;
import android.content.Context;
import android.content.ContextWrapper;
import android.graphics.Point;
@@ -111,6 +113,8 @@ public class VirtualDeviceManagerServiceTest {
@Mock
IThermalService mIThermalServiceMock;
private PowerManager mPowerManager;
@Mock
private IAudioSessionCallback mCallback;
@Before
public void setUp() {
@@ -249,6 +253,12 @@ public class VirtualDeviceManagerServiceTest {
VENDOR_ID, PRODUCT_ID, BINDER, new Point(WIDTH, HEIGHT)));
}
@Test
public void onAudioSessionStarting_noDisplay_failsSecurityException() {
assertThrows(SecurityException.class,
() -> mDeviceImpl.onAudioSessionStarting(DISPLAY_ID, mCallback));
}
@Test
public void createVirtualKeyboard_noPermission_failsSecurityException() {
mDeviceImpl.mVirtualDisplayIds.add(DISPLAY_ID);
@@ -282,6 +292,22 @@ public class VirtualDeviceManagerServiceTest {
VENDOR_ID, PRODUCT_ID, BINDER, new Point(WIDTH, HEIGHT)));
}
@Test
public void onAudioSessionStarting_noPermission_failsSecurityException() {
mDeviceImpl.mVirtualDisplayIds.add(DISPLAY_ID);
doCallRealMethod().when(mContext).enforceCallingOrSelfPermission(
eq(Manifest.permission.CREATE_VIRTUAL_DEVICE), anyString());
assertThrows(SecurityException.class,
() -> mDeviceImpl.onAudioSessionStarting(DISPLAY_ID, mCallback));
}
@Test
public void onAudioSessionEnded_noPermission_failsSecurityException() {
doCallRealMethod().when(mContext).enforceCallingOrSelfPermission(
eq(Manifest.permission.CREATE_VIRTUAL_DEVICE), anyString());
assertThrows(SecurityException.class, () -> mDeviceImpl.onAudioSessionEnded());
}
@Test
public void createVirtualKeyboard_hasDisplay_obtainFileDescriptor() {
mDeviceImpl.mVirtualDisplayIds.add(DISPLAY_ID);
@@ -315,6 +341,25 @@ public class VirtualDeviceManagerServiceTest {
eq(PRODUCT_ID), anyString(), eq(HEIGHT), eq(WIDTH));
}
@Test
public void onAudioSessionStarting_hasVirtualAudioController() {
mDeviceImpl.onVirtualDisplayCreatedLocked(DISPLAY_ID);
mDeviceImpl.onAudioSessionStarting(DISPLAY_ID, mCallback);
assertThat(mDeviceImpl.getVirtualAudioControllerForTesting()).isNotNull();
}
@Test
public void onAudioSessionEnded_noVirtualAudioController() {
mDeviceImpl.onVirtualDisplayCreatedLocked(DISPLAY_ID);
mDeviceImpl.onAudioSessionStarting(DISPLAY_ID, mCallback);
mDeviceImpl.onAudioSessionEnded();
assertThat(mDeviceImpl.getVirtualAudioControllerForTesting()).isNull();
}
@Test
public void sendKeyEvent_noFd() {
assertThrows(

202
services/tests/servicestests/src/com/android/server/companion/virtual/audio/VirtualAudioControllerTest.java Normal file
View File

@@ -0,0 +1,202 @@
/*
* Copyright (C) 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.server.companion.virtual.audio;
import static android.media.AudioAttributes.FLAG_SECURE;
import static android.media.AudioPlaybackConfiguration.PLAYER_STATE_STARTED;
import static android.view.WindowManager.LayoutParams.SYSTEM_FLAG_HIDE_NON_SYSTEM_OVERLAY_WINDOWS;
import static com.google.common.truth.Truth.assertThat;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import android.companion.virtual.audio.IAudioSessionCallback;
import android.content.Context;
import android.content.ContextWrapper;
import android.media.AudioPlaybackConfiguration;
import android.media.AudioRecordingConfiguration;
import android.media.MediaRecorder;
import android.media.PlayerBase;
import android.os.Parcel;
import android.os.RemoteException;
import android.util.ArraySet;
import androidx.test.InstrumentationRegistry;
import androidx.test.runner.AndroidJUnit4;
import com.android.server.companion.virtual.GenericWindowPolicyController;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.MockitoAnnotations;
import java.util.ArrayList;
import java.util.List;
@RunWith(AndroidJUnit4.class)
public class VirtualAudioControllerTest {
private static final int APP1_UID = 100;
private static final int APP2_UID = 200;
private Context mContext;
private VirtualAudioController mVirtualAudioController;
private GenericWindowPolicyController mGenericWindowPolicyController;
@Mock IAudioSessionCallback mCallback;
@Before
public void setUp() {
MockitoAnnotations.initMocks(this);
mContext = Mockito.spy(new ContextWrapper(InstrumentationRegistry.getTargetContext()));
mVirtualAudioController = new VirtualAudioController(mContext);
mGenericWindowPolicyController = new GenericWindowPolicyController(
FLAG_SECURE,
SYSTEM_FLAG_HIDE_NON_SYSTEM_OVERLAY_WINDOWS,
/* allowedUsers= */ new ArraySet<>(),
/* allowedActivities= */ new ArraySet<>(),
/* blockedActivities= */ new ArraySet<>(),
/* activityListener= */null,
/* activityBlockedCallback= */ null);
}
@Test
public void startListening_receivesCallback() throws RemoteException {
ArraySet<Integer> runningUids = new ArraySet<>();
runningUids.add(APP1_UID);
int[] appUids = new int[] {APP1_UID};
mVirtualAudioController.startListening(mGenericWindowPolicyController, mCallback);
mGenericWindowPolicyController.onRunningAppsChanged(runningUids);
verify(mCallback).onAppsNeedingAudioRoutingChanged(appUids);
}
@Test
public void stopListening_removesCallback() throws RemoteException {
ArraySet<Integer> runningUids = new ArraySet<>();
runningUids.add(APP1_UID);
int[] appUids = new int[] {APP1_UID};
mVirtualAudioController.startListening(mGenericWindowPolicyController, mCallback);
mVirtualAudioController.stopListening();
mGenericWindowPolicyController.onRunningAppsChanged(runningUids);
verify(mCallback, never()).onAppsNeedingAudioRoutingChanged(appUids);
}
@Test
public void onRunningAppsChanged_notifiesAudioRoutingModified() throws RemoteException {
mVirtualAudioController.startListening(mGenericWindowPolicyController, mCallback);
ArraySet<Integer> runningUids = new ArraySet<>();
runningUids.add(APP1_UID);
mVirtualAudioController.onRunningAppsChanged(runningUids);
int[] appUids = new int[] {APP1_UID};
verify(mCallback).onAppsNeedingAudioRoutingChanged(appUids);
}
@Test
public void onRunningAppsChanged_audioIsPlaying_doesNothing() throws RemoteException {
mVirtualAudioController.startListening(mGenericWindowPolicyController, mCallback);
mVirtualAudioController.addPlayingAppsForTesting(APP2_UID);
ArraySet<Integer> runningUids = new ArraySet<>();
runningUids.add(APP1_UID);
mVirtualAudioController.onRunningAppsChanged(runningUids);
int[] appUids = new int[]{APP1_UID};
verify(mCallback, never()).onAppsNeedingAudioRoutingChanged(appUids);
}
@Test
public void onRunningAppsChanged_lastPlayingAppRemoved_delaysReroutingAudio() {
ArraySet<Integer> runningUids = new ArraySet<>();
runningUids.add(APP1_UID);
runningUids.add(APP2_UID);
mVirtualAudioController.onRunningAppsChanged(runningUids);
mVirtualAudioController.addPlayingAppsForTesting(APP2_UID);
ArraySet<Integer> appUids = new ArraySet<>();
appUids.add(APP1_UID);
mVirtualAudioController.onRunningAppsChanged(appUids);
assertThat(mVirtualAudioController.hasPendingRunnable()).isTrue();
}
@Test
public void onPlaybackConfigChanged_sendsCallback() throws RemoteException {
mVirtualAudioController.startListening(mGenericWindowPolicyController, mCallback);
ArraySet<Integer> runningUids = new ArraySet<>();
runningUids.add(APP1_UID);
mVirtualAudioController.onRunningAppsChanged(runningUids);
List<AudioPlaybackConfiguration> configs = createPlaybackConfigurations(runningUids);
mVirtualAudioController.onPlaybackConfigChanged(configs);
verify(mCallback).onPlaybackConfigChanged(configs);
}
@Test
public void onRecordingConfigChanged_sendsCallback() throws RemoteException {
mVirtualAudioController.startListening(mGenericWindowPolicyController, mCallback);
ArraySet<Integer> runningUids = new ArraySet<>();
runningUids.add(APP1_UID);
mVirtualAudioController.onRunningAppsChanged(runningUids);
List<AudioRecordingConfiguration> configs = createRecordingConfigurations(runningUids);
mVirtualAudioController.onRecordingConfigChanged(configs);
verify(mCallback).onRecordingConfigChanged(configs);
}
private List<AudioPlaybackConfiguration> createPlaybackConfigurations(
ArraySet<Integer> appUids) {
List<AudioPlaybackConfiguration> configs = new ArrayList<>();
for (int appUid : appUids) {
PlayerBase.PlayerIdCard playerIdCard =
PlayerBase.PlayerIdCard.CREATOR.createFromParcel(Parcel.obtain());
AudioPlaybackConfiguration audioPlaybackConfiguration =
new AudioPlaybackConfiguration(
playerIdCard, /* piid= */ 1000, appUid, /* pid= */ 1000);
audioPlaybackConfiguration.handleStateEvent(PLAYER_STATE_STARTED, /* deviceId= */1);
configs.add(audioPlaybackConfiguration);
}
return configs;
}
private List<AudioRecordingConfiguration> createRecordingConfigurations(
ArraySet<Integer> appUids) {
List<AudioRecordingConfiguration> configs = new ArrayList<>();
for (int appUid : appUids) {
AudioRecordingConfiguration audioRecordingConfiguration =
new AudioRecordingConfiguration(
/* uid= */ appUid,
/* session= */ 1000,
MediaRecorder.AudioSource.MIC,
/* clientFormat= */ null,
/* devFormat= */ null,
/* patchHandle= */ 1000,
"com.android.example");
configs.add(audioRecordingConfiguration);
}
return configs;
}
}