diff --git a/core/api/system-current.txt b/core/api/system-current.txt index 1536728502c02..b077cd7b846cf 100644 --- a/core/api/system-current.txt +++ b/core/api/system-current.txt @@ -2750,6 +2750,7 @@ package android.companion.virtual { method public void addActivityListener(@NonNull android.companion.virtual.VirtualDeviceManager.ActivityListener); method public void addActivityListener(@NonNull android.companion.virtual.VirtualDeviceManager.ActivityListener, @NonNull java.util.concurrent.Executor); method @RequiresPermission(android.Manifest.permission.CREATE_VIRTUAL_DEVICE) public void close(); + method @NonNull @RequiresPermission(android.Manifest.permission.CREATE_VIRTUAL_DEVICE) public android.companion.virtual.audio.VirtualAudioDevice createVirtualAudioDevice(@NonNull android.hardware.display.VirtualDisplay, @Nullable java.util.concurrent.Executor, @Nullable android.companion.virtual.audio.VirtualAudioDevice.AudioConfigurationChangeCallback); method @Nullable public android.hardware.display.VirtualDisplay createVirtualDisplay(int, int, int, @Nullable android.view.Surface, int, @Nullable android.os.Handler, @Nullable android.hardware.display.VirtualDisplay.Callback); method @NonNull @RequiresPermission(android.Manifest.permission.CREATE_VIRTUAL_DEVICE) public android.hardware.input.VirtualKeyboard createVirtualKeyboard(@NonNull android.hardware.display.VirtualDisplay, @NonNull String, int, int); method @NonNull @RequiresPermission(android.Manifest.permission.CREATE_VIRTUAL_DEVICE) public android.hardware.input.VirtualMouse createVirtualMouse(@NonNull android.hardware.display.VirtualDisplay, @NonNull String, int, int); @@ -2782,6 +2783,39 @@ package android.companion.virtual { } +package android.companion.virtual.audio { + + public final class AudioCapture { + ctor public AudioCapture(); + method public int getRecordingState(); + method public int read(@NonNull java.nio.ByteBuffer, int); + method public void startRecording(); + method public void stop(); + } + + public final class AudioInjection { + ctor public AudioInjection(); + method public int getPlayState(); + method public void play(); + method public void stop(); + method public int write(@NonNull java.nio.ByteBuffer, int, int); + } + + public final class VirtualAudioDevice implements java.io.Closeable { + method @RequiresPermission(android.Manifest.permission.MODIFY_AUDIO_ROUTING) public void close(); + method @Nullable public android.companion.virtual.audio.AudioCapture getAudioCapture(); + method @Nullable public android.companion.virtual.audio.AudioInjection getAudioInjection(); + method @NonNull @RequiresPermission(android.Manifest.permission.MODIFY_AUDIO_ROUTING) public android.companion.virtual.audio.AudioCapture startAudioCapture(@NonNull android.media.AudioFormat); + method @NonNull @RequiresPermission(android.Manifest.permission.MODIFY_AUDIO_ROUTING) public android.companion.virtual.audio.AudioInjection startAudioInjection(@NonNull android.media.AudioFormat); + } + + public static interface VirtualAudioDevice.AudioConfigurationChangeCallback { + method public void onPlaybackConfigChanged(@NonNull java.util.List); + method public void onRecordingConfigChanged(@NonNull java.util.List); + } + +} + package android.content { public class ApexEnvironment { diff --git a/core/java/android/companion/virtual/IVirtualDevice.aidl b/core/java/android/companion/virtual/IVirtualDevice.aidl index 8fc24fd2d7f22..e2859998efd4c 100644 --- a/core/java/android/companion/virtual/IVirtualDevice.aidl +++ b/core/java/android/companion/virtual/IVirtualDevice.aidl @@ -17,6 +17,7 @@ package android.companion.virtual; import android.app.PendingIntent; +import android.companion.virtual.audio.IAudioSessionCallback; import android.graphics.Point; import android.graphics.PointF; import android.hardware.input.VirtualKeyEvent; @@ -45,6 +46,15 @@ interface IVirtualDevice { */ void close(); + /** + * Notifies of an audio session being started. + */ + void onAudioSessionStarting( + int displayId, + IAudioSessionCallback callback); + + void onAudioSessionEnded(); + void createVirtualKeyboard( int displayId, String inputDeviceName, diff --git a/core/java/android/companion/virtual/VirtualDeviceManager.java b/core/java/android/companion/virtual/VirtualDeviceManager.java index 1dbe04cfda8e7..fc0efb1340858 100644 --- a/core/java/android/companion/virtual/VirtualDeviceManager.java +++ b/core/java/android/companion/virtual/VirtualDeviceManager.java @@ -25,6 +25,8 @@ import android.annotation.SystemService; import android.app.Activity; import android.app.PendingIntent; import android.companion.AssociationInfo; +import android.companion.virtual.audio.VirtualAudioDevice; +import android.companion.virtual.audio.VirtualAudioDevice.AudioConfigurationChangeCallback; import android.content.ComponentName; import android.content.Context; import android.graphics.Point; @@ -338,6 +340,30 @@ public final class VirtualDeviceManager { } } + /** + * Creates a VirtualAudioDevice, capable of recording audio emanating from this device, + * or injecting audio from another device. + * + *

Note: This object does not support capturing privileged playback, such as voice call + * audio. + * + * @param display The target virtual display to capture from and inject into. + * @param executor The {@link Executor} object for the thread on which to execute + * the callback. If null, the {@link Executor} associated with + * the main {@link Looper} will be used. + * @param callback Interface to be notified when playback or recording configuration of + * applications running on virtual display is changed. + * @return A {@link VirtualAudioDevice} instance. + */ + @RequiresPermission(android.Manifest.permission.CREATE_VIRTUAL_DEVICE) + @NonNull + public VirtualAudioDevice createVirtualAudioDevice( + @NonNull VirtualDisplay display, + @Nullable Executor executor, + @Nullable AudioConfigurationChangeCallback callback) { + return new VirtualAudioDevice(mContext, mVirtualDevice, display, executor, callback); + } + /** * Sets the visibility of the pointer icon for this VirtualDevice's associated displays. * diff --git a/core/java/android/companion/virtual/audio/AudioCapture.java b/core/java/android/companion/virtual/audio/AudioCapture.java new file mode 100644 index 0000000000000..ebe17dba5775e --- /dev/null +++ b/core/java/android/companion/virtual/audio/AudioCapture.java @@ -0,0 +1,124 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.companion.virtual.audio; + +import static android.media.AudioRecord.RECORDSTATE_RECORDING; +import static android.media.AudioRecord.RECORDSTATE_STOPPED; + +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.annotation.SuppressLint; +import android.annotation.SystemApi; +import android.media.AudioRecord; +import android.util.Log; + +import com.android.internal.annotations.GuardedBy; + +import java.nio.ByteBuffer; + +/** + * Wrapper around {@link AudioRecord} that allows for the underlying {@link AudioRecord} to + * be swapped out while recording is ongoing. + * + * @hide + */ +// The stop() actually doesn't release resources, so should not force implementing Closeable. +@SuppressLint("NotCloseable") +@SystemApi +public final class AudioCapture { + private static final String TAG = "AudioCapture"; + + private final Object mLock = new Object(); + + @GuardedBy("mLock") + @Nullable + private AudioRecord mAudioRecord; + + @GuardedBy("mLock") + private int mRecordingState = RECORDSTATE_STOPPED; + + /** + * Sets the {@link AudioRecord} to handle audio capturing. + * Callers may call this multiple times with different audio records to change + * the underlying {@link AudioRecord} without stopping and re-starting recording. + * + * @param audioRecord The underlying {@link AudioRecord} to use for capture, + * or null if no audio (i.e. silence) should be captured while still keeping the + * record in a recording state. + */ + void setAudioRecord(@Nullable AudioRecord audioRecord) { + Log.d(TAG, "set AudioRecord with " + audioRecord); + synchronized (mLock) { + // Release old reference. + if (mAudioRecord != null) { + mAudioRecord.release(); + } + // Sync recording state for new reference. + if (audioRecord != null) { + if (mRecordingState == RECORDSTATE_RECORDING + && audioRecord.getRecordingState() != RECORDSTATE_RECORDING) { + audioRecord.startRecording(); + } + if (mRecordingState == RECORDSTATE_STOPPED + && audioRecord.getRecordingState() != RECORDSTATE_STOPPED) { + audioRecord.stop(); + } + } + mAudioRecord = audioRecord; + } + } + + /** See {@link AudioRecord#read(ByteBuffer, int)}. */ + public int read(@NonNull ByteBuffer audioBuffer, int sizeInBytes) { + final int sizeRead; + synchronized (mLock) { + if (mAudioRecord != null) { + sizeRead = mAudioRecord.read(audioBuffer, sizeInBytes); + } else { + sizeRead = 0; + } + } + return sizeRead; + } + + /** See {@link AudioRecord#startRecording()}. */ + public void startRecording() { + synchronized (mLock) { + mRecordingState = RECORDSTATE_RECORDING; + if (mAudioRecord != null && mAudioRecord.getRecordingState() != RECORDSTATE_RECORDING) { + mAudioRecord.startRecording(); + } + } + } + + /** See {@link AudioRecord#stop()}. */ + public void stop() { + synchronized (mLock) { + mRecordingState = RECORDSTATE_STOPPED; + if (mAudioRecord != null && mAudioRecord.getRecordingState() != RECORDSTATE_STOPPED) { + mAudioRecord.stop(); + } + } + } + + /** See {@link AudioRecord#getRecordingState()}. */ + public int getRecordingState() { + synchronized (mLock) { + return mRecordingState; + } + } +} diff --git a/core/java/android/companion/virtual/audio/AudioInjection.java b/core/java/android/companion/virtual/audio/AudioInjection.java new file mode 100644 index 0000000000000..0519075038472 --- /dev/null +++ b/core/java/android/companion/virtual/audio/AudioInjection.java @@ -0,0 +1,124 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.companion.virtual.audio; + +import static android.media.AudioTrack.PLAYSTATE_PLAYING; +import static android.media.AudioTrack.PLAYSTATE_STOPPED; + +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.annotation.SuppressLint; +import android.annotation.SystemApi; +import android.media.AudioTrack; +import android.util.Log; + +import com.android.internal.annotations.GuardedBy; + +import java.nio.ByteBuffer; + +/** + * Wrapper around {@link AudioTrack} that allows for the underlying {@link AudioTrack} to + * be swapped out while playout is ongoing. + * + * @hide + */ +// The stop() actually doesn't release resources, so should not force implementing Closeable. +@SuppressLint("NotCloseable") +@SystemApi +public final class AudioInjection { + private static final String TAG = "AudioInjection"; + + private final Object mLock = new Object(); + + @GuardedBy("mLock") + @Nullable + private AudioTrack mAudioTrack; + + @GuardedBy("mLock") + private int mPlayState = PLAYSTATE_STOPPED; + + /** + * Sets the {@link AudioTrack} to handle audio injection. + * Callers may call this multiple times with different audio tracks to change + * the underlying {@link AudioTrack} without stopping and re-starting injection. + * + * @param audioTrack The underlying {@link AudioTrack} to use for injection, + * or null if no audio (i.e. silence) should be injected while still keeping the + * record in a playing state. + */ + void setAudioTrack(@Nullable AudioTrack audioTrack) { + Log.d(TAG, "set AudioTrack with " + audioTrack); + synchronized (mLock) { + // Release old reference. + if (mAudioTrack != null) { + mAudioTrack.release(); + } + // Sync play state for new reference. + if (audioTrack != null) { + if (mPlayState == PLAYSTATE_PLAYING + && audioTrack.getPlayState() != PLAYSTATE_PLAYING) { + audioTrack.play(); + } + if (mPlayState == PLAYSTATE_STOPPED + && audioTrack.getPlayState() != PLAYSTATE_STOPPED) { + audioTrack.stop(); + } + } + mAudioTrack = audioTrack; + } + } + + /** See {@link AudioTrack#write(ByteBuffer, int, int)}. */ + public int write(@NonNull ByteBuffer audioBuffer, int sizeInBytes, int writeMode) { + final int sizeWrite; + synchronized (mLock) { + if (mAudioTrack != null) { + sizeWrite = mAudioTrack.write(audioBuffer, sizeInBytes, writeMode); + } else { + sizeWrite = 0; + } + } + return sizeWrite; + } + + /** See {@link AudioTrack#play()}. */ + public void play() { + synchronized (mLock) { + mPlayState = PLAYSTATE_PLAYING; + if (mAudioTrack != null && mAudioTrack.getPlayState() != PLAYSTATE_PLAYING) { + mAudioTrack.play(); + } + } + } + + /** See {@link AudioTrack#stop()}. */ + public void stop() { + synchronized (mLock) { + mPlayState = PLAYSTATE_STOPPED; + if (mAudioTrack != null && mAudioTrack.getPlayState() != PLAYSTATE_STOPPED) { + mAudioTrack.stop(); + } + } + } + + /** See {@link AudioTrack#getPlayState()}. */ + public int getPlayState() { + synchronized (mLock) { + return mPlayState; + } + } +} diff --git a/core/java/android/companion/virtual/audio/IAudioSessionCallback.aidl b/core/java/android/companion/virtual/audio/IAudioSessionCallback.aidl new file mode 100644 index 0000000000000..e22ce148a9be9 --- /dev/null +++ b/core/java/android/companion/virtual/audio/IAudioSessionCallback.aidl @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.companion.virtual.audio; + +import android.media.AudioPlaybackConfiguration; +import android.media.AudioRecordingConfiguration; + +/** + * Callback to control audio rerouting, notify playback and recording state of applications running + * on virtual device. + * + * @hide + */ +oneway interface IAudioSessionCallback { + + /** Updates the set of applications that need to have their audio rerouted. */ + void onAppsNeedingAudioRoutingChanged(in int[] appUids); + + /** + * Called whenever the playback configuration of applications running on virtual device has + * changed. + */ + void onPlaybackConfigChanged(in List configs); + + /** + * Called whenever the recording configuration of applications running on virtual device has + * changed. + */ + void onRecordingConfigChanged(in List configs); +} diff --git a/core/java/android/companion/virtual/audio/VirtualAudioDevice.java b/core/java/android/companion/virtual/audio/VirtualAudioDevice.java new file mode 100644 index 0000000000000..38e37ec0c316c --- /dev/null +++ b/core/java/android/companion/virtual/audio/VirtualAudioDevice.java @@ -0,0 +1,167 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.companion.virtual.audio; + +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.annotation.RequiresPermission; +import android.annotation.SystemApi; +import android.companion.virtual.IVirtualDevice; +import android.content.Context; +import android.hardware.display.VirtualDisplay; +import android.media.AudioFormat; +import android.media.AudioPlaybackConfiguration; +import android.media.AudioRecordingConfiguration; +import android.os.RemoteException; + +import java.io.Closeable; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.Executor; + +/** + * The class stores an {@link AudioCapture} for audio capturing and an {@link AudioInjection} for + * audio injection. + * + * @hide + */ +@SystemApi +public final class VirtualAudioDevice implements Closeable { + + /** + * Interface to be notified when playback or recording configuration of applications running on + * virtual display was changed. + * + * @hide + */ + @SystemApi + public interface AudioConfigurationChangeCallback { + /** + * Notifies when playback configuration of applications running on virtual display was + * changed. + */ + void onPlaybackConfigChanged(@NonNull List configs); + + /** + * Notifies when recording configuration of applications running on virtual display was + * changed. + */ + void onRecordingConfigChanged(@NonNull List configs); + } + + private final Context mContext; + private final IVirtualDevice mVirtualDevice; + private final VirtualDisplay mVirtualDisplay; + private final AudioConfigurationChangeCallback mCallback; + private final Executor mExecutor; + @Nullable + private VirtualAudioSession mOngoingSession; + + /** + * @hide + */ + public VirtualAudioDevice(Context context, IVirtualDevice virtualDevice, + VirtualDisplay virtualDisplay, Executor executor, + AudioConfigurationChangeCallback callback) { + mContext = context; + mVirtualDevice = virtualDevice; + mVirtualDisplay = virtualDisplay; + mExecutor = executor; + mCallback = callback; + } + + /** + * Begins injecting audio from a remote device into this device. + * + * @return An {@link AudioInjection} containing the injected audio. + */ + @RequiresPermission(android.Manifest.permission.MODIFY_AUDIO_ROUTING) + @NonNull + public AudioInjection startAudioInjection(@NonNull AudioFormat injectionFormat) { + Objects.requireNonNull(injectionFormat, "injectionFormat must not be null"); + + if (mOngoingSession != null && mOngoingSession.getAudioInjection() != null) { + throw new IllegalStateException("Cannot start an audio injection while a session is " + + "ongoing. Call close() on this device first to end the previous injection."); + } + if (mOngoingSession == null) { + mOngoingSession = new VirtualAudioSession(mContext, mCallback, mExecutor); + } + + try { + mVirtualDevice.onAudioSessionStarting(mVirtualDisplay.getDisplay().getDisplayId(), + /* callback= */ mOngoingSession); + } catch (RemoteException e) { + throw e.rethrowFromSystemServer(); + } + return mOngoingSession.startAudioInjection(injectionFormat); + } + + /** + * Begins recording audio emanating from this device. + * + * @return An {@link AudioCapture} containing the recorded audio. + */ + @RequiresPermission(android.Manifest.permission.MODIFY_AUDIO_ROUTING) + @NonNull + public AudioCapture startAudioCapture(@NonNull AudioFormat captureFormat) { + Objects.requireNonNull(captureFormat, "captureFormat must not be null"); + + if (mOngoingSession != null && mOngoingSession.getAudioCapture() != null) { + throw new IllegalStateException("Cannot start an audio capture while a session is " + + "ongoing. Call close() on this device first to end the previous session."); + } + if (mOngoingSession == null) { + mOngoingSession = new VirtualAudioSession(mContext, mCallback, mExecutor); + } + + try { + mVirtualDevice.onAudioSessionStarting(mVirtualDisplay.getDisplay().getDisplayId(), + /* callback= */ mOngoingSession); + } catch (RemoteException e) { + throw e.rethrowFromSystemServer(); + } + return mOngoingSession.startAudioCapture(captureFormat); + } + + /** Returns the {@link AudioCapture} instance. */ + @Nullable + public AudioCapture getAudioCapture() { + return mOngoingSession != null ? mOngoingSession.getAudioCapture() : null; + } + + /** Returns the {@link AudioInjection} instance. */ + @Nullable + public AudioInjection getAudioInjection() { + return mOngoingSession != null ? mOngoingSession.getAudioInjection() : null; + } + + @RequiresPermission(android.Manifest.permission.MODIFY_AUDIO_ROUTING) + @Override + public void close() { + if (mOngoingSession != null) { + mOngoingSession.close(); + mOngoingSession = null; + + try { + mVirtualDevice.onAudioSessionEnded(); + } catch (RemoteException e) { + throw e.rethrowFromSystemServer(); + } + } + } +} diff --git a/core/java/android/companion/virtual/audio/VirtualAudioSession.java b/core/java/android/companion/virtual/audio/VirtualAudioSession.java new file mode 100644 index 0000000000000..4f4c33a8f7197 --- /dev/null +++ b/core/java/android/companion/virtual/audio/VirtualAudioSession.java @@ -0,0 +1,305 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.companion.virtual.audio; + +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.annotation.RequiresPermission; +import android.companion.virtual.audio.VirtualAudioDevice.AudioConfigurationChangeCallback; +import android.content.Context; +import android.media.AudioFormat; +import android.media.AudioManager; +import android.media.AudioPlaybackConfiguration; +import android.media.AudioRecord; +import android.media.AudioRecordingConfiguration; +import android.media.AudioTrack; +import android.media.audiopolicy.AudioMix; +import android.media.audiopolicy.AudioMixingRule; +import android.media.audiopolicy.AudioPolicy; +import android.util.IntArray; +import android.util.Log; + +import com.android.internal.annotations.GuardedBy; +import com.android.internal.annotations.VisibleForTesting; + +import java.io.Closeable; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.Executor; + +/** + * Manages an ongiong audio session in which audio can be captured (recorded) and/or + * injected from a remote device. + * + * @hide + */ +@VisibleForTesting +public final class VirtualAudioSession extends IAudioSessionCallback.Stub implements + Closeable { + private static final String TAG = "VirtualAudioSession"; + + private final Context mContext; + /** The {@link Executor} for sending {@link AudioConfigurationChangeCallback} to the caller */ + private final Executor mExecutor; + @Nullable + private final AudioConfigurationChangeCallback mCallback; + private final Object mLock = new Object(); + @GuardedBy("mLock") + private final IntArray mReroutedAppUids = new IntArray(); + @Nullable + @GuardedBy("mLock") + private AudioPolicy mAudioPolicy; + @Nullable + @GuardedBy("mLock") + private AudioFormat mCaptureFormat; + @Nullable + @GuardedBy("mLock") + private AudioFormat mInjectionFormat; + @Nullable + @GuardedBy("mLock") + private AudioCapture mAudioCapture; + @Nullable + @GuardedBy("mLock") + private AudioInjection mAudioInjection; + + @VisibleForTesting + public VirtualAudioSession(Context context, + @Nullable AudioConfigurationChangeCallback callback, @Nullable Executor executor) { + mContext = context; + mCallback = callback; + mExecutor = executor != null ? executor : context.getMainExecutor(); + } + + /** + * Begins recording audio emanating from this device. + * + * @return An {@link AudioCapture} containing the recorded audio. + */ + @VisibleForTesting + @NonNull + public AudioCapture startAudioCapture(@NonNull AudioFormat captureFormat) { + Objects.requireNonNull(captureFormat, "captureFormat must not be null"); + + synchronized (mLock) { + if (mAudioCapture != null) { + throw new IllegalStateException( + "Cannot start capture while another capture is ongoing."); + } + + mCaptureFormat = captureFormat; + mAudioCapture = new AudioCapture(); + mAudioCapture.startRecording(); + return mAudioCapture; + } + } + + /** + * Begins injecting audio from a remote device into this device. + * + * @return An {@link AudioInjection} containing the injected audio. + */ + @VisibleForTesting + @NonNull + public AudioInjection startAudioInjection(@NonNull AudioFormat injectionFormat) { + Objects.requireNonNull(injectionFormat, "injectionFormat must not be null"); + + synchronized (mLock) { + if (mAudioInjection != null) { + throw new IllegalStateException( + "Cannot start injection while injection is already ongoing."); + } + + mInjectionFormat = injectionFormat; + mAudioInjection = new AudioInjection(); + mAudioInjection.play(); + return mAudioInjection; + } + } + + /** @hide */ + @VisibleForTesting + @Nullable + public AudioCapture getAudioCapture() { + synchronized (mLock) { + return mAudioCapture; + } + } + + /** @hide */ + @VisibleForTesting + @Nullable + public AudioInjection getAudioInjection() { + synchronized (mLock) { + return mAudioInjection; + } + } + + @RequiresPermission(android.Manifest.permission.MODIFY_AUDIO_ROUTING) + @Override + public void onAppsNeedingAudioRoutingChanged(int[] appUids) { + synchronized (mLock) { + if (Arrays.equals(mReroutedAppUids.toArray(), appUids)) { + return; + } + } + + releaseAudioStreams(); + + if (appUids.length == 0) { + return; + } + + createAudioStreams(appUids); + } + + @RequiresPermission(android.Manifest.permission.MODIFY_AUDIO_ROUTING) + @Override + public void close() { + releaseAudioStreams(); + synchronized (mLock) { + mAudioCapture = null; + mAudioInjection = null; + mCaptureFormat = null; + mInjectionFormat = null; + } + } + + @RequiresPermission(android.Manifest.permission.MODIFY_AUDIO_ROUTING) + private void createAudioStreams(int[] appUids) { + synchronized (mLock) { + if (mCaptureFormat == null && mInjectionFormat == null) { + throw new IllegalStateException( + "At least one of captureFormat and injectionFormat must be specified."); + } + if (mAudioPolicy != null) { + throw new IllegalStateException( + "Cannot create audio streams while the audio policy is registered. Call " + + "releaseAudioStreams() first to unregister the previous audio " + + "policy." + ); + } + + mReroutedAppUids.clear(); + for (int appUid : appUids) { + mReroutedAppUids.add(appUid); + } + + AudioMix audioRecordMix = null; + AudioMix audioTrackMix = null; + AudioPolicy.Builder builder = new AudioPolicy.Builder(mContext); + if (mCaptureFormat != null) { + audioRecordMix = createAudioRecordMix(mCaptureFormat, appUids); + builder.addMix(audioRecordMix); + } + if (mInjectionFormat != null) { + audioTrackMix = createAudioTrackMix(mInjectionFormat, appUids); + builder.addMix(audioTrackMix); + } + mAudioPolicy = builder.build(); + AudioManager audioManager = mContext.getSystemService(AudioManager.class); + if (audioManager.registerAudioPolicy(mAudioPolicy) == AudioManager.ERROR) { + Log.e(TAG, "Failed to register audio policy!"); + } + + AudioRecord audioRecord = + audioRecordMix != null ? mAudioPolicy.createAudioRecordSink(audioRecordMix) + : null; + AudioTrack audioTrack = + audioTrackMix != null ? mAudioPolicy.createAudioTrackSource(audioTrackMix) + : null; + + if (mAudioCapture != null) { + mAudioCapture.setAudioRecord(audioRecord); + } + if (mAudioInjection != null) { + mAudioInjection.setAudioTrack(audioTrack); + } + } + } + + @RequiresPermission(android.Manifest.permission.MODIFY_AUDIO_ROUTING) + private void releaseAudioStreams() { + synchronized (mLock) { + if (mAudioCapture != null) { + mAudioCapture.setAudioRecord(null); + } + if (mAudioInjection != null) { + mAudioInjection.setAudioTrack(null); + } + mReroutedAppUids.clear(); + if (mAudioPolicy != null) { + AudioManager audioManager = mContext.getSystemService(AudioManager.class); + audioManager.unregisterAudioPolicy(mAudioPolicy); + mAudioPolicy = null; + Log.i(TAG, "AudioPolicy unregistered"); + } + } + } + + @Override + public void onPlaybackConfigChanged(List configs) { + if (mCallback != null) { + mExecutor.execute(() -> mCallback.onPlaybackConfigChanged(configs)); + } + } + + @Override + public void onRecordingConfigChanged(List configs) { + if (mCallback != null) { + mExecutor.execute(() -> mCallback.onRecordingConfigChanged(configs)); + } + } + + /** @hide */ + @VisibleForTesting + public IntArray getReroutedAppUids() { + synchronized (mLock) { + return mReroutedAppUids; + } + } + + private static AudioMix createAudioRecordMix(@NonNull AudioFormat audioFormat, int[] appUids) { + AudioMixingRule.Builder builder = new AudioMixingRule.Builder(); + builder.setTargetMixRole(AudioMixingRule.MIX_ROLE_PLAYERS); + for (int uid : appUids) { + builder.addMixRule(AudioMixingRule.RULE_MATCH_UID, uid); + } + AudioMixingRule audioMixingRule = builder.allowPrivilegedPlaybackCapture(false).build(); + AudioMix audioMix = + new AudioMix.Builder(audioMixingRule) + .setFormat(audioFormat) + .setRouteFlags(AudioMix.ROUTE_FLAG_LOOP_BACK) + .build(); + return audioMix; + } + + private static AudioMix createAudioTrackMix(@NonNull AudioFormat audioFormat, int[] appUids) { + AudioMixingRule.Builder builder = new AudioMixingRule.Builder(); + builder.setTargetMixRole(AudioMixingRule.MIX_ROLE_INJECTOR); + for (int uid : appUids) { + builder.addMixRule(AudioMixingRule.RULE_MATCH_UID, uid); + } + AudioMixingRule audioMixingRule = builder.build(); + AudioMix audioMix = + new AudioMix.Builder(audioMixingRule) + .setFormat(audioFormat) + .setRouteFlags(AudioMix.ROUTE_FLAG_LOOP_BACK) + .build(); + return audioMix; + } +} diff --git a/core/tests/coretests/src/android/companion/virtual/audio/VirtualAudioSessionTest.java b/core/tests/coretests/src/android/companion/virtual/audio/VirtualAudioSessionTest.java new file mode 100644 index 0000000000000..d66cb712b1ca2 --- /dev/null +++ b/core/tests/coretests/src/android/companion/virtual/audio/VirtualAudioSessionTest.java @@ -0,0 +1,182 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.companion.virtual.audio; + +import static android.media.AudioFormat.CHANNEL_IN_MONO; +import static android.media.AudioFormat.CHANNEL_OUT_MONO; +import static android.media.AudioFormat.ENCODING_PCM_16BIT; + +import static com.google.common.truth.Truth.assertThat; + +import static org.mockito.Mockito.verify; +import static org.testng.Assert.assertThrows; + +import android.companion.virtual.audio.VirtualAudioDevice.AudioConfigurationChangeCallback; +import android.content.Context; +import android.content.ContextWrapper; +import android.media.AudioFormat; +import android.media.AudioPlaybackConfiguration; +import android.media.AudioRecordingConfiguration; +import android.platform.test.annotations.Presubmit; + +import androidx.test.InstrumentationRegistry; +import androidx.test.runner.AndroidJUnit4; + +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +@Presubmit +@RunWith(AndroidJUnit4.class) +public class VirtualAudioSessionTest { + + @Rule + public final MockitoRule mockito = MockitoJUnit.rule(); + @Mock + private AudioConfigurationChangeCallback mCallback; + private static final int APP_UID = 100; + private static final int APP_UID2 = 200; + private static final AudioFormat AUDIO_CAPTURE_FORMAT = + new AudioFormat.Builder() + .setSampleRate(48000) + .setEncoding(ENCODING_PCM_16BIT) + .setChannelMask(CHANNEL_IN_MONO) + .build(); + private static final AudioFormat AUDIO_INJECT_FORMAT = + new AudioFormat.Builder() + .setSampleRate(48000) + .setEncoding(ENCODING_PCM_16BIT) + .setChannelMask(CHANNEL_OUT_MONO) + .build(); + private Context mContext; + private VirtualAudioSession mVirtualAudioSession; + + @Before + public void setUp() { + MockitoAnnotations.initMocks(this); + mContext = Mockito.spy(new ContextWrapper(InstrumentationRegistry.getTargetContext())); + mVirtualAudioSession = new VirtualAudioSession( + mContext, mCallback, /* executor= */ null); + } + + @Test + public void startAudioCapture_isSuccessful() { + AudioCapture audioCapture = mVirtualAudioSession.startAudioCapture(AUDIO_CAPTURE_FORMAT); + + assertThat(audioCapture).isNotNull(); + assertThat(mVirtualAudioSession.getAudioCapture()).isEqualTo(audioCapture); + } + + @Test + public void startAudioCapture_audioCaptureAlreadyStarted_throws() { + mVirtualAudioSession.startAudioCapture(AUDIO_CAPTURE_FORMAT); + + assertThrows(IllegalStateException.class, + () -> mVirtualAudioSession.startAudioCapture(AUDIO_CAPTURE_FORMAT)); + } + + @Test + public void startAudioInjection_isSuccessful() { + AudioInjection audioInjection = mVirtualAudioSession.startAudioInjection( + AUDIO_INJECT_FORMAT); + + assertThat(audioInjection).isNotNull(); + assertThat(mVirtualAudioSession.getAudioInjection()).isEqualTo(audioInjection); + } + + @Test + public void startAudioInjection_audioInjectionAlreadyStarted_throws() { + mVirtualAudioSession.startAudioInjection(AUDIO_INJECT_FORMAT); + + assertThrows(IllegalStateException.class, + () -> mVirtualAudioSession.startAudioInjection(AUDIO_INJECT_FORMAT)); + } + + @Test + public void onAppsNeedingAudioRoutingChanged_neverStartAudioCaptureOrInjection_throws() { + int[] uids = new int[]{APP_UID}; + + assertThrows(IllegalStateException.class, + () -> mVirtualAudioSession.onAppsNeedingAudioRoutingChanged(uids)); + } + + @Test + public void onAppsNeedingAudioRoutingChanged_cachesReroutedApps() { + mVirtualAudioSession.startAudioCapture(AUDIO_CAPTURE_FORMAT); + mVirtualAudioSession.startAudioInjection(AUDIO_INJECT_FORMAT); + int[] appUids = new int[]{APP_UID}; + + mVirtualAudioSession.onAppsNeedingAudioRoutingChanged(appUids); + + assertThat(Arrays.equals(mVirtualAudioSession.getReroutedAppUids().toArray(), + appUids)).isTrue(); + } + + @Test + public void onAppsNeedingAudioRoutingChanged_receiveManyTimes_reroutedAppsSizeIsCorrect() { + mVirtualAudioSession.startAudioCapture(AUDIO_CAPTURE_FORMAT); + mVirtualAudioSession.startAudioInjection(AUDIO_INJECT_FORMAT); + int[] appUids = new int[]{APP_UID, APP_UID2}; + + mVirtualAudioSession.onAppsNeedingAudioRoutingChanged(new int[]{1234}); + mVirtualAudioSession.onAppsNeedingAudioRoutingChanged(new int[]{5678}); + mVirtualAudioSession.onAppsNeedingAudioRoutingChanged(appUids); + + assertThat(Arrays.equals(mVirtualAudioSession.getReroutedAppUids().toArray(), + appUids)).isTrue(); + assertThat(mVirtualAudioSession.getReroutedAppUids().size()).isEqualTo(2); + } + + @Test + public void close_releasesCaptureAndInjection() { + mVirtualAudioSession.startAudioCapture(AUDIO_CAPTURE_FORMAT); + mVirtualAudioSession.startAudioInjection(AUDIO_INJECT_FORMAT); + + mVirtualAudioSession.close(); + + assertThat(mVirtualAudioSession.getAudioCapture()).isNull(); + assertThat(mVirtualAudioSession.getAudioInjection()).isNull(); + } + + @Test + public void onPlaybackConfigChanged_sendsCallback() { + List configs = new ArrayList<>(); + + mVirtualAudioSession.onPlaybackConfigChanged(configs); + + verify(mCallback).onPlaybackConfigChanged(configs); + } + + @Test + public void onRecordingConfigChanged_sendCallback() { + List configs = new ArrayList<>(); + + mVirtualAudioSession.onRecordingConfigChanged(configs); + + verify(mCallback).onRecordingConfigChanged(configs); + } +} diff --git a/services/companion/java/com/android/server/companion/virtual/GenericWindowPolicyController.java b/services/companion/java/com/android/server/companion/virtual/GenericWindowPolicyController.java index dafcc60fb651d..4afa96c8072d1 100644 --- a/services/companion/java/com/android/server/companion/virtual/GenericWindowPolicyController.java +++ b/services/companion/java/com/android/server/companion/virtual/GenericWindowPolicyController.java @@ -47,9 +47,17 @@ import java.util.function.Consumer; /** * A controller to control the policies of the windows that can be displayed on the virtual display. */ -class GenericWindowPolicyController extends DisplayWindowPolicyController { +public class GenericWindowPolicyController extends DisplayWindowPolicyController { - private static final String TAG = "VirtualDeviceManager"; + private static final String TAG = "GenericWindowPolicyController"; + + /** Interface to listen running applications change on virtual display. */ + public interface RunningAppsChangedListener { + /** + * Notifies the running applications change. + */ + void onRunningAppsChanged(ArraySet runningUids); + } private static final ComponentName BLOCKED_APP_STREAMING_COMPONENT = new ComponentName("android", BlockedAppStreamingActivity.class.getName()); @@ -74,6 +82,9 @@ class GenericWindowPolicyController extends DisplayWindowPolicyController { @Nullable private final ActivityListener mActivityListener; private final Handler mHandler = new Handler(Looper.getMainLooper()); + @Nullable + private RunningAppsChangedListener mRunningAppsChangedListener; + /** * Creates a window policy controller that is generic to the different use cases of virtual * device. @@ -84,7 +95,7 @@ class GenericWindowPolicyController extends DisplayWindowPolicyController { * @param activityListener Activity listener to listen for activity changes. The display ID * is not populated in this callback and is always {@link Display#INVALID_DISPLAY}. */ - GenericWindowPolicyController(int windowFlags, int systemWindowFlags, + public GenericWindowPolicyController(int windowFlags, int systemWindowFlags, @NonNull ArraySet allowedUsers, @Nullable Set allowedActivities, @Nullable Set blockedActivities, @@ -98,6 +109,11 @@ class GenericWindowPolicyController extends DisplayWindowPolicyController { mActivityListener = activityListener; } + /** Sets listener for running applications change. */ + public void setRunningAppsChangedListener(@Nullable RunningAppsChangedListener listener) { + mRunningAppsChangedListener = listener; + } + @Override public boolean canContainActivities(@NonNull List activities) { // Can't display all the activities if any of them don't want to be displayed. @@ -139,6 +155,9 @@ class GenericWindowPolicyController extends DisplayWindowPolicyController { // Post callback on the main thread so it doesn't block activity launching mHandler.post(() -> mActivityListener.onDisplayEmpty(Display.INVALID_DISPLAY)); } + if (mRunningAppsChangedListener != null) { + mRunningAppsChangedListener.onRunningAppsChanged(runningUids); + } } /** diff --git a/services/companion/java/com/android/server/companion/virtual/VirtualDeviceImpl.java b/services/companion/java/com/android/server/companion/virtual/VirtualDeviceImpl.java index 075d96dd7ad42..3fa1c865bda16 100644 --- a/services/companion/java/com/android/server/companion/virtual/VirtualDeviceImpl.java +++ b/services/companion/java/com/android/server/companion/virtual/VirtualDeviceImpl.java @@ -23,6 +23,7 @@ import static android.view.WindowManager.LayoutParams.FLAG_SECURE; import static android.view.WindowManager.LayoutParams.SYSTEM_FLAG_HIDE_NON_SYSTEM_OVERLAY_WINDOWS; import android.annotation.NonNull; +import android.annotation.RequiresPermission; import android.app.Activity; import android.app.ActivityOptions; import android.app.PendingIntent; @@ -32,6 +33,7 @@ import android.companion.virtual.IVirtualDevice; import android.companion.virtual.IVirtualDeviceActivityListener; import android.companion.virtual.VirtualDeviceManager.ActivityListener; import android.companion.virtual.VirtualDeviceParams; +import android.companion.virtual.audio.IAudioSessionCallback; import android.content.ComponentName; import android.content.Context; import android.content.Intent; @@ -60,6 +62,7 @@ import android.window.DisplayWindowPolicyController; import com.android.internal.annotations.GuardedBy; import com.android.internal.annotations.VisibleForTesting; import com.android.internal.app.BlockedAppStreamingActivity; +import com.android.server.companion.virtual.audio.VirtualAudioController; import java.io.FileDescriptor; import java.io.PrintWriter; @@ -78,6 +81,7 @@ final class VirtualDeviceImpl extends IVirtualDevice.Stub private final PendingTrampolineCallback mPendingTrampolineCallback; private final int mOwnerUid; private final InputController mInputController; + private VirtualAudioController mVirtualAudioController; @VisibleForTesting final Set mVirtualDisplayIds = new ArraySet<>(); private final OnDeviceCloseListener mListener; @@ -245,6 +249,46 @@ final class VirtualDeviceImpl extends IVirtualDevice.Stub close(); } + @VisibleForTesting + VirtualAudioController getVirtualAudioControllerForTesting() { + return mVirtualAudioController; + } + + @RequiresPermission(android.Manifest.permission.CREATE_VIRTUAL_DEVICE) + @Override // Binder call + public void onAudioSessionStarting(int displayId, IAudioSessionCallback callback) { + mContext.enforceCallingOrSelfPermission( + android.Manifest.permission.CREATE_VIRTUAL_DEVICE, + "Permission required to start audio session"); + synchronized (mVirtualDeviceLock) { + if (!mVirtualDisplayIds.contains(displayId)) { + throw new SecurityException( + "Cannot start audio session for a display not associated with this virtual " + + "device"); + } + + if (mVirtualAudioController == null) { + mVirtualAudioController = new VirtualAudioController(mContext); + GenericWindowPolicyController gwpc = mWindowPolicyControllers.get(displayId); + mVirtualAudioController.startListening(gwpc, callback); + } + } + } + + @RequiresPermission(android.Manifest.permission.CREATE_VIRTUAL_DEVICE) + @Override // Binder call + public void onAudioSessionEnded() { + mContext.enforceCallingOrSelfPermission( + android.Manifest.permission.CREATE_VIRTUAL_DEVICE, + "Permission required to stop audio session"); + synchronized (mVirtualDeviceLock) { + if (mVirtualAudioController != null) { + mVirtualAudioController.stopListening(); + mVirtualAudioController = null; + } + } + } + @Override // Binder call public void createVirtualKeyboard( int displayId, diff --git a/services/companion/java/com/android/server/companion/virtual/audio/AudioPlaybackDetector.java b/services/companion/java/com/android/server/companion/virtual/audio/AudioPlaybackDetector.java new file mode 100644 index 0000000000000..2d7291300e23e --- /dev/null +++ b/services/companion/java/com/android/server/companion/virtual/audio/AudioPlaybackDetector.java @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.companion.virtual.audio; + +import android.annotation.NonNull; +import android.content.Context; +import android.media.AudioManager; +import android.media.AudioPlaybackConfiguration; + +import java.util.List; + +/** + * Wrapper class for other classes to listen {@link #onPlaybackConfigChanged(List)} by implementing + * {@link AudioPlaybackCallback} instead of inheriting the + * {@link AudioManager.AudioPlaybackCallback}. + */ +final class AudioPlaybackDetector extends AudioManager.AudioPlaybackCallback { + + /** + * Interface to listen {@link #onPlaybackConfigChanged(List)} from + * {@link AudioManager.AudioPlaybackCallback}. + */ + interface AudioPlaybackCallback { + void onPlaybackConfigChanged(List configs); + } + + private final AudioManager mAudioManager; + private AudioPlaybackCallback mAudioPlaybackCallback; + + AudioPlaybackDetector(Context context) { + mAudioManager = context.getSystemService(AudioManager.class); + } + + void register(@NonNull AudioPlaybackCallback callback) { + mAudioPlaybackCallback = callback; + mAudioManager.registerAudioPlaybackCallback(/* cb= */ this, /* handler= */ null); + } + + void unregister() { + mAudioPlaybackCallback = null; + mAudioManager.unregisterAudioPlaybackCallback(/* cb= */ this); + } + + @Override + public void onPlaybackConfigChanged(List configs) { + super.onPlaybackConfigChanged(configs); + if (mAudioPlaybackCallback != null) { + mAudioPlaybackCallback.onPlaybackConfigChanged(configs); + } + } +} diff --git a/services/companion/java/com/android/server/companion/virtual/audio/AudioRecordingDetector.java b/services/companion/java/com/android/server/companion/virtual/audio/AudioRecordingDetector.java new file mode 100644 index 0000000000000..c20414529f5b9 --- /dev/null +++ b/services/companion/java/com/android/server/companion/virtual/audio/AudioRecordingDetector.java @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.companion.virtual.audio; + +import android.annotation.NonNull; +import android.content.Context; +import android.media.AudioManager; +import android.media.AudioRecordingConfiguration; + +import java.util.List; + +/** + * Wrapper class for other classes to listen {@link #onRecordingConfigChanged(List)} by implementing + * {@link AudioRecordingCallback} instead of inheriting the + * {@link AudioManager.AudioRecordingCallback}. + */ +final class AudioRecordingDetector extends AudioManager.AudioRecordingCallback { + + /** + * Interface to listen {@link #onRecordingConfigChanged(List)} from + * {@link AudioManager.AudioRecordingCallback}. + */ + interface AudioRecordingCallback { + void onRecordingConfigChanged(List configs); + } + + private final AudioManager mAudioManager; + private AudioRecordingCallback mAudioRecordingCallback; + + AudioRecordingDetector(Context context) { + mAudioManager = context.getSystemService(AudioManager.class); + } + + void register(@NonNull AudioRecordingCallback callback) { + mAudioRecordingCallback = callback; + mAudioManager.registerAudioRecordingCallback(/* cb= */ this, /* handler= */ null); + } + + void unregister() { + mAudioRecordingCallback = null; + mAudioManager.unregisterAudioRecordingCallback(/* cb= */ this); + } + + @Override + public void onRecordingConfigChanged(List configs) { + super.onRecordingConfigChanged(configs); + if (mAudioRecordingCallback != null) { + mAudioRecordingCallback.onRecordingConfigChanged(configs); + } + } +} diff --git a/services/companion/java/com/android/server/companion/virtual/audio/VirtualAudioController.java b/services/companion/java/com/android/server/companion/virtual/audio/VirtualAudioController.java new file mode 100644 index 0000000000000..1dc87d68aeaee --- /dev/null +++ b/services/companion/java/com/android/server/companion/virtual/audio/VirtualAudioController.java @@ -0,0 +1,294 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.companion.virtual.audio; + +import static android.media.AudioPlaybackConfiguration.PLAYER_STATE_STARTED; + +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.annotation.RequiresPermission; +import android.companion.virtual.audio.IAudioSessionCallback; +import android.content.Context; +import android.media.AudioManager; +import android.media.AudioPlaybackConfiguration; +import android.media.AudioRecordingConfiguration; +import android.os.Handler; +import android.os.Looper; +import android.os.RemoteException; +import android.util.ArraySet; +import android.util.Slog; + +import com.android.internal.annotations.GuardedBy; +import com.android.internal.annotations.VisibleForTesting; +import com.android.server.companion.virtual.GenericWindowPolicyController; +import com.android.server.companion.virtual.GenericWindowPolicyController.RunningAppsChangedListener; +import com.android.server.companion.virtual.audio.AudioPlaybackDetector.AudioPlaybackCallback; +import com.android.server.companion.virtual.audio.AudioRecordingDetector.AudioRecordingCallback; + +import java.util.ArrayList; +import java.util.List; + +/** + * Manages audio streams associated with a {@link VirtualAudioDevice}. Responsible for monitoring + * running applications and playback configuration changes in order to correctly re-route audio and + * then notify clients of these changes. + */ +public final class VirtualAudioController implements AudioPlaybackCallback, + AudioRecordingCallback, RunningAppsChangedListener { + private static final String TAG = "VirtualAudioController"; + private static final int UPDATE_REROUTING_APPS_DELAY_MS = 2000; + + private final Context mContext; + private final Handler mHandler = new Handler(Looper.getMainLooper()); + private final Runnable mUpdateAudioRoutingRunnable = this::notifyAppsNeedingAudioRoutingChanged; + private final AudioPlaybackDetector mAudioPlaybackDetector; + private final AudioRecordingDetector mAudioRecordingDetector; + private final Object mLock = new Object(); + @GuardedBy("mLock") + private final ArraySet mRunningAppUids = new ArraySet<>(); + @GuardedBy("mLock") + private ArraySet mPlayingAppUids = new ArraySet<>(); + private GenericWindowPolicyController mGenericWindowPolicyController; + private final Object mCallbackLock = new Object(); + @Nullable + @GuardedBy("mCallbackLock") + private IAudioSessionCallback mCallback; + + public VirtualAudioController(Context context) { + mContext = context; + mAudioPlaybackDetector = new AudioPlaybackDetector(context); + mAudioRecordingDetector = new AudioRecordingDetector(context); + } + + /** + * Starts to listen to running applications and audio configuration changes on virtual display + * for audio capture and injection. + */ + public void startListening( + @NonNull GenericWindowPolicyController genericWindowPolicyController, + @Nullable IAudioSessionCallback callback) { + mGenericWindowPolicyController = genericWindowPolicyController; + mGenericWindowPolicyController.setRunningAppsChangedListener(/* listener= */ this); + synchronized (mCallbackLock) { + mCallback = callback; + } + synchronized (mLock) { + mRunningAppUids.clear(); + mPlayingAppUids.clear(); + } + mAudioPlaybackDetector.register(/* callback= */ this); + mAudioRecordingDetector.register(/* callback= */ this); + } + + /** + * Stops listening to running applications and audio configuration changes on virtual display + * for audio capture and injection. + */ + public void stopListening() { + if (mHandler.hasCallbacks(mUpdateAudioRoutingRunnable)) { + mHandler.removeCallbacks(mUpdateAudioRoutingRunnable); + } + mAudioPlaybackDetector.unregister(); + mAudioRecordingDetector.unregister(); + if (mGenericWindowPolicyController != null) { + mGenericWindowPolicyController.setRunningAppsChangedListener(/* listener= */ null); + mGenericWindowPolicyController = null; + } + synchronized (mCallbackLock) { + mCallback = null; + } + } + + @Override + public void onRunningAppsChanged(ArraySet runningUids) { + synchronized (mLock) { + if (mRunningAppUids.equals(runningUids)) { + // Ignore no-op events. + return; + } + mRunningAppUids.clear(); + mRunningAppUids.addAll(runningUids); + + ArraySet oldPlayingAppUids = mPlayingAppUids; + + // Update the list of playing apps after caching the old list, and before checking if + // the list of playing apps is empty. This is a subset of the running apps, so we need + // to update this here as well. + AudioManager audioManager = mContext.getSystemService(AudioManager.class); + List configs = + audioManager.getActivePlaybackConfigurations(); + mPlayingAppUids = findPlayingAppUids(configs, mRunningAppUids); + + // Do not change rerouted applications while any application is playing, or the sound + // will be leaked from phone during the transition. Delay the change until we detect + // there is no application is playing in onPlaybackConfigChanged(). + if (!mPlayingAppUids.isEmpty()) { + Slog.i(TAG, "Audio is playing, do not change rerouted apps"); + return; + } + + // An application previously playing audio was removed from the display. + if (!oldPlayingAppUids.isEmpty()) { + // Delay changing the rerouted application when the last application playing audio + // was removed from virtual device, or the sound will be leaked from phone side + // during the transition. + Slog.i(TAG, "The last playing app removed, delay change rerouted apps"); + if (mHandler.hasCallbacks(mUpdateAudioRoutingRunnable)) { + mHandler.removeCallbacks(mUpdateAudioRoutingRunnable); + } + mHandler.postDelayed(mUpdateAudioRoutingRunnable, UPDATE_REROUTING_APPS_DELAY_MS); + return; + } + } + + // Normal case with no application playing, just update routing. + notifyAppsNeedingAudioRoutingChanged(); + } + + @Override + public void onPlaybackConfigChanged(List configs) { + updatePlayingApplications(configs); + + List audioPlaybackConfigurations; + synchronized (mLock) { + // Filter configurations of applications running on virtual device. + audioPlaybackConfigurations = findPlaybackConfigurations(configs, mRunningAppUids); + } + synchronized (mCallbackLock) { + if (mCallback != null) { + try { + mCallback.onPlaybackConfigChanged(audioPlaybackConfigurations); + } catch (RemoteException e) { + Slog.e(TAG, "RemoteException when calling onPlaybackConfigChanged", e); + } + } + } + } + + @RequiresPermission(android.Manifest.permission.MODIFY_AUDIO_ROUTING) + @Override + public void onRecordingConfigChanged(List configs) { + List audioRecordingConfigurations; + synchronized (mLock) { + // Filter configurations of applications running on virtual device. + audioRecordingConfigurations = findRecordingConfigurations(configs, mRunningAppUids); + } + synchronized (mCallbackLock) { + if (mCallback != null) { + try { + mCallback.onRecordingConfigChanged(audioRecordingConfigurations); + } catch (RemoteException e) { + Slog.e(TAG, "RemoteException when calling onRecordingConfigChanged", e); + } + } + } + } + + private void updatePlayingApplications(List configs) { + synchronized (mLock) { + ArraySet playingAppUids = findPlayingAppUids(configs, mRunningAppUids); + if (mPlayingAppUids.equals(playingAppUids)) { + return; + } + mPlayingAppUids = playingAppUids; + } + + // Updated rerouted apps, even if the app is already playing. It originally should be done + // when onRunningAppsChanged() is called, but we don't want to interrupt the audio + // streaming and cause the sound leak from phone when it's playing, so delay until here. + notifyAppsNeedingAudioRoutingChanged(); + } + + private void notifyAppsNeedingAudioRoutingChanged() { + if (mHandler.hasCallbacks(mUpdateAudioRoutingRunnable)) { + mHandler.removeCallbacks(mUpdateAudioRoutingRunnable); + } + + int[] runningUids; + synchronized (mLock) { + runningUids = new int[mRunningAppUids.size()]; + for (int i = 0; i < mRunningAppUids.size(); i++) { + runningUids[i] = mRunningAppUids.valueAt(i); + } + } + + synchronized (mCallbackLock) { + if (mCallback != null) { + try { + mCallback.onAppsNeedingAudioRoutingChanged(runningUids); + } catch (RemoteException e) { + Slog.e(TAG, "RemoteException when calling updateReroutingApps", e); + } + } + } + } + + /** + * Finds uid of playing applications from the given running applications. + * + * @param configs a list of playback configs which get from {@link AudioManager} + */ + private static ArraySet findPlayingAppUids(List configs, + ArraySet runningAppUids) { + ArraySet playingAppUids = new ArraySet<>(); + for (AudioPlaybackConfiguration config : configs) { + if (runningAppUids.contains(config.getClientUid()) + && config.getPlayerState() == PLAYER_STATE_STARTED) { + playingAppUids.add(config.getClientUid()); + } + } + return playingAppUids; + } + + /** Finds a list of {@link AudioPlaybackConfiguration} for the given running applications. */ + private static List findPlaybackConfigurations( + List configs, + ArraySet runningAppUids) { + List runningConfigs = new ArrayList<>(); + for (AudioPlaybackConfiguration config : configs) { + if (runningAppUids.contains(config.getClientUid())) { + runningConfigs.add(config); + } + } + return runningConfigs; + } + + /** Finds a list of {@link AudioRecordingConfiguration} for the given running applications. */ + @RequiresPermission(android.Manifest.permission.MODIFY_AUDIO_ROUTING) + private static List findRecordingConfigurations( + List configs, ArraySet runningAppUids) { + List runningConfigs = new ArrayList<>(); + for (AudioRecordingConfiguration config : configs) { + if (runningAppUids.contains(config.getClientUid())) { + runningConfigs.add(config); + } + } + return runningConfigs; + } + + @VisibleForTesting + boolean hasPendingRunnable() { + return mHandler.hasCallbacks(mUpdateAudioRoutingRunnable); + } + + @VisibleForTesting + void addPlayingAppsForTesting(int appUid) { + synchronized (mLock) { + mPlayingAppUids.add(appUid); + } + } +} diff --git a/services/tests/servicestests/src/com/android/server/companion/virtual/VirtualDeviceManagerServiceTest.java b/services/tests/servicestests/src/com/android/server/companion/virtual/VirtualDeviceManagerServiceTest.java index 3b4aece5997e2..2ae285409b731 100644 --- a/services/tests/servicestests/src/com/android/server/companion/virtual/VirtualDeviceManagerServiceTest.java +++ b/services/tests/servicestests/src/com/android/server/companion/virtual/VirtualDeviceManagerServiceTest.java @@ -16,6 +16,7 @@ package com.android.server.companion.virtual; +import static com.google.common.truth.Truth.assertThat; import static com.google.common.truth.Truth.assertWithMessage; import static org.mockito.ArgumentMatchers.any; @@ -39,6 +40,7 @@ import android.app.admin.DevicePolicyManager; import android.companion.AssociationInfo; import android.companion.virtual.IVirtualDeviceActivityListener; import android.companion.virtual.VirtualDeviceParams; +import android.companion.virtual.audio.IAudioSessionCallback; import android.content.Context; import android.content.ContextWrapper; import android.graphics.Point; @@ -111,6 +113,8 @@ public class VirtualDeviceManagerServiceTest { @Mock IThermalService mIThermalServiceMock; private PowerManager mPowerManager; + @Mock + private IAudioSessionCallback mCallback; @Before public void setUp() { @@ -249,6 +253,12 @@ public class VirtualDeviceManagerServiceTest { VENDOR_ID, PRODUCT_ID, BINDER, new Point(WIDTH, HEIGHT))); } + @Test + public void onAudioSessionStarting_noDisplay_failsSecurityException() { + assertThrows(SecurityException.class, + () -> mDeviceImpl.onAudioSessionStarting(DISPLAY_ID, mCallback)); + } + @Test public void createVirtualKeyboard_noPermission_failsSecurityException() { mDeviceImpl.mVirtualDisplayIds.add(DISPLAY_ID); @@ -282,6 +292,22 @@ public class VirtualDeviceManagerServiceTest { VENDOR_ID, PRODUCT_ID, BINDER, new Point(WIDTH, HEIGHT))); } + @Test + public void onAudioSessionStarting_noPermission_failsSecurityException() { + mDeviceImpl.mVirtualDisplayIds.add(DISPLAY_ID); + doCallRealMethod().when(mContext).enforceCallingOrSelfPermission( + eq(Manifest.permission.CREATE_VIRTUAL_DEVICE), anyString()); + assertThrows(SecurityException.class, + () -> mDeviceImpl.onAudioSessionStarting(DISPLAY_ID, mCallback)); + } + + @Test + public void onAudioSessionEnded_noPermission_failsSecurityException() { + doCallRealMethod().when(mContext).enforceCallingOrSelfPermission( + eq(Manifest.permission.CREATE_VIRTUAL_DEVICE), anyString()); + assertThrows(SecurityException.class, () -> mDeviceImpl.onAudioSessionEnded()); + } + @Test public void createVirtualKeyboard_hasDisplay_obtainFileDescriptor() { mDeviceImpl.mVirtualDisplayIds.add(DISPLAY_ID); @@ -315,6 +341,25 @@ public class VirtualDeviceManagerServiceTest { eq(PRODUCT_ID), anyString(), eq(HEIGHT), eq(WIDTH)); } + @Test + public void onAudioSessionStarting_hasVirtualAudioController() { + mDeviceImpl.onVirtualDisplayCreatedLocked(DISPLAY_ID); + + mDeviceImpl.onAudioSessionStarting(DISPLAY_ID, mCallback); + + assertThat(mDeviceImpl.getVirtualAudioControllerForTesting()).isNotNull(); + } + + @Test + public void onAudioSessionEnded_noVirtualAudioController() { + mDeviceImpl.onVirtualDisplayCreatedLocked(DISPLAY_ID); + mDeviceImpl.onAudioSessionStarting(DISPLAY_ID, mCallback); + + mDeviceImpl.onAudioSessionEnded(); + + assertThat(mDeviceImpl.getVirtualAudioControllerForTesting()).isNull(); + } + @Test public void sendKeyEvent_noFd() { assertThrows( diff --git a/services/tests/servicestests/src/com/android/server/companion/virtual/audio/VirtualAudioControllerTest.java b/services/tests/servicestests/src/com/android/server/companion/virtual/audio/VirtualAudioControllerTest.java new file mode 100644 index 0000000000000..3160272ef9b1c --- /dev/null +++ b/services/tests/servicestests/src/com/android/server/companion/virtual/audio/VirtualAudioControllerTest.java @@ -0,0 +1,202 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.companion.virtual.audio; + +import static android.media.AudioAttributes.FLAG_SECURE; +import static android.media.AudioPlaybackConfiguration.PLAYER_STATE_STARTED; +import static android.view.WindowManager.LayoutParams.SYSTEM_FLAG_HIDE_NON_SYSTEM_OVERLAY_WINDOWS; + +import static com.google.common.truth.Truth.assertThat; + +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; + +import android.companion.virtual.audio.IAudioSessionCallback; +import android.content.Context; +import android.content.ContextWrapper; +import android.media.AudioPlaybackConfiguration; +import android.media.AudioRecordingConfiguration; +import android.media.MediaRecorder; +import android.media.PlayerBase; +import android.os.Parcel; +import android.os.RemoteException; +import android.util.ArraySet; + +import androidx.test.InstrumentationRegistry; +import androidx.test.runner.AndroidJUnit4; + +import com.android.server.companion.virtual.GenericWindowPolicyController; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; + +import java.util.ArrayList; +import java.util.List; + +@RunWith(AndroidJUnit4.class) +public class VirtualAudioControllerTest { + private static final int APP1_UID = 100; + private static final int APP2_UID = 200; + + private Context mContext; + private VirtualAudioController mVirtualAudioController; + private GenericWindowPolicyController mGenericWindowPolicyController; + @Mock IAudioSessionCallback mCallback; + + @Before + public void setUp() { + MockitoAnnotations.initMocks(this); + mContext = Mockito.spy(new ContextWrapper(InstrumentationRegistry.getTargetContext())); + mVirtualAudioController = new VirtualAudioController(mContext); + mGenericWindowPolicyController = new GenericWindowPolicyController( + FLAG_SECURE, + SYSTEM_FLAG_HIDE_NON_SYSTEM_OVERLAY_WINDOWS, + /* allowedUsers= */ new ArraySet<>(), + /* allowedActivities= */ new ArraySet<>(), + /* blockedActivities= */ new ArraySet<>(), + /* activityListener= */null, + /* activityBlockedCallback= */ null); + } + + @Test + public void startListening_receivesCallback() throws RemoteException { + ArraySet runningUids = new ArraySet<>(); + runningUids.add(APP1_UID); + int[] appUids = new int[] {APP1_UID}; + + mVirtualAudioController.startListening(mGenericWindowPolicyController, mCallback); + + mGenericWindowPolicyController.onRunningAppsChanged(runningUids); + verify(mCallback).onAppsNeedingAudioRoutingChanged(appUids); + } + + @Test + public void stopListening_removesCallback() throws RemoteException { + ArraySet runningUids = new ArraySet<>(); + runningUids.add(APP1_UID); + int[] appUids = new int[] {APP1_UID}; + mVirtualAudioController.startListening(mGenericWindowPolicyController, mCallback); + + mVirtualAudioController.stopListening(); + + mGenericWindowPolicyController.onRunningAppsChanged(runningUids); + verify(mCallback, never()).onAppsNeedingAudioRoutingChanged(appUids); + } + + @Test + public void onRunningAppsChanged_notifiesAudioRoutingModified() throws RemoteException { + mVirtualAudioController.startListening(mGenericWindowPolicyController, mCallback); + + ArraySet runningUids = new ArraySet<>(); + runningUids.add(APP1_UID); + mVirtualAudioController.onRunningAppsChanged(runningUids); + + int[] appUids = new int[] {APP1_UID}; + verify(mCallback).onAppsNeedingAudioRoutingChanged(appUids); + } + + @Test + public void onRunningAppsChanged_audioIsPlaying_doesNothing() throws RemoteException { + mVirtualAudioController.startListening(mGenericWindowPolicyController, mCallback); + mVirtualAudioController.addPlayingAppsForTesting(APP2_UID); + + ArraySet runningUids = new ArraySet<>(); + runningUids.add(APP1_UID); + mVirtualAudioController.onRunningAppsChanged(runningUids); + + int[] appUids = new int[]{APP1_UID}; + verify(mCallback, never()).onAppsNeedingAudioRoutingChanged(appUids); + } + + @Test + public void onRunningAppsChanged_lastPlayingAppRemoved_delaysReroutingAudio() { + ArraySet runningUids = new ArraySet<>(); + runningUids.add(APP1_UID); + runningUids.add(APP2_UID); + mVirtualAudioController.onRunningAppsChanged(runningUids); + mVirtualAudioController.addPlayingAppsForTesting(APP2_UID); + + ArraySet appUids = new ArraySet<>(); + appUids.add(APP1_UID); + mVirtualAudioController.onRunningAppsChanged(appUids); + + assertThat(mVirtualAudioController.hasPendingRunnable()).isTrue(); + } + + @Test + public void onPlaybackConfigChanged_sendsCallback() throws RemoteException { + mVirtualAudioController.startListening(mGenericWindowPolicyController, mCallback); + ArraySet runningUids = new ArraySet<>(); + runningUids.add(APP1_UID); + mVirtualAudioController.onRunningAppsChanged(runningUids); + List configs = createPlaybackConfigurations(runningUids); + + mVirtualAudioController.onPlaybackConfigChanged(configs); + + verify(mCallback).onPlaybackConfigChanged(configs); + } + + @Test + public void onRecordingConfigChanged_sendsCallback() throws RemoteException { + mVirtualAudioController.startListening(mGenericWindowPolicyController, mCallback); + ArraySet runningUids = new ArraySet<>(); + runningUids.add(APP1_UID); + mVirtualAudioController.onRunningAppsChanged(runningUids); + List configs = createRecordingConfigurations(runningUids); + + mVirtualAudioController.onRecordingConfigChanged(configs); + + verify(mCallback).onRecordingConfigChanged(configs); + } + + private List createPlaybackConfigurations( + ArraySet appUids) { + List configs = new ArrayList<>(); + for (int appUid : appUids) { + PlayerBase.PlayerIdCard playerIdCard = + PlayerBase.PlayerIdCard.CREATOR.createFromParcel(Parcel.obtain()); + AudioPlaybackConfiguration audioPlaybackConfiguration = + new AudioPlaybackConfiguration( + playerIdCard, /* piid= */ 1000, appUid, /* pid= */ 1000); + audioPlaybackConfiguration.handleStateEvent(PLAYER_STATE_STARTED, /* deviceId= */1); + configs.add(audioPlaybackConfiguration); + } + return configs; + } + + private List createRecordingConfigurations( + ArraySet appUids) { + List configs = new ArrayList<>(); + for (int appUid : appUids) { + AudioRecordingConfiguration audioRecordingConfiguration = + new AudioRecordingConfiguration( + /* uid= */ appUid, + /* session= */ 1000, + MediaRecorder.AudioSource.MIC, + /* clientFormat= */ null, + /* devFormat= */ null, + /* patchHandle= */ 1000, + "com.android.example"); + configs.add(audioRecordingConfiguration); + } + return configs; + } +}