diff --git a/core/java/android/companion/virtual/VirtualDeviceManager.java b/core/java/android/companion/virtual/VirtualDeviceManager.java index 4d6997076b4ed..57bdf2d5212cd 100644 --- a/core/java/android/companion/virtual/VirtualDeviceManager.java +++ b/core/java/android/companion/virtual/VirtualDeviceManager.java @@ -29,11 +29,14 @@ import android.app.PendingIntent; import android.companion.AssociationInfo; import android.companion.virtual.audio.VirtualAudioDevice; import android.companion.virtual.audio.VirtualAudioDevice.AudioConfigurationChangeCallback; +import android.companion.virtual.camera.VirtualCameraDevice; +import android.companion.virtual.camera.VirtualCameraInput; import android.companion.virtual.sensor.VirtualSensor; import android.companion.virtual.sensor.VirtualSensorConfig; import android.content.ComponentName; import android.content.Context; import android.graphics.Point; +import android.hardware.camera2.CameraCharacteristics; import android.hardware.display.DisplayManager; import android.hardware.display.DisplayManager.VirtualDisplayFlag; import android.hardware.display.DisplayManagerGlobal; @@ -293,9 +296,11 @@ public final class VirtualDeviceManager { } }; @Nullable - private VirtualAudioDevice mVirtualAudioDevice; + private VirtualCameraDevice mVirtualCameraDevice; @NonNull - private List mVirtualSensors = new ArrayList<>(); + private final List mVirtualSensors = new ArrayList<>(); + @Nullable + private VirtualAudioDevice mVirtualAudioDevice; @RequiresPermission(android.Manifest.permission.CREATE_VIRTUAL_DEVICE) private VirtualDevice( @@ -715,6 +720,34 @@ public final class VirtualDeviceManager { return mVirtualAudioDevice; } + /** + * Creates a new virtual camera. If a virtual camera was already created, it will be closed. + * + * @param cameraName name of the virtual camera. + * @param characteristics camera characteristics. + * @param virtualCameraInput callback that provides input to camera. + * @param executor Executor on which camera input will be sent into system. Don't + * use the Main Thread for this executor. + * @return newly created camera; + * + * @hide + */ + @RequiresPermission(android.Manifest.permission.CREATE_VIRTUAL_DEVICE) + @NonNull + public VirtualCameraDevice createVirtualCameraDevice( + @NonNull String cameraName, + @NonNull CameraCharacteristics characteristics, + @NonNull VirtualCameraInput virtualCameraInput, + @NonNull Executor executor) { + if (mVirtualCameraDevice != null) { + mVirtualCameraDevice.close(); + } + int deviceId = getDeviceId(); + mVirtualCameraDevice = new VirtualCameraDevice( + deviceId, cameraName, characteristics, virtualCameraInput, executor); + return mVirtualCameraDevice; + } + /** * Sets the visibility of the pointer icon for this VirtualDevice's associated displays. * diff --git a/core/java/android/companion/virtual/camera/VirtualCameraDevice.java b/core/java/android/companion/virtual/camera/VirtualCameraDevice.java new file mode 100644 index 0000000000000..a7eba873445c7 --- /dev/null +++ b/core/java/android/companion/virtual/camera/VirtualCameraDevice.java @@ -0,0 +1,82 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.companion.virtual.camera; + +import android.hardware.camera2.CameraCharacteristics; + +import androidx.annotation.NonNull; + +import java.util.Locale; +import java.util.Objects; +import java.util.concurrent.Executor; + +/** + * Virtual camera that is used to send image data into system. + * + * @hide + */ +public final class VirtualCameraDevice implements AutoCloseable { + + @NonNull + private final String mCameraDeviceName; + @NonNull + private final CameraCharacteristics mCameraCharacteristics; + @NonNull + private final VirtualCameraOutput mCameraOutput; + private boolean mCameraRegistered = false; + + /** + * VirtualCamera device constructor. + * + * @param virtualDeviceId ID of virtual device to which camera will be added. + * @param cameraName must be unique for each camera per virtual device. + * @param characteristics of camera that will be passed into system in order to describe + * camera. + * @param virtualCameraInput component that provides image data. + * @param executor on which to collect image data and pass it into system. + */ + public VirtualCameraDevice(int virtualDeviceId, @NonNull String cameraName, + @NonNull CameraCharacteristics characteristics, + @NonNull VirtualCameraInput virtualCameraInput, @NonNull Executor executor) { + Objects.requireNonNull(cameraName); + mCameraCharacteristics = Objects.requireNonNull(characteristics); + mCameraDeviceName = generateCameraDeviceName(virtualDeviceId, cameraName); + mCameraOutput = new VirtualCameraOutput(virtualCameraInput, executor); + registerCamera(); + } + + private static String generateCameraDeviceName(int deviceId, @NonNull String cameraName) { + return String.format(Locale.ENGLISH, "%d_%s", deviceId, Objects.requireNonNull(cameraName)); + } + + @Override + public void close() { + if (!mCameraRegistered) { + return; + } + + mCameraOutput.closeStream(); + } + + private void registerCamera() { + if (mCameraRegistered) { + return; + } + + mCameraRegistered = true; + } +} diff --git a/core/java/android/companion/virtual/camera/VirtualCameraInput.java b/core/java/android/companion/virtual/camera/VirtualCameraInput.java new file mode 100644 index 0000000000000..690a64b7fd23d --- /dev/null +++ b/core/java/android/companion/virtual/camera/VirtualCameraInput.java @@ -0,0 +1,54 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.companion.virtual.camera; + +import android.annotation.NonNull; +import android.hardware.camera2.params.InputConfiguration; + +import java.io.InputStream; + +/*** + * Used for sending image data into virtual camera. + *

+ * The system will call {@link #openStream(InputConfiguration)} to signal when you + * should start sending Camera image data. + * When Camera is no longer needed, or there is change in configuration + * {@link #closeStream()} will be called. At that time finish sending current + * image data and then close the stream. + *

+ * If Camera image data is needed again, {@link #openStream(InputConfiguration)} will be + * called by the system. + * + * @hide + */ +public interface VirtualCameraInput { + + /** + * Opens a new image stream for the provided {@link InputConfiguration}. + * + * @param inputConfiguration image data configuration. + * @return image data stream. + */ + @NonNull + InputStream openStream(@NonNull InputConfiguration inputConfiguration); + + /** + * Stop sending image data and close {@link InputStream} provided in {@link + * #openStream(InputConfiguration)}. Do nothing if there is currently no active stream. + */ + void closeStream(); +} diff --git a/core/java/android/companion/virtual/camera/VirtualCameraOutput.java b/core/java/android/companion/virtual/camera/VirtualCameraOutput.java new file mode 100644 index 0000000000000..fa1c3ad23ab84 --- /dev/null +++ b/core/java/android/companion/virtual/camera/VirtualCameraOutput.java @@ -0,0 +1,197 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.companion.virtual.camera; + +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.hardware.camera2.params.InputConfiguration; +import android.os.ParcelFileDescriptor; +import android.util.Log; + +import com.android.internal.annotations.VisibleForTesting; + +import java.io.FileDescriptor; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Objects; +import java.util.concurrent.Executor; + +/** + * Component for providing Camera data to the system. + *

+ * {@link #getStreamDescriptor(InputConfiguration)} will be called by the system when Camera should + * start sending image data. Camera data will continue to be sent into {@link ParcelFileDescriptor} + * until {@link #closeStream()} is called by the system, at which point {@link ParcelFileDescriptor} + * will be closed. + * + * @hide + */ +@VisibleForTesting +public class VirtualCameraOutput { + + private static final String TAG = "VirtualCameraDeviceImpl"; + + @NonNull + private final VirtualCameraInput mVirtualCameraInput; + @NonNull + private final Executor mExecutor; + @Nullable + private VirtualCameraStream mCameraStream; + + @VisibleForTesting + public VirtualCameraOutput(@NonNull VirtualCameraInput cameraInput, + @NonNull Executor executor) { + mVirtualCameraInput = Objects.requireNonNull(cameraInput); + mExecutor = Objects.requireNonNull(executor); + } + + /** + * Get a read Descriptor on which Camera HAL will receive data. At any point in time there can + * exist a maximum of one active {@link ParcelFileDescriptor}. + * Calling this method with a different {@link InputConfiguration} is going to close the + * previously created file descriptor. + * + * @param imageConfiguration for which to create the {@link ParcelFileDescriptor}. + * @return Newly created ParcelFileDescriptor if stream param is different from previous or if + * this is first time call. Will return null if there was an error during Descriptor + * creation process. + */ + @Nullable + @VisibleForTesting + public ParcelFileDescriptor getStreamDescriptor( + @NonNull InputConfiguration imageConfiguration) { + Objects.requireNonNull(imageConfiguration); + + // Reuse same descriptor if stream is the same, otherwise create a new one. + try { + if (mCameraStream == null) { + mCameraStream = new VirtualCameraStream(imageConfiguration, mExecutor); + } else if (!mCameraStream.isSameConfiguration(imageConfiguration)) { + mCameraStream.close(); + mCameraStream = new VirtualCameraStream(imageConfiguration, mExecutor); + } + } catch (IOException exception) { + Log.e(TAG, "Unable to open file descriptor.", exception); + return null; + } + + InputStream imageStream = mVirtualCameraInput.openStream(imageConfiguration); + mCameraStream.startSending(imageStream); + return mCameraStream.getDescriptor(); + } + + /** + * Closes currently opened stream. If there is no stream, do nothing. + */ + @VisibleForTesting + public void closeStream() { + mVirtualCameraInput.closeStream(); + if (mCameraStream != null) { + mCameraStream.close(); + mCameraStream = null; + } + + try { + mVirtualCameraInput.closeStream(); + } catch (Exception e) { + Log.e(TAG, "Error during closing stream.", e); + } + } + + private static class VirtualCameraStream implements AutoCloseable { + + private static final String TAG = "VirtualCameraStream"; + private static final int BUFFER_SIZE = 1024; + + private static final int SENDING_STATE_INITIAL = 0; + private static final int SENDING_STATE_IN_PROGRESS = 1; + private static final int SENDING_STATE_CLOSED = 2; + + @NonNull + private final InputConfiguration mImageConfiguration; + @NonNull + private final Executor mExecutor; + @Nullable + private final ParcelFileDescriptor mReadDescriptor; + @Nullable + private final ParcelFileDescriptor mWriteDescriptor; + private int mSendingState; + + VirtualCameraStream(@NonNull InputConfiguration imageConfiguration, + @NonNull Executor executor) throws IOException { + mSendingState = SENDING_STATE_INITIAL; + mImageConfiguration = Objects.requireNonNull(imageConfiguration); + mExecutor = Objects.requireNonNull(executor); + ParcelFileDescriptor[] parcels = ParcelFileDescriptor.createPipe(); + mReadDescriptor = parcels[0]; + mWriteDescriptor = parcels[1]; + } + + boolean isSameConfiguration(@NonNull InputConfiguration imageConfiguration) { + return mImageConfiguration == Objects.requireNonNull(imageConfiguration); + } + + @Nullable + ParcelFileDescriptor getDescriptor() { + return mReadDescriptor; + } + + public void startSending(@NonNull InputStream inputStream) { + Objects.requireNonNull(inputStream); + + if (mSendingState != SENDING_STATE_INITIAL) { + return; + } + + mSendingState = SENDING_STATE_IN_PROGRESS; + mExecutor.execute(() -> sendData(inputStream)); + } + + @Override + public void close() { + mSendingState = SENDING_STATE_CLOSED; + try { + mReadDescriptor.close(); + } catch (IOException e) { + Log.e(TAG, "Unable to close read descriptor.", e); + } + try { + mWriteDescriptor.close(); + } catch (IOException e) { + Log.e(TAG, "Unable to close write descriptor.", e); + } + } + + private void sendData(@NonNull InputStream inputStream) { + Objects.requireNonNull(inputStream); + + byte[] buffer = new byte[BUFFER_SIZE]; + FileDescriptor fd = mWriteDescriptor.getFileDescriptor(); + try (FileOutputStream outputStream = new FileOutputStream(fd)) { + while (mSendingState == SENDING_STATE_IN_PROGRESS) { + int bytesRead = inputStream.read(buffer, 0, BUFFER_SIZE); + if (bytesRead < 1) continue; + + outputStream.write(buffer, 0, bytesRead); + } + } catch (IOException e) { + Log.e(TAG, "Error while sending camera data.", e); + } + } + } +} diff --git a/core/tests/coretests/src/android/companion/virtual/camera/VirtualCameraOutputTest.java b/core/tests/coretests/src/android/companion/virtual/camera/VirtualCameraOutputTest.java new file mode 100644 index 0000000000000..694b3128998c8 --- /dev/null +++ b/core/tests/coretests/src/android/companion/virtual/camera/VirtualCameraOutputTest.java @@ -0,0 +1,128 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.companion.virtual.camera; + +import static com.google.common.truth.Truth.assertThat; + +import static org.junit.Assert.fail; + +import android.graphics.PixelFormat; +import android.hardware.camera2.params.InputConfiguration; +import android.os.ParcelFileDescriptor; +import android.util.Log; + +import androidx.annotation.NonNull; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +public class VirtualCameraOutputTest { + + private static final String TAG = "VirtualCameraOutputTest"; + + private ExecutorService mExecutor; + + private InputConfiguration mConfiguration; + + @Before + public void setUp() { + mExecutor = Executors.newSingleThreadExecutor(); + mConfiguration = new InputConfiguration(64, 64, PixelFormat.RGB_888); + } + + @After + public void cleanUp() { + mExecutor.shutdownNow(); + } + + @Test + public void createStreamDescriptor_successfulDataStream() { + byte[] cameraData = new byte[]{1, 2, 3, 4, 5}; + VirtualCameraInput input = createCameraInput(cameraData); + VirtualCameraOutput output = new VirtualCameraOutput(input, mExecutor); + ParcelFileDescriptor descriptor = output.getStreamDescriptor(mConfiguration); + + try (FileInputStream fis = new FileInputStream(descriptor.getFileDescriptor())) { + byte[] receivedData = fis.readNBytes(cameraData.length); + + output.closeStream(); + assertThat(receivedData).isEqualTo(cameraData); + } catch (IOException exception) { + fail("Unable to read bytes from FileInputStream. Message: " + exception.getMessage()); + } + } + + @Test + public void createStreamDescriptor_multipleCallsSameStream() { + VirtualCameraInput input = createCameraInput(new byte[]{0}); + VirtualCameraOutput output = new VirtualCameraOutput(input, mExecutor); + + ParcelFileDescriptor firstDescriptor = output.getStreamDescriptor(mConfiguration); + ParcelFileDescriptor secondDescriptor = output.getStreamDescriptor(mConfiguration); + + assertThat(firstDescriptor).isSameInstanceAs(secondDescriptor); + } + + @Test + public void createStreamDescriptor_differentStreams() { + VirtualCameraInput input = createCameraInput(new byte[]{0}); + VirtualCameraOutput callback = new VirtualCameraOutput(input, mExecutor); + + InputConfiguration differentConfig = new InputConfiguration(mConfiguration.getWidth() + 1, + mConfiguration.getHeight() + 1, mConfiguration.getFormat()); + + ParcelFileDescriptor firstDescriptor = callback.getStreamDescriptor(mConfiguration); + ParcelFileDescriptor secondDescriptor = callback.getStreamDescriptor(differentConfig); + + assertThat(firstDescriptor).isNotSameInstanceAs(secondDescriptor); + } + + private VirtualCameraInput createCameraInput(byte[] data) { + return new VirtualCameraInput() { + private ByteArrayInputStream mInputStream = null; + + @Override + @NonNull + public InputStream openStream(@NonNull InputConfiguration inputConfiguration) { + closeStream(); + mInputStream = new ByteArrayInputStream(data); + return mInputStream; + } + + @Override + public void closeStream() { + if (mInputStream == null) { + return; + } + try { + mInputStream.close(); + } catch (IOException e) { + Log.e(TAG, "Unable to close image stream.", e); + } + mInputStream = null; + } + }; + } +}