Merge changes I4f225864,Ic95b7014 into tm-qpr-dev

* changes:
  Include initialAudio size in metrics and logs
  Add initialAudio field to HotwordAudioStream

core/java/android/service/voice/HotwordAudioStream.java

@@ -27,6 +27,7 @@ import android.os.ParcelFileDescriptor;
 import android.os.Parcelable;
 import android.os.PersistableBundle;
 
+import java.util.Arrays;
 import java.util.Objects;
 
 /**
@@ -59,8 +60,17 @@ public final class HotwordAudioStream implements Parcelable {
     private final AudioFormat mAudioFormat;
 
     /**
-     * This stream starts with the audio bytes used for hotword detection, but continues streaming
+     * This stream typically starts with the audio bytes used for hotword detection, but continues
-     * the audio until the stream is shutdown by the {@link HotwordDetectionService}.
+     * streaming the audio (e.g., with the query) until the stream is shutdown by the
+     * {@link HotwordDetectionService}. The data format is expected to match
+     * {@link #getAudioFormat()}.
+     *
+     * <p>
+     * Alternatively, the {@link HotwordDetectionService} may use {@link #getInitialAudio()}
+     * to pass the start of the audio instead of streaming it here. This may prevent added latency
+     * caused by the streaming buffer (see {@link #KEY_AUDIO_STREAM_COPY_BUFFER_LENGTH_BYTES}) not
+     * being large enough to handle this initial chunk of audio.
+     * </p>
      */
     @NonNull
     @UnsupportedAppUsage
@@ -138,6 +148,32 @@ public final class HotwordAudioStream implements Parcelable {
         }
     }
 
+    /**
+     * The start of the audio used for hotword detection. The data format is expected to match
+     * {@link #getAudioFormat()}.
+     *
+     * <p>
+     * The {@link HotwordDetectionService} may use this instead of using
+     * {@link #getAudioStreamParcelFileDescriptor()} to stream these initial bytes of audio. This
+     * may prevent added latency caused by the streaming buffer (see
+     * {@link #KEY_AUDIO_STREAM_COPY_BUFFER_LENGTH_BYTES}) not being large enough to handle this
+     * initial chunk of audio.
+     * </p>
+     */
+    @NonNull
+    @UnsupportedAppUsage
+    private final byte[] mInitialAudio;
+
+    private static final byte[] DEFAULT_INITIAL_EMPTY_AUDIO = {};
+
+    private static byte[] defaultInitialAudio() {
+        return DEFAULT_INITIAL_EMPTY_AUDIO;
+    }
+
+    private String initialAudioToString() {
+        return "length=" + mInitialAudio.length;
+    }
+
     /**
      * Provides an instance of {@link Builder} with state corresponding to this instance.
      * @hide
@@ -145,7 +181,8 @@ public final class HotwordAudioStream implements Parcelable {
     public Builder buildUpon() {
         return new Builder(mAudioFormat, mAudioStreamParcelFileDescriptor)
             .setTimestamp(mTimestamp)
-            .setMetadata(mMetadata);
+            .setMetadata(mMetadata)
+            .setInitialAudio(mInitialAudio);
     }
 
     /* package-private */
@@ -153,7 +190,8 @@ public final class HotwordAudioStream implements Parcelable {
             @NonNull AudioFormat audioFormat,
             @NonNull ParcelFileDescriptor audioStreamParcelFileDescriptor,
             @Nullable AudioTimestamp timestamp,
-            @NonNull PersistableBundle metadata) {
+            @NonNull PersistableBundle metadata,
+            @NonNull byte[] initialAudio) {
         this.mAudioFormat = audioFormat;
         com.android.internal.util.AnnotationValidations.validate(
                 NonNull.class, null, mAudioFormat);
@@ -164,6 +202,9 @@ public final class HotwordAudioStream implements Parcelable {
         this.mMetadata = metadata;
         com.android.internal.util.AnnotationValidations.validate(
                 NonNull.class, null, mMetadata);
+        this.mInitialAudio = initialAudio;
+        com.android.internal.util.AnnotationValidations.validate(
+                NonNull.class, null, mInitialAudio);
 
         // onConstructed(); // You can define this method to get a callback
     }
@@ -178,8 +219,17 @@ public final class HotwordAudioStream implements Parcelable {
     }
 
     /**
-     * This stream starts with the audio bytes used for hotword detection, but continues streaming
+     * This stream typically starts with the audio bytes used for hotword detection, but continues
-     * the audio until the stream is shutdown by the {@link HotwordDetectionService}.
+     * streaming the audio (e.g., with the query) until the stream is shutdown by the
+     * {@link HotwordDetectionService}. The data format is expected to match
+     * {@link #getAudioFormat()}.
+     *
+     * <p>
+     * Alternatively, the {@link HotwordDetectionService} may use {@link #getInitialAudio()}
+     * to pass the start of the audio instead of streaming it here. This may prevent added latency
+     * caused by the streaming buffer (see {@link #KEY_AUDIO_STREAM_COPY_BUFFER_LENGTH_BYTES}) not
+     * being large enough to handle this initial chunk of audio.
+     * </p>
      */
     @UnsupportedAppUsage
     @NonNull
@@ -220,6 +270,24 @@ public final class HotwordAudioStream implements Parcelable {
         return mMetadata;
     }
 
+    /**
+     * The start of the audio used for hotword detection. The data format is expected to match
+     * {@link #getAudioFormat()}.
+     *
+     * <p>
+     * The {@link HotwordDetectionService} may use this instead of using
+     * {@link #getAudioStreamParcelFileDescriptor()} to stream these initial bytes of audio. This
+     * may prevent added latency caused by the streaming buffer (see
+     * {@link #KEY_AUDIO_STREAM_COPY_BUFFER_LENGTH_BYTES}) not being large enough to handle this
+     * initial chunk of audio.
+     * </p>
+     */
+    @UnsupportedAppUsage
+    @NonNull
+    public byte[] getInitialAudio() {
+        return mInitialAudio;
+    }
+
     @Override
     public String toString() {
         // You can override field toString logic by defining methods like:
@@ -229,7 +297,8 @@ public final class HotwordAudioStream implements Parcelable {
                 + "audioFormat = " + mAudioFormat + ", "
                 + "audioStreamParcelFileDescriptor = " + mAudioStreamParcelFileDescriptor + ", "
                 + "timestamp = " + timestampToString() + ", "
-                + "metadata = " + mMetadata + " }";
+                + "metadata = " + mMetadata + ", "
+                + "initialAudio = " + initialAudioToString() + " }";
     }
 
     @Override
@@ -247,7 +316,8 @@ public final class HotwordAudioStream implements Parcelable {
                 && Objects.equals(mAudioStreamParcelFileDescriptor,
                 that.mAudioStreamParcelFileDescriptor)
                 && Objects.equals(mTimestamp, that.mTimestamp)
-                && Objects.equals(mMetadata, that.mMetadata);
+                && Objects.equals(mMetadata, that.mMetadata)
+                && Arrays.equals(mInitialAudio, that.mInitialAudio);
     }
 
     @Override
@@ -260,6 +330,7 @@ public final class HotwordAudioStream implements Parcelable {
         _hash = 31 * _hash + Objects.hashCode(mAudioStreamParcelFileDescriptor);
         _hash = 31 * _hash + Objects.hashCode(mTimestamp);
         _hash = 31 * _hash + Objects.hashCode(mMetadata);
+        _hash = 31 * _hash + Arrays.hashCode(mInitialAudio);
         return _hash;
     }
 
@@ -275,6 +346,7 @@ public final class HotwordAudioStream implements Parcelable {
         dest.writeTypedObject(mAudioStreamParcelFileDescriptor, flags);
         parcelTimestamp(dest, flags);
         dest.writeTypedObject(mMetadata, flags);
+        dest.writeByteArray(mInitialAudio);
     }
 
     @Override
@@ -296,6 +368,7 @@ public final class HotwordAudioStream implements Parcelable {
         AudioTimestamp timestamp = unparcelTimestamp(in);
         PersistableBundle metadata = (PersistableBundle) in.readTypedObject(
                 PersistableBundle.CREATOR);
+        byte[] initialAudio = in.createByteArray();
 
         this.mAudioFormat = audioFormat;
         com.android.internal.util.AnnotationValidations.validate(
@@ -307,6 +380,9 @@ public final class HotwordAudioStream implements Parcelable {
         this.mMetadata = metadata;
         com.android.internal.util.AnnotationValidations.validate(
                 NonNull.class, null, mMetadata);
+        this.mInitialAudio = initialAudio;
+        com.android.internal.util.AnnotationValidations.validate(
+                NonNull.class, null, mInitialAudio);
 
         // onConstructed(); // You can define this method to get a callback
     }
@@ -339,17 +415,29 @@ public final class HotwordAudioStream implements Parcelable {
         private AudioTimestamp mTimestamp;
         @NonNull
         private PersistableBundle mMetadata;
+        @NonNull
+        private byte[] mInitialAudio;
 
         private long mBuilderFieldsSet = 0L;
 
         /**
          * Creates a new Builder.
          *
-         * @param audioFormat                     The {@link AudioFormat} of the audio stream.
+         * @param audioFormat
-         * @param audioStreamParcelFileDescriptor This stream starts with the audio bytes used for
+         *   The {@link AudioFormat} of the audio stream.
-         *                                        hotword detection, but continues streaming
+         * @param audioStreamParcelFileDescriptor
-         *                                        the audio until the stream is shutdown by the
+         *   This stream typically starts with the audio bytes used for hotword detection, but
-         *                                        {@link HotwordDetectionService}.
+         *   continues streaming the audio (e.g., with the query) until the stream is shutdown by
+         *   the {@link HotwordDetectionService}. The data format is expected to match
+         *   {@link #getAudioFormat()}.
+         *
+         *   <p>
+         *   Alternatively, the {@link HotwordDetectionService} may use {@link #getInitialAudio()}
+         *   to pass the start of the audio instead of streaming it here. This may prevent added
+         *   latency caused by the streaming buffer
+         *   (see {@link #KEY_AUDIO_STREAM_COPY_BUFFER_LENGTH_BYTES}) not being large enough to
+         *   handle this initial chunk of audio.
+         *   </p>
          */
         @UnsupportedAppUsage
         public Builder(
@@ -376,9 +464,18 @@ public final class HotwordAudioStream implements Parcelable {
         }
 
         /**
-         * This stream starts with the audio bytes used for hotword detection, but continues
+         * This stream typically starts with the audio bytes used for hotword detection, but
-         * streaming
+         * continues streaming the audio (e.g., with the query) until the stream is shutdown by the
-         * the audio until the stream is shutdown by the {@link HotwordDetectionService}.
+         * {@link HotwordDetectionService}. The data format is expected to match
+         * {@link #getAudioFormat()}.
+         *
+         * <p>
+         * Alternatively, the {@link HotwordDetectionService} may use {@link #getInitialAudio()}
+         * to pass the start of the audio instead of streaming it here. This may prevent added
+         * latency caused by the streaming buffer
+         * (see {@link #KEY_AUDIO_STREAM_COPY_BUFFER_LENGTH_BYTES}) not being large enough to handle
+         * this initial chunk of audio.
+         * </p>
          */
         @UnsupportedAppUsage
         @NonNull
@@ -428,12 +525,33 @@ public final class HotwordAudioStream implements Parcelable {
             return this;
         }
 
+        /**
+         * The start of the audio used for hotword detection. The data format is expected to match
+         * {@link #getAudioFormat()}.
+         *
+         * <p>
+         * The {@link HotwordDetectionService} may use this instead of using
+         * {@link #getAudioStreamParcelFileDescriptor()} to stream these initial bytes of audio.
+         * This may prevent added latency caused by the streaming buffer (see
+         * {@link #KEY_AUDIO_STREAM_COPY_BUFFER_LENGTH_BYTES}) not being large enough to handle this
+         * initial chunk of audio.
+         * </p>
+         */
+        @UnsupportedAppUsage
+        @NonNull
+        public Builder setInitialAudio(@NonNull byte[] value) {
+            checkNotUsed();
+            mBuilderFieldsSet |= 0x10;
+            mInitialAudio = value;
+            return this;
+        }
+
         /** Builds the instance. This builder should not be touched after calling this! */
         @UnsupportedAppUsage
         @NonNull
         public HotwordAudioStream build() {
             checkNotUsed();
-            mBuilderFieldsSet |= 0x10; // Mark builder used
+            mBuilderFieldsSet |= 0x20; // Mark builder used
 
             if ((mBuilderFieldsSet & 0x4) == 0) {
                 mTimestamp = defaultTimestamp();
@@ -441,16 +559,20 @@ public final class HotwordAudioStream implements Parcelable {
             if ((mBuilderFieldsSet & 0x8) == 0) {
                 mMetadata = defaultMetadata();
             }
+            if ((mBuilderFieldsSet & 0x10) == 0) {
+                mInitialAudio = defaultInitialAudio();
+            }
             HotwordAudioStream o = new HotwordAudioStream(
                     mAudioFormat,
                     mAudioStreamParcelFileDescriptor,
                     mTimestamp,
-                    mMetadata);
+                    mMetadata,
+                    mInitialAudio);
             return o;
         }
 
         private void checkNotUsed() {
-            if ((mBuilderFieldsSet & 0x10) != 0) {
+            if ((mBuilderFieldsSet & 0x20) != 0) {
                 throw new IllegalStateException(
                         "This Builder should not be reused. Use a new Builder instance instead");
             }

services/voiceinteraction/java/com/android/server/voiceinteraction/HotwordAudioStreamCopier.java

@@ -109,6 +109,7 @@ final class HotwordAudioStreamCopier {
         List<HotwordAudioStream> newAudioStreams = new ArrayList<>(audioStreams.size());
         List<CopyTaskInfo> copyTaskInfos = new ArrayList<>(audioStreams.size());
         int totalMetadataBundleSizeBytes = 0;
+        int totalInitialAudioSizeBytes = 0;
         for (HotwordAudioStream audioStream : audioStreams) {
             ParcelFileDescriptor[] clientPipe = ParcelFileDescriptor.createReliablePipe();
             ParcelFileDescriptor clientAudioSource = clientPipe[0];
@@ -137,6 +138,10 @@ final class HotwordAudioStreamCopier {
                 }
             }
 
+            // We are including the non-streamed initial audio
+            // (HotwordAudioStream.getInitialAudio()) bytes in the "stream" size metrics.
+            totalInitialAudioSizeBytes += audioStream.getInitialAudio().length;
+
             ParcelFileDescriptor serviceAudioSource =
                     audioStream.getAudioStreamParcelFileDescriptor();
             copyTaskInfos.add(new CopyTaskInfo(serviceAudioSource, clientAudioSink,
@@ -146,7 +151,7 @@ final class HotwordAudioStreamCopier {
         String resultTaskId = TASK_ID_PREFIX + System.identityHashCode(result);
         mExecutorService.execute(
                 new HotwordDetectedResultCopyTask(resultTaskId, copyTaskInfos,
-                        totalMetadataBundleSizeBytes));
+                        totalMetadataBundleSizeBytes, totalInitialAudioSizeBytes));
 
         return result.buildUpon().setAudioStreams(newAudioStreams).build();
     }
@@ -167,13 +172,15 @@ final class HotwordAudioStreamCopier {
         private final String mResultTaskId;
         private final List<CopyTaskInfo> mCopyTaskInfos;
         private final int mTotalMetadataSizeBytes;
+        private final int mTotalInitialAudioSizeBytes;
         private final ExecutorService mExecutorService = Executors.newCachedThreadPool();
 
         HotwordDetectedResultCopyTask(String resultTaskId, List<CopyTaskInfo> copyTaskInfos,
-                int totalMetadataSizeBytes) {
+                int totalMetadataSizeBytes, int totalInitialAudioSizeBytes) {
             mResultTaskId = resultTaskId;
             mCopyTaskInfos = copyTaskInfos;
             mTotalMetadataSizeBytes = totalMetadataSizeBytes;
+            mTotalInitialAudioSizeBytes = totalInitialAudioSizeBytes;
         }
 
         @Override
@@ -195,25 +202,30 @@ final class HotwordAudioStreamCopier {
                 try {
                     HotwordMetricsLogger.writeAudioEgressEvent(mDetectorType,
                             HOTWORD_AUDIO_EGRESS_EVENT_REPORTED__EVENT__STARTED,
-                            mVoiceInteractorUid, /* streamSizeBytes= */ 0, mTotalMetadataSizeBytes,
+                            mVoiceInteractorUid, mTotalInitialAudioSizeBytes,
-                            size);
+                            mTotalMetadataSizeBytes, size);
                     // TODO(b/244599891): Set timeout, close after inactivity
                     mExecutorService.invokeAll(tasks);
 
-                    int totalStreamSizeBytes = 0;
+                    // We are including the non-streamed initial audio
+                    // (HotwordAudioStream.getInitialAudio()) bytes in the "stream" size metrics.
+                    int totalStreamSizeBytes = mTotalInitialAudioSizeBytes;
                     for (SingleAudioStreamCopyTask task : tasks) {
                         totalStreamSizeBytes += task.mTotalCopiedBytes;
                     }
 
-                    Slog.i(TAG, mResultTaskId + ": Task was completed. Total bytes streamed: "
+                    Slog.i(TAG, mResultTaskId + ": Task was completed. Total bytes egressed: "
-                            + totalStreamSizeBytes + ", total metadata bundle size bytes: "
+                            + totalStreamSizeBytes + " (including " + mTotalInitialAudioSizeBytes
+                            + " bytes NOT streamed), total metadata bundle size bytes: "
                             + mTotalMetadataSizeBytes);
                     HotwordMetricsLogger.writeAudioEgressEvent(mDetectorType,
                             HOTWORD_AUDIO_EGRESS_EVENT_REPORTED__EVENT__ENDED,
                             mVoiceInteractorUid, totalStreamSizeBytes, mTotalMetadataSizeBytes,
                             size);
                 } catch (InterruptedException e) {
-                    int totalStreamSizeBytes = 0;
+                    // We are including the non-streamed initial audio
+                    // (HotwordAudioStream.getInitialAudio()) bytes in the "stream" size metrics.
+                    int totalStreamSizeBytes = mTotalInitialAudioSizeBytes;
                     for (SingleAudioStreamCopyTask task : tasks) {
                         totalStreamSizeBytes += task.mTotalCopiedBytes;
                     }
@@ -222,8 +234,9 @@ final class HotwordAudioStreamCopier {
                             HOTWORD_AUDIO_EGRESS_EVENT_REPORTED__EVENT__INTERRUPTED_EXCEPTION,
                             mVoiceInteractorUid, totalStreamSizeBytes, mTotalMetadataSizeBytes,
                             size);
-                    Slog.e(TAG, mResultTaskId + ": Task was interrupted. Total bytes streamed: "
+                    Slog.i(TAG, mResultTaskId + ": Task was interrupted. Total bytes egressed: "
-                            + totalStreamSizeBytes + ", total metadata bundle size bytes: "
+                            + totalStreamSizeBytes + " (including " + mTotalInitialAudioSizeBytes
+                            + " bytes NOT streamed), total metadata bundle size bytes: "
                             + mTotalMetadataSizeBytes);
                     bestEffortPropagateError(e.getMessage());
                 } finally {