Merge "cameraservice: Add metrics for extension sessions" into udc-dev

core/java/android/hardware/CameraSessionStats.java

@@ -65,6 +65,7 @@ public class CameraSessionStats implements Parcelable {
     private String mUserTag;
     private int mVideoStabilizationMode;
     private int mSessionIndex;
+    private CameraExtensionSessionStats mCameraExtensionSessionStats;
 
     public CameraSessionStats() {
         mFacing = -1;
@@ -82,6 +83,7 @@ public class CameraSessionStats implements Parcelable {
         mStreamStats = new ArrayList<CameraStreamStats>();
         mVideoStabilizationMode = -1;
         mSessionIndex = 0;
+        mCameraExtensionSessionStats = new CameraExtensionSessionStats();
     }
 
     public CameraSessionStats(String cameraId, int facing, int newCameraState,
@@ -101,6 +103,7 @@ public class CameraSessionStats implements Parcelable {
         mInternalReconfigure = internalReconfigure;
         mStreamStats = new ArrayList<CameraStreamStats>();
         mSessionIndex = sessionIdx;
+        mCameraExtensionSessionStats = new CameraExtensionSessionStats();
     }
 
     public static final @android.annotation.NonNull Parcelable.Creator<CameraSessionStats> CREATOR =
@@ -145,6 +148,7 @@ public class CameraSessionStats implements Parcelable {
         dest.writeString(mUserTag);
         dest.writeInt(mVideoStabilizationMode);
         dest.writeInt(mSessionIndex);
+        mCameraExtensionSessionStats.writeToParcel(dest, 0);
     }
 
     public void readFromParcel(Parcel in) {
@@ -170,6 +174,7 @@ public class CameraSessionStats implements Parcelable {
         mUserTag = in.readString();
         mVideoStabilizationMode = in.readInt();
         mSessionIndex = in.readInt();
+        mCameraExtensionSessionStats = CameraExtensionSessionStats.CREATOR.createFromParcel(in);
     }
 
     public String getCameraId() {
@@ -243,4 +248,8 @@ public class CameraSessionStats implements Parcelable {
     public int getSessionIndex() {
         return mSessionIndex;
     }
+
+    public CameraExtensionSessionStats getExtensionSessionStats() {
+        return mCameraExtensionSessionStats;
+    }
 }

core/java/android/hardware/camera2/CameraManager.java

@@ -30,6 +30,7 @@ import android.compat.annotation.Overridable;
 import android.content.Context;
 import android.content.pm.PackageManager;
 import android.graphics.Point;
+import android.hardware.CameraExtensionSessionStats;
 import android.hardware.CameraStatus;
 import android.hardware.ICameraService;
 import android.hardware.ICameraServiceListener;
@@ -1726,6 +1727,30 @@ public final class CameraManager {
         }
     }
 
+    /**
+     * Reports {@link CameraExtensionSessionStats} to the {@link ICameraService} to be logged for
+     * currently active session. Validation is done downstream.
+     *
+     * @param extStats Extension Session stats to be logged by cameraservice
+     *
+     * @return the key to be used with the next call.
+     *         See {@link ICameraService#reportExtensionSessionStats}.
+     * @hide
+     */
+    public static String reportExtensionSessionStats(CameraExtensionSessionStats extStats) {
+        ICameraService cameraService = CameraManagerGlobal.get().getCameraService();
+        if (cameraService == null) {
+            Log.e(TAG, "CameraService not available. Not reporting extension stats.");
+            return "";
+        }
+        try {
+            return cameraService.reportExtensionSessionStats(extStats);
+        } catch (RemoteException e) {
+            Log.e(TAG, "Failed to report extension session stats to cameraservice.", e);
+        }
+        return "";
+    }
+
     /**
      * A per-process global camera manager instance, to retain a connection to the camera service,
      * and to distribute camera availability notices to API-registered callbacks

core/java/android/hardware/camera2/impl/CameraAdvancedExtensionSessionImpl.java

@@ -53,6 +53,7 @@ import android.hardware.camera2.params.DynamicRangeProfiles;
 import android.hardware.camera2.params.ExtensionSessionConfiguration;
 import android.hardware.camera2.params.OutputConfiguration;
 import android.hardware.camera2.params.SessionConfiguration;
+import android.hardware.camera2.utils.ExtensionSessionStatsAggregator;
 import android.hardware.camera2.utils.SurfaceUtils;
 import android.media.Image;
 import android.media.ImageReader;
@@ -96,6 +97,7 @@ public final class CameraAdvancedExtensionSessionImpl extends CameraExtensionSes
     private CameraCaptureSession mCaptureSession = null;
     private ISessionProcessorImpl mSessionProcessor = null;
     private final InitializeSessionHandler mInitializeHandler;
+    private final ExtensionSessionStatsAggregator mStatsAggregator;
 
     private boolean mInitialized;
 
@@ -205,6 +207,10 @@ public final class CameraAdvancedExtensionSessionImpl extends CameraExtensionSes
                 extender, cameraDevice, characteristicsMapNative, repeatingRequestSurface,
                 burstCaptureSurface, postviewSurface, config.getStateCallback(),
                 config.getExecutor(), sessionId);
+
+        ret.mStatsAggregator.setClientName(ctx.getOpPackageName());
+        ret.mStatsAggregator.setExtensionType(config.getExtension());
+
         ret.initialize();
 
         return ret;
@@ -234,6 +240,9 @@ public final class CameraAdvancedExtensionSessionImpl extends CameraExtensionSes
         mInitializeHandler = new InitializeSessionHandler();
         mSessionId = sessionId;
         mInterfaceLock = cameraDevice.mInterfaceLock;
+
+        mStatsAggregator = new ExtensionSessionStatsAggregator(mCameraDevice.getId(),
+                /*isAdvanced=*/true);
     }
 
     /**
@@ -523,11 +532,26 @@ public final class CameraAdvancedExtensionSessionImpl extends CameraExtensionSes
                     Log.e(TAG, "Failed to stop the repeating request or end the session,"
                             + " , extension service does not respond!") ;
                 }
+                // Commit stats before closing the capture session
+                mStatsAggregator.commit(/*isFinal*/true);
                 mCaptureSession.close();
             }
         }
     }
 
+    /**
+     * Called by {@link CameraDeviceImpl} right before the capture session is closed, and before it
+     * calls {@link #release}
+     */
+    public void commitStats() {
+        synchronized (mInterfaceLock) {
+            if (mInitialized) {
+                // Only commit stats if a capture session was initialized
+                mStatsAggregator.commit(/*isFinal*/true);
+            }
+        }
+    }
+
     public void release(boolean skipCloseNotification) {
         boolean notifyClose = false;
 
@@ -608,6 +632,8 @@ public final class CameraAdvancedExtensionSessionImpl extends CameraExtensionSes
         public void onConfigured(@NonNull CameraCaptureSession session) {
             synchronized (mInterfaceLock) {
                 mCaptureSession = session;
+                // Commit basic stats as soon as the capture session is created
+                mStatsAggregator.commit(/*isFinal*/false);
             }
 
             try {

core/java/android/hardware/camera2/impl/CameraDeviceImpl.java

@@ -700,6 +700,14 @@ public class CameraDeviceImpl extends CameraDevice
                         + " input configuration yet.");
             }
 
+            if (mCurrentExtensionSession != null) {
+                mCurrentExtensionSession.commitStats();
+            }
+
+            if (mCurrentAdvancedExtensionSession != null) {
+                mCurrentAdvancedExtensionSession.commitStats();
+            }
+
             // Notify current session that it's going away, before starting camera operations
             // After this call completes, the session is not allowed to call into CameraDeviceImpl
             if (mCurrentSession != null) {
@@ -1414,6 +1422,15 @@ public class CameraDeviceImpl extends CameraDevice
                 mOfflineSwitchService = null;
             }
 
+            // Let extension sessions commit stats before disconnecting remoteDevice
+            if (mCurrentExtensionSession != null) {
+                mCurrentExtensionSession.commitStats();
+            }
+
+            if (mCurrentAdvancedExtensionSession != null) {
+                mCurrentAdvancedExtensionSession.commitStats();
+            }
+
             if (mRemoteDevice != null) {
                 mRemoteDevice.disconnect();
                 mRemoteDevice.unlinkToDeath(this, /*flags*/0);

core/java/android/hardware/camera2/impl/CameraExtensionSessionImpl.java

@@ -30,7 +30,6 @@ import android.hardware.camera2.CameraCharacteristics;
 import android.hardware.camera2.CameraDevice;
 import android.hardware.camera2.CameraExtensionCharacteristics;
 import android.hardware.camera2.CameraExtensionSession;
-import android.hardware.camera2.CameraManager;
 import android.hardware.camera2.CaptureFailure;
 import android.hardware.camera2.CaptureRequest;
 import android.hardware.camera2.CaptureResult;
@@ -49,6 +48,7 @@ import android.hardware.camera2.params.DynamicRangeProfiles;
 import android.hardware.camera2.params.ExtensionSessionConfiguration;
 import android.hardware.camera2.params.OutputConfiguration;
 import android.hardware.camera2.params.SessionConfiguration;
+import android.hardware.camera2.utils.ExtensionSessionStatsAggregator;
 import android.hardware.camera2.utils.SurfaceUtils;
 import android.media.Image;
 import android.media.ImageReader;
@@ -90,6 +90,7 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession {
     private final int mSessionId;
     private final Set<CaptureRequest.Key> mSupportedRequestKeys;
     private final Set<CaptureResult.Key> mSupportedResultKeys;
+    private final ExtensionSessionStatsAggregator mStatsAggregator;
     private boolean mCaptureResultsSupported;
 
     private CameraCaptureSession mCaptureSession = null;
@@ -242,6 +243,9 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession {
                 extensionChars.getAvailableCaptureRequestKeys(config.getExtension()),
                 extensionChars.getAvailableCaptureResultKeys(config.getExtension()));
 
+        session.mStatsAggregator.setClientName(ctx.getOpPackageName());
+        session.mStatsAggregator.setExtensionType(config.getExtension());
+
         session.initialize();
 
         return session;
@@ -280,6 +284,9 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession {
         mSupportedResultKeys = resultKeys;
         mCaptureResultsSupported = !resultKeys.isEmpty();
         mInterfaceLock = cameraDevice.mInterfaceLock;
+
+        mStatsAggregator = new ExtensionSessionStatsAggregator(mCameraDevice.getId(),
+                /*isAdvanced=*/false);
     }
 
     private void initializeRepeatingRequestPipeline() throws RemoteException {
@@ -793,11 +800,27 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession {
                             new CloseRequestHandler(mRepeatingRequestImageCallback), mHandler);
                 }
 
+                mStatsAggregator.commit(/*isFinal*/true); // Commit stats before closing session
                 mCaptureSession.close();
             }
         }
     }
 
+    /**
+     * Called by {@link CameraDeviceImpl} right before the capture session is closed, and before it
+     * calls {@link #release}
+     *
+     * @hide
+     */
+    public void commitStats() {
+        synchronized (mInterfaceLock) {
+            if (mInitialized) {
+                // Only commit stats if a capture session was initialized
+                mStatsAggregator.commit(/*isFinal*/true);
+            }
+        }
+    }
+
     private void setInitialCaptureRequest(List<CaptureStageImpl> captureStageList,
                                           InitialRequestHandler requestHandler)
             throws CameraAccessException {
@@ -955,6 +978,8 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession {
         public void onConfigured(@NonNull CameraCaptureSession session) {
             synchronized (mInterfaceLock) {
                 mCaptureSession = session;
+                // Commit basic stats as soon as the capture session is created
+                mStatsAggregator.commit(/*isFinal*/false);
                 try {
                     finishPipelineInitialization();
                     CameraExtensionCharacteristics.initializeSession(mInitializeHandler);

core/java/android/hardware/camera2/utils/ExtensionSessionStatsAggregator.java

@@ -0,0 +1,121 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.hardware.camera2.utils;
+
+import android.annotation.NonNull;
+import android.hardware.CameraExtensionSessionStats;
+import android.hardware.camera2.CameraManager;
+import android.util.Log;
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+/**
+ * Utility class to aggregate metrics specific to Camera Extensions and pass them to
+ * {@link CameraManager}. {@link android.hardware.camera2.CameraExtensionSession} should call
+ * {@link #commit} before closing the session.
+ *
+ * @hide
+ */
+public class ExtensionSessionStatsAggregator {
+    private static final boolean DEBUG = false;
+    private static final String TAG = ExtensionSessionStatsAggregator.class.getSimpleName();
+
+    private final ExecutorService mExecutor = Executors.newSingleThreadExecutor();
+
+    private final Object mLock = new Object(); // synchronizes access to all fields of the class
+    private boolean mIsDone = false; // marks the aggregator as "done".
+                                     // Mutations and commits become no-op if this is true.
+    private final CameraExtensionSessionStats mStats;
+
+    public ExtensionSessionStatsAggregator(@NonNull String cameraId, boolean isAdvanced) {
+        if (DEBUG) {
+            Log.v(TAG, "Creating new Extension Session Stats Aggregator");
+        }
+        mStats = new CameraExtensionSessionStats();
+        mStats.key = "";
+        mStats.cameraId = cameraId;
+        mStats.isAdvanced = isAdvanced;
+    }
+
+    /**
+     * Set client package name
+     *
+     * @param clientName package name of the client that these stats are associated with.
+     */
+    public void setClientName(@NonNull String clientName) {
+        synchronized (mLock) {
+            if (mIsDone) {
+                return;
+            }
+            if (DEBUG) {
+                Log.v(TAG, "Setting clientName: " + clientName);
+            }
+            mStats.clientName = clientName;
+        }
+    }
+
+    /**
+     * Set extension type.
+     *
+     * @param extensionType Type of extension. Must match one of
+     *                      {@code CameraExtensionCharacteristics#EXTENSION_*}
+     */
+    public void setExtensionType(int extensionType) {
+        synchronized (mLock) {
+            if (mIsDone) {
+                return;
+            }
+            if (DEBUG) {
+                Log.v(TAG, "Setting type: " + extensionType);
+            }
+            mStats.type = extensionType;
+        }
+    }
+
+    /**
+     * Asynchronously commits the stats to CameraManager on a background thread.
+     *
+     * @param isFinal marks the stats as final and prevents any further commits or changes. This
+     *                should be set to true when the stats are considered final for logging,
+     *                for example right before the capture session is about to close
+     */
+    public void commit(boolean isFinal) {
+        // Call binder on a background thread to reduce latencies from metrics logging.
+        mExecutor.execute(() -> {
+            synchronized (mLock) {
+                if (mIsDone) {
+                    return;
+                }
+                mIsDone = isFinal;
+                if (DEBUG) {
+                    Log.v(TAG, "Committing: " + prettyPrintStats(mStats));
+                }
+                mStats.key = CameraManager.reportExtensionSessionStats(mStats);
+            }
+        });
+    }
+
+    private static String prettyPrintStats(@NonNull CameraExtensionSessionStats stats) {
+        return CameraExtensionSessionStats.class.getSimpleName() + ":\n"
+                + "  key: '" + stats.key + "'\n"
+                + "  cameraId: '" + stats.cameraId + "'\n"
+                + "  clientName: '" + stats.clientName + "'\n"
+                + "  type: '" + stats.type + "'\n"
+                + "  isAdvanced: '" + stats.isAdvanced + "'\n";
+    }
+}

services/core/java/com/android/server/camera/CameraServiceProxy.java

@@ -38,6 +38,7 @@ import android.content.pm.PackageManager;
 import android.content.pm.ParceledListSlice;
 import android.content.res.Configuration;
 import android.graphics.Rect;
+import android.hardware.CameraExtensionSessionStats;
 import android.hardware.CameraSessionStats;
 import android.hardware.CameraStreamStats;
 import android.hardware.ICameraService;
@@ -247,6 +248,7 @@ public class CameraServiceProxy extends SystemService
         public final int mSessionIndex;
 
         private long mDurationOrStartTimeMs;  // Either start time, or duration once completed
+        public CameraExtensionSessionStats mExtSessionStats = null;
 
         CameraUsageEvent(String cameraId, int facing, String clientName, int apiLevel,
                 boolean isNdk, int action, int latencyMs, int operatingMode, boolean deviceError,
@@ -269,7 +271,7 @@ public class CameraServiceProxy extends SystemService
         public void markCompleted(int internalReconfigure, long requestCount,
                 long resultErrorCount, boolean deviceError,
                 List<CameraStreamStats>  streamStats, String userTag,
-                int videoStabilizationMode) {
+                int videoStabilizationMode, CameraExtensionSessionStats extStats) {
             if (mCompleted) {
                 return;
             }
@@ -282,6 +284,7 @@ public class CameraServiceProxy extends SystemService
             mStreamStats = streamStats;
             mUserTag = userTag;
             mVideoStabilizationMode = videoStabilizationMode;
+            mExtSessionStats = extStats;
             if (CameraServiceProxy.DEBUG) {
                 Slog.v(TAG, "A camera facing " + cameraFacingToString(mCameraFacing) +
                         " was in use by " + mClientName + " for " +
@@ -825,6 +828,36 @@ public class CameraServiceProxy extends SystemService
                     Slog.w(TAG, "Unknown camera facing: " + e.mCameraFacing);
             }
 
+            int extensionType = FrameworkStatsLog.CAMERA_ACTION_EVENT__EXT_TYPE__EXTENSION_NONE;
+            boolean extensionIsAdvanced = false;
+            if (e.mExtSessionStats != null) {
+                switch (e.mExtSessionStats.type) {
+                    case CameraExtensionSessionStats.Type.EXTENSION_AUTOMATIC:
+                        extensionType = FrameworkStatsLog
+                                .CAMERA_ACTION_EVENT__EXT_TYPE__EXTENSION_AUTOMATIC;
+                        break;
+                    case CameraExtensionSessionStats.Type.EXTENSION_FACE_RETOUCH:
+                        extensionType = FrameworkStatsLog
+                                .CAMERA_ACTION_EVENT__EXT_TYPE__EXTENSION_FACE_RETOUCH;
+                        break;
+                    case CameraExtensionSessionStats.Type.EXTENSION_BOKEH:
+                        extensionType =
+                                FrameworkStatsLog.CAMERA_ACTION_EVENT__EXT_TYPE__EXTENSION_BOKEH;
+                        break;
+                    case CameraExtensionSessionStats.Type.EXTENSION_HDR:
+                        extensionType =
+                                FrameworkStatsLog.CAMERA_ACTION_EVENT__EXT_TYPE__EXTENSION_HDR;
+                        break;
+                    case CameraExtensionSessionStats.Type.EXTENSION_NIGHT:
+                        extensionType =
+                                FrameworkStatsLog.CAMERA_ACTION_EVENT__EXT_TYPE__EXTENSION_NIGHT;
+                        break;
+                    default:
+                        Slog.w(TAG, "Unknown extension type: " + e.mExtSessionStats.type);
+                }
+                extensionIsAdvanced = e.mExtSessionStats.isAdvanced;
+            }
+
             int streamCount = 0;
             if (e.mStreamStats != null) {
                 streamCount = e.mStreamStats.size();
@@ -847,7 +880,9 @@ public class CameraServiceProxy extends SystemService
                         + ", userTag is " + e.mUserTag
                         + ", videoStabilizationMode " + e.mVideoStabilizationMode
                         + ", logId " + e.mLogId
-                        + ", sessionIndex " + e.mSessionIndex);
+                        + ", sessionIndex " + e.mSessionIndex
+                        + ", mExtSessionStats {type " + extensionType
+                        + " isAdvanced " + extensionIsAdvanced + "}");
             }
             // Convert from CameraStreamStats to CameraStreamProto
             CameraStreamProto[] streamProtos = new CameraStreamProto[MAX_STREAM_STATISTICS];
@@ -907,7 +942,8 @@ public class CameraServiceProxy extends SystemService
                     MessageNano.toByteArray(streamProtos[2]),
                     MessageNano.toByteArray(streamProtos[3]),
                     MessageNano.toByteArray(streamProtos[4]),
-                    e.mUserTag, e.mVideoStabilizationMode, e.mLogId, e.mSessionIndex);
+                    e.mUserTag, e.mVideoStabilizationMode, e.mLogId, e.mSessionIndex,
+                    extensionType, extensionIsAdvanced);
         }
     }
 
@@ -1098,6 +1134,7 @@ public class CameraServiceProxy extends SystemService
         int videoStabilizationMode = cameraState.getVideoStabilizationMode();
         long logId = cameraState.getLogId();
         int sessionIdx = cameraState.getSessionIndex();
+        CameraExtensionSessionStats extSessionStats = cameraState.getExtensionSessionStats();
         synchronized(mLock) {
             // Update active camera list and notify NFC if necessary
             boolean wasEmpty = mActiveCameraUsage.isEmpty();
@@ -1152,7 +1189,8 @@ public class CameraServiceProxy extends SystemService
                         Slog.w(TAG, "Camera " + cameraId + " was already marked as active");
                         oldEvent.markCompleted(/*internalReconfigure*/0, /*requestCount*/0,
                                 /*resultErrorCount*/0, /*deviceError*/false, streamStats,
-                                /*userTag*/"", /*videoStabilizationMode*/-1);
+                                /*userTag*/"", /*videoStabilizationMode*/-1,
+                                new CameraExtensionSessionStats());
                         mCameraUsageHistory.add(oldEvent);
                     }
                     break;
@@ -1163,7 +1201,7 @@ public class CameraServiceProxy extends SystemService
 
                         doneEvent.markCompleted(internalReconfigureCount, requestCount,
                                 resultErrorCount, deviceError, streamStats, userTag,
-                                videoStabilizationMode);
+                                videoStabilizationMode, extSessionStats);
                         mCameraUsageHistory.add(doneEvent);
                         // Do not double count device error
                         deviceError = false;