From 51ef38a47e7e994cd15e37567dcff75c5bdb45e0 Mon Sep 17 00:00:00 2001 From: Avichal Rakesh Date: Mon, 1 May 2023 17:53:52 -0700 Subject: [PATCH] cameraservice: Add metrics for extension sessions We want to log camera extension usage in cameraservice. This CL adds the scaffolding for it. Specifically, it does the following: - Updates CameraSessionStats.java to match that from CameraSessionStats.h. - Updates CameraServiceProxy to log CameraExtensionSessionStats along with CameraSessionStats. - Adds ExtensionSessionStatsAggregator responsible for collecting and sending extension stats to cameraservice. - Updates CameraExtensionSessions to use ExtensionSessionStatsAggregator. Bug: 261470491 Test: Only adds metrics logging. No functional change. `atest CtsCameraTestCases` pass. `statsd_testdrive 227` confirms that extension metrics are logged correctly. Change-Id: I40128f41c25daca8b4cc87a862f67990f8928caf --- .../android/hardware/CameraSessionStats.java | 9 ++ .../hardware/camera2/CameraManager.java | 25 ++++ .../CameraAdvancedExtensionSessionImpl.java | 26 ++++ .../camera2/impl/CameraDeviceImpl.java | 17 +++ .../impl/CameraExtensionSessionImpl.java | 27 +++- .../ExtensionSessionStatsAggregator.java | 121 ++++++++++++++++++ .../server/camera/CameraServiceProxy.java | 48 ++++++- 7 files changed, 267 insertions(+), 6 deletions(-) create mode 100644 core/java/android/hardware/camera2/utils/ExtensionSessionStatsAggregator.java diff --git a/core/java/android/hardware/CameraSessionStats.java b/core/java/android/hardware/CameraSessionStats.java index d59295ebe940a..1c2cbbc819713 100644 --- a/core/java/android/hardware/CameraSessionStats.java +++ b/core/java/android/hardware/CameraSessionStats.java @@ -65,6 +65,7 @@ public class CameraSessionStats implements Parcelable { private String mUserTag; private int mVideoStabilizationMode; private int mSessionIndex; + private CameraExtensionSessionStats mCameraExtensionSessionStats; public CameraSessionStats() { mFacing = -1; @@ -82,6 +83,7 @@ public class CameraSessionStats implements Parcelable { mStreamStats = new ArrayList(); mVideoStabilizationMode = -1; mSessionIndex = 0; + mCameraExtensionSessionStats = new CameraExtensionSessionStats(); } public CameraSessionStats(String cameraId, int facing, int newCameraState, @@ -101,6 +103,7 @@ public class CameraSessionStats implements Parcelable { mInternalReconfigure = internalReconfigure; mStreamStats = new ArrayList(); mSessionIndex = sessionIdx; + mCameraExtensionSessionStats = new CameraExtensionSessionStats(); } public static final @android.annotation.NonNull Parcelable.Creator CREATOR = @@ -145,6 +148,7 @@ public class CameraSessionStats implements Parcelable { dest.writeString(mUserTag); dest.writeInt(mVideoStabilizationMode); dest.writeInt(mSessionIndex); + mCameraExtensionSessionStats.writeToParcel(dest, 0); } public void readFromParcel(Parcel in) { @@ -170,6 +174,7 @@ public class CameraSessionStats implements Parcelable { mUserTag = in.readString(); mVideoStabilizationMode = in.readInt(); mSessionIndex = in.readInt(); + mCameraExtensionSessionStats = CameraExtensionSessionStats.CREATOR.createFromParcel(in); } public String getCameraId() { @@ -243,4 +248,8 @@ public class CameraSessionStats implements Parcelable { public int getSessionIndex() { return mSessionIndex; } + + public CameraExtensionSessionStats getExtensionSessionStats() { + return mCameraExtensionSessionStats; + } } diff --git a/core/java/android/hardware/camera2/CameraManager.java b/core/java/android/hardware/camera2/CameraManager.java index c6fc69efb85cf..51501b558fba9 100644 --- a/core/java/android/hardware/camera2/CameraManager.java +++ b/core/java/android/hardware/camera2/CameraManager.java @@ -30,6 +30,7 @@ import android.compat.annotation.Overridable; import android.content.Context; import android.content.pm.PackageManager; import android.graphics.Point; +import android.hardware.CameraExtensionSessionStats; import android.hardware.CameraStatus; import android.hardware.ICameraService; import android.hardware.ICameraServiceListener; @@ -1726,6 +1727,30 @@ public final class CameraManager { } } + /** + * Reports {@link CameraExtensionSessionStats} to the {@link ICameraService} to be logged for + * currently active session. Validation is done downstream. + * + * @param extStats Extension Session stats to be logged by cameraservice + * + * @return the key to be used with the next call. + * See {@link ICameraService#reportExtensionSessionStats}. + * @hide + */ + public static String reportExtensionSessionStats(CameraExtensionSessionStats extStats) { + ICameraService cameraService = CameraManagerGlobal.get().getCameraService(); + if (cameraService == null) { + Log.e(TAG, "CameraService not available. Not reporting extension stats."); + return ""; + } + try { + return cameraService.reportExtensionSessionStats(extStats); + } catch (RemoteException e) { + Log.e(TAG, "Failed to report extension session stats to cameraservice.", e); + } + return ""; + } + /** * A per-process global camera manager instance, to retain a connection to the camera service, * and to distribute camera availability notices to API-registered callbacks diff --git a/core/java/android/hardware/camera2/impl/CameraAdvancedExtensionSessionImpl.java b/core/java/android/hardware/camera2/impl/CameraAdvancedExtensionSessionImpl.java index e787779d0f572..d87226c2b8950 100644 --- a/core/java/android/hardware/camera2/impl/CameraAdvancedExtensionSessionImpl.java +++ b/core/java/android/hardware/camera2/impl/CameraAdvancedExtensionSessionImpl.java @@ -53,6 +53,7 @@ import android.hardware.camera2.params.DynamicRangeProfiles; import android.hardware.camera2.params.ExtensionSessionConfiguration; import android.hardware.camera2.params.OutputConfiguration; import android.hardware.camera2.params.SessionConfiguration; +import android.hardware.camera2.utils.ExtensionSessionStatsAggregator; import android.hardware.camera2.utils.SurfaceUtils; import android.media.Image; import android.media.ImageReader; @@ -96,6 +97,7 @@ public final class CameraAdvancedExtensionSessionImpl extends CameraExtensionSes private CameraCaptureSession mCaptureSession = null; private ISessionProcessorImpl mSessionProcessor = null; private final InitializeSessionHandler mInitializeHandler; + private final ExtensionSessionStatsAggregator mStatsAggregator; private boolean mInitialized; @@ -205,6 +207,10 @@ public final class CameraAdvancedExtensionSessionImpl extends CameraExtensionSes extender, cameraDevice, characteristicsMapNative, repeatingRequestSurface, burstCaptureSurface, postviewSurface, config.getStateCallback(), config.getExecutor(), sessionId); + + ret.mStatsAggregator.setClientName(ctx.getOpPackageName()); + ret.mStatsAggregator.setExtensionType(config.getExtension()); + ret.initialize(); return ret; @@ -234,6 +240,9 @@ public final class CameraAdvancedExtensionSessionImpl extends CameraExtensionSes mInitializeHandler = new InitializeSessionHandler(); mSessionId = sessionId; mInterfaceLock = cameraDevice.mInterfaceLock; + + mStatsAggregator = new ExtensionSessionStatsAggregator(mCameraDevice.getId(), + /*isAdvanced=*/true); } /** @@ -523,11 +532,26 @@ public final class CameraAdvancedExtensionSessionImpl extends CameraExtensionSes Log.e(TAG, "Failed to stop the repeating request or end the session," + " , extension service does not respond!") ; } + // Commit stats before closing the capture session + mStatsAggregator.commit(/*isFinal*/true); mCaptureSession.close(); } } } + /** + * Called by {@link CameraDeviceImpl} right before the capture session is closed, and before it + * calls {@link #release} + */ + public void commitStats() { + synchronized (mInterfaceLock) { + if (mInitialized) { + // Only commit stats if a capture session was initialized + mStatsAggregator.commit(/*isFinal*/true); + } + } + } + public void release(boolean skipCloseNotification) { boolean notifyClose = false; @@ -608,6 +632,8 @@ public final class CameraAdvancedExtensionSessionImpl extends CameraExtensionSes public void onConfigured(@NonNull CameraCaptureSession session) { synchronized (mInterfaceLock) { mCaptureSession = session; + // Commit basic stats as soon as the capture session is created + mStatsAggregator.commit(/*isFinal*/false); } try { diff --git a/core/java/android/hardware/camera2/impl/CameraDeviceImpl.java b/core/java/android/hardware/camera2/impl/CameraDeviceImpl.java index b8e451c2f8163..e23bbc6c347d9 100644 --- a/core/java/android/hardware/camera2/impl/CameraDeviceImpl.java +++ b/core/java/android/hardware/camera2/impl/CameraDeviceImpl.java @@ -700,6 +700,14 @@ public class CameraDeviceImpl extends CameraDevice + " input configuration yet."); } + if (mCurrentExtensionSession != null) { + mCurrentExtensionSession.commitStats(); + } + + if (mCurrentAdvancedExtensionSession != null) { + mCurrentAdvancedExtensionSession.commitStats(); + } + // Notify current session that it's going away, before starting camera operations // After this call completes, the session is not allowed to call into CameraDeviceImpl if (mCurrentSession != null) { @@ -1414,6 +1422,15 @@ public class CameraDeviceImpl extends CameraDevice mOfflineSwitchService = null; } + // Let extension sessions commit stats before disconnecting remoteDevice + if (mCurrentExtensionSession != null) { + mCurrentExtensionSession.commitStats(); + } + + if (mCurrentAdvancedExtensionSession != null) { + mCurrentAdvancedExtensionSession.commitStats(); + } + if (mRemoteDevice != null) { mRemoteDevice.disconnect(); mRemoteDevice.unlinkToDeath(this, /*flags*/0); diff --git a/core/java/android/hardware/camera2/impl/CameraExtensionSessionImpl.java b/core/java/android/hardware/camera2/impl/CameraExtensionSessionImpl.java index 8e7c7e0cfca89..9d2bb7e77e5aa 100644 --- a/core/java/android/hardware/camera2/impl/CameraExtensionSessionImpl.java +++ b/core/java/android/hardware/camera2/impl/CameraExtensionSessionImpl.java @@ -30,7 +30,6 @@ import android.hardware.camera2.CameraCharacteristics; import android.hardware.camera2.CameraDevice; import android.hardware.camera2.CameraExtensionCharacteristics; import android.hardware.camera2.CameraExtensionSession; -import android.hardware.camera2.CameraManager; import android.hardware.camera2.CaptureFailure; import android.hardware.camera2.CaptureRequest; import android.hardware.camera2.CaptureResult; @@ -49,6 +48,7 @@ import android.hardware.camera2.params.DynamicRangeProfiles; import android.hardware.camera2.params.ExtensionSessionConfiguration; import android.hardware.camera2.params.OutputConfiguration; import android.hardware.camera2.params.SessionConfiguration; +import android.hardware.camera2.utils.ExtensionSessionStatsAggregator; import android.hardware.camera2.utils.SurfaceUtils; import android.media.Image; import android.media.ImageReader; @@ -90,6 +90,7 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession { private final int mSessionId; private final Set mSupportedRequestKeys; private final Set mSupportedResultKeys; + private final ExtensionSessionStatsAggregator mStatsAggregator; private boolean mCaptureResultsSupported; private CameraCaptureSession mCaptureSession = null; @@ -242,6 +243,9 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession { extensionChars.getAvailableCaptureRequestKeys(config.getExtension()), extensionChars.getAvailableCaptureResultKeys(config.getExtension())); + session.mStatsAggregator.setClientName(ctx.getOpPackageName()); + session.mStatsAggregator.setExtensionType(config.getExtension()); + session.initialize(); return session; @@ -280,6 +284,9 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession { mSupportedResultKeys = resultKeys; mCaptureResultsSupported = !resultKeys.isEmpty(); mInterfaceLock = cameraDevice.mInterfaceLock; + + mStatsAggregator = new ExtensionSessionStatsAggregator(mCameraDevice.getId(), + /*isAdvanced=*/false); } private void initializeRepeatingRequestPipeline() throws RemoteException { @@ -793,11 +800,27 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession { new CloseRequestHandler(mRepeatingRequestImageCallback), mHandler); } + mStatsAggregator.commit(/*isFinal*/true); // Commit stats before closing session mCaptureSession.close(); } } } + /** + * Called by {@link CameraDeviceImpl} right before the capture session is closed, and before it + * calls {@link #release} + * + * @hide + */ + public void commitStats() { + synchronized (mInterfaceLock) { + if (mInitialized) { + // Only commit stats if a capture session was initialized + mStatsAggregator.commit(/*isFinal*/true); + } + } + } + private void setInitialCaptureRequest(List captureStageList, InitialRequestHandler requestHandler) throws CameraAccessException { @@ -955,6 +978,8 @@ public final class CameraExtensionSessionImpl extends CameraExtensionSession { public void onConfigured(@NonNull CameraCaptureSession session) { synchronized (mInterfaceLock) { mCaptureSession = session; + // Commit basic stats as soon as the capture session is created + mStatsAggregator.commit(/*isFinal*/false); try { finishPipelineInitialization(); CameraExtensionCharacteristics.initializeSession(mInitializeHandler); diff --git a/core/java/android/hardware/camera2/utils/ExtensionSessionStatsAggregator.java b/core/java/android/hardware/camera2/utils/ExtensionSessionStatsAggregator.java new file mode 100644 index 0000000000000..8cd5e83241add --- /dev/null +++ b/core/java/android/hardware/camera2/utils/ExtensionSessionStatsAggregator.java @@ -0,0 +1,121 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.hardware.camera2.utils; + +import android.annotation.NonNull; +import android.hardware.CameraExtensionSessionStats; +import android.hardware.camera2.CameraManager; +import android.util.Log; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +/** + * Utility class to aggregate metrics specific to Camera Extensions and pass them to + * {@link CameraManager}. {@link android.hardware.camera2.CameraExtensionSession} should call + * {@link #commit} before closing the session. + * + * @hide + */ +public class ExtensionSessionStatsAggregator { + private static final boolean DEBUG = false; + private static final String TAG = ExtensionSessionStatsAggregator.class.getSimpleName(); + + private final ExecutorService mExecutor = Executors.newSingleThreadExecutor(); + + private final Object mLock = new Object(); // synchronizes access to all fields of the class + private boolean mIsDone = false; // marks the aggregator as "done". + // Mutations and commits become no-op if this is true. + private final CameraExtensionSessionStats mStats; + + public ExtensionSessionStatsAggregator(@NonNull String cameraId, boolean isAdvanced) { + if (DEBUG) { + Log.v(TAG, "Creating new Extension Session Stats Aggregator"); + } + mStats = new CameraExtensionSessionStats(); + mStats.key = ""; + mStats.cameraId = cameraId; + mStats.isAdvanced = isAdvanced; + } + + /** + * Set client package name + * + * @param clientName package name of the client that these stats are associated with. + */ + public void setClientName(@NonNull String clientName) { + synchronized (mLock) { + if (mIsDone) { + return; + } + if (DEBUG) { + Log.v(TAG, "Setting clientName: " + clientName); + } + mStats.clientName = clientName; + } + } + + /** + * Set extension type. + * + * @param extensionType Type of extension. Must match one of + * {@code CameraExtensionCharacteristics#EXTENSION_*} + */ + public void setExtensionType(int extensionType) { + synchronized (mLock) { + if (mIsDone) { + return; + } + if (DEBUG) { + Log.v(TAG, "Setting type: " + extensionType); + } + mStats.type = extensionType; + } + } + + /** + * Asynchronously commits the stats to CameraManager on a background thread. + * + * @param isFinal marks the stats as final and prevents any further commits or changes. This + * should be set to true when the stats are considered final for logging, + * for example right before the capture session is about to close + */ + public void commit(boolean isFinal) { + // Call binder on a background thread to reduce latencies from metrics logging. + mExecutor.execute(() -> { + synchronized (mLock) { + if (mIsDone) { + return; + } + mIsDone = isFinal; + if (DEBUG) { + Log.v(TAG, "Committing: " + prettyPrintStats(mStats)); + } + mStats.key = CameraManager.reportExtensionSessionStats(mStats); + } + }); + } + + private static String prettyPrintStats(@NonNull CameraExtensionSessionStats stats) { + return CameraExtensionSessionStats.class.getSimpleName() + ":\n" + + " key: '" + stats.key + "'\n" + + " cameraId: '" + stats.cameraId + "'\n" + + " clientName: '" + stats.clientName + "'\n" + + " type: '" + stats.type + "'\n" + + " isAdvanced: '" + stats.isAdvanced + "'\n"; + } +} diff --git a/services/core/java/com/android/server/camera/CameraServiceProxy.java b/services/core/java/com/android/server/camera/CameraServiceProxy.java index 0315352be133b..0b04159194d11 100644 --- a/services/core/java/com/android/server/camera/CameraServiceProxy.java +++ b/services/core/java/com/android/server/camera/CameraServiceProxy.java @@ -38,6 +38,7 @@ import android.content.pm.PackageManager; import android.content.pm.ParceledListSlice; import android.content.res.Configuration; import android.graphics.Rect; +import android.hardware.CameraExtensionSessionStats; import android.hardware.CameraSessionStats; import android.hardware.CameraStreamStats; import android.hardware.ICameraService; @@ -247,6 +248,7 @@ public class CameraServiceProxy extends SystemService public final int mSessionIndex; private long mDurationOrStartTimeMs; // Either start time, or duration once completed + public CameraExtensionSessionStats mExtSessionStats = null; CameraUsageEvent(String cameraId, int facing, String clientName, int apiLevel, boolean isNdk, int action, int latencyMs, int operatingMode, boolean deviceError, @@ -269,7 +271,7 @@ public class CameraServiceProxy extends SystemService public void markCompleted(int internalReconfigure, long requestCount, long resultErrorCount, boolean deviceError, List streamStats, String userTag, - int videoStabilizationMode) { + int videoStabilizationMode, CameraExtensionSessionStats extStats) { if (mCompleted) { return; } @@ -282,6 +284,7 @@ public class CameraServiceProxy extends SystemService mStreamStats = streamStats; mUserTag = userTag; mVideoStabilizationMode = videoStabilizationMode; + mExtSessionStats = extStats; if (CameraServiceProxy.DEBUG) { Slog.v(TAG, "A camera facing " + cameraFacingToString(mCameraFacing) + " was in use by " + mClientName + " for " + @@ -825,6 +828,36 @@ public class CameraServiceProxy extends SystemService Slog.w(TAG, "Unknown camera facing: " + e.mCameraFacing); } + int extensionType = FrameworkStatsLog.CAMERA_ACTION_EVENT__EXT_TYPE__EXTENSION_NONE; + boolean extensionIsAdvanced = false; + if (e.mExtSessionStats != null) { + switch (e.mExtSessionStats.type) { + case CameraExtensionSessionStats.Type.EXTENSION_AUTOMATIC: + extensionType = FrameworkStatsLog + .CAMERA_ACTION_EVENT__EXT_TYPE__EXTENSION_AUTOMATIC; + break; + case CameraExtensionSessionStats.Type.EXTENSION_FACE_RETOUCH: + extensionType = FrameworkStatsLog + .CAMERA_ACTION_EVENT__EXT_TYPE__EXTENSION_FACE_RETOUCH; + break; + case CameraExtensionSessionStats.Type.EXTENSION_BOKEH: + extensionType = + FrameworkStatsLog.CAMERA_ACTION_EVENT__EXT_TYPE__EXTENSION_BOKEH; + break; + case CameraExtensionSessionStats.Type.EXTENSION_HDR: + extensionType = + FrameworkStatsLog.CAMERA_ACTION_EVENT__EXT_TYPE__EXTENSION_HDR; + break; + case CameraExtensionSessionStats.Type.EXTENSION_NIGHT: + extensionType = + FrameworkStatsLog.CAMERA_ACTION_EVENT__EXT_TYPE__EXTENSION_NIGHT; + break; + default: + Slog.w(TAG, "Unknown extension type: " + e.mExtSessionStats.type); + } + extensionIsAdvanced = e.mExtSessionStats.isAdvanced; + } + int streamCount = 0; if (e.mStreamStats != null) { streamCount = e.mStreamStats.size(); @@ -847,7 +880,9 @@ public class CameraServiceProxy extends SystemService + ", userTag is " + e.mUserTag + ", videoStabilizationMode " + e.mVideoStabilizationMode + ", logId " + e.mLogId - + ", sessionIndex " + e.mSessionIndex); + + ", sessionIndex " + e.mSessionIndex + + ", mExtSessionStats {type " + extensionType + + " isAdvanced " + extensionIsAdvanced + "}"); } // Convert from CameraStreamStats to CameraStreamProto CameraStreamProto[] streamProtos = new CameraStreamProto[MAX_STREAM_STATISTICS]; @@ -907,7 +942,8 @@ public class CameraServiceProxy extends SystemService MessageNano.toByteArray(streamProtos[2]), MessageNano.toByteArray(streamProtos[3]), MessageNano.toByteArray(streamProtos[4]), - e.mUserTag, e.mVideoStabilizationMode, e.mLogId, e.mSessionIndex); + e.mUserTag, e.mVideoStabilizationMode, e.mLogId, e.mSessionIndex, + extensionType, extensionIsAdvanced); } } @@ -1098,6 +1134,7 @@ public class CameraServiceProxy extends SystemService int videoStabilizationMode = cameraState.getVideoStabilizationMode(); long logId = cameraState.getLogId(); int sessionIdx = cameraState.getSessionIndex(); + CameraExtensionSessionStats extSessionStats = cameraState.getExtensionSessionStats(); synchronized(mLock) { // Update active camera list and notify NFC if necessary boolean wasEmpty = mActiveCameraUsage.isEmpty(); @@ -1152,7 +1189,8 @@ public class CameraServiceProxy extends SystemService Slog.w(TAG, "Camera " + cameraId + " was already marked as active"); oldEvent.markCompleted(/*internalReconfigure*/0, /*requestCount*/0, /*resultErrorCount*/0, /*deviceError*/false, streamStats, - /*userTag*/"", /*videoStabilizationMode*/-1); + /*userTag*/"", /*videoStabilizationMode*/-1, + new CameraExtensionSessionStats()); mCameraUsageHistory.add(oldEvent); } break; @@ -1163,7 +1201,7 @@ public class CameraServiceProxy extends SystemService doneEvent.markCompleted(internalReconfigureCount, requestCount, resultErrorCount, deviceError, streamStats, userTag, - videoStabilizationMode); + videoStabilizationMode, extSessionStats); mCameraUsageHistory.add(doneEvent); // Do not double count device error deviceError = false;