From ba66b256aeaefe3dfffd20664bdff2ab67b52ae0 Mon Sep 17 00:00:00 2001 From: Chavi Weingarten Date: Mon, 23 Jan 2023 20:27:27 +0000 Subject: [PATCH] Add JNI and public methods for TrustedPresentationCallback Test: TrustedPresentationCallbackTest Bug: 256993331 Change-Id: Iade93f73c55264886ef647e4fd24af614fe871d2 --- core/api/current.txt | 6 + core/java/android/view/SurfaceControl.java | 182 ++++++++++++++++++++- core/jni/android_view_SurfaceControl.cpp | 127 ++++++++++++++ 3 files changed, 314 insertions(+), 1 deletion(-) diff --git a/core/api/current.txt b/core/api/current.txt index 53890eb5539df..71288f0a3741d 100644 --- a/core/api/current.txt +++ b/core/api/current.txt @@ -51183,6 +51183,7 @@ package android.view { method @NonNull public android.view.SurfaceControl.Transaction addTransactionCommittedListener(@NonNull java.util.concurrent.Executor, @NonNull android.view.SurfaceControl.TransactionCommittedListener); method public void apply(); method @NonNull public android.view.SurfaceControl.Transaction clearFrameRate(@NonNull android.view.SurfaceControl); + method @NonNull public android.view.SurfaceControl.Transaction clearTrustedPresentationCallback(@NonNull android.view.SurfaceControl); method public void close(); method public int describeContents(); method @NonNull public android.view.SurfaceControl.Transaction merge(@NonNull android.view.SurfaceControl.Transaction); @@ -51203,6 +51204,7 @@ package android.view { method @NonNull public android.view.SurfaceControl.Transaction setOpaque(@NonNull android.view.SurfaceControl, boolean); method @NonNull public android.view.SurfaceControl.Transaction setPosition(@NonNull android.view.SurfaceControl, float, float); method @NonNull public android.view.SurfaceControl.Transaction setScale(@NonNull android.view.SurfaceControl, float, float); + method @NonNull public android.view.SurfaceControl.Transaction setTrustedPresentationCallback(@NonNull android.view.SurfaceControl, @NonNull android.view.SurfaceControl.TrustedPresentationThresholds, @NonNull java.util.concurrent.Executor, @NonNull java.util.function.Consumer); method @NonNull public android.view.SurfaceControl.Transaction setVisibility(@NonNull android.view.SurfaceControl, boolean); method public void writeToParcel(@NonNull android.os.Parcel, int); field @NonNull public static final android.os.Parcelable.Creator CREATOR; @@ -51212,6 +51214,10 @@ package android.view { method public void onTransactionCommitted(); } + public static class SurfaceControl.TrustedPresentationThresholds { + ctor public SurfaceControl.TrustedPresentationThresholds(@FloatRange(from=0.0f, fromInclusive=false, to=1.0f) float, @FloatRange(from=0.0f, fromInclusive=false, to=1.0f) float, @IntRange(from=1) int); + } + public class SurfaceControlViewHost { ctor public SurfaceControlViewHost(@NonNull android.content.Context, @NonNull android.view.Display, @Nullable android.os.IBinder); method @Nullable public android.view.SurfaceControlViewHost.SurfacePackage getSurfacePackage(); diff --git a/core/java/android/view/SurfaceControl.java b/core/java/android/view/SurfaceControl.java index fedb09823a994..54e1a53ed7c99 100644 --- a/core/java/android/view/SurfaceControl.java +++ b/core/java/android/view/SurfaceControl.java @@ -280,7 +280,12 @@ public final class SurfaceControl implements Parcelable { private static native void nativeSetDefaultApplyToken(IBinder token); private static native IBinder nativeGetDefaultApplyToken(); private static native boolean nativeBootFinished(); - + private static native long nativeCreateTpc(TrustedPresentationCallback callback); + private static native long getNativeTrustedPresentationCallbackFinalizer(); + private static native void nativeSetTrustedPresentationCallback(long transactionObj, + long nativeObject, long nativeTpc, TrustedPresentationThresholds thresholds); + private static native void nativeClearTrustedPresentationCallback(long transactionObj, + long nativeObject); /** * Transforms that can be applied to buffers as they are displayed to a window. @@ -465,6 +470,8 @@ public final class SurfaceControl implements Parcelable { @GuardedBy("mLock") private int mHeight; + private TrustedPresentationCallback mTrustedPresentationCallback; + private WeakReference mLocalOwnerView; static GlobalTransactionWrapper sGlobalTransaction; @@ -2399,6 +2406,79 @@ public final class SurfaceControl implements Parcelable { void onTransactionCommitted(); } + /** + * Threshold values that are sent with + * {@link Transaction#setTrustedPresentationCallback(SurfaceControl, + * TrustedPresentationThresholds, Executor, Consumer)} + */ + public static class TrustedPresentationThresholds { + private float mMinAlpha; + private float mMinFractionRendered; + private int mStabilityRequirementMs; + + /** + * Creates a TrustedPresentationThresholds that's used when calling + * {@link Transaction#setTrustedPresentationCallback(SurfaceControl, + * TrustedPresentationThresholds, Executor, Consumer)} + * + * @param minAlpha The min alpha the {@link SurfaceControl} is required to + * have to be considered inside the threshold. + * @param minFractionRendered The min fraction of the SurfaceControl that was resented + * to the user to be considered inside the threshold. + * @param stabilityRequirementMs The time in milliseconds required for the + * {@link SurfaceControl} to be in the threshold. + * @throws IllegalArgumentException If threshold values are invalid. + */ + public TrustedPresentationThresholds( + @FloatRange(from = 0f, fromInclusive = false, to = 1f) float minAlpha, + @FloatRange(from = 0f, fromInclusive = false, to = 1f) float minFractionRendered, + @IntRange(from = 1) int stabilityRequirementMs) { + mMinAlpha = minAlpha; + mMinFractionRendered = minFractionRendered; + mStabilityRequirementMs = stabilityRequirementMs; + + checkValid(); + } + + private void checkValid() { + if (mMinAlpha <= 0 || mMinFractionRendered <= 0 || mStabilityRequirementMs < 1) { + throw new IllegalArgumentException( + "TrustedPresentationThresholds values are invalid"); + } + } + } + + /** + * Register a TrustedPresentationCallback for a particular SurfaceControl so it can be notified + * when the specified Threshold has been crossed. + * + * @hide + */ + public abstract static class TrustedPresentationCallback { + private final long mNativeObject; + + private static final NativeAllocationRegistry sRegistry = + NativeAllocationRegistry.createMalloced( + TrustedPresentationCallback.class.getClassLoader(), + getNativeTrustedPresentationCallbackFinalizer()); + + private final Runnable mFreeNativeResources; + + private TrustedPresentationCallback() { + mNativeObject = nativeCreateTpc(this); + mFreeNativeResources = sRegistry.registerNativeAllocation(this, mNativeObject); + } + + /** + * Invoked when the SurfaceControl that this TrustedPresentationCallback was registered for + * enters or exits the threshold bounds. + * + * @param inTrustedPresentationState true when the SurfaceControl entered the + * presentation state, false when it has left. + */ + public abstract void onTrustedPresentationChanged(boolean inTrustedPresentationState); + } + /** * An atomic set of changes to a set of SurfaceControl. */ @@ -3776,6 +3856,106 @@ public final class SurfaceControl implements Parcelable { return this; } + /** + * Sets a callback to receive feedback about the presentation of a {@link SurfaceControl}. + * When the {@link SurfaceControl} is presented according to the passed in + * {@link TrustedPresentationThresholds}, it is said to "enter the state", and receives the + * callback with {@code true}. When the conditions fall out of thresholds, it is then + * said to leave the state. + *

+ * There are a few simple thresholds: + *

    + *
  • minAlpha: Lower bound on computed alpha
    • + *
  • minFractionRendered: Lower bounds on fraction of pixels that were rendered
    • + *
  • stabilityThresholdMs: A time that alpha and fraction rendered must remain within + * bounds before we can "enter the state"
    • + *
+ *

+ * The fraction of pixels rendered is a computation based on scale, crop + * and occlusion. The calculation may be somewhat counterintuitive, so we + * can work through an example. Imagine we have a SurfaceControl with a 100x100 buffer + * which is occluded by (10x100) pixels on the left, and cropped by (100x10) pixels + * on the top. Furthermore imagine this SurfaceControl is scaled by 0.9 in both dimensions. + * (c=crop,o=occluded,b=both,x=none) + * + *

+ * + * + * + * + * + * + *
bccc
oxxx
oxxx
oxxx
+ *
+ * + *

+ * We first start by computing fr=xscale*yscale=0.9*0.9=0.81, indicating + * that "81%" of the pixels were rendered. This corresponds to what was 100 + * pixels being displayed in 81 pixels. This is somewhat of an abuse of + * language, as the information of merged pixels isn't totally lost, but + * we err on the conservative side. + *

+ * We then repeat a similar process for the crop and covered regions and + * accumulate the results: fr = fr * (fractionNotCropped) * (fractionNotCovered) + * So for this example we would get 0.9*0.9*0.9*0.9=0.65... + *

+ * Notice that this is not completely accurate, as we have double counted + * the region marked as b. However we only wanted a "lower bound" and so it + * is ok to err in this direction. Selection of the threshold will ultimately + * be somewhat arbitrary, and so there are some somewhat arbitrary decisions in + * this API as well. + *

+ * @param sc The {@link SurfaceControl} to set the + * {@link TrustedPresentationCallback} on + * @param thresholds The {@link TrustedPresentationThresholds} that will specify when the to + * invoke the callback. + * @param executor The {@link Executor} where the callback will be invoked on. + * @param listener The {@link Consumer} that will receive the callbacks when entered or + * exited the threshold. + * @return This transaction + * @see TrustedPresentationThresholds + */ + @NonNull + public Transaction setTrustedPresentationCallback(@NonNull SurfaceControl sc, + @NonNull TrustedPresentationThresholds thresholds, @NonNull Executor executor, + @NonNull Consumer listener) { + checkPreconditions(sc); + TrustedPresentationCallback tpc = new TrustedPresentationCallback() { + @Override + public void onTrustedPresentationChanged(boolean inTrustedPresentationState) { + executor.execute( + () -> listener.accept(inTrustedPresentationState)); + } + }; + + if (sc.mTrustedPresentationCallback != null) { + sc.mTrustedPresentationCallback.mFreeNativeResources.run(); + } + + nativeSetTrustedPresentationCallback(mNativeObject, sc.mNativeObject, + tpc.mNativeObject, thresholds); + sc.mTrustedPresentationCallback = tpc; + return this; + } + + /** + * Clears the {@link TrustedPresentationCallback} for a specific {@link SurfaceControl} + * + * @param sc The SurfaceControl that the {@link TrustedPresentationCallback} should be + * cleared from + * @return This transaction + */ + @NonNull + public Transaction clearTrustedPresentationCallback(@NonNull SurfaceControl sc) { + checkPreconditions(sc); + nativeClearTrustedPresentationCallback(mNativeObject, sc.mNativeObject); + if (sc.mTrustedPresentationCallback != null) { + sc.mTrustedPresentationCallback.mFreeNativeResources.run(); + sc.mTrustedPresentationCallback = null; + } + return this; + } + /** * Writes the transaction to parcel, clearing the transaction as if it had been applied so * it can be used to store future transactions. It's the responsibility of the parcel diff --git a/core/jni/android_view_SurfaceControl.cpp b/core/jni/android_view_SurfaceControl.cpp index 225ed2cff7624..97a0f50e760d8 100644 --- a/core/jni/android_view_SurfaceControl.cpp +++ b/core/jni/android_view_SurfaceControl.cpp @@ -246,6 +246,17 @@ static struct { jmethodID invokeReleaseCallback; } gSurfaceControlClassInfo; +static struct { + jclass clazz; + jfieldID mMinAlpha; + jfieldID mMinFractionRendered; + jfieldID mStabilityRequirementMs; +} gTrustedPresentationThresholdsClassInfo; + +static struct { + jmethodID onTrustedPresentationChanged; +} gTrustedPresentationCallbackClassInfo; + constexpr ui::Dataspace pickDataspaceFromColorMode(const ui::ColorMode colorMode) { switch (colorMode) { case ui::ColorMode::DISPLAY_P3: @@ -328,6 +339,47 @@ private: } }; +class TrustedPresentationCallbackWrapper { +public: + explicit TrustedPresentationCallbackWrapper(JNIEnv* env, jobject trustedPresentationListener) { + env->GetJavaVM(&mVm); + mTrustedPresentationCallback = env->NewGlobalRef(trustedPresentationListener); + LOG_ALWAYS_FATAL_IF(!mTrustedPresentationCallback, "Failed to make global ref"); + } + + ~TrustedPresentationCallbackWrapper() { + getenv()->DeleteGlobalRef(mTrustedPresentationCallback); + } + + void onTrustedPresentationChanged(bool inTrustedPresentationState) { + JNIEnv* env = getenv(); + env->CallVoidMethod(mTrustedPresentationCallback, + gTrustedPresentationCallbackClassInfo.onTrustedPresentationChanged, + inTrustedPresentationState); + } + + void addCallbackRef(const sp& callbackRef) { + mCallbackRef = callbackRef; + } + + static void onTrustedPresentationChangedThunk(void* context, bool inTrustedPresentationState) { + TrustedPresentationCallbackWrapper* listener = + reinterpret_cast(context); + listener->onTrustedPresentationChanged(inTrustedPresentationState); + } + +private: + jobject mTrustedPresentationCallback; + JavaVM* mVm; + sp mCallbackRef; + + JNIEnv* getenv() { + JNIEnv* env; + mVm->GetEnv(reinterpret_cast(&env), JNI_VERSION_1_6); + return env; + } +}; + // ---------------------------------------------------------------------------- static jlong nativeCreateTransaction(JNIEnv* env, jclass clazz) { @@ -1806,6 +1858,42 @@ static void nativeAddTransactionCommittedListener(JNIEnv* env, jclass clazz, jlo context); } +static void nativeSetTrustedPresentationCallback(JNIEnv* env, jclass clazz, jlong transactionObj, + jlong nativeObject, + jlong trustedPresentationCallbackObject, + jobject trustedPresentationThresholds) { + auto transaction = reinterpret_cast(transactionObj); + auto ctrl = reinterpret_cast(nativeObject); + + TrustedPresentationThresholds thresholds; + thresholds.minAlpha = env->GetFloatField(trustedPresentationThresholds, + gTrustedPresentationThresholdsClassInfo.mMinAlpha); + thresholds.minFractionRendered = + env->GetFloatField(trustedPresentationThresholds, + gTrustedPresentationThresholdsClassInfo.mMinFractionRendered); + thresholds.stabilityRequirementMs = + env->GetIntField(trustedPresentationThresholds, + gTrustedPresentationThresholdsClassInfo.mStabilityRequirementMs); + + sp callbackRef; + TrustedPresentationCallbackWrapper* wrapper = + reinterpret_cast( + trustedPresentationCallbackObject); + transaction->setTrustedPresentationCallback(ctrl, + TrustedPresentationCallbackWrapper:: + onTrustedPresentationChangedThunk, + thresholds, wrapper, callbackRef); + wrapper->addCallbackRef(callbackRef); +} + +static void nativeClearTrustedPresentationCallback(JNIEnv* env, jclass clazz, jlong transactionObj, + jlong nativeObject) { + auto transaction = reinterpret_cast(transactionObj); + auto ctrl = reinterpret_cast(nativeObject); + + transaction->clearTrustedPresentationCallback(ctrl); +} + class JankDataListenerWrapper : public JankDataListener { public: JankDataListenerWrapper(JNIEnv* env, jobject onJankDataListenerObject) { @@ -1919,6 +2007,21 @@ static jboolean nativeBootFinished(JNIEnv* env, jclass clazz) { return error == OK ? JNI_TRUE : JNI_FALSE; } +jlong nativeCreateTpc(JNIEnv* env, jclass clazz, jobject trustedPresentationCallback) { + return reinterpret_cast( + new TrustedPresentationCallbackWrapper(env, trustedPresentationCallback)); +} + +void destroyNativeTpc(void* ptr) { + TrustedPresentationCallbackWrapper* callback = + reinterpret_cast(ptr); + delete callback; +} + +static jlong getNativeTrustedPresentationCallbackFinalizer(JNIEnv* env, jclass clazz) { + return static_cast(reinterpret_cast(&destroyNativeTpc)); +} + // ---------------------------------------------------------------------------- SurfaceControl* android_view_SurfaceControl_getNativeSurfaceControl(JNIEnv* env, @@ -2145,6 +2248,10 @@ static const JNINativeMethod sSurfaceControlMethods[] = { (void*)nativeSetDropInputMode }, {"nativeAddTransactionCommittedListener", "(JLandroid/view/SurfaceControl$TransactionCommittedListener;)V", (void*) nativeAddTransactionCommittedListener }, + {"nativeSetTrustedPresentationCallback", "(JJJLandroid/view/SurfaceControl$TrustedPresentationThresholds;)V", + (void*) nativeSetTrustedPresentationCallback }, + {"nativeClearTrustedPresentationCallback", "(JJ)V", + (void*) nativeClearTrustedPresentationCallback }, {"nativeSanitize", "(J)V", (void*) nativeSanitize }, {"nativeSetDestinationFrame", "(JJIIII)V", @@ -2155,6 +2262,9 @@ static const JNINativeMethod sSurfaceControlMethods[] = { (void*)nativeGetDefaultApplyToken }, {"nativeBootFinished", "()Z", (void*)nativeBootFinished }, + {"nativeCreateTpc", "(Landroid/view/SurfaceControl$TrustedPresentationCallback;)J", + (void*)nativeCreateTpc}, + {"getNativeTrustedPresentationCallbackFinalizer", "()J", (void*)getNativeTrustedPresentationCallbackFinalizer }, // clang-format on }; @@ -2381,6 +2491,23 @@ int register_android_view_SurfaceControl(JNIEnv* env) gTransactionClassInfo.mNativeObject = GetFieldIDOrDie(env, gTransactionClassInfo.clazz, "mNativeObject", "J"); + jclass trustedPresentationThresholdsClazz = + FindClassOrDie(env, "android/view/SurfaceControl$TrustedPresentationThresholds"); + gTrustedPresentationThresholdsClassInfo.clazz = + MakeGlobalRefOrDie(env, trustedPresentationThresholdsClazz); + gTrustedPresentationThresholdsClassInfo.mMinAlpha = + GetFieldIDOrDie(env, trustedPresentationThresholdsClazz, "mMinAlpha", "F"); + gTrustedPresentationThresholdsClassInfo.mMinFractionRendered = + GetFieldIDOrDie(env, trustedPresentationThresholdsClazz, "mMinFractionRendered", "F"); + gTrustedPresentationThresholdsClassInfo.mStabilityRequirementMs = + GetFieldIDOrDie(env, trustedPresentationThresholdsClazz, "mStabilityRequirementMs", + "I"); + + jclass trustedPresentationCallbackClazz = + FindClassOrDie(env, "android/view/SurfaceControl$TrustedPresentationCallback"); + gTrustedPresentationCallbackClassInfo.onTrustedPresentationChanged = + GetMethodIDOrDie(env, trustedPresentationCallbackClazz, "onTrustedPresentationChanged", + "(Z)V"); return err; }