Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Handle reperenting of InlineContentView

Browse Source 
When an inline content view is reparented its surface is
getting offset and not being under the view itelf. This is
because the surface views manage the postion of their
surface and are assuming a location based off the window's
surface position. However after a reparenting the inline
content view's surface position becomes relative to that
of the new parent surface view. For example, two surface
views at position (10, 10) being reparented would reslut
in the surface of the parent being at (10, 10) while the
surface of the child being at (20, 20) while both views
would still be at (10, 10).

To address this we are intecepting when an inline content
view's surface is reparented and get a weak reference to
the view that owns the new parent surface. We then position
the inline content view's surface relative to the view that
owns the new parent surface, i.e. we position the surface
such that its location would not change because of the
fact it is being reparented.

While at this make sure the inline content view is marked
as not important for a11y to ernsure the a11y plugins don't
try to click on the view tree in the app's process but
insted on the views in the remote proccess, i.e. on the
embedded view tree.

bug:153826463

Test: atest android.widget.cts.inline.InlineContentViewTest#testReparenting

Change-Id: I2cff4b88d404a740bc447668e948eabccad084d2
This commit is contained in:
Svet Ganov
2020-06-18 00:47:22 -07:00
committed by Svetoslav Ganov
parent 449851f8e2
commit a5b4990d96
4 changed files with 350 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
api/test-current.txt
View File

@@ -5505,6 +5505,19 @@ package android.widget {
}
package android.widget.inline {
public class InlineContentView extends android.view.ViewGroup {
method public void setChildSurfacePackageUpdater(@Nullable android.widget.inline.InlineContentView.SurfacePackageUpdater);
}
public static interface InlineContentView.SurfacePackageUpdater {
method public void getSurfacePackage(@NonNull java.util.function.Consumer<android.view.SurfaceControlViewHost.SurfacePackage>);
method public void onSurfacePackageReleased();
}
}
package android.window {
public final class DisplayAreaInfo implements android.os.Parcelable {

145
core/java/android/view/SurfaceControl.java
View File

@@ -63,8 +63,10 @@ import dalvik.system.CloseGuard;
import libcore.util.NativeAllocationRegistry;
import java.io.Closeable;
import java.lang.ref.WeakReference;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.ArrayList;
import java.util.Objects;
/**
@@ -226,24 +228,86 @@ public final class SurfaceControl implements Parcelable {
private static native void nativeSetFixedTransformHint(long transactionObj, long nativeObject,
int transformHint);
@Nullable
@GuardedBy("sLock")
private ArrayList<OnReparentListener> mReparentListeners;
/**
* Listener to observe surface reparenting.
*
* @hide
*/
public interface OnReparentListener {
/**
* Callback for reparenting surfaces.
*
* Important: You should only interact with the provided surface control
* only if you have a contract with its owner to avoid them closing it
* under you or vise versa.
*
* @param transaction The transaction that would commit reparenting.
* @param parent The future parent surface.
*/
void onReparent(@NonNull Transaction transaction, @Nullable SurfaceControl parent);
}
private final CloseGuard mCloseGuard = CloseGuard.get();
private String mName;
/**
/**
* @hide
*/
public long mNativeObject;
private long mNativeHandle;
// TODO: Move this to native.
private final Object mSizeLock = new Object();
@GuardedBy("mSizeLock")
// TODO: Move width/height to native and fix locking through out.
private final Object mLock = new Object();
@GuardedBy("mLock")
private int mWidth;
@GuardedBy("mSizeLock")
@GuardedBy("mLock")
private int mHeight;
private WeakReference<View> mLocalOwnerView;
static Transaction sGlobalTransaction;
static long sTransactionNestCount = 0;
/**
* Adds a reparenting listener.
*
* @param listener The listener.
* @return Whether listener was added.
*
* @hide
*/
public boolean addOnReparentListener(@NonNull OnReparentListener listener) {
synchronized (mLock) {
if (mReparentListeners == null) {
mReparentListeners = new ArrayList<>(1);
}
return mReparentListeners.add(listener);
}
}
/**
* Removes a reparenting listener.
*
* @param listener The listener.
* @return Whether listener was removed.
*
* @hide
*/
public boolean removeOnReparentListener(@NonNull OnReparentListener listener) {
synchronized (mLock) {
final boolean removed = mReparentListeners.remove(listener);
if (mReparentListeners.isEmpty()) {
mReparentListeners = null;
}
return removed;
}
}
/* flags used in constructor (keep in sync with ISurfaceComposerClient.h) */
/**
@@ -455,6 +519,7 @@ public final class SurfaceControl implements Parcelable {
mName = other.mName;
mWidth = other.mWidth;
mHeight = other.mHeight;
mLocalOwnerView = other.mLocalOwnerView;
assignNativeObject(nativeCopyFromSurfaceControl(other.mNativeObject));
}
@@ -553,6 +618,7 @@ public final class SurfaceControl implements Parcelable {
private int mHeight;
private int mFormat = PixelFormat.OPAQUE;
private String mName;
private WeakReference<View> mLocalOwnerView;
private SurfaceControl mParent;
private SparseIntArray mMetadata;
@@ -587,7 +653,8 @@ public final class SurfaceControl implements Parcelable {
"Only buffer layers can set a valid buffer size.");
}
return new SurfaceControl(
mSession, mName, mWidth, mHeight, mFormat, mFlags, mParent, mMetadata);
mSession, mName, mWidth, mHeight, mFormat, mFlags, mParent, mMetadata,
mLocalOwnerView);
}
/**
@@ -601,6 +668,27 @@ public final class SurfaceControl implements Parcelable {
return this;
}
/**
* Set the local owner view for the surface. This view is only
* valid in the same process and is not transferred in an IPC.
*
* Note: This is used for cases where we want to know the view
* that manages the surface control while intercepting reparenting.
* A specific example is InlineContentView which exposes is surface
* control for reparenting as a way to implement clipping of several
* InlineContentView instances within a certain area.
*
* @param view The owner view.
* @return This builder.
*
* @hide
*/
@NonNull
public Builder setLocalOwnerView(@NonNull View view) {
mLocalOwnerView = new WeakReference<>(view);
return this;
}
/**
* Set the initial size of the controlled surface's buffers in pixels.
*
@@ -858,7 +946,7 @@ public final class SurfaceControl implements Parcelable {
* @throws throws OutOfResourcesException If the SurfaceControl cannot be created.
*/
private SurfaceControl(SurfaceSession session, String name, int w, int h, int format, int flags,
SurfaceControl parent, SparseIntArray metadata)
SurfaceControl parent, SparseIntArray metadata, WeakReference<View> localOwnerView)
throws OutOfResourcesException, IllegalArgumentException {
if (name == null) {
throw new IllegalArgumentException("name must not be null");
@@ -867,6 +955,7 @@ public final class SurfaceControl implements Parcelable {
mName = name;
mWidth = w;
mHeight = h;
mLocalOwnerView = localOwnerView;
Parcel metaParcel = Parcel.obtain();
try {
if (metadata != null && metadata.size() > 0) {
@@ -1307,7 +1396,7 @@ public final class SurfaceControl implements Parcelable {
* @hide
*/
public int getWidth() {
synchronized (mSizeLock) {
synchronized (mLock) {
return mWidth;
}
}
@@ -1316,11 +1405,22 @@ public final class SurfaceControl implements Parcelable {
* @hide
*/
public int getHeight() {
synchronized (mSizeLock) {
synchronized (mLock) {
return mHeight;
}
}
/**
* Gets the local view that owns this surface.
*
* @return The owner view.
*
* @hide
*/
public @Nullable View getLocalOwnerView() {
return (mLocalOwnerView != null) ? mLocalOwnerView.get() : null;
}
@Override
public String toString() {
return "Surface(name=" + mName + ")/@0x" +
@@ -2165,6 +2265,9 @@ public final class SurfaceControl implements Parcelable {
public long mNativeObject;
private final ArrayMap<SurfaceControl, Point> mResizedSurfaces = new ArrayMap<>();
private final ArrayMap<SurfaceControl, SurfaceControl> mReparentedSurfaces =
new ArrayMap<>();
Runnable mFreeNativeResources;
private static final float[] INVALID_COLOR = {-1, -1, -1};
@@ -2205,6 +2308,8 @@ public final class SurfaceControl implements Parcelable {
*/
@Override
public void close() {
mResizedSurfaces.clear();
mReparentedSurfaces.clear();
mFreeNativeResources.run();
mNativeObject = 0;
}
@@ -2215,6 +2320,7 @@ public final class SurfaceControl implements Parcelable {
*/
public void apply(boolean sync) {
applyResizedSurfaces();
notifyReparentedSurfaces();
nativeApplyTransaction(mNativeObject, sync);
}
@@ -2222,7 +2328,7 @@ public final class SurfaceControl implements Parcelable {
for (int i = mResizedSurfaces.size() - 1; i >= 0; i--) {
final Point size = mResizedSurfaces.valueAt(i);
final SurfaceControl surfaceControl = mResizedSurfaces.keyAt(i);
synchronized (surfaceControl.mSizeLock) {
synchronized (surfaceControl.mLock) {
surfaceControl.mWidth = size.x;
surfaceControl.mHeight = size.y;
}
@@ -2230,6 +2336,22 @@ public final class SurfaceControl implements Parcelable {
mResizedSurfaces.clear();
}
private void notifyReparentedSurfaces() {
final int reparentCount = mReparentedSurfaces.size();
for (int i = reparentCount - 1; i >= 0; i--) {
final SurfaceControl child = mReparentedSurfaces.keyAt(i);
synchronized (child.mLock) {
final int listenerCount = (child.mReparentListeners != null)
? child.mReparentListeners.size() : 0;
for (int j = 0; j < listenerCount; j++) {
final OnReparentListener listener = child.mReparentListeners.get(j);
listener.onReparent(this, mReparentedSurfaces.valueAt(i));
}
mReparentedSurfaces.removeAt(i);
}
}
}
/**
* Toggle the visibility of a given Layer and it's sub-tree.
*
@@ -2632,6 +2754,7 @@ public final class SurfaceControl implements Parcelable {
otherObject = newParent.mNativeObject;
}
nativeReparent(mNativeObject, sc.mNativeObject, otherObject);
mReparentedSurfaces.put(sc, newParent);
return this;
}
@@ -2912,6 +3035,8 @@ public final class SurfaceControl implements Parcelable {
}
mResizedSurfaces.putAll(other.mResizedSurfaces);
other.mResizedSurfaces.clear();
mReparentedSurfaces.putAll(other.mReparentedSurfaces);
other.mReparentedSurfaces.clear();
nativeMergeTransaction(mNativeObject, other.mNativeObject);
return this;
}

68
core/java/android/view/SurfaceView.java
View File

@@ -44,7 +44,6 @@ import android.os.SystemClock;
import android.util.AttributeSet;
import android.util.Log;
import android.view.SurfaceControl.Transaction;
import android.view.SurfaceControlViewHost;
import android.view.accessibility.AccessibilityNodeInfo;
import android.view.accessibility.IAccessibilityEmbeddedConnection;
@@ -988,6 +987,7 @@ public class SurfaceView extends View implements ViewRootImpl.SurfaceChangedCall
mSurfaceControl = new SurfaceControl.Builder(mSurfaceSession)
.setName(name)
.setLocalOwnerView(this)
.setOpaque((mSurfaceFlags & SurfaceControl.OPAQUE) != 0)
.setBufferSize(mSurfaceWidth, mSurfaceHeight)
.setFormat(mFormat)
@@ -996,6 +996,7 @@ public class SurfaceView extends View implements ViewRootImpl.SurfaceChangedCall
.build();
mBackgroundControl = new SurfaceControl.Builder(mSurfaceSession)
.setName("Background for -" + name)
.setLocalOwnerView(this)
.setOpaque(true)
.setColorLayer()
.setParent(mSurfaceControl)
@@ -1051,11 +1052,12 @@ public class SurfaceView extends View implements ViewRootImpl.SurfaceChangedCall
// we still need to latch a buffer).
// b/28866173
if (sizeChanged || creating || !mRtHandlingPositionUpdates) {
mTmpTransaction.setPosition(mSurfaceControl, mScreenRect.left,
mScreenRect.top);
mTmpTransaction.setMatrix(mSurfaceControl,
mScreenRect.width() / (float) mSurfaceWidth, 0.0f, 0.0f,
mScreenRect.height() / (float) mSurfaceHeight);
onSetSurfacePositionAndScaleRT(mTmpTransaction, mSurfaceControl,
mScreenRect.left, /*positionLeft*/
mScreenRect.top /*positionTop*/ ,
mScreenRect.width() / (float) mSurfaceWidth /*postScaleX*/,
mScreenRect.height() / (float) mSurfaceHeight /*postScaleY*/);
// Set a window crop when creating the surface or changing its size to
// crop the buffer to the surface size since the buffer producer may
// use SCALING_MODE_SCALE and submit a larger size than the surface
@@ -1211,6 +1213,40 @@ public class SurfaceView extends View implements ViewRootImpl.SurfaceChangedCall
Surface viewRootSurface, long nextViewRootFrameNumber) {
}
/**
* Sets the surface position and scale. Can be called on
* the UI thread as well as on the renderer thread.
*
* @param transaction Transaction in which to execute.
* @param surface Surface whose location to set.
* @param positionLeft The left position to set.
* @param positionTop The top position to set.
* @param postScaleX The X axis post scale
* @param postScaleY The Y axis post scale
*
* @hide
*/
protected void onSetSurfacePositionAndScaleRT(@NonNull Transaction transaction,
@NonNull SurfaceControl surface, int positionLeft, int positionTop,
float postScaleX, float postScaleY) {
transaction.setPosition(surface, positionLeft, positionTop);
transaction.setMatrix(surface, postScaleX /*dsdx*/, 0f /*dtdx*/,
0f /*dsdy*/, postScaleY /*dtdy*/);
}
/** @hide */
public void requestUpdateSurfacePositionAndScale() {
if (mSurfaceControl == null) {
return;
}
onSetSurfacePositionAndScaleRT(mTmpTransaction, mSurfaceControl,
mScreenRect.left, /*positionLeft*/
mScreenRect.top/*positionTop*/ ,
mScreenRect.width() / (float) mSurfaceWidth /*postScaleX*/,
mScreenRect.height() / (float) mSurfaceHeight /*postScaleY*/);
mTmpTransaction.apply();
}
private void applySurfaceTransforms(SurfaceControl surface, SurfaceControl.Transaction t,
Rect position, long frameNumber) {
final ViewRootImpl viewRoot = getViewRootImpl();
@@ -1219,16 +1255,26 @@ public class SurfaceView extends View implements ViewRootImpl.SurfaceChangedCall
frameNumber);
}
t.setPosition(surface, position.left, position.top);
t.setMatrix(surface,
position.width() / (float) mSurfaceWidth,
0.0f, 0.0f,
position.height() / (float) mSurfaceHeight);
onSetSurfacePositionAndScaleRT(t, surface,
position.left /*positionLeft*/,
position.top /*positionTop*/,
position.width() / (float) mSurfaceWidth /*postScaleX*/,
position.height() / (float) mSurfaceHeight /*postScaleY*/);
if (mViewVisibility) {
t.show(surface);
}
}
/**
* @return The last render position of the backing surface or an empty rect.
*
* @hide
*/
public @NonNull Rect getSurfaceRenderPosition() {
return mRTLastReportedPosition;
}
private void setParentSpaceRectangle(Rect position, long frameNumber) {
final ViewRootImpl viewRoot = getViewRootImpl();
final boolean useBLAST = viewRoot.isDrawingToBLASTTransaction();

163
core/java/android/widget/inline/InlineContentView.java
View File

@@ -18,18 +18,22 @@ package android.widget.inline;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.annotation.TestApi;
import android.content.Context;
import android.graphics.PixelFormat;
import android.graphics.Rect;
import android.graphics.PointF;
import android.util.AttributeSet;
import android.util.Log;
import android.view.SurfaceControl;
import android.view.SurfaceControlViewHost;
import android.view.SurfaceHolder;
import android.view.SurfaceView;
import android.view.View;
import android.view.ViewGroup;
import android.view.ViewTreeObserver.OnPreDrawListener;
import android.view.ViewTreeObserver;
import java.lang.ref.WeakReference;
import java.util.function.Consumer;
/**
@@ -88,8 +92,10 @@ public class InlineContentView extends ViewGroup {
*
* @hide
*/
@TestApi
public interface SurfacePackageUpdater {
/**
* Called when the previous surface package is released due to view being detached
* from the window.
@@ -101,14 +107,16 @@ public class InlineContentView extends ViewGroup {
*
* @param consumer consumes the updated surface package.
*/
void getSurfacePackage(Consumer<SurfaceControlViewHost.SurfacePackage> consumer);
void getSurfacePackage(@NonNull Consumer<SurfaceControlViewHost.SurfacePackage> consumer);
}
@NonNull
private final SurfaceHolder.Callback mSurfaceCallback = new SurfaceHolder.Callback() {
@Override
public void surfaceCreated(@NonNull SurfaceHolder holder) {
mSurfaceControlCallback.onCreated(mSurfaceView.getSurfaceControl());
final SurfaceControl surfaceControl = mSurfaceView.getSurfaceControl();
surfaceControl.addOnReparentListener(mOnReparentListener);
mSurfaceControlCallback.onCreated(surfaceControl);
}
@Override
@@ -119,29 +127,58 @@ public class InlineContentView extends ViewGroup {
@Override
public void surfaceDestroyed(@NonNull SurfaceHolder holder) {
mSurfaceControlCallback.onDestroyed(mSurfaceView.getSurfaceControl());
final SurfaceControl surfaceControl = mSurfaceView.getSurfaceControl();
surfaceControl.removeOnReparentListener(mOnReparentListener);
mSurfaceControlCallback.onDestroyed(surfaceControl);
}
};
@NonNull
private final SurfaceControl.OnReparentListener mOnReparentListener =
new SurfaceControl.OnReparentListener() {
@Override
public void onReparent(SurfaceControl.Transaction transaction,
SurfaceControl parent) {
final View parentSurfaceOwnerView = (parent != null)
? parent.getLocalOwnerView() : null;
if (parentSurfaceOwnerView instanceof SurfaceView) {
mParentSurfaceOwnerView = new WeakReference<>(
(SurfaceView) parentSurfaceOwnerView);
} else {
mParentSurfaceOwnerView = null;
}
}
};
@NonNull
private final ViewTreeObserver.OnDrawListener mOnDrawListener =
new ViewTreeObserver.OnDrawListener() {
@Override
public void onDraw() {
computeParentPositionAndScale();
final int visibility = InlineContentView.this.isShown() ? VISIBLE : GONE;
mSurfaceView.setVisibility(visibility);
}
};
@NonNull
private final SurfaceView mSurfaceView;
@Nullable
private WeakReference<SurfaceView> mParentSurfaceOwnerView;
@Nullable
private int[] mParentPosition;
@Nullable
private PointF mParentScale;
@Nullable
private SurfaceControlCallback mSurfaceControlCallback;
@Nullable
private SurfacePackageUpdater mSurfacePackageUpdater;
@NonNull
private final OnPreDrawListener mDrawListener = new OnPreDrawListener() {
@Override
public boolean onPreDraw() {
int visibility = InlineContentView.this.isShown() ? VISIBLE : GONE;
mSurfaceView.setVisibility(visibility);
return true;
}
};
/**
* @inheritDoc
* @hide
@@ -192,10 +229,33 @@ public class InlineContentView extends ViewGroup {
public InlineContentView(@NonNull Context context, @Nullable AttributeSet attrs,
int defStyleAttr, int defStyleRes) {
super(context, attrs, defStyleAttr, defStyleRes);
mSurfaceView = new SurfaceView(context, attrs, defStyleAttr, defStyleRes);
mSurfaceView = new SurfaceView(context, attrs, defStyleAttr, defStyleRes) {
@Override
protected void onSetSurfacePositionAndScaleRT(
@NonNull SurfaceControl.Transaction transaction,
@NonNull SurfaceControl surface, int positionLeft, int positionTop,
float postScaleX, float postScaleY) {
// If we have a parent position, we need to make our coordinates relative
// to the parent in the rendering space.
if (mParentPosition != null) {
positionLeft = (int) ((positionLeft - mParentPosition[0]) / mParentScale.x);
positionTop = (int) ((positionTop - mParentPosition[1]) / mParentScale.y);
}
// Any scaling done to the parent or its predecessors would be applied
// via the surfaces parent -> child relation, so we only propagate any
// scaling set on the InlineContentView itself.
postScaleX = InlineContentView.this.getScaleX();
postScaleY = InlineContentView.this.getScaleY();
super.onSetSurfacePositionAndScaleRT(transaction, surface, positionLeft,
positionTop, postScaleX, postScaleY);
}
};
mSurfaceView.setZOrderOnTop(true);
mSurfaceView.getHolder().setFormat(PixelFormat.TRANSPARENT);
addView(mSurfaceView);
setImportantForAccessibility(IMPORTANT_FOR_ACCESSIBILITY_NO);
}
/**
@@ -203,6 +263,7 @@ public class InlineContentView extends ViewGroup {
*
* @hide
*/
@TestApi
public void setChildSurfacePackageUpdater(
@Nullable SurfacePackageUpdater surfacePackageUpdater) {
mSurfacePackageUpdater = surfacePackageUpdater;
@@ -221,8 +282,9 @@ public class InlineContentView extends ViewGroup {
}
});
}
mSurfaceView.setVisibility(VISIBLE);
getViewTreeObserver().addOnPreDrawListener(mDrawListener);
mSurfaceView.setVisibility(getVisibility());
getViewTreeObserver().addOnDrawListener(mOnDrawListener);
}
@Override
@@ -232,7 +294,9 @@ public class InlineContentView extends ViewGroup {
if (mSurfacePackageUpdater != null) {
mSurfacePackageUpdater.onSurfacePackageReleased();
}
getViewTreeObserver().removeOnPreDrawListener(mDrawListener);
getViewTreeObserver().removeOnDrawListener(mOnDrawListener);
mSurfaceView.setVisibility(View.GONE);
}
@Override
@@ -279,4 +343,67 @@ public class InlineContentView extends ViewGroup {
public boolean setZOrderedOnTop(boolean onTop) {
return mSurfaceView.setZOrderedOnTop(onTop, /*allowDynamicChange*/ true);
}
private void computeParentPositionAndScale() {
boolean contentPositionOrScaleChanged = false;
// This method can be called on the UI or render thread but for the cases
// it is called these threads are not running concurrently, so no need to lock.
final SurfaceView parentSurfaceOwnerView = (mParentSurfaceOwnerView != null)
? mParentSurfaceOwnerView.get() : null;
if (parentSurfaceOwnerView != null) {
if (mParentPosition == null) {
mParentPosition = new int[2];
}
final int oldParentPositionX = mParentPosition[0];
final int oldParentPositionY = mParentPosition[1];
parentSurfaceOwnerView.getLocationInSurface(mParentPosition);
if (oldParentPositionX != mParentPosition[0]
|| oldParentPositionY != mParentPosition[1]) {
contentPositionOrScaleChanged = true;
}
if (mParentScale == null) {
mParentScale = new PointF();
}
final float lastParentSurfaceWidth = parentSurfaceOwnerView
.getSurfaceRenderPosition().width();
final float oldParentScaleX = mParentScale.x;
if (lastParentSurfaceWidth > 0) {
mParentScale.x = lastParentSurfaceWidth /
(float) parentSurfaceOwnerView.getWidth();
} else {
mParentScale.x = 1.0f;
}
if (!contentPositionOrScaleChanged
&& Float.compare(oldParentScaleX, mParentScale.x) != 0) {
contentPositionOrScaleChanged = true;
}
final float lastParentSurfaceHeight = parentSurfaceOwnerView
.getSurfaceRenderPosition().height();
final float oldParentScaleY = mParentScale.y;
if (lastParentSurfaceHeight > 0) {
mParentScale.y = lastParentSurfaceHeight
/ (float) parentSurfaceOwnerView.getHeight();
} else {
mParentScale.y = 1.0f;
}
if (!contentPositionOrScaleChanged
&& Float.compare(oldParentScaleY, mParentScale.y) != 0) {
contentPositionOrScaleChanged = true;
}
} else if (mParentPosition != null || mParentScale != null) {
contentPositionOrScaleChanged = true;
mParentPosition = null;
mParentScale = null;
}
if (contentPositionOrScaleChanged) {
mSurfaceView.requestUpdateSurfacePositionAndScale();
}
}
}