Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Descreses the counts of windows_change_bounds events

Browse Source 
When the display magnification is activated, the window bounds of
the windows callbacked from the SF will be the real position after
the magnifying. We need to use the inverse matrix of magnification spec
to compute the original bounds without the magnification effect. And we
found if the magnification spec changed too quickly, the computed window
bounds won't be correct because the callbacked windows will get one
frame behind.

This symptom will make the window bounds of the same window isn't the
same, and send the windows_change_bounds event out.

To descreses the counts of the window_change_bounds events, we stores
two consecutive magnification specs, and found out which one is the
proper one closing the identity matrix for generating the inverse
matrix based on below formula under window is at the stable state based
on the assumption, We don't consider the transition effect because this
condition wouldn't happen with changing the magnification spec
simultaneously:
WindowState#transform * MagnificationSpecMatrix * inputWindowHandle#transform
= IdentityMatrix

Bug: 191736824
Test: a11y CTS & unit tests
Test: Manual testing including the A11y services
Change-Id: Ib447d696feaa2758fe3192f7e570cce8019f40df
This commit is contained in:
Jacky Kao
2021-11-17 15:27:42 +08:00
parent 914cef5f32
commit dabdcc6738
2 changed files with 281 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
services/core/java/com/android/server/wm/AccessibilityController.java
View File

@@ -263,6 +263,8 @@ final class AccessibilityController {
FLAGS_MAGNIFICATION_CALLBACK | FLAGS_WINDOWS_FOR_ACCESSIBILITY_CALLBACK,
"displayId=" + displayId + "; spec={" + spec + "}");
}
mAccessibilityWindowsPopulator.setMagnificationSpec(displayId, spec);
final DisplayMagnifier displayMagnifier = mDisplayMagnifiers.get(displayId);
if (displayMagnifier != null) {
displayMagnifier.setMagnificationSpec(spec);
@@ -454,19 +456,6 @@ final class AccessibilityController {
return null;
}
boolean getMagnificationSpecForDisplay(int displayId, MagnificationSpec outSpec) {
if (mAccessibilityTracing.isTracingEnabled(FLAGS_MAGNIFICATION_CALLBACK)) {
mAccessibilityTracing.logTrace(TAG + ".getMagnificationSpecForDisplay",
FLAGS_MAGNIFICATION_CALLBACK, "displayId=" + displayId);
}
final DisplayMagnifier displayMagnifier = mDisplayMagnifiers.get(displayId);
if (displayMagnifier == null) {
return false;
}
return displayMagnifier.getMagnificationSpec(outSpec);
}
boolean hasCallbacks() {
if (mAccessibilityTracing.isTracingEnabled(FLAGS_MAGNIFICATION_CALLBACK
| FLAGS_WINDOWS_FOR_ACCESSIBILITY_CALLBACK)) {
@@ -768,25 +757,6 @@ final class AccessibilityController {
return spec;
}
boolean getMagnificationSpec(MagnificationSpec outSpec) {
if (mAccessibilityTracing.isTracingEnabled(FLAGS_MAGNIFICATION_CALLBACK)) {
mAccessibilityTracing.logTrace(LOG_TAG + ".getMagnificationSpec",
FLAGS_MAGNIFICATION_CALLBACK);
}
MagnificationSpec spec = mMagnifedViewport.getMagnificationSpec();
if (spec == null) {
return false;
}
outSpec.setTo(spec);
if (mAccessibilityTracing.isTracingEnabled(FLAGS_MAGNIFICATION_CALLBACK)) {
mAccessibilityTracing.logTrace(LOG_TAG + ".getMagnificationSpec",
FLAGS_MAGNIFICATION_CALLBACK, "outSpec={" + outSpec + "}");
}
return true;
}
void getMagnificationRegion(Region outMagnificationRegion) {
if (mAccessibilityTracing.isTracingEnabled(FLAGS_MAGNIFICATION_CALLBACK)) {
mAccessibilityTracing.logTrace(LOG_TAG + ".getMagnificationRegion",

323
services/core/java/com/android/server/wm/AccessibilityWindowsPopulator.java
View File

@@ -41,7 +41,9 @@ import android.window.WindowInfosListener;
import com.android.internal.annotations.GuardedBy;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* This class is the accessibility windows population adapter.
@@ -68,13 +70,24 @@ public final class AccessibilityWindowsPopulator extends WindowInfosListener {
private final SparseArray<Matrix> mMagnificationSpecInverseMatrix = new SparseArray<>();
@GuardedBy("mLock")
private final SparseArray<DisplayInfo> mDisplayInfos = new SparseArray<>();
private final SparseArray<MagnificationSpec> mCurrentMagnificationSpec = new SparseArray<>();
@GuardedBy("mLock")
private final SparseArray<MagnificationSpec> mPreviousMagnificationSpec = new SparseArray<>();
@GuardedBy("mLock")
private final List<InputWindowHandle> mVisibleWindows = new ArrayList<>();
@GuardedBy("mLock")
private boolean mWindowsNotificationEnabled = false;
@GuardedBy("mLock")
private final Map<IBinder, Matrix> mWindowsTransformMatrixMap = new HashMap<>();
private final Object mLock = new Object();
private final Handler mHandler;
private final Matrix mTempMatrix1 = new Matrix();
private final Matrix mTempMatrix2 = new Matrix();
private final float[] mTempFloat1 = new float[9];
private final float[] mTempFloat2 = new float[9];
private final float[] mTempFloat3 = new float[9];
AccessibilityWindowsPopulator(WindowManagerService service,
AccessibilityController accessibilityController) {
mService = service;
@@ -132,28 +145,55 @@ public final class AccessibilityWindowsPopulator extends WindowInfosListener {
@Override
public void onWindowInfosChanged(InputWindowHandle[] windowHandles,
DisplayInfo[] displayInfos) {
synchronized (mLock) {
mVisibleWindows.clear();
for (InputWindowHandle window : windowHandles) {
if (window.visible && window.getWindow() != null) {
mVisibleWindows.add(window);
}
final List<InputWindowHandle> tempVisibleWindows = new ArrayList<>();
for (InputWindowHandle window : windowHandles) {
if (window.visible && window.getWindow() != null) {
tempVisibleWindows.add(window);
}
}
final HashMap<IBinder, Matrix> windowsTransformMatrixMap =
getWindowsTransformMatrix(tempVisibleWindows);
synchronized (mLock) {
mWindowsTransformMatrixMap.clear();
mWindowsTransformMatrixMap.putAll(windowsTransformMatrixMap);
mVisibleWindows.clear();
mVisibleWindows.addAll(tempVisibleWindows);
mDisplayInfos.clear();
for (final DisplayInfo displayInfo : displayInfos) {
mDisplayInfos.put(displayInfo.mDisplayId, displayInfo);
}
if (mWindowsNotificationEnabled) {
if (!mHandler.hasMessages(
MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_TIMEOUT)) {
mHandler.sendEmptyMessageDelayed(
MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_TIMEOUT,
WINDOWS_CHANGED_NOTIFICATION_MAX_DURATION_TIMES_MS);
}
populateVisibleWindowHandlesAndNotifyWindowsChangeIfNeededLocked();
if (!mHandler.hasMessages(
MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_TIMEOUT)) {
mHandler.sendEmptyMessageDelayed(
MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_TIMEOUT,
WINDOWS_CHANGED_NOTIFICATION_MAX_DURATION_TIMES_MS);
}
populateVisibleWindowHandlesAndNotifyWindowsChangeIfNeeded();
}
}
private HashMap<IBinder, Matrix> getWindowsTransformMatrix(List<InputWindowHandle> windows) {
synchronized (mService.mGlobalLock) {
final HashMap<IBinder, Matrix> windowsTransformMatrixMap = new HashMap<>();
for (InputWindowHandle inputWindowHandle : windows) {
final IWindow iWindow = inputWindowHandle.getWindow();
final WindowState windowState = iWindow != null ? mService.mWindowMap.get(
iWindow.asBinder()) : null;
if (windowState != null && windowState.shouldMagnify()) {
final Matrix transformMatrix = new Matrix();
windowState.getTransformationMatrix(sTempFloats, transformMatrix);
windowsTransformMatrixMap.put(iWindow.asBinder(), transformMatrix);
}
}
return windowsTransformMatrixMap;
}
}
@@ -171,14 +211,43 @@ public final class AccessibilityWindowsPopulator extends WindowInfosListener {
}
mWindowsNotificationEnabled = register;
if (mWindowsNotificationEnabled) {
populateVisibleWindowHandlesAndNotifyWindowsChangeIfNeededLocked();
populateVisibleWindowHandlesAndNotifyWindowsChangeIfNeeded();
} else {
releaseResources();
}
}
}
private void populateVisibleWindowHandlesAndNotifyWindowsChangeIfNeededLocked() {
/**
* Sets the magnification spec for calculating the window bounds of all windows
* reported from the surface flinger in the magnifying.
*
* @param displayId The display Id.
* @param spec THe magnification spec.
*/
public void setMagnificationSpec(int displayId, MagnificationSpec spec) {
synchronized (mLock) {
MagnificationSpec currentMagnificationSpec = mCurrentMagnificationSpec.get(displayId);
if (currentMagnificationSpec == null) {
currentMagnificationSpec = new MagnificationSpec();
currentMagnificationSpec.setTo(spec);
mCurrentMagnificationSpec.put(displayId, currentMagnificationSpec);
return;
}
MagnificationSpec previousMagnificationSpec = mPreviousMagnificationSpec.get(displayId);
if (previousMagnificationSpec == null) {
previousMagnificationSpec = new MagnificationSpec();
mPreviousMagnificationSpec.put(displayId, previousMagnificationSpec);
}
previousMagnificationSpec.setTo(currentMagnificationSpec);
currentMagnificationSpec.setTo(spec);
}
}
@GuardedBy("mLock")
private void populateVisibleWindowHandlesAndNotifyWindowsChangeIfNeeded() {
final SparseArray<List<InputWindowHandle>> tempWindowHandleList = new SparseArray<>();
for (final InputWindowHandle windowHandle : mVisibleWindows) {
@@ -188,15 +257,15 @@ public final class AccessibilityWindowsPopulator extends WindowInfosListener {
if (inputWindowHandles == null) {
inputWindowHandles = new ArrayList<>();
tempWindowHandleList.put(windowHandle.displayId, inputWindowHandles);
generateMagnificationSpecInverseMatrixLocked(windowHandle.displayId);
}
inputWindowHandles.add(windowHandle);
}
findMagnificationSpecInverseMatrixIfNeeded(tempWindowHandleList);
final List<Integer> displayIdsForWindowsChanged = new ArrayList<>();
getDisplaysForWindowsChangedLocked(displayIdsForWindowsChanged, tempWindowHandleList,
getDisplaysForWindowsChanged(displayIdsForWindowsChanged, tempWindowHandleList,
mInputWindowHandlesOnDisplays);
// Clones all windows from the callback of the surface flinger.
mInputWindowHandlesOnDisplays.clear();
for (int i = 0; i < tempWindowHandleList.size(); i++) {
@@ -204,7 +273,7 @@ public final class AccessibilityWindowsPopulator extends WindowInfosListener {
mInputWindowHandlesOnDisplays.put(displayId, tempWindowHandleList.get(displayId));
}
if (displayIdsForWindowsChanged.size() > 0) {
if (!displayIdsForWindowsChanged.isEmpty()) {
if (!mHandler.hasMessages(MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED)) {
mHandler.obtainMessage(MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED,
displayIdsForWindowsChanged).sendToTarget();
@@ -217,7 +286,8 @@ public final class AccessibilityWindowsPopulator extends WindowInfosListener {
SURFACE_FLINGER_CALLBACK_WINDOWS_STABLE_TIMES_MS);
}
private void getDisplaysForWindowsChangedLocked(List<Integer> outDisplayIdsForWindowsChanged,
@GuardedBy("mLock")
private static void getDisplaysForWindowsChanged(List<Integer> outDisplayIdsForWindowsChanged,
SparseArray<List<InputWindowHandle>> newWindowsList,
SparseArray<List<InputWindowHandle>> oldWindowsList) {
for (int i = 0; i < newWindowsList.size(); i++) {
@@ -225,13 +295,14 @@ public final class AccessibilityWindowsPopulator extends WindowInfosListener {
final List<InputWindowHandle> newWindows = newWindowsList.get(displayId);
final List<InputWindowHandle> oldWindows = oldWindowsList.get(displayId);
if (hasWindowsChangedLocked(newWindows, oldWindows)) {
if (hasWindowsChanged(newWindows, oldWindows)) {
outDisplayIdsForWindowsChanged.add(displayId);
}
}
}
private boolean hasWindowsChangedLocked(List<InputWindowHandle> newWindows,
@GuardedBy("mLock")
private static boolean hasWindowsChanged(List<InputWindowHandle> newWindows,
List<InputWindowHandle> oldWindows) {
if (oldWindows == null || oldWindows.size() != newWindows.size()) {
return true;
@@ -253,34 +324,195 @@ public final class AccessibilityWindowsPopulator extends WindowInfosListener {
return false;
}
private void generateMagnificationSpecInverseMatrixLocked(int displayId) {
MagnificationSpec spec = new MagnificationSpec();
if (!mAccessibilityController.getMagnificationSpecForDisplay(displayId, spec)) {
mMagnificationSpecInverseMatrix.remove(displayId);
return;
@GuardedBy("mLock")
private void findMagnificationSpecInverseMatrixIfNeeded(SparseArray<List<InputWindowHandle>>
windowHandleList) {
MagnificationSpec currentMagnificationSpec;
MagnificationSpec previousMagnificationSpec;
for (int i = 0; i < windowHandleList.size(); i++) {
final int displayId = windowHandleList.keyAt(i);
List<InputWindowHandle> inputWindowHandles = windowHandleList.get(displayId);
final MagnificationSpec currentSpec = mCurrentMagnificationSpec.get(displayId);
if (currentSpec == null) {
continue;
}
currentMagnificationSpec = new MagnificationSpec();
currentMagnificationSpec.setTo(currentSpec);
final MagnificationSpec previousSpec = mPreviousMagnificationSpec.get(displayId);
if (previousSpec == null) {
final Matrix inverseMatrixForCurrentSpec = new Matrix();
generateInverseMatrix(currentMagnificationSpec, inverseMatrixForCurrentSpec);
mMagnificationSpecInverseMatrix.put(displayId, inverseMatrixForCurrentSpec);
continue;
}
previousMagnificationSpec = new MagnificationSpec();
previousMagnificationSpec.setTo(previousSpec);
generateInverseMatrixBasedOnProperMagnificationSpecForDisplay(inputWindowHandles,
currentMagnificationSpec, previousMagnificationSpec);
}
sTempFloats[Matrix.MSCALE_X] = spec.scale;
sTempFloats[Matrix.MSKEW_Y] = 0;
sTempFloats[Matrix.MSKEW_X] = 0;
sTempFloats[Matrix.MSCALE_Y] = spec.scale;
sTempFloats[Matrix.MTRANS_X] = spec.offsetX;
sTempFloats[Matrix.MTRANS_Y] = spec.offsetY;
sTempFloats[Matrix.MPERSP_0] = 0;
sTempFloats[Matrix.MPERSP_1] = 0;
sTempFloats[Matrix.MPERSP_2] = 1;
}
@GuardedBy("mLock")
private void generateInverseMatrixBasedOnProperMagnificationSpecForDisplay(
List<InputWindowHandle> inputWindowHandles, MagnificationSpec currentMagnificationSpec,
MagnificationSpec previousMagnificationSpec) {
// To decrease the counts of holding the WindowManagerService#mGlogalLock in
// the method, getWindowTransformMatrix(), this for loop begins from the bottom
// to top of the z-order windows.
for (int index = inputWindowHandles.size() - 1; index >= 0; index--) {
final Matrix windowTransformMatrix = mTempMatrix2;
final InputWindowHandle windowHandle = inputWindowHandles.get(index);
final IBinder iBinder = windowHandle.getWindow().asBinder();
if (getWindowTransformMatrix(iBinder, windowTransformMatrix)) {
generateMagnificationSpecInverseMatrix(windowHandle, currentMagnificationSpec,
previousMagnificationSpec, windowTransformMatrix);
break;
}
}
}
@GuardedBy("mLock")
private boolean getWindowTransformMatrix(IBinder iBinder, Matrix outTransform) {
final Matrix windowMatrix = iBinder != null
? mWindowsTransformMatrixMap.get(iBinder) : null;
if (windowMatrix == null) {
return false;
}
outTransform.set(windowMatrix);
return true;
}
/**
* Generates the inverse matrix based on the proper magnification spec.
* The magnification spec associated with the InputWindowHandle might not the current
* spec set by WM, which might be the previous one. To find the appropriate spec,
* we store two consecutive magnification specs, and found out which one is the proper
* one closing the identity matrix for generating the inverse matrix.
*
* @param inputWindowHandle The window from the surface flinger.
* @param currentMagnificationSpec The current magnification spec.
* @param previousMagnificationSpec The previous magnification spec.
* @param transformMatrix The transform matrix of the window doesn't consider the
* magnifying effect.
*/
@GuardedBy("mLock")
private void generateMagnificationSpecInverseMatrix(InputWindowHandle inputWindowHandle,
@NonNull MagnificationSpec currentMagnificationSpec,
@NonNull MagnificationSpec previousMagnificationSpec, Matrix transformMatrix) {
final float[] identityMatrixFloatsForCurrentSpec = mTempFloat1;
computeIdentityMatrix(inputWindowHandle, currentMagnificationSpec,
transformMatrix, identityMatrixFloatsForCurrentSpec);
final float[] identityMatrixFloatsForPreviousSpec = mTempFloat2;
computeIdentityMatrix(inputWindowHandle, previousMagnificationSpec,
transformMatrix, identityMatrixFloatsForPreviousSpec);
Matrix inverseMatrixForMagnificationSpec = new Matrix();
if (selectProperMagnificationSpecByComparingIdentityDegree(
identityMatrixFloatsForCurrentSpec, identityMatrixFloatsForPreviousSpec)) {
generateInverseMatrix(currentMagnificationSpec,
inverseMatrixForMagnificationSpec);
// Choosing the current spec means the previous spec is out of date,
// so removing it. And if the current spec is no magnifying, meaning
// the magnifying is done so removing the inverse matrix of this display.
mPreviousMagnificationSpec.remove(inputWindowHandle.displayId);
if (currentMagnificationSpec.isNop()) {
mCurrentMagnificationSpec.remove(inputWindowHandle.displayId);
mMagnificationSpecInverseMatrix.remove(inputWindowHandle.displayId);
return;
}
} else {
generateInverseMatrix(previousMagnificationSpec,
inverseMatrixForMagnificationSpec);
}
mMagnificationSpecInverseMatrix.put(inputWindowHandle.displayId,
inverseMatrixForMagnificationSpec);
}
/**
* Computes the identity matrix for generating the
* inverse matrix based on below formula under window is at the stable state:
* inputWindowHandle#transform * MagnificationSpecMatrix * WindowState#transform
* = IdentityMatrix
*/
@GuardedBy("mLock")
private void computeIdentityMatrix(InputWindowHandle inputWindowHandle,
@NonNull MagnificationSpec magnificationSpec,
Matrix transformMatrix, float[] magnifyMatrixFloats) {
final Matrix specMatrix = mTempMatrix1;
transformMagnificationSpecToMatrix(magnificationSpec, specMatrix);
final Matrix resultMatrix = new Matrix(inputWindowHandle.transform);
resultMatrix.preConcat(specMatrix);
resultMatrix.preConcat(transformMatrix);
resultMatrix.getValues(magnifyMatrixFloats);
}
/**
* @return true if selecting the magnification spec one, otherwise selecting the
* magnification spec two.
*/
@GuardedBy("mLock")
private boolean selectProperMagnificationSpecByComparingIdentityDegree(
float[] magnifyMatrixFloatsForSpecOne,
float[] magnifyMatrixFloatsForSpecTwo) {
final float[] IdentityMatrixValues = mTempFloat3;
Matrix.IDENTITY_MATRIX.getValues(IdentityMatrixValues);
final float scaleDiffForSpecOne = Math.abs(IdentityMatrixValues[Matrix.MSCALE_X]
- magnifyMatrixFloatsForSpecOne[Matrix.MSCALE_X]);
final float scaleDiffForSpecTwo = Math.abs(IdentityMatrixValues[Matrix.MSCALE_X]
- magnifyMatrixFloatsForSpecTwo[Matrix.MSCALE_X]);
final float offsetXDiffForSpecOne = Math.abs(IdentityMatrixValues[Matrix.MTRANS_X]
- magnifyMatrixFloatsForSpecOne[Matrix.MTRANS_X]);
final float offsetXDiffForSpecTwo = Math.abs(IdentityMatrixValues[Matrix.MTRANS_X]
- magnifyMatrixFloatsForSpecTwo[Matrix.MTRANS_X]);
final float offsetYDiffForSpecOne = Math.abs(IdentityMatrixValues[Matrix.MTRANS_Y]
- magnifyMatrixFloatsForSpecOne[Matrix.MTRANS_Y]);
final float offsetYDiffForSpecTwo = Math.abs(IdentityMatrixValues[Matrix.MTRANS_Y]
- magnifyMatrixFloatsForSpecTwo[Matrix.MTRANS_Y]);
final float offsetDiffForSpecOne = offsetXDiffForSpecOne
+ offsetYDiffForSpecOne;
final float offsetDiffForSpecTwo = offsetXDiffForSpecTwo
+ offsetYDiffForSpecTwo;
return Float.compare(scaleDiffForSpecTwo, scaleDiffForSpecOne) > 0
|| (Float.compare(scaleDiffForSpecTwo, scaleDiffForSpecOne) == 0
&& Float.compare(offsetDiffForSpecTwo, offsetDiffForSpecOne) > 0);
}
@GuardedBy("mLock")
private static void generateInverseMatrix(MagnificationSpec spec, Matrix outMatrix) {
outMatrix.reset();
final Matrix tempMatrix = new Matrix();
tempMatrix.setValues(sTempFloats);
final Matrix inverseMatrix = new Matrix();
final boolean result = tempMatrix.invert(inverseMatrix);
transformMagnificationSpecToMatrix(spec, tempMatrix);
final boolean result = tempMatrix.invert(outMatrix);
if (!result) {
Slog.e(TAG, "Can't inverse the magnification spec matrix with the "
+ "magnification spec = " + spec + " on the displayId = " + displayId);
return;
+ "magnification spec = " + spec);
outMatrix.reset();
}
mMagnificationSpecInverseMatrix.set(displayId, inverseMatrix);
}
@GuardedBy("mLock")
private static void transformMagnificationSpecToMatrix(MagnificationSpec spec,
Matrix outMatrix) {
outMatrix.reset();
outMatrix.postScale(spec.scale, spec.scale);
outMatrix.postTranslate(spec.offsetX, spec.offsetY);
}
private void notifyWindowsChanged(@NonNull List<Integer> displayIdsForWindowsChanged) {
@@ -310,6 +542,9 @@ public final class AccessibilityWindowsPopulator extends WindowInfosListener {
mMagnificationSpecInverseMatrix.clear();
mVisibleWindows.clear();
mDisplayInfos.clear();
mCurrentMagnificationSpec.clear();
mPreviousMagnificationSpec.clear();
mWindowsTransformMatrixMap.clear();
mWindowsNotificationEnabled = false;
mHandler.removeCallbacksAndMessages(null);
}