From dabdcc6738d92a6e6018466f564d8305ab5f9c67 Mon Sep 17 00:00:00 2001 From: Jacky Kao Date: Wed, 17 Nov 2021 15:27:42 +0800 Subject: [PATCH] Descreses the counts of windows_change_bounds events When the display magnification is activated, the window bounds of the windows callbacked from the SF will be the real position after the magnifying. We need to use the inverse matrix of magnification spec to compute the original bounds without the magnification effect. And we found if the magnification spec changed too quickly, the computed window bounds won't be correct because the callbacked windows will get one frame behind. This symptom will make the window bounds of the same window isn't the same, and send the windows_change_bounds event out. To descreses the counts of the window_change_bounds events, we stores two consecutive magnification specs, and found out which one is the proper one closing the identity matrix for generating the inverse matrix based on below formula under window is at the stable state based on the assumption, We don't consider the transition effect because this condition wouldn't happen with changing the magnification spec simultaneously: WindowState#transform * MagnificationSpecMatrix * inputWindowHandle#transform = IdentityMatrix Bug: 191736824 Test: a11y CTS & unit tests Test: Manual testing including the A11y services Change-Id: Ib447d696feaa2758fe3192f7e570cce8019f40df --- .../server/wm/AccessibilityController.java | 34 +- .../wm/AccessibilityWindowsPopulator.java | 323 +++++++++++++++--- 2 files changed, 281 insertions(+), 76 deletions(-) diff --git a/services/core/java/com/android/server/wm/AccessibilityController.java b/services/core/java/com/android/server/wm/AccessibilityController.java index 8f703c5c77612..57f77d5b7b945 100644 --- a/services/core/java/com/android/server/wm/AccessibilityController.java +++ b/services/core/java/com/android/server/wm/AccessibilityController.java @@ -263,6 +263,8 @@ final class AccessibilityController { FLAGS_MAGNIFICATION_CALLBACK | FLAGS_WINDOWS_FOR_ACCESSIBILITY_CALLBACK, "displayId=" + displayId + "; spec={" + spec + "}"); } + mAccessibilityWindowsPopulator.setMagnificationSpec(displayId, spec); + final DisplayMagnifier displayMagnifier = mDisplayMagnifiers.get(displayId); if (displayMagnifier != null) { displayMagnifier.setMagnificationSpec(spec); @@ -454,19 +456,6 @@ final class AccessibilityController { return null; } - boolean getMagnificationSpecForDisplay(int displayId, MagnificationSpec outSpec) { - if (mAccessibilityTracing.isTracingEnabled(FLAGS_MAGNIFICATION_CALLBACK)) { - mAccessibilityTracing.logTrace(TAG + ".getMagnificationSpecForDisplay", - FLAGS_MAGNIFICATION_CALLBACK, "displayId=" + displayId); - } - final DisplayMagnifier displayMagnifier = mDisplayMagnifiers.get(displayId); - if (displayMagnifier == null) { - return false; - } - - return displayMagnifier.getMagnificationSpec(outSpec); - } - boolean hasCallbacks() { if (mAccessibilityTracing.isTracingEnabled(FLAGS_MAGNIFICATION_CALLBACK | FLAGS_WINDOWS_FOR_ACCESSIBILITY_CALLBACK)) { @@ -768,25 +757,6 @@ final class AccessibilityController { return spec; } - boolean getMagnificationSpec(MagnificationSpec outSpec) { - if (mAccessibilityTracing.isTracingEnabled(FLAGS_MAGNIFICATION_CALLBACK)) { - mAccessibilityTracing.logTrace(LOG_TAG + ".getMagnificationSpec", - FLAGS_MAGNIFICATION_CALLBACK); - } - MagnificationSpec spec = mMagnifedViewport.getMagnificationSpec(); - if (spec == null) { - return false; - } - - outSpec.setTo(spec); - if (mAccessibilityTracing.isTracingEnabled(FLAGS_MAGNIFICATION_CALLBACK)) { - mAccessibilityTracing.logTrace(LOG_TAG + ".getMagnificationSpec", - FLAGS_MAGNIFICATION_CALLBACK, "outSpec={" + outSpec + "}"); - } - - return true; - } - void getMagnificationRegion(Region outMagnificationRegion) { if (mAccessibilityTracing.isTracingEnabled(FLAGS_MAGNIFICATION_CALLBACK)) { mAccessibilityTracing.logTrace(LOG_TAG + ".getMagnificationRegion", diff --git a/services/core/java/com/android/server/wm/AccessibilityWindowsPopulator.java b/services/core/java/com/android/server/wm/AccessibilityWindowsPopulator.java index c0fb83ba294c6..43317adca04a6 100644 --- a/services/core/java/com/android/server/wm/AccessibilityWindowsPopulator.java +++ b/services/core/java/com/android/server/wm/AccessibilityWindowsPopulator.java @@ -41,7 +41,9 @@ import android.window.WindowInfosListener; import com.android.internal.annotations.GuardedBy; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; /** * This class is the accessibility windows population adapter. @@ -68,13 +70,24 @@ public final class AccessibilityWindowsPopulator extends WindowInfosListener { private final SparseArray mMagnificationSpecInverseMatrix = new SparseArray<>(); @GuardedBy("mLock") private final SparseArray mDisplayInfos = new SparseArray<>(); + private final SparseArray mCurrentMagnificationSpec = new SparseArray<>(); + @GuardedBy("mLock") + private final SparseArray mPreviousMagnificationSpec = new SparseArray<>(); @GuardedBy("mLock") private final List mVisibleWindows = new ArrayList<>(); @GuardedBy("mLock") private boolean mWindowsNotificationEnabled = false; + @GuardedBy("mLock") + private final Map mWindowsTransformMatrixMap = new HashMap<>(); private final Object mLock = new Object(); private final Handler mHandler; + private final Matrix mTempMatrix1 = new Matrix(); + private final Matrix mTempMatrix2 = new Matrix(); + private final float[] mTempFloat1 = new float[9]; + private final float[] mTempFloat2 = new float[9]; + private final float[] mTempFloat3 = new float[9]; + AccessibilityWindowsPopulator(WindowManagerService service, AccessibilityController accessibilityController) { mService = service; @@ -132,28 +145,55 @@ public final class AccessibilityWindowsPopulator extends WindowInfosListener { @Override public void onWindowInfosChanged(InputWindowHandle[] windowHandles, DisplayInfo[] displayInfos) { - synchronized (mLock) { - mVisibleWindows.clear(); - for (InputWindowHandle window : windowHandles) { - if (window.visible && window.getWindow() != null) { - mVisibleWindows.add(window); - } + final List tempVisibleWindows = new ArrayList<>(); + + for (InputWindowHandle window : windowHandles) { + if (window.visible && window.getWindow() != null) { + tempVisibleWindows.add(window); } + } + final HashMap windowsTransformMatrixMap = + getWindowsTransformMatrix(tempVisibleWindows); + + synchronized (mLock) { + mWindowsTransformMatrixMap.clear(); + mWindowsTransformMatrixMap.putAll(windowsTransformMatrixMap); + + mVisibleWindows.clear(); + mVisibleWindows.addAll(tempVisibleWindows); mDisplayInfos.clear(); for (final DisplayInfo displayInfo : displayInfos) { mDisplayInfos.put(displayInfo.mDisplayId, displayInfo); } - if (mWindowsNotificationEnabled) { - if (!mHandler.hasMessages( - MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_TIMEOUT)) { - mHandler.sendEmptyMessageDelayed( - MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_TIMEOUT, - WINDOWS_CHANGED_NOTIFICATION_MAX_DURATION_TIMES_MS); - } - populateVisibleWindowHandlesAndNotifyWindowsChangeIfNeededLocked(); + if (!mHandler.hasMessages( + MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_TIMEOUT)) { + mHandler.sendEmptyMessageDelayed( + MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_TIMEOUT, + WINDOWS_CHANGED_NOTIFICATION_MAX_DURATION_TIMES_MS); } + populateVisibleWindowHandlesAndNotifyWindowsChangeIfNeeded(); + } + } + + private HashMap getWindowsTransformMatrix(List windows) { + synchronized (mService.mGlobalLock) { + final HashMap windowsTransformMatrixMap = new HashMap<>(); + + for (InputWindowHandle inputWindowHandle : windows) { + final IWindow iWindow = inputWindowHandle.getWindow(); + final WindowState windowState = iWindow != null ? mService.mWindowMap.get( + iWindow.asBinder()) : null; + + if (windowState != null && windowState.shouldMagnify()) { + final Matrix transformMatrix = new Matrix(); + windowState.getTransformationMatrix(sTempFloats, transformMatrix); + windowsTransformMatrixMap.put(iWindow.asBinder(), transformMatrix); + } + } + + return windowsTransformMatrixMap; } } @@ -171,14 +211,43 @@ public final class AccessibilityWindowsPopulator extends WindowInfosListener { } mWindowsNotificationEnabled = register; if (mWindowsNotificationEnabled) { - populateVisibleWindowHandlesAndNotifyWindowsChangeIfNeededLocked(); + populateVisibleWindowHandlesAndNotifyWindowsChangeIfNeeded(); } else { releaseResources(); } } } - private void populateVisibleWindowHandlesAndNotifyWindowsChangeIfNeededLocked() { + /** + * Sets the magnification spec for calculating the window bounds of all windows + * reported from the surface flinger in the magnifying. + * + * @param displayId The display Id. + * @param spec THe magnification spec. + */ + public void setMagnificationSpec(int displayId, MagnificationSpec spec) { + synchronized (mLock) { + MagnificationSpec currentMagnificationSpec = mCurrentMagnificationSpec.get(displayId); + if (currentMagnificationSpec == null) { + currentMagnificationSpec = new MagnificationSpec(); + currentMagnificationSpec.setTo(spec); + mCurrentMagnificationSpec.put(displayId, currentMagnificationSpec); + + return; + } + + MagnificationSpec previousMagnificationSpec = mPreviousMagnificationSpec.get(displayId); + if (previousMagnificationSpec == null) { + previousMagnificationSpec = new MagnificationSpec(); + mPreviousMagnificationSpec.put(displayId, previousMagnificationSpec); + } + previousMagnificationSpec.setTo(currentMagnificationSpec); + currentMagnificationSpec.setTo(spec); + } + } + + @GuardedBy("mLock") + private void populateVisibleWindowHandlesAndNotifyWindowsChangeIfNeeded() { final SparseArray> tempWindowHandleList = new SparseArray<>(); for (final InputWindowHandle windowHandle : mVisibleWindows) { @@ -188,15 +257,15 @@ public final class AccessibilityWindowsPopulator extends WindowInfosListener { if (inputWindowHandles == null) { inputWindowHandles = new ArrayList<>(); tempWindowHandleList.put(windowHandle.displayId, inputWindowHandles); - generateMagnificationSpecInverseMatrixLocked(windowHandle.displayId); } inputWindowHandles.add(windowHandle); } + findMagnificationSpecInverseMatrixIfNeeded(tempWindowHandleList); final List displayIdsForWindowsChanged = new ArrayList<>(); - - getDisplaysForWindowsChangedLocked(displayIdsForWindowsChanged, tempWindowHandleList, + getDisplaysForWindowsChanged(displayIdsForWindowsChanged, tempWindowHandleList, mInputWindowHandlesOnDisplays); + // Clones all windows from the callback of the surface flinger. mInputWindowHandlesOnDisplays.clear(); for (int i = 0; i < tempWindowHandleList.size(); i++) { @@ -204,7 +273,7 @@ public final class AccessibilityWindowsPopulator extends WindowInfosListener { mInputWindowHandlesOnDisplays.put(displayId, tempWindowHandleList.get(displayId)); } - if (displayIdsForWindowsChanged.size() > 0) { + if (!displayIdsForWindowsChanged.isEmpty()) { if (!mHandler.hasMessages(MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED)) { mHandler.obtainMessage(MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED, displayIdsForWindowsChanged).sendToTarget(); @@ -217,7 +286,8 @@ public final class AccessibilityWindowsPopulator extends WindowInfosListener { SURFACE_FLINGER_CALLBACK_WINDOWS_STABLE_TIMES_MS); } - private void getDisplaysForWindowsChangedLocked(List outDisplayIdsForWindowsChanged, + @GuardedBy("mLock") + private static void getDisplaysForWindowsChanged(List outDisplayIdsForWindowsChanged, SparseArray> newWindowsList, SparseArray> oldWindowsList) { for (int i = 0; i < newWindowsList.size(); i++) { @@ -225,13 +295,14 @@ public final class AccessibilityWindowsPopulator extends WindowInfosListener { final List newWindows = newWindowsList.get(displayId); final List oldWindows = oldWindowsList.get(displayId); - if (hasWindowsChangedLocked(newWindows, oldWindows)) { + if (hasWindowsChanged(newWindows, oldWindows)) { outDisplayIdsForWindowsChanged.add(displayId); } } } - private boolean hasWindowsChangedLocked(List newWindows, + @GuardedBy("mLock") + private static boolean hasWindowsChanged(List newWindows, List oldWindows) { if (oldWindows == null || oldWindows.size() != newWindows.size()) { return true; @@ -253,34 +324,195 @@ public final class AccessibilityWindowsPopulator extends WindowInfosListener { return false; } - private void generateMagnificationSpecInverseMatrixLocked(int displayId) { - MagnificationSpec spec = new MagnificationSpec(); - if (!mAccessibilityController.getMagnificationSpecForDisplay(displayId, spec)) { - mMagnificationSpecInverseMatrix.remove(displayId); - return; + @GuardedBy("mLock") + private void findMagnificationSpecInverseMatrixIfNeeded(SparseArray> + windowHandleList) { + MagnificationSpec currentMagnificationSpec; + MagnificationSpec previousMagnificationSpec; + for (int i = 0; i < windowHandleList.size(); i++) { + final int displayId = windowHandleList.keyAt(i); + List inputWindowHandles = windowHandleList.get(displayId); + + final MagnificationSpec currentSpec = mCurrentMagnificationSpec.get(displayId); + if (currentSpec == null) { + continue; + } + currentMagnificationSpec = new MagnificationSpec(); + currentMagnificationSpec.setTo(currentSpec); + + final MagnificationSpec previousSpec = mPreviousMagnificationSpec.get(displayId); + + if (previousSpec == null) { + final Matrix inverseMatrixForCurrentSpec = new Matrix(); + generateInverseMatrix(currentMagnificationSpec, inverseMatrixForCurrentSpec); + mMagnificationSpecInverseMatrix.put(displayId, inverseMatrixForCurrentSpec); + continue; + } + previousMagnificationSpec = new MagnificationSpec(); + previousMagnificationSpec.setTo(previousSpec); + + generateInverseMatrixBasedOnProperMagnificationSpecForDisplay(inputWindowHandles, + currentMagnificationSpec, previousMagnificationSpec); } - sTempFloats[Matrix.MSCALE_X] = spec.scale; - sTempFloats[Matrix.MSKEW_Y] = 0; - sTempFloats[Matrix.MSKEW_X] = 0; - sTempFloats[Matrix.MSCALE_Y] = spec.scale; - sTempFloats[Matrix.MTRANS_X] = spec.offsetX; - sTempFloats[Matrix.MTRANS_Y] = spec.offsetY; - sTempFloats[Matrix.MPERSP_0] = 0; - sTempFloats[Matrix.MPERSP_1] = 0; - sTempFloats[Matrix.MPERSP_2] = 1; + } + + @GuardedBy("mLock") + private void generateInverseMatrixBasedOnProperMagnificationSpecForDisplay( + List inputWindowHandles, MagnificationSpec currentMagnificationSpec, + MagnificationSpec previousMagnificationSpec) { + // To decrease the counts of holding the WindowManagerService#mGlogalLock in + // the method, getWindowTransformMatrix(), this for loop begins from the bottom + // to top of the z-order windows. + for (int index = inputWindowHandles.size() - 1; index >= 0; index--) { + final Matrix windowTransformMatrix = mTempMatrix2; + final InputWindowHandle windowHandle = inputWindowHandles.get(index); + final IBinder iBinder = windowHandle.getWindow().asBinder(); + + if (getWindowTransformMatrix(iBinder, windowTransformMatrix)) { + generateMagnificationSpecInverseMatrix(windowHandle, currentMagnificationSpec, + previousMagnificationSpec, windowTransformMatrix); + + break; + } + } + } + + @GuardedBy("mLock") + private boolean getWindowTransformMatrix(IBinder iBinder, Matrix outTransform) { + final Matrix windowMatrix = iBinder != null + ? mWindowsTransformMatrixMap.get(iBinder) : null; + + if (windowMatrix == null) { + return false; + } + outTransform.set(windowMatrix); + + return true; + } + + /** + * Generates the inverse matrix based on the proper magnification spec. + * The magnification spec associated with the InputWindowHandle might not the current + * spec set by WM, which might be the previous one. To find the appropriate spec, + * we store two consecutive magnification specs, and found out which one is the proper + * one closing the identity matrix for generating the inverse matrix. + * + * @param inputWindowHandle The window from the surface flinger. + * @param currentMagnificationSpec The current magnification spec. + * @param previousMagnificationSpec The previous magnification spec. + * @param transformMatrix The transform matrix of the window doesn't consider the + * magnifying effect. + */ + @GuardedBy("mLock") + private void generateMagnificationSpecInverseMatrix(InputWindowHandle inputWindowHandle, + @NonNull MagnificationSpec currentMagnificationSpec, + @NonNull MagnificationSpec previousMagnificationSpec, Matrix transformMatrix) { + + final float[] identityMatrixFloatsForCurrentSpec = mTempFloat1; + computeIdentityMatrix(inputWindowHandle, currentMagnificationSpec, + transformMatrix, identityMatrixFloatsForCurrentSpec); + final float[] identityMatrixFloatsForPreviousSpec = mTempFloat2; + computeIdentityMatrix(inputWindowHandle, previousMagnificationSpec, + transformMatrix, identityMatrixFloatsForPreviousSpec); + + Matrix inverseMatrixForMagnificationSpec = new Matrix(); + if (selectProperMagnificationSpecByComparingIdentityDegree( + identityMatrixFloatsForCurrentSpec, identityMatrixFloatsForPreviousSpec)) { + generateInverseMatrix(currentMagnificationSpec, + inverseMatrixForMagnificationSpec); + + // Choosing the current spec means the previous spec is out of date, + // so removing it. And if the current spec is no magnifying, meaning + // the magnifying is done so removing the inverse matrix of this display. + mPreviousMagnificationSpec.remove(inputWindowHandle.displayId); + if (currentMagnificationSpec.isNop()) { + mCurrentMagnificationSpec.remove(inputWindowHandle.displayId); + mMagnificationSpecInverseMatrix.remove(inputWindowHandle.displayId); + return; + } + } else { + generateInverseMatrix(previousMagnificationSpec, + inverseMatrixForMagnificationSpec); + } + + mMagnificationSpecInverseMatrix.put(inputWindowHandle.displayId, + inverseMatrixForMagnificationSpec); + } + + /** + * Computes the identity matrix for generating the + * inverse matrix based on below formula under window is at the stable state: + * inputWindowHandle#transform * MagnificationSpecMatrix * WindowState#transform + * = IdentityMatrix + */ + @GuardedBy("mLock") + private void computeIdentityMatrix(InputWindowHandle inputWindowHandle, + @NonNull MagnificationSpec magnificationSpec, + Matrix transformMatrix, float[] magnifyMatrixFloats) { + final Matrix specMatrix = mTempMatrix1; + transformMagnificationSpecToMatrix(magnificationSpec, specMatrix); + + final Matrix resultMatrix = new Matrix(inputWindowHandle.transform); + resultMatrix.preConcat(specMatrix); + resultMatrix.preConcat(transformMatrix); + + resultMatrix.getValues(magnifyMatrixFloats); + } + + /** + * @return true if selecting the magnification spec one, otherwise selecting the + * magnification spec two. + */ + @GuardedBy("mLock") + private boolean selectProperMagnificationSpecByComparingIdentityDegree( + float[] magnifyMatrixFloatsForSpecOne, + float[] magnifyMatrixFloatsForSpecTwo) { + final float[] IdentityMatrixValues = mTempFloat3; + Matrix.IDENTITY_MATRIX.getValues(IdentityMatrixValues); + + final float scaleDiffForSpecOne = Math.abs(IdentityMatrixValues[Matrix.MSCALE_X] + - magnifyMatrixFloatsForSpecOne[Matrix.MSCALE_X]); + final float scaleDiffForSpecTwo = Math.abs(IdentityMatrixValues[Matrix.MSCALE_X] + - magnifyMatrixFloatsForSpecTwo[Matrix.MSCALE_X]); + final float offsetXDiffForSpecOne = Math.abs(IdentityMatrixValues[Matrix.MTRANS_X] + - magnifyMatrixFloatsForSpecOne[Matrix.MTRANS_X]); + final float offsetXDiffForSpecTwo = Math.abs(IdentityMatrixValues[Matrix.MTRANS_X] + - magnifyMatrixFloatsForSpecTwo[Matrix.MTRANS_X]); + final float offsetYDiffForSpecOne = Math.abs(IdentityMatrixValues[Matrix.MTRANS_Y] + - magnifyMatrixFloatsForSpecOne[Matrix.MTRANS_Y]); + final float offsetYDiffForSpecTwo = Math.abs(IdentityMatrixValues[Matrix.MTRANS_Y] + - magnifyMatrixFloatsForSpecTwo[Matrix.MTRANS_Y]); + final float offsetDiffForSpecOne = offsetXDiffForSpecOne + + offsetYDiffForSpecOne; + final float offsetDiffForSpecTwo = offsetXDiffForSpecTwo + + offsetYDiffForSpecTwo; + + return Float.compare(scaleDiffForSpecTwo, scaleDiffForSpecOne) > 0 + || (Float.compare(scaleDiffForSpecTwo, scaleDiffForSpecOne) == 0 + && Float.compare(offsetDiffForSpecTwo, offsetDiffForSpecOne) > 0); + } + + @GuardedBy("mLock") + private static void generateInverseMatrix(MagnificationSpec spec, Matrix outMatrix) { + outMatrix.reset(); final Matrix tempMatrix = new Matrix(); - tempMatrix.setValues(sTempFloats); - - final Matrix inverseMatrix = new Matrix(); - final boolean result = tempMatrix.invert(inverseMatrix); + transformMagnificationSpecToMatrix(spec, tempMatrix); + final boolean result = tempMatrix.invert(outMatrix); if (!result) { Slog.e(TAG, "Can't inverse the magnification spec matrix with the " - + "magnification spec = " + spec + " on the displayId = " + displayId); - return; + + "magnification spec = " + spec); + outMatrix.reset(); } - mMagnificationSpecInverseMatrix.set(displayId, inverseMatrix); + } + + @GuardedBy("mLock") + private static void transformMagnificationSpecToMatrix(MagnificationSpec spec, + Matrix outMatrix) { + outMatrix.reset(); + outMatrix.postScale(spec.scale, spec.scale); + outMatrix.postTranslate(spec.offsetX, spec.offsetY); } private void notifyWindowsChanged(@NonNull List displayIdsForWindowsChanged) { @@ -310,6 +542,9 @@ public final class AccessibilityWindowsPopulator extends WindowInfosListener { mMagnificationSpecInverseMatrix.clear(); mVisibleWindows.clear(); mDisplayInfos.clear(); + mCurrentMagnificationSpec.clear(); + mPreviousMagnificationSpec.clear(); + mWindowsTransformMatrixMap.clear(); mWindowsNotificationEnabled = false; mHandler.removeCallbacksAndMessages(null); }