From a2fae6b5d6ef8f1e66a9000c8861531f9b0d7723 Mon Sep 17 00:00:00 2001 From: Jacky Kao Date: Wed, 3 Nov 2021 14:49:53 +0800 Subject: [PATCH 1/4] Uses SF callback windows for the A11yWindowInfo [1/n] 1. Implements AccessibilityWindowsPopulator class including the new accessibilityWindow class generated by the inputWindowHandle from the surface flinger. 2. Replaces the windowState by the accessibilityWindow used in the AccessibiltyController class for computing the changed windows reported to the A11y framework. 3. The embedded hiarerchy windows would be reported to A11y framework due to they are reported from the surface fligner. This will make the traversal function is broken when the Auotfill features is using talkback. We find out the embedded hiarerchy windows by its window token and don't add them into the A11y windows list. 4. Removes the modal windows calculation of the unAccountedSpace including the task fragment due to we intersec the region between the frame and touch region for all windows. 5. When reported windows counts from the surface flinger changed, calling the accessibilty controller class to compute the changed windows and send the results to the A11y framework immediately. This can avoid the CTS failure with no corresponding window found when the activity is launched due to the timing issue between the callbacks for computing changed windows from the WM and the callback for changed windows from the surface flinger. Bug: 191736824 Test: a11y CTS & unit tests Test: Manual testing including the A11y services Change-Id: Ie9150ddc8e53ca88ad8cd7ba9376837e31436518 --- .../AccessibilityWindowManager.java | 19 + .../server/wm/AccessibilityController.java | 294 ++++------ .../wm/AccessibilityWindowsPopulator.java | 534 ++++++++++++++++++ 3 files changed, 651 insertions(+), 196 deletions(-) create mode 100644 services/core/java/com/android/server/wm/AccessibilityWindowsPopulator.java diff --git a/services/accessibility/java/com/android/server/accessibility/AccessibilityWindowManager.java b/services/accessibility/java/com/android/server/accessibility/AccessibilityWindowManager.java index 59c1461e4abca..aba32ec465b56 100644 --- a/services/accessibility/java/com/android/server/accessibility/AccessibilityWindowManager.java +++ b/services/accessibility/java/com/android/server/accessibility/AccessibilityWindowManager.java @@ -667,6 +667,10 @@ public class AccessibilityWindowManager { return null; } + // Don't need to add the embedded hierarchy windows into the accessibility windows list. + if (mHostEmbeddedMap.size() > 0 && isEmbeddedHierarchyWindowsLocked(windowId)) { + return null; + } final AccessibilityWindowInfo reportedWindow = AccessibilityWindowInfo.obtain(); reportedWindow.setId(windowId); @@ -699,6 +703,21 @@ public class AccessibilityWindowManager { return reportedWindow; } + private boolean isEmbeddedHierarchyWindowsLocked(int windowId) { + final IBinder leashToken = mWindowIdMap.get(windowId); + if (leashToken == null) { + return false; + } + + for (int i = 0; i < mHostEmbeddedMap.size(); i++) { + if (mHostEmbeddedMap.keyAt(i).equals(leashToken)) { + return true; + } + } + + return false; + } + private int getTypeForWindowManagerWindowType(int windowType) { switch (windowType) { case WindowManager.LayoutParams.TYPE_APPLICATION: diff --git a/services/core/java/com/android/server/wm/AccessibilityController.java b/services/core/java/com/android/server/wm/AccessibilityController.java index 0396a1101c644..14fb94d41b3b9 100644 --- a/services/core/java/com/android/server/wm/AccessibilityController.java +++ b/services/core/java/com/android/server/wm/AccessibilityController.java @@ -45,7 +45,6 @@ import static com.android.server.accessibility.AccessibilityTraceProto.WINDOW_MA import static com.android.server.wm.WindowManagerDebugConfig.TAG_WITH_CLASS_NAME; import static com.android.server.wm.WindowManagerDebugConfig.TAG_WM; import static com.android.server.wm.WindowTracing.WINSCOPE_EXT; -import static com.android.server.wm.utils.RegionUtils.forEachRect; import android.accessibilityservice.AccessibilityTrace; import android.animation.ObjectAnimator; @@ -101,6 +100,7 @@ import com.android.internal.util.TraceBuffer; import com.android.internal.util.function.pooled.PooledLambda; import com.android.server.LocalServices; import com.android.server.policy.WindowManagerPolicy; +import com.android.server.wm.AccessibilityWindowsPopulator.AccessibilityWindow; import com.android.server.wm.WindowManagerInternal.AccessibilityControllerInternal; import com.android.server.wm.WindowManagerInternal.MagnificationCallbacks; import com.android.server.wm.WindowManagerInternal.WindowsForAccessibilityCallback; @@ -133,19 +133,22 @@ final class AccessibilityController { private static final Rect EMPTY_RECT = new Rect(); private static final float[] sTempFloats = new float[9]; - private SparseArray mDisplayMagnifiers = new SparseArray<>(); - private SparseArray mWindowsForAccessibilityObserver = + private final SparseArray mDisplayMagnifiers = new SparseArray<>(); + private final SparseArray mWindowsForAccessibilityObserver = new SparseArray<>(); private SparseArray mFocusedWindow = new SparseArray<>(); private int mFocusedDisplay = -1; private boolean mIsImeVisible = false; // Set to true if initializing window population complete. private boolean mAllObserversInitialized = true; + private final AccessibilityWindowsPopulator mAccessibilityWindowsPopulator; AccessibilityController(WindowManagerService service) { mService = service; mAccessibilityTracing = AccessibilityController.getAccessibilityControllerInternal(service); + + mAccessibilityWindowsPopulator = new AccessibilityWindowsPopulator(mService, this); } boolean setMagnificationCallbacks(int displayId, MagnificationCallbacks callbacks) { @@ -209,7 +212,9 @@ final class AccessibilityController { } mWindowsForAccessibilityObserver.remove(displayId); } - observer = new WindowsForAccessibilityObserver(mService, displayId, callback); + mAccessibilityWindowsPopulator.setWindowsNotification(true); + observer = new WindowsForAccessibilityObserver(mService, displayId, callback, + mAccessibilityWindowsPopulator); mWindowsForAccessibilityObserver.put(displayId, observer); mAllObserversInitialized &= observer.mInitialized; } else { @@ -224,6 +229,10 @@ final class AccessibilityController { } } mWindowsForAccessibilityObserver.remove(displayId); + + if (mWindowsForAccessibilityObserver.size() <= 0) { + mAccessibilityWindowsPopulator.setWindowsNotification(false); + } } } @@ -455,6 +464,19 @@ final class AccessibilityController { return null; } + boolean getMagnificationSpecForDisplay(int displayId, MagnificationSpec outSpec) { + if (mAccessibilityTracing.isTracingEnabled(FLAGS_MAGNIFICATION_CALLBACK)) { + mAccessibilityTracing.logTrace(TAG + ".getMagnificationSpecForDisplay", + FLAGS_MAGNIFICATION_CALLBACK, "displayId=" + displayId); + } + final DisplayMagnifier displayMagnifier = mDisplayMagnifiers.get(displayId); + if (displayMagnifier == null) { + return false; + } + + return displayMagnifier.getMagnificationSpec(outSpec); + } + boolean hasCallbacks() { if (mAccessibilityTracing.isTracingEnabled(FLAGS_MAGNIFICATION_CALLBACK | FLAGS_WINDOWS_FOR_ACCESSIBILITY_CALLBACK)) { @@ -756,6 +778,25 @@ final class AccessibilityController { return spec; } + boolean getMagnificationSpec(MagnificationSpec outSpec) { + if (mAccessibilityTracing.isTracingEnabled(FLAGS_MAGNIFICATION_CALLBACK)) { + mAccessibilityTracing.logTrace(LOG_TAG + ".getMagnificationSpec", + FLAGS_MAGNIFICATION_CALLBACK); + } + MagnificationSpec spec = mMagnifedViewport.getMagnificationSpec(); + if (spec == null) { + return false; + } + + outSpec.setTo(spec); + if (mAccessibilityTracing.isTracingEnabled(FLAGS_MAGNIFICATION_CALLBACK)) { + mAccessibilityTracing.logTrace(LOG_TAG + ".getMagnificationSpec", + FLAGS_MAGNIFICATION_CALLBACK, "outSpec={" + outSpec + "}"); + } + + return true; + } + void getMagnificationRegion(Region outMagnificationRegion) { if (mAccessibilityTracing.isTracingEnabled(FLAGS_MAGNIFICATION_CALLBACK)) { mAccessibilityTracing.logTrace(LOG_TAG + ".getMagnificationRegion", @@ -1403,20 +1444,18 @@ final class AccessibilityController { private static final boolean DEBUG = false; - private final SparseArray mTempWindowStates = new SparseArray<>(); + private final List mTempA11yWindows = new ArrayList<>(); private final Set mTempBinderSet = new ArraySet<>(); - private final RectF mTempRectF = new RectF(); - - private final Matrix mTempMatrix = new Matrix(); - private final Point mTempPoint = new Point(); private final Region mTempRegion = new Region(); private final Region mTempRegion1 = new Region(); + private final Region mTempRegion2 = new Region(); + private final WindowManagerService mService; private final Handler mHandler; @@ -1431,10 +1470,11 @@ final class AccessibilityController { // Set to true if initializing window population complete. private boolean mInitialized; + private final AccessibilityWindowsPopulator mA11yWindowsPopulator; WindowsForAccessibilityObserver(WindowManagerService windowManagerService, - int displayId, - WindowsForAccessibilityCallback callback) { + int displayId, WindowsForAccessibilityCallback callback, + AccessibilityWindowsPopulator accessibilityWindowsPopulator) { mService = windowManagerService; mCallback = callback; mDisplayId = displayId; @@ -1443,6 +1483,7 @@ final class AccessibilityController { AccessibilityController.getAccessibilityControllerInternal(mService); mRecurringAccessibilityEventsIntervalMillis = ViewConfiguration .getSendRecurringAccessibilityEventsInterval(); + mA11yWindowsPopulator = accessibilityWindowsPopulator; computeChangedWindows(true); } @@ -1466,52 +1507,6 @@ final class AccessibilityController { } } - boolean shellRootIsAbove(WindowState windowState, ShellRoot shellRoot) { - int wsLayer = mService.mPolicy.getWindowLayerLw(windowState); - int shellLayer = mService.mPolicy.getWindowLayerFromTypeLw(shellRoot.getWindowType(), - true); - return shellLayer >= wsLayer; - } - - int addShellRootsIfAbove(WindowState windowState, ArrayList shellRoots, - int shellRootIndex, List windows, Set addedWindows, - Region unaccountedSpace, boolean focusedWindowAdded) { - while (shellRootIndex < shellRoots.size() - && shellRootIsAbove(windowState, shellRoots.get(shellRootIndex))) { - ShellRoot shellRoot = shellRoots.get(shellRootIndex); - shellRootIndex++; - final WindowInfo info = shellRoot.getWindowInfo(); - if (info == null) { - continue; - } - - info.layer = addedWindows.size(); - windows.add(info); - addedWindows.add(info.token); - unaccountedSpace.op(info.regionInScreen, unaccountedSpace, - Region.Op.REVERSE_DIFFERENCE); - if (unaccountedSpace.isEmpty() && focusedWindowAdded) { - break; - } - } - return shellRootIndex; - } - - private ArrayList getSortedShellRoots( - SparseArray originalShellRoots) { - ArrayList sortedShellRoots = new ArrayList<>(originalShellRoots.size()); - for (int i = originalShellRoots.size() - 1; i >= 0; --i) { - sortedShellRoots.add(originalShellRoots.valueAt(i)); - } - - sortedShellRoots.sort((left, right) -> - mService.mPolicy.getWindowLayerFromTypeLw(right.getWindowType(), true) - - mService.mPolicy.getWindowLayerFromTypeLw(left.getWindowType(), - true)); - - return sortedShellRoots; - } - /** * Check if windows have changed, and send them to the accessibility subsystem if they have. * @@ -1561,44 +1556,29 @@ final class AccessibilityController { Region unaccountedSpace = mTempRegion; unaccountedSpace.set(0, 0, screenWidth, screenHeight); - final SparseArray visibleWindows = mTempWindowStates; - populateVisibleWindowsOnScreen(visibleWindows); + final List visibleWindows = mTempA11yWindows; + mA11yWindowsPopulator.populateVisibleWindowsOnScreenLocked( + mDisplayId, visibleWindows); Set addedWindows = mTempBinderSet; addedWindows.clear(); boolean focusedWindowAdded = false; final int visibleWindowCount = visibleWindows.size(); - ArrayList skipRemainingWindowsForTaskFragments = new ArrayList<>(); - - ArrayList shellRoots = getSortedShellRoots(dc.mShellRoots); // Iterate until we figure out what is touchable for the entire screen. - int shellRootIndex = 0; - for (int i = visibleWindowCount - 1; i >= 0; i--) { - final WindowState windowState = visibleWindows.valueAt(i); - int prevShellRootIndex = shellRootIndex; - shellRootIndex = addShellRootsIfAbove(windowState, shellRoots, shellRootIndex, - windows, addedWindows, unaccountedSpace, focusedWindowAdded); - - // If a Shell Root was added, it could have accounted for all the space already. - if (shellRootIndex > prevShellRootIndex && unaccountedSpace.isEmpty() - && focusedWindowAdded) { - break; - } - - final Region regionInScreen = new Region(); - computeWindowRegionInScreen(windowState, regionInScreen); - if (windowMattersToAccessibility(windowState, - regionInScreen, unaccountedSpace, - skipRemainingWindowsForTaskFragments)) { - addPopulatedWindowInfo(windowState, regionInScreen, windows, addedWindows); - if (windowMattersToUnaccountedSpaceComputation(windowState)) { - updateUnaccountedSpace(windowState, regionInScreen, unaccountedSpace, - skipRemainingWindowsForTaskFragments); + for (int i = 0; i < visibleWindowCount; i++) { + final AccessibilityWindow a11yWindow = visibleWindows.get(i); + final Region regionInWindow = new Region(); + a11yWindow.getTouchableRegionInWindow(regionInWindow); + if (windowMattersToAccessibility(a11yWindow, regionInWindow, + unaccountedSpace)) { + addPopulatedWindowInfo(a11yWindow, regionInWindow, windows, addedWindows); + if (windowMattersToUnaccountedSpaceComputation(a11yWindow)) { + updateUnaccountedSpace(a11yWindow, unaccountedSpace); } - focusedWindowAdded |= windowState.isFocused(); - } else if (isUntouchableNavigationBar(windowState, mTempRegion1)) { + focusedWindowAdded |= a11yWindow.isFocused(); + } else if (a11yWindow.isUntouchableNavigationBar()) { // If this widow is navigation bar without touchable region, accounting the // region of navigation bar inset because all touch events from this region // would be received by launcher, i.e. this region is a un-touchable one @@ -1647,47 +1627,38 @@ final class AccessibilityController { // Some windows should be excluded from unaccounted space computation, though they still // should be reported - private boolean windowMattersToUnaccountedSpaceComputation(WindowState windowState) { + private boolean windowMattersToUnaccountedSpaceComputation(AccessibilityWindow a11yWindow) { // Do not account space of trusted non-touchable windows, except the split-screen // divider. // If it's not trusted, touch events are not sent to the windows behind it. - if (((windowState.mAttrs.flags & WindowManager.LayoutParams.FLAG_NOT_TOUCHABLE) != 0) - && (windowState.mAttrs.type != TYPE_DOCK_DIVIDER) - && windowState.isTrustedOverlay()) { + if (((a11yWindow.getFlags() & WindowManager.LayoutParams.FLAG_NOT_TOUCHABLE) != 0) + && (a11yWindow.getType() != TYPE_DOCK_DIVIDER) + && a11yWindow.isTrustedOverlay()) { return false; } - if (windowState.mAttrs.type - == WindowManager.LayoutParams.TYPE_ACCESSIBILITY_OVERLAY) { + if (a11yWindow.getType() == WindowManager.LayoutParams.TYPE_ACCESSIBILITY_OVERLAY) { return false; } return true; } - private boolean windowMattersToAccessibility(WindowState windowState, - Region regionInScreen, Region unaccountedSpace, - ArrayList skipRemainingWindowsForTaskFragments) { - final RecentsAnimationController controller = mService.getRecentsAnimationController(); - if (controller != null && controller.shouldIgnoreForAccessibility(windowState)) { + private boolean windowMattersToAccessibility(AccessibilityWindow a11yWindow, + Region regionInScreen, Region unaccountedSpace) { + if (a11yWindow.ignoreRecentsAnimationForAccessibility()) { return false; } - if (windowState.isFocused()) { + if (a11yWindow.isFocused()) { return true; } - // If the window is part of a task that we're finished with - ignore. - final TaskFragment taskFragment = windowState.getTaskFragment(); - if (taskFragment != null - && skipRemainingWindowsForTaskFragments.contains(taskFragment)) { - return false; - } - // Ignore non-touchable windows, except the split-screen divider, which is // occasionally non-touchable but still useful for identifying split-screen // mode. - if (((windowState.mAttrs.flags & WindowManager.LayoutParams.FLAG_NOT_TOUCHABLE) != 0) - && (windowState.mAttrs.type != TYPE_DOCK_DIVIDER)) { + if (((a11yWindow.getFlags() + & WindowManager.LayoutParams.FLAG_NOT_TOUCHABLE) != 0) + && (a11yWindow.getType() != TYPE_DOCK_DIVIDER)) { return false; } @@ -1697,88 +1668,36 @@ final class AccessibilityController { } // Add windows of certain types not covered by modal windows. - if (isReportedWindowType(windowState.mAttrs.type)) { + if (isReportedWindowType(a11yWindow.getType())) { return true; } return false; } - private void updateUnaccountedSpace(WindowState windowState, Region regionInScreen, - Region unaccountedSpace, - ArrayList skipRemainingWindowsForTaskFragments) { - // Account for the space this window takes if the window - // is not an accessibility overlay which does not change - // the reported windows. - unaccountedSpace.op(regionInScreen, unaccountedSpace, - Region.Op.REVERSE_DIFFERENCE); - - // If a window is modal it prevents other windows from being touched - if ((windowState.mAttrs.flags & (WindowManager.LayoutParams.FLAG_NOT_FOCUSABLE - | WindowManager.LayoutParams.FLAG_NOT_TOUCH_MODAL)) == 0) { - if (!windowState.hasTapExcludeRegion()) { - // Account for all space in the task, whether the windows in it are - // touchable or not. The modal window blocks all touches from the task's - // area. - unaccountedSpace.op(windowState.getDisplayFrame(), unaccountedSpace, - Region.Op.REVERSE_DIFFERENCE); - } else { - // If a window has tap exclude region, we need to account it. - final Region displayRegion = new Region(windowState.getDisplayFrame()); - final Region tapExcludeRegion = new Region(); - windowState.getTapExcludeRegion(tapExcludeRegion); - displayRegion.op(tapExcludeRegion, displayRegion, - Region.Op.REVERSE_DIFFERENCE); - unaccountedSpace.op(displayRegion, unaccountedSpace, - Region.Op.REVERSE_DIFFERENCE); - } - - final TaskFragment taskFragment = windowState.getTaskFragment(); - if (taskFragment != null) { - // If the window is associated with a particular task, we can skip the - // rest of the windows for that task. - skipRemainingWindowsForTaskFragments.add(taskFragment); - } else if (!windowState.hasTapExcludeRegion()) { - // If the window is not associated with a particular task, then it is - // globally modal. In this case we can skip all remaining windows when - // it doesn't has tap exclude region. - unaccountedSpace.setEmpty(); - } - } - - // Account for the space of letterbox. - if (windowState.areAppWindowBoundsLetterboxed()) { - unaccountedSpace.op(getLetterboxBounds(windowState), unaccountedSpace, + private void updateUnaccountedSpace(AccessibilityWindow a11yWindow, + Region unaccountedSpace) { + if (a11yWindow.getType() + != WindowManager.LayoutParams.TYPE_ACCESSIBILITY_OVERLAY) { + // Account for the space this window takes if the window + // is not an accessibility overlay which does not change + // the reported windows. + final Region touchableRegion = mTempRegion2; + a11yWindow.getTouchableRegionInScreen(touchableRegion); + unaccountedSpace.op(touchableRegion, unaccountedSpace, Region.Op.REVERSE_DIFFERENCE); + // Account for the space of letterbox. + final Region letterboxBounds = mTempRegion1; + if (a11yWindow.setLetterBoxBoundsIfNeeded(letterboxBounds)) { + unaccountedSpace.op(letterboxBounds, + unaccountedSpace, Region.Op.REVERSE_DIFFERENCE); + } } } - private void computeWindowRegionInScreen(WindowState windowState, Region outRegion) { - // Get the touchable frame. - Region touchableRegion = mTempRegion1; - windowState.getTouchableRegion(touchableRegion); - - // Map the frame to get what appears on the screen. - Matrix matrix = mTempMatrix; - populateTransformationMatrix(windowState, matrix); - - forEachRect(touchableRegion, rect -> { - // Move to origin as all transforms are captured by the matrix. - RectF windowFrame = mTempRectF; - windowFrame.set(rect); - windowFrame.offset(-windowState.getFrame().left, -windowState.getFrame().top); - - matrix.mapRect(windowFrame); - - // Union all rects. - outRegion.union(new Rect((int) windowFrame.left, (int) windowFrame.top, - (int) windowFrame.right, (int) windowFrame.bottom)); - }); - } - - private static void addPopulatedWindowInfo(WindowState windowState, Region regionInScreen, - List out, Set tokenOut) { - final WindowInfo window = windowState.getWindowInfo(); + private static void addPopulatedWindowInfo(AccessibilityWindow a11yWindow, + Region regionInScreen, List out, Set tokenOut) { + final WindowInfo window = a11yWindow.getWindowInfo(); window.regionInScreen.set(regionInScreen); window.layer = tokenOut.size(); out.add(window); @@ -1805,23 +1724,6 @@ final class AccessibilityController { && windowType != WindowManager.LayoutParams.TYPE_PRIVATE_PRESENTATION); } - private void populateVisibleWindowsOnScreen(SparseArray outWindows) { - final List tempWindowStatesList = new ArrayList<>(); - final DisplayContent dc = mService.mRoot.getDisplayContent(mDisplayId); - if (dc == null) { - return; - } - - dc.forAllWindows(w -> { - if (w.isVisible()) { - tempWindowStatesList.add(w); - } - }, false /* traverseTopToBottom */); - for (int i = 0; i < tempWindowStatesList.size(); i++) { - outWindows.put(i, tempWindowStatesList.get(i)); - } - } - private WindowState getTopFocusWindow() { return mService.mRoot.getTopFocusedDisplayContent().mCurrentFocus; } diff --git a/services/core/java/com/android/server/wm/AccessibilityWindowsPopulator.java b/services/core/java/com/android/server/wm/AccessibilityWindowsPopulator.java new file mode 100644 index 0000000000000..0260f8584fa0e --- /dev/null +++ b/services/core/java/com/android/server/wm/AccessibilityWindowsPopulator.java @@ -0,0 +1,534 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.wm; + +import static android.view.WindowManager.LayoutParams.TYPE_DOCK_DIVIDER; + +import static com.android.server.wm.utils.RegionUtils.forEachRect; + +import android.annotation.NonNull; +import android.graphics.Matrix; +import android.graphics.Rect; +import android.graphics.RectF; +import android.graphics.Region; +import android.os.Handler; +import android.os.Looper; +import android.os.Message; +import android.util.Slog; +import android.util.SparseArray; +import android.view.IWindow; +import android.view.InputWindowHandle; +import android.view.MagnificationSpec; +import android.view.WindowInfo; +import android.view.WindowManager; +import android.window.WindowInfosListener; + +import com.android.internal.annotations.GuardedBy; + +import java.util.ArrayList; +import java.util.List; + +/** + * This class is the accessibility windows population adapter. + */ +public final class AccessibilityWindowsPopulator extends WindowInfosListener { + + private static final String TAG = AccessibilityWindowsPopulator.class.getSimpleName(); + private static final float[] sTempFloats = new float[9]; + + private final WindowManagerService mService; + private final AccessibilityController mAccessibilityController; + @GuardedBy("mLock") + private final SparseArray> mInputWindowHandlesOnDisplays = + new SparseArray<>(); + @GuardedBy("mLock") + private final SparseArray mMagnificationSpecInverseMatrix = new SparseArray<>(); + @GuardedBy("mLock") + private final List mVisibleWindows = new ArrayList<>(); + @GuardedBy("mLock") + private boolean mWindowsNotificationEnabled = false; + private final Object mLock = new Object(); + private final Handler mHandler; + + AccessibilityWindowsPopulator(WindowManagerService service, + AccessibilityController accessibilityController) { + mService = service; + mAccessibilityController = accessibilityController; + mHandler = new MyHandler(mService.mH.getLooper()); + + register(); + } + + /** + * Gets the visible windows list with the window layer on the specified display. + * + * @param displayId The display. + * @param outWindows The visible windows list. The z-order of each window in the list + * is from the top to bottom. + */ + public void populateVisibleWindowsOnScreenLocked(int displayId, + List outWindows) { + List inputWindowHandles; + final Matrix inverseMatrix = new Matrix(); + + synchronized (mLock) { + inputWindowHandles = mInputWindowHandlesOnDisplays.get(displayId); + if (inputWindowHandles == null) { + outWindows.clear(); + + return; + } + + inverseMatrix.set(mMagnificationSpecInverseMatrix.get(displayId)); + } + + for (final InputWindowHandle windowHandle : inputWindowHandles) { + final AccessibilityWindow accessibilityWindow = + AccessibilityWindow.initializeData(mService, windowHandle, inverseMatrix); + + outWindows.add(accessibilityWindow); + } + } + + @Override + public void onWindowInfosChanged(InputWindowHandle[] windowHandles) { + synchronized (mLock) { + mVisibleWindows.clear(); + for (InputWindowHandle window : windowHandles) { + if (window.visible && window.getWindow() != null) { + mVisibleWindows.add(window); + } + } + if (mWindowsNotificationEnabled) { + populateVisibleWindowHandlesAndNotifyWindowsChangeIfNeededLocked(); + } + } + } + + /** + * Sets to notify the accessibilityController to compute changed windows on + * the display after populating the visible windows if the windows reported + * from the surface flinger changes. + * + * @param register {@code true} means starting windows population. + */ + public void setWindowsNotification(boolean register) { + synchronized (mLock) { + if (mWindowsNotificationEnabled == register) { + return; + } + mWindowsNotificationEnabled = register; + if (mWindowsNotificationEnabled) { + populateVisibleWindowHandlesAndNotifyWindowsChangeIfNeededLocked(); + } else { + releaseResources(); + } + } + } + + private void populateVisibleWindowHandlesAndNotifyWindowsChangeIfNeededLocked() { + final SparseArray> tempWindowHandleList = new SparseArray<>(); + + for (final InputWindowHandle windowHandle : mVisibleWindows) { + List inputWindowHandles = tempWindowHandleList.get( + windowHandle.displayId); + + if (inputWindowHandles == null) { + inputWindowHandles = new ArrayList<>(); + tempWindowHandleList.put(windowHandle.displayId, inputWindowHandles); + generateMagnificationSpecInverseMatrixLocked(windowHandle.displayId); + } + inputWindowHandles.add(windowHandle); + } + + final List displayIdsForWindowsChanged = new ArrayList<>(); + + getDisplaysForWindowsChangedLocked(displayIdsForWindowsChanged, tempWindowHandleList, + mInputWindowHandlesOnDisplays); + // Clones all windows from the callback of the surface flinger. + mInputWindowHandlesOnDisplays.clear(); + for (int i = 0; i < tempWindowHandleList.size(); i++) { + final int displayId = tempWindowHandleList.keyAt(i); + mInputWindowHandlesOnDisplays.put(displayId, tempWindowHandleList.get(displayId)); + } + if (displayIdsForWindowsChanged.size() > 0 + && !mHandler.hasMessages(MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED)) { + mHandler.obtainMessage(MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED, + displayIdsForWindowsChanged).sendToTarget(); + } + } + + private void getDisplaysForWindowsChangedLocked(List outDisplayIdsForWindowsChanged, + SparseArray> newWindowsList, + SparseArray> oldWindowsList) { + for (int i = 0; i < newWindowsList.size(); i++) { + final int displayId = newWindowsList.keyAt(i); + final List newWindows = newWindowsList.get(displayId); + final List oldWindows = oldWindowsList.get(displayId); + + if (hasWindowsChangedLocked(newWindows, oldWindows)) { + outDisplayIdsForWindowsChanged.add(displayId); + } + } + } + + private boolean hasWindowsChangedLocked(List newWindows, + List oldWindows) { + if (oldWindows == null || oldWindows.size() != newWindows.size()) { + return true; + } + + final int windowsCount = newWindows.size(); + // Since we always traverse windows from high to low layer, + // the old and new windows at the same index should be the + // same, otherwise something changed. + for (int i = 0; i < windowsCount; i++) { + final InputWindowHandle newWindow = newWindows.get(i); + final InputWindowHandle oldWindow = oldWindows.get(i); + + if (!newWindow.getWindow().asBinder().equals(oldWindow.getWindow().asBinder())) { + return true; + } + } + + return false; + } + + private void generateMagnificationSpecInverseMatrixLocked(int displayId) { + MagnificationSpec spec = new MagnificationSpec(); + if (!mAccessibilityController.getMagnificationSpecForDisplay(displayId, spec)) { + mMagnificationSpecInverseMatrix.remove(displayId); + return; + } + sTempFloats[Matrix.MSCALE_X] = spec.scale; + sTempFloats[Matrix.MSKEW_Y] = 0; + sTempFloats[Matrix.MSKEW_X] = 0; + sTempFloats[Matrix.MSCALE_Y] = spec.scale; + sTempFloats[Matrix.MTRANS_X] = spec.offsetX; + sTempFloats[Matrix.MTRANS_Y] = spec.offsetY; + sTempFloats[Matrix.MPERSP_0] = 0; + sTempFloats[Matrix.MPERSP_1] = 0; + sTempFloats[Matrix.MPERSP_2] = 1; + + final Matrix tempMatrix = new Matrix(); + tempMatrix.setValues(sTempFloats); + + final Matrix inverseMatrix = new Matrix(); + final boolean result = tempMatrix.invert(inverseMatrix); + + if (!result) { + Slog.e(TAG, "Can't inverse the magnification spec matrix with the " + + "magnification spec = " + spec + " on the displayId = " + displayId); + return; + } + mMagnificationSpecInverseMatrix.set(displayId, inverseMatrix); + } + + private void notifyWindowsChanged(@NonNull List displayIdsForWindowsChanged) { + for (int i = 0; i < displayIdsForWindowsChanged.size(); i++) { + mAccessibilityController.performComputeChangedWindowsNot( + displayIdsForWindowsChanged.get(i), false); + } + } + + @GuardedBy("mLock") + private void releaseResources() { + mInputWindowHandlesOnDisplays.clear(); + mMagnificationSpecInverseMatrix.clear(); + mVisibleWindows.clear(); + mWindowsNotificationEnabled = false; + } + + private class MyHandler extends Handler { + public static final int MESSAGE_NOTIFY_WINDOWS_CHANGED = 1; + + MyHandler(Looper looper) { + super(looper, null, false); + } + + @Override + public void handleMessage(Message message) { + if (message.what == MESSAGE_NOTIFY_WINDOWS_CHANGED) { + final List displayIdsForWindowsChanged = (List) message.obj; + notifyWindowsChanged(displayIdsForWindowsChanged); + } + } + } + + /** + * This class represents information about a window from the + * surface flinger to the accessibility framework. + */ + public static class AccessibilityWindow { + private static final Region TEMP_REGION = new Region(); + private static final RectF TEMP_RECTF = new RectF(); + // Data + private IWindow mWindow; + private int mDisplayId; + private int mFlags; + private int mType; + private int mPrivateFlags; + private boolean mIsFocused; + private boolean mShouldMagnify; + private boolean mIgnoreDuetoRecentsAnimation; + private boolean mIsTrustedOverlay; + private final Region mTouchableRegionInScreen = new Region(); + private final Region mTouchableRegionInWindow = new Region(); + private final Region mLetterBoxBounds = new Region(); + private WindowInfo mWindowInfo; + + /** + * Returns the instance after initializing the internal data. + * @param service The window manager service. + * @param inputWindowHandle The window from the surface flinger. + * @param inverseMatrix The magnification spec inverse matrix. + */ + public static AccessibilityWindow initializeData(WindowManagerService service, + InputWindowHandle inputWindowHandle, Matrix inverseMatrix) { + final IWindow window = inputWindowHandle.getWindow(); + final WindowState windowState = window != null ? service.mWindowMap.get( + window.asBinder()) : null; + + final AccessibilityWindow instance = new AccessibilityWindow(); + + instance.mWindow = inputWindowHandle.getWindow(); + instance.mDisplayId = inputWindowHandle.displayId; + instance.mFlags = inputWindowHandle.layoutParamsFlags; + instance.mType = inputWindowHandle.layoutParamsType; + + // TODO (b/199357848): gets the private flag of the window from other way. + instance.mPrivateFlags = windowState != null ? windowState.mAttrs.privateFlags : 0; + // TODO (b/199358208) : using new way to implement the focused window. + instance.mIsFocused = windowState != null && windowState.isFocused(); + instance.mShouldMagnify = windowState == null || windowState.shouldMagnify(); + + final RecentsAnimationController controller = service.getRecentsAnimationController(); + instance.mIgnoreDuetoRecentsAnimation = windowState != null && controller != null + && controller.shouldIgnoreForAccessibility(windowState); + instance.mIsTrustedOverlay = inputWindowHandle.trustedOverlay; + + // TODO (b/199358388) : gets the letterbox bounds of the window from other way. + if (windowState != null && windowState.areAppWindowBoundsLetterboxed()) { + getLetterBoxBounds(windowState, instance.mLetterBoxBounds); + } + + final Rect windowFrame = new Rect(inputWindowHandle.frameLeft, + inputWindowHandle.frameTop, inputWindowHandle.frameRight, + inputWindowHandle.frameBottom); + getTouchableRegionInWindow(instance.mShouldMagnify, inputWindowHandle.touchableRegion, + instance.mTouchableRegionInWindow, windowFrame, inverseMatrix); + getUnMagnifiedTouchableRegion(instance.mShouldMagnify, + inputWindowHandle.touchableRegion, instance.mTouchableRegionInScreen, + inverseMatrix); + instance.mWindowInfo = windowState != null + ? windowState.getWindowInfo() : getWindowInfoForWindowlessWindows(instance); + + return instance; + } + + /** + * Returns the touchable region in the screen. + * @param outRegion The touchable region. + */ + public void getTouchableRegionInScreen(Region outRegion) { + outRegion.set(mTouchableRegionInScreen); + } + + /** + * Returns the touchable region in the window. + * @param outRegion The touchable region. + */ + public void getTouchableRegionInWindow(Region outRegion) { + outRegion.set(mTouchableRegionInWindow); + } + + /** + * @return the layout parameter flag {@link android.view.WindowManager.LayoutParams#flags}. + */ + public int getFlags() { + return mFlags; + } + + /** + * @return the layout parameter type {@link android.view.WindowManager.LayoutParams#type}. + */ + public int getType() { + return mType; + } + + /** + * @return the layout parameter private flag + * {@link android.view.WindowManager.LayoutParams#privateFlags}. + */ + public int getPrivateFlag() { + return mPrivateFlags; + } + + /** + * @return the windowInfo {@link WindowInfo}. + */ + public WindowInfo getWindowInfo() { + return mWindowInfo; + } + + /** + * Gets the letter box bounds if activity bounds are letterboxed + * or letterboxed for display cutout. + * + * @return {@code true} there's a letter box bounds. + */ + public Boolean setLetterBoxBoundsIfNeeded(Region outBounds) { + if (mLetterBoxBounds.isEmpty()) { + return false; + } + + outBounds.set(mLetterBoxBounds); + return true; + } + + /** + * @return true if this window should be magnified. + */ + public boolean shouldMagnify() { + return mShouldMagnify; + } + + /** + * @return true if this window is focused. + */ + public boolean isFocused() { + return mIsFocused; + } + + /** + * @return true if it's running the recent animation but not the target app. + */ + public boolean ignoreRecentsAnimationForAccessibility() { + return mIgnoreDuetoRecentsAnimation; + } + + /** + * @return true if this window is the trusted overlay. + */ + public boolean isTrustedOverlay() { + return mIsTrustedOverlay; + } + + /** + * @return true if this window is the navigation bar with the gesture mode. + */ + public boolean isUntouchableNavigationBar() { + if (mType != WindowManager.LayoutParams.TYPE_NAVIGATION_BAR) { + return false; + } + + return mTouchableRegionInScreen.isEmpty(); + } + + private static void getTouchableRegionInWindow(boolean shouldMagnify, Region inRegion, + Region outRegion, Rect frame, Matrix inverseMatrix) { + // Some modal windows, like the activity with Theme.dialog, has the full screen + // as its touchable region, but its window frame is smaller than the touchable + // region. The region we report should be the touchable area in the window frame + // for the consistency and match developers expectation. + // So we need to make the intersection between the frame and touchable region to + // obtain the real touch region in the screen. + Region touchRegion = TEMP_REGION; + touchRegion.set(inRegion); + touchRegion.op(frame, Region.Op.INTERSECT); + + getUnMagnifiedTouchableRegion(shouldMagnify, touchRegion, outRegion, inverseMatrix); + } + + /** + * Gets the un-magnified touchable region. If this window can be magnified and magnifying, + * we will transform the input touchable region by applying the inverse matrix of the + * magnification spec to get the un-magnified touchable region. + * @param shouldMagnify The window can be magnified. + * @param inRegion The touchable region of this window. + * @param outRegion The un-magnified touchable region of this window. + * @param inverseMatrix The inverse matrix of the magnification spec. + */ + private static void getUnMagnifiedTouchableRegion(boolean shouldMagnify, Region inRegion, + Region outRegion, Matrix inverseMatrix) { + if (!shouldMagnify || inverseMatrix.isIdentity()) { + outRegion.set(inRegion); + return; + } + + forEachRect(inRegion, rect -> { + // Move to origin as all transforms are captured by the matrix. + RectF windowFrame = TEMP_RECTF; + windowFrame.set(rect); + + inverseMatrix.mapRect(windowFrame); + // Union all rects. + outRegion.union(new Rect((int) windowFrame.left, (int) windowFrame.top, + (int) windowFrame.right, (int) windowFrame.bottom)); + }); + } + + private static WindowInfo getWindowInfoForWindowlessWindows(AccessibilityWindow window) { + WindowInfo windowInfo = WindowInfo.obtain(); + windowInfo.displayId = window.mDisplayId; + windowInfo.type = window.mType; + windowInfo.token = window.mWindow.asBinder(); + windowInfo.hasFlagWatchOutsideTouch = (window.mFlags + & WindowManager.LayoutParams.FLAG_WATCH_OUTSIDE_TOUCH) != 0; + windowInfo.inPictureInPicture = false; + + // There only are two windowless windows now, one is split window, and the other + // one is PIP. + if (windowInfo.type == TYPE_DOCK_DIVIDER) { + windowInfo.title = "Splitscreen Divider"; + } else { + windowInfo.title = "Picture-in-Picture menu"; + } + + return windowInfo; + } + + private static void getLetterBoxBounds(WindowState windowState, Region outRegion) { + final Rect letterboxInsets = windowState.mActivityRecord.getLetterboxInsets(); + final Rect nonLetterboxRect = windowState.getBounds(); + + nonLetterboxRect.inset(letterboxInsets); + outRegion.set(windowState.getBounds()); + outRegion.op(nonLetterboxRect, Region.Op.DIFFERENCE); + } + + @Override + public String toString() { + String builder = "A11yWindow=[" + mWindow + + ", displayId=" + mDisplayId + + ", flag=0x" + Integer.toHexString(mFlags) + + ", type=" + mType + + ", privateFlag=0x" + Integer.toHexString(mPrivateFlags) + + ", focused=" + mIsFocused + + ", magnify=" + mShouldMagnify + + ", ignoreDuetoRecentsAnimation=" + mIgnoreDuetoRecentsAnimation + + ", mIsTrustedOverlay=" + mIsTrustedOverlay + + ", regionInScreen=" + mTouchableRegionInScreen + + ", touchableRegion=" + mTouchableRegionInWindow + + ", letterBoxBounds=" + mLetterBoxBounds + + ", windowInfo=" + mWindowInfo + + "]"; + + return builder; + } + } +} From 148938787075c6039054b30505673fd25d97b0ee Mon Sep 17 00:00:00 2001 From: Jacky Kao Date: Thu, 4 Nov 2021 13:53:10 +0800 Subject: [PATCH 2/4] Uses SF callback windows for the A11yWindowInfo [2/n] 1. The SF callbacks the windows even they are in the transition state. This will make many useless A11y events reporting to the A11y services, like windows change with bounds change event. To avoid this symptom, we assume the callback windows become stable and compute the reported windows to the A11y framework if the SF callback isn't sent within 2 frames time (about 35ms). 2. To avoid the endless SF callback always in a short time, then A11y framework can't obtain any windows, we set a maximum time, 500ms, which is longer than the whole animation, to compute the reported windows. 3. Removes the legacy signals from the WM to compute the reported windows due to they can be covered by the signals from the SF callback. Bug: 191736824 Test: a11y CTS & unit tests Test: Manual testing including the A11y services Change-Id: Ibe2096e388769a12e3bcbaa472239ec9e458a396 --- .../server/wm/AccessibilityController.java | 10 --- .../wm/AccessibilityWindowsPopulator.java | 67 +++++++++++++++++-- .../java/com/android/server/wm/ShellRoot.java | 4 -- .../com/android/server/wm/WindowState.java | 3 - 4 files changed, 60 insertions(+), 24 deletions(-) diff --git a/services/core/java/com/android/server/wm/AccessibilityController.java b/services/core/java/com/android/server/wm/AccessibilityController.java index 14fb94d41b3b9..03abe8931e28c 100644 --- a/services/core/java/com/android/server/wm/AccessibilityController.java +++ b/services/core/java/com/android/server/wm/AccessibilityController.java @@ -318,11 +318,6 @@ final class AccessibilityController { if (displayMagnifier != null) { displayMagnifier.onDisplaySizeChanged(displayContent); } - final WindowsForAccessibilityObserver windowsForA11yObserver = - mWindowsForAccessibilityObserver.get(displayId); - if (windowsForA11yObserver != null) { - windowsForA11yObserver.scheduleComputeChangedWindows(); - } } void onAppWindowTransition(int displayId, int transition) { @@ -350,11 +345,6 @@ final class AccessibilityController { if (displayMagnifier != null) { displayMagnifier.onWindowTransition(windowState, transition); } - final WindowsForAccessibilityObserver windowsForA11yObserver = - mWindowsForAccessibilityObserver.get(displayId); - if (windowsForA11yObserver != null) { - windowsForA11yObserver.scheduleComputeChangedWindows(); - } } void onWindowFocusChangedNot(int displayId) { diff --git a/services/core/java/com/android/server/wm/AccessibilityWindowsPopulator.java b/services/core/java/com/android/server/wm/AccessibilityWindowsPopulator.java index 0260f8584fa0e..cf1ff7713d707 100644 --- a/services/core/java/com/android/server/wm/AccessibilityWindowsPopulator.java +++ b/services/core/java/com/android/server/wm/AccessibilityWindowsPopulator.java @@ -48,6 +48,14 @@ import java.util.List; public final class AccessibilityWindowsPopulator extends WindowInfosListener { private static final String TAG = AccessibilityWindowsPopulator.class.getSimpleName(); + // If the surface flinger callback is not coming within in 2 frames time, i.e. about + // 35ms, then assuming the windows become stable. + private static final int SURFACE_FLINGER_CALLBACK_WINDOWS_STABLE_TIMES_MS = 35; + // To avoid the surface flinger callbacks always comes within in 2 frames, then no windows + // are reported to the A11y framework, and the animation duration time is 500ms, so setting + // this value as the max timeout value to force computing changed windows. + private static final int WINDOWS_CHANGED_NOTIFICATION_MAX_DURATION_TIMES_MS = 500; + private static final float[] sTempFloats = new float[9]; private final WindowManagerService mService; @@ -114,6 +122,12 @@ public final class AccessibilityWindowsPopulator extends WindowInfosListener { } } if (mWindowsNotificationEnabled) { + if (!mHandler.hasMessages( + MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_TIMEOUT)) { + mHandler.sendEmptyMessageDelayed( + MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_TIMEOUT, + WINDOWS_CHANGED_NOTIFICATION_MAX_DURATION_TIMES_MS); + } populateVisibleWindowHandlesAndNotifyWindowsChangeIfNeededLocked(); } } @@ -165,11 +179,18 @@ public final class AccessibilityWindowsPopulator extends WindowInfosListener { final int displayId = tempWindowHandleList.keyAt(i); mInputWindowHandlesOnDisplays.put(displayId, tempWindowHandleList.get(displayId)); } - if (displayIdsForWindowsChanged.size() > 0 - && !mHandler.hasMessages(MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED)) { - mHandler.obtainMessage(MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED, - displayIdsForWindowsChanged).sendToTarget(); + + if (displayIdsForWindowsChanged.size() > 0) { + if (!mHandler.hasMessages(MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED)) { + mHandler.obtainMessage(MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED, + displayIdsForWindowsChanged).sendToTarget(); + } + + return; } + mHandler.removeMessages(MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_UI_STABLE); + mHandler.sendEmptyMessageDelayed(MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_UI_STABLE, + SURFACE_FLINGER_CALLBACK_WINDOWS_STABLE_TIMES_MS); } private void getDisplaysForWindowsChangedLocked(List outDisplayIdsForWindowsChanged, @@ -239,22 +260,39 @@ public final class AccessibilityWindowsPopulator extends WindowInfosListener { } private void notifyWindowsChanged(@NonNull List displayIdsForWindowsChanged) { + mHandler.removeMessages(MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_TIMEOUT); + for (int i = 0; i < displayIdsForWindowsChanged.size(); i++) { mAccessibilityController.performComputeChangedWindowsNot( displayIdsForWindowsChanged.get(i), false); } } + private void forceUpdateWindows() { + final List displayIdsForWindowsChanged = new ArrayList<>(); + + synchronized (mLock) { + for (int i = 0; i < mInputWindowHandlesOnDisplays.size(); i++) { + final int displayId = mInputWindowHandlesOnDisplays.keyAt(i); + displayIdsForWindowsChanged.add(displayId); + } + } + notifyWindowsChanged(displayIdsForWindowsChanged); + } + @GuardedBy("mLock") private void releaseResources() { mInputWindowHandlesOnDisplays.clear(); mMagnificationSpecInverseMatrix.clear(); mVisibleWindows.clear(); mWindowsNotificationEnabled = false; + mHandler.removeCallbacksAndMessages(null); } private class MyHandler extends Handler { public static final int MESSAGE_NOTIFY_WINDOWS_CHANGED = 1; + public static final int MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_UI_STABLE = 2; + public static final int MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_TIMEOUT = 3; MyHandler(Looper looper) { super(looper, null, false); @@ -262,9 +300,24 @@ public final class AccessibilityWindowsPopulator extends WindowInfosListener { @Override public void handleMessage(Message message) { - if (message.what == MESSAGE_NOTIFY_WINDOWS_CHANGED) { - final List displayIdsForWindowsChanged = (List) message.obj; - notifyWindowsChanged(displayIdsForWindowsChanged); + switch (message.what) { + case MESSAGE_NOTIFY_WINDOWS_CHANGED: { + final List displayIdsForWindowsChanged = (List) message.obj; + notifyWindowsChanged(displayIdsForWindowsChanged); + } break; + + case MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_UI_STABLE: { + forceUpdateWindows(); + } break; + + case MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_TIMEOUT: { + Slog.w(TAG, "Windows change within in 2 frames continuously over 500 ms " + + "and notify windows changed immediately"); + mHandler.removeMessages( + MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_UI_STABLE); + + forceUpdateWindows(); + } break; } } } diff --git a/services/core/java/com/android/server/wm/ShellRoot.java b/services/core/java/com/android/server/wm/ShellRoot.java index 2eab3bab0bd37..8ec9ef23a3f6b 100644 --- a/services/core/java/com/android/server/wm/ShellRoot.java +++ b/services/core/java/com/android/server/wm/ShellRoot.java @@ -196,9 +196,5 @@ public class ShellRoot { mAccessibilityWindow = null; } } - if (mDisplayContent.mWmService.mAccessibilityController.hasCallbacks()) { - mDisplayContent.mWmService.mAccessibilityController.onSomeWindowResizedOrMoved( - mDisplayContent.getDisplayId()); - } } } diff --git a/services/core/java/com/android/server/wm/WindowState.java b/services/core/java/com/android/server/wm/WindowState.java index 87ef09ee5982a..26acf43cf9edf 100644 --- a/services/core/java/com/android/server/wm/WindowState.java +++ b/services/core/java/com/android/server/wm/WindowState.java @@ -4992,9 +4992,6 @@ class WindowState extends WindowContainer implements WindowManagerP if (isAnimating()) { return; } - if (mWmService.mAccessibilityController.hasCallbacks()) { - mWmService.mAccessibilityController.onSomeWindowResizedOrMoved(getDisplayId()); - } if (!isSelfOrAncestorWindowAnimatingExit()) { return; From 35433f34399ec97cbd955d7f3fea362828461f3e Mon Sep 17 00:00:00 2001 From: ryanlwlin Date: Mon, 22 Nov 2021 11:36:07 +0800 Subject: [PATCH 3/4] Use shellRoot to identify PIP menu window Pip menu is an untouchable window which is exculded from accessibility windows list. However it is touchable when users touch in the pip mode window. To fix it, we add this window to visible windows list by comparing the window token. Bug: 191736824 Test: a11y CTS & unit tests Change-Id: I3d30d93bb92db467f37b7e659e2baea1a69cab16 --- .../server/wm/AccessibilityController.java | 5 +- .../wm/AccessibilityWindowsPopulator.java | 27 ++++++++--- .../java/com/android/server/wm/ShellRoot.java | 47 +++---------------- 3 files changed, 30 insertions(+), 49 deletions(-) diff --git a/services/core/java/com/android/server/wm/AccessibilityController.java b/services/core/java/com/android/server/wm/AccessibilityController.java index 03abe8931e28c..8f703c5c77612 100644 --- a/services/core/java/com/android/server/wm/AccessibilityController.java +++ b/services/core/java/com/android/server/wm/AccessibilityController.java @@ -1645,10 +1645,11 @@ final class AccessibilityController { // Ignore non-touchable windows, except the split-screen divider, which is // occasionally non-touchable but still useful for identifying split-screen - // mode. + // mode and the PIP menu. if (((a11yWindow.getFlags() & WindowManager.LayoutParams.FLAG_NOT_TOUCHABLE) != 0) - && (a11yWindow.getType() != TYPE_DOCK_DIVIDER)) { + && (a11yWindow.getType() != TYPE_DOCK_DIVIDER + && !a11yWindow.isPIPMenu())) { return false; } diff --git a/services/core/java/com/android/server/wm/AccessibilityWindowsPopulator.java b/services/core/java/com/android/server/wm/AccessibilityWindowsPopulator.java index cf1ff7713d707..246f9acf55bb8 100644 --- a/services/core/java/com/android/server/wm/AccessibilityWindowsPopulator.java +++ b/services/core/java/com/android/server/wm/AccessibilityWindowsPopulator.java @@ -26,6 +26,7 @@ import android.graphics.Rect; import android.graphics.RectF; import android.graphics.Region; import android.os.Handler; +import android.os.IBinder; import android.os.Looper; import android.os.Message; import android.util.Slog; @@ -104,9 +105,14 @@ public final class AccessibilityWindowsPopulator extends WindowInfosListener { inverseMatrix.set(mMagnificationSpecInverseMatrix.get(displayId)); } + final DisplayContent dc = mService.mRoot.getDisplayContent(displayId); + final ShellRoot shellroot = dc.mShellRoots.get(WindowManager.SHELL_ROOT_LAYER_PIP); + final IBinder pipMenuIBinder = + shellroot != null ? shellroot.getAccessibilityWindowToken() : null; for (final InputWindowHandle windowHandle : inputWindowHandles) { final AccessibilityWindow accessibilityWindow = - AccessibilityWindow.initializeData(mService, windowHandle, inverseMatrix); + AccessibilityWindow.initializeData(mService, windowHandle, inverseMatrix, + pipMenuIBinder); outWindows.add(accessibilityWindow); } @@ -335,6 +341,7 @@ public final class AccessibilityWindowsPopulator extends WindowInfosListener { private int mFlags; private int mType; private int mPrivateFlags; + private boolean mIsPIPMenu; private boolean mIsFocused; private boolean mShouldMagnify; private boolean mIgnoreDuetoRecentsAnimation; @@ -351,7 +358,7 @@ public final class AccessibilityWindowsPopulator extends WindowInfosListener { * @param inverseMatrix The magnification spec inverse matrix. */ public static AccessibilityWindow initializeData(WindowManagerService service, - InputWindowHandle inputWindowHandle, Matrix inverseMatrix) { + InputWindowHandle inputWindowHandle, Matrix inverseMatrix, IBinder pipIBinder) { final IWindow window = inputWindowHandle.getWindow(); final WindowState windowState = window != null ? service.mWindowMap.get( window.asBinder()) : null; @@ -362,6 +369,7 @@ public final class AccessibilityWindowsPopulator extends WindowInfosListener { instance.mDisplayId = inputWindowHandle.displayId; instance.mFlags = inputWindowHandle.layoutParamsFlags; instance.mType = inputWindowHandle.layoutParamsType; + instance.mIsPIPMenu = inputWindowHandle.getWindow().asBinder().equals(pipIBinder); // TODO (b/199357848): gets the private flag of the window from other way. instance.mPrivateFlags = windowState != null ? windowState.mAttrs.privateFlags : 0; @@ -492,6 +500,13 @@ public final class AccessibilityWindowsPopulator extends WindowInfosListener { return mTouchableRegionInScreen.isEmpty(); } + /** + * @return true if this window is PIP menu. + */ + public boolean isPIPMenu() { + return mIsPIPMenu; + } + private static void getTouchableRegionInWindow(boolean shouldMagnify, Region inRegion, Region outRegion, Rect frame, Matrix inverseMatrix) { // Some modal windows, like the activity with Theme.dialog, has the full screen @@ -548,10 +563,9 @@ public final class AccessibilityWindowsPopulator extends WindowInfosListener { // one is PIP. if (windowInfo.type == TYPE_DOCK_DIVIDER) { windowInfo.title = "Splitscreen Divider"; - } else { + } else if (window.mIsPIPMenu) { windowInfo.title = "Picture-in-Picture menu"; } - return windowInfo; } @@ -566,18 +580,19 @@ public final class AccessibilityWindowsPopulator extends WindowInfosListener { @Override public String toString() { - String builder = "A11yWindow=[" + mWindow + String builder = "A11yWindow=[" + mWindow.asBinder() + ", displayId=" + mDisplayId + ", flag=0x" + Integer.toHexString(mFlags) + ", type=" + mType + ", privateFlag=0x" + Integer.toHexString(mPrivateFlags) + ", focused=" + mIsFocused - + ", magnify=" + mShouldMagnify + + ", mShouldMagnify=" + mShouldMagnify + ", ignoreDuetoRecentsAnimation=" + mIgnoreDuetoRecentsAnimation + ", mIsTrustedOverlay=" + mIsTrustedOverlay + ", regionInScreen=" + mTouchableRegionInScreen + ", touchableRegion=" + mTouchableRegionInWindow + ", letterBoxBounds=" + mLetterBoxBounds + + ", isPIPMenu=" + mIsPIPMenu + ", windowInfo=" + mWindowInfo + "]"; diff --git a/services/core/java/com/android/server/wm/ShellRoot.java b/services/core/java/com/android/server/wm/ShellRoot.java index 8ec9ef23a3f6b..f9d7b53e4e780 100644 --- a/services/core/java/com/android/server/wm/ShellRoot.java +++ b/services/core/java/com/android/server/wm/ShellRoot.java @@ -25,15 +25,14 @@ import static com.android.server.wm.SurfaceAnimator.ANIMATION_TYPE_WINDOW_ANIMAT import static com.android.server.wm.WindowManagerService.MAX_ANIMATION_DURATION; import android.annotation.NonNull; +import android.annotation.Nullable; import android.graphics.Point; -import android.graphics.Rect; import android.os.IBinder; import android.os.RemoteException; import android.util.Slog; import android.view.DisplayInfo; import android.view.IWindow; import android.view.SurfaceControl; -import android.view.WindowInfo; import android.view.WindowManager; import android.view.animation.Animation; @@ -136,46 +135,12 @@ public class ShellRoot { ANIMATION_TYPE_WINDOW_ANIMATION); } - WindowInfo getWindowInfo() { - if (mShellRootLayer != SHELL_ROOT_LAYER_DIVIDER - && mShellRootLayer != SHELL_ROOT_LAYER_PIP) { - return null; + @Nullable + IBinder getAccessibilityWindowToken() { + if (mAccessibilityWindow != null) { + return mAccessibilityWindow.asBinder(); } - if (mShellRootLayer == SHELL_ROOT_LAYER_DIVIDER) { - return null; - } - if (mShellRootLayer == SHELL_ROOT_LAYER_PIP - && mDisplayContent.getDefaultTaskDisplayArea().getRootPinnedTask() == null) { - return null; - } - if (mAccessibilityWindow == null) { - return null; - } - WindowInfo windowInfo = WindowInfo.obtain(); - windowInfo.displayId = mToken.getDisplayArea().getDisplayContent().mDisplayId; - windowInfo.type = mToken.windowType; - windowInfo.layer = mToken.getWindowLayerFromType(); - windowInfo.token = mAccessibilityWindow.asBinder(); - windowInfo.focused = false; - windowInfo.hasFlagWatchOutsideTouch = false; - final Rect regionRect = new Rect(); - - - // DividerView - if (mShellRootLayer == SHELL_ROOT_LAYER_DIVIDER) { - windowInfo.inPictureInPicture = false; - mDisplayContent.getDockedDividerController().getTouchRegion(regionRect); - windowInfo.regionInScreen.set(regionRect); - windowInfo.title = "Splitscreen Divider"; - } - // PipMenuView - if (mShellRootLayer == SHELL_ROOT_LAYER_PIP) { - windowInfo.inPictureInPicture = true; - mDisplayContent.getDefaultTaskDisplayArea().getRootPinnedTask().getBounds(regionRect); - windowInfo.regionInScreen.set(regionRect); - windowInfo.title = "Picture-in-Picture menu"; - } - return windowInfo; + return null; } void setAccessibilityWindow(IWindow window) { From 914cef5f32e823d3480afa731b5260dcbe660da5 Mon Sep 17 00:00:00 2001 From: Jacky Kao Date: Tue, 30 Nov 2021 10:14:28 +0800 Subject: [PATCH 4/4] Apply display transform to window bounds and node bounds Currently, the node bounds and the window bounds are both in unrotated coordinates, it means if their layout is not changed, then the bound wouldn't be change after rotating the device. This effects Uiobject2, which is used for uiautomator, because it clips the nodes bounds based on current display metrics. see ag/13805539. To fix it, we transfer them back to the rotated coordinates based on the displayInfo called back from the surface flinger. (Co-Author: Ryan) Bug: 191736824 Test: a11y CTS & unit tests Test: atest NexusLauncherTests & Launcher3Tests Change-Id: Ia92e70d98f58cb68c8e464bc142d20640ad8c53c --- .../wm/AccessibilityWindowsPopulator.java | 48 ++++++++++++++----- 1 file changed, 36 insertions(+), 12 deletions(-) diff --git a/services/core/java/com/android/server/wm/AccessibilityWindowsPopulator.java b/services/core/java/com/android/server/wm/AccessibilityWindowsPopulator.java index 246f9acf55bb8..c0fb83ba294c6 100644 --- a/services/core/java/com/android/server/wm/AccessibilityWindowsPopulator.java +++ b/services/core/java/com/android/server/wm/AccessibilityWindowsPopulator.java @@ -67,6 +67,8 @@ public final class AccessibilityWindowsPopulator extends WindowInfosListener { @GuardedBy("mLock") private final SparseArray mMagnificationSpecInverseMatrix = new SparseArray<>(); @GuardedBy("mLock") + private final SparseArray mDisplayInfos = new SparseArray<>(); + @GuardedBy("mLock") private final List mVisibleWindows = new ArrayList<>(); @GuardedBy("mLock") private boolean mWindowsNotificationEnabled = false; @@ -93,6 +95,7 @@ public final class AccessibilityWindowsPopulator extends WindowInfosListener { List outWindows) { List inputWindowHandles; final Matrix inverseMatrix = new Matrix(); + final Matrix displayMatrix = new Matrix(); synchronized (mLock) { inputWindowHandles = mInputWindowHandlesOnDisplays.get(displayId); @@ -101,25 +104,34 @@ public final class AccessibilityWindowsPopulator extends WindowInfosListener { return; } - inverseMatrix.set(mMagnificationSpecInverseMatrix.get(displayId)); + + final DisplayInfo displayInfo = mDisplayInfos.get(displayId); + if (displayInfo != null) { + displayMatrix.set(displayInfo.mTransform); + } else { + Slog.w(TAG, "The displayInfo of this displayId (" + displayId + ") called " + + "back from the surface fligner is null"); + } } final DisplayContent dc = mService.mRoot.getDisplayContent(displayId); final ShellRoot shellroot = dc.mShellRoots.get(WindowManager.SHELL_ROOT_LAYER_PIP); final IBinder pipMenuIBinder = shellroot != null ? shellroot.getAccessibilityWindowToken() : null; + for (final InputWindowHandle windowHandle : inputWindowHandles) { final AccessibilityWindow accessibilityWindow = AccessibilityWindow.initializeData(mService, windowHandle, inverseMatrix, - pipMenuIBinder); + pipMenuIBinder, displayMatrix); outWindows.add(accessibilityWindow); } } @Override - public void onWindowInfosChanged(InputWindowHandle[] windowHandles) { + public void onWindowInfosChanged(InputWindowHandle[] windowHandles, + DisplayInfo[] displayInfos) { synchronized (mLock) { mVisibleWindows.clear(); for (InputWindowHandle window : windowHandles) { @@ -127,6 +139,12 @@ public final class AccessibilityWindowsPopulator extends WindowInfosListener { mVisibleWindows.add(window); } } + + mDisplayInfos.clear(); + for (final DisplayInfo displayInfo : displayInfos) { + mDisplayInfos.put(displayInfo.mDisplayId, displayInfo); + } + if (mWindowsNotificationEnabled) { if (!mHandler.hasMessages( MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_TIMEOUT)) { @@ -291,6 +309,7 @@ public final class AccessibilityWindowsPopulator extends WindowInfosListener { mInputWindowHandlesOnDisplays.clear(); mMagnificationSpecInverseMatrix.clear(); mVisibleWindows.clear(); + mDisplayInfos.clear(); mWindowsNotificationEnabled = false; mHandler.removeCallbacksAndMessages(null); } @@ -358,7 +377,8 @@ public final class AccessibilityWindowsPopulator extends WindowInfosListener { * @param inverseMatrix The magnification spec inverse matrix. */ public static AccessibilityWindow initializeData(WindowManagerService service, - InputWindowHandle inputWindowHandle, Matrix inverseMatrix, IBinder pipIBinder) { + InputWindowHandle inputWindowHandle, Matrix inverseMatrix, IBinder pipIBinder, + Matrix displayMatrix) { final IWindow window = inputWindowHandle.getWindow(); final WindowState windowState = window != null ? service.mWindowMap.get( window.asBinder()) : null; @@ -391,10 +411,10 @@ public final class AccessibilityWindowsPopulator extends WindowInfosListener { inputWindowHandle.frameTop, inputWindowHandle.frameRight, inputWindowHandle.frameBottom); getTouchableRegionInWindow(instance.mShouldMagnify, inputWindowHandle.touchableRegion, - instance.mTouchableRegionInWindow, windowFrame, inverseMatrix); + instance.mTouchableRegionInWindow, windowFrame, inverseMatrix, displayMatrix); getUnMagnifiedTouchableRegion(instance.mShouldMagnify, inputWindowHandle.touchableRegion, instance.mTouchableRegionInScreen, - inverseMatrix); + inverseMatrix, displayMatrix); instance.mWindowInfo = windowState != null ? windowState.getWindowInfo() : getWindowInfoForWindowlessWindows(instance); @@ -508,7 +528,7 @@ public final class AccessibilityWindowsPopulator extends WindowInfosListener { } private static void getTouchableRegionInWindow(boolean shouldMagnify, Region inRegion, - Region outRegion, Rect frame, Matrix inverseMatrix) { + Region outRegion, Rect frame, Matrix inverseMatrix, Matrix displayMatrix) { // Some modal windows, like the activity with Theme.dialog, has the full screen // as its touchable region, but its window frame is smaller than the touchable // region. The region we report should be the touchable area in the window frame @@ -519,7 +539,8 @@ public final class AccessibilityWindowsPopulator extends WindowInfosListener { touchRegion.set(inRegion); touchRegion.op(frame, Region.Op.INTERSECT); - getUnMagnifiedTouchableRegion(shouldMagnify, touchRegion, outRegion, inverseMatrix); + getUnMagnifiedTouchableRegion(shouldMagnify, touchRegion, outRegion, inverseMatrix, + displayMatrix); } /** @@ -530,10 +551,12 @@ public final class AccessibilityWindowsPopulator extends WindowInfosListener { * @param inRegion The touchable region of this window. * @param outRegion The un-magnified touchable region of this window. * @param inverseMatrix The inverse matrix of the magnification spec. + * @param displayMatrix The display transform matrix which takes display coordinates to + * logical display coordinates. */ private static void getUnMagnifiedTouchableRegion(boolean shouldMagnify, Region inRegion, - Region outRegion, Matrix inverseMatrix) { - if (!shouldMagnify || inverseMatrix.isIdentity()) { + Region outRegion, Matrix inverseMatrix, Matrix displayMatrix) { + if ((!shouldMagnify || inverseMatrix.isIdentity()) && displayMatrix.isIdentity()) { outRegion.set(inRegion); return; } @@ -544,6 +567,7 @@ public final class AccessibilityWindowsPopulator extends WindowInfosListener { windowFrame.set(rect); inverseMatrix.mapRect(windowFrame); + displayMatrix.mapRect(windowFrame); // Union all rects. outRegion.union(new Rect((int) windowFrame.left, (int) windowFrame.top, (int) windowFrame.right, (int) windowFrame.bottom)); @@ -586,9 +610,9 @@ public final class AccessibilityWindowsPopulator extends WindowInfosListener { + ", type=" + mType + ", privateFlag=0x" + Integer.toHexString(mPrivateFlags) + ", focused=" + mIsFocused - + ", mShouldMagnify=" + mShouldMagnify + + ", shouldMagnify=" + mShouldMagnify + ", ignoreDuetoRecentsAnimation=" + mIgnoreDuetoRecentsAnimation - + ", mIsTrustedOverlay=" + mIsTrustedOverlay + + ", isTrustedOverlay=" + mIsTrustedOverlay + ", regionInScreen=" + mTouchableRegionInScreen + ", touchableRegion=" + mTouchableRegionInWindow + ", letterBoxBounds=" + mLetterBoxBounds