Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Uses SF callback windows for the A11yWindowInfo [1/n]

Browse Source 
1. Implements AccessibilityWindowsPopulator class including the
new accessibilityWindow class generated by the inputWindowHandle
from the surface flinger.

2. Replaces the windowState by the accessibilityWindow used in the
AccessibiltyController class for computing the changed windows
reported to the A11y framework.

3. The embedded hiarerchy windows would be reported to A11y framework
due to they are reported from the surface fligner. This will make
the traversal function is broken when the Auotfill features is using
talkback. We find out the embedded hiarerchy windows by its window
token and don't add them into the A11y windows list.

4. Removes the modal windows calculation of the unAccountedSpace
including the task fragment due to we intersec the region between
the frame and touch region for all windows.

5. When reported windows counts from the surface flinger changed,
calling the accessibilty controller class to compute the changed
windows and send the results to the A11y framework immediately.
This can avoid the CTS failure with no corresponding window found
when the activity is launched due to the timing issue between the
callbacks for computing changed windows from the WM and the
callback for changed windows from the surface flinger.

Bug: 191736824
Test: a11y CTS & unit tests
Test: Manual testing including the A11y services
Change-Id: Ie9150ddc8e53ca88ad8cd7ba9376837e31436518
This commit is contained in:
Jacky Kao
2021-11-03 14:49:53 +08:00
parent 12dd1d6747
commit a2fae6b5d6
3 changed files with 651 additions and 196 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
services/accessibility/java/com/android/server/accessibility/AccessibilityWindowManager.java
View File

@@ -667,6 +667,10 @@ public class AccessibilityWindowManager {
return null;
}
// Don't need to add the embedded hierarchy windows into the accessibility windows list.
if (mHostEmbeddedMap.size() > 0 && isEmbeddedHierarchyWindowsLocked(windowId)) {
return null;
}
final AccessibilityWindowInfo reportedWindow = AccessibilityWindowInfo.obtain();
reportedWindow.setId(windowId);
@@ -699,6 +703,21 @@ public class AccessibilityWindowManager {
return reportedWindow;
}
private boolean isEmbeddedHierarchyWindowsLocked(int windowId) {
final IBinder leashToken = mWindowIdMap.get(windowId);
if (leashToken == null) {
return false;
}
for (int i = 0; i < mHostEmbeddedMap.size(); i++) {
if (mHostEmbeddedMap.keyAt(i).equals(leashToken)) {
return true;
}
}
return false;
}
private int getTypeForWindowManagerWindowType(int windowType) {
switch (windowType) {
case WindowManager.LayoutParams.TYPE_APPLICATION:

294
services/core/java/com/android/server/wm/AccessibilityController.java
View File

@@ -45,7 +45,6 @@ import static com.android.server.accessibility.AccessibilityTraceProto.WINDOW_MA
import static com.android.server.wm.WindowManagerDebugConfig.TAG_WITH_CLASS_NAME;
import static com.android.server.wm.WindowManagerDebugConfig.TAG_WM;
import static com.android.server.wm.WindowTracing.WINSCOPE_EXT;
import static com.android.server.wm.utils.RegionUtils.forEachRect;
import android.accessibilityservice.AccessibilityTrace;
import android.animation.ObjectAnimator;
@@ -101,6 +100,7 @@ import com.android.internal.util.TraceBuffer;
import com.android.internal.util.function.pooled.PooledLambda;
import com.android.server.LocalServices;
import com.android.server.policy.WindowManagerPolicy;
import com.android.server.wm.AccessibilityWindowsPopulator.AccessibilityWindow;
import com.android.server.wm.WindowManagerInternal.AccessibilityControllerInternal;
import com.android.server.wm.WindowManagerInternal.MagnificationCallbacks;
import com.android.server.wm.WindowManagerInternal.WindowsForAccessibilityCallback;
@@ -133,19 +133,22 @@ final class AccessibilityController {
private static final Rect EMPTY_RECT = new Rect();
private static final float[] sTempFloats = new float[9];
private SparseArray<DisplayMagnifier> mDisplayMagnifiers = new SparseArray<>();
private SparseArray<WindowsForAccessibilityObserver> mWindowsForAccessibilityObserver =
private final SparseArray<DisplayMagnifier> mDisplayMagnifiers = new SparseArray<>();
private final SparseArray<WindowsForAccessibilityObserver> mWindowsForAccessibilityObserver =
new SparseArray<>();
private SparseArray<IBinder> mFocusedWindow = new SparseArray<>();
private int mFocusedDisplay = -1;
private boolean mIsImeVisible = false;
// Set to true if initializing window population complete.
private boolean mAllObserversInitialized = true;
private final AccessibilityWindowsPopulator mAccessibilityWindowsPopulator;
AccessibilityController(WindowManagerService service) {
mService = service;
mAccessibilityTracing =
AccessibilityController.getAccessibilityControllerInternal(service);
mAccessibilityWindowsPopulator = new AccessibilityWindowsPopulator(mService, this);
}
boolean setMagnificationCallbacks(int displayId, MagnificationCallbacks callbacks) {
@@ -209,7 +212,9 @@ final class AccessibilityController {
}
mWindowsForAccessibilityObserver.remove(displayId);
}
observer = new WindowsForAccessibilityObserver(mService, displayId, callback);
mAccessibilityWindowsPopulator.setWindowsNotification(true);
observer = new WindowsForAccessibilityObserver(mService, displayId, callback,
mAccessibilityWindowsPopulator);
mWindowsForAccessibilityObserver.put(displayId, observer);
mAllObserversInitialized &= observer.mInitialized;
} else {
@@ -224,6 +229,10 @@ final class AccessibilityController {
}
}
mWindowsForAccessibilityObserver.remove(displayId);
if (mWindowsForAccessibilityObserver.size() <= 0) {
mAccessibilityWindowsPopulator.setWindowsNotification(false);
}
}
}
@@ -455,6 +464,19 @@ final class AccessibilityController {
return null;
}
boolean getMagnificationSpecForDisplay(int displayId, MagnificationSpec outSpec) {
if (mAccessibilityTracing.isTracingEnabled(FLAGS_MAGNIFICATION_CALLBACK)) {
mAccessibilityTracing.logTrace(TAG + ".getMagnificationSpecForDisplay",
FLAGS_MAGNIFICATION_CALLBACK, "displayId=" + displayId);
}
final DisplayMagnifier displayMagnifier = mDisplayMagnifiers.get(displayId);
if (displayMagnifier == null) {
return false;
}
return displayMagnifier.getMagnificationSpec(outSpec);
}
boolean hasCallbacks() {
if (mAccessibilityTracing.isTracingEnabled(FLAGS_MAGNIFICATION_CALLBACK
| FLAGS_WINDOWS_FOR_ACCESSIBILITY_CALLBACK)) {
@@ -756,6 +778,25 @@ final class AccessibilityController {
return spec;
}
boolean getMagnificationSpec(MagnificationSpec outSpec) {
if (mAccessibilityTracing.isTracingEnabled(FLAGS_MAGNIFICATION_CALLBACK)) {
mAccessibilityTracing.logTrace(LOG_TAG + ".getMagnificationSpec",
FLAGS_MAGNIFICATION_CALLBACK);
}
MagnificationSpec spec = mMagnifedViewport.getMagnificationSpec();
if (spec == null) {
return false;
}
outSpec.setTo(spec);
if (mAccessibilityTracing.isTracingEnabled(FLAGS_MAGNIFICATION_CALLBACK)) {
mAccessibilityTracing.logTrace(LOG_TAG + ".getMagnificationSpec",
FLAGS_MAGNIFICATION_CALLBACK, "outSpec={" + outSpec + "}");
}
return true;
}
void getMagnificationRegion(Region outMagnificationRegion) {
if (mAccessibilityTracing.isTracingEnabled(FLAGS_MAGNIFICATION_CALLBACK)) {
mAccessibilityTracing.logTrace(LOG_TAG + ".getMagnificationRegion",
@@ -1403,20 +1444,18 @@ final class AccessibilityController {
private static final boolean DEBUG = false;
private final SparseArray<WindowState> mTempWindowStates = new SparseArray<>();
private final List<AccessibilityWindow> mTempA11yWindows = new ArrayList<>();
private final Set<IBinder> mTempBinderSet = new ArraySet<>();
private final RectF mTempRectF = new RectF();
private final Matrix mTempMatrix = new Matrix();
private final Point mTempPoint = new Point();
private final Region mTempRegion = new Region();
private final Region mTempRegion1 = new Region();
private final Region mTempRegion2 = new Region();
private final WindowManagerService mService;
private final Handler mHandler;
@@ -1431,10 +1470,11 @@ final class AccessibilityController {
// Set to true if initializing window population complete.
private boolean mInitialized;
private final AccessibilityWindowsPopulator mA11yWindowsPopulator;
WindowsForAccessibilityObserver(WindowManagerService windowManagerService,
int displayId,
WindowsForAccessibilityCallback callback) {
int displayId, WindowsForAccessibilityCallback callback,
AccessibilityWindowsPopulator accessibilityWindowsPopulator) {
mService = windowManagerService;
mCallback = callback;
mDisplayId = displayId;
@@ -1443,6 +1483,7 @@ final class AccessibilityController {
AccessibilityController.getAccessibilityControllerInternal(mService);
mRecurringAccessibilityEventsIntervalMillis = ViewConfiguration
.getSendRecurringAccessibilityEventsInterval();
mA11yWindowsPopulator = accessibilityWindowsPopulator;
computeChangedWindows(true);
}
@@ -1466,52 +1507,6 @@ final class AccessibilityController {
}
}
boolean shellRootIsAbove(WindowState windowState, ShellRoot shellRoot) {
int wsLayer = mService.mPolicy.getWindowLayerLw(windowState);
int shellLayer = mService.mPolicy.getWindowLayerFromTypeLw(shellRoot.getWindowType(),
true);
return shellLayer >= wsLayer;
}
int addShellRootsIfAbove(WindowState windowState, ArrayList<ShellRoot> shellRoots,
int shellRootIndex, List<WindowInfo> windows, Set<IBinder> addedWindows,
Region unaccountedSpace, boolean focusedWindowAdded) {
while (shellRootIndex < shellRoots.size()
&& shellRootIsAbove(windowState, shellRoots.get(shellRootIndex))) {
ShellRoot shellRoot = shellRoots.get(shellRootIndex);
shellRootIndex++;
final WindowInfo info = shellRoot.getWindowInfo();
if (info == null) {
continue;
}
info.layer = addedWindows.size();
windows.add(info);
addedWindows.add(info.token);
unaccountedSpace.op(info.regionInScreen, unaccountedSpace,
Region.Op.REVERSE_DIFFERENCE);
if (unaccountedSpace.isEmpty() && focusedWindowAdded) {
break;
}
}
return shellRootIndex;
}
private ArrayList<ShellRoot> getSortedShellRoots(
SparseArray<ShellRoot> originalShellRoots) {
ArrayList<ShellRoot> sortedShellRoots = new ArrayList<>(originalShellRoots.size());
for (int i = originalShellRoots.size() - 1; i >= 0; --i) {
sortedShellRoots.add(originalShellRoots.valueAt(i));
}
sortedShellRoots.sort((left, right) ->
mService.mPolicy.getWindowLayerFromTypeLw(right.getWindowType(), true)
- mService.mPolicy.getWindowLayerFromTypeLw(left.getWindowType(),
true));
return sortedShellRoots;
}
/**
* Check if windows have changed, and send them to the accessibility subsystem if they have.
*
@@ -1561,44 +1556,29 @@ final class AccessibilityController {
Region unaccountedSpace = mTempRegion;
unaccountedSpace.set(0, 0, screenWidth, screenHeight);
final SparseArray<WindowState> visibleWindows = mTempWindowStates;
populateVisibleWindowsOnScreen(visibleWindows);
final List<AccessibilityWindow> visibleWindows = mTempA11yWindows;
mA11yWindowsPopulator.populateVisibleWindowsOnScreenLocked(
mDisplayId, visibleWindows);
Set<IBinder> addedWindows = mTempBinderSet;
addedWindows.clear();
boolean focusedWindowAdded = false;
final int visibleWindowCount = visibleWindows.size();
ArrayList<TaskFragment> skipRemainingWindowsForTaskFragments = new ArrayList<>();
ArrayList<ShellRoot> shellRoots = getSortedShellRoots(dc.mShellRoots);
// Iterate until we figure out what is touchable for the entire screen.
int shellRootIndex = 0;
for (int i = visibleWindowCount - 1; i >= 0; i--) {
final WindowState windowState = visibleWindows.valueAt(i);
int prevShellRootIndex = shellRootIndex;
shellRootIndex = addShellRootsIfAbove(windowState, shellRoots, shellRootIndex,
windows, addedWindows, unaccountedSpace, focusedWindowAdded);
// If a Shell Root was added, it could have accounted for all the space already.
if (shellRootIndex > prevShellRootIndex && unaccountedSpace.isEmpty()
&& focusedWindowAdded) {
break;
}
final Region regionInScreen = new Region();
computeWindowRegionInScreen(windowState, regionInScreen);
if (windowMattersToAccessibility(windowState,
regionInScreen, unaccountedSpace,
skipRemainingWindowsForTaskFragments)) {
addPopulatedWindowInfo(windowState, regionInScreen, windows, addedWindows);
if (windowMattersToUnaccountedSpaceComputation(windowState)) {
updateUnaccountedSpace(windowState, regionInScreen, unaccountedSpace,
skipRemainingWindowsForTaskFragments);
for (int i = 0; i < visibleWindowCount; i++) {
final AccessibilityWindow a11yWindow = visibleWindows.get(i);
final Region regionInWindow = new Region();
a11yWindow.getTouchableRegionInWindow(regionInWindow);
if (windowMattersToAccessibility(a11yWindow, regionInWindow,
unaccountedSpace)) {
addPopulatedWindowInfo(a11yWindow, regionInWindow, windows, addedWindows);
if (windowMattersToUnaccountedSpaceComputation(a11yWindow)) {
updateUnaccountedSpace(a11yWindow, unaccountedSpace);
}
focusedWindowAdded |= windowState.isFocused();
} else if (isUntouchableNavigationBar(windowState, mTempRegion1)) {
focusedWindowAdded |= a11yWindow.isFocused();
} else if (a11yWindow.isUntouchableNavigationBar()) {
// If this widow is navigation bar without touchable region, accounting the
// region of navigation bar inset because all touch events from this region
// would be received by launcher, i.e. this region is a un-touchable one
@@ -1647,47 +1627,38 @@ final class AccessibilityController {
// Some windows should be excluded from unaccounted space computation, though they still
// should be reported
private boolean windowMattersToUnaccountedSpaceComputation(WindowState windowState) {
private boolean windowMattersToUnaccountedSpaceComputation(AccessibilityWindow a11yWindow) {
// Do not account space of trusted non-touchable windows, except the split-screen
// divider.
// If it's not trusted, touch events are not sent to the windows behind it.
if (((windowState.mAttrs.flags & WindowManager.LayoutParams.FLAG_NOT_TOUCHABLE) != 0)
&& (windowState.mAttrs.type != TYPE_DOCK_DIVIDER)
&& windowState.isTrustedOverlay()) {
if (((a11yWindow.getFlags() & WindowManager.LayoutParams.FLAG_NOT_TOUCHABLE) != 0)
&& (a11yWindow.getType() != TYPE_DOCK_DIVIDER)
&& a11yWindow.isTrustedOverlay()) {
return false;
}
if (windowState.mAttrs.type
== WindowManager.LayoutParams.TYPE_ACCESSIBILITY_OVERLAY) {
if (a11yWindow.getType() == WindowManager.LayoutParams.TYPE_ACCESSIBILITY_OVERLAY) {
return false;
}
return true;
}
private boolean windowMattersToAccessibility(WindowState windowState,
Region regionInScreen, Region unaccountedSpace,
ArrayList<TaskFragment> skipRemainingWindowsForTaskFragments) {
final RecentsAnimationController controller = mService.getRecentsAnimationController();
if (controller != null && controller.shouldIgnoreForAccessibility(windowState)) {
private boolean windowMattersToAccessibility(AccessibilityWindow a11yWindow,
Region regionInScreen, Region unaccountedSpace) {
if (a11yWindow.ignoreRecentsAnimationForAccessibility()) {
return false;
}
if (windowState.isFocused()) {
if (a11yWindow.isFocused()) {
return true;
}
// If the window is part of a task that we're finished with - ignore.
final TaskFragment taskFragment = windowState.getTaskFragment();
if (taskFragment != null
&& skipRemainingWindowsForTaskFragments.contains(taskFragment)) {
return false;
}
// Ignore non-touchable windows, except the split-screen divider, which is
// occasionally non-touchable but still useful for identifying split-screen
// mode.
if (((windowState.mAttrs.flags & WindowManager.LayoutParams.FLAG_NOT_TOUCHABLE) != 0)
&& (windowState.mAttrs.type != TYPE_DOCK_DIVIDER)) {
if (((a11yWindow.getFlags()
& WindowManager.LayoutParams.FLAG_NOT_TOUCHABLE) != 0)
&& (a11yWindow.getType() != TYPE_DOCK_DIVIDER)) {
return false;
}
@@ -1697,88 +1668,36 @@ final class AccessibilityController {
}
// Add windows of certain types not covered by modal windows.
if (isReportedWindowType(windowState.mAttrs.type)) {
if (isReportedWindowType(a11yWindow.getType())) {
return true;
}
return false;
}
private void updateUnaccountedSpace(WindowState windowState, Region regionInScreen,
Region unaccountedSpace,
ArrayList<TaskFragment> skipRemainingWindowsForTaskFragments) {
// Account for the space this window takes if the window
// is not an accessibility overlay which does not change
// the reported windows.
unaccountedSpace.op(regionInScreen, unaccountedSpace,
Region.Op.REVERSE_DIFFERENCE);
// If a window is modal it prevents other windows from being touched
if ((windowState.mAttrs.flags & (WindowManager.LayoutParams.FLAG_NOT_FOCUSABLE
| WindowManager.LayoutParams.FLAG_NOT_TOUCH_MODAL)) == 0) {
if (!windowState.hasTapExcludeRegion()) {
// Account for all space in the task, whether the windows in it are
// touchable or not. The modal window blocks all touches from the task's
// area.
unaccountedSpace.op(windowState.getDisplayFrame(), unaccountedSpace,
Region.Op.REVERSE_DIFFERENCE);
} else {
// If a window has tap exclude region, we need to account it.
final Region displayRegion = new Region(windowState.getDisplayFrame());
final Region tapExcludeRegion = new Region();
windowState.getTapExcludeRegion(tapExcludeRegion);
displayRegion.op(tapExcludeRegion, displayRegion,
Region.Op.REVERSE_DIFFERENCE);
unaccountedSpace.op(displayRegion, unaccountedSpace,
Region.Op.REVERSE_DIFFERENCE);
}
final TaskFragment taskFragment = windowState.getTaskFragment();
if (taskFragment != null) {
// If the window is associated with a particular task, we can skip the
// rest of the windows for that task.
skipRemainingWindowsForTaskFragments.add(taskFragment);
} else if (!windowState.hasTapExcludeRegion()) {
// If the window is not associated with a particular task, then it is
// globally modal. In this case we can skip all remaining windows when
// it doesn't has tap exclude region.
unaccountedSpace.setEmpty();
}
}
// Account for the space of letterbox.
if (windowState.areAppWindowBoundsLetterboxed()) {
unaccountedSpace.op(getLetterboxBounds(windowState), unaccountedSpace,
private void updateUnaccountedSpace(AccessibilityWindow a11yWindow,
Region unaccountedSpace) {
if (a11yWindow.getType()
!= WindowManager.LayoutParams.TYPE_ACCESSIBILITY_OVERLAY) {
// Account for the space this window takes if the window
// is not an accessibility overlay which does not change
// the reported windows.
final Region touchableRegion = mTempRegion2;
a11yWindow.getTouchableRegionInScreen(touchableRegion);
unaccountedSpace.op(touchableRegion, unaccountedSpace,
Region.Op.REVERSE_DIFFERENCE);
// Account for the space of letterbox.
final Region letterboxBounds = mTempRegion1;
if (a11yWindow.setLetterBoxBoundsIfNeeded(letterboxBounds)) {
unaccountedSpace.op(letterboxBounds,
unaccountedSpace, Region.Op.REVERSE_DIFFERENCE);
}
}
}
private void computeWindowRegionInScreen(WindowState windowState, Region outRegion) {
// Get the touchable frame.
Region touchableRegion = mTempRegion1;
windowState.getTouchableRegion(touchableRegion);
// Map the frame to get what appears on the screen.
Matrix matrix = mTempMatrix;
populateTransformationMatrix(windowState, matrix);
forEachRect(touchableRegion, rect -> {
// Move to origin as all transforms are captured by the matrix.
RectF windowFrame = mTempRectF;
windowFrame.set(rect);
windowFrame.offset(-windowState.getFrame().left, -windowState.getFrame().top);
matrix.mapRect(windowFrame);
// Union all rects.
outRegion.union(new Rect((int) windowFrame.left, (int) windowFrame.top,
(int) windowFrame.right, (int) windowFrame.bottom));
});
}
private static void addPopulatedWindowInfo(WindowState windowState, Region regionInScreen,
List<WindowInfo> out, Set<IBinder> tokenOut) {
final WindowInfo window = windowState.getWindowInfo();
private static void addPopulatedWindowInfo(AccessibilityWindow a11yWindow,
Region regionInScreen, List<WindowInfo> out, Set<IBinder> tokenOut) {
final WindowInfo window = a11yWindow.getWindowInfo();
window.regionInScreen.set(regionInScreen);
window.layer = tokenOut.size();
out.add(window);
@@ -1805,23 +1724,6 @@ final class AccessibilityController {
&& windowType != WindowManager.LayoutParams.TYPE_PRIVATE_PRESENTATION);
}
private void populateVisibleWindowsOnScreen(SparseArray<WindowState> outWindows) {
final List<WindowState> tempWindowStatesList = new ArrayList<>();
final DisplayContent dc = mService.mRoot.getDisplayContent(mDisplayId);
if (dc == null) {
return;
}
dc.forAllWindows(w -> {
if (w.isVisible()) {
tempWindowStatesList.add(w);
}
}, false /* traverseTopToBottom */);
for (int i = 0; i < tempWindowStatesList.size(); i++) {
outWindows.put(i, tempWindowStatesList.get(i));
}
}
private WindowState getTopFocusWindow() {
return mService.mRoot.getTopFocusedDisplayContent().mCurrentFocus;
}

534
services/core/java/com/android/server/wm/AccessibilityWindowsPopulator.java Normal file
View File

@@ -0,0 +1,534 @@
/*
* Copyright (C) 2021 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.server.wm;
import static android.view.WindowManager.LayoutParams.TYPE_DOCK_DIVIDER;
import static com.android.server.wm.utils.RegionUtils.forEachRect;
import android.annotation.NonNull;
import android.graphics.Matrix;
import android.graphics.Rect;
import android.graphics.RectF;
import android.graphics.Region;
import android.os.Handler;
import android.os.Looper;
import android.os.Message;
import android.util.Slog;
import android.util.SparseArray;
import android.view.IWindow;
import android.view.InputWindowHandle;
import android.view.MagnificationSpec;
import android.view.WindowInfo;
import android.view.WindowManager;
import android.window.WindowInfosListener;
import com.android.internal.annotations.GuardedBy;
import java.util.ArrayList;
import java.util.List;
/**
* This class is the accessibility windows population adapter.
*/
public final class AccessibilityWindowsPopulator extends WindowInfosListener {
private static final String TAG = AccessibilityWindowsPopulator.class.getSimpleName();
private static final float[] sTempFloats = new float[9];
private final WindowManagerService mService;
private final AccessibilityController mAccessibilityController;
@GuardedBy("mLock")
private final SparseArray<List<InputWindowHandle>> mInputWindowHandlesOnDisplays =
new SparseArray<>();
@GuardedBy("mLock")
private final SparseArray<Matrix> mMagnificationSpecInverseMatrix = new SparseArray<>();
@GuardedBy("mLock")
private final List<InputWindowHandle> mVisibleWindows = new ArrayList<>();
@GuardedBy("mLock")
private boolean mWindowsNotificationEnabled = false;
private final Object mLock = new Object();
private final Handler mHandler;
AccessibilityWindowsPopulator(WindowManagerService service,
AccessibilityController accessibilityController) {
mService = service;
mAccessibilityController = accessibilityController;
mHandler = new MyHandler(mService.mH.getLooper());
register();
}
/**
* Gets the visible windows list with the window layer on the specified display.
*
* @param displayId The display.
* @param outWindows The visible windows list. The z-order of each window in the list
* is from the top to bottom.
*/
public void populateVisibleWindowsOnScreenLocked(int displayId,
List<AccessibilityWindow> outWindows) {
List<InputWindowHandle> inputWindowHandles;
final Matrix inverseMatrix = new Matrix();
synchronized (mLock) {
inputWindowHandles = mInputWindowHandlesOnDisplays.get(displayId);
if (inputWindowHandles == null) {
outWindows.clear();
return;
}
inverseMatrix.set(mMagnificationSpecInverseMatrix.get(displayId));
}
for (final InputWindowHandle windowHandle : inputWindowHandles) {
final AccessibilityWindow accessibilityWindow =
AccessibilityWindow.initializeData(mService, windowHandle, inverseMatrix);
outWindows.add(accessibilityWindow);
}
}
@Override
public void onWindowInfosChanged(InputWindowHandle[] windowHandles) {
synchronized (mLock) {
mVisibleWindows.clear();
for (InputWindowHandle window : windowHandles) {
if (window.visible && window.getWindow() != null) {
mVisibleWindows.add(window);
}
}
if (mWindowsNotificationEnabled) {
populateVisibleWindowHandlesAndNotifyWindowsChangeIfNeededLocked();
}
}
}
/**
* Sets to notify the accessibilityController to compute changed windows on
* the display after populating the visible windows if the windows reported
* from the surface flinger changes.
*
* @param register {@code true} means starting windows population.
*/
public void setWindowsNotification(boolean register) {
synchronized (mLock) {
if (mWindowsNotificationEnabled == register) {
return;
}
mWindowsNotificationEnabled = register;
if (mWindowsNotificationEnabled) {
populateVisibleWindowHandlesAndNotifyWindowsChangeIfNeededLocked();
} else {
releaseResources();
}
}
}
private void populateVisibleWindowHandlesAndNotifyWindowsChangeIfNeededLocked() {
final SparseArray<List<InputWindowHandle>> tempWindowHandleList = new SparseArray<>();
for (final InputWindowHandle windowHandle : mVisibleWindows) {
List<InputWindowHandle> inputWindowHandles = tempWindowHandleList.get(
windowHandle.displayId);
if (inputWindowHandles == null) {
inputWindowHandles = new ArrayList<>();
tempWindowHandleList.put(windowHandle.displayId, inputWindowHandles);
generateMagnificationSpecInverseMatrixLocked(windowHandle.displayId);
}
inputWindowHandles.add(windowHandle);
}
final List<Integer> displayIdsForWindowsChanged = new ArrayList<>();
getDisplaysForWindowsChangedLocked(displayIdsForWindowsChanged, tempWindowHandleList,
mInputWindowHandlesOnDisplays);
// Clones all windows from the callback of the surface flinger.
mInputWindowHandlesOnDisplays.clear();
for (int i = 0; i < tempWindowHandleList.size(); i++) {
final int displayId = tempWindowHandleList.keyAt(i);
mInputWindowHandlesOnDisplays.put(displayId, tempWindowHandleList.get(displayId));
}
if (displayIdsForWindowsChanged.size() > 0
&& !mHandler.hasMessages(MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED)) {
mHandler.obtainMessage(MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED,
displayIdsForWindowsChanged).sendToTarget();
}
}
private void getDisplaysForWindowsChangedLocked(List<Integer> outDisplayIdsForWindowsChanged,
SparseArray<List<InputWindowHandle>> newWindowsList,
SparseArray<List<InputWindowHandle>> oldWindowsList) {
for (int i = 0; i < newWindowsList.size(); i++) {
final int displayId = newWindowsList.keyAt(i);
final List<InputWindowHandle> newWindows = newWindowsList.get(displayId);
final List<InputWindowHandle> oldWindows = oldWindowsList.get(displayId);
if (hasWindowsChangedLocked(newWindows, oldWindows)) {
outDisplayIdsForWindowsChanged.add(displayId);
}
}
}
private boolean hasWindowsChangedLocked(List<InputWindowHandle> newWindows,
List<InputWindowHandle> oldWindows) {
if (oldWindows == null || oldWindows.size() != newWindows.size()) {
return true;
}
final int windowsCount = newWindows.size();
// Since we always traverse windows from high to low layer,
// the old and new windows at the same index should be the
// same, otherwise something changed.
for (int i = 0; i < windowsCount; i++) {
final InputWindowHandle newWindow = newWindows.get(i);
final InputWindowHandle oldWindow = oldWindows.get(i);
if (!newWindow.getWindow().asBinder().equals(oldWindow.getWindow().asBinder())) {
return true;
}
}
return false;
}
private void generateMagnificationSpecInverseMatrixLocked(int displayId) {
MagnificationSpec spec = new MagnificationSpec();
if (!mAccessibilityController.getMagnificationSpecForDisplay(displayId, spec)) {
mMagnificationSpecInverseMatrix.remove(displayId);
return;
}
sTempFloats[Matrix.MSCALE_X] = spec.scale;
sTempFloats[Matrix.MSKEW_Y] = 0;
sTempFloats[Matrix.MSKEW_X] = 0;
sTempFloats[Matrix.MSCALE_Y] = spec.scale;
sTempFloats[Matrix.MTRANS_X] = spec.offsetX;
sTempFloats[Matrix.MTRANS_Y] = spec.offsetY;
sTempFloats[Matrix.MPERSP_0] = 0;
sTempFloats[Matrix.MPERSP_1] = 0;
sTempFloats[Matrix.MPERSP_2] = 1;
final Matrix tempMatrix = new Matrix();
tempMatrix.setValues(sTempFloats);
final Matrix inverseMatrix = new Matrix();
final boolean result = tempMatrix.invert(inverseMatrix);
if (!result) {
Slog.e(TAG, "Can't inverse the magnification spec matrix with the "
+ "magnification spec = " + spec + " on the displayId = " + displayId);
return;
}
mMagnificationSpecInverseMatrix.set(displayId, inverseMatrix);
}
private void notifyWindowsChanged(@NonNull List<Integer> displayIdsForWindowsChanged) {
for (int i = 0; i < displayIdsForWindowsChanged.size(); i++) {
mAccessibilityController.performComputeChangedWindowsNot(
displayIdsForWindowsChanged.get(i), false);
}
}
@GuardedBy("mLock")
private void releaseResources() {
mInputWindowHandlesOnDisplays.clear();
mMagnificationSpecInverseMatrix.clear();
mVisibleWindows.clear();
mWindowsNotificationEnabled = false;
}
private class MyHandler extends Handler {
public static final int MESSAGE_NOTIFY_WINDOWS_CHANGED = 1;
MyHandler(Looper looper) {
super(looper, null, false);
}
@Override
public void handleMessage(Message message) {
if (message.what == MESSAGE_NOTIFY_WINDOWS_CHANGED) {
final List<Integer> displayIdsForWindowsChanged = (List<Integer>) message.obj;
notifyWindowsChanged(displayIdsForWindowsChanged);
}
}
}
/**
* This class represents information about a window from the
* surface flinger to the accessibility framework.
*/
public static class AccessibilityWindow {
private static final Region TEMP_REGION = new Region();
private static final RectF TEMP_RECTF = new RectF();
// Data
private IWindow mWindow;
private int mDisplayId;
private int mFlags;
private int mType;
private int mPrivateFlags;
private boolean mIsFocused;
private boolean mShouldMagnify;
private boolean mIgnoreDuetoRecentsAnimation;
private boolean mIsTrustedOverlay;
private final Region mTouchableRegionInScreen = new Region();
private final Region mTouchableRegionInWindow = new Region();
private final Region mLetterBoxBounds = new Region();
private WindowInfo mWindowInfo;
/**
* Returns the instance after initializing the internal data.
* @param service The window manager service.
* @param inputWindowHandle The window from the surface flinger.
* @param inverseMatrix The magnification spec inverse matrix.
*/
public static AccessibilityWindow initializeData(WindowManagerService service,
InputWindowHandle inputWindowHandle, Matrix inverseMatrix) {
final IWindow window = inputWindowHandle.getWindow();
final WindowState windowState = window != null ? service.mWindowMap.get(
window.asBinder()) : null;
final AccessibilityWindow instance = new AccessibilityWindow();
instance.mWindow = inputWindowHandle.getWindow();
instance.mDisplayId = inputWindowHandle.displayId;
instance.mFlags = inputWindowHandle.layoutParamsFlags;
instance.mType = inputWindowHandle.layoutParamsType;
// TODO (b/199357848): gets the private flag of the window from other way.
instance.mPrivateFlags = windowState != null ? windowState.mAttrs.privateFlags : 0;
// TODO (b/199358208) : using new way to implement the focused window.
instance.mIsFocused = windowState != null && windowState.isFocused();
instance.mShouldMagnify = windowState == null || windowState.shouldMagnify();
final RecentsAnimationController controller = service.getRecentsAnimationController();
instance.mIgnoreDuetoRecentsAnimation = windowState != null && controller != null
&& controller.shouldIgnoreForAccessibility(windowState);
instance.mIsTrustedOverlay = inputWindowHandle.trustedOverlay;
// TODO (b/199358388) : gets the letterbox bounds of the window from other way.
if (windowState != null && windowState.areAppWindowBoundsLetterboxed()) {
getLetterBoxBounds(windowState, instance.mLetterBoxBounds);
}
final Rect windowFrame = new Rect(inputWindowHandle.frameLeft,
inputWindowHandle.frameTop, inputWindowHandle.frameRight,
inputWindowHandle.frameBottom);
getTouchableRegionInWindow(instance.mShouldMagnify, inputWindowHandle.touchableRegion,
instance.mTouchableRegionInWindow, windowFrame, inverseMatrix);
getUnMagnifiedTouchableRegion(instance.mShouldMagnify,
inputWindowHandle.touchableRegion, instance.mTouchableRegionInScreen,
inverseMatrix);
instance.mWindowInfo = windowState != null
? windowState.getWindowInfo() : getWindowInfoForWindowlessWindows(instance);
return instance;
}
/**
* Returns the touchable region in the screen.
* @param outRegion The touchable region.
*/
public void getTouchableRegionInScreen(Region outRegion) {
outRegion.set(mTouchableRegionInScreen);
}
/**
* Returns the touchable region in the window.
* @param outRegion The touchable region.
*/
public void getTouchableRegionInWindow(Region outRegion) {
outRegion.set(mTouchableRegionInWindow);
}
/**
* @return the layout parameter flag {@link android.view.WindowManager.LayoutParams#flags}.
*/
public int getFlags() {
return mFlags;
}
/**
* @return the layout parameter type {@link android.view.WindowManager.LayoutParams#type}.
*/
public int getType() {
return mType;
}
/**
* @return the layout parameter private flag
* {@link android.view.WindowManager.LayoutParams#privateFlags}.
*/
public int getPrivateFlag() {
return mPrivateFlags;
}
/**
* @return the windowInfo {@link WindowInfo}.
*/
public WindowInfo getWindowInfo() {
return mWindowInfo;
}
/**
* Gets the letter box bounds if activity bounds are letterboxed
* or letterboxed for display cutout.
*
* @return {@code true} there's a letter box bounds.
*/
public Boolean setLetterBoxBoundsIfNeeded(Region outBounds) {
if (mLetterBoxBounds.isEmpty()) {
return false;
}
outBounds.set(mLetterBoxBounds);
return true;
}
/**
* @return true if this window should be magnified.
*/
public boolean shouldMagnify() {
return mShouldMagnify;
}
/**
* @return true if this window is focused.
*/
public boolean isFocused() {
return mIsFocused;
}
/**
* @return true if it's running the recent animation but not the target app.
*/
public boolean ignoreRecentsAnimationForAccessibility() {
return mIgnoreDuetoRecentsAnimation;
}
/**
* @return true if this window is the trusted overlay.
*/
public boolean isTrustedOverlay() {
return mIsTrustedOverlay;
}
/**
* @return true if this window is the navigation bar with the gesture mode.
*/
public boolean isUntouchableNavigationBar() {
if (mType != WindowManager.LayoutParams.TYPE_NAVIGATION_BAR) {
return false;
}
return mTouchableRegionInScreen.isEmpty();
}
private static void getTouchableRegionInWindow(boolean shouldMagnify, Region inRegion,
Region outRegion, Rect frame, Matrix inverseMatrix) {
// Some modal windows, like the activity with Theme.dialog, has the full screen
// as its touchable region, but its window frame is smaller than the touchable
// region. The region we report should be the touchable area in the window frame
// for the consistency and match developers expectation.
// So we need to make the intersection between the frame and touchable region to
// obtain the real touch region in the screen.
Region touchRegion = TEMP_REGION;
touchRegion.set(inRegion);
touchRegion.op(frame, Region.Op.INTERSECT);
getUnMagnifiedTouchableRegion(shouldMagnify, touchRegion, outRegion, inverseMatrix);
}
/**
* Gets the un-magnified touchable region. If this window can be magnified and magnifying,
* we will transform the input touchable region by applying the inverse matrix of the
* magnification spec to get the un-magnified touchable region.
* @param shouldMagnify The window can be magnified.
* @param inRegion The touchable region of this window.
* @param outRegion The un-magnified touchable region of this window.
* @param inverseMatrix The inverse matrix of the magnification spec.
*/
private static void getUnMagnifiedTouchableRegion(boolean shouldMagnify, Region inRegion,
Region outRegion, Matrix inverseMatrix) {
if (!shouldMagnify || inverseMatrix.isIdentity()) {
outRegion.set(inRegion);
return;
}
forEachRect(inRegion, rect -> {
// Move to origin as all transforms are captured by the matrix.
RectF windowFrame = TEMP_RECTF;
windowFrame.set(rect);
inverseMatrix.mapRect(windowFrame);
// Union all rects.
outRegion.union(new Rect((int) windowFrame.left, (int) windowFrame.top,
(int) windowFrame.right, (int) windowFrame.bottom));
});
}
private static WindowInfo getWindowInfoForWindowlessWindows(AccessibilityWindow window) {
WindowInfo windowInfo = WindowInfo.obtain();
windowInfo.displayId = window.mDisplayId;
windowInfo.type = window.mType;
windowInfo.token = window.mWindow.asBinder();
windowInfo.hasFlagWatchOutsideTouch = (window.mFlags
& WindowManager.LayoutParams.FLAG_WATCH_OUTSIDE_TOUCH) != 0;
windowInfo.inPictureInPicture = false;
// There only are two windowless windows now, one is split window, and the other
// one is PIP.
if (windowInfo.type == TYPE_DOCK_DIVIDER) {
windowInfo.title = "Splitscreen Divider";
} else {
windowInfo.title = "Picture-in-Picture menu";
}
return windowInfo;
}
private static void getLetterBoxBounds(WindowState windowState, Region outRegion) {
final Rect letterboxInsets = windowState.mActivityRecord.getLetterboxInsets();
final Rect nonLetterboxRect = windowState.getBounds();
nonLetterboxRect.inset(letterboxInsets);
outRegion.set(windowState.getBounds());
outRegion.op(nonLetterboxRect, Region.Op.DIFFERENCE);
}
@Override
public String toString() {
String builder = "A11yWindow=[" + mWindow
+ ", displayId=" + mDisplayId
+ ", flag=0x" + Integer.toHexString(mFlags)
+ ", type=" + mType
+ ", privateFlag=0x" + Integer.toHexString(mPrivateFlags)
+ ", focused=" + mIsFocused
+ ", magnify=" + mShouldMagnify
+ ", ignoreDuetoRecentsAnimation=" + mIgnoreDuetoRecentsAnimation
+ ", mIsTrustedOverlay=" + mIsTrustedOverlay
+ ", regionInScreen=" + mTouchableRegionInScreen
+ ", touchableRegion=" + mTouchableRegionInWindow
+ ", letterBoxBounds=" + mLetterBoxBounds
+ ", windowInfo=" + mWindowInfo
+ "]";
return builder;
}
}
}