Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge changes from topic "SF_Solution" into tm-dev

Browse Source 
* changes:
  Apply display transform to window bounds and node bounds
  Use shellRoot to identify PIP menu window
  Uses SF callback windows for the A11yWindowInfo [2/n]
  Uses SF callback windows for the A11yWindowInfo [1/n]
This commit is contained in:
TreeHugger Robot
2022-02-24 15:32:36 +00:00
committed by Android (Google) Code Review
parent 554859be00 914cef5f32
commit 2a01b66100
5 changed files with 751 additions and 255 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
services/accessibility/java/com/android/server/accessibility/AccessibilityWindowManager.java
View File

@@ -667,6 +667,10 @@ public class AccessibilityWindowManager {
return null;
}
// Don't need to add the embedded hierarchy windows into the accessibility windows list.
if (mHostEmbeddedMap.size() > 0 && isEmbeddedHierarchyWindowsLocked(windowId)) {
return null;
}
final AccessibilityWindowInfo reportedWindow = AccessibilityWindowInfo.obtain();
reportedWindow.setId(windowId);
@@ -699,6 +703,21 @@ public class AccessibilityWindowManager {
return reportedWindow;
}
private boolean isEmbeddedHierarchyWindowsLocked(int windowId) {
final IBinder leashToken = mWindowIdMap.get(windowId);
if (leashToken == null) {
return false;
}
for (int i = 0; i < mHostEmbeddedMap.size(); i++) {
if (mHostEmbeddedMap.keyAt(i).equals(leashToken)) {
return true;
}
}
return false;
}
private int getTypeForWindowManagerWindowType(int windowType) {
switch (windowType) {
case WindowManager.LayoutParams.TYPE_APPLICATION:

307
services/core/java/com/android/server/wm/AccessibilityController.java
View File

@@ -45,7 +45,6 @@ import static com.android.server.accessibility.AccessibilityTraceProto.WINDOW_MA
import static com.android.server.wm.WindowManagerDebugConfig.TAG_WITH_CLASS_NAME;
import static com.android.server.wm.WindowManagerDebugConfig.TAG_WM;
import static com.android.server.wm.WindowTracing.WINSCOPE_EXT;
import static com.android.server.wm.utils.RegionUtils.forEachRect;
import android.accessibilityservice.AccessibilityTrace;
import android.animation.ObjectAnimator;
@@ -101,6 +100,7 @@ import com.android.internal.util.TraceBuffer;
import com.android.internal.util.function.pooled.PooledLambda;
import com.android.server.LocalServices;
import com.android.server.policy.WindowManagerPolicy;
import com.android.server.wm.AccessibilityWindowsPopulator.AccessibilityWindow;
import com.android.server.wm.WindowManagerInternal.AccessibilityControllerInternal;
import com.android.server.wm.WindowManagerInternal.MagnificationCallbacks;
import com.android.server.wm.WindowManagerInternal.WindowsForAccessibilityCallback;
@@ -133,19 +133,22 @@ final class AccessibilityController {
private static final Rect EMPTY_RECT = new Rect();
private static final float[] sTempFloats = new float[9];
private SparseArray<DisplayMagnifier> mDisplayMagnifiers = new SparseArray<>();
private SparseArray<WindowsForAccessibilityObserver> mWindowsForAccessibilityObserver =
private final SparseArray<DisplayMagnifier> mDisplayMagnifiers = new SparseArray<>();
private final SparseArray<WindowsForAccessibilityObserver> mWindowsForAccessibilityObserver =
new SparseArray<>();
private SparseArray<IBinder> mFocusedWindow = new SparseArray<>();
private int mFocusedDisplay = -1;
private boolean mIsImeVisible = false;
// Set to true if initializing window population complete.
private boolean mAllObserversInitialized = true;
private final AccessibilityWindowsPopulator mAccessibilityWindowsPopulator;
AccessibilityController(WindowManagerService service) {
mService = service;
mAccessibilityTracing =
AccessibilityController.getAccessibilityControllerInternal(service);
mAccessibilityWindowsPopulator = new AccessibilityWindowsPopulator(mService, this);
}
boolean setMagnificationCallbacks(int displayId, MagnificationCallbacks callbacks) {
@@ -209,7 +212,9 @@ final class AccessibilityController {
}
mWindowsForAccessibilityObserver.remove(displayId);
}
observer = new WindowsForAccessibilityObserver(mService, displayId, callback);
mAccessibilityWindowsPopulator.setWindowsNotification(true);
observer = new WindowsForAccessibilityObserver(mService, displayId, callback,
mAccessibilityWindowsPopulator);
mWindowsForAccessibilityObserver.put(displayId, observer);
mAllObserversInitialized &= observer.mInitialized;
} else {
@@ -224,6 +229,10 @@ final class AccessibilityController {
}
}
mWindowsForAccessibilityObserver.remove(displayId);
if (mWindowsForAccessibilityObserver.size() <= 0) {
mAccessibilityWindowsPopulator.setWindowsNotification(false);
}
}
}
@@ -309,11 +318,6 @@ final class AccessibilityController {
if (displayMagnifier != null) {
displayMagnifier.onDisplaySizeChanged(displayContent);
}
final WindowsForAccessibilityObserver windowsForA11yObserver =
mWindowsForAccessibilityObserver.get(displayId);
if (windowsForA11yObserver != null) {
windowsForA11yObserver.scheduleComputeChangedWindows();
}
}
void onAppWindowTransition(int displayId, int transition) {
@@ -341,11 +345,6 @@ final class AccessibilityController {
if (displayMagnifier != null) {
displayMagnifier.onWindowTransition(windowState, transition);
}
final WindowsForAccessibilityObserver windowsForA11yObserver =
mWindowsForAccessibilityObserver.get(displayId);
if (windowsForA11yObserver != null) {
windowsForA11yObserver.scheduleComputeChangedWindows();
}
}
void onWindowFocusChangedNot(int displayId) {
@@ -455,6 +454,19 @@ final class AccessibilityController {
return null;
}
boolean getMagnificationSpecForDisplay(int displayId, MagnificationSpec outSpec) {
if (mAccessibilityTracing.isTracingEnabled(FLAGS_MAGNIFICATION_CALLBACK)) {
mAccessibilityTracing.logTrace(TAG + ".getMagnificationSpecForDisplay",
FLAGS_MAGNIFICATION_CALLBACK, "displayId=" + displayId);
}
final DisplayMagnifier displayMagnifier = mDisplayMagnifiers.get(displayId);
if (displayMagnifier == null) {
return false;
}
return displayMagnifier.getMagnificationSpec(outSpec);
}
boolean hasCallbacks() {
if (mAccessibilityTracing.isTracingEnabled(FLAGS_MAGNIFICATION_CALLBACK
| FLAGS_WINDOWS_FOR_ACCESSIBILITY_CALLBACK)) {
@@ -756,6 +768,25 @@ final class AccessibilityController {
return spec;
}
boolean getMagnificationSpec(MagnificationSpec outSpec) {
if (mAccessibilityTracing.isTracingEnabled(FLAGS_MAGNIFICATION_CALLBACK)) {
mAccessibilityTracing.logTrace(LOG_TAG + ".getMagnificationSpec",
FLAGS_MAGNIFICATION_CALLBACK);
}
MagnificationSpec spec = mMagnifedViewport.getMagnificationSpec();
if (spec == null) {
return false;
}
outSpec.setTo(spec);
if (mAccessibilityTracing.isTracingEnabled(FLAGS_MAGNIFICATION_CALLBACK)) {
mAccessibilityTracing.logTrace(LOG_TAG + ".getMagnificationSpec",
FLAGS_MAGNIFICATION_CALLBACK, "outSpec={" + outSpec + "}");
}
return true;
}
void getMagnificationRegion(Region outMagnificationRegion) {
if (mAccessibilityTracing.isTracingEnabled(FLAGS_MAGNIFICATION_CALLBACK)) {
mAccessibilityTracing.logTrace(LOG_TAG + ".getMagnificationRegion",
@@ -1403,20 +1434,18 @@ final class AccessibilityController {
private static final boolean DEBUG = false;
private final SparseArray<WindowState> mTempWindowStates = new SparseArray<>();
private final List<AccessibilityWindow> mTempA11yWindows = new ArrayList<>();
private final Set<IBinder> mTempBinderSet = new ArraySet<>();
private final RectF mTempRectF = new RectF();
private final Matrix mTempMatrix = new Matrix();
private final Point mTempPoint = new Point();
private final Region mTempRegion = new Region();
private final Region mTempRegion1 = new Region();
private final Region mTempRegion2 = new Region();
private final WindowManagerService mService;
private final Handler mHandler;
@@ -1431,10 +1460,11 @@ final class AccessibilityController {
// Set to true if initializing window population complete.
private boolean mInitialized;
private final AccessibilityWindowsPopulator mA11yWindowsPopulator;
WindowsForAccessibilityObserver(WindowManagerService windowManagerService,
int displayId,
WindowsForAccessibilityCallback callback) {
int displayId, WindowsForAccessibilityCallback callback,
AccessibilityWindowsPopulator accessibilityWindowsPopulator) {
mService = windowManagerService;
mCallback = callback;
mDisplayId = displayId;
@@ -1443,6 +1473,7 @@ final class AccessibilityController {
AccessibilityController.getAccessibilityControllerInternal(mService);
mRecurringAccessibilityEventsIntervalMillis = ViewConfiguration
.getSendRecurringAccessibilityEventsInterval();
mA11yWindowsPopulator = accessibilityWindowsPopulator;
computeChangedWindows(true);
}
@@ -1466,52 +1497,6 @@ final class AccessibilityController {
}
}
boolean shellRootIsAbove(WindowState windowState, ShellRoot shellRoot) {
int wsLayer = mService.mPolicy.getWindowLayerLw(windowState);
int shellLayer = mService.mPolicy.getWindowLayerFromTypeLw(shellRoot.getWindowType(),
true);
return shellLayer >= wsLayer;
}
int addShellRootsIfAbove(WindowState windowState, ArrayList<ShellRoot> shellRoots,
int shellRootIndex, List<WindowInfo> windows, Set<IBinder> addedWindows,
Region unaccountedSpace, boolean focusedWindowAdded) {
while (shellRootIndex < shellRoots.size()
&& shellRootIsAbove(windowState, shellRoots.get(shellRootIndex))) {
ShellRoot shellRoot = shellRoots.get(shellRootIndex);
shellRootIndex++;
final WindowInfo info = shellRoot.getWindowInfo();
if (info == null) {
continue;
}
info.layer = addedWindows.size();
windows.add(info);
addedWindows.add(info.token);
unaccountedSpace.op(info.regionInScreen, unaccountedSpace,
Region.Op.REVERSE_DIFFERENCE);
if (unaccountedSpace.isEmpty() && focusedWindowAdded) {
break;
}
}
return shellRootIndex;
}
private ArrayList<ShellRoot> getSortedShellRoots(
SparseArray<ShellRoot> originalShellRoots) {
ArrayList<ShellRoot> sortedShellRoots = new ArrayList<>(originalShellRoots.size());
for (int i = originalShellRoots.size() - 1; i >= 0; --i) {
sortedShellRoots.add(originalShellRoots.valueAt(i));
}
sortedShellRoots.sort((left, right) ->
mService.mPolicy.getWindowLayerFromTypeLw(right.getWindowType(), true)
- mService.mPolicy.getWindowLayerFromTypeLw(left.getWindowType(),
true));
return sortedShellRoots;
}
/**
* Check if windows have changed, and send them to the accessibility subsystem if they have.
*
@@ -1561,44 +1546,29 @@ final class AccessibilityController {
Region unaccountedSpace = mTempRegion;
unaccountedSpace.set(0, 0, screenWidth, screenHeight);
final SparseArray<WindowState> visibleWindows = mTempWindowStates;
populateVisibleWindowsOnScreen(visibleWindows);
final List<AccessibilityWindow> visibleWindows = mTempA11yWindows;
mA11yWindowsPopulator.populateVisibleWindowsOnScreenLocked(
mDisplayId, visibleWindows);
Set<IBinder> addedWindows = mTempBinderSet;
addedWindows.clear();
boolean focusedWindowAdded = false;
final int visibleWindowCount = visibleWindows.size();
ArrayList<TaskFragment> skipRemainingWindowsForTaskFragments = new ArrayList<>();
ArrayList<ShellRoot> shellRoots = getSortedShellRoots(dc.mShellRoots);
// Iterate until we figure out what is touchable for the entire screen.
int shellRootIndex = 0;
for (int i = visibleWindowCount - 1; i >= 0; i--) {
final WindowState windowState = visibleWindows.valueAt(i);
int prevShellRootIndex = shellRootIndex;
shellRootIndex = addShellRootsIfAbove(windowState, shellRoots, shellRootIndex,
windows, addedWindows, unaccountedSpace, focusedWindowAdded);
// If a Shell Root was added, it could have accounted for all the space already.
if (shellRootIndex > prevShellRootIndex && unaccountedSpace.isEmpty()
&& focusedWindowAdded) {
break;
}
final Region regionInScreen = new Region();
computeWindowRegionInScreen(windowState, regionInScreen);
if (windowMattersToAccessibility(windowState,
regionInScreen, unaccountedSpace,
skipRemainingWindowsForTaskFragments)) {
addPopulatedWindowInfo(windowState, regionInScreen, windows, addedWindows);
if (windowMattersToUnaccountedSpaceComputation(windowState)) {
updateUnaccountedSpace(windowState, regionInScreen, unaccountedSpace,
skipRemainingWindowsForTaskFragments);
for (int i = 0; i < visibleWindowCount; i++) {
final AccessibilityWindow a11yWindow = visibleWindows.get(i);
final Region regionInWindow = new Region();
a11yWindow.getTouchableRegionInWindow(regionInWindow);
if (windowMattersToAccessibility(a11yWindow, regionInWindow,
unaccountedSpace)) {
addPopulatedWindowInfo(a11yWindow, regionInWindow, windows, addedWindows);
if (windowMattersToUnaccountedSpaceComputation(a11yWindow)) {
updateUnaccountedSpace(a11yWindow, unaccountedSpace);
}
focusedWindowAdded |= windowState.isFocused();
} else if (isUntouchableNavigationBar(windowState, mTempRegion1)) {
focusedWindowAdded |= a11yWindow.isFocused();
} else if (a11yWindow.isUntouchableNavigationBar()) {
// If this widow is navigation bar without touchable region, accounting the
// region of navigation bar inset because all touch events from this region
// would be received by launcher, i.e. this region is a un-touchable one
@@ -1647,47 +1617,39 @@ final class AccessibilityController {
// Some windows should be excluded from unaccounted space computation, though they still
// should be reported
private boolean windowMattersToUnaccountedSpaceComputation(WindowState windowState) {
private boolean windowMattersToUnaccountedSpaceComputation(AccessibilityWindow a11yWindow) {
// Do not account space of trusted non-touchable windows, except the split-screen
// divider.
// If it's not trusted, touch events are not sent to the windows behind it.
if (((windowState.mAttrs.flags & WindowManager.LayoutParams.FLAG_NOT_TOUCHABLE) != 0)
&& (windowState.mAttrs.type != TYPE_DOCK_DIVIDER)
&& windowState.isTrustedOverlay()) {
if (((a11yWindow.getFlags() & WindowManager.LayoutParams.FLAG_NOT_TOUCHABLE) != 0)
&& (a11yWindow.getType() != TYPE_DOCK_DIVIDER)
&& a11yWindow.isTrustedOverlay()) {
return false;
}
if (windowState.mAttrs.type
== WindowManager.LayoutParams.TYPE_ACCESSIBILITY_OVERLAY) {
if (a11yWindow.getType() == WindowManager.LayoutParams.TYPE_ACCESSIBILITY_OVERLAY) {
return false;
}
return true;
}
private boolean windowMattersToAccessibility(WindowState windowState,
Region regionInScreen, Region unaccountedSpace,
ArrayList<TaskFragment> skipRemainingWindowsForTaskFragments) {
final RecentsAnimationController controller = mService.getRecentsAnimationController();
if (controller != null && controller.shouldIgnoreForAccessibility(windowState)) {
private boolean windowMattersToAccessibility(AccessibilityWindow a11yWindow,
Region regionInScreen, Region unaccountedSpace) {
if (a11yWindow.ignoreRecentsAnimationForAccessibility()) {
return false;
}
if (windowState.isFocused()) {
if (a11yWindow.isFocused()) {
return true;
}
// If the window is part of a task that we're finished with - ignore.
final TaskFragment taskFragment = windowState.getTaskFragment();
if (taskFragment != null
&& skipRemainingWindowsForTaskFragments.contains(taskFragment)) {
return false;
}
// Ignore non-touchable windows, except the split-screen divider, which is
// occasionally non-touchable but still useful for identifying split-screen
// mode.
if (((windowState.mAttrs.flags & WindowManager.LayoutParams.FLAG_NOT_TOUCHABLE) != 0)
&& (windowState.mAttrs.type != TYPE_DOCK_DIVIDER)) {
// mode and the PIP menu.
if (((a11yWindow.getFlags()
& WindowManager.LayoutParams.FLAG_NOT_TOUCHABLE) != 0)
&& (a11yWindow.getType() != TYPE_DOCK_DIVIDER
&& !a11yWindow.isPIPMenu())) {
return false;
}
@@ -1697,88 +1659,36 @@ final class AccessibilityController {
}
// Add windows of certain types not covered by modal windows.
if (isReportedWindowType(windowState.mAttrs.type)) {
if (isReportedWindowType(a11yWindow.getType())) {
return true;
}
return false;
}
private void updateUnaccountedSpace(WindowState windowState, Region regionInScreen,
Region unaccountedSpace,
ArrayList<TaskFragment> skipRemainingWindowsForTaskFragments) {
// Account for the space this window takes if the window
// is not an accessibility overlay which does not change
// the reported windows.
unaccountedSpace.op(regionInScreen, unaccountedSpace,
Region.Op.REVERSE_DIFFERENCE);
// If a window is modal it prevents other windows from being touched
if ((windowState.mAttrs.flags & (WindowManager.LayoutParams.FLAG_NOT_FOCUSABLE
| WindowManager.LayoutParams.FLAG_NOT_TOUCH_MODAL)) == 0) {
if (!windowState.hasTapExcludeRegion()) {
// Account for all space in the task, whether the windows in it are
// touchable or not. The modal window blocks all touches from the task's
// area.
unaccountedSpace.op(windowState.getDisplayFrame(), unaccountedSpace,
Region.Op.REVERSE_DIFFERENCE);
} else {
// If a window has tap exclude region, we need to account it.
final Region displayRegion = new Region(windowState.getDisplayFrame());
final Region tapExcludeRegion = new Region();
windowState.getTapExcludeRegion(tapExcludeRegion);
displayRegion.op(tapExcludeRegion, displayRegion,
Region.Op.REVERSE_DIFFERENCE);
unaccountedSpace.op(displayRegion, unaccountedSpace,
Region.Op.REVERSE_DIFFERENCE);
}
final TaskFragment taskFragment = windowState.getTaskFragment();
if (taskFragment != null) {
// If the window is associated with a particular task, we can skip the
// rest of the windows for that task.
skipRemainingWindowsForTaskFragments.add(taskFragment);
} else if (!windowState.hasTapExcludeRegion()) {
// If the window is not associated with a particular task, then it is
// globally modal. In this case we can skip all remaining windows when
// it doesn't has tap exclude region.
unaccountedSpace.setEmpty();
}
}
// Account for the space of letterbox.
if (windowState.areAppWindowBoundsLetterboxed()) {
unaccountedSpace.op(getLetterboxBounds(windowState), unaccountedSpace,
private void updateUnaccountedSpace(AccessibilityWindow a11yWindow,
Region unaccountedSpace) {
if (a11yWindow.getType()
!= WindowManager.LayoutParams.TYPE_ACCESSIBILITY_OVERLAY) {
// Account for the space this window takes if the window
// is not an accessibility overlay which does not change
// the reported windows.
final Region touchableRegion = mTempRegion2;
a11yWindow.getTouchableRegionInScreen(touchableRegion);
unaccountedSpace.op(touchableRegion, unaccountedSpace,
Region.Op.REVERSE_DIFFERENCE);
// Account for the space of letterbox.
final Region letterboxBounds = mTempRegion1;
if (a11yWindow.setLetterBoxBoundsIfNeeded(letterboxBounds)) {
unaccountedSpace.op(letterboxBounds,
unaccountedSpace, Region.Op.REVERSE_DIFFERENCE);
}
}
}
private void computeWindowRegionInScreen(WindowState windowState, Region outRegion) {
// Get the touchable frame.
Region touchableRegion = mTempRegion1;
windowState.getTouchableRegion(touchableRegion);
// Map the frame to get what appears on the screen.
Matrix matrix = mTempMatrix;
populateTransformationMatrix(windowState, matrix);
forEachRect(touchableRegion, rect -> {
// Move to origin as all transforms are captured by the matrix.
RectF windowFrame = mTempRectF;
windowFrame.set(rect);
windowFrame.offset(-windowState.getFrame().left, -windowState.getFrame().top);
matrix.mapRect(windowFrame);
// Union all rects.
outRegion.union(new Rect((int) windowFrame.left, (int) windowFrame.top,
(int) windowFrame.right, (int) windowFrame.bottom));
});
}
private static void addPopulatedWindowInfo(WindowState windowState, Region regionInScreen,
List<WindowInfo> out, Set<IBinder> tokenOut) {
final WindowInfo window = windowState.getWindowInfo();
private static void addPopulatedWindowInfo(AccessibilityWindow a11yWindow,
Region regionInScreen, List<WindowInfo> out, Set<IBinder> tokenOut) {
final WindowInfo window = a11yWindow.getWindowInfo();
window.regionInScreen.set(regionInScreen);
window.layer = tokenOut.size();
out.add(window);
@@ -1805,23 +1715,6 @@ final class AccessibilityController {
&& windowType != WindowManager.LayoutParams.TYPE_PRIVATE_PRESENTATION);
}
private void populateVisibleWindowsOnScreen(SparseArray<WindowState> outWindows) {
final List<WindowState> tempWindowStatesList = new ArrayList<>();
final DisplayContent dc = mService.mRoot.getDisplayContent(mDisplayId);
if (dc == null) {
return;
}
dc.forAllWindows(w -> {
if (w.isVisible()) {
tempWindowStatesList.add(w);
}
}, false /* traverseTopToBottom */);
for (int i = 0; i < tempWindowStatesList.size(); i++) {
outWindows.put(i, tempWindowStatesList.get(i));
}
}
private WindowState getTopFocusWindow() {
return mService.mRoot.getTopFocusedDisplayContent().mCurrentFocus;
}

626
services/core/java/com/android/server/wm/AccessibilityWindowsPopulator.java Normal file
View File

@@ -0,0 +1,626 @@
/*
* Copyright (C) 2021 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.server.wm;
import static android.view.WindowManager.LayoutParams.TYPE_DOCK_DIVIDER;
import static com.android.server.wm.utils.RegionUtils.forEachRect;
import android.annotation.NonNull;
import android.graphics.Matrix;
import android.graphics.Rect;
import android.graphics.RectF;
import android.graphics.Region;
import android.os.Handler;
import android.os.IBinder;
import android.os.Looper;
import android.os.Message;
import android.util.Slog;
import android.util.SparseArray;
import android.view.IWindow;
import android.view.InputWindowHandle;
import android.view.MagnificationSpec;
import android.view.WindowInfo;
import android.view.WindowManager;
import android.window.WindowInfosListener;
import com.android.internal.annotations.GuardedBy;
import java.util.ArrayList;
import java.util.List;
/**
* This class is the accessibility windows population adapter.
*/
public final class AccessibilityWindowsPopulator extends WindowInfosListener {
private static final String TAG = AccessibilityWindowsPopulator.class.getSimpleName();
// If the surface flinger callback is not coming within in 2 frames time, i.e. about
// 35ms, then assuming the windows become stable.
private static final int SURFACE_FLINGER_CALLBACK_WINDOWS_STABLE_TIMES_MS = 35;
// To avoid the surface flinger callbacks always comes within in 2 frames, then no windows
// are reported to the A11y framework, and the animation duration time is 500ms, so setting
// this value as the max timeout value to force computing changed windows.
private static final int WINDOWS_CHANGED_NOTIFICATION_MAX_DURATION_TIMES_MS = 500;
private static final float[] sTempFloats = new float[9];
private final WindowManagerService mService;
private final AccessibilityController mAccessibilityController;
@GuardedBy("mLock")
private final SparseArray<List<InputWindowHandle>> mInputWindowHandlesOnDisplays =
new SparseArray<>();
@GuardedBy("mLock")
private final SparseArray<Matrix> mMagnificationSpecInverseMatrix = new SparseArray<>();
@GuardedBy("mLock")
private final SparseArray<DisplayInfo> mDisplayInfos = new SparseArray<>();
@GuardedBy("mLock")
private final List<InputWindowHandle> mVisibleWindows = new ArrayList<>();
@GuardedBy("mLock")
private boolean mWindowsNotificationEnabled = false;
private final Object mLock = new Object();
private final Handler mHandler;
AccessibilityWindowsPopulator(WindowManagerService service,
AccessibilityController accessibilityController) {
mService = service;
mAccessibilityController = accessibilityController;
mHandler = new MyHandler(mService.mH.getLooper());
register();
}
/**
* Gets the visible windows list with the window layer on the specified display.
*
* @param displayId The display.
* @param outWindows The visible windows list. The z-order of each window in the list
* is from the top to bottom.
*/
public void populateVisibleWindowsOnScreenLocked(int displayId,
List<AccessibilityWindow> outWindows) {
List<InputWindowHandle> inputWindowHandles;
final Matrix inverseMatrix = new Matrix();
final Matrix displayMatrix = new Matrix();
synchronized (mLock) {
inputWindowHandles = mInputWindowHandlesOnDisplays.get(displayId);
if (inputWindowHandles == null) {
outWindows.clear();
return;
}
inverseMatrix.set(mMagnificationSpecInverseMatrix.get(displayId));
final DisplayInfo displayInfo = mDisplayInfos.get(displayId);
if (displayInfo != null) {
displayMatrix.set(displayInfo.mTransform);
} else {
Slog.w(TAG, "The displayInfo of this displayId (" + displayId + ") called "
+ "back from the surface fligner is null");
}
}
final DisplayContent dc = mService.mRoot.getDisplayContent(displayId);
final ShellRoot shellroot = dc.mShellRoots.get(WindowManager.SHELL_ROOT_LAYER_PIP);
final IBinder pipMenuIBinder =
shellroot != null ? shellroot.getAccessibilityWindowToken() : null;
for (final InputWindowHandle windowHandle : inputWindowHandles) {
final AccessibilityWindow accessibilityWindow =
AccessibilityWindow.initializeData(mService, windowHandle, inverseMatrix,
pipMenuIBinder, displayMatrix);
outWindows.add(accessibilityWindow);
}
}
@Override
public void onWindowInfosChanged(InputWindowHandle[] windowHandles,
DisplayInfo[] displayInfos) {
synchronized (mLock) {
mVisibleWindows.clear();
for (InputWindowHandle window : windowHandles) {
if (window.visible && window.getWindow() != null) {
mVisibleWindows.add(window);
}
}
mDisplayInfos.clear();
for (final DisplayInfo displayInfo : displayInfos) {
mDisplayInfos.put(displayInfo.mDisplayId, displayInfo);
}
if (mWindowsNotificationEnabled) {
if (!mHandler.hasMessages(
MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_TIMEOUT)) {
mHandler.sendEmptyMessageDelayed(
MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_TIMEOUT,
WINDOWS_CHANGED_NOTIFICATION_MAX_DURATION_TIMES_MS);
}
populateVisibleWindowHandlesAndNotifyWindowsChangeIfNeededLocked();
}
}
}
/**
* Sets to notify the accessibilityController to compute changed windows on
* the display after populating the visible windows if the windows reported
* from the surface flinger changes.
*
* @param register {@code true} means starting windows population.
*/
public void setWindowsNotification(boolean register) {
synchronized (mLock) {
if (mWindowsNotificationEnabled == register) {
return;
}
mWindowsNotificationEnabled = register;
if (mWindowsNotificationEnabled) {
populateVisibleWindowHandlesAndNotifyWindowsChangeIfNeededLocked();
} else {
releaseResources();
}
}
}
private void populateVisibleWindowHandlesAndNotifyWindowsChangeIfNeededLocked() {
final SparseArray<List<InputWindowHandle>> tempWindowHandleList = new SparseArray<>();
for (final InputWindowHandle windowHandle : mVisibleWindows) {
List<InputWindowHandle> inputWindowHandles = tempWindowHandleList.get(
windowHandle.displayId);
if (inputWindowHandles == null) {
inputWindowHandles = new ArrayList<>();
tempWindowHandleList.put(windowHandle.displayId, inputWindowHandles);
generateMagnificationSpecInverseMatrixLocked(windowHandle.displayId);
}
inputWindowHandles.add(windowHandle);
}
final List<Integer> displayIdsForWindowsChanged = new ArrayList<>();
getDisplaysForWindowsChangedLocked(displayIdsForWindowsChanged, tempWindowHandleList,
mInputWindowHandlesOnDisplays);
// Clones all windows from the callback of the surface flinger.
mInputWindowHandlesOnDisplays.clear();
for (int i = 0; i < tempWindowHandleList.size(); i++) {
final int displayId = tempWindowHandleList.keyAt(i);
mInputWindowHandlesOnDisplays.put(displayId, tempWindowHandleList.get(displayId));
}
if (displayIdsForWindowsChanged.size() > 0) {
if (!mHandler.hasMessages(MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED)) {
mHandler.obtainMessage(MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED,
displayIdsForWindowsChanged).sendToTarget();
}
return;
}
mHandler.removeMessages(MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_UI_STABLE);
mHandler.sendEmptyMessageDelayed(MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_UI_STABLE,
SURFACE_FLINGER_CALLBACK_WINDOWS_STABLE_TIMES_MS);
}
private void getDisplaysForWindowsChangedLocked(List<Integer> outDisplayIdsForWindowsChanged,
SparseArray<List<InputWindowHandle>> newWindowsList,
SparseArray<List<InputWindowHandle>> oldWindowsList) {
for (int i = 0; i < newWindowsList.size(); i++) {
final int displayId = newWindowsList.keyAt(i);
final List<InputWindowHandle> newWindows = newWindowsList.get(displayId);
final List<InputWindowHandle> oldWindows = oldWindowsList.get(displayId);
if (hasWindowsChangedLocked(newWindows, oldWindows)) {
outDisplayIdsForWindowsChanged.add(displayId);
}
}
}
private boolean hasWindowsChangedLocked(List<InputWindowHandle> newWindows,
List<InputWindowHandle> oldWindows) {
if (oldWindows == null || oldWindows.size() != newWindows.size()) {
return true;
}
final int windowsCount = newWindows.size();
// Since we always traverse windows from high to low layer,
// the old and new windows at the same index should be the
// same, otherwise something changed.
for (int i = 0; i < windowsCount; i++) {
final InputWindowHandle newWindow = newWindows.get(i);
final InputWindowHandle oldWindow = oldWindows.get(i);
if (!newWindow.getWindow().asBinder().equals(oldWindow.getWindow().asBinder())) {
return true;
}
}
return false;
}
private void generateMagnificationSpecInverseMatrixLocked(int displayId) {
MagnificationSpec spec = new MagnificationSpec();
if (!mAccessibilityController.getMagnificationSpecForDisplay(displayId, spec)) {
mMagnificationSpecInverseMatrix.remove(displayId);
return;
}
sTempFloats[Matrix.MSCALE_X] = spec.scale;
sTempFloats[Matrix.MSKEW_Y] = 0;
sTempFloats[Matrix.MSKEW_X] = 0;
sTempFloats[Matrix.MSCALE_Y] = spec.scale;
sTempFloats[Matrix.MTRANS_X] = spec.offsetX;
sTempFloats[Matrix.MTRANS_Y] = spec.offsetY;
sTempFloats[Matrix.MPERSP_0] = 0;
sTempFloats[Matrix.MPERSP_1] = 0;
sTempFloats[Matrix.MPERSP_2] = 1;
final Matrix tempMatrix = new Matrix();
tempMatrix.setValues(sTempFloats);
final Matrix inverseMatrix = new Matrix();
final boolean result = tempMatrix.invert(inverseMatrix);
if (!result) {
Slog.e(TAG, "Can't inverse the magnification spec matrix with the "
+ "magnification spec = " + spec + " on the displayId = " + displayId);
return;
}
mMagnificationSpecInverseMatrix.set(displayId, inverseMatrix);
}
private void notifyWindowsChanged(@NonNull List<Integer> displayIdsForWindowsChanged) {
mHandler.removeMessages(MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_TIMEOUT);
for (int i = 0; i < displayIdsForWindowsChanged.size(); i++) {
mAccessibilityController.performComputeChangedWindowsNot(
displayIdsForWindowsChanged.get(i), false);
}
}
private void forceUpdateWindows() {
final List<Integer> displayIdsForWindowsChanged = new ArrayList<>();
synchronized (mLock) {
for (int i = 0; i < mInputWindowHandlesOnDisplays.size(); i++) {
final int displayId = mInputWindowHandlesOnDisplays.keyAt(i);
displayIdsForWindowsChanged.add(displayId);
}
}
notifyWindowsChanged(displayIdsForWindowsChanged);
}
@GuardedBy("mLock")
private void releaseResources() {
mInputWindowHandlesOnDisplays.clear();
mMagnificationSpecInverseMatrix.clear();
mVisibleWindows.clear();
mDisplayInfos.clear();
mWindowsNotificationEnabled = false;
mHandler.removeCallbacksAndMessages(null);
}
private class MyHandler extends Handler {
public static final int MESSAGE_NOTIFY_WINDOWS_CHANGED = 1;
public static final int MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_UI_STABLE = 2;
public static final int MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_TIMEOUT = 3;
MyHandler(Looper looper) {
super(looper, null, false);
}
@Override
public void handleMessage(Message message) {
switch (message.what) {
case MESSAGE_NOTIFY_WINDOWS_CHANGED: {
final List<Integer> displayIdsForWindowsChanged = (List<Integer>) message.obj;
notifyWindowsChanged(displayIdsForWindowsChanged);
} break;
case MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_UI_STABLE: {
forceUpdateWindows();
} break;
case MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_TIMEOUT: {
Slog.w(TAG, "Windows change within in 2 frames continuously over 500 ms "
+ "and notify windows changed immediately");
mHandler.removeMessages(
MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_UI_STABLE);
forceUpdateWindows();
} break;
}
}
}
/**
* This class represents information about a window from the
* surface flinger to the accessibility framework.
*/
public static class AccessibilityWindow {
private static final Region TEMP_REGION = new Region();
private static final RectF TEMP_RECTF = new RectF();
// Data
private IWindow mWindow;
private int mDisplayId;
private int mFlags;
private int mType;
private int mPrivateFlags;
private boolean mIsPIPMenu;
private boolean mIsFocused;
private boolean mShouldMagnify;
private boolean mIgnoreDuetoRecentsAnimation;
private boolean mIsTrustedOverlay;
private final Region mTouchableRegionInScreen = new Region();
private final Region mTouchableRegionInWindow = new Region();
private final Region mLetterBoxBounds = new Region();
private WindowInfo mWindowInfo;
/**
* Returns the instance after initializing the internal data.
* @param service The window manager service.
* @param inputWindowHandle The window from the surface flinger.
* @param inverseMatrix The magnification spec inverse matrix.
*/
public static AccessibilityWindow initializeData(WindowManagerService service,
InputWindowHandle inputWindowHandle, Matrix inverseMatrix, IBinder pipIBinder,
Matrix displayMatrix) {
final IWindow window = inputWindowHandle.getWindow();
final WindowState windowState = window != null ? service.mWindowMap.get(
window.asBinder()) : null;
final AccessibilityWindow instance = new AccessibilityWindow();
instance.mWindow = inputWindowHandle.getWindow();
instance.mDisplayId = inputWindowHandle.displayId;
instance.mFlags = inputWindowHandle.layoutParamsFlags;
instance.mType = inputWindowHandle.layoutParamsType;
instance.mIsPIPMenu = inputWindowHandle.getWindow().asBinder().equals(pipIBinder);
// TODO (b/199357848): gets the private flag of the window from other way.
instance.mPrivateFlags = windowState != null ? windowState.mAttrs.privateFlags : 0;
// TODO (b/199358208) : using new way to implement the focused window.
instance.mIsFocused = windowState != null && windowState.isFocused();
instance.mShouldMagnify = windowState == null || windowState.shouldMagnify();
final RecentsAnimationController controller = service.getRecentsAnimationController();
instance.mIgnoreDuetoRecentsAnimation = windowState != null && controller != null
&& controller.shouldIgnoreForAccessibility(windowState);
instance.mIsTrustedOverlay = inputWindowHandle.trustedOverlay;
// TODO (b/199358388) : gets the letterbox bounds of the window from other way.
if (windowState != null && windowState.areAppWindowBoundsLetterboxed()) {
getLetterBoxBounds(windowState, instance.mLetterBoxBounds);
}
final Rect windowFrame = new Rect(inputWindowHandle.frameLeft,
inputWindowHandle.frameTop, inputWindowHandle.frameRight,
inputWindowHandle.frameBottom);
getTouchableRegionInWindow(instance.mShouldMagnify, inputWindowHandle.touchableRegion,
instance.mTouchableRegionInWindow, windowFrame, inverseMatrix, displayMatrix);
getUnMagnifiedTouchableRegion(instance.mShouldMagnify,
inputWindowHandle.touchableRegion, instance.mTouchableRegionInScreen,
inverseMatrix, displayMatrix);
instance.mWindowInfo = windowState != null
? windowState.getWindowInfo() : getWindowInfoForWindowlessWindows(instance);
return instance;
}
/**
* Returns the touchable region in the screen.
* @param outRegion The touchable region.
*/
public void getTouchableRegionInScreen(Region outRegion) {
outRegion.set(mTouchableRegionInScreen);
}
/**
* Returns the touchable region in the window.
* @param outRegion The touchable region.
*/
public void getTouchableRegionInWindow(Region outRegion) {
outRegion.set(mTouchableRegionInWindow);
}
/**
* @return the layout parameter flag {@link android.view.WindowManager.LayoutParams#flags}.
*/
public int getFlags() {
return mFlags;
}
/**
* @return the layout parameter type {@link android.view.WindowManager.LayoutParams#type}.
*/
public int getType() {
return mType;
}
/**
* @return the layout parameter private flag
* {@link android.view.WindowManager.LayoutParams#privateFlags}.
*/
public int getPrivateFlag() {
return mPrivateFlags;
}
/**
* @return the windowInfo {@link WindowInfo}.
*/
public WindowInfo getWindowInfo() {
return mWindowInfo;
}
/**
* Gets the letter box bounds if activity bounds are letterboxed
* or letterboxed for display cutout.
*
* @return {@code true} there's a letter box bounds.
*/
public Boolean setLetterBoxBoundsIfNeeded(Region outBounds) {
if (mLetterBoxBounds.isEmpty()) {
return false;
}
outBounds.set(mLetterBoxBounds);
return true;
}
/**
* @return true if this window should be magnified.
*/
public boolean shouldMagnify() {
return mShouldMagnify;
}
/**
* @return true if this window is focused.
*/
public boolean isFocused() {
return mIsFocused;
}
/**
* @return true if it's running the recent animation but not the target app.
*/
public boolean ignoreRecentsAnimationForAccessibility() {
return mIgnoreDuetoRecentsAnimation;
}
/**
* @return true if this window is the trusted overlay.
*/
public boolean isTrustedOverlay() {
return mIsTrustedOverlay;
}
/**
* @return true if this window is the navigation bar with the gesture mode.
*/
public boolean isUntouchableNavigationBar() {
if (mType != WindowManager.LayoutParams.TYPE_NAVIGATION_BAR) {
return false;
}
return mTouchableRegionInScreen.isEmpty();
}
/**
* @return true if this window is PIP menu.
*/
public boolean isPIPMenu() {
return mIsPIPMenu;
}
private static void getTouchableRegionInWindow(boolean shouldMagnify, Region inRegion,
Region outRegion, Rect frame, Matrix inverseMatrix, Matrix displayMatrix) {
// Some modal windows, like the activity with Theme.dialog, has the full screen
// as its touchable region, but its window frame is smaller than the touchable
// region. The region we report should be the touchable area in the window frame
// for the consistency and match developers expectation.
// So we need to make the intersection between the frame and touchable region to
// obtain the real touch region in the screen.
Region touchRegion = TEMP_REGION;
touchRegion.set(inRegion);
touchRegion.op(frame, Region.Op.INTERSECT);
getUnMagnifiedTouchableRegion(shouldMagnify, touchRegion, outRegion, inverseMatrix,
displayMatrix);
}
/**
* Gets the un-magnified touchable region. If this window can be magnified and magnifying,
* we will transform the input touchable region by applying the inverse matrix of the
* magnification spec to get the un-magnified touchable region.
* @param shouldMagnify The window can be magnified.
* @param inRegion The touchable region of this window.
* @param outRegion The un-magnified touchable region of this window.
* @param inverseMatrix The inverse matrix of the magnification spec.
* @param displayMatrix The display transform matrix which takes display coordinates to
* logical display coordinates.
*/
private static void getUnMagnifiedTouchableRegion(boolean shouldMagnify, Region inRegion,
Region outRegion, Matrix inverseMatrix, Matrix displayMatrix) {
if ((!shouldMagnify || inverseMatrix.isIdentity()) && displayMatrix.isIdentity()) {
outRegion.set(inRegion);
return;
}
forEachRect(inRegion, rect -> {
// Move to origin as all transforms are captured by the matrix.
RectF windowFrame = TEMP_RECTF;
windowFrame.set(rect);
inverseMatrix.mapRect(windowFrame);
displayMatrix.mapRect(windowFrame);
// Union all rects.
outRegion.union(new Rect((int) windowFrame.left, (int) windowFrame.top,
(int) windowFrame.right, (int) windowFrame.bottom));
});
}
private static WindowInfo getWindowInfoForWindowlessWindows(AccessibilityWindow window) {
WindowInfo windowInfo = WindowInfo.obtain();
windowInfo.displayId = window.mDisplayId;
windowInfo.type = window.mType;
windowInfo.token = window.mWindow.asBinder();
windowInfo.hasFlagWatchOutsideTouch = (window.mFlags
& WindowManager.LayoutParams.FLAG_WATCH_OUTSIDE_TOUCH) != 0;
windowInfo.inPictureInPicture = false;
// There only are two windowless windows now, one is split window, and the other
// one is PIP.
if (windowInfo.type == TYPE_DOCK_DIVIDER) {
windowInfo.title = "Splitscreen Divider";
} else if (window.mIsPIPMenu) {
windowInfo.title = "Picture-in-Picture menu";
}
return windowInfo;
}
private static void getLetterBoxBounds(WindowState windowState, Region outRegion) {
final Rect letterboxInsets = windowState.mActivityRecord.getLetterboxInsets();
final Rect nonLetterboxRect = windowState.getBounds();
nonLetterboxRect.inset(letterboxInsets);
outRegion.set(windowState.getBounds());
outRegion.op(nonLetterboxRect, Region.Op.DIFFERENCE);
}
@Override
public String toString() {
String builder = "A11yWindow=[" + mWindow.asBinder()
+ ", displayId=" + mDisplayId
+ ", flag=0x" + Integer.toHexString(mFlags)
+ ", type=" + mType
+ ", privateFlag=0x" + Integer.toHexString(mPrivateFlags)
+ ", focused=" + mIsFocused
+ ", shouldMagnify=" + mShouldMagnify
+ ", ignoreDuetoRecentsAnimation=" + mIgnoreDuetoRecentsAnimation
+ ", isTrustedOverlay=" + mIsTrustedOverlay
+ ", regionInScreen=" + mTouchableRegionInScreen
+ ", touchableRegion=" + mTouchableRegionInWindow
+ ", letterBoxBounds=" + mLetterBoxBounds
+ ", isPIPMenu=" + mIsPIPMenu
+ ", windowInfo=" + mWindowInfo
+ "]";
return builder;
}
}
}

51
services/core/java/com/android/server/wm/ShellRoot.java
View File

@@ -25,15 +25,14 @@ import static com.android.server.wm.SurfaceAnimator.ANIMATION_TYPE_WINDOW_ANIMAT
import static com.android.server.wm.WindowManagerService.MAX_ANIMATION_DURATION;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.graphics.Point;
import android.graphics.Rect;
import android.os.IBinder;
import android.os.RemoteException;
import android.util.Slog;
import android.view.DisplayInfo;
import android.view.IWindow;
import android.view.SurfaceControl;
import android.view.WindowInfo;
import android.view.WindowManager;
import android.view.animation.Animation;
@@ -136,46 +135,12 @@ public class ShellRoot {
ANIMATION_TYPE_WINDOW_ANIMATION);
}
WindowInfo getWindowInfo() {
if (mShellRootLayer != SHELL_ROOT_LAYER_DIVIDER
&& mShellRootLayer != SHELL_ROOT_LAYER_PIP) {
return null;
@Nullable
IBinder getAccessibilityWindowToken() {
if (mAccessibilityWindow != null) {
return mAccessibilityWindow.asBinder();
}
if (mShellRootLayer == SHELL_ROOT_LAYER_DIVIDER) {
return null;
}
if (mShellRootLayer == SHELL_ROOT_LAYER_PIP
&& mDisplayContent.getDefaultTaskDisplayArea().getRootPinnedTask() == null) {
return null;
}
if (mAccessibilityWindow == null) {
return null;
}
WindowInfo windowInfo = WindowInfo.obtain();
windowInfo.displayId = mToken.getDisplayArea().getDisplayContent().mDisplayId;
windowInfo.type = mToken.windowType;
windowInfo.layer = mToken.getWindowLayerFromType();
windowInfo.token = mAccessibilityWindow.asBinder();
windowInfo.focused = false;
windowInfo.hasFlagWatchOutsideTouch = false;
final Rect regionRect = new Rect();
// DividerView
if (mShellRootLayer == SHELL_ROOT_LAYER_DIVIDER) {
windowInfo.inPictureInPicture = false;
mDisplayContent.getDockedDividerController().getTouchRegion(regionRect);
windowInfo.regionInScreen.set(regionRect);
windowInfo.title = "Splitscreen Divider";
}
// PipMenuView
if (mShellRootLayer == SHELL_ROOT_LAYER_PIP) {
windowInfo.inPictureInPicture = true;
mDisplayContent.getDefaultTaskDisplayArea().getRootPinnedTask().getBounds(regionRect);
windowInfo.regionInScreen.set(regionRect);
windowInfo.title = "Picture-in-Picture menu";
}
return windowInfo;
return null;
}
void setAccessibilityWindow(IWindow window) {
@@ -196,9 +161,5 @@ public class ShellRoot {
mAccessibilityWindow = null;
}
}
if (mDisplayContent.mWmService.mAccessibilityController.hasCallbacks()) {
mDisplayContent.mWmService.mAccessibilityController.onSomeWindowResizedOrMoved(
mDisplayContent.getDisplayId());
}
}
}

3
services/core/java/com/android/server/wm/WindowState.java
View File

@@ -4992,9 +4992,6 @@ class WindowState extends WindowContainer<WindowState> implements WindowManagerP
if (isAnimating()) {
return;
}
if (mWmService.mAccessibilityController.hasCallbacks()) {
mWmService.mAccessibilityController.onSomeWindowResizedOrMoved(getDisplayId());
}
if (!isSelfOrAncestorWindowAnimatingExit()) {
return;