diff --git a/services/accessibility/java/com/android/server/accessibility/AccessibilityWindowManager.java b/services/accessibility/java/com/android/server/accessibility/AccessibilityWindowManager.java index 59c1461e4abca..aba32ec465b56 100644 --- a/services/accessibility/java/com/android/server/accessibility/AccessibilityWindowManager.java +++ b/services/accessibility/java/com/android/server/accessibility/AccessibilityWindowManager.java @@ -667,6 +667,10 @@ public class AccessibilityWindowManager { return null; } + // Don't need to add the embedded hierarchy windows into the accessibility windows list. + if (mHostEmbeddedMap.size() > 0 && isEmbeddedHierarchyWindowsLocked(windowId)) { + return null; + } final AccessibilityWindowInfo reportedWindow = AccessibilityWindowInfo.obtain(); reportedWindow.setId(windowId); @@ -699,6 +703,21 @@ public class AccessibilityWindowManager { return reportedWindow; } + private boolean isEmbeddedHierarchyWindowsLocked(int windowId) { + final IBinder leashToken = mWindowIdMap.get(windowId); + if (leashToken == null) { + return false; + } + + for (int i = 0; i < mHostEmbeddedMap.size(); i++) { + if (mHostEmbeddedMap.keyAt(i).equals(leashToken)) { + return true; + } + } + + return false; + } + private int getTypeForWindowManagerWindowType(int windowType) { switch (windowType) { case WindowManager.LayoutParams.TYPE_APPLICATION: diff --git a/services/core/java/com/android/server/wm/AccessibilityController.java b/services/core/java/com/android/server/wm/AccessibilityController.java index 0396a1101c644..8f703c5c77612 100644 --- a/services/core/java/com/android/server/wm/AccessibilityController.java +++ b/services/core/java/com/android/server/wm/AccessibilityController.java @@ -45,7 +45,6 @@ import static com.android.server.accessibility.AccessibilityTraceProto.WINDOW_MA import static com.android.server.wm.WindowManagerDebugConfig.TAG_WITH_CLASS_NAME; import static com.android.server.wm.WindowManagerDebugConfig.TAG_WM; import static com.android.server.wm.WindowTracing.WINSCOPE_EXT; -import static com.android.server.wm.utils.RegionUtils.forEachRect; import android.accessibilityservice.AccessibilityTrace; import android.animation.ObjectAnimator; @@ -101,6 +100,7 @@ import com.android.internal.util.TraceBuffer; import com.android.internal.util.function.pooled.PooledLambda; import com.android.server.LocalServices; import com.android.server.policy.WindowManagerPolicy; +import com.android.server.wm.AccessibilityWindowsPopulator.AccessibilityWindow; import com.android.server.wm.WindowManagerInternal.AccessibilityControllerInternal; import com.android.server.wm.WindowManagerInternal.MagnificationCallbacks; import com.android.server.wm.WindowManagerInternal.WindowsForAccessibilityCallback; @@ -133,19 +133,22 @@ final class AccessibilityController { private static final Rect EMPTY_RECT = new Rect(); private static final float[] sTempFloats = new float[9]; - private SparseArray mDisplayMagnifiers = new SparseArray<>(); - private SparseArray mWindowsForAccessibilityObserver = + private final SparseArray mDisplayMagnifiers = new SparseArray<>(); + private final SparseArray mWindowsForAccessibilityObserver = new SparseArray<>(); private SparseArray mFocusedWindow = new SparseArray<>(); private int mFocusedDisplay = -1; private boolean mIsImeVisible = false; // Set to true if initializing window population complete. private boolean mAllObserversInitialized = true; + private final AccessibilityWindowsPopulator mAccessibilityWindowsPopulator; AccessibilityController(WindowManagerService service) { mService = service; mAccessibilityTracing = AccessibilityController.getAccessibilityControllerInternal(service); + + mAccessibilityWindowsPopulator = new AccessibilityWindowsPopulator(mService, this); } boolean setMagnificationCallbacks(int displayId, MagnificationCallbacks callbacks) { @@ -209,7 +212,9 @@ final class AccessibilityController { } mWindowsForAccessibilityObserver.remove(displayId); } - observer = new WindowsForAccessibilityObserver(mService, displayId, callback); + mAccessibilityWindowsPopulator.setWindowsNotification(true); + observer = new WindowsForAccessibilityObserver(mService, displayId, callback, + mAccessibilityWindowsPopulator); mWindowsForAccessibilityObserver.put(displayId, observer); mAllObserversInitialized &= observer.mInitialized; } else { @@ -224,6 +229,10 @@ final class AccessibilityController { } } mWindowsForAccessibilityObserver.remove(displayId); + + if (mWindowsForAccessibilityObserver.size() <= 0) { + mAccessibilityWindowsPopulator.setWindowsNotification(false); + } } } @@ -309,11 +318,6 @@ final class AccessibilityController { if (displayMagnifier != null) { displayMagnifier.onDisplaySizeChanged(displayContent); } - final WindowsForAccessibilityObserver windowsForA11yObserver = - mWindowsForAccessibilityObserver.get(displayId); - if (windowsForA11yObserver != null) { - windowsForA11yObserver.scheduleComputeChangedWindows(); - } } void onAppWindowTransition(int displayId, int transition) { @@ -341,11 +345,6 @@ final class AccessibilityController { if (displayMagnifier != null) { displayMagnifier.onWindowTransition(windowState, transition); } - final WindowsForAccessibilityObserver windowsForA11yObserver = - mWindowsForAccessibilityObserver.get(displayId); - if (windowsForA11yObserver != null) { - windowsForA11yObserver.scheduleComputeChangedWindows(); - } } void onWindowFocusChangedNot(int displayId) { @@ -455,6 +454,19 @@ final class AccessibilityController { return null; } + boolean getMagnificationSpecForDisplay(int displayId, MagnificationSpec outSpec) { + if (mAccessibilityTracing.isTracingEnabled(FLAGS_MAGNIFICATION_CALLBACK)) { + mAccessibilityTracing.logTrace(TAG + ".getMagnificationSpecForDisplay", + FLAGS_MAGNIFICATION_CALLBACK, "displayId=" + displayId); + } + final DisplayMagnifier displayMagnifier = mDisplayMagnifiers.get(displayId); + if (displayMagnifier == null) { + return false; + } + + return displayMagnifier.getMagnificationSpec(outSpec); + } + boolean hasCallbacks() { if (mAccessibilityTracing.isTracingEnabled(FLAGS_MAGNIFICATION_CALLBACK | FLAGS_WINDOWS_FOR_ACCESSIBILITY_CALLBACK)) { @@ -756,6 +768,25 @@ final class AccessibilityController { return spec; } + boolean getMagnificationSpec(MagnificationSpec outSpec) { + if (mAccessibilityTracing.isTracingEnabled(FLAGS_MAGNIFICATION_CALLBACK)) { + mAccessibilityTracing.logTrace(LOG_TAG + ".getMagnificationSpec", + FLAGS_MAGNIFICATION_CALLBACK); + } + MagnificationSpec spec = mMagnifedViewport.getMagnificationSpec(); + if (spec == null) { + return false; + } + + outSpec.setTo(spec); + if (mAccessibilityTracing.isTracingEnabled(FLAGS_MAGNIFICATION_CALLBACK)) { + mAccessibilityTracing.logTrace(LOG_TAG + ".getMagnificationSpec", + FLAGS_MAGNIFICATION_CALLBACK, "outSpec={" + outSpec + "}"); + } + + return true; + } + void getMagnificationRegion(Region outMagnificationRegion) { if (mAccessibilityTracing.isTracingEnabled(FLAGS_MAGNIFICATION_CALLBACK)) { mAccessibilityTracing.logTrace(LOG_TAG + ".getMagnificationRegion", @@ -1403,20 +1434,18 @@ final class AccessibilityController { private static final boolean DEBUG = false; - private final SparseArray mTempWindowStates = new SparseArray<>(); + private final List mTempA11yWindows = new ArrayList<>(); private final Set mTempBinderSet = new ArraySet<>(); - private final RectF mTempRectF = new RectF(); - - private final Matrix mTempMatrix = new Matrix(); - private final Point mTempPoint = new Point(); private final Region mTempRegion = new Region(); private final Region mTempRegion1 = new Region(); + private final Region mTempRegion2 = new Region(); + private final WindowManagerService mService; private final Handler mHandler; @@ -1431,10 +1460,11 @@ final class AccessibilityController { // Set to true if initializing window population complete. private boolean mInitialized; + private final AccessibilityWindowsPopulator mA11yWindowsPopulator; WindowsForAccessibilityObserver(WindowManagerService windowManagerService, - int displayId, - WindowsForAccessibilityCallback callback) { + int displayId, WindowsForAccessibilityCallback callback, + AccessibilityWindowsPopulator accessibilityWindowsPopulator) { mService = windowManagerService; mCallback = callback; mDisplayId = displayId; @@ -1443,6 +1473,7 @@ final class AccessibilityController { AccessibilityController.getAccessibilityControllerInternal(mService); mRecurringAccessibilityEventsIntervalMillis = ViewConfiguration .getSendRecurringAccessibilityEventsInterval(); + mA11yWindowsPopulator = accessibilityWindowsPopulator; computeChangedWindows(true); } @@ -1466,52 +1497,6 @@ final class AccessibilityController { } } - boolean shellRootIsAbove(WindowState windowState, ShellRoot shellRoot) { - int wsLayer = mService.mPolicy.getWindowLayerLw(windowState); - int shellLayer = mService.mPolicy.getWindowLayerFromTypeLw(shellRoot.getWindowType(), - true); - return shellLayer >= wsLayer; - } - - int addShellRootsIfAbove(WindowState windowState, ArrayList shellRoots, - int shellRootIndex, List windows, Set addedWindows, - Region unaccountedSpace, boolean focusedWindowAdded) { - while (shellRootIndex < shellRoots.size() - && shellRootIsAbove(windowState, shellRoots.get(shellRootIndex))) { - ShellRoot shellRoot = shellRoots.get(shellRootIndex); - shellRootIndex++; - final WindowInfo info = shellRoot.getWindowInfo(); - if (info == null) { - continue; - } - - info.layer = addedWindows.size(); - windows.add(info); - addedWindows.add(info.token); - unaccountedSpace.op(info.regionInScreen, unaccountedSpace, - Region.Op.REVERSE_DIFFERENCE); - if (unaccountedSpace.isEmpty() && focusedWindowAdded) { - break; - } - } - return shellRootIndex; - } - - private ArrayList getSortedShellRoots( - SparseArray originalShellRoots) { - ArrayList sortedShellRoots = new ArrayList<>(originalShellRoots.size()); - for (int i = originalShellRoots.size() - 1; i >= 0; --i) { - sortedShellRoots.add(originalShellRoots.valueAt(i)); - } - - sortedShellRoots.sort((left, right) -> - mService.mPolicy.getWindowLayerFromTypeLw(right.getWindowType(), true) - - mService.mPolicy.getWindowLayerFromTypeLw(left.getWindowType(), - true)); - - return sortedShellRoots; - } - /** * Check if windows have changed, and send them to the accessibility subsystem if they have. * @@ -1561,44 +1546,29 @@ final class AccessibilityController { Region unaccountedSpace = mTempRegion; unaccountedSpace.set(0, 0, screenWidth, screenHeight); - final SparseArray visibleWindows = mTempWindowStates; - populateVisibleWindowsOnScreen(visibleWindows); + final List visibleWindows = mTempA11yWindows; + mA11yWindowsPopulator.populateVisibleWindowsOnScreenLocked( + mDisplayId, visibleWindows); Set addedWindows = mTempBinderSet; addedWindows.clear(); boolean focusedWindowAdded = false; final int visibleWindowCount = visibleWindows.size(); - ArrayList skipRemainingWindowsForTaskFragments = new ArrayList<>(); - - ArrayList shellRoots = getSortedShellRoots(dc.mShellRoots); // Iterate until we figure out what is touchable for the entire screen. - int shellRootIndex = 0; - for (int i = visibleWindowCount - 1; i >= 0; i--) { - final WindowState windowState = visibleWindows.valueAt(i); - int prevShellRootIndex = shellRootIndex; - shellRootIndex = addShellRootsIfAbove(windowState, shellRoots, shellRootIndex, - windows, addedWindows, unaccountedSpace, focusedWindowAdded); - - // If a Shell Root was added, it could have accounted for all the space already. - if (shellRootIndex > prevShellRootIndex && unaccountedSpace.isEmpty() - && focusedWindowAdded) { - break; - } - - final Region regionInScreen = new Region(); - computeWindowRegionInScreen(windowState, regionInScreen); - if (windowMattersToAccessibility(windowState, - regionInScreen, unaccountedSpace, - skipRemainingWindowsForTaskFragments)) { - addPopulatedWindowInfo(windowState, regionInScreen, windows, addedWindows); - if (windowMattersToUnaccountedSpaceComputation(windowState)) { - updateUnaccountedSpace(windowState, regionInScreen, unaccountedSpace, - skipRemainingWindowsForTaskFragments); + for (int i = 0; i < visibleWindowCount; i++) { + final AccessibilityWindow a11yWindow = visibleWindows.get(i); + final Region regionInWindow = new Region(); + a11yWindow.getTouchableRegionInWindow(regionInWindow); + if (windowMattersToAccessibility(a11yWindow, regionInWindow, + unaccountedSpace)) { + addPopulatedWindowInfo(a11yWindow, regionInWindow, windows, addedWindows); + if (windowMattersToUnaccountedSpaceComputation(a11yWindow)) { + updateUnaccountedSpace(a11yWindow, unaccountedSpace); } - focusedWindowAdded |= windowState.isFocused(); - } else if (isUntouchableNavigationBar(windowState, mTempRegion1)) { + focusedWindowAdded |= a11yWindow.isFocused(); + } else if (a11yWindow.isUntouchableNavigationBar()) { // If this widow is navigation bar without touchable region, accounting the // region of navigation bar inset because all touch events from this region // would be received by launcher, i.e. this region is a un-touchable one @@ -1647,47 +1617,39 @@ final class AccessibilityController { // Some windows should be excluded from unaccounted space computation, though they still // should be reported - private boolean windowMattersToUnaccountedSpaceComputation(WindowState windowState) { + private boolean windowMattersToUnaccountedSpaceComputation(AccessibilityWindow a11yWindow) { // Do not account space of trusted non-touchable windows, except the split-screen // divider. // If it's not trusted, touch events are not sent to the windows behind it. - if (((windowState.mAttrs.flags & WindowManager.LayoutParams.FLAG_NOT_TOUCHABLE) != 0) - && (windowState.mAttrs.type != TYPE_DOCK_DIVIDER) - && windowState.isTrustedOverlay()) { + if (((a11yWindow.getFlags() & WindowManager.LayoutParams.FLAG_NOT_TOUCHABLE) != 0) + && (a11yWindow.getType() != TYPE_DOCK_DIVIDER) + && a11yWindow.isTrustedOverlay()) { return false; } - if (windowState.mAttrs.type - == WindowManager.LayoutParams.TYPE_ACCESSIBILITY_OVERLAY) { + if (a11yWindow.getType() == WindowManager.LayoutParams.TYPE_ACCESSIBILITY_OVERLAY) { return false; } return true; } - private boolean windowMattersToAccessibility(WindowState windowState, - Region regionInScreen, Region unaccountedSpace, - ArrayList skipRemainingWindowsForTaskFragments) { - final RecentsAnimationController controller = mService.getRecentsAnimationController(); - if (controller != null && controller.shouldIgnoreForAccessibility(windowState)) { + private boolean windowMattersToAccessibility(AccessibilityWindow a11yWindow, + Region regionInScreen, Region unaccountedSpace) { + if (a11yWindow.ignoreRecentsAnimationForAccessibility()) { return false; } - if (windowState.isFocused()) { + if (a11yWindow.isFocused()) { return true; } - // If the window is part of a task that we're finished with - ignore. - final TaskFragment taskFragment = windowState.getTaskFragment(); - if (taskFragment != null - && skipRemainingWindowsForTaskFragments.contains(taskFragment)) { - return false; - } - // Ignore non-touchable windows, except the split-screen divider, which is // occasionally non-touchable but still useful for identifying split-screen - // mode. - if (((windowState.mAttrs.flags & WindowManager.LayoutParams.FLAG_NOT_TOUCHABLE) != 0) - && (windowState.mAttrs.type != TYPE_DOCK_DIVIDER)) { + // mode and the PIP menu. + if (((a11yWindow.getFlags() + & WindowManager.LayoutParams.FLAG_NOT_TOUCHABLE) != 0) + && (a11yWindow.getType() != TYPE_DOCK_DIVIDER + && !a11yWindow.isPIPMenu())) { return false; } @@ -1697,88 +1659,36 @@ final class AccessibilityController { } // Add windows of certain types not covered by modal windows. - if (isReportedWindowType(windowState.mAttrs.type)) { + if (isReportedWindowType(a11yWindow.getType())) { return true; } return false; } - private void updateUnaccountedSpace(WindowState windowState, Region regionInScreen, - Region unaccountedSpace, - ArrayList skipRemainingWindowsForTaskFragments) { - // Account for the space this window takes if the window - // is not an accessibility overlay which does not change - // the reported windows. - unaccountedSpace.op(regionInScreen, unaccountedSpace, - Region.Op.REVERSE_DIFFERENCE); - - // If a window is modal it prevents other windows from being touched - if ((windowState.mAttrs.flags & (WindowManager.LayoutParams.FLAG_NOT_FOCUSABLE - | WindowManager.LayoutParams.FLAG_NOT_TOUCH_MODAL)) == 0) { - if (!windowState.hasTapExcludeRegion()) { - // Account for all space in the task, whether the windows in it are - // touchable or not. The modal window blocks all touches from the task's - // area. - unaccountedSpace.op(windowState.getDisplayFrame(), unaccountedSpace, - Region.Op.REVERSE_DIFFERENCE); - } else { - // If a window has tap exclude region, we need to account it. - final Region displayRegion = new Region(windowState.getDisplayFrame()); - final Region tapExcludeRegion = new Region(); - windowState.getTapExcludeRegion(tapExcludeRegion); - displayRegion.op(tapExcludeRegion, displayRegion, - Region.Op.REVERSE_DIFFERENCE); - unaccountedSpace.op(displayRegion, unaccountedSpace, - Region.Op.REVERSE_DIFFERENCE); - } - - final TaskFragment taskFragment = windowState.getTaskFragment(); - if (taskFragment != null) { - // If the window is associated with a particular task, we can skip the - // rest of the windows for that task. - skipRemainingWindowsForTaskFragments.add(taskFragment); - } else if (!windowState.hasTapExcludeRegion()) { - // If the window is not associated with a particular task, then it is - // globally modal. In this case we can skip all remaining windows when - // it doesn't has tap exclude region. - unaccountedSpace.setEmpty(); - } - } - - // Account for the space of letterbox. - if (windowState.areAppWindowBoundsLetterboxed()) { - unaccountedSpace.op(getLetterboxBounds(windowState), unaccountedSpace, + private void updateUnaccountedSpace(AccessibilityWindow a11yWindow, + Region unaccountedSpace) { + if (a11yWindow.getType() + != WindowManager.LayoutParams.TYPE_ACCESSIBILITY_OVERLAY) { + // Account for the space this window takes if the window + // is not an accessibility overlay which does not change + // the reported windows. + final Region touchableRegion = mTempRegion2; + a11yWindow.getTouchableRegionInScreen(touchableRegion); + unaccountedSpace.op(touchableRegion, unaccountedSpace, Region.Op.REVERSE_DIFFERENCE); + // Account for the space of letterbox. + final Region letterboxBounds = mTempRegion1; + if (a11yWindow.setLetterBoxBoundsIfNeeded(letterboxBounds)) { + unaccountedSpace.op(letterboxBounds, + unaccountedSpace, Region.Op.REVERSE_DIFFERENCE); + } } } - private void computeWindowRegionInScreen(WindowState windowState, Region outRegion) { - // Get the touchable frame. - Region touchableRegion = mTempRegion1; - windowState.getTouchableRegion(touchableRegion); - - // Map the frame to get what appears on the screen. - Matrix matrix = mTempMatrix; - populateTransformationMatrix(windowState, matrix); - - forEachRect(touchableRegion, rect -> { - // Move to origin as all transforms are captured by the matrix. - RectF windowFrame = mTempRectF; - windowFrame.set(rect); - windowFrame.offset(-windowState.getFrame().left, -windowState.getFrame().top); - - matrix.mapRect(windowFrame); - - // Union all rects. - outRegion.union(new Rect((int) windowFrame.left, (int) windowFrame.top, - (int) windowFrame.right, (int) windowFrame.bottom)); - }); - } - - private static void addPopulatedWindowInfo(WindowState windowState, Region regionInScreen, - List out, Set tokenOut) { - final WindowInfo window = windowState.getWindowInfo(); + private static void addPopulatedWindowInfo(AccessibilityWindow a11yWindow, + Region regionInScreen, List out, Set tokenOut) { + final WindowInfo window = a11yWindow.getWindowInfo(); window.regionInScreen.set(regionInScreen); window.layer = tokenOut.size(); out.add(window); @@ -1805,23 +1715,6 @@ final class AccessibilityController { && windowType != WindowManager.LayoutParams.TYPE_PRIVATE_PRESENTATION); } - private void populateVisibleWindowsOnScreen(SparseArray outWindows) { - final List tempWindowStatesList = new ArrayList<>(); - final DisplayContent dc = mService.mRoot.getDisplayContent(mDisplayId); - if (dc == null) { - return; - } - - dc.forAllWindows(w -> { - if (w.isVisible()) { - tempWindowStatesList.add(w); - } - }, false /* traverseTopToBottom */); - for (int i = 0; i < tempWindowStatesList.size(); i++) { - outWindows.put(i, tempWindowStatesList.get(i)); - } - } - private WindowState getTopFocusWindow() { return mService.mRoot.getTopFocusedDisplayContent().mCurrentFocus; } diff --git a/services/core/java/com/android/server/wm/AccessibilityWindowsPopulator.java b/services/core/java/com/android/server/wm/AccessibilityWindowsPopulator.java new file mode 100644 index 0000000000000..c0fb83ba294c6 --- /dev/null +++ b/services/core/java/com/android/server/wm/AccessibilityWindowsPopulator.java @@ -0,0 +1,626 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.wm; + +import static android.view.WindowManager.LayoutParams.TYPE_DOCK_DIVIDER; + +import static com.android.server.wm.utils.RegionUtils.forEachRect; + +import android.annotation.NonNull; +import android.graphics.Matrix; +import android.graphics.Rect; +import android.graphics.RectF; +import android.graphics.Region; +import android.os.Handler; +import android.os.IBinder; +import android.os.Looper; +import android.os.Message; +import android.util.Slog; +import android.util.SparseArray; +import android.view.IWindow; +import android.view.InputWindowHandle; +import android.view.MagnificationSpec; +import android.view.WindowInfo; +import android.view.WindowManager; +import android.window.WindowInfosListener; + +import com.android.internal.annotations.GuardedBy; + +import java.util.ArrayList; +import java.util.List; + +/** + * This class is the accessibility windows population adapter. + */ +public final class AccessibilityWindowsPopulator extends WindowInfosListener { + + private static final String TAG = AccessibilityWindowsPopulator.class.getSimpleName(); + // If the surface flinger callback is not coming within in 2 frames time, i.e. about + // 35ms, then assuming the windows become stable. + private static final int SURFACE_FLINGER_CALLBACK_WINDOWS_STABLE_TIMES_MS = 35; + // To avoid the surface flinger callbacks always comes within in 2 frames, then no windows + // are reported to the A11y framework, and the animation duration time is 500ms, so setting + // this value as the max timeout value to force computing changed windows. + private static final int WINDOWS_CHANGED_NOTIFICATION_MAX_DURATION_TIMES_MS = 500; + + private static final float[] sTempFloats = new float[9]; + + private final WindowManagerService mService; + private final AccessibilityController mAccessibilityController; + @GuardedBy("mLock") + private final SparseArray> mInputWindowHandlesOnDisplays = + new SparseArray<>(); + @GuardedBy("mLock") + private final SparseArray mMagnificationSpecInverseMatrix = new SparseArray<>(); + @GuardedBy("mLock") + private final SparseArray mDisplayInfos = new SparseArray<>(); + @GuardedBy("mLock") + private final List mVisibleWindows = new ArrayList<>(); + @GuardedBy("mLock") + private boolean mWindowsNotificationEnabled = false; + private final Object mLock = new Object(); + private final Handler mHandler; + + AccessibilityWindowsPopulator(WindowManagerService service, + AccessibilityController accessibilityController) { + mService = service; + mAccessibilityController = accessibilityController; + mHandler = new MyHandler(mService.mH.getLooper()); + + register(); + } + + /** + * Gets the visible windows list with the window layer on the specified display. + * + * @param displayId The display. + * @param outWindows The visible windows list. The z-order of each window in the list + * is from the top to bottom. + */ + public void populateVisibleWindowsOnScreenLocked(int displayId, + List outWindows) { + List inputWindowHandles; + final Matrix inverseMatrix = new Matrix(); + final Matrix displayMatrix = new Matrix(); + + synchronized (mLock) { + inputWindowHandles = mInputWindowHandlesOnDisplays.get(displayId); + if (inputWindowHandles == null) { + outWindows.clear(); + + return; + } + inverseMatrix.set(mMagnificationSpecInverseMatrix.get(displayId)); + + final DisplayInfo displayInfo = mDisplayInfos.get(displayId); + if (displayInfo != null) { + displayMatrix.set(displayInfo.mTransform); + } else { + Slog.w(TAG, "The displayInfo of this displayId (" + displayId + ") called " + + "back from the surface fligner is null"); + } + } + + final DisplayContent dc = mService.mRoot.getDisplayContent(displayId); + final ShellRoot shellroot = dc.mShellRoots.get(WindowManager.SHELL_ROOT_LAYER_PIP); + final IBinder pipMenuIBinder = + shellroot != null ? shellroot.getAccessibilityWindowToken() : null; + + for (final InputWindowHandle windowHandle : inputWindowHandles) { + final AccessibilityWindow accessibilityWindow = + AccessibilityWindow.initializeData(mService, windowHandle, inverseMatrix, + pipMenuIBinder, displayMatrix); + + outWindows.add(accessibilityWindow); + } + } + + @Override + public void onWindowInfosChanged(InputWindowHandle[] windowHandles, + DisplayInfo[] displayInfos) { + synchronized (mLock) { + mVisibleWindows.clear(); + for (InputWindowHandle window : windowHandles) { + if (window.visible && window.getWindow() != null) { + mVisibleWindows.add(window); + } + } + + mDisplayInfos.clear(); + for (final DisplayInfo displayInfo : displayInfos) { + mDisplayInfos.put(displayInfo.mDisplayId, displayInfo); + } + + if (mWindowsNotificationEnabled) { + if (!mHandler.hasMessages( + MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_TIMEOUT)) { + mHandler.sendEmptyMessageDelayed( + MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_TIMEOUT, + WINDOWS_CHANGED_NOTIFICATION_MAX_DURATION_TIMES_MS); + } + populateVisibleWindowHandlesAndNotifyWindowsChangeIfNeededLocked(); + } + } + } + + /** + * Sets to notify the accessibilityController to compute changed windows on + * the display after populating the visible windows if the windows reported + * from the surface flinger changes. + * + * @param register {@code true} means starting windows population. + */ + public void setWindowsNotification(boolean register) { + synchronized (mLock) { + if (mWindowsNotificationEnabled == register) { + return; + } + mWindowsNotificationEnabled = register; + if (mWindowsNotificationEnabled) { + populateVisibleWindowHandlesAndNotifyWindowsChangeIfNeededLocked(); + } else { + releaseResources(); + } + } + } + + private void populateVisibleWindowHandlesAndNotifyWindowsChangeIfNeededLocked() { + final SparseArray> tempWindowHandleList = new SparseArray<>(); + + for (final InputWindowHandle windowHandle : mVisibleWindows) { + List inputWindowHandles = tempWindowHandleList.get( + windowHandle.displayId); + + if (inputWindowHandles == null) { + inputWindowHandles = new ArrayList<>(); + tempWindowHandleList.put(windowHandle.displayId, inputWindowHandles); + generateMagnificationSpecInverseMatrixLocked(windowHandle.displayId); + } + inputWindowHandles.add(windowHandle); + } + + final List displayIdsForWindowsChanged = new ArrayList<>(); + + getDisplaysForWindowsChangedLocked(displayIdsForWindowsChanged, tempWindowHandleList, + mInputWindowHandlesOnDisplays); + // Clones all windows from the callback of the surface flinger. + mInputWindowHandlesOnDisplays.clear(); + for (int i = 0; i < tempWindowHandleList.size(); i++) { + final int displayId = tempWindowHandleList.keyAt(i); + mInputWindowHandlesOnDisplays.put(displayId, tempWindowHandleList.get(displayId)); + } + + if (displayIdsForWindowsChanged.size() > 0) { + if (!mHandler.hasMessages(MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED)) { + mHandler.obtainMessage(MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED, + displayIdsForWindowsChanged).sendToTarget(); + } + + return; + } + mHandler.removeMessages(MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_UI_STABLE); + mHandler.sendEmptyMessageDelayed(MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_UI_STABLE, + SURFACE_FLINGER_CALLBACK_WINDOWS_STABLE_TIMES_MS); + } + + private void getDisplaysForWindowsChangedLocked(List outDisplayIdsForWindowsChanged, + SparseArray> newWindowsList, + SparseArray> oldWindowsList) { + for (int i = 0; i < newWindowsList.size(); i++) { + final int displayId = newWindowsList.keyAt(i); + final List newWindows = newWindowsList.get(displayId); + final List oldWindows = oldWindowsList.get(displayId); + + if (hasWindowsChangedLocked(newWindows, oldWindows)) { + outDisplayIdsForWindowsChanged.add(displayId); + } + } + } + + private boolean hasWindowsChangedLocked(List newWindows, + List oldWindows) { + if (oldWindows == null || oldWindows.size() != newWindows.size()) { + return true; + } + + final int windowsCount = newWindows.size(); + // Since we always traverse windows from high to low layer, + // the old and new windows at the same index should be the + // same, otherwise something changed. + for (int i = 0; i < windowsCount; i++) { + final InputWindowHandle newWindow = newWindows.get(i); + final InputWindowHandle oldWindow = oldWindows.get(i); + + if (!newWindow.getWindow().asBinder().equals(oldWindow.getWindow().asBinder())) { + return true; + } + } + + return false; + } + + private void generateMagnificationSpecInverseMatrixLocked(int displayId) { + MagnificationSpec spec = new MagnificationSpec(); + if (!mAccessibilityController.getMagnificationSpecForDisplay(displayId, spec)) { + mMagnificationSpecInverseMatrix.remove(displayId); + return; + } + sTempFloats[Matrix.MSCALE_X] = spec.scale; + sTempFloats[Matrix.MSKEW_Y] = 0; + sTempFloats[Matrix.MSKEW_X] = 0; + sTempFloats[Matrix.MSCALE_Y] = spec.scale; + sTempFloats[Matrix.MTRANS_X] = spec.offsetX; + sTempFloats[Matrix.MTRANS_Y] = spec.offsetY; + sTempFloats[Matrix.MPERSP_0] = 0; + sTempFloats[Matrix.MPERSP_1] = 0; + sTempFloats[Matrix.MPERSP_2] = 1; + + final Matrix tempMatrix = new Matrix(); + tempMatrix.setValues(sTempFloats); + + final Matrix inverseMatrix = new Matrix(); + final boolean result = tempMatrix.invert(inverseMatrix); + + if (!result) { + Slog.e(TAG, "Can't inverse the magnification spec matrix with the " + + "magnification spec = " + spec + " on the displayId = " + displayId); + return; + } + mMagnificationSpecInverseMatrix.set(displayId, inverseMatrix); + } + + private void notifyWindowsChanged(@NonNull List displayIdsForWindowsChanged) { + mHandler.removeMessages(MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_TIMEOUT); + + for (int i = 0; i < displayIdsForWindowsChanged.size(); i++) { + mAccessibilityController.performComputeChangedWindowsNot( + displayIdsForWindowsChanged.get(i), false); + } + } + + private void forceUpdateWindows() { + final List displayIdsForWindowsChanged = new ArrayList<>(); + + synchronized (mLock) { + for (int i = 0; i < mInputWindowHandlesOnDisplays.size(); i++) { + final int displayId = mInputWindowHandlesOnDisplays.keyAt(i); + displayIdsForWindowsChanged.add(displayId); + } + } + notifyWindowsChanged(displayIdsForWindowsChanged); + } + + @GuardedBy("mLock") + private void releaseResources() { + mInputWindowHandlesOnDisplays.clear(); + mMagnificationSpecInverseMatrix.clear(); + mVisibleWindows.clear(); + mDisplayInfos.clear(); + mWindowsNotificationEnabled = false; + mHandler.removeCallbacksAndMessages(null); + } + + private class MyHandler extends Handler { + public static final int MESSAGE_NOTIFY_WINDOWS_CHANGED = 1; + public static final int MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_UI_STABLE = 2; + public static final int MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_TIMEOUT = 3; + + MyHandler(Looper looper) { + super(looper, null, false); + } + + @Override + public void handleMessage(Message message) { + switch (message.what) { + case MESSAGE_NOTIFY_WINDOWS_CHANGED: { + final List displayIdsForWindowsChanged = (List) message.obj; + notifyWindowsChanged(displayIdsForWindowsChanged); + } break; + + case MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_UI_STABLE: { + forceUpdateWindows(); + } break; + + case MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_TIMEOUT: { + Slog.w(TAG, "Windows change within in 2 frames continuously over 500 ms " + + "and notify windows changed immediately"); + mHandler.removeMessages( + MyHandler.MESSAGE_NOTIFY_WINDOWS_CHANGED_BY_UI_STABLE); + + forceUpdateWindows(); + } break; + } + } + } + + /** + * This class represents information about a window from the + * surface flinger to the accessibility framework. + */ + public static class AccessibilityWindow { + private static final Region TEMP_REGION = new Region(); + private static final RectF TEMP_RECTF = new RectF(); + // Data + private IWindow mWindow; + private int mDisplayId; + private int mFlags; + private int mType; + private int mPrivateFlags; + private boolean mIsPIPMenu; + private boolean mIsFocused; + private boolean mShouldMagnify; + private boolean mIgnoreDuetoRecentsAnimation; + private boolean mIsTrustedOverlay; + private final Region mTouchableRegionInScreen = new Region(); + private final Region mTouchableRegionInWindow = new Region(); + private final Region mLetterBoxBounds = new Region(); + private WindowInfo mWindowInfo; + + /** + * Returns the instance after initializing the internal data. + * @param service The window manager service. + * @param inputWindowHandle The window from the surface flinger. + * @param inverseMatrix The magnification spec inverse matrix. + */ + public static AccessibilityWindow initializeData(WindowManagerService service, + InputWindowHandle inputWindowHandle, Matrix inverseMatrix, IBinder pipIBinder, + Matrix displayMatrix) { + final IWindow window = inputWindowHandle.getWindow(); + final WindowState windowState = window != null ? service.mWindowMap.get( + window.asBinder()) : null; + + final AccessibilityWindow instance = new AccessibilityWindow(); + + instance.mWindow = inputWindowHandle.getWindow(); + instance.mDisplayId = inputWindowHandle.displayId; + instance.mFlags = inputWindowHandle.layoutParamsFlags; + instance.mType = inputWindowHandle.layoutParamsType; + instance.mIsPIPMenu = inputWindowHandle.getWindow().asBinder().equals(pipIBinder); + + // TODO (b/199357848): gets the private flag of the window from other way. + instance.mPrivateFlags = windowState != null ? windowState.mAttrs.privateFlags : 0; + // TODO (b/199358208) : using new way to implement the focused window. + instance.mIsFocused = windowState != null && windowState.isFocused(); + instance.mShouldMagnify = windowState == null || windowState.shouldMagnify(); + + final RecentsAnimationController controller = service.getRecentsAnimationController(); + instance.mIgnoreDuetoRecentsAnimation = windowState != null && controller != null + && controller.shouldIgnoreForAccessibility(windowState); + instance.mIsTrustedOverlay = inputWindowHandle.trustedOverlay; + + // TODO (b/199358388) : gets the letterbox bounds of the window from other way. + if (windowState != null && windowState.areAppWindowBoundsLetterboxed()) { + getLetterBoxBounds(windowState, instance.mLetterBoxBounds); + } + + final Rect windowFrame = new Rect(inputWindowHandle.frameLeft, + inputWindowHandle.frameTop, inputWindowHandle.frameRight, + inputWindowHandle.frameBottom); + getTouchableRegionInWindow(instance.mShouldMagnify, inputWindowHandle.touchableRegion, + instance.mTouchableRegionInWindow, windowFrame, inverseMatrix, displayMatrix); + getUnMagnifiedTouchableRegion(instance.mShouldMagnify, + inputWindowHandle.touchableRegion, instance.mTouchableRegionInScreen, + inverseMatrix, displayMatrix); + instance.mWindowInfo = windowState != null + ? windowState.getWindowInfo() : getWindowInfoForWindowlessWindows(instance); + + return instance; + } + + /** + * Returns the touchable region in the screen. + * @param outRegion The touchable region. + */ + public void getTouchableRegionInScreen(Region outRegion) { + outRegion.set(mTouchableRegionInScreen); + } + + /** + * Returns the touchable region in the window. + * @param outRegion The touchable region. + */ + public void getTouchableRegionInWindow(Region outRegion) { + outRegion.set(mTouchableRegionInWindow); + } + + /** + * @return the layout parameter flag {@link android.view.WindowManager.LayoutParams#flags}. + */ + public int getFlags() { + return mFlags; + } + + /** + * @return the layout parameter type {@link android.view.WindowManager.LayoutParams#type}. + */ + public int getType() { + return mType; + } + + /** + * @return the layout parameter private flag + * {@link android.view.WindowManager.LayoutParams#privateFlags}. + */ + public int getPrivateFlag() { + return mPrivateFlags; + } + + /** + * @return the windowInfo {@link WindowInfo}. + */ + public WindowInfo getWindowInfo() { + return mWindowInfo; + } + + /** + * Gets the letter box bounds if activity bounds are letterboxed + * or letterboxed for display cutout. + * + * @return {@code true} there's a letter box bounds. + */ + public Boolean setLetterBoxBoundsIfNeeded(Region outBounds) { + if (mLetterBoxBounds.isEmpty()) { + return false; + } + + outBounds.set(mLetterBoxBounds); + return true; + } + + /** + * @return true if this window should be magnified. + */ + public boolean shouldMagnify() { + return mShouldMagnify; + } + + /** + * @return true if this window is focused. + */ + public boolean isFocused() { + return mIsFocused; + } + + /** + * @return true if it's running the recent animation but not the target app. + */ + public boolean ignoreRecentsAnimationForAccessibility() { + return mIgnoreDuetoRecentsAnimation; + } + + /** + * @return true if this window is the trusted overlay. + */ + public boolean isTrustedOverlay() { + return mIsTrustedOverlay; + } + + /** + * @return true if this window is the navigation bar with the gesture mode. + */ + public boolean isUntouchableNavigationBar() { + if (mType != WindowManager.LayoutParams.TYPE_NAVIGATION_BAR) { + return false; + } + + return mTouchableRegionInScreen.isEmpty(); + } + + /** + * @return true if this window is PIP menu. + */ + public boolean isPIPMenu() { + return mIsPIPMenu; + } + + private static void getTouchableRegionInWindow(boolean shouldMagnify, Region inRegion, + Region outRegion, Rect frame, Matrix inverseMatrix, Matrix displayMatrix) { + // Some modal windows, like the activity with Theme.dialog, has the full screen + // as its touchable region, but its window frame is smaller than the touchable + // region. The region we report should be the touchable area in the window frame + // for the consistency and match developers expectation. + // So we need to make the intersection between the frame and touchable region to + // obtain the real touch region in the screen. + Region touchRegion = TEMP_REGION; + touchRegion.set(inRegion); + touchRegion.op(frame, Region.Op.INTERSECT); + + getUnMagnifiedTouchableRegion(shouldMagnify, touchRegion, outRegion, inverseMatrix, + displayMatrix); + } + + /** + * Gets the un-magnified touchable region. If this window can be magnified and magnifying, + * we will transform the input touchable region by applying the inverse matrix of the + * magnification spec to get the un-magnified touchable region. + * @param shouldMagnify The window can be magnified. + * @param inRegion The touchable region of this window. + * @param outRegion The un-magnified touchable region of this window. + * @param inverseMatrix The inverse matrix of the magnification spec. + * @param displayMatrix The display transform matrix which takes display coordinates to + * logical display coordinates. + */ + private static void getUnMagnifiedTouchableRegion(boolean shouldMagnify, Region inRegion, + Region outRegion, Matrix inverseMatrix, Matrix displayMatrix) { + if ((!shouldMagnify || inverseMatrix.isIdentity()) && displayMatrix.isIdentity()) { + outRegion.set(inRegion); + return; + } + + forEachRect(inRegion, rect -> { + // Move to origin as all transforms are captured by the matrix. + RectF windowFrame = TEMP_RECTF; + windowFrame.set(rect); + + inverseMatrix.mapRect(windowFrame); + displayMatrix.mapRect(windowFrame); + // Union all rects. + outRegion.union(new Rect((int) windowFrame.left, (int) windowFrame.top, + (int) windowFrame.right, (int) windowFrame.bottom)); + }); + } + + private static WindowInfo getWindowInfoForWindowlessWindows(AccessibilityWindow window) { + WindowInfo windowInfo = WindowInfo.obtain(); + windowInfo.displayId = window.mDisplayId; + windowInfo.type = window.mType; + windowInfo.token = window.mWindow.asBinder(); + windowInfo.hasFlagWatchOutsideTouch = (window.mFlags + & WindowManager.LayoutParams.FLAG_WATCH_OUTSIDE_TOUCH) != 0; + windowInfo.inPictureInPicture = false; + + // There only are two windowless windows now, one is split window, and the other + // one is PIP. + if (windowInfo.type == TYPE_DOCK_DIVIDER) { + windowInfo.title = "Splitscreen Divider"; + } else if (window.mIsPIPMenu) { + windowInfo.title = "Picture-in-Picture menu"; + } + return windowInfo; + } + + private static void getLetterBoxBounds(WindowState windowState, Region outRegion) { + final Rect letterboxInsets = windowState.mActivityRecord.getLetterboxInsets(); + final Rect nonLetterboxRect = windowState.getBounds(); + + nonLetterboxRect.inset(letterboxInsets); + outRegion.set(windowState.getBounds()); + outRegion.op(nonLetterboxRect, Region.Op.DIFFERENCE); + } + + @Override + public String toString() { + String builder = "A11yWindow=[" + mWindow.asBinder() + + ", displayId=" + mDisplayId + + ", flag=0x" + Integer.toHexString(mFlags) + + ", type=" + mType + + ", privateFlag=0x" + Integer.toHexString(mPrivateFlags) + + ", focused=" + mIsFocused + + ", shouldMagnify=" + mShouldMagnify + + ", ignoreDuetoRecentsAnimation=" + mIgnoreDuetoRecentsAnimation + + ", isTrustedOverlay=" + mIsTrustedOverlay + + ", regionInScreen=" + mTouchableRegionInScreen + + ", touchableRegion=" + mTouchableRegionInWindow + + ", letterBoxBounds=" + mLetterBoxBounds + + ", isPIPMenu=" + mIsPIPMenu + + ", windowInfo=" + mWindowInfo + + "]"; + + return builder; + } + } +} diff --git a/services/core/java/com/android/server/wm/ShellRoot.java b/services/core/java/com/android/server/wm/ShellRoot.java index 2eab3bab0bd37..f9d7b53e4e780 100644 --- a/services/core/java/com/android/server/wm/ShellRoot.java +++ b/services/core/java/com/android/server/wm/ShellRoot.java @@ -25,15 +25,14 @@ import static com.android.server.wm.SurfaceAnimator.ANIMATION_TYPE_WINDOW_ANIMAT import static com.android.server.wm.WindowManagerService.MAX_ANIMATION_DURATION; import android.annotation.NonNull; +import android.annotation.Nullable; import android.graphics.Point; -import android.graphics.Rect; import android.os.IBinder; import android.os.RemoteException; import android.util.Slog; import android.view.DisplayInfo; import android.view.IWindow; import android.view.SurfaceControl; -import android.view.WindowInfo; import android.view.WindowManager; import android.view.animation.Animation; @@ -136,46 +135,12 @@ public class ShellRoot { ANIMATION_TYPE_WINDOW_ANIMATION); } - WindowInfo getWindowInfo() { - if (mShellRootLayer != SHELL_ROOT_LAYER_DIVIDER - && mShellRootLayer != SHELL_ROOT_LAYER_PIP) { - return null; + @Nullable + IBinder getAccessibilityWindowToken() { + if (mAccessibilityWindow != null) { + return mAccessibilityWindow.asBinder(); } - if (mShellRootLayer == SHELL_ROOT_LAYER_DIVIDER) { - return null; - } - if (mShellRootLayer == SHELL_ROOT_LAYER_PIP - && mDisplayContent.getDefaultTaskDisplayArea().getRootPinnedTask() == null) { - return null; - } - if (mAccessibilityWindow == null) { - return null; - } - WindowInfo windowInfo = WindowInfo.obtain(); - windowInfo.displayId = mToken.getDisplayArea().getDisplayContent().mDisplayId; - windowInfo.type = mToken.windowType; - windowInfo.layer = mToken.getWindowLayerFromType(); - windowInfo.token = mAccessibilityWindow.asBinder(); - windowInfo.focused = false; - windowInfo.hasFlagWatchOutsideTouch = false; - final Rect regionRect = new Rect(); - - - // DividerView - if (mShellRootLayer == SHELL_ROOT_LAYER_DIVIDER) { - windowInfo.inPictureInPicture = false; - mDisplayContent.getDockedDividerController().getTouchRegion(regionRect); - windowInfo.regionInScreen.set(regionRect); - windowInfo.title = "Splitscreen Divider"; - } - // PipMenuView - if (mShellRootLayer == SHELL_ROOT_LAYER_PIP) { - windowInfo.inPictureInPicture = true; - mDisplayContent.getDefaultTaskDisplayArea().getRootPinnedTask().getBounds(regionRect); - windowInfo.regionInScreen.set(regionRect); - windowInfo.title = "Picture-in-Picture menu"; - } - return windowInfo; + return null; } void setAccessibilityWindow(IWindow window) { @@ -196,9 +161,5 @@ public class ShellRoot { mAccessibilityWindow = null; } } - if (mDisplayContent.mWmService.mAccessibilityController.hasCallbacks()) { - mDisplayContent.mWmService.mAccessibilityController.onSomeWindowResizedOrMoved( - mDisplayContent.getDisplayId()); - } } } diff --git a/services/core/java/com/android/server/wm/WindowState.java b/services/core/java/com/android/server/wm/WindowState.java index 87ef09ee5982a..26acf43cf9edf 100644 --- a/services/core/java/com/android/server/wm/WindowState.java +++ b/services/core/java/com/android/server/wm/WindowState.java @@ -4992,9 +4992,6 @@ class WindowState extends WindowContainer implements WindowManagerP if (isAnimating()) { return; } - if (mWmService.mAccessibilityController.hasCallbacks()) { - mWmService.mAccessibilityController.onSomeWindowResizedOrMoved(getDisplayId()); - } if (!isSelfOrAncestorWindowAnimatingExit()) { return;