Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Gets node bound when the FLAG_RETRIEVE_INTERACTIVE_WINDOWS is unset

Browse Source 
Since the WindowInfo is null if the FLAG_RETRIEVE_INTERACTIVE_WINDOWS is
unset, we would fail to calculate the node bound correctly.

The reason behind this failure is that the accessibility framework only
tracks windows when the FLAG_RETRIEVE_INTERACTIVE_WINDOWS is set.

As being a short term solution, we could retrieve the transformation matrix
and magnification spec from WindowState, making us have this CL.

However, as a long term solution, we should keep tracking WindowInfo in
DisplayWindowsObserver whether the flag is set or unset and mitigate the
additional system overhead.

Bug: 229379635
Test: atest android.accessibilityservice.cts
      atest com.android.server.accessibility
      We use a test apk to test the node bound value when the flag is
      unset equals to the value from WindowInfo when the flag is set.
Change-Id: I02e7e74d027dc65605f5e8fcc10d3c808e316476
This commit is contained in:
Daniel Hsieh
2022-04-26 12:50:38 +00:00
parent b1eb715bca
commit 81f7e655e5
6 changed files with 132 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

71
services/accessibility/java/com/android/server/accessibility/AbstractAccessibilityServiceConnection.java
View File

@@ -243,6 +243,9 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ
int getCurrentUserIdLocked();
Pair<float[], MagnificationSpec> getWindowTransformationMatrixAndMagnificationSpec(
int windowId);
boolean isAccessibilityButtonShown();
/**
@@ -552,8 +555,6 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ
final int resolvedWindowId;
RemoteAccessibilityConnection connection;
Region partialInteractiveRegion = Region.obtain();
final MagnificationSpec spec;
final float[] transformMatrix;
synchronized (mLock) {
mUsesAccessibilityCache = true;
if (!hasRightsToCurrentUserLocked()) {
@@ -577,11 +578,11 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ
partialInteractiveRegion.recycle();
partialInteractiveRegion = null;
}
final Pair<float[], MagnificationSpec> transformMatrixAndSpec =
getTransformMatrixAndSpecLocked(resolvedWindowId);
transformMatrix = transformMatrixAndSpec.first;
spec = transformMatrixAndSpec.second;
}
final Pair<float[], MagnificationSpec> transformMatrixAndSpec =
getWindowTransformationMatrixAndMagnificationSpec(resolvedWindowId);
final float[] transformMatrix = transformMatrixAndSpec.first;
final MagnificationSpec spec = transformMatrixAndSpec.second;
if (!mSecurityPolicy.checkAccessibilityAccess(this)) {
return null;
}
@@ -629,8 +630,6 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ
final int resolvedWindowId;
RemoteAccessibilityConnection connection;
Region partialInteractiveRegion = Region.obtain();
final MagnificationSpec spec;
final float [] transformMatrix;
synchronized (mLock) {
mUsesAccessibilityCache = true;
if (!hasRightsToCurrentUserLocked()) {
@@ -654,11 +653,11 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ
partialInteractiveRegion.recycle();
partialInteractiveRegion = null;
}
final Pair<float[], MagnificationSpec> transformMatrixAndSpec =
getTransformMatrixAndSpecLocked(resolvedWindowId);
transformMatrix = transformMatrixAndSpec.first;
spec = transformMatrixAndSpec.second;
}
final Pair<float[], MagnificationSpec> transformMatrixAndSpec =
getWindowTransformationMatrixAndMagnificationSpec(resolvedWindowId);
final float[] transformMatrix = transformMatrixAndSpec.first;
final MagnificationSpec spec = transformMatrixAndSpec.second;
if (!mSecurityPolicy.checkAccessibilityAccess(this)) {
return null;
}
@@ -707,8 +706,6 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ
final int resolvedWindowId;
RemoteAccessibilityConnection connection;
Region partialInteractiveRegion = Region.obtain();
final MagnificationSpec spec;
final float[] transformMatrix;
synchronized (mLock) {
mUsesAccessibilityCache = true;
if (!hasRightsToCurrentUserLocked()) {
@@ -732,11 +729,11 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ
partialInteractiveRegion.recycle();
partialInteractiveRegion = null;
}
final Pair<float[], MagnificationSpec> transformMatrixAndSpec =
getTransformMatrixAndSpecLocked(resolvedWindowId);
transformMatrix = transformMatrixAndSpec.first;
spec = transformMatrixAndSpec.second;
}
final Pair<float[], MagnificationSpec> transformMatrixAndSpec =
getWindowTransformationMatrixAndMagnificationSpec(resolvedWindowId);
final float[] transformMatrix = transformMatrixAndSpec.first;
final MagnificationSpec spec = transformMatrixAndSpec.second;
if (!mSecurityPolicy.checkAccessibilityAccess(this)) {
return null;
}
@@ -787,8 +784,6 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ
final int resolvedWindowId;
RemoteAccessibilityConnection connection;
Region partialInteractiveRegion = Region.obtain();
final MagnificationSpec spec;
final float[] transformMatrix;
synchronized (mLock) {
if (!hasRightsToCurrentUserLocked()) {
return null;
@@ -812,11 +807,11 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ
partialInteractiveRegion.recycle();
partialInteractiveRegion = null;
}
final Pair<float[], MagnificationSpec> transformMatrixAndSpec =
getTransformMatrixAndSpecLocked(resolvedWindowId);
transformMatrix = transformMatrixAndSpec.first;
spec = transformMatrixAndSpec.second;
}
final Pair<float[], MagnificationSpec> transformMatrixAndSpec =
getWindowTransformationMatrixAndMagnificationSpec(resolvedWindowId);
final float[] transformMatrix = transformMatrixAndSpec.first;
final MagnificationSpec spec = transformMatrixAndSpec.second;
if (!mSecurityPolicy.checkAccessibilityAccess(this)) {
return null;
}
@@ -866,8 +861,6 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ
final int resolvedWindowId;
RemoteAccessibilityConnection connection;
Region partialInteractiveRegion = Region.obtain();
final MagnificationSpec spec;
final float[] transformMatrix;
synchronized (mLock) {
if (!hasRightsToCurrentUserLocked()) {
return null;
@@ -890,12 +883,11 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ
partialInteractiveRegion.recycle();
partialInteractiveRegion = null;
}
final Pair<float[], MagnificationSpec> transformMatrixAndSpec =
getTransformMatrixAndSpecLocked(resolvedWindowId);
transformMatrix = transformMatrixAndSpec.first;
spec = transformMatrixAndSpec.second;
}
final Pair<float[], MagnificationSpec> transformMatrixAndSpec =
getWindowTransformationMatrixAndMagnificationSpec(resolvedWindowId);
final float[] transformMatrix = transformMatrixAndSpec.first;
final MagnificationSpec spec = transformMatrixAndSpec.second;
if (!mSecurityPolicy.checkAccessibilityAccess(this)) {
return null;
}
@@ -1672,21 +1664,10 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ
mInvocationHandler.startInputLocked(startInputToken, inputContext, editorInfo, restarting);
}
@Nullable
Pair<float[], MagnificationSpec> getTransformMatrixAndSpecLocked(int resolvedWindowId) {
final WindowInfo windowInfo =
mA11yWindowManager.findWindowInfoByIdLocked(resolvedWindowId);
if (windowInfo == null) {
Slog.w(LOG_TAG, "getTransformMatrixAndSpec, windowInfo is null window id = "
+ resolvedWindowId);
return new Pair<>(null, null);
}
final MagnificationSpec spec = new MagnificationSpec();
spec.setTo(windowInfo.mMagnificationSpec);
return new Pair<>(windowInfo.mTransformMatrix, spec);
private Pair<float[], MagnificationSpec> getWindowTransformationMatrixAndMagnificationSpec(
int resolvedWindowId) {
return mSystemSupport.getWindowTransformationMatrixAndMagnificationSpec(resolvedWindowId);
}
/**

45
services/accessibility/java/com/android/server/accessibility/AccessibilityManagerService.java
View File

@@ -68,6 +68,7 @@ import android.content.pm.PackageManagerInternal;
import android.content.pm.ResolveInfo;
import android.content.pm.ServiceInfo;
import android.database.ContentObserver;
import android.graphics.Matrix;
import android.graphics.Point;
import android.graphics.Rect;
import android.graphics.Region;
@@ -99,6 +100,7 @@ import android.text.TextUtils;
import android.text.TextUtils.SimpleStringSplitter;
import android.util.ArraySet;
import android.util.IntArray;
import android.util.Pair;
import android.util.Slog;
import android.util.SparseArray;
import android.view.Display;
@@ -448,6 +450,41 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub
return mIsAccessibilityButtonShown;
}
@Override
public Pair<float[], MagnificationSpec> getWindowTransformationMatrixAndMagnificationSpec(
int windowId) {
WindowInfo windowInfo;
synchronized (mLock) {
windowInfo = mA11yWindowManager.findWindowInfoByIdLocked(windowId);
}
if (windowInfo != null) {
final MagnificationSpec spec = new MagnificationSpec();
spec.setTo(windowInfo.mMagnificationSpec);
return new Pair<>(windowInfo.mTransformMatrix, spec);
} else {
// If the framework doesn't track windows, we fall back to get the pair of
// transformation matrix and MagnificationSpe from the WindowManagerService's
// WindowState.
IBinder token;
synchronized (mLock) {
token = mA11yWindowManager.getWindowTokenForUserAndWindowIdLocked(mCurrentUserId,
windowId);
}
Pair<Matrix, MagnificationSpec> pair =
mWindowManagerService.getWindowTransformationMatrixAndMagnificationSpec(token);
final float[] outTransformationMatrix = new float[9];
final Matrix tmpMatrix = pair.first;
final MagnificationSpec spec = pair.second;
if (!spec.isNop()) {
tmpMatrix.postScale(spec.scale, spec.scale);
tmpMatrix.postTranslate(spec.offsetX, spec.offsetY);
}
tmpMatrix.getValues(outTransformationMatrix);
return new Pair<>(outTransformationMatrix, pair.second);
}
}
@Override
public void onServiceInfoChangedLocked(AccessibilityUserState userState) {
mSecurityPolicy.onBoundServicesChangedLocked(userState.mUserId,
@@ -3726,12 +3763,12 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub
boundsInScreenBeforeMagnification.centerY());
// Invert magnification if needed.
final WindowInfo windowInfo = mA11yWindowManager.findWindowInfoByIdLocked(
focus.getWindowId());
final Pair<float[], MagnificationSpec> pair =
getWindowTransformationMatrixAndMagnificationSpec(focus.getWindowId());
MagnificationSpec spec = null;
if (windowInfo != null) {
if (pair != null && pair.second != null) {
spec = new MagnificationSpec();
spec.setTo(windowInfo.mMagnificationSpec);
spec.setTo(pair.second);
}
if (spec != null && !spec.isNop()) {

37
services/core/java/com/android/server/wm/AccessibilityController.java
View File

@@ -446,6 +446,43 @@ final class AccessibilityController {
// Not relevant for the window observer.
}
public Pair<Matrix, MagnificationSpec> getWindowTransformationMatrixAndMagnificationSpec(
IBinder token) {
synchronized (mService.mGlobalLock) {
final Matrix transformationMatrix = new Matrix();
final MagnificationSpec magnificationSpec = new MagnificationSpec();
final WindowState windowState = mService.mWindowMap.get(token);
if (windowState != null) {
windowState.getTransformationMatrix(new float[9], transformationMatrix);
if (hasCallbacks()) {
final MagnificationSpec otherMagnificationSpec =
getMagnificationSpecForWindow(windowState);
if (otherMagnificationSpec != null && !otherMagnificationSpec.isNop()) {
magnificationSpec.setTo(otherMagnificationSpec);
}
}
}
return new Pair<>(transformationMatrix, magnificationSpec);
}
}
MagnificationSpec getMagnificationSpecForWindow(WindowState windowState) {
if (mAccessibilityTracing.isTracingEnabled(FLAGS_MAGNIFICATION_CALLBACK)) {
mAccessibilityTracing.logTrace(TAG + ".getMagnificationSpecForWindow",
FLAGS_MAGNIFICATION_CALLBACK,
"windowState={" + windowState + "}");
}
final int displayId = windowState.getDisplayId();
final DisplayMagnifier displayMagnifier = mDisplayMagnifiers.get(displayId);
if (displayMagnifier != null) {
return displayMagnifier.getMagnificationSpecForWindow(windowState);
}
return null;
}
boolean hasCallbacks() {
if (mAccessibilityTracing.isTracingEnabled(FLAGS_MAGNIFICATION_CALLBACK
| FLAGS_WINDOWS_FOR_ACCESSIBILITY_CALLBACK)) {

13
services/core/java/com/android/server/wm/WindowManagerInternal.java
View File

@@ -23,11 +23,13 @@ import android.annotation.NonNull;
import android.annotation.Nullable;
import android.content.ClipData;
import android.content.Context;
import android.graphics.Matrix;
import android.graphics.Rect;
import android.graphics.Region;
import android.hardware.display.DisplayManagerInternal;
import android.os.Bundle;
import android.os.IBinder;
import android.util.Pair;
import android.view.Display;
import android.view.IInputFilter;
import android.view.IRemoteAnimationFinishedCallback;
@@ -450,6 +452,17 @@ public abstract class WindowManagerInternal {
*/
public abstract void getWindowFrame(IBinder token, Rect outBounds);
/**
* Get the transformation matrix and MagnificationSpec given its token.
*
* @param token The token.
* @return The pair of the transformation matrix and magnification spec.
*/
// TODO (b/231663133): Long term solution for tracking window when the
// FLAG_RETRIEVE_INTERACTIVE_WINDOWS is unset.
public abstract Pair<Matrix, MagnificationSpec>
getWindowTransformationMatrixAndMagnificationSpec(IBinder token);
/**
* Opens the global actions dialog.
*/

9
services/core/java/com/android/server/wm/WindowManagerService.java
View File

@@ -172,6 +172,7 @@ import android.content.res.Resources;
import android.content.res.TypedArray;
import android.database.ContentObserver;
import android.graphics.Bitmap;
import android.graphics.Matrix;
import android.graphics.Point;
import android.graphics.PointF;
import android.graphics.Rect;
@@ -223,6 +224,7 @@ import android.util.ArraySet;
import android.util.DisplayMetrics;
import android.util.EventLog;
import android.util.MergedConfiguration;
import android.util.Pair;
import android.util.Slog;
import android.util.SparseBooleanArray;
import android.util.TimeUtils;
@@ -7677,6 +7679,13 @@ public class WindowManagerService extends IWindowManager.Stub
}
}
@Override
public Pair<Matrix, MagnificationSpec> getWindowTransformationMatrixAndMagnificationSpec(
IBinder token) {
return mAccessibilityController
.getWindowTransformationMatrixAndMagnificationSpec(token);
}
@Override
public void waitForAllWindowsDrawn(Runnable callback, long timeout, int displayId) {
final WindowContainer container = displayId == INVALID_DISPLAY

6
services/tests/servicestests/src/com/android/server/accessibility/AbstractAccessibilityServiceConnectionTest.java
View File

@@ -89,8 +89,10 @@ import android.os.PowerManager;
import android.os.Process;
import android.os.RemoteCallback;
import android.os.RemoteException;
import android.util.Pair;
import android.view.Display;
import android.view.KeyEvent;
import android.view.MagnificationSpec;
import android.view.accessibility.AccessibilityNodeInfo;
import android.view.accessibility.AccessibilityWindowInfo;
import android.view.accessibility.IAccessibilityInteractionConnection;
@@ -145,6 +147,8 @@ public class AbstractAccessibilityServiceConnectionTest {
private static final int USER_ID = 1;
private static final int USER_ID2 = 2;
private static final int INTERACTION_ID = 199;
private static final Pair<float[], MagnificationSpec> FAKE_MATRIX_AND_MAG_SPEC =
new Pair<>(new float[9], new MagnificationSpec());
private static final int PID = Process.myPid();
private static final long TID = Process.myTid();
private static final int UID = Process.myUid();
@@ -188,6 +192,8 @@ public class AbstractAccessibilityServiceConnectionTest {
.thenReturn(mMockFingerprintGestureDispatcher);
when(mMockSystemSupport.getMagnificationProcessor())
.thenReturn(mMockMagnificationProcessor);
when(mMockSystemSupport.getWindowTransformationMatrixAndMagnificationSpec(anyInt()))
.thenReturn(FAKE_MATRIX_AND_MAG_SPEC);
PowerManager powerManager =
new PowerManager(mMockContext, mMockIPowerManager, mMockIThermalService, mHandler);