diff --git a/services/accessibility/java/com/android/server/accessibility/AbstractAccessibilityServiceConnection.java b/services/accessibility/java/com/android/server/accessibility/AbstractAccessibilityServiceConnection.java index 562d11a0b1971..fb13d01ffdda3 100644 --- a/services/accessibility/java/com/android/server/accessibility/AbstractAccessibilityServiceConnection.java +++ b/services/accessibility/java/com/android/server/accessibility/AbstractAccessibilityServiceConnection.java @@ -243,6 +243,9 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ int getCurrentUserIdLocked(); + Pair getWindowTransformationMatrixAndMagnificationSpec( + int windowId); + boolean isAccessibilityButtonShown(); /** @@ -552,8 +555,6 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ final int resolvedWindowId; RemoteAccessibilityConnection connection; Region partialInteractiveRegion = Region.obtain(); - final MagnificationSpec spec; - final float[] transformMatrix; synchronized (mLock) { mUsesAccessibilityCache = true; if (!hasRightsToCurrentUserLocked()) { @@ -577,11 +578,11 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ partialInteractiveRegion.recycle(); partialInteractiveRegion = null; } - final Pair transformMatrixAndSpec = - getTransformMatrixAndSpecLocked(resolvedWindowId); - transformMatrix = transformMatrixAndSpec.first; - spec = transformMatrixAndSpec.second; } + final Pair transformMatrixAndSpec = + getWindowTransformationMatrixAndMagnificationSpec(resolvedWindowId); + final float[] transformMatrix = transformMatrixAndSpec.first; + final MagnificationSpec spec = transformMatrixAndSpec.second; if (!mSecurityPolicy.checkAccessibilityAccess(this)) { return null; } @@ -629,8 +630,6 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ final int resolvedWindowId; RemoteAccessibilityConnection connection; Region partialInteractiveRegion = Region.obtain(); - final MagnificationSpec spec; - final float [] transformMatrix; synchronized (mLock) { mUsesAccessibilityCache = true; if (!hasRightsToCurrentUserLocked()) { @@ -654,11 +653,11 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ partialInteractiveRegion.recycle(); partialInteractiveRegion = null; } - final Pair transformMatrixAndSpec = - getTransformMatrixAndSpecLocked(resolvedWindowId); - transformMatrix = transformMatrixAndSpec.first; - spec = transformMatrixAndSpec.second; } + final Pair transformMatrixAndSpec = + getWindowTransformationMatrixAndMagnificationSpec(resolvedWindowId); + final float[] transformMatrix = transformMatrixAndSpec.first; + final MagnificationSpec spec = transformMatrixAndSpec.second; if (!mSecurityPolicy.checkAccessibilityAccess(this)) { return null; } @@ -707,8 +706,6 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ final int resolvedWindowId; RemoteAccessibilityConnection connection; Region partialInteractiveRegion = Region.obtain(); - final MagnificationSpec spec; - final float[] transformMatrix; synchronized (mLock) { mUsesAccessibilityCache = true; if (!hasRightsToCurrentUserLocked()) { @@ -732,11 +729,11 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ partialInteractiveRegion.recycle(); partialInteractiveRegion = null; } - final Pair transformMatrixAndSpec = - getTransformMatrixAndSpecLocked(resolvedWindowId); - transformMatrix = transformMatrixAndSpec.first; - spec = transformMatrixAndSpec.second; } + final Pair transformMatrixAndSpec = + getWindowTransformationMatrixAndMagnificationSpec(resolvedWindowId); + final float[] transformMatrix = transformMatrixAndSpec.first; + final MagnificationSpec spec = transformMatrixAndSpec.second; if (!mSecurityPolicy.checkAccessibilityAccess(this)) { return null; } @@ -787,8 +784,6 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ final int resolvedWindowId; RemoteAccessibilityConnection connection; Region partialInteractiveRegion = Region.obtain(); - final MagnificationSpec spec; - final float[] transformMatrix; synchronized (mLock) { if (!hasRightsToCurrentUserLocked()) { return null; @@ -812,11 +807,11 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ partialInteractiveRegion.recycle(); partialInteractiveRegion = null; } - final Pair transformMatrixAndSpec = - getTransformMatrixAndSpecLocked(resolvedWindowId); - transformMatrix = transformMatrixAndSpec.first; - spec = transformMatrixAndSpec.second; } + final Pair transformMatrixAndSpec = + getWindowTransformationMatrixAndMagnificationSpec(resolvedWindowId); + final float[] transformMatrix = transformMatrixAndSpec.first; + final MagnificationSpec spec = transformMatrixAndSpec.second; if (!mSecurityPolicy.checkAccessibilityAccess(this)) { return null; } @@ -866,8 +861,6 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ final int resolvedWindowId; RemoteAccessibilityConnection connection; Region partialInteractiveRegion = Region.obtain(); - final MagnificationSpec spec; - final float[] transformMatrix; synchronized (mLock) { if (!hasRightsToCurrentUserLocked()) { return null; @@ -890,12 +883,11 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ partialInteractiveRegion.recycle(); partialInteractiveRegion = null; } - - final Pair transformMatrixAndSpec = - getTransformMatrixAndSpecLocked(resolvedWindowId); - transformMatrix = transformMatrixAndSpec.first; - spec = transformMatrixAndSpec.second; } + final Pair transformMatrixAndSpec = + getWindowTransformationMatrixAndMagnificationSpec(resolvedWindowId); + final float[] transformMatrix = transformMatrixAndSpec.first; + final MagnificationSpec spec = transformMatrixAndSpec.second; if (!mSecurityPolicy.checkAccessibilityAccess(this)) { return null; } @@ -1672,21 +1664,10 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ mInvocationHandler.startInputLocked(startInputToken, inputContext, editorInfo, restarting); } - - @Nullable - Pair getTransformMatrixAndSpecLocked(int resolvedWindowId) { - final WindowInfo windowInfo = - mA11yWindowManager.findWindowInfoByIdLocked(resolvedWindowId); - if (windowInfo == null) { - Slog.w(LOG_TAG, "getTransformMatrixAndSpec, windowInfo is null window id = " - + resolvedWindowId); - return new Pair<>(null, null); - } - - final MagnificationSpec spec = new MagnificationSpec(); - spec.setTo(windowInfo.mMagnificationSpec); - return new Pair<>(windowInfo.mTransformMatrix, spec); + private Pair getWindowTransformationMatrixAndMagnificationSpec( + int resolvedWindowId) { + return mSystemSupport.getWindowTransformationMatrixAndMagnificationSpec(resolvedWindowId); } /** diff --git a/services/accessibility/java/com/android/server/accessibility/AccessibilityManagerService.java b/services/accessibility/java/com/android/server/accessibility/AccessibilityManagerService.java index ac0c051794b30..d0ade03afed4a 100644 --- a/services/accessibility/java/com/android/server/accessibility/AccessibilityManagerService.java +++ b/services/accessibility/java/com/android/server/accessibility/AccessibilityManagerService.java @@ -68,6 +68,7 @@ import android.content.pm.PackageManagerInternal; import android.content.pm.ResolveInfo; import android.content.pm.ServiceInfo; import android.database.ContentObserver; +import android.graphics.Matrix; import android.graphics.Point; import android.graphics.Rect; import android.graphics.Region; @@ -99,6 +100,7 @@ import android.text.TextUtils; import android.text.TextUtils.SimpleStringSplitter; import android.util.ArraySet; import android.util.IntArray; +import android.util.Pair; import android.util.Slog; import android.util.SparseArray; import android.view.Display; @@ -448,6 +450,41 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub return mIsAccessibilityButtonShown; } + @Override + public Pair getWindowTransformationMatrixAndMagnificationSpec( + int windowId) { + WindowInfo windowInfo; + synchronized (mLock) { + windowInfo = mA11yWindowManager.findWindowInfoByIdLocked(windowId); + } + if (windowInfo != null) { + final MagnificationSpec spec = new MagnificationSpec(); + spec.setTo(windowInfo.mMagnificationSpec); + return new Pair<>(windowInfo.mTransformMatrix, spec); + } else { + // If the framework doesn't track windows, we fall back to get the pair of + // transformation matrix and MagnificationSpe from the WindowManagerService's + // WindowState. + IBinder token; + synchronized (mLock) { + token = mA11yWindowManager.getWindowTokenForUserAndWindowIdLocked(mCurrentUserId, + windowId); + } + Pair pair = + mWindowManagerService.getWindowTransformationMatrixAndMagnificationSpec(token); + final float[] outTransformationMatrix = new float[9]; + final Matrix tmpMatrix = pair.first; + final MagnificationSpec spec = pair.second; + if (!spec.isNop()) { + tmpMatrix.postScale(spec.scale, spec.scale); + tmpMatrix.postTranslate(spec.offsetX, spec.offsetY); + } + tmpMatrix.getValues(outTransformationMatrix); + + return new Pair<>(outTransformationMatrix, pair.second); + } + } + @Override public void onServiceInfoChangedLocked(AccessibilityUserState userState) { mSecurityPolicy.onBoundServicesChangedLocked(userState.mUserId, @@ -3726,12 +3763,12 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub boundsInScreenBeforeMagnification.centerY()); // Invert magnification if needed. - final WindowInfo windowInfo = mA11yWindowManager.findWindowInfoByIdLocked( - focus.getWindowId()); + final Pair pair = + getWindowTransformationMatrixAndMagnificationSpec(focus.getWindowId()); MagnificationSpec spec = null; - if (windowInfo != null) { + if (pair != null && pair.second != null) { spec = new MagnificationSpec(); - spec.setTo(windowInfo.mMagnificationSpec); + spec.setTo(pair.second); } if (spec != null && !spec.isNop()) { diff --git a/services/core/java/com/android/server/wm/AccessibilityController.java b/services/core/java/com/android/server/wm/AccessibilityController.java index 63619e543a6d6..f1dbad61a76dc 100644 --- a/services/core/java/com/android/server/wm/AccessibilityController.java +++ b/services/core/java/com/android/server/wm/AccessibilityController.java @@ -446,6 +446,43 @@ final class AccessibilityController { // Not relevant for the window observer. } + public Pair getWindowTransformationMatrixAndMagnificationSpec( + IBinder token) { + synchronized (mService.mGlobalLock) { + final Matrix transformationMatrix = new Matrix(); + final MagnificationSpec magnificationSpec = new MagnificationSpec(); + + final WindowState windowState = mService.mWindowMap.get(token); + if (windowState != null) { + windowState.getTransformationMatrix(new float[9], transformationMatrix); + + if (hasCallbacks()) { + final MagnificationSpec otherMagnificationSpec = + getMagnificationSpecForWindow(windowState); + if (otherMagnificationSpec != null && !otherMagnificationSpec.isNop()) { + magnificationSpec.setTo(otherMagnificationSpec); + } + } + } + + return new Pair<>(transformationMatrix, magnificationSpec); + } + } + + MagnificationSpec getMagnificationSpecForWindow(WindowState windowState) { + if (mAccessibilityTracing.isTracingEnabled(FLAGS_MAGNIFICATION_CALLBACK)) { + mAccessibilityTracing.logTrace(TAG + ".getMagnificationSpecForWindow", + FLAGS_MAGNIFICATION_CALLBACK, + "windowState={" + windowState + "}"); + } + final int displayId = windowState.getDisplayId(); + final DisplayMagnifier displayMagnifier = mDisplayMagnifiers.get(displayId); + if (displayMagnifier != null) { + return displayMagnifier.getMagnificationSpecForWindow(windowState); + } + return null; + } + boolean hasCallbacks() { if (mAccessibilityTracing.isTracingEnabled(FLAGS_MAGNIFICATION_CALLBACK | FLAGS_WINDOWS_FOR_ACCESSIBILITY_CALLBACK)) { diff --git a/services/core/java/com/android/server/wm/WindowManagerInternal.java b/services/core/java/com/android/server/wm/WindowManagerInternal.java index 42fad6d14fb9c..d53e6f3958a9d 100644 --- a/services/core/java/com/android/server/wm/WindowManagerInternal.java +++ b/services/core/java/com/android/server/wm/WindowManagerInternal.java @@ -23,11 +23,13 @@ import android.annotation.NonNull; import android.annotation.Nullable; import android.content.ClipData; import android.content.Context; +import android.graphics.Matrix; import android.graphics.Rect; import android.graphics.Region; import android.hardware.display.DisplayManagerInternal; import android.os.Bundle; import android.os.IBinder; +import android.util.Pair; import android.view.Display; import android.view.IInputFilter; import android.view.IRemoteAnimationFinishedCallback; @@ -450,6 +452,17 @@ public abstract class WindowManagerInternal { */ public abstract void getWindowFrame(IBinder token, Rect outBounds); + /** + * Get the transformation matrix and MagnificationSpec given its token. + * + * @param token The token. + * @return The pair of the transformation matrix and magnification spec. + */ + // TODO (b/231663133): Long term solution for tracking window when the + // FLAG_RETRIEVE_INTERACTIVE_WINDOWS is unset. + public abstract Pair + getWindowTransformationMatrixAndMagnificationSpec(IBinder token); + /** * Opens the global actions dialog. */ diff --git a/services/core/java/com/android/server/wm/WindowManagerService.java b/services/core/java/com/android/server/wm/WindowManagerService.java index 2f4dc515fd117..6720830a16b57 100644 --- a/services/core/java/com/android/server/wm/WindowManagerService.java +++ b/services/core/java/com/android/server/wm/WindowManagerService.java @@ -172,6 +172,7 @@ import android.content.res.Resources; import android.content.res.TypedArray; import android.database.ContentObserver; import android.graphics.Bitmap; +import android.graphics.Matrix; import android.graphics.Point; import android.graphics.PointF; import android.graphics.Rect; @@ -223,6 +224,7 @@ import android.util.ArraySet; import android.util.DisplayMetrics; import android.util.EventLog; import android.util.MergedConfiguration; +import android.util.Pair; import android.util.Slog; import android.util.SparseBooleanArray; import android.util.TimeUtils; @@ -7677,6 +7679,13 @@ public class WindowManagerService extends IWindowManager.Stub } } + @Override + public Pair getWindowTransformationMatrixAndMagnificationSpec( + IBinder token) { + return mAccessibilityController + .getWindowTransformationMatrixAndMagnificationSpec(token); + } + @Override public void waitForAllWindowsDrawn(Runnable callback, long timeout, int displayId) { final WindowContainer container = displayId == INVALID_DISPLAY diff --git a/services/tests/servicestests/src/com/android/server/accessibility/AbstractAccessibilityServiceConnectionTest.java b/services/tests/servicestests/src/com/android/server/accessibility/AbstractAccessibilityServiceConnectionTest.java index 08df4381ad62b..19df5a2ddd8b2 100644 --- a/services/tests/servicestests/src/com/android/server/accessibility/AbstractAccessibilityServiceConnectionTest.java +++ b/services/tests/servicestests/src/com/android/server/accessibility/AbstractAccessibilityServiceConnectionTest.java @@ -89,8 +89,10 @@ import android.os.PowerManager; import android.os.Process; import android.os.RemoteCallback; import android.os.RemoteException; +import android.util.Pair; import android.view.Display; import android.view.KeyEvent; +import android.view.MagnificationSpec; import android.view.accessibility.AccessibilityNodeInfo; import android.view.accessibility.AccessibilityWindowInfo; import android.view.accessibility.IAccessibilityInteractionConnection; @@ -145,6 +147,8 @@ public class AbstractAccessibilityServiceConnectionTest { private static final int USER_ID = 1; private static final int USER_ID2 = 2; private static final int INTERACTION_ID = 199; + private static final Pair FAKE_MATRIX_AND_MAG_SPEC = + new Pair<>(new float[9], new MagnificationSpec()); private static final int PID = Process.myPid(); private static final long TID = Process.myTid(); private static final int UID = Process.myUid(); @@ -188,6 +192,8 @@ public class AbstractAccessibilityServiceConnectionTest { .thenReturn(mMockFingerprintGestureDispatcher); when(mMockSystemSupport.getMagnificationProcessor()) .thenReturn(mMockMagnificationProcessor); + when(mMockSystemSupport.getWindowTransformationMatrixAndMagnificationSpec(anyInt())) + .thenReturn(FAKE_MATRIX_AND_MAG_SPEC); PowerManager powerManager = new PowerManager(mMockContext, mMockIPowerManager, mMockIThermalService, mHandler);