Merge "Gets node bound when the FLAG_RETRIEVE_INTERACTIVE_WINDOWS is unset" into tm-dev

services/accessibility/java/com/android/server/accessibility/AbstractAccessibilityServiceConnection.java

@@ -242,6 +242,9 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ
 
         int getCurrentUserIdLocked();
 
+        Pair<float[], MagnificationSpec> getWindowTransformationMatrixAndMagnificationSpec(
+                int windowId);
+
         boolean isAccessibilityButtonShown();
 
         /**
@@ -551,8 +554,6 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ
         final int resolvedWindowId;
         RemoteAccessibilityConnection connection;
         Region partialInteractiveRegion = Region.obtain();
-        final MagnificationSpec spec;
-        final float[] transformMatrix;
         synchronized (mLock) {
             mUsesAccessibilityCache = true;
             if (!hasRightsToCurrentUserLocked()) {
@@ -576,11 +577,11 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ
                 partialInteractiveRegion.recycle();
                 partialInteractiveRegion = null;
             }
-            final Pair<float[], MagnificationSpec> transformMatrixAndSpec =
-                    getTransformMatrixAndSpecLocked(resolvedWindowId);
-            transformMatrix = transformMatrixAndSpec.first;
-            spec = transformMatrixAndSpec.second;
         }
+        final Pair<float[], MagnificationSpec> transformMatrixAndSpec =
+                getWindowTransformationMatrixAndMagnificationSpec(resolvedWindowId);
+        final float[] transformMatrix = transformMatrixAndSpec.first;
+        final MagnificationSpec spec = transformMatrixAndSpec.second;
         if (!mSecurityPolicy.checkAccessibilityAccess(this)) {
             return null;
         }
@@ -628,8 +629,6 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ
         final int resolvedWindowId;
         RemoteAccessibilityConnection connection;
         Region partialInteractiveRegion = Region.obtain();
-        final MagnificationSpec spec;
-        final float [] transformMatrix;
         synchronized (mLock) {
             mUsesAccessibilityCache = true;
             if (!hasRightsToCurrentUserLocked()) {
@@ -653,11 +652,11 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ
                 partialInteractiveRegion.recycle();
                 partialInteractiveRegion = null;
             }
-            final Pair<float[], MagnificationSpec> transformMatrixAndSpec =
-                    getTransformMatrixAndSpecLocked(resolvedWindowId);
-            transformMatrix = transformMatrixAndSpec.first;
-            spec = transformMatrixAndSpec.second;
         }
+        final Pair<float[], MagnificationSpec> transformMatrixAndSpec =
+                getWindowTransformationMatrixAndMagnificationSpec(resolvedWindowId);
+        final float[] transformMatrix = transformMatrixAndSpec.first;
+        final MagnificationSpec spec = transformMatrixAndSpec.second;
         if (!mSecurityPolicy.checkAccessibilityAccess(this)) {
             return null;
         }
@@ -706,8 +705,6 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ
         final int resolvedWindowId;
         RemoteAccessibilityConnection connection;
         Region partialInteractiveRegion = Region.obtain();
-        final MagnificationSpec spec;
-        final float[] transformMatrix;
         synchronized (mLock) {
             mUsesAccessibilityCache = true;
             if (!hasRightsToCurrentUserLocked()) {
@@ -731,11 +728,11 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ
                 partialInteractiveRegion.recycle();
                 partialInteractiveRegion = null;
             }
-            final Pair<float[], MagnificationSpec> transformMatrixAndSpec =
-                    getTransformMatrixAndSpecLocked(resolvedWindowId);
-            transformMatrix = transformMatrixAndSpec.first;
-            spec = transformMatrixAndSpec.second;
         }
+        final Pair<float[], MagnificationSpec> transformMatrixAndSpec =
+                getWindowTransformationMatrixAndMagnificationSpec(resolvedWindowId);
+        final float[] transformMatrix = transformMatrixAndSpec.first;
+        final MagnificationSpec spec = transformMatrixAndSpec.second;
         if (!mSecurityPolicy.checkAccessibilityAccess(this)) {
             return null;
         }
@@ -786,8 +783,6 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ
         final int resolvedWindowId;
         RemoteAccessibilityConnection connection;
         Region partialInteractiveRegion = Region.obtain();
-        final MagnificationSpec spec;
-        final float[] transformMatrix;
         synchronized (mLock) {
             if (!hasRightsToCurrentUserLocked()) {
                 return null;
@@ -811,11 +806,11 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ
                 partialInteractiveRegion.recycle();
                 partialInteractiveRegion = null;
             }
-            final Pair<float[], MagnificationSpec> transformMatrixAndSpec =
-                    getTransformMatrixAndSpecLocked(resolvedWindowId);
-            transformMatrix = transformMatrixAndSpec.first;
-            spec = transformMatrixAndSpec.second;
         }
+        final Pair<float[], MagnificationSpec> transformMatrixAndSpec =
+                getWindowTransformationMatrixAndMagnificationSpec(resolvedWindowId);
+        final float[] transformMatrix = transformMatrixAndSpec.first;
+        final MagnificationSpec spec = transformMatrixAndSpec.second;
         if (!mSecurityPolicy.checkAccessibilityAccess(this)) {
             return null;
         }
@@ -865,8 +860,6 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ
         final int resolvedWindowId;
         RemoteAccessibilityConnection connection;
         Region partialInteractiveRegion = Region.obtain();
-        final MagnificationSpec spec;
-        final float[] transformMatrix;
         synchronized (mLock) {
             if (!hasRightsToCurrentUserLocked()) {
                 return null;
@@ -889,12 +882,11 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ
                 partialInteractiveRegion.recycle();
                 partialInteractiveRegion = null;
             }
-
-            final Pair<float[], MagnificationSpec> transformMatrixAndSpec =
-                    getTransformMatrixAndSpecLocked(resolvedWindowId);
-            transformMatrix = transformMatrixAndSpec.first;
-            spec = transformMatrixAndSpec.second;
         }
+        final Pair<float[], MagnificationSpec> transformMatrixAndSpec =
+                getWindowTransformationMatrixAndMagnificationSpec(resolvedWindowId);
+        final float[] transformMatrix = transformMatrixAndSpec.first;
+        final MagnificationSpec spec = transformMatrixAndSpec.second;
         if (!mSecurityPolicy.checkAccessibilityAccess(this)) {
             return null;
         }
@@ -1672,21 +1664,10 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ
         mInvocationHandler.startInputLocked(connection, editorInfo, restarting);
     }
 
-
-
     @Nullable
-    Pair<float[], MagnificationSpec> getTransformMatrixAndSpecLocked(int resolvedWindowId) {
+    private Pair<float[], MagnificationSpec> getWindowTransformationMatrixAndMagnificationSpec(
-        final WindowInfo windowInfo =
+            int resolvedWindowId) {
-                mA11yWindowManager.findWindowInfoByIdLocked(resolvedWindowId);
+        return mSystemSupport.getWindowTransformationMatrixAndMagnificationSpec(resolvedWindowId);
-        if (windowInfo == null) {
-            Slog.w(LOG_TAG, "getTransformMatrixAndSpec, windowInfo is null window id = "
-                    + resolvedWindowId);
-            return new Pair<>(null, null);
-        }
-
-        final MagnificationSpec spec = new MagnificationSpec();
-        spec.setTo(windowInfo.mMagnificationSpec);
-        return new Pair<>(windowInfo.mTransformMatrix, spec);
     }
 
     /**

services/accessibility/java/com/android/server/accessibility/AccessibilityManagerService.java

@@ -68,6 +68,7 @@ import android.content.pm.PackageManagerInternal;
 import android.content.pm.ResolveInfo;
 import android.content.pm.ServiceInfo;
 import android.database.ContentObserver;
+import android.graphics.Matrix;
 import android.graphics.Point;
 import android.graphics.Rect;
 import android.graphics.Region;
@@ -99,6 +100,7 @@ import android.text.TextUtils;
 import android.text.TextUtils.SimpleStringSplitter;
 import android.util.ArraySet;
 import android.util.IntArray;
+import android.util.Pair;
 import android.util.Slog;
 import android.util.SparseArray;
 import android.view.Display;
@@ -457,6 +459,41 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub
         return mIsAccessibilityButtonShown;
     }
 
+    @Override
+    public Pair<float[], MagnificationSpec> getWindowTransformationMatrixAndMagnificationSpec(
+            int windowId) {
+        WindowInfo windowInfo;
+        synchronized (mLock) {
+            windowInfo = mA11yWindowManager.findWindowInfoByIdLocked(windowId);
+        }
+        if (windowInfo != null) {
+            final MagnificationSpec spec = new MagnificationSpec();
+            spec.setTo(windowInfo.mMagnificationSpec);
+            return new Pair<>(windowInfo.mTransformMatrix, spec);
+        } else {
+            // If the framework doesn't track windows, we fall back to get the pair of
+            // transformation matrix and MagnificationSpe from the WindowManagerService's
+            // WindowState.
+            IBinder token;
+            synchronized (mLock) {
+                token = mA11yWindowManager.getWindowTokenForUserAndWindowIdLocked(mCurrentUserId,
+                        windowId);
+            }
+            Pair<Matrix, MagnificationSpec> pair =
+                    mWindowManagerService.getWindowTransformationMatrixAndMagnificationSpec(token);
+            final float[] outTransformationMatrix = new float[9];
+            final Matrix tmpMatrix = pair.first;
+            final MagnificationSpec spec = pair.second;
+            if (!spec.isNop()) {
+                tmpMatrix.postScale(spec.scale, spec.scale);
+                tmpMatrix.postTranslate(spec.offsetX, spec.offsetY);
+            }
+            tmpMatrix.getValues(outTransformationMatrix);
+
+            return new Pair<>(outTransformationMatrix, pair.second);
+        }
+    }
+
     @Override
     public void onServiceInfoChangedLocked(AccessibilityUserState userState) {
         mSecurityPolicy.onBoundServicesChangedLocked(userState.mUserId,
@@ -3750,12 +3787,12 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub
                         boundsInScreenBeforeMagnification.centerY());
 
                 // Invert magnification if needed.
-                final WindowInfo windowInfo = mA11yWindowManager.findWindowInfoByIdLocked(
+                final Pair<float[], MagnificationSpec> pair =
-                        focus.getWindowId());
+                        getWindowTransformationMatrixAndMagnificationSpec(focus.getWindowId());
                 MagnificationSpec spec = null;
-                if (windowInfo != null) {
+                if (pair != null && pair.second != null) {
                     spec = new MagnificationSpec();
-                    spec.setTo(windowInfo.mMagnificationSpec);
+                    spec.setTo(pair.second);
                 }
 
                 if (spec != null && !spec.isNop()) {

services/core/java/com/android/server/wm/AccessibilityController.java

@@ -446,6 +446,43 @@ final class AccessibilityController {
         // Not relevant for the window observer.
     }
 
+    public Pair<Matrix, MagnificationSpec> getWindowTransformationMatrixAndMagnificationSpec(
+            IBinder token) {
+        synchronized (mService.mGlobalLock) {
+            final Matrix transformationMatrix = new Matrix();
+            final MagnificationSpec magnificationSpec = new MagnificationSpec();
+
+            final WindowState windowState = mService.mWindowMap.get(token);
+            if (windowState != null) {
+                windowState.getTransformationMatrix(new float[9], transformationMatrix);
+
+                if (hasCallbacks()) {
+                    final MagnificationSpec otherMagnificationSpec =
+                            getMagnificationSpecForWindow(windowState);
+                    if (otherMagnificationSpec != null && !otherMagnificationSpec.isNop()) {
+                        magnificationSpec.setTo(otherMagnificationSpec);
+                    }
+                }
+            }
+
+            return new Pair<>(transformationMatrix, magnificationSpec);
+        }
+    }
+
+    MagnificationSpec getMagnificationSpecForWindow(WindowState windowState) {
+        if (mAccessibilityTracing.isTracingEnabled(FLAGS_MAGNIFICATION_CALLBACK)) {
+            mAccessibilityTracing.logTrace(TAG + ".getMagnificationSpecForWindow",
+                    FLAGS_MAGNIFICATION_CALLBACK,
+                    "windowState={" + windowState + "}");
+        }
+        final int displayId = windowState.getDisplayId();
+        final DisplayMagnifier displayMagnifier = mDisplayMagnifiers.get(displayId);
+        if (displayMagnifier != null) {
+            return displayMagnifier.getMagnificationSpecForWindow(windowState);
+        }
+        return null;
+    }
+
     boolean hasCallbacks() {
         if (mAccessibilityTracing.isTracingEnabled(FLAGS_MAGNIFICATION_CALLBACK
                 | FLAGS_WINDOWS_FOR_ACCESSIBILITY_CALLBACK)) {

services/core/java/com/android/server/wm/WindowManagerInternal.java

@@ -23,11 +23,13 @@ import android.annotation.NonNull;
 import android.annotation.Nullable;
 import android.content.ClipData;
 import android.content.Context;
+import android.graphics.Matrix;
 import android.graphics.Rect;
 import android.graphics.Region;
 import android.hardware.display.DisplayManagerInternal;
 import android.os.Bundle;
 import android.os.IBinder;
+import android.util.Pair;
 import android.view.Display;
 import android.view.IInputFilter;
 import android.view.IRemoteAnimationFinishedCallback;
@@ -458,6 +460,17 @@ public abstract class WindowManagerInternal {
      */
     public abstract void getWindowFrame(IBinder token, Rect outBounds);
 
+    /**
+     * Get the transformation matrix and MagnificationSpec given its token.
+     *
+     * @param token The token.
+     * @return The pair of the transformation matrix and magnification spec.
+     */
+    // TODO (b/231663133): Long term solution for tracking window when the
+    //                     FLAG_RETRIEVE_INTERACTIVE_WINDOWS is unset.
+    public abstract Pair<Matrix, MagnificationSpec>
+            getWindowTransformationMatrixAndMagnificationSpec(IBinder token);
+
     /**
      * Opens the global actions dialog.
      */

services/core/java/com/android/server/wm/WindowManagerService.java

@@ -173,6 +173,7 @@ import android.content.res.Resources;
 import android.content.res.TypedArray;
 import android.database.ContentObserver;
 import android.graphics.Bitmap;
+import android.graphics.Matrix;
 import android.graphics.Point;
 import android.graphics.PointF;
 import android.graphics.Rect;
@@ -224,6 +225,7 @@ import android.util.ArraySet;
 import android.util.DisplayMetrics;
 import android.util.EventLog;
 import android.util.MergedConfiguration;
+import android.util.Pair;
 import android.util.Slog;
 import android.util.SparseBooleanArray;
 import android.util.TimeUtils;
@@ -7768,6 +7770,13 @@ public class WindowManagerService extends IWindowManager.Stub
             }
         }
 
+        @Override
+        public Pair<Matrix, MagnificationSpec> getWindowTransformationMatrixAndMagnificationSpec(
+                IBinder token) {
+            return mAccessibilityController
+                    .getWindowTransformationMatrixAndMagnificationSpec(token);
+        }
+
         @Override
         public void waitForAllWindowsDrawn(Runnable callback, long timeout, int displayId) {
             final WindowContainer container = displayId == INVALID_DISPLAY

services/tests/servicestests/src/com/android/server/accessibility/AbstractAccessibilityServiceConnectionTest.java

@@ -89,8 +89,10 @@ import android.os.PowerManager;
 import android.os.Process;
 import android.os.RemoteCallback;
 import android.os.RemoteException;
+import android.util.Pair;
 import android.view.Display;
 import android.view.KeyEvent;
+import android.view.MagnificationSpec;
 import android.view.accessibility.AccessibilityNodeInfo;
 import android.view.accessibility.AccessibilityWindowInfo;
 import android.view.accessibility.IAccessibilityInteractionConnection;
@@ -145,6 +147,8 @@ public class AbstractAccessibilityServiceConnectionTest {
     private static final int USER_ID = 1;
     private static final int USER_ID2 = 2;
     private static final int INTERACTION_ID = 199;
+    private static final Pair<float[], MagnificationSpec> FAKE_MATRIX_AND_MAG_SPEC =
+            new Pair<>(new float[9], new MagnificationSpec());
     private static final int PID = Process.myPid();
     private static final long TID = Process.myTid();
     private static final int UID = Process.myUid();
@@ -188,6 +192,8 @@ public class AbstractAccessibilityServiceConnectionTest {
                 .thenReturn(mMockFingerprintGestureDispatcher);
         when(mMockSystemSupport.getMagnificationProcessor())
                 .thenReturn(mMockMagnificationProcessor);
+        when(mMockSystemSupport.getWindowTransformationMatrixAndMagnificationSpec(anyInt()))
+                .thenReturn(FAKE_MATRIX_AND_MAG_SPEC);
 
         PowerManager powerManager =
                 new PowerManager(mMockContext, mMockIPowerManager, mMockIThermalService, mHandler);