From 8cdb9fedb964b2f5c84add75724fd9e1f2026785 Mon Sep 17 00:00:00 2001 From: Daniel Hsieh Date: Thu, 16 Dec 2021 13:15:26 +0000 Subject: [PATCH] Support following typing foucs in window mode [2/n]. There are 3 milestones in this feature. 1. Refactor the callbacks for Accessibility in WindowManagerInternal. 2. Implement this feature in such new architecture. 3. Implement the setting choice in preference page. This CL is for the 2nd milestone. We move the window magnification to the typing focus' center position based on the condition of whether a user takes control or not. We only make a movement when the control is not taken by a user or we don't preform the movement for the window magnification. There are 2 methods for a user to take the control. 1. A user use 1 finger to drag the window magnification. 2. A user use 2 finger to drag the window magnification. There is 1 method for a user to release the control. 1. When IME is shown, the control would be released. So, we can decide whether we should make a movement to typing focus given the condition of who take the control. Bug: 194668976 Test: atest MagnificationControllerTest atest WindowMagnificationTest atest WindowMagnificationControllerTest atest WindowMagnificationManagerTest Change-Id: I145f893d412b74c20afe1685449370d1dba99961 --- ...WindowMagnificationConnectionCallback.aidl | 9 ++ .../accessibility/WindowMagnification.java | 7 ++ .../WindowMagnificationConnectionImpl.java | 10 ++ .../WindowMagnificationController.java | 1 + .../WindowMagnifierCallback.java | 10 ++ .../WindowMagnificationTest.java | 10 ++ .../MagnificationController.java | 1 + .../WindowMagnificationManager.java | 107 +++++++++++++++++- .../WindowMagnificationManagerTest.java | 98 ++++++++++++++++ 9 files changed, 248 insertions(+), 5 deletions(-) diff --git a/core/java/android/view/accessibility/IWindowMagnificationConnectionCallback.aidl b/core/java/android/view/accessibility/IWindowMagnificationConnectionCallback.aidl index 1cb6825e426e0..722546eb06e46 100644 --- a/core/java/android/view/accessibility/IWindowMagnificationConnectionCallback.aidl +++ b/core/java/android/view/accessibility/IWindowMagnificationConnectionCallback.aidl @@ -67,4 +67,13 @@ import android.graphics.Rect; */ void onAccessibilityActionPerformed(int displayId); + /** + * Called when the user is performing dragging gesture. It is started after the offset + * between the down location and the move event location exceed + * {@link ViewConfiguration#getScaledTouchSlop()}. + * + * @param displayId The logical display id. + */ + void onDrag(int displayId); + } diff --git a/packages/SystemUI/src/com/android/systemui/accessibility/WindowMagnification.java b/packages/SystemUI/src/com/android/systemui/accessibility/WindowMagnification.java index a10efa982701d..4784bc12099b2 100644 --- a/packages/SystemUI/src/com/android/systemui/accessibility/WindowMagnification.java +++ b/packages/SystemUI/src/com/android/systemui/accessibility/WindowMagnification.java @@ -224,6 +224,13 @@ public class WindowMagnification extends CoreStartable implements WindowMagnifie } } + @Override + public void onDrag(int displayId) { + if (mWindowMagnificationConnectionImpl != null) { + mWindowMagnificationConnectionImpl.onDrag(displayId); + } + } + @Override public void requestWindowMagnificationConnection(boolean connect) { if (connect) { diff --git a/packages/SystemUI/src/com/android/systemui/accessibility/WindowMagnificationConnectionImpl.java b/packages/SystemUI/src/com/android/systemui/accessibility/WindowMagnificationConnectionImpl.java index 2133da202ce92..1d22633455e96 100644 --- a/packages/SystemUI/src/com/android/systemui/accessibility/WindowMagnificationConnectionImpl.java +++ b/packages/SystemUI/src/com/android/systemui/accessibility/WindowMagnificationConnectionImpl.java @@ -142,4 +142,14 @@ class WindowMagnificationConnectionImpl extends IWindowMagnificationConnection.S } } } + + void onDrag(int displayId) { + if (mConnectionCallback != null) { + try { + mConnectionCallback.onDrag(displayId); + } catch (RemoteException e) { + Log.e(TAG, "Failed to inform taking control by a user", e); + } + } + } } diff --git a/packages/SystemUI/src/com/android/systemui/accessibility/WindowMagnificationController.java b/packages/SystemUI/src/com/android/systemui/accessibility/WindowMagnificationController.java index b064ba904120f..0ffcea4f85701 100644 --- a/packages/SystemUI/src/com/android/systemui/accessibility/WindowMagnificationController.java +++ b/packages/SystemUI/src/com/android/systemui/accessibility/WindowMagnificationController.java @@ -960,6 +960,7 @@ class WindowMagnificationController implements View.OnTouchListener, SurfaceHold @Override public boolean onDrag(float offsetX, float offsetY) { moveWindowMagnifier(offsetX, offsetY); + mWindowMagnifierCallback.onDrag(mDisplayId); return true; } diff --git a/packages/SystemUI/src/com/android/systemui/accessibility/WindowMagnifierCallback.java b/packages/SystemUI/src/com/android/systemui/accessibility/WindowMagnifierCallback.java index 628a5e88b89e2..bdded10dfa1d4 100644 --- a/packages/SystemUI/src/com/android/systemui/accessibility/WindowMagnifierCallback.java +++ b/packages/SystemUI/src/com/android/systemui/accessibility/WindowMagnifierCallback.java @@ -17,6 +17,7 @@ package com.android.systemui.accessibility; import android.graphics.Rect; +import android.view.ViewConfiguration; /** * A callback to inform {@link com.android.server.accessibility.AccessibilityManagerService} about @@ -53,4 +54,13 @@ interface WindowMagnifierCallback { * @param displayId The logical display id. */ void onAccessibilityActionPerformed(int displayId); + + /** + * Called when the user is performing dragging gesture. It is started after the offset + * between the down location and the move event location exceed + * {@link ViewConfiguration#getScaledTouchSlop()}. + * + * @param displayId The logical display id. + */ + void onDrag(int displayId); } diff --git a/packages/SystemUI/tests/src/com/android/systemui/accessibility/WindowMagnificationTest.java b/packages/SystemUI/tests/src/com/android/systemui/accessibility/WindowMagnificationTest.java index c898150d857cd..343658d312729 100644 --- a/packages/SystemUI/tests/src/com/android/systemui/accessibility/WindowMagnificationTest.java +++ b/packages/SystemUI/tests/src/com/android/systemui/accessibility/WindowMagnificationTest.java @@ -148,6 +148,16 @@ public class WindowMagnificationTest extends SysuiTestCase { verify(mConnectionCallback).onAccessibilityActionPerformed(TEST_DISPLAY); } + @Test + public void onDrag_enabled_notifyCallback() throws RemoteException { + mCommandQueue.requestWindowMagnificationConnection(true); + waitForIdleSync(); + + mWindowMagnification.onDrag(TEST_DISPLAY); + + verify(mConnectionCallback).onDrag(TEST_DISPLAY); + } + @Test public void onConfigurationChanged_updateModeSwitches() { final Configuration config = new Configuration(); diff --git a/services/accessibility/java/com/android/server/accessibility/magnification/MagnificationController.java b/services/accessibility/java/com/android/server/accessibility/magnification/MagnificationController.java index 037dc1f6795ca..3d703063434a6 100644 --- a/services/accessibility/java/com/android/server/accessibility/magnification/MagnificationController.java +++ b/services/accessibility/java/com/android/server/accessibility/magnification/MagnificationController.java @@ -426,6 +426,7 @@ public class MagnificationController implements WindowMagnificationManager.Callb synchronized (mLock) { mImeWindowVisible = shown; } + getWindowMagnificationMgr().onImeWindowVisibilityChanged(shown); logMagnificationModeWithImeOnIfNeeded(); } diff --git a/services/accessibility/java/com/android/server/accessibility/magnification/WindowMagnificationManager.java b/services/accessibility/java/com/android/server/accessibility/magnification/WindowMagnificationManager.java index c4a577d6e4611..75ea1c1c074c7 100644 --- a/services/accessibility/java/com/android/server/accessibility/magnification/WindowMagnificationManager.java +++ b/services/accessibility/java/com/android/server/accessibility/magnification/WindowMagnificationManager.java @@ -291,13 +291,66 @@ public class WindowMagnificationManager implements @Override public void onRectangleOnScreenRequested(int displayId, int left, int top, int right, int bottom) { - // TODO(b/194668976): We will implement following typing focus in window mode after - // our refactor. + + float toCenterX = (float) (left + right) / 2; + float toCenterY = (float) (top + bottom) / 2; + + if (!isPositionInSourceBounds(displayId, toCenterX, toCenterY) + && isTrackingTypingFocusEnabled(displayId)) { + enableWindowMagnification(displayId, Float.NaN, toCenterX, toCenterY); + } + } + + /** + * Enable or disable tracking typing focus for the specific magnification window. + * + * The tracking typing focus should be set to enabled with the following conditions: + * 1. IME is shown. + * + * The tracking typing focus should be set to disabled with the following conditions: + * 1. A user drags the magnification window by 1 finger. + * 2. A user scroll the magnification window by 2 fingers. + * + * @param displayId The logical display id. + * @param trackingTypingFocusEnabled Enabled or disable the function of tracking typing focus. + */ + private void setTrackingTypingFocusEnabled(int displayId, boolean trackingTypingFocusEnabled) { + synchronized (mLock) { + WindowMagnifier magnifier = mWindowMagnifiers.get(displayId); + if (magnifier == null) { + return; + } + magnifier.setTrackingTypingFocusEnabled(trackingTypingFocusEnabled); + } + } + + /** + * Enable tracking typing focus function for all magnifications. + */ + private void enableAllTrackingTypingFocus() { + synchronized (mLock) { + for (int i = 0; i < mWindowMagnifiers.size(); i++) { + WindowMagnifier magnifier = mWindowMagnifiers.valueAt(i); + magnifier.setTrackingTypingFocusEnabled(true); + } + } + } + + /** + * Called when the IME window visibility changed. + * + * @param shown {@code true} means the IME window shows on the screen. Otherwise, it's hidden. + */ + void onImeWindowVisibilityChanged(boolean shown) { + if (shown) { + enableAllTrackingTypingFocus(); + } } @Override public boolean processScroll(int displayId, float distanceX, float distanceY) { moveWindowMagnification(displayId, -distanceX, -distanceY); + setTrackingTypingFocusEnabled(displayId, false); return /* event consumed: */ true; } @@ -469,6 +522,16 @@ public class WindowMagnificationManager implements } } + boolean isPositionInSourceBounds(int displayId, float x, float y) { + synchronized (mLock) { + WindowMagnifier magnifier = mWindowMagnifiers.get(displayId); + if (magnifier == null) { + return false; + } + return magnifier.isPositionInSourceBounds(x, y); + } + } + /** * Indicates whether window magnification is enabled on specified display. * @@ -598,6 +661,16 @@ public class WindowMagnificationManager implements } } + boolean isTrackingTypingFocusEnabled(int displayId) { + synchronized (mLock) { + WindowMagnifier magnifier = mWindowMagnifiers.get(displayId); + if (magnifier == null) { + return false; + } + return magnifier.isTrackingTypingFocusEnabled(); + } + } + /** * Populates magnified bounds on the screen. And the populated magnified bounds would be * empty If window magnifier is not activated. @@ -713,6 +786,17 @@ public class WindowMagnificationManager implements mCallback.onAccessibilityActionPerformed(displayId); } + @Override + public void onDrag(int displayId) { + if (mTrace.isA11yTracingEnabledForTypes( + FLAGS_WINDOW_MAGNIFICATION_CONNECTION_CALLBACK)) { + mTrace.logTrace(TAG + "ConnectionCallback.onDrag", + FLAGS_WINDOW_MAGNIFICATION_CONNECTION_CALLBACK, + "displayId=" + displayId); + } + setTrackingTypingFocusEnabled(displayId, false); + } + @Override public void binderDied() { synchronized (mLock) { @@ -749,7 +833,9 @@ public class WindowMagnificationManager implements private int mIdOfLastServiceToControl = INVALID_SERVICE_ID; - private PointF mMagnificationFrameOffsetRatio = new PointF(0f, 0f); + private final PointF mMagnificationFrameOffsetRatio = new PointF(0f, 0f); + + private boolean mTrackingTypingFocusEnabled = true; WindowMagnifier(int displayId, WindowMagnificationManager windowMagnificationManager) { mDisplayId = displayId; @@ -790,7 +876,6 @@ public class WindowMagnificationManager implements } } - @GuardedBy("mLock") boolean disableWindowMagnificationInternal( @Nullable MagnificationAnimationCallback animationResultCallback) { if (!mEnabled) { @@ -800,6 +885,7 @@ public class WindowMagnificationManager implements mDisplayId, animationResultCallback)) { mEnabled = false; mIdOfLastServiceToControl = INVALID_SERVICE_ID; + mTrackingTypingFocusEnabled = false; return true; } return false; @@ -848,7 +934,18 @@ public class WindowMagnificationManager implements return count; } - @GuardedBy("mLock") + boolean isPositionInSourceBounds(float x, float y) { + return mSourceBounds.contains((int) x, (int) y); + } + + void setTrackingTypingFocusEnabled(boolean trackingTypingFocusEnabled) { + mTrackingTypingFocusEnabled = trackingTypingFocusEnabled; + } + + boolean isTrackingTypingFocusEnabled() { + return mTrackingTypingFocusEnabled; + } + boolean isEnabled() { return mEnabled; } diff --git a/services/tests/servicestests/src/com/android/server/accessibility/magnification/WindowMagnificationManagerTest.java b/services/tests/servicestests/src/com/android/server/accessibility/magnification/WindowMagnificationManagerTest.java index a62c0d5e1eaf1..f3959e622ed5c 100644 --- a/services/tests/servicestests/src/com/android/server/accessibility/magnification/WindowMagnificationManagerTest.java +++ b/services/tests/servicestests/src/com/android/server/accessibility/magnification/WindowMagnificationManagerTest.java @@ -273,6 +273,104 @@ public class WindowMagnificationManagerTest { MagnificationScaleProvider.MAX_SCALE); } + @Test + public void onRectangleOnScreenRequested_trackingDisabledByOnDrag_withoutMovingMagnification() + throws RemoteException { + mWindowMagnificationManager.setConnection(mMockConnection.getConnection()); + mWindowMagnificationManager.enableWindowMagnification(TEST_DISPLAY, 3.0f, 50f, 50f); + final Region outRegion = new Region(); + mWindowMagnificationManager.getMagnificationSourceBounds(TEST_DISPLAY, outRegion); + final Rect requestedRect = outRegion.getBounds(); + requestedRect.offsetTo(requestedRect.right + 10, requestedRect.bottom + 10); + mMockConnection.getConnectionCallback().onDrag(TEST_DISPLAY); + + mWindowMagnificationManager.onRectangleOnScreenRequested(TEST_DISPLAY, + requestedRect.left, requestedRect.top, requestedRect.right, requestedRect.bottom); + + verify(mMockConnection.getConnection(), never()).enableWindowMagnification(eq(TEST_DISPLAY), + eq(3f), eq(requestedRect.exactCenterX()), eq(requestedRect.exactCenterY()), + eq(0f), eq(0f), notNull()); + } + + + @Test + public void onRectangleOnScreenRequested_trackingDisabledByScroll_withoutMovingMagnification() + throws RemoteException { + final float distanceX = 10f; + final float distanceY = 10f; + mWindowMagnificationManager.setConnection(mMockConnection.getConnection()); + mWindowMagnificationManager.enableWindowMagnification(TEST_DISPLAY, 3.0f, 50f, 50f); + final Region outRegion = new Region(); + mWindowMagnificationManager.getMagnificationSourceBounds(TEST_DISPLAY, outRegion); + final Rect requestedRect = outRegion.getBounds(); + requestedRect.offsetTo(requestedRect.right + 10, requestedRect.bottom + 10); + mWindowMagnificationManager.processScroll(TEST_DISPLAY, distanceX, distanceY); + + mWindowMagnificationManager.onRectangleOnScreenRequested(TEST_DISPLAY, + requestedRect.left, requestedRect.top, requestedRect.right, requestedRect.bottom); + + verify(mMockConnection.getConnection(), never()).enableWindowMagnification(eq(TEST_DISPLAY), + eq(3f), eq(requestedRect.exactCenterX()), eq(requestedRect.exactCenterY()), + eq(0f), eq(0f), notNull()); + } + + @Test + public void onRectangleOnScreenRequested_requestRectangleInBound_withoutMovingMagnification() + throws RemoteException { + mWindowMagnificationManager.setConnection(mMockConnection.getConnection()); + mWindowMagnificationManager.enableWindowMagnification(TEST_DISPLAY, 3.0f, 50f, 50f); + final Region outRegion = new Region(); + mWindowMagnificationManager.getMagnificationSourceBounds(TEST_DISPLAY, outRegion); + final Rect requestedRect = outRegion.getBounds(); + requestedRect.inset(-10, -10); + + mWindowMagnificationManager.onRectangleOnScreenRequested(TEST_DISPLAY, + requestedRect.left, requestedRect.top, requestedRect.right, requestedRect.bottom); + + verify(mMockConnection.getConnection(), never()).enableWindowMagnification(eq(TEST_DISPLAY), + eq(3f), eq(500f), eq(500f), eq(0f), eq(0f), notNull()); + } + + @Test + public void onRectangleOnScreenRequested_trackingEnabledByDefault_movingMagnification() + throws RemoteException { + mWindowMagnificationManager.setConnection(mMockConnection.getConnection()); + mWindowMagnificationManager.enableWindowMagnification(TEST_DISPLAY, 3.0f, 50f, 50f); + final Region outRegion = new Region(); + mWindowMagnificationManager.getMagnificationSourceBounds(TEST_DISPLAY, outRegion); + final Rect requestedRect = outRegion.getBounds(); + requestedRect.offsetTo(requestedRect.right + 10, requestedRect.bottom + 10); + + mWindowMagnificationManager.onRectangleOnScreenRequested(TEST_DISPLAY, + requestedRect.left, requestedRect.top, requestedRect.right, requestedRect.bottom); + + verify(mMockConnection.getConnection()).enableWindowMagnification(eq(TEST_DISPLAY), eq(3f), + eq(requestedRect.exactCenterX()), eq(requestedRect.exactCenterY()), + eq(0f), eq(0f), notNull()); + } + + @Test + public void onRectangleOnScreenRequested_trackingEnabledByDragAndReset_movingMagnification() + throws RemoteException { + final PointF initialPoint = new PointF(50f, 50f); + mWindowMagnificationManager.setConnection(mMockConnection.getConnection()); + mWindowMagnificationManager.enableWindowMagnification(TEST_DISPLAY, 3.0f, + initialPoint.x, initialPoint.y); + mMockConnection.getConnectionCallback().onDrag(TEST_DISPLAY); + mWindowMagnificationManager.onImeWindowVisibilityChanged(true); + final Region outRegion = new Region(); + mWindowMagnificationManager.getMagnificationSourceBounds(TEST_DISPLAY, outRegion); + final Rect requestedRect = outRegion.getBounds(); + requestedRect.offsetTo(requestedRect.right + 10, requestedRect.bottom + 10); + + mWindowMagnificationManager.onRectangleOnScreenRequested(TEST_DISPLAY, + requestedRect.left, requestedRect.top, requestedRect.right, requestedRect.bottom); + + verify(mMockConnection.getConnection()).enableWindowMagnification(eq(TEST_DISPLAY), + eq(3f), eq(requestedRect.exactCenterX()), eq(requestedRect.exactCenterY()), + eq(0f), eq(0f), notNull()); + } + @Test public void moveWindowMagnifier_enabled_invokeConnectionMethod() throws RemoteException { mWindowMagnificationManager.setConnection(mMockConnection.getConnection());