From f8fb7b24bc5f2dc82ef042008df6a119394fbf89 Mon Sep 17 00:00:00 2001 From: Jacky Kao Date: Tue, 26 Jan 2021 10:16:39 +0800 Subject: [PATCH] Logs magnification feature behavior. (2/2). Uses the atom MagnificationModeWithImeOnReported in westworld to log the activated mode when the IME window is shown on the screen. Adding a new callback API in the MagnificationCallback to monitor the IME window visibility changes. The A11y framework registers the callback when the magnification is enabled. It logs the related data when it receives the IME window visibility changes through this callback and the magnification is in the activation. Bug: 154021596 Test: a11y CTS & unit tests Test: make statsd_testdrive && ./out/host/linux-x86/bin/statsd_testdrive 346 Merged-In: I49b02e00d5a1131b388eeb923440f59a2b4f81a6 Change-Id: I49b02e00d5a1131b388eeb923440f59a2b4f81a6 (cherry picked from commit 1aa113d8dde452f7196d55da74186e518691c197) --- core/java/android/provider/Settings.java | 6 ++ .../util/AccessibilityStatsLogUtils.java | 11 ++++ .../FullScreenMagnificationController.java | 42 +++++++++++--- .../MagnificationController.java | 57 ++++++++++++++++++- .../server/wm/AccessibilityController.java | 28 ++++++++- .../server/wm/WindowManagerInternal.java | 15 +++-- .../com/android/server/wm/WindowState.java | 3 + ...FullScreenMagnificationControllerTest.java | 15 ++++- ...ScreenMagnificationGestureHandlerTest.java | 6 +- .../MagnificationControllerTest.java | 41 +++++++++++++ 10 files changed, 203 insertions(+), 21 deletions(-) diff --git a/core/java/android/provider/Settings.java b/core/java/android/provider/Settings.java index 719c38392fb10..13458e6e3fe0c 100644 --- a/core/java/android/provider/Settings.java +++ b/core/java/android/provider/Settings.java @@ -9725,6 +9725,12 @@ public final class Settings { public static final String ACCESSIBILITY_MAGNIFICATION_MODE = "accessibility_magnification_mode"; + /** + * Magnification mode value that is a default value for the magnification logging feature. + * @hide + */ + public static final int ACCESSIBILITY_MAGNIFICATION_MODE_NONE = 0x0; + /** * Magnification mode value that magnifies whole display. * @hide diff --git a/core/java/com/android/internal/accessibility/util/AccessibilityStatsLogUtils.java b/core/java/com/android/internal/accessibility/util/AccessibilityStatsLogUtils.java index 7baa53bcd56d2..a600a948222a4 100644 --- a/core/java/com/android/internal/accessibility/util/AccessibilityStatsLogUtils.java +++ b/core/java/com/android/internal/accessibility/util/AccessibilityStatsLogUtils.java @@ -133,6 +133,17 @@ public final class AccessibilityStatsLogUtils { duration); } + /** + * Logs the activated mode of the magnification when the IME window is shown on the screen. + * Calls this when the magnification is enabled and the IME window is shown on the screen. + * + * @param mode The activated magnification mode. + */ + public static void logMagnificationModeWithImeOn(int mode) { + FrameworkStatsLog.write(FrameworkStatsLog.MAGNIFICATION_MODE_WITH_IME_ON_REPORTED, + convertToLoggingMagnificationMode(mode)); + } + private static int convertToLoggingShortcutType(@ShortcutType int shortcutType) { switch (shortcutType) { case ACCESSIBILITY_BUTTON: diff --git a/services/accessibility/java/com/android/server/accessibility/magnification/FullScreenMagnificationController.java b/services/accessibility/java/com/android/server/accessibility/magnification/FullScreenMagnificationController.java index 5b74cbd9621e9..1f66bfdb20c11 100644 --- a/services/accessibility/java/com/android/server/accessibility/magnification/FullScreenMagnificationController.java +++ b/services/accessibility/java/com/android/server/accessibility/magnification/FullScreenMagnificationController.java @@ -79,7 +79,7 @@ public class FullScreenMagnificationController { private final ScreenStateObserver mScreenStateObserver; - private final MagnificationRequestObserver mMagnificationRequestObserver; + private final MagnificationInfoChangedCallback mMagnificationInfoChangedCallback; private int mUserId; @@ -284,6 +284,14 @@ public class FullScreenMagnificationController { mControllerCtx.getHandler().sendMessage(m); } + @Override + public void onImeWindowVisibilityChanged(boolean shown) { + final Message m = PooledLambda.obtainMessage( + FullScreenMagnificationController::notifyImeWindowVisibilityChanged, + FullScreenMagnificationController.this, shown); + mControllerCtx.getHandler().sendMessage(m); + } + /** * Update our copy of the current magnification region * @@ -329,7 +337,7 @@ public class FullScreenMagnificationController { final boolean lastMagnificationActivated = mMagnificationActivated; mMagnificationActivated = spec.scale > 1.0f; if (mMagnificationActivated != lastMagnificationActivated) { - mMagnificationRequestObserver.onFullScreenMagnificationActivationState( + mMagnificationInfoChangedCallback.onFullScreenMagnificationActivationState( mMagnificationActivated); } } @@ -498,7 +506,7 @@ public class FullScreenMagnificationController { sendSpecToAnimation(mCurrentMagnificationSpec, animationCallback); if (isMagnifying() && (id != INVALID_ID)) { mIdOfLastServiceToMagnify = id; - mMagnificationRequestObserver.onRequestMagnificationSpec(mDisplayId, + mMagnificationInfoChangedCallback.onRequestMagnificationSpec(mDisplayId, mIdOfLastServiceToMagnify); } return changed; @@ -631,12 +639,12 @@ public class FullScreenMagnificationController { */ public FullScreenMagnificationController(@NonNull Context context, @NonNull AccessibilityManagerService ams, @NonNull Object lock, - @NonNull MagnificationRequestObserver magnificationRequestObserver) { + @NonNull MagnificationInfoChangedCallback magnificationInfoChangedCallback) { this(new ControllerContext(context, ams, LocalServices.getService(WindowManagerInternal.class), new Handler(context.getMainLooper()), context.getResources().getInteger(R.integer.config_longAnimTime)), lock, - magnificationRequestObserver); + magnificationInfoChangedCallback); } /** @@ -645,12 +653,12 @@ public class FullScreenMagnificationController { @VisibleForTesting public FullScreenMagnificationController(@NonNull ControllerContext ctx, @NonNull Object lock, - @NonNull MagnificationRequestObserver magnificationRequestObserver) { + @NonNull MagnificationInfoChangedCallback magnificationInfoChangedCallback) { mControllerCtx = ctx; mLock = lock; mMainThreadId = mControllerCtx.getContext().getMainLooper().getThread().getId(); mScreenStateObserver = new ScreenStateObserver(mControllerCtx.getContext(), this); - mMagnificationRequestObserver = magnificationRequestObserver; + mMagnificationInfoChangedCallback = magnificationInfoChangedCallback; } /** @@ -1167,6 +1175,16 @@ public class FullScreenMagnificationController { } } + /** + * Notifies that the IME window visibility changed. + * + * @param shown {@code true} means the IME window shows on the screen. Otherwise it's + * hidden. + */ + void notifyImeWindowVisibilityChanged(boolean shown) { + mMagnificationInfoChangedCallback.onImeWindowVisibilityChanged(shown); + } + /** * Returns {@code true} if the magnifiable regions of the display is forced to be shown. * @@ -1528,7 +1546,7 @@ public class FullScreenMagnificationController { return animate ? STUB_ANIMATION_CALLBACK : null; } - interface MagnificationRequestObserver { + interface MagnificationInfoChangedCallback { /** * Called when the {@link MagnificationSpec} is changed with non-default @@ -1545,7 +1563,13 @@ public class FullScreenMagnificationController { * * @param activated {@code true} if the magnification is activated, otherwise {@code false}. */ - @GuardedBy("mLock") void onFullScreenMagnificationActivationState(boolean activated); + + /** + * Called when the IME window visibility changed. + * @param shown {@code true} means the IME window shows on the screen. Otherwise it's + * hidden. + */ + void onImeWindowVisibilityChanged(boolean shown); } } diff --git a/services/accessibility/java/com/android/server/accessibility/magnification/MagnificationController.java b/services/accessibility/java/com/android/server/accessibility/magnification/MagnificationController.java index 2073c7001fa44..878ebc5bbfbde 100644 --- a/services/accessibility/java/com/android/server/accessibility/magnification/MagnificationController.java +++ b/services/accessibility/java/com/android/server/accessibility/magnification/MagnificationController.java @@ -18,6 +18,7 @@ package com.android.server.accessibility.magnification; import static android.provider.Settings.Secure.ACCESSIBILITY_MAGNIFICATION_MODE_ALL; import static android.provider.Settings.Secure.ACCESSIBILITY_MAGNIFICATION_MODE_FULLSCREEN; +import static android.provider.Settings.Secure.ACCESSIBILITY_MAGNIFICATION_MODE_NONE; import static android.provider.Settings.Secure.ACCESSIBILITY_MAGNIFICATION_MODE_WINDOW; import android.annotation.NonNull; @@ -33,6 +34,7 @@ import android.util.SparseArray; import android.view.accessibility.MagnificationAnimationCallback; import com.android.internal.accessibility.util.AccessibilityStatsLogUtils; +import com.android.internal.annotations.GuardedBy; import com.android.internal.annotations.VisibleForTesting; import com.android.server.accessibility.AccessibilityManagerService; @@ -58,7 +60,7 @@ import com.android.server.accessibility.AccessibilityManagerService; */ public class MagnificationController implements WindowMagnificationManager.Callback, MagnificationGestureHandler.Callback, - FullScreenMagnificationController.MagnificationRequestObserver { + FullScreenMagnificationController.MagnificationInfoChangedCallback { private static final boolean DEBUG = false; private static final String TAG = "MagnificationController"; @@ -73,6 +75,10 @@ public class MagnificationController implements WindowMagnificationManager.Callb private WindowMagnificationManager mWindowMagnificationMgr; private int mMagnificationCapabilities = ACCESSIBILITY_MAGNIFICATION_MODE_FULLSCREEN; + @GuardedBy("mLock") + private int mActivatedMode = ACCESSIBILITY_MAGNIFICATION_MODE_NONE; + @GuardedBy("mLock") + private boolean mImeWindowVisible = false; private long mWindowModeEnabledTime = 0; private long mFullScreenModeEnabledTime = 0; @@ -216,9 +222,18 @@ public class MagnificationController implements WindowMagnificationManager.Callb public void onWindowMagnificationActivationState(boolean activated) { if (activated) { mWindowModeEnabledTime = SystemClock.uptimeMillis(); + + synchronized (mLock) { + mActivatedMode = ACCESSIBILITY_MAGNIFICATION_MODE_WINDOW; + } + logMagnificationModeWithImeOnIfNeeded(); } else { logMagnificationUsageState(ACCESSIBILITY_MAGNIFICATION_MODE_WINDOW, SystemClock.uptimeMillis() - mWindowModeEnabledTime); + + synchronized (mLock) { + mActivatedMode = ACCESSIBILITY_MAGNIFICATION_MODE_NONE; + } } } @@ -226,12 +241,29 @@ public class MagnificationController implements WindowMagnificationManager.Callb public void onFullScreenMagnificationActivationState(boolean activated) { if (activated) { mFullScreenModeEnabledTime = SystemClock.uptimeMillis(); + + synchronized (mLock) { + mActivatedMode = ACCESSIBILITY_MAGNIFICATION_MODE_FULLSCREEN; + } + logMagnificationModeWithImeOnIfNeeded(); } else { logMagnificationUsageState(ACCESSIBILITY_MAGNIFICATION_MODE_FULLSCREEN, SystemClock.uptimeMillis() - mFullScreenModeEnabledTime); + + synchronized (mLock) { + mActivatedMode = ACCESSIBILITY_MAGNIFICATION_MODE_NONE; + } } } + @Override + public void onImeWindowVisibilityChanged(boolean shown) { + synchronized (mLock) { + mImeWindowVisible = shown; + } + logMagnificationModeWithImeOnIfNeeded(); + } + /** * Wrapper method of logging the magnification activated mode and its duration of the usage * when the magnification is disabled. @@ -244,6 +276,17 @@ public class MagnificationController implements WindowMagnificationManager.Callb AccessibilityStatsLogUtils.logMagnificationUsageState(mode, duration); } + /** + * Wrapper method of logging the activated mode of the magnification when the IME window + * is shown on the screen. + * + * @param mode The activated magnification mode. + */ + @VisibleForTesting + public void logMagnificationModeWithIme(int mode) { + AccessibilityStatsLogUtils.logMagnificationModeWithImeOn(mode); + } + /** * Updates the active user ID of {@link FullScreenMagnificationController} and {@link * WindowMagnificationManager}. @@ -295,6 +338,18 @@ public class MagnificationController implements WindowMagnificationManager.Callb } } + private void logMagnificationModeWithImeOnIfNeeded() { + final int mode; + + synchronized (mLock) { + if (!mImeWindowVisible || mActivatedMode == ACCESSIBILITY_MAGNIFICATION_MODE_NONE) { + return; + } + mode = mActivatedMode; + } + logMagnificationModeWithIme(mode); + } + /** * Getter of {@link FullScreenMagnificationController}. * diff --git a/services/core/java/com/android/server/wm/AccessibilityController.java b/services/core/java/com/android/server/wm/AccessibilityController.java index b947c8883d074..16692e169a3d3 100644 --- a/services/core/java/com/android/server/wm/AccessibilityController.java +++ b/services/core/java/com/android/server/wm/AccessibilityController.java @@ -514,6 +514,18 @@ final class AccessibilityController { } } + void onImeSurfaceShownChanged(WindowState windowState, boolean shown) { + if (mAccessibilityTracing.isEnabled()) { + mAccessibilityTracing.logState(TAG + ".onImeSurfaceShownChanged", + "windowState=" + windowState + "; shown=" + shown); + } + final int displayId = windowState.getDisplayId(); + final DisplayMagnifier displayMagnifier = mDisplayMagnifiers.get(displayId); + if (displayMagnifier != null) { + displayMagnifier.onImeSurfaceShownChanged(shown); + } + } + private static void populateTransformationMatrix(WindowState windowState, Matrix outMatrix) { windowState.getTransformationMatrix(sTempFloats, outMatrix); @@ -766,6 +778,15 @@ final class AccessibilityController { } } + void onImeSurfaceShownChanged(boolean shown) { + if (mAccessibilityTracing.isEnabled()) { + mAccessibilityTracing.logState( + LOG_TAG + ".onImeSurfaceShownChanged", "shown=" + shown); + } + mHandler.obtainMessage(MyHandler.MESSAGE_NOTIFY_IME_WINDOW_VISIBILITY_CHANGED, + shown ? 1 : 0, 0).sendToTarget(); + } + MagnificationSpec getMagnificationSpecForWindow(WindowState windowState) { if (mAccessibilityTracing.isEnabled()) { mAccessibilityTracing.logState(LOG_TAG + ".getMagnificationSpecForWindow", @@ -1337,6 +1358,7 @@ final class AccessibilityController { public static final int MESSAGE_NOTIFY_USER_CONTEXT_CHANGED = 3; public static final int MESSAGE_NOTIFY_ROTATION_CHANGED = 4; public static final int MESSAGE_SHOW_MAGNIFIED_REGION_BOUNDS_IF_NEEDED = 5; + public static final int MESSAGE_NOTIFY_IME_WINDOW_VISIBILITY_CHANGED = 6; MyHandler(Looper looper) { super(looper); @@ -1380,6 +1402,11 @@ final class AccessibilityController { } } } break; + + case MESSAGE_NOTIFY_IME_WINDOW_VISIBILITY_CHANGED: { + final boolean shown = message.arg1 == 1; + mCallbacks.onImeWindowVisibilityChanged(shown); + } break; } } } @@ -2123,5 +2150,4 @@ final class AccessibilityController { } } } - } diff --git a/services/core/java/com/android/server/wm/WindowManagerInternal.java b/services/core/java/com/android/server/wm/WindowManagerInternal.java index 8148f15981b32..ab515d448f303 100644 --- a/services/core/java/com/android/server/wm/WindowManagerInternal.java +++ b/services/core/java/com/android/server/wm/WindowManagerInternal.java @@ -113,7 +113,7 @@ public abstract class WindowManagerInternal { * * @param magnificationRegion the current magnification region */ - public void onMagnificationRegionChanged(Region magnificationRegion); + void onMagnificationRegionChanged(Region magnificationRegion); /** * Called when an application requests a rectangle on the screen to allow @@ -124,20 +124,27 @@ public abstract class WindowManagerInternal { * @param right The rectangle right. * @param bottom The rectangle bottom. */ - public void onRectangleOnScreenRequested(int left, int top, int right, int bottom); + void onRectangleOnScreenRequested(int left, int top, int right, int bottom); /** * Notifies that the rotation changed. * * @param rotation The current rotation. */ - public void onRotationChanged(int rotation); + void onRotationChanged(int rotation); /** * Notifies that the context of the user changed. For example, an application * was started. */ - public void onUserContextChanged(); + void onUserContextChanged(); + + /** + * Notifies that the IME window visibility changed. + * @param shown {@code true} means the IME window shows on the screen. Otherwise it's + * hidden. + */ + void onImeWindowVisibilityChanged(boolean shown); } /** diff --git a/services/core/java/com/android/server/wm/WindowState.java b/services/core/java/com/android/server/wm/WindowState.java index 48d4fc5464b00..f3fb9ef545891 100644 --- a/services/core/java/com/android/server/wm/WindowState.java +++ b/services/core/java/com/android/server/wm/WindowState.java @@ -3453,6 +3453,9 @@ class WindowState extends WindowContainer implements WindowManagerP if (mAttrs.type >= FIRST_SYSTEM_WINDOW && mAttrs.type != TYPE_TOAST) { mWmService.mAtmService.mActiveUids.onNonAppSurfaceVisibilityChanged(mOwnerUid, shown); } + if (mIsImWindow && mWmService.mAccessibilityController != null) { + mWmService.mAccessibilityController.onImeSurfaceShownChanged(this, shown); + } } private void logExclusionRestrictions(int side) { diff --git a/services/tests/servicestests/src/com/android/server/accessibility/magnification/FullScreenMagnificationControllerTest.java b/services/tests/servicestests/src/com/android/server/accessibility/magnification/FullScreenMagnificationControllerTest.java index 29691fb3583a3..502f64a1e50b2 100644 --- a/services/tests/servicestests/src/com/android/server/accessibility/magnification/FullScreenMagnificationControllerTest.java +++ b/services/tests/servicestests/src/com/android/server/accessibility/magnification/FullScreenMagnificationControllerTest.java @@ -16,7 +16,7 @@ package com.android.server.accessibility.magnification; -import static com.android.server.accessibility.magnification.FullScreenMagnificationController.MagnificationRequestObserver; +import static com.android.server.accessibility.magnification.FullScreenMagnificationController.MagnificationInfoChangedCallback; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -96,8 +96,8 @@ public class FullScreenMagnificationControllerTest { final WindowManagerInternal mMockWindowManager = mock(WindowManagerInternal.class); private final MagnificationAnimationCallback mAnimationCallback = mock( MagnificationAnimationCallback.class); - private final MagnificationRequestObserver mRequestObserver = mock( - MagnificationRequestObserver.class); + private final MagnificationInfoChangedCallback mRequestObserver = mock( + MagnificationInfoChangedCallback.class); final MessageCapturingHandler mMessageCapturingHandler = new MessageCapturingHandler(null); ValueAnimator mMockValueAnimator; @@ -1145,6 +1145,15 @@ public class FullScreenMagnificationControllerTest { verify(mRequestObserver).onFullScreenMagnificationActivationState(eq(false)); } + @Test + public void testImeWindowIsShown_serviceNotified() { + register(DISPLAY_0); + MagnificationCallbacks callbacks = getMagnificationCallbacks(DISPLAY_0); + callbacks.onImeWindowVisibilityChanged(true); + mMessageCapturingHandler.sendAllMessages(); + verify(mRequestObserver).onImeWindowVisibilityChanged(eq(true)); + } + private void setScaleToMagnifying() { register(DISPLAY_0); float scale = 2.0f; diff --git a/services/tests/servicestests/src/com/android/server/accessibility/magnification/FullScreenMagnificationGestureHandlerTest.java b/services/tests/servicestests/src/com/android/server/accessibility/magnification/FullScreenMagnificationGestureHandlerTest.java index 28a6ff7e3287a..f881f04e5b073 100644 --- a/services/tests/servicestests/src/com/android/server/accessibility/magnification/FullScreenMagnificationGestureHandlerTest.java +++ b/services/tests/servicestests/src/com/android/server/accessibility/magnification/FullScreenMagnificationGestureHandlerTest.java @@ -53,7 +53,7 @@ import androidx.test.runner.AndroidJUnit4; import com.android.server.accessibility.AccessibilityManagerService; import com.android.server.accessibility.EventStreamTransformation; -import com.android.server.accessibility.magnification.FullScreenMagnificationController.MagnificationRequestObserver; +import com.android.server.accessibility.magnification.FullScreenMagnificationController.MagnificationInfoChangedCallback; import com.android.server.testutils.OffsettableClock; import com.android.server.testutils.TestHandler; import com.android.server.wm.WindowManagerInternal; @@ -126,7 +126,7 @@ public class FullScreenMagnificationGestureHandlerTest { @Mock MagnificationGestureHandler.Callback mMockCallback; @Mock - MagnificationRequestObserver mMagnificationRequestObserver; + MagnificationInfoChangedCallback mMagnificationInfoChangedCallback; @Mock WindowMagnificationPromptController mWindowMagnificationPromptController; @@ -151,7 +151,7 @@ public class FullScreenMagnificationGestureHandlerTest { when(mockController.getAnimationDuration()).thenReturn(1000L); when(mockWindowManager.setMagnificationCallbacks(eq(DISPLAY_0), any())).thenReturn(true); mFullScreenMagnificationController = new FullScreenMagnificationController(mockController, - new Object(), mMagnificationRequestObserver) { + new Object(), mMagnificationInfoChangedCallback) { @Override public boolean magnificationRegionContains(int displayId, float x, float y) { return true; diff --git a/services/tests/servicestests/src/com/android/server/accessibility/magnification/MagnificationControllerTest.java b/services/tests/servicestests/src/com/android/server/accessibility/magnification/MagnificationControllerTest.java index cf231979171b3..3dff36e8f379d 100644 --- a/services/tests/servicestests/src/com/android/server/accessibility/magnification/MagnificationControllerTest.java +++ b/services/tests/servicestests/src/com/android/server/accessibility/magnification/MagnificationControllerTest.java @@ -486,6 +486,47 @@ public class MagnificationControllerTest { eq(MODE_WINDOW)); } + @Test + public void imeWindowStateShown_windowMagnifying_logWindowMode() { + mMagnificationController.onWindowMagnificationActivationState(true); + + mMagnificationController.onImeWindowVisibilityChanged(true); + + verify(mMagnificationController).logMagnificationModeWithIme( + eq(ACCESSIBILITY_MAGNIFICATION_MODE_WINDOW)); + } + + @Test + public void imeWindowStateShown_fullScreenMagnifying_logFullScreenMode() { + mMagnificationController.onFullScreenMagnificationActivationState(true); + + mMagnificationController.onImeWindowVisibilityChanged(true); + + verify(mMagnificationController).logMagnificationModeWithIme( + eq(ACCESSIBILITY_MAGNIFICATION_MODE_FULLSCREEN)); + } + + @Test + public void imeWindowStateShown_noMagnifying_noLogAnyMode() { + mMagnificationController.onImeWindowVisibilityChanged(true); + + verify(mMagnificationController, never()).logMagnificationModeWithIme(anyInt()); + } + + @Test + public void imeWindowStateHidden_windowMagnifying_noLogAnyMode() { + mMagnificationController.onFullScreenMagnificationActivationState(true); + + verify(mMagnificationController, never()).logMagnificationModeWithIme(anyInt()); + } + + @Test + public void imeWindowStateHidden_fullScreenMagnifying_noLogAnyMode() { + mMagnificationController.onWindowMagnificationActivationState(true); + + verify(mMagnificationController, never()).logMagnificationModeWithIme(anyInt()); + } + private void setMagnificationEnabled(int mode) throws RemoteException { setMagnificationEnabled(mode, MAGNIFIED_CENTER_X, MAGNIFIED_CENTER_Y);