From d3dcd106ef9262f44dda36cb681c19944ae4c32a Mon Sep 17 00:00:00 2001 From: Zixuan Qu Date: Tue, 15 Nov 2022 06:14:55 +0000 Subject: [PATCH] Override the locale hints when the app is running on a virtual device. 1) In VirtualDeviceManagerInternal, add a method that returns the preferred language based on for apps running on Virtual Device. 2) From InputMethodManagerService, check if the language hints in EditorInfo need to overrided upon starting input by calling the method above. Bug: 237537306 Test: atest FrameworksInputMethodSystemServerTests Change-Id: I907cf4573472b0ad538ba67a49891a19ae104ad9 Change-Id: Ifdfe17a14830887d61f0ed44425ffbe45c014607 --- .../hardware/input/VirtualKeyboardConfig.java | 4 + .../companion/virtual/VirtualDeviceImpl.java | 13 +++ .../virtual/VirtualDeviceManagerService.java | 16 +++ .../virtual/VirtualDeviceManagerInternal.java | 16 +++ .../InputMethodManagerService.java | 13 +++ ...odManagerServiceWindowGainedFocusTest.java | 21 ++++ .../VirtualDeviceManagerServiceTest.java | 100 ++++++++++++++++++ 7 files changed, 183 insertions(+) diff --git a/core/java/android/hardware/input/VirtualKeyboardConfig.java b/core/java/android/hardware/input/VirtualKeyboardConfig.java index 28d8a0ff553ec..d788df49bfc56 100644 --- a/core/java/android/hardware/input/VirtualKeyboardConfig.java +++ b/core/java/android/hardware/input/VirtualKeyboardConfig.java @@ -124,6 +124,10 @@ public final class VirtualKeyboardConfig extends VirtualInputDeviceConfig implem * Note that the preferred layout is not guaranteed. If the specified language is * well-formed but not supported, the keyboard will be using English US QWERTY layout. * + * In case where the owning Virtual Device has created multiple virtual keyboards, only the + * {@code languageTag} of the most recent virtual keyboard will be kept to hint the locale + * of the Virtual Device. + * * @throws IllegalArgumentException if either of the language or country is not present in * the language tag. */ diff --git a/services/companion/java/com/android/server/companion/virtual/VirtualDeviceImpl.java b/services/companion/java/com/android/server/companion/virtual/VirtualDeviceImpl.java index b6cd160af80e1..38913b1d8c9ee 100644 --- a/services/companion/java/com/android/server/companion/virtual/VirtualDeviceImpl.java +++ b/services/companion/java/com/android/server/companion/virtual/VirtualDeviceImpl.java @@ -60,6 +60,7 @@ import android.hardware.input.VirtualTouchEvent; import android.hardware.input.VirtualTouchscreenConfig; import android.os.Binder; import android.os.IBinder; +import android.os.LocaleList; import android.os.Looper; import android.os.PowerManager; import android.os.RemoteException; @@ -123,6 +124,9 @@ final class VirtualDeviceImpl extends IVirtualDevice.Stub // The default setting for showing the pointer on new displays. @GuardedBy("mVirtualDeviceLock") private boolean mDefaultShowPointerIcon = true; + @GuardedBy("mVirtualDeviceLock") + @Nullable + private LocaleList mLocaleList = null; private ActivityListener createListenerAdapter() { return new ActivityListener() { @@ -247,6 +251,13 @@ final class VirtualDeviceImpl extends IVirtualDevice.Stub return mParams.getName(); } + /** Returns the locale of the device. */ + LocaleList getDeviceLocaleList() { + synchronized (mVirtualDeviceLock) { + return mLocaleList; + } + } + /** Returns the policy specified for this policy type */ public @VirtualDeviceParams.DevicePolicy int getDevicePolicy( @VirtualDeviceParams.PolicyType int policyType) { @@ -334,6 +345,7 @@ final class VirtualDeviceImpl extends IVirtualDevice.Stub mVirtualAudioController.stopListening(); mVirtualAudioController = null; } + mLocaleList = null; } mOnDeviceCloseListener.onClose(mDeviceId); mAppToken.unlinkToDeath(this, 0); @@ -435,6 +447,7 @@ final class VirtualDeviceImpl extends IVirtualDevice.Stub "Cannot create a virtual keyboard for a display not associated with " + "this virtual device"); } + mLocaleList = LocaleList.forLanguageTags(config.getLanguageTag()); } final long ident = Binder.clearCallingIdentity(); try { diff --git a/services/companion/java/com/android/server/companion/virtual/VirtualDeviceManagerService.java b/services/companion/java/com/android/server/companion/virtual/VirtualDeviceManagerService.java index da2c5162e6e18..563444f3f17f0 100644 --- a/services/companion/java/com/android/server/companion/virtual/VirtualDeviceManagerService.java +++ b/services/companion/java/com/android/server/companion/virtual/VirtualDeviceManagerService.java @@ -39,6 +39,7 @@ import android.hardware.display.VirtualDisplayConfig; import android.os.Binder; import android.os.Handler; import android.os.IBinder; +import android.os.LocaleList; import android.os.Looper; import android.os.Parcel; import android.os.Process; @@ -579,6 +580,21 @@ public class VirtualDeviceManagerService extends SystemService { return ((VirtualDeviceImpl) virtualDevice).getBaseVirtualDisplayFlags(); } + @Override + @Nullable + public LocaleList getPreferredLocaleListForUid(int uid) { + // TODO: b/263188984 support the case where an app is running on multiple VDs + synchronized (mVirtualDeviceManagerLock) { + for (int i = 0; i < mAppsOnVirtualDevices.size(); i++) { + if (mAppsOnVirtualDevices.valueAt(i).contains(uid)) { + int deviceId = mAppsOnVirtualDevices.keyAt(i); + return mVirtualDevices.get(deviceId).getDeviceLocaleList(); + } + } + } + return null; + } + @Override public boolean isAppRunningOnAnyVirtualDevice(int uid) { synchronized (mVirtualDeviceManagerLock) { diff --git a/services/core/java/com/android/server/companion/virtual/VirtualDeviceManagerInternal.java b/services/core/java/com/android/server/companion/virtual/VirtualDeviceManagerInternal.java index 09bec5e1d028f..e3ea1a6a3de77 100644 --- a/services/core/java/com/android/server/companion/virtual/VirtualDeviceManagerInternal.java +++ b/services/core/java/com/android/server/companion/virtual/VirtualDeviceManagerInternal.java @@ -17,7 +17,9 @@ package com.android.server.companion.virtual; import android.annotation.NonNull; +import android.annotation.Nullable; import android.companion.virtual.IVirtualDevice; +import android.os.LocaleList; import java.util.Set; @@ -110,6 +112,20 @@ public abstract class VirtualDeviceManagerInternal { */ public abstract int getBaseVirtualDisplayFlags(IVirtualDevice virtualDevice); + /** + * Returns the preferred locale hints of the Virtual Device on which the given app is running, + * or {@code null} if the hosting virtual device doesn't have a virtual keyboard or the app is + * not on any virtual device. + * + * If an app is on multiple virtual devices, the locale of the virtual device created the + * earliest will be returned. + * + * See {@link android.hardware.input.VirtualKeyboardConfig#setLanguageTag() for how the locale + * is specified for virtual keyboard. + */ + @Nullable + public abstract LocaleList getPreferredLocaleListForUid(int uid); + /** * Returns true if the given {@code uid} is currently running on any virtual devices. This is * determined by whether the app has any activities in the task stack on a virtual-device-owned diff --git a/services/core/java/com/android/server/inputmethod/InputMethodManagerService.java b/services/core/java/com/android/server/inputmethod/InputMethodManagerService.java index 97c8305f546f0..5840acfad6757 100644 --- a/services/core/java/com/android/server/inputmethod/InputMethodManagerService.java +++ b/services/core/java/com/android/server/inputmethod/InputMethodManagerService.java @@ -183,6 +183,7 @@ import com.android.server.LocalServices; import com.android.server.ServiceThread; import com.android.server.SystemServerInitThreadPool; import com.android.server.SystemService; +import com.android.server.companion.virtual.VirtualDeviceManagerInternal; import com.android.server.input.InputManagerInternal; import com.android.server.inputmethod.InputMethodManagerInternal.InputMethodListListener; import com.android.server.inputmethod.InputMethodSubtypeSwitchingController.ImeSubtypeListItem; @@ -304,6 +305,8 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub */ @Nullable private AudioManagerInternal mAudioManagerInternal = null; + @Nullable + private VirtualDeviceManagerInternal mVdmInternal = null; // All known input methods. final ArrayList mMethodList = new ArrayList<>(); @@ -2533,6 +2536,16 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub mCurVirtualDisplayToScreenMatrix = getVirtualDisplayToScreenMatrixLocked(cs.mSelfReportedDisplayId, mDisplayIdToShowIme); + // Override the locale hints if the app is running on a virtual device. + if (mVdmInternal == null) { + mVdmInternal = LocalServices.getService(VirtualDeviceManagerInternal.class); + } + if (mVdmInternal != null && editorInfo.hintLocales == null) { + LocaleList hintsFromVirtualDevice = mVdmInternal.getPreferredLocaleListForUid(cs.mUid); + if (hintsFromVirtualDevice != null) { + editorInfo.hintLocales = hintsFromVirtualDevice; + } + } mCurEditorInfo = editorInfo; // If configured, we want to avoid starting up the IME if it is not supposed to be showing diff --git a/services/tests/InputMethodSystemServerTests/src/com/android/server/inputmethod/InputMethodManagerServiceWindowGainedFocusTest.java b/services/tests/InputMethodSystemServerTests/src/com/android/server/inputmethod/InputMethodManagerServiceWindowGainedFocusTest.java index ffa2729434553..c6b355c2868fa 100644 --- a/services/tests/InputMethodSystemServerTests/src/com/android/server/inputmethod/InputMethodManagerServiceWindowGainedFocusTest.java +++ b/services/tests/InputMethodSystemServerTests/src/com/android/server/inputmethod/InputMethodManagerServiceWindowGainedFocusTest.java @@ -28,6 +28,8 @@ import static android.view.WindowManager.LayoutParams.SOFT_INPUT_STATE_UNCHANGED import static android.view.WindowManager.LayoutParams.SOFT_INPUT_STATE_UNSPECIFIED; import static android.view.WindowManager.LayoutParams.SOFT_INPUT_STATE_VISIBLE; +import static com.android.dx.mockito.inline.extended.ExtendedMockito.doReturn; + import static com.google.common.truth.Truth.assertThat; import static org.mockito.ArgumentMatchers.any; @@ -35,6 +37,7 @@ import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.Mockito.when; import android.os.IBinder; +import android.os.LocaleList; import android.os.RemoteException; import android.util.Log; import android.view.inputmethod.EditorInfo; @@ -47,11 +50,14 @@ import com.android.internal.inputmethod.InputBindResult; import com.android.internal.inputmethod.InputMethodDebug; import com.android.internal.inputmethod.StartInputFlags; import com.android.internal.inputmethod.StartInputReason; +import com.android.server.LocalServices; +import com.android.server.companion.virtual.VirtualDeviceManagerInternal; import com.android.server.wm.WindowManagerInternal; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +import org.mockito.Mock; import java.util.ArrayList; import java.util.List; @@ -84,6 +90,8 @@ public class InputMethodManagerServiceWindowGainedFocusTest }; private static final int DEFAULT_SOFT_INPUT_FLAG = StartInputFlags.VIEW_HAS_FOCUS | StartInputFlags.IS_TEXT_EDITOR; + @Mock + VirtualDeviceManagerInternal mMockVdmInternal; @Parameterized.Parameters(name = "softInputState={0}, softInputAdjustment={1}") public static List softInputModeConfigs() { @@ -256,6 +264,19 @@ public class InputMethodManagerServiceWindowGainedFocusTest mMockImeOnBackInvokedDispatcher /* imeDispatcher */); } + @Test + public void startInputOrWindowGainedFocus_localeHintsOverride() throws RemoteException { + doReturn(mMockVdmInternal).when( + () -> LocalServices.getService(VirtualDeviceManagerInternal.class)); + LocaleList overrideLocale = LocaleList.forLanguageTags("zh-CN"); + doReturn(overrideLocale).when(mMockVdmInternal).getPreferredLocaleListForUid(anyInt()); + mockHasImeFocusAndRestoreImeVisibility(false /* restoreImeVisibility */); + + assertThat(startInputOrWindowGainedFocus(DEFAULT_SOFT_INPUT_FLAG, + true /* forwardNavigation */)).isEqualTo(SUCCESS_WAITING_IME_BINDING_RESULT); + assertThat(mEditorInfo.hintLocales).isEqualTo(overrideLocale); + } + private void mockHasImeFocusAndRestoreImeVisibility(boolean restoreImeVisibility) { when(mMockWindowManagerInternal.hasInputMethodClientFocus( any(), anyInt(), anyInt(), anyInt())) diff --git a/services/tests/servicestests/src/com/android/server/companion/virtual/VirtualDeviceManagerServiceTest.java b/services/tests/servicestests/src/com/android/server/companion/virtual/VirtualDeviceManagerServiceTest.java index 0a4ae6f0b2060..cb1955e2f8fba 100644 --- a/services/tests/servicestests/src/com/android/server/companion/virtual/VirtualDeviceManagerServiceTest.java +++ b/services/tests/servicestests/src/com/android/server/companion/virtual/VirtualDeviceManagerServiceTest.java @@ -83,6 +83,7 @@ import android.os.Handler; import android.os.IBinder; import android.os.IPowerManager; import android.os.IThermalService; +import android.os.LocaleList; import android.os.PowerManager; import android.os.Process; import android.os.RemoteException; @@ -501,6 +502,63 @@ public class VirtualDeviceManagerServiceTest { mDeviceImpl.getDeviceId(), secondDevice.getDeviceId()); } + @Test + public void getPreferredLocaleListForApp_keyboardAttached_returnLocaleHints() { + mDeviceImpl.mVirtualDisplayIds.add(DISPLAY_ID); + + mDeviceImpl.createVirtualKeyboard(KEYBOARD_CONFIG, BINDER); + + mVdms.notifyRunningAppsChanged(mDeviceImpl.getDeviceId(), Sets.newArraySet(UID_1)); + + LocaleList localeList = mLocalService.getPreferredLocaleListForUid(UID_1); + assertThat(localeList).isEqualTo( + LocaleList.forLanguageTags(KEYBOARD_CONFIG.getLanguageTag())); + } + + @Test + public void getPreferredLocaleListForApp_noKeyboardAttached_nullLocaleHints() { + mVdms.notifyRunningAppsChanged(mDeviceImpl.getDeviceId(), Sets.newArraySet(UID_1)); + + // no preceding call to createVirtualKeyboard() + assertThat(mLocalService.getPreferredLocaleListForUid(UID_1)).isNull(); + } + + @Test + public void getPreferredLocaleListForApp_appOnMultipleVD_localeOnFirstVDReturned() { + int secondDeviceId = VIRTUAL_DEVICE_ID + 1; + VirtualDeviceImpl secondDevice = createVirtualDevice(secondDeviceId, DEVICE_OWNER_UID_2); + Binder secondBinder = new Binder("secondBinder"); + VirtualKeyboardConfig firstKeyboardConfig = + new VirtualKeyboardConfig.Builder() + .setVendorId(VENDOR_ID) + .setProductId(PRODUCT_ID) + .setInputDeviceName(DEVICE_NAME) + .setAssociatedDisplayId(DISPLAY_ID) + .setLanguageTag("zh-CN") + .build(); + VirtualKeyboardConfig secondKeyboardConfig = + new VirtualKeyboardConfig.Builder() + .setVendorId(VENDOR_ID) + .setProductId(PRODUCT_ID) + .setInputDeviceName(DEVICE_NAME) + .setAssociatedDisplayId(DISPLAY_ID_2) + .setLanguageTag("fr-FR") + .build(); + + mDeviceImpl.mVirtualDisplayIds.add(DISPLAY_ID); + secondDevice.mVirtualDisplayIds.add(DISPLAY_ID_2); + + mDeviceImpl.createVirtualKeyboard(firstKeyboardConfig, BINDER); + secondDevice.createVirtualKeyboard(secondKeyboardConfig, secondBinder); + + mVdms.notifyRunningAppsChanged(mDeviceImpl.getDeviceId(), Sets.newArraySet(UID_1)); + mVdms.notifyRunningAppsChanged(secondDevice.getDeviceId(), Sets.newArraySet(UID_1)); + + LocaleList localeList = mLocalService.getPreferredLocaleListForUid(UID_1); + assertThat(localeList).isEqualTo( + LocaleList.forLanguageTags(firstKeyboardConfig.getLanguageTag())); + } + @Test public void onVirtualDisplayRemovedLocked_doesNotThrowException() { mDeviceImpl.onVirtualDisplayCreatedLocked( @@ -882,6 +940,48 @@ public class VirtualDeviceManagerServiceTest { eq(PRODUCT_ID), anyString()); } + @Test + public void createVirtualKeyboard_keyboardCreated_localeUpdated() { + mDeviceImpl.mVirtualDisplayIds.add(DISPLAY_ID); + mDeviceImpl.createVirtualKeyboard(KEYBOARD_CONFIG, BINDER); + assertWithMessage("Virtual keyboard should register fd when the display matches") + .that(mInputController.getInputDeviceDescriptors()) + .isNotEmpty(); + verify(mNativeWrapperMock).openUinputKeyboard(eq(DEVICE_NAME), eq(VENDOR_ID), + eq(PRODUCT_ID), anyString()); + assertThat(mDeviceImpl.getDeviceLocaleList()).isEqualTo( + LocaleList.forLanguageTags(KEYBOARD_CONFIG.getLanguageTag())); + } + + @Test + public void createVirtualKeyboard_keyboardWithoutExplicitLayoutInfo_localeUpdatedWithDefault() { + VirtualKeyboardConfig configWithoutExplicitLayoutInfo = + new VirtualKeyboardConfig.Builder() + .setVendorId(VENDOR_ID) + .setProductId(PRODUCT_ID) + .setInputDeviceName(DEVICE_NAME) + .setAssociatedDisplayId(DISPLAY_ID) + .build(); + + mDeviceImpl.mVirtualDisplayIds.add(DISPLAY_ID); + mDeviceImpl.createVirtualKeyboard(configWithoutExplicitLayoutInfo, BINDER); + assertWithMessage("Virtual keyboard should register fd when the display matches") + .that(mInputController.getInputDeviceDescriptors()) + .isNotEmpty(); + verify(mNativeWrapperMock).openUinputKeyboard(eq(DEVICE_NAME), eq(VENDOR_ID), + eq(PRODUCT_ID), anyString()); + assertThat(mDeviceImpl.getDeviceLocaleList()).isEqualTo( + LocaleList.forLanguageTags(VirtualKeyboardConfig.DEFAULT_LANGUAGE_TAG)); + } + + @Test + public void virtualDeviceWithoutKeyboard_noLocaleUpdate() { + mDeviceImpl.mVirtualDisplayIds.add(DISPLAY_ID); + + // no preceding call to createVirtualKeyboard() + assertThat(mDeviceImpl.getDeviceLocaleList()).isNull(); + } + @Test public void createVirtualMouse_hasDisplay_obtainFileDescriptor() { mDeviceImpl.mVirtualDisplayIds.add(DISPLAY_ID);