diff --git a/core/java/android/view/inputmethod/CursorAnchorInfo.java b/core/java/android/view/inputmethod/CursorAnchorInfo.java index 7f07146776ab0..a17089169b614 100644 --- a/core/java/android/view/inputmethod/CursorAnchorInfo.java +++ b/core/java/android/view/inputmethod/CursorAnchorInfo.java @@ -415,7 +415,7 @@ public final class CursorAnchorInfo implements Parcelable { "required when positional parameters are specified."); } } - return new CursorAnchorInfo(this); + return CursorAnchorInfo.create(this); } /** @@ -440,31 +440,93 @@ public final class CursorAnchorInfo implements Parcelable { } } - private CursorAnchorInfo(final Builder builder) { - mSelectionStart = builder.mSelectionStart; - mSelectionEnd = builder.mSelectionEnd; - mComposingTextStart = builder.mComposingTextStart; - mComposingText = builder.mComposingText; - mInsertionMarkerFlags = builder.mInsertionMarkerFlags; - mInsertionMarkerHorizontal = builder.mInsertionMarkerHorizontal; - mInsertionMarkerTop = builder.mInsertionMarkerTop; - mInsertionMarkerBaseline = builder.mInsertionMarkerBaseline; - mInsertionMarkerBottom = builder.mInsertionMarkerBottom; - mCharacterBoundsArray = builder.mCharacterBoundsArrayBuilder != null - ? builder.mCharacterBoundsArrayBuilder.build() : null; - mEditorBoundsInfo = builder.mEditorBoundsInfo; - mMatrixValues = new float[9]; + private static CursorAnchorInfo create(Builder builder) { + final SparseRectFArray characterBoundsArray = + builder.mCharacterBoundsArrayBuilder != null + ? builder.mCharacterBoundsArrayBuilder.build() + : null; + final float[] matrixValues = new float[9]; if (builder.mMatrixInitialized) { - System.arraycopy(builder.mMatrixValues, 0, mMatrixValues, 0, 9); + System.arraycopy(builder.mMatrixValues, 0, matrixValues, 0, 9); } else { - Matrix.IDENTITY_MATRIX.getValues(mMatrixValues); + Matrix.IDENTITY_MATRIX.getValues(matrixValues); } + return new CursorAnchorInfo(builder.mSelectionStart, builder.mSelectionEnd, + builder.mComposingTextStart, builder.mComposingText, builder.mInsertionMarkerFlags, + builder.mInsertionMarkerHorizontal, builder.mInsertionMarkerTop, + builder.mInsertionMarkerBaseline, builder.mInsertionMarkerBottom, + characterBoundsArray, builder.mEditorBoundsInfo, matrixValues); + } + + private CursorAnchorInfo(int selectionStart, int selectionEnd, int composingTextStart, + @Nullable CharSequence composingText, int insertionMarkerFlags, + float insertionMarkerHorizontal, float insertionMarkerTop, + float insertionMarkerBaseline, float insertionMarkerBottom, + @Nullable SparseRectFArray characterBoundsArray, + @Nullable EditorBoundsInfo editorBoundsInfo, + @NonNull float[] matrixValues) { + mSelectionStart = selectionStart; + mSelectionEnd = selectionEnd; + mComposingTextStart = composingTextStart; + mComposingText = composingText; + mInsertionMarkerFlags = insertionMarkerFlags; + mInsertionMarkerHorizontal = insertionMarkerHorizontal; + mInsertionMarkerTop = insertionMarkerTop; + mInsertionMarkerBaseline = insertionMarkerBaseline; + mInsertionMarkerBottom = insertionMarkerBottom; + mCharacterBoundsArray = characterBoundsArray; + mEditorBoundsInfo = editorBoundsInfo; + mMatrixValues = matrixValues; + // To keep hash function simple, we only use some complex objects for hash. - int hash = Objects.hashCode(mComposingText); - hash *= 31; - hash += Arrays.hashCode(mMatrixValues); - mHashCode = hash; + int hashCode = Objects.hashCode(mComposingText); + hashCode *= 31; + hashCode += Arrays.hashCode(mMatrixValues); + mHashCode = hashCode; + } + + /** + * Creates a new instance of {@link CursorAnchorInfo} by applying {@code parentMatrix} to + * the coordinate transformation matrix. + * + * @param original {@link CursorAnchorInfo} to be cloned from. + * @param parentMatrix {@link Matrix} to be applied to {@code original.getMatrix()} + * @return A new instance of {@link CursorAnchorInfo} whose {@link CursorAnchorInfo#getMatrix()} + * returns {@code parentMatrix * original.getMatrix()}. + * @hide + */ + public static CursorAnchorInfo createForAdditionalParentMatrix(CursorAnchorInfo original, + @NonNull Matrix parentMatrix) { + return new CursorAnchorInfo(original.mSelectionStart, original.mSelectionEnd, + original.mComposingTextStart, original.mComposingText, + original.mInsertionMarkerFlags, original.mInsertionMarkerHorizontal, + original.mInsertionMarkerTop, original.mInsertionMarkerBaseline, + original.mInsertionMarkerBottom, original.mCharacterBoundsArray, + original.mEditorBoundsInfo, computeMatrixValues(parentMatrix, original)); + } + + /** + * Returns a float array that represents {@link Matrix} elements for + * {@code parentMatrix * info.getMatrix()}. + * + * @param parentMatrix {@link Matrix} to be multiplied. + * @param info {@link CursorAnchorInfo} to provide {@link Matrix} to be multiplied. + * @return {@code parentMatrix * info.getMatrix()}. + */ + private static float[] computeMatrixValues(@NonNull Matrix parentMatrix, + @NonNull CursorAnchorInfo info) { + if (parentMatrix.isIdentity()) { + return info.mMatrixValues; + } + + final Matrix newMatrix = new Matrix(); + newMatrix.setValues(info.mMatrixValues); + newMatrix.postConcat(parentMatrix); + + final float[] matrixValues = new float[9]; + newMatrix.getValues(matrixValues); + return matrixValues; } /** diff --git a/core/java/android/view/inputmethod/InputMethodManager.java b/core/java/android/view/inputmethod/InputMethodManager.java index da20e336627ce..805f8e7551a56 100644 --- a/core/java/android/view/inputmethod/InputMethodManager.java +++ b/core/java/android/view/inputmethod/InputMethodManager.java @@ -53,6 +53,7 @@ import android.content.ComponentName; import android.content.ContentResolver; import android.content.Context; import android.content.pm.PackageManager; +import android.graphics.Matrix; import android.graphics.Rect; import android.inputmethodservice.InputMethodService; import android.os.Binder; @@ -452,6 +453,18 @@ public final class InputMethodManager { */ private CursorAnchorInfo mCursorAnchorInfo = null; + /** + * A special {@link Matrix} that can be provided by the system when this instance is running + * inside a virtual display. + * + *

If this is non-{@code null}, {@link #updateCursorAnchorInfo(View, CursorAnchorInfo)} + * should be adjusted with this {@link Matrix}.

+ * + *

{@code null} when not used.

+ */ + @GuardedBy("mH") + private Matrix mVirtualDisplayToScreenMatrix = null; + /** * As reported by {@link InputBindResult}. This value is determined by * {@link com.android.internal.R.styleable#InputMethod_suppressesSpellChecking}. @@ -528,6 +541,7 @@ public final class InputMethodManager { static final int MSG_REPORT_FULLSCREEN_MODE = 10; static final int MSG_BIND_ACCESSIBILITY_SERVICE = 11; static final int MSG_UNBIND_ACCESSIBILITY_SERVICE = 12; + static final int MSG_UPDATE_VIRTUAL_DISPLAY_TO_SCREEN_MATRIX = 30; private static boolean isAutofillUIShowing(View servedView) { AutofillManager afm = servedView.getContext().getSystemService(AutofillManager.class); @@ -892,6 +906,7 @@ public final class InputMethodManager { InputMethodSessionWrapper.createOrNull(res.method); mCurId = res.id; mBindSequence = res.sequence; + mVirtualDisplayToScreenMatrix = res.getVirtualDisplayToScreenMatrix(); mIsInputMethodSuppressingSpellChecker = res.isInputMethodSuppressingSpellChecker; } @@ -1063,6 +1078,45 @@ public final class InputMethodManager { } return; } + case MSG_UPDATE_VIRTUAL_DISPLAY_TO_SCREEN_MATRIX: { + final float[] matrixValues = (float[]) msg.obj; + final int bindSequence = msg.arg1; + synchronized (mH) { + if (mBindSequence != bindSequence) { + return; + } + if (matrixValues == null || mVirtualDisplayToScreenMatrix == null) { + // Either InputBoundResult#mVirtualDisplayToScreenMatrixValues is null + // OR this app is unbound from the parent VirtualDisplay. In this case, + // calling updateCursorAnchorInfo() isn't safe. Only clear the matrix. + mVirtualDisplayToScreenMatrix = null; + return; + } + + final float[] currentValues = new float[9]; + mVirtualDisplayToScreenMatrix.getValues(currentValues); + if (Arrays.equals(currentValues, matrixValues)) { + return; + } + mVirtualDisplayToScreenMatrix.setValues(matrixValues); + + if (mCursorAnchorInfo == null || mCurrentInputMethodSession == null + || mServedInputConnection == null) { + return; + } + final boolean isMonitoring = (mRequestUpdateCursorAnchorInfoMonitorMode + & InputConnection.CURSOR_UPDATE_MONITOR) != 0; + if (!isMonitoring) { + return; + } + // Since the host VirtualDisplay is moved, we need to issue + // IMS#updateCursorAnchorInfo() again. + mCurrentInputMethodSession.updateCursorAnchorInfo( + CursorAnchorInfo.createForAdditionalParentMatrix( + mCursorAnchorInfo, mVirtualDisplayToScreenMatrix)); + } + return; + } } } } @@ -1127,6 +1181,12 @@ public final class InputMethodManager { .sendToTarget(); } + @Override + public void updateVirtualDisplayToScreenMatrix(int bindSequence, float[] matrixValues) { + mH.obtainMessage(MSG_UPDATE_VIRTUAL_DISPLAY_TO_SCREEN_MATRIX, bindSequence, 0, + matrixValues).sendToTarget(); + } + @Override public void setImeTraceEnabled(boolean enabled) { ImeTracing.getInstance().setEnabled(enabled); @@ -1596,7 +1656,9 @@ public final class InputMethodManager { * Disconnect any existing input connection, clearing the served view. */ @UnsupportedAppUsage + @GuardedBy("mH") void finishInputLocked() { + mVirtualDisplayToScreenMatrix = null; mIsInputMethodSuppressingSpellChecker = false; setNextServedViewLocked(null); if (getServedViewLocked() != null) { @@ -2275,6 +2337,7 @@ public final class InputMethodManager { + InputMethodDebug.startInputFlagsToString(startInputFlags)); return false; } + mVirtualDisplayToScreenMatrix = res.getVirtualDisplayToScreenMatrix(); mIsInputMethodSuppressingSpellChecker = res.isInputMethodSuppressingSpellChecker; if (res.id != null) { setInputChannelLocked(res.channel); @@ -2695,7 +2758,13 @@ public final class InputMethodManager { return; } if (DEBUG) Log.v(TAG, "updateCursorAnchorInfo: " + cursorAnchorInfo); - mCurrentInputMethodSession.updateCursorAnchorInfo(cursorAnchorInfo); + if (mVirtualDisplayToScreenMatrix != null) { + mCurrentInputMethodSession.updateCursorAnchorInfo( + CursorAnchorInfo.createForAdditionalParentMatrix( + cursorAnchorInfo, mVirtualDisplayToScreenMatrix)); + } else { + mCurrentInputMethodSession.updateCursorAnchorInfo(cursorAnchorInfo); + } mCursorAnchorInfo = cursorAnchorInfo; // Clear immediate bit (if any). mRequestUpdateCursorAnchorInfoMonitorMode &= ~CURSOR_UPDATE_IMMEDIATE; @@ -3268,6 +3337,43 @@ public final class InputMethodManager { } } + /** + * An internal API for {@link android.hardware.display.VirtualDisplay} to report where its + * embedded virtual display is placed. + * + * @param childDisplayId Display ID of the embedded virtual display. + * @param matrix {@link Matrix} to convert virtual display screen coordinates to + * the host screen coordinates. {@code null} to clear the relationship. + * @hide + */ + public void reportVirtualDisplayGeometry(int childDisplayId, @Nullable Matrix matrix) { + try { + final float[] matrixValues; + if (matrix == null) { + matrixValues = null; + } else { + matrixValues = new float[9]; + matrix.getValues(matrixValues); + } + mService.reportVirtualDisplayGeometryAsync(mClient, childDisplayId, matrixValues); + } catch (RemoteException e) { + throw e.rethrowFromSystemServer(); + } + } + + /** + * An internal API that returns if the current display has a transformation matrix to apply. + * + * @return {@code true} if {@link Matrix} to convert virtual display screen coordinates to + * the host screen coordinates is set. + * @hide + */ + public boolean hasVirtualDisplayToScreenMatrix() { + synchronized (mH) { + return mVirtualDisplayToScreenMatrix != null; + } + } + /** * Force switch to the last used input method and subtype. If the last input method didn't have * any subtypes, the framework will simply switch to the last input method with no subtype diff --git a/core/java/com/android/internal/inputmethod/InputBindResult.java b/core/java/com/android/internal/inputmethod/InputBindResult.java index e83840177a733..f7341a5c45748 100644 --- a/core/java/com/android/internal/inputmethod/InputBindResult.java +++ b/core/java/com/android/internal/inputmethod/InputBindResult.java @@ -19,9 +19,11 @@ package com.android.internal.inputmethod; import static java.lang.annotation.RetentionPolicy.SOURCE; import android.annotation.IntDef; +import android.annotation.Nullable; import android.content.ComponentName; import android.content.Intent; import android.content.ServiceConnection; +import android.graphics.Matrix; import android.os.IBinder; import android.os.Parcel; import android.os.Parcelable; @@ -202,11 +204,28 @@ public final class InputBindResult implements Parcelable { */ public final int sequence; + @Nullable + private final float[] mVirtualDisplayToScreenMatrixValues; + /** * {@code true} if the IME explicitly specifies {@code suppressesSpellChecker="true"}. */ public final boolean isInputMethodSuppressingSpellChecker; + /** + * @return {@link Matrix} that corresponds to {@link #mVirtualDisplayToScreenMatrixValues}. + * {@code null} if {@link #mVirtualDisplayToScreenMatrixValues} is {@code null}. + */ + @Nullable + public Matrix getVirtualDisplayToScreenMatrix() { + if (mVirtualDisplayToScreenMatrixValues == null) { + return null; + } + final Matrix matrix = new Matrix(); + matrix.setValues(mVirtualDisplayToScreenMatrixValues); + return matrix; + } + /** * Creates a new instance of {@link InputBindResult}. * @@ -225,6 +244,7 @@ public final class InputBindResult implements Parcelable { public InputBindResult(@ResultCode int result, IInputMethodSession method, SparseArray accessibilitySessions, InputChannel channel, String id, int sequence, + @Nullable Matrix virtualDisplayToScreenMatrix, boolean isInputMethodSuppressingSpellChecker) { this.result = result; this.method = method; @@ -232,6 +252,12 @@ public final class InputBindResult implements Parcelable { this.channel = channel; this.id = id; this.sequence = sequence; + if (virtualDisplayToScreenMatrix == null) { + mVirtualDisplayToScreenMatrixValues = null; + } else { + mVirtualDisplayToScreenMatrixValues = new float[9]; + virtualDisplayToScreenMatrix.getValues(mVirtualDisplayToScreenMatrixValues); + } this.isInputMethodSuppressingSpellChecker = isInputMethodSuppressingSpellChecker; } @@ -258,6 +284,7 @@ public final class InputBindResult implements Parcelable { } id = source.readString(); sequence = source.readInt(); + mVirtualDisplayToScreenMatrixValues = source.createFloatArray(); isInputMethodSuppressingSpellChecker = source.readBoolean(); } @@ -268,6 +295,7 @@ public final class InputBindResult implements Parcelable { public String toString() { return "InputBindResult{result=" + getResultString() + " method=" + method + " id=" + id + " sequence=" + sequence + + " virtualDisplayToScreenMatrix=" + getVirtualDisplayToScreenMatrix() + " isInputMethodSuppressingSpellChecker=" + isInputMethodSuppressingSpellChecker + "}"; } @@ -299,6 +327,7 @@ public final class InputBindResult implements Parcelable { } dest.writeString(id); dest.writeInt(sequence); + dest.writeFloatArray(mVirtualDisplayToScreenMatrixValues); dest.writeBoolean(isInputMethodSuppressingSpellChecker); } @@ -366,7 +395,7 @@ public final class InputBindResult implements Parcelable { } private static InputBindResult error(@ResultCode int result) { - return new InputBindResult(result, null, null, null, null, -1, false); + return new InputBindResult(result, null, null, null, null, -1, null, false); } /** diff --git a/core/java/com/android/internal/inputmethod/RemoteInputConnectionImpl.java b/core/java/com/android/internal/inputmethod/RemoteInputConnectionImpl.java index f81fd376fef0c..7f11e3018849e 100644 --- a/core/java/com/android/internal/inputmethod/RemoteInputConnectionImpl.java +++ b/core/java/com/android/internal/inputmethod/RemoteInputConnectionImpl.java @@ -987,7 +987,8 @@ public final class RemoteInputConnectionImpl extends IInputContext.Stub { Log.w(TAG, "requestCursorAnchorInfo on inactive InputConnection"); return false; } - if (mParentInputMethodManager.getDisplayId() != imeDisplayId) { + if (mParentInputMethodManager.getDisplayId() != imeDisplayId + && !mParentInputMethodManager.hasVirtualDisplayToScreenMatrix()) { // requestCursorUpdates() is not currently supported across displays. return false; } diff --git a/core/java/com/android/internal/view/IInputMethodClient.aidl b/core/java/com/android/internal/view/IInputMethodClient.aidl index 8430c0859bc9f..8952474924bb7 100644 --- a/core/java/com/android/internal/view/IInputMethodClient.aidl +++ b/core/java/com/android/internal/view/IInputMethodClient.aidl @@ -30,6 +30,7 @@ oneway interface IInputMethodClient { void setActive(boolean active, boolean fullscreen, boolean reportToImeController); void scheduleStartInputIfNecessary(boolean fullscreen); void reportFullscreenMode(boolean fullscreen); + void updateVirtualDisplayToScreenMatrix(int bindSequence, in float[] matrixValues); void setImeTraceEnabled(boolean enabled); void throwExceptionFromSystem(String message); } diff --git a/core/java/com/android/internal/view/IInputMethodManager.aidl b/core/java/com/android/internal/view/IInputMethodManager.aidl index c5346b95440ba..616411fee5f26 100644 --- a/core/java/com/android/internal/view/IInputMethodManager.aidl +++ b/core/java/com/android/internal/view/IInputMethodManager.aidl @@ -69,6 +69,9 @@ interface IInputMethodManager { // TODO(Bug 113914148): Consider removing this. int getInputMethodWindowVisibleHeight(in IInputMethodClient client); + oneway void reportVirtualDisplayGeometryAsync(in IInputMethodClient parentClient, + int childDisplayId, in float[] matrixValues); + oneway void reportPerceptibleAsync(in IBinder windowToken, boolean perceptible); /** Remove the IME surface. Requires INTERNAL_SYSTEM_WINDOW permission. */ void removeImeSurface(); diff --git a/core/tests/coretests/src/android/view/inputmethod/CursorAnchorInfoTest.java b/core/tests/coretests/src/android/view/inputmethod/CursorAnchorInfoTest.java new file mode 100644 index 0000000000000..9d7d71d8d5396 --- /dev/null +++ b/core/tests/coretests/src/android/view/inputmethod/CursorAnchorInfoTest.java @@ -0,0 +1,63 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.view.inputmethod; + +import static org.junit.Assert.assertEquals; + +import android.graphics.Matrix; + +import androidx.test.filters.SmallTest; +import androidx.test.runner.AndroidJUnit4; + +import org.junit.Test; +import org.junit.runner.RunWith; + +@SmallTest +@RunWith(AndroidJUnit4.class) +public class CursorAnchorInfoTest { + @Test + public void testCreateForAdditionalParentMatrix() { + final Matrix originalMatrix = new Matrix(); + originalMatrix.setTranslate(10.0f, 20.0f); + final CursorAnchorInfo.Builder builder = new CursorAnchorInfo.Builder(); + builder.setMatrix(originalMatrix); + + final CursorAnchorInfo originalInstance = builder.build(); + + assertEquals(originalMatrix, originalInstance.getMatrix()); + + final Matrix additionalParentMatrix = new Matrix(); + additionalParentMatrix.setTranslate(1.0f, 2.0f); + + final Matrix newMatrix = new Matrix(originalMatrix); + newMatrix.postConcat(additionalParentMatrix); + + builder.reset(); + builder.setMatrix(newMatrix); + // An instance created by the standard Builder class. + final CursorAnchorInfo newInstanceByBuilder = builder.build(); + + // An instance created by an @hide method. + final CursorAnchorInfo newInstanceByMethod = + CursorAnchorInfo.createForAdditionalParentMatrix( + originalInstance, additionalParentMatrix); + + assertEquals(newMatrix, newInstanceByBuilder.getMatrix()); + assertEquals(newMatrix, newInstanceByMethod.getMatrix()); + assertEquals(newInstanceByBuilder.hashCode(), newInstanceByMethod.hashCode()); + } +} diff --git a/services/core/java/com/android/server/inputmethod/InputMethodBindingController.java b/services/core/java/com/android/server/inputmethod/InputMethodBindingController.java index d2d80ffde5328..0de523c5f3f62 100644 --- a/services/core/java/com/android/server/inputmethod/InputMethodBindingController.java +++ b/services/core/java/com/android/server/inputmethod/InputMethodBindingController.java @@ -427,7 +427,7 @@ final class InputMethodBindingController { addFreshWindowToken(); return new InputBindResult( InputBindResult.ResultCode.SUCCESS_WAITING_IME_BINDING, - null, null, null, mCurId, mCurSeq, false); + null, null, null, mCurId, mCurSeq, null, false); } Slog.w(InputMethodManagerService.TAG, diff --git a/services/core/java/com/android/server/inputmethod/InputMethodManagerService.java b/services/core/java/com/android/server/inputmethod/InputMethodManagerService.java index 1e037ab11031c..3c3140551bc31 100644 --- a/services/core/java/com/android/server/inputmethod/InputMethodManagerService.java +++ b/services/core/java/com/android/server/inputmethod/InputMethodManagerService.java @@ -88,6 +88,8 @@ import android.content.pm.ServiceInfo; import android.content.res.Configuration; import android.content.res.Resources; import android.database.ContentObserver; +import android.graphics.Matrix; +import android.hardware.display.DisplayManagerInternal; import android.hardware.input.InputManagerInternal; import android.inputmethodservice.InputMethodService; import android.media.AudioManagerInternal; @@ -124,6 +126,7 @@ import android.util.Slog; import android.util.SparseArray; import android.util.SparseBooleanArray; import android.util.proto.ProtoOutputStream; +import android.view.DisplayInfo; import android.view.IWindowManager; import android.view.InputChannel; import android.view.View; @@ -298,6 +301,7 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub final PackageManagerInternal mPackageManagerInternal; final InputManagerInternal mInputManagerInternal; final ImePlatformCompatUtils mImePlatformCompatUtils; + private final DisplayManagerInternal mDisplayManagerInternal; final boolean mHasFeature; private final ArrayMap> mAdditionalSubtypeMap = new ArrayMap<>(); @@ -465,6 +469,35 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub @GuardedBy("ImfLock.class") final ArrayMap mClients = new ArrayMap<>(); + private static final class VirtualDisplayInfo { + /** + * {@link ClientState} where {@link android.hardware.display.VirtualDisplay} is running. + */ + private final ClientState mParentClient; + /** + * {@link Matrix} to convert screen coordinates in the embedded virtual display to + * screen coordinates where {@link #mParentClient} exists. + */ + private final Matrix mMatrix; + + VirtualDisplayInfo(ClientState parentClient, Matrix matrix) { + mParentClient = parentClient; + mMatrix = matrix; + } + } + + /** + * A mapping table from virtual display IDs created for + * {@link android.hardware.display.VirtualDisplay} to its parent IME client where the embedded + * virtual display is running. + * + *

Note: this can be used only for virtual display IDs created by + * {@link android.hardware.display.VirtualDisplay}.

+ */ + @GuardedBy("ImfLock.class") + private final SparseArray mVirtualDisplayIdToParentMap = + new SparseArray<>(); + /** * Set once the system is ready to run third party code. */ @@ -561,6 +594,16 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub */ EditorInfo mCurAttribute; + /** + * A special {@link Matrix} to convert virtual screen coordinates to the IME target display + * coordinates. + * + *

Used only while the IME client is running in a virtual display. {@code null} + * otherwise.

+ */ + @Nullable + private Matrix mCurVirtualDisplayToScreenMatrix = null; + /** * Id obtained with {@link InputMethodInfo#getId()} for the input method that we are currently * connected to or in the process of connecting to. @@ -1660,6 +1703,7 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub mInputManagerInternal = LocalServices.getService(InputManagerInternal.class); mImePlatformCompatUtils = new ImePlatformCompatUtils(); mImeDisplayValidator = mWindowManagerInternal::getDisplayImePolicy; + mDisplayManagerInternal = LocalServices.getService(DisplayManagerInternal.class); mAppOpsManager = mContext.getSystemService(AppOpsManager.class); mUserManager = mContext.getSystemService(UserManager.class); mUserManagerInternal = LocalServices.getService(UserManagerInternal.class); @@ -2333,6 +2377,15 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub client.asBinder().unlinkToDeath(cs.clientDeathRecipient, 0); clearClientSessionLocked(cs); clearClientSessionForAccessibilityLocked(cs); + + final int numItems = mVirtualDisplayIdToParentMap.size(); + for (int i = numItems - 1; i >= 0; --i) { + final VirtualDisplayInfo info = mVirtualDisplayIdToParentMap.valueAt(i); + if (info.mParentClient == cs) { + mVirtualDisplayIdToParentMap.removeAt(i); + } + } + if (mCurClient == cs) { hideCurrentInputLocked( mCurFocusedWindow, 0, null, SoftInputShowHideReason.HIDE_REMOVE_CLIENT); @@ -2348,6 +2401,7 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub } mBoundToAccessibility = false; mCurClient = null; + mCurVirtualDisplayToScreenMatrix = null; } if (mCurFocusedWindowClient == cs) { mCurFocusedWindowClient = null; @@ -2432,6 +2486,7 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub mCurClient.sessionRequested = false; mCurClient.mSessionRequestedForAccessibility = false; mCurClient = null; + mCurVirtualDisplayToScreenMatrix = null; mMenuController.hideInputMethodMenuLocked(); } @@ -2517,7 +2572,33 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub return new InputBindResult(InputBindResult.ResultCode.SUCCESS_WITH_IME_SESSION, session.session, accessibilityInputMethodSessions, (session.channel != null ? session.channel.dup() : null), - curId, getSequenceNumberLocked(), suppressesSpellChecker); + curId, getSequenceNumberLocked(), mCurVirtualDisplayToScreenMatrix, + suppressesSpellChecker); + } + + @GuardedBy("ImfLock.class") + @Nullable + private Matrix getVirtualDisplayToScreenMatrixLocked(int clientDisplayId, int imeDisplayId) { + if (clientDisplayId == imeDisplayId) { + return null; + } + int displayId = clientDisplayId; + Matrix matrix = null; + while (true) { + final VirtualDisplayInfo info = mVirtualDisplayIdToParentMap.get(displayId); + if (info == null) { + return null; + } + if (matrix == null) { + matrix = new Matrix(info.mMatrix); + } else { + matrix.postConcat(info.mMatrix); + } + if (info.mParentClient.selfReportedDisplayId == imeDisplayId) { + return matrix; + } + displayId = info.mParentClient.selfReportedDisplayId; + } } @GuardedBy("ImfLock.class") @@ -2551,7 +2632,8 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub return new InputBindResult( InputBindResult.ResultCode.SUCCESS_WITH_ACCESSIBILITY_SESSION, imeSession, accessibilityInputMethodSessions, null, - getCurIdLocked(), getSequenceNumberLocked(), false); + getCurIdLocked(), getSequenceNumberLocked(), mCurVirtualDisplayToScreenMatrix, + false); } return null; } @@ -2594,7 +2676,7 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub // party code. return new InputBindResult( InputBindResult.ResultCode.ERROR_SYSTEM_NOT_READY, - null, null, null, selectedMethodId, getSequenceNumberLocked(), false); + null, null, null, selectedMethodId, getSequenceNumberLocked(), null, false); } if (!InputMethodUtils.checkIfPackageBelongsToUid(mAppOpsManager, cs.uid, @@ -2625,6 +2707,9 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub advanceSequenceNumberLocked(); mCurClient = cs; mCurInputContext = inputContext; + mCurVirtualDisplayToScreenMatrix = + getVirtualDisplayToScreenMatrixLocked(cs.selfReportedDisplayId, + mDisplayIdToShowIme); mCurAttribute = attribute; // If configured, we want to avoid starting up the IME if it is not supposed to be showing @@ -2728,7 +2813,7 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub requestClientSessionForAccessibilityLocked(cs); return new InputBindResult( InputBindResult.ResultCode.SUCCESS_WAITING_IME_SESSION, - null, null, null, getCurIdLocked(), getSequenceNumberLocked(), false); + null, null, null, getCurIdLocked(), getSequenceNumberLocked(), null, false); } else { long bindingDuration = SystemClock.uptimeMillis() - getLastBindTimeLocked(); if (bindingDuration < TIME_TO_RECONNECT) { @@ -2741,7 +2826,8 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub // to see if we can get back in touch with the service. return new InputBindResult( InputBindResult.ResultCode.SUCCESS_WAITING_IME_BINDING, - null, null, null, getCurIdLocked(), getSequenceNumberLocked(), false); + null, null, null, getCurIdLocked(), getSequenceNumberLocked(), null, + false); } else { EventLog.writeEvent(EventLogTags.IMF_FORCE_RECONNECT_IME, getSelectedMethodIdLocked(), bindingDuration, 0); @@ -3794,7 +3880,7 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub } return new InputBindResult( InputBindResult.ResultCode.SUCCESS_REPORT_WINDOW_FOCUS_ONLY, - null, null, null, null, -1, false); + null, null, null, null, -1, null, false); } mCurFocusedWindow = windowToken; @@ -4313,6 +4399,104 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub mHandler.obtainMessage(MSG_REMOVE_IME_SURFACE).sendToTarget(); } + @Override + public void reportVirtualDisplayGeometryAsync(IInputMethodClient parentClient, + int childDisplayId, float[] matrixValues) { + try { + final DisplayInfo displayInfo = mDisplayManagerInternal.getDisplayInfo(childDisplayId); + if (displayInfo == null) { + throw new IllegalArgumentException( + "Cannot find display for non-existent displayId: " + childDisplayId); + } + final int callingUid = Binder.getCallingUid(); + if (callingUid != displayInfo.ownerUid) { + throw new SecurityException("The caller doesn't own the display."); + } + + synchronized (ImfLock.class) { + final ClientState cs = mClients.get(parentClient.asBinder()); + if (cs == null) { + return; + } + + // null matrixValues means that the entry needs to be removed. + if (matrixValues == null) { + final VirtualDisplayInfo info = + mVirtualDisplayIdToParentMap.get(childDisplayId); + if (info == null) { + return; + } + if (info.mParentClient != cs) { + throw new SecurityException("Only the owner client can clear" + + " VirtualDisplayGeometry for display #" + childDisplayId); + } + mVirtualDisplayIdToParentMap.remove(childDisplayId); + return; + } + + VirtualDisplayInfo info = mVirtualDisplayIdToParentMap.get(childDisplayId); + if (info != null && info.mParentClient != cs) { + throw new InvalidParameterException("Display #" + childDisplayId + + " is already registered by " + info.mParentClient); + } + if (info == null) { + if (!mWindowManagerInternal.isUidAllowedOnDisplay(childDisplayId, cs.uid)) { + throw new SecurityException(cs + " cannot access to display #" + + childDisplayId); + } + info = new VirtualDisplayInfo(cs, new Matrix()); + mVirtualDisplayIdToParentMap.put(childDisplayId, info); + } + info.mMatrix.setValues(matrixValues); + + if (mCurClient == null || mCurClient.curSession == null) { + return; + } + + Matrix matrix = null; + int displayId = mCurClient.selfReportedDisplayId; + boolean needToNotify = false; + while (true) { + needToNotify |= (displayId == childDisplayId); + final VirtualDisplayInfo next = mVirtualDisplayIdToParentMap.get(displayId); + if (next == null) { + break; + } + if (matrix == null) { + matrix = new Matrix(next.mMatrix); + } else { + matrix.postConcat(next.mMatrix); + } + if (next.mParentClient.selfReportedDisplayId == mCurTokenDisplayId) { + if (needToNotify) { + final float[] values = new float[9]; + matrix.getValues(values); + try { + mCurClient.client.updateVirtualDisplayToScreenMatrix( + getSequenceNumberLocked(), values); + } catch (RemoteException e) { + Slog.e(TAG, + "Exception calling updateVirtualDisplayToScreenMatrix()", + e); + + } + } + break; + } + displayId = info.mParentClient.selfReportedDisplayId; + } + } + } catch (Throwable t) { + if (parentClient != null) { + try { + parentClient.throwExceptionFromSystem(t.toString()); + } catch (RemoteException e) { + Slog.e(TAG, "Exception calling throwExceptionFromSystem()", e); + } + } + } + } + @Override public void removeImeSurfaceFromWindowAsync(IBinder windowToken) { // No permission check, because we'll only execute the request if the calling window is