diff --git a/core/api/current.txt b/core/api/current.txt index 25e10cda322f8..7e55cb199ead9 100644 --- a/core/api/current.txt +++ b/core/api/current.txt @@ -3061,6 +3061,7 @@ package android.accessibilityservice { method @NonNull public final android.accessibilityservice.AccessibilityButtonController getAccessibilityButtonController(); method @NonNull public final android.accessibilityservice.AccessibilityButtonController getAccessibilityButtonController(int); method @NonNull @RequiresPermission(android.Manifest.permission.USE_FINGERPRINT) public final android.accessibilityservice.FingerprintGestureController getFingerprintGestureController(); + method @Nullable public final android.accessibilityservice.InputMethod getInputMethod(); method @NonNull public final android.accessibilityservice.AccessibilityService.MagnificationController getMagnificationController(); method public android.view.accessibility.AccessibilityNodeInfo getRootInActiveWindow(); method public final android.accessibilityservice.AccessibilityServiceInfo getServiceInfo(); @@ -3073,6 +3074,7 @@ package android.accessibilityservice { method public boolean isNodeInCache(@NonNull android.view.accessibility.AccessibilityNodeInfo); method public abstract void onAccessibilityEvent(android.view.accessibility.AccessibilityEvent); method public final android.os.IBinder onBind(android.content.Intent); + method @NonNull public android.accessibilityservice.InputMethod onCreateInputMethod(); method @Deprecated protected boolean onGesture(int); method public boolean onGesture(@NonNull android.accessibilityservice.AccessibilityGestureEvent); method public abstract void onInterrupt(); @@ -3260,6 +3262,7 @@ package android.accessibilityservice { field public static final int FEEDBACK_VISUAL = 8; // 0x8 field public static final int FLAG_ENABLE_ACCESSIBILITY_VOLUME = 128; // 0x80 field public static final int FLAG_INCLUDE_NOT_IMPORTANT_VIEWS = 2; // 0x2 + field public static final int FLAG_INPUT_METHOD_EDITOR = 32768; // 0x8000 field public static final int FLAG_REPORT_VIEW_IDS = 16; // 0x10 field public static final int FLAG_REQUEST_2_FINGER_PASSTHROUGH = 8192; // 0x2000 field public static final int FLAG_REQUEST_ACCESSIBILITY_BUTTON = 256; // 0x100 @@ -3320,6 +3323,28 @@ package android.accessibilityservice { method public boolean willContinue(); } + public class InputMethod { + ctor protected InputMethod(@NonNull android.accessibilityservice.AccessibilityService); + method @Nullable public final android.accessibilityservice.InputMethod.AccessibilityInputConnection getCurrentInputConnection(); + method @Nullable public final android.view.inputmethod.EditorInfo getCurrentInputEditorInfo(); + method public final boolean getCurrentInputStarted(); + method public void onFinishInput(); + method public void onStartInput(@NonNull android.view.inputmethod.EditorInfo, boolean); + method public void onUpdateSelection(int, int, int, int, int, int); + } + + public final class InputMethod.AccessibilityInputConnection { + method public void clearMetaKeyStates(int); + method public void commitText(@NonNull CharSequence, int, @Nullable android.view.inputmethod.TextAttribute); + method public void deleteSurroundingText(int, int); + method public int getCursorCapsMode(int); + method @Nullable public android.view.inputmethod.SurroundingText getSurroundingText(@IntRange(from=0) int, @IntRange(from=0) int, int); + method public void performContextMenuAction(int); + method public void performEditorAction(int); + method public void sendKeyEvent(@NonNull android.view.KeyEvent); + method public void setSelection(int, int); + } + public final class MagnificationConfig implements android.os.Parcelable { method public int describeContents(); method public float getCenterX(); diff --git a/core/java/android/accessibilityservice/AccessibilityService.java b/core/java/android/accessibilityservice/AccessibilityService.java index 50473f1d8c845..cf3ca20315d8d 100644 --- a/core/java/android/accessibilityservice/AccessibilityService.java +++ b/core/java/android/accessibilityservice/AccessibilityService.java @@ -40,6 +40,8 @@ import android.graphics.ParcelableColorSpace; import android.graphics.Region; import android.hardware.HardwareBuffer; import android.hardware.display.DisplayManager; +import android.inputmethodservice.IInputMethodSessionWrapper; +import android.inputmethodservice.RemoteInputConnection; import android.os.Build; import android.os.Bundle; import android.os.Handler; @@ -65,14 +67,23 @@ import android.view.accessibility.AccessibilityInteractionClient; import android.view.accessibility.AccessibilityNodeInfo; import android.view.accessibility.AccessibilityNodeInfo.AccessibilityAction; import android.view.accessibility.AccessibilityWindowInfo; +import android.view.inputmethod.EditorInfo; +import android.view.inputmethod.InputBinding; +import android.view.inputmethod.InputConnection; +import android.view.inputmethod.InputMethodSession; +import com.android.internal.inputmethod.CancellationGroup; import com.android.internal.os.HandlerCaller; import com.android.internal.os.SomeArgs; import com.android.internal.util.Preconditions; import com.android.internal.util.function.pooled.PooledLambda; +import com.android.internal.view.IInputContext; +import com.android.internal.view.IInputMethodSession; +import com.android.internal.view.IInputSessionWithIdCallback; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; +import java.lang.ref.WeakReference; import java.util.Collections; import java.util.List; import java.util.concurrent.Executor; @@ -627,6 +638,21 @@ public abstract class AccessibilityService extends Service { void onAccessibilityButtonAvailabilityChanged(boolean available); /** This is called when the system action list is changed. */ void onSystemActionsChanged(); + /** This is called when an app requests ime sessions or when the service is enabled. */ + void createImeSession(IInputSessionWithIdCallback callback); + /** + * This is called when InputMethodManagerService requests to set the session enabled or + * disabled + */ + void setImeSessionEnabled(InputMethodSession session, boolean enabled); + /** This is called when an app binds input or when the service is enabled. */ + void bindInput(InputBinding binding); + /** This is called when an app unbinds input or when the service is disabled. */ + void unbindInput(); + /** This is called when an app starts input or when the service is enabled. */ + void startInput(@Nullable InputConnection inputConnection, + @NonNull EditorInfo editorInfo, boolean restarting, + @NonNull IBinder startInputToken); } /** @@ -763,6 +789,8 @@ public abstract class AccessibilityService extends Service { new SparseArray<>(0); private SoftKeyboardController mSoftKeyboardController; + private InputMethod mInputMethod; + private boolean mInputMethodInitialized = false; private final SparseArray mAccessibilityButtonControllers = new SparseArray<>(0); @@ -797,6 +825,17 @@ public abstract class AccessibilityService extends Service { for (int i = 0; i < mMagnificationControllers.size(); i++) { mMagnificationControllers.valueAt(i).onServiceConnectedLocked(); } + AccessibilityServiceInfo info = getServiceInfo(); + if (info != null) { + boolean requestIme = (info.flags + & AccessibilityServiceInfo.FLAG_INPUT_METHOD_EDITOR) != 0; + if (requestIme && !mInputMethodInitialized) { + mInputMethod = onCreateInputMethod(); + mInputMethodInitialized = true; + } + } else { + Log.e(LOG_TAG, "AccessibilityServiceInfo is null in dispatchServiceConnected"); + } } if (mSoftKeyboardController != null) { mSoftKeyboardController.onServiceConnected(); @@ -1849,6 +1888,32 @@ public abstract class AccessibilityService extends Service { } } + /** + * The default implementation returns our default {@link InputMethod}. Subclasses can override + * it to provide their own customized version. Accessibility services need to set the + * {@link AccessibilityServiceInfo#FLAG_INPUT_METHOD_EDITOR} flag to use input method APIs. + * + * @return the InputMethod. + */ + @NonNull + public InputMethod onCreateInputMethod() { + return new InputMethod(this); + } + + /** + * Returns the InputMethod instance after the system calls {@link #onCreateInputMethod()}, + * which may be used to input text or get editable text selection change notifications. It will + * return null if the accessibility service doesn't set the + * {@link AccessibilityServiceInfo#FLAG_INPUT_METHOD_EDITOR} flag or the system doesn't call + * {@link #onCreateInputMethod()}. + * + * @return the InputMethod instance + */ + @Nullable + public final InputMethod getInputMethod() { + return mInputMethod; + } + private void onSoftKeyboardShowModeChanged(int showMode) { if (mSoftKeyboardController != null) { mSoftKeyboardController.dispatchSoftKeyboardShowModeChanged(showMode); @@ -2657,6 +2722,47 @@ public abstract class AccessibilityService extends Service { public void onSystemActionsChanged() { AccessibilityService.this.onSystemActionsChanged(); } + + @Override + public void createImeSession(IInputSessionWithIdCallback callback) { + if (mInputMethod != null) { + mInputMethod.createImeSession(callback); + } + } + + @Override + public void setImeSessionEnabled(InputMethodSession session, boolean enabled) { + if (mInputMethod != null) { + mInputMethod.setImeSessionEnabled(session, enabled); + } + } + + @Override + public void bindInput(InputBinding binding) { + if (mInputMethod != null) { + mInputMethod.bindInput(binding); + } + } + + @Override + public void unbindInput() { + if (mInputMethod != null) { + mInputMethod.unbindInput(); + } + } + + @Override + public void startInput(@Nullable InputConnection inputConnection, + @NonNull EditorInfo editorInfo, boolean restarting, + @NonNull IBinder startInputToken) { + if (mInputMethod != null) { + if (restarting) { + mInputMethod.restartInput(inputConnection, editorInfo); + } else { + mInputMethod.startInput(inputConnection, editorInfo); + } + } + } }); } @@ -2682,6 +2788,11 @@ public abstract class AccessibilityService extends Service { private static final int DO_ACCESSIBILITY_BUTTON_CLICKED = 12; private static final int DO_ACCESSIBILITY_BUTTON_AVAILABILITY_CHANGED = 13; private static final int DO_ON_SYSTEM_ACTIONS_CHANGED = 14; + private static final int DO_CREATE_IME_SESSION = 15; + private static final int DO_SET_IME_SESSION_ENABLED = 16; + private static final int DO_BIND_INPUT = 17; + private static final int DO_UNBIND_INPUT = 18; + private static final int DO_START_INPUT = 19; private final HandlerCaller mCaller; @@ -2690,6 +2801,22 @@ public abstract class AccessibilityService extends Service { private int mConnectionId = AccessibilityInteractionClient.NO_ID; + /** + * This is not {@null} only between {@link #bindInput(InputBinding)} and + * {@link #unbindInput()} so that {@link RemoteInputConnection} can query if + * {@link #unbindInput()} has already been called or not, mainly to avoid unnecessary + * blocking operations. + * + *

This field must be set and cleared only from the binder thread(s), where the system + * guarantees that {@link #bindInput(InputBinding)}, + * {@link #startInput(IBinder, IInputContext, EditorInfo, boolean)}, and + * {@link #unbindInput()} are called with the same order as the original calls + * in {@link com.android.server.inputmethod.InputMethodManagerService}. + * See {@link IBinder#FLAG_ONEWAY} for detailed semantics.

+ */ + @Nullable + CancellationGroup mCancellationGroup = null; + public IAccessibilityServiceClientWrapper(Context context, Looper looper, Callbacks callback) { mCallback = callback; @@ -2783,6 +2910,70 @@ public abstract class AccessibilityService extends Service { mCaller.sendMessage(mCaller.obtainMessage(DO_ON_SYSTEM_ACTIONS_CHANGED)); } + /** This is called when an app requests ime sessions or when the service is enabled. */ + public void createImeSession(IInputSessionWithIdCallback callback) { + final Message message = mCaller.obtainMessageO(DO_CREATE_IME_SESSION, callback); + mCaller.sendMessage(message); + } + + /** + * This is called when InputMethodManagerService requests to set the session enabled or + * disabled + */ + public void setImeSessionEnabled(IInputMethodSession session, boolean enabled) { + try { + InputMethodSession ls = ((IInputMethodSessionWrapper) + session).getInternalInputMethodSession(); + if (ls == null) { + Log.w(LOG_TAG, "Session is already finished: " + session); + return; + } + mCaller.sendMessage(mCaller.obtainMessageIO( + DO_SET_IME_SESSION_ENABLED, enabled ? 1 : 0, ls)); + } catch (ClassCastException e) { + Log.w(LOG_TAG, "Incoming session not of correct type: " + session, e); + } + } + + /** This is called when an app binds input or when the service is enabled. */ + public void bindInput(InputBinding binding) { + if (mCancellationGroup != null) { + Log.e(LOG_TAG, "bindInput must be paired with unbindInput."); + } + mCancellationGroup = new CancellationGroup(); + InputConnection ic = new RemoteInputConnection(new WeakReference<>(() -> mContext), + IInputContext.Stub.asInterface(binding.getConnectionToken()), + mCancellationGroup); + InputBinding nu = new InputBinding(ic, binding); + final Message message = mCaller.obtainMessageO(DO_BIND_INPUT, nu); + mCaller.sendMessage(message); + } + + /** This is called when an app unbinds input or when the service is disabled. */ + public void unbindInput() { + if (mCancellationGroup != null) { + // Signal the flag then forget it. + mCancellationGroup.cancelAll(); + mCancellationGroup = null; + } else { + Log.e(LOG_TAG, "unbindInput must be paired with bindInput."); + } + mCaller.sendMessage(mCaller.obtainMessage(DO_UNBIND_INPUT)); + } + + /** This is called when an app starts input or when the service is enabled. */ + public void startInput(IBinder startInputToken, IInputContext inputContext, + EditorInfo editorInfo, boolean restarting) { + if (mCancellationGroup == null) { + Log.e(LOG_TAG, "startInput must be called after bindInput."); + mCancellationGroup = new CancellationGroup(); + } + final Message message = mCaller.obtainMessageOOOOII(DO_START_INPUT, startInputToken, + inputContext, editorInfo, mCancellationGroup, restarting ? 1 : 0, + 0 /* unused */); + mCaller.sendMessage(message); + } + @Override public void onMotionEvent(MotionEvent event) { final Message message = PooledLambda.obtainMessage( @@ -2948,6 +3139,50 @@ public abstract class AccessibilityService extends Service { } return; } + case DO_CREATE_IME_SESSION: { + if (mConnectionId != AccessibilityInteractionClient.NO_ID) { + IInputSessionWithIdCallback callback = + (IInputSessionWithIdCallback) message.obj; + mCallback.createImeSession(callback); + } + return; + } + case DO_SET_IME_SESSION_ENABLED: { + if (mConnectionId != AccessibilityInteractionClient.NO_ID) { + mCallback.setImeSessionEnabled((InputMethodSession) message.obj, + message.arg1 != 0); + } + return; + } + case DO_BIND_INPUT: { + if (mConnectionId != AccessibilityInteractionClient.NO_ID) { + mCallback.bindInput((InputBinding) message.obj); + } + return; + } + case DO_UNBIND_INPUT: { + if (mConnectionId != AccessibilityInteractionClient.NO_ID) { + mCallback.unbindInput(); + } + return; + } + case DO_START_INPUT: { + if (mConnectionId != AccessibilityInteractionClient.NO_ID) { + final SomeArgs args = (SomeArgs) message.obj; + final IBinder startInputToken = (IBinder) args.arg1; + final IInputContext inputContext = (IInputContext) args.arg2; + final EditorInfo info = (EditorInfo) args.arg3; + final CancellationGroup cancellationGroup = (CancellationGroup) args.arg4; + final boolean restarting = args.argi5 == 1; + final InputConnection ic = inputContext != null + ? new RemoteInputConnection(new WeakReference<>(() -> mContext), + inputContext, cancellationGroup) : null; + info.makeCompatible(mContext.getApplicationInfo().targetSdkVersion); + mCallback.startInput(ic, info, restarting, startInputToken); + args.recycle(); + } + return; + } default: Log.w(LOG_TAG, "Unknown message type " + message.what); } diff --git a/core/java/android/accessibilityservice/AccessibilityServiceInfo.java b/core/java/android/accessibilityservice/AccessibilityServiceInfo.java index 6988048c34e1e..85e78540332fc 100644 --- a/core/java/android/accessibilityservice/AccessibilityServiceInfo.java +++ b/core/java/android/accessibilityservice/AccessibilityServiceInfo.java @@ -390,6 +390,15 @@ public class AccessibilityServiceInfo implements Parcelable { */ public static final int FLAG_SEND_MOTION_EVENTS = 0x0004000; + /** + * This flag makes the AccessibilityService an input method editor with a subset of input + * method editor capabilities: get the {@link android.view.inputmethod.InputConnection} and get + * text selection change notifications. + * + * @see AccessibilityService#getInputMethod() + */ + public static final int FLAG_INPUT_METHOD_EDITOR = 0x0008000; + /** {@hide} */ public static final int FLAG_FORCE_DIRECT_BOOT_AWARE = 0x00010000; @@ -497,6 +506,7 @@ public class AccessibilityServiceInfo implements Parcelable { * @see #FLAG_ENABLE_ACCESSIBILITY_VOLUME * @see #FLAG_REQUEST_ACCESSIBILITY_BUTTON * @see #FLAG_REQUEST_SHORTCUT_WARNING_DIALOG_SPOKEN_FEEDBACK + * @see #FLAG_INPUT_METHOD_EDITOR */ public int flags; @@ -1356,6 +1366,8 @@ public class AccessibilityServiceInfo implements Parcelable { return "FLAG_REQUEST_FINGERPRINT_GESTURES"; case FLAG_REQUEST_SHORTCUT_WARNING_DIALOG_SPOKEN_FEEDBACK: return "FLAG_REQUEST_SHORTCUT_WARNING_DIALOG_SPOKEN_FEEDBACK"; + case FLAG_INPUT_METHOD_EDITOR: + return "FLAG_INPUT_METHOD_EDITOR"; default: return null; } diff --git a/core/java/android/accessibilityservice/IAccessibilityServiceClient.aidl b/core/java/android/accessibilityservice/IAccessibilityServiceClient.aidl index 375383d5d8588..94da61f82d294 100644 --- a/core/java/android/accessibilityservice/IAccessibilityServiceClient.aidl +++ b/core/java/android/accessibilityservice/IAccessibilityServiceClient.aidl @@ -24,6 +24,11 @@ import android.accessibilityservice.AccessibilityGestureEvent; import android.accessibilityservice.MagnificationConfig; import android.view.KeyEvent; import android.view.MotionEvent; +import android.view.inputmethod.EditorInfo; +import android.view.inputmethod.InputBinding; +import com.android.internal.view.IInputContext; +import com.android.internal.view.IInputMethodSession; +import com.android.internal.view.IInputSessionWithIdCallback; /** * Top-level interface to an accessibility service component. @@ -63,4 +68,15 @@ import android.view.MotionEvent; void onAccessibilityButtonAvailabilityChanged(boolean available); void onSystemActionsChanged(); + + void createImeSession(IInputSessionWithIdCallback callback); + + void setImeSessionEnabled(IInputMethodSession session, boolean enabled); + + void bindInput(in InputBinding binding); + + void unbindInput(); + + void startInput(in IBinder startInputToken, in IInputContext inputContext, + in EditorInfo editorInfo, boolean restarting); } diff --git a/core/java/android/accessibilityservice/InputMethod.java b/core/java/android/accessibilityservice/InputMethod.java new file mode 100644 index 0000000000000..001d804b22d6a --- /dev/null +++ b/core/java/android/accessibilityservice/InputMethod.java @@ -0,0 +1,637 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.accessibilityservice; + +import static android.os.Trace.TRACE_TAG_WINDOW_MANAGER; + +import android.annotation.CallbackExecutor; +import android.annotation.IntRange; +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.annotation.SuppressLint; +import android.graphics.Rect; +import android.inputmethodservice.IInputMethodSessionWrapper; +import android.inputmethodservice.RemoteInputConnection; +import android.os.Bundle; +import android.os.RemoteException; +import android.os.Trace; +import android.util.Log; +import android.view.KeyCharacterMap; +import android.view.KeyEvent; +import android.view.MotionEvent; +import android.view.inputmethod.CompletionInfo; +import android.view.inputmethod.CursorAnchorInfo; +import android.view.inputmethod.EditorInfo; +import android.view.inputmethod.ExtractedText; +import android.view.inputmethod.InputBinding; +import android.view.inputmethod.InputConnection; +import android.view.inputmethod.InputMethodManager; +import android.view.inputmethod.InputMethodSession; +import android.view.inputmethod.SurroundingText; +import android.view.inputmethod.TextAttribute; + +import com.android.internal.view.IInputContext; +import com.android.internal.view.IInputSessionWithIdCallback; + +import java.util.concurrent.Executor; + +/** + * This class provides input method APIs. Some public methods such as + * @link #onUpdateSelection(int, int, int, int, int, int)} do nothing by default and service + * developers should override them as needed. Developers should also override + * {@link AccessibilityService#onCreateInputMethod()} to return + * their custom InputMethod implementation. Accessibility services also need to set the + * {@link AccessibilityServiceInfo#FLAG_INPUT_METHOD_EDITOR} flag to use input method APIs. + */ +public class InputMethod { + private static final String LOG_TAG = "A11yInputMethod"; + + private final AccessibilityService mService; + private InputBinding mInputBinding; + private InputConnection mInputConnection; + private boolean mInputStarted; + private InputConnection mStartedInputConnection; + private EditorInfo mInputEditorInfo; + + protected InputMethod(@NonNull AccessibilityService service) { + mService = service; + } + + /** + * Retrieve the currently active InputConnection that is bound to + * the input method, or null if there is none. + */ + @Nullable + public final AccessibilityInputConnection getCurrentInputConnection() { + InputConnection ic = mStartedInputConnection; + if (ic != null) { + return new AccessibilityInputConnection(ic); + } + if (mInputConnection != null) { + return new AccessibilityInputConnection(mInputConnection); + } + return null; + } + + /** + * Whether the input has started. + */ + public final boolean getCurrentInputStarted() { + return mInputStarted; + } + + /** + * Get the EditorInfo which describes several attributes of a text editing object + * that an accessibility service is communicating with (typically an EditText). + */ + @Nullable + public final EditorInfo getCurrentInputEditorInfo() { + return mInputEditorInfo; + } + + /** + * Called to inform the accessibility service that text input has started in an + * editor. You should use this callback to initialize the state of your + * input to match the state of the editor given to it. + * + * @param attribute The attributes of the editor that input is starting + * in. + * @param restarting Set to true if input is restarting in the same + * editor such as because the application has changed the text in + * the editor. Otherwise will be false, indicating this is a new + * session with the editor. + */ + public void onStartInput(@NonNull EditorInfo attribute, boolean restarting) { + // Intentionally empty + } + + /** + * Called to inform the accessibility service that text input has finished in + * the last editor. At this point there may be a call to + * {@link #onStartInput(EditorInfo, boolean)} to perform input in a + * new editor, or the accessibility service may be left idle. This method is + * not called when input restarts in the same editor. + * + *

The default + * implementation uses the InputConnection to clear any active composing + * text; you can override this (not calling the base class implementation) + * to perform whatever behavior you would like. + */ + public void onFinishInput() { + InputConnection ic = mStartedInputConnection != null ? mStartedInputConnection + : mInputConnection; + if (ic != null) { + ic.finishComposingText(); + } + } + + /** + * Called when the application has reported a new selection region of + * the text. This is called whether or not the accessibility service has requested + * extracted text updates, although if so it will not receive this call + * if the extracted text has changed as well. + * + *

Be careful about changing the text in reaction to this call with + * methods such as setComposingText, commitText or + * deleteSurroundingText. If the cursor moves as a result, this method + * will be called again, which may result in an infinite loop. + */ + public void onUpdateSelection(int oldSelStart, int oldSelEnd, int newSelStart, + int newSelEnd, int candidatesStart, int candidatesEnd) { + // Intentionally empty + } + + final void createImeSession(IInputSessionWithIdCallback callback) { + InputMethodSession session = onCreateInputMethodSessionInterface(); + try { + IInputMethodSessionWrapper wrap = + new IInputMethodSessionWrapper(mService, session, null); + callback.sessionCreated(wrap, mService.getConnectionId()); + } catch (RemoteException ignored) { + } + } + + final void setImeSessionEnabled(@NonNull InputMethodSession session, boolean enabled) { + ((InputMethodSessionForAccessibility) session).setEnabled(enabled); + } + + final void bindInput(@NonNull InputBinding binding) { + Trace.traceBegin(TRACE_TAG_WINDOW_MANAGER, "AccessibilityService.bindInput"); + mInputBinding = binding; + mInputConnection = binding.getConnection(); + Log.v(LOG_TAG, "bindInput(): binding=" + binding + + " ic=" + mInputConnection); + Trace.traceEnd(TRACE_TAG_WINDOW_MANAGER); + } + + final void unbindInput() { + Log.v(LOG_TAG, "unbindInput(): binding=" + mInputBinding + + " ic=" + mInputConnection); + // Unbind input is per process per display. + mInputBinding = null; + mInputConnection = null; + } + + final void startInput(@Nullable InputConnection ic, @NonNull EditorInfo attribute) { + Log.v(LOG_TAG, "startInput(): editor=" + attribute); + Trace.traceBegin(TRACE_TAG_WINDOW_MANAGER, "IMS.startInput"); + doStartInput(ic, attribute, false /* restarting */); + Trace.traceEnd(TRACE_TAG_WINDOW_MANAGER); + } + + final void restartInput(@Nullable InputConnection ic, @NonNull EditorInfo attribute) { + Log.v(LOG_TAG, "restartInput(): editor=" + attribute); + Trace.traceBegin(TRACE_TAG_WINDOW_MANAGER, "IMS.restartInput"); + doStartInput(ic, attribute, true /* restarting */); + Trace.traceEnd(TRACE_TAG_WINDOW_MANAGER); + } + + + final void doStartInput(InputConnection ic, EditorInfo attribute, boolean restarting) { + if (!restarting && mInputStarted) { + doFinishInput(); + } + mInputStarted = true; + mStartedInputConnection = ic; + mInputEditorInfo = attribute; + Log.v(LOG_TAG, "CALL: onStartInput"); + onStartInput(attribute, restarting); + } + + final void doFinishInput() { + Log.v(LOG_TAG, "CALL: doFinishInput"); + if (mInputStarted) { + Log.v(LOG_TAG, "CALL: onFinishInput"); + onFinishInput(); + } + mInputStarted = false; + mStartedInputConnection = null; + } + + private InputMethodSession onCreateInputMethodSessionInterface() { + return new InputMethodSessionForAccessibility(); + } + + /** + * This class provides the allowed list of {@link InputConnection} APIs for + * accessibility services. + */ + public final class AccessibilityInputConnection { + private InputConnection mIc; + AccessibilityInputConnection(InputConnection ic) { + this.mIc = ic; + } + + /** + * Commit text to the text box and set the new cursor position. This method is + * used to allow the IME to provide extra information while setting up text. + * + *

This method commits the contents of the currently composing text, and then + * moves the cursor according to {@code newCursorPosition}. If there + * is no composing text when this method is called, the new text is + * inserted at the cursor position, removing text inside the selection + * if any. + * + *

Calling this method will cause the editor to call + * {@link #onUpdateSelection(int, int, int, int, + * int, int)} on the current accessibility service after the batch input is over. + * Editor authors, for this to happen you need to + * make the changes known to the accessibility service by calling + * {@link InputMethodManager#updateSelection(View, int, int, int, int)}, + * but be careful to wait until the batch edit is over if one is + * in progress.

+ * + * @param text The text to commit. This may include styles. + * @param newCursorPosition The new cursor position around the text, + * in Java characters. If > 0, this is relative to the end + * of the text - 1; if <= 0, this is relative to the start + * of the text. So a value of 1 will always advance the cursor + * to the position after the full text being inserted. Note that + * this means you can't position the cursor within the text, + * because the editor can make modifications to the text + * you are providing so it is not possible to correctly specify + * locations there. + * @param textAttribute The extra information about the text. + */ + public void commitText(@NonNull CharSequence text, int newCursorPosition, + @Nullable TextAttribute textAttribute) { + if (mIc != null) { + mIc.commitText(text, newCursorPosition, textAttribute); + } + } + + /** + * Set the selection of the text editor. To set the cursor + * position, start and end should have the same value. + * + *

Since this moves the cursor, calling this method will cause + * the editor to call + * {@link android.inputmethodservice.InputMethodService#onUpdateSelection(int, int, int, + * int,int, int)} on the current IME after the batch input is over. + * Editor authors, for this to happen you need to + * make the changes known to the input method by calling + * {@link InputMethodManager#updateSelection(View, int, int, int, int)}, + * but be careful to wait until the batch edit is over if one is + * in progress.

+ * + *

This has no effect on the composing region which must stay + * unchanged. The order of start and end is not important. In + * effect, the region from start to end and the region from end to + * start is the same. Editor authors, be ready to accept a start + * that is greater than end.

+ * + * @param start the character index where the selection should start. + * @param end the character index where the selection should end. + */ + public void setSelection(int start, int end) { + if (mIc != null) { + mIc.setSelection(start, end); + } + } + + /** + * Gets the surrounding text around the current cursor, with beforeLength + * characters of text before the cursor (start of the selection), afterLength + * characters of text after the cursor (end of the selection), and all of the selected + * text. The range are for java characters, not glyphs that can be multiple characters. + * + *

This method may fail either if the input connection has become invalid (such as its + * process crashing), or the client is taking too long to respond with the text (it is + * given a couple seconds to return), or the protocol is not supported. In any of these + * cases, null is returned. + * + *

This method does not affect the text in the editor in any way, nor does it affect the + * selection or composing spans.

+ * + *

If {@link InputConnection#GET_TEXT_WITH_STYLES} is supplied as flags, the editor + * should return a {@link android.text.Spanned} with all the spans set on the text.

+ * + *

Accessibility service authors: please consider this will trigger an + * IPC round-trip that will take some time. Assume this method consumes a lot of time. + * + * @param beforeLength The expected length of the text before the cursor. + * @param afterLength The expected length of the text after the cursor. + * @param flags Supplies additional options controlling how the text is returned. May be + * either {@code 0} or {@link InputConnection#GET_TEXT_WITH_STYLES}. + * @return an {@link android.view.inputmethod.SurroundingText} object describing the + * surrounding text and state of selection, or null if the input connection is no longer + * valid, or the editor can't comply with the request for some reason, or the application + * does not implement this method. The length of the returned text might be less than the + * sum of beforeLength and afterLength . + * @throws IllegalArgumentException if {@code beforeLength} or {@code afterLength} is + * negative. + */ + @Nullable + public SurroundingText getSurroundingText( + @IntRange(from = 0) int beforeLength, @IntRange(from = 0) int afterLength, + @InputConnection.GetTextType int flags) { + if (mIc != null) { + return mIc.getSurroundingText(beforeLength, afterLength, flags); + } + return null; + } + + /** + * Delete beforeLength characters of text before the + * current cursor position, and delete afterLength + * characters of text after the current cursor position, excluding + * the selection. Before and after refer to the order of the + * characters in the string, not to their visual representation: + * this means you don't have to figure out the direction of the + * text and can just use the indices as-is. + * + *

The lengths are supplied in Java chars, not in code points + * or in glyphs.

+ * + *

Since this method only operates on text before and after the + * selection, it can't affect the contents of the selection. This + * may affect the composing span if the span includes characters + * that are to be deleted, but otherwise will not change it. If + * some characters in the composing span are deleted, the + * composing span will persist but get shortened by however many + * chars inside it have been removed.

+ * + *

Accessibility service authors: please be careful not to + * delete only half of a surrogate pair. Also take care not to + * delete more characters than are in the editor, as that may have + * ill effects on the application. Calling this method will cause + * the editor to call + * {@link android.inputmethodservice.InputMethodService#onUpdateSelection(int, int, int, int, + * int, int)} on your service after the batch input is over.

+ * + *

Editor authors: please be careful of race + * conditions in implementing this call. An IME can make a change + * to the text or change the selection position and use this + * method right away; you need to make sure the effects are + * consistent with the results of the latest edits. Also, although + * the IME should not send lengths bigger than the contents of the + * string, you should check the values for overflows and trim the + * indices to the size of the contents to avoid crashes. Since + * this changes the contents of the editor, you need to make the + * changes known to the input method by calling + * {@link InputMethodManager#updateSelection(View, int, int, int, int)}, + * but be careful to wait until the batch edit is over if one is + * in progress.

+ * + * @param beforeLength The number of characters before the cursor to be deleted, in code + * unit. If this is greater than the number of existing characters between the + * beginning of the text and the cursor, then this method does not fail but deletes + * all the characters in that range. + * @param afterLength The number of characters after the cursor to be deleted, in code unit. + * If this is greater than the number of existing characters between the cursor and + * the end of the text, then this method does not fail but deletes all the characters + * in that range. + */ + public void deleteSurroundingText(int beforeLength, int afterLength) { + if (mIc != null) { + mIc.deleteSurroundingText(beforeLength, afterLength); + } + } + + /** + * Send a key event to the process that is currently attached + * through this input connection. The event will be dispatched + * like a normal key event, to the currently focused view; this + * generally is the view that is providing this InputConnection, + * but due to the asynchronous nature of this protocol that can + * not be guaranteed and the focus may have changed by the time + * the event is received. + * + *

This method can be used to send key events to the + * application. For example, an on-screen keyboard may use this + * method to simulate a hardware keyboard. There are three types + * of standard keyboards, numeric (12-key), predictive (20-key) + * and ALPHA (QWERTY). You can specify the keyboard type by + * specify the device id of the key event.

+ * + *

You will usually want to set the flag + * {@link KeyEvent#FLAG_SOFT_KEYBOARD KeyEvent.FLAG_SOFT_KEYBOARD} + * on all key event objects you give to this API; the flag will + * not be set for you.

+ * + *

Note that it's discouraged to send such key events in normal + * operation; this is mainly for use with + * {@link android.text.InputType#TYPE_NULL} type text fields. Use + * the {@link #commitText} family of methods to send text to the + * application instead.

+ * + * @param event The key event. + * + * @see KeyEvent + * @see KeyCharacterMap#NUMERIC + * @see KeyCharacterMap#PREDICTIVE + * @see KeyCharacterMap#ALPHA + */ + public void sendKeyEvent(@NonNull KeyEvent event) { + if (mIc != null) { + mIc.sendKeyEvent(event); + } + } + + /** + * Have the editor perform an action it has said it can do. + * + * @param editorAction This must be one of the action constants for + * {@link EditorInfo#imeOptions EditorInfo.imeOptions}, such as + * {@link EditorInfo#IME_ACTION_GO EditorInfo.EDITOR_ACTION_GO}, or the value of + * {@link EditorInfo#actionId EditorInfo.actionId} if a custom action is available. + */ + public void performEditorAction(int editorAction) { + if (mIc != null) { + mIc.performEditorAction(editorAction); + } + } + + /** + * Perform a context menu action on the field. The given id may be one of: + * {@link android.R.id#selectAll}, + * {@link android.R.id#startSelectingText}, {@link android.R.id#stopSelectingText}, + * {@link android.R.id#cut}, {@link android.R.id#copy}, + * {@link android.R.id#paste}, {@link android.R.id#copyUrl}, + * or {@link android.R.id#switchInputMethod} + */ + public void performContextMenuAction(int id) { + if (mIc != null) { + mIc.performContextMenuAction(id); + } + } + + /** + * Retrieve the current capitalization mode in effect at the + * current cursor position in the text. See + * {@link android.text.TextUtils#getCapsMode TextUtils.getCapsMode} + * for more information. + * + *

This method may fail either if the input connection has + * become invalid (such as its process crashing) or the client is + * taking too long to respond with the text (it is given a couple + * seconds to return). In either case, 0 is returned.

+ * + *

This method does not affect the text in the editor in any + * way, nor does it affect the selection or composing spans.

+ * + *

Editor authors: please be careful of race + * conditions in implementing this call. An IME can change the + * cursor position and use this method right away; you need to make + * sure the returned value is consistent with the results of the + * latest edits and changes to the cursor position.

+ * + * @param reqModes The desired modes to retrieve, as defined by + * {@link android.text.TextUtils#getCapsMode TextUtils.getCapsMode}. These + * constants are defined so that you can simply pass the current + * {@link EditorInfo#inputType TextBoxAttribute.contentType} value + * directly in to here. + * @return the caps mode flags that are in effect at the current + * cursor position. See TYPE_TEXT_FLAG_CAPS_* in {@link android.text.InputType}. + */ + public int getCursorCapsMode(int reqModes) { + if (mIc != null) { + return mIc.getCursorCapsMode(reqModes); + } + return 0; + } + + /** + * Clear the given meta key pressed states in the given input + * connection. + * + *

This can be used by the accessibility service to clear the meta key states set + * by a hardware keyboard with latched meta keys, if the editor + * keeps track of these.

+ * + * @param states The states to be cleared, may be one or more bits as + * per {@link KeyEvent#getMetaState() KeyEvent.getMetaState()}. + */ + public void clearMetaKeyStates(int states) { + if (mIc != null) { + mIc.clearMetaKeyStates(states); + } + } + } + + /** + * Concrete implementation of InputMethodSession that provides all of the standard behavior + * for an input method session. + */ + private final class InputMethodSessionForAccessibility implements InputMethodSession { + boolean mEnabled = true; + + public void setEnabled(boolean enabled) { + mEnabled = enabled; + } + + @Override + public void finishInput() { + if (mEnabled) { + doFinishInput(); + } + } + + @Override + public void updateSelection(int oldSelStart, int oldSelEnd, int newSelStart, + int newSelEnd, int candidatesStart, int candidatesEnd) { + if (mEnabled) { + InputMethod.this.onUpdateSelection(oldSelEnd, oldSelEnd, newSelStart, + newSelEnd, candidatesStart, candidatesEnd); + } + } + + @Override + public void viewClicked(boolean focusChanged) { + } + + @Override + public void updateCursor(@NonNull Rect newCursor) { + } + + @Override + public void displayCompletions( + @SuppressLint("ArrayReturn") @NonNull CompletionInfo[] completions) { + } + + @Override + public void updateExtractedText(int token, @NonNull ExtractedText text) { + } + + public void dispatchKeyEvent(int seq, @NonNull KeyEvent event, + @NonNull @CallbackExecutor Executor executor, @NonNull EventCallback callback) { + } + + @Override + public void dispatchKeyEvent(int seq, @NonNull KeyEvent event, + @NonNull EventCallback callback) { + } + + public void dispatchTrackballEvent(int seq, @NonNull MotionEvent event, + @NonNull @CallbackExecutor Executor executor, @NonNull EventCallback callback) { + } + + @Override + public void dispatchTrackballEvent(int seq, @NonNull MotionEvent event, + @NonNull EventCallback callback) { + } + + public void dispatchGenericMotionEvent(int seq, @NonNull MotionEvent event, + @NonNull @CallbackExecutor Executor executor, @NonNull EventCallback callback) { + } + + @Override + public void dispatchGenericMotionEvent(int seq, @NonNull MotionEvent event, + @NonNull EventCallback callback) { + } + + @Override + public void appPrivateCommand(@NonNull String action, @NonNull Bundle data) { + } + + @Override + public void toggleSoftInput(int showFlags, int hideFlags) { + } + + @Override + public void updateCursorAnchorInfo(@NonNull CursorAnchorInfo cursorAnchorInfo) { + } + + @Override + public void notifyImeHidden() { + } + + @Override + public void removeImeSurface() { + } + + /** + * {@inheritDoc} + */ + @Override + public void invalidateInputInternal(EditorInfo editorInfo, IInputContext inputContext, + int sessionId) { + // TODO(b/217788708): Add automated test. + if (mStartedInputConnection instanceof RemoteInputConnection) { + final RemoteInputConnection ric = + (RemoteInputConnection) mStartedInputConnection; + if (!ric.isSameConnection(inputContext)) { + // This is not an error, and can be safely ignored. + return; + } + editorInfo.makeCompatible( + mService.getApplicationInfo().targetSdkVersion); + restartInput(new RemoteInputConnection(ric, sessionId), editorInfo); + } + } + } +} diff --git a/core/java/android/app/UiAutomation.java b/core/java/android/app/UiAutomation.java index b41b5f005f1f3..2af8905fa3afa 100644 --- a/core/java/android/app/UiAutomation.java +++ b/core/java/android/app/UiAutomation.java @@ -63,9 +63,14 @@ import android.view.accessibility.AccessibilityInteractionClient; import android.view.accessibility.AccessibilityNodeInfo; import android.view.accessibility.AccessibilityWindowInfo; import android.view.accessibility.IAccessibilityInteractionConnection; +import android.view.inputmethod.EditorInfo; +import android.view.inputmethod.InputBinding; +import android.view.inputmethod.InputConnection; +import android.view.inputmethod.InputMethodSession; import com.android.internal.annotations.GuardedBy; import com.android.internal.util.function.pooled.PooledLambda; +import com.android.internal.view.IInputSessionWithIdCallback; import libcore.io.IoUtils; @@ -1565,6 +1570,29 @@ public final class UiAutomation { /* do nothing */ } + @Override + public void createImeSession(IInputSessionWithIdCallback callback) { + /* do nothing */ + } + + @Override + public void setImeSessionEnabled(InputMethodSession session, boolean enabled) { + } + + @Override + public void bindInput(InputBinding binding) { + } + + @Override + public void unbindInput() { + } + + @Override + public void startInput(@Nullable InputConnection inputConnection, + @NonNull EditorInfo editorInfo, boolean restarting, + @NonNull IBinder startInputToken) { + } + @Override public boolean onGesture(AccessibilityGestureEvent gestureEvent) { /* do nothing */ diff --git a/core/java/android/inputmethodservice/IInputMethodSessionWrapper.java b/core/java/android/inputmethodservice/IInputMethodSessionWrapper.java index eccbb403b3068..75356d1ce9944 100644 --- a/core/java/android/inputmethodservice/IInputMethodSessionWrapper.java +++ b/core/java/android/inputmethodservice/IInputMethodSessionWrapper.java @@ -41,7 +41,9 @@ import com.android.internal.os.SomeArgs; import com.android.internal.view.IInputContext; import com.android.internal.view.IInputMethodSession; -class IInputMethodSessionWrapper extends IInputMethodSession.Stub +/** @hide */ +// TODO(b/215636776): move IInputMethodSessionWrapper to proper package +public class IInputMethodSessionWrapper extends IInputMethodSession.Stub implements HandlerCaller.Callback { private static final String TAG = "InputMethodWrapper"; diff --git a/core/java/android/inputmethodservice/InputMethodServiceInternal.java b/core/java/android/inputmethodservice/InputMethodServiceInternal.java index 7cd4ff61b2c80..09dbb27359b03 100644 --- a/core/java/android/inputmethodservice/InputMethodServiceInternal.java +++ b/core/java/android/inputmethodservice/InputMethodServiceInternal.java @@ -18,6 +18,7 @@ package android.inputmethodservice; import android.annotation.NonNull; import android.annotation.Nullable; +import android.annotation.SuppressLint; import android.content.Context; import android.os.Bundle; import android.view.inputmethod.InputConnection; @@ -31,8 +32,11 @@ import java.io.PrintWriter; * framework classes for internal use. * *

CAVEATS: {@link AbstractInputMethodService} does not support all the methods here.

+ * + * @hide */ -interface InputMethodServiceInternal { +// TODO(b/215636776): move InputMethodServiceInternal to proper package +public interface InputMethodServiceInternal { /** * @return {@link Context} associated with the service. */ @@ -70,7 +74,8 @@ interface InputMethodServiceInternal { * closed for you after you return. * @param args additional arguments to the dump request. */ - default void dump(FileDescriptor fd, PrintWriter fout, String[] args) { + default void dump(@SuppressLint("UseParcelFileDescriptor") @NonNull FileDescriptor fd, + @NonNull PrintWriter fout, @NonNull String[] args) { } /** @@ -81,6 +86,6 @@ interface InputMethodServiceInternal { * @param where {@code where} parameter to be passed. * @param icProto {@code icProto} parameter to be passed. */ - default void triggerServiceDump(String where, @Nullable byte[] icProto) { + default void triggerServiceDump(@NonNull String where, @Nullable byte[] icProto) { } } diff --git a/core/java/android/inputmethodservice/RemoteInputConnection.java b/core/java/android/inputmethodservice/RemoteInputConnection.java index 9ef2579d04e20..6b7815d0f7323 100644 --- a/core/java/android/inputmethodservice/RemoteInputConnection.java +++ b/core/java/android/inputmethodservice/RemoteInputConnection.java @@ -53,8 +53,11 @@ import java.util.concurrent.CompletableFuture; * *

See also {@link IInputContext} for the actual {@link android.os.Binder} IPC protocols under * the hood.

+ * + * @hide */ -final class RemoteInputConnection implements InputConnection { +// TODO(b/215636776): move RemoteInputConnection to proper package +public final class RemoteInputConnection implements InputConnection { private static final String TAG = "RemoteInputConnection"; private static final int MAX_WAIT_TIME_MILLIS = 2000; @@ -95,7 +98,7 @@ final class RemoteInputConnection implements InputConnection { @NonNull private final CancellationGroup mCancellationGroup; - RemoteInputConnection( + public RemoteInputConnection( @NonNull WeakReference inputMethodService, IInputContext inputContext, @NonNull CancellationGroup cancellationGroup) { mImsInternal = new InputMethodServiceInternalHolder(inputMethodService); @@ -108,7 +111,7 @@ final class RemoteInputConnection implements InputConnection { return mInvoker.isSameConnection(inputContext); } - RemoteInputConnection(@NonNull RemoteInputConnection original, int sessionId) { + public RemoteInputConnection(@NonNull RemoteInputConnection original, int sessionId) { mImsInternal = original.mImsInternal; mInvoker = original.mInvoker.cloneWithSessionId(sessionId); mCancellationGroup = original.mCancellationGroup; diff --git a/core/java/android/view/inputmethod/InputMethodManager.java b/core/java/android/view/inputmethod/InputMethodManager.java index f480b24bdba31..c713a54fdf7d3 100644 --- a/core/java/android/view/inputmethod/InputMethodManager.java +++ b/core/java/android/view/inputmethod/InputMethodManager.java @@ -115,6 +115,7 @@ import java.util.Map; import java.util.Objects; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; /** * Central system API to the overall input method framework (IMF) architecture, @@ -426,6 +427,8 @@ public final class InputMethodManager { int mCursorSelEnd; int mCursorCandStart; int mCursorCandEnd; + int mInitialSelStart; + int mInitialSelEnd; /** * The instance that has previously been sent to the input method. @@ -468,6 +471,13 @@ public final class InputMethodManager { @Nullable @GuardedBy("mH") private InputMethodSessionWrapper mCurrentInputMethodSession = null; + /** + * Encapsulates IPCs to the currently connected AccessibilityServices. + */ + @Nullable + @GuardedBy("mH") + private final SparseArray mAccessibilityInputMethodSession = + new SparseArray<>(); InputChannel mCurChannel; ImeInputEventSender mCurSender; @@ -499,6 +509,8 @@ public final class InputMethodManager { static final int MSG_TIMEOUT_INPUT_EVENT = 6; static final int MSG_FLUSH_INPUT_EVENT = 7; static final int MSG_REPORT_FULLSCREEN_MODE = 10; + static final int MSG_BIND_ACCESSIBILITY_SERVICE = 11; + static final int MSG_UNBIND_ACCESSIBILITY_SERVICE = 12; private static boolean isAutofillUIShowing(View servedView) { AutofillManager afm = servedView.getContext().getSystemService(AutofillManager.class); @@ -626,6 +638,7 @@ public final class InputMethodManager { if (mCurrentInputMethodSession != null) { mCurrentInputMethodSession.finishInput(); } + forAccessibilitySessions(InputMethodSessionWrapper::finishInput); } } @@ -881,6 +894,7 @@ public final class InputMethodManager { if (mBindSequence != sequence) { return; } + clearAllAccessibilityBindingLocked(); clearBindingLocked(); // If we were actively using the last input method, then // we would like to re-connect to the next input method. @@ -896,6 +910,73 @@ public final class InputMethodManager { } return; } + case MSG_BIND_ACCESSIBILITY_SERVICE: { + final int id = msg.arg1; + final InputBindResult res = (InputBindResult) msg.obj; + if (DEBUG) { + Log.i(TAG, "handleMessage: MSG_BIND_ACCESSIBILITY " + res.sequence + + "," + res.id); + } + synchronized (mH) { + if (mBindSequence < 0 || mBindSequence != res.sequence) { + Log.w(TAG, "Ignoring onBind: cur seq=" + mBindSequence + + ", given seq=" + res.sequence); + if (res.channel != null && res.channel != mCurChannel) { + res.channel.dispose(); + } + return; + } + + // Since IMM can start inputting text before a11y sessions are back, + // we send a notification so that the a11y service knows the session is + // registered and update the a11y service with the current cursor positions. + if (res.accessibilitySessions != null) { + InputMethodSessionWrapper wrapper = + InputMethodSessionWrapper.createOrNull( + res.accessibilitySessions.get(id)); + if (wrapper != null) { + mAccessibilityInputMethodSession.put(id, wrapper); + if (mServedInputConnection != null) { + wrapper.updateSelection(mInitialSelStart, mInitialSelEnd, + mCursorSelStart, mCursorSelEnd, mCursorCandStart, + mCursorCandEnd); + } else { + // If an a11y service binds before input starts, we should still + // send a notification because the a11y service doesn't know it + // binds before or after input starts, it may wonder if it binds + // after input starts, why it doesn't receive a notification of + // the current cursor positions. + wrapper.updateSelection(-1, -1, + -1, -1, -1, + -1); + } + } + } + mBindSequence = res.sequence; + } + startInputInner(StartInputReason.BOUND_ACCESSIBILITY_SESSION_TO_IMMS, null, + 0, 0, 0); + return; + } + case MSG_UNBIND_ACCESSIBILITY_SERVICE: { + final int sequence = msg.arg1; + final int id = msg.arg2; + if (DEBUG) { + Log.i(TAG, "handleMessage: MSG_UNBIND_ACCESSIBILITY_SERVICE " + + sequence + " id=" + id); + } + synchronized (mH) { + if (mBindSequence != sequence) { + if (DEBUG) { + Log.i(TAG, "mBindSequence =" + mBindSequence + " sequence =" + + sequence + " id=" + id); + } + return; + } + clearAccessibilityBindingLocked(id); + } + return; + } case MSG_SET_ACTIVE: { final boolean active = msg.arg1 != 0; final boolean fullscreen = msg.arg2 != 0; @@ -995,11 +1076,21 @@ public final class InputMethodManager { mH.obtainMessage(MSG_BIND, res).sendToTarget(); } + @Override + public void onBindAccessibilityService(InputBindResult res, int id) { + mH.obtainMessage(MSG_BIND_ACCESSIBILITY_SERVICE, id, 0, res).sendToTarget(); + } + @Override public void onUnbindMethod(int sequence, @UnbindReason int unbindReason) { mH.obtainMessage(MSG_UNBIND, sequence, unbindReason).sendToTarget(); } + @Override + public void onUnbindAccessibilityService(int sequence, int id) { + mH.obtainMessage(MSG_UNBIND_ACCESSIBILITY_SERVICE, sequence, id).sendToTarget(); + } + @Override public void setActive(boolean active, boolean fullscreen, boolean reportToImeController) { mH.obtainMessage(MSG_SET_ACTIVE, active ? 1 : 0, fullscreen ? 1 : 0, @@ -1420,16 +1511,36 @@ public final class InputMethodManager { /** * Reset all of the state associated with being bound to an input method. */ + @GuardedBy("mH") void clearBindingLocked() { if (DEBUG) Log.v(TAG, "Clearing binding!"); clearConnectionLocked(); setInputChannelLocked(null); + // We only reset sequence number for input method, but not accessibility. mBindSequence = -1; mCurId = null; mCurMethod = null; // for @UnsupportedAppUsage mCurrentInputMethodSession = null; } + /** + * Reset all of the state associated with being bound to an accessibility service. + */ + @GuardedBy("mH") + void clearAccessibilityBindingLocked(int id) { + if (DEBUG) Log.v(TAG, "Clearing accessibility binding " + id); + mAccessibilityInputMethodSession.remove(id); + } + + /** + * Reset all of the state associated with being bound to all ccessibility services. + */ + @GuardedBy("mH") + void clearAllAccessibilityBindingLocked() { + if (DEBUG) Log.v(TAG, "Clearing all accessibility bindings"); + mAccessibilityInputMethodSession.clear(); + } + void setInputChannelLocked(InputChannel channel) { if (mCurChannel == channel) { return; @@ -1938,6 +2049,8 @@ public final class InputMethodManager { editorInfo.setInitialSurroundingTextInternal(textSnapshot.getSurroundingText()); mCurrentInputMethodSession.invalidateInput(editorInfo, mServedInputConnection, sessionId); + forAccessibilitySessions(wrapper -> wrapper.invalidateInput(editorInfo, + mServedInputConnection, sessionId)); } } @@ -2080,6 +2193,8 @@ public final class InputMethodManager { if (ic != null) { mCursorSelStart = tba.initialSelStart; mCursorSelEnd = tba.initialSelEnd; + mInitialSelStart = mCursorSelStart; + mInitialSelEnd = mCursorSelEnd; mCursorCandStart = -1; mCursorCandEnd = -1; mCursorRect.setEmpty(); @@ -2128,6 +2243,17 @@ public final class InputMethodManager { mBindSequence = res.sequence; mCurMethod = res.method; // for @UnsupportedAppUsage mCurrentInputMethodSession = InputMethodSessionWrapper.createOrNull(res.method); + mAccessibilityInputMethodSession.clear(); + if (res.accessibilitySessions != null) { + for (int i = 0; i < res.accessibilitySessions.size(); i++) { + InputMethodSessionWrapper wrapper = InputMethodSessionWrapper.createOrNull( + res.accessibilitySessions.valueAt(i)); + if (wrapper != null) { + mAccessibilityInputMethodSession.append( + res.accessibilitySessions.keyAt(i), wrapper); + } + } + } mCurId = res.id; } else if (res.channel != null && res.channel != mCurChannel) { res.channel.dispose(); @@ -2137,8 +2263,10 @@ public final class InputMethodManager { mRestartOnNextWindowFocus = true; break; } - if (mCurrentInputMethodSession != null && mCompletions != null) { - mCurrentInputMethodSession.displayCompletions(mCompletions); + if (mCompletions != null) { + if (mCurrentInputMethodSession != null) { + mCurrentInputMethodSession.displayCompletions(mCompletions); + } } } @@ -2369,6 +2497,8 @@ public final class InputMethodManager { mCursorCandEnd = candidatesEnd; mCurrentInputMethodSession.updateSelection( oldSelStart, oldSelEnd, selStart, selEnd, candidatesStart, candidatesEnd); + forAccessibilitySessions(wrapper -> wrapper.updateSelection(oldSelStart, + oldSelEnd, selStart, selEnd, candidatesStart, candidatesEnd)); } } } @@ -3233,6 +3363,11 @@ public final class InputMethodManager { } else { p.println(" mCurMethod= null"); } + for (int i = 0; i < mAccessibilityInputMethodSession.size(); i++) { + p.println(" mAccessibilityInputMethodSession(" + + mAccessibilityInputMethodSession.keyAt(i) + ")=" + + mAccessibilityInputMethodSession.valueAt(i)); + } p.println(" mCurRootView=" + mCurRootView); p.println(" mServedView=" + getServedViewLocked()); p.println(" mNextServedView=" + getNextServedViewLocked()); @@ -3377,4 +3512,10 @@ public final class InputMethodManager { } } } + + private void forAccessibilitySessions(Consumer consumer) { + for (int i = 0; i < mAccessibilityInputMethodSession.size(); i++) { + consumer.accept(mAccessibilityInputMethodSession.valueAt(i)); + } + } } diff --git a/core/java/com/android/internal/inputmethod/InputBindResult.java b/core/java/com/android/internal/inputmethod/InputBindResult.java index 1357bac30667c..e83840177a733 100644 --- a/core/java/com/android/internal/inputmethod/InputBindResult.java +++ b/core/java/com/android/internal/inputmethod/InputBindResult.java @@ -25,6 +25,7 @@ import android.content.ServiceConnection; import android.os.IBinder; import android.os.Parcel; import android.os.Parcelable; +import android.util.SparseArray; import android.view.InputChannel; import com.android.internal.view.IInputMethodSession; @@ -53,7 +54,8 @@ public final class InputBindResult implements Parcelable { ResultCode.ERROR_NOT_IME_TARGET_WINDOW, ResultCode.ERROR_NO_EDITOR, ResultCode.ERROR_DISPLAY_ID_MISMATCH, - ResultCode.ERROR_INVALID_DISPLAY_ID + ResultCode.ERROR_INVALID_DISPLAY_ID, + ResultCode.SUCCESS_WITH_ACCESSIBILITY_SESSION }) public @interface ResultCode { /** @@ -168,6 +170,7 @@ public final class InputBindResult implements Parcelable { * display. */ int ERROR_INVALID_DISPLAY_ID = 15; + int SUCCESS_WITH_ACCESSIBILITY_SESSION = 16; } @ResultCode @@ -178,6 +181,11 @@ public final class InputBindResult implements Parcelable { */ public final IInputMethodSession method; + /** + * The accessibility services. + */ + public SparseArray accessibilitySessions; + /** * The input channel used to send input events to this IME. */ @@ -204,6 +212,8 @@ public final class InputBindResult implements Parcelable { * * @param result A result code defined in {@link ResultCode}. * @param method {@link IInputMethodSession} to interact with the IME. + * @param accessibilitySessions {@link IInputMethodSession} to interact with accessibility + * services. * @param channel {@link InputChannel} to forward input events to the IME. * @param id The {@link String} representations of the IME, which is the same as * {@link android.view.inputmethod.InputMethodInfo#getId()} and @@ -213,10 +223,12 @@ public final class InputBindResult implements Parcelable { * {@code suppressesSpellChecker="true"}. */ public InputBindResult(@ResultCode int result, - IInputMethodSession method, InputChannel channel, String id, int sequence, + IInputMethodSession method, SparseArray accessibilitySessions, + InputChannel channel, String id, int sequence, boolean isInputMethodSuppressingSpellChecker) { this.result = result; this.method = method; + this.accessibilitySessions = accessibilitySessions; this.channel = channel; this.id = id; this.sequence = sequence; @@ -226,6 +238,19 @@ public final class InputBindResult implements Parcelable { private InputBindResult(Parcel source) { result = source.readInt(); method = IInputMethodSession.Stub.asInterface(source.readStrongBinder()); + int n = source.readInt(); + if (n < 0) { + accessibilitySessions = null; + } else { + accessibilitySessions = new SparseArray<>(n); + while (n > 0) { + int key = source.readInt(); + IInputMethodSession value = + IInputMethodSession.Stub.asInterface(source.readStrongBinder()); + accessibilitySessions.append(key, value); + n--; + } + } if (source.readInt() != 0) { channel = InputChannel.CREATOR.createFromParcel(source); } else { @@ -254,6 +279,18 @@ public final class InputBindResult implements Parcelable { public void writeToParcel(Parcel dest, int flags) { dest.writeInt(result); dest.writeStrongInterface(method); + if (accessibilitySessions == null) { + dest.writeInt(-1); + } else { + int n = accessibilitySessions.size(); + dest.writeInt(n); + int i = 0; + while (i < n) { + dest.writeInt(accessibilitySessions.keyAt(i)); + dest.writeStrongInterface(accessibilitySessions.valueAt(i)); + i++; + } + } if (channel != null) { dest.writeInt(1); channel.writeToParcel(dest, flags); @@ -329,7 +366,7 @@ public final class InputBindResult implements Parcelable { } private static InputBindResult error(@ResultCode int result) { - return new InputBindResult(result, null, null, null, -1, false); + return new InputBindResult(result, null, null, null, null, -1, false); } /** diff --git a/core/java/com/android/internal/inputmethod/InputMethodDebug.java b/core/java/com/android/internal/inputmethod/InputMethodDebug.java index bf094dbd8f1e7..d6697684f79e7 100644 --- a/core/java/com/android/internal/inputmethod/InputMethodDebug.java +++ b/core/java/com/android/internal/inputmethod/InputMethodDebug.java @@ -64,6 +64,8 @@ public final class InputMethodDebug { return "DEACTIVATED_BY_IMMS"; case StartInputReason.SESSION_CREATED_BY_IME: return "SESSION_CREATED_BY_IME"; + case StartInputReason.BOUND_ACCESSIBILITY_SESSION_TO_IMMS: + return "BOUND_ACCESSIBILITY_SESSION_TO_IMMS"; default: return "Unknown=" + reason; } @@ -91,6 +93,8 @@ public final class InputMethodDebug { return "SWITCH_IME_FAILED"; case UnbindReason.SWITCH_USER: return "SWITCH_USER"; + case UnbindReason.ACCESSIBILITY_SERVICE_DISABLED: + return "ACCESSIBILITY_SERVICE_DISABLED"; default: return "Unknown=" + reason; } diff --git a/core/java/com/android/internal/inputmethod/StartInputReason.java b/core/java/com/android/internal/inputmethod/StartInputReason.java index 2ba708dd93124..1263466703ace 100644 --- a/core/java/com/android/internal/inputmethod/StartInputReason.java +++ b/core/java/com/android/internal/inputmethod/StartInputReason.java @@ -38,7 +38,9 @@ import java.lang.annotation.Retention; StartInputReason.UNBOUND_FROM_IMMS, StartInputReason.ACTIVATED_BY_IMMS, StartInputReason.DEACTIVATED_BY_IMMS, - StartInputReason.SESSION_CREATED_BY_IME}) + StartInputReason.SESSION_CREATED_BY_IME, + StartInputReason.SESSION_CREATED_BY_ACCESSIBILITY, + StartInputReason.BOUND_ACCESSIBILITY_SESSION_TO_IMMS}) public @interface StartInputReason { /** * Reason is not specified. @@ -96,4 +98,14 @@ public @interface StartInputReason { * {@link com.android.internal.view.IInputSessionCallback#sessionCreated}. */ int SESSION_CREATED_BY_IME = 10; + /** + * {@link android.accessibilityservice.AccessibilityService} is responding to + * {@link com.android.internal.view.IInputSessionWithIdCallback#sessionCreated}. + */ + int SESSION_CREATED_BY_ACCESSIBILITY = 11; + /** + * {@link android.view.inputmethod.InputMethodManager} is responding to + * {@link com.android.internal.view.IInputMethodClient#onBindAccessibilityService(InputBindResult, int)}. + */ + int BOUND_ACCESSIBILITY_SESSION_TO_IMMS = 12; } diff --git a/core/java/com/android/internal/inputmethod/UnbindReason.java b/core/java/com/android/internal/inputmethod/UnbindReason.java index f0f18f11abe71..e9266251371e3 100644 --- a/core/java/com/android/internal/inputmethod/UnbindReason.java +++ b/core/java/com/android/internal/inputmethod/UnbindReason.java @@ -34,7 +34,9 @@ import java.lang.annotation.Retention; UnbindReason.DISCONNECT_IME, UnbindReason.NO_IME, UnbindReason.SWITCH_IME_FAILED, - UnbindReason.SWITCH_USER}) + UnbindReason.SWITCH_USER, + UnbindReason.ACCESSIBILITY_SERVICE_DISABLED +}) public @interface UnbindReason { /** * Reason is not specified. @@ -66,4 +68,5 @@ public @interface UnbindReason { * user's active IME. */ int SWITCH_USER = 6; + int ACCESSIBILITY_SERVICE_DISABLED = 7; } diff --git a/core/java/com/android/internal/view/IInputMethodClient.aidl b/core/java/com/android/internal/view/IInputMethodClient.aidl index e72afdd78ba43..8430c0859bc9f 100644 --- a/core/java/com/android/internal/view/IInputMethodClient.aidl +++ b/core/java/com/android/internal/view/IInputMethodClient.aidl @@ -24,7 +24,9 @@ import com.android.internal.inputmethod.InputBindResult; */ oneway interface IInputMethodClient { void onBindMethod(in InputBindResult res); + void onBindAccessibilityService(in InputBindResult res, int id); void onUnbindMethod(int sequence, int unbindReason); + void onUnbindAccessibilityService(int sequence, int id); void setActive(boolean active, boolean fullscreen, boolean reportToImeController); void scheduleStartInputIfNecessary(boolean fullscreen); void reportFullscreenMode(boolean fullscreen); diff --git a/core/java/com/android/internal/view/IInputSessionWithIdCallback.aidl b/core/java/com/android/internal/view/IInputSessionWithIdCallback.aidl new file mode 100644 index 0000000000000..8fbdefe5931e8 --- /dev/null +++ b/core/java/com/android/internal/view/IInputSessionWithIdCallback.aidl @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + package com.android.internal.view; + + import com.android.internal.view.IInputMethodSession; + +/** + * Helper interface for IInputMethod to allow the input method to notify the client when a new + * session has been created. + */ +oneway interface IInputSessionWithIdCallback { + void sessionCreated(IInputMethodSession session, int id); +} \ No newline at end of file diff --git a/core/res/res/values/attrs.xml b/core/res/res/values/attrs.xml index 8df5871577916..090b1c527d15b 100644 --- a/core/res/res/values/attrs.xml +++ b/core/res/res/values/attrs.xml @@ -3910,6 +3910,8 @@ + + diff --git a/services/accessibility/java/com/android/server/accessibility/AbstractAccessibilityServiceConnection.java b/services/accessibility/java/com/android/server/accessibility/AbstractAccessibilityServiceConnection.java index 8b62a64f57d4c..7f103144b7fb7 100644 --- a/services/accessibility/java/com/android/server/accessibility/AbstractAccessibilityServiceConnection.java +++ b/services/accessibility/java/com/android/server/accessibility/AbstractAccessibilityServiceConnection.java @@ -25,6 +25,7 @@ import static android.accessibilityservice.AccessibilityTrace.FLAGS_ACCESSIBILIT import static android.accessibilityservice.AccessibilityTrace.FLAGS_ACCESSIBILITY_SERVICE_CLIENT; import static android.accessibilityservice.AccessibilityTrace.FLAGS_ACCESSIBILITY_SERVICE_CONNECTION; import static android.accessibilityservice.AccessibilityTrace.FLAGS_WINDOW_MANAGER_INTERNAL; +import static android.os.Trace.TRACE_TAG_WINDOW_MANAGER; import static android.view.WindowManager.LayoutParams.TYPE_ACCESSIBILITY_OVERLAY; import static android.view.accessibility.AccessibilityInteractionClient.CALL_STACK; import static android.view.accessibility.AccessibilityInteractionClient.IGNORE_CALL_STACK; @@ -66,6 +67,7 @@ import android.os.RemoteCallback; import android.os.RemoteException; import android.os.ServiceManager; import android.os.SystemClock; +import android.os.Trace; import android.provider.Settings; import android.util.Slog; import android.util.SparseArray; @@ -80,15 +82,21 @@ import android.view.accessibility.AccessibilityEvent; import android.view.accessibility.AccessibilityNodeInfo; import android.view.accessibility.AccessibilityWindowInfo; import android.view.accessibility.IAccessibilityInteractionConnectionCallback; +import android.view.inputmethod.EditorInfo; +import android.view.inputmethod.InputBinding; import com.android.internal.annotations.GuardedBy; import com.android.internal.compat.IPlatformCompat; import com.android.internal.os.SomeArgs; import com.android.internal.util.DumpUtils; import com.android.internal.util.function.pooled.PooledLambda; +import com.android.internal.view.IInputContext; +import com.android.internal.view.IInputMethodSession; +import com.android.internal.view.IInputSessionWithIdCallback; import com.android.server.LocalServices; import com.android.server.accessibility.AccessibilityWindowManager.RemoteAccessibilityConnection; import com.android.server.accessibility.magnification.MagnificationProcessor; +import com.android.server.inputmethod.InputMethodManagerInternal; import com.android.server.wm.ActivityTaskManagerInternal; import com.android.server.wm.WindowManagerInternal; @@ -180,6 +188,8 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ boolean mLastAccessibilityButtonCallbackState; + boolean mRequestImeApis; + int mFetchFlags; long mNotificationTimeout; @@ -271,6 +281,9 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ void onDoubleTapAndHold(int displayId); + void requestImeLocked(AccessibilityServiceConnection connection); + + void unbindImeLocked(AccessibilityServiceConnection connection); } public AbstractAccessibilityServiceConnection(Context context, ComponentName componentName, @@ -374,6 +387,9 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ & AccessibilityServiceInfo.FLAG_REQUEST_FINGERPRINT_GESTURES) != 0; mRequestAccessibilityButton = (info.flags & AccessibilityServiceInfo.FLAG_REQUEST_ACCESSIBILITY_BUTTON) != 0; + // TODO(b/218193835): request ime when ime flag is set and clean up when ime flag is unset + mRequestImeApis = (info.flags + & AccessibilityServiceInfo.FLAG_INPUT_METHOD_EDITOR) != 0; } protected boolean supportsFlagForNotImportantViews(AccessibilityServiceInfo info) { @@ -1610,6 +1626,27 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ mInvocationHandler.notifyAccessibilityButtonAvailabilityChangedLocked(available); } + public void createImeSessionLocked() { + mInvocationHandler.createImeSessionLocked(); + } + + public void setImeSessionEnabledLocked(IInputMethodSession session, boolean enabled) { + mInvocationHandler.setImeSessionEnabledLocked(session, enabled); + } + + public void bindInputLocked(InputBinding binding) { + mInvocationHandler.bindInputLocked(binding); + } + + public void unbindInputLocked() { + mInvocationHandler.unbindInputLocked(); + } + + public void startInputLocked(IBinder startInputToken, IInputContext inputContext, + EditorInfo editorInfo, boolean restarting) { + mInvocationHandler.startInputLocked(startInputToken, inputContext, editorInfo, restarting); + } + /** * Called by the invocation handler to notify the service that the * state of magnification has changed. @@ -1732,6 +1769,84 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ } } + private void createImeSessionInternal() { + final IAccessibilityServiceClient listener = getServiceInterfaceSafely(); + if (listener != null) { + try { + if (svcClientTracingEnabled()) { + logTraceSvcClient("createImeSession", ""); + } + AccessibilityCallback callback = new AccessibilityCallback(); + listener.createImeSession(callback); + } catch (RemoteException re) { + Slog.e(LOG_TAG, + "Error requesting IME session from " + mService, re); + } + } + } + + private void setImeSessionEnabledInternal(IInputMethodSession session, boolean enabled) { + final IAccessibilityServiceClient listener = getServiceInterfaceSafely(); + if (listener != null && session != null) { + try { + if (svcClientTracingEnabled()) { + logTraceSvcClient("createImeSession", ""); + } + listener.setImeSessionEnabled(session, enabled); + } catch (RemoteException re) { + Slog.e(LOG_TAG, + "Error requesting IME session from " + mService, re); + } + } + } + + private void bindInputInternal(InputBinding binding) { + final IAccessibilityServiceClient listener = getServiceInterfaceSafely(); + if (listener != null) { + try { + if (svcClientTracingEnabled()) { + logTraceSvcClient("bindInput", binding.toString()); + } + listener.bindInput(binding); + } catch (RemoteException re) { + Slog.e(LOG_TAG, + "Error binding input to " + mService, re); + } + } + } + + private void unbindInputInternal() { + final IAccessibilityServiceClient listener = getServiceInterfaceSafely(); + if (listener != null) { + try { + if (svcClientTracingEnabled()) { + logTraceSvcClient("unbindInput", ""); + } + listener.unbindInput(); + } catch (RemoteException re) { + Slog.e(LOG_TAG, + "Error unbinding input to " + mService, re); + } + } + } + + private void startInputInternal(IBinder startInputToken, IInputContext inputContext, + EditorInfo editorInfo, boolean restarting) { + final IAccessibilityServiceClient listener = getServiceInterfaceSafely(); + if (listener != null) { + try { + if (svcClientTracingEnabled()) { + logTraceSvcClient("startInput", startInputToken + " " + + inputContext + " " + editorInfo + restarting); + } + listener.startInput(startInputToken, inputContext, editorInfo, restarting); + } catch (RemoteException re) { + Slog.e(LOG_TAG, + "Error starting input to " + mService, re); + } + } + } + protected IAccessibilityServiceClient getServiceInterfaceSafely() { synchronized (mLock) { return mServiceInterface; @@ -1925,6 +2040,11 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ private static final int MSG_ON_ACCESSIBILITY_BUTTON_CLICKED = 7; private static final int MSG_ON_ACCESSIBILITY_BUTTON_AVAILABILITY_CHANGED = 8; private static final int MSG_ON_SYSTEM_ACTIONS_CHANGED = 9; + private static final int MSG_CREATE_IME_SESSION = 10; + private static final int MSG_SET_IME_SESSION_ENABLED = 11; + private static final int MSG_BIND_INPUT = 12; + private static final int MSG_UNBIND_INPUT = 13; + private static final int MSG_START_INPUT = 14; /** List of magnification callback states, mapping from displayId -> Boolean */ @GuardedBy("mlock") @@ -1974,6 +2094,29 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ notifySystemActionsChangedInternal(); break; } + case MSG_CREATE_IME_SESSION: + createImeSessionInternal(); + break; + case MSG_SET_IME_SESSION_ENABLED: + final boolean enabled = (message.arg1 != 0); + final IInputMethodSession session = (IInputMethodSession) message.obj; + setImeSessionEnabledInternal(session, enabled); + break; + case MSG_BIND_INPUT: + final InputBinding binding = (InputBinding) message.obj; + bindInputInternal(binding); + break; + case MSG_UNBIND_INPUT: + unbindInputInternal(); + break; + case MSG_START_INPUT: + final boolean restarting = (message.arg1 != 0); + final SomeArgs args = (SomeArgs) message.obj; + final IBinder startInputToken = (IBinder) args.arg1; + final IInputContext inputContext = (IInputContext) args.arg2; + final EditorInfo editorInfo = (EditorInfo) args.arg3; + startInputInternal(startInputToken, inputContext, editorInfo, restarting); + break; default: { throw new IllegalArgumentException("Unknown message: " + type); } @@ -2036,6 +2179,37 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ (available ? 1 : 0), 0); msg.sendToTarget(); } + + public void createImeSessionLocked() { + final Message msg = obtainMessage(MSG_CREATE_IME_SESSION); + msg.sendToTarget(); + } + + public void setImeSessionEnabledLocked(IInputMethodSession session, boolean enabled) { + final Message msg = obtainMessage(MSG_SET_IME_SESSION_ENABLED, (enabled ? 1 : 0), + 0, session); + msg.sendToTarget(); + } + + public void bindInputLocked(InputBinding binding) { + final Message msg = obtainMessage(MSG_BIND_INPUT, binding); + msg.sendToTarget(); + } + + public void unbindInputLocked() { + final Message msg = obtainMessage(MSG_UNBIND_INPUT); + msg.sendToTarget(); + } + + public void startInputLocked(IBinder startInputToken, IInputContext inputContext, + EditorInfo editorInfo, boolean restarting) { + final SomeArgs args = SomeArgs.obtain(); + args.arg1 = startInputToken; + args.arg2 = inputContext; + args.arg3 = editorInfo; + final Message msg = obtainMessage(MSG_START_INPUT, restarting ? 1 : 0, 0, args); + msg.sendToTarget(); + } } public boolean isServiceHandlesDoubleTapEnabled() { @@ -2185,4 +2359,18 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ Binder.restoreCallingIdentity(identity); } } + + private static final class AccessibilityCallback extends IInputSessionWithIdCallback.Stub { + @Override + public void sessionCreated(IInputMethodSession session, int id) { + Trace.traceBegin(TRACE_TAG_WINDOW_MANAGER, "IMMS.sessionCreated"); + final long ident = Binder.clearCallingIdentity(); + try { + InputMethodManagerInternal.get().onSessionForAccessibilityCreated(id, session); + } finally { + Binder.restoreCallingIdentity(ident); + } + Trace.traceEnd(TRACE_TAG_WINDOW_MANAGER); + } + } } \ No newline at end of file diff --git a/services/accessibility/java/com/android/server/accessibility/AccessibilityManagerService.java b/services/accessibility/java/com/android/server/accessibility/AccessibilityManagerService.java index 9467fa9bf9eb0..62da981bd9a74 100644 --- a/services/accessibility/java/com/android/server/accessibility/AccessibilityManagerService.java +++ b/services/accessibility/java/com/android/server/accessibility/AccessibilityManagerService.java @@ -115,6 +115,8 @@ import android.view.accessibility.IAccessibilityInteractionConnection; import android.view.accessibility.IAccessibilityManager; import android.view.accessibility.IAccessibilityManagerClient; import android.view.accessibility.IWindowMagnificationConnection; +import android.view.inputmethod.EditorInfo; +import android.view.inputmethod.InputBinding; import com.android.internal.R; import com.android.internal.accessibility.AccessibilityShortcutController; @@ -128,12 +130,16 @@ import com.android.internal.os.BackgroundThread; import com.android.internal.util.ArrayUtils; import com.android.internal.util.DumpUtils; import com.android.internal.util.IntPair; +import com.android.internal.view.IInputContext; +import com.android.internal.view.IInputMethodSession; +import com.android.server.AccessibilityManagerInternal; import com.android.server.LocalServices; import com.android.server.SystemService; import com.android.server.accessibility.magnification.MagnificationController; import com.android.server.accessibility.magnification.MagnificationProcessor; import com.android.server.accessibility.magnification.MagnificationScaleProvider; import com.android.server.accessibility.magnification.WindowMagnificationManager; +import com.android.server.inputmethod.InputMethodManagerInternal; import com.android.server.pm.UserManagerInternal; import com.android.server.wm.ActivityTaskManagerInternal; import com.android.server.wm.WindowManagerInternal; @@ -231,7 +237,7 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub private final MainHandler mMainHandler; - // Lazily initialized - access through getSystemActionPerfomer() + // Lazily initialized - access through getSystemActionPerformer() private SystemActionPerformer mSystemActionPerformer; private InteractionBridge mInteractionBridge; @@ -273,6 +279,13 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub private Point mTempPoint = new Point(); private boolean mIsAccessibilityButtonShown; + private InputBinding mInputBinding; + IBinder mStartInputToken; + IInputContext mInputContext; + EditorInfo mEditorInfo; + boolean mRestarting; + boolean mInputSessionRequested; + private AccessibilityUserState getCurrentUserStateLocked() { return getUserStateLocked(mCurrentUserId); } @@ -298,6 +311,42 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub } } + private static final class LocalServiceImpl extends AccessibilityManagerInternal { + @NonNull + private final AccessibilityManagerService mService; + + LocalServiceImpl(@NonNull AccessibilityManagerService service) { + mService = service; + } + + @Override + public void setImeSessionEnabled(SparseArray sessions, + boolean enabled) { + mService.setImeSessionEnabled(sessions, enabled); + } + + @Override + public void unbindInput() { + mService.unbindInput(); + } + + @Override + public void bindInput(InputBinding binding) { + mService.bindInput(binding); + } + + @Override + public void createImeSession(ArraySet ignoreSet) { + mService.createImeSession(ignoreSet); + } + + @Override + public void startInput(IBinder startInputToken, IInputContext inputContext, + EditorInfo editorInfo, boolean restarting) { + mService.startInput(startInputToken, inputContext, editorInfo, restarting); + } + } + public static final class Lifecycle extends SystemService { private final AccessibilityManagerService mService; @@ -308,6 +357,8 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub @Override public void onStart() { + LocalServices.addService(AccessibilityManagerInternal.class, + new LocalServiceImpl(mService)); publishBinderService(Context.ACCESSIBILITY_SERVICE, mService); } @@ -4240,6 +4291,45 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub this, displayId)); } + @Override + public void requestImeLocked(AccessibilityServiceConnection connection) { + mMainHandler.sendMessage(obtainMessage( + AccessibilityManagerService::createSessionForConnection, this, connection)); + mMainHandler.sendMessage(obtainMessage( + AccessibilityManagerService::bindAndStartInputForConnection, this, connection)); + } + + @Override + public void unbindImeLocked(AccessibilityServiceConnection connection) { + mMainHandler.sendMessage(obtainMessage( + AccessibilityManagerService::unbindInputForConnection, this, connection)); + } + + private void createSessionForConnection(AccessibilityServiceConnection connection) { + synchronized (mLock) { + if (mInputSessionRequested) { + connection.createImeSessionLocked(); + } + } + } + + private void bindAndStartInputForConnection(AccessibilityServiceConnection connection) { + synchronized (mLock) { + if (mInputBinding != null) { + connection.bindInputLocked(mInputBinding); + connection.startInputLocked(mStartInputToken, mInputContext, mEditorInfo, + mRestarting); + } + } + } + + private void unbindInputForConnection(AccessibilityServiceConnection connection) { + InputMethodManagerInternal.get().unbindAccessibilityFromCurrentClient(connection.mId); + synchronized (mLock) { + connection.unbindInputLocked(); + } + } + private void onDoubleTapAndHoldInternal(int displayId) { synchronized (mLock) { if (mHasInputFilter && mInputFilter != null) { @@ -4268,4 +4358,100 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub public AccessibilityTraceManager getTraceManager() { return mTraceManager; } + + /** + * Bind input for accessibility services which request ime capabilities. + * + * @param binding Information given to an accessibility service about a client connecting to it. + */ + public void bindInput(InputBinding binding) { + AccessibilityUserState userState; + synchronized (mLock) { + // Keep records of these in case new Accessibility Services are enabled. + mInputBinding = binding; + userState = getCurrentUserStateLocked(); + for (int i = userState.mBoundServices.size() - 1; i >= 0; i--) { + final AccessibilityServiceConnection service = userState.mBoundServices.get(i); + if (service.requestImeApis()) { + service.bindInputLocked(binding); + } + } + } + } + + /** + * Unbind input for accessibility services which request ime capabilities. + */ + public void unbindInput() { + AccessibilityUserState userState; + // TODO(b/218182733): Resolve the Imf lock and mLock possible deadlock + synchronized (mLock) { + userState = getCurrentUserStateLocked(); + for (int i = userState.mBoundServices.size() - 1; i >= 0; i--) { + final AccessibilityServiceConnection service = userState.mBoundServices.get(i); + if (service.requestImeApis()) { + service.unbindInputLocked(); + } + } + } + } + + /** + * Start input for accessibility services which request ime capabilities. + */ + public void startInput(IBinder startInputToken, IInputContext inputContext, + EditorInfo editorInfo, boolean restarting) { + AccessibilityUserState userState; + synchronized (mLock) { + // Keep records of these in case new Accessibility Services are enabled. + mStartInputToken = startInputToken; + mInputContext = inputContext; + mEditorInfo = editorInfo; + mRestarting = restarting; + userState = getCurrentUserStateLocked(); + for (int i = userState.mBoundServices.size() - 1; i >= 0; i--) { + final AccessibilityServiceConnection service = userState.mBoundServices.get(i); + if (service.requestImeApis()) { + service.startInputLocked(startInputToken, inputContext, editorInfo, restarting); + } + } + } + } + + /** + * Request input sessions from all accessibility services which request ime capabilities and + * whose id is not in the ignoreSet + */ + public void createImeSession(ArraySet ignoreSet) { + AccessibilityUserState userState; + synchronized (mLock) { + mInputSessionRequested = true; + userState = getCurrentUserStateLocked(); + for (int i = userState.mBoundServices.size() - 1; i >= 0; i--) { + final AccessibilityServiceConnection service = userState.mBoundServices.get(i); + if ((!ignoreSet.contains(service.mId)) && service.requestImeApis()) { + service.createImeSessionLocked(); + } + } + } + } + + /** + * Enable or disable the sessions. + * + * @param sessions Sessions to enable or disable. + * @param enabled True if enable the sessions or false if disable the sessions. + */ + public void setImeSessionEnabled(SparseArray sessions, boolean enabled) { + AccessibilityUserState userState; + synchronized (mLock) { + userState = getCurrentUserStateLocked(); + for (int i = userState.mBoundServices.size() - 1; i >= 0; i--) { + final AccessibilityServiceConnection service = userState.mBoundServices.get(i); + if (sessions.contains(service.mId) && service.requestImeApis()) { + service.setImeSessionEnabledLocked(sessions.get(service.mId), enabled); + } + } + } + } } diff --git a/services/accessibility/java/com/android/server/accessibility/AccessibilityServiceConnection.java b/services/accessibility/java/com/android/server/accessibility/AccessibilityServiceConnection.java index 8f7260f0df834..06310284b56c2 100644 --- a/services/accessibility/java/com/android/server/accessibility/AccessibilityServiceConnection.java +++ b/services/accessibility/java/com/android/server/accessibility/AccessibilityServiceConnection.java @@ -127,6 +127,9 @@ class AccessibilityServiceConnection extends AbstractAccessibilityServiceConnect } public void unbindLocked() { + if (requestImeApis()) { + mSystemSupport.unbindImeLocked(this); + } mContext.unbindService(this); AccessibilityUserState userState = mUserStateWeakReference.get(); if (userState == null) return; @@ -188,6 +191,9 @@ class AccessibilityServiceConnection extends AbstractAccessibilityServiceConnect // the new configuration (for example, initializing the input filter). mMainHandler.sendMessage(obtainMessage( AccessibilityServiceConnection::initializeService, this)); + if (requestImeApis()) { + mSystemSupport.requestImeLocked(this); + } } } @@ -371,6 +377,9 @@ class AccessibilityServiceConnection extends AbstractAccessibilityServiceConnect if (!isConnectedLocked()) { return; } + if (requestImeApis()) { + mSystemSupport.unbindImeLocked(this); + } mAccessibilityServiceInfo.crashed = true; AccessibilityUserState userState = mUserStateWeakReference.get(); if (userState != null) { @@ -512,6 +521,10 @@ class AccessibilityServiceConnection extends AbstractAccessibilityServiceConnect mMainHandler.sendMessage(msg); } + public boolean requestImeApis() { + return mRequestImeApis; + } + private void notifyMotionEventInternal(MotionEvent event) { final IAccessibilityServiceClient listener = getServiceInterfaceSafely(); if (listener != null) { diff --git a/services/core/java/com/android/server/AccessibilityManagerInternal.java b/services/core/java/com/android/server/AccessibilityManagerInternal.java new file mode 100644 index 0000000000000..28f6db1c800b5 --- /dev/null +++ b/services/core/java/com/android/server/AccessibilityManagerInternal.java @@ -0,0 +1,86 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server; + +import android.annotation.NonNull; +import android.os.IBinder; +import android.util.ArraySet; +import android.util.SparseArray; +import android.view.inputmethod.EditorInfo; +import android.view.inputmethod.InputBinding; + +import com.android.internal.view.IInputContext; +import com.android.internal.view.IInputMethodSession; + +/** + * Accessibility manager local system service interface. + */ +public abstract class AccessibilityManagerInternal { + /** Enable or disable the sessions. */ + public abstract void setImeSessionEnabled(SparseArray sessions, + boolean enabled); + + /** Unbind input for all accessibility services which require ime capabilities. */ + public abstract void unbindInput(); + + /** Bind input for all accessibility services which require ime capabilities. */ + public abstract void bindInput(InputBinding binding); + + /** + * Request input session from all accessibility services which require ime capabilities and + * whose id is not in the ignoreSet. + */ + public abstract void createImeSession(ArraySet ignoreSet); + + /** Start input for all accessibility services which require ime capabilities. */ + public abstract void startInput(IBinder startInputToken, IInputContext inputContext, + EditorInfo editorInfo, boolean restarting); + + private static final AccessibilityManagerInternal NOP = new AccessibilityManagerInternal() { + @Override + public void setImeSessionEnabled(SparseArray sessions, + boolean enabled) { + } + + @Override + public void unbindInput() { + } + + @Override + public void bindInput(InputBinding binding) { + } + + @Override + public void createImeSession(ArraySet ignoreSet) { + } + + @Override + public void startInput(IBinder startInputToken, IInputContext inputContext, + EditorInfo editorInfo, boolean restarting) { + } + }; + + /** + * @return Global instance if exists. Otherwise, a fallback no-op instance. + */ + @NonNull + public static AccessibilityManagerInternal get() { + final AccessibilityManagerInternal instance = + LocalServices.getService(AccessibilityManagerInternal.class); + return instance != null ? instance : NOP; + } +} diff --git a/services/core/java/com/android/server/inputmethod/InputMethodBindingController.java b/services/core/java/com/android/server/inputmethod/InputMethodBindingController.java index b81478285a627..b2f500a59ba96 100644 --- a/services/core/java/com/android/server/inputmethod/InputMethodBindingController.java +++ b/services/core/java/com/android/server/inputmethod/InputMethodBindingController.java @@ -422,7 +422,7 @@ final class InputMethodBindingController { addFreshWindowToken(); return new InputBindResult( InputBindResult.ResultCode.SUCCESS_WAITING_IME_BINDING, - null, null, mCurId, mCurSeq, false); + null, null, null, mCurId, mCurSeq, false); } Slog.w(InputMethodManagerService.TAG, diff --git a/services/core/java/com/android/server/inputmethod/InputMethodManagerInternal.java b/services/core/java/com/android/server/inputmethod/InputMethodManagerInternal.java index 80c83e97256e7..29dcdfaa1bba4 100644 --- a/services/core/java/com/android/server/inputmethod/InputMethodManagerInternal.java +++ b/services/core/java/com/android/server/inputmethod/InputMethodManagerInternal.java @@ -25,6 +25,7 @@ import android.view.inputmethod.InputMethodInfo; import com.android.internal.inputmethod.SoftInputShowHideReason; import com.android.internal.view.IInlineSuggestionsRequestCallback; +import com.android.internal.view.IInputMethodSession; import com.android.internal.view.InlineSuggestionsRequestInfo; import com.android.server.LocalServices; @@ -148,6 +149,24 @@ public abstract class InputMethodManagerInternal { */ public abstract void updateImeWindowStatus(boolean disableImeIcon); + /** + * Callback when the IInputMethodSession from the accessibility service with the specified + * accessibilityConnectionId is created. + * + * @param accessibilityConnectionId The connection id of the accessibility service. + * @param session The session passed back from the accessibility service. + */ + public abstract void onSessionForAccessibilityCreated(int accessibilityConnectionId, + IInputMethodSession session); + + /** + * Unbind the accessibility service with the specified accessibilityConnectionId from current + * client. + * + * @param accessibilityConnectionId The connection id of the accessibility service. + */ + public abstract void unbindAccessibilityFromCurrentClient(int accessibilityConnectionId); + /** * Fake implementation of {@link InputMethodManagerInternal}. All the methods do nothing. */ @@ -211,6 +230,15 @@ public abstract class InputMethodManagerInternal { @Override public void updateImeWindowStatus(boolean disableImeIcon) { } + + @Override + public void onSessionForAccessibilityCreated(int accessibilityConnectionId, + IInputMethodSession session) { + } + + @Override + public void unbindAccessibilityFromCurrentClient(int accessibilityConnectionId) { + } }; /** diff --git a/services/core/java/com/android/server/inputmethod/InputMethodManagerService.java b/services/core/java/com/android/server/inputmethod/InputMethodManagerService.java index ba15a047af4ab..7068ed13376f1 100644 --- a/services/core/java/com/android/server/inputmethod/InputMethodManagerService.java +++ b/services/core/java/com/android/server/inputmethod/InputMethodManagerService.java @@ -94,6 +94,7 @@ import android.media.AudioManagerInternal; import android.net.Uri; import android.os.Binder; import android.os.Bundle; +import android.os.DeadObjectException; import android.os.Debug; import android.os.Handler; import android.os.IBinder; @@ -120,6 +121,7 @@ import android.util.Pair; import android.util.PrintWriterPrinter; import android.util.Printer; import android.util.Slog; +import android.util.SparseArray; import android.util.proto.ProtoOutputStream; import android.view.IWindowManager; import android.view.InputChannel; @@ -171,6 +173,7 @@ import com.android.internal.view.IInputMethodManager; import com.android.internal.view.IInputMethodSession; import com.android.internal.view.IInputSessionCallback; import com.android.internal.view.InlineSuggestionsRequestInfo; +import com.android.server.AccessibilityManagerInternal; import com.android.server.EventLogTags; import com.android.server.LocalServices; import com.android.server.ServiceThread; @@ -228,7 +231,9 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub private static final int MSG_START_HANDWRITING = 1100; private static final int MSG_UNBIND_CLIENT = 3000; + private static final int MSG_UNBIND_ACCESSIBILITY_SERVICE = 3001; private static final int MSG_BIND_CLIENT = 3010; + private static final int MSG_BIND_ACCESSIBILITY_SERVICE = 3011; private static final int MSG_SET_ACTIVE = 3020; private static final int MSG_SET_INTERACTIVE = 3030; private static final int MSG_REPORT_FULLSCREEN_MODE = 3045; @@ -351,6 +356,33 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub } } + /** + * Record session state for an accessibility service. + */ + private static class AccessibilitySessionState { + final ClientState mClient; + // Id of the accessibility service. + final int mId; + + public IInputMethodSession mSession; + + @Override + public String toString() { + return "AccessibilitySessionState{uid " + mClient.uid + " pid " + mClient.pid + + " id " + Integer.toHexString(mId) + + " session " + Integer.toHexString( + System.identityHashCode(mSession)) + + "}"; + } + + AccessibilitySessionState(ClientState client, int id, + IInputMethodSession session) { + mClient = client; + mId = id; + mSession = session; + } + } + private static final class ClientDeathRecipient implements IBinder.DeathRecipient { private final InputMethodManagerService mImms; private final IInputMethodClient mClient; @@ -376,7 +408,9 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub final ClientDeathRecipient clientDeathRecipient; boolean sessionRequested; + boolean mSessionRequestedForAccessibility; SessionState curSession; + SparseArray mAccessibilitySessions = new SparseArray<>(); @Override public String toString() { @@ -637,10 +671,16 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub boolean mBoundToMethod; /** + * Have we called bindInput() for accessibility services? + */ + boolean mBoundToAccessibility; + + /** * Currently enabled session. */ @GuardedBy("ImfLock.class") SessionState mEnabledSession; + SparseArray mEnabledAccessibilitySessions = new SparseArray<>(); /** * True if the device is currently interactive with user. The value is true initially. @@ -2188,6 +2228,7 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub if (cs != null) { client.asBinder().unlinkToDeath(cs.clientDeathRecipient, 0); clearClientSessionLocked(cs); + clearClientSessionForAccessibilityLocked(cs); if (mCurClient == cs) { hideCurrentInputLocked( mCurFocusedWindow, 0, null, SoftInputShowHideReason.HIDE_REMOVE_CLIENT); @@ -2195,9 +2236,13 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub mBoundToMethod = false; IInputMethodInvoker curMethod = getCurMethodLocked(); if (curMethod != null) { + // When we unbind input, we are unbinding the client, so we always + // unbind ime and a11y together. curMethod.unbindInput(); + AccessibilityManagerInternal.get().unbindInput(); } } + mBoundToAccessibility = false; mCurClient = null; } if (mCurFocusedWindowClient == cs) { @@ -2215,6 +2260,24 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub return mHandler.obtainMessage(what, 0, 0, args); } + @NonNull + private Message obtainMessageOOO(int what, Object arg1, Object arg2, Object arg3) { + SomeArgs args = SomeArgs.obtain(); + args.arg1 = arg1; + args.arg2 = arg2; + args.arg3 = arg3; + return mHandler.obtainMessage(what, 0, 0, args); + } + + @NonNull + private Message obtainMessageIIOO(int what, int arg1, int arg2, + Object arg3, Object arg4) { + SomeArgs args = SomeArgs.obtain(); + args.arg1 = arg3; + args.arg2 = arg4; + return mHandler.obtainMessage(what, arg1, arg2, args); + } + @NonNull private Message obtainMessageIIIO(int what, int argi1, int argi2, int argi3, Object arg1) { final SomeArgs args = SomeArgs.obtain(); @@ -2252,13 +2315,18 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub curMethod.unbindInput(); } } + mBoundToAccessibility = false; + // Since we set active false to current client and set mCurClient to null, let's unbind + // all accessibility too. That means, when input method get disconnected (including + // switching ime), we also unbind accessibility scheduleSetActiveToClient(mCurClient, false /* active */, false /* fullscreen */, false /* reportToImeController */); executeOrSendMessage(mCurClient.client, mHandler.obtainMessage( MSG_UNBIND_CLIENT, getSequenceNumberLocked(), unbindClientReason, mCurClient.client)); mCurClient.sessionRequested = false; + mCurClient.mSessionRequestedForAccessibility = false; mCurClient = null; mMenuController.hideInputMethodMenuLocked(); @@ -2340,11 +2408,63 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub final InputMethodInfo curInputMethodInfo = mMethodMap.get(curId); final boolean suppressesSpellChecker = curInputMethodInfo != null && curInputMethodInfo.suppressesSpellChecker(); + final SparseArray accessibilityInputMethodSessions = + createAccessibilityInputMethodSessions(mCurClient.mAccessibilitySessions); return new InputBindResult(InputBindResult.ResultCode.SUCCESS_WITH_IME_SESSION, - session.session, (session.channel != null ? session.channel.dup() : null), + session.session, accessibilityInputMethodSessions, + (session.channel != null ? session.channel.dup() : null), curId, getSequenceNumberLocked(), suppressesSpellChecker); } + @GuardedBy("ImfLock.class") + @Nullable + InputBindResult attachNewAccessibilityLocked(@StartInputReason int startInputReason, + boolean initial, int id) { + if (!mBoundToAccessibility) { + AccessibilityManagerInternal.get().bindInput(mCurClient.binding); + mBoundToAccessibility = true; + } + + // TODO(b/187453053): grantImplicitAccess to accessibility services access? if so, need to + // record accessibility services uid. + + final AccessibilitySessionState accessibilitySession = + mCurClient.mAccessibilitySessions.get(id); + // We don't start input when session for a11y is created. We start input when + // input method start input, a11y manager service is always on. + if (startInputReason != StartInputReason.SESSION_CREATED_BY_ACCESSIBILITY) { + final Binder startInputToken = new Binder(); + setEnabledSessionForAccessibilityLocked(mCurClient.mAccessibilitySessions); + AccessibilityManagerInternal.get().startInput(startInputToken, mCurInputContext, + mCurAttribute, !initial /* restarting */); + } + + if (accessibilitySession != null) { + final SessionState session = mCurClient.curSession; + IInputMethodSession imeSession = session == null ? null : session.session; + final SparseArray accessibilityInputMethodSessions = + createAccessibilityInputMethodSessions(mCurClient.mAccessibilitySessions); + return new InputBindResult( + InputBindResult.ResultCode.SUCCESS_WITH_ACCESSIBILITY_SESSION, + imeSession, accessibilityInputMethodSessions, null, + getCurIdLocked(), getSequenceNumberLocked(), false); + } + return null; + } + + private SparseArray createAccessibilityInputMethodSessions( + SparseArray accessibilitySessions) { + final SparseArray accessibilityInputMethodSessions = + new SparseArray<>(); + if (accessibilitySessions != null) { + for (int i = 0; i < accessibilitySessions.size(); i++) { + accessibilityInputMethodSessions.append(accessibilitySessions.keyAt(i), + accessibilitySessions.valueAt(i).mSession); + } + } + return accessibilityInputMethodSessions; + } + /** * Called by {@link #startInputOrWindowGainedFocusInternalLocked} to bind/unbind/attach the * selected InputMethod to the given focused IME client. @@ -2370,7 +2490,7 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub // party code. return new InputBindResult( InputBindResult.ResultCode.ERROR_SYSTEM_NOT_READY, - null, null, selectedMethodId, getSequenceNumberLocked(), false); + null, null, null, selectedMethodId, getSequenceNumberLocked(), false); } if (!InputMethodUtils.checkIfPackageBelongsToUid(mAppOpsManager, cs.uid, @@ -2422,6 +2542,17 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub if (cs.curSession != null) { // Fast case: if we are already connected to the input method, // then just return it. + // This doesn't mean a11y sessions are there. When a11y service is + // enabled while this client is switched out, this client doesn't have the session. + // A11yManagerService will only request missing sessions (will not request existing + // sessions again). Note when an a11y service is disabled, it will clear its + // session from all clients, so we don't need to worry about disabled a11y services. + cs.mSessionRequestedForAccessibility = false; + requestClientSessionForAccessibilityLocked(cs); + // we can always attach to accessibility because AccessibilityManagerService is + // always on. + attachNewAccessibilityLocked(startInputReason, + (startInputFlags & StartInputFlags.INITIAL_CONNECTION) != 0, -1); return attachNewInputLocked(startInputReason, (startInputFlags & StartInputFlags.INITIAL_CONNECTION) != 0); } @@ -2464,9 +2595,10 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub // Return to client, and we will get back with it when // we have had a session made for it. requestClientSessionLocked(cs); + requestClientSessionForAccessibilityLocked(cs); return new InputBindResult( InputBindResult.ResultCode.SUCCESS_WAITING_IME_SESSION, - null, null, getCurIdLocked(), getSequenceNumberLocked(), false); + null, null, null, getCurIdLocked(), getSequenceNumberLocked(), false); } else { long bindingDuration = SystemClock.uptimeMillis() - getLastBindTimeLocked(); if (bindingDuration < TIME_TO_RECONNECT) { @@ -2479,7 +2611,7 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub // to see if we can get back in touch with the service. return new InputBindResult( InputBindResult.ResultCode.SUCCESS_WAITING_IME_BINDING, - null, null, getCurIdLocked(), getSequenceNumberLocked(), false); + null, null, null, getCurIdLocked(), getSequenceNumberLocked(), false); } else { EventLog.writeEvent(EventLogTags.IMF_FORCE_RECONNECT_IME, getSelectedMethodIdLocked(), bindingDuration, 0); @@ -2565,6 +2697,8 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub method, session, channel); InputBindResult res = attachNewInputLocked( StartInputReason.SESSION_CREATED_BY_IME, true); + attachNewAccessibilityLocked(StartInputReason.SESSION_CREATED_BY_IME, + true, -1); if (res.method != null) { executeOrSendMessage(mCurClient.client, obtainMessageOO( MSG_BIND_CLIENT, mCurClient.client, res)); @@ -2602,7 +2736,9 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub void reRequestCurrentClientSessionLocked() { if (mCurClient != null) { clearClientSessionLocked(mCurClient); + clearClientSessionForAccessibilityLocked(mCurClient); requestClientSessionLocked(mCurClient); + requestClientSessionForAccessibilityLocked(mCurClient); } } @@ -2645,6 +2781,19 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub } } + @GuardedBy("ImfLock.class") + void requestClientSessionForAccessibilityLocked(ClientState cs) { + if (!cs.mSessionRequestedForAccessibility) { + if (DEBUG) Slog.v(TAG, "Creating new accessibility sessions for client " + cs); + cs.mSessionRequestedForAccessibility = true; + ArraySet ignoreSet = new ArraySet<>(); + for (int i = 0; i < cs.mAccessibilitySessions.size(); i++) { + ignoreSet.add(cs.mAccessibilitySessions.keyAt(i)); + } + AccessibilityManagerInternal.get().createImeSession(ignoreSet); + } + } + @GuardedBy("ImfLock.class") void clearClientSessionLocked(ClientState cs) { finishSessionLocked(cs.curSession); @@ -2652,6 +2801,24 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub cs.sessionRequested = false; } + @GuardedBy("ImfLock.class") + void clearClientSessionForAccessibilityLocked(ClientState cs) { + for (int i = 0; i < cs.mAccessibilitySessions.size(); i++) { + finishSessionForAccessibilityLocked(cs.mAccessibilitySessions.valueAt(i)); + } + cs.mAccessibilitySessions.clear(); + cs.mSessionRequestedForAccessibility = false; + } + + @GuardedBy("ImfLock.class") + void clearClientSessionForAccessibilityLocked(ClientState cs, int id) { + AccessibilitySessionState session = cs.mAccessibilitySessions.get(id); + if (session != null) { + finishSessionForAccessibilityLocked(session); + cs.mAccessibilitySessions.remove(id); + } + } + @GuardedBy("ImfLock.class") private void finishSessionLocked(SessionState sessionState) { if (sessionState != null) { @@ -2671,16 +2838,35 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub } } + @GuardedBy("ImfLock.class") + private void finishSessionForAccessibilityLocked(AccessibilitySessionState sessionState) { + if (sessionState != null) { + if (sessionState.mSession != null) { + try { + sessionState.mSession.finishSession(); + } catch (RemoteException e) { + Slog.w(TAG, "Session failed to close due to remote exception", e); + } + sessionState.mSession = null; + } + } + } + @GuardedBy("ImfLock.class") void clearClientSessionsLocked() { if (getCurMethodLocked() != null) { final int numClients = mClients.size(); for (int i = 0; i < numClients; ++i) { clearClientSessionLocked(mClients.valueAt(i)); + clearClientSessionForAccessibilityLocked(mClients.valueAt(i)); } finishSessionLocked(mEnabledSession); + for (int i = 0; i < mEnabledAccessibilitySessions.size(); i++) { + finishSessionForAccessibilityLocked(mEnabledAccessibilitySessions.valueAt(i)); + } mEnabledSession = null; + mEnabledAccessibilitySessions.clear(); scheduleNotifyImeUidToAudioService(Process.INVALID_UID); } hideStatusBarIconLocked(); @@ -3455,7 +3641,7 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub } return new InputBindResult( InputBindResult.ResultCode.SUCCESS_REPORT_WINDOW_FOCUS_ONLY, - null, null, null, -1, false); + null, null, null, null, -1, false); } mCurFocusedWindow = windowToken; @@ -4250,6 +4436,41 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub } } + @GuardedBy("ImfLock.class") + void setEnabledSessionForAccessibilityLocked( + SparseArray accessibilitySessions) { + // mEnabledAccessibilitySessions could the same object as accessibilitySessions. + SparseArray disabledSessions = new SparseArray<>(); + for (int i = 0; i < mEnabledAccessibilitySessions.size(); i++) { + if (!accessibilitySessions.contains(mEnabledAccessibilitySessions.keyAt(i))) { + AccessibilitySessionState sessionState = mEnabledAccessibilitySessions.valueAt(i); + if (sessionState != null) { + disabledSessions.append(mEnabledAccessibilitySessions.keyAt(i), + sessionState.mSession); + } + } + } + if (disabledSessions.size() > 0) { + AccessibilityManagerInternal.get().setImeSessionEnabled(disabledSessions, + false); + } + SparseArray enabledSessions = new SparseArray<>(); + for (int i = 0; i < accessibilitySessions.size(); i++) { + if (!mEnabledAccessibilitySessions.contains(accessibilitySessions.keyAt(i))) { + AccessibilitySessionState sessionState = accessibilitySessions.valueAt(i); + if (sessionState != null) { + enabledSessions.append(accessibilitySessions.keyAt(i), sessionState.mSession); + } + } + } + if (enabledSessions.size() > 0) { + AccessibilityManagerInternal.get().setImeSessionEnabled(enabledSessions, + true); + } + mEnabledAccessibilitySessions = accessibilitySessions; + } + + @SuppressWarnings("unchecked") @UiThread @Override public boolean handleMessage(Message msg) { @@ -4319,13 +4540,34 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub // --------------------------------------------------------- - case MSG_UNBIND_CLIENT: + case MSG_UNBIND_CLIENT: { try { - ((IInputMethodClient)msg.obj).onUnbindMethod(msg.arg1, msg.arg2); + // This unbinds all accessibility services too. + ((IInputMethodClient) msg.obj).onUnbindMethod(msg.arg1, msg.arg2); } catch (RemoteException e) { // There is nothing interesting about the last client dying. + if (!(e instanceof DeadObjectException)) { + Slog.w(TAG, "RemoteException when unbinding input method service or" + + "accessibility services"); + } } return true; + } + case MSG_UNBIND_ACCESSIBILITY_SERVICE: { + args = (SomeArgs) msg.obj; + IInputMethodClient client = (IInputMethodClient) args.arg1; + int id = (int) args.arg2; + try { + client.onUnbindAccessibilityService(msg.arg1, id); + } catch (RemoteException e) { + // There is nothing interesting about the last client dying. + if (!(e instanceof DeadObjectException)) { + Slog.w(TAG, "RemoteException when unbinding accessibility services"); + } + } + args.recycle(); + return true; + } case MSG_BIND_CLIENT: { args = (SomeArgs)msg.obj; IInputMethodClient client = (IInputMethodClient)args.arg1; @@ -4344,6 +4586,25 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub args.recycle(); return true; } + case MSG_BIND_ACCESSIBILITY_SERVICE: { + args = (SomeArgs) msg.obj; + IInputMethodClient client = (IInputMethodClient) args.arg1; + InputBindResult res = (InputBindResult) args.arg2; + int id = (int) args.arg3; + try { + client.onBindAccessibilityService(res, id); + } catch (RemoteException e) { + Slog.w(TAG, "Client died receiving input method " + args.arg2); + } finally { + // Dispose the channel if the accessibility service is not local to this process + // because the remote proxy will get its own copy when unparceled. + if (res.channel != null && Binder.isProxy(client)) { + res.channel.dispose(); + } + } + args.recycle(); + return true; + } case MSG_SET_ACTIVE: { args = (SomeArgs) msg.obj; final ClientState clientState = (ClientState) args.arg1; @@ -5025,6 +5286,59 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub mHandler.obtainMessage(MSG_UPDATE_IME_WINDOW_STATUS, disableImeIcon ? 1 : 0, 0) .sendToTarget(); } + + @Override + public void onSessionForAccessibilityCreated(int accessibilityConnectionId, + IInputMethodSession session) { + synchronized (ImfLock.class) { + if (mCurClient != null) { + clearClientSessionForAccessibilityLocked(mCurClient, accessibilityConnectionId); + mCurClient.mAccessibilitySessions.put(accessibilityConnectionId, + new AccessibilitySessionState(mCurClient, accessibilityConnectionId, + session)); + InputBindResult res = attachNewAccessibilityLocked( + StartInputReason.SESSION_CREATED_BY_ACCESSIBILITY, true, + accessibilityConnectionId); + executeOrSendMessage(mCurClient.client, obtainMessageOOO( + MSG_BIND_ACCESSIBILITY_SERVICE, mCurClient.client, res, + accessibilityConnectionId)); + } + } + } + + @Override + public void unbindAccessibilityFromCurrentClient(int accessibilityConnectionId) { + synchronized (ImfLock.class) { + if (mCurClient != null) { + if (DEBUG) { + Slog.v(TAG, "unbindAccessibilityFromCurrentClientLocked: client=" + + mCurClient.client.asBinder()); + } + // A11yManagerService unbinds the disabled accessibility service. We don't need + // to do it here. + @UnbindReason int unbindClientReason = + UnbindReason.ACCESSIBILITY_SERVICE_DISABLED; + executeOrSendMessage(mCurClient.client, obtainMessageIIOO( + MSG_UNBIND_ACCESSIBILITY_SERVICE, getSequenceNumberLocked(), + unbindClientReason, mCurClient.client, accessibilityConnectionId)); + } + // We only have sessions when we bound to an input method. Remove this session + // from all clients. + if (getCurMethodLocked() != null) { + final int numClients = mClients.size(); + for (int i = 0; i < numClients; ++i) { + clearClientSessionForAccessibilityLocked(mClients.valueAt(i), + accessibilityConnectionId); + } + AccessibilitySessionState session = mEnabledAccessibilitySessions.get( + accessibilityConnectionId); + if (session != null) { + finishSessionForAccessibilityLocked(session); + mEnabledAccessibilitySessions.remove(accessibilityConnectionId); + } + } + } + } } @BinderThread @@ -5184,6 +5498,8 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub p.println(" client=" + ci.client); p.println(" inputContext=" + ci.inputContext); p.println(" sessionRequested=" + ci.sessionRequested); + p.println(" sessionRequestedForAccessibility=" + + ci.mSessionRequestedForAccessibility); p.println(" curSession=" + ci.curSession); } p.println(" mCurMethodId=" + getSelectedMethodIdLocked());