From fb17e5ae7a9e1a095d114d8dde76f14578b6c233 Mon Sep 17 00:00:00 2001 From: yingleiw Date: Mon, 13 Dec 2021 14:00:06 -0800 Subject: [PATCH 1/3] Allow a11y services to enter text via the path IMEs use InputMethodService is the primary and a11y is the secondary. InputMethodService is not affected by a11y status. When the session from input method is established, app can start input (pass input context to input method). When an a11y session comes back, it will be passed to the app. When InputMethodManagerService binds to/start input with InputMethodService, it does the same to a11y services which requested IME functionalities. It is possible that input method can edit text before ally sessios are established. So the EditorInfo passed to a11y could be stale. So when an a11y session is passed to client, client will send a notification (input method doesn't have this extra notification) for the current selection. I think since the time for a11y session establish shouldn't be long, and we get the current state later, it should be fine for a11y services. When input method is disconnected from app (client) (even for input method switching), we cleared a11y and sessions too. When input method request sessions, we must rerequest sessions for a11y. This is mainly because when we unbindCurrentClientLocked(SWITCH_IME), we set active to false for the current client. Suppose we don't want to change the current structure of input method, an inactive client probably should clear accessibility sessions too. When we switch to a client which already has a session with input method, there might be some a11y sessions with this client, and some a11y services might be disabled or enabled while the client was switched out. We pass unchanged a11y sessions to client, and request sessions for newly enabled a11y services. When an a11y service is disabled, it removes its session from all clients in InputMethodManagerService. Test: type word through modified "switchToInputMethod". Tested session notification through logs. Tested client switching, input method switching, a11y service enabled/disable, multiple a11y services, a11y service enabled before device reboot. Also tested work profile. Bug: 187453053 Change-Id: Ia651a811093a939d00c081be1961e24ed3ad0356 --- core/api/current.txt | 24 + .../AccessibilityService.java | 229 +++++++ .../IAccessibilityServiceClient.aidl | 16 + .../accessibilityservice/InputMethod.java | 637 ++++++++++++++++++ core/java/android/app/UiAutomation.java | 28 + .../IInputMethodSessionWrapper.java | 4 +- .../InputMethodServiceInternal.java | 11 +- .../RemoteInputConnection.java | 9 +- .../view/inputmethod/InputMethodManager.java | 134 +++- .../internal/inputmethod/InputBindResult.java | 4 +- .../inputmethod/InputMethodDebug.java | 4 + .../inputmethod/StartInputReason.java | 14 +- .../internal/inputmethod/UnbindReason.java | 5 +- .../internal/view/IInputMethodClient.aidl | 2 + .../view/IInputSessionWithIdCallback.aidl | 27 + ...bstractAccessibilityServiceConnection.java | 183 +++++ .../AccessibilityManagerService.java | 195 +++++- .../AccessibilityServiceConnection.java | 5 + .../server/AccessibilityManagerInternal.java | 82 +++ .../InputMethodManagerInternal.java | 28 + .../InputMethodManagerService.java | 283 +++++++- 21 files changed, 1909 insertions(+), 15 deletions(-) create mode 100644 core/java/android/accessibilityservice/InputMethod.java create mode 100644 core/java/com/android/internal/view/IInputSessionWithIdCallback.aidl create mode 100644 services/core/java/com/android/server/AccessibilityManagerInternal.java diff --git a/core/api/current.txt b/core/api/current.txt index fd53d880bccdd..b30ca65aae1e8 100644 --- a/core/api/current.txt +++ b/core/api/current.txt @@ -3059,6 +3059,7 @@ package android.accessibilityservice { method @NonNull public final android.accessibilityservice.AccessibilityButtonController getAccessibilityButtonController(); method @NonNull public final android.accessibilityservice.AccessibilityButtonController getAccessibilityButtonController(int); method @NonNull @RequiresPermission(android.Manifest.permission.USE_FINGERPRINT) public final android.accessibilityservice.FingerprintGestureController getFingerprintGestureController(); + method @Nullable public final android.accessibilityservice.InputMethod getInputMethod(); method @NonNull public final android.accessibilityservice.AccessibilityService.MagnificationController getMagnificationController(); method public android.view.accessibility.AccessibilityNodeInfo getRootInActiveWindow(); method public final android.accessibilityservice.AccessibilityServiceInfo getServiceInfo(); @@ -3071,6 +3072,7 @@ package android.accessibilityservice { method public boolean isNodeInCache(@NonNull android.view.accessibility.AccessibilityNodeInfo); method public abstract void onAccessibilityEvent(android.view.accessibility.AccessibilityEvent); method public final android.os.IBinder onBind(android.content.Intent); + method @NonNull public android.accessibilityservice.InputMethod onCreateInputMethod(); method @Deprecated protected boolean onGesture(int); method public boolean onGesture(@NonNull android.accessibilityservice.AccessibilityGestureEvent); method public abstract void onInterrupt(); @@ -3317,6 +3319,28 @@ package android.accessibilityservice { method public boolean willContinue(); } + public class InputMethod { + ctor protected InputMethod(@NonNull android.accessibilityservice.AccessibilityService); + method @Nullable public final android.accessibilityservice.InputMethod.AccessibilityInputConnection getCurrentInputConnection(); + method @Nullable public final android.view.inputmethod.EditorInfo getCurrentInputEditorInfo(); + method public final boolean getCurrentInputStarted(); + method public void onFinishInput(); + method public void onStartInput(@NonNull android.view.inputmethod.EditorInfo, boolean); + method public void onUpdateSelection(int, int, int, int, int, int); + } + + public final class InputMethod.AccessibilityInputConnection { + method public void clearMetaKeyStates(int); + method public void commitText(@NonNull CharSequence, int, @Nullable android.view.inputmethod.TextAttribute); + method public void deleteSurroundingText(int, int); + method public int getCursorCapsMode(int); + method @Nullable public android.view.inputmethod.SurroundingText getSurroundingText(@IntRange(from=0) int, @IntRange(from=0) int, int); + method public void performContextMenuAction(int); + method public void performEditorAction(int); + method public void sendKeyEvent(@NonNull android.view.KeyEvent); + method public void setSelection(int, int); + } + public final class MagnificationConfig implements android.os.Parcelable { method public int describeContents(); method public float getCenterX(); diff --git a/core/java/android/accessibilityservice/AccessibilityService.java b/core/java/android/accessibilityservice/AccessibilityService.java index 50473f1d8c845..621d9282784d0 100644 --- a/core/java/android/accessibilityservice/AccessibilityService.java +++ b/core/java/android/accessibilityservice/AccessibilityService.java @@ -40,6 +40,8 @@ import android.graphics.ParcelableColorSpace; import android.graphics.Region; import android.hardware.HardwareBuffer; import android.hardware.display.DisplayManager; +import android.inputmethodservice.IInputMethodSessionWrapper; +import android.inputmethodservice.RemoteInputConnection; import android.os.Build; import android.os.Bundle; import android.os.Handler; @@ -65,14 +67,23 @@ import android.view.accessibility.AccessibilityInteractionClient; import android.view.accessibility.AccessibilityNodeInfo; import android.view.accessibility.AccessibilityNodeInfo.AccessibilityAction; import android.view.accessibility.AccessibilityWindowInfo; +import android.view.inputmethod.EditorInfo; +import android.view.inputmethod.InputBinding; +import android.view.inputmethod.InputConnection; +import android.view.inputmethod.InputMethodSession; +import com.android.internal.inputmethod.CancellationGroup; import com.android.internal.os.HandlerCaller; import com.android.internal.os.SomeArgs; import com.android.internal.util.Preconditions; import com.android.internal.util.function.pooled.PooledLambda; +import com.android.internal.view.IInputContext; +import com.android.internal.view.IInputMethodSession; +import com.android.internal.view.IInputSessionWithIdCallback; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; +import java.lang.ref.WeakReference; import java.util.Collections; import java.util.List; import java.util.concurrent.Executor; @@ -627,6 +638,21 @@ public abstract class AccessibilityService extends Service { void onAccessibilityButtonAvailabilityChanged(boolean available); /** This is called when the system action list is changed. */ void onSystemActionsChanged(); + /** This is called when an app requests ime sessions or when the service is enabled. */ + void createImeSession(IInputSessionWithIdCallback callback); + /** + * This is called when InputMethodManagerService requests to set the session enabled or + * disabled + */ + void setImeSessionEnabled(InputMethodSession session, boolean enabled); + /** This is called when an app binds input or when the service is enabled. */ + void bindInput(InputBinding binding); + /** This is called when an app unbinds input or when the service is disabled. */ + void unbindInput(); + /** This is called when an app starts input or when the service is enabled. */ + void startInput(@Nullable InputConnection inputConnection, + @NonNull EditorInfo editorInfo, boolean restarting, + @NonNull IBinder startInputToken); } /** @@ -763,6 +789,8 @@ public abstract class AccessibilityService extends Service { new SparseArray<>(0); private SoftKeyboardController mSoftKeyboardController; + private InputMethod mInputMethod; + private boolean mInputMethodInitialized = false; private final SparseArray mAccessibilityButtonControllers = new SparseArray<>(0); @@ -797,6 +825,11 @@ public abstract class AccessibilityService extends Service { for (int i = 0; i < mMagnificationControllers.size(); i++) { mMagnificationControllers.valueAt(i).onServiceConnectedLocked(); } + // TODO(b/187453053): If service requested ime capabilities + if (!mInputMethodInitialized) { + mInputMethod = onCreateInputMethod(); + mInputMethodInitialized = true; + } } if (mSoftKeyboardController != null) { mSoftKeyboardController.onServiceConnected(); @@ -1849,6 +1882,32 @@ public abstract class AccessibilityService extends Service { } } + /** + * The default implementation returns our default {@link InputMethod}. Subclasses can override + * it to provide their own customized version. Accessibility services need to set the + * {@link AccessibilityServiceInfo#FLAG_REQUEST_IME_APIS} flag to use input method APIs. + * + * @return the InputMethod. + */ + @NonNull + public InputMethod onCreateInputMethod() { + return new InputMethod(this); + } + + /** + * Returns the InputMethod instance after the system calls {@link #onCreateInputMethod()}, + * which may be used to input text or get editable text selection change notifications. It will + * return null if the accessibility service doesn't set the + * {@link AccessibilityServiceInfo#FLAG_REQUEST_IME_APIS} flag or the system doesn't call + * {@link #onCreateInputMethod()}. + * + * @return the InputMethod instance + */ + @Nullable + public final InputMethod getInputMethod() { + return mInputMethod; + } + private void onSoftKeyboardShowModeChanged(int showMode) { if (mSoftKeyboardController != null) { mSoftKeyboardController.dispatchSoftKeyboardShowModeChanged(showMode); @@ -2657,6 +2716,47 @@ public abstract class AccessibilityService extends Service { public void onSystemActionsChanged() { AccessibilityService.this.onSystemActionsChanged(); } + + @Override + public void createImeSession(IInputSessionWithIdCallback callback) { + if (mInputMethod != null) { + mInputMethod.createImeSession(callback); + } + } + + @Override + public void setImeSessionEnabled(InputMethodSession session, boolean enabled) { + if (mInputMethod != null) { + mInputMethod.setImeSessionEnabled(session, enabled); + } + } + + @Override + public void bindInput(InputBinding binding) { + if (mInputMethod != null) { + mInputMethod.bindInput(binding); + } + } + + @Override + public void unbindInput() { + if (mInputMethod != null) { + mInputMethod.unbindInput(); + } + } + + @Override + public void startInput(@Nullable InputConnection inputConnection, + @NonNull EditorInfo editorInfo, boolean restarting, + @NonNull IBinder startInputToken) { + if (mInputMethod != null) { + if (restarting) { + mInputMethod.restartInput(inputConnection, editorInfo); + } else { + mInputMethod.startInput(inputConnection, editorInfo); + } + } + } }); } @@ -2682,6 +2782,11 @@ public abstract class AccessibilityService extends Service { private static final int DO_ACCESSIBILITY_BUTTON_CLICKED = 12; private static final int DO_ACCESSIBILITY_BUTTON_AVAILABILITY_CHANGED = 13; private static final int DO_ON_SYSTEM_ACTIONS_CHANGED = 14; + private static final int DO_CREATE_IME_SESSION = 15; + private static final int DO_SET_IME_SESSION_ENABLED = 16; + private static final int DO_BIND_INPUT = 17; + private static final int DO_UNBIND_INPUT = 18; + private static final int DO_START_INPUT = 19; private final HandlerCaller mCaller; @@ -2690,6 +2795,22 @@ public abstract class AccessibilityService extends Service { private int mConnectionId = AccessibilityInteractionClient.NO_ID; + /** + * This is not {@null} only between {@link #bindInput(InputBinding)} and + * {@link #unbindInput()} so that {@link RemoteInputConnection} can query if + * {@link #unbindInput()} has already been called or not, mainly to avoid unnecessary + * blocking operations. + * + *

This field must be set and cleared only from the binder thread(s), where the system + * guarantees that {@link #bindInput(InputBinding)}, + * {@link #startInput(IBinder, IInputContext, EditorInfo, boolean)}, and + * {@link #unbindInput()} are called with the same order as the original calls + * in {@link com.android.server.inputmethod.InputMethodManagerService}. + * See {@link IBinder#FLAG_ONEWAY} for detailed semantics.

+ */ + @Nullable + CancellationGroup mCancellationGroup = null; + public IAccessibilityServiceClientWrapper(Context context, Looper looper, Callbacks callback) { mCallback = callback; @@ -2783,6 +2904,70 @@ public abstract class AccessibilityService extends Service { mCaller.sendMessage(mCaller.obtainMessage(DO_ON_SYSTEM_ACTIONS_CHANGED)); } + /** This is called when an app requests ime sessions or when the service is enabled. */ + public void createImeSession(IInputSessionWithIdCallback callback) { + final Message message = mCaller.obtainMessageO(DO_CREATE_IME_SESSION, callback); + mCaller.sendMessage(message); + } + + /** + * This is called when InputMethodManagerService requests to set the session enabled or + * disabled + */ + public void setImeSessionEnabled(IInputMethodSession session, boolean enabled) { + try { + InputMethodSession ls = ((IInputMethodSessionWrapper) + session).getInternalInputMethodSession(); + if (ls == null) { + Log.w(LOG_TAG, "Session is already finished: " + session); + return; + } + mCaller.sendMessage(mCaller.obtainMessageIO( + DO_SET_IME_SESSION_ENABLED, enabled ? 1 : 0, ls)); + } catch (ClassCastException e) { + Log.w(LOG_TAG, "Incoming session not of correct type: " + session, e); + } + } + + /** This is called when an app binds input or when the service is enabled. */ + public void bindInput(InputBinding binding) { + if (mCancellationGroup != null) { + Log.e(LOG_TAG, "bindInput must be paired with unbindInput."); + } + mCancellationGroup = new CancellationGroup(); + InputConnection ic = new RemoteInputConnection(new WeakReference<>(() -> mContext), + IInputContext.Stub.asInterface(binding.getConnectionToken()), + mCancellationGroup); + InputBinding nu = new InputBinding(ic, binding); + final Message message = mCaller.obtainMessageO(DO_BIND_INPUT, nu); + mCaller.sendMessage(message); + } + + /** This is called when an app unbinds input or when the service is disabled. */ + public void unbindInput() { + if (mCancellationGroup != null) { + // Signal the flag then forget it. + mCancellationGroup.cancelAll(); + mCancellationGroup = null; + } else { + Log.e(LOG_TAG, "unbindInput must be paired with bindInput."); + } + mCaller.sendMessage(mCaller.obtainMessage(DO_UNBIND_INPUT)); + } + + /** This is called when an app starts input or when the service is enabled. */ + public void startInput(IBinder startInputToken, IInputContext inputContext, + EditorInfo editorInfo, boolean restarting) { + if (mCancellationGroup == null) { + Log.e(LOG_TAG, "startInput must be called after bindInput."); + mCancellationGroup = new CancellationGroup(); + } + final Message message = mCaller.obtainMessageOOOOII(DO_START_INPUT, startInputToken, + inputContext, editorInfo, mCancellationGroup, restarting ? 1 : 0, + 0 /* unused */); + mCaller.sendMessage(message); + } + @Override public void onMotionEvent(MotionEvent event) { final Message message = PooledLambda.obtainMessage( @@ -2948,6 +3133,50 @@ public abstract class AccessibilityService extends Service { } return; } + case DO_CREATE_IME_SESSION: { + if (mConnectionId != AccessibilityInteractionClient.NO_ID) { + IInputSessionWithIdCallback callback = + (IInputSessionWithIdCallback) message.obj; + mCallback.createImeSession(callback); + } + return; + } + case DO_SET_IME_SESSION_ENABLED: { + if (mConnectionId != AccessibilityInteractionClient.NO_ID) { + mCallback.setImeSessionEnabled((InputMethodSession) message.obj, + message.arg1 != 0); + } + return; + } + case DO_BIND_INPUT: { + if (mConnectionId != AccessibilityInteractionClient.NO_ID) { + mCallback.bindInput((InputBinding) message.obj); + } + return; + } + case DO_UNBIND_INPUT: { + if (mConnectionId != AccessibilityInteractionClient.NO_ID) { + mCallback.unbindInput(); + } + return; + } + case DO_START_INPUT: { + if (mConnectionId != AccessibilityInteractionClient.NO_ID) { + final SomeArgs args = (SomeArgs) message.obj; + final IBinder startInputToken = (IBinder) args.arg1; + final IInputContext inputContext = (IInputContext) args.arg2; + final EditorInfo info = (EditorInfo) args.arg3; + final CancellationGroup cancellationGroup = (CancellationGroup) args.arg4; + final boolean restarting = args.argi5 == 1; + final InputConnection ic = inputContext != null + ? new RemoteInputConnection(new WeakReference<>(() -> mContext), + inputContext, cancellationGroup) : null; + info.makeCompatible(mContext.getApplicationInfo().targetSdkVersion); + mCallback.startInput(ic, info, restarting, startInputToken); + args.recycle(); + } + return; + } default: Log.w(LOG_TAG, "Unknown message type " + message.what); } diff --git a/core/java/android/accessibilityservice/IAccessibilityServiceClient.aidl b/core/java/android/accessibilityservice/IAccessibilityServiceClient.aidl index 375383d5d8588..94da61f82d294 100644 --- a/core/java/android/accessibilityservice/IAccessibilityServiceClient.aidl +++ b/core/java/android/accessibilityservice/IAccessibilityServiceClient.aidl @@ -24,6 +24,11 @@ import android.accessibilityservice.AccessibilityGestureEvent; import android.accessibilityservice.MagnificationConfig; import android.view.KeyEvent; import android.view.MotionEvent; +import android.view.inputmethod.EditorInfo; +import android.view.inputmethod.InputBinding; +import com.android.internal.view.IInputContext; +import com.android.internal.view.IInputMethodSession; +import com.android.internal.view.IInputSessionWithIdCallback; /** * Top-level interface to an accessibility service component. @@ -63,4 +68,15 @@ import android.view.MotionEvent; void onAccessibilityButtonAvailabilityChanged(boolean available); void onSystemActionsChanged(); + + void createImeSession(IInputSessionWithIdCallback callback); + + void setImeSessionEnabled(IInputMethodSession session, boolean enabled); + + void bindInput(in InputBinding binding); + + void unbindInput(); + + void startInput(in IBinder startInputToken, in IInputContext inputContext, + in EditorInfo editorInfo, boolean restarting); } diff --git a/core/java/android/accessibilityservice/InputMethod.java b/core/java/android/accessibilityservice/InputMethod.java new file mode 100644 index 0000000000000..1684bea274a90 --- /dev/null +++ b/core/java/android/accessibilityservice/InputMethod.java @@ -0,0 +1,637 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.accessibilityservice; + +import static android.os.Trace.TRACE_TAG_WINDOW_MANAGER; + +import android.annotation.CallbackExecutor; +import android.annotation.IntRange; +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.annotation.SuppressLint; +import android.graphics.Rect; +import android.inputmethodservice.IInputMethodSessionWrapper; +import android.inputmethodservice.RemoteInputConnection; +import android.os.Bundle; +import android.os.RemoteException; +import android.os.Trace; +import android.util.Log; +import android.view.KeyCharacterMap; +import android.view.KeyEvent; +import android.view.MotionEvent; +import android.view.inputmethod.CompletionInfo; +import android.view.inputmethod.CursorAnchorInfo; +import android.view.inputmethod.EditorInfo; +import android.view.inputmethod.ExtractedText; +import android.view.inputmethod.InputBinding; +import android.view.inputmethod.InputConnection; +import android.view.inputmethod.InputMethodManager; +import android.view.inputmethod.InputMethodSession; +import android.view.inputmethod.SurroundingText; +import android.view.inputmethod.TextAttribute; + +import com.android.internal.view.IInputContext; +import com.android.internal.view.IInputSessionWithIdCallback; + +import java.util.concurrent.Executor; + +/** + * This class provides input method APIs. Some public methods such as + * @link #onUpdateSelection(int, int, int, int, int, int)} do nothing by default and service + * developers should override them as needed. Developers should also override + * {@link AccessibilityService#onCreateInputMethod()} to return + * their custom InputMethod implementation. Accessibility services also need to set the + * {@link AccessibilityServiceInfo#FLAG_REQUEST_IME_APIS} flag to use input method APIs. + */ +public class InputMethod { + private static final String LOG_TAG = "A11yInputMethod"; + + private final AccessibilityService mService; + private InputBinding mInputBinding; + private InputConnection mInputConnection; + private boolean mInputStarted; + private InputConnection mStartedInputConnection; + private EditorInfo mInputEditorInfo; + + protected InputMethod(@NonNull AccessibilityService service) { + mService = service; + } + + /** + * Retrieve the currently active InputConnection that is bound to + * the input method, or null if there is none. + */ + @Nullable + public final AccessibilityInputConnection getCurrentInputConnection() { + InputConnection ic = mStartedInputConnection; + if (ic != null) { + return new AccessibilityInputConnection(ic); + } + if (mInputConnection != null) { + return new AccessibilityInputConnection(mInputConnection); + } + return null; + } + + /** + * Whether the input has started. + */ + public final boolean getCurrentInputStarted() { + return mInputStarted; + } + + /** + * Get the EditorInfo which describes several attributes of a text editing object + * that an accessibility service is communicating with (typically an EditText). + */ + @Nullable + public final EditorInfo getCurrentInputEditorInfo() { + return mInputEditorInfo; + } + + /** + * Called to inform the accessibility service that text input has started in an + * editor. You should use this callback to initialize the state of your + * input to match the state of the editor given to it. + * + * @param attribute The attributes of the editor that input is starting + * in. + * @param restarting Set to true if input is restarting in the same + * editor such as because the application has changed the text in + * the editor. Otherwise will be false, indicating this is a new + * session with the editor. + */ + public void onStartInput(@NonNull EditorInfo attribute, boolean restarting) { + // Intentionally empty + } + + /** + * Called to inform the accessibility service that text input has finished in + * the last editor. At this point there may be a call to + * {@link #onStartInput(EditorInfo, boolean)} to perform input in a + * new editor, or the accessibility service may be left idle. This method is + * not called when input restarts in the same editor. + * + *

The default + * implementation uses the InputConnection to clear any active composing + * text; you can override this (not calling the base class implementation) + * to perform whatever behavior you would like. + */ + public void onFinishInput() { + InputConnection ic = mStartedInputConnection != null ? mStartedInputConnection + : mInputConnection; + if (ic != null) { + ic.finishComposingText(); + } + } + + /** + * Called when the application has reported a new selection region of + * the text. This is called whether or not the accessibility service has requested + * extracted text updates, although if so it will not receive this call + * if the extracted text has changed as well. + * + *

Be careful about changing the text in reaction to this call with + * methods such as setComposingText, commitText or + * deleteSurroundingText. If the cursor moves as a result, this method + * will be called again, which may result in an infinite loop. + */ + public void onUpdateSelection(int oldSelStart, int oldSelEnd, int newSelStart, + int newSelEnd, int candidatesStart, int candidatesEnd) { + // Intentionally empty + } + + final void createImeSession(IInputSessionWithIdCallback callback) { + InputMethodSession session = onCreateInputMethodSessionInterface(); + try { + IInputMethodSessionWrapper wrap = + new IInputMethodSessionWrapper(mService, session, null); + callback.sessionCreated(wrap, mService.getConnectionId()); + } catch (RemoteException ignored) { + } + } + + final void setImeSessionEnabled(@NonNull InputMethodSession session, boolean enabled) { + ((InputMethodSessionForAccessibility) session).setEnabled(enabled); + } + + final void bindInput(@NonNull InputBinding binding) { + Trace.traceBegin(TRACE_TAG_WINDOW_MANAGER, "AccessibilityService.bindInput"); + mInputBinding = binding; + mInputConnection = binding.getConnection(); + Log.v(LOG_TAG, "bindInput(): binding=" + binding + + " ic=" + mInputConnection); + Trace.traceEnd(TRACE_TAG_WINDOW_MANAGER); + } + + final void unbindInput() { + Log.v(LOG_TAG, "unbindInput(): binding=" + mInputBinding + + " ic=" + mInputConnection); + // Unbind input is per process per display. + mInputBinding = null; + mInputConnection = null; + } + + final void startInput(@Nullable InputConnection ic, @NonNull EditorInfo attribute) { + Log.v(LOG_TAG, "startInput(): editor=" + attribute); + Trace.traceBegin(TRACE_TAG_WINDOW_MANAGER, "IMS.startInput"); + doStartInput(ic, attribute, false /* restarting */); + Trace.traceEnd(TRACE_TAG_WINDOW_MANAGER); + } + + final void restartInput(@Nullable InputConnection ic, @NonNull EditorInfo attribute) { + Log.v(LOG_TAG, "restartInput(): editor=" + attribute); + Trace.traceBegin(TRACE_TAG_WINDOW_MANAGER, "IMS.restartInput"); + doStartInput(ic, attribute, true /* restarting */); + Trace.traceEnd(TRACE_TAG_WINDOW_MANAGER); + } + + + final void doStartInput(InputConnection ic, EditorInfo attribute, boolean restarting) { + if (!restarting && mInputStarted) { + doFinishInput(); + } + mInputStarted = true; + mStartedInputConnection = ic; + mInputEditorInfo = attribute; + Log.v(LOG_TAG, "CALL: onStartInput"); + onStartInput(attribute, restarting); + } + + final void doFinishInput() { + Log.v(LOG_TAG, "CALL: doFinishInput"); + if (mInputStarted) { + Log.v(LOG_TAG, "CALL: onFinishInput"); + onFinishInput(); + } + mInputStarted = false; + mStartedInputConnection = null; + } + + private InputMethodSession onCreateInputMethodSessionInterface() { + return new InputMethodSessionForAccessibility(); + } + + /** + * This class provides the allowed list of {@link InputConnection} APIs for + * accessibility services. + */ + public final class AccessibilityInputConnection { + private InputConnection mIc; + AccessibilityInputConnection(InputConnection ic) { + this.mIc = ic; + } + + /** + * Commit text to the text box and set the new cursor position. This method is + * used to allow the IME to provide extra information while setting up text. + * + *

This method commits the contents of the currently composing text, and then + * moves the cursor according to {@code newCursorPosition}. If there + * is no composing text when this method is called, the new text is + * inserted at the cursor position, removing text inside the selection + * if any. + * + *

Calling this method will cause the editor to call + * {@link #onUpdateSelection(int, int, int, int, + * int, int)} on the current accessibility service after the batch input is over. + * Editor authors, for this to happen you need to + * make the changes known to the accessibility service by calling + * {@link InputMethodManager#updateSelection(View, int, int, int, int)}, + * but be careful to wait until the batch edit is over if one is + * in progress.

+ * + * @param text The text to commit. This may include styles. + * @param newCursorPosition The new cursor position around the text, + * in Java characters. If > 0, this is relative to the end + * of the text - 1; if <= 0, this is relative to the start + * of the text. So a value of 1 will always advance the cursor + * to the position after the full text being inserted. Note that + * this means you can't position the cursor within the text, + * because the editor can make modifications to the text + * you are providing so it is not possible to correctly specify + * locations there. + * @param textAttribute The extra information about the text. + */ + public void commitText(@NonNull CharSequence text, int newCursorPosition, + @Nullable TextAttribute textAttribute) { + if (mIc != null) { + mIc.commitText(text, newCursorPosition, textAttribute); + } + } + + /** + * Set the selection of the text editor. To set the cursor + * position, start and end should have the same value. + * + *

Since this moves the cursor, calling this method will cause + * the editor to call + * {@link android.inputmethodservice.InputMethodService#onUpdateSelection(int, int, int, + * int,int, int)} on the current IME after the batch input is over. + * Editor authors, for this to happen you need to + * make the changes known to the input method by calling + * {@link InputMethodManager#updateSelection(View, int, int, int, int)}, + * but be careful to wait until the batch edit is over if one is + * in progress.

+ * + *

This has no effect on the composing region which must stay + * unchanged. The order of start and end is not important. In + * effect, the region from start to end and the region from end to + * start is the same. Editor authors, be ready to accept a start + * that is greater than end.

+ * + * @param start the character index where the selection should start. + * @param end the character index where the selection should end. + */ + public void setSelection(int start, int end) { + if (mIc != null) { + mIc.setSelection(start, end); + } + } + + /** + * Gets the surrounding text around the current cursor, with beforeLength + * characters of text before the cursor (start of the selection), afterLength + * characters of text after the cursor (end of the selection), and all of the selected + * text. The range are for java characters, not glyphs that can be multiple characters. + * + *

This method may fail either if the input connection has become invalid (such as its + * process crashing), or the client is taking too long to respond with the text (it is + * given a couple seconds to return), or the protocol is not supported. In any of these + * cases, null is returned. + * + *

This method does not affect the text in the editor in any way, nor does it affect the + * selection or composing spans.

+ * + *

If {@link InputConnection#GET_TEXT_WITH_STYLES} is supplied as flags, the editor + * should return a {@link android.text.Spanned} with all the spans set on the text.

+ * + *

Accessibility service authors: please consider this will trigger an + * IPC round-trip that will take some time. Assume this method consumes a lot of time. + * + * @param beforeLength The expected length of the text before the cursor. + * @param afterLength The expected length of the text after the cursor. + * @param flags Supplies additional options controlling how the text is returned. May be + * either {@code 0} or {@link InputConnection#GET_TEXT_WITH_STYLES}. + * @return an {@link android.view.inputmethod.SurroundingText} object describing the + * surrounding text and state of selection, or null if the input connection is no longer + * valid, or the editor can't comply with the request for some reason, or the application + * does not implement this method. The length of the returned text might be less than the + * sum of beforeLength and afterLength . + * @throws IllegalArgumentException if {@code beforeLength} or {@code afterLength} is + * negative. + */ + @Nullable + public SurroundingText getSurroundingText( + @IntRange(from = 0) int beforeLength, @IntRange(from = 0) int afterLength, + @InputConnection.GetTextType int flags) { + if (mIc != null) { + return mIc.getSurroundingText(beforeLength, afterLength, flags); + } + return null; + } + + /** + * Delete beforeLength characters of text before the + * current cursor position, and delete afterLength + * characters of text after the current cursor position, excluding + * the selection. Before and after refer to the order of the + * characters in the string, not to their visual representation: + * this means you don't have to figure out the direction of the + * text and can just use the indices as-is. + * + *

The lengths are supplied in Java chars, not in code points + * or in glyphs.

+ * + *

Since this method only operates on text before and after the + * selection, it can't affect the contents of the selection. This + * may affect the composing span if the span includes characters + * that are to be deleted, but otherwise will not change it. If + * some characters in the composing span are deleted, the + * composing span will persist but get shortened by however many + * chars inside it have been removed.

+ * + *

Accessibility service authors: please be careful not to + * delete only half of a surrogate pair. Also take care not to + * delete more characters than are in the editor, as that may have + * ill effects on the application. Calling this method will cause + * the editor to call + * {@link android.inputmethodservice.InputMethodService#onUpdateSelection(int, int, int, int, + * int, int)} on your service after the batch input is over.

+ * + *

Editor authors: please be careful of race + * conditions in implementing this call. An IME can make a change + * to the text or change the selection position and use this + * method right away; you need to make sure the effects are + * consistent with the results of the latest edits. Also, although + * the IME should not send lengths bigger than the contents of the + * string, you should check the values for overflows and trim the + * indices to the size of the contents to avoid crashes. Since + * this changes the contents of the editor, you need to make the + * changes known to the input method by calling + * {@link InputMethodManager#updateSelection(View, int, int, int, int)}, + * but be careful to wait until the batch edit is over if one is + * in progress.

+ * + * @param beforeLength The number of characters before the cursor to be deleted, in code + * unit. If this is greater than the number of existing characters between the + * beginning of the text and the cursor, then this method does not fail but deletes + * all the characters in that range. + * @param afterLength The number of characters after the cursor to be deleted, in code unit. + * If this is greater than the number of existing characters between the cursor and + * the end of the text, then this method does not fail but deletes all the characters + * in that range. + */ + public void deleteSurroundingText(int beforeLength, int afterLength) { + if (mIc != null) { + mIc.deleteSurroundingText(beforeLength, afterLength); + } + } + + /** + * Send a key event to the process that is currently attached + * through this input connection. The event will be dispatched + * like a normal key event, to the currently focused view; this + * generally is the view that is providing this InputConnection, + * but due to the asynchronous nature of this protocol that can + * not be guaranteed and the focus may have changed by the time + * the event is received. + * + *

This method can be used to send key events to the + * application. For example, an on-screen keyboard may use this + * method to simulate a hardware keyboard. There are three types + * of standard keyboards, numeric (12-key), predictive (20-key) + * and ALPHA (QWERTY). You can specify the keyboard type by + * specify the device id of the key event.

+ * + *

You will usually want to set the flag + * {@link KeyEvent#FLAG_SOFT_KEYBOARD KeyEvent.FLAG_SOFT_KEYBOARD} + * on all key event objects you give to this API; the flag will + * not be set for you.

+ * + *

Note that it's discouraged to send such key events in normal + * operation; this is mainly for use with + * {@link android.text.InputType#TYPE_NULL} type text fields. Use + * the {@link #commitText} family of methods to send text to the + * application instead.

+ * + * @param event The key event. + * + * @see KeyEvent + * @see KeyCharacterMap#NUMERIC + * @see KeyCharacterMap#PREDICTIVE + * @see KeyCharacterMap#ALPHA + */ + public void sendKeyEvent(@NonNull KeyEvent event) { + if (mIc != null) { + mIc.sendKeyEvent(event); + } + } + + /** + * Have the editor perform an action it has said it can do. + * + * @param editorAction This must be one of the action constants for + * {@link EditorInfo#imeOptions EditorInfo.imeOptions}, such as + * {@link EditorInfo#IME_ACTION_GO EditorInfo.EDITOR_ACTION_GO}, or the value of + * {@link EditorInfo#actionId EditorInfo.actionId} if a custom action is available. + */ + public void performEditorAction(int editorAction) { + if (mIc != null) { + mIc.performEditorAction(editorAction); + } + } + + /** + * Perform a context menu action on the field. The given id may be one of: + * {@link android.R.id#selectAll}, + * {@link android.R.id#startSelectingText}, {@link android.R.id#stopSelectingText}, + * {@link android.R.id#cut}, {@link android.R.id#copy}, + * {@link android.R.id#paste}, {@link android.R.id#copyUrl}, + * or {@link android.R.id#switchInputMethod} + */ + public void performContextMenuAction(int id) { + if (mIc != null) { + mIc.performContextMenuAction(id); + } + } + + /** + * Retrieve the current capitalization mode in effect at the + * current cursor position in the text. See + * {@link android.text.TextUtils#getCapsMode TextUtils.getCapsMode} + * for more information. + * + *

This method may fail either if the input connection has + * become invalid (such as its process crashing) or the client is + * taking too long to respond with the text (it is given a couple + * seconds to return). In either case, 0 is returned.

+ * + *

This method does not affect the text in the editor in any + * way, nor does it affect the selection or composing spans.

+ * + *

Editor authors: please be careful of race + * conditions in implementing this call. An IME can change the + * cursor position and use this method right away; you need to make + * sure the returned value is consistent with the results of the + * latest edits and changes to the cursor position.

+ * + * @param reqModes The desired modes to retrieve, as defined by + * {@link android.text.TextUtils#getCapsMode TextUtils.getCapsMode}. These + * constants are defined so that you can simply pass the current + * {@link EditorInfo#inputType TextBoxAttribute.contentType} value + * directly in to here. + * @return the caps mode flags that are in effect at the current + * cursor position. See TYPE_TEXT_FLAG_CAPS_* in {@link android.text.InputType}. + */ + public int getCursorCapsMode(int reqModes) { + if (mIc != null) { + return mIc.getCursorCapsMode(reqModes); + } + return 0; + } + + /** + * Clear the given meta key pressed states in the given input + * connection. + * + *

This can be used by the accessibility service to clear the meta key states set + * by a hardware keyboard with latched meta keys, if the editor + * keeps track of these.

+ * + * @param states The states to be cleared, may be one or more bits as + * per {@link KeyEvent#getMetaState() KeyEvent.getMetaState()}. + */ + public void clearMetaKeyStates(int states) { + if (mIc != null) { + mIc.clearMetaKeyStates(states); + } + } + } + + /** + * Concrete implementation of InputMethodSession that provides all of the standard behavior + * for an input method session. + */ + private final class InputMethodSessionForAccessibility implements InputMethodSession { + boolean mEnabled = true; + + public void setEnabled(boolean enabled) { + mEnabled = enabled; + } + + @Override + public void finishInput() { + if (mEnabled) { + doFinishInput(); + } + } + + @Override + public void updateSelection(int oldSelStart, int oldSelEnd, int newSelStart, + int newSelEnd, int candidatesStart, int candidatesEnd) { + if (mEnabled) { + InputMethod.this.onUpdateSelection(oldSelEnd, oldSelEnd, newSelStart, + newSelEnd, candidatesStart, candidatesEnd); + } + } + + @Override + public void viewClicked(boolean focusChanged) { + } + + @Override + public void updateCursor(@NonNull Rect newCursor) { + } + + @Override + public void displayCompletions( + @SuppressLint("ArrayReturn") @NonNull CompletionInfo[] completions) { + } + + @Override + public void updateExtractedText(int token, @NonNull ExtractedText text) { + } + + public void dispatchKeyEvent(int seq, @NonNull KeyEvent event, + @NonNull @CallbackExecutor Executor executor, @NonNull EventCallback callback) { + } + + @Override + public void dispatchKeyEvent(int seq, @NonNull KeyEvent event, + @NonNull EventCallback callback) { + } + + public void dispatchTrackballEvent(int seq, @NonNull MotionEvent event, + @NonNull @CallbackExecutor Executor executor, @NonNull EventCallback callback) { + } + + @Override + public void dispatchTrackballEvent(int seq, @NonNull MotionEvent event, + @NonNull EventCallback callback) { + } + + public void dispatchGenericMotionEvent(int seq, @NonNull MotionEvent event, + @NonNull @CallbackExecutor Executor executor, @NonNull EventCallback callback) { + } + + @Override + public void dispatchGenericMotionEvent(int seq, @NonNull MotionEvent event, + @NonNull EventCallback callback) { + } + + @Override + public void appPrivateCommand(@NonNull String action, @NonNull Bundle data) { + } + + @Override + public void toggleSoftInput(int showFlags, int hideFlags) { + } + + @Override + public void updateCursorAnchorInfo(@NonNull CursorAnchorInfo cursorAnchorInfo) { + } + + @Override + public void notifyImeHidden() { + } + + @Override + public void removeImeSurface() { + } + + /** + * {@inheritDoc} + */ + @Override + public void invalidateInputInternal(EditorInfo editorInfo, IInputContext inputContext, + int sessionId) { + // TODO(b/217788708): Add automated test. + if (mStartedInputConnection instanceof RemoteInputConnection) { + final RemoteInputConnection ric = + (RemoteInputConnection) mStartedInputConnection; + if (!ric.isSameConnection(inputContext)) { + // This is not an error, and can be safely ignored. + return; + } + editorInfo.makeCompatible( + mService.getApplicationInfo().targetSdkVersion); + restartInput(new RemoteInputConnection(ric, sessionId), editorInfo); + } + } + } +} diff --git a/core/java/android/app/UiAutomation.java b/core/java/android/app/UiAutomation.java index b41b5f005f1f3..2af8905fa3afa 100644 --- a/core/java/android/app/UiAutomation.java +++ b/core/java/android/app/UiAutomation.java @@ -63,9 +63,14 @@ import android.view.accessibility.AccessibilityInteractionClient; import android.view.accessibility.AccessibilityNodeInfo; import android.view.accessibility.AccessibilityWindowInfo; import android.view.accessibility.IAccessibilityInteractionConnection; +import android.view.inputmethod.EditorInfo; +import android.view.inputmethod.InputBinding; +import android.view.inputmethod.InputConnection; +import android.view.inputmethod.InputMethodSession; import com.android.internal.annotations.GuardedBy; import com.android.internal.util.function.pooled.PooledLambda; +import com.android.internal.view.IInputSessionWithIdCallback; import libcore.io.IoUtils; @@ -1565,6 +1570,29 @@ public final class UiAutomation { /* do nothing */ } + @Override + public void createImeSession(IInputSessionWithIdCallback callback) { + /* do nothing */ + } + + @Override + public void setImeSessionEnabled(InputMethodSession session, boolean enabled) { + } + + @Override + public void bindInput(InputBinding binding) { + } + + @Override + public void unbindInput() { + } + + @Override + public void startInput(@Nullable InputConnection inputConnection, + @NonNull EditorInfo editorInfo, boolean restarting, + @NonNull IBinder startInputToken) { + } + @Override public boolean onGesture(AccessibilityGestureEvent gestureEvent) { /* do nothing */ diff --git a/core/java/android/inputmethodservice/IInputMethodSessionWrapper.java b/core/java/android/inputmethodservice/IInputMethodSessionWrapper.java index eccbb403b3068..75356d1ce9944 100644 --- a/core/java/android/inputmethodservice/IInputMethodSessionWrapper.java +++ b/core/java/android/inputmethodservice/IInputMethodSessionWrapper.java @@ -41,7 +41,9 @@ import com.android.internal.os.SomeArgs; import com.android.internal.view.IInputContext; import com.android.internal.view.IInputMethodSession; -class IInputMethodSessionWrapper extends IInputMethodSession.Stub +/** @hide */ +// TODO(b/215636776): move IInputMethodSessionWrapper to proper package +public class IInputMethodSessionWrapper extends IInputMethodSession.Stub implements HandlerCaller.Callback { private static final String TAG = "InputMethodWrapper"; diff --git a/core/java/android/inputmethodservice/InputMethodServiceInternal.java b/core/java/android/inputmethodservice/InputMethodServiceInternal.java index 7cd4ff61b2c80..09dbb27359b03 100644 --- a/core/java/android/inputmethodservice/InputMethodServiceInternal.java +++ b/core/java/android/inputmethodservice/InputMethodServiceInternal.java @@ -18,6 +18,7 @@ package android.inputmethodservice; import android.annotation.NonNull; import android.annotation.Nullable; +import android.annotation.SuppressLint; import android.content.Context; import android.os.Bundle; import android.view.inputmethod.InputConnection; @@ -31,8 +32,11 @@ import java.io.PrintWriter; * framework classes for internal use. * *

CAVEATS: {@link AbstractInputMethodService} does not support all the methods here.

+ * + * @hide */ -interface InputMethodServiceInternal { +// TODO(b/215636776): move InputMethodServiceInternal to proper package +public interface InputMethodServiceInternal { /** * @return {@link Context} associated with the service. */ @@ -70,7 +74,8 @@ interface InputMethodServiceInternal { * closed for you after you return. * @param args additional arguments to the dump request. */ - default void dump(FileDescriptor fd, PrintWriter fout, String[] args) { + default void dump(@SuppressLint("UseParcelFileDescriptor") @NonNull FileDescriptor fd, + @NonNull PrintWriter fout, @NonNull String[] args) { } /** @@ -81,6 +86,6 @@ interface InputMethodServiceInternal { * @param where {@code where} parameter to be passed. * @param icProto {@code icProto} parameter to be passed. */ - default void triggerServiceDump(String where, @Nullable byte[] icProto) { + default void triggerServiceDump(@NonNull String where, @Nullable byte[] icProto) { } } diff --git a/core/java/android/inputmethodservice/RemoteInputConnection.java b/core/java/android/inputmethodservice/RemoteInputConnection.java index 9ef2579d04e20..6b7815d0f7323 100644 --- a/core/java/android/inputmethodservice/RemoteInputConnection.java +++ b/core/java/android/inputmethodservice/RemoteInputConnection.java @@ -53,8 +53,11 @@ import java.util.concurrent.CompletableFuture; * *

See also {@link IInputContext} for the actual {@link android.os.Binder} IPC protocols under * the hood.

+ * + * @hide */ -final class RemoteInputConnection implements InputConnection { +// TODO(b/215636776): move RemoteInputConnection to proper package +public final class RemoteInputConnection implements InputConnection { private static final String TAG = "RemoteInputConnection"; private static final int MAX_WAIT_TIME_MILLIS = 2000; @@ -95,7 +98,7 @@ final class RemoteInputConnection implements InputConnection { @NonNull private final CancellationGroup mCancellationGroup; - RemoteInputConnection( + public RemoteInputConnection( @NonNull WeakReference inputMethodService, IInputContext inputContext, @NonNull CancellationGroup cancellationGroup) { mImsInternal = new InputMethodServiceInternalHolder(inputMethodService); @@ -108,7 +111,7 @@ final class RemoteInputConnection implements InputConnection { return mInvoker.isSameConnection(inputContext); } - RemoteInputConnection(@NonNull RemoteInputConnection original, int sessionId) { + public RemoteInputConnection(@NonNull RemoteInputConnection original, int sessionId) { mImsInternal = original.mImsInternal; mInvoker = original.mInvoker.cloneWithSessionId(sessionId); mCancellationGroup = original.mCancellationGroup; diff --git a/core/java/android/view/inputmethod/InputMethodManager.java b/core/java/android/view/inputmethod/InputMethodManager.java index f480b24bdba31..94b2215b30a8c 100644 --- a/core/java/android/view/inputmethod/InputMethodManager.java +++ b/core/java/android/view/inputmethod/InputMethodManager.java @@ -115,6 +115,7 @@ import java.util.Map; import java.util.Objects; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; /** * Central system API to the overall input method framework (IMF) architecture, @@ -426,6 +427,8 @@ public final class InputMethodManager { int mCursorSelEnd; int mCursorCandStart; int mCursorCandEnd; + int mInitialSelStart; + int mInitialSelEnd; /** * The instance that has previously been sent to the input method. @@ -468,6 +471,13 @@ public final class InputMethodManager { @Nullable @GuardedBy("mH") private InputMethodSessionWrapper mCurrentInputMethodSession = null; + /** + * Encapsulates IPCs to the currently connected AccessibilityServices. + */ + @Nullable + @GuardedBy("mH") + private final SparseArray mAccessibilityInputMethodSession = + new SparseArray<>(); InputChannel mCurChannel; ImeInputEventSender mCurSender; @@ -499,6 +509,8 @@ public final class InputMethodManager { static final int MSG_TIMEOUT_INPUT_EVENT = 6; static final int MSG_FLUSH_INPUT_EVENT = 7; static final int MSG_REPORT_FULLSCREEN_MODE = 10; + static final int MSG_BIND_ACCESSIBILITY_SERVICE = 11; + static final int MSG_UNBIND_ACCESSIBILITY_SERVICE = 12; private static boolean isAutofillUIShowing(View servedView) { AutofillManager afm = servedView.getContext().getSystemService(AutofillManager.class); @@ -626,6 +638,7 @@ public final class InputMethodManager { if (mCurrentInputMethodSession != null) { mCurrentInputMethodSession.finishInput(); } + forAccessibilitySessions(InputMethodSessionWrapper::finishInput); } } @@ -881,6 +894,7 @@ public final class InputMethodManager { if (mBindSequence != sequence) { return; } + clearAllAccessibilityBindingLocked(); clearBindingLocked(); // If we were actively using the last input method, then // we would like to re-connect to the next input method. @@ -896,6 +910,70 @@ public final class InputMethodManager { } return; } + case MSG_BIND_ACCESSIBILITY_SERVICE: { + final int id = msg.arg1; + final InputBindResult res = (InputBindResult) msg.obj; + if (DEBUG) { + Log.i(TAG, "handleMessage: MSG_BIND_ACCESSIBILITY " + res.sequence + + "," + res.id); + } + synchronized (mH) { + if (mBindSequence < 0 || mBindSequence != res.sequence) { + Log.w(TAG, "Ignoring onBind: cur seq=" + mBindSequence + + ", given seq=" + res.sequence); + if (res.channel != null && res.channel != mCurChannel) { + res.channel.dispose(); + } + return; + } + + // Since IMM can start inputting text before a11y sessions are back, + // we send a notification so that the a11y service knows the session is + // registered and update the a11y service with the current cursor positions. + InputMethodSessionWrapper wrapper = + InputMethodSessionWrapper.createOrNull(res.method); + if (wrapper != null) { + mAccessibilityInputMethodSession.put(id, wrapper); + if (mServedInputConnection != null) { + wrapper.updateSelection(mInitialSelStart, mInitialSelEnd, + mCursorSelStart, mCursorSelEnd, mCursorCandStart, + mCursorCandEnd); + } else { + // If an a11y service binds before input starts, we should still + // send a notification because the a11y service doesn't know it + // binds before or after input starts, it may wonder if it binds + // after input starts, why it doesn't receive a notification of + // the current cursor positions. + wrapper.updateSelection(-1, -1, + -1, -1, -1, + -1); + } + } + mBindSequence = res.sequence; + } + startInputInner(StartInputReason.BOUND_ACCESSIBILITY_SESSION_TO_IMMS, null, + 0, 0, 0); + return; + } + case MSG_UNBIND_ACCESSIBILITY_SERVICE: { + final int sequence = msg.arg1; + final int id = msg.arg2; + if (DEBUG) { + Log.i(TAG, "handleMessage: MSG_UNBIND_ACCESSIBILITY_SERVICE " + + sequence + " id=" + id); + } + synchronized (mH) { + if (mBindSequence != sequence) { + if (DEBUG) { + Log.i(TAG, "mBindSequence =" + mBindSequence + " sequence =" + + sequence + " id=" + id); + } + return; + } + clearAccessibilityBindingLocked(id); + } + return; + } case MSG_SET_ACTIVE: { final boolean active = msg.arg1 != 0; final boolean fullscreen = msg.arg2 != 0; @@ -995,11 +1073,21 @@ public final class InputMethodManager { mH.obtainMessage(MSG_BIND, res).sendToTarget(); } + @Override + public void onBindAccessibilityService(InputBindResult res, int id) { + mH.obtainMessage(MSG_BIND_ACCESSIBILITY_SERVICE, id, 0, res).sendToTarget(); + } + @Override public void onUnbindMethod(int sequence, @UnbindReason int unbindReason) { mH.obtainMessage(MSG_UNBIND, sequence, unbindReason).sendToTarget(); } + @Override + public void onUnbindAccessibilityService(int sequence, int id) { + mH.obtainMessage(MSG_UNBIND_ACCESSIBILITY_SERVICE, sequence, id).sendToTarget(); + } + @Override public void setActive(boolean active, boolean fullscreen, boolean reportToImeController) { mH.obtainMessage(MSG_SET_ACTIVE, active ? 1 : 0, fullscreen ? 1 : 0, @@ -1424,12 +1512,31 @@ public final class InputMethodManager { if (DEBUG) Log.v(TAG, "Clearing binding!"); clearConnectionLocked(); setInputChannelLocked(null); + // We only reset sequence number for input method, but not accessibility. mBindSequence = -1; mCurId = null; mCurMethod = null; // for @UnsupportedAppUsage mCurrentInputMethodSession = null; } + /** + * Reset all of the state associated with being bound to an accessibility service. + */ + @GuardedBy("mH") + void clearAccessibilityBindingLocked(int id) { + if (DEBUG) Log.v(TAG, "Clearing accessibility binding " + id); + mAccessibilityInputMethodSession.remove(id); + } + + /** + * Reset all of the state associated with being bound to all ccessibility services. + */ + @GuardedBy("mH") + void clearAllAccessibilityBindingLocked() { + if (DEBUG) Log.v(TAG, "Clearing all accessibility bindings"); + mAccessibilityInputMethodSession.clear(); + } + void setInputChannelLocked(InputChannel channel) { if (mCurChannel == channel) { return; @@ -1938,6 +2045,8 @@ public final class InputMethodManager { editorInfo.setInitialSurroundingTextInternal(textSnapshot.getSurroundingText()); mCurrentInputMethodSession.invalidateInput(editorInfo, mServedInputConnection, sessionId); + forAccessibilitySessions(wrapper -> wrapper.invalidateInput(editorInfo, + mServedInputConnection, sessionId)); } } @@ -2080,6 +2189,8 @@ public final class InputMethodManager { if (ic != null) { mCursorSelStart = tba.initialSelStart; mCursorSelEnd = tba.initialSelEnd; + mInitialSelStart = mCursorSelStart; + mInitialSelEnd = mCursorSelEnd; mCursorCandStart = -1; mCursorCandEnd = -1; mCursorRect.setEmpty(); @@ -2124,6 +2235,10 @@ public final class InputMethodManager { } mIsInputMethodSuppressingSpellChecker = res.isInputMethodSuppressingSpellChecker; if (res.id != null) { + // we might need to put a11y sessions and channels into res and restore them here. + // Currently we have a workaround to request a11y session after each client + // switching, even when the new client is opened before and is in memory (has + // existing a11y sessions). setInputChannelLocked(res.channel); mBindSequence = res.sequence; mCurMethod = res.method; // for @UnsupportedAppUsage @@ -2137,8 +2252,10 @@ public final class InputMethodManager { mRestartOnNextWindowFocus = true; break; } - if (mCurrentInputMethodSession != null && mCompletions != null) { - mCurrentInputMethodSession.displayCompletions(mCompletions); + if (mCompletions != null) { + if (mCurrentInputMethodSession != null) { + mCurrentInputMethodSession.displayCompletions(mCompletions); + } } } @@ -2369,6 +2486,8 @@ public final class InputMethodManager { mCursorCandEnd = candidatesEnd; mCurrentInputMethodSession.updateSelection( oldSelStart, oldSelEnd, selStart, selEnd, candidatesStart, candidatesEnd); + forAccessibilitySessions(wrapper -> wrapper.updateSelection(oldSelStart, + oldSelEnd, selStart, selEnd, candidatesStart, candidatesEnd)); } } } @@ -3233,6 +3352,11 @@ public final class InputMethodManager { } else { p.println(" mCurMethod= null"); } + for (int i = 0; i < mAccessibilityInputMethodSession.size(); i++) { + p.println(" mAccessibilityInputMethodSession(" + + mAccessibilityInputMethodSession.keyAt(i) + ")=" + + mAccessibilityInputMethodSession.valueAt(i)); + } p.println(" mCurRootView=" + mCurRootView); p.println(" mServedView=" + getServedViewLocked()); p.println(" mNextServedView=" + getNextServedViewLocked()); @@ -3377,4 +3501,10 @@ public final class InputMethodManager { } } } + + private void forAccessibilitySessions(Consumer consumer) { + for (int i = 0; i < mAccessibilityInputMethodSession.size(); i++) { + consumer.accept(mAccessibilityInputMethodSession.valueAt(i)); + } + } } diff --git a/core/java/com/android/internal/inputmethod/InputBindResult.java b/core/java/com/android/internal/inputmethod/InputBindResult.java index 1357bac30667c..1bc46f61429eb 100644 --- a/core/java/com/android/internal/inputmethod/InputBindResult.java +++ b/core/java/com/android/internal/inputmethod/InputBindResult.java @@ -53,7 +53,8 @@ public final class InputBindResult implements Parcelable { ResultCode.ERROR_NOT_IME_TARGET_WINDOW, ResultCode.ERROR_NO_EDITOR, ResultCode.ERROR_DISPLAY_ID_MISMATCH, - ResultCode.ERROR_INVALID_DISPLAY_ID + ResultCode.ERROR_INVALID_DISPLAY_ID, + ResultCode.SUCCESS_WITH_ACCESSIBILITY_SESSION }) public @interface ResultCode { /** @@ -168,6 +169,7 @@ public final class InputBindResult implements Parcelable { * display. */ int ERROR_INVALID_DISPLAY_ID = 15; + int SUCCESS_WITH_ACCESSIBILITY_SESSION = 16; } @ResultCode diff --git a/core/java/com/android/internal/inputmethod/InputMethodDebug.java b/core/java/com/android/internal/inputmethod/InputMethodDebug.java index bf094dbd8f1e7..d6697684f79e7 100644 --- a/core/java/com/android/internal/inputmethod/InputMethodDebug.java +++ b/core/java/com/android/internal/inputmethod/InputMethodDebug.java @@ -64,6 +64,8 @@ public final class InputMethodDebug { return "DEACTIVATED_BY_IMMS"; case StartInputReason.SESSION_CREATED_BY_IME: return "SESSION_CREATED_BY_IME"; + case StartInputReason.BOUND_ACCESSIBILITY_SESSION_TO_IMMS: + return "BOUND_ACCESSIBILITY_SESSION_TO_IMMS"; default: return "Unknown=" + reason; } @@ -91,6 +93,8 @@ public final class InputMethodDebug { return "SWITCH_IME_FAILED"; case UnbindReason.SWITCH_USER: return "SWITCH_USER"; + case UnbindReason.ACCESSIBILITY_SERVICE_DISABLED: + return "ACCESSIBILITY_SERVICE_DISABLED"; default: return "Unknown=" + reason; } diff --git a/core/java/com/android/internal/inputmethod/StartInputReason.java b/core/java/com/android/internal/inputmethod/StartInputReason.java index 2ba708dd93124..1263466703ace 100644 --- a/core/java/com/android/internal/inputmethod/StartInputReason.java +++ b/core/java/com/android/internal/inputmethod/StartInputReason.java @@ -38,7 +38,9 @@ import java.lang.annotation.Retention; StartInputReason.UNBOUND_FROM_IMMS, StartInputReason.ACTIVATED_BY_IMMS, StartInputReason.DEACTIVATED_BY_IMMS, - StartInputReason.SESSION_CREATED_BY_IME}) + StartInputReason.SESSION_CREATED_BY_IME, + StartInputReason.SESSION_CREATED_BY_ACCESSIBILITY, + StartInputReason.BOUND_ACCESSIBILITY_SESSION_TO_IMMS}) public @interface StartInputReason { /** * Reason is not specified. @@ -96,4 +98,14 @@ public @interface StartInputReason { * {@link com.android.internal.view.IInputSessionCallback#sessionCreated}. */ int SESSION_CREATED_BY_IME = 10; + /** + * {@link android.accessibilityservice.AccessibilityService} is responding to + * {@link com.android.internal.view.IInputSessionWithIdCallback#sessionCreated}. + */ + int SESSION_CREATED_BY_ACCESSIBILITY = 11; + /** + * {@link android.view.inputmethod.InputMethodManager} is responding to + * {@link com.android.internal.view.IInputMethodClient#onBindAccessibilityService(InputBindResult, int)}. + */ + int BOUND_ACCESSIBILITY_SESSION_TO_IMMS = 12; } diff --git a/core/java/com/android/internal/inputmethod/UnbindReason.java b/core/java/com/android/internal/inputmethod/UnbindReason.java index f0f18f11abe71..e9266251371e3 100644 --- a/core/java/com/android/internal/inputmethod/UnbindReason.java +++ b/core/java/com/android/internal/inputmethod/UnbindReason.java @@ -34,7 +34,9 @@ import java.lang.annotation.Retention; UnbindReason.DISCONNECT_IME, UnbindReason.NO_IME, UnbindReason.SWITCH_IME_FAILED, - UnbindReason.SWITCH_USER}) + UnbindReason.SWITCH_USER, + UnbindReason.ACCESSIBILITY_SERVICE_DISABLED +}) public @interface UnbindReason { /** * Reason is not specified. @@ -66,4 +68,5 @@ public @interface UnbindReason { * user's active IME. */ int SWITCH_USER = 6; + int ACCESSIBILITY_SERVICE_DISABLED = 7; } diff --git a/core/java/com/android/internal/view/IInputMethodClient.aidl b/core/java/com/android/internal/view/IInputMethodClient.aidl index e72afdd78ba43..8430c0859bc9f 100644 --- a/core/java/com/android/internal/view/IInputMethodClient.aidl +++ b/core/java/com/android/internal/view/IInputMethodClient.aidl @@ -24,7 +24,9 @@ import com.android.internal.inputmethod.InputBindResult; */ oneway interface IInputMethodClient { void onBindMethod(in InputBindResult res); + void onBindAccessibilityService(in InputBindResult res, int id); void onUnbindMethod(int sequence, int unbindReason); + void onUnbindAccessibilityService(int sequence, int id); void setActive(boolean active, boolean fullscreen, boolean reportToImeController); void scheduleStartInputIfNecessary(boolean fullscreen); void reportFullscreenMode(boolean fullscreen); diff --git a/core/java/com/android/internal/view/IInputSessionWithIdCallback.aidl b/core/java/com/android/internal/view/IInputSessionWithIdCallback.aidl new file mode 100644 index 0000000000000..8fbdefe5931e8 --- /dev/null +++ b/core/java/com/android/internal/view/IInputSessionWithIdCallback.aidl @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + package com.android.internal.view; + + import com.android.internal.view.IInputMethodSession; + +/** + * Helper interface for IInputMethod to allow the input method to notify the client when a new + * session has been created. + */ +oneway interface IInputSessionWithIdCallback { + void sessionCreated(IInputMethodSession session, int id); +} \ No newline at end of file diff --git a/services/accessibility/java/com/android/server/accessibility/AbstractAccessibilityServiceConnection.java b/services/accessibility/java/com/android/server/accessibility/AbstractAccessibilityServiceConnection.java index 8b62a64f57d4c..0f65ebca9b3ac 100644 --- a/services/accessibility/java/com/android/server/accessibility/AbstractAccessibilityServiceConnection.java +++ b/services/accessibility/java/com/android/server/accessibility/AbstractAccessibilityServiceConnection.java @@ -25,6 +25,7 @@ import static android.accessibilityservice.AccessibilityTrace.FLAGS_ACCESSIBILIT import static android.accessibilityservice.AccessibilityTrace.FLAGS_ACCESSIBILITY_SERVICE_CLIENT; import static android.accessibilityservice.AccessibilityTrace.FLAGS_ACCESSIBILITY_SERVICE_CONNECTION; import static android.accessibilityservice.AccessibilityTrace.FLAGS_WINDOW_MANAGER_INTERNAL; +import static android.os.Trace.TRACE_TAG_WINDOW_MANAGER; import static android.view.WindowManager.LayoutParams.TYPE_ACCESSIBILITY_OVERLAY; import static android.view.accessibility.AccessibilityInteractionClient.CALL_STACK; import static android.view.accessibility.AccessibilityInteractionClient.IGNORE_CALL_STACK; @@ -66,6 +67,7 @@ import android.os.RemoteCallback; import android.os.RemoteException; import android.os.ServiceManager; import android.os.SystemClock; +import android.os.Trace; import android.provider.Settings; import android.util.Slog; import android.util.SparseArray; @@ -80,15 +82,21 @@ import android.view.accessibility.AccessibilityEvent; import android.view.accessibility.AccessibilityNodeInfo; import android.view.accessibility.AccessibilityWindowInfo; import android.view.accessibility.IAccessibilityInteractionConnectionCallback; +import android.view.inputmethod.EditorInfo; +import android.view.inputmethod.InputBinding; import com.android.internal.annotations.GuardedBy; import com.android.internal.compat.IPlatformCompat; import com.android.internal.os.SomeArgs; import com.android.internal.util.DumpUtils; import com.android.internal.util.function.pooled.PooledLambda; +import com.android.internal.view.IInputContext; +import com.android.internal.view.IInputMethodSession; +import com.android.internal.view.IInputSessionWithIdCallback; import com.android.server.LocalServices; import com.android.server.accessibility.AccessibilityWindowManager.RemoteAccessibilityConnection; import com.android.server.accessibility.magnification.MagnificationProcessor; +import com.android.server.inputmethod.InputMethodManagerInternal; import com.android.server.wm.ActivityTaskManagerInternal; import com.android.server.wm.WindowManagerInternal; @@ -271,6 +279,9 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ void onDoubleTapAndHold(int displayId); + void requestImeLocked(AccessibilityServiceConnection connection); + + void unbindImeLocked(AccessibilityServiceConnection connection); } public AbstractAccessibilityServiceConnection(Context context, ComponentName componentName, @@ -1610,6 +1621,27 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ mInvocationHandler.notifyAccessibilityButtonAvailabilityChangedLocked(available); } + public void createImeSessionLocked() { + mInvocationHandler.createImeSessionLocked(); + } + + public void setImeSessionEnabledLocked(IInputMethodSession session, boolean enabled) { + mInvocationHandler.setImeSessionEnabledLocked(session, enabled); + } + + public void bindInputLocked(InputBinding binding) { + mInvocationHandler.bindInputLocked(binding); + } + + public void unbindInputLocked() { + mInvocationHandler.unbindInputLocked(); + } + + public void startInputLocked(IBinder startInputToken, IInputContext inputContext, + EditorInfo editorInfo, boolean restarting) { + mInvocationHandler.startInputLocked(startInputToken, inputContext, editorInfo, restarting); + } + /** * Called by the invocation handler to notify the service that the * state of magnification has changed. @@ -1732,6 +1764,84 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ } } + private void createImeSessionInternal() { + final IAccessibilityServiceClient listener = getServiceInterfaceSafely(); + if (listener != null) { + try { + if (svcClientTracingEnabled()) { + logTraceSvcClient("createImeSession", ""); + } + AccessibilityCallback callback = new AccessibilityCallback(); + listener.createImeSession(callback); + } catch (RemoteException re) { + Slog.e(LOG_TAG, + "Error requesting IME session from " + mService, re); + } + } + } + + private void setImeSessionEnabledInternal(IInputMethodSession session, boolean enabled) { + final IAccessibilityServiceClient listener = getServiceInterfaceSafely(); + if (listener != null && session != null) { + try { + if (svcClientTracingEnabled()) { + logTraceSvcClient("createImeSession", ""); + } + listener.setImeSessionEnabled(session, enabled); + } catch (RemoteException re) { + Slog.e(LOG_TAG, + "Error requesting IME session from " + mService, re); + } + } + } + + private void bindInputInternal(InputBinding binding) { + final IAccessibilityServiceClient listener = getServiceInterfaceSafely(); + if (listener != null) { + try { + if (svcClientTracingEnabled()) { + logTraceSvcClient("bindInput", binding.toString()); + } + listener.bindInput(binding); + } catch (RemoteException re) { + Slog.e(LOG_TAG, + "Error binding input to " + mService, re); + } + } + } + + private void unbindInputInternal() { + final IAccessibilityServiceClient listener = getServiceInterfaceSafely(); + if (listener != null) { + try { + if (svcClientTracingEnabled()) { + logTraceSvcClient("unbindInput", ""); + } + listener.unbindInput(); + } catch (RemoteException re) { + Slog.e(LOG_TAG, + "Error unbinding input to " + mService, re); + } + } + } + + private void startInputInternal(IBinder startInputToken, IInputContext inputContext, + EditorInfo editorInfo, boolean restarting) { + final IAccessibilityServiceClient listener = getServiceInterfaceSafely(); + if (listener != null) { + try { + if (svcClientTracingEnabled()) { + logTraceSvcClient("startInput", startInputToken + " " + + inputContext + " " + editorInfo + restarting); + } + listener.startInput(startInputToken, inputContext, editorInfo, restarting); + } catch (RemoteException re) { + Slog.e(LOG_TAG, + "Error starting input to " + mService, re); + } + } + } + protected IAccessibilityServiceClient getServiceInterfaceSafely() { synchronized (mLock) { return mServiceInterface; @@ -1925,6 +2035,11 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ private static final int MSG_ON_ACCESSIBILITY_BUTTON_CLICKED = 7; private static final int MSG_ON_ACCESSIBILITY_BUTTON_AVAILABILITY_CHANGED = 8; private static final int MSG_ON_SYSTEM_ACTIONS_CHANGED = 9; + private static final int MSG_CREATE_IME_SESSION = 10; + private static final int MSG_SET_IME_SESSION_ENABLED = 11; + private static final int MSG_BIND_INPUT = 12; + private static final int MSG_UNBIND_INPUT = 13; + private static final int MSG_START_INPUT = 14; /** List of magnification callback states, mapping from displayId -> Boolean */ @GuardedBy("mlock") @@ -1974,6 +2089,29 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ notifySystemActionsChangedInternal(); break; } + case MSG_CREATE_IME_SESSION: + createImeSessionInternal(); + break; + case MSG_SET_IME_SESSION_ENABLED: + final boolean enabled = (message.arg1 != 0); + final IInputMethodSession session = (IInputMethodSession) message.obj; + setImeSessionEnabledInternal(session, enabled); + break; + case MSG_BIND_INPUT: + final InputBinding binding = (InputBinding) message.obj; + bindInputInternal(binding); + break; + case MSG_UNBIND_INPUT: + unbindInputInternal(); + break; + case MSG_START_INPUT: + final boolean restarting = (message.arg1 != 0); + final SomeArgs args = (SomeArgs) message.obj; + final IBinder startInputToken = (IBinder) args.arg1; + final IInputContext inputContext = (IInputContext) args.arg2; + final EditorInfo editorInfo = (EditorInfo) args.arg3; + startInputInternal(startInputToken, inputContext, editorInfo, restarting); + break; default: { throw new IllegalArgumentException("Unknown message: " + type); } @@ -2036,6 +2174,37 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ (available ? 1 : 0), 0); msg.sendToTarget(); } + + public void createImeSessionLocked() { + final Message msg = obtainMessage(MSG_CREATE_IME_SESSION); + msg.sendToTarget(); + } + + public void setImeSessionEnabledLocked(IInputMethodSession session, boolean enabled) { + final Message msg = obtainMessage(MSG_SET_IME_SESSION_ENABLED, (enabled ? 1 : 0), + 0, session); + msg.sendToTarget(); + } + + public void bindInputLocked(InputBinding binding) { + final Message msg = obtainMessage(MSG_BIND_INPUT, binding); + msg.sendToTarget(); + } + + public void unbindInputLocked() { + final Message msg = obtainMessage(MSG_UNBIND_INPUT); + msg.sendToTarget(); + } + + public void startInputLocked(IBinder startInputToken, IInputContext inputContext, + EditorInfo editorInfo, boolean restarting) { + final SomeArgs args = SomeArgs.obtain(); + args.arg1 = startInputToken; + args.arg2 = inputContext; + args.arg3 = editorInfo; + final Message msg = obtainMessage(MSG_START_INPUT, restarting ? 1 : 0, 0, args); + msg.sendToTarget(); + } } public boolean isServiceHandlesDoubleTapEnabled() { @@ -2185,4 +2354,18 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ Binder.restoreCallingIdentity(identity); } } + + private static final class AccessibilityCallback extends IInputSessionWithIdCallback.Stub { + @Override + public void sessionCreated(IInputMethodSession session, int id) { + Trace.traceBegin(TRACE_TAG_WINDOW_MANAGER, "IMMS.sessionCreated"); + final long ident = Binder.clearCallingIdentity(); + try { + InputMethodManagerInternal.get().onSessionForAccessibilityCreated(id, session); + } finally { + Binder.restoreCallingIdentity(ident); + } + Trace.traceEnd(TRACE_TAG_WINDOW_MANAGER); + } + } } \ No newline at end of file diff --git a/services/accessibility/java/com/android/server/accessibility/AccessibilityManagerService.java b/services/accessibility/java/com/android/server/accessibility/AccessibilityManagerService.java index 5b580d9d829c6..2f049f7edc1fb 100644 --- a/services/accessibility/java/com/android/server/accessibility/AccessibilityManagerService.java +++ b/services/accessibility/java/com/android/server/accessibility/AccessibilityManagerService.java @@ -115,6 +115,8 @@ import android.view.accessibility.IAccessibilityInteractionConnection; import android.view.accessibility.IAccessibilityManager; import android.view.accessibility.IAccessibilityManagerClient; import android.view.accessibility.IWindowMagnificationConnection; +import android.view.inputmethod.EditorInfo; +import android.view.inputmethod.InputBinding; import com.android.internal.R; import com.android.internal.accessibility.AccessibilityShortcutController; @@ -128,12 +130,16 @@ import com.android.internal.os.BackgroundThread; import com.android.internal.util.ArrayUtils; import com.android.internal.util.DumpUtils; import com.android.internal.util.IntPair; +import com.android.internal.view.IInputContext; +import com.android.internal.view.IInputMethodSession; +import com.android.server.AccessibilityManagerInternal; import com.android.server.LocalServices; import com.android.server.SystemService; import com.android.server.accessibility.magnification.MagnificationController; import com.android.server.accessibility.magnification.MagnificationProcessor; import com.android.server.accessibility.magnification.MagnificationScaleProvider; import com.android.server.accessibility.magnification.WindowMagnificationManager; +import com.android.server.inputmethod.InputMethodManagerInternal; import com.android.server.pm.UserManagerInternal; import com.android.server.wm.ActivityTaskManagerInternal; import com.android.server.wm.WindowManagerInternal; @@ -231,7 +237,7 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub private final MainHandler mMainHandler; - // Lazily initialized - access through getSystemActionPerfomer() + // Lazily initialized - access through getSystemActionPerformer() private SystemActionPerformer mSystemActionPerformer; private InteractionBridge mInteractionBridge; @@ -273,6 +279,13 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub private Point mTempPoint = new Point(); private boolean mIsAccessibilityButtonShown; + private InputBinding mInputBinding; + IBinder mStartInputToken; + IInputContext mInputContext; + EditorInfo mEditorInfo; + boolean mRestarting; + boolean mInputSessionRequested; + private AccessibilityUserState getCurrentUserStateLocked() { return getUserStateLocked(mCurrentUserId); } @@ -298,6 +311,42 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub } } + private static final class LocalServiceImpl extends AccessibilityManagerInternal { + @NonNull + private final AccessibilityManagerService mService; + + LocalServiceImpl(@NonNull AccessibilityManagerService service) { + mService = service; + } + + @Override + public void setImeSessionEnabled(SparseArray sessions, + boolean enabled) { + mService.setImeSessionEnabled(sessions, enabled); + } + + @Override + public void unbindInput() { + mService.unbindInput(); + } + + @Override + public void bindInput(InputBinding binding) { + mService.bindInput(binding); + } + + @Override + public void createImeSession() { + mService.createImeSession(); + } + + @Override + public void startInput(IBinder startInputToken, IInputContext inputContext, + EditorInfo editorInfo, boolean restarting) { + mService.startInput(startInputToken, inputContext, editorInfo, restarting); + } + } + public static final class Lifecycle extends SystemService { private final AccessibilityManagerService mService; @@ -308,6 +357,8 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub @Override public void onStart() { + LocalServices.addService(AccessibilityManagerInternal.class, + new LocalServiceImpl(mService)); publishBinderService(Context.ACCESSIBILITY_SERVICE, mService); } @@ -4240,6 +4291,45 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub this, displayId)); } + @Override + public void requestImeLocked(AccessibilityServiceConnection connection) { + mMainHandler.sendMessage(obtainMessage( + AccessibilityManagerService::createSessionForConnection, this, connection)); + mMainHandler.sendMessage(obtainMessage( + AccessibilityManagerService::bindAndStartInputForConnection, this, connection)); + } + + @Override + public void unbindImeLocked(AccessibilityServiceConnection connection) { + mMainHandler.sendMessage(obtainMessage( + AccessibilityManagerService::unbindInputForConnection, this, connection)); + } + + private void createSessionForConnection(AccessibilityServiceConnection connection) { + synchronized (mLock) { + if (mInputSessionRequested) { + connection.createImeSessionLocked(); + } + } + } + + private void bindAndStartInputForConnection(AccessibilityServiceConnection connection) { + synchronized (mLock) { + if (mInputBinding != null) { + connection.bindInputLocked(mInputBinding); + connection.startInputLocked(mStartInputToken, mInputContext, mEditorInfo, + mRestarting); + } + } + } + + private void unbindInputForConnection(AccessibilityServiceConnection connection) { + InputMethodManagerInternal.get().unbindAccessibilityFromCurrentClient(connection.mId); + synchronized (mLock) { + connection.unbindInputLocked(); + } + } + private void onDoubleTapAndHoldInternal(int displayId) { synchronized (mLock) { if (mHasInputFilter && mInputFilter != null) { @@ -4268,4 +4358,107 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub public AccessibilityTraceManager getTraceManager() { return mTraceManager; } + + /** + * Bind input for accessibility services which request ime capabilities. + * + * @param binding Information given to an accessibility service about a client connecting to it. + */ + public void bindInput(InputBinding binding) { + AccessibilityUserState userState; + synchronized (mLock) { + // Keep records of these in case new Accessibility Services are enabled. + mInputBinding = binding; + userState = getCurrentUserStateLocked(); + for (int i = userState.mBoundServices.size() - 1; i >= 0; i--) { + final AccessibilityServiceConnection service = userState.mBoundServices.get(i); + // TODO(b/187453053): mRequestedIme implementation + //if (service.mRequestedIme) { + service.bindInputLocked(binding); + //} + } + } + } + + /** + * Unbind input for accessibility services which request ime capabilities. + */ + public void unbindInput() { + AccessibilityUserState userState; + // TODO(b/218182733): Resolve the Imf lock and mLock possible deadlock + synchronized (mLock) { + userState = getCurrentUserStateLocked(); + for (int i = userState.mBoundServices.size() - 1; i >= 0; i--) { + final AccessibilityServiceConnection service = userState.mBoundServices.get(i); + // TODO(187453053): mRequestedIme implementation + //if (service.mRequestedIme) { + service.unbindInputLocked(); + //} + } + } + } + + /** + * Start input for accessibility services which request ime capabilities. + */ + public void startInput(IBinder startInputToken, IInputContext inputContext, + EditorInfo editorInfo, boolean restarting) { + //TODO(b/187453053): including the java doc + AccessibilityUserState userState; + synchronized (mLock) { + // Keep records of these in case new Accessibility Services are enabled. + mStartInputToken = startInputToken; + mInputContext = inputContext; + mEditorInfo = editorInfo; + mRestarting = restarting; + userState = getCurrentUserStateLocked(); + for (int i = userState.mBoundServices.size() - 1; i >= 0; i--) { + final AccessibilityServiceConnection service = userState.mBoundServices.get(i); + // TODO(b/187453053): mRequestedIme implementation + //if (service.mRequestedIme) { + service.startInputLocked(startInputToken, inputContext, editorInfo, restarting); + //} + } + } + } + + /** + * Request input sessions from all accessibility services which request ime capabilities. + */ + public void createImeSession() { + AccessibilityUserState userState; + synchronized (mLock) { + mInputSessionRequested = true; + userState = getCurrentUserStateLocked(); + for (int i = userState.mBoundServices.size() - 1; i >= 0; i--) { + final AccessibilityServiceConnection service = userState.mBoundServices.get(i); + // TODO(b/187453053): mRequestedIme implementation + //if (service.mRequestedIme) { + service.createImeSessionLocked(); + //} + } + } + } + + /** + * Enable or disable the sessions. + * + * @param sessions Sessions to enable or disable. + * @param enabled True if enable the sessions or false if disable the sessions. + */ + public void setImeSessionEnabled(SparseArray sessions, boolean enabled) { + AccessibilityUserState userState; + synchronized (mLock) { + userState = getCurrentUserStateLocked(); + for (int i = userState.mBoundServices.size() - 1; i >= 0; i--) { + final AccessibilityServiceConnection service = userState.mBoundServices.get(i); + // TODO(b/187453053): mRequestedIme implementation + if (sessions.contains(service.mId)) { + //if (service.mRequestedIme) { + service.setImeSessionEnabledLocked(sessions.get(service.mId), enabled); + //} + } + } + } + } } diff --git a/services/accessibility/java/com/android/server/accessibility/AccessibilityServiceConnection.java b/services/accessibility/java/com/android/server/accessibility/AccessibilityServiceConnection.java index 8f7260f0df834..e06ee77974ec7 100644 --- a/services/accessibility/java/com/android/server/accessibility/AccessibilityServiceConnection.java +++ b/services/accessibility/java/com/android/server/accessibility/AccessibilityServiceConnection.java @@ -127,6 +127,8 @@ class AccessibilityServiceConnection extends AbstractAccessibilityServiceConnect } public void unbindLocked() { + // If requested ime + mSystemSupport.unbindImeLocked(this); mContext.unbindService(this); AccessibilityUserState userState = mUserStateWeakReference.get(); if (userState == null) return; @@ -188,6 +190,8 @@ class AccessibilityServiceConnection extends AbstractAccessibilityServiceConnect // the new configuration (for example, initializing the input filter). mMainHandler.sendMessage(obtainMessage( AccessibilityServiceConnection::initializeService, this)); + //if (service.mRequestedIme) { + mSystemSupport.requestImeLocked(this); } } @@ -371,6 +375,7 @@ class AccessibilityServiceConnection extends AbstractAccessibilityServiceConnect if (!isConnectedLocked()) { return; } + mSystemSupport.unbindImeLocked(this); mAccessibilityServiceInfo.crashed = true; AccessibilityUserState userState = mUserStateWeakReference.get(); if (userState != null) { diff --git a/services/core/java/com/android/server/AccessibilityManagerInternal.java b/services/core/java/com/android/server/AccessibilityManagerInternal.java new file mode 100644 index 0000000000000..c02e94d7bc668 --- /dev/null +++ b/services/core/java/com/android/server/AccessibilityManagerInternal.java @@ -0,0 +1,82 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server; + +import android.annotation.NonNull; +import android.os.IBinder; +import android.util.SparseArray; +import android.view.inputmethod.EditorInfo; +import android.view.inputmethod.InputBinding; + +import com.android.internal.view.IInputContext; +import com.android.internal.view.IInputMethodSession; + +/** + * Accessibility manager local system service interface. + */ +public abstract class AccessibilityManagerInternal { + /** Enable or disable the sessions. */ + public abstract void setImeSessionEnabled(SparseArray sessions, + boolean enabled); + + /** Unbind input for all accessibility services which require ime capabilities. */ + public abstract void unbindInput(); + + /** Bind input for all accessibility services which require ime capabilities. */ + public abstract void bindInput(InputBinding binding); + + /** Request input session from all accessibility services which require ime capabilities. */ + public abstract void createImeSession(); + + /** Start input for all accessibility services which require ime capabilities. */ + public abstract void startInput(IBinder startInputToken, IInputContext inputContext, + EditorInfo editorInfo, boolean restarting); + + private static final AccessibilityManagerInternal NOP = new AccessibilityManagerInternal() { + @Override + public void setImeSessionEnabled(SparseArray sessions, + boolean enabled) { + } + + @Override + public void unbindInput() { + } + + @Override + public void bindInput(InputBinding binding) { + } + + @Override + public void createImeSession() { + } + + @Override + public void startInput(IBinder startInputToken, IInputContext inputContext, + EditorInfo editorInfo, boolean restarting) { + } + }; + + /** + * @return Global instance if exists. Otherwise, a fallback no-op instance. + */ + @NonNull + public static AccessibilityManagerInternal get() { + final AccessibilityManagerInternal instance = + LocalServices.getService(AccessibilityManagerInternal.class); + return instance != null ? instance : NOP; + } +} diff --git a/services/core/java/com/android/server/inputmethod/InputMethodManagerInternal.java b/services/core/java/com/android/server/inputmethod/InputMethodManagerInternal.java index 80c83e97256e7..29dcdfaa1bba4 100644 --- a/services/core/java/com/android/server/inputmethod/InputMethodManagerInternal.java +++ b/services/core/java/com/android/server/inputmethod/InputMethodManagerInternal.java @@ -25,6 +25,7 @@ import android.view.inputmethod.InputMethodInfo; import com.android.internal.inputmethod.SoftInputShowHideReason; import com.android.internal.view.IInlineSuggestionsRequestCallback; +import com.android.internal.view.IInputMethodSession; import com.android.internal.view.InlineSuggestionsRequestInfo; import com.android.server.LocalServices; @@ -148,6 +149,24 @@ public abstract class InputMethodManagerInternal { */ public abstract void updateImeWindowStatus(boolean disableImeIcon); + /** + * Callback when the IInputMethodSession from the accessibility service with the specified + * accessibilityConnectionId is created. + * + * @param accessibilityConnectionId The connection id of the accessibility service. + * @param session The session passed back from the accessibility service. + */ + public abstract void onSessionForAccessibilityCreated(int accessibilityConnectionId, + IInputMethodSession session); + + /** + * Unbind the accessibility service with the specified accessibilityConnectionId from current + * client. + * + * @param accessibilityConnectionId The connection id of the accessibility service. + */ + public abstract void unbindAccessibilityFromCurrentClient(int accessibilityConnectionId); + /** * Fake implementation of {@link InputMethodManagerInternal}. All the methods do nothing. */ @@ -211,6 +230,15 @@ public abstract class InputMethodManagerInternal { @Override public void updateImeWindowStatus(boolean disableImeIcon) { } + + @Override + public void onSessionForAccessibilityCreated(int accessibilityConnectionId, + IInputMethodSession session) { + } + + @Override + public void unbindAccessibilityFromCurrentClient(int accessibilityConnectionId) { + } }; /** diff --git a/services/core/java/com/android/server/inputmethod/InputMethodManagerService.java b/services/core/java/com/android/server/inputmethod/InputMethodManagerService.java index ba15a047af4ab..736da2dd27cc0 100644 --- a/services/core/java/com/android/server/inputmethod/InputMethodManagerService.java +++ b/services/core/java/com/android/server/inputmethod/InputMethodManagerService.java @@ -94,6 +94,7 @@ import android.media.AudioManagerInternal; import android.net.Uri; import android.os.Binder; import android.os.Bundle; +import android.os.DeadObjectException; import android.os.Debug; import android.os.Handler; import android.os.IBinder; @@ -120,6 +121,7 @@ import android.util.Pair; import android.util.PrintWriterPrinter; import android.util.Printer; import android.util.Slog; +import android.util.SparseArray; import android.util.proto.ProtoOutputStream; import android.view.IWindowManager; import android.view.InputChannel; @@ -171,6 +173,7 @@ import com.android.internal.view.IInputMethodManager; import com.android.internal.view.IInputMethodSession; import com.android.internal.view.IInputSessionCallback; import com.android.internal.view.InlineSuggestionsRequestInfo; +import com.android.server.AccessibilityManagerInternal; import com.android.server.EventLogTags; import com.android.server.LocalServices; import com.android.server.ServiceThread; @@ -228,7 +231,9 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub private static final int MSG_START_HANDWRITING = 1100; private static final int MSG_UNBIND_CLIENT = 3000; + private static final int MSG_UNBIND_ACCESSIBILITY_SERVICE = 3001; private static final int MSG_BIND_CLIENT = 3010; + private static final int MSG_BIND_ACCESSIBILITY_SERVICE = 3011; private static final int MSG_SET_ACTIVE = 3020; private static final int MSG_SET_INTERACTIVE = 3030; private static final int MSG_REPORT_FULLSCREEN_MODE = 3045; @@ -351,6 +356,33 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub } } + /** + * Record session state for an accessibility service. + */ + private static class AccessibilitySessionState { + final ClientState mClient; + // Id of the accessibility service. + final int mId; + + public IInputMethodSession mSession; + + @Override + public String toString() { + return "AccessibilitySessionState{uid " + mClient.uid + " pid " + mClient.pid + + " id " + Integer.toHexString(mId) + + " session " + Integer.toHexString( + System.identityHashCode(mSession)) + + "}"; + } + + AccessibilitySessionState(ClientState client, int id, + IInputMethodSession session) { + mClient = client; + mId = id; + mSession = session; + } + } + private static final class ClientDeathRecipient implements IBinder.DeathRecipient { private final InputMethodManagerService mImms; private final IInputMethodClient mClient; @@ -376,7 +408,9 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub final ClientDeathRecipient clientDeathRecipient; boolean sessionRequested; + boolean mSessionRequestedForAccessibility; SessionState curSession; + SparseArray mAccessibilitySessions = new SparseArray<>(); @Override public String toString() { @@ -637,10 +671,16 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub boolean mBoundToMethod; /** + * Have we called bindInput() for accessibility services? + */ + boolean mBoundToAccessibility; + + /** * Currently enabled session. */ @GuardedBy("ImfLock.class") SessionState mEnabledSession; + SparseArray mEnabledAccessibilitySessions = new SparseArray<>(); /** * True if the device is currently interactive with user. The value is true initially. @@ -2188,6 +2228,7 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub if (cs != null) { client.asBinder().unlinkToDeath(cs.clientDeathRecipient, 0); clearClientSessionLocked(cs); + clearClientSessionForAccessibilityLocked(cs); if (mCurClient == cs) { hideCurrentInputLocked( mCurFocusedWindow, 0, null, SoftInputShowHideReason.HIDE_REMOVE_CLIENT); @@ -2195,9 +2236,13 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub mBoundToMethod = false; IInputMethodInvoker curMethod = getCurMethodLocked(); if (curMethod != null) { + // When we unbind input, we are unbinding the client, so we always + // unbind ime and a11y together. curMethod.unbindInput(); + AccessibilityManagerInternal.get().unbindInput(); } } + mBoundToAccessibility = false; mCurClient = null; } if (mCurFocusedWindowClient == cs) { @@ -2215,6 +2260,24 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub return mHandler.obtainMessage(what, 0, 0, args); } + @NonNull + private Message obtainMessageOOO(int what, Object arg1, Object arg2, Object arg3) { + SomeArgs args = SomeArgs.obtain(); + args.arg1 = arg1; + args.arg2 = arg2; + args.arg3 = arg3; + return mHandler.obtainMessage(what, 0, 0, args); + } + + @NonNull + private Message obtainMessageIIOO(int what, int arg1, int arg2, + Object arg3, Object arg4) { + SomeArgs args = SomeArgs.obtain(); + args.arg1 = arg3; + args.arg2 = arg4; + return mHandler.obtainMessage(what, arg1, arg2, args); + } + @NonNull private Message obtainMessageIIIO(int what, int argi1, int argi2, int argi3, Object arg1) { final SomeArgs args = SomeArgs.obtain(); @@ -2252,13 +2315,18 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub curMethod.unbindInput(); } } + mBoundToAccessibility = false; + // Since we set active false to current client and set mCurClient to null, let's unbind + // all accessibility too. That means, when input method get disconnected (including + // switching ime), we also unbind accessibility scheduleSetActiveToClient(mCurClient, false /* active */, false /* fullscreen */, false /* reportToImeController */); executeOrSendMessage(mCurClient.client, mHandler.obtainMessage( MSG_UNBIND_CLIENT, getSequenceNumberLocked(), unbindClientReason, mCurClient.client)); mCurClient.sessionRequested = false; + mCurClient.mSessionRequestedForAccessibility = false; mCurClient = null; mMenuController.hideInputMethodMenuLocked(); @@ -2345,6 +2413,38 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub curId, getSequenceNumberLocked(), suppressesSpellChecker); } + @GuardedBy("ImfLock.class") + @Nullable + InputBindResult attachNewAccessibilityLocked(@StartInputReason int startInputReason, + boolean initial, int id) { + if (!mBoundToAccessibility) { + AccessibilityManagerInternal.get().bindInput(mCurClient.binding); + mBoundToAccessibility = true; + } + + // TODO(b/187453053): grantImplicitAccess to accessibility services access? if so, need to + // record accessibility services uid. + + final AccessibilitySessionState accessibilitySession = + mCurClient.mAccessibilitySessions.get(id); + // We don't start input when session for a11y is created. We start input when + // input method start input, a11y manager service is always on. + if (startInputReason != StartInputReason.SESSION_CREATED_BY_ACCESSIBILITY) { + final Binder startInputToken = new Binder(); + setEnabledSessionForAccessibilityLocked(mCurClient.mAccessibilitySessions); + AccessibilityManagerInternal.get().startInput(startInputToken, mCurInputContext, + mCurAttribute, !initial /* restarting */); + } + + if (accessibilitySession != null) { + return new InputBindResult( + InputBindResult.ResultCode.SUCCESS_WITH_ACCESSIBILITY_SESSION, + accessibilitySession.mSession, null, + getCurIdLocked(), getSequenceNumberLocked(), false); + } + return null; + } + /** * Called by {@link #startInputOrWindowGainedFocusInternalLocked} to bind/unbind/attach the * selected InputMethod to the given focused IME client. @@ -2419,9 +2519,20 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub // We expect the caller has already verified that the client is allowed to access this // display ID. if (isSelectedMethodBoundLocked()) { + // TODO(b/187453053): this doesn't mean a11y sessions are there. When a11y service is + // enabled while this client is switched out, this client doesn't have the session. We + // need to remove disabled sessions and add new sessions and pass them to imm through + // the input result. if (cs.curSession != null) { // Fast case: if we are already connected to the input method, // then just return it. + // we can always attach to accessibility because AccessibilityManagerService is + // always on. + // This is a workaround to the method describe above + cs.mSessionRequestedForAccessibility = false; + requestClientSessionForAccessibilityLocked(cs); + attachNewAccessibilityLocked(startInputReason, + (startInputFlags & StartInputFlags.INITIAL_CONNECTION) != 0, -1); return attachNewInputLocked(startInputReason, (startInputFlags & StartInputFlags.INITIAL_CONNECTION) != 0); } @@ -2464,6 +2575,7 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub // Return to client, and we will get back with it when // we have had a session made for it. requestClientSessionLocked(cs); + requestClientSessionForAccessibilityLocked(cs); return new InputBindResult( InputBindResult.ResultCode.SUCCESS_WAITING_IME_SESSION, null, null, getCurIdLocked(), getSequenceNumberLocked(), false); @@ -2565,6 +2677,8 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub method, session, channel); InputBindResult res = attachNewInputLocked( StartInputReason.SESSION_CREATED_BY_IME, true); + attachNewAccessibilityLocked(StartInputReason.SESSION_CREATED_BY_IME, + true, -1); if (res.method != null) { executeOrSendMessage(mCurClient.client, obtainMessageOO( MSG_BIND_CLIENT, mCurClient.client, res)); @@ -2602,7 +2716,9 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub void reRequestCurrentClientSessionLocked() { if (mCurClient != null) { clearClientSessionLocked(mCurClient); + clearClientSessionForAccessibilityLocked(mCurClient); requestClientSessionLocked(mCurClient); + requestClientSessionForAccessibilityLocked(mCurClient); } } @@ -2645,6 +2761,15 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub } } + @GuardedBy("ImfLock.class") + void requestClientSessionForAccessibilityLocked(ClientState cs) { + if (!cs.mSessionRequestedForAccessibility) { + if (DEBUG) Slog.v(TAG, "Creating new accessibility sessions for client " + cs); + cs.mSessionRequestedForAccessibility = true; + AccessibilityManagerInternal.get().createImeSession(); + } + } + @GuardedBy("ImfLock.class") void clearClientSessionLocked(ClientState cs) { finishSessionLocked(cs.curSession); @@ -2652,6 +2777,24 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub cs.sessionRequested = false; } + @GuardedBy("ImfLock.class") + void clearClientSessionForAccessibilityLocked(ClientState cs) { + for (int i = 0; i < cs.mAccessibilitySessions.size(); i++) { + finishSessionForAccessibilityLocked(cs.mAccessibilitySessions.valueAt(i)); + } + cs.mAccessibilitySessions.clear(); + cs.mSessionRequestedForAccessibility = false; + } + + @GuardedBy("ImfLock.class") + void clearClientSessionForAccessibilityLocked(ClientState cs, int id) { + AccessibilitySessionState session = cs.mAccessibilitySessions.get(id); + if (session != null) { + finishSessionForAccessibilityLocked(session); + cs.mAccessibilitySessions.remove(id); + } + } + @GuardedBy("ImfLock.class") private void finishSessionLocked(SessionState sessionState) { if (sessionState != null) { @@ -2671,16 +2814,35 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub } } + @GuardedBy("ImfLock.class") + private void finishSessionForAccessibilityLocked(AccessibilitySessionState sessionState) { + if (sessionState != null) { + if (sessionState.mSession != null) { + try { + sessionState.mSession.finishSession(); + } catch (RemoteException e) { + Slog.w(TAG, "Session failed to close due to remote exception", e); + } + sessionState.mSession = null; + } + } + } + @GuardedBy("ImfLock.class") void clearClientSessionsLocked() { if (getCurMethodLocked() != null) { final int numClients = mClients.size(); for (int i = 0; i < numClients; ++i) { clearClientSessionLocked(mClients.valueAt(i)); + clearClientSessionForAccessibilityLocked(mClients.valueAt(i)); } finishSessionLocked(mEnabledSession); + for (int i = 0; i < mEnabledAccessibilitySessions.size(); i++) { + finishSessionForAccessibilityLocked(mEnabledAccessibilitySessions.valueAt(i)); + } mEnabledSession = null; + mEnabledAccessibilitySessions.clear(); scheduleNotifyImeUidToAudioService(Process.INVALID_UID); } hideStatusBarIconLocked(); @@ -4250,6 +4412,41 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub } } + @GuardedBy("ImfLock.class") + void setEnabledSessionForAccessibilityLocked( + SparseArray accessibilitySessions) { + // mEnabledAccessibilitySessions could the same object as accessibilitySessions. + SparseArray disabledSessions = new SparseArray<>(); + for (int i = 0; i < mEnabledAccessibilitySessions.size(); i++) { + if (!accessibilitySessions.contains(mEnabledAccessibilitySessions.keyAt(i))) { + AccessibilitySessionState sessionState = mEnabledAccessibilitySessions.valueAt(i); + if (sessionState != null) { + disabledSessions.append(mEnabledAccessibilitySessions.keyAt(i), + sessionState.mSession); + } + } + } + if (disabledSessions.size() > 0) { + AccessibilityManagerInternal.get().setImeSessionEnabled(disabledSessions, + false); + } + SparseArray enabledSessions = new SparseArray<>(); + for (int i = 0; i < accessibilitySessions.size(); i++) { + if (!mEnabledAccessibilitySessions.contains(accessibilitySessions.keyAt(i))) { + AccessibilitySessionState sessionState = accessibilitySessions.valueAt(i); + if (sessionState != null) { + enabledSessions.append(accessibilitySessions.keyAt(i), sessionState.mSession); + } + } + } + if (enabledSessions.size() > 0) { + AccessibilityManagerInternal.get().setImeSessionEnabled(enabledSessions, + true); + } + mEnabledAccessibilitySessions = accessibilitySessions; + } + + @SuppressWarnings("unchecked") @UiThread @Override public boolean handleMessage(Message msg) { @@ -4319,13 +4516,34 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub // --------------------------------------------------------- - case MSG_UNBIND_CLIENT: + case MSG_UNBIND_CLIENT: { try { - ((IInputMethodClient)msg.obj).onUnbindMethod(msg.arg1, msg.arg2); + // This unbinds all accessibility services too. + ((IInputMethodClient) msg.obj).onUnbindMethod(msg.arg1, msg.arg2); } catch (RemoteException e) { // There is nothing interesting about the last client dying. + if (!(e instanceof DeadObjectException)) { + Slog.w(TAG, "RemoteException when unbinding input method service or" + + "accessibility services"); + } } return true; + } + case MSG_UNBIND_ACCESSIBILITY_SERVICE: { + args = (SomeArgs) msg.obj; + IInputMethodClient client = (IInputMethodClient) args.arg1; + int id = (int) args.arg2; + try { + client.onUnbindAccessibilityService(msg.arg1, id); + } catch (RemoteException e) { + // There is nothing interesting about the last client dying. + if (!(e instanceof DeadObjectException)) { + Slog.w(TAG, "RemoteException when unbinding accessibility services"); + } + } + args.recycle(); + return true; + } case MSG_BIND_CLIENT: { args = (SomeArgs)msg.obj; IInputMethodClient client = (IInputMethodClient)args.arg1; @@ -4344,6 +4562,25 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub args.recycle(); return true; } + case MSG_BIND_ACCESSIBILITY_SERVICE: { + args = (SomeArgs) msg.obj; + IInputMethodClient client = (IInputMethodClient) args.arg1; + InputBindResult res = (InputBindResult) args.arg2; + int id = (int) args.arg3; + try { + client.onBindAccessibilityService(res, id); + } catch (RemoteException e) { + Slog.w(TAG, "Client died receiving input method " + args.arg2); + } finally { + // Dispose the channel if the accessibility service is not local to this process + // because the remote proxy will get its own copy when unparceled. + if (res.channel != null && Binder.isProxy(client)) { + res.channel.dispose(); + } + } + args.recycle(); + return true; + } case MSG_SET_ACTIVE: { args = (SomeArgs) msg.obj; final ClientState clientState = (ClientState) args.arg1; @@ -5025,6 +5262,46 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub mHandler.obtainMessage(MSG_UPDATE_IME_WINDOW_STATUS, disableImeIcon ? 1 : 0, 0) .sendToTarget(); } + + @Override + public void onSessionForAccessibilityCreated(int accessibilityConnectionId, + IInputMethodSession session) { + synchronized (ImfLock.class) { + if (mCurClient != null) { + clearClientSessionForAccessibilityLocked(mCurClient, accessibilityConnectionId); + mCurClient.mAccessibilitySessions.put(accessibilityConnectionId, + new AccessibilitySessionState(mCurClient, accessibilityConnectionId, + session)); + InputBindResult res = attachNewAccessibilityLocked( + StartInputReason.SESSION_CREATED_BY_ACCESSIBILITY, true, + accessibilityConnectionId); + if ((res != null) && (res.method != null)) { + executeOrSendMessage(mCurClient.client, obtainMessageOOO( + MSG_BIND_ACCESSIBILITY_SERVICE, mCurClient.client, res, + accessibilityConnectionId)); + } + } + } + } + + @Override + public void unbindAccessibilityFromCurrentClient(int accessibilityConnectionId) { + synchronized (ImfLock.class) { + if (mCurClient != null) { + if (DEBUG) { + Slog.v(TAG, "unbindAccessibilityFromCurrentClientLocked: client=" + + mCurClient.client.asBinder()); + } + // A11yManagerService unbinds the disabled accessibility service. We don't need + // to do it here. + @UnbindReason int unbindClientReason = + UnbindReason.ACCESSIBILITY_SERVICE_DISABLED; + executeOrSendMessage(mCurClient.client, obtainMessageIIOO( + MSG_UNBIND_ACCESSIBILITY_SERVICE, getSequenceNumberLocked(), + unbindClientReason, mCurClient.client, accessibilityConnectionId)); + } + } + } } @BinderThread @@ -5184,6 +5461,8 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub p.println(" client=" + ci.client); p.println(" inputContext=" + ci.inputContext); p.println(" sessionRequested=" + ci.sessionRequested); + p.println(" sessionRequestedForAccessibility=" + + ci.mSessionRequestedForAccessibility); p.println(" curSession=" + ci.curSession); } p.println(" mCurMethodId=" + getSelectedMethodIdLocked()); From 8cb76d59ae3ddfe27f9ad3ba6a840e6255b75cd0 Mon Sep 17 00:00:00 2001 From: yingleiw Date: Mon, 24 Jan 2022 18:37:54 -0800 Subject: [PATCH 2/3] Add FLAG_INPUT_METHOD_EDITOR to a11y An accessibility service needs to set FLAG_INPUT_METHOD_EDITOR accessibilityFlags to use ime apis. Capability is not added per Phil's suggestion. Bug: 187453053 Test: CTS test added. Change-Id: I78703c48e343ba09c2a0bf61c3447eea7337cb49 --- core/api/current.txt | 1 + .../AccessibilityService.java | 18 ++++++---- .../AccessibilityServiceInfo.java | 12 +++++++ .../accessibilityservice/InputMethod.java | 2 +- core/res/res/values/attrs.xml | 2 ++ ...bstractAccessibilityServiceConnection.java | 5 +++ .../AccessibilityManagerService.java | 34 +++++++------------ .../AccessibilityServiceConnection.java | 18 +++++++--- 8 files changed, 59 insertions(+), 33 deletions(-) diff --git a/core/api/current.txt b/core/api/current.txt index b30ca65aae1e8..c506e50fa74eb 100644 --- a/core/api/current.txt +++ b/core/api/current.txt @@ -3259,6 +3259,7 @@ package android.accessibilityservice { field public static final int FEEDBACK_VISUAL = 8; // 0x8 field public static final int FLAG_ENABLE_ACCESSIBILITY_VOLUME = 128; // 0x80 field public static final int FLAG_INCLUDE_NOT_IMPORTANT_VIEWS = 2; // 0x2 + field public static final int FLAG_INPUT_METHOD_EDITOR = 32768; // 0x8000 field public static final int FLAG_REPORT_VIEW_IDS = 16; // 0x10 field public static final int FLAG_REQUEST_2_FINGER_PASSTHROUGH = 8192; // 0x2000 field public static final int FLAG_REQUEST_ACCESSIBILITY_BUTTON = 256; // 0x100 diff --git a/core/java/android/accessibilityservice/AccessibilityService.java b/core/java/android/accessibilityservice/AccessibilityService.java index 621d9282784d0..cf3ca20315d8d 100644 --- a/core/java/android/accessibilityservice/AccessibilityService.java +++ b/core/java/android/accessibilityservice/AccessibilityService.java @@ -825,10 +825,16 @@ public abstract class AccessibilityService extends Service { for (int i = 0; i < mMagnificationControllers.size(); i++) { mMagnificationControllers.valueAt(i).onServiceConnectedLocked(); } - // TODO(b/187453053): If service requested ime capabilities - if (!mInputMethodInitialized) { - mInputMethod = onCreateInputMethod(); - mInputMethodInitialized = true; + AccessibilityServiceInfo info = getServiceInfo(); + if (info != null) { + boolean requestIme = (info.flags + & AccessibilityServiceInfo.FLAG_INPUT_METHOD_EDITOR) != 0; + if (requestIme && !mInputMethodInitialized) { + mInputMethod = onCreateInputMethod(); + mInputMethodInitialized = true; + } + } else { + Log.e(LOG_TAG, "AccessibilityServiceInfo is null in dispatchServiceConnected"); } } if (mSoftKeyboardController != null) { @@ -1885,7 +1891,7 @@ public abstract class AccessibilityService extends Service { /** * The default implementation returns our default {@link InputMethod}. Subclasses can override * it to provide their own customized version. Accessibility services need to set the - * {@link AccessibilityServiceInfo#FLAG_REQUEST_IME_APIS} flag to use input method APIs. + * {@link AccessibilityServiceInfo#FLAG_INPUT_METHOD_EDITOR} flag to use input method APIs. * * @return the InputMethod. */ @@ -1898,7 +1904,7 @@ public abstract class AccessibilityService extends Service { * Returns the InputMethod instance after the system calls {@link #onCreateInputMethod()}, * which may be used to input text or get editable text selection change notifications. It will * return null if the accessibility service doesn't set the - * {@link AccessibilityServiceInfo#FLAG_REQUEST_IME_APIS} flag or the system doesn't call + * {@link AccessibilityServiceInfo#FLAG_INPUT_METHOD_EDITOR} flag or the system doesn't call * {@link #onCreateInputMethod()}. * * @return the InputMethod instance diff --git a/core/java/android/accessibilityservice/AccessibilityServiceInfo.java b/core/java/android/accessibilityservice/AccessibilityServiceInfo.java index 1167d0b1034f8..f945367fb2dba 100644 --- a/core/java/android/accessibilityservice/AccessibilityServiceInfo.java +++ b/core/java/android/accessibilityservice/AccessibilityServiceInfo.java @@ -390,6 +390,15 @@ public class AccessibilityServiceInfo implements Parcelable { */ public static final int FLAG_SEND_MOTION_EVENTS = 0x0004000; + /** + * This flag makes the AccessibilityService an input method editor with a subset of input + * method editor capabilities: get the {@link android.view.inputmethod.InputConnection} and get + * text selection change notifications. + * + * @see AccessibilityService#getInputMethod() + */ + public static final int FLAG_INPUT_METHOD_EDITOR = 0x0008000; + /** {@hide} */ public static final int FLAG_FORCE_DIRECT_BOOT_AWARE = 0x00010000; @@ -497,6 +506,7 @@ public class AccessibilityServiceInfo implements Parcelable { * @see #FLAG_ENABLE_ACCESSIBILITY_VOLUME * @see #FLAG_REQUEST_ACCESSIBILITY_BUTTON * @see #FLAG_REQUEST_SHORTCUT_WARNING_DIALOG_SPOKEN_FEEDBACK + * @see #FLAG_INPUT_METHOD_EDITOR */ public int flags; @@ -1332,6 +1342,8 @@ public class AccessibilityServiceInfo implements Parcelable { return "FLAG_REQUEST_FINGERPRINT_GESTURES"; case FLAG_REQUEST_SHORTCUT_WARNING_DIALOG_SPOKEN_FEEDBACK: return "FLAG_REQUEST_SHORTCUT_WARNING_DIALOG_SPOKEN_FEEDBACK"; + case FLAG_INPUT_METHOD_EDITOR: + return "FLAG_INPUT_METHOD_EDITOR"; default: return null; } diff --git a/core/java/android/accessibilityservice/InputMethod.java b/core/java/android/accessibilityservice/InputMethod.java index 1684bea274a90..001d804b22d6a 100644 --- a/core/java/android/accessibilityservice/InputMethod.java +++ b/core/java/android/accessibilityservice/InputMethod.java @@ -55,7 +55,7 @@ import java.util.concurrent.Executor; * developers should override them as needed. Developers should also override * {@link AccessibilityService#onCreateInputMethod()} to return * their custom InputMethod implementation. Accessibility services also need to set the - * {@link AccessibilityServiceInfo#FLAG_REQUEST_IME_APIS} flag to use input method APIs. + * {@link AccessibilityServiceInfo#FLAG_INPUT_METHOD_EDITOR} flag to use input method APIs. */ public class InputMethod { private static final String LOG_TAG = "A11yInputMethod"; diff --git a/core/res/res/values/attrs.xml b/core/res/res/values/attrs.xml index d774fd4e397a7..7d12d5947f7e4 100644 --- a/core/res/res/values/attrs.xml +++ b/core/res/res/values/attrs.xml @@ -3910,6 +3910,8 @@ + + diff --git a/services/accessibility/java/com/android/server/accessibility/AbstractAccessibilityServiceConnection.java b/services/accessibility/java/com/android/server/accessibility/AbstractAccessibilityServiceConnection.java index 0f65ebca9b3ac..7f103144b7fb7 100644 --- a/services/accessibility/java/com/android/server/accessibility/AbstractAccessibilityServiceConnection.java +++ b/services/accessibility/java/com/android/server/accessibility/AbstractAccessibilityServiceConnection.java @@ -188,6 +188,8 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ boolean mLastAccessibilityButtonCallbackState; + boolean mRequestImeApis; + int mFetchFlags; long mNotificationTimeout; @@ -385,6 +387,9 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ & AccessibilityServiceInfo.FLAG_REQUEST_FINGERPRINT_GESTURES) != 0; mRequestAccessibilityButton = (info.flags & AccessibilityServiceInfo.FLAG_REQUEST_ACCESSIBILITY_BUTTON) != 0; + // TODO(b/218193835): request ime when ime flag is set and clean up when ime flag is unset + mRequestImeApis = (info.flags + & AccessibilityServiceInfo.FLAG_INPUT_METHOD_EDITOR) != 0; } protected boolean supportsFlagForNotImportantViews(AccessibilityServiceInfo info) { diff --git a/services/accessibility/java/com/android/server/accessibility/AccessibilityManagerService.java b/services/accessibility/java/com/android/server/accessibility/AccessibilityManagerService.java index 2f049f7edc1fb..48d6229a1dfbb 100644 --- a/services/accessibility/java/com/android/server/accessibility/AccessibilityManagerService.java +++ b/services/accessibility/java/com/android/server/accessibility/AccessibilityManagerService.java @@ -4372,10 +4372,9 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub userState = getCurrentUserStateLocked(); for (int i = userState.mBoundServices.size() - 1; i >= 0; i--) { final AccessibilityServiceConnection service = userState.mBoundServices.get(i); - // TODO(b/187453053): mRequestedIme implementation - //if (service.mRequestedIme) { - service.bindInputLocked(binding); - //} + if (service.requestImeApis()) { + service.bindInputLocked(binding); + } } } } @@ -4390,10 +4389,9 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub userState = getCurrentUserStateLocked(); for (int i = userState.mBoundServices.size() - 1; i >= 0; i--) { final AccessibilityServiceConnection service = userState.mBoundServices.get(i); - // TODO(187453053): mRequestedIme implementation - //if (service.mRequestedIme) { - service.unbindInputLocked(); - //} + if (service.requestImeApis()) { + service.unbindInputLocked(); + } } } } @@ -4403,7 +4401,6 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub */ public void startInput(IBinder startInputToken, IInputContext inputContext, EditorInfo editorInfo, boolean restarting) { - //TODO(b/187453053): including the java doc AccessibilityUserState userState; synchronized (mLock) { // Keep records of these in case new Accessibility Services are enabled. @@ -4414,10 +4411,9 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub userState = getCurrentUserStateLocked(); for (int i = userState.mBoundServices.size() - 1; i >= 0; i--) { final AccessibilityServiceConnection service = userState.mBoundServices.get(i); - // TODO(b/187453053): mRequestedIme implementation - //if (service.mRequestedIme) { - service.startInputLocked(startInputToken, inputContext, editorInfo, restarting); - //} + if (service.requestImeApis()) { + service.startInputLocked(startInputToken, inputContext, editorInfo, restarting); + } } } } @@ -4432,10 +4428,9 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub userState = getCurrentUserStateLocked(); for (int i = userState.mBoundServices.size() - 1; i >= 0; i--) { final AccessibilityServiceConnection service = userState.mBoundServices.get(i); - // TODO(b/187453053): mRequestedIme implementation - //if (service.mRequestedIme) { - service.createImeSessionLocked(); - //} + if (service.requestImeApis()) { + service.createImeSessionLocked(); + } } } } @@ -4452,11 +4447,8 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub userState = getCurrentUserStateLocked(); for (int i = userState.mBoundServices.size() - 1; i >= 0; i--) { final AccessibilityServiceConnection service = userState.mBoundServices.get(i); - // TODO(b/187453053): mRequestedIme implementation - if (sessions.contains(service.mId)) { - //if (service.mRequestedIme) { + if (sessions.contains(service.mId) && service.requestImeApis()) { service.setImeSessionEnabledLocked(sessions.get(service.mId), enabled); - //} } } } diff --git a/services/accessibility/java/com/android/server/accessibility/AccessibilityServiceConnection.java b/services/accessibility/java/com/android/server/accessibility/AccessibilityServiceConnection.java index e06ee77974ec7..06310284b56c2 100644 --- a/services/accessibility/java/com/android/server/accessibility/AccessibilityServiceConnection.java +++ b/services/accessibility/java/com/android/server/accessibility/AccessibilityServiceConnection.java @@ -127,8 +127,9 @@ class AccessibilityServiceConnection extends AbstractAccessibilityServiceConnect } public void unbindLocked() { - // If requested ime - mSystemSupport.unbindImeLocked(this); + if (requestImeApis()) { + mSystemSupport.unbindImeLocked(this); + } mContext.unbindService(this); AccessibilityUserState userState = mUserStateWeakReference.get(); if (userState == null) return; @@ -190,8 +191,9 @@ class AccessibilityServiceConnection extends AbstractAccessibilityServiceConnect // the new configuration (for example, initializing the input filter). mMainHandler.sendMessage(obtainMessage( AccessibilityServiceConnection::initializeService, this)); - //if (service.mRequestedIme) { - mSystemSupport.requestImeLocked(this); + if (requestImeApis()) { + mSystemSupport.requestImeLocked(this); + } } } @@ -375,7 +377,9 @@ class AccessibilityServiceConnection extends AbstractAccessibilityServiceConnect if (!isConnectedLocked()) { return; } - mSystemSupport.unbindImeLocked(this); + if (requestImeApis()) { + mSystemSupport.unbindImeLocked(this); + } mAccessibilityServiceInfo.crashed = true; AccessibilityUserState userState = mUserStateWeakReference.get(); if (userState != null) { @@ -517,6 +521,10 @@ class AccessibilityServiceConnection extends AbstractAccessibilityServiceConnect mMainHandler.sendMessage(msg); } + public boolean requestImeApis() { + return mRequestImeApis; + } + private void notifyMotionEventInternal(MotionEvent event) { final IAccessibilityServiceClient listener = getServiceInterfaceSafely(); if (listener != null) { From 107b9413a6a252ce8859850c1f5dcdd57aa403bc Mon Sep 17 00:00:00 2001 From: yingleiw Date: Wed, 26 Jan 2022 21:07:27 -0800 Subject: [PATCH 3/3] Add accessibilitySessions to InputBindResult This way, we can pass back a11y sessions from direct startInput() return value. And a11yManagerService will only request sessions from a11y services which the client doesn't have sessions yet (will not request existing a11y sessions again). Bug: 187453053 Test: tested manually with talkback. Tested client switching within "recent apps". Change-Id: I8efdc9886ce33185a2195b741668c12e319ea660 --- .../view/inputmethod/InputMethodManager.java | 53 +++++++------ .../internal/inputmethod/InputBindResult.java | 39 +++++++++- .../AccessibilityManagerService.java | 11 +-- .../server/AccessibilityManagerInternal.java | 10 ++- .../InputMethodBindingController.java | 2 +- .../InputMethodManagerService.java | 75 ++++++++++++++----- 6 files changed, 139 insertions(+), 51 deletions(-) diff --git a/core/java/android/view/inputmethod/InputMethodManager.java b/core/java/android/view/inputmethod/InputMethodManager.java index 94b2215b30a8c..c713a54fdf7d3 100644 --- a/core/java/android/view/inputmethod/InputMethodManager.java +++ b/core/java/android/view/inputmethod/InputMethodManager.java @@ -930,23 +930,26 @@ public final class InputMethodManager { // Since IMM can start inputting text before a11y sessions are back, // we send a notification so that the a11y service knows the session is // registered and update the a11y service with the current cursor positions. - InputMethodSessionWrapper wrapper = - InputMethodSessionWrapper.createOrNull(res.method); - if (wrapper != null) { - mAccessibilityInputMethodSession.put(id, wrapper); - if (mServedInputConnection != null) { - wrapper.updateSelection(mInitialSelStart, mInitialSelEnd, - mCursorSelStart, mCursorSelEnd, mCursorCandStart, - mCursorCandEnd); - } else { - // If an a11y service binds before input starts, we should still - // send a notification because the a11y service doesn't know it - // binds before or after input starts, it may wonder if it binds - // after input starts, why it doesn't receive a notification of - // the current cursor positions. - wrapper.updateSelection(-1, -1, - -1, -1, -1, - -1); + if (res.accessibilitySessions != null) { + InputMethodSessionWrapper wrapper = + InputMethodSessionWrapper.createOrNull( + res.accessibilitySessions.get(id)); + if (wrapper != null) { + mAccessibilityInputMethodSession.put(id, wrapper); + if (mServedInputConnection != null) { + wrapper.updateSelection(mInitialSelStart, mInitialSelEnd, + mCursorSelStart, mCursorSelEnd, mCursorCandStart, + mCursorCandEnd); + } else { + // If an a11y service binds before input starts, we should still + // send a notification because the a11y service doesn't know it + // binds before or after input starts, it may wonder if it binds + // after input starts, why it doesn't receive a notification of + // the current cursor positions. + wrapper.updateSelection(-1, -1, + -1, -1, -1, + -1); + } } } mBindSequence = res.sequence; @@ -1508,6 +1511,7 @@ public final class InputMethodManager { /** * Reset all of the state associated with being bound to an input method. */ + @GuardedBy("mH") void clearBindingLocked() { if (DEBUG) Log.v(TAG, "Clearing binding!"); clearConnectionLocked(); @@ -2235,14 +2239,21 @@ public final class InputMethodManager { } mIsInputMethodSuppressingSpellChecker = res.isInputMethodSuppressingSpellChecker; if (res.id != null) { - // we might need to put a11y sessions and channels into res and restore them here. - // Currently we have a workaround to request a11y session after each client - // switching, even when the new client is opened before and is in memory (has - // existing a11y sessions). setInputChannelLocked(res.channel); mBindSequence = res.sequence; mCurMethod = res.method; // for @UnsupportedAppUsage mCurrentInputMethodSession = InputMethodSessionWrapper.createOrNull(res.method); + mAccessibilityInputMethodSession.clear(); + if (res.accessibilitySessions != null) { + for (int i = 0; i < res.accessibilitySessions.size(); i++) { + InputMethodSessionWrapper wrapper = InputMethodSessionWrapper.createOrNull( + res.accessibilitySessions.valueAt(i)); + if (wrapper != null) { + mAccessibilityInputMethodSession.append( + res.accessibilitySessions.keyAt(i), wrapper); + } + } + } mCurId = res.id; } else if (res.channel != null && res.channel != mCurChannel) { res.channel.dispose(); diff --git a/core/java/com/android/internal/inputmethod/InputBindResult.java b/core/java/com/android/internal/inputmethod/InputBindResult.java index 1bc46f61429eb..e83840177a733 100644 --- a/core/java/com/android/internal/inputmethod/InputBindResult.java +++ b/core/java/com/android/internal/inputmethod/InputBindResult.java @@ -25,6 +25,7 @@ import android.content.ServiceConnection; import android.os.IBinder; import android.os.Parcel; import android.os.Parcelable; +import android.util.SparseArray; import android.view.InputChannel; import com.android.internal.view.IInputMethodSession; @@ -180,6 +181,11 @@ public final class InputBindResult implements Parcelable { */ public final IInputMethodSession method; + /** + * The accessibility services. + */ + public SparseArray accessibilitySessions; + /** * The input channel used to send input events to this IME. */ @@ -206,6 +212,8 @@ public final class InputBindResult implements Parcelable { * * @param result A result code defined in {@link ResultCode}. * @param method {@link IInputMethodSession} to interact with the IME. + * @param accessibilitySessions {@link IInputMethodSession} to interact with accessibility + * services. * @param channel {@link InputChannel} to forward input events to the IME. * @param id The {@link String} representations of the IME, which is the same as * {@link android.view.inputmethod.InputMethodInfo#getId()} and @@ -215,10 +223,12 @@ public final class InputBindResult implements Parcelable { * {@code suppressesSpellChecker="true"}. */ public InputBindResult(@ResultCode int result, - IInputMethodSession method, InputChannel channel, String id, int sequence, + IInputMethodSession method, SparseArray accessibilitySessions, + InputChannel channel, String id, int sequence, boolean isInputMethodSuppressingSpellChecker) { this.result = result; this.method = method; + this.accessibilitySessions = accessibilitySessions; this.channel = channel; this.id = id; this.sequence = sequence; @@ -228,6 +238,19 @@ public final class InputBindResult implements Parcelable { private InputBindResult(Parcel source) { result = source.readInt(); method = IInputMethodSession.Stub.asInterface(source.readStrongBinder()); + int n = source.readInt(); + if (n < 0) { + accessibilitySessions = null; + } else { + accessibilitySessions = new SparseArray<>(n); + while (n > 0) { + int key = source.readInt(); + IInputMethodSession value = + IInputMethodSession.Stub.asInterface(source.readStrongBinder()); + accessibilitySessions.append(key, value); + n--; + } + } if (source.readInt() != 0) { channel = InputChannel.CREATOR.createFromParcel(source); } else { @@ -256,6 +279,18 @@ public final class InputBindResult implements Parcelable { public void writeToParcel(Parcel dest, int flags) { dest.writeInt(result); dest.writeStrongInterface(method); + if (accessibilitySessions == null) { + dest.writeInt(-1); + } else { + int n = accessibilitySessions.size(); + dest.writeInt(n); + int i = 0; + while (i < n) { + dest.writeInt(accessibilitySessions.keyAt(i)); + dest.writeStrongInterface(accessibilitySessions.valueAt(i)); + i++; + } + } if (channel != null) { dest.writeInt(1); channel.writeToParcel(dest, flags); @@ -331,7 +366,7 @@ public final class InputBindResult implements Parcelable { } private static InputBindResult error(@ResultCode int result) { - return new InputBindResult(result, null, null, null, -1, false); + return new InputBindResult(result, null, null, null, null, -1, false); } /** diff --git a/services/accessibility/java/com/android/server/accessibility/AccessibilityManagerService.java b/services/accessibility/java/com/android/server/accessibility/AccessibilityManagerService.java index 48d6229a1dfbb..c4e5b8173c3e1 100644 --- a/services/accessibility/java/com/android/server/accessibility/AccessibilityManagerService.java +++ b/services/accessibility/java/com/android/server/accessibility/AccessibilityManagerService.java @@ -336,8 +336,8 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub } @Override - public void createImeSession() { - mService.createImeSession(); + public void createImeSession(ArraySet ignoreSet) { + mService.createImeSession(ignoreSet); } @Override @@ -4419,16 +4419,17 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub } /** - * Request input sessions from all accessibility services which request ime capabilities. + * Request input sessions from all accessibility services which request ime capabilities and + * whose id is not in the ignoreSet */ - public void createImeSession() { + public void createImeSession(ArraySet ignoreSet) { AccessibilityUserState userState; synchronized (mLock) { mInputSessionRequested = true; userState = getCurrentUserStateLocked(); for (int i = userState.mBoundServices.size() - 1; i >= 0; i--) { final AccessibilityServiceConnection service = userState.mBoundServices.get(i); - if (service.requestImeApis()) { + if ((!ignoreSet.contains(service.mId)) && service.requestImeApis()) { service.createImeSessionLocked(); } } diff --git a/services/core/java/com/android/server/AccessibilityManagerInternal.java b/services/core/java/com/android/server/AccessibilityManagerInternal.java index c02e94d7bc668..28f6db1c800b5 100644 --- a/services/core/java/com/android/server/AccessibilityManagerInternal.java +++ b/services/core/java/com/android/server/AccessibilityManagerInternal.java @@ -18,6 +18,7 @@ package com.android.server; import android.annotation.NonNull; import android.os.IBinder; +import android.util.ArraySet; import android.util.SparseArray; import android.view.inputmethod.EditorInfo; import android.view.inputmethod.InputBinding; @@ -39,8 +40,11 @@ public abstract class AccessibilityManagerInternal { /** Bind input for all accessibility services which require ime capabilities. */ public abstract void bindInput(InputBinding binding); - /** Request input session from all accessibility services which require ime capabilities. */ - public abstract void createImeSession(); + /** + * Request input session from all accessibility services which require ime capabilities and + * whose id is not in the ignoreSet. + */ + public abstract void createImeSession(ArraySet ignoreSet); /** Start input for all accessibility services which require ime capabilities. */ public abstract void startInput(IBinder startInputToken, IInputContext inputContext, @@ -61,7 +65,7 @@ public abstract class AccessibilityManagerInternal { } @Override - public void createImeSession() { + public void createImeSession(ArraySet ignoreSet) { } @Override diff --git a/services/core/java/com/android/server/inputmethod/InputMethodBindingController.java b/services/core/java/com/android/server/inputmethod/InputMethodBindingController.java index b81478285a627..b2f500a59ba96 100644 --- a/services/core/java/com/android/server/inputmethod/InputMethodBindingController.java +++ b/services/core/java/com/android/server/inputmethod/InputMethodBindingController.java @@ -422,7 +422,7 @@ final class InputMethodBindingController { addFreshWindowToken(); return new InputBindResult( InputBindResult.ResultCode.SUCCESS_WAITING_IME_BINDING, - null, null, mCurId, mCurSeq, false); + null, null, null, mCurId, mCurSeq, false); } Slog.w(InputMethodManagerService.TAG, diff --git a/services/core/java/com/android/server/inputmethod/InputMethodManagerService.java b/services/core/java/com/android/server/inputmethod/InputMethodManagerService.java index 736da2dd27cc0..7068ed13376f1 100644 --- a/services/core/java/com/android/server/inputmethod/InputMethodManagerService.java +++ b/services/core/java/com/android/server/inputmethod/InputMethodManagerService.java @@ -2408,8 +2408,11 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub final InputMethodInfo curInputMethodInfo = mMethodMap.get(curId); final boolean suppressesSpellChecker = curInputMethodInfo != null && curInputMethodInfo.suppressesSpellChecker(); + final SparseArray accessibilityInputMethodSessions = + createAccessibilityInputMethodSessions(mCurClient.mAccessibilitySessions); return new InputBindResult(InputBindResult.ResultCode.SUCCESS_WITH_IME_SESSION, - session.session, (session.channel != null ? session.channel.dup() : null), + session.session, accessibilityInputMethodSessions, + (session.channel != null ? session.channel.dup() : null), curId, getSequenceNumberLocked(), suppressesSpellChecker); } @@ -2437,14 +2440,31 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub } if (accessibilitySession != null) { + final SessionState session = mCurClient.curSession; + IInputMethodSession imeSession = session == null ? null : session.session; + final SparseArray accessibilityInputMethodSessions = + createAccessibilityInputMethodSessions(mCurClient.mAccessibilitySessions); return new InputBindResult( InputBindResult.ResultCode.SUCCESS_WITH_ACCESSIBILITY_SESSION, - accessibilitySession.mSession, null, + imeSession, accessibilityInputMethodSessions, null, getCurIdLocked(), getSequenceNumberLocked(), false); } return null; } + private SparseArray createAccessibilityInputMethodSessions( + SparseArray accessibilitySessions) { + final SparseArray accessibilityInputMethodSessions = + new SparseArray<>(); + if (accessibilitySessions != null) { + for (int i = 0; i < accessibilitySessions.size(); i++) { + accessibilityInputMethodSessions.append(accessibilitySessions.keyAt(i), + accessibilitySessions.valueAt(i).mSession); + } + } + return accessibilityInputMethodSessions; + } + /** * Called by {@link #startInputOrWindowGainedFocusInternalLocked} to bind/unbind/attach the * selected InputMethod to the given focused IME client. @@ -2470,7 +2490,7 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub // party code. return new InputBindResult( InputBindResult.ResultCode.ERROR_SYSTEM_NOT_READY, - null, null, selectedMethodId, getSequenceNumberLocked(), false); + null, null, null, selectedMethodId, getSequenceNumberLocked(), false); } if (!InputMethodUtils.checkIfPackageBelongsToUid(mAppOpsManager, cs.uid, @@ -2519,18 +2539,18 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub // We expect the caller has already verified that the client is allowed to access this // display ID. if (isSelectedMethodBoundLocked()) { - // TODO(b/187453053): this doesn't mean a11y sessions are there. When a11y service is - // enabled while this client is switched out, this client doesn't have the session. We - // need to remove disabled sessions and add new sessions and pass them to imm through - // the input result. if (cs.curSession != null) { // Fast case: if we are already connected to the input method, // then just return it. - // we can always attach to accessibility because AccessibilityManagerService is - // always on. - // This is a workaround to the method describe above + // This doesn't mean a11y sessions are there. When a11y service is + // enabled while this client is switched out, this client doesn't have the session. + // A11yManagerService will only request missing sessions (will not request existing + // sessions again). Note when an a11y service is disabled, it will clear its + // session from all clients, so we don't need to worry about disabled a11y services. cs.mSessionRequestedForAccessibility = false; requestClientSessionForAccessibilityLocked(cs); + // we can always attach to accessibility because AccessibilityManagerService is + // always on. attachNewAccessibilityLocked(startInputReason, (startInputFlags & StartInputFlags.INITIAL_CONNECTION) != 0, -1); return attachNewInputLocked(startInputReason, @@ -2578,7 +2598,7 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub requestClientSessionForAccessibilityLocked(cs); return new InputBindResult( InputBindResult.ResultCode.SUCCESS_WAITING_IME_SESSION, - null, null, getCurIdLocked(), getSequenceNumberLocked(), false); + null, null, null, getCurIdLocked(), getSequenceNumberLocked(), false); } else { long bindingDuration = SystemClock.uptimeMillis() - getLastBindTimeLocked(); if (bindingDuration < TIME_TO_RECONNECT) { @@ -2591,7 +2611,7 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub // to see if we can get back in touch with the service. return new InputBindResult( InputBindResult.ResultCode.SUCCESS_WAITING_IME_BINDING, - null, null, getCurIdLocked(), getSequenceNumberLocked(), false); + null, null, null, getCurIdLocked(), getSequenceNumberLocked(), false); } else { EventLog.writeEvent(EventLogTags.IMF_FORCE_RECONNECT_IME, getSelectedMethodIdLocked(), bindingDuration, 0); @@ -2766,7 +2786,11 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub if (!cs.mSessionRequestedForAccessibility) { if (DEBUG) Slog.v(TAG, "Creating new accessibility sessions for client " + cs); cs.mSessionRequestedForAccessibility = true; - AccessibilityManagerInternal.get().createImeSession(); + ArraySet ignoreSet = new ArraySet<>(); + for (int i = 0; i < cs.mAccessibilitySessions.size(); i++) { + ignoreSet.add(cs.mAccessibilitySessions.keyAt(i)); + } + AccessibilityManagerInternal.get().createImeSession(ignoreSet); } } @@ -3617,7 +3641,7 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub } return new InputBindResult( InputBindResult.ResultCode.SUCCESS_REPORT_WINDOW_FOCUS_ONLY, - null, null, null, -1, false); + null, null, null, null, -1, false); } mCurFocusedWindow = windowToken; @@ -5275,11 +5299,9 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub InputBindResult res = attachNewAccessibilityLocked( StartInputReason.SESSION_CREATED_BY_ACCESSIBILITY, true, accessibilityConnectionId); - if ((res != null) && (res.method != null)) { - executeOrSendMessage(mCurClient.client, obtainMessageOOO( - MSG_BIND_ACCESSIBILITY_SERVICE, mCurClient.client, res, - accessibilityConnectionId)); - } + executeOrSendMessage(mCurClient.client, obtainMessageOOO( + MSG_BIND_ACCESSIBILITY_SERVICE, mCurClient.client, res, + accessibilityConnectionId)); } } } @@ -5300,6 +5322,21 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub MSG_UNBIND_ACCESSIBILITY_SERVICE, getSequenceNumberLocked(), unbindClientReason, mCurClient.client, accessibilityConnectionId)); } + // We only have sessions when we bound to an input method. Remove this session + // from all clients. + if (getCurMethodLocked() != null) { + final int numClients = mClients.size(); + for (int i = 0; i < numClients; ++i) { + clearClientSessionForAccessibilityLocked(mClients.valueAt(i), + accessibilityConnectionId); + } + AccessibilitySessionState session = mEnabledAccessibilitySessions.get( + accessibilityConnectionId); + if (session != null) { + finishSessionForAccessibilityLocked(session); + mEnabledAccessibilitySessions.remove(accessibilityConnectionId); + } + } } } }