From 107b9413a6a252ce8859850c1f5dcdd57aa403bc Mon Sep 17 00:00:00 2001 From: yingleiw Date: Wed, 26 Jan 2022 21:07:27 -0800 Subject: [PATCH] Add accessibilitySessions to InputBindResult This way, we can pass back a11y sessions from direct startInput() return value. And a11yManagerService will only request sessions from a11y services which the client doesn't have sessions yet (will not request existing a11y sessions again). Bug: 187453053 Test: tested manually with talkback. Tested client switching within "recent apps". Change-Id: I8efdc9886ce33185a2195b741668c12e319ea660 --- .../view/inputmethod/InputMethodManager.java | 53 +++++++------ .../internal/inputmethod/InputBindResult.java | 39 +++++++++- .../AccessibilityManagerService.java | 11 +-- .../server/AccessibilityManagerInternal.java | 10 ++- .../InputMethodBindingController.java | 2 +- .../InputMethodManagerService.java | 75 ++++++++++++++----- 6 files changed, 139 insertions(+), 51 deletions(-) diff --git a/core/java/android/view/inputmethod/InputMethodManager.java b/core/java/android/view/inputmethod/InputMethodManager.java index 94b2215b30a8c..c713a54fdf7d3 100644 --- a/core/java/android/view/inputmethod/InputMethodManager.java +++ b/core/java/android/view/inputmethod/InputMethodManager.java @@ -930,23 +930,26 @@ public final class InputMethodManager { // Since IMM can start inputting text before a11y sessions are back, // we send a notification so that the a11y service knows the session is // registered and update the a11y service with the current cursor positions. - InputMethodSessionWrapper wrapper = - InputMethodSessionWrapper.createOrNull(res.method); - if (wrapper != null) { - mAccessibilityInputMethodSession.put(id, wrapper); - if (mServedInputConnection != null) { - wrapper.updateSelection(mInitialSelStart, mInitialSelEnd, - mCursorSelStart, mCursorSelEnd, mCursorCandStart, - mCursorCandEnd); - } else { - // If an a11y service binds before input starts, we should still - // send a notification because the a11y service doesn't know it - // binds before or after input starts, it may wonder if it binds - // after input starts, why it doesn't receive a notification of - // the current cursor positions. - wrapper.updateSelection(-1, -1, - -1, -1, -1, - -1); + if (res.accessibilitySessions != null) { + InputMethodSessionWrapper wrapper = + InputMethodSessionWrapper.createOrNull( + res.accessibilitySessions.get(id)); + if (wrapper != null) { + mAccessibilityInputMethodSession.put(id, wrapper); + if (mServedInputConnection != null) { + wrapper.updateSelection(mInitialSelStart, mInitialSelEnd, + mCursorSelStart, mCursorSelEnd, mCursorCandStart, + mCursorCandEnd); + } else { + // If an a11y service binds before input starts, we should still + // send a notification because the a11y service doesn't know it + // binds before or after input starts, it may wonder if it binds + // after input starts, why it doesn't receive a notification of + // the current cursor positions. + wrapper.updateSelection(-1, -1, + -1, -1, -1, + -1); + } } } mBindSequence = res.sequence; @@ -1508,6 +1511,7 @@ public final class InputMethodManager { /** * Reset all of the state associated with being bound to an input method. */ + @GuardedBy("mH") void clearBindingLocked() { if (DEBUG) Log.v(TAG, "Clearing binding!"); clearConnectionLocked(); @@ -2235,14 +2239,21 @@ public final class InputMethodManager { } mIsInputMethodSuppressingSpellChecker = res.isInputMethodSuppressingSpellChecker; if (res.id != null) { - // we might need to put a11y sessions and channels into res and restore them here. - // Currently we have a workaround to request a11y session after each client - // switching, even when the new client is opened before and is in memory (has - // existing a11y sessions). setInputChannelLocked(res.channel); mBindSequence = res.sequence; mCurMethod = res.method; // for @UnsupportedAppUsage mCurrentInputMethodSession = InputMethodSessionWrapper.createOrNull(res.method); + mAccessibilityInputMethodSession.clear(); + if (res.accessibilitySessions != null) { + for (int i = 0; i < res.accessibilitySessions.size(); i++) { + InputMethodSessionWrapper wrapper = InputMethodSessionWrapper.createOrNull( + res.accessibilitySessions.valueAt(i)); + if (wrapper != null) { + mAccessibilityInputMethodSession.append( + res.accessibilitySessions.keyAt(i), wrapper); + } + } + } mCurId = res.id; } else if (res.channel != null && res.channel != mCurChannel) { res.channel.dispose(); diff --git a/core/java/com/android/internal/inputmethod/InputBindResult.java b/core/java/com/android/internal/inputmethod/InputBindResult.java index 1bc46f61429eb..e83840177a733 100644 --- a/core/java/com/android/internal/inputmethod/InputBindResult.java +++ b/core/java/com/android/internal/inputmethod/InputBindResult.java @@ -25,6 +25,7 @@ import android.content.ServiceConnection; import android.os.IBinder; import android.os.Parcel; import android.os.Parcelable; +import android.util.SparseArray; import android.view.InputChannel; import com.android.internal.view.IInputMethodSession; @@ -180,6 +181,11 @@ public final class InputBindResult implements Parcelable { */ public final IInputMethodSession method; + /** + * The accessibility services. + */ + public SparseArray accessibilitySessions; + /** * The input channel used to send input events to this IME. */ @@ -206,6 +212,8 @@ public final class InputBindResult implements Parcelable { * * @param result A result code defined in {@link ResultCode}. * @param method {@link IInputMethodSession} to interact with the IME. + * @param accessibilitySessions {@link IInputMethodSession} to interact with accessibility + * services. * @param channel {@link InputChannel} to forward input events to the IME. * @param id The {@link String} representations of the IME, which is the same as * {@link android.view.inputmethod.InputMethodInfo#getId()} and @@ -215,10 +223,12 @@ public final class InputBindResult implements Parcelable { * {@code suppressesSpellChecker="true"}. */ public InputBindResult(@ResultCode int result, - IInputMethodSession method, InputChannel channel, String id, int sequence, + IInputMethodSession method, SparseArray accessibilitySessions, + InputChannel channel, String id, int sequence, boolean isInputMethodSuppressingSpellChecker) { this.result = result; this.method = method; + this.accessibilitySessions = accessibilitySessions; this.channel = channel; this.id = id; this.sequence = sequence; @@ -228,6 +238,19 @@ public final class InputBindResult implements Parcelable { private InputBindResult(Parcel source) { result = source.readInt(); method = IInputMethodSession.Stub.asInterface(source.readStrongBinder()); + int n = source.readInt(); + if (n < 0) { + accessibilitySessions = null; + } else { + accessibilitySessions = new SparseArray<>(n); + while (n > 0) { + int key = source.readInt(); + IInputMethodSession value = + IInputMethodSession.Stub.asInterface(source.readStrongBinder()); + accessibilitySessions.append(key, value); + n--; + } + } if (source.readInt() != 0) { channel = InputChannel.CREATOR.createFromParcel(source); } else { @@ -256,6 +279,18 @@ public final class InputBindResult implements Parcelable { public void writeToParcel(Parcel dest, int flags) { dest.writeInt(result); dest.writeStrongInterface(method); + if (accessibilitySessions == null) { + dest.writeInt(-1); + } else { + int n = accessibilitySessions.size(); + dest.writeInt(n); + int i = 0; + while (i < n) { + dest.writeInt(accessibilitySessions.keyAt(i)); + dest.writeStrongInterface(accessibilitySessions.valueAt(i)); + i++; + } + } if (channel != null) { dest.writeInt(1); channel.writeToParcel(dest, flags); @@ -331,7 +366,7 @@ public final class InputBindResult implements Parcelable { } private static InputBindResult error(@ResultCode int result) { - return new InputBindResult(result, null, null, null, -1, false); + return new InputBindResult(result, null, null, null, null, -1, false); } /** diff --git a/services/accessibility/java/com/android/server/accessibility/AccessibilityManagerService.java b/services/accessibility/java/com/android/server/accessibility/AccessibilityManagerService.java index 48d6229a1dfbb..c4e5b8173c3e1 100644 --- a/services/accessibility/java/com/android/server/accessibility/AccessibilityManagerService.java +++ b/services/accessibility/java/com/android/server/accessibility/AccessibilityManagerService.java @@ -336,8 +336,8 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub } @Override - public void createImeSession() { - mService.createImeSession(); + public void createImeSession(ArraySet ignoreSet) { + mService.createImeSession(ignoreSet); } @Override @@ -4419,16 +4419,17 @@ public class AccessibilityManagerService extends IAccessibilityManager.Stub } /** - * Request input sessions from all accessibility services which request ime capabilities. + * Request input sessions from all accessibility services which request ime capabilities and + * whose id is not in the ignoreSet */ - public void createImeSession() { + public void createImeSession(ArraySet ignoreSet) { AccessibilityUserState userState; synchronized (mLock) { mInputSessionRequested = true; userState = getCurrentUserStateLocked(); for (int i = userState.mBoundServices.size() - 1; i >= 0; i--) { final AccessibilityServiceConnection service = userState.mBoundServices.get(i); - if (service.requestImeApis()) { + if ((!ignoreSet.contains(service.mId)) && service.requestImeApis()) { service.createImeSessionLocked(); } } diff --git a/services/core/java/com/android/server/AccessibilityManagerInternal.java b/services/core/java/com/android/server/AccessibilityManagerInternal.java index c02e94d7bc668..28f6db1c800b5 100644 --- a/services/core/java/com/android/server/AccessibilityManagerInternal.java +++ b/services/core/java/com/android/server/AccessibilityManagerInternal.java @@ -18,6 +18,7 @@ package com.android.server; import android.annotation.NonNull; import android.os.IBinder; +import android.util.ArraySet; import android.util.SparseArray; import android.view.inputmethod.EditorInfo; import android.view.inputmethod.InputBinding; @@ -39,8 +40,11 @@ public abstract class AccessibilityManagerInternal { /** Bind input for all accessibility services which require ime capabilities. */ public abstract void bindInput(InputBinding binding); - /** Request input session from all accessibility services which require ime capabilities. */ - public abstract void createImeSession(); + /** + * Request input session from all accessibility services which require ime capabilities and + * whose id is not in the ignoreSet. + */ + public abstract void createImeSession(ArraySet ignoreSet); /** Start input for all accessibility services which require ime capabilities. */ public abstract void startInput(IBinder startInputToken, IInputContext inputContext, @@ -61,7 +65,7 @@ public abstract class AccessibilityManagerInternal { } @Override - public void createImeSession() { + public void createImeSession(ArraySet ignoreSet) { } @Override diff --git a/services/core/java/com/android/server/inputmethod/InputMethodBindingController.java b/services/core/java/com/android/server/inputmethod/InputMethodBindingController.java index b81478285a627..b2f500a59ba96 100644 --- a/services/core/java/com/android/server/inputmethod/InputMethodBindingController.java +++ b/services/core/java/com/android/server/inputmethod/InputMethodBindingController.java @@ -422,7 +422,7 @@ final class InputMethodBindingController { addFreshWindowToken(); return new InputBindResult( InputBindResult.ResultCode.SUCCESS_WAITING_IME_BINDING, - null, null, mCurId, mCurSeq, false); + null, null, null, mCurId, mCurSeq, false); } Slog.w(InputMethodManagerService.TAG, diff --git a/services/core/java/com/android/server/inputmethod/InputMethodManagerService.java b/services/core/java/com/android/server/inputmethod/InputMethodManagerService.java index 736da2dd27cc0..7068ed13376f1 100644 --- a/services/core/java/com/android/server/inputmethod/InputMethodManagerService.java +++ b/services/core/java/com/android/server/inputmethod/InputMethodManagerService.java @@ -2408,8 +2408,11 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub final InputMethodInfo curInputMethodInfo = mMethodMap.get(curId); final boolean suppressesSpellChecker = curInputMethodInfo != null && curInputMethodInfo.suppressesSpellChecker(); + final SparseArray accessibilityInputMethodSessions = + createAccessibilityInputMethodSessions(mCurClient.mAccessibilitySessions); return new InputBindResult(InputBindResult.ResultCode.SUCCESS_WITH_IME_SESSION, - session.session, (session.channel != null ? session.channel.dup() : null), + session.session, accessibilityInputMethodSessions, + (session.channel != null ? session.channel.dup() : null), curId, getSequenceNumberLocked(), suppressesSpellChecker); } @@ -2437,14 +2440,31 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub } if (accessibilitySession != null) { + final SessionState session = mCurClient.curSession; + IInputMethodSession imeSession = session == null ? null : session.session; + final SparseArray accessibilityInputMethodSessions = + createAccessibilityInputMethodSessions(mCurClient.mAccessibilitySessions); return new InputBindResult( InputBindResult.ResultCode.SUCCESS_WITH_ACCESSIBILITY_SESSION, - accessibilitySession.mSession, null, + imeSession, accessibilityInputMethodSessions, null, getCurIdLocked(), getSequenceNumberLocked(), false); } return null; } + private SparseArray createAccessibilityInputMethodSessions( + SparseArray accessibilitySessions) { + final SparseArray accessibilityInputMethodSessions = + new SparseArray<>(); + if (accessibilitySessions != null) { + for (int i = 0; i < accessibilitySessions.size(); i++) { + accessibilityInputMethodSessions.append(accessibilitySessions.keyAt(i), + accessibilitySessions.valueAt(i).mSession); + } + } + return accessibilityInputMethodSessions; + } + /** * Called by {@link #startInputOrWindowGainedFocusInternalLocked} to bind/unbind/attach the * selected InputMethod to the given focused IME client. @@ -2470,7 +2490,7 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub // party code. return new InputBindResult( InputBindResult.ResultCode.ERROR_SYSTEM_NOT_READY, - null, null, selectedMethodId, getSequenceNumberLocked(), false); + null, null, null, selectedMethodId, getSequenceNumberLocked(), false); } if (!InputMethodUtils.checkIfPackageBelongsToUid(mAppOpsManager, cs.uid, @@ -2519,18 +2539,18 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub // We expect the caller has already verified that the client is allowed to access this // display ID. if (isSelectedMethodBoundLocked()) { - // TODO(b/187453053): this doesn't mean a11y sessions are there. When a11y service is - // enabled while this client is switched out, this client doesn't have the session. We - // need to remove disabled sessions and add new sessions and pass them to imm through - // the input result. if (cs.curSession != null) { // Fast case: if we are already connected to the input method, // then just return it. - // we can always attach to accessibility because AccessibilityManagerService is - // always on. - // This is a workaround to the method describe above + // This doesn't mean a11y sessions are there. When a11y service is + // enabled while this client is switched out, this client doesn't have the session. + // A11yManagerService will only request missing sessions (will not request existing + // sessions again). Note when an a11y service is disabled, it will clear its + // session from all clients, so we don't need to worry about disabled a11y services. cs.mSessionRequestedForAccessibility = false; requestClientSessionForAccessibilityLocked(cs); + // we can always attach to accessibility because AccessibilityManagerService is + // always on. attachNewAccessibilityLocked(startInputReason, (startInputFlags & StartInputFlags.INITIAL_CONNECTION) != 0, -1); return attachNewInputLocked(startInputReason, @@ -2578,7 +2598,7 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub requestClientSessionForAccessibilityLocked(cs); return new InputBindResult( InputBindResult.ResultCode.SUCCESS_WAITING_IME_SESSION, - null, null, getCurIdLocked(), getSequenceNumberLocked(), false); + null, null, null, getCurIdLocked(), getSequenceNumberLocked(), false); } else { long bindingDuration = SystemClock.uptimeMillis() - getLastBindTimeLocked(); if (bindingDuration < TIME_TO_RECONNECT) { @@ -2591,7 +2611,7 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub // to see if we can get back in touch with the service. return new InputBindResult( InputBindResult.ResultCode.SUCCESS_WAITING_IME_BINDING, - null, null, getCurIdLocked(), getSequenceNumberLocked(), false); + null, null, null, getCurIdLocked(), getSequenceNumberLocked(), false); } else { EventLog.writeEvent(EventLogTags.IMF_FORCE_RECONNECT_IME, getSelectedMethodIdLocked(), bindingDuration, 0); @@ -2766,7 +2786,11 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub if (!cs.mSessionRequestedForAccessibility) { if (DEBUG) Slog.v(TAG, "Creating new accessibility sessions for client " + cs); cs.mSessionRequestedForAccessibility = true; - AccessibilityManagerInternal.get().createImeSession(); + ArraySet ignoreSet = new ArraySet<>(); + for (int i = 0; i < cs.mAccessibilitySessions.size(); i++) { + ignoreSet.add(cs.mAccessibilitySessions.keyAt(i)); + } + AccessibilityManagerInternal.get().createImeSession(ignoreSet); } } @@ -3617,7 +3641,7 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub } return new InputBindResult( InputBindResult.ResultCode.SUCCESS_REPORT_WINDOW_FOCUS_ONLY, - null, null, null, -1, false); + null, null, null, null, -1, false); } mCurFocusedWindow = windowToken; @@ -5275,11 +5299,9 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub InputBindResult res = attachNewAccessibilityLocked( StartInputReason.SESSION_CREATED_BY_ACCESSIBILITY, true, accessibilityConnectionId); - if ((res != null) && (res.method != null)) { - executeOrSendMessage(mCurClient.client, obtainMessageOOO( - MSG_BIND_ACCESSIBILITY_SERVICE, mCurClient.client, res, - accessibilityConnectionId)); - } + executeOrSendMessage(mCurClient.client, obtainMessageOOO( + MSG_BIND_ACCESSIBILITY_SERVICE, mCurClient.client, res, + accessibilityConnectionId)); } } } @@ -5300,6 +5322,21 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub MSG_UNBIND_ACCESSIBILITY_SERVICE, getSequenceNumberLocked(), unbindClientReason, mCurClient.client, accessibilityConnectionId)); } + // We only have sessions when we bound to an input method. Remove this session + // from all clients. + if (getCurMethodLocked() != null) { + final int numClients = mClients.size(); + for (int i = 0; i < numClients; ++i) { + clearClientSessionForAccessibilityLocked(mClients.valueAt(i), + accessibilityConnectionId); + } + AccessibilitySessionState session = mEnabledAccessibilitySessions.get( + accessibilityConnectionId); + if (session != null) { + finishSessionForAccessibilityLocked(session); + mEnabledAccessibilitySessions.remove(accessibilityConnectionId); + } + } } } }