From cc160389793db54c3a11a0a1ce3eed47e9820398 Mon Sep 17 00:00:00 2001 From: Feng Cao Date: Wed, 1 Apr 2020 18:11:54 -0700 Subject: [PATCH] Split autofill InlineSuggestionSession to two classes * The AutofillInlineSession is renamed from InlineSuggestionSession, to avoid confusion from having the same name as the inputmethod/InlineSuggestionSession. * The convoluted synchronization and CompletableFuture are removed to simplify the flow and reduce potential bug. Everything requires the same mLock as the Session now, so that they are synchronized and deadlock-free. * The AutofillInlineSessionController is introduced, to prepare for the follow on patch where we will make each AutofillInlineSession to be only responsible for one field. Test: manual Test: atest CtsAutoFillServiceTestCases Bug: 151123764 Change-Id: I92cc558351b57f04553709569b8d41b928b9b49d --- .../AutofillInlineSessionController.java | 131 ++++++ ...tofillInlineSuggestionsRequestSession.java | 419 ++++++++++++++++++ .../autofill/InlineSuggestionSession.java | 403 ----------------- .../com/android/server/autofill/Session.java | 36 +- 4 files changed, 572 insertions(+), 417 deletions(-) create mode 100644 services/autofill/java/com/android/server/autofill/AutofillInlineSessionController.java create mode 100644 services/autofill/java/com/android/server/autofill/AutofillInlineSuggestionsRequestSession.java delete mode 100644 services/autofill/java/com/android/server/autofill/InlineSuggestionSession.java diff --git a/services/autofill/java/com/android/server/autofill/AutofillInlineSessionController.java b/services/autofill/java/com/android/server/autofill/AutofillInlineSessionController.java new file mode 100644 index 0000000000000..3612e093c8bd2 --- /dev/null +++ b/services/autofill/java/com/android/server/autofill/AutofillInlineSessionController.java @@ -0,0 +1,131 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.autofill; + +import android.annotation.NonNull; +import android.content.ComponentName; +import android.os.Bundle; +import android.os.Handler; +import android.view.autofill.AutofillId; +import android.view.inputmethod.InlineSuggestionsRequest; +import android.view.inputmethod.InlineSuggestionsResponse; + +import com.android.internal.annotations.GuardedBy; +import com.android.server.inputmethod.InputMethodManagerInternal; + +import java.util.Collections; +import java.util.Optional; +import java.util.function.Consumer; + + +/** + * Controls the interaction with the IME for the inline suggestion sessions. + */ +final class AutofillInlineSessionController { + @NonNull + private final InputMethodManagerInternal mInputMethodManagerInternal; + private final int mUserId; + @NonNull + private final ComponentName mComponentName; + @NonNull + private final Object mLock; + @NonNull + private final Handler mHandler; + + @GuardedBy("mLock") + private AutofillInlineSuggestionsRequestSession mSession; + + AutofillInlineSessionController(InputMethodManagerInternal inputMethodManagerInternal, + int userId, ComponentName componentName, Handler handler, Object lock) { + mInputMethodManagerInternal = inputMethodManagerInternal; + mUserId = userId; + mComponentName = componentName; + mHandler = handler; + mLock = lock; + } + + + /** + * Requests the IME to create an {@link InlineSuggestionsRequest} for {@code autofillId}. + * + * @param autofillId the Id of the field for which the request is for. + * @param requestConsumer the callback which will be invoked when IME responded or if it times + * out waiting for IME response. + */ + @GuardedBy("mLock") + void onCreateInlineSuggestionsRequestLocked(@NonNull AutofillId autofillId, + @NonNull Consumer requestConsumer, @NonNull Bundle uiExtras) { + // TODO(b/151123764): rename the method to better reflect what it does. + if (mSession != null) { + // Send an empty response to IME and destroy the existing session. + mSession.onInlineSuggestionsResponseLocked(mSession.getAutofillIdLocked(), + new InlineSuggestionsResponse(Collections.EMPTY_LIST)); + mSession.destroySessionLocked(); + } + // TODO(b/151123764): consider reusing the same AutofillInlineSession object for the + // same field. + mSession = new AutofillInlineSuggestionsRequestSession(mInputMethodManagerInternal, mUserId, + mComponentName, mHandler, mLock, autofillId, requestConsumer, uiExtras); + mSession.onCreateInlineSuggestionsRequestLocked(); + + } + + /** + * Returns the {@link InlineSuggestionsRequest} provided by IME for the last request. + * + *

The caller is responsible for making sure Autofill hears back from IME before calling + * this method, using the {@code requestConsumer} provided when calling {@link + * #onCreateInlineSuggestionsRequestLocked(AutofillId, Consumer, Bundle)}. + */ + @GuardedBy("mLock") + Optional getInlineSuggestionsRequestLocked() { + if (mSession != null) { + return mSession.getInlineSuggestionsRequestLocked(); + } + return Optional.empty(); + } + + /** + * Requests the IME to hide the current suggestions, if any. Returns true if the message is sent + * to the IME. + */ + @GuardedBy("mLock") + boolean hideInlineSuggestionsUiLocked(@NonNull AutofillId autofillId) { + if (mSession != null) { + return mSession.onInlineSuggestionsResponseLocked(autofillId, + new InlineSuggestionsResponse(Collections.EMPTY_LIST)); + } + return false; + } + + /** + * Requests showing the inline suggestion in the IME when the IME becomes visible and is focused + * on the {@code autofillId}. + * + * @return false if there is no session, or if the IME callback is not available in the session. + */ + @GuardedBy("mLock") + boolean onInlineSuggestionsResponseLocked(@NonNull AutofillId autofillId, + @NonNull InlineSuggestionsResponse inlineSuggestionsResponse) { + // TODO(b/151123764): rename the method to better reflect what it does. + if (mSession != null) { + return mSession.onInlineSuggestionsResponseLocked(autofillId, + inlineSuggestionsResponse); + } + return false; + } +} diff --git a/services/autofill/java/com/android/server/autofill/AutofillInlineSuggestionsRequestSession.java b/services/autofill/java/com/android/server/autofill/AutofillInlineSuggestionsRequestSession.java new file mode 100644 index 0000000000000..ca230b6936ff1 --- /dev/null +++ b/services/autofill/java/com/android/server/autofill/AutofillInlineSuggestionsRequestSession.java @@ -0,0 +1,419 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.autofill; + +import static com.android.internal.util.function.pooled.PooledLambda.obtainMessage; +import static com.android.server.autofill.Helper.sDebug; + +import android.annotation.BinderThread; +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.content.ComponentName; +import android.os.Bundle; +import android.os.Handler; +import android.os.RemoteException; +import android.util.Log; +import android.util.Slog; +import android.view.autofill.AutofillId; +import android.view.inputmethod.InlineSuggestionsRequest; +import android.view.inputmethod.InlineSuggestionsResponse; + +import com.android.internal.annotations.GuardedBy; +import com.android.internal.view.IInlineSuggestionsRequestCallback; +import com.android.internal.view.IInlineSuggestionsResponseCallback; +import com.android.internal.view.InlineSuggestionsRequestInfo; +import com.android.server.inputmethod.InputMethodManagerInternal; + +import java.lang.ref.WeakReference; +import java.util.Optional; +import java.util.function.Consumer; + +/** + * Maintains an inline suggestion session with the IME. + * + *

Each session corresponds to one request from the Autofill manager service to create an + * {@link InlineSuggestionsRequest}. It's responsible for receiving callbacks from the IME and + * sending {@link android.view.inputmethod.InlineSuggestionsResponse} to IME. + */ +final class AutofillInlineSuggestionsRequestSession { + + private static final String TAG = AutofillInlineSuggestionsRequestSession.class.getSimpleName(); + private static final int INLINE_REQUEST_TIMEOUT_MS = 200; + + @NonNull + private final InputMethodManagerInternal mInputMethodManagerInternal; + private final int mUserId; + @NonNull + private final ComponentName mComponentName; + @NonNull + private final Object mLock; + @NonNull + private final Handler mHandler; + @NonNull + private final Bundle mUiExtras; + + @GuardedBy("mLock") + @NonNull + private AutofillId mAutofillId; + @GuardedBy("mLock") + @Nullable + private Consumer mImeRequestConsumer; + + @GuardedBy("mLock") + private boolean mImeRequestReceived; + @GuardedBy("mLock") + @Nullable + private InlineSuggestionsRequest mImeRequest; + @GuardedBy("mLock") + @Nullable + private IInlineSuggestionsResponseCallback mResponseCallback; + @GuardedBy("mLock") + @Nullable + private Runnable mTimeoutCallback; + + @GuardedBy("mLock") + @Nullable + private AutofillId mImeCurrentFieldId; + @GuardedBy("mLock") + private boolean mImeInputStarted; + @GuardedBy("mLock") + private boolean mImeInputViewStarted; + @GuardedBy("mLock") + @Nullable + private InlineSuggestionsResponse mInlineSuggestionsResponse; + @GuardedBy("mLock") + private boolean mPreviousResponseIsNotEmpty; + + @GuardedBy("mLock") + private boolean mDestroyed = false; + + AutofillInlineSuggestionsRequestSession( + @NonNull InputMethodManagerInternal inputMethodManagerInternal, int userId, + @NonNull ComponentName componentName, @NonNull Handler handler, @NonNull Object lock, + @NonNull AutofillId autofillId, + @NonNull Consumer requestConsumer, @NonNull Bundle uiExtras) { + mInputMethodManagerInternal = inputMethodManagerInternal; + mUserId = userId; + mComponentName = componentName; + mHandler = handler; + mLock = lock; + mUiExtras = uiExtras; + + mAutofillId = autofillId; + mImeRequestConsumer = requestConsumer; + } + + @GuardedBy("mLock") + @NonNull + AutofillId getAutofillIdLocked() { + return mAutofillId; + } + + /** + * Returns the {@link InlineSuggestionsRequest} provided by IME. + * + *

The caller is responsible for making sure Autofill hears back from IME before calling + * this method, using the {@link #mImeRequestConsumer}. + */ + @GuardedBy("mLock") + Optional getInlineSuggestionsRequestLocked() { + if (mDestroyed) { + return Optional.empty(); + } + return Optional.ofNullable(mImeRequest); + } + + /** + * Requests showing the inline suggestion in the IME when the IME becomes visible and is focused + * on the {@code autofillId}. + * + * @return false if the IME callback is not available. + */ + @GuardedBy("mLock") + boolean onInlineSuggestionsResponseLocked(@NonNull AutofillId autofillId, + @NonNull InlineSuggestionsResponse inlineSuggestionsResponse) { + if (mDestroyed) { + return false; + } + if (sDebug) Log.d(TAG, "onInlineSuggestionsResponseLocked called for:" + autofillId); + if (mImeRequest == null || mResponseCallback == null) { + return false; + } + // TODO(b/151123764): each session should only correspond to one field. + mAutofillId = autofillId; + mInlineSuggestionsResponse = inlineSuggestionsResponse; + maybeUpdateResponseToImeLocked(); + return true; + } + + /** + * This method must be called when the session is destroyed, to avoid further callbacks from/to + * the IME. + */ + @GuardedBy("mLock") + void destroySessionLocked() { + mDestroyed = true; + } + + /** + * Requests the IME to create an {@link InlineSuggestionsRequest}. + * + *

This method should only be called once per session. + */ + @GuardedBy("mLock") + void onCreateInlineSuggestionsRequestLocked() { + if (sDebug) Log.d(TAG, "onCreateInlineSuggestionsRequestLocked called: " + mAutofillId); + if (mDestroyed) { + return; + } + mInputMethodManagerInternal.onCreateInlineSuggestionsRequest(mUserId, + new InlineSuggestionsRequestInfo(mComponentName, mAutofillId, mUiExtras), + new InlineSuggestionsRequestCallbackImpl(this)); + mTimeoutCallback = () -> { + Log.w(TAG, "Timed out waiting for IME callback InlineSuggestionsRequest."); + handleOnReceiveImeRequest(null, null); + }; + mHandler.postDelayed(mTimeoutCallback, INLINE_REQUEST_TIMEOUT_MS); + } + + /** + * Optionally sends inline response to the IME, depending on the current state. + */ + @GuardedBy("mLock") + private void maybeUpdateResponseToImeLocked() { + if (sDebug) Log.d(TAG, "maybeUpdateResponseToImeLocked called"); + if (mDestroyed || mResponseCallback == null) { + return; + } + if (!mImeInputViewStarted && mPreviousResponseIsNotEmpty) { + // 1. if previous response is not empty, and IME just become invisible, then send + // empty response to make sure existing responses don't stick around on the IME. + // Although the inline suggestions should disappear when IME hides which removes them + // from the view hierarchy, but we still send an empty response to be extra safe. + + // TODO(b/149945531): clear the existing suggestions when IME is hide, once the bug is + // fixed. + //if (sDebug) Log.d(TAG, "Send empty inline response"); + //updateResponseToImeUncheckLocked(new InlineSuggestionsResponse(Collections + // .EMPTY_LIST)); + //mPreviousResponseIsNotEmpty = false; + } else if (mImeInputViewStarted && mInlineSuggestionsResponse != null && match(mAutofillId, + mImeCurrentFieldId)) { + // 2. if IME is visible, and response is not null, send the response + boolean isEmptyResponse = mInlineSuggestionsResponse.getInlineSuggestions().isEmpty(); + if (isEmptyResponse && !mPreviousResponseIsNotEmpty) { + // No-op if both the previous response and current response are empty. + return; + } + if (sDebug) { + Log.d(TAG, "Send inline response: " + + mInlineSuggestionsResponse.getInlineSuggestions().size()); + } + updateResponseToImeUncheckLocked(mInlineSuggestionsResponse); + // TODO(b/149945531): don't set the response to null so it's cached, once the bug is + // fixed. + mInlineSuggestionsResponse = null; + mPreviousResponseIsNotEmpty = !isEmptyResponse; + } + } + + /** + * Sends the {@code response} to the IME, assuming all the relevant checks are already done. + */ + @GuardedBy("mLock") + private void updateResponseToImeUncheckLocked(InlineSuggestionsResponse response) { + if (mDestroyed) { + return; + } + try { + mResponseCallback.onInlineSuggestionsResponse(mAutofillId, response); + } catch (RemoteException e) { + Slog.e(TAG, "RemoteException sending InlineSuggestionsResponse to IME"); + } + } + + /** + * Handles the {@code request} and {@code callback} received from the IME. + * + *

Should only invoked in the {@link #mHandler} thread. + */ + private void handleOnReceiveImeRequest(@Nullable InlineSuggestionsRequest request, + @Nullable IInlineSuggestionsResponseCallback callback) { + synchronized (mLock) { + if (mDestroyed || mImeRequestReceived) { + return; + } + mImeRequestReceived = true; + + if (mTimeoutCallback != null) { + if (sDebug) Log.d(TAG, "removing timeout callback"); + mHandler.removeCallbacks(mTimeoutCallback); + mTimeoutCallback = null; + } + if (request != null && callback != null) { + mImeRequest = request; + mResponseCallback = callback; + handleOnReceiveImeStatusUpdated(mAutofillId, true, false); + } + if (mImeRequestConsumer != null) { + // Note that mImeRequest is only set if both request and callback are non-null. + mImeRequestConsumer.accept(mImeRequest); + mImeRequestConsumer = null; + } + } + } + + /** + * Handles the IME status updates received from the IME. + * + *

Should only be invoked in the {@link #mHandler} thread. + */ + private void handleOnReceiveImeStatusUpdated(boolean imeInputStarted, + boolean imeInputViewStarted) { + synchronized (mLock) { + if (mDestroyed) { + return; + } + if (mImeCurrentFieldId != null) { + boolean imeInputStartedChanged = (mImeInputStarted != imeInputStarted); + boolean imeInputViewStartedChanged = (mImeInputViewStarted != imeInputViewStarted); + mImeInputStarted = imeInputStarted; + mImeInputViewStarted = imeInputViewStarted; + if (imeInputStartedChanged || imeInputViewStartedChanged) { + maybeUpdateResponseToImeLocked(); + } + } + } + } + + /** + * Handles the IME status updates received from the IME. + * + *

Should only be invoked in the {@link #mHandler} thread. + */ + private void handleOnReceiveImeStatusUpdated(@Nullable AutofillId imeFieldId, + boolean imeInputStarted, boolean imeInputViewStarted) { + synchronized (mLock) { + if (mDestroyed) { + return; + } + if (imeFieldId != null) { + mImeCurrentFieldId = imeFieldId; + } + handleOnReceiveImeStatusUpdated(imeInputStarted, imeInputViewStarted); + } + } + + private static final class InlineSuggestionsRequestCallbackImpl extends + IInlineSuggestionsRequestCallback.Stub { + + private final WeakReference mSession; + + private InlineSuggestionsRequestCallbackImpl( + AutofillInlineSuggestionsRequestSession session) { + mSession = new WeakReference<>(session); + } + + @BinderThread + @Override + public void onInlineSuggestionsUnsupported() throws RemoteException { + if (sDebug) Log.d(TAG, "onInlineSuggestionsUnsupported() called."); + final AutofillInlineSuggestionsRequestSession session = mSession.get(); + if (session != null) { + session.mHandler.sendMessage(obtainMessage( + AutofillInlineSuggestionsRequestSession::handleOnReceiveImeRequest, session, + null, null)); + } + } + + @BinderThread + @Override + public void onInlineSuggestionsRequest(InlineSuggestionsRequest request, + IInlineSuggestionsResponseCallback callback) { + if (sDebug) Log.d(TAG, "onInlineSuggestionsRequest() received: " + request); + final AutofillInlineSuggestionsRequestSession session = mSession.get(); + if (session != null) { + session.mHandler.sendMessage(obtainMessage( + AutofillInlineSuggestionsRequestSession::handleOnReceiveImeRequest, session, + request, callback)); + } + } + + @Override + public void onInputMethodStartInput(AutofillId imeFieldId) throws RemoteException { + if (sDebug) Log.d(TAG, "onInputMethodStartInput() received on " + imeFieldId); + final AutofillInlineSuggestionsRequestSession session = mSession.get(); + if (session != null) { + session.mHandler.sendMessage(obtainMessage( + AutofillInlineSuggestionsRequestSession::handleOnReceiveImeStatusUpdated, + session, imeFieldId, true, false)); + } + } + + @Override + public void onInputMethodShowInputRequested(boolean requestResult) throws RemoteException { + if (sDebug) { + Log.d(TAG, "onInputMethodShowInputRequested() received: " + requestResult); + } + } + + @BinderThread + @Override + public void onInputMethodStartInputView() { + if (sDebug) Log.d(TAG, "onInputMethodStartInputView() received"); + final AutofillInlineSuggestionsRequestSession session = mSession.get(); + if (session != null) { + session.mHandler.sendMessage(obtainMessage( + AutofillInlineSuggestionsRequestSession::handleOnReceiveImeStatusUpdated, + session, true, true)); + } + } + + @BinderThread + @Override + public void onInputMethodFinishInputView() { + if (sDebug) Log.d(TAG, "onInputMethodFinishInputView() received"); + final AutofillInlineSuggestionsRequestSession session = mSession.get(); + if (session != null) { + session.mHandler.sendMessage(obtainMessage( + AutofillInlineSuggestionsRequestSession::handleOnReceiveImeStatusUpdated, + session, true, false)); + } + } + + @Override + public void onInputMethodFinishInput() throws RemoteException { + if (sDebug) Log.d(TAG, "onInputMethodFinishInput() received"); + final AutofillInlineSuggestionsRequestSession session = mSession.get(); + if (session != null) { + session.mHandler.sendMessage(obtainMessage( + AutofillInlineSuggestionsRequestSession::handleOnReceiveImeStatusUpdated, + session, false, false)); + } + } + } + + private static boolean match(@Nullable AutofillId autofillId, + @Nullable AutofillId imeClientFieldId) { + // The IME doesn't have information about the virtual view id for the child views in the + // web view, so we are only comparing the parent view id here. This means that for cases + // where there are two input fields in the web view, they will have the same view id + // (although different virtual child id), and we will not be able to distinguish them. + return autofillId != null && imeClientFieldId != null + && autofillId.getViewId() == imeClientFieldId.getViewId(); + } +} diff --git a/services/autofill/java/com/android/server/autofill/InlineSuggestionSession.java b/services/autofill/java/com/android/server/autofill/InlineSuggestionSession.java deleted file mode 100644 index e2d511212a71a..0000000000000 --- a/services/autofill/java/com/android/server/autofill/InlineSuggestionSession.java +++ /dev/null @@ -1,403 +0,0 @@ -/* - * Copyright (C) 2020 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.android.server.autofill; - -import static com.android.server.autofill.Helper.sDebug; - -import android.annotation.BinderThread; -import android.annotation.NonNull; -import android.annotation.Nullable; -import android.content.ComponentName; -import android.os.Bundle; -import android.os.Handler; -import android.os.RemoteException; -import android.util.Log; -import android.util.Slog; -import android.view.autofill.AutofillId; -import android.view.inputmethod.InlineSuggestionsRequest; -import android.view.inputmethod.InlineSuggestionsResponse; - -import com.android.internal.annotations.GuardedBy; -import com.android.internal.view.IInlineSuggestionsRequestCallback; -import com.android.internal.view.IInlineSuggestionsResponseCallback; -import com.android.internal.view.InlineSuggestionsRequestInfo; -import com.android.server.inputmethod.InputMethodManagerInternal; - -import java.util.Collections; -import java.util.Optional; -import java.util.concurrent.CompletableFuture; -import java.util.function.Consumer; - -/** - * Maintains an autofill inline suggestion session that communicates with the IME. - * - *

- * The same session may be reused for multiple input fields involved in the same autofill - * {@link Session}. Therefore, one {@link InlineSuggestionsRequest} and one - * {@link IInlineSuggestionsResponseCallback} may be used to generate and callback with inline - * suggestions for different input fields. - * - *

- * This class is the sole place in Autofill responsible for directly communicating with the IME. It - * receives the IME input view start/finish events, with the associated IME field Id. It uses the - * information to decide when to send the {@link InlineSuggestionsResponse} to IME. As a result, - * some of the response will be cached locally and only be sent when the IME is ready to show them. - * - *

- * See {@link android.inputmethodservice.InlineSuggestionSession} comments for InputMethodService - * side flow. - * - *

- * This class should hold the same lock as {@link Session} as they call into each other. - */ -final class InlineSuggestionSession { - - private static final String TAG = "AfInlineSuggestionSession"; - private static final int INLINE_REQUEST_TIMEOUT_MS = 200; - - @NonNull - private final InputMethodManagerInternal mInputMethodManagerInternal; - private final int mUserId; - @NonNull - private final ComponentName mComponentName; - @NonNull - private final Object mLock; - @NonNull - private final ImeStatusListener mImeStatusListener; - @NonNull - private final Handler mHandler; - - /** - * To avoid the race condition, one should not access {@code mPendingImeResponse} without - * holding the {@code mLock}. For consuming the existing value, tt's recommended to use - * {@link #getPendingImeResponse()} to get a copy of the reference to avoid blocking call. - */ - @GuardedBy("mLock") - @Nullable - private CompletableFuture mPendingImeResponse; - - @GuardedBy("mLock") - @Nullable - private AutofillResponse mPendingAutofillResponse; - - @GuardedBy("mLock") - private boolean mIsLastResponseNonEmpty = false; - - @Nullable - @GuardedBy("mLock") - private AutofillId mImeFieldId = null; - - @GuardedBy("mLock") - private boolean mImeInputViewStarted = false; - - InlineSuggestionSession(InputMethodManagerInternal inputMethodManagerInternal, - int userId, ComponentName componentName, Handler handler, Object lock) { - mInputMethodManagerInternal = inputMethodManagerInternal; - mUserId = userId; - mComponentName = componentName; - mHandler = handler; - mLock = lock; - mImeStatusListener = new ImeStatusListener() { - @Override - public void onInputMethodStartInput(AutofillId imeFieldId) { - synchronized (mLock) { - mImeFieldId = imeFieldId; - mImeInputViewStarted = false; - } - } - - @Override - public void onInputMethodStartInputView() { - synchronized (mLock) { - mImeInputViewStarted = true; - AutofillResponse pendingAutofillResponse = mPendingAutofillResponse; - if (pendingAutofillResponse != null - && pendingAutofillResponse.mAutofillId.equalsIgnoreSession( - mImeFieldId)) { - mPendingAutofillResponse = null; - onInlineSuggestionsResponseLocked(pendingAutofillResponse.mAutofillId, - pendingAutofillResponse.mResponse); - } - } - } - - @Override - public void onInputMethodFinishInputView() { - synchronized (mLock) { - mImeInputViewStarted = false; - } - } - - @Override - public void onInputMethodFinishInput() { - synchronized (mLock) { - mImeFieldId = null; - } - } - }; - } - - public void onCreateInlineSuggestionsRequest(@NonNull AutofillId autofillId, - @NonNull Consumer requestConsumer) { - if (sDebug) Log.d(TAG, "onCreateInlineSuggestionsRequest called for " + autofillId); - - synchronized (mLock) { - // Clean up all the state about the previous request. - hideInlineSuggestionsUi(autofillId); - mImeFieldId = null; - mImeInputViewStarted = false; - if (mPendingImeResponse != null && !mPendingImeResponse.isDone()) { - mPendingImeResponse.complete(null); - } - mPendingImeResponse = new CompletableFuture<>(); - // TODO(b/146454892): pipe the uiExtras from the ExtServices. - mInputMethodManagerInternal.onCreateInlineSuggestionsRequest( - mUserId, - new InlineSuggestionsRequestInfo(mComponentName, autofillId, new Bundle()), - new InlineSuggestionsRequestCallbackImpl(autofillId, mPendingImeResponse, - mImeStatusListener, requestConsumer, mHandler, mLock)); - } - } - - public Optional getInlineSuggestionsRequest() { - final CompletableFuture pendingImeResponse = getPendingImeResponse(); - if (pendingImeResponse == null || !pendingImeResponse.isDone()) { - return Optional.empty(); - } - return Optional.ofNullable(pendingImeResponse.getNow(null)).map(ImeResponse::getRequest); - } - - public boolean hideInlineSuggestionsUi(@NonNull AutofillId autofillId) { - synchronized (mLock) { - if (mIsLastResponseNonEmpty) { - return onInlineSuggestionsResponseLocked(autofillId, - new InlineSuggestionsResponse(Collections.EMPTY_LIST)); - } - return false; - } - } - - public boolean onInlineSuggestionsResponse(@NonNull AutofillId autofillId, - @NonNull InlineSuggestionsResponse inlineSuggestionsResponse) { - synchronized (mLock) { - return onInlineSuggestionsResponseLocked(autofillId, inlineSuggestionsResponse); - } - } - - private boolean onInlineSuggestionsResponseLocked(@NonNull AutofillId autofillId, - @NonNull InlineSuggestionsResponse inlineSuggestionsResponse) { - final CompletableFuture completedImsResponse = getPendingImeResponse(); - if (completedImsResponse == null || !completedImsResponse.isDone()) { - if (sDebug) Log.d(TAG, "onInlineSuggestionsResponseLocked without IMS request"); - return false; - } - // There is no need to wait on the CompletableFuture since it should have been completed. - ImeResponse imeResponse = completedImsResponse.getNow(null); - if (imeResponse == null) { - if (sDebug) Log.d(TAG, "onInlineSuggestionsResponseLocked with pending IMS response"); - return false; - } - - // TODO(b/151846600): IME doesn't have access to the virtual id of the webview, so we - // only compare the view id for now. - if (!mImeInputViewStarted || mImeFieldId == null - || autofillId.getViewId() != mImeFieldId.getViewId()) { - if (sDebug) { - Log.d(TAG, - "onInlineSuggestionsResponseLocked not sent because input view is not " - + "started for " + autofillId); - } - mPendingAutofillResponse = new AutofillResponse(autofillId, inlineSuggestionsResponse); - // TODO(b/149442582): Although we are not sending the response to IME right away, we - // still return true to indicate that the response may be sent eventually, such that - // the dropdown UI will not be shown. This may not be the desired behavior in the - // auto-focus case where IME isn't shown after switching back to an activity. We may - // revisit this. - return true; - } - - try { - imeResponse.mCallback.onInlineSuggestionsResponse(autofillId, - inlineSuggestionsResponse); - mIsLastResponseNonEmpty = !inlineSuggestionsResponse.getInlineSuggestions().isEmpty(); - if (sDebug) { - Log.d(TAG, "Autofill sends inline response to IME: " - + inlineSuggestionsResponse.getInlineSuggestions().size()); - } - return true; - } catch (RemoteException e) { - Slog.e(TAG, "RemoteException sending InlineSuggestionsResponse to IME"); - return false; - } - } - - @Nullable - @GuardedBy("mLock") - private CompletableFuture getPendingImeResponse() { - synchronized (mLock) { - return mPendingImeResponse; - } - } - - private static final class InlineSuggestionsRequestCallbackImpl - extends IInlineSuggestionsRequestCallback.Stub { - - private final Object mLock; - private final AutofillId mAutofillId; - @GuardedBy("mLock") - private final CompletableFuture mResponse; - @GuardedBy("mLock") - private final Consumer mRequestConsumer; - private final ImeStatusListener mImeStatusListener; - private final Handler mHandler; - private final Runnable mTimeoutCallback; - - private InlineSuggestionsRequestCallbackImpl(AutofillId autofillId, - CompletableFuture response, - ImeStatusListener imeStatusListener, - Consumer requestConsumer, - Handler handler, Object lock) { - mAutofillId = autofillId; - mResponse = response; - mImeStatusListener = imeStatusListener; - mRequestConsumer = requestConsumer; - mLock = lock; - - mHandler = handler; - mTimeoutCallback = () -> { - Log.w(TAG, "Timed out waiting for IME callback InlineSuggestionsRequest."); - completeIfNot(null); - }; - mHandler.postDelayed(mTimeoutCallback, INLINE_REQUEST_TIMEOUT_MS); - } - - private void completeIfNot(@Nullable ImeResponse response) { - synchronized (mLock) { - if (mResponse.isDone()) { - return; - } - mResponse.complete(response); - mRequestConsumer.accept(response == null ? null : response.mRequest); - mHandler.removeCallbacks(mTimeoutCallback); - } - } - - @BinderThread - @Override - public void onInlineSuggestionsUnsupported() throws RemoteException { - if (sDebug) Log.d(TAG, "onInlineSuggestionsUnsupported() called."); - completeIfNot(null); - } - - @BinderThread - @Override - public void onInlineSuggestionsRequest(InlineSuggestionsRequest request, - IInlineSuggestionsResponseCallback callback) { - if (sDebug) Log.d(TAG, "onInlineSuggestionsRequest() received: " + request); - mImeStatusListener.onInputMethodStartInput(mAutofillId); - if (request != null && callback != null) { - completeIfNot(new ImeResponse(request, callback)); - } else { - completeIfNot(null); - } - } - - @Override - public void onInputMethodStartInput(AutofillId imeFieldId) throws RemoteException { - if (sDebug) Log.d(TAG, "onInputMethodStartInput() received on " + imeFieldId); - mImeStatusListener.onInputMethodStartInput(imeFieldId); - } - - @Override - public void onInputMethodShowInputRequested(boolean requestResult) throws RemoteException { - if (sDebug) { - Log.d(TAG, "onInputMethodShowInputRequested() received: " + requestResult); - } - // TODO(b/151123764): use this signal to adjust the timeout on Autofill side waiting for - // IME to show. - } - - @BinderThread - @Override - public void onInputMethodStartInputView() { - if (sDebug) Log.d(TAG, "onInputMethodStartInputView() received"); - mImeStatusListener.onInputMethodStartInputView(); - } - - @BinderThread - @Override - public void onInputMethodFinishInputView() { - if (sDebug) Log.d(TAG, "onInputMethodFinishInputView() received"); - mImeStatusListener.onInputMethodFinishInputView(); - } - - @Override - public void onInputMethodFinishInput() throws RemoteException { - if (sDebug) Log.d(TAG, "onInputMethodFinishInput() received"); - mImeStatusListener.onInputMethodFinishInput(); - } - } - - private interface ImeStatusListener { - void onInputMethodStartInput(AutofillId imeFieldId); - - void onInputMethodStartInputView(); - - void onInputMethodFinishInputView(); - - void onInputMethodFinishInput(); - } - - /** - * A data class wrapping Autofill responses for the inline suggestion request. - */ - private static class AutofillResponse { - @NonNull - final AutofillId mAutofillId; - - @NonNull - final InlineSuggestionsResponse mResponse; - - AutofillResponse(@NonNull AutofillId autofillId, - @NonNull InlineSuggestionsResponse response) { - mAutofillId = autofillId; - mResponse = response; - } - - } - - /** - * A data class wrapping IME responses for the create inline suggestions request. - */ - private static class ImeResponse { - @NonNull - final InlineSuggestionsRequest mRequest; - - @NonNull - final IInlineSuggestionsResponseCallback mCallback; - - ImeResponse(@NonNull InlineSuggestionsRequest request, - @NonNull IInlineSuggestionsResponseCallback callback) { - mRequest = request; - mCallback = callback; - } - - InlineSuggestionsRequest getRequest() { - return mRequest; - } - } -} diff --git a/services/autofill/java/com/android/server/autofill/Session.java b/services/autofill/java/com/android/server/autofill/Session.java index 12905696ff98b..4ecffd8d29b03 100644 --- a/services/autofill/java/com/android/server/autofill/Session.java +++ b/services/autofill/java/com/android/server/autofill/Session.java @@ -304,7 +304,7 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState private boolean mForAugmentedAutofillOnly; @Nullable - private final InlineSuggestionSession mInlineSuggestionSession; + private final AutofillInlineSessionController mInlineSessionController; /** * Receiver of assist data from the app's {@link Activity}. @@ -720,8 +720,9 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState Consumer inlineSuggestionsRequestConsumer = mAssistReceiver.newAutofillRequestLocked(/*isInlineRequest=*/ true); if (inlineSuggestionsRequestConsumer != null) { - mInlineSuggestionSession.onCreateInlineSuggestionsRequest(mCurrentViewId, - inlineSuggestionsRequestConsumer); + // TODO(b/146454892): pipe the uiExtras from the ExtServices. + mInlineSessionController.onCreateInlineSuggestionsRequestLocked(mCurrentViewId, + inlineSuggestionsRequestConsumer, Bundle.EMPTY); } } else { mAssistReceiver.newAutofillRequestLocked(/*isInlineRequest=*/ false); @@ -777,8 +778,8 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState mForAugmentedAutofillOnly = forAugmentedAutofillOnly; setClientLocked(client); - mInlineSuggestionSession = new InlineSuggestionSession(inputMethodManagerInternal, userId, - componentName, handler, mLock); + mInlineSessionController = new AutofillInlineSessionController(inputMethodManagerInternal, + userId, componentName, handler, mLock); mMetricsLogger.write(newLogMaker(MetricsEvent.AUTOFILL_SESSION_STARTED) .addTaggedData(MetricsEvent.FIELD_AUTOFILL_FLAGS, flags)); @@ -2561,7 +2562,7 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState if (sVerbose) Slog.v(TAG, "Exiting view " + id); mUi.hideFillUi(this); hideAugmentedAutofillLocked(viewState); - mInlineSuggestionSession.hideInlineSuggestionsUi(mCurrentViewId); + mInlineSessionController.hideInlineSuggestionsUiLocked(mCurrentViewId); mCurrentViewId = null; } break; @@ -2779,7 +2780,7 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState private boolean requestShowInlineSuggestionsLocked(@NonNull FillResponse response, @Nullable String filterText) { final Optional inlineSuggestionsRequest = - mInlineSuggestionSession.getInlineSuggestionsRequest(); + mInlineSessionController.getInlineSuggestionsRequestLocked(); if (!inlineSuggestionsRequest.isPresent()) { Log.w(TAG, "InlineSuggestionsRequest unavailable"); return false; @@ -2801,7 +2802,8 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState inlineSuggestionsRequest.get(), response, filterText, mCurrentViewId, this, () -> { synchronized (mLock) { - mInlineSuggestionSession.hideInlineSuggestionsUi(mCurrentViewId); + mInlineSessionController.hideInlineSuggestionsUiLocked( + mCurrentViewId); } }, remoteRenderService); if (inlineSuggestionsResponse == null) { @@ -2809,7 +2811,7 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState return false; } - return mInlineSuggestionSession.onInlineSuggestionsResponse(mCurrentViewId, + return mInlineSessionController.onInlineSuggestionsResponseLocked(mCurrentViewId, inlineSuggestionsResponse); } @@ -3106,8 +3108,13 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState focusedId, currentValue, inlineSuggestionsRequest, /*inlineSuggestionsCallback=*/ - response -> mInlineSuggestionSession.onInlineSuggestionsResponse( - mCurrentViewId, response), + response -> { + synchronized (mLock) { + return mInlineSessionController + .onInlineSuggestionsResponseLocked( + mCurrentViewId, response); + } + }, /*onErrorCallback=*/ () -> { synchronized (mLock) { cancelAugmentedAutofillLocked(); @@ -3125,11 +3132,12 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState && (mForAugmentedAutofillOnly || !isInlineSuggestionsEnabledByAutofillProviderLocked())) { if (sDebug) Slog.d(TAG, "Create inline request for augmented autofill"); - mInlineSuggestionSession.onCreateInlineSuggestionsRequest(mCurrentViewId, - /*requestConsumer=*/ requestAugmentedAutofill); + // TODO(b/146454892): pipe the uiExtras from the ExtServices. + mInlineSessionController.onCreateInlineSuggestionsRequestLocked(mCurrentViewId, + /*requestConsumer=*/ requestAugmentedAutofill, Bundle.EMPTY); } else { requestAugmentedAutofill.accept( - mInlineSuggestionSession.getInlineSuggestionsRequest().orElse(null)); + mInlineSessionController.getInlineSuggestionsRequestLocked().orElse(null)); } if (mAugmentedAutofillDestroyer == null) { mAugmentedAutofillDestroyer = () -> remoteService.onDestroyAutofillWindowsRequest();