Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge "Notify autofill with the IME start/finish input view events" into rvc-dev

Browse Source
This commit is contained in:
Feng Cao
2020-03-03 18:36:11 +00:00
committed by Android (Google) Code Review
parent 2c2afb92e3 7c85eb79a7
commit c9611acb62
6 changed files with 245 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
core/java/android/inputmethodservice/InlineSuggestionSession.java
View File

@@ -28,6 +28,7 @@ import android.os.Looper;
import android.os.RemoteException;
import android.util.Log;
import android.view.autofill.AutofillId;
import android.view.inputmethod.EditorInfo;
import android.view.inputmethod.InlineSuggestionsRequest;
import android.view.inputmethod.InlineSuggestionsResponse;
@@ -45,10 +46,27 @@ import java.util.function.Supplier;
* Each session corresponds to one {@link InlineSuggestionsRequest} and one {@link
* IInlineSuggestionsResponseCallback}, but there may be multiple invocations of the response
* callback for the same field or different fields in the same component.
*
* <p>
* The data flow from IMS point of view is:
* Before calling {@link InputMethodService#onStartInputView(EditorInfo, boolean)}, call the {@link
* #notifyOnStartInputView(AutofillId)}
* ->
* [async] {@link IInlineSuggestionsRequestCallback#onInputMethodStartInputView(AutofillId)}
* --- process boundary ---
* ->
* {@link com.android.server.inputmethod.InputMethodManagerService
* .InlineSuggestionsRequestCallbackDecorator#onInputMethodStartInputView(AutofillId)}
* ->
* {@link com.android.server.autofill.InlineSuggestionSession
* .InlineSuggestionsRequestCallbackImpl#onInputMethodStartInputView(AutofillId)}
*
* <p>
* The data flow for {@link #notifyOnFinishInputView(AutofillId)} is similar.
*/
class InlineSuggestionSession {
private static final String TAG = InlineSuggestionSession.class.getSimpleName();
private static final String TAG = "ImsInlineSuggestionSession";
private final Handler mHandler = new Handler(Looper.getMainLooper(), null, true);
@@ -77,7 +95,8 @@ class InlineSuggestionSession {
@NonNull Supplier<AutofillId> clientAutofillIdSupplier,
@NonNull Supplier<InlineSuggestionsRequest> requestSupplier,
@NonNull Supplier<IBinder> hostInputTokenSupplier,
@NonNull Consumer<InlineSuggestionsResponse> responseConsumer) {
@NonNull Consumer<InlineSuggestionsResponse> responseConsumer,
boolean inputViewStarted) {
mComponentName = componentName;
mCallback = callback;
mResponseCallback = new InlineSuggestionsResponseCallbackImpl(this);
@@ -87,7 +106,25 @@ class InlineSuggestionSession {
mHostInputTokenSupplier = hostInputTokenSupplier;
mResponseConsumer = responseConsumer;
makeInlineSuggestionsRequest();
makeInlineSuggestionsRequest(inputViewStarted);
}
void notifyOnStartInputView(AutofillId imeFieldId) {
if (DEBUG) Log.d(TAG, "notifyOnStartInputView");
try {
mCallback.onInputMethodStartInputView(imeFieldId);
} catch (RemoteException e) {
Log.w(TAG, "onInputMethodStartInputView() remote exception:" + e);
}
}
void notifyOnFinishInputView(AutofillId imeFieldId) {
if (DEBUG) Log.d(TAG, "notifyOnFinishInputView");
try {
mCallback.onInputMethodFinishInputView(imeFieldId);
} catch (RemoteException e) {
Log.w(TAG, "onInputMethodFinishInputView() remote exception:" + e);
}
}
/**
@@ -103,7 +140,7 @@ class InlineSuggestionSession {
* Autofill Session through
* {@link IInlineSuggestionsRequestCallback#onInlineSuggestionsRequest}.
*/
private void makeInlineSuggestionsRequest() {
private void makeInlineSuggestionsRequest(boolean inputViewStarted) {
try {
final InlineSuggestionsRequest request = mRequestSupplier.get();
if (request == null) {
@@ -113,7 +150,8 @@ class InlineSuggestionSession {
mCallback.onInlineSuggestionsUnsupported();
} else {
request.setHostInputToken(mHostInputTokenSupplier.get());
mCallback.onInlineSuggestionsRequest(request, mResponseCallback);
mCallback.onInlineSuggestionsRequest(request, mResponseCallback,
mClientAutofillIdSupplier.get(), inputViewStarted);
}
} catch (RemoteException e) {
Log.w(TAG, "makeInlinedSuggestionsRequest() remote exception:" + e);
@@ -128,16 +166,15 @@ class InlineSuggestionSession {
}
return;
}
// TODO(b/149522488): Verify fieldId against {@code mClientAutofillIdSupplier.get()} using
// {@link AutofillId#equalsIgnoreSession(AutofillId)}. Right now, this seems to be
// falsely alarmed quite often, depending whether autofill suggestions arrive earlier
// than the IMS EditorInfo updates or not.
if (!mComponentName.getPackageName().equals(mClientPackageNameSupplier.get())) {
if (!mComponentName.getPackageName().equals(mClientPackageNameSupplier.get())
|| !fieldId.equalsIgnoreSession(mClientAutofillIdSupplier.get())) {
if (DEBUG) {
Log.d(TAG,
"handleOnInlineSuggestionsResponse() called on the wrong package "
"handleOnInlineSuggestionsResponse() called on the wrong package/field "
+ "name: " + mComponentName.getPackageName() + " v.s. "
+ mClientPackageNameSupplier.get());
+ mClientPackageNameSupplier.get() + ", " + fieldId + " v.s. "
+ mClientAutofillIdSupplier.get());
}
return;
}

39
core/java/android/inputmethodservice/InputMethodService.java
View File

@@ -444,6 +444,16 @@ public class InputMethodService extends AbstractInputMethodService {
final Insets mTmpInsets = new Insets();
final int[] mTmpLocation = new int[2];
/**
* We use a separate {@code mInlineLock} to make sure {@code mInlineSuggestionSession} is
* only accessed synchronously. Although when the lock is introduced, all the calls are from
* the main thread so the lock is not really necessarily (but for the same reason it also
* doesn't hurt), it's still being added as a safety guard to make sure in the future we
* don't add more code causing race condition when updating the {@code
* mInlineSuggestionSession}.
*/
private final Object mInlineLock = new Object();
@GuardedBy("mInlineLock")
@Nullable
private InlineSuggestionSession mInlineSuggestionSession;
@@ -822,13 +832,15 @@ public class InputMethodService extends AbstractInputMethodService {
return;
}
if (mInlineSuggestionSession != null) {
mInlineSuggestionSession.invalidateSession();
synchronized (mInlineLock) {
if (mInlineSuggestionSession != null) {
mInlineSuggestionSession.invalidateSession();
}
mInlineSuggestionSession = new InlineSuggestionSession(requestInfo.getComponentName(),
callback, this::getEditorInfoPackageName, this::getEditorInfoAutofillId,
() -> onCreateInlineSuggestionsRequest(requestInfo.getUiExtras()),
this::getHostInputToken, this::onInlineSuggestionsResponse, mInputViewStarted);
}
mInlineSuggestionSession = new InlineSuggestionSession(requestInfo.getComponentName(),
callback, this::getEditorInfoPackageName, this::getEditorInfoAutofillId,
() -> onCreateInlineSuggestionsRequest(requestInfo.getUiExtras()),
this::getHostInputToken, this::onInlineSuggestionsResponse);
}
@Nullable
@@ -2193,6 +2205,11 @@ public class InputMethodService extends AbstractInputMethodService {
if (!mInputViewStarted) {
if (DEBUG) Log.v(TAG, "CALL: onStartInputView");
mInputViewStarted = true;
synchronized (mInlineLock) {
if (mInlineSuggestionSession != null) {
mInlineSuggestionSession.notifyOnStartInputView(getEditorInfoAutofillId());
}
}
onStartInputView(mInputEditorInfo, false);
}
} else if (!mCandidatesViewStarted) {
@@ -2233,6 +2250,11 @@ public class InputMethodService extends AbstractInputMethodService {
private void finishViews(boolean finishingInput) {
if (mInputViewStarted) {
if (DEBUG) Log.v(TAG, "CALL: onFinishInputView");
synchronized (mInlineLock) {
if (mInlineSuggestionSession != null) {
mInlineSuggestionSession.notifyOnFinishInputView(getEditorInfoAutofillId());
}
}
onFinishInputView(finishingInput);
} else if (mCandidatesViewStarted) {
if (DEBUG) Log.v(TAG, "CALL: onFinishCandidatesView");
@@ -2345,6 +2367,11 @@ public class InputMethodService extends AbstractInputMethodService {
if (mShowInputRequested) {
if (DEBUG) Log.v(TAG, "CALL: onStartInputView");
mInputViewStarted = true;
synchronized (mInlineLock) {
if (mInlineSuggestionSession != null) {
mInlineSuggestionSession.notifyOnStartInputView(getEditorInfoAutofillId());
}
}
onStartInputView(mInputEditorInfo, restarting);
startExtractingText(true);
} else if (mCandidatesVisibility == View.VISIBLE) {

7
core/java/android/view/inputmethod/EditorInfo.java
View File

@@ -431,8 +431,7 @@ public class EditorInfo implements InputType, Parcelable {
* <p> Marked as hide since it's only used by framework.</p>
* @hide
*/
@NonNull
public AutofillId autofillId = new AutofillId(View.NO_ID);
public AutofillId autofillId;
/**
* Identifier for the editor's field. This is optional, and may be
@@ -832,7 +831,7 @@ public class EditorInfo implements InputType, Parcelable {
TextUtils.writeToParcel(hintText, dest, flags);
TextUtils.writeToParcel(label, dest, flags);
dest.writeString(packageName);
autofillId.writeToParcel(dest, flags);
dest.writeParcelable(autofillId, flags);
dest.writeInt(fieldId);
dest.writeString(fieldName);
dest.writeBundle(extras);
@@ -864,7 +863,7 @@ public class EditorInfo implements InputType, Parcelable {
res.hintText = TextUtils.CHAR_SEQUENCE_CREATOR.createFromParcel(source);
res.label = TextUtils.CHAR_SEQUENCE_CREATOR.createFromParcel(source);
res.packageName = source.readString();
res.autofillId = AutofillId.CREATOR.createFromParcel(source);
res.autofillId = source.readParcelable(AutofillId.class.getClassLoader());
res.fieldId = source.readInt();
res.fieldName = source.readString();
res.extras = source.readBundle();

6
core/java/com/android/internal/view/IInlineSuggestionsRequestCallback.aidl
View File

@@ -16,6 +16,7 @@
package com.android.internal.view;
import android.view.autofill.AutofillId;
import android.view.inputmethod.InlineSuggestionsRequest;
import com.android.internal.view.IInlineSuggestionsResponseCallback;
@@ -27,5 +28,8 @@ import com.android.internal.view.IInlineSuggestionsResponseCallback;
oneway interface IInlineSuggestionsRequestCallback {
void onInlineSuggestionsUnsupported();
void onInlineSuggestionsRequest(in InlineSuggestionsRequest request,
in IInlineSuggestionsResponseCallback callback);
in IInlineSuggestionsResponseCallback callback, in AutofillId imeFieldId,
boolean inputViewStarted);
void onInputMethodStartInputView(in AutofillId imeFieldId);
void onInputMethodFinishInputView(in AutofillId imeFieldId);
}

157
services/autofill/java/com/android/server/autofill/InlineSuggestionSession.java
View File

@@ -18,6 +18,7 @@ package com.android.server.autofill;
import static com.android.server.autofill.Helper.sDebug;
import android.annotation.BinderThread;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.content.ComponentName;
@@ -50,14 +51,24 @@ import java.util.concurrent.TimeoutException;
* The same session may be reused for multiple input fields involved in the same autofill
* {@link Session}. Therefore, one {@link InlineSuggestionsRequest} and one
* {@link IInlineSuggestionsResponseCallback} may be used to generate and callback with inline
* suggestions for different input fields.
* suggestions for different input fields.
*
* <p>
* This class is the sole place in Autofill responsible for directly communicating with the IME. It
* receives the IME input view start/finish events, with the associated IME field Id. It uses the
* information to decide when to send the {@link InlineSuggestionsResponse} to IME. As a result,
* some of the response will be cached locally and only be sent when the IME is ready to show them.
*
* <p>
* See {@link android.inputmethodservice.InlineSuggestionSession} comments for InputMethodService
* side flow.
*
* <p>
* This class is thread safe.
*/
final class InlineSuggestionSession {
private static final String TAG = "InlineSuggestionSession";
private static final String TAG = "AfInlineSuggestionSession";
private static final int INLINE_REQUEST_TIMEOUT_MS = 1000;
@NonNull
@@ -67,33 +78,83 @@ final class InlineSuggestionSession {
private final ComponentName mComponentName;
@NonNull
private final Object mLock;
@NonNull
private final ImeStatusListener mImeStatusListener;
/**
* To avoid the race condition, one should not access {@code mPendingImeResponse} without
* holding the {@code mLock}. For consuming the existing value, tt's recommended to use
* {@link #getPendingImeResponse()} to get a copy of the reference to avoid blocking call.
*/
@GuardedBy("mLock")
@Nullable
private CompletableFuture<ImeResponse> mPendingImeResponse;
@GuardedBy("mLock")
@Nullable
private AutofillResponse mPendingAutofillResponse;
@GuardedBy("mLock")
private boolean mIsLastResponseNonEmpty = false;
@Nullable
@GuardedBy("mLock")
private AutofillId mImeFieldId = null;
@GuardedBy("mLock")
private boolean mImeInputViewStarted = false;
InlineSuggestionSession(InputMethodManagerInternal inputMethodManagerInternal,
int userId, ComponentName componentName) {
mInputMethodManagerInternal = inputMethodManagerInternal;
mUserId = userId;
mComponentName = componentName;
mLock = new Object();
mImeStatusListener = new ImeStatusListener() {
@Override
public void onInputMethodStartInputView(AutofillId imeFieldId) {
synchronized (mLock) {
mImeFieldId = imeFieldId;
mImeInputViewStarted = true;
AutofillResponse pendingAutofillResponse = mPendingAutofillResponse;
if (pendingAutofillResponse != null
&& pendingAutofillResponse.mAutofillId.equalsIgnoreSession(
mImeFieldId)) {
mPendingAutofillResponse = null;
onInlineSuggestionsResponseLocked(pendingAutofillResponse.mAutofillId,
pendingAutofillResponse.mResponse);
}
}
}
@Override
public void onInputMethodFinishInputView(AutofillId imeFieldId) {
synchronized (mLock) {
mImeFieldId = imeFieldId;
mImeInputViewStarted = false;
}
}
};
}
public void onCreateInlineSuggestionsRequest(@NonNull AutofillId autofillId) {
if (sDebug) Log.d(TAG, "onCreateInlineSuggestionsRequest called for " + autofillId);
synchronized (mLock) {
cancelCurrentRequest();
// Clean up all the state about the previous request.
hideInlineSuggestionsUi(autofillId);
mImeFieldId = null;
mImeInputViewStarted = false;
if (mPendingImeResponse != null && !mPendingImeResponse.isDone()) {
mPendingImeResponse.complete(null);
}
mPendingImeResponse = new CompletableFuture<>();
// TODO(b/146454892): pipe the uiExtras from the ExtServices.
mInputMethodManagerInternal.onCreateInlineSuggestionsRequest(
mUserId,
new InlineSuggestionsRequestInfo(mComponentName, autofillId, new Bundle()),
new InlineSuggestionsRequestCallbackImpl(mPendingImeResponse));
new InlineSuggestionsRequestCallbackImpl(mPendingImeResponse,
mImeStatusListener));
}
}
@@ -116,10 +177,8 @@ final class InlineSuggestionSession {
}
public boolean hideInlineSuggestionsUi(@NonNull AutofillId autofillId) {
if (sDebug) Log.d(TAG, "Called hideInlineSuggestionsUi for " + autofillId);
synchronized (mLock) {
if (mIsLastResponseNonEmpty) {
if (sDebug) Log.d(TAG, "Send empty suggestion to IME");
return onInlineSuggestionsResponseLocked(autofillId,
new InlineSuggestionsResponse(Collections.EMPTY_LIST));
}
@@ -138,14 +197,32 @@ final class InlineSuggestionSession {
@NonNull InlineSuggestionsResponse inlineSuggestionsResponse) {
final CompletableFuture<ImeResponse> completedImsResponse = getPendingImeResponse();
if (completedImsResponse == null || !completedImsResponse.isDone()) {
if (sDebug) Log.d(TAG, "onInlineSuggestionsResponseLocked without IMS request");
return false;
}
// There is no need to wait on the CompletableFuture since it should have been completed
// when {@link #waitAndGetInlineSuggestionsRequest()} was called.
ImeResponse imeResponse = completedImsResponse.getNow(null);
if (imeResponse == null) {
if (sDebug) Log.d(TAG, "onInlineSuggestionsResponseLocked with pending IMS response");
return false;
}
if (!mImeInputViewStarted || !autofillId.equalsIgnoreSession(mImeFieldId)) {
if (sDebug) {
Log.d(TAG,
"onInlineSuggestionsResponseLocked not sent because input view is not "
+ "started for " + autofillId);
}
mPendingAutofillResponse = new AutofillResponse(autofillId, inlineSuggestionsResponse);
// TODO(b/149442582): Although we are not sending the response to IME right away, we
// still return true to indicate that the response may be sent eventually, such that
// the dropdown UI will not be shown. This may not be the desired behavior in the
// auto-focus case where IME isn't shown after switching back to an activity. We may
// revisit this.
return true;
}
try {
imeResponse.mCallback.onInlineSuggestionsResponse(autofillId,
inlineSuggestionsResponse);
@@ -161,13 +238,6 @@ final class InlineSuggestionSession {
}
}
private void cancelCurrentRequest() {
CompletableFuture<ImeResponse> pendingImeResponse = getPendingImeResponse();
if (pendingImeResponse != null && !pendingImeResponse.isDone()) {
pendingImeResponse.complete(null);
}
}
@Nullable
@GuardedBy("mLock")
private CompletableFuture<ImeResponse> getPendingImeResponse() {
@@ -180,31 +250,84 @@ final class InlineSuggestionSession {
extends IInlineSuggestionsRequestCallback.Stub {
private final CompletableFuture<ImeResponse> mResponse;
private final ImeStatusListener mImeStatusListener;
private InlineSuggestionsRequestCallbackImpl(CompletableFuture<ImeResponse> response) {
private InlineSuggestionsRequestCallbackImpl(CompletableFuture<ImeResponse> response,
ImeStatusListener imeStatusListener) {
mResponse = response;
mImeStatusListener = imeStatusListener;
}
@BinderThread
@Override
public void onInlineSuggestionsUnsupported() throws RemoteException {
if (sDebug) Log.d(TAG, "onInlineSuggestionsUnsupported() called.");
mResponse.complete(null);
}
@BinderThread
@Override
public void onInlineSuggestionsRequest(InlineSuggestionsRequest request,
IInlineSuggestionsResponseCallback callback) {
if (sDebug) Log.d(TAG, "onInlineSuggestionsRequest() received: " + request);
IInlineSuggestionsResponseCallback callback, AutofillId imeFieldId,
boolean inputViewStarted) {
if (sDebug) {
Log.d(TAG,
"onInlineSuggestionsRequest() received: " + request + ", inputViewStarted="
+ inputViewStarted + ", imeFieldId=" + imeFieldId);
}
if (inputViewStarted) {
mImeStatusListener.onInputMethodStartInputView(imeFieldId);
} else {
mImeStatusListener.onInputMethodFinishInputView(imeFieldId);
}
if (request != null && callback != null) {
mResponse.complete(new ImeResponse(request, callback));
} else {
mResponse.complete(null);
}
}
@BinderThread
@Override
public void onInputMethodStartInputView(AutofillId imeFieldId) {
if (sDebug) Log.d(TAG, "onInputMethodStartInputView() received on " + imeFieldId);
mImeStatusListener.onInputMethodStartInputView(imeFieldId);
}
@BinderThread
@Override
public void onInputMethodFinishInputView(AutofillId imeFieldId) {
if (sDebug) Log.d(TAG, "onInputMethodFinishInputView() received on " + imeFieldId);
mImeStatusListener.onInputMethodFinishInputView(imeFieldId);
}
}
private interface ImeStatusListener {
void onInputMethodStartInputView(AutofillId imeFieldId);
void onInputMethodFinishInputView(AutofillId imeFieldId);
}
/**
* A data class wrapping IME responses for the inline suggestion request.
* A data class wrapping Autofill responses for the inline suggestion request.
*/
private static class AutofillResponse {
@NonNull
final AutofillId mAutofillId;
@NonNull
final InlineSuggestionsResponse mResponse;
AutofillResponse(@NonNull AutofillId autofillId,
@NonNull InlineSuggestionsResponse response) {
mAutofillId = autofillId;
mResponse = response;
}
}
/**
* A data class wrapping IME responses for the create inline suggestions request.
*/
private static class ImeResponse {
@NonNull

17
services/core/java/com/android/server/inputmethod/InputMethodManagerService.java
View File

@@ -109,6 +109,7 @@ import android.view.ViewGroup;
import android.view.Window;
import android.view.WindowManager.LayoutParams;
import android.view.WindowManager.LayoutParams.SoftInputModeFlags;
import android.view.autofill.AutofillId;
import android.view.inputmethod.EditorInfo;
import android.view.inputmethod.InlineSuggestionsRequest;
import android.view.inputmethod.InputBinding;
@@ -2006,7 +2007,9 @@ public class InputMethodManagerService extends IInputMethodManager.Stub
@Override
public void onInlineSuggestionsRequest(InlineSuggestionsRequest request,
IInlineSuggestionsResponseCallback callback) throws RemoteException {
IInlineSuggestionsResponseCallback callback, AutofillId imeFieldId,
boolean inputViewStarted)
throws RemoteException {
if (!mImePackageName.equals(request.getHostPackageName())) {
throw new SecurityException(
"Host package name in the provide request=[" + request.getHostPackageName()
@@ -2014,7 +2017,17 @@ public class InputMethodManagerService extends IInputMethodManager.Stub
+ "].");
}
request.setHostDisplayId(mImeDisplayId);
mCallback.onInlineSuggestionsRequest(request, callback);
mCallback.onInlineSuggestionsRequest(request, callback, imeFieldId, inputViewStarted);
}
@Override
public void onInputMethodStartInputView(AutofillId imeFieldId) throws RemoteException {
mCallback.onInputMethodStartInputView(imeFieldId);
}
@Override
public void onInputMethodFinishInputView(AutofillId imeFieldId) throws RemoteException {
mCallback.onInputMethodFinishInputView(imeFieldId);
}
}