From bc67f2e15813ec05c07f18ef961e7303fd344ff1 Mon Sep 17 00:00:00 2001 From: Adam He Date: Wed, 13 Nov 2019 14:34:56 -0800 Subject: [PATCH] API for autofill integration with keyboard. Bug: 137800469 Test: manual verification Change-Id: Id222500c373898d576661cacb7a1cb51061041d4 --- api/current.txt | 6 +- .../IInputMethodWrapper.java | 18 ++ .../InputMethodService.java | 185 +++++++++++++ .../android/service/autofill/FillRequest.java | 107 ++++++-- .../service/autofill/FillResponse.java | 40 ++- .../view/inline/InlineContentView.java | 5 +- .../view/inputmethod/InlineSuggestion.java | 2 +- .../android/view/inputmethod/InputMethod.java | 19 ++ .../android/internal/view/IInputMethod.aidl | 6 + .../autofill/AutofillManagerServiceImpl.java | 6 +- .../com/android/server/autofill/Session.java | 248 +++++++++++++++++- .../server/autofill/ui/AutoFillUI.java | 77 ++++++ .../InputMethodManagerInternal.java | 28 ++ .../InputMethodManagerService.java | 42 +++ .../MultiClientInputMethodManagerService.java | 15 ++ 15 files changed, 769 insertions(+), 35 deletions(-) diff --git a/api/current.txt b/api/current.txt index f1c03a4e576d3..8bb772dab77cc 100644 --- a/api/current.txt +++ b/api/current.txt @@ -22611,6 +22611,7 @@ package android.inputmethodservice { method public void onConfigureWindow(android.view.Window, boolean, boolean); method public android.view.View onCreateCandidatesView(); method public android.view.View onCreateExtractTextView(); + method @Nullable public android.view.inputmethod.InlineSuggestionsRequest onCreateInlineSuggestionsRequest(); method public android.inputmethodservice.AbstractInputMethodService.AbstractInputMethodImpl onCreateInputMethodInterface(); method public android.inputmethodservice.AbstractInputMethodService.AbstractInputMethodSessionImpl onCreateInputMethodSessionInterface(); method public android.view.View onCreateInputView(); @@ -22627,6 +22628,7 @@ package android.inputmethodservice { method public void onFinishInput(); method public void onFinishInputView(boolean); method public void onInitializeInterface(); + method public boolean onInlineSuggestionsResponse(@NonNull android.view.inputmethod.InlineSuggestionsResponse); method public boolean onKeyDown(int, android.view.KeyEvent); method public boolean onKeyLongPress(int, android.view.KeyEvent); method public boolean onKeyMultiple(int, int, android.view.KeyEvent); @@ -41446,7 +41448,8 @@ package android.service.autofill { method @NonNull public java.util.List getFillContexts(); method public int getFlags(); method public int getId(); - method public void writeToParcel(android.os.Parcel, int); + method @Nullable public android.view.inputmethod.InlineSuggestionsRequest getInlineSuggestionsRequest(); + method public void writeToParcel(@NonNull android.os.Parcel, int); field @NonNull public static final android.os.Parcelable.Creator CREATOR; field public static final int FLAG_COMPATIBILITY_MODE_REQUEST = 2; // 0x2 field public static final int FLAG_MANUAL_REQUEST = 1; // 0x1 @@ -41463,6 +41466,7 @@ package android.service.autofill { public static final class FillResponse.Builder { ctor public FillResponse.Builder(); method @NonNull public android.service.autofill.FillResponse.Builder addDataset(@Nullable android.service.autofill.Dataset); + method @NonNull public android.service.autofill.FillResponse.Builder addInlineSuggestionSlice(@NonNull android.app.slice.Slice); method @NonNull public android.service.autofill.FillResponse build(); method @NonNull public android.service.autofill.FillResponse.Builder disableAutofill(long); method @NonNull public android.service.autofill.FillResponse.Builder setAuthentication(@NonNull android.view.autofill.AutofillId[], @Nullable android.content.IntentSender, @Nullable android.widget.RemoteViews); diff --git a/core/java/android/inputmethodservice/IInputMethodWrapper.java b/core/java/android/inputmethodservice/IInputMethodWrapper.java index a47f601033cfa..62f0196693eae 100644 --- a/core/java/android/inputmethodservice/IInputMethodWrapper.java +++ b/core/java/android/inputmethodservice/IInputMethodWrapper.java @@ -19,6 +19,7 @@ package android.inputmethodservice; import android.annotation.BinderThread; import android.annotation.MainThread; import android.annotation.UnsupportedAppUsage; +import android.content.ComponentName; import android.content.Context; import android.content.pm.PackageManager; import android.os.Binder; @@ -28,6 +29,7 @@ import android.os.RemoteException; import android.os.ResultReceiver; import android.util.Log; import android.view.InputChannel; +import android.view.autofill.AutofillId; import android.view.inputmethod.EditorInfo; import android.view.inputmethod.InputBinding; import android.view.inputmethod.InputConnection; @@ -39,6 +41,7 @@ import android.view.inputmethod.InputMethodSubtype; import com.android.internal.inputmethod.IInputMethodPrivilegedOperations; import com.android.internal.os.HandlerCaller; import com.android.internal.os.SomeArgs; +import com.android.internal.view.IInlineSuggestionsRequestCallback; import com.android.internal.view.IInputContext; import com.android.internal.view.IInputMethod; import com.android.internal.view.IInputMethodSession; @@ -72,6 +75,7 @@ class IInputMethodWrapper extends IInputMethod.Stub private static final int DO_SHOW_SOFT_INPUT = 60; private static final int DO_HIDE_SOFT_INPUT = 70; private static final int DO_CHANGE_INPUTMETHOD_SUBTYPE = 80; + private static final int DO_CREATE_INLINE_SUGGESTIONS_REQUEST = 90; final WeakReference mTarget; final Context mContext; @@ -225,6 +229,11 @@ class IInputMethodWrapper extends IInputMethod.Stub case DO_CHANGE_INPUTMETHOD_SUBTYPE: inputMethod.changeInputMethodSubtype((InputMethodSubtype)msg.obj); return; + case DO_CREATE_INLINE_SUGGESTIONS_REQUEST: + SomeArgs args = (SomeArgs) msg.obj; + inputMethod.onCreateInlineSuggestionsRequest((ComponentName) args.arg1, + (AutofillId) args.arg2, (IInlineSuggestionsRequestCallback) args.arg3); + return; } Log.w(TAG, "Unhandled message code: " + msg.what); } @@ -265,6 +274,15 @@ class IInputMethodWrapper extends IInputMethod.Stub mCaller.obtainMessageIOO(DO_INITIALIZE_INTERNAL, displayId, token, privOps)); } + @BinderThread + @Override + public void onCreateInlineSuggestionsRequest(ComponentName componentName, AutofillId autofillId, + IInlineSuggestionsRequestCallback cb) { + mCaller.executeOrSendMessage( + mCaller.obtainMessageOOO(DO_CREATE_INLINE_SUGGESTIONS_REQUEST, componentName, + autofillId, cb)); + } + @BinderThread @Override public void bindInput(InputBinding binding) { diff --git a/core/java/android/inputmethodservice/InputMethodService.java b/core/java/android/inputmethodservice/InputMethodService.java index 43842c5c3403e..da70359332a10 100644 --- a/core/java/android/inputmethodservice/InputMethodService.java +++ b/core/java/android/inputmethodservice/InputMethodService.java @@ -22,6 +22,8 @@ import static android.view.ViewGroup.LayoutParams.WRAP_CONTENT; import static android.view.ViewRootImpl.NEW_INSETS_MODE_NONE; import static android.view.WindowManager.LayoutParams.FLAG_DRAWS_SYSTEM_BAR_BACKGROUNDS; +import static com.android.internal.util.function.pooled.PooledLambda.obtainMessage; + import static java.lang.annotation.RetentionPolicy.SOURCE; import android.annotation.AnyThread; @@ -34,6 +36,7 @@ import android.annotation.Nullable; import android.annotation.UnsupportedAppUsage; import android.app.ActivityManager; import android.app.Dialog; +import android.content.ComponentName; import android.content.Context; import android.content.res.Configuration; import android.content.res.Resources; @@ -46,6 +49,8 @@ import android.os.Build; import android.os.Bundle; import android.os.Handler; import android.os.IBinder; +import android.os.Looper; +import android.os.RemoteException; import android.os.ResultReceiver; import android.os.SystemClock; import android.provider.Settings; @@ -68,11 +73,14 @@ import android.view.ViewTreeObserver; import android.view.Window; import android.view.WindowManager; import android.view.animation.AnimationUtils; +import android.view.autofill.AutofillId; import android.view.inputmethod.CompletionInfo; import android.view.inputmethod.CursorAnchorInfo; import android.view.inputmethod.EditorInfo; import android.view.inputmethod.ExtractedText; import android.view.inputmethod.ExtractedTextRequest; +import android.view.inputmethod.InlineSuggestionsRequest; +import android.view.inputmethod.InlineSuggestionsResponse; import android.view.inputmethod.InputBinding; import android.view.inputmethod.InputConnection; import android.view.inputmethod.InputContentInfo; @@ -89,11 +97,14 @@ import com.android.internal.inputmethod.IInputContentUriToken; import com.android.internal.inputmethod.IInputMethodPrivilegedOperations; import com.android.internal.inputmethod.InputMethodPrivilegedOperations; import com.android.internal.inputmethod.InputMethodPrivilegedOperationsRegistry; +import com.android.internal.view.IInlineSuggestionsRequestCallback; +import com.android.internal.view.IInlineSuggestionsResponseCallback; import java.io.FileDescriptor; import java.io.PrintWriter; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; +import java.lang.ref.WeakReference; import java.util.Collections; /** @@ -434,6 +445,14 @@ public class InputMethodService extends AbstractInputMethodService { final Insets mTmpInsets = new Insets(); final int[] mTmpLocation = new int[2]; + @Nullable + private InlineSuggestionsRequestInfo mInlineSuggestionsRequestInfo = null; + + @Nullable + private InlineSuggestionsResponseCallbackImpl mInlineSuggestionsResponseCallback = null; + + private final Handler mHandler = new Handler(Looper.getMainLooper(), null, true); + final ViewTreeObserver.OnComputeInternalInsetsListener mInsetsComputer = info -> { onComputeInsets(mTmpInsets); if (isExtractViewShown()) { @@ -491,6 +510,18 @@ public class InputMethodService extends AbstractInputMethodService { attachToken(token); } + /** + * {@inheritDoc} + * @hide + */ + @MainThread + @Override + public void onCreateInlineSuggestionsRequest(ComponentName componentName, + AutofillId autofillId, IInlineSuggestionsRequestCallback cb) { + Log.d(TAG, "InputMethodService received onCreateInlineSuggestionsRequest()"); + handleOnCreateInlineSuggestionsRequest(componentName, autofillId, cb); + } + /** * {@inheritDoc} */ @@ -668,6 +699,103 @@ public class InputMethodService extends AbstractInputMethodService { } } + // TODO(b/137800469): Add detailed docs explaining the inline suggestions process. + /** + * Returns an {@link InlineSuggestionsRequest} to be sent to Autofill. + * + *

Should be implemented by subclasses.

+ */ + public @Nullable InlineSuggestionsRequest onCreateInlineSuggestionsRequest() { + return null; + } + + /** + * Called when Autofill responds back with {@link InlineSuggestionsResponse} containing + * inline suggestions. + * + *

Should be implemented by subclasses.

+ * + * @param response {@link InlineSuggestionsResponse} passed back by Autofill. + * @return Whether the IME will use and render the inline suggestions. + */ + public boolean onInlineSuggestionsResponse(@NonNull InlineSuggestionsResponse response) { + return false; + } + + /** + * Returns whether inline suggestions are enabled on this service. + * + * TODO(b/137800469): check XML for value. + */ + private boolean isInlineSuggestionsEnabled() { + return true; + } + + /** + * Sends an {@link InlineSuggestionsRequest} obtained from + * {@link #onCreateInlineSuggestionsRequest()} to the current Autofill Session through + * {@link IInlineSuggestionsRequestCallback#onInlineSuggestionsRequest}. + */ + private void makeInlineSuggestionsRequest() { + if (mInlineSuggestionsRequestInfo == null) { + Log.w(TAG, "makeInlineSuggestionsRequest() called with null requestInfo cache"); + return; + } + + final IInlineSuggestionsRequestCallback requestCallback = + mInlineSuggestionsRequestInfo.mCallback; + try { + final InlineSuggestionsRequest request = onCreateInlineSuggestionsRequest(); + if (request == null) { + Log.w(TAG, "onCreateInlineSuggestionsRequest() returned null request"); + requestCallback.onInlineSuggestionsUnsupported(); + } else { + if (mInlineSuggestionsResponseCallback == null) { + mInlineSuggestionsResponseCallback = + new InlineSuggestionsResponseCallbackImpl(this, + mInlineSuggestionsRequestInfo.mComponentName, + mInlineSuggestionsRequestInfo.mFocusedId); + } + requestCallback.onInlineSuggestionsRequest(request, + mInlineSuggestionsResponseCallback); + } + } catch (RemoteException e) { + Log.w(TAG, "makeInlinedSuggestionsRequest() remote exception:" + e); + } + } + + private void handleOnCreateInlineSuggestionsRequest(@NonNull ComponentName componentName, + @NonNull AutofillId autofillId, @NonNull IInlineSuggestionsRequestCallback callback) { + mInlineSuggestionsRequestInfo = new InlineSuggestionsRequestInfo(componentName, autofillId, + callback); + + if (!isInlineSuggestionsEnabled()) { + try { + callback.onInlineSuggestionsUnsupported(); + } catch (RemoteException e) { + Log.w(TAG, "handleMakeInlineSuggestionsRequest() RemoteException:" + e); + } + return; + } + + if (!mInputStarted) { + Log.w(TAG, "onStartInput() not called yet"); + return; + } + + makeInlineSuggestionsRequest(); + } + + private void handleOnInlineSuggestionsResponse(@NonNull ComponentName componentName, + @NonNull AutofillId autofillId, @NonNull InlineSuggestionsResponse response) { + if (!mInlineSuggestionsRequestInfo.validate(componentName)) { + Log.d(TAG, "Response component=" + componentName + " differs from request component=" + + mInlineSuggestionsRequestInfo.mComponentName + ", ignoring response"); + return; + } + onInlineSuggestionsResponse(response); + } + private void notifyImeHidden() { setImeWindowStatus(IME_ACTIVE | IME_INVISIBLE, mBackDisposition); onPreRenderedWindowVisibilityChanged(false /* setVisible */); @@ -685,6 +813,63 @@ public class InputMethodService extends AbstractInputMethodService { inputFrameRootView.setSystemGestureExclusionRects(Collections.singletonList(r)); } + /** + * Internal implementation of {@link IInlineSuggestionsResponseCallback}. + */ + private static final class InlineSuggestionsResponseCallbackImpl + extends IInlineSuggestionsResponseCallback.Stub { + private final WeakReference mInputMethodService; + + private final ComponentName mRequestComponentName; + private final AutofillId mRequestAutofillId; + + private InlineSuggestionsResponseCallbackImpl(InputMethodService inputMethodService, + ComponentName componentName, AutofillId autofillId) { + mInputMethodService = new WeakReference<>(inputMethodService); + mRequestComponentName = componentName; + mRequestAutofillId = autofillId; + } + + @Override + public void onInlineSuggestionsResponse(InlineSuggestionsResponse response) + throws RemoteException { + final InputMethodService service = mInputMethodService.get(); + if (service != null) { + service.mHandler.sendMessage(obtainMessage( + InputMethodService::handleOnInlineSuggestionsResponse, service, + mRequestComponentName, mRequestAutofillId, response)); + } + } + } + + /** + * Information about incoming requests from Autofill Frameworks for inline suggestions. + */ + private static final class InlineSuggestionsRequestInfo { + final ComponentName mComponentName; + final AutofillId mFocusedId; + final IInlineSuggestionsRequestCallback mCallback; + + InlineSuggestionsRequestInfo(ComponentName componentName, AutofillId focusedId, + IInlineSuggestionsRequestCallback callback) { + this.mComponentName = componentName; + this.mFocusedId = focusedId; + this.mCallback = callback; + } + + /** + * Returns whether the cached {@link ComponentName} matches the passed in activity. + */ + public boolean validate(ComponentName componentName) { + final boolean result = componentName.equals(mComponentName); + if (!result) { + Log.d(TAG, "Cached request info ComponentName=" + mComponentName + + " differs from received ComponentName=" + componentName); + } + return result; + } + } + /** * Concrete implementation of * {@link AbstractInputMethodService.AbstractInputMethodSessionImpl} that provides diff --git a/core/java/android/service/autofill/FillRequest.java b/core/java/android/service/autofill/FillRequest.java index e53ebada55fb9..72e9ad047ed7a 100644 --- a/core/java/android/service/autofill/FillRequest.java +++ b/core/java/android/service/autofill/FillRequest.java @@ -23,6 +23,7 @@ import android.os.Bundle; import android.os.Parcel; import android.os.Parcelable; import android.view.View; +import android.view.inputmethod.InlineSuggestionsRequest; import com.android.internal.util.DataClass; import com.android.internal.util.Preconditions; @@ -116,20 +117,34 @@ public final class FillRequest implements Parcelable { */ private final @RequestFlags int mFlags; + /** + * Gets the {@link android.view.inputmethod.InlineSuggestionsRequest} associated + * with this request. + * + * TODO(b/137800469): Add more doc describing how to handle the inline suggestions request. + * + * @return the suggestionspec + */ + private final @Nullable InlineSuggestionsRequest mInlineSuggestionsRequest; + private void onConstructed() { Preconditions.checkCollectionElementsNotNull(mFillContexts, "contexts"); } - // Code below generated by codegen v1.0.0. + // Code below generated by codegen v1.0.14. // // DO NOT MODIFY! + // CHECKSTYLE:OFF Generated code // // To regenerate run: // $ codegen $ANDROID_BUILD_TOP/frameworks/base/core/java/android/service/autofill/FillRequest.java // - // CHECKSTYLE:OFF Generated code + // To exclude the generated code from IntelliJ auto-formatting enable (one-time): + // Settings > Editor > Code Style > Formatter Control + //@formatter:off + /** @hide */ @IntDef(flag = true, prefix = "FLAG_", value = { @@ -184,6 +199,11 @@ public final class FillRequest implements Parcelable { * * @return any combination of {@link #FLAG_MANUAL_REQUEST} and * {@link #FLAG_COMPATIBILITY_MODE_REQUEST}. + * @param inlineSuggestionsRequest + * Gets the {@link android.view.inputmethod.InlineSuggestionsRequest} associated + * with this request. + * + * TODO(b/137800469): Add more doc describing how to handle the inline suggestions request. * @hide */ @DataClass.Generated.Member @@ -191,7 +211,8 @@ public final class FillRequest implements Parcelable { int id, @NonNull List fillContexts, @Nullable Bundle clientState, - @RequestFlags int flags) { + @RequestFlags int flags, + @Nullable InlineSuggestionsRequest inlineSuggestionsRequest) { this.mId = id; this.mFillContexts = fillContexts; com.android.internal.util.AnnotationValidations.validate( @@ -203,6 +224,7 @@ public final class FillRequest implements Parcelable { mFlags, FLAG_MANUAL_REQUEST | FLAG_COMPATIBILITY_MODE_REQUEST); + this.mInlineSuggestionsRequest = inlineSuggestionsRequest; onConstructed(); } @@ -256,6 +278,19 @@ public final class FillRequest implements Parcelable { return mFlags; } + /** + * Gets the {@link android.view.inputmethod.InlineSuggestionsRequest} associated + * with this request. + * + * TODO(b/137800469): Add more doc describing how to handle the inline suggestions request. + * + * @return the suggestionspec + */ + @DataClass.Generated.Member + public @Nullable InlineSuggestionsRequest getInlineSuggestionsRequest() { + return mInlineSuggestionsRequest; + } + @Override @DataClass.Generated.Member public String toString() { @@ -266,29 +301,63 @@ public final class FillRequest implements Parcelable { "id = " + mId + ", " + "fillContexts = " + mFillContexts + ", " + "clientState = " + mClientState + ", " + - "flags = " + requestFlagsToString(mFlags) + + "flags = " + requestFlagsToString(mFlags) + ", " + + "inlineSuggestionsRequest = " + mInlineSuggestionsRequest + " }"; } @Override @DataClass.Generated.Member - public void writeToParcel(Parcel dest, int flags) { + public void writeToParcel(@NonNull Parcel dest, int flags) { // You can override field parcelling by defining methods like: // void parcelFieldName(Parcel dest, int flags) { ... } byte flg = 0; if (mClientState != null) flg |= 0x4; + if (mInlineSuggestionsRequest != null) flg |= 0x10; dest.writeByte(flg); dest.writeInt(mId); dest.writeParcelableList(mFillContexts, flags); if (mClientState != null) dest.writeBundle(mClientState); dest.writeInt(mFlags); + if (mInlineSuggestionsRequest != null) dest.writeTypedObject(mInlineSuggestionsRequest, flags); } @Override @DataClass.Generated.Member public int describeContents() { return 0; } + /** @hide */ + @SuppressWarnings({"unchecked", "RedundantCast"}) + @DataClass.Generated.Member + /* package-private */ FillRequest(@NonNull Parcel in) { + // You can override field unparcelling by defining methods like: + // static FieldType unparcelFieldName(Parcel in) { ... } + + byte flg = in.readByte(); + int id = in.readInt(); + List fillContexts = new ArrayList<>(); + in.readParcelableList(fillContexts, FillContext.class.getClassLoader()); + Bundle clientState = (flg & 0x4) == 0 ? null : in.readBundle(); + int flags = in.readInt(); + InlineSuggestionsRequest inlineSuggestionsRequest = (flg & 0x10) == 0 ? null : (InlineSuggestionsRequest) in.readTypedObject(InlineSuggestionsRequest.CREATOR); + + this.mId = id; + this.mFillContexts = fillContexts; + com.android.internal.util.AnnotationValidations.validate( + NonNull.class, null, mFillContexts); + this.mClientState = clientState; + this.mFlags = flags; + + Preconditions.checkFlagsArgument( + mFlags, + FLAG_MANUAL_REQUEST + | FLAG_COMPATIBILITY_MODE_REQUEST); + this.mInlineSuggestionsRequest = inlineSuggestionsRequest; + + onConstructed(); + } + @DataClass.Generated.Member public static final @NonNull Parcelable.Creator CREATOR = new Parcelable.Creator() { @@ -298,31 +367,21 @@ public final class FillRequest implements Parcelable { } @Override - @SuppressWarnings({"unchecked", "RedundantCast"}) - public FillRequest createFromParcel(Parcel in) { - // You can override field unparcelling by defining methods like: - // static FieldType unparcelFieldName(Parcel in) { ... } - - byte flg = in.readByte(); - int id = in.readInt(); - List fillContexts = new ArrayList<>(); - in.readParcelableList(fillContexts, FillContext.class.getClassLoader()); - Bundle clientState = (flg & 0x4) == 0 ? null : in.readBundle(); - int flags = in.readInt(); - return new FillRequest( - id, - fillContexts, - clientState, - flags); + public FillRequest createFromParcel(@NonNull Parcel in) { + return new FillRequest(in); } }; @DataClass.Generated( - time = 1565152134349L, - codegenVersion = "1.0.0", + time = 1575928271155L, + codegenVersion = "1.0.14", sourceFile = "frameworks/base/core/java/android/service/autofill/FillRequest.java", - inputSignatures = "public static final @android.service.autofill.FillRequest.RequestFlags int FLAG_MANUAL_REQUEST\npublic static final @android.service.autofill.FillRequest.RequestFlags int FLAG_COMPATIBILITY_MODE_REQUEST\npublic static final int INVALID_REQUEST_ID\nprivate final int mId\nprivate final @android.annotation.NonNull java.util.List mFillContexts\nprivate final @android.annotation.Nullable android.os.Bundle mClientState\nprivate final @android.service.autofill.FillRequest.RequestFlags int mFlags\nprivate void onConstructed()\nclass FillRequest extends java.lang.Object implements [android.os.Parcelable]\n@com.android.internal.util.DataClass(genToString=true, genHiddenConstructor=true, genHiddenConstDefs=true)") + inputSignatures = "public static final @android.service.autofill.FillRequest.RequestFlags int FLAG_MANUAL_REQUEST\npublic static final @android.service.autofill.FillRequest.RequestFlags int FLAG_COMPATIBILITY_MODE_REQUEST\npublic static final int INVALID_REQUEST_ID\nprivate final int mId\nprivate final @android.annotation.NonNull java.util.List mFillContexts\nprivate final @android.annotation.Nullable android.os.Bundle mClientState\nprivate final @android.service.autofill.FillRequest.RequestFlags int mFlags\nprivate final @android.annotation.Nullable android.view.inputmethod.InlineSuggestionsRequest mInlineSuggestionsRequest\nprivate void onConstructed()\nclass FillRequest extends java.lang.Object implements [android.os.Parcelable]\n@com.android.internal.util.DataClass(genToString=true, genHiddenConstructor=true, genHiddenConstDefs=true)") @Deprecated private void __metadata() {} + + //@formatter:on + // End of generated code + } diff --git a/core/java/android/service/autofill/FillResponse.java b/core/java/android/service/autofill/FillResponse.java index c99fe6149aab3..02a6390a08bb7 100644 --- a/core/java/android/service/autofill/FillResponse.java +++ b/core/java/android/service/autofill/FillResponse.java @@ -25,6 +25,7 @@ import android.annotation.NonNull; import android.annotation.Nullable; import android.annotation.TestApi; import android.app.Activity; +import android.app.slice.Slice; import android.content.IntentSender; import android.content.pm.ParceledListSlice; import android.os.Bundle; @@ -86,6 +87,7 @@ public final class FillResponse implements Parcelable { private int mRequestId; private final @Nullable UserData mUserData; private final @Nullable int[] mCancelIds; + private final @Nullable ParceledListSlice mInlineSuggestionSlices; private FillResponse(@NonNull Builder builder) { mDatasets = (builder.mDatasets != null) ? new ParceledListSlice<>(builder.mDatasets) : null; @@ -103,6 +105,8 @@ public final class FillResponse implements Parcelable { mRequestId = INVALID_REQUEST_ID; mUserData = builder.mUserData; mCancelIds = builder.mCancelIds; + mInlineSuggestionSlices = (builder.mInlineSuggestionSlices != null) + ? new ParceledListSlice<>(builder.mInlineSuggestionSlices) : null; } /** @hide */ @@ -195,6 +199,11 @@ public final class FillResponse implements Parcelable { return mCancelIds; } + /** @hide */ + public List getInlineSuggestionSlices() { + return (mInlineSuggestionSlices != null) ? mInlineSuggestionSlices.getList() : null; + } + /** * Builder for {@link FillResponse} objects. You must to provide at least * one dataset or set an authentication intent with a presentation view. @@ -215,6 +224,7 @@ public final class FillResponse implements Parcelable { private boolean mDestroyed; private UserData mUserData; private int[] mCancelIds; + private ArrayList mInlineSuggestionSlices; /** * Triggers a custom UI before before autofilling the screen with any data set in this @@ -569,6 +579,20 @@ public final class FillResponse implements Parcelable { return this; } + /** + * TODO(b/137800469): add javadoc + */ + @NonNull + public Builder addInlineSuggestionSlice(@NonNull Slice inlineSuggestionSlice) { + throwIfDestroyed(); + throwIfAuthenticationCalled(); + if (mInlineSuggestionSlices == null) { + mInlineSuggestionSlices = new ArrayList<>(); + } + mInlineSuggestionSlices.add(inlineSuggestionSlice); + return this; + } + /** * Builds a new {@link FillResponse} instance. * @@ -670,7 +694,9 @@ public final class FillResponse implements Parcelable { if (mCancelIds != null) { builder.append(", mCancelIds=").append(mCancelIds.length); } - + if (mInlineSuggestionSlices != null) { + builder.append(", inlinedSuggestions=").append(mInlineSuggestionSlices.getList()); + } return builder.append("]").toString(); } @@ -699,7 +725,7 @@ public final class FillResponse implements Parcelable { parcel.writeParcelableArray(mFieldClassificationIds, flags); parcel.writeInt(mFlags); parcel.writeIntArray(mCancelIds); - + parcel.writeParcelable(mInlineSuggestionSlices, flags); parcel.writeInt(mRequestId); } @@ -755,6 +781,16 @@ public final class FillResponse implements Parcelable { final int[] cancelIds = parcel.createIntArray(); builder.setCancelTargetIds(cancelIds); + final ParceledListSlice parceledInlineSuggestionSlices = + parcel.readParcelable(null); + if (parceledInlineSuggestionSlices != null) { + final List inlineSuggestionSlices = parceledInlineSuggestionSlices.getList(); + final int size = inlineSuggestionSlices.size(); + for (int i = 0; i < size; i++) { + builder.addInlineSuggestionSlice(inlineSuggestionSlices.get(i)); + } + } + final FillResponse response = builder.build(); response.setRequestId(parcel.readInt()); diff --git a/core/java/android/view/inline/InlineContentView.java b/core/java/android/view/inline/InlineContentView.java index 4bc2176e82792..b143fad778ec0 100644 --- a/core/java/android/view/inline/InlineContentView.java +++ b/core/java/android/view/inline/InlineContentView.java @@ -50,7 +50,10 @@ public class InlineContentView extends SurfaceView { @Override public void surfaceDestroyed(SurfaceHolder holder) { - // TODO(b/137800469): implement this. + new SurfaceControl.Transaction() + .setVisibility(surfaceControl, false) + .reparent(surfaceControl, null) + .apply(); } }); } diff --git a/core/java/android/view/inputmethod/InlineSuggestion.java b/core/java/android/view/inputmethod/InlineSuggestion.java index 25e34d36aa9a1..c10144e6ee25c 100644 --- a/core/java/android/view/inputmethod/InlineSuggestion.java +++ b/core/java/android/view/inputmethod/InlineSuggestion.java @@ -271,7 +271,7 @@ public final class InlineSuggestion implements Parcelable { }; @DataClass.Generated( - time = 1574446220398L, + time = 1575933636929L, codegenVersion = "1.0.14", sourceFile = "frameworks/base/core/java/android/view/inputmethod/InlineSuggestion.java", inputSignatures = "private static final java.lang.String TAG\nprivate final @android.annotation.NonNull android.view.inputmethod.InlineSuggestionInfo mInfo\nprivate final @android.annotation.Nullable com.android.internal.view.inline.IInlineContentProvider mContentProvider\npublic void inflate(android.content.Context,android.util.Size,java.util.concurrent.Executor,java.util.function.Consumer)\nclass InlineSuggestion extends java.lang.Object implements [android.os.Parcelable]\n@com.android.internal.util.DataClass(genEqualsHashCode=true, genToString=true, genHiddenConstDefs=true, genHiddenConstructor=true)") diff --git a/core/java/android/view/inputmethod/InputMethod.java b/core/java/android/view/inputmethod/InputMethod.java index 112653aa34e38..1e5a3b0276090 100644 --- a/core/java/android/view/inputmethod/InputMethod.java +++ b/core/java/android/view/inputmethod/InputMethod.java @@ -21,11 +21,16 @@ import android.annotation.NonNull; import android.annotation.Nullable; import android.annotation.SdkConstant; import android.annotation.SdkConstant.SdkConstantType; +import android.content.ComponentName; import android.inputmethodservice.InputMethodService; import android.os.IBinder; +import android.os.RemoteException; import android.os.ResultReceiver; +import android.util.Log; +import android.view.autofill.AutofillId; import com.android.internal.inputmethod.IInputMethodPrivilegedOperations; +import com.android.internal.view.IInlineSuggestionsRequestCallback; /** * The InputMethod interface represents an input method which can generate key @@ -103,6 +108,20 @@ public interface InputMethod { attachToken(token); } + /** + * Called to notify the IME that Autofill Frameworks requested an inline suggestions request. + * + * @hide + */ + default void onCreateInlineSuggestionsRequest(ComponentName componentName, + AutofillId autofillId, IInlineSuggestionsRequestCallback cb) { + try { + cb.onInlineSuggestionsUnsupported(); + } catch (RemoteException e) { + Log.w("InputMethod", "RemoteException calling onInlineSuggestionsUnsupported: " + e); + } + } + /** * Called first thing after an input method is created, this supplies a * unique token for the session it has with the system service. It is diff --git a/core/java/com/android/internal/view/IInputMethod.aidl b/core/java/com/android/internal/view/IInputMethod.aidl index 2ee902ab6468a..58aaa80b51bea 100644 --- a/core/java/com/android/internal/view/IInputMethod.aidl +++ b/core/java/com/android/internal/view/IInputMethod.aidl @@ -16,13 +16,16 @@ package com.android.internal.view; +import android.content.ComponentName; import android.os.IBinder; import android.os.ResultReceiver; +import android.view.autofill.AutofillId; import android.view.InputChannel; import android.view.inputmethod.EditorInfo; import android.view.inputmethod.InputBinding; import android.view.inputmethod.InputMethodSubtype; import com.android.internal.inputmethod.IInputMethodPrivilegedOperations; +import com.android.internal.view.IInlineSuggestionsRequestCallback; import com.android.internal.view.IInputContext; import com.android.internal.view.IInputMethodSession; import com.android.internal.view.IInputSessionCallback; @@ -35,6 +38,9 @@ import com.android.internal.view.IInputSessionCallback; oneway interface IInputMethod { void initializeInternal(IBinder token, int displayId, IInputMethodPrivilegedOperations privOps); + void onCreateInlineSuggestionsRequest(in ComponentName componentName, in AutofillId autofillId, + in IInlineSuggestionsRequestCallback cb); + void bindInput(in InputBinding binding); void unbindInput(); diff --git a/services/autofill/java/com/android/server/autofill/AutofillManagerServiceImpl.java b/services/autofill/java/com/android/server/autofill/AutofillManagerServiceImpl.java index d7ed2e9abde44..202f90068fc1a 100644 --- a/services/autofill/java/com/android/server/autofill/AutofillManagerServiceImpl.java +++ b/services/autofill/java/com/android/server/autofill/AutofillManagerServiceImpl.java @@ -82,6 +82,7 @@ import com.android.server.autofill.AutofillManagerService.AutofillCompatState; import com.android.server.autofill.RemoteAugmentedAutofillService.RemoteAugmentedAutofillServiceCallbacks; import com.android.server.autofill.ui.AutoFillUI; import com.android.server.infra.AbstractPerUserSystemService; +import com.android.server.inputmethod.InputMethodManagerInternal; import java.io.PrintWriter; import java.util.ArrayList; @@ -168,6 +169,8 @@ final class AutofillManagerServiceImpl @Nullable private ServiceInfo mRemoteAugmentedAutofillServiceInfo; + private final InputMethodManagerInternal mInputMethodManagerInternal; + AutofillManagerServiceImpl(AutofillManagerService master, Object lock, LocalLog uiLatencyHistory, LocalLog wtfHistory, int userId, AutoFillUI ui, AutofillCompatState autofillCompatState, @@ -179,6 +182,7 @@ final class AutofillManagerServiceImpl mUi = ui; mFieldClassificationStrategy = new FieldClassificationStrategy(getContext(), userId); mAutofillCompatState = autofillCompatState; + mInputMethodManagerInternal = LocalServices.getService(InputMethodManagerInternal.class); updateLocked(disabled); } @@ -493,7 +497,7 @@ final class AutofillManagerServiceImpl sessionId, taskId, uid, activityToken, appCallbackToken, hasCallback, mUiLatencyHistory, mWtfHistory, serviceComponentName, componentName, compatMode, bindInstantServiceAllowed, forAugmentedAutofillOnly, - flags); + flags, mInputMethodManagerInternal); mSessions.put(newSession.id, newSession); return newSession; diff --git a/services/autofill/java/com/android/server/autofill/Session.java b/services/autofill/java/com/android/server/autofill/Session.java index 3b2da911a8490..67bcccd1d7def 100644 --- a/services/autofill/java/com/android/server/autofill/Session.java +++ b/services/autofill/java/com/android/server/autofill/Session.java @@ -42,6 +42,8 @@ import android.app.IAssistDataReceiver; import android.app.assist.AssistStructure; import android.app.assist.AssistStructure.AutofillOverlay; import android.app.assist.AssistStructure.ViewNode; +import android.app.slice.Slice; +import android.app.slice.SliceItem; import android.content.ComponentName; import android.content.Context; import android.content.Intent; @@ -80,6 +82,8 @@ import android.text.TextUtils; import android.util.ArrayMap; import android.util.ArraySet; import android.util.LocalLog; +import android.util.Log; +import android.util.Size; import android.util.Slog; import android.util.SparseArray; import android.util.TimeUtils; @@ -90,22 +94,38 @@ import android.view.autofill.AutofillManager.SmartSuggestionMode; import android.view.autofill.AutofillValue; import android.view.autofill.IAutoFillManagerClient; import android.view.autofill.IAutofillWindowPresenter; +import android.view.inline.InlinePresentationSpec; +import android.view.inputmethod.InlineSuggestion; +import android.view.inputmethod.InlineSuggestionInfo; +import android.view.inputmethod.InlineSuggestionsRequest; +import android.view.inputmethod.InlineSuggestionsResponse; import com.android.internal.R; import com.android.internal.annotations.GuardedBy; import com.android.internal.logging.MetricsLogger; import com.android.internal.logging.nano.MetricsProto.MetricsEvent; import com.android.internal.util.ArrayUtils; +import com.android.internal.view.IInlineSuggestionsRequestCallback; +import com.android.internal.view.IInlineSuggestionsResponseCallback; +import com.android.internal.view.inline.IInlineContentCallback; +import com.android.internal.view.inline.IInlineContentProvider; import com.android.server.autofill.ui.AutoFillUI; import com.android.server.autofill.ui.PendingUi; +import com.android.server.inputmethod.InputMethodManagerInternal; import java.io.PrintWriter; +import java.lang.ref.WeakReference; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Objects; +import java.util.concurrent.CancellationException; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; /** @@ -290,6 +310,23 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState @GuardedBy("mLock") private boolean mForAugmentedAutofillOnly; + @NonNull + private final InputMethodManagerInternal mInputMethodManagerInternal; + + @GuardedBy("mLock") + @Nullable + private CompletableFuture mSuggestionsRequestFuture; + + @GuardedBy("mLock") + @Nullable + private CompletableFuture + mInlineSuggestionsResponseCallbackFuture; + + @Nullable + private InlineSuggestionsRequestCallback mInlineSuggestionsRequestCallback; + + private static final int INLINE_REQUEST_TIMEOUT_MS = 1000; + /** * Receiver of assist data from the app's {@link Activity}. */ @@ -386,7 +423,23 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState final ArrayList contexts = mergePreviousSessionLocked(/* forSave= */ false); - request = new FillRequest(requestId, contexts, mClientState, flags); + + InlineSuggestionsRequest suggestionsRequest = null; + if (mSuggestionsRequestFuture != null) { + try { + suggestionsRequest = mSuggestionsRequestFuture.get( + INLINE_REQUEST_TIMEOUT_MS, TimeUnit.MILLISECONDS); + } catch (TimeoutException e) { + Log.w(TAG, "Exception getting inline suggestions request in time: " + e); + } catch (CancellationException e) { + Log.w(TAG, "Inline suggestions request cancelled"); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + } + + request = new FillRequest(requestId, contexts, mClientState, flags, + suggestionsRequest); } mRemoteFillService.onFillRequest(request); @@ -568,6 +621,70 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState } } + /** + * Returns whether inline suggestions are enabled for Autofill. + */ + // TODO(b/137800469): Implement this + private boolean isInlineSuggestionsEnabled() { + return true; + } + + /** + * Ask the IME to make an inline suggestions request if enabled. + */ + private void maybeRequestInlineSuggestionsRequestThenFillLocked(@NonNull ViewState viewState, + int newState, int flags) { + if (isInlineSuggestionsEnabled()) { + mSuggestionsRequestFuture = new CompletableFuture<>(); + mInlineSuggestionsResponseCallbackFuture = new CompletableFuture<>(); + + if (mInlineSuggestionsRequestCallback == null) { + mInlineSuggestionsRequestCallback = new InlineSuggestionsRequestCallback(this); + } + + mInputMethodManagerInternal.onCreateInlineSuggestionsRequest( + mComponentName, mCurrentViewId, mInlineSuggestionsRequestCallback); + } + + requestNewFillResponseLocked(viewState, newState, flags); + } + + private static final class InlineSuggestionsRequestCallback + extends IInlineSuggestionsRequestCallback.Stub { + private final WeakReference mSession; + + private InlineSuggestionsRequestCallback(Session session) { + mSession = new WeakReference<>(session); + } + + @Override + public void onInlineSuggestionsUnsupported() throws RemoteException { + Log.i(TAG, "inline suggestions request unsupported, " + + "falling back to regular autofill"); + final Session session = mSession.get(); + if (session != null) { + synchronized (session.mLock) { + session.mSuggestionsRequestFuture.cancel(true); + session.mInlineSuggestionsResponseCallbackFuture.cancel(true); + } + } + } + + @Override + public void onInlineSuggestionsRequest(InlineSuggestionsRequest request, + IInlineSuggestionsResponseCallback callback) throws RemoteException { + Log.i(TAG, "onInlineSuggestionsRequest() received: " + + request); + final Session session = mSession.get(); + if (session != null) { + synchronized (session.mLock) { + session.mSuggestionsRequestFuture.complete(request); + session.mInlineSuggestionsResponseCallbackFuture.complete(callback); + } + } + } + } + /** * Reads a new structure and then request a new fill response from the fill service. */ @@ -584,6 +701,7 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState triggerAugmentedAutofillLocked(); return; } + viewState.setState(newState); int requestId; @@ -636,7 +754,8 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState @NonNull IBinder client, boolean hasCallback, @NonNull LocalLog uiLatencyHistory, @NonNull LocalLog wtfHistory, @Nullable ComponentName serviceComponentName, @NonNull ComponentName componentName, boolean compatMode, - boolean bindInstantServiceAllowed, boolean forAugmentedAutofillOnly, int flags) { + boolean bindInstantServiceAllowed, boolean forAugmentedAutofillOnly, int flags, + @NonNull InputMethodManagerInternal inputMethodManagerInternal) { if (sessionId < 0) { wtf(null, "Non-positive sessionId: %s", sessionId); } @@ -661,6 +780,8 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState mForAugmentedAutofillOnly = forAugmentedAutofillOnly; setClientLocked(client); + mInputMethodManagerInternal = inputMethodManagerInternal; + mMetricsLogger.write(newLogMaker(MetricsEvent.AUTOFILL_SESSION_STARTED) .addTaggedData(MetricsEvent.FIELD_AUTOFILL_FLAGS, flags)); } @@ -2208,7 +2329,8 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState if ((flags & FLAG_MANUAL_REQUEST) != 0) { mForAugmentedAutofillOnly = false; if (sDebug) Slog.d(TAG, "Re-starting session on view " + id + " and flags " + flags); - requestNewFillResponseLocked(viewState, ViewState.STATE_RESTARTED_SESSION, flags); + maybeRequestInlineSuggestionsRequestThenFillLocked(viewState, + ViewState.STATE_RESTARTED_SESSION, flags); return; } @@ -2218,7 +2340,8 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState Slog.d(TAG, "Starting partition or augmented request for view id " + id + ": " + viewState.getStateAsString()); } - requestNewFillResponseLocked(viewState, ViewState.STATE_STARTED_PARTITION, flags); + maybeRequestInlineSuggestionsRequestThenFillLocked(viewState, + ViewState.STATE_STARTED_PARTITION, flags); } else { if (sVerbose) { Slog.v(TAG, "Not starting new partition for view " + id + ": " @@ -2325,7 +2448,8 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState // View is triggering autofill. mCurrentViewId = viewState.id; viewState.update(value, virtualBounds, flags); - requestNewFillResponseLocked(viewState, ViewState.STATE_STARTED_SESSION, flags); + maybeRequestInlineSuggestionsRequestThenFillLocked(viewState, + ViewState.STATE_STARTED_SESSION, flags); break; case ACTION_VALUE_CHANGED: if (mCompatMode && (viewState.getState() & ViewState.STATE_URL_BAR) != 0) { @@ -2527,6 +2651,16 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState wtf(null, "onFillReady(): no service label or icon"); return; } + + final List inlineSuggestionSlices = response.getInlineSuggestionSlices(); + if (inlineSuggestionSlices != null) { + if (requestShowInlineSuggestions(inlineSuggestionSlices, response)) { + //TODO(b/137800469): Add logging instead of bypassing below logic. + return; + } + } + + getUiForShowing().showFillUi(filledId, response, filterText, mService.getServicePackageName(), mComponentName, serviceLabel, serviceIcon, this, id, mCompatMode); @@ -2558,6 +2692,109 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState } } + /** + * Returns whether we made a request to show inline suggestions. + */ + private boolean requestShowInlineSuggestions(List inlineSuggestionSlices, + FillResponse response) { + IInlineSuggestionsResponseCallback inlineContentCallback = null; + synchronized (mLock) { + if (mInlineSuggestionsResponseCallbackFuture != null) { + try { + inlineContentCallback = mInlineSuggestionsResponseCallbackFuture.get( + INLINE_REQUEST_TIMEOUT_MS, TimeUnit.MILLISECONDS); + } catch (TimeoutException e) { + Log.w(TAG, "Exception getting inline suggestions callback in time: " + e); + } catch (CancellationException e) { + Log.w(TAG, "Inline suggestions callback cancelled"); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + } + } + + if (inlineContentCallback == null) { + Log.w(TAG, "Session input method callback is not set yet"); + return false; + } + + final List datasets = response.getDatasets(); + if (datasets == null) { + Log.w(TAG, "response returned null datasets"); + return false; + } + + final ArrayList inlineSuggestions = new ArrayList<>(); + final int slicesSize = inlineSuggestionSlices.size(); + if (datasets.size() < slicesSize) { + Log.w(TAG, "Too many slices provided, not enough corresponding datasets"); + return false; + } + + for (int sliceIndex = 0; sliceIndex < slicesSize; sliceIndex++) { + Log.i(TAG, "Reading slice-" + sliceIndex + " at requestshowinlinesuggestions"); + final Slice inlineSuggestionSlice = inlineSuggestionSlices.get(sliceIndex); + final List sliceItems = inlineSuggestionSlice.getItems(); + + final int itemsSize = sliceItems.size(); + int minWidth = -1; + int maxWidth = -1; + int minHeight = -1; + int maxHeight = -1; + for (int itemIndex = 0; itemIndex < itemsSize; itemIndex++) { + final SliceItem item = sliceItems.get(itemIndex); + final String subtype = item.getSubType(); + switch (item.getSubType()) { + case "SUBTYPE_MIN_WIDTH": + minWidth = item.getInt(); + break; + case "SUBTYPE_MAX_WIDTH": + maxWidth = item.getInt(); + break; + case "SUBTYPE_MIN_HEIGHT": + minHeight = item.getInt(); + break; + case "SUBTYPE_MAX_HEIGHT": + maxHeight = item.getInt(); + break; + default: + Log.i(TAG, "unrecognized inline suggestions subtype: " + subtype); + } + } + + if (minWidth < 0 || maxWidth < 0 || minHeight < 0 || maxHeight < 0) { + Log.w(TAG, "missing inline suggestion requirements"); + return false; + } + + final InlinePresentationSpec spec = new InlinePresentationSpec.Builder( + new Size(minWidth, minHeight), new Size(maxWidth, maxHeight)).build(); + final InlineSuggestionInfo inlineSuggestionInfo = new InlineSuggestionInfo( + spec, InlineSuggestionInfo.SOURCE_AUTOFILL, new String[] { "" }); + final Dataset dataset = datasets.get(sliceIndex); + + inlineSuggestions.add(new InlineSuggestion(inlineSuggestionInfo, + new IInlineContentProvider.Stub() { + @Override + public void provideContent(int width, int height, + IInlineContentCallback callback) throws RemoteException { + getUiForShowing().getSuggestionSurfaceForShowing(dataset, response, + mCurrentViewId, width, height, callback); + } + })); + } + + try { + inlineContentCallback.onInlineSuggestionsResponse( + new InlineSuggestionsResponse(inlineSuggestions)); + } catch (RemoteException e) { + Log.w(TAG, "onFillReady() remote error calling onInlineSuggestionsResponse()"); + return false; + } + + return true; + } + boolean isDestroyed() { synchronized (mLock) { return mDestroyed; @@ -2831,6 +3068,7 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState final AutofillId focusedId = AutofillId.withoutSession(mCurrentViewId); + // TODO(b/137800469): implement inlined suggestions for augmented autofill remoteService.onRequestAutofillLocked(id, mClient, taskId, mComponentName, focusedId, currentValue); diff --git a/services/autofill/java/com/android/server/autofill/ui/AutoFillUI.java b/services/autofill/java/com/android/server/autofill/ui/AutoFillUI.java index 7e8edf212bdd2..32f55b73517cb 100644 --- a/services/autofill/java/com/android/server/autofill/ui/AutoFillUI.java +++ b/services/autofill/java/com/android/server/autofill/ui/AutoFillUI.java @@ -24,6 +24,8 @@ import android.content.ComponentName; import android.content.Context; import android.content.Intent; import android.content.IntentSender; +import android.graphics.Color; +import android.graphics.PixelFormat; import android.graphics.drawable.Drawable; import android.metrics.LogMaker; import android.os.Bundle; @@ -37,13 +39,19 @@ import android.service.autofill.ValueFinder; import android.text.TextUtils; import android.util.Slog; import android.view.KeyEvent; +import android.view.SurfaceControl; +import android.view.WindowManager; +import android.view.WindowlessViewRoot; import android.view.autofill.AutofillId; import android.view.autofill.AutofillManager; +import android.view.autofill.AutofillValue; import android.view.autofill.IAutofillWindowPresenter; +import android.widget.TextView; import android.widget.Toast; import com.android.internal.logging.MetricsLogger; import com.android.internal.logging.nano.MetricsProto.MetricsEvent; +import com.android.internal.view.inline.IInlineContentCallback; import com.android.server.LocalServices; import com.android.server.UiModeManagerInternal; import com.android.server.UiThread; @@ -170,6 +178,75 @@ public final class AutoFillUI { }); } + /** + * TODO(b/137800469): Fill in javadoc. + * TODO(b/137800469): peoperly manage lifecycle of suggestions surfaces. + */ + public void getSuggestionSurfaceForShowing(@NonNull Dataset dataset, + @NonNull FillResponse response, AutofillId autofillId, int width, int height, + IInlineContentCallback cb) { + if (dataset == null) { + Slog.w(TAG, "getSuggestionSurfaceForShowing() called with null dataset"); + } + mHandler.post(() -> { + final SurfaceControl suggestionSurface = inflateInlineSuggestion(dataset, response, + autofillId, width, height); + + try { + cb.onContent(suggestionSurface); + } catch (RemoteException e) { + Slog.w(TAG, "RemoteException replying onContent(" + suggestionSurface + "): " + e); + } + }); + } + + /** + * TODO(b/137800469): Fill in javadoc, generate custom templated view for inline suggestions. + * TODO: Move to ExtServices. + * + * @return a {@link SurfaceControl} with the inflated content embedded in it. + */ + private SurfaceControl inflateInlineSuggestion(@NonNull Dataset dataset, + @NonNull FillResponse response, AutofillId autofillId, int width, int height) { + Slog.i(TAG, "inflate() called"); + final Context context = mContext; + final int index = dataset.getFieldIds().indexOf(autofillId); + if (index < 0) { + Slog.w(TAG, "inflateInlineSuggestion(): AutofillId=" + autofillId + + " not found in dataset"); + } + + final AutofillValue datasetValue = dataset.getFieldValues().get(index); + final SurfaceControl sc = new SurfaceControl.Builder() + // TODO(b/137800469): sanitize name + .setName("af suggestion") + .build(); + + //TODO(b/137800469): Pass in inputToken from IME. + final WindowlessViewRoot wvr = new WindowlessViewRoot(context, context.getDisplay(), sc, + null); + + TextView textView = new TextView(context); + textView.setText(datasetValue.getTextValue()); + textView.setBackgroundColor(Color.WHITE); + textView.setTextColor(Color.BLACK); + textView.setOnClickListener(v -> { + Slog.d(TAG, "Inline suggestion clicked"); + hideFillUiUiThread(mCallback, true); + if (mCallback != null) { + final int datasetIndex = response.getDatasets().indexOf(dataset); + mCallback.fill(response.getRequestId(), datasetIndex, dataset); + } + }); + + WindowManager.LayoutParams lp = + new WindowManager.LayoutParams(width, height, + WindowManager.LayoutParams.TYPE_APPLICATION, 0, PixelFormat.OPAQUE); + wvr.addView(textView, lp); + + return sc; + } + /** * Shows the fill UI, removing the previous fill UI if the has changed. * diff --git a/services/core/java/com/android/server/inputmethod/InputMethodManagerInternal.java b/services/core/java/com/android/server/inputmethod/InputMethodManagerInternal.java index 944a95dda99b2..44c8971ba4c27 100644 --- a/services/core/java/com/android/server/inputmethod/InputMethodManagerInternal.java +++ b/services/core/java/com/android/server/inputmethod/InputMethodManagerInternal.java @@ -18,8 +18,14 @@ package com.android.server.inputmethod; import android.annotation.NonNull; import android.annotation.UserIdInt; +import android.content.ComponentName; +import android.os.RemoteException; +import android.util.Log; +import android.view.autofill.AutofillId; +import android.view.inputmethod.InlineSuggestionsRequest; import android.view.inputmethod.InputMethodInfo; +import com.android.internal.view.IInlineSuggestionsRequestCallback; import com.android.server.LocalServices; import java.util.Collections; @@ -56,6 +62,17 @@ public abstract class InputMethodManagerInternal { */ public abstract List getEnabledInputMethodListAsUser(@UserIdInt int userId); + /** + * Called by the Autofill Frameworks to request an {@link InlineSuggestionsRequest} from + * the input method. + * + * @param componentName {@link ComponentName} of current app/activity. + * @param autofillId {@link AutofillId} of currently focused field. + * @param cb {@link IInlineSuggestionsRequestCallback} used to pass back the request object. + */ + public abstract void onCreateInlineSuggestionsRequest(ComponentName componentName, + AutofillId autofillId, IInlineSuggestionsRequestCallback cb); + /** * Fake implementation of {@link InputMethodManagerInternal}. All the methods do nothing. */ @@ -78,6 +95,17 @@ public abstract class InputMethodManagerInternal { public List getEnabledInputMethodListAsUser(int userId) { return Collections.emptyList(); } + + @Override + public void onCreateInlineSuggestionsRequest(ComponentName componentName, + AutofillId autofillId, IInlineSuggestionsRequestCallback cb) { + try { + cb.onInlineSuggestionsUnsupported(); + } catch (RemoteException e) { + Log.w("IMManagerInternal", "RemoteException calling" + + " onInlineSuggestionsUnsupported: " + e); + } + } }; /** diff --git a/services/core/java/com/android/server/inputmethod/InputMethodManagerService.java b/services/core/java/com/android/server/inputmethod/InputMethodManagerService.java index 471fa72b1957c..5865dc471b303 100644 --- a/services/core/java/com/android/server/inputmethod/InputMethodManagerService.java +++ b/services/core/java/com/android/server/inputmethod/InputMethodManagerService.java @@ -109,6 +109,7 @@ import android.view.ViewGroup; import android.view.Window; import android.view.WindowManager.LayoutParams; import android.view.WindowManager.LayoutParams.SoftInputModeFlags; +import android.view.autofill.AutofillId; import android.view.inputmethod.EditorInfo; import android.view.inputmethod.InputBinding; import android.view.inputmethod.InputConnection; @@ -140,6 +141,7 @@ import com.android.internal.os.SomeArgs; import com.android.internal.os.TransferPipe; import com.android.internal.util.DumpUtils; import com.android.internal.util.IndentingPrintWriter; +import com.android.internal.view.IInlineSuggestionsRequestCallback; import com.android.internal.view.IInputContext; import com.android.internal.view.IInputMethod; import com.android.internal.view.IInputMethodClient; @@ -211,6 +213,8 @@ public class InputMethodManagerService extends IInputMethodManager.Stub static final int MSG_SYSTEM_UNLOCK_USER = 5000; + static final int MSG_INLINE_SUGGESTIONS_REQUEST = 6000; + static final long TIME_TO_RECONNECT = 3 * 1000; static final int SECURE_SUGGESTION_SPANS_MAX_SIZE = 20; @@ -1785,6 +1789,16 @@ public class InputMethodManagerService extends IInputMethodManager.Stub return settings.getEnabledInputMethodListLocked(); } + @GuardedBy("mMethodMap") + private void onCreateInlineSuggestionsRequestLocked(ComponentName componentName, + AutofillId autofillId, IInlineSuggestionsRequestCallback callback) { + if (mCurMethod != null) { + executeOrSendMessage(mCurMethod, + mCaller.obtainMessageOOOO(MSG_INLINE_SUGGESTIONS_REQUEST, mCurMethod, + componentName, autofillId, callback)); + } + } + /** * @param imiId if null, returns enabled subtypes for the current imi * @return enabled subtypes of the specified imi @@ -3874,6 +3888,21 @@ public class InputMethodManagerService extends IInputMethodManager.Stub final int userId = msg.arg1; onUnlockUser(userId); return true; + + // --------------------------------------------------------------- + case MSG_INLINE_SUGGESTIONS_REQUEST: + args = (SomeArgs) msg.obj; + final ComponentName componentName = (ComponentName) args.arg2; + final AutofillId autofillId = (AutofillId) args.arg3; + final IInlineSuggestionsRequestCallback callback = + (IInlineSuggestionsRequestCallback) args.arg4; + try { + ((IInputMethod) args.arg1).onCreateInlineSuggestionsRequest(componentName, + autofillId, callback); + } catch (RemoteException e) { + Slog.w(TAG, "RemoteException calling onCreateInlineSuggestionsRequest(): " + e); + } + return true; } return false; } @@ -4434,6 +4463,13 @@ public class InputMethodManagerService extends IInputMethodManager.Stub } } + private void onCreateInlineSuggestionsRequest(ComponentName componentName, + AutofillId autofillId, IInlineSuggestionsRequestCallback callback) { + synchronized (mMethodMap) { + onCreateInlineSuggestionsRequestLocked(componentName, autofillId, callback); + } + } + private static final class LocalServiceImpl extends InputMethodManagerInternal { @NonNull private final InputMethodManagerService mService; @@ -4464,6 +4500,12 @@ public class InputMethodManagerService extends IInputMethodManager.Stub public List getEnabledInputMethodListAsUser(int userId) { return mService.getEnabledInputMethodListAsUser(userId); } + + @Override + public void onCreateInlineSuggestionsRequest(ComponentName componentName, + AutofillId autofillId, IInlineSuggestionsRequestCallback cb) { + mService.onCreateInlineSuggestionsRequest(componentName, autofillId, cb); + } } @BinderThread diff --git a/services/core/java/com/android/server/inputmethod/MultiClientInputMethodManagerService.java b/services/core/java/com/android/server/inputmethod/MultiClientInputMethodManagerService.java index 02e29e0b2ab5a..c13d55adda08c 100644 --- a/services/core/java/com/android/server/inputmethod/MultiClientInputMethodManagerService.java +++ b/services/core/java/com/android/server/inputmethod/MultiClientInputMethodManagerService.java @@ -59,10 +59,12 @@ import android.provider.Settings; import android.text.TextUtils; import android.util.ArrayMap; import android.util.ArraySet; +import android.util.Log; import android.util.Slog; import android.util.SparseArray; import android.view.InputChannel; import android.view.WindowManager.LayoutParams.SoftInputModeFlags; +import android.view.autofill.AutofillId; import android.view.inputmethod.EditorInfo; import android.view.inputmethod.InputConnectionInspector.MissingMethodFlags; import android.view.inputmethod.InputMethodInfo; @@ -82,6 +84,7 @@ import com.android.internal.os.TransferPipe; import com.android.internal.util.DumpUtils; import com.android.internal.util.IndentingPrintWriter; import com.android.internal.util.function.pooled.PooledLambda; +import com.android.internal.view.IInlineSuggestionsRequestCallback; import com.android.internal.view.IInputContext; import com.android.internal.view.IInputMethodClient; import com.android.internal.view.IInputMethodManager; @@ -187,6 +190,18 @@ public final class MultiClientInputMethodManagerService { @UserIdInt int userId) { return userIdToInputMethodInfoMapper.getAsList(userId); } + + @Override + public void onCreateInlineSuggestionsRequest(ComponentName componentName, + AutofillId autofillId, IInlineSuggestionsRequestCallback cb) { + try { + //TODO(b/137800469): support multi client IMEs. + cb.onInlineSuggestionsUnsupported(); + } catch (RemoteException e) { + Log.w("MultiClientIMManager", "RemoteException calling" + + " onInlineSuggestionsUnsupported: " + e); + } + } }); }