From 58531839827f05299e2e436a01236cec19b19337 Mon Sep 17 00:00:00 2001 From: Feng Cao Date: Thu, 16 Jan 2020 16:10:16 -0800 Subject: [PATCH] Fix on click listener for the inline suggestion, and some refactoring. Test: manual Bug: 147116534 Change-Id: Ie413ab51e29873cba71d42a5a69a1c7bef1a747c --- .../autofill/InlineSuggestionFactory.java | 120 ++++++++++++------ .../RemoteAugmentedAutofillService.java | 13 +- .../com/android/server/autofill/Session.java | 40 +----- .../server/autofill/ui/AutoFillUI.java | 32 ----- .../autofill/ui/InlineSuggestionUi.java | 23 +--- 5 files changed, 100 insertions(+), 128 deletions(-) diff --git a/services/autofill/java/com/android/server/autofill/InlineSuggestionFactory.java b/services/autofill/java/com/android/server/autofill/InlineSuggestionFactory.java index 3af15c7a0e5cc..5e6f97e4f2826 100644 --- a/services/autofill/java/com/android/server/autofill/InlineSuggestionFactory.java +++ b/services/autofill/java/com/android/server/autofill/InlineSuggestionFactory.java @@ -20,23 +20,24 @@ import static com.android.server.autofill.Helper.sDebug; import android.annotation.NonNull; import android.annotation.Nullable; +import android.app.slice.Slice; import android.content.Context; -import android.os.Handler; import android.os.RemoteException; import android.service.autofill.Dataset; +import android.service.autofill.InlinePresentation; import android.util.Slog; import android.view.SurfaceControl; import android.view.View; import android.view.autofill.AutofillId; import android.view.autofill.IAutoFillManagerClient; -import android.view.inline.InlinePresentationSpec; import android.view.inputmethod.InlineSuggestion; import android.view.inputmethod.InlineSuggestionInfo; -import android.view.inputmethod.InlineSuggestionsRequest; import android.view.inputmethod.InlineSuggestionsResponse; import com.android.internal.view.inline.IInlineContentCallback; import com.android.internal.view.inline.IInlineContentProvider; +import com.android.server.UiThread; +import com.android.server.autofill.ui.AutoFillUI; import com.android.server.autofill.ui.InlineSuggestionUi; import java.util.ArrayList; @@ -56,23 +57,59 @@ public final class InlineSuggestionFactory { int sessionId, @NonNull Dataset[] datasets, @NonNull AutofillId autofillId, - @NonNull InlineSuggestionsRequest request, - @NonNull Handler uiHandler, @NonNull Context context, @NonNull IAutoFillManagerClient client) { + if (sDebug) Slog.d(TAG, "createAugmentedInlineSuggestionsResponse called"); + + final ArrayList inlineSuggestions = new ArrayList<>(); + final InlineSuggestionUi inlineSuggestionUi = new InlineSuggestionUi(context); + for (Dataset dataset : datasets) { + final int fieldIndex = dataset.getFieldIds().indexOf(autofillId); + if (fieldIndex < 0) { + Slog.w(TAG, "AutofillId=" + autofillId + " not found in dataset"); + return null; + } + final InlinePresentation inlinePresentation = dataset.getFieldInlinePresentation( + fieldIndex); + if (inlinePresentation == null) { + Slog.w(TAG, "InlinePresentation not found in dataset"); + return null; + } + InlineSuggestion inlineSuggestion = createAugmentedInlineSuggestion(sessionId, dataset, + inlinePresentation, inlineSuggestionUi, client); + inlineSuggestions.add(inlineSuggestion); + } + return new InlineSuggestionsResponse(inlineSuggestions); + } + + /** + * Creates an {@link InlineSuggestionsResponse} with the {@code datasets} provided by the + * autofill service. + */ + public static InlineSuggestionsResponse createInlineSuggestionsResponse(int requestId, + @NonNull Dataset[] datasets, + @NonNull AutofillId autofillId, + @NonNull Context context, + @NonNull AutoFillUI.AutoFillUiCallback client) { if (sDebug) Slog.d(TAG, "createInlineSuggestionsResponse called"); final ArrayList inlineSuggestions = new ArrayList<>(); final InlineSuggestionUi inlineSuggestionUi = new InlineSuggestionUi(context); for (Dataset dataset : datasets) { - // TODO(b/146453195): use the spec in the dataset. - InlinePresentationSpec spec = request.getPresentationSpecs().get(0); - if (spec == null) { - Slog.w(TAG, "InlinePresentationSpec is not provided in the response data set"); - continue; + final int fieldIndex = dataset.getFieldIds().indexOf(autofillId); + if (fieldIndex < 0) { + Slog.w(TAG, "AutofillId=" + autofillId + " not found in dataset"); + return null; } - InlineSuggestion inlineSuggestion = createAugmentedInlineSuggestion(sessionId, dataset, - autofillId, spec, uiHandler, inlineSuggestionUi, client); + final InlinePresentation inlinePresentation = dataset.getFieldInlinePresentation( + fieldIndex); + if (inlinePresentation == null) { + Slog.w(TAG, "InlinePresentation not found in dataset"); + return null; + } + InlineSuggestion inlineSuggestion = createInlineSuggestion(requestId, dataset, + fieldIndex, + inlinePresentation, inlineSuggestionUi, client); inlineSuggestions.add(inlineSuggestion); } return new InlineSuggestionsResponse(inlineSuggestions); @@ -80,33 +117,55 @@ public final class InlineSuggestionFactory { private static InlineSuggestion createAugmentedInlineSuggestion(int sessionId, @NonNull Dataset dataset, - @NonNull AutofillId autofillId, - @NonNull InlinePresentationSpec spec, - @NonNull Handler uiHandler, + @NonNull InlinePresentation inlinePresentation, @NonNull InlineSuggestionUi inlineSuggestionUi, @NonNull IAutoFillManagerClient client) { // TODO(b/146453195): fill in the autofill hint properly. final InlineSuggestionInfo inlineSuggestionInfo = new InlineSuggestionInfo( - spec, InlineSuggestionInfo.SOURCE_PLATFORM, new String[]{""}); - final View.OnClickListener onClickListener = createOnClickListener(sessionId, dataset, - client); + inlinePresentation.getInlinePresentationSpec(), + InlineSuggestionInfo.SOURCE_PLATFORM, new String[]{""}); + final View.OnClickListener onClickListener = v -> { + try { + client.autofill(sessionId, dataset.getFieldIds(), dataset.getFieldValues()); + } catch (RemoteException e) { + Slog.w(TAG, "Encounter exception autofilling the values"); + } + }; final InlineSuggestion inlineSuggestion = new InlineSuggestion(inlineSuggestionInfo, - createInlineContentProvider(autofillId, dataset, uiHandler, inlineSuggestionUi, + createInlineContentProvider(inlinePresentation.getSlice(), inlineSuggestionUi, + onClickListener)); + return inlineSuggestion; + } + + private static InlineSuggestion createInlineSuggestion(int requestId, + @NonNull Dataset dataset, + int fieldIndex, + @NonNull InlinePresentation inlinePresentation, + @NonNull InlineSuggestionUi inlineSuggestionUi, + @NonNull AutoFillUI.AutoFillUiCallback client) { + // TODO(b/146453195): fill in the autofill hint properly. + final InlineSuggestionInfo inlineSuggestionInfo = new InlineSuggestionInfo( + inlinePresentation.getInlinePresentationSpec(), + InlineSuggestionInfo.SOURCE_AUTOFILL, new String[]{""}); + final View.OnClickListener onClickListener = v -> { + client.fill(requestId, fieldIndex, dataset); + }; + final InlineSuggestion inlineSuggestion = new InlineSuggestion(inlineSuggestionInfo, + createInlineContentProvider(inlinePresentation.getSlice(), inlineSuggestionUi, onClickListener)); return inlineSuggestion; } private static IInlineContentProvider.Stub createInlineContentProvider( - @NonNull AutofillId autofillId, @NonNull Dataset dataset, @NonNull Handler uiHandler, - @NonNull InlineSuggestionUi inlineSuggestionUi, + @NonNull Slice slice, @NonNull InlineSuggestionUi inlineSuggestionUi, @Nullable View.OnClickListener onClickListener) { return new IInlineContentProvider.Stub() { @Override public void provideContent(int width, int height, IInlineContentCallback callback) { - uiHandler.post(() -> { - SurfaceControl sc = inlineSuggestionUi.inflate(dataset, autofillId, - width, height, onClickListener); + UiThread.getHandler().post(() -> { + SurfaceControl sc = inlineSuggestionUi.inflate(slice, width, height, + onClickListener); try { callback.onContent(sc); } catch (RemoteException e) { @@ -117,19 +176,6 @@ public final class InlineSuggestionFactory { }; } - private static View.OnClickListener createOnClickListener(int sessionId, - @NonNull Dataset dataset, - @NonNull IAutoFillManagerClient client) { - return v -> { - if (sDebug) Slog.d(TAG, "Inline suggestion clicked"); - try { - client.autofill(sessionId, dataset.getFieldIds(), dataset.getFieldValues()); - } catch (RemoteException e) { - Slog.w(TAG, "Encounter exception autofilling the values"); - } - }; - } - private InlineSuggestionFactory() { } } diff --git a/services/autofill/java/com/android/server/autofill/RemoteAugmentedAutofillService.java b/services/autofill/java/com/android/server/autofill/RemoteAugmentedAutofillService.java index 2a7357b8fca64..5fbdd25ea6c6d 100644 --- a/services/autofill/java/com/android/server/autofill/RemoteAugmentedAutofillService.java +++ b/services/autofill/java/com/android/server/autofill/RemoteAugmentedAutofillService.java @@ -161,8 +161,7 @@ final class RemoteAugmentedAutofillService @Override public void onSuccess(@Nullable Dataset[] inlineSuggestionsData) { maybeHandleInlineSuggestions(sessionId, inlineSuggestionsData, - focusedId, inlineSuggestionsRequest, - inlineSuggestionsCallback, client); + focusedId, inlineSuggestionsCallback, client); requestAutofill.complete(null); } @@ -226,19 +225,15 @@ final class RemoteAugmentedAutofillService private void maybeHandleInlineSuggestions(int sessionId, @Nullable Dataset[] inlineSuggestionsData, @NonNull AutofillId focusedId, - @Nullable InlineSuggestionsRequest inlineSuggestionsRequest, @Nullable IInlineSuggestionsResponseCallback inlineSuggestionsCallback, @NonNull IAutoFillManagerClient client) { - if (inlineSuggestionsRequest == null - || ArrayUtils.isEmpty(inlineSuggestionsData) - || inlineSuggestionsCallback == null) { + if (ArrayUtils.isEmpty(inlineSuggestionsData) || inlineSuggestionsCallback == null) { return; } try { inlineSuggestionsCallback.onInlineSuggestionsResponse( - InlineSuggestionFactory.createAugmentedInlineSuggestionsResponse( - sessionId, inlineSuggestionsData, focusedId, inlineSuggestionsRequest, - getJobHandler(), mContext, client)); + InlineSuggestionFactory.createAugmentedInlineSuggestionsResponse(sessionId, + inlineSuggestionsData, focusedId, mContext, client)); } catch (RemoteException e) { Slog.w(TAG, "Exception sending inline suggestions response back to IME."); } diff --git a/services/autofill/java/com/android/server/autofill/Session.java b/services/autofill/java/com/android/server/autofill/Session.java index cdd75107db24f..ee37de56ca159 100644 --- a/services/autofill/java/com/android/server/autofill/Session.java +++ b/services/autofill/java/com/android/server/autofill/Session.java @@ -71,7 +71,6 @@ import android.service.autofill.FieldClassificationUserData; import android.service.autofill.FillContext; import android.service.autofill.FillRequest; import android.service.autofill.FillResponse; -import android.service.autofill.InlinePresentation; import android.service.autofill.InternalSanitizer; import android.service.autofill.InternalValidator; import android.service.autofill.SaveInfo; @@ -93,9 +92,6 @@ import android.view.autofill.AutofillManager.SmartSuggestionMode; import android.view.autofill.AutofillValue; import android.view.autofill.IAutoFillManagerClient; import android.view.autofill.IAutofillWindowPresenter; -import android.view.inline.InlinePresentationSpec; -import android.view.inputmethod.InlineSuggestion; -import android.view.inputmethod.InlineSuggestionInfo; import android.view.inputmethod.InlineSuggestionsRequest; import android.view.inputmethod.InlineSuggestionsResponse; @@ -106,8 +102,6 @@ import com.android.internal.logging.nano.MetricsProto.MetricsEvent; import com.android.internal.util.ArrayUtils; import com.android.internal.view.IInlineSuggestionsRequestCallback; import com.android.internal.view.IInlineSuggestionsResponseCallback; -import com.android.internal.view.inline.IInlineContentCallback; -import com.android.internal.view.inline.IInlineContentProvider; import com.android.server.autofill.ui.AutoFillUI; import com.android.server.autofill.ui.PendingUi; import com.android.server.inputmethod.InputMethodManagerInternal; @@ -148,6 +142,7 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState private final Handler mHandler; private final Object mLock; private final AutoFillUI mUi; + private final Context mContext; private final MetricsLogger mMetricsLogger = new MetricsLogger(); @@ -773,6 +768,7 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState mLock = lock; mUi = ui; mHandler = handler; + mContext = context; mRemoteFillService = serviceComponentName == null ? null : new RemoteFillService(context, serviceComponentName, userId, this, bindInstantServiceAllowed); @@ -2733,35 +2729,11 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState return false; } - final int size = datasets.size(); - final ArrayList inlineSuggestions = new ArrayList<>(); - - for (int index = 0; index < size; index++) { - final Dataset dataset = datasets.get(index); - //TODO(b/146453536): Use the proper presentation/spec for currently focused view. - final InlinePresentation inlinePresentation = dataset.getFieldInlinePresentation(0); - if (inlinePresentation == null) { - if (sDebug) Log.d(TAG, "Missing InlinePresentation on dataset=" + dataset); - continue; - } - final InlinePresentationSpec spec = inlinePresentation.getInlinePresentationSpec(); - final InlineSuggestionInfo inlineSuggestionInfo = new InlineSuggestionInfo( - spec, InlineSuggestionInfo.SOURCE_AUTOFILL, new String[] { "" }); - - inlineSuggestions.add(new InlineSuggestion(inlineSuggestionInfo, - new IInlineContentProvider.Stub() { - @Override - public void provideContent(int width, int height, - IInlineContentCallback callback) throws RemoteException { - getUiForShowing().getSuggestionSurfaceForShowing(dataset, response, - mCurrentViewId, width, height, callback); - } - })); - } - + InlineSuggestionsResponse inlineSuggestionsResponse = + InlineSuggestionFactory.createInlineSuggestionsResponse(response.getRequestId(), + datasets.toArray(new Dataset[]{}), mCurrentViewId, mContext, this); try { - inlineContentCallback.onInlineSuggestionsResponse( - new InlineSuggestionsResponse(inlineSuggestions)); + inlineContentCallback.onInlineSuggestionsResponse(inlineSuggestionsResponse); } catch (RemoteException e) { Log.w(TAG, "onFillReady() remote error calling onInlineSuggestionsResponse()"); return false; diff --git a/services/autofill/java/com/android/server/autofill/ui/AutoFillUI.java b/services/autofill/java/com/android/server/autofill/ui/AutoFillUI.java index 0511bf2f3b741..26bb7c35b9dc4 100644 --- a/services/autofill/java/com/android/server/autofill/ui/AutoFillUI.java +++ b/services/autofill/java/com/android/server/autofill/ui/AutoFillUI.java @@ -37,7 +37,6 @@ import android.service.autofill.ValueFinder; import android.text.TextUtils; import android.util.Slog; import android.view.KeyEvent; -import android.view.SurfaceControl; import android.view.autofill.AutofillId; import android.view.autofill.AutofillManager; import android.view.autofill.IAutofillWindowPresenter; @@ -45,7 +44,6 @@ import android.widget.Toast; import com.android.internal.logging.MetricsLogger; import com.android.internal.logging.nano.MetricsProto.MetricsEvent; -import com.android.internal.view.inline.IInlineContentCallback; import com.android.server.LocalServices; import com.android.server.UiModeManagerInternal; import com.android.server.UiThread; @@ -172,36 +170,6 @@ public final class AutoFillUI { }); } - /** - * TODO(b/137800469): Fill in javadoc. - * TODO(b/137800469): peoperly manage lifecycle of suggestions surfaces. - */ - public void getSuggestionSurfaceForShowing(@NonNull Dataset dataset, - @NonNull FillResponse response, AutofillId autofillId, int width, int height, - IInlineContentCallback cb) { - if (dataset == null) { - Slog.w(TAG, "getSuggestionSurfaceForShowing() called with null dataset"); - } - mHandler.post(() -> { - final InlineSuggestionUi inlineSuggestionUi = new InlineSuggestionUi(mContext); - final SurfaceControl suggestionSurface = inlineSuggestionUi.inflate(dataset, - autofillId, width, height, v -> { - Slog.d(TAG, "Inline suggestion clicked"); - hideFillUiUiThread(mCallback, true); - if (mCallback != null) { - final int datasetIndex = response.getDatasets().indexOf(dataset); - mCallback.fill(response.getRequestId(), datasetIndex, dataset); - } - }); - - try { - cb.onContent(suggestionSurface); - } catch (RemoteException e) { - Slog.w(TAG, "RemoteException replying onContent(" + suggestionSurface + "): " + e); - } - }); - } - /** * Shows the fill UI, removing the previous fill UI if the has changed. * diff --git a/services/autofill/java/com/android/server/autofill/ui/InlineSuggestionUi.java b/services/autofill/java/com/android/server/autofill/ui/InlineSuggestionUi.java index 1e3ee888a272e..2460732d17dc7 100644 --- a/services/autofill/java/com/android/server/autofill/ui/InlineSuggestionUi.java +++ b/services/autofill/java/com/android/server/autofill/ui/InlineSuggestionUi.java @@ -28,17 +28,13 @@ import android.content.Context; import android.graphics.PixelFormat; import android.graphics.drawable.Icon; import android.os.IBinder; -import android.service.autofill.Dataset; import android.util.Log; -import android.util.Slog; import android.view.LayoutInflater; import android.view.SurfaceControl; import android.view.SurfaceControlViewHost; import android.view.View; import android.view.ViewGroup; import android.view.WindowManager; -import android.view.autofill.AutofillId; -import android.view.autofill.AutofillValue; import android.widget.ImageView; import android.widget.TextView; @@ -69,23 +65,18 @@ public class InlineSuggestionUi { */ @MainThread @Nullable - public SurfaceControl inflate(@NonNull Dataset dataset, @NonNull AutofillId autofillId, - int width, int height, @Nullable View.OnClickListener onClickListener) { + public SurfaceControl inflate(@NonNull Slice slice, int width, int height, + @Nullable View.OnClickListener onClickListener) { Log.d(TAG, "Inflating the inline suggestion UI"); - final int index = dataset.getFieldIds().indexOf(autofillId); - if (index < 0) { - Slog.w(TAG, "inflateInlineSuggestion(): AutofillId=" + autofillId - + " not found in dataset"); - return null; - } - final AutofillValue datasetValue = dataset.getFieldValues().get(index); + //TODO(b/137800469): Pass in inputToken from IME. final SurfaceControlViewHost wvr = new SurfaceControlViewHost(mContext, mContext.getDisplay(), (IBinder) null); final SurfaceControl sc = wvr.getSurfacePackage().getSurfaceControl(); - - final ViewGroup suggestionView = - (ViewGroup) renderSlice(dataset.getFieldInlinePresentation(index).getSlice()); + final ViewGroup suggestionView = (ViewGroup) renderSlice(slice); + if (onClickListener != null) { + suggestionView.setOnClickListener(onClickListener); + } WindowManager.LayoutParams lp = new WindowManager.LayoutParams(width, height,