From 17d9d2f22f890a2252da8ca33f5dec17b4deae7c Mon Sep 17 00:00:00 2001 From: Svet Ganov Date: Sat, 18 Jan 2020 10:12:11 -0800 Subject: [PATCH] Scrollable inline keyboard suggestions Inlined keyboard suggestions are provided by the OS while hosted by the keyboard. The clicks are detected by the OS trusted code to release the data. Not all suggestions may fit on the keyboard suggestion strip and the user may need to scroll. To support this we are delegating the touch event stream to the keyboard window as soon as the OS detects that the user is not clicking, i.e. after a pointer went down and moved more than the touch slop. Bug:146535667 Test: atest CtsAutofillTestCases atest inputflinger_tests atest com.android.server.wm.DragDropControllerTests atest android.server.wm.DragDropTest atest android.server.wm.CrossAppDragAndDropTests added dedicated tests to inputflinger_tests Change-Id: Iefb732f70a7b593b31ab2f323b0c277e3819f7b7 --- .../augmented/AugmentedAutofillService.java | 70 +++++++++------- .../autofill/augmented/FillCallback.java | 6 +- .../autofill/augmented/FillController.java | 9 +- .../autofill/augmented/FillRequest.java | 4 +- .../autofill/augmented/FillWindow.java | 41 ++++++---- .../autofill/augmented/IFillCallback.aidl | 2 +- .../autofill/AutofillManagerServiceImpl.java | 20 ++--- .../RemoteAugmentedAutofillService.java | 25 +++--- .../com/android/server/autofill/Session.java | 14 +++- .../{ => ui}/InlineSuggestionFactory.java | 66 +++++++-------- .../autofill/ui/InlineSuggestionRoot.java | 82 +++++++++++++++++++ .../autofill/ui/InlineSuggestionUi.java | 16 ++-- .../server/input/InputManagerService.java | 33 ++++++-- .../server/wm/WindowManagerInternal.java | 10 +++ .../server/wm/WindowManagerService.java | 23 ++++++ ...droid_server_input_InputManagerService.cpp | 19 ++--- .../server/wm/DragDropControllerTests.java | 6 +- 17 files changed, 304 insertions(+), 142 deletions(-) rename services/autofill/java/com/android/server/autofill/{ => ui}/InlineSuggestionFactory.java (80%) create mode 100644 services/autofill/java/com/android/server/autofill/ui/InlineSuggestionRoot.java diff --git a/core/java/android/service/autofill/augmented/AugmentedAutofillService.java b/core/java/android/service/autofill/augmented/AugmentedAutofillService.java index 368c94cdc7904..79852d3c0fcad 100644 --- a/core/java/android/service/autofill/augmented/AugmentedAutofillService.java +++ b/core/java/android/service/autofill/augmented/AugmentedAutofillService.java @@ -31,7 +31,6 @@ import android.content.ComponentName; import android.content.Intent; import android.graphics.Rect; import android.os.Build; -import android.os.Bundle; import android.os.CancellationSignal; import android.os.Handler; import android.os.IBinder; @@ -93,7 +92,7 @@ public abstract class AugmentedAutofillService extends Service { // Used for metrics / debug only private ComponentName mServiceComponentName; - private final IAugmentedAutofillService mInterface = new IAugmentedAutofillService.Stub() { + private final class AugmentedAutofillServiceImpl extends IAugmentedAutofillService.Stub { @Override public void onConnected(boolean debug, boolean verbose) { @@ -137,7 +136,7 @@ public abstract class AugmentedAutofillService extends Service { public final IBinder onBind(Intent intent) { mServiceComponentName = intent.getComponent(); if (SERVICE_INTERFACE.equals(intent.getAction())) { - return mInterface.asBinder(); + return new AugmentedAutofillServiceImpl(); } Log.w(TAG, "Tried to bind to wrong intent (should be " + SERVICE_INTERFACE + ": " + intent); return null; @@ -352,11 +351,13 @@ public abstract class AugmentedAutofillService extends Service { static final int REPORT_EVENT_NO_RESPONSE = 1; static final int REPORT_EVENT_UI_SHOWN = 2; static final int REPORT_EVENT_UI_DESTROYED = 3; + static final int REPORT_EVENT_INLINE_RESPONSE = 4; @IntDef(prefix = { "REPORT_EVENT_" }, value = { REPORT_EVENT_NO_RESPONSE, REPORT_EVENT_UI_SHOWN, - REPORT_EVENT_UI_DESTROYED + REPORT_EVENT_UI_DESTROYED, + REPORT_EVENT_INLINE_RESPONSE }) @Retention(RetentionPolicy.SOURCE) @interface ReportEvent{} @@ -365,8 +366,8 @@ public abstract class AugmentedAutofillService extends Service { private final Object mLock = new Object(); private final IAugmentedAutofillManagerClient mClient; private final int mSessionId; - public final int taskId; - public final ComponentName componentName; + public final int mTaskId; + public final ComponentName mComponentName; // Used for metrics / debug only private String mServicePackageName; @GuardedBy("mLock") @@ -406,8 +407,8 @@ public abstract class AugmentedAutofillService extends Service { mSessionId = sessionId; mClient = IAugmentedAutofillManagerClient.Stub.asInterface(client); mCallback = callback; - this.taskId = taskId; - this.componentName = componentName; + mTaskId = taskId; + mComponentName = componentName; mServicePackageName = serviceComponentName.getPackageName(); mFocusedId = focusedId; mFocusedValue = focusedValue; @@ -514,22 +515,24 @@ public abstract class AugmentedAutofillService extends Service { } } - public void onInlineSuggestionsDataReady(@NonNull List inlineSuggestionsData, - @Nullable Bundle clientState) { + void reportResult(@Nullable List inlineSuggestionsData) { try { - mCallback.onSuccess(inlineSuggestionsData.toArray(new Dataset[]{}), clientState); + final Dataset[] inlineSuggestions = (inlineSuggestionsData != null) + ? inlineSuggestionsData.toArray(new Dataset[inlineSuggestionsData.size()]) + : null; + mCallback.onSuccess(inlineSuggestions); } catch (RemoteException e) { Log.e(TAG, "Error calling back with the inline suggestions data: " + e); } } - // Used (mostly) for metrics. - public void report(@ReportEvent int event) { - if (sVerbose) Log.v(TAG, "report(): " + event); + void logEvent(@ReportEvent int event) { + if (sVerbose) Log.v(TAG, "returnAndLogResult(): " + event); long duration = -1; int type = MetricsEvent.TYPE_UNKNOWN; + switch (event) { - case REPORT_EVENT_NO_RESPONSE: + case REPORT_EVENT_NO_RESPONSE: { type = MetricsEvent.TYPE_SUCCESS; if (mFirstOnSuccessTime == 0) { mFirstOnSuccessTime = SystemClock.elapsedRealtime(); @@ -538,40 +541,49 @@ public abstract class AugmentedAutofillService extends Service { Log.d(TAG, "Service responded nothing in " + formatDuration(duration)); } } - try { - mCallback.onSuccess(/* inlineSuggestionsData= */null, /* clientState=*/ - null); - } catch (RemoteException e) { - Log.e(TAG, "Error reporting success: " + e); + } break; + + case REPORT_EVENT_INLINE_RESPONSE: { + // TODO: Define a constant and log this event + // type = MetricsEvent.TYPE_SUCCESS_INLINE; + if (mFirstOnSuccessTime == 0) { + mFirstOnSuccessTime = SystemClock.elapsedRealtime(); + duration = mFirstOnSuccessTime - mFirstRequestTime; + if (sDebug) { + Log.d(TAG, "Service responded nothing in " + formatDuration(duration)); + } } - break; - case REPORT_EVENT_UI_SHOWN: + } break; + + case REPORT_EVENT_UI_SHOWN: { type = MetricsEvent.TYPE_OPEN; if (mUiFirstShownTime == 0) { mUiFirstShownTime = SystemClock.elapsedRealtime(); duration = mUiFirstShownTime - mFirstRequestTime; if (sDebug) Log.d(TAG, "UI shown in " + formatDuration(duration)); } - break; - case REPORT_EVENT_UI_DESTROYED: + } break; + + case REPORT_EVENT_UI_DESTROYED: { type = MetricsEvent.TYPE_CLOSE; if (mUiFirstDestroyedTime == 0) { mUiFirstDestroyedTime = SystemClock.elapsedRealtime(); - duration = mUiFirstDestroyedTime - mFirstRequestTime; + duration = mUiFirstDestroyedTime - mFirstRequestTime; if (sDebug) Log.d(TAG, "UI destroyed in " + formatDuration(duration)); } - break; + } break; + default: Log.w(TAG, "invalid event reported: " + event); } - logResponse(type, mServicePackageName, componentName, mSessionId, duration); + logResponse(type, mServicePackageName, mComponentName, mSessionId, duration); } public void dump(@NonNull String prefix, @NonNull PrintWriter pw) { pw.print(prefix); pw.print("sessionId: "); pw.println(mSessionId); - pw.print(prefix); pw.print("taskId: "); pw.println(taskId); + pw.print(prefix); pw.print("taskId: "); pw.println(mTaskId); pw.print(prefix); pw.print("component: "); - pw.println(componentName.flattenToShortString()); + pw.println(mComponentName.flattenToShortString()); pw.print(prefix); pw.print("focusedId: "); pw.println(mFocusedId); if (mFocusedValue != null) { pw.print(prefix); pw.print("focusedValue: "); pw.println(mFocusedValue); diff --git a/core/java/android/service/autofill/augmented/FillCallback.java b/core/java/android/service/autofill/augmented/FillCallback.java index d0ffd7b4d8d41..19eff57269ae4 100644 --- a/core/java/android/service/autofill/augmented/FillCallback.java +++ b/core/java/android/service/autofill/augmented/FillCallback.java @@ -54,13 +54,15 @@ public final class FillCallback { if (sDebug) Log.d(TAG, "onSuccess(): " + response); if (response == null) { - mProxy.report(AutofillProxy.REPORT_EVENT_NO_RESPONSE); + mProxy.logEvent(AutofillProxy.REPORT_EVENT_NO_RESPONSE); + mProxy.reportResult(null /*inlineSuggestions*/); return; } List inlineSuggestions = response.getInlineSuggestions(); if (inlineSuggestions != null && !inlineSuggestions.isEmpty()) { - mProxy.onInlineSuggestionsDataReady(inlineSuggestions, response.getClientState()); + mProxy.logEvent(AutofillProxy.REPORT_EVENT_INLINE_RESPONSE); + mProxy.reportResult(inlineSuggestions); return; } diff --git a/core/java/android/service/autofill/augmented/FillController.java b/core/java/android/service/autofill/augmented/FillController.java index 63ec2d886cafc..7d552d62fa721 100644 --- a/core/java/android/service/autofill/augmented/FillController.java +++ b/core/java/android/service/autofill/augmented/FillController.java @@ -62,12 +62,13 @@ public final class FillController { try { mProxy.autofill(values); - final FillWindow fillWindow = mProxy.getFillWindow(); - if (fillWindow != null) { - fillWindow.destroy(); - } } catch (RemoteException e) { e.rethrowAsRuntimeException(); } + + final FillWindow fillWindow = mProxy.getFillWindow(); + if (fillWindow != null) { + fillWindow.destroy(); + } } } diff --git a/core/java/android/service/autofill/augmented/FillRequest.java b/core/java/android/service/autofill/augmented/FillRequest.java index ca49e7daf0548..6927cf6541e08 100644 --- a/core/java/android/service/autofill/augmented/FillRequest.java +++ b/core/java/android/service/autofill/augmented/FillRequest.java @@ -53,7 +53,7 @@ public final class FillRequest { * Gets the task of the activity associated with this request. */ public int getTaskId() { - return mProxy.taskId; + return mProxy.mTaskId; } /** @@ -61,7 +61,7 @@ public final class FillRequest { */ @NonNull public ComponentName getActivityComponent() { - return mProxy.componentName; + return mProxy.mComponentName; } /** diff --git a/core/java/android/service/autofill/augmented/FillWindow.java b/core/java/android/service/autofill/augmented/FillWindow.java index 5d003706ac830..077df6cf16ef1 100644 --- a/core/java/android/service/autofill/augmented/FillWindow.java +++ b/core/java/android/service/autofill/augmented/FillWindow.java @@ -21,6 +21,7 @@ import static android.service.autofill.augmented.AugmentedAutofillService.sVerbo import static com.android.internal.util.function.pooled.PooledLambda.obtainMessage; import android.annotation.NonNull; +import android.annotation.Nullable; import android.annotation.SystemApi; import android.annotation.TestApi; import android.graphics.Rect; @@ -41,6 +42,7 @@ import com.android.internal.util.Preconditions; import dalvik.system.CloseGuard; import java.io.PrintWriter; +import java.lang.ref.WeakReference; /** * Handle to a window used to display the augmented autofill UI. @@ -70,23 +72,22 @@ public final class FillWindow implements AutoCloseable { private final CloseGuard mCloseGuard = CloseGuard.get(); private final @NonNull Handler mUiThreadHandler = new Handler(Looper.getMainLooper()); - private final @NonNull FillWindowPresenter mFillWindowPresenter = new FillWindowPresenter(); @GuardedBy("mLock") - private WindowManager mWm; + private @NonNull WindowManager mWm; @GuardedBy("mLock") private View mFillView; @GuardedBy("mLock") private boolean mShowing; @GuardedBy("mLock") - private Rect mBounds; + private @Nullable Rect mBounds; @GuardedBy("mLock") private boolean mUpdateCalled; @GuardedBy("mLock") private boolean mDestroyed; - private AutofillProxy mProxy; + private @NonNull AutofillProxy mProxy; /** * Updates the content of the window. @@ -172,11 +173,11 @@ public final class FillWindow implements AutoCloseable { try { mProxy.requestShowFillUi(mBounds.right - mBounds.left, mBounds.bottom - mBounds.top, - /*anchorBounds=*/ null, mFillWindowPresenter); + /*anchorBounds=*/ null, new FillWindowPresenter(this)); } catch (RemoteException e) { Log.w(TAG, "Error requesting to show fill window", e); } - mProxy.report(AutofillProxy.REPORT_EVENT_UI_SHOWN); + mProxy.logEvent(AutofillProxy.REPORT_EVENT_UI_SHOWN); } } } @@ -244,7 +245,7 @@ public final class FillWindow implements AutoCloseable { if (mUpdateCalled) { mFillView.setOnClickListener(null); hide(); - mProxy.report(AutofillProxy.REPORT_EVENT_UI_DESTROYED); + mProxy.logEvent(AutofillProxy.REPORT_EVENT_UI_DESTROYED); } mDestroyed = true; mCloseGuard.close(); @@ -254,9 +255,7 @@ public final class FillWindow implements AutoCloseable { @Override protected void finalize() throws Throwable { try { - if (mCloseGuard != null) { - mCloseGuard.warnIfOpen(); - } + mCloseGuard.warnIfOpen(); destroy(); } finally { super.finalize(); @@ -289,22 +288,36 @@ public final class FillWindow implements AutoCloseable { /** @hide */ @Override - public void close() throws Exception { + public void close() { destroy(); } - private final class FillWindowPresenter extends IAutofillWindowPresenter.Stub { + private static final class FillWindowPresenter extends IAutofillWindowPresenter.Stub { + private final @NonNull WeakReference mFillWindowReference; + + FillWindowPresenter(@NonNull FillWindow fillWindow) { + mFillWindowReference = new WeakReference<>(fillWindow); + } + @Override public void show(WindowManager.LayoutParams p, Rect transitionEpicenter, boolean fitsSystemWindows, int layoutDirection) { if (sDebug) Log.d(TAG, "FillWindowPresenter.show()"); - mUiThreadHandler.sendMessage(obtainMessage(FillWindow::handleShow, FillWindow.this, p)); + final FillWindow fillWindow = mFillWindowReference.get(); + if (fillWindow != null) { + fillWindow.mUiThreadHandler.sendMessage( + obtainMessage(FillWindow::handleShow, fillWindow, p)); + } } @Override public void hide(Rect transitionEpicenter) { if (sDebug) Log.d(TAG, "FillWindowPresenter.hide()"); - mUiThreadHandler.sendMessage(obtainMessage(FillWindow::handleHide, FillWindow.this)); + final FillWindow fillWindow = mFillWindowReference.get(); + if (fillWindow != null) { + fillWindow.mUiThreadHandler.sendMessage( + obtainMessage(FillWindow::handleHide, fillWindow)); + } } } } diff --git a/core/java/android/service/autofill/augmented/IFillCallback.aidl b/core/java/android/service/autofill/augmented/IFillCallback.aidl index 31e77f3587823..d9837211be19c 100644 --- a/core/java/android/service/autofill/augmented/IFillCallback.aidl +++ b/core/java/android/service/autofill/augmented/IFillCallback.aidl @@ -28,7 +28,7 @@ import android.service.autofill.Dataset; */ interface IFillCallback { void onCancellable(in ICancellationSignal cancellation); - void onSuccess(in @nullable Dataset[] inlineSuggestionsData, in @nullable Bundle clientState); + void onSuccess(in @nullable Dataset[] inlineSuggestionsData); boolean isCompleted(); void cancel(); } diff --git a/services/autofill/java/com/android/server/autofill/AutofillManagerServiceImpl.java b/services/autofill/java/com/android/server/autofill/AutofillManagerServiceImpl.java index 1a4fc32a76cd1..1cb9313d9bf99 100644 --- a/services/autofill/java/com/android/server/autofill/AutofillManagerServiceImpl.java +++ b/services/autofill/java/com/android/server/autofill/AutofillManagerServiceImpl.java @@ -818,27 +818,26 @@ final class AutofillManagerServiceImpl } } - void logAugmentedAutofillSelected(int sessionId, @Nullable String suggestionId, - @Nullable Bundle clientState) { + void logAugmentedAutofillSelected(int sessionId, @Nullable String suggestionId) { synchronized (mLock) { if (mAugmentedAutofillEventHistory == null || mAugmentedAutofillEventHistory.getSessionId() != sessionId) { return; } mAugmentedAutofillEventHistory.addEvent( - new Event(Event.TYPE_DATASET_SELECTED, suggestionId, clientState, null, null, + new Event(Event.TYPE_DATASET_SELECTED, suggestionId, null, null, null, null, null, null, null, null, null)); } } - void logAugmentedAutofillShown(int sessionId, @Nullable Bundle clientState) { + void logAugmentedAutofillShown(int sessionId) { synchronized (mLock) { if (mAugmentedAutofillEventHistory == null || mAugmentedAutofillEventHistory.getSessionId() != sessionId) { return; } mAugmentedAutofillEventHistory.addEvent( - new Event(Event.TYPE_DATASETS_SHOWN, null, clientState, null, null, null, + new Event(Event.TYPE_DATASETS_SHOWN, null, null, null, null, null, null, null, null, null, null)); } @@ -1227,16 +1226,15 @@ final class AutofillManagerServiceImpl } @Override - public void logAugmentedAutofillShown(int sessionId, Bundle clientState) { - AutofillManagerServiceImpl.this.logAugmentedAutofillShown(sessionId, - clientState); + public void logAugmentedAutofillShown(int sessionId) { + AutofillManagerServiceImpl.this.logAugmentedAutofillShown(sessionId); } @Override - public void logAugmentedAutofillSelected(int sessionId, String suggestionId, - Bundle clientState) { + public void logAugmentedAutofillSelected(int sessionId, + String suggestionId) { AutofillManagerServiceImpl.this.logAugmentedAutofillSelected(sessionId, - suggestionId, clientState); + suggestionId); } @Override diff --git a/services/autofill/java/com/android/server/autofill/RemoteAugmentedAutofillService.java b/services/autofill/java/com/android/server/autofill/RemoteAugmentedAutofillService.java index 5e6f6fea4dda2..880c401581149 100644 --- a/services/autofill/java/com/android/server/autofill/RemoteAugmentedAutofillService.java +++ b/services/autofill/java/com/android/server/autofill/RemoteAugmentedAutofillService.java @@ -55,6 +55,7 @@ import com.android.internal.logging.nano.MetricsProto.MetricsEvent; import com.android.internal.os.IResultReceiver; import com.android.internal.util.ArrayUtils; import com.android.internal.view.IInlineSuggestionsResponseCallback; +import com.android.server.autofill.ui.InlineSuggestionFactory; import java.util.concurrent.CancellationException; import java.util.concurrent.TimeUnit; @@ -144,7 +145,8 @@ final class RemoteAugmentedAutofillService int taskId, @NonNull ComponentName activityComponent, @NonNull AutofillId focusedId, @Nullable AutofillValue focusedValue, @Nullable InlineSuggestionsRequest inlineSuggestionsRequest, - @Nullable IInlineSuggestionsResponseCallback inlineSuggestionsCallback) { + @Nullable IInlineSuggestionsResponseCallback inlineSuggestionsCallback, + @NonNull Runnable onErrorCallback) { long requestTime = SystemClock.elapsedRealtime(); AtomicReference cancellationRef = new AtomicReference<>(); @@ -161,12 +163,12 @@ final class RemoteAugmentedAutofillService focusedId, focusedValue, requestTime, inlineSuggestionsRequest, new IFillCallback.Stub() { @Override - public void onSuccess(@Nullable Dataset[] inlineSuggestionsData, - @Nullable Bundle clientState) { + public void onSuccess(@Nullable Dataset[] inlineSuggestionsData) { mCallbacks.resetLastResponse(); maybeRequestShowInlineSuggestions(sessionId, inlineSuggestionsData, focusedId, - inlineSuggestionsCallback, client, clientState); + inlineSuggestionsCallback, client, + onErrorCallback); requestAutofill.complete(null); } @@ -231,29 +233,31 @@ final class RemoteAugmentedAutofillService private void maybeRequestShowInlineSuggestions(int sessionId, @Nullable Dataset[] inlineSuggestionsData, @NonNull AutofillId focusedId, @Nullable IInlineSuggestionsResponseCallback inlineSuggestionsCallback, - @NonNull IAutoFillManagerClient client, @Nullable Bundle clientState) { + @NonNull IAutoFillManagerClient client, @NonNull Runnable onErrorCallback) { if (ArrayUtils.isEmpty(inlineSuggestionsData) || inlineSuggestionsCallback == null) { return; } mCallbacks.setLastResponse(sessionId); + try { inlineSuggestionsCallback.onInlineSuggestionsResponse( InlineSuggestionFactory.createAugmentedInlineSuggestionsResponse( inlineSuggestionsData, focusedId, mContext, dataset -> { mCallbacks.logAugmentedAutofillSelected(sessionId, - dataset.getId(), clientState); + dataset.getId()); try { client.autofill(sessionId, dataset.getFieldIds(), dataset.getFieldValues()); } catch (RemoteException e) { Slog.w(TAG, "Encounter exception autofilling the values"); } - })); + }, onErrorCallback)); } catch (RemoteException e) { Slog.w(TAG, "Exception sending inline suggestions response back to IME."); } - mCallbacks.logAugmentedAutofillShown(sessionId, clientState); + + mCallbacks.logAugmentedAutofillShown(sessionId); } @Override @@ -275,9 +279,8 @@ final class RemoteAugmentedAutofillService void setLastResponse(int sessionId); - void logAugmentedAutofillShown(int sessionId, @Nullable Bundle clientState); + void logAugmentedAutofillShown(int sessionId); - void logAugmentedAutofillSelected(int sessionId, @Nullable String suggestionId, - @Nullable Bundle clientState); + void logAugmentedAutofillSelected(int sessionId, @Nullable String suggestionId); } } diff --git a/services/autofill/java/com/android/server/autofill/Session.java b/services/autofill/java/com/android/server/autofill/Session.java index 415ecd8cfd632..7e5123c820547 100644 --- a/services/autofill/java/com/android/server/autofill/Session.java +++ b/services/autofill/java/com/android/server/autofill/Session.java @@ -103,6 +103,7 @@ import com.android.internal.util.ArrayUtils; import com.android.internal.view.IInlineSuggestionsRequestCallback; import com.android.internal.view.IInlineSuggestionsResponseCallback; import com.android.server.autofill.ui.AutoFillUI; +import com.android.server.autofill.ui.InlineSuggestionFactory; import com.android.server.autofill.ui.PendingUi; import com.android.server.inputmethod.InputMethodManagerInternal; @@ -2681,7 +2682,6 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState } } - getUiForShowing().showFillUi(filledId, response, filterText, mService.getServicePackageName(), mComponentName, serviceLabel, serviceIcon, this, id, mCompatMode); @@ -2733,7 +2733,11 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState InlineSuggestionsResponse inlineSuggestionsResponse = InlineSuggestionFactory.createInlineSuggestionsResponse(response.getRequestId(), - datasets.toArray(new Dataset[]{}), mCurrentViewId, mContext, this); + datasets.toArray(new Dataset[]{}), mCurrentViewId, mContext, this, () -> { + synchronized (mLock) { + requestHideFillUi(mCurrentViewId); + } + }); try { inlineContentCallback.onInlineSuggestionsResponse(inlineSuggestionsResponse); } catch (RemoteException e) { @@ -3024,7 +3028,11 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState mInlineSuggestionsRequestCallback != null ? mInlineSuggestionsRequestCallback.getResponseCallback() : null; remoteService.onRequestAutofillLocked(id, mClient, taskId, mComponentName, focusedId, - currentValue, inlineSuggestionsRequest, inlineSuggestionsResponseCallback); + currentValue, inlineSuggestionsRequest, inlineSuggestionsResponseCallback, () -> { + synchronized (mLock) { + cancelAugmentedAutofillLocked(); + } + }); if (mAugmentedAutofillDestroyer == null) { mAugmentedAutofillDestroyer = () -> remoteService.onDestroyAutofillWindowsRequest(); diff --git a/services/autofill/java/com/android/server/autofill/InlineSuggestionFactory.java b/services/autofill/java/com/android/server/autofill/ui/InlineSuggestionFactory.java similarity index 80% rename from services/autofill/java/com/android/server/autofill/InlineSuggestionFactory.java rename to services/autofill/java/com/android/server/autofill/ui/InlineSuggestionFactory.java index cb6c8f5d3ea57..38a5b5b1cdaab 100644 --- a/services/autofill/java/com/android/server/autofill/InlineSuggestionFactory.java +++ b/services/autofill/java/com/android/server/autofill/ui/InlineSuggestionFactory.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.android.server.autofill; +package com.android.server.autofill.ui; import static com.android.server.autofill.Helper.sDebug; @@ -32,18 +32,13 @@ import android.view.inputmethod.InlineSuggestion; import android.view.inputmethod.InlineSuggestionInfo; import android.view.inputmethod.InlineSuggestionsResponse; +import com.android.internal.util.function.QuadFunction; import com.android.internal.view.inline.IInlineContentCallback; import com.android.internal.view.inline.IInlineContentProvider; import com.android.server.UiThread; -import com.android.server.autofill.ui.AutoFillUI; -import com.android.server.autofill.ui.InlineSuggestionUi; import java.util.ArrayList; - -/** - * @hide - */ public final class InlineSuggestionFactory { private static final String TAG = "InlineSuggestionFactory"; @@ -65,28 +60,12 @@ public final class InlineSuggestionFactory { @NonNull Dataset[] datasets, @NonNull AutofillId autofillId, @NonNull Context context, - @NonNull InlineSuggestionUiCallback inlineSuggestionUiCallback) { - if (sDebug) Slog.d(TAG, "createAugmentedInlineSuggestionsResponse called"); - - final ArrayList inlineSuggestions = new ArrayList<>(); - final InlineSuggestionUi inlineSuggestionUi = new InlineSuggestionUi(context); - for (Dataset dataset : datasets) { - final int fieldIndex = dataset.getFieldIds().indexOf(autofillId); - if (fieldIndex < 0) { - Slog.w(TAG, "AutofillId=" + autofillId + " not found in dataset"); - return null; - } - final InlinePresentation inlinePresentation = dataset.getFieldInlinePresentation( - fieldIndex); - if (inlinePresentation == null) { - Slog.w(TAG, "InlinePresentation not found in dataset"); - return null; - } - InlineSuggestion inlineSuggestion = createAugmentedInlineSuggestion(dataset, - inlinePresentation, inlineSuggestionUi, inlineSuggestionUiCallback); - inlineSuggestions.add(inlineSuggestion); - } - return new InlineSuggestionsResponse(inlineSuggestions); + @NonNull InlineSuggestionUiCallback inlineSuggestionUiCallback, + @NonNull Runnable onErrorCallback) { + return createInlineSuggestionsResponseInternal(datasets, autofillId, + context, onErrorCallback, (dataset, inlinePresentation, inlineSuggestionUi, + filedIndex) -> createAugmentedInlineSuggestion(dataset, + inlinePresentation, inlineSuggestionUi, inlineSuggestionUiCallback)); } /** @@ -97,11 +76,26 @@ public final class InlineSuggestionFactory { @NonNull Dataset[] datasets, @NonNull AutofillId autofillId, @NonNull Context context, - @NonNull AutoFillUI.AutoFillUiCallback client) { - if (sDebug) Slog.d(TAG, "createInlineSuggestionsResponse called"); + @NonNull AutoFillUI.AutoFillUiCallback client, + @NonNull Runnable onErrorCallback) { + return createInlineSuggestionsResponseInternal(datasets, autofillId, + context, onErrorCallback, (dataset, inlinePresentation, inlineSuggestionUi, + filedIndex) -> createInlineSuggestion(requestId, dataset, filedIndex, + inlinePresentation, inlineSuggestionUi, client)); + } + + private static InlineSuggestionsResponse createInlineSuggestionsResponseInternal( + @NonNull Dataset[] datasets, + @NonNull AutofillId autofillId, + @NonNull Context context, + @NonNull Runnable onErrorCallback, + @NonNull QuadFunction suggestionFactory) { + if (sDebug) Slog.d(TAG, "createAugmentedInlineSuggestionsResponse called"); final ArrayList inlineSuggestions = new ArrayList<>(); - final InlineSuggestionUi inlineSuggestionUi = new InlineSuggestionUi(context); + final InlineSuggestionUi inlineSuggestionUi = new InlineSuggestionUi(context, + onErrorCallback); for (Dataset dataset : datasets) { final int fieldIndex = dataset.getFieldIds().indexOf(autofillId); if (fieldIndex < 0) { @@ -114,9 +108,8 @@ public final class InlineSuggestionFactory { Slog.w(TAG, "InlinePresentation not found in dataset"); return null; } - InlineSuggestion inlineSuggestion = createInlineSuggestion(requestId, dataset, - fieldIndex, - inlinePresentation, inlineSuggestionUi, client); + InlineSuggestion inlineSuggestion = suggestionFactory.apply(dataset, + inlinePresentation, inlineSuggestionUi, fieldIndex); inlineSuggestions.add(inlineSuggestion); } return new InlineSuggestionsResponse(inlineSuggestions); @@ -131,9 +124,8 @@ public final class InlineSuggestionFactory { inlinePresentation.getInlinePresentationSpec(), InlineSuggestionInfo.SOURCE_PLATFORM, new String[]{""}, InlineSuggestionInfo.TYPE_SUGGESTION); - final View.OnClickListener onClickListener = v -> { + final View.OnClickListener onClickListener = v -> inlineSuggestionUiCallback.autofill(dataset); - }; final InlineSuggestion inlineSuggestion = new InlineSuggestion(inlineSuggestionInfo, createInlineContentProvider(inlinePresentation, inlineSuggestionUi, onClickListener)); diff --git a/services/autofill/java/com/android/server/autofill/ui/InlineSuggestionRoot.java b/services/autofill/java/com/android/server/autofill/ui/InlineSuggestionRoot.java new file mode 100644 index 0000000000000..8d476d72c639c --- /dev/null +++ b/services/autofill/java/com/android/server/autofill/ui/InlineSuggestionRoot.java @@ -0,0 +1,82 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.autofill.ui; + +import android.annotation.NonNull; +import android.annotation.SuppressLint; +import android.content.Context; +import android.util.Log; +import android.util.MathUtils; +import android.view.MotionEvent; +import android.view.ViewConfiguration; +import android.widget.FrameLayout; + +import com.android.server.LocalServices; +import com.android.server.wm.WindowManagerInternal; + +/** + * This class is the root view for an inline suggestion. It is responsible for + * detecting the click on the item and to also transfer input focus to the IME + * window if we detect the user is scrolling. + */ + // TODO(b/146453086) Move to ExtServices and add @SystemApi to transfer touch focus +@SuppressLint("ViewConstructor") +class InlineSuggestionRoot extends FrameLayout { + private static final String LOG_TAG = InlineSuggestionRoot.class.getSimpleName(); + + private final @NonNull Runnable mOnErrorCallback; + private final int mTouchSlop; + + private float mDownX; + private float mDownY; + + InlineSuggestionRoot(@NonNull Context context, @NonNull Runnable onErrorCallback) { + super(context); + mOnErrorCallback = onErrorCallback; + mTouchSlop = ViewConfiguration.get(context).getScaledTouchSlop(); + } + + @Override + @SuppressLint("ClickableViewAccessibility") + public boolean onTouchEvent(@NonNull MotionEvent event) { + switch (event.getActionMasked()) { + case MotionEvent.ACTION_DOWN: { + mDownX = event.getX(); + mDownY = event.getY(); + } break; + + case MotionEvent.ACTION_MOVE: { + final float distance = MathUtils.dist(mDownX, mDownY, + event.getX(), event.getY()); + if (distance > mTouchSlop) { + transferTouchFocusToImeWindow(); + } + } break; + } + return super.onTouchEvent(event); + } + + private void transferTouchFocusToImeWindow() { + final WindowManagerInternal windowManagerInternal = LocalServices.getService( + WindowManagerInternal.class); + if (!windowManagerInternal.transferTouchFocusToImeWindow(getViewRootImpl().getInputToken(), + getContext().getDisplayId())) { + Log.e(LOG_TAG, "Cannot transfer touch focus from suggestion to IME"); + mOnErrorCallback.run(); + } + } +} diff --git a/services/autofill/java/com/android/server/autofill/ui/InlineSuggestionUi.java b/services/autofill/java/com/android/server/autofill/ui/InlineSuggestionUi.java index 2adefeabfcc80..bf148a642bbd2 100644 --- a/services/autofill/java/com/android/server/autofill/ui/InlineSuggestionUi.java +++ b/services/autofill/java/com/android/server/autofill/ui/InlineSuggestionUi.java @@ -67,10 +67,12 @@ public class InlineSuggestionUi { // (int)}. This name is a single string of the form "package:type/entry". private static final Pattern RESOURCE_NAME_PATTERN = Pattern.compile("([^:]+):([^/]+)/(\\S+)"); - private final Context mContext; + private final @NonNull Context mContext; + private final @NonNull Runnable mOnErrorCallback; - public InlineSuggestionUi(Context context) { + InlineSuggestionUi(@NonNull Context context, @NonNull Runnable onErrorCallback) { this.mContext = context; + mOnErrorCallback = onErrorCallback; } /** @@ -94,15 +96,17 @@ public class InlineSuggestionUi { } final View suggestionView = renderSlice(inlinePresentation.getSlice(), contextThemeWrapper); - if (onClickListener != null) { - suggestionView.setOnClickListener(onClickListener); - } + + final InlineSuggestionRoot suggestionRoot = new InlineSuggestionRoot( + mContext, mOnErrorCallback); + suggestionRoot.addView(suggestionView); + suggestionRoot.setOnClickListener(onClickListener); WindowManager.LayoutParams lp = new WindowManager.LayoutParams(width, height, WindowManager.LayoutParams.TYPE_APPLICATION, 0, PixelFormat.TRANSPARENT); - wvr.addView(suggestionView, lp); + wvr.addView(suggestionRoot, lp); return sc; } diff --git a/services/core/java/com/android/server/input/InputManagerService.java b/services/core/java/com/android/server/input/InputManagerService.java index 90358ca1f133c..a8f706cdd6297 100644 --- a/services/core/java/com/android/server/input/InputManagerService.java +++ b/services/core/java/com/android/server/input/InputManagerService.java @@ -223,7 +223,7 @@ public class InputManagerService extends IInputManager.Stub int displayId, InputApplicationHandle application); private static native void nativeSetFocusedDisplay(long ptr, int displayId); private static native boolean nativeTransferTouchFocus(long ptr, - InputChannel fromChannel, InputChannel toChannel); + IBinder fromChannelToken, IBinder toChannelToken); private static native void nativeSetPointerSpeed(long ptr, int speed); private static native void nativeSetShowTouches(long ptr, boolean enabled); private static native void nativeSetInteractive(long ptr, boolean interactive); @@ -1554,14 +1554,29 @@ public class InputManagerService extends IInputManager.Stub * @return True if the transfer was successful. False if the window with the * specified channel did not actually have touch focus at the time of the request. */ - public boolean transferTouchFocus(InputChannel fromChannel, InputChannel toChannel) { - if (fromChannel == null) { - throw new IllegalArgumentException("fromChannel must not be null."); - } - if (toChannel == null) { - throw new IllegalArgumentException("toChannel must not be null."); - } - return nativeTransferTouchFocus(mPtr, fromChannel, toChannel); + public boolean transferTouchFocus(@NonNull InputChannel fromChannel, + @NonNull InputChannel toChannel) { + return nativeTransferTouchFocus(mPtr, fromChannel.getToken(), toChannel.getToken()); + } + + /** + * Atomically transfers touch focus from one window to another as identified by + * their input channels. It is possible for multiple windows to have + * touch focus if they support split touch dispatch + * {@link android.view.WindowManager.LayoutParams#FLAG_SPLIT_TOUCH} but this + * method only transfers touch focus of the specified window without affecting + * other windows that may also have touch focus at the same time. + * @param fromChannelToken The channel token of a window that currently has touch focus. + * @param toChannelToken The channel token of the window that should receive touch focus in + * place of the first. + * @return True if the transfer was successful. False if the window with the + * specified channel did not actually have touch focus at the time of the request. + */ + public boolean transferTouchFocus(@NonNull IBinder fromChannelToken, + @NonNull IBinder toChannelToken) { + Objects.nonNull(fromChannelToken); + Objects.nonNull(toChannelToken); + return nativeTransferTouchFocus(mPtr, fromChannelToken, toChannelToken); } @Override // Binder call diff --git a/services/core/java/com/android/server/wm/WindowManagerInternal.java b/services/core/java/com/android/server/wm/WindowManagerInternal.java index 6e243f0b937ac..59eee9cc94041 100644 --- a/services/core/java/com/android/server/wm/WindowManagerInternal.java +++ b/services/core/java/com/android/server/wm/WindowManagerInternal.java @@ -562,4 +562,14 @@ public abstract class WindowManagerInternal { */ public abstract void setAccessibilityIdToSurfaceMetadata( IBinder windowToken, int accessibilityWindowId); + + /** + * Transfers input focus from a given input token to that of the IME window. + * + * @param sourceInputToken The source token. + * @param displayId The display hosting the IME window. + * @return Whether transfer was successful. + */ + public abstract boolean transferTouchFocusToImeWindow(@NonNull IBinder sourceInputToken, + int displayId); } diff --git a/services/core/java/com/android/server/wm/WindowManagerService.java b/services/core/java/com/android/server/wm/WindowManagerService.java index a5b99b0a26a3f..6e1f46bb5e42d 100644 --- a/services/core/java/com/android/server/wm/WindowManagerService.java +++ b/services/core/java/com/android/server/wm/WindowManagerService.java @@ -7558,6 +7558,29 @@ public class WindowManagerService extends IWindowManager.Stub } } } + + @Override + public boolean transferTouchFocusToImeWindow(@NonNull IBinder sourceInputToken, + int displayId) { + final IBinder destinationInputToken; + + synchronized (mGlobalLock) { + final DisplayContent displayContent = mRoot.getDisplayContent(displayId); + if (displayContent == null) { + return false; + } + final WindowState imeWindow = displayContent.mInputMethodWindow; + if (imeWindow == null) { + return false; + } + if (imeWindow.mInputChannel == null) { + return false; + } + destinationInputToken = imeWindow.mInputChannel.getToken(); + } + + return mInputManager.transferTouchFocus(sourceInputToken, destinationInputToken); + } } void registerAppFreezeListener(AppFreezeListener listener) { diff --git a/services/core/jni/com_android_server_input_InputManagerService.cpp b/services/core/jni/com_android_server_input_InputManagerService.cpp index 212a3a6386344..49db3d5b2b64c 100644 --- a/services/core/jni/com_android_server_input_InputManagerService.cpp +++ b/services/core/jni/com_android_server_input_InputManagerService.cpp @@ -1563,20 +1563,17 @@ static void nativeSetSystemUiVisibility(JNIEnv* /* env */, } static jboolean nativeTransferTouchFocus(JNIEnv* env, - jclass /* clazz */, jlong ptr, jobject fromChannelObj, jobject toChannelObj) { - NativeInputManager* im = reinterpret_cast(ptr); - - sp fromChannel = - android_view_InputChannel_getInputChannel(env, fromChannelObj); - sp toChannel = - android_view_InputChannel_getInputChannel(env, toChannelObj); - - if (fromChannel == nullptr || toChannel == nullptr) { + jclass /* clazz */, jlong ptr, jobject fromChannelTokenObj, jobject toChannelTokenObj) { + if (fromChannelTokenObj == nullptr || toChannelTokenObj == nullptr) { return JNI_FALSE; } + sp fromChannelToken = ibinderForJavaObject(env, fromChannelTokenObj); + sp toChannelToken = ibinderForJavaObject(env, toChannelTokenObj); + + NativeInputManager* im = reinterpret_cast(ptr); if (im->getInputManager()->getDispatcher()->transferTouchFocus( - fromChannel->getConnectionToken(), toChannel->getConnectionToken())) { + fromChannelToken, toChannelToken)) { return JNI_TRUE; } else { return JNI_FALSE; @@ -1784,7 +1781,7 @@ static const JNINativeMethod gInputManagerMethods[] = { (void*) nativeSetInputDispatchMode }, { "nativeSetSystemUiVisibility", "(JI)V", (void*) nativeSetSystemUiVisibility }, - { "nativeTransferTouchFocus", "(JLandroid/view/InputChannel;Landroid/view/InputChannel;)Z", + { "nativeTransferTouchFocus", "(JLandroid/os/IBinder;Landroid/os/IBinder;)Z", (void*) nativeTransferTouchFocus }, { "nativeSetPointerSpeed", "(JI)V", (void*) nativeSetPointerSpeed }, diff --git a/services/tests/wmtests/src/com/android/server/wm/DragDropControllerTests.java b/services/tests/wmtests/src/com/android/server/wm/DragDropControllerTests.java index 5acc0f2735546..956c200022e95 100644 --- a/services/tests/wmtests/src/com/android/server/wm/DragDropControllerTests.java +++ b/services/tests/wmtests/src/com/android/server/wm/DragDropControllerTests.java @@ -126,7 +126,8 @@ public class DragDropControllerTests extends WindowTestsBase { mTarget = new TestDragDropController(mWm, mWm.mH.getLooper()); mWindow = createDropTargetWindow("Drag test window", 0); doReturn(mWindow).when(mDisplayContent).getTouchableWinAtPointLocked(0, 0); - when(mWm.mInputManager.transferTouchFocus(any(), any())).thenReturn(true); + when(mWm.mInputManager.transferTouchFocus(any(InputChannel.class), + any(InputChannel.class))).thenReturn(true); mWm.mWindowMap.put(mWindow.mClient.asBinder(), mWindow); } @@ -176,7 +177,8 @@ public class DragDropControllerTests extends WindowTestsBase { .setFormat(PixelFormat.TRANSLUCENT) .build(); - assertTrue(mWm.mInputManager.transferTouchFocus(null, null)); + assertTrue(mWm.mInputManager.transferTouchFocus(new InputChannel(), + new InputChannel())); mToken = mTarget.performDrag( new SurfaceSession(), 0, 0, mWindow.mClient, flag, surface, 0, 0, 0, 0, 0, data);