From b46851c964ab00c7a38a0420e8ed27c2489c32c2 Mon Sep 17 00:00:00 2001 From: Feng Cao Date: Wed, 29 Apr 2020 18:22:49 -0700 Subject: [PATCH] Support re-attaching the inline suggestion view to window * Before this change, when the inline suggestion view is detached from the IME window (e.g. due to layout change), the remote view content will not show again even after the view is reattached to the window on the IME side. This patch fixes it by requesting the remote view owner (the ext services) for a new SurfacePackage when the view is re-attached to the window (see javadoc of SurfaceControlViewHost for why this works). * This patch also fixes the issue where the SurfaceControlViewHost was never released in the ext services. This is done by notifying the ext services (through system server) when the view is detached from the IME window, and then the system server will release the SurfaceControlViewHost if after N(=500) ms the view is not re-attached to a window. * After the SurfaceControlViewHost is released, if the view is re-attached to the window later, a new SurfaceControlViewHost will be created to back the same InlineContentView. * The current code structure also lays a foundation for a subsequent change to allow reusing the same remote view for inline suggestions during filtering. Test: atest CtsAutofillServiceTestCases (sanity test) Bug: 153615023 Bug: 154683107 Change-Id: Idc587e1e82a96b792c351796464821b7aad7cd89 --- .../service/autofill/IInlineSuggestionUi.aidl | 29 ++ .../autofill/IInlineSuggestionUiCallback.aidl | 6 +- .../ISurfacePackageResultCallback.aidl | 28 ++ .../InlineSuggestionRenderService.java | 121 +++++++- .../view/inputmethod/InlineSuggestion.java | 270 ++++++++++++---- .../widget/inline/InlineContentView.java | 137 ++++++--- .../view/inline/IInlineContentProvider.aidl | 2 + ...tofillInlineSuggestionsRequestSession.java | 4 +- .../RemoteInlineSuggestionRenderService.java | 2 +- .../ui/InlineContentProviderImpl.java | 140 +++++++++ .../autofill/ui/InlineSuggestionFactory.java | 102 ++---- .../autofill/ui/RemoteInlineSuggestionUi.java | 291 ++++++++++++++++++ .../RemoteInlineSuggestionViewConnector.java | 125 ++++++++ 13 files changed, 1069 insertions(+), 188 deletions(-) create mode 100644 core/java/android/service/autofill/IInlineSuggestionUi.aidl create mode 100644 core/java/android/service/autofill/ISurfacePackageResultCallback.aidl create mode 100644 services/autofill/java/com/android/server/autofill/ui/InlineContentProviderImpl.java create mode 100644 services/autofill/java/com/android/server/autofill/ui/RemoteInlineSuggestionUi.java create mode 100644 services/autofill/java/com/android/server/autofill/ui/RemoteInlineSuggestionViewConnector.java diff --git a/core/java/android/service/autofill/IInlineSuggestionUi.aidl b/core/java/android/service/autofill/IInlineSuggestionUi.aidl new file mode 100644 index 0000000000000..7289853064f87 --- /dev/null +++ b/core/java/android/service/autofill/IInlineSuggestionUi.aidl @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.service.autofill; + +import android.service.autofill.ISurfacePackageResultCallback; + +/** + * Interface to interact with a remote inline suggestion UI. + * + * @hide + */ +oneway interface IInlineSuggestionUi { + void getSurfacePackage(ISurfacePackageResultCallback callback); + void releaseSurfaceControlViewHost(); +} diff --git a/core/java/android/service/autofill/IInlineSuggestionUiCallback.aidl b/core/java/android/service/autofill/IInlineSuggestionUiCallback.aidl index 172cfef9fee26..97eb790b9acce 100644 --- a/core/java/android/service/autofill/IInlineSuggestionUiCallback.aidl +++ b/core/java/android/service/autofill/IInlineSuggestionUiCallback.aidl @@ -18,17 +18,19 @@ package android.service.autofill; import android.content.IntentSender; import android.os.IBinder; +import android.service.autofill.IInlineSuggestionUi; import android.view.SurfaceControlViewHost; /** - * Interface to receive events from inline suggestions. + * Interface to receive events from a remote inline suggestion UI. * * @hide */ oneway interface IInlineSuggestionUiCallback { void onClick(); void onLongClick(); - void onContent(in SurfaceControlViewHost.SurfacePackage surface, int width, int height); + void onContent(in IInlineSuggestionUi content, in SurfaceControlViewHost.SurfacePackage surface, + int width, int height); void onError(); void onTransferTouchFocusToImeWindow(in IBinder sourceInputToken, int displayId); void onStartIntentSender(in IntentSender intentSender); diff --git a/core/java/android/service/autofill/ISurfacePackageResultCallback.aidl b/core/java/android/service/autofill/ISurfacePackageResultCallback.aidl new file mode 100644 index 0000000000000..0c2c624952eb3 --- /dev/null +++ b/core/java/android/service/autofill/ISurfacePackageResultCallback.aidl @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.service.autofill; + +import android.view.SurfaceControlViewHost; + +/** + * Interface to receive a SurfaceControlViewHost.SurfacePackage. + * + * @hide + */ +oneway interface ISurfacePackageResultCallback { + void onResult(in SurfaceControlViewHost.SurfacePackage result); +} diff --git a/core/java/android/service/autofill/InlineSuggestionRenderService.java b/core/java/android/service/autofill/InlineSuggestionRenderService.java index 6c22b1936d747..3ea443bab3f80 100644 --- a/core/java/android/service/autofill/InlineSuggestionRenderService.java +++ b/core/java/android/service/autofill/InlineSuggestionRenderService.java @@ -33,6 +33,7 @@ import android.os.Looper; import android.os.RemoteCallback; import android.os.RemoteException; import android.util.Log; +import android.util.LruCache; import android.util.Size; import android.view.Display; import android.view.SurfaceControlViewHost; @@ -40,6 +41,8 @@ import android.view.View; import android.view.ViewGroup; import android.view.WindowManager; +import java.lang.ref.WeakReference; + /** * A service that renders an inline presentation view given the {@link InlinePresentation}. * @@ -65,6 +68,27 @@ public abstract class InlineSuggestionRenderService extends Service { private IInlineSuggestionUiCallback mCallback; + + /** + * A local LRU cache keeping references to the inflated {@link SurfaceControlViewHost}s, so + * they can be released properly when no longer used. Each view needs to be tracked separately, + * therefore for simplicity we use the hash code of the value object as key in the cache. + */ + private final LruCache mActiveInlineSuggestions = + new LruCache(30) { + @Override + public void entryRemoved(boolean evicted, InlineSuggestionUiImpl key, + Boolean oldValue, + Boolean newValue) { + if (evicted) { + Log.w(TAG, + "Hit max=100 entries in the cache. Releasing oldest one to make " + + "space."); + key.releaseSurfaceControlViewHost(); + } + } + }; + /** * If the specified {@code width}/{@code height} is an exact value, then it will be returned as * is, otherwise the method tries to measure a size that is just large enough to fit the view @@ -169,8 +193,14 @@ public abstract class InlineSuggestionRenderService extends Service { return true; }); - sendResult(callback, host.getSurfacePackage(), measuredSize.getWidth(), - measuredSize.getHeight()); + try { + InlineSuggestionUiImpl uiImpl = new InlineSuggestionUiImpl(host, mHandler); + mActiveInlineSuggestions.put(uiImpl, true); + callback.onContent(new InlineSuggestionUiWrapper(uiImpl), host.getSurfacePackage(), + measuredSize.getWidth(), measuredSize.getHeight()); + } catch (RemoteException e) { + Log.w(TAG, "RemoteException calling onContent()"); + } } finally { updateDisplay(Display.DEFAULT_DISPLAY); } @@ -181,12 +211,87 @@ public abstract class InlineSuggestionRenderService extends Service { callback.sendResult(rendererInfo); } - private void sendResult(@NonNull IInlineSuggestionUiCallback callback, - @Nullable SurfaceControlViewHost.SurfacePackage surface, int width, int height) { - try { - callback.onContent(surface, width, height); - } catch (RemoteException e) { - Log.w(TAG, "RemoteException calling onContent(" + surface + ")"); + /** + * A wrapper class around the {@link InlineSuggestionUiImpl} to ensure it's not strongly + * reference by the remote system server process. + */ + private static final class InlineSuggestionUiWrapper extends + android.service.autofill.IInlineSuggestionUi.Stub { + + private final WeakReference mUiImpl; + + InlineSuggestionUiWrapper(InlineSuggestionUiImpl uiImpl) { + mUiImpl = new WeakReference<>(uiImpl); + } + + @Override + public void releaseSurfaceControlViewHost() { + final InlineSuggestionUiImpl uiImpl = mUiImpl.get(); + if (uiImpl != null) { + uiImpl.releaseSurfaceControlViewHost(); + } + } + + @Override + public void getSurfacePackage(ISurfacePackageResultCallback callback) { + final InlineSuggestionUiImpl uiImpl = mUiImpl.get(); + if (uiImpl != null) { + uiImpl.getSurfacePackage(callback); + } + } + } + + /** + * Keeps track of a SurfaceControlViewHost to ensure it's released when its lifecycle ends. + * + *

This class is thread safe, because all the outside calls are piped into a single + * handler thread to be processed. + */ + private final class InlineSuggestionUiImpl { + + @Nullable + private SurfaceControlViewHost mViewHost; + @NonNull + private final Handler mHandler; + + InlineSuggestionUiImpl(SurfaceControlViewHost viewHost, Handler handler) { + this.mViewHost = viewHost; + this.mHandler = handler; + } + + /** + * Call {@link SurfaceControlViewHost#release()} to release it. After this, this view is + * not usable, and any further calls to the + * {@link #getSurfacePackage(ISurfacePackageResultCallback)} will get {@code null} result. + */ + public void releaseSurfaceControlViewHost() { + mHandler.post(() -> { + if (mViewHost == null) { + return; + } + Log.v(TAG, "Releasing inline suggestion view host"); + mViewHost.release(); + mViewHost = null; + InlineSuggestionRenderService.this.mActiveInlineSuggestions.remove( + InlineSuggestionUiImpl.this); + Log.v(TAG, "Removed the inline suggestion from the cache, current size=" + + InlineSuggestionRenderService.this.mActiveInlineSuggestions.size()); + }); + } + + /** + * Sends back a new {@link android.view.SurfaceControlViewHost.SurfacePackage} if the view + * is not released, {@code null} otherwise. + */ + public void getSurfacePackage(ISurfacePackageResultCallback callback) { + Log.d(TAG, "getSurfacePackage"); + mHandler.post(() -> { + try { + callback.onResult(mViewHost == null ? null : mViewHost.getSurfacePackage()); + } catch (RemoteException e) { + Log.w(TAG, "RemoteException calling onSurfacePackage"); + } + }); } } diff --git a/core/java/android/view/inputmethod/InlineSuggestion.java b/core/java/android/view/inputmethod/InlineSuggestion.java index 6b1a480986c8e..4c72474435a42 100644 --- a/core/java/android/view/inputmethod/InlineSuggestion.java +++ b/core/java/android/view/inputmethod/InlineSuggestion.java @@ -18,11 +18,13 @@ package android.view.inputmethod; import android.annotation.BinderThread; import android.annotation.CallbackExecutor; +import android.annotation.MainThread; import android.annotation.NonNull; import android.annotation.Nullable; import android.annotation.TestApi; import android.content.Context; -import android.os.AsyncTask; +import android.os.Handler; +import android.os.Looper; import android.os.Parcel; import android.os.Parcelable; import android.os.RemoteException; @@ -42,26 +44,26 @@ import java.util.concurrent.Executor; import java.util.function.Consumer; /** - * This class represents an inline suggestion which is made by one app - * and can be embedded into the UI of another. Suggestions may contain - * sensitive information not known to the host app which needs to be - * protected from spoofing. To address that the suggestion view inflated - * on demand for embedding is created in such a way that the hosting app - * cannot introspect its content and cannot interact with it. + * This class represents an inline suggestion which is made by one app and can be embedded into the + * UI of another. Suggestions may contain sensitive information not known to the host app which + * needs to be protected from spoofing. To address that the suggestion view inflated on demand for + * embedding is created in such a way that the hosting app cannot introspect its content and cannot + * interact with it. */ -@DataClass( - genEqualsHashCode = true, - genToString = true, - genHiddenConstDefs = true, +@DataClass(genEqualsHashCode = true, genToString = true, genHiddenConstDefs = true, genHiddenConstructor = true) -@DataClass.Suppress({"getContentProvider"}) public final class InlineSuggestion implements Parcelable { private static final String TAG = "InlineSuggestion"; - private final @NonNull InlineSuggestionInfo mInfo; + @NonNull + private final InlineSuggestionInfo mInfo; - private final @Nullable IInlineContentProvider mContentProvider; + /** + * @hide + */ + @Nullable + private final IInlineContentProvider mContentProvider; /** * Used to keep a strong reference to the callback so it doesn't get garbage collected. @@ -69,7 +71,8 @@ public final class InlineSuggestion implements Parcelable { * @hide */ @DataClass.ParcelWith(InlineContentCallbackImplParceling.class) - private @Nullable InlineContentCallbackImpl mInlineContentCallback; + @Nullable + private InlineContentCallbackImpl mInlineContentCallback; /** * Creates a new {@link InlineSuggestion}, for testing purpose. @@ -87,8 +90,7 @@ public final class InlineSuggestion implements Parcelable { * * @hide */ - public InlineSuggestion( - @NonNull InlineSuggestionInfo info, + public InlineSuggestion(@NonNull InlineSuggestionInfo info, @Nullable IInlineContentProvider contentProvider) { this(info, contentProvider, /* inlineContentCallback */ null); } @@ -96,25 +98,30 @@ public final class InlineSuggestion implements Parcelable { /** * Inflates a view with the content of this suggestion at a specific size. * - *

The size must be either 1) between the - * {@link android.widget.inline.InlinePresentationSpec#getMinSize() min size} and the - * {@link android.widget.inline.InlinePresentationSpec#getMaxSize() max size} of the - * presentation spec returned by {@link InlineSuggestionInfo#getInlinePresentationSpec()}, - * or 2) {@link ViewGroup.LayoutParams#WRAP_CONTENT}. If the size is set to - * {@link ViewGroup.LayoutParams#WRAP_CONTENT}, then the size of the inflated view will be just - * large enough to fit the content, while still conforming to the min / max size specified by - * the {@link android.widget.inline.InlinePresentationSpec}. + *

Each dimension of the size must satisfy one of the following conditions: + * + *

    + *
  1. between {@link android.widget.inline.InlinePresentationSpec#getMinSize()} and + * {@link android.widget.inline.InlinePresentationSpec#getMaxSize()} of the presentation spec + * from {@code mInfo} + *
  2. {@link ViewGroup.LayoutParams#WRAP_CONTENT} + *
+ * + * If the size is set to {@link + * ViewGroup.LayoutParams#WRAP_CONTENT}, then the size of the inflated view will be just large + * enough to fit the content, while still conforming to the min / max size specified by the + * {@link android.widget.inline.InlinePresentationSpec}. * *

The caller can attach an {@link android.view.View.OnClickListener} and/or an - * {@link android.view.View.OnLongClickListener} to the view in the - * {@code callback} to receive click and long click events on the view. + * {@link android.view.View.OnLongClickListener} to the view in the {@code callback} to receive + * click and long click events on the view. * * @param context Context in which to inflate the view. - * @param size The size at which to inflate the suggestion. For each dimension, it maybe - * an exact value or {@link ViewGroup.LayoutParams#WRAP_CONTENT}. - * @param callback Callback for receiving the inflated view, where the - * {@link ViewGroup.LayoutParams} of the view is set as the actual size of - * the underlying remote view. + * @param size The size at which to inflate the suggestion. For each dimension, it maybe an + * exact value or {@link ViewGroup.LayoutParams#WRAP_CONTENT}. + * @param callback Callback for receiving the inflated view, where the {@link + * ViewGroup.LayoutParams} of the view is set as the actual size of the + * underlying remote view. * @throws IllegalArgumentException If an invalid argument is passed. * @throws IllegalStateException If this method is already called. */ @@ -130,19 +137,17 @@ public final class InlineSuggestion implements Parcelable { + ", nor wrap_content"); } mInlineContentCallback = getInlineContentCallback(context, callbackExecutor, callback); - AsyncTask.THREAD_POOL_EXECUTOR.execute(() -> { - if (mContentProvider == null) { - callback.accept(/* view */ null); - return; - } - try { - mContentProvider.provideContent(size.getWidth(), size.getHeight(), - new InlineContentCallbackWrapper(mInlineContentCallback)); - } catch (RemoteException e) { - Slog.w(TAG, "Error creating suggestion content surface: " + e); - callback.accept(/* view */ null); - } - }); + if (mContentProvider == null) { + callbackExecutor.execute(() -> callback.accept(/* view */ null)); + return; + } + try { + mContentProvider.provideContent(size.getWidth(), size.getHeight(), + new InlineContentCallbackWrapper(mInlineContentCallback)); + } catch (RemoteException e) { + Slog.w(TAG, "Error creating suggestion content surface: " + e); + callbackExecutor.execute(() -> callback.accept(/* view */ null)); + } } /** @@ -161,9 +166,14 @@ public final class InlineSuggestion implements Parcelable { if (mInlineContentCallback != null) { throw new IllegalStateException("Already called #inflate()"); } - return new InlineContentCallbackImpl(context, callbackExecutor, callback); + return new InlineContentCallbackImpl(context, mContentProvider, callbackExecutor, + callback); } + /** + * A wrapper class around the {@link InlineContentCallbackImpl} to ensure it's not strongly + * reference by the remote system server process. + */ private static final class InlineContentCallbackWrapper extends IInlineContentCallback.Stub { private final WeakReference mCallbackImpl; @@ -201,17 +211,68 @@ public final class InlineSuggestion implements Parcelable { } } + /** + * Handles the communication between the inline suggestion view in current (IME) process and + * the remote view provided from the system server. + * + *

This class is thread safe, because all the outside calls are piped into a single + * handler thread to be processed. + */ private static final class InlineContentCallbackImpl { - private final @NonNull Context mContext; - private final @NonNull Executor mCallbackExecutor; - private final @NonNull Consumer mCallback; - private @Nullable InlineContentView mView; + @NonNull + private final Handler mMainHandler = new Handler(Looper.getMainLooper()); + + @NonNull + private final Context mContext; + @Nullable + private final IInlineContentProvider mInlineContentProvider; + @NonNull + private final Executor mCallbackExecutor; + + /** + * Callback from the client (IME) that will receive the inflated suggestion view. It'll + * only be called once when the view SurfacePackage is first sent back to the client. Any + * updates to the view due to attach to window and detach from window events will be + * handled under the hood, transparent from the client. + */ + @NonNull + private final Consumer mCallback; + + /** + * Indicates whether the first content has been received or not. + */ + private boolean mFirstContentReceived = false; + + /** + * The client (IME) side view which internally wraps a remote view. It'll be set when + * {@link #onContent(SurfaceControlViewHost.SurfacePackage, int, int)} is called, which + * should only happen once in the lifecycle of this inline suggestion instance. + */ + @Nullable + private InlineContentView mView; + + /** + * The SurfacePackage pointing to the remote view. It's cached here to be sent to the next + * available consumer. + */ + @Nullable + private SurfaceControlViewHost.SurfacePackage mSurfacePackage; + + /** + * The callback (from the {@link InlineContentView}) which consumes the surface package. + * It's cached here to be called when the SurfacePackage is returned from the remote + * view owning process. + */ + @Nullable + private Consumer mSurfacePackageConsumer; InlineContentCallbackImpl(@NonNull Context context, + @Nullable IInlineContentProvider inlineContentProvider, @NonNull @CallbackExecutor Executor callbackExecutor, @NonNull Consumer callback) { mContext = context; + mInlineContentProvider = inlineContentProvider; mCallbackExecutor = callbackExecutor; mCallback = callback; } @@ -219,28 +280,110 @@ public final class InlineSuggestion implements Parcelable { @BinderThread public void onContent(SurfaceControlViewHost.SurfacePackage content, int width, int height) { - if (content == null) { + mMainHandler.post(() -> handleOnContent(content, width, height)); + } + + @MainThread + private void handleOnContent(SurfaceControlViewHost.SurfacePackage content, int width, + int height) { + if (!mFirstContentReceived) { + handleOnFirstContentReceived(content, width, height); + mFirstContentReceived = true; + } else { + handleOnSurfacePackage(content); + } + } + + /** + * Called when the view content is returned for the first time. + */ + @MainThread + private void handleOnFirstContentReceived(SurfaceControlViewHost.SurfacePackage content, + int width, int height) { + mSurfacePackage = content; + if (mSurfacePackage == null) { mCallbackExecutor.execute(() -> mCallback.accept(/* view */null)); } else { mView = new InlineContentView(mContext); mView.setLayoutParams(new ViewGroup.LayoutParams(width, height)); - mView.setChildSurfacePackage(content); + mView.setChildSurfacePackageUpdater(getSurfacePackageUpdater()); mCallbackExecutor.execute(() -> mCallback.accept(mView)); } } + /** + * Called when any subsequent SurfacePackage is returned from the remote view owning + * process. + */ + @MainThread + private void handleOnSurfacePackage(SurfaceControlViewHost.SurfacePackage surfacePackage) { + mSurfacePackage = surfacePackage; + if (mSurfacePackage != null && mSurfacePackageConsumer != null) { + mSurfacePackageConsumer.accept(mSurfacePackage); + mSurfacePackageConsumer = null; + } + } + + @MainThread + private void handleOnSurfacePackageReleased() { + mSurfacePackage = null; + try { + mInlineContentProvider.onSurfacePackageReleased(); + } catch (RemoteException e) { + Slog.w(TAG, "Error calling onSurfacePackageReleased(): " + e); + } + } + + @MainThread + private void handleGetSurfacePackage( + Consumer consumer) { + if (mSurfacePackage != null) { + consumer.accept(mSurfacePackage); + } else { + mSurfacePackageConsumer = consumer; + try { + mInlineContentProvider.requestSurfacePackage(); + } catch (RemoteException e) { + Slog.w(TAG, "Error calling getSurfacePackage(): " + e); + consumer.accept(null); + mSurfacePackageConsumer = null; + } + } + } + + private InlineContentView.SurfacePackageUpdater getSurfacePackageUpdater() { + return new InlineContentView.SurfacePackageUpdater() { + @Override + public void onSurfacePackageReleased() { + mMainHandler.post( + () -> InlineContentCallbackImpl.this.handleOnSurfacePackageReleased()); + } + + @Override + public void getSurfacePackage( + Consumer consumer) { + mMainHandler.post( + () -> InlineContentCallbackImpl.this.handleGetSurfacePackage(consumer)); + } + }; + } + @BinderThread public void onClick() { - if (mView != null && mView.hasOnClickListeners()) { - mView.callOnClick(); - } + mMainHandler.post(() -> { + if (mView != null && mView.hasOnClickListeners()) { + mView.callOnClick(); + } + }); } @BinderThread public void onLongClick() { - if (mView != null && mView.hasOnLongClickListeners()) { - mView.performLongClick(); - } + mMainHandler.post(() -> { + if (mView != null && mView.hasOnLongClickListeners()) { + mView.performLongClick(); + } + }); } } @@ -262,6 +405,7 @@ public final class InlineSuggestion implements Parcelable { + // Code below generated by codegen v1.0.15. // // DO NOT MODIFY! @@ -301,6 +445,14 @@ public final class InlineSuggestion implements Parcelable { return mInfo; } + /** + * @hide + */ + @DataClass.Generated.Member + public @Nullable IInlineContentProvider getContentProvider() { + return mContentProvider; + } + /** * Used to keep a strong reference to the callback so it doesn't get garbage collected. * @@ -421,7 +573,7 @@ public final class InlineSuggestion implements Parcelable { }; @DataClass.Generated( - time = 1587771173367L, + time = 1588308946517L, codegenVersion = "1.0.15", sourceFile = "frameworks/base/core/java/android/view/inputmethod/InlineSuggestion.java", inputSignatures = "private static final java.lang.String TAG\nprivate final @android.annotation.NonNull android.view.inputmethod.InlineSuggestionInfo mInfo\nprivate final @android.annotation.Nullable com.android.internal.view.inline.IInlineContentProvider mContentProvider\nprivate @com.android.internal.util.DataClass.ParcelWith(android.view.inputmethod.InlineSuggestion.InlineContentCallbackImplParceling.class) @android.annotation.Nullable android.view.inputmethod.InlineSuggestion.InlineContentCallbackImpl mInlineContentCallback\npublic static @android.annotation.TestApi @android.annotation.NonNull android.view.inputmethod.InlineSuggestion newInlineSuggestion(android.view.inputmethod.InlineSuggestionInfo)\npublic void inflate(android.content.Context,android.util.Size,java.util.concurrent.Executor,java.util.function.Consumer)\nprivate static boolean isValid(int,int,int)\nprivate synchronized android.view.inputmethod.InlineSuggestion.InlineContentCallbackImpl getInlineContentCallback(android.content.Context,java.util.concurrent.Executor,java.util.function.Consumer)\nclass InlineSuggestion extends java.lang.Object implements [android.os.Parcelable]\n@com.android.internal.util.DataClass(genEqualsHashCode=true, genToString=true, genHiddenConstDefs=true, genHiddenConstructor=true)") diff --git a/core/java/android/widget/inline/InlineContentView.java b/core/java/android/widget/inline/InlineContentView.java index 4f2af63626cf0..8657e828a3f61 100644 --- a/core/java/android/widget/inline/InlineContentView.java +++ b/core/java/android/widget/inline/InlineContentView.java @@ -21,40 +21,45 @@ import android.annotation.Nullable; import android.content.Context; import android.graphics.PixelFormat; import android.util.AttributeSet; +import android.util.Log; import android.view.SurfaceControl; import android.view.SurfaceControlViewHost; import android.view.SurfaceHolder; import android.view.SurfaceView; import android.view.ViewGroup; +import java.util.function.Consumer; + /** - * This class represents a view that holds opaque content from another app that - * you can inline in your UI. + * This class represents a view that holds opaque content from another app that you can inline in + * your UI. * *

Since the content presented by this view is from another security domain,it is - * shown on a remote surface preventing the host application from accessing that content. - * Also the host application cannot interact with the inlined content by injecting touch - * events or clicking programmatically. + * shown on a remote surface preventing the host application from accessing that content. Also the + * host application cannot interact with the inlined content by injecting touch events or clicking + * programmatically. * *

This view can be overlaid by other windows, i.e. redressed, but if this is the case - * the inined UI would not be interactive. Sometimes this is desirable, e.g. animating - * transitions. + * the inlined UI would not be interactive. Sometimes this is desirable, e.g. animating transitions. * *

By default the surface backing this view is shown on top of the hosting window such - * that the inlined content is interactive. However, you can temporarily move the surface - * under the hosting window which could be useful in some cases, e.g. animating transitions. - * At this point the inlined content will not be interactive and the touch events would - * be delivered to your app. - *

- * Instances of this class are created by the platform and can be programmatically attached - * to your UI. Once you attach and detach this view it can not longer be reused and you - * should obtain a new view from the platform via the dedicated APIs. + * that the inlined content is interactive. However, you can temporarily move the surface under the + * hosting window which could be useful in some cases, e.g. animating transitions. At this point the + * inlined content will not be interactive and the touch events would be delivered to your app. + * + *

Instances of this class are created by the platform and can be programmatically attached to + * your UI. Once the view is attached to the window, you may detach and reattach it to the window. + * It should work seamlessly from the hosting process's point of view. */ public class InlineContentView extends ViewGroup { + private static final String TAG = "InlineContentView"; + + private static final boolean DEBUG = false; + /** - * Callback for observing the lifecycle of the surface control - * that manipulates the backing secure embedded UI surface. + * Callback for observing the lifecycle of the surface control that manipulates the backing + * secure embedded UI surface. */ public interface SurfaceControlCallback { /** @@ -72,15 +77,41 @@ public class InlineContentView extends ViewGroup { void onDestroyed(@NonNull SurfaceControl surfaceControl); } - private final @NonNull SurfaceHolder.Callback mSurfaceCallback = new SurfaceHolder.Callback() { + /** + * Callback for sending an updated surface package in case the previous one is released + * from the detached from window event, and for getting notified of such event. + * + * This is expected to be provided to the {@link InlineContentView} so it can get updates + * from and send updates to the remote content (i.e. surface package) provider. + * + * @hide + */ + public interface SurfacePackageUpdater { + + /** + * Called when the previous surface package is released due to view being detached + * from the window. + */ + void onSurfacePackageReleased(); + + /** + * Called to request an updated surface package. + * + * @param consumer consumes the updated surface package. + */ + void getSurfacePackage(Consumer consumer); + } + + @NonNull + private final SurfaceHolder.Callback mSurfaceCallback = new SurfaceHolder.Callback() { @Override public void surfaceCreated(@NonNull SurfaceHolder holder) { mSurfaceControlCallback.onCreated(mSurfaceView.getSurfaceControl()); } @Override - public void surfaceChanged(@NonNull SurfaceHolder holder, - int format, int width, int height) { + public void surfaceChanged(@NonNull SurfaceHolder holder, int format, int width, + int height) { /* do nothing */ } @@ -90,13 +121,17 @@ public class InlineContentView extends ViewGroup { } }; - private final @NonNull SurfaceView mSurfaceView; + @NonNull + private final SurfaceView mSurfaceView; - private @Nullable SurfaceControlCallback mSurfaceControlCallback; + @Nullable + private SurfaceControlCallback mSurfaceControlCallback; + + @Nullable + private SurfacePackageUpdater mSurfacePackageUpdater; /** * @inheritDoc - * * @hide */ public InlineContentView(@NonNull Context context) { @@ -105,7 +140,6 @@ public class InlineContentView extends ViewGroup { /** * @inheritDoc - * * @hide */ public InlineContentView(@NonNull Context context, @Nullable AttributeSet attrs) { @@ -114,7 +148,6 @@ public class InlineContentView extends ViewGroup { /** * @inheritDoc - * * @hide */ public InlineContentView(@NonNull Context context, @Nullable AttributeSet attrs, @@ -123,20 +156,18 @@ public class InlineContentView extends ViewGroup { } /** - * Gets the surface control. If the surface is not created this method - * returns {@code null}. + * Gets the surface control. If the surface is not created this method returns {@code null}. * * @return The surface control. - * * @see #setSurfaceControlCallback(SurfaceControlCallback) */ - public @Nullable SurfaceControl getSurfaceControl() { + @Nullable + public SurfaceControl getSurfaceControl() { return mSurfaceView.getSurfaceControl(); } /** * @inheritDoc - * * @hide */ public InlineContentView(@NonNull Context context, @Nullable AttributeSet attrs, @@ -149,14 +180,35 @@ public class InlineContentView extends ViewGroup { } /** - * Sets the embedded UI. - * @param surfacePackage The embedded UI. + * Sets the embedded UI provider. * * @hide */ - public void setChildSurfacePackage( - @Nullable SurfaceControlViewHost.SurfacePackage surfacePackage) { - mSurfaceView.setChildSurfacePackage(surfacePackage); + public void setChildSurfacePackageUpdater( + @Nullable SurfacePackageUpdater surfacePackageUpdater) { + mSurfacePackageUpdater = surfacePackageUpdater; + } + + @Override + protected void onAttachedToWindow() { + if (DEBUG) Log.v(TAG, "onAttachedToWindow"); + super.onAttachedToWindow(); + if (mSurfacePackageUpdater != null) { + mSurfacePackageUpdater.getSurfacePackage( + sp -> { + if (DEBUG) Log.v(TAG, "Received new SurfacePackage"); + mSurfaceView.setChildSurfacePackage(sp); + }); + } + } + + @Override + protected void onDetachedFromWindow() { + if (DEBUG) Log.v(TAG, "onDetachedFromWindow"); + super.onDetachedFromWindow(); + if (mSurfacePackageUpdater != null) { + mSurfacePackageUpdater.onSurfacePackageReleased(); + } } @Override @@ -165,8 +217,8 @@ public class InlineContentView extends ViewGroup { } /** - * Sets a callback to observe the lifecycle of the surface control for - * managing the backing surface. + * Sets a callback to observe the lifecycle of the surface control for managing the backing + * surface. * * @param callback The callback to set or {@code null} to clear. */ @@ -182,7 +234,6 @@ public class InlineContentView extends ViewGroup { /** * @return Whether the surface backing this view appears on top of its parent. - * * @see #setZOrderedOnTop(boolean) */ public boolean isZOrderedOnTop() { @@ -190,17 +241,15 @@ public class InlineContentView extends ViewGroup { } /** - * Controls whether the backing surface is placed on top of this view's window. - * Normally, it is placed on top of the window, to allow interaction - * with the inlined UI. Via this method, you can place the surface below the - * window. This means that all of the contents of the window this view is in - * will be visible on top of its surface. + * Controls whether the backing surface is placed on top of this view's window. Normally, it is + * placed on top of the window, to allow interaction with the inlined UI. Via this method, you + * can place the surface below the window. This means that all of the contents of the window + * this view is in will be visible on top of its surface. * *

The Z ordering can be changed dynamically if the backing surface is * created, otherwise the ordering would be applied at surface construction time. * * @param onTop Whether to show the surface on top of this view's window. - * * @see #isZOrderedOnTop() */ public boolean setZOrderedOnTop(boolean onTop) { diff --git a/core/java/com/android/internal/view/inline/IInlineContentProvider.aidl b/core/java/com/android/internal/view/inline/IInlineContentProvider.aidl index 08a349c21c8b6..78df3eb660a52 100644 --- a/core/java/com/android/internal/view/inline/IInlineContentProvider.aidl +++ b/core/java/com/android/internal/view/inline/IInlineContentProvider.aidl @@ -24,4 +24,6 @@ import com.android.internal.view.inline.IInlineContentCallback; */ oneway interface IInlineContentProvider { void provideContent(int width, int height, in IInlineContentCallback callback); + void requestSurfacePackage(); + void onSurfacePackageReleased(); } diff --git a/services/autofill/java/com/android/server/autofill/AutofillInlineSuggestionsRequestSession.java b/services/autofill/java/com/android/server/autofill/AutofillInlineSuggestionsRequestSession.java index ce11c76e5c6a2..22451e1d992ea 100644 --- a/services/autofill/java/com/android/server/autofill/AutofillInlineSuggestionsRequestSession.java +++ b/services/autofill/java/com/android/server/autofill/AutofillInlineSuggestionsRequestSession.java @@ -37,6 +37,7 @@ import com.android.internal.annotations.GuardedBy; import com.android.internal.view.IInlineSuggestionsRequestCallback; import com.android.internal.view.IInlineSuggestionsResponseCallback; import com.android.internal.view.InlineSuggestionsRequestInfo; +import com.android.server.autofill.ui.InlineSuggestionFactory; import com.android.server.inputmethod.InputMethodManagerInternal; import java.lang.ref.WeakReference; @@ -242,7 +243,8 @@ final class AutofillInlineSuggestionsRequestSession { } if (sDebug) Log.d(TAG, "Send inline response: " + response.getInlineSuggestions().size()); try { - mResponseCallback.onInlineSuggestionsResponse(mAutofillId, response); + mResponseCallback.onInlineSuggestionsResponse(mAutofillId, + InlineSuggestionFactory.copy(response)); } catch (RemoteException e) { Slog.e(TAG, "RemoteException sending InlineSuggestionsResponse to IME"); } diff --git a/services/autofill/java/com/android/server/autofill/RemoteInlineSuggestionRenderService.java b/services/autofill/java/com/android/server/autofill/RemoteInlineSuggestionRenderService.java index 255adcd92da3f..617c111c6c385 100644 --- a/services/autofill/java/com/android/server/autofill/RemoteInlineSuggestionRenderService.java +++ b/services/autofill/java/com/android/server/autofill/RemoteInlineSuggestionRenderService.java @@ -47,7 +47,7 @@ public final class RemoteInlineSuggestionRenderService extends private static final String TAG = "RemoteInlineSuggestionRenderService"; - private final int mIdleUnbindTimeoutMs = 5000; + private final long mIdleUnbindTimeoutMs = PERMANENT_BOUND_TIMEOUT_MS; RemoteInlineSuggestionRenderService(Context context, ComponentName componentName, String serviceInterface, int userId, InlineSuggestionRenderCallbacks callback, diff --git a/services/autofill/java/com/android/server/autofill/ui/InlineContentProviderImpl.java b/services/autofill/java/com/android/server/autofill/ui/InlineContentProviderImpl.java new file mode 100644 index 0000000000000..819f2b813a5e5 --- /dev/null +++ b/services/autofill/java/com/android/server/autofill/ui/InlineContentProviderImpl.java @@ -0,0 +1,140 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.autofill.ui; + +import static com.android.server.autofill.Helper.sVerbose; + +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.os.Handler; +import android.util.Slog; + +import com.android.internal.view.inline.IInlineContentCallback; +import com.android.internal.view.inline.IInlineContentProvider; +import com.android.server.FgThread; + +/** + * We create one instance of this class for each {@link android.view.inputmethod.InlineSuggestion} + * instance. Each inline suggestion instance will only be sent to the remote IME process once. In + * case of filtering and resending the suggestion when keyboard state changes between hide and + * show, a new instance of this class will be created using {@link #copy()}, with the same backing + * {@link RemoteInlineSuggestionUi}. When the + * {@link #provideContent(int, int, IInlineContentCallback)} is called the first time (it's only + * allowed to be called at most once), the passed in width/height is used to determine whether + * the existing {@link RemoteInlineSuggestionUi} provided in the constructor can be reused, or a + * new one should be created to suit the new size requirement for the view. In normal cases, + * we should not expect the size requirement to change, although in theory the public API allows + * the IME to do that. + * + *

This design is to enable us to be able to reuse the backing remote view while still keeping + * the callbacks relatively well aligned. For example, if we allow multiple remote IME binder + * callbacks to call into one instance of this class, then binder A may call in with width/height + * X for which we create a view (i.e. {@link RemoteInlineSuggestionUi}) for it, + * + * See also {@link RemoteInlineSuggestionUi} for relevant information. + */ +public final class InlineContentProviderImpl extends IInlineContentProvider.Stub { + + // TODO(b/153615023): consider not holding strong reference to heavy objects in this stub, to + // avoid memory leak in case the client app is holding the remote reference for a longer + // time than expected. Essentially we need strong reference in the system process to + // the member variables, but weak reference to them in the IInlineContentProvider.Stub. + + private static final String TAG = InlineContentProviderImpl.class.getSimpleName(); + + private final Handler mHandler = FgThread.getHandler();; + + @NonNull + private final RemoteInlineSuggestionViewConnector mRemoteInlineSuggestionViewConnector; + @Nullable + private RemoteInlineSuggestionUi mRemoteInlineSuggestionUi; + + private boolean mProvideContentCalled = false; + + InlineContentProviderImpl( + @NonNull RemoteInlineSuggestionViewConnector remoteInlineSuggestionViewConnector, + @Nullable RemoteInlineSuggestionUi remoteInlineSuggestionUi) { + mRemoteInlineSuggestionViewConnector = remoteInlineSuggestionViewConnector; + mRemoteInlineSuggestionUi = remoteInlineSuggestionUi; + } + + /** + * Returns a new instance of this class, with the same {@code mInlineSuggestionRenderer} and + * {@code mRemoteInlineSuggestionUi}. The latter may or may not be reusable depending on the + * size information provided when the client calls {@link #provideContent(int, int, + * IInlineContentCallback)}. + */ + @NonNull + public InlineContentProviderImpl copy() { + return new InlineContentProviderImpl(mRemoteInlineSuggestionViewConnector, + mRemoteInlineSuggestionUi); + } + + /** + * Provides a SurfacePackage associated with the inline suggestion view to the IME. If such + * view doesn't exit, then create a new one. This method should be called once per lifecycle + * of this object. Any further calls to the method will be ignored. + */ + @Override + public void provideContent(int width, int height, IInlineContentCallback callback) { + mHandler.post(() -> handleProvideContent(width, height, callback)); + } + + @Override + public void requestSurfacePackage() { + mHandler.post(this::handleGetSurfacePackage); + } + + @Override + public void onSurfacePackageReleased() { + mHandler.post(this::handleOnSurfacePackageReleased); + } + + private void handleProvideContent(int width, int height, IInlineContentCallback callback) { + if (sVerbose) Slog.v(TAG, "handleProvideContent"); + if (mProvideContentCalled) { + // This method should only be called once. + return; + } + mProvideContentCalled = true; + if (mRemoteInlineSuggestionUi == null || !mRemoteInlineSuggestionUi.match(width, height)) { + mRemoteInlineSuggestionUi = new RemoteInlineSuggestionUi( + mRemoteInlineSuggestionViewConnector, + width, height, mHandler); + } + mRemoteInlineSuggestionUi.setInlineContentCallback(callback); + mRemoteInlineSuggestionUi.requestSurfacePackage(); + } + + private void handleGetSurfacePackage() { + if (sVerbose) Slog.v(TAG, "handleGetSurfacePackage"); + if (!mProvideContentCalled || mRemoteInlineSuggestionUi == null) { + // provideContent should be called first, and remote UI should not be null. + return; + } + mRemoteInlineSuggestionUi.requestSurfacePackage(); + } + + private void handleOnSurfacePackageReleased() { + if (sVerbose) Slog.v(TAG, "handleOnSurfacePackageReleased"); + if (!mProvideContentCalled || mRemoteInlineSuggestionUi == null) { + // provideContent should be called first, and remote UI should not be null. + return; + } + mRemoteInlineSuggestionUi.surfacePackageReleased(); + } +} diff --git a/services/autofill/java/com/android/server/autofill/ui/InlineSuggestionFactory.java b/services/autofill/java/com/android/server/autofill/ui/InlineSuggestionFactory.java index 79c9efa48d737..e74463a8584b0 100644 --- a/services/autofill/java/com/android/server/autofill/ui/InlineSuggestionFactory.java +++ b/services/autofill/java/com/android/server/autofill/ui/InlineSuggestionFactory.java @@ -24,14 +24,11 @@ import android.annotation.Nullable; import android.content.Intent; import android.content.IntentSender; import android.os.IBinder; -import android.os.RemoteException; import android.service.autofill.Dataset; import android.service.autofill.FillResponse; -import android.service.autofill.IInlineSuggestionUiCallback; import android.service.autofill.InlinePresentation; import android.text.TextUtils; import android.util.Slog; -import android.view.SurfaceControlViewHost; import android.view.autofill.AutofillId; import android.view.autofill.AutofillManager; import android.view.autofill.AutofillValue; @@ -41,12 +38,8 @@ import android.view.inputmethod.InlineSuggestionsRequest; import android.view.inputmethod.InlineSuggestionsResponse; import android.widget.inline.InlinePresentationSpec; -import com.android.internal.view.inline.IInlineContentCallback; import com.android.internal.view.inline.IInlineContentProvider; -import com.android.server.LocalServices; -import com.android.server.UiThread; import com.android.server.autofill.RemoteInlineSuggestionRenderService; -import com.android.server.inputmethod.InputMethodManagerInternal; import java.util.ArrayList; import java.util.List; @@ -72,6 +65,27 @@ public final class InlineSuggestionFactory { void startIntentSender(@NonNull IntentSender intentSender, @NonNull Intent intent); } + /** + * Returns a copy of the response, that internally copies the {@link IInlineContentProvider} + * so that it's not reused by the remote IME process across different inline suggestions. + * See {@link InlineContentProviderImpl} for why this is needed. + */ + @NonNull + public static InlineSuggestionsResponse copy(@NonNull InlineSuggestionsResponse response) { + final ArrayList copiedInlineSuggestions = new ArrayList<>(); + for (InlineSuggestion inlineSuggestion : response.getInlineSuggestions()) { + final IInlineContentProvider contentProvider = inlineSuggestion.getContentProvider(); + if (contentProvider instanceof InlineContentProviderImpl) { + copiedInlineSuggestions.add(new + InlineSuggestion(inlineSuggestion.getInfo(), + ((InlineContentProviderImpl) contentProvider).copy())); + } else { + copiedInlineSuggestions.add(inlineSuggestion); + } + } + return new InlineSuggestionsResponse(copiedInlineSuggestions); + } + /** * Creates an {@link InlineSuggestionsResponse} with the {@code datasets} provided by the * autofill service, potentially filtering the datasets. @@ -276,78 +290,20 @@ public final class InlineSuggestionFactory { inlinePresentation.isPinned()); } - private static IInlineContentProvider.Stub createInlineContentProvider( + private static IInlineContentProvider createInlineContentProvider( @NonNull InlinePresentation inlinePresentation, @Nullable Runnable onClickAction, @NonNull Runnable onErrorCallback, @NonNull Consumer intentSenderConsumer, @Nullable RemoteInlineSuggestionRenderService remoteRenderService, @Nullable IBinder hostInputToken, int displayId) { - return new IInlineContentProvider.Stub() { - @Override - public void provideContent(int width, int height, IInlineContentCallback callback) { - UiThread.getHandler().post(() -> { - final IInlineSuggestionUiCallback uiCallback = createInlineSuggestionUiCallback( - callback, onClickAction, onErrorCallback, intentSenderConsumer); - - if (remoteRenderService == null) { - Slog.e(TAG, "RemoteInlineSuggestionRenderService is null"); - return; - } - - remoteRenderService.renderSuggestion(uiCallback, inlinePresentation, - width, height, hostInputToken, displayId); - }); - } - }; - } - - private static IInlineSuggestionUiCallback.Stub createInlineSuggestionUiCallback( - @NonNull IInlineContentCallback callback, @NonNull Runnable onAutofillCallback, - @NonNull Runnable onErrorCallback, - @NonNull Consumer intentSenderConsumer) { - return new IInlineSuggestionUiCallback.Stub() { - @Override - public void onClick() throws RemoteException { - onAutofillCallback.run(); - callback.onClick(); - } - - @Override - public void onLongClick() throws RemoteException { - callback.onLongClick(); - } - - @Override - public void onContent(SurfaceControlViewHost.SurfacePackage surface, int width, - int height) - throws RemoteException { - callback.onContent(surface, width, height); - surface.release(); - } - - @Override - public void onError() throws RemoteException { - onErrorCallback.run(); - } - - @Override - public void onTransferTouchFocusToImeWindow(IBinder sourceInputToken, int displayId) - throws RemoteException { - final InputMethodManagerInternal inputMethodManagerInternal = - LocalServices.getService(InputMethodManagerInternal.class); - if (!inputMethodManagerInternal.transferTouchFocusToImeWindow(sourceInputToken, - displayId)) { - Slog.e(TAG, "Cannot transfer touch focus from suggestion to IME"); - onErrorCallback.run(); - } - } - - @Override - public void onStartIntentSender(IntentSender intentSender) { - intentSenderConsumer.accept(intentSender); - } - }; + RemoteInlineSuggestionViewConnector + remoteInlineSuggestionViewConnector = new RemoteInlineSuggestionViewConnector( + remoteRenderService, inlinePresentation, hostInputToken, displayId, onClickAction, + onErrorCallback, intentSenderConsumer); + InlineContentProviderImpl inlineContentProvider = new InlineContentProviderImpl( + remoteInlineSuggestionViewConnector, null); + return inlineContentProvider; } private InlineSuggestionFactory() { diff --git a/services/autofill/java/com/android/server/autofill/ui/RemoteInlineSuggestionUi.java b/services/autofill/java/com/android/server/autofill/ui/RemoteInlineSuggestionUi.java new file mode 100644 index 0000000000000..00a5283c9b1fc --- /dev/null +++ b/services/autofill/java/com/android/server/autofill/ui/RemoteInlineSuggestionUi.java @@ -0,0 +1,291 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.autofill.ui; + +import static com.android.server.autofill.Helper.sDebug; + +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.content.IntentSender; +import android.os.Handler; +import android.os.IBinder; +import android.os.RemoteException; +import android.service.autofill.IInlineSuggestionUi; +import android.service.autofill.IInlineSuggestionUiCallback; +import android.service.autofill.ISurfacePackageResultCallback; +import android.util.Slog; +import android.view.SurfaceControlViewHost; + +import com.android.internal.view.inline.IInlineContentCallback; + +/** + * The instance of this class lives in the system server, orchestrating the communication between + * the remote process owning embedded view (i.e. ExtServices) and the remote process hosting the + * embedded view (i.e. IME). It's also responsible for releasing the embedded view from the owning + * process when it's not longer needed in the hosting process. + * + *

An instance of this class may be reused to associate with multiple instances of + * {@link InlineContentProviderImpl}s, each of which wraps a callback from the IME. But at any + * given time, there is only one active IME callback which this class will callback into. + * + *

This class is thread safe, because all the outside calls are piped into the same single + * thread handler to be processed. + * + * TODO(b/154683107): implement the reference counting in case there are multiple active + * SurfacePackages at the same time. This will not happen for now since all the InlineSuggestions + * sharing the same UI will be sent to the same IME window, so the previous view will be detached + * before the new view are attached to the window. + */ +final class RemoteInlineSuggestionUi { + + private static final String TAG = RemoteInlineSuggestionUi.class.getSimpleName(); + + // The delay time to release the remote inline suggestion view (in the renderer + // process) after receiving a signal about the surface package being released due to being + // detached from the window in the host app (in the IME process). The release will be + // canceled if the host app reattaches the view to a window within this delay time. + // TODO(b/154683107): try out using the Chroreographer to schedule the release right at the + // next frame. Basically if the view is not re-attached to the window immediately in the next + // frame after it was detached, then it will be released. + private static final long RELEASE_REMOTE_VIEW_HOST_DELAY_MS = 200; + + @NonNull + private final Handler mHandler; + @NonNull + private final RemoteInlineSuggestionViewConnector mRemoteInlineSuggestionViewConnector; + private final int mWidth; + private final int mHeight; + @NonNull + private final InlineSuggestionUiCallbackImpl mInlineSuggestionUiCallback; + + @Nullable + private IInlineContentCallback mInlineContentCallback; // from IME + + /** + * Remote inline suggestion view, backed by an instance of {@link SurfaceControlViewHost} in + * the render service process. We takes care of releasing it when there is no remote + * reference to it (from IME), and we will create a new instance of the view when it's needed + * by IME again. + */ + @Nullable + private IInlineSuggestionUi mInlineSuggestionUi; + private boolean mWaitingForUiCreation = false; + private int mActualWidth; + private int mActualHeight; + + @Nullable + private Runnable mDelayedReleaseViewRunnable; + + RemoteInlineSuggestionUi( + @NonNull RemoteInlineSuggestionViewConnector remoteInlineSuggestionViewConnector, + int width, int height, Handler handler) { + mHandler = handler; + mRemoteInlineSuggestionViewConnector = remoteInlineSuggestionViewConnector; + mWidth = width; + mHeight = height; + mInlineSuggestionUiCallback = new InlineSuggestionUiCallbackImpl(); + } + + /** + * Updates the callback from the IME process. It'll swap out the previous IME callback, and + * all the subsequent callback events (onClick, onLongClick, touch event transfer, etc) will + * be directed to the new callback. + */ + void setInlineContentCallback(@NonNull IInlineContentCallback inlineContentCallback) { + mHandler.post(() -> { + mInlineContentCallback = inlineContentCallback; + }); + } + + /** + * Handles the request from the IME process to get a new surface package. May create a new + * view in the renderer process if the existing view is already released. + */ + void requestSurfacePackage() { + mHandler.post(this::handleRequestSurfacePackage); + } + + /** + * Handles the signal from the IME process that the previously sent surface package has been + * released. + */ + void surfacePackageReleased() { + mHandler.post(this::handleSurfacePackageReleased); + } + + /** + * Returns true if the provided size matches the remote view's size. + */ + boolean match(int width, int height) { + return mWidth == width && mHeight == height; + } + + private void handleSurfacePackageReleased() { + cancelPendingReleaseViewRequest(); + + // Schedule a delayed release view request + mDelayedReleaseViewRunnable = () -> { + if (mInlineSuggestionUi != null) { + try { + mInlineSuggestionUi.releaseSurfaceControlViewHost(); + mInlineSuggestionUi = null; + } catch (RemoteException e) { + Slog.w(TAG, "RemoteException calling releaseSurfaceControlViewHost"); + } + } + mDelayedReleaseViewRunnable = null; + }; + mHandler.postDelayed(mDelayedReleaseViewRunnable, RELEASE_REMOTE_VIEW_HOST_DELAY_MS); + } + + private void handleRequestSurfacePackage() { + cancelPendingReleaseViewRequest(); + + if (mInlineSuggestionUi == null) { + if (mWaitingForUiCreation) { + // This could happen in the following case: the remote embedded view was released + // when previously detached from window. An event after that to re-attached to + // the window will cause us calling the renderSuggestion again. Now, before the + // render call returns a new surface package, if the view is detached and + // re-attached to the window, causing this method to be called again, we will get + // to this state. This request will be ignored and the surface package will still + // be sent back once the view is rendered. + if (sDebug) Slog.d(TAG, "Inline suggestion ui is not ready"); + } else { + mRemoteInlineSuggestionViewConnector.renderSuggestion(mWidth, mHeight, + mInlineSuggestionUiCallback); + mWaitingForUiCreation = true; + } + } else { + try { + mInlineSuggestionUi.getSurfacePackage(new ISurfacePackageResultCallback.Stub() { + @Override + public void onResult(SurfaceControlViewHost.SurfacePackage result) + throws RemoteException { + if (sDebug) Slog.d(TAG, "Sending new SurfacePackage to IME"); + mInlineContentCallback.onContent(result, mActualWidth, mActualHeight); + } + }); + } catch (RemoteException e) { + Slog.w(TAG, "RemoteException calling getSurfacePackage."); + } + } + } + + private void cancelPendingReleaseViewRequest() { + if (mDelayedReleaseViewRunnable != null) { + mHandler.removeCallbacks(mDelayedReleaseViewRunnable); + mDelayedReleaseViewRunnable = null; + } + } + + /** + * This is called when a new inline suggestion UI is inflated from the ext services. + */ + private void handleInlineSuggestionUiReady(IInlineSuggestionUi content, + SurfaceControlViewHost.SurfacePackage surfacePackage, int width, int height) { + mInlineSuggestionUi = content; + mWaitingForUiCreation = false; + mActualWidth = width; + mActualHeight = height; + if (mInlineContentCallback != null) { + try { + mInlineContentCallback.onContent(surfacePackage, mActualWidth, mActualHeight); + } catch (RemoteException e) { + Slog.w(TAG, "RemoteException calling onContent"); + } + } + if (surfacePackage != null) { + surfacePackage.release(); + } + } + + private void handleOnClick() { + // Autofill the value + mRemoteInlineSuggestionViewConnector.onClick(); + + // Notify the remote process (IME) that hosts the embedded UI that it's clicked + if (mInlineContentCallback != null) { + try { + mInlineContentCallback.onClick(); + } catch (RemoteException e) { + Slog.w(TAG, "RemoteException calling onClick"); + } + } + } + + private void handleOnLongClick() { + // Notify the remote process (IME) that hosts the embedded UI that it's long clicked + if (mInlineContentCallback != null) { + try { + mInlineContentCallback.onLongClick(); + } catch (RemoteException e) { + Slog.w(TAG, "RemoteException calling onLongClick"); + } + } + } + + private void handleOnError() { + mRemoteInlineSuggestionViewConnector.onError(); + } + + private void handleOnTransferTouchFocusToImeWindow(IBinder sourceInputToken, int displayId) { + mRemoteInlineSuggestionViewConnector.onTransferTouchFocusToImeWindow(sourceInputToken, + displayId); + } + + private void handleOnStartIntentSender(IntentSender intentSender) { + mRemoteInlineSuggestionViewConnector.onStartIntentSender(intentSender); + } + + /** + * Responsible for communicating with the inline suggestion view owning process. + */ + private class InlineSuggestionUiCallbackImpl extends IInlineSuggestionUiCallback.Stub { + + @Override + public void onClick() { + mHandler.post(RemoteInlineSuggestionUi.this::handleOnClick); + } + + @Override + public void onLongClick() { + mHandler.post(RemoteInlineSuggestionUi.this::handleOnLongClick); + } + + @Override + public void onContent(IInlineSuggestionUi content, + SurfaceControlViewHost.SurfacePackage surface, int width, int height) { + mHandler.post(() -> handleInlineSuggestionUiReady(content, surface, width, height)); + } + + @Override + public void onError() { + mHandler.post(RemoteInlineSuggestionUi.this::handleOnError); + } + + @Override + public void onTransferTouchFocusToImeWindow(IBinder sourceInputToken, int displayId) { + mHandler.post(() -> handleOnTransferTouchFocusToImeWindow(sourceInputToken, displayId)); + } + + @Override + public void onStartIntentSender(IntentSender intentSender) { + mHandler.post(() -> handleOnStartIntentSender(intentSender)); + } + } +} diff --git a/services/autofill/java/com/android/server/autofill/ui/RemoteInlineSuggestionViewConnector.java b/services/autofill/java/com/android/server/autofill/ui/RemoteInlineSuggestionViewConnector.java new file mode 100644 index 0000000000000..9d23c171800d9 --- /dev/null +++ b/services/autofill/java/com/android/server/autofill/ui/RemoteInlineSuggestionViewConnector.java @@ -0,0 +1,125 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.autofill.ui; + +import static com.android.server.autofill.Helper.sDebug; + +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.content.IntentSender; +import android.os.IBinder; +import android.service.autofill.IInlineSuggestionUiCallback; +import android.service.autofill.InlinePresentation; +import android.util.Slog; + +import com.android.server.LocalServices; +import com.android.server.autofill.RemoteInlineSuggestionRenderService; +import com.android.server.inputmethod.InputMethodManagerInternal; + +import java.util.function.Consumer; + +/** + * Wraps the parameters needed to create a new inline suggestion view in the remote renderer + * service, and handles the callback from the events on the created remote view. + */ +final class RemoteInlineSuggestionViewConnector { + private static final String TAG = RemoteInlineSuggestionViewConnector.class.getSimpleName(); + + @Nullable + private final RemoteInlineSuggestionRenderService mRemoteRenderService; + @NonNull + private final InlinePresentation mInlinePresentation; + @Nullable + private final IBinder mHostInputToken; + private final int mDisplayId; + + @NonNull + private final Runnable mOnAutofillCallback; + @NonNull + private final Runnable mOnErrorCallback; + @NonNull + private final Consumer mStartIntentSenderFromClientApp; + + RemoteInlineSuggestionViewConnector( + @Nullable RemoteInlineSuggestionRenderService remoteRenderService, + @NonNull InlinePresentation inlinePresentation, + @Nullable IBinder hostInputToken, + int displayId, + @NonNull Runnable onAutofillCallback, + @NonNull Runnable onErrorCallback, + @NonNull Consumer startIntentSenderFromClientApp) { + mRemoteRenderService = remoteRenderService; + mInlinePresentation = inlinePresentation; + mHostInputToken = hostInputToken; + mDisplayId = displayId; + + mOnAutofillCallback = onAutofillCallback; + mOnErrorCallback = onErrorCallback; + mStartIntentSenderFromClientApp = startIntentSenderFromClientApp; + } + + /** + * Calls the remote renderer service to create a new inline suggestion view. + * + * @return true if the call is made to the remote renderer service, false otherwise. + */ + public boolean renderSuggestion(int width, int height, + @NonNull IInlineSuggestionUiCallback callback) { + if (mRemoteRenderService != null) { + if (sDebug) Slog.d(TAG, "Request to recreate the UI"); + mRemoteRenderService.renderSuggestion(callback, mInlinePresentation, width, height, + mHostInputToken, mDisplayId); + return true; + } + return false; + } + + /** + * Handles the callback for the event of remote view being clicked. + */ + public void onClick() { + mOnAutofillCallback.run(); + } + + /** + * Handles the callback for the remote error when creating or interacting with the view. + */ + public void onError() { + mOnErrorCallback.run(); + } + + /** + * Handles the callback for transferring the touch event on the remote view to the IME + * process. + */ + public void onTransferTouchFocusToImeWindow(IBinder sourceInputToken, int displayId) { + final InputMethodManagerInternal inputMethodManagerInternal = + LocalServices.getService(InputMethodManagerInternal.class); + if (!inputMethodManagerInternal.transferTouchFocusToImeWindow(sourceInputToken, + displayId)) { + Slog.e(TAG, "Cannot transfer touch focus from suggestion to IME"); + mOnErrorCallback.run(); + } + } + + /** + * Handles starting an intent sender from the client app's process. + */ + public void onStartIntentSender(IntentSender intentSender) { + mStartIntentSenderFromClientApp.accept(intentSender); + } +}