diff --git a/core/java/android/service/autofill/IInlineSuggestionUi.aidl b/core/java/android/service/autofill/IInlineSuggestionUi.aidl new file mode 100644 index 0000000000000..7289853064f87 --- /dev/null +++ b/core/java/android/service/autofill/IInlineSuggestionUi.aidl @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.service.autofill; + +import android.service.autofill.ISurfacePackageResultCallback; + +/** + * Interface to interact with a remote inline suggestion UI. + * + * @hide + */ +oneway interface IInlineSuggestionUi { + void getSurfacePackage(ISurfacePackageResultCallback callback); + void releaseSurfaceControlViewHost(); +} diff --git a/core/java/android/service/autofill/IInlineSuggestionUiCallback.aidl b/core/java/android/service/autofill/IInlineSuggestionUiCallback.aidl index 172cfef9fee26..97eb790b9acce 100644 --- a/core/java/android/service/autofill/IInlineSuggestionUiCallback.aidl +++ b/core/java/android/service/autofill/IInlineSuggestionUiCallback.aidl @@ -18,17 +18,19 @@ package android.service.autofill; import android.content.IntentSender; import android.os.IBinder; +import android.service.autofill.IInlineSuggestionUi; import android.view.SurfaceControlViewHost; /** - * Interface to receive events from inline suggestions. + * Interface to receive events from a remote inline suggestion UI. * * @hide */ oneway interface IInlineSuggestionUiCallback { void onClick(); void onLongClick(); - void onContent(in SurfaceControlViewHost.SurfacePackage surface, int width, int height); + void onContent(in IInlineSuggestionUi content, in SurfaceControlViewHost.SurfacePackage surface, + int width, int height); void onError(); void onTransferTouchFocusToImeWindow(in IBinder sourceInputToken, int displayId); void onStartIntentSender(in IntentSender intentSender); diff --git a/core/java/android/service/autofill/ISurfacePackageResultCallback.aidl b/core/java/android/service/autofill/ISurfacePackageResultCallback.aidl new file mode 100644 index 0000000000000..0c2c624952eb3 --- /dev/null +++ b/core/java/android/service/autofill/ISurfacePackageResultCallback.aidl @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.service.autofill; + +import android.view.SurfaceControlViewHost; + +/** + * Interface to receive a SurfaceControlViewHost.SurfacePackage. + * + * @hide + */ +oneway interface ISurfacePackageResultCallback { + void onResult(in SurfaceControlViewHost.SurfacePackage result); +} diff --git a/core/java/android/service/autofill/InlineSuggestionRenderService.java b/core/java/android/service/autofill/InlineSuggestionRenderService.java index 6c22b1936d747..3ea443bab3f80 100644 --- a/core/java/android/service/autofill/InlineSuggestionRenderService.java +++ b/core/java/android/service/autofill/InlineSuggestionRenderService.java @@ -33,6 +33,7 @@ import android.os.Looper; import android.os.RemoteCallback; import android.os.RemoteException; import android.util.Log; +import android.util.LruCache; import android.util.Size; import android.view.Display; import android.view.SurfaceControlViewHost; @@ -40,6 +41,8 @@ import android.view.View; import android.view.ViewGroup; import android.view.WindowManager; +import java.lang.ref.WeakReference; + /** * A service that renders an inline presentation view given the {@link InlinePresentation}. * @@ -65,6 +68,27 @@ public abstract class InlineSuggestionRenderService extends Service { private IInlineSuggestionUiCallback mCallback; + + /** + * A local LRU cache keeping references to the inflated {@link SurfaceControlViewHost}s, so + * they can be released properly when no longer used. Each view needs to be tracked separately, + * therefore for simplicity we use the hash code of the value object as key in the cache. + */ + private final LruCache mActiveInlineSuggestions = + new LruCache(30) { + @Override + public void entryRemoved(boolean evicted, InlineSuggestionUiImpl key, + Boolean oldValue, + Boolean newValue) { + if (evicted) { + Log.w(TAG, + "Hit max=100 entries in the cache. Releasing oldest one to make " + + "space."); + key.releaseSurfaceControlViewHost(); + } + } + }; + /** * If the specified {@code width}/{@code height} is an exact value, then it will be returned as * is, otherwise the method tries to measure a size that is just large enough to fit the view @@ -169,8 +193,14 @@ public abstract class InlineSuggestionRenderService extends Service { return true; }); - sendResult(callback, host.getSurfacePackage(), measuredSize.getWidth(), - measuredSize.getHeight()); + try { + InlineSuggestionUiImpl uiImpl = new InlineSuggestionUiImpl(host, mHandler); + mActiveInlineSuggestions.put(uiImpl, true); + callback.onContent(new InlineSuggestionUiWrapper(uiImpl), host.getSurfacePackage(), + measuredSize.getWidth(), measuredSize.getHeight()); + } catch (RemoteException e) { + Log.w(TAG, "RemoteException calling onContent()"); + } } finally { updateDisplay(Display.DEFAULT_DISPLAY); } @@ -181,12 +211,87 @@ public abstract class InlineSuggestionRenderService extends Service { callback.sendResult(rendererInfo); } - private void sendResult(@NonNull IInlineSuggestionUiCallback callback, - @Nullable SurfaceControlViewHost.SurfacePackage surface, int width, int height) { - try { - callback.onContent(surface, width, height); - } catch (RemoteException e) { - Log.w(TAG, "RemoteException calling onContent(" + surface + ")"); + /** + * A wrapper class around the {@link InlineSuggestionUiImpl} to ensure it's not strongly + * reference by the remote system server process. + */ + private static final class InlineSuggestionUiWrapper extends + android.service.autofill.IInlineSuggestionUi.Stub { + + private final WeakReference mUiImpl; + + InlineSuggestionUiWrapper(InlineSuggestionUiImpl uiImpl) { + mUiImpl = new WeakReference<>(uiImpl); + } + + @Override + public void releaseSurfaceControlViewHost() { + final InlineSuggestionUiImpl uiImpl = mUiImpl.get(); + if (uiImpl != null) { + uiImpl.releaseSurfaceControlViewHost(); + } + } + + @Override + public void getSurfacePackage(ISurfacePackageResultCallback callback) { + final InlineSuggestionUiImpl uiImpl = mUiImpl.get(); + if (uiImpl != null) { + uiImpl.getSurfacePackage(callback); + } + } + } + + /** + * Keeps track of a SurfaceControlViewHost to ensure it's released when its lifecycle ends. + * + *

This class is thread safe, because all the outside calls are piped into a single + * handler thread to be processed. + */ + private final class InlineSuggestionUiImpl { + + @Nullable + private SurfaceControlViewHost mViewHost; + @NonNull + private final Handler mHandler; + + InlineSuggestionUiImpl(SurfaceControlViewHost viewHost, Handler handler) { + this.mViewHost = viewHost; + this.mHandler = handler; + } + + /** + * Call {@link SurfaceControlViewHost#release()} to release it. After this, this view is + * not usable, and any further calls to the + * {@link #getSurfacePackage(ISurfacePackageResultCallback)} will get {@code null} result. + */ + public void releaseSurfaceControlViewHost() { + mHandler.post(() -> { + if (mViewHost == null) { + return; + } + Log.v(TAG, "Releasing inline suggestion view host"); + mViewHost.release(); + mViewHost = null; + InlineSuggestionRenderService.this.mActiveInlineSuggestions.remove( + InlineSuggestionUiImpl.this); + Log.v(TAG, "Removed the inline suggestion from the cache, current size=" + + InlineSuggestionRenderService.this.mActiveInlineSuggestions.size()); + }); + } + + /** + * Sends back a new {@link android.view.SurfaceControlViewHost.SurfacePackage} if the view + * is not released, {@code null} otherwise. + */ + public void getSurfacePackage(ISurfacePackageResultCallback callback) { + Log.d(TAG, "getSurfacePackage"); + mHandler.post(() -> { + try { + callback.onResult(mViewHost == null ? null : mViewHost.getSurfacePackage()); + } catch (RemoteException e) { + Log.w(TAG, "RemoteException calling onSurfacePackage"); + } + }); } } diff --git a/core/java/android/view/inputmethod/InlineSuggestion.java b/core/java/android/view/inputmethod/InlineSuggestion.java index 6b1a480986c8e..4c72474435a42 100644 --- a/core/java/android/view/inputmethod/InlineSuggestion.java +++ b/core/java/android/view/inputmethod/InlineSuggestion.java @@ -18,11 +18,13 @@ package android.view.inputmethod; import android.annotation.BinderThread; import android.annotation.CallbackExecutor; +import android.annotation.MainThread; import android.annotation.NonNull; import android.annotation.Nullable; import android.annotation.TestApi; import android.content.Context; -import android.os.AsyncTask; +import android.os.Handler; +import android.os.Looper; import android.os.Parcel; import android.os.Parcelable; import android.os.RemoteException; @@ -42,26 +44,26 @@ import java.util.concurrent.Executor; import java.util.function.Consumer; /** - * This class represents an inline suggestion which is made by one app - * and can be embedded into the UI of another. Suggestions may contain - * sensitive information not known to the host app which needs to be - * protected from spoofing. To address that the suggestion view inflated - * on demand for embedding is created in such a way that the hosting app - * cannot introspect its content and cannot interact with it. + * This class represents an inline suggestion which is made by one app and can be embedded into the + * UI of another. Suggestions may contain sensitive information not known to the host app which + * needs to be protected from spoofing. To address that the suggestion view inflated on demand for + * embedding is created in such a way that the hosting app cannot introspect its content and cannot + * interact with it. */ -@DataClass( - genEqualsHashCode = true, - genToString = true, - genHiddenConstDefs = true, +@DataClass(genEqualsHashCode = true, genToString = true, genHiddenConstDefs = true, genHiddenConstructor = true) -@DataClass.Suppress({"getContentProvider"}) public final class InlineSuggestion implements Parcelable { private static final String TAG = "InlineSuggestion"; - private final @NonNull InlineSuggestionInfo mInfo; + @NonNull + private final InlineSuggestionInfo mInfo; - private final @Nullable IInlineContentProvider mContentProvider; + /** + * @hide + */ + @Nullable + private final IInlineContentProvider mContentProvider; /** * Used to keep a strong reference to the callback so it doesn't get garbage collected. @@ -69,7 +71,8 @@ public final class InlineSuggestion implements Parcelable { * @hide */ @DataClass.ParcelWith(InlineContentCallbackImplParceling.class) - private @Nullable InlineContentCallbackImpl mInlineContentCallback; + @Nullable + private InlineContentCallbackImpl mInlineContentCallback; /** * Creates a new {@link InlineSuggestion}, for testing purpose. @@ -87,8 +90,7 @@ public final class InlineSuggestion implements Parcelable { * * @hide */ - public InlineSuggestion( - @NonNull InlineSuggestionInfo info, + public InlineSuggestion(@NonNull InlineSuggestionInfo info, @Nullable IInlineContentProvider contentProvider) { this(info, contentProvider, /* inlineContentCallback */ null); } @@ -96,25 +98,30 @@ public final class InlineSuggestion implements Parcelable { /** * Inflates a view with the content of this suggestion at a specific size. * - *

The size must be either 1) between the - * {@link android.widget.inline.InlinePresentationSpec#getMinSize() min size} and the - * {@link android.widget.inline.InlinePresentationSpec#getMaxSize() max size} of the - * presentation spec returned by {@link InlineSuggestionInfo#getInlinePresentationSpec()}, - * or 2) {@link ViewGroup.LayoutParams#WRAP_CONTENT}. If the size is set to - * {@link ViewGroup.LayoutParams#WRAP_CONTENT}, then the size of the inflated view will be just - * large enough to fit the content, while still conforming to the min / max size specified by - * the {@link android.widget.inline.InlinePresentationSpec}. + *

Each dimension of the size must satisfy one of the following conditions: + * + *

    + *
  1. between {@link android.widget.inline.InlinePresentationSpec#getMinSize()} and + * {@link android.widget.inline.InlinePresentationSpec#getMaxSize()} of the presentation spec + * from {@code mInfo} + *
  2. {@link ViewGroup.LayoutParams#WRAP_CONTENT} + *
+ * + * If the size is set to {@link + * ViewGroup.LayoutParams#WRAP_CONTENT}, then the size of the inflated view will be just large + * enough to fit the content, while still conforming to the min / max size specified by the + * {@link android.widget.inline.InlinePresentationSpec}. * *

The caller can attach an {@link android.view.View.OnClickListener} and/or an - * {@link android.view.View.OnLongClickListener} to the view in the - * {@code callback} to receive click and long click events on the view. + * {@link android.view.View.OnLongClickListener} to the view in the {@code callback} to receive + * click and long click events on the view. * * @param context Context in which to inflate the view. - * @param size The size at which to inflate the suggestion. For each dimension, it maybe - * an exact value or {@link ViewGroup.LayoutParams#WRAP_CONTENT}. - * @param callback Callback for receiving the inflated view, where the - * {@link ViewGroup.LayoutParams} of the view is set as the actual size of - * the underlying remote view. + * @param size The size at which to inflate the suggestion. For each dimension, it maybe an + * exact value or {@link ViewGroup.LayoutParams#WRAP_CONTENT}. + * @param callback Callback for receiving the inflated view, where the {@link + * ViewGroup.LayoutParams} of the view is set as the actual size of the + * underlying remote view. * @throws IllegalArgumentException If an invalid argument is passed. * @throws IllegalStateException If this method is already called. */ @@ -130,19 +137,17 @@ public final class InlineSuggestion implements Parcelable { + ", nor wrap_content"); } mInlineContentCallback = getInlineContentCallback(context, callbackExecutor, callback); - AsyncTask.THREAD_POOL_EXECUTOR.execute(() -> { - if (mContentProvider == null) { - callback.accept(/* view */ null); - return; - } - try { - mContentProvider.provideContent(size.getWidth(), size.getHeight(), - new InlineContentCallbackWrapper(mInlineContentCallback)); - } catch (RemoteException e) { - Slog.w(TAG, "Error creating suggestion content surface: " + e); - callback.accept(/* view */ null); - } - }); + if (mContentProvider == null) { + callbackExecutor.execute(() -> callback.accept(/* view */ null)); + return; + } + try { + mContentProvider.provideContent(size.getWidth(), size.getHeight(), + new InlineContentCallbackWrapper(mInlineContentCallback)); + } catch (RemoteException e) { + Slog.w(TAG, "Error creating suggestion content surface: " + e); + callbackExecutor.execute(() -> callback.accept(/* view */ null)); + } } /** @@ -161,9 +166,14 @@ public final class InlineSuggestion implements Parcelable { if (mInlineContentCallback != null) { throw new IllegalStateException("Already called #inflate()"); } - return new InlineContentCallbackImpl(context, callbackExecutor, callback); + return new InlineContentCallbackImpl(context, mContentProvider, callbackExecutor, + callback); } + /** + * A wrapper class around the {@link InlineContentCallbackImpl} to ensure it's not strongly + * reference by the remote system server process. + */ private static final class InlineContentCallbackWrapper extends IInlineContentCallback.Stub { private final WeakReference mCallbackImpl; @@ -201,17 +211,68 @@ public final class InlineSuggestion implements Parcelable { } } + /** + * Handles the communication between the inline suggestion view in current (IME) process and + * the remote view provided from the system server. + * + *

This class is thread safe, because all the outside calls are piped into a single + * handler thread to be processed. + */ private static final class InlineContentCallbackImpl { - private final @NonNull Context mContext; - private final @NonNull Executor mCallbackExecutor; - private final @NonNull Consumer mCallback; - private @Nullable InlineContentView mView; + @NonNull + private final Handler mMainHandler = new Handler(Looper.getMainLooper()); + + @NonNull + private final Context mContext; + @Nullable + private final IInlineContentProvider mInlineContentProvider; + @NonNull + private final Executor mCallbackExecutor; + + /** + * Callback from the client (IME) that will receive the inflated suggestion view. It'll + * only be called once when the view SurfacePackage is first sent back to the client. Any + * updates to the view due to attach to window and detach from window events will be + * handled under the hood, transparent from the client. + */ + @NonNull + private final Consumer mCallback; + + /** + * Indicates whether the first content has been received or not. + */ + private boolean mFirstContentReceived = false; + + /** + * The client (IME) side view which internally wraps a remote view. It'll be set when + * {@link #onContent(SurfaceControlViewHost.SurfacePackage, int, int)} is called, which + * should only happen once in the lifecycle of this inline suggestion instance. + */ + @Nullable + private InlineContentView mView; + + /** + * The SurfacePackage pointing to the remote view. It's cached here to be sent to the next + * available consumer. + */ + @Nullable + private SurfaceControlViewHost.SurfacePackage mSurfacePackage; + + /** + * The callback (from the {@link InlineContentView}) which consumes the surface package. + * It's cached here to be called when the SurfacePackage is returned from the remote + * view owning process. + */ + @Nullable + private Consumer mSurfacePackageConsumer; InlineContentCallbackImpl(@NonNull Context context, + @Nullable IInlineContentProvider inlineContentProvider, @NonNull @CallbackExecutor Executor callbackExecutor, @NonNull Consumer callback) { mContext = context; + mInlineContentProvider = inlineContentProvider; mCallbackExecutor = callbackExecutor; mCallback = callback; } @@ -219,28 +280,110 @@ public final class InlineSuggestion implements Parcelable { @BinderThread public void onContent(SurfaceControlViewHost.SurfacePackage content, int width, int height) { - if (content == null) { + mMainHandler.post(() -> handleOnContent(content, width, height)); + } + + @MainThread + private void handleOnContent(SurfaceControlViewHost.SurfacePackage content, int width, + int height) { + if (!mFirstContentReceived) { + handleOnFirstContentReceived(content, width, height); + mFirstContentReceived = true; + } else { + handleOnSurfacePackage(content); + } + } + + /** + * Called when the view content is returned for the first time. + */ + @MainThread + private void handleOnFirstContentReceived(SurfaceControlViewHost.SurfacePackage content, + int width, int height) { + mSurfacePackage = content; + if (mSurfacePackage == null) { mCallbackExecutor.execute(() -> mCallback.accept(/* view */null)); } else { mView = new InlineContentView(mContext); mView.setLayoutParams(new ViewGroup.LayoutParams(width, height)); - mView.setChildSurfacePackage(content); + mView.setChildSurfacePackageUpdater(getSurfacePackageUpdater()); mCallbackExecutor.execute(() -> mCallback.accept(mView)); } } + /** + * Called when any subsequent SurfacePackage is returned from the remote view owning + * process. + */ + @MainThread + private void handleOnSurfacePackage(SurfaceControlViewHost.SurfacePackage surfacePackage) { + mSurfacePackage = surfacePackage; + if (mSurfacePackage != null && mSurfacePackageConsumer != null) { + mSurfacePackageConsumer.accept(mSurfacePackage); + mSurfacePackageConsumer = null; + } + } + + @MainThread + private void handleOnSurfacePackageReleased() { + mSurfacePackage = null; + try { + mInlineContentProvider.onSurfacePackageReleased(); + } catch (RemoteException e) { + Slog.w(TAG, "Error calling onSurfacePackageReleased(): " + e); + } + } + + @MainThread + private void handleGetSurfacePackage( + Consumer consumer) { + if (mSurfacePackage != null) { + consumer.accept(mSurfacePackage); + } else { + mSurfacePackageConsumer = consumer; + try { + mInlineContentProvider.requestSurfacePackage(); + } catch (RemoteException e) { + Slog.w(TAG, "Error calling getSurfacePackage(): " + e); + consumer.accept(null); + mSurfacePackageConsumer = null; + } + } + } + + private InlineContentView.SurfacePackageUpdater getSurfacePackageUpdater() { + return new InlineContentView.SurfacePackageUpdater() { + @Override + public void onSurfacePackageReleased() { + mMainHandler.post( + () -> InlineContentCallbackImpl.this.handleOnSurfacePackageReleased()); + } + + @Override + public void getSurfacePackage( + Consumer consumer) { + mMainHandler.post( + () -> InlineContentCallbackImpl.this.handleGetSurfacePackage(consumer)); + } + }; + } + @BinderThread public void onClick() { - if (mView != null && mView.hasOnClickListeners()) { - mView.callOnClick(); - } + mMainHandler.post(() -> { + if (mView != null && mView.hasOnClickListeners()) { + mView.callOnClick(); + } + }); } @BinderThread public void onLongClick() { - if (mView != null && mView.hasOnLongClickListeners()) { - mView.performLongClick(); - } + mMainHandler.post(() -> { + if (mView != null && mView.hasOnLongClickListeners()) { + mView.performLongClick(); + } + }); } } @@ -262,6 +405,7 @@ public final class InlineSuggestion implements Parcelable { + // Code below generated by codegen v1.0.15. // // DO NOT MODIFY! @@ -301,6 +445,14 @@ public final class InlineSuggestion implements Parcelable { return mInfo; } + /** + * @hide + */ + @DataClass.Generated.Member + public @Nullable IInlineContentProvider getContentProvider() { + return mContentProvider; + } + /** * Used to keep a strong reference to the callback so it doesn't get garbage collected. * @@ -421,7 +573,7 @@ public final class InlineSuggestion implements Parcelable { }; @DataClass.Generated( - time = 1587771173367L, + time = 1588308946517L, codegenVersion = "1.0.15", sourceFile = "frameworks/base/core/java/android/view/inputmethod/InlineSuggestion.java", inputSignatures = "private static final java.lang.String TAG\nprivate final @android.annotation.NonNull android.view.inputmethod.InlineSuggestionInfo mInfo\nprivate final @android.annotation.Nullable com.android.internal.view.inline.IInlineContentProvider mContentProvider\nprivate @com.android.internal.util.DataClass.ParcelWith(android.view.inputmethod.InlineSuggestion.InlineContentCallbackImplParceling.class) @android.annotation.Nullable android.view.inputmethod.InlineSuggestion.InlineContentCallbackImpl mInlineContentCallback\npublic static @android.annotation.TestApi @android.annotation.NonNull android.view.inputmethod.InlineSuggestion newInlineSuggestion(android.view.inputmethod.InlineSuggestionInfo)\npublic void inflate(android.content.Context,android.util.Size,java.util.concurrent.Executor,java.util.function.Consumer)\nprivate static boolean isValid(int,int,int)\nprivate synchronized android.view.inputmethod.InlineSuggestion.InlineContentCallbackImpl getInlineContentCallback(android.content.Context,java.util.concurrent.Executor,java.util.function.Consumer)\nclass InlineSuggestion extends java.lang.Object implements [android.os.Parcelable]\n@com.android.internal.util.DataClass(genEqualsHashCode=true, genToString=true, genHiddenConstDefs=true, genHiddenConstructor=true)") diff --git a/core/java/android/widget/inline/InlineContentView.java b/core/java/android/widget/inline/InlineContentView.java index 4f2af63626cf0..8657e828a3f61 100644 --- a/core/java/android/widget/inline/InlineContentView.java +++ b/core/java/android/widget/inline/InlineContentView.java @@ -21,40 +21,45 @@ import android.annotation.Nullable; import android.content.Context; import android.graphics.PixelFormat; import android.util.AttributeSet; +import android.util.Log; import android.view.SurfaceControl; import android.view.SurfaceControlViewHost; import android.view.SurfaceHolder; import android.view.SurfaceView; import android.view.ViewGroup; +import java.util.function.Consumer; + /** - * This class represents a view that holds opaque content from another app that - * you can inline in your UI. + * This class represents a view that holds opaque content from another app that you can inline in + * your UI. * *

Since the content presented by this view is from another security domain,it is - * shown on a remote surface preventing the host application from accessing that content. - * Also the host application cannot interact with the inlined content by injecting touch - * events or clicking programmatically. + * shown on a remote surface preventing the host application from accessing that content. Also the + * host application cannot interact with the inlined content by injecting touch events or clicking + * programmatically. * *

This view can be overlaid by other windows, i.e. redressed, but if this is the case - * the inined UI would not be interactive. Sometimes this is desirable, e.g. animating - * transitions. + * the inlined UI would not be interactive. Sometimes this is desirable, e.g. animating transitions. * *

By default the surface backing this view is shown on top of the hosting window such - * that the inlined content is interactive. However, you can temporarily move the surface - * under the hosting window which could be useful in some cases, e.g. animating transitions. - * At this point the inlined content will not be interactive and the touch events would - * be delivered to your app. - *

- * Instances of this class are created by the platform and can be programmatically attached - * to your UI. Once you attach and detach this view it can not longer be reused and you - * should obtain a new view from the platform via the dedicated APIs. + * that the inlined content is interactive. However, you can temporarily move the surface under the + * hosting window which could be useful in some cases, e.g. animating transitions. At this point the + * inlined content will not be interactive and the touch events would be delivered to your app. + * + *

Instances of this class are created by the platform and can be programmatically attached to + * your UI. Once the view is attached to the window, you may detach and reattach it to the window. + * It should work seamlessly from the hosting process's point of view. */ public class InlineContentView extends ViewGroup { + private static final String TAG = "InlineContentView"; + + private static final boolean DEBUG = false; + /** - * Callback for observing the lifecycle of the surface control - * that manipulates the backing secure embedded UI surface. + * Callback for observing the lifecycle of the surface control that manipulates the backing + * secure embedded UI surface. */ public interface SurfaceControlCallback { /** @@ -72,15 +77,41 @@ public class InlineContentView extends ViewGroup { void onDestroyed(@NonNull SurfaceControl surfaceControl); } - private final @NonNull SurfaceHolder.Callback mSurfaceCallback = new SurfaceHolder.Callback() { + /** + * Callback for sending an updated surface package in case the previous one is released + * from the detached from window event, and for getting notified of such event. + * + * This is expected to be provided to the {@link InlineContentView} so it can get updates + * from and send updates to the remote content (i.e. surface package) provider. + * + * @hide + */ + public interface SurfacePackageUpdater { + + /** + * Called when the previous surface package is released due to view being detached + * from the window. + */ + void onSurfacePackageReleased(); + + /** + * Called to request an updated surface package. + * + * @param consumer consumes the updated surface package. + */ + void getSurfacePackage(Consumer consumer); + } + + @NonNull + private final SurfaceHolder.Callback mSurfaceCallback = new SurfaceHolder.Callback() { @Override public void surfaceCreated(@NonNull SurfaceHolder holder) { mSurfaceControlCallback.onCreated(mSurfaceView.getSurfaceControl()); } @Override - public void surfaceChanged(@NonNull SurfaceHolder holder, - int format, int width, int height) { + public void surfaceChanged(@NonNull SurfaceHolder holder, int format, int width, + int height) { /* do nothing */ } @@ -90,13 +121,17 @@ public class InlineContentView extends ViewGroup { } }; - private final @NonNull SurfaceView mSurfaceView; + @NonNull + private final SurfaceView mSurfaceView; - private @Nullable SurfaceControlCallback mSurfaceControlCallback; + @Nullable + private SurfaceControlCallback mSurfaceControlCallback; + + @Nullable + private SurfacePackageUpdater mSurfacePackageUpdater; /** * @inheritDoc - * * @hide */ public InlineContentView(@NonNull Context context) { @@ -105,7 +140,6 @@ public class InlineContentView extends ViewGroup { /** * @inheritDoc - * * @hide */ public InlineContentView(@NonNull Context context, @Nullable AttributeSet attrs) { @@ -114,7 +148,6 @@ public class InlineContentView extends ViewGroup { /** * @inheritDoc - * * @hide */ public InlineContentView(@NonNull Context context, @Nullable AttributeSet attrs, @@ -123,20 +156,18 @@ public class InlineContentView extends ViewGroup { } /** - * Gets the surface control. If the surface is not created this method - * returns {@code null}. + * Gets the surface control. If the surface is not created this method returns {@code null}. * * @return The surface control. - * * @see #setSurfaceControlCallback(SurfaceControlCallback) */ - public @Nullable SurfaceControl getSurfaceControl() { + @Nullable + public SurfaceControl getSurfaceControl() { return mSurfaceView.getSurfaceControl(); } /** * @inheritDoc - * * @hide */ public InlineContentView(@NonNull Context context, @Nullable AttributeSet attrs, @@ -149,14 +180,35 @@ public class InlineContentView extends ViewGroup { } /** - * Sets the embedded UI. - * @param surfacePackage The embedded UI. + * Sets the embedded UI provider. * * @hide */ - public void setChildSurfacePackage( - @Nullable SurfaceControlViewHost.SurfacePackage surfacePackage) { - mSurfaceView.setChildSurfacePackage(surfacePackage); + public void setChildSurfacePackageUpdater( + @Nullable SurfacePackageUpdater surfacePackageUpdater) { + mSurfacePackageUpdater = surfacePackageUpdater; + } + + @Override + protected void onAttachedToWindow() { + if (DEBUG) Log.v(TAG, "onAttachedToWindow"); + super.onAttachedToWindow(); + if (mSurfacePackageUpdater != null) { + mSurfacePackageUpdater.getSurfacePackage( + sp -> { + if (DEBUG) Log.v(TAG, "Received new SurfacePackage"); + mSurfaceView.setChildSurfacePackage(sp); + }); + } + } + + @Override + protected void onDetachedFromWindow() { + if (DEBUG) Log.v(TAG, "onDetachedFromWindow"); + super.onDetachedFromWindow(); + if (mSurfacePackageUpdater != null) { + mSurfacePackageUpdater.onSurfacePackageReleased(); + } } @Override @@ -165,8 +217,8 @@ public class InlineContentView extends ViewGroup { } /** - * Sets a callback to observe the lifecycle of the surface control for - * managing the backing surface. + * Sets a callback to observe the lifecycle of the surface control for managing the backing + * surface. * * @param callback The callback to set or {@code null} to clear. */ @@ -182,7 +234,6 @@ public class InlineContentView extends ViewGroup { /** * @return Whether the surface backing this view appears on top of its parent. - * * @see #setZOrderedOnTop(boolean) */ public boolean isZOrderedOnTop() { @@ -190,17 +241,15 @@ public class InlineContentView extends ViewGroup { } /** - * Controls whether the backing surface is placed on top of this view's window. - * Normally, it is placed on top of the window, to allow interaction - * with the inlined UI. Via this method, you can place the surface below the - * window. This means that all of the contents of the window this view is in - * will be visible on top of its surface. + * Controls whether the backing surface is placed on top of this view's window. Normally, it is + * placed on top of the window, to allow interaction with the inlined UI. Via this method, you + * can place the surface below the window. This means that all of the contents of the window + * this view is in will be visible on top of its surface. * *

The Z ordering can be changed dynamically if the backing surface is * created, otherwise the ordering would be applied at surface construction time. * * @param onTop Whether to show the surface on top of this view's window. - * * @see #isZOrderedOnTop() */ public boolean setZOrderedOnTop(boolean onTop) { diff --git a/core/java/com/android/internal/view/inline/IInlineContentProvider.aidl b/core/java/com/android/internal/view/inline/IInlineContentProvider.aidl index 08a349c21c8b6..78df3eb660a52 100644 --- a/core/java/com/android/internal/view/inline/IInlineContentProvider.aidl +++ b/core/java/com/android/internal/view/inline/IInlineContentProvider.aidl @@ -24,4 +24,6 @@ import com.android.internal.view.inline.IInlineContentCallback; */ oneway interface IInlineContentProvider { void provideContent(int width, int height, in IInlineContentCallback callback); + void requestSurfacePackage(); + void onSurfacePackageReleased(); } diff --git a/services/autofill/java/com/android/server/autofill/AutofillInlineSuggestionsRequestSession.java b/services/autofill/java/com/android/server/autofill/AutofillInlineSuggestionsRequestSession.java index ce11c76e5c6a2..22451e1d992ea 100644 --- a/services/autofill/java/com/android/server/autofill/AutofillInlineSuggestionsRequestSession.java +++ b/services/autofill/java/com/android/server/autofill/AutofillInlineSuggestionsRequestSession.java @@ -37,6 +37,7 @@ import com.android.internal.annotations.GuardedBy; import com.android.internal.view.IInlineSuggestionsRequestCallback; import com.android.internal.view.IInlineSuggestionsResponseCallback; import com.android.internal.view.InlineSuggestionsRequestInfo; +import com.android.server.autofill.ui.InlineSuggestionFactory; import com.android.server.inputmethod.InputMethodManagerInternal; import java.lang.ref.WeakReference; @@ -242,7 +243,8 @@ final class AutofillInlineSuggestionsRequestSession { } if (sDebug) Log.d(TAG, "Send inline response: " + response.getInlineSuggestions().size()); try { - mResponseCallback.onInlineSuggestionsResponse(mAutofillId, response); + mResponseCallback.onInlineSuggestionsResponse(mAutofillId, + InlineSuggestionFactory.copy(response)); } catch (RemoteException e) { Slog.e(TAG, "RemoteException sending InlineSuggestionsResponse to IME"); } diff --git a/services/autofill/java/com/android/server/autofill/RemoteInlineSuggestionRenderService.java b/services/autofill/java/com/android/server/autofill/RemoteInlineSuggestionRenderService.java index 255adcd92da3f..617c111c6c385 100644 --- a/services/autofill/java/com/android/server/autofill/RemoteInlineSuggestionRenderService.java +++ b/services/autofill/java/com/android/server/autofill/RemoteInlineSuggestionRenderService.java @@ -47,7 +47,7 @@ public final class RemoteInlineSuggestionRenderService extends private static final String TAG = "RemoteInlineSuggestionRenderService"; - private final int mIdleUnbindTimeoutMs = 5000; + private final long mIdleUnbindTimeoutMs = PERMANENT_BOUND_TIMEOUT_MS; RemoteInlineSuggestionRenderService(Context context, ComponentName componentName, String serviceInterface, int userId, InlineSuggestionRenderCallbacks callback, diff --git a/services/autofill/java/com/android/server/autofill/ui/InlineContentProviderImpl.java b/services/autofill/java/com/android/server/autofill/ui/InlineContentProviderImpl.java new file mode 100644 index 0000000000000..819f2b813a5e5 --- /dev/null +++ b/services/autofill/java/com/android/server/autofill/ui/InlineContentProviderImpl.java @@ -0,0 +1,140 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.autofill.ui; + +import static com.android.server.autofill.Helper.sVerbose; + +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.os.Handler; +import android.util.Slog; + +import com.android.internal.view.inline.IInlineContentCallback; +import com.android.internal.view.inline.IInlineContentProvider; +import com.android.server.FgThread; + +/** + * We create one instance of this class for each {@link android.view.inputmethod.InlineSuggestion} + * instance. Each inline suggestion instance will only be sent to the remote IME process once. In + * case of filtering and resending the suggestion when keyboard state changes between hide and + * show, a new instance of this class will be created using {@link #copy()}, with the same backing + * {@link RemoteInlineSuggestionUi}. When the + * {@link #provideContent(int, int, IInlineContentCallback)} is called the first time (it's only + * allowed to be called at most once), the passed in width/height is used to determine whether + * the existing {@link RemoteInlineSuggestionUi} provided in the constructor can be reused, or a + * new one should be created to suit the new size requirement for the view. In normal cases, + * we should not expect the size requirement to change, although in theory the public API allows + * the IME to do that. + * + *

This design is to enable us to be able to reuse the backing remote view while still keeping + * the callbacks relatively well aligned. For example, if we allow multiple remote IME binder + * callbacks to call into one instance of this class, then binder A may call in with width/height + * X for which we create a view (i.e. {@link RemoteInlineSuggestionUi}) for it, + * + * See also {@link RemoteInlineSuggestionUi} for relevant information. + */ +public final class InlineContentProviderImpl extends IInlineContentProvider.Stub { + + // TODO(b/153615023): consider not holding strong reference to heavy objects in this stub, to + // avoid memory leak in case the client app is holding the remote reference for a longer + // time than expected. Essentially we need strong reference in the system process to + // the member variables, but weak reference to them in the IInlineContentProvider.Stub. + + private static final String TAG = InlineContentProviderImpl.class.getSimpleName(); + + private final Handler mHandler = FgThread.getHandler();; + + @NonNull + private final RemoteInlineSuggestionViewConnector mRemoteInlineSuggestionViewConnector; + @Nullable + private RemoteInlineSuggestionUi mRemoteInlineSuggestionUi; + + private boolean mProvideContentCalled = false; + + InlineContentProviderImpl( + @NonNull RemoteInlineSuggestionViewConnector remoteInlineSuggestionViewConnector, + @Nullable RemoteInlineSuggestionUi remoteInlineSuggestionUi) { + mRemoteInlineSuggestionViewConnector = remoteInlineSuggestionViewConnector; + mRemoteInlineSuggestionUi = remoteInlineSuggestionUi; + } + + /** + * Returns a new instance of this class, with the same {@code mInlineSuggestionRenderer} and + * {@code mRemoteInlineSuggestionUi}. The latter may or may not be reusable depending on the + * size information provided when the client calls {@link #provideContent(int, int, + * IInlineContentCallback)}. + */ + @NonNull + public InlineContentProviderImpl copy() { + return new InlineContentProviderImpl(mRemoteInlineSuggestionViewConnector, + mRemoteInlineSuggestionUi); + } + + /** + * Provides a SurfacePackage associated with the inline suggestion view to the IME. If such + * view doesn't exit, then create a new one. This method should be called once per lifecycle + * of this object. Any further calls to the method will be ignored. + */ + @Override + public void provideContent(int width, int height, IInlineContentCallback callback) { + mHandler.post(() -> handleProvideContent(width, height, callback)); + } + + @Override + public void requestSurfacePackage() { + mHandler.post(this::handleGetSurfacePackage); + } + + @Override + public void onSurfacePackageReleased() { + mHandler.post(this::handleOnSurfacePackageReleased); + } + + private void handleProvideContent(int width, int height, IInlineContentCallback callback) { + if (sVerbose) Slog.v(TAG, "handleProvideContent"); + if (mProvideContentCalled) { + // This method should only be called once. + return; + } + mProvideContentCalled = true; + if (mRemoteInlineSuggestionUi == null || !mRemoteInlineSuggestionUi.match(width, height)) { + mRemoteInlineSuggestionUi = new RemoteInlineSuggestionUi( + mRemoteInlineSuggestionViewConnector, + width, height, mHandler); + } + mRemoteInlineSuggestionUi.setInlineContentCallback(callback); + mRemoteInlineSuggestionUi.requestSurfacePackage(); + } + + private void handleGetSurfacePackage() { + if (sVerbose) Slog.v(TAG, "handleGetSurfacePackage"); + if (!mProvideContentCalled || mRemoteInlineSuggestionUi == null) { + // provideContent should be called first, and remote UI should not be null. + return; + } + mRemoteInlineSuggestionUi.requestSurfacePackage(); + } + + private void handleOnSurfacePackageReleased() { + if (sVerbose) Slog.v(TAG, "handleOnSurfacePackageReleased"); + if (!mProvideContentCalled || mRemoteInlineSuggestionUi == null) { + // provideContent should be called first, and remote UI should not be null. + return; + } + mRemoteInlineSuggestionUi.surfacePackageReleased(); + } +} diff --git a/services/autofill/java/com/android/server/autofill/ui/InlineSuggestionFactory.java b/services/autofill/java/com/android/server/autofill/ui/InlineSuggestionFactory.java index 79c9efa48d737..e74463a8584b0 100644 --- a/services/autofill/java/com/android/server/autofill/ui/InlineSuggestionFactory.java +++ b/services/autofill/java/com/android/server/autofill/ui/InlineSuggestionFactory.java @@ -24,14 +24,11 @@ import android.annotation.Nullable; import android.content.Intent; import android.content.IntentSender; import android.os.IBinder; -import android.os.RemoteException; import android.service.autofill.Dataset; import android.service.autofill.FillResponse; -import android.service.autofill.IInlineSuggestionUiCallback; import android.service.autofill.InlinePresentation; import android.text.TextUtils; import android.util.Slog; -import android.view.SurfaceControlViewHost; import android.view.autofill.AutofillId; import android.view.autofill.AutofillManager; import android.view.autofill.AutofillValue; @@ -41,12 +38,8 @@ import android.view.inputmethod.InlineSuggestionsRequest; import android.view.inputmethod.InlineSuggestionsResponse; import android.widget.inline.InlinePresentationSpec; -import com.android.internal.view.inline.IInlineContentCallback; import com.android.internal.view.inline.IInlineContentProvider; -import com.android.server.LocalServices; -import com.android.server.UiThread; import com.android.server.autofill.RemoteInlineSuggestionRenderService; -import com.android.server.inputmethod.InputMethodManagerInternal; import java.util.ArrayList; import java.util.List; @@ -72,6 +65,27 @@ public final class InlineSuggestionFactory { void startIntentSender(@NonNull IntentSender intentSender, @NonNull Intent intent); } + /** + * Returns a copy of the response, that internally copies the {@link IInlineContentProvider} + * so that it's not reused by the remote IME process across different inline suggestions. + * See {@link InlineContentProviderImpl} for why this is needed. + */ + @NonNull + public static InlineSuggestionsResponse copy(@NonNull InlineSuggestionsResponse response) { + final ArrayList copiedInlineSuggestions = new ArrayList<>(); + for (InlineSuggestion inlineSuggestion : response.getInlineSuggestions()) { + final IInlineContentProvider contentProvider = inlineSuggestion.getContentProvider(); + if (contentProvider instanceof InlineContentProviderImpl) { + copiedInlineSuggestions.add(new + InlineSuggestion(inlineSuggestion.getInfo(), + ((InlineContentProviderImpl) contentProvider).copy())); + } else { + copiedInlineSuggestions.add(inlineSuggestion); + } + } + return new InlineSuggestionsResponse(copiedInlineSuggestions); + } + /** * Creates an {@link InlineSuggestionsResponse} with the {@code datasets} provided by the * autofill service, potentially filtering the datasets. @@ -276,78 +290,20 @@ public final class InlineSuggestionFactory { inlinePresentation.isPinned()); } - private static IInlineContentProvider.Stub createInlineContentProvider( + private static IInlineContentProvider createInlineContentProvider( @NonNull InlinePresentation inlinePresentation, @Nullable Runnable onClickAction, @NonNull Runnable onErrorCallback, @NonNull Consumer intentSenderConsumer, @Nullable RemoteInlineSuggestionRenderService remoteRenderService, @Nullable IBinder hostInputToken, int displayId) { - return new IInlineContentProvider.Stub() { - @Override - public void provideContent(int width, int height, IInlineContentCallback callback) { - UiThread.getHandler().post(() -> { - final IInlineSuggestionUiCallback uiCallback = createInlineSuggestionUiCallback( - callback, onClickAction, onErrorCallback, intentSenderConsumer); - - if (remoteRenderService == null) { - Slog.e(TAG, "RemoteInlineSuggestionRenderService is null"); - return; - } - - remoteRenderService.renderSuggestion(uiCallback, inlinePresentation, - width, height, hostInputToken, displayId); - }); - } - }; - } - - private static IInlineSuggestionUiCallback.Stub createInlineSuggestionUiCallback( - @NonNull IInlineContentCallback callback, @NonNull Runnable onAutofillCallback, - @NonNull Runnable onErrorCallback, - @NonNull Consumer intentSenderConsumer) { - return new IInlineSuggestionUiCallback.Stub() { - @Override - public void onClick() throws RemoteException { - onAutofillCallback.run(); - callback.onClick(); - } - - @Override - public void onLongClick() throws RemoteException { - callback.onLongClick(); - } - - @Override - public void onContent(SurfaceControlViewHost.SurfacePackage surface, int width, - int height) - throws RemoteException { - callback.onContent(surface, width, height); - surface.release(); - } - - @Override - public void onError() throws RemoteException { - onErrorCallback.run(); - } - - @Override - public void onTransferTouchFocusToImeWindow(IBinder sourceInputToken, int displayId) - throws RemoteException { - final InputMethodManagerInternal inputMethodManagerInternal = - LocalServices.getService(InputMethodManagerInternal.class); - if (!inputMethodManagerInternal.transferTouchFocusToImeWindow(sourceInputToken, - displayId)) { - Slog.e(TAG, "Cannot transfer touch focus from suggestion to IME"); - onErrorCallback.run(); - } - } - - @Override - public void onStartIntentSender(IntentSender intentSender) { - intentSenderConsumer.accept(intentSender); - } - }; + RemoteInlineSuggestionViewConnector + remoteInlineSuggestionViewConnector = new RemoteInlineSuggestionViewConnector( + remoteRenderService, inlinePresentation, hostInputToken, displayId, onClickAction, + onErrorCallback, intentSenderConsumer); + InlineContentProviderImpl inlineContentProvider = new InlineContentProviderImpl( + remoteInlineSuggestionViewConnector, null); + return inlineContentProvider; } private InlineSuggestionFactory() { diff --git a/services/autofill/java/com/android/server/autofill/ui/RemoteInlineSuggestionUi.java b/services/autofill/java/com/android/server/autofill/ui/RemoteInlineSuggestionUi.java new file mode 100644 index 0000000000000..00a5283c9b1fc --- /dev/null +++ b/services/autofill/java/com/android/server/autofill/ui/RemoteInlineSuggestionUi.java @@ -0,0 +1,291 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.autofill.ui; + +import static com.android.server.autofill.Helper.sDebug; + +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.content.IntentSender; +import android.os.Handler; +import android.os.IBinder; +import android.os.RemoteException; +import android.service.autofill.IInlineSuggestionUi; +import android.service.autofill.IInlineSuggestionUiCallback; +import android.service.autofill.ISurfacePackageResultCallback; +import android.util.Slog; +import android.view.SurfaceControlViewHost; + +import com.android.internal.view.inline.IInlineContentCallback; + +/** + * The instance of this class lives in the system server, orchestrating the communication between + * the remote process owning embedded view (i.e. ExtServices) and the remote process hosting the + * embedded view (i.e. IME). It's also responsible for releasing the embedded view from the owning + * process when it's not longer needed in the hosting process. + * + *

An instance of this class may be reused to associate with multiple instances of + * {@link InlineContentProviderImpl}s, each of which wraps a callback from the IME. But at any + * given time, there is only one active IME callback which this class will callback into. + * + *

This class is thread safe, because all the outside calls are piped into the same single + * thread handler to be processed. + * + * TODO(b/154683107): implement the reference counting in case there are multiple active + * SurfacePackages at the same time. This will not happen for now since all the InlineSuggestions + * sharing the same UI will be sent to the same IME window, so the previous view will be detached + * before the new view are attached to the window. + */ +final class RemoteInlineSuggestionUi { + + private static final String TAG = RemoteInlineSuggestionUi.class.getSimpleName(); + + // The delay time to release the remote inline suggestion view (in the renderer + // process) after receiving a signal about the surface package being released due to being + // detached from the window in the host app (in the IME process). The release will be + // canceled if the host app reattaches the view to a window within this delay time. + // TODO(b/154683107): try out using the Chroreographer to schedule the release right at the + // next frame. Basically if the view is not re-attached to the window immediately in the next + // frame after it was detached, then it will be released. + private static final long RELEASE_REMOTE_VIEW_HOST_DELAY_MS = 200; + + @NonNull + private final Handler mHandler; + @NonNull + private final RemoteInlineSuggestionViewConnector mRemoteInlineSuggestionViewConnector; + private final int mWidth; + private final int mHeight; + @NonNull + private final InlineSuggestionUiCallbackImpl mInlineSuggestionUiCallback; + + @Nullable + private IInlineContentCallback mInlineContentCallback; // from IME + + /** + * Remote inline suggestion view, backed by an instance of {@link SurfaceControlViewHost} in + * the render service process. We takes care of releasing it when there is no remote + * reference to it (from IME), and we will create a new instance of the view when it's needed + * by IME again. + */ + @Nullable + private IInlineSuggestionUi mInlineSuggestionUi; + private boolean mWaitingForUiCreation = false; + private int mActualWidth; + private int mActualHeight; + + @Nullable + private Runnable mDelayedReleaseViewRunnable; + + RemoteInlineSuggestionUi( + @NonNull RemoteInlineSuggestionViewConnector remoteInlineSuggestionViewConnector, + int width, int height, Handler handler) { + mHandler = handler; + mRemoteInlineSuggestionViewConnector = remoteInlineSuggestionViewConnector; + mWidth = width; + mHeight = height; + mInlineSuggestionUiCallback = new InlineSuggestionUiCallbackImpl(); + } + + /** + * Updates the callback from the IME process. It'll swap out the previous IME callback, and + * all the subsequent callback events (onClick, onLongClick, touch event transfer, etc) will + * be directed to the new callback. + */ + void setInlineContentCallback(@NonNull IInlineContentCallback inlineContentCallback) { + mHandler.post(() -> { + mInlineContentCallback = inlineContentCallback; + }); + } + + /** + * Handles the request from the IME process to get a new surface package. May create a new + * view in the renderer process if the existing view is already released. + */ + void requestSurfacePackage() { + mHandler.post(this::handleRequestSurfacePackage); + } + + /** + * Handles the signal from the IME process that the previously sent surface package has been + * released. + */ + void surfacePackageReleased() { + mHandler.post(this::handleSurfacePackageReleased); + } + + /** + * Returns true if the provided size matches the remote view's size. + */ + boolean match(int width, int height) { + return mWidth == width && mHeight == height; + } + + private void handleSurfacePackageReleased() { + cancelPendingReleaseViewRequest(); + + // Schedule a delayed release view request + mDelayedReleaseViewRunnable = () -> { + if (mInlineSuggestionUi != null) { + try { + mInlineSuggestionUi.releaseSurfaceControlViewHost(); + mInlineSuggestionUi = null; + } catch (RemoteException e) { + Slog.w(TAG, "RemoteException calling releaseSurfaceControlViewHost"); + } + } + mDelayedReleaseViewRunnable = null; + }; + mHandler.postDelayed(mDelayedReleaseViewRunnable, RELEASE_REMOTE_VIEW_HOST_DELAY_MS); + } + + private void handleRequestSurfacePackage() { + cancelPendingReleaseViewRequest(); + + if (mInlineSuggestionUi == null) { + if (mWaitingForUiCreation) { + // This could happen in the following case: the remote embedded view was released + // when previously detached from window. An event after that to re-attached to + // the window will cause us calling the renderSuggestion again. Now, before the + // render call returns a new surface package, if the view is detached and + // re-attached to the window, causing this method to be called again, we will get + // to this state. This request will be ignored and the surface package will still + // be sent back once the view is rendered. + if (sDebug) Slog.d(TAG, "Inline suggestion ui is not ready"); + } else { + mRemoteInlineSuggestionViewConnector.renderSuggestion(mWidth, mHeight, + mInlineSuggestionUiCallback); + mWaitingForUiCreation = true; + } + } else { + try { + mInlineSuggestionUi.getSurfacePackage(new ISurfacePackageResultCallback.Stub() { + @Override + public void onResult(SurfaceControlViewHost.SurfacePackage result) + throws RemoteException { + if (sDebug) Slog.d(TAG, "Sending new SurfacePackage to IME"); + mInlineContentCallback.onContent(result, mActualWidth, mActualHeight); + } + }); + } catch (RemoteException e) { + Slog.w(TAG, "RemoteException calling getSurfacePackage."); + } + } + } + + private void cancelPendingReleaseViewRequest() { + if (mDelayedReleaseViewRunnable != null) { + mHandler.removeCallbacks(mDelayedReleaseViewRunnable); + mDelayedReleaseViewRunnable = null; + } + } + + /** + * This is called when a new inline suggestion UI is inflated from the ext services. + */ + private void handleInlineSuggestionUiReady(IInlineSuggestionUi content, + SurfaceControlViewHost.SurfacePackage surfacePackage, int width, int height) { + mInlineSuggestionUi = content; + mWaitingForUiCreation = false; + mActualWidth = width; + mActualHeight = height; + if (mInlineContentCallback != null) { + try { + mInlineContentCallback.onContent(surfacePackage, mActualWidth, mActualHeight); + } catch (RemoteException e) { + Slog.w(TAG, "RemoteException calling onContent"); + } + } + if (surfacePackage != null) { + surfacePackage.release(); + } + } + + private void handleOnClick() { + // Autofill the value + mRemoteInlineSuggestionViewConnector.onClick(); + + // Notify the remote process (IME) that hosts the embedded UI that it's clicked + if (mInlineContentCallback != null) { + try { + mInlineContentCallback.onClick(); + } catch (RemoteException e) { + Slog.w(TAG, "RemoteException calling onClick"); + } + } + } + + private void handleOnLongClick() { + // Notify the remote process (IME) that hosts the embedded UI that it's long clicked + if (mInlineContentCallback != null) { + try { + mInlineContentCallback.onLongClick(); + } catch (RemoteException e) { + Slog.w(TAG, "RemoteException calling onLongClick"); + } + } + } + + private void handleOnError() { + mRemoteInlineSuggestionViewConnector.onError(); + } + + private void handleOnTransferTouchFocusToImeWindow(IBinder sourceInputToken, int displayId) { + mRemoteInlineSuggestionViewConnector.onTransferTouchFocusToImeWindow(sourceInputToken, + displayId); + } + + private void handleOnStartIntentSender(IntentSender intentSender) { + mRemoteInlineSuggestionViewConnector.onStartIntentSender(intentSender); + } + + /** + * Responsible for communicating with the inline suggestion view owning process. + */ + private class InlineSuggestionUiCallbackImpl extends IInlineSuggestionUiCallback.Stub { + + @Override + public void onClick() { + mHandler.post(RemoteInlineSuggestionUi.this::handleOnClick); + } + + @Override + public void onLongClick() { + mHandler.post(RemoteInlineSuggestionUi.this::handleOnLongClick); + } + + @Override + public void onContent(IInlineSuggestionUi content, + SurfaceControlViewHost.SurfacePackage surface, int width, int height) { + mHandler.post(() -> handleInlineSuggestionUiReady(content, surface, width, height)); + } + + @Override + public void onError() { + mHandler.post(RemoteInlineSuggestionUi.this::handleOnError); + } + + @Override + public void onTransferTouchFocusToImeWindow(IBinder sourceInputToken, int displayId) { + mHandler.post(() -> handleOnTransferTouchFocusToImeWindow(sourceInputToken, displayId)); + } + + @Override + public void onStartIntentSender(IntentSender intentSender) { + mHandler.post(() -> handleOnStartIntentSender(intentSender)); + } + } +} diff --git a/services/autofill/java/com/android/server/autofill/ui/RemoteInlineSuggestionViewConnector.java b/services/autofill/java/com/android/server/autofill/ui/RemoteInlineSuggestionViewConnector.java new file mode 100644 index 0000000000000..9d23c171800d9 --- /dev/null +++ b/services/autofill/java/com/android/server/autofill/ui/RemoteInlineSuggestionViewConnector.java @@ -0,0 +1,125 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.server.autofill.ui; + +import static com.android.server.autofill.Helper.sDebug; + +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.content.IntentSender; +import android.os.IBinder; +import android.service.autofill.IInlineSuggestionUiCallback; +import android.service.autofill.InlinePresentation; +import android.util.Slog; + +import com.android.server.LocalServices; +import com.android.server.autofill.RemoteInlineSuggestionRenderService; +import com.android.server.inputmethod.InputMethodManagerInternal; + +import java.util.function.Consumer; + +/** + * Wraps the parameters needed to create a new inline suggestion view in the remote renderer + * service, and handles the callback from the events on the created remote view. + */ +final class RemoteInlineSuggestionViewConnector { + private static final String TAG = RemoteInlineSuggestionViewConnector.class.getSimpleName(); + + @Nullable + private final RemoteInlineSuggestionRenderService mRemoteRenderService; + @NonNull + private final InlinePresentation mInlinePresentation; + @Nullable + private final IBinder mHostInputToken; + private final int mDisplayId; + + @NonNull + private final Runnable mOnAutofillCallback; + @NonNull + private final Runnable mOnErrorCallback; + @NonNull + private final Consumer mStartIntentSenderFromClientApp; + + RemoteInlineSuggestionViewConnector( + @Nullable RemoteInlineSuggestionRenderService remoteRenderService, + @NonNull InlinePresentation inlinePresentation, + @Nullable IBinder hostInputToken, + int displayId, + @NonNull Runnable onAutofillCallback, + @NonNull Runnable onErrorCallback, + @NonNull Consumer startIntentSenderFromClientApp) { + mRemoteRenderService = remoteRenderService; + mInlinePresentation = inlinePresentation; + mHostInputToken = hostInputToken; + mDisplayId = displayId; + + mOnAutofillCallback = onAutofillCallback; + mOnErrorCallback = onErrorCallback; + mStartIntentSenderFromClientApp = startIntentSenderFromClientApp; + } + + /** + * Calls the remote renderer service to create a new inline suggestion view. + * + * @return true if the call is made to the remote renderer service, false otherwise. + */ + public boolean renderSuggestion(int width, int height, + @NonNull IInlineSuggestionUiCallback callback) { + if (mRemoteRenderService != null) { + if (sDebug) Slog.d(TAG, "Request to recreate the UI"); + mRemoteRenderService.renderSuggestion(callback, mInlinePresentation, width, height, + mHostInputToken, mDisplayId); + return true; + } + return false; + } + + /** + * Handles the callback for the event of remote view being clicked. + */ + public void onClick() { + mOnAutofillCallback.run(); + } + + /** + * Handles the callback for the remote error when creating or interacting with the view. + */ + public void onError() { + mOnErrorCallback.run(); + } + + /** + * Handles the callback for transferring the touch event on the remote view to the IME + * process. + */ + public void onTransferTouchFocusToImeWindow(IBinder sourceInputToken, int displayId) { + final InputMethodManagerInternal inputMethodManagerInternal = + LocalServices.getService(InputMethodManagerInternal.class); + if (!inputMethodManagerInternal.transferTouchFocusToImeWindow(sourceInputToken, + displayId)) { + Slog.e(TAG, "Cannot transfer touch focus from suggestion to IME"); + mOnErrorCallback.run(); + } + } + + /** + * Handles starting an intent sender from the client app's process. + */ + public void onStartIntentSender(IntentSender intentSender) { + mStartIntentSenderFromClientApp.accept(intentSender); + } +}