Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge "Support re-attaching the inline suggestion view to window" into rvc-dev

Browse Source
This commit is contained in:
Feng Cao
2020-05-07 05:23:15 +00:00
committed by Android (Google) Code Review
parent f7d1f1a40c b46851c964
commit 254545cae5
13 changed files with 1069 additions and 188 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
core/java/android/service/autofill/IInlineSuggestionUi.aidl Normal file
View File

@@ -0,0 +1,29 @@
/*
* Copyright (C) 2020 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package android.service.autofill;
import android.service.autofill.ISurfacePackageResultCallback;
/**
* Interface to interact with a remote inline suggestion UI.
*
* @hide
*/
oneway interface IInlineSuggestionUi {
void getSurfacePackage(ISurfacePackageResultCallback callback);
void releaseSurfaceControlViewHost();
}

6
core/java/android/service/autofill/IInlineSuggestionUiCallback.aidl
View File

@@ -18,17 +18,19 @@ package android.service.autofill;
import android.content.IntentSender;
import android.os.IBinder;
import android.service.autofill.IInlineSuggestionUi;
import android.view.SurfaceControlViewHost;
/**
* Interface to receive events from inline suggestions.
* Interface to receive events from a remote inline suggestion UI.
*
* @hide
*/
oneway interface IInlineSuggestionUiCallback {
void onClick();
void onLongClick();
void onContent(in SurfaceControlViewHost.SurfacePackage surface, int width, int height);
void onContent(in IInlineSuggestionUi content, in SurfaceControlViewHost.SurfacePackage surface,
int width, int height);
void onError();
void onTransferTouchFocusToImeWindow(in IBinder sourceInputToken, int displayId);
void onStartIntentSender(in IntentSender intentSender);

28
core/java/android/service/autofill/ISurfacePackageResultCallback.aidl Normal file
View File

@@ -0,0 +1,28 @@
/*
* Copyright (C) 2020 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package android.service.autofill;
import android.view.SurfaceControlViewHost;
/**
* Interface to receive a SurfaceControlViewHost.SurfacePackage.
*
* @hide
*/
oneway interface ISurfacePackageResultCallback {
void onResult(in SurfaceControlViewHost.SurfacePackage result);
}

121
core/java/android/service/autofill/InlineSuggestionRenderService.java
View File

@@ -33,6 +33,7 @@ import android.os.Looper;
import android.os.RemoteCallback;
import android.os.RemoteException;
import android.util.Log;
import android.util.LruCache;
import android.util.Size;
import android.view.Display;
import android.view.SurfaceControlViewHost;
@@ -40,6 +41,8 @@ import android.view.View;
import android.view.ViewGroup;
import android.view.WindowManager;
import java.lang.ref.WeakReference;
/**
* A service that renders an inline presentation view given the {@link InlinePresentation}.
*
@@ -65,6 +68,27 @@ public abstract class InlineSuggestionRenderService extends Service {
private IInlineSuggestionUiCallback mCallback;
/**
* A local LRU cache keeping references to the inflated {@link SurfaceControlViewHost}s, so
* they can be released properly when no longer used. Each view needs to be tracked separately,
* therefore for simplicity we use the hash code of the value object as key in the cache.
*/
private final LruCache<InlineSuggestionUiImpl, Boolean> mActiveInlineSuggestions =
new LruCache<InlineSuggestionUiImpl, Boolean>(30) {
@Override
public void entryRemoved(boolean evicted, InlineSuggestionUiImpl key,
Boolean oldValue,
Boolean newValue) {
if (evicted) {
Log.w(TAG,
"Hit max=100 entries in the cache. Releasing oldest one to make "
+ "space.");
key.releaseSurfaceControlViewHost();
}
}
};
/**
* If the specified {@code width}/{@code height} is an exact value, then it will be returned as
* is, otherwise the method tries to measure a size that is just large enough to fit the view
@@ -169,8 +193,14 @@ public abstract class InlineSuggestionRenderService extends Service {
return true;
});
sendResult(callback, host.getSurfacePackage(), measuredSize.getWidth(),
measuredSize.getHeight());
try {
InlineSuggestionUiImpl uiImpl = new InlineSuggestionUiImpl(host, mHandler);
mActiveInlineSuggestions.put(uiImpl, true);
callback.onContent(new InlineSuggestionUiWrapper(uiImpl), host.getSurfacePackage(),
measuredSize.getWidth(), measuredSize.getHeight());
} catch (RemoteException e) {
Log.w(TAG, "RemoteException calling onContent()");
}
} finally {
updateDisplay(Display.DEFAULT_DISPLAY);
}
@@ -181,12 +211,87 @@ public abstract class InlineSuggestionRenderService extends Service {
callback.sendResult(rendererInfo);
}
private void sendResult(@NonNull IInlineSuggestionUiCallback callback,
@Nullable SurfaceControlViewHost.SurfacePackage surface, int width, int height) {
try {
callback.onContent(surface, width, height);
} catch (RemoteException e) {
Log.w(TAG, "RemoteException calling onContent(" + surface + ")");
/**
* A wrapper class around the {@link InlineSuggestionUiImpl} to ensure it's not strongly
* reference by the remote system server process.
*/
private static final class InlineSuggestionUiWrapper extends
android.service.autofill.IInlineSuggestionUi.Stub {
private final WeakReference<InlineSuggestionUiImpl> mUiImpl;
InlineSuggestionUiWrapper(InlineSuggestionUiImpl uiImpl) {
mUiImpl = new WeakReference<>(uiImpl);
}
@Override
public void releaseSurfaceControlViewHost() {
final InlineSuggestionUiImpl uiImpl = mUiImpl.get();
if (uiImpl != null) {
uiImpl.releaseSurfaceControlViewHost();
}
}
@Override
public void getSurfacePackage(ISurfacePackageResultCallback callback) {
final InlineSuggestionUiImpl uiImpl = mUiImpl.get();
if (uiImpl != null) {
uiImpl.getSurfacePackage(callback);
}
}
}
/**
* Keeps track of a SurfaceControlViewHost to ensure it's released when its lifecycle ends.
*
* <p>This class is thread safe, because all the outside calls are piped into a single
* handler thread to be processed.
*/
private final class InlineSuggestionUiImpl {
@Nullable
private SurfaceControlViewHost mViewHost;
@NonNull
private final Handler mHandler;
InlineSuggestionUiImpl(SurfaceControlViewHost viewHost, Handler handler) {
this.mViewHost = viewHost;
this.mHandler = handler;
}
/**
* Call {@link SurfaceControlViewHost#release()} to release it. After this, this view is
* not usable, and any further calls to the
* {@link #getSurfacePackage(ISurfacePackageResultCallback)} will get {@code null} result.
*/
public void releaseSurfaceControlViewHost() {
mHandler.post(() -> {
if (mViewHost == null) {
return;
}
Log.v(TAG, "Releasing inline suggestion view host");
mViewHost.release();
mViewHost = null;
InlineSuggestionRenderService.this.mActiveInlineSuggestions.remove(
InlineSuggestionUiImpl.this);
Log.v(TAG, "Removed the inline suggestion from the cache, current size="
+ InlineSuggestionRenderService.this.mActiveInlineSuggestions.size());
});
}
/**
* Sends back a new {@link android.view.SurfaceControlViewHost.SurfacePackage} if the view
* is not released, {@code null} otherwise.
*/
public void getSurfacePackage(ISurfacePackageResultCallback callback) {
Log.d(TAG, "getSurfacePackage");
mHandler.post(() -> {
try {
callback.onResult(mViewHost == null ? null : mViewHost.getSurfacePackage());
} catch (RemoteException e) {
Log.w(TAG, "RemoteException calling onSurfacePackage");
}
});
}
}

270
core/java/android/view/inputmethod/InlineSuggestion.java
View File

@@ -18,11 +18,13 @@ package android.view.inputmethod;
import android.annotation.BinderThread;
import android.annotation.CallbackExecutor;
import android.annotation.MainThread;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.annotation.TestApi;
import android.content.Context;
import android.os.AsyncTask;
import android.os.Handler;
import android.os.Looper;
import android.os.Parcel;
import android.os.Parcelable;
import android.os.RemoteException;
@@ -42,26 +44,26 @@ import java.util.concurrent.Executor;
import java.util.function.Consumer;
/**
* This class represents an inline suggestion which is made by one app
* and can be embedded into the UI of another. Suggestions may contain
* sensitive information not known to the host app which needs to be
* protected from spoofing. To address that the suggestion view inflated
* on demand for embedding is created in such a way that the hosting app
* cannot introspect its content and cannot interact with it.
* This class represents an inline suggestion which is made by one app and can be embedded into the
* UI of another. Suggestions may contain sensitive information not known to the host app which
* needs to be protected from spoofing. To address that the suggestion view inflated on demand for
* embedding is created in such a way that the hosting app cannot introspect its content and cannot
* interact with it.
*/
@DataClass(
genEqualsHashCode = true,
genToString = true,
genHiddenConstDefs = true,
@DataClass(genEqualsHashCode = true, genToString = true, genHiddenConstDefs = true,
genHiddenConstructor = true)
@DataClass.Suppress({"getContentProvider"})
public final class InlineSuggestion implements Parcelable {
private static final String TAG = "InlineSuggestion";
private final @NonNull InlineSuggestionInfo mInfo;
@NonNull
private final InlineSuggestionInfo mInfo;
private final @Nullable IInlineContentProvider mContentProvider;
/**
* @hide
*/
@Nullable
private final IInlineContentProvider mContentProvider;
/**
* Used to keep a strong reference to the callback so it doesn't get garbage collected.
@@ -69,7 +71,8 @@ public final class InlineSuggestion implements Parcelable {
* @hide
*/
@DataClass.ParcelWith(InlineContentCallbackImplParceling.class)
private @Nullable InlineContentCallbackImpl mInlineContentCallback;
@Nullable
private InlineContentCallbackImpl mInlineContentCallback;
/**
* Creates a new {@link InlineSuggestion}, for testing purpose.
@@ -87,8 +90,7 @@ public final class InlineSuggestion implements Parcelable {
*
* @hide
*/
public InlineSuggestion(
@NonNull InlineSuggestionInfo info,
public InlineSuggestion(@NonNull InlineSuggestionInfo info,
@Nullable IInlineContentProvider contentProvider) {
this(info, contentProvider, /* inlineContentCallback */ null);
}
@@ -96,25 +98,30 @@ public final class InlineSuggestion implements Parcelable {
/**
* Inflates a view with the content of this suggestion at a specific size.
*
* <p> The size must be either 1) between the
* {@link android.widget.inline.InlinePresentationSpec#getMinSize() min size} and the
* {@link android.widget.inline.InlinePresentationSpec#getMaxSize() max size} of the
* presentation spec returned by {@link InlineSuggestionInfo#getInlinePresentationSpec()},
* or 2) {@link ViewGroup.LayoutParams#WRAP_CONTENT}. If the size is set to
* {@link ViewGroup.LayoutParams#WRAP_CONTENT}, then the size of the inflated view will be just
* large enough to fit the content, while still conforming to the min / max size specified by
* the {@link android.widget.inline.InlinePresentationSpec}.
* <p> Each dimension of the size must satisfy one of the following conditions:
*
* <ol>
* <li>between {@link android.widget.inline.InlinePresentationSpec#getMinSize()} and
* {@link android.widget.inline.InlinePresentationSpec#getMaxSize()} of the presentation spec
* from {@code mInfo}
* <li>{@link ViewGroup.LayoutParams#WRAP_CONTENT}
* </ol>
*
* If the size is set to {@link
* ViewGroup.LayoutParams#WRAP_CONTENT}, then the size of the inflated view will be just large
* enough to fit the content, while still conforming to the min / max size specified by the
* {@link android.widget.inline.InlinePresentationSpec}.
*
* <p> The caller can attach an {@link android.view.View.OnClickListener} and/or an
* {@link android.view.View.OnLongClickListener} to the view in the
* {@code callback} to receive click and long click events on the view.
* {@link android.view.View.OnLongClickListener} to the view in the {@code callback} to receive
* click and long click events on the view.
*
* @param context Context in which to inflate the view.
* @param size The size at which to inflate the suggestion. For each dimension, it maybe
* an exact value or {@link ViewGroup.LayoutParams#WRAP_CONTENT}.
* @param callback Callback for receiving the inflated view, where the
* {@link ViewGroup.LayoutParams} of the view is set as the actual size of
* the underlying remote view.
* @param size The size at which to inflate the suggestion. For each dimension, it maybe an
* exact value or {@link ViewGroup.LayoutParams#WRAP_CONTENT}.
* @param callback Callback for receiving the inflated view, where the {@link
* ViewGroup.LayoutParams} of the view is set as the actual size of the
* underlying remote view.
* @throws IllegalArgumentException If an invalid argument is passed.
* @throws IllegalStateException If this method is already called.
*/
@@ -130,19 +137,17 @@ public final class InlineSuggestion implements Parcelable {
+ ", nor wrap_content");
}
mInlineContentCallback = getInlineContentCallback(context, callbackExecutor, callback);
AsyncTask.THREAD_POOL_EXECUTOR.execute(() -> {
if (mContentProvider == null) {
callback.accept(/* view */ null);
return;
}
try {
mContentProvider.provideContent(size.getWidth(), size.getHeight(),
new InlineContentCallbackWrapper(mInlineContentCallback));
} catch (RemoteException e) {
Slog.w(TAG, "Error creating suggestion content surface: " + e);
callback.accept(/* view */ null);
}
});
if (mContentProvider == null) {
callbackExecutor.execute(() -> callback.accept(/* view */ null));
return;
}
try {
mContentProvider.provideContent(size.getWidth(), size.getHeight(),
new InlineContentCallbackWrapper(mInlineContentCallback));
} catch (RemoteException e) {
Slog.w(TAG, "Error creating suggestion content surface: " + e);
callbackExecutor.execute(() -> callback.accept(/* view */ null));
}
}
/**
@@ -161,9 +166,14 @@ public final class InlineSuggestion implements Parcelable {
if (mInlineContentCallback != null) {
throw new IllegalStateException("Already called #inflate()");
}
return new InlineContentCallbackImpl(context, callbackExecutor, callback);
return new InlineContentCallbackImpl(context, mContentProvider, callbackExecutor,
callback);
}
/**
* A wrapper class around the {@link InlineContentCallbackImpl} to ensure it's not strongly
* reference by the remote system server process.
*/
private static final class InlineContentCallbackWrapper extends IInlineContentCallback.Stub {
private final WeakReference<InlineContentCallbackImpl> mCallbackImpl;
@@ -201,17 +211,68 @@ public final class InlineSuggestion implements Parcelable {
}
}
/**
* Handles the communication between the inline suggestion view in current (IME) process and
* the remote view provided from the system server.
*
* <p>This class is thread safe, because all the outside calls are piped into a single
* handler thread to be processed.
*/
private static final class InlineContentCallbackImpl {
private final @NonNull Context mContext;
private final @NonNull Executor mCallbackExecutor;
private final @NonNull Consumer<InlineContentView> mCallback;
private @Nullable InlineContentView mView;
@NonNull
private final Handler mMainHandler = new Handler(Looper.getMainLooper());
@NonNull
private final Context mContext;
@Nullable
private final IInlineContentProvider mInlineContentProvider;
@NonNull
private final Executor mCallbackExecutor;
/**
* Callback from the client (IME) that will receive the inflated suggestion view. It'll
* only be called once when the view SurfacePackage is first sent back to the client. Any
* updates to the view due to attach to window and detach from window events will be
* handled under the hood, transparent from the client.
*/
@NonNull
private final Consumer<InlineContentView> mCallback;
/**
* Indicates whether the first content has been received or not.
*/
private boolean mFirstContentReceived = false;
/**
* The client (IME) side view which internally wraps a remote view. It'll be set when
* {@link #onContent(SurfaceControlViewHost.SurfacePackage, int, int)} is called, which
* should only happen once in the lifecycle of this inline suggestion instance.
*/
@Nullable
private InlineContentView mView;
/**
* The SurfacePackage pointing to the remote view. It's cached here to be sent to the next
* available consumer.
*/
@Nullable
private SurfaceControlViewHost.SurfacePackage mSurfacePackage;
/**
* The callback (from the {@link InlineContentView}) which consumes the surface package.
* It's cached here to be called when the SurfacePackage is returned from the remote
* view owning process.
*/
@Nullable
private Consumer<SurfaceControlViewHost.SurfacePackage> mSurfacePackageConsumer;
InlineContentCallbackImpl(@NonNull Context context,
@Nullable IInlineContentProvider inlineContentProvider,
@NonNull @CallbackExecutor Executor callbackExecutor,
@NonNull Consumer<InlineContentView> callback) {
mContext = context;
mInlineContentProvider = inlineContentProvider;
mCallbackExecutor = callbackExecutor;
mCallback = callback;
}
@@ -219,28 +280,110 @@ public final class InlineSuggestion implements Parcelable {
@BinderThread
public void onContent(SurfaceControlViewHost.SurfacePackage content, int width,
int height) {
if (content == null) {
mMainHandler.post(() -> handleOnContent(content, width, height));
}
@MainThread
private void handleOnContent(SurfaceControlViewHost.SurfacePackage content, int width,
int height) {
if (!mFirstContentReceived) {
handleOnFirstContentReceived(content, width, height);
mFirstContentReceived = true;
} else {
handleOnSurfacePackage(content);
}
}
/**
* Called when the view content is returned for the first time.
*/
@MainThread
private void handleOnFirstContentReceived(SurfaceControlViewHost.SurfacePackage content,
int width, int height) {
mSurfacePackage = content;
if (mSurfacePackage == null) {
mCallbackExecutor.execute(() -> mCallback.accept(/* view */null));
} else {
mView = new InlineContentView(mContext);
mView.setLayoutParams(new ViewGroup.LayoutParams(width, height));
mView.setChildSurfacePackage(content);
mView.setChildSurfacePackageUpdater(getSurfacePackageUpdater());
mCallbackExecutor.execute(() -> mCallback.accept(mView));
}
}
/**
* Called when any subsequent SurfacePackage is returned from the remote view owning
* process.
*/
@MainThread
private void handleOnSurfacePackage(SurfaceControlViewHost.SurfacePackage surfacePackage) {
mSurfacePackage = surfacePackage;
if (mSurfacePackage != null && mSurfacePackageConsumer != null) {
mSurfacePackageConsumer.accept(mSurfacePackage);
mSurfacePackageConsumer = null;
}
}
@MainThread
private void handleOnSurfacePackageReleased() {
mSurfacePackage = null;
try {
mInlineContentProvider.onSurfacePackageReleased();
} catch (RemoteException e) {
Slog.w(TAG, "Error calling onSurfacePackageReleased(): " + e);
}
}
@MainThread
private void handleGetSurfacePackage(
Consumer<SurfaceControlViewHost.SurfacePackage> consumer) {
if (mSurfacePackage != null) {
consumer.accept(mSurfacePackage);
} else {
mSurfacePackageConsumer = consumer;
try {
mInlineContentProvider.requestSurfacePackage();
} catch (RemoteException e) {
Slog.w(TAG, "Error calling getSurfacePackage(): " + e);
consumer.accept(null);
mSurfacePackageConsumer = null;
}
}
}
private InlineContentView.SurfacePackageUpdater getSurfacePackageUpdater() {
return new InlineContentView.SurfacePackageUpdater() {
@Override
public void onSurfacePackageReleased() {
mMainHandler.post(
() -> InlineContentCallbackImpl.this.handleOnSurfacePackageReleased());
}
@Override
public void getSurfacePackage(
Consumer<SurfaceControlViewHost.SurfacePackage> consumer) {
mMainHandler.post(
() -> InlineContentCallbackImpl.this.handleGetSurfacePackage(consumer));
}
};
}
@BinderThread
public void onClick() {
if (mView != null && mView.hasOnClickListeners()) {
mView.callOnClick();
}
mMainHandler.post(() -> {
if (mView != null && mView.hasOnClickListeners()) {
mView.callOnClick();
}
});
}
@BinderThread
public void onLongClick() {
if (mView != null && mView.hasOnLongClickListeners()) {
mView.performLongClick();
}
mMainHandler.post(() -> {
if (mView != null && mView.hasOnLongClickListeners()) {
mView.performLongClick();
}
});
}
}
@@ -262,6 +405,7 @@ public final class InlineSuggestion implements Parcelable {
// Code below generated by codegen v1.0.15.
//
// DO NOT MODIFY!
@@ -301,6 +445,14 @@ public final class InlineSuggestion implements Parcelable {
return mInfo;
}
/**
* @hide
*/
@DataClass.Generated.Member
public @Nullable IInlineContentProvider getContentProvider() {
return mContentProvider;
}
/**
* Used to keep a strong reference to the callback so it doesn't get garbage collected.
*
@@ -421,7 +573,7 @@ public final class InlineSuggestion implements Parcelable {
};
@DataClass.Generated(
time = 1587771173367L,
time = 1588308946517L,
codegenVersion = "1.0.15",
sourceFile = "frameworks/base/core/java/android/view/inputmethod/InlineSuggestion.java",
inputSignatures = "private static final java.lang.String TAG\nprivate final @android.annotation.NonNull android.view.inputmethod.InlineSuggestionInfo mInfo\nprivate final @android.annotation.Nullable com.android.internal.view.inline.IInlineContentProvider mContentProvider\nprivate @com.android.internal.util.DataClass.ParcelWith(android.view.inputmethod.InlineSuggestion.InlineContentCallbackImplParceling.class) @android.annotation.Nullable android.view.inputmethod.InlineSuggestion.InlineContentCallbackImpl mInlineContentCallback\npublic static @android.annotation.TestApi @android.annotation.NonNull android.view.inputmethod.InlineSuggestion newInlineSuggestion(android.view.inputmethod.InlineSuggestionInfo)\npublic void inflate(android.content.Context,android.util.Size,java.util.concurrent.Executor,java.util.function.Consumer<android.widget.inline.InlineContentView>)\nprivate static boolean isValid(int,int,int)\nprivate synchronized android.view.inputmethod.InlineSuggestion.InlineContentCallbackImpl getInlineContentCallback(android.content.Context,java.util.concurrent.Executor,java.util.function.Consumer<android.widget.inline.InlineContentView>)\nclass InlineSuggestion extends java.lang.Object implements [android.os.Parcelable]\n@com.android.internal.util.DataClass(genEqualsHashCode=true, genToString=true, genHiddenConstDefs=true, genHiddenConstructor=true)")

137
core/java/android/widget/inline/InlineContentView.java
View File

@@ -21,40 +21,45 @@ import android.annotation.Nullable;
import android.content.Context;
import android.graphics.PixelFormat;
import android.util.AttributeSet;
import android.util.Log;
import android.view.SurfaceControl;
import android.view.SurfaceControlViewHost;
import android.view.SurfaceHolder;
import android.view.SurfaceView;
import android.view.ViewGroup;
import java.util.function.Consumer;
/**
* This class represents a view that holds opaque content from another app that
* you can inline in your UI.
* This class represents a view that holds opaque content from another app that you can inline in
* your UI.
*
* <p>Since the content presented by this view is from another security domain,it is
* shown on a remote surface preventing the host application from accessing that content.
* Also the host application cannot interact with the inlined content by injecting touch
* events or clicking programmatically.
* shown on a remote surface preventing the host application from accessing that content. Also the
* host application cannot interact with the inlined content by injecting touch events or clicking
* programmatically.
*
* <p>This view can be overlaid by other windows, i.e. redressed, but if this is the case
* the inined UI would not be interactive. Sometimes this is desirable, e.g. animating
* transitions.
* the inlined UI would not be interactive. Sometimes this is desirable, e.g. animating transitions.
*
* <p>By default the surface backing this view is shown on top of the hosting window such
* that the inlined content is interactive. However, you can temporarily move the surface
* under the hosting window which could be useful in some cases, e.g. animating transitions.
* At this point the inlined content will not be interactive and the touch events would
* be delivered to your app.
* <p>
* Instances of this class are created by the platform and can be programmatically attached
* to your UI. Once you attach and detach this view it can not longer be reused and you
* should obtain a new view from the platform via the dedicated APIs.
* that the inlined content is interactive. However, you can temporarily move the surface under the
* hosting window which could be useful in some cases, e.g. animating transitions. At this point the
* inlined content will not be interactive and the touch events would be delivered to your app.
*
* <p> Instances of this class are created by the platform and can be programmatically attached to
* your UI. Once the view is attached to the window, you may detach and reattach it to the window.
* It should work seamlessly from the hosting process's point of view.
*/
public class InlineContentView extends ViewGroup {
private static final String TAG = "InlineContentView";
private static final boolean DEBUG = false;
/**
* Callback for observing the lifecycle of the surface control
* that manipulates the backing secure embedded UI surface.
* Callback for observing the lifecycle of the surface control that manipulates the backing
* secure embedded UI surface.
*/
public interface SurfaceControlCallback {
/**
@@ -72,15 +77,41 @@ public class InlineContentView extends ViewGroup {
void onDestroyed(@NonNull SurfaceControl surfaceControl);
}
private final @NonNull SurfaceHolder.Callback mSurfaceCallback = new SurfaceHolder.Callback() {
/**
* Callback for sending an updated surface package in case the previous one is released
* from the detached from window event, and for getting notified of such event.
*
* This is expected to be provided to the {@link InlineContentView} so it can get updates
* from and send updates to the remote content (i.e. surface package) provider.
*
* @hide
*/
public interface SurfacePackageUpdater {
/**
* Called when the previous surface package is released due to view being detached
* from the window.
*/
void onSurfacePackageReleased();
/**
* Called to request an updated surface package.
*
* @param consumer consumes the updated surface package.
*/
void getSurfacePackage(Consumer<SurfaceControlViewHost.SurfacePackage> consumer);
}
@NonNull
private final SurfaceHolder.Callback mSurfaceCallback = new SurfaceHolder.Callback() {
@Override
public void surfaceCreated(@NonNull SurfaceHolder holder) {
mSurfaceControlCallback.onCreated(mSurfaceView.getSurfaceControl());
}
@Override
public void surfaceChanged(@NonNull SurfaceHolder holder,
int format, int width, int height) {
public void surfaceChanged(@NonNull SurfaceHolder holder, int format, int width,
int height) {
/* do nothing */
}
@@ -90,13 +121,17 @@ public class InlineContentView extends ViewGroup {
}
};
private final @NonNull SurfaceView mSurfaceView;
@NonNull
private final SurfaceView mSurfaceView;
private @Nullable SurfaceControlCallback mSurfaceControlCallback;
@Nullable
private SurfaceControlCallback mSurfaceControlCallback;
@Nullable
private SurfacePackageUpdater mSurfacePackageUpdater;
/**
* @inheritDoc
*
* @hide
*/
public InlineContentView(@NonNull Context context) {
@@ -105,7 +140,6 @@ public class InlineContentView extends ViewGroup {
/**
* @inheritDoc
*
* @hide
*/
public InlineContentView(@NonNull Context context, @Nullable AttributeSet attrs) {
@@ -114,7 +148,6 @@ public class InlineContentView extends ViewGroup {
/**
* @inheritDoc
*
* @hide
*/
public InlineContentView(@NonNull Context context, @Nullable AttributeSet attrs,
@@ -123,20 +156,18 @@ public class InlineContentView extends ViewGroup {
}
/**
* Gets the surface control. If the surface is not created this method
* returns {@code null}.
* Gets the surface control. If the surface is not created this method returns {@code null}.
*
* @return The surface control.
*
* @see #setSurfaceControlCallback(SurfaceControlCallback)
*/
public @Nullable SurfaceControl getSurfaceControl() {
@Nullable
public SurfaceControl getSurfaceControl() {
return mSurfaceView.getSurfaceControl();
}
/**
* @inheritDoc
*
* @hide
*/
public InlineContentView(@NonNull Context context, @Nullable AttributeSet attrs,
@@ -149,14 +180,35 @@ public class InlineContentView extends ViewGroup {
}
/**
* Sets the embedded UI.
* @param surfacePackage The embedded UI.
* Sets the embedded UI provider.
*
* @hide
*/
public void setChildSurfacePackage(
@Nullable SurfaceControlViewHost.SurfacePackage surfacePackage) {
mSurfaceView.setChildSurfacePackage(surfacePackage);
public void setChildSurfacePackageUpdater(
@Nullable SurfacePackageUpdater surfacePackageUpdater) {
mSurfacePackageUpdater = surfacePackageUpdater;
}
@Override
protected void onAttachedToWindow() {
if (DEBUG) Log.v(TAG, "onAttachedToWindow");
super.onAttachedToWindow();
if (mSurfacePackageUpdater != null) {
mSurfacePackageUpdater.getSurfacePackage(
sp -> {
if (DEBUG) Log.v(TAG, "Received new SurfacePackage");
mSurfaceView.setChildSurfacePackage(sp);
});
}
}
@Override
protected void onDetachedFromWindow() {
if (DEBUG) Log.v(TAG, "onDetachedFromWindow");
super.onDetachedFromWindow();
if (mSurfacePackageUpdater != null) {
mSurfacePackageUpdater.onSurfacePackageReleased();
}
}
@Override
@@ -165,8 +217,8 @@ public class InlineContentView extends ViewGroup {
}
/**
* Sets a callback to observe the lifecycle of the surface control for
* managing the backing surface.
* Sets a callback to observe the lifecycle of the surface control for managing the backing
* surface.
*
* @param callback The callback to set or {@code null} to clear.
*/
@@ -182,7 +234,6 @@ public class InlineContentView extends ViewGroup {
/**
* @return Whether the surface backing this view appears on top of its parent.
*
* @see #setZOrderedOnTop(boolean)
*/
public boolean isZOrderedOnTop() {
@@ -190,17 +241,15 @@ public class InlineContentView extends ViewGroup {
}
/**
* Controls whether the backing surface is placed on top of this view's window.
* Normally, it is placed on top of the window, to allow interaction
* with the inlined UI. Via this method, you can place the surface below the
* window. This means that all of the contents of the window this view is in
* will be visible on top of its surface.
* Controls whether the backing surface is placed on top of this view's window. Normally, it is
* placed on top of the window, to allow interaction with the inlined UI. Via this method, you
* can place the surface below the window. This means that all of the contents of the window
* this view is in will be visible on top of its surface.
*
* <p> The Z ordering can be changed dynamically if the backing surface is
* created, otherwise the ordering would be applied at surface construction time.
*
* @param onTop Whether to show the surface on top of this view's window.
*
* @see #isZOrderedOnTop()
*/
public boolean setZOrderedOnTop(boolean onTop) {

2
core/java/com/android/internal/view/inline/IInlineContentProvider.aidl
View File

@@ -24,4 +24,6 @@ import com.android.internal.view.inline.IInlineContentCallback;
*/
oneway interface IInlineContentProvider {
void provideContent(int width, int height, in IInlineContentCallback callback);
void requestSurfacePackage();
void onSurfacePackageReleased();
}

4
services/autofill/java/com/android/server/autofill/AutofillInlineSuggestionsRequestSession.java
View File

@@ -37,6 +37,7 @@ import com.android.internal.annotations.GuardedBy;
import com.android.internal.view.IInlineSuggestionsRequestCallback;
import com.android.internal.view.IInlineSuggestionsResponseCallback;
import com.android.internal.view.InlineSuggestionsRequestInfo;
import com.android.server.autofill.ui.InlineSuggestionFactory;
import com.android.server.inputmethod.InputMethodManagerInternal;
import java.lang.ref.WeakReference;
@@ -242,7 +243,8 @@ final class AutofillInlineSuggestionsRequestSession {
}
if (sDebug) Log.d(TAG, "Send inline response: " + response.getInlineSuggestions().size());
try {
mResponseCallback.onInlineSuggestionsResponse(mAutofillId, response);
mResponseCallback.onInlineSuggestionsResponse(mAutofillId,
InlineSuggestionFactory.copy(response));
} catch (RemoteException e) {
Slog.e(TAG, "RemoteException sending InlineSuggestionsResponse to IME");
}

2
services/autofill/java/com/android/server/autofill/RemoteInlineSuggestionRenderService.java
View File

@@ -47,7 +47,7 @@ public final class RemoteInlineSuggestionRenderService extends
private static final String TAG = "RemoteInlineSuggestionRenderService";
private final int mIdleUnbindTimeoutMs = 5000;
private final long mIdleUnbindTimeoutMs = PERMANENT_BOUND_TIMEOUT_MS;
RemoteInlineSuggestionRenderService(Context context, ComponentName componentName,
String serviceInterface, int userId, InlineSuggestionRenderCallbacks callback,

140
services/autofill/java/com/android/server/autofill/ui/InlineContentProviderImpl.java Normal file
View File

@@ -0,0 +1,140 @@
/*
* Copyright (C) 2020 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.server.autofill.ui;
import static com.android.server.autofill.Helper.sVerbose;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.os.Handler;
import android.util.Slog;
import com.android.internal.view.inline.IInlineContentCallback;
import com.android.internal.view.inline.IInlineContentProvider;
import com.android.server.FgThread;
/**
* We create one instance of this class for each {@link android.view.inputmethod.InlineSuggestion}
* instance. Each inline suggestion instance will only be sent to the remote IME process once. In
* case of filtering and resending the suggestion when keyboard state changes between hide and
* show, a new instance of this class will be created using {@link #copy()}, with the same backing
* {@link RemoteInlineSuggestionUi}. When the
* {@link #provideContent(int, int, IInlineContentCallback)} is called the first time (it's only
* allowed to be called at most once), the passed in width/height is used to determine whether
* the existing {@link RemoteInlineSuggestionUi} provided in the constructor can be reused, or a
* new one should be created to suit the new size requirement for the view. In normal cases,
* we should not expect the size requirement to change, although in theory the public API allows
* the IME to do that.
*
* <p>This design is to enable us to be able to reuse the backing remote view while still keeping
* the callbacks relatively well aligned. For example, if we allow multiple remote IME binder
* callbacks to call into one instance of this class, then binder A may call in with width/height
* X for which we create a view (i.e. {@link RemoteInlineSuggestionUi}) for it,
*
* See also {@link RemoteInlineSuggestionUi} for relevant information.
*/
public final class InlineContentProviderImpl extends IInlineContentProvider.Stub {
// TODO(b/153615023): consider not holding strong reference to heavy objects in this stub, to
// avoid memory leak in case the client app is holding the remote reference for a longer
// time than expected. Essentially we need strong reference in the system process to
// the member variables, but weak reference to them in the IInlineContentProvider.Stub.
private static final String TAG = InlineContentProviderImpl.class.getSimpleName();
private final Handler mHandler = FgThread.getHandler();;
@NonNull
private final RemoteInlineSuggestionViewConnector mRemoteInlineSuggestionViewConnector;
@Nullable
private RemoteInlineSuggestionUi mRemoteInlineSuggestionUi;
private boolean mProvideContentCalled = false;
InlineContentProviderImpl(
@NonNull RemoteInlineSuggestionViewConnector remoteInlineSuggestionViewConnector,
@Nullable RemoteInlineSuggestionUi remoteInlineSuggestionUi) {
mRemoteInlineSuggestionViewConnector = remoteInlineSuggestionViewConnector;
mRemoteInlineSuggestionUi = remoteInlineSuggestionUi;
}
/**
* Returns a new instance of this class, with the same {@code mInlineSuggestionRenderer} and
* {@code mRemoteInlineSuggestionUi}. The latter may or may not be reusable depending on the
* size information provided when the client calls {@link #provideContent(int, int,
* IInlineContentCallback)}.
*/
@NonNull
public InlineContentProviderImpl copy() {
return new InlineContentProviderImpl(mRemoteInlineSuggestionViewConnector,
mRemoteInlineSuggestionUi);
}
/**
* Provides a SurfacePackage associated with the inline suggestion view to the IME. If such
* view doesn't exit, then create a new one. This method should be called once per lifecycle
* of this object. Any further calls to the method will be ignored.
*/
@Override
public void provideContent(int width, int height, IInlineContentCallback callback) {
mHandler.post(() -> handleProvideContent(width, height, callback));
}
@Override
public void requestSurfacePackage() {
mHandler.post(this::handleGetSurfacePackage);
}
@Override
public void onSurfacePackageReleased() {
mHandler.post(this::handleOnSurfacePackageReleased);
}
private void handleProvideContent(int width, int height, IInlineContentCallback callback) {
if (sVerbose) Slog.v(TAG, "handleProvideContent");
if (mProvideContentCalled) {
// This method should only be called once.
return;
}
mProvideContentCalled = true;
if (mRemoteInlineSuggestionUi == null || !mRemoteInlineSuggestionUi.match(width, height)) {
mRemoteInlineSuggestionUi = new RemoteInlineSuggestionUi(
mRemoteInlineSuggestionViewConnector,
width, height, mHandler);
}
mRemoteInlineSuggestionUi.setInlineContentCallback(callback);
mRemoteInlineSuggestionUi.requestSurfacePackage();
}
private void handleGetSurfacePackage() {
if (sVerbose) Slog.v(TAG, "handleGetSurfacePackage");
if (!mProvideContentCalled || mRemoteInlineSuggestionUi == null) {
// provideContent should be called first, and remote UI should not be null.
return;
}
mRemoteInlineSuggestionUi.requestSurfacePackage();
}
private void handleOnSurfacePackageReleased() {
if (sVerbose) Slog.v(TAG, "handleOnSurfacePackageReleased");
if (!mProvideContentCalled || mRemoteInlineSuggestionUi == null) {
// provideContent should be called first, and remote UI should not be null.
return;
}
mRemoteInlineSuggestionUi.surfacePackageReleased();
}
}

102
services/autofill/java/com/android/server/autofill/ui/InlineSuggestionFactory.java
View File

@@ -24,14 +24,11 @@ import android.annotation.Nullable;
import android.content.Intent;
import android.content.IntentSender;
import android.os.IBinder;
import android.os.RemoteException;
import android.service.autofill.Dataset;
import android.service.autofill.FillResponse;
import android.service.autofill.IInlineSuggestionUiCallback;
import android.service.autofill.InlinePresentation;
import android.text.TextUtils;
import android.util.Slog;
import android.view.SurfaceControlViewHost;
import android.view.autofill.AutofillId;
import android.view.autofill.AutofillManager;
import android.view.autofill.AutofillValue;
@@ -41,12 +38,8 @@ import android.view.inputmethod.InlineSuggestionsRequest;
import android.view.inputmethod.InlineSuggestionsResponse;
import android.widget.inline.InlinePresentationSpec;
import com.android.internal.view.inline.IInlineContentCallback;
import com.android.internal.view.inline.IInlineContentProvider;
import com.android.server.LocalServices;
import com.android.server.UiThread;
import com.android.server.autofill.RemoteInlineSuggestionRenderService;
import com.android.server.inputmethod.InputMethodManagerInternal;
import java.util.ArrayList;
import java.util.List;
@@ -72,6 +65,27 @@ public final class InlineSuggestionFactory {
void startIntentSender(@NonNull IntentSender intentSender, @NonNull Intent intent);
}
/**
* Returns a copy of the response, that internally copies the {@link IInlineContentProvider}
* so that it's not reused by the remote IME process across different inline suggestions.
* See {@link InlineContentProviderImpl} for why this is needed.
*/
@NonNull
public static InlineSuggestionsResponse copy(@NonNull InlineSuggestionsResponse response) {
final ArrayList<InlineSuggestion> copiedInlineSuggestions = new ArrayList<>();
for (InlineSuggestion inlineSuggestion : response.getInlineSuggestions()) {
final IInlineContentProvider contentProvider = inlineSuggestion.getContentProvider();
if (contentProvider instanceof InlineContentProviderImpl) {
copiedInlineSuggestions.add(new
InlineSuggestion(inlineSuggestion.getInfo(),
((InlineContentProviderImpl) contentProvider).copy()));
} else {
copiedInlineSuggestions.add(inlineSuggestion);
}
}
return new InlineSuggestionsResponse(copiedInlineSuggestions);
}
/**
* Creates an {@link InlineSuggestionsResponse} with the {@code datasets} provided by the
* autofill service, potentially filtering the datasets.
@@ -276,78 +290,20 @@ public final class InlineSuggestionFactory {
inlinePresentation.isPinned());
}
private static IInlineContentProvider.Stub createInlineContentProvider(
private static IInlineContentProvider createInlineContentProvider(
@NonNull InlinePresentation inlinePresentation, @Nullable Runnable onClickAction,
@NonNull Runnable onErrorCallback,
@NonNull Consumer<IntentSender> intentSenderConsumer,
@Nullable RemoteInlineSuggestionRenderService remoteRenderService,
@Nullable IBinder hostInputToken,
int displayId) {
return new IInlineContentProvider.Stub() {
@Override
public void provideContent(int width, int height, IInlineContentCallback callback) {
UiThread.getHandler().post(() -> {
final IInlineSuggestionUiCallback uiCallback = createInlineSuggestionUiCallback(
callback, onClickAction, onErrorCallback, intentSenderConsumer);
if (remoteRenderService == null) {
Slog.e(TAG, "RemoteInlineSuggestionRenderService is null");
return;
}
remoteRenderService.renderSuggestion(uiCallback, inlinePresentation,
width, height, hostInputToken, displayId);
});
}
};
}
private static IInlineSuggestionUiCallback.Stub createInlineSuggestionUiCallback(
@NonNull IInlineContentCallback callback, @NonNull Runnable onAutofillCallback,
@NonNull Runnable onErrorCallback,
@NonNull Consumer<IntentSender> intentSenderConsumer) {
return new IInlineSuggestionUiCallback.Stub() {
@Override
public void onClick() throws RemoteException {
onAutofillCallback.run();
callback.onClick();
}
@Override
public void onLongClick() throws RemoteException {
callback.onLongClick();
}
@Override
public void onContent(SurfaceControlViewHost.SurfacePackage surface, int width,
int height)
throws RemoteException {
callback.onContent(surface, width, height);
surface.release();
}
@Override
public void onError() throws RemoteException {
onErrorCallback.run();
}
@Override
public void onTransferTouchFocusToImeWindow(IBinder sourceInputToken, int displayId)
throws RemoteException {
final InputMethodManagerInternal inputMethodManagerInternal =
LocalServices.getService(InputMethodManagerInternal.class);
if (!inputMethodManagerInternal.transferTouchFocusToImeWindow(sourceInputToken,
displayId)) {
Slog.e(TAG, "Cannot transfer touch focus from suggestion to IME");
onErrorCallback.run();
}
}
@Override
public void onStartIntentSender(IntentSender intentSender) {
intentSenderConsumer.accept(intentSender);
}
};
RemoteInlineSuggestionViewConnector
remoteInlineSuggestionViewConnector = new RemoteInlineSuggestionViewConnector(
remoteRenderService, inlinePresentation, hostInputToken, displayId, onClickAction,
onErrorCallback, intentSenderConsumer);
InlineContentProviderImpl inlineContentProvider = new InlineContentProviderImpl(
remoteInlineSuggestionViewConnector, null);
return inlineContentProvider;
}
private InlineSuggestionFactory() {

291
services/autofill/java/com/android/server/autofill/ui/RemoteInlineSuggestionUi.java Normal file
View File

@@ -0,0 +1,291 @@
/*
* Copyright (C) 2020 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.server.autofill.ui;
import static com.android.server.autofill.Helper.sDebug;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.content.IntentSender;
import android.os.Handler;
import android.os.IBinder;
import android.os.RemoteException;
import android.service.autofill.IInlineSuggestionUi;
import android.service.autofill.IInlineSuggestionUiCallback;
import android.service.autofill.ISurfacePackageResultCallback;
import android.util.Slog;
import android.view.SurfaceControlViewHost;
import com.android.internal.view.inline.IInlineContentCallback;
/**
* The instance of this class lives in the system server, orchestrating the communication between
* the remote process owning embedded view (i.e. ExtServices) and the remote process hosting the
* embedded view (i.e. IME). It's also responsible for releasing the embedded view from the owning
* process when it's not longer needed in the hosting process.
*
* <p>An instance of this class may be reused to associate with multiple instances of
* {@link InlineContentProviderImpl}s, each of which wraps a callback from the IME. But at any
* given time, there is only one active IME callback which this class will callback into.
*
* <p>This class is thread safe, because all the outside calls are piped into the same single
* thread handler to be processed.
*
* TODO(b/154683107): implement the reference counting in case there are multiple active
* SurfacePackages at the same time. This will not happen for now since all the InlineSuggestions
* sharing the same UI will be sent to the same IME window, so the previous view will be detached
* before the new view are attached to the window.
*/
final class RemoteInlineSuggestionUi {
private static final String TAG = RemoteInlineSuggestionUi.class.getSimpleName();
// The delay time to release the remote inline suggestion view (in the renderer
// process) after receiving a signal about the surface package being released due to being
// detached from the window in the host app (in the IME process). The release will be
// canceled if the host app reattaches the view to a window within this delay time.
// TODO(b/154683107): try out using the Chroreographer to schedule the release right at the
// next frame. Basically if the view is not re-attached to the window immediately in the next
// frame after it was detached, then it will be released.
private static final long RELEASE_REMOTE_VIEW_HOST_DELAY_MS = 200;
@NonNull
private final Handler mHandler;
@NonNull
private final RemoteInlineSuggestionViewConnector mRemoteInlineSuggestionViewConnector;
private final int mWidth;
private final int mHeight;
@NonNull
private final InlineSuggestionUiCallbackImpl mInlineSuggestionUiCallback;
@Nullable
private IInlineContentCallback mInlineContentCallback; // from IME
/**
* Remote inline suggestion view, backed by an instance of {@link SurfaceControlViewHost} in
* the render service process. We takes care of releasing it when there is no remote
* reference to it (from IME), and we will create a new instance of the view when it's needed
* by IME again.
*/
@Nullable
private IInlineSuggestionUi mInlineSuggestionUi;
private boolean mWaitingForUiCreation = false;
private int mActualWidth;
private int mActualHeight;
@Nullable
private Runnable mDelayedReleaseViewRunnable;
RemoteInlineSuggestionUi(
@NonNull RemoteInlineSuggestionViewConnector remoteInlineSuggestionViewConnector,
int width, int height, Handler handler) {
mHandler = handler;
mRemoteInlineSuggestionViewConnector = remoteInlineSuggestionViewConnector;
mWidth = width;
mHeight = height;
mInlineSuggestionUiCallback = new InlineSuggestionUiCallbackImpl();
}
/**
* Updates the callback from the IME process. It'll swap out the previous IME callback, and
* all the subsequent callback events (onClick, onLongClick, touch event transfer, etc) will
* be directed to the new callback.
*/
void setInlineContentCallback(@NonNull IInlineContentCallback inlineContentCallback) {
mHandler.post(() -> {
mInlineContentCallback = inlineContentCallback;
});
}
/**
* Handles the request from the IME process to get a new surface package. May create a new
* view in the renderer process if the existing view is already released.
*/
void requestSurfacePackage() {
mHandler.post(this::handleRequestSurfacePackage);
}
/**
* Handles the signal from the IME process that the previously sent surface package has been
* released.
*/
void surfacePackageReleased() {
mHandler.post(this::handleSurfacePackageReleased);
}
/**
* Returns true if the provided size matches the remote view's size.
*/
boolean match(int width, int height) {
return mWidth == width && mHeight == height;
}
private void handleSurfacePackageReleased() {
cancelPendingReleaseViewRequest();
// Schedule a delayed release view request
mDelayedReleaseViewRunnable = () -> {
if (mInlineSuggestionUi != null) {
try {
mInlineSuggestionUi.releaseSurfaceControlViewHost();
mInlineSuggestionUi = null;
} catch (RemoteException e) {
Slog.w(TAG, "RemoteException calling releaseSurfaceControlViewHost");
}
}
mDelayedReleaseViewRunnable = null;
};
mHandler.postDelayed(mDelayedReleaseViewRunnable, RELEASE_REMOTE_VIEW_HOST_DELAY_MS);
}
private void handleRequestSurfacePackage() {
cancelPendingReleaseViewRequest();
if (mInlineSuggestionUi == null) {
if (mWaitingForUiCreation) {
// This could happen in the following case: the remote embedded view was released
// when previously detached from window. An event after that to re-attached to
// the window will cause us calling the renderSuggestion again. Now, before the
// render call returns a new surface package, if the view is detached and
// re-attached to the window, causing this method to be called again, we will get
// to this state. This request will be ignored and the surface package will still
// be sent back once the view is rendered.
if (sDebug) Slog.d(TAG, "Inline suggestion ui is not ready");
} else {
mRemoteInlineSuggestionViewConnector.renderSuggestion(mWidth, mHeight,
mInlineSuggestionUiCallback);
mWaitingForUiCreation = true;
}
} else {
try {
mInlineSuggestionUi.getSurfacePackage(new ISurfacePackageResultCallback.Stub() {
@Override
public void onResult(SurfaceControlViewHost.SurfacePackage result)
throws RemoteException {
if (sDebug) Slog.d(TAG, "Sending new SurfacePackage to IME");
mInlineContentCallback.onContent(result, mActualWidth, mActualHeight);
}
});
} catch (RemoteException e) {
Slog.w(TAG, "RemoteException calling getSurfacePackage.");
}
}
}
private void cancelPendingReleaseViewRequest() {
if (mDelayedReleaseViewRunnable != null) {
mHandler.removeCallbacks(mDelayedReleaseViewRunnable);
mDelayedReleaseViewRunnable = null;
}
}
/**
* This is called when a new inline suggestion UI is inflated from the ext services.
*/
private void handleInlineSuggestionUiReady(IInlineSuggestionUi content,
SurfaceControlViewHost.SurfacePackage surfacePackage, int width, int height) {
mInlineSuggestionUi = content;
mWaitingForUiCreation = false;
mActualWidth = width;
mActualHeight = height;
if (mInlineContentCallback != null) {
try {
mInlineContentCallback.onContent(surfacePackage, mActualWidth, mActualHeight);
} catch (RemoteException e) {
Slog.w(TAG, "RemoteException calling onContent");
}
}
if (surfacePackage != null) {
surfacePackage.release();
}
}
private void handleOnClick() {
// Autofill the value
mRemoteInlineSuggestionViewConnector.onClick();
// Notify the remote process (IME) that hosts the embedded UI that it's clicked
if (mInlineContentCallback != null) {
try {
mInlineContentCallback.onClick();
} catch (RemoteException e) {
Slog.w(TAG, "RemoteException calling onClick");
}
}
}
private void handleOnLongClick() {
// Notify the remote process (IME) that hosts the embedded UI that it's long clicked
if (mInlineContentCallback != null) {
try {
mInlineContentCallback.onLongClick();
} catch (RemoteException e) {
Slog.w(TAG, "RemoteException calling onLongClick");
}
}
}
private void handleOnError() {
mRemoteInlineSuggestionViewConnector.onError();
}
private void handleOnTransferTouchFocusToImeWindow(IBinder sourceInputToken, int displayId) {
mRemoteInlineSuggestionViewConnector.onTransferTouchFocusToImeWindow(sourceInputToken,
displayId);
}
private void handleOnStartIntentSender(IntentSender intentSender) {
mRemoteInlineSuggestionViewConnector.onStartIntentSender(intentSender);
}
/**
* Responsible for communicating with the inline suggestion view owning process.
*/
private class InlineSuggestionUiCallbackImpl extends IInlineSuggestionUiCallback.Stub {
@Override
public void onClick() {
mHandler.post(RemoteInlineSuggestionUi.this::handleOnClick);
}
@Override
public void onLongClick() {
mHandler.post(RemoteInlineSuggestionUi.this::handleOnLongClick);
}
@Override
public void onContent(IInlineSuggestionUi content,
SurfaceControlViewHost.SurfacePackage surface, int width, int height) {
mHandler.post(() -> handleInlineSuggestionUiReady(content, surface, width, height));
}
@Override
public void onError() {
mHandler.post(RemoteInlineSuggestionUi.this::handleOnError);
}
@Override
public void onTransferTouchFocusToImeWindow(IBinder sourceInputToken, int displayId) {
mHandler.post(() -> handleOnTransferTouchFocusToImeWindow(sourceInputToken, displayId));
}
@Override
public void onStartIntentSender(IntentSender intentSender) {
mHandler.post(() -> handleOnStartIntentSender(intentSender));
}
}
}

125
services/autofill/java/com/android/server/autofill/ui/RemoteInlineSuggestionViewConnector.java Normal file
View File

@@ -0,0 +1,125 @@
/*
* Copyright (C) 2020 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.server.autofill.ui;
import static com.android.server.autofill.Helper.sDebug;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.content.IntentSender;
import android.os.IBinder;
import android.service.autofill.IInlineSuggestionUiCallback;
import android.service.autofill.InlinePresentation;
import android.util.Slog;
import com.android.server.LocalServices;
import com.android.server.autofill.RemoteInlineSuggestionRenderService;
import com.android.server.inputmethod.InputMethodManagerInternal;
import java.util.function.Consumer;
/**
* Wraps the parameters needed to create a new inline suggestion view in the remote renderer
* service, and handles the callback from the events on the created remote view.
*/
final class RemoteInlineSuggestionViewConnector {
private static final String TAG = RemoteInlineSuggestionViewConnector.class.getSimpleName();
@Nullable
private final RemoteInlineSuggestionRenderService mRemoteRenderService;
@NonNull
private final InlinePresentation mInlinePresentation;
@Nullable
private final IBinder mHostInputToken;
private final int mDisplayId;
@NonNull
private final Runnable mOnAutofillCallback;
@NonNull
private final Runnable mOnErrorCallback;
@NonNull
private final Consumer<IntentSender> mStartIntentSenderFromClientApp;
RemoteInlineSuggestionViewConnector(
@Nullable RemoteInlineSuggestionRenderService remoteRenderService,
@NonNull InlinePresentation inlinePresentation,
@Nullable IBinder hostInputToken,
int displayId,
@NonNull Runnable onAutofillCallback,
@NonNull Runnable onErrorCallback,
@NonNull Consumer<IntentSender> startIntentSenderFromClientApp) {
mRemoteRenderService = remoteRenderService;
mInlinePresentation = inlinePresentation;
mHostInputToken = hostInputToken;
mDisplayId = displayId;
mOnAutofillCallback = onAutofillCallback;
mOnErrorCallback = onErrorCallback;
mStartIntentSenderFromClientApp = startIntentSenderFromClientApp;
}
/**
* Calls the remote renderer service to create a new inline suggestion view.
*
* @return true if the call is made to the remote renderer service, false otherwise.
*/
public boolean renderSuggestion(int width, int height,
@NonNull IInlineSuggestionUiCallback callback) {
if (mRemoteRenderService != null) {
if (sDebug) Slog.d(TAG, "Request to recreate the UI");
mRemoteRenderService.renderSuggestion(callback, mInlinePresentation, width, height,
mHostInputToken, mDisplayId);
return true;
}
return false;
}
/**
* Handles the callback for the event of remote view being clicked.
*/
public void onClick() {
mOnAutofillCallback.run();
}
/**
* Handles the callback for the remote error when creating or interacting with the view.
*/
public void onError() {
mOnErrorCallback.run();
}
/**
* Handles the callback for transferring the touch event on the remote view to the IME
* process.
*/
public void onTransferTouchFocusToImeWindow(IBinder sourceInputToken, int displayId) {
final InputMethodManagerInternal inputMethodManagerInternal =
LocalServices.getService(InputMethodManagerInternal.class);
if (!inputMethodManagerInternal.transferTouchFocusToImeWindow(sourceInputToken,
displayId)) {
Slog.e(TAG, "Cannot transfer touch focus from suggestion to IME");
mOnErrorCallback.run();
}
}
/**
* Handles starting an intent sender from the client app's process.
*/
public void onStartIntentSender(IntentSender intentSender) {
mStartIntentSenderFromClientApp.accept(intentSender);
}
}