Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move rendering logic for inline suggestions to ExtServices.

Browse Source 
Added a render service in ExtServices to connect to the renderer in
androidx.autofill for inline suggestion slices.

Cleaned up old UI rendering code that lived in system_server.

Bug: 146453086
Test: atest ExtServicesUnitTests
Change-Id: I25a7ea438afe524683671c850625ae80dacccfaa
This commit is contained in:
Adam He
2020-02-04 14:55:39 -08:00
parent 3eed349051
commit 4011625638
15 changed files with 309 additions and 371 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
api/system-current.txt
View File

@@ -14063,6 +14063,14 @@ package android.view.contentcapture {
}
package android.view.inline {
public final class InlinePresentationSpec implements android.os.Parcelable {
method @Nullable public String getStyle();
}
}
package android.webkit {
public abstract class CookieManager {

13
core/java/android/inputmethodservice/InputMethodService.java
View File

@@ -789,6 +789,7 @@ public class InputMethodService extends AbstractInputMethodService {
Log.w(TAG, "onCreateInlineSuggestionsRequest() returned null request");
requestCallback.onInlineSuggestionsUnsupported();
} else {
request.setHostInputToken(getHostInputToken());
final IInlineSuggestionsResponseCallback inlineSuggestionsResponseCallback =
new InlineSuggestionsResponseCallbackImpl(this,
mInlineSuggestionsRequestInfo.mComponentName,
@@ -833,6 +834,18 @@ public class InputMethodService extends AbstractInputMethodService {
onInlineSuggestionsResponse(response);
}
/**
* Returns the {@link IBinder} input token from the host view root.
*/
@Nullable
private IBinder getHostInputToken() {
ViewRootImpl viewRoot = null;
if (mRootView != null) {
viewRoot = mRootView.getViewRootImpl();
}
return viewRoot == null ? null : viewRoot.getInputToken();
}
private void notifyImeHidden() {
setImeWindowStatus(IME_ACTIVE | IME_INVISIBLE, mBackDisposition);
onPreRenderedWindowVisibilityChanged(false /* setVisible */);

6
core/java/android/service/autofill/IInlineSuggestionRenderService.aidl
View File

@@ -16,6 +16,7 @@
package android.service.autofill;
import android.os.IBinder;
import android.service.autofill.IInlineSuggestionUiCallback;
import android.service.autofill.InlinePresentation;
@@ -25,6 +26,7 @@ import android.service.autofill.InlinePresentation;
* @hide
*/
oneway interface IInlineSuggestionRenderService {
void renderSuggestion(in IInlineSuggestionUiCallback callback, in InlinePresentation presentation,
int width, int height);
void renderSuggestion(in IInlineSuggestionUiCallback callback,
in InlinePresentation presentation, int width, int height,
in IBinder hostInputToken);
}

5
core/java/android/service/autofill/IInlineSuggestionUiCallback.aidl
View File

@@ -16,6 +16,7 @@
package android.service.autofill;
import android.os.IBinder;
import android.view.SurfaceControl;
/**
@@ -24,6 +25,8 @@ import android.view.SurfaceControl;
* @hide
*/
oneway interface IInlineSuggestionUiCallback {
void autofill();
void onAutofill();
void onContent(in SurfaceControl surface);
void onError();
void onTransferTouchFocusToImeWindow(in IBinder sourceInputToken, int displayId);
}

46
core/java/android/service/autofill/InlineSuggestionRenderService.java
View File

@@ -24,11 +24,16 @@ import android.annotation.TestApi;
import android.app.Service;
import android.app.slice.Slice;
import android.content.Intent;
import android.graphics.PixelFormat;
import android.os.Handler;
import android.os.IBinder;
import android.os.Looper;
import android.os.RemoteException;
import android.util.Log;
import android.view.SurfaceControl;
import android.view.SurfaceControlViewHost;
import android.view.View;
import android.view.WindowManager;
/**
* A service that renders an inline presentation given the {@link InlinePresentation} containing
@@ -55,8 +60,40 @@ public abstract class InlineSuggestionRenderService extends Service {
private final Handler mHandler = new Handler(Looper.getMainLooper(), null, true);
private void handleRenderSuggestion(IInlineSuggestionUiCallback callback,
InlinePresentation presentation, int width, int height) {
//TODO(b/146453086): implementation in ExtService
InlinePresentation presentation, int width, int height, IBinder hostInputToken) {
if (hostInputToken == null) {
try {
callback.onError();
} catch (RemoteException e) {
Log.w(TAG, "RemoteException calling onError()");
}
return;
}
final SurfaceControlViewHost host = new SurfaceControlViewHost(this, this.getDisplay(),
hostInputToken);
final SurfaceControl surface = host.getSurfacePackage().getSurfaceControl();
final View suggestionView = onRenderSuggestion(presentation, width, height);
final InlineSuggestionRoot suggestionRoot = new InlineSuggestionRoot(this, callback);
suggestionRoot.addView(suggestionView);
suggestionRoot.setOnClickListener((v) -> {
try {
callback.onAutofill();
} catch (RemoteException e) {
Log.w(TAG, "RemoteException calling onAutofill()");
}
});
WindowManager.LayoutParams lp =
new WindowManager.LayoutParams(width, height,
WindowManager.LayoutParams.TYPE_APPLICATION, 0, PixelFormat.TRANSPARENT);
host.addView(suggestionRoot, lp);
try {
callback.onContent(surface);
} catch (RemoteException e) {
Log.w(TAG, "RemoteException calling onContent(" + surface + ")");
}
}
@Override
@@ -66,11 +103,12 @@ public abstract class InlineSuggestionRenderService extends Service {
return new IInlineSuggestionRenderService.Stub() {
@Override
public void renderSuggestion(@NonNull IInlineSuggestionUiCallback callback,
@NonNull InlinePresentation presentation, int width, int height) {
@NonNull InlinePresentation presentation, int width, int height,
@Nullable IBinder hostInputToken) {
mHandler.sendMessage(obtainMessage(
InlineSuggestionRenderService::handleRenderSuggestion,
InlineSuggestionRenderService.this, callback, presentation,
width, height));
width, height, hostInputToken));
}
}.asBinder();
}

37
services/autofill/java/com/android/server/autofill/ui/InlineSuggestionRoot.java → core/java/android/service/autofill/InlineSuggestionRoot.java
View File

@@ -14,39 +14,39 @@
* limitations under the License.
*/
package com.android.server.autofill.ui;
package android.service.autofill;
import android.annotation.NonNull;
import android.annotation.SuppressLint;
import android.content.Context;
import android.os.RemoteException;
import android.util.Log;
import android.util.MathUtils;
import android.view.MotionEvent;
import android.view.ViewConfiguration;
import android.widget.FrameLayout;
import com.android.server.LocalServices;
import com.android.server.wm.WindowManagerInternal;
/**
* This class is the root view for an inline suggestion. It is responsible for
* detecting the click on the item and to also transfer input focus to the IME
* window if we detect the user is scrolling.
*
* @hide
*/
// TODO(b/146453086) Move to ExtServices and add @SystemApi to transfer touch focus
@SuppressLint("ViewConstructor")
class InlineSuggestionRoot extends FrameLayout {
private static final String LOG_TAG = InlineSuggestionRoot.class.getSimpleName();
public class InlineSuggestionRoot extends FrameLayout {
private static final String TAG = "InlineSuggestionRoot";
private final @NonNull Runnable mOnErrorCallback;
private final @NonNull IInlineSuggestionUiCallback mCallback;
private final int mTouchSlop;
private float mDownX;
private float mDownY;
InlineSuggestionRoot(@NonNull Context context, @NonNull Runnable onErrorCallback) {
public InlineSuggestionRoot(@NonNull Context context,
@NonNull IInlineSuggestionUiCallback callback) {
super(context);
mOnErrorCallback = onErrorCallback;
mCallback = callback;
mTouchSlop = ViewConfiguration.get(context).getScaledTouchSlop();
setFocusable(false);
}
@@ -64,20 +64,15 @@ class InlineSuggestionRoot extends FrameLayout {
final float distance = MathUtils.dist(mDownX, mDownY,
event.getX(), event.getY());
if (distance > mTouchSlop) {
transferTouchFocusToImeWindow();
try {
mCallback.onTransferTouchFocusToImeWindow(getViewRootImpl().getInputToken(),
getContext().getDisplayId());
} catch (RemoteException e) {
Log.w(TAG, "RemoteException transferring touch focus to IME");
}
}
} break;
}
return super.onTouchEvent(event);
}
private void transferTouchFocusToImeWindow() {
final WindowManagerInternal windowManagerInternal = LocalServices.getService(
WindowManagerInternal.class);
if (!windowManagerInternal.transferTouchFocusToImeWindow(getViewRootImpl().getInputToken(),
getContext().getDisplayId())) {
Log.e(LOG_TAG, "Cannot transfer touch focus from suggestion to IME");
mOnErrorCallback.run();
}
}
}

6
core/java/android/view/inline/InlinePresentationSpec.java
View File

@@ -18,6 +18,7 @@ package android.view.inline;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.annotation.SystemApi;
import android.os.Parcelable;
import android.util.Size;
@@ -55,6 +56,7 @@ public final class InlinePresentationSpec implements Parcelable {
/**
* @hide
*/
@SystemApi
public @Nullable String getStyle() {
return mStyle;
}
@@ -281,10 +283,10 @@ public final class InlinePresentationSpec implements Parcelable {
}
@DataClass.Generated(
time = 1577145109444L,
time = 1581117017522L,
codegenVersion = "1.0.14",
sourceFile = "frameworks/base/core/java/android/view/inline/InlinePresentationSpec.java",
inputSignatures = "private final @android.annotation.NonNull android.util.Size mMinSize\nprivate final @android.annotation.NonNull android.util.Size mMaxSize\nprivate final @android.annotation.Nullable java.lang.String mStyle\nprivate static java.lang.String defaultStyle()\npublic @android.annotation.Nullable java.lang.String getStyle()\nclass InlinePresentationSpec extends java.lang.Object implements [android.os.Parcelable]\n@com.android.internal.util.DataClass(genEqualsHashCode=true, genToString=true, genBuilder=true)\nclass BaseBuilder extends java.lang.Object implements []")
inputSignatures = "private final @android.annotation.NonNull android.util.Size mMinSize\nprivate final @android.annotation.NonNull android.util.Size mMaxSize\nprivate final @android.annotation.Nullable java.lang.String mStyle\nprivate static java.lang.String defaultStyle()\npublic @android.annotation.SystemApi @android.annotation.Nullable java.lang.String getStyle()\nclass InlinePresentationSpec extends java.lang.Object implements [android.os.Parcelable]\n@com.android.internal.util.DataClass(genEqualsHashCode=true, genToString=true, genBuilder=true)\nclass BaseBuilder extends java.lang.Object implements []")
@Deprecated
private void __metadata() {}

2
core/java/android/view/inputmethod/InlineSuggestion.java
View File

@@ -286,7 +286,7 @@ public final class InlineSuggestion implements Parcelable {
};
@DataClass.Generated(
time = 1578972138081L,
time = 1581377984320L,
codegenVersion = "1.0.14",
sourceFile = "frameworks/base/core/java/android/view/inputmethod/InlineSuggestion.java",
inputSignatures = "private static final java.lang.String TAG\nprivate final @android.annotation.NonNull android.view.inputmethod.InlineSuggestionInfo mInfo\nprivate final @android.annotation.Nullable com.android.internal.view.inline.IInlineContentProvider mContentProvider\npublic static @android.annotation.TestApi @android.annotation.NonNull android.view.inputmethod.InlineSuggestion newInlineSuggestion(android.view.inputmethod.InlineSuggestionInfo)\npublic void inflate(android.content.Context,android.util.Size,java.util.concurrent.Executor,java.util.function.Consumer<android.view.View>)\nclass InlineSuggestion extends java.lang.Object implements [android.os.Parcelable]\n@com.android.internal.util.DataClass(genEqualsHashCode=true, genToString=true, genHiddenConstDefs=true, genHiddenConstructor=true)")

82
core/java/android/view/inputmethod/InlineSuggestionsRequest.java
View File

@@ -19,6 +19,7 @@ package android.view.inputmethod;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.app.ActivityThread;
import android.os.IBinder;
import android.os.Parcelable;
import android.view.inline.InlinePresentationSpec;
@@ -58,6 +59,14 @@ public final class InlineSuggestionsRequest implements Parcelable {
*/
private @NonNull String mHostPackageName;
/**
* The host input token of the IME that made the request. This will be set by the system for
* safety reasons.
*
* @hide
*/
private @Nullable IBinder mHostInputToken;
/**
* @hide
* @see {@link #mHostPackageName}.
@@ -66,6 +75,14 @@ public final class InlineSuggestionsRequest implements Parcelable {
mHostPackageName = hostPackageName;
}
/**
* @hide
* @see {@link #mHostInputToken}.
*/
public void setHostInputToken(IBinder hostInputToken) {
mHostInputToken = hostInputToken;
}
private void onConstructed() {
Preconditions.checkState(mMaxSuggestionCount >= mPresentationSpecs.size());
}
@@ -78,11 +95,17 @@ public final class InlineSuggestionsRequest implements Parcelable {
return ActivityThread.currentPackageName();
}
private static IBinder defaultHostInputToken() {
return null;
}
/** @hide */
abstract static class BaseBuilder {
abstract Builder setPresentationSpecs(@NonNull List<InlinePresentationSpec> value);
abstract Builder setHostPackageName(@Nullable String value);
abstract Builder setHostInputToken(IBinder hostInputToken);
}
@@ -104,7 +127,8 @@ public final class InlineSuggestionsRequest implements Parcelable {
/* package-private */ InlineSuggestionsRequest(
int maxSuggestionCount,
@NonNull List<InlinePresentationSpec> presentationSpecs,
@NonNull String hostPackageName) {
@NonNull String hostPackageName,
@Nullable IBinder hostInputToken) {
this.mMaxSuggestionCount = maxSuggestionCount;
this.mPresentationSpecs = presentationSpecs;
com.android.internal.util.AnnotationValidations.validate(
@@ -112,6 +136,7 @@ public final class InlineSuggestionsRequest implements Parcelable {
this.mHostPackageName = hostPackageName;
com.android.internal.util.AnnotationValidations.validate(
NonNull.class, null, mHostPackageName);
this.mHostInputToken = hostInputToken;
onConstructed();
}
@@ -145,6 +170,17 @@ public final class InlineSuggestionsRequest implements Parcelable {
return mHostPackageName;
}
/**
* The host input token of the IME that made the request. This will be set by the system for
* safety reasons.
*
* @hide
*/
@DataClass.Generated.Member
public @Nullable IBinder getHostInputToken() {
return mHostInputToken;
}
@Override
@DataClass.Generated.Member
public String toString() {
@@ -154,7 +190,8 @@ public final class InlineSuggestionsRequest implements Parcelable {
return "InlineSuggestionsRequest { " +
"maxSuggestionCount = " + mMaxSuggestionCount + ", " +
"presentationSpecs = " + mPresentationSpecs + ", " +
"hostPackageName = " + mHostPackageName +
"hostPackageName = " + mHostPackageName + ", " +
"hostInputToken = " + mHostInputToken +
" }";
}
@@ -173,7 +210,8 @@ public final class InlineSuggestionsRequest implements Parcelable {
return true
&& mMaxSuggestionCount == that.mMaxSuggestionCount
&& java.util.Objects.equals(mPresentationSpecs, that.mPresentationSpecs)
&& java.util.Objects.equals(mHostPackageName, that.mHostPackageName);
&& java.util.Objects.equals(mHostPackageName, that.mHostPackageName)
&& java.util.Objects.equals(mHostInputToken, that.mHostInputToken);
}
@Override
@@ -186,6 +224,7 @@ public final class InlineSuggestionsRequest implements Parcelable {
_hash = 31 * _hash + mMaxSuggestionCount;
_hash = 31 * _hash + java.util.Objects.hashCode(mPresentationSpecs);
_hash = 31 * _hash + java.util.Objects.hashCode(mHostPackageName);
_hash = 31 * _hash + java.util.Objects.hashCode(mHostInputToken);
return _hash;
}
@@ -195,9 +234,13 @@ public final class InlineSuggestionsRequest implements Parcelable {
// You can override field parcelling by defining methods like:
// void parcelFieldName(Parcel dest, int flags) { ... }
byte flg = 0;
if (mHostInputToken != null) flg |= 0x8;
dest.writeByte(flg);
dest.writeInt(mMaxSuggestionCount);
dest.writeParcelableList(mPresentationSpecs, flags);
dest.writeString(mHostPackageName);
if (mHostInputToken != null) dest.writeStrongBinder(mHostInputToken);
}
@Override
@@ -211,10 +254,12 @@ public final class InlineSuggestionsRequest implements Parcelable {
// You can override field unparcelling by defining methods like:
// static FieldType unparcelFieldName(Parcel in) { ... }
byte flg = in.readByte();
int maxSuggestionCount = in.readInt();
List<InlinePresentationSpec> presentationSpecs = new ArrayList<>();
in.readParcelableList(presentationSpecs, InlinePresentationSpec.class.getClassLoader());
String hostPackageName = in.readString();
IBinder hostInputToken = (flg & 0x8) == 0 ? null : in.readStrongBinder();
this.mMaxSuggestionCount = maxSuggestionCount;
this.mPresentationSpecs = presentationSpecs;
@@ -223,6 +268,7 @@ public final class InlineSuggestionsRequest implements Parcelable {
this.mHostPackageName = hostPackageName;
com.android.internal.util.AnnotationValidations.validate(
NonNull.class, null, mHostPackageName);
this.mHostInputToken = hostInputToken;
onConstructed();
}
@@ -251,6 +297,7 @@ public final class InlineSuggestionsRequest implements Parcelable {
private int mMaxSuggestionCount;
private @NonNull List<InlinePresentationSpec> mPresentationSpecs;
private @NonNull String mHostPackageName;
private @Nullable IBinder mHostInputToken;
private long mBuilderFieldsSet = 0L;
@@ -320,10 +367,25 @@ public final class InlineSuggestionsRequest implements Parcelable {
return this;
}
/**
* The host input token of the IME that made the request. This will be set by the system for
* safety reasons.
*
* @hide
*/
@DataClass.Generated.Member
@Override
@NonNull Builder setHostInputToken(@Nullable IBinder value) {
checkNotUsed();
mBuilderFieldsSet |= 0x8;
mHostInputToken = value;
return this;
}
/** Builds the instance. This builder should not be touched after calling this! */
public @NonNull InlineSuggestionsRequest build() {
checkNotUsed();
mBuilderFieldsSet |= 0x8; // Mark builder used
mBuilderFieldsSet |= 0x10; // Mark builder used
if ((mBuilderFieldsSet & 0x1) == 0) {
mMaxSuggestionCount = defaultMaxSuggestionCount();
@@ -331,15 +393,19 @@ public final class InlineSuggestionsRequest implements Parcelable {
if ((mBuilderFieldsSet & 0x4) == 0) {
mHostPackageName = defaultHostPackageName();
}
if ((mBuilderFieldsSet & 0x8) == 0) {
mHostInputToken = defaultHostInputToken();
}
InlineSuggestionsRequest o = new InlineSuggestionsRequest(
mMaxSuggestionCount,
mPresentationSpecs,
mHostPackageName);
mHostPackageName,
mHostInputToken);
return o;
}
private void checkNotUsed() {
if ((mBuilderFieldsSet & 0x8) != 0) {
if ((mBuilderFieldsSet & 0x10) != 0) {
throw new IllegalStateException(
"This Builder should not be reused. Use a new Builder instance instead");
}
@@ -347,10 +413,10 @@ public final class InlineSuggestionsRequest implements Parcelable {
}
@DataClass.Generated(
time = 1578948035951L,
time = 1581555687721L,
codegenVersion = "1.0.14",
sourceFile = "frameworks/base/core/java/android/view/inputmethod/InlineSuggestionsRequest.java",
inputSignatures = "public static final int SUGGESTION_COUNT_UNLIMITED\nprivate final int mMaxSuggestionCount\nprivate final @android.annotation.NonNull java.util.List<android.view.inline.InlinePresentationSpec> mPresentationSpecs\nprivate @android.annotation.NonNull java.lang.String mHostPackageName\npublic void setHostPackageName(java.lang.String)\nprivate void onConstructed()\nprivate static int defaultMaxSuggestionCount()\nprivate static java.lang.String defaultHostPackageName()\nclass InlineSuggestionsRequest extends java.lang.Object implements [android.os.Parcelable]\n@com.android.internal.util.DataClass(genEqualsHashCode=true, genToString=true, genBuilder=true)\nabstract android.view.inputmethod.InlineSuggestionsRequest.Builder setPresentationSpecs(java.util.List<android.view.inline.InlinePresentationSpec>)\nabstract android.view.inputmethod.InlineSuggestionsRequest.Builder setHostPackageName(java.lang.String)\nclass BaseBuilder extends java.lang.Object implements []")
inputSignatures = "public static final int SUGGESTION_COUNT_UNLIMITED\nprivate final int mMaxSuggestionCount\nprivate final @android.annotation.NonNull java.util.List<android.view.inline.InlinePresentationSpec> mPresentationSpecs\nprivate @android.annotation.NonNull java.lang.String mHostPackageName\nprivate @android.annotation.Nullable android.os.IBinder mHostInputToken\npublic void setHostPackageName(java.lang.String)\npublic void setHostInputToken(android.os.IBinder)\nprivate void onConstructed()\nprivate static int defaultMaxSuggestionCount()\nprivate static java.lang.String defaultHostPackageName()\nprivate static android.os.IBinder defaultHostInputToken()\nclass InlineSuggestionsRequest extends java.lang.Object implements [android.os.Parcelable]\n@com.android.internal.util.DataClass(genEqualsHashCode=true, genToString=true, genBuilder=true)\nabstract android.view.inputmethod.InlineSuggestionsRequest.Builder setPresentationSpecs(java.util.List<android.view.inline.InlinePresentationSpec>)\nabstract android.view.inputmethod.InlineSuggestionsRequest.Builder setHostPackageName(java.lang.String)\nabstract android.view.inputmethod.InlineSuggestionsRequest.Builder setHostInputToken(android.os.IBinder)\nclass BaseBuilder extends java.lang.Object implements []")
@Deprecated
private void __metadata() {}

38
services/autofill/java/com/android/server/autofill/AutofillManagerServiceImpl.java
View File

@@ -119,6 +119,9 @@ final class AutofillManagerServiceImpl
private final LocalLog mUiLatencyHistory;
private final LocalLog mWtfHistory;
private final FieldClassificationStrategy mFieldClassificationStrategy;
@GuardedBy("mLock")
@Nullable
private RemoteInlineSuggestionRenderService mRemoteInlineSuggestionRenderService;
/**
@@ -236,17 +239,8 @@ final class AutofillManagerServiceImpl
sendStateToClients(/* resetClient= */ false);
}
updateRemoteAugmentedAutofillService();
updateRemoteInlineSuggestionRenderServiceLocked();
final ComponentName componentName = RemoteInlineSuggestionRenderService
.getServiceComponentName(getContext(), mUserId);
if (componentName != null) {
mRemoteInlineSuggestionRenderService = new RemoteInlineSuggestionRenderService(
getContext(), componentName, InlineSuggestionRenderService.SERVICE_INTERFACE,
mUserId, new InlineSuggestionRenderCallbacksImpl(),
mMaster.isBindInstantServiceAllowed(), mMaster.verbose);
} else {
mRemoteInlineSuggestionRenderService = null;
}
return enabledChanged;
}
@@ -1644,7 +1638,29 @@ final class AutofillManagerServiceImpl
return mFieldClassificationStrategy.getDefaultAlgorithm();
}
RemoteInlineSuggestionRenderService getRemoteInlineSuggestionRenderService() {
private void updateRemoteInlineSuggestionRenderServiceLocked() {
if (mRemoteInlineSuggestionRenderService != null) {
if (sVerbose) {
Slog.v(TAG, "updateRemoteInlineSuggestionRenderService(): "
+ "destroying old remote service");
}
mRemoteInlineSuggestionRenderService = null;
}
mRemoteInlineSuggestionRenderService = getRemoteInlineSuggestionRenderServiceLocked();
}
RemoteInlineSuggestionRenderService getRemoteInlineSuggestionRenderServiceLocked() {
final ComponentName componentName = RemoteInlineSuggestionRenderService
.getServiceComponentName(getContext(), mUserId);
if (mRemoteInlineSuggestionRenderService == null) {
mRemoteInlineSuggestionRenderService = new RemoteInlineSuggestionRenderService(
getContext(), componentName, InlineSuggestionRenderService.SERVICE_INTERFACE,
mUserId, new InlineSuggestionRenderCallbacksImpl(),
mMaster.isBindInstantServiceAllowed(), mMaster.verbose);
}
return mRemoteInlineSuggestionRenderService;
}

10
services/autofill/java/com/android/server/autofill/RemoteAugmentedAutofillService.java
View File

@@ -146,7 +146,8 @@ final class RemoteAugmentedAutofillService
@Nullable AutofillValue focusedValue,
@Nullable InlineSuggestionsRequest inlineSuggestionsRequest,
@Nullable IInlineSuggestionsResponseCallback inlineSuggestionsCallback,
@NonNull Runnable onErrorCallback) {
@NonNull Runnable onErrorCallback,
@NonNull RemoteInlineSuggestionRenderService remoteRenderService) {
long requestTime = SystemClock.elapsedRealtime();
AtomicReference<ICancellationSignal> cancellationRef = new AtomicReference<>();
@@ -168,7 +169,7 @@ final class RemoteAugmentedAutofillService
maybeRequestShowInlineSuggestions(sessionId,
inlineSuggestionsRequest, inlineSuggestionsData,
focusedId, inlineSuggestionsCallback, client,
onErrorCallback);
onErrorCallback, remoteRenderService);
requestAutofill.complete(null);
}
@@ -234,7 +235,8 @@ final class RemoteAugmentedAutofillService
@Nullable InlineSuggestionsRequest request, @Nullable Dataset[] inlineSuggestionsData,
@NonNull AutofillId focusedId,
@Nullable IInlineSuggestionsResponseCallback inlineSuggestionsCallback,
@NonNull IAutoFillManagerClient client, @NonNull Runnable onErrorCallback) {
@NonNull IAutoFillManagerClient client, @NonNull Runnable onErrorCallback,
@NonNull RemoteInlineSuggestionRenderService remoteRenderService) {
if (ArrayUtils.isEmpty(inlineSuggestionsData) || inlineSuggestionsCallback == null
|| request == null) {
return;
@@ -254,7 +256,7 @@ final class RemoteAugmentedAutofillService
} catch (RemoteException e) {
Slog.w(TAG, "Encounter exception autofilling the values");
}
}, onErrorCallback));
}, onErrorCallback, remoteRenderService));
} catch (RemoteException e) {
Slog.w(TAG, "Exception sending inline suggestions response back to IME.");
}

13
services/autofill/java/com/android/server/autofill/RemoteInlineSuggestionRenderService.java
View File

@@ -36,7 +36,10 @@ import android.util.Slog;
import com.android.internal.infra.AbstractMultiplePendingRequestsRemoteService;
final class RemoteInlineSuggestionRenderService extends
/**
* Remote service to help connect to InlineSuggestionRenderService in ExtServices.
*/
public final class RemoteInlineSuggestionRenderService extends
AbstractMultiplePendingRequestsRemoteService<RemoteInlineSuggestionRenderService,
IInlineSuggestionRenderService> {
@@ -81,9 +84,11 @@ final class RemoteInlineSuggestionRenderService extends
* Called by {@link Session} to generate a call to the
* {@link RemoteInlineSuggestionRenderService} to request rendering a slice .
*/
void renderSuggestion(@NonNull IInlineSuggestionUiCallback callback,
@NonNull InlinePresentation presentation, int width, int height) {
scheduleAsyncRequest((s) -> s.renderSuggestion(callback, presentation, width, height));
public void renderSuggestion(@NonNull IInlineSuggestionUiCallback callback,
@NonNull InlinePresentation presentation, int width, int height,
@Nullable IBinder hostInputToken) {
scheduleAsyncRequest(
(s) -> s.renderSuggestion(callback, presentation, width, height, hostInputToken));
}
@Nullable

4
services/autofill/java/com/android/server/autofill/Session.java
View File

@@ -2677,7 +2677,7 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState
synchronized (mLock) {
requestHideFillUi(mCurrentViewId);
}
});
}, mService.getRemoteInlineSuggestionRenderServiceLocked());
try {
imeResponse.getCallback().onInlineSuggestionsResponse(inlineSuggestionsResponse);
} catch (RemoteException e) {
@@ -2981,7 +2981,7 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState
synchronized (mLock) {
cancelAugmentedAutofillLocked();
}
});
}, mService.getRemoteInlineSuggestionRenderServiceLocked());
if (mAugmentedAutofillDestroyer == null) {
mAugmentedAutofillDestroyer = () -> remoteService.onDestroyAutofillWindowsRequest();

153
services/autofill/java/com/android/server/autofill/ui/InlineSuggestionFactory.java
View File

@@ -22,13 +22,14 @@ import static com.android.server.autofill.Helper.sVerbose;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.content.Context;
import android.os.IBinder;
import android.os.RemoteException;
import android.service.autofill.Dataset;
import android.service.autofill.IInlineSuggestionUiCallback;
import android.service.autofill.InlinePresentation;
import android.text.TextUtils;
import android.util.Slog;
import android.view.SurfaceControl;
import android.view.View;
import android.view.autofill.AutofillId;
import android.view.autofill.AutofillValue;
import android.view.inline.InlinePresentationSpec;
@@ -40,11 +41,14 @@ import android.widget.Toast;
import com.android.internal.view.inline.IInlineContentCallback;
import com.android.internal.view.inline.IInlineContentProvider;
import com.android.server.LocalServices;
import com.android.server.UiThread;
import com.android.server.autofill.RemoteInlineSuggestionRenderService;
import com.android.server.wm.WindowManagerInternal;
import java.util.ArrayList;
import java.util.List;
import java.util.function.BiFunction;
import java.util.function.BiConsumer;
import java.util.regex.Pattern;
public final class InlineSuggestionFactory {
@@ -60,24 +64,6 @@ public final class InlineSuggestionFactory {
void autofill(@NonNull Dataset dataset);
}
/**
* Creates an {@link InlineSuggestionsResponse} with the {@code datasets} provided by
* augmented autofill service.
*/
public static InlineSuggestionsResponse createAugmentedInlineSuggestionsResponse(
@NonNull InlineSuggestionsRequest request,
@NonNull Dataset[] datasets,
@NonNull AutofillId autofillId,
@NonNull Context context,
@NonNull InlineSuggestionUiCallback inlineSuggestionUiCallback,
@NonNull Runnable onErrorCallback) {
if (sDebug) Slog.d(TAG, "createAugmentedInlineSuggestionsResponse called");
return createInlineSuggestionsResponseInternal(/* isAugmented= */ true, request,
datasets, /* filterText= */ null, /* inlineActions= */ null, autofillId, context,
onErrorCallback,
(dataset, filedIndex) -> (v -> inlineSuggestionUiCallback.autofill(dataset)));
}
/**
* Creates an {@link InlineSuggestionsResponse} with the {@code datasets} provided by the
* autofill service, potentially filtering the datasets.
@@ -90,11 +76,33 @@ public final class InlineSuggestionFactory {
@NonNull AutofillId autofillId,
@NonNull Context context,
@NonNull AutoFillUI.AutoFillUiCallback client,
@NonNull Runnable onErrorCallback) {
@NonNull Runnable onErrorCallback,
@Nullable RemoteInlineSuggestionRenderService remoteRenderService) {
if (sDebug) Slog.d(TAG, "createInlineSuggestionsResponse called");
return createInlineSuggestionsResponseInternal(/* isAugmented= */ false, request, datasets,
filterText, inlineActions, autofillId, context, onErrorCallback,
(dataset, filedIndex) -> (v -> client.fill(requestId, filedIndex, dataset)));
(dataset, datasetIndex) -> client.fill(requestId, datasetIndex, dataset),
remoteRenderService);
}
/**
* Creates an {@link InlineSuggestionsResponse} with the {@code datasets} provided by augmented
* autofill service.
*/
public static InlineSuggestionsResponse createAugmentedInlineSuggestionsResponse(
@NonNull InlineSuggestionsRequest request,
@NonNull Dataset[] datasets,
@NonNull AutofillId autofillId,
@NonNull Context context,
@NonNull InlineSuggestionUiCallback inlineSuggestionUiCallback,
@NonNull Runnable onErrorCallback,
@Nullable RemoteInlineSuggestionRenderService remoteRenderService) {
if (sDebug) Slog.d(TAG, "createAugmentedInlineSuggestionsResponse called");
return createInlineSuggestionsResponseInternal(/* isAugmented= */ true, request,
datasets, /* filterText= */ null, /* inlineActions= */ null, autofillId, context,
onErrorCallback,
(dataset, fieldIndex) -> inlineSuggestionUiCallback.autofill(dataset),
remoteRenderService);
}
private static InlineSuggestionsResponse createInlineSuggestionsResponseInternal(
@@ -102,10 +110,9 @@ public final class InlineSuggestionFactory {
@NonNull Dataset[] datasets, @Nullable String filterText,
@Nullable List<InlinePresentation> inlineActions, @NonNull AutofillId autofillId,
@NonNull Context context, @NonNull Runnable onErrorCallback,
@NonNull BiFunction<Dataset, Integer, View.OnClickListener> onClickListenerFactory) {
@NonNull BiConsumer<Dataset, Integer> onClickFactory,
@Nullable RemoteInlineSuggestionRenderService remoteRenderService) {
final ArrayList<InlineSuggestion> inlineSuggestions = new ArrayList<>();
final InlineSuggestionUi inlineSuggestionUi = new InlineSuggestionUi(context,
onErrorCallback);
for (int i = 0; i < datasets.length; i++) {
final Dataset dataset = datasets[i];
final int fieldIndex = dataset.getFieldIds().indexOf(autofillId);
@@ -124,14 +131,15 @@ public final class InlineSuggestionFactory {
}
InlineSuggestion inlineSuggestion = createInlineSuggestion(isAugmented, dataset,
fieldIndex, mergedInlinePresentation(request, i, inlinePresentation),
inlineSuggestionUi, onClickListenerFactory);
onClickFactory, remoteRenderService, onErrorCallback,
request.getHostInputToken());
inlineSuggestions.add(inlineSuggestion);
}
if (inlineActions != null) {
for (InlinePresentation inlinePresentation : inlineActions) {
final InlineSuggestion inlineAction = createInlineAction(isAugmented, context,
mergedInlinePresentation(request, 0, inlinePresentation),
inlineSuggestionUi);
remoteRenderService, onErrorCallback, request.getHostInputToken());
inlineSuggestions.add(inlineAction);
}
}
@@ -173,40 +181,40 @@ public final class InlineSuggestionFactory {
private static InlineSuggestion createInlineAction(boolean isAugmented,
@NonNull Context context,
@NonNull InlinePresentation inlinePresentation,
@NonNull InlineSuggestionUi inlineSuggestionUi) {
@Nullable RemoteInlineSuggestionRenderService remoteRenderService,
@NonNull Runnable onErrorCallback, @Nullable IBinder hostInputToken) {
// TODO(b/146453195): fill in the autofill hint properly.
final InlineSuggestionInfo inlineSuggestionInfo = new InlineSuggestionInfo(
inlinePresentation.getInlinePresentationSpec(),
isAugmented ? InlineSuggestionInfo.SOURCE_PLATFORM
: InlineSuggestionInfo.SOURCE_AUTOFILL, new String[]{""},
InlineSuggestionInfo.TYPE_ACTION);
final View.OnClickListener onClickListener = v -> {
// TODO(b/148567875): Launch the intent provided through the slice. This
// should be part of the UI renderer therefore will be moved to the support
// library.
final Runnable onClickAction = () -> {
Toast.makeText(context, "icon clicked", Toast.LENGTH_SHORT).show();
};
return new InlineSuggestion(inlineSuggestionInfo,
createInlineContentProvider(inlinePresentation, inlineSuggestionUi,
onClickListener));
createInlineContentProvider(inlinePresentation, onClickAction, onErrorCallback,
remoteRenderService, hostInputToken));
}
private static InlineSuggestion createInlineSuggestion(boolean isAugmented,
@NonNull Dataset dataset,
int fieldIndex, @NonNull InlinePresentation inlinePresentation,
@NonNull InlineSuggestionUi inlineSuggestionUi,
@NonNull BiFunction<Dataset, Integer, View.OnClickListener> onClickListenerFactory) {
@NonNull Dataset dataset, int fieldIndex,
@NonNull InlinePresentation inlinePresentation,
@NonNull BiConsumer<Dataset, Integer> onClickFactory,
@NonNull RemoteInlineSuggestionRenderService remoteRenderService,
@NonNull Runnable onErrorCallback, @Nullable IBinder hostInputToken) {
// TODO(b/146453195): fill in the autofill hint properly.
final InlineSuggestionInfo inlineSuggestionInfo = new InlineSuggestionInfo(
inlinePresentation.getInlinePresentationSpec(),
isAugmented ? InlineSuggestionInfo.SOURCE_PLATFORM
: InlineSuggestionInfo.SOURCE_AUTOFILL, new String[]{""},
InlineSuggestionInfo.TYPE_SUGGESTION);
final View.OnClickListener onClickListener = onClickListenerFactory.apply(dataset,
fieldIndex);
final InlineSuggestion inlineSuggestion = new InlineSuggestion(inlineSuggestionInfo,
createInlineContentProvider(inlinePresentation, inlineSuggestionUi,
onClickListener));
createInlineContentProvider(inlinePresentation,
() -> onClickFactory.accept(dataset, fieldIndex), onErrorCallback,
remoteRenderService, hostInputToken));
return inlineSuggestion;
}
@@ -231,27 +239,64 @@ public final class InlineSuggestionFactory {
}
private static IInlineContentProvider.Stub createInlineContentProvider(
@NonNull InlinePresentation inlinePresentation,
@NonNull InlineSuggestionUi inlineSuggestionUi,
@Nullable View.OnClickListener onClickListener) {
@NonNull InlinePresentation inlinePresentation, @Nullable Runnable onClickAction,
@NonNull Runnable onErrorCallback,
@Nullable RemoteInlineSuggestionRenderService remoteRenderService,
@Nullable IBinder hostInputToken) {
return new IInlineContentProvider.Stub() {
@Override
public void provideContent(int width, int height,
IInlineContentCallback callback) {
public void provideContent(int width, int height, IInlineContentCallback callback) {
UiThread.getHandler().post(() -> {
SurfaceControl sc = inlineSuggestionUi.inflate(inlinePresentation, width,
height,
onClickListener);
try {
callback.onContent(sc);
} catch (RemoteException e) {
Slog.w(TAG, "Encounter exception calling back with inline content.");
final IInlineSuggestionUiCallback uiCallback = createInlineSuggestionUiCallback(
callback, onClickAction, onErrorCallback);
if (remoteRenderService == null) {
Slog.e(TAG, "RemoteInlineSuggestionRenderService is null");
return;
}
remoteRenderService.renderSuggestion(uiCallback, inlinePresentation,
width, height, hostInputToken);
});
}
};
}
private static IInlineSuggestionUiCallback.Stub createInlineSuggestionUiCallback(
@NonNull IInlineContentCallback callback, @NonNull Runnable onAutofillCallback,
@NonNull Runnable onErrorCallback) {
return new IInlineSuggestionUiCallback.Stub() {
@Override
public void onAutofill() throws RemoteException {
onAutofillCallback.run();
}
@Override
public void onContent(SurfaceControl surface)
throws RemoteException {
callback.onContent(surface);
}
@Override
public void onError() throws RemoteException {
onErrorCallback.run();
}
@Override
public void onTransferTouchFocusToImeWindow(IBinder sourceInputToken, int displayId)
throws RemoteException {
//TODO(b/149574510): Move logic to IMMS
final WindowManagerInternal windowManagerInternal = LocalServices.getService(
WindowManagerInternal.class);
if (!windowManagerInternal.transferTouchFocusToImeWindow(sourceInputToken,
displayId)) {
Slog.e(TAG, "Cannot transfer touch focus from suggestion to IME");
onErrorCallback.run();
}
}
};
}
private InlineSuggestionFactory() {
}
}

257
services/autofill/java/com/android/server/autofill/ui/InlineSuggestionUi.java
View File

@@ -1,257 +0,0 @@
/*
* Copyright (C) 2019 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.server.autofill.ui;
import static android.app.slice.SliceItem.FORMAT_IMAGE;
import static android.app.slice.SliceItem.FORMAT_TEXT;
import android.annotation.MainThread;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.app.slice.Slice;
import android.app.slice.SliceItem;
import android.content.Context;
import android.content.pm.PackageManager;
import android.content.res.Resources;
import android.content.res.TypedArray;
import android.graphics.PixelFormat;
import android.graphics.drawable.Icon;
import android.graphics.fonts.SystemFonts;
import android.os.IBinder;
import android.service.autofill.InlinePresentation;
import android.text.TextUtils;
import android.util.Log;
import android.view.ContextThemeWrapper;
import android.view.LayoutInflater;
import android.view.SurfaceControl;
import android.view.SurfaceControlViewHost;
import android.view.View;
import android.view.ViewGroup;
import android.view.WindowManager;
import android.widget.ImageView;
import android.widget.TextView;
import com.android.internal.R;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* This is a temporary inline suggestion UI inflater which will be replaced by the ExtServices
* implementation.
*
* TODO(b/146453086): remove this class once autofill ext service is implemented.
*
* @hide
*/
public class InlineSuggestionUi {
private static final String TAG = "InlineSuggestionUi";
// The pattern to match the value can be obtained by calling {@code Resources#getResourceName
// (int)}. This name is a single string of the form "package:type/entry".
private static final Pattern RESOURCE_NAME_PATTERN = Pattern.compile("([^:]+):([^/]+)/(\\S+)");
private final @NonNull Context mContext;
private final @NonNull Runnable mOnErrorCallback;
InlineSuggestionUi(@NonNull Context context, @NonNull Runnable onErrorCallback) {
this.mContext = context;
mOnErrorCallback = onErrorCallback;
}
/**
* Returns a {@link SurfaceControl} with the inflated content embedded in it.
*/
@MainThread
@Nullable
public SurfaceControl inflate(@NonNull InlinePresentation inlinePresentation, int width,
int height, @Nullable View.OnClickListener onClickListener) {
Log.d(TAG, "Inflating the inline suggestion UI");
//TODO(b/137800469): Pass in inputToken from IME.
final SurfaceControlViewHost wvr = new SurfaceControlViewHost(mContext,
mContext.getDisplay(), (IBinder) null);
final SurfaceControl sc = wvr.getSurfacePackage().getSurfaceControl();
Context contextThemeWrapper = getContextThemeWrapper(mContext,
inlinePresentation.getInlinePresentationSpec().getStyle());
if (contextThemeWrapper == null) {
contextThemeWrapper = getDefaultContextThemeWrapper(mContext);
}
final View suggestionView = renderSlice(inlinePresentation.getSlice(),
contextThemeWrapper);
final InlineSuggestionRoot suggestionRoot = new InlineSuggestionRoot(
mContext, mOnErrorCallback);
suggestionRoot.addView(suggestionView);
suggestionRoot.setOnClickListener(onClickListener);
WindowManager.LayoutParams lp =
new WindowManager.LayoutParams(width, height,
WindowManager.LayoutParams.TYPE_APPLICATION, 0,
PixelFormat.TRANSPARENT);
wvr.addView(suggestionRoot, lp);
return sc;
}
private static View renderSlice(Slice slice, Context context) {
final LayoutInflater inflater = LayoutInflater.from(context);
final ViewGroup suggestionView =
(ViewGroup) inflater.inflate(R.layout.autofill_inline_suggestion, null);
final ImageView startIconView =
suggestionView.findViewById(R.id.autofill_inline_suggestion_start_icon);
final TextView titleView =
suggestionView.findViewById(R.id.autofill_inline_suggestion_title);
final TextView subtitleView =
suggestionView.findViewById(R.id.autofill_inline_suggestion_subtitle);
final ImageView endIconView =
suggestionView.findViewById(R.id.autofill_inline_suggestion_end_icon);
boolean hasStartIcon = false;
boolean hasEndIcon = false;
boolean hasSubtitle = false;
final List<SliceItem> sliceItems = slice.getItems();
for (int i = 0; i < sliceItems.size(); i++) {
final SliceItem sliceItem = sliceItems.get(i);
if (sliceItem.getFormat().equals(FORMAT_IMAGE)) {
final Icon sliceIcon = sliceItem.getIcon();
if (i == 0) { // start icon
startIconView.setImageIcon(sliceIcon);
hasStartIcon = true;
} else { // end icon
endIconView.setImageIcon(sliceIcon);
hasEndIcon = true;
}
} else if (sliceItem.getFormat().equals(FORMAT_TEXT)) {
final List<String> sliceHints = sliceItem.getHints();
final String sliceText = sliceItem.getText().toString();
if (sliceHints.contains("inline_title")) { // title
titleView.setText(sliceText);
} else { // subtitle
subtitleView.setText(sliceText);
hasSubtitle = true;
}
}
}
if (!hasStartIcon) {
startIconView.setVisibility(View.GONE);
}
if (!hasEndIcon) {
endIconView.setVisibility(View.GONE);
}
if (!hasSubtitle) {
subtitleView.setVisibility(View.GONE);
}
return suggestionView;
}
private Context getDefaultContextThemeWrapper(@NonNull Context context) {
Resources.Theme theme = context.getResources().newTheme();
theme.applyStyle(android.R.style.Theme_AutofillInlineSuggestion, true);
return new ContextThemeWrapper(context, theme);
}
/**
* Returns a context wrapping the theme in the provided {@code style}, or null if {@code
* style} doesn't pass validation.
*/
@Nullable
private static Context getContextThemeWrapper(@NonNull Context context,
@Nullable String style) {
if (style == null) {
return null;
}
Matcher matcher = RESOURCE_NAME_PATTERN.matcher(style);
if (!matcher.matches()) {
Log.d(TAG, "Can not parse the style=" + style);
return null;
}
String packageName = matcher.group(1);
String type = matcher.group(2);
String entry = matcher.group(3);
if (TextUtils.isEmpty(packageName) || TextUtils.isEmpty(type) || TextUtils.isEmpty(entry)) {
Log.d(TAG, "Can not proceed with empty field values in the style=" + style);
return null;
}
Resources resources = null;
try {
resources = context.getPackageManager().getResourcesForApplication(
packageName);
} catch (PackageManager.NameNotFoundException e) {
return null;
}
int resId = resources.getIdentifier(entry, type, packageName);
if (resId == Resources.ID_NULL) {
return null;
}
Resources.Theme theme = resources.newTheme();
theme.applyStyle(resId, true);
if (!validateBaseTheme(theme, resId)) {
Log.d(TAG, "Provided theme is not a child of Theme.InlineSuggestion, ignoring it.");
return null;
}
if (!validateFontFamilyForTextViewStyles(theme)) {
Log.d(TAG,
"Provided theme specifies a font family that is not system font, ignoring it.");
return null;
}
return new ContextThemeWrapper(context, theme);
}
private static boolean validateFontFamilyForTextViewStyles(Resources.Theme theme) {
return validateFontFamily(theme, android.R.attr.autofillInlineSuggestionTitle)
&& validateFontFamily(theme, android.R.attr.autofillInlineSuggestionSubtitle);
}
private static boolean validateFontFamily(Resources.Theme theme, int styleAttr) {
TypedArray ta = null;
try {
ta = theme.obtainStyledAttributes(null, new int[]{android.R.attr.fontFamily},
styleAttr,
0);
if (ta.getIndexCount() == 0) {
return true;
}
String fontFamily = ta.getString(ta.getIndex(0));
return SystemFonts.getRawSystemFallbackMap().containsKey(fontFamily);
} finally {
if (ta != null) {
ta.recycle();
}
}
}
private static boolean validateBaseTheme(Resources.Theme theme, int styleAttr) {
TypedArray ta = null;
try {
ta = theme.obtainStyledAttributes(null,
new int[]{android.R.attr.isAutofillInlineSuggestionTheme}, styleAttr, 0);
if (ta.getIndexCount() == 0) {
return false;
}
return ta.getBoolean(ta.getIndex(0), false);
} finally {
if (ta != null) {
ta.recycle();
}
}
}
}