Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge "Few improvements on Augmented Autofill."

Browse Source
This commit is contained in:
TreeHugger Robot
2018-12-05 23:19:06 +00:00
committed by Android (Google) Code Review
parent 9da2ff0fdc bf93cdcb91
commit fdb30ae286
12 changed files with 183 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
api/system-current.txt
View File

@@ -4992,6 +4992,7 @@ package android.service.intelligence {
}
public final class FillRequest {
method public android.view.autofill.AutofillValue getFocusedAutofillValue();
method public android.view.autofill.AutofillId getFocusedId();
method public android.service.intelligence.PresentationParams getPresentationParams();
method public android.service.intelligence.InteractionSessionId getSessionId();

8
core/java/android/service/intelligence/FillCallback.java
View File

@@ -15,8 +15,10 @@
*/
package android.service.intelligence;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.annotation.SystemApi;
import android.service.intelligence.SmartSuggestionsService.AutofillProxy;
/**
* Callback used to indicate at {@link FillRequest} has been fulfilled.
@@ -25,8 +27,11 @@ import android.annotation.SystemApi;
*/
@SystemApi
public final class FillCallback {
private final AutofillProxy mProxy;
FillCallback() {}
FillCallback(@NonNull AutofillProxy proxy) {
mProxy = proxy;
}
/**
* Sets the response associated with the request.
@@ -35,6 +40,7 @@ public final class FillCallback {
* could not provide autofill for the request.
*/
public void onSuccess(@Nullable FillResponse response) {
mProxy.report(AutofillProxy.REPORT_EVENT_ON_SUCCESS);
final FillWindow fillWindow = response.getFillWindow();
if (fillWindow != null) {
fillWindow.show();

9
core/java/android/service/intelligence/FillRequest.java
View File

@@ -20,6 +20,7 @@ import android.annotation.Nullable;
import android.annotation.SystemApi;
import android.service.intelligence.SmartSuggestionsService.AutofillProxy;
import android.view.autofill.AutofillId;
import android.view.autofill.AutofillValue;
/**
* Represents a request to augment-fill an activity.
@@ -51,6 +52,14 @@ public final class FillRequest {
return mProxy.focusedId;
}
/**
* Gets the current value of the field that triggered the request.
*/
@NonNull
public AutofillValue getFocusedAutofillValue() {
return mProxy.focusedValue;
}
/**
* Gets the Smart Suggestions object used to embed the autofill UI.
*

38
core/java/android/service/intelligence/FillWindow.java
View File

@@ -23,6 +23,7 @@ import android.annotation.SystemApi;
import android.app.Dialog;
import android.graphics.Rect;
import android.service.intelligence.PresentationParams.Area;
import android.service.intelligence.SmartSuggestionsService.AutofillProxy;
import android.util.Log;
import android.view.Gravity;
import android.view.View;
@@ -33,6 +34,8 @@ import android.view.WindowManager;
import com.android.internal.annotations.GuardedBy;
import com.android.internal.util.Preconditions;
import dalvik.system.CloseGuard;
import java.io.PrintWriter;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
@@ -73,6 +76,7 @@ public final class FillWindow {
@interface Flags{}
private final Object mLock = new Object();
private final CloseGuard mCloseGuard = CloseGuard.get();
@GuardedBy("mLock")
private Dialog mDialog;
@@ -80,6 +84,8 @@ public final class FillWindow {
@GuardedBy("mLock")
private boolean mDestroyed;
private AutofillProxy mProxy;
/**
* Updates the content of the window.
*
@@ -123,6 +129,8 @@ public final class FillWindow {
synchronized (mLock) {
checkNotDestroyedLocked();
mProxy = area.proxy;
// TODO(b/111330312): once we have the SurfaceControl approach, we should update the
// window instead of destroying. In fact, it might be better to allocate a full window
// initially, which is transparent (and let touches get through) everywhere but in the
@@ -133,6 +141,7 @@ public final class FillWindow {
// etc.
mDialog = new Dialog(rootView.getContext());
mCloseGuard.open("destroy");
final Window window = mDialog.getWindow();
window.setType(WindowManager.LayoutParams.TYPE_APPLICATION_OVERLAY);
@@ -156,7 +165,7 @@ public final class FillWindow {
Log.d(TAG, "Created FillWindow: params= " + smartSuggestion + " view=" + rootView);
}
area.proxy.setFillWindow(this);
mProxy.setFillWindow(this);
return true;
}
}
@@ -173,6 +182,9 @@ public final class FillWindow {
}
mDialog.show();
if (mProxy != null) {
mProxy.report(AutofillProxy.REPORT_EVENT_UI_SHOWN);
}
}
}
@@ -182,15 +194,29 @@ public final class FillWindow {
* <p>Once destroyed, this window cannot be used anymore
*/
public void destroy() {
if (DEBUG) Log.d(TAG, "destroy(): mDestroyed = " + mDestroyed);
if (DEBUG) Log.d(TAG, "destroy(): mDestroyed=" + mDestroyed + " mDialog=" + mDialog);
synchronized (this) {
if (mDestroyed) return;
if (mDestroyed || mDialog == null) return;
if (mDialog != null) {
mDialog.dismiss();
mDialog = null;
mDialog.dismiss();
mDialog = null;
if (mProxy != null) {
mProxy.report(AutofillProxy.REPORT_EVENT_UI_DESTROYED);
}
mCloseGuard.close();
}
}
@Override
protected void finalize() throws Throwable {
try {
if (mCloseGuard != null) {
mCloseGuard.warnIfOpen();
}
destroy();
} finally {
super.finalize();
}
}

4
core/java/android/service/intelligence/IIntelligenceService.aidl
View File

@@ -23,6 +23,7 @@ import android.service.intelligence.InteractionContext;
import android.service.intelligence.SnapshotData;
import android.view.autofill.AutofillId;
import android.view.autofill.AutofillValue;
import android.view.intelligence.ContentCaptureEvent;
import java.util.List;
@@ -45,7 +46,8 @@ oneway interface IIntelligenceService {
in SnapshotData snapshotData);
void onAutofillRequest(in InteractionSessionId sessionId, in IBinder autofillManagerClient,
int autofilSessionId, in AutofillId focusedId);
int autofilSessionId, in AutofillId focusedId,
in AutofillValue focusedValue, long requestTime);
void onDestroyAutofillWindowsRequest(in InteractionSessionId sessionId);
}

104
core/java/android/service/intelligence/SmartSuggestionsService.java
View File

@@ -18,6 +18,7 @@ package android.service.intelligence;
import static com.android.internal.util.function.pooled.PooledLambda.obtainMessage;
import android.annotation.CallSuper;
import android.annotation.IntDef;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.annotation.SystemApi;
@@ -30,10 +31,13 @@ import android.os.Handler;
import android.os.IBinder;
import android.os.Looper;
import android.os.RemoteException;
import android.os.SystemClock;
import android.service.intelligence.PresentationParams.SystemPopupPresentationParams;
import android.util.ArrayMap;
import android.util.Log;
import android.util.Pair;
import android.util.Slog;
import android.util.TimeUtils;
import android.view.autofill.AutofillId;
import android.view.autofill.AutofillValue;
import android.view.autofill.IAugmentedAutofillManagerClient;
@@ -43,6 +47,8 @@ import com.android.internal.annotations.GuardedBy;
import java.io.FileDescriptor;
import java.io.PrintWriter;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
@@ -110,9 +116,11 @@ public abstract class SmartSuggestionsService extends Service {
@Override
public void onAutofillRequest(InteractionSessionId sessionId, IBinder client,
int autofilSessionId, AutofillId focusedId) {
int autofilSessionId, AutofillId focusedId, AutofillValue focusedValue,
long requestTime) {
mHandler.sendMessage(obtainMessage(SmartSuggestionsService::handleOnAutofillRequest,
SmartSuggestionsService.this, sessionId, client, autofilSessionId, focusedId));
SmartSuggestionsService.this, sessionId, client, autofilSessionId, focusedId,
focusedValue, requestTime));
}
@Override
@@ -229,13 +237,15 @@ public abstract class SmartSuggestionsService extends Service {
@NonNull ContentCaptureEventsRequest request);
private void handleOnAutofillRequest(@NonNull InteractionSessionId sessionId,
@NonNull IBinder client, int autofillSessionId, @NonNull AutofillId focusedId) {
@NonNull IBinder client, int autofillSessionId, @NonNull AutofillId focusedId,
@Nullable AutofillValue focusedValue, long requestTime) {
if (mAutofillProxies == null) {
mAutofillProxies = new ArrayMap<>();
}
AutofillProxy proxy = mAutofillProxies.get(sessionId);
if (proxy == null) {
proxy = new AutofillProxy(sessionId, client, autofillSessionId, focusedId);
proxy = new AutofillProxy(sessionId, client, autofillSessionId, focusedId, focusedValue,
requestTime);
mAutofillProxies.put(sessionId, proxy);
} else {
// TODO(b/111330312): figure out if it's ok to reuse the proxy; add logging
@@ -244,7 +254,7 @@ public abstract class SmartSuggestionsService extends Service {
// TODO(b/111330312): set cancellation signal
final CancellationSignal cancellationSignal = null;
onFillRequest(sessionId, new FillRequest(proxy), cancellationSignal,
new FillController(proxy), new FillCallback());
new FillController(proxy), new FillCallback(proxy));
}
/**
@@ -332,11 +342,32 @@ public abstract class SmartSuggestionsService extends Service {
/** @hide */
static final class AutofillProxy {
static final int REPORT_EVENT_ON_SUCCESS = 1;
static final int REPORT_EVENT_UI_SHOWN = 2;
static final int REPORT_EVENT_UI_DESTROYED = 3;
@IntDef(prefix = { "REPORT_EVENT_" }, value = {
REPORT_EVENT_ON_SUCCESS,
REPORT_EVENT_UI_SHOWN,
REPORT_EVENT_UI_DESTROYED
})
@Retention(RetentionPolicy.SOURCE)
@interface ReportEvent{}
private final Object mLock = new Object();
private final IAugmentedAutofillManagerClient mClient;
private final int mAutofillSessionId;
public final InteractionSessionId sessionId;
public final AutofillId focusedId;
public final AutofillValue focusedValue;
// Objects used to log metrics
private final long mRequestTime;
private long mOnSuccessTime;
private long mUiFirstShownTime;
private long mUiFirstDestroyedTime;
@GuardedBy("mLock")
private SystemPopupPresentationParams mSmartSuggestion;
@@ -345,11 +376,14 @@ public abstract class SmartSuggestionsService extends Service {
private FillWindow mFillWindow;
private AutofillProxy(@NonNull InteractionSessionId sessionId, @NonNull IBinder client,
int autofillSessionId, @NonNull AutofillId focusedId) {
int autofillSessionId, @NonNull AutofillId focusedId,
@Nullable AutofillValue focusedValue, long requestTime) {
this.sessionId = sessionId;
mClient = IAugmentedAutofillManagerClient.Stub.asInterface(client);
mAutofillSessionId = autofillSessionId;
this.focusedId = focusedId;
this.focusedValue = focusedValue;
this.mRequestTime = requestTime;
// TODO(b/111330312): linkToDeath
}
@@ -400,9 +434,50 @@ public abstract class SmartSuggestionsService extends Service {
}
}
// Used for metrics.
public void report(@ReportEvent int event) {
switch (event) {
case REPORT_EVENT_ON_SUCCESS:
if (mOnSuccessTime == 0) {
mOnSuccessTime = SystemClock.elapsedRealtime();
if (DEBUG) {
Slog.d(TAG, "Service responsed in "
+ TimeUtils.formatDuration(mOnSuccessTime - mRequestTime));
}
}
break;
case REPORT_EVENT_UI_SHOWN:
if (mUiFirstShownTime == 0) {
mUiFirstShownTime = SystemClock.elapsedRealtime();
if (DEBUG) {
Slog.d(TAG, "UI shown in "
+ TimeUtils.formatDuration(mUiFirstShownTime - mRequestTime));
}
}
break;
case REPORT_EVENT_UI_DESTROYED:
if (mUiFirstDestroyedTime == 0) {
mUiFirstDestroyedTime = SystemClock.elapsedRealtime();
if (DEBUG) {
Slog.d(TAG, "UI destroyed in "
+ TimeUtils.formatDuration(
mUiFirstDestroyedTime - mRequestTime));
}
}
break;
default:
Slog.w(TAG, "invalid event reported: " + event);
}
// TODO(b/111330312): log metrics as well
}
public void dump(@NonNull String prefix, @NonNull PrintWriter pw) {
pw.print(prefix); pw.print("afSessionId: "); pw.println(mAutofillSessionId);
pw.print(prefix); pw.print("focusedId: "); pw.println(focusedId);
if (focusedValue != null) {
pw.print(prefix); pw.print("focusedValue: "); pw.println(focusedValue);
}
pw.print(prefix); pw.print("client: "); pw.println(mClient);
final String prefix2 = prefix + " ";
if (mFillWindow != null) {
@@ -413,6 +488,23 @@ public abstract class SmartSuggestionsService extends Service {
pw.print(prefix); pw.println("smartSuggestion:");
mSmartSuggestion.dump(prefix2, pw);
}
if (mOnSuccessTime > 0) {
final long responseTime = mOnSuccessTime - mRequestTime;
pw.print(prefix); pw.print("response time: ");
TimeUtils.formatDuration(responseTime, pw); pw.println();
}
if (mUiFirstShownTime > 0) {
final long uiRenderingTime = mUiFirstShownTime - mRequestTime;
pw.print(prefix); pw.print("UI rendering time: ");
TimeUtils.formatDuration(uiRenderingTime, pw); pw.println();
}
if (mUiFirstDestroyedTime > 0) {
final long uiTotalTime = mUiFirstDestroyedTime - mRequestTime;
pw.print(prefix); pw.print("UI life time: ");
TimeUtils.formatDuration(uiTotalTime, pw); pw.println();
}
}
private void destroy() {

4
services/autofill/java/com/android/server/autofill/Session.java
View File

@@ -2579,11 +2579,13 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState
+ " when server returned null for session " + this.id);
}
final AutofillValue currentValue = mViewStates.get(mCurrentViewId).getCurrentValue();
// TODO(b/111330312): we might need to add a new state in the AutofillManager to optimize
// furgher AFM -> AFMS calls.
// TODO(b/119638958): add CTS tests
return intelligenceManagerInternal.requestAutofill(mService.getUserId(), mClient,
mActivityToken, this.id, mCurrentViewId);
mActivityToken, this.id, mCurrentViewId, currentValue);
}
@GuardedBy("mLock")

6
services/core/java/com/android/server/intelligence/IntelligenceManagerInternal.java
View File

@@ -16,10 +16,12 @@
package com.android.server.intelligence;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.annotation.UserIdInt;
import android.os.Bundle;
import android.os.IBinder;
import android.view.autofill.AutofillId;
import android.view.autofill.AutofillValue;
import android.view.autofill.IAutoFillManagerClient;
/**
@@ -53,6 +55,7 @@ public abstract class IntelligenceManagerInternal {
* @param activityToken activity that originated this request.
* @param autofillSessionId autofill session id (must be used on {@code client} calls.
* @param focusedId id of the the field that triggered this request.
* @param focusedValue current value of the field that triggered this request.
*
* @return {@code false} if the service cannot handle this request, {@code true} otherwise.
* <b>NOTE: </b> it must return right away; typically it will return {@code false} if the
@@ -60,7 +63,8 @@ public abstract class IntelligenceManagerInternal {
*/
public abstract AugmentedAutofillCallback requestAutofill(@UserIdInt int userId,
@NonNull IAutoFillManagerClient client, @NonNull IBinder activityToken,
int autofillSessionId, @NonNull AutofillId focusedId);
int autofillSessionId, @NonNull AutofillId focusedId,
@Nullable AutofillValue focusedValue);
/**
* Callback used by the Autofill Session to communicate with the Augmented Autofill service.

8
services/intelligence/java/com/android/server/intelligence/ContentCaptureSession.java
View File

@@ -16,6 +16,7 @@
package com.android.server.intelligence;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.content.ComponentName;
import android.content.Context;
import android.os.IBinder;
@@ -25,6 +26,7 @@ import android.service.intelligence.SmartSuggestionsService;
import android.service.intelligence.SnapshotData;
import android.util.Slog;
import android.view.autofill.AutofillId;
import android.view.autofill.AutofillValue;
import android.view.autofill.IAutoFillManagerClient;
import android.view.intelligence.ContentCaptureEvent;
@@ -98,8 +100,10 @@ final class ContentCaptureSession implements RemoteIntelligenceServiceCallbacks
* Requests the service to autofill the given field.
*/
public AugmentedAutofillCallback requestAutofillLocked(@NonNull IAutoFillManagerClient client,
int autofillSessionId, @NonNull AutofillId focusedId) {
mRemoteService.onRequestAutofillLocked(mId, client, autofillSessionId, focusedId);
int autofillSessionId, @NonNull AutofillId focusedId,
@Nullable AutofillValue focusedValue) {
mRemoteService.onRequestAutofillLocked(mId, client, autofillSessionId, focusedId,
focusedValue);
if (mAutofillCallback == null) {
mAutofillCallback = () -> mRemoteService.onDestroyAutofillWindowsRequest(mId);
}

6
services/intelligence/java/com/android/server/intelligence/IntelligenceManagerService.java
View File

@@ -35,6 +35,7 @@ import android.os.UserManager;
import android.service.intelligence.InteractionSessionId;
import android.util.Slog;
import android.view.autofill.AutofillId;
import android.view.autofill.AutofillValue;
import android.view.autofill.IAutoFillManagerClient;
import android.view.intelligence.ContentCaptureEvent;
import android.view.intelligence.IIntelligenceManager;
@@ -257,12 +258,13 @@ public final class IntelligenceManagerService extends
@Override
public AugmentedAutofillCallback requestAutofill(@UserIdInt int userId,
@NonNull IAutoFillManagerClient client, @NonNull IBinder activityToken,
int autofillSessionId, @NonNull AutofillId focusedId) {
int autofillSessionId, @NonNull AutofillId focusedId,
@Nullable AutofillValue focusedValue) {
synchronized (mLock) {
final IntelligencePerUserService service = peekServiceForUserLocked(userId);
if (service != null) {
return service.requestAutofill(client, activityToken, autofillSessionId,
focusedId);
focusedId, focusedValue);
}
}
return null;

7
services/intelligence/java/com/android/server/intelligence/IntelligencePerUserService.java
View File

@@ -40,6 +40,7 @@ import android.text.TextUtils;
import android.util.ArrayMap;
import android.util.Slog;
import android.view.autofill.AutofillId;
import android.view.autofill.AutofillValue;
import android.view.autofill.IAutoFillManagerClient;
import android.view.intelligence.ContentCaptureEvent;
import android.view.intelligence.ContentCaptureManager;
@@ -289,13 +290,15 @@ final class IntelligencePerUserService
}
public AugmentedAutofillCallback requestAutofill(@NonNull IAutoFillManagerClient client,
@NonNull IBinder activityToken, int autofillSessionId, @NonNull AutofillId focusedId) {
@NonNull IBinder activityToken, int autofillSessionId, @NonNull AutofillId focusedId,
@Nullable AutofillValue focusedValue) {
synchronized (mLock) {
final ContentCaptureSession session = getSession(activityToken);
if (session != null) {
// TODO(b/111330312): log metrics
if (mMaster.verbose) Slog.v(TAG, "requestAugmentedAutofill()");
return session.requestAutofillLocked(client, autofillSessionId, focusedId);
return session.requestAutofillLocked(client, autofillSessionId, focusedId,
focusedValue);
}
if (mMaster.debug) {
Slog.d(TAG, "requestAutofill(): no session for " + activityToken);

14
services/intelligence/java/com/android/server/intelligence/RemoteIntelligenceService.java
View File

@@ -23,6 +23,7 @@ import android.os.Bundle;
import android.os.IBinder;
import android.os.IInterface;
import android.os.RemoteException;
import android.os.SystemClock;
import android.service.intelligence.ContentCaptureEventsRequest;
import android.service.intelligence.IIntelligenceService;
import android.service.intelligence.InteractionContext;
@@ -32,6 +33,7 @@ import android.text.format.DateUtils;
import android.util.Slog;
import android.view.autofill.AutofillId;
import android.view.autofill.AutofillManager;
import android.view.autofill.AutofillValue;
import android.view.autofill.IAutoFillManagerClient;
import android.view.intelligence.ContentCaptureEvent;
@@ -114,10 +116,10 @@ final class RemoteIntelligenceService
*/
public void onRequestAutofillLocked(@NonNull InteractionSessionId sessionId,
@NonNull IAutoFillManagerClient client, int autofillSessionId,
@NonNull AutofillId focusedId) {
@NonNull AutofillId focusedId, @Nullable AutofillValue focusedValue) {
cancelScheduledUnbind();
scheduleRequest(new PendingAutofillRequest(this, sessionId, client, autofillSessionId,
focusedId));
focusedId, focusedValue));
}
/**
@@ -222,16 +224,20 @@ final class RemoteIntelligenceService
private static final class PendingAutofillRequest extends MyPendingRequest {
private final @NonNull AutofillId mFocusedId;
private final @Nullable AutofillValue mFocusedValue;
private final @NonNull IAutoFillManagerClient mClient;
private final int mAutofillSessionId;
private final long mRequestTime = SystemClock.elapsedRealtime();
protected PendingAutofillRequest(@NonNull RemoteIntelligenceService service,
@NonNull InteractionSessionId sessionId, @NonNull IAutoFillManagerClient client,
int autofillSessionId, @NonNull AutofillId focusedId) {
int autofillSessionId, @NonNull AutofillId focusedId,
@Nullable AutofillValue focusedValue) {
super(service, sessionId);
mClient = client;
mAutofillSessionId = autofillSessionId;
mFocusedId = focusedId;
mFocusedValue = focusedValue;
}
@Override // from MyPendingRequest
@@ -243,7 +249,7 @@ final class RemoteIntelligenceService
final IBinder realClient = resultData
.getBinder(AutofillManager.EXTRA_AUGMENTED_AUTOFILL_CLIENT);
remoteService.mService.onAutofillRequest(mSessionId, realClient,
mAutofillSessionId, mFocusedId);
mAutofillSessionId, mFocusedId, mFocusedValue, mRequestTime);
}
};