Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge "Pass focused field's ViewNode information in the augmented autofill request"

Browse Source
This commit is contained in:
TreeHugger Robot
2021-01-07 20:25:15 +00:00
committed by Android (Google) Code Review
parent a37c840348 41797aa777
commit 677c97ddf8
10 changed files with 271 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
core/api/system-current.txt
View File

@@ -9212,6 +9212,7 @@ package android.service.autofill.augmented {
method @NonNull public android.content.ComponentName getActivityComponent();
method @NonNull public android.view.autofill.AutofillId getFocusedId();
method @NonNull public android.view.autofill.AutofillValue getFocusedValue();
method @Nullable public android.app.assist.AssistStructure.ViewNode getFocusedViewNode();
method @Nullable public android.view.inputmethod.InlineSuggestionsRequest getInlineSuggestionsRequest();
method @Nullable public android.service.autofill.augmented.PresentationParams getPresentationParams();
method public int getTaskId();

2
core/api/test-current.txt
View File

@@ -1714,6 +1714,7 @@ package android.service.autofill.augmented {
method @NonNull public android.content.ComponentName getActivityComponent();
method @NonNull public android.view.autofill.AutofillId getFocusedId();
method @NonNull public android.view.autofill.AutofillValue getFocusedValue();
method @Nullable public android.app.assist.AssistStructure.ViewNode getFocusedViewNode();
method @Nullable public android.view.inputmethod.InlineSuggestionsRequest getInlineSuggestionsRequest();
method @Nullable public android.service.autofill.augmented.PresentationParams getPresentationParams();
method public int getTaskId();
@@ -2199,6 +2200,7 @@ package android.view.autofill {
ctor public AutofillId(int, int);
ctor public AutofillId(@NonNull android.view.autofill.AutofillId, long, int);
method public boolean equalsIgnoreSession(@Nullable android.view.autofill.AutofillId);
method public boolean isNonVirtual();
method @NonNull public static android.view.autofill.AutofillId withoutSession(@NonNull android.view.autofill.AutofillId);
}

5
core/java/android/app/assist/AssistStructure.aidl
View File

@@ -17,3 +17,8 @@
package android.app.assist;
parcelable AssistStructure;
/**
* {@hide}
*/
parcelable AssistStructure.ViewNodeParcelable;

173
core/java/android/app/assist/AssistStructure.java
View File

@@ -271,7 +271,8 @@ public class AssistStructure implements Parcelable {
+ ", views=" + mNumWrittenViews
+ ", level=" + (mCurViewStackPos+levelAdj));
out.writeInt(VALIDATE_VIEW_TOKEN);
int flags = child.writeSelfToParcel(out, pwriter, mSanitizeOnWrite, mTmpMatrix);
int flags = child.writeSelfToParcel(out, pwriter, mSanitizeOnWrite,
mTmpMatrix, /*willWriteChildren=*/true);
mNumWrittenViews++;
// If the child has children, push it on the stack to write them next.
if ((flags&ViewNode.FLAGS_HAS_CHILDREN) != 0) {
@@ -724,11 +725,51 @@ public class AssistStructure implements Parcelable {
public ViewNode() {
}
ViewNode(@NonNull Parcel in) {
initializeFromParcelWithoutChildren(in, /*preader=*/null, /*tmpMatrix=*/null);
}
ViewNode(ParcelTransferReader reader, int nestingLevel) {
final Parcel in = reader.readParcel(VALIDATE_VIEW_TOKEN, nestingLevel);
reader.mNumReadViews++;
final PooledStringReader preader = reader.mStringReader;
mClassName = preader.readString();
initializeFromParcelWithoutChildren(in, Objects.requireNonNull(reader.mStringReader),
Objects.requireNonNull(reader.mTmpMatrix));
if ((mFlags & FLAGS_HAS_CHILDREN) != 0) {
final int numChildren = in.readInt();
if (DEBUG_PARCEL_TREE || DEBUG_PARCEL_CHILDREN) {
Log.d(TAG,
"Preparing to read " + numChildren
+ " children: @ #" + reader.mNumReadViews
+ ", level " + nestingLevel);
}
mChildren = new ViewNode[numChildren];
for (int i = 0; i < numChildren; i++) {
mChildren[i] = new ViewNode(reader, nestingLevel + 1);
}
}
}
private static void writeString(@NonNull Parcel out, @Nullable PooledStringWriter pwriter,
@Nullable String str) {
if (pwriter != null) {
pwriter.writeString(str);
} else {
out.writeString(str);
}
}
@Nullable
private static String readString(@NonNull Parcel in, @Nullable PooledStringReader preader) {
if (preader != null) {
return preader.readString();
}
return in.readString();
}
// This does not read the child nodes.
void initializeFromParcelWithoutChildren(Parcel in, @Nullable PooledStringReader preader,
@Nullable float[] tmpMatrix) {
mClassName = readString(in, preader);
mFlags = in.readInt();
final int flags = mFlags;
mAutofillFlags = in.readInt();
@@ -736,10 +777,10 @@ public class AssistStructure implements Parcelable {
if ((flags&FLAGS_HAS_ID) != 0) {
mId = in.readInt();
if (mId != View.NO_ID) {
mIdEntry = preader.readString();
mIdEntry = readString(in, preader);
if (mIdEntry != null) {
mIdType = preader.readString();
mIdPackage = preader.readString();
mIdType = readString(in, preader);
mIdPackage = readString(in, preader);
}
}
}
@@ -784,10 +825,10 @@ public class AssistStructure implements Parcelable {
mMaxLength = in.readInt();
}
if ((autofillFlags & AUTOFILL_FLAGS_HAS_TEXT_ID_ENTRY) != 0) {
mTextIdEntry = preader.readString();
mTextIdEntry = readString(in, preader);
}
if ((autofillFlags & AUTOFILL_FLAGS_HAS_HINT_ID_ENTRY) != 0) {
mHintIdEntry = preader.readString();
mHintIdEntry = readString(in, preader);
}
}
if ((flags&FLAGS_HAS_LARGE_COORDS) != 0) {
@@ -809,8 +850,11 @@ public class AssistStructure implements Parcelable {
}
if ((flags&FLAGS_HAS_MATRIX) != 0) {
mMatrix = new Matrix();
in.readFloatArray(reader.mTmpMatrix);
mMatrix.setValues(reader.mTmpMatrix);
if (tmpMatrix == null) {
tmpMatrix = new float[9];
}
in.readFloatArray(tmpMatrix);
mMatrix.setValues(tmpMatrix);
}
if ((flags&FLAGS_HAS_ELEVATION) != 0) {
mElevation = in.readFloat();
@@ -839,21 +883,16 @@ public class AssistStructure implements Parcelable {
if ((flags&FLAGS_HAS_EXTRAS) != 0) {
mExtras = in.readBundle();
}
if ((flags&FLAGS_HAS_CHILDREN) != 0) {
final int NCHILDREN = in.readInt();
if (DEBUG_PARCEL_TREE || DEBUG_PARCEL_CHILDREN) Log.d(TAG,
"Preparing to read " + NCHILDREN
+ " children: @ #" + reader.mNumReadViews
+ ", level " + nestingLevel);
mChildren = new ViewNode[NCHILDREN];
for (int i=0; i<NCHILDREN; i++) {
mChildren[i] = new ViewNode(reader, nestingLevel + 1);
}
}
}
int writeSelfToParcel(Parcel out, PooledStringWriter pwriter, boolean sanitizeOnWrite,
float[] tmpMatrix) {
/**
* This does not write the child nodes.
*
* @param willWriteChildren whether child nodes will be written to the parcel or not after
* calling this method.
*/
int writeSelfToParcel(@NonNull Parcel out, @Nullable PooledStringWriter pwriter,
boolean sanitizeOnWrite, @Nullable float[] tmpMatrix, boolean willWriteChildren) {
// Guard used to skip non-sanitized data when writing for autofill.
boolean writeSensitive = true;
@@ -903,7 +942,7 @@ public class AssistStructure implements Parcelable {
if (mExtras != null) {
flags |= FLAGS_HAS_EXTRAS;
}
if (mChildren != null) {
if (mChildren != null && willWriteChildren) {
flags |= FLAGS_HAS_CHILDREN;
}
if (mAutofillId != null) {
@@ -946,7 +985,7 @@ public class AssistStructure implements Parcelable {
autofillFlags |= AUTOFILL_FLAGS_HAS_HINT_ID_ENTRY;
}
pwriter.writeString(mClassName);
writeString(out, pwriter, mClassName);
int writtenFlags = flags;
if (autofillFlags != 0 && (mSanitized || !sanitizeOnWrite)) {
@@ -966,10 +1005,10 @@ public class AssistStructure implements Parcelable {
if ((flags&FLAGS_HAS_ID) != 0) {
out.writeInt(mId);
if (mId != View.NO_ID) {
pwriter.writeString(mIdEntry);
writeString(out, pwriter, mIdEntry);
if (mIdEntry != null) {
pwriter.writeString(mIdType);
pwriter.writeString(mIdPackage);
writeString(out, pwriter, mIdType);
writeString(out, pwriter, mIdPackage);
}
}
}
@@ -1020,10 +1059,10 @@ public class AssistStructure implements Parcelable {
out.writeInt(mMaxLength);
}
if ((autofillFlags & AUTOFILL_FLAGS_HAS_TEXT_ID_ENTRY) != 0) {
pwriter.writeString(mTextIdEntry);
writeString(out, pwriter, mTextIdEntry);
}
if ((autofillFlags & AUTOFILL_FLAGS_HAS_HINT_ID_ENTRY) != 0) {
pwriter.writeString(mHintIdEntry);
writeString(out, pwriter, mHintIdEntry);
}
}
if ((flags&FLAGS_HAS_LARGE_COORDS) != 0) {
@@ -1040,6 +1079,9 @@ public class AssistStructure implements Parcelable {
out.writeInt(mScrollY);
}
if ((flags&FLAGS_HAS_MATRIX) != 0) {
if (tmpMatrix == null) {
tmpMatrix = new float[9];
}
mMatrix.getValues(tmpMatrix);
out.writeFloatArray(tmpMatrix);
}
@@ -1694,6 +1736,57 @@ public class AssistStructure implements Parcelable {
}
}
/**
* A parcelable wrapper class around {@link ViewNode}.
*
* <p>This class, when parceled and unparceled, does not carry the child nodes.
*
* @hide
*/
public static final class ViewNodeParcelable implements Parcelable {
@NonNull
private final ViewNode mViewNode;
public ViewNodeParcelable(@NonNull ViewNode viewNode) {
mViewNode = viewNode;
}
public ViewNodeParcelable(@NonNull Parcel in) {
mViewNode = new ViewNode(in);
}
@NonNull
public ViewNode getViewNode() {
return mViewNode;
}
@Override
public int describeContents() {
return 0;
}
@Override
public void writeToParcel(@NonNull Parcel parcel, int flags) {
mViewNode.writeSelfToParcel(parcel, /*pwriter=*/null, /*sanitizeOnWrite=*/false,
/*tmpMatrix*/null, /*willWriteChildren=*/ false);
}
@NonNull
public static final Parcelable.Creator<ViewNodeParcelable> CREATOR =
new Parcelable.Creator<ViewNodeParcelable>() {
@Override
public ViewNodeParcelable createFromParcel(@NonNull Parcel in) {
return new ViewNodeParcelable(in);
}
@Override
public ViewNodeParcelable[] newArray(int size) {
return new ViewNodeParcelable[size];
}
};
}
/**
* POJO used to override some autofill-related values when the node is parcelized.
*
@@ -1704,17 +1797,35 @@ public class AssistStructure implements Parcelable {
public AutofillValue value;
}
static class ViewNodeBuilder extends ViewStructure {
/**
* @hide
*/
public static class ViewNodeBuilder extends ViewStructure {
final AssistStructure mAssist;
final ViewNode mNode;
final boolean mAsync;
/**
* Used to instantiate a builder for a stand-alone {@link ViewNode} which is not associated
* to a properly created {@link AssistStructure}.
*/
public ViewNodeBuilder() {
mAssist = new AssistStructure();
mNode = new ViewNode();
mAsync = false;
}
ViewNodeBuilder(AssistStructure assist, ViewNode node, boolean async) {
mAssist = assist;
mNode = node;
mAsync = async;
}
@NonNull
public ViewNode getViewNode() {
return mNode;
}
@Override
public void setId(int id, String packageName, String typeName, String entryName) {
mNode.mId = id;

24
core/java/android/service/autofill/augmented/AugmentedAutofillService.java
View File

@@ -26,6 +26,8 @@ import android.annotation.NonNull;
import android.annotation.Nullable;
import android.annotation.SystemApi;
import android.app.Service;
import android.app.assist.AssistStructure.ViewNode;
import android.app.assist.AssistStructure.ViewNodeParcelable;
import android.content.ComponentName;
import android.content.Intent;
import android.graphics.Rect;
@@ -415,6 +417,8 @@ public abstract class AugmentedAutofillService extends Service {
@GuardedBy("mLock")
private AutofillValue mFocusedValue;
@GuardedBy("mLock")
private ViewNode mFocusedViewNode;
@GuardedBy("mLock")
private IFillCallback mCallback;
/**
@@ -532,6 +536,7 @@ public abstract class AugmentedAutofillService extends Service {
synchronized (mLock) {
mFocusedId = focusedId;
mFocusedValue = focusedValue;
mFocusedViewNode = null;
if (mCallback != null) {
try {
if (!mCallback.isCompleted()) {
@@ -570,6 +575,25 @@ public abstract class AugmentedAutofillService extends Service {
}
}
@Nullable
public ViewNode getFocusedViewNode() {
synchronized (mLock) {
if (mFocusedViewNode == null) {
try {
final ViewNodeParcelable viewNodeParcelable = mClient.getViewNodeParcelable(
mFocusedId);
if (viewNodeParcelable != null) {
mFocusedViewNode = viewNodeParcelable.getViewNode();
}
} catch (RemoteException e) {
Log.e(TAG, "Error getting the ViewNode of the focused view: " + e);
return null;
}
}
return mFocusedViewNode;
}
}
void logEvent(@ReportEvent int event) {
if (sVerbose) Log.v(TAG, "returnAndLogResult(): " + event);
long duration = -1;

17
core/java/android/service/autofill/augmented/FillRequest.java
View File

@@ -19,6 +19,7 @@ import android.annotation.NonNull;
import android.annotation.Nullable;
import android.annotation.SystemApi;
import android.annotation.TestApi;
import android.app.assist.AssistStructure.ViewNode;
import android.content.ComponentName;
import android.service.autofill.augmented.AugmentedAutofillService.AutofillProxy;
import android.view.autofill.AutofillId;
@@ -80,6 +81,14 @@ public final class FillRequest {
return mProxy.getFocusedValue();
}
/**
* Gets the current {@link ViewNode} information of the field that triggered the request.
*/
@Nullable
public ViewNode getFocusedViewNode() {
return mProxy.getFocusedViewNode();
}
/**
* Gets the Smart Suggestions object used to embed the autofill UI.
*
@@ -98,7 +107,7 @@ public final class FillRequest {
// Code below generated by codegen v1.0.14.
// Code below generated by codegen v1.0.22.
//
// DO NOT MODIFY!
// CHECKSTYLE:OFF Generated code
@@ -151,10 +160,10 @@ public final class FillRequest {
}
@DataClass.Generated(
time = 1577399314707L,
codegenVersion = "1.0.14",
time = 1608160139217L,
codegenVersion = "1.0.22",
sourceFile = "frameworks/base/core/java/android/service/autofill/augmented/FillRequest.java",
inputSignatures = "private final @android.annotation.NonNull android.service.autofill.augmented.AugmentedAutofillService.AutofillProxy mProxy\nprivate final @android.annotation.Nullable android.view.inputmethod.InlineSuggestionsRequest mInlineSuggestionsRequest\npublic int getTaskId()\npublic @android.annotation.NonNull android.content.ComponentName getActivityComponent()\npublic @android.annotation.NonNull android.view.autofill.AutofillId getFocusedId()\npublic @android.annotation.NonNull android.view.autofill.AutofillValue getFocusedValue()\npublic @android.annotation.Nullable android.service.autofill.augmented.PresentationParams getPresentationParams()\n java.lang.String proxyToString()\nclass FillRequest extends java.lang.Object implements []\n@com.android.internal.util.DataClass(genToString=true, genBuilder=false, genHiddenConstructor=true)")
inputSignatures = "private final @android.annotation.NonNull android.service.autofill.augmented.AugmentedAutofillService.AutofillProxy mProxy\nprivate final @android.annotation.Nullable android.view.inputmethod.InlineSuggestionsRequest mInlineSuggestionsRequest\npublic int getTaskId()\npublic @android.annotation.NonNull android.content.ComponentName getActivityComponent()\npublic @android.annotation.NonNull android.view.autofill.AutofillId getFocusedId()\npublic @android.annotation.NonNull android.view.autofill.AutofillValue getFocusedValue()\npublic @android.annotation.Nullable android.app.assist.AssistStructure.ViewNode getFocusedViewNode()\npublic @android.annotation.Nullable android.service.autofill.augmented.PresentationParams getPresentationParams()\n java.lang.String proxyToString()\nclass FillRequest extends java.lang.Object implements []\n@com.android.internal.util.DataClass(genToString=true, genBuilder=false, genHiddenConstructor=true)")
@Deprecated
private void __metadata() {}

1
core/java/android/view/autofill/AutofillId.java
View File

@@ -143,6 +143,7 @@ public final class AutofillId implements Parcelable {
*
* @hide
*/
@TestApi
public boolean isNonVirtual() {
return !isVirtualInt() && !isVirtualLong();
}

34
core/java/android/view/autofill/AutofillManager.java
View File

@@ -32,6 +32,9 @@ import android.annotation.RequiresFeature;
import android.annotation.SystemApi;
import android.annotation.SystemService;
import android.annotation.TestApi;
import android.app.assist.AssistStructure.ViewNode;
import android.app.assist.AssistStructure.ViewNodeBuilder;
import android.app.assist.AssistStructure.ViewNodeParcelable;
import android.content.AutofillOptions;
import android.content.ClipData;
import android.content.ComponentName;
@@ -64,6 +67,8 @@ import android.view.Choreographer;
import android.view.ContentInfo;
import android.view.KeyEvent;
import android.view.View;
import android.view.ViewRootImpl;
import android.view.WindowManager;
import android.view.accessibility.AccessibilityEvent;
import android.view.accessibility.AccessibilityManager;
import android.view.accessibility.AccessibilityNodeInfo;
@@ -3581,6 +3586,35 @@ public final class AutofillManager {
mAfm = new WeakReference<>(autofillManager);
}
@Nullable
@Override
public ViewNodeParcelable getViewNodeParcelable(@NonNull AutofillId id) {
final AutofillManager afm = mAfm.get();
if (afm == null) return null;
final View view = getView(afm, id);
if (view == null) {
Log.w(TAG, "getViewNodeParcelable(" + id + "): could not find view");
return null;
}
final ViewRootImpl root = view.getViewRootImpl();
if (root != null
&& (root.getWindowFlags() & WindowManager.LayoutParams.FLAG_SECURE) == 0) {
ViewNodeBuilder viewStructure = new ViewNodeBuilder();
viewStructure.setAutofillId(view.getAutofillId());
view.onProvideAutofillStructure(viewStructure, /* flags= */ 0);
// TODO(b/141703532): We don't call View#onProvideAutofillVirtualStructure for
// efficiency reason. But this also means we will return null for virtual views
// for now. We will add a new API to fetch the view node info of the virtual
// child view.
ViewNode viewNode = viewStructure.getViewNode();
if (viewNode != null && id.equals(viewNode.getAutofillId())) {
return new ViewNodeParcelable(viewNode);
}
}
return null;
}
@Override
public Rect getViewCoordinates(@NonNull AutofillId id) {
final AutofillManager afm = mAfm.get();

6
core/java/android/view/autofill/IAugmentedAutofillManagerClient.aidl
View File

@@ -18,6 +18,7 @@ package android.view.autofill;
import java.util.List;
import android.app.assist.AssistStructure;
import android.graphics.Rect;
import android.view.autofill.AutofillId;
import android.view.autofill.AutofillValue;
@@ -35,6 +36,11 @@ interface IAugmentedAutofillManagerClient {
*/
Rect getViewCoordinates(in AutofillId id);
/**
* Gets the autofill view structure of the input field view.
*/
AssistStructure.ViewNodeParcelable getViewNodeParcelable(in AutofillId id);
/**
* Autofills the activity with the contents of the values.
*/

43
core/tests/coretests/src/android/app/assist/AssistStructureTest.java
View File

@@ -23,11 +23,14 @@ import static android.view.View.IMPORTANT_FOR_AUTOFILL_YES;
import static com.google.common.truth.Truth.assertThat;
import android.app.assist.AssistStructure.ViewNode;
import android.app.assist.AssistStructure.ViewNodeBuilder;
import android.app.assist.AssistStructure.ViewNodeParcelable;
import android.content.Context;
import android.os.Parcel;
import android.os.SystemClock;
import android.text.InputFilter;
import android.util.Log;
import android.view.View;
import android.view.autofill.AutofillId;
import android.widget.EditText;
import android.widget.FrameLayout;
@@ -219,6 +222,28 @@ public class AssistStructureTest {
}
}
@Test
public void testViewNodeParcelableForAutofill() {
Log.d(TAG, "Adding view with " + BIG_VIEW_SIZE + " chars");
View view = newBigView();
mActivity.addView(view);
waitUntilViewsAreLaidOff();
assertThat(view.getViewRootImpl()).isNotNull();
ViewNodeBuilder viewStructure = new ViewNodeBuilder();
viewStructure.setAutofillId(view.getAutofillId());
view.onProvideAutofillStructure(viewStructure, /* flags= */ 0);
ViewNodeParcelable viewNodeParcelable = new ViewNodeParcelable(viewStructure.getViewNode());
// Check properties on "original" view node.
assertBigView(viewNodeParcelable.getViewNode());
// Check properties on "cloned" view node.
ViewNodeParcelable clone = cloneThroughParcel(viewNodeParcelable);
assertBigView(clone.getViewNode());
}
private EditText newSmallView() {
EditText view = new EditText(mContext);
view.setText("I AM GROOT");
@@ -272,6 +297,24 @@ public class AssistStructureTest {
assertThat(hint.charAt(BIG_VIEW_SIZE - 1)).isEqualTo(BIG_VIEW_CHAR);
}
private ViewNodeParcelable cloneThroughParcel(ViewNodeParcelable viewNodeParcelable) {
Parcel parcel = Parcel.obtain();
try {
// Write to parcel
parcel.setDataPosition(0); // Validity Check
viewNodeParcelable.writeToParcel(parcel, NO_FLAGS);
// Read from parcel
parcel.setDataPosition(0);
ViewNodeParcelable clone = ViewNodeParcelable.CREATOR.createFromParcel(parcel);
assertThat(clone).isNotNull();
return clone;
} finally {
parcel.recycle();
}
}
private AssistStructure cloneThroughParcel(AssistStructure structure) {
Parcel parcel = Parcel.obtain();