NOTE: the important for content capture values were the first flags added and are set in + * the rightmost position, so we don't need to shift them + */ + private static final int PFLAG4_IMPORTANT_FOR_CONTENT_CAPTURE_MASK = + IMPORTANT_FOR_CONTENT_CAPTURE_AUTO | IMPORTANT_FOR_CONTENT_CAPTURE_YES + | IMPORTANT_FOR_CONTENT_CAPTURE_NO + | IMPORTANT_FOR_CONTENT_CAPTURE_YES_EXCLUDE_DESCENDANTS + | IMPORTANT_FOR_CONTENT_CAPTURE_NO_EXCLUDE_DESCENDANTS; + + /* + * Variables used to control when the IntelligenceManager.notifyNodeAdded()/removed() methods + * should be called. + * + * The idea is to call notifyNodeAdded() after the view is layout and visible, then call + * notifyNodeRemoved() when it's gone (without known when it was removed from the parent). + * + * TODO(b/111276913): the current algortighm could probably be optimized and some of them + * removed + */ + private static final int PFLAG4_NOTIFIED_CONTENT_CAPTURE_ON_LAYOUT = 0x10; + private static final int PFLAG4_NOTIFIED_CONTENT_CAPTURE_ADDED = 0x20; + private static final int PFLAG4_LAST_CONTENT_CAPTURE_NOTIFICATION_TYPE = 0x40; + + private static final int CONTENT_CAPTURE_NOTIFICATION_TYPE_APPEARED = 1; + private static final int CONTENT_CAPTURE_NOTIFICATION_TYPE_DISAPPEARED = 0; + + /** @hide */ + @IntDef(flag = true, prefix = { "CONTENT_CAPTURE_NOTIFICATION_TYPE_" }, value = { + CONTENT_CAPTURE_NOTIFICATION_TYPE_APPEARED, + CONTENT_CAPTURE_NOTIFICATION_TYPE_DISAPPEARED + }) + @Retention(RetentionPolicy.SOURCE) + public @interface ContentCaptureNotificationType {} + + /* End of masks for mPrivateFlags4 */ + /** * Always allow a user to over-scroll this view, provided it is a * view that can scroll. @@ -3861,6 +4010,8 @@ public class View implements Drawable.Callback, KeyEvent.Callback, @UnsupportedAppUsage int mPrivateFlags3; + private int mPrivateFlags4; + /** * This view's request for the visibility of the status bar. * @hide @@ -5803,6 +5954,7 @@ public class View implements Drawable.Callback, KeyEvent.Callback, mAttributes = trimmed; } + @Override public String toString() { StringBuilder out = new StringBuilder(128); out.append(getClass().getName()); @@ -5875,6 +6027,9 @@ public class View implements Drawable.Callback, KeyEvent.Callback, } } } + if (mAutofillId != null) { + out.append(" aid="); out.append(mAutofillId); + } out.append("}"); return out.toString(); } @@ -7888,7 +8043,8 @@ public class View implements Drawable.Callback, KeyEvent.Callback, * fills in all data that can be inferred from the view itself. */ public void onProvideStructure(ViewStructure structure) { - onProvideStructureForAssistOrAutofill(structure, false, 0); + onProvideStructureForAssistOrAutofillOrViewCapture(structure, /* forAutofill = */ false, + /* forViewCapture= */ false, /* flags= */ 0); } /** @@ -7961,11 +8117,46 @@ public class View implements Drawable.Callback, KeyEvent.Callback, * @see #AUTOFILL_FLAG_INCLUDE_NOT_IMPORTANT_VIEWS */ public void onProvideAutofillStructure(ViewStructure structure, @AutofillFlags int flags) { - onProvideStructureForAssistOrAutofill(structure, true, flags); + onProvideStructureForAssistOrAutofillOrViewCapture(structure, /* forAutofill = */ true, + /* forViewCapture= */ false, flags); } - private void onProvideStructureForAssistOrAutofill(ViewStructure structure, - boolean forAutofill, @AutofillFlags int flags) { + /** + * Populates a {@link ViewStructure} for Content Capture. + * + *
This method is called after a view is that is eligible for Content Capture + * (for example, if it {@link #isImportantForAutofill()}, an intelligence service is enabled for + * the user, and the activity rendering the view is enabled for Content Capture) is laid out and + * is visible. + * + *
Note: the following methods of the {@code structure} will be ignored: + *
See {@link #setImportantForContentCapture(int)} and + * {@link #isImportantForContentCapture()} for more info about this mode. + * + * @return {@link #IMPORTANT_FOR_CONTENT_CAPTURE_AUTO} by default, or value passed to + * {@link #setImportantForContentCapture(int)}. + * + * @attr ref android.R.styleable#View_importantForContentCapture + */ + @ViewDebug.ExportedProperty(mapping = { + @ViewDebug.IntToString(from = IMPORTANT_FOR_CONTENT_CAPTURE_AUTO, to = "auto"), + @ViewDebug.IntToString(from = IMPORTANT_FOR_CONTENT_CAPTURE_YES, to = "yes"), + @ViewDebug.IntToString(from = IMPORTANT_FOR_CONTENT_CAPTURE_NO, to = "no"), + @ViewDebug.IntToString(from = IMPORTANT_FOR_CONTENT_CAPTURE_YES_EXCLUDE_DESCENDANTS, + to = "yesExcludeDescendants"), + @ViewDebug.IntToString(from = IMPORTANT_FOR_CONTENT_CAPTURE_NO_EXCLUDE_DESCENDANTS, + to = "noExcludeDescendants")}) + public @ContentCaptureImportance int getImportantForContentCapture() { + // NOTE: the important for content capture values were the first flags added and are set in + // the rightmost position, so we don't need to shift them + return mPrivateFlags4 & PFLAG4_IMPORTANT_FOR_CONTENT_CAPTURE_MASK; + } + + /** + * Sets the mode for determining whether this view is considered important for content capture. + * + *
The platform determines the importance for autofill automatically but you + * can use this method to customize the behavior. Typically, a view that provides text should + * be marked as {@link #IMPORTANT_FOR_CONTENT_CAPTURE_YES}. + * + * @param mode {@link #IMPORTANT_FOR_CONTENT_CAPTURE_AUTO}, + * {@link #IMPORTANT_FOR_CONTENT_CAPTURE_YES}, {@link #IMPORTANT_FOR_CONTENT_CAPTURE_NO}, + * {@link #IMPORTANT_FOR_CONTENT_CAPTURE_YES_EXCLUDE_DESCENDANTS}, + * or {@link #IMPORTANT_FOR_CONTENT_CAPTURE_NO_EXCLUDE_DESCENDANTS}. + * + * @attr ref android.R.styleable#View_importantForContentCapture + */ + public void setImportantForContentCapture(@ContentCaptureImportance int mode) { + // Reset first + mPrivateFlags4 &= ~PFLAG4_IMPORTANT_FOR_CONTENT_CAPTURE_MASK; + // Then set again + // NOTE: the important for content capture values were the first flags added and are set in + // the rightmost position, so we don't need to shift them + mPrivateFlags4 |= (mode & PFLAG4_IMPORTANT_FOR_CONTENT_CAPTURE_MASK); + } + + /** + * Hints the Android System whether this view is considered important for Content Capture, based + * on the value explicitly set by {@link #setImportantForContentCapture(int)} and heuristics + * when it's {@link #IMPORTANT_FOR_CONTENT_CAPTURE_AUTO}. + * + * @return whether the view is considered important for autofill. + * + * @see #setImportantForContentCapture(int) + * @see #IMPORTANT_FOR_CONTENT_CAPTURE_AUTO + * @see #IMPORTANT_FOR_CONTENT_CAPTURE_YES + * @see #IMPORTANT_FOR_CONTENT_CAPTURE_NO + * @see #IMPORTANT_FOR_CONTENT_CAPTURE_YES_EXCLUDE_DESCENDANTS + * @see #IMPORTANT_FOR_CONTENT_CAPTURE_NO_EXCLUDE_DESCENDANTS + */ + public final boolean isImportantForContentCapture() { + // Check parent mode to ensure we're important + ViewParent parent = mParent; + while (parent instanceof View) { + final int parentImportance = ((View) parent).getImportantForContentCapture(); + if (parentImportance == IMPORTANT_FOR_CONTENT_CAPTURE_NO_EXCLUDE_DESCENDANTS + || parentImportance == IMPORTANT_FOR_CONTENT_CAPTURE_YES_EXCLUDE_DESCENDANTS) { + if (Log.isLoggable(CONTENT_CAPTURE_LOG_TAG, Log.VERBOSE)) { + Log.v(CONTENT_CAPTURE_LOG_TAG, "View (" + this + ") is not important for " + + "content capture because parent " + parent + "'s importance is " + + parentImportance); + } + return false; + } + parent = parent.getParent(); + } + + final int importance = getImportantForContentCapture(); + + // First, check the explicit states. + if (importance == IMPORTANT_FOR_CONTENT_CAPTURE_YES_EXCLUDE_DESCENDANTS + || importance == IMPORTANT_FOR_CONTENT_CAPTURE_YES) { + return true; + } + if (importance == IMPORTANT_FOR_CONTENT_CAPTURE_NO_EXCLUDE_DESCENDANTS + || importance == IMPORTANT_FOR_CONTENT_CAPTURE_NO) { + if (Log.isLoggable(CONTENT_CAPTURE_LOG_TAG, Log.VERBOSE)) { + Log.v(CONTENT_CAPTURE_LOG_TAG, "View (" + this + ") is not important for content " + + "capture because its importance is " + importance); + } + return false; + } + + // Then use some heuristics to handle AUTO. + if (importance != IMPORTANT_FOR_CONTENT_CAPTURE_AUTO) { + Log.w(CONTENT_CAPTURE_LOG_TAG, "invalid content capture importance (" + importance + + " on view " + this); + return false; + } + + // View group is important if at least one children also is + //TODO(b/111276913): decide if we really need to send the relevant parents or just the + // leaves (with absolute coordinates). If it's the latter, then we need to update this + // javadoc and ViewGroup's implementation. + if (this instanceof ViewGroup) { + final ViewGroup group = (ViewGroup) this; + for (int i = 0; i < group.getChildCount(); i++) { + final View child = group.getChildAt(i); + if (child.isImportantForContentCapture()) { + return true; + } + } + } + + // If the app developer explicitly set hints or autofill hintsfor it, it's important. + if (getAutofillHints() != null) { + return true; + } + + // Otherwise, assume it's not important... + return false; + } + + /** + * Helper used to notify the {@link IntelligenceManager}anager when the view is removed or + * added, based on whether it's laid out and visible, and without knowing if the parent removed + * it from the view + * hierarchy. + */ + // TODO(b/111276913): make sure the current algorithm covers all cases. For example, it should + // probably be called every time notifyEnterOrExitForAutoFillIfNeeded() is called as well. + private void notifyNodeAddedOrRemovedForContentCaptureIfNeeded( + @ContentCaptureNotificationType int type) { + if (type != CONTENT_CAPTURE_NOTIFICATION_TYPE_APPEARED + && type != CONTENT_CAPTURE_NOTIFICATION_TYPE_DISAPPEARED) { + // Sanity check so it does not screw up the flags + Log.wtf(CONTENT_CAPTURE_LOG_TAG, "notifyNodeAddedOrRemovedForContentCaptureIfNeeded(): " + + "invalid type " + type + " for " + this); + return; + } + + if (!isImportantForContentCapture()) return; + + final IntelligenceManager im = mContext.getSystemService(IntelligenceManager.class); + if (im == null || !im.isContentCaptureEnabled()) return; + + // Make sure event is notified just once, and reset the + // PFLAG4_LAST_CONTENT_CAPTURE_NOTIFICATION_TYPE flag + boolean ignoreNotification = false; + if (type == CONTENT_CAPTURE_NOTIFICATION_TYPE_APPEARED) { + if ((mPrivateFlags4 & PFLAG4_LAST_CONTENT_CAPTURE_NOTIFICATION_TYPE) + == CONTENT_CAPTURE_NOTIFICATION_TYPE_APPEARED) { + ignoreNotification = true; + } else { + mPrivateFlags4 |= PFLAG4_LAST_CONTENT_CAPTURE_NOTIFICATION_TYPE; + } + } else { + if ((mPrivateFlags4 & PFLAG4_LAST_CONTENT_CAPTURE_NOTIFICATION_TYPE) + == CONTENT_CAPTURE_NOTIFICATION_TYPE_DISAPPEARED) { + ignoreNotification = true; + } else { + mPrivateFlags4 &= ~PFLAG4_LAST_CONTENT_CAPTURE_NOTIFICATION_TYPE; + } + } + if (ignoreNotification) { + if (Log.isLoggable(CONTENT_CAPTURE_LOG_TAG, Log.VERBOSE)) { + // TODO(b/111276913): remove this log statement if the algorithm is not improved + // (right now it's called too many times when the activity is stopped and/or views + // disappear + Log.v(CONTENT_CAPTURE_LOG_TAG, "notifyNodeAddedOrRemovedForContentCaptureIfNeeded(" + + type + "): ignoring repeated notification on " + this); + } + return; + } + + if (type == CONTENT_CAPTURE_NOTIFICATION_TYPE_APPEARED) { + final ViewStructure structure = im.newViewStructure(this); + boolean notifyMgr = onProvideContentCaptureStructure(structure, /* flags= */ 0); + if (notifyMgr) { + im.notifyViewAppeared(structure); + } + mPrivateFlags4 |= PFLAG4_NOTIFIED_CONTENT_CAPTURE_ADDED; + } else { + if ((mPrivateFlags4 & PFLAG4_NOTIFIED_CONTENT_CAPTURE_ADDED) == 0) { + return; // skip initial notification + } + im.notifyViewDisappeared(getAutofillId()); + } + } + @Nullable private AutofillManager getAutofillManager() { return mContext.getSystemService(AutofillManager.class); @@ -13094,6 +13483,9 @@ public class View implements Drawable.Callback, KeyEvent.Callback, : AccessibilityEvent.CONTENT_CHANGE_TYPE_PANE_DISAPPEARED); } } + notifyNodeAddedOrRemovedForContentCaptureIfNeeded(isVisible + ? CONTENT_CAPTURE_NOTIFICATION_TYPE_APPEARED + : CONTENT_CAPTURE_NOTIFICATION_TYPE_DISAPPEARED); } /** @@ -18630,6 +19022,8 @@ public class View implements Drawable.Callback, KeyEvent.Callback, } notifyEnterOrExitForAutoFillIfNeeded(false); + notifyNodeAddedOrRemovedForContentCaptureIfNeeded( + CONTENT_CAPTURE_NOTIFICATION_TYPE_DISAPPEARED); } /** @@ -20934,6 +21328,13 @@ public class View implements Drawable.Callback, KeyEvent.Callback, mPrivateFlags3 &= ~PFLAG3_NOTIFY_AUTOFILL_ENTER_ON_LAYOUT; notifyEnterOrExitForAutoFillIfNeeded(true); } + + if ((mViewFlags & VISIBILITY_MASK) == VISIBLE + && (mPrivateFlags4 & PFLAG4_NOTIFIED_CONTENT_CAPTURE_ON_LAYOUT) == 0) { + notifyNodeAddedOrRemovedForContentCaptureIfNeeded( + CONTENT_CAPTURE_NOTIFICATION_TYPE_APPEARED); + mPrivateFlags4 |= PFLAG4_NOTIFIED_CONTENT_CAPTURE_ON_LAYOUT; + } } private boolean hasParentWantsFocus() { diff --git a/core/java/android/view/ViewStructure.java b/core/java/android/view/ViewStructure.java index 38dcdd30d843e..6efb6f38d1185 100644 --- a/core/java/android/view/ViewStructure.java +++ b/core/java/android/view/ViewStructure.java @@ -24,7 +24,6 @@ import android.os.Bundle; import android.os.LocaleList; import android.util.Pair; import android.view.View.AutofillImportance; -import android.view.ViewStructure.HtmlInfo; import android.view.autofill.AutofillId; import android.view.autofill.AutofillValue; diff --git a/core/java/android/view/intelligence/ContentCaptureEvent.java b/core/java/android/view/intelligence/ContentCaptureEvent.java index 2530ae3b31240..befcb55b1f731 100644 --- a/core/java/android/view/intelligence/ContentCaptureEvent.java +++ b/core/java/android/view/intelligence/ContentCaptureEvent.java @@ -16,12 +16,17 @@ package android.view.intelligence; import android.annotation.IntDef; +import android.annotation.NonNull; import android.annotation.Nullable; import android.annotation.SystemApi; import android.os.Parcel; import android.os.Parcelable; +import android.os.SystemClock; import android.view.autofill.AutofillId; +import com.android.internal.util.Preconditions; + +import java.io.PrintWriter; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; @@ -60,14 +65,14 @@ public final class ContentCaptureEvent implements Parcelable { * *
The metadata of the node is available through {@link #getViewNode()}. */ - public static final int TYPE_VIEW_ADDED = 5; + public static final int TYPE_VIEW_APPEARED = 5; /** * Called when a node has been removed from the screen and is not visible to the user anymore. * *
The id of the node is available through {@link #getId()}. */ - public static final int TYPE_VIEW_REMOVED = 6; + public static final int TYPE_VIEW_DISAPPEARED = 6; /** * Called when the text of a node has been changed. @@ -85,8 +90,8 @@ public final class ContentCaptureEvent implements Parcelable { TYPE_ACTIVITY_PAUSED, TYPE_ACTIVITY_RESUMED, TYPE_ACTIVITY_STOPPED, - TYPE_VIEW_ADDED, - TYPE_VIEW_REMOVED, + TYPE_VIEW_APPEARED, + TYPE_VIEW_DISAPPEARED, TYPE_VIEW_TEXT_CHANGED }) @Retention(RetentionPolicy.SOURCE) @@ -95,7 +100,9 @@ public final class ContentCaptureEvent implements Parcelable { private final int mType; private final long mEventTime; private final int mFlags; - + private @Nullable AutofillId mId; + private @Nullable ViewNode mNode; + private @Nullable CharSequence mText; /** @hide */ public ContentCaptureEvent(int type, long eventTime, int flags) { @@ -104,12 +111,42 @@ public final class ContentCaptureEvent implements Parcelable { mFlags = flags; } + + /** @hide */ + public ContentCaptureEvent(int type, int flags) { + this(type, SystemClock.uptimeMillis(), flags); + } + + /** @hide */ + public ContentCaptureEvent(int type) { + this(type, /* flags= */ 0); + } + + /** @hide */ + public ContentCaptureEvent setAutofillId(@NonNull AutofillId id) { + mId = Preconditions.checkNotNull(id); + return this; + } + + /** @hide */ + public ContentCaptureEvent setViewNode(@NonNull ViewNode node) { + mNode = Preconditions.checkNotNull(node); + return this; + } + + /** @hide */ + public ContentCaptureEvent setText(@Nullable CharSequence text) { + mText = text; + return this; + } + /** * Gets the type of the event. * * @return one of {@link #TYPE_ACTIVITY_STARTED}, {@link #TYPE_ACTIVITY_RESUMED}, * {@link #TYPE_ACTIVITY_PAUSED}, {@link #TYPE_ACTIVITY_STOPPED}, - * {@link #TYPE_VIEW_ADDED}, {@link #TYPE_VIEW_REMOVED}, or {@link #TYPE_VIEW_TEXT_CHANGED}. + * {@link #TYPE_VIEW_APPEARED}, {@link #TYPE_VIEW_DISAPPEARED}, + * or {@link #TYPE_VIEW_TEXT_CHANGED}. */ public @EventType int getType() { return mType; @@ -135,21 +172,21 @@ public final class ContentCaptureEvent implements Parcelable { /** * Gets the whole metadata of the node associated with the event. * - *
Only set on {@link #TYPE_VIEW_ADDED} events. + *
Only set on {@link #TYPE_VIEW_APPEARED} events. */ @Nullable public ViewNode getViewNode() { - return null; + return mNode; } /** * Gets the {@link AutofillId} of the node associated with the event. * - *
Only set on {@link #TYPE_VIEW_REMOVED} and {@link #TYPE_VIEW_TEXT_CHANGED} events. + *
Only set on {@link #TYPE_VIEW_DISAPPEARED} and {@link #TYPE_VIEW_TEXT_CHANGED} events.
*/
@Nullable
public AutofillId getId() {
- return null;
+ return mId;
}
/**
@@ -159,16 +196,41 @@ public final class ContentCaptureEvent implements Parcelable {
*/
@Nullable
public CharSequence getText() {
- return null;
+ return mText;
+ }
+
+ /** @hide */
+ public void dump(@NonNull PrintWriter pw) {
+ pw.print("type="); pw.print(getTypeAsString(mType));
+ pw.print(", time="); pw.print(mEventTime);
+ if (mFlags > 0) {
+ pw.print(", flags="); pw.print(mFlags);
+ }
+ if (mId != null) {
+ pw.print(", id="); pw.print(mId);
+ }
+ if (mNode != null) {
+ pw.print(", id="); pw.print(mNode.getAutofillId());
+ }
}
@Override
public String toString() {
final StringBuilder string = new StringBuilder("ContentCaptureEvent[type=")
- .append(getTypeAsString(mType)).append(", time=").append(mEventTime);
+ .append(getTypeAsString(mType));
if (mFlags > 0) {
string.append(", flags=").append(mFlags);
}
+ if (mId != null) {
+ string.append(", id=").append(mId);
+ }
+ if (mNode != null) {
+ final String className = mNode.getClassName();
+ if (mNode != null) {
+ string.append(", class=").append(className);
+ }
+ string.append(", id=").append(mNode.getAutofillId());
+ }
return string.append(']').toString();
}
@@ -182,6 +244,9 @@ public final class ContentCaptureEvent implements Parcelable {
parcel.writeInt(mType);
parcel.writeLong(mEventTime);
parcel.writeInt(mFlags);
+ parcel.writeParcelable(mId, flags);
+ ViewNode.writeToParcel(parcel, mNode, flags);
+ parcel.writeCharSequence(mText);
}
public static final Parcelable.Creator If the session is not started after the limit is reached, it's discarded.
+ */
+ private static final int MAX_DELAYED_SIZE = 20;
+
private final Context mContext;
@Nullable
@@ -99,10 +117,24 @@ public final class IntelligenceManager {
@GuardedBy("mLock")
private ComponentName mComponentName;
+ // TODO(b/111276913): create using maximum batch size as capacity
+ /**
+ * List of events held to be sent as a batch.
+ */
+ @GuardedBy("mLock")
+ private final ArrayList Typically called "manually" by views that handle their own virtual view hierarchy, or
+ * automatically by the Android System for views that return {@code true} on
+ * {@link View#onProvideContentCaptureStructure(ViewStructure, int)}.
+ *
+ * @param node node that has been added.
+ */
+ public void notifyViewAppeared(@NonNull ViewStructure node) {
+ Preconditions.checkNotNull(node);
+ if (!isContentCaptureEnabled()) return;
+
+ if (!(node instanceof ViewNode.ViewStructureImpl)) {
+ throw new IllegalArgumentException("Invalid node class: " + node.getClass());
+ }
+ sendEvent(new ContentCaptureEvent(TYPE_VIEW_APPEARED)
+ .setViewNode(((ViewNode.ViewStructureImpl) node).mNode));
+ }
+
+ /**
+ * Notifies the Intelligence Service that a node has been removed from the view structure.
+ *
+ * Typically called "manually" by views that handle their own virtual view hierarchy, or
+ * automatically by the Android System for standard views.
+ *
+ * @param id id of the node that has been removed.
+ */
+ public void notifyViewDisappeared(@NonNull AutofillId id) {
+ Preconditions.checkNotNull(id);
+ if (!isContentCaptureEnabled()) return;
+
+ sendEvent(new ContentCaptureEvent(TYPE_VIEW_DISAPPEARED).setAutofillId(id));
+ }
+
+ /**
+ * Notifies the Intelligence Service that the value of a text node has been changed.
+ *
+ * @param id of the node.
+ * @param text new text.
+ * @param flags either {@code 0} or {@link #FLAG_USER_INPUT} when the value was explicitly
+ * changed by the user (for example, through the keyboard).
+ */
+ public void notifyViewTextChanged(@NonNull AutofillId id, @Nullable CharSequence text,
+ int flags) {
+ Preconditions.checkNotNull(id);
+ if (!isContentCaptureEnabled()) return;
+
+ sendEvent(new ContentCaptureEvent(TYPE_VIEW_TEXT_CHANGED, flags).setAutofillId(id)
+ .setText(text));
+ }
+
+ /**
+ * Creates a {@link ViewStructure} for a "standard" view.
+ *
+ * @hide
+ */
+ @NonNull
+ public ViewStructure newViewStructure(@NonNull View view) {
+ return new ViewNode.ViewStructureImpl(view);
+ }
+
+ /**
+ * Creates a {@link ViewStructure} for a "virtual" view, so it can be passed to
+ * {@link #notifyViewAppeared(ViewStructure)} by the view managing the virtual view hierarchy.
+ *
+ * @param parentId id of the virtual view parent (it can be obtained by calling
+ * {@link ViewStructure#getAutofillId()} on the parent).
+ * @param virtualId id of the virtual child, relative to the parent.
+ *
+ * @return a new {@link ViewStructure} that can be used for Content Capture purposes.
+ */
+ @NonNull
+ public ViewStructure newVirtualViewStructure(@NonNull AutofillId parentId, int virtualId) {
+ return new ViewNode.ViewStructureImpl(parentId, virtualId);
+ }
+
/**
* Returns the component name of the {@code android.service.intelligence.IntelligenceService}
* that is enabled for the current user.
@@ -322,15 +475,29 @@ public final class IntelligenceManager {
pw.print(prefix2); pw.print("enabled: "); pw.println(isContentCaptureEnabled());
pw.print(prefix2); pw.print("id: "); pw.println(mId);
pw.print(prefix2); pw.print("state: "); pw.print(mState); pw.print(" (");
- pw.print(getStateAsStringLocked()); pw.println(")");
- pw.print(prefix2); pw.print("appToken: "); pw.println(mApplicationToken);
- pw.print(prefix2); pw.print("componentName: "); pw.println(mComponentName);
+ pw.print(getStateAsString(mState)); pw.println(")");
+ pw.print(prefix2); pw.print("app token: "); pw.println(mApplicationToken);
+ pw.print(prefix2); pw.print("component name: ");
+ pw.println(mComponentName == null ? "null" : mComponentName.flattenToShortString());
+ final int numberEvents = mEvents.size();
+ pw.print(prefix2); pw.print("batched events: "); pw.println(numberEvents);
+ if (numberEvents > 0) {
+ for (int i = 0; i < numberEvents; i++) {
+ final ContentCaptureEvent event = mEvents.get(i);
+ pw.println(i); pw.print(": "); event.dump(pw); pw.println();
+ }
+
+ }
}
}
+ /**
+ * Gets a string that can be used to identify the activity on logging statements.
+ */
@GuardedBy("mLock")
- private String getStateAsStringLocked() {
- return getStateAsString(mState);
+ private String getActivityDebugNameLocked() {
+ return mComponentName == null ? mContext.getPackageName()
+ : mComponentName.flattenToShortString();
}
@NonNull
diff --git a/core/java/android/view/intelligence/ViewNode.java b/core/java/android/view/intelligence/ViewNode.java
index 357ecf599f7a2..cc78e6b4aa6de 100644
--- a/core/java/android/view/intelligence/ViewNode.java
+++ b/core/java/android/view/intelligence/ViewNode.java
@@ -15,10 +15,24 @@
*/
package android.view.intelligence;
+import android.annotation.NonNull;
import android.annotation.Nullable;
import android.annotation.SystemApi;
import android.app.assist.AssistStructure;
+import android.graphics.Matrix;
+import android.graphics.Rect;
+import android.os.Bundle;
+import android.os.LocaleList;
+import android.os.Parcel;
+import android.util.Log;
+import android.view.View;
+import android.view.ViewParent;
+import android.view.ViewStructure;
+import android.view.ViewStructure.HtmlInfo.Builder;
import android.view.autofill.AutofillId;
+import android.view.autofill.AutofillValue;
+
+import com.android.internal.util.Preconditions;
//TODO(b/111276913): add javadocs / implement Parcelable / implement
//TODO(b/111276913): for now it's extending ViewNode directly as it needs most of its properties,
@@ -28,6 +42,16 @@ import android.view.autofill.AutofillId;
@SystemApi
public final class ViewNode extends AssistStructure.ViewNode {
+ private static final String TAG = "ViewNode";
+
+ private AutofillId mParentAutofillId;
+
+ // TODO(b/111276913): temporarily setting some fields here while they're not accessible from the
+ // superclass
+ private AutofillId mAutofillId;
+ private CharSequence mText;
+ private String mClassName;
+
/** @hide */
public ViewNode() {
}
@@ -38,7 +62,343 @@ public final class ViewNode extends AssistStructure.ViewNode {
*/
@Nullable
public AutofillId getParentAutofillId() {
- //TODO(b/111276913): implement
- return null;
+ return mParentAutofillId;
+ }
+
+ // TODO(b/111276913): temporarily overwriting some methods
+ @Override
+ public AutofillId getAutofillId() {
+ return mAutofillId;
+ }
+ @Override
+ public CharSequence getText() {
+ return mText;
+ }
+ @Override
+ public String getClassName() {
+ return mClassName;
+ }
+
+ /** @hide */
+ public static void writeToParcel(@NonNull Parcel parcel, @Nullable ViewNode node, int flags) {
+ if (node == null) {
+ parcel.writeParcelable(null, flags);
+ return;
+ }
+ parcel.writeParcelable(node.mAutofillId, flags);
+ parcel.writeParcelable(node.mParentAutofillId, flags);
+ parcel.writeCharSequence(node.mText);
+ parcel.writeString(node.mClassName);
+ }
+
+ /** @hide */
+ public static @Nullable ViewNode readFromParcel(@NonNull Parcel parcel) {
+ final AutofillId id = parcel.readParcelable(null);
+ if (id == null) return null;
+
+ final ViewNode node = new ViewNode();
+
+ node.mAutofillId = id;
+ node.mParentAutofillId = parcel.readParcelable(null);
+ node.mText = parcel.readCharSequence();
+ node.mClassName = parcel.readString();
+
+ return node;
+ }
+
+ /** @hide */
+ static final class ViewStructureImpl extends ViewStructure {
+
+ final ViewNode mNode = new ViewNode();
+
+ ViewStructureImpl(@NonNull View view) {
+ mNode.mAutofillId = Preconditions.checkNotNull(view).getAutofillId();
+ final ViewParent parent = view.getParent();
+ if (parent instanceof View) {
+ mNode.mParentAutofillId = ((View) parent).getAutofillId();
+ }
+ }
+
+ ViewStructureImpl(@NonNull AutofillId parentId, int virtualId) {
+ mNode.mParentAutofillId = Preconditions.checkNotNull(parentId);
+ mNode.mAutofillId = new AutofillId(parentId, virtualId);
+ }
+
+ @Override
+ public void setId(int id, String packageName, String typeName, String entryName) {
+ // TODO(b/111276913): implement or move to superclass
+ }
+
+ @Override
+ public void setDimens(int left, int top, int scrollX, int scrollY, int width, int height) {
+ // TODO(b/111276913): implement or move to superclass
+ }
+
+ @Override
+ public void setTransformation(Matrix matrix) {
+ // TODO(b/111276913): implement or move to superclass
+ }
+
+ @Override
+ public void setElevation(float elevation) {
+ // TODO(b/111276913): implement or move to superclass
+ }
+
+ @Override
+ public void setAlpha(float alpha) {
+ // TODO(b/111276913): implement or move to superclass
+ }
+
+ @Override
+ public void setVisibility(int visibility) {
+ // TODO(b/111276913): implement or move to superclass
+ }
+
+ @Override
+ public void setAssistBlocked(boolean state) {
+ // TODO(b/111276913): implement or move to superclass
+ }
+
+ @Override
+ public void setEnabled(boolean state) {
+ // TODO(b/111276913): implement or move to superclass
+ }
+
+ @Override
+ public void setClickable(boolean state) {
+ // TODO(b/111276913): implement or move to superclass
+ }
+
+ @Override
+ public void setLongClickable(boolean state) {
+ // TODO(b/111276913): implement or move to superclass
+ }
+
+ @Override
+ public void setContextClickable(boolean state) {
+ // TODO(b/111276913): implement or move to superclass
+ }
+
+ @Override
+ public void setFocusable(boolean state) {
+ // TODO(b/111276913): implement or move to superclass
+ }
+
+ @Override
+ public void setFocused(boolean state) {
+ // TODO(b/111276913): implement or move to superclass
+ }
+
+ @Override
+ public void setAccessibilityFocused(boolean state) {
+ // TODO(b/111276913): implement or move to superclass
+ }
+
+ @Override
+ public void setCheckable(boolean state) {
+ // TODO(b/111276913): implement or move to superclass
+ }
+
+ @Override
+ public void setChecked(boolean state) {
+ // TODO(b/111276913): implement or move to superclass
+ }
+
+ @Override
+ public void setSelected(boolean state) {
+ // TODO(b/111276913): implement or move to superclass
+ }
+
+ @Override
+ public void setActivated(boolean state) {
+ // TODO(b/111276913): implement or move to superclass
+ }
+
+ @Override
+ public void setOpaque(boolean opaque) {
+ // TODO(b/111276913): implement or move to superclass
+ }
+
+ @Override
+ public void setClassName(String className) {
+ // TODO(b/111276913): temporarily setting directly; should be done on superclass instead
+ mNode.mClassName = className;
+ }
+
+ @Override
+ public void setContentDescription(CharSequence contentDescription) {
+ // TODO(b/111276913): implement or move to superclass
+ }
+
+ @Override
+ public void setText(CharSequence text) {
+ // TODO(b/111276913): temporarily setting directly; should be done on superclass instead
+ mNode.mText = text;
+ }
+
+ @Override
+ public void setText(CharSequence text, int selectionStart, int selectionEnd) {
+ // TODO(b/111276913): implement or move to superclass
+ }
+
+ @Override
+ public void setTextStyle(float size, int fgColor, int bgColor, int style) {
+ // TODO(b/111276913): implement or move to superclass
+ }
+
+ @Override
+ public void setTextLines(int[] charOffsets, int[] baselines) {
+ // TODO(b/111276913): implement or move to superclass
+ }
+
+ @Override
+ public void setHint(CharSequence hint) {
+ // TODO(b/111276913): implement or move to superclass
+ }
+
+ @Override
+ public CharSequence getText() {
+ // TODO(b/111276913): temporarily getting directly; should be done on superclass instead
+ return mNode.mText;
+ }
+
+ @Override
+ public int getTextSelectionStart() {
+ // TODO(b/111276913): implement or move to superclass
+ return 0;
+ }
+
+ @Override
+ public int getTextSelectionEnd() {
+ // TODO(b/111276913): implement or move to superclass
+ return 0;
+ }
+
+ @Override
+ public CharSequence getHint() {
+ // TODO(b/111276913): implement or move to superclass
+ return null;
+ }
+
+ @Override
+ public Bundle getExtras() {
+ // TODO(b/111276913): implement or move to superclass
+ return null;
+ }
+
+ @Override
+ public boolean hasExtras() {
+ // TODO(b/111276913): implement or move to superclass
+ return false;
+ }
+
+ @Override
+ public void setChildCount(int num) {
+ Log.w(TAG, "setChildCount() is not supported");
+ }
+
+ @Override
+ public int addChildCount(int num) {
+ Log.w(TAG, "addChildCount() is not supported");
+ return 0;
+ }
+
+ @Override
+ public int getChildCount() {
+ Log.w(TAG, "getChildCount() is not supported");
+ return 0;
+ }
+
+ @Override
+ public ViewStructure newChild(int index) {
+ Log.w(TAG, "newChild() is not supported");
+ return null;
+ }
+
+ @Override
+ public ViewStructure asyncNewChild(int index) {
+ Log.w(TAG, "asyncNewChild() is not supported");
+ return null;
+ }
+
+ @Override
+ public AutofillId getAutofillId() {
+ // TODO(b/111276913): temporarily getting directly; should be done on superclass instead
+ return mNode.mAutofillId;
+ }
+
+ @Override
+ public void setAutofillId(AutofillId id) {
+ // TODO(b/111276913): temporarily setting directly; should be done on superclass instead
+ mNode.mAutofillId = id;
+ }
+
+ @Override
+ public void setAutofillId(AutofillId parentId, int virtualId) {
+ // TODO(b/111276913): temporarily setting directly; should be done on superclass instead
+ mNode.mAutofillId = new AutofillId(parentId, virtualId);
+ }
+
+ @Override
+ public void setAutofillType(int type) {
+ // TODO(b/111276913): implement or move to superclass
+ }
+
+ @Override
+ public void setAutofillHints(String[] hint) {
+ // TODO(b/111276913): implement or move to superclass
+ }
+
+ @Override
+ public void setAutofillValue(AutofillValue value) {
+ // TODO(b/111276913): implement or move to superclass
+ }
+
+ @Override
+ public void setAutofillOptions(CharSequence[] options) {
+ // TODO(b/111276913): implement or move to superclass
+ }
+
+ @Override
+ public void setInputType(int inputType) {
+ // TODO(b/111276913): implement or move to superclass
+ }
+
+ @Override
+ public void setDataIsSensitive(boolean sensitive) {
+ // TODO(b/111276913): implement or move to superclass
+ }
+
+ @Override
+ public void asyncCommit() {
+ Log.w(TAG, "asyncCommit() is not supported");
+ }
+
+ @Override
+ public Rect getTempRect() {
+ // TODO(b/111276913): implement or move to superclass
+ return null;
+ }
+
+ @Override
+ public void setWebDomain(String domain) {
+ Log.w(TAG, "setWebDomain() is not supported");
+ }
+
+ @Override
+ public void setLocaleList(LocaleList localeList) {
+ // TODO(b/111276913): implement or move to superclass
+ }
+
+ @Override
+ public Builder newHtmlInfoBuilder(String tagName) {
+ Log.w(TAG, "newHtmlInfoBuilder() is not supported");
+ return null;
+ }
+
+ @Override
+ public void setHtmlInfo(HtmlInfo htmlInfo) {
+ Log.w(TAG, "setHtmlInfo() is not supported");
+ }
}
}
diff --git a/core/java/android/widget/Switch.java b/core/java/android/widget/Switch.java
index 8bfc151b1a839..d55c09f7fcb60 100644
--- a/core/java/android/widget/Switch.java
+++ b/core/java/android/widget/Switch.java
@@ -1422,17 +1422,24 @@ public class Switch extends CompoundButton {
@Override
public void onProvideStructure(ViewStructure structure) {
super.onProvideStructure(structure);
- onProvideAutoFillStructureForAssistOrAutofill(structure);
+ onProvideStructureForAssistOrAutofillOrViewCapture(structure);
}
@Override
public void onProvideAutofillStructure(ViewStructure structure, int flags) {
super.onProvideAutofillStructure(structure, flags);
- onProvideAutoFillStructureForAssistOrAutofill(structure);
+ onProvideStructureForAssistOrAutofillOrViewCapture(structure);
}
- // NOTE: currently there is no difference for Assist or AutoFill, so it doesn't take flags
- private void onProvideAutoFillStructureForAssistOrAutofill(ViewStructure structure) {
+ @Override
+ public boolean onProvideContentCaptureStructure(ViewStructure structure, int flags) {
+ final boolean notifyManager = super.onProvideContentCaptureStructure(structure, flags);
+ onProvideStructureForAssistOrAutofillOrViewCapture(structure);
+ return notifyManager;
+ }
+
+ // NOTE: currently there is no difference for any type, so it doesn't take flags
+ private void onProvideStructureForAssistOrAutofillOrViewCapture(ViewStructure structure) {
CharSequence switchText = isChecked() ? mTextOn : mTextOff;
if (!TextUtils.isEmpty(switchText)) {
CharSequence oldText = structure.getText();
diff --git a/core/java/android/widget/TextView.java b/core/java/android/widget/TextView.java
index 572670fc662b4..3bdd7b8e91be6 100644
--- a/core/java/android/widget/TextView.java
+++ b/core/java/android/widget/TextView.java
@@ -166,6 +166,7 @@ import android.view.inputmethod.ExtractedText;
import android.view.inputmethod.ExtractedTextRequest;
import android.view.inputmethod.InputConnection;
import android.view.inputmethod.InputMethodManager;
+import android.view.intelligence.IntelligenceManager;
import android.view.textclassifier.TextClassification;
import android.view.textclassifier.TextClassificationContext;
import android.view.textclassifier.TextClassificationManager;
@@ -948,6 +949,9 @@ public class TextView extends View implements ViewTreeObserver.OnPreDrawListener
if (getImportantForAutofill() == IMPORTANT_FOR_AUTOFILL_AUTO) {
setImportantForAutofill(IMPORTANT_FOR_AUTOFILL_YES);
}
+ if (getImportantForContentCapture() == IMPORTANT_FOR_CONTENT_CAPTURE_AUTO) {
+ setImportantForContentCapture(IMPORTANT_FOR_CONTENT_CAPTURE_YES);
+ }
setTextInternal("");
@@ -6072,7 +6076,7 @@ public class TextView extends View implements ViewTreeObserver.OnPreDrawListener
if (needEditableForNotification) {
sendAfterTextChanged((Editable) text);
} else {
- notifyAutoFillManagerAfterTextChanged();
+ notifyManagersAfterTextChanged();
}
// SelectionModifierCursorController depends on textCanBeSelected, which depends on text
@@ -10121,23 +10125,33 @@ public class TextView extends View implements ViewTreeObserver.OnPreDrawListener
}
// Always notify AutoFillManager - it will return right away if autofill is disabled.
- notifyAutoFillManagerAfterTextChanged();
+ notifyManagersAfterTextChanged();
hideErrorIfUnchanged();
}
- private void notifyAutoFillManagerAfterTextChanged() {
- // It is important to not check whether the view is important for autofill
- // since the user can trigger autofill manually on not important views.
- if (!isAutofillable()) {
- return;
- }
- final AutofillManager afm = mContext.getSystemService(AutofillManager.class);
- if (afm != null) {
- if (android.view.autofill.Helper.sVerbose) {
- Log.v(LOG_TAG, "notifyAutoFillManagerAfterTextChanged");
+ private void notifyManagersAfterTextChanged() {
+
+ // Autofill
+ if (isAutofillable()) {
+ // It is important to not check whether the view is important for autofill
+ // since the user can trigger autofill manually on not important views.
+ final AutofillManager afm = mContext.getSystemService(AutofillManager.class);
+ if (afm != null) {
+ if (android.view.autofill.Helper.sVerbose) {
+ Log.v(LOG_TAG, "notifyAutoFillManagerAfterTextChanged");
+ }
+ afm.notifyValueChanged(TextView.this);
+ }
+ }
+
+ // ContentCapture
+ if (isImportantForContentCapture() && isTextEditable()) {
+ final IntelligenceManager im = mContext.getSystemService(IntelligenceManager.class);
+ if (im != null && im.isContentCaptureEnabled()) {
+ // TODO(b/111276913): pass flags when edited by user / add CTS test
+ im.notifyViewTextChanged(getAutofillId(), getText(), /* flags= */ 0);
}
- afm.notifyValueChanged(TextView.this);
}
}
@@ -10900,21 +10914,33 @@ public class TextView extends View implements ViewTreeObserver.OnPreDrawListener
@Override
public void onProvideStructure(ViewStructure structure) {
super.onProvideStructure(structure);
- onProvideAutoStructureForAssistOrAutofill(structure, false);
+ onProvideStructureForAssistOrAutofillOrViewCapture(structure, /* forAutofill = */ false,
+ /* forViewCapture= */ false);
}
@Override
public void onProvideAutofillStructure(ViewStructure structure, int flags) {
super.onProvideAutofillStructure(structure, flags);
- onProvideAutoStructureForAssistOrAutofill(structure, true);
+ onProvideStructureForAssistOrAutofillOrViewCapture(structure, /* forAutofill = */ true,
+ /* forViewCapture= */ false);
}
- private void onProvideAutoStructureForAssistOrAutofill(ViewStructure structure,
- boolean forAutofill) {
+ @Override
+ public boolean onProvideContentCaptureStructure(ViewStructure structure, int flags) {
+ final boolean notifyManager = super.onProvideContentCaptureStructure(structure, flags);
+ onProvideStructureForAssistOrAutofillOrViewCapture(structure, /* forAutofill = */ false,
+ /* forViewCapture= */ true);
+ return notifyManager;
+ }
+
+ private void onProvideStructureForAssistOrAutofillOrViewCapture(ViewStructure structure,
+ boolean forAutofill, boolean forViewCapture) {
final boolean isPassword = hasPasswordTransformationMethod()
|| isPasswordInputType(getInputType());
- if (forAutofill) {
- structure.setDataIsSensitive(!mTextSetFromXmlOrResourceId);
+ if (forAutofill || forViewCapture) {
+ if (forAutofill) {
+ structure.setDataIsSensitive(!mTextSetFromXmlOrResourceId);
+ }
if (mTextId != ResourceId.ID_NULL) {
try {
structure.setTextIdEntry(getResources().getResourceEntryName(mTextId));
@@ -10927,7 +10953,7 @@ public class TextView extends View implements ViewTreeObserver.OnPreDrawListener
}
}
- if (!isPassword || forAutofill) {
+ if (!isPassword || forAutofill || forViewCapture) {
if (mLayout == null) {
assumeLayout();
}
@@ -11043,7 +11069,8 @@ public class TextView extends View implements ViewTreeObserver.OnPreDrawListener
// of the View (and can be any drawable) or a BackgroundColorSpan inside the text.
structure.setTextStyle(getTextSize(), getCurrentTextColor(),
AssistStructure.ViewNode.TEXT_COLOR_UNDEFINED /* bgColor */, style);
- } else {
+ }
+ if (forAutofill || forViewCapture) {
structure.setMinTextEms(getMinEms());
structure.setMaxTextEms(getMaxEms());
int maxLength = -1;
diff --git a/core/res/res/values/attrs.xml b/core/res/res/values/attrs.xml
index 68ec34229d482..a99b9421e3b3d 100644
--- a/core/res/res/values/attrs.xml
+++ b/core/res/res/values/attrs.xml
@@ -2448,6 +2448,25 @@