From 88eae3bc69435ba251843b824bbabd5f46f87196 Mon Sep 17 00:00:00 2001 From: Felipe Leme Date: Wed, 7 Nov 2018 15:11:56 -0800 Subject: [PATCH] Added moar ContentCapture APIs (and their initial implementation). There are 4 new APIs on View: - boolean setImportantForContentCapture() - boolean getImportantForContentCapture() - boolean isImportantForContentCapture() - boolean onProvideContentCaptureStructure() And 4 on IntelligenceManager: - void notifyViewAppeared() - void notifyViewDisappeared() - void notifyViewTextChanged() - ViewStructure newVirtualViewStructure() These methods are similar to the equivalent methods that are used for Autofill and/or Assist, except for the following differences: - The view hierarchy nodes are reported as they are rendered, rather than at once in a tree, recursively. Hence, the ViewStructure implementation does not implement the methods that add children to it, and views that provide virtual hierarchies must manually call IntelligenceManager to create the ViewStructure to their children and notify when their children are added and removed. - It does not support methods added for Autofill to handle HTML pages (such as setHtmlInfo() and setWewbDomain()), as they're not important in the Content Capture context. - Similarly, it also does not support setDataIsSensitive(), because the Intelligence service does not have the same restrictions as the Autofill service. The CL also provides the initial implementation of these APIs, although still full of TODOs (for example, we're not holding the events to send as a batch yet). Test: m -j update-api doc-comment-check-docs Bug: 117944706 Change-Id: I43f06ce82bfe3b14d8d13fb3b2ebee223db83284 --- api/current.txt | 14 + api/system-current.txt | 4 +- api/test-current.txt | 6 +- core/java/android/view/View.java | 429 +++++++++++++++++- core/java/android/view/ViewStructure.java | 1 - .../intelligence/ContentCaptureEvent.java | 110 ++++- .../intelligence/IntelligenceManager.java | 249 ++++++++-- .../android/view/intelligence/ViewNode.java | 364 ++++++++++++++- core/java/android/widget/Switch.java | 15 +- core/java/android/widget/TextView.java | 69 ++- core/res/res/values/attrs.xml | 19 + core/res/res/values/public.xml | 1 + 12 files changed, 1175 insertions(+), 106 deletions(-) diff --git a/api/current.txt b/api/current.txt index 533c70f9221cc..aa73481f35de0 100755 --- a/api/current.txt +++ b/api/current.txt @@ -740,6 +740,7 @@ package android { field public static final int immersive = 16843456; // 0x10102c0 field public static final int importantForAccessibility = 16843690; // 0x10103aa field public static final int importantForAutofill = 16844120; // 0x1010558 + field public static final int importantForContentCapture = 16844182; // 0x1010596 field public static final int inAnimation = 16843127; // 0x1010177 field public static final int includeFontPadding = 16843103; // 0x101015f field public static final int includeInGlobalSearch = 16843374; // 0x101026e @@ -48785,6 +48786,7 @@ package android.view { method public int getId(); method public int getImportantForAccessibility(); method public int getImportantForAutofill(); + method public int getImportantForContentCapture(); method public boolean getKeepScreenOn(); method public android.view.KeyEvent.DispatcherState getKeyDispatcherState(); method public int getLabelFor(); @@ -48918,6 +48920,7 @@ package android.view { method public boolean isHovered(); method public boolean isImportantForAccessibility(); method public final boolean isImportantForAutofill(); + method public final boolean isImportantForContentCapture(); method public boolean isInEditMode(); method public boolean isInLayout(); method public boolean isInTouchMode(); @@ -48992,6 +48995,7 @@ package android.view { method public void onPopulateAccessibilityEvent(android.view.accessibility.AccessibilityEvent); method public void onProvideAutofillStructure(android.view.ViewStructure, int); method public void onProvideAutofillVirtualStructure(android.view.ViewStructure, int); + method public boolean onProvideContentCaptureStructure(android.view.ViewStructure, int); method public void onProvideStructure(android.view.ViewStructure); method public void onProvideVirtualStructure(android.view.ViewStructure); method public android.view.PointerIcon onResolvePointerIcon(android.view.MotionEvent, int); @@ -49110,6 +49114,7 @@ package android.view { method public void setId(int); method public void setImportantForAccessibility(int); method public void setImportantForAutofill(int); + method public void setImportantForContentCapture(int); method public void setKeepScreenOn(boolean); method public void setKeyboardNavigationCluster(boolean); method public void setLabelFor(int); @@ -49280,6 +49285,11 @@ package android.view { field public static final int IMPORTANT_FOR_AUTOFILL_NO_EXCLUDE_DESCENDANTS = 8; // 0x8 field public static final int IMPORTANT_FOR_AUTOFILL_YES = 1; // 0x1 field public static final int IMPORTANT_FOR_AUTOFILL_YES_EXCLUDE_DESCENDANTS = 4; // 0x4 + field public static final int IMPORTANT_FOR_CONTENT_CAPTURE_AUTO = 0; // 0x0 + field public static final int IMPORTANT_FOR_CONTENT_CAPTURE_NO = 2; // 0x2 + field public static final int IMPORTANT_FOR_CONTENT_CAPTURE_NO_EXCLUDE_DESCENDANTS = 8; // 0x8 + field public static final int IMPORTANT_FOR_CONTENT_CAPTURE_YES = 1; // 0x1 + field public static final int IMPORTANT_FOR_CONTENT_CAPTURE_YES_EXCLUDE_DESCENDANTS = 4; // 0x4 field public static final int INVISIBLE = 4; // 0x4 field public static final int KEEP_SCREEN_ON = 67108864; // 0x4000000 field public static final int LAYER_TYPE_HARDWARE = 2; // 0x2 @@ -51734,6 +51744,10 @@ package android.view.intelligence { method public void disableContentCapture(); method public android.content.ComponentName getIntelligenceServiceComponentName(); method public boolean isContentCaptureEnabled(); + method public android.view.ViewStructure newVirtualViewStructure(android.view.autofill.AutofillId, int); + method public void notifyViewAppeared(android.view.ViewStructure); + method public void notifyViewDisappeared(android.view.autofill.AutofillId); + method public void notifyViewTextChanged(android.view.autofill.AutofillId, java.lang.CharSequence, int); field public static final int FLAG_USER_INPUT = 1; // 0x1 } diff --git a/api/system-current.txt b/api/system-current.txt index 576df26859926..7b8e7a12d262f 100644 --- a/api/system-current.txt +++ b/api/system-current.txt @@ -6994,8 +6994,8 @@ package android.view.intelligence { field public static final int TYPE_ACTIVITY_RESUMED = 2; // 0x2 field public static final int TYPE_ACTIVITY_STARTED = 1; // 0x1 field public static final int TYPE_ACTIVITY_STOPPED = 4; // 0x4 - field public static final int TYPE_VIEW_ADDED = 5; // 0x5 - field public static final int TYPE_VIEW_REMOVED = 6; // 0x6 + field public static final int TYPE_VIEW_APPEARED = 5; // 0x5 + field public static final int TYPE_VIEW_DISAPPEARED = 6; // 0x6 field public static final int TYPE_VIEW_TEXT_CHANGED = 7; // 0x7 } diff --git a/api/test-current.txt b/api/test-current.txt index 8b8c5426f195a..88f4556602de5 100644 --- a/api/test-current.txt +++ b/api/test-current.txt @@ -1637,9 +1637,9 @@ package android.view { } public abstract interface WindowManager implements android.view.ViewManager { - method public abstract void setShouldShowIme(int, boolean); - method public abstract void setShouldShowWithInsecureKeyguard(int, boolean); - method public abstract void setShouldShowSystemDecors(int, boolean); + method public default void setShouldShowIme(int, boolean); + method public default void setShouldShowSystemDecors(int, boolean); + method public default void setShouldShowWithInsecureKeyguard(int, boolean); } public static class WindowManager.LayoutParams extends android.view.ViewGroup.LayoutParams implements android.os.Parcelable { diff --git a/core/java/android/view/View.java b/core/java/android/view/View.java index 453d7885f4d2f..d7440e896573c 100644 --- a/core/java/android/view/View.java +++ b/core/java/android/view/View.java @@ -112,6 +112,7 @@ import android.view.autofill.AutofillValue; import android.view.inputmethod.EditorInfo; import android.view.inputmethod.InputConnection; import android.view.inputmethod.InputMethodManager; +import android.view.intelligence.IntelligenceManager; import android.widget.Checkable; import android.widget.FrameLayout; import android.widget.ScrollBarDrawable; @@ -798,6 +799,11 @@ public class View implements Drawable.Callback, KeyEvent.Callback, // set if a session is not started. private static final String AUTOFILL_LOG_TAG = "View.Autofill"; + /** + * The logging tag used by this class when logging content capture-related messages. + */ + private static final String CONTENT_CAPTURE_LOG_TAG = "View.ContentCapture"; + /** * When set to true, apps will draw debugging information about their layouts. * @@ -1337,6 +1343,59 @@ public class View implements Drawable.Callback, KeyEvent.Callback, */ public static final int AUTOFILL_FLAG_INCLUDE_NOT_IMPORTANT_VIEWS = 0x1; + /** @hide */ + @IntDef(prefix = { "IMPORTANT_FOR_CONTENT_CAPTURE_" }, value = { + IMPORTANT_FOR_CONTENT_CAPTURE_AUTO, + IMPORTANT_FOR_CONTENT_CAPTURE_YES, + IMPORTANT_FOR_CONTENT_CAPTURE_NO, + IMPORTANT_FOR_CONTENT_CAPTURE_YES_EXCLUDE_DESCENDANTS, + IMPORTANT_FOR_CONTENT_CAPTURE_NO_EXCLUDE_DESCENDANTS + }) + @Retention(RetentionPolicy.SOURCE) + public @interface ContentCaptureImportance {} + + /** + * Automatically determine whether a view is important for content capture. + * + * @see #isImportantForContentCapture() + * @see #setImportantForContentCapture(int) + */ + public static final int IMPORTANT_FOR_CONTENT_CAPTURE_AUTO = 0x0; + + /** + * The view is important for content capture, and its children (if any) will be traversed. + * + * @see #isImportantForContentCapture() + * @see #setImportantForContentCapture(int) + */ + public static final int IMPORTANT_FOR_CONTENT_CAPTURE_YES = 0x1; + + /** + * The view is not important for content capture, but its children (if any) will be traversed. + * + * @see #isImportantForContentCapture() + * @see #setImportantForContentCapture(int) + */ + public static final int IMPORTANT_FOR_CONTENT_CAPTURE_NO = 0x2; + + /** + * The view is important for content capture, but its children (if any) will not be traversed. + * + * @see #isImportantForContentCapture() + * @see #setImportantForContentCapture(int) + */ + public static final int IMPORTANT_FOR_CONTENT_CAPTURE_YES_EXCLUDE_DESCENDANTS = 0x4; + + /** + * The view is not important for content capture, and its children (if any) will not be + * traversed. + * + * @see #isImportantForContentCapture() + * @see #setImportantForContentCapture(int) + */ + public static final int IMPORTANT_FOR_CONTENT_CAPTURE_NO_EXCLUDE_DESCENDANTS = 0x8; + + /** * This view is enabled. Interpretation varies by subclass. * Use with ENABLED_MASK when calling setFlags. @@ -2243,7 +2302,44 @@ public class View implements Drawable.Callback, KeyEvent.Callback, @UnsupportedAppUsage protected Object mTag = null; - // for mPrivateFlags: + /* + * Masks for mPrivateFlags, as generated by dumpFlags(): + * + * |-------|-------|-------|-------| + * 1 PFLAG_WANTS_FOCUS + * 1 PFLAG_FOCUSED + * 1 PFLAG_SELECTED + * 1 PFLAG_IS_ROOT_NAMESPACE + * 1 PFLAG_HAS_BOUNDS + * 1 PFLAG_DRAWN + * 1 PFLAG_DRAW_ANIMATION + * 1 PFLAG_SKIP_DRAW + * 1 PFLAG_REQUEST_TRANSPARENT_REGIONS + * 1 PFLAG_DRAWABLE_STATE_DIRTY + * 1 PFLAG_MEASURED_DIMENSION_SET + * 1 PFLAG_FORCE_LAYOUT + * 1 PFLAG_LAYOUT_REQUIRED + * 1 PFLAG_PRESSED + * 1 PFLAG_DRAWING_CACHE_VALID + * 1 PFLAG_ANIMATION_STARTED + * 1 PFLAG_SAVE_STATE_CALLED + * 1 PFLAG_ALPHA_SET + * 1 PFLAG_SCROLL_CONTAINER + * 1 PFLAG_SCROLL_CONTAINER_ADDED + * 1 PFLAG_DIRTY + * 1 PFLAG_DIRTY_MASK + * 1 PFLAG_OPAQUE_BACKGROUND + * 1 PFLAG_OPAQUE_SCROLLBARS + * 11 PFLAG_OPAQUE_MASK + * 1 PFLAG_PREPRESSED + * 1 PFLAG_CANCEL_NEXT_UP_EVENT + * 1 PFLAG_AWAKEN_SCROLL_BARS_ON_ATTACH + * 1 PFLAG_HOVERED + * 1 PFLAG_NOTIFY_AUTOFILL_MANAGER_ON_CLICK + * 1 PFLAG_ACTIVATED + * 1 PFLAG_INVALIDATED + * |-------|-------|-------|-------| + */ /** {@hide} */ static final int PFLAG_WANTS_FOCUS = 0x00000001; /** {@hide} */ @@ -2393,7 +2489,9 @@ public class View implements Drawable.Callback, KeyEvent.Callback, */ static final int PFLAG_INVALIDATED = 0x80000000; - /** + /* End of masks for mPrivateFlags */ + + /* * Masks for mPrivateFlags2, as generated by dumpFlags(): * * |-------|-------|-------|-------| @@ -2934,7 +3032,7 @@ public class View implements Drawable.Callback, KeyEvent.Callback, /* End of masks for mPrivateFlags2 */ - /** + /* * Masks for mPrivateFlags3, as generated by dumpFlags(): * * |-------|-------|-------|-------| @@ -3270,6 +3368,57 @@ public class View implements Drawable.Callback, KeyEvent.Callback, /* End of masks for mPrivateFlags3 */ + /* + * Masks for mPrivateFlags4, as generated by dumpFlags(): + * + * |-------|-------|-------|-------| + * 1111 PFLAG4_IMPORTANT_FOR_CONTENT_CAPTURE_MASK + * 1 PFLAG4_NOTIFIED_CONTENT_CAPTURE_ON_LAYOUT + * 1 PFLAG4_NOTIFIED_CONTENT_CAPTURE_ADDED + * 1 PFLAG4_LAST_CONTENT_CAPTURE_NOTIFICATION_TYPE + * |-------|-------|-------|-------| + */ + + /** + * Mask for obtaining the bits which specify how to determine + * whether a view is important for autofill. + * + *

NOTE: the important for content capture values were the first flags added and are set in + * the rightmost position, so we don't need to shift them + */ + private static final int PFLAG4_IMPORTANT_FOR_CONTENT_CAPTURE_MASK = + IMPORTANT_FOR_CONTENT_CAPTURE_AUTO | IMPORTANT_FOR_CONTENT_CAPTURE_YES + | IMPORTANT_FOR_CONTENT_CAPTURE_NO + | IMPORTANT_FOR_CONTENT_CAPTURE_YES_EXCLUDE_DESCENDANTS + | IMPORTANT_FOR_CONTENT_CAPTURE_NO_EXCLUDE_DESCENDANTS; + + /* + * Variables used to control when the IntelligenceManager.notifyNodeAdded()/removed() methods + * should be called. + * + * The idea is to call notifyNodeAdded() after the view is layout and visible, then call + * notifyNodeRemoved() when it's gone (without known when it was removed from the parent). + * + * TODO(b/111276913): the current algortighm could probably be optimized and some of them + * removed + */ + private static final int PFLAG4_NOTIFIED_CONTENT_CAPTURE_ON_LAYOUT = 0x10; + private static final int PFLAG4_NOTIFIED_CONTENT_CAPTURE_ADDED = 0x20; + private static final int PFLAG4_LAST_CONTENT_CAPTURE_NOTIFICATION_TYPE = 0x40; + + private static final int CONTENT_CAPTURE_NOTIFICATION_TYPE_APPEARED = 1; + private static final int CONTENT_CAPTURE_NOTIFICATION_TYPE_DISAPPEARED = 0; + + /** @hide */ + @IntDef(flag = true, prefix = { "CONTENT_CAPTURE_NOTIFICATION_TYPE_" }, value = { + CONTENT_CAPTURE_NOTIFICATION_TYPE_APPEARED, + CONTENT_CAPTURE_NOTIFICATION_TYPE_DISAPPEARED + }) + @Retention(RetentionPolicy.SOURCE) + public @interface ContentCaptureNotificationType {} + + /* End of masks for mPrivateFlags4 */ + /** * Always allow a user to over-scroll this view, provided it is a * view that can scroll. @@ -3861,6 +4010,8 @@ public class View implements Drawable.Callback, KeyEvent.Callback, @UnsupportedAppUsage int mPrivateFlags3; + private int mPrivateFlags4; + /** * This view's request for the visibility of the status bar. * @hide @@ -5803,6 +5954,7 @@ public class View implements Drawable.Callback, KeyEvent.Callback, mAttributes = trimmed; } + @Override public String toString() { StringBuilder out = new StringBuilder(128); out.append(getClass().getName()); @@ -5875,6 +6027,9 @@ public class View implements Drawable.Callback, KeyEvent.Callback, } } } + if (mAutofillId != null) { + out.append(" aid="); out.append(mAutofillId); + } out.append("}"); return out.toString(); } @@ -7888,7 +8043,8 @@ public class View implements Drawable.Callback, KeyEvent.Callback, * fills in all data that can be inferred from the view itself. */ public void onProvideStructure(ViewStructure structure) { - onProvideStructureForAssistOrAutofill(structure, false, 0); + onProvideStructureForAssistOrAutofillOrViewCapture(structure, /* forAutofill = */ false, + /* forViewCapture= */ false, /* flags= */ 0); } /** @@ -7961,11 +8117,46 @@ public class View implements Drawable.Callback, KeyEvent.Callback, * @see #AUTOFILL_FLAG_INCLUDE_NOT_IMPORTANT_VIEWS */ public void onProvideAutofillStructure(ViewStructure structure, @AutofillFlags int flags) { - onProvideStructureForAssistOrAutofill(structure, true, flags); + onProvideStructureForAssistOrAutofillOrViewCapture(structure, /* forAutofill = */ true, + /* forViewCapture= */ false, flags); } - private void onProvideStructureForAssistOrAutofill(ViewStructure structure, - boolean forAutofill, @AutofillFlags int flags) { + /** + * Populates a {@link ViewStructure} for Content Capture. + * + *

This method is called after a view is that is eligible for Content Capture + * (for example, if it {@link #isImportantForAutofill()}, an intelligence service is enabled for + * the user, and the activity rendering the view is enabled for Content Capture) is laid out and + * is visible. + * + *

Note: the following methods of the {@code structure} will be ignored: + *

+ * + * @return whether the IntelligenceService should be notified that the view was added (through + * the {@link IntelligenceManager#notifyViewAppeared(ViewStructure)} method) to the view + * hierarchy. Most views should return {@code true} here, but views that contains virtual + * hierarchy might opt to return {@code false} and notify the manager independently, as the + * virtual views are rendered. + */ + public boolean onProvideContentCaptureStructure(@NonNull ViewStructure structure, int flags) { + onProvideStructureForAssistOrAutofillOrViewCapture(structure, /* forAutofill = */ false, + /* forViewCapture= */ true, flags); + return true; + } + + private void onProvideStructureForAssistOrAutofillOrViewCapture(ViewStructure structure, + boolean forAutofill, boolean forViewCapture, @AutofillFlags int flags) { final int id = mID; if (id != NO_ID && !isViewIdGenerated(id)) { String pkg, type, entry; @@ -7981,8 +8172,11 @@ public class View implements Drawable.Callback, KeyEvent.Callback, } else { structure.setId(id, null, null, null); } + if (forViewCapture) { + structure.setDataIsSensitive(false); + } - if (forAutofill) { + if (forAutofill || forViewCapture) { final @AutofillType int autofillType = getAutofillType(); // Don't need to fill autofill info if view does not support it. // For example, only TextViews that are editable support autofill @@ -8530,9 +8724,8 @@ public class View implements Drawable.Callback, KeyEvent.Callback, if (parentImportance == IMPORTANT_FOR_AUTOFILL_NO_EXCLUDE_DESCENDANTS || parentImportance == IMPORTANT_FOR_AUTOFILL_YES_EXCLUDE_DESCENDANTS) { if (Log.isLoggable(AUTOFILL_LOG_TAG, Log.VERBOSE)) { - Log.v(AUTOFILL_LOG_TAG, "View (autofillId=" + getAutofillViewId() + ", " - + getClass() + ") is not important for autofill because parent " - + parent + "'s importance is " + parentImportance); + Log.v(AUTOFILL_LOG_TAG, "View (" + this + ") is not important for autofill " + + "because parent " + parent + "'s importance is " + parentImportance); } return false; } @@ -8549,14 +8742,18 @@ public class View implements Drawable.Callback, KeyEvent.Callback, if (importance == IMPORTANT_FOR_AUTOFILL_NO_EXCLUDE_DESCENDANTS || importance == IMPORTANT_FOR_AUTOFILL_NO) { if (Log.isLoggable(AUTOFILL_LOG_TAG, Log.VERBOSE)) { - Log.v(AUTOFILL_LOG_TAG, "View (autofillId=" + getAutofillViewId() + ", " - + getClass() + ") is not important for autofill because its " - + "importance is " + importance); + Log.v(AUTOFILL_LOG_TAG, "View (" + this + ") is not important for autofill " + + "because its importance is " + importance); } return false; } // Then use some heuristics to handle AUTO. + if (importance != IMPORTANT_FOR_AUTOFILL_AUTO) { + Log.w(AUTOFILL_LOG_TAG, "invalid autofill importance (" + importance + " on view " + + this); + return false; + } // Always include views that have an explicit resource id. final int id = mID; @@ -8584,6 +8781,198 @@ public class View implements Drawable.Callback, KeyEvent.Callback, return false; } + /** + * Gets the mode for determining whether this view is important for content capture. + * + *

See {@link #setImportantForContentCapture(int)} and + * {@link #isImportantForContentCapture()} for more info about this mode. + * + * @return {@link #IMPORTANT_FOR_CONTENT_CAPTURE_AUTO} by default, or value passed to + * {@link #setImportantForContentCapture(int)}. + * + * @attr ref android.R.styleable#View_importantForContentCapture + */ + @ViewDebug.ExportedProperty(mapping = { + @ViewDebug.IntToString(from = IMPORTANT_FOR_CONTENT_CAPTURE_AUTO, to = "auto"), + @ViewDebug.IntToString(from = IMPORTANT_FOR_CONTENT_CAPTURE_YES, to = "yes"), + @ViewDebug.IntToString(from = IMPORTANT_FOR_CONTENT_CAPTURE_NO, to = "no"), + @ViewDebug.IntToString(from = IMPORTANT_FOR_CONTENT_CAPTURE_YES_EXCLUDE_DESCENDANTS, + to = "yesExcludeDescendants"), + @ViewDebug.IntToString(from = IMPORTANT_FOR_CONTENT_CAPTURE_NO_EXCLUDE_DESCENDANTS, + to = "noExcludeDescendants")}) + public @ContentCaptureImportance int getImportantForContentCapture() { + // NOTE: the important for content capture values were the first flags added and are set in + // the rightmost position, so we don't need to shift them + return mPrivateFlags4 & PFLAG4_IMPORTANT_FOR_CONTENT_CAPTURE_MASK; + } + + /** + * Sets the mode for determining whether this view is considered important for content capture. + * + *

The platform determines the importance for autofill automatically but you + * can use this method to customize the behavior. Typically, a view that provides text should + * be marked as {@link #IMPORTANT_FOR_CONTENT_CAPTURE_YES}. + * + * @param mode {@link #IMPORTANT_FOR_CONTENT_CAPTURE_AUTO}, + * {@link #IMPORTANT_FOR_CONTENT_CAPTURE_YES}, {@link #IMPORTANT_FOR_CONTENT_CAPTURE_NO}, + * {@link #IMPORTANT_FOR_CONTENT_CAPTURE_YES_EXCLUDE_DESCENDANTS}, + * or {@link #IMPORTANT_FOR_CONTENT_CAPTURE_NO_EXCLUDE_DESCENDANTS}. + * + * @attr ref android.R.styleable#View_importantForContentCapture + */ + public void setImportantForContentCapture(@ContentCaptureImportance int mode) { + // Reset first + mPrivateFlags4 &= ~PFLAG4_IMPORTANT_FOR_CONTENT_CAPTURE_MASK; + // Then set again + // NOTE: the important for content capture values were the first flags added and are set in + // the rightmost position, so we don't need to shift them + mPrivateFlags4 |= (mode & PFLAG4_IMPORTANT_FOR_CONTENT_CAPTURE_MASK); + } + + /** + * Hints the Android System whether this view is considered important for Content Capture, based + * on the value explicitly set by {@link #setImportantForContentCapture(int)} and heuristics + * when it's {@link #IMPORTANT_FOR_CONTENT_CAPTURE_AUTO}. + * + * @return whether the view is considered important for autofill. + * + * @see #setImportantForContentCapture(int) + * @see #IMPORTANT_FOR_CONTENT_CAPTURE_AUTO + * @see #IMPORTANT_FOR_CONTENT_CAPTURE_YES + * @see #IMPORTANT_FOR_CONTENT_CAPTURE_NO + * @see #IMPORTANT_FOR_CONTENT_CAPTURE_YES_EXCLUDE_DESCENDANTS + * @see #IMPORTANT_FOR_CONTENT_CAPTURE_NO_EXCLUDE_DESCENDANTS + */ + public final boolean isImportantForContentCapture() { + // Check parent mode to ensure we're important + ViewParent parent = mParent; + while (parent instanceof View) { + final int parentImportance = ((View) parent).getImportantForContentCapture(); + if (parentImportance == IMPORTANT_FOR_CONTENT_CAPTURE_NO_EXCLUDE_DESCENDANTS + || parentImportance == IMPORTANT_FOR_CONTENT_CAPTURE_YES_EXCLUDE_DESCENDANTS) { + if (Log.isLoggable(CONTENT_CAPTURE_LOG_TAG, Log.VERBOSE)) { + Log.v(CONTENT_CAPTURE_LOG_TAG, "View (" + this + ") is not important for " + + "content capture because parent " + parent + "'s importance is " + + parentImportance); + } + return false; + } + parent = parent.getParent(); + } + + final int importance = getImportantForContentCapture(); + + // First, check the explicit states. + if (importance == IMPORTANT_FOR_CONTENT_CAPTURE_YES_EXCLUDE_DESCENDANTS + || importance == IMPORTANT_FOR_CONTENT_CAPTURE_YES) { + return true; + } + if (importance == IMPORTANT_FOR_CONTENT_CAPTURE_NO_EXCLUDE_DESCENDANTS + || importance == IMPORTANT_FOR_CONTENT_CAPTURE_NO) { + if (Log.isLoggable(CONTENT_CAPTURE_LOG_TAG, Log.VERBOSE)) { + Log.v(CONTENT_CAPTURE_LOG_TAG, "View (" + this + ") is not important for content " + + "capture because its importance is " + importance); + } + return false; + } + + // Then use some heuristics to handle AUTO. + if (importance != IMPORTANT_FOR_CONTENT_CAPTURE_AUTO) { + Log.w(CONTENT_CAPTURE_LOG_TAG, "invalid content capture importance (" + importance + + " on view " + this); + return false; + } + + // View group is important if at least one children also is + //TODO(b/111276913): decide if we really need to send the relevant parents or just the + // leaves (with absolute coordinates). If it's the latter, then we need to update this + // javadoc and ViewGroup's implementation. + if (this instanceof ViewGroup) { + final ViewGroup group = (ViewGroup) this; + for (int i = 0; i < group.getChildCount(); i++) { + final View child = group.getChildAt(i); + if (child.isImportantForContentCapture()) { + return true; + } + } + } + + // If the app developer explicitly set hints or autofill hintsfor it, it's important. + if (getAutofillHints() != null) { + return true; + } + + // Otherwise, assume it's not important... + return false; + } + + /** + * Helper used to notify the {@link IntelligenceManager}anager when the view is removed or + * added, based on whether it's laid out and visible, and without knowing if the parent removed + * it from the view + * hierarchy. + */ + // TODO(b/111276913): make sure the current algorithm covers all cases. For example, it should + // probably be called every time notifyEnterOrExitForAutoFillIfNeeded() is called as well. + private void notifyNodeAddedOrRemovedForContentCaptureIfNeeded( + @ContentCaptureNotificationType int type) { + if (type != CONTENT_CAPTURE_NOTIFICATION_TYPE_APPEARED + && type != CONTENT_CAPTURE_NOTIFICATION_TYPE_DISAPPEARED) { + // Sanity check so it does not screw up the flags + Log.wtf(CONTENT_CAPTURE_LOG_TAG, "notifyNodeAddedOrRemovedForContentCaptureIfNeeded(): " + + "invalid type " + type + " for " + this); + return; + } + + if (!isImportantForContentCapture()) return; + + final IntelligenceManager im = mContext.getSystemService(IntelligenceManager.class); + if (im == null || !im.isContentCaptureEnabled()) return; + + // Make sure event is notified just once, and reset the + // PFLAG4_LAST_CONTENT_CAPTURE_NOTIFICATION_TYPE flag + boolean ignoreNotification = false; + if (type == CONTENT_CAPTURE_NOTIFICATION_TYPE_APPEARED) { + if ((mPrivateFlags4 & PFLAG4_LAST_CONTENT_CAPTURE_NOTIFICATION_TYPE) + == CONTENT_CAPTURE_NOTIFICATION_TYPE_APPEARED) { + ignoreNotification = true; + } else { + mPrivateFlags4 |= PFLAG4_LAST_CONTENT_CAPTURE_NOTIFICATION_TYPE; + } + } else { + if ((mPrivateFlags4 & PFLAG4_LAST_CONTENT_CAPTURE_NOTIFICATION_TYPE) + == CONTENT_CAPTURE_NOTIFICATION_TYPE_DISAPPEARED) { + ignoreNotification = true; + } else { + mPrivateFlags4 &= ~PFLAG4_LAST_CONTENT_CAPTURE_NOTIFICATION_TYPE; + } + } + if (ignoreNotification) { + if (Log.isLoggable(CONTENT_CAPTURE_LOG_TAG, Log.VERBOSE)) { + // TODO(b/111276913): remove this log statement if the algorithm is not improved + // (right now it's called too many times when the activity is stopped and/or views + // disappear + Log.v(CONTENT_CAPTURE_LOG_TAG, "notifyNodeAddedOrRemovedForContentCaptureIfNeeded(" + + type + "): ignoring repeated notification on " + this); + } + return; + } + + if (type == CONTENT_CAPTURE_NOTIFICATION_TYPE_APPEARED) { + final ViewStructure structure = im.newViewStructure(this); + boolean notifyMgr = onProvideContentCaptureStructure(structure, /* flags= */ 0); + if (notifyMgr) { + im.notifyViewAppeared(structure); + } + mPrivateFlags4 |= PFLAG4_NOTIFIED_CONTENT_CAPTURE_ADDED; + } else { + if ((mPrivateFlags4 & PFLAG4_NOTIFIED_CONTENT_CAPTURE_ADDED) == 0) { + return; // skip initial notification + } + im.notifyViewDisappeared(getAutofillId()); + } + } + @Nullable private AutofillManager getAutofillManager() { return mContext.getSystemService(AutofillManager.class); @@ -13094,6 +13483,9 @@ public class View implements Drawable.Callback, KeyEvent.Callback, : AccessibilityEvent.CONTENT_CHANGE_TYPE_PANE_DISAPPEARED); } } + notifyNodeAddedOrRemovedForContentCaptureIfNeeded(isVisible + ? CONTENT_CAPTURE_NOTIFICATION_TYPE_APPEARED + : CONTENT_CAPTURE_NOTIFICATION_TYPE_DISAPPEARED); } /** @@ -18630,6 +19022,8 @@ public class View implements Drawable.Callback, KeyEvent.Callback, } notifyEnterOrExitForAutoFillIfNeeded(false); + notifyNodeAddedOrRemovedForContentCaptureIfNeeded( + CONTENT_CAPTURE_NOTIFICATION_TYPE_DISAPPEARED); } /** @@ -20934,6 +21328,13 @@ public class View implements Drawable.Callback, KeyEvent.Callback, mPrivateFlags3 &= ~PFLAG3_NOTIFY_AUTOFILL_ENTER_ON_LAYOUT; notifyEnterOrExitForAutoFillIfNeeded(true); } + + if ((mViewFlags & VISIBILITY_MASK) == VISIBLE + && (mPrivateFlags4 & PFLAG4_NOTIFIED_CONTENT_CAPTURE_ON_LAYOUT) == 0) { + notifyNodeAddedOrRemovedForContentCaptureIfNeeded( + CONTENT_CAPTURE_NOTIFICATION_TYPE_APPEARED); + mPrivateFlags4 |= PFLAG4_NOTIFIED_CONTENT_CAPTURE_ON_LAYOUT; + } } private boolean hasParentWantsFocus() { diff --git a/core/java/android/view/ViewStructure.java b/core/java/android/view/ViewStructure.java index 38dcdd30d843e..6efb6f38d1185 100644 --- a/core/java/android/view/ViewStructure.java +++ b/core/java/android/view/ViewStructure.java @@ -24,7 +24,6 @@ import android.os.Bundle; import android.os.LocaleList; import android.util.Pair; import android.view.View.AutofillImportance; -import android.view.ViewStructure.HtmlInfo; import android.view.autofill.AutofillId; import android.view.autofill.AutofillValue; diff --git a/core/java/android/view/intelligence/ContentCaptureEvent.java b/core/java/android/view/intelligence/ContentCaptureEvent.java index 2530ae3b31240..befcb55b1f731 100644 --- a/core/java/android/view/intelligence/ContentCaptureEvent.java +++ b/core/java/android/view/intelligence/ContentCaptureEvent.java @@ -16,12 +16,17 @@ package android.view.intelligence; import android.annotation.IntDef; +import android.annotation.NonNull; import android.annotation.Nullable; import android.annotation.SystemApi; import android.os.Parcel; import android.os.Parcelable; +import android.os.SystemClock; import android.view.autofill.AutofillId; +import com.android.internal.util.Preconditions; + +import java.io.PrintWriter; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; @@ -60,14 +65,14 @@ public final class ContentCaptureEvent implements Parcelable { * *

The metadata of the node is available through {@link #getViewNode()}. */ - public static final int TYPE_VIEW_ADDED = 5; + public static final int TYPE_VIEW_APPEARED = 5; /** * Called when a node has been removed from the screen and is not visible to the user anymore. * *

The id of the node is available through {@link #getId()}. */ - public static final int TYPE_VIEW_REMOVED = 6; + public static final int TYPE_VIEW_DISAPPEARED = 6; /** * Called when the text of a node has been changed. @@ -85,8 +90,8 @@ public final class ContentCaptureEvent implements Parcelable { TYPE_ACTIVITY_PAUSED, TYPE_ACTIVITY_RESUMED, TYPE_ACTIVITY_STOPPED, - TYPE_VIEW_ADDED, - TYPE_VIEW_REMOVED, + TYPE_VIEW_APPEARED, + TYPE_VIEW_DISAPPEARED, TYPE_VIEW_TEXT_CHANGED }) @Retention(RetentionPolicy.SOURCE) @@ -95,7 +100,9 @@ public final class ContentCaptureEvent implements Parcelable { private final int mType; private final long mEventTime; private final int mFlags; - + private @Nullable AutofillId mId; + private @Nullable ViewNode mNode; + private @Nullable CharSequence mText; /** @hide */ public ContentCaptureEvent(int type, long eventTime, int flags) { @@ -104,12 +111,42 @@ public final class ContentCaptureEvent implements Parcelable { mFlags = flags; } + + /** @hide */ + public ContentCaptureEvent(int type, int flags) { + this(type, SystemClock.uptimeMillis(), flags); + } + + /** @hide */ + public ContentCaptureEvent(int type) { + this(type, /* flags= */ 0); + } + + /** @hide */ + public ContentCaptureEvent setAutofillId(@NonNull AutofillId id) { + mId = Preconditions.checkNotNull(id); + return this; + } + + /** @hide */ + public ContentCaptureEvent setViewNode(@NonNull ViewNode node) { + mNode = Preconditions.checkNotNull(node); + return this; + } + + /** @hide */ + public ContentCaptureEvent setText(@Nullable CharSequence text) { + mText = text; + return this; + } + /** * Gets the type of the event. * * @return one of {@link #TYPE_ACTIVITY_STARTED}, {@link #TYPE_ACTIVITY_RESUMED}, * {@link #TYPE_ACTIVITY_PAUSED}, {@link #TYPE_ACTIVITY_STOPPED}, - * {@link #TYPE_VIEW_ADDED}, {@link #TYPE_VIEW_REMOVED}, or {@link #TYPE_VIEW_TEXT_CHANGED}. + * {@link #TYPE_VIEW_APPEARED}, {@link #TYPE_VIEW_DISAPPEARED}, + * or {@link #TYPE_VIEW_TEXT_CHANGED}. */ public @EventType int getType() { return mType; @@ -135,21 +172,21 @@ public final class ContentCaptureEvent implements Parcelable { /** * Gets the whole metadata of the node associated with the event. * - *

Only set on {@link #TYPE_VIEW_ADDED} events. + *

Only set on {@link #TYPE_VIEW_APPEARED} events. */ @Nullable public ViewNode getViewNode() { - return null; + return mNode; } /** * Gets the {@link AutofillId} of the node associated with the event. * - *

Only set on {@link #TYPE_VIEW_REMOVED} and {@link #TYPE_VIEW_TEXT_CHANGED} events. + *

Only set on {@link #TYPE_VIEW_DISAPPEARED} and {@link #TYPE_VIEW_TEXT_CHANGED} events. */ @Nullable public AutofillId getId() { - return null; + return mId; } /** @@ -159,16 +196,41 @@ public final class ContentCaptureEvent implements Parcelable { */ @Nullable public CharSequence getText() { - return null; + return mText; + } + + /** @hide */ + public void dump(@NonNull PrintWriter pw) { + pw.print("type="); pw.print(getTypeAsString(mType)); + pw.print(", time="); pw.print(mEventTime); + if (mFlags > 0) { + pw.print(", flags="); pw.print(mFlags); + } + if (mId != null) { + pw.print(", id="); pw.print(mId); + } + if (mNode != null) { + pw.print(", id="); pw.print(mNode.getAutofillId()); + } } @Override public String toString() { final StringBuilder string = new StringBuilder("ContentCaptureEvent[type=") - .append(getTypeAsString(mType)).append(", time=").append(mEventTime); + .append(getTypeAsString(mType)); if (mFlags > 0) { string.append(", flags=").append(mFlags); } + if (mId != null) { + string.append(", id=").append(mId); + } + if (mNode != null) { + final String className = mNode.getClassName(); + if (mNode != null) { + string.append(", class=").append(className); + } + string.append(", id=").append(mNode.getAutofillId()); + } return string.append(']').toString(); } @@ -182,6 +244,9 @@ public final class ContentCaptureEvent implements Parcelable { parcel.writeInt(mType); parcel.writeLong(mEventTime); parcel.writeInt(mFlags); + parcel.writeParcelable(mId, flags); + ViewNode.writeToParcel(parcel, mNode, flags); + parcel.writeCharSequence(mText); } public static final Parcelable.Creator CREATOR = @@ -192,7 +257,17 @@ public final class ContentCaptureEvent implements Parcelable { final int type = parcel.readInt(); final long eventTime = parcel.readLong(); final int flags = parcel.readInt(); - return new ContentCaptureEvent(type, eventTime, flags); + final ContentCaptureEvent event = new ContentCaptureEvent(type, eventTime, flags); + final AutofillId id = parcel.readParcelable(null); + if (id != null) { + event.setAutofillId(id); + } + final ViewNode node = ViewNode.readFromParcel(parcel); + if (node != null) { + event.setViewNode(node); + } + event.setText(parcel.readCharSequence()); + return event; } @Override @@ -201,7 +276,6 @@ public final class ContentCaptureEvent implements Parcelable { } }; - /** @hide */ public static String getTypeAsString(@EventType int type) { switch (type) { @@ -213,10 +287,10 @@ public final class ContentCaptureEvent implements Parcelable { return "ACTIVITY_PAUSED"; case TYPE_ACTIVITY_STOPPED: return "ACTIVITY_STOPPED"; - case TYPE_VIEW_ADDED: - return "VIEW_ADDED"; - case TYPE_VIEW_REMOVED: - return "VIEW_REMOVED"; + case TYPE_VIEW_APPEARED: + return "VIEW_APPEARED"; + case TYPE_VIEW_DISAPPEARED: + return "VIEW_DISAPPEARED"; case TYPE_VIEW_TEXT_CHANGED: return "VIEW_TEXT_CHANGED"; default: diff --git a/core/java/android/view/intelligence/IntelligenceManager.java b/core/java/android/view/intelligence/IntelligenceManager.java index 9bf6c2ce6184e..c02fb3218a2e7 100644 --- a/core/java/android/view/intelligence/IntelligenceManager.java +++ b/core/java/android/view/intelligence/IntelligenceManager.java @@ -15,6 +15,12 @@ */ package android.view.intelligence; +import static android.view.intelligence.ContentCaptureEvent.TYPE_VIEW_APPEARED; +import static android.view.intelligence.ContentCaptureEvent.TYPE_VIEW_DISAPPEARED; +import static android.view.intelligence.ContentCaptureEvent.TYPE_VIEW_TEXT_CHANGED; + +import static com.android.internal.util.function.pooled.PooledLambda.obtainMessage; + import android.annotation.NonNull; import android.annotation.Nullable; import android.annotation.SystemApi; @@ -22,11 +28,15 @@ import android.annotation.SystemService; import android.content.ComponentName; import android.content.Context; import android.os.Bundle; +import android.os.Handler; +import android.os.HandlerThread; import android.os.IBinder; import android.os.RemoteException; -import android.os.SystemClock; import android.service.intelligence.InteractionSessionId; import android.util.Log; +import android.view.View; +import android.view.ViewStructure; +import android.view.autofill.AutofillId; import android.view.intelligence.ContentCaptureEvent.EventType; import com.android.internal.annotations.GuardedBy; @@ -34,8 +44,7 @@ import com.android.internal.os.IResultReceiver; import com.android.internal.util.Preconditions; import java.io.PrintWriter; -import java.util.Arrays; -import java.util.List; +import java.util.ArrayList; import java.util.Set; /** @@ -46,8 +55,9 @@ public final class IntelligenceManager { private static final String TAG = "IntelligenceManager"; - // TODO(b/111276913): define a way to dynamically set it (for example, using settings?) + // TODO(b/111276913): define a way to dynamically set them(for example, using settings?) private static final boolean VERBOSE = false; + private static final boolean DEBUG = true; // STOPSHIP if not set to false /** * Used to indicate that a text change was caused by user input (for example, through IME). @@ -55,7 +65,6 @@ public final class IntelligenceManager { //TODO(b/111276913): link to notifyTextChanged() method once available public static final int FLAG_USER_INPUT = 0x1; - /** * Initial state, when there is no session. * @@ -77,6 +86,15 @@ public final class IntelligenceManager { */ public static final int STATE_ACTIVE = 2; + private static final String BG_THREAD_NAME = "intel_svc_streamer_thread"; + + /** + * Maximum number of events that are delayed for an app. + * + *

If the session is not started after the limit is reached, it's discarded. + */ + private static final int MAX_DELAYED_SIZE = 20; + private final Context mContext; @Nullable @@ -99,10 +117,24 @@ public final class IntelligenceManager { @GuardedBy("mLock") private ComponentName mComponentName; + // TODO(b/111276913): create using maximum batch size as capacity + /** + * List of events held to be sent as a batch. + */ + @GuardedBy("mLock") + private final ArrayList mEvents = new ArrayList<>(); + + private final Handler mHandler; + /** @hide */ public IntelligenceManager(@NonNull Context context, @Nullable IIntelligenceManager service) { mContext = Preconditions.checkNotNull(context, "context cannot be null"); mService = service; + + // TODO(b/111276913): use an existing bg thread instead... + final HandlerThread bgThread = new HandlerThread(BG_THREAD_NAME); + bgThread.start(); + mHandler = Handler.createAsync(bgThread.getLooper()); } /** @hide */ @@ -111,8 +143,9 @@ public final class IntelligenceManager { synchronized (mLock) { if (mState != STATE_UNKNOWN) { + // TODO(b/111276913): revisit this scenario Log.w(TAG, "ignoring onActivityStarted(" + token + ") while on state " - + getStateAsStringLocked()); + + getStateAsString(mState)); return; } mState = STATE_WAITING_FOR_SERVER; @@ -121,8 +154,8 @@ public final class IntelligenceManager { mComponentName = componentName; if (VERBOSE) { - Log.v(TAG, "onActivityStarted(): token=" + token + ", act=" + componentName - + ", id=" + mId); + Log.v(TAG, "onActivityCreated(): token=" + token + ", act=" + + getActivityDebugNameLocked() + ", id=" + mId); } final int flags = 0; // TODO(b/111276913): get proper flags @@ -138,12 +171,12 @@ public final class IntelligenceManager { } else { // TODO(b/111276913): handle other cases like disabled by // service - mState = STATE_UNKNOWN; + resetStateLocked(); } if (VERBOSE) { Log.v(TAG, "onActivityStarted() result: code=" + resultCode + ", id=" + mId - + ", state=" + getStateAsStringLocked()); + + ", state=" + getStateAsString(mState)); } } } @@ -154,6 +187,60 @@ public final class IntelligenceManager { } } + //TODO(b/111276913): should buffer event (and call service on handler thread), instead of + // calling right away + private void sendEvent(@NonNull ContentCaptureEvent event) { + mHandler.sendMessage(obtainMessage(IntelligenceManager::handleSendEvent, this, event)); + } + + private void handleSendEvent(@NonNull ContentCaptureEvent event) { + + synchronized (mLock) { + mEvents.add(event); + final int numberEvents = mEvents.size(); + if (mState != STATE_ACTIVE) { + if (numberEvents >= MAX_DELAYED_SIZE) { + // Typically happens on system apps that are started before the system service + // is ready (like com.android.settings/.FallbackHome) + //TODO(b/111276913): try to ignore session while system is not ready / boot + // not complete instead. + Log.w(TAG, "Closing session for " + getActivityDebugNameLocked() + + " after " + numberEvents + " delayed events"); + // TODO(b/111276913): blacklist activity / use special flag to indicate that + // when it's launched again + resetStateLocked(); + return; + } + + if (VERBOSE) { + Log.v(TAG, "Delaying " + numberEvents + " events for " + + getActivityDebugNameLocked() + " while on state " + + getStateAsString(mState)); + } + return; + } + + if (mId == null) { + // Sanity check - should not happen + Log.wtf(TAG, "null session id for " + mComponentName); + return; + } + + //TODO(b/111276913): right now we're sending sending right away (unless not ready), but + // we should hold the events and flush later. + try { + if (DEBUG) { + Log.d(TAG, "Sending " + numberEvents + " event(s) for " + + getActivityDebugNameLocked()); + } + mService.sendEvents(mContext.getUserId(), mId, mEvents); + mEvents.clear(); + } catch (RemoteException e) { + throw e.rethrowFromSystemServer(); + } + } + } + /** * Used for intermediate events (i.e, other than created and destroyed). * @@ -161,28 +248,11 @@ public final class IntelligenceManager { */ public void onActivityLifecycleEvent(@EventType int type) { if (!isContentCaptureEnabled()) return; - - //TODO(b/111276913): should buffer event (and call service on handler thread), instead of - // calling right away - final ContentCaptureEvent event = new ContentCaptureEvent(type, SystemClock.uptimeMillis(), - 0); - final List events = Arrays.asList(event); - - synchronized (mLock) { - //TODO(b/111276913): check session state; for example, how to handle if it's waiting for - // remote id - - if (VERBOSE) { - Log.v(TAG, "onActivityLifecycleEvent() for " + mComponentName.flattenToShortString() - + ": " + ContentCaptureEvent.getTypeAsString(type)); - } - - try { - mService.sendEvents(mContext.getUserId(), mId, events); - } catch (RemoteException e) { - throw e.rethrowFromSystemServer(); - } + if (VERBOSE) { + Log.v(TAG, "onActivityLifecycleEvent() for " + getActivityDebugNameLocked() + + ": " + ContentCaptureEvent.getTypeAsString(type)); } + sendEvent(new ContentCaptureEvent(type)); } /** @hide */ @@ -194,22 +264,105 @@ public final class IntelligenceManager { // id) and send it to the cache of batched commands if (VERBOSE) { - Log.v(TAG, "onActivityDestroyed(): state=" + getStateAsStringLocked() + Log.v(TAG, "onActivityDestroyed(): state=" + getStateAsString(mState) + ", mId=" + mId); } try { mService.finishSession(mContext.getUserId(), mId); - mState = STATE_UNKNOWN; - mId = null; - mApplicationToken = null; - mComponentName = null; + resetStateLocked(); } catch (RemoteException e) { throw e.rethrowFromSystemServer(); } } } + @GuardedBy("mLock") + private void resetStateLocked() { + mState = STATE_UNKNOWN; + mId = null; + mApplicationToken = null; + mComponentName = null; + mEvents.clear(); + } + + /** + * Notifies the Intelligence Service that a node has been added to the view structure. + * + *

Typically called "manually" by views that handle their own virtual view hierarchy, or + * automatically by the Android System for views that return {@code true} on + * {@link View#onProvideContentCaptureStructure(ViewStructure, int)}. + * + * @param node node that has been added. + */ + public void notifyViewAppeared(@NonNull ViewStructure node) { + Preconditions.checkNotNull(node); + if (!isContentCaptureEnabled()) return; + + if (!(node instanceof ViewNode.ViewStructureImpl)) { + throw new IllegalArgumentException("Invalid node class: " + node.getClass()); + } + sendEvent(new ContentCaptureEvent(TYPE_VIEW_APPEARED) + .setViewNode(((ViewNode.ViewStructureImpl) node).mNode)); + } + + /** + * Notifies the Intelligence Service that a node has been removed from the view structure. + * + *

Typically called "manually" by views that handle their own virtual view hierarchy, or + * automatically by the Android System for standard views. + * + * @param id id of the node that has been removed. + */ + public void notifyViewDisappeared(@NonNull AutofillId id) { + Preconditions.checkNotNull(id); + if (!isContentCaptureEnabled()) return; + + sendEvent(new ContentCaptureEvent(TYPE_VIEW_DISAPPEARED).setAutofillId(id)); + } + + /** + * Notifies the Intelligence Service that the value of a text node has been changed. + * + * @param id of the node. + * @param text new text. + * @param flags either {@code 0} or {@link #FLAG_USER_INPUT} when the value was explicitly + * changed by the user (for example, through the keyboard). + */ + public void notifyViewTextChanged(@NonNull AutofillId id, @Nullable CharSequence text, + int flags) { + Preconditions.checkNotNull(id); + if (!isContentCaptureEnabled()) return; + + sendEvent(new ContentCaptureEvent(TYPE_VIEW_TEXT_CHANGED, flags).setAutofillId(id) + .setText(text)); + } + + /** + * Creates a {@link ViewStructure} for a "standard" view. + * + * @hide + */ + @NonNull + public ViewStructure newViewStructure(@NonNull View view) { + return new ViewNode.ViewStructureImpl(view); + } + + /** + * Creates a {@link ViewStructure} for a "virtual" view, so it can be passed to + * {@link #notifyViewAppeared(ViewStructure)} by the view managing the virtual view hierarchy. + * + * @param parentId id of the virtual view parent (it can be obtained by calling + * {@link ViewStructure#getAutofillId()} on the parent). + * @param virtualId id of the virtual child, relative to the parent. + * + * @return a new {@link ViewStructure} that can be used for Content Capture purposes. + */ + @NonNull + public ViewStructure newVirtualViewStructure(@NonNull AutofillId parentId, int virtualId) { + return new ViewNode.ViewStructureImpl(parentId, virtualId); + } + /** * Returns the component name of the {@code android.service.intelligence.IntelligenceService} * that is enabled for the current user. @@ -322,15 +475,29 @@ public final class IntelligenceManager { pw.print(prefix2); pw.print("enabled: "); pw.println(isContentCaptureEnabled()); pw.print(prefix2); pw.print("id: "); pw.println(mId); pw.print(prefix2); pw.print("state: "); pw.print(mState); pw.print(" ("); - pw.print(getStateAsStringLocked()); pw.println(")"); - pw.print(prefix2); pw.print("appToken: "); pw.println(mApplicationToken); - pw.print(prefix2); pw.print("componentName: "); pw.println(mComponentName); + pw.print(getStateAsString(mState)); pw.println(")"); + pw.print(prefix2); pw.print("app token: "); pw.println(mApplicationToken); + pw.print(prefix2); pw.print("component name: "); + pw.println(mComponentName == null ? "null" : mComponentName.flattenToShortString()); + final int numberEvents = mEvents.size(); + pw.print(prefix2); pw.print("batched events: "); pw.println(numberEvents); + if (numberEvents > 0) { + for (int i = 0; i < numberEvents; i++) { + final ContentCaptureEvent event = mEvents.get(i); + pw.println(i); pw.print(": "); event.dump(pw); pw.println(); + } + + } } } + /** + * Gets a string that can be used to identify the activity on logging statements. + */ @GuardedBy("mLock") - private String getStateAsStringLocked() { - return getStateAsString(mState); + private String getActivityDebugNameLocked() { + return mComponentName == null ? mContext.getPackageName() + : mComponentName.flattenToShortString(); } @NonNull diff --git a/core/java/android/view/intelligence/ViewNode.java b/core/java/android/view/intelligence/ViewNode.java index 357ecf599f7a2..cc78e6b4aa6de 100644 --- a/core/java/android/view/intelligence/ViewNode.java +++ b/core/java/android/view/intelligence/ViewNode.java @@ -15,10 +15,24 @@ */ package android.view.intelligence; +import android.annotation.NonNull; import android.annotation.Nullable; import android.annotation.SystemApi; import android.app.assist.AssistStructure; +import android.graphics.Matrix; +import android.graphics.Rect; +import android.os.Bundle; +import android.os.LocaleList; +import android.os.Parcel; +import android.util.Log; +import android.view.View; +import android.view.ViewParent; +import android.view.ViewStructure; +import android.view.ViewStructure.HtmlInfo.Builder; import android.view.autofill.AutofillId; +import android.view.autofill.AutofillValue; + +import com.android.internal.util.Preconditions; //TODO(b/111276913): add javadocs / implement Parcelable / implement //TODO(b/111276913): for now it's extending ViewNode directly as it needs most of its properties, @@ -28,6 +42,16 @@ import android.view.autofill.AutofillId; @SystemApi public final class ViewNode extends AssistStructure.ViewNode { + private static final String TAG = "ViewNode"; + + private AutofillId mParentAutofillId; + + // TODO(b/111276913): temporarily setting some fields here while they're not accessible from the + // superclass + private AutofillId mAutofillId; + private CharSequence mText; + private String mClassName; + /** @hide */ public ViewNode() { } @@ -38,7 +62,343 @@ public final class ViewNode extends AssistStructure.ViewNode { */ @Nullable public AutofillId getParentAutofillId() { - //TODO(b/111276913): implement - return null; + return mParentAutofillId; + } + + // TODO(b/111276913): temporarily overwriting some methods + @Override + public AutofillId getAutofillId() { + return mAutofillId; + } + @Override + public CharSequence getText() { + return mText; + } + @Override + public String getClassName() { + return mClassName; + } + + /** @hide */ + public static void writeToParcel(@NonNull Parcel parcel, @Nullable ViewNode node, int flags) { + if (node == null) { + parcel.writeParcelable(null, flags); + return; + } + parcel.writeParcelable(node.mAutofillId, flags); + parcel.writeParcelable(node.mParentAutofillId, flags); + parcel.writeCharSequence(node.mText); + parcel.writeString(node.mClassName); + } + + /** @hide */ + public static @Nullable ViewNode readFromParcel(@NonNull Parcel parcel) { + final AutofillId id = parcel.readParcelable(null); + if (id == null) return null; + + final ViewNode node = new ViewNode(); + + node.mAutofillId = id; + node.mParentAutofillId = parcel.readParcelable(null); + node.mText = parcel.readCharSequence(); + node.mClassName = parcel.readString(); + + return node; + } + + /** @hide */ + static final class ViewStructureImpl extends ViewStructure { + + final ViewNode mNode = new ViewNode(); + + ViewStructureImpl(@NonNull View view) { + mNode.mAutofillId = Preconditions.checkNotNull(view).getAutofillId(); + final ViewParent parent = view.getParent(); + if (parent instanceof View) { + mNode.mParentAutofillId = ((View) parent).getAutofillId(); + } + } + + ViewStructureImpl(@NonNull AutofillId parentId, int virtualId) { + mNode.mParentAutofillId = Preconditions.checkNotNull(parentId); + mNode.mAutofillId = new AutofillId(parentId, virtualId); + } + + @Override + public void setId(int id, String packageName, String typeName, String entryName) { + // TODO(b/111276913): implement or move to superclass + } + + @Override + public void setDimens(int left, int top, int scrollX, int scrollY, int width, int height) { + // TODO(b/111276913): implement or move to superclass + } + + @Override + public void setTransformation(Matrix matrix) { + // TODO(b/111276913): implement or move to superclass + } + + @Override + public void setElevation(float elevation) { + // TODO(b/111276913): implement or move to superclass + } + + @Override + public void setAlpha(float alpha) { + // TODO(b/111276913): implement or move to superclass + } + + @Override + public void setVisibility(int visibility) { + // TODO(b/111276913): implement or move to superclass + } + + @Override + public void setAssistBlocked(boolean state) { + // TODO(b/111276913): implement or move to superclass + } + + @Override + public void setEnabled(boolean state) { + // TODO(b/111276913): implement or move to superclass + } + + @Override + public void setClickable(boolean state) { + // TODO(b/111276913): implement or move to superclass + } + + @Override + public void setLongClickable(boolean state) { + // TODO(b/111276913): implement or move to superclass + } + + @Override + public void setContextClickable(boolean state) { + // TODO(b/111276913): implement or move to superclass + } + + @Override + public void setFocusable(boolean state) { + // TODO(b/111276913): implement or move to superclass + } + + @Override + public void setFocused(boolean state) { + // TODO(b/111276913): implement or move to superclass + } + + @Override + public void setAccessibilityFocused(boolean state) { + // TODO(b/111276913): implement or move to superclass + } + + @Override + public void setCheckable(boolean state) { + // TODO(b/111276913): implement or move to superclass + } + + @Override + public void setChecked(boolean state) { + // TODO(b/111276913): implement or move to superclass + } + + @Override + public void setSelected(boolean state) { + // TODO(b/111276913): implement or move to superclass + } + + @Override + public void setActivated(boolean state) { + // TODO(b/111276913): implement or move to superclass + } + + @Override + public void setOpaque(boolean opaque) { + // TODO(b/111276913): implement or move to superclass + } + + @Override + public void setClassName(String className) { + // TODO(b/111276913): temporarily setting directly; should be done on superclass instead + mNode.mClassName = className; + } + + @Override + public void setContentDescription(CharSequence contentDescription) { + // TODO(b/111276913): implement or move to superclass + } + + @Override + public void setText(CharSequence text) { + // TODO(b/111276913): temporarily setting directly; should be done on superclass instead + mNode.mText = text; + } + + @Override + public void setText(CharSequence text, int selectionStart, int selectionEnd) { + // TODO(b/111276913): implement or move to superclass + } + + @Override + public void setTextStyle(float size, int fgColor, int bgColor, int style) { + // TODO(b/111276913): implement or move to superclass + } + + @Override + public void setTextLines(int[] charOffsets, int[] baselines) { + // TODO(b/111276913): implement or move to superclass + } + + @Override + public void setHint(CharSequence hint) { + // TODO(b/111276913): implement or move to superclass + } + + @Override + public CharSequence getText() { + // TODO(b/111276913): temporarily getting directly; should be done on superclass instead + return mNode.mText; + } + + @Override + public int getTextSelectionStart() { + // TODO(b/111276913): implement or move to superclass + return 0; + } + + @Override + public int getTextSelectionEnd() { + // TODO(b/111276913): implement or move to superclass + return 0; + } + + @Override + public CharSequence getHint() { + // TODO(b/111276913): implement or move to superclass + return null; + } + + @Override + public Bundle getExtras() { + // TODO(b/111276913): implement or move to superclass + return null; + } + + @Override + public boolean hasExtras() { + // TODO(b/111276913): implement or move to superclass + return false; + } + + @Override + public void setChildCount(int num) { + Log.w(TAG, "setChildCount() is not supported"); + } + + @Override + public int addChildCount(int num) { + Log.w(TAG, "addChildCount() is not supported"); + return 0; + } + + @Override + public int getChildCount() { + Log.w(TAG, "getChildCount() is not supported"); + return 0; + } + + @Override + public ViewStructure newChild(int index) { + Log.w(TAG, "newChild() is not supported"); + return null; + } + + @Override + public ViewStructure asyncNewChild(int index) { + Log.w(TAG, "asyncNewChild() is not supported"); + return null; + } + + @Override + public AutofillId getAutofillId() { + // TODO(b/111276913): temporarily getting directly; should be done on superclass instead + return mNode.mAutofillId; + } + + @Override + public void setAutofillId(AutofillId id) { + // TODO(b/111276913): temporarily setting directly; should be done on superclass instead + mNode.mAutofillId = id; + } + + @Override + public void setAutofillId(AutofillId parentId, int virtualId) { + // TODO(b/111276913): temporarily setting directly; should be done on superclass instead + mNode.mAutofillId = new AutofillId(parentId, virtualId); + } + + @Override + public void setAutofillType(int type) { + // TODO(b/111276913): implement or move to superclass + } + + @Override + public void setAutofillHints(String[] hint) { + // TODO(b/111276913): implement or move to superclass + } + + @Override + public void setAutofillValue(AutofillValue value) { + // TODO(b/111276913): implement or move to superclass + } + + @Override + public void setAutofillOptions(CharSequence[] options) { + // TODO(b/111276913): implement or move to superclass + } + + @Override + public void setInputType(int inputType) { + // TODO(b/111276913): implement or move to superclass + } + + @Override + public void setDataIsSensitive(boolean sensitive) { + // TODO(b/111276913): implement or move to superclass + } + + @Override + public void asyncCommit() { + Log.w(TAG, "asyncCommit() is not supported"); + } + + @Override + public Rect getTempRect() { + // TODO(b/111276913): implement or move to superclass + return null; + } + + @Override + public void setWebDomain(String domain) { + Log.w(TAG, "setWebDomain() is not supported"); + } + + @Override + public void setLocaleList(LocaleList localeList) { + // TODO(b/111276913): implement or move to superclass + } + + @Override + public Builder newHtmlInfoBuilder(String tagName) { + Log.w(TAG, "newHtmlInfoBuilder() is not supported"); + return null; + } + + @Override + public void setHtmlInfo(HtmlInfo htmlInfo) { + Log.w(TAG, "setHtmlInfo() is not supported"); + } } } diff --git a/core/java/android/widget/Switch.java b/core/java/android/widget/Switch.java index 8bfc151b1a839..d55c09f7fcb60 100644 --- a/core/java/android/widget/Switch.java +++ b/core/java/android/widget/Switch.java @@ -1422,17 +1422,24 @@ public class Switch extends CompoundButton { @Override public void onProvideStructure(ViewStructure structure) { super.onProvideStructure(structure); - onProvideAutoFillStructureForAssistOrAutofill(structure); + onProvideStructureForAssistOrAutofillOrViewCapture(structure); } @Override public void onProvideAutofillStructure(ViewStructure structure, int flags) { super.onProvideAutofillStructure(structure, flags); - onProvideAutoFillStructureForAssistOrAutofill(structure); + onProvideStructureForAssistOrAutofillOrViewCapture(structure); } - // NOTE: currently there is no difference for Assist or AutoFill, so it doesn't take flags - private void onProvideAutoFillStructureForAssistOrAutofill(ViewStructure structure) { + @Override + public boolean onProvideContentCaptureStructure(ViewStructure structure, int flags) { + final boolean notifyManager = super.onProvideContentCaptureStructure(structure, flags); + onProvideStructureForAssistOrAutofillOrViewCapture(structure); + return notifyManager; + } + + // NOTE: currently there is no difference for any type, so it doesn't take flags + private void onProvideStructureForAssistOrAutofillOrViewCapture(ViewStructure structure) { CharSequence switchText = isChecked() ? mTextOn : mTextOff; if (!TextUtils.isEmpty(switchText)) { CharSequence oldText = structure.getText(); diff --git a/core/java/android/widget/TextView.java b/core/java/android/widget/TextView.java index 572670fc662b4..3bdd7b8e91be6 100644 --- a/core/java/android/widget/TextView.java +++ b/core/java/android/widget/TextView.java @@ -166,6 +166,7 @@ import android.view.inputmethod.ExtractedText; import android.view.inputmethod.ExtractedTextRequest; import android.view.inputmethod.InputConnection; import android.view.inputmethod.InputMethodManager; +import android.view.intelligence.IntelligenceManager; import android.view.textclassifier.TextClassification; import android.view.textclassifier.TextClassificationContext; import android.view.textclassifier.TextClassificationManager; @@ -948,6 +949,9 @@ public class TextView extends View implements ViewTreeObserver.OnPreDrawListener if (getImportantForAutofill() == IMPORTANT_FOR_AUTOFILL_AUTO) { setImportantForAutofill(IMPORTANT_FOR_AUTOFILL_YES); } + if (getImportantForContentCapture() == IMPORTANT_FOR_CONTENT_CAPTURE_AUTO) { + setImportantForContentCapture(IMPORTANT_FOR_CONTENT_CAPTURE_YES); + } setTextInternal(""); @@ -6072,7 +6076,7 @@ public class TextView extends View implements ViewTreeObserver.OnPreDrawListener if (needEditableForNotification) { sendAfterTextChanged((Editable) text); } else { - notifyAutoFillManagerAfterTextChanged(); + notifyManagersAfterTextChanged(); } // SelectionModifierCursorController depends on textCanBeSelected, which depends on text @@ -10121,23 +10125,33 @@ public class TextView extends View implements ViewTreeObserver.OnPreDrawListener } // Always notify AutoFillManager - it will return right away if autofill is disabled. - notifyAutoFillManagerAfterTextChanged(); + notifyManagersAfterTextChanged(); hideErrorIfUnchanged(); } - private void notifyAutoFillManagerAfterTextChanged() { - // It is important to not check whether the view is important for autofill - // since the user can trigger autofill manually on not important views. - if (!isAutofillable()) { - return; - } - final AutofillManager afm = mContext.getSystemService(AutofillManager.class); - if (afm != null) { - if (android.view.autofill.Helper.sVerbose) { - Log.v(LOG_TAG, "notifyAutoFillManagerAfterTextChanged"); + private void notifyManagersAfterTextChanged() { + + // Autofill + if (isAutofillable()) { + // It is important to not check whether the view is important for autofill + // since the user can trigger autofill manually on not important views. + final AutofillManager afm = mContext.getSystemService(AutofillManager.class); + if (afm != null) { + if (android.view.autofill.Helper.sVerbose) { + Log.v(LOG_TAG, "notifyAutoFillManagerAfterTextChanged"); + } + afm.notifyValueChanged(TextView.this); + } + } + + // ContentCapture + if (isImportantForContentCapture() && isTextEditable()) { + final IntelligenceManager im = mContext.getSystemService(IntelligenceManager.class); + if (im != null && im.isContentCaptureEnabled()) { + // TODO(b/111276913): pass flags when edited by user / add CTS test + im.notifyViewTextChanged(getAutofillId(), getText(), /* flags= */ 0); } - afm.notifyValueChanged(TextView.this); } } @@ -10900,21 +10914,33 @@ public class TextView extends View implements ViewTreeObserver.OnPreDrawListener @Override public void onProvideStructure(ViewStructure structure) { super.onProvideStructure(structure); - onProvideAutoStructureForAssistOrAutofill(structure, false); + onProvideStructureForAssistOrAutofillOrViewCapture(structure, /* forAutofill = */ false, + /* forViewCapture= */ false); } @Override public void onProvideAutofillStructure(ViewStructure structure, int flags) { super.onProvideAutofillStructure(structure, flags); - onProvideAutoStructureForAssistOrAutofill(structure, true); + onProvideStructureForAssistOrAutofillOrViewCapture(structure, /* forAutofill = */ true, + /* forViewCapture= */ false); } - private void onProvideAutoStructureForAssistOrAutofill(ViewStructure structure, - boolean forAutofill) { + @Override + public boolean onProvideContentCaptureStructure(ViewStructure structure, int flags) { + final boolean notifyManager = super.onProvideContentCaptureStructure(structure, flags); + onProvideStructureForAssistOrAutofillOrViewCapture(structure, /* forAutofill = */ false, + /* forViewCapture= */ true); + return notifyManager; + } + + private void onProvideStructureForAssistOrAutofillOrViewCapture(ViewStructure structure, + boolean forAutofill, boolean forViewCapture) { final boolean isPassword = hasPasswordTransformationMethod() || isPasswordInputType(getInputType()); - if (forAutofill) { - structure.setDataIsSensitive(!mTextSetFromXmlOrResourceId); + if (forAutofill || forViewCapture) { + if (forAutofill) { + structure.setDataIsSensitive(!mTextSetFromXmlOrResourceId); + } if (mTextId != ResourceId.ID_NULL) { try { structure.setTextIdEntry(getResources().getResourceEntryName(mTextId)); @@ -10927,7 +10953,7 @@ public class TextView extends View implements ViewTreeObserver.OnPreDrawListener } } - if (!isPassword || forAutofill) { + if (!isPassword || forAutofill || forViewCapture) { if (mLayout == null) { assumeLayout(); } @@ -11043,7 +11069,8 @@ public class TextView extends View implements ViewTreeObserver.OnPreDrawListener // of the View (and can be any drawable) or a BackgroundColorSpan inside the text. structure.setTextStyle(getTextSize(), getCurrentTextColor(), AssistStructure.ViewNode.TEXT_COLOR_UNDEFINED /* bgColor */, style); - } else { + } + if (forAutofill || forViewCapture) { structure.setMinTextEms(getMinEms()); structure.setMaxTextEms(getMaxEms()); int maxLength = -1; diff --git a/core/res/res/values/attrs.xml b/core/res/res/values/attrs.xml index 68ec34229d482..a99b9421e3b3d 100644 --- a/core/res/res/values/attrs.xml +++ b/core/res/res/values/attrs.xml @@ -2448,6 +2448,25 @@ + + + + + + + + + + + + + + diff --git a/core/res/res/values/public.xml b/core/res/res/values/public.xml index 86879c30553f4..73dae0801b8e1 100644 --- a/core/res/res/values/public.xml +++ b/core/res/res/values/public.xml @@ -2919,6 +2919,7 @@ +