Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added moar ContentCapture APIs (and their initial implementation).

Browse Source 
There are 4 new APIs on View:
  - boolean setImportantForContentCapture()
  - boolean getImportantForContentCapture()
  - boolean isImportantForContentCapture()
  - boolean onProvideContentCaptureStructure()

And 4 on IntelligenceManager:
  - void notifyViewAppeared()
  - void notifyViewDisappeared()
  - void notifyViewTextChanged()
  - ViewStructure newVirtualViewStructure()

These methods are similar to the equivalent methods that are used for Autofill
and/or Assist, except for the following differences:

- The view hierarchy nodes are reported as they are rendered, rather than at
  once in a tree, recursively. Hence, the ViewStructure implementation does
  not implement the methods that add children to it, and views that provide
  virtual hierarchies must manually call IntelligenceManager to create the
  ViewStructure to their children and notify when their children are added and
  removed.
- It does not support methods added for Autofill to handle HTML pages (such
  as setHtmlInfo() and setWewbDomain()), as they're not important in the
  Content Capture context.
- Similarly, it also does not support setDataIsSensitive(), because the
  Intelligence service does not have the same restrictions as the Autofill
  service.

The CL also provides the initial implementation of these APIs, although still
full of TODOs (for example, we're not holding the events to send as a batch
yet).

Test: m -j update-api doc-comment-check-docs
Bug: 117944706

Change-Id: I43f06ce82bfe3b14d8d13fb3b2ebee223db83284
This commit is contained in:
Felipe Leme
2018-11-07 15:11:56 -08:00
parent 9c045fe697
commit 88eae3bc69
12 changed files with 1175 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files

429
core/java/android/view/View.java
View File

@@ -112,6 +112,7 @@ import android.view.autofill.AutofillValue;
import android.view.inputmethod.EditorInfo;
import android.view.inputmethod.InputConnection;
import android.view.inputmethod.InputMethodManager;
import android.view.intelligence.IntelligenceManager;
import android.widget.Checkable;
import android.widget.FrameLayout;
import android.widget.ScrollBarDrawable;
@@ -798,6 +799,11 @@ public class View implements Drawable.Callback, KeyEvent.Callback,
// set if a session is not started.
private static final String AUTOFILL_LOG_TAG = "View.Autofill";
/**
* The logging tag used by this class when logging content capture-related messages.
*/
private static final String CONTENT_CAPTURE_LOG_TAG = "View.ContentCapture";
/**
* When set to true, apps will draw debugging information about their layouts.
*
@@ -1337,6 +1343,59 @@ public class View implements Drawable.Callback, KeyEvent.Callback,
*/
public static final int AUTOFILL_FLAG_INCLUDE_NOT_IMPORTANT_VIEWS = 0x1;
/** @hide */
@IntDef(prefix = { "IMPORTANT_FOR_CONTENT_CAPTURE_" }, value = {
IMPORTANT_FOR_CONTENT_CAPTURE_AUTO,
IMPORTANT_FOR_CONTENT_CAPTURE_YES,
IMPORTANT_FOR_CONTENT_CAPTURE_NO,
IMPORTANT_FOR_CONTENT_CAPTURE_YES_EXCLUDE_DESCENDANTS,
IMPORTANT_FOR_CONTENT_CAPTURE_NO_EXCLUDE_DESCENDANTS
})
@Retention(RetentionPolicy.SOURCE)
public @interface ContentCaptureImportance {}
/**
* Automatically determine whether a view is important for content capture.
*
* @see #isImportantForContentCapture()
* @see #setImportantForContentCapture(int)
*/
public static final int IMPORTANT_FOR_CONTENT_CAPTURE_AUTO = 0x0;
/**
* The view is important for content capture, and its children (if any) will be traversed.
*
* @see #isImportantForContentCapture()
* @see #setImportantForContentCapture(int)
*/
public static final int IMPORTANT_FOR_CONTENT_CAPTURE_YES = 0x1;
/**
* The view is not important for content capture, but its children (if any) will be traversed.
*
* @see #isImportantForContentCapture()
* @see #setImportantForContentCapture(int)
*/
public static final int IMPORTANT_FOR_CONTENT_CAPTURE_NO = 0x2;
/**
* The view is important for content capture, but its children (if any) will not be traversed.
*
* @see #isImportantForContentCapture()
* @see #setImportantForContentCapture(int)
*/
public static final int IMPORTANT_FOR_CONTENT_CAPTURE_YES_EXCLUDE_DESCENDANTS = 0x4;
/**
* The view is not important for content capture, and its children (if any) will not be
* traversed.
*
* @see #isImportantForContentCapture()
* @see #setImportantForContentCapture(int)
*/
public static final int IMPORTANT_FOR_CONTENT_CAPTURE_NO_EXCLUDE_DESCENDANTS = 0x8;
/**
* This view is enabled. Interpretation varies by subclass.
* Use with ENABLED_MASK when calling setFlags.
@@ -2243,7 +2302,44 @@ public class View implements Drawable.Callback, KeyEvent.Callback,
@UnsupportedAppUsage
protected Object mTag = null;
// for mPrivateFlags:
/*
* Masks for mPrivateFlags, as generated by dumpFlags():
*
* |-------|-------|-------|-------|
* 1 PFLAG_WANTS_FOCUS
* 1 PFLAG_FOCUSED
* 1 PFLAG_SELECTED
* 1 PFLAG_IS_ROOT_NAMESPACE
* 1 PFLAG_HAS_BOUNDS
* 1 PFLAG_DRAWN
* 1 PFLAG_DRAW_ANIMATION
* 1 PFLAG_SKIP_DRAW
* 1 PFLAG_REQUEST_TRANSPARENT_REGIONS
* 1 PFLAG_DRAWABLE_STATE_DIRTY
* 1 PFLAG_MEASURED_DIMENSION_SET
* 1 PFLAG_FORCE_LAYOUT
* 1 PFLAG_LAYOUT_REQUIRED
* 1 PFLAG_PRESSED
* 1 PFLAG_DRAWING_CACHE_VALID
* 1 PFLAG_ANIMATION_STARTED
* 1 PFLAG_SAVE_STATE_CALLED
* 1 PFLAG_ALPHA_SET
* 1 PFLAG_SCROLL_CONTAINER
* 1 PFLAG_SCROLL_CONTAINER_ADDED
* 1 PFLAG_DIRTY
* 1 PFLAG_DIRTY_MASK
* 1 PFLAG_OPAQUE_BACKGROUND
* 1 PFLAG_OPAQUE_SCROLLBARS
* 11 PFLAG_OPAQUE_MASK
* 1 PFLAG_PREPRESSED
* 1 PFLAG_CANCEL_NEXT_UP_EVENT
* 1 PFLAG_AWAKEN_SCROLL_BARS_ON_ATTACH
* 1 PFLAG_HOVERED
* 1 PFLAG_NOTIFY_AUTOFILL_MANAGER_ON_CLICK
* 1 PFLAG_ACTIVATED
* 1 PFLAG_INVALIDATED
* |-------|-------|-------|-------|
*/
/** {@hide} */
static final int PFLAG_WANTS_FOCUS = 0x00000001;
/** {@hide} */
@@ -2393,7 +2489,9 @@ public class View implements Drawable.Callback, KeyEvent.Callback,
*/
static final int PFLAG_INVALIDATED = 0x80000000;
/**
/* End of masks for mPrivateFlags */
/*
* Masks for mPrivateFlags2, as generated by dumpFlags():
*
* |-------|-------|-------|-------|
@@ -2934,7 +3032,7 @@ public class View implements Drawable.Callback, KeyEvent.Callback,
/* End of masks for mPrivateFlags2 */
/**
/*
* Masks for mPrivateFlags3, as generated by dumpFlags():
*
* |-------|-------|-------|-------|
@@ -3270,6 +3368,57 @@ public class View implements Drawable.Callback, KeyEvent.Callback,
/* End of masks for mPrivateFlags3 */
/*
* Masks for mPrivateFlags4, as generated by dumpFlags():
*
* |-------|-------|-------|-------|
* 1111 PFLAG4_IMPORTANT_FOR_CONTENT_CAPTURE_MASK
* 1 PFLAG4_NOTIFIED_CONTENT_CAPTURE_ON_LAYOUT
* 1 PFLAG4_NOTIFIED_CONTENT_CAPTURE_ADDED
* 1 PFLAG4_LAST_CONTENT_CAPTURE_NOTIFICATION_TYPE
* |-------|-------|-------|-------|
*/
/**
* Mask for obtaining the bits which specify how to determine
* whether a view is important for autofill.
*
* <p>NOTE: the important for content capture values were the first flags added and are set in
* the rightmost position, so we don't need to shift them
*/
private static final int PFLAG4_IMPORTANT_FOR_CONTENT_CAPTURE_MASK =
IMPORTANT_FOR_CONTENT_CAPTURE_AUTO | IMPORTANT_FOR_CONTENT_CAPTURE_YES
| IMPORTANT_FOR_CONTENT_CAPTURE_NO
| IMPORTANT_FOR_CONTENT_CAPTURE_YES_EXCLUDE_DESCENDANTS
| IMPORTANT_FOR_CONTENT_CAPTURE_NO_EXCLUDE_DESCENDANTS;
/*
* Variables used to control when the IntelligenceManager.notifyNodeAdded()/removed() methods
* should be called.
*
* The idea is to call notifyNodeAdded() after the view is layout and visible, then call
* notifyNodeRemoved() when it's gone (without known when it was removed from the parent).
*
* TODO(b/111276913): the current algortighm could probably be optimized and some of them
* removed
*/
private static final int PFLAG4_NOTIFIED_CONTENT_CAPTURE_ON_LAYOUT = 0x10;
private static final int PFLAG4_NOTIFIED_CONTENT_CAPTURE_ADDED = 0x20;
private static final int PFLAG4_LAST_CONTENT_CAPTURE_NOTIFICATION_TYPE = 0x40;
private static final int CONTENT_CAPTURE_NOTIFICATION_TYPE_APPEARED = 1;
private static final int CONTENT_CAPTURE_NOTIFICATION_TYPE_DISAPPEARED = 0;
/** @hide */
@IntDef(flag = true, prefix = { "CONTENT_CAPTURE_NOTIFICATION_TYPE_" }, value = {
CONTENT_CAPTURE_NOTIFICATION_TYPE_APPEARED,
CONTENT_CAPTURE_NOTIFICATION_TYPE_DISAPPEARED
})
@Retention(RetentionPolicy.SOURCE)
public @interface ContentCaptureNotificationType {}
/* End of masks for mPrivateFlags4 */
/**
* Always allow a user to over-scroll this view, provided it is a
* view that can scroll.
@@ -3861,6 +4010,8 @@ public class View implements Drawable.Callback, KeyEvent.Callback,
@UnsupportedAppUsage
int mPrivateFlags3;
private int mPrivateFlags4;
/**
* This view's request for the visibility of the status bar.
* @hide
@@ -5803,6 +5954,7 @@ public class View implements Drawable.Callback, KeyEvent.Callback,
mAttributes = trimmed;
}
@Override
public String toString() {
StringBuilder out = new StringBuilder(128);
out.append(getClass().getName());
@@ -5875,6 +6027,9 @@ public class View implements Drawable.Callback, KeyEvent.Callback,
}
}
}
if (mAutofillId != null) {
out.append(" aid="); out.append(mAutofillId);
}
out.append("}");
return out.toString();
}
@@ -7888,7 +8043,8 @@ public class View implements Drawable.Callback, KeyEvent.Callback,
* fills in all data that can be inferred from the view itself.
*/
public void onProvideStructure(ViewStructure structure) {
onProvideStructureForAssistOrAutofill(structure, false, 0);
onProvideStructureForAssistOrAutofillOrViewCapture(structure, /* forAutofill = */ false,
/* forViewCapture= */ false, /* flags= */ 0);
}
/**
@@ -7961,11 +8117,46 @@ public class View implements Drawable.Callback, KeyEvent.Callback,
* @see #AUTOFILL_FLAG_INCLUDE_NOT_IMPORTANT_VIEWS
*/
public void onProvideAutofillStructure(ViewStructure structure, @AutofillFlags int flags) {
onProvideStructureForAssistOrAutofill(structure, true, flags);
onProvideStructureForAssistOrAutofillOrViewCapture(structure, /* forAutofill = */ true,
/* forViewCapture= */ false, flags);
}
private void onProvideStructureForAssistOrAutofill(ViewStructure structure,
boolean forAutofill, @AutofillFlags int flags) {
/**
* Populates a {@link ViewStructure} for Content Capture.
*
* <p>This method is called after a view is that is eligible for Content Capture
* (for example, if it {@link #isImportantForAutofill()}, an intelligence service is enabled for
* the user, and the activity rendering the view is enabled for Content Capture) is laid out and
* is visible.
*
* <p><b>Note: </b>the following methods of the {@code structure} will be ignored:
* <ul>
* <li>{@link ViewStructure#setChildCount(int)}
* <li>{@link ViewStructure#addChildCount(int)}
* <li>{@link ViewStructure#getChildCount()}
* <li>{@link ViewStructure#newChild(int)}
* <li>{@link ViewStructure#asyncNewChild(int)}
* <li>{@link ViewStructure#asyncCommit()}
* <li>{@link ViewStructure#setWebDomain(String)}
* <li>{@link ViewStructure#newHtmlInfoBuilder(String)}
* <li>{@link ViewStructure#setHtmlInfo(android.view.ViewStructure.HtmlInfo)}
* <li>{@link ViewStructure#setDataIsSensitive(boolean)}
* </ul>
*
* @return whether the IntelligenceService should be notified that the view was added (through
* the {@link IntelligenceManager#notifyViewAppeared(ViewStructure)} method) to the view
* hierarchy. Most views should return {@code true} here, but views that contains virtual
* hierarchy might opt to return {@code false} and notify the manager independently, as the
* virtual views are rendered.
*/
public boolean onProvideContentCaptureStructure(@NonNull ViewStructure structure, int flags) {
onProvideStructureForAssistOrAutofillOrViewCapture(structure, /* forAutofill = */ false,
/* forViewCapture= */ true, flags);
return true;
}
private void onProvideStructureForAssistOrAutofillOrViewCapture(ViewStructure structure,
boolean forAutofill, boolean forViewCapture, @AutofillFlags int flags) {
final int id = mID;
if (id != NO_ID && !isViewIdGenerated(id)) {
String pkg, type, entry;
@@ -7981,8 +8172,11 @@ public class View implements Drawable.Callback, KeyEvent.Callback,
} else {
structure.setId(id, null, null, null);
}
if (forViewCapture) {
structure.setDataIsSensitive(false);
}
if (forAutofill) {
if (forAutofill || forViewCapture) {
final @AutofillType int autofillType = getAutofillType();
// Don't need to fill autofill info if view does not support it.
// For example, only TextViews that are editable support autofill
@@ -8530,9 +8724,8 @@ public class View implements Drawable.Callback, KeyEvent.Callback,
if (parentImportance == IMPORTANT_FOR_AUTOFILL_NO_EXCLUDE_DESCENDANTS
|| parentImportance == IMPORTANT_FOR_AUTOFILL_YES_EXCLUDE_DESCENDANTS) {
if (Log.isLoggable(AUTOFILL_LOG_TAG, Log.VERBOSE)) {
Log.v(AUTOFILL_LOG_TAG, "View (autofillId=" + getAutofillViewId() + ", "
+ getClass() + ") is not important for autofill because parent "
+ parent + "'s importance is " + parentImportance);
Log.v(AUTOFILL_LOG_TAG, "View (" + this + ") is not important for autofill "
+ "because parent " + parent + "'s importance is " + parentImportance);
}
return false;
}
@@ -8549,14 +8742,18 @@ public class View implements Drawable.Callback, KeyEvent.Callback,
if (importance == IMPORTANT_FOR_AUTOFILL_NO_EXCLUDE_DESCENDANTS
|| importance == IMPORTANT_FOR_AUTOFILL_NO) {
if (Log.isLoggable(AUTOFILL_LOG_TAG, Log.VERBOSE)) {
Log.v(AUTOFILL_LOG_TAG, "View (autofillId=" + getAutofillViewId() + ", "
+ getClass() + ") is not important for autofill because its "
+ "importance is " + importance);
Log.v(AUTOFILL_LOG_TAG, "View (" + this + ") is not important for autofill "
+ "because its importance is " + importance);
}
return false;
}
// Then use some heuristics to handle AUTO.
if (importance != IMPORTANT_FOR_AUTOFILL_AUTO) {
Log.w(AUTOFILL_LOG_TAG, "invalid autofill importance (" + importance + " on view "
+ this);
return false;
}
// Always include views that have an explicit resource id.
final int id = mID;
@@ -8584,6 +8781,198 @@ public class View implements Drawable.Callback, KeyEvent.Callback,
return false;
}
/**
* Gets the mode for determining whether this view is important for content capture.
*
* <p>See {@link #setImportantForContentCapture(int)} and
* {@link #isImportantForContentCapture()} for more info about this mode.
*
* @return {@link #IMPORTANT_FOR_CONTENT_CAPTURE_AUTO} by default, or value passed to
* {@link #setImportantForContentCapture(int)}.
*
* @attr ref android.R.styleable#View_importantForContentCapture
*/
@ViewDebug.ExportedProperty(mapping = {
@ViewDebug.IntToString(from = IMPORTANT_FOR_CONTENT_CAPTURE_AUTO, to = "auto"),
@ViewDebug.IntToString(from = IMPORTANT_FOR_CONTENT_CAPTURE_YES, to = "yes"),
@ViewDebug.IntToString(from = IMPORTANT_FOR_CONTENT_CAPTURE_NO, to = "no"),
@ViewDebug.IntToString(from = IMPORTANT_FOR_CONTENT_CAPTURE_YES_EXCLUDE_DESCENDANTS,
to = "yesExcludeDescendants"),
@ViewDebug.IntToString(from = IMPORTANT_FOR_CONTENT_CAPTURE_NO_EXCLUDE_DESCENDANTS,
to = "noExcludeDescendants")})
public @ContentCaptureImportance int getImportantForContentCapture() {
// NOTE: the important for content capture values were the first flags added and are set in
// the rightmost position, so we don't need to shift them
return mPrivateFlags4 & PFLAG4_IMPORTANT_FOR_CONTENT_CAPTURE_MASK;
}
/**
* Sets the mode for determining whether this view is considered important for content capture.
*
* <p>The platform determines the importance for autofill automatically but you
* can use this method to customize the behavior. Typically, a view that provides text should
* be marked as {@link #IMPORTANT_FOR_CONTENT_CAPTURE_YES}.
*
* @param mode {@link #IMPORTANT_FOR_CONTENT_CAPTURE_AUTO},
* {@link #IMPORTANT_FOR_CONTENT_CAPTURE_YES}, {@link #IMPORTANT_FOR_CONTENT_CAPTURE_NO},
* {@link #IMPORTANT_FOR_CONTENT_CAPTURE_YES_EXCLUDE_DESCENDANTS},
* or {@link #IMPORTANT_FOR_CONTENT_CAPTURE_NO_EXCLUDE_DESCENDANTS}.
*
* @attr ref android.R.styleable#View_importantForContentCapture
*/
public void setImportantForContentCapture(@ContentCaptureImportance int mode) {
// Reset first
mPrivateFlags4 &= ~PFLAG4_IMPORTANT_FOR_CONTENT_CAPTURE_MASK;
// Then set again
// NOTE: the important for content capture values were the first flags added and are set in
// the rightmost position, so we don't need to shift them
mPrivateFlags4 |= (mode & PFLAG4_IMPORTANT_FOR_CONTENT_CAPTURE_MASK);
}
/**
* Hints the Android System whether this view is considered important for Content Capture, based
* on the value explicitly set by {@link #setImportantForContentCapture(int)} and heuristics
* when it's {@link #IMPORTANT_FOR_CONTENT_CAPTURE_AUTO}.
*
* @return whether the view is considered important for autofill.
*
* @see #setImportantForContentCapture(int)
* @see #IMPORTANT_FOR_CONTENT_CAPTURE_AUTO
* @see #IMPORTANT_FOR_CONTENT_CAPTURE_YES
* @see #IMPORTANT_FOR_CONTENT_CAPTURE_NO
* @see #IMPORTANT_FOR_CONTENT_CAPTURE_YES_EXCLUDE_DESCENDANTS
* @see #IMPORTANT_FOR_CONTENT_CAPTURE_NO_EXCLUDE_DESCENDANTS
*/
public final boolean isImportantForContentCapture() {
// Check parent mode to ensure we're important
ViewParent parent = mParent;
while (parent instanceof View) {
final int parentImportance = ((View) parent).getImportantForContentCapture();
if (parentImportance == IMPORTANT_FOR_CONTENT_CAPTURE_NO_EXCLUDE_DESCENDANTS
|| parentImportance == IMPORTANT_FOR_CONTENT_CAPTURE_YES_EXCLUDE_DESCENDANTS) {
if (Log.isLoggable(CONTENT_CAPTURE_LOG_TAG, Log.VERBOSE)) {
Log.v(CONTENT_CAPTURE_LOG_TAG, "View (" + this + ") is not important for "
+ "content capture because parent " + parent + "'s importance is "
+ parentImportance);
}
return false;
}
parent = parent.getParent();
}
final int importance = getImportantForContentCapture();
// First, check the explicit states.
if (importance == IMPORTANT_FOR_CONTENT_CAPTURE_YES_EXCLUDE_DESCENDANTS
|| importance == IMPORTANT_FOR_CONTENT_CAPTURE_YES) {
return true;
}
if (importance == IMPORTANT_FOR_CONTENT_CAPTURE_NO_EXCLUDE_DESCENDANTS
|| importance == IMPORTANT_FOR_CONTENT_CAPTURE_NO) {
if (Log.isLoggable(CONTENT_CAPTURE_LOG_TAG, Log.VERBOSE)) {
Log.v(CONTENT_CAPTURE_LOG_TAG, "View (" + this + ") is not important for content "
+ "capture because its importance is " + importance);
}
return false;
}
// Then use some heuristics to handle AUTO.
if (importance != IMPORTANT_FOR_CONTENT_CAPTURE_AUTO) {
Log.w(CONTENT_CAPTURE_LOG_TAG, "invalid content capture importance (" + importance
+ " on view " + this);
return false;
}
// View group is important if at least one children also is
//TODO(b/111276913): decide if we really need to send the relevant parents or just the
// leaves (with absolute coordinates). If it's the latter, then we need to update this
// javadoc and ViewGroup's implementation.
if (this instanceof ViewGroup) {
final ViewGroup group = (ViewGroup) this;
for (int i = 0; i < group.getChildCount(); i++) {
final View child = group.getChildAt(i);
if (child.isImportantForContentCapture()) {
return true;
}
}
}
// If the app developer explicitly set hints or autofill hintsfor it, it's important.
if (getAutofillHints() != null) {
return true;
}
// Otherwise, assume it's not important...
return false;
}
/**
* Helper used to notify the {@link IntelligenceManager}anager when the view is removed or
* added, based on whether it's laid out and visible, and without knowing if the parent removed
* it from the view
* hierarchy.
*/
// TODO(b/111276913): make sure the current algorithm covers all cases. For example, it should
// probably be called every time notifyEnterOrExitForAutoFillIfNeeded() is called as well.
private void notifyNodeAddedOrRemovedForContentCaptureIfNeeded(
@ContentCaptureNotificationType int type) {
if (type != CONTENT_CAPTURE_NOTIFICATION_TYPE_APPEARED
&& type != CONTENT_CAPTURE_NOTIFICATION_TYPE_DISAPPEARED) {
// Sanity check so it does not screw up the flags
Log.wtf(CONTENT_CAPTURE_LOG_TAG, "notifyNodeAddedOrRemovedForContentCaptureIfNeeded(): "
+ "invalid type " + type + " for " + this);
return;
}
if (!isImportantForContentCapture()) return;
final IntelligenceManager im = mContext.getSystemService(IntelligenceManager.class);
if (im == null || !im.isContentCaptureEnabled()) return;
// Make sure event is notified just once, and reset the
// PFLAG4_LAST_CONTENT_CAPTURE_NOTIFICATION_TYPE flag
boolean ignoreNotification = false;
if (type == CONTENT_CAPTURE_NOTIFICATION_TYPE_APPEARED) {
if ((mPrivateFlags4 & PFLAG4_LAST_CONTENT_CAPTURE_NOTIFICATION_TYPE)
== CONTENT_CAPTURE_NOTIFICATION_TYPE_APPEARED) {
ignoreNotification = true;
} else {
mPrivateFlags4 |= PFLAG4_LAST_CONTENT_CAPTURE_NOTIFICATION_TYPE;
}
} else {
if ((mPrivateFlags4 & PFLAG4_LAST_CONTENT_CAPTURE_NOTIFICATION_TYPE)
== CONTENT_CAPTURE_NOTIFICATION_TYPE_DISAPPEARED) {
ignoreNotification = true;
} else {
mPrivateFlags4 &= ~PFLAG4_LAST_CONTENT_CAPTURE_NOTIFICATION_TYPE;
}
}
if (ignoreNotification) {
if (Log.isLoggable(CONTENT_CAPTURE_LOG_TAG, Log.VERBOSE)) {
// TODO(b/111276913): remove this log statement if the algorithm is not improved
// (right now it's called too many times when the activity is stopped and/or views
// disappear
Log.v(CONTENT_CAPTURE_LOG_TAG, "notifyNodeAddedOrRemovedForContentCaptureIfNeeded("
+ type + "): ignoring repeated notification on " + this);
}
return;
}
if (type == CONTENT_CAPTURE_NOTIFICATION_TYPE_APPEARED) {
final ViewStructure structure = im.newViewStructure(this);
boolean notifyMgr = onProvideContentCaptureStructure(structure, /* flags= */ 0);
if (notifyMgr) {
im.notifyViewAppeared(structure);
}
mPrivateFlags4 |= PFLAG4_NOTIFIED_CONTENT_CAPTURE_ADDED;
} else {
if ((mPrivateFlags4 & PFLAG4_NOTIFIED_CONTENT_CAPTURE_ADDED) == 0) {
return; // skip initial notification
}
im.notifyViewDisappeared(getAutofillId());
}
}
@Nullable
private AutofillManager getAutofillManager() {
return mContext.getSystemService(AutofillManager.class);
@@ -13094,6 +13483,9 @@ public class View implements Drawable.Callback, KeyEvent.Callback,
: AccessibilityEvent.CONTENT_CHANGE_TYPE_PANE_DISAPPEARED);
}
}
notifyNodeAddedOrRemovedForContentCaptureIfNeeded(isVisible
? CONTENT_CAPTURE_NOTIFICATION_TYPE_APPEARED
: CONTENT_CAPTURE_NOTIFICATION_TYPE_DISAPPEARED);
}
/**
@@ -18630,6 +19022,8 @@ public class View implements Drawable.Callback, KeyEvent.Callback,
}
notifyEnterOrExitForAutoFillIfNeeded(false);
notifyNodeAddedOrRemovedForContentCaptureIfNeeded(
CONTENT_CAPTURE_NOTIFICATION_TYPE_DISAPPEARED);
}
/**
@@ -20934,6 +21328,13 @@ public class View implements Drawable.Callback, KeyEvent.Callback,
mPrivateFlags3 &= ~PFLAG3_NOTIFY_AUTOFILL_ENTER_ON_LAYOUT;
notifyEnterOrExitForAutoFillIfNeeded(true);
}
if ((mViewFlags & VISIBILITY_MASK) == VISIBLE
&& (mPrivateFlags4 & PFLAG4_NOTIFIED_CONTENT_CAPTURE_ON_LAYOUT) == 0) {
notifyNodeAddedOrRemovedForContentCaptureIfNeeded(
CONTENT_CAPTURE_NOTIFICATION_TYPE_APPEARED);
mPrivateFlags4 |= PFLAG4_NOTIFIED_CONTENT_CAPTURE_ON_LAYOUT;
}
}
private boolean hasParentWantsFocus() {

1
core/java/android/view/ViewStructure.java
View File

@@ -24,7 +24,6 @@ import android.os.Bundle;
import android.os.LocaleList;
import android.util.Pair;
import android.view.View.AutofillImportance;
import android.view.ViewStructure.HtmlInfo;
import android.view.autofill.AutofillId;
import android.view.autofill.AutofillValue;

110
core/java/android/view/intelligence/ContentCaptureEvent.java
View File

@@ -16,12 +16,17 @@
package android.view.intelligence;
import android.annotation.IntDef;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.annotation.SystemApi;
import android.os.Parcel;
import android.os.Parcelable;
import android.os.SystemClock;
import android.view.autofill.AutofillId;
import com.android.internal.util.Preconditions;
import java.io.PrintWriter;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
@@ -60,14 +65,14 @@ public final class ContentCaptureEvent implements Parcelable {
*
* <p>The metadata of the node is available through {@link #getViewNode()}.
*/
public static final int TYPE_VIEW_ADDED = 5;
public static final int TYPE_VIEW_APPEARED = 5;
/**
* Called when a node has been removed from the screen and is not visible to the user anymore.
*
* <p>The id of the node is available through {@link #getId()}.
*/
public static final int TYPE_VIEW_REMOVED = 6;
public static final int TYPE_VIEW_DISAPPEARED = 6;
/**
* Called when the text of a node has been changed.
@@ -85,8 +90,8 @@ public final class ContentCaptureEvent implements Parcelable {
TYPE_ACTIVITY_PAUSED,
TYPE_ACTIVITY_RESUMED,
TYPE_ACTIVITY_STOPPED,
TYPE_VIEW_ADDED,
TYPE_VIEW_REMOVED,
TYPE_VIEW_APPEARED,
TYPE_VIEW_DISAPPEARED,
TYPE_VIEW_TEXT_CHANGED
})
@Retention(RetentionPolicy.SOURCE)
@@ -95,7 +100,9 @@ public final class ContentCaptureEvent implements Parcelable {
private final int mType;
private final long mEventTime;
private final int mFlags;
private @Nullable AutofillId mId;
private @Nullable ViewNode mNode;
private @Nullable CharSequence mText;
/** @hide */
public ContentCaptureEvent(int type, long eventTime, int flags) {
@@ -104,12 +111,42 @@ public final class ContentCaptureEvent implements Parcelable {
mFlags = flags;
}
/** @hide */
public ContentCaptureEvent(int type, int flags) {
this(type, SystemClock.uptimeMillis(), flags);
}
/** @hide */
public ContentCaptureEvent(int type) {
this(type, /* flags= */ 0);
}
/** @hide */
public ContentCaptureEvent setAutofillId(@NonNull AutofillId id) {
mId = Preconditions.checkNotNull(id);
return this;
}
/** @hide */
public ContentCaptureEvent setViewNode(@NonNull ViewNode node) {
mNode = Preconditions.checkNotNull(node);
return this;
}
/** @hide */
public ContentCaptureEvent setText(@Nullable CharSequence text) {
mText = text;
return this;
}
/**
* Gets the type of the event.
*
* @return one of {@link #TYPE_ACTIVITY_STARTED}, {@link #TYPE_ACTIVITY_RESUMED},
* {@link #TYPE_ACTIVITY_PAUSED}, {@link #TYPE_ACTIVITY_STOPPED},
* {@link #TYPE_VIEW_ADDED}, {@link #TYPE_VIEW_REMOVED}, or {@link #TYPE_VIEW_TEXT_CHANGED}.
* {@link #TYPE_VIEW_APPEARED}, {@link #TYPE_VIEW_DISAPPEARED},
* or {@link #TYPE_VIEW_TEXT_CHANGED}.
*/
public @EventType int getType() {
return mType;
@@ -135,21 +172,21 @@ public final class ContentCaptureEvent implements Parcelable {
/**
* Gets the whole metadata of the node associated with the event.
*
* <p>Only set on {@link #TYPE_VIEW_ADDED} events.
* <p>Only set on {@link #TYPE_VIEW_APPEARED} events.
*/
@Nullable
public ViewNode getViewNode() {
return null;
return mNode;
}
/**
* Gets the {@link AutofillId} of the node associated with the event.
*
* <p>Only set on {@link #TYPE_VIEW_REMOVED} and {@link #TYPE_VIEW_TEXT_CHANGED} events.
* <p>Only set on {@link #TYPE_VIEW_DISAPPEARED} and {@link #TYPE_VIEW_TEXT_CHANGED} events.
*/
@Nullable
public AutofillId getId() {
return null;
return mId;
}
/**
@@ -159,16 +196,41 @@ public final class ContentCaptureEvent implements Parcelable {
*/
@Nullable
public CharSequence getText() {
return null;
return mText;
}
/** @hide */
public void dump(@NonNull PrintWriter pw) {
pw.print("type="); pw.print(getTypeAsString(mType));
pw.print(", time="); pw.print(mEventTime);
if (mFlags > 0) {
pw.print(", flags="); pw.print(mFlags);
}
if (mId != null) {
pw.print(", id="); pw.print(mId);
}
if (mNode != null) {
pw.print(", id="); pw.print(mNode.getAutofillId());
}
}
@Override
public String toString() {
final StringBuilder string = new StringBuilder("ContentCaptureEvent[type=")
.append(getTypeAsString(mType)).append(", time=").append(mEventTime);
.append(getTypeAsString(mType));
if (mFlags > 0) {
string.append(", flags=").append(mFlags);
}
if (mId != null) {
string.append(", id=").append(mId);
}
if (mNode != null) {
final String className = mNode.getClassName();
if (mNode != null) {
string.append(", class=").append(className);
}
string.append(", id=").append(mNode.getAutofillId());
}
return string.append(']').toString();
}
@@ -182,6 +244,9 @@ public final class ContentCaptureEvent implements Parcelable {
parcel.writeInt(mType);
parcel.writeLong(mEventTime);
parcel.writeInt(mFlags);
parcel.writeParcelable(mId, flags);
ViewNode.writeToParcel(parcel, mNode, flags);
parcel.writeCharSequence(mText);
}
public static final Parcelable.Creator<ContentCaptureEvent> CREATOR =
@@ -192,7 +257,17 @@ public final class ContentCaptureEvent implements Parcelable {
final int type = parcel.readInt();
final long eventTime = parcel.readLong();
final int flags = parcel.readInt();
return new ContentCaptureEvent(type, eventTime, flags);
final ContentCaptureEvent event = new ContentCaptureEvent(type, eventTime, flags);
final AutofillId id = parcel.readParcelable(null);
if (id != null) {
event.setAutofillId(id);
}
final ViewNode node = ViewNode.readFromParcel(parcel);
if (node != null) {
event.setViewNode(node);
}
event.setText(parcel.readCharSequence());
return event;
}
@Override
@@ -201,7 +276,6 @@ public final class ContentCaptureEvent implements Parcelable {
}
};
/** @hide */
public static String getTypeAsString(@EventType int type) {
switch (type) {
@@ -213,10 +287,10 @@ public final class ContentCaptureEvent implements Parcelable {
return "ACTIVITY_PAUSED";
case TYPE_ACTIVITY_STOPPED:
return "ACTIVITY_STOPPED";
case TYPE_VIEW_ADDED:
return "VIEW_ADDED";
case TYPE_VIEW_REMOVED:
return "VIEW_REMOVED";
case TYPE_VIEW_APPEARED:
return "VIEW_APPEARED";
case TYPE_VIEW_DISAPPEARED:
return "VIEW_DISAPPEARED";
case TYPE_VIEW_TEXT_CHANGED:
return "VIEW_TEXT_CHANGED";
default:

249
core/java/android/view/intelligence/IntelligenceManager.java
View File

@@ -15,6 +15,12 @@
*/
package android.view.intelligence;
import static android.view.intelligence.ContentCaptureEvent.TYPE_VIEW_APPEARED;
import static android.view.intelligence.ContentCaptureEvent.TYPE_VIEW_DISAPPEARED;
import static android.view.intelligence.ContentCaptureEvent.TYPE_VIEW_TEXT_CHANGED;
import static com.android.internal.util.function.pooled.PooledLambda.obtainMessage;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.annotation.SystemApi;
@@ -22,11 +28,15 @@ import android.annotation.SystemService;
import android.content.ComponentName;
import android.content.Context;
import android.os.Bundle;
import android.os.Handler;
import android.os.HandlerThread;
import android.os.IBinder;
import android.os.RemoteException;
import android.os.SystemClock;
import android.service.intelligence.InteractionSessionId;
import android.util.Log;
import android.view.View;
import android.view.ViewStructure;
import android.view.autofill.AutofillId;
import android.view.intelligence.ContentCaptureEvent.EventType;
import com.android.internal.annotations.GuardedBy;
@@ -34,8 +44,7 @@ import com.android.internal.os.IResultReceiver;
import com.android.internal.util.Preconditions;
import java.io.PrintWriter;
import java.util.Arrays;
import java.util.List;
import java.util.ArrayList;
import java.util.Set;
/**
@@ -46,8 +55,9 @@ public final class IntelligenceManager {
private static final String TAG = "IntelligenceManager";
// TODO(b/111276913): define a way to dynamically set it (for example, using settings?)
// TODO(b/111276913): define a way to dynamically set them(for example, using settings?)
private static final boolean VERBOSE = false;
private static final boolean DEBUG = true; // STOPSHIP if not set to false
/**
* Used to indicate that a text change was caused by user input (for example, through IME).
@@ -55,7 +65,6 @@ public final class IntelligenceManager {
//TODO(b/111276913): link to notifyTextChanged() method once available
public static final int FLAG_USER_INPUT = 0x1;
/**
* Initial state, when there is no session.
*
@@ -77,6 +86,15 @@ public final class IntelligenceManager {
*/
public static final int STATE_ACTIVE = 2;
private static final String BG_THREAD_NAME = "intel_svc_streamer_thread";
/**
* Maximum number of events that are delayed for an app.
*
* <p>If the session is not started after the limit is reached, it's discarded.
*/
private static final int MAX_DELAYED_SIZE = 20;
private final Context mContext;
@Nullable
@@ -99,10 +117,24 @@ public final class IntelligenceManager {
@GuardedBy("mLock")
private ComponentName mComponentName;
// TODO(b/111276913): create using maximum batch size as capacity
/**
* List of events held to be sent as a batch.
*/
@GuardedBy("mLock")
private final ArrayList<ContentCaptureEvent> mEvents = new ArrayList<>();
private final Handler mHandler;
/** @hide */
public IntelligenceManager(@NonNull Context context, @Nullable IIntelligenceManager service) {
mContext = Preconditions.checkNotNull(context, "context cannot be null");
mService = service;
// TODO(b/111276913): use an existing bg thread instead...
final HandlerThread bgThread = new HandlerThread(BG_THREAD_NAME);
bgThread.start();
mHandler = Handler.createAsync(bgThread.getLooper());
}
/** @hide */
@@ -111,8 +143,9 @@ public final class IntelligenceManager {
synchronized (mLock) {
if (mState != STATE_UNKNOWN) {
// TODO(b/111276913): revisit this scenario
Log.w(TAG, "ignoring onActivityStarted(" + token + ") while on state "
+ getStateAsStringLocked());
+ getStateAsString(mState));
return;
}
mState = STATE_WAITING_FOR_SERVER;
@@ -121,8 +154,8 @@ public final class IntelligenceManager {
mComponentName = componentName;
if (VERBOSE) {
Log.v(TAG, "onActivityStarted(): token=" + token + ", act=" + componentName
+ ", id=" + mId);
Log.v(TAG, "onActivityCreated(): token=" + token + ", act="
+ getActivityDebugNameLocked() + ", id=" + mId);
}
final int flags = 0; // TODO(b/111276913): get proper flags
@@ -138,12 +171,12 @@ public final class IntelligenceManager {
} else {
// TODO(b/111276913): handle other cases like disabled by
// service
mState = STATE_UNKNOWN;
resetStateLocked();
}
if (VERBOSE) {
Log.v(TAG, "onActivityStarted() result: code=" + resultCode
+ ", id=" + mId
+ ", state=" + getStateAsStringLocked());
+ ", state=" + getStateAsString(mState));
}
}
}
@@ -154,6 +187,60 @@ public final class IntelligenceManager {
}
}
//TODO(b/111276913): should buffer event (and call service on handler thread), instead of
// calling right away
private void sendEvent(@NonNull ContentCaptureEvent event) {
mHandler.sendMessage(obtainMessage(IntelligenceManager::handleSendEvent, this, event));
}
private void handleSendEvent(@NonNull ContentCaptureEvent event) {
synchronized (mLock) {
mEvents.add(event);
final int numberEvents = mEvents.size();
if (mState != STATE_ACTIVE) {
if (numberEvents >= MAX_DELAYED_SIZE) {
// Typically happens on system apps that are started before the system service
// is ready (like com.android.settings/.FallbackHome)
//TODO(b/111276913): try to ignore session while system is not ready / boot
// not complete instead.
Log.w(TAG, "Closing session for " + getActivityDebugNameLocked()
+ " after " + numberEvents + " delayed events");
// TODO(b/111276913): blacklist activity / use special flag to indicate that
// when it's launched again
resetStateLocked();
return;
}
if (VERBOSE) {
Log.v(TAG, "Delaying " + numberEvents + " events for "
+ getActivityDebugNameLocked() + " while on state "
+ getStateAsString(mState));
}
return;
}
if (mId == null) {
// Sanity check - should not happen
Log.wtf(TAG, "null session id for " + mComponentName);
return;
}
//TODO(b/111276913): right now we're sending sending right away (unless not ready), but
// we should hold the events and flush later.
try {
if (DEBUG) {
Log.d(TAG, "Sending " + numberEvents + " event(s) for "
+ getActivityDebugNameLocked());
}
mService.sendEvents(mContext.getUserId(), mId, mEvents);
mEvents.clear();
} catch (RemoteException e) {
throw e.rethrowFromSystemServer();
}
}
}
/**
* Used for intermediate events (i.e, other than created and destroyed).
*
@@ -161,28 +248,11 @@ public final class IntelligenceManager {
*/
public void onActivityLifecycleEvent(@EventType int type) {
if (!isContentCaptureEnabled()) return;
//TODO(b/111276913): should buffer event (and call service on handler thread), instead of
// calling right away
final ContentCaptureEvent event = new ContentCaptureEvent(type, SystemClock.uptimeMillis(),
0);
final List<ContentCaptureEvent> events = Arrays.asList(event);
synchronized (mLock) {
//TODO(b/111276913): check session state; for example, how to handle if it's waiting for
// remote id
if (VERBOSE) {
Log.v(TAG, "onActivityLifecycleEvent() for " + mComponentName.flattenToShortString()
+ ": " + ContentCaptureEvent.getTypeAsString(type));
}
try {
mService.sendEvents(mContext.getUserId(), mId, events);
} catch (RemoteException e) {
throw e.rethrowFromSystemServer();
}
if (VERBOSE) {
Log.v(TAG, "onActivityLifecycleEvent() for " + getActivityDebugNameLocked()
+ ": " + ContentCaptureEvent.getTypeAsString(type));
}
sendEvent(new ContentCaptureEvent(type));
}
/** @hide */
@@ -194,22 +264,105 @@ public final class IntelligenceManager {
// id) and send it to the cache of batched commands
if (VERBOSE) {
Log.v(TAG, "onActivityDestroyed(): state=" + getStateAsStringLocked()
Log.v(TAG, "onActivityDestroyed(): state=" + getStateAsString(mState)
+ ", mId=" + mId);
}
try {
mService.finishSession(mContext.getUserId(), mId);
mState = STATE_UNKNOWN;
mId = null;
mApplicationToken = null;
mComponentName = null;
resetStateLocked();
} catch (RemoteException e) {
throw e.rethrowFromSystemServer();
}
}
}
@GuardedBy("mLock")
private void resetStateLocked() {
mState = STATE_UNKNOWN;
mId = null;
mApplicationToken = null;
mComponentName = null;
mEvents.clear();
}
/**
* Notifies the Intelligence Service that a node has been added to the view structure.
*
* <p>Typically called "manually" by views that handle their own virtual view hierarchy, or
* automatically by the Android System for views that return {@code true} on
* {@link View#onProvideContentCaptureStructure(ViewStructure, int)}.
*
* @param node node that has been added.
*/
public void notifyViewAppeared(@NonNull ViewStructure node) {
Preconditions.checkNotNull(node);
if (!isContentCaptureEnabled()) return;
if (!(node instanceof ViewNode.ViewStructureImpl)) {
throw new IllegalArgumentException("Invalid node class: " + node.getClass());
}
sendEvent(new ContentCaptureEvent(TYPE_VIEW_APPEARED)
.setViewNode(((ViewNode.ViewStructureImpl) node).mNode));
}
/**
* Notifies the Intelligence Service that a node has been removed from the view structure.
*
* <p>Typically called "manually" by views that handle their own virtual view hierarchy, or
* automatically by the Android System for standard views.
*
* @param id id of the node that has been removed.
*/
public void notifyViewDisappeared(@NonNull AutofillId id) {
Preconditions.checkNotNull(id);
if (!isContentCaptureEnabled()) return;
sendEvent(new ContentCaptureEvent(TYPE_VIEW_DISAPPEARED).setAutofillId(id));
}
/**
* Notifies the Intelligence Service that the value of a text node has been changed.
*
* @param id of the node.
* @param text new text.
* @param flags either {@code 0} or {@link #FLAG_USER_INPUT} when the value was explicitly
* changed by the user (for example, through the keyboard).
*/
public void notifyViewTextChanged(@NonNull AutofillId id, @Nullable CharSequence text,
int flags) {
Preconditions.checkNotNull(id);
if (!isContentCaptureEnabled()) return;
sendEvent(new ContentCaptureEvent(TYPE_VIEW_TEXT_CHANGED, flags).setAutofillId(id)
.setText(text));
}
/**
* Creates a {@link ViewStructure} for a "standard" view.
*
* @hide
*/
@NonNull
public ViewStructure newViewStructure(@NonNull View view) {
return new ViewNode.ViewStructureImpl(view);
}
/**
* Creates a {@link ViewStructure} for a "virtual" view, so it can be passed to
* {@link #notifyViewAppeared(ViewStructure)} by the view managing the virtual view hierarchy.
*
* @param parentId id of the virtual view parent (it can be obtained by calling
* {@link ViewStructure#getAutofillId()} on the parent).
* @param virtualId id of the virtual child, relative to the parent.
*
* @return a new {@link ViewStructure} that can be used for Content Capture purposes.
*/
@NonNull
public ViewStructure newVirtualViewStructure(@NonNull AutofillId parentId, int virtualId) {
return new ViewNode.ViewStructureImpl(parentId, virtualId);
}
/**
* Returns the component name of the {@code android.service.intelligence.IntelligenceService}
* that is enabled for the current user.
@@ -322,15 +475,29 @@ public final class IntelligenceManager {
pw.print(prefix2); pw.print("enabled: "); pw.println(isContentCaptureEnabled());
pw.print(prefix2); pw.print("id: "); pw.println(mId);
pw.print(prefix2); pw.print("state: "); pw.print(mState); pw.print(" (");
pw.print(getStateAsStringLocked()); pw.println(")");
pw.print(prefix2); pw.print("appToken: "); pw.println(mApplicationToken);
pw.print(prefix2); pw.print("componentName: "); pw.println(mComponentName);
pw.print(getStateAsString(mState)); pw.println(")");
pw.print(prefix2); pw.print("app token: "); pw.println(mApplicationToken);
pw.print(prefix2); pw.print("component name: ");
pw.println(mComponentName == null ? "null" : mComponentName.flattenToShortString());
final int numberEvents = mEvents.size();
pw.print(prefix2); pw.print("batched events: "); pw.println(numberEvents);
if (numberEvents > 0) {
for (int i = 0; i < numberEvents; i++) {
final ContentCaptureEvent event = mEvents.get(i);
pw.println(i); pw.print(": "); event.dump(pw); pw.println();
}
}
}
}
/**
* Gets a string that can be used to identify the activity on logging statements.
*/
@GuardedBy("mLock")
private String getStateAsStringLocked() {
return getStateAsString(mState);
private String getActivityDebugNameLocked() {
return mComponentName == null ? mContext.getPackageName()
: mComponentName.flattenToShortString();
}
@NonNull

364
core/java/android/view/intelligence/ViewNode.java
View File

@@ -15,10 +15,24 @@
*/
package android.view.intelligence;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.annotation.SystemApi;
import android.app.assist.AssistStructure;
import android.graphics.Matrix;
import android.graphics.Rect;
import android.os.Bundle;
import android.os.LocaleList;
import android.os.Parcel;
import android.util.Log;
import android.view.View;
import android.view.ViewParent;
import android.view.ViewStructure;
import android.view.ViewStructure.HtmlInfo.Builder;
import android.view.autofill.AutofillId;
import android.view.autofill.AutofillValue;
import com.android.internal.util.Preconditions;
//TODO(b/111276913): add javadocs / implement Parcelable / implement
//TODO(b/111276913): for now it's extending ViewNode directly as it needs most of its properties,
@@ -28,6 +42,16 @@ import android.view.autofill.AutofillId;
@SystemApi
public final class ViewNode extends AssistStructure.ViewNode {
private static final String TAG = "ViewNode";
private AutofillId mParentAutofillId;
// TODO(b/111276913): temporarily setting some fields here while they're not accessible from the
// superclass
private AutofillId mAutofillId;
private CharSequence mText;
private String mClassName;
/** @hide */
public ViewNode() {
}
@@ -38,7 +62,343 @@ public final class ViewNode extends AssistStructure.ViewNode {
*/
@Nullable
public AutofillId getParentAutofillId() {
//TODO(b/111276913): implement
return null;
return mParentAutofillId;
}
// TODO(b/111276913): temporarily overwriting some methods
@Override
public AutofillId getAutofillId() {
return mAutofillId;
}
@Override
public CharSequence getText() {
return mText;
}
@Override
public String getClassName() {
return mClassName;
}
/** @hide */
public static void writeToParcel(@NonNull Parcel parcel, @Nullable ViewNode node, int flags) {
if (node == null) {
parcel.writeParcelable(null, flags);
return;
}
parcel.writeParcelable(node.mAutofillId, flags);
parcel.writeParcelable(node.mParentAutofillId, flags);
parcel.writeCharSequence(node.mText);
parcel.writeString(node.mClassName);
}
/** @hide */
public static @Nullable ViewNode readFromParcel(@NonNull Parcel parcel) {
final AutofillId id = parcel.readParcelable(null);
if (id == null) return null;
final ViewNode node = new ViewNode();
node.mAutofillId = id;
node.mParentAutofillId = parcel.readParcelable(null);
node.mText = parcel.readCharSequence();
node.mClassName = parcel.readString();
return node;
}
/** @hide */
static final class ViewStructureImpl extends ViewStructure {
final ViewNode mNode = new ViewNode();
ViewStructureImpl(@NonNull View view) {
mNode.mAutofillId = Preconditions.checkNotNull(view).getAutofillId();
final ViewParent parent = view.getParent();
if (parent instanceof View) {
mNode.mParentAutofillId = ((View) parent).getAutofillId();
}
}
ViewStructureImpl(@NonNull AutofillId parentId, int virtualId) {
mNode.mParentAutofillId = Preconditions.checkNotNull(parentId);
mNode.mAutofillId = new AutofillId(parentId, virtualId);
}
@Override
public void setId(int id, String packageName, String typeName, String entryName) {
// TODO(b/111276913): implement or move to superclass
}
@Override
public void setDimens(int left, int top, int scrollX, int scrollY, int width, int height) {
// TODO(b/111276913): implement or move to superclass
}
@Override
public void setTransformation(Matrix matrix) {
// TODO(b/111276913): implement or move to superclass
}
@Override
public void setElevation(float elevation) {
// TODO(b/111276913): implement or move to superclass
}
@Override
public void setAlpha(float alpha) {
// TODO(b/111276913): implement or move to superclass
}
@Override
public void setVisibility(int visibility) {
// TODO(b/111276913): implement or move to superclass
}
@Override
public void setAssistBlocked(boolean state) {
// TODO(b/111276913): implement or move to superclass
}
@Override
public void setEnabled(boolean state) {
// TODO(b/111276913): implement or move to superclass
}
@Override
public void setClickable(boolean state) {
// TODO(b/111276913): implement or move to superclass
}
@Override
public void setLongClickable(boolean state) {
// TODO(b/111276913): implement or move to superclass
}
@Override
public void setContextClickable(boolean state) {
// TODO(b/111276913): implement or move to superclass
}
@Override
public void setFocusable(boolean state) {
// TODO(b/111276913): implement or move to superclass
}
@Override
public void setFocused(boolean state) {
// TODO(b/111276913): implement or move to superclass
}
@Override
public void setAccessibilityFocused(boolean state) {
// TODO(b/111276913): implement or move to superclass
}
@Override
public void setCheckable(boolean state) {
// TODO(b/111276913): implement or move to superclass
}
@Override
public void setChecked(boolean state) {
// TODO(b/111276913): implement or move to superclass
}
@Override
public void setSelected(boolean state) {
// TODO(b/111276913): implement or move to superclass
}
@Override
public void setActivated(boolean state) {
// TODO(b/111276913): implement or move to superclass
}
@Override
public void setOpaque(boolean opaque) {
// TODO(b/111276913): implement or move to superclass
}
@Override
public void setClassName(String className) {
// TODO(b/111276913): temporarily setting directly; should be done on superclass instead
mNode.mClassName = className;
}
@Override
public void setContentDescription(CharSequence contentDescription) {
// TODO(b/111276913): implement or move to superclass
}
@Override
public void setText(CharSequence text) {
// TODO(b/111276913): temporarily setting directly; should be done on superclass instead
mNode.mText = text;
}
@Override
public void setText(CharSequence text, int selectionStart, int selectionEnd) {
// TODO(b/111276913): implement or move to superclass
}
@Override
public void setTextStyle(float size, int fgColor, int bgColor, int style) {
// TODO(b/111276913): implement or move to superclass
}
@Override
public void setTextLines(int[] charOffsets, int[] baselines) {
// TODO(b/111276913): implement or move to superclass
}
@Override
public void setHint(CharSequence hint) {
// TODO(b/111276913): implement or move to superclass
}
@Override
public CharSequence getText() {
// TODO(b/111276913): temporarily getting directly; should be done on superclass instead
return mNode.mText;
}
@Override
public int getTextSelectionStart() {
// TODO(b/111276913): implement or move to superclass
return 0;
}
@Override
public int getTextSelectionEnd() {
// TODO(b/111276913): implement or move to superclass
return 0;
}
@Override
public CharSequence getHint() {
// TODO(b/111276913): implement or move to superclass
return null;
}
@Override
public Bundle getExtras() {
// TODO(b/111276913): implement or move to superclass
return null;
}
@Override
public boolean hasExtras() {
// TODO(b/111276913): implement or move to superclass
return false;
}
@Override
public void setChildCount(int num) {
Log.w(TAG, "setChildCount() is not supported");
}
@Override
public int addChildCount(int num) {
Log.w(TAG, "addChildCount() is not supported");
return 0;
}
@Override
public int getChildCount() {
Log.w(TAG, "getChildCount() is not supported");
return 0;
}
@Override
public ViewStructure newChild(int index) {
Log.w(TAG, "newChild() is not supported");
return null;
}
@Override
public ViewStructure asyncNewChild(int index) {
Log.w(TAG, "asyncNewChild() is not supported");
return null;
}
@Override
public AutofillId getAutofillId() {
// TODO(b/111276913): temporarily getting directly; should be done on superclass instead
return mNode.mAutofillId;
}
@Override
public void setAutofillId(AutofillId id) {
// TODO(b/111276913): temporarily setting directly; should be done on superclass instead
mNode.mAutofillId = id;
}
@Override
public void setAutofillId(AutofillId parentId, int virtualId) {
// TODO(b/111276913): temporarily setting directly; should be done on superclass instead
mNode.mAutofillId = new AutofillId(parentId, virtualId);
}
@Override
public void setAutofillType(int type) {
// TODO(b/111276913): implement or move to superclass
}
@Override
public void setAutofillHints(String[] hint) {
// TODO(b/111276913): implement or move to superclass
}
@Override
public void setAutofillValue(AutofillValue value) {
// TODO(b/111276913): implement or move to superclass
}
@Override
public void setAutofillOptions(CharSequence[] options) {
// TODO(b/111276913): implement or move to superclass
}
@Override
public void setInputType(int inputType) {
// TODO(b/111276913): implement or move to superclass
}
@Override
public void setDataIsSensitive(boolean sensitive) {
// TODO(b/111276913): implement or move to superclass
}
@Override
public void asyncCommit() {
Log.w(TAG, "asyncCommit() is not supported");
}
@Override
public Rect getTempRect() {
// TODO(b/111276913): implement or move to superclass
return null;
}
@Override
public void setWebDomain(String domain) {
Log.w(TAG, "setWebDomain() is not supported");
}
@Override
public void setLocaleList(LocaleList localeList) {
// TODO(b/111276913): implement or move to superclass
}
@Override
public Builder newHtmlInfoBuilder(String tagName) {
Log.w(TAG, "newHtmlInfoBuilder() is not supported");
return null;
}
@Override
public void setHtmlInfo(HtmlInfo htmlInfo) {
Log.w(TAG, "setHtmlInfo() is not supported");
}
}
}

15
core/java/android/widget/Switch.java
View File

@@ -1422,17 +1422,24 @@ public class Switch extends CompoundButton {
@Override
public void onProvideStructure(ViewStructure structure) {
super.onProvideStructure(structure);
onProvideAutoFillStructureForAssistOrAutofill(structure);
onProvideStructureForAssistOrAutofillOrViewCapture(structure);
}
@Override
public void onProvideAutofillStructure(ViewStructure structure, int flags) {
super.onProvideAutofillStructure(structure, flags);
onProvideAutoFillStructureForAssistOrAutofill(structure);
onProvideStructureForAssistOrAutofillOrViewCapture(structure);
}
// NOTE: currently there is no difference for Assist or AutoFill, so it doesn't take flags
private void onProvideAutoFillStructureForAssistOrAutofill(ViewStructure structure) {
@Override
public boolean onProvideContentCaptureStructure(ViewStructure structure, int flags) {
final boolean notifyManager = super.onProvideContentCaptureStructure(structure, flags);
onProvideStructureForAssistOrAutofillOrViewCapture(structure);
return notifyManager;
}
// NOTE: currently there is no difference for any type, so it doesn't take flags
private void onProvideStructureForAssistOrAutofillOrViewCapture(ViewStructure structure) {
CharSequence switchText = isChecked() ? mTextOn : mTextOff;
if (!TextUtils.isEmpty(switchText)) {
CharSequence oldText = structure.getText();

69
core/java/android/widget/TextView.java
View File

@@ -166,6 +166,7 @@ import android.view.inputmethod.ExtractedText;
import android.view.inputmethod.ExtractedTextRequest;
import android.view.inputmethod.InputConnection;
import android.view.inputmethod.InputMethodManager;
import android.view.intelligence.IntelligenceManager;
import android.view.textclassifier.TextClassification;
import android.view.textclassifier.TextClassificationContext;
import android.view.textclassifier.TextClassificationManager;
@@ -948,6 +949,9 @@ public class TextView extends View implements ViewTreeObserver.OnPreDrawListener
if (getImportantForAutofill() == IMPORTANT_FOR_AUTOFILL_AUTO) {
setImportantForAutofill(IMPORTANT_FOR_AUTOFILL_YES);
}
if (getImportantForContentCapture() == IMPORTANT_FOR_CONTENT_CAPTURE_AUTO) {
setImportantForContentCapture(IMPORTANT_FOR_CONTENT_CAPTURE_YES);
}
setTextInternal("");
@@ -6072,7 +6076,7 @@ public class TextView extends View implements ViewTreeObserver.OnPreDrawListener
if (needEditableForNotification) {
sendAfterTextChanged((Editable) text);
} else {
notifyAutoFillManagerAfterTextChanged();
notifyManagersAfterTextChanged();
}
// SelectionModifierCursorController depends on textCanBeSelected, which depends on text
@@ -10121,23 +10125,33 @@ public class TextView extends View implements ViewTreeObserver.OnPreDrawListener
}
// Always notify AutoFillManager - it will return right away if autofill is disabled.
notifyAutoFillManagerAfterTextChanged();
notifyManagersAfterTextChanged();
hideErrorIfUnchanged();
}
private void notifyAutoFillManagerAfterTextChanged() {
// It is important to not check whether the view is important for autofill
// since the user can trigger autofill manually on not important views.
if (!isAutofillable()) {
return;
}
final AutofillManager afm = mContext.getSystemService(AutofillManager.class);
if (afm != null) {
if (android.view.autofill.Helper.sVerbose) {
Log.v(LOG_TAG, "notifyAutoFillManagerAfterTextChanged");
private void notifyManagersAfterTextChanged() {
// Autofill
if (isAutofillable()) {
// It is important to not check whether the view is important for autofill
// since the user can trigger autofill manually on not important views.
final AutofillManager afm = mContext.getSystemService(AutofillManager.class);
if (afm != null) {
if (android.view.autofill.Helper.sVerbose) {
Log.v(LOG_TAG, "notifyAutoFillManagerAfterTextChanged");
}
afm.notifyValueChanged(TextView.this);
}
}
// ContentCapture
if (isImportantForContentCapture() && isTextEditable()) {
final IntelligenceManager im = mContext.getSystemService(IntelligenceManager.class);
if (im != null && im.isContentCaptureEnabled()) {
// TODO(b/111276913): pass flags when edited by user / add CTS test
im.notifyViewTextChanged(getAutofillId(), getText(), /* flags= */ 0);
}
afm.notifyValueChanged(TextView.this);
}
}
@@ -10900,21 +10914,33 @@ public class TextView extends View implements ViewTreeObserver.OnPreDrawListener
@Override
public void onProvideStructure(ViewStructure structure) {
super.onProvideStructure(structure);
onProvideAutoStructureForAssistOrAutofill(structure, false);
onProvideStructureForAssistOrAutofillOrViewCapture(structure, /* forAutofill = */ false,
/* forViewCapture= */ false);
}
@Override
public void onProvideAutofillStructure(ViewStructure structure, int flags) {
super.onProvideAutofillStructure(structure, flags);
onProvideAutoStructureForAssistOrAutofill(structure, true);
onProvideStructureForAssistOrAutofillOrViewCapture(structure, /* forAutofill = */ true,
/* forViewCapture= */ false);
}
private void onProvideAutoStructureForAssistOrAutofill(ViewStructure structure,
boolean forAutofill) {
@Override
public boolean onProvideContentCaptureStructure(ViewStructure structure, int flags) {
final boolean notifyManager = super.onProvideContentCaptureStructure(structure, flags);
onProvideStructureForAssistOrAutofillOrViewCapture(structure, /* forAutofill = */ false,
/* forViewCapture= */ true);
return notifyManager;
}
private void onProvideStructureForAssistOrAutofillOrViewCapture(ViewStructure structure,
boolean forAutofill, boolean forViewCapture) {
final boolean isPassword = hasPasswordTransformationMethod()
|| isPasswordInputType(getInputType());
if (forAutofill) {
structure.setDataIsSensitive(!mTextSetFromXmlOrResourceId);
if (forAutofill || forViewCapture) {
if (forAutofill) {
structure.setDataIsSensitive(!mTextSetFromXmlOrResourceId);
}
if (mTextId != ResourceId.ID_NULL) {
try {
structure.setTextIdEntry(getResources().getResourceEntryName(mTextId));
@@ -10927,7 +10953,7 @@ public class TextView extends View implements ViewTreeObserver.OnPreDrawListener
}
}
if (!isPassword || forAutofill) {
if (!isPassword || forAutofill || forViewCapture) {
if (mLayout == null) {
assumeLayout();
}
@@ -11043,7 +11069,8 @@ public class TextView extends View implements ViewTreeObserver.OnPreDrawListener
// of the View (and can be any drawable) or a BackgroundColorSpan inside the text.
structure.setTextStyle(getTextSize(), getCurrentTextColor(),
AssistStructure.ViewNode.TEXT_COLOR_UNDEFINED /* bgColor */, style);
} else {
}
if (forAutofill || forViewCapture) {
structure.setMinTextEms(getMinEms());
structure.setMaxTextEms(getMaxEms());
int maxLength = -1;