From 7e568c91776b13911af517ead723a7a87cff0c10 Mon Sep 17 00:00:00 2001 From: Oli Lan Date: Thu, 25 Feb 2021 14:40:59 +0000 Subject: [PATCH] Add text classification results to clip description and clip data. With this change, text classification is performed on text in clip data when it is set as the primary clip, if it is below a certain length. This allows information about which entities (e.g. URLs, addresses) are present in the text to be added to the ClipDescription. With toast notifications being added when an app accesses clipboard, this change will allow apps to tell if clipboard contains information they are interested in before they access the clip data. This will allow unnecessary clipboard accesses to be avoided. If entities are detected in the clip data, the results of classification are added to the ClipData so that users of this API do not then need to repeat text classification themselves in order to locate the entities. Bug: 180577441 Test: new CTS tests added to ClipDescriptionTest Change-Id: I334db4805b8b72e0e00c721fe6efe64ec0661a8a --- core/api/current.txt | 7 ++ core/java/android/content/ClipData.java | 29 +++++ .../java/android/content/ClipDescription.java | 110 ++++++++++++++++++ .../view/textclassifier/TextClassifier.java | 4 +- .../server/clipboard/ClipboardService.java | 89 ++++++++++++++ 5 files changed, 238 insertions(+), 1 deletion(-) diff --git a/core/api/current.txt b/core/api/current.txt index 17acbf3e17718..cb4e9cd272f0a 100644 --- a/core/api/current.txt +++ b/core/api/current.txt @@ -9908,6 +9908,7 @@ package android.content { method public String getHtmlText(); method public android.content.Intent getIntent(); method public CharSequence getText(); + method @Nullable public android.view.textclassifier.TextLinks getTextLinks(); method public android.net.Uri getUri(); } @@ -9917,6 +9918,8 @@ package android.content { method public static boolean compareMimeTypes(String, String); method public int describeContents(); method public String[] filterMimeTypes(String); + method public int getClassificationStatus(); + method @FloatRange(from=0.0, to=1.0) public float getConfidenceScore(@NonNull String); method public android.os.PersistableBundle getExtras(); method public CharSequence getLabel(); method public String getMimeType(int); @@ -9926,6 +9929,9 @@ package android.content { method public boolean isStyledText(); method public void setExtras(android.os.PersistableBundle); method public void writeToParcel(android.os.Parcel, int); + field public static final int CLASSIFICATION_COMPLETE = 3; // 0x3 + field public static final int CLASSIFICATION_NOT_COMPLETE = 1; // 0x1 + field public static final int CLASSIFICATION_NOT_PERFORMED = 2; // 0x2 field @NonNull public static final android.os.Parcelable.Creator CREATOR; field public static final String MIMETYPE_TEXT_HTML = "text/html"; field public static final String MIMETYPE_TEXT_INTENT = "text/vnd.android.intent"; @@ -51927,6 +51933,7 @@ package android.view.textclassifier { field public static final String TYPE_PHONE = "phone"; field public static final String TYPE_UNKNOWN = ""; field public static final String TYPE_URL = "url"; + field public static final String WIDGET_TYPE_CLIPBOARD = "clipboard"; field public static final String WIDGET_TYPE_CUSTOM_EDITTEXT = "customedit"; field public static final String WIDGET_TYPE_CUSTOM_TEXTVIEW = "customview"; field public static final String WIDGET_TYPE_CUSTOM_UNSELECTABLE_TEXTVIEW = "nosel-customview"; diff --git a/core/java/android/content/ClipData.java b/core/java/android/content/ClipData.java index f3ecbf6c08f34..6ab1975f97176 100644 --- a/core/java/android/content/ClipData.java +++ b/core/java/android/content/ClipData.java @@ -21,6 +21,7 @@ import static android.content.ContentResolver.SCHEME_ANDROID_RESOURCE; import static android.content.ContentResolver.SCHEME_CONTENT; import static android.content.ContentResolver.SCHEME_FILE; +import android.annotation.Nullable; import android.compat.annotation.UnsupportedAppUsage; import android.content.pm.ActivityInfo; import android.content.res.AssetFileDescriptor; @@ -38,6 +39,7 @@ import android.text.TextUtils; import android.text.style.URLSpan; import android.util.Log; import android.util.proto.ProtoOutputStream; +import android.view.textclassifier.TextLinks; import com.android.internal.util.ArrayUtils; @@ -204,6 +206,7 @@ public class ClipData implements Parcelable { Uri mUri; // Additional activity info resolved by the system ActivityInfo mActivityInfo; + private TextLinks mTextLinks; /** @hide */ public Item(Item other) { @@ -331,6 +334,29 @@ public class ClipData implements Parcelable { mActivityInfo = info; } + /** + * Returns the results of text classification run on the raw text contained in this item, + * if it was performed, and if any entities were found in the text. Classification is + * generally only performed on the first item in clip data, and only if the text is below a + * certain length. + * + *

Returns {@code null} if classification was not performed, or if no entities were + * found in the text. + * + * @see ClipDescription#getConfidenceScore(String) + */ + @Nullable + public TextLinks getTextLinks() { + return mTextLinks; + } + + /** + * @hide + */ + public void setTextLinks(TextLinks textLinks) { + mTextLinks = textLinks; + } + /** * Turn this item into text, regardless of the type of data it * actually contains. @@ -1183,6 +1209,7 @@ public class ClipData implements Parcelable { dest.writeTypedObject(item.mIntent, flags); dest.writeTypedObject(item.mUri, flags); dest.writeTypedObject(item.mActivityInfo, flags); + dest.writeTypedObject(item.mTextLinks, flags); } } @@ -1201,8 +1228,10 @@ public class ClipData implements Parcelable { Intent intent = in.readTypedObject(Intent.CREATOR); Uri uri = in.readTypedObject(Uri.CREATOR); ActivityInfo info = in.readTypedObject(ActivityInfo.CREATOR); + TextLinks textLinks = in.readTypedObject(TextLinks.CREATOR); Item item = new Item(text, htmlText, intent, uri); item.setActivityInfo(info); + item.setTextLinks(textLinks); mItems.add(item); } } diff --git a/core/java/android/content/ClipDescription.java b/core/java/android/content/ClipDescription.java index d48f83223d3e2..f49362e9300e8 100644 --- a/core/java/android/content/ClipDescription.java +++ b/core/java/android/content/ClipDescription.java @@ -16,16 +16,25 @@ package android.content; +import android.annotation.FloatRange; +import android.annotation.IntDef; import android.annotation.NonNull; +import android.os.Bundle; import android.os.Parcel; import android.os.Parcelable; import android.os.PersistableBundle; import android.text.TextUtils; +import android.util.ArrayMap; import android.util.TimeUtils; import android.util.proto.ProtoOutputStream; +import android.view.textclassifier.TextClassifier; +import android.view.textclassifier.TextLinks; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; import java.util.ArrayList; import java.util.Arrays; +import java.util.Map; /** * Meta-data describing the contents of a {@link ClipData}. Provides enough @@ -115,12 +124,39 @@ public class ClipDescription implements Parcelable { */ public static final String EXTRA_ACTIVITY_OPTIONS = "android.intent.extra.ACTIVITY_OPTIONS"; + /** @hide */ + @Retention(RetentionPolicy.SOURCE) + @IntDef(value = + { CLASSIFICATION_NOT_COMPLETE, CLASSIFICATION_NOT_PERFORMED, CLASSIFICATION_COMPLETE}) + @interface ClassificationStatus {} + + /** + * Value returned by {@link #getConfidenceScore(String)} if text classification has not been + * completed on the associated clip. This will be always be the case if the clip has not been + * copied to clipboard, or if there is no associated clip. + */ + public static final int CLASSIFICATION_NOT_COMPLETE = 1; + + /** + * Value returned by {@link #getConfidenceScore(String)} if text classification was not and will + * not be performed on the associated clip. This may be the case if the clip does not contain + * text in its first item, or if the text is too long. + */ + public static final int CLASSIFICATION_NOT_PERFORMED = 2; + + /** + * Value returned by {@link #getConfidenceScore(String)} if text classification has been + * completed. + */ + public static final int CLASSIFICATION_COMPLETE = 3; final CharSequence mLabel; private final ArrayList mMimeTypes; private PersistableBundle mExtras; private long mTimeStamp; private boolean mIsStyledText; + private final ArrayMap mEntityConfidence = new ArrayMap<>(); + private int mClassificationStatus = CLASSIFICATION_NOT_COMPLETE; /** * Create a new clip. @@ -346,6 +382,61 @@ public class ClipDescription implements Parcelable { mIsStyledText = isStyledText; } + /** + * Sets the current status of text classification for the associated clip. + * + * @hide + */ + public void setClassificationStatus(@ClassificationStatus int status) { + mClassificationStatus = status; + } + + /** + * Returns a score indicating confidence that an instance of the given entity is present in the + * first item of the clip data, if that item is plain text and text classification has been + * performed. The value ranges from 0 (low confidence) to 1 (high confidence). 0 indicates that + * the entity was not found in the classified text. + * + *

Entities should be as defined in the {@link TextClassifier} class, such as + * {@link TextClassifier#TYPE_ADDRESS}, {@link TextClassifier#TYPE_URL}, or + * {@link TextClassifier#TYPE_EMAIL}. + * + *

If the result is positive for any entity, the full classification result as a + * {@link TextLinks} object may be obtained using the {@link ClipData.Item#getTextLinks()} + * method. + * + * @throws IllegalStateException if {@link #getClassificationStatus()} is not + * {@link #CLASSIFICATION_COMPLETE} + */ + @FloatRange(from = 0.0, to = 1.0) + public float getConfidenceScore(@NonNull @TextClassifier.EntityType String entity) { + if (mClassificationStatus != CLASSIFICATION_COMPLETE) { + throw new IllegalStateException("Classification not complete"); + } + return mEntityConfidence.getOrDefault(entity, 0f); + } + + /** + * Returns {@link #CLASSIFICATION_COMPLETE} if text classification has been performed on the + * associated {@link ClipData}. If this is the case then {@link #getConfidenceScore} may be used + * to retrieve information about entities within the text. Otherwise, returns + * {@link #CLASSIFICATION_NOT_COMPLETE} if classification has not yet returned results, or + * {@link #CLASSIFICATION_NOT_PERFORMED} if classification was not attempted (e.g. because the + * text was too long). + */ + public @ClassificationStatus int getClassificationStatus() { + return mClassificationStatus; + } + + /** + * @hide + */ + public void setConfidenceScores(Map confidences) { + mEntityConfidence.clear(); + mEntityConfidence.putAll(confidences); + mClassificationStatus = CLASSIFICATION_COMPLETE; + } + @Override public String toString() { StringBuilder b = new StringBuilder(128); @@ -451,6 +542,23 @@ public class ClipDescription implements Parcelable { dest.writePersistableBundle(mExtras); dest.writeLong(mTimeStamp); dest.writeBoolean(mIsStyledText); + dest.writeInt(mClassificationStatus); + dest.writeBundle(confidencesToBundle()); + } + + private Bundle confidencesToBundle() { + Bundle bundle = new Bundle(); + int size = mEntityConfidence.size(); + for (int i = 0; i < size; i++) { + bundle.putFloat(mEntityConfidence.keyAt(i), mEntityConfidence.valueAt(i)); + } + return bundle; + } + + private void readBundleToConfidences(Bundle bundle) { + for (String key : bundle.keySet()) { + mEntityConfidence.put(key, bundle.getFloat(key)); + } } ClipDescription(Parcel in) { @@ -459,6 +567,8 @@ public class ClipDescription implements Parcelable { mExtras = in.readPersistableBundle(); mTimeStamp = in.readLong(); mIsStyledText = in.readBoolean(); + mClassificationStatus = in.readInt(); + readBundleToConfidences(in.readBundle()); } public static final @android.annotation.NonNull Parcelable.Creator CREATOR = diff --git a/core/java/android/view/textclassifier/TextClassifier.java b/core/java/android/view/textclassifier/TextClassifier.java index 6d5077a99c62d..ef50045363540 100644 --- a/core/java/android/view/textclassifier/TextClassifier.java +++ b/core/java/android/view/textclassifier/TextClassifier.java @@ -145,7 +145,7 @@ public interface TextClassifier { @StringDef({WIDGET_TYPE_TEXTVIEW, WIDGET_TYPE_EDITTEXT, WIDGET_TYPE_UNSELECTABLE_TEXTVIEW, WIDGET_TYPE_WEBVIEW, WIDGET_TYPE_EDIT_WEBVIEW, WIDGET_TYPE_CUSTOM_TEXTVIEW, WIDGET_TYPE_CUSTOM_EDITTEXT, WIDGET_TYPE_CUSTOM_UNSELECTABLE_TEXTVIEW, - WIDGET_TYPE_NOTIFICATION, WIDGET_TYPE_UNKNOWN}) + WIDGET_TYPE_NOTIFICATION, WIDGET_TYPE_CLIPBOARD, WIDGET_TYPE_UNKNOWN }) @interface WidgetType {} /** The widget involved in the text classification context is a standard @@ -172,6 +172,8 @@ public interface TextClassifier { String WIDGET_TYPE_CUSTOM_UNSELECTABLE_TEXTVIEW = "nosel-customview"; /** The widget involved in the text classification context is a notification */ String WIDGET_TYPE_NOTIFICATION = "notification"; + /** The text classification context is for use with the system clipboard. */ + String WIDGET_TYPE_CLIPBOARD = "clipboard"; /** The widget involved in the text classification context is of an unknown/unspecified type. */ String WIDGET_TYPE_UNKNOWN = "unknown"; diff --git a/services/core/java/com/android/server/clipboard/ClipboardService.java b/services/core/java/com/android/server/clipboard/ClipboardService.java index 5020917f8eb18..a0d95d0eba624 100644 --- a/services/core/java/com/android/server/clipboard/ClipboardService.java +++ b/services/core/java/com/android/server/clipboard/ClipboardService.java @@ -22,6 +22,7 @@ import android.Manifest; import android.annotation.NonNull; import android.annotation.Nullable; import android.annotation.UserIdInt; +import android.annotation.WorkerThread; import android.app.ActivityManagerInternal; import android.app.AppGlobals; import android.app.AppOpsManager; @@ -43,6 +44,8 @@ import android.content.pm.PackageManager; import android.content.pm.UserInfo; import android.net.Uri; import android.os.Binder; +import android.os.Handler; +import android.os.HandlerThread; import android.os.IBinder; import android.os.IUserManager; import android.os.Parcel; @@ -55,9 +58,14 @@ import android.os.UserManager; import android.provider.DeviceConfig; import android.provider.Settings; import android.text.TextUtils; +import android.util.ArrayMap; import android.util.Slog; import android.util.SparseArray; import android.view.autofill.AutofillManagerInternal; +import android.view.textclassifier.TextClassificationContext; +import android.view.textclassifier.TextClassificationManager; +import android.view.textclassifier.TextClassifier; +import android.view.textclassifier.TextLinks; import android.widget.Toast; import com.android.internal.R; @@ -169,6 +177,8 @@ public class ClipboardService extends SystemService { // DeviceConfig properties private static final String PROPERTY_SHOW_ACCESS_NOTIFICATIONS = "show_access_notifications"; private static final boolean DEFAULT_SHOW_ACCESS_NOTIFICATIONS = true; + private static final String PROPERTY_MAX_CLASSIFICATION_LENGTH = "max_classification_length"; + private static final int DEFAULT_MAX_CLASSIFICATION_LENGTH = 400; private final ActivityManagerInternal mAmInternal; private final IUriGrantsManager mUgm; @@ -179,8 +189,10 @@ public class ClipboardService extends SystemService { private final AppOpsManager mAppOps; private final ContentCaptureManagerInternal mContentCaptureInternal; private final AutofillManagerInternal mAutofillInternal; + private final TextClassificationManager mTextClassificationManager; private final IBinder mPermissionOwner; private final HostClipboardMonitor mHostClipboardMonitor; + private final Handler mWorkerHandler; @GuardedBy("mLock") private final SparseArray mClipboards = new SparseArray<>(); @@ -188,6 +200,9 @@ public class ClipboardService extends SystemService { @GuardedBy("mLock") private boolean mShowAccessNotifications = DEFAULT_SHOW_ACCESS_NOTIFICATIONS; + @GuardedBy("mLock") + private int mMaxClassificationLength = DEFAULT_MAX_CLASSIFICATION_LENGTH; + private final Object mLock = new Object(); /** @@ -205,6 +220,8 @@ public class ClipboardService extends SystemService { mAppOps = (AppOpsManager) getContext().getSystemService(Context.APP_OPS_SERVICE); mContentCaptureInternal = LocalServices.getService(ContentCaptureManagerInternal.class); mAutofillInternal = LocalServices.getService(AutofillManagerInternal.class); + mTextClassificationManager = (TextClassificationManager) + getContext().getSystemService(Context.TEXT_CLASSIFICATION_SERVICE); final IBinder permOwner = mUgmInternal.newUriPermissionOwner("clipboard"); mPermissionOwner = permOwner; if (IS_EMULATOR) { @@ -231,6 +248,10 @@ public class ClipboardService extends SystemService { updateConfig(); DeviceConfig.addOnPropertiesChangedListener(DeviceConfig.NAMESPACE_CLIPBOARD, getContext().getMainExecutor(), properties -> updateConfig()); + + HandlerThread workerThread = new HandlerThread(TAG); + workerThread.start(); + mWorkerHandler = workerThread.getThreadHandler(); } @Override @@ -249,6 +270,8 @@ public class ClipboardService extends SystemService { synchronized (mLock) { mShowAccessNotifications = DeviceConfig.getBoolean(DeviceConfig.NAMESPACE_CLIPBOARD, PROPERTY_SHOW_ACCESS_NOTIFICATIONS, DEFAULT_SHOW_ACCESS_NOTIFICATIONS); + mMaxClassificationLength = DeviceConfig.getInt(DeviceConfig.NAMESPACE_CLIPBOARD, + PROPERTY_MAX_CLASSIFICATION_LENGTH, DEFAULT_MAX_CLASSIFICATION_LENGTH); } } @@ -588,6 +611,10 @@ public class ClipboardService extends SystemService { } } + if (clip != null) { + startClassificationLocked(clip); + } + // Update this user final int userId = UserHandle.getUserId(uid); setPrimaryClipInternalLocked(getClipboardLocked(userId), clip, uid, sourcePackage); @@ -686,6 +713,68 @@ public class ClipboardService extends SystemService { } } + @GuardedBy("mLock") + private void startClassificationLocked(@NonNull ClipData clip) { + TextClassifier classifier; + final long ident = Binder.clearCallingIdentity(); + try { + classifier = mTextClassificationManager.createTextClassificationSession( + new TextClassificationContext.Builder( + getContext().getPackageName(), + TextClassifier.WIDGET_TYPE_CLIPBOARD + ).build() + ); + } finally { + Binder.restoreCallingIdentity(ident); + } + + if (clip.getItemCount() == 0) { + clip.getDescription().setClassificationStatus( + ClipDescription.CLASSIFICATION_NOT_PERFORMED); + return; + } + CharSequence text = clip.getItemAt(0).getText(); + if (TextUtils.isEmpty(text) || text.length() > mMaxClassificationLength + || text.length() > classifier.getMaxGenerateLinksTextLength()) { + clip.getDescription().setClassificationStatus( + ClipDescription.CLASSIFICATION_NOT_PERFORMED); + return; + } + + mWorkerHandler.post(() -> doClassification(text, clip, classifier)); + } + + @WorkerThread + private void doClassification( + CharSequence text, ClipData clip, TextClassifier classifier) { + TextLinks.Request request = new TextLinks.Request.Builder(text).build(); + TextLinks links; + try { + links = classifier.generateLinks(request); + } finally { + classifier.destroy(); + } + + // Find the highest confidence for each entity in the text. + ArrayMap confidences = new ArrayMap<>(); + for (TextLinks.TextLink link : links.getLinks()) { + for (int i = 0; i < link.getEntityCount(); i++) { + String entity = link.getEntity(i); + float conf = link.getConfidenceScore(entity); + if (conf > confidences.getOrDefault(entity, 0f)) { + confidences.put(entity, conf); + } + } + } + + synchronized (mLock) { + clip.getDescription().setConfidenceScores(confidences); + if (!links.getLinks().isEmpty()) { + clip.getItemAt(0).setTextLinks(links); + } + } + } + private boolean isDeviceLocked(@UserIdInt int userId) { final long token = Binder.clearCallingIdentity(); try {