diff --git a/core/api/current.txt b/core/api/current.txt index 42a6f0661ec53..63e35f11ecea6 100644 --- a/core/api/current.txt +++ b/core/api/current.txt @@ -9910,6 +9910,7 @@ package android.content { method public String getHtmlText(); method public android.content.Intent getIntent(); method public CharSequence getText(); + method @Nullable public android.view.textclassifier.TextLinks getTextLinks(); method public android.net.Uri getUri(); } @@ -9919,6 +9920,8 @@ package android.content { method public static boolean compareMimeTypes(String, String); method public int describeContents(); method public String[] filterMimeTypes(String); + method public int getClassificationStatus(); + method @FloatRange(from=0.0, to=1.0) public float getConfidenceScore(@NonNull String); method public android.os.PersistableBundle getExtras(); method public CharSequence getLabel(); method public String getMimeType(int); @@ -9928,6 +9931,9 @@ package android.content { method public boolean isStyledText(); method public void setExtras(android.os.PersistableBundle); method public void writeToParcel(android.os.Parcel, int); + field public static final int CLASSIFICATION_COMPLETE = 3; // 0x3 + field public static final int CLASSIFICATION_NOT_COMPLETE = 1; // 0x1 + field public static final int CLASSIFICATION_NOT_PERFORMED = 2; // 0x2 field @NonNull public static final android.os.Parcelable.Creator CREATOR; field public static final String MIMETYPE_TEXT_HTML = "text/html"; field public static final String MIMETYPE_TEXT_INTENT = "text/vnd.android.intent"; @@ -51917,6 +51923,7 @@ package android.view.textclassifier { field public static final String TYPE_PHONE = "phone"; field public static final String TYPE_UNKNOWN = ""; field public static final String TYPE_URL = "url"; + field public static final String WIDGET_TYPE_CLIPBOARD = "clipboard"; field public static final String WIDGET_TYPE_CUSTOM_EDITTEXT = "customedit"; field public static final String WIDGET_TYPE_CUSTOM_TEXTVIEW = "customview"; field public static final String WIDGET_TYPE_CUSTOM_UNSELECTABLE_TEXTVIEW = "nosel-customview"; diff --git a/core/java/android/content/ClipData.java b/core/java/android/content/ClipData.java index f3ecbf6c08f34..6ab1975f97176 100644 --- a/core/java/android/content/ClipData.java +++ b/core/java/android/content/ClipData.java @@ -21,6 +21,7 @@ import static android.content.ContentResolver.SCHEME_ANDROID_RESOURCE; import static android.content.ContentResolver.SCHEME_CONTENT; import static android.content.ContentResolver.SCHEME_FILE; +import android.annotation.Nullable; import android.compat.annotation.UnsupportedAppUsage; import android.content.pm.ActivityInfo; import android.content.res.AssetFileDescriptor; @@ -38,6 +39,7 @@ import android.text.TextUtils; import android.text.style.URLSpan; import android.util.Log; import android.util.proto.ProtoOutputStream; +import android.view.textclassifier.TextLinks; import com.android.internal.util.ArrayUtils; @@ -204,6 +206,7 @@ public class ClipData implements Parcelable { Uri mUri; // Additional activity info resolved by the system ActivityInfo mActivityInfo; + private TextLinks mTextLinks; /** @hide */ public Item(Item other) { @@ -331,6 +334,29 @@ public class ClipData implements Parcelable { mActivityInfo = info; } + /** + * Returns the results of text classification run on the raw text contained in this item, + * if it was performed, and if any entities were found in the text. Classification is + * generally only performed on the first item in clip data, and only if the text is below a + * certain length. + * + *

Returns {@code null} if classification was not performed, or if no entities were + * found in the text. + * + * @see ClipDescription#getConfidenceScore(String) + */ + @Nullable + public TextLinks getTextLinks() { + return mTextLinks; + } + + /** + * @hide + */ + public void setTextLinks(TextLinks textLinks) { + mTextLinks = textLinks; + } + /** * Turn this item into text, regardless of the type of data it * actually contains. @@ -1183,6 +1209,7 @@ public class ClipData implements Parcelable { dest.writeTypedObject(item.mIntent, flags); dest.writeTypedObject(item.mUri, flags); dest.writeTypedObject(item.mActivityInfo, flags); + dest.writeTypedObject(item.mTextLinks, flags); } } @@ -1201,8 +1228,10 @@ public class ClipData implements Parcelable { Intent intent = in.readTypedObject(Intent.CREATOR); Uri uri = in.readTypedObject(Uri.CREATOR); ActivityInfo info = in.readTypedObject(ActivityInfo.CREATOR); + TextLinks textLinks = in.readTypedObject(TextLinks.CREATOR); Item item = new Item(text, htmlText, intent, uri); item.setActivityInfo(info); + item.setTextLinks(textLinks); mItems.add(item); } } diff --git a/core/java/android/content/ClipDescription.java b/core/java/android/content/ClipDescription.java index d48f83223d3e2..f49362e9300e8 100644 --- a/core/java/android/content/ClipDescription.java +++ b/core/java/android/content/ClipDescription.java @@ -16,16 +16,25 @@ package android.content; +import android.annotation.FloatRange; +import android.annotation.IntDef; import android.annotation.NonNull; +import android.os.Bundle; import android.os.Parcel; import android.os.Parcelable; import android.os.PersistableBundle; import android.text.TextUtils; +import android.util.ArrayMap; import android.util.TimeUtils; import android.util.proto.ProtoOutputStream; +import android.view.textclassifier.TextClassifier; +import android.view.textclassifier.TextLinks; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; import java.util.ArrayList; import java.util.Arrays; +import java.util.Map; /** * Meta-data describing the contents of a {@link ClipData}. Provides enough @@ -115,12 +124,39 @@ public class ClipDescription implements Parcelable { */ public static final String EXTRA_ACTIVITY_OPTIONS = "android.intent.extra.ACTIVITY_OPTIONS"; + /** @hide */ + @Retention(RetentionPolicy.SOURCE) + @IntDef(value = + { CLASSIFICATION_NOT_COMPLETE, CLASSIFICATION_NOT_PERFORMED, CLASSIFICATION_COMPLETE}) + @interface ClassificationStatus {} + + /** + * Value returned by {@link #getConfidenceScore(String)} if text classification has not been + * completed on the associated clip. This will be always be the case if the clip has not been + * copied to clipboard, or if there is no associated clip. + */ + public static final int CLASSIFICATION_NOT_COMPLETE = 1; + + /** + * Value returned by {@link #getConfidenceScore(String)} if text classification was not and will + * not be performed on the associated clip. This may be the case if the clip does not contain + * text in its first item, or if the text is too long. + */ + public static final int CLASSIFICATION_NOT_PERFORMED = 2; + + /** + * Value returned by {@link #getConfidenceScore(String)} if text classification has been + * completed. + */ + public static final int CLASSIFICATION_COMPLETE = 3; final CharSequence mLabel; private final ArrayList mMimeTypes; private PersistableBundle mExtras; private long mTimeStamp; private boolean mIsStyledText; + private final ArrayMap mEntityConfidence = new ArrayMap<>(); + private int mClassificationStatus = CLASSIFICATION_NOT_COMPLETE; /** * Create a new clip. @@ -346,6 +382,61 @@ public class ClipDescription implements Parcelable { mIsStyledText = isStyledText; } + /** + * Sets the current status of text classification for the associated clip. + * + * @hide + */ + public void setClassificationStatus(@ClassificationStatus int status) { + mClassificationStatus = status; + } + + /** + * Returns a score indicating confidence that an instance of the given entity is present in the + * first item of the clip data, if that item is plain text and text classification has been + * performed. The value ranges from 0 (low confidence) to 1 (high confidence). 0 indicates that + * the entity was not found in the classified text. + * + *

Entities should be as defined in the {@link TextClassifier} class, such as + * {@link TextClassifier#TYPE_ADDRESS}, {@link TextClassifier#TYPE_URL}, or + * {@link TextClassifier#TYPE_EMAIL}. + * + *

If the result is positive for any entity, the full classification result as a + * {@link TextLinks} object may be obtained using the {@link ClipData.Item#getTextLinks()} + * method. + * + * @throws IllegalStateException if {@link #getClassificationStatus()} is not + * {@link #CLASSIFICATION_COMPLETE} + */ + @FloatRange(from = 0.0, to = 1.0) + public float getConfidenceScore(@NonNull @TextClassifier.EntityType String entity) { + if (mClassificationStatus != CLASSIFICATION_COMPLETE) { + throw new IllegalStateException("Classification not complete"); + } + return mEntityConfidence.getOrDefault(entity, 0f); + } + + /** + * Returns {@link #CLASSIFICATION_COMPLETE} if text classification has been performed on the + * associated {@link ClipData}. If this is the case then {@link #getConfidenceScore} may be used + * to retrieve information about entities within the text. Otherwise, returns + * {@link #CLASSIFICATION_NOT_COMPLETE} if classification has not yet returned results, or + * {@link #CLASSIFICATION_NOT_PERFORMED} if classification was not attempted (e.g. because the + * text was too long). + */ + public @ClassificationStatus int getClassificationStatus() { + return mClassificationStatus; + } + + /** + * @hide + */ + public void setConfidenceScores(Map confidences) { + mEntityConfidence.clear(); + mEntityConfidence.putAll(confidences); + mClassificationStatus = CLASSIFICATION_COMPLETE; + } + @Override public String toString() { StringBuilder b = new StringBuilder(128); @@ -451,6 +542,23 @@ public class ClipDescription implements Parcelable { dest.writePersistableBundle(mExtras); dest.writeLong(mTimeStamp); dest.writeBoolean(mIsStyledText); + dest.writeInt(mClassificationStatus); + dest.writeBundle(confidencesToBundle()); + } + + private Bundle confidencesToBundle() { + Bundle bundle = new Bundle(); + int size = mEntityConfidence.size(); + for (int i = 0; i < size; i++) { + bundle.putFloat(mEntityConfidence.keyAt(i), mEntityConfidence.valueAt(i)); + } + return bundle; + } + + private void readBundleToConfidences(Bundle bundle) { + for (String key : bundle.keySet()) { + mEntityConfidence.put(key, bundle.getFloat(key)); + } } ClipDescription(Parcel in) { @@ -459,6 +567,8 @@ public class ClipDescription implements Parcelable { mExtras = in.readPersistableBundle(); mTimeStamp = in.readLong(); mIsStyledText = in.readBoolean(); + mClassificationStatus = in.readInt(); + readBundleToConfidences(in.readBundle()); } public static final @android.annotation.NonNull Parcelable.Creator CREATOR = diff --git a/core/java/android/view/textclassifier/TextClassifier.java b/core/java/android/view/textclassifier/TextClassifier.java index 6d5077a99c62d..ef50045363540 100644 --- a/core/java/android/view/textclassifier/TextClassifier.java +++ b/core/java/android/view/textclassifier/TextClassifier.java @@ -145,7 +145,7 @@ public interface TextClassifier { @StringDef({WIDGET_TYPE_TEXTVIEW, WIDGET_TYPE_EDITTEXT, WIDGET_TYPE_UNSELECTABLE_TEXTVIEW, WIDGET_TYPE_WEBVIEW, WIDGET_TYPE_EDIT_WEBVIEW, WIDGET_TYPE_CUSTOM_TEXTVIEW, WIDGET_TYPE_CUSTOM_EDITTEXT, WIDGET_TYPE_CUSTOM_UNSELECTABLE_TEXTVIEW, - WIDGET_TYPE_NOTIFICATION, WIDGET_TYPE_UNKNOWN}) + WIDGET_TYPE_NOTIFICATION, WIDGET_TYPE_CLIPBOARD, WIDGET_TYPE_UNKNOWN }) @interface WidgetType {} /** The widget involved in the text classification context is a standard @@ -172,6 +172,8 @@ public interface TextClassifier { String WIDGET_TYPE_CUSTOM_UNSELECTABLE_TEXTVIEW = "nosel-customview"; /** The widget involved in the text classification context is a notification */ String WIDGET_TYPE_NOTIFICATION = "notification"; + /** The text classification context is for use with the system clipboard. */ + String WIDGET_TYPE_CLIPBOARD = "clipboard"; /** The widget involved in the text classification context is of an unknown/unspecified type. */ String WIDGET_TYPE_UNKNOWN = "unknown"; diff --git a/services/core/java/com/android/server/clipboard/ClipboardService.java b/services/core/java/com/android/server/clipboard/ClipboardService.java index b831e7ca0903b..5bf15dc70ff94 100644 --- a/services/core/java/com/android/server/clipboard/ClipboardService.java +++ b/services/core/java/com/android/server/clipboard/ClipboardService.java @@ -22,6 +22,7 @@ import android.Manifest; import android.annotation.NonNull; import android.annotation.Nullable; import android.annotation.UserIdInt; +import android.annotation.WorkerThread; import android.app.ActivityManagerInternal; import android.app.AppGlobals; import android.app.AppOpsManager; @@ -43,6 +44,8 @@ import android.content.pm.PackageManager; import android.content.pm.UserInfo; import android.net.Uri; import android.os.Binder; +import android.os.Handler; +import android.os.HandlerThread; import android.os.IBinder; import android.os.IUserManager; import android.os.Parcel; @@ -55,10 +58,15 @@ import android.os.UserManager; import android.provider.DeviceConfig; import android.provider.Settings; import android.text.TextUtils; +import android.util.ArrayMap; import android.util.Slog; import android.util.SparseArray; import android.util.SparseBooleanArray; import android.view.autofill.AutofillManagerInternal; +import android.view.textclassifier.TextClassificationContext; +import android.view.textclassifier.TextClassificationManager; +import android.view.textclassifier.TextClassifier; +import android.view.textclassifier.TextLinks; import android.widget.Toast; import com.android.internal.R; @@ -170,6 +178,8 @@ public class ClipboardService extends SystemService { // DeviceConfig properties private static final String PROPERTY_SHOW_ACCESS_NOTIFICATIONS = "show_access_notifications"; private static final boolean DEFAULT_SHOW_ACCESS_NOTIFICATIONS = true; + private static final String PROPERTY_MAX_CLASSIFICATION_LENGTH = "max_classification_length"; + private static final int DEFAULT_MAX_CLASSIFICATION_LENGTH = 400; private final ActivityManagerInternal mAmInternal; private final IUriGrantsManager mUgm; @@ -180,8 +190,10 @@ public class ClipboardService extends SystemService { private final AppOpsManager mAppOps; private final ContentCaptureManagerInternal mContentCaptureInternal; private final AutofillManagerInternal mAutofillInternal; + private final TextClassificationManager mTextClassificationManager; private final IBinder mPermissionOwner; private final HostClipboardMonitor mHostClipboardMonitor; + private final Handler mWorkerHandler; @GuardedBy("mLock") private final SparseArray mClipboards = new SparseArray<>(); @@ -189,6 +201,9 @@ public class ClipboardService extends SystemService { @GuardedBy("mLock") private boolean mShowAccessNotifications = DEFAULT_SHOW_ACCESS_NOTIFICATIONS; + @GuardedBy("mLock") + private int mMaxClassificationLength = DEFAULT_MAX_CLASSIFICATION_LENGTH; + private final Object mLock = new Object(); /** @@ -206,6 +221,8 @@ public class ClipboardService extends SystemService { mAppOps = (AppOpsManager) getContext().getSystemService(Context.APP_OPS_SERVICE); mContentCaptureInternal = LocalServices.getService(ContentCaptureManagerInternal.class); mAutofillInternal = LocalServices.getService(AutofillManagerInternal.class); + mTextClassificationManager = (TextClassificationManager) + getContext().getSystemService(Context.TEXT_CLASSIFICATION_SERVICE); final IBinder permOwner = mUgmInternal.newUriPermissionOwner("clipboard"); mPermissionOwner = permOwner; if (IS_EMULATOR) { @@ -232,6 +249,10 @@ public class ClipboardService extends SystemService { updateConfig(); DeviceConfig.addOnPropertiesChangedListener(DeviceConfig.NAMESPACE_CLIPBOARD, getContext().getMainExecutor(), properties -> updateConfig()); + + HandlerThread workerThread = new HandlerThread(TAG); + workerThread.start(); + mWorkerHandler = workerThread.getThreadHandler(); } @Override @@ -250,6 +271,8 @@ public class ClipboardService extends SystemService { synchronized (mLock) { mShowAccessNotifications = DeviceConfig.getBoolean(DeviceConfig.NAMESPACE_CLIPBOARD, PROPERTY_SHOW_ACCESS_NOTIFICATIONS, DEFAULT_SHOW_ACCESS_NOTIFICATIONS); + mMaxClassificationLength = DeviceConfig.getInt(DeviceConfig.NAMESPACE_CLIPBOARD, + PROPERTY_MAX_CLASSIFICATION_LENGTH, DEFAULT_MAX_CLASSIFICATION_LENGTH); } } @@ -592,6 +615,10 @@ public class ClipboardService extends SystemService { } } + if (clip != null) { + startClassificationLocked(clip); + } + // Update this user final int userId = UserHandle.getUserId(uid); setPrimaryClipInternalLocked(getClipboardLocked(userId), clip, uid, sourcePackage); @@ -691,6 +718,68 @@ public class ClipboardService extends SystemService { } } + @GuardedBy("mLock") + private void startClassificationLocked(@NonNull ClipData clip) { + TextClassifier classifier; + final long ident = Binder.clearCallingIdentity(); + try { + classifier = mTextClassificationManager.createTextClassificationSession( + new TextClassificationContext.Builder( + getContext().getPackageName(), + TextClassifier.WIDGET_TYPE_CLIPBOARD + ).build() + ); + } finally { + Binder.restoreCallingIdentity(ident); + } + + if (clip.getItemCount() == 0) { + clip.getDescription().setClassificationStatus( + ClipDescription.CLASSIFICATION_NOT_PERFORMED); + return; + } + CharSequence text = clip.getItemAt(0).getText(); + if (TextUtils.isEmpty(text) || text.length() > mMaxClassificationLength + || text.length() > classifier.getMaxGenerateLinksTextLength()) { + clip.getDescription().setClassificationStatus( + ClipDescription.CLASSIFICATION_NOT_PERFORMED); + return; + } + + mWorkerHandler.post(() -> doClassification(text, clip, classifier)); + } + + @WorkerThread + private void doClassification( + CharSequence text, ClipData clip, TextClassifier classifier) { + TextLinks.Request request = new TextLinks.Request.Builder(text).build(); + TextLinks links; + try { + links = classifier.generateLinks(request); + } finally { + classifier.destroy(); + } + + // Find the highest confidence for each entity in the text. + ArrayMap confidences = new ArrayMap<>(); + for (TextLinks.TextLink link : links.getLinks()) { + for (int i = 0; i < link.getEntityCount(); i++) { + String entity = link.getEntity(i); + float conf = link.getConfidenceScore(entity); + if (conf > confidences.getOrDefault(entity, 0f)) { + confidences.put(entity, conf); + } + } + } + + synchronized (mLock) { + clip.getDescription().setConfidenceScores(confidences); + if (!links.getLinks().isEmpty()) { + clip.getItemAt(0).setTextLinks(links); + } + } + } + private boolean isDeviceLocked(@UserIdInt int userId) { final long token = Binder.clearCallingIdentity(); try {