Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge "Add text classification results to clip description and clip data." into sc-dev

Browse Source
This commit is contained in:
Oli Lan
2021-03-15 17:26:35 +00:00
committed by Android (Google) Code Review
parent 54ec981b86 7e568c9177
commit fe4a8f58fb
5 changed files with 238 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
core/api/current.txt
View File

@@ -9910,6 +9910,7 @@ package android.content {
method public String getHtmlText();
method public android.content.Intent getIntent();
method public CharSequence getText();
method @Nullable public android.view.textclassifier.TextLinks getTextLinks();
method public android.net.Uri getUri();
}
@@ -9919,6 +9920,8 @@ package android.content {
method public static boolean compareMimeTypes(String, String);
method public int describeContents();
method public String[] filterMimeTypes(String);
method public int getClassificationStatus();
method @FloatRange(from=0.0, to=1.0) public float getConfidenceScore(@NonNull String);
method public android.os.PersistableBundle getExtras();
method public CharSequence getLabel();
method public String getMimeType(int);
@@ -9928,6 +9931,9 @@ package android.content {
method public boolean isStyledText();
method public void setExtras(android.os.PersistableBundle);
method public void writeToParcel(android.os.Parcel, int);
field public static final int CLASSIFICATION_COMPLETE = 3; // 0x3
field public static final int CLASSIFICATION_NOT_COMPLETE = 1; // 0x1
field public static final int CLASSIFICATION_NOT_PERFORMED = 2; // 0x2
field @NonNull public static final android.os.Parcelable.Creator<android.content.ClipDescription> CREATOR;
field public static final String MIMETYPE_TEXT_HTML = "text/html";
field public static final String MIMETYPE_TEXT_INTENT = "text/vnd.android.intent";
@@ -51917,6 +51923,7 @@ package android.view.textclassifier {
field public static final String TYPE_PHONE = "phone";
field public static final String TYPE_UNKNOWN = "";
field public static final String TYPE_URL = "url";
field public static final String WIDGET_TYPE_CLIPBOARD = "clipboard";
field public static final String WIDGET_TYPE_CUSTOM_EDITTEXT = "customedit";
field public static final String WIDGET_TYPE_CUSTOM_TEXTVIEW = "customview";
field public static final String WIDGET_TYPE_CUSTOM_UNSELECTABLE_TEXTVIEW = "nosel-customview";

29
core/java/android/content/ClipData.java
View File

@@ -21,6 +21,7 @@ import static android.content.ContentResolver.SCHEME_ANDROID_RESOURCE;
import static android.content.ContentResolver.SCHEME_CONTENT;
import static android.content.ContentResolver.SCHEME_FILE;
import android.annotation.Nullable;
import android.compat.annotation.UnsupportedAppUsage;
import android.content.pm.ActivityInfo;
import android.content.res.AssetFileDescriptor;
@@ -38,6 +39,7 @@ import android.text.TextUtils;
import android.text.style.URLSpan;
import android.util.Log;
import android.util.proto.ProtoOutputStream;
import android.view.textclassifier.TextLinks;
import com.android.internal.util.ArrayUtils;
@@ -204,6 +206,7 @@ public class ClipData implements Parcelable {
Uri mUri;
// Additional activity info resolved by the system
ActivityInfo mActivityInfo;
private TextLinks mTextLinks;
/** @hide */
public Item(Item other) {
@@ -331,6 +334,29 @@ public class ClipData implements Parcelable {
mActivityInfo = info;
}
/**
* Returns the results of text classification run on the raw text contained in this item,
* if it was performed, and if any entities were found in the text. Classification is
* generally only performed on the first item in clip data, and only if the text is below a
* certain length.
*
* <p>Returns {@code null} if classification was not performed, or if no entities were
* found in the text.
*
* @see ClipDescription#getConfidenceScore(String)
*/
@Nullable
public TextLinks getTextLinks() {
return mTextLinks;
}
/**
* @hide
*/
public void setTextLinks(TextLinks textLinks) {
mTextLinks = textLinks;
}
/**
* Turn this item into text, regardless of the type of data it
* actually contains.
@@ -1183,6 +1209,7 @@ public class ClipData implements Parcelable {
dest.writeTypedObject(item.mIntent, flags);
dest.writeTypedObject(item.mUri, flags);
dest.writeTypedObject(item.mActivityInfo, flags);
dest.writeTypedObject(item.mTextLinks, flags);
}
}
@@ -1201,8 +1228,10 @@ public class ClipData implements Parcelable {
Intent intent = in.readTypedObject(Intent.CREATOR);
Uri uri = in.readTypedObject(Uri.CREATOR);
ActivityInfo info = in.readTypedObject(ActivityInfo.CREATOR);
TextLinks textLinks = in.readTypedObject(TextLinks.CREATOR);
Item item = new Item(text, htmlText, intent, uri);
item.setActivityInfo(info);
item.setTextLinks(textLinks);
mItems.add(item);
}
}

110
core/java/android/content/ClipDescription.java
View File

@@ -16,16 +16,25 @@
package android.content;
import android.annotation.FloatRange;
import android.annotation.IntDef;
import android.annotation.NonNull;
import android.os.Bundle;
import android.os.Parcel;
import android.os.Parcelable;
import android.os.PersistableBundle;
import android.text.TextUtils;
import android.util.ArrayMap;
import android.util.TimeUtils;
import android.util.proto.ProtoOutputStream;
import android.view.textclassifier.TextClassifier;
import android.view.textclassifier.TextLinks;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Map;
/**
* Meta-data describing the contents of a {@link ClipData}. Provides enough
@@ -115,12 +124,39 @@ public class ClipDescription implements Parcelable {
*/
public static final String EXTRA_ACTIVITY_OPTIONS = "android.intent.extra.ACTIVITY_OPTIONS";
/** @hide */
@Retention(RetentionPolicy.SOURCE)
@IntDef(value =
{ CLASSIFICATION_NOT_COMPLETE, CLASSIFICATION_NOT_PERFORMED, CLASSIFICATION_COMPLETE})
@interface ClassificationStatus {}
/**
* Value returned by {@link #getConfidenceScore(String)} if text classification has not been
* completed on the associated clip. This will be always be the case if the clip has not been
* copied to clipboard, or if there is no associated clip.
*/
public static final int CLASSIFICATION_NOT_COMPLETE = 1;
/**
* Value returned by {@link #getConfidenceScore(String)} if text classification was not and will
* not be performed on the associated clip. This may be the case if the clip does not contain
* text in its first item, or if the text is too long.
*/
public static final int CLASSIFICATION_NOT_PERFORMED = 2;
/**
* Value returned by {@link #getConfidenceScore(String)} if text classification has been
* completed.
*/
public static final int CLASSIFICATION_COMPLETE = 3;
final CharSequence mLabel;
private final ArrayList<String> mMimeTypes;
private PersistableBundle mExtras;
private long mTimeStamp;
private boolean mIsStyledText;
private final ArrayMap<String, Float> mEntityConfidence = new ArrayMap<>();
private int mClassificationStatus = CLASSIFICATION_NOT_COMPLETE;
/**
* Create a new clip.
@@ -346,6 +382,61 @@ public class ClipDescription implements Parcelable {
mIsStyledText = isStyledText;
}
/**
* Sets the current status of text classification for the associated clip.
*
* @hide
*/
public void setClassificationStatus(@ClassificationStatus int status) {
mClassificationStatus = status;
}
/**
* Returns a score indicating confidence that an instance of the given entity is present in the
* first item of the clip data, if that item is plain text and text classification has been
* performed. The value ranges from 0 (low confidence) to 1 (high confidence). 0 indicates that
* the entity was not found in the classified text.
*
* <p>Entities should be as defined in the {@link TextClassifier} class, such as
* {@link TextClassifier#TYPE_ADDRESS}, {@link TextClassifier#TYPE_URL}, or
* {@link TextClassifier#TYPE_EMAIL}.
*
* <p>If the result is positive for any entity, the full classification result as a
* {@link TextLinks} object may be obtained using the {@link ClipData.Item#getTextLinks()}
* method.
*
* @throws IllegalStateException if {@link #getClassificationStatus()} is not
* {@link #CLASSIFICATION_COMPLETE}
*/
@FloatRange(from = 0.0, to = 1.0)
public float getConfidenceScore(@NonNull @TextClassifier.EntityType String entity) {
if (mClassificationStatus != CLASSIFICATION_COMPLETE) {
throw new IllegalStateException("Classification not complete");
}
return mEntityConfidence.getOrDefault(entity, 0f);
}
/**
* Returns {@link #CLASSIFICATION_COMPLETE} if text classification has been performed on the
* associated {@link ClipData}. If this is the case then {@link #getConfidenceScore} may be used
* to retrieve information about entities within the text. Otherwise, returns
* {@link #CLASSIFICATION_NOT_COMPLETE} if classification has not yet returned results, or
* {@link #CLASSIFICATION_NOT_PERFORMED} if classification was not attempted (e.g. because the
* text was too long).
*/
public @ClassificationStatus int getClassificationStatus() {
return mClassificationStatus;
}
/**
* @hide
*/
public void setConfidenceScores(Map<String, Float> confidences) {
mEntityConfidence.clear();
mEntityConfidence.putAll(confidences);
mClassificationStatus = CLASSIFICATION_COMPLETE;
}
@Override
public String toString() {
StringBuilder b = new StringBuilder(128);
@@ -451,6 +542,23 @@ public class ClipDescription implements Parcelable {
dest.writePersistableBundle(mExtras);
dest.writeLong(mTimeStamp);
dest.writeBoolean(mIsStyledText);
dest.writeInt(mClassificationStatus);
dest.writeBundle(confidencesToBundle());
}
private Bundle confidencesToBundle() {
Bundle bundle = new Bundle();
int size = mEntityConfidence.size();
for (int i = 0; i < size; i++) {
bundle.putFloat(mEntityConfidence.keyAt(i), mEntityConfidence.valueAt(i));
}
return bundle;
}
private void readBundleToConfidences(Bundle bundle) {
for (String key : bundle.keySet()) {
mEntityConfidence.put(key, bundle.getFloat(key));
}
}
ClipDescription(Parcel in) {
@@ -459,6 +567,8 @@ public class ClipDescription implements Parcelable {
mExtras = in.readPersistableBundle();
mTimeStamp = in.readLong();
mIsStyledText = in.readBoolean();
mClassificationStatus = in.readInt();
readBundleToConfidences(in.readBundle());
}
public static final @android.annotation.NonNull Parcelable.Creator<ClipDescription> CREATOR =

4
core/java/android/view/textclassifier/TextClassifier.java
View File

@@ -145,7 +145,7 @@ public interface TextClassifier {
@StringDef({WIDGET_TYPE_TEXTVIEW, WIDGET_TYPE_EDITTEXT, WIDGET_TYPE_UNSELECTABLE_TEXTVIEW,
WIDGET_TYPE_WEBVIEW, WIDGET_TYPE_EDIT_WEBVIEW, WIDGET_TYPE_CUSTOM_TEXTVIEW,
WIDGET_TYPE_CUSTOM_EDITTEXT, WIDGET_TYPE_CUSTOM_UNSELECTABLE_TEXTVIEW,
WIDGET_TYPE_NOTIFICATION, WIDGET_TYPE_UNKNOWN})
WIDGET_TYPE_NOTIFICATION, WIDGET_TYPE_CLIPBOARD, WIDGET_TYPE_UNKNOWN })
@interface WidgetType {}
/** The widget involved in the text classification context is a standard
@@ -172,6 +172,8 @@ public interface TextClassifier {
String WIDGET_TYPE_CUSTOM_UNSELECTABLE_TEXTVIEW = "nosel-customview";
/** The widget involved in the text classification context is a notification */
String WIDGET_TYPE_NOTIFICATION = "notification";
/** The text classification context is for use with the system clipboard. */
String WIDGET_TYPE_CLIPBOARD = "clipboard";
/** The widget involved in the text classification context is of an unknown/unspecified type. */
String WIDGET_TYPE_UNKNOWN = "unknown";

89
services/core/java/com/android/server/clipboard/ClipboardService.java
View File

@@ -22,6 +22,7 @@ import android.Manifest;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.annotation.UserIdInt;
import android.annotation.WorkerThread;
import android.app.ActivityManagerInternal;
import android.app.AppGlobals;
import android.app.AppOpsManager;
@@ -43,6 +44,8 @@ import android.content.pm.PackageManager;
import android.content.pm.UserInfo;
import android.net.Uri;
import android.os.Binder;
import android.os.Handler;
import android.os.HandlerThread;
import android.os.IBinder;
import android.os.IUserManager;
import android.os.Parcel;
@@ -55,10 +58,15 @@ import android.os.UserManager;
import android.provider.DeviceConfig;
import android.provider.Settings;
import android.text.TextUtils;
import android.util.ArrayMap;
import android.util.Slog;
import android.util.SparseArray;
import android.util.SparseBooleanArray;
import android.view.autofill.AutofillManagerInternal;
import android.view.textclassifier.TextClassificationContext;
import android.view.textclassifier.TextClassificationManager;
import android.view.textclassifier.TextClassifier;
import android.view.textclassifier.TextLinks;
import android.widget.Toast;
import com.android.internal.R;
@@ -170,6 +178,8 @@ public class ClipboardService extends SystemService {
// DeviceConfig properties
private static final String PROPERTY_SHOW_ACCESS_NOTIFICATIONS = "show_access_notifications";
private static final boolean DEFAULT_SHOW_ACCESS_NOTIFICATIONS = true;
private static final String PROPERTY_MAX_CLASSIFICATION_LENGTH = "max_classification_length";
private static final int DEFAULT_MAX_CLASSIFICATION_LENGTH = 400;
private final ActivityManagerInternal mAmInternal;
private final IUriGrantsManager mUgm;
@@ -180,8 +190,10 @@ public class ClipboardService extends SystemService {
private final AppOpsManager mAppOps;
private final ContentCaptureManagerInternal mContentCaptureInternal;
private final AutofillManagerInternal mAutofillInternal;
private final TextClassificationManager mTextClassificationManager;
private final IBinder mPermissionOwner;
private final HostClipboardMonitor mHostClipboardMonitor;
private final Handler mWorkerHandler;
@GuardedBy("mLock")
private final SparseArray<PerUserClipboard> mClipboards = new SparseArray<>();
@@ -189,6 +201,9 @@ public class ClipboardService extends SystemService {
@GuardedBy("mLock")
private boolean mShowAccessNotifications = DEFAULT_SHOW_ACCESS_NOTIFICATIONS;
@GuardedBy("mLock")
private int mMaxClassificationLength = DEFAULT_MAX_CLASSIFICATION_LENGTH;
private final Object mLock = new Object();
/**
@@ -206,6 +221,8 @@ public class ClipboardService extends SystemService {
mAppOps = (AppOpsManager) getContext().getSystemService(Context.APP_OPS_SERVICE);
mContentCaptureInternal = LocalServices.getService(ContentCaptureManagerInternal.class);
mAutofillInternal = LocalServices.getService(AutofillManagerInternal.class);
mTextClassificationManager = (TextClassificationManager)
getContext().getSystemService(Context.TEXT_CLASSIFICATION_SERVICE);
final IBinder permOwner = mUgmInternal.newUriPermissionOwner("clipboard");
mPermissionOwner = permOwner;
if (IS_EMULATOR) {
@@ -232,6 +249,10 @@ public class ClipboardService extends SystemService {
updateConfig();
DeviceConfig.addOnPropertiesChangedListener(DeviceConfig.NAMESPACE_CLIPBOARD,
getContext().getMainExecutor(), properties -> updateConfig());
HandlerThread workerThread = new HandlerThread(TAG);
workerThread.start();
mWorkerHandler = workerThread.getThreadHandler();
}
@Override
@@ -250,6 +271,8 @@ public class ClipboardService extends SystemService {
synchronized (mLock) {
mShowAccessNotifications = DeviceConfig.getBoolean(DeviceConfig.NAMESPACE_CLIPBOARD,
PROPERTY_SHOW_ACCESS_NOTIFICATIONS, DEFAULT_SHOW_ACCESS_NOTIFICATIONS);
mMaxClassificationLength = DeviceConfig.getInt(DeviceConfig.NAMESPACE_CLIPBOARD,
PROPERTY_MAX_CLASSIFICATION_LENGTH, DEFAULT_MAX_CLASSIFICATION_LENGTH);
}
}
@@ -592,6 +615,10 @@ public class ClipboardService extends SystemService {
}
}
if (clip != null) {
startClassificationLocked(clip);
}
// Update this user
final int userId = UserHandle.getUserId(uid);
setPrimaryClipInternalLocked(getClipboardLocked(userId), clip, uid, sourcePackage);
@@ -691,6 +718,68 @@ public class ClipboardService extends SystemService {
}
}
@GuardedBy("mLock")
private void startClassificationLocked(@NonNull ClipData clip) {
TextClassifier classifier;
final long ident = Binder.clearCallingIdentity();
try {
classifier = mTextClassificationManager.createTextClassificationSession(
new TextClassificationContext.Builder(
getContext().getPackageName(),
TextClassifier.WIDGET_TYPE_CLIPBOARD
).build()
);
} finally {
Binder.restoreCallingIdentity(ident);
}
if (clip.getItemCount() == 0) {
clip.getDescription().setClassificationStatus(
ClipDescription.CLASSIFICATION_NOT_PERFORMED);
return;
}
CharSequence text = clip.getItemAt(0).getText();
if (TextUtils.isEmpty(text) || text.length() > mMaxClassificationLength
|| text.length() > classifier.getMaxGenerateLinksTextLength()) {
clip.getDescription().setClassificationStatus(
ClipDescription.CLASSIFICATION_NOT_PERFORMED);
return;
}
mWorkerHandler.post(() -> doClassification(text, clip, classifier));
}
@WorkerThread
private void doClassification(
CharSequence text, ClipData clip, TextClassifier classifier) {
TextLinks.Request request = new TextLinks.Request.Builder(text).build();
TextLinks links;
try {
links = classifier.generateLinks(request);
} finally {
classifier.destroy();
}
// Find the highest confidence for each entity in the text.
ArrayMap<String, Float> confidences = new ArrayMap<>();
for (TextLinks.TextLink link : links.getLinks()) {
for (int i = 0; i < link.getEntityCount(); i++) {
String entity = link.getEntity(i);
float conf = link.getConfidenceScore(entity);
if (conf > confidences.getOrDefault(entity, 0f)) {
confidences.put(entity, conf);
}
}
}
synchronized (mLock) {
clip.getDescription().setConfidenceScores(confidences);
if (!links.getLinks().isEmpty()) {
clip.getItemAt(0).setTextLinks(links);
}
}
}
private boolean isDeviceLocked(@UserIdInt int userId) {
final long token = Binder.clearCallingIdentity();
try {