From 8710ea13c13aa22f82cdcb0fc461ed4cd8b2aa63 Mon Sep 17 00:00:00 2001 From: Abodunrinwa Toki Date: Tue, 24 Jan 2017 10:34:13 -0800 Subject: [PATCH] start/invalidate selection actionMode asynchronously Text selection is now started/updated using AsyncTasks that first get a result from the TextClassifier before actually starting/invalidating the actionMode on the UI thread with the result. Bug: 34778899 Test: cts-tradefed run cts-dev -m CtsWidgetTestCases -t android.widget.cts.TextViewTest#testSmartSelection Change-Id: Ie7eda04bb64335744d88b8874363d46af4f94742 --- core/java/android/widget/Editor.java | 129 ++++---- .../widget/SelectionActionModeHelper.java | 288 ++++++++++++++++++ core/java/android/widget/TextView.java | 9 +- 3 files changed, 356 insertions(+), 70 deletions(-) create mode 100644 core/java/android/widget/SelectionActionModeHelper.java diff --git a/core/java/android/widget/Editor.java b/core/java/android/widget/Editor.java index a2cb491bbb7ad..ee31b829580c7 100644 --- a/core/java/android/widget/Editor.java +++ b/core/java/android/widget/Editor.java @@ -107,7 +107,6 @@ import android.view.inputmethod.ExtractedTextRequest; import android.view.inputmethod.InputConnection; import android.view.inputmethod.InputMethodManager; import android.view.textclassifier.TextClassificationResult; -import android.view.textclassifier.TextSelection; import android.widget.AdapterView.OnItemClickListener; import android.widget.TextView.Drawables; import android.widget.TextView.OnEditorActionListener; @@ -171,7 +170,7 @@ public class Editor { private InsertionPointCursorController mInsertionPointCursorController; SelectionModifierCursorController mSelectionModifierCursorController; // Action mode used when text is selected or when actions on an insertion cursor are triggered. - ActionMode mTextActionMode; + private ActionMode mTextActionMode; private boolean mInsertionControllerEnabled; private boolean mSelectionControllerEnabled; @@ -238,7 +237,7 @@ public class Editor { private boolean mPreserveSelection; private boolean mRestartActionModeOnNextRefresh; - private TextClassificationResult mTextClassificationResult; + private SelectionActionModeHelper mSelectionActionModeHelper; boolean mIsBeingLongClicked; @@ -294,7 +293,7 @@ public class Editor { private Rect mTempRect; - private TextView mTextView; + private final TextView mTextView; final ProcessTextIntentActionsHandler mProcessTextIntentActionsHandler; @@ -1891,7 +1890,7 @@ public class Editor { mInsertionPointCursorController.invalidateHandle(); } if (mTextActionMode != null) { - invalidateActionMode(getTextClassifierInfo(false)); + invalidateActionModeAsync(); } } @@ -1984,12 +1983,12 @@ public class Editor { if (mRestartActionModeOnNextRefresh) { // To avoid distraction, newly start action mode only when selection action // mode is being restarted. - startSelectionActionMode(null); + startSelectionActionMode(); } } else if (selectionController == null || !selectionController.isActive()) { // Insertion action mode is active. Avoid dismissing the selection. stopTextActionModeWithPreservingSelection(); - startSelectionActionMode(null); + startSelectionActionMode(); } else { mTextActionMode.invalidateContentRect(); } @@ -2026,55 +2025,46 @@ public class Editor { } } + @NonNull + TextView getTextView() { + return mTextView; + } + + @Nullable + ActionMode getTextActionMode() { + return mTextActionMode; + } + + void setRestartActionModeOnNextRefresh(boolean value) { + mRestartActionModeOnNextRefresh = value; + } + /** - * Starts a Selection Action Mode with the current selection and ensures the selection handles - * are shown if there is a selection. This should be used when the mode is started from a - * non-touch event. - * - * @return true if the selection mode was actually started. + * Asynchronously starts a selection action mode using the TextClassifier. */ - boolean startSelectionActionMode(@Nullable TextClassificationResult textClassificationResult) { - mTextClassificationResult = textClassificationResult; - boolean selectionStarted = startSelectionActionModeInternal(); - if (selectionStarted) { - getSelectionController().show(); + void startSelectionActionModeAsync() { + getSelectionActionModeHelper().startActionModeAsync(); + } + + /** + * Synchronously starts a selection action mode without the TextClassifier. + */ + void startSelectionActionMode() { + getSelectionActionModeHelper().startActionMode(); + } + + /** + * Asynchronously invalidates an action mode using the TextClassifier. + */ + private void invalidateActionModeAsync() { + getSelectionActionModeHelper().invalidateActionModeAsync(); + } + + private SelectionActionModeHelper getSelectionActionModeHelper() { + if (mSelectionActionModeHelper == null) { + mSelectionActionModeHelper = new SelectionActionModeHelper(this); } - mRestartActionModeOnNextRefresh = false; - return selectionStarted; - } - - private boolean startSelectionActionModeWithTextAssistant() { - return startSelectionActionMode(getTextClassifierInfo(true)); - } - - private void invalidateActionMode(TextClassificationResult textClassificationResult) { - mTextClassificationResult = textClassificationResult; - mTextActionMode.invalidate(); - } - - // TODO: Make this a non-blocking call. - private TextClassificationResult getTextClassifierInfo(boolean updateSelection) { - // TODO: Trim the text so that only text necessary to provide context of the selected - // text is sent to the assistant. - final int trimStartIndex = 0; - final int trimEndIndex = mTextView.getText().length(); - CharSequence trimmedText = - mTextView.getText().subSequence(trimStartIndex, trimEndIndex); - int startIndex = mTextView.getSelectionStart() - trimStartIndex; - int endIndex = mTextView.getSelectionEnd() - trimStartIndex; - - if (updateSelection) { - TextSelection textSelection = mTextView.getTextClassifier() - .suggestSelection(trimmedText, startIndex, endIndex); - startIndex = Math.max(0, textSelection.getSelectionStartIndex() + trimStartIndex); - endIndex = Math.min(mTextView.getText().length(), - textSelection.getSelectionEndIndex() + trimStartIndex); - Selection.setSelection((Spannable) mTextView.getText(), startIndex, endIndex); - return getTextClassifierInfo(false); - } - - return mTextView.getTextClassifier() - .getTextClassificationResult(trimmedText, startIndex, endIndex); + return mSelectionActionModeHelper; } /** @@ -2117,13 +2107,13 @@ public class Editor { return true; } - private boolean startSelectionActionModeInternal() { + boolean startSelectionActionModeInternal() { if (extractedTextModeWillBeStarted()) { return false; } if (mTextActionMode != null) { // Text action mode is already started - invalidateActionMode(getTextClassifierInfo(false)); + invalidateActionModeAsync(); return false; } @@ -2314,7 +2304,8 @@ public class Editor { return mInsertionPointCursorController; } - private SelectionModifierCursorController getSelectionController() { + @Nullable + SelectionModifierCursorController getSelectionController() { if (!mSelectionControllerEnabled) { return null; } @@ -3813,7 +3804,7 @@ public class Editor { mode.setSubtitle(null); mode.setTitleOptionalHint(true); populateMenuWithItems(menu); - updateAssistMenuItem(menu, mTextClassificationResult); + updateAssistMenuItem(menu); Callback customCallback = getCustomCallback(); if (customCallback != null) { @@ -3881,7 +3872,7 @@ public class Editor { public boolean onPrepareActionMode(ActionMode mode, Menu menu) { updateSelectAllItem(menu); updateReplaceItem(menu); - updateAssistMenuItem(menu, mTextClassificationResult); + updateAssistMenuItem(menu); Callback customCallback = getCustomCallback(); if (customCallback != null) { @@ -3914,9 +3905,10 @@ public class Editor { } } - private void updateAssistMenuItem( - Menu menu, TextClassificationResult textClassificationResult) { + private void updateAssistMenuItem(Menu menu) { menu.removeItem(TextView.ID_ASSIST); + final TextClassificationResult textClassificationResult = + getSelectionActionModeHelper().getTextClassificationResult(); if (textClassificationResult != null) { final Drawable icon = textClassificationResult.getIcon(); final CharSequence label = textClassificationResult.getLabel(); @@ -3941,7 +3933,8 @@ public class Editor { if (customCallback != null && customCallback.onActionItemClicked(mode, item)) { return true; } - final TextClassificationResult textClassificationResult = mTextClassificationResult; + final TextClassificationResult textClassificationResult = + getSelectionActionModeHelper().getTextClassificationResult(); if (TextView.ID_ASSIST == item.getItemId() && textClassificationResult != null) { final OnClickListener onClickListener = textClassificationResult.getOnClickListener(); @@ -3964,8 +3957,8 @@ public class Editor { @Override public void onDestroyActionMode(ActionMode mode) { // Clear mTextActionMode not to recursively destroy action mode by clearing selection. + getSelectionActionModeHelper().cancelAsyncTask(); mTextActionMode = null; - mTextClassificationResult = null; Callback customCallback = getCustomCallback(); if (customCallback != null) { customCallback.onDestroyActionMode(mode); @@ -4783,7 +4776,7 @@ public class Editor { } positionAtCursorOffset(offset, false); if (mTextActionMode != null) { - invalidateActionMode(getTextClassifierInfo(false)); + invalidateActionModeAsync(); } } @@ -4867,7 +4860,7 @@ public class Editor { } updateDrawable(); if (mTextActionMode != null) { - invalidateActionMode(getTextClassifierInfo(false)); + invalidateActionModeAsync(); } } @@ -5516,8 +5509,12 @@ public class Editor { if (mTextView.hasSelection()) { // Do not invoke the text assistant if this was a drag selection. - startSelectionActionMode( - mHaventMovedEnoughToStartDrag ? getTextClassifierInfo(true) : null); + if (mHaventMovedEnoughToStartDrag) { + startSelectionActionModeAsync(); + } else { + startSelectionActionMode(); + } + } break; } diff --git a/core/java/android/widget/SelectionActionModeHelper.java b/core/java/android/widget/SelectionActionModeHelper.java new file mode 100644 index 0000000000000..770d9eec792ac --- /dev/null +++ b/core/java/android/widget/SelectionActionModeHelper.java @@ -0,0 +1,288 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.widget; + +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.annotation.UiThread; +import android.annotation.WorkerThread; +import android.os.AsyncTask; +import android.text.Selection; +import android.text.Spannable; +import android.text.TextUtils; +import android.view.ActionMode; +import android.view.textclassifier.TextClassificationResult; +import android.view.textclassifier.TextClassifier; +import android.view.textclassifier.TextSelection; +import android.widget.Editor.SelectionModifierCursorController; + +import com.android.internal.util.Preconditions; + +import java.util.function.Consumer; +import java.util.function.Supplier; + +/** + * Helper class for starting selection action mode + * (synchronously without the TextClassifier, asynchronously with the TextClassifier). + */ +@UiThread +final class SelectionActionModeHelper { + + /** + * Maximum time (in milliseconds) to wait for a result before timing out. + */ + // TODO: Consider making this a ViewConfiguration. + private static final int TIMEOUT_DURATION = 200; + + private final Editor mEditor; + private final TextClassificationHelper mTextClassificationHelper; + + private TextClassificationResult mTextClassificationResult; + private AsyncTask mTextClassificationAsyncTask; + + SelectionActionModeHelper(@NonNull Editor editor) { + mEditor = Preconditions.checkNotNull(editor); + final TextView textView = mEditor.getTextView(); + mTextClassificationHelper = new TextClassificationHelper( + textView.getTextClassifier(), textView.getText(), + textView.getSelectionStart(), textView.getSelectionEnd()); + } + + public void startActionModeAsync() { + cancelAsyncTask(); + if (isNoOpTextClassifier()) { + // No need to make an async call for a no-op TextClassifier. + startActionMode(null); + } else { + resetTextClassificationHelper(); + mTextClassificationAsyncTask = new TextClassificationAsyncTask( + mEditor.getTextView(), TIMEOUT_DURATION, + mTextClassificationHelper::suggestSelection, this::startActionMode) + .execute(); + } + } + + public void startActionMode() { + startActionMode(null); + } + + public void invalidateActionModeAsync() { + cancelAsyncTask(); + if (isNoOpTextClassifier()) { + // No need to make an async call for a no-op TextClassifier. + invalidateActionMode(null); + } else { + resetTextClassificationHelper(); + mTextClassificationAsyncTask = new TextClassificationAsyncTask( + mEditor.getTextView(), TIMEOUT_DURATION, + mTextClassificationHelper::classifyText, this::invalidateActionMode) + .execute(); + } + } + + public void cancelAsyncTask() { + if (mTextClassificationAsyncTask != null) { + mTextClassificationAsyncTask.cancel(true); + mTextClassificationAsyncTask = null; + } + mTextClassificationResult = null; + } + + @Nullable + public TextClassificationResult getTextClassificationResult() { + return mTextClassificationResult; + } + + private boolean isNoOpTextClassifier() { + return mEditor.getTextView().getTextClassifier() == TextClassifier.NO_OP; + } + + private void startActionMode(@Nullable SelectionResult result) { + final CharSequence text = mEditor.getTextView().getText(); + if (result != null && text instanceof Spannable) { + Selection.setSelection((Spannable) text, result.mStart, result.mEnd); + mTextClassificationResult = result.mResult; + } else { + mTextClassificationResult = null; + } + if (mEditor.startSelectionActionModeInternal()) { + final SelectionModifierCursorController controller = mEditor.getSelectionController(); + if (controller != null) { + controller.show(); + } + } + mEditor.setRestartActionModeOnNextRefresh(false); + mTextClassificationAsyncTask = null; + } + + private void invalidateActionMode(@Nullable SelectionResult result) { + mTextClassificationResult = result != null ? result.mResult : null; + final ActionMode actionMode = mEditor.getTextActionMode(); + if (actionMode != null) { + actionMode.invalidate(); + } + mTextClassificationAsyncTask = null; + } + + private void resetTextClassificationHelper() { + final TextView textView = mEditor.getTextView(); + mTextClassificationHelper.reset(textView.getTextClassifier(), textView.getText(), + textView.getSelectionStart(), textView.getSelectionEnd()); + } + + /** + * AsyncTask for running a query on a background thread and returning the result on the + * UiThread. The AsyncTask times out after a specified time, returning a null result if the + * query has not yet returned. + */ + private static final class TextClassificationAsyncTask + extends AsyncTask { + + private final int mTimeOutDuration; + private final Supplier mSelectionResultSupplier; + private final Consumer mSelectionResultCallback; + private final TextView mTextView; + private final String mOriginalText; + + /** + * @param textView the TextView + * @param timeOut time in milliseconds to timeout the query if it has not completed + * @param selectionResultSupplier fetches the selection results. Runs on a background thread + * @param selectionResultCallback receives the selection results. Runs on the UiThread + */ + TextClassificationAsyncTask( + @NonNull TextView textView, int timeOut, + @NonNull Supplier selectionResultSupplier, + @NonNull Consumer selectionResultCallback) { + mTextView = Preconditions.checkNotNull(textView); + mTimeOutDuration = timeOut; + mSelectionResultSupplier = Preconditions.checkNotNull(selectionResultSupplier); + mSelectionResultCallback = Preconditions.checkNotNull(selectionResultCallback); + // Make a copy of the original text. + mOriginalText = mTextView.getText().toString(); + } + + @Override + @WorkerThread + protected SelectionResult doInBackground(Void... params) { + final Runnable onTimeOut = this::onTimeOut; + mTextView.postDelayed(onTimeOut, mTimeOutDuration); + final SelectionResult result = mSelectionResultSupplier.get(); + mTextView.removeCallbacks(onTimeOut); + return result; + } + + @Override + @UiThread + protected void onPostExecute(SelectionResult result) { + result = TextUtils.equals(mOriginalText, mTextView.getText()) ? result : null; + mSelectionResultCallback.accept(result); + } + + private void onTimeOut() { + if (getStatus() == Status.RUNNING) { + onPostExecute(null); + } + cancel(true); + } + } + + /** + * Helper class for querying the TextClassifier. + * It trims text so that only text necessary to provide context of the selected text is + * sent to the TextClassifier. + */ + private static final class TextClassificationHelper { + + private static final int TRIM_DELTA = 50; // characters + + private TextClassifier mTextClassifier; + + /** The original TextView text. **/ + private String mText; + /** Start index relative to mText. */ + private int mSelectionStart; + /** End index relative to mText. */ + private int mSelectionEnd; + + /** Trimmed text starting from mTrimStart in mText. */ + private CharSequence mTrimmedText; + /** Index indicating the start of mTrimmedText in mText. */ + private int mTrimStart; + /** Start index relative to mTrimmedText */ + private int mRelativeStart; + /** End index relative to mTrimmedText */ + private int mRelativeEnd; + + TextClassificationHelper(TextClassifier textClassifier, + CharSequence text, int selectionStart, int selectionEnd) { + reset(textClassifier, text, selectionStart, selectionEnd); + } + + @UiThread + public void reset(TextClassifier textClassifier, + CharSequence text, int selectionStart, int selectionEnd) { + mTextClassifier = Preconditions.checkNotNull(textClassifier); + mText = Preconditions.checkNotNull(text).toString(); + mSelectionStart = selectionStart; + mSelectionEnd = selectionEnd; + } + + @WorkerThread + public SelectionResult classifyText() { + trimText(); + return new SelectionResult( + mSelectionStart, + mSelectionEnd, + mTextClassifier.getTextClassificationResult( + mTrimmedText, mRelativeStart, mRelativeEnd)); + } + + @WorkerThread + public SelectionResult suggestSelection() { + trimText(); + final TextSelection sel = mTextClassifier.suggestSelection( + mTrimmedText, mRelativeStart, mRelativeEnd); + mSelectionStart = Math.max(0, sel.getSelectionStartIndex() + mTrimStart); + mSelectionEnd = Math.min(mText.length(), sel.getSelectionEndIndex() + mTrimStart); + return classifyText(); + } + + private void trimText() { + mTrimStart = Math.max(0, mSelectionStart - TRIM_DELTA); + final int referenceEnd = Math.min(mText.length(), mSelectionEnd + TRIM_DELTA); + mTrimmedText = mText.subSequence(mTrimStart, referenceEnd); + mRelativeStart = mSelectionStart - mTrimStart; + mRelativeEnd = mSelectionEnd - mTrimStart; + } + } + + /** + * Selection result. + */ + private static final class SelectionResult { + private final int mStart; + private final int mEnd; + private final TextClassificationResult mResult; + + SelectionResult(int start, int end, TextClassificationResult result) { + mStart = start; + mEnd = end; + mResult = Preconditions.checkNotNull(result); + } + } +} diff --git a/core/java/android/widget/TextView.java b/core/java/android/widget/TextView.java index 4a8ec94962d14..65167e85acdc1 100644 --- a/core/java/android/widget/TextView.java +++ b/core/java/android/widget/TextView.java @@ -6635,7 +6635,7 @@ public class TextView extends View implements ViewTreeObserver.OnPreDrawListener */ public boolean handleBackInTextActionModeIfNeeded(KeyEvent event) { // Do nothing unless mEditor is in text action mode. - if (mEditor == null || mEditor.mTextActionMode == null) { + if (mEditor == null || mEditor.getTextActionMode() == null) { return false; } @@ -6819,7 +6819,7 @@ public class TextView extends View implements ViewTreeObserver.OnPreDrawListener // Has to be done on key down (and not on key up) to correctly be intercepted. case KeyEvent.KEYCODE_BACK: - if (mEditor != null && mEditor.mTextActionMode != null) { + if (mEditor != null && mEditor.getTextActionMode() != null) { stopTextActionMode(); return KEY_EVENT_HANDLED; } @@ -9012,7 +9012,8 @@ public class TextView extends View implements ViewTreeObserver.OnPreDrawListener if (mEditor != null) { mEditor.refreshTextActionMode(); - if (!hasSelection() && mEditor.mTextActionMode == null && hasTransientState()) { + if (!hasSelection() + && mEditor.getTextActionMode() == null && hasTransientState()) { // User generated selection has been removed. setHasTransientState(false); } @@ -10009,7 +10010,7 @@ public class TextView extends View implements ViewTreeObserver.OnPreDrawListener Selection.setSelection((Spannable) text, start, end); // Make sure selection mode is engaged. if (mEditor != null) { - mEditor.startSelectionActionMode(null); + mEditor.startSelectionActionModeAsync(); } return true; }