From 3e3ff1a3d298f74246597cb10529e52e96b0d7b9 Mon Sep 17 00:00:00 2001 From: Taran Singh Date: Wed, 20 Jul 2022 23:58:13 -0700 Subject: [PATCH] Introduce Scribe Rich gestures API During a stylus handwriting session, user can perform a stylus gesture operation like Select, delete, insert on an area of text. IME needs APIs to perform these gestures on text. This CL introduces API signature for Select, delete, insert gestures. Design doc: go/scribe-gestures-api Bug: 239783077 Bug: 210039666 Test: Manually build & compile HandwritingIme Change-Id: I53bcb62e03ac1c371feb60d1385c88c921754092 --- core/api/current.txt | 55 +++++ .../IRemoteInputConnectionInvoker.java | 28 +++ .../RemoteInputConnection.java | 11 + .../view/inputmethod/DeleteGesture.aidl | 19 ++ .../view/inputmethod/DeleteGesture.java | 193 ++++++++++++++++++ .../view/inputmethod/HandwritingGesture.java | 94 +++++++++ .../view/inputmethod/InputConnection.java | 14 ++ .../inputmethod/InputConnectionWrapper.java | 15 ++ .../view/inputmethod/InsertGesture.aidl | 19 ++ .../view/inputmethod/InsertGesture.java | 172 ++++++++++++++++ .../view/inputmethod/SelectGesture.aidl | 19 ++ .../view/inputmethod/SelectGesture.java | 191 +++++++++++++++++ .../inputmethod/IRemoteInputConnection.aidl | 13 ++ .../RemoteInputConnectionImpl.java | 66 ++++++ 14 files changed, 909 insertions(+) create mode 100644 core/java/android/view/inputmethod/DeleteGesture.aidl create mode 100644 core/java/android/view/inputmethod/DeleteGesture.java create mode 100644 core/java/android/view/inputmethod/HandwritingGesture.java create mode 100644 core/java/android/view/inputmethod/InsertGesture.aidl create mode 100644 core/java/android/view/inputmethod/InsertGesture.java create mode 100644 core/java/android/view/inputmethod/SelectGesture.aidl create mode 100644 core/java/android/view/inputmethod/SelectGesture.java diff --git a/core/api/current.txt b/core/api/current.txt index 3efe2ef5baeaf..8134df40c524c 100644 --- a/core/api/current.txt +++ b/core/api/current.txt @@ -52985,6 +52985,22 @@ package android.view.inputmethod { method public android.view.inputmethod.CursorAnchorInfo.Builder setSelectionRange(int, int); } + public final class DeleteGesture extends android.view.inputmethod.HandwritingGesture implements android.os.Parcelable { + method public int describeContents(); + method @NonNull public android.graphics.RectF getDeletionArea(); + method public int getGranularity(); + method public void writeToParcel(@NonNull android.os.Parcel, int); + field @NonNull public static final android.os.Parcelable.Creator CREATOR; + } + + public static final class DeleteGesture.Builder { + ctor public DeleteGesture.Builder(); + method @NonNull public android.view.inputmethod.DeleteGesture build(); + method @NonNull public android.view.inputmethod.DeleteGesture.Builder setDeletionArea(@NonNull android.graphics.RectF); + method @NonNull public android.view.inputmethod.DeleteGesture.Builder setFallbackText(@Nullable String); + method @NonNull public android.view.inputmethod.DeleteGesture.Builder setGranularity(int); + } + public final class EditorBoundsInfo implements android.os.Parcelable { method public int describeContents(); method @Nullable public android.graphics.RectF getEditorBounds(); @@ -53079,6 +53095,12 @@ package android.view.inputmethod { field public int token; } + public abstract class HandwritingGesture { + method @Nullable public String getFallbackText(); + field public static final int GRANULARITY_CHARACTER = 2; // 0x2 + field public static final int GRANULARITY_WORD = 1; // 0x1 + } + public final class InlineSuggestion implements android.os.Parcelable { method public int describeContents(); method @NonNull public android.view.inputmethod.InlineSuggestionInfo getInfo(); @@ -53169,6 +53191,7 @@ package android.view.inputmethod { method @Nullable public CharSequence getTextBeforeCursor(@IntRange(from=0) int, int); method public boolean performContextMenuAction(int); method public boolean performEditorAction(int); + method public default void performHandwritingGesture(@NonNull android.view.inputmethod.HandwritingGesture, @Nullable java.util.concurrent.Executor, @Nullable java.util.function.IntConsumer); method public boolean performPrivateCommand(String, android.os.Bundle); method public default boolean performSpellCheck(); method public boolean reportFullscreenMode(boolean); @@ -53390,6 +53413,38 @@ package android.view.inputmethod { method public android.view.inputmethod.InputMethodSubtype.InputMethodSubtypeBuilder setSubtypeNameResId(int); } + public final class InsertGesture extends android.view.inputmethod.HandwritingGesture implements android.os.Parcelable { + method public int describeContents(); + method @Nullable public android.graphics.PointF getInsertionPoint(); + method @Nullable public String getTextToInsert(); + method public void writeToParcel(@NonNull android.os.Parcel, int); + field @NonNull public static final android.os.Parcelable.Creator CREATOR; + } + + public static final class InsertGesture.Builder { + ctor public InsertGesture.Builder(); + method @NonNull public android.view.inputmethod.InsertGesture build(); + method @NonNull public android.view.inputmethod.InsertGesture.Builder setFallbackText(@Nullable String); + method @NonNull public android.view.inputmethod.InsertGesture.Builder setInsertionPoint(@NonNull android.graphics.PointF); + method @NonNull public android.view.inputmethod.InsertGesture.Builder setTextToInsert(@NonNull String); + } + + public final class SelectGesture extends android.view.inputmethod.HandwritingGesture implements android.os.Parcelable { + method public int describeContents(); + method public int getGranularity(); + method @NonNull public android.graphics.RectF getSelectionArea(); + method public void writeToParcel(@NonNull android.os.Parcel, int); + field @NonNull public static final android.os.Parcelable.Creator CREATOR; + } + + public static final class SelectGesture.Builder { + ctor public SelectGesture.Builder(); + method @NonNull public android.view.inputmethod.SelectGesture build(); + method @NonNull public android.view.inputmethod.SelectGesture.Builder setFallbackText(@Nullable String); + method @NonNull public android.view.inputmethod.SelectGesture.Builder setGranularity(int); + method @NonNull public android.view.inputmethod.SelectGesture.Builder setSelectionArea(@NonNull android.graphics.RectF); + } + public final class SurroundingText implements android.os.Parcelable { ctor public SurroundingText(@NonNull CharSequence, @IntRange(from=0) int, @IntRange(from=0) int, @IntRange(from=0xffffffff) int); method public int describeContents(); diff --git a/core/java/android/inputmethodservice/IRemoteInputConnectionInvoker.java b/core/java/android/inputmethodservice/IRemoteInputConnectionInvoker.java index e38e611227865..3260713a7d14b 100644 --- a/core/java/android/inputmethodservice/IRemoteInputConnectionInvoker.java +++ b/core/java/android/inputmethodservice/IRemoteInputConnectionInvoker.java @@ -17,6 +17,7 @@ package android.inputmethodservice; import android.annotation.AnyThread; +import android.annotation.CallbackExecutor; import android.annotation.NonNull; import android.annotation.Nullable; import android.os.Bundle; @@ -24,9 +25,13 @@ import android.os.RemoteException; import android.view.KeyEvent; import android.view.inputmethod.CompletionInfo; import android.view.inputmethod.CorrectionInfo; +import android.view.inputmethod.DeleteGesture; import android.view.inputmethod.ExtractedText; import android.view.inputmethod.ExtractedTextRequest; +import android.view.inputmethod.HandwritingGesture; import android.view.inputmethod.InputContentInfo; +import android.view.inputmethod.InsertGesture; +import android.view.inputmethod.SelectGesture; import android.view.inputmethod.SurroundingText; import android.view.inputmethod.TextAttribute; @@ -35,6 +40,8 @@ import com.android.internal.inputmethod.IRemoteInputConnection; import com.android.internal.inputmethod.InputConnectionCommandHeader; import java.util.Objects; +import java.util.concurrent.Executor; +import java.util.function.IntConsumer; /** * A stateless wrapper of {@link com.android.internal.inputmethod.IRemoteInputConnection} to @@ -591,6 +598,27 @@ final class IRemoteInputConnectionInvoker { } } + @AnyThread + public void performHandwritingGesture( + @NonNull HandwritingGesture gesture, @Nullable @CallbackExecutor Executor executor, + @Nullable IntConsumer consumer) { + // TODO(b/210039666): implement resultReceiver + try { + if (gesture instanceof SelectGesture) { + mConnection.performHandwritingSelectGesture( + createHeader(), (SelectGesture) gesture, null); + } else if (gesture instanceof InsertGesture) { + mConnection.performHandwritingInsertGesture( + createHeader(), (InsertGesture) gesture, null); + } else if (gesture instanceof DeleteGesture) { + mConnection.performHandwritingDeleteGesture( + createHeader(), (DeleteGesture) gesture, null); + } + } catch (RemoteException e) { + // TODO(b/210039666): return result + } + } + /** * Invokes {@link IRemoteInputConnection#requestCursorUpdates(InputConnectionCommandHeader, int, * int, AndroidFuture)}. diff --git a/core/java/android/inputmethodservice/RemoteInputConnection.java b/core/java/android/inputmethodservice/RemoteInputConnection.java index 2711c4f0db1f5..694293c62bd73 100644 --- a/core/java/android/inputmethodservice/RemoteInputConnection.java +++ b/core/java/android/inputmethodservice/RemoteInputConnection.java @@ -17,6 +17,7 @@ package android.inputmethodservice; import android.annotation.AnyThread; +import android.annotation.CallbackExecutor; import android.annotation.IntRange; import android.annotation.NonNull; import android.annotation.Nullable; @@ -28,6 +29,7 @@ import android.view.inputmethod.CompletionInfo; import android.view.inputmethod.CorrectionInfo; import android.view.inputmethod.ExtractedText; import android.view.inputmethod.ExtractedTextRequest; +import android.view.inputmethod.HandwritingGesture; import android.view.inputmethod.InputConnection; import android.view.inputmethod.InputContentInfo; import android.view.inputmethod.SurroundingText; @@ -41,6 +43,8 @@ import com.android.internal.inputmethod.InputConnectionProtoDumper; import java.lang.ref.WeakReference; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executor; +import java.util.function.IntConsumer; /** * Takes care of remote method invocations of {@link InputConnection} in the IME side. @@ -410,6 +414,13 @@ final class RemoteInputConnection implements InputConnection { return mInvoker.performPrivateCommand(action, data); } + @AnyThread + public void performHandwritingGesture( + @NonNull HandwritingGesture gesture, @Nullable @CallbackExecutor Executor executor, + @Nullable IntConsumer consumer) { + mInvoker.performHandwritingGesture(gesture, executor, consumer); + } + @AnyThread public boolean requestCursorUpdates(int cursorUpdateMode) { if (mCancellationGroup.isCanceled()) { diff --git a/core/java/android/view/inputmethod/DeleteGesture.aidl b/core/java/android/view/inputmethod/DeleteGesture.aidl new file mode 100644 index 0000000000000..e9f31dd470ef4 --- /dev/null +++ b/core/java/android/view/inputmethod/DeleteGesture.aidl @@ -0,0 +1,19 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.view.inputmethod; + +parcelable DeleteGesture; \ No newline at end of file diff --git a/core/java/android/view/inputmethod/DeleteGesture.java b/core/java/android/view/inputmethod/DeleteGesture.java new file mode 100644 index 0000000000000..257254e5737a9 --- /dev/null +++ b/core/java/android/view/inputmethod/DeleteGesture.java @@ -0,0 +1,193 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.view.inputmethod; + +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.annotation.SuppressLint; +import android.graphics.RectF; +import android.os.Parcel; +import android.os.Parcelable; +import android.widget.TextView; + +import java.util.Objects; + +/** + * A sub-class of {@link HandwritingGesture} for deleting an area of text. + * This class holds the information required for deletion of text in + * toolkit widgets like {@link TextView}. + */ +public final class DeleteGesture extends HandwritingGesture implements Parcelable { + + private @Granularity int mGranularity; + private RectF mArea; + + private DeleteGesture(@Granularity int granularity, RectF area, String fallbackText) { + mArea = area; + mGranularity = granularity; + mFallbackText = fallbackText; + } + + private DeleteGesture(@NonNull final Parcel source) { + mFallbackText = source.readString8(); + mGranularity = source.readInt(); + mArea = source.readTypedObject(RectF.CREATOR); + } + + /** + * Returns Granular level on which text should be operated. + * @see HandwritingGesture#GRANULARITY_CHARACTER + * @see HandwritingGesture#GRANULARITY_WORD + */ + @Granularity + public int getGranularity() { + return mGranularity; + } + + /** + * Returns the deletion area {@link RectF} in screen coordinates. + * + * Getter for deletion area set with {@link DeleteGesture.Builder#setDeletionArea(RectF)}. + * {@code null} if area was not set. + */ + @NonNull + public RectF getDeletionArea() { + return mArea; + } + + /** + * Builder for {@link DeleteGesture}. This class is not designed to be thread-safe. + */ + public static final class Builder { + private int mGranularity; + private RectF mArea; + private String mFallbackText; + + /** + * Set text deletion granularity. Intersecting words/characters will be + * included in the operation. + * @param granularity {@link HandwritingGesture#GRANULARITY_WORD} or + * {@link HandwritingGesture#GRANULARITY_CHARACTER}. + * @return {@link Builder}. + */ + @NonNull + @SuppressLint("MissingGetterMatchingBuilder") + public Builder setGranularity(@Granularity int granularity) { + mGranularity = granularity; + return this; + } + + /** + * Set rectangular single/multiline text deletion area intersecting with text. + * + * The resulting deletion would be performed for all text intersecting rectangle. The + * deletion includes the first word/character in the rectangle, and the last + * word/character in the rectangle, and includes everything in between even if it's not + * in the rectangle. + * + * Intersection is determined using + * {@link #setGranularity(int)}. e.g. {@link HandwritingGesture#GRANULARITY_WORD} includes + * all the words with their width/height center included in the deletion rectangle. + * @param area {@link RectF} (in screen coordinates) for which text will be deleted. + * @see HandwritingGesture#GRANULARITY_WORD + * @see HandwritingGesture#GRANULARITY_CHARACTER + */ + @SuppressLint("MissingGetterMatchingBuilder") + @NonNull + public Builder setDeletionArea(@NonNull RectF area) { + mArea = area; + return this; + } + + /** + * Set fallback text that will be committed at current cursor position if there is no + * applicable text beneath the area of gesture. + * @param fallbackText text to set + */ + @NonNull + public Builder setFallbackText(@Nullable String fallbackText) { + mFallbackText = fallbackText; + return this; + } + + /** + * @return {@link DeleteGesture} using parameters in this {@link DeleteGesture.Builder}. + * @throws IllegalArgumentException if one or more positional parameters are not specified. + */ + @NonNull + public DeleteGesture build() { + if (mArea == null || mArea.isEmpty()) { + throw new IllegalArgumentException("Deletion area must be set."); + } + if (mGranularity <= GRANULARITY_UNDEFINED) { + throw new IllegalArgumentException("Deletion granularity must be set."); + } + return new DeleteGesture(mGranularity, mArea, mFallbackText); + } + } + + /** + * Used to make this class parcelable. + */ + public static final @android.annotation.NonNull Creator CREATOR = + new Creator() { + @Override + public DeleteGesture createFromParcel(Parcel source) { + return new DeleteGesture(source); + } + + @Override + public DeleteGesture[] newArray(int size) { + return new DeleteGesture[size]; + } + }; + + @Override + public int hashCode() { + return Objects.hash(mArea, mGranularity, mFallbackText); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof DeleteGesture)) return false; + + DeleteGesture that = (DeleteGesture) o; + + if (mGranularity != that.mGranularity) return false; + if (!Objects.equals(mFallbackText, that.mFallbackText)) return false; + return Objects.equals(mArea, that.mArea); + } + + @Override + public int describeContents() { + return 0; + } + + /** + * Used to package this object into a {@link Parcel}. + * + * @param dest The {@link Parcel} to be written. + * @param flags The flags used for parceling. + */ + @Override + public void writeToParcel(@NonNull Parcel dest, int flags) { + dest.writeString8(mFallbackText); + dest.writeInt(mGranularity); + dest.writeTypedObject(mArea, flags); + } +} diff --git a/core/java/android/view/inputmethod/HandwritingGesture.java b/core/java/android/view/inputmethod/HandwritingGesture.java new file mode 100644 index 0000000000000..15824aeb0aebf --- /dev/null +++ b/core/java/android/view/inputmethod/HandwritingGesture.java @@ -0,0 +1,94 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.view.inputmethod; + +import android.annotation.IntDef; +import android.annotation.Nullable; +import android.graphics.RectF; +import android.inputmethodservice.InputMethodService; +import android.view.MotionEvent; + +import java.util.concurrent.Executor; +import java.util.function.IntConsumer; + +/** + * Base class for Stylus handwriting gesture. + * + * During a stylus handwriting session, user can perform a stylus gesture operation like + * {@link SelectGesture}, {@link DeleteGesture}, {@link InsertGesture} on an + * area of text. IME is responsible for listening to Stylus {@link MotionEvent} using + * {@link InputMethodService#onStylusHandwritingMotionEvent} and interpret if it can translate to a + * gesture operation. + * While creating Gesture operations {@link SelectGesture}, {@link DeleteGesture}, + * , {@code Granularity} helps pick the correct granular level of text like word level + * {@link #GRANULARITY_WORD}, or character level {@link #GRANULARITY_CHARACTER}. + * + * @see InputConnection#performHandwritingGesture(HandwritingGesture, Executor, IntConsumer) + * @see InputMethodService#onStartStylusHandwriting() + */ +public abstract class HandwritingGesture { + + HandwritingGesture() {} + + static final int GRANULARITY_UNDEFINED = 0; + + /** + * Operate text per word basis. e.g. if selection includes width-wise center of the word, + * whole word is selected. + *

Strategy of operating at a granular level is maintained in the UI toolkit. + * A character/word/line is included if its center is within the gesture rectangle. + * e.g. if a selection {@link RectF} with {@link #GRANULARITY_WORD} includes width-wise + * center of the word, it should be selected. + * Similarly, text in a line should be included in the operation if rectangle includes + * line height center.

+ * Refer to https://www.unicode.org/reports/tr29/#Word_Boundaries for more detail on how word + * breaks are decided. + */ + public static final int GRANULARITY_WORD = 1; + + /** + * Operate on text per character basis. i.e. each character is selected based on its + * intersection with selection rectangle. + *

Strategy of operating at a granular level is maintained in the UI toolkit. + * A character/word/line is included if its center is within the gesture rectangle. + * e.g. if a selection {@link RectF} with {@link #GRANULARITY_CHARACTER} includes width-wise + * center of the character, it should be selected. + * Similarly, text in a line should be included in the operation if rectangle includes + * line height center.

+ */ + public static final int GRANULARITY_CHARACTER = 2; + + /** + * Granular level on which text should be operated. + */ + @IntDef({GRANULARITY_CHARACTER, GRANULARITY_WORD}) + @interface Granularity {} + + @Nullable + String mFallbackText; + + /** + * The fallback text that will be committed at current cursor position if there is no applicable + * text beneath the area of gesture. + * For example, select can fail if gesture is drawn over area that has no text beneath. + * example 2: join can fail if the gesture is drawn over text but there is no whitespace. + */ + @Nullable + public String getFallbackText() { + return mFallbackText; + } +} diff --git a/core/java/android/view/inputmethod/InputConnection.java b/core/java/android/view/inputmethod/InputConnection.java index dac1be6f5a134..7b0270a1859fc 100644 --- a/core/java/android/view/inputmethod/InputConnection.java +++ b/core/java/android/view/inputmethod/InputConnection.java @@ -16,6 +16,7 @@ package android.view.inputmethod; +import android.annotation.CallbackExecutor; import android.annotation.IntDef; import android.annotation.IntRange; import android.annotation.NonNull; @@ -31,6 +32,8 @@ import com.android.internal.util.Preconditions; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; +import java.util.concurrent.Executor; +import java.util.function.IntConsumer; /** * The InputConnection interface is the communication channel from an @@ -967,6 +970,17 @@ public interface InputConnection { */ boolean performPrivateCommand(String action, Bundle data); + /** + * Perform a handwriting gesture on text. + * + * @param gesture the gesture to perform + * @param executor if the caller passes a non-null consumer TODO(b/210039666): complete doc + * @param consumer if the caller passes a non-null receiver, the editor must invoke this + */ + default void performHandwritingGesture( + @NonNull HandwritingGesture gesture, @Nullable @CallbackExecutor Executor executor, + @Nullable IntConsumer consumer) {} + /** * The editor is requested to call * {@link InputMethodManager#updateCursorAnchorInfo(android.view.View, CursorAnchorInfo)} at diff --git a/core/java/android/view/inputmethod/InputConnectionWrapper.java b/core/java/android/view/inputmethod/InputConnectionWrapper.java index 7a88a75f93ade..56beddf2ef383 100644 --- a/core/java/android/view/inputmethod/InputConnectionWrapper.java +++ b/core/java/android/view/inputmethod/InputConnectionWrapper.java @@ -16,6 +16,7 @@ package android.view.inputmethod; +import android.annotation.CallbackExecutor; import android.annotation.IntRange; import android.annotation.NonNull; import android.annotation.Nullable; @@ -25,6 +26,9 @@ import android.view.KeyEvent; import com.android.internal.util.Preconditions; +import java.util.concurrent.Executor; +import java.util.function.IntConsumer; + /** *

Wrapper class for proxying calls to another InputConnection. Subclass and have fun! */ @@ -318,6 +322,17 @@ public class InputConnectionWrapper implements InputConnection { return mTarget.performPrivateCommand(action, data); } + /** + * {@inheritDoc} + * @throws NullPointerException if the target is {@code null}. + */ + @Override + public void performHandwritingGesture( + @NonNull HandwritingGesture gesture, @Nullable @CallbackExecutor Executor executor, + @Nullable IntConsumer consumer) { + mTarget.performHandwritingGesture(gesture, executor, consumer); + } + /** * {@inheritDoc} * @throws NullPointerException if the target is {@code null}. diff --git a/core/java/android/view/inputmethod/InsertGesture.aidl b/core/java/android/view/inputmethod/InsertGesture.aidl new file mode 100644 index 0000000000000..9cdb14a8b3ab9 --- /dev/null +++ b/core/java/android/view/inputmethod/InsertGesture.aidl @@ -0,0 +1,19 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.view.inputmethod; + +parcelable InsertGesture; \ No newline at end of file diff --git a/core/java/android/view/inputmethod/InsertGesture.java b/core/java/android/view/inputmethod/InsertGesture.java new file mode 100644 index 0000000000000..2cf015a33283d --- /dev/null +++ b/core/java/android/view/inputmethod/InsertGesture.java @@ -0,0 +1,172 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.view.inputmethod; + +import android.annotation.NonNull; +import android.annotation.SuppressLint; +import android.graphics.PointF; +import android.os.Parcel; +import android.os.Parcelable; +import android.text.TextUtils; +import android.widget.TextView; + +import androidx.annotation.Nullable; + +import java.util.Objects; + +/** + * A sub-class of {@link HandwritingGesture} for inserting text at the defined insertion point. + * This class holds the information required for insertion of text in + * toolkit widgets like {@link TextView}. + */ +public final class InsertGesture extends HandwritingGesture implements Parcelable { + + private String mTextToInsert; + private PointF mPoint; + + private InsertGesture(String text, PointF point, String fallbackText) { + mPoint = point; + mTextToInsert = text; + mFallbackText = fallbackText; + } + + private InsertGesture(final Parcel source) { + mFallbackText = source.readString8(); + mTextToInsert = source.readString8(); + mPoint = source.readTypedObject(PointF.CREATOR); + } + + /** Returns the text that will be inserted at {@link #getInsertionPoint()} **/ + @Nullable + public String getTextToInsert() { + return mTextToInsert; + } + + /** + * Returns the insertion point {@link PointF} (in screen coordinates) where + * {@link #getTextToInsert()} will be inserted. + */ + @Nullable + public PointF getInsertionPoint() { + return mPoint; + } + + /** + * Builder for {@link InsertGesture}. This class is not designed to be thread-safe. + */ + public static final class Builder { + private String mText; + private PointF mPoint; + private String mFallbackText; + + /** set the text that will be inserted at {@link #setInsertionPoint(PointF)} **/ + @NonNull + @SuppressLint("MissingGetterMatchingBuilder") + public Builder setTextToInsert(@NonNull String text) { + mText = text; + return this; + } + + /** + * Sets the insertion point (in screen coordinates) where {@link #setTextToInsert(String)} + * should be inserted. + */ + @NonNull + @SuppressLint("MissingGetterMatchingBuilder") + public Builder setInsertionPoint(@NonNull PointF point) { + mPoint = point; + return this; + } + + /** + * Set fallback text that will be committed at current cursor position if there is no + * applicable text beneath the area of gesture. + * @param fallbackText text to set + */ + @NonNull + public Builder setFallbackText(@Nullable String fallbackText) { + mFallbackText = fallbackText; + return this; + } + + /** + * @return {@link InsertGesture} using parameters in this {@link InsertGesture.Builder}. + * @throws IllegalArgumentException if one or more positional parameters are not specified. + */ + @NonNull + public InsertGesture build() { + if (mPoint == null) { + throw new IllegalArgumentException("Insertion point must be set."); + } + if (TextUtils.isEmpty(mText)) { + throw new IllegalArgumentException("Text to insert must be non-empty."); + } + return new InsertGesture(mText, mPoint, mFallbackText); + } + } + + /** + * Used to make this class parcelable. + */ + public static final @android.annotation.NonNull Creator CREATOR = + new Creator() { + @Override + public InsertGesture createFromParcel(Parcel source) { + return new InsertGesture(source); + } + + @Override + public InsertGesture[] newArray(int size) { + return new InsertGesture[size]; + } + }; + + @Override + public int hashCode() { + return Objects.hash(mPoint, mTextToInsert, mFallbackText); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof InsertGesture)) return false; + + InsertGesture that = (InsertGesture) o; + + if (!Objects.equals(mFallbackText, that.mFallbackText)) return false; + if (!Objects.equals(mTextToInsert, that.mTextToInsert)) return false; + return Objects.equals(mPoint, that.mPoint); + } + + @Override + public int describeContents() { + return 0; + } + + /** + * Used to package this object into a {@link Parcel}. + * + * @param dest The {@link Parcel} to be written. + * @param flags The flags used for parceling. + */ + @Override + public void writeToParcel(@NonNull Parcel dest, int flags) { + dest.writeString8(mFallbackText); + dest.writeString8(mTextToInsert); + dest.writeTypedObject(mPoint, flags); + } +} diff --git a/core/java/android/view/inputmethod/SelectGesture.aidl b/core/java/android/view/inputmethod/SelectGesture.aidl new file mode 100644 index 0000000000000..65da4f340e8c5 --- /dev/null +++ b/core/java/android/view/inputmethod/SelectGesture.aidl @@ -0,0 +1,19 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.view.inputmethod; + +parcelable SelectGesture; \ No newline at end of file diff --git a/core/java/android/view/inputmethod/SelectGesture.java b/core/java/android/view/inputmethod/SelectGesture.java new file mode 100644 index 0000000000000..f3cd71e1eed98 --- /dev/null +++ b/core/java/android/view/inputmethod/SelectGesture.java @@ -0,0 +1,191 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.view.inputmethod; + +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.annotation.SuppressLint; +import android.graphics.RectF; +import android.os.Parcel; +import android.os.Parcelable; +import android.widget.TextView; + +import java.util.Objects; + +/** + * A sub-class of {@link HandwritingGesture} for selecting an area of text. + * This class holds the information required for selection of text in + * toolkit widgets like {@link TextView}. + */ +public final class SelectGesture extends HandwritingGesture implements Parcelable { + + private @Granularity int mGranularity; + private RectF mArea; + + private SelectGesture(int granularity, RectF area, String fallbackText) { + mArea = area; + mGranularity = granularity; + mFallbackText = fallbackText; + } + + private SelectGesture(@NonNull Parcel source) { + mFallbackText = source.readString8(); + mGranularity = source.readInt(); + mArea = source.readTypedObject(RectF.CREATOR); + } + + /** + * Returns Granular level on which text should be operated. + * @see #GRANULARITY_CHARACTER + * @see #GRANULARITY_WORD + */ + @Granularity + public int getGranularity() { + return mGranularity; + } + + /** + * Returns the Selection area {@link RectF} in screen coordinates. + * + * Getter for selection area set with {@link Builder#setSelectionArea(RectF)}. {@code null} + * if area was not set. + */ + @NonNull + public RectF getSelectionArea() { + return mArea; + } + + + /** + * Builder for {@link SelectGesture}. This class is not designed to be thread-safe. + */ + public static final class Builder { + private int mGranularity; + private RectF mArea; + private String mFallbackText; + + /** + * Define text selection granularity. Intersecting words/characters will be + * included in the operation. + * @param granularity {@link HandwritingGesture#GRANULARITY_WORD} or + * {@link HandwritingGesture#GRANULARITY_CHARACTER}. + * @return {@link Builder}. + */ + @NonNull + @SuppressLint("MissingGetterMatchingBuilder") + public Builder setGranularity(@Granularity int granularity) { + mGranularity = granularity; + return this; + } + + /** + * Set rectangular single/multiline text selection area intersecting with text. + * + * The resulting selection would be performed for all text intersecting rectangle. The + * selection includes the first word/character in the rectangle, and the last + * word/character in the rectangle, and includes everything in between even if it's not + * in the rectangle. + * + * Intersection is determined using + * {@link #setGranularity(int)}. e.g. {@link HandwritingGesture#GRANULARITY_WORD} includes + * all the words with their width/height center included in the selection rectangle. + * @param area {@link RectF} (in screen coordinates) for which text will be selection. + */ + @NonNull + @SuppressLint("MissingGetterMatchingBuilder") + public Builder setSelectionArea(@NonNull RectF area) { + mArea = area; + return this; + } + + /** + * Set fallback text that will be committed at current cursor position if there is no + * applicable text beneath the area of gesture. + * @param fallbackText text to set + */ + @NonNull + public Builder setFallbackText(@Nullable String fallbackText) { + mFallbackText = fallbackText; + return this; + } + + /** + * @return {@link SelectGesture} using parameters in this {@link InsertGesture.Builder}. + * @throws IllegalArgumentException if one or more positional parameters are not specified. + */ + @NonNull + public SelectGesture build() { + if (mArea == null || mArea.isEmpty()) { + throw new IllegalArgumentException("Selection area must be set."); + } + if (mGranularity <= GRANULARITY_UNDEFINED) { + throw new IllegalArgumentException("Selection granularity must be set."); + } + return new SelectGesture(mGranularity, mArea, mFallbackText); + } + } + + /** + * Used to make this class parcelable. + */ + public static final @android.annotation.NonNull Parcelable.Creator CREATOR = + new Parcelable.Creator() { + @Override + public SelectGesture createFromParcel(Parcel source) { + return new SelectGesture(source); + } + + @Override + public SelectGesture[] newArray(int size) { + return new SelectGesture[size]; + } + }; + + @Override + public int hashCode() { + return Objects.hash(mGranularity, mArea, mFallbackText); + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof SelectGesture)) return false; + + SelectGesture that = (SelectGesture) o; + + if (mGranularity != that.mGranularity) return false; + if (!Objects.equals(mFallbackText, that.mFallbackText)) return false; + return Objects.equals(mArea, that.mArea); + } + + @Override + public int describeContents() { + return 0; + } + + /** + * Used to package this object into a {@link Parcel}. + * + * @param dest The {@link Parcel} to be written. + * @param flags The flags used for parceling. + */ + @Override + public void writeToParcel(@NonNull Parcel dest, int flags) { + dest.writeString8(mFallbackText); + dest.writeInt(mGranularity); + dest.writeTypedObject(mArea, flags); + } +} diff --git a/core/java/com/android/internal/inputmethod/IRemoteInputConnection.aidl b/core/java/com/android/internal/inputmethod/IRemoteInputConnection.aidl index a7dd6f13b02de..7a219c61bafb0 100644 --- a/core/java/com/android/internal/inputmethod/IRemoteInputConnection.aidl +++ b/core/java/com/android/internal/inputmethod/IRemoteInputConnection.aidl @@ -17,11 +17,15 @@ package com.android.internal.inputmethod; import android.os.Bundle; +import android.os.ResultReceiver; import android.view.KeyEvent; import android.view.inputmethod.CompletionInfo; import android.view.inputmethod.CorrectionInfo; +import android.view.inputmethod.DeleteGesture; import android.view.inputmethod.ExtractedTextRequest; import android.view.inputmethod.InputContentInfo; +import android.view.inputmethod.InsertGesture; +import android.view.inputmethod.SelectGesture; import android.view.inputmethod.TextAttribute; import com.android.internal.infra.AndroidFuture; @@ -86,6 +90,15 @@ import com.android.internal.inputmethod.InputConnectionCommandHeader; void performPrivateCommand(in InputConnectionCommandHeader header, String action, in Bundle data); + void performHandwritingSelectGesture(in InputConnectionCommandHeader header, + in SelectGesture gesture, in ResultReceiver resultReceiver); + + void performHandwritingInsertGesture(in InputConnectionCommandHeader header, + in InsertGesture gesture, in ResultReceiver resultReceiver); + + void performHandwritingDeleteGesture(in InputConnectionCommandHeader header, + in DeleteGesture gesture, in ResultReceiver resultReceiver); + void setComposingRegion(in InputConnectionCommandHeader header, int start, int end); void setComposingRegionWithTextAttribute(in InputConnectionCommandHeader header, int start, diff --git a/core/java/com/android/internal/inputmethod/RemoteInputConnectionImpl.java b/core/java/com/android/internal/inputmethod/RemoteInputConnectionImpl.java index b63ce1b22cdbb..c65a69f057976 100644 --- a/core/java/com/android/internal/inputmethod/RemoteInputConnectionImpl.java +++ b/core/java/com/android/internal/inputmethod/RemoteInputConnectionImpl.java @@ -31,6 +31,7 @@ import android.annotation.Nullable; import android.os.Bundle; import android.os.Handler; import android.os.Looper; +import android.os.ResultReceiver; import android.os.Trace; import android.util.Log; import android.util.proto.ProtoOutputStream; @@ -39,11 +40,15 @@ import android.view.View; import android.view.ViewRootImpl; import android.view.inputmethod.CompletionInfo; import android.view.inputmethod.CorrectionInfo; +import android.view.inputmethod.DeleteGesture; import android.view.inputmethod.DumpableInputConnection; import android.view.inputmethod.ExtractedTextRequest; +import android.view.inputmethod.HandwritingGesture; import android.view.inputmethod.InputConnection; import android.view.inputmethod.InputContentInfo; import android.view.inputmethod.InputMethodManager; +import android.view.inputmethod.InsertGesture; +import android.view.inputmethod.SelectGesture; import android.view.inputmethod.TextAttribute; import android.view.inputmethod.TextSnapshot; @@ -968,6 +973,67 @@ public final class RemoteInputConnectionImpl extends IRemoteInputConnection.Stub }); } + @Dispatching(cancellable = true) + @Override + public void performHandwritingSelectGesture( + InputConnectionCommandHeader header, SelectGesture gesture, + ResultReceiver resultReceiver) { + performHandwritingGestureInternal(header, gesture, resultReceiver); + } + + @Dispatching(cancellable = true) + @Override + public void performHandwritingInsertGesture( + InputConnectionCommandHeader header, InsertGesture gesture, + ResultReceiver resultReceiver) { + performHandwritingGestureInternal(header, gesture, resultReceiver); + } + + @Dispatching(cancellable = true) + @Override + public void performHandwritingDeleteGesture( + InputConnectionCommandHeader header, DeleteGesture gesture, + ResultReceiver resultReceiver) { + performHandwritingGestureInternal(header, gesture, resultReceiver); + } + + private void performHandwritingGestureInternal( + InputConnectionCommandHeader header, T gesture, ResultReceiver resultReceiver) { + dispatchWithTracing("performHandwritingGesture", () -> { + if (header.mSessionId != mCurrentSessionId.get()) { + return; // cancelled + } + InputConnection ic = getInputConnection(); + if (ic == null || !isActive()) { + Log.w(TAG, "performHandwritingGesture on inactive InputConnection"); + return; + } + // TODO(b/210039666): implement resultReceiver + ic.performHandwritingGesture(gesture, null, null); + }); + } + + /** + * Dispatches {@link InputConnection#requestCursorUpdates(int)}. + * + *

This method is intended to be called only from {@link InputMethodManager}.

+ * @param cursorUpdateMode the mode for {@link InputConnection#requestCursorUpdates(int, int)} + * @param cursorUpdateFilter the filter for + * {@link InputConnection#requestCursorUpdates(int, int)} + * @param imeDisplayId displayId on which IME is displayed. + */ + @Dispatching(cancellable = true) + public void requestCursorUpdatesFromImm(int cursorUpdateMode, int cursorUpdateFilter, + int imeDisplayId) { + final int currentSessionId = mCurrentSessionId.get(); + dispatchWithTracing("requestCursorUpdatesFromImm", () -> { + if (currentSessionId != mCurrentSessionId.get()) { + return; // cancelled + } + requestCursorUpdatesInternal(cursorUpdateMode, cursorUpdateFilter, imeDisplayId); + }); + } + @Dispatching(cancellable = true) @Override public void requestCursorUpdates(InputConnectionCommandHeader header, int cursorUpdateMode,