From 8f2ae0b7b1b2dcc57dab7369671b694d7123a56f Mon Sep 17 00:00:00 2001 From: Taran Singh Date: Tue, 1 Feb 2022 18:36:49 +0000 Subject: [PATCH] Minimal test IME with Stylus HW support Minimal IME with Handwriting support that can draw ink trail and commit random text after each session. Test: Manually 1. m -j HandwritingIme 2. adb install out/../HandwritingIme.apk 3. Simulate stylus with I7723be718354872509a9e85beda10acdbe25dd48 4. and enable the property in #3 with adb shell setprop true 5. Open any app with editor and swipe on EditText. 6. HW ink window with ink will appear. Bug: 217957587 Change-Id: I53808ec261bd4364797b93d57cf6a8ddfd615390 --- tests/HandwritingIme/Android.bp | 35 ++++ tests/HandwritingIme/AndroidManifest.xml | 35 ++++ tests/HandwritingIme/res/xml/ime.xml | 20 +++ .../test/handwritingime/HandwritingIme.java | 111 ++++++++++++ .../android/test/handwritingime/InkView.java | 164 ++++++++++++++++++ 5 files changed, 365 insertions(+) create mode 100644 tests/HandwritingIme/Android.bp create mode 100644 tests/HandwritingIme/AndroidManifest.xml create mode 100644 tests/HandwritingIme/res/xml/ime.xml create mode 100644 tests/HandwritingIme/src/com/google/android/test/handwritingime/HandwritingIme.java create mode 100644 tests/HandwritingIme/src/com/google/android/test/handwritingime/InkView.java diff --git a/tests/HandwritingIme/Android.bp b/tests/HandwritingIme/Android.bp new file mode 100644 index 0000000000000..1f552bf4dc6d2 --- /dev/null +++ b/tests/HandwritingIme/Android.bp @@ -0,0 +1,35 @@ +// Copyright (C) 2022 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package { + // See: http://go/android-license-faq + // A large-scale-change added 'default_applicable_licenses' to import + // all of the 'license_kinds' from "frameworks_base_license" + // to get the below license kinds: + // SPDX-license-identifier-Apache-2.0 + default_applicable_licenses: ["frameworks_base_license"], +} + +android_test { + name: "HandwritingIme", + srcs: ["src/**/*.java"], + resource_dirs: ["res"], + certificate: "platform", + platform_apis: true, + static_libs: [ + "androidx.core_core", + "androidx.appcompat_appcompat", + "com.google.android.material_material", + ], +} diff --git a/tests/HandwritingIme/AndroidManifest.xml b/tests/HandwritingIme/AndroidManifest.xml new file mode 100644 index 0000000000000..1445d95c28795 --- /dev/null +++ b/tests/HandwritingIme/AndroidManifest.xml @@ -0,0 +1,35 @@ + + + + + + + + + + + + + + + diff --git a/tests/HandwritingIme/res/xml/ime.xml b/tests/HandwritingIme/res/xml/ime.xml new file mode 100644 index 0000000000000..2e84a03894293 --- /dev/null +++ b/tests/HandwritingIme/res/xml/ime.xml @@ -0,0 +1,20 @@ + + + + + \ No newline at end of file diff --git a/tests/HandwritingIme/src/com/google/android/test/handwritingime/HandwritingIme.java b/tests/HandwritingIme/src/com/google/android/test/handwritingime/HandwritingIme.java new file mode 100644 index 0000000000000..18f962398fb84 --- /dev/null +++ b/tests/HandwritingIme/src/com/google/android/test/handwritingime/HandwritingIme.java @@ -0,0 +1,111 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.test.handwritingime; + +import android.annotation.Nullable; +import android.inputmethodservice.InputMethodService; +import android.util.Log; +import android.view.MotionEvent; +import android.view.View; +import android.view.ViewGroup; +import android.view.Window; +import android.widget.FrameLayout; +import android.widget.Toast; + +import java.util.Random; + +public class HandwritingIme extends InputMethodService { + + public static final int HEIGHT_DP = 100; + + private Window mInkWindow; + private InkView mInk; + + static final String TAG = "HandwritingIme"; + + interface HandwritingFinisher { + void finish(); + } + + interface StylusListener { + void onStylusEvent(MotionEvent me); + } + + final class StylusConsumer implements StylusListener { + @Override + public void onStylusEvent(MotionEvent me) { + HandwritingIme.this.onStylusEvent(me); + } + } + + final class HandwritingFinisherImpl implements HandwritingFinisher { + + HandwritingFinisherImpl() {} + + @Override + public void finish() { + finishStylusHandwriting(); + Log.d(TAG, "HandwritingIme called finishStylusHandwriting() "); + } + } + + private void onStylusEvent(@Nullable MotionEvent event) { + // TODO Hookup recognizer here + if (event.getAction() == MotionEvent.ACTION_UP) { + sendKeyChar((char) (56 + new Random().nextInt(66))); + } + } + + @Override + public View onCreateInputView() { + Log.d(TAG, "onCreateInputView"); + final ViewGroup view = new FrameLayout(this); + final View inner = new View(this); + final float density = getResources().getDisplayMetrics().density; + final int height = (int) (HEIGHT_DP * density); + view.setPadding(0, 0, 0, 0); + view.addView(inner, new FrameLayout.LayoutParams( + FrameLayout.LayoutParams.MATCH_PARENT, height)); + inner.setBackgroundColor(0xff0110fe); // blue + + return view; + } + + public void onPrepareStylusHandwriting() { + Log.d(TAG, "onPrepareStylusHandwriting "); + if (mInk == null) { + mInk = new InkView(this, new HandwritingFinisherImpl(), new StylusConsumer()); + } + } + + @Override + public boolean onStartStylusHandwriting() { + Log.d(TAG, "onStartStylusHandwriting "); + Toast.makeText(this, "START HW", Toast.LENGTH_SHORT).show(); + mInkWindow = getStylusHandwritingWindow(); + mInkWindow.setContentView(mInk, mInk.getLayoutParams()); + return true; + } + + @Override + public void onFinishStylusHandwriting() { + Log.d(TAG, "onFinishStylusHandwriting "); + Toast.makeText(this, "Finish HW", Toast.LENGTH_SHORT).show(); + // Free-up + ((ViewGroup) mInk.getParent()).removeView(mInk); + mInk = null; + } +} diff --git a/tests/HandwritingIme/src/com/google/android/test/handwritingime/InkView.java b/tests/HandwritingIme/src/com/google/android/test/handwritingime/InkView.java new file mode 100644 index 0000000000000..4ffdc9211f1dd --- /dev/null +++ b/tests/HandwritingIme/src/com/google/android/test/handwritingime/InkView.java @@ -0,0 +1,164 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.android.test.handwritingime; + +import android.content.Context; +import android.graphics.Canvas; +import android.graphics.Color; +import android.graphics.Insets; +import android.graphics.Paint; +import android.graphics.Path; +import android.view.MotionEvent; +import android.view.View; +import android.view.ViewGroup; +import android.view.WindowInsets; +import android.view.WindowManager; +import android.view.WindowMetrics; + +class InkView extends View { + private static final long FINISH_TIMEOUT = 2500; + private final HandwritingIme.HandwritingFinisher mHwCanceller; + private final HandwritingIme.StylusConsumer mConsumer; + private Paint mPaint; + private Path mPath; + private float mX, mY; + private static final float STYLUS_MOVE_TOLERANCE = 1; + private Runnable mFinishRunnable; + + InkView(Context context, HandwritingIme.HandwritingFinisher hwController, + HandwritingIme.StylusConsumer consumer) { + super(context); + mHwCanceller = hwController; + mConsumer = consumer; + + mPaint = new Paint(); + mPaint.setAntiAlias(true); + mPaint.setDither(true); + mPaint.setColor(Color.GREEN); + mPaint.setStyle(Paint.Style.STROKE); + mPaint.setStrokeJoin(Paint.Join.ROUND); + mPaint.setStrokeCap(Paint.Cap.ROUND); + mPaint.setStrokeWidth(14); + + mPath = new Path(); + + WindowManager wm = context.getSystemService(WindowManager.class); + WindowMetrics metrics = wm.getCurrentWindowMetrics(); + Insets insets = metrics.getWindowInsets() + .getInsetsIgnoringVisibility(WindowInsets.Type.systemBars()); + setLayoutParams(new ViewGroup.LayoutParams( + metrics.getBounds().width() - insets.left - insets.right, + metrics.getBounds().height() - insets.top - insets.bottom)); + } + + @Override + protected void onDraw(Canvas canvas) { + super.onDraw(canvas); + + canvas.drawPath(mPath, mPaint); + canvas.drawARGB(20, 255, 50, 50); + } + + private void stylusStart(float x, float y) { + mPath.moveTo(x, y); + mX = x; + mY = y; + } + + private void stylusMove(float x, float y) { + float dx = Math.abs(x - mX); + float dy = Math.abs(y - mY); + if (mPath.isEmpty()) { + stylusStart(x, y); + } + if (dx >= STYLUS_MOVE_TOLERANCE || dy >= STYLUS_MOVE_TOLERANCE) { + mPath.quadTo(mX, mY, (x + mX) / 2, (y + mY) / 2); + mX = x; + mY = y; + } + } + + private void stylusFinish() { + mPath.lineTo(mX, mY); + // TODO: support offscreen? e.g. mCanvas.drawPath(mPath, mPaint); + mPath.reset(); + mX = 0; + mY = 0; + + } + + + @Override + public boolean onTouchEvent(MotionEvent event) { + if (event.getToolType(0) == MotionEvent.TOOL_TYPE_STYLUS) { + mConsumer.onStylusEvent(event); + android.util.Log.w(HandwritingIme.TAG, "INK touch onStylusEvent " + event); + float x = event.getX(); + float y = event.getY(); + switch (event.getAction()) { + case MotionEvent.ACTION_DOWN: + cancelTimer(); + stylusStart(x, y); + invalidate(); + break; + case MotionEvent.ACTION_MOVE: + stylusMove(x, y); + invalidate(); + break; + + case MotionEvent.ACTION_UP: + scheduleTimer(); + break; + + } + return true; + } + return false; + } + + private void cancelTimer() { + if (mFinishRunnable != null) { + if (getHandler() != null) { + getHandler().removeCallbacks(mFinishRunnable); + } + mFinishRunnable = null; + } + if (getHandler() != null) { + getHandler().removeCallbacksAndMessages(null); + } + } + + private void scheduleTimer() { + cancelTimer(); + if (getHandler() != null) { + postDelayed(getFinishRunnable(), FINISH_TIMEOUT); + } + } + + private Runnable getFinishRunnable() { + mFinishRunnable = () -> { + android.util.Log.e(HandwritingIme.TAG, "Hw view timer finishHandwriting "); + mHwCanceller.finish(); + stylusFinish(); + mPath.reset(); + invalidate(); + }; + + return mFinishRunnable; + } + +}