Add translation APIs for virtual views.

Bug: 178046780 Test: manual Test: atest CtsTranslationTestCases CTS-Coverage-Bug: 177960696 Change-Id: I99caed0d42b3a164a25c2707eba7057c42c19140
2021-03-24 22:24:37 +08:00
parent 948bd86481
commit e1b23cd423
6 changed files with 167 additions and 20 deletions
--- a/core/api/current.txt
+++ b/core/api/current.txt
@@ -48771,6 +48771,7 @@ package android.view {
    method protected int[] onCreateDrawableState(int);
    method public android.view.inputmethod.InputConnection onCreateInputConnection(android.view.inputmethod.EditorInfo);
    method @Nullable public android.view.translation.ViewTranslationRequest onCreateTranslationRequest(@NonNull int[]);
+    method public void onCreateTranslationRequests(@NonNull long[], @NonNull int[], @NonNull java.util.function.Consumer<android.view.translation.ViewTranslationRequest>);
    method @CallSuper protected void onDetachedFromWindow();
    method protected void onDisplayHint(int);
    method public boolean onDragEvent(android.view.DragEvent);
@@ -48816,6 +48817,7 @@ package android.view {
    method public boolean onTouchEvent(android.view.MotionEvent);
    method public boolean onTrackballEvent(android.view.MotionEvent);
    method public void onTranslationResponse(@NonNull android.view.translation.ViewTranslationResponse);
+    method public void onTranslationResponse(@NonNull android.util.LongSparseArray<android.view.translation.ViewTranslationResponse>);
    method @CallSuper public void onVisibilityAggregated(boolean);
    method protected void onVisibilityChanged(@NonNull android.view.View, int);
    method public void onWindowFocusChanged(boolean);
--- a/core/api/system-current.txt
+++ b/core/api/system-current.txt
@@ -14719,6 +14719,7 @@ package android.webkit {
    method public default boolean onCheckIsTextEditor();
    method public void onConfigurationChanged(android.content.res.Configuration);
    method public android.view.inputmethod.InputConnection onCreateInputConnection(android.view.inputmethod.EditorInfo);
+    method @Nullable public default void onCreateTranslationRequests(@NonNull long[], @NonNull int[], @NonNull java.util.function.Consumer<android.view.translation.ViewTranslationRequest>);
    method public void onDetachedFromWindow();
    method public boolean onDragEvent(android.view.DragEvent);
    method public void onDraw(android.graphics.Canvas);
@@ -14743,6 +14744,7 @@ package android.webkit {
    method public void onStartTemporaryDetach();
    method public boolean onTouchEvent(android.view.MotionEvent);
    method public boolean onTrackballEvent(android.view.MotionEvent);
+    method public default void onTranslationResponse(@NonNull android.util.LongSparseArray<android.view.translation.ViewTranslationResponse>);
    method public void onVisibilityChanged(android.view.View, int);
    method public void onWindowFocusChanged(boolean);
    method public void onWindowVisibilityChanged(int);
--- a/core/java/android/view/View.java
+++ b/core/java/android/view/View.java
@@ -107,6 +107,7 @@ import android.util.AttributeSet;
 import android.util.FloatProperty;
 import android.util.LayoutDirection;
 import android.util.Log;
+import android.util.LongSparseArray;
 import android.util.LongSparseLongArray;
 import android.util.Pair;
 import android.util.Pools.SynchronizedPool;
@@ -30740,6 +30741,30 @@ public class View implements Drawable.Callback, KeyEvent.Callback,
        return null;
    }

+    /**
+     * Returns a {@link ViewTranslationRequest} list which represents the content to be translated
+     * in the virtual view. This is called if this view returned a virtual view structure
+     * from {@link #onProvideContentCaptureStructure} and the system determined that those virtual
+     * views were relevant for translation.
+     *
+     * <p>The default implementation does nothing.</p>
+     *
+     * @param virtualChildIds the virtual child ids which represents the child views in the virtual
+     * view.
+     * @param supportedFormats the supported translation formats. For now, the only possible value
+     * is the {@link android.view.translation.TranslationSpec#DATA_FORMAT_TEXT}.
+     * @param requestsCollector a {@link ViewTranslationRequest} collector that will be called
+     * multiple times to collect the information to be translated in the virtual view. One
+     * {@link ViewTranslationRequest} per virtual child. The {@link ViewTranslationRequest} must
+     * contains the {@link AutofillId} corresponding to the virtualChildIds.
+     */
+    @SuppressLint("NullableCollection")
+    public void onCreateTranslationRequests(@NonNull long[] virtualChildIds,
+            @NonNull @DataFormat int[] supportedFormats,
+            @NonNull Consumer<ViewTranslationRequest> requestsCollector) {
+        // no-op
+    }
+
    /**
     * Returns a {@link ViewTranslationCallback} that is used to display/hide the translated
     * information. If the View supports displaying translated content, it should implement
@@ -30781,6 +30806,21 @@ public class View implements Drawable.Callback, KeyEvent.Callback,
        // no-op
    }

+    /**
+     * Called when the content from {@link View#onCreateTranslationRequest} had been translated by
+     * the TranslationService.
+     *
+     * <p> The default implementation does nothing.</p>
+     *
+     * @param response a {@link ViewTranslationResponse} SparseArray for the request that send by
+     * {@link View#onCreateTranslationRequests} that contains the translated information which can
+     * be shown in the view. The key of SparseArray is
+     * the virtual child ids.
+     */
+    public void onTranslationResponse(@NonNull LongSparseArray<ViewTranslationResponse> response) {
+        // no-op
+    }
+
    /**
     * Dispatch to collect the {@link ViewTranslationRequest}s for translation purpose by traversing
     * the hierarchy when the app requests ui translation. Typically, this method should only be
@@ -30788,7 +30828,8 @@ public class View implements Drawable.Callback, KeyEvent.Callback,
     * classes should override {@link View#onCreateTranslationRequest}. When requested to start the
     * ui translation, the system will call this method to traverse the view hierarchy to call
     * {@link View#onCreateTranslationRequest} to build {@link ViewTranslationRequest}s and create a
-     * {@link android.view.translation.Translator} to translate the requests.
+     * {@link android.view.translation.Translator} to translate the requests. All the
+     * {@link ViewTranslationRequest}s will be added when the traversal is done.
     *
     * <p> The default implementation will call {@link View#onCreateTranslationRequest} to build
     * {@link ViewTranslationRequest} if the view should be translated. </p>
@@ -30808,14 +30849,15 @@ public class View implements Drawable.Callback, KeyEvent.Callback,
            @NonNull List<ViewTranslationRequest> requests) {
        AutofillId autofillId = getAutofillId();
        if (viewIds.containsKey(autofillId)) {
-            ViewTranslationRequest request = null;
            if (viewIds.get(autofillId) == null) {
-                request = onCreateTranslationRequest(supportedFormats);
+                ViewTranslationRequest request = onCreateTranslationRequest(supportedFormats);
                if (request != null && request.getKeys().size() > 0) {
                    requests.add(request);
                }
            } else {
-                // TODO: handle virtual view
+                onCreateTranslationRequests(viewIds.get(autofillId), supportedFormats, request -> {
+                    requests.add(request);
+                });
            }
        }
    }
--- a/core/java/android/view/translation/UiTranslationController.java
+++ b/core/java/android/view/translation/UiTranslationController.java
@@ -31,6 +31,7 @@ import android.os.HandlerThread;
 import android.os.Process;
 import android.util.ArrayMap;
 import android.util.Log;
+import android.util.LongSparseArray;
 import android.util.Pair;
 import android.util.SparseArray;
 import android.util.SparseIntArray;
@@ -238,29 +239,90 @@ public class UiTranslationController {
        final SparseArray<ViewTranslationResponse> translatedResult =
                response.getViewTranslationResponses();
        final SparseArray<ViewTranslationResponse> viewsResult = new SparseArray<>();
-        final SparseArray<ViewTranslationResponse> virtualViewsResult = new SparseArray<>();
-        final List<AutofillId> viewIds = new ArrayList<>();
+        final SparseArray<LongSparseArray<ViewTranslationResponse>> virtualViewsResult =
+                new SparseArray<>();
+        // TODO: use another structure to prevent autoboxing?
+        final List<Integer> viewIds = new ArrayList<>();
+
        for (int i = 0; i < translatedResult.size(); i++) {
            final ViewTranslationResponse result = translatedResult.valueAt(i);
            final AutofillId autofillId = result.getAutofillId();
+            if (!viewIds.contains(autofillId.getViewId())) {
+                viewIds.add(autofillId.getViewId());
+            }
            if (autofillId.isNonVirtual()) {
                viewsResult.put(translatedResult.keyAt(i), result);
-                viewIds.add(autofillId);
            } else {
-                virtualViewsResult.put(translatedResult.keyAt(i), result);
+                final boolean isVirtualViewAdded =
+                        virtualViewsResult.indexOfKey(autofillId.getViewId()) > 0;
+                final LongSparseArray<ViewTranslationResponse> childIds =
+                        isVirtualViewAdded ? virtualViewsResult.get(autofillId.getViewId())
+                                : new LongSparseArray<>();
+                childIds.put(autofillId.getVirtualChildLongId(), result);
+                if (!isVirtualViewAdded) {
+                    virtualViewsResult.put(autofillId.getViewId(), childIds);
+                }
            }
        }
+        // Traverse tree and get views by the responsed AutofillId
+        findViewsTraversalByAutofillIds(viewIds);
+
        if (viewsResult.size() > 0) {
-            onTranslationCompleted(viewsResult, viewIds);
+            onTranslationCompleted(viewsResult);
+        }
+        if (virtualViewsResult.size() > 0) {
+            onVirtualViewTranslationCompleted(virtualViewsResult);
+        }
+    }
+
+    /**
+     * The method is used to handle the translation result for the vertual views.
+     */
+    private void onVirtualViewTranslationCompleted(
+            SparseArray<LongSparseArray<ViewTranslationResponse>> translatedResult) {
+        if (!mActivity.isResumed()) {
+            if (DEBUG) {
+                Log.v(TAG, "onTranslationCompleted: Activity is not resumed.");
+            }
+            return;
+        }
+        synchronized (mLock) {
+            if (mCurrentState == STATE_UI_TRANSLATION_FINISHED) {
+                Log.w(TAG, "onTranslationCompleted: the translation state is finished now. "
+                        + "Skip to show the translated text.");
+                return;
+            }
+            for (int i = 0; i < translatedResult.size(); i++) {
+                final AutofillId autofillId = new AutofillId(translatedResult.keyAt(i));
+                final View view = mViews.get(autofillId).get();
+                if (view == null) {
+                    Log.w(TAG, "onTranslationCompleted: the view for autofill id " + autofillId
+                            + " may be gone.");
+                    continue;
+                }
+                final LongSparseArray<ViewTranslationResponse> virtualChildResponse =
+                        translatedResult.valueAt(i);
+                mActivity.runOnUiThread(() -> {
+                    if (view.getViewTranslationCallback() == null) {
+                        if (DEBUG) {
+                            Log.d(TAG, view + " doesn't support showing translation because of "
+                                    + "null ViewTranslationCallback.");
+                        }
+                        return;
+                    }
+                    view.onTranslationResponse(virtualChildResponse);
+                    if (view.getViewTranslationCallback() != null) {
+                        view.getViewTranslationCallback().onShowTranslation(view);
+                    }
+                });
+            }
        }
-        //TODO(b/177960696): call virtual views onTranslationCompleted()
    }

    /**
     * The method is used to handle the translation result for non-vertual views.
     */
-    private void onTranslationCompleted(SparseArray<ViewTranslationResponse> translatedResult,
-            List<AutofillId> viewIds) {
+    private void onTranslationCompleted(SparseArray<ViewTranslationResponse> translatedResult) {
        if (!mActivity.isResumed()) {
            if (DEBUG) {
                Log.v(TAG, "onTranslationCompleted: Activity is not resumed.");
@@ -277,8 +339,6 @@ public class UiTranslationController {
                        + "Skip to show the translated text.");
                return;
            }
-            // Traverse tree and get views by the responsed AutofillId
-            findViewsTraversalByAutofillIds(viewIds);
            for (int i = 0; i < resultCount; i++) {
                final ViewTranslationResponse response = translatedResult.valueAt(i);
                final AutofillId autofillId = response.getAutofillId();
@@ -300,7 +360,9 @@ public class UiTranslationController {
                        return;
                    }
                    view.onTranslationResponse(response);
-                    view.getViewTranslationCallback().onShowTranslation(view);
+                    if (view.getViewTranslationCallback() != null) {
+                        view.getViewTranslationCallback().onShowTranslation(view);
+                    }
                });
            }
        }
@@ -359,7 +421,7 @@ public class UiTranslationController {
                        childs = new long[childCount];
                        viewIds.put(virtualViewAutofillId, childs);
                    }
-                    int end = childs.length;
+                    int end = childs.length - 1;
                    childs[end] = autofillId.getVirtualChildLongId();
                }
            }
@@ -403,7 +465,7 @@ public class UiTranslationController {
        return new int[] {TranslationSpec.DATA_FORMAT_TEXT};
    }

-    private void findViewsTraversalByAutofillIds(List<AutofillId> sourceViewIds) {
+    private void findViewsTraversalByAutofillIds(List<Integer> sourceViewIds) {
        final ArrayList<ViewRootImpl> roots =
                WindowManagerGlobal.getInstance().getRootViews(mActivity.getActivityToken());
        for (int rootNum = 0; rootNum < roots.size(); rootNum++) {
@@ -417,7 +479,7 @@ public class UiTranslationController {
    }

    private void findViewsTraversalByAutofillIds(ViewGroup viewGroup,
-            List<AutofillId> sourceViewIds) {
+            List<Integer> sourceViewIds) {
        final int childCount = viewGroup.getChildCount();
        for (int i = 0; i < childCount; ++i) {
            final View child = viewGroup.getChildAt(i);
@@ -429,9 +491,9 @@ public class UiTranslationController {
        }
    }

-    private void addViewIfNeeded(List<AutofillId> sourceViewIds, View view) {
+    private void addViewIfNeeded(List<Integer> sourceViewIds, View view) {
        final AutofillId autofillId = view.getAutofillId();
-        if (sourceViewIds.contains(autofillId)) {
+        if (sourceViewIds.contains(autofillId.getViewId()) && !mViews.containsKey(autofillId)) {
            mViews.put(autofillId, new WeakReference<>(view));
        }
    }
--- a/core/java/android/webkit/WebView.java
+++ b/core/java/android/webkit/WebView.java
@@ -45,6 +45,7 @@ import android.os.StrictMode;
 import android.print.PrintDocumentAdapter;
 import android.util.AttributeSet;
 import android.util.Log;
+import android.util.LongSparseArray;
 import android.util.SparseArray;
 import android.view.DragEvent;
 import android.view.KeyEvent;
@@ -64,6 +65,9 @@ import android.view.inputmethod.EditorInfo;
 import android.view.inputmethod.InputConnection;
 import android.view.inspector.InspectableProperty;
 import android.view.textclassifier.TextClassifier;
+import android.view.translation.TranslationSpec.DataFormat;
+import android.view.translation.ViewTranslationRequest;
+import android.view.translation.ViewTranslationResponse;
 import android.widget.AbsoluteLayout;

 import java.io.BufferedWriter;
@@ -73,6 +77,7 @@ import java.lang.annotation.RetentionPolicy;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.Executor;
+import java.util.function.Consumer;

 /**
 * A View that displays web pages.
@@ -2854,6 +2859,20 @@ public class WebView extends AbsoluteLayout
        return mProvider.getViewDelegate().isVisibleToUserForAutofill(virtualId);
    }

+    @Override
+    @Nullable
+    public void onCreateTranslationRequests(@NonNull long[] virtualChildIds,
+            @NonNull @DataFormat int[] supportedFormats,
+            @NonNull Consumer<ViewTranslationRequest> requestsCollector) {
+        mProvider.getViewDelegate().onCreateTranslationRequests(virtualChildIds, supportedFormats,
+                requestsCollector);
+    }
+
+    @Override
+    public void onTranslationResponse(@NonNull LongSparseArray<ViewTranslationResponse> response) {
+        mProvider.getViewDelegate().onTranslationResponse(response);
+    }
+
    /** @hide */
    @Override
    public void onInitializeAccessibilityNodeInfoInternal(AccessibilityNodeInfo info) {
--- a/core/java/android/webkit/WebViewProvider.java
+++ b/core/java/android/webkit/WebViewProvider.java
@@ -18,6 +18,7 @@ package android.webkit;

 import android.annotation.NonNull;
 import android.annotation.Nullable;
+import android.annotation.SuppressLint;
 import android.annotation.SystemApi;
 import android.content.Intent;
 import android.content.res.Configuration;
@@ -33,6 +34,7 @@ import android.os.Bundle;
 import android.os.Handler;
 import android.os.Message;
 import android.print.PrintDocumentAdapter;
+import android.util.LongSparseArray;
 import android.util.SparseArray;
 import android.view.DragEvent;
 import android.view.KeyEvent;
@@ -47,6 +49,9 @@ import android.view.autofill.AutofillValue;
 import android.view.inputmethod.EditorInfo;
 import android.view.inputmethod.InputConnection;
 import android.view.textclassifier.TextClassifier;
+import android.view.translation.TranslationSpec.DataFormat;
+import android.view.translation.ViewTranslationRequest;
+import android.view.translation.ViewTranslationResponse;
 import android.webkit.WebView.HitTestResult;
 import android.webkit.WebView.PictureListener;
 import android.webkit.WebView.VisualStateCallback;
@@ -56,6 +61,7 @@ import java.io.BufferedWriter;
 import java.io.File;
 import java.util.Map;
 import java.util.concurrent.Executor;
+import java.util.function.Consumer;

 /**
 * WebView backend provider interface: this interface is the abstract backend to a WebView
@@ -358,6 +364,20 @@ public interface WebViewProvider {
                @SuppressWarnings("unused") int flags) {
        }

+        @SuppressLint("NullableCollection")
+        @Nullable
+        default void onCreateTranslationRequests(
+                @NonNull @SuppressWarnings("unused") long[] virtualChildIds,
+                @NonNull @SuppressWarnings("unused") @DataFormat int[] supportedFormats,
+                @NonNull @SuppressWarnings("unused")
+                        Consumer<ViewTranslationRequest> requestsCollector) {
+        }
+
+        default void onTranslationResponse(
+                @NonNull @SuppressWarnings("unused")
+                        LongSparseArray<ViewTranslationResponse> response) {
+        }
+
        public AccessibilityNodeProvider getAccessibilityNodeProvider();

        public void onInitializeAccessibilityNodeInfo(AccessibilityNodeInfo info);