diff --git a/core/api/current.txt b/core/api/current.txt index 3773d3b09299f..4584b69809ba8 100644 --- a/core/api/current.txt +++ b/core/api/current.txt @@ -48771,6 +48771,7 @@ package android.view { method protected int[] onCreateDrawableState(int); method public android.view.inputmethod.InputConnection onCreateInputConnection(android.view.inputmethod.EditorInfo); method @Nullable public android.view.translation.ViewTranslationRequest onCreateTranslationRequest(@NonNull int[]); + method public void onCreateTranslationRequests(@NonNull long[], @NonNull int[], @NonNull java.util.function.Consumer); method @CallSuper protected void onDetachedFromWindow(); method protected void onDisplayHint(int); method public boolean onDragEvent(android.view.DragEvent); @@ -48816,6 +48817,7 @@ package android.view { method public boolean onTouchEvent(android.view.MotionEvent); method public boolean onTrackballEvent(android.view.MotionEvent); method public void onTranslationResponse(@NonNull android.view.translation.ViewTranslationResponse); + method public void onTranslationResponse(@NonNull android.util.LongSparseArray); method @CallSuper public void onVisibilityAggregated(boolean); method protected void onVisibilityChanged(@NonNull android.view.View, int); method public void onWindowFocusChanged(boolean); diff --git a/core/api/system-current.txt b/core/api/system-current.txt index 49b8a901bcbba..0220c02ab1a46 100644 --- a/core/api/system-current.txt +++ b/core/api/system-current.txt @@ -14719,6 +14719,7 @@ package android.webkit { method public default boolean onCheckIsTextEditor(); method public void onConfigurationChanged(android.content.res.Configuration); method public android.view.inputmethod.InputConnection onCreateInputConnection(android.view.inputmethod.EditorInfo); + method @Nullable public default void onCreateTranslationRequests(@NonNull long[], @NonNull int[], @NonNull java.util.function.Consumer); method public void onDetachedFromWindow(); method public boolean onDragEvent(android.view.DragEvent); method public void onDraw(android.graphics.Canvas); @@ -14743,6 +14744,7 @@ package android.webkit { method public void onStartTemporaryDetach(); method public boolean onTouchEvent(android.view.MotionEvent); method public boolean onTrackballEvent(android.view.MotionEvent); + method public default void onTranslationResponse(@NonNull android.util.LongSparseArray); method public void onVisibilityChanged(android.view.View, int); method public void onWindowFocusChanged(boolean); method public void onWindowVisibilityChanged(int); diff --git a/core/java/android/view/View.java b/core/java/android/view/View.java index fb528995b2f7b..615dd82fb8488 100644 --- a/core/java/android/view/View.java +++ b/core/java/android/view/View.java @@ -107,6 +107,7 @@ import android.util.AttributeSet; import android.util.FloatProperty; import android.util.LayoutDirection; import android.util.Log; +import android.util.LongSparseArray; import android.util.LongSparseLongArray; import android.util.Pair; import android.util.Pools.SynchronizedPool; @@ -30740,6 +30741,30 @@ public class View implements Drawable.Callback, KeyEvent.Callback, return null; } + /** + * Returns a {@link ViewTranslationRequest} list which represents the content to be translated + * in the virtual view. This is called if this view returned a virtual view structure + * from {@link #onProvideContentCaptureStructure} and the system determined that those virtual + * views were relevant for translation. + * + *

The default implementation does nothing.

+ * + * @param virtualChildIds the virtual child ids which represents the child views in the virtual + * view. + * @param supportedFormats the supported translation formats. For now, the only possible value + * is the {@link android.view.translation.TranslationSpec#DATA_FORMAT_TEXT}. + * @param requestsCollector a {@link ViewTranslationRequest} collector that will be called + * multiple times to collect the information to be translated in the virtual view. One + * {@link ViewTranslationRequest} per virtual child. The {@link ViewTranslationRequest} must + * contains the {@link AutofillId} corresponding to the virtualChildIds. + */ + @SuppressLint("NullableCollection") + public void onCreateTranslationRequests(@NonNull long[] virtualChildIds, + @NonNull @DataFormat int[] supportedFormats, + @NonNull Consumer requestsCollector) { + // no-op + } + /** * Returns a {@link ViewTranslationCallback} that is used to display/hide the translated * information. If the View supports displaying translated content, it should implement @@ -30781,6 +30806,21 @@ public class View implements Drawable.Callback, KeyEvent.Callback, // no-op } + /** + * Called when the content from {@link View#onCreateTranslationRequest} had been translated by + * the TranslationService. + * + *

The default implementation does nothing.

+ * + * @param response a {@link ViewTranslationResponse} SparseArray for the request that send by + * {@link View#onCreateTranslationRequests} that contains the translated information which can + * be shown in the view. The key of SparseArray is + * the virtual child ids. + */ + public void onTranslationResponse(@NonNull LongSparseArray response) { + // no-op + } + /** * Dispatch to collect the {@link ViewTranslationRequest}s for translation purpose by traversing * the hierarchy when the app requests ui translation. Typically, this method should only be @@ -30788,7 +30828,8 @@ public class View implements Drawable.Callback, KeyEvent.Callback, * classes should override {@link View#onCreateTranslationRequest}. When requested to start the * ui translation, the system will call this method to traverse the view hierarchy to call * {@link View#onCreateTranslationRequest} to build {@link ViewTranslationRequest}s and create a - * {@link android.view.translation.Translator} to translate the requests. + * {@link android.view.translation.Translator} to translate the requests. All the + * {@link ViewTranslationRequest}s will be added when the traversal is done. * *

The default implementation will call {@link View#onCreateTranslationRequest} to build * {@link ViewTranslationRequest} if the view should be translated.

@@ -30808,14 +30849,15 @@ public class View implements Drawable.Callback, KeyEvent.Callback, @NonNull List requests) { AutofillId autofillId = getAutofillId(); if (viewIds.containsKey(autofillId)) { - ViewTranslationRequest request = null; if (viewIds.get(autofillId) == null) { - request = onCreateTranslationRequest(supportedFormats); + ViewTranslationRequest request = onCreateTranslationRequest(supportedFormats); if (request != null && request.getKeys().size() > 0) { requests.add(request); } } else { - // TODO: handle virtual view + onCreateTranslationRequests(viewIds.get(autofillId), supportedFormats, request -> { + requests.add(request); + }); } } } diff --git a/core/java/android/view/translation/UiTranslationController.java b/core/java/android/view/translation/UiTranslationController.java index cfe892f8c8c91..6ba40c3cc5d9a 100644 --- a/core/java/android/view/translation/UiTranslationController.java +++ b/core/java/android/view/translation/UiTranslationController.java @@ -31,6 +31,7 @@ import android.os.HandlerThread; import android.os.Process; import android.util.ArrayMap; import android.util.Log; +import android.util.LongSparseArray; import android.util.Pair; import android.util.SparseArray; import android.util.SparseIntArray; @@ -238,29 +239,90 @@ public class UiTranslationController { final SparseArray translatedResult = response.getViewTranslationResponses(); final SparseArray viewsResult = new SparseArray<>(); - final SparseArray virtualViewsResult = new SparseArray<>(); - final List viewIds = new ArrayList<>(); + final SparseArray> virtualViewsResult = + new SparseArray<>(); + // TODO: use another structure to prevent autoboxing? + final List viewIds = new ArrayList<>(); + for (int i = 0; i < translatedResult.size(); i++) { final ViewTranslationResponse result = translatedResult.valueAt(i); final AutofillId autofillId = result.getAutofillId(); + if (!viewIds.contains(autofillId.getViewId())) { + viewIds.add(autofillId.getViewId()); + } if (autofillId.isNonVirtual()) { viewsResult.put(translatedResult.keyAt(i), result); - viewIds.add(autofillId); } else { - virtualViewsResult.put(translatedResult.keyAt(i), result); + final boolean isVirtualViewAdded = + virtualViewsResult.indexOfKey(autofillId.getViewId()) > 0; + final LongSparseArray childIds = + isVirtualViewAdded ? virtualViewsResult.get(autofillId.getViewId()) + : new LongSparseArray<>(); + childIds.put(autofillId.getVirtualChildLongId(), result); + if (!isVirtualViewAdded) { + virtualViewsResult.put(autofillId.getViewId(), childIds); + } } } + // Traverse tree and get views by the responsed AutofillId + findViewsTraversalByAutofillIds(viewIds); + if (viewsResult.size() > 0) { - onTranslationCompleted(viewsResult, viewIds); + onTranslationCompleted(viewsResult); + } + if (virtualViewsResult.size() > 0) { + onVirtualViewTranslationCompleted(virtualViewsResult); + } + } + + /** + * The method is used to handle the translation result for the vertual views. + */ + private void onVirtualViewTranslationCompleted( + SparseArray> translatedResult) { + if (!mActivity.isResumed()) { + if (DEBUG) { + Log.v(TAG, "onTranslationCompleted: Activity is not resumed."); + } + return; + } + synchronized (mLock) { + if (mCurrentState == STATE_UI_TRANSLATION_FINISHED) { + Log.w(TAG, "onTranslationCompleted: the translation state is finished now. " + + "Skip to show the translated text."); + return; + } + for (int i = 0; i < translatedResult.size(); i++) { + final AutofillId autofillId = new AutofillId(translatedResult.keyAt(i)); + final View view = mViews.get(autofillId).get(); + if (view == null) { + Log.w(TAG, "onTranslationCompleted: the view for autofill id " + autofillId + + " may be gone."); + continue; + } + final LongSparseArray virtualChildResponse = + translatedResult.valueAt(i); + mActivity.runOnUiThread(() -> { + if (view.getViewTranslationCallback() == null) { + if (DEBUG) { + Log.d(TAG, view + " doesn't support showing translation because of " + + "null ViewTranslationCallback."); + } + return; + } + view.onTranslationResponse(virtualChildResponse); + if (view.getViewTranslationCallback() != null) { + view.getViewTranslationCallback().onShowTranslation(view); + } + }); + } } - //TODO(b/177960696): call virtual views onTranslationCompleted() } /** * The method is used to handle the translation result for non-vertual views. */ - private void onTranslationCompleted(SparseArray translatedResult, - List viewIds) { + private void onTranslationCompleted(SparseArray translatedResult) { if (!mActivity.isResumed()) { if (DEBUG) { Log.v(TAG, "onTranslationCompleted: Activity is not resumed."); @@ -277,8 +339,6 @@ public class UiTranslationController { + "Skip to show the translated text."); return; } - // Traverse tree and get views by the responsed AutofillId - findViewsTraversalByAutofillIds(viewIds); for (int i = 0; i < resultCount; i++) { final ViewTranslationResponse response = translatedResult.valueAt(i); final AutofillId autofillId = response.getAutofillId(); @@ -300,7 +360,9 @@ public class UiTranslationController { return; } view.onTranslationResponse(response); - view.getViewTranslationCallback().onShowTranslation(view); + if (view.getViewTranslationCallback() != null) { + view.getViewTranslationCallback().onShowTranslation(view); + } }); } } @@ -359,7 +421,7 @@ public class UiTranslationController { childs = new long[childCount]; viewIds.put(virtualViewAutofillId, childs); } - int end = childs.length; + int end = childs.length - 1; childs[end] = autofillId.getVirtualChildLongId(); } } @@ -403,7 +465,7 @@ public class UiTranslationController { return new int[] {TranslationSpec.DATA_FORMAT_TEXT}; } - private void findViewsTraversalByAutofillIds(List sourceViewIds) { + private void findViewsTraversalByAutofillIds(List sourceViewIds) { final ArrayList roots = WindowManagerGlobal.getInstance().getRootViews(mActivity.getActivityToken()); for (int rootNum = 0; rootNum < roots.size(); rootNum++) { @@ -417,7 +479,7 @@ public class UiTranslationController { } private void findViewsTraversalByAutofillIds(ViewGroup viewGroup, - List sourceViewIds) { + List sourceViewIds) { final int childCount = viewGroup.getChildCount(); for (int i = 0; i < childCount; ++i) { final View child = viewGroup.getChildAt(i); @@ -429,9 +491,9 @@ public class UiTranslationController { } } - private void addViewIfNeeded(List sourceViewIds, View view) { + private void addViewIfNeeded(List sourceViewIds, View view) { final AutofillId autofillId = view.getAutofillId(); - if (sourceViewIds.contains(autofillId)) { + if (sourceViewIds.contains(autofillId.getViewId()) && !mViews.containsKey(autofillId)) { mViews.put(autofillId, new WeakReference<>(view)); } } diff --git a/core/java/android/webkit/WebView.java b/core/java/android/webkit/WebView.java index 487d13e40bf32..6d4fa658b3e9c 100644 --- a/core/java/android/webkit/WebView.java +++ b/core/java/android/webkit/WebView.java @@ -45,6 +45,7 @@ import android.os.StrictMode; import android.print.PrintDocumentAdapter; import android.util.AttributeSet; import android.util.Log; +import android.util.LongSparseArray; import android.util.SparseArray; import android.view.DragEvent; import android.view.KeyEvent; @@ -64,6 +65,9 @@ import android.view.inputmethod.EditorInfo; import android.view.inputmethod.InputConnection; import android.view.inspector.InspectableProperty; import android.view.textclassifier.TextClassifier; +import android.view.translation.TranslationSpec.DataFormat; +import android.view.translation.ViewTranslationRequest; +import android.view.translation.ViewTranslationResponse; import android.widget.AbsoluteLayout; import java.io.BufferedWriter; @@ -73,6 +77,7 @@ import java.lang.annotation.RetentionPolicy; import java.util.List; import java.util.Map; import java.util.concurrent.Executor; +import java.util.function.Consumer; /** * A View that displays web pages. @@ -2854,6 +2859,20 @@ public class WebView extends AbsoluteLayout return mProvider.getViewDelegate().isVisibleToUserForAutofill(virtualId); } + @Override + @Nullable + public void onCreateTranslationRequests(@NonNull long[] virtualChildIds, + @NonNull @DataFormat int[] supportedFormats, + @NonNull Consumer requestsCollector) { + mProvider.getViewDelegate().onCreateTranslationRequests(virtualChildIds, supportedFormats, + requestsCollector); + } + + @Override + public void onTranslationResponse(@NonNull LongSparseArray response) { + mProvider.getViewDelegate().onTranslationResponse(response); + } + /** @hide */ @Override public void onInitializeAccessibilityNodeInfoInternal(AccessibilityNodeInfo info) { diff --git a/core/java/android/webkit/WebViewProvider.java b/core/java/android/webkit/WebViewProvider.java index 18a110b2be0fb..264736008782d 100644 --- a/core/java/android/webkit/WebViewProvider.java +++ b/core/java/android/webkit/WebViewProvider.java @@ -18,6 +18,7 @@ package android.webkit; import android.annotation.NonNull; import android.annotation.Nullable; +import android.annotation.SuppressLint; import android.annotation.SystemApi; import android.content.Intent; import android.content.res.Configuration; @@ -33,6 +34,7 @@ import android.os.Bundle; import android.os.Handler; import android.os.Message; import android.print.PrintDocumentAdapter; +import android.util.LongSparseArray; import android.util.SparseArray; import android.view.DragEvent; import android.view.KeyEvent; @@ -47,6 +49,9 @@ import android.view.autofill.AutofillValue; import android.view.inputmethod.EditorInfo; import android.view.inputmethod.InputConnection; import android.view.textclassifier.TextClassifier; +import android.view.translation.TranslationSpec.DataFormat; +import android.view.translation.ViewTranslationRequest; +import android.view.translation.ViewTranslationResponse; import android.webkit.WebView.HitTestResult; import android.webkit.WebView.PictureListener; import android.webkit.WebView.VisualStateCallback; @@ -56,6 +61,7 @@ import java.io.BufferedWriter; import java.io.File; import java.util.Map; import java.util.concurrent.Executor; +import java.util.function.Consumer; /** * WebView backend provider interface: this interface is the abstract backend to a WebView @@ -358,6 +364,20 @@ public interface WebViewProvider { @SuppressWarnings("unused") int flags) { } + @SuppressLint("NullableCollection") + @Nullable + default void onCreateTranslationRequests( + @NonNull @SuppressWarnings("unused") long[] virtualChildIds, + @NonNull @SuppressWarnings("unused") @DataFormat int[] supportedFormats, + @NonNull @SuppressWarnings("unused") + Consumer requestsCollector) { + } + + default void onTranslationResponse( + @NonNull @SuppressWarnings("unused") + LongSparseArray response) { + } + public AccessibilityNodeProvider getAccessibilityNodeProvider(); public void onInitializeAccessibilityNodeInfo(AccessibilityNodeInfo info);