From 7f9a46a3d36ceb0520f01ac8988a31ee033479f0 Mon Sep 17 00:00:00 2001 From: Joanne Chung Date: Thu, 31 Dec 2020 18:16:52 +0800 Subject: [PATCH] Implement the behavior after receiving auto translation request. Send the TranslationRequest which collected from the UI toolkit to the TranslationService. And apply the result to the UI toolkit to show the translated text. Bug: 172969740 Bug: 177887860 Test: Verified by using a sample app and made sure the full flow work and the text was replaced with translated text. Change-Id: I1db4d1eabe5bc65d8785f13f760af54f74e46c97 --- core/java/android/app/Activity.java | 15 +- .../android/view/translation/Translator.java | 48 ++++ .../translation/UiTranslationController.java | 249 +++++++++++++++++- 3 files changed, 304 insertions(+), 8 deletions(-) diff --git a/core/java/android/app/Activity.java b/core/java/android/app/Activity.java index 415105f88f254..f366df55be4c4 100644 --- a/core/java/android/app/Activity.java +++ b/core/java/android/app/Activity.java @@ -2669,6 +2669,10 @@ public class Activity extends ContextThemeWrapper dispatchActivityDestroyed(); notifyContentCaptureManagerIfNeeded(CONTENT_CAPTURE_STOP); + + if (mUiTranslationController != null) { + mUiTranslationController.onActivityDestroyed(); + } } /** @@ -8450,9 +8454,8 @@ public class Activity extends ContextThemeWrapper } /** @hide */ - @Override @Nullable - public final View autofillClientFindViewByAutofillIdTraversal(AutofillId autofillId) { + public View findViewByAutofillIdTraversal(@NonNull AutofillId autofillId) { final ArrayList roots = WindowManagerGlobal.getInstance().getRootViews(getActivityToken()); for (int rootNum = 0; rootNum < roots.size(); rootNum++) { @@ -8465,10 +8468,16 @@ public class Activity extends ContextThemeWrapper } } } - return null; } + /** @hide */ + @Override + @Nullable + public final View autofillClientFindViewByAutofillIdTraversal(AutofillId autofillId) { + return findViewByAutofillIdTraversal(autofillId); + } + /** @hide */ @Override public final @NonNull boolean[] autofillClientGetViewVisibility( diff --git a/core/java/android/view/translation/Translator.java b/core/java/android/view/translation/Translator.java index 675f32b19d17f..22c3e57ecc957 100644 --- a/core/java/android/view/translation/Translator.java +++ b/core/java/android/view/translation/Translator.java @@ -28,6 +28,7 @@ import android.os.Bundle; import android.os.Handler; import android.os.IBinder; import android.os.RemoteException; +import android.service.translation.ITranslationCallback; import android.util.Log; import com.android.internal.annotations.GuardedBy; @@ -36,9 +37,11 @@ import com.android.internal.util.SyncResultReceiver; import java.lang.ref.WeakReference; import java.util.ArrayList; +import java.util.List; import java.util.Objects; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; /** * The {@link Translator} for translation, defined by a source and a dest {@link TranslationSpec}. @@ -295,4 +298,49 @@ public class Translator { } // TODO: add methods for UI-toolkit case. + /** @hide */ + public void requestUiTranslate(@NonNull List requests, + @NonNull Consumer responseCallback) { + if (mDirectServiceBinder == null) { + Log.wtf(TAG, "Translator created without proper initialization."); + return; + } + final android.service.translation.TranslationRequest request = + new android.service.translation.TranslationRequest + .Builder(getNextRequestId(), mSourceSpec, mDestSpec, requests) + .build(); + final ITranslationCallback callback = + new TranslationResponseCallbackImpl(responseCallback); + try { + mDirectServiceBinder.onTranslationRequest(request, mId, callback, null); + } catch (RemoteException e) { + Log.w(TAG, "RemoteException calling flushRequest"); + } + } + + private static class TranslationResponseCallbackImpl extends ITranslationCallback.Stub { + + private final WeakReference> mResponseCallback; + + TranslationResponseCallbackImpl(Consumer responseCallback) { + mResponseCallback = new WeakReference<>(responseCallback); + } + + @Override + public void onTranslationComplete(TranslationResponse response) throws RemoteException { + provideTranslationResponse(response); + } + + @Override + public void onError() throws RemoteException { + provideTranslationResponse(null); + } + + private void provideTranslationResponse(TranslationResponse response) { + final Consumer responseCallback = mResponseCallback.get(); + if (responseCallback != null) { + responseCallback.accept(response); + } + } + } } diff --git a/core/java/android/view/translation/UiTranslationController.java b/core/java/android/view/translation/UiTranslationController.java index a810c2e6fb41e..fa4614628102e 100644 --- a/core/java/android/view/translation/UiTranslationController.java +++ b/core/java/android/view/translation/UiTranslationController.java @@ -16,35 +16,274 @@ package android.view.translation; +import static android.view.translation.UiTranslationManager.STATE_UI_TRANSLATION_FINISHED; +import static android.view.translation.UiTranslationManager.STATE_UI_TRANSLATION_PAUSED; +import static android.view.translation.UiTranslationManager.STATE_UI_TRANSLATION_RESUMED; +import static android.view.translation.UiTranslationManager.STATE_UI_TRANSLATION_STARTED; + +import android.annotation.NonNull; +import android.annotation.WorkerThread; import android.app.Activity; import android.content.Context; +import android.os.Handler; +import android.os.HandlerThread; +import android.os.Process; +import android.util.ArrayMap; +import android.util.Log; +import android.util.Pair; +import android.view.View; import android.view.autofill.AutofillId; +import android.view.translation.UiTranslationManager.UiTranslationState; +import com.android.internal.util.function.pooled.PooledLambda; + +import java.lang.ref.WeakReference; +import java.util.ArrayList; import java.util.List; +import java.util.function.Consumer; /** - * A controller to manage the ui translation requests. + * A controller to manage the ui translation requests for the {@link Activity}. * * @hide */ public class UiTranslationController { private static final String TAG = "UiTranslationController"; - + @NonNull private final Activity mActivity; - + @NonNull private final Context mContext; + @NonNull + private final Object mLock = new Object(); + + // Each Translator is distinguished by sourceSpec and desSepc. + @NonNull + private final ArrayMap, Translator> mTranslators; + @NonNull + private final ArrayMap> mViews; + @NonNull + private final HandlerThread mWorkerThread; + @NonNull + private final Handler mWorkerHandler; public UiTranslationController(Activity activity, Context context) { mActivity = activity; mContext = context; + mViews = new ArrayMap<>(); + mTranslators = new ArrayMap<>(); + + mWorkerThread = + new HandlerThread("UiTranslationController_" + mActivity.getComponentName(), + Process.THREAD_PRIORITY_FOREGROUND); + mWorkerThread.start(); + mWorkerHandler = mWorkerThread.getThreadHandler(); } /** * Update the Ui translation state. */ - public void updateUiTranslationState(int state, TranslationSpec sourceSpec, + public void updateUiTranslationState(@UiTranslationState int state, TranslationSpec sourceSpec, TranslationSpec destSpec, List views) { - // Implement it. Deal with the each states + if (!mActivity.isResumed()) { + return; + } + switch (state) { + case STATE_UI_TRANSLATION_STARTED: + final Pair specs = + new Pair<>(sourceSpec, destSpec); + if (!mTranslators.containsKey(specs)) { + mWorkerHandler.sendMessage(PooledLambda.obtainMessage( + UiTranslationController::createTranslatorAndStart, + UiTranslationController.this, sourceSpec, destSpec, views)); + } else { + onUiTranslationStarted(mTranslators.get(specs), views); + } + break; + case STATE_UI_TRANSLATION_PAUSED: + runForEachView((view) -> view.onPauseUiTranslation(), STATE_UI_TRANSLATION_PAUSED); + break; + case STATE_UI_TRANSLATION_RESUMED: + runForEachView((view) -> view.onRestoreUiTranslation(), + STATE_UI_TRANSLATION_PAUSED); + break; + case STATE_UI_TRANSLATION_FINISHED: + destroyTranslators(); + runForEachView((view) -> view.onFinishUiTranslation(), STATE_UI_TRANSLATION_PAUSED); + break; + default: + Log.w(TAG, "onAutoTranslationStateChange(): unknown state: " + state); + } + } + + /** + * Called when the Activity is destroyed. + */ + public void onActivityDestroyed() { + synchronized (mLock) { + mViews.clear(); + destroyTranslators(); + mWorkerThread.quitSafely(); + } + } + + /** + * The method is used by {@link Translator}, it will be called when the translation is done. The + * translation result can be get from here. + */ + public void onTranslationCompleted(TranslationResponse response) { + if (response == null || response.getTranslationStatus() + != TranslationResponse.TRANSLATION_STATUS_SUCCESS) { + Log.w(TAG, "Fail result from TranslationService, response: " + response); + return; + } + final List translatedResult = response.getTranslations(); + onTranslationCompleted(translatedResult); + } + + private void onTranslationCompleted(List translatedResult) { + if (!mActivity.isResumed()) { + return; + } + final int resultCount = translatedResult.size(); + synchronized (mLock) { + for (int i = 0; i < resultCount; i++) { + final TranslationRequest request = translatedResult.get(i); + final AutofillId autofillId = request.getAutofillId(); + if (autofillId == null) { + continue; + } + final View view = mViews.get(autofillId).get(); + if (view == null) { + Log.w(TAG, "onTranslationCompleted: the Veiew for autofill id " + autofillId + + " may be gone."); + continue; + } + mActivity.runOnUiThread(() -> view.onTranslationComplete(request)); + } + } + } + + /** + * Called when there is an ui translation request comes to request view translation. + */ + @WorkerThread + private void createTranslatorAndStart(TranslationSpec sourceSpec, TranslationSpec destSpec, + List views) { + // Create Translator + final Translator translator = createTranslatorIfNeeded(sourceSpec, destSpec); + if (translator == null) { + Log.w(TAG, "Can not create Translator for sourceSpec:" + sourceSpec + " destSpec:" + + destSpec); + return; + } + onUiTranslationStarted(translator, views); + } + + @WorkerThread + private void sendTranslationRequest(Translator translator, + ArrayList requests) { + translator.requestUiTranslate(requests, this::onTranslationCompleted); + } + + /** + * Called when there is an ui translation request comes to request view translation. + */ + private void onUiTranslationStarted(Translator translator, List views) { + synchronized (mLock) { + if (views == null || views.size() == 0) { + throw new IllegalArgumentException("Invalid empty views: " + views); + } + // Find Views collect the translation data + // TODO(b/178084101): try to optimize, e.g. to this in a single traversal + final int viewCounts = views.size(); + final ArrayList requests = new ArrayList<>(); + for (int i = 0; i < viewCounts; i++) { + final AutofillId viewAutofillId = views.get(i); + final View view = mActivity.findViewByAutofillIdTraversal(viewAutofillId); + if (view == null) { + Log.w(TAG, "Can not find the View for autofill id= " + viewAutofillId); + continue; + } + mViews.put(viewAutofillId, new WeakReference<>(view)); + mActivity.runOnUiThread(() -> { + final TranslationRequest translationRequest = view.onCreateTranslationRequest(); + if (translationRequest != null + && translationRequest.getTranslationText().length() > 0) { + requests.add(translationRequest); + } + if (requests.size() == viewCounts) { + Log.v(TAG, "onUiTranslationStarted: send " + requests.size() + " request."); + mWorkerHandler.sendMessage(PooledLambda.obtainMessage( + UiTranslationController::sendTranslationRequest, + UiTranslationController.this, translator, requests)); + } + }); + } + } + } + + private void runForEachView(Consumer action, @UiTranslationState int state) { + synchronized (mLock) { + mActivity.runOnUiThread(() -> { + final int viewCounts = mViews.size(); + for (int i = 0; i < viewCounts; i++) { + final View view = mViews.valueAt(i).get(); + if (view == null) { + Log.w(TAG, "The View for autofill id " + mViews.keyAt(i) + + " may be gone for state " + stateToString(state)); + continue; + } + action.accept(view); + } + if (state == STATE_UI_TRANSLATION_FINISHED) { + mViews.clear(); + } + }); + } + } + + private Translator createTranslatorIfNeeded( + TranslationSpec sourceSpec, TranslationSpec destSpec) { + final TranslationManager tm = mContext.getSystemService(TranslationManager.class); + if (tm == null) { + Log.e(TAG, "Can not find TranslationManager when trying to create translator."); + return null; + } + final Translator translator = tm.createTranslator(sourceSpec, destSpec); + if (translator != null) { + final Pair specs = new Pair<>(sourceSpec, destSpec); + mTranslators.put(specs, translator); + } + return translator; + } + + private void destroyTranslators() { + synchronized (mLock) { + final int count = mTranslators.size(); + for (int i = 0; i < count; i++) { + Translator translator = mTranslators.valueAt(i); + translator.destroy(); + } + mTranslators.clear(); + } + } + + /** + * Returns a string representation of the state. + */ + public static String stateToString(@UiTranslationState int state) { + switch (state) { + case STATE_UI_TRANSLATION_STARTED: + return "UI_TRANSLATION_STARTED"; + case STATE_UI_TRANSLATION_PAUSED: + return "UI_TRANSLATION_PAUSED"; + case STATE_UI_TRANSLATION_RESUMED: + return "UI_TRANSLATION_RESUMED"; + case STATE_UI_TRANSLATION_FINISHED: + return "UI_TRANSLATION_FINISHED"; + default: + return "Unknown state (" + state + ")"; + } } }