Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement the behavior after receiving auto translation request.

Browse Source 
Send the TranslationRequest which collected from the UI toolkit to
the TranslationService. And apply the result to the UI toolkit to
show the translated text.

Bug: 172969740
Bug: 177887860
Test: Verified by using a sample app and made sure the full flow work
and the text was replaced with translated text.

Change-Id: I1db4d1eabe5bc65d8785f13f760af54f74e46c97
This commit is contained in:
Joanne Chung
2020-12-31 18:16:52 +08:00
parent 62f17faa91
commit 7f9a46a3d3
3 changed files with 304 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
core/java/android/app/Activity.java
View File

@@ -2669,6 +2669,10 @@ public class Activity extends ContextThemeWrapper
dispatchActivityDestroyed();
notifyContentCaptureManagerIfNeeded(CONTENT_CAPTURE_STOP);
if (mUiTranslationController != null) {
mUiTranslationController.onActivityDestroyed();
}
}
/**
@@ -8450,9 +8454,8 @@ public class Activity extends ContextThemeWrapper
}
/** @hide */
@Override
@Nullable
public final View autofillClientFindViewByAutofillIdTraversal(AutofillId autofillId) {
public View findViewByAutofillIdTraversal(@NonNull AutofillId autofillId) {
final ArrayList<ViewRootImpl> roots =
WindowManagerGlobal.getInstance().getRootViews(getActivityToken());
for (int rootNum = 0; rootNum < roots.size(); rootNum++) {
@@ -8465,10 +8468,16 @@ public class Activity extends ContextThemeWrapper
}
}
}
return null;
}
/** @hide */
@Override
@Nullable
public final View autofillClientFindViewByAutofillIdTraversal(AutofillId autofillId) {
return findViewByAutofillIdTraversal(autofillId);
}
/** @hide */
@Override
public final @NonNull boolean[] autofillClientGetViewVisibility(

48
core/java/android/view/translation/Translator.java
View File

@@ -28,6 +28,7 @@ import android.os.Bundle;
import android.os.Handler;
import android.os.IBinder;
import android.os.RemoteException;
import android.service.translation.ITranslationCallback;
import android.util.Log;
import com.android.internal.annotations.GuardedBy;
@@ -36,9 +37,11 @@ import com.android.internal.util.SyncResultReceiver;
import java.lang.ref.WeakReference;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
/**
* The {@link Translator} for translation, defined by a source and a dest {@link TranslationSpec}.
@@ -295,4 +298,49 @@ public class Translator {
}
// TODO: add methods for UI-toolkit case.
/** @hide */
public void requestUiTranslate(@NonNull List<TranslationRequest> requests,
@NonNull Consumer<TranslationResponse> responseCallback) {
if (mDirectServiceBinder == null) {
Log.wtf(TAG, "Translator created without proper initialization.");
return;
}
final android.service.translation.TranslationRequest request =
new android.service.translation.TranslationRequest
.Builder(getNextRequestId(), mSourceSpec, mDestSpec, requests)
.build();
final ITranslationCallback callback =
new TranslationResponseCallbackImpl(responseCallback);
try {
mDirectServiceBinder.onTranslationRequest(request, mId, callback, null);
} catch (RemoteException e) {
Log.w(TAG, "RemoteException calling flushRequest");
}
}
private static class TranslationResponseCallbackImpl extends ITranslationCallback.Stub {
private final WeakReference<Consumer<TranslationResponse>> mResponseCallback;
TranslationResponseCallbackImpl(Consumer<TranslationResponse> responseCallback) {
mResponseCallback = new WeakReference<>(responseCallback);
}
@Override
public void onTranslationComplete(TranslationResponse response) throws RemoteException {
provideTranslationResponse(response);
}
@Override
public void onError() throws RemoteException {
provideTranslationResponse(null);
}
private void provideTranslationResponse(TranslationResponse response) {
final Consumer<TranslationResponse> responseCallback = mResponseCallback.get();
if (responseCallback != null) {
responseCallback.accept(response);
}
}
}
}

249
core/java/android/view/translation/UiTranslationController.java
View File

@@ -16,35 +16,274 @@
package android.view.translation;
import static android.view.translation.UiTranslationManager.STATE_UI_TRANSLATION_FINISHED;
import static android.view.translation.UiTranslationManager.STATE_UI_TRANSLATION_PAUSED;
import static android.view.translation.UiTranslationManager.STATE_UI_TRANSLATION_RESUMED;
import static android.view.translation.UiTranslationManager.STATE_UI_TRANSLATION_STARTED;
import android.annotation.NonNull;
import android.annotation.WorkerThread;
import android.app.Activity;
import android.content.Context;
import android.os.Handler;
import android.os.HandlerThread;
import android.os.Process;
import android.util.ArrayMap;
import android.util.Log;
import android.util.Pair;
import android.view.View;
import android.view.autofill.AutofillId;
import android.view.translation.UiTranslationManager.UiTranslationState;
import com.android.internal.util.function.pooled.PooledLambda;
import java.lang.ref.WeakReference;
import java.util.ArrayList;
import java.util.List;
import java.util.function.Consumer;
/**
* A controller to manage the ui translation requests.
* A controller to manage the ui translation requests for the {@link Activity}.
*
* @hide
*/
public class UiTranslationController {
private static final String TAG = "UiTranslationController";
@NonNull
private final Activity mActivity;
@NonNull
private final Context mContext;
@NonNull
private final Object mLock = new Object();
// Each Translator is distinguished by sourceSpec and desSepc.
@NonNull
private final ArrayMap<Pair<TranslationSpec, TranslationSpec>, Translator> mTranslators;
@NonNull
private final ArrayMap<AutofillId, WeakReference<View>> mViews;
@NonNull
private final HandlerThread mWorkerThread;
@NonNull
private final Handler mWorkerHandler;
public UiTranslationController(Activity activity, Context context) {
mActivity = activity;
mContext = context;
mViews = new ArrayMap<>();
mTranslators = new ArrayMap<>();
mWorkerThread =
new HandlerThread("UiTranslationController_" + mActivity.getComponentName(),
Process.THREAD_PRIORITY_FOREGROUND);
mWorkerThread.start();
mWorkerHandler = mWorkerThread.getThreadHandler();
}
/**
* Update the Ui translation state.
*/
public void updateUiTranslationState(int state, TranslationSpec sourceSpec,
public void updateUiTranslationState(@UiTranslationState int state, TranslationSpec sourceSpec,
TranslationSpec destSpec, List<AutofillId> views) {
// Implement it. Deal with the each states
if (!mActivity.isResumed()) {
return;
}
switch (state) {
case STATE_UI_TRANSLATION_STARTED:
final Pair<TranslationSpec, TranslationSpec> specs =
new Pair<>(sourceSpec, destSpec);
if (!mTranslators.containsKey(specs)) {
mWorkerHandler.sendMessage(PooledLambda.obtainMessage(
UiTranslationController::createTranslatorAndStart,
UiTranslationController.this, sourceSpec, destSpec, views));
} else {
onUiTranslationStarted(mTranslators.get(specs), views);
}
break;
case STATE_UI_TRANSLATION_PAUSED:
runForEachView((view) -> view.onPauseUiTranslation(), STATE_UI_TRANSLATION_PAUSED);
break;
case STATE_UI_TRANSLATION_RESUMED:
runForEachView((view) -> view.onRestoreUiTranslation(),
STATE_UI_TRANSLATION_PAUSED);
break;
case STATE_UI_TRANSLATION_FINISHED:
destroyTranslators();
runForEachView((view) -> view.onFinishUiTranslation(), STATE_UI_TRANSLATION_PAUSED);
break;
default:
Log.w(TAG, "onAutoTranslationStateChange(): unknown state: " + state);
}
}
/**
* Called when the Activity is destroyed.
*/
public void onActivityDestroyed() {
synchronized (mLock) {
mViews.clear();
destroyTranslators();
mWorkerThread.quitSafely();
}
}
/**
* The method is used by {@link Translator}, it will be called when the translation is done. The
* translation result can be get from here.
*/
public void onTranslationCompleted(TranslationResponse response) {
if (response == null || response.getTranslationStatus()
!= TranslationResponse.TRANSLATION_STATUS_SUCCESS) {
Log.w(TAG, "Fail result from TranslationService, response: " + response);
return;
}
final List<TranslationRequest> translatedResult = response.getTranslations();
onTranslationCompleted(translatedResult);
}
private void onTranslationCompleted(List<TranslationRequest> translatedResult) {
if (!mActivity.isResumed()) {
return;
}
final int resultCount = translatedResult.size();
synchronized (mLock) {
for (int i = 0; i < resultCount; i++) {
final TranslationRequest request = translatedResult.get(i);
final AutofillId autofillId = request.getAutofillId();
if (autofillId == null) {
continue;
}
final View view = mViews.get(autofillId).get();
if (view == null) {
Log.w(TAG, "onTranslationCompleted: the Veiew for autofill id " + autofillId
+ " may be gone.");
continue;
}
mActivity.runOnUiThread(() -> view.onTranslationComplete(request));
}
}
}
/**
* Called when there is an ui translation request comes to request view translation.
*/
@WorkerThread
private void createTranslatorAndStart(TranslationSpec sourceSpec, TranslationSpec destSpec,
List<AutofillId> views) {
// Create Translator
final Translator translator = createTranslatorIfNeeded(sourceSpec, destSpec);
if (translator == null) {
Log.w(TAG, "Can not create Translator for sourceSpec:" + sourceSpec + " destSpec:"
+ destSpec);
return;
}
onUiTranslationStarted(translator, views);
}
@WorkerThread
private void sendTranslationRequest(Translator translator,
ArrayList<TranslationRequest> requests) {
translator.requestUiTranslate(requests, this::onTranslationCompleted);
}
/**
* Called when there is an ui translation request comes to request view translation.
*/
private void onUiTranslationStarted(Translator translator, List<AutofillId> views) {
synchronized (mLock) {
if (views == null || views.size() == 0) {
throw new IllegalArgumentException("Invalid empty views: " + views);
}
// Find Views collect the translation data
// TODO(b/178084101): try to optimize, e.g. to this in a single traversal
final int viewCounts = views.size();
final ArrayList<TranslationRequest> requests = new ArrayList<>();
for (int i = 0; i < viewCounts; i++) {
final AutofillId viewAutofillId = views.get(i);
final View view = mActivity.findViewByAutofillIdTraversal(viewAutofillId);
if (view == null) {
Log.w(TAG, "Can not find the View for autofill id= " + viewAutofillId);
continue;
}
mViews.put(viewAutofillId, new WeakReference<>(view));
mActivity.runOnUiThread(() -> {
final TranslationRequest translationRequest = view.onCreateTranslationRequest();
if (translationRequest != null
&& translationRequest.getTranslationText().length() > 0) {
requests.add(translationRequest);
}
if (requests.size() == viewCounts) {
Log.v(TAG, "onUiTranslationStarted: send " + requests.size() + " request.");
mWorkerHandler.sendMessage(PooledLambda.obtainMessage(
UiTranslationController::sendTranslationRequest,
UiTranslationController.this, translator, requests));
}
});
}
}
}
private void runForEachView(Consumer<View> action, @UiTranslationState int state) {
synchronized (mLock) {
mActivity.runOnUiThread(() -> {
final int viewCounts = mViews.size();
for (int i = 0; i < viewCounts; i++) {
final View view = mViews.valueAt(i).get();
if (view == null) {
Log.w(TAG, "The View for autofill id " + mViews.keyAt(i)
+ " may be gone for state " + stateToString(state));
continue;
}
action.accept(view);
}
if (state == STATE_UI_TRANSLATION_FINISHED) {
mViews.clear();
}
});
}
}
private Translator createTranslatorIfNeeded(
TranslationSpec sourceSpec, TranslationSpec destSpec) {
final TranslationManager tm = mContext.getSystemService(TranslationManager.class);
if (tm == null) {
Log.e(TAG, "Can not find TranslationManager when trying to create translator.");
return null;
}
final Translator translator = tm.createTranslator(sourceSpec, destSpec);
if (translator != null) {
final Pair<TranslationSpec, TranslationSpec> specs = new Pair<>(sourceSpec, destSpec);
mTranslators.put(specs, translator);
}
return translator;
}
private void destroyTranslators() {
synchronized (mLock) {
final int count = mTranslators.size();
for (int i = 0; i < count; i++) {
Translator translator = mTranslators.valueAt(i);
translator.destroy();
}
mTranslators.clear();
}
}
/**
* Returns a string representation of the state.
*/
public static String stateToString(@UiTranslationState int state) {
switch (state) {
case STATE_UI_TRANSLATION_STARTED:
return "UI_TRANSLATION_STARTED";
case STATE_UI_TRANSLATION_PAUSED:
return "UI_TRANSLATION_PAUSED";
case STATE_UI_TRANSLATION_RESUMED:
return "UI_TRANSLATION_RESUMED";
case STATE_UI_TRANSLATION_FINISHED:
return "UI_TRANSLATION_FINISHED";
default:
return "Unknown state (" + state + ")";
}
}
}