From 30704c1b853cb23564bf7f3f68baa80ddc5e91e8 Mon Sep 17 00:00:00 2001 From: Ahaan Ugale Date: Thu, 11 Mar 2021 17:51:10 -0800 Subject: [PATCH] Add UiTranslationStateCallback. This is primarily intended for Input Methods, so they can offer complementary translation experiences to the UI Translation. But it could also be useful to the applications being translated too, so it's added to UiTranslationManager. The states are intentionally different from the ones in the View API - these are about the UI Translation feature lifecycle, whereas the View states are about the display states (shown/hidden). Bug: 177500482 Test: manual - with a sample IME CTS-Coverage-Bug: 182853869 Change-Id: I8255e18c29a229c8ea3b89098b76124ee8955bdc --- core/api/current.txt | 11 ++ core/java/android/os/RemoteCallbackList.java | 18 +++ .../view/translation/ITranslationManager.aidl | 4 + .../translation/UiTranslationManager.java | 146 +++++++++++++++++- .../UiTranslationStateCallback.java | 48 ++++++ .../TranslationManagerService.java | 23 +++ .../TranslationManagerServiceImpl.java | 53 +++++++ 7 files changed, 300 insertions(+), 3 deletions(-) create mode 100644 core/java/android/view/translation/UiTranslationStateCallback.java diff --git a/core/api/current.txt b/core/api/current.txt index 4a1c325ebff78..90acb6301fe0b 100644 --- a/core/api/current.txt +++ b/core/api/current.txt @@ -52387,6 +52387,17 @@ package android.view.translation { method @Nullable @WorkerThread public android.view.translation.TranslationResponse translate(@NonNull android.view.translation.TranslationRequest); } + public final class UiTranslationManager { + method public void registerUiTranslationStateCallback(@NonNull java.util.concurrent.Executor, @NonNull android.view.translation.UiTranslationStateCallback); + method public void unregisterUiTranslationStateCallback(@NonNull android.view.translation.UiTranslationStateCallback); + } + + public interface UiTranslationStateCallback { + method public void onFinished(); + method public void onPaused(); + method public void onStarted(@NonNull String, @NonNull String); + } + public final class ViewTranslationRequest implements android.os.Parcelable { method public int describeContents(); method @NonNull public android.view.autofill.AutofillId getAutofillId(); diff --git a/core/java/android/os/RemoteCallbackList.java b/core/java/android/os/RemoteCallbackList.java index df4ade09753ba..d89c3d591d46e 100644 --- a/core/java/android/os/RemoteCallbackList.java +++ b/core/java/android/os/RemoteCallbackList.java @@ -21,6 +21,7 @@ import android.util.ArrayMap; import android.util.Slog; import java.io.PrintWriter; +import java.util.function.BiConsumer; import java.util.function.Consumer; /** @@ -353,6 +354,23 @@ public class RemoteCallbackList { } } + /** + * Performs {@code action} on each callback and associated cookie, calling {@link + * #beginBroadcast()}/{@link #finishBroadcast()} before/after looping. + * + * @hide + */ + public void broadcast(BiConsumer action) { + int itemCount = beginBroadcast(); + try { + for (int i = 0; i < itemCount; i++) { + action.accept(getBroadcastItem(i), (C) getBroadcastCookie(i)); + } + } finally { + finishBroadcast(); + } + } + /** * Returns the number of registered callbacks. Note that the number of registered * callbacks may differ from the value returned by {@link #beginBroadcast()} since diff --git a/core/java/android/view/translation/ITranslationManager.aidl b/core/java/android/view/translation/ITranslationManager.aidl index 7f6c4b474d3aa..d347f31eb9346 100644 --- a/core/java/android/view/translation/ITranslationManager.aidl +++ b/core/java/android/view/translation/ITranslationManager.aidl @@ -17,6 +17,7 @@ package android.view.translation; import android.os.IBinder; +import android.os.IRemoteCallback; import android.view.autofill.AutofillId; import android.view.translation.TranslationSpec; import com.android.internal.os.IResultReceiver; @@ -40,4 +41,7 @@ oneway interface ITranslationManager { void updateUiTranslationStateByTaskId(int state, in TranslationSpec sourceSpec, in TranslationSpec destSpec, in List viewIds, int taskId, int userId); + + void registerUiTranslationStateCallback(in IRemoteCallback callback, int userId); + void unregisterUiTranslationStateCallback(in IRemoteCallback callback, int userId); } diff --git a/core/java/android/view/translation/UiTranslationManager.java b/core/java/android/view/translation/UiTranslationManager.java index 7c73e701b7c8e..9fba95f29bdc3 100644 --- a/core/java/android/view/translation/UiTranslationManager.java +++ b/core/java/android/view/translation/UiTranslationManager.java @@ -16,28 +16,36 @@ package android.view.translation; +import android.annotation.CallbackExecutor; import android.annotation.IntDef; import android.annotation.NonNull; import android.annotation.RequiresPermission; import android.annotation.SystemApi; import android.app.assist.ActivityId; import android.content.Context; +import android.os.Binder; +import android.os.Bundle; +import android.os.IRemoteCallback; import android.os.RemoteException; +import android.util.ArrayMap; +import android.util.Log; import android.view.View; import android.view.autofill.AutofillId; +import com.android.internal.annotations.GuardedBy; + import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.util.List; +import java.util.Map; import java.util.Objects; +import java.util.concurrent.Executor; +// TODO(b/178044703): Describe what UI Translation is. /** * The {@link UiTranslationManager} class provides ways for apps to use the ui translation * function in framework. - * - * @hide */ -@SystemApi public final class UiTranslationManager { private static final String TAG = "UiTranslationManager"; @@ -88,6 +96,14 @@ public final class UiTranslationManager { public @interface UiTranslationState { } + // Keys for the data transmitted in the internal UI Translation state callback. + /** @hide */ + public static final String EXTRA_STATE = "state"; + /** @hide */ + public static final String EXTRA_SOURCE_LOCALE = "source_locale"; + /** @hide */ + public static final String EXTRA_TARGET_LOCALE = "target_locale"; + @NonNull private final Context mContext; @@ -111,9 +127,12 @@ public final class UiTranslationManager { * @param destSpec {@link TranslationSpec} for the translated data. * @param viewIds A list of the {@link View}'s {@link AutofillId} which needs to be translated * @param taskId the Activity Task id which needs ui translation + * + * @hide */ // TODO, hide the APIs @RequiresPermission(android.Manifest.permission.MANAGE_UI_TRANSLATION) + @SystemApi public void startTranslation(@NonNull TranslationSpec sourceSpec, @NonNull TranslationSpec destSpec, @NonNull List viewIds, int taskId) { @@ -141,8 +160,11 @@ public final class UiTranslationManager { * @throws IllegalArgumentException if the no {@link View}'s {@link AutofillId} in the list * @throws NullPointerException the sourceSpec, destSpec, viewIds, activityId or * {@link android.app.assist.ActivityId#getToken()} is {@code null} + * + * @hide */ @RequiresPermission(android.Manifest.permission.MANAGE_UI_TRANSLATION) + @SystemApi public void startTranslation(@NonNull TranslationSpec sourceSpec, @NonNull TranslationSpec destSpec, @NonNull List viewIds, @NonNull ActivityId activityId) { @@ -171,9 +193,12 @@ public final class UiTranslationManager { * NOTE: Please use {@code finishTranslation(ActivityId)} instead. * * @param taskId the Activity Task id which needs ui translation + * + * @hide */ // TODO, hide the APIs @RequiresPermission(android.Manifest.permission.MANAGE_UI_TRANSLATION) + @SystemApi public void finishTranslation(int taskId) { try { mService.updateUiTranslationStateByTaskId(STATE_UI_TRANSLATION_FINISHED, @@ -191,8 +216,11 @@ public final class UiTranslationManager { * @param activityId the identifier for the Activity which needs ui translation * @throws NullPointerException the activityId or * {@link android.app.assist.ActivityId#getToken()} is {@code null} + * + * @hide */ @RequiresPermission(android.Manifest.permission.MANAGE_UI_TRANSLATION) + @SystemApi public void finishTranslation(@NonNull ActivityId activityId) { try { Objects.requireNonNull(activityId); @@ -212,9 +240,12 @@ public final class UiTranslationManager { * NOTE: Please use {@code pauseTranslation(ActivityId)} instead. * * @param taskId the Activity Task id which needs ui translation + * + * @hide */ // TODO, hide the APIs @RequiresPermission(android.Manifest.permission.MANAGE_UI_TRANSLATION) + @SystemApi public void pauseTranslation(int taskId) { try { mService.updateUiTranslationStateByTaskId(STATE_UI_TRANSLATION_PAUSED, @@ -232,8 +263,11 @@ public final class UiTranslationManager { * @param activityId the identifier for the Activity which needs ui translation * @throws NullPointerException the activityId or * {@link android.app.assist.ActivityId#getToken()} is {@code null} + * + * @hide */ @RequiresPermission(android.Manifest.permission.MANAGE_UI_TRANSLATION) + @SystemApi public void pauseTranslation(@NonNull ActivityId activityId) { try { Objects.requireNonNull(activityId); @@ -253,9 +287,12 @@ public final class UiTranslationManager { * NOTE: Please use {@code resumeTranslation(ActivityId)} instead. * * @param taskId the Activity Task id which needs ui translation + * + * @hide */ // TODO, hide the APIs @RequiresPermission(android.Manifest.permission.MANAGE_UI_TRANSLATION) + @SystemApi public void resumeTranslation(int taskId) { try { mService.updateUiTranslationStateByTaskId(STATE_UI_TRANSLATION_RESUMED, @@ -273,8 +310,11 @@ public final class UiTranslationManager { * @param activityId the identifier for the Activity which needs ui translation * @throws NullPointerException the activityId or * {@link android.app.assist.ActivityId#getToken()} is {@code null} + * + * @hide */ @RequiresPermission(android.Manifest.permission.MANAGE_UI_TRANSLATION) + @SystemApi public void resumeTranslation(@NonNull ActivityId activityId) { try { Objects.requireNonNull(activityId); @@ -286,4 +326,104 @@ public final class UiTranslationManager { throw e.rethrowFromSystemServer(); } } + + // TODO(b/178044703): Fix the View API link when it becomes public. + /** + * Register for notifications of UI Translation state changes on the foreground activity. This + * is available to the owning application itself and also the current input method. + *

+ * The application whose UI is being translated can use this to customize the UI Translation + * behavior in ways that aren't made easy by methods like + * View#onCreateTranslationRequest(). + *

+ * Input methods can use this to offer complementary features to UI Translation; for example, + * enabling outgoing message translation when the system is translating incoming messages in a + * communication app. + * + * @param callback the callback to register for receiving the state change + * notifications + */ + public void registerUiTranslationStateCallback( + @NonNull @CallbackExecutor Executor executor, + @NonNull UiTranslationStateCallback callback) { + Objects.requireNonNull(executor); + Objects.requireNonNull(callback); + synchronized (mCallbacks) { + if (mCallbacks.containsKey(callback)) { + Log.w(TAG, "registerUiTranslationStateCallback: callback already registered;" + + " ignoring."); + return; + } + final IRemoteCallback remoteCallback = + new UiTranslationStateRemoteCallback(executor, callback); + try { + mService.registerUiTranslationStateCallback(remoteCallback, mContext.getUserId()); + } catch (RemoteException e) { + throw e.rethrowFromSystemServer(); + } + mCallbacks.put(callback, remoteCallback); + } + } + + /** + * Unregister {@code callback}. + * + * @see #registerUiTranslationStateCallback(Executor, UiTranslationStateCallback) + */ + public void unregisterUiTranslationStateCallback(@NonNull UiTranslationStateCallback callback) { + Objects.requireNonNull(callback); + + synchronized (mCallbacks) { + final IRemoteCallback remoteCallback = mCallbacks.get(callback); + if (remoteCallback == null) { + Log.w(TAG, "unregisterUiTranslationStateCallback: callback not found; ignoring."); + return; + } + try { + mService.unregisterUiTranslationStateCallback(remoteCallback, mContext.getUserId()); + } catch (RemoteException e) { + throw e.rethrowFromSystemServer(); + } + mCallbacks.remove(callback); + } + } + + @NonNull + @GuardedBy("mCallbacks") + private final Map mCallbacks = new ArrayMap<>(); + + private static class UiTranslationStateRemoteCallback extends IRemoteCallback.Stub { + private final Executor mExecutor; + private final UiTranslationStateCallback mCallback; + + UiTranslationStateRemoteCallback(Executor executor, + UiTranslationStateCallback callback) { + mExecutor = executor; + mCallback = callback; + } + + @Override + public void sendResult(Bundle bundle) { + Binder.clearCallingIdentity(); + mExecutor.execute(() -> { + int state = bundle.getInt(EXTRA_STATE); + switch (state) { + case STATE_UI_TRANSLATION_STARTED: + case STATE_UI_TRANSLATION_RESUMED: + mCallback.onStarted( + bundle.getString(EXTRA_SOURCE_LOCALE), + bundle.getString(EXTRA_TARGET_LOCALE)); + break; + case STATE_UI_TRANSLATION_PAUSED: + mCallback.onPaused(); + break; + case STATE_UI_TRANSLATION_FINISHED: + mCallback.onFinished(); + break; + default: + Log.wtf(TAG, "Unexpected translation state:" + state); + } + }); + } + } } diff --git a/core/java/android/view/translation/UiTranslationStateCallback.java b/core/java/android/view/translation/UiTranslationStateCallback.java new file mode 100644 index 0000000000000..1946b703935d8 --- /dev/null +++ b/core/java/android/view/translation/UiTranslationStateCallback.java @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.view.translation; + +import android.annotation.NonNull; + +import java.util.concurrent.Executor; + +/** + * Callback for listening to UI Translation state changes. See {@link + * UiTranslationManager#registerUiTranslationStateCallback(Executor, UiTranslationStateCallback)}. + */ +public interface UiTranslationStateCallback { + + /** + * The system is requesting translation of the UI from {@code sourceLocale} to {@code + * targetLocale}. + *

+ * This is also called if either the requested {@code sourceLocale} or {@code targetLocale} has + * changed; or called again after {@link #onPaused()}. + */ + void onStarted(@NonNull String sourceLocale, @NonNull String targetLocale); + + /** + * The system is requesting that the application temporarily show the UI contents in their + * original language. + */ + void onPaused(); + + /** + * The UI Translation session has ended. + */ + void onFinished(); +} diff --git a/services/translation/java/com/android/server/translation/TranslationManagerService.java b/services/translation/java/com/android/server/translation/TranslationManagerService.java index 8874e0afd7165..72e1e33491ffa 100644 --- a/services/translation/java/com/android/server/translation/TranslationManagerService.java +++ b/services/translation/java/com/android/server/translation/TranslationManagerService.java @@ -27,6 +27,7 @@ import android.content.Context; import android.content.pm.PackageManager; import android.os.Binder; import android.os.IBinder; +import android.os.IRemoteCallback; import android.os.RemoteException; import android.os.ResultReceiver; import android.os.ShellCallback; @@ -203,6 +204,28 @@ public final class TranslationManagerService } } + @Override + public void registerUiTranslationStateCallback(IRemoteCallback callback, int userId) { + TranslationManagerServiceImpl service; + synchronized (mLock) { + service = getServiceForUserLocked(userId); + } + if (service != null) { + service.registerUiTranslationStateCallback(callback, Binder.getCallingUid()); + } + } + + @Override + public void unregisterUiTranslationStateCallback(IRemoteCallback callback, int userId) { + TranslationManagerServiceImpl service; + synchronized (mLock) { + service = getServiceForUserLocked(userId); + } + if (service != null) { + service.unregisterUiTranslationStateCallback(callback); + } + } + /** * Dump the service state into the given stream. You run "adb shell dumpsys translation". */ diff --git a/services/translation/java/com/android/server/translation/TranslationManagerServiceImpl.java b/services/translation/java/com/android/server/translation/TranslationManagerServiceImpl.java index ab6ac12c90fa7..1ca07cb8d9281 100644 --- a/services/translation/java/com/android/server/translation/TranslationManagerServiceImpl.java +++ b/services/translation/java/com/android/server/translation/TranslationManagerServiceImpl.java @@ -17,17 +17,24 @@ package com.android.server.translation; import static android.view.translation.TranslationManager.STATUS_SYNC_CALL_SUCCESS; +import static android.view.translation.UiTranslationManager.EXTRA_SOURCE_LOCALE; +import static android.view.translation.UiTranslationManager.EXTRA_STATE; +import static android.view.translation.UiTranslationManager.EXTRA_TARGET_LOCALE; import android.annotation.NonNull; import android.annotation.Nullable; import android.content.ComponentName; import android.content.pm.PackageManager; import android.content.pm.ServiceInfo; +import android.os.Bundle; import android.os.IBinder; +import android.os.IRemoteCallback; +import android.os.RemoteCallbackList; import android.os.RemoteException; import android.service.translation.TranslationServiceInfo; import android.util.Slog; import android.view.autofill.AutofillId; +import android.view.inputmethod.InputMethodInfo; import android.view.translation.TranslationSpec; import android.view.translation.UiTranslationManager.UiTranslationState; @@ -36,6 +43,7 @@ import com.android.internal.os.IResultReceiver; import com.android.internal.util.SyncResultReceiver; import com.android.server.LocalServices; import com.android.server.infra.AbstractPerUserSystemService; +import com.android.server.inputmethod.InputMethodManagerInternal; import com.android.server.wm.ActivityTaskManagerInternal; import com.android.server.wm.ActivityTaskManagerInternal.ActivityTokens; @@ -174,5 +182,50 @@ final class TranslationManagerServiceImpl extends } catch (RemoteException e) { Slog.w(TAG, "Update UiTranslationState fail: " + e); } + invokeCallbacks(state, sourceSpec, destSpec); } + + private void invokeCallbacks( + int state, TranslationSpec sourceSpec, TranslationSpec targetSpec) { + Bundle res = new Bundle(); + res.putInt(EXTRA_STATE, state); + // TODO(177500482): Store the locale pair so it can be sent for RESUME events. + if (sourceSpec != null) { + res.putString(EXTRA_SOURCE_LOCALE, sourceSpec.getLanguage()); + res.putString(EXTRA_TARGET_LOCALE, targetSpec.getLanguage()); + } + // TODO(177500482): Only support the *current* Input Method. + List enabledInputMethods = + LocalServices.getService(InputMethodManagerInternal.class) + .getEnabledInputMethodListAsUser(mUserId); + mCallbacks.broadcast((callback, uid) -> { + // Code here is non-optimal since it's temporary.. + boolean isIme = false; + for (InputMethodInfo inputMethod : enabledInputMethods) { + if ((int) uid == inputMethod.getServiceInfo().applicationInfo.uid) { + isIme = true; + } + } + // TODO(177500482): Invoke it for the application being translated too. + if (!isIme) { + return; + } + try { + callback.sendResult(res); + } catch (RemoteException e) { + Slog.w(TAG, "Failed to invoke UiTranslationStateCallback: " + e); + } + }); + } + + public void registerUiTranslationStateCallback(IRemoteCallback callback, int sourceUid) { + mCallbacks.register(callback, sourceUid); + // TODO(177500482): trigger the callback here if we're already translating the UI. + } + + public void unregisterUiTranslationStateCallback(IRemoteCallback callback) { + mCallbacks.unregister(callback); + } + + private final RemoteCallbackList mCallbacks = new RemoteCallbackList<>(); }