Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Screenshot Notification Smart Action: AiAi and - Sys UI integration

Browse Source 
This change creates ScreenshotNotificationSmartActionsProvider, which is overridden
in GoogleSystemUI.
Calling AiAi is guarded by a device config flag created in cl/277143225.

Test: Manually tested the code in this CL and corresponding change in SystemUIGoogle.
Took a screenshot and verified that AiAi gets invoked and the screenshot notification
shows smart actions.
Ran new tests added in this CL 'atest ScreenshotNotificationSmartActionsTest'
Bug: 143556894

Change-Id: I439a4be9aac53fb02b566ae4d438afe3edf2b37a
This commit is contained in:
Satakshi
2019-10-29 10:57:43 -07:00
parent 3ad8b97d66
commit d2010f2628
7 changed files with 336 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
core/java/com/android/internal/config/sysui/SystemUiDeviceConfigFlags.java
View File

@@ -47,6 +47,20 @@ public final class SystemUiDeviceConfigFlags {
*/
public static final String NAS_MAX_SUGGESTIONS = "nas_max_suggestions";
// Flags related to screenshot intelligence
/**
* (bool) Whether to enable smart actions in screenshot notifications.
*/
public static final String ENABLE_SCREENSHOT_NOTIFICATION_SMART_ACTIONS =
"enable_screenshot_notification_smart_actions";
/**
* (int) Timeout value in ms to get smart actions for screenshot notification.
*/
public static final String SCREENSHOT_NOTIFICATION_SMART_ACTIONS_TIMEOUT_MS =
"screenshot_notification_smart_actions_timeout_ms";
// Flags related to controls
/**

3
packages/SystemUI/AndroidManifest.xml
View File

@@ -60,6 +60,9 @@
<uses-permission android:name="android.permission.GET_APP_OPS_STATS" />
<uses-permission android:name="android.permission.USE_RESERVED_DISK" />
<!-- to invoke ContentSuggestionsService -->
<uses-permission android:name="android.permission.MANAGE_CONTENT_SUGGESTIONS"/>
<!-- Networking and telephony -->
<uses-permission android:name="android.permission.BLUETOOTH" />
<uses-permission android:name="android.permission.BLUETOOTH_ADMIN" />

10
packages/SystemUI/src/com/android/systemui/SystemUIFactory.java
View File

@@ -33,6 +33,7 @@ import com.android.systemui.dagger.SystemUIRootComponent;
import com.android.systemui.keyguard.DismissCallbackRegistry;
import com.android.systemui.plugins.FalsingManager;
import com.android.systemui.plugins.statusbar.StatusBarStateController;
import com.android.systemui.screenshot.ScreenshotNotificationSmartActionsProvider;
import com.android.systemui.statusbar.KeyguardIndicationController;
import com.android.systemui.statusbar.NotificationMediaManager;
import com.android.systemui.statusbar.notification.NotificationWakeUpCoordinator;
@@ -124,6 +125,15 @@ public class SystemUIFactory {
return new StatusBarKeyguardViewManager(context, viewMediatorCallback, lockPatternUtils);
}
/**
* Creates an instance of ScreenshotNotificationSmartActionsProvider.
* This method is overridden in vendor specific implementation of Sys UI.
*/
public ScreenshotNotificationSmartActionsProvider
createScreenshotNotificationSmartActionsProvider() {
return new ScreenshotNotificationSmartActionsProvider();
}
public KeyguardBouncer createKeyguardBouncer(Context context, ViewMediatorCallback callback,
LockPatternUtils lockPatternUtils, ViewGroup container,
DismissCallbackRegistry dismissCallbackRegistry,

107
packages/SystemUI/src/com/android/systemui/screenshot/GlobalScreenshot.java
View File

@@ -17,6 +17,7 @@
package com.android.systemui.screenshot;
import static android.content.Context.NOTIFICATION_SERVICE;
import static android.os.AsyncTask.THREAD_POOL_EXECUTOR;
import static android.provider.DeviceConfig.NAMESPACE_SYSTEMUI;
import static android.view.WindowManager.LayoutParams.LAYOUT_IN_DISPLAY_CUTOUT_MODE_ALWAYS;
@@ -29,7 +30,9 @@ import android.animation.AnimatorSet;
import android.animation.ValueAnimator;
import android.animation.ValueAnimator.AnimatorUpdateListener;
import android.annotation.Nullable;
import android.app.ActivityManager;
import android.app.ActivityOptions;
import android.app.ActivityTaskManager;
import android.app.Notification;
import android.app.Notification.BigPictureStyle;
import android.app.NotificationManager;
@@ -42,6 +45,7 @@ import android.content.ComponentName;
import android.content.ContentResolver;
import android.content.Context;
import android.content.Intent;
import android.content.pm.UserInfo;
import android.content.res.Configuration;
import android.content.res.Resources;
import android.graphics.Bitmap;
@@ -64,7 +68,10 @@ import android.os.Looper;
import android.os.Message;
import android.os.PowerManager;
import android.os.Process;
import android.os.RemoteException;
import android.os.SystemClock;
import android.os.UserHandle;
import android.os.UserManager;
import android.provider.DeviceConfig;
import android.provider.MediaStore;
import android.text.TextUtils;
@@ -82,9 +89,12 @@ import android.view.animation.Interpolator;
import android.widget.ImageView;
import android.widget.Toast;
import com.android.internal.annotations.VisibleForTesting;
import com.android.internal.config.sysui.SystemUiDeviceConfigFlags;
import com.android.internal.messages.nano.SystemMessageProto.SystemMessage;
import com.android.systemui.R;
import com.android.systemui.SystemUI;
import com.android.systemui.SystemUIFactory;
import com.android.systemui.dagger.qualifiers.MainResources;
import com.android.systemui.shared.system.ActivityManagerWrapper;
import com.android.systemui.statusbar.phone.StatusBar;
@@ -97,8 +107,11 @@ import java.io.IOException;
import java.io.OutputStream;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
@@ -156,6 +169,8 @@ public class GlobalScreenshot {
private final BigPictureStyle mNotificationStyle;
private final int mImageWidth;
private final int mImageHeight;
private final Handler mHandler;
private final ScreenshotNotificationSmartActionsProvider mSmartActionsProvider;
SaveImageInBackgroundTask(Context context, SaveImageInBackgroundData data,
NotificationManager nManager) {
@@ -167,6 +182,11 @@ public class GlobalScreenshot {
String imageDate = new SimpleDateFormat("yyyyMMdd-HHmmss").format(new Date(mImageTime));
mImageFileName = String.format(SCREENSHOT_FILE_NAME_TEMPLATE, imageDate);
// Initialize screenshot notification smart actions provider.
mHandler = new Handler();
mSmartActionsProvider =
SystemUIFactory.getInstance().createScreenshotNotificationSmartActionsProvider();
// Create the large notification icon
mImageWidth = data.image.getWidth();
mImageHeight = data.image.getHeight();
@@ -242,6 +262,23 @@ public class GlobalScreenshot {
mNotificationStyle.bigLargeIcon((Bitmap) null);
}
private int getUserHandleOfForegroundApplication(Context context) {
// This logic matches
// com.android.systemui.statusbar.phone.PhoneStatusBarPolicy#updateManagedProfile
try {
return ActivityTaskManager.getService().getLastResumedActivityUserId();
} catch (RemoteException e) {
Slog.w(TAG, "getUserHandleOfForegroundApplication: ", e);
return context.getUserId();
}
}
private boolean isManagedProfile(Context context) {
UserManager manager = UserManager.get(context);
UserInfo info = manager.getUserInfo(getUserHandleOfForegroundApplication(context));
return info.isManagedProfile();
}
/**
* Generates a new hardware bitmap with specified values, copying the content from the
* passed in bitmap.
@@ -268,6 +305,12 @@ public class GlobalScreenshot {
Context context = mParams.context;
Bitmap image = mParams.image;
boolean smartActionsEnabled = DeviceConfig.getBoolean(DeviceConfig.NAMESPACE_SYSTEMUI,
SystemUiDeviceConfigFlags.ENABLE_SCREENSHOT_NOTIFICATION_SMART_ACTIONS, true);
CompletableFuture<List<Notification.Action>> smartActionsFuture = getSmartActionsFuture(
context, image, mSmartActionsProvider, mHandler, smartActionsEnabled,
isManagedProfile(context));
Resources r = context.getResources();
try {
@@ -378,6 +421,18 @@ public class GlobalScreenshot {
mParams.imageUri = uri;
mParams.image = null;
mParams.errorMsgResId = 0;
if (smartActionsEnabled) {
int timeoutMs = DeviceConfig.getInt(DeviceConfig.NAMESPACE_SYSTEMUI,
SystemUiDeviceConfigFlags
.SCREENSHOT_NOTIFICATION_SMART_ACTIONS_TIMEOUT_MS,
1000);
List<Notification.Action> smartActions = getSmartActions(smartActionsFuture,
timeoutMs);
for (Notification.Action action : smartActions) {
mNotificationBuilder.addAction(action);
}
}
} catch (Exception e) {
// IOException/UnsupportedOperationException may be thrown if external storage is
// not mounted
@@ -1039,6 +1094,58 @@ public class GlobalScreenshot {
nManager.notify(SystemMessage.NOTE_GLOBAL_SCREENSHOT, n);
}
@VisibleForTesting
static CompletableFuture<List<Notification.Action>> getSmartActionsFuture(Context context,
Bitmap image, ScreenshotNotificationSmartActionsProvider smartActionsProvider,
Handler handler, boolean smartActionsEnabled, boolean isManagedProfile) {
if (!smartActionsEnabled) {
Slog.i(TAG, "Screenshot Intelligence not enabled, returning empty list.");
return CompletableFuture.completedFuture(Collections.emptyList());
}
if (image.getConfig() != Bitmap.Config.HARDWARE) {
Slog.w(TAG, String.format(
"Bitmap expected: Hardware, Bitmap found: %s. Returning empty list.",
image.getConfig()));
return CompletableFuture.completedFuture(Collections.emptyList());
}
Slog.d(TAG, "Screenshot from a managed profile: " + isManagedProfile);
CompletableFuture<List<Notification.Action>> smartActionsFuture;
try {
ActivityManager.RunningTaskInfo runningTask =
ActivityManagerWrapper.getInstance().getRunningTask();
ComponentName componentName =
(runningTask != null && runningTask.topActivity != null)
? runningTask.topActivity
: new ComponentName("", "");
smartActionsFuture = smartActionsProvider.getActions(image, context,
THREAD_POOL_EXECUTOR,
handler,
componentName,
isManagedProfile);
} catch (Throwable e) {
smartActionsFuture = CompletableFuture.completedFuture(Collections.emptyList());
Slog.e(TAG, "Failed to get future for screenshot notification smart actions.", e);
}
return smartActionsFuture;
}
@VisibleForTesting
static List<Notification.Action> getSmartActions(
CompletableFuture<List<Notification.Action>> smartActionsFuture, int timeoutMs) {
try {
long startTimeMs = SystemClock.uptimeMillis();
List<Notification.Action> actions = smartActionsFuture.get(timeoutMs,
TimeUnit.MILLISECONDS);
Slog.d(TAG, String.format("Wait time for smart actions: %d ms",
SystemClock.uptimeMillis() - startTimeMs));
return actions;
} catch (Throwable e) {
Slog.e(TAG, "Failed to obtain screenshot notification smart actions.", e);
return Collections.emptyList();
}
}
/**
* Receiver to proxy the share or edit intent, used to clean up the notification and send
* appropriate signals to the system (ie. to dismiss the keyguard if necessary).

58
packages/SystemUI/src/com/android/systemui/screenshot/ScreenshotNotificationSmartActionsProvider.java Normal file
View File

@@ -0,0 +1,58 @@
/*
* Copyright (C) 2019 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.systemui.screenshot;
import android.app.Notification;
import android.content.ComponentName;
import android.content.Context;
import android.graphics.Bitmap;
import android.os.Handler;
import android.util.Log;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.Executor;
/**
* This class can be overridden by a vendor-specific sys UI implementation,
* in order to provide smart actions in the screenshot notification.
*/
public class ScreenshotNotificationSmartActionsProvider {
private static final String TAG = "ScreenshotActions";
/**
* Default implementation that returns an empty list.
* This method is overridden in vendor-specific Sys UI implementation.
*
* @param bitmap The bitmap of the screenshot. The bitmap config must be {@link
* HARDWARE}.
* @param context The current app {@link Context}.
* @param executor A {@link Executor} that can be used to execute tasks in parallel.
* @param handler A {@link Handler} to possibly run UI-thread code.
* @param componentName Contains package and activity class names where the screenshot was
* taken. This is used as an additional signal to generate and rank more
* relevant actions.
* @param isManagedProfile The screenshot was taken for a work profile app.
*/
public CompletableFuture<List<Notification.Action>> getActions(Bitmap bitmap, Context context,
Executor executor, Handler handler, ComponentName componentName,
boolean isManagedProfile) {
Log.d(TAG, "Returning empty smart action list.");
return CompletableFuture.completedFuture(Collections.emptyList());
}
}

143
packages/SystemUI/tests/src/com/android/systemui/screenshot/ScreenshotNotificationSmartActionsTest.java Normal file
View File

@@ -0,0 +1,143 @@
/*
* Copyright (C) 2019 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.systemui.screenshot;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import android.app.Notification;
import android.graphics.Bitmap;
import android.os.Handler;
import android.testing.AndroidTestingRunner;
import androidx.test.filters.SmallTest;
import com.android.systemui.SystemUIFactory;
import com.android.systemui.SysuiTestCase;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
@SmallTest
@RunWith(AndroidTestingRunner.class)
/**
* Tests for exception handling and bitmap configuration in adding smart actions to Screenshot
* Notification.
*/
public class ScreenshotNotificationSmartActionsTest extends SysuiTestCase {
private ScreenshotNotificationSmartActionsProvider mSmartActionsProvider;
private Handler mHandler;
@Before
public void setup() {
mSmartActionsProvider = mock(
ScreenshotNotificationSmartActionsProvider.class);
mHandler = mock(Handler.class);
}
// Tests any exception thrown in getting smart actions future does not affect regular
// screenshot flow.
@Test
public void testExceptionHandlingInGetSmartActionsFuture()
throws Exception {
Bitmap bitmap = mock(Bitmap.class);
when(bitmap.getConfig()).thenReturn(Bitmap.Config.HARDWARE);
ScreenshotNotificationSmartActionsProvider smartActionsProvider = mock(
ScreenshotNotificationSmartActionsProvider.class);
when(smartActionsProvider.getActions(any(), any(), any(), any(), any(),
eq(false))).thenThrow(
RuntimeException.class);
CompletableFuture<List<Notification.Action>> smartActionsFuture =
GlobalScreenshot.getSmartActionsFuture(mContext, bitmap,
smartActionsProvider, mHandler, true, false);
Assert.assertNotNull(smartActionsFuture);
List<Notification.Action> smartActions = smartActionsFuture.get(5, TimeUnit.MILLISECONDS);
Assert.assertEquals(Collections.emptyList(), smartActions);
}
// Tests any exception thrown in waiting for smart actions future to complete does
// not affect regular screenshot flow.
@Test
public void testExceptionHandlingInGetSmartActions()
throws Exception {
CompletableFuture<List<Notification.Action>> smartActionsFuture = mock(
CompletableFuture.class);
int timeoutMs = 1000;
when(smartActionsFuture.get(timeoutMs, TimeUnit.MILLISECONDS)).thenThrow(
RuntimeException.class);
List<Notification.Action> actions = GlobalScreenshot.getSmartActions(
smartActionsFuture, timeoutMs);
Assert.assertEquals(Collections.emptyList(), actions);
}
// Tests for a non-hardware bitmap, ScreenshotNotificationSmartActionsProvider is never invoked
// and a completed future is returned.
@Test
public void testUnsupportedBitmapConfiguration()
throws Exception {
Bitmap bitmap = mock(Bitmap.class);
when(bitmap.getConfig()).thenReturn(Bitmap.Config.RGB_565);
CompletableFuture<List<Notification.Action>> smartActionsFuture =
GlobalScreenshot.getSmartActionsFuture(mContext, bitmap,
mSmartActionsProvider, mHandler, true, true);
verify(mSmartActionsProvider, never()).getActions(any(), any(), any(), any(), any(),
eq(false));
Assert.assertNotNull(smartActionsFuture);
List<Notification.Action> smartActions = smartActionsFuture.get(5, TimeUnit.MILLISECONDS);
Assert.assertEquals(Collections.emptyList(), smartActions);
}
// Tests for a hardware bitmap, ScreenshotNotificationSmartActionsProvider is invoked once.
@Test
public void testScreenshotNotificationSmartActionsProviderInvokedOnce() {
Bitmap bitmap = mock(Bitmap.class);
when(bitmap.getConfig()).thenReturn(Bitmap.Config.HARDWARE);
GlobalScreenshot.getSmartActionsFuture(mContext, bitmap, mSmartActionsProvider,
mHandler, true, true);
verify(mSmartActionsProvider, times(1))
.getActions(any(), any(), any(), any(), any(), eq(true));
}
// Tests for a hardware bitmap, a completed future is returned.
@Test
public void testSupportedBitmapConfiguration()
throws Exception {
Bitmap bitmap = mock(Bitmap.class);
when(bitmap.getConfig()).thenReturn(Bitmap.Config.HARDWARE);
ScreenshotNotificationSmartActionsProvider actionsProvider =
SystemUIFactory.getInstance().createScreenshotNotificationSmartActionsProvider();
CompletableFuture<List<Notification.Action>> smartActionsFuture =
GlobalScreenshot.getSmartActionsFuture(mContext, bitmap,
actionsProvider,
mHandler, true, true);
Assert.assertNotNull(smartActionsFuture);
List<Notification.Action> smartActions = smartActionsFuture.get(5, TimeUnit.MILLISECONDS);
Assert.assertEquals(smartActions.size(), 0);
}
}

3
services/contentsuggestions/java/com/android/server/contentsuggestions/ContentSuggestionsManagerService.java
View File

@@ -16,7 +16,6 @@
package com.android.server.contentsuggestions;
import static android.Manifest.permission.BIND_CONTENT_SUGGESTIONS_SERVICE;
import static android.Manifest.permission.MANAGE_CONTENT_SUGGESTIONS;
import static android.content.pm.PackageManager.PERMISSION_GRANTED;
@@ -96,7 +95,7 @@ public class ContentSuggestionsManagerService extends
private void enforceCaller(int userId, String func) {
Context ctx = getContext();
if (ctx.checkCallingPermission(BIND_CONTENT_SUGGESTIONS_SERVICE) == PERMISSION_GRANTED
if (ctx.checkCallingPermission(MANAGE_CONTENT_SUGGESTIONS) == PERMISSION_GRANTED
|| mServiceNameResolver.isTemporary(userId)
|| mActivityTaskManagerInternal.isCallerRecents(Binder.getCallingUid())) {
return;