Partially move wallpaper local color extraction to background

We move only the color extraction, not the
screenshot done with PixelCopy.

Test: treehugger
Test: perfetto trace
Test: atest WallpaperManagerTest
Bug: 268057764
Change-Id: I7c28e00834f581393e35a856fafc7282bc066dc5

core/java/android/service/wallpaper/WallpaperService.java

@@ -60,6 +60,7 @@ import android.hardware.display.DisplayManager.DisplayListener;
 import android.os.Build;
 import android.os.Bundle;
 import android.os.Handler;
+import android.os.HandlerThread;
 import android.os.IBinder;
 import android.os.Looper;
 import android.os.Message;
@@ -97,6 +98,7 @@ import android.view.WindowManagerGlobal;
 import android.window.ClientWindowFrames;
 import android.window.ScreenCapture;
 
+import com.android.internal.annotations.GuardedBy;
 import com.android.internal.annotations.VisibleForTesting;
 import com.android.internal.os.HandlerCaller;
 import com.android.internal.view.BaseIWindow;
@@ -104,9 +106,10 @@ import com.android.internal.view.BaseSurfaceHolder;
 
 import java.io.FileDescriptor;
 import java.io.PrintWriter;
-import java.util.Arrays;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Objects;
+import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Supplier;
@@ -167,11 +170,12 @@ public abstract class WallpaperService extends Service {
     private static final int MSG_REPORT_SHOWN = 10150;
     private static final int MSG_UPDATE_DIMMING = 10200;
     private static final int MSG_WALLPAPER_FLAGS_CHANGED = 10210;
-    private static final List<Float> PROHIBITED_STEPS = Arrays.asList(0f, Float.POSITIVE_INFINITY,
-            Float.NEGATIVE_INFINITY);
 
+    /** limit calls to {@link Engine#onComputeColors} to at most once per second */
     private static final int NOTIFY_COLORS_RATE_LIMIT_MS = 1000;
-    private static final int PROCESS_LOCAL_COLORS_INTERVAL_MS = 1000;
+
+    /** limit calls to {@link Engine#processLocalColorsInternal} to at most once per 2 seconds */
+    private static final int PROCESS_LOCAL_COLORS_INTERVAL_MS = 2000;
 
     private static final boolean ENABLE_WALLPAPER_DIMMING =
             SystemProperties.getBoolean("persist.debug.enable_wallpaper_dimming", true);
@@ -180,6 +184,9 @@ public abstract class WallpaperService extends Service {
 
     private final ArrayMap<IBinder, IWallpaperEngineWrapper> mActiveEngines = new ArrayMap<>();
 
+    private Handler mBackgroundHandler;
+    private HandlerThread mBackgroundThread;
+
     static final class WallpaperCommand {
         String action;
         int x;
@@ -198,14 +205,6 @@ public abstract class WallpaperService extends Service {
      */
     public class Engine {
         IWallpaperEngineWrapper mIWallpaperEngine;
-        final ArraySet<RectF> mLocalColorAreas = new ArraySet<>(4);
-        final ArraySet<RectF> mLocalColorsToAdd = new ArraySet<>(4);
-
-        // 2D matrix [x][y] to represent a page of a portion of a window
-        EngineWindowPage[] mWindowPages = new EngineWindowPage[0];
-        Bitmap mLastScreenshot;
-        int mLastWindowPage = -1;
-        private boolean mResetWindowPages;
 
         // Copies from mIWallpaperEngine.
         HandlerCaller mCaller;
@@ -266,11 +265,34 @@ public abstract class WallpaperService extends Service {
 
         final Object mLock = new Object();
         boolean mOffsetMessageEnqueued;
+
         @UnsupportedAppUsage(maxTargetSdk = Build.VERSION_CODES.P, trackingBug = 115609023)
-        float mPendingXOffset;
+        @GuardedBy("mLock")
-        float mPendingYOffset;
+        private float mPendingXOffset;
-        float mPendingXOffsetStep;
+        @GuardedBy("mLock")
-        float mPendingYOffsetStep;
+        private float mPendingYOffset;
+        @GuardedBy("mLock")
+        private float mPendingXOffsetStep;
+        @GuardedBy("mLock")
+        private float mPendingYOffsetStep;
+
+        /**
+         * local color extraction related fields. When a user calls `addLocalColorAreas`
+         */
+        @GuardedBy("mLock")
+        private final ArraySet<RectF> mLocalColorAreas = new ArraySet<>(4);
+
+        @GuardedBy("mLock")
+        private final ArraySet<RectF> mLocalColorsToAdd = new ArraySet<>(4);
+        private long mLastProcessLocalColorsTimestamp;
+        private AtomicBoolean mProcessLocalColorsPending = new AtomicBoolean(false);
+        private int mPixelCopyCount = 0;
+        // 2D matrix [x][y] to represent a page of a portion of a window
+        @GuardedBy("mLock")
+        private EngineWindowPage[] mWindowPages = new EngineWindowPage[0];
+        private Bitmap mLastScreenshot;
+        private boolean mResetWindowPages;
+
         boolean mPendingSync;
         MotionEvent mPendingMove;
         boolean mIsInAmbientMode;
@@ -279,12 +301,8 @@ public abstract class WallpaperService extends Service {
         private long mLastColorInvalidation;
         private final Runnable mNotifyColorsChanged = this::notifyColorsChanged;
 
-        // used to throttle processLocalColors
-        private long mLastProcessLocalColorsTimestamp;
-        private AtomicBoolean mProcessLocalColorsPending = new AtomicBoolean(false);
         private final Supplier<Long> mClockFunction;
         private final Handler mHandler;
-
         private Display mDisplay;
         private Context mDisplayContext;
         private int mDisplayState;
@@ -854,7 +872,7 @@ public abstract class WallpaperService extends Service {
                             + "was not established.");
                 }
                 mResetWindowPages = true;
-                processLocalColors(mPendingXOffset, mPendingXOffsetStep);
+                processLocalColors();
             } catch (RemoteException e) {
                 Log.w(TAG, "Can't notify system because wallpaper connection was lost.", e);
             }
@@ -1389,10 +1407,9 @@ public abstract class WallpaperService extends Service {
                         mIsCreating = false;
                         mSurfaceCreated = true;
                         if (redrawNeeded) {
-                            resetWindowPages();
                             mSession.finishDrawing(mWindow, null /* postDrawTransaction */,
                                                    Integer.MAX_VALUE);
-                            processLocalColors(mPendingXOffset, mPendingXOffsetStep);
+                            processLocalColors();
                         }
                         reposition();
                         reportEngineShown(shouldWaitForEngineShown());
@@ -1536,7 +1553,7 @@ public abstract class WallpaperService extends Service {
             if (!mDestroyed) {
                 mVisible = visible;
                 reportVisibility(false);
-                if (mReportedVisible) processLocalColors(mPendingXOffset, mPendingXOffsetStep);
+                if (mReportedVisible) processLocalColors();
             } else {
                 AnimationHandler.requestAnimatorsEnabled(visible, this);
             }
@@ -1639,14 +1656,14 @@ public abstract class WallpaperService extends Service {
             }
 
             // setup local color extraction data
-            processLocalColors(xOffset, xOffsetStep);
+            processLocalColors();
         }
 
         /**
          * Thread-safe util to call {@link #processLocalColorsInternal} with a minimum interval of
          * {@link #PROCESS_LOCAL_COLORS_INTERVAL_MS} between two calls.
          */
-        private void processLocalColors(float xOffset, float xOffsetStep) {
+        private void processLocalColors() {
             if (mProcessLocalColorsPending.compareAndSet(false, true)) {
                 final long now = mClockFunction.get();
                 final long timeSinceLastColorProcess = now - mLastProcessLocalColorsTimestamp;
@@ -1656,23 +1673,41 @@ public abstract class WallpaperService extends Service {
                 mHandler.postDelayed(() -> {
                     mLastProcessLocalColorsTimestamp = now + timeToWait;
                     mProcessLocalColorsPending.set(false);
-                    processLocalColorsInternal(xOffset, xOffsetStep);
+                    processLocalColorsInternal();
                 }, timeToWait);
             }
         }
 
-        private void processLocalColorsInternal(float xOffset, float xOffsetStep) {
+        /**
-            // implemented by the wallpaper
+         * Default implementation of the local color extraction.
+         * This will take a screenshot of the whole wallpaper on the main thread.
+         * Then, in a background thread, for each launcher page, for each area that needs color
+         * extraction in this page, creates a sub-bitmap and call {@link WallpaperColors#fromBitmap}
+         * to extract the colors. Every time a launcher page has been processed, call
+         * {@link #notifyLocalColorsChanged} with the color and areas of this page.
+         */
+        private void processLocalColorsInternal() {
             if (supportsLocalColorExtraction()) return;
+            float xOffset;
+            float xOffsetStep;
+            float wallpaperDimAmount;
+            int xPage;
+            int xPages;
+            Set<RectF> areas;
+            EngineWindowPage current;
+
+            synchronized (mLock) {
+                xOffset = mPendingXOffset;
+                xOffsetStep = mPendingXOffsetStep;
+                wallpaperDimAmount = mWallpaperDimAmount;
+
                 if (DEBUG) {
                     Log.d(TAG, "processLocalColors " + xOffset + " of step "
                             + xOffsetStep);
                 }
-            //below is the default implementation
                 if (xOffset % xOffsetStep > MIN_PAGE_ALLOWED_MARGIN
                         || !mSurfaceHolder.getSurface().isValid()) return;
                 int xCurrentPage;
-            int xPages;
                 if (!validStep(xOffsetStep)) {
                     if (DEBUG) {
                         Log.w(TAG, "invalid offset step " + xOffsetStep);
@@ -1696,10 +1731,8 @@ public abstract class WallpaperService extends Service {
                 float finalXOffsetStep = xOffsetStep;
                 float finalXOffset = xOffset;
 
-            Trace.beginSection("WallpaperService#processLocalColors");
                 resetWindowPages();
-            int xPage = xCurrentPage;
+                xPage = xCurrentPage;
-            EngineWindowPage current;
                 if (mWindowPages.length == 0 || (mWindowPages.length != xPages)) {
                     mWindowPages = new EngineWindowPage[xPages];
                     initWindowPages(mWindowPages, finalXOffsetStep);
@@ -1726,10 +1759,12 @@ public abstract class WallpaperService extends Service {
                     xPage = mWindowPages.length - 1;
                 }
                 current = mWindowPages[xPage];
-            updatePage(current, xPage, xPages, finalXOffsetStep);
+                areas = new HashSet<>(current.getAreas());
-            Trace.endSection();
+            }
+            updatePage(current, areas, xPage, xPages, wallpaperDimAmount);
         }
 
+        @GuardedBy("mLock")
         private void initWindowPages(EngineWindowPage[] windowPages, float step) {
             for (int i = 0; i < windowPages.length; i++) {
                 windowPages[i] = new EngineWindowPage();
@@ -1746,16 +1781,16 @@ public abstract class WallpaperService extends Service {
             }
         }
 
-        void updatePage(EngineWindowPage currentPage, int pageIndx, int numPages,
+        void updatePage(EngineWindowPage currentPage, Set<RectF> areas, int pageIndx, int numPages,
-                float xOffsetStep) {
+                float wallpaperDimAmount) {
+
             // in case the clock is zero, we start with negative time
             long current = SystemClock.elapsedRealtime() - DEFAULT_UPDATE_SCREENSHOT_DURATION;
             long lapsed = current - currentPage.getLastUpdateTime();
             // Always update the page when the last update time is <= 0
             // This is important especially when the device first boots
-            if (lapsed < DEFAULT_UPDATE_SCREENSHOT_DURATION) {
+            if (lapsed < DEFAULT_UPDATE_SCREENSHOT_DURATION) return;
-                return;
+
-            }
             Surface surface = mSurfaceHolder.getSurface();
             if (!surface.isValid()) return;
             boolean widthIsLarger = mSurfaceSize.x > mSurfaceSize.y;
@@ -1768,43 +1803,59 @@ public abstract class WallpaperService extends Service {
                 Log.e(TAG, "wrong width and height values of bitmap " + width + " " + height);
                 return;
             }
+            final String pixelCopySectionName = "WallpaperService#pixelCopy";
+            final int pixelCopyCount = mPixelCopyCount++;
+            Trace.beginAsyncSection(pixelCopySectionName, pixelCopyCount);
             Bitmap screenShot = Bitmap.createBitmap(width, height,
                     Bitmap.Config.ARGB_8888);
             final Bitmap finalScreenShot = screenShot;
-            Trace.beginSection("WallpaperService#pixelCopy");
+            try {
+                // TODO(b/274427458) check if this can be done in the background.
                 PixelCopy.request(surface, screenShot, (res) -> {
-                Trace.endSection();
+                    Trace.endAsyncSection(pixelCopySectionName, pixelCopyCount);
-                if (DEBUG) Log.d(TAG, "result of pixel copy is " + res);
+                    if (DEBUG) {
+                        Log.d(TAG, "result of pixel copy is: "
+                                + (res == PixelCopy.SUCCESS ? "SUCCESS" : "FAILURE"));
+                    }
                     if (res != PixelCopy.SUCCESS) {
                         Bitmap lastBitmap = currentPage.getBitmap();
                         // assign the last bitmap taken for now
                         currentPage.setBitmap(mLastScreenshot);
                         Bitmap lastScreenshot = mLastScreenshot;
-                    if (lastScreenshot != null && !lastScreenshot.isRecycled()
+                        if (lastScreenshot != null && !Objects.equals(lastBitmap, lastScreenshot)) {
-                            && !Objects.equals(lastBitmap, lastScreenshot)) {
+                            updatePageColors(
-                        updatePageColors(currentPage, pageIndx, numPages, xOffsetStep);
+                                    currentPage, areas, pageIndx, numPages, wallpaperDimAmount);
                         }
                     } else {
                         mLastScreenshot = finalScreenShot;
-                    // going to hold this lock for a while
                         currentPage.setBitmap(finalScreenShot);
                         currentPage.setLastUpdateTime(current);
-                    updatePageColors(currentPage, pageIndx, numPages, xOffsetStep);
+                        updatePageColors(
+                                currentPage, areas, pageIndx, numPages, wallpaperDimAmount);
+                    }
+                }, mBackgroundHandler);
+            } catch (IllegalArgumentException e) {
+                // this can potentially happen if the surface is invalidated right between the
+                // surface.isValid() check and the PixelCopy operation.
+                // in this case, stop: we'll compute colors on the next processLocalColors call.
+                Log.w(TAG, "Cancelling processLocalColors: exception caught during PixelCopy");
             }
-            }, mHandler);
-
         }
         // locked by the passed page
-        private void updatePageColors(EngineWindowPage page, int pageIndx, int numPages,
+        private void updatePageColors(EngineWindowPage page, Set<RectF> areas,
-                float xOffsetStep) {
+                int pageIndx, int numPages, float wallpaperDimAmount) {
             if (page.getBitmap() == null) return;
+            if (!mBackgroundHandler.getLooper().isCurrentThread()) {
+                throw new IllegalStateException(
+                        "ProcessLocalColors should be called from the background thread");
+            }
             Trace.beginSection("WallpaperService#updatePageColors");
             if (DEBUG) {
                 Log.d(TAG, "updatePageColorsLocked for page " + pageIndx + " with areas "
                         + page.getAreas().size() + " and bitmap size of "
                         + page.getBitmap().getWidth() + " x " + page.getBitmap().getHeight());
             }
-            for (RectF area: page.getAreas()) {
+            for (RectF area: areas) {
                 if (area == null) continue;
                 RectF subArea = generateSubRect(area, pageIndx, numPages);
                 Bitmap b = page.getBitmap();
@@ -1814,12 +1865,12 @@ public abstract class WallpaperService extends Service {
                 int height = Math.round(b.getHeight() * subArea.height());
                 Bitmap target;
                 try {
-                    target = Bitmap.createBitmap(page.getBitmap(), x, y, width, height);
+                    target = Bitmap.createBitmap(b, x, y, width, height);
                 } catch (Exception e) {
                     Log.e(TAG, "Error creating page local color bitmap", e);
                     continue;
                 }
-                WallpaperColors color = WallpaperColors.fromBitmap(target, mWallpaperDimAmount);
+                WallpaperColors color = WallpaperColors.fromBitmap(target, wallpaperDimAmount);
                 target.recycle();
                 WallpaperColors currentColor = page.getColors(area);
 
@@ -1836,12 +1887,14 @@ public abstract class WallpaperService extends Service {
                                 + " local color callback for area" + area + " for page " + pageIndx
                                 + " of " + numPages);
                     }
+                    mHandler.post(() -> {
                         try {
                             mConnection.onLocalWallpaperColorsChanged(area, color,
                                     mDisplayContext.getDisplayId());
                         } catch (RemoteException e) {
                             Log.e(TAG, "Error calling Connection.onLocalWallpaperColorsChanged", e);
                         }
+                    });
                 }
             }
             Trace.endSection();
@@ -1867,16 +1920,17 @@ public abstract class WallpaperService extends Service {
             return new RectF(left, in.top, right, in.bottom);
         }
 
+        @GuardedBy("mLock")
         private void resetWindowPages() {
             if (supportsLocalColorExtraction()) return;
             if (!mResetWindowPages) return;
             mResetWindowPages = false;
-            mLastWindowPage = -1;
             for (int i = 0; i < mWindowPages.length; i++) {
                 mWindowPages[i].setLastUpdateTime(0L);
             }
         }
 
+        @GuardedBy("mLock")
         private int getRectFPage(RectF area, float step) {
             if (!isValid(area)) return 0;
             if (!validStep(step)) return 0;
@@ -1897,12 +1951,12 @@ public abstract class WallpaperService extends Service {
             if (DEBUG) {
                 Log.d(TAG, "addLocalColorsAreas adding local color areas " + regions);
             }
-            mHandler.post(() -> {
+            mBackgroundHandler.post(() -> {
+                synchronized (mLock) {
                     mLocalColorsToAdd.addAll(regions);
-                processLocalColors(mPendingXOffset, mPendingYOffset);
+                }
+                processLocalColors();
             });
-
-
         }
 
         /**
@@ -1912,7 +1966,8 @@ public abstract class WallpaperService extends Service {
          */
         public void removeLocalColorsAreas(@NonNull List<RectF> regions) {
             if (supportsLocalColorExtraction()) return;
-            mHandler.post(() -> {
+            mBackgroundHandler.post(() -> {
+                synchronized (mLock) {
                     float step = mPendingXOffsetStep;
                     mLocalColorsToAdd.removeAll(regions);
                     mLocalColorAreas.removeAll(regions);
@@ -1924,6 +1979,7 @@ public abstract class WallpaperService extends Service {
                             mWindowPages[i].removeArea(regions.get(j));
                         }
                     }
+                }
             });
         }
 
@@ -1939,7 +1995,7 @@ public abstract class WallpaperService extends Service {
         }
 
         private boolean validStep(float step) {
-            return !PROHIBITED_STEPS.contains(step) && step > 0. && step <= 1.;
+            return !Float.isNaN(step) && step > 0f && step <= 1f;
         }
 
         void doCommand(WallpaperCommand cmd) {
@@ -2578,6 +2634,9 @@ public abstract class WallpaperService extends Service {
     @Override
     public void onCreate() {
         Trace.beginSection("WPMS.onCreate");
+        mBackgroundThread = new HandlerThread("DefaultWallpaperLocalColorExtractor");
+        mBackgroundThread.start();
+        mBackgroundHandler = new Handler(mBackgroundThread.getLooper());
         super.onCreate();
         Trace.endSection();
     }
@@ -2590,6 +2649,7 @@ public abstract class WallpaperService extends Service {
             engineWrapper.destroy();
         }
         mActiveEngines.clear();
+        mBackgroundThread.quitSafely();
         Trace.endSection();
     }