Merge "Add check for optimize after each mutation operation." into sc-dev

apex/appsearch/service/java/com/android/server/appsearch/AppSearchConfig.java

@@ -64,6 +64,12 @@ public final class AppSearchConfig implements AutoCloseable {
     static final int DEFAULT_LIMIT_CONFIG_MAX_DOCUMENT_SIZE_BYTES = 512 * 1024; // 512KiB
     @VisibleForTesting
     static final int DEFAULT_LIMIT_CONFIG_MAX_DOCUMENT_COUNT = 20_000;
+    @VisibleForTesting
+    static final int DEFAULT_BYTES_OPTIMIZE_THRESHOLD = 1 * 1024 * 1024; // 1 MiB
+    @VisibleForTesting
+    static final int DEFAULT_TIME_OPTIMIZE_THRESHOLD_MILLIS = Integer.MAX_VALUE;
+    @VisibleForTesting
+    static final int DEFAULT_DOC_COUNT_OPTIMIZE_THRESHOLD = 10_000;
 
     /*
      * Keys for ALL the flags stored in DeviceConfig.
@@ -79,6 +85,9 @@ public final class AppSearchConfig implements AutoCloseable {
             "limit_config_max_document_size_bytes";
     public static final String KEY_LIMIT_CONFIG_MAX_DOCUMENT_COUNT =
             "limit_config_max_document_docunt";
+    public static final String KEY_BYTES_OPTIMIZE_THRESHOLD = "bytes_optimize_threshold";
+    public static final String KEY_TIME_OPTIMIZE_THRESHOLD_MILLIS = "time_optimize_threshold";
+    public static final String KEY_DOC_COUNT_OPTIMIZE_THRESHOLD = "doc_count_optimize_threshold";
 
     // Array contains all the corresponding keys for the cached values.
     private static final String[] KEYS_TO_ALL_CACHED_VALUES = {
@@ -88,6 +97,9 @@ public final class AppSearchConfig implements AutoCloseable {
             KEY_SAMPLING_INTERVAL_FOR_PUT_DOCUMENT_STATS,
             KEY_LIMIT_CONFIG_MAX_DOCUMENT_SIZE_BYTES,
             KEY_LIMIT_CONFIG_MAX_DOCUMENT_COUNT,
+            KEY_BYTES_OPTIMIZE_THRESHOLD,
+            KEY_TIME_OPTIMIZE_THRESHOLD_MILLIS,
+            KEY_DOC_COUNT_OPTIMIZE_THRESHOLD
     };
 
     // Lock needed for all the operations in this class.
@@ -251,6 +263,48 @@ public final class AppSearchConfig implements AutoCloseable {
         }
     }
 
+    /**
+     * Returns the cached optimize byte size threshold.
+     *
+     * An AppSearch Optimize job will be triggered if the bytes size of garbage resource exceeds
+     * this threshold.
+     */
+    int getCachedBytesOptimizeThreshold() {
+        synchronized (mLock) {
+            throwIfClosedLocked();
+            return mBundleLocked.getInt(KEY_BYTES_OPTIMIZE_THRESHOLD,
+                    DEFAULT_BYTES_OPTIMIZE_THRESHOLD);
+        }
+    }
+
+    /**
+     * Returns the cached optimize time interval threshold.
+     *
+     * An AppSearch Optimize job will be triggered if the time since last optimize job exceeds
+     * this threshold.
+     */
+    int getCachedTimeOptimizeThresholdMs() {
+        synchronized (mLock) {
+            throwIfClosedLocked();
+            return mBundleLocked.getInt(KEY_TIME_OPTIMIZE_THRESHOLD_MILLIS,
+                    DEFAULT_TIME_OPTIMIZE_THRESHOLD_MILLIS);
+        }
+    }
+
+    /**
+     * Returns the cached optimize document count threshold threshold.
+     *
+     * An AppSearch Optimize job will be triggered if the number of document of garbage resource
+     * exceeds this threshold.
+     */
+    int getCachedDocCountOptimizeThreshold() {
+        synchronized (mLock) {
+            throwIfClosedLocked();
+            return mBundleLocked.getInt(KEY_DOC_COUNT_OPTIMIZE_THRESHOLD,
+                    DEFAULT_DOC_COUNT_OPTIMIZE_THRESHOLD);
+        }
+    }
+
     @GuardedBy("mLock")
     private void throwIfClosedLocked() {
         if (mIsClosedLocked) {
@@ -307,6 +361,24 @@ public final class AppSearchConfig implements AutoCloseable {
                             properties.getInt(key, DEFAULT_LIMIT_CONFIG_MAX_DOCUMENT_COUNT));
                 }
                 break;
+            case KEY_BYTES_OPTIMIZE_THRESHOLD:
+                synchronized (mLock) {
+                    mBundleLocked.putInt(key, properties.getInt(key,
+                            DEFAULT_BYTES_OPTIMIZE_THRESHOLD));
+                }
+                break;
+            case KEY_TIME_OPTIMIZE_THRESHOLD_MILLIS:
+                synchronized (mLock) {
+                    mBundleLocked.putInt(key, properties.getInt(key,
+                            DEFAULT_TIME_OPTIMIZE_THRESHOLD_MILLIS));
+                }
+                break;
+            case KEY_DOC_COUNT_OPTIMIZE_THRESHOLD:
+                synchronized (mLock) {
+                    mBundleLocked.putInt(key, properties.getInt(key,
+                            DEFAULT_DOC_COUNT_OPTIMIZE_THRESHOLD));
+                }
+                break;
             default:
                 break;
         }

apex/appsearch/service/java/com/android/server/appsearch/AppSearchManagerService.java

@@ -364,6 +364,12 @@ public class AppSearchManagerService extends SystemService {
                     ++operationSuccessCount;
                     invokeCallbackOnResult(callback,
                             AppSearchResult.newSuccessfulResult(setSchemaResponse.getBundle()));
+
+                    // setSchema will sync the schemas in the request to AppSearch, any existing
+                    // schemas which  is not included in the request will be delete if we force
+                    // override incompatible schemas. And all documents of these types will be
+                    // deleted as well. We should checkForOptimize for these deletion.
+                    checkForOptimize(instance);
                 } catch (Throwable t) {
                     ++operationFailureCount;
                     statusCode = throwableToFailedResult(t).getResultCode();
@@ -505,6 +511,10 @@ public class AppSearchManagerService extends SystemService {
                     // Now that the batch has been written. Persist the newly written data.
                     instance.getAppSearchImpl().persistToDisk(PersistType.Code.LITE);
                     invokeCallbackOnResult(callback, resultBuilder.build());
+
+                    // The existing documents with same ID will be deleted, so there may be some
+                    // resources that could be released after optimize().
+                    checkForOptimize(instance, /*mutateBatchSize=*/ documentBundles.size());
                 } catch (Throwable t) {
                     ++operationFailureCount;
                     statusCode = throwableToFailedResult(t).getResultCode();
@@ -1023,6 +1033,8 @@ public class AppSearchManagerService extends SystemService {
                     // Now that the batch has been written. Persist the newly written data.
                     instance.getAppSearchImpl().persistToDisk(PersistType.Code.LITE);
                     invokeCallbackOnResult(callback, resultBuilder.build());
+
+                    checkForOptimize(instance, ids.size());
                 } catch (Throwable t) {
                     ++operationFailureCount;
                     statusCode = throwableToFailedResult(t).getResultCode();
@@ -1092,6 +1104,8 @@ public class AppSearchManagerService extends SystemService {
                     instance.getAppSearchImpl().persistToDisk(PersistType.Code.LITE);
                     ++operationSuccessCount;
                     invokeCallbackOnResult(callback, AppSearchResult.newSuccessfulResult(null));
+
+                    checkForOptimize(instance);
                 } catch (Throwable t) {
                     ++operationFailureCount;
                     statusCode = throwableToFailedResult(t).getResultCode();
@@ -1472,4 +1486,24 @@ public class AppSearchManagerService extends SystemService {
             }
         }
     }
+
+    private void checkForOptimize(AppSearchUserInstance instance, int mutateBatchSize) {
+        EXECUTOR.execute(() -> {
+            try {
+                instance.getAppSearchImpl().checkForOptimize(mutateBatchSize);
+            } catch (AppSearchException e) {
+                Log.w(TAG, "Error occurred when check for optimize", e);
+            }
+        });
+    }
+
+    private void checkForOptimize(AppSearchUserInstance instance) {
+        EXECUTOR.execute(() -> {
+            try {
+                instance.getAppSearchImpl().checkForOptimize();
+            } catch (AppSearchException e) {
+                Log.w(TAG, "Error occurred when check for optimize", e);
+            }
+        });
+    }
 }

apex/appsearch/service/java/com/android/server/appsearch/AppSearchUserInstanceManager.java

@@ -27,7 +27,6 @@ import android.util.Log;
 
 import com.android.internal.annotations.GuardedBy;
 import com.android.server.appsearch.external.localstorage.AppSearchImpl;
-import com.android.server.appsearch.external.localstorage.FrameworkOptimizeStrategy;
 import com.android.server.appsearch.external.localstorage.stats.InitializeStats;
 import com.android.server.appsearch.stats.PlatformLogger;
 import com.android.server.appsearch.visibilitystore.VisibilityStoreImpl;
@@ -177,7 +176,7 @@ public final class AppSearchUserInstanceManager {
                 icingDir,
                 new FrameworkLimitConfig(config),
                 initStatsBuilder,
-                new FrameworkOptimizeStrategy());
+                new FrameworkOptimizeStrategy(config));
 
         long prepareVisibilityStoreLatencyStartMillis = SystemClock.elapsedRealtime();
         VisibilityStoreImpl visibilityStore =

apex/appsearch/service/java/com/android/server/appsearch/FrameworkOptimizeStrategy.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2021 The Android Open Source Project
+ * Copyright (C) 2021 The Android Open Source Project
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -13,14 +13,17 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package com.android.server.appsearch.external.localstorage;
+package com.android.server.appsearch;
 
 import android.annotation.NonNull;
 
-import com.android.internal.annotations.VisibleForTesting;
+import com.android.server.appsearch.external.localstorage.AppSearchImpl;
+import com.android.server.appsearch.external.localstorage.OptimizeStrategy;
 
 import com.google.android.icing.proto.GetOptimizeInfoResultProto;
 
+import java.util.Objects;
+
 /**
  * An implementation of {@link OptimizeStrategy} will determine when to trigger {@link
  * AppSearchImpl#optimize()} in Jetpack environment.
@@ -28,17 +31,18 @@ import com.google.android.icing.proto.GetOptimizeInfoResultProto;
  * @hide
  */
 public class FrameworkOptimizeStrategy implements OptimizeStrategy {
-
+    private final AppSearchConfig mAppSearchConfig;
-    @VisibleForTesting static final int DOC_COUNT_OPTIMIZE_THRESHOLD = 100_000;
+    FrameworkOptimizeStrategy(@NonNull AppSearchConfig config) {
-    @VisibleForTesting static final int BYTES_OPTIMIZE_THRESHOLD = 1 * 1024 * 1024 * 1024; // 1GB
+        mAppSearchConfig = Objects.requireNonNull(config);
-
+    }
-    @VisibleForTesting
-    static final long TIME_OPTIMIZE_THRESHOLD_MILLIS = 7 * 24 * 60 * 60 * 1000; // 1 week
 
     @Override
     public boolean shouldOptimize(@NonNull GetOptimizeInfoResultProto optimizeInfo) {
-        return optimizeInfo.getOptimizableDocs() >= DOC_COUNT_OPTIMIZE_THRESHOLD
+        return optimizeInfo.getOptimizableDocs()
-                || optimizeInfo.getEstimatedOptimizableBytes() >= BYTES_OPTIMIZE_THRESHOLD
+                    >= mAppSearchConfig.getCachedDocCountOptimizeThreshold()
-                || optimizeInfo.getTimeSinceLastOptimizeMs() >= TIME_OPTIMIZE_THRESHOLD_MILLIS;
+                || optimizeInfo.getEstimatedOptimizableBytes()
+                    >= mAppSearchConfig.getCachedBytesOptimizeThreshold()
+                || optimizeInfo.getTimeSinceLastOptimizeMs()
+                    >= mAppSearchConfig.getCachedTimeOptimizeThresholdMs();
     }
 }

services/tests/mockingservicestests/src/com/android/server/appsearch/AppSearchConfigTest.java

@@ -54,6 +54,12 @@ public class AppSearchConfigTest {
                 AppSearchConfig.DEFAULT_LIMIT_CONFIG_MAX_DOCUMENT_SIZE_BYTES);
         assertThat(appSearchConfig.getCachedLimitConfigMaxDocumentCount()).isEqualTo(
                 AppSearchConfig.DEFAULT_LIMIT_CONFIG_MAX_DOCUMENT_COUNT);
+        assertThat(appSearchConfig.getCachedBytesOptimizeThreshold()).isEqualTo(
+                AppSearchConfig.DEFAULT_BYTES_OPTIMIZE_THRESHOLD);
+        assertThat(appSearchConfig.getCachedTimeOptimizeThresholdMs()).isEqualTo(
+                AppSearchConfig.DEFAULT_TIME_OPTIMIZE_THRESHOLD_MILLIS);
+        assertThat(appSearchConfig.getCachedDocCountOptimizeThreshold()).isEqualTo(
+                AppSearchConfig.DEFAULT_DOC_COUNT_OPTIMIZE_THRESHOLD);
     }
 
     @Test
@@ -163,10 +169,8 @@ public class AppSearchConfigTest {
 
     /**
      * Tests if we fall back to {@link AppSearchConfig#DEFAULT_SAMPLING_INTERVAL} if both default
-     * sampling
+     * sampling interval and custom value are not set in DeviceConfig, and there is some other
-     * interval and custom value are not set in DeviceConfig, and there is some other sampling
+     * sampling interval set.
-     * interval
-     * set.
      */
     @Test
     public void testFallbackToDefaultSamplingValue_useHardCodedDefault() {
@@ -269,7 +273,7 @@ public class AppSearchConfigTest {
     }
 
     @Test
-    public void testCustomizedValue() {
+    public void testCustomizedValue_maxDocument() {
         DeviceConfig.setProperty(DeviceConfig.NAMESPACE_APPSEARCH,
                 AppSearchConfig.KEY_LIMIT_CONFIG_MAX_DOCUMENT_SIZE_BYTES,
                 Integer.toString(2001),
@@ -284,6 +288,64 @@ public class AppSearchConfigTest {
         assertThat(appSearchConfig.getCachedLimitConfigMaxDocumentCount()).isEqualTo(2002);
     }
 
+    @Test
+    public void testCustomizedValue_optimizeThreshold() {
+        DeviceConfig.setProperty(DeviceConfig.NAMESPACE_APPSEARCH,
+                AppSearchConfig.KEY_BYTES_OPTIMIZE_THRESHOLD,
+                Integer.toString(147147),
+                false);
+        DeviceConfig.setProperty(DeviceConfig.NAMESPACE_APPSEARCH,
+                AppSearchConfig.KEY_TIME_OPTIMIZE_THRESHOLD_MILLIS,
+                Integer.toString(258258),
+                false);
+        DeviceConfig.setProperty(DeviceConfig.NAMESPACE_APPSEARCH,
+                AppSearchConfig.KEY_DOC_COUNT_OPTIMIZE_THRESHOLD,
+                Integer.toString(369369),
+                false);
+
+        AppSearchConfig appSearchConfig = AppSearchConfig.create(DIRECT_EXECUTOR);
+
+        assertThat(appSearchConfig.getCachedBytesOptimizeThreshold()).isEqualTo(147147);
+        assertThat(appSearchConfig.getCachedTimeOptimizeThresholdMs()).isEqualTo(258258);
+        assertThat(appSearchConfig.getCachedDocCountOptimizeThreshold()).isEqualTo(369369);
+    }
+
+    @Test
+    public void testCustomizedValueOverride_optimizeThreshold() {
+        DeviceConfig.setProperty(DeviceConfig.NAMESPACE_APPSEARCH,
+                AppSearchConfig.KEY_BYTES_OPTIMIZE_THRESHOLD,
+                Integer.toString(147147),
+                false);
+        DeviceConfig.setProperty(DeviceConfig.NAMESPACE_APPSEARCH,
+                AppSearchConfig.KEY_TIME_OPTIMIZE_THRESHOLD_MILLIS,
+                Integer.toString(258258),
+                false);
+        DeviceConfig.setProperty(DeviceConfig.NAMESPACE_APPSEARCH,
+                AppSearchConfig.KEY_DOC_COUNT_OPTIMIZE_THRESHOLD,
+                Integer.toString(369369),
+                false);
+
+        AppSearchConfig appSearchConfig = AppSearchConfig.create(DIRECT_EXECUTOR);
+
+        // Override
+        DeviceConfig.setProperty(DeviceConfig.NAMESPACE_APPSEARCH,
+                AppSearchConfig.KEY_BYTES_OPTIMIZE_THRESHOLD,
+                Integer.toString(741741),
+                false);
+        DeviceConfig.setProperty(DeviceConfig.NAMESPACE_APPSEARCH,
+                AppSearchConfig.KEY_TIME_OPTIMIZE_THRESHOLD_MILLIS,
+                Integer.toString(852852),
+                false);
+        DeviceConfig.setProperty(DeviceConfig.NAMESPACE_APPSEARCH,
+                AppSearchConfig.KEY_DOC_COUNT_OPTIMIZE_THRESHOLD,
+                Integer.toString(963963),
+                false);
+
+        assertThat(appSearchConfig.getCachedBytesOptimizeThreshold()).isEqualTo(741741);
+        assertThat(appSearchConfig.getCachedTimeOptimizeThresholdMs()).isEqualTo(852852);
+        assertThat(appSearchConfig.getCachedDocCountOptimizeThreshold()).isEqualTo(963963);
+    }
+
     @Test
     public void testNotUsable_afterClose() {
         AppSearchConfig appSearchConfig = AppSearchConfig.create(DIRECT_EXECUTOR);
@@ -302,5 +364,14 @@ public class AppSearchConfigTest {
         Assert.assertThrows("Trying to use a closed AppSearchConfig instance.",
                 IllegalStateException.class,
                 () -> appSearchConfig.getCachedSamplingIntervalForPutDocumentStats());
+        Assert.assertThrows("Trying to use a closed AppSearchConfig instance.",
+                IllegalStateException.class,
+                () -> appSearchConfig.getCachedBytesOptimizeThreshold());
+        Assert.assertThrows("Trying to use a closed AppSearchConfig instance.",
+                IllegalStateException.class,
+                () -> appSearchConfig.getCachedTimeOptimizeThresholdMs());
+        Assert.assertThrows("Trying to use a closed AppSearchConfig instance.",
+                IllegalStateException.class,
+                () -> appSearchConfig.getCachedDocCountOptimizeThreshold());
     }
 }

services/tests/servicestests/src/com/android/server/appsearch/FrameworkOptimizeStrategyTest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2021 The Android Open Source Project
+ * Copyright (C) 2021 The Android Open Source Project
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -13,12 +13,9 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+package com.android.server.appsearch;
 
-package com.android.server.appsearch.external.localstorage;
+import static com.android.internal.util.ConcurrentUtils.DIRECT_EXECUTOR;
-
-import static com.android.server.appsearch.external.localstorage.FrameworkOptimizeStrategy.BYTES_OPTIMIZE_THRESHOLD;
-import static com.android.server.appsearch.external.localstorage.FrameworkOptimizeStrategy.DOC_COUNT_OPTIMIZE_THRESHOLD;
-import static com.android.server.appsearch.external.localstorage.FrameworkOptimizeStrategy.TIME_OPTIMIZE_THRESHOLD_MILLIS;
 
 import static com.google.common.truth.Truth.assertThat;
 
@@ -28,26 +25,17 @@ import com.android.server.appsearch.icing.proto.StatusProto;
 import org.junit.Test;
 
 public class FrameworkOptimizeStrategyTest {
-    FrameworkOptimizeStrategy mFrameworkOptimizeStrategy = new FrameworkOptimizeStrategy();
+    AppSearchConfig mAppSearchConfig = AppSearchConfig.create(DIRECT_EXECUTOR);
-
+    FrameworkOptimizeStrategy mFrameworkOptimizeStrategy =
-    @Test
+            new FrameworkOptimizeStrategy(mAppSearchConfig);
-    public void testShouldOptimize_docCountThreshold() {
-        GetOptimizeInfoResultProto optimizeInfo =
-                GetOptimizeInfoResultProto.newBuilder()
-                        .setTimeSinceLastOptimizeMs(0)
-                        .setEstimatedOptimizableBytes(BYTES_OPTIMIZE_THRESHOLD)
-                        .setOptimizableDocs(0)
-                        .setStatus(StatusProto.newBuilder().setCode(StatusProto.Code.OK).build())
-                        .build();
-        assertThat(mFrameworkOptimizeStrategy.shouldOptimize(optimizeInfo)).isTrue();
-    }
 
     @Test
     public void testShouldOptimize_byteThreshold() {
         GetOptimizeInfoResultProto optimizeInfo =
                 GetOptimizeInfoResultProto.newBuilder()
-                        .setTimeSinceLastOptimizeMs(TIME_OPTIMIZE_THRESHOLD_MILLIS)
+                        .setTimeSinceLastOptimizeMs(0)
-                        .setEstimatedOptimizableBytes(0)
+                        .setEstimatedOptimizableBytes(
+                                mAppSearchConfig.getCachedBytesOptimizeThreshold())
                         .setOptimizableDocs(0)
                         .setStatus(StatusProto.newBuilder().setCode(StatusProto.Code.OK).build())
                         .build();
@@ -56,11 +44,25 @@ public class FrameworkOptimizeStrategyTest {
 
     @Test
     public void testShouldNotOptimize_timeThreshold() {
+        GetOptimizeInfoResultProto optimizeInfo =
+                GetOptimizeInfoResultProto.newBuilder()
+                        .setTimeSinceLastOptimizeMs(
+                                mAppSearchConfig.getCachedTimeOptimizeThresholdMs())
+                        .setEstimatedOptimizableBytes(0)
+                        .setOptimizableDocs(0)
+                        .setStatus(StatusProto.newBuilder().setCode(StatusProto.Code.OK).build())
+                        .build();
+        assertThat(mFrameworkOptimizeStrategy.shouldOptimize(optimizeInfo)).isTrue();
+    }
+
+    @Test
+    public void testShouldOptimize_docCountThreshold() {
         GetOptimizeInfoResultProto optimizeInfo =
                 GetOptimizeInfoResultProto.newBuilder()
                         .setTimeSinceLastOptimizeMs(0)
                         .setEstimatedOptimizableBytes(0)
-                        .setOptimizableDocs(DOC_COUNT_OPTIMIZE_THRESHOLD)
+                        .setOptimizableDocs(
+                                mAppSearchConfig.getCachedDocCountOptimizeThreshold())
                         .setStatus(StatusProto.newBuilder().setCode(StatusProto.Code.OK).build())
                         .build();
         assertThat(mFrameworkOptimizeStrategy.shouldOptimize(optimizeInfo)).isTrue();