Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add check for optimize after each mutation operation.

Browse Source 
Optimize is an important job for AppSearch to work properly.
It could collect garbage and release resources. Without it, the garbage
resource will last forever and the device's space will be fulled up by
zombie documents.

The algorithm to trigger optimize is:
1: Query garbage resource info after a batch of mutation operations.
2: Only trigger optimize if there are enough resource could be released.

The behavior exists in AppSearch Jetpack for a while and working properly
for our Jetpack dogfooders.

Bug: 175255572
Test: FrameworkOptimizeStrategyTest
Test: AppSearchConfigTest
Test: AppSearchImplTest
Change-Id: Ib7ae475cc035d1b69969df1e22ac409895e0e3fa
This commit is contained in:
Terry Wang
2021-07-01 19:11:39 -07:00
parent 733537f1a6
commit da0e9a9b37
6 changed files with 223 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
apex/appsearch/service/java/com/android/server/appsearch/AppSearchConfig.java
View File

@@ -64,6 +64,12 @@ public final class AppSearchConfig implements AutoCloseable {
static final int DEFAULT_LIMIT_CONFIG_MAX_DOCUMENT_SIZE_BYTES = 512 * 1024; // 512KiB
@VisibleForTesting
static final int DEFAULT_LIMIT_CONFIG_MAX_DOCUMENT_COUNT = 20_000;
@VisibleForTesting
static final int DEFAULT_BYTES_OPTIMIZE_THRESHOLD = 1 * 1024 * 1024; // 1 MiB
@VisibleForTesting
static final int DEFAULT_TIME_OPTIMIZE_THRESHOLD_MILLIS = Integer.MAX_VALUE;
@VisibleForTesting
static final int DEFAULT_DOC_COUNT_OPTIMIZE_THRESHOLD = 10_000;
/*
* Keys for ALL the flags stored in DeviceConfig.
@@ -79,6 +85,9 @@ public final class AppSearchConfig implements AutoCloseable {
"limit_config_max_document_size_bytes";
public static final String KEY_LIMIT_CONFIG_MAX_DOCUMENT_COUNT =
"limit_config_max_document_docunt";
public static final String KEY_BYTES_OPTIMIZE_THRESHOLD = "bytes_optimize_threshold";
public static final String KEY_TIME_OPTIMIZE_THRESHOLD_MILLIS = "time_optimize_threshold";
public static final String KEY_DOC_COUNT_OPTIMIZE_THRESHOLD = "doc_count_optimize_threshold";
// Array contains all the corresponding keys for the cached values.
private static final String[] KEYS_TO_ALL_CACHED_VALUES = {
@@ -88,6 +97,9 @@ public final class AppSearchConfig implements AutoCloseable {
KEY_SAMPLING_INTERVAL_FOR_PUT_DOCUMENT_STATS,
KEY_LIMIT_CONFIG_MAX_DOCUMENT_SIZE_BYTES,
KEY_LIMIT_CONFIG_MAX_DOCUMENT_COUNT,
KEY_BYTES_OPTIMIZE_THRESHOLD,
KEY_TIME_OPTIMIZE_THRESHOLD_MILLIS,
KEY_DOC_COUNT_OPTIMIZE_THRESHOLD
};
// Lock needed for all the operations in this class.
@@ -251,6 +263,48 @@ public final class AppSearchConfig implements AutoCloseable {
}
}
/**
* Returns the cached optimize byte size threshold.
*
* An AppSearch Optimize job will be triggered if the bytes size of garbage resource exceeds
* this threshold.
*/
int getCachedBytesOptimizeThreshold() {
synchronized (mLock) {
throwIfClosedLocked();
return mBundleLocked.getInt(KEY_BYTES_OPTIMIZE_THRESHOLD,
DEFAULT_BYTES_OPTIMIZE_THRESHOLD);
}
}
/**
* Returns the cached optimize time interval threshold.
*
* An AppSearch Optimize job will be triggered if the time since last optimize job exceeds
* this threshold.
*/
int getCachedTimeOptimizeThresholdMs() {
synchronized (mLock) {
throwIfClosedLocked();
return mBundleLocked.getInt(KEY_TIME_OPTIMIZE_THRESHOLD_MILLIS,
DEFAULT_TIME_OPTIMIZE_THRESHOLD_MILLIS);
}
}
/**
* Returns the cached optimize document count threshold threshold.
*
* An AppSearch Optimize job will be triggered if the number of document of garbage resource
* exceeds this threshold.
*/
int getCachedDocCountOptimizeThreshold() {
synchronized (mLock) {
throwIfClosedLocked();
return mBundleLocked.getInt(KEY_DOC_COUNT_OPTIMIZE_THRESHOLD,
DEFAULT_DOC_COUNT_OPTIMIZE_THRESHOLD);
}
}
@GuardedBy("mLock")
private void throwIfClosedLocked() {
if (mIsClosedLocked) {
@@ -307,6 +361,24 @@ public final class AppSearchConfig implements AutoCloseable {
properties.getInt(key, DEFAULT_LIMIT_CONFIG_MAX_DOCUMENT_COUNT));
}
break;
case KEY_BYTES_OPTIMIZE_THRESHOLD:
synchronized (mLock) {
mBundleLocked.putInt(key, properties.getInt(key,
DEFAULT_BYTES_OPTIMIZE_THRESHOLD));
}
break;
case KEY_TIME_OPTIMIZE_THRESHOLD_MILLIS:
synchronized (mLock) {
mBundleLocked.putInt(key, properties.getInt(key,
DEFAULT_TIME_OPTIMIZE_THRESHOLD_MILLIS));
}
break;
case KEY_DOC_COUNT_OPTIMIZE_THRESHOLD:
synchronized (mLock) {
mBundleLocked.putInt(key, properties.getInt(key,
DEFAULT_DOC_COUNT_OPTIMIZE_THRESHOLD));
}
break;
default:
break;
}

34
apex/appsearch/service/java/com/android/server/appsearch/AppSearchManagerService.java
View File

@@ -364,6 +364,12 @@ public class AppSearchManagerService extends SystemService {
++operationSuccessCount;
invokeCallbackOnResult(callback,
AppSearchResult.newSuccessfulResult(setSchemaResponse.getBundle()));
// setSchema will sync the schemas in the request to AppSearch, any existing
// schemas which is not included in the request will be delete if we force
// override incompatible schemas. And all documents of these types will be
// deleted as well. We should checkForOptimize for these deletion.
checkForOptimize(instance);
} catch (Throwable t) {
++operationFailureCount;
statusCode = throwableToFailedResult(t).getResultCode();
@@ -505,6 +511,10 @@ public class AppSearchManagerService extends SystemService {
// Now that the batch has been written. Persist the newly written data.
instance.getAppSearchImpl().persistToDisk(PersistType.Code.LITE);
invokeCallbackOnResult(callback, resultBuilder.build());
// The existing documents with same ID will be deleted, so there may be some
// resources that could be released after optimize().
checkForOptimize(instance, /*mutateBatchSize=*/ documentBundles.size());
} catch (Throwable t) {
++operationFailureCount;
statusCode = throwableToFailedResult(t).getResultCode();
@@ -1023,6 +1033,8 @@ public class AppSearchManagerService extends SystemService {
// Now that the batch has been written. Persist the newly written data.
instance.getAppSearchImpl().persistToDisk(PersistType.Code.LITE);
invokeCallbackOnResult(callback, resultBuilder.build());
checkForOptimize(instance, ids.size());
} catch (Throwable t) {
++operationFailureCount;
statusCode = throwableToFailedResult(t).getResultCode();
@@ -1092,6 +1104,8 @@ public class AppSearchManagerService extends SystemService {
instance.getAppSearchImpl().persistToDisk(PersistType.Code.LITE);
++operationSuccessCount;
invokeCallbackOnResult(callback, AppSearchResult.newSuccessfulResult(null));
checkForOptimize(instance);
} catch (Throwable t) {
++operationFailureCount;
statusCode = throwableToFailedResult(t).getResultCode();
@@ -1472,4 +1486,24 @@ public class AppSearchManagerService extends SystemService {
}
}
}
private void checkForOptimize(AppSearchUserInstance instance, int mutateBatchSize) {
EXECUTOR.execute(() -> {
try {
instance.getAppSearchImpl().checkForOptimize(mutateBatchSize);
} catch (AppSearchException e) {
Log.w(TAG, "Error occurred when check for optimize", e);
}
});
}
private void checkForOptimize(AppSearchUserInstance instance) {
EXECUTOR.execute(() -> {
try {
instance.getAppSearchImpl().checkForOptimize();
} catch (AppSearchException e) {
Log.w(TAG, "Error occurred when check for optimize", e);
}
});
}
}

3
apex/appsearch/service/java/com/android/server/appsearch/AppSearchUserInstanceManager.java
View File

@@ -27,7 +27,6 @@ import android.util.Log;
import com.android.internal.annotations.GuardedBy;
import com.android.server.appsearch.external.localstorage.AppSearchImpl;
import com.android.server.appsearch.external.localstorage.FrameworkOptimizeStrategy;
import com.android.server.appsearch.external.localstorage.stats.InitializeStats;
import com.android.server.appsearch.stats.PlatformLogger;
import com.android.server.appsearch.visibilitystore.VisibilityStoreImpl;
@@ -177,7 +176,7 @@ public final class AppSearchUserInstanceManager {
icingDir,
new FrameworkLimitConfig(config),
initStatsBuilder,
new FrameworkOptimizeStrategy());
new FrameworkOptimizeStrategy(config));
long prepareVisibilityStoreLatencyStartMillis = SystemClock.elapsedRealtime();
VisibilityStoreImpl visibilityStore =

28
apex/appsearch/service/java/com/android/server/appsearch/external/localstorage/FrameworkOptimizeStrategy.java → apex/appsearch/service/java/com/android/server/appsearch/FrameworkOptimizeStrategy.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2021 The Android Open Source Project
* Copyright (C) 2021 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -13,14 +13,17 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.server.appsearch.external.localstorage;
package com.android.server.appsearch;
import android.annotation.NonNull;
import com.android.internal.annotations.VisibleForTesting;
import com.android.server.appsearch.external.localstorage.AppSearchImpl;
import com.android.server.appsearch.external.localstorage.OptimizeStrategy;
import com.google.android.icing.proto.GetOptimizeInfoResultProto;
import java.util.Objects;
/**
* An implementation of {@link OptimizeStrategy} will determine when to trigger {@link
* AppSearchImpl#optimize()} in Jetpack environment.
@@ -28,17 +31,18 @@ import com.google.android.icing.proto.GetOptimizeInfoResultProto;
* @hide
*/
public class FrameworkOptimizeStrategy implements OptimizeStrategy {
@VisibleForTesting static final int DOC_COUNT_OPTIMIZE_THRESHOLD = 100_000;
@VisibleForTesting static final int BYTES_OPTIMIZE_THRESHOLD = 1 * 1024 * 1024 * 1024; // 1GB
@VisibleForTesting
static final long TIME_OPTIMIZE_THRESHOLD_MILLIS = 7 * 24 * 60 * 60 * 1000; // 1 week
private final AppSearchConfig mAppSearchConfig;
FrameworkOptimizeStrategy(@NonNull AppSearchConfig config) {
mAppSearchConfig = Objects.requireNonNull(config);
}
@Override
public boolean shouldOptimize(@NonNull GetOptimizeInfoResultProto optimizeInfo) {
return optimizeInfo.getOptimizableDocs() >= DOC_COUNT_OPTIMIZE_THRESHOLD
|| optimizeInfo.getEstimatedOptimizableBytes() >= BYTES_OPTIMIZE_THRESHOLD
|| optimizeInfo.getTimeSinceLastOptimizeMs() >= TIME_OPTIMIZE_THRESHOLD_MILLIS;
return optimizeInfo.getOptimizableDocs()
>= mAppSearchConfig.getCachedDocCountOptimizeThreshold()
|| optimizeInfo.getEstimatedOptimizableBytes()
>= mAppSearchConfig.getCachedBytesOptimizeThreshold()
|| optimizeInfo.getTimeSinceLastOptimizeMs()
>= mAppSearchConfig.getCachedTimeOptimizeThresholdMs();
}
}

81
services/tests/mockingservicestests/src/com/android/server/appsearch/AppSearchConfigTest.java
View File

@@ -54,6 +54,12 @@ public class AppSearchConfigTest {
AppSearchConfig.DEFAULT_LIMIT_CONFIG_MAX_DOCUMENT_SIZE_BYTES);
assertThat(appSearchConfig.getCachedLimitConfigMaxDocumentCount()).isEqualTo(
AppSearchConfig.DEFAULT_LIMIT_CONFIG_MAX_DOCUMENT_COUNT);
assertThat(appSearchConfig.getCachedBytesOptimizeThreshold()).isEqualTo(
AppSearchConfig.DEFAULT_BYTES_OPTIMIZE_THRESHOLD);
assertThat(appSearchConfig.getCachedTimeOptimizeThresholdMs()).isEqualTo(
AppSearchConfig.DEFAULT_TIME_OPTIMIZE_THRESHOLD_MILLIS);
assertThat(appSearchConfig.getCachedDocCountOptimizeThreshold()).isEqualTo(
AppSearchConfig.DEFAULT_DOC_COUNT_OPTIMIZE_THRESHOLD);
}
@Test
@@ -163,10 +169,8 @@ public class AppSearchConfigTest {
/**
* Tests if we fall back to {@link AppSearchConfig#DEFAULT_SAMPLING_INTERVAL} if both default
* sampling
* interval and custom value are not set in DeviceConfig, and there is some other sampling
* interval
* set.
* sampling interval and custom value are not set in DeviceConfig, and there is some other
* sampling interval set.
*/
@Test
public void testFallbackToDefaultSamplingValue_useHardCodedDefault() {
@@ -269,7 +273,7 @@ public class AppSearchConfigTest {
}
@Test
public void testCustomizedValue() {
public void testCustomizedValue_maxDocument() {
DeviceConfig.setProperty(DeviceConfig.NAMESPACE_APPSEARCH,
AppSearchConfig.KEY_LIMIT_CONFIG_MAX_DOCUMENT_SIZE_BYTES,
Integer.toString(2001),
@@ -284,6 +288,64 @@ public class AppSearchConfigTest {
assertThat(appSearchConfig.getCachedLimitConfigMaxDocumentCount()).isEqualTo(2002);
}
@Test
public void testCustomizedValue_optimizeThreshold() {
DeviceConfig.setProperty(DeviceConfig.NAMESPACE_APPSEARCH,
AppSearchConfig.KEY_BYTES_OPTIMIZE_THRESHOLD,
Integer.toString(147147),
false);
DeviceConfig.setProperty(DeviceConfig.NAMESPACE_APPSEARCH,
AppSearchConfig.KEY_TIME_OPTIMIZE_THRESHOLD_MILLIS,
Integer.toString(258258),
false);
DeviceConfig.setProperty(DeviceConfig.NAMESPACE_APPSEARCH,
AppSearchConfig.KEY_DOC_COUNT_OPTIMIZE_THRESHOLD,
Integer.toString(369369),
false);
AppSearchConfig appSearchConfig = AppSearchConfig.create(DIRECT_EXECUTOR);
assertThat(appSearchConfig.getCachedBytesOptimizeThreshold()).isEqualTo(147147);
assertThat(appSearchConfig.getCachedTimeOptimizeThresholdMs()).isEqualTo(258258);
assertThat(appSearchConfig.getCachedDocCountOptimizeThreshold()).isEqualTo(369369);
}
@Test
public void testCustomizedValueOverride_optimizeThreshold() {
DeviceConfig.setProperty(DeviceConfig.NAMESPACE_APPSEARCH,
AppSearchConfig.KEY_BYTES_OPTIMIZE_THRESHOLD,
Integer.toString(147147),
false);
DeviceConfig.setProperty(DeviceConfig.NAMESPACE_APPSEARCH,
AppSearchConfig.KEY_TIME_OPTIMIZE_THRESHOLD_MILLIS,
Integer.toString(258258),
false);
DeviceConfig.setProperty(DeviceConfig.NAMESPACE_APPSEARCH,
AppSearchConfig.KEY_DOC_COUNT_OPTIMIZE_THRESHOLD,
Integer.toString(369369),
false);
AppSearchConfig appSearchConfig = AppSearchConfig.create(DIRECT_EXECUTOR);
// Override
DeviceConfig.setProperty(DeviceConfig.NAMESPACE_APPSEARCH,
AppSearchConfig.KEY_BYTES_OPTIMIZE_THRESHOLD,
Integer.toString(741741),
false);
DeviceConfig.setProperty(DeviceConfig.NAMESPACE_APPSEARCH,
AppSearchConfig.KEY_TIME_OPTIMIZE_THRESHOLD_MILLIS,
Integer.toString(852852),
false);
DeviceConfig.setProperty(DeviceConfig.NAMESPACE_APPSEARCH,
AppSearchConfig.KEY_DOC_COUNT_OPTIMIZE_THRESHOLD,
Integer.toString(963963),
false);
assertThat(appSearchConfig.getCachedBytesOptimizeThreshold()).isEqualTo(741741);
assertThat(appSearchConfig.getCachedTimeOptimizeThresholdMs()).isEqualTo(852852);
assertThat(appSearchConfig.getCachedDocCountOptimizeThreshold()).isEqualTo(963963);
}
@Test
public void testNotUsable_afterClose() {
AppSearchConfig appSearchConfig = AppSearchConfig.create(DIRECT_EXECUTOR);
@@ -302,5 +364,14 @@ public class AppSearchConfigTest {
Assert.assertThrows("Trying to use a closed AppSearchConfig instance.",
IllegalStateException.class,
() -> appSearchConfig.getCachedSamplingIntervalForPutDocumentStats());
Assert.assertThrows("Trying to use a closed AppSearchConfig instance.",
IllegalStateException.class,
() -> appSearchConfig.getCachedBytesOptimizeThreshold());
Assert.assertThrows("Trying to use a closed AppSearchConfig instance.",
IllegalStateException.class,
() -> appSearchConfig.getCachedTimeOptimizeThresholdMs());
Assert.assertThrows("Trying to use a closed AppSearchConfig instance.",
IllegalStateException.class,
() -> appSearchConfig.getCachedDocCountOptimizeThreshold());
}
}

46
services/tests/servicestests/src/com/android/server/appsearch/external/localstorage/FrameworkOptimizeStrategyTest.java → services/tests/servicestests/src/com/android/server/appsearch/FrameworkOptimizeStrategyTest.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2021 The Android Open Source Project
* Copyright (C) 2021 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -13,12 +13,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.server.appsearch;
package com.android.server.appsearch.external.localstorage;
import static com.android.server.appsearch.external.localstorage.FrameworkOptimizeStrategy.BYTES_OPTIMIZE_THRESHOLD;
import static com.android.server.appsearch.external.localstorage.FrameworkOptimizeStrategy.DOC_COUNT_OPTIMIZE_THRESHOLD;
import static com.android.server.appsearch.external.localstorage.FrameworkOptimizeStrategy.TIME_OPTIMIZE_THRESHOLD_MILLIS;
import static com.android.internal.util.ConcurrentUtils.DIRECT_EXECUTOR;
import static com.google.common.truth.Truth.assertThat;
@@ -28,26 +25,17 @@ import com.android.server.appsearch.icing.proto.StatusProto;
import org.junit.Test;
public class FrameworkOptimizeStrategyTest {
FrameworkOptimizeStrategy mFrameworkOptimizeStrategy = new FrameworkOptimizeStrategy();
@Test
public void testShouldOptimize_docCountThreshold() {
GetOptimizeInfoResultProto optimizeInfo =
GetOptimizeInfoResultProto.newBuilder()
.setTimeSinceLastOptimizeMs(0)
.setEstimatedOptimizableBytes(BYTES_OPTIMIZE_THRESHOLD)
.setOptimizableDocs(0)
.setStatus(StatusProto.newBuilder().setCode(StatusProto.Code.OK).build())
.build();
assertThat(mFrameworkOptimizeStrategy.shouldOptimize(optimizeInfo)).isTrue();
}
AppSearchConfig mAppSearchConfig = AppSearchConfig.create(DIRECT_EXECUTOR);
FrameworkOptimizeStrategy mFrameworkOptimizeStrategy =
new FrameworkOptimizeStrategy(mAppSearchConfig);
@Test
public void testShouldOptimize_byteThreshold() {
GetOptimizeInfoResultProto optimizeInfo =
GetOptimizeInfoResultProto.newBuilder()
.setTimeSinceLastOptimizeMs(TIME_OPTIMIZE_THRESHOLD_MILLIS)
.setEstimatedOptimizableBytes(0)
.setTimeSinceLastOptimizeMs(0)
.setEstimatedOptimizableBytes(
mAppSearchConfig.getCachedBytesOptimizeThreshold())
.setOptimizableDocs(0)
.setStatus(StatusProto.newBuilder().setCode(StatusProto.Code.OK).build())
.build();
@@ -56,11 +44,25 @@ public class FrameworkOptimizeStrategyTest {
@Test
public void testShouldNotOptimize_timeThreshold() {
GetOptimizeInfoResultProto optimizeInfo =
GetOptimizeInfoResultProto.newBuilder()
.setTimeSinceLastOptimizeMs(
mAppSearchConfig.getCachedTimeOptimizeThresholdMs())
.setEstimatedOptimizableBytes(0)
.setOptimizableDocs(0)
.setStatus(StatusProto.newBuilder().setCode(StatusProto.Code.OK).build())
.build();
assertThat(mFrameworkOptimizeStrategy.shouldOptimize(optimizeInfo)).isTrue();
}
@Test
public void testShouldOptimize_docCountThreshold() {
GetOptimizeInfoResultProto optimizeInfo =
GetOptimizeInfoResultProto.newBuilder()
.setTimeSinceLastOptimizeMs(0)
.setEstimatedOptimizableBytes(0)
.setOptimizableDocs(DOC_COUNT_OPTIMIZE_THRESHOLD)
.setOptimizableDocs(
mAppSearchConfig.getCachedDocCountOptimizeThreshold())
.setStatus(StatusProto.newBuilder().setCode(StatusProto.Code.OK).build())
.build();
assertThat(mFrameworkOptimizeStrategy.shouldOptimize(optimizeInfo)).isTrue();