[IPMS] Implement regular maintenance

Implement regular maintenance of IpMemoryStoreService. Regular
maintenance is scheduled for when the device is idle with access
power and a minimum interval of one day.

Bug: 113554482
Test: atest NetworkStackTests
Change-Id: Id3985e30d12307fc2e9fcbe782caaf97a627cef3
This commit is contained in:
paulhu
2019-03-26 01:39:10 +08:00
parent 154a0d0b38
commit 028d7a55fd
5 changed files with 484 additions and 6 deletions

View File

@@ -25,5 +25,9 @@
android:defaultToDeviceProtectedStorage="true"
android:directBootAware="true"
android:usesCleartextTraffic="true">
<service android:name="com.android.server.connectivity.ipmemorystore.RegularMaintenanceJobService"
android:permission="android.permission.BIND_JOB_SERVICE" >
</service>
</application>
</manifest>

View File

@@ -139,8 +139,9 @@ public class IpMemoryStoreDatabase {
/** The SQLite DB helper */
public static class DbHelper extends SQLiteOpenHelper {
// Update this whenever changing the schema.
private static final int SCHEMA_VERSION = 3;
private static final int SCHEMA_VERSION = 4;
private static final String DATABASE_FILENAME = "IpMemoryStore.db";
private static final String TRIGGER_NAME = "delete_cascade_to_private";
public DbHelper(@NonNull final Context context) {
super(context, DATABASE_FILENAME, null, SCHEMA_VERSION);
@@ -152,6 +153,7 @@ public class IpMemoryStoreDatabase {
public void onCreate(@NonNull final SQLiteDatabase db) {
db.execSQL(NetworkAttributesContract.CREATE_TABLE);
db.execSQL(PrivateDataContract.CREATE_TABLE);
createTrigger(db);
}
/** Called when the database is upgraded */
@@ -172,6 +174,10 @@ public class IpMemoryStoreDatabase {
+ " " + NetworkAttributesContract.COLTYPE_ASSIGNEDV4ADDRESSEXPIRY;
db.execSQL(sqlUpgradeAddressExpiry);
}
if (oldVersion < 4) {
createTrigger(db);
}
} catch (SQLiteException e) {
Log.e(TAG, "Could not upgrade to the new version", e);
// create database with new version
@@ -188,8 +194,20 @@ public class IpMemoryStoreDatabase {
// Downgrades always nuke all data and recreate an empty table.
db.execSQL(NetworkAttributesContract.DROP_TABLE);
db.execSQL(PrivateDataContract.DROP_TABLE);
db.execSQL("DROP TRIGGER " + TRIGGER_NAME);
onCreate(db);
}
private void createTrigger(@NonNull final SQLiteDatabase db) {
final String createTrigger = "CREATE TRIGGER " + TRIGGER_NAME
+ " DELETE ON " + NetworkAttributesContract.TABLENAME
+ " BEGIN"
+ " DELETE FROM " + PrivateDataContract.TABLENAME + " WHERE OLD."
+ NetworkAttributesContract.COLNAME_L2KEY
+ "=" + PrivateDataContract.COLNAME_L2KEY
+ "; END;";
db.execSQL(createTrigger);
}
}
@NonNull
@@ -336,7 +354,7 @@ public class IpMemoryStoreDatabase {
}
// If the attributes are null, this will only write the expiry.
// Returns an int out of Status.{SUCCESS,ERROR_*}
// Returns an int out of Status.{SUCCESS, ERROR_*}
static int storeNetworkAttributes(@NonNull final SQLiteDatabase db, @NonNull final String key,
final long expiry, @Nullable final NetworkAttributes attributes) {
final ContentValues cv = toContentValues(key, attributes, expiry);
@@ -361,7 +379,7 @@ public class IpMemoryStoreDatabase {
return Status.ERROR_STORAGE;
}
// Returns an int out of Status.{SUCCESS,ERROR_*}
// Returns an int out of Status.{SUCCESS, ERROR_*}
static int storeBlob(@NonNull final SQLiteDatabase db, @NonNull final String key,
@NonNull final String clientId, @NonNull final String name,
@NonNull final byte[] data) {
@@ -524,6 +542,93 @@ public class IpMemoryStoreDatabase {
return bestKey;
}
// Drops all records that are expired. Relevance has decayed to zero of these records. Returns
// an int out of Status.{SUCCESS, ERROR_*}
static int dropAllExpiredRecords(@NonNull final SQLiteDatabase db) {
db.beginTransaction();
try {
// Deletes NetworkAttributes that have expired.
db.delete(NetworkAttributesContract.TABLENAME,
NetworkAttributesContract.COLNAME_EXPIRYDATE + " < ?",
new String[]{Long.toString(System.currentTimeMillis())});
db.setTransactionSuccessful();
} catch (SQLiteException e) {
Log.e(TAG, "Could not delete data from memory store", e);
return Status.ERROR_STORAGE;
} finally {
db.endTransaction();
}
// Execute vacuuming here if above operation has no exception. If above operation got
// exception, vacuuming can be ignored for reducing unnecessary consumption.
try {
db.execSQL("VACUUM");
} catch (SQLiteException e) {
// Do nothing.
}
return Status.SUCCESS;
}
// Drops number of records that start from the lowest expiryDate. Returns an int out of
// Status.{SUCCESS, ERROR_*}
static int dropNumberOfRecords(@NonNull final SQLiteDatabase db, int number) {
if (number <= 0) {
return Status.ERROR_ILLEGAL_ARGUMENT;
}
// Queries number of NetworkAttributes that start from the lowest expiryDate.
final Cursor cursor = db.query(NetworkAttributesContract.TABLENAME,
new String[] {NetworkAttributesContract.COLNAME_EXPIRYDATE}, // columns
null, // selection
null, // selectionArgs
null, // groupBy
null, // having
NetworkAttributesContract.COLNAME_EXPIRYDATE, // orderBy
Integer.toString(number)); // limit
if (cursor == null || cursor.getCount() <= 0) return Status.ERROR_GENERIC;
cursor.moveToLast();
//Get the expiryDate from last record.
final long expiryDate = getLong(cursor, NetworkAttributesContract.COLNAME_EXPIRYDATE, 0);
cursor.close();
db.beginTransaction();
try {
// Deletes NetworkAttributes that expiryDate are lower than given value.
db.delete(NetworkAttributesContract.TABLENAME,
NetworkAttributesContract.COLNAME_EXPIRYDATE + " <= ?",
new String[]{Long.toString(expiryDate)});
db.setTransactionSuccessful();
} catch (SQLiteException e) {
Log.e(TAG, "Could not delete data from memory store", e);
return Status.ERROR_STORAGE;
} finally {
db.endTransaction();
}
// Execute vacuuming here if above operation has no exception. If above operation got
// exception, vacuuming can be ignored for reducing unnecessary consumption.
try {
db.execSQL("VACUUM");
} catch (SQLiteException e) {
// Do nothing.
}
return Status.SUCCESS;
}
static int getTotalRecordNumber(@NonNull final SQLiteDatabase db) {
// Query the total number of NetworkAttributes
final Cursor cursor = db.query(NetworkAttributesContract.TABLENAME,
new String[] {"COUNT(*)"}, // columns
null, // selection
null, // selectionArgs
null, // groupBy
null, // having
null); // orderBy
cursor.moveToFirst();
return cursor == null ? 0 : cursor.getInt(0);
}
// Helper methods
private static String getString(final Cursor cursor, final String columnName) {
final int columnIndex = cursor.getColumnIndex(columnName);

View File

@@ -22,6 +22,7 @@ import static android.net.ipmemorystore.Status.ERROR_ILLEGAL_ARGUMENT;
import static android.net.ipmemorystore.Status.SUCCESS;
import static com.android.server.connectivity.ipmemorystore.IpMemoryStoreDatabase.EXPIRY_ERROR;
import static com.android.server.connectivity.ipmemorystore.RegularMaintenanceJobService.InterruptMaintenance;
import android.annotation.NonNull;
import android.annotation.Nullable;
@@ -43,6 +44,9 @@ import android.net.ipmemorystore.StatusParcelable;
import android.os.RemoteException;
import android.util.Log;
import com.android.internal.annotations.VisibleForTesting;
import java.io.File;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -57,8 +61,17 @@ import java.util.concurrent.Executors;
public class IpMemoryStoreService extends IIpMemoryStore.Stub {
private static final String TAG = IpMemoryStoreService.class.getSimpleName();
private static final int MAX_CONCURRENT_THREADS = 4;
private static final int DATABASE_SIZE_THRESHOLD = 10 * 1024 * 1024; //10MB
private static final int MAX_DROP_RECORD_TIMES = 500;
private static final int MIN_DELETE_NUM = 5;
private static final boolean DBG = true;
// Error codes below are internal and used for notifying status beteween IpMemoryStore modules.
static final int ERROR_INTERNAL_BASE = -1_000_000_000;
// This error code is used for maintenance only to notify RegularMaintenanceJobService that
// full maintenance job has been interrupted.
static final int ERROR_INTERNAL_INTERRUPTED = ERROR_INTERNAL_BASE - 1;
@NonNull
final Context mContext;
@Nullable
@@ -111,6 +124,7 @@ public class IpMemoryStoreService extends IIpMemoryStore.Stub {
// with judicious subclassing of ThreadPoolExecutor, but that's a lot of dangerous
// complexity for little benefit in this case.
mExecutor = Executors.newWorkStealingPool(MAX_CONCURRENT_THREADS);
RegularMaintenanceJobService.schedule(mContext, this);
}
/**
@@ -125,6 +139,7 @@ public class IpMemoryStoreService extends IIpMemoryStore.Stub {
// guarantee the threads can be terminated in any given amount of time.
mExecutor.shutdownNow();
if (mDb != null) mDb.close();
RegularMaintenanceJobService.unschedule(mContext);
}
/** Helper function to make a status object */
@@ -394,4 +409,89 @@ public class IpMemoryStoreService extends IIpMemoryStore.Stub {
}
});
}
/** Get db size threshold. */
@VisibleForTesting
protected int getDbSizeThreshold() {
return DATABASE_SIZE_THRESHOLD;
}
private long getDbSize() {
final File dbFile = new File(mDb.getPath());
try {
return dbFile.length();
} catch (final SecurityException e) {
if (DBG) Log.e(TAG, "Read db size access deny.", e);
// Return zero value if can't get disk usage exactly.
return 0;
}
}
/** Check if db size is over the threshold. */
@VisibleForTesting
boolean isDbSizeOverThreshold() {
return getDbSize() > getDbSizeThreshold();
}
/**
* Full maintenance.
*
* @param listener A listener to inform of the completion of this call.
*/
void fullMaintenance(@NonNull final IOnStatusListener listener,
@NonNull final InterruptMaintenance interrupt) {
mExecutor.execute(() -> {
try {
if (null == mDb) {
listener.onComplete(makeStatus(ERROR_DATABASE_CANNOT_BE_OPENED));
return;
}
// Interrupt maintenance because the scheduling job has been canceled.
if (checkForInterrupt(listener, interrupt)) return;
int result = SUCCESS;
// Drop all records whose relevance has decayed to zero.
// This is the first step to decrease memory store size.
result = IpMemoryStoreDatabase.dropAllExpiredRecords(mDb);
if (checkForInterrupt(listener, interrupt)) return;
// Aggregate historical data in passes
// TODO : Waiting for historical data implement.
// Check if db size meets the storage goal(10MB). If not, keep dropping records and
// aggregate historical data until the storage goal is met. Use for loop with 500
// times restriction to prevent infinite loop (Deleting records always fail and db
// size is still over the threshold)
for (int i = 0; isDbSizeOverThreshold() && i < MAX_DROP_RECORD_TIMES; i++) {
if (checkForInterrupt(listener, interrupt)) return;
final int totalNumber = IpMemoryStoreDatabase.getTotalRecordNumber(mDb);
final long dbSize = getDbSize();
final float decreaseRate = (dbSize == 0)
? 0 : (float) (dbSize - getDbSizeThreshold()) / (float) dbSize;
final int deleteNumber = Math.max(
(int) (totalNumber * decreaseRate), MIN_DELETE_NUM);
result = IpMemoryStoreDatabase.dropNumberOfRecords(mDb, deleteNumber);
if (checkForInterrupt(listener, interrupt)) return;
// Aggregate historical data
// TODO : Waiting for historical data implement.
}
listener.onComplete(makeStatus(result));
} catch (final RemoteException e) {
// Client at the other end died
}
});
}
private boolean checkForInterrupt(@NonNull final IOnStatusListener listener,
@NonNull final InterruptMaintenance interrupt) throws RemoteException {
if (!interrupt.isInterrupted()) return false;
listener.onComplete(makeStatus(ERROR_INTERNAL_INTERRUPTED));
return true;
}
}

View File

@@ -0,0 +1,140 @@
/*
* Copyright (C) 2019 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.server.connectivity.ipmemorystore;
import android.app.job.JobInfo;
import android.app.job.JobParameters;
import android.app.job.JobScheduler;
import android.app.job.JobService;
import android.content.ComponentName;
import android.content.Context;
import android.net.ipmemorystore.IOnStatusListener;
import android.net.ipmemorystore.Status;
import android.net.ipmemorystore.StatusParcelable;
import android.os.IBinder;
import android.os.RemoteException;
import android.util.Log;
import java.util.ArrayList;
import java.util.concurrent.TimeUnit;
/**
* Regular maintenance job service.
* @hide
*/
public final class RegularMaintenanceJobService extends JobService {
// Must be unique within the system server uid.
public static final int REGULAR_MAINTENANCE_ID = 3345678;
/**
* Class for interrupt check of maintenance job.
*/
public static final class InterruptMaintenance {
private volatile boolean mIsInterrupted;
private final int mJobId;
public InterruptMaintenance(int jobId) {
mJobId = jobId;
mIsInterrupted = false;
}
public int getJobId() {
return mJobId;
}
public void setInterrupted(boolean interrupt) {
mIsInterrupted = interrupt;
}
public boolean isInterrupted() {
return mIsInterrupted;
}
}
private static final ArrayList<InterruptMaintenance> sInterruptList = new ArrayList<>();
private static IpMemoryStoreService sIpMemoryStoreService;
@Override
public boolean onStartJob(JobParameters params) {
if (sIpMemoryStoreService == null) {
Log.wtf("RegularMaintenanceJobService",
"Can not start job because sIpMemoryStoreService is null.");
return false;
}
final InterruptMaintenance im = new InterruptMaintenance(params.getJobId());
sInterruptList.add(im);
sIpMemoryStoreService.fullMaintenance(new IOnStatusListener() {
@Override
public void onComplete(final StatusParcelable statusParcelable) throws RemoteException {
final Status result = new Status(statusParcelable);
if (!result.isSuccess()) {
Log.e("RegularMaintenanceJobService", "Regular maintenance failed."
+ " Error is " + result.resultCode);
}
sInterruptList.remove(im);
jobFinished(params, !result.isSuccess());
}
@Override
public IBinder asBinder() {
return null;
}
}, im);
return true;
}
@Override
public boolean onStopJob(JobParameters params) {
final int jobId = params.getJobId();
for (InterruptMaintenance im : sInterruptList) {
if (im.getJobId() == jobId) {
im.setInterrupted(true);
}
}
return true;
}
/** Schedule regular maintenance job */
static void schedule(Context context, IpMemoryStoreService ipMemoryStoreService) {
final JobScheduler jobScheduler =
(JobScheduler) context.getSystemService(Context.JOB_SCHEDULER_SERVICE);
final ComponentName maintenanceJobName =
new ComponentName(context, RegularMaintenanceJobService.class);
// Regular maintenance is scheduled for when the device is idle with access power and a
// minimum interval of one day.
final JobInfo regularMaintenanceJob =
new JobInfo.Builder(REGULAR_MAINTENANCE_ID, maintenanceJobName)
.setRequiresDeviceIdle(true)
.setRequiresCharging(true)
.setRequiresBatteryNotLow(true)
.setPeriodic(TimeUnit.HOURS.toMillis(24)).build();
jobScheduler.schedule(regularMaintenanceJob);
sIpMemoryStoreService = ipMemoryStoreService;
}
/** Unschedule regular maintenance job */
static void unschedule(Context context) {
final JobScheduler jobScheduler =
(JobScheduler) context.getSystemService(Context.JOB_SCHEDULER_SERVICE);
jobScheduler.cancel(REGULAR_MAINTENANCE_ID);
sIpMemoryStoreService = null;
}
}

View File

@@ -16,6 +16,8 @@
package com.android.server.connectivity.ipmemorystore;
import static com.android.server.connectivity.ipmemorystore.RegularMaintenanceJobService.InterruptMaintenance;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
@@ -24,6 +26,7 @@ import static org.junit.Assert.fail;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.doReturn;
import android.app.job.JobScheduler;
import android.content.Context;
import android.net.ipmemorystore.Blob;
import android.net.ipmemorystore.IOnBlobRetrievedListener;
@@ -37,6 +40,7 @@ import android.net.ipmemorystore.SameL3NetworkResponse;
import android.net.ipmemorystore.SameL3NetworkResponseParcelable;
import android.net.ipmemorystore.Status;
import android.net.ipmemorystore.StatusParcelable;
import android.os.ConditionVariable;
import android.os.IBinder;
import android.os.RemoteException;
@@ -69,6 +73,9 @@ public class IpMemoryStoreServiceTest {
private static final String TEST_CLIENT_ID = "testClientId";
private static final String TEST_DATA_NAME = "testData";
private static final int TEST_DATABASE_SIZE_THRESHOLD = 100 * 1024; //100KB
private static final int DEFAULT_TIMEOUT_MS = 5000;
private static final int LONG_TIMEOUT_MS = 30000;
private static final int FAKE_KEY_COUNT = 20;
private static final String[] FAKE_KEYS;
static {
@@ -80,6 +87,8 @@ public class IpMemoryStoreServiceTest {
@Mock
private Context mMockContext;
@Mock
private JobScheduler mMockJobScheduler;
private File mDbFile;
private IpMemoryStoreService mService;
@@ -91,7 +100,22 @@ public class IpMemoryStoreServiceTest {
final File dir = context.getFilesDir();
mDbFile = new File(dir, "test.db");
doReturn(mDbFile).when(mMockContext).getDatabasePath(anyString());
mService = new IpMemoryStoreService(mMockContext);
doReturn(mMockJobScheduler).when(mMockContext)
.getSystemService(Context.JOB_SCHEDULER_SERVICE);
mService = new IpMemoryStoreService(mMockContext) {
@Override
protected int getDbSizeThreshold() {
return TEST_DATABASE_SIZE_THRESHOLD;
}
@Override
boolean isDbSizeOverThreshold() {
// Add a 100ms delay here for pausing maintenance job a while. Interrupted flag can
// be set at this time.
waitForMs(100);
return super.isDbSizeOverThreshold();
}
};
}
@After
@@ -200,10 +224,15 @@ public class IpMemoryStoreServiceTest {
// Helper method to factorize some boilerplate
private void doLatched(final String timeoutMessage, final Consumer<CountDownLatch> functor) {
doLatched(timeoutMessage, functor, DEFAULT_TIMEOUT_MS);
}
private void doLatched(final String timeoutMessage, final Consumer<CountDownLatch> functor,
final int timeout) {
final CountDownLatch latch = new CountDownLatch(1);
functor.accept(latch);
try {
if (!latch.await(5000, TimeUnit.MILLISECONDS)) {
if (!latch.await(timeout, TimeUnit.MILLISECONDS)) {
fail(timeoutMessage);
}
} catch (InterruptedException e) {
@@ -224,6 +253,46 @@ public class IpMemoryStoreServiceTest {
})));
}
/** Insert large data that db size will be over threshold for maintenance test usage. */
private void insertFakeDataAndOverThreshold() {
try {
final NetworkAttributes.Builder na = new NetworkAttributes.Builder();
na.setAssignedV4Address((Inet4Address) Inet4Address.getByName("1.2.3.4"));
na.setGroupHint("hint1");
na.setMtu(219);
na.setDnsAddresses(Arrays.asList(Inet6Address.getByName("0A1C:2E40:480A::1CA6")));
final byte[] data = new byte[]{-3, 6, 8, -9, 12, -128, 0, 89, 112, 91, -34};
final long time = System.currentTimeMillis() - 1;
for (int i = 0; i < 1000; i++) {
int errorCode = IpMemoryStoreDatabase.storeNetworkAttributes(
mService.mDb,
"fakeKey" + i,
// Let first 100 records get expiry.
i < 100 ? time : time + TimeUnit.HOURS.toMillis(i),
na.build());
assertEquals(errorCode, Status.SUCCESS);
errorCode = IpMemoryStoreDatabase.storeBlob(
mService.mDb, "fakeKey" + i, TEST_CLIENT_ID, TEST_DATA_NAME, data);
assertEquals(errorCode, Status.SUCCESS);
}
// After added 5000 records, db size is larger than fake threshold(100KB).
assertTrue(mService.isDbSizeOverThreshold());
} catch (final UnknownHostException e) {
fail("Insert fake data fail");
}
}
/** Wait for assigned time. */
private void waitForMs(long ms) {
try {
Thread.sleep(ms);
} catch (final InterruptedException e) {
fail("Thread was interrupted");
}
}
@Test
public void testNetworkAttributes() throws UnknownHostException {
final NetworkAttributes.Builder na = new NetworkAttributes.Builder();
@@ -344,7 +413,7 @@ public class IpMemoryStoreServiceTest {
status.isSuccess());
assertEquals(l2Key, key);
assertEquals(name, TEST_DATA_NAME);
Arrays.equals(b.data, data);
assertTrue(Arrays.equals(b.data, data));
latch.countDown();
})));
@@ -506,4 +575,64 @@ public class IpMemoryStoreServiceTest {
latch.countDown();
})));
}
@Test
public void testFullMaintenance() {
insertFakeDataAndOverThreshold();
final InterruptMaintenance im = new InterruptMaintenance(0/* Fake JobId */);
// Do full maintenance and then db size should go down and meet the threshold.
doLatched("Maintenance unexpectedly completed successfully", latch ->
mService.fullMaintenance(onStatus((status) -> {
assertTrue("Execute full maintenance failed: "
+ status.resultCode, status.isSuccess());
latch.countDown();
}), im), LONG_TIMEOUT_MS);
// Assume that maintenance is successful, db size shall meet the threshold.
assertFalse(mService.isDbSizeOverThreshold());
}
@Test
public void testInterruptMaintenance() {
insertFakeDataAndOverThreshold();
final InterruptMaintenance im = new InterruptMaintenance(0/* Fake JobId */);
// Test interruption immediately.
im.setInterrupted(true);
// Do full maintenance and the expectation is not completed by interruption.
doLatched("Maintenance unexpectedly completed successfully", latch ->
mService.fullMaintenance(onStatus((status) -> {
assertFalse(status.isSuccess());
latch.countDown();
}), im), LONG_TIMEOUT_MS);
// Assume that no data are removed, db size shall be over the threshold.
assertTrue(mService.isDbSizeOverThreshold());
// Reset the flag and test interruption during maintenance.
im.setInterrupted(false);
final ConditionVariable latch = new ConditionVariable();
// Do full maintenance and the expectation is not completed by interruption.
mService.fullMaintenance(onStatus((status) -> {
assertFalse(status.isSuccess());
latch.open();
}), im);
// Give a little bit of time for maintenance to start up for realism
waitForMs(50);
// Interrupt maintenance job.
im.setInterrupted(true);
if (!latch.block(LONG_TIMEOUT_MS)) {
fail("Maintenance unexpectedly completed successfully");
}
// Assume that only do dropAllExpiredRecords method in previous maintenance, db size shall
// still be over the threshold.
assertTrue(mService.isDbSizeOverThreshold());
}
}