From 506e3f17c5bcb5f4e5f4c104518a8f0c9d919339 Mon Sep 17 00:00:00 2001 From: Edgar Arriaga Date: Tue, 12 Apr 2022 18:29:53 -0700 Subject: [PATCH] Add sql statement cache miss info and test This patch adds a test for checking that statement cache works and it also improves dumpsys diagnostics to include per database stats as previously it only showed per connection stats which could have data loss if a connection was dropped for any reason and not put back in the pool and it also provides the aggregated view. Test: dumpsys meminfo Test: atest SQLiteDatabasePerfTest#testSelectCacheMissRate Bug: 228498581 Change-Id: Ia49264e8f0e6db83234c2ba5943fd9e25fd5d8b2 --- .../database/SQLiteDatabasePerfTest.java | 47 +++++++++++++++++++ core/api/test-current.txt | 4 +- core/java/android/app/ActivityThread.java | 44 +++++++++++++---- .../database/sqlite/SQLiteConnection.java | 11 +++-- .../database/sqlite/SQLiteConnectionPool.java | 21 +++++++++ .../database/sqlite/SQLiteDatabase.java | 28 +++++++++++ .../android/database/sqlite/SQLiteDebug.java | 22 ++++++--- .../server/activitymanagerservice.proto | 3 ++ 8 files changed, 159 insertions(+), 21 deletions(-) diff --git a/apct-tests/perftests/core/src/android/database/SQLiteDatabasePerfTest.java b/apct-tests/perftests/core/src/android/database/SQLiteDatabasePerfTest.java index 762b16c23ef79..b7460cd6ead45 100644 --- a/apct-tests/perftests/core/src/android/database/SQLiteDatabasePerfTest.java +++ b/apct-tests/perftests/core/src/android/database/SQLiteDatabasePerfTest.java @@ -24,10 +24,12 @@ import android.content.Context; import android.database.sqlite.SQLiteDatabase; import android.perftests.utils.BenchmarkState; import android.perftests.utils.PerfStatusReporter; +import android.util.Log; import androidx.test.InstrumentationRegistry; import androidx.test.filters.LargeTest; import androidx.test.runner.AndroidJUnit4; import java.io.File; +import java.util.ArrayList; import java.util.Random; import org.junit.After; import org.junit.Before; @@ -106,6 +108,51 @@ public class SQLiteDatabasePerfTest { } } + @Test + public void testSelectCacheMissRate() { + BenchmarkState state = mPerfStatusReporter.getBenchmarkState(); + + insertT1TestDataSet(); + + ArrayList queryPool = new ArrayList<>(); + queryPool.add("SELECT _ID, COL_A, COL_B, COL_C FROM T1 WHERE _ID=?"); + queryPool.add("SELECT _ID FROM T1 WHERE _ID=?"); + queryPool.add("SELECT COL_A FROM T1 WHERE _ID=?"); + queryPool.add("SELECT COL_B FROM T1 WHERE _ID=?"); + queryPool.add("SELECT COL_C FROM T1 WHERE _ID=?"); + queryPool.add("SELECT _ID, COL_A FROM T1 WHERE _ID=?"); + queryPool.add("SELECT _ID, COL_B FROM T1 WHERE _ID=?"); + queryPool.add("SELECT _ID, COL_C FROM T1 WHERE _ID=?"); + queryPool.add("SELECT COL_A, COL_B FROM T1 WHERE _ID=?"); + queryPool.add("SELECT COL_A, COL_C FROM T1 WHERE _ID=?"); + queryPool.add("SELECT COL_B, COL_C FROM T1 WHERE _ID=?"); + while (state.keepRunning()) { + Random rnd = new Random(0); + + int queries = 1000; + for (int iQuery = 0; iQuery < queries; ++iQuery) { + int queryIndex = rnd.nextInt(queryPool.size()); + int index = rnd.nextInt(DEFAULT_DATASET_SIZE); + + try (Cursor cursor = mDatabase.rawQuery( + queryPool.get(queryIndex), new String[] {String.valueOf(index)})) { + assertTrue(cursor.moveToNext()); + } + } + } + + Log.d("testSelectMemory", + "cacheMissRate: " + mDatabase.getStatementCacheMissRate() + + "Total Statements: " + mDatabase.getTotalPreparedStatements() + + ". Misses: " + mDatabase.getTotalStatementCacheMisses()); + + // Make sure caching is working and our miss rate should definitely be less than 100% + // however, we would expect this number to be actually closer to 0. + assertTrue(mDatabase.getStatementCacheMissRate() < 1); + mDatabase.close(); + mContext.deleteDatabase(DB_NAME); + } + @Test public void testSelectMultipleRows() { insertT1TestDataSet(); diff --git a/core/api/test-current.txt b/core/api/test-current.txt index 3a808c796634d..631f0fbd0b0cf 100644 --- a/core/api/test-current.txt +++ b/core/api/test-current.txt @@ -975,8 +975,8 @@ package android.database.sqlite { } public static class SQLiteDebug.DbStats { - ctor public SQLiteDebug.DbStats(String, long, long, int, int, int, int); - field public String cache; + ctor public SQLiteDebug.DbStats(@NonNull String, long, long, int, int, int, int, boolean); + field public final boolean arePoolStats; field public String dbName; field public long dbSize; field public int lookaside; diff --git a/core/java/android/app/ActivityThread.java b/core/java/android/app/ActivityThread.java index 05df1065ad30d..8a9ec0984ec78 100644 --- a/core/java/android/app/ActivityThread.java +++ b/core/java/android/app/ActivityThread.java @@ -1026,7 +1026,10 @@ public final class ActivityThread extends ClientTransactionHandler } private class ApplicationThread extends IApplicationThread.Stub { - private static final String DB_INFO_FORMAT = " %8s %8s %14s %14s %s"; + private static final String DB_CONNECTION_INFO_HEADER = " %8s %8s %14s %5s %5s %5s %s"; + private static final String DB_CONNECTION_INFO_FORMAT = " %8s %8s %14s %5d %5d %5d %s"; + private static final String DB_POOL_INFO_HEADER = " %13s %13s %13s %s"; + private static final String DB_POOL_INFO_FORMAT = " %13d %13d %13d %s"; public final void scheduleReceiver(Intent intent, ActivityInfo info, CompatibilityInfo compatInfo, int resultCode, String data, Bundle extras, @@ -1454,8 +1457,9 @@ public final class ActivityThread extends ClientTransactionHandler pw.print(','); pw.print(dbStats.pageSize); pw.print(','); pw.print(dbStats.dbSize); pw.print(','); pw.print(dbStats.lookaside); - pw.print(','); pw.print(dbStats.cache); - pw.print(','); pw.print(dbStats.cache); + pw.print(','); pw.print(dbStats.cacheHits); + pw.print(','); pw.print(dbStats.cacheMisses); + pw.print(','); pw.print(dbStats.cacheSize); } pw.println(); @@ -1490,15 +1494,34 @@ public final class ActivityThread extends ClientTransactionHandler int N = stats.dbStats.size(); if (N > 0) { pw.println(" DATABASES"); - printRow(pw, DB_INFO_FORMAT, "pgsz", "dbsz", "Lookaside(b)", "cache", - "Dbname"); + printRow(pw, DB_CONNECTION_INFO_HEADER, "pgsz", "dbsz", "Lookaside(b)", + "cache hits", "cache misses", "cache size", "Dbname"); + pw.println("PER CONNECTION STATS"); for (int i = 0; i < N; i++) { DbStats dbStats = stats.dbStats.get(i); - printRow(pw, DB_INFO_FORMAT, + if (dbStats.arePoolStats) { + // these will be printed after + continue; + } + printRow(pw, DB_CONNECTION_INFO_FORMAT, (dbStats.pageSize > 0) ? String.valueOf(dbStats.pageSize) : " ", (dbStats.dbSize > 0) ? String.valueOf(dbStats.dbSize) : " ", (dbStats.lookaside > 0) ? String.valueOf(dbStats.lookaside) : " ", - dbStats.cache, dbStats.dbName); + dbStats.cacheHits, dbStats.cacheMisses, dbStats.cacheSize, + dbStats.dbName); + } + // Print stats accumulated through all the connections that have existed in the + // pool since it was opened. + pw.println("POOL STATS"); + printRow(pw, DB_POOL_INFO_HEADER, "cache hits", "cache misses", "cache size", + "Dbname"); + for (int i = 0; i < N; i++) { + DbStats dbStats = stats.dbStats.get(i); + if (!dbStats.arePoolStats) { + continue; + } + printRow(pw, DB_POOL_INFO_FORMAT, dbStats.cacheHits, dbStats.cacheMisses, + dbStats.cacheSize, dbStats.dbName); } } @@ -1623,7 +1646,12 @@ public final class ActivityThread extends ClientTransactionHandler proto.write(MemInfoDumpProto.AppData.SqlStats.Database.DB_SIZE, dbStats.dbSize); proto.write(MemInfoDumpProto.AppData.SqlStats.Database.LOOKASIDE_B, dbStats.lookaside); - proto.write(MemInfoDumpProto.AppData.SqlStats.Database.CACHE, dbStats.cache); + proto.write( + MemInfoDumpProto.AppData.SqlStats.Database.CACHE_HITS, dbStats.cacheHits); + proto.write(MemInfoDumpProto.AppData.SqlStats.Database.CACHE_MISSES, + dbStats.cacheMisses); + proto.write( + MemInfoDumpProto.AppData.SqlStats.Database.CACHE_SIZE, dbStats.cacheSize); proto.end(dToken); } proto.end(sToken); diff --git a/core/java/android/database/sqlite/SQLiteConnection.java b/core/java/android/database/sqlite/SQLiteConnection.java index 6d6ec06182d65..4b3eb3a000ff6 100644 --- a/core/java/android/database/sqlite/SQLiteConnection.java +++ b/core/java/android/database/sqlite/SQLiteConnection.java @@ -1053,6 +1053,7 @@ public final class SQLiteConnection implements CancellationSignal.OnCancelListen } private PreparedStatement acquirePreparedStatement(String sql) { + ++mPool.mTotalPrepareStatements; PreparedStatement statement = mPreparedStatementCache.get(sql); boolean skipCache = false; if (statement != null) { @@ -1064,7 +1065,7 @@ public final class SQLiteConnection implements CancellationSignal.OnCancelListen // statement but do not cache it. skipCache = true; } - + ++mPool.mTotalPrepareStatementCacheMiss; final long statementPtr = nativePrepareStatement(mConnectionPtr, sql); try { final int numParameters = nativeGetParameterCount(mConnectionPtr, statementPtr); @@ -1320,7 +1321,8 @@ public final class SQLiteConnection implements CancellationSignal.OnCancelListen if (!path.isEmpty()) { label.append(": ").append(path); } - dbStatsList.add(new DbStats(label.toString(), pageCount, pageSize, 0, 0, 0, 0)); + dbStatsList.add( + new DbStats(label.toString(), pageCount, pageSize, 0, 0, 0, 0, false)); } } catch (SQLiteException ex) { // Ignore. @@ -1349,9 +1351,8 @@ public final class SQLiteConnection implements CancellationSignal.OnCancelListen label = mConfiguration.path + " (" + mConnectionId + ")"; } return new DbStats(label, pageCount, pageSize, lookaside, - mPreparedStatementCache.hitCount(), - mPreparedStatementCache.missCount(), - mPreparedStatementCache.size()); + mPreparedStatementCache.hitCount(), mPreparedStatementCache.missCount(), + mPreparedStatementCache.size(), false); } @Override diff --git a/core/java/android/database/sqlite/SQLiteConnectionPool.java b/core/java/android/database/sqlite/SQLiteConnectionPool.java index db0cac3eb9c7d..069c264313e79 100644 --- a/core/java/android/database/sqlite/SQLiteConnectionPool.java +++ b/core/java/android/database/sqlite/SQLiteConnectionPool.java @@ -32,6 +32,7 @@ import android.util.Printer; import com.android.internal.annotations.GuardedBy; import com.android.internal.annotations.VisibleForTesting; +import dalvik.annotation.optimization.NeverCompile; import dalvik.system.CloseGuard; import java.io.Closeable; @@ -103,6 +104,10 @@ public final class SQLiteConnectionPool implements Closeable { new ArrayList(); private SQLiteConnection mAvailablePrimaryConnection; + // Prepare statement cache statistics + public int mTotalPrepareStatementCacheMiss = 0; + public int mTotalPrepareStatements = 0; + @GuardedBy("mLock") private IdleConnectionHandler mIdleConnectionHandler; @@ -507,6 +512,12 @@ public final class SQLiteConnectionPool implements Closeable { for (SQLiteConnection connection : mAcquiredConnections.keySet()) { connection.collectDbStatsUnsafe(dbStatsList); } + + // Global pool stats + DbStats poolStats = new DbStats(mConfiguration.path, 0, 0, 0, + mTotalPrepareStatements - mTotalPrepareStatementCacheMiss, + mTotalPrepareStatementCacheMiss, mTotalPrepareStatements, true); + dbStatsList.add(poolStats); } } @@ -1203,6 +1214,16 @@ public final class SQLiteConnectionPool implements Closeable { } } + /** @hide */ + @NeverCompile + public double getStatementCacheMissRate() { + if (mTotalPrepareStatements == 0) { + // no statements executed thus no miss rate. + return 0; + } + return (double) mTotalPrepareStatementCacheMiss / (double) mTotalPrepareStatements; + } + public long getTotalStatementsTime() { return mTotalStatementsTime.get(); } diff --git a/core/java/android/database/sqlite/SQLiteDatabase.java b/core/java/android/database/sqlite/SQLiteDatabase.java index 0d0615a28af3d..c08294f42c45d 100644 --- a/core/java/android/database/sqlite/SQLiteDatabase.java +++ b/core/java/android/database/sqlite/SQLiteDatabase.java @@ -45,6 +45,8 @@ import android.util.Log; import android.util.Pair; import android.util.Printer; import com.android.internal.util.Preconditions; + +import dalvik.annotation.optimization.NeverCompile; import dalvik.system.CloseGuard; import java.io.File; import java.io.FileFilter; @@ -2233,6 +2235,16 @@ public final class SQLiteDatabase extends SQLiteClosable { } } + /** @hide */ + @NeverCompile + public double getStatementCacheMissRate() { + synchronized (mLock) { + throwIfNotOpenLocked(); + + return mConnectionPoolLocked.getStatementCacheMissRate(); + } + } + /** * Sets whether foreign key constraints are enabled for the database. *

@@ -2489,6 +2501,22 @@ public final class SQLiteDatabase extends SQLiteClosable { return connectionPools; } + /** @hide */ + @NeverCompile + public int getTotalPreparedStatements() { + throwIfNotOpenLocked(); + + return mConnectionPoolLocked.mTotalPrepareStatements; + } + + /** @hide */ + @NeverCompile + public int getTotalStatementCacheMisses() { + throwIfNotOpenLocked(); + + return mConnectionPoolLocked.mTotalPrepareStatementCacheMiss; + } + /** * Dump detailed information about all open databases in the current process. * Used by bug report. diff --git a/core/java/android/database/sqlite/SQLiteDebug.java b/core/java/android/database/sqlite/SQLiteDebug.java index b84a8d24d42a3..93d74b1fbdff5 100644 --- a/core/java/android/database/sqlite/SQLiteDebug.java +++ b/core/java/android/database/sqlite/SQLiteDebug.java @@ -16,6 +16,7 @@ package android.database.sqlite; +import android.annotation.NonNull; import android.annotation.TestApi; import android.compat.annotation.UnsupportedAppUsage; import android.os.Build; @@ -23,7 +24,6 @@ import android.os.Process; import android.os.SystemProperties; import android.util.Log; import android.util.Printer; - import java.util.ArrayList; /** @@ -173,16 +173,26 @@ public final class SQLiteDebug { @UnsupportedAppUsage public int lookaside; - /** statement cache stats: hits/misses/cachesize */ - public String cache; + /** @hide */ + final public int cacheHits; + /** @hide */ + final public int cacheMisses; + /** @hide */ + final public int cacheSize; - public DbStats(String dbName, long pageCount, long pageSize, int lookaside, - int hits, int misses, int cachesize) { + /** true if connection specific stats or whole connection pool if false */ + public final boolean arePoolStats; + + public DbStats(@NonNull String dbName, long pageCount, long pageSize, int lookaside, + int hits, int misses, int cachesize, boolean arePoolStats) { this.dbName = dbName; this.pageSize = pageSize / 1024; dbSize = (pageCount * pageSize) / 1024; this.lookaside = lookaside; - this.cache = hits + "/" + misses + "/" + cachesize; + this.cacheHits = hits; + this.cacheMisses = misses; + this.cacheSize = cachesize; + this.arePoolStats = arePoolStats; } } diff --git a/core/proto/android/server/activitymanagerservice.proto b/core/proto/android/server/activitymanagerservice.proto index 6ec71baa836a7..5099dd20a6d55 100644 --- a/core/proto/android/server/activitymanagerservice.proto +++ b/core/proto/android/server/activitymanagerservice.proto @@ -266,6 +266,9 @@ message MemInfoDumpProto { optional int32 lookaside_b = 4; // Statement cache stats: hits/misses/cachesize optional string cache = 5; + optional int32 cache_hits = 6; + optional int32 cache_misses = 7; + optional int32 cache_size = 8; } repeated Database databases = 4; }