diff --git a/core/java/android/privacy/internal/longitudinalreporting/LongitudinalReportingEncoder.java b/core/java/android/privacy/internal/longitudinalreporting/LongitudinalReportingEncoder.java index 219868d663d23..dd97f1ef6a022 100644 --- a/core/java/android/privacy/internal/longitudinalreporting/LongitudinalReportingEncoder.java +++ b/core/java/android/privacy/internal/longitudinalreporting/LongitudinalReportingEncoder.java @@ -19,6 +19,7 @@ package android.privacy.internal.longitudinalreporting; import android.privacy.DifferentialPrivacyEncoder; import android.privacy.internal.rappor.RapporConfig; import android.privacy.internal.rappor.RapporEncoder; +import android.util.Log; import com.android.internal.annotations.VisibleForTesting; @@ -48,6 +49,9 @@ import com.android.internal.annotations.VisibleForTesting; */ public class LongitudinalReportingEncoder implements DifferentialPrivacyEncoder { + private static final String TAG = "LongitudinalEncoder"; + private static final boolean DEBUG = false; + // Suffix that will be added to Rappor's encoder id. There's a (relatively) small risk some // other Rappor encoder may re-use the same encoder id. private static final String PRR1_ENCODER_ID = "prr1_encoder_id"; @@ -121,11 +125,18 @@ public class LongitudinalReportingEncoder implements DifferentialPrivacyEncoder @Override public byte[] encodeBoolean(boolean original) { + if (DEBUG) { + Log.d(TAG, "encodeBoolean, encoderId:" + mConfig.getEncoderId() + ", original: " + + original); + } if (mFakeValue != null) { // Use the fake value generated in PRR. original = mFakeValue.booleanValue(); + if (DEBUG) Log.d(TAG, "Use fake value: " + original); } - return mIRREncoder.encodeBoolean(original); + byte[] result = mIRREncoder.encodeBoolean(original); + if (DEBUG) Log.d(TAG, "result: " + ((result[0] & 0x1) != 0)); + return result; } @Override diff --git a/services/core/java/com/android/server/net/watchlist/NetworkWatchlistService.java b/services/core/java/com/android/server/net/watchlist/NetworkWatchlistService.java index 6907c58f13b02..29b1339e8022c 100644 --- a/services/core/java/com/android/server/net/watchlist/NetworkWatchlistService.java +++ b/services/core/java/com/android/server/net/watchlist/NetworkWatchlistService.java @@ -171,7 +171,7 @@ public class NetworkWatchlistService extends INetworkWatchlistManager.Stub { Slog.w(TAG, "Only shell is allowed to call network watchlist shell commands"); return; } - (new NetworkWatchlistShellCommand(mContext)).exec(this, in, out, err, args, callback, + (new NetworkWatchlistShellCommand(this, mContext)).exec(this, in, out, err, args, callback, resultReceiver); } @@ -262,6 +262,21 @@ public class NetworkWatchlistService extends INetworkWatchlistManager.Stub { mNetworkWatchlistHandler.reportWatchlistIfNecessary(); } + /** + * Force generate watchlist report for testing. + * + * @param lastReportTime Watchlist report will cotain all records before this time. + * @return True if operation success. + */ + public boolean forceReportWatchlistForTest(long lastReportTime) { + if (mConfig.isConfigSecure()) { + // Should not force generate report under production config. + return false; + } + mNetworkWatchlistHandler.forceReportWatchlistForTest(lastReportTime); + return true; + } + @Override protected void dump(FileDescriptor fd, PrintWriter pw, String[] args) { if (!DumpUtils.checkDumpPermission(mContext, TAG, pw)) return; diff --git a/services/core/java/com/android/server/net/watchlist/NetworkWatchlistShellCommand.java b/services/core/java/com/android/server/net/watchlist/NetworkWatchlistShellCommand.java index 9533823df8084..17c5868a53f75 100644 --- a/services/core/java/com/android/server/net/watchlist/NetworkWatchlistShellCommand.java +++ b/services/core/java/com/android/server/net/watchlist/NetworkWatchlistShellCommand.java @@ -19,9 +19,11 @@ package com.android.server.net.watchlist; import android.content.Context; import android.content.Intent; import android.net.NetworkWatchlistManager; +import android.os.Binder; import android.os.ParcelFileDescriptor; import android.os.RemoteException; import android.os.ShellCommand; +import android.provider.Settings; import java.io.FileInputStream; import java.io.IOException; @@ -34,10 +36,12 @@ import java.io.PrintWriter; */ class NetworkWatchlistShellCommand extends ShellCommand { - final NetworkWatchlistManager mNetworkWatchlistManager; + final Context mContext; + final NetworkWatchlistService mService; - NetworkWatchlistShellCommand(Context context) { - mNetworkWatchlistManager = new NetworkWatchlistManager(context); + NetworkWatchlistShellCommand(NetworkWatchlistService service, Context context) { + mContext = context; + mService = service; } @Override @@ -51,11 +55,13 @@ class NetworkWatchlistShellCommand extends ShellCommand { switch(cmd) { case "set-test-config": return runSetTestConfig(); + case "force-generate-report": + return runForceGenerateReport(); default: return handleDefaultCommands(cmd); } - } catch (RemoteException e) { - pw.println("Remote exception: " + e); + } catch (Exception e) { + pw.println("Exception: " + e); } return -1; } @@ -73,22 +79,44 @@ class NetworkWatchlistShellCommand extends ShellCommand { WatchlistConfig.getInstance().setTestMode(fileStream); } pw.println("Success!"); - } catch (RuntimeException | IOException ex) { + } catch (Exception ex) { pw.println("Error: " + ex.toString()); return -1; } return 0; } + private int runForceGenerateReport() throws RemoteException { + final PrintWriter pw = getOutPrintWriter(); + final long ident = Binder.clearCallingIdentity(); + try { + // Reset last report time + if (!WatchlistConfig.getInstance().isConfigSecure()) { + pw.println("Error: Cannot force generate report under production config"); + return -1; + } + Settings.Global.putLong(mContext.getContentResolver(), + Settings.Global.NETWORK_WATCHLIST_LAST_REPORT_TIME, 0L); + mService.forceReportWatchlistForTest(System.currentTimeMillis()); + pw.println("Success!"); + } catch (Exception ex) { + pw.println("Error: " + ex); + return -1; + } finally { + Binder.restoreCallingIdentity(ident); + } + return 0; + } + @Override public void onHelp() { final PrintWriter pw = getOutPrintWriter(); pw.println("Network watchlist manager commands:"); pw.println(" help"); pw.println(" Print this help text."); - pw.println(""); pw.println(" set-test-config your_watchlist_config.xml"); - pw.println(); - Intent.printIntentArgsHelp(pw , ""); + pw.println(" Set network watchlist test config file."); + pw.println(" force-generate-report"); + pw.println(" Force generate watchlist test report."); } } diff --git a/services/core/java/com/android/server/net/watchlist/PrivacyUtils.java b/services/core/java/com/android/server/net/watchlist/PrivacyUtils.java index c1231fa342e76..408a9ed31d188 100644 --- a/services/core/java/com/android/server/net/watchlist/PrivacyUtils.java +++ b/services/core/java/com/android/server/net/watchlist/PrivacyUtils.java @@ -19,6 +19,7 @@ package com.android.server.net.watchlist; import android.privacy.DifferentialPrivacyEncoder; import android.privacy.internal.longitudinalreporting.LongitudinalReportingConfig; import android.privacy.internal.longitudinalreporting.LongitudinalReportingEncoder; +import android.util.Slog; import com.android.internal.annotations.VisibleForTesting; @@ -32,6 +33,7 @@ import java.util.Map; class PrivacyUtils { private static final String TAG = "PrivacyUtils"; + private static final boolean DEBUG = NetworkWatchlistService.DEBUG; /** * Parameters used for encoding watchlist reports. @@ -84,6 +86,7 @@ class PrivacyUtils { @VisibleForTesting static Map createDpEncodedReportMap(boolean isSecure, byte[] userSecret, List appDigestList, WatchlistReportDbHelper.AggregatedResult aggregatedResult) { + if (DEBUG) Slog.i(TAG, "createDpEncodedReportMap start"); final int appDigestListSize = appDigestList.size(); final HashMap resultMap = new HashMap<>(appDigestListSize); for (int i = 0; i < appDigestListSize; i++) { @@ -93,6 +96,7 @@ class PrivacyUtils { ? createSecureDPEncoder(userSecret, appDigest) : createInsecureDPEncoderForTest(appDigest); final boolean visitedWatchlist = aggregatedResult.appDigestList.contains(appDigest); + if (DEBUG) Slog.i(TAG, appDigest + ": " + visitedWatchlist); // Get the least significant bit of first byte, and set result to True if it is 1 boolean encodedVisitedWatchlist = ((int) encoder.encodeBoolean(visitedWatchlist)[0] & 0x1) == 0x1; diff --git a/services/core/java/com/android/server/net/watchlist/WatchlistLoggingHandler.java b/services/core/java/com/android/server/net/watchlist/WatchlistLoggingHandler.java index e8b39c04a8308..b331b9c0d9ae4 100644 --- a/services/core/java/com/android/server/net/watchlist/WatchlistLoggingHandler.java +++ b/services/core/java/com/android/server/net/watchlist/WatchlistLoggingHandler.java @@ -43,6 +43,7 @@ import java.io.File; import java.io.IOException; import java.security.NoSuchAlgorithmException; import java.util.ArrayList; +import java.util.GregorianCalendar; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -61,6 +62,8 @@ class WatchlistLoggingHandler extends Handler { static final int LOG_WATCHLIST_EVENT_MSG = 1; @VisibleForTesting static final int REPORT_RECORDS_IF_NECESSARY_MSG = 2; + @VisibleForTesting + static final int FORCE_REPORT_RECORDS_NOW_FOR_TEST_MSG = 3; private static final long ONE_DAY_MS = TimeUnit.DAYS.toMillis(1); private static final String DROPBOX_TAG = "network_watchlist_report"; @@ -110,7 +113,15 @@ class WatchlistLoggingHandler extends Handler { break; } case REPORT_RECORDS_IF_NECESSARY_MSG: - tryAggregateRecords(); + tryAggregateRecords(getLastMidnightTime()); + break; + case FORCE_REPORT_RECORDS_NOW_FOR_TEST_MSG: + if (msg.obj instanceof Long) { + long lastRecordTime = (Long) msg.obj; + tryAggregateRecords(lastRecordTime); + } else { + Slog.e(TAG, "Msg.obj needs to be a Long object."); + } break; default: { Slog.d(TAG, "WatchlistLoggingHandler received an unknown of message."); @@ -146,6 +157,12 @@ class WatchlistLoggingHandler extends Handler { sendMessage(msg); } + public void forceReportWatchlistForTest(long lastReportTime) { + final Message msg = obtainMessage(FORCE_REPORT_RECORDS_NOW_FOR_TEST_MSG); + msg.obj = lastReportTime; + sendMessage(msg); + } + /** * Insert network traffic event to watchlist async queue processor. */ @@ -177,8 +194,14 @@ class WatchlistLoggingHandler extends Handler { } private boolean insertRecord(int uid, String cncHost, long timestamp) { + if (DEBUG) { + Slog.i(TAG, "trying to insert record with host: " + cncHost + ", uid: " + uid); + } if (!mConfig.isConfigSecure() && !isPackageTestOnly(uid)) { // Skip package if config is not secure and package is not TestOnly app. + if (DEBUG) { + Slog.i(TAG, "uid: " + uid + " is not test only package"); + } return true; } final byte[] digest = getDigestFromUid(uid); @@ -187,50 +210,56 @@ class WatchlistLoggingHandler extends Handler { return false; } final boolean result = mDbHelper.insertNewRecord(digest, cncHost, timestamp); - tryAggregateRecords(); return result; } - private boolean shouldReportNetworkWatchlist() { + private boolean shouldReportNetworkWatchlist(long lastRecordTime) { final long lastReportTime = Settings.Global.getLong(mResolver, Settings.Global.NETWORK_WATCHLIST_LAST_REPORT_TIME, 0L); - final long currentTimestamp = System.currentTimeMillis(); - if (currentTimestamp < lastReportTime) { + if (lastRecordTime < lastReportTime) { Slog.i(TAG, "Last report time is larger than current time, reset report"); - mDbHelper.cleanup(); + mDbHelper.cleanup(lastReportTime); return false; } - return currentTimestamp >= lastReportTime + ONE_DAY_MS; + return lastRecordTime >= lastReportTime + ONE_DAY_MS; } - private void tryAggregateRecords() { - // Check if it's necessary to generate watchlist report now. - if (!shouldReportNetworkWatchlist()) { - Slog.i(TAG, "No need to aggregate record yet."); - return; - } - Slog.i(TAG, "Start aggregating watchlist records."); - if (mDropBoxManager != null && mDropBoxManager.isTagEnabled(DROPBOX_TAG)) { - Settings.Global.putLong(mResolver, - Settings.Global.NETWORK_WATCHLIST_LAST_REPORT_TIME, - System.currentTimeMillis()); - final WatchlistReportDbHelper.AggregatedResult aggregatedResult = - mDbHelper.getAggregatedRecords(); - if (aggregatedResult == null) { - Slog.i(TAG, "Cannot get result from database"); + private void tryAggregateRecords(long lastRecordTime) { + long startTime = System.currentTimeMillis(); + try { + // Check if it's necessary to generate watchlist report now. + if (!shouldReportNetworkWatchlist(lastRecordTime)) { + Slog.i(TAG, "No need to aggregate record yet."); return; } - // Get all digests for watchlist report, it should include all installed - // application digests and previously recorded app digests. - final List digestsForReport = getAllDigestsForReport(aggregatedResult); - final byte[] secretKey = mSettings.getPrivacySecretKey(); - final byte[] encodedResult = ReportEncoder.encodeWatchlistReport(mConfig, - secretKey, digestsForReport, aggregatedResult); - if (encodedResult != null) { - addEncodedReportToDropBox(encodedResult); + Slog.i(TAG, "Start aggregating watchlist records."); + if (mDropBoxManager != null && mDropBoxManager.isTagEnabled(DROPBOX_TAG)) { + Settings.Global.putLong(mResolver, + Settings.Global.NETWORK_WATCHLIST_LAST_REPORT_TIME, + lastRecordTime); + final WatchlistReportDbHelper.AggregatedResult aggregatedResult = + mDbHelper.getAggregatedRecords(lastRecordTime); + if (aggregatedResult == null) { + Slog.i(TAG, "Cannot get result from database"); + return; + } + // Get all digests for watchlist report, it should include all installed + // application digests and previously recorded app digests. + final List digestsForReport = getAllDigestsForReport(aggregatedResult); + final byte[] secretKey = mSettings.getPrivacySecretKey(); + final byte[] encodedResult = ReportEncoder.encodeWatchlistReport(mConfig, + secretKey, digestsForReport, aggregatedResult); + if (encodedResult != null) { + addEncodedReportToDropBox(encodedResult); + } + } else { + Slog.w(TAG, "Network Watchlist dropbox tag is not enabled"); } + mDbHelper.cleanup(lastRecordTime); + } finally { + long endTime = System.currentTimeMillis(); + Slog.i(TAG, "Milliseconds spent on tryAggregateRecords(): " + (endTime - startTime)); } - mDbHelper.cleanup(); } /** @@ -379,4 +408,19 @@ class WatchlistLoggingHandler extends Handler { } return subDomainList.toArray(new String[0]); } + + static long getLastMidnightTime() { + return getMidnightTimestamp(0); + } + + static long getMidnightTimestamp(int daysBefore) { + java.util.Calendar date = new GregorianCalendar(); + // reset hour, minutes, seconds and millis + date.set(java.util.Calendar.HOUR_OF_DAY, 0); + date.set(java.util.Calendar.MINUTE, 0); + date.set(java.util.Calendar.SECOND, 0); + date.set(java.util.Calendar.MILLISECOND, 0); + date.add(java.util.Calendar.DAY_OF_MONTH, -daysBefore); + return date.getTimeInMillis(); + } } diff --git a/services/core/java/com/android/server/net/watchlist/WatchlistReportDbHelper.java b/services/core/java/com/android/server/net/watchlist/WatchlistReportDbHelper.java index 4b577bb9c919f..632ab81b131e0 100644 --- a/services/core/java/com/android/server/net/watchlist/WatchlistReportDbHelper.java +++ b/services/core/java/com/android/server/net/watchlist/WatchlistReportDbHelper.java @@ -141,11 +141,10 @@ class WatchlistReportDbHelper extends SQLiteOpenHelper { } /** - * Aggregate all records before most recent local midnight in database, and return a + * Aggregate all records in database before input timestamp, and return a * rappor encoded result. */ - public AggregatedResult getAggregatedRecords() { - final long lastMidnightTime = getLastMidnightTime(); + public AggregatedResult getAggregatedRecords(long untilTimestamp) { final String selectStatement = WhiteListReportContract.TIMESTAMP + " < ?"; final SQLiteDatabase db = getReadableDatabase(); @@ -153,7 +152,7 @@ class WatchlistReportDbHelper extends SQLiteOpenHelper { try { c = db.query(true /* distinct */, WhiteListReportContract.TABLE, DIGEST_DOMAIN_PROJECTION, selectStatement, - new String[]{"" + lastMidnightTime}, null, null, + new String[]{Long.toString(untilTimestamp)}, null, null, null, null); if (c == null) { return null; @@ -181,29 +180,13 @@ class WatchlistReportDbHelper extends SQLiteOpenHelper { } /** - * Remove all the records before most recent local midnight. + * Remove all the records before input timestamp. * * @return True if success. */ - public boolean cleanup() { + public boolean cleanup(long untilTimestamp) { final SQLiteDatabase db = getWritableDatabase(); - final long midnightTime = getLastMidnightTime(); - final String clause = WhiteListReportContract.TIMESTAMP + "< " + midnightTime; + final String clause = WhiteListReportContract.TIMESTAMP + "< " + untilTimestamp; return db.delete(WhiteListReportContract.TABLE, clause, null) != 0; } - - static long getLastMidnightTime() { - return getMidnightTimestamp(0); - } - - static long getMidnightTimestamp(int daysBefore) { - java.util.Calendar date = new GregorianCalendar(); - // reset hour, minutes, seconds and millis - date.set(java.util.Calendar.HOUR_OF_DAY, 0); - date.set(java.util.Calendar.MINUTE, 0); - date.set(java.util.Calendar.SECOND, 0); - date.set(java.util.Calendar.MILLISECOND, 0); - date.add(java.util.Calendar.DAY_OF_MONTH, -daysBefore); - return date.getTimeInMillis(); - } } \ No newline at end of file