f6d9f9d84d
Test device config changes are propagated correctly to the watchdog. * Keep polling until the config value becomes the expected one. * Remove listeners at the end of each test - too many device config listeners will delay the notification significantly ( > 1 min) according to my local experiments. Bug: 181820350 Test: atest PackageWatchdogTest Change-Id: Ia5e89b69a8052f49e6400f6b22313249d523786c
1595 lines
69 KiB
Java
1595 lines
69 KiB
Java
/*
|
|
* Copyright (C) 2019 The Android Open Source Project
|
|
*
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*/
|
|
|
|
package com.android.server;
|
|
|
|
import static android.service.watchdog.ExplicitHealthCheckService.PackageConfig;
|
|
|
|
import static com.android.dx.mockito.inline.extended.ExtendedMockito.doAnswer;
|
|
|
|
import static com.google.common.truth.Truth.assertThat;
|
|
|
|
import static org.junit.Assert.assertTrue;
|
|
import static org.mockito.ArgumentMatchers.anyInt;
|
|
import static org.mockito.ArgumentMatchers.anyLong;
|
|
import static org.mockito.ArgumentMatchers.anyString;
|
|
import static org.mockito.Mockito.reset;
|
|
import static org.mockito.Mockito.spy;
|
|
import static org.mockito.Mockito.verify;
|
|
import static org.mockito.Mockito.when;
|
|
|
|
import android.Manifest;
|
|
import android.content.Context;
|
|
import android.content.pm.PackageInfo;
|
|
import android.content.pm.PackageManager;
|
|
import android.content.pm.VersionedPackage;
|
|
import android.net.ConnectivityModuleConnector;
|
|
import android.net.ConnectivityModuleConnector.ConnectivityModuleHealthListener;
|
|
import android.os.Handler;
|
|
import android.os.SystemProperties;
|
|
import android.os.test.TestLooper;
|
|
import android.provider.DeviceConfig;
|
|
import android.util.AtomicFile;
|
|
import android.util.LongArrayQueue;
|
|
import android.util.TypedXmlPullParser;
|
|
import android.util.TypedXmlSerializer;
|
|
import android.util.Xml;
|
|
|
|
import androidx.test.InstrumentationRegistry;
|
|
|
|
import com.android.dx.mockito.inline.extended.ExtendedMockito;
|
|
import com.android.internal.util.XmlUtils;
|
|
import com.android.server.PackageWatchdog.HealthCheckState;
|
|
import com.android.server.PackageWatchdog.MonitoredPackage;
|
|
import com.android.server.PackageWatchdog.PackageHealthObserver;
|
|
import com.android.server.PackageWatchdog.PackageHealthObserverImpact;
|
|
|
|
import org.junit.After;
|
|
import org.junit.Before;
|
|
import org.junit.Test;
|
|
import org.mockito.ArgumentCaptor;
|
|
import org.mockito.Captor;
|
|
import org.mockito.Mock;
|
|
import org.mockito.MockitoAnnotations;
|
|
import org.mockito.MockitoSession;
|
|
import org.mockito.quality.Strictness;
|
|
import org.mockito.stubbing.Answer;
|
|
|
|
import java.io.File;
|
|
import java.io.FileOutputStream;
|
|
import java.util.ArrayList;
|
|
import java.util.Arrays;
|
|
import java.util.Collections;
|
|
import java.util.HashMap;
|
|
import java.util.List;
|
|
import java.util.Set;
|
|
import java.util.concurrent.TimeUnit;
|
|
import java.util.function.Consumer;
|
|
import java.util.function.Supplier;
|
|
|
|
/**
|
|
* Test PackageWatchdog.
|
|
*/
|
|
public class PackageWatchdogTest {
|
|
private static final long RETRY_MAX_COUNT = 30;
|
|
private static final long RETRY_TIMEOUT_MILLIS = 500;
|
|
|
|
private static final String APP_A = "com.package.a";
|
|
private static final String APP_B = "com.package.b";
|
|
private static final String APP_C = "com.package.c";
|
|
private static final String APP_D = "com.package.d";
|
|
private static final long VERSION_CODE = 1L;
|
|
private static final String OBSERVER_NAME_1 = "observer1";
|
|
private static final String OBSERVER_NAME_2 = "observer2";
|
|
private static final String OBSERVER_NAME_3 = "observer3";
|
|
private static final String OBSERVER_NAME_4 = "observer4";
|
|
private static final long SHORT_DURATION = TimeUnit.SECONDS.toMillis(1);
|
|
private static final long LONG_DURATION = TimeUnit.SECONDS.toMillis(5);
|
|
private final TestClock mTestClock = new TestClock();
|
|
private TestLooper mTestLooper;
|
|
private Context mSpyContext;
|
|
// Keep track of all created watchdogs to apply device config changes
|
|
private List<PackageWatchdog> mAllocatedWatchdogs;
|
|
@Mock
|
|
private ConnectivityModuleConnector mConnectivityModuleConnector;
|
|
@Mock
|
|
private PackageManager mMockPackageManager;
|
|
@Captor
|
|
private ArgumentCaptor<ConnectivityModuleHealthListener> mConnectivityModuleCallbackCaptor;
|
|
private MockitoSession mSession;
|
|
private HashMap<String, String> mSystemSettingsMap;
|
|
|
|
private boolean retry(Supplier<Boolean> supplier) throws Exception {
|
|
for (int i = 0; i < RETRY_MAX_COUNT; ++i) {
|
|
if (supplier.get()) {
|
|
return true;
|
|
}
|
|
Thread.sleep(RETRY_TIMEOUT_MILLIS);
|
|
}
|
|
return false;
|
|
}
|
|
|
|
@Before
|
|
public void setUp() throws Exception {
|
|
MockitoAnnotations.initMocks(this);
|
|
new File(InstrumentationRegistry.getContext().getFilesDir(),
|
|
"package-watchdog.xml").delete();
|
|
adoptShellPermissions(Manifest.permission.READ_DEVICE_CONFIG,
|
|
Manifest.permission.WRITE_DEVICE_CONFIG);
|
|
mTestLooper = new TestLooper();
|
|
mSpyContext = spy(InstrumentationRegistry.getContext());
|
|
when(mSpyContext.getPackageManager()).thenReturn(mMockPackageManager);
|
|
when(mMockPackageManager.getPackageInfo(anyString(), anyInt())).then(inv -> {
|
|
final PackageInfo res = new PackageInfo();
|
|
res.packageName = inv.getArgument(0);
|
|
res.setLongVersionCode(VERSION_CODE);
|
|
return res;
|
|
});
|
|
mSession = ExtendedMockito.mockitoSession()
|
|
.initMocks(this)
|
|
.strictness(Strictness.LENIENT)
|
|
.spyStatic(SystemProperties.class)
|
|
.startMocking();
|
|
mSystemSettingsMap = new HashMap<>();
|
|
|
|
|
|
// Mock SystemProperties setter and various getters
|
|
doAnswer((Answer<Void>) invocationOnMock -> {
|
|
String key = invocationOnMock.getArgument(0);
|
|
String value = invocationOnMock.getArgument(1);
|
|
|
|
mSystemSettingsMap.put(key, value);
|
|
return null;
|
|
}
|
|
).when(() -> SystemProperties.set(anyString(), anyString()));
|
|
|
|
doAnswer((Answer<Integer>) invocationOnMock -> {
|
|
String key = invocationOnMock.getArgument(0);
|
|
int defaultValue = invocationOnMock.getArgument(1);
|
|
|
|
String storedValue = mSystemSettingsMap.get(key);
|
|
return storedValue == null ? defaultValue : Integer.parseInt(storedValue);
|
|
}
|
|
).when(() -> SystemProperties.getInt(anyString(), anyInt()));
|
|
|
|
doAnswer((Answer<Long>) invocationOnMock -> {
|
|
String key = invocationOnMock.getArgument(0);
|
|
long defaultValue = invocationOnMock.getArgument(1);
|
|
|
|
String storedValue = mSystemSettingsMap.get(key);
|
|
return storedValue == null ? defaultValue : Long.parseLong(storedValue);
|
|
}
|
|
).when(() -> SystemProperties.getLong(anyString(), anyLong()));
|
|
|
|
DeviceConfig.setProperty(DeviceConfig.NAMESPACE_ROLLBACK,
|
|
PackageWatchdog.PROPERTY_WATCHDOG_EXPLICIT_HEALTH_CHECK_ENABLED,
|
|
Boolean.toString(true), false);
|
|
|
|
DeviceConfig.setProperty(DeviceConfig.NAMESPACE_ROLLBACK,
|
|
PackageWatchdog.PROPERTY_WATCHDOG_TRIGGER_FAILURE_COUNT,
|
|
Integer.toString(PackageWatchdog.DEFAULT_TRIGGER_FAILURE_COUNT), false);
|
|
|
|
mAllocatedWatchdogs = new ArrayList<>();
|
|
}
|
|
|
|
@After
|
|
public void tearDown() throws Exception {
|
|
dropShellPermissions();
|
|
mSession.finishMocking();
|
|
// Clean up listeners since too many listeners will delay notifications significantly
|
|
for (PackageWatchdog watchdog : mAllocatedWatchdogs) {
|
|
watchdog.removePropertyChangedListener();
|
|
}
|
|
mAllocatedWatchdogs.clear();
|
|
}
|
|
|
|
@Test
|
|
public void testRegistration_singleObserver() {
|
|
PackageWatchdog watchdog = createWatchdog();
|
|
TestObserver observer = new TestObserver(OBSERVER_NAME_1);
|
|
|
|
watchdog.startObservingHealth(observer, Arrays.asList(APP_A), SHORT_DURATION);
|
|
raiseFatalFailureAndDispatch(watchdog,
|
|
Arrays.asList(new VersionedPackage(APP_A, VERSION_CODE)),
|
|
PackageWatchdog.FAILURE_REASON_UNKNOWN);
|
|
|
|
// The failed packages should be the same as the registered ones to ensure registration is
|
|
// done successfully
|
|
assertThat(observer.mHealthCheckFailedPackages).containsExactly(APP_A);
|
|
}
|
|
|
|
@Test
|
|
public void testRegistration_multiObservers() {
|
|
PackageWatchdog watchdog = createWatchdog();
|
|
TestObserver observer1 = new TestObserver(OBSERVER_NAME_1);
|
|
TestObserver observer2 = new TestObserver(OBSERVER_NAME_2);
|
|
|
|
watchdog.startObservingHealth(observer1, Arrays.asList(APP_A), SHORT_DURATION);
|
|
watchdog.startObservingHealth(observer2, Arrays.asList(APP_A, APP_B), SHORT_DURATION);
|
|
raiseFatalFailureAndDispatch(watchdog,
|
|
Arrays.asList(new VersionedPackage(APP_A, VERSION_CODE),
|
|
new VersionedPackage(APP_B, VERSION_CODE)),
|
|
PackageWatchdog.FAILURE_REASON_UNKNOWN);
|
|
|
|
// The failed packages should be the same as the registered ones to ensure registration is
|
|
// done successfully
|
|
assertThat(observer1.mHealthCheckFailedPackages).containsExactly(APP_A);
|
|
assertThat(observer2.mHealthCheckFailedPackages).containsExactly(APP_A, APP_B);
|
|
}
|
|
|
|
@Test
|
|
public void testUnregistration_singleObserver() {
|
|
PackageWatchdog watchdog = createWatchdog();
|
|
TestObserver observer = new TestObserver(OBSERVER_NAME_1);
|
|
|
|
watchdog.startObservingHealth(observer, Arrays.asList(APP_A), SHORT_DURATION);
|
|
watchdog.unregisterHealthObserver(observer);
|
|
raiseFatalFailureAndDispatch(watchdog,
|
|
Arrays.asList(new VersionedPackage(APP_A, VERSION_CODE)),
|
|
PackageWatchdog.FAILURE_REASON_UNKNOWN);
|
|
|
|
// We should have no failed packages to ensure unregistration is done successfully
|
|
assertThat(observer.mHealthCheckFailedPackages).isEmpty();
|
|
}
|
|
|
|
@Test
|
|
public void testUnregistration_multiObservers() {
|
|
PackageWatchdog watchdog = createWatchdog();
|
|
TestObserver observer1 = new TestObserver(OBSERVER_NAME_1);
|
|
TestObserver observer2 = new TestObserver(OBSERVER_NAME_2);
|
|
|
|
watchdog.startObservingHealth(observer1, Arrays.asList(APP_A), SHORT_DURATION);
|
|
watchdog.startObservingHealth(observer2, Arrays.asList(APP_A), SHORT_DURATION);
|
|
watchdog.unregisterHealthObserver(observer2);
|
|
raiseFatalFailureAndDispatch(watchdog,
|
|
Arrays.asList(new VersionedPackage(APP_A, VERSION_CODE)),
|
|
PackageWatchdog.FAILURE_REASON_UNKNOWN);
|
|
|
|
// observer1 should receive failed packages as intended.
|
|
assertThat(observer1.mHealthCheckFailedPackages).containsExactly(APP_A);
|
|
// observer2 should have no failed packages to ensure unregistration is done successfully
|
|
assertThat(observer2.mHealthCheckFailedPackages).isEmpty();
|
|
}
|
|
|
|
@Test
|
|
public void testExpiration_singleObserver() {
|
|
PackageWatchdog watchdog = createWatchdog();
|
|
TestObserver observer = new TestObserver(OBSERVER_NAME_1);
|
|
|
|
watchdog.startObservingHealth(observer, Arrays.asList(APP_A), SHORT_DURATION);
|
|
moveTimeForwardAndDispatch(SHORT_DURATION);
|
|
raiseFatalFailureAndDispatch(watchdog,
|
|
Arrays.asList(new VersionedPackage(APP_A, VERSION_CODE)),
|
|
PackageWatchdog.FAILURE_REASON_UNKNOWN);
|
|
|
|
// We should have no failed packages for the fatal failure is raised after expiration
|
|
assertThat(observer.mHealthCheckFailedPackages).isEmpty();
|
|
}
|
|
|
|
@Test
|
|
public void testExpiration_multiObservers() {
|
|
PackageWatchdog watchdog = createWatchdog();
|
|
TestObserver observer1 = new TestObserver(OBSERVER_NAME_1);
|
|
TestObserver observer2 = new TestObserver(OBSERVER_NAME_2);
|
|
|
|
watchdog.startObservingHealth(observer1, Arrays.asList(APP_A), SHORT_DURATION);
|
|
watchdog.startObservingHealth(observer2, Arrays.asList(APP_A), LONG_DURATION);
|
|
moveTimeForwardAndDispatch(SHORT_DURATION);
|
|
raiseFatalFailureAndDispatch(watchdog,
|
|
Arrays.asList(new VersionedPackage(APP_A, VERSION_CODE)),
|
|
PackageWatchdog.FAILURE_REASON_UNKNOWN);
|
|
|
|
// We should have no failed packages for the fatal failure is raised after expiration
|
|
assertThat(observer1.mHealthCheckFailedPackages).isEmpty();
|
|
// We should have failed packages since observer2 hasn't expired
|
|
assertThat(observer2.mHealthCheckFailedPackages).containsExactly(APP_A);
|
|
}
|
|
|
|
/** Observing already observed package extends the observation time. */
|
|
@Test
|
|
public void testObserveAlreadyObservedPackage() {
|
|
PackageWatchdog watchdog = createWatchdog();
|
|
TestObserver observer = new TestObserver(OBSERVER_NAME_1);
|
|
|
|
// Start observing APP_A
|
|
watchdog.startObservingHealth(observer, Arrays.asList(APP_A), SHORT_DURATION);
|
|
|
|
// Then advance time half-way
|
|
moveTimeForwardAndDispatch(SHORT_DURATION / 2);
|
|
|
|
// Start observing APP_A again
|
|
watchdog.startObservingHealth(observer, Arrays.asList(APP_A), SHORT_DURATION);
|
|
|
|
// Then advance time such that it should have expired were it not for the second observation
|
|
moveTimeForwardAndDispatch((SHORT_DURATION / 2) + 1);
|
|
|
|
raiseFatalFailureAndDispatch(watchdog,
|
|
Arrays.asList(new VersionedPackage(APP_A, VERSION_CODE)),
|
|
PackageWatchdog.FAILURE_REASON_UNKNOWN);
|
|
|
|
// Verify that we receive failed packages as expected for APP_A not expired
|
|
assertThat(observer.mHealthCheckFailedPackages).containsExactly(APP_A);
|
|
}
|
|
|
|
/**
|
|
* Test package observers are persisted and loaded on startup
|
|
*/
|
|
@Test
|
|
public void testPersistence() {
|
|
PackageWatchdog watchdog1 = createWatchdog();
|
|
TestObserver observer1 = new TestObserver(OBSERVER_NAME_1);
|
|
TestObserver observer2 = new TestObserver(OBSERVER_NAME_2);
|
|
|
|
watchdog1.startObservingHealth(observer1, Arrays.asList(APP_A), SHORT_DURATION);
|
|
watchdog1.startObservingHealth(observer2, Arrays.asList(APP_A, APP_B), SHORT_DURATION);
|
|
// Then advance time and run IO Handler so file is saved
|
|
mTestLooper.dispatchAll();
|
|
// Then start a new watchdog
|
|
PackageWatchdog watchdog2 = createWatchdog();
|
|
// Then resume observer1 and observer2
|
|
watchdog2.registerHealthObserver(observer1);
|
|
watchdog2.registerHealthObserver(observer2);
|
|
raiseFatalFailureAndDispatch(watchdog2,
|
|
Arrays.asList(new VersionedPackage(APP_A, VERSION_CODE),
|
|
new VersionedPackage(APP_B, VERSION_CODE)),
|
|
PackageWatchdog.FAILURE_REASON_UNKNOWN);
|
|
|
|
// We should receive failed packages as expected to ensure observers are persisted and
|
|
// resumed correctly
|
|
assertThat(observer1.mHealthCheckFailedPackages).containsExactly(APP_A);
|
|
assertThat(observer2.mHealthCheckFailedPackages).containsExactly(APP_A, APP_B);
|
|
}
|
|
|
|
/**
|
|
* Test package failure under threshold does not notify observers
|
|
*/
|
|
@Test
|
|
public void testNoPackageFailureBeforeThreshold() throws Exception {
|
|
PackageWatchdog watchdog = createWatchdog();
|
|
TestObserver observer1 = new TestObserver(OBSERVER_NAME_1);
|
|
TestObserver observer2 = new TestObserver(OBSERVER_NAME_2);
|
|
|
|
watchdog.startObservingHealth(observer2, Arrays.asList(APP_A), SHORT_DURATION);
|
|
watchdog.startObservingHealth(observer1, Arrays.asList(APP_A), SHORT_DURATION);
|
|
|
|
// Then fail APP_A below the threshold
|
|
for (int i = 0; i < watchdog.getTriggerFailureCount() - 1; i++) {
|
|
watchdog.onPackageFailure(Arrays.asList(new VersionedPackage(APP_A, VERSION_CODE)),
|
|
PackageWatchdog.FAILURE_REASON_UNKNOWN);
|
|
}
|
|
|
|
// Run handler so package failures are dispatched to observers
|
|
mTestLooper.dispatchAll();
|
|
|
|
// Verify that observers are not notified
|
|
assertThat(observer1.mHealthCheckFailedPackages).isEmpty();
|
|
assertThat(observer2.mHealthCheckFailedPackages).isEmpty();
|
|
}
|
|
|
|
/**
|
|
* Test package failure and does not notify any observer because they are not observing
|
|
* the failed packages.
|
|
*/
|
|
@Test
|
|
public void testPackageFailureDifferentPackageNotifyNone() throws Exception {
|
|
PackageWatchdog watchdog = createWatchdog();
|
|
TestObserver observer1 = new TestObserver(OBSERVER_NAME_1);
|
|
TestObserver observer2 = new TestObserver(OBSERVER_NAME_2);
|
|
|
|
|
|
watchdog.startObservingHealth(observer2, Arrays.asList(APP_A), SHORT_DURATION);
|
|
watchdog.startObservingHealth(observer1, Arrays.asList(APP_B), SHORT_DURATION);
|
|
|
|
// Then fail APP_C (not observed) above the threshold
|
|
raiseFatalFailureAndDispatch(watchdog,
|
|
Arrays.asList(new VersionedPackage(APP_C, VERSION_CODE)),
|
|
PackageWatchdog.FAILURE_REASON_UNKNOWN);
|
|
|
|
// Verify that observers are not notified
|
|
assertThat(observer1.mHealthCheckFailedPackages).isEmpty();
|
|
assertThat(observer2.mHealthCheckFailedPackages).isEmpty();
|
|
}
|
|
|
|
/**
|
|
* Test package failure and does not notify any observer because the failed package version
|
|
* does not match the available rollback-from-version.
|
|
*/
|
|
@Test
|
|
public void testPackageFailureDifferentVersionNotifyNone() throws Exception {
|
|
PackageWatchdog watchdog = createWatchdog();
|
|
long differentVersionCode = 2L;
|
|
TestObserver observer = new TestObserver(OBSERVER_NAME_1) {
|
|
@Override
|
|
public int onHealthCheckFailed(VersionedPackage versionedPackage,
|
|
int failureReason, int mitigationCount) {
|
|
if (versionedPackage.getVersionCode() == VERSION_CODE) {
|
|
// Only rollback for specific versionCode
|
|
return PackageHealthObserverImpact.USER_IMPACT_MEDIUM;
|
|
}
|
|
return PackageHealthObserverImpact.USER_IMPACT_NONE;
|
|
}
|
|
};
|
|
|
|
watchdog.startObservingHealth(observer, Arrays.asList(APP_A), SHORT_DURATION);
|
|
|
|
// Then fail APP_A (different version) above the threshold
|
|
raiseFatalFailureAndDispatch(watchdog,
|
|
Arrays.asList(new VersionedPackage(APP_A, differentVersionCode)),
|
|
PackageWatchdog.FAILURE_REASON_UNKNOWN);
|
|
|
|
// Verify that observers are not notified
|
|
assertThat(observer.mHealthCheckFailedPackages).isEmpty();
|
|
}
|
|
|
|
|
|
/**
|
|
* Test package failure and notifies only least impact observers.
|
|
*/
|
|
@Test
|
|
public void testPackageFailureNotifyAllDifferentImpacts() throws Exception {
|
|
PackageWatchdog watchdog = createWatchdog();
|
|
TestObserver observerNone = new TestObserver(OBSERVER_NAME_1,
|
|
PackageHealthObserverImpact.USER_IMPACT_NONE);
|
|
TestObserver observerHigh = new TestObserver(OBSERVER_NAME_2,
|
|
PackageHealthObserverImpact.USER_IMPACT_HIGH);
|
|
TestObserver observerMid = new TestObserver(OBSERVER_NAME_3,
|
|
PackageHealthObserverImpact.USER_IMPACT_MEDIUM);
|
|
TestObserver observerLow = new TestObserver(OBSERVER_NAME_4,
|
|
PackageHealthObserverImpact.USER_IMPACT_LOW);
|
|
|
|
// Start observing for all impact observers
|
|
watchdog.startObservingHealth(observerNone, Arrays.asList(APP_A, APP_B, APP_C, APP_D),
|
|
SHORT_DURATION);
|
|
watchdog.startObservingHealth(observerHigh, Arrays.asList(APP_A, APP_B, APP_C),
|
|
SHORT_DURATION);
|
|
watchdog.startObservingHealth(observerMid, Arrays.asList(APP_A, APP_B),
|
|
SHORT_DURATION);
|
|
watchdog.startObservingHealth(observerLow, Arrays.asList(APP_A),
|
|
SHORT_DURATION);
|
|
|
|
// Then fail all apps above the threshold
|
|
raiseFatalFailureAndDispatch(watchdog,
|
|
Arrays.asList(new VersionedPackage(APP_A, VERSION_CODE),
|
|
new VersionedPackage(APP_B, VERSION_CODE),
|
|
new VersionedPackage(APP_C, VERSION_CODE),
|
|
new VersionedPackage(APP_D, VERSION_CODE)),
|
|
PackageWatchdog.FAILURE_REASON_UNKNOWN);
|
|
|
|
// Verify least impact observers are notifed of package failures
|
|
List<String> observerNonePackages = observerNone.mMitigatedPackages;
|
|
List<String> observerHighPackages = observerHigh.mMitigatedPackages;
|
|
List<String> observerMidPackages = observerMid.mMitigatedPackages;
|
|
List<String> observerLowPackages = observerLow.mMitigatedPackages;
|
|
|
|
// APP_D failure observed by only observerNone is not caught cos its impact is none
|
|
assertThat(observerNonePackages).isEmpty();
|
|
// APP_C failure is caught by observerHigh cos it's the lowest impact observer
|
|
assertThat(observerHighPackages).containsExactly(APP_C);
|
|
// APP_B failure is caught by observerMid cos it's the lowest impact observer
|
|
assertThat(observerMidPackages).containsExactly(APP_B);
|
|
// APP_A failure is caught by observerLow cos it's the lowest impact observer
|
|
assertThat(observerLowPackages).containsExactly(APP_A);
|
|
}
|
|
|
|
/**
|
|
* Test package failure and least impact observers are notified successively.
|
|
* State transistions:
|
|
*
|
|
* <ul>
|
|
* <li>(observer1:low, observer2:mid) -> {observer1}
|
|
* <li>(observer1:high, observer2:mid) -> {observer2}
|
|
* <li>(observer1:high, observer2:none) -> {observer1}
|
|
* <li>(observer1:none, observer2:none) -> {}
|
|
* <ul>
|
|
*/
|
|
@Test
|
|
public void testPackageFailureNotifyLeastImpactSuccessively() throws Exception {
|
|
PackageWatchdog watchdog = createWatchdog();
|
|
TestObserver observerFirst = new TestObserver(OBSERVER_NAME_1,
|
|
PackageHealthObserverImpact.USER_IMPACT_LOW);
|
|
TestObserver observerSecond = new TestObserver(OBSERVER_NAME_2,
|
|
PackageHealthObserverImpact.USER_IMPACT_MEDIUM);
|
|
|
|
// Start observing for observerFirst and observerSecond with failure handling
|
|
watchdog.startObservingHealth(observerFirst, Arrays.asList(APP_A), LONG_DURATION);
|
|
watchdog.startObservingHealth(observerSecond, Arrays.asList(APP_A), LONG_DURATION);
|
|
|
|
// Then fail APP_A above the threshold
|
|
raiseFatalFailureAndDispatch(watchdog,
|
|
Arrays.asList(new VersionedPackage(APP_A, VERSION_CODE)),
|
|
PackageWatchdog.FAILURE_REASON_UNKNOWN);
|
|
|
|
// Verify only observerFirst is notifed
|
|
assertThat(observerFirst.mMitigatedPackages).containsExactly(APP_A);
|
|
assertThat(observerSecond.mMitigatedPackages).isEmpty();
|
|
|
|
// After observerFirst handles failure, next action it has is high impact
|
|
observerFirst.mImpact = PackageHealthObserverImpact.USER_IMPACT_HIGH;
|
|
observerFirst.mMitigatedPackages.clear();
|
|
observerSecond.mMitigatedPackages.clear();
|
|
|
|
// Then fail APP_A again above the threshold
|
|
raiseFatalFailureAndDispatch(watchdog,
|
|
Arrays.asList(new VersionedPackage(APP_A, VERSION_CODE)),
|
|
PackageWatchdog.FAILURE_REASON_UNKNOWN);
|
|
|
|
// Verify only observerSecond is notifed cos it has least impact
|
|
assertThat(observerSecond.mMitigatedPackages).containsExactly(APP_A);
|
|
assertThat(observerFirst.mMitigatedPackages).isEmpty();
|
|
|
|
// After observerSecond handles failure, it has no further actions
|
|
observerSecond.mImpact = PackageHealthObserverImpact.USER_IMPACT_NONE;
|
|
observerFirst.mMitigatedPackages.clear();
|
|
observerSecond.mMitigatedPackages.clear();
|
|
|
|
// Then fail APP_A again above the threshold
|
|
raiseFatalFailureAndDispatch(watchdog,
|
|
Arrays.asList(new VersionedPackage(APP_A, VERSION_CODE)),
|
|
PackageWatchdog.FAILURE_REASON_UNKNOWN);
|
|
|
|
// Verify only observerFirst is notifed cos it has the only action
|
|
assertThat(observerFirst.mMitigatedPackages).containsExactly(APP_A);
|
|
assertThat(observerSecond.mMitigatedPackages).isEmpty();
|
|
|
|
// After observerFirst handles failure, it too has no further actions
|
|
observerFirst.mImpact = PackageHealthObserverImpact.USER_IMPACT_NONE;
|
|
observerFirst.mMitigatedPackages.clear();
|
|
observerSecond.mMitigatedPackages.clear();
|
|
|
|
// Then fail APP_A again above the threshold
|
|
raiseFatalFailureAndDispatch(watchdog,
|
|
Arrays.asList(new VersionedPackage(APP_A, VERSION_CODE)),
|
|
PackageWatchdog.FAILURE_REASON_UNKNOWN);
|
|
|
|
// Verify no observer is notified cos no actions left
|
|
assertThat(observerFirst.mMitigatedPackages).isEmpty();
|
|
assertThat(observerSecond.mMitigatedPackages).isEmpty();
|
|
}
|
|
|
|
/**
|
|
* Test package failure and notifies only one observer even with observer impact tie.
|
|
*/
|
|
@Test
|
|
public void testPackageFailureNotifyOneSameImpact() throws Exception {
|
|
PackageWatchdog watchdog = createWatchdog();
|
|
TestObserver observer1 = new TestObserver(OBSERVER_NAME_1,
|
|
PackageHealthObserverImpact.USER_IMPACT_HIGH);
|
|
TestObserver observer2 = new TestObserver(OBSERVER_NAME_2,
|
|
PackageHealthObserverImpact.USER_IMPACT_HIGH);
|
|
|
|
// Start observing for observer1 and observer2 with failure handling
|
|
watchdog.startObservingHealth(observer2, Arrays.asList(APP_A), SHORT_DURATION);
|
|
watchdog.startObservingHealth(observer1, Arrays.asList(APP_A), SHORT_DURATION);
|
|
|
|
// Then fail APP_A above the threshold
|
|
raiseFatalFailureAndDispatch(watchdog,
|
|
Arrays.asList(new VersionedPackage(APP_A, VERSION_CODE)),
|
|
PackageWatchdog.FAILURE_REASON_UNKNOWN);
|
|
|
|
// Verify only one observer is notifed
|
|
assertThat(observer1.mMitigatedPackages).containsExactly(APP_A);
|
|
assertThat(observer2.mMitigatedPackages).isEmpty();
|
|
}
|
|
|
|
/**
|
|
* Test package passing explicit health checks does not fail and vice versa.
|
|
*/
|
|
@Test
|
|
public void testExplicitHealthChecks() throws Exception {
|
|
TestController controller = new TestController();
|
|
PackageWatchdog watchdog = createWatchdog(controller, true /* withPackagesReady */);
|
|
TestObserver observer1 = new TestObserver(OBSERVER_NAME_1,
|
|
PackageHealthObserverImpact.USER_IMPACT_HIGH);
|
|
TestObserver observer2 = new TestObserver(OBSERVER_NAME_2,
|
|
PackageHealthObserverImpact.USER_IMPACT_HIGH);
|
|
TestObserver observer3 = new TestObserver(OBSERVER_NAME_3,
|
|
PackageHealthObserverImpact.USER_IMPACT_HIGH);
|
|
|
|
|
|
// Start observing with explicit health checks for APP_A and APP_B respectively
|
|
// with observer1 and observer2
|
|
controller.setSupportedPackages(Arrays.asList(APP_A, APP_B));
|
|
watchdog.startObservingHealth(observer1, Arrays.asList(APP_A), SHORT_DURATION);
|
|
watchdog.startObservingHealth(observer2, Arrays.asList(APP_B), SHORT_DURATION);
|
|
|
|
// Run handler so requests are dispatched to the controller
|
|
mTestLooper.dispatchAll();
|
|
|
|
// Verify we requested health checks for APP_A and APP_B
|
|
List<String> requestedPackages = controller.getRequestedPackages();
|
|
assertThat(requestedPackages).containsExactly(APP_A, APP_B);
|
|
|
|
// Then health check passed for APP_A (observer1 is aware)
|
|
controller.setPackagePassed(APP_A);
|
|
|
|
// Then start observing APP_A with explicit health checks for observer3.
|
|
// Observer3 didn't exist when we got the explicit health check above, so
|
|
// it starts out with a non-passing explicit health check and has to wait for a pass
|
|
// otherwise it would be notified of APP_A failure on expiry
|
|
watchdog.startObservingHealth(observer3, Arrays.asList(APP_A), SHORT_DURATION);
|
|
|
|
// Then expire observers
|
|
moveTimeForwardAndDispatch(SHORT_DURATION);
|
|
|
|
// Verify we cancelled all requests on expiry
|
|
assertThat(controller.getRequestedPackages()).isEmpty();
|
|
|
|
// Verify observer1 is not notified
|
|
assertThat(observer1.mMitigatedPackages).isEmpty();
|
|
|
|
// Verify observer2 is notifed because health checks for APP_B never passed
|
|
assertThat(observer2.mMitigatedPackages).containsExactly(APP_B);
|
|
|
|
// Verify observer3 is notifed because health checks for APP_A did not pass before expiry
|
|
assertThat(observer3.mMitigatedPackages).containsExactly(APP_A);
|
|
}
|
|
|
|
/**
|
|
* Test explicit health check state can be disabled and enabled correctly.
|
|
*/
|
|
@Test
|
|
public void testExplicitHealthCheckStateChanges() throws Exception {
|
|
TestController controller = new TestController();
|
|
PackageWatchdog watchdog = createWatchdog(controller, true /* withPackagesReady */);
|
|
TestObserver observer = new TestObserver(OBSERVER_NAME_1,
|
|
PackageHealthObserverImpact.USER_IMPACT_MEDIUM);
|
|
|
|
// Start observing with explicit health checks for APP_A and APP_B
|
|
controller.setSupportedPackages(Arrays.asList(APP_A, APP_B, APP_C));
|
|
watchdog.startObservingHealth(observer, Arrays.asList(APP_A), SHORT_DURATION);
|
|
watchdog.startObservingHealth(observer, Arrays.asList(APP_B), LONG_DURATION);
|
|
|
|
// Run handler so requests are dispatched to the controller
|
|
mTestLooper.dispatchAll();
|
|
|
|
// Verify we requested health checks for APP_A and APP_B
|
|
List<String> requestedPackages = controller.getRequestedPackages();
|
|
assertThat(requestedPackages).containsExactly(APP_A, APP_B);
|
|
|
|
// Disable explicit health checks (marks APP_A and APP_B as passed)
|
|
setExplicitHealthCheckEnabled(false);
|
|
|
|
// Run handler so requests/cancellations are dispatched to the controller
|
|
mTestLooper.dispatchAll();
|
|
|
|
// Verify all checks are cancelled
|
|
assertThat(controller.getRequestedPackages()).isEmpty();
|
|
|
|
// Then expire APP_A
|
|
moveTimeForwardAndDispatch(SHORT_DURATION);
|
|
|
|
// Verify APP_A is not failed (APP_B) is not expired yet
|
|
assertThat(observer.mMitigatedPackages).isEmpty();
|
|
|
|
// Re-enable explicit health checks
|
|
setExplicitHealthCheckEnabled(true);
|
|
|
|
// Run handler so requests/cancellations are dispatched to the controller
|
|
mTestLooper.dispatchAll();
|
|
|
|
// Verify no requests are made cos APP_A is expired and APP_B was marked as passed
|
|
assertThat(controller.getRequestedPackages()).isEmpty();
|
|
|
|
// Then set new supported packages
|
|
controller.setSupportedPackages(Arrays.asList(APP_C));
|
|
// Start observing APP_A and APP_C; only APP_C has support for explicit health checks
|
|
watchdog.startObservingHealth(observer, Arrays.asList(APP_A, APP_C), SHORT_DURATION);
|
|
|
|
// Run handler so requests/cancellations are dispatched to the controller
|
|
mTestLooper.dispatchAll();
|
|
|
|
// Verify requests are only made for APP_C
|
|
requestedPackages = controller.getRequestedPackages();
|
|
assertThat(requestedPackages).containsExactly(APP_C);
|
|
|
|
// Then expire APP_A and APP_C
|
|
moveTimeForwardAndDispatch(SHORT_DURATION);
|
|
|
|
// Verify only APP_C is failed because explicit health checks was not supported for APP_A
|
|
assertThat(observer.mMitigatedPackages).containsExactly(APP_C);
|
|
}
|
|
|
|
/**
|
|
* Tests failure when health check duration is different from package observation duration
|
|
* Failure is also notified only once.
|
|
*/
|
|
@Test
|
|
public void testExplicitHealthCheckFailureBeforeExpiry() throws Exception {
|
|
TestController controller = new TestController();
|
|
PackageWatchdog watchdog = createWatchdog(controller, true /* withPackagesReady */);
|
|
TestObserver observer = new TestObserver(OBSERVER_NAME_1,
|
|
PackageHealthObserverImpact.USER_IMPACT_MEDIUM);
|
|
|
|
// Start observing with explicit health checks for APP_A and
|
|
// package observation duration == LONG_DURATION
|
|
// health check duration == SHORT_DURATION (set by default in the TestController)
|
|
controller.setSupportedPackages(Arrays.asList(APP_A));
|
|
watchdog.startObservingHealth(observer, Arrays.asList(APP_A), LONG_DURATION);
|
|
|
|
// Then APP_A has exceeded health check duration
|
|
moveTimeForwardAndDispatch(SHORT_DURATION);
|
|
|
|
// Verify that health check is failed
|
|
assertThat(observer.mMitigatedPackages).containsExactly(APP_A);
|
|
|
|
// Clear failed packages and forward time to expire the observation duration
|
|
observer.mMitigatedPackages.clear();
|
|
moveTimeForwardAndDispatch(LONG_DURATION);
|
|
|
|
// Verify that health check failure is not notified again
|
|
assertThat(observer.mMitigatedPackages).isEmpty();
|
|
}
|
|
|
|
/**
|
|
* Tests failure when health check duration is different from package observation duration
|
|
* Failure is also notified only once.
|
|
*/
|
|
@Test
|
|
public void testExplicitHealthCheckFailureAfterExpiry() {
|
|
TestController controller = new TestController();
|
|
PackageWatchdog watchdog = createWatchdog(controller, true /* withPackagesReady */);
|
|
TestObserver observer = new TestObserver(OBSERVER_NAME_1,
|
|
PackageHealthObserverImpact.USER_IMPACT_MEDIUM);
|
|
|
|
// Start observing with explicit health checks for APP_A and
|
|
// package observation duration == SHORT_DURATION / 2
|
|
// health check duration == SHORT_DURATION (set by default in the TestController)
|
|
controller.setSupportedPackages(Arrays.asList(APP_A));
|
|
watchdog.startObservingHealth(observer, Arrays.asList(APP_A), SHORT_DURATION / 2);
|
|
|
|
// Forward time to expire the observation duration
|
|
moveTimeForwardAndDispatch(SHORT_DURATION / 2);
|
|
|
|
// Verify that health check is failed
|
|
assertThat(observer.mMitigatedPackages).containsExactly(APP_A);
|
|
|
|
// Clear failed packages and forward time to expire the health check duration
|
|
observer.mMitigatedPackages.clear();
|
|
moveTimeForwardAndDispatch(SHORT_DURATION);
|
|
|
|
// Verify that health check failure is not notified again
|
|
assertThat(observer.mMitigatedPackages).isEmpty();
|
|
}
|
|
|
|
/** Tests {@link MonitoredPackage} health check state transitions. */
|
|
@Test
|
|
public void testPackageHealthCheckStateTransitions() {
|
|
TestController controller = new TestController();
|
|
PackageWatchdog wd = createWatchdog(controller, true /* withPackagesReady */);
|
|
MonitoredPackage m1 = wd.newMonitoredPackage(APP_A, LONG_DURATION,
|
|
false /* hasPassedHealthCheck */);
|
|
MonitoredPackage m2 = wd.newMonitoredPackage(APP_B, LONG_DURATION, false);
|
|
MonitoredPackage m3 = wd.newMonitoredPackage(APP_C, LONG_DURATION, false);
|
|
MonitoredPackage m4 = wd.newMonitoredPackage(APP_D, LONG_DURATION, SHORT_DURATION, true,
|
|
new LongArrayQueue());
|
|
|
|
// Verify transition: inactive -> active -> passed
|
|
// Verify initially inactive
|
|
assertThat(m1.getHealthCheckStateLocked()).isEqualTo(HealthCheckState.INACTIVE);
|
|
// Verify still inactive, until we #setHealthCheckActiveLocked
|
|
assertThat(m1.handleElapsedTimeLocked(SHORT_DURATION)).isEqualTo(HealthCheckState.INACTIVE);
|
|
// Verify now active
|
|
assertThat(m1.setHealthCheckActiveLocked(SHORT_DURATION)).isEqualTo(
|
|
HealthCheckState.ACTIVE);
|
|
// Verify now passed
|
|
assertThat(m1.tryPassHealthCheckLocked()).isEqualTo(HealthCheckState.PASSED);
|
|
|
|
// Verify transition: inactive -> active -> failed
|
|
// Verify initially inactive
|
|
assertThat(m2.getHealthCheckStateLocked()).isEqualTo(HealthCheckState.INACTIVE);
|
|
// Verify now active
|
|
assertThat(m2.setHealthCheckActiveLocked(SHORT_DURATION)).isEqualTo(
|
|
HealthCheckState.ACTIVE);
|
|
// Verify now failed
|
|
assertThat(m2.handleElapsedTimeLocked(SHORT_DURATION)).isEqualTo(HealthCheckState.FAILED);
|
|
|
|
// Verify transition: inactive -> failed
|
|
// Verify initially inactive
|
|
assertThat(m3.getHealthCheckStateLocked()).isEqualTo(HealthCheckState.INACTIVE);
|
|
// Verify now failed because package expired
|
|
assertThat(m3.handleElapsedTimeLocked(LONG_DURATION)).isEqualTo(HealthCheckState.FAILED);
|
|
// Verify remains failed even when asked to pass
|
|
assertThat(m3.tryPassHealthCheckLocked()).isEqualTo(HealthCheckState.FAILED);
|
|
|
|
// Verify transition: passed
|
|
assertThat(m4.getHealthCheckStateLocked()).isEqualTo(HealthCheckState.PASSED);
|
|
// Verify remains passed even if health check fails
|
|
assertThat(m4.handleElapsedTimeLocked(SHORT_DURATION)).isEqualTo(HealthCheckState.PASSED);
|
|
// Verify remains passed even if package expires
|
|
assertThat(m4.handleElapsedTimeLocked(LONG_DURATION)).isEqualTo(HealthCheckState.PASSED);
|
|
}
|
|
|
|
@Test
|
|
public void testNetworkStackFailure() {
|
|
final PackageWatchdog wd = createWatchdog();
|
|
|
|
// Start observing with failure handling
|
|
TestObserver observer = new TestObserver(OBSERVER_NAME_1,
|
|
PackageHealthObserverImpact.USER_IMPACT_HIGH);
|
|
wd.startObservingHealth(observer, Collections.singletonList(APP_A), SHORT_DURATION);
|
|
|
|
// Notify of NetworkStack failure
|
|
mConnectivityModuleCallbackCaptor.getValue().onNetworkStackFailure(APP_A);
|
|
|
|
// Run handler so package failures are dispatched to observers
|
|
mTestLooper.dispatchAll();
|
|
|
|
// Verify the NetworkStack observer is notified
|
|
assertThat(observer.mMitigatedPackages).containsExactly(APP_A);
|
|
}
|
|
|
|
/** Test default values are used when device property is invalid. */
|
|
@Test
|
|
public void testInvalidConfig_watchdogTriggerFailureCount() {
|
|
DeviceConfig.setProperty(DeviceConfig.NAMESPACE_ROLLBACK,
|
|
PackageWatchdog.PROPERTY_WATCHDOG_TRIGGER_FAILURE_COUNT,
|
|
Integer.toString(-1), /*makeDefault*/false);
|
|
PackageWatchdog watchdog = createWatchdog();
|
|
TestObserver observer = new TestObserver(OBSERVER_NAME_1);
|
|
|
|
watchdog.startObservingHealth(observer, Arrays.asList(APP_A), SHORT_DURATION);
|
|
// Fail APP_A below the threshold which should not trigger package failures
|
|
for (int i = 0; i < PackageWatchdog.DEFAULT_TRIGGER_FAILURE_COUNT - 1; i++) {
|
|
watchdog.onPackageFailure(Arrays.asList(new VersionedPackage(APP_A, VERSION_CODE)),
|
|
PackageWatchdog.FAILURE_REASON_UNKNOWN);
|
|
}
|
|
mTestLooper.dispatchAll();
|
|
assertThat(observer.mHealthCheckFailedPackages).isEmpty();
|
|
|
|
// One more to trigger the package failure
|
|
watchdog.onPackageFailure(Arrays.asList(new VersionedPackage(APP_A, VERSION_CODE)),
|
|
PackageWatchdog.FAILURE_REASON_UNKNOWN);
|
|
mTestLooper.dispatchAll();
|
|
assertThat(observer.mHealthCheckFailedPackages).containsExactly(APP_A);
|
|
}
|
|
|
|
/** Test default values are used when device property is invalid. */
|
|
@Test
|
|
public void testInvalidConfig_watchdogTriggerDurationMillis() {
|
|
DeviceConfig.setProperty(DeviceConfig.NAMESPACE_ROLLBACK,
|
|
PackageWatchdog.PROPERTY_WATCHDOG_TRIGGER_FAILURE_COUNT,
|
|
Integer.toString(2), /*makeDefault*/false);
|
|
DeviceConfig.setProperty(DeviceConfig.NAMESPACE_ROLLBACK,
|
|
PackageWatchdog.PROPERTY_WATCHDOG_TRIGGER_DURATION_MILLIS,
|
|
Integer.toString(-1), /*makeDefault*/false);
|
|
PackageWatchdog watchdog = createWatchdog();
|
|
TestObserver observer = new TestObserver(OBSERVER_NAME_1);
|
|
|
|
watchdog.startObservingHealth(observer, Arrays.asList(APP_A, APP_B), Long.MAX_VALUE);
|
|
watchdog.onPackageFailure(Arrays.asList(new VersionedPackage(APP_A, VERSION_CODE)),
|
|
PackageWatchdog.FAILURE_REASON_UNKNOWN);
|
|
moveTimeForwardAndDispatch(PackageWatchdog.DEFAULT_TRIGGER_FAILURE_DURATION_MS + 1);
|
|
watchdog.onPackageFailure(Arrays.asList(new VersionedPackage(APP_A, VERSION_CODE)),
|
|
PackageWatchdog.FAILURE_REASON_UNKNOWN);
|
|
mTestLooper.dispatchAll();
|
|
|
|
// We shouldn't receive APP_A since the interval of 2 failures is greater than
|
|
// DEFAULT_TRIGGER_FAILURE_DURATION_MS.
|
|
assertThat(observer.mHealthCheckFailedPackages).isEmpty();
|
|
|
|
watchdog.onPackageFailure(Arrays.asList(new VersionedPackage(APP_B, VERSION_CODE)),
|
|
PackageWatchdog.FAILURE_REASON_UNKNOWN);
|
|
moveTimeForwardAndDispatch(PackageWatchdog.DEFAULT_TRIGGER_FAILURE_DURATION_MS - 1);
|
|
watchdog.onPackageFailure(Arrays.asList(new VersionedPackage(APP_B, VERSION_CODE)),
|
|
PackageWatchdog.FAILURE_REASON_UNKNOWN);
|
|
mTestLooper.dispatchAll();
|
|
|
|
// We should receive APP_B since the interval of 2 failures is less than
|
|
// DEFAULT_TRIGGER_FAILURE_DURATION_MS.
|
|
assertThat(observer.mHealthCheckFailedPackages).containsExactly(APP_B);
|
|
}
|
|
|
|
/**
|
|
* Test default monitoring duration is used when PackageWatchdog#startObservingHealth is offered
|
|
* an invalid durationMs.
|
|
*/
|
|
@Test
|
|
public void testInvalidMonitoringDuration_beforeExpiry() {
|
|
PackageWatchdog watchdog = createWatchdog();
|
|
TestObserver observer = new TestObserver(OBSERVER_NAME_1);
|
|
|
|
watchdog.startObservingHealth(observer, Arrays.asList(APP_A), -1);
|
|
// Note: Don't move too close to the expiration time otherwise the handler will be thrashed
|
|
// by PackageWatchdog#scheduleNextSyncStateLocked which keeps posting runnables with very
|
|
// small timeouts.
|
|
moveTimeForwardAndDispatch(PackageWatchdog.DEFAULT_OBSERVING_DURATION_MS - 100);
|
|
raiseFatalFailureAndDispatch(watchdog,
|
|
Arrays.asList(new VersionedPackage(APP_A, VERSION_CODE)),
|
|
PackageWatchdog.FAILURE_REASON_UNKNOWN);
|
|
|
|
// We should receive APP_A since the observer hasn't expired
|
|
assertThat(observer.mHealthCheckFailedPackages).containsExactly(APP_A);
|
|
}
|
|
|
|
/**
|
|
* Test default monitoring duration is used when PackageWatchdog#startObservingHealth is offered
|
|
* an invalid durationMs.
|
|
*/
|
|
@Test
|
|
public void testInvalidMonitoringDuration_afterExpiry() {
|
|
PackageWatchdog watchdog = createWatchdog();
|
|
TestObserver observer = new TestObserver(OBSERVER_NAME_1);
|
|
|
|
watchdog.startObservingHealth(observer, Arrays.asList(APP_A), -1);
|
|
moveTimeForwardAndDispatch(PackageWatchdog.DEFAULT_OBSERVING_DURATION_MS + 1);
|
|
raiseFatalFailureAndDispatch(watchdog,
|
|
Arrays.asList(new VersionedPackage(APP_A, VERSION_CODE)),
|
|
PackageWatchdog.FAILURE_REASON_UNKNOWN);
|
|
|
|
// We should receive nothing since the observer has expired
|
|
assertThat(observer.mHealthCheckFailedPackages).isEmpty();
|
|
}
|
|
|
|
/** Test we are notified when enough failures are triggered within any window. */
|
|
@Test
|
|
public void testFailureTriggerWindow() {
|
|
DeviceConfig.setProperty(DeviceConfig.NAMESPACE_ROLLBACK,
|
|
PackageWatchdog.PROPERTY_WATCHDOG_TRIGGER_FAILURE_COUNT,
|
|
Integer.toString(3), /*makeDefault*/false);
|
|
DeviceConfig.setProperty(DeviceConfig.NAMESPACE_ROLLBACK,
|
|
PackageWatchdog.PROPERTY_WATCHDOG_TRIGGER_DURATION_MILLIS,
|
|
Integer.toString(1000), /*makeDefault*/false);
|
|
PackageWatchdog watchdog = createWatchdog();
|
|
TestObserver observer = new TestObserver(OBSERVER_NAME_1);
|
|
|
|
watchdog.startObservingHealth(observer, Arrays.asList(APP_A), Long.MAX_VALUE);
|
|
// Raise 2 failures at t=0 and t=900 respectively
|
|
watchdog.onPackageFailure(Arrays.asList(new VersionedPackage(APP_A, VERSION_CODE)),
|
|
PackageWatchdog.FAILURE_REASON_UNKNOWN);
|
|
moveTimeForwardAndDispatch(900);
|
|
watchdog.onPackageFailure(Arrays.asList(new VersionedPackage(APP_A, VERSION_CODE)),
|
|
PackageWatchdog.FAILURE_REASON_UNKNOWN);
|
|
|
|
// Raise 2 failures at t=1100
|
|
moveTimeForwardAndDispatch(200);
|
|
watchdog.onPackageFailure(Arrays.asList(new VersionedPackage(APP_A, VERSION_CODE)),
|
|
PackageWatchdog.FAILURE_REASON_UNKNOWN);
|
|
watchdog.onPackageFailure(Arrays.asList(new VersionedPackage(APP_A, VERSION_CODE)),
|
|
PackageWatchdog.FAILURE_REASON_UNKNOWN);
|
|
mTestLooper.dispatchAll();
|
|
|
|
// We should receive APP_A since there are 3 failures within 1000ms window
|
|
assertThat(observer.mHealthCheckFailedPackages).containsExactly(APP_A);
|
|
}
|
|
|
|
/** Test that observers execute correctly for failures reasons that go through thresholding. */
|
|
@Test
|
|
public void testNonImmediateFailureReasons() {
|
|
PackageWatchdog watchdog = createWatchdog();
|
|
TestObserver observer1 = new TestObserver(OBSERVER_NAME_1);
|
|
TestObserver observer2 = new TestObserver(OBSERVER_NAME_2);
|
|
|
|
watchdog.startObservingHealth(observer1, Arrays.asList(APP_A), SHORT_DURATION);
|
|
watchdog.startObservingHealth(observer2, Arrays.asList(APP_B), SHORT_DURATION);
|
|
|
|
raiseFatalFailureAndDispatch(watchdog, Arrays.asList(new VersionedPackage(APP_A,
|
|
VERSION_CODE)), PackageWatchdog.FAILURE_REASON_APP_CRASH);
|
|
raiseFatalFailureAndDispatch(watchdog, Arrays.asList(new VersionedPackage(APP_B,
|
|
VERSION_CODE)), PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING);
|
|
|
|
assertThat(observer1.getLastFailureReason()).isEqualTo(
|
|
PackageWatchdog.FAILURE_REASON_APP_CRASH);
|
|
assertThat(observer2.getLastFailureReason()).isEqualTo(
|
|
PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING);
|
|
}
|
|
|
|
/** Test that observers execute correctly for failures reasons that skip thresholding. */
|
|
@Test
|
|
public void testImmediateFailures() {
|
|
PackageWatchdog watchdog = createWatchdog();
|
|
TestObserver observer1 = new TestObserver(OBSERVER_NAME_1);
|
|
|
|
watchdog.startObservingHealth(observer1, Arrays.asList(APP_A), SHORT_DURATION);
|
|
|
|
raiseFatalFailureAndDispatch(watchdog, Arrays.asList(new VersionedPackage(APP_A,
|
|
VERSION_CODE)), PackageWatchdog.FAILURE_REASON_NATIVE_CRASH);
|
|
raiseFatalFailureAndDispatch(watchdog, Arrays.asList(new VersionedPackage(APP_B,
|
|
VERSION_CODE)), PackageWatchdog.FAILURE_REASON_EXPLICIT_HEALTH_CHECK);
|
|
|
|
assertThat(observer1.mMitigatedPackages).containsExactly(APP_A, APP_B);
|
|
}
|
|
|
|
/**
|
|
* Test that a persistent observer will mitigate failures if it wishes to observe a package.
|
|
*/
|
|
@Test
|
|
public void testPersistentObserverWatchesPackage() {
|
|
PackageWatchdog watchdog = createWatchdog();
|
|
TestObserver persistentObserver = new TestObserver(OBSERVER_NAME_1);
|
|
persistentObserver.setPersistent(true);
|
|
persistentObserver.setMayObservePackages(true);
|
|
|
|
watchdog.startObservingHealth(persistentObserver, Arrays.asList(APP_B), SHORT_DURATION);
|
|
|
|
raiseFatalFailureAndDispatch(watchdog, Arrays.asList(new VersionedPackage(APP_A,
|
|
VERSION_CODE)), PackageWatchdog.FAILURE_REASON_UNKNOWN);
|
|
assertThat(persistentObserver.mHealthCheckFailedPackages).containsExactly(APP_A);
|
|
}
|
|
|
|
/**
|
|
* Test that a persistent observer will not mitigate failures if it does not wish to observe
|
|
* a given package.
|
|
*/
|
|
@Test
|
|
public void testPersistentObserverDoesNotWatchPackage() {
|
|
PackageWatchdog watchdog = createWatchdog();
|
|
TestObserver persistentObserver = new TestObserver(OBSERVER_NAME_1);
|
|
persistentObserver.setPersistent(true);
|
|
persistentObserver.setMayObservePackages(false);
|
|
|
|
watchdog.startObservingHealth(persistentObserver, Arrays.asList(APP_B), SHORT_DURATION);
|
|
|
|
raiseFatalFailureAndDispatch(watchdog, Arrays.asList(new VersionedPackage(APP_A,
|
|
VERSION_CODE)), PackageWatchdog.FAILURE_REASON_UNKNOWN);
|
|
assertThat(persistentObserver.mHealthCheckFailedPackages).isEmpty();
|
|
}
|
|
|
|
|
|
/** Ensure that boot loop mitigation is done when the number of boots meets the threshold. */
|
|
@Test
|
|
public void testBootLoopDetection_meetsThreshold() {
|
|
PackageWatchdog watchdog = createWatchdog();
|
|
TestObserver bootObserver = new TestObserver(OBSERVER_NAME_1);
|
|
watchdog.registerHealthObserver(bootObserver);
|
|
for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT; i++) {
|
|
watchdog.noteBoot();
|
|
}
|
|
assertThat(bootObserver.mitigatedBootLoop()).isTrue();
|
|
}
|
|
|
|
|
|
/**
|
|
* Ensure that boot loop mitigation is not done when the number of boots does not meet the
|
|
* threshold.
|
|
*/
|
|
@Test
|
|
public void testBootLoopDetection_doesNotMeetThreshold() {
|
|
PackageWatchdog watchdog = createWatchdog();
|
|
TestObserver bootObserver = new TestObserver(OBSERVER_NAME_1);
|
|
watchdog.registerHealthObserver(bootObserver);
|
|
for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT - 1; i++) {
|
|
watchdog.noteBoot();
|
|
}
|
|
assertThat(bootObserver.mitigatedBootLoop()).isFalse();
|
|
}
|
|
|
|
/**
|
|
* Ensure that boot loop mitigation is done for the observer with the lowest user impact
|
|
*/
|
|
@Test
|
|
public void testBootLoopMitigationDoneForLowestUserImpact() {
|
|
PackageWatchdog watchdog = createWatchdog();
|
|
TestObserver bootObserver1 = new TestObserver(OBSERVER_NAME_1);
|
|
bootObserver1.setImpact(PackageHealthObserverImpact.USER_IMPACT_LOW);
|
|
TestObserver bootObserver2 = new TestObserver(OBSERVER_NAME_2);
|
|
bootObserver2.setImpact(PackageHealthObserverImpact.USER_IMPACT_MEDIUM);
|
|
watchdog.registerHealthObserver(bootObserver1);
|
|
watchdog.registerHealthObserver(bootObserver2);
|
|
for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT; i++) {
|
|
watchdog.noteBoot();
|
|
}
|
|
assertThat(bootObserver1.mitigatedBootLoop()).isTrue();
|
|
assertThat(bootObserver2.mitigatedBootLoop()).isFalse();
|
|
}
|
|
|
|
/**
|
|
* Ensure that the correct mitigation counts are sent to the boot loop observer.
|
|
*/
|
|
@Test
|
|
public void testMultipleBootLoopMitigation() {
|
|
PackageWatchdog watchdog = createWatchdog();
|
|
TestObserver bootObserver = new TestObserver(OBSERVER_NAME_1);
|
|
watchdog.registerHealthObserver(bootObserver);
|
|
for (int i = 0; i < 4; i++) {
|
|
for (int j = 0; j < PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT; j++) {
|
|
watchdog.noteBoot();
|
|
}
|
|
}
|
|
|
|
moveTimeForwardAndDispatch(PackageWatchdog.DEFAULT_DEESCALATION_WINDOW_MS + 1);
|
|
|
|
for (int i = 0; i < 4; i++) {
|
|
for (int j = 0; j < PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT; j++) {
|
|
watchdog.noteBoot();
|
|
}
|
|
}
|
|
|
|
assertThat(bootObserver.mBootMitigationCounts).isEqualTo(List.of(1, 2, 3, 4, 1, 2, 3, 4));
|
|
}
|
|
|
|
/**
|
|
* Ensure that passing a null list of failed packages does not cause any mitigation logic to
|
|
* execute.
|
|
*/
|
|
@Test
|
|
public void testNullFailedPackagesList() {
|
|
PackageWatchdog watchdog = createWatchdog();
|
|
TestObserver observer1 = new TestObserver(OBSERVER_NAME_1);
|
|
watchdog.startObservingHealth(observer1, List.of(APP_A), LONG_DURATION);
|
|
|
|
raiseFatalFailureAndDispatch(watchdog, null, PackageWatchdog.FAILURE_REASON_APP_CRASH);
|
|
assertThat(observer1.mMitigatedPackages).isEmpty();
|
|
}
|
|
|
|
/**
|
|
* Test to verify that Package Watchdog syncs health check requests with the controller
|
|
* correctly, and that the requests are only synced when the set of observed packages
|
|
* changes.
|
|
*/
|
|
@Test
|
|
public void testSyncHealthCheckRequests() {
|
|
TestController testController = spy(TestController.class);
|
|
testController.setSupportedPackages(List.of(APP_A, APP_B, APP_C));
|
|
PackageWatchdog watchdog = createWatchdog(testController, true);
|
|
|
|
TestObserver testObserver1 = new TestObserver(OBSERVER_NAME_1);
|
|
watchdog.registerHealthObserver(testObserver1);
|
|
watchdog.startObservingHealth(testObserver1, List.of(APP_A), LONG_DURATION);
|
|
mTestLooper.dispatchAll();
|
|
|
|
TestObserver testObserver2 = new TestObserver(OBSERVER_NAME_2);
|
|
watchdog.registerHealthObserver(testObserver2);
|
|
watchdog.startObservingHealth(testObserver2, List.of(APP_B), LONG_DURATION);
|
|
mTestLooper.dispatchAll();
|
|
|
|
TestObserver testObserver3 = new TestObserver(OBSERVER_NAME_3);
|
|
watchdog.registerHealthObserver(testObserver3);
|
|
watchdog.startObservingHealth(testObserver3, List.of(APP_C), LONG_DURATION);
|
|
mTestLooper.dispatchAll();
|
|
|
|
watchdog.unregisterHealthObserver(testObserver1);
|
|
mTestLooper.dispatchAll();
|
|
|
|
watchdog.unregisterHealthObserver(testObserver2);
|
|
mTestLooper.dispatchAll();
|
|
|
|
watchdog.unregisterHealthObserver(testObserver3);
|
|
mTestLooper.dispatchAll();
|
|
|
|
List<Set> expectedSyncRequests = List.of(
|
|
Set.of(),
|
|
Set.of(APP_A),
|
|
Set.of(APP_A, APP_B),
|
|
Set.of(APP_A, APP_B, APP_C),
|
|
Set.of(APP_B, APP_C),
|
|
Set.of(APP_C),
|
|
Set.of()
|
|
);
|
|
assertThat(testController.getSyncRequests()).isEqualTo(expectedSyncRequests);
|
|
}
|
|
|
|
/**
|
|
* Ensure that the failure history of a package is preserved when making duplicate calls to
|
|
* observe the package.
|
|
*/
|
|
@Test
|
|
public void testFailureHistoryIsPreserved() {
|
|
PackageWatchdog watchdog = createWatchdog();
|
|
TestObserver observer = new TestObserver(OBSERVER_NAME_1);
|
|
watchdog.startObservingHealth(observer, List.of(APP_A), SHORT_DURATION);
|
|
for (int i = 0; i < PackageWatchdog.DEFAULT_TRIGGER_FAILURE_COUNT - 1; i++) {
|
|
watchdog.onPackageFailure(List.of(new VersionedPackage(APP_A, VERSION_CODE)),
|
|
PackageWatchdog.FAILURE_REASON_UNKNOWN);
|
|
}
|
|
mTestLooper.dispatchAll();
|
|
assertThat(observer.mMitigatedPackages).isEmpty();
|
|
watchdog.startObservingHealth(observer, List.of(APP_A), LONG_DURATION);
|
|
watchdog.onPackageFailure(List.of(new VersionedPackage(APP_A, VERSION_CODE)),
|
|
PackageWatchdog.FAILURE_REASON_UNKNOWN);
|
|
mTestLooper.dispatchAll();
|
|
assertThat(observer.mMitigatedPackages).isEqualTo(List.of(APP_A));
|
|
}
|
|
|
|
/**
|
|
* Ensure that the sliding window logic results in the correct mitigation count being sent to
|
|
* an observer.
|
|
*/
|
|
@Test
|
|
public void testMitigationSlidingWindow() {
|
|
PackageWatchdog watchdog = createWatchdog();
|
|
TestObserver observer = new TestObserver(OBSERVER_NAME_1);
|
|
watchdog.startObservingHealth(observer, List.of(APP_A),
|
|
PackageWatchdog.DEFAULT_OBSERVING_DURATION_MS * 2);
|
|
|
|
|
|
raiseFatalFailureAndDispatch(watchdog, Arrays.asList(new VersionedPackage(APP_A,
|
|
VERSION_CODE)), PackageWatchdog.FAILURE_REASON_UNKNOWN);
|
|
|
|
moveTimeForwardAndDispatch(TimeUnit.MINUTES.toMillis(10));
|
|
|
|
raiseFatalFailureAndDispatch(watchdog, Arrays.asList(new VersionedPackage(APP_A,
|
|
VERSION_CODE)), PackageWatchdog.FAILURE_REASON_UNKNOWN);
|
|
raiseFatalFailureAndDispatch(watchdog, Arrays.asList(new VersionedPackage(APP_A,
|
|
VERSION_CODE)), PackageWatchdog.FAILURE_REASON_UNKNOWN);
|
|
|
|
moveTimeForwardAndDispatch(PackageWatchdog.DEFAULT_DEESCALATION_WINDOW_MS);
|
|
|
|
// The first failure will be outside the threshold.
|
|
raiseFatalFailureAndDispatch(watchdog, Arrays.asList(new VersionedPackage(APP_A,
|
|
VERSION_CODE)), PackageWatchdog.FAILURE_REASON_UNKNOWN);
|
|
|
|
moveTimeForwardAndDispatch(TimeUnit.MINUTES.toMillis(20));
|
|
|
|
// The next 2 failures will also be outside the threshold.
|
|
raiseFatalFailureAndDispatch(watchdog, Arrays.asList(new VersionedPackage(APP_A,
|
|
VERSION_CODE)), PackageWatchdog.FAILURE_REASON_UNKNOWN);
|
|
raiseFatalFailureAndDispatch(watchdog, Arrays.asList(new VersionedPackage(APP_A,
|
|
VERSION_CODE)), PackageWatchdog.FAILURE_REASON_UNKNOWN);
|
|
|
|
assertThat(observer.mMitigationCounts).isEqualTo(List.of(1, 2, 3, 3, 2, 3));
|
|
}
|
|
|
|
@Test
|
|
public void testNormalizingMitigationCalls() {
|
|
PackageWatchdog watchdog = createWatchdog();
|
|
|
|
LongArrayQueue mitigationCalls = new LongArrayQueue();
|
|
mitigationCalls.addLast(1000);
|
|
mitigationCalls.addLast(2000);
|
|
mitigationCalls.addLast(3000);
|
|
|
|
MonitoredPackage pkg = watchdog.newMonitoredPackage(
|
|
"test", 123, 456, true, mitigationCalls);
|
|
|
|
// Make current system uptime 10000ms.
|
|
moveTimeForwardAndDispatch(9999);
|
|
|
|
LongArrayQueue expectedCalls = pkg.normalizeMitigationCalls();
|
|
|
|
assertThat(expectedCalls.size()).isEqualTo(mitigationCalls.size());
|
|
|
|
for (int i = 0; i < mitigationCalls.size(); i++) {
|
|
assertThat(expectedCalls.get(i)).isEqualTo(mitigationCalls.get(i) - 10000);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Ensure that a {@link MonitoredPackage} may be correctly written and read in order to persist
|
|
* across reboots.
|
|
*/
|
|
@Test
|
|
public void testWritingAndReadingMonitoredPackage() throws Exception {
|
|
PackageWatchdog watchdog = createWatchdog();
|
|
|
|
LongArrayQueue mitigationCalls = new LongArrayQueue();
|
|
mitigationCalls.addLast(1000);
|
|
mitigationCalls.addLast(2000);
|
|
mitigationCalls.addLast(3000);
|
|
MonitoredPackage writePkg = watchdog.newMonitoredPackage(
|
|
"test.package", 1000, 2000, true, mitigationCalls);
|
|
|
|
// Move time forward so that the current uptime is 4000ms. Therefore, the written mitigation
|
|
// calls will each be reduced by 4000.
|
|
moveTimeForwardAndDispatch(3999);
|
|
LongArrayQueue expectedCalls = new LongArrayQueue();
|
|
expectedCalls.addLast(-3000);
|
|
expectedCalls.addLast(-2000);
|
|
expectedCalls.addLast(-1000);
|
|
MonitoredPackage expectedPkg = watchdog.newMonitoredPackage(
|
|
"test.package", 1000, 2000, true, expectedCalls);
|
|
|
|
// Write the package
|
|
File tmpFile = File.createTempFile("package-watchdog-test", ".xml");
|
|
AtomicFile testFile = new AtomicFile(tmpFile);
|
|
FileOutputStream stream = testFile.startWrite();
|
|
TypedXmlSerializer outputSerializer = Xml.resolveSerializer(stream);
|
|
outputSerializer.startDocument(null, true);
|
|
writePkg.writeLocked(outputSerializer);
|
|
outputSerializer.endDocument();
|
|
testFile.finishWrite(stream);
|
|
|
|
// Read the package
|
|
TypedXmlPullParser parser = Xml.resolvePullParser(testFile.openRead());
|
|
XmlUtils.beginDocument(parser, "package");
|
|
MonitoredPackage readPkg = watchdog.parseMonitoredPackage(parser);
|
|
|
|
assertTrue(readPkg.isEqualTo(expectedPkg));
|
|
}
|
|
|
|
/**
|
|
* Tests device config changes are propagated correctly.
|
|
*/
|
|
@Test
|
|
public void testDeviceConfigChange_explicitHealthCheckEnabled() throws Exception {
|
|
TestController controller = new TestController();
|
|
PackageWatchdog watchdog = createWatchdog(controller, true /* withPackagesReady */);
|
|
assertThat(controller.mIsEnabled).isTrue();
|
|
|
|
DeviceConfig.setProperty(DeviceConfig.NAMESPACE_ROLLBACK,
|
|
PackageWatchdog.PROPERTY_WATCHDOG_EXPLICIT_HEALTH_CHECK_ENABLED,
|
|
Boolean.toString(false), /*makeDefault*/false);
|
|
retry(() -> !controller.mIsEnabled);
|
|
assertThat(controller.mIsEnabled).isFalse();
|
|
}
|
|
|
|
/**
|
|
* Tests device config changes are propagated correctly.
|
|
*/
|
|
@Test
|
|
public void testDeviceConfigChange_triggerFailureCount() throws Exception {
|
|
PackageWatchdog watchdog = createWatchdog();
|
|
|
|
DeviceConfig.setProperty(DeviceConfig.NAMESPACE_ROLLBACK,
|
|
PackageWatchdog.PROPERTY_WATCHDOG_TRIGGER_FAILURE_COUNT,
|
|
Integer.toString(777), false);
|
|
retry(() -> watchdog.getTriggerFailureCount() == 777);
|
|
assertThat(watchdog.getTriggerFailureCount()).isEqualTo(777);
|
|
|
|
DeviceConfig.setProperty(DeviceConfig.NAMESPACE_ROLLBACK,
|
|
PackageWatchdog.PROPERTY_WATCHDOG_TRIGGER_FAILURE_COUNT,
|
|
Integer.toString(0), false);
|
|
retry(() -> watchdog.getTriggerFailureCount()
|
|
== PackageWatchdog.DEFAULT_TRIGGER_FAILURE_COUNT);
|
|
assertThat(watchdog.getTriggerFailureCount()).isEqualTo(
|
|
PackageWatchdog.DEFAULT_TRIGGER_FAILURE_COUNT);
|
|
}
|
|
|
|
/**
|
|
* Tests device config changes are propagated correctly.
|
|
*/
|
|
@Test
|
|
public void testDeviceConfigChange_triggerFailureDurationMs() throws Exception {
|
|
PackageWatchdog watchdog = createWatchdog();
|
|
|
|
DeviceConfig.setProperty(DeviceConfig.NAMESPACE_ROLLBACK,
|
|
PackageWatchdog.PROPERTY_WATCHDOG_TRIGGER_DURATION_MILLIS,
|
|
Integer.toString(888), false);
|
|
retry(() -> watchdog.getTriggerFailureDurationMs() == 888);
|
|
assertThat(watchdog.getTriggerFailureDurationMs()).isEqualTo(888);
|
|
|
|
DeviceConfig.setProperty(DeviceConfig.NAMESPACE_ROLLBACK,
|
|
PackageWatchdog.PROPERTY_WATCHDOG_TRIGGER_DURATION_MILLIS,
|
|
Integer.toString(0), false);
|
|
retry(() -> watchdog.getTriggerFailureDurationMs()
|
|
== PackageWatchdog.DEFAULT_TRIGGER_FAILURE_DURATION_MS);
|
|
assertThat(watchdog.getTriggerFailureDurationMs()).isEqualTo(
|
|
PackageWatchdog.DEFAULT_TRIGGER_FAILURE_DURATION_MS);
|
|
}
|
|
|
|
private void adoptShellPermissions(String... permissions) {
|
|
InstrumentationRegistry
|
|
.getInstrumentation()
|
|
.getUiAutomation()
|
|
.adoptShellPermissionIdentity(permissions);
|
|
}
|
|
|
|
private void dropShellPermissions() {
|
|
InstrumentationRegistry
|
|
.getInstrumentation()
|
|
.getUiAutomation()
|
|
.dropShellPermissionIdentity();
|
|
}
|
|
|
|
private void setExplicitHealthCheckEnabled(boolean enabled) {
|
|
DeviceConfig.setProperty(DeviceConfig.NAMESPACE_ROLLBACK,
|
|
PackageWatchdog.PROPERTY_WATCHDOG_EXPLICIT_HEALTH_CHECK_ENABLED,
|
|
Boolean.toString(enabled), /*makeDefault*/false);
|
|
// Call updateConfigs() so device config changes take effect immediately
|
|
for (PackageWatchdog watchdog : mAllocatedWatchdogs) {
|
|
watchdog.updateConfigs();
|
|
}
|
|
}
|
|
|
|
private void moveTimeForwardAndDispatch(long milliSeconds) {
|
|
// Exhaust all due runnables now which shouldn't be executed after time-leap
|
|
mTestLooper.dispatchAll();
|
|
mTestClock.moveTimeForward(milliSeconds);
|
|
mTestLooper.moveTimeForward(milliSeconds);
|
|
mTestLooper.dispatchAll();
|
|
}
|
|
|
|
/** Trigger package failures above the threshold. */
|
|
private void raiseFatalFailureAndDispatch(PackageWatchdog watchdog,
|
|
List<VersionedPackage> packages, int failureReason) {
|
|
long triggerFailureCount = watchdog.getTriggerFailureCount();
|
|
if (failureReason == PackageWatchdog.FAILURE_REASON_EXPLICIT_HEALTH_CHECK
|
|
|| failureReason == PackageWatchdog.FAILURE_REASON_NATIVE_CRASH) {
|
|
triggerFailureCount = 1;
|
|
}
|
|
for (int i = 0; i < triggerFailureCount; i++) {
|
|
watchdog.onPackageFailure(packages, failureReason);
|
|
}
|
|
mTestLooper.dispatchAll();
|
|
}
|
|
|
|
private PackageWatchdog createWatchdog() {
|
|
return createWatchdog(new TestController(), true /* withPackagesReady */);
|
|
}
|
|
|
|
private PackageWatchdog createWatchdog(TestController controller, boolean withPackagesReady) {
|
|
AtomicFile policyFile =
|
|
new AtomicFile(new File(mSpyContext.getFilesDir(), "package-watchdog.xml"));
|
|
Handler handler = new Handler(mTestLooper.getLooper());
|
|
PackageWatchdog watchdog =
|
|
new PackageWatchdog(mSpyContext, policyFile, handler, handler, controller,
|
|
mConnectivityModuleConnector, mTestClock);
|
|
// Verify controller is not automatically started
|
|
assertThat(controller.mIsEnabled).isFalse();
|
|
if (withPackagesReady) {
|
|
// Only capture the NetworkStack callback for the latest registered watchdog
|
|
reset(mConnectivityModuleConnector);
|
|
watchdog.onPackagesReady();
|
|
// Verify controller by default is started when packages are ready
|
|
assertThat(controller.mIsEnabled).isTrue();
|
|
|
|
verify(mConnectivityModuleConnector).registerHealthListener(
|
|
mConnectivityModuleCallbackCaptor.capture());
|
|
}
|
|
mAllocatedWatchdogs.add(watchdog);
|
|
return watchdog;
|
|
}
|
|
|
|
private static class TestObserver implements PackageHealthObserver {
|
|
private final String mName;
|
|
private int mImpact;
|
|
private int mLastFailureReason;
|
|
private boolean mIsPersistent = false;
|
|
private boolean mMayObservePackages = false;
|
|
private boolean mMitigatedBootLoop = false;
|
|
final List<String> mHealthCheckFailedPackages = new ArrayList<>();
|
|
final List<String> mMitigatedPackages = new ArrayList<>();
|
|
final List<Integer> mMitigationCounts = new ArrayList<>();
|
|
final List<Integer> mBootMitigationCounts = new ArrayList<>();
|
|
|
|
TestObserver(String name) {
|
|
mName = name;
|
|
mImpact = PackageHealthObserverImpact.USER_IMPACT_MEDIUM;
|
|
}
|
|
|
|
TestObserver(String name, int impact) {
|
|
mName = name;
|
|
mImpact = impact;
|
|
}
|
|
|
|
public int onHealthCheckFailed(VersionedPackage versionedPackage, int failureReason,
|
|
int mitigationCount) {
|
|
mHealthCheckFailedPackages.add(versionedPackage.getPackageName());
|
|
return mImpact;
|
|
}
|
|
|
|
public boolean execute(VersionedPackage versionedPackage, int failureReason,
|
|
int mitigationCount) {
|
|
mMitigatedPackages.add(versionedPackage.getPackageName());
|
|
mMitigationCounts.add(mitigationCount);
|
|
mLastFailureReason = failureReason;
|
|
return true;
|
|
}
|
|
|
|
public String getName() {
|
|
return mName;
|
|
}
|
|
|
|
public boolean isPersistent() {
|
|
return mIsPersistent;
|
|
}
|
|
|
|
public boolean mayObservePackage(String packageName) {
|
|
return mMayObservePackages;
|
|
}
|
|
|
|
public int onBootLoop(int level) {
|
|
return mImpact;
|
|
}
|
|
|
|
public boolean executeBootLoopMitigation(int level) {
|
|
mMitigatedBootLoop = true;
|
|
mBootMitigationCounts.add(level);
|
|
return true;
|
|
}
|
|
|
|
public boolean mitigatedBootLoop() {
|
|
return mMitigatedBootLoop;
|
|
}
|
|
|
|
public int getLastFailureReason() {
|
|
return mLastFailureReason;
|
|
}
|
|
|
|
public void setPersistent(boolean persistent) {
|
|
mIsPersistent = persistent;
|
|
}
|
|
|
|
public void setImpact(int impact) {
|
|
mImpact = impact;
|
|
}
|
|
|
|
public void setMayObservePackages(boolean mayObservePackages) {
|
|
mMayObservePackages = mayObservePackages;
|
|
}
|
|
}
|
|
|
|
private static class TestController extends ExplicitHealthCheckController {
|
|
TestController() {
|
|
super(null /* controller */);
|
|
}
|
|
|
|
private boolean mIsEnabled;
|
|
private List<String> mSupportedPackages = new ArrayList<>();
|
|
private List<String> mRequestedPackages = new ArrayList<>();
|
|
private Consumer<String> mPassedConsumer;
|
|
private Consumer<List<PackageConfig>> mSupportedConsumer;
|
|
private Runnable mNotifySyncRunnable;
|
|
private List<Set> mSyncRequests = new ArrayList<>();
|
|
|
|
@Override
|
|
public void setEnabled(boolean enabled) {
|
|
mIsEnabled = enabled;
|
|
if (!mIsEnabled) {
|
|
mSupportedPackages.clear();
|
|
}
|
|
}
|
|
|
|
@Override
|
|
public void setCallbacks(Consumer<String> passedConsumer,
|
|
Consumer<List<PackageConfig>> supportedConsumer, Runnable notifySyncRunnable) {
|
|
mPassedConsumer = passedConsumer;
|
|
mSupportedConsumer = supportedConsumer;
|
|
mNotifySyncRunnable = notifySyncRunnable;
|
|
}
|
|
|
|
@Override
|
|
public void syncRequests(Set<String> packages) {
|
|
mSyncRequests.add(packages);
|
|
mRequestedPackages.clear();
|
|
if (mIsEnabled) {
|
|
packages.retainAll(mSupportedPackages);
|
|
mRequestedPackages.addAll(packages);
|
|
List<PackageConfig> packageConfigs = new ArrayList<>();
|
|
for (String packageName: packages) {
|
|
packageConfigs.add(new PackageConfig(packageName, SHORT_DURATION));
|
|
}
|
|
mSupportedConsumer.accept(packageConfigs);
|
|
} else {
|
|
mSupportedConsumer.accept(Collections.emptyList());
|
|
}
|
|
}
|
|
|
|
public void setSupportedPackages(List<String> packages) {
|
|
mSupportedPackages.clear();
|
|
mSupportedPackages.addAll(packages);
|
|
}
|
|
|
|
public void setPackagePassed(String packageName) {
|
|
mPassedConsumer.accept(packageName);
|
|
}
|
|
|
|
public List<String> getRequestedPackages() {
|
|
if (mIsEnabled) {
|
|
return mRequestedPackages;
|
|
} else {
|
|
return Collections.emptyList();
|
|
}
|
|
}
|
|
|
|
public List<Set> getSyncRequests() {
|
|
return mSyncRequests;
|
|
}
|
|
}
|
|
|
|
private static class TestClock implements PackageWatchdog.SystemClock {
|
|
// Note 0 is special to the internal clock of PackageWatchdog. We need to start from
|
|
// a non-zero value in order not to disrupt the logic of PackageWatchdog.
|
|
private long mUpTimeMillis = 1;
|
|
@Override
|
|
public long uptimeMillis() {
|
|
return mUpTimeMillis;
|
|
}
|
|
public void moveTimeForward(long milliSeconds) {
|
|
mUpTimeMillis += milliSeconds;
|
|
}
|
|
}
|
|
}
|