From c429f690e54db1bdc4631aab7c0191e30ccf1d51 Mon Sep 17 00:00:00 2001 From: Jason Monk Date: Tue, 27 Jun 2017 13:13:49 -0400 Subject: [PATCH] Add system to avoid a crash stopping tests from running Add a system such that when a crash happens on the main thread, it is delayed until the tests complete, then thrown. Also mark some old flaky tests as not flaky as they should be good now. Test: runtest systemui Change-Id: Ic18a38daf4a93dab63549d834fa00f295644fbf1 Fixes: 62935720, 62251903, 62529569 --- packages/SystemUI/tests/AndroidManifest.xml | 2 +- .../android/systemui/RoundedCornersTest.java | 9 +- .../com/android/systemui/SysuiTestCase.java | 14 +- .../systemui/doze/DozeTriggersTest.java | 1 - .../com/android/systemui/qs/QSDetailTest.java | 4 +- .../com/android/systemui/qs/QSFooterTest.java | 1 - .../com/android/systemui/qs/QSPanelTest.java | 3 - .../ExpandableNotificationRowTest.java | 8 +- .../NotificationContentViewTest.java | 25 +-- .../NotificationCustomViewWrapperTest.java | 5 - .../NotificationChildrenContainerTest.java | 6 - .../tests/notification/AndroidManifest.xml | 2 +- .../testing/TestableInstrumentation.java | 181 ++++++++++++++++++ .../src/android/testing/TestableLooper.java | 19 +- 14 files changed, 225 insertions(+), 55 deletions(-) create mode 100644 tests/testables/src/android/testing/TestableInstrumentation.java diff --git a/packages/SystemUI/tests/AndroidManifest.xml b/packages/SystemUI/tests/AndroidManifest.xml index b12fd1c9ba893..9bb21808e9bcb 100644 --- a/packages/SystemUI/tests/AndroidManifest.xml +++ b/packages/SystemUI/tests/AndroidManifest.xml @@ -48,7 +48,7 @@ android:process=":killable" /> - diff --git a/packages/SystemUI/tests/src/com/android/systemui/RoundedCornersTest.java b/packages/SystemUI/tests/src/com/android/systemui/RoundedCornersTest.java index 945af34c1df10..507515a8c2e92 100644 --- a/packages/SystemUI/tests/src/com/android/systemui/RoundedCornersTest.java +++ b/packages/SystemUI/tests/src/com/android/systemui/RoundedCornersTest.java @@ -21,7 +21,6 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; @@ -30,7 +29,6 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import android.app.Fragment; -import android.support.test.filters.FlakyTest; import android.support.test.filters.SmallTest; import android.testing.AndroidTestingRunner; import android.view.Display; @@ -48,12 +46,10 @@ import com.android.systemui.tuner.TunablePadding.TunablePaddingService; import com.android.systemui.tuner.TunerService; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; @RunWith(AndroidTestingRunner.class) -@FlakyTest @SmallTest public class RoundedCornersTest extends SysuiTestCase { @@ -72,6 +68,7 @@ public class RoundedCornersTest extends SysuiTestCase { mWindowManager = mock(WindowManager.class); mView = spy(new StatusBarWindowView(mContext, null)); when(mStatusBar.getStatusBarWindow()).thenReturn(mView); + when(mStatusBar.getNavigationBarWindow()).thenReturn(mView); mContext.putComponent(StatusBar.class, mStatusBar); Display display = mContext.getSystemService(WindowManager.class).getDefaultDisplay(); @@ -94,6 +91,8 @@ public class RoundedCornersTest extends SysuiTestCase { @Test public void testNoRounding() { mContext.getOrCreateTestableResources().addOverride(dimen.rounded_corner_radius, 0); + mContext.getOrCreateTestableResources() + .addOverride(dimen.rounded_corner_content_padding, 0); mRoundedCorners.start(); // No views added. @@ -107,6 +106,8 @@ public class RoundedCornersTest extends SysuiTestCase { @Test public void testRounding() { mContext.getOrCreateTestableResources().addOverride(dimen.rounded_corner_radius, 20); + mContext.getOrCreateTestableResources() + .addOverride(dimen.rounded_corner_content_padding, 20); mRoundedCorners.start(); // Add 2 windows for rounded corners (top and bottom). diff --git a/packages/SystemUI/tests/src/com/android/systemui/SysuiTestCase.java b/packages/SystemUI/tests/src/com/android/systemui/SysuiTestCase.java index 361a20fb7bdc6..66d00dd6c5c67 100644 --- a/packages/SystemUI/tests/src/com/android/systemui/SysuiTestCase.java +++ b/packages/SystemUI/tests/src/com/android/systemui/SysuiTestCase.java @@ -31,6 +31,8 @@ import android.util.Log; import org.junit.After; import org.junit.Before; import org.junit.Rule; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; @@ -56,10 +58,14 @@ public abstract class SysuiTestCase { mRealInstrumentation = InstrumentationRegistry.getInstrumentation(); Instrumentation inst = spy(mRealInstrumentation); - when(inst.getContext()).thenThrow(new RuntimeException( - "SysUI Tests should use SysuiTestCase#getContext or SysuiTestCase#mContext")); - when(inst.getTargetContext()).thenThrow(new RuntimeException( - "SysUI Tests should use SysuiTestCase#getContext or SysuiTestCase#mContext")); + when(inst.getContext()).thenAnswer(invocation -> { + throw new RuntimeException( + "SysUI Tests should use SysuiTestCase#getContext or SysuiTestCase#mContext"); + }); + when(inst.getTargetContext()).thenAnswer(invocation -> { + throw new RuntimeException( + "SysUI Tests should use SysuiTestCase#getContext or SysuiTestCase#mContext"); + }); InstrumentationRegistry.registerInstance(inst, InstrumentationRegistry.getArguments()); } diff --git a/packages/SystemUI/tests/src/com/android/systemui/doze/DozeTriggersTest.java b/packages/SystemUI/tests/src/com/android/systemui/doze/DozeTriggersTest.java index 2363b2aadc102..8641faca5d6e2 100644 --- a/packages/SystemUI/tests/src/com/android/systemui/doze/DozeTriggersTest.java +++ b/packages/SystemUI/tests/src/com/android/systemui/doze/DozeTriggersTest.java @@ -27,7 +27,6 @@ import android.app.Instrumentation; import android.os.Handler; import android.os.Looper; import android.support.test.InstrumentationRegistry; -import android.support.test.filters.FlakyTest; import android.support.test.filters.SmallTest; import android.support.test.runner.AndroidJUnit4; diff --git a/packages/SystemUI/tests/src/com/android/systemui/qs/QSDetailTest.java b/packages/SystemUI/tests/src/com/android/systemui/qs/QSDetailTest.java index 6a85511daca2a..ed47fbb890774 100644 --- a/packages/SystemUI/tests/src/com/android/systemui/qs/QSDetailTest.java +++ b/packages/SystemUI/tests/src/com/android/systemui/qs/QSDetailTest.java @@ -23,10 +23,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import org.junit.After; -import org.junit.Ignore; import android.support.test.filters.SmallTest; -import android.support.test.filters.FlakyTest; import android.testing.AndroidTestingRunner; import android.testing.TestableLooper; import android.testing.TestableLooper.RunWithLooper; @@ -41,6 +38,7 @@ import com.android.systemui.plugins.ActivityStarter; import com.android.systemui.plugins.qs.DetailAdapter; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/packages/SystemUI/tests/src/com/android/systemui/qs/QSFooterTest.java b/packages/SystemUI/tests/src/com/android/systemui/qs/QSFooterTest.java index f4fda065860e4..d25bbe1e4904d 100644 --- a/packages/SystemUI/tests/src/com/android/systemui/qs/QSFooterTest.java +++ b/packages/SystemUI/tests/src/com/android/systemui/qs/QSFooterTest.java @@ -21,7 +21,6 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import android.support.test.filters.SmallTest; -import android.support.test.filters.FlakyTest; import android.testing.AndroidTestingRunner; import android.testing.TestableLooper; import android.testing.TestableLooper.RunWithLooper; diff --git a/packages/SystemUI/tests/src/com/android/systemui/qs/QSPanelTest.java b/packages/SystemUI/tests/src/com/android/systemui/qs/QSPanelTest.java index 637b2440af09d..85cdfcc2ce093 100644 --- a/packages/SystemUI/tests/src/com/android/systemui/qs/QSPanelTest.java +++ b/packages/SystemUI/tests/src/com/android/systemui/qs/QSPanelTest.java @@ -20,7 +20,6 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import android.support.test.filters.SmallTest; -import android.support.test.filters.FlakyTest; import android.testing.AndroidTestingRunner; import android.testing.TestableLooper; import android.testing.TestableLooper.RunWithLooper; @@ -31,7 +30,6 @@ import com.android.systemui.SysuiTestCase; import com.android.systemui.qs.customize.QSCustomizer; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; @@ -40,7 +38,6 @@ import java.util.Collections; @RunWith(AndroidTestingRunner.class) @RunWithLooper @SmallTest -@FlakyTest public class QSPanelTest extends SysuiTestCase { private MetricsLogger mMetricsLogger; diff --git a/packages/SystemUI/tests/src/com/android/systemui/statusbar/ExpandableNotificationRowTest.java b/packages/SystemUI/tests/src/com/android/systemui/statusbar/ExpandableNotificationRowTest.java index 664ea710d61e8..2f6511c8481eb 100644 --- a/packages/SystemUI/tests/src/com/android/systemui/statusbar/ExpandableNotificationRowTest.java +++ b/packages/SystemUI/tests/src/com/android/systemui/statusbar/ExpandableNotificationRowTest.java @@ -21,27 +21,21 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; -import android.content.Context; -import android.support.test.InstrumentationRegistry; -import android.support.test.annotation.UiThreadTest; -import android.support.test.filters.FlakyTest; import android.support.test.filters.SmallTest; import android.support.test.runner.AndroidJUnit4; import android.view.View; +import com.android.systemui.SysuiTestCase; import com.android.systemui.statusbar.notification.AboveShelfChangedListener; import com.android.systemui.statusbar.stack.NotificationChildrenContainer; -import com.android.systemui.SysuiTestCase; import org.junit.Assert; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; @SmallTest @RunWith(AndroidJUnit4.class) -@FlakyTest public class ExpandableNotificationRowTest extends SysuiTestCase { private ExpandableNotificationRow mGroup; diff --git a/packages/SystemUI/tests/src/com/android/systemui/statusbar/NotificationContentViewTest.java b/packages/SystemUI/tests/src/com/android/systemui/statusbar/NotificationContentViewTest.java index 4dce2f5f87ed4..436849c9d700a 100644 --- a/packages/SystemUI/tests/src/com/android/systemui/statusbar/NotificationContentViewTest.java +++ b/packages/SystemUI/tests/src/com/android/systemui/statusbar/NotificationContentViewTest.java @@ -16,30 +16,25 @@ package com.android.systemui.statusbar; -import android.content.Context; -import android.support.test.InstrumentationRegistry; -import android.support.test.annotation.UiThreadTest; -import android.support.test.filters.FlakyTest; -import android.support.test.filters.SmallTest; -import android.support.test.runner.AndroidJUnit4; -import android.view.View; - -import org.junit.Assert; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; -import org.junit.runner.RunWith; - import static org.mockito.ArgumentMatchers.anyFloat; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.spy; +import android.support.test.annotation.UiThreadTest; +import android.support.test.filters.SmallTest; +import android.support.test.runner.AndroidJUnit4; +import android.view.View; + import com.android.systemui.SysuiTestCase; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; + @SmallTest @RunWith(AndroidJUnit4.class) -@FlakyTest public class NotificationContentViewTest extends SysuiTestCase { NotificationContentView mView; diff --git a/packages/SystemUI/tests/src/com/android/systemui/statusbar/NotificationCustomViewWrapperTest.java b/packages/SystemUI/tests/src/com/android/systemui/statusbar/NotificationCustomViewWrapperTest.java index d18e63bcc1519..6e59d10aad3cd 100644 --- a/packages/SystemUI/tests/src/com/android/systemui/statusbar/NotificationCustomViewWrapperTest.java +++ b/packages/SystemUI/tests/src/com/android/systemui/statusbar/NotificationCustomViewWrapperTest.java @@ -16,10 +16,7 @@ package com.android.systemui.statusbar; -import android.content.Context; -import android.support.test.InstrumentationRegistry; import android.support.test.annotation.UiThreadTest; -import android.support.test.filters.FlakyTest; import android.support.test.filters.SmallTest; import android.support.test.runner.AndroidJUnit4; import android.view.View; @@ -32,13 +29,11 @@ import com.android.systemui.statusbar.notification.NotificationViewWrapper; import org.junit.Assert; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; @SmallTest @RunWith(AndroidJUnit4.class) -@FlakyTest public class NotificationCustomViewWrapperTest extends SysuiTestCase { private ExpandableNotificationRow mRow; diff --git a/packages/SystemUI/tests/src/com/android/systemui/statusbar/stack/NotificationChildrenContainerTest.java b/packages/SystemUI/tests/src/com/android/systemui/statusbar/stack/NotificationChildrenContainerTest.java index bc6833df1e304..5ac965cf4042e 100644 --- a/packages/SystemUI/tests/src/com/android/systemui/statusbar/stack/NotificationChildrenContainerTest.java +++ b/packages/SystemUI/tests/src/com/android/systemui/statusbar/stack/NotificationChildrenContainerTest.java @@ -16,10 +16,6 @@ package com.android.systemui.statusbar.stack; -import android.content.Context; -import android.support.test.InstrumentationRegistry; -import android.support.test.annotation.UiThreadTest; -import android.support.test.filters.FlakyTest; import android.support.test.filters.SmallTest; import android.support.test.runner.AndroidJUnit4; import android.view.NotificationHeaderView; @@ -31,13 +27,11 @@ import com.android.systemui.statusbar.NotificationTestHelper; import org.junit.Assert; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; @SmallTest @RunWith(AndroidJUnit4.class) -@FlakyTest public class NotificationChildrenContainerTest extends SysuiTestCase { private ExpandableNotificationRow mGroup; diff --git a/services/tests/notification/AndroidManifest.xml b/services/tests/notification/AndroidManifest.xml index 99d9c7b873019..c20020abb85c7 100644 --- a/services/tests/notification/AndroidManifest.xml +++ b/services/tests/notification/AndroidManifest.xml @@ -31,7 +31,7 @@ diff --git a/tests/testables/src/android/testing/TestableInstrumentation.java b/tests/testables/src/android/testing/TestableInstrumentation.java new file mode 100644 index 0000000000000..93fed859cf391 --- /dev/null +++ b/tests/testables/src/android/testing/TestableInstrumentation.java @@ -0,0 +1,181 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package android.testing; + +import android.os.Bundle; +import android.os.Handler; +import android.os.Looper; +import android.os.Message; +import android.os.TestLooperManager; +import android.support.test.runner.AndroidJUnitRunner; +import android.util.Log; + +import java.util.ArrayList; + +/** + * Wrapper around instrumentation that spins up a TestLooperManager around + * the main looper whenever a test is not using it to attempt to stop crashes + * from stopping other tests from running. + */ +public class TestableInstrumentation extends AndroidJUnitRunner { + + private static final String TAG = "TestableInstrumentation"; + + private static final int MAX_CRASHES = 5; + private static MainLooperManager sManager; + + @Override + public void onCreate(Bundle arguments) { + sManager = new MainLooperManager(); + Log.setWtfHandler((tag, what, system) -> { + if (system) { + Log.e(TAG, "WTF!!", what); + } else { + // These normally kill the app, but we don't want that in a test, instead we want + // it to throw. + throw new RuntimeException(what); + } + }); + super.onCreate(arguments); + } + + @Override + public void finish(int resultCode, Bundle results) { + sManager.destroy(); + super.finish(resultCode, results); + } + + public static void acquireMain() { + if (sManager != null) { + sManager.acquireMain(); + } + } + + public static void releaseMain() { + if (sManager != null) { + sManager.releaseMain(); + } + } + + public class MainLooperManager implements Runnable { + + private final ArrayList mExceptions = new ArrayList<>(); + private Message mStopMessage; + private final Handler mMainHandler; + private TestLooperManager mManager; + + public MainLooperManager() { + mMainHandler = new Handler(Looper.getMainLooper()); + startManaging(); + } + + @Override + public void run() { + try { + synchronized (this) { + // Let the thing starting us know we are up and ready to run. + notify(); + } + while (true) { + Message m = mManager.next(); + if (m == mStopMessage) { + mManager.recycle(m); + return; + } + try { + mManager.execute(m); + } catch (Throwable t) { + if (!checkStack(t) || (mExceptions.size() == MAX_CRASHES)) { + throw t; + } + mExceptions.add(t); + Log.d(TAG, "Ignoring exception to run more tests", t); + } + mManager.recycle(m); + } + } finally { + mManager.release(); + synchronized (this) { + // Let the caller know we are done managing the main thread. + notify(); + } + } + } + + private boolean checkStack(Throwable t) { + StackTraceElement topStack = t.getStackTrace()[0]; + String className = topStack.getClassName(); + if (className.equals(TestLooperManager.class.getName())) { + topStack = t.getCause().getStackTrace()[0]; + className = topStack.getClassName(); + } + // Only interested in blocking exceptions from the app itself, not from android + // framework. + return !className.startsWith("android.") + && !className.startsWith("com.android.internal"); + } + + public void destroy() { + mStopMessage.sendToTarget(); + if (mExceptions.size() != 0) { + throw new RuntimeException("Exception caught during tests", mExceptions.get(0)); + } + } + + public void acquireMain() { + synchronized (this) { + mStopMessage.sendToTarget(); + try { + wait(); + } catch (InterruptedException e) { + } + } + } + + public void releaseMain() { + startManaging(); + } + + private void startManaging() { + mStopMessage = mMainHandler.obtainMessage(); + synchronized (this) { + mManager = acquireLooperManager(Looper.getMainLooper()); + // This bit needs to happen on a background thread or it will hang if called + // from the same thread we are looking to block. + new Thread(() -> { + // Post a message to the main handler that will manage executing all future + // messages. + mMainHandler.post(this); + while (!mManager.hasMessages(mMainHandler, null, this)); + // Lastly run the message that executes this so it can manage the main thread. + Message next = mManager.next(); + // Run through messages until we reach ours. + while (next.getCallback() != this) { + mManager.execute(next); + mManager.recycle(next); + next = mManager.next(); + } + mManager.execute(next); + }).start(); + if (Looper.myLooper() != Looper.getMainLooper()) { + try { + wait(); + } catch (InterruptedException e) { + } + } + } + } + } +} diff --git a/tests/testables/src/android/testing/TestableLooper.java b/tests/testables/src/android/testing/TestableLooper.java index f6c3cb3ec4981..f1a70921a4690 100644 --- a/tests/testables/src/android/testing/TestableLooper.java +++ b/tests/testables/src/android/testing/TestableLooper.java @@ -29,7 +29,6 @@ import java.lang.annotation.ElementType; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; -import java.lang.reflect.Field; import java.util.Map; /** @@ -49,7 +48,7 @@ public class TestableLooper { private TestLooperManager mQueueWrapper; public TestableLooper(Looper l) throws Exception { - this(InstrumentationRegistry.getInstrumentation().acquireLooperManager(l), l); + this(acquireLooperManager(l), l); } private TestableLooper(TestLooperManager wrapper, Looper l) throws Exception { @@ -78,6 +77,9 @@ public class TestableLooper { */ public void destroy() throws NoSuchFieldException, IllegalAccessException { mQueueWrapper.release(); + if (mLooper == Looper.getMainLooper()) { + TestableInstrumentation.releaseMain(); + } } /** @@ -196,6 +198,13 @@ public class TestableLooper { } } + private static TestLooperManager acquireLooperManager(Looper l) { + if (l == Looper.getMainLooper()) { + TestableInstrumentation.acquireMain(); + } + return InstrumentationRegistry.getInstrumentation().acquireLooperManager(l); + } + private static final Map sLoopers = new ArrayMap<>(); /** @@ -247,8 +256,7 @@ public class TestableLooper { } boolean set = mTestableLooper.mQueueWrapper == null; if (set) { - mTestableLooper.mQueueWrapper = InstrumentationRegistry.getInstrumentation() - .acquireLooperManager(mLooper); + mTestableLooper.mQueueWrapper = acquireLooperManager(mLooper); } try { Object[] ret = new Object[1]; @@ -283,6 +291,9 @@ public class TestableLooper { if (set) { mTestableLooper.mQueueWrapper.release(); mTestableLooper.mQueueWrapper = null; + if (mLooper == Looper.getMainLooper()) { + TestableInstrumentation.releaseMain(); + } } } }