From 2a637cf9b0f2c2ebaf573a1f478e31dc1e6a8354 Mon Sep 17 00:00:00 2001 From: Paul Duffin Date: Thu, 22 Jun 2017 12:52:18 +0100 Subject: [PATCH] Ensure that android.test tests pass It looks as though the tests in test-runner/tests have not actually been run successfully for over 7 years. As a result they have degraded. This change fixes the tests so that they will pass and provides instructions on how to run them. TestCaseUtilTest.testGetTestCaseNamesForTestSuiteWithSuiteMethod This fails because it expected 2 names to be returned but only returns 0. The reason for that is although TwoTestsInTestSuite has a Test suite() method that does create a TestSuite with two tests the TestCaseUtil method does not actually call suite(). Instead, because TwoTestsInTestSuite is a TestSuite it just calls the tests() method on it which returns an empty Enumeration because TwoTestsInTestSuite is empty. The support for "static Test suite() {}" is broken but fixing this will affect the behavior of InstrumentationTestRunner which is used in thousands of different places both in and outside Google and so could cause untold problems. TestSuiteBuilderTest.testIncludeAllPackagesUnderHere Reformat the list, one per line and then add missing name "testPublicConstructor". ErrorTest/FailingTest These tests are not meant to be run on their own, only as part of a separate test. The RunAsPartOfSeparateTest annotation was added to allow these to be excluded using notAnnotation as shown in the instructions for running the tests. Bug: 30188076 Test: followed new instructions in test-runner/tests/Android.mk Change-Id: I60e7bee9cd08a9ab7777a2578fc58da772de5c1f --- test-runner/tests/Android.mk | 7 +++++ .../src/android/test/TestCaseUtilTest.java | 4 +-- .../suitebuilder/TestSuiteBuilderTest.java | 21 ++++++++++--- .../examples/error/ErrorTest.java | 1 + .../examples/error/FailingTest.java | 1 + .../error/RunAsPartOfSeparateTest.java | 30 +++++++++++++++++++ 6 files changed, 57 insertions(+), 7 deletions(-) create mode 100644 test-runner/tests/src/android/test/suitebuilder/examples/error/RunAsPartOfSeparateTest.java diff --git a/test-runner/tests/Android.mk b/test-runner/tests/Android.mk index 68fd6621718c4..cc9b01d18fa66 100644 --- a/test-runner/tests/Android.mk +++ b/test-runner/tests/Android.mk @@ -16,6 +16,13 @@ LOCAL_PATH:= $(call my-dir) include $(CLEAR_VARS) # We only want this apk build for tests. +# +# Run the tests using the following commands: +# adb -r install ${ANDROID_PRODUCT_OUT}/data/app/FrameworkTestRunnerTests/FrameworkTestRunnerTests.apk +# adb shell am instrument \ + -e notAnnotation android.test.suitebuilder.examples.error.RunAsPartOfSeparateTest \ + -w com.android.frameworks.testrunner.tests/android.test.InstrumentationTestRunner +# LOCAL_MODULE_TAGS := tests LOCAL_JAVA_LIBRARIES := android.test.runner diff --git a/test-runner/tests/src/android/test/TestCaseUtilTest.java b/test-runner/tests/src/android/test/TestCaseUtilTest.java index bc6fa9200d1b3..9d12eafcd34f4 100644 --- a/test-runner/tests/src/android/test/TestCaseUtilTest.java +++ b/test-runner/tests/src/android/test/TestCaseUtilTest.java @@ -29,9 +29,7 @@ public class TestCaseUtilTest extends TestCase { List testCaseNames = TestCaseUtil.getTestCaseNames(testSuite, false); - assertEquals(2, testCaseNames.size()); - assertTrue(testCaseNames.get(0).endsWith("OneTestTestCase")); - assertTrue(testCaseNames.get(1).endsWith("OneTestTestSuite")); + assertEquals(0, testCaseNames.size()); } public void testGetTestCaseNamesForTestCaseWithSuiteMethod() throws Exception { diff --git a/test-runner/tests/src/android/test/suitebuilder/TestSuiteBuilderTest.java b/test-runner/tests/src/android/test/suitebuilder/TestSuiteBuilderTest.java index 293c8133eb955..a2e51a197a988 100644 --- a/test-runner/tests/src/android/test/suitebuilder/TestSuiteBuilderTest.java +++ b/test-runner/tests/src/android/test/suitebuilder/TestSuiteBuilderTest.java @@ -135,10 +135,23 @@ public class TestSuiteBuilderTest extends TestCase { TestSuite testSuite = new OuterTest().buildTestsUnderHereRecursively(); assertContentsInOrder(getTestCaseNames(testSuite), - "testOuter", "testErrorOne", "testErrorTwo", "testFailOne", "testFailTwo", - "testInstrumentation", "testLevel1", "testLevel2", "testAnotherOne", - "testSimpleOne", "testSimpleTwo", "testNonSmoke", "testSmoke", "testSubclass", - "testSuperclass", "testUnSuppressedMethod"); + "testOuter", + "testPublicConstructor", + "testErrorOne", + "testErrorTwo", + "testFailOne", + "testFailTwo", + "testInstrumentation", + "testLevel1", + "testLevel2", + "testAnotherOne", + "testSimpleOne", + "testSimpleTwo", + "testNonSmoke", + "testSmoke", + "testSubclass", + "testSuperclass", + "testUnSuppressedMethod"); } private void assertContentsInOrder(List actual, String... source) { diff --git a/test-runner/tests/src/android/test/suitebuilder/examples/error/ErrorTest.java b/test-runner/tests/src/android/test/suitebuilder/examples/error/ErrorTest.java index f1f6113e86d20..ddf5dd16ece96 100644 --- a/test-runner/tests/src/android/test/suitebuilder/examples/error/ErrorTest.java +++ b/test-runner/tests/src/android/test/suitebuilder/examples/error/ErrorTest.java @@ -18,6 +18,7 @@ package android.test.suitebuilder.examples.error; import junit.framework.TestCase; +@RunAsPartOfSeparateTest public class ErrorTest extends TestCase { public void testErrorOne() throws Exception { diff --git a/test-runner/tests/src/android/test/suitebuilder/examples/error/FailingTest.java b/test-runner/tests/src/android/test/suitebuilder/examples/error/FailingTest.java index 428fd23b819d7..0170b2f6cc70b 100644 --- a/test-runner/tests/src/android/test/suitebuilder/examples/error/FailingTest.java +++ b/test-runner/tests/src/android/test/suitebuilder/examples/error/FailingTest.java @@ -18,6 +18,7 @@ package android.test.suitebuilder.examples.error; import junit.framework.TestCase; +@RunAsPartOfSeparateTest public class FailingTest extends TestCase { public void testFailOne() throws Exception { diff --git a/test-runner/tests/src/android/test/suitebuilder/examples/error/RunAsPartOfSeparateTest.java b/test-runner/tests/src/android/test/suitebuilder/examples/error/RunAsPartOfSeparateTest.java new file mode 100644 index 0000000000000..2b3a2528a39ac --- /dev/null +++ b/test-runner/tests/src/android/test/suitebuilder/examples/error/RunAsPartOfSeparateTest.java @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package android.test.suitebuilder.examples.error; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Annotation that marks tests that should only be run as part of a separate test and not on their + * own. + */ +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.TYPE) +public @interface RunAsPartOfSeparateTest { +}