Improve InstrumentationTestRunner exception handling.

This commit fixes two somewhat related problems:
  - Attempting to run a method which does not exist caused a runtime
exception which stopped the test run. Change this so the runner reports
an individual test failure instead
  - A runtime exception during the test run would cause it to stop
completely, with no information dumped to the logcat or stdout. Now
exceptions are trapped and reported to stdout.

Also added associated unit tests to test these two conditions.

Related bug 2812262.

Change-Id: I383f9b9bad99f14cb51071800fa9bdbf6a6a1119

test-runner/src/android/test/InstrumentationTestRunner.java

@@ -496,9 +496,18 @@ public class InstrumentationTestRunner extends Instrumentation implements TestSu
         return null;
     }
 
+    /**
+     * Initialize the current thread as a looper.
+     * <p/>
+     * Exposed for unit testing.
+     */
+    void prepareLooper() {
+        Looper.prepare();
+    }
+
     @Override
     public void onStart() {
-        Looper.prepare();
+        prepareLooper();
 
         if (mJustCount) {
             mResults.putString(Instrumentation.REPORT_KEY_IDENTIFIER, REPORT_VALUE_ID);
@@ -521,6 +530,11 @@ public class InstrumentationTestRunner extends Instrumentation implements TestSu
                 long runTime = System.currentTimeMillis() - startTime;
 
                 resultPrinter.print(mTestRunner.getTestResult(), runTime);
+            } catch (Throwable t) {
+                // catch all exceptions so a more verbose error message can be outputted
+                writer.println(String.format("Test run aborted due to unexpected exception: %s",
+                                t.getMessage()));
+                t.printStackTrace(writer);
             } finally {
                 mResults.putString(Instrumentation.REPORT_KEY_STREAMRESULT,
                         String.format("\nTest results for %s=%s",
@@ -762,9 +776,11 @@ public class InstrumentationTestRunner extends Instrumentation implements TestSu
                             TimedTest.class).includeDetailedStats();
                 }
             } catch (SecurityException e) {
-                throw new IllegalStateException(e);
+                // ignore - the test with given name cannot be accessed. Will be handled during
+                // test execution
             } catch (NoSuchMethodException e) {
-                throw new IllegalStateException(e);
+                // ignore- the test with given name does not exist. Will be handled during test
+                // execution
             }
 
             if (mIsTimedTest && mIncludeDetailedStats) {

test-runner/tests/src/android/test/InstrumentationTestRunnerTest.java

@@ -16,6 +16,7 @@
 
 package android.test;
 
+import android.app.Instrumentation;
 import android.content.Context;
 import android.os.Bundle;
 import android.test.mock.MockContext;
@@ -89,6 +90,42 @@ public class InstrumentationTestRunnerTest extends TestCase {
         
     }
 
+    /**
+     * Test that runtime exceptions during runTest are handled gracefully
+     */
+    public void testUnhandledException() throws Exception {
+        StubAndroidTestRunner stubAndroidTestRunner = new StubAndroidTestRunner() {
+            @Override
+            public void runTest() {
+                throw new RuntimeException();
+            }
+        };
+        StubInstrumentationTestRunner instrumentationTestRunner = new StubInstrumentationTestRunner(
+                new StubContext("com.google.foo.tests"),
+                new StubContext(mTargetContextPackageName), stubAndroidTestRunner);
+        instrumentationTestRunner.onCreate(new Bundle());
+        instrumentationTestRunner.onStart();
+        assertTrue("Instrumentation did not finish", instrumentationTestRunner.isFinished());
+        // ensure a meaningful error message placed in results
+        String resultsData = instrumentationTestRunner.mResults.getString(
+                Instrumentation.REPORT_KEY_STREAMRESULT);
+        assertTrue("Instrumentation results is missing RuntimeException",
+                resultsData.contains("RuntimeException"));
+    }
+
+    /**
+     * Test that specifying a method which does not exist is handled gracefully
+     */
+    public void testBadMethodArgument() throws Exception {
+        String testClassName = PlaceHolderTest.class.getName();
+        String invalidMethodName = "testNoExist";
+        String classAndMethod = testClassName + "#" + invalidMethodName;
+        mInstrumentationTestRunner.onCreate(createBundle(
+                InstrumentationTestRunner.ARGUMENT_TEST_CLASS, classAndMethod));
+        assertTestRunnerCalledWithExpectedParameters(testClassName,
+                invalidMethodName);
+    }
+
     public void testDelayParameter() throws Exception {
         int delayMsec = 1000;
         Bundle args = new Bundle();
@@ -170,6 +207,7 @@ public class InstrumentationTestRunnerTest extends TestCase {
         private TestSuite mTestSuite;
         private TestSuite mDefaultTestSuite;
         private String mPackageNameForDefaultTests;
+        private Bundle mResults;
 
         public StubInstrumentationTestRunner(Context context, Context targetContext,
                 AndroidTestRunner androidTestRunner) {
@@ -200,6 +238,7 @@ public class InstrumentationTestRunnerTest extends TestCase {
 
         public void finish(int resultCode, Bundle results) {
             mFinished = true;
+            mResults = results;
         }
 
         public boolean isStarted() {
@@ -221,6 +260,11 @@ public class InstrumentationTestRunnerTest extends TestCase {
         public String getPackageNameForDefaultTests() {
             return mPackageNameForDefaultTests;
         }
+
+        @Override
+        void prepareLooper() {
+            // ignore
+        }
     }
 
     private static class StubContext extends MockContext {