diff --git a/tests/DumpRenderTree2/src/com/android/dumprendertree2/AbstractResult.java b/tests/DumpRenderTree2/src/com/android/dumprendertree2/AbstractResult.java index 7cbb3970c8da2..6048338af4966 100644 --- a/tests/DumpRenderTree2/src/com/android/dumprendertree2/AbstractResult.java +++ b/tests/DumpRenderTree2/src/com/android/dumprendertree2/AbstractResult.java @@ -46,12 +46,14 @@ public abstract class AbstractResult implements Comparable { public abstract AbstractResult createResult(Bundle bundle); } + /** + * A code representing the result of comparing actual and expected results. + */ public enum ResultCode { - PASS("Passed"), - FAIL_RESULT_DIFFERS("Result differs"), - FAIL_NO_EXPECTED_RESULT("No expected result"), - FAIL_TIMED_OUT("Timed out"), - FAIL_CRASHED("Crashed"); + RESULTS_MATCH("Results match"), + RESULTS_DIFFER("Results differ"), + NO_EXPECTED_RESULT("No expected result"), + NO_ACTUAL_RESULT("No actual result"); private String mTitle; @@ -123,13 +125,47 @@ public abstract class AbstractResult implements Comparable { public abstract String getActualTextResult(); /** - * Returns the code of this result. + * Returns the status code representing the result of comparing actual and expected results. * * @return - * the code of this result + * the status code from comparing actual and expected results */ public abstract ResultCode getResultCode(); + /** + * Returns whether this test crashed. + * + * @return + * whether this test crashed + */ + public abstract boolean didCrash(); + + /** + * Returns whether this test timed out. + * + * @return + * whether this test timed out + */ + public abstract boolean didTimeOut(); + + /** + * Sets that this test timed out. + */ + public abstract void setDidTimeOut(); + + /** + * Returns whether the test passed. + * + * @return + * whether the test passed + */ + public boolean didPass() { + // Tests that crash can't have timed out or have an actual result. + assert !(didCrash() && didTimeOut()); + assert !(didCrash() && getResultCode() != ResultCode.NO_ACTUAL_RESULT); + return !didCrash() && !didTimeOut() && getResultCode() == ResultCode.RESULTS_MATCH; + } + /** * Return the type of the result data. * @@ -150,4 +186,4 @@ public abstract class AbstractResult implements Comparable { public abstract String getDiffAsHtml(); public abstract Bundle getBundle(); -} \ No newline at end of file +} diff --git a/tests/DumpRenderTree2/src/com/android/dumprendertree2/CrashedDummyResult.java b/tests/DumpRenderTree2/src/com/android/dumprendertree2/CrashedDummyResult.java index 31da7767ffa71..483116895f8e1 100644 --- a/tests/DumpRenderTree2/src/com/android/dumprendertree2/CrashedDummyResult.java +++ b/tests/DumpRenderTree2/src/com/android/dumprendertree2/CrashedDummyResult.java @@ -61,7 +61,23 @@ public class CrashedDummyResult extends AbstractResult { @Override public ResultCode getResultCode() { - return ResultCode.FAIL_CRASHED; + return ResultCode.NO_ACTUAL_RESULT; + } + + @Override + public boolean didCrash() { + return true; + } + + @Override + public boolean didTimeOut() { + return false; + } + + @Override + public void setDidTimeOut() { + /** This method is not applicable for this type of result */ + assert false; } @Override @@ -106,4 +122,4 @@ public class CrashedDummyResult extends AbstractResult { public void setExpectedTextResultPath(String relativePath) { /** TODO */ } -} \ No newline at end of file +} diff --git a/tests/DumpRenderTree2/src/com/android/dumprendertree2/LayoutTestsExecutor.java b/tests/DumpRenderTree2/src/com/android/dumprendertree2/LayoutTestsExecutor.java index efea1925b47b9..b9fc2740d8764 100644 --- a/tests/DumpRenderTree2/src/com/android/dumprendertree2/LayoutTestsExecutor.java +++ b/tests/DumpRenderTree2/src/com/android/dumprendertree2/LayoutTestsExecutor.java @@ -426,6 +426,7 @@ public class LayoutTestsExecutor extends Activity { assert mCurrentState.isRunningState() : "mCurrentState = " + mCurrentState.name(); Log.i(LOG_TAG, "onTestFinished(): " + mCurrentTestRelativePath); + mResultHandler.removeMessages(MSG_TEST_TIMED_OUT); obtainActualResultsFromWebView(); } @@ -441,6 +442,9 @@ public class LayoutTestsExecutor extends Activity { mCurrentState = CurrentState.OBTAINING_RESULT; + if (mCurrentTestTimedOut) { + mCurrentResult.setDidTimeOut(); + } mCurrentResult.obtainActualResults(mCurrentWebView, mResultHandler.obtainMessage(MSG_ACTUAL_RESULT_OBTAINED)); } @@ -452,7 +456,6 @@ public class LayoutTestsExecutor extends Activity { Log.i(LOG_TAG, "onActualResultsObtained(): " + mCurrentTestRelativePath); mCurrentState = CurrentState.IDLE; - mResultHandler.removeMessages(MSG_TEST_TIMED_OUT); reportResultToService(); mCurrentTestIndex++; updateProgressBar(); @@ -470,9 +473,6 @@ public class LayoutTestsExecutor extends Activity { Bundle bundle = mCurrentResult.getBundle(); bundle.putInt("testIndex", mCurrentTestIndex); - if (mCurrentTestTimedOut) { - bundle.putString("resultCode", AbstractResult.ResultCode.FAIL_TIMED_OUT.name()); - } if (!mTestsList.isEmpty()) { bundle.putString("nextTest", mTestsList.get(0)); } diff --git a/tests/DumpRenderTree2/src/com/android/dumprendertree2/Summarizer.java b/tests/DumpRenderTree2/src/com/android/dumprendertree2/Summarizer.java index e31e74799c466..b1dea6e8a536a 100644 --- a/tests/DumpRenderTree2/src/com/android/dumprendertree2/Summarizer.java +++ b/tests/DumpRenderTree2/src/com/android/dumprendertree2/Summarizer.java @@ -75,9 +75,12 @@ public class Summarizer { " width: 20px;}" + "h3 span.sqr {" + " text-decoration: none;" + - " color: #8ee100;" + " float: left;" + " width: 20px;}" + + "h3 span.sqr_pass {" + + " color: #8ee100;}" + + "h3 span.sqr_fail {" + + " color: #c30000;}" + "span.source {" + " display: block;" + " font-size: 10px;" + @@ -145,16 +148,16 @@ public class Summarizer { " text-transform: uppercase;" + " padding: 3px;" + " -webkit-border-radius: 4px;}" + - "span." + AbstractResult.ResultCode.FAIL_RESULT_DIFFERS.name() + "{" + + "span." + AbstractResult.ResultCode.RESULTS_DIFFER.name() + "{" + " background-color: #ccc;" + " color: black;}" + - "span." + AbstractResult.ResultCode.FAIL_NO_EXPECTED_RESULT.name() + "{" + + "span." + AbstractResult.ResultCode.NO_EXPECTED_RESULT.name() + "{" + " background-color: #a700e4;" + " color: #fff;}" + - "span." + AbstractResult.ResultCode.FAIL_TIMED_OUT.name() + "{" + + "span.timed_out {" + " background-color: #f3cb00;" + " color: black;}" + - "span." + AbstractResult.ResultCode.FAIL_CRASHED.name() + "{" + + "span.crashed {" + " background-color: #c30000;" + " color: #fff;}" + "span.noLtc {" + @@ -208,11 +211,11 @@ public class Summarizer { public void appendTest(AbstractResult result) { String relativePath = result.getRelativePath(); - if (result.getResultCode() == AbstractResult.ResultCode.FAIL_CRASHED) { + if (result.didCrash()) { mCrashedTestsCount++; } - if (result.getResultCode() == AbstractResult.ResultCode.PASS) { + if (result.didPass()) { if (mFileFilter.isFail(relativePath)) { mUnexpectedPasses.add(result); } else { @@ -276,13 +279,10 @@ public class Summarizer { createTopSummaryTable(webKitRevision, html); - createResultsListWithDiff(html, "Unexpected failures", mUnexpectedFailures); - - createResultsListNoDiff(html, "Unexpected passes", mUnexpectedPasses); - - createResultsListWithDiff(html, "Expected failures", mExpectedFailures); - - createResultsListNoDiff(html, "Expected passes", mExpectedPasses); + createResultsList(html, "Unexpected failures", mUnexpectedFailures); + createResultsList(html, "Unexpected passes", mUnexpectedPasses); + createResultsList(html, "Expected failures", mExpectedFailures); + createResultsList(html, "Expected passes", mExpectedPasses); html.append(""); @@ -357,8 +357,8 @@ public class Summarizer { html.append(""); } - private void createResultsListWithDiff(StringBuilder html, String title, - List resultsList) { + private void createResultsList( + StringBuilder html, String title, List resultsList) { String relativePath; String id = ""; AbstractResult.ResultCode resultCode; @@ -368,7 +368,6 @@ public class Summarizer { for (AbstractResult result : resultsList) { relativePath = result.getRelativePath(); resultCode = result.getResultCode(); - assert resultCode != AbstractResult.ResultCode.PASS : "resultCode=" + resultCode; html.append("

"); @@ -378,61 +377,73 @@ public class Summarizer { * to cause any problems in this case */ id = relativePath.replace(File.separator, ":"); - html.append(""); - html.append(""); - html.append("" + relativePath + ""); - html.append(""); - html.append(" "); - html.append(resultCode.toString()); - html.append(""); + /** Write the test name */ + if (resultCode == AbstractResult.ResultCode.RESULTS_DIFFER) { + html.append(""); + html.append(""); + html.append("" + relativePath + ""); + html.append(""); + } else { + html.append(""); + html.append(""); + html.append("" + result.getRelativePath() + ""); + html.append(""); + } - /** Detect missing LTC function */ - String additionalTextOutputString = result.getAdditionalTextOutputString(); - if (additionalTextOutputString != null && - additionalTextOutputString.contains("com.android.dumprendertree") && - additionalTextOutputString.contains("has no method")) { - if (additionalTextOutputString.contains("LayoutTestController")) { - html.append(" LTC function missing"); - } - if (additionalTextOutputString.contains("EventSender")) { - html.append(" "); - html.append("ES function missing"); - } + if (!result.didPass()) { + appendTags(html, result); } html.append("

"); appendExpectedResultsSources(result, html); - html.append("
"); - html.append(result.getDiffAsHtml()); - html.append("Hide"); - html.append(" | "); - html.append("Show source"); - html.append("
"); + if (resultCode == AbstractResult.ResultCode.RESULTS_DIFFER) { + html.append("
"); + html.append(result.getDiffAsHtml()); + html.append("Hide"); + html.append(" | "); + html.append("Show source"); + html.append("
"); + } html.append("
"); } } - private void createResultsListNoDiff(StringBuilder html, String title, - List resultsList) { - Collections.sort(resultsList); - html.append("

" + title + " [" + resultsList.size() + "]

"); - for (AbstractResult result : resultsList) { - html.append("

"); - html.append(""); - html.append(""); - html.append("" + result.getRelativePath() + ""); - html.append(""); - html.append("

"); - appendExpectedResultsSources(result, html); - html.append("
"); + private void appendTags(StringBuilder html, AbstractResult result) { + /** Tag tests which crash, time out or where results don't match */ + if (result.didCrash()) { + html.append(" Crashed"); + } else { + if (result.didTimeOut()) { + html.append(" Timed out"); + } + AbstractResult.ResultCode resultCode = result.getResultCode(); + if (resultCode != AbstractResult.ResultCode.RESULTS_MATCH) { + html.append(" "); + html.append(resultCode.toString()); + html.append(""); + } + } + + /** Detect missing LTC function */ + String additionalTextOutputString = result.getAdditionalTextOutputString(); + if (additionalTextOutputString != null && + additionalTextOutputString.contains("com.android.dumprendertree") && + additionalTextOutputString.contains("has no method")) { + if (additionalTextOutputString.contains("LayoutTestController")) { + html.append(" LTC function missing"); + } + if (additionalTextOutputString.contains("EventSender")) { + html.append(" "); + html.append("ES function missing"); + } } } diff --git a/tests/DumpRenderTree2/src/com/android/dumprendertree2/TextResult.java b/tests/DumpRenderTree2/src/com/android/dumprendertree2/TextResult.java index 9664efe301150..21e54300d4444 100644 --- a/tests/DumpRenderTree2/src/com/android/dumprendertree2/TextResult.java +++ b/tests/DumpRenderTree2/src/com/android/dumprendertree2/TextResult.java @@ -39,6 +39,7 @@ public class TextResult extends AbstractResult { private String mExpectedResultPath; private String mActualResult; private String mRelativePath; + private boolean mDidTimeOut; private ResultCode mResultCode; private Message mResultObtainedMsg; @@ -74,10 +75,7 @@ public class TextResult extends AbstractResult { mActualResult = bundle.getString("actualTextualResult"); setAdditionalTextOutputString(bundle.getString("additionalTextOutputString")); mRelativePath = bundle.getString("relativePath"); - String resultCode = bundle.getString("resultCode"); - if (resultCode != null) { - mResultCode = ResultCode.valueOf(resultCode); - } + mDidTimeOut = bundle.getBoolean("didTimeOut"); } @Override @@ -87,16 +85,30 @@ public class TextResult extends AbstractResult { } if (mExpectedResult == null) { - mResultCode = AbstractResult.ResultCode.FAIL_NO_EXPECTED_RESULT; + mResultCode = AbstractResult.ResultCode.NO_EXPECTED_RESULT; } else if (!mExpectedResult.equals(mActualResult)) { - mResultCode = AbstractResult.ResultCode.FAIL_RESULT_DIFFERS; + mResultCode = AbstractResult.ResultCode.RESULTS_DIFFER; } else { - mResultCode = AbstractResult.ResultCode.PASS; + mResultCode = AbstractResult.ResultCode.RESULTS_MATCH; } - return mResultCode; } + @Override + public boolean didCrash() { + return false; + } + + @Override + public boolean didTimeOut() { + return mDidTimeOut; + } + + @Override + public void setDidTimeOut() { + mDidTimeOut = true; + } + @Override public byte[] getActualImageResult() { return null; @@ -239,9 +251,7 @@ public class TextResult extends AbstractResult { bundle.putString("actualTextualResult", getActualTextResult()); bundle.putString("additionalTextOutputString", getAdditionalTextOutputString()); bundle.putString("relativePath", mRelativePath); - if (mResultCode != null) { - bundle.putString("resultCode", mResultCode.name()); - } + bundle.putBoolean("didTimeOut", mDidTimeOut); bundle.putString("type", getType().name()); return bundle; } @@ -250,4 +260,4 @@ public class TextResult extends AbstractResult { public String getRelativePath() { return mRelativePath; } -} \ No newline at end of file +}