Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update DumpRenderTree2 to handle failing tests where no meaningful diff is available

Browse Source 
Currently, DumpRenderTree2 asigns the following result codes
{PASS, FAIL_RESULT_DIFFERS, FAIL_NO_EXPECTED_RESULT, FAIL_TIMED_OUT,
FAIL_CRASHED}

This is not strictly correct, as a test may fail for any of three
reasons - crashing, timing out, or the the result not being as
expected. Therefore, it's possible for a test to fail even if the
result is as expected.

This patch updates AbstractResult to handle each of the three reasons
for failure separately. We then test all three to determine if the
test has passed.

This allows us to correctly report whether or not the result differs
from expected for a failing test.

Change-Id: I7adcfe72c4dd0bd3de2e1b868d9807be6eb5bddf
This commit is contained in:
Steve Block
2010-09-17 11:39:22 +01:00
parent c0847c55ea
commit 8a6def0247
5 changed files with 159 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files

52
tests/DumpRenderTree2/src/com/android/dumprendertree2/AbstractResult.java
View File

@@ -46,12 +46,14 @@ public abstract class AbstractResult implements Comparable<AbstractResult> {
public abstract AbstractResult createResult(Bundle bundle);
}
/**
* A code representing the result of comparing actual and expected results.
*/
public enum ResultCode {
PASS("Passed"),
FAIL_RESULT_DIFFERS("Result differs"),
FAIL_NO_EXPECTED_RESULT("No expected result"),
FAIL_TIMED_OUT("Timed out"),
FAIL_CRASHED("Crashed");
RESULTS_MATCH("Results match"),
RESULTS_DIFFER("Results differ"),
NO_EXPECTED_RESULT("No expected result"),
NO_ACTUAL_RESULT("No actual result");
private String mTitle;
@@ -123,13 +125,47 @@ public abstract class AbstractResult implements Comparable<AbstractResult> {
public abstract String getActualTextResult();
/**
* Returns the code of this result.
* Returns the status code representing the result of comparing actual and expected results.
*
* @return
* the code of this result
* the status code from comparing actual and expected results
*/
public abstract ResultCode getResultCode();
/**
* Returns whether this test crashed.
*
* @return
* whether this test crashed
*/
public abstract boolean didCrash();
/**
* Returns whether this test timed out.
*
* @return
* whether this test timed out
*/
public abstract boolean didTimeOut();
/**
* Sets that this test timed out.
*/
public abstract void setDidTimeOut();
/**
* Returns whether the test passed.
*
* @return
* whether the test passed
*/
public boolean didPass() {
// Tests that crash can't have timed out or have an actual result.
assert !(didCrash() && didTimeOut());
assert !(didCrash() && getResultCode() != ResultCode.NO_ACTUAL_RESULT);
return !didCrash() && !didTimeOut() && getResultCode() == ResultCode.RESULTS_MATCH;
}
/**
* Return the type of the result data.
*
@@ -150,4 +186,4 @@ public abstract class AbstractResult implements Comparable<AbstractResult> {
public abstract String getDiffAsHtml();
public abstract Bundle getBundle();
}
}

20
tests/DumpRenderTree2/src/com/android/dumprendertree2/CrashedDummyResult.java
View File

@@ -61,7 +61,23 @@ public class CrashedDummyResult extends AbstractResult {
@Override
public ResultCode getResultCode() {
return ResultCode.FAIL_CRASHED;
return ResultCode.NO_ACTUAL_RESULT;
}
@Override
public boolean didCrash() {
return true;
}
@Override
public boolean didTimeOut() {
return false;
}
@Override
public void setDidTimeOut() {
/** This method is not applicable for this type of result */
assert false;
}
@Override
@@ -106,4 +122,4 @@ public class CrashedDummyResult extends AbstractResult {
public void setExpectedTextResultPath(String relativePath) {
/** TODO */
}
}
}

8
tests/DumpRenderTree2/src/com/android/dumprendertree2/LayoutTestsExecutor.java
View File

@@ -426,6 +426,7 @@ public class LayoutTestsExecutor extends Activity {
assert mCurrentState.isRunningState() : "mCurrentState = " + mCurrentState.name();
Log.i(LOG_TAG, "onTestFinished(): " + mCurrentTestRelativePath);
mResultHandler.removeMessages(MSG_TEST_TIMED_OUT);
obtainActualResultsFromWebView();
}
@@ -441,6 +442,9 @@ public class LayoutTestsExecutor extends Activity {
mCurrentState = CurrentState.OBTAINING_RESULT;
if (mCurrentTestTimedOut) {
mCurrentResult.setDidTimeOut();
}
mCurrentResult.obtainActualResults(mCurrentWebView,
mResultHandler.obtainMessage(MSG_ACTUAL_RESULT_OBTAINED));
}
@@ -452,7 +456,6 @@ public class LayoutTestsExecutor extends Activity {
Log.i(LOG_TAG, "onActualResultsObtained(): " + mCurrentTestRelativePath);
mCurrentState = CurrentState.IDLE;
mResultHandler.removeMessages(MSG_TEST_TIMED_OUT);
reportResultToService();
mCurrentTestIndex++;
updateProgressBar();
@@ -470,9 +473,6 @@ public class LayoutTestsExecutor extends Activity {
Bundle bundle = mCurrentResult.getBundle();
bundle.putInt("testIndex", mCurrentTestIndex);
if (mCurrentTestTimedOut) {
bundle.putString("resultCode", AbstractResult.ResultCode.FAIL_TIMED_OUT.name());
}
if (!mTestsList.isEmpty()) {
bundle.putString("nextTest", mTestsList.get(0));
}

131
tests/DumpRenderTree2/src/com/android/dumprendertree2/Summarizer.java
View File

@@ -75,9 +75,12 @@ public class Summarizer {
" width: 20px;}" +
"h3 span.sqr {" +
" text-decoration: none;" +
" color: #8ee100;" +
" float: left;" +
" width: 20px;}" +
"h3 span.sqr_pass {" +
" color: #8ee100;}" +
"h3 span.sqr_fail {" +
" color: #c30000;}" +
"span.source {" +
" display: block;" +
" font-size: 10px;" +
@@ -145,16 +148,16 @@ public class Summarizer {
" text-transform: uppercase;" +
" padding: 3px;" +
" -webkit-border-radius: 4px;}" +
"span." + AbstractResult.ResultCode.FAIL_RESULT_DIFFERS.name() + "{" +
"span." + AbstractResult.ResultCode.RESULTS_DIFFER.name() + "{" +
" background-color: #ccc;" +
" color: black;}" +
"span." + AbstractResult.ResultCode.FAIL_NO_EXPECTED_RESULT.name() + "{" +
"span." + AbstractResult.ResultCode.NO_EXPECTED_RESULT.name() + "{" +
" background-color: #a700e4;" +
" color: #fff;}" +
"span." + AbstractResult.ResultCode.FAIL_TIMED_OUT.name() + "{" +
"span.timed_out {" +
" background-color: #f3cb00;" +
" color: black;}" +
"span." + AbstractResult.ResultCode.FAIL_CRASHED.name() + "{" +
"span.crashed {" +
" background-color: #c30000;" +
" color: #fff;}" +
"span.noLtc {" +
@@ -208,11 +211,11 @@ public class Summarizer {
public void appendTest(AbstractResult result) {
String relativePath = result.getRelativePath();
if (result.getResultCode() == AbstractResult.ResultCode.FAIL_CRASHED) {
if (result.didCrash()) {
mCrashedTestsCount++;
}
if (result.getResultCode() == AbstractResult.ResultCode.PASS) {
if (result.didPass()) {
if (mFileFilter.isFail(relativePath)) {
mUnexpectedPasses.add(result);
} else {
@@ -276,13 +279,10 @@ public class Summarizer {
createTopSummaryTable(webKitRevision, html);
createResultsListWithDiff(html, "Unexpected failures", mUnexpectedFailures);
createResultsListNoDiff(html, "Unexpected passes", mUnexpectedPasses);
createResultsListWithDiff(html, "Expected failures", mExpectedFailures);
createResultsListNoDiff(html, "Expected passes", mExpectedPasses);
createResultsList(html, "Unexpected failures", mUnexpectedFailures);
createResultsList(html, "Unexpected passes", mUnexpectedPasses);
createResultsList(html, "Expected failures", mExpectedFailures);
createResultsList(html, "Expected passes", mExpectedPasses);
html.append("</body></html>");
@@ -357,8 +357,8 @@ public class Summarizer {
html.append("</tr>");
}
private void createResultsListWithDiff(StringBuilder html, String title,
List<AbstractResult> resultsList) {
private void createResultsList(
StringBuilder html, String title, List<AbstractResult> resultsList) {
String relativePath;
String id = "";
AbstractResult.ResultCode resultCode;
@@ -368,7 +368,6 @@ public class Summarizer {
for (AbstractResult result : resultsList) {
relativePath = result.getRelativePath();
resultCode = result.getResultCode();
assert resultCode != AbstractResult.ResultCode.PASS : "resultCode=" + resultCode;
html.append("<h3>");
@@ -378,61 +377,73 @@ public class Summarizer {
* to cause any problems in this case
*/
id = relativePath.replace(File.separator, ":");
html.append("<a href=\"#\" onClick=\"toggleDisplay('" + id + "');");
html.append("return false;\">");
html.append("<span class=\"tri\" id=\"tri." + id + "\">&#x25b6; </span>");
html.append("<span class=\"path\">" + relativePath + "</span>");
html.append("</a>");
html.append(" <span class=\"listItem " + resultCode.name() + "\">");
html.append(resultCode.toString());
html.append("</span>");
/** Write the test name */
if (resultCode == AbstractResult.ResultCode.RESULTS_DIFFER) {
html.append("<a href=\"#\" onClick=\"toggleDisplay('" + id + "');");
html.append("return false;\">");
html.append("<span class=\"tri\" id=\"tri." + id + "\">&#x25b6; </span>");
html.append("<span class=\"path\">" + relativePath + "</span>");
html.append("</a>");
} else {
html.append("<a href=\"" + getViewSourceUrl(result.getRelativePath()).toString() + "\"");
html.append(" target=\"_blank\">");
html.append("<span class=\"sqr sqr_" + (result.didPass() ? "pass" : "fail"));
html.append("\">&#x25a0; </span>");
html.append("<span class=\"path\">" + result.getRelativePath() + "</span>");
html.append("</a>");
}
/** Detect missing LTC function */
String additionalTextOutputString = result.getAdditionalTextOutputString();
if (additionalTextOutputString != null &&
additionalTextOutputString.contains("com.android.dumprendertree") &&
additionalTextOutputString.contains("has no method")) {
if (additionalTextOutputString.contains("LayoutTestController")) {
html.append(" <span class=\"listItem noLtc\">LTC function missing</span>");
}
if (additionalTextOutputString.contains("EventSender")) {
html.append(" <span class=\"listItem noEventSender\">");
html.append("ES function missing</span>");
}
if (!result.didPass()) {
appendTags(html, result);
}
html.append("</h3>");
appendExpectedResultsSources(result, html);
html.append("<div class=\"diff\" style=\"display: none;\" id=\"" + id + "\">");
html.append(result.getDiffAsHtml());
html.append("<a href=\"#\" onClick=\"toggleDisplay('" + id + "');");
html.append("return false;\">Hide</a>");
html.append(" | ");
html.append("<a href=\"" + getViewSourceUrl(relativePath).toString() + "\"");
html.append(" target=\"_blank\">Show source</a>");
html.append("</div>");
if (resultCode == AbstractResult.ResultCode.RESULTS_DIFFER) {
html.append("<div class=\"diff\" style=\"display: none;\" id=\"" + id + "\">");
html.append(result.getDiffAsHtml());
html.append("<a href=\"#\" onClick=\"toggleDisplay('" + id + "');");
html.append("return false;\">Hide</a>");
html.append(" | ");
html.append("<a href=\"" + getViewSourceUrl(relativePath).toString() + "\"");
html.append(" target=\"_blank\">Show source</a>");
html.append("</div>");
}
html.append("<div class=\"space\"></div>");
}
}
private void createResultsListNoDiff(StringBuilder html, String title,
List<AbstractResult> resultsList) {
Collections.sort(resultsList);
html.append("<h2>" + title + " [" + resultsList.size() + "]</h2>");
for (AbstractResult result : resultsList) {
html.append("<h3>");
html.append("<a href=\"" + getViewSourceUrl(result.getRelativePath()).toString() +
"\"");
html.append(" target=\"_blank\">");
html.append("<span class=\"sqr\">&#x25a0; </span>");
html.append("<span class=\"path\">" + result.getRelativePath() + "</span>");
html.append("</a>");
html.append("</h3>");
appendExpectedResultsSources(result, html);
html.append("<div class=\"space\"></div>");
private void appendTags(StringBuilder html, AbstractResult result) {
/** Tag tests which crash, time out or where results don't match */
if (result.didCrash()) {
html.append(" <span class=\"listItem crashed\">Crashed</span>");
} else {
if (result.didTimeOut()) {
html.append(" <span class=\"listItem timed_out\">Timed out</span>");
}
AbstractResult.ResultCode resultCode = result.getResultCode();
if (resultCode != AbstractResult.ResultCode.RESULTS_MATCH) {
html.append(" <span class=\"listItem " + resultCode.name() + "\">");
html.append(resultCode.toString());
html.append("</span>");
}
}
/** Detect missing LTC function */
String additionalTextOutputString = result.getAdditionalTextOutputString();
if (additionalTextOutputString != null &&
additionalTextOutputString.contains("com.android.dumprendertree") &&
additionalTextOutputString.contains("has no method")) {
if (additionalTextOutputString.contains("LayoutTestController")) {
html.append(" <span class=\"listItem noLtc\">LTC function missing</span>");
}
if (additionalTextOutputString.contains("EventSender")) {
html.append(" <span class=\"listItem noEventSender\">");
html.append("ES function missing</span>");
}
}
}

34
tests/DumpRenderTree2/src/com/android/dumprendertree2/TextResult.java
View File

@@ -39,6 +39,7 @@ public class TextResult extends AbstractResult {
private String mExpectedResultPath;
private String mActualResult;
private String mRelativePath;
private boolean mDidTimeOut;
private ResultCode mResultCode;
private Message mResultObtainedMsg;
@@ -74,10 +75,7 @@ public class TextResult extends AbstractResult {
mActualResult = bundle.getString("actualTextualResult");
setAdditionalTextOutputString(bundle.getString("additionalTextOutputString"));
mRelativePath = bundle.getString("relativePath");
String resultCode = bundle.getString("resultCode");
if (resultCode != null) {
mResultCode = ResultCode.valueOf(resultCode);
}
mDidTimeOut = bundle.getBoolean("didTimeOut");
}
@Override
@@ -87,16 +85,30 @@ public class TextResult extends AbstractResult {
}
if (mExpectedResult == null) {
mResultCode = AbstractResult.ResultCode.FAIL_NO_EXPECTED_RESULT;
mResultCode = AbstractResult.ResultCode.NO_EXPECTED_RESULT;
} else if (!mExpectedResult.equals(mActualResult)) {
mResultCode = AbstractResult.ResultCode.FAIL_RESULT_DIFFERS;
mResultCode = AbstractResult.ResultCode.RESULTS_DIFFER;
} else {
mResultCode = AbstractResult.ResultCode.PASS;
mResultCode = AbstractResult.ResultCode.RESULTS_MATCH;
}
return mResultCode;
}
@Override
public boolean didCrash() {
return false;
}
@Override
public boolean didTimeOut() {
return mDidTimeOut;
}
@Override
public void setDidTimeOut() {
mDidTimeOut = true;
}
@Override
public byte[] getActualImageResult() {
return null;
@@ -239,9 +251,7 @@ public class TextResult extends AbstractResult {
bundle.putString("actualTextualResult", getActualTextResult());
bundle.putString("additionalTextOutputString", getAdditionalTextOutputString());
bundle.putString("relativePath", mRelativePath);
if (mResultCode != null) {
bundle.putString("resultCode", mResultCode.name());
}
bundle.putBoolean("didTimeOut", mDidTimeOut);
bundle.putString("type", getType().name());
return bundle;
}
@@ -250,4 +260,4 @@ public class TextResult extends AbstractResult {
public String getRelativePath() {
return mRelativePath;
}
}
}