Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Adds to DumpRenderTree the ability to ignore the results of tests

Browse Source 
Change-Id: I7c16d9713fc35c773b810f9d5ce6700f8d9a28e4
This commit is contained in:
Steve Block
2010-03-09 13:54:09 +00:00
parent 73f7537b30
commit cf0fd7892b
5 changed files with 98 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
tests/DumpRenderTree/assets/run_layout_tests.py
View File

@@ -246,6 +246,7 @@ def main(options, args):
result_files = ["/sdcard/layout_tests_passed.txt",
"/sdcard/layout_tests_failed.txt",
"/sdcard/layout_tests_ignored.txt",
"/sdcard/layout_tests_nontext.txt"]
for file in result_files:
shell_cmd_str = adb_cmd + " pull " + file + " " + results_dir
@@ -263,10 +264,13 @@ def main(options, args):
logging.info(str(passed_tests) + " passed")
failed_tests = CountLineNumber(results_dir + "/layout_tests_failed.txt")
logging.info(str(failed_tests) + " failed")
ignored_tests = CountLineNumber(results_dir + "/layout_tests_ignored.txt")
logging.info(str(ignored_tests) + " ignored results")
crashed_tests = CountLineNumber(results_dir + "/layout_tests_crashed.txt")
logging.info(str(crashed_tests) + " crashed")
nontext_tests = CountLineNumber(results_dir + "/layout_tests_nontext.txt")
logging.info(str(nontext_tests) + " no dumpAsText")
logging.info(str(passed_tests + failed_tests + ignored_tests + crashed_tests + nontext_tests) + " TOTAL")
logging.info("Results are stored under: " + results_dir + "\n")

48
tests/DumpRenderTree/src/com/android/dumprendertree/FileFilter.java
View File

@@ -23,27 +23,33 @@ public class FileFilter {
private static final String LOGTAG = "FileFilter";
// Returns whether we should ignore this test and skip running it.
// Currently we use this only for tests that crash the browser.
// TODO: Once these crashes are fixed, we should probably eliminate this
// method, as no test should crash.
public static boolean ignoreTest(String file) {
// treat files like directories for the time being.
for (int i = 0; i < ignoreTestList.length; i ++) {
for (int i = 0; i < ignoreTestList.length; i++) {
if (file.endsWith(ignoreTestList[i])) {
Log.v(LOGTAG, "File path in list of ignored tests: " + file);
return true;
}
}
for (int i = 0; i < ignoreTestDirs.length; i++) {
if (file.endsWith(ignoreTestDirs[i])) {
Log.v(LOGTAG, "File path in list of ignored directories: " + file);
return false;
}
// Returns whether a directory does not contain layout tests and so can be
// ignored.
public static boolean isNonTestDir(String file) {
for (int i = 0; i < nonTestDirs.length; i++) {
if (file.endsWith(nonTestDirs[i])) {
return true;
}
}
// We should run tests for which the expected result is wrong, as there is
// value in checking that they don't cause crashes.
// TODO: Run these tests but ignore the result.
return ignoreResults(file);
return false;
}
public static boolean ignoreResults(String file) {
// Returns whether we should ignore the result of this test.
public static boolean ignoreResult(String file) {
for (int i = 0; i < ignoreResultList.size(); i++) {
if (file.endsWith(ignoreResultList.get(i))) {
Log.v(LOGTAG, "File path in list of ignored results: " + file);
@@ -51,7 +57,7 @@ public class FileFilter {
}
}
return false;
}
}
final static Vector<String> ignoreResultList = new Vector<String>();
@@ -59,21 +65,16 @@ public class FileFilter {
fillIgnoreResultList();
}
static final String[] ignoreTestDirs = {
static final String[] nonTestDirs = {
".", // ignore hidden directories and files
"resources", // ignore resource directories
".svn", // don't run anything under .svn folder
"profiler", // profiler is not supported
"svg", // svg is not supported
"platform", // platform specific
"platform" // No-Android specific tests
};
static final String [] ignoreTestList = {
"editing/selection/move-left-right.html",
static final String[] ignoreTestList = {
"fast/js/regexp-charclass-crash.html", // RegExp is too large, causing OOM
"storage/domstorage/localstorage/private-browsing-affects-storage.html", // No notion of private browsing.
"storage/domstorage/sessionstorage/private-browsing-affects-storage.html", // No notion of private browsing.
"storage/private-browsing-readonly.html", // No notion of private browsing.
"editing/selection/move-left-right.html" // Causes DumpRenderTree to hang
};
static void fillIgnoreResultList() {
@@ -113,6 +114,7 @@ public class FileFilter {
ignoreResultList.add("fast/workers/shared-worker-shared.html"); // shared workers not supported
ignoreResultList.add("fast/workers/shared-worker-simple.html"); // shared workers not supported
// TODO: These need to be triaged
ignoreResultList.add("fast/css/case-transform.html"); // will not fix #619707
ignoreResultList.add("fast/dom/Element/offsetLeft-offsetTop-body-quirk.html"); // different screen size result in extra spaces in Apple compared to us
ignoreResultList.add("fast/dom/Window/Plug-ins.html"); // need test plugin
@@ -183,6 +185,12 @@ public class FileFilter {
ignoreResultList.add("fast/parser/script-tag-with-trailing-slash.html"); // not capturing the console messages
ignoreResultList.add("fast/replaced/image-map.html"); // requires eventSender.mouseDown(),mouseUp()
ignoreResultList.add("fast/text/plain-text-line-breaks.html"); // extra spacing because iFrames rendered next to each other on Apple
ignoreResultList.add("profiler"); // profiler is not supported
ignoreResultList.add("storage/domstorage/localstorage/private-browsing-affects-storage.html"); // No notion of private browsing.
ignoreResultList.add("storage/domstorage/sessionstorage/private-browsing-affects-storage.html"); // No notion of private browsing.
ignoreResultList.add("storage/private-browsing-readonly.html"); // No notion of private browsing.
ignoreResultList.add("svg"); // svg is not supported
}
}

39
tests/DumpRenderTree/src/com/android/dumprendertree/FsUtils.java
View File

@@ -43,38 +43,45 @@ public class FsUtils {
}
public static void findLayoutTestsRecursively(BufferedOutputStream bos,
String dir) throws IOException {
String dir, boolean ignoreResultsInDir) throws IOException {
Log.v(LOGTAG, "Searching tests under " + dir);
File d = new File(dir);
if (!d.isDirectory()) {
throw new AssertionError("A directory expected, but got " + dir);
}
ignoreResultsInDir |= FileFilter.ignoreResult(dir);
String[] files = d.list();
for (int i = 0; i < files.length; i++) {
String s = dir + "/" + files[i];
if (s.endsWith("TEMPLATE.html")) {
continue;
}
if (FileFilter.ignoreTest(s)) {
Log.v(LOGTAG, " Ignoring: " + s);
continue;
}
if (s.toLowerCase().endsWith(".html")
|| s.toLowerCase().endsWith(".xml")) {
bos.write(s.getBytes());
bos.write('\n');
continue;
}
File f = new File(s);
if (f.isDirectory()) {
findLayoutTestsRecursively(bos, s);
// If this is not a test directory, we don't recurse into it.
if (!FileFilter.isNonTestDir(s)) {
Log.v(LOGTAG, "Recursing on " + s);
findLayoutTestsRecursively(bos, s, ignoreResultsInDir);
}
continue;
}
Log.v(LOGTAG, "Skipping " + s);
// If this test should be ignored, we skip it completely.
if (FileFilter.ignoreTest(s)) {
Log.v(LOGTAG, "Ignoring: " + s);
continue;
}
if ((s.toLowerCase().endsWith(".html") || s.toLowerCase().endsWith(".xml"))
&& !s.endsWith("TEMPLATE.html")) {
Log.v(LOGTAG, "Recording " + s);
bos.write(s.getBytes());
// If the result of this test should be ignored, we still run the test.
if (ignoreResultsInDir || FileFilter.ignoreResult(s)) {
bos.write((" IGNORE_RESULT").getBytes());
}
bos.write('\n');
}
}
}

50
tests/DumpRenderTree/src/com/android/dumprendertree/LayoutTestsAutoTest.java
View File

@@ -38,14 +38,16 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.util.Vector;
// TestRecorder creates three files ...
// TestRecorder creates four files ...
// - passing tests
// - failing tests
// - tests for which results are ignored
// - tests with no text results available
// TestRecorder does not have the ability to clear the results.
class MyTestRecorder {
private BufferedOutputStream mBufferedOutputPassedStream;
private BufferedOutputStream mBufferedOutputFailedStream;
private BufferedOutputStream mBufferedOutputIgnoreResultStream;
private BufferedOutputStream mBufferedOutputNoResultStream;
public void passed(String layout_file) {
@@ -68,6 +70,16 @@ class MyTestRecorder {
}
}
public void ignoreResult(String layout_file) {
try {
mBufferedOutputIgnoreResultStream.write(layout_file.getBytes());
mBufferedOutputIgnoreResultStream.write('\n');
mBufferedOutputIgnoreResultStream.flush();
} catch(Exception e) {
e.printStackTrace();
}
}
public void noResult(String layout_file) {
try {
mBufferedOutputNoResultStream.write(layout_file.getBytes());
@@ -82,12 +94,15 @@ class MyTestRecorder {
try {
File resultsPassedFile = new File("/sdcard/layout_tests_passed.txt");
File resultsFailedFile = new File("/sdcard/layout_tests_failed.txt");
File resultsIgnoreResultFile = new File("/sdcard/layout_tests_ignored.txt");
File noExpectedResultFile = new File("/sdcard/layout_tests_nontext.txt");
mBufferedOutputPassedStream =
new BufferedOutputStream(new FileOutputStream(resultsPassedFile, resume));
mBufferedOutputFailedStream =
new BufferedOutputStream(new FileOutputStream(resultsFailedFile, resume));
mBufferedOutputIgnoreResultStream =
new BufferedOutputStream(new FileOutputStream(resultsIgnoreResultFile, resume));
mBufferedOutputNoResultStream =
new BufferedOutputStream(new FileOutputStream(noExpectedResultFile, resume));
} catch (Exception e) {
@@ -99,6 +114,7 @@ class MyTestRecorder {
try {
mBufferedOutputPassedStream.close();
mBufferedOutputFailedStream.close();
mBufferedOutputIgnoreResultStream.close();
mBufferedOutputNoResultStream.close();
} catch (Exception e) {
e.printStackTrace();
@@ -132,6 +148,8 @@ public class LayoutTestsAutoTest extends ActivityInstrumentationTestCase2<TestSh
private MyTestRecorder mResultRecorder;
private Vector<String> mTestList;
// Whether we should ignore the result for the corresponding test. Ordered same as mTestList.
private Vector<Boolean> mTestListIgnoreResult;
private boolean mRebaselineResults;
// The JavaScript engine currently in use. This determines which set of Android-specific
// expected test results we use.
@@ -158,8 +176,11 @@ public class LayoutTestsAutoTest extends ActivityInstrumentationTestCase2<TestSh
BufferedReader inReader = new BufferedReader(new FileReader(LAYOUT_TESTS_LIST_FILE));
String line = inReader.readLine();
while (line != null) {
if (line.startsWith(mTestPathPrefix))
mTestList.add(line);
if (line.startsWith(mTestPathPrefix)) {
String[] components = line.split(" ");
mTestList.add(components[0]);
mTestListIgnoreResult.add(components.length > 1 && components[1].equals("IGNORE_RESULT"));
}
line = inReader.readLine();
}
inReader.close();
@@ -176,6 +197,7 @@ public class LayoutTestsAutoTest extends ActivityInstrumentationTestCase2<TestSh
for (int i = 0; i < mTestList.size(); i++) {
if (mTestList.elementAt(i).equals(line)) {
mTestList = new Vector<String>(mTestList.subList(i+1, mTestList.size()));
mTestListIgnoreResult = new Vector<Boolean>(mTestListIgnoreResult.subList(i+1, mTestListIgnoreResult.size()));
break;
}
}
@@ -236,14 +258,24 @@ public class LayoutTestsAutoTest extends ActivityInstrumentationTestCase2<TestSh
mResultRecorder.passed(file);
}
private void ignoreResultCase(String file) {
Log.v("Layout test:", file + " ignore result");
mResultRecorder.ignoreResult(file);
}
private void noResultCase(String file) {
Log.v("Layout test:", file + " no expected result");
mResultRecorder.noResult(file);
}
private void processResult(String testFile, String actualResultFile, String expectedResultFile) {
private void processResult(String testFile, String actualResultFile, String expectedResultFile, boolean ignoreResult) {
Log.v(LOGTAG, " Processing result: " + testFile);
if (ignoreResult) {
ignoreResultCase(testFile);
return;
}
File actual = new File(actualResultFile);
File expected = new File(expectedResultFile);
if (actual.exists() && expected.exists()) {
@@ -266,7 +298,7 @@ public class LayoutTestsAutoTest extends ActivityInstrumentationTestCase2<TestSh
}
}
private void runTestAndWaitUntilDone(TestShellActivity activity, String test, int timeout) {
private void runTestAndWaitUntilDone(TestShellActivity activity, String test, int timeout, boolean ignoreResult) {
activity.setCallback(new TestShellCallback() {
public void finished() {
synchronized (LayoutTestsAutoTest.this) {
@@ -320,7 +352,7 @@ public class LayoutTestsAutoTest extends ActivityInstrumentationTestCase2<TestSh
expectedResultFile = getAndroidExpectedResultFile(expectedResultFile);
}
processResult(test, resultFile, expectedResultFile);
processResult(test, resultFile, expectedResultFile, ignoreResult);
}
}
@@ -336,6 +368,7 @@ public class LayoutTestsAutoTest extends ActivityInstrumentationTestCase2<TestSh
}
this.mTestList = new Vector<String>();
this.mTestListIgnoreResult = new Vector<Boolean>();
// Read settings
mTestPathPrefix = (new File(LAYOUT_TESTS_ROOT + runner.mTestPath)).getAbsolutePath();
@@ -372,9 +405,10 @@ public class LayoutTestsAutoTest extends ActivityInstrumentationTestCase2<TestSh
}
for (int i = 0; i < mTestList.size(); i++) {
String s = mTestList.elementAt(i);
boolean ignoreResult = mTestListIgnoreResult.elementAt(i);
FsUtils.updateTestStatus(TEST_STATUS_FILE, s);
// Run tests
runTestAndWaitUntilDone(activity, s, runner.mTimeoutInMillis);
runTestAndWaitUntilDone(activity, s, runner.mTimeoutInMillis, ignoreResult);
}
FsUtils.updateTestStatus(TEST_STATUS_FILE, "#DONE");
@@ -399,7 +433,7 @@ public class LayoutTestsAutoTest extends ActivityInstrumentationTestCase2<TestSh
try {
File tests_list = new File(LAYOUT_TESTS_LIST_FILE);
BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(tests_list, false));
FsUtils.findLayoutTestsRecursively(bos, getTestPath());
FsUtils.findLayoutTestsRecursively(bos, getTestPath(), false); // Don't ignore results
bos.flush();
bos.close();
} catch (Exception e) {

3
tests/DumpRenderTree/src/com/android/dumprendertree/Menu.java
View File

@@ -68,7 +68,7 @@ public class Menu extends FileList {
try {
File tests_list = new File(LAYOUT_TESTS_LIST_FILE);
BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(tests_list, false));
FsUtils.findLayoutTestsRecursively(bos, path);
FsUtils.findLayoutTestsRecursively(bos, path, false); // Don't ignore results
bos.flush();
bos.close();
} catch (Exception e) {
@@ -77,4 +77,3 @@ public class Menu extends FileList {
}
}