0006952a8d
Currently test code use getActivity to get DRT's assets. However the side effect is that this will actually launch the activity. This is not affecting any tests yet, however there are cleaner ways to do this. The instrumentation framework provides a getTargetContext call which will give access to app under test's context, and therefore granting access to its assets. Change-Id: I0560b0fa5681f80bcb9296beec3fd0549c40cc8e
500 lines
19 KiB
Java
500 lines
19 KiB
Java
/*
|
|
* Copyright (C) 2008 The Android Open Source Project
|
|
*
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*/
|
|
|
|
package com.android.dumprendertree;
|
|
|
|
import com.android.dumprendertree.TestShellActivity.DumpDataType;
|
|
import com.android.dumprendertree.forwarder.AdbUtils;
|
|
import com.android.dumprendertree.forwarder.ForwardService;
|
|
|
|
import android.content.Context;
|
|
import android.content.Intent;
|
|
import android.os.Environment;
|
|
import android.test.ActivityInstrumentationTestCase2;
|
|
import android.util.Log;
|
|
|
|
import java.io.BufferedOutputStream;
|
|
import java.io.BufferedReader;
|
|
import java.io.File;
|
|
import java.io.FileNotFoundException;
|
|
import java.io.FileOutputStream;
|
|
import java.io.FileReader;
|
|
import java.io.IOException;
|
|
import java.io.InputStream;
|
|
import java.io.OutputStream;
|
|
import java.util.Vector;
|
|
|
|
// TestRecorder creates four files ...
|
|
// - passing tests
|
|
// - failing tests
|
|
// - tests for which results are ignored
|
|
// - tests with no text results available
|
|
// TestRecorder does not have the ability to clear the results.
|
|
class MyTestRecorder {
|
|
private BufferedOutputStream mBufferedOutputPassedStream;
|
|
private BufferedOutputStream mBufferedOutputFailedStream;
|
|
private BufferedOutputStream mBufferedOutputIgnoreResultStream;
|
|
private BufferedOutputStream mBufferedOutputNoResultStream;
|
|
|
|
public void passed(String layout_file) {
|
|
try {
|
|
mBufferedOutputPassedStream.write(layout_file.getBytes());
|
|
mBufferedOutputPassedStream.write('\n');
|
|
mBufferedOutputPassedStream.flush();
|
|
} catch(Exception e) {
|
|
e.printStackTrace();
|
|
}
|
|
}
|
|
|
|
public void failed(String layout_file) {
|
|
try {
|
|
mBufferedOutputFailedStream.write(layout_file.getBytes());
|
|
mBufferedOutputFailedStream.write('\n');
|
|
mBufferedOutputFailedStream.flush();
|
|
} catch(Exception e) {
|
|
e.printStackTrace();
|
|
}
|
|
}
|
|
|
|
public void ignoreResult(String layout_file) {
|
|
try {
|
|
mBufferedOutputIgnoreResultStream.write(layout_file.getBytes());
|
|
mBufferedOutputIgnoreResultStream.write('\n');
|
|
mBufferedOutputIgnoreResultStream.flush();
|
|
} catch(Exception e) {
|
|
e.printStackTrace();
|
|
}
|
|
}
|
|
|
|
public void noResult(String layout_file) {
|
|
try {
|
|
mBufferedOutputNoResultStream.write(layout_file.getBytes());
|
|
mBufferedOutputNoResultStream.write('\n');
|
|
mBufferedOutputNoResultStream.flush();
|
|
} catch(Exception e) {
|
|
e.printStackTrace();
|
|
}
|
|
}
|
|
|
|
public MyTestRecorder(boolean resume) {
|
|
try {
|
|
File externalDir = Environment.getExternalStorageDirectory();
|
|
File resultsPassedFile = new File(externalDir, "layout_tests_passed.txt");
|
|
File resultsFailedFile = new File(externalDir, "layout_tests_failed.txt");
|
|
File resultsIgnoreResultFile = new File(externalDir, "layout_tests_ignored.txt");
|
|
File noExpectedResultFile = new File(externalDir, "layout_tests_nontext.txt");
|
|
|
|
mBufferedOutputPassedStream =
|
|
new BufferedOutputStream(new FileOutputStream(resultsPassedFile, resume));
|
|
mBufferedOutputFailedStream =
|
|
new BufferedOutputStream(new FileOutputStream(resultsFailedFile, resume));
|
|
mBufferedOutputIgnoreResultStream =
|
|
new BufferedOutputStream(new FileOutputStream(resultsIgnoreResultFile, resume));
|
|
mBufferedOutputNoResultStream =
|
|
new BufferedOutputStream(new FileOutputStream(noExpectedResultFile, resume));
|
|
} catch (Exception e) {
|
|
e.printStackTrace();
|
|
}
|
|
}
|
|
|
|
public void close() {
|
|
try {
|
|
mBufferedOutputPassedStream.close();
|
|
mBufferedOutputFailedStream.close();
|
|
mBufferedOutputIgnoreResultStream.close();
|
|
mBufferedOutputNoResultStream.close();
|
|
} catch (Exception e) {
|
|
e.printStackTrace();
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
public class LayoutTestsAutoTest extends ActivityInstrumentationTestCase2<TestShellActivity> {
|
|
|
|
private static final String LOGTAG = "LayoutTests";
|
|
static final int DEFAULT_TIMEOUT_IN_MILLIS = 5000;
|
|
|
|
static final String EXTERNAL_DIR = Environment.getExternalStorageDirectory().toString();
|
|
static final String LAYOUT_TESTS_ROOT = EXTERNAL_DIR + "/android/layout_tests/";
|
|
static final String LAYOUT_TESTS_RESULT_DIR = EXTERNAL_DIR + "/android/layout_tests_results/";
|
|
static final String ANDROID_EXPECTED_RESULT_DIR = EXTERNAL_DIR + "/android/expected_results/";
|
|
static final String LAYOUT_TESTS_LIST_FILE = EXTERNAL_DIR + "/android/layout_tests_list.txt";
|
|
static final String TEST_STATUS_FILE = EXTERNAL_DIR + "/android/running_test.txt";
|
|
static final String LAYOUT_TESTS_RESULTS_REFERENCE_FILES[] = {
|
|
"results/layout_tests_passed.txt",
|
|
"results/layout_tests_failed.txt",
|
|
"results/layout_tests_nontext.txt",
|
|
"results/layout_tests_crashed.txt",
|
|
"run_layout_tests.py"
|
|
};
|
|
|
|
static final String LAYOUT_RESULTS_FAILED_RESULT_FILE = "results/layout_tests_failed.txt";
|
|
static final String LAYOUT_RESULTS_NONTEXT_RESULT_FILE = "results/layout_tests_nontext.txt";
|
|
static final String LAYOUT_RESULTS_CRASHED_RESULT_FILE = "results/layout_tests_crashed.txt";
|
|
static final String LAYOUT_TESTS_RUNNER = "run_layout_tests.py";
|
|
|
|
private MyTestRecorder mResultRecorder;
|
|
private Vector<String> mTestList;
|
|
// Whether we should ignore the result for the corresponding test. Ordered same as mTestList.
|
|
private Vector<Boolean> mTestListIgnoreResult;
|
|
private boolean mRebaselineResults;
|
|
// The JavaScript engine currently in use. This determines which set of Android-specific
|
|
// expected test results we use.
|
|
private String mJsEngine;
|
|
private String mTestPathPrefix;
|
|
private boolean mFinished;
|
|
private int mTestCount;
|
|
private int mResumeIndex;
|
|
|
|
public LayoutTestsAutoTest() {
|
|
super(TestShellActivity.class);
|
|
}
|
|
|
|
private void getTestList() {
|
|
// Read test list.
|
|
try {
|
|
BufferedReader inReader = new BufferedReader(new FileReader(LAYOUT_TESTS_LIST_FILE));
|
|
String line = inReader.readLine();
|
|
while (line != null) {
|
|
if (line.startsWith(mTestPathPrefix)) {
|
|
String[] components = line.split(" ");
|
|
mTestList.add(components[0]);
|
|
mTestListIgnoreResult.add(components.length > 1 && components[1].equals("IGNORE_RESULT"));
|
|
}
|
|
line = inReader.readLine();
|
|
}
|
|
inReader.close();
|
|
Log.v(LOGTAG, "Test list has " + mTestList.size() + " test(s).");
|
|
} catch (Exception e) {
|
|
Log.e(LOGTAG, "Error while reading test list : " + e.getMessage());
|
|
}
|
|
mTestCount = mTestList.size();
|
|
}
|
|
|
|
private void resumeTestList() {
|
|
// read out the test name it stoped last time.
|
|
try {
|
|
String line = FsUtils.readTestStatus(TEST_STATUS_FILE);
|
|
for (int i = 0; i < mTestList.size(); i++) {
|
|
if (mTestList.elementAt(i).equals(line)) {
|
|
mTestList = new Vector<String>(mTestList.subList(i+1, mTestList.size()));
|
|
mTestListIgnoreResult = new Vector<Boolean>(mTestListIgnoreResult.subList(i+1, mTestListIgnoreResult.size()));
|
|
mResumeIndex = i + 1;
|
|
break;
|
|
}
|
|
}
|
|
} catch (Exception e) {
|
|
Log.e(LOGTAG, "Error reading " + TEST_STATUS_FILE);
|
|
}
|
|
}
|
|
|
|
private void clearTestStatus() {
|
|
// Delete TEST_STATUS_FILE
|
|
try {
|
|
File f = new File(TEST_STATUS_FILE);
|
|
if (f.delete())
|
|
Log.v(LOGTAG, "Deleted " + TEST_STATUS_FILE);
|
|
else
|
|
Log.e(LOGTAG, "Fail to delete " + TEST_STATUS_FILE);
|
|
} catch (Exception e) {
|
|
Log.e(LOGTAG, "Fail to delete " + TEST_STATUS_FILE + " : " + e.getMessage());
|
|
}
|
|
}
|
|
|
|
private String getResultFile(String test) {
|
|
String shortName = test.substring(0, test.lastIndexOf('.'));
|
|
// Write actual results to result directory.
|
|
return shortName.replaceFirst(LAYOUT_TESTS_ROOT, LAYOUT_TESTS_RESULT_DIR) + "-result.txt";
|
|
}
|
|
|
|
// Gets the file which contains WebKit's expected results for this test.
|
|
private String getExpectedResultFile(String test) {
|
|
// The generic result is at <path>/<name>-expected.txt
|
|
// First try the Android-specific result at
|
|
// platform/android-<js-engine>/<path>/<name>-expected.txt
|
|
// then
|
|
// platform/android/<path>/<name>-expected.txt
|
|
int pos = test.lastIndexOf('.');
|
|
if (pos == -1)
|
|
return null;
|
|
String genericExpectedResult = test.substring(0, pos) + "-expected.txt";
|
|
String androidExpectedResultsDir = "platform/android-" + mJsEngine + "/";
|
|
String androidExpectedResult = genericExpectedResult.replaceFirst(LAYOUT_TESTS_ROOT,
|
|
LAYOUT_TESTS_ROOT + androidExpectedResultsDir);
|
|
File f = new File(androidExpectedResult);
|
|
if (f.exists())
|
|
return androidExpectedResult;
|
|
androidExpectedResultsDir = "platform/android/";
|
|
androidExpectedResult = genericExpectedResult.replaceFirst(LAYOUT_TESTS_ROOT,
|
|
LAYOUT_TESTS_ROOT + androidExpectedResultsDir);
|
|
f = new File(androidExpectedResult);
|
|
return f.exists() ? androidExpectedResult : genericExpectedResult;
|
|
}
|
|
|
|
// Gets the file which contains the actual results of running the test on
|
|
// Android, generated by a previous run which set a new baseline.
|
|
private String getAndroidExpectedResultFile(String expectedResultFile) {
|
|
return expectedResultFile.replaceFirst(LAYOUT_TESTS_ROOT, ANDROID_EXPECTED_RESULT_DIR);
|
|
}
|
|
|
|
// Wrap up
|
|
private void failedCase(String file) {
|
|
Log.w("Layout test: ", file + " failed");
|
|
mResultRecorder.failed(file);
|
|
}
|
|
|
|
private void passedCase(String file) {
|
|
Log.v("Layout test:", file + " passed");
|
|
mResultRecorder.passed(file);
|
|
}
|
|
|
|
private void ignoreResultCase(String file) {
|
|
Log.v("Layout test:", file + " ignore result");
|
|
mResultRecorder.ignoreResult(file);
|
|
}
|
|
|
|
private void noResultCase(String file) {
|
|
Log.v("Layout test:", file + " no expected result");
|
|
mResultRecorder.noResult(file);
|
|
}
|
|
|
|
private void processResult(String testFile, String actualResultFile, String expectedResultFile, boolean ignoreResult) {
|
|
Log.v(LOGTAG, " Processing result: " + testFile);
|
|
|
|
if (ignoreResult) {
|
|
ignoreResultCase(testFile);
|
|
return;
|
|
}
|
|
|
|
File actual = new File(actualResultFile);
|
|
File expected = new File(expectedResultFile);
|
|
if (actual.exists() && expected.exists()) {
|
|
try {
|
|
if (FsUtils.diffIgnoreSpaces(actualResultFile, expectedResultFile)) {
|
|
passedCase(testFile);
|
|
} else {
|
|
failedCase(testFile);
|
|
}
|
|
} catch (FileNotFoundException ex) {
|
|
Log.e(LOGTAG, "File not found : " + ex.getMessage());
|
|
} catch (IOException ex) {
|
|
Log.e(LOGTAG, "IO Error : " + ex.getMessage());
|
|
}
|
|
return;
|
|
}
|
|
|
|
if (!expected.exists()) {
|
|
noResultCase(testFile);
|
|
}
|
|
}
|
|
|
|
private void runTestAndWaitUntilDone(TestShellActivity activity, String test, int timeout, boolean ignoreResult, int testNumber) {
|
|
activity.setCallback(new TestShellCallback() {
|
|
public void finished() {
|
|
synchronized (LayoutTestsAutoTest.this) {
|
|
mFinished = true;
|
|
LayoutTestsAutoTest.this.notifyAll();
|
|
}
|
|
}
|
|
|
|
public void timedOut(String url) {
|
|
Log.v(LOGTAG, "layout timeout: " + url);
|
|
}
|
|
});
|
|
|
|
String resultFile = getResultFile(test);
|
|
if (resultFile == null) {
|
|
// Simply ignore this test.
|
|
return;
|
|
}
|
|
if (mRebaselineResults) {
|
|
String expectedResultFile = getExpectedResultFile(test);
|
|
File f = new File(expectedResultFile);
|
|
if (f.exists()) {
|
|
return; // don't run test and don't overwrite default tests.
|
|
}
|
|
|
|
resultFile = getAndroidExpectedResultFile(expectedResultFile);
|
|
}
|
|
|
|
mFinished = false;
|
|
Intent intent = new Intent(Intent.ACTION_VIEW);
|
|
intent.setClass(activity, TestShellActivity.class);
|
|
intent.addFlags(Intent.FLAG_ACTIVITY_SINGLE_TOP);
|
|
intent.putExtra(TestShellActivity.TEST_URL, FsUtils.getTestUrl(test));
|
|
intent.putExtra(TestShellActivity.RESULT_FILE, resultFile);
|
|
intent.putExtra(TestShellActivity.TIMEOUT_IN_MILLIS, timeout);
|
|
intent.putExtra(TestShellActivity.TOTAL_TEST_COUNT, mTestCount);
|
|
intent.putExtra(TestShellActivity.CURRENT_TEST_NUMBER, testNumber);
|
|
intent.putExtra(TestShellActivity.STOP_ON_REF_ERROR, true);
|
|
activity.startActivity(intent);
|
|
|
|
// Wait until done.
|
|
synchronized (this) {
|
|
while(!mFinished){
|
|
try {
|
|
this.wait();
|
|
} catch (InterruptedException e) { }
|
|
}
|
|
}
|
|
|
|
if (!mRebaselineResults) {
|
|
String expectedResultFile = getExpectedResultFile(test);
|
|
File f = new File(expectedResultFile);
|
|
if (!f.exists()) {
|
|
expectedResultFile = getAndroidExpectedResultFile(expectedResultFile);
|
|
}
|
|
|
|
processResult(test, resultFile, expectedResultFile, ignoreResult);
|
|
}
|
|
}
|
|
|
|
// Invokes running of layout tests
|
|
// and waits till it has finished running.
|
|
public void executeLayoutTests(boolean resume) {
|
|
LayoutTestsAutoRunner runner = (LayoutTestsAutoRunner) getInstrumentation();
|
|
// A convenient method to be called by another activity.
|
|
|
|
if (runner.mTestPath == null) {
|
|
Log.e(LOGTAG, "No test specified");
|
|
return;
|
|
}
|
|
|
|
this.mTestList = new Vector<String>();
|
|
this.mTestListIgnoreResult = new Vector<Boolean>();
|
|
|
|
// Read settings
|
|
mTestPathPrefix = (new File(LAYOUT_TESTS_ROOT + runner.mTestPath)).getAbsolutePath();
|
|
mRebaselineResults = runner.mRebaseline;
|
|
// V8 is the default JavaScript engine.
|
|
mJsEngine = runner.mJsEngine == null ? "v8" : runner.mJsEngine;
|
|
|
|
int timeout = runner.mTimeoutInMillis;
|
|
if (timeout <= 0) {
|
|
timeout = DEFAULT_TIMEOUT_IN_MILLIS;
|
|
}
|
|
|
|
this.mResultRecorder = new MyTestRecorder(resume);
|
|
|
|
if (!resume)
|
|
clearTestStatus();
|
|
|
|
getTestList();
|
|
if (resume)
|
|
resumeTestList();
|
|
|
|
TestShellActivity activity = getActivity();
|
|
activity.setDefaultDumpDataType(DumpDataType.EXT_REPR);
|
|
|
|
// Run tests.
|
|
int addr = -1;
|
|
try{
|
|
addr = AdbUtils.resolve("android-browser-test.mtv.corp.google.com");
|
|
} catch (IOException ioe) {
|
|
Log.w(LOGTAG, "error while resolving test host name", ioe);
|
|
}
|
|
if(addr == -1) {
|
|
Log.w(LOGTAG, "failed to resolve test host. http tests will fail.");
|
|
}
|
|
for (int i = 0; i < mTestList.size(); i++) {
|
|
String s = mTestList.elementAt(i);
|
|
boolean ignoreResult = mTestListIgnoreResult.elementAt(i);
|
|
FsUtils.updateTestStatus(TEST_STATUS_FILE, s);
|
|
// Run tests
|
|
// i is 0 based, but test count is 1 based so add 1 to i here.
|
|
runTestAndWaitUntilDone(activity, s, runner.mTimeoutInMillis, ignoreResult,
|
|
i + 1 + mResumeIndex);
|
|
}
|
|
|
|
FsUtils.updateTestStatus(TEST_STATUS_FILE, "#DONE");
|
|
ForwardService.getForwardService().stopForwardService();
|
|
activity.finish();
|
|
}
|
|
|
|
private String getTestPath() {
|
|
LayoutTestsAutoRunner runner = (LayoutTestsAutoRunner) getInstrumentation();
|
|
|
|
String test_path = LAYOUT_TESTS_ROOT;
|
|
if (runner.mTestPath != null) {
|
|
test_path += runner.mTestPath;
|
|
}
|
|
test_path = new File(test_path).getAbsolutePath();
|
|
Log.v("LayoutTestsAutoTest", " Test path : " + test_path);
|
|
|
|
return test_path;
|
|
}
|
|
|
|
public void generateTestList() {
|
|
try {
|
|
File tests_list = new File(LAYOUT_TESTS_LIST_FILE);
|
|
BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(tests_list, false));
|
|
FsUtils.writeLayoutTestListRecursively(bos, getTestPath(), false); // Don't ignore results
|
|
bos.flush();
|
|
bos.close();
|
|
} catch (Exception e) {
|
|
Log.e(LOGTAG, "Error when creating test list: " + e.getMessage());
|
|
}
|
|
}
|
|
|
|
// Running all the layout tests at once sometimes
|
|
// causes the dumprendertree to run out of memory.
|
|
// So, additional tests are added to run the tests
|
|
// in chunks.
|
|
public void startLayoutTests() {
|
|
try {
|
|
File tests_list = new File(LAYOUT_TESTS_LIST_FILE);
|
|
if (!tests_list.exists())
|
|
generateTestList();
|
|
} catch (Exception e) {
|
|
e.printStackTrace();
|
|
}
|
|
|
|
executeLayoutTests(false);
|
|
}
|
|
|
|
public void resumeLayoutTests() {
|
|
executeLayoutTests(true);
|
|
}
|
|
|
|
public void copyResultsAndRunnerAssetsToCache() {
|
|
try {
|
|
Context targetContext = getInstrumentation().getTargetContext();
|
|
File cacheDir = targetContext.getCacheDir();
|
|
|
|
for( int i=0; i< LAYOUT_TESTS_RESULTS_REFERENCE_FILES.length; i++) {
|
|
InputStream in = targetContext.getAssets().open(
|
|
LAYOUT_TESTS_RESULTS_REFERENCE_FILES[i]);
|
|
OutputStream out = new FileOutputStream(new File(cacheDir,
|
|
LAYOUT_TESTS_RESULTS_REFERENCE_FILES[i]));
|
|
|
|
byte[] buf = new byte[2048];
|
|
int len;
|
|
|
|
while ((len = in.read(buf)) >= 0 ) {
|
|
out.write(buf, 0, len);
|
|
}
|
|
out.close();
|
|
in.close();
|
|
}
|
|
}catch (IOException e) {
|
|
e.printStackTrace();
|
|
}
|
|
|
|
}
|
|
|
|
}
|