Android/frameworks_base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added reliability test usding DumpRenderTree

Browse Source
This commit is contained in:
Guang Zhu
2009-04-29 14:49:03 -07:00
parent 1e6dab1852
commit 4010ac35b1
7 changed files with 524 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
tests/DumpRenderTree/AndroidManifest.xml
View File

@@ -31,4 +31,5 @@
android:targetPackage="com.android.dumprendertree"
android:label="Layout test automation runner"
/>
<uses-permission android:name="android.permission.INTERNET"></uses-permission>
</manifest>

214
tests/DumpRenderTree/assets/run_reliability_tests.py Executable file
View File

@@ -0,0 +1,214 @@
#!/usr/bin/python2.4
"""Run reliability tests using Android instrumentation.
A test file consists of list web sites to test is needed as a parameter
Usage:
run_reliability_tests.py path/to/url/list
"""
import logging
import optparse
import random
import subprocess
import sys
import time
TEST_LIST_FILE = "/sdcard/android/reliability_tests_list.txt"
TEST_STATUS_FILE = "/sdcard/android/reliability_running_test.txt"
TEST_TIMEOUT_FILE = "/sdcard/android/reliability_timeout_test.txt"
HTTP_URL_FILE = "urllist_http"
HTTPS_URL_FILE = "urllist_https"
NUM_URLS = 25
def DumpRenderTreeFinished(adb_cmd):
"""Check if DumpRenderTree finished running.
Args:
adb_cmd: adb command string
Returns:
True if DumpRenderTree has finished, False otherwise
"""
# pull test status file and look for "#DONE"
shell_cmd_str = adb_cmd + " shell cat " + TEST_STATUS_FILE
adb_output = subprocess.Popen(shell_cmd_str,
shell=True, stdout=subprocess.PIPE,
stderr=subprocess.PIPE).communicate()[0]
return adb_output.strip() == "#DONE"
def RandomPick(file_name, approx_size, num_needed):
"""Randomly pick lines from the text file specifed.
Args:
file_name: the text file where lines should be picked from
approx_size: an approximate size of the text file
num_needed: how many lines are needed from the file
Returns:
an array of string
"""
p = float(num_needed) / approx_size
num_picked = 0
lines = []
random.seed()
while num_picked < num_needed:
file_handle = open(file_name, "r")
for line in file_handle:
line = line.strip()
if float(random.randint(0, approx_size)) / approx_size < p:
lines.append(line)
num_picked += 1
if num_picked == num_needed:
break
file_handle.close()
return lines
def main(options, args):
"""Send the url list to device and start testing, restart if crashed."""
generate_url = False
# Set up logging format.
log_level = logging.INFO
if options.verbose:
log_level = logging.DEBUG
logging.basicConfig(level=log_level,
format="%(message)s")
# Include all tests if none are specified.
if not args:
path = "/tmp/url_list_%d.txt" % time.time()
generate_url = True
logging.info("A URL list is not provided, will be automatically generated.")
else:
path = args[0]
if not options.crash_file:
print "missing crash file name, use --crash-file to specify"
sys.exit(1)
else:
crashed_file = options.crash_file
if not options.timeout_file:
print "missing timeout file, use --timeout-file to specify"
sys.exit(1)
else:
timedout_file = options.timeout_file
http = RandomPick(HTTP_URL_FILE, 500000, NUM_URLS)
https = RandomPick(HTTPS_URL_FILE, 45000, NUM_URLS)
if generate_url:
file_handle = open(path, "w")
for i in range(0, NUM_URLS):
file_handle.write(http[i] + "\n")
file_handle.write(https[i] + "\n")
file_handle.close()
adb_cmd = "adb "
if options.adb_options:
adb_cmd += options.adb_options + " "
# push url list to device
test_cmd = adb_cmd + " push \"" + path + "\" \"" + TEST_LIST_FILE + "\""
proc = subprocess.Popen(test_cmd, shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
(adb_output, adb_error) = proc.communicate()
if proc.returncode != 0:
logging.error("failed to push url list to device.")
logging.error(adb_output)
logging.error(adb_error)
sys.exit(1)
logging.info("Running the test ...")
# Count crashed tests.
crashed_tests = []
if options.time_out_ms:
timeout_ms = options.time_out_ms
# Run test until it's done
test_cmd_prefix = adb_cmd + " shell am instrument"
test_cmd_postfix = " -w com.android.dumprendertree/.LayoutTestsAutoRunner"
# Call ReliabilityTestsAutoTest#startReliabilityTests
test_cmd = (test_cmd_prefix + " -e class "
"com.android.dumprendertree.ReliabilityTestsAutoTest#"
"startReliabilityTests -e timeout " + timeout_ms
+ test_cmd_postfix)
time_start = time.time()
adb_output = subprocess.Popen(test_cmd, shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE).communicate()[0]
while not DumpRenderTreeFinished(adb_cmd):
logging.error("DumpRenderTree exited before all URLs are visited.")
shell_cmd_str = adb_cmd + " shell cat " + TEST_STATUS_FILE
crashed_test = subprocess.Popen(shell_cmd_str, shell=True,
stdout=subprocess.PIPE).communicate()[0]
logging.info(crashed_test + " CRASHED")
crashed_tests.append(crashed_test)
logging.info("Resuming reliability test runner...")
test_cmd = (test_cmd_prefix + " -e class "
"com.android.dumprendertree.ReliabilityTestsAutoTest#"
"resumeReliabilityTests -e timeout " + timeout_ms
+ test_cmd_postfix)
adb_output = subprocess.Popen(test_cmd, shell=True, stdout=subprocess.PIPE,
stderr=subprocess.PIPE).communicate()[0]
time_end = time.time()
fp = open("time_stat", "a")
fp.writelines("%.2f\n" % ((time_end - time_start) / NUM_URLS / 2))
fp.close()
if (adb_output.find("INSTRUMENTATION_FAILED") != -1 or
adb_output.find("Process crashed.") != -1):
logging.error("Error happened : " + adb_output)
sys.exit(1)
logging.info(adb_output)
logging.info("Done\n")
if crashed_tests:
file_handle = open(crashed_file, "w")
file_handle.writelines("\n".join(crashed_tests))
logging.info("Crashed URL list stored in: " + crashed_file)
file_handle.close()
else:
logging.info("No crash found.")
test_cmd = (adb_cmd + "pull \"" + TEST_TIMEOUT_FILE + "\" \""
+ timedout_file + "\"")
subprocess.Popen(test_cmd, shell=True, stdout=subprocess.PIPE,
stderr=subprocess.PIPE).communicate()
if "__main__" == __name__:
option_parser = optparse.OptionParser()
option_parser.add_option("", "--time-out-ms",
default=60000,
help="set the timeout for each test")
option_parser.add_option("", "--verbose", action="store_true",
default=False,
help="include debug-level logging")
option_parser.add_option("", "--adb-options",
default=None,
help="pass options to adb, such as -d -e, etc")
option_parser.add_option("", "--crash-file",
default="reliability_crashed_sites.txt",
help="the list of sites that cause browser to crash")
option_parser.add_option("", "--timeout-file",
default="reliability_timedout_sites.txt",
help="the list of sites that timedout during test.")
opts, arguments = option_parser.parse_args()
main(opts, arguments)

20
tests/DumpRenderTree/src/com/android/dumprendertree/LayoutTestsAutoTest.java
View File

@@ -41,6 +41,7 @@ class MyTestRecorder {
private BufferedOutputStream mBufferedOutputPassedStream;
private BufferedOutputStream mBufferedOutputFailedStream;
private BufferedOutputStream mBufferedOutputNoresultStream;
private BufferedOutputStream mBufferedOutputTimedoutStream;
public void passed(String layout_file) {
try {
@@ -72,11 +73,22 @@ class MyTestRecorder {
}
}
public void timedout(String url) {
try {
mBufferedOutputTimedoutStream.write(url.getBytes());
mBufferedOutputTimedoutStream.write('\n');
mBufferedOutputTimedoutStream.flush();
} catch (Exception e) {
e.printStackTrace();
}
}
public MyTestRecorder(boolean resume) {
try {
File resultsPassedFile = new File("/sdcard/layout_tests_passed.txt");
File resultsFailedFile = new File("/sdcard/layout_tests_failed.txt");
File noExpectedResultFile = new File("/sdcard/layout_tests_nontext.txt");
File resultTimedoutFile = new File("/sdcard/layout_tests_timedout.txt");
mBufferedOutputPassedStream =
new BufferedOutputStream(new FileOutputStream(resultsPassedFile, resume));
@@ -84,6 +96,8 @@ class MyTestRecorder {
new BufferedOutputStream(new FileOutputStream(resultsFailedFile, resume));
mBufferedOutputNoresultStream =
new BufferedOutputStream(new FileOutputStream(noExpectedResultFile, resume));
mBufferedOutputTimedoutStream =
new BufferedOutputStream(new FileOutputStream(resultTimedoutFile, resume));
} catch (Exception e) {
e.printStackTrace();
}
@@ -94,6 +108,7 @@ class MyTestRecorder {
mBufferedOutputPassedStream.close();
mBufferedOutputFailedStream.close();
mBufferedOutputNoresultStream.close();
mBufferedOutputTimedoutStream.close();
} catch (Exception e) {
e.printStackTrace();
}
@@ -281,7 +296,10 @@ public class LayoutTestsAutoTest extends ActivityInstrumentationTestCase2<TestSh
mFinished = true;
LayoutTestsAutoTest.this.notifyAll();
}
}
}
public void timedOut(String url) {
}
});
String resultFile = getResultFile(test);

5
tests/DumpRenderTree/src/com/android/dumprendertree/LoadTestsAutoTest.java
View File

@@ -42,7 +42,7 @@ public class LoadTestsAutoTest extends ActivityInstrumentationTestCase2<TestShel
private boolean mFinished;
static final String LOAD_TEST_RUNNER_FILES[] = {
"run_page_cycler.py"
};
};
public LoadTestsAutoTest() {
super("com.android.dumprendertree", TestShellActivity.class);
@@ -134,6 +134,9 @@ public class LoadTestsAutoTest extends ActivityInstrumentationTestCase2<TestShel
LoadTestsAutoTest.this.notifyAll();
}
}
public void timedOut(String url) {
}
});
mFinished = false;

209
tests/DumpRenderTree/src/com/android/dumprendertree/ReliabilityTestsAutoTest.java Normal file
View File

@@ -0,0 +1,209 @@
package com.android.dumprendertree;
import com.android.dumprendertree.TestShellActivity.DumpDataType;
import android.content.Intent;
import android.test.ActivityInstrumentationTestCase2;
import android.util.Log;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.List;
import java.util.Vector;
public class ReliabilityTestsAutoTest extends ActivityInstrumentationTestCase2<TestShellActivity> {
private static final String LOGTAG = "ReliabilityTests";
private static final String TEST_LIST_FILE = "/sdcard/android/reliability_tests_list.txt";
private static final String TEST_STATUS_FILE = "/sdcard/android/reliability_running_test.txt";
private static final String TEST_TIMEOUT_FILE = "/sdcard/android/reliability_timeout_test.txt";
static final String RELIABILITY_TEST_RUNNER_FILES[] = {
"run_reliability_tests.py"
};
private boolean finished;
private List<String> testList;
public ReliabilityTestsAutoTest() {
super("com.android.dumprendertree", TestShellActivity.class);
}
private void getTestList() {
// Read test list.
testList = new Vector<String>();
try {
BufferedReader inReader = new BufferedReader(new FileReader(TEST_LIST_FILE));
String line;
while ((line = inReader.readLine()) != null) {
testList.add(line);
}
inReader.close();
Log.v(LOGTAG, "Test list has " + testList.size() + " test(s).");
} catch (Exception e) {
Log.e(LOGTAG, "Error while reading test list : " + e.getMessage());
}
}
private void resumeTestList() {
// read out the test name it stopped last time.
try {
BufferedReader inReader = new BufferedReader(new FileReader(TEST_STATUS_FILE));
String line = inReader.readLine();
for (int i = 0; i < testList.size(); i++) {
if (testList.get(i).equals(line)) {
testList = new Vector<String>(testList.subList(i+1, testList.size()));
break;
}
}
inReader.close();
} catch (Exception e) {
Log.e(LOGTAG, "Error reading " + TEST_STATUS_FILE);
}
}
private void clearTestStatus() {
// Delete TEST_STATUS_FILE
try {
File f = new File(TEST_STATUS_FILE);
if (f.delete())
Log.v(LOGTAG, "Deleted " + TEST_STATUS_FILE);
else
Log.e(LOGTAG, "Fail to delete " + TEST_STATUS_FILE);
} catch (Exception e) {
Log.e(LOGTAG, "Fail to delete " + TEST_STATUS_FILE + " : " + e.getMessage());
}
}
private void clearTestTimeout() {
// Delete TEST_TIMEOUT_FILE
try {
File f = new File(TEST_TIMEOUT_FILE);
if (f.delete())
Log.v(LOGTAG, "Deleted " + TEST_TIMEOUT_FILE);
else
Log.e(LOGTAG, "Fail to delete " + TEST_TIMEOUT_FILE);
} catch (Exception e) {
Log.e(LOGTAG, "Fail to delete " + TEST_TIMEOUT_FILE + " : " + e.getMessage());
}
}
private void updateTestStatus(String s) {
// Write TEST_STATUS_FILE
try {
BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(TEST_STATUS_FILE));
bos.write(s.getBytes());
bos.close();
} catch (Exception e) {
Log.e(LOGTAG, "Cannot update file " + TEST_STATUS_FILE);
}
}
private void writeTimeoutFile(String s) {
// Write TEST_TIMEOUT_FILE
try {
BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(TEST_TIMEOUT_FILE, true));
bos.write(s.getBytes());
bos.write('\n');
bos.close();
} catch (Exception e) {
Log.e(LOGTAG, "Cannot update file " + TEST_TIMEOUT_FILE);
}
}
private void runReliabilityTest(boolean resume) {
LayoutTestsAutoRunner runner = (LayoutTestsAutoRunner) getInstrumentation();
getTestList();
if(!resume)
clearTestStatus();
else
resumeTestList();
TestShellActivity activity = getActivity();
activity.setDefaultDumpDataType(DumpDataType.NO_OP);
// Run tests.
for (int i = 0; i < testList.size(); i++) {
String s = testList.get(i);
updateTestStatus(s);
// Run tests
runTestAndWaitUntilDone(activity, s, runner.mTimeoutInMillis);
}
updateTestStatus("#DONE");
activity.finish();
}
private void runTestAndWaitUntilDone(TestShellActivity activity, String url, int timeout) {
activity.setCallback(new TestShellCallback() {
public void finished() {
synchronized (ReliabilityTestsAutoTest.this) {
finished = true;
ReliabilityTestsAutoTest.this.notifyAll();
}
}
public void timedOut(String url) {
writeTimeoutFile(url);
}
});
finished = false;
Intent intent = new Intent(Intent.ACTION_VIEW);
intent.setClass(activity, TestShellActivity.class);
intent.addFlags(Intent.FLAG_ACTIVITY_SINGLE_TOP);
intent.putExtra(TestShellActivity.TEST_URL, url);
intent.putExtra(TestShellActivity.TIMEOUT_IN_MILLIS, timeout);
activity.startActivity(intent);
// Wait until done.
synchronized (this) {
while(!finished){
try {
this.wait();
} catch (InterruptedException e) { }
}
}
}
public void startReliabilityTests() {
clearTestTimeout();
runReliabilityTest(false);
}
public void resumeReliabilityTests() {
runReliabilityTest(true);
}
public void copyRunnerAssetsToCache() {
try {
String out_dir = getActivity().getApplicationContext()
.getCacheDir().getPath() + "/";
for( int i=0; i< RELIABILITY_TEST_RUNNER_FILES.length; i++) {
InputStream in = getActivity().getAssets().open(
RELIABILITY_TEST_RUNNER_FILES[i]);
OutputStream out = new FileOutputStream(
out_dir + RELIABILITY_TEST_RUNNER_FILES[i]);
byte[] buf = new byte[2048];
int len;
while ((len = in.read(buf)) >= 0 ) {
out.write(buf, 0, len);
}
out.close();
in.close();
}
}catch (IOException e) {
e.printStackTrace();
}
}
}

95
tests/DumpRenderTree/src/com/android/dumprendertree/TestShellActivity.java
View File

@@ -16,29 +16,40 @@
package com.android.dumprendertree;
import android.app.Activity;
import android.content.Intent;
import android.graphics.Bitmap;
import android.net.http.SslError;
import android.os.Bundle;
import android.os.Handler;
import android.os.Message;
import android.util.Log;
import android.view.ViewGroup;
import android.webkit.HttpAuthHandler;
import android.webkit.JsPromptResult;
import android.webkit.JsResult;
import android.webkit.SslErrorHandler;
import android.webkit.WebChromeClient;
import android.webkit.WebSettings;
import android.webkit.WebView;
import android.webkit.WebViewClient;
import android.widget.LinearLayout;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.Vector;
import android.app.Activity;
import android.content.Intent;
import android.util.Log;
import android.webkit.JsPromptResult;
import android.webkit.JsResult;
import android.view.ViewGroup;
import android.webkit.WebChromeClient;
import android.webkit.WebSettings;
import android.webkit.WebView;
import android.widget.LinearLayout;
import android.os.*;
public class TestShellActivity extends Activity implements LayoutTestController {
static enum DumpDataType {DUMP_AS_TEXT, EXT_REPR, NO_OP}
public class AsyncHandler extends Handler {
@Override
public void handleMessage(Message msg) {
if (msg.what == MSG_TIMEOUT) {
mTimedOut = true;
mCallback.timedOut(mWebView.getUrl());
requestWebKitData();
return;
} else if (msg.what == MSG_WEBKIT_DATA) {
@@ -57,10 +68,16 @@ public class TestShellActivity extends Activity implements LayoutTestController
throw new AssertionError("Requested webkit data twice: " + mWebView.getUrl());
mRequestedWebKitData = true;
if (mDumpAsText) {
mWebView.documentAsText(callback);
} else {
mWebView.externalRepresentation(callback);
switch (mDumpDataType) {
case DUMP_AS_TEXT:
mWebView.documentAsText(callback);
break;
case EXT_REPR:
mWebView.externalRepresentation(callback);
break;
default:
finished();
break;
}
}
@@ -75,6 +92,41 @@ public class TestShellActivity extends Activity implements LayoutTestController
mWebView = new WebView(this);
mWebView.getSettings().setJavaScriptEnabled(true);
mWebView.setWebChromeClient(mChromeClient);
mWebView.setWebViewClient(new WebViewClient(){
@Override
public void onPageFinished(WebView view, String url) {
Log.v(LOGTAG, "onPageFinished, url=" + url);
super.onPageFinished(view, url);
}
@Override
public void onPageStarted(WebView view, String url, Bitmap favicon) {
Log.v(LOGTAG, "onPageStarted, url=" + url);
super.onPageStarted(view, url, favicon);
}
@Override
public void onReceivedError(WebView view, int errorCode, String description,
String failingUrl) {
Log.v(LOGTAG, "onReceivedError, errorCode=" + errorCode
+ ", desc=" + description + ", url=" + failingUrl);
super.onReceivedError(view, errorCode, description, failingUrl);
}
@Override
public void onReceivedHttpAuthRequest(WebView view, HttpAuthHandler handler,
String host, String realm) {
handler.cancel();
}
@Override
public void onReceivedSslError(WebView view, SslErrorHandler handler,
SslError error) {
handler.proceed();
}
});
mEventSender = new WebViewEventSender(mWebView);
mCallbackProxy = new CallbackProxy(mEventSender, this);
@@ -186,10 +238,14 @@ public class TestShellActivity extends Activity implements LayoutTestController
}
}
public void setDefaultDumpDataType(DumpDataType defaultDumpDataType) {
mDefaultDumpDataType = defaultDumpDataType;
}
// .......................................
// LayoutTestController Functions
public void dumpAsText() {
mDumpAsText = true;
mDumpDataType = DumpDataType.DUMP_AS_TEXT;
if (mWebView != null) {
String url = mWebView.getUrl();
Log.v(LOGTAG, "dumpAsText called: "+url);
@@ -382,7 +438,7 @@ public class TestShellActivity extends Activity implements LayoutTestController
private void resetTestStatus() {
mWaitUntilDone = false;
mDumpAsText = false;
mDumpDataType = mDefaultDumpDataType;
mTimedOut = false;
mDumpTitleChanges = false;
mRequestedWebKitData = false;
@@ -406,7 +462,8 @@ public class TestShellActivity extends Activity implements LayoutTestController
private boolean mFinishedRunning;
// Layout test controller variables.
private boolean mDumpAsText;
private DumpDataType mDumpDataType;
private DumpDataType mDefaultDumpDataType = DumpDataType.EXT_REPR;
private boolean mWaitUntilDone;
private boolean mDumpTitleChanges;
private StringBuffer mTitleChanges;

1
tests/DumpRenderTree/src/com/android/dumprendertree/TestShellCallback.java
View File

@@ -18,4 +18,5 @@ package com.android.dumprendertree;
public interface TestShellCallback {
public void finished();
public void timedOut(String url);
}