added reliability test usding DumpRenderTree

tests/DumpRenderTree/AndroidManifest.xml

@@ -31,4 +31,5 @@
         android:targetPackage="com.android.dumprendertree"
         android:label="Layout test automation runner"
     />
+    <uses-permission android:name="android.permission.INTERNET"></uses-permission>
 </manifest>

tests/DumpRenderTree/assets/run_reliability_tests.py

@@ -0,0 +1,214 @@
+#!/usr/bin/python2.4
+
+"""Run reliability tests using Android instrumentation.
+
+  A test file consists of list web sites to test is needed as a parameter
+
+  Usage:
+    run_reliability_tests.py path/to/url/list
+"""
+
+import logging
+import optparse
+import random
+import subprocess
+import sys
+import time
+
+TEST_LIST_FILE = "/sdcard/android/reliability_tests_list.txt"
+TEST_STATUS_FILE = "/sdcard/android/reliability_running_test.txt"
+TEST_TIMEOUT_FILE = "/sdcard/android/reliability_timeout_test.txt"
+HTTP_URL_FILE = "urllist_http"
+HTTPS_URL_FILE = "urllist_https"
+NUM_URLS = 25
+
+
+def DumpRenderTreeFinished(adb_cmd):
+  """Check if DumpRenderTree finished running.
+
+  Args:
+    adb_cmd: adb command string
+
+  Returns:
+    True if DumpRenderTree has finished, False otherwise
+  """
+
+  # pull test status file and look for "#DONE"
+  shell_cmd_str = adb_cmd + " shell cat " + TEST_STATUS_FILE
+  adb_output = subprocess.Popen(shell_cmd_str,
+                                shell=True, stdout=subprocess.PIPE,
+                                stderr=subprocess.PIPE).communicate()[0]
+  return adb_output.strip() == "#DONE"
+
+
+def RandomPick(file_name, approx_size, num_needed):
+  """Randomly pick lines from the text file specifed.
+
+  Args:
+    file_name: the text file where lines should be picked from
+    approx_size: an approximate size of the text file
+    num_needed: how many lines are needed from the file
+
+  Returns:
+    an array of string
+  """
+  p = float(num_needed) / approx_size
+  num_picked = 0
+  lines = []
+  random.seed()
+
+  while num_picked < num_needed:
+    file_handle = open(file_name, "r")
+    for line in file_handle:
+      line = line.strip()
+      if float(random.randint(0, approx_size)) / approx_size < p:
+        lines.append(line)
+        num_picked += 1
+        if num_picked == num_needed:
+          break
+    file_handle.close()
+  return lines
+
+
+def main(options, args):
+  """Send the url list to device and start testing, restart if crashed."""
+
+  generate_url = False
+
+  # Set up logging format.
+  log_level = logging.INFO
+  if options.verbose:
+    log_level = logging.DEBUG
+  logging.basicConfig(level=log_level,
+                      format="%(message)s")
+
+  # Include all tests if none are specified.
+  if not args:
+    path = "/tmp/url_list_%d.txt" % time.time()
+    generate_url = True
+    logging.info("A URL list is not provided, will be automatically generated.")
+  else:
+    path = args[0]
+
+  if not options.crash_file:
+    print "missing crash file name, use --crash-file to specify"
+    sys.exit(1)
+  else:
+    crashed_file = options.crash_file
+
+  if not options.timeout_file:
+    print "missing timeout file, use --timeout-file to specify"
+    sys.exit(1)
+  else:
+    timedout_file = options.timeout_file
+
+  http = RandomPick(HTTP_URL_FILE, 500000, NUM_URLS)
+  https = RandomPick(HTTPS_URL_FILE, 45000, NUM_URLS)
+
+  if generate_url:
+    file_handle = open(path, "w")
+    for i in range(0, NUM_URLS):
+      file_handle.write(http[i] + "\n")
+      file_handle.write(https[i] + "\n")
+    file_handle.close()
+
+  adb_cmd = "adb "
+  if options.adb_options:
+    adb_cmd += options.adb_options + " "
+
+  # push url list to device
+  test_cmd = adb_cmd + " push \"" + path + "\" \"" + TEST_LIST_FILE + "\""
+  proc = subprocess.Popen(test_cmd, shell=True,
+                          stdout=subprocess.PIPE,
+                          stderr=subprocess.PIPE)
+  (adb_output, adb_error) = proc.communicate()
+  if proc.returncode != 0:
+    logging.error("failed to push url list to device.")
+    logging.error(adb_output)
+    logging.error(adb_error)
+    sys.exit(1)
+
+  logging.info("Running the test ...")
+
+  # Count crashed tests.
+  crashed_tests = []
+
+  if options.time_out_ms:
+    timeout_ms = options.time_out_ms
+
+  # Run test until it's done
+  test_cmd_prefix = adb_cmd + " shell am instrument"
+  test_cmd_postfix = " -w com.android.dumprendertree/.LayoutTestsAutoRunner"
+
+  # Call ReliabilityTestsAutoTest#startReliabilityTests
+  test_cmd = (test_cmd_prefix + " -e class "
+              "com.android.dumprendertree.ReliabilityTestsAutoTest#"
+              "startReliabilityTests -e timeout " + timeout_ms
+              + test_cmd_postfix)
+
+  time_start = time.time()
+  adb_output = subprocess.Popen(test_cmd, shell=True,
+                                stdout=subprocess.PIPE,
+                                stderr=subprocess.PIPE).communicate()[0]
+  while not DumpRenderTreeFinished(adb_cmd):
+    logging.error("DumpRenderTree exited before all URLs are visited.")
+    shell_cmd_str = adb_cmd + " shell cat " + TEST_STATUS_FILE
+    crashed_test = subprocess.Popen(shell_cmd_str, shell=True,
+                                    stdout=subprocess.PIPE).communicate()[0]
+    logging.info(crashed_test + " CRASHED")
+    crashed_tests.append(crashed_test)
+    logging.info("Resuming reliability test runner...")
+
+    test_cmd = (test_cmd_prefix + " -e class "
+                "com.android.dumprendertree.ReliabilityTestsAutoTest#"
+                "resumeReliabilityTests -e timeout " + timeout_ms
+                + test_cmd_postfix)
+    adb_output = subprocess.Popen(test_cmd, shell=True, stdout=subprocess.PIPE,
+                                  stderr=subprocess.PIPE).communicate()[0]
+
+  time_end = time.time()
+  fp = open("time_stat", "a")
+  fp.writelines("%.2f\n" % ((time_end - time_start) / NUM_URLS / 2))
+  fp.close()
+  if (adb_output.find("INSTRUMENTATION_FAILED") != -1 or
+      adb_output.find("Process crashed.") != -1):
+    logging.error("Error happened : " + adb_output)
+    sys.exit(1)
+
+  logging.info(adb_output)
+  logging.info("Done\n")
+
+  if crashed_tests:
+    file_handle = open(crashed_file, "w")
+    file_handle.writelines("\n".join(crashed_tests))
+    logging.info("Crashed URL list stored in: " + crashed_file)
+    file_handle.close()
+  else:
+    logging.info("No crash found.")
+
+  test_cmd = (adb_cmd + "pull \"" + TEST_TIMEOUT_FILE + "\" \""
+              + timedout_file +  "\"")
+
+  subprocess.Popen(test_cmd, shell=True, stdout=subprocess.PIPE,
+                   stderr=subprocess.PIPE).communicate()
+
+
+if "__main__" == __name__:
+  option_parser = optparse.OptionParser()
+  option_parser.add_option("", "--time-out-ms",
+                           default=60000,
+                           help="set the timeout for each test")
+  option_parser.add_option("", "--verbose", action="store_true",
+                           default=False,
+                           help="include debug-level logging")
+  option_parser.add_option("", "--adb-options",
+                           default=None,
+                           help="pass options to adb, such as -d -e, etc")
+  option_parser.add_option("", "--crash-file",
+                           default="reliability_crashed_sites.txt",
+                           help="the list of sites that cause browser to crash")
+  option_parser.add_option("", "--timeout-file",
+                           default="reliability_timedout_sites.txt",
+                           help="the list of sites that timedout during test.")
+  opts, arguments = option_parser.parse_args()
+  main(opts, arguments)

tests/DumpRenderTree/src/com/android/dumprendertree/LayoutTestsAutoTest.java

@@ -41,6 +41,7 @@ class MyTestRecorder {
     private BufferedOutputStream mBufferedOutputPassedStream;
     private BufferedOutputStream mBufferedOutputFailedStream;
     private BufferedOutputStream mBufferedOutputNoresultStream;
+    private BufferedOutputStream mBufferedOutputTimedoutStream;
     
     public void passed(String layout_file) {
         try {
@@ -72,11 +73,22 @@ class MyTestRecorder {
         }
     }
     
+    public void timedout(String url) {
+        try {
+            mBufferedOutputTimedoutStream.write(url.getBytes());
+            mBufferedOutputTimedoutStream.write('\n');
+            mBufferedOutputTimedoutStream.flush();
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+    
     public MyTestRecorder(boolean resume) {
         try {
             File resultsPassedFile = new File("/sdcard/layout_tests_passed.txt");
             File resultsFailedFile = new File("/sdcard/layout_tests_failed.txt");
             File noExpectedResultFile = new File("/sdcard/layout_tests_nontext.txt");
+            File resultTimedoutFile = new File("/sdcard/layout_tests_timedout.txt");
           
             mBufferedOutputPassedStream =
                 new BufferedOutputStream(new FileOutputStream(resultsPassedFile, resume));
@@ -84,6 +96,8 @@ class MyTestRecorder {
                 new BufferedOutputStream(new FileOutputStream(resultsFailedFile, resume));
             mBufferedOutputNoresultStream =
                 new BufferedOutputStream(new FileOutputStream(noExpectedResultFile, resume));
+            mBufferedOutputTimedoutStream =
+                new BufferedOutputStream(new FileOutputStream(resultTimedoutFile, resume));
         } catch (Exception e) {
             e.printStackTrace();
         }
@@ -94,6 +108,7 @@ class MyTestRecorder {
             mBufferedOutputPassedStream.close();
             mBufferedOutputFailedStream.close();
             mBufferedOutputNoresultStream.close();
+            mBufferedOutputTimedoutStream.close();
         } catch (Exception e) {
             e.printStackTrace();
         }
@@ -282,6 +297,9 @@ public class LayoutTestsAutoTest extends ActivityInstrumentationTestCase2<TestSh
                     LayoutTestsAutoTest.this.notifyAll();
                 }
             }
+            
+            public void timedOut(String url) {
+            }
         });
 
         String resultFile = getResultFile(test);

tests/DumpRenderTree/src/com/android/dumprendertree/LoadTestsAutoTest.java

@@ -42,7 +42,7 @@ public class LoadTestsAutoTest extends ActivityInstrumentationTestCase2<TestShel
     private boolean mFinished;
     static final String LOAD_TEST_RUNNER_FILES[] = {
         "run_page_cycler.py"
-  };
+    };
 
     public LoadTestsAutoTest() {
         super("com.android.dumprendertree", TestShellActivity.class);
@@ -134,6 +134,9 @@ public class LoadTestsAutoTest extends ActivityInstrumentationTestCase2<TestShel
                     LoadTestsAutoTest.this.notifyAll();
                 }
             }
+            
+            public void timedOut(String url) {
+            }
         });
 
         mFinished = false;

tests/DumpRenderTree/src/com/android/dumprendertree/ReliabilityTestsAutoTest.java

@@ -0,0 +1,209 @@
+package com.android.dumprendertree;
+
+import com.android.dumprendertree.TestShellActivity.DumpDataType;
+
+import android.content.Intent;
+import android.test.ActivityInstrumentationTestCase2;
+import android.util.Log;
+
+import java.io.BufferedOutputStream;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.List;
+import java.util.Vector;
+
+public class ReliabilityTestsAutoTest extends ActivityInstrumentationTestCase2<TestShellActivity> {
+
+    private static final String LOGTAG = "ReliabilityTests";
+    private static final String TEST_LIST_FILE = "/sdcard/android/reliability_tests_list.txt";
+    private static final String TEST_STATUS_FILE = "/sdcard/android/reliability_running_test.txt";
+    private static final String TEST_TIMEOUT_FILE = "/sdcard/android/reliability_timeout_test.txt";
+    static final String RELIABILITY_TEST_RUNNER_FILES[] = {
+        "run_reliability_tests.py"
+    };
+
+    private boolean finished;
+    private List<String> testList; 
+
+    public ReliabilityTestsAutoTest() {
+        super("com.android.dumprendertree", TestShellActivity.class);
+    }
+
+    private void getTestList() {
+        // Read test list.
+        testList = new Vector<String>();
+        try {
+            BufferedReader inReader = new BufferedReader(new FileReader(TEST_LIST_FILE));
+            String line;
+            while ((line = inReader.readLine()) != null) {
+                testList.add(line);
+            }
+            inReader.close();
+            Log.v(LOGTAG, "Test list has " + testList.size() + " test(s).");
+        } catch (Exception e) {
+            Log.e(LOGTAG, "Error while reading test list : " + e.getMessage());
+        }
+    }
+
+    private void resumeTestList() {
+        // read out the test name it stopped last time.
+        try {
+            BufferedReader inReader = new BufferedReader(new FileReader(TEST_STATUS_FILE));
+            String line = inReader.readLine();
+            for (int i = 0; i < testList.size(); i++) {
+                if (testList.get(i).equals(line)) {
+                    testList = new Vector<String>(testList.subList(i+1, testList.size()));
+                    break;
+                }
+            }
+            inReader.close();
+        } catch (Exception e) {
+            Log.e(LOGTAG, "Error reading " + TEST_STATUS_FILE);
+        }
+    }
+
+    private void clearTestStatus() {
+        // Delete TEST_STATUS_FILE
+        try {
+            File f = new File(TEST_STATUS_FILE);
+            if (f.delete())
+                Log.v(LOGTAG, "Deleted " + TEST_STATUS_FILE);
+            else
+                Log.e(LOGTAG, "Fail to delete " + TEST_STATUS_FILE);
+        } catch (Exception e) {
+            Log.e(LOGTAG, "Fail to delete " + TEST_STATUS_FILE + " : " + e.getMessage());
+        }
+    }
+
+    private void clearTestTimeout() {
+        // Delete TEST_TIMEOUT_FILE
+        try {
+            File f = new File(TEST_TIMEOUT_FILE);
+            if (f.delete())
+                Log.v(LOGTAG, "Deleted " + TEST_TIMEOUT_FILE);
+            else
+                Log.e(LOGTAG, "Fail to delete " + TEST_TIMEOUT_FILE);
+        } catch (Exception e) {
+            Log.e(LOGTAG, "Fail to delete " + TEST_TIMEOUT_FILE + " : " + e.getMessage());
+        }
+    }
+
+    private void updateTestStatus(String s) {
+        // Write TEST_STATUS_FILE
+        try {
+            BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(TEST_STATUS_FILE));
+            bos.write(s.getBytes());
+            bos.close();
+        } catch (Exception e) {
+            Log.e(LOGTAG, "Cannot update file " + TEST_STATUS_FILE);
+        }
+    }
+
+    private void writeTimeoutFile(String s) {
+        // Write TEST_TIMEOUT_FILE
+        try {
+            BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(TEST_TIMEOUT_FILE, true));
+            bos.write(s.getBytes());
+            bos.write('\n');
+            bos.close();
+        } catch (Exception e) {
+            Log.e(LOGTAG, "Cannot update file " + TEST_TIMEOUT_FILE);
+        }
+    }
+
+    private void runReliabilityTest(boolean resume) {
+        LayoutTestsAutoRunner runner = (LayoutTestsAutoRunner) getInstrumentation();
+
+        getTestList();
+        if(!resume)
+            clearTestStatus();
+        else
+            resumeTestList();
+
+        TestShellActivity activity = getActivity();
+        activity.setDefaultDumpDataType(DumpDataType.NO_OP);
+        // Run tests.
+        for (int i = 0; i < testList.size(); i++) {
+            String s = testList.get(i);
+            updateTestStatus(s);
+            // Run tests
+            runTestAndWaitUntilDone(activity, s, runner.mTimeoutInMillis);
+        }
+
+        updateTestStatus("#DONE");
+
+        activity.finish();
+    }
+
+    private void runTestAndWaitUntilDone(TestShellActivity activity, String url, int timeout) {
+        activity.setCallback(new TestShellCallback() {
+            public void finished() {
+                synchronized (ReliabilityTestsAutoTest.this) {
+                    finished = true;
+                    ReliabilityTestsAutoTest.this.notifyAll();
+                }
+            }
+
+            public void timedOut(String url) {
+                writeTimeoutFile(url);
+            }
+        });
+
+        finished = false;
+        Intent intent = new Intent(Intent.ACTION_VIEW);
+        intent.setClass(activity, TestShellActivity.class);
+        intent.addFlags(Intent.FLAG_ACTIVITY_SINGLE_TOP);
+        intent.putExtra(TestShellActivity.TEST_URL, url);
+        intent.putExtra(TestShellActivity.TIMEOUT_IN_MILLIS, timeout);
+        activity.startActivity(intent);
+
+        // Wait until done.
+        synchronized (this) {
+            while(!finished){
+                try {
+                    this.wait();
+                } catch (InterruptedException e) { }
+            }
+        }
+    }
+
+    public void startReliabilityTests() {
+        clearTestTimeout();
+        runReliabilityTest(false);
+    }
+
+    public void resumeReliabilityTests() {
+        runReliabilityTest(true);
+    }
+
+    public void copyRunnerAssetsToCache() {
+        try {
+            String out_dir = getActivity().getApplicationContext()
+            .getCacheDir().getPath() + "/";
+
+            for( int i=0; i< RELIABILITY_TEST_RUNNER_FILES.length; i++) {
+                InputStream in = getActivity().getAssets().open(
+                        RELIABILITY_TEST_RUNNER_FILES[i]);
+                OutputStream out = new FileOutputStream(
+                        out_dir + RELIABILITY_TEST_RUNNER_FILES[i]);
+
+                byte[] buf = new byte[2048];
+                int len;
+
+                while ((len = in.read(buf)) >= 0 ) {
+                    out.write(buf, 0, len);
+                }
+                out.close();
+                in.close();
+            }
+        }catch (IOException e) {
+            e.printStackTrace();
+        }
+
+    }
+}

tests/DumpRenderTree/src/com/android/dumprendertree/TestShellActivity.java

@@ -16,29 +16,40 @@
 
 package com.android.dumprendertree;
 
+import android.app.Activity;
+import android.content.Intent;
+import android.graphics.Bitmap;
+import android.net.http.SslError;
+import android.os.Bundle;
+import android.os.Handler;
+import android.os.Message;
+import android.util.Log;
+import android.view.ViewGroup;
+import android.webkit.HttpAuthHandler;
+import android.webkit.JsPromptResult;
+import android.webkit.JsResult;
+import android.webkit.SslErrorHandler;
+import android.webkit.WebChromeClient;
+import android.webkit.WebSettings;
+import android.webkit.WebView;
+import android.webkit.WebViewClient;
+import android.widget.LinearLayout;
+
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.util.Vector;
 
-import android.app.Activity;
-import android.content.Intent;
-import android.util.Log;
-import android.webkit.JsPromptResult;
-import android.webkit.JsResult;
-import android.view.ViewGroup;
-import android.webkit.WebChromeClient;
-import android.webkit.WebSettings;
-import android.webkit.WebView;
-import android.widget.LinearLayout;
-import android.os.*;
-
 public class TestShellActivity extends Activity implements LayoutTestController {
+    
+    static enum DumpDataType {DUMP_AS_TEXT, EXT_REPR, NO_OP}
+    
     public class AsyncHandler extends Handler {
         @Override
         public void handleMessage(Message msg) {
             if (msg.what == MSG_TIMEOUT) {
                 mTimedOut = true;
+                mCallback.timedOut(mWebView.getUrl());
                 requestWebKitData();
                 return;
             } else if (msg.what == MSG_WEBKIT_DATA) {
@@ -57,10 +68,16 @@ public class TestShellActivity extends Activity implements LayoutTestController
             throw new AssertionError("Requested webkit data twice: " + mWebView.getUrl());
         
         mRequestedWebKitData = true;
-        if (mDumpAsText) { 
+        switch (mDumpDataType) {
-            mWebView.documentAsText(callback);
+            case DUMP_AS_TEXT:
-        } else {
+                mWebView.documentAsText(callback);
-            mWebView.externalRepresentation(callback);
+                break;
+            case EXT_REPR:
+                mWebView.externalRepresentation(callback);
+                break;
+            default:
+                finished();
+                break;
         }
     } 
 
@@ -75,6 +92,41 @@ public class TestShellActivity extends Activity implements LayoutTestController
         mWebView = new WebView(this);
         mWebView.getSettings().setJavaScriptEnabled(true);
         mWebView.setWebChromeClient(mChromeClient);
+        mWebView.setWebViewClient(new WebViewClient(){
+
+            @Override
+            public void onPageFinished(WebView view, String url) {
+                Log.v(LOGTAG, "onPageFinished, url=" + url);
+                super.onPageFinished(view, url);
+            }
+
+            @Override
+            public void onPageStarted(WebView view, String url, Bitmap favicon) {
+                Log.v(LOGTAG, "onPageStarted, url=" + url);
+                super.onPageStarted(view, url, favicon);
+            }
+
+            @Override
+            public void onReceivedError(WebView view, int errorCode, String description,
+                    String failingUrl) {
+                Log.v(LOGTAG, "onReceivedError, errorCode=" + errorCode
+                        + ", desc=" + description + ", url=" + failingUrl);
+                super.onReceivedError(view, errorCode, description, failingUrl);
+            }
+
+            @Override
+            public void onReceivedHttpAuthRequest(WebView view, HttpAuthHandler handler,
+                    String host, String realm) {
+                handler.cancel();
+            }
+
+            @Override
+            public void onReceivedSslError(WebView view, SslErrorHandler handler,
+                    SslError error) {
+                handler.proceed();
+            }
+
+        });
         mEventSender = new WebViewEventSender(mWebView);
         mCallbackProxy = new CallbackProxy(mEventSender, this);
 
@@ -186,10 +238,14 @@ public class TestShellActivity extends Activity implements LayoutTestController
         }
     }
    
+    public void setDefaultDumpDataType(DumpDataType defaultDumpDataType) {
+        mDefaultDumpDataType = defaultDumpDataType;
+    }
+
     // .......................................
     // LayoutTestController Functions
     public void dumpAsText() {
-        mDumpAsText = true;
+        mDumpDataType = DumpDataType.DUMP_AS_TEXT;
         if (mWebView != null) {
             String url = mWebView.getUrl();
             Log.v(LOGTAG, "dumpAsText called: "+url);
@@ -382,7 +438,7 @@ public class TestShellActivity extends Activity implements LayoutTestController
     
     private void resetTestStatus() {
         mWaitUntilDone = false;
-        mDumpAsText = false;
+        mDumpDataType = mDefaultDumpDataType;
         mTimedOut = false;
         mDumpTitleChanges = false;
         mRequestedWebKitData = false;
@@ -406,7 +462,8 @@ public class TestShellActivity extends Activity implements LayoutTestController
     private boolean mFinishedRunning;
 
     // Layout test controller variables.
-    private boolean mDumpAsText;
+    private DumpDataType mDumpDataType;
+    private DumpDataType mDefaultDumpDataType = DumpDataType.EXT_REPR;
     private boolean mWaitUntilDone;
     private boolean mDumpTitleChanges;
     private StringBuffer mTitleChanges;

tests/DumpRenderTree/src/com/android/dumprendertree/TestShellCallback.java

@@ -18,4 +18,5 @@ package com.android.dumprendertree;
 
 public interface TestShellCallback {
     public void finished();
+    public void timedOut(String url);
 }