From 9bdc94b7a42a07d7dafcdf2cbadbb9c736b979d2 Mon Sep 17 00:00:00 2001
From: Dan Egnor <egnor@google.com>
Date: Thu, 4 Mar 2010 14:20:31 -0800
Subject: [PATCH] Improve watchdog diagnostics.

Capture stack traces from the system process using the same
mechanism as ANRs (which will initialize traces.txt, etc).
Also record the watchdog reset in the dropbox for uploading.

Bug: 2475557
---
 .../java/com/android/server/Watchdog.java     | 35 +++++++++----------
 .../server/am/ActivityManagerService.java     |  7 ++--
 2 files changed, 22 insertions(+), 20 deletions(-)

diff --git a/services/java/com/android/server/Watchdog.java b/services/java/com/android/server/Watchdog.java
index ee69715391b92..f97f50a45a125 100644
--- a/services/java/com/android/server/Watchdog.java
+++ b/services/java/com/android/server/Watchdog.java
@@ -34,8 +34,10 @@ import android.os.SystemProperties;
 import android.provider.Settings;
 import android.util.Config;
 import android.util.EventLog;
+import android.util.Log;
 import android.util.Slog;
 
+import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Calendar;
@@ -804,19 +806,14 @@ public class Watchdog extends Thread {
                 // to timeout on is asleep as well and won't have a chance to run. Causing a false
                 // positive on when to kill things.
                 long start = SystemClock.uptimeMillis();
-                do {
+                while (timeout > 0 && !mForceKillSystem) {
                     try {
-                        wait(timeout);
+                        wait(timeout);  // notifyAll() is called when mForceKillSystem is set
                     } catch (InterruptedException e) {
-                        if (SystemProperties.getBoolean("ro.secure", false)) {
-                            // If this is a secure build, just log the error.
-                            Slog.e("WatchDog", "Woof! Woof! Interrupter!");
-                        } else {
-                            throw new AssertionError("Someone interrupted the watchdog");
-                        }
+                        Log.wtf(TAG, e);
                     }
                     timeout = TIME_TO_WAIT - (SystemClock.uptimeMillis() - start);
-                } while (timeout > 0 && !mForceKillSystem);
+                }
 
                 if (mCompleted && !mForceKillSystem) {
                     // The monitors have returned.
@@ -825,22 +822,24 @@ public class Watchdog extends Thread {
             }
 
             // If we got here, that means that the system is most likely hung.
-            // First send a SIGQUIT so that we can see where it was hung. Then
-            // kill this process so that the system will restart.
+            // First collect stack traces from all threads of the system process.
+            // Then kill this process so that the system will restart.
+
             String name = (mCurrentMonitor != null) ? mCurrentMonitor.getClass().getName() : "null";
             EventLog.writeEvent(EventLogTags.WATCHDOG, name);
-            Process.sendSignal(Process.myPid(), Process.SIGNAL_QUIT);
 
-            // Wait a bit longer before killing so we can make sure that the stacks are captured.
-            try {
-                Thread.sleep(10*1000);
-            } catch (InterruptedException e) {
-            }
+            ArrayList pids = new ArrayList();
+            pids.add(Process.myPid());
+            File stack = ActivityManagerService.dumpStackTraces(pids);
+            mActivity.addErrorToDropBox("watchdog", null, null, null, name, null, stack, null);
 
             // Only kill the process if the debugger is not attached.
             if (!Debug.isDebuggerConnected()) {
-                Slog.i(TAG, "Watchdog is killing the system process");
+                Slog.w(TAG, "*** WATCHDOG KILLING SYSTEM PROCESS: " + name);
                 Process.killProcess(Process.myPid());
+                System.exit(10);
+            } else {
+                Slog.w(TAG, "Debugger connected: Watchdog is *not* killing the system process");
             }
         }
     }
diff --git a/services/java/com/android/server/am/ActivityManagerService.java b/services/java/com/android/server/am/ActivityManagerService.java
index a263b23f32332..9018872eba108 100644
--- a/services/java/com/android/server/am/ActivityManagerService.java
+++ b/services/java/com/android/server/am/ActivityManagerService.java
@@ -4662,7 +4662,7 @@ public final class ActivityManagerService extends ActivityManagerNative implemen
      * @param pids of dalvik VM processes to dump stack traces for
      * @return file containing stack traces, or null if no dump file is configured
      */
-    private static File dumpStackTraces(ArrayList<Integer> pids) {
+    public static File dumpStackTraces(ArrayList<Integer> pids) {
         String tracesPath = SystemProperties.get("dalvik.vm.stack-trace-file", null);
         if (tracesPath == null || tracesPath.length() == 0) {
             return null;
@@ -8958,10 +8958,13 @@ public final class ActivityManagerService extends ActivityManagerNative implemen
      * @param logFile to include in the report, null if none
      * @param crashInfo giving an application stack trace, null if absent
      */
-    private void addErrorToDropBox(String eventType,
+    public void addErrorToDropBox(String eventType,
             ProcessRecord process, HistoryRecord activity, HistoryRecord parent,
             String subject, String report, File logFile,
             ApplicationErrorReport.CrashInfo crashInfo) {
+        // NOTE -- this must never acquire the ActivityManagerService lock,
+        // otherwise the watchdog may be prevented from resetting the system.
+
         String dropboxTag;
         if (process == null || process.pid == MY_PID) {
             dropboxTag = "system_server_" + eventType;