Add statsd.

It doesn't start yet by default. When you start it manually, it sets
itself up as a binder system service and starts a thread to read the
event log.

Test: Run statsd, observe output. also run stats_test
Change-Id: If435d6a80fef3c1d957aedb61699bf5e9aae7e56

Android.mk

@@ -270,6 +270,7 @@ LOCAL_SRC_FILES += \
 	core/java/android/os/IRecoverySystemProgressListener.aidl \
 	core/java/android/os/IRemoteCallback.aidl \
 	core/java/android/os/ISchedulingPolicyService.aidl \
+	core/java/android/os/IStatsManager.aidl \
 	core/java/android/os/IThermalEventListener.aidl \
 	core/java/android/os/IThermalService.aidl \
 	core/java/android/os/IUpdateLock.aidl \

cmds/statsd/Android.mk

@@ -0,0 +1,106 @@
+# Copyright (C) 2017 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+LOCAL_PATH:= $(call my-dir)
+
+# =========
+# statsd
+# =========
+
+include $(CLEAR_VARS)
+
+LOCAL_MODULE := statsd
+
+LOCAL_SRC_FILES := \
+    ../../core/java/android/os/IStatsManager.aidl \
+    src/StatsService.cpp \
+    src/LogEntryPrinter.cpp \
+    src/LogReader.cpp \
+    src/main.cpp
+
+LOCAL_CFLAGS += \
+    -Wall \
+    -Werror \
+    -Wno-missing-field-initializers \
+    -Wno-unused-variable \
+    -Wno-unused-function \
+    -Wno-unused-parameter
+
+ifeq (debug,)
+    LOCAL_CFLAGS += \
+            -g -O0
+else
+    # optimize for size (protobuf glop can get big)
+    LOCAL_CFLAGS += \
+            -Os
+endif
+
+LOCAL_AIDL_INCLUDES := $(LOCAL_PATH)/../../core/java
+LOCAL_C_INCLUDES += $(LOCAL_PATH)/src
+
+LOCAL_SHARED_LIBRARIES := \
+        libbase \
+        libbinder \
+        libcutils \
+        libincident \
+        liblog \
+        libselinux \
+        libutils
+
+LOCAL_MODULE_CLASS := EXECUTABLES
+
+#LOCAL_INIT_RC := statsd.rc
+
+include $(BUILD_EXECUTABLE)
+
+# ==============
+# statsd_test
+# ==============
+
+include $(CLEAR_VARS)
+
+LOCAL_MODULE := statsd_test
+LOCAL_COMPATIBILITY_SUITE := device-tests
+LOCAL_MODULE_TAGS := tests
+
+LOCAL_CFLAGS += \
+    -Wall \
+    -Werror \
+    -Wno-missing-field-initializers \
+    -Wno-unused-variable \
+    -Wno-unused-function \
+    -Wno-unused-parameter
+
+LOCAL_C_INCLUDES += $(LOCAL_PATH)/src
+
+LOCAL_SRC_FILES := \
+    ../../core/java/android/os/IStatsManager.aidl \
+    src/StatsService.cpp \
+    src/LogEntryPrinter.cpp \
+    src/LogReader.cpp \
+    tests/LogReader_test.cpp \
+
+LOCAL_STATIC_LIBRARIES := \
+    libgmock \
+
+LOCAL_SHARED_LIBRARIES := \
+    libbase \
+    libbinder \
+    libcutils \
+    liblog \
+    libselinux \
+    libutils
+
+include $(BUILD_NATIVE_TEST)
+

cmds/statsd/src/LogEntryPrinter.cpp

@@ -0,0 +1,61 @@
+/*
+ * Copyright (C) 2017 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <LogEntryPrinter.h>
+
+#include <log/event_tag_map.h>
+#include <log/logprint.h>
+#include <utils/Errors.h>
+
+using namespace android;
+
+LogEntryPrinter::LogEntryPrinter(int out)
+    :m_out(out)
+{
+    // Initialize the EventTagMap, which is how we know the names of the numeric event tags.
+    // If this fails, we can't print well, but something will print.
+    m_tags = android_openEventTagMap(NULL);
+
+    // Printing format
+    m_format = android_log_format_new();
+    android_log_setPrintFormat(m_format, FORMAT_THREADTIME);
+}
+
+LogEntryPrinter::~LogEntryPrinter()
+{
+    if (m_tags != NULL) {
+        android_closeEventTagMap(m_tags);
+    }
+    android_log_format_free(m_format);
+}
+
+void
+LogEntryPrinter::OnLogEvent(const log_msg& msg)
+{
+    status_t err;
+    AndroidLogEntry entry;
+    char buf[1024];
+
+    err = android_log_processBinaryLogBuffer(&(const_cast<log_msg*>(&msg)->entry_v1),
+                &entry, m_tags, buf, sizeof(buf));
+    if (err == NO_ERROR) {
+        android_log_printLogLine(m_format, m_out, &entry);
+    } else {
+        printf("log entry: %s\n", buf);
+        fflush(stdout);
+    }
+}
+

cmds/statsd/src/LogEntryPrinter.h

@@ -0,0 +1,54 @@
+/*
+ * Copyright (C) 2017 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LOG_ENTRY_PRINTER_H
+#define LOG_ENTRY_PRINTER_H
+
+#include "LogReader.h"
+
+#include <log/logprint.h>
+
+#include <stdio.h>
+
+/**
+ * Decodes the log entry and prints it to the supplied file descriptor.
+ */
+class LogEntryPrinter : public LogListener
+{
+public:
+    LogEntryPrinter(int out);
+    virtual ~LogEntryPrinter();
+
+    virtual void OnLogEvent(const log_msg& msg);
+
+private:
+    /**
+     * Where to write to.
+     */
+    int m_out;
+
+    /**
+     * Numeric to string tag name mapping.
+     */
+    EventTagMap* m_tags;
+
+    /**
+     * Pretty printing format.
+     */
+    AndroidLogFormat* m_format;
+};
+
+#endif // LOG_ENTRY_PRINTER_H

cmds/statsd/src/LogReader.cpp

@@ -0,0 +1,143 @@
+/*
+ * Copyright (C) 2017 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "LogReader.h"
+
+#include <log/log_read.h>
+
+#include <utils/Errors.h>
+
+#include <time.h>
+#include <unistd.h>
+
+using namespace android;
+using namespace std;
+
+#define SNOOZE_INITIAL_MS 100
+#define SNOOZE_MAX_MS (10 * 60 * 1000) // Ten minutes
+
+
+// ================================================================================
+LogListener::LogListener()
+{
+}
+
+LogListener::~LogListener()
+{
+}
+
+
+// ================================================================================
+LogReader::LogReader()
+{
+}
+
+LogReader::~LogReader()
+{
+}
+
+void
+LogReader::AddListener(const sp<LogListener>& listener)
+{
+    m_listeners.push_back(listener);
+}
+
+void
+LogReader::Run()
+{
+    int nextSnoozeMs = SNOOZE_INITIAL_MS;
+
+    // In an ideal world, this outer loop will only ever run one iteration, but it
+    // exists to handle crashes in logd.  The inner loop inside connect_and_read()
+    // reads from logd forever, but if that read fails, we fall out to the outer
+    // loop, do the backoff (resetting the backoff timeout if we successfully read
+    // something), and then try again.
+    while (true) {
+        // Connect and read
+        int lineCount = connect_and_read();
+
+        // Figure out how long to sleep.
+        if (lineCount > 0) {
+            // If we managed to read at least one line, reset the backoff
+            nextSnoozeMs = SNOOZE_INITIAL_MS;
+        } else {
+            // Otherwise, expontial backoff
+            nextSnoozeMs *= 1.5f;
+            if (nextSnoozeMs > 10 * 60 * 1000) {
+                // Don't wait for toooo long.
+                nextSnoozeMs = SNOOZE_MAX_MS;
+            }
+        }
+
+        // Sleep
+        timespec ts;
+        timespec rem;
+        ts.tv_sec = nextSnoozeMs / 1000;
+        ts.tv_nsec = (nextSnoozeMs % 1000) * 1000000L;
+        while (nanosleep(&ts, &rem) == -1) {
+            if (errno == EINTR) {
+                ts = rem;
+            }
+            // other errors are basically impossible
+        }
+    }
+}
+
+int
+LogReader::connect_and_read()
+{
+    int lineCount = 0;
+    status_t err;
+    logger_list* loggers;
+    logger* eventLogger;
+
+    // Prepare the logging context
+    loggers = android_logger_list_alloc(ANDROID_LOG_RDONLY,
+            /* don't stop after N lines */ 0,
+            /* no pid restriction */ 0);
+
+    // Open the buffer(s)
+    eventLogger = android_logger_open(loggers, LOG_ID_EVENTS);
+
+    // Read forever
+    if (eventLogger) {
+        while (true) {
+            log_msg msg;
+
+            // Read a message
+            err = android_logger_list_read(loggers, &msg);
+            if (err < 0) {
+                fprintf(stderr, "logcat read failure: %s\n", strerror(err));
+                break;
+            }
+
+            // Record that we read one (used above to know how to snooze).
+            lineCount++;
+
+            // Call the listeners
+            for (vector<sp<LogListener> >::iterator it = m_listeners.begin();
+                    it != m_listeners.end(); it++) {
+                (*it)->OnLogEvent(msg);
+            }
+        }
+    }
+
+    // Free the logger list and close the individual loggers
+    android_logger_list_free(loggers);
+
+    return lineCount;
+}
+

cmds/statsd/src/LogReader.h

@@ -0,0 +1,81 @@
+/*
+ * Copyright (C) 2017 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LOGREADER_H
+#define LOGREADER_H
+
+#include <log/log_read.h>
+
+#include <utils/RefBase.h>
+
+#include <vector>
+
+/**
+ * Callback for LogReader 
+ */
+class LogListener : public virtual android::RefBase
+{
+public:
+    LogListener();
+    virtual ~LogListener();
+
+    // TODO: Rather than using log_msg, which doesn't have any real internal structure
+    // here, we should pull this out into our own LogEntry class.
+    virtual void OnLogEvent(const log_msg& msg) = 0;
+};
+
+/**
+ * Class to read logs from logd.
+ */
+class LogReader : public virtual android::RefBase
+{
+public:
+    /**
+     * Construct the LogReader with a pointer back to the StatsService
+     */
+    LogReader();
+
+    /**
+     * Destructor.
+     */
+    virtual ~LogReader();
+
+    /**
+     * Add a LogListener class.
+     */
+    void AddListener(const android::sp<LogListener>& listener);
+
+   /**
+    * Run the main LogReader loop
+    */
+    void Run();
+
+private:
+    /**
+     * List of listeners to call back on when we do get an event.
+     */
+    std::vector<android::sp<LogListener> > m_listeners;
+
+    /**
+     * Connect to a single instance of logd, and read until there's a read error.
+     * Logd can crash, exit, be killed etc.
+     *
+     * Returns the number of lines that were read.
+     */
+    int connect_and_read();
+};
+
+#endif // LOGREADER_H

cmds/statsd/src/StatsService.cpp

@@ -0,0 +1,76 @@
+/*
+ * Copyright (C) 2016 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define LOG_TAG "statsd"
+
+#include "StatsService.h"
+
+#include <binder/IPCThreadState.h>
+#include <binder/IServiceManager.h>
+#include <cutils/log.h>
+#include <private/android_filesystem_config.h>
+#include <utils/Looper.h>
+
+#include <unistd.h>
+#include <stdio.h>
+
+using namespace android;
+
+// ================================================================================
+StatsService::StatsService(const sp<Looper>& handlerLooper)
+{
+    ALOGD("stats service constructed");
+}
+
+StatsService::~StatsService()
+{
+}
+
+status_t
+StatsService::dump(int fd, const Vector<String16>& args)
+{
+    FILE* out = fdopen(fd, "w");
+    if (out == NULL) {
+        return NO_MEMORY;  // the fd is already open
+    }
+
+    fprintf(out, "StatsService::dump:");
+    ALOGD("StatsService::dump:");
+    const int N = args.size();
+    for (int i=0; i<N; i++) {
+        fprintf(out, " %s", String8(args[i]).string());
+        ALOGD("   %s", String8(args[i]).string());
+    }
+    fprintf(out, "\n");
+
+    fclose(out);
+    return NO_ERROR;
+}
+
+Status
+StatsService::systemRunning()
+{
+    if (IPCThreadState::self()->getCallingUid() != AID_SYSTEM) {
+        return Status::fromExceptionCode(Status::EX_SECURITY,
+                "Only system uid can call systemRunning");
+    }
+
+    // When system_server is up and running, schedule the dropbox task to run.
+    ALOGD("StatsService::systemRunning");
+
+    return Status::ok();
+}
+

cmds/statsd/src/StatsService.h

@@ -0,0 +1,44 @@
+/*
+ * Copyright (C) 2016 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef STATS_SERVICE_H
+#define STATS_SERVICE_H
+
+#include <android/os/BnStatsManager.h>
+#include <utils/Looper.h>
+
+#include <deque>
+#include <mutex>
+
+using namespace android;
+using namespace android::base;
+using namespace android::binder;
+using namespace android::os;
+using namespace std;
+
+
+// ================================================================================
+class StatsService : public BnStatsManager {
+public:
+    StatsService(const sp<Looper>& handlerLooper);
+    virtual ~StatsService();
+
+    virtual status_t dump(int fd, const Vector<String16>& args);
+    virtual Status systemRunning();
+
+};
+
+#endif // STATS_SERVICE_H

cmds/statsd/src/main.cpp

@@ -0,0 +1,144 @@
+/*
+ * Copyright (C) 2016 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define LOG_TAG "statsd"
+
+#include "LogEntryPrinter.h"
+#include "LogReader.h"
+#include "StatsService.h"
+
+#include <binder/IInterface.h>
+#include <binder/IPCThreadState.h>
+#include <binder/IServiceManager.h>
+#include <binder/ProcessState.h>
+#include <binder/Status.h>
+#include <cutils/log.h>
+#include <utils/Looper.h>
+#include <utils/StrongPointer.h>
+
+#include <stdio.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+using namespace android;
+
+// ================================================================================
+/**
+ * Thread function data.
+ */
+struct log_reader_thread_data {
+    sp<StatsService> service;
+};
+
+/**
+ * Thread func for where the log reader runs.
+ */
+static void*
+log_reader_thread_func(void* cookie)
+{
+    log_reader_thread_data* data = static_cast<log_reader_thread_data*>(cookie);
+
+    sp<LogReader> reader = new LogReader();
+
+    // Put the printer one first, so it will print before the real ones.
+    if (true) {
+        reader->AddListener(new LogEntryPrinter(STDOUT_FILENO));
+    }
+
+    // TODO: Construct and add real LogListners here.
+
+    reader->Run();
+
+    ALOGW("statsd LogReader.Run() is not supposed to return.");
+
+    delete data;
+    return NULL;
+}
+
+/**
+ * Creates and starts the thread to own the LogReader.
+ */
+static status_t
+start_log_reader_thread(const sp<StatsService>& service)
+{
+    status_t err;
+    pthread_attr_t attr;
+    pthread_t thread;
+
+    // Thread data.
+    log_reader_thread_data* data = new log_reader_thread_data();
+    data->service = service;
+
+    // Create the thread
+    err = pthread_attr_init(&attr);
+    if (err != NO_ERROR) {
+        return err;
+    }
+    // TODO: Do we need to tweak thread priority?
+    err = pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
+    if (err != NO_ERROR) {
+        pthread_attr_destroy(&attr);
+        return err;
+    }
+    err = pthread_create(&thread, &attr, log_reader_thread_func, static_cast<void*>(data));
+    if (err != NO_ERROR) {
+        pthread_attr_destroy(&attr);
+        return err;
+    }
+    pthread_attr_destroy(&attr);
+
+    return NO_ERROR;
+}
+
+// ================================================================================
+int
+main(int /*argc*/, char** /*argv*/)
+{
+    status_t err;
+
+    // Set up the looper
+    sp<Looper> looper(Looper::prepare(0 /* opts */));
+
+    // Set up the binder
+    sp<ProcessState> ps(ProcessState::self());
+    ps->setThreadPoolMaxThreadCount(1); // everything is oneway, let it queue and save ram
+    ps->startThreadPool();
+    ps->giveThreadPoolName();
+    IPCThreadState::self()->disableBackgroundScheduling(true);
+
+    // Create the service
+    sp<StatsService> service = new StatsService(looper);
+    if (defaultServiceManager()->addService(String16("stats"), service) != 0) {
+        ALOGE("Failed to add service");
+        return -1;
+    }
+
+    // Start the log reader thread
+    err = start_log_reader_thread(service);
+    if (err != NO_ERROR) {
+        return 1;
+    }
+
+    // Loop forever -- the reports run on this thread in a handler, and the
+    // binder calls remain responsive in their pool of one thread.
+    while (true) {
+        looper->pollAll(-1 /* timeoutMillis */);
+    }
+    ALOGW("statsd escaped from its loop.");
+
+    return 1;
+}

cmds/statsd/statsd.rc

@@ -0,0 +1,16 @@
+# Copyright (C) 2017 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+service statsd /system/bin/statsd
+    class main

cmds/statsd/tests/LogReader_test.cpp

@@ -0,0 +1,24 @@
+// Copyright (C) 2017 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#define LOG_TAG "statsd_test"
+
+#include <gtest/gtest.h>
+
+#include <stdio.h>
+
+TEST(LogReaderTest, TestNothingAtAll) {
+    printf("yay!");
+}
+

core/java/android/os/IStatsManager.aidl

@@ -0,0 +1,28 @@
+/**
+ * Copyright (c) 2016, The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.os;
+
+/**
+  * Binder interface to communicate with the statistics collection service.
+  * {@hide}
+  */
+oneway interface IStatsManager {
+    /**
+     * Tell the incident daemon that the android system server is up and running.
+     */
+    void systemRunning();
+}