Merge branch 'feature-source-health' into devel

admin/listmounts.xsl

@@ -16,10 +16,24 @@
                         <section class="box">
                             <h3 class="box_title">Mountpoint <code><xsl:value-of select="@mount" /></code></h3>
                             <!-- Mount nav -->
-                            <xsl:call-template name="mountnav" />
+                            <div class="side-by-side">
+                                <div class="trafficlight colour-{health/text()}">&#160;</div>
+                                <xsl:call-template name="mountnav" />
+                            </div>
                             <xsl:call-template name="player" />
                             <p><xsl:value-of select="listeners" /> Listener(s)</p>
 
+                            <xsl:if test="maintenance/*">
+                                <h4>Maintenance</h4>
+                                <ul class="maintenance-container">
+                                    <xsl:for-each select="maintenance/*">
+                                        <li class="maintenance-level-{@maintenance-level}">
+                                            <p><xsl:value-of select="text()" /></p>
+                                        </li>
+                                    </xsl:for-each>
+                                </ul>
+                            </xsl:if>
+
                             <!-- Mount Authentication -->
                             <xsl:if test="authentication">
                                 <h4>Mount Authentication</h4>

src/admin.c

@@ -167,12 +167,6 @@
 #define DEFAULT_HTML_REQUEST                ""
 #define BUILDM3U_RAW_REQUEST                "buildm3u"
 
-typedef enum {
-    ADMIN_DASHBOARD_STATUS_OK = 0,
-    ADMIN_DASHBOARD_STATUS_WARNING = 1,
-    ADMIN_DASHBOARD_STATUS_ERROR = 2
-} admin_dashboard_status_t;
-
 typedef struct {
     const char *prefix;
     size_t length;
@@ -476,6 +470,33 @@ xmlNodePtr admin_build_rootnode(xmlDocPtr doc, const char *name)
     return rootnode;
 }
 
+static inline void admin_build_sourcelist__add_flag(xmlNodePtr parent, source_flags_t flags, source_flags_t flag, bool invert, const char *name, const char *description)
+{
+    xmlNodePtr node;
+    source_flags_t testflags = SOURCE_FLAGS_GOOD|flag;
+
+    if (invert ? (flags & flag) : !(flags & flag))
+        return;
+
+    node = xmlNewTextChild(parent, NULL, XMLSTR("flag"), XMLSTR(description));
+    xmlSetProp(node, XMLSTR("value"), XMLSTR(name));
+
+    if (invert)
+        testflags &= ~flag;
+
+    switch (source_get_health_by_flags(testflags)) {
+        case HEALTH_OK:
+            xmlSetProp(node, XMLSTR("maintenance-level"), XMLSTR("info"));
+            break;
+        case HEALTH_WARNING:
+            xmlSetProp(node, XMLSTR("maintenance-level"), XMLSTR("warning"));
+            break;
+        case HEALTH_ERROR:
+            xmlSetProp(node, XMLSTR("maintenance-level"), XMLSTR("error"));
+            break;
+    }
+}
+
 /* build an XML doc containing information about currently running sources.
  * If a mountpoint is passed then that source will not be added to the XML
  * doc even if the source is running */
@@ -536,6 +557,9 @@ xmlDocPtr admin_build_sourcelist(const char *mount, client_t *client, admin_form
             config_release_config();
 
             if (source->running) {
+                const source_flags_t flags = source->flags;
+                xmlNodePtr maintenancenode;
+
                 if (source->client) {
                     snprintf(buf, sizeof(buf), "%lu",
                         (unsigned long)(now - source->con->con_time));
@@ -547,6 +571,25 @@ xmlDocPtr admin_build_sourcelist(const char *mount, client_t *client, admin_form
                 }
                 xmlNewTextChild(srcnode, NULL, XMLSTR("content-type"),
                     XMLSTR(source->format->contenttype));
+
+                switch (source_get_health(source)) {
+                    case HEALTH_OK:
+                        xmlNewTextChild(srcnode, NULL, XMLSTR("health"), XMLSTR("green"));
+                        break;
+                    case HEALTH_WARNING:
+                        xmlNewTextChild(srcnode, NULL, XMLSTR("health"), XMLSTR("yellow"));
+                        break;
+                    case HEALTH_ERROR:
+                        xmlNewTextChild(srcnode, NULL, XMLSTR("health"), XMLSTR("red"));
+                        break;
+                }
+                maintenancenode = xmlNewChild(srcnode, NULL, XMLSTR("maintenance"), NULL);
+                xmlSetProp(maintenancenode, XMLSTR("comment"), XMLSTR("This is an experimental node. Do not use!"));
+
+                admin_build_sourcelist__add_flag(maintenancenode, flags, SOURCE_FLAG_GOT_DATA, true, "no-got-data", "No data has yet been received.");
+                admin_build_sourcelist__add_flag(maintenancenode, flags, SOURCE_FLAG_FORMAT_GENERIC, false, "format-generic", "Legacy or unsupported streaming format is used.");
+                admin_build_sourcelist__add_flag(maintenancenode, flags, SOURCE_FLAG_LEGACY_METADATA, false, "legacy-metadata", "Legacy metadata on non-legacy source received. This is likely a bug in the source client.");
+                admin_build_sourcelist__add_flag(maintenancenode, flags, SOURCE_FLAG_AGED, true, "no-aged", "The stream did not mature yet.");
             }
 
             snprintf(buf, sizeof(buf), "%"PRIu64, source->dumpfile_written);
@@ -1212,6 +1255,7 @@ static void command_metadata(client_t *client,
     if (source->parser && source->parser->req_type == httpp_req_put) {
         ICECAST_LOG_ERROR("Got legacy SOURCE-style metadata update command on "
             "source connected with PUT at mountpoint %H", source->mount);
+        source_set_flags(source, SOURCE_FLAG_LEGACY_METADATA);
     }
 
     COMMAND_REQUIRE(client, "mode", action);
@@ -1256,6 +1300,7 @@ static void command_metadata(client_t *client,
     } else {
         ICECAST_LOG_ERROR("Got legacy shoutcast-style metadata update command "
             "on source that does not accept it at mountpoint %H", source->mount);
+        source_set_flags(source, SOURCE_FLAG_LEGACY_METADATA);
 
         admin_send_response_simple(client, source, response, "Mountpoint will not accept URL updates", 1);
         return;
@@ -1629,19 +1674,10 @@ static void __reportxml_add_maintenance(reportxml_node_t *parent, reportxml_data
     refobject_unref(incident);
 }
 
-static admin_dashboard_status_t command_dashboard__atbest(admin_dashboard_status_t a, admin_dashboard_status_t b)
-{
-    if (a > b) {
-        return a;
-    } else {
-        return b;
-    }
-}
-
 #if HAVE_GETRLIMIT && HAVE_SYS_RESOURCE_H
-static admin_dashboard_status_t command_dashboard__getrlimit(ice_config_t *config, reportxml_node_t *parent, reportxml_database_t *db)
+static health_t command_dashboard__getrlimit(ice_config_t *config, reportxml_node_t *parent, reportxml_database_t *db)
 {
-    admin_dashboard_status_t ret = ADMIN_DASHBOARD_STATUS_OK;
+    health_t ret = HEALTH_OK;
     struct rlimit limit;
 
     if (getrlimit(RLIMIT_NOFILE, &limit) == 0) {
@@ -1651,7 +1687,7 @@ static admin_dashboard_status_t command_dashboard__getrlimit(ice_config_t *confi
                 // We assume that we need one FD per client, at max three per source (e.g. for auth), and at max 24 additional for logfiles and similar.
                 // This is just an estimation.
                 __reportxml_add_maintenance(parent, db, "a93a842a-9664-43a9-b707-7f358066fe2b", "error", "Global client, and source limit is bigger than suitable for current open file limit.", NULL);
-                ret = command_dashboard__atbest(ret, ADMIN_DASHBOARD_STATUS_ERROR);
+                ret = health_atbest(ret, HEALTH_ERROR);
             }
         }
     }
@@ -1674,7 +1710,7 @@ static void command_dashboard           (client_t *client, source_t *source, adm
     reportxml_t *report = client_get_reportxml("0aa76ea1-bf42-49d1-887e-ca95fb307dc4", NULL, NULL);
     reportxml_node_t *reportnode = reportxml_get_node_by_type(report, REPORTXML_NODE_TYPE_REPORT, 0);
     reportxml_node_t *incident = reportxml_get_node_by_type(report, REPORTXML_NODE_TYPE_INCIDENT, 0);
-    admin_dashboard_status_t status = ADMIN_DASHBOARD_STATUS_OK;
+    health_t health = HEALTH_OK;
     reportxml_node_t *resource;
     reportxml_node_t *node;
     bool has_sources;
@@ -1713,13 +1749,13 @@ static void command_dashboard           (client_t *client, source_t *source, adm
     refobject_unref(node);
 
     if (config->config_problems || has_too_many_clients || !inet6_enabled) {
-        status = command_dashboard__atbest(status, ADMIN_DASHBOARD_STATUS_ERROR);
+        health = health_atbest(health, HEALTH_ERROR);
     } else if (!has_sources || has_many_clients) {
-        status = command_dashboard__atbest(status, ADMIN_DASHBOARD_STATUS_WARNING);
+        health = health_atbest(health, HEALTH_WARNING);
     }
 
 #ifdef DEVEL_LOGGING
-    status = command_dashboard__atbest(status, ADMIN_DASHBOARD_STATUS_WARNING);
+    health = health_atbest(health, HEALTH_WARNING);
     __reportxml_add_maintenance(reportnode, config->reportxml_db, "c704804e-d3b9-4544-898b-d477078135de", "warning", "Developer logging is active. This mode is not for production.", NULL);
 #endif
 
@@ -1760,20 +1796,13 @@ static void command_dashboard           (client_t *client, source_t *source, adm
     }
 
 #if HAVE_GETRLIMIT && HAVE_SYS_RESOURCE_H
-    status = command_dashboard__atbest(status, command_dashboard__getrlimit(config, reportnode, config->reportxml_db));
+    if (true) {
+        health_t limits = command_dashboard__getrlimit(config, reportnode, config->reportxml_db);
+        health = health_atbest(health, limits);
+    }
 #endif
 
-    switch (status) {
+    reportxml_helper_add_value_health(resource, "status", health);
-        case ADMIN_DASHBOARD_STATUS_OK:
-            reportxml_helper_add_value_enum(resource, "status", "green");
-        break;
-        case ADMIN_DASHBOARD_STATUS_WARNING:
-            reportxml_helper_add_value_enum(resource, "status", "yellow");
-        break;
-        case ADMIN_DASHBOARD_STATUS_ERROR:
-            reportxml_helper_add_value_enum(resource, "status", "red");
-        break;
-    }
 
     reportxml_node_add_child(incident, resource);
     refobject_unref(resource);

src/connection.c

@@ -952,6 +952,9 @@ int connection_complete_source(source_t *source, int response)
             format_type = FORMAT_TYPE_GENERIC;
         }
 
+        if (format_type == FORMAT_TYPE_GENERIC)
+           source_set_flags(source, SOURCE_FLAG_FORMAT_GENERIC);
+
         if (format_get_plugin (format_type, source) < 0) {
             global_unlock();
             config_release_config();

src/icecasttypes.h

@@ -22,6 +22,14 @@
 #define XMLNS_LEGACY_STATS      "http://icecast.org/specs/legacystats-0.0.1"
 #define XMLNS_LEGACY_RESPONSE   "http://icecast.org/specs/legacyresponse-0.0.1"
 
+typedef enum {
+    HEALTH_OK = 0,
+    HEALTH_WARNING = 1,
+    HEALTH_ERROR = 2
+} health_t;
+
+#define health_atbest(a, b) (((a) > (b)) ? (a) : (b))
+
 /* ---[ client.[ch] ]--- */
 
 typedef struct _client_tag client_t;

src/reportxml_helper.c

@@ -43,6 +43,21 @@ void reportxml_helper_add_value_int(reportxml_node_t *parent, const char *member
     reportxml_helper_add_value(parent, "int", member, buf);
 }
 
+void reportxml_helper_add_value_health(reportxml_node_t *parent, const char *member, health_t val)
+{
+    switch (val) {
+        case HEALTH_OK:
+            reportxml_helper_add_value_enum(parent, member, "green");
+        break;
+        case HEALTH_WARNING:
+            reportxml_helper_add_value_enum(parent, member, "yellow");
+        break;
+        case HEALTH_ERROR:
+            reportxml_helper_add_value_enum(parent, member, "red");
+        break;
+    }
+}
+
 void reportxml_helper_add_text(reportxml_node_t *parent, const char *definition, const char *text)
 {
     reportxml_node_t *textnode = reportxml_node_new(REPORTXML_NODE_TYPE_TEXT, NULL, definition, NULL);

src/reportxml_helper.h

@@ -20,6 +20,7 @@ void reportxml_helper_add_value(reportxml_node_t *parent, const char *type, cons
 #define reportxml_helper_add_value_boolean(parent,member,value) reportxml_helper_add_value((parent), "boolean", (member), (value) ? "true" : "false")
 #define reportxml_helper_add_value_flag(parent,member,value) reportxml_helper_add_value((parent), "flag", (member), (value) ? "true" : "false")
 void reportxml_helper_add_value_int(reportxml_node_t *parent, const char *member, long long int val);
+void reportxml_helper_add_value_health(reportxml_node_t *parent, const char *member, health_t val);
 
 void reportxml_helper_add_text(reportxml_node_t *parent, const char *definition, const char *text);
 

src/source.c

@@ -290,6 +290,8 @@ void source_clear_source (source_t *source)
     source->hidden = 0;
     source->shoutcast_compat = 0;
     source->last_stats_update = 0;
+    source->create_time = 0;
+    source->flags = 0;
     util_dict_free(source->audio_info);
     source->audio_info = NULL;
 
@@ -500,12 +502,18 @@ static refbuf_t *get_next_buffer (source_t *source)
         }
 
         if (current >= (source->last_stats_update + 5)) {
+            time_t age = current - source->create_time;
 
             stats_event_args(source->mount, "total_bytes_read", "%"PRIu64, source->format->read_bytes);
             stats_event_args(source->mount, "total_bytes_sent", "%"PRIu64, source->format->sent_bytes);
             if (source->dumpfile) {
                 stats_event_args(source->mount, "dumpfile_written", "%"PRIu64, source->dumpfile_written);
             }
+
+            if (age > 30) { /* TODO: Should this be configurable? */
+                source_set_flags(source, SOURCE_FLAG_AGED);
+            }
+
             source->last_stats_update = current;
         }
         if (fds < 0) {
@@ -537,6 +545,9 @@ static refbuf_t *get_next_buffer (source_t *source)
             break;
     }
 
+    if (refbuf)
+        source_set_flags(source, SOURCE_FLAG_GOT_DATA);
+
     return refbuf;
 }
 
@@ -637,6 +648,7 @@ static void source_init (source_t *source)
 {
     char listenurl[512];
     const char *str;
+    time_t now;
 
     str = httpp_getvar(source->parser, "ice-audio-info");
     source->audio_info = util_dict_new();
@@ -664,8 +676,11 @@ static void source_init (source_t *source)
     stats_event_time (source->mount, "stream_start");
     stats_event_time_iso8601 (source->mount, "stream_start_iso8601");
 
+    now = time(NULL);
+    source->last_read = now;
+    source->create_time = now;
+
     ICECAST_LOG_DEBUG("Source creation complete");
-    source->last_read = time (NULL);
     source->prev_listeners = -1;
     source->running = 1;
 
@@ -1013,6 +1028,8 @@ static void source_apply_mount (ice_config_t *config, source_t *source, mount_pr
     avl_tree_rlock (source->client_tree);
     stats_event_args (source->mount, "listener_peak", "%lu", source->peak_listeners);
 
+    source->flags &= ~SOURCE_FLAGS_CLEARABLE;
+
     if (mountinfo)
     {
         source->max_listeners = mountinfo->max_listeners;
@@ -1515,3 +1532,39 @@ void source_kill_dumpfile(source_t *source)
     stats_event(source->mount, "dumpfile_written", NULL);
     stats_event(source->mount, "dumpfile_start", NULL);
 }
+
+health_t source_get_health(source_t *source)
+{
+    const source_flags_t flags = source->flags;
+    return source_get_health_by_flags(flags);
+}
+
+health_t source_get_health_by_flags(source_flags_t flags)
+{
+    health_t health = HEALTH_OK;
+
+    if (!(flags & SOURCE_FLAG_GOT_DATA))
+        health = health_atbest(health, HEALTH_ERROR);
+
+    if (flags & SOURCE_FLAG_FORMAT_GENERIC)
+        health = health_atbest(health, HEALTH_WARNING);
+
+    if (flags & SOURCE_FLAG_LEGACY_METADATA)
+        health = health_atbest(health, HEALTH_ERROR);
+
+    if (!(flags & SOURCE_FLAG_AGED))
+        health = health_atbest(health, HEALTH_WARNING);
+
+    return health;
+}
+
+void source_set_flags(source_t *source, source_flags_t flags)
+{
+    /* check if we need to do anything at all */
+    if ((source->flags & flags) == flags)
+        return;
+
+    thread_mutex_lock(&source->lock);
+    source->flags |= flags;
+    thread_mutex_unlock(&source->lock);
+}

src/source.h

@@ -15,6 +15,7 @@
 #define __SOURCE_H__
 
 #include <stdio.h>
+#include <stdint.h>
 #include <stdbool.h>
 
 #include "common/thread/thread.h"
@@ -27,12 +28,23 @@
 #include "format.h"
 #include "playlist.h"
 
+typedef uint_least32_t source_flags_t;
+
+#define SOURCE_FLAG_GOT_DATA        ((source_flags_t)0x00000001U)
+#define SOURCE_FLAG_FORMAT_GENERIC  ((source_flags_t)0x00000002U)
+#define SOURCE_FLAG_LEGACY_METADATA ((source_flags_t)0x00000004U)
+#define SOURCE_FLAG_AGED            ((source_flags_t)0x00000008U)
+
+#define SOURCE_FLAGS_CLEARABLE      (SOURCE_FLAG_LEGACY_METADATA)
+#define SOURCE_FLAGS_GOOD           (SOURCE_FLAG_GOT_DATA|SOURCE_FLAG_AGED)
+
 struct source_tag {
     mutex_t lock;
     client_t *client;
     connection_t *con;
     http_parser_t *parser;
     time_t last_stats_update;
+    time_t create_time;
     
     char *mount; // TODO: Should we at some point migrate away from this to only use identifier?
     mount_identifier_t *identifier;
@@ -43,6 +55,7 @@ struct source_tag {
     /* set to zero to request the source to shutdown without causing a global
      * shutdown */
     int running;
+    source_flags_t flags;
 
     struct _format_plugin_tag *format;
 
@@ -114,6 +127,9 @@ void source_recheck_mounts (int update_all);
 /* Writes a buffer of raw data to a dumpfile. returns true if the write was successful (and complete). */
 bool source_write_dumpfile(source_t *source, const void *buffer, size_t len);
 void source_kill_dumpfile(source_t *source);
+health_t source_get_health(source_t *source);
+health_t source_get_health_by_flags(source_flags_t flags);
+void source_set_flags(source_t *source, source_flags_t flags);
 
 extern mutex_t move_clients_mutex;