Fix bug #895. Most if not all non-Ogg streams send metadata as non-UTF8, typically

ISO-8859-1 is assumed as there is no real clarity wrt the spec. In most cases people
send ASCII so it's not an issue, but for some, the extended characters they send
can cause problems with XML processing.  As stats and YP require UTF8 we need to
translate them and block invalid cases.

For the moment, for non-Ogg streams only, we assume that the metadata needs converting
from ISO-8859-1. Ogg streams are UTF8 so no conversion needed. You can override the
default with a charset mount option.


svn path=/icecast/trunk/icecast/; revision=13615

src/cfgfile.c

@@ -133,6 +133,7 @@ static void config_clear_mount (mount_proxy *mount)
     xmlFree (mount->stream_genre);
     xmlFree (mount->bitrate);
     xmlFree (mount->type);
+    xmlFree (mount->charset);
     xmlFree (mount->cluster_password);
 
     xmlFree (mount->auth_type);
@@ -581,6 +582,10 @@ static void _parse_mount(xmlDocPtr doc, xmlNodePtr node,
             mount->max_listeners = atoi(tmp);
             if(tmp) xmlFree(tmp);
         }
+        else if (strcmp(node->name, "charset") == 0) {
+            mount->charset = (char *)xmlNodeListGetString(doc,
+                    node->xmlChildrenNode, 1);
+        }
         else if (strcmp(node->name, "mp3-metadata-interval") == 0) {
             tmp = (char *)xmlNodeListGetString(doc, node->xmlChildrenNode, 1);
             mount->mp3_meta_interval = atoi(tmp);

src/cfgfile.h

@@ -64,6 +64,7 @@ typedef struct _mount_proxy {
     unsigned int queue_size_limit;
     int hidden; /* Do we list this on the xsl pages */
     unsigned int source_timeout;  /* source timeout in seconds */
+    char *charset;  /* character set if not utf8 */
     int mp3_meta_interval; /* outgoing per-stream metadata interval */
 
     char *auth_type; /* Authentication type */

src/format.h

@@ -40,6 +40,7 @@ typedef struct _format_plugin_tag
     char *mount;
 
     const char *contenttype;
+    char *charset;
     uint64_t read_bytes;
     uint64_t sent_bytes;
 

src/format_mp3.c

@@ -184,7 +184,7 @@ static void filter_shoutcast_metadata (source_t *source, char *metadata, unsigne
             {
                 memcpy (p, metadata+13, len);
                 logging_playlist (source->mount, p, source->listeners);
-                stats_event (source->mount, "title", p);
+                stats_event_conv (source->mount, "title", p, source->format->charset);
                 yp_touch (source->mount);
                 free (p);
             }
@@ -197,10 +197,21 @@ static void format_mp3_apply_settings (client_t *client, format_plugin_t *format
 {
     mp3_state *source_mp3 = format->_state;
 
-    if (mount == NULL || mount->mp3_meta_interval < 0)
+    source_mp3->interval = -1;
+    free (format->charset);
+    format->charset = NULL;
+
+    if (mount)
+    {
+        if (mount->mp3_meta_interval > 0)
+            source_mp3->interval = mount->mp3_meta_interval;
+        if (mount->charset)
+            format->charset = strdup (mount->charset);
+    }
+    if (source_mp3->interval <= 0)
     {
         const char *metadata = httpp_getvar (client->parser, "icy-metaint");
-        source_mp3->interval = -1;
+        source_mp3->interval = ICY_METADATA_INTERVAL;
         if (metadata)
         {
             int interval = atoi (metadata);
@@ -208,9 +219,12 @@ static void format_mp3_apply_settings (client_t *client, format_plugin_t *format
                 source_mp3->interval = interval;
         }
     }
-    else
-        source_mp3->interval = mount->mp3_meta_interval;
-    DEBUG1 ("mp3 interval %d", source_mp3->interval);
+
+    if (format->charset == NULL)
+        format->charset = strdup ("ISO8859-1");
+
+    DEBUG1 ("sending metadata interval %d", source_mp3->interval);
+    DEBUG1 ("charset %s", format->charset);
 }
 
 
@@ -277,7 +291,7 @@ static void mp3_set_title (source_t *source)
 static int send_mp3_metadata (client_t *client, refbuf_t *associated)
 {
     int ret = 0;
-    unsigned char *metadata;
+    char *metadata;
     int meta_len;
     mp3_client_data *client_mp3 = client->format_data;
 
@@ -411,6 +425,7 @@ static void format_mp3_free_plugin(format_plugin_t *self)
     thread_mutex_destroy (&state->url_lock);
     free (state->url_artist);
     free (state->url_title);
+    free (self->charset);
     refbuf_release (state->metadata);
     refbuf_release (state->read_data);
     free(state);
@@ -509,7 +524,7 @@ static refbuf_t *mp3_get_filter_meta (source_t *source)
 
     refbuf = source_mp3->read_data;
     source_mp3->read_data = NULL;
-    src = refbuf->data;
+    src = (unsigned char *)refbuf->data;
 
     if (source_mp3->update_metadata)
     {

src/source.c

@@ -994,7 +994,7 @@ static void source_apply_mount (source_t *source, mount_proxy *mountinfo)
             str = "Unspecified name";
         } while (0);
     }
-    stats_event (source->mount, "server_name", str);
+    stats_event_conv (source->mount, "server_name", str, source->format->charset);
 
     /* stream description */
     if (mountinfo && mountinfo->stream_description)
@@ -1011,7 +1011,7 @@ static void source_apply_mount (source_t *source, mount_proxy *mountinfo)
             str = "Unspecified description";
         } while (0);
     }
-    stats_event (source->mount, "server_description", str);
+    stats_event_conv (source->mount, "server_description", str, source->format->charset);
 
     /* stream URL */
     if (mountinfo && mountinfo->stream_url)
@@ -1044,7 +1044,7 @@ static void source_apply_mount (source_t *source, mount_proxy *mountinfo)
             str = "various";
         } while (0);
     }
-    stats_event (source->mount, "genre", str);
+    stats_event_conv (source->mount, "genre", str, source->format->charset);
 
     /* stream bitrate */
     if (mountinfo && mountinfo->bitrate)

src/stats.c

@@ -201,11 +201,55 @@ void stats_event(const char *source, const char *name, const char *value)
 {
     stats_event_t *event;
 
+    if (value && xmlCheckUTF8 ((unsigned char *)value) == 0)
+    {
+        WARN2 ("seen non-UTF8 data, probably incorrect metadata (%s, %s)", name, value);
+        return;
+    }
     event = build_event (source, name, value);
     if (event)
         queue_global_event (event);
 }
 
+
+/* wrapper for stats_event, this takes a charset to convert from */
+void stats_event_conv(const char *mount, const char *name, const char *value, const char *charset)
+{
+    const char *metadata = value;
+    xmlBufferPtr conv = xmlBufferCreate ();
+
+    if (charset)
+    {
+        xmlCharEncodingHandlerPtr handle = xmlFindCharEncodingHandler (charset);
+
+        if (handle)
+        {
+            xmlBufferPtr raw = xmlBufferCreate ();
+            xmlBufferAdd (raw, (const xmlChar *)value, strlen (value));
+            if (xmlCharEncInFunc (handle, conv, raw) > 0)
+                metadata = (char *)xmlBufferContent (conv);
+            xmlBufferFree (raw);
+            xmlCharEncCloseFunc (handle);
+        }
+        else
+            WARN1 ("No charset found for \"%s\"", charset);
+    }
+
+    stats_event (mount, name, metadata);
+
+    /* special case for title updates, log converted title */
+    if (mount && strcmp (name, "title") == 0)
+    {
+        char *s = stats_get_value ((char*)mount, "listeners");
+        int listeners = 0;
+        if (s)
+            listeners = atoi (s);
+        free (s);
+        logging_playlist (mount, metadata, listeners);
+    }
+    xmlBufferFree (conv);
+}
+
 /* make stat hidden (non-zero). name can be NULL if it applies to a whole
  * source stats tree. */
 void stats_event_hidden (const char *source, const char *name, int hidden)

src/stats.h

@@ -80,6 +80,8 @@ void stats_get_streamlist (char *buffer, size_t remaining);
 void stats_clear_virtual_mounts (void);
 
 void stats_event(const char *source, const char *name, const char *value);
+void stats_event_conv(const char *mount, const char *name,
+        const char *value, const char *charset);
 void stats_event_args(const char *source, char *name, char *format, ...);
 void stats_event_inc(const char *source, const char *name);
 void stats_event_add(const char *source, const char *name, unsigned long value);