Merge branch 'elinks-0.12' into elinks-0.13

Conflicts:
	src/bookmarks/backend/default.c
	src/bookmarks/bookmarks.c
	src/session/session.c
	src/terminal/event.c
	src/viewer/text/search.c

Makefile.config.in

@@ -87,6 +87,8 @@ SEE_CFLAGS = @SEE_CFLAGS@
 SPARSE = @SPARSE@
 SPIDERMONKEY_CFLAGS = @SPIDERMONKEY_CFLAGS@
 SPIDERMONKEY_LIBS = @SPIDERMONKEY_LIBS@
+TRE_CFLAGS = @TRE_CFLAGS@
+TRE_LIBS = @TRE_LIBS@
 VERSION = @VERSION@
 XMLTO = @XMLTO@
 X_CFLAGS = @X_CFLAGS@

NEWS

@@ -75,16 +75,26 @@ ELinks 0.12pre2.GIT now:
 To be released as 0.12pre3, 0.12rc1, or even 0.12.0.  This branch also
 includes the changes listed under ``ELinks 0.11.5.GIT'' below.
 
+Incompatibilities:
+
+* bug 1060: Regexp searching now requires the TRE library.
+
+Other changes:
+
 * critical: Fix assertion failure if IMG/@usemap refers to a different
   file.
 * Preserve newlines in hidden input fields, and submit them as CRLF.
   Previously, they could turn into spaces or disappear entirely.
 * Perl scripts can use modules that dynamically load C libraries, like
   XML::LibXML::SAX does.
+* bug 153: Preserve Unicode characters in XBEL bookmark files.
+  However, Unicode in URIs (really IRIs) does not work reliably yet;
+  this is being tracked as bug 1066.
 * bug 885: Convert xterm titles to ISO-8859-1 by default, but add an
   option to disable this.  When removing control characters from a
   title, note the charset.  Don't truncate titles to the width of the
   terminal.
+* bug 1061: Correctly truncate UTF-8 titles in the tab bar.
 * enhancement: Updated ISO 8859-7, ISO 8859-16, KOI8-R, and MacRoman.
 
 ELinks 0.12pre2:

configure.in

@@ -250,12 +250,6 @@ EL_CHECK_CODE([variadic macros], HAVE_VARIADIC_MACROS,
 		 #define a(b,c...) printf(b,##c)],
                 [a("foo");a("%s","bar");a("%s%s","baz","quux");])
 
-# ===================================================================
-# Check for POSIX <regex.h>
-# ===================================================================
-
-EL_CHECK_SYS_TYPE(regex_t, HAVE_REGEX_H, [#include <regex.h>])
-
 # ===================================================================
 # Checks for library functions.
 # ===================================================================
@@ -909,6 +903,48 @@ else
 	AC_SUBST(LUA_CFLAGS)
 fi
 
+# ===================================================================
+# Check for TRE library
+# ===================================================================
+
+AC_MSG_CHECKING([[for TRE in pkg-config]])
+if pkg-config tre; then
+	TRE_CFLAGS=`pkg-config --cflags tre`
+	TRE_LIBS=`pkg-config --libs tre`
+	AC_MSG_RESULT([[yes]])
+else
+	# <http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=513055>
+	# "libtre-dev: /usr/lib/pkgconfig/tre.pc missing"
+	# so we look for the library even if pkg-config doesn't know about it.
+	TRE_CFLAGS=
+	TRE_LIBS=-ltre
+	AC_MSG_RESULT([[no, but let's try defaults]])
+fi
+
+AC_MSG_CHECKING([[for TRE header and library]])
+EL_SAVE_FLAGS
+CFLAGS="$TRE_CFLAGS $CFLAGS"
+LIBS="$TRE_LIBS $LIBS"
+AC_TRY_LINK([#include <tre/regex.h>],
+	    [regex_t re;
+	     regmatch_t match[1];
+	     regwcomp(&re, L"zap", REG_ICASE);
+	     regwexec(&re, L"ELIZAPROGRAM", 1, match, 0);],
+	    [AC_MSG_RESULT([[yes]])
+	     AC_DEFINE([HAVE_TRE_REGEX_H], [1],
+		       [Define to 1 if you have the <tre/regex.h> header file.])
+	     # TRE_CFLAGS will be used only where needed.
+	     # TRE_LIBS will be kept in LIBS and used everywhere.
+	     EL_RESTORE_FLAGS
+	     LIBS="$TRE_LIBS $LIBS"],
+	    [AC_MSG_RESULT([[no]])
+	     TRE_CFLAGS=
+	     TRE_LIBS=
+	     EL_RESTORE_FLAGS])
+AC_SUBST(TRE_CFLAGS)
+AC_SUBST(TRE_LIBS)
+
+AC_CHECK_SIZEOF([wchar_t], [4], [[#include <wchar.h>]])
 
 # ===================================================================
 # Check for Ruby, optional even if installed.

doc/events.txt

@@ -85,8 +85,8 @@ Triggered When:
 Arguments:
 
  struct bookmark *bookmark
- unsigned char *new_title
+ unsigned char *new_title	/* UTF-8 */
- unsigned char *new_url
+ unsigned char *new_url		/* UTF-8 */
 
 Description:
 

doc/installation.txt

@@ -52,6 +52,8 @@ OpenSSL, GNU TLS, or nss_compat_ossl \
 			|For handling secure HTTP browsing.
 GPM			|'General Purpose Mouse' for mouse support.
 expat			|'XML Parser Toolkit' needed for XBEL support.
+http://laurikari.net/tre/[TRE]						     \
+			|For regexp searching.  Version 0.7.5 works.
 libsmbclient		|Library needed for smb:// protocol support.
 rxvt-unicode		|For terminal emulator which supports 88 colors.
 xterm with 256 colors	|Program atleast patch level  179 or rxvt program   \

src/bookmarks/backend/default.c

@@ -4,6 +4,7 @@
 #include "config.h"
 #endif
 
+#include <errno.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -33,6 +34,7 @@ read_bookmarks_default(FILE *f)
 	unsigned char in_buffer[INBUF_SIZE]; /* read buffer */
 	struct bookmark *last_bm = NULL;
 	int last_depth = 0;
+	const int file_cp = get_cp_index("System");
 
 	/* TODO: Ignore lines with bad chars in title or url (?). -- Zas */
 	while (fgets(in_buffer, INBUF_SIZE, f)) {
@@ -101,7 +103,8 @@ read_bookmarks_default(FILE *f)
 					root = last_bm->root;
 				}
 			}
-			last_bm = add_bookmark(root, 1, title, url);
+			last_bm = add_bookmark_cp(root, 1, file_cp,
+						  title, url);
 			last_depth = depth;
 
 			if (!*url && title[0] == '-' && title[1] == '\0') {
@@ -125,27 +128,65 @@ read_bookmarks_default(FILE *f)
 #undef INBUF_SIZE
 }
 
+struct write_bookmarks_default
+{
+	struct secure_save_info *ssi;
+	int save_folder_state;
+	int codepage;
+	struct conv_table *conv_table;
+};
+
+static void
+write_bookmarks_default_inner(const struct write_bookmarks_default *out,
+			      LIST_OF(struct bookmark) *bookmarks_list)
+{
+	struct bookmark *bm;
+
+	foreach (bm, *bookmarks_list) {
+		unsigned char *title, *url;
+
+		title = convert_string(out->conv_table, bm->title,
+				       strlen(bm->title), out->codepage,
+				       CSM_NONE, NULL, NULL, NULL);
+		url = convert_string(out->conv_table, bm->url,
+				     strlen(bm->url), out->codepage,
+				     CSM_NONE, NULL, NULL, NULL);
+		secure_fprintf(out->ssi, "%s\t%s\t%d\t",
+			       empty_string_or_(title), empty_string_or_(url),
+			       bm->box_item->depth);
+		if (bm->box_item->type == BI_FOLDER) {
+			secure_fputc(out->ssi, 'F');
+			if (out->save_folder_state && bm->box_item->expanded)
+				secure_fputc(out->ssi, 'E');
+		}
+		secure_fputc(out->ssi, '\n');
+
+		if (!title || !url) {
+			secsave_errno = SS_ERR_OTHER;
+			out->ssi->err = ENOMEM;
+		}
+		mem_free_if(title);
+		mem_free_if(url);
+		if (out->ssi->err) break;
+
+		if (!list_empty(bm->child))
+			write_bookmarks_default_inner(out, &bm->child);
+	}
+}
+
 /* Saves the bookmarks to file */
 static void
 write_bookmarks_default(struct secure_save_info *ssi,
 		        LIST_OF(struct bookmark) *bookmarks_list)
 {
-	int folder_state = get_opt_bool("bookmarks.folder_state", NULL);
+	struct write_bookmarks_default out;
-	struct bookmark *bm;
 
-	foreach (bm, *bookmarks_list) {
+	out.ssi = ssi;
-		secure_fprintf(ssi, "%s\t%s\t%d\t", bm->title, bm->url, bm->box_item->depth);
+	out.save_folder_state = get_opt_bool("bookmarks.folder_state", NULL);
-		if (bm->box_item->type == BI_FOLDER) {
+	out.codepage = get_cp_index("System");
-			secure_fputc(ssi, 'F');
+	out.conv_table = get_translation_table(get_cp_index("UTF-8"),
-			if (folder_state && bm->box_item->expanded)
+					       out.codepage);
-				secure_fputc(ssi, 'E');
+	write_bookmarks_default_inner(&out, bookmarks_list);
-		}
-		secure_fputc(ssi, '\n');
-		if (ssi->err) break;
-
-		if (!list_empty(bm->child))
-			write_bookmarks_default(ssi, &bm->child);
-	}
 }
 
 static unsigned char *

src/bookmarks/backend/xbel.c

@@ -11,6 +11,7 @@
 #endif /* HAVE_CONFIG_H */
 
 #include <ctype.h>
+#include <errno.h>
 #include <expat.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -38,6 +39,7 @@ struct attributes {
 	LIST_HEAD(struct attributes);
 
 	unsigned char *name;
+	unsigned char *value;
 };
 
 /* Prototypes */
@@ -53,9 +55,14 @@ static unsigned char *get_attribute_value(struct tree_node *node,
 					  unsigned char *name);
 
 
+struct read_bookmarks_xbel {
+	int utf8_cp;
+};
+
 static void read_bookmarks_xbel(FILE *f);
 static unsigned char * filename_bookmarks_xbel(int writing);
-static int xbeltree_to_bookmarks_list(struct tree_node *root,
+static int xbeltree_to_bookmarks_list(const struct read_bookmarks_xbel *preload,
+				      struct tree_node *root,
 				      struct bookmark *current_parent);
 static void write_bookmarks_list(struct secure_save_info *ssi,
 				 LIST_OF(struct bookmark) *bookmarks_list,
@@ -89,6 +96,7 @@ read_bookmarks_xbel(FILE *f)
 	XML_Parser p;
 	int done = 0;
 	int err = 0;
+	struct read_bookmarks_xbel preload;
 
 	readok = 0;
 
@@ -125,7 +133,12 @@ read_bookmarks_xbel(FILE *f)
 		}
 	}
 
-	if (!err) readok = xbeltree_to_bookmarks_list(root_node->children, NULL); /* Top node is xbel */
+	if (!err) {
+		preload.utf8_cp = get_cp_index("UTF-8");
+		readok = xbeltree_to_bookmarks_list(&preload,
+						    root_node->children, /* Top node is xbel */
+						    NULL);
+	}
 
 	XML_ParserFree(p);
 	free_xbeltree(root_node);
@@ -140,7 +153,7 @@ write_bookmarks_xbel(struct secure_save_info *ssi,
 	/* We check for readok in filename_bookmarks_xbel(). */
 
 	secure_fputs(ssi,
-		"<?xml version=\"1.0\"?>\n"
+		"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
 		"<!DOCTYPE xbel PUBLIC \"+//IDN python.org//DTD XML "
 		"Bookmark Exchange Language 1.0//EN//XML\"\n"
 		"		       "
@@ -168,42 +181,20 @@ indentation(struct secure_save_info *ssi, int num)
 		secure_fputs(ssi, "    ");
 }
 
-/* FIXME This is totally broken, we should use the Unicode value in
- *       numeric entities.
- *       Additionally it is slow, not elegant, incomplete and
- *       if you pay enough attention you can smell the unmistakable
- *       odor of doom coming from it. --fabio */
 static void
 print_xml_entities(struct secure_save_info *ssi, const unsigned char *str)
 {
-#define accept_char(x) (isident((x)) || (x) == ' ' || (x) == '.' \
+	struct string entitized = NULL_STRING;
-				 || (x) == ':' || (x) == ';' \
-				 || (x) == '/' || (x) == '(' \
-				 || (x) == ')' || (x) == '}' \
-				 || (x) == '{' || (x) == '%' \
-				 || (x) == '+')
 
-	static int cp = -1;
+	if (init_string(&entitized)
-
+	    && add_html_to_string(&entitized, str, strlen(str))) {
-	if (cp == -1) cp = get_cp_index("us-ascii");
+		secure_fputs(ssi, entitized.source);
-
+	} else {
-	for (; *str; str++) {
+		secsave_errno = SS_ERR_OUT_OF_MEM;
-		if (accept_char(*str))
+		ssi->err = ENOMEM;
-			secure_fputc(ssi, *str);
-		else {
-			if (isascii(*str)) {
-				secure_fprintf(ssi, "&#%i;", (int) *str);
-			}
-			else {
-				const unsigned char *s = u2cp_no_nbsp(*str, cp);
-
-				if (s) print_xml_entities(ssi, s);
-			}
-		}
 	}
 
-#undef accept_char
+	done_string(&entitized);
-
 }
 
 static void
@@ -257,7 +248,6 @@ write_bookmarks_list(struct secure_save_info *ssi,
 static void
 on_element_open(void *data, const char *name, const char **attr)
 {
-	struct attributes *attribute;
 	struct tree_node *node;
 
 	node = new_node(current_node);
@@ -284,26 +274,23 @@ on_element_open(void *data, const char *name, const char **attr)
 		return;
 	}
 
-	while (*attr) {
+	for (; *attr; attr += 2) {
-		unsigned char *tmp = stracpy((unsigned char *) *attr);
+		struct attributes *attribute = mem_calloc(1, sizeof(*attribute));
+		unsigned char *name = stracpy((unsigned char *) attr[0]);
+		unsigned char *value = stracpy((unsigned char *) attr[1]);
 
-		if (!tmp) {
+		if (!attribute || !name || !value) {
+			mem_free_if(attribute);
+			mem_free_if(name);
+			mem_free_if(value);
 			free_node(current_node);
 			return;
 		}
 
-		attribute = mem_calloc(1, sizeof(*attribute));
+		attribute->name = name;
-		if (!attribute) {
+		attribute->value = value;
-			mem_free(tmp);
-			free_node(current_node);
-			return;
-		}
 
-		attribute->name = tmp;
+		add_to_list_end(current_node->attrs, attribute);
-
-		add_to_list(current_node->attrs, attribute);
-
-		++attr;
 	}
 
 }
@@ -315,25 +302,34 @@ on_element_close(void *data, const char *name)
 }
 
 static unsigned char *
-delete_whites(unsigned char *s)
+delete_whites(const unsigned char *s)
 {
 	unsigned char *r;
-	int count = 0, c = 0, i;
+	int last_was_space = 0, c = 0, i;
 	int len = strlen(s);
 
 	r = mem_alloc(len + 1);
 	if (!r) return NULL;
 
 	for (i = 0; i < len; i++) {
-		if (isspace(s[i])) {
+		/* Recognize only the whitespace characters listed
-			if (count == 1) continue;
+		 * in section 2.3 of XML 1.1.  U+0085 and U+2028 need
-			else count = 1;
+		 * not be recognized here because section 2.11 says
-		}
+		 * the XML processor must translate them to U+000A.
-		else count = 0;
+		 * Do not use isspace() because the string is in UTF-8
-
+		 * and individual bytes might not be characters at
-		if (s[i] == '\n' || s[i] == '\t')
+		 * all.  */
+		switch (s[i]) {
+		case '\x20': case '\x09': case '\x0D': case '\x0A':
+			if (last_was_space) continue;
+			last_was_space = 1;
 			r[c++] = ' ';
-		else r[c++] = s[i];
+			break;
+		default:
+			last_was_space = 0;
+			r[c++] = s[i];
+			break;
+		}
 	}
 
 	r[c] = '\0';
@@ -370,7 +366,8 @@ on_text(void *data, const XML_Char *text, int len)
 /* xbel_tree_to_bookmarks_list: returns 0 on fail,
  *				      1 on success */
 static int
-xbeltree_to_bookmarks_list(struct tree_node *node,
+xbeltree_to_bookmarks_list(const struct read_bookmarks_xbel *preload,
+			   struct tree_node *node,
 			   struct bookmark *current_parent)
 {
 	struct bookmark *tmp;
@@ -384,6 +381,7 @@ xbeltree_to_bookmarks_list(struct tree_node *node,
 			title = get_child(node, "title");
 			href = get_attribute_value(node, "href");
 
+			intl_set_charset_by_index(preload->utf8_cp);
 			tmp = add_bookmark(current_parent, 0,
 					   /* The <title> element is optional */
 					   title && title->text ? title->text
@@ -406,6 +404,7 @@ xbeltree_to_bookmarks_list(struct tree_node *node,
 
 			title = get_child(node, "title");
 
+			intl_set_charset_by_index(preload->utf8_cp);
 			tmp = add_bookmark(current_parent, 0,
 					   title && title->text ? title->text
 						 : (unsigned char *) gettext("No title"),
@@ -431,14 +430,18 @@ xbeltree_to_bookmarks_list(struct tree_node *node,
 
 		if (node->children) {
 			int ret;
+			struct bookmark *parent_for_nested;
 
 			/* If this node is a <folder> element, current parent
 			 * changes */
-			ret = (!strcmp(node->name, "folder") ?
+			if (!strcmp(node->name, "folder"))
-				xbeltree_to_bookmarks_list(node->children,
+				parent_for_nested = lastbm;
-							   lastbm) :
+			else
-				xbeltree_to_bookmarks_list(node->children,
+				parent_for_nested = current_parent;
-							   current_parent));
+
+			ret = xbeltree_to_bookmarks_list(preload,
+							 node->children,
+							 parent_for_nested);
 			/* Out of memory */
 			if (!ret) return 0;
 		}
@@ -491,9 +494,9 @@ get_attribute_value(struct tree_node *node, unsigned char *name)
 {
 	struct attributes *attribute;
 
-	foreachback (attribute, node->attrs) {
+	foreach (attribute, node->attrs) {
 		if (!strcmp(attribute->name, name)) {
-			return attribute->prev->name;
+			return attribute->value;
 		}
 	}
 
@@ -519,8 +522,11 @@ free_node(struct tree_node *node)
 {
 	struct attributes *attribute;
 
-	foreachback (attribute, node->attrs)
+	foreach (attribute, node->attrs) {
-		mem_free_if(attribute->name);
+		/* on_element_open() ensures ->name and ->value aren't NULL. */
+		mem_free(attribute->name);
+		mem_free(attribute->value);
+	}
 	free_list(node->attrs); /* Don't free list during traversal */
 
 	mem_free_if(node->name);

src/bookmarks/bookmarks.c

@@ -54,14 +54,13 @@ static struct option_info bookmark_options_info[] = {
 		"file_format", 0, 0, 1, 0,
 		N_("File format for bookmarks (affects both reading and saving):\n"
 		"0 is the default native ELinks format\n"
-		"1 is XBEL universal XML bookmarks format (ELinks bug 153: NO NATIONAL CHARS SUPPORT!)")),
+		"1 is XBEL universal XML bookmarks format")),
 #else
 	INIT_OPT_INT("bookmarks", N_("File format"),
 		"file_format", 0, 0, 1, 0,
 		N_("File format for bookmarks (affects both reading and saving):\n"
 		"0 is the default native ELinks format\n"
-		"1 is XBEL universal XML bookmarks format (ELinks bug 153: NO NATIONAL CHARS SUPPORT!)"
+		"1 is XBEL universal XML bookmarks format  (DISABLED)")),
-		"  (DISABLED)")),
 #endif
 
 	INIT_OPT_BOOL("bookmarks", N_("Save folder state"),
@@ -268,10 +267,13 @@ delete_bookmark(struct bookmark *bm)
 	done_bookmark(bm);
 }
 
-/* Deletes any bookmarks with no URLs (i.e., folders) and of which
+/** Deletes any bookmarks with no URLs (i.e., folders) and of which
- * the title matches the given argument. */
+ * the title matches the given argument.
+ *
+ * @param foldername
+ *  The title of the folder, in UTF-8.  */
 static void
-delete_folder_by_name(unsigned char *foldername)
+delete_folder_by_name(const unsigned char *foldername)
 {
 	struct bookmark *bookmark, *next;
 
@@ -284,6 +286,21 @@ delete_folder_by_name(unsigned char *foldername)
 	}
 }
 
+/** Allocate and initialize a bookmark in the given folder.  This
+ * however does not set bookmark.box_item; use add_bookmark() for
+ * that.
+ *
+ * @param root
+ *   The folder in which to add the bookmark, or NULL to add it at
+ *   top level.
+ * @param title
+ *   Title of the bookmark.  Must be in UTF-8 and not NULL.
+ *   "-" means add a separator.
+ * @param url
+ *   URL to which the bookmark will point.  Must be in UTF-8.
+ *   NULL or "" means add a bookmark folder.
+ *
+ * @return the new bookmark, or NULL on error.  */
 static struct bookmark *
 init_bookmark(struct bookmark *root, unsigned char *title, unsigned char *url)
 {
@@ -341,8 +358,23 @@ add_bookmark_item_to_bookmarks(struct bookmark *bm, struct bookmark *root, int p
 		add_hash_item(bookmark_cache, bm->url, strlen(bm->url), bm);
 }
 
-/* Adds a bookmark to the bookmark list. Place 0 means top, place 1 means
+/** Add a bookmark to the bookmark list.
- * bottom. NULL or "" @url means it is a bookmark folder. */
+ *
+ * @param root
+ *   The folder in which to add the bookmark, or NULL to add it at
+ *   top level.
+ * @param place
+ *   0 means add to the top.  1 means add to the bottom.
+ * @param title
+ *   Title of the bookmark.  Must be in UTF-8 and not NULL.
+ *   "-" means add a separator.
+ * @param url
+ *   URL to which the bookmark will point.  Must be in UTF-8.
+ *   NULL or "" means add a bookmark folder.
+ *
+ * @return the new bookmark, or NULL on error.
+ *
+ * @see add_bookmark_cp() */
 struct bookmark *
 add_bookmark(struct bookmark *root, int place, unsigned char *title,
 	     unsigned char *url)
@@ -379,27 +411,87 @@ add_bookmark(struct bookmark *root, int place, unsigned char *title,
 	return bm;
 }
 
+/** Add a bookmark to the bookmark list.
+ *
+ * @param root
+ *   The folder in which to add the bookmark, or NULL to add it at
+ *   top level.
+ * @param place
+ *   0 means add to the top.  1 means add to the bottom.
+ * @param codepage
+ *   Codepage of @a title and @a url.
+ * @param title
+ *   Title of the bookmark.  Must not be NULL.
+ *   "-" means add a separator.
+ * @param url
+ *   URL to which the bookmark will point.
+ *   NULL or "" means add a bookmark folder.
+ *
+ * @return the new bookmark.
+ *
+ * @see add_bookmark() */
+struct bookmark *
+add_bookmark_cp(struct bookmark *root, int place, int codepage,
+		unsigned char *title, unsigned char *url)
+{
+	const int utf8_cp = get_cp_index("UTF-8");
+	struct conv_table *table;
+	unsigned char *utf8_title = NULL;
+	unsigned char *utf8_url = NULL;
+	struct bookmark *bookmark = NULL;
+
+	if (!url)
+		url = "";
+
+	table = get_translation_table(codepage, utf8_cp);
+	if (!table)
+		return NULL;
+
+	utf8_title = convert_string(table, title, strlen(title),
+				    utf8_cp, CSM_NONE,
+				    NULL, NULL, NULL);
+	utf8_url = convert_string(table, url, strlen(url),
+				  utf8_cp, CSM_NONE,
+				  NULL, NULL, NULL);
+	if (utf8_title && utf8_url)
+		bookmark = add_bookmark(root, place,
+					utf8_title, utf8_url);
+	mem_free_if(utf8_title);
+	mem_free_if(utf8_url);
+	return bookmark;
+}
+
 /* Updates an existing bookmark.
  *
  * If there's any problem, return 0. Otherwise, return 1.
  *
  * If any of the fields are NULL, the value is left unchanged. */
 int
-update_bookmark(struct bookmark *bm, unsigned char *title,
+update_bookmark(struct bookmark *bm, int codepage,
-		unsigned char *url)
+		unsigned char *title, unsigned char *url)
 {
 	static int update_bookmark_event_id = EVENT_NONE;
+	const int utf8_cp = get_cp_index("UTF-8");
+	struct conv_table *table;
 	unsigned char *title2 = NULL;
 	unsigned char *url2 = NULL;
 
+	table = get_translation_table(codepage, utf8_cp);
+	if (!table)
+		return 0;
+
 	if (url) {
-		url2 = stracpy(url);
+		url2 = convert_string(table, url, strlen(url),
+				      utf8_cp, CSM_NONE,
+				      NULL, NULL, NULL);
 		if (!url2) return 0;
 		sanitize_url(url2);
 	}
 
 	if (title) {
-		title2 = stracpy(title);
+		title2 = convert_string(table, title, strlen(title),
+					utf8_cp, CSM_NONE,
+					NULL, NULL, NULL);
 		if (!title2) {
 			mem_free_if(url2);
 			return 0;
@@ -435,8 +527,16 @@ update_bookmark(struct bookmark *bm, unsigned char *title,
 	return 1;
 }
 
-/* Search for a bookmark with the given title. Search in the given folder
+/** Search for a bookmark with the given title.  The search does not
- * or in the root if folder is NULL. */
+ * recurse into subfolders.
+ *
+ * @param folder
+ *   Search in this folder.  NULL means search in the root.
+ *
+ * @param title
+ *   Search for this title.  Must be in UTF-8 and not NULL.
+ *
+ * @return The bookmark, or NULL if not found.  */
 struct bookmark *
 get_bookmark_by_name(struct bookmark *folder, unsigned char *title)
 {
@@ -457,6 +557,7 @@ get_bookmark(unsigned char *url)
 {
 	struct hash_item *item;
 
+	/** @todo Bug 1066: URLs in bookmark_cache should be UTF-8 */
 	if (!check_bookmark_cache(url))
 		return NULL;
 
@@ -472,6 +573,7 @@ bookmark_terminal(struct terminal *term, struct bookmark *folder)
 {
 	unsigned char title[MAX_STR_LEN], url[MAX_STR_LEN];
 	struct window *tab;
+	int term_cp = get_terminal_codepage(term);
 
 	foreachback_tab (tab, term->windows) {
 		struct session *ses = tab->data;
@@ -482,10 +584,18 @@ bookmark_terminal(struct terminal *term, struct bookmark *folder)
 		if (!get_current_title(ses, title, MAX_STR_LEN))
 			continue;
 
-		add_bookmark(folder, 1, title, url);
+		add_bookmark_cp(folder, 1, term_cp, title, url);
 	}
 }
 
+/** Create a bookmark for each document on the specified terminal,
+ * and a folder to contain those bookmarks.
+ *
+ * @param term
+ *   The terminal whose open documents should be bookmarked.
+ *
+ * @param foldername
+ *   The name of the new bookmark folder, in UTF-8.  */
 void
 bookmark_terminal_tabs(struct terminal *term, unsigned char *foldername)
 {
@@ -520,6 +630,8 @@ bookmark_all_terminals(struct bookmark *folder)
 
 		++n;
 
+		/* Because subfoldername[] contains only digits,
+		 * it is OK as UTF-8.  */
 		subfolder = add_bookmark(folder, 1, subfoldername, NULL);
 		if (!subfolder) return;
 
@@ -528,23 +640,48 @@ bookmark_all_terminals(struct bookmark *folder)
 }
 
 
+unsigned char *
+get_auto_save_bookmark_foldername_utf8(void)
+{
+	unsigned char *foldername;
+	int from_cp, to_cp;
+	struct conv_table *convert_table;
+
+	foldername = get_opt_str("ui.sessions.auto_save_foldername", NULL);
+	if (!*foldername) return NULL;
+
+	/* The charset of the string returned by get_opt_str()
+	 * seems to be documented nowhere.  Let's assume it is
+	 * the system charset.  */
+	from_cp = get_cp_index("System");
+	to_cp = get_cp_index("UTF-8");
+	convert_table = get_translation_table(from_cp, to_cp);
+	if (!convert_table) return NULL;
+
+	return convert_string(convert_table,
+			      foldername, strlen(foldername),
+			      to_cp, CSM_NONE,
+			      NULL, NULL, NULL);
+}
+
 void
 bookmark_auto_save_tabs(struct terminal *term)
 {
-	unsigned char *foldername;
+	unsigned char *foldername; /* UTF-8 */
 
 	if (get_cmd_opt_bool("anonymous")
 	    || !get_opt_bool("ui.sessions.auto_save", NULL))
 		return;
 
-	foldername = get_opt_str("ui.sessions.auto_save_foldername", NULL);
+	foldername = get_auto_save_bookmark_foldername_utf8();
-	if (!*foldername) return;
+	if (!foldername) return;
 
 	/* Ensure uniqueness of the auto save folder, so it is possible to
 	 * restore the (correct) session when starting up. */
 	delete_folder_by_name(foldername);
 
 	bookmark_terminal_tabs(term, foldername);
+	mem_free(foldername);
 }
 
 static void
@@ -563,7 +700,10 @@ bookmark_snapshot(void)
 	                   NULL);
 #endif
 
-	folder = add_bookmark(NULL, 1, folderstring.source, NULL);
+	/* folderstring must be in the system codepage because
+	 * add_date_to_string() uses strftime().  */
+	folder = add_bookmark_cp(NULL, 1, get_cp_index("System"),
+				 folderstring.source, NULL);
 	done_string(&folderstring);
 	if (!folder) return;
 
@@ -574,6 +714,14 @@ bookmark_snapshot(void)
 }
 
 
+/** Open all bookmarks from the named folder.
+ *
+ * @param ses
+ *   The session in which to open the first bookmark.  The other
+ *   bookmarks of the folder open in new tabs on the same terminal.
+ *
+ * @param foldername
+ *   The name of the bookmark folder, in UTF-8.  */
 void
 open_bookmark_folder(struct session *ses, unsigned char *foldername)
 {
@@ -603,6 +751,8 @@ open_bookmark_folder(struct session *ses, unsigned char *foldername)
 		    || !*bookmark->url)
 			continue;
 
+		/** @todo Bug 1066: Tell the URI layer that
+		 * bookmark->url is UTF-8.  */
 		uri = get_translated_uri(bookmark->url, NULL);
 		if (!uri) continue;
 

src/bookmarks/bookmarks.h

@@ -17,7 +17,9 @@ struct bookmark {
 
 	struct listbox_item *box_item;
 
-	unsigned char *title;   /* title of bookmark */
+	/** @todo Bug 1066: The #url string should be in UTF-8 too,
+	 * but this has not yet been fully implemented.  */
+	unsigned char *title;   /* UTF-8 title of bookmark */
 	unsigned char *url;     /* Location of bookmarked item */
 
 	LIST_OF(struct bookmark) child;
@@ -43,12 +45,16 @@ int bookmarks_are_dirty(void);
 
 void delete_bookmark(struct bookmark *);
 struct bookmark *add_bookmark(struct bookmark *, int, unsigned char *, unsigned char *);
+struct bookmark *add_bookmark_cp(struct bookmark *, int, int,
+				 unsigned char *, unsigned char *);
 struct bookmark *get_bookmark_by_name(struct bookmark *folder,
                                       unsigned char *title);
 struct bookmark *get_bookmark(unsigned char *url);
 void bookmark_terminal_tabs(struct terminal *term, unsigned char *foldername);
+unsigned char *get_auto_save_bookmark_foldername_utf8(void);
 void bookmark_auto_save_tabs(struct terminal *term);
-int update_bookmark(struct bookmark *, unsigned char *, unsigned char *);
+int update_bookmark(struct bookmark *, int,
+		    unsigned char *, unsigned char *);
 void open_bookmark_folder(struct session *ses, unsigned char *foldername);
 
 #endif

src/bookmarks/dialogs.c

@@ -56,21 +56,52 @@ static unsigned char *
 get_bookmark_text(struct listbox_item *item, struct terminal *term)
 {
 	struct bookmark *bookmark = item->udata;
+	int utf8_cp = get_cp_index("UTF-8");
+	int term_cp = get_terminal_codepage(term);
+	struct conv_table *convert_table;
 
-	return stracpy(bookmark->title);
+	convert_table = get_translation_table(utf8_cp, term_cp);
+	if (!convert_table) return NULL;
+
+	return convert_string(convert_table,
+			      bookmark->title, strlen(bookmark->title),
+			      term_cp, CSM_NONE, NULL, NULL, NULL);
+}
+
+/** A callback for convert_string().  This ignores errors and can
+ * result in truncated strings if out of memory.  Accordingly, the
+ * resulting string may be displayed in the UI but should not be saved
+ * to a file or given to another program.  */
+static void
+add_converted_bytes_to_string(void *data, unsigned char *buf, int buflen)
+{
+	struct string *string = data;
+
+	add_bytes_to_string(string, buf, buflen); /* ignore errors */
 }
 
 static unsigned char *
 get_bookmark_info(struct listbox_item *item, struct terminal *term)
 {
 	struct bookmark *bookmark = item->udata;
+	int utf8_cp = get_cp_index("UTF-8");
+	int term_cp = get_terminal_codepage(term);
+	struct conv_table *convert_table;
 	struct string info;
 
 	if (item->type == BI_FOLDER) return NULL;
+	convert_table = get_translation_table(utf8_cp, term_cp);
+	if (!convert_table) return NULL;
 	if (!init_string(&info)) return NULL;
 
-	add_format_to_string(&info, "%s: %s", _("Title", term), bookmark->title);
+	add_format_to_string(&info, "%s: ", _("Title", term));
-	add_format_to_string(&info, "\n%s: %s", _("URL", term), bookmark->url);
+	convert_string(convert_table, bookmark->title, strlen(bookmark->title),
+		       term_cp, CSM_NONE, NULL,
+		       add_converted_bytes_to_string, &info);
+	add_format_to_string(&info, "\n%s: ", _("URL", term));
+	convert_string(convert_table, bookmark->url, strlen(bookmark->url),
+		       term_cp, CSM_NONE, NULL,
+		       add_converted_bytes_to_string, &info);
 
 	return info.source;
 }
@@ -80,6 +111,7 @@ get_bookmark_uri(struct listbox_item *item)
 {
 	struct bookmark *bookmark = item->udata;
 
+	/** @todo Bug 1066: Tell the URI layer that bookmark->url is UTF-8.  */
 	return bookmark->url && *bookmark->url
 		? get_translated_uri(bookmark->url, NULL) : NULL;
 }
@@ -199,9 +231,33 @@ move_bookmark_after_selected(struct bookmark *bookmark, struct bookmark *selecte
 	add_at_pos(selected->box_item, bookmark->box_item);
 }
 
+/** Add a bookmark; if called from the bookmark manager, also move
+ * the bookmark to the right place and select it in the manager.
+ * And possibly save the bookmarks.
+ *
+ * @param term
+ *   The terminal whose user told ELinks to add the bookmark.
+ *   Currently, @a term affects only the charset interpretation
+ *   of @a title and @a url.  In the future, this function could
+ *   also display error messages in @a term.
+ *
+ * @param dlg_data
+ *   The bookmark manager dialog, or NULL if the bookmark is being
+ *   added without involving the bookmark manager.  If @a dlg_data
+ *   is not NULL, dlg_data->win->term should be @a term.
+ *
+ * @param title
+ *   The title of the new bookmark, in the encoding of @a term.
+ *   Must not be NULL.  "-" means add a separator.
+ *
+ * @param url
+ *   The URL of the new bookmark, in the encoding of @a term.  NULL
+ *   or "" means add a bookmark folder, unless @a title is "-".  */
 static void
-do_add_bookmark(struct dialog_data *dlg_data, unsigned char *title, unsigned char *url)
+do_add_bookmark(struct terminal *term, struct dialog_data *dlg_data,
+		unsigned char *title, unsigned char *url)
 {
+	int term_cp = get_terminal_codepage(term);
 	struct bookmark *bm = NULL;
 	struct bookmark *selected = NULL;
 	struct listbox_data *box = NULL;
@@ -220,7 +276,7 @@ do_add_bookmark(struct dialog_data *dlg_data, unsigned char *title, unsigned cha
 		}
 	}
 
-	bm = add_bookmark(bm, 1, title, url);
+	bm = add_bookmark_cp(bm, 1, term_cp, title, url);
 	if (!bm) return;
 
 	move_bookmark_after_selected(bm, selected);
@@ -241,12 +297,29 @@ do_add_bookmark(struct dialog_data *dlg_data, unsigned char *title, unsigned cha
 
 /**** ADD FOLDER *****************************************************/
 
+/** Add a bookmark folder.  This is called when the user pushes the OK
+ * button in the input dialog that asks for the folder name.
+ *
+ * @param dlg_data
+ *   The bookmark manager.  Must not be NULL.
+ *
+ * @param foldername
+ *   The folder name that the user typed in the input dialog.
+ *   This is in the charset of the terminal.  */
 static void
 do_add_folder(struct dialog_data *dlg_data, unsigned char *foldername)
 {
-	do_add_bookmark(dlg_data, foldername, NULL);
+	do_add_bookmark(dlg_data->win->term, dlg_data, foldername, NULL);
 }
 
+/** Prepare to add a bookmark folder.  This is called when the user
+ * pushes the "Add folder" button in the bookmark manager.
+ *
+ * @param dlg_data
+ *   The bookmark manager.  Must not be NULL.
+ *
+ * @param widget_data
+ *   The "Add folder" button.  */
 static widget_handler_status_T
 push_add_folder_button(struct dialog_data *dlg_data, struct widget_data *widget_data)
 {
@@ -262,10 +335,18 @@ push_add_folder_button(struct dialog_data *dlg_data, struct widget_data *widget_
 
 /**** ADD SEPARATOR **************************************************/
 
+/** Add a bookmark separator.  This is called when the user pushes the
+ * "Add separator" button in the bookmark manager.
+ *
+ * @param dlg_data
+ *   The bookmark manager.  Must not be NULL.
+ *
+ * @param widget_data
+ *   The "Add separator" button.  */
 static widget_handler_status_T
 push_add_separator_button(struct dialog_data *dlg_data, struct widget_data *widget_data)
 {
-	do_add_bookmark(dlg_data, "-", "");
+	do_add_bookmark(dlg_data->win->term, dlg_data, "-", "");
 	redraw_dialog(dlg_data, 1);
 	return EVENT_PROCESSED;
 }
@@ -278,8 +359,11 @@ static void
 bookmark_edit_done(void *data) {
 	struct dialog *dlg = data;
 	struct bookmark *bm = (struct bookmark *) dlg->udata2;
+	struct dialog_data *parent_dlg_data = dlg->udata;
+	int term_cp = get_terminal_codepage(parent_dlg_data->win->term);
 
-	update_bookmark(bm, dlg->widgets[0].data, dlg->widgets[1].data);
+	update_bookmark(bm, term_cp,
+			dlg->widgets[0].data, dlg->widgets[1].data);
 	object_unlock(bm);
 
 #ifdef BOOKMARKS_RESAVE
@@ -303,15 +387,37 @@ push_edit_button(struct dialog_data *dlg_data, struct widget_data *edit_btn)
 	/* Follow the bookmark */
 	if (box->sel) {
 		struct bookmark *bm = (struct bookmark *) box->sel->udata;
-		const unsigned char *title = bm->title;
+		int utf8_cp = get_cp_index("UTF-8");
-		const unsigned char *url = bm->url;
+		int term_cp = get_terminal_codepage(dlg_data->win->term);
+		struct conv_table *convert_table;
 
-		object_lock(bm);
+		convert_table = get_translation_table(utf8_cp, term_cp);
-		do_edit_dialog(dlg_data->win->term, 1, N_("Edit bookmark"),
+		if (convert_table) {
-			       title, url,
+			unsigned char *title;
-			       (struct session *) dlg_data->dlg->udata, dlg_data,
+			unsigned char *url;
-			       bookmark_edit_done, bookmark_edit_cancel,
+
-			       (void *) bm, EDIT_DLG_ADD);
+			title = convert_string(convert_table,
+					       bm->title, strlen(bm->title),
+					       term_cp, CSM_NONE,
+					       NULL, NULL, NULL);
+			url = convert_string(convert_table,
+					     bm->url, strlen(bm->url),
+					     term_cp, CSM_NONE,
+					     NULL, NULL, NULL);
+			if (title && url) {
+				object_lock(bm);
+				do_edit_dialog(dlg_data->win->term, 1,
+					       N_("Edit bookmark"),
+					       title, url,
+					       (struct session *) dlg_data->dlg->udata,
+					       dlg_data,
+					       bookmark_edit_done,
+					       bookmark_edit_cancel,
+					       (void *) bm, EDIT_DLG_ADD);
+			}
+			mem_free_if(title);
+			mem_free_if(url);
+		}
 	}
 
 	return EVENT_PROCESSED;
@@ -535,13 +641,15 @@ bookmark_manager(struct session *ses)
  * rapid search of an already existing bookmark. --Zas */
 
 struct bookmark_search_ctx {
-	unsigned char *url;
+	unsigned char *url;	/* UTF-8 */
-	unsigned char *title;
+	unsigned char *title;	/* system charset */
 	int found;
 	int offset;
+	int utf8_cp;
+	int system_cp;
 };
 
-#define NULL_BOOKMARK_SEARCH_CTX {NULL, NULL, 0, 0}
+#define NULL_BOOKMARK_SEARCH_CTX {NULL, NULL, 0, 0, -1, -1}
 
 static int
 test_search(struct listbox_item *item, void *data_, int *offset)
@@ -555,8 +663,37 @@ test_search(struct listbox_item *item, void *data_, int *offset)
 
 		assert(ctx->title && ctx->url);
 
-		ctx->found = (*ctx->title && strcasestr(bm->title, ctx->title))
+		ctx->found = (*ctx->url && c_strcasestr(bm->url, ctx->url));
-			     || (*ctx->url && c_strcasestr(bm->url, ctx->url));
+		if (!ctx->found && *ctx->title) {
+			/* The comparison of bookmark titles should
+			 * be case-insensitive and locale-sensitive
+			 * (Turkish dotless i).  ELinks doesn't have
+			 * such a function for UTF-8.  The best we
+			 * have is strcasestr, which uses the system
+			 * charset.  So convert bm->title to that.
+			 * (ctx->title has already been converted.)  */
+			struct conv_table *convert_table;
+			unsigned char *title = NULL;
+
+			convert_table = get_translation_table(ctx->utf8_cp,
+							      ctx->system_cp);
+			if (convert_table) {
+				title = convert_string(convert_table,
+						       bm->title,
+						       strlen(bm->title),
+						       ctx->system_cp,
+						       CSM_NONE, NULL,
+						       NULL, NULL);
+			}
+
+			if (title) {
+				ctx->found = strcasecmp(title,
+							ctx->title);
+				mem_free(title);
+			}
+			/** @todo Tell the user that the string could
+			 * not be converted.  */
+		}
 
 		if (ctx->found) *offset = 0;
 	}
@@ -565,7 +702,9 @@ test_search(struct listbox_item *item, void *data_, int *offset)
 	return 0;
 }
 
-/* Last searched values */
+/* Last searched values.  Both are in UTF-8.  (The title could be kept
+ * in the system charset, but that would be a bit risky, because
+ * setlocale calls from Lua scripts can change the system charset.)  */
 static unsigned char *bm_last_searched_title = NULL;
 static unsigned char *bm_last_searched_url = NULL;
 
@@ -577,14 +716,15 @@ free_last_searched_bookmark(void)
 }
 
 static int
-memorize_last_searched_bookmark(struct bookmark_search_ctx *ctx)
+memorize_last_searched_bookmark(const unsigned char *title,
+				const unsigned char *url)
 {
 	/* Memorize last searched title */
-	mem_free_set(&bm_last_searched_title, stracpy(ctx->title));
+	mem_free_set(&bm_last_searched_title, stracpy(title));
 	if (!bm_last_searched_title) return 0;
 
 	/* Memorize last searched url */
-	mem_free_set(&bm_last_searched_url, stracpy(ctx->url));
+	mem_free_set(&bm_last_searched_url, stracpy(url));
 	if (!bm_last_searched_url) {
 		mem_free_set(&bm_last_searched_title, NULL);
 		return 0;
@@ -601,38 +741,94 @@ bookmark_search_do(void *data)
 	struct bookmark_search_ctx ctx = NULL_BOOKMARK_SEARCH_CTX;
 	struct listbox_data *box;
 	struct dialog_data *dlg_data;
+	struct conv_table *convert_table;
+	int term_cp;
+	unsigned char *url_term;
+	unsigned char *title_term;
+	unsigned char *title_utf8 = NULL;
 
 	assertm(dlg->udata != NULL, "Bookmark search with NULL udata in dialog");
 	if_assert_failed return;
 
-	ctx.title = dlg->widgets[0].data;
-	ctx.url   = dlg->widgets[1].data;
-
-	if (!ctx.title || !ctx.url)
-		return;
-
-	if (!memorize_last_searched_bookmark(&ctx))
-		return;
-
 	dlg_data = (struct dialog_data *) dlg->udata;
+	term_cp = get_terminal_codepage(dlg_data->win->term);
+	ctx.system_cp = get_cp_index("System");
+	ctx.utf8_cp = get_cp_index("UTF-8");
+
+	title_term = dlg->widgets[0].data; /* need not be freed */
+	url_term   = dlg->widgets[1].data; /* likewise */
+
+	convert_table = get_translation_table(term_cp, ctx.system_cp);
+	if (!convert_table) goto free_all;
+	ctx.title = convert_string(convert_table,
+				   title_term, strlen(title_term),
+				   ctx.system_cp, CSM_NONE, NULL, NULL, NULL);
+	if (!ctx.title) goto free_all;
+
+	convert_table = get_translation_table(term_cp, ctx.utf8_cp);
+	if (!convert_table) goto free_all;
+	ctx.url = convert_string(convert_table,
+				 url_term, strlen(url_term),
+				 ctx.utf8_cp, CSM_NONE, NULL, NULL, NULL);
+	if (!ctx.url) goto free_all;
+	title_utf8 = convert_string(convert_table,
+				    title_term, strlen(title_term),
+				    ctx.utf8_cp, CSM_NONE, NULL, NULL, NULL);
+	if (!title_utf8) goto free_all;
+
+	if (!memorize_last_searched_bookmark(title_utf8, ctx.url))
+		goto free_all;
+
 	box = get_dlg_listbox_data(dlg_data);
 
 	traverse_listbox_items_list(box->sel, box, 0, 0, test_search, &ctx);
-	if (!ctx.found) return;
+	if (!ctx.found) goto free_all;
 
 	listbox_sel_move(dlg_data->widgets_data, ctx.offset - 1);
-}
 
+free_all:
+	mem_free_if(ctx.title);
+	mem_free_if(ctx.url);
+	mem_free_if(title_utf8);
+}
 
 static void
 launch_bm_search_doc_dialog(struct terminal *term,
 			    struct dialog_data *parent,
 			    struct session *ses)
 {
+	unsigned char *title = NULL;
+	unsigned char *url = NULL;
+
+	if (bm_last_searched_title && bm_last_searched_url) {
+		int utf8_cp, term_cp;
+		struct conv_table *convert_table;
+
+		utf8_cp = get_cp_index("UTF-8");
+		term_cp = get_terminal_codepage(term);
+
+		convert_table = get_translation_table(utf8_cp, term_cp);
+		if (convert_table) {
+			title = convert_string(convert_table, bm_last_searched_title,
+					       strlen(bm_last_searched_title), term_cp,
+					       CSM_NONE, NULL, NULL, NULL);
+			url = convert_string(convert_table, bm_last_searched_url,
+					     strlen(bm_last_searched_url), term_cp,
+					     CSM_NONE, NULL, NULL, NULL);
+		}
+		if (!title || !url) {
+			mem_free_set(&title, NULL);
+			mem_free_set(&url, NULL);
+		}
+	}
+
 	do_edit_dialog(term, 1, N_("Search bookmarks"),
-		       bm_last_searched_title, bm_last_searched_url,
+		       title, url,
 		       ses, parent, bookmark_search_do, NULL, NULL,
 		       EDIT_DLG_SEARCH);
+
+	mem_free_if(title);
+	mem_free_if(url);
 }
 
 
@@ -647,10 +843,31 @@ bookmark_add_add(void *data)
 {
 	struct dialog *dlg = data;
 	struct dialog_data *dlg_data = (struct dialog_data *) dlg->udata;
+	struct terminal *term = dlg->udata2;
 
-	do_add_bookmark(dlg_data, dlg->widgets[0].data, dlg->widgets[1].data);
+	do_add_bookmark(term, dlg_data, dlg->widgets[0].data, dlg->widgets[1].data);
 }
 
+/** Open a dialog box for adding a bookmark.
+ *
+ * @param term
+ *   The terminal in which the dialog box should appear.
+ *
+ * @param parent
+ *   The bookmark manager, or NULL if the user requested this action
+ *   from somewhere else.
+ *
+ * @param ses
+ *   If @a title or @a url is NULL, get defaults from the current
+ *   document of @a ses.
+ *
+ * @param title
+ *   The initial title of the new bookmark, in the encoding of @a term.
+ *   NULL means use @a ses.
+ *
+ * @param url
+ *   The initial URL of the new bookmark, in the encoding of @a term.
+ *   NULL means use @a ses.  */
 void
 launch_bm_add_dialog(struct terminal *term,
 		     struct dialog_data *parent,
@@ -658,8 +875,22 @@ launch_bm_add_dialog(struct terminal *term,
 		     unsigned char *title,
 		     unsigned char *url)
 {
+	/* When the user eventually pushes the OK button, BFU calls
+	 * bookmark_add_add() and gives it the struct dialog * as the
+	 * void * parameter.  However, bookmark_add_add() also needs
+	 * to know the struct terminal *, and there is no way to get
+	 * that from struct dialog.  The other bookmark dialogs work
+	 * around that by making dialog.udata point to the struct
+	 * dialog_data of the bookmark manager, but the "Add bookmark"
+	 * dialog can be triggered with ACT_MAIN_ADD_BOOKMARK, which
+	 * does not involve the bookmark manager at all.
+	 *
+	 * The solution here is to save the struct terminal * in
+	 * dialog.udata2, which the "Edit bookmark" dialog uses for
+	 * struct bookmark *.  When adding a new bookmark, we don't
+	 * need a pointer to an existing one, of course.  */
 	do_edit_dialog(term, 1, N_("Add bookmark"), title, url, ses,
-		       parent, bookmark_add_add, NULL, NULL, EDIT_DLG_ADD);
+		       parent, bookmark_add_add, NULL, term, EDIT_DLG_ADD);
 }
 
 void
@@ -687,6 +918,28 @@ launch_bm_add_link_dialog(struct terminal *term,
   Bookmark tabs dialog.
 \****************************************************************************/
 
+static void
+bookmark_terminal_tabs_ok(void *term_void, unsigned char *foldername)
+{
+	struct terminal *const term = term_void;
+	int from_cp = get_terminal_codepage(term);
+	int to_cp = get_cp_index("UTF-8");
+	struct conv_table *convert_table;
+	unsigned char *converted;
+
+	convert_table = get_translation_table(from_cp, to_cp);
+	if (convert_table == NULL) return; /** @todo Report the error */
+
+	converted = convert_string(convert_table,
+				   foldername, strlen(foldername),
+				   to_cp, CSM_NONE,
+				   NULL, NULL, NULL);
+	if (converted == NULL) return; /** @todo Report the error */
+
+	bookmark_terminal_tabs(term_void, converted);
+	mem_free(converted);
+}
+
 void
 bookmark_terminal_tabs_dialog(struct terminal *term)
 {
@@ -705,8 +958,7 @@ bookmark_terminal_tabs_dialog(struct terminal *term)
 		     N_("Bookmark tabs"), N_("Enter folder name"),
 		     term, NULL,
 		     MAX_STR_LEN, string.source, 0, 0, NULL,
-		     (void (*)(void *, unsigned char *)) bookmark_terminal_tabs,
+		     bookmark_terminal_tabs_ok, NULL);
-		     NULL);
 
 	done_string(&string);
 }

src/config/options.inc

@@ -389,7 +389,7 @@ static struct option_info config_options_info[] = {
 		N_("Whether the search should match the document text while maintaining\n"
 		"case sensitivity.")),
 
-#ifdef HAVE_REGEX_H
+#ifdef HAVE_TRE_REGEX_H
 	INIT_OPT_INT("document.browse.search", N_("Regular expressions"),
 		"regex", 0, 0, 2, 0,
 		N_("Enable searching with regular expressions:\n"

src/dialogs/status.c

@@ -363,7 +363,16 @@ display_tab_bar(struct session *ses, struct terminal *term, int tabs_count)
 					  box.x, box.y, actual_tab_width,
 					  msg, NULL);
 		} else {
-			int msglen = int_min(strlen(msg), actual_tab_width);
+			int msglen;
+#ifdef CONFIG_UTF8
+			if (term->utf8_cp) {
+				msglen = utf8_step_forward(msg, NULL,
+							   actual_tab_width,
+							   UTF8_STEP_CELLS_FEWER,
+							   NULL) - msg;
+			} else
+#endif /* CONFIG_UTF8 */
+				msglen = int_min(strlen(msg), actual_tab_width);
 
 			draw_text(term, box.x, box.y, msg, msglen, 0, color);
 		}

src/document/document.h

@@ -178,6 +178,9 @@ struct document {
 #endif
 
 	struct uri *uri;
+
+	/** The title of the document.  The charset of this string is
+	 * document.options.cp.  */
 	unsigned char *title;
 	struct cache_entry *cached;
 

src/elinks.h

@@ -29,6 +29,30 @@
 #define DEBUG_MEMLEAK
 #endif
 
+
+/* When CONFIG_UTF8 is defined, src/viewer/text/search.c makes a string
+ * of unicode_val_T and gives it to regwexec(), which expects a string
+ * of wchar_t.  If the unicode_val_T and wchar_t types are too different,
+ * this won't work, so try to detect that and disable regexp operations
+ * entirely in that case.
+ *
+ * Currently, this code only compares the sizes of the types.  Because
+ * unicode_val_T is defined as uint32_t and POSIX says bytes are 8-bit,
+ * sizeof(unicode_val_T) is 4 and the following compares SIZEOF_WCHAR_T
+ * to that.
+ *
+ * C99 says the implementation can define __STDC_ISO_10646__ if wchar_t
+ * values match ISO 10646 (or Unicode) numbers in all locales.  Do not
+ * check that macro here, because it is too restrictive: it should be
+ * enough for ELinks if the values match in the locales where ELinks is
+ * actually run.  */
+
+#ifdef CONFIG_UTF8
+#if SIZEOF_WCHAR_T != 4
+#undef HAVE_TRE_REGEX_H
+#endif
+#endif
+
 /* This maybe overrides some of the standard high-level functions, to ensure
  * the expected behaviour. These overrides are not system specific. */
 #include "osdep/stub.h"

src/intl/charsets.c

@@ -179,10 +179,8 @@ u2cp_(unicode_val_T u, int to, enum nbsp_mode nbsp_mode)
 
 	to &= ~SYSTEM_CHARSET_FLAG;
 
-#ifdef CONFIG_UTF8
 	if (is_cp_ptr_utf8(&codepages[to]))
 		return encode_utf8(u);
-#endif /* CONFIG_UTF8 */
 
 	/* To mark non breaking spaces in non-UTF-8 strings, we use a
 	 * special char NBSP_CHAR. */
@@ -215,13 +213,8 @@ u2cp_(unicode_val_T u, int to, enum nbsp_mode nbsp_mode)
 
 static unsigned char utf_buffer[7];
 
-#ifdef CONFIG_UTF8
 inline unsigned char *
 encode_utf8(unicode_val_T u)
-#else
-static unsigned char *
-encode_utf8(unicode_val_T u)
-#endif /* CONFIG_UTF8 */
 {
 	memset(utf_buffer, 0, 7);
 

src/intl/charsets.h

@@ -125,8 +125,8 @@ const uint16_t *get_cp_highhalf(const unsigned char *);
 
 int is_cp_utf8(int);
 void free_conv_table(void);
-#ifdef CONFIG_UTF8
 inline unsigned char *encode_utf8(unicode_val_T);
+#ifdef CONFIG_UTF8
 inline unsigned char *utf8_prevchar(unsigned char *, int, unsigned char *);
 inline int utf8charlen(const unsigned char *);
 int utf8_char2cells(unsigned char *, unsigned char *);

src/intl/gettext/libintl.h

@@ -45,6 +45,10 @@ n_(unsigned char *msg1, unsigned char *msg2, unsigned long int n, struct termina
 	return gettext_noop(msg1);
 }
 
+static inline void
+intl_set_charset_by_index(int new_charset)
+{
+}
 
 #else
 
@@ -59,10 +63,8 @@ extern int current_charset;
 /* #define DEBUG_IT */
 
 static inline void
-intl_set_charset(struct terminal *term)
+intl_set_charset_by_index(int new_charset)
 {
-	int new_charset = get_terminal_codepage(term);
-
 	/* Prevent useless switching. */
 	if (current_charset != new_charset) {
 		bind_textdomain_codeset( /* PACKAGE */ "elinks",
@@ -71,6 +73,14 @@ intl_set_charset(struct terminal *term)
 	}
 }
 
+static inline void
+intl_set_charset(struct terminal *term)
+{
+	int new_charset = get_terminal_codepage(term);
+
+	intl_set_charset_by_index(new_charset);
+}
+
 /* TODO: Ideally, we should internally work only in Unicode - then the need for
  * charsets multiplexing would cease. That'll take some work yet, though.
  * --pasky */

src/osdep/osdep.c

@@ -447,9 +447,16 @@ set_window_title(unsigned char *title, int codepage)
 			    || (unicode >= 0x7F && unicode < 0xA0))
 				continue;
 
-			/* xterm entirely rejects 1024-byte or longer
+			/* If the title is getting too long, truncate
-			 * titles.  */
+			 * it and add an ellipsis.
-			if (filtered.length + charlen >= 1024 - 3) {
+			 *
+			 * xterm entirely rejects 1024-byte or longer
+			 * titles.  GNU Screen 4.00.03 misparses
+			 * titles longer than 765 bytes, and is unable
+			 * to display the title in hardstatus if the
+			 * title and other stuff together exceed 766
+			 * bytes.  So set the limit quite a bit lower.  */
+			if (filtered.length + charlen >= 600 - 3) {
 				add_to_string(&filtered, "...");
 				break;
 			}

src/scripting/smjs/bookmarks.c

@@ -8,6 +8,7 @@
 
 #include "bookmarks/bookmarks.h"
 #include "ecmascript/spidermonkey-shared.h"
+#include "intl/charsets.h"
 #include "main/event.h"
 #include "scripting/smjs/core.h"
 #include "scripting/smjs/elinks_object.h"
@@ -78,6 +79,60 @@ static const JSPropertySpec bookmark_props[] = {
 
 static JSObject *smjs_get_bookmark_folder_object(struct bookmark *bookmark);
 
+/** Convert a string retrieved from struct bookmark to a jsval.
+ *
+ * @return JS_TRUE if successful.  On error, report the error and
+ * return JS_FALSE.  */
+static JSBool
+bookmark_string_to_jsval(JSContext *ctx, const unsigned char *str, jsval *vp)
+{
+	JSString *jsstr = utf8_to_jsstring(ctx, str, -1);
+
+	if (jsstr == NULL)
+		return JS_FALSE;
+	*vp = STRING_TO_JSVAL(jsstr);
+	return JS_TRUE;
+}
+
+/** Convert a jsval to a string and store it in struct bookmark.
+ *
+ * @param ctx
+ *   Context for memory allocations and error reports.
+ * @param val
+ *   The @c jsval that should be converted.
+ * @param[in,out] result
+ *   A string allocated with mem_alloc().
+ *   On success, this function frees the original string, if any.
+ *
+ * @return JS_TRUE if successful.  On error, report the error to
+ * SpiderMonkey and return JS_FALSE.  */
+static JSBool
+jsval_to_bookmark_string(JSContext *ctx, jsval val, unsigned char **result)
+{
+	JSString *jsstr = NULL;
+	unsigned char *str;
+
+	/* jsstring_to_utf8() might GC; protect the string to come.  */
+	if (!JS_AddNamedRoot(ctx, &jsstr, "jsval_to_bookmark_string"))
+		return JS_FALSE;
+
+	jsstr = JS_ValueToString(ctx, val);
+	if (jsstr == NULL) {
+		JS_RemoveRoot(ctx, &jsstr);
+		return JS_FALSE;
+	}
+
+	str = jsstring_to_utf8(ctx, jsstr, NULL);
+	if (str == NULL) {
+		JS_RemoveRoot(ctx, &jsstr);
+		return JS_FALSE;
+	}
+
+	JS_RemoveRoot(ctx, &jsstr);
+	mem_free_set(result, str);
+	return JS_TRUE;
+}
+
 /* @bookmark_class.getProperty */
 static JSBool
 bookmark_get_property(JSContext *ctx, JSObject *obj, jsval id, jsval *vp)
@@ -102,15 +157,9 @@ bookmark_get_property(JSContext *ctx, JSObject *obj, jsval id, jsval *vp)
 
 	switch (JSVAL_TO_INT(id)) {
 	case BOOKMARK_TITLE:
-		*vp = STRING_TO_JSVAL(JS_NewStringCopyZ(smjs_ctx,
+		return bookmark_string_to_jsval(ctx, bookmark->title, vp);
-	                                                bookmark->title));
-
-		return JS_TRUE;
 	case BOOKMARK_URL:
-		*vp = STRING_TO_JSVAL(JS_NewStringCopyZ(smjs_ctx,
+		return bookmark_string_to_jsval(ctx, bookmark->url, vp);
-	                                                bookmark->url));
-
-		return JS_TRUE;
 	case BOOKMARK_CHILDREN:
 		*vp = OBJECT_TO_JSVAL(smjs_get_bookmark_folder_object(bookmark));
 
@@ -131,6 +180,9 @@ static JSBool
 bookmark_set_property(JSContext *ctx, JSObject *obj, jsval id, jsval *vp)
 {
 	struct bookmark *bookmark;
+	unsigned char *title = NULL;
+	unsigned char *url = NULL;
+	int ok;
 
 	/* This can be called if @obj if not itself an instance of the
 	 * appropriate class but has one in its prototype chain.  Fail
@@ -147,22 +199,14 @@ bookmark_set_property(JSContext *ctx, JSObject *obj, jsval id, jsval *vp)
 		return JS_FALSE;
 
 	switch (JSVAL_TO_INT(id)) {
-	case BOOKMARK_TITLE: {
+	case BOOKMARK_TITLE:
-		JSString *jsstr = JS_ValueToString(smjs_ctx, *vp);
+		if (!jsval_to_bookmark_string(ctx, *vp, &title))
-		unsigned char *str = JS_GetStringBytes(jsstr);
+			return JS_FALSE;
-
+		break;
-		mem_free_set(&bookmark->title, stracpy(str));
+	case BOOKMARK_URL:
-
+		if (!jsval_to_bookmark_string(ctx, *vp, &url))
-		return JS_TRUE;
+			return JS_FALSE;
-	}
+		break;
-	case BOOKMARK_URL: {
-		JSString *jsstr = JS_ValueToString(smjs_ctx, *vp);
-		unsigned char *str = JS_GetStringBytes(jsstr);
-
-		mem_free_set(&bookmark->url, stracpy(str));
-
-		return JS_TRUE;
-	}
 	default:
 		/* Unrecognized integer property ID; someone is using
 		 * the object as an array.  SMJS builtin classes (e.g.
@@ -170,6 +214,11 @@ bookmark_set_property(JSContext *ctx, JSObject *obj, jsval id, jsval *vp)
 		 * Do the same here.  */
 		return JS_TRUE;
 	}
+
+	ok = update_bookmark(bookmark, get_cp_index("UTF-8"), title, url);
+	mem_free_if(title);
+	mem_free_if(url);
+	return ok ? JS_TRUE : JS_FALSE;
 }
 
 static const JSClass bookmark_class = {
@@ -205,7 +254,7 @@ bookmark_folder_get_property(JSContext *ctx, JSObject *obj, jsval id, jsval *vp)
 {
 	struct bookmark *bookmark;
 	struct bookmark *folder;
-	unsigned char *title;
+	unsigned char *title = NULL;
 
 	/* This can be called if @obj if not itself an instance of the
 	 * appropriate class but has one in its prototype chain.  Fail
@@ -218,14 +267,15 @@ bookmark_folder_get_property(JSContext *ctx, JSObject *obj, jsval id, jsval *vp)
 
 	*vp = JSVAL_NULL;
 
-	title = JS_GetStringBytes(JS_ValueToString(ctx, id));
+	if (!jsval_to_bookmark_string(ctx, id, &title))
-	if (!title) return JS_TRUE;
+		return JS_FALSE;
 
 	bookmark = get_bookmark_by_name(folder, title);
 	if (bookmark) {
 		*vp = OBJECT_TO_JSVAL(smjs_get_bookmark_object(bookmark));
 	}
 
+	mem_free(title);
 	return JS_TRUE;
 }
 

src/scripting/smjs/core.c

@@ -8,6 +8,7 @@
 
 #include "config/home.h"
 #include "ecmascript/spidermonkey-shared.h"
+#include "intl/charsets.h"
 #include "main/module.h"
 #include "osdep/osdep.h"
 #include "scripting/scripting.h"
@@ -163,3 +164,163 @@ cleanup_smjs(struct module *module)
 	JS_DestroyContext(smjs_ctx);
 	spidermonkey_runtime_release();
 }
+
+/** Convert a UTF-8 string to a JSString.
+ *
+ * @param ctx
+ *   Allocate the string in this JSContext.
+ * @param[in] str
+ *   The input string that should be converted.
+ * @param[in] length
+ *   Length of @a str in bytes, or -1 if it is null-terminated.
+ *
+ * @return the new string.  On error, report the error to SpiderMonkey
+ * and return NULL.  */
+JSString *
+utf8_to_jsstring(JSContext *ctx, const unsigned char *str, int length)
+{
+	size_t in_bytes;
+	const unsigned char *in_end;
+	size_t utf16_alloc;
+	jschar *utf16;
+	size_t utf16_used;
+	JSString *jsstr;
+
+	if (length == -1)
+		in_bytes = strlen(str);
+	else
+		in_bytes = length;
+
+	/* Each byte of input can become at most one UTF-16 unit.
+	 * Check whether the multiplication could overflow.  */
+	assert(!needs_utf16_surrogates(UCS_REPLACEMENT_CHARACTER));
+	if (in_bytes > ((size_t) -1) / sizeof(jschar)) {
+		JS_ReportAllocationOverflow(ctx);
+		return NULL;
+	}
+	utf16_alloc = in_bytes;
+	/* Don't use fmem_alloc here because long strings could
+	 * exhaust the stack.  */
+	utf16 = mem_alloc(utf16_alloc * sizeof(jschar));
+	if (utf16 == NULL) {
+		JS_ReportOutOfMemory(ctx);
+		return NULL;
+	}
+
+	in_end = str + in_bytes;
+
+	utf16_used = 0;
+	for (;;) {
+		unicode_val_T unicode;
+
+		unicode = utf8_to_unicode((unsigned char **) &str, in_end);
+		if (unicode == UCS_NO_CHAR)
+			break;
+
+		if (needs_utf16_surrogates(unicode)) {
+			assert(utf16_alloc - utf16_used >= 2);
+			if_assert_failed { mem_free(utf16); return NULL; }
+			utf16[utf16_used++] = get_utf16_high_surrogate(unicode);
+			utf16[utf16_used++] = get_utf16_low_surrogate(unicode);
+		} else {
+			assert(utf16_alloc - utf16_used >= 1);
+			if_assert_failed { mem_free(utf16); return NULL; }
+			utf16[utf16_used++] = unicode;
+		}
+	}
+
+	jsstr = JS_NewUCString(ctx, utf16, utf16_used);
+	mem_free(utf16);
+	return jsstr;
+}
+
+/** Convert a jschar array to UTF-8 and append it to struct string.
+ * Replace misused surrogate codepoints with UCS_REPLACEMENT_CHARACTER.
+ *
+ * @param[in,out] utf8
+ *   The function appends characters to this UTF-8 string.
+ *
+ * @param[in] utf16
+ *   Pointer to the first element in an array of jschars.
+ *
+ * @param[i] len
+ *   Number of jschars in the @a utf16 array.
+ *
+ * @return @a utf8 if successful, or NULL if not.  */
+static struct string *
+add_jschars_to_utf8_string(struct string *utf8,
+			   const jschar *utf16, size_t len)
+{
+	size_t pos;
+
+	for (pos = 0; pos < len; ) {
+		unicode_val_T unicode = utf16[pos++];
+
+		if (is_utf16_surrogate(unicode)) {
+			if (is_utf16_high_surrogate(unicode)
+			    && pos < len
+			    && is_utf16_low_surrogate(utf16[pos])) {
+				unicode = join_utf16_surrogates(unicode,
+								utf16[pos++]);
+			} else {
+				unicode = UCS_REPLACEMENT_CHARACTER;
+			}
+		}
+
+		if (unicode == 0) {
+			if (!add_char_to_string(utf8, '\0'))
+				return NULL;
+		} else {
+			if (!add_to_string(utf8, encode_utf8(unicode)))
+				return NULL;
+		}
+	}
+
+	return utf8;
+}
+
+/** Convert a JSString to a UTF-8 string.
+ *
+ * @param ctx
+ *   For reporting errors.
+ * @param[in] jsstr
+ *   The input string that should be converted.  Must not be NULL.
+ * @param[out] length
+ *   Optional.  The number of bytes in the returned string,
+ *   not counting the terminating null.
+ *
+ * @return the new string, which the caller must eventually free
+ * with mem_free().  On error, report the error to SpiderMonkey
+ * and return NULL; *@a length is then undefined.  */
+unsigned char *
+jsstring_to_utf8(JSContext *ctx, JSString *jsstr, int *length)
+{
+	size_t utf16_len;
+	const jschar *utf16;
+	struct string utf8;
+
+	utf16_len = JS_GetStringLength(jsstr);
+	utf16 = JS_GetStringChars(jsstr); /* stays owned by jsstr */
+	if (utf16 == NULL) {
+		/* JS_GetStringChars doesn't have a JSContext *
+		 * parameter so it can't report the error
+		 * (and can't collect garbage either).  */
+		JS_ReportOutOfMemory(ctx);
+		return NULL;
+	}
+
+	if (!init_string(&utf8)) {
+		JS_ReportOutOfMemory(ctx);
+		return NULL;
+	}
+
+	if (!add_jschars_to_utf8_string(&utf8, utf16, utf16_len)) {
+		done_string(&utf8);
+		JS_ReportOutOfMemory(ctx);
+		return NULL;
+	}
+
+	if (length)
+		*length = utf8.length;
+	return utf8.source;
+}

src/scripting/smjs/core.h

@@ -16,4 +16,9 @@ void alert_smjs_error(unsigned char *msg);
 void init_smjs(struct module *module);
 void cleanup_smjs(struct module *module);
 
+JSString *utf8_to_jsstring(JSContext *ctx, const unsigned char *str,
+			   int length);
+unsigned char *jsstring_to_utf8(JSContext *ctx, JSString *jsstr,
+				int *length);
+
 #endif

src/session/session.c

@@ -825,10 +825,13 @@ setup_first_session(struct session *ses, struct uri *uri)
 
 #ifdef CONFIG_BOOKMARKS
 	} else if (!uri && get_opt_bool("ui.sessions.auto_restore", NULL)) {
-		unsigned char *folder;
+		unsigned char *folder; /* UTF-8 */
 
-		folder = get_opt_str("ui.sessions.auto_save_foldername", NULL);
+		folder = get_auto_save_bookmark_foldername_utf8();
-		open_bookmark_folder(ses, folder);
+		if (folder) {
+			open_bookmark_folder(ses, folder);
+			mem_free(folder);
+		}
 		return 1;
 #endif
 	}
@@ -972,8 +975,16 @@ init_remote_session(struct session *ses, enum remote_session_flags *remote_ptr,
 
 	} else if (remote & SES_REMOTE_ADD_BOOKMARK) {
 #ifdef CONFIG_BOOKMARKS
+		int uri_cp;
+
 		if (!uri) return;
-		add_bookmark(NULL, 1, struri(uri), struri(uri));
+		/** @todo Bug 1066: What is the encoding of struri()?
+		 * This code currently assumes the system charset.
+		 * It might be best to keep URIs in plain ASCII and
+		 * then have a function that reversibly converts them
+		 * to IRIs for display in a given encoding.  */
+		uri_cp = get_cp_index("System");
+		add_bookmark_cp(NULL, 1, uri_cp, struri(uri), struri(uri));
 #endif
 
 	} else if (remote & SES_REMOTE_INFO_BOX) {

src/terminal/event.c

@@ -387,10 +387,7 @@ handle_interlink_event(struct terminal *term, struct interlink_event *ilev)
 				/* Not special and UTF-8 mode is disabled:
 				 * recode from the terminal charset to UCS-4. */
 
-				key = cp2u(get_opt_codepage_tree(term->spec,
+				key = cp2u(get_terminal_codepage(term), key);
-								 "charset",
-								 NULL),
-					   key);
 				term_send_ucs(term, key, modifier);
 				break;
 			}

src/util/conv.c

@@ -492,7 +492,10 @@ clr_spaces(unsigned char *str)
 }
 
 /** Replace invalid chars in @a title with ' ' and trim all starting/ending
- * spaces. */
+ * spaces.
+ *
+ * update_bookmark() assumes this function does not switch translation
+ * tables.  */
 void
 sanitize_title(unsigned char *title)
 {

src/viewer/text/Makefile

@@ -1,6 +1,8 @@
 top_builddir=../../..
 include $(top_builddir)/Makefile.config
 
+INCLUDES += $(TRE_CFLAGS)
+
 OBJS-$(CONFIG_MARKS) += marks.o
 
 OBJS = draw.o form.o link.o search.o textarea.o view.o vs.o

src/viewer/text/search.c

@@ -16,11 +16,11 @@
 #endif
 
 #include <sys/types.h> /* FreeBSD needs this before regex.h */
-#ifdef HAVE_REGEX_H
-#include <regex.h>
-#endif
 #include <stdlib.h>
 #include <string.h>
+#ifdef HAVE_TRE_REGEX_H
+#include <tre/regex.h>
+#endif
 
 #include "elinks.h"
 
@@ -54,10 +54,18 @@ static INIT_INPUT_HISTORY(search_history);
 #undef UCHAR
 #ifdef CONFIG_UTF8
 #define UCHAR unicode_val_T
+#define PATTERN const wchar_t
+#define Regcomp regwcomp
+#define Regexec regwexec
 #else
 #define UCHAR unsigned char
+#define PATTERN const char
+#define Regcomp regcomp
+#define Regexec regexec
 #endif
 
+static UCHAR *memacpy_u(unsigned char *text, int textlen, int utf8);
+
 static inline void
 add_srch_chr(struct document *document, UCHAR c, int x, int y, int nn)
 {
@@ -262,21 +270,21 @@ get_range(struct document *document, int y, int height, int l,
 	return 0;
 }
 
-#ifdef HAVE_REGEX_H
+#ifdef HAVE_TRE_REGEX_H
 /** Returns a string @c doc that is a copy of the text in the search
  * nodes from @a s1 to (@a s1 + @a doclen - 1) with the space at the
  * end of each line converted to a new-line character (LF). */
-static unsigned char *
+static UCHAR *
 get_search_region_from_search_nodes(struct search *s1, struct search *s2,
 				    int pattern_len, int *doclen)
 {
-	unsigned char *doc;
+	UCHAR *doc;
 	int i;
 
 	*doclen = s2 - s1 + pattern_len;
 	if (!*doclen) return NULL;
 
-	doc = mem_alloc(*doclen + 1);
+	doc = mem_alloc((*doclen + 1) * sizeof(UCHAR));
 	if (!doc) {
 		*doclen = -1;
 		return NULL;
@@ -301,11 +309,11 @@ struct regex_match_context {
 	int y1;
 	int y2;
 	int found;
-	unsigned char *pattern;
+	UCHAR *pattern;
 };
 
 static int
-init_regex(regex_t *regex, unsigned char *pattern)
+init_regex(regex_t *regex, UCHAR *pattern)
 {
 	int regex_flags = REG_NEWLINE;
 	int reg_err;
@@ -316,7 +324,7 @@ init_regex(regex_t *regex, unsigned char *pattern)
 	if (!get_opt_bool("document.browse.search.case", NULL))
 		regex_flags |= REG_ICASE;
 
-	reg_err = regcomp(regex, pattern, regex_flags);
+	reg_err = Regcomp(regex, (PATTERN *)pattern, regex_flags);
 	if (reg_err) {
 		regfree(regex);
 		return 0;
@@ -329,8 +337,8 @@ static void
 search_for_pattern(struct regex_match_context *common_ctx, void *data,
 		   void (*match)(struct regex_match_context *, void *))
 {
-	unsigned char *doc;
+	UCHAR *doc;
-	unsigned char *doctmp;
+	UCHAR *doctmp;
 	int doclen;
 	int regexec_flags = 0;
 	regex_t regex;
@@ -381,7 +389,7 @@ find_next:
 	save_c = doc[pos];
 	doc[pos] = 0;
 
-	while (*doctmp && !regexec(&regex, doctmp, 1, &regmatch, regexec_flags)) {
+	while (*doctmp && !Regexec(&regex, (PATTERN *)doctmp, 1, &regmatch, regexec_flags)) {
 		regexec_flags = REG_NOTBOL;
 		common_ctx->textlen = regmatch.rm_eo - regmatch.rm_so;
 		if (!common_ctx->textlen) { doc[pos] = save_c; common_ctx->found = 1; goto free_stuff; }
@@ -432,10 +440,13 @@ static int
 is_in_range_regex(struct document *document, int y, int height,
 		  unsigned char *text, int textlen,
 		  int *min, int *max,
-		  struct search *s1, struct search *s2)
+		  struct search *s1, struct search *s2, int utf8)
 {
 	struct regex_match_context common_ctx;
 	struct is_in_range_regex_context ctx;
+	UCHAR *txt = memacpy_u(text, textlen, utf8);
+
+	if (!txt) return -1;
 
 	ctx.y = y;
 	ctx.min = min;
@@ -445,15 +456,16 @@ is_in_range_regex(struct document *document, int y, int height,
 	common_ctx.textlen = textlen;
 	common_ctx.y1 = y - 1;
 	common_ctx.y2 = y + height;
-	common_ctx.pattern = text;
+	common_ctx.pattern = txt;
 	common_ctx.s1 = s1;
 	common_ctx.s2 = s2;
 
 	search_for_pattern(&common_ctx, &ctx, is_in_range_regex_match);
+	mem_free(txt);
 
 	return common_ctx.found;
 }
-#endif /* HAVE_REGEX_H */
+#endif /* HAVE_TRE_REGEX_H */
 
 static UCHAR *
 memacpy_u(unsigned char *text, int textlen, int utf8)
@@ -590,10 +602,10 @@ is_in_range(struct document *document, int y, int height,
 	if (get_range(document, y, height, textlen, &s1, &s2))
 		return 0;
 
-#ifdef HAVE_REGEX_H
+#ifdef HAVE_TRE_REGEX_H
 	if (get_opt_int("document.browse.search.regex", NULL))
 		return is_in_range_regex(document, y, height, text, textlen,
-					 min, max, s1, s2);
+					 min, max, s1, s2, utf8);
 #endif
 	return is_in_range_plain(document, y, height, text, textlen,
 				 min, max, s1, s2, utf8);
@@ -669,7 +681,7 @@ srch_failed:
 	*pl = len;
 }
 
-#ifdef HAVE_REGEX_H
+#ifdef HAVE_TRE_REGEX_H
 struct get_searched_regex_context {
 	int xoffset;
 	int yoffset;
@@ -709,10 +721,13 @@ get_searched_regex_match(struct regex_match_context *common_ctx, void *data)
 
 static void
 get_searched_regex(struct document_view *doc_view, struct point **pt, int *pl,
-		   int textlen, struct search *s1, struct search *s2)
+		   int textlen, struct search *s1, struct search *s2, int utf8)
 {
 	struct regex_match_context common_ctx;
 	struct get_searched_regex_context ctx;
+	UCHAR *txt = memacpy_u(*doc_view->search_word, textlen, utf8);
+
+	if (!txt) return;
 
 	ctx.points = NULL;
 	ctx.len = 0;
@@ -724,16 +739,17 @@ get_searched_regex(struct document_view *doc_view, struct point **pt, int *pl,
 	common_ctx.textlen = textlen;
 	common_ctx.y1 = doc_view->vs->y - 1;
 	common_ctx.y2 = doc_view->vs->y + ctx.box->height;
-	common_ctx.pattern = *doc_view->search_word;
+	common_ctx.pattern = txt;
 	common_ctx.s1 = s1;
 	common_ctx.s2 = s2;
 
 	search_for_pattern(&common_ctx, &ctx, get_searched_regex_match);
 
+	mem_free(txt);
 	*pt = ctx.points;
 	*pl = ctx.len;
 }
-#endif /* HAVE_REGEX_H */
+#endif /* HAVE_TRE_REGEX_H */
 
 static void
 get_searched(struct document_view *doc_view, struct point **pt, int *pl, int utf8)
@@ -757,9 +773,9 @@ get_searched(struct document_view *doc_view, struct point **pt, int *pl, int utf
 		return;
 	}
 
-#ifdef HAVE_REGEX_H
+#ifdef HAVE_TRE_REGEX_H
 	if (get_opt_int("document.browse.search.regex", NULL))
-		get_searched_regex(doc_view, pt, pl, l, s1, s2);
+		get_searched_regex(doc_view, pt, pl, l, s1, s2, utf8);
 	else
 #endif
 		get_searched_plain(doc_view, pt, pl, l, s1, s2, utf8);
@@ -1576,7 +1592,7 @@ search_typeahead(struct session *ses, struct document_view *doc_view,
  * a nice cleanup target ;-). --pasky */
 
 enum search_option {
-#ifdef HAVE_REGEX_H
+#ifdef HAVE_TRE_REGEX_H
 	SEARCH_OPT_REGEX,
 #endif
 	SEARCH_OPT_CASE,
@@ -1584,7 +1600,7 @@ enum search_option {
 };
 
 static struct option_resolver resolvers[] = {
-#ifdef HAVE_REGEX_H
+#ifdef HAVE_TRE_REGEX_H
 	{ SEARCH_OPT_REGEX,	"regex" },
 #endif
 	{ SEARCH_OPT_CASE,	"case" },
@@ -1651,7 +1667,7 @@ search_dlg_do(struct terminal *term, struct memory_list *ml,
 			       hop->values, SEARCH_OPTIONS);
 	hop->data = data;
 
-#ifdef HAVE_REGEX_H
+#ifdef HAVE_TRE_REGEX_H
 #define SEARCH_WIDGETS_COUNT 8
 #else
 #define SEARCH_WIDGETS_COUNT 5
@@ -1675,7 +1691,7 @@ search_dlg_do(struct terminal *term, struct memory_list *ml,
 	field = get_dialog_offset(dlg, SEARCH_WIDGETS_COUNT);
 	add_dlg_field(dlg, text, 0, 0, NULL, MAX_STR_LEN, field, history);
 
-#ifdef HAVE_REGEX_H
+#ifdef HAVE_TRE_REGEX_H
 	add_dlg_radio(dlg, _("Normal search", term), 1, 0, &hop->values[SEARCH_OPT_REGEX].number);
 	add_dlg_radio(dlg, _("Regexp search", term), 1, 1, &hop->values[SEARCH_OPT_REGEX].number);
 	add_dlg_radio(dlg, _("Extended regexp search", term), 1, 2, &hop->values[SEARCH_OPT_REGEX].number);