terminal UTF-8: term_event_keyboard.key is UCS-4, #ifdef CONFIG_UTF_8.

Form fields and BFU text-input widgets then convert from UCS-4 to UTF-8.
If not all UTF-8 bytes fit, they don't insert anything.  Thus it is no
longer possible to get invalid UTF-8 by hitting the length limit.

It is unclear to me which charset is supposed to be used for strings
in internal buffers.  I made BFU insert UTF-8 whenever CONFIG_UTF_8,
but form fields use the charset of the terminal; that may have to be
changed.

As a side effect, this change should solve bug 782, because
term_send_ucs no longer encodes in UTF-8 if CONFIG_UTF_8 is defined.
I think the UTF-8 and codepage encoding calls I added are safe, too.
A similar bug may still surface somewhere else, but 782 could be
closed for now.

This change also lays the foundation for binding actions to non-ASCII
keys, but the keystroke name parser doesn't yet support that.
The CONFIG_UTF_8 mode does not currently support non-ASCII characters
in hot keys, either.

src/bfu/inpfield.c

@@ -680,32 +680,27 @@ kbd_field(struct dialog_data *dlg_data, struct widget_data *widget_data)
 			if (check_kbd_textinput_key(ev)) {
 				unsigned char *text = widget_data->cdata;
 				int textlen = strlen(text);
+#ifdef CONFIG_UTF_8
+				const unsigned char *ins = encode_utf_8(get_kbd_key(ev));
+				int inslen = utf8charlen(ins);
+#else  /* !CONFIG_UTF_8 */
+				const int inslen = 1;
+#endif /* !CONFIG_UTF_8 */
 
-				if (textlen >= widget_data->widget->datalen - 1)
+				if (textlen >= widget_data->widget->datalen - inslen)
 					goto display_field;
 
 				/* Shift to position of the cursor */
 				textlen -= widget_data->info.field.cpos;
-				text	+= widget_data->info.field.cpos++;
+				text	+= widget_data->info.field.cpos;
 
-				memmove(text + 1, text, textlen + 1);
+				memmove(text + inslen, text, textlen + 1);
-				*text = get_kbd_key(ev);
 #ifdef CONFIG_UTF_8
-				if (term->utf8) {
+				memcpy(text, ins, inslen);
-					static unsigned char buf[7];
+#else  /* !CONFIG_UTF_8 */
-					unsigned char *t = buf;
+				*text = get_kbd_key(ev);
-					static int i = 0;
+#endif /* !CONFIG_UTF_8 */
-					unicode_val_T data;
+				widget_data->info.field.cpos += inslen;
-
-					buf[i++] = *text;
-					buf[i] = '\0';
-					data = utf_8_to_unicode(&t, buf + i);
-					if (i == 6) i = 0;
-					if (data == UCS_NO_CHAR)
-						return EVENT_PROCESSED;
-					else i = 0;
-				}
-#endif /* CONFIG_UTF_8 */
 				goto display_field;
 			}
 	}

src/terminal/event.c

@@ -129,9 +129,15 @@ term_send_event(struct terminal *term, struct term_event *ev)
 static void
 term_send_ucs(struct terminal *term, unicode_val_T u, int modifier)
 {
-	unsigned char *recoded;
+#ifdef CONFIG_UTF_8
 	struct term_event ev;
 
+	set_kbd_term_event(&ev, u, modifier);
+	term_send_event(term, &ev);
+#else  /* !CONFIG_UTF_8 */
+	struct term_event ev;
+	unsigned char *recoded;
+
 	set_kbd_term_event(&ev, KBD_UNDEF, modifier);
 	recoded = u2cp_no_nbsp(u, get_opt_codepage_tree(term->spec, "charset"));
 	if (!recoded) recoded = "*";
@@ -140,6 +146,7 @@ term_send_ucs(struct terminal *term, unicode_val_T u, int modifier)
 		term_send_event(term, &ev);
 		recoded++;
 	}
+#endif /* !CONFIG_UTF_8 */
 }
 
 static void
@@ -282,18 +289,14 @@ handle_interlink_event(struct terminal *term, struct interlink_event *ilev)
 
 		/* Character Conversions.  */
 #ifdef CONFIG_UTF_8
-		/* struct term_event_keyboard carries bytes in the
+		/* struct term_event_keyboard carries UCS-4.
-		 * charset of the terminal.
 		 * - If the "utf_8_io" option (i.e. term->utf8) is
 		 *   true or the "charset" option refers to UTF-8,
 		 *   then handle_interlink_event() converts from UTF-8
-		 *   to UCS-4, and term_send_ucs() converts from UCS-4
+		 *   to UCS-4.
-		 *   to the codepage specified with the "charset" option.
 		 * - Otherwise, handle_interlink_event() converts from
 		 *   the codepage specified with the "charset" option
-		 *   to UCS-4, and term_send_ucs() converts right back.
+		 *   to UCS-4.  */
-		 * TO DO: Change struct term_event_keyboard to carry
-		 * UCS-4 instead, reducing these conversions.  */
 		utf8_io = term->utf8
 			|| is_cp_utf8(get_opt_codepage_tree(term->spec, "charset"));
 #else

src/terminal/event.h

@@ -165,8 +165,18 @@ void in_term(struct terminal *);
 #define get_kbd_modifier(event)		(kbd_get_modifier(&(event)->info.keyboard))
 #define check_kbd_modifier(event, mod)	(kbd_modifier_is(&(event)->info.keyboard, (mod)))
 
-#define check_kbd_textinput_key(event)	(get_kbd_key(event) >= ' ' && get_kbd_key(event) < 256 && check_kbd_modifier(event, KBD_MOD_NONE))
+#define check_kbd_textinput_key(event)	(get_kbd_key(event) >= ' ' && check_kbd_modifier(event, KBD_MOD_NONE))
-#define check_kbd_label_key(event)	(get_kbd_key(event) > ' ' && get_kbd_key(event) < 256)
+#ifdef CONFIG_UTF_8
+/* We must currently limit hotkeys of labels to ASCII, because
+ * get_kbd_key(event) is in UCS-4 and various event handlers pass it
+ * to toupper() if check_kbd_label_key() returns true.
+ * TO DO: Change the event handlers to use unicode_fold_label_case()
+ * instead.  The code that extracts the hotkey from the label string
+ * will also have to be changed.  */
+#define check_kbd_label_key(event)	(get_kbd_key(event) > ' ' && get_kbd_key(event) <= 0x7F)
+#else  /* !CONFIG_UTF_8 */
+#define check_kbd_label_key(event)	(get_kbd_key(event) > ' ')
+#endif /* !CONFIG_UTF_8 */
 
 
 /* For mouse events handling */

src/terminal/kbd.h

@@ -7,8 +7,8 @@ struct term_event_keyboard {
 	/* Values <= -0x100 are special; e.g. KBD_ENTER.
 	 * Values between -0xFF and -2 are not used yet; treat as special.
 	 * Value  == -1 is KBD_UNDEF; not sent via socket. 
-	 * Values between 0 and 0xFF are bytes received from the terminal.
+	 * Values >= 0 are characters received from the terminal;
-	 * Values >= 0x100 are not used.  */
+	 * in UCS-4 #ifdef CONFIG_UTF_8. */
 	int key;
 	int modifier;
 };

src/viewer/text/form.c

@@ -1712,8 +1712,8 @@ field_op(struct session *ses, struct document_view *doc_view,
 			}
 
 			if (form_field_is_readonly(fc)
-					|| strlen(fs->value) >= fc->maxlength
 #ifndef CONFIG_UTF_8
+					|| strlen(fs->value) >= fc->maxlength
 					|| !insert_in_string(&fs->value, fs->state, "?", 1)
 #endif /* CONFIG_UTF_8 */
 					)
@@ -1721,42 +1721,31 @@ field_op(struct session *ses, struct document_view *doc_view,
 				status = FRAME_EVENT_OK;
 				break;
 			}
-#ifdef CONFIG_UTF_8
-			if (utf8) {
-				static unsigned char buf[7];
-				static int i = 0;
-				unicode_val_T data;
-				unsigned char *t;
 
-				t = buf;
+#ifdef CONFIG_UTF_8
-				buf[i++] = get_kbd_key(ev);
+			{
-				buf[i] = 0;
+				/* The charset of the terminal; we assume
-				data = utf_8_to_unicode(&t, buf + i);
+				 * fs->value is in this charset.
-				if (data != UCS_NO_CHAR) {
+				 * (Is that OK?)  */
-					if (!insert_in_string(&fs->value, fs->state, buf, i)) {
+				int cp = get_opt_codepage_tree(ses->tab->term->spec,
-					    	i = 0;
+							       "charset");
-						return FRAME_EVENT_OK;
+
-					}
+				text = u2cp_no_nbsp(get_kbd_key(ev), cp);
-					fs->state += i;
+				length = strlen(text);
-					if (fc->type == FC_PASSWORD)
+
-						fs->state_cell++;
+				if (strlen(fs->value) + length > fc->maxlength
-					else if (fc->type == FC_TEXTAREA)
+				    || !insert_in_string(&fs->value, fs->state, text, length)) {
-						fs->state_cell = 0;
+					status = FRAME_EVENT_OK;
-					else
-						fs->state_cell += unicode_to_cell(data);
-					i = 0;
 					break;
 				}
 
-				if (i == 6) {
+				fs->state += length;
-					i = 0;
+				if (fc->type == FC_PASSWORD)
-				}
+					fs->state_cell += (is_cp_utf8(cp) ? 1 : length);
-				return FRAME_EVENT_OK;
+				else if (fc->type == FC_TEXTAREA)
-
+					fs->state_cell = 0;
-			} else {
+				else
-				if (!insert_in_string(&fs->value, fs->state, "?", 1))
+					fs->state_cell += (is_cp_utf8(cp) ? unicode_to_cell(get_kbd_key(ev)) : length);
-					return FRAME_EVENT_OK;
-				fs->value[fs->state++] = get_kbd_key(ev);
 			}
 #else
 			fs->value[fs->state++] = get_kbd_key(ev);