Convert strings to UTF-8 if possible during file save and load

.gitignore

@@ -13,6 +13,7 @@ Makefile.in
 /configure
 /stamp-h1
 
+/build-aux/compile
 /build-aux/config.guess
 /build-aux/config.rpath
 /build-aux/config.sub
@@ -22,6 +23,7 @@ Makefile.in
 
 /build-aux/snippet/arg-nonnull.h
 /build-aux/snippet/c++defs.h
+/build-aux/snippet/unused-parameter.h
 /build-aux/snippet/warn-on-use.h
 /build-aux/snippet/_Noreturn.h
 

configure.ac

@@ -50,6 +50,9 @@ AX_C___ATTRIBUTE__
 AC_TYPE_SIZE_T
 AC_TYPE_SSIZE_T
 
+AM_GNU_GETTEXT([external])
+AM_GNU_GETTEXT_VERSION([0.18.1])
+
 gl_INIT
 
 AX_WITH_CURSES
@@ -57,9 +60,6 @@ AS_IF([test "x$ax_cv_curses" != xyes || test "x$ax_cv_curses_color" != xyes], [
 	AC_MSG_ERROR([requires an X/Open-compatible Curses library with colour])
 ])
 
-AM_GNU_GETTEXT([external])
-AM_GNU_GETTEXT_VERSION([0.18.1])
-
 AC_CONFIG_FILES([
 	Makefile
 	lib/Makefile

lib/.gitignore

@@ -2,6 +2,11 @@ Makefile.am
 alloca.in.h
 asnprintf.c
 ctype.in.h
+c-ctype.c
+c-ctype.h
+c-strcase.h
+c-strcasecmp.c
+c-strncasecmp.c
 dosname.h
 errno.in.h
 float.c
@@ -19,6 +24,15 @@ getopt1.c
 getopt_int.h
 gettext.h
 gettimeofday.c
+iconv.c
+iconv.in.h
+iconv_close.c
+iconv_open-aix.gperf
+iconv_open-hpux.gperf
+iconv_open-irix.gperf
+iconv_open-osf.gperf
+iconv_open-solaris.gperf
+iconv_open.c
 isnan.c
 isnand.c
 isnand-nolibm.h
@@ -26,6 +40,7 @@ isnanf.c
 isnanf-nolibm.h
 isnanl.c
 isnanl-nolibm.h
+langinfo.in.h
 locale.in.h
 malloc.c
 math.in.h
@@ -54,6 +69,8 @@ stdio.in.h
 stdio-impl.h
 stdlib.in.h
 strdup.c
+striconv.c
+striconv.h
 string.in.h
 strncat.c
 strstr.c
@@ -62,6 +79,8 @@ sys_stat.in.h
 sys_time.in.h
 time.in.h
 unistd.in.h
+unistr.in.h
+unitypes.in.h
 vasnprintf.c
 vasnprintf.h
 verify.h
@@ -75,6 +94,13 @@ arg-nonnull.h
 c++defs.h
 ctype.h
 getopt.h
+iconv.h
+iconv_open-aix.h
+iconv_open-hpux.h
+iconv_open-irix.h
+iconv_open-osf.h
+iconv_open-solaris.h
+langinfo.h
 locale.h
 math.h
 stdio.h
@@ -83,5 +109,13 @@ string.h
 sys
 time.h
 unistd.h
+unistr.h
+unitypes.h
+unused-parameter.h
 warn-on-use.h
 wchar.h
+
+unistr/.dirstamp
+unistr/u8-mbtoucr.c
+unistr/u8-uctomb-aux.c
+unistr/u8-uctomb.c

m4/.gitignore

@@ -25,7 +25,10 @@ gnulib-common.m4
 gnulib-comp.m4
 gnulib-tool.m4
 iconv.m4
+iconv_h.m4
+iconv_open.m4
 include_next.m4
+inline.m4
 intdiv0.m4
 intldir.m4
 intl.m4
@@ -37,12 +40,14 @@ inttypes_h.m4
 isnand.m4
 isnanf.m4
 isnanl.m4
+langinfo_h.m4
 largefile.m4
 lcmessage.m4
 ldexpl.m4
 lib-ld.m4
 lib-link.m4
 lib-prefix.m4
+libunistring-base.m4
 locale_h.m4
 lock.m4
 longlong.m4

m4/gnulib-cache.m4

@@ -15,7 +15,7 @@
 
 
 # Specification in the form of a command-line invocation:
-#   gnulib-tool --import --dir=. --lib=libgnu --source-base=lib --m4-base=m4 --doc-base=doc --tests-base=tests --aux-dir=build-aux --no-conditional-dependencies --no-libtool --macro-prefix=gl assert config-h ctype fprintf-posix getopt-gnu gettext gettext-h gettimeofday locale printf-posix snprintf-posix stat stdarg stdbool stdio strdup-posix string strncat strstr sys_stat sys_time unistd vfprintf-posix vsnprintf-posix
+#   gnulib-tool --import --dir=. --lib=libgnu --source-base=lib --m4-base=m4 --doc-base=doc --tests-base=tests --aux-dir=build-aux --no-conditional-dependencies --no-libtool --macro-prefix=gl assert config-h ctype fprintf-posix getopt-gnu gettext gettext-h gettimeofday langinfo locale printf-posix snprintf-posix stat stdarg stdbool stdio strdup-posix striconv string strncat strstr sys_stat sys_time unistd vfprintf-posix vsnprintf-posix
 
 # Specification in the form of a few gnulib-tool.m4 macro invocations:
 gl_LOCAL_DIR([])
@@ -28,6 +28,7 @@ gl_MODULES([
   gettext
   gettext-h
   gettimeofday
+  langinfo
   locale
   printf-posix
   snprintf-posix
@@ -36,6 +37,7 @@ gl_MODULES([
   stdbool
   stdio
   strdup-posix
+  striconv
   string
   strncat
   strstr

src/Makefile.am

@@ -43,6 +43,6 @@ trader_SOURCES	= \
 
 trader_CPPFLAGS	= -I$(top_builddir)/lib -I$(top_srcdir)/lib	\
 		  -DLOCALEDIR=\"$(localedir)\"
-trader_LDADD	= @CURSES_LIB@ $(top_builddir)/lib/libgnu.a @LIBINTL@
+trader_LDADD	= @CURSES_LIB@ @LIBICONV@ $(top_builddir)/lib/libgnu.a @LIBINTL@
 
 EXTRA_DIST	= README

src/fileio.c

@@ -52,7 +52,8 @@ static const unsigned char game_file_crypt_key[] = {
 #define load_game_scanf(_fmt, _var, _cond)				\
     do {								\
 	if (fgets(buf, BUFSIZE, file) == NULL) {			\
-	    err_exit(_("%s: missing field on line %d"), filename, lineno);  \
+	    err_exit(_("%s: missing field on line %d"),			\
+		     filename, lineno);					\
 	}								\
 	if (sscanf(unscramble(crypt_key, buf, BUFSIZE), _fmt "\n",	\
 		   &(_var)) != 1) {					\
@@ -81,32 +82,76 @@ static const unsigned char game_file_crypt_key[] = {
 	(_var) = b;							\
     } while (0)
 
-#define load_game_read_string(_var)					\
+#ifdef USE_UTF8_GAME_FILE
+#  define load_game_read_string(_var)					\
     do {								\
 	char *s;							\
 	int len;							\
 									\
 	if (fgets(buf, BUFSIZE, file) == NULL) {			\
-	    err_exit(_("%s: missing field on line %d"), filename, lineno);  \
+	    err_exit(_("%s: missing field on line %d"),			\
+		     filename, lineno);					\
 	}								\
 	if (strlen(unscramble(crypt_key, buf, BUFSIZE)) == 0) {		\
-	    err_exit(_("%s: illegal value on line %d"), filename, lineno);  \
+	    err_exit(_("%s: illegal value on line %d"),			\
+		     filename, lineno);					\
 	}								\
-	lineno++;							\
+	if (need_icd) {							\
-									\
+	    s = str_cd_iconv(buf, icd);					\
-	s = malloc(strlen(buf) + 1);					\
+	    if (s == NULL) {						\
-	if (s == NULL) {						\
+		if (errno == EILSEQ) {					\
-	    err_exit_nomem();						\
+		    err_exit(_("%s: illegal characters on line %d"),	\
+			     filename, lineno);				\
+		} else {						\
+		    errno_exit("str_cd_iconv()");			\
+		}							\
+	    }								\
+	} else {							\
+	    s = malloc(strlen(buf) + 1);				\
+	    if (s == NULL) {						\
+		err_exit_nomem();					\
+	    }								\
+	    strcpy(s, buf);						\
 	}								\
 									\
-	strcpy(s, buf);							\
 	len = strlen(s);						\
 	if (len > 0 && s[len - 1] == '\n') {				\
 	    s[len - 1] = '\0';						\
 	}								\
 									\
+	lineno++;							\
 	(_var) = s;							\
     } while (0)
+#else // ! USE_UTF8_GAME_FILE
+#  define load_game_read_string(_var)					\
+    do {								\
+	char *s;							\
+	int len;							\
+									\
+	if (fgets(buf, BUFSIZE, file) == NULL) {			\
+	    err_exit(_("%s: missing field on line %d"),			\
+		     filename, lineno);					\
+	}								\
+	if (strlen(unscramble(crypt_key, buf, BUFSIZE)) == 0) {		\
+	    err_exit(_("%s: illegal value on line %d"),			\
+		     filename, lineno);					\
+	}								\
+									\
+	s = malloc(strlen(buf) + 1);					\
+	if (s == NULL) {						\
+	    err_exit_nomem();						\
+	}								\
+	strcpy(s, buf);							\
+									\
+	len = strlen(s);						\
+	if (len > 0 && s[len - 1] == '\n') {				\
+	    s[len - 1] = '\0';						\
+	}								\
+									\
+	lineno++;							\
+	(_var) = s;							\
+    } while (0)
+#endif // ! USE_UTF8_GAME_FILE
 
 
 // Macros used in save_game()
@@ -126,8 +171,30 @@ static const unsigned char game_file_crypt_key[] = {
     save_game_printf("%2.20e", _var)
 #define save_game_write_bool(_var)					\
     save_game_printf("%d", (int) _var)
-#define save_game_write_string(_var)					\
+
+#ifdef USE_UTF8_GAME_FILE
+#  define save_game_write_string(_var)					\
+    do {								\
+	if (need_icd) {							\
+	    char *s = str_cd_iconv(_var, icd);				\
+	    if (s == NULL) {						\
+		if (errno == EILSEQ) {					\
+		    err_exit(_("%s: could not convert string"),		\
+			     filename);					\
+		} else {						\
+		    errno_exit("str_cd_iconv()");			\
+		}							\
+	    }								\
+	    save_game_printf("%s", s);					\
+	    free(s);							\
+	} else {							\
+	    save_game_printf("%s", _var);				\
+	}								\
+    } while (0)
+#else // ! USE_UTF8_GAME_FILE
+#  define save_game_write_string(_var)					\
     save_game_printf("%s", _var)
+#endif // ! USE_UTF8_GAME_FILE
 
 
 /************************************************************************
@@ -144,12 +211,18 @@ bool load_game (int num)
 {
     char *buf, *filename;
     FILE *file;
+    char *codeset, *codeset_nl;
     int saved_errno, lineno;
     char *prev_locale;
 
     int crypt_key;
     int n, i, j;
 
+#ifdef USE_UTF8_GAME_FILE
+    iconv_t icd;
+    bool need_icd;
+#endif
+
 
     assert(num >= 1 && num <= 9);
 
@@ -196,6 +269,39 @@ bool load_game (int num)
 	return false;
     }
 
+#ifdef USE_UTF8_GAME_FILE
+    // Make sure all strings are read in UTF-8 format for consistency
+    codeset = nl_langinfo(CODESET);
+    if (codeset == NULL) {
+	errno_exit("nl_langinfo(CODESET)");
+    }
+    need_icd = (strcmp(codeset, GAME_FILE_CHARSET) != 0);
+    if (need_icd) {
+	icd = iconv_open(codeset, GAME_FILE_CHARSET);
+	if (icd == (iconv_t) -1) {
+	    errno_exit("iconv_open()");
+	}
+    } else {
+	icd = (iconv_t) -1;
+    }
+    codeset_nl = strdup(GAME_FILE_CHARSET "\n");
+    if (codeset_nl == NULL) {
+	err_exit_nomem();
+    }
+#else // ! USE_UTF8_GAME_FILE
+    // Make sure all strings are read in the correct codeset
+    codeset = nl_langinfo(CODESET);
+    if (codeset == NULL) {
+	errno_exit("nl_langinfo(CODESET)");
+    }
+    codeset_nl = malloc(strlen(codeset) + 2);
+    if (codeset_nl == NULL) {
+	err_exit_nomem();
+    }
+    strcpy(codeset_nl, codeset);
+    strcat(codeset_nl, "\n");
+#endif // ! USE_UTF8_GAME_FILE
+
     // Change the formatting of numbers to the POSIX locale for consistency
     prev_locale = strdup(setlocale(LC_NUMERIC, NULL));
     if (prev_locale == NULL) {
@@ -217,8 +323,15 @@ bool load_game (int num)
 	err_exit(_("%s: saved under a different version of Star Traders"),
 		 filename);
     }
+    if (fgets(buf, BUFSIZE, file) == NULL) {
+	err_exit(_("%s: missing subheader in game file"), filename);
+    }
+    if (strcmp(buf, codeset_nl) != 0) {
+	err_exit(_("%s: saved under an incompatible character encoding"),
+		 filename);
+    }
 
-    lineno = 3;
+    lineno = 4;
 
     // Read in the game file encryption key
     if (fscanf(file, "%i\n", &crypt_key) != 1) {
@@ -291,9 +404,16 @@ bool load_game (int num)
     // Change the formatting of numbers back to the user-supplied locale
     setlocale(LC_NUMERIC, prev_locale);
 
+#ifdef USE_UTF8_GAME_FILE
+    if (need_icd) {
+	iconv_close(icd);
+    }
+#endif
+
     free(buf);
     free(filename);
     free(prev_locale);
+    free(codeset_nl);
     return true;
 }
 
@@ -306,12 +426,18 @@ bool save_game (int num)
     const char *data_dir;
     char *buf, *filename;
     FILE *file;
+    char *codeset;
     int saved_errno;
     char *prev_locale;
     struct stat statbuf;
     int crypt_key;
     int i, j, x, y;
 
+#ifdef USE_UTF8_GAME_FILE
+    iconv_t icd;
+    bool need_icd;
+#endif
+
 
     assert(num >= 1 && num <= 9);
 
@@ -375,6 +501,30 @@ bool save_game (int num)
 	return false;
     }
 
+#ifdef USE_UTF8_GAME_FILE
+    // Make sure all strings are output in UTF-8 format for consistency
+    codeset = nl_langinfo(CODESET);
+    if (codeset == NULL) {
+	errno_exit("nl_langinfo(CODESET)");
+    }
+    need_icd = (strcmp(codeset, GAME_FILE_CHARSET) != 0);
+    if (need_icd) {
+	icd = iconv_open(codeset, GAME_FILE_CHARSET);
+	if (icd == (iconv_t) -1) {
+	    errno_exit("iconv_open()");
+	}
+    } else {
+	icd = (iconv_t) -1;
+    }
+    codeset = GAME_FILE_CHARSET;	// Now contains output codeset
+#else // ! USE_UTF8_GAME_FILE
+    // Make sure all strings are output in the correct codeset
+    codeset = nl_langinfo(CODESET);
+    if (codeset == NULL) {
+	errno_exit("nl_langinfo(CODESET)");
+    }
+#endif // ! USE_UTF8_GAME_FILE
+
     // Change the formatting of numbers to the POSIX locale for consistency
     prev_locale = strdup(setlocale(LC_NUMERIC, NULL));
     if (prev_locale == NULL) {
@@ -384,7 +534,7 @@ bool save_game (int num)
 
     // Write out the game file header and encryption key
     fprintf(file, "%s\n" "%s\n", GAME_FILE_HEADER, GAME_FILE_API_VERSION);
-    fprintf(file, "%d\n", crypt_key);
+    fprintf(file, "%s\n" "%d\n", codeset, crypt_key);
 
     // Write out various game variables
     save_game_write_int(MAX_X);
@@ -443,6 +593,12 @@ bool save_game (int num)
     // Change the formatting of numbers back to the user-supplied locale
     setlocale(LC_NUMERIC, prev_locale);
 
+#ifdef USE_UTF8_GAME_FILE
+    if (need_icd) {
+	iconv_close(icd);
+    }
+#endif
+
     free(buf);
     free(filename);
     free(prev_locale);

src/system.h

@@ -67,6 +67,7 @@
 #include <sys/stat.h>
 #include <sys/time.h>
 #include <monetary.h>
+#include <langinfo.h>
 
 
 // Headers defined by the GNU C Library
@@ -82,6 +83,15 @@
 #define N_(string)	gettext_noop(string)
 
 
+// Character set conversion for game files
+
+#undef USE_UTF8_GAME_FILE
+#ifdef HAVE_ICONV
+#  define USE_UTF8_GAME_FILE	1
+#  include "striconv.h"
+#endif
+
+
 // X/Open-compatible Curses library
 
 #if defined(HAVE_NCURSESW_CURSES_H)

src/trader.h

@@ -53,8 +53,12 @@
 ************************************************************************/
 
 #define GAME_FILE_HEADER	"Star Traders Saved Game"
-#define GAME_FILE_API_VERSION	"7.0"	// For game loads and saves
+#define GAME_FILE_API_VERSION	"File API 7.2"	// For game loads and saves
-#define GAME_FILE_SENTINEL	42	// End of game file sentinel
+#define GAME_FILE_SENTINEL	42		// End of game file sentinel
+
+#ifdef USE_UTF8_GAME_FILE
+#  define GAME_FILE_CHARSET	"UTF-8"		// For strings in game file
+#endif
 
 #define BUFSIZE			1024	// For various string buffers