From 32ed55a0cd8f46126c1d757455ba46a21800812e Mon Sep 17 00:00:00 2001 From: John Zaitseff Date: Mon, 8 Aug 2011 11:15:44 +1000 Subject: [PATCH] Convert strings to UTF-8 if possible during file save and load --- .gitignore | 2 + configure.ac | 6 +- lib/.gitignore | 34 +++++++++ m4/.gitignore | 5 ++ m4/gnulib-cache.m4 | 4 +- src/Makefile.am | 2 +- src/fileio.c | 182 +++++++++++++++++++++++++++++++++++++++++---- src/system.h | 10 +++ src/trader.h | 8 +- 9 files changed, 233 insertions(+), 20 deletions(-) diff --git a/.gitignore b/.gitignore index 0247e11..2f1337e 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ Makefile.in /configure /stamp-h1 +/build-aux/compile /build-aux/config.guess /build-aux/config.rpath /build-aux/config.sub @@ -22,6 +23,7 @@ Makefile.in /build-aux/snippet/arg-nonnull.h /build-aux/snippet/c++defs.h +/build-aux/snippet/unused-parameter.h /build-aux/snippet/warn-on-use.h /build-aux/snippet/_Noreturn.h diff --git a/configure.ac b/configure.ac index 77fa26c..c65dafc 100644 --- a/configure.ac +++ b/configure.ac @@ -50,6 +50,9 @@ AX_C___ATTRIBUTE__ AC_TYPE_SIZE_T AC_TYPE_SSIZE_T +AM_GNU_GETTEXT([external]) +AM_GNU_GETTEXT_VERSION([0.18.1]) + gl_INIT AX_WITH_CURSES @@ -57,9 +60,6 @@ AS_IF([test "x$ax_cv_curses" != xyes || test "x$ax_cv_curses_color" != xyes], [ AC_MSG_ERROR([requires an X/Open-compatible Curses library with colour]) ]) -AM_GNU_GETTEXT([external]) -AM_GNU_GETTEXT_VERSION([0.18.1]) - AC_CONFIG_FILES([ Makefile lib/Makefile diff --git a/lib/.gitignore b/lib/.gitignore index 58e90ee..723eb71 100644 --- a/lib/.gitignore +++ b/lib/.gitignore @@ -2,6 +2,11 @@ Makefile.am alloca.in.h asnprintf.c ctype.in.h +c-ctype.c +c-ctype.h +c-strcase.h +c-strcasecmp.c +c-strncasecmp.c dosname.h errno.in.h float.c @@ -19,6 +24,15 @@ getopt1.c getopt_int.h gettext.h gettimeofday.c +iconv.c +iconv.in.h +iconv_close.c +iconv_open-aix.gperf +iconv_open-hpux.gperf +iconv_open-irix.gperf +iconv_open-osf.gperf +iconv_open-solaris.gperf +iconv_open.c isnan.c isnand.c isnand-nolibm.h @@ -26,6 +40,7 @@ isnanf.c isnanf-nolibm.h isnanl.c isnanl-nolibm.h +langinfo.in.h locale.in.h malloc.c math.in.h @@ -54,6 +69,8 @@ stdio.in.h stdio-impl.h stdlib.in.h strdup.c +striconv.c +striconv.h string.in.h strncat.c strstr.c @@ -62,6 +79,8 @@ sys_stat.in.h sys_time.in.h time.in.h unistd.in.h +unistr.in.h +unitypes.in.h vasnprintf.c vasnprintf.h verify.h @@ -75,6 +94,13 @@ arg-nonnull.h c++defs.h ctype.h getopt.h +iconv.h +iconv_open-aix.h +iconv_open-hpux.h +iconv_open-irix.h +iconv_open-osf.h +iconv_open-solaris.h +langinfo.h locale.h math.h stdio.h @@ -83,5 +109,13 @@ string.h sys time.h unistd.h +unistr.h +unitypes.h +unused-parameter.h warn-on-use.h wchar.h + +unistr/.dirstamp +unistr/u8-mbtoucr.c +unistr/u8-uctomb-aux.c +unistr/u8-uctomb.c diff --git a/m4/.gitignore b/m4/.gitignore index 8c17497..76961e1 100644 --- a/m4/.gitignore +++ b/m4/.gitignore @@ -25,7 +25,10 @@ gnulib-common.m4 gnulib-comp.m4 gnulib-tool.m4 iconv.m4 +iconv_h.m4 +iconv_open.m4 include_next.m4 +inline.m4 intdiv0.m4 intldir.m4 intl.m4 @@ -37,12 +40,14 @@ inttypes_h.m4 isnand.m4 isnanf.m4 isnanl.m4 +langinfo_h.m4 largefile.m4 lcmessage.m4 ldexpl.m4 lib-ld.m4 lib-link.m4 lib-prefix.m4 +libunistring-base.m4 locale_h.m4 lock.m4 longlong.m4 diff --git a/m4/gnulib-cache.m4 b/m4/gnulib-cache.m4 index b0b1044..da62d61 100644 --- a/m4/gnulib-cache.m4 +++ b/m4/gnulib-cache.m4 @@ -15,7 +15,7 @@ # Specification in the form of a command-line invocation: -# gnulib-tool --import --dir=. --lib=libgnu --source-base=lib --m4-base=m4 --doc-base=doc --tests-base=tests --aux-dir=build-aux --no-conditional-dependencies --no-libtool --macro-prefix=gl assert config-h ctype fprintf-posix getopt-gnu gettext gettext-h gettimeofday locale printf-posix snprintf-posix stat stdarg stdbool stdio strdup-posix string strncat strstr sys_stat sys_time unistd vfprintf-posix vsnprintf-posix +# gnulib-tool --import --dir=. --lib=libgnu --source-base=lib --m4-base=m4 --doc-base=doc --tests-base=tests --aux-dir=build-aux --no-conditional-dependencies --no-libtool --macro-prefix=gl assert config-h ctype fprintf-posix getopt-gnu gettext gettext-h gettimeofday langinfo locale printf-posix snprintf-posix stat stdarg stdbool stdio strdup-posix striconv string strncat strstr sys_stat sys_time unistd vfprintf-posix vsnprintf-posix # Specification in the form of a few gnulib-tool.m4 macro invocations: gl_LOCAL_DIR([]) @@ -28,6 +28,7 @@ gl_MODULES([ gettext gettext-h gettimeofday + langinfo locale printf-posix snprintf-posix @@ -36,6 +37,7 @@ gl_MODULES([ stdbool stdio strdup-posix + striconv string strncat strstr diff --git a/src/Makefile.am b/src/Makefile.am index 692d7f6..57feebf 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -43,6 +43,6 @@ trader_SOURCES = \ trader_CPPFLAGS = -I$(top_builddir)/lib -I$(top_srcdir)/lib \ -DLOCALEDIR=\"$(localedir)\" -trader_LDADD = @CURSES_LIB@ $(top_builddir)/lib/libgnu.a @LIBINTL@ +trader_LDADD = @CURSES_LIB@ @LIBICONV@ $(top_builddir)/lib/libgnu.a @LIBINTL@ EXTRA_DIST = README diff --git a/src/fileio.c b/src/fileio.c index 94e9677..6721854 100644 --- a/src/fileio.c +++ b/src/fileio.c @@ -52,7 +52,8 @@ static const unsigned char game_file_crypt_key[] = { #define load_game_scanf(_fmt, _var, _cond) \ do { \ if (fgets(buf, BUFSIZE, file) == NULL) { \ - err_exit(_("%s: missing field on line %d"), filename, lineno); \ + err_exit(_("%s: missing field on line %d"), \ + filename, lineno); \ } \ if (sscanf(unscramble(crypt_key, buf, BUFSIZE), _fmt "\n", \ &(_var)) != 1) { \ @@ -81,32 +82,76 @@ static const unsigned char game_file_crypt_key[] = { (_var) = b; \ } while (0) -#define load_game_read_string(_var) \ +#ifdef USE_UTF8_GAME_FILE +# define load_game_read_string(_var) \ do { \ char *s; \ int len; \ \ if (fgets(buf, BUFSIZE, file) == NULL) { \ - err_exit(_("%s: missing field on line %d"), filename, lineno); \ + err_exit(_("%s: missing field on line %d"), \ + filename, lineno); \ } \ if (strlen(unscramble(crypt_key, buf, BUFSIZE)) == 0) { \ - err_exit(_("%s: illegal value on line %d"), filename, lineno); \ + err_exit(_("%s: illegal value on line %d"), \ + filename, lineno); \ } \ - lineno++; \ - \ - s = malloc(strlen(buf) + 1); \ - if (s == NULL) { \ - err_exit_nomem(); \ + if (need_icd) { \ + s = str_cd_iconv(buf, icd); \ + if (s == NULL) { \ + if (errno == EILSEQ) { \ + err_exit(_("%s: illegal characters on line %d"), \ + filename, lineno); \ + } else { \ + errno_exit("str_cd_iconv()"); \ + } \ + } \ + } else { \ + s = malloc(strlen(buf) + 1); \ + if (s == NULL) { \ + err_exit_nomem(); \ + } \ + strcpy(s, buf); \ } \ \ - strcpy(s, buf); \ len = strlen(s); \ if (len > 0 && s[len - 1] == '\n') { \ s[len - 1] = '\0'; \ } \ \ + lineno++; \ (_var) = s; \ } while (0) +#else // ! USE_UTF8_GAME_FILE +# define load_game_read_string(_var) \ + do { \ + char *s; \ + int len; \ + \ + if (fgets(buf, BUFSIZE, file) == NULL) { \ + err_exit(_("%s: missing field on line %d"), \ + filename, lineno); \ + } \ + if (strlen(unscramble(crypt_key, buf, BUFSIZE)) == 0) { \ + err_exit(_("%s: illegal value on line %d"), \ + filename, lineno); \ + } \ + \ + s = malloc(strlen(buf) + 1); \ + if (s == NULL) { \ + err_exit_nomem(); \ + } \ + strcpy(s, buf); \ + \ + len = strlen(s); \ + if (len > 0 && s[len - 1] == '\n') { \ + s[len - 1] = '\0'; \ + } \ + \ + lineno++; \ + (_var) = s; \ + } while (0) +#endif // ! USE_UTF8_GAME_FILE // Macros used in save_game() @@ -126,8 +171,30 @@ static const unsigned char game_file_crypt_key[] = { save_game_printf("%2.20e", _var) #define save_game_write_bool(_var) \ save_game_printf("%d", (int) _var) -#define save_game_write_string(_var) \ + +#ifdef USE_UTF8_GAME_FILE +# define save_game_write_string(_var) \ + do { \ + if (need_icd) { \ + char *s = str_cd_iconv(_var, icd); \ + if (s == NULL) { \ + if (errno == EILSEQ) { \ + err_exit(_("%s: could not convert string"), \ + filename); \ + } else { \ + errno_exit("str_cd_iconv()"); \ + } \ + } \ + save_game_printf("%s", s); \ + free(s); \ + } else { \ + save_game_printf("%s", _var); \ + } \ + } while (0) +#else // ! USE_UTF8_GAME_FILE +# define save_game_write_string(_var) \ save_game_printf("%s", _var) +#endif // ! USE_UTF8_GAME_FILE /************************************************************************ @@ -144,12 +211,18 @@ bool load_game (int num) { char *buf, *filename; FILE *file; + char *codeset, *codeset_nl; int saved_errno, lineno; char *prev_locale; int crypt_key; int n, i, j; +#ifdef USE_UTF8_GAME_FILE + iconv_t icd; + bool need_icd; +#endif + assert(num >= 1 && num <= 9); @@ -196,6 +269,39 @@ bool load_game (int num) return false; } +#ifdef USE_UTF8_GAME_FILE + // Make sure all strings are read in UTF-8 format for consistency + codeset = nl_langinfo(CODESET); + if (codeset == NULL) { + errno_exit("nl_langinfo(CODESET)"); + } + need_icd = (strcmp(codeset, GAME_FILE_CHARSET) != 0); + if (need_icd) { + icd = iconv_open(codeset, GAME_FILE_CHARSET); + if (icd == (iconv_t) -1) { + errno_exit("iconv_open()"); + } + } else { + icd = (iconv_t) -1; + } + codeset_nl = strdup(GAME_FILE_CHARSET "\n"); + if (codeset_nl == NULL) { + err_exit_nomem(); + } +#else // ! USE_UTF8_GAME_FILE + // Make sure all strings are read in the correct codeset + codeset = nl_langinfo(CODESET); + if (codeset == NULL) { + errno_exit("nl_langinfo(CODESET)"); + } + codeset_nl = malloc(strlen(codeset) + 2); + if (codeset_nl == NULL) { + err_exit_nomem(); + } + strcpy(codeset_nl, codeset); + strcat(codeset_nl, "\n"); +#endif // ! USE_UTF8_GAME_FILE + // Change the formatting of numbers to the POSIX locale for consistency prev_locale = strdup(setlocale(LC_NUMERIC, NULL)); if (prev_locale == NULL) { @@ -217,8 +323,15 @@ bool load_game (int num) err_exit(_("%s: saved under a different version of Star Traders"), filename); } + if (fgets(buf, BUFSIZE, file) == NULL) { + err_exit(_("%s: missing subheader in game file"), filename); + } + if (strcmp(buf, codeset_nl) != 0) { + err_exit(_("%s: saved under an incompatible character encoding"), + filename); + } - lineno = 3; + lineno = 4; // Read in the game file encryption key if (fscanf(file, "%i\n", &crypt_key) != 1) { @@ -291,9 +404,16 @@ bool load_game (int num) // Change the formatting of numbers back to the user-supplied locale setlocale(LC_NUMERIC, prev_locale); +#ifdef USE_UTF8_GAME_FILE + if (need_icd) { + iconv_close(icd); + } +#endif + free(buf); free(filename); free(prev_locale); + free(codeset_nl); return true; } @@ -306,12 +426,18 @@ bool save_game (int num) const char *data_dir; char *buf, *filename; FILE *file; + char *codeset; int saved_errno; char *prev_locale; struct stat statbuf; int crypt_key; int i, j, x, y; +#ifdef USE_UTF8_GAME_FILE + iconv_t icd; + bool need_icd; +#endif + assert(num >= 1 && num <= 9); @@ -375,6 +501,30 @@ bool save_game (int num) return false; } +#ifdef USE_UTF8_GAME_FILE + // Make sure all strings are output in UTF-8 format for consistency + codeset = nl_langinfo(CODESET); + if (codeset == NULL) { + errno_exit("nl_langinfo(CODESET)"); + } + need_icd = (strcmp(codeset, GAME_FILE_CHARSET) != 0); + if (need_icd) { + icd = iconv_open(codeset, GAME_FILE_CHARSET); + if (icd == (iconv_t) -1) { + errno_exit("iconv_open()"); + } + } else { + icd = (iconv_t) -1; + } + codeset = GAME_FILE_CHARSET; // Now contains output codeset +#else // ! USE_UTF8_GAME_FILE + // Make sure all strings are output in the correct codeset + codeset = nl_langinfo(CODESET); + if (codeset == NULL) { + errno_exit("nl_langinfo(CODESET)"); + } +#endif // ! USE_UTF8_GAME_FILE + // Change the formatting of numbers to the POSIX locale for consistency prev_locale = strdup(setlocale(LC_NUMERIC, NULL)); if (prev_locale == NULL) { @@ -384,7 +534,7 @@ bool save_game (int num) // Write out the game file header and encryption key fprintf(file, "%s\n" "%s\n", GAME_FILE_HEADER, GAME_FILE_API_VERSION); - fprintf(file, "%d\n", crypt_key); + fprintf(file, "%s\n" "%d\n", codeset, crypt_key); // Write out various game variables save_game_write_int(MAX_X); @@ -443,6 +593,12 @@ bool save_game (int num) // Change the formatting of numbers back to the user-supplied locale setlocale(LC_NUMERIC, prev_locale); +#ifdef USE_UTF8_GAME_FILE + if (need_icd) { + iconv_close(icd); + } +#endif + free(buf); free(filename); free(prev_locale); diff --git a/src/system.h b/src/system.h index 3456ad3..6b03183 100644 --- a/src/system.h +++ b/src/system.h @@ -67,6 +67,7 @@ #include #include #include +#include // Headers defined by the GNU C Library @@ -82,6 +83,15 @@ #define N_(string) gettext_noop(string) +// Character set conversion for game files + +#undef USE_UTF8_GAME_FILE +#ifdef HAVE_ICONV +# define USE_UTF8_GAME_FILE 1 +# include "striconv.h" +#endif + + // X/Open-compatible Curses library #if defined(HAVE_NCURSESW_CURSES_H) diff --git a/src/trader.h b/src/trader.h index 3fd27d3..1cc785b 100644 --- a/src/trader.h +++ b/src/trader.h @@ -53,8 +53,12 @@ ************************************************************************/ #define GAME_FILE_HEADER "Star Traders Saved Game" -#define GAME_FILE_API_VERSION "7.0" // For game loads and saves -#define GAME_FILE_SENTINEL 42 // End of game file sentinel +#define GAME_FILE_API_VERSION "File API 7.2" // For game loads and saves +#define GAME_FILE_SENTINEL 42 // End of game file sentinel + +#ifdef USE_UTF8_GAME_FILE +# define GAME_FILE_CHARSET "UTF-8" // For strings in game file +#endif #define BUFSIZE 1024 // For various string buffers