From a2c7af990b11e1772b9f3e26c372213aa0fb8b03 Mon Sep 17 00:00:00 2001 From: Jonas Fonseca Date: Tue, 5 Feb 2008 01:53:12 +0100 Subject: [PATCH 01/31] Change Cogito commands to git --- doc/release.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/release.txt b/doc/release.txt index b0806caf..28e1882b 100644 --- a/doc/release.txt +++ b/doc/release.txt @@ -17,13 +17,14 @@ When releasing a new version - Create a signed tag having the version ("ELinks X.X.X") as the subject and using the changelog create above as the body. Use something like: - $ cg-tag -s -M changelog.file elinks-X.X.X + $ git tag -s -F changelog.file elinks-X.X.X - Append ".GIT" to the VERSION variable in the top of configure.in. - Commit only this change. - Push these changes plus tag using: - $ cg-push -t elinks-X.X.X + $ git push + $ git push --tags 2. Tasks on http://elinks.cz/ (part 1): From 3b93dcc4726d4363215233a780df34772d76ee09 Mon Sep 17 00:00:00 2001 From: Kalle Olavi Niemitalo Date: Sat, 9 Feb 2008 00:24:45 +0200 Subject: [PATCH 02/31] Debian bug 464384: fix cast warning in alignof --- NEWS | 1 + src/osdep/generic.h | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 06215f07..ac4c3377 100644 --- a/NEWS +++ b/NEWS @@ -131,6 +131,7 @@ Miscellaneous: Build system and compile-time errors (ignore if you don't build ELinks): +* serious Debian bug 464384: fix cast warning in alignof * bug 725: fix version checking for Ruby in 'configure' * enhancement: if make -k was used and a sub-Make fails, build the rest before propagating diff --git a/src/osdep/generic.h b/src/osdep/generic.h index 9223cd50..357b602d 100644 --- a/src/osdep/generic.h +++ b/src/osdep/generic.h @@ -102,7 +102,7 @@ safe_write(int fd, const void *buf, size_t count) { /* Alignment of types. */ #define alignof(TYPE) \ - ((int) &((struct { unsigned char dummy1; TYPE dummy2; } *) 0)->dummy2) + offsetof(struct { unsigned char dummy1; TYPE dummy2; }, dummy2) /* Using this macro to copy structs is both faster and safer than * memcpy(destination, source, sizeof(source)). Please, use this macro instead From df07b7839c6063ddf8c7f08853777b938d0a7a34 Mon Sep 17 00:00:00 2001 From: Kalle Olavi Niemitalo Date: Sat, 9 Feb 2008 13:27:33 +0200 Subject: [PATCH 03/31] configure: Don't try to link with Lua 5.1. ELinks does not yet work with Lua 5.1 (see bug 742), so the configure script should not suggest that it does. When bug 742 is eventually fixed, ELinks should probably prefer Lua 5.1 over 5.0, as the newer version seems likely to be kept installed longer. (cherry picked from commit dc747a6ec37baf3e21e99988a65b150a25a27ba4) --- configure.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.in b/configure.in index 1740106e..f80ed18a 100644 --- a/configure.in +++ b/configure.in @@ -888,7 +888,7 @@ if test -z "$disable_lua"; then withval=""; fi for luadir in "$withval" "" /usr /usr/local; do - for suffix in "" 50 51; do + for suffix in "" 50; do if test "$cf_result" = no && ( test -f "$luadir/include/lua.h" || \ test -f "$luadir/include/lua$suffix/lua.h" ) ; then LUA_LIBS="-L$luadir/lib -llua$suffix -llualib$suffix -lm" From 9ba7079735ed9f073296275c98d7a85b13cdb490 Mon Sep 17 00:00:00 2001 From: Laurent MONIN Date: Sat, 3 Mar 2007 23:49:00 +0100 Subject: [PATCH 04/31] Patch: fix for bug 938 This patch prevents handle_itrm_stdin() and clear_handlers(itrm->in.std) to be called when -remote is set and in.std < 0. It adds two assertions for in.std >= 0 in handle_itrm_stdin() and unhandle_itrm_stdin(). May be a bad fix, please test and review. [Added a NEWS entry. --KON] --- NEWS | 1 + src/terminal/kbd.c | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/NEWS b/NEWS index ac4c3377..d2b66d5c 100644 --- a/NEWS +++ b/NEWS @@ -319,6 +319,7 @@ To be released as 0.11.4. hostname * bug 712: GnuTLS works on https://www-s.uiuc.edu/[] * fix active and passive FTP over IPv6 +* bug 938: elinks -remote no longer needs a controlling tty * bug 978: Python's webbrowser.open_new_tab(URL) works since now * minor bug 951 in user SMJS: garbage-collect SMJS objects on 'File -> Flush all caches' to work around their holding cache entries busy diff --git a/src/terminal/kbd.c b/src/terminal/kbd.c index 1f2a6917..f37a9cfd 100644 --- a/src/terminal/kbd.c +++ b/src/terminal/kbd.c @@ -336,10 +336,12 @@ handle_trm(int std_in, int std_out, int sock_in, int sock_out, int ctl_in, #ifdef CONFIG_MOUSE enable_mouse(); #endif + handle_itrm_stdin(itrm); + } else { + /* elinks -remote may not have a valid stdin if not run from a tty (bug 938) */ + if (std_in >= 0) handle_itrm_stdin(itrm); } - handle_itrm_stdin(itrm); - if (sock_in != std_out) set_handlers(sock_in, (select_handler_T) in_sock, NULL, (select_handler_T) free_itrm, itrm); @@ -437,7 +439,8 @@ free_itrm(struct itrm *itrm) mem_free_set(&itrm->orig_title, NULL); - clear_handlers(itrm->in.std); + /* elinks -remote may not have a valid stdin if not run from a tty (bug 938) */ + if (!itrm->remote || itrm->in.std >= 0) clear_handlers(itrm->in.std); clear_handlers(itrm->in.sock); clear_handlers(itrm->out.std); clear_handlers(itrm->out.sock); @@ -1194,6 +1197,9 @@ in_kbd(struct itrm *itrm) static void handle_itrm_stdin(struct itrm *itrm) { + assert(itrm->in.std >= 0); + if_assert_failed return; + set_handlers(itrm->in.std, (select_handler_T) in_kbd, NULL, (select_handler_T) free_itrm, itrm); } @@ -1204,6 +1210,9 @@ handle_itrm_stdin(struct itrm *itrm) static void unhandle_itrm_stdin(struct itrm *itrm) { + assert(itrm->in.std >= 0); + if_assert_failed return; + set_handlers(itrm->in.std, (select_handler_T) NULL, NULL, (select_handler_T) free_itrm, itrm); } From d529a1f24da6f85ac2ce67f4f4c1f7084485f538 Mon Sep 17 00:00:00 2001 From: Kalle Olavi Niemitalo Date: Sat, 9 Feb 2008 14:07:07 +0200 Subject: [PATCH 05/31] NEWS sync from 0.11.4rc0.GIT Specifically 0.11.4rc0.GIT (c72730628a721f7df6ae2e62e8b4b436f64cf665). I changed the heading for that version though. --- NEWS | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/NEWS b/NEWS index d2b66d5c..c8cbfd3d 100644 --- a/NEWS +++ b/NEWS @@ -80,8 +80,6 @@ Miscellaneous: reloaded. See elinks-users mail from 28 Oct 2005. * several accesskey fixes * in Lua: don't write to the string returned by lua_tostring -* minor bug 54, Debian bug 338402: don't force the terminal to 8 bits - with no parity, and don't disable XON/OFF flow control either * minor bug 972: preserve the background color and underlining in spaces when justifying * minor bug 284: render closing bracket for HTML element SUB in the @@ -135,7 +133,6 @@ Build system and compile-time errors (ignore if you don't build ELinks): * bug 725: fix version checking for Ruby in 'configure' * enhancement: if make -k was used and a sub-Make fails, build the rest before propagating -* enhancement: avoid compilation of vernum.c in 'make install' * enhancement: make uninstall * experimental enhancements: --with-python=DIRECTORY, --with-gc=DIRECTORY * experimental enhancement: Win32 port (build with MinGW MSYS) @@ -244,6 +241,7 @@ have already been considered. earlier versions. - fix compilation under gcc 4.x. Backported from gentoo portage. Commit bcabd8b7951f3319199811088e607501296ee573 in ELinks 0.11.3. + - enhancement: avoid compilation of vernum.c in 'make install' * Reverted changes: - (new feature) document.write, reverted in 2c087e52e74528a720621186b91880463e039d50 @@ -296,8 +294,8 @@ have already been considered. * Really retry forever when connection.retries = 0 ////////////////////////////////////////////////////////////////////// -ELinks 0.11.3.GIT now: ----------------------- +ELinks 0.11.4rc0.GIT now: +------------------------- To be released as 0.11.4. @@ -321,6 +319,8 @@ To be released as 0.11.4. * fix active and passive FTP over IPv6 * bug 938: elinks -remote no longer needs a controlling tty * bug 978: Python's webbrowser.open_new_tab(URL) works since now +* minor bug 54, Debian bug 338402: don't force the terminal to 8 bits + with no parity, and don't disable XON/OFF flow control either * minor bug 951 in user SMJS: garbage-collect SMJS objects on 'File -> Flush all caches' to work around their holding cache entries busy * minor bug 396: never show empty filename in the what-to-do dialog From 6555359f8e56b6e08bd1a8f40fd363506a0e3ff5 Mon Sep 17 00:00:00 2001 From: Kalle Olavi Niemitalo Date: Sat, 9 Feb 2008 15:07:04 +0200 Subject: [PATCH 06/31] Debian bug 464384: fix cast warning in ssl_connect There are warnings about casts in the Debian amd64 build logs: http://buildd.debian.org/fetch.cgi?&pkg=elinks&ver=0.11.3-2&arch=amd64&stamp=1200348983&file=log [CC] src/intl/gettext/dcigettext.o /build/buildd/elinks-0.11.3/src/intl/gettext/dcigettext.c: In function '_nl_find_msg': /build/buildd/elinks-0.11.3/src/intl/gettext/dcigettext.c:745: warning: cast from pointer to integer of different size /build/buildd/elinks-0.11.3/src/intl/gettext/dcigettext.c:746: warning: cast from pointer to integer of different size ... [CC] src/network/ssl/socket.o /build/buildd/elinks-0.11.3/src/network/ssl/socket.c: In function 'ssl_connect': /build/buildd/elinks-0.11.3/src/network/ssl/socket.c:219: warning: cast to pointer from integer of different size The warnings in _nl_find_msg were caused by alignof, which I already fixed. This commit ought to fix the gnutls_transport_set_ptr call in ssl_connect. This warning did not yet happen in bug 464384 because the others broke the build before it got that far. --- NEWS | 2 +- src/network/ssl/socket.c | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index c8cbfd3d..6ff7805e 100644 --- a/NEWS +++ b/NEWS @@ -129,7 +129,7 @@ Miscellaneous: Build system and compile-time errors (ignore if you don't build ELinks): -* serious Debian bug 464384: fix cast warning in alignof +* serious Debian bug 464384: fix cast warnings in alignof and ssl_connect * bug 725: fix version checking for Ruby in 'configure' * enhancement: if make -k was used and a sub-Make fails, build the rest before propagating diff --git a/src/network/ssl/socket.c b/src/network/ssl/socket.c index 19b89413..eabd8a3f 100644 --- a/src/network/ssl/socket.c +++ b/src/network/ssl/socket.c @@ -154,8 +154,15 @@ ssl_connect(struct socket *socket) } #elif defined(CONFIG_GNUTLS) + /* GnuTLS uses function pointers for network I/O. The default + * functions take a file descriptor, but it must be passed in + * as a pointer. GnuTLS uses the GNUTLS_INT_TO_POINTER and + * GNUTLS_POINTER_TO_INT macros for these conversions, but + * those are unfortunately not in any public header. So + * ELinks must just cast the pointer the best it can and hope + * that the conversions match. */ gnutls_transport_set_ptr(*((ssl_t *) socket->ssl), - (gnutls_transport_ptr) socket->fd); + (gnutls_transport_ptr) (longptr_T) socket->fd); /* TODO: Some certificates fuss. --pasky */ #endif From 61019c31304f89141248b0381974d1e3886cf160 Mon Sep 17 00:00:00 2001 From: Kalle Olavi Niemitalo Date: Sun, 10 Feb 2008 11:20:33 +0200 Subject: [PATCH 07/31] Debian bug 464384: fix OFF_T_FORMAT mismatches on amd64 On AMD64 apparently, off_t is long but ELinks detected SIZEOF_OFF_T == 8 and defined OFF_T_FORMAT as "lld", which expects long long and so causes GCC to warn about a mismatching format specifier. Because --enable-debug adds -Werror to $CFLAGS, this warning breaks the build. When both SIZEOF_LONG and SIZEOF_LONG_LONG are 8, ELinks cannot know which type it should use. To fix this, do not attempt to find a format specifier for off_t itself. Instead cast all printed off_t values to a new typedef off_print_T that is large enough, and replace OFF_T_FORMAT with OFF_PRINT_FORMAT which is suitable for off_print_T altough not necessarily for off_t. ELinks already had a similar scheme with time_print_T and TIME_PRINT_FORMAT. --- src/cache/cache.c | 8 +++++--- src/cache/dialogs.c | 8 ++++---- src/dialogs/document.c | 5 +++-- src/osdep/types.h | 22 +++++++++++++++++----- src/protocol/fsp/fsp.c | 3 ++- src/protocol/ftp/ftp.c | 3 ++- src/protocol/smb/smb2.c | 3 ++- 7 files changed, 35 insertions(+), 17 deletions(-) diff --git a/src/cache/cache.c b/src/cache/cache.c index db9efe1e..8b050800 100644 --- a/src/cache/cache.c +++ b/src/cache/cache.c @@ -40,9 +40,11 @@ static void truncate_entry(struct cache_entry *cached, off_t offset, int final); #define dump_frag(frag, count) \ do { \ - DBG(" [%d] f=%p offset=%" OFF_T_FORMAT " length=%" OFF_T_FORMAT \ - " real_length=%" OFF_T_FORMAT, \ - count, frag, frag->offset, frag->length, frag->real_length); \ + DBG(" [%d] f=%p offset=%" OFF_PRINT_FORMAT \ + " length=%" OFF_PRINT_FORMAT \ + " real_length=%" OFF_PRINT_FORMAT, \ + count, frag, (off_print_T) frag->offset, \ + (off_print_T) frag->length, (off_print_T) frag->real_length); \ } while (0) #define dump_frags(entry, comment) \ diff --git a/src/cache/dialogs.c b/src/cache/dialogs.c index b0a33b34..29a94461 100644 --- a/src/cache/dialogs.c +++ b/src/cache/dialogs.c @@ -82,10 +82,10 @@ get_cache_entry_info(struct listbox_item *item, struct terminal *term) } } - add_format_to_string(&msg, "\n%s: %" OFF_T_FORMAT, _("Size", term), - cached->length); - add_format_to_string(&msg, "\n%s: %" OFF_T_FORMAT, _("Loaded size", term), - cached->data_size); + add_format_to_string(&msg, "\n%s: %" OFF_PRINT_FORMAT, _("Size", term), + (off_print_T) cached->length); + add_format_to_string(&msg, "\n%s: %" OFF_PRINT_FORMAT, _("Loaded size", term), + (off_print_T) cached->data_size); if (cached->content_type) { add_format_to_string(&msg, "\n%s: %s", _("Content type", term), cached->content_type); diff --git a/src/dialogs/document.c b/src/dialogs/document.c index ba55ea2b..458e8394 100644 --- a/src/dialogs/document.c +++ b/src/dialogs/document.c @@ -152,8 +152,9 @@ document_info_dialog(struct session *ses) if (cached) { unsigned char *a; - add_format_to_string(&msg, "\n%s: %" OFF_T_FORMAT, - _("Size", term), cached->length); + add_format_to_string(&msg, "\n%s: %" OFF_PRINT_FORMAT, + _("Size", term), + (off_print_T) cached->length); if (cached->incomplete) { add_format_to_string(&msg, " (%s)", _("incomplete", term)); diff --git a/src/osdep/types.h b/src/osdep/types.h index 2404ac01..f1c42f83 100644 --- a/src/osdep/types.h +++ b/src/osdep/types.h @@ -146,12 +146,24 @@ typedef unsigned long long uint32_t; */ typedef long longptr_T; -/* Define internal off_t format macro for printing variables. */ -#if HAVE_OFF_T == 1 && SIZEOF_OFF_T == 8 -#define OFF_T_FORMAT "lld" +/* To print off_t offset, ELinks does: + * + * printf("%" OFF_PRINT_FORMAT, (off_print_T) offset); + * + * The cast is necessary because it is not possible to guess + * a printf format for off_t itself based on what we have here. + * The off_t type might be either long or long long, and the format + * string must match even if both types have the same representation, + * because GCC warns about mismatches and --enable-debug adds -Werror + * to $CFLAGS. */ +#if !HAVE_OFF_T || SIZEOF_OFF_T <= SIZEOF_LONG +typedef long off_print_T; +# define OFF_PRINT_FORMAT "ld" +#elif HAVE_LONG_LONG && SIZEOF_OFF_T <= SIZEOF_LONG_LONG +typedef long long off_print_T; +# define OFF_PRINT_FORMAT "lld" #else -/* For ELinks, off_t defaults to long. */ -#define OFF_T_FORMAT "ld" +# error "cannot figure out how to print off_t values" #endif #endif diff --git a/src/protocol/fsp/fsp.c b/src/protocol/fsp/fsp.c index 06b24efc..efa6e2e4 100644 --- a/src/protocol/fsp/fsp.c +++ b/src/protocol/fsp/fsp.c @@ -314,7 +314,8 @@ do_fsp(struct connection *conn) #endif /* Send filesize */ - fprintf(stderr, "%" OFF_T_FORMAT "\n", (off_t)(sb.st_size)); + fprintf(stderr, "%" OFF_PRINT_FORMAT "\n", + (off_print_T) sb.st_size); fclose(stderr); while ((r = fsp_fread(buf, 1, READ_SIZE, file)) > 0) diff --git a/src/protocol/ftp/ftp.c b/src/protocol/ftp/ftp.c index 13201275..eefe8b1a 100644 --- a/src/protocol/ftp/ftp.c +++ b/src/protocol/ftp/ftp.c @@ -1170,7 +1170,8 @@ display_dir_entry(struct cache_entry *cached, off_t *pos, int *tries, add_to_string(&string, " 1 ftp ftp "); if (ftp_info->size != FTP_SIZE_UNKNOWN) { - add_format_to_string(&string, "%12" OFF_T_FORMAT " ", ftp_info->size); + add_format_to_string(&string, "%12" OFF_PRINT_FORMAT " ", + (off_print_T) ftp_info->size); } else { add_to_string(&string, " - "); } diff --git a/src/protocol/smb/smb2.c b/src/protocol/smb/smb2.c index 30908b32..0a839dbf 100644 --- a/src/protocol/smb/smb2.c +++ b/src/protocol/smb/smb2.c @@ -349,7 +349,8 @@ do_smb(struct connection *conn) smb_error(res); } /* filesize */ - fprintf(stderr, "%" OFF_T_FORMAT, sb.st_size); + fprintf(stderr, "%" OFF_PRINT_FORMAT, + (off_print_T) sb.st_size); fclose(stderr); while ((r = smbc_read(file, buf, READ_SIZE)) > 0) { From 76d803bbb9aa0e26810bf81f3600e31ec3f7de06 Mon Sep 17 00:00:00 2001 From: Kalle Olavi Niemitalo Date: Sun, 10 Feb 2008 14:30:26 +0200 Subject: [PATCH 08/31] config: Count backslashed newlines in str_rd. If a newline has a backslash in front of it, then str_rd replaces it with a space. However, the newline was in the original config file, so the line number must still be incremented. --- src/config/opttypes.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config/opttypes.c b/src/config/opttypes.c index 72e23c96..e79e42f0 100644 --- a/src/config/opttypes.c +++ b/src/config/opttypes.c @@ -282,7 +282,7 @@ str_rd(struct option *opt, unsigned char **file, int *line) while (*str && (commandline || !isquote(*str))) { if (*str == '\\') { /* FIXME: This won't work on crlf systems. */ - if (str[1] == '\n') { str[1] = ' '; str++; } + if (str[1] == '\n') { str[1] = ' '; str++; (*line)++; } /* When there's quote char, we will just move on there, * thus we will never test for it in while () condition * and we will treat it just as '"', ignoring the From 4a1e296c3b636e9b16bd134f44c38ec6d99f6302 Mon Sep 17 00:00:00 2001 From: Y Giridhar Appaji Nag Date: Sun, 10 Feb 2008 12:30:02 +0100 Subject: [PATCH 09/31] Bug 1002: Fix build failure on Debian GNU/Linux arch powerpc Change char id to enum bittorrent_message_id id to prevent FTBFS on powerpc and s390 (cherry picked from commit 01b0c812275c884823ec20d122a7035a7df3861f, with a conflict) --- NEWS | 2 ++ src/protocol/bittorrent/common.h | 4 +--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/NEWS b/NEWS index 6ff7805e..87eb7b41 100644 --- a/NEWS +++ b/NEWS @@ -330,6 +330,8 @@ To be released as 0.11.4. * minor: don't assume sizeof(int)==4 in bittorrent * trivial bug 947: document.html.wrap_nbsp also affects text in tables * trivial bug 997: fix unlikely stack corruption in active FTP +* build bug 1002: fix ``comparison is always true due to limited range + of data type'' warning on PowerPC and s390 * build bug 950: fix ``config/install-sh: No such file or directory'' on SunOS * build bug 936: fix errors about undefined off_t (autoheader diff --git a/src/protocol/bittorrent/common.h b/src/protocol/bittorrent/common.h index 9d44b761..26a86325 100644 --- a/src/protocol/bittorrent/common.h +++ b/src/protocol/bittorrent/common.h @@ -108,9 +108,7 @@ struct bittorrent_peer_request { uint16_t block; /**< The block index in the piece. */ - /* This holds the message id when the request struct is used for queuing - * pending messages. */ - char id; /**< @-> enum bittorrent_message_id */ + enum bittorrent_message_id id; /**< ID of queued pending message. */ unsigned int cloned:1; /**< The request was cloned. */ unsigned int requested:1; /**< Whether it has been requested. */ From 24f8c65010ffcf35ca97d33135792ac08406ed71 Mon Sep 17 00:00:00 2001 From: Witold Filipczyk Date: Thu, 14 Feb 2008 18:55:10 +0100 Subject: [PATCH 10/31] pl.po: Typo --- po/pl.po | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/po/pl.po b/po/pl.po index df3d3717..c1ac4f86 100644 --- a/po/pl.po +++ b/po/pl.po @@ -2274,7 +2274,7 @@ msgid "" msgstr "" "Czy próbować połączyć się z serwerem przy pomocy IPv4?\n" "Nie dotykaj tej opcji.\n" -"Pamiętaj, żę możesz wymusić dany protokół przy połączeniu\n" +"Pamiętaj, że możesz wymusić dany protokół przy połączeniu\n" "używając URL-a w stylu np. http4://elinks.cz/." #: src/config/options.inc:118 From 172771bcfcd0066c24bc753c961ce33df48745da Mon Sep 17 00:00:00 2001 From: Kalle Olavi Niemitalo Date: Sun, 17 Feb 2008 18:27:06 +0200 Subject: [PATCH 11/31] configure.in: Remove obsolete comment. --- configure.in | 1 - 1 file changed, 1 deletion(-) diff --git a/configure.in b/configure.in index f80ed18a..680a60b1 100644 --- a/configure.in +++ b/configure.in @@ -377,7 +377,6 @@ dnl =================================================================== dnl Checks for libraries. dnl =================================================================== -dnl Replace `main' with a function in -lsocket: AC_CHECK_FUNC(socket, cf_result=yes, cf_result=no) if test "$cf_result" = no; then AC_CHECK_LIB(socket, socket) From 4672bad9c73321019c1a2a7695761b8188bd1a8f Mon Sep 17 00:00:00 2001 From: Kalle Olavi Niemitalo Date: Sun, 17 Feb 2008 18:40:19 +0200 Subject: [PATCH 12/31] configure.in: Change most "dnl" comments to "#". Autoconf and m4 copy the "#" comments from configure.in to configure. This should make it easier to find the part of configure that was expanded from a specific check in configure.in. --- configure.in | 328 ++++++++++++++++++++++++++------------------------- 1 file changed, 167 insertions(+), 161 deletions(-) diff --git a/configure.in b/configure.in index 680a60b1..d365ec43 100644 --- a/configure.in +++ b/configure.in @@ -1,5 +1,10 @@ dnl Process this file with autoconf to produce a configure script. +dnl There are two types of comments in this file. +dnl Comments that refer to Autoconf macros begin with "dnl", and m4 +dnl discards them. Other comments begin with "#", and they get copied +dnl to the configure script, hopefully making it easier to read. + dnl Autoconf 2.13 generates an incomplete config.h.in; see ELinks bug 936. dnl Autoconf 2.59 is installed in the computer that generates our nightly dnl snapshots, so we need to be compatible with that. @@ -37,18 +42,18 @@ if test -e Makefile.config; then AC_MSG_RESULT(done) fi -dnl =================================================================== -dnl Load feature configuration file and start logging features. -dnl =================================================================== +# =================================================================== +# Load feature configuration file and start logging features. +# =================================================================== features="features.conf" AC_CHECK_FILE("$srcdir/$features", [ . $srcdir/$features ]) AC_CHECK_FILE("$builddir/$features", [ . $builddir/$features ]) echo "Feature summary:" > features.log -dnl =================================================================== -dnl Checks for programs. -dnl =================================================================== +# =================================================================== +# Checks for programs. +# =================================================================== AC_PROG_CC AC_PROG_AWK @@ -116,17 +121,17 @@ EL_CONFIG_DEPENDS(CONFIG_MANUAL, [MANUAL_ASCIIDOC MANUAL_XMLTO MANUAL_JW], [Manu EL_CONFIG_DEPENDS(CONFIG_MANPAGE, [MAN_ASCIIDOC MAN_XMLTO], [Man Page Formats]) EL_CONFIG_DEPENDS(CONFIG_APIDOCS, [CONFIG_DOXYGEN], [API Documentation]) -dnl gcc specific options (to be continued at the bottom of configure) +# gcc specific options (to be continued at the bottom of configure) if test "x$ac_cv_c_compiler_gnu" = "xyes"; then - dnl We want to see all warnings and live with none. - dnl We can't set up -Werror here as there may be some warnings in test - dnl suite of configure, and we don't want to fail them. + # We want to see all warnings and live with none. + # We can't set up -Werror here as there may be some warnings in test + # suite of configure, and we don't want to fail them. CFLAGS="$CFLAGS -Wall" fi -dnl =================================================================== -dnl Checks for special OSes. -dnl =================================================================== +# =================================================================== +# Checks for special OSes. +# =================================================================== dnl EL_CHECK_COMPILER_MACRO(define, name, flagname) AC_DEFUN([EL_CHECK_COMPILER_MACROS], @@ -172,9 +177,9 @@ fi AC_MSG_RESULT($CONFIG_OS_UNIX) AC_SUBST(CONFIG_OS_UNIX) -dnl =================================================================== -dnl Checks for header files. -dnl =================================================================== +# =================================================================== +# Checks for header files. +# =================================================================== AC_HEADER_DIRENT AC_HEADER_STDC @@ -208,9 +213,9 @@ AC_CHECK_HEADERS(sys/un.h, [CONFIG_INTERLINK=no]) AC_SUBST(CONFIG_INTERLINK) -dnl =================================================================== -dnl Checks for typedefs, structures, and compiler characteristics. -dnl =================================================================== +# =================================================================== +# Checks for typedefs, structures, and compiler characteristics. +# =================================================================== AC_STRUCT_TM AC_C_CONST @@ -234,41 +239,41 @@ AC_CHECK_SIZEOF(long, 4) test "x$HAVE_LONG_LONG" = xyes && AC_CHECK_SIZEOF(long long, 8) test "x$HAVE_OFF_T" = xyes && AC_CHECK_SIZEOF(off_t, 4) -dnl Check for variadic macros +# Check for variadic macros EL_CHECK_CODE([variadic macros], HAVE_VARIADIC_MACROS, [#include #define a(b,c...) printf(b,##c)], [a("foo");a("%s","bar");a("%s%s","baz","quux");]) -dnl Check for -rdynamic -dnl -dnl gcc -rdynamic calls ld -export-dynamic, which adds all symbols of -dnl the executable to its dynamic symbol table. ELinks uses this for -dnl two purposes: -dnl -dnl 1. If ELinks detects a bug, it can try to display a backtrace by -dnl calling backtrace_symbols_fd() in the GNU libc. The glibc-2.3.6 -dnl manual states users of GNU ld must pass -rdynamic to make the -dnl symbols available to the program. -dnl -dnl 2. It would eventually be nice to dynamically load shared -dnl libraries as plugins (bug 73). The plugins must be able to -dnl call ELinks functions. This can be implemented either by -dnl registering all callable functions in ELinks-specific data -dnl structures, or by letting the dynamic linker handle them. -dnl The latter way requires something equivalent to -rdynamic. -dnl -dnl Because backtraces are not needed for correct operation, and bug -dnl 73 is not yet being fixed, the configure script and makefiles -dnl should not complain to the user if they find that -rdynamic does -dnl not work. Besides, it was reported at elinks-users on 2006-09-12 -dnl that gcc-3.4.2 with "ld: Software Generation Utilities - Solaris -dnl Link Editors: 5.8-1.284" on Sun Solaris 8 Sparc does not support -dnl -rdynamic but does something equivalent automatically. (This was -dnl tested with "nm -D elinks | grep redraw_from_window".) -dnl -dnl FIXME: This check doesn't work. Something to do with the compiler -dnl happily ignoring it and stderr not being checked for error messages. +# Check for -rdynamic +# +# gcc -rdynamic calls ld -export-dynamic, which adds all symbols of +# the executable to its dynamic symbol table. ELinks uses this for +# two purposes: +# +# 1. If ELinks detects a bug, it can try to display a backtrace by +# calling backtrace_symbols_fd() in the GNU libc. The glibc-2.3.6 +# manual states users of GNU ld must pass -rdynamic to make the +# symbols available to the program. +# +# 2. It would eventually be nice to dynamically load shared +# libraries as plugins (bug 73). The plugins must be able to +# call ELinks functions. This can be implemented either by +# registering all callable functions in ELinks-specific data +# structures, or by letting the dynamic linker handle them. +# The latter way requires something equivalent to -rdynamic. +# +# Because backtraces are not needed for correct operation, and bug +# 73 is not yet being fixed, the configure script and makefiles +# should not complain to the user if they find that -rdynamic does +# not work. Besides, it was reported at elinks-users on 2006-09-12 +# that gcc-3.4.2 with "ld: Software Generation Utilities - Solaris +# Link Editors: 5.8-1.284" on Sun Solaris 8 Sparc does not support +# -rdynamic but does something equivalent automatically. (This was +# tested with "nm -D elinks | grep redraw_from_window".) +# +# FIXME: This check doesn't work. Something to do with the compiler +# happily ignoring it and stderr not being checked for error messages. AC_MSG_CHECKING([for -rdynamic]) LDFLAGS_X="$LDFLAGS" LDFLAGS="$LDFLAGS -rdynamic" @@ -276,15 +281,15 @@ AC_LINK_IFELSE([AC_LANG_PROGRAM([[]], [[]])],[have_rdynamic=yes],[have_rdynamic= test "$have_rdynamic" = no && LDFLAGS="$LDFLAGS_X" AC_MSG_RESULT($have_rdynamic) -dnl =================================================================== -dnl Check for POSIX -dnl =================================================================== +# =================================================================== +# Check for POSIX +# =================================================================== EL_CHECK_SYS_TYPE(regex_t, HAVE_REGEX_H, [#include ]) -dnl =================================================================== -dnl Checks for library functions. -dnl =================================================================== +# =================================================================== +# Checks for library functions. +# =================================================================== AC_PROG_GCC_TRADITIONAL AC_FUNC_MEMCMP @@ -305,9 +310,9 @@ AC_CHECK_FUNCS(setenv putenv, HAVE_SETENV_OR_PUTENV=yes) AC_CHECK_FUNCS(getuid, HAVE_GETUID=yes) AC_CHECK_FUNCS(geteuid, HAVE_GETEUID=yes) -dnl These aren't probably needed now, as they are commented in links.h. -dnl I've no idea about their historical background, but I keep them here -dnl just in the case they will help later. --pasky +# These aren't probably needed now, as they are commented in links.h. +# I've no idea about their historical background, but I keep them here +# just in the case they will help later. --pasky AC_CHECK_FUNCS(getpid, HAVE_GETPID=yes) AC_CHECK_FUNCS(setpgid getpgid setpgrp getpgrp) AC_CHECK_FUNCS(raise, HAVE_RAISE=yes) @@ -371,11 +376,12 @@ if test x"$el_cv_HAVE_C99_VSNPRINTF" = x"yes"; then EL_DEFINE(HAVE_C99_VSNPRINTF, [C99 compliant vsnprintf()]) fi -AC_CHECK_LIB(dl, dlopen) # OpenSSL and Lua frequently needs it +# OpenSSL and Lua frequently need dlopen +AC_CHECK_LIB(dl, dlopen) -dnl =================================================================== -dnl Checks for libraries. -dnl =================================================================== +# =================================================================== +# Checks for libraries. +# =================================================================== AC_CHECK_FUNC(socket, cf_result=yes, cf_result=no) if test "$cf_result" = no; then @@ -397,18 +403,18 @@ if test "$cf_result" = no; then fi fi -dnl =================================================================== -dnl Checks for packaging specific options. -dnl =================================================================== +# =================================================================== +# Checks for packaging specific options. +# =================================================================== AC_ARG_WITH(xterm, [ --with-xterm how to invoke the X terminal emulator], [ if test "$withval" != no && test "$withval" != yes; then AC_DEFINE_UNQUOTED(XTERM, "$withval", [How to invoke XTerm]) fi ]) -dnl =================================================================== -dnl Checks for a libraries, optional even if installed. -dnl =================================================================== +# =================================================================== +# Checks for a libraries, optional even if installed. +# =================================================================== dnl EL_CHECK_OPTIONAL_LIBRARY(define, name, header, lib, function) AC_DEFUN([EL_CHECK_OPTIONAL_LIBRARY], @@ -491,9 +497,9 @@ fi EL_ARG_ENABLE(CONFIG_LZMA, lzma, [lzma], [ --enable-lzma enable lzma encoding support]) -dnl =================================================================== -dnl Check for GSSAPI, optional even if installed. -dnl =================================================================== +# =================================================================== +# Check for GSSAPI, optional even if installed. +# =================================================================== enable_gssapi="no"; @@ -515,9 +521,9 @@ fi AC_SUBST(CONFIG_GSSAPI) -dnl =================================================================== -dnl Bookmark and XBEL support -dnl =================================================================== +# =================================================================== +# Bookmark and XBEL support +# =================================================================== EL_SAVE_FLAGS @@ -545,34 +551,34 @@ if test "$CONFIG_XBEL_BOOKMARKS" != yes; then EL_RESTORE_FLAGS fi -dnl =================================================================== -dnl Checks for BSD sysmouse -dnl =================================================================== +# =================================================================== +# Checks for BSD sysmouse +# =================================================================== HAVE_SYSMOUSE_HEADER="no" # Either of these header files provides the (same) sysmouse interface AC_CHECK_HEADERS(sys/consio.h machine/console.h, [HAVE_SYSMOUSE_HEADER="yes"]) -dnl =================================================================== -dnl Checks for OS/2 -dnl =================================================================== +# =================================================================== +# Checks for OS/2 +# =================================================================== if test "$CONFIG_OS_OS2" = yes; then EL_CONFIG_OS_OS2 fi -dnl =================================================================== -dnl Checks for Win32 -dnl =================================================================== +# =================================================================== +# Checks for Win32 +# =================================================================== if test "$CONFIG_OS_WIN32" = yes; then EL_CONFIG_OS_WIN32 fi -dnl =================================================================== -dnl Check for SEE (Simple Ecmascript Engine) -dnl =================================================================== +# =================================================================== +# Check for SEE (Simple Ecmascript Engine) +# =================================================================== AC_ARG_WITH(see, [ --with-see enable Simple Ecmascript Engine (SEE) support], [ if test "x$withval" != xno; then enable_see=yes; fi ]) @@ -618,9 +624,9 @@ if test "$enable_see" = "yes"; then fi fi -dnl =================================================================== -dnl Check for SpiderMonkey, optional even if installed. -dnl =================================================================== +# =================================================================== +# Check for SpiderMonkey, optional even if installed. +# =================================================================== AC_ARG_WITH(spidermonkey, [ --without-spidermonkey disable SpiderMonkey Mozilla JavaScript engine support], [if test "$withval" = no; then disable_spidermonkey=yes; fi]) @@ -669,9 +675,9 @@ AC_SUBST(CONFIG_ECMASCRIPT_SEE) AC_SUBST(CONFIG_ECMASCRIPT_SMJS) -dnl =================================================================== -dnl Optional Spidermonkey-based ECMAScript browser scripting -dnl =================================================================== +# =================================================================== +# Optional Spidermonkey-based ECMAScript browser scripting +# =================================================================== AC_ARG_ENABLE(sm-scripting, [ --disable-sm-scripting ECMAScript browser scripting (requires Spidermonkey)], @@ -693,9 +699,9 @@ if test "x$CONFIG_ECMASCRIPT_SMJS" = xyes || AC_SUBST(CONFIG_SPIDERMONKEY) fi -dnl =================================================================== -dnl Check for Guile, optional even if installed. -dnl =================================================================== +# =================================================================== +# Check for Guile, optional even if installed. +# =================================================================== enable_guile="no"; @@ -750,9 +756,9 @@ else AC_MSG_RESULT(no); fi -dnl =================================================================== -dnl Check for Perl -dnl =================================================================== +# =================================================================== +# Check for Perl +# =================================================================== enable_perl="no"; AC_ARG_WITH(perl, [ --with-perl enable Perl support], @@ -807,9 +813,9 @@ else AC_SUBST(PERL_CFLAGS) fi -dnl =================================================================== -dnl Check for Python -dnl =================================================================== +# =================================================================== +# Check for Python +# =================================================================== enable_python="no"; AC_ARG_WITH(python, [ --with-python=[DIR] enable Python support], @@ -868,12 +874,12 @@ else fi -dnl =================================================================== -dnl Check for Lua, optional even if installed. -dnl =================================================================== +# =================================================================== +# Check for Lua, optional even if installed. +# =================================================================== -dnl Do this the long way, as FreeBSD reportedly needs -L for -dnl anything other than /usr/lib, and Lua is very often in /usr/local/lib. +# Do this the long way, as FreeBSD reportedly needs -L for +# anything other than /usr/lib, and Lua is very often in /usr/local/lib. AC_ARG_WITH(lua, [ --without-lua disable Lua support], [if test "$withval" = no; then disable_lua=yes; fi]) @@ -927,15 +933,15 @@ else fi -dnl =================================================================== -dnl Check for Ruby, optional even if installed. -dnl =================================================================== +# =================================================================== +# Check for Ruby, optional even if installed. +# =================================================================== EL_CONFIG_SCRIPTING_RUBY -dnl =================================================================== -dnl Setup global scripting -dnl =================================================================== +# =================================================================== +# Setup global scripting +# =================================================================== EL_CONFIG_DEPENDS(CONFIG_SCRIPTING, [CONFIG_SCRIPTING_GUILE CONFIG_SCRIPTING_LUA CONFIG_SCRIPTING_PERL CONFIG_SCRIPTING_PYTHON CONFIG_SCRIPTING_RUBY CONFIG_SCRIPTING_SPIDERMONKEY], [Browser scripting]) AC_SUBST(CONFIG_SCRIPTING_GUILE) @@ -947,14 +953,14 @@ AC_SUBST(CONFIG_SCRIPTING_SPIDERMONKEY) AC_SUBST(CONFIG_SCRIPTING) -dnl =================================================================== -dnl Check for SSL support. -dnl =================================================================== +# =================================================================== +# Check for SSL support. +# =================================================================== -dnl We by default use OpenSSL, and we always prefer it. However, when GNUTLS -dnl is enabled, we won't try to use OpenSSL anymore. +# We by default use OpenSSL, and we always prefer it. However, when GNUTLS +# is enabled, we won't try to use OpenSSL anymore. -dnl For wiping SSL hooks.. +# For wiping SSL hooks.. #ifdef CONFIG_SSL disable_openssl="" @@ -976,7 +982,7 @@ AC_ARG_WITH(openssl, [ --without-openssl disable OpenSSL support], AC_ARG_WITH(openssl, [ --with-openssl[=DIR] enable OpenSSL support (default)]) openssl_withval="$withval" -dnl ---- OpenSSL +# ---- OpenSSL AC_MSG_CHECKING([for OpenSSL]) @@ -1027,7 +1033,7 @@ fi AC_MSG_RESULT($cf_result) CONFIG_GNUTLS_OPENSSL_COMPAT=no -dnl ---- GNU TLS +# ---- GNU TLS dnl We can't have AC_MSG_CHECKING here, because AC_PATH_PROG prints its own and dnl it looks ugly then. @@ -1092,7 +1098,7 @@ fi AC_MSG_CHECKING([for GNU TLS (1.2 or later)]) AC_MSG_RESULT($cf_result) -dnl Final SSL setup +# Final SSL setup EL_CONFIG_DEPENDS(CONFIG_SSL, [CONFIG_OPENSSL CONFIG_GNUTLS], [SSL]) AC_SUBST(CONFIG_GNUTLS_OPENSSL_COMPAT) @@ -1104,9 +1110,9 @@ AC_SUBST(CONFIG_GNUTLS) AC_MSG_CHECKING([whether to be or not to be]) AC_MSG_RESULT([needs to be determined experimentally]) -dnl =================================================================== -dnl Check for IPv6 support and related functions. -dnl =================================================================== +# =================================================================== +# Check for IPv6 support and related functions. +# =================================================================== EL_CHECK_NET_TYPE(struct sockaddr_storage, HAVE_SA_STORAGE, []) EL_CHECK_NET_TYPE(struct sockaddr_in6, HAVE_SA_IN6, [#include ]) @@ -1121,9 +1127,9 @@ if test "$HAVE_GETADDRINFO" != yes; then fi -dnl =================================================================== -dnl Checking for X11 (window title restoring). -dnl =================================================================== +# =================================================================== +# Checking for X11 (window title restoring). +# =================================================================== AC_PATH_X if test x"$no_x" != xyes; then @@ -1150,17 +1156,17 @@ if test x"$no_x" != xyes; then fi -dnl =================================================================== -dnl Backtraces displaying support. -dnl =================================================================== +# =================================================================== +# Backtraces displaying support. +# =================================================================== AC_CHECK_HEADERS(execinfo.h, HAVE_EXECINFO=yes, HAVE_EXECINFO=no) # possible checks for other system-specific means go here -dnl =================================================================== -dnl Gettext grey zone. Beware. -dnl =================================================================== +# =================================================================== +# Gettext grey zone. Beware. +# =================================================================== ALL_LINGUAS="af be bg ca cs da de el es et fi fr gl hr hu id is it lt nl nb pl pt pt_BR ro ru sk sr sv tr uk" @@ -1172,9 +1178,9 @@ dnl cf_result=$((48#z - 48#a + 1)); dnl AC_MSG_RESULT($cf_result) -dnl =================================================================== -dnl Compile-time features control -dnl =================================================================== +# =================================================================== +# Compile-time features control +# =================================================================== EL_ARG_ENABLE(CONFIG_COOKIES, cookies, [Cookies], [ --disable-cookies disable cookie support]) @@ -1213,9 +1219,9 @@ EL_ARG_DEPEND(CONFIG_CGI, cgi, [HAVE_SETENV_OR_PUTENV:yes], [Local CGI], EL_ARG_ENABLE(CONFIG_FINGER, finger, [Finger protocol], [ --enable-finger enable finger protocol support]) -dnl =================================================================== -dnl FSP protocol -dnl =================================================================== +# =================================================================== +# FSP protocol +# =================================================================== EL_SAVE_FLAGS if test "x${enable_fsp}" != xno; then @@ -1250,9 +1256,9 @@ EL_ARG_ENABLE(CONFIG_GOPHER, gopher, [Gopher protocol], EL_ARG_ENABLE(CONFIG_NNTP, nntp, [NNTP protocol], [ --enable-nntp enable nntp protocol support]) -dnl =================================================================== -dnl SMB protocol support. -dnl =================================================================== +# =================================================================== +# SMB protocol support. +# =================================================================== EL_SAVE_FLAGS if test "x${enable_smb}" != xno; then @@ -1309,8 +1315,8 @@ EL_ARG_ENABLE(CONFIG_CSS, css, [Cascading Style Sheets], EL_ARG_DEPEND(CONFIG_HTML_HIGHLIGHT, html-highlight, [CONFIG_CSS:yes], [HTML highlighting], [ --enable-html-highlight HTML highlighting using DOM engine]) -dnl Everything in the tree already uses CONFIG_DOM -dnl so resolve CONFIG_HTML_HIGHLIGHT to CONFIG_DOM +# Everything in the tree already uses CONFIG_DOM +# so resolve CONFIG_HTML_HIGHLIGHT to CONFIG_DOM EL_CONFIG_DEPENDS(CONFIG_DOM, [CONFIG_HTML_HIGHLIGHT], [DOM engine]) EL_ARG_DEPEND(CONFIG_BACKTRACE, backtrace, [HAVE_EXECINFO:yes], [Backtrace], @@ -1343,15 +1349,15 @@ AC_ARG_ENABLE(weehoofooboomookerchoo, [AC_MSG_ERROR(Are you strange, or what?)]) -dnl == EMX hack +# == EMX hack test "$CONFIG_OS_OS2" = yes && LDFLAGS="$LDFLAGS -Zexe" test "$CONFIG_OS_OS2" = yes && LDFLAGS=`echo "$LDFLAGS" | sed "s/-Zbin-files//g"` -dnl =================================================================== -dnl Export directory paths -dnl =================================================================== +# =================================================================== +# Export directory paths +# =================================================================== # Set up the ``entry points'' if they were not supplied by builder test "x$prefix" = xNONE && prefix=$ac_default_prefix @@ -1391,13 +1397,13 @@ AC_SUBST(LIBDIR) EL_LOG_CONFIG(CONFDIR, [System configuration directory], []) EL_LOG_CONFIG(LOCALEDIR, [Locale catalogs directory], []) -dnl =================================================================== -dnl A little fine tuning of gcc specific options (continued) -dnl =================================================================== +# =================================================================== +# A little fine tuning of gcc specific options (continued) +# =================================================================== if test "x$ac_cv_c_compiler_gnu" = "xyes"; then if test "$CONFIG_DEBUG" = "yes"; then - dnl We want to see all warnings and live with none (in debug mode). + # We want to see all warnings and live with none (in debug mode). CFLAGS="$CFLAGS -Werror" fi @@ -1447,18 +1453,18 @@ EL_LOG_CONFIG(CPPFLAGS, [Preprocessor flags (CPPFLAGS)], []) EL_LOG_CONFIG(LDFLAGS, [Linker flags (LDFLAGS)], []) EL_LOG_CONFIG(LIBS, [Library flags (LIBS)], []) -dnl =================================================================== -dnl Colored make output -dnl =================================================================== +# =================================================================== +# Colored make output +# =================================================================== if test $(`which tput` colors) -ge 4; then MAKE_COLOR=1 AC_SUBST(MAKE_COLOR) fi -dnl =================================================================== -dnl Generated files -dnl =================================================================== +# =================================================================== +# Generated files +# =================================================================== AC_CONFIG_FILES([ \ Makefile.config \ @@ -1483,9 +1489,9 @@ if test "$abs_srcdir" != "$builddir"; then echo "*" > "$builddir/.gitignore" fi -dnl =================================================================== -dnl Configuration summary -dnl =================================================================== +# =================================================================== +# Configuration summary +# =================================================================== AC_MSG_RESULT(The following feature summary has been saved to features.log) cat features.log From 1127b6624022ee884d7b48a8e127660f076f4380 Mon Sep 17 00:00:00 2001 From: Kalle Olavi Niemitalo Date: Sun, 17 Feb 2008 21:49:09 +0200 Subject: [PATCH 13/31] NEWS update --- NEWS | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/NEWS b/NEWS index 87eb7b41..1b56f06e 100644 --- a/NEWS +++ b/NEWS @@ -12,7 +12,7 @@ To be released as ELinks 0.12.0. ////////////////////////////////////////////////////////////////////// This list now contains all the important changes from ELinks 0.11.0 to -ELinks 0.12.GIT (054852ae234342765ab69da0eb95a18f96bd6028) and related +ELinks 0.12.GIT (4672bad9c73321019c1a2a7695761b8188bd1a8f) and related bug numbers. Each section is sorted by severity and grouped by topic. The list no doubt includes several changes that are not really @@ -129,7 +129,8 @@ Miscellaneous: Build system and compile-time errors (ignore if you don't build ELinks): -* serious Debian bug 464384: fix cast warnings in alignof and ssl_connect +* serious Debian bug 464384: fix warnings in alignof, ssl_connect, and + printing of off_t values * bug 725: fix version checking for Ruby in 'configure' * enhancement: if make -k was used and a sub-Make fails, build the rest before propagating @@ -320,13 +321,14 @@ To be released as 0.11.4. * bug 938: elinks -remote no longer needs a controlling tty * bug 978: Python's webbrowser.open_new_tab(URL) works since now * minor bug 54, Debian bug 338402: don't force the terminal to 8 bits - with no parity, and don't disable XON/OFF flow control either + with no parity, and don't disable XON/XOFF flow control either * minor bug 951 in user SMJS: garbage-collect SMJS objects on 'File -> Flush all caches' to work around their holding cache entries busy * minor bug 396: never show empty filename in the what-to-do dialog * minor bug 461: ensure contrast in blank areas, to keep the cursor visible * minor bug 928: properly display no-break spaces in a UTF-8 document if the terminal uses some other charset +* minor bug 987: English spelling and grammar corrections * minor: don't assume sizeof(int)==4 in bittorrent * trivial bug 947: document.html.wrap_nbsp also affects text in tables * trivial bug 997: fix unlikely stack corruption in active FTP From 3fd6ae4a2de7f89fb5d46ef85c79967c95eccdd5 Mon Sep 17 00:00:00 2001 From: Witold Filipczyk Date: Mon, 21 Jan 2008 14:44:10 +0100 Subject: [PATCH 14/31] Three CGI tests for content-encoding. The output should be: Two lines should be visible. The second line. All three tests fail currently when the ELinks is invoked like this: $ elinks -no-connect path_to_chunked_test --- test/cgi/chunked_bzip2.py | 36 ++++++++++++++++++++++++++++++++++++ test/cgi/chunked_deflate.py | 22 ++++++++++++++++++++++ test/cgi/chunked_gzip.py | 36 ++++++++++++++++++++++++++++++++++++ 3 files changed, 94 insertions(+) create mode 100755 test/cgi/chunked_bzip2.py create mode 100755 test/cgi/chunked_deflate.py create mode 100755 test/cgi/chunked_gzip.py diff --git a/test/cgi/chunked_bzip2.py b/test/cgi/chunked_bzip2.py new file mode 100755 index 00000000..c34dc132 --- /dev/null +++ b/test/cgi/chunked_bzip2.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python +import bz2, os + +data1 = 'Two lines should be visible.
' +data2 = 'The second line.' + +f1 = bz2.BZ2File("/tmp/1.bz2", mode = "wb") +f1.write(data1) +f1.close() +f2 = bz2.BZ2File("/tmp/2.bz2", mode = "wb") +f2.write(data2) +f2.close() + +f = open("/tmp/1.bz2") +cd1 = f.read() +f.close() +f3 = open("/tmp/2.bz2") +cd2 = f3.read() +f3.close() + +os.unlink("/tmp/1.bz2") +os.unlink("/tmp/2.bz2") + +calosc = cd1 + cd2 +length = len(calosc) +how_many = 40 + +len1 = hex(how_many)[2:] +len2 = hex(length - how_many)[2:] + +os.write(1, "HTTP/1.0 200 OK\r\nDate: Sun, 20 Jan 2008 15:24:00 GMT\r\nServer: ddd\r\nTransfer-Encoding: chunked\r\nContent-Encoding: bzip2\r\nConnection: close\r\nContent-Type: text/html; charset=ISO-8859-1\r\n\r\n") +os.write(1, "%s\r\n" % len1) +os.write(1, calosc[:how_many]) +os.write(1, "\r\n%s\r\n" % len2) +os.write(1, calosc[how_many:]) +os.write(1, "\r\n0\r\n\r\n") diff --git a/test/cgi/chunked_deflate.py b/test/cgi/chunked_deflate.py new file mode 100755 index 00000000..59d7ee0b --- /dev/null +++ b/test/cgi/chunked_deflate.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +import os, zlib + +data1 = 'Two lines should be visible.
' +data2 = 'The second line.' + +cd1 = zlib.compress(data1) +cd2 = zlib.compress(data2) + +calosc = cd1 + cd2 +length = len(calosc) + +how_many = 40 +len1 = hex(how_many)[2:] +len2 = hex(length - how_many)[2:] + +os.write(1, "HTTP/1.0 200 OK\r\nDate: Sun, 20 Jan 2008 15:24:00 GMT\r\nServer: ddd\r\nTransfer-Encoding: chunked\r\nContent-Encoding: deflate\r\nConnection: close\r\nContent-Type: text/html; charset=ISO-8859-1\r\n\r\n") +os.write(1, "%s\r\n" % len1) +os.write(1, calosc[:how_many]) +os.write(1, "\r\n%s\r\n" % len2) +os.write(1, calosc[how_many:]) +os.write(1, "\r\n0\r\n\r\n") diff --git a/test/cgi/chunked_gzip.py b/test/cgi/chunked_gzip.py new file mode 100755 index 00000000..411fa99f --- /dev/null +++ b/test/cgi/chunked_gzip.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python +import gzip, os + +data1 = 'Two lines should be visible.
' +data2 = 'The second line.' + +f1 = gzip.GzipFile("/tmp/1.gz", mode = "wb") +f1.write(data1) +f1.close() +f2 = gzip.GzipFile("/tmp/2.gz", mode = "wb") +f2.write(data2) +f2.close() + +f = open("/tmp/1.gz") +cd1 = f.read() +f.close() +f3 = open("/tmp/2.gz") +cd2 = f3.read() +f3.close() + +os.unlink("/tmp/1.gz") +os.unlink("/tmp/2.gz") + +calosc = cd1 + cd2 +length = len(calosc) +how_many = 40 + +len1 = hex(how_many)[2:] +len2 = hex(length - how_many)[2:] + +os.write(1, "HTTP/1.0 200 OK\r\nDate: Sun, 20 Jan 2008 15:24:00 GMT\r\nServer: ddd\r\nTransfer-Encoding: chunked\r\nContent-Encoding: gzip\r\nConnection: close\r\nContent-Type: text/html; charset=ISO-8859-1\r\n\r\n") +os.write(1, "%s\r\n" % len1) +os.write(1, calosc[:how_many]) +os.write(1, "\r\n%s\r\n" % len2) +os.write(1, calosc[how_many:]) +os.write(1, "\r\n0\r\n\r\n") From 99c144381a9ccf942c6f6dc556c2fc9687f0398b Mon Sep 17 00:00:00 2001 From: Witold Filipczyk Date: Wed, 13 Feb 2008 11:12:58 +0100 Subject: [PATCH 15/31] chunked encoding tests: All these tests fail. --- test/cgi/chunked_bzip2.py | 33 +++++++++++---------------------- test/cgi/chunked_deflate.py | 28 +++++++++++----------------- test/cgi/chunked_gzip.py | 33 +++++++++++---------------------- 3 files changed, 33 insertions(+), 61 deletions(-) diff --git a/test/cgi/chunked_bzip2.py b/test/cgi/chunked_bzip2.py index c34dc132..0c8dfb00 100755 --- a/test/cgi/chunked_bzip2.py +++ b/test/cgi/chunked_bzip2.py @@ -1,36 +1,25 @@ #!/usr/bin/env python -import bz2, os +import bz2, os, time -data1 = 'Two lines should be visible.
' -data2 = 'The second line.' +data1 = 'Two lines should be visible.
The second line.' f1 = bz2.BZ2File("/tmp/1.bz2", mode = "wb") f1.write(data1) f1.close() -f2 = bz2.BZ2File("/tmp/2.bz2", mode = "wb") -f2.write(data2) -f2.close() f = open("/tmp/1.bz2") cd1 = f.read() f.close() -f3 = open("/tmp/2.bz2") -cd2 = f3.read() -f3.close() os.unlink("/tmp/1.bz2") -os.unlink("/tmp/2.bz2") -calosc = cd1 + cd2 -length = len(calosc) -how_many = 40 +length = len(cd1) +next_chunk = hex(length - 10)[2:] -len1 = hex(how_many)[2:] -len2 = hex(length - how_many)[2:] - -os.write(1, "HTTP/1.0 200 OK\r\nDate: Sun, 20 Jan 2008 15:24:00 GMT\r\nServer: ddd\r\nTransfer-Encoding: chunked\r\nContent-Encoding: bzip2\r\nConnection: close\r\nContent-Type: text/html; charset=ISO-8859-1\r\n\r\n") -os.write(1, "%s\r\n" % len1) -os.write(1, calosc[:how_many]) -os.write(1, "\r\n%s\r\n" % len2) -os.write(1, calosc[how_many:]) -os.write(1, "\r\n0\r\n\r\n") +os.write(1, "Date: Sun, 20 Jan 2008 15:24:00 GMT\r\nServer: ddd\r\nTransfer-Encoding: chunked\r\nContent-Encoding: bzip2\r\nConnection: close\r\nContent-Type: text/html; charset=ISO-8859-1\r\n") +os.write(1, "\r\na\r\n") +os.write(1, cd1[:10]) +time.sleep(2) +os.write(1, "\r\n%s\r\n" % next_chunk) +os.write(1, cd1[10:]) +os.write(1, "\r\n0\r\n") diff --git a/test/cgi/chunked_deflate.py b/test/cgi/chunked_deflate.py index 59d7ee0b..45f06828 100755 --- a/test/cgi/chunked_deflate.py +++ b/test/cgi/chunked_deflate.py @@ -1,22 +1,16 @@ #!/usr/bin/env python -import os, zlib - -data1 = 'Two lines should be visible.
' -data2 = 'The second line.' +import os, time, zlib +data1 = 'Two lines should be visible.
The second line.' cd1 = zlib.compress(data1) -cd2 = zlib.compress(data2) -calosc = cd1 + cd2 -length = len(calosc) +length = len(cd1) +next_chunk = hex(length - 10)[2:] -how_many = 40 -len1 = hex(how_many)[2:] -len2 = hex(length - how_many)[2:] - -os.write(1, "HTTP/1.0 200 OK\r\nDate: Sun, 20 Jan 2008 15:24:00 GMT\r\nServer: ddd\r\nTransfer-Encoding: chunked\r\nContent-Encoding: deflate\r\nConnection: close\r\nContent-Type: text/html; charset=ISO-8859-1\r\n\r\n") -os.write(1, "%s\r\n" % len1) -os.write(1, calosc[:how_many]) -os.write(1, "\r\n%s\r\n" % len2) -os.write(1, calosc[how_many:]) -os.write(1, "\r\n0\r\n\r\n") +os.write(1, "Date: Sun, 20 Jan 2008 15:24:00 GMT\r\nServer: ddd\r\nTransfer-Encoding: chunked\r\nContent-Encoding: deflate\r\nConnection: close\r\nContent-Type: text/html; charset=ISO-8859-1\r\n") +os.write(1, "\r\na\r\n") +os.write(1, cd1[:10]) +time.sleep(2) +os.write(1, "\r\n%s\r\n" % next_chunk) +os.write(1, cd1[10:]) +os.write(1, "\r\n0\r\n") diff --git a/test/cgi/chunked_gzip.py b/test/cgi/chunked_gzip.py index 411fa99f..cd20c9f8 100755 --- a/test/cgi/chunked_gzip.py +++ b/test/cgi/chunked_gzip.py @@ -1,36 +1,25 @@ #!/usr/bin/env python -import gzip, os +import gzip, os, time -data1 = 'Two lines should be visible.
' -data2 = 'The second line.' +data1 = 'Two lines should be visible.
The second line.' f1 = gzip.GzipFile("/tmp/1.gz", mode = "wb") f1.write(data1) f1.close() -f2 = gzip.GzipFile("/tmp/2.gz", mode = "wb") -f2.write(data2) -f2.close() f = open("/tmp/1.gz") cd1 = f.read() f.close() -f3 = open("/tmp/2.gz") -cd2 = f3.read() -f3.close() os.unlink("/tmp/1.gz") -os.unlink("/tmp/2.gz") -calosc = cd1 + cd2 -length = len(calosc) -how_many = 40 +length = len(cd1) +next_chunk = hex(length - 10)[2:] -len1 = hex(how_many)[2:] -len2 = hex(length - how_many)[2:] - -os.write(1, "HTTP/1.0 200 OK\r\nDate: Sun, 20 Jan 2008 15:24:00 GMT\r\nServer: ddd\r\nTransfer-Encoding: chunked\r\nContent-Encoding: gzip\r\nConnection: close\r\nContent-Type: text/html; charset=ISO-8859-1\r\n\r\n") -os.write(1, "%s\r\n" % len1) -os.write(1, calosc[:how_many]) -os.write(1, "\r\n%s\r\n" % len2) -os.write(1, calosc[how_many:]) -os.write(1, "\r\n0\r\n\r\n") +os.write(1, "Date: Sun, 20 Jan 2008 15:24:00 GMT\r\nServer: ddd\r\nTransfer-Encoding: chunked\r\nContent-Encoding: gzip\r\nConnection: close\r\nContent-Type: text/html; charset=ISO-8859-1\r\n") +os.write(1, "\r\na\r\n") +os.write(1, cd1[:10]) +time.sleep(2) +os.write(1, "\r\n%s\r\n" % next_chunk) +os.write(1, cd1[10:]) +os.write(1, "\r\n0\r\n") From 8e0938d2fcb93f3d1d50d0b013d24710c77af8e5 Mon Sep 17 00:00:00 2001 From: Witold Filipczyk Date: Thu, 14 Feb 2008 10:39:56 +0100 Subject: [PATCH 16/31] Encoding tests rewritten. They do not create files in /tmp. --- test/cgi/chunked_bzip2.py | 11 +---------- test/cgi/chunked_gzip.py | 13 ++++++------- 2 files changed, 7 insertions(+), 17 deletions(-) diff --git a/test/cgi/chunked_bzip2.py b/test/cgi/chunked_bzip2.py index 0c8dfb00..3e07a174 100755 --- a/test/cgi/chunked_bzip2.py +++ b/test/cgi/chunked_bzip2.py @@ -2,16 +2,7 @@ import bz2, os, time data1 = 'Two lines should be visible.
The second line.' - -f1 = bz2.BZ2File("/tmp/1.bz2", mode = "wb") -f1.write(data1) -f1.close() - -f = open("/tmp/1.bz2") -cd1 = f.read() -f.close() - -os.unlink("/tmp/1.bz2") +cd1 = bz2.compress(data1) length = len(cd1) next_chunk = hex(length - 10)[2:] diff --git a/test/cgi/chunked_gzip.py b/test/cgi/chunked_gzip.py index cd20c9f8..e01f6070 100755 --- a/test/cgi/chunked_gzip.py +++ b/test/cgi/chunked_gzip.py @@ -1,17 +1,16 @@ #!/usr/bin/env python -import gzip, os, time +import gzip, os, time, StringIO + +output = StringIO.StringIO() data1 = 'Two lines should be visible.
The second line.' -f1 = gzip.GzipFile("/tmp/1.gz", mode = "wb") +f1 = gzip.GzipFile("/tmp/1.gz", mode = "wb", fileobj=output) f1.write(data1) f1.close() -f = open("/tmp/1.gz") -cd1 = f.read() -f.close() - -os.unlink("/tmp/1.gz") +cd1 = output.getvalue() +output.close() length = len(cd1) next_chunk = hex(length - 10)[2:] From cfb2ef63d622eef0e605b1be03d29bfba18d45dc Mon Sep 17 00:00:00 2001 From: Witold Filipczyk Date: Mon, 11 Feb 2008 15:30:14 +0100 Subject: [PATCH 17/31] encoding: Fixed a chunked decompression and added the deflate. --- src/encoding/Makefile | 2 +- src/encoding/deflate.c | 200 ++++++++++++++++++++++++ src/encoding/{gzip.h => deflate.h} | 6 +- src/encoding/encoding.c | 3 +- src/encoding/encoding.h | 1 + src/encoding/gzip.c | 241 ----------------------------- src/protocol/http/http.c | 59 +++---- 7 files changed, 230 insertions(+), 282 deletions(-) create mode 100644 src/encoding/deflate.c rename src/encoding/{gzip.h => deflate.h} (50%) delete mode 100644 src/encoding/gzip.c diff --git a/src/encoding/Makefile b/src/encoding/Makefile index d51b4166..938497ba 100644 --- a/src/encoding/Makefile +++ b/src/encoding/Makefile @@ -2,7 +2,7 @@ top_builddir=../.. include $(top_builddir)/Makefile.config OBJS-$(CONFIG_BZIP2) += bzip2.o -OBJS-$(CONFIG_GZIP) += gzip.o +OBJS-$(CONFIG_GZIP) += deflate.o OBJS-$(CONFIG_LZMA) += lzma.o LzmaDecode.o OBJS = encoding.o diff --git a/src/encoding/deflate.c b/src/encoding/deflate.c new file mode 100644 index 00000000..c211942f --- /dev/null +++ b/src/encoding/deflate.c @@ -0,0 +1,200 @@ +/* deflate/gzip encoding backend */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#ifdef HAVE_UNISTD_H +#include +#endif +#ifdef HAVE_ZLIB_H +#include +#endif +#include + +#include "elinks.h" + +#include "encoding/deflate.h" +#include "encoding/encoding.h" +#include "util/memory.h" + +/* How many bytes of compressed data to read before decompressing. */ +#define ELINKS_DEFLATE_BUFFER_LENGTH 5000 + +struct deflate_enc_data { + z_stream deflate_stream; + + /* The file descriptor from which we read. */ + int fdread; + + unsigned int last_read:1; + + /* A buffer for data that has been read from the file but not + * yet decompressed. z_stream.next_in and z_stream.avail_in + * refer to this buffer. */ + unsigned char buf[ELINKS_DEFLATE_BUFFER_LENGTH]; +}; + +static int +deflate_open(struct stream_encoded *stream, int fd) +{ + /* A zero-initialized z_stream. The compiler ensures that all + * pointer members in it are null. (Can't do this with memset + * because C99 does not require all-bits-zero to be a null + * pointer.) */ + static const z_stream null_z_stream = {0}; + int err; + + struct deflate_enc_data *data = mem_alloc(sizeof(*data)); + + stream->data = NULL; + if (!data) { + return -1; + } + + /* Initialize all members of *data, except data->buf[], which + * will be initialized on demand by deflate_read. */ + copy_struct(&data->deflate_stream, &null_z_stream); + data->fdread = fd; + data->last_read = 0; + + err = inflateInit2(&data->deflate_stream, MAX_WBITS | 32); + if (err != Z_OK) { + mem_free(data); + return -1; + } + stream->data = data; + + return 0; +} + +static int +deflate_read(struct stream_encoded *stream, unsigned char *buf, int len) +{ + struct deflate_enc_data *data = (struct deflate_enc_data *) stream->data; + int err = 0; + + if (!data) return -1; + + assert(len > 0); + + if (data->last_read) return 0; + + data->deflate_stream.avail_out = len; + data->deflate_stream.next_out = buf; + + do { + if (data->deflate_stream.avail_in == 0) { + int l = safe_read(data->fdread, data->buf, + ELINKS_DEFLATE_BUFFER_LENGTH); + + if (l == -1) { + if (errno == EAGAIN) + break; + else + return -1; /* I/O error */ + } else if (l == 0) { + /* EOF. It is error: we wait for more bytes */ + return -1; + } + + data->deflate_stream.next_in = data->buf; + data->deflate_stream.avail_in = l; + } + err = inflate(&data->deflate_stream, Z_SYNC_FLUSH); + if (err == Z_STREAM_END) { + data->last_read = 1; + break; + } else if (err != Z_OK) { + return -1; + } + } while (data->deflate_stream.avail_out > 0); + + assert(len - data->deflate_stream.avail_out == data->deflate_stream.next_out - buf); + return len - data->deflate_stream.avail_out; +} + +static unsigned char * +deflate_decode_buffer(unsigned char *data, int len, int *new_len) +{ + z_stream stream; + unsigned char *buffer = NULL; + int error; + + if (!len) return NULL; + memset(&stream, 0, sizeof(z_stream)); + stream.next_in = data; + stream.avail_in = len; + + if (inflateInit2(&stream, MAX_WBITS | 32) != Z_OK) + return NULL; + + do { + unsigned char *new_buffer; + size_t size = stream.total_out + MAX_STR_LEN; + + new_buffer = mem_realloc(buffer, size); + if (!new_buffer) { + error = Z_MEM_ERROR; + break; + } + + buffer = new_buffer; + stream.next_out = buffer + stream.total_out; + stream.avail_out = MAX_STR_LEN; + + error = inflate(&stream, Z_SYNC_FLUSH); + if (error == Z_STREAM_END) { + *new_len = stream.total_out; + error = Z_OK; + break; + } + } while (error == Z_OK && stream.avail_in > 0); + + inflateEnd(&stream); + + if (error != Z_OK) { + if (buffer) mem_free(buffer); + *new_len = 0; + return NULL; + } + + return buffer; +} + +static void +deflate_close(struct stream_encoded *stream) +{ + struct deflate_enc_data *data = (struct deflate_enc_data *) stream->data; + + if (data) { + inflateEnd(&data->deflate_stream); + close(data->fdread); + mem_free(data); + stream->data = 0; + } +} + +static const unsigned char *const deflate_extensions[] = { NULL }; + +const struct decoding_backend deflate_decoding_backend = { + "deflate", + deflate_extensions, + deflate_open, + deflate_read, + deflate_decode_buffer, + deflate_close, +}; + +static const unsigned char *const gzip_extensions[] = { ".gz", ".tgz", NULL }; + +const struct decoding_backend gzip_decoding_backend = { + "gzip", + gzip_extensions, + deflate_open, + deflate_read, + deflate_decode_buffer, + deflate_close, +}; diff --git a/src/encoding/gzip.h b/src/encoding/deflate.h similarity index 50% rename from src/encoding/gzip.h rename to src/encoding/deflate.h index c13038e3..4d3d2c3a 100644 --- a/src/encoding/gzip.h +++ b/src/encoding/deflate.h @@ -1,11 +1,13 @@ -#ifndef EL__ENCODING_GZIP_H -#define EL__ENCODING_GZIP_H +#ifndef EL__ENCODING_DEFLATE_H +#define EL__ENCODING_DEFLATE_H #include "encoding/encoding.h" #ifdef CONFIG_GZIP +extern const struct decoding_backend deflate_decoding_backend; extern const struct decoding_backend gzip_decoding_backend; #else +#define deflate_decoding_backend dummy_decoding_backend #define gzip_decoding_backend dummy_decoding_backend #endif diff --git a/src/encoding/encoding.c b/src/encoding/encoding.c index 42dc62b1..4d8c2d35 100644 --- a/src/encoding/encoding.c +++ b/src/encoding/encoding.c @@ -84,7 +84,7 @@ static const struct decoding_backend dummy_decoding_backend = { /* Dynamic backend area */ #include "encoding/bzip2.h" -#include "encoding/gzip.h" +#include "encoding/deflate.h" #include "encoding/lzma.h" static const struct decoding_backend *const decoding_backends[] = { @@ -92,6 +92,7 @@ static const struct decoding_backend *const decoding_backends[] = { &gzip_decoding_backend, &bzip2_decoding_backend, &lzma_decoding_backend, + &deflate_decoding_backend, }; diff --git a/src/encoding/encoding.h b/src/encoding/encoding.h index 8b32c3b3..00b4ab2c 100644 --- a/src/encoding/encoding.h +++ b/src/encoding/encoding.h @@ -9,6 +9,7 @@ enum stream_encoding { ENCODING_GZIP, ENCODING_BZIP2, ENCODING_LZMA, + ENCODING_DEFLATE, /* Max. number of known encoding including ENCODING_NONE. */ ENCODINGS_KNOWN, diff --git a/src/encoding/gzip.c b/src/encoding/gzip.c deleted file mode 100644 index a4680007..00000000 --- a/src/encoding/gzip.c +++ /dev/null @@ -1,241 +0,0 @@ -/* Gzip encoding (ENCODING_GZIP) backend */ - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#include -#include -#ifdef HAVE_UNISTD_H -#include -#endif -#ifdef HAVE_ZLIB_H -#include -#endif - -#include "elinks.h" - -#include "encoding/encoding.h" -#include "encoding/gzip.h" -#include "osdep/osdep.h" -#include "util/memory.h" - - -static int -gzip_open(struct stream_encoded *stream, int fd) -{ - stream->data = (void *) gzdopen(fd, "rb"); - if (!stream->data) return -1; - - return 0; -} - -static int -gzip_read(struct stream_encoded *stream, unsigned char *data, int len) -{ - gzclearerr((gzFile *) stream->data); - return gzread((gzFile *) stream->data, data, len); -} - - -/* The following code for decoding gzip in memory is a mix of code from zlib's - * gzio.c file copyrighted 1995-2002 by Jean-loup Gailly and the costumized - * header extraction in the linux kernels lib/inflate.c file not copyrighted - * 1992 by Mark Adler. */ - -static int gzip_header_magic[2] = { 0x1f, 0x8b }; - -enum gzip_header_flag { - GZIP_ASCII_TEXT = 0x01, /* File probably ascii text (unused) */ - GZIP_HEADER_CRC = 0x02, /* Header CRC present */ - GZIP_EXTRA_FIELD = 0x04, /* Extra field present */ - GZIP_ORIG_NAME = 0x08, /* Original file name present */ - GZIP_COMMENT = 0x10, /* File comment present */ - GZIP_RESERVED = 0xE0, /* bits 5..7: reserved */ -}; - -/* Read a byte from a gz_stream; update next_in and avail_in. Return EOF for - * end of file. */ -static int -get_gzip_byte(z_stream *stream) -{ - if (stream->avail_in == 0) - return EOF; - - stream->avail_in--; - - return *(stream->next_in)++; -} - -#define skip_gzip_bytes(stream, bytes) \ - do { int i = bytes; while (i-- > 0) get_gzip_byte(stream); } while (0) - -#define skip_gzip_string(stream) \ - do { int i; while ((i = get_gzip_byte(stream)) != 0 && i != EOF) ; } while (0) - -/* Check the gzip header of a gz_stream opened for reading. Set the stream mode - * to transparent if the gzip magic header is not present; set s->err to - * Z_DATA_ERROR if the magic header is present but the rest of the header is - * incorrect. */ -static int -skip_gzip_header(z_stream *stream) -{ - unsigned int len; - int method; /* method byte */ - int flags; /* flags byte */ - - /* Check the gzip magic header */ - for (len = 0; len < 2; len++) { - int byte = get_gzip_byte(stream); - - if (byte != gzip_header_magic[len]) { - if (len != 0) { - stream->avail_in++; - stream->next_in--; - } - - if (byte != EOF) { - stream->avail_in++; - stream->next_in--; - } - - return stream->avail_in != 0 ? Z_OK : Z_STREAM_END; - } - } - - method = get_gzip_byte(stream); - flags = get_gzip_byte(stream); - - if (method != Z_DEFLATED || (flags & GZIP_RESERVED) != 0) - return Z_DATA_ERROR; - - /* Discard time, xflags and OS code: */ - skip_gzip_bytes(stream, 6); - - if (flags & GZIP_EXTRA_FIELD) { - /* Skip the extra field */ - len = (unsigned int) get_gzip_byte(stream); - len += ((unsigned int) get_gzip_byte(stream)) << 8; - - /* If EOF is encountered @len is garbage, but the loop below - * will quit anyway. */ - while (len-- > 0 && get_gzip_byte(stream) != EOF) ; - } - - /* Skip the original file name */ - if (flags & GZIP_ORIG_NAME) - skip_gzip_string(stream); - - /* Skip the .gz file comment */ - if (flags & GZIP_COMMENT) - skip_gzip_string(stream); - - /* Skip the header CRC */ - if (flags & GZIP_HEADER_CRC) - skip_gzip_bytes(stream, 2); - - return Z_OK; -} - - -/* Freaking dammit. This is impossible for me to get working. */ -static unsigned char * -gzip_decode_buffer(unsigned char *data, int len, int *new_len) -{ - unsigned char *buffer = NULL; - int error = Z_OK; - int tries, wbits; - - /* This WBITS loop thing was something I got from - * http://lists.infradead.org/pipermail/linux-mtd/2002-March/004429.html - * but it doesn't fix it. :/ --jonas */ - /* -MAX_WBITS impiles -> suppress zlib header and adler32. try first - * with -MAX_WBITS, if that fails, try MAX_WBITS to be backwards - * compatible */ - wbits = -MAX_WBITS; - - for (tries = 0; tries < 2; tries++) { - z_stream stream; - - memset(&stream, 0, sizeof(z_stream)); - - /* FIXME: Use inflateInit2() to configure low memory - * usage for CONFIG_SMALL configurations. --jonas */ - error = inflateInit2(&stream, wbits); - if (error != Z_OK) break; - - stream.next_in = (char *)data; - stream.avail_in = len; - - error = skip_gzip_header(&stream); - if (error != Z_OK) { - stream.next_in = (char *)data; - stream.avail_in = len; - } - - do { - unsigned char *new_buffer; - size_t size = stream.total_out + MAX_STR_LEN; - - assert(stream.total_out >= 0); - assert(stream.next_in); - - new_buffer = mem_realloc(buffer, size); - if (!new_buffer) { - error = Z_MEM_ERROR; - break; - } - - buffer = new_buffer; - stream.next_out = buffer + stream.total_out; - stream.avail_out = MAX_STR_LEN; - - error = inflate(&stream, Z_NO_FLUSH); - if (error == Z_STREAM_END) { - /* Here gzio.c has some detection of - * concatenated .gz files and will do a gzip - * header skip and an inflateReset() call - * before continuing. It partly uses CRC to - * detect that. */ - *new_len = stream.total_out; - error = Z_OK; - break; - } - - } while (error == Z_OK && stream.avail_in > 0); - - inflateEnd(&stream); - - if (error != Z_DATA_ERROR) - break; - - /* Try again with next wbits */ - wbits = -wbits; - } - - if (error != Z_OK) { - if (buffer) mem_free(buffer); - *new_len = 0; - return NULL; - } - - return buffer; -} - - -static void -gzip_close(struct stream_encoded *stream) -{ - gzclose((gzFile *) stream->data); -} - -static const unsigned char *const gzip_extensions[] = { ".gz", ".tgz", NULL }; - -const struct decoding_backend gzip_decoding_backend = { - "gzip", - gzip_extensions, - gzip_open, - gzip_read, - gzip_decode_buffer, - gzip_close, -}; diff --git a/src/protocol/http/http.c b/src/protocol/http/http.c index bf675121..ab27239d 100644 --- a/src/protocol/http/http.c +++ b/src/protocol/http/http.c @@ -759,7 +759,7 @@ http_send_header(struct socket *socket) add_to_string(&header, ", "); #endif - add_to_string(&header, "gzip"); + add_to_string(&header, "deflate, gzip"); #endif add_crlf_to_string(&header); #endif @@ -995,28 +995,23 @@ decompress_data(struct connection *conn, unsigned char *data, int len, int *new_len) { struct http_connection_info *http = conn->info; - /* to_read is number of bytes to be read from the decoder. It is 65536 - * (then we are just emptying the decoder buffer as we finished the walk - * through the incoming stream already) or PIPE_BUF / 2 (when we are - * still walking through the stream - then we write PIPE_BUF / 2 to the - * pipe and read it back to the decoder ASAP; the point is that we can't - * write more than PIPE_BUF to the pipe at once, but we also have to - * never let read_encoded() (gzread(), in fact) to empty the pipe - that - * causes further malfunction of zlib :[ ... so we will make sure that - * we will always have at least PIPE_BUF / 2 + 1 in the pipe (returning - * early otherwise)). */ enum { NORMAL, FINISHING } state = NORMAL; int did_read = 0; int *length_of_block; unsigned char *output = NULL; - length_of_block = (http->length == LEN_CHUNKED ? &http->chunk_remaining - : &http->length); - #define BIG_READ 65536 - if (!*length_of_block) { - /* Going to finish this decoding bussiness. */ - state = FINISHING; + + if (http->length == LEN_CHUNKED) { + if (http->chunk_remaining == CHUNK_ZERO_SIZE) + state = FINISHING; + length_of_block = &http->chunk_remaining; + } else { + length_of_block = &http->length; + if (!*length_of_block) { + /* Going to finish this decoding bussiness. */ + state = FINISHING; + } } if (conn->content_encoding == ENCODING_NONE) { @@ -1035,16 +1030,9 @@ decompress_data(struct connection *conn, unsigned char *data, int len, } do { - /* The initial value is used only when state == NORMAL. - * Unconditional initialization avoids a GCC warning. */ - int to_read = PIPE_BUF / 2; - if (state == NORMAL) { /* ... we aren't finishing yet. */ - int written; - - written = safe_write(conn->stream_pipes[1], data, - len > to_read ? to_read : len); + int written = safe_write(conn->stream_pipes[1], data, len); if (written > 0) { data += written; @@ -1084,14 +1072,13 @@ decompress_data(struct connection *conn, unsigned char *data, int len, did_read = read_encoded(conn->stream, output + *new_len, BIG_READ); if (did_read > 0) *new_len += did_read; - else if (did_read == -1) { - mem_free_set(&output, NULL); - *new_len = 0; - break; /* Loop prevention (bug 517), is this correct ? --Zas */ + else { + if (did_read < 0) state = FINISHING; + break; } - } while (len || did_read == BIG_READ); + } while (len || (did_read == BIG_READ)); - shutdown_connection_stream(conn); + if (state == FINISHING) shutdown_connection_stream(conn); return output; } @@ -1218,11 +1205,8 @@ read_chunked_http_data(struct connection *conn, struct read_buffer *rb) } else { unsigned char *data; int data_len; - int len; int zero = (http->chunk_remaining == CHUNK_ZERO_SIZE); - - if (zero) http->chunk_remaining = 0; - len = http->chunk_remaining; + int len = zero ? 0 : http->chunk_remaining; /* Maybe everything necessary didn't come yet.. */ int_upper_bound(&len, rb->length); @@ -1863,6 +1847,8 @@ again: if (file_encoding != ENCODING_GZIP && (!strcasecmp(d, "gzip") || !strcasecmp(d, "x-gzip"))) conn->content_encoding = ENCODING_GZIP; + if (!strcasecmp(d, "deflate") || !strcasecmp(d, "x-deflate")) + conn->content_encoding = ENCODING_DEFLATE; #endif #ifdef CONFIG_BZIP2 @@ -1879,8 +1865,7 @@ again: conn->cached->encoding_info = stracpy(get_encoding_name(conn->content_encoding)); } - if (http->length == -1 - || (PRE_HTTP_1_1(http->recv_version) && http->close)) + if (http->length == -1 || http->close) socket->state = SOCKET_END_ONCLOSE; read_http_data(socket, rb); From 0b363eb2457df73bf48b8842db35f5832a130705 Mon Sep 17 00:00:00 2001 From: Witold Filipczyk Date: Sun, 17 Feb 2008 12:39:33 +0100 Subject: [PATCH 18/31] content encoding: handle the case when the len is 0 in the chunked encoding. We do not want to start decompression when the len is 0 with the chunked encoding. --- src/protocol/http/http.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/protocol/http/http.c b/src/protocol/http/http.c index ab27239d..69ce6709 100644 --- a/src/protocol/http/http.c +++ b/src/protocol/http/http.c @@ -1034,7 +1034,7 @@ decompress_data(struct connection *conn, unsigned char *data, int len, /* ... we aren't finishing yet. */ int written = safe_write(conn->stream_pipes[1], data, len); - if (written > 0) { + if (written >= 0) { data += written; len -= written; From 474f1f4268d24ba7782ab57ae47c87113ec2a8f5 Mon Sep 17 00:00:00 2001 From: Kalle Olavi Niemitalo Date: Sun, 17 Feb 2008 18:55:41 +0200 Subject: [PATCH 19/31] encoding: Set *new_len even if data was truncated. Previously, bzip2_decode_buffer and deflate_decode_buffer left *new_len unchanged if the compressed input data ended unexpectedly. This behaviour was also inherited by decode_encoded_buffer, whose only caller render_encoded_document preinitializes the variable and so did not crash. With this change, the functions now store in *new_len the number of bytes that were successfully decoded, even if more bytes were expected. An error should perhaps be reported to the user, but I don't think the previous version did that either, as it returned a non-NULL pointer. --- src/encoding/bzip2.c | 11 ++++++----- src/encoding/deflate.c | 11 ++++++----- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/encoding/bzip2.c b/src/encoding/bzip2.c index 87434b15..8e7139a6 100644 --- a/src/encoding/bzip2.c +++ b/src/encoding/bzip2.c @@ -138,6 +138,8 @@ bzip2_decode_buffer(unsigned char *data, int len, int *new_len) unsigned char *buffer = NULL; int error; + *new_len = 0; /* default, left there if an error occurs */ + memset(&stream, 0, sizeof(bz_stream)); stream.next_in = data; stream.avail_in = len; @@ -168,7 +170,6 @@ bzip2_decode_buffer(unsigned char *data, int len, int *new_len) error = BZ2_bzDecompress(&stream); if (error == BZ_STREAM_END) { - *new_len = stream.total_out_lo32; error = BZ_OK; break; } @@ -181,13 +182,13 @@ bzip2_decode_buffer(unsigned char *data, int len, int *new_len) BZ2_bzDecompressEnd(&stream); - if (error != BZ_OK) { + if (error == BZ_OK) { + *new_len = stream.total_out_lo32; + return buffer; + } else { if (buffer) mem_free(buffer); - *new_len = 0; return NULL; } - - return buffer; } static void diff --git a/src/encoding/deflate.c b/src/encoding/deflate.c index c211942f..95b33c23 100644 --- a/src/encoding/deflate.c +++ b/src/encoding/deflate.c @@ -123,6 +123,8 @@ deflate_decode_buffer(unsigned char *data, int len, int *new_len) unsigned char *buffer = NULL; int error; + *new_len = 0; /* default, left there if an error occurs */ + if (!len) return NULL; memset(&stream, 0, sizeof(z_stream)); stream.next_in = data; @@ -147,7 +149,6 @@ deflate_decode_buffer(unsigned char *data, int len, int *new_len) error = inflate(&stream, Z_SYNC_FLUSH); if (error == Z_STREAM_END) { - *new_len = stream.total_out; error = Z_OK; break; } @@ -155,13 +156,13 @@ deflate_decode_buffer(unsigned char *data, int len, int *new_len) inflateEnd(&stream); - if (error != Z_OK) { + if (error == Z_OK) { + *new_len = stream.total_out; + return buffer; + } else { if (buffer) mem_free(buffer); - *new_len = 0; return NULL; } - - return buffer; } static void From a7232792dd0c343af0e561971634ce0ec3ce4a1d Mon Sep 17 00:00:00 2001 From: Kalle Olavi Niemitalo Date: Sun, 17 Feb 2008 19:08:04 +0200 Subject: [PATCH 20/31] encoding: Don't leak a pipe fd if open_encoded() fails. --- src/network/connection.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/network/connection.c b/src/network/connection.c index 6eeb8939..879bd84d 100644 --- a/src/network/connection.c +++ b/src/network/connection.c @@ -374,6 +374,10 @@ shutdown_connection_stream(struct connection *conn) if (conn->stream) { close_encoded(conn->stream); conn->stream = NULL; + } else if (conn->stream_pipes[0] >= 0) { + /* close_encoded() usually closes this end of the pipe, + * but open_encoded() apparently failed this time. */ + close(conn->stream_pipes[0]); } if (conn->stream_pipes[1] >= 0) close(conn->stream_pipes[1]); From ac030867542ec46c825f88a8ce5773b5a8fb0ce5 Mon Sep 17 00:00:00 2001 From: Kalle Olavi Niemitalo Date: Sun, 17 Feb 2008 18:44:48 +0200 Subject: [PATCH 21/31] configure.in: Explain why to check for gzclearerr. --- configure.in | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/configure.in b/configure.in index d365ec43..82c4f027 100644 --- a/configure.in +++ b/configure.in @@ -480,6 +480,11 @@ AC_DEFUN([EL_CONFIG_OPTIONAL_LIBRARY], EL_CONFIG_OPTIONAL_LIBRARY(CONFIG_GPM, gpm, gpm.h, gpm, Gpm_Open, [ --without-gpm disable gpm (mouse) support]) +# ELinks calls deflateInit2 with windowBits = MAX_WBITS | 32, to +# enable automatic decoding of both zlib and gzip headers. This +# feature was added in zlib 1.2.0.2; earlier versions return an error. +# The gzclearerr function was also added in zlib 1.2.0.2, so check for +# that, even though ELinks does not actually call gzclearerr. EL_CONFIG_OPTIONAL_LIBRARY(CONFIG_GZIP, zlib, zlib.h, z, gzclearerr, [ --without-zlib disable zlib support]) From a9da2d5802c76d46379e8f9567a4e5b7a334dd25 Mon Sep 17 00:00:00 2001 From: Kalle Olavi Niemitalo Date: Sun, 17 Feb 2008 20:44:47 +0200 Subject: [PATCH 22/31] NEWS: mention bug 534 --- NEWS | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/NEWS b/NEWS index 1b56f06e..bde3aee0 100644 --- a/NEWS +++ b/NEWS @@ -63,12 +63,13 @@ Miscellaneous: * critical bug 869: long mailcap entry buffer overflow (non-security) when downloading * tabs opened by -remote now go behind existing dialogs +* major bug 534, enhancement 517: fix HTTP gzip and bzip2 + decompression, and add deflate and LZMA * major bug 503: various fixes in parsing and updating of elinks.conf * Debian bug 257762: turn terminal transparency off by default * bug 724: better parsing of escape sequences and control sequences from the terminal * bug 948: fix wrong UTF-8 output after the charset menu was used -* gzip_read: always call gzclearerr * bug 816: convert entity references in input/@value only once * bug 916: if a mailcap entry has no %s, provide the file as stdin * bug 744: don't change ``//'' to ``/'' in URIs @@ -93,8 +94,6 @@ Miscellaneous: itself acts as a pager * enhancement 790: If-Modified-Since and If-None-Match * enhancement: HTTP negotiate-auth using GSSAPI -* enhancement 517: fixed and enabled HTTP bzip2 decompression -* enhancement: LZMA decompression * enhancement: FSP progress indicator and password prompt * enhancement: autocreate directories needed to download a file * enhancement: ``Add server'' button in the cookie manager From ae5446e9bd7cf11f90fbf20bdc5e9c2d1529ebc2 Mon Sep 17 00:00:00 2001 From: Witold Filipczyk Date: Thu, 21 Feb 2008 14:54:19 +0100 Subject: [PATCH 23/31] pl.po: fixes. --- po/pl.po | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/po/pl.po b/po/pl.po index c1ac4f86..a35e1ff4 100644 --- a/po/pl.po +++ b/po/pl.po @@ -5206,7 +5206,7 @@ msgstr[2] "%ld sesji" #: src/dialogs/info.c:252 msgid "Memory allocated" -msgstr "Pamięć zaalokowana" +msgstr "Przydzielona pamięć" #: src/dialogs/info.c:260 #, c-format @@ -6576,7 +6576,7 @@ msgstr "Włącz obsługę mime.types." #: src/mime/backend/mimetypes.c:58 msgid "The search path for mime.types files. Colon-separated list of files." -msgstr "Odzielona dwukropkami lista ścieżek do plików mime.types." +msgstr "Oddzielona dwukropkami lista ścieżek do plików mime.types." #: src/mime/dialogs.c:66 msgid "Delete extension" @@ -6816,9 +6816,9 @@ msgid "" "programs for handling it. Downloading URLs using external\n" "programs is not supported." msgstr "" -"Ten URL zawiera protokół nie obsługiwany natywnie przez\n" -"ELinks co oznacza iż ELinks będzie polegał na zewnętrznym\n" -"programie do jego obsługi. Ściąganie URLi za pomocą\n" +"Ten URL zawiera protokół nie obsługiwany bezpośrednio przez\n" +"ELinksa co oznacza iż ELinks będzie polegał na zewnętrznym\n" +"programie do jego obsługi. Ściąganie URL-i za pomocą\n" "zewnętrznych programów nie jest w tej chwili wspierane." #: src/network/state.c:71 @@ -6875,7 +6875,7 @@ msgstr "Błąd SSL" #: src/network/state.c:91 msgid "This version of ELinks does not contain SSL/TLS support" -msgstr "Ta wersja ELinksa nie zawiera wsparcia dla SSL/TSL" +msgstr "Ta wersja ELinksa nie zawiera wsparcia dla SSL/TLS" #: src/network/state.c:94 msgid "JavaScript support is not enabled" @@ -6949,7 +6949,7 @@ msgstr "Błąd BitTorrenta" #: src/network/state.c:124 msgid "The BitTorrent metainfo file contained errors" -msgstr "Plik metainformacji BitTorrent zawiera błędy" +msgstr "Plik metainformacji BitTorrenta zawiera błędy" #: src/network/state.c:125 msgid "The tracker requesting failed" @@ -7118,7 +7118,7 @@ msgstr "Maksymalny numer portu" #: src/protocol/bittorrent/bittorrent.c:44 msgid "The maximum port to try and listen on." -msgstr "Maksymalny numer portu do nasłychiwania." +msgstr "Maksymalny numer portu do nasłuchiwania." #. ****************************************************************** #. Tracker connection options: @@ -8335,7 +8335,7 @@ msgstr "Dane uwierzytelniające" msgid "Credentials file passed to smbclient via -A option." msgstr "" "Plik z danymi uwierzytelniającymi przekazywany\n" -"do smbclient-a przy użyciu opcji -A." +"do smbclienta przy użyciu opcji -A." # #. name: From 8492e6b1af49deab4c554b51eb2d23565957ab28 Mon Sep 17 00:00:00 2001 From: Witold Filipczyk Date: Mon, 18 Feb 2008 14:15:53 +0100 Subject: [PATCH 24/31] lzma: used the lzma library from tukaani.org. --- configure.in | 4 +- src/encoding/LzmaDecode.c | 588 -------------------------------------- src/encoding/LzmaDecode.h | 131 --------- src/encoding/Makefile | 2 +- src/encoding/lzma.c | 225 +++++++-------- src/protocol/http/http.c | 5 + 6 files changed, 110 insertions(+), 845 deletions(-) delete mode 100644 src/encoding/LzmaDecode.c delete mode 100644 src/encoding/LzmaDecode.h diff --git a/configure.in b/configure.in index 82c4f027..aa802662 100644 --- a/configure.in +++ b/configure.in @@ -499,8 +499,8 @@ if test "x{with_gc}" != xno; then [ --with-gc enable Boehm's garbage collector]) fi -EL_ARG_ENABLE(CONFIG_LZMA, lzma, [lzma], - [ --enable-lzma enable lzma encoding support]) +EL_CONFIG_OPTIONAL_LIBRARY(CONFIG_LZMA, lzma, lzma.h, lzma, lzma_code, + [ --without-lzma disable lzma encoding support]) # =================================================================== # Check for GSSAPI, optional even if installed. diff --git a/src/encoding/LzmaDecode.c b/src/encoding/LzmaDecode.c deleted file mode 100644 index 976a8649..00000000 --- a/src/encoding/LzmaDecode.c +++ /dev/null @@ -1,588 +0,0 @@ -/* - LzmaDecode.c - LZMA Decoder (optimized for Speed version) - - LZMA SDK 4.22 Copyright (c) 1999-2005 Igor Pavlov (2005-06-10) - http://www.7-zip.org/ - - LZMA SDK is licensed under two licenses: - 1) GNU Lesser General Public License (GNU LGPL) - 2) Common Public License (CPL) - It means that you can select one of these two licenses and - follow rules of that license. - - SPECIAL EXCEPTION: - Igor Pavlov, as the author of this Code, expressly permits you to - statically or dynamically link your Code (or bind by name) to the - interfaces of this file without subjecting your linked Code to the - terms of the CPL or GNU LGPL. Any modifications or additions - to this file, however, are subject to the LGPL or CPL terms. -*/ - -#include "encoding/LzmaDecode.h" - -#ifndef Byte -#define Byte unsigned char -#endif - -#define kNumTopBits 24 -#define kTopValue ((UInt32)1 << kNumTopBits) - -#define kNumBitModelTotalBits 11 -#define kBitModelTotal (1 << kNumBitModelTotalBits) -#define kNumMoveBits 5 - -#define RC_READ_BYTE (*Buffer++) - -#define RC_INIT2 Code = 0; Range = 0xFFFFFFFF; \ - { int i; for(i = 0; i < 5; i++) { RC_TEST; Code = (Code << 8) | RC_READ_BYTE; }} - -#ifdef _LZMA_IN_CB - -#define RC_TEST { if (Buffer == BufferLim) \ - { SizeT size; int result = InCallback->Read(InCallback, &Buffer, &size); if (result != LZMA_RESULT_OK) return result; \ - BufferLim = Buffer + size; if (size == 0) return LZMA_RESULT_DATA_ERROR; }} - -#define RC_INIT Buffer = BufferLim = 0; RC_INIT2 - -#else - -#define RC_TEST { if (Buffer == BufferLim) return LZMA_RESULT_DATA_ERROR; } - -#define RC_INIT(buffer, bufferSize) Buffer = buffer; BufferLim = buffer + bufferSize; RC_INIT2 - -#endif - -#define RC_NORMALIZE if (Range < kTopValue) { RC_TEST; Range <<= 8; Code = (Code << 8) | RC_READ_BYTE; } - -#define IfBit0(p) RC_NORMALIZE; bound = (Range >> kNumBitModelTotalBits) * *(p); if (Code < bound) -#define UpdateBit0(p) Range = bound; *(p) += (kBitModelTotal - *(p)) >> kNumMoveBits; -#define UpdateBit1(p) Range -= bound; Code -= bound; *(p) -= (*(p)) >> kNumMoveBits; - -#define RC_GET_BIT2(p, mi, A0, A1) IfBit0(p) \ - { UpdateBit0(p); mi <<= 1; A0; } else \ - { UpdateBit1(p); mi = (mi + mi) + 1; A1; } - -#define RC_GET_BIT(p, mi) RC_GET_BIT2(p, mi, ; , ;) - -#define RangeDecoderBitTreeDecode(probs, numLevels, res) \ - { int i = numLevels; res = 1; \ - do { CProb *p = probs + res; RC_GET_BIT(p, res) } while(--i != 0); \ - res -= (1 << numLevels); } - - -#define kNumPosBitsMax 4 -#define kNumPosStatesMax (1 << kNumPosBitsMax) - -#define kLenNumLowBits 3 -#define kLenNumLowSymbols (1 << kLenNumLowBits) -#define kLenNumMidBits 3 -#define kLenNumMidSymbols (1 << kLenNumMidBits) -#define kLenNumHighBits 8 -#define kLenNumHighSymbols (1 << kLenNumHighBits) - -#define LenChoice 0 -#define LenChoice2 (LenChoice + 1) -#define LenLow (LenChoice2 + 1) -#define LenMid (LenLow + (kNumPosStatesMax << kLenNumLowBits)) -#define LenHigh (LenMid + (kNumPosStatesMax << kLenNumMidBits)) -#define kNumLenProbs (LenHigh + kLenNumHighSymbols) - - -#define kNumStates 12 -#define kNumLitStates 7 - -#define kStartPosModelIndex 4 -#define kEndPosModelIndex 14 -#define kNumFullDistances (1 << (kEndPosModelIndex >> 1)) - -#define kNumPosSlotBits 6 -#define kNumLenToPosStates 4 - -#define kNumAlignBits 4 -#define kAlignTableSize (1 << kNumAlignBits) - -#define kMatchMinLen 2 - -#define IsMatch 0 -#define IsRep (IsMatch + (kNumStates << kNumPosBitsMax)) -#define IsRepG0 (IsRep + kNumStates) -#define IsRepG1 (IsRepG0 + kNumStates) -#define IsRepG2 (IsRepG1 + kNumStates) -#define IsRep0Long (IsRepG2 + kNumStates) -#define PosSlot (IsRep0Long + (kNumStates << kNumPosBitsMax)) -#define SpecPos (PosSlot + (kNumLenToPosStates << kNumPosSlotBits)) -#define Align (SpecPos + kNumFullDistances - kEndPosModelIndex) -#define LenCoder (Align + kAlignTableSize) -#define RepLenCoder (LenCoder + kNumLenProbs) -#define Literal (RepLenCoder + kNumLenProbs) - -#if Literal != LZMA_BASE_SIZE -StopCompilingDueBUG -#endif - -int LzmaDecodeProperties(CLzmaProperties *propsRes, const unsigned char *propsData, int size) -{ - unsigned char prop0; - if (size < LZMA_PROPERTIES_SIZE) - return LZMA_RESULT_DATA_ERROR; - prop0 = propsData[0]; - if (prop0 >= (9 * 5 * 5)) - return LZMA_RESULT_DATA_ERROR; - { - for (propsRes->pb = 0; prop0 >= (9 * 5); propsRes->pb++, prop0 -= (9 * 5)); - for (propsRes->lp = 0; prop0 >= 9; propsRes->lp++, prop0 -= 9); - propsRes->lc = prop0; - /* - unsigned char remainder = (unsigned char)(prop0 / 9); - propsRes->lc = prop0 % 9; - propsRes->pb = remainder / 5; - propsRes->lp = remainder % 5; - */ - } - - #ifdef _LZMA_OUT_READ - { - int i; - propsRes->DictionarySize = 0; - for (i = 0; i < 4; i++) - propsRes->DictionarySize += (UInt32)(propsData[1 + i]) << (i * 8); - if (propsRes->DictionarySize == 0) - propsRes->DictionarySize = 1; - } - #endif - return LZMA_RESULT_OK; -} - -#define kLzmaStreamWasFinishedId (-1) - -int LzmaDecode(CLzmaDecoderState *vs, - #ifdef _LZMA_IN_CB - ILzmaInCallback *InCallback, - #else - const unsigned char *inStream, SizeT inSize, SizeT *inSizeProcessed, - #endif - unsigned char *outStream, SizeT outSize, SizeT *outSizeProcessed) -{ - CProb *p = vs->Probs; - SizeT nowPos = 0; - Byte previousByte = 0; - UInt32 posStateMask = (1 << (vs->Properties.pb)) - 1; - UInt32 literalPosMask = (1 << (vs->Properties.lp)) - 1; - int lc = vs->Properties.lc; - - #ifdef _LZMA_OUT_READ - - UInt32 Range = vs->Range; - UInt32 Code = vs->Code; - #ifdef _LZMA_IN_CB - const Byte *Buffer = vs->Buffer; - const Byte *BufferLim = vs->BufferLim; - #else - const Byte *Buffer = inStream; - const Byte *BufferLim = inStream + inSize; - #endif - int state = vs->State; - UInt32 rep0 = vs->Reps[0], rep1 = vs->Reps[1], rep2 = vs->Reps[2], rep3 = vs->Reps[3]; - int len = vs->RemainLen; - UInt32 globalPos = vs->GlobalPos; - UInt32 distanceLimit = vs->DistanceLimit; - - Byte *dictionary = vs->Dictionary; - UInt32 dictionarySize = vs->Properties.DictionarySize; - UInt32 dictionaryPos = vs->DictionaryPos; - - Byte tempDictionary[4]; - - #ifndef _LZMA_IN_CB - *inSizeProcessed = 0; - #endif - *outSizeProcessed = 0; - if (len == kLzmaStreamWasFinishedId) - return LZMA_RESULT_OK; - - if (dictionarySize == 0) - { - dictionary = tempDictionary; - dictionarySize = 1; - tempDictionary[0] = vs->TempDictionary[0]; - } - - if (len == kLzmaNeedInitId) - { - { - UInt32 numProbs = Literal + ((UInt32)LZMA_LIT_SIZE << (lc + vs->Properties.lp)); - UInt32 i; - for (i = 0; i < numProbs; i++) - p[i] = kBitModelTotal >> 1; - rep0 = rep1 = rep2 = rep3 = 1; - state = 0; - globalPos = 0; - distanceLimit = 0; - dictionaryPos = 0; - dictionary[dictionarySize - 1] = 0; - #ifdef _LZMA_IN_CB - RC_INIT; - #else - RC_INIT(inStream, inSize); - #endif - } - len = 0; - } - while(len != 0 && nowPos < outSize) - { - UInt32 pos = dictionaryPos - rep0; - if (pos >= dictionarySize) - pos += dictionarySize; - outStream[nowPos++] = dictionary[dictionaryPos] = dictionary[pos]; - if (++dictionaryPos == dictionarySize) - dictionaryPos = 0; - len--; - } - if (dictionaryPos == 0) - previousByte = dictionary[dictionarySize - 1]; - else - previousByte = dictionary[dictionaryPos - 1]; - - #else /* if !_LZMA_OUT_READ */ - - int state = 0; - UInt32 rep0 = 1, rep1 = 1, rep2 = 1, rep3 = 1; - int len = 0; - const Byte *Buffer; - const Byte *BufferLim; - UInt32 Range; - UInt32 Code; - - #ifndef _LZMA_IN_CB - *inSizeProcessed = 0; - #endif - *outSizeProcessed = 0; - - { - UInt32 i; - UInt32 numProbs = Literal + ((UInt32)LZMA_LIT_SIZE << (lc + vs->Properties.lp)); - for (i = 0; i < numProbs; i++) - p[i] = kBitModelTotal >> 1; - } - - #ifdef _LZMA_IN_CB - RC_INIT; - #else - RC_INIT(inStream, inSize); - #endif - - #endif /* _LZMA_OUT_READ */ - - while(nowPos < outSize) - { - CProb *prob; - UInt32 bound; - int posState = (int)( - (nowPos - #ifdef _LZMA_OUT_READ - + globalPos - #endif - ) - & posStateMask); - - prob = p + IsMatch + (state << kNumPosBitsMax) + posState; - IfBit0(prob) - { - int symbol = 1; - UpdateBit0(prob) - prob = p + Literal + (LZMA_LIT_SIZE * - ((( - (nowPos - #ifdef _LZMA_OUT_READ - + globalPos - #endif - ) - & literalPosMask) << lc) + (previousByte >> (8 - lc)))); - - if (state >= kNumLitStates) - { - int matchByte; - #ifdef _LZMA_OUT_READ - UInt32 pos = dictionaryPos - rep0; - if (pos >= dictionarySize) - pos += dictionarySize; - matchByte = dictionary[pos]; - #else - matchByte = outStream[nowPos - rep0]; - #endif - do - { - int bit; - CProb *probLit; - matchByte <<= 1; - bit = (matchByte & 0x100); - probLit = prob + 0x100 + bit + symbol; - RC_GET_BIT2(probLit, symbol, if (bit != 0) break, if (bit == 0) break) - } - while (symbol < 0x100); - } - while (symbol < 0x100) - { - CProb *probLit = prob + symbol; - RC_GET_BIT(probLit, symbol) - } - previousByte = (Byte)symbol; - - outStream[nowPos++] = previousByte; - #ifdef _LZMA_OUT_READ - if (distanceLimit < dictionarySize) - distanceLimit++; - - dictionary[dictionaryPos] = previousByte; - if (++dictionaryPos == dictionarySize) - dictionaryPos = 0; - #endif - if (state < 4) state = 0; - else if (state < 10) state -= 3; - else state -= 6; - } - else - { - UpdateBit1(prob); - prob = p + IsRep + state; - IfBit0(prob) - { - UpdateBit0(prob); - rep3 = rep2; - rep2 = rep1; - rep1 = rep0; - state = state < kNumLitStates ? 0 : 3; - prob = p + LenCoder; - } - else - { - UpdateBit1(prob); - prob = p + IsRepG0 + state; - IfBit0(prob) - { - UpdateBit0(prob); - prob = p + IsRep0Long + (state << kNumPosBitsMax) + posState; - IfBit0(prob) - { - #ifdef _LZMA_OUT_READ - UInt32 pos; - #endif - UpdateBit0(prob); - - #ifdef _LZMA_OUT_READ - if (distanceLimit == 0) - #else - if (nowPos == 0) - #endif - return LZMA_RESULT_DATA_ERROR; - - state = state < kNumLitStates ? 9 : 11; - #ifdef _LZMA_OUT_READ - pos = dictionaryPos - rep0; - if (pos >= dictionarySize) - pos += dictionarySize; - previousByte = dictionary[pos]; - dictionary[dictionaryPos] = previousByte; - if (++dictionaryPos == dictionarySize) - dictionaryPos = 0; - #else - previousByte = outStream[nowPos - rep0]; - #endif - outStream[nowPos++] = previousByte; - #ifdef _LZMA_OUT_READ - if (distanceLimit < dictionarySize) - distanceLimit++; - #endif - - continue; - } - else - { - UpdateBit1(prob); - } - } - else - { - UInt32 distance; - UpdateBit1(prob); - prob = p + IsRepG1 + state; - IfBit0(prob) - { - UpdateBit0(prob); - distance = rep1; - } - else - { - UpdateBit1(prob); - prob = p + IsRepG2 + state; - IfBit0(prob) - { - UpdateBit0(prob); - distance = rep2; - } - else - { - UpdateBit1(prob); - distance = rep3; - rep3 = rep2; - } - rep2 = rep1; - } - rep1 = rep0; - rep0 = distance; - } - state = state < kNumLitStates ? 8 : 11; - prob = p + RepLenCoder; - } - { - int numBits, offset; - CProb *probLen = prob + LenChoice; - IfBit0(probLen) - { - UpdateBit0(probLen); - probLen = prob + LenLow + (posState << kLenNumLowBits); - offset = 0; - numBits = kLenNumLowBits; - } - else - { - UpdateBit1(probLen); - probLen = prob + LenChoice2; - IfBit0(probLen) - { - UpdateBit0(probLen); - probLen = prob + LenMid + (posState << kLenNumMidBits); - offset = kLenNumLowSymbols; - numBits = kLenNumMidBits; - } - else - { - UpdateBit1(probLen); - probLen = prob + LenHigh; - offset = kLenNumLowSymbols + kLenNumMidSymbols; - numBits = kLenNumHighBits; - } - } - RangeDecoderBitTreeDecode(probLen, numBits, len); - len += offset; - } - - if (state < 4) - { - int posSlot; - state += kNumLitStates; - prob = p + PosSlot + - ((len < kNumLenToPosStates ? len : kNumLenToPosStates - 1) << - kNumPosSlotBits); - RangeDecoderBitTreeDecode(prob, kNumPosSlotBits, posSlot); - if (posSlot >= kStartPosModelIndex) - { - int numDirectBits = ((posSlot >> 1) - 1); - rep0 = (2 | ((UInt32)posSlot & 1)); - if (posSlot < kEndPosModelIndex) - { - rep0 <<= numDirectBits; - prob = p + SpecPos + rep0 - posSlot - 1; - } - else - { - numDirectBits -= kNumAlignBits; - do - { - RC_NORMALIZE - Range >>= 1; - rep0 <<= 1; - if (Code >= Range) - { - Code -= Range; - rep0 |= 1; - } - } - while (--numDirectBits != 0); - prob = p + Align; - rep0 <<= kNumAlignBits; - numDirectBits = kNumAlignBits; - } - { - int i = 1; - int mi = 1; - do - { - CProb *prob3 = prob + mi; - RC_GET_BIT2(prob3, mi, ; , rep0 |= i); - i <<= 1; - } - while(--numDirectBits != 0); - } - } - else - rep0 = posSlot; - if (++rep0 == (UInt32)(0)) - { - /* it's for stream version */ - len = kLzmaStreamWasFinishedId; - break; - } - } - - len += kMatchMinLen; - #ifdef _LZMA_OUT_READ - if (rep0 > distanceLimit) - #else - if (rep0 > nowPos) - #endif - return LZMA_RESULT_DATA_ERROR; - - #ifdef _LZMA_OUT_READ - if (dictionarySize - distanceLimit > (UInt32)len) - distanceLimit += len; - else - distanceLimit = dictionarySize; - #endif - - do - { - #ifdef _LZMA_OUT_READ - UInt32 pos = dictionaryPos - rep0; - if (pos >= dictionarySize) - pos += dictionarySize; - previousByte = dictionary[pos]; - dictionary[dictionaryPos] = previousByte; - if (++dictionaryPos == dictionarySize) - dictionaryPos = 0; - #else - previousByte = outStream[nowPos - rep0]; - #endif - len--; - outStream[nowPos++] = previousByte; - } - while(len != 0 && nowPos < outSize); - } - } - RC_NORMALIZE; - - #ifdef _LZMA_OUT_READ - vs->Range = Range; - vs->Code = Code; - vs->DictionaryPos = dictionaryPos; - vs->GlobalPos = globalPos + (UInt32)nowPos; - vs->DistanceLimit = distanceLimit; - vs->Reps[0] = rep0; - vs->Reps[1] = rep1; - vs->Reps[2] = rep2; - vs->Reps[3] = rep3; - vs->State = state; - vs->RemainLen = len; - vs->TempDictionary[0] = tempDictionary[0]; - #endif - - #ifdef _LZMA_IN_CB - vs->Buffer = Buffer; - vs->BufferLim = BufferLim; - #else - *inSizeProcessed = (SizeT)(Buffer - inStream); - #endif - *outSizeProcessed = nowPos; - return LZMA_RESULT_OK; -} diff --git a/src/encoding/LzmaDecode.h b/src/encoding/LzmaDecode.h deleted file mode 100644 index cfbfeb59..00000000 --- a/src/encoding/LzmaDecode.h +++ /dev/null @@ -1,131 +0,0 @@ -/* - LzmaDecode.h - LZMA Decoder interface - - LZMA SDK 4.21 Copyright (c) 1999-2005 Igor Pavlov (2005-06-08) - http://www.7-zip.org/ - - LZMA SDK is licensed under two licenses: - 1) GNU Lesser General Public License (GNU LGPL) - 2) Common Public License (CPL) - It means that you can select one of these two licenses and - follow rules of that license. - - SPECIAL EXCEPTION: - Igor Pavlov, as the author of this code, expressly permits you to - statically or dynamically link your code (or bind by name) to the - interfaces of this file without subjecting your linked code to the - terms of the CPL or GNU LGPL. Any modifications or additions - to this file, however, are subject to the LGPL or CPL terms. -*/ - -#ifndef __LZMADECODE_H -#define __LZMADECODE_H - -/* #define _LZMA_IN_CB */ -/* Use callback for input data */ - -/* #define _LZMA_OUT_READ */ -/* Use read function for output data */ - -/* #define _LZMA_PROB32 */ -/* It can increase speed on some 32-bit CPUs, - but memory usage will be doubled in that case */ - -#define _LZMA_LOC_OPT 1 -/* Enable local speed optimizations inside code */ - -/* #define _LZMA_SYSTEM_SIZE_T */ -/* Use system's size_t. You can use it to enable 64-bit sizes supporting*/ - -#ifndef UInt32 -#ifdef _LZMA_UINT32_IS_ULONG -#define UInt32 unsigned long -#else -#define UInt32 unsigned int -#endif -#endif - -#ifndef SizeT -#ifdef _LZMA_SYSTEM_SIZE_T -#include -#define SizeT size_t -#else -#define SizeT UInt32 -#endif -#endif - -#ifdef _LZMA_PROB32 -#define CProb UInt32 -#else -#define CProb unsigned short -#endif - -#define LZMA_RESULT_OK 0 -#define LZMA_RESULT_DATA_ERROR 1 - -#ifdef _LZMA_IN_CB -typedef struct _ILzmaInCallback -{ - int (*Read)(void *object, const unsigned char **buffer, SizeT *bufferSize); -} ILzmaInCallback; -#endif - -#define LZMA_BASE_SIZE 1846 -#define LZMA_LIT_SIZE 768 - -#define LZMA_PROPERTIES_SIZE 5 - -typedef struct _CLzmaProperties -{ - int lc; - int lp; - int pb; - #ifdef _LZMA_OUT_READ - UInt32 DictionarySize; - #endif -}CLzmaProperties; - -int LzmaDecodeProperties(CLzmaProperties *propsRes, const unsigned char *propsData, int size); - -#define LzmaGetNumProbs(Properties) (LZMA_BASE_SIZE + (LZMA_LIT_SIZE << ((Properties)->lc + (Properties)->lp))) - -#define kLzmaNeedInitId (-2) - -typedef struct _CLzmaDecoderState -{ - CLzmaProperties Properties; - CProb *Probs; - - #ifdef _LZMA_IN_CB - const unsigned char *Buffer; - const unsigned char *BufferLim; - #endif - - #ifdef _LZMA_OUT_READ - unsigned char *Dictionary; - UInt32 Range; - UInt32 Code; - UInt32 DictionaryPos; - UInt32 GlobalPos; - UInt32 DistanceLimit; - UInt32 Reps[4]; - int State; - int RemainLen; - unsigned char TempDictionary[4]; - #endif -} CLzmaDecoderState; - -#ifdef _LZMA_OUT_READ -#define LzmaDecoderInit(vs) { (vs)->RemainLen = kLzmaNeedInitId; } -#endif - -int LzmaDecode(CLzmaDecoderState *vs, - #ifdef _LZMA_IN_CB - ILzmaInCallback *inCallback, - #else - const unsigned char *inStream, SizeT inSize, SizeT *inSizeProcessed, - #endif - unsigned char *outStream, SizeT outSize, SizeT *outSizeProcessed); - -#endif diff --git a/src/encoding/Makefile b/src/encoding/Makefile index 938497ba..61ed5e64 100644 --- a/src/encoding/Makefile +++ b/src/encoding/Makefile @@ -3,7 +3,7 @@ include $(top_builddir)/Makefile.config OBJS-$(CONFIG_BZIP2) += bzip2.o OBJS-$(CONFIG_GZIP) += deflate.o -OBJS-$(CONFIG_LZMA) += lzma.o LzmaDecode.o +OBJS-$(CONFIG_LZMA) += lzma.o OBJS = encoding.o diff --git a/src/encoding/lzma.c b/src/encoding/lzma.c index f4d13459..e2e2f76f 100644 --- a/src/encoding/lzma.c +++ b/src/encoding/lzma.c @@ -11,101 +11,45 @@ #include #endif +#include +#include + #include "elinks.h" -#include "encoding/LzmaDecode.h" #include "encoding/encoding.h" #include "encoding/lzma.h" #include "util/memory.h" -#define LZMAMAXOUTPUT 2097152 +#define ELINKS_BZ_BUFFER_LENGTH 5000 struct lzma_enc_data { - unsigned char *output; - off_t current; - off_t outSize; + lzma_stream flzma_stream; + int fdread; + int last_read; + unsigned char buf[ELINKS_BZ_BUFFER_LENGTH]; }; -static void -lzma_cleanup(struct lzma_enc_data *data) -{ - mem_free_if(data->output); - mem_free(data); -} - static int lzma_open(struct stream_encoded *stream, int fd) { - CLzmaDecoderState state; - struct stat buf; - struct lzma_enc_data *data; - ssize_t nb; - size_t inSize, inProcessed, outProcessed; - int res; - unsigned char *input, *inData; - unsigned int i; + struct lzma_enc_data *data = mem_alloc(sizeof(*data)); + int err; - if (fstat(fd, &buf)) return -1; - if (!S_ISREG(buf.st_mode)) return -1; - if (buf.st_size < LZMA_PROPERTIES_SIZE + 8) return -1; - data = mem_calloc(1, sizeof(*data)); - if (!data) return -1; - input = mem_alloc(buf.st_size); - if (!input) { + stream->data = NULL; + if (!data) { + return -1; + } + + copy_struct(&data->flzma_stream, &LZMA_STREAM_INIT_VAR); + data->fdread = fd; + data->last_read = 0; + + err = lzma_auto_decoder(&data->flzma_stream, NULL, NULL); + if (err != LZMA_OK) { mem_free(data); return -1; } - nb = safe_read(fd, input, buf.st_size); - close(fd); - if (nb != buf.st_size) { - mem_free(input); - lzma_cleanup(data); - return -1; - } - if (LzmaDecodeProperties(&state.Properties, input, - LZMA_PROPERTIES_SIZE) != LZMA_RESULT_OK) { - mem_free(input); - lzma_cleanup(data); - return -1; - } - state.Probs = (CProb *)mem_alloc(LzmaGetNumProbs( - &state.Properties) * sizeof(CProb)); - if (!state.Probs) { - mem_free(input); - lzma_cleanup(data); - return -1; - } - inSize = buf.st_size - LZMA_PROPERTIES_SIZE - 8; - inData = input + LZMA_PROPERTIES_SIZE; - data->outSize = 0; - /* The size is 8 bytes long, but who wants such big files */ - for (i = 0; i < 4; i++) { - unsigned char b = inData[i]; - data->outSize += (unsigned int)(b) << (i * 8); - } - if (data->outSize == 0xffffffff) data->outSize = LZMAMAXOUTPUT; - - data->output = mem_alloc(data->outSize); - if (!data->output) { - mem_free(state.Probs); - mem_free(input); - lzma_cleanup(data); - return -1; - } - inData += 8; - res = LzmaDecode(&state, inData, inSize, &inProcessed, - data->output, data->outSize, &outProcessed); - if (res) { - mem_free(state.Probs); - mem_free(input); - lzma_cleanup(data); - return -1; - } - data->outSize = outProcessed; - data->current = 0; - mem_free(input); - mem_free(state.Probs); stream->data = data; return 0; @@ -115,63 +59,93 @@ static int lzma_read(struct stream_encoded *stream, unsigned char *buf, int len) { struct lzma_enc_data *data = (struct lzma_enc_data *) stream->data; + int err = 0; - if (data->current + len > data->outSize) - len = data->outSize - data->current; + if (!data) return -1; - if (len < 0) return -1; - memcpy(buf, data->output + data->current, len); - data->current += len; + assert(len > 0); - return len; + if (data->last_read) return 0; + + data->flzma_stream.avail_out = len; + data->flzma_stream.next_out = buf; + + do { + if (data->flzma_stream.avail_in == 0) { + int l = safe_read(data->fdread, data->buf, + ELINKS_BZ_BUFFER_LENGTH); + + if (l == -1) { + if (errno == EAGAIN) + break; + else + return -1; /* I/O error */ + } else if (l == 0) { + /* EOF. It is error: we wait for more bytes */ + return -1; + } + + data->flzma_stream.next_in = data->buf; + data->flzma_stream.avail_in = l; + } + + err = lzma_code(&data->flzma_stream, LZMA_RUN); + if (err == LZMA_STREAM_END) { + data->last_read = 1; + break; + } else if (err != LZMA_OK) { + return -1; + } + } while (data->flzma_stream.avail_out > 0); + + assert(len - data->flzma_stream.avail_out == data->flzma_stream.next_out - buf); + return len - data->flzma_stream.avail_out; } static unsigned char * lzma_decode_buffer(unsigned char *data, int len, int *new_len) { - CLzmaDecoderState state; - size_t inSize, inProcessed, outProcessed; - int res, outSize; - unsigned char *inData; - unsigned char *output; - unsigned int i; + lzma_stream stream = LZMA_STREAM_INIT; + unsigned char *buffer = NULL; + int error; - if (len < LZMA_PROPERTIES_SIZE + 8) return NULL; - if (LzmaDecodeProperties(&state.Properties, data, - LZMA_PROPERTIES_SIZE) != LZMA_RESULT_OK) { - return NULL; - } - state.Probs = (CProb *)mem_alloc(LzmaGetNumProbs( - &state.Properties) * sizeof(CProb)); - if (!state.Probs) { + stream.next_in = data; + stream.avail_in = len; + + if (lzma_auto_decoder(&stream, NULL, NULL) != LZMA_OK) + return NULL; + + do { + unsigned char *new_buffer; + size_t size = stream.total_out + MAX_STR_LEN; + + new_buffer = mem_realloc(buffer, size); + if (!new_buffer) { + error = LZMA_MEM_ERROR; + break; + } + + buffer = new_buffer; + stream.next_out = buffer + stream.total_out; + stream.avail_out = MAX_STR_LEN; + + error = lzma_code(&stream, LZMA_RUN); + if (error == LZMA_STREAM_END) { + *new_len = stream.total_out; + error = LZMA_OK; + break; + } + } while (error == LZMA_OK && stream.avail_in > 0); + + lzma_end(&stream); + + if (error != LZMA_OK) { + if (buffer) mem_free(buffer); + *new_len = 0; return NULL; } - inSize = len - LZMA_PROPERTIES_SIZE - 8; - inData = data + LZMA_PROPERTIES_SIZE; - outSize = 0; - for (i = 0; i < 4; i++) { - unsigned char b = inData[i]; - outSize += (unsigned int)(b) << (i * 8); - } - if (outSize == 0xffffffff) outSize = LZMAMAXOUTPUT; - - output = mem_alloc(outSize); - if (!output) { - mem_free(state.Probs); - return NULL; - } - inData += 8; - res = LzmaDecode(&state, inData, inSize, &inProcessed, output, - outSize, &outProcessed); - if (res) { - mem_free(state.Probs); - mem_free(output); - return NULL; - } - *new_len = outProcessed; - - return output; + return buffer; } static void @@ -179,7 +153,12 @@ lzma_close(struct stream_encoded *stream) { struct lzma_enc_data *data = (struct lzma_enc_data *) stream->data; - lzma_cleanup(data); + if (data) { + lzma_end(&data->flzma_stream); + close(data->fdread); + mem_free(data); + stream->data = 0; + } } static const unsigned char *const lzma_extensions[] = { ".lzma", NULL }; diff --git a/src/protocol/http/http.c b/src/protocol/http/http.c index 69ce6709..82b08818 100644 --- a/src/protocol/http/http.c +++ b/src/protocol/http/http.c @@ -1857,6 +1857,11 @@ again: conn->content_encoding = ENCODING_BZIP2; #endif +#ifdef CONFIG_LZMA + if (file_encoding != ENCODING_LZMA + && (!strcasecmp(d, "lzma") || !strcasecmp(d, "x-lzma"))) + conn->content_encoding = ENCODING_LZMA; +#endif mem_free(d); } From 902522fbdf1ec0f2d526a2ad50b34d420fad26a1 Mon Sep 17 00:00:00 2001 From: Witold Filipczyk Date: Mon, 18 Feb 2008 14:30:49 +0100 Subject: [PATCH 25/31] content encoding: New function accept_encoding_header. --- src/encoding/encoding.c | 27 +++++++++++++++++++++++++++ src/encoding/encoding.h | 2 ++ src/protocol/http/http.c | 20 ++------------------ 3 files changed, 31 insertions(+), 18 deletions(-) diff --git a/src/encoding/encoding.c b/src/encoding/encoding.c index 4d8c2d35..99f75cc6 100644 --- a/src/encoding/encoding.c +++ b/src/encoding/encoding.c @@ -344,3 +344,30 @@ read_encoded_file(struct string *filename, struct string *page) close(fd); return state; } + +void +accept_encoding_header(struct string *header) +{ +#if defined(CONFIG_GZIP) || defined(CONFIG_BZIP2) || defined(CONFIG_LZMA) + int comma = 0; + + add_to_string(header, "Accept-Encoding: "); + +#ifdef CONFIG_BZIP2 + add_to_string(header, "bzip2"); + comma = 1; +#endif + +#ifdef CONFIG_GZIP + if (comma) add_to_string(header, ", "); + add_to_string(header, "deflate, gzip"); + comma = 1; +#endif + +#ifdef CONFIG_LZMA + if (comma) add_to_string(header, ", "); + add_to_string(header, "lzma"); +#endif + add_crlf_to_string(header); +#endif +} diff --git a/src/encoding/encoding.h b/src/encoding/encoding.h index 00b4ab2c..26688e56 100644 --- a/src/encoding/encoding.h +++ b/src/encoding/encoding.h @@ -45,4 +45,6 @@ read_file(struct stream_encoded *stream, int readsize, struct string *page); /* Reads the file with the given @filename into the string @source. */ enum connection_state read_encoded_file(struct string *filename, struct string *source); +void accept_encoding_header(struct string *header); + #endif diff --git a/src/protocol/http/http.c b/src/protocol/http/http.c index 82b08818..fc94c9e6 100644 --- a/src/protocol/http/http.c +++ b/src/protocol/http/http.c @@ -23,6 +23,7 @@ #include "cache/cache.h" #include "config/options.h" #include "cookies/cookies.h" +#include "encoding/encoding.h" #include "intl/charsets.h" #include "intl/gettext/libintl.h" #include "main/module.h" @@ -745,24 +746,7 @@ http_send_header(struct socket *socket) add_to_string(&header, "Accept: */*"); add_crlf_to_string(&header); - /* TODO: Make this encoding.c function. */ -#if defined(CONFIG_GZIP) || defined(CONFIG_BZIP2) - add_to_string(&header, "Accept-Encoding: "); - -#ifdef CONFIG_BZIP2 - add_to_string(&header, "bzip2"); -#endif - -#ifdef CONFIG_GZIP - -#ifdef CONFIG_BZIP2 - add_to_string(&header, ", "); -#endif - - add_to_string(&header, "deflate, gzip"); -#endif - add_crlf_to_string(&header); -#endif + accept_encoding_header(&header); if (!accept_charset) { init_accept_charset(); From 26607d22631d4e06bd6b3dccb60ee9ee4d0ef78e Mon Sep 17 00:00:00 2001 From: Witold Filipczyk Date: Mon, 18 Feb 2008 17:43:34 +0100 Subject: [PATCH 26/31] decompress_data: Do not leak memory in mem_realloc. --- src/protocol/http/http.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/protocol/http/http.c b/src/protocol/http/http.c index fc94c9e6..588b01b6 100644 --- a/src/protocol/http/http.c +++ b/src/protocol/http/http.c @@ -1014,6 +1014,8 @@ decompress_data(struct connection *conn, unsigned char *data, int len, } do { + unsigned char *tmp; + if (state == NORMAL) { /* ... we aren't finishing yet. */ int written = safe_write(conn->stream_pipes[1], data, len); @@ -1050,8 +1052,9 @@ decompress_data(struct connection *conn, unsigned char *data, int len, if (!conn->stream) return NULL; } - output = (unsigned char *) mem_realloc(output, *new_len + BIG_READ); - if (!output) break; + tmp = mem_realloc(output, *new_len + BIG_READ); + if (!tmp) break; + output = tmp; did_read = read_encoded(conn->stream, output + *new_len, BIG_READ); From a03698d695882477fc77369835bf7e7065699d54 Mon Sep 17 00:00:00 2001 From: Kalle Olavi Niemitalo Date: Sun, 24 Feb 2008 19:19:58 +0200 Subject: [PATCH 27/31] lzma: Set *new_len even if data was truncated. Commit 474f1f4268d24ba7782ab57ae47c87113ec2a8f5 did the same to bzip2 and deflate. --- src/encoding/lzma.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/encoding/lzma.c b/src/encoding/lzma.c index e2e2f76f..0cbbaf32 100644 --- a/src/encoding/lzma.c +++ b/src/encoding/lzma.c @@ -109,6 +109,8 @@ lzma_decode_buffer(unsigned char *data, int len, int *new_len) unsigned char *buffer = NULL; int error; + *new_len = 0; /* default, left there if an error occurs */ + stream.next_in = data; stream.avail_in = len; @@ -131,7 +133,6 @@ lzma_decode_buffer(unsigned char *data, int len, int *new_len) error = lzma_code(&stream, LZMA_RUN); if (error == LZMA_STREAM_END) { - *new_len = stream.total_out; error = LZMA_OK; break; } @@ -139,13 +140,13 @@ lzma_decode_buffer(unsigned char *data, int len, int *new_len) lzma_end(&stream); - if (error != LZMA_OK) { + if (error == LZMA_OK) { + *new_len = stream.total_out; + return buffer; + } else { if (buffer) mem_free(buffer); - *new_len = 0; return NULL; } - - return buffer; } static void From 79b69e0bf96c5cfc289025c8bc9c27c2a297294b Mon Sep 17 00:00:00 2001 From: Kalle Olavi Niemitalo Date: Sun, 24 Feb 2008 20:32:43 +0200 Subject: [PATCH 28/31] encoding: documentation --- NEWS | 2 +- doc/installation.txt | 8 +++++--- features.conf | 43 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 4 deletions(-) diff --git a/NEWS b/NEWS index bde3aee0..d92a0c29 100644 --- a/NEWS +++ b/NEWS @@ -64,7 +64,7 @@ Miscellaneous: when downloading * tabs opened by -remote now go behind existing dialogs * major bug 534, enhancement 517: fix HTTP gzip and bzip2 - decompression, and add deflate and LZMA + decompression, and add deflate and LZMA (requires LZMA Utils) * major bug 503: various fixes in parsing and updating of elinks.conf * Debian bug 257762: turn terminal transparency off by default * bug 724: better parsing of escape sequences and control diff --git a/doc/installation.txt b/doc/installation.txt index 8c9d7073..25ef0010 100644 --- a/doc/installation.txt +++ b/doc/installation.txt @@ -42,9 +42,11 @@ ifndef::installation-webpage[] See <> \ endif::installation-webpage[] for more info. -zlib 1.2.0.2 or later For handling compressed documents both locally and \ - sent from server. -bzip2 For handling local compressed documents. +zlib 1.2.0.2 or later For handling gzip or deflate compressed documents \ + both locally and sent from server. +bzip2 Likewise, for bzip2 compressed documents. +LZMA Utils Likewise, for LZMA compressed documents. \ + Version 4.32.5 should work. OpenSSL or GNU TLS For handling secure HTTP browsing. GPM 'General Purpose Mouse' for mouse support. expat 'XML Parser Toolkit' needed for XBEL support. diff --git a/features.conf b/features.conf index ad253c54..e859bb1a 100644 --- a/features.conf +++ b/features.conf @@ -175,6 +175,49 @@ CONFIG_MAILCAP=yes CONFIG_MIMETYPES=yes +### Gzip and Deflate Decompression +# +# This makes ELinks send "Accept-Encoding: deflate, gzip" in HTTP +# requests and decompress any documents received in those formats. +# It works with local *.gz files as well. +# +# Default: enabled if zlib is installed and new enough + +CONFIG_GZIP=yes + + +### Bzip2 Decompression +# +# This makes ELinks decompress local *.bz2 files. +# Also, ELinks sends "Accept-Encoding: bzip2" in HTTP requests +# and decompresses any documents received in that format, +# but this encoding has not been registered at +# http://www.iana.org/assignments/http-parameters[], +# so most servers probably won't use it. +# +# Default: enabled if the library is installed + +CONFIG_BZIP2=yes + + +### LZMA Decompression +# +# This makes ELinks decompress local *.lzma files. +# Also, ELinks sends "Accept-Encoding: lzma" in HTTP requests +# and decompresses any documents received in that format, +# but this encoding has not been registered at +# http://www.iana.org/assignments/http-parameters[], +# so most servers probably won't use it. +# +# To use this, first install http://tukaani.org/lzma/[LZMA Utils]. +# Version 4.32.5 works; 4.42.2alpha also works and understands a +# newer LZMA file format. ELinks cannot use LZMA SDK from 7-Zip. +# +# Default: enabled if the library is installed + +CONFIG_LZMA=yes + + ### IPv6 Protocol Support # # You know this thing that was designed to obsolete IPv4 but only pasky, From 31255d4a4d21b0c4c6d87cfa9badac0d75f97287 Mon Sep 17 00:00:00 2001 From: Kalle Olavi Niemitalo Date: Sun, 24 Feb 2008 22:13:00 +0200 Subject: [PATCH 29/31] grafthistory: git-* commands are deprecated The fully dashed form git-update-server-info is deprecated in Git 1.5.4 and will cease to work in Git 1.6.0 (unless PATH is modified). Use git update-server-info instead. --- contrib/grafthistory.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/grafthistory.sh b/contrib/grafthistory.sh index b725c244..821e5eba 100755 --- a/contrib/grafthistory.sh +++ b/contrib/grafthistory.sh @@ -32,4 +32,4 @@ echo 43a9a406737fd22a8558c47c74b4ad04d4c92a2b 730242dcf2cdeed13eae7e8b0c5f47bb03 echo "[grafthistory] Refreshing the dumb server info wrt. new packs" cd .. -git-update-server-info +git update-server-info From bc42af691be3a8698b868da9dccc059e09794d7a Mon Sep 17 00:00:00 2001 From: Kalle Olavi Niemitalo Date: Sun, 24 Feb 2008 22:23:39 +0200 Subject: [PATCH 30/31] grafthistory: exit if a command fails --- contrib/grafthistory.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/grafthistory.sh b/contrib/grafthistory.sh index 821e5eba..19504ab9 100755 --- a/contrib/grafthistory.sh +++ b/contrib/grafthistory.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # Graft the ELinks development history to the current tree. # From 6139c2ffb5127a75aec176167e22298c693d339a Mon Sep 17 00:00:00 2001 From: Kalle Olavi Niemitalo Date: Sun, 24 Feb 2008 22:24:33 +0200 Subject: [PATCH 31/31] grafthistory: comment about downloading the pack index --- contrib/grafthistory.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/contrib/grafthistory.sh b/contrib/grafthistory.sh index 19504ab9..447a6388 100755 --- a/contrib/grafthistory.sh +++ b/contrib/grafthistory.sh @@ -19,6 +19,9 @@ cd "$GIT_DIR" echo "[grafthistory] Downloading the history" mkdir -p objects/pack cd objects/pack +# pack-0d6c5c67aab3b9d5d9b245da5929c15d79124a48.idx is 3163784 bytes long. +# Downloading it takes less than 6 seconds here, whereas generating it +# with git index-pack takes over 4 minutes (750 MHz Duron, git 1.5.4.1). wget -c http://elinks.cz/elinks-history.git/objects/pack/pack-0d6c5c67aab3b9d5d9b245da5929c15d79124a48.idx wget -c http://elinks.cz/elinks-history.git/objects/pack/pack-0d6c5c67aab3b9d5d9b245da5929c15d79124a48.pack