Removed code using pipes for decompression and simplified decompress_data.

Workarounds for sites, which send incorrect data, probably won't work.

src/document/renderer.c

@@ -229,14 +229,21 @@ render_encoded_document(struct cache_entry *cached, struct document *document)
 		if (encoding != ENCODING_NONE) {
 			int length = 0;
 			unsigned char *source;
+			struct stream_encoded *stream = open_encoded(-1, encoding);
 
-			source = decode_encoded_buffer(encoding, buffer.source,
+			if (!stream) {
-					       buffer.length, &length);
-			if (source) {
-				buffer.source = source;
-				buffer.length = length;
-			} else {
 				encoding = ENCODING_NONE;
+			} else {
+				source = decode_encoded_buffer(stream, encoding, buffer.source,
+					       buffer.length, &length);
+				close_encoded(stream);
+
+				if (source) {
+					buffer.source = source;
+					buffer.length = length;
+				} else {
+					encoding = ENCODING_NONE;
+				}
 			}
 		}
 	}

src/encoding/bzip2.c

@@ -36,7 +36,8 @@ struct bz2_enc_data {
 	 * end-of-stream marker and all data has been decompressed.
 	 * Then we neither read from the file nor call BZ2_bzDecompress
 	 * any more.  */
-	int last_read;
+	int last_read:1;
+	int after_end:1;
 
 	/* A buffer for data that has been read from the file but not
 	 * yet decompressed.  fbz_stream.next_in and fbz_stream.avail_in
@@ -132,31 +133,30 @@ bzip2_read(struct stream_encoded *stream, unsigned char *buf, int len)
 #endif
 
 static unsigned char *
-bzip2_decode_buffer(unsigned char *data, int len, int *new_len)
+bzip2_decode_buffer(struct stream_encoded *st, unsigned char *data, int len, int *new_len)
 {
-	bz_stream stream;
+	struct bz2_enc_data *enc_data = (struct bz2_enc_data *)st->data;
+	bz_stream *stream = &enc_data->fbz_stream;
 	unsigned char *buffer = NULL;
 	int error;
 
 	*new_len = 0;	  /* default, left there if an error occurs */
 
-	memset(&stream, 0, sizeof(bz_stream));
+	stream->next_in = data;
-	stream.next_in = data;
+	stream->avail_in = len;
-	stream.avail_in = len;
+	stream->total_out_lo32 = 0;
-
+	stream->total_out_hi32 = 0;
-	if (BZ2_bzDecompressInit(&stream, 0, BZIP2_SMALL) != BZ_OK)
-		return NULL;
 
 	do {
 		unsigned char *new_buffer;
-		size_t size = stream.total_out_lo32 + MAX_STR_LEN;
+		size_t size = stream->total_out_lo32 + MAX_STR_LEN;
 
 		/* FIXME: support for 64 bit.  real size is
 		 *
 		 * 	(total_in_hi32 << * 32) + total_in_lo32
 		 *
 		 * --jonas */
-		assertm(!stream.total_out_hi32, "64 bzip2 decoding not supported");
+		assertm(!stream->total_out_hi32, "64 bzip2 decoding not supported");
 
 		new_buffer = mem_realloc(buffer, size);
 		if (!new_buffer) {
@@ -165,12 +165,11 @@ bzip2_decode_buffer(unsigned char *data, int len, int *new_len)
 		}
 
 		buffer		 = new_buffer;
-		stream.next_out  = buffer + stream.total_out_lo32;
+		stream->next_out  = buffer + stream->total_out_lo32;
-		stream.avail_out = MAX_STR_LEN;
+		stream->avail_out = MAX_STR_LEN;
 
-		error = BZ2_bzDecompress(&stream);
+		error = BZ2_bzDecompress(stream);
 		if (error == BZ_STREAM_END) {
-			error = BZ_OK;
 			break;
 		}
 
@@ -178,12 +177,16 @@ bzip2_decode_buffer(unsigned char *data, int len, int *new_len)
 		 * is reached. At least lindi- reported that it caused a
 		 * reproducable infinite loop. Maybe it has to do with decoding
 		 * an incomplete file. */
-	} while (error == BZ_OK && stream.avail_in > 0);
+	} while (error == BZ_OK && stream->avail_in > 0);
 
-	BZ2_bzDecompressEnd(&stream);
+	if (error == BZ_STREAM_END) {
+		BZ2_bzDecompressEnd(stream);
+		enc_data->after_end = 1;
+		error = BZ_OK;
+	}
 
 	if (error == BZ_OK) {
-		*new_len = stream.total_out_lo32;
+		*new_len = stream->total_out_lo32;
 		return buffer;
 	} else {
 		if (buffer) mem_free(buffer);
@@ -197,8 +200,12 @@ bzip2_close(struct stream_encoded *stream)
 	struct bz2_enc_data *data = (struct bz2_enc_data *) stream->data;
 
 	if (data) {
-		BZ2_bzDecompressEnd(&data->fbz_stream);
+		if (!data->after_end) {
-		close(data->fdread);
+			BZ2_bzDecompressEnd(&data->fbz_stream);
+		}
+		if (data->fdread != -1) {
+			close(data->fdread);
+		}
 		mem_free(data);
 		stream->data = 0;
 	}

src/encoding/deflate.c

@@ -31,6 +31,7 @@ struct deflate_enc_data {
 
 	unsigned int last_read:1;
 	unsigned int after_first_read:1;
+	unsigned int after_end:1;
 
 	/* A buffer for data that has been read from the file but not
 	 * yet decompressed.  z_stream.next_in and z_stream.avail_in
@@ -170,25 +171,23 @@ restart:
 }
 
 static unsigned char *
-deflate_decode_buffer(int window_size, unsigned char *data, int len, int *new_len)
+deflate_decode_buffer(struct stream_encoded *st, int window_size, unsigned char *data, int len, int *new_len)
 {
-	z_stream stream;
+	struct deflate_enc_data *enc_data = (struct deflate_enc_data *) st->data;
+	z_stream *stream = &enc_data->deflate_stream;
 	unsigned char *buffer = NULL;
 	int error;
 
 	*new_len = 0;	  /* default, left there if an error occurs */
 
 	if (!len) return NULL;
-	memset(&stream, 0, sizeof(z_stream));
+	stream->next_in = data;
-	stream.next_in = data;
+	stream->avail_in = len;
-	stream.avail_in = len;
+	stream->total_out = 0;
-
-	if (inflateInit2(&stream, window_size) != Z_OK)
-		return NULL;
 
 	do {
 		unsigned char *new_buffer;
-		size_t size = stream.total_out + MAX_STR_LEN;
+		size_t size = stream->total_out + MAX_STR_LEN;
 
 		new_buffer = mem_realloc(buffer, size);
 		if (!new_buffer) {
@@ -197,20 +196,23 @@ deflate_decode_buffer(int window_size, unsigned char *data, int len, int *new_le
 		}
 
 		buffer		 = new_buffer;
-		stream.next_out  = buffer + stream.total_out;
+		stream->next_out  = buffer + stream->total_out;
-		stream.avail_out = MAX_STR_LEN;
+		stream->avail_out = MAX_STR_LEN;
 
-		error = inflate(&stream, Z_SYNC_FLUSH);
+		error = inflate(stream, Z_SYNC_FLUSH);
 		if (error == Z_STREAM_END) {
-			error = Z_OK;
 			break;
 		}
-	} while (error == Z_OK && stream.avail_in > 0);
+	} while (error == Z_OK && stream->avail_in > 0);
 
-	inflateEnd(&stream);
+	if (error == Z_STREAM_END) {
+		inflateEnd(stream);
+		enc_data->after_end = 1;
+		error = Z_OK;
+	}
 
 	if (error == Z_OK) {
-		*new_len = stream.total_out;
+		*new_len = stream->total_out;
 		return buffer;
 	} else {
 		if (buffer) mem_free(buffer);
@@ -219,17 +221,17 @@ deflate_decode_buffer(int window_size, unsigned char *data, int len, int *new_le
 }
 
 static unsigned char *
-deflate_raw_decode_buffer(unsigned char *data, int len, int *new_len)
+deflate_raw_decode_buffer(struct stream_encoded *st, unsigned char *data, int len, int *new_len)
 {
 	/* raw DEFLATE with neither zlib nor gzip header */
-	return deflate_decode_buffer(-MAX_WBITS, data, len, new_len);
+	return deflate_decode_buffer(st, -MAX_WBITS, data, len, new_len);
 }
 
 static unsigned char *
-deflate_gzip_decode_buffer(unsigned char *data, int len, int *new_len)
+deflate_gzip_decode_buffer(struct stream_encoded *st, unsigned char *data, int len, int *new_len)
 {
 	/* detect gzip header, else assume zlib header */
-	return deflate_decode_buffer(MAX_WBITS + 32, data, len, new_len);
+	return deflate_decode_buffer(st, MAX_WBITS + 32, data, len, new_len);
 }
 
 static void
@@ -238,8 +240,12 @@ deflate_close(struct stream_encoded *stream)
 	struct deflate_enc_data *data = (struct deflate_enc_data *) stream->data;
 
 	if (data) {
-		inflateEnd(&data->deflate_stream);
+		if (!data->after_end) {
-		close(data->fdread);
+			inflateEnd(&data->deflate_stream);
+		}
+		if (data->fdread != -1) {
+			close(data->fdread);
+		}
 		mem_free(data);
 		stream->data = 0;
 	}

src/encoding/encoding.c

@@ -52,7 +52,7 @@ dummy_read(struct stream_encoded *stream, unsigned char *data, int len)
 }
 
 static unsigned char *
-dummy_decode_buffer(unsigned char *data, int len, int *new_len)
+dummy_decode_buffer(struct stream_encoded *stream, unsigned char *data, int len, int *new_len)
 {
 	unsigned char *buffer = memacpy(data, len);
 
@@ -131,10 +131,10 @@ read_encoded(struct stream_encoded *stream, unsigned char *data, int len)
  * for parts of files. @data contains the original data, @len bytes
  * long. The resulting decoded data chunk is *@new_len bytes long. */
 unsigned char *
-decode_encoded_buffer(enum stream_encoding encoding, unsigned char *data, int len,
+decode_encoded_buffer(struct stream_encoded *stream, enum stream_encoding encoding, unsigned char *data, int len,
 		      int *new_len)
 {
-	return decoding_backends[encoding]->decode_buffer(data, len, new_len);
+	return decoding_backends[encoding]->decode_buffer(stream, data, len, new_len);
 }
 
 /* Closes encoded stream. Note that fd associated with the stream will be

src/encoding/encoding.h

@@ -25,13 +25,13 @@ struct decoding_backend {
 	const unsigned char *const *extensions;
 	int (*open)(struct stream_encoded *stream, int fd);
 	int (*read)(struct stream_encoded *stream, unsigned char *data, int len);
-	unsigned char *(*decode_buffer)(unsigned char *data, int len, int *new_len);
+	unsigned char *(*decode_buffer)(struct stream_encoded *stream, unsigned char *data, int len, int *new_len);
 	void (*close)(struct stream_encoded *stream);
 };
 
 struct stream_encoded *open_encoded(int, enum stream_encoding);
 int read_encoded(struct stream_encoded *, unsigned char *, int);
-unsigned char *decode_encoded_buffer(enum stream_encoding encoding, unsigned char *data, int len, int *new_len);
+unsigned char *decode_encoded_buffer(struct stream_encoded *stream, enum stream_encoding encoding, unsigned char *data, int len, int *new_len);
 void close_encoded(struct stream_encoded *);
 
 const unsigned char *const *listext_encoded(enum stream_encoding);

src/encoding/lzma.c

@@ -27,7 +27,8 @@
 struct lzma_enc_data {
 	lzma_stream flzma_stream;
 	int fdread;
-	int last_read;
+	int last_read:1;
+	int after_end:1;
 	unsigned char buf[ELINKS_BZ_BUFFER_LENGTH];
 };
 
@@ -105,23 +106,25 @@ lzma_read(struct stream_encoded *stream, unsigned char *buf, int len)
 }
 
 static unsigned char *
-lzma_decode_buffer(unsigned char *data, int len, int *new_len)
+lzma_decode_buffer(struct stream_encoded *st, unsigned char *data, int len, int *new_len)
 {
-	lzma_stream stream = LZMA_STREAM_INIT;
+	struct lzma_enc_data *enc_data = (struct lzma_enc_data *) st->data;
+	lzma_stream *stream = &enc_data->flzma_stream;
 	unsigned char *buffer = NULL;
 	int error;
 
 	*new_len = 0;	  /* default, left there if an error occurs */
 
-	stream.next_in = data;
+	stream->next_in = data;
-	stream.avail_in = len;
+	stream->avail_in = len;
+	stream->total_out = 0;
 
-	if (lzma_auto_decoder(&stream, ELINKS_LZMA_MEMORY_LIMIT, 0) != LZMA_OK)
+	if (lzma_auto_decoder(stream, ELINKS_LZMA_MEMORY_LIMIT, 0) != LZMA_OK)
 		return NULL;
 
 	do {
 		unsigned char *new_buffer;
-		size_t size = stream.total_out + MAX_STR_LEN;
+		size_t size = stream->total_out + MAX_STR_LEN;
 
 		new_buffer = mem_realloc(buffer, size);
 		if (!new_buffer) {
@@ -130,20 +133,24 @@ lzma_decode_buffer(unsigned char *data, int len, int *new_len)
 		}
 
 		buffer		 = new_buffer;
-		stream.next_out  = buffer + stream.total_out;
+		stream->next_out  = buffer + stream->total_out;
-		stream.avail_out = MAX_STR_LEN;
+		stream->avail_out = MAX_STR_LEN;
 
-		error = lzma_code(&stream, LZMA_RUN);
+		error = lzma_code(stream, LZMA_RUN);
 		if (error == LZMA_STREAM_END) {
 			error = LZMA_OK;
 			break;
 		}
-	} while (error == LZMA_OK && stream.avail_in > 0);
+	} while (error == LZMA_OK && stream->avail_in > 0);
 
-	lzma_end(&stream);
+	if (error == LZMA_STREAM_END) {
+		lzma_end(stream);
+		enc_data->after_end = 1;
+		error = LZMA_OK;
+	}
 
 	if (error == LZMA_OK) {
-		*new_len = stream.total_out;
+		*new_len = stream->total_out;
 		return buffer;
 	} else {
 		if (buffer) mem_free(buffer);
@@ -157,8 +164,12 @@ lzma_close(struct stream_encoded *stream)
 	struct lzma_enc_data *data = (struct lzma_enc_data *) stream->data;
 
 	if (data) {
-		lzma_end(&data->flzma_stream);
+		if (!data->after_end) {
-		close(data->fdread);
+			lzma_end(&data->flzma_stream);
+		}
+		if (data->fdread != -1) {
+			close(data->fdread);
+		}
 		mem_free(data);
 		stream->data = 0;
 	}

src/network/connection.c

@@ -308,7 +308,6 @@ init_connection(struct uri *uri, struct uri *proxied_uri, struct uri *referrer,
 	conn->cache_mode = cache_mode;
 
 	conn->content_encoding = ENCODING_NONE;
-	conn->stream_pipes[0] = conn->stream_pipes[1] = -1;
 	init_list(conn->downloads);
 	conn->est_length = -1;
 	conn->timer = TIMER_ID_UNDEF;
@@ -402,14 +401,7 @@ shutdown_connection_stream(struct connection *conn)
 	if (conn->stream) {
 		close_encoded(conn->stream);
 		conn->stream = NULL;
-	} else if (conn->stream_pipes[0] >= 0) {
-		/* close_encoded() usually closes this end of the pipe,
-		 * but open_encoded() apparently failed this time.  */
-		close(conn->stream_pipes[0]);
 	}
-	if (conn->stream_pipes[1] >= 0)
-		close(conn->stream_pipes[1]);
-	conn->stream_pipes[0] = conn->stream_pipes[1] = -1;
 }
 
 static void

src/network/connection.h

@@ -61,7 +61,6 @@ struct connection {
 
 	int tries;
 	timer_id_T timer;
-	int stream_pipes[2];
 
 	unsigned int running:1;
 	unsigned int unrestartable:1;

src/protocol/http/http.c

@@ -612,7 +612,6 @@ accept_encoding_header(struct string *header)
 }
 
 #define POST_BUFFER_SIZE 16384
-#define BIG_READ 655360
 
 static void
 send_more_post_data(struct socket *socket)
@@ -1072,7 +1071,6 @@ decompress_data(struct connection *conn, unsigned char *data, int len,
 {
 	struct http_connection_info *http = conn->info;
 	enum { NORMAL, FINISHING } state = NORMAL;
-	int did_read = 0;
 	int *length_of_block;
 	unsigned char *output = NULL;
 
@@ -1096,74 +1094,26 @@ decompress_data(struct connection *conn, unsigned char *data, int len,
 
 	*new_len = 0; /* new_len must be zero if we would ever return NULL */
 
-	if (conn->stream_pipes[0] == -1
+	if (!conn->stream) {
-	    && (c_pipe(conn->stream_pipes) < 0
+		conn->stream = open_encoded(-1, conn->content_encoding);
-		|| set_nonblocking_fd(conn->stream_pipes[0]) < 0
+		if (!conn->stream) return NULL;
-		|| set_nonblocking_fd(conn->stream_pipes[1]) < 0)) {
-		return NULL;
 	}
 
-	do {
+	output = decode_encoded_buffer(conn->stream, conn->content_encoding, data, len, new_len);
-		unsigned char *tmp;
 
-		if (state == NORMAL) {
+	if (*length_of_block > 0) {
-			/* ... we aren't finishing yet. */
+		*length_of_block -= len;
-			int written = safe_write(conn->stream_pipes[1], data, len);
+	}
-
+	/* http->length is 0 at the end of block for all modes: keep-alive,
-			if (written >= 0) {
+	 * non-keep-alive and chunked */
-				data += written;
+	if (!http->length) {
-				len -= written;
+		/* That's all, folks - let's finish this. */
-
+		state = FINISHING;
-				/* In non-keep-alive connections http->length == -1, so the test below */
+	}
-				if (*length_of_block > 0)
-					*length_of_block -= written;
-				/* http->length is 0 at the end of block for all modes: keep-alive,
-				 * non-keep-alive and chunked */
-				if (!http->length) {
-					/* That's all, folks - let's finish this. */
-					state = FINISHING;
-				} else if (!len) {
-					/* We've done for this round (but not done
-					 * completely). Thus we will get out with
-					 * what we have and leave what we wrote to
-					 * the next round - we have to do that since
-					 * we MUST NOT ever empty the pipe completely
-					 * - this would cause a disaster for
-					 * read_encoded(), which would simply not
-					 * work right then. */
-					return output;
-				}
-			}
-		}
-
-		if (!conn->stream) {
-			conn->stream = open_encoded(conn->stream_pipes[0],
-					conn->content_encoding);
-			if (!conn->stream) return NULL;
-		}
-
-		tmp = mem_realloc(output, *new_len + BIG_READ);
-		if (!tmp) break;
-		output = tmp;
-
-		did_read = read_encoded(conn->stream, output + *new_len, BIG_READ);
-
-		/* Do not break from the loop if did_read == 0.  It
-		 * means no decoded data is available yet, but some may
-		 * become available later.  This happens especially with
-		 * the bzip2 decoder, which needs an entire compressed
-		 * block as input before it generates any output.  */
-		if (did_read < 0) {
-			state = FINISHING;
-			break;
-		}
-		*new_len += did_read;
-	} while (len || (did_read == BIG_READ));
 
 	if (state == FINISHING) shutdown_connection_stream(conn);
 	return output;
 }
-#undef BIG_READ
 
 static int
 is_line_in_buffer(struct read_buffer *rb)