aniani/elinks
1
0
Fork 0
mirror of https://github.com/rkd77/elinks.git synced 2024-10-23 07:20:10 -04:00
Code

Removed code using pipes for decompression and simplified decompress_data.

Browse Source 
Workarounds for sites, which send incorrect data, probably won't work.
This commit is contained in:
Witold Filipczyk 2010-09-24 16:12:35 +02:00 committed by Witold Filipczyk
parent a9c02bbf01
commit 7933724dc8
9 changed files with 112 additions and 140 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
src/document/renderer.c
View File

@ -229,14 +229,21 @@ render_encoded_document(struct cache_entry *cached, struct document *document)
if (encoding != ENCODING_NONE) {
int length = 0;
unsigned char *source;
struct stream_encoded *stream = open_encoded(-1, encoding);
source = decode_encoded_buffer(encoding, buffer.source,
buffer.length, &length);
if (source) {
buffer.source = source;
buffer.length = length;
} else {
if (!stream) {
encoding = ENCODING_NONE;
} else {
source = decode_encoded_buffer(stream, encoding, buffer.source,
buffer.length, &length);
close_encoded(stream);
if (source) {
buffer.source = source;
buffer.length = length;
} else {
encoding = ENCODING_NONE;
}
}
}
}

47
src/encoding/bzip2.c
View File

@ -36,7 +36,8 @@ struct bz2_enc_data {
* end-of-stream marker and all data has been decompressed.
* Then we neither read from the file nor call BZ2_bzDecompress
* any more. */
int last_read;
int last_read:1;
int after_end:1;
/* A buffer for data that has been read from the file but not
* yet decompressed. fbz_stream.next_in and fbz_stream.avail_in
@ -132,31 +133,30 @@ bzip2_read(struct stream_encoded *stream, unsigned char *buf, int len)
#endif
static unsigned char *
bzip2_decode_buffer(unsigned char *data, int len, int *new_len)
bzip2_decode_buffer(struct stream_encoded *st, unsigned char *data, int len, int *new_len)
{
bz_stream stream;
struct bz2_enc_data *enc_data = (struct bz2_enc_data *)st->data;
bz_stream *stream = &enc_data->fbz_stream;
unsigned char *buffer = NULL;
int error;
*new_len = 0; /* default, left there if an error occurs */
memset(&stream, 0, sizeof(bz_stream));
stream.next_in = data;
stream.avail_in = len;
if (BZ2_bzDecompressInit(&stream, 0, BZIP2_SMALL) != BZ_OK)
return NULL;
stream->next_in = data;
stream->avail_in = len;
stream->total_out_lo32 = 0;
stream->total_out_hi32 = 0;
do {
unsigned char *new_buffer;
size_t size = stream.total_out_lo32 + MAX_STR_LEN;
size_t size = stream->total_out_lo32 + MAX_STR_LEN;
/* FIXME: support for 64 bit. real size is
*
* (total_in_hi32 << * 32) + total_in_lo32
*
* --jonas */
assertm(!stream.total_out_hi32, "64 bzip2 decoding not supported");
assertm(!stream->total_out_hi32, "64 bzip2 decoding not supported");
new_buffer = mem_realloc(buffer, size);
if (!new_buffer) {
@ -165,12 +165,11 @@ bzip2_decode_buffer(unsigned char *data, int len, int *new_len)
}
buffer = new_buffer;
stream.next_out = buffer + stream.total_out_lo32;
stream.avail_out = MAX_STR_LEN;
stream->next_out = buffer + stream->total_out_lo32;
stream->avail_out = MAX_STR_LEN;
error = BZ2_bzDecompress(&stream);
error = BZ2_bzDecompress(stream);
if (error == BZ_STREAM_END) {
error = BZ_OK;
break;
}
@ -178,12 +177,16 @@ bzip2_decode_buffer(unsigned char *data, int len, int *new_len)
* is reached. At least lindi- reported that it caused a
* reproducable infinite loop. Maybe it has to do with decoding
* an incomplete file. */
} while (error == BZ_OK && stream.avail_in > 0);
} while (error == BZ_OK && stream->avail_in > 0);
BZ2_bzDecompressEnd(&stream);
if (error == BZ_STREAM_END) {
BZ2_bzDecompressEnd(stream);
enc_data->after_end = 1;
error = BZ_OK;
}
if (error == BZ_OK) {
*new_len = stream.total_out_lo32;
*new_len = stream->total_out_lo32;
return buffer;
} else {
if (buffer) mem_free(buffer);
@ -197,8 +200,12 @@ bzip2_close(struct stream_encoded *stream)
struct bz2_enc_data *data = (struct bz2_enc_data *) stream->data;
if (data) {
BZ2_bzDecompressEnd(&data->fbz_stream);
close(data->fdread);
if (!data->after_end) {
BZ2_bzDecompressEnd(&data->fbz_stream);
}
if (data->fdread != -1) {
close(data->fdread);
}
mem_free(data);
stream->data = 0;
}

50
src/encoding/deflate.c
View File

@ -31,6 +31,7 @@ struct deflate_enc_data {
unsigned int last_read:1;
unsigned int after_first_read:1;
unsigned int after_end:1;
/* A buffer for data that has been read from the file but not
* yet decompressed. z_stream.next_in and z_stream.avail_in
@ -170,25 +171,23 @@ restart:
}
static unsigned char *
deflate_decode_buffer(int window_size, unsigned char *data, int len, int *new_len)
deflate_decode_buffer(struct stream_encoded *st, int window_size, unsigned char *data, int len, int *new_len)
{
z_stream stream;
struct deflate_enc_data *enc_data = (struct deflate_enc_data *) st->data;
z_stream *stream = &enc_data->deflate_stream;
unsigned char *buffer = NULL;
int error;
*new_len = 0; /* default, left there if an error occurs */
if (!len) return NULL;
memset(&stream, 0, sizeof(z_stream));
stream.next_in = data;
stream.avail_in = len;
if (inflateInit2(&stream, window_size) != Z_OK)
return NULL;
stream->next_in = data;
stream->avail_in = len;
stream->total_out = 0;
do {
unsigned char *new_buffer;
size_t size = stream.total_out + MAX_STR_LEN;
size_t size = stream->total_out + MAX_STR_LEN;
new_buffer = mem_realloc(buffer, size);
if (!new_buffer) {
@ -197,20 +196,23 @@ deflate_decode_buffer(int window_size, unsigned char *data, int len, int *new_le
}
buffer = new_buffer;
stream.next_out = buffer + stream.total_out;
stream.avail_out = MAX_STR_LEN;
stream->next_out = buffer + stream->total_out;
stream->avail_out = MAX_STR_LEN;
error = inflate(&stream, Z_SYNC_FLUSH);
error = inflate(stream, Z_SYNC_FLUSH);
if (error == Z_STREAM_END) {
error = Z_OK;
break;
}
} while (error == Z_OK && stream.avail_in > 0);
} while (error == Z_OK && stream->avail_in > 0);
inflateEnd(&stream);
if (error == Z_STREAM_END) {
inflateEnd(stream);
enc_data->after_end = 1;
error = Z_OK;
}
if (error == Z_OK) {
*new_len = stream.total_out;
*new_len = stream->total_out;
return buffer;
} else {
if (buffer) mem_free(buffer);
@ -219,17 +221,17 @@ deflate_decode_buffer(int window_size, unsigned char *data, int len, int *new_le
}
static unsigned char *
deflate_raw_decode_buffer(unsigned char *data, int len, int *new_len)
deflate_raw_decode_buffer(struct stream_encoded *st, unsigned char *data, int len, int *new_len)
{
/* raw DEFLATE with neither zlib nor gzip header */
return deflate_decode_buffer(-MAX_WBITS, data, len, new_len);
return deflate_decode_buffer(st, -MAX_WBITS, data, len, new_len);
}
static unsigned char *
deflate_gzip_decode_buffer(unsigned char *data, int len, int *new_len)
deflate_gzip_decode_buffer(struct stream_encoded *st, unsigned char *data, int len, int *new_len)
{
/* detect gzip header, else assume zlib header */
return deflate_decode_buffer(MAX_WBITS + 32, data, len, new_len);
return deflate_decode_buffer(st, MAX_WBITS + 32, data, len, new_len);
}
static void
@ -238,8 +240,12 @@ deflate_close(struct stream_encoded *stream)
struct deflate_enc_data *data = (struct deflate_enc_data *) stream->data;
if (data) {
inflateEnd(&data->deflate_stream);
close(data->fdread);
if (!data->after_end) {
inflateEnd(&data->deflate_stream);
}
if (data->fdread != -1) {
close(data->fdread);
}
mem_free(data);
stream->data = 0;
}

6
src/encoding/encoding.c
View File

@ -52,7 +52,7 @@ dummy_read(struct stream_encoded *stream, unsigned char *data, int len)
}
static unsigned char *
dummy_decode_buffer(unsigned char *data, int len, int *new_len)
dummy_decode_buffer(struct stream_encoded *stream, unsigned char *data, int len, int *new_len)
{
unsigned char *buffer = memacpy(data, len);
@ -131,10 +131,10 @@ read_encoded(struct stream_encoded *stream, unsigned char *data, int len)
* for parts of files. @data contains the original data, @len bytes
* long. The resulting decoded data chunk is *@new_len bytes long. */
unsigned char *
decode_encoded_buffer(enum stream_encoding encoding, unsigned char *data, int len,
decode_encoded_buffer(struct stream_encoded *stream, enum stream_encoding encoding, unsigned char *data, int len,
int *new_len)
{
return decoding_backends[encoding]->decode_buffer(data, len, new_len);
return decoding_backends[encoding]->decode_buffer(stream, data, len, new_len);
}
/* Closes encoded stream. Note that fd associated with the stream will be

4
src/encoding/encoding.h
View File

@ -25,13 +25,13 @@ struct decoding_backend {
const unsigned char *const *extensions;
int (*open)(struct stream_encoded *stream, int fd);
int (*read)(struct stream_encoded *stream, unsigned char *data, int len);
unsigned char *(*decode_buffer)(unsigned char *data, int len, int *new_len);
unsigned char *(*decode_buffer)(struct stream_encoded *stream, unsigned char *data, int len, int *new_len);
void (*close)(struct stream_encoded *stream);
};
struct stream_encoded *open_encoded(int, enum stream_encoding);
int read_encoded(struct stream_encoded *, unsigned char *, int);
unsigned char *decode_encoded_buffer(enum stream_encoding encoding, unsigned char *data, int len, int *new_len);
unsigned char *decode_encoded_buffer(struct stream_encoded *stream, enum stream_encoding encoding, unsigned char *data, int len, int *new_len);
void close_encoded(struct stream_encoded *);
const unsigned char *const *listext_encoded(enum stream_encoding);

41
src/encoding/lzma.c
View File

@ -27,7 +27,8 @@
struct lzma_enc_data {
lzma_stream flzma_stream;
int fdread;
int last_read;
int last_read:1;
int after_end:1;
unsigned char buf[ELINKS_BZ_BUFFER_LENGTH];
};
@ -105,23 +106,25 @@ lzma_read(struct stream_encoded *stream, unsigned char *buf, int len)
}
static unsigned char *
lzma_decode_buffer(unsigned char *data, int len, int *new_len)
lzma_decode_buffer(struct stream_encoded *st, unsigned char *data, int len, int *new_len)
{
lzma_stream stream = LZMA_STREAM_INIT;
struct lzma_enc_data *enc_data = (struct lzma_enc_data *) st->data;
lzma_stream *stream = &enc_data->flzma_stream;
unsigned char *buffer = NULL;
int error;
*new_len = 0; /* default, left there if an error occurs */
stream.next_in = data;
stream.avail_in = len;
stream->next_in = data;
stream->avail_in = len;
stream->total_out = 0;
if (lzma_auto_decoder(&stream, ELINKS_LZMA_MEMORY_LIMIT, 0) != LZMA_OK)
if (lzma_auto_decoder(stream, ELINKS_LZMA_MEMORY_LIMIT, 0) != LZMA_OK)
return NULL;
do {
unsigned char *new_buffer;
size_t size = stream.total_out + MAX_STR_LEN;
size_t size = stream->total_out + MAX_STR_LEN;
new_buffer = mem_realloc(buffer, size);
if (!new_buffer) {
@ -130,20 +133,24 @@ lzma_decode_buffer(unsigned char *data, int len, int *new_len)
}
buffer = new_buffer;
stream.next_out = buffer + stream.total_out;
stream.avail_out = MAX_STR_LEN;
stream->next_out = buffer + stream->total_out;
stream->avail_out = MAX_STR_LEN;
error = lzma_code(&stream, LZMA_RUN);
error = lzma_code(stream, LZMA_RUN);
if (error == LZMA_STREAM_END) {
error = LZMA_OK;
break;
}
} while (error == LZMA_OK && stream.avail_in > 0);
} while (error == LZMA_OK && stream->avail_in > 0);
lzma_end(&stream);
if (error == LZMA_STREAM_END) {
lzma_end(stream);
enc_data->after_end = 1;
error = LZMA_OK;
}
if (error == LZMA_OK) {
*new_len = stream.total_out;
*new_len = stream->total_out;
return buffer;
} else {
if (buffer) mem_free(buffer);
@ -157,8 +164,12 @@ lzma_close(struct stream_encoded *stream)
struct lzma_enc_data *data = (struct lzma_enc_data *) stream->data;
if (data) {
lzma_end(&data->flzma_stream);
close(data->fdread);
if (!data->after_end) {
lzma_end(&data->flzma_stream);
}
if (data->fdread != -1) {
close(data->fdread);
}
mem_free(data);
stream->data = 0;
}

8
src/network/connection.c
View File

@ -308,7 +308,6 @@ init_connection(struct uri *uri, struct uri *proxied_uri, struct uri *referrer,
conn->cache_mode = cache_mode;
conn->content_encoding = ENCODING_NONE;
conn->stream_pipes[0] = conn->stream_pipes[1] = -1;
init_list(conn->downloads);
conn->est_length = -1;
conn->timer = TIMER_ID_UNDEF;
@ -402,14 +401,7 @@ shutdown_connection_stream(struct connection *conn)
if (conn->stream) {
close_encoded(conn->stream);
conn->stream = NULL;
} else if (conn->stream_pipes[0] >= 0) {
/* close_encoded() usually closes this end of the pipe,
* but open_encoded() apparently failed this time. */
close(conn->stream_pipes[0]);
}
if (conn->stream_pipes[1] >= 0)
close(conn->stream_pipes[1]);
conn->stream_pipes[0] = conn->stream_pipes[1] = -1;
}
static void

1
src/network/connection.h
View File

@ -61,7 +61,6 @@ struct connection {
int tries;
timer_id_T timer;
int stream_pipes[2];
unsigned int running:1;
unsigned int unrestartable:1;

76
src/protocol/http/http.c
View File

@ -612,7 +612,6 @@ accept_encoding_header(struct string *header)
}
#define POST_BUFFER_SIZE 16384
#define BIG_READ 655360
static void
send_more_post_data(struct socket *socket)
@ -1072,7 +1071,6 @@ decompress_data(struct connection *conn, unsigned char *data, int len,
{
struct http_connection_info *http = conn->info;
enum { NORMAL, FINISHING } state = NORMAL;
int did_read = 0;
int *length_of_block;
unsigned char *output = NULL;
@ -1096,74 +1094,26 @@ decompress_data(struct connection *conn, unsigned char *data, int len,
*new_len = 0; /* new_len must be zero if we would ever return NULL */
if (conn->stream_pipes[0] == -1
&& (c_pipe(conn->stream_pipes) < 0
|| set_nonblocking_fd(conn->stream_pipes[0]) < 0
|| set_nonblocking_fd(conn->stream_pipes[1]) < 0)) {
return NULL;
if (!conn->stream) {
conn->stream = open_encoded(-1, conn->content_encoding);
if (!conn->stream) return NULL;
}
do {
unsigned char *tmp;
output = decode_encoded_buffer(conn->stream, conn->content_encoding, data, len, new_len);
if (state == NORMAL) {
/* ... we aren't finishing yet. */
int written = safe_write(conn->stream_pipes[1], data, len);
if (written >= 0) {
data += written;
len -= written;
/* In non-keep-alive connections http->length == -1, so the test below */
if (*length_of_block > 0)
*length_of_block -= written;
/* http->length is 0 at the end of block for all modes: keep-alive,
* non-keep-alive and chunked */
if (!http->length) {
/* That's all, folks - let's finish this. */
state = FINISHING;
} else if (!len) {
/* We've done for this round (but not done
* completely). Thus we will get out with
* what we have and leave what we wrote to
* the next round - we have to do that since
* we MUST NOT ever empty the pipe completely
* - this would cause a disaster for
* read_encoded(), which would simply not
* work right then. */
return output;
}
}
}
if (!conn->stream) {
conn->stream = open_encoded(conn->stream_pipes[0],
conn->content_encoding);
if (!conn->stream) return NULL;
}
tmp = mem_realloc(output, *new_len + BIG_READ);
if (!tmp) break;
output = tmp;
did_read = read_encoded(conn->stream, output + *new_len, BIG_READ);
/* Do not break from the loop if did_read == 0. It
* means no decoded data is available yet, but some may
* become available later. This happens especially with
* the bzip2 decoder, which needs an entire compressed
* block as input before it generates any output. */
if (did_read < 0) {
state = FINISHING;
break;
}
*new_len += did_read;
} while (len || (did_read == BIG_READ));
if (*length_of_block > 0) {
*length_of_block -= len;
}
/* http->length is 0 at the end of block for all modes: keep-alive,
* non-keep-alive and chunked */
if (!http->length) {
/* That's all, folks - let's finish this. */
state = FINISHING;
}
if (state == FINISHING) shutdown_connection_stream(conn);
return output;
}
#undef BIG_READ
static int
is_line_in_buffer(struct read_buffer *rb)