Index: configure.in =================================================================== --- configure.in (revision 2139) +++ configure.in (working copy) @@ -240,6 +240,34 @@ dnl Checks for libraries. dnl +dnl $with_zlib +AC_ARG_WITH(zlib, +[ --with-zlib[[=DIR]] use libz in DIR],[ + if test "$withval" != "no" -a "$withval" != "yes"; then + Z_DIR=$withval + CPPFLAGS="${CPPFLAGS} -I$withval/include" + LDFLAGS="${LDFLAGS} -L$withval/lib" + fi +]) +if test "$with_zlib" = "no"; then + echo "Disabling compression support" +else + AC_CHECK_HEADERS(zlib.h, + AC_CHECK_LIB(z, gzopen,[ + AC_DEFINE([HAVE_ZLIB], [], [Have compression library]) + if test "x${Z_DIR}" != "x"; then + CPPFLAGS="${CPPFLAGS} -I${Z_DIR}/include" + LIBS="${LIBS} -L${Z_DIR}/lib -lz" + [case ${host} in + *-*-solaris*) + LIBS="-L${Z_DIR}/lib -R${Z_DIR}/lib -lz" + ;; + esac] + else + LIBS="-lz" + fi])) +fi + if test x"$with_ssl" = xgnutls then dnl Now actually check for -lssl Index: src/options.h =================================================================== --- src/options.h (revision 2139) +++ src/options.h (working copy) @@ -214,6 +214,11 @@ bool ipv4_only; /* IPv4 connections have been requested. */ bool ipv6_only; /* IPv4 connections have been requested. */ #endif + +#ifdef HAVE_ZLIB + int compress; /* enable gzip compression requests */ +#endif + enum { prefer_ipv4, prefer_ipv6, Index: src/init.c =================================================================== --- src/init.c (revision 2139) +++ src/init.c (working copy) @@ -123,6 +123,9 @@ { "certificatetype", &opt.cert_type, cmd_cert_type }, { "checkcertificate", &opt.check_cert, cmd_boolean }, #endif +#ifdef HAVE_ZLIB + { "compress", &opt.compress, cmd_boolean }, +#endif { "connecttimeout", &opt.connect_timeout, cmd_time }, { "continue", &opt.always_rest, cmd_boolean }, { "convertlinks", &opt.convert_links, cmd_boolean }, Index: src/http.c =================================================================== --- src/http.c (revision 2139) +++ src/http.c (working copy) @@ -1315,6 +1315,9 @@ bool inhibit_keep_alive = !opt.http_keep_alive || opt.ignore_length || proxy != NULL; + /* Content encoding */ + char *encoding; + /* Headers sent when using POST. */ wgint post_data_size = 0; @@ -1384,6 +1387,12 @@ SET_USER_AGENT (req); request_set_header (req, "Accept", "*/*", rel_none); +#ifdef HAVE_ZLIB + /* XXX order needs to be settable via the command-line */ + if (opt.compress) + request_set_header (req, "Accept-Encoding", "deflate,gzip", rel_none); +#endif + /* Find the username and password for authentication. */ user = u->user; passwd = u->passwd; @@ -1524,6 +1533,7 @@ for the Digest authorization scheme.) */ keep_alive = false; + encoding = 0; /* Establish the connection. */ @@ -1934,6 +1944,9 @@ *tmp = '\0'; } } + + encoding = resp_header_strdup (resp, "Content-Encoding"); + hs->newloc = resp_header_strdup (resp, "Location"); hs->remote_time = resp_header_strdup (resp, "Last-Modified"); @@ -2183,6 +2196,11 @@ /* If the server ignored our range request, instruct fd_read_body to skip the first RESTVAL bytes of body. */ flags |= rb_skip_startpos; + if (encoding && (strncasecmp(encoding, "deflate", 8) == 0)) + flags |= rb_content_deflate; + if (encoding && (strncasecmp(encoding, "gzip", 4) == 0)) + flags |= rb_content_gzip; + hs->len = hs->restval; hs->rd_size = 0; hs->res = fd_read_body (sock, fp, contlen != -1 ? contlen : 0, Index: src/retr.c =================================================================== --- src/retr.c (revision 2139) +++ src/retr.c (working copy) @@ -38,6 +38,8 @@ #include #include +#include + #include "wget.h" #include "utils.h" #include "retr.h" @@ -170,6 +172,12 @@ return !ferror (out); } +/* Buffer size for zlib decompression, of course, bigger is better + since it means less cycles per inflate call. */ +#ifdef HAVE_ZLIB +#define ZLIB_INFLATE_BUFFER_SIZE 0x10000 +#endif + /* Read the contents of file descriptor FD until it the connection terminates or a read error occurs. The data is read in portions of up to 16K and written to OUT as it arrives. If opt.verbose is set, @@ -217,6 +225,19 @@ wgint sum_read = 0; wgint sum_written = 0; +#ifdef HAVE_ZLIB + /* zlib specific variables defines */ + int wbits = -MAX_WBITS; + int err; + unsigned char debuf[ZLIB_INFLATE_BUFFER_SIZE]; + unsigned long last = 0; + z_stream z; + + z.zalloc = (alloc_func)Z_NULL; + z.zfree = (free_func)Z_NULL; + z.opaque = (voidpf)0; +#endif + if (flags & rb_skip_startpos) skip = startpos; @@ -296,11 +317,67 @@ if (ret > 0) { sum_read += ret; - if (!write_data (out, dlbuf, ret, &skip, &sum_written)) - { - ret = -2; - goto out; - } + if ((flags & rb_content_deflate) || (flags & rb_content_gzip)) + { + #ifdef HAVE_ZLIB + z.next_in = (unsigned char *) dlbuf; + z.avail_in = (unsigned int) ret; + + /* first time thru read the compressed header and init zlib */ + if (last == 0) { + if (flags & rb_content_gzip) + wbits = MAX_WBITS + 32; + + if ((err = inflateInit2(&z, wbits)) != Z_OK) { + logprintf (LOG_NOTQUIET, "inflateInit2() : %s\n", z.msg); + ret = -2; + goto out; + } + } + + for (;;) { + z.avail_out = (int)sizeof(debuf); + z.next_out = debuf; + + if ((err = inflate(&z, Z_SYNC_FLUSH)) != Z_OK) { + if (err != Z_STREAM_END) { + logprintf (LOG_NOTQUIET, "inflate() : %s\n", z.msg); + ret = -2; + goto out; + } + + } + + /* XXX don't think we can skip a compressed buf */ + if (!write_data(out, debuf, (z.total_out - last), + &skip, &sum_written)) + { + ret = -2; + goto out; + } + + + last = z.total_out; + + if (z.avail_in == 0) { + if (err == Z_STREAM_END) + /* XXX http.c checks for these to + match hstat.len/contlen */ + sum_written = sum_read; + + break; + } + + } + #endif + } + else if (!write_data (out, dlbuf, ret, &skip, &sum_written)) + { + ret = -2; + goto out; + } + + } if (opt.limit_rate) @@ -318,6 +395,10 @@ ret = -1; out: +#ifdef HAVE_ZLIB + inflateEnd(&z); +#endif + if (progress) progress_finish (progress, ptimer_read (timer)); Index: src/retr.h =================================================================== --- src/retr.h (revision 2139) +++ src/retr.h (working copy) @@ -40,7 +40,9 @@ /* Flags for fd_read_body. */ enum { rb_read_exactly = 1, - rb_skip_startpos = 2 + rb_skip_startpos = 2, + rb_content_deflate = 4, + rb_content_gzip = 16 }; int fd_read_body (int, FILE *, wgint, wgint, wgint *, wgint *, double *, int); Index: src/main.c =================================================================== --- src/main.c (revision 2139) +++ src/main.c (working copy) @@ -141,6 +141,9 @@ { IF_SSL ("check-certificate"), 0, OPT_BOOLEAN, "checkcertificate", -1 }, { "clobber", 0, OPT__CLOBBER, NULL, optional_argument }, { "connect-timeout", 0, OPT_VALUE, "connecttimeout", -1 }, +#ifdef HAVE_ZLIB + { "compress", 'z', OPT_BOOLEAN, "compress", -1 }, +#endif { "continue", 'c', OPT_BOOLEAN, "continue", -1 }, { "convert-links", 'k', OPT_BOOLEAN, "convertlinks", -1 }, { "cookies", 0, OPT_BOOLEAN, "cookies", -1 }, @@ -518,6 +521,10 @@ --post-data=STRING use the POST method; send STRING as the data.\n"), N_("\ --post-file=FILE use the POST method; send contents of FILE.\n"), +#ifdef HAVE_ZLIB + N_("\ + -z --compress request gzip http compression\n"), +#endif "\n", #ifdef HAVE_SSL