MINOR: compression: pass the content type to the algorithms' init function

Now that we have slz 1.4-dev4 with support for text-aware compression
using an alternate huffman tree, let's implement support for enabling it.
In order to permit this, the algorithms' init() function now takes a new
argument, <ctype>, which is either COMP_CT_GENERIC (0) for unknown or
binary contents, or COMP_CT_TEXT (1) for text contents.

comp_prepare_compress_request() and select_compression_response_header()
set it to COMP_CT_TEXT when the Content-Type header starts with "text/",
regardless of the "compression type" lists. The config-time checks of the
algorithms use COMP_CT_GENERIC.

The slz algorithms select SLZ_CT_TEXT after their init for text contents.
libslz has no dedicated macro for the content types, but they were
introduced right after SLZ_MAX_OVERHEAD(), which is thus used to detect an
external libslz that supports them. zlib and identity ignore the argument.

On an 81 kB HTML file, the response is 22.0 kB when served as text/html
instead of 25.4-27.1 kB when served as application/octet-stream at level 1
(the size depends on how the server sends the data).

diff --git a/doc/configuration.txt b/doc/configuration.txt
index 700fee9806a59eb33c916e717e5e2aa737e1b1c5..2698b146253ed2b64f179ae94ed6f01750247bb3 100644 (file)
--- a/doc/configuration.txt
+++ b/doc/configuration.txt
@@ -7463,6 +7463,10 @@ compression type <mime type> ...
   there is Accept-Encoding header in request, HAProxy will compress the
   matching response.
 
+  The Content-Type response header will be inspected, and its contents may be
+  used by some libraries to tune their compression algorithm. For example,
+  libslz uses a more efficient variant for types starting with "text/".
+
   Compression is disabled when:
     * the request does not advertise a supported compression algorithm in the
       "Accept-Encoding" header

diff --git a/include/haproxy/compression-t.h b/include/haproxy/compression-t.h
index 1b71cdc145d4e2aa4129a709ec7fc6cb2ebe4a7b..a2b2a77f1f23271d05ed4a34129c1a1bbe68bb29 100644 (file)
--- a/include/haproxy/compression-t.h
+++ b/include/haproxy/compression-t.h
@@ -70,6 +70,15 @@ struct comp_ctx {
        int cur_lvl;
 };
 
+/* Content types passed to the algorithms' init() functions, so that they can
+ * adapt their encoding to the contents. Algorithms which don't make any
+ * difference simply ignore them.
+ */
+enum comp_ctype {
+       COMP_CT_GENERIC = 0, /* unknown or binary contents */
+       COMP_CT_TEXT    = 1, /* "text/..." contents */
+};
+
 /* Thanks to MSIE/IIS, the "deflate" name is ambiguous, as according to the RFC
  * it's a zlib-wrapped deflate stream, but MSIE only understands a raw deflate
  * stream. For this reason some people prefer to emit a raw deflate stream on
@@ -83,7 +92,7 @@ struct comp_algo {
        char *ua_name;  /* name for the user-agent */
        int ua_name_len;
 
-       int (*init)(struct comp_ctx **comp_ctx, int level);
+       int (*init)(struct comp_ctx **comp_ctx, int level, int ctype);
        int (*add_data)(struct comp_ctx *comp_ctx, const char *in_data, int in_len, struct buffer *out);
        int (*flush)(struct comp_ctx *comp_ctx, struct buffer *out);
        int (*finish)(struct comp_ctx *comp_ctx, struct buffer *out);

diff --git a/src/compression.c b/src/compression.c
index 1d98cb7599717bbb58ac04c9b1753dfbde05a593..7b1acfe0c3a4b501735829f01fb860dcf78f7f22 100644 (file)
--- a/src/compression.c
+++ b/src/compression.c
@@ -62,7 +62,7 @@ static int global_tune_zlibwindowsize = MAX_WBITS;  /* zlib window size */
 
 unsigned int compress_min_idle = 0;
 
-static int comp_identity_init(struct comp_ctx **comp_ctx, int level);
+static int comp_identity_init(struct comp_ctx **comp_ctx, int level, int ctype);
 static int comp_identity_add_data(struct comp_ctx *comp_ctx, const char *in_data, int in_len, struct buffer *out);
 static int comp_identity_flush(struct comp_ctx *comp_ctx, struct buffer *out);
 static int comp_identity_finish(struct comp_ctx *comp_ctx, struct buffer *out);
@@ -70,9 +70,9 @@ static int comp_identity_end(struct comp_ctx **comp_ctx);
 
 #if defined(USE_SLZ)
 
-static int comp_rfc1950_init(struct comp_ctx **comp_ctx, int level);
-static int comp_rfc1951_init(struct comp_ctx **comp_ctx, int level);
-static int comp_rfc1952_init(struct comp_ctx **comp_ctx, int level);
+static int comp_rfc1950_init(struct comp_ctx **comp_ctx, int level, int ctype);
+static int comp_rfc1951_init(struct comp_ctx **comp_ctx, int level, int ctype);
+static int comp_rfc1952_init(struct comp_ctx **comp_ctx, int level, int ctype);
 static int comp_rfc195x_add_data(struct comp_ctx *comp_ctx, const char *in_data, int in_len, struct buffer *out);
 static int comp_rfc195x_flush(struct comp_ctx *comp_ctx, struct buffer *out);
 static int comp_rfc195x_finish(struct comp_ctx *comp_ctx, struct buffer *out);
@@ -87,9 +87,9 @@ static int decomp_uslz_end(struct decomp_ctx **decomp_ctx);
 
 #elif defined(USE_ZLIB)
 
-static int comp_gzip_init(struct comp_ctx **comp_ctx, int level);
-static int comp_raw_def_init(struct comp_ctx **comp_ctx, int level);
-static int comp_deflate_init(struct comp_ctx **comp_ctx, int level);
+static int comp_gzip_init(struct comp_ctx **comp_ctx, int level, int ctype);
+static int comp_raw_def_init(struct comp_ctx **comp_ctx, int level, int ctype);
+static int comp_deflate_init(struct comp_ctx **comp_ctx, int level, int ctype);
 static int comp_deflate_add_data(struct comp_ctx *comp_ctx, const char *in_data, int in_len, struct buffer *out);
 static int comp_deflate_flush(struct comp_ctx *comp_ctx, struct buffer *out);
 static int comp_deflate_finish(struct comp_ctx *comp_ctx, struct buffer *out);
@@ -288,7 +288,7 @@ static inline __maybe_unused int deinit_decomp_ctx(struct decomp_ctx **decomp_ct
 /*
  * Init the identity algorithm
  */
-static int comp_identity_init(struct comp_ctx **comp_ctx, int level)
+static int comp_identity_init(struct comp_ctx **comp_ctx, int level, int ctype)
 {
        return 0;
 }
@@ -333,34 +333,57 @@ static int comp_identity_end(struct comp_ctx **comp_ctx)
 
 #ifdef USE_SLZ
 
+/* Selects SLZ's content type matching <ctype> on the stream of <comp_ctx>
+ * after its init. libslz has no dedicated macro for the content types, but
+ * they come right after SLZ_MAX_OVERHEAD(), which is thus used to detect an
+ * older external libslz that doesn't support them, in which case the default
+ * encoding is always used.
+ */
+static inline void comp_slz_set_ctype(struct comp_ctx *comp_ctx, int ctype)
+{
+#if defined(SLZ_MAX_OVERHEAD)
+       if (ctype == COMP_CT_TEXT)
+               slz_set_content_type(&comp_ctx->strm, SLZ_CT_TEXT);
+#endif
+}
+
 /* SLZ's gzip format (RFC1952). Returns < 0 on error. */
-static int comp_rfc1952_init(struct comp_ctx **comp_ctx, int level)
+static int comp_rfc1952_init(struct comp_ctx **comp_ctx, int level, int ctype)
 {
        if (init_comp_ctx(comp_ctx) < 0)
                return -1;
 
        (*comp_ctx)->cur_lvl = level;
-       return slz_rfc1952_init(&(*comp_ctx)->strm, level);
+       if (slz_rfc1952_init(&(*comp_ctx)->strm, level) < 0)
+               return -1;
+       comp_slz_set_ctype(*comp_ctx, ctype);
+       return 0;
 }
 
 /* SLZ's raw deflate format (RFC1951). Returns < 0 on error. */
-static int comp_rfc1951_init(struct comp_ctx **comp_ctx, int level)
+static int comp_rfc1951_init(struct comp_ctx **comp_ctx, int level, int ctype)
 {
        if (init_comp_ctx(comp_ctx) < 0)
                return -1;
 
        (*comp_ctx)->cur_lvl = level;
-       return slz_rfc1951_init(&(*comp_ctx)->strm, level);
+       if (slz_rfc1951_init(&(*comp_ctx)->strm, level) < 0)
+               return -1;
+       comp_slz_set_ctype(*comp_ctx, ctype);
+       return 0;
 }
 
 /* SLZ's zlib format (RFC1950). Returns < 0 on error. */
-static int comp_rfc1950_init(struct comp_ctx **comp_ctx, int level)
+static int comp_rfc1950_init(struct comp_ctx **comp_ctx, int level, int ctype)
 {
        if (init_comp_ctx(comp_ctx) < 0)
                return -1;
 
        (*comp_ctx)->cur_lvl = level;
-       return slz_rfc1950_init(&(*comp_ctx)->strm, level);
+       if (slz_rfc1950_init(&(*comp_ctx)->strm, level) < 0)
+               return -1;
+       comp_slz_set_ctype(*comp_ctx, ctype);
+       return 0;
 }
 
 /* Return the size of consumed data. The caller is responsible for guaranteeing
@@ -657,7 +680,7 @@ static void free_zlib(void *opaque, void *ptr)
 /**************************
 ****  gzip algorithm   ****
 ***************************/
-static int comp_gzip_init(struct comp_ctx **comp_ctx, int level)
+static int comp_gzip_init(struct comp_ctx **comp_ctx, int level, int ctype)
 {
        z_stream *strm;
 
@@ -677,7 +700,7 @@ static int comp_gzip_init(struct comp_ctx **comp_ctx, int level)
 }
 
 /* Raw deflate algorithm */
-static int comp_raw_def_init(struct comp_ctx **comp_ctx, int level)
+static int comp_raw_def_init(struct comp_ctx **comp_ctx, int level, int ctype)
 {
        z_stream *strm;
 
@@ -699,7 +722,7 @@ static int comp_raw_def_init(struct comp_ctx **comp_ctx, int level)
 **** Deflate algorithm ****
 ***************************/
 
-static int comp_deflate_init(struct comp_ctx **comp_ctx, int level)
+static int comp_deflate_init(struct comp_ctx **comp_ctx, int level, int ctype)
 {
        z_stream *strm;
 

diff --git a/src/flt_http_comp.c b/src/flt_http_comp.c
index 3a3a6c60da833138e27515191d7d7d20157013bb..f76fe212ea05e5893454a7967abd8b66ca4104cc 100644 (file)
--- a/src/flt_http_comp.c
+++ b/src/flt_http_comp.c
@@ -127,6 +127,7 @@ comp_prepare_compress_request(struct comp_state *st, struct stream *s, struct ht
        struct http_hdr_ctx ctx;
        struct comp_type *comp_type;
        unsigned int comp_minsize = 0;
+       int ctype = COMP_CT_GENERIC;
 
        ctx.blk = NULL;
        /* Already compressed, don't bother */
@@ -152,6 +153,10 @@ comp_prepare_compress_request(struct comp_state *st, struct stream *s, struct ht
         */
        ctx.blk = NULL;
        if (http_find_header(htx, ist("Content-Type"), &ctx, 1)) {
+               /* text contents may be encoded differently by the algorithm */
+               if (ctx.value.len >= 5 && strncasecmp("text/", ctx.value.ptr, 5) == 0)
+                       ctype = COMP_CT_TEXT;
+
                if ((s->be->comp && (comp_type = s->be->comp->types_req)) ||
                    (strm_fe(s)->comp && (comp_type = strm_fe(s)->comp->types_req))) {
                        for (; comp_type; comp_type = comp_type->next) {
@@ -190,7 +195,7 @@ comp_prepare_compress_request(struct comp_state *st, struct stream *s, struct ht
                goto fail; /* no algo selected: nothing to do */
 
        /* initialize compression */
-       if (st->comp_algo->init(&st->comp_ctx, global.tune.comp_maxlevel) < 0)
+       if (st->comp_algo->init(&st->comp_ctx, global.tune.comp_maxlevel, ctype) < 0)
                goto fail;
 
        return;
@@ -683,6 +688,7 @@ select_compression_response_header(struct comp_state *st, struct stream *s, stru
        struct http_hdr_ctx ctx;
        struct comp_type *comp_type;
        unsigned int comp_minsize = 0;
+       int ctype = COMP_CT_GENERIC;
 
        /* no common compression algorithm was found in request header */
        if (st->comp_algo == NULL)
@@ -754,6 +760,10 @@ select_compression_response_header(struct comp_state *st, struct stream *s, stru
                if (ctx.value.len >= 9 && strncasecmp("multipart", ctx.value.ptr, 9) == 0)
                        goto fail;
 
+               /* text contents may be encoded differently by the algorithm */
+               if (ctx.value.len >= 5 && strncasecmp("text/", ctx.value.ptr, 5) == 0)
+                       ctype = COMP_CT_TEXT;
+
                if ((s->be->comp && (comp_type = s->be->comp->types_res)) ||
                    (strm_fe(s)->comp && (comp_type = strm_fe(s)->comp->types_res))) {
                        for (; comp_type; comp_type = comp_type->next) {
@@ -783,7 +793,7 @@ select_compression_response_header(struct comp_state *st, struct stream *s, stru
                goto fail;
 
        /* initialize compression */
-       if (st->comp_algo->init(&st->comp_ctx, global.tune.comp_maxlevel) < 0)
+       if (st->comp_algo->init(&st->comp_ctx, global.tune.comp_maxlevel, ctype) < 0)
                goto fail;
        msg->flags |= HTTP_MSGF_COMPRESSING;
        return 1;
@@ -912,7 +922,7 @@ parse_compression_options(char **args, int section, struct proxy *proxy,
                                goto end;
                        }
 
-                       if (proxy->comp->algos_res->init(&ctx, 9) == 0)
+                       if (proxy->comp->algos_res->init(&ctx, 9, COMP_CT_GENERIC) == 0)
                                proxy->comp->algos_res->end(&ctx);
                        else {
                                memprintf(err, "'%s' : Can't init '%s' algorithm.",
@@ -941,7 +951,7 @@ parse_compression_options(char **args, int section, struct proxy *proxy,
                        goto end;
                }
 
-               if (proxy->comp->algo_req->init(&ctx, 9) == 0)
+               if (proxy->comp->algo_req->init(&ctx, 9, COMP_CT_GENERIC) == 0)
                        proxy->comp->algo_req->end(&ctx);
                else {
                        memprintf(err, "'%s' : Can't init '%s' algorithm.",