From: Willy Tarreau Date: Sun, 1 Nov 2026 21:33:14 +0000 (+0100) Subject: MINOR: compression: pass the content type to the algorithms' init function X-Git-Tag: v3.5-dev11~57 X-Git-Url: https://git.kaiwu.me/$%7BGITURL%7D$%7Bcid%7D?a=commitdiff_plain;h=00bb955ca96e3f499596e98e950e500d680c2018;p=haproxy.git MINOR: compression: pass the content type to the algorithms' init function Now that we have slz 1.4-dev4 with support for text-aware compression using an alternate huffman tree, let's implement support for enabling it. In order to permit this, the algorithms' init() function now takes a new argument, , which is either COMP_CT_GENERIC (0) for unknown or binary contents, or COMP_CT_TEXT (1) for text contents. comp_prepare_compress_request() and select_compression_response_header() set it to COMP_CT_TEXT when the Content-Type header starts with "text/", regardless of the "compression type" lists. The config-time checks of the algorithms use COMP_CT_GENERIC. The slz algorithms select SLZ_CT_TEXT after their init for text contents. libslz has no dedicated macro for the content types, but they were introduced right after SLZ_MAX_OVERHEAD(), which is thus used to detect an external libslz that supports them. zlib and identity ignore the argument. On an 81 kB HTML file, the response is 22.0 kB when served as text/html instead of 25.4-27.1 kB when served as application/octet-stream at level 1 (the size depends on how the server sends the data). --- diff --git a/doc/configuration.txt b/doc/configuration.txt index 700fee980..2698b1462 100644 --- a/doc/configuration.txt +++ b/doc/configuration.txt @@ -7463,6 +7463,10 @@ compression type ... there is Accept-Encoding header in request, HAProxy will compress the matching response. + The Content-Type response header will be inspected, and its contents may be + used by some libraries to tune their compression algorithm. For example, + libslz uses a more efficient variant for types starting with "text/". + Compression is disabled when: * the request does not advertise a supported compression algorithm in the "Accept-Encoding" header diff --git a/include/haproxy/compression-t.h b/include/haproxy/compression-t.h index 1b71cdc14..a2b2a77f1 100644 --- a/include/haproxy/compression-t.h +++ b/include/haproxy/compression-t.h @@ -70,6 +70,15 @@ struct comp_ctx { int cur_lvl; }; +/* Content types passed to the algorithms' init() functions, so that they can + * adapt their encoding to the contents. Algorithms which don't make any + * difference simply ignore them. + */ +enum comp_ctype { + COMP_CT_GENERIC = 0, /* unknown or binary contents */ + COMP_CT_TEXT = 1, /* "text/..." contents */ +}; + /* Thanks to MSIE/IIS, the "deflate" name is ambiguous, as according to the RFC * it's a zlib-wrapped deflate stream, but MSIE only understands a raw deflate * stream. For this reason some people prefer to emit a raw deflate stream on @@ -83,7 +92,7 @@ struct comp_algo { char *ua_name; /* name for the user-agent */ int ua_name_len; - int (*init)(struct comp_ctx **comp_ctx, int level); + int (*init)(struct comp_ctx **comp_ctx, int level, int ctype); int (*add_data)(struct comp_ctx *comp_ctx, const char *in_data, int in_len, struct buffer *out); int (*flush)(struct comp_ctx *comp_ctx, struct buffer *out); int (*finish)(struct comp_ctx *comp_ctx, struct buffer *out); diff --git a/src/compression.c b/src/compression.c index 1d98cb759..7b1acfe0c 100644 --- a/src/compression.c +++ b/src/compression.c @@ -62,7 +62,7 @@ static int global_tune_zlibwindowsize = MAX_WBITS; /* zlib window size */ unsigned int compress_min_idle = 0; -static int comp_identity_init(struct comp_ctx **comp_ctx, int level); +static int comp_identity_init(struct comp_ctx **comp_ctx, int level, int ctype); static int comp_identity_add_data(struct comp_ctx *comp_ctx, const char *in_data, int in_len, struct buffer *out); static int comp_identity_flush(struct comp_ctx *comp_ctx, struct buffer *out); static int comp_identity_finish(struct comp_ctx *comp_ctx, struct buffer *out); @@ -70,9 +70,9 @@ static int comp_identity_end(struct comp_ctx **comp_ctx); #if defined(USE_SLZ) -static int comp_rfc1950_init(struct comp_ctx **comp_ctx, int level); -static int comp_rfc1951_init(struct comp_ctx **comp_ctx, int level); -static int comp_rfc1952_init(struct comp_ctx **comp_ctx, int level); +static int comp_rfc1950_init(struct comp_ctx **comp_ctx, int level, int ctype); +static int comp_rfc1951_init(struct comp_ctx **comp_ctx, int level, int ctype); +static int comp_rfc1952_init(struct comp_ctx **comp_ctx, int level, int ctype); static int comp_rfc195x_add_data(struct comp_ctx *comp_ctx, const char *in_data, int in_len, struct buffer *out); static int comp_rfc195x_flush(struct comp_ctx *comp_ctx, struct buffer *out); static int comp_rfc195x_finish(struct comp_ctx *comp_ctx, struct buffer *out); @@ -87,9 +87,9 @@ static int decomp_uslz_end(struct decomp_ctx **decomp_ctx); #elif defined(USE_ZLIB) -static int comp_gzip_init(struct comp_ctx **comp_ctx, int level); -static int comp_raw_def_init(struct comp_ctx **comp_ctx, int level); -static int comp_deflate_init(struct comp_ctx **comp_ctx, int level); +static int comp_gzip_init(struct comp_ctx **comp_ctx, int level, int ctype); +static int comp_raw_def_init(struct comp_ctx **comp_ctx, int level, int ctype); +static int comp_deflate_init(struct comp_ctx **comp_ctx, int level, int ctype); static int comp_deflate_add_data(struct comp_ctx *comp_ctx, const char *in_data, int in_len, struct buffer *out); static int comp_deflate_flush(struct comp_ctx *comp_ctx, struct buffer *out); static int comp_deflate_finish(struct comp_ctx *comp_ctx, struct buffer *out); @@ -288,7 +288,7 @@ static inline __maybe_unused int deinit_decomp_ctx(struct decomp_ctx **decomp_ct /* * Init the identity algorithm */ -static int comp_identity_init(struct comp_ctx **comp_ctx, int level) +static int comp_identity_init(struct comp_ctx **comp_ctx, int level, int ctype) { return 0; } @@ -333,34 +333,57 @@ static int comp_identity_end(struct comp_ctx **comp_ctx) #ifdef USE_SLZ +/* Selects SLZ's content type matching on the stream of + * after its init. libslz has no dedicated macro for the content types, but + * they come right after SLZ_MAX_OVERHEAD(), which is thus used to detect an + * older external libslz that doesn't support them, in which case the default + * encoding is always used. + */ +static inline void comp_slz_set_ctype(struct comp_ctx *comp_ctx, int ctype) +{ +#if defined(SLZ_MAX_OVERHEAD) + if (ctype == COMP_CT_TEXT) + slz_set_content_type(&comp_ctx->strm, SLZ_CT_TEXT); +#endif +} + /* SLZ's gzip format (RFC1952). Returns < 0 on error. */ -static int comp_rfc1952_init(struct comp_ctx **comp_ctx, int level) +static int comp_rfc1952_init(struct comp_ctx **comp_ctx, int level, int ctype) { if (init_comp_ctx(comp_ctx) < 0) return -1; (*comp_ctx)->cur_lvl = level; - return slz_rfc1952_init(&(*comp_ctx)->strm, level); + if (slz_rfc1952_init(&(*comp_ctx)->strm, level) < 0) + return -1; + comp_slz_set_ctype(*comp_ctx, ctype); + return 0; } /* SLZ's raw deflate format (RFC1951). Returns < 0 on error. */ -static int comp_rfc1951_init(struct comp_ctx **comp_ctx, int level) +static int comp_rfc1951_init(struct comp_ctx **comp_ctx, int level, int ctype) { if (init_comp_ctx(comp_ctx) < 0) return -1; (*comp_ctx)->cur_lvl = level; - return slz_rfc1951_init(&(*comp_ctx)->strm, level); + if (slz_rfc1951_init(&(*comp_ctx)->strm, level) < 0) + return -1; + comp_slz_set_ctype(*comp_ctx, ctype); + return 0; } /* SLZ's zlib format (RFC1950). Returns < 0 on error. */ -static int comp_rfc1950_init(struct comp_ctx **comp_ctx, int level) +static int comp_rfc1950_init(struct comp_ctx **comp_ctx, int level, int ctype) { if (init_comp_ctx(comp_ctx) < 0) return -1; (*comp_ctx)->cur_lvl = level; - return slz_rfc1950_init(&(*comp_ctx)->strm, level); + if (slz_rfc1950_init(&(*comp_ctx)->strm, level) < 0) + return -1; + comp_slz_set_ctype(*comp_ctx, ctype); + return 0; } /* Return the size of consumed data. The caller is responsible for guaranteeing @@ -657,7 +680,7 @@ static void free_zlib(void *opaque, void *ptr) /************************** **** gzip algorithm **** ***************************/ -static int comp_gzip_init(struct comp_ctx **comp_ctx, int level) +static int comp_gzip_init(struct comp_ctx **comp_ctx, int level, int ctype) { z_stream *strm; @@ -677,7 +700,7 @@ static int comp_gzip_init(struct comp_ctx **comp_ctx, int level) } /* Raw deflate algorithm */ -static int comp_raw_def_init(struct comp_ctx **comp_ctx, int level) +static int comp_raw_def_init(struct comp_ctx **comp_ctx, int level, int ctype) { z_stream *strm; @@ -699,7 +722,7 @@ static int comp_raw_def_init(struct comp_ctx **comp_ctx, int level) **** Deflate algorithm **** ***************************/ -static int comp_deflate_init(struct comp_ctx **comp_ctx, int level) +static int comp_deflate_init(struct comp_ctx **comp_ctx, int level, int ctype) { z_stream *strm; diff --git a/src/flt_http_comp.c b/src/flt_http_comp.c index 3a3a6c60d..f76fe212e 100644 --- a/src/flt_http_comp.c +++ b/src/flt_http_comp.c @@ -127,6 +127,7 @@ comp_prepare_compress_request(struct comp_state *st, struct stream *s, struct ht struct http_hdr_ctx ctx; struct comp_type *comp_type; unsigned int comp_minsize = 0; + int ctype = COMP_CT_GENERIC; ctx.blk = NULL; /* Already compressed, don't bother */ @@ -152,6 +153,10 @@ comp_prepare_compress_request(struct comp_state *st, struct stream *s, struct ht */ ctx.blk = NULL; if (http_find_header(htx, ist("Content-Type"), &ctx, 1)) { + /* text contents may be encoded differently by the algorithm */ + if (ctx.value.len >= 5 && strncasecmp("text/", ctx.value.ptr, 5) == 0) + ctype = COMP_CT_TEXT; + if ((s->be->comp && (comp_type = s->be->comp->types_req)) || (strm_fe(s)->comp && (comp_type = strm_fe(s)->comp->types_req))) { for (; comp_type; comp_type = comp_type->next) { @@ -190,7 +195,7 @@ comp_prepare_compress_request(struct comp_state *st, struct stream *s, struct ht goto fail; /* no algo selected: nothing to do */ /* initialize compression */ - if (st->comp_algo->init(&st->comp_ctx, global.tune.comp_maxlevel) < 0) + if (st->comp_algo->init(&st->comp_ctx, global.tune.comp_maxlevel, ctype) < 0) goto fail; return; @@ -683,6 +688,7 @@ select_compression_response_header(struct comp_state *st, struct stream *s, stru struct http_hdr_ctx ctx; struct comp_type *comp_type; unsigned int comp_minsize = 0; + int ctype = COMP_CT_GENERIC; /* no common compression algorithm was found in request header */ if (st->comp_algo == NULL) @@ -754,6 +760,10 @@ select_compression_response_header(struct comp_state *st, struct stream *s, stru if (ctx.value.len >= 9 && strncasecmp("multipart", ctx.value.ptr, 9) == 0) goto fail; + /* text contents may be encoded differently by the algorithm */ + if (ctx.value.len >= 5 && strncasecmp("text/", ctx.value.ptr, 5) == 0) + ctype = COMP_CT_TEXT; + if ((s->be->comp && (comp_type = s->be->comp->types_res)) || (strm_fe(s)->comp && (comp_type = strm_fe(s)->comp->types_res))) { for (; comp_type; comp_type = comp_type->next) { @@ -783,7 +793,7 @@ select_compression_response_header(struct comp_state *st, struct stream *s, stru goto fail; /* initialize compression */ - if (st->comp_algo->init(&st->comp_ctx, global.tune.comp_maxlevel) < 0) + if (st->comp_algo->init(&st->comp_ctx, global.tune.comp_maxlevel, ctype) < 0) goto fail; msg->flags |= HTTP_MSGF_COMPRESSING; return 1; @@ -912,7 +922,7 @@ parse_compression_options(char **args, int section, struct proxy *proxy, goto end; } - if (proxy->comp->algos_res->init(&ctx, 9) == 0) + if (proxy->comp->algos_res->init(&ctx, 9, COMP_CT_GENERIC) == 0) proxy->comp->algos_res->end(&ctx); else { memprintf(err, "'%s' : Can't init '%s' algorithm.", @@ -941,7 +951,7 @@ parse_compression_options(char **args, int section, struct proxy *proxy, goto end; } - if (proxy->comp->algo_req->init(&ctx, 9) == 0) + if (proxy->comp->algo_req->init(&ctx, 9, COMP_CT_GENERIC) == 0) proxy->comp->algo_req->end(&ctx); else { memprintf(err, "'%s' : Can't init '%s' algorithm.",