From: Aurelien DARRAGON Date: Wed, 12 Aug 2026 09:58:52 +0000 (+0200) Subject: MINOR: (de)compression: add decompression infrastructure X-Git-Tag: v3.5-dev6~53 X-Git-Url: https://git.kaiwu.me/?a=commitdiff_plain;h=545bfa6a6b72eeb85452680ccb89b0c1ee7c4065;p=haproxy.git MINOR: (de)compression: add decompression infrastructure Add basic decompression infrastructure to haproxy compression API, which will be used in upcoming commits to implement TCP and HTTP decompression. For now decompression exclusively leverages uslz (slz decompression API). As such it supports the following decompression options: - uslz-auto: this is automatic format detection from libslz/uslz: it is able to guess if the stream is gzip or zlib (deflate) one, and if it is none of the two, it fallsback on raw-reflate. - gzip: if the stream is valid gzip, it will be decompressed, else the filter will refuse to decompress the stream. - "deflate": if the stream is valid deflate (zlib), it will be decompressed, else the filter will refuse to decompress the stream. - "raw-deflate": the stream will be assumed to be raw deflate (deflate without enveloppe, also known as (rfc1951), the filter will try to decompress it unless the stream is invalid. --- diff --git a/include/haproxy/compression.h b/include/haproxy/compression.h index 851ea2393..eb67c7103 100644 --- a/include/haproxy/compression.h +++ b/include/haproxy/compression.h @@ -24,11 +24,13 @@ #define _HAPROXY_COMP_H #include +#include extern unsigned int compress_min_idle; int comp_append_type(struct comp_type **types, const char *type); int comp_append_algo(struct comp_algo **algos, const char *algo); +int decomp_append_algo(struct decomp_algo **algos, const char *algo); #ifdef USE_ZLIB extern long zlib_used_memory; diff --git a/include/haproxy/decompression-t.h b/include/haproxy/decompression-t.h new file mode 100644 index 000000000..700896359 --- /dev/null +++ b/include/haproxy/decompression-t.h @@ -0,0 +1,102 @@ +/* + * include/haproxy/decompression-t.h + * This file defines everything related to decompression. + * + * Copyright 2026 HAProxy Technologies + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation, version 2.1 + * exclusively. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef _HAPROXY_DECOMP_T_H +#define _HAPROXY_DECOMP_T_H + +#if defined(USE_SLZ) +#ifdef USE_ZLIB +#error "Cannot build with both USE_SLZ and USE_ZLIB at the same time." +#endif +#include +#elif defined(USE_ZLIB) +#include +#endif + +#include + +/* Decompression flags */ + +#define DECOMP_FL_NONE 0x00000000 +#define DECOMP_FL_DIR_REQ 0x00000001 /* decompress requests */ +#define DECOMP_FL_DIR_RES 0x00000002 /* decompress responses */ + +/* Per direction decompression flags */ + +/* DECOMP_DIR_FL_MODE modes are mutually exclusive */ +#define DECOMP_DIR_FL_MODE_MASK 0x0007 +#define DECOMP_DIR_FL_MODE_NONE 0x0001 /* default: do nothing */ +#define DECOMP_DIR_FL_MODE_AUTO 0x0002 /* best effort */ +#define DECOMP_DIR_FL_MODE_ALWAYS 0x0004 /* guarantees the output stream will be decompressed */ + +/* Decompression ctx flags */ + +#define DECOMP_CTX_FL_DONE 0x0001 /* decompression done */ + +/* for either request or response decompression */ +struct decomp_dir { + struct decomp_algo *algos; /* algos available for decompression */ + uint16_t flags; /* DECOMP_DIR_FL_* flags */ +}; +struct decomp { + struct decomp_dir req; + struct decomp_dir res; + unsigned int flags; /* DECOMP_FL_* flags */ +}; + +struct decomp_ctx { + uint16_t flags; /* DECOMP_CTX_FL_* flags */ + char *drain; /* pointer to pending (unconsumed) data in decompression buffer */ + uint32_t drain_len; /* pending bytes in decompression buffer */ +#if defined(USE_SLZ) + struct uslz_stream state; + int fmt; /* SLZ_FMT_* value, if != SLZ_FMT_NONE library will be initialized + * with uslz_init_fmt() and format will be enforced (this disables + * auto-detection) + */ + char out_buf[32768]; /* decompression buffer */ +#endif +}; + +struct decomp_algo { + char *cfg_name; /* config name */ + int cfg_name_len; + + char *ua_name; /* name for the user-agent */ + int ua_name_len; + + int (*init)(struct decomp_ctx **decomp_ctx); + int (*stream_new)(struct decomp_ctx *decomp_ctx); + int (*add_data)(struct decomp_ctx *decomp_ctx, const char *in_data, int in_len); + int (*finish)(struct decomp_ctx *decomp_ctx); + int (*end)(struct decomp_ctx **decomp_ctx); + struct decomp_algo *next; +}; + +#endif /* _HAPROXY_DECOMP_T_H */ + +/* + * Local variables: + * c-indent-level: 8 + * c-basic-offset: 8 + * End: + */ + diff --git a/src/compression.c b/src/compression.c index d09e31251..ee67e74a7 100644 --- a/src/compression.c +++ b/src/compression.c @@ -77,6 +77,13 @@ static int comp_rfc195x_add_data(struct comp_ctx *comp_ctx, const char *in_data, static int comp_rfc195x_flush(struct comp_ctx *comp_ctx, struct buffer *out); static int comp_rfc195x_finish(struct comp_ctx *comp_ctx, struct buffer *out); static int comp_rfc195x_end(struct comp_ctx **comp_ctx); +static int decomp_uslz_init(struct decomp_ctx **decomp_ctx); +static int decomp_uslz_gzip_init(struct decomp_ctx **decomp_ctx); +static int decomp_uslz_deflate_init(struct decomp_ctx **decomp_ctx); +static int decomp_uslz_raw_deflate_init(struct decomp_ctx **decomp_ctx); +static int decomp_uslz_stream_new(struct decomp_ctx *decomp_ctx); +static int decomp_uslz_add_data(struct decomp_ctx *decomp_ctx, const char *in_data, int in_len); +static int decomp_uslz_end(struct decomp_ctx **decomp_ctx); #elif defined(USE_ZLIB) @@ -106,6 +113,18 @@ const struct comp_algo comp_algos[] = { NULL, 0, NULL, 0, NULL , NULL, NULL, NULL, NULL } }; +const struct decomp_algo decomp_algos[] = +{ +#if defined(USE_SLZ) + { "uslz-auto", 9, "uslz-auto", 9, decomp_uslz_init, decomp_uslz_stream_new, decomp_uslz_add_data, NULL, decomp_uslz_end }, + { "deflate", 7, "deflate", 7, decomp_uslz_deflate_init, decomp_uslz_stream_new, decomp_uslz_add_data, NULL, decomp_uslz_end }, + { "raw-deflate", 11, "deflate", 7, decomp_uslz_raw_deflate_init, decomp_uslz_stream_new, decomp_uslz_add_data, NULL, decomp_uslz_end }, + { "gzip", 4, "gzip", 4, decomp_uslz_gzip_init, decomp_uslz_stream_new, decomp_uslz_add_data, NULL, decomp_uslz_end }, +#elif defined(USE_ZLIB) +#endif /* USE_ZLIB */ + { NULL, 0, NULL, 0, NULL , NULL, NULL, NULL, NULL } +}; + /* * Add a content-type in the configuration * Returns 0 in case of success, 1 in case of allocation failure. @@ -155,8 +174,28 @@ int comp_append_algo(struct comp_algo **algos, const char *algo) return -1; } +int decomp_append_algo(struct decomp_algo **algos, const char *algo) +{ + struct decomp_algo *decomp_algo; + int i; + + for (i = 0; decomp_algos[i].cfg_name; i++) { + if (strcmp(algo, decomp_algos[i].cfg_name) == 0) { + decomp_algo = calloc(1, sizeof(*decomp_algo)); + if (!decomp_algo) + return 1; + memmove(decomp_algo, &decomp_algos[i], sizeof(struct decomp_algo)); + decomp_algo->next = *algos; + *algos = decomp_algo; + return 0; + } + } + return -1; +} + #if defined(USE_ZLIB) || defined(USE_SLZ) DECLARE_STATIC_TYPED_POOL(pool_comp_ctx, "comp_ctx", struct comp_ctx); +DECLARE_STATIC_TYPED_POOL(pool_decomp_ctx, "decomp_ctx", struct decomp_ctx); /* * Alloc the comp_ctx @@ -189,6 +228,25 @@ static inline int init_comp_ctx(struct comp_ctx **comp_ctx) return 0; } +/* + * Alloc the decomp_ctx + */ +static inline __maybe_unused int init_decomp_ctx(struct decomp_ctx **decomp_ctx) +{ + *decomp_ctx = pool_alloc(pool_decomp_ctx); + if (*decomp_ctx == NULL) + return -1; + +#if defined(USE_SLZ) + (*decomp_ctx)->fmt = SLZ_FMT_NONE; +#endif + (*decomp_ctx)->drain_len = 0; + (*decomp_ctx)->drain = NULL; + (*decomp_ctx)->flags = 0; + + return 0; +} + /* * Dealloc the comp_ctx */ @@ -206,8 +264,21 @@ static inline int deinit_comp_ctx(struct comp_ctx **comp_ctx) #endif return 0; } -#endif +/* + * Dealloc the decomp_ctx + */ +static inline __maybe_unused int deinit_decomp_ctx(struct decomp_ctx **decomp_ctx) +{ + if (!*decomp_ctx) + return 0; + + pool_free(pool_decomp_ctx, *decomp_ctx); + *decomp_ctx = NULL; + return 0; +} + +#endif /**************************** **** Identity algorithm **** @@ -398,6 +469,101 @@ static int comp_rfc195x_end(struct comp_ctx **comp_ctx) return 0; } +/* SLZ's auto deflate (supports RFC1950, 51 and 52). + * Returns < 0 on error. + */ +static int decomp_uslz_init(struct decomp_ctx **decomp_ctx) +{ + if (init_decomp_ctx(decomp_ctx) < 0) + return -1; + + return 0; +} + +/* only accept zlib deflate (rfc1950 stream) */ +static int decomp_uslz_deflate_init(struct decomp_ctx **decomp_ctx) +{ + if (decomp_uslz_init(decomp_ctx) < 0) + return -1; + + (*decomp_ctx)->fmt = SLZ_FMT_ZLIB; + + return 0; +} + +/* only accept gzip (rfc1952 stream) */ +static int decomp_uslz_gzip_init(struct decomp_ctx **decomp_ctx) +{ + if (decomp_uslz_init(decomp_ctx) < 0) + return -1; + + (*decomp_ctx)->fmt = SLZ_FMT_GZIP; + + return 0; +} + +/* only accept raw deflate stream (rfc1951 stream) */ +static int decomp_uslz_raw_deflate_init(struct decomp_ctx **decomp_ctx) +{ + if (decomp_uslz_init(decomp_ctx) < 0) + return -1; + + (*decomp_ctx)->fmt = SLZ_FMT_DEFLATE; + + return 0; +} + +static int decomp_uslz_stream_new(struct decomp_ctx *decomp_ctx) +{ + if (decomp_ctx->fmt == SLZ_FMT_NONE) { + if (!uslz_init(&decomp_ctx->state, (unsigned char *)decomp_ctx->out_buf, sizeof(decomp_ctx->out_buf))) + return -1; + } + else if (!uslz_init_fmt(&decomp_ctx->state, (unsigned char *)decomp_ctx->out_buf, sizeof(decomp_ctx->out_buf), decomp_ctx->fmt)) + return -1; + + return 0; +} + +/* Return the size of consumed data or -1. The output buffer is unused at this + * point, we only keep a reference to the input data or a copy of them if the + * reference is already used. + */ +static int decomp_uslz_add_data(struct decomp_ctx *decomp_ctx, const char *in_data, int in_len) +{ + unsigned char *decoded = NULL; + long decoded_len = 0; + long consumed = 0; + enum uslz_decode_ret decode_ret; + uint32_t crc; + + decode_ret = uslz_decode(&decomp_ctx->state, (unsigned char *)in_data, in_len, &decoded, &decoded_len, &consumed, &crc); + decomp_ctx->drain = (char *)decoded; + decomp_ctx->drain_len = decoded_len; + switch (decode_ret) { + case USLZ_DECODE_SUCCESS: + case USLZ_DECODE_OUT_OF_SPACE: + if (decode_ret == USLZ_DECODE_SUCCESS) + decomp_ctx->flags |= DECOMP_CTX_FL_DONE; + break; + + case USLZ_DECODE_OUT_OF_DATA: + break; + default: + /* decompression error. TODO: trace the error code and its meaning? */ + return -1; + } + + return consumed; +} + +/* we just need to free the decomp_ctx here, nothing was allocated */ +static int decomp_uslz_end(struct decomp_ctx **decomp_ctx) +{ + deinit_decomp_ctx(decomp_ctx); + return 0; +} + #elif defined(USE_ZLIB) /* ! USE_SLZ */ /*