From: Willy Tarreau Date: Wed, 21 Oct 2026 20:43:44 +0000 (+0200) Subject: MINOR: sample: add a "tr" converter X-Git-Url: https://git.kaiwu.me/web/%22data:,/doc/static/gitweb.js?a=commitdiff_plain;h=6bf3333f7abe7cd8053022d48fa789ade0fb9788;p=haproxy.git MINOR: sample: add a "tr" converter This converter works like the unix "tr" utility: it translates or deletes characters from the input sample. As on the command line, it takes 2 or 3 arguments: - optional flags, which must start with '-': 'd' to delete characters instead of translating them, 'c' to use the complement of the first set ; - the set of characters to match ; - the replacement set, mandatory without 'd' and forbidden with it. For example, tr('a-z','A-Z') upper-cases the input, and tr(-dc,'[:digit:]') only keeps the digits. A set starting with '-' as the first argument must be written with a backslash ('\-'), except for "-" alone, again as with the unix utility. Sets support ranges ("a-z"), the POSIX character classes ("[:alpha:]" etc, ASCII only), and the usual escapes (\NNN octal, \n, \t, \\ etc). When the replacement set is shorter than the first one, its last char is repeated. When complemented, the chars are enumerated in ascending order. The sets are resolved at config parsing time into a 256-byte table stored in the second argument, and the flags are stored as an integer in the first one, so that the runtime processing only consists in one table lookup per byte. The deletion loop is branchless so that it remains fast on large contents such as a request body. It relies on smp_prepare_out() so that read-only inputs, which are the most common ones, are not duplicated first but directly transformed into a trash chunk. The converter works on binary contents and preserves the input sample's type. The set parsing functions tr_set_init(), tr_read_char() and tr_set_next() are exported, with struct tr_set_ctx in sample-t.h, so that other converters may use the same set syntax. The doc and a reg-test were added. --- diff --git a/doc/configuration.txt b/doc/configuration.txt index ede927fc0..1eea8c3e9 100644 --- a/doc/configuration.txt +++ b/doc/configuration.txt @@ -22423,6 +22423,7 @@ tcp.options_list binary binary tcp.seq binary integer tcp.src binary integer tcp.win binary integer +tr([-flags,]set1[,set2]) binary binary ub64dec string string ub64enc string string ungrpc(field_number[,field_type]) binary binary / int @@ -24633,6 +24634,70 @@ tcp.win 16-bit unsigned quantity, without applying the window scaling factor. See also "fc_saved_syn", "tcp-ss", and "ip.data". +tr([-,][,]) + Translates or deletes characters from the input sample, similarly to the + unix "tr" utility. It works on binary contents (e.g. a request body) as well + as strings, and preserves the input sample's type. The sets are resolved + once at configuration parsing time, so that the processing is very fast. + As on the unix command line, the flags are optional and come first, and + are distinguished from by their leading '-'. + + is a '-' followed by one or more of the following letters: + - 'd': delete the characters from instead of translating + them. must not be set in this case. + - 'c': use the complement of , i.e. all the characters that + are not in , in ascending order. + + is the set of characters to translate or delete. Since a first + argument starting with '-' designates flags, a starting + with '-' must be written with a backslash (e.g. '\-a'), unless it + is only "-" and flags are not used. + + is the set of replacement characters, which is mandatory when the + 'd' flag is not set. Each character from is replaced with + the character at the same position in . If is shorter + than , its last character is repeated as needed. If a + character appears multiple times in , the last mapping wins. + + As with the unix "tr" utility, a set is a list of characters which may + contain: + - ranges of characters in ascending order, such as "a-z" or "0-9". + A '-' at the beginning or at the end of a set is a regular character ; + + - character classes "[:alnum:]", "[:alpha:]", "[:blank:]", "[:cntrl:]", + "[:digit:]", "[:graph:]", "[:lower:]", "[:print:]", "[:punct:]", + "[:space:]", "[:upper:]", "[:xdigit:]", which only cover ASCII + characters, and whose characters are enumerated in ascending order + (e.g. "[:lower:]" may be translated to "[:upper:]") ; + + - backslash-escaped characters: "\NNN" for the character whose octal + value is NNN (1 to 3 digits), "\a", "\b", "\f", "\n", "\r", "\t", + "\v", and "\" followed by any other character stands for this + character (e.g. "\\" for a backslash, or "\-" for a dash). + + Note that, contrary to regular expressions, brackets do not enclose sets: + "[a-z]" designates the 28 characters '[', 'a' to 'z', and ']'. It only + works when translating to "[A-Z]" because '[' and ']' are then mapped to + themselves. + + Since backslashes and some characters such as commas are also processed by + the configuration and argument parsers, it is recommended to enclose sets + within single quotes, and to double the backslashes when the whole + expression is enclosed in double quotes, as in the examples below. + + Example: + # upper-case the "user" query string parameter + http-request set-var(txn.user) "url_param(user),tr('a-z','A-Z')" + + # replace all non-alphanumeric chars with underscores + http-request set-var(txn.id) "req.hdr(x-id),tr(-c,'[:alnum:]',_)" + + # only keep digits and spaces from the request body + http-request set-var(txn.digits) "req.body,tr(-dc,'[:digit:] ')" + + # delete control characters (0 to 31 and 127) + http-request set-var(txn.text) "req.body,tr(-d,'\\0-\\37\\177')" + ub64dec This converter is the base64url variant of b64dec converter. base64url encoding is the "URL and Filename Safe Alphabet" variant of base64 encoding. diff --git a/include/haproxy/sample-t.h b/include/haproxy/sample-t.h index a7c16c621..631959558 100644 --- a/include/haproxy/sample-t.h +++ b/include/haproxy/sample-t.h @@ -279,6 +279,16 @@ struct sample_conv { struct thread_exec_ctx exec_ctx; /* execution context */ }; +/* context used to walk over the characters of a set definition using the + * syntax of the "tr" converter (see tr_set_init()). + */ +struct tr_set_ctx { + const char *ptr; /* next char to parse */ + const char *end; /* end of the set definition */ + int (*cls)(int c); /* non-NULL when walking over a character class */ + int cur, last; /* next and last chars of the current range/class */ +}; + /* sample conversion expression */ struct sample_conv_expr { struct list list; /* member of a sample_expr */ diff --git a/include/haproxy/sample.h b/include/haproxy/sample.h index c0baaca04..e33049c3f 100644 --- a/include/haproxy/sample.h +++ b/include/haproxy/sample.h @@ -64,6 +64,9 @@ int c_none(struct sample *smp); int c_pseudo(struct sample *smp); int smp_dup(struct sample *smp); int smp_prepare_out(struct sample *smp); +void tr_set_init(struct tr_set_ctx *ctx, const struct buffer *set); +int tr_read_char(struct tr_set_ctx *ctx); +int tr_set_next(struct tr_set_ctx *ctx, char **err); int sample_check_arg_base64(struct arg *arg, char **err); /* diff --git a/reg-tests/converter/tr.vtc b/reg-tests/converter/tr.vtc new file mode 100644 index 000000000..a544f1e1f --- /dev/null +++ b/reg-tests/converter/tr.vtc @@ -0,0 +1,89 @@ +varnishtest "tr converter test" + +feature ignore_unknown_macro + +haproxy h1 -conf { + global + .if feature(THREAD) + thread-groups 1 + .endif + + defaults + mode http + timeout connect "${HAPROXY_TEST_TIMEOUT-5s}" + timeout client "${HAPROXY_TEST_TIMEOUT-5s}" + timeout server "${HAPROXY_TEST_TIMEOUT-5s}" + + frontend fe + bind "fd@${fe}" + + http-request set-var(txn.in) str('Hello World 123') + http-request set-var(txn.bin) bin(78310032ff33e979) + http-request return status 200 + http-after-response set-header x-upper "%[var(txn.in),tr(a-z,A-Z)]" + http-after-response set-header x-class "%[var(txn.in),tr('[:lower:]','[:upper:]')]" + http-after-response set-header x-del "%[var(txn.in),tr(-d,aeiou)]" + http-after-response set-header x-delc "%[var(txn.in),tr(-dc,a-z)]" + http-after-response set-header x-compl "%[var(txn.in),tr(-c,a-zA-Z,_)]" + http-after-response set-header x-short "%[str(aabbcc),tr(abc,xy)]" + http-after-response set-header x-dash "%[str(a-1-b),tr(-d,0-9-)]" + http-after-response set-header x-oct "%[var(txn.in),tr(-d,'\\040')]" + http-after-response set-header x-bksl "%[str('a\\\\b'),tr(-d,'\\\\')]" + http-after-response set-header x-octr "%[str(abc),tr(a-c,'\\101-C')]" + http-after-response set-header x-empty "%[var(txn.in),tr(-dc,'')]" + http-after-response set-header x-bin1 "%[var(txn.bin),tr(-dc,'[:digit:]')]" + http-after-response set-header x-bin2 "%[var(txn.bin),tr(-cd,'\\0-\\177'),hex]" + http-after-response set-header x-bin3 "%[var(txn.bin),tr(-c,'\\1-\\177',.),hex]" + http-after-response set-header x-rw1 "%[var(txn.in),tr(a-z,A-Z),tr(-d,'O ')]" + http-after-response set-header x-rw2 "%[var(txn.in),lower,tr(a-z0-9,A-Z.)]" + http-after-response set-header x-dash1 "%[str(a-b-c),tr(-,_)]" + http-after-response set-header x-dash2 "%[str(a-b-c),tr('\\-a',+A)]" + http-after-response set-header x-dash3 "%[str(a-b-c),tr(-d,'\\-a')]" + http-after-response set-header x-rw3 "%[var(txn.bin),tr(-c,'\\1-\\177',.),tr(-d,.),hex]" +} -start + +# flags must start with '-', so that a third set is rejected +haproxy h2 -conf-BAD {too many sets} { + defaults + mode http + + frontend fe + bind "fd@${fe2}" + http-request set-var(txn.x) str(abc),tr(d,a-z,A-Z) +} + +# unknown flag +haproxy h3 -conf-BAD {invalid flag 'a'} { + defaults + mode http + + frontend fe + bind "fd@${fe3}" + http-request set-var(txn.x) str(abc),tr(-a,b) +} + +client c1 -connect ${h1_fe_sock} { + txreq -url "/" + rxresp + expect resp.status == 200 + expect resp.http.x-upper == "HELLO WORLD 123" + expect resp.http.x-class == "HELLO WORLD 123" + expect resp.http.x-del == "Hll Wrld 123" + expect resp.http.x-delc == "elloorld" + expect resp.http.x-compl == "Hello_World____" + expect resp.http.x-short == "xxyyyy" + expect resp.http.x-dash == "ab" + expect resp.http.x-oct == "HelloWorld123" + expect resp.http.x-bksl == "ab" + expect resp.http.x-octr == "ABC" + expect resp.http.x-empty == "" + expect resp.http.x-bin1 == "123" + expect resp.http.x-bin2 == "783100323379" + expect resp.http.x-bin3 == "78312E322E332E79" + expect resp.http.x-rw1 == "HELLWRLD123" + expect resp.http.x-rw2 == "HELLO WORLD ..." + expect resp.http.x-dash1 == "a_b_c" + expect resp.http.x-dash2 == "A+b+c" + expect resp.http.x-dash3 == "bc" + expect resp.http.x-rw3 == "7831323379" +} -run diff --git a/src/sample.c b/src/sample.c index bf5bc34a4..eddef4be3 100644 --- a/src/sample.c +++ b/src/sample.c @@ -4817,6 +4817,303 @@ static int sample_conv_rtrim(const struct arg *arg_p, struct sample *smp, void * return 1; } +/* flags for the "tr" converter, stored as an integer in its first argument */ +#define TR_F_DELETE 0x01 /* 'd': delete chars from the set */ +#define TR_F_COMPLEMENT 0x02 /* 'c': use the complement of the set */ + +/* character classes supported by the "tr" converter, ASCII only */ +static const struct { + const char *name; + int (*fct)(int c); +} tr_classes[] = { + { "alnum", isalnum }, { "alpha", isalpha }, { "blank", isblank }, + { "cntrl", iscntrl }, { "digit", isdigit }, { "graph", isgraph }, + { "lower", islower }, { "print", isprint }, { "punct", ispunct }, + { "space", isspace }, { "upper", isupper }, { "xdigit", isxdigit }, +}; + +/* Prepares to walk over the characters of the "tr" set definition + * using tr_set_next(). + */ +void tr_set_init(struct tr_set_ctx *ctx, const struct buffer *set) +{ + ctx->ptr = set->area; + ctx->end = set->area + set->data; + ctx->cls = NULL; + ctx->cur = 1; + ctx->last = 0; +} + +/* Reads one possibly escaped character from , which must not be empty, + * and returns it. Supported escapes are the same as for the unix "tr" utility: + * octal values of up to 3 digits (\NNN), \a, \b, \f, \n, \r, \t, \v, and any + * other char preceded by a backslash stands for itself (e.g. "\\" or "\-"). + */ +int tr_read_char(struct tr_set_ctx *ctx) +{ + const char *p = ctx->ptr; + int c = (uchar)*p++; + int i; + + if (c == '\\' && p < ctx->end) { + c = (uchar)*p++; + if (c >= '0' && c <= '7') { + c -= '0'; + for (i = 1; i < 3 && p < ctx->end && *p >= '0' && *p <= '7' && + c * 8 + (*p - '0') <= 255; i++) + c = c * 8 + (*p++ - '0'); + } + else switch (c) { + case 'a': c = '\a'; break; + case 'b': c = '\b'; break; + case 'f': c = '\f'; break; + case 'n': c = '\n'; break; + case 'r': c = '\r'; break; + case 't': c = '\t'; break; + case 'v': c = '\v'; break; + } + } + ctx->ptr = p; + return c; +} + +/* Returns the next character of the "tr" set being walked over in , in + * the order they appear, with ranges ("a-z") and classes ("[:alpha:]") + * expanded in ascending order. Returns -1 once the end is reached, or -2 on + * error with filled. + */ +int tr_set_next(struct tr_set_ctx *ctx, char **err) +{ + const char *p; + uint i; + int c; + + while (1) { + /* continue to walk over the current range or class first */ + while (ctx->cur <= ctx->last) { + c = ctx->cur++; + if (!ctx->cls || (c < 0x80 && ctx->cls(c))) + return c; + } + + if (ctx->ptr >= ctx->end) + return -1; + + if (ctx->end - ctx->ptr >= 4 && ctx->ptr[0] == '[' && ctx->ptr[1] == ':') { + /* possibly a "[:class:]" */ + for (p = ctx->ptr + 2; p + 1 < ctx->end; p++) + if (p[0] == ':' && p[1] == ']') + break; + + if (p + 1 < ctx->end) { + for (i = 0; i < sizeof(tr_classes) / sizeof(*tr_classes); i++) { + if (strlen(tr_classes[i].name) == (size_t)(p - (ctx->ptr + 2)) && + memcmp(tr_classes[i].name, ctx->ptr + 2, p - (ctx->ptr + 2)) == 0) + break; + } + + if (i == sizeof(tr_classes) / sizeof(*tr_classes)) { + memprintf(err, "unknown character class '%.*s'", + (int)(p + 2 - ctx->ptr), ctx->ptr); + return -2; + } + ctx->cls = tr_classes[i].fct; + ctx->cur = 0; + ctx->last = 0x7f; + ctx->ptr = p + 2; + continue; + } + } + + p = ctx->ptr; + c = tr_read_char(ctx); + if (ctx->end - ctx->ptr >= 2 && *ctx->ptr == '-') { + /* range: "-" is not the last char */ + ctx->ptr++; + ctx->last = tr_read_char(ctx); + if (ctx->last < c) { + memprintf(err, "range '%.*s' is in reverse order", + (int)(ctx->ptr - p), p); + return -2; + } + ctx->cls = NULL; + ctx->cur = c + 1; + } + return c; + } +} + +/* Checks the "tr" converter's arguments and turns them into their final form. + * As with the unix "tr" utility, the arguments are an optional string of flags + * starting with '-', the set of chars to match, and the replacement set, which + * is mandatory unless the 'd' flag is set. A lone '-' as the first argument + * is a set. Upon success, arg[0] becomes an integer carrying the TR_F_* flags, + * arg[1] becomes a 256-byte table indexed by input chars, and arg[2] is + * released. In delete mode, the table indicates which chars to keep (1) or + * delete (0), otherwise it indicates the output char. + */ +static int sample_conv_tr_check(struct arg *args, struct sample_conv *conv, + const char *file, int line, char **err) +{ + struct tr_set_ctx set1, set2; + const struct buffer *s1, *s2; + uchar in_set1[256]; + uchar *table = NULL; + int flags = 0; + int c, r, last_r; + size_t i; + + if (args[0].data.str.data > 1 && args[0].data.str.area[0] == '-') { + for (i = 1; i < args[0].data.str.data; i++) { + if (args[0].data.str.area[i] == 'd') + flags |= TR_F_DELETE; + else if (args[0].data.str.area[i] == 'c') + flags |= TR_F_COMPLEMENT; + else { + memprintf(err, "invalid flag '%c' in '%.*s', only 'c' and 'd' are supported" + " (a set starting with '-' must be written '\\-...')", + args[0].data.str.area[i], (int)args[0].data.str.data, + args[0].data.str.area); + goto fail; + } + } + s1 = &args[1].data.str; + s2 = (args[2].type == ARGT_STR) ? &args[2].data.str : NULL; + } + else if (args[2].type == ARGT_STR) { + memprintf(err, "too many sets, flags must start with '-' (e.g. '-d')"); + goto fail; + } + else { + s1 = &args[0].data.str; + s2 = &args[1].data.str; + } + + if (flags & TR_F_DELETE) { + if (s2 && s2->data) { + memprintf(err, "a replacement set is not supported with flag 'd'"); + goto fail; + } + } + else if (!s2 || !s2->data) { + memprintf(err, "a non-empty replacement set is required without flag 'd'"); + goto fail; + } + + table = malloc(256); + if (!table) { + memprintf(err, "out of memory"); + goto fail; + } + + memset(in_set1, 0, sizeof(in_set1)); + tr_set_init(&set1, s1); + while ((c = tr_set_next(&set1, err)) >= 0) + in_set1[c] = 1; + + if (c == -2) + goto fail; + + if (flags & TR_F_DELETE) { + for (c = 0; c < 256; c++) + table[c] = !(in_set1[c] ^ !!(flags & TR_F_COMPLEMENT)); + } + else { + /* chars from set1 are mapped to the char at the same position + * in set2, whose last char is repeated as needed. The set1 chars + * are walked over in ascending order when complemented. + */ + for (c = 0; c < 256; c++) + table[c] = c; + + tr_set_init(&set1, s1); + tr_set_init(&set2, s2); + last_r = -1; + c = -1; + while (1) { + if (flags & TR_F_COMPLEMENT) { + while (++c < 256 && in_set1[c]) + ; + if (c == 256) + break; + } + else if ((c = tr_set_next(&set1, err)) < 0) + break; /* no error possible, it was already parsed */ + + r = tr_set_next(&set2, err); + if (r == -2) + goto fail; + if (r == -1) + r = last_r; /* set2 is not empty, last_r is valid */ + last_r = r; + table[c] = r; + } + + /* check the remaining part of set2 for errors */ + while ((r = tr_set_next(&set2, err)) >= 0) + ; + if (r == -2) + goto fail; + } + + chunk_destroy(&args[0].data.str); + args[0].type = ARGT_SINT; + args[0].data.sint = flags; + + if (args[2].type == ARGT_STR) { + chunk_destroy(&args[2].data.str); + args[2].type = ARGT_STOP; + } + + chunk_destroy(&args[1].data.str); + args[1].data.str.area = (char *)table; + args[1].data.str.size = 256; + args[1].data.str.data = 256; + return 1; + + fail: + free(table); + return 0; +} + +/* This sample function works like the unix "tr" utility: depending on the + * flags in arg[0], it either deletes chars or translates them, using the + * 256-byte table prepared in arg[1] by sample_conv_tr_check(). The sample + * type is preserved. + */ +static int sample_conv_tr(const struct arg *arg_p, struct sample *smp, void *private) +{ + const uchar *table = (const uchar *)arg_p[1].data.str.area; + const uchar *src = (const uchar *)smp->data.u.str.area; + uchar *dst; + size_t in, out, len; + uchar c; + + /* may alias , we never write ahead of where we read */ + if (!smp_prepare_out(smp)) + return 0; + + dst = (uchar *)smp->data.u.str.area; + len = smp->data.u.str.data; + + if (arg_p[0].data.sint & TR_F_DELETE) { + /* always copy the char but only advance over kept ones, this + * avoids unpredictable branches. + */ + for (in = out = 0; in < len; in++) { + c = src[in]; + dst[out] = c; + out += table[c]; + } + smp->data.u.str.data = out; + } + else { + for (in = 0; in < len; in++) + dst[in] = table[src[in]]; + } + return 1; +} + /* This function checks the "json_query" converter's arguments. */ static int sample_check_json_query(struct arg *arg, struct sample_conv *conv, const char *file, int line, char **err) @@ -6218,6 +6515,7 @@ static struct sample_conv_kw_list sample_conv_kws = {ILH, { { "cut_crlf", sample_conv_cut_crlf, 0, NULL, SMP_T_STR, SMP_T_STR }, { "ltrim", sample_conv_ltrim, ARG1(1,STR), NULL, SMP_T_STR, SMP_T_STR }, { "rtrim", sample_conv_rtrim, ARG1(1,STR), NULL, SMP_T_STR, SMP_T_STR }, + { "tr", sample_conv_tr, ARG3(2,STR,STR,STR), sample_conv_tr_check, SMP_T_BIN, SMP_T_BIN }, { "json_query", sample_conv_json_query, ARG2(1,STR,STR), sample_check_json_query , SMP_T_STR, SMP_T_ANY }, #ifdef USE_OPENSSL