Incomplete numerical values (e.g.: '%B' when logasap is used),
which are normally prefixed with '+' without encoding, will be
encoded as-is. Also, '+E' option will be ignored.
+ As described in section 8.6, characters outside the 32..126 range
+ are escaped one byte at a time, using the JSON escape sequences,
+ so that the output remains pure printable ASCII whatever the
+ input. This includes valid UTF-8 sequences, meaning that non-ASCII
+ text will appear as its individual bytes once decoded (e.g.
+ U+00E9, encoded as 0xC3 0xA9, is logged as "\u00C3\u00A9"). See
+ "utf8" below to preserve such text.
+ * utf8: combined with "json", permits to let valid multi-byte UTF-8 sequences
+ pass unescaped. The C1 control characters (U+0080 to U+009F) are
+ still escaped, represented by their code point. Invalid sequences are
+ escaped one byte at a time so that the output always remains valid
+ UTF-8. Note that since valid UTF-8 is emitted as-is, a client may use
+ some special characters to alter the rendering of a log line on a
+ terminal or in a log viewer, such as direction changes, zero-width
+ characters, or line and paragraph separators. Thus it is recommended
+ to only use it for contents coming from trusted sources, or when logs
+ are not meant to be displayed on a terminal. It has no effect without
+ "json".
* cbor: automatically encode value in CBOR format
(when set globally, only named logformat items are considered)
By default, cbor encoded data is represented in HEX form so
log-format-sd %{+Q,+E}o\ [exampleSDID@1234\ header=%[capture.req.hdr(0)]]
log-format "%{+json}o %(request)r %(custom_expr)[str(custom)]"
+ log-format "%{+json,+utf8}o %(request)r %(custom_expr)[str(custom)]"
log-format "%{+cbor}o %(request)r %(custom_expr)[str(custom)]"
Please refer to the table below for currently defined aliases :
digits, or properly delimited binary blocks that will not be sent to a terminal
but will be consumed by a CBOR parser.
+The json encoding uses the JSON escape sequences instead, and the +utf8 option
+also lets valid multi-byte UTF-8 sequences pass as-is (see the "utf8" flag in
+section 8.2.6).
+
Note that the space character (' ') is not encoded in headers, which can cause
issues for tools relying on space count to locate fields. A typical header
containing spaces is "User-Agent".
# JSON encoding has no reason to be more permissive than the plain one.
# Escaping them all also makes the output valid UTF-8, which RFC 8259 #8.1
# requires, without having to guess what the logged bytes represent: a header
-# value is arbitrary bytes, not text.
+# value is arbitrary bytes, not text. The +utf8 option bypasses this, letting
+# valid multi-byte UTF-8 sequences pass as-is, and still escapes all other
+# bytes above 126 one at a time so that the output remains valid UTF-8.
haproxy h1 -conf {
global
size 16384
format raw
+ ring rawring
+ size 16384
+ format raw
+
defaults
mode http
timeout connect "${HAPROXY_TEST_TIMEOUT-5s}"
log ring@myring local0
log-format "{\"hdr\":%{+json}[capture.req.hdr(0)],\"bin\":%{+json}[str(Gw==),b64dec]}"
http-request return status 200
+
+ frontend fe_raw
+ bind "fd@${fe_raw}"
+ capture request header x len 64
+ log ring@rawring local0
+ log-format "%{+json,+utf8}o %(hdr)[capture.req.hdr(0)] %(bin)[str(w6nCm+KC),b64dec]"
+ http-request return status 200
} -start
# an ESC, which has no two-character form, a tab, which has one, a double quote
expect resp.status == 200
} -run
+# in raw mode: a valid UTF-8 sequence, a lone continuation byte, an overlong
+# form, an encoded surrogate, a C1 control character, a 4-byte sequence, one
+# beyond U+10FFFF and a sequence truncated by the end of the value
+client c3 -connect ${h1_fe_raw_sock} {
+ txreq -url "/raw" -hdr "x: A\xc3\xa9B\xaaC\xc0\x81D\xed\xa0\x80E\xc2\x9bF\xf0\x9f\x98\x80G\xf4\x90\x80\x80H\xe2\x82"
+ rxresp
+ expect resp.status == 200
+} -run
+
delay 0.2
# the ESC comes out as the six-character form and the tab as the short one, in
send "show events myring -s"
expect !~ "\\?"
}
+
+# in raw mode, the valid UTF-8 sequences are emitted as-is, the C1 control
+# character as its code point's escape sequence, and all other bytes above the
+# printable range get one escape per byte, so that the output remains valid
+# UTF-8
+haproxy h1 -cli {
+ send "show events rawring"
+ expect ~ "\"hdr\": \"A\xc3\xa9B\\\\u00AAC\\\\u00C0\\\\u0081D\\\\u00ED\\\\u00A0\\\\u0080E\\\\u009BF\xf0\x9f\x98\x80G\\\\u00F4\\\\u0090\\\\u0080\\\\u0080H\\\\u00E2\\\\u0082\""
+}
+
+# same in the binary sample path, where the truncated sequence ends the sample
+haproxy h1 -cli {
+ send "show events rawring"
+ expect ~ "\"bin\": \"\xc3\xa9\\\\u009B\\\\u00E2\\\\u0082\""
+}
{ "X", LOG_OPT_HEXA },
{ "E", LOG_OPT_ESC },
{ "bin", LOG_OPT_BIN },
+ { "utf8", LOG_OPT_UTF8 },
{ "json", LOG_OPT_ENCODE_JSON },
{ "cbor", LOG_OPT_ENCODE_CBOR },
{ 0, 0 }
int options; /* LOG_OPT_* options */
int typecast; /* same as logformat_node->typecast */
int in_text; /* inside variable-length text */
+ int utf8_skip; /* JSON: continuation bytes of an already emitted UTF-8 sequence */
};
static THREAD_LOCAL char lf_buildbuf[256]; /* fixed size buffer for building small strings */
return start;
}
+/* Checks whether <s> starts with a multi-byte UTF-8 sequence which is valid as
+ * per RFC 3629 and does not span more than <len> bytes: no overlong form, no
+ * surrogate (U+D800..U+Dfff), nothing beyond U+10FFFF (U+FFFE and U+FFFF,
+ * which utf8_next() rejects as well, are simply escaped). The sequence may be
+ * terminated by a NUL byte before <len> bytes, since it is not a continuation
+ * byte. On success the code point is stored into <c> and the sequence length
+ * (2 to 4) is returned, otherwise zero is returned.
+ */
+static inline int _lf_json_utf8_len(const char *s, size_t len, unsigned int *c)
+{
+ unsigned char ret;
+
+ ret = utf8_next(s, MIN(len, 4), c);
+ if (utf8_return_code(ret) != UTF8_CODE_OK)
+ return 0;
+
+ ret = utf8_return_length(ret);
+ if (ret < 2 || ret > 4 || *c > 0x10FFFF)
+ return 0;
+
+ return ret;
+}
+
+/* Writes at <start> the valid UTF-8 sequence of <len> bytes found at <seq> and
+ * representing code point <c>, as reported by _lf_json_utf8_len(). It is
+ * copied as-is, except for the C1 control characters (U+0080 .. U+009F) which,
+ * just like the C0 ones (U+0000 .. U+001F), could be interpreted by a
+ * terminal, and which are written as their \u00XX escape sequence instead.
+ *
+ * The function returns the address of the byte immediately after the last
+ * written byte, or NULL if the whole sequence does not fit before <stop>, so
+ * that a sequence is never truncated. It will not append a terminating NULL
+ * byte.
+ */
+static inline char *_lf_json_utf8_seq(char *start, char *stop,
+ const char *seq, int len, unsigned int c)
+{
+ if (c < 0xa0)
+ return _lf_json_escape_seq(start, stop, c);
+
+ if (start + len > stop)
+ return NULL;
+ memcpy(start, seq, len);
+ return start + len;
+}
+
/* helper function for _lf_encode_bytes() to encode a single byte
* and escape it with <escape> if found in <map> or escape it with
- * '\' if found in json_escape_map
+ * '\' if found in json_escape_map. With LOG_OPT_UTF8, valid multi-byte
+ * UTF-8 sequences are emitted at once when their first byte is met
+ * instead, the following ones being skipped.
*
* The function assumes that at least 1 byte is available for writing
*
const char **pending, uint8_t cbor_string_prefix,
struct lf_buildctx *ctx)
{
+ unsigned int c;
+ int len;
+
+ if (ctx->utf8_skip) {
+ /* already emitted with the sequence's first byte */
+ ctx->utf8_skip--;
+ return start;
+ }
+
if (!ha_bit_test((unsigned char)(*byte), map)) {
if (!ha_bit_test((unsigned char)(*byte), json_escape_map))
*start++ = *byte;
+ else if ((ctx->options & LOG_OPT_UTF8) &&
+ (len = _lf_json_utf8_len(byte, end ? end - byte : 4, &c))) {
+ start = _lf_json_utf8_seq(start, stop, byte, len, c);
+ if (start)
+ ctx->utf8_skip = len - 1;
+ }
else
start = _lf_json_escape_seq(start, stop, *byte);
}
const char **pending, uint8_t cbor_string_prefix,
struct lf_buildctx *ctx);
- if (ctx->options & LOG_OPT_ENCODE_JSON)
+ if (ctx->options & LOG_OPT_ENCODE_JSON) {
encode_byte = _lf_json_escape_byte;
+ ctx->utf8_skip = 0;
+ }
else if (ctx->options & LOG_OPT_ENCODE_CBOR)
encode_byte = _lf_cbor_map_escape_byte;
else if (ctx->options & LOG_OPT_ESC)
/* Same as escape_string() but for a JSON string content: the bytes tagged in
* json_escape_map cannot all be expressed with the two-character form that
- * escape_string() emits, the others need the \u00XX one.
+ * escape_string() emits, the others need the \u00XX one. When <raw> is set,
+ * valid multi-byte UTF-8 sequences are emitted as-is instead (see
+ * _lf_json_utf8_seq()).
*
* Return the address of the \0 character, or NULL on error
*/
static char *_lf_json_escape_string(char *start, char *stop,
- const char *string, const char *string_stop)
+ const char *string, const char *string_stop,
+ int raw)
{
if (start < stop) {
stop--; /* reserve one byte for the final '\0' */
while (start < stop && string < string_stop && *string != '\0') {
- if (!ha_bit_test((unsigned char)(*string), json_escape_map))
- *start++ = *string;
- else {
- char *next = _lf_json_escape_seq(start, stop, *string);
+ unsigned int c;
+ char *next;
+ int len;
+
+ if (!ha_bit_test((unsigned char)(*string), json_escape_map)) {
+ *start++ = *string++;
+ continue;
+ }
- if (!next)
- break; /* does not fit, truncate here */
- start = next;
+ len = raw ? _lf_json_utf8_len(string, string_stop - string, &c) : 0;
+ if (len)
+ next = _lf_json_utf8_seq(start, stop, string, len, c);
+ else {
+ len = 1;
+ next = _lf_json_escape_seq(start, stop, *string);
}
- string++;
+
+ if (!next)
+ break; /* does not fit, truncate here */
+ start = next;
+ string += len;
}
*start = '\0';
return start;
else if (ctx->options & LOG_OPT_ENCODE_JSON) {
char *ret;
- ret = _lf_json_escape_string(dst, dst + size, src, src + len);
+ ret = _lf_json_escape_string(dst, dst + size, src, src + len,
+ !!(ctx->options & LOG_OPT_UTF8));
if (ret == NULL)
return NULL;
len = ret - dst;
* spares us from having to care about the UTF-8 validity that RFC 8259
* #8.1 demands, since a header value is made of arbitrary bytes and not
* of text. On top of that, RFC 8259 #7 mandates escaping the double
- * quote, the backslash and everything below 0x20.
+ * quote, the backslash and everything below 0x20. The raw JSON encoding
+ * is the only exception: its encoders detect the valid multi-byte UTF-8
+ * sequences by themselves and emit them as-is.
*/
memset(json_escape_map, 0, sizeof(json_escape_map));
projects / haproxy.git / commitdiff