From: Willy Tarreau Date: Mon, 16 Nov 2026 07:53:51 +0000 (+0100) Subject: IMPORT: slz: update slz to 1.4-dev5-24 X-Git-Url: https://git.kaiwu.me/web/%22data:,/doc/static/gitweb.js?a=commitdiff_plain;h=5ccaa1aa28cb74a5688b8a0c14737748ad6f548d;p=haproxy.git IMPORT: slz: update slz to 1.4-dev5-24 This fixes a risk of oversized output at level 3 or text, includes automatic switching between fixed and text trees at levels >= 2 and with CT_TEXT, and introduces support for a ring for short repetitive events compression (e.g. SSE). --- diff --git a/include/import/slz-tables.h b/include/import/slz-tables.h index 1ab37fd61..f5e8f9a73 100644 --- a/include/import/slz-tables.h +++ b/include/import/slz-tables.h @@ -228,6 +228,57 @@ static const uint32_t text_len[259] = { 0x0a0397, }; +/* number of bits the text tree would save on each literal over the fixed + * tree, see SLZ_AUTO_IN. + */ +static const int8_t lit_tsave[256] = { + -3, -3, -3, -3, -3, -3, -3, -3, -3, -2, 1, -3, -3, -3, -3, -3, + -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, + 3, -2, 1, -3, -3, -3, -2, -1, 0, 0, -2, -2, 1, 1, 1, 0, + 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, -1, -1, 0, -1, -2, + -3, 0, -1, 0, 0, 0, -1, -1, -1, 0, -3, -3, 0, 0, 0, 0, + 0, -3, 0, 0, 0, -1, -2, -2, -2, -2, -3, -2, -2, -2, -3, 0, + -2, 2, 0, 1, 2, 3, 1, 0, 1, 2, -2, -1, 2, 1, 2, 2, + 1, -3, 2, 2, 2, 1, 0, 0, 0, 0, -2, -2, -2, -1, -3, -3, + -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, +}; + +/* number of bits the text tree would save on each match length, and on + * each distance, indexed by the reversed 5-bit fixed code found in bits 5-9 + * of fh_dist_table[], see SLZ_AUTO_IN. + */ +static const int8_t len_tsave[259] = { + 0, 0, 0, 7, 4, 3, 2, 2, 2, 1, 1, 1, 1, 1, 1, 0, + 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, -1, + -1, -1, -1, 0, 0, 0, 0, 0, 0, 0, 0, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, + -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, + -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, + -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, + -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, + -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, + -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, + -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, + -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, + -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, + -3, -3, -2, +}; + +static const int8_t dist_tsave[32] = { + -5, 1, -1, 1, -3, 1, 1, 0, -4, 1, 0, 0, -2, 1, 1, 0, + -5, 1, -1, 0, -3, 0, 1, -1, -3, 1, 0, -1, -2, 0, 1, 0, +}; + /* distance codes indexed by the reversed 5-bit fixed code, used to build * text_dist_table[] at boot when it's not precomputed, see text_dist(). */ diff --git a/include/import/slz.h b/include/import/slz.h index aa94b77b6..826c8e9b6 100644 --- a/include/import/slz.h +++ b/include/import/slz.h @@ -66,17 +66,21 @@ enum { SLZ_FMT_DEFLATE, /* RFC1951: raw deflate, and no crc */ }; -/* Content types, which select the huffman tree used to encode the data. The - * default one uses the fixed tree from RFC1951. The text one uses a tree - * trained on web contents (html, css, js, json, svg...), which saves about 15% - * of the huffman-coded bits on such contents and still works on any text, at - * the expense of a header of about 80 bytes each time a block is opened. It - * is not suited to binary contents, and is not worth it below 1-2 kB. It is - * set using slz_set_content_type(). +/* Content types, a hint about the data to compress, see + * slz_set_content_type(). They also name the huffman tree used to encode + * them: the fixed tree from RFC1951, and a tree trained on web contents + * (html, css, js, json, svg...), which saves about 15% of the huffman-coded + * bits on such contents and still works on any text, at the expense of a + * header of about 80 bytes each time a block is opened. The event one is for + * streams made of small events flushed one at a time (SSE, logs...): it uses + * the fixed tree, and references up to the last 2 kB passed to the stream, + * which requires a ring allocated by the caller, see slz_ring_size. Without a + * ring, it is the same as no hint. */ enum { - SLZ_CT_GENERIC = 0, /* default: fixed huffman tree */ - SLZ_CT_TEXT, /* text contents: tree trained on web contents */ + SLZ_CT_GENERIC = 0, /* default: no hint; also the fixed huffman tree */ + SLZ_CT_TEXT, /* mostly text contents; also the text tree */ + SLZ_CT_EVENT, /* small events flushed one at a time, see slz_ring_size */ }; struct slz_stream { @@ -93,8 +97,8 @@ struct slz_stream { unsigned char *outbuf; /* set by encode() */ uint32_t qbits; /* number of bits in queue, < 8 on 32-bit, < 32 on 64-bit */ uint32_t state; /* one of slz_state */ - uint8_t unused:1; /* free for future use */ - uint8_t content_type:1; /* SLZ_CT_*, see slz_set_content_type() */ + uint8_t cur_tree:1; /* SLZ_CT_*: tree of the current block, see SLZ_AUTO_IN in slz.c */ + uint8_t content_type:2; /* SLZ_CT_*, see slz_set_content_type() */ uint8_t format:2; /* SLZ_FMT_* */ uint8_t level:4; /* 0 = no compression, 1..15 = compression */ uint16_t debt; /* number of bits by which the output is above the @@ -106,14 +110,32 @@ struct slz_stream { */ uint32_t crc32; uint32_t ilen; + int32_t tree_save; /* bits the other tree would have saved, see SLZ_AUTO_IN */ + uint32_t tree_switch; /* switch to the other tree after the next match */ + void *ring; /* SLZ_CT_EVENT only: history kept between calls, see + * slz_ring_size. Reset by the init functions. + */ }; +/* Size in bytes of the ring that the SLZ_CT_EVENT content type needs per + * stream (about 4 kB). The caller allocates it, zeroes it, and sets it in the + * stream's field after the init function and before the first encode + * call; the stream doesn't free it. 64-byte alignment is preferred but not + * required. The same ring may be reused by another stream after being zeroed + * again. If data are passed to the stream while the ring is not set (or with + * another content type), the history it holds is dropped at the next call. + */ +extern const int slz_ring_size; + +/* lets applications detect the event content type with #ifdef slz_ring_size */ +#define slz_ring_size slz_ring_size + /* Maximum number of bytes by which the encoder's output may exceed the same * data sent as stored blocks, which is the input plus 5 bytes per 65535. It * comes from the 9-bit literals of the fixed huffman encoding, or from the - * literals and the block headers of the text content type, may be reached - * within a single call, and has to be added to the output buffer size of each - * call. It is checked against SLZ_MAX_DEBT at build time in slz.c. + * literals and the block headers of the text tree (levels 2 and above), may + * be reached within a single call, and has to be added to the output buffer + * size of each call. It is checked against SLZ_MAX_DEBT at build time in slz.c. */ #define SLZ_MAX_DEBT_BYTES 40 @@ -206,7 +228,7 @@ static inline long slz_encode(struct slz_stream *strm, void *out, * possibly pending bits from the queue, the possible EOB and the final empty * block, rounding to the next byte, then 4 bytes for the CRC when doing * zlib/gzip, then another 4 bytes for the input length for gzip. That may - * amount to 6+4+4 = 14 bytes, that the caller must ensure are available before + * amount to 7+4+4 = 15 bytes, that the caller must ensure are available before * calling the function. Note that if the initial header was never sent, it * will be sent first as well (up to 10 extra bytes). */ @@ -224,15 +246,16 @@ static inline int slz_finish(struct slz_stream *strm, void *buf) return ret; } -/* Flushes any pending data for stream into buffer , then emits an - * empty literal block to byte-align the output, allowing to completely flush - * the queue. Note that if the initial header was never sent, it will be sent - * first as well (0, 2 or 10 extra bytes). This requires that the output buffer - * still has this plus the 6 bytes needed to flush the queue, the possible EOB - * and the (BFINAL,BTYPE) bits, plus 4 bytes for LEN+NLEN, or a total of 20 - * bytes in the worst case. The number of bytes emitted is returned. It is - * guaranteed that the queue is empty on return. This may cause some overhead - * by adding needless 5-byte blocks if called to often. +/* Emits into buffer all the bits pending in stream at the time of + * the call, so that everything passed so far is decodable from the emitted + * bytes. If they don't end on a byte boundary, the least is added to cross + * it (usually the current block's EOB, 7 bits with the fixed tree, see + * slz_rfc1951_flush()), and up to 7 of the bits added may stay queued. The + * output is thus not necessarily aligned on return. Note that if the initial + * header was never sent, it will be sent first as well (0, 2 or 10 extra + * bytes). This requires that the output buffer still has this plus the 6 + * bytes needed to emit the queue and a possible EOB, or a total of 16 bytes + * in the worst case. The number of bytes emitted is returned. */ static inline int slz_flush(struct slz_stream *strm, void *buf) { @@ -248,21 +271,28 @@ static inline int slz_flush(struct slz_stream *strm, void *buf) return ret; } -/* Sets the content type of stream to , one of SLZ_CT_*, which - * selects the huffman tree used to encode the next blocks. It is meant to be - * called right after one of the init functions, which reset it to - * SLZ_CT_GENERIC. It may also be called later, but only works when no block - * is open, which is not guaranteed even after a flush since a flush doesn't - * close a block if the output is already aligned. It returns 0 on success, or - * -1 if a block is still open or the type is unknown, in which case nothing - * is changed. +/* Hints that the contents compressed with stream are mostly text when + * is SLZ_CT_TEXT, or removes the hint with SLZ_CT_GENERIC (the + * default). From level 2 on, the encoder chooses the huffman tree on the fly + * anyway, and starts with the text one. With the hint, levels 1 and 2 work + * as level 3, whose accounting of the references finds text with few + * literals, and level 1 starts with the text tree as well. SLZ_CT_EVENT + * hints that the stream is made of small events flushed one at a time, and + * switches to the event encoder once a ring is set, see slz_ring_size. It + * is meant to be called right after one of the init functions, which reset + * it to SLZ_CT_GENERIC. It returns 0 on success, or -1 if a block was + * already opened or the type is unknown, in which case nothing is changed. */ static inline int slz_set_content_type(struct slz_stream *strm, int type) { - if ((unsigned int)type > SLZ_CT_TEXT || + if ((unsigned int)type > SLZ_CT_EVENT || strm->state == SLZ_ST_FIXED || strm->state == SLZ_ST_LAST) return -1; strm->content_type = type; + if (strm->level >= 2 || (strm->level && type == SLZ_CT_TEXT)) + strm->cur_tree = SLZ_CT_TEXT; + else + strm->cur_tree = SLZ_CT_GENERIC; return 0; } diff --git a/src/slz.c b/src/slz.c index 99cf64eba..5a495d31e 100644 --- a/src/slz.c +++ b/src/slz.c @@ -110,16 +110,6 @@ static uint32_t text_dist_table[32768]; # define SLZ_LITERAL_SKIP 1 #endif -/* Enables growing a match backwards over the literals that are still pending, - * see slz_rfc1951_encode(). Costs a little CPU on every match and gains about - * 1% of output. Tests show that matching up to 2 bytes brings almost optimal - * results at minimal extra cost, and that compression gains at 1 byte are - * higher than the CPU increase they cause, and tend to result in higher - * compression speed due to the savings on encoding. The only way to disable - * this optimization is to comment out this line. - */ -#define SLZ_BACKWARD_MATCH ((const int[]){ 0, 1, 2, 8 }) - /* Log2 of the size of the hash table used for the references table. */ #ifndef SLZ_HASH_BITS # define SLZ_HASH_BITS 13 @@ -265,9 +255,11 @@ union ref { }; /* This hash provides good average results on HTML contents, and is among the - * few which provide almost optimal results on various different pages. + * few which provide almost optimal results on various different pages. It + * returns bits, which must be a constant. */ -static inline uint32_t slz_hash(uint32_t a) +static inline __attribute__((always_inline)) +uint32_t slz_hash_bits(uint32_t a, const int bits) { #if defined(__ARM_FEATURE_CRC32) # if defined(__ARM_ARCH_ISA_A64) @@ -292,7 +284,13 @@ static inline uint32_t slz_hash(uint32_t a) #else a = ((a << 19) + (a << 6) - a); #endif - return a >> (32 - SLZ_HASH_BITS); + return a >> (32 - bits); +} + +/* the hash of the main encoder's reference table, on SLZ_HASH_BITS bits */ +static inline uint32_t slz_hash(uint32_t a) +{ + return slz_hash_bits(a, SLZ_HASH_BITS); } /* This function compares buffers and and reads 32 or 64 bits at a time @@ -579,6 +577,55 @@ static inline void copy_32b(struct slz_stream *__restrict strm, uint32_t x) strm->outbuf += 4; } +/* From level 2 on, the huffman tree is chosen on the fly, within a call, + * between the fixed one and the text one. The encoder then switches between + * them as it goes, with the same reference table, and the stream's + * tells which one is in use. The literals sent with a tree are compared to + * what the other tree would have cost them, and the difference is accumulated + * in (a cumulated sum, never below zero): + * - the fixed tree's literals sent in a huffman block add what the text + * tree would have saved on them, computed by the out-of-line function + * that sends them, and its stored blocks reset it (binary data); + * - from level 3, the fixed tree's references add what the text tree + * would have saved on their length and distance codes (len_tsave[] and + * dist_tsave[]). On text with few literals, such as C headers made of + * #defines, that's where nearly all of its savings are, which the + * literals alone take too long to notice, if ever. It costs 0.2 to 1% + * of speed for 0.02 to 0.8% of output, hence not at level 2; + * - the text tree's literals add their , known when they are + * sent, i.e. as if the fixed tree took 8 bits for each of them, whether + * they are sent in a huffman block or in a stored one. + * Above SLZ_AUTO_IN bits (fixed -> text) or SLZ_AUTO_OUT bits (text -> + * fixed), is set and the encoder switches after the next match, + * closing its block, and continues from there with the other tree. In + * addition, with the text tree, 64 pending literals that take 9 bits or more + * on average switch at once: this is binary data, which the fixed tree can + * skip, so the encoder processes them again from the first of them. + * + * Streams start with the text tree (see the init functions): on text, it + * saves the 160 to 290 bytes the fixed tree took to switch, i.e. 8% of a + * 4 kB object, while the start of a binary object costs about 80 bytes + * until the text tree gives up. + */ +#define SLZ_AUTO_IN 1280 +#define SLZ_AUTO_OUT 256 + +/* accounts more bits that the other tree would have saved on literals + * sent with , and requests a switch past the threshold. + */ +static inline void auto_account(struct slz_stream *__restrict strm, int32_t d, int ctype) +{ + int32_t s = strm->tree_save + d; + + if (s < 0) + s = 0; + if (s > (ctype == SLZ_CT_TEXT ? SLZ_AUTO_OUT : SLZ_AUTO_IN)) { + strm->tree_switch = 1; + s = 0; + } + strm->tree_save = s; +} + /* Everything below that depends on the huffman tree takes a argument * (SLZ_CT_*), that must be a constant in the encoder, so that each of its * specialisations only contains the code for its own tree and names the @@ -650,11 +697,21 @@ static inline uint32_t len_code(long mlen, int ctype) /* copies litterals from . at once. must not be null and must * be lower than or equal to 65535, which can be guaranteed when using * SLZ_LITERAL_SKIP with limited literal lengths. indicates that there - * are data past buf + . Use copy_lit_small() instead. + * are data past buf + . Use copy_lit_small() instead. In this function + * and the other copy_lit*() ones, a non-zero (a constant) adds + * what the text tree would have saved on the fixed tree's literals to the + * stream's , or resets it for those sent in a stored block, see + * SLZ_AUTO_IN. It is ignored with the text tree, whose encoder does it. */ static inline __attribute__((always_inline)) -void _copy_lit_small(struct slz_stream *__restrict strm, const void *buf, uint32_t len, int more, int ctype) +void _copy_lit_small(struct slz_stream *__restrict strm, const void *buf, uint32_t len, int more, int ctype, int track_save) { + /* stored literals with the fixed tree: binary data. With the text + * tree, the encoder accounts them itself. + */ + if (track_save && ctype == SLZ_CT_GENERIC) + strm->tree_save = 0; + if (ctype == SLZ_CT_TEXT) { /* same as below with the text tree's EOB, which is not * zero: 3 to 21 bits. @@ -688,44 +745,57 @@ void _copy_lit_small(struct slz_stream *__restrict strm, const void *buf, uint32 strm->outbuf += len; } -/* out-of-line versions of _copy_lit_small() for each content type. They must +/* out-of-line versions of _copy_lit_small() for each tree. They must * not be inlined in the encoder, which is sensitive to register pressure. */ static __attribute__((noinline)) void copy_lit_small_generic(struct slz_stream *__restrict strm, const void *buf, uint32_t len, int more) { - _copy_lit_small(strm, buf, len, more, SLZ_CT_GENERIC); + _copy_lit_small(strm, buf, len, more, SLZ_CT_GENERIC, 0); +} + +static __attribute__((noinline)) +void copy_lit_small_generic_auto(struct slz_stream *__restrict strm, const void *buf, uint32_t len, int more) +{ + _copy_lit_small(strm, buf, len, more, SLZ_CT_GENERIC, 1); } static __attribute__((noinline)) void copy_lit_small_text(struct slz_stream *__restrict strm, const void *buf, uint32_t len, int more) { - _copy_lit_small(strm, buf, len, more, SLZ_CT_TEXT); + _copy_lit_small(strm, buf, len, more, SLZ_CT_TEXT, 0); } /* calls the version of _copy_lit_small() for */ static inline __attribute__((always_inline)) -void copy_lit_small(struct slz_stream *__restrict strm, const void *buf, uint32_t len, int more, int ctype) +void copy_lit_small(struct slz_stream *__restrict strm, const void *buf, uint32_t len, int more, int ctype, int track_save) { if (ctype == SLZ_CT_TEXT) copy_lit_small_text(strm, buf, len, more); + else if (track_save) + copy_lit_small_generic_auto(strm, buf, len, more); else copy_lit_small_generic(strm, buf, len, more); } /* copies litterals from . indicates that there are data past - * buf + . must not be null. Use copy_lit() instead. + * buf + . must not be null. Use copy_lit() instead. It's only used + * with the fixed tree, the text tree's literals leave through copy_lit_small() + * since they never accumulate past 0xffc0 bytes. */ static inline __attribute__((always_inline)) -void _copy_lit(struct slz_stream *__restrict strm, const void *buf, uint32_t len, int more, int ctype) +void _copy_lit(struct slz_stream *__restrict strm, const void *buf, uint32_t len, int more, int track_save) { uint32_t len2; if (len <= 65535) - return copy_lit_small(strm, buf, len, more, ctype); + return copy_lit_small(strm, buf, len, more, SLZ_CT_GENERIC, track_save); + + if (track_save) + strm->tree_save = 0; if (strm->state != SLZ_ST_EOB) - send_eob(strm, ctype); + send_eob(strm, SLZ_CT_GENERIC); len2 = 65535; len -= len2; @@ -750,25 +820,25 @@ void _copy_lit(struct slz_stream *__restrict strm, const void *buf, uint32_t len strm->state = more ? SLZ_ST_EOB : SLZ_ST_DONE; } -/* out-of-line versions of _copy_lit() for each content type */ +/* out-of-line versions of _copy_lit() with and without */ static __attribute__((noinline,unused)) void copy_lit_generic(struct slz_stream *__restrict strm, const void *buf, uint32_t len, int more) { - _copy_lit(strm, buf, len, more, SLZ_CT_GENERIC); + _copy_lit(strm, buf, len, more, 0); } -static __attribute__((noinline,unused)) -void copy_lit_text(struct slz_stream *__restrict strm, const void *buf, uint32_t len, int more) +static __attribute__((noinline)) +void copy_lit_generic_auto(struct slz_stream *__restrict strm, const void *buf, uint32_t len, int more) { - _copy_lit(strm, buf, len, more, SLZ_CT_TEXT); + _copy_lit(strm, buf, len, more, 1); } -/* calls the version of _copy_lit() for */ +/* calls the version of _copy_lit() for */ static inline __attribute__((always_inline)) -void copy_lit(struct slz_stream *__restrict strm, const void *buf, uint32_t len, int more, int ctype) +void copy_lit(struct slz_stream *__restrict strm, const void *buf, uint32_t len, int more, int track_save) { - if (ctype == SLZ_CT_TEXT) - copy_lit_text(strm, buf, len, more); + if (track_save) + copy_lit_generic_auto(strm, buf, len, more); else copy_lit_generic(strm, buf, len, more); } @@ -777,7 +847,7 @@ void copy_lit(struct slz_stream *__restrict strm, const void *buf, uint32_t len, * buf + . must not be null. Use copy_lit_huff() instead. */ static inline __attribute__((always_inline)) -void _copy_lit_huff(struct slz_stream *__restrict strm, const unsigned char *buf, uint32_t len, int more, int ctype) +void _copy_lit_huff(struct slz_stream *__restrict strm, const unsigned char *buf, uint32_t len, int more, int ctype, int track_save) { uint32_t pos; @@ -795,30 +865,57 @@ void _copy_lit_huff(struct slz_stream *__restrict strm, const unsigned char *buf } pos = 0; + if (track_save && ctype == SLZ_CT_GENERIC) { + /* what the text tree would have saved on these literals, from + * lit_tsave[], next to their fixed codes from fixed_huff[]. + * A 64-bit table holding both, summed at once, was 1.9% slower + * on a Cortex-A720 and within 1% elsewhere. + */ + int32_t sum = 0; + + do { + uint32_t c = buf[pos++]; + uint32_t code = fixed_huff[c]; + + sum += lit_tsave[c]; + enqueue24(strm, code >> 4, code & 15); + } while (pos < len); + auto_account(strm, sum, ctype); + return; + } + do { send_huff(strm, buf[pos++], ctype); } while (pos < len); } -/* out-of-line versions of _copy_lit_huff() for each content type */ +/* out-of-line versions of _copy_lit_huff() for each tree */ static __attribute__((noinline)) void copy_lit_huff_generic(struct slz_stream *__restrict strm, const unsigned char *buf, uint32_t len, int more) { - _copy_lit_huff(strm, buf, len, more, SLZ_CT_GENERIC); + _copy_lit_huff(strm, buf, len, more, SLZ_CT_GENERIC, 0); +} + +static __attribute__((noinline)) +void copy_lit_huff_generic_auto(struct slz_stream *__restrict strm, const unsigned char *buf, uint32_t len, int more) +{ + _copy_lit_huff(strm, buf, len, more, SLZ_CT_GENERIC, 1); } static __attribute__((noinline)) void copy_lit_huff_text(struct slz_stream *__restrict strm, const unsigned char *buf, uint32_t len, int more) { - _copy_lit_huff(strm, buf, len, more, SLZ_CT_TEXT); + _copy_lit_huff(strm, buf, len, more, SLZ_CT_TEXT, 0); } /* calls the version of _copy_lit_huff() for */ static inline __attribute__((always_inline)) -void copy_lit_huff(struct slz_stream *__restrict strm, const unsigned char *buf, uint32_t len, int more, int ctype) +void copy_lit_huff(struct slz_stream *__restrict strm, const unsigned char *buf, uint32_t len, int more, int ctype, int track_save) { if (ctype == SLZ_CT_TEXT) copy_lit_huff_text(strm, buf, len, more); + else if (track_save) + copy_lit_huff_generic_auto(strm, buf, len, more); else copy_lit_huff_generic(strm, buf, len, more); } @@ -903,8 +1000,8 @@ static inline void insert_ref(union ref *refs, const unsigned char *in, unsigned * credit, and they are then removed from it. The balance is not allowed to * drop below zero, i.e. to accumulate more than SLZ_CREDIT bits of savings: * these allow a few groups of literals to remain in fixed huffman instead of - * paying for a return to it, which improves the compression ratio at levels - * 2 and 3 (from 32 to 300 it only grows, but costs more CPU past 64, since + * paying for a return to it, which improves the compression ratio at level 3 + * (and at the former level 4; from 32 to 300 it only grows, but costs more CPU past 64, since * more literals are huffman-encoded). The balance is carried between calls in * debt>, and a stream starts with no savings at all. * @@ -916,7 +1013,7 @@ static inline void insert_ref(union ref *refs, const unsigned char *in, unsigned * SLZ_CREDIT + SLZ_MAX_DEBT + SLZ_SWITCH_COST, which is what bounds the excess * of a stream over the same data sent as stored blocks to 252 bits. In * practice it reaches 154 bits on crafted data and 149 on silesia.tar, where - * values 100 to 1000 trade a little ratio between levels 2 and 3. + * values 100 to 1000 trade a little ratio between levels 3 and the former 4. * * The same threshold also marks the stored mode inside a call: whenever the * encoder leaves fixed huffman mode (literal skip, full stored block), the @@ -929,16 +1026,20 @@ static inline void insert_ref(union ref *refs, const unsigned char *in, unsigned * it saves by itself, and the switch to fixed huffman is charged when it is * performed by that reference, but not when the first literals perform it. * - * At level 1, literals are not counted one at a time, which costs too much in - * the main loop. Instead, a group of literals sent in fixed huffman mode before - * a reference is charged one bit per byte as if all of them took 9 bits, once - * the decision to send it in fixed huffman mode is taken. The balance is thus - * still an upper bound of the excess, and the group itself, whose decision did - * not include it, is shorter than 64 bytes there (longer ones are skipped and - * leave as a stored block), so it adds less than 64 bits to the bound, which - * is thus 315 bits, published in bytes as SLZ_MAX_DEBT_BYTES in slz.h. On real - * data, the reference that follows almost always saves more than the charge, - * so this rarely changes a decision. + * At encoder levels 1 and 2 (compression levels 1 to 3, see the + * argument of _slz_rfc1951_encode()), literals are not counted one at a time, + * which costs too much in the main loop. Instead, a group of literals sent in + * fixed huffman mode before a reference is charged one bit per byte as if all + * of them took 9 bits, once the decision to send it in fixed huffman mode is + * taken. The balance is thus still an upper bound of the excess, and the group + * itself, whose decision did not include it, is shorter than 64 bytes there + * (longer ones are skipped and leave as a stored block), so it adds less than + * 64 bits to the bound, which is thus 315 bits, published in bytes as + * SLZ_MAX_DEBT_BYTES in slz.h. On real data, the reference that follows almost + * always saves more than the charge, so this rarely changes a decision. This + * relies on the literal skip: without it (SLZ_LITERAL_SKIP=0), a group may be + * up to 65472 bytes long and would add as many bits to the bound, so all + * levels count literals one at a time then. */ #define SLZ_MAX_DEBT 200 #define SLZ_CREDIT 64 @@ -966,13 +1067,47 @@ typedef char slz_max_debt_bytes_check[(SLZ_MAX_DEBT_BYTES * 8 >= SLZ_MAX_DEBT + * silesia.tar in stored blocks. Since the output of each call must also stay * within SLZ_MAX_OVERHEAD(), only the savings of the current call may be used, * and those carried from previous calls are dropped when a call starts (see - * _slz_rfc1951_encode_text()). Past 2000 bits, little is gained. This needs a + * _slz_rfc1951_encode()). Past 2000 bits, little is gained. This needs a * larger counter, , which is only used with the text tree so that * the code for the fixed one doesn't have to deal with 16-bit values. */ #define SLZ_TEXT_CREDIT 4096 #define SLZ_TEXT_MAX_DEBT (SLZ_MAX_DEBT + SLZ_TEXT_CREDIT) +/* Each tree's part only maintains its own counter, or , so + * the excess must be carried from one to the other when the tree changes, or + * a call or a stream alternating between them could reach the sum of their + * bounds. Both counters remain within the larger bound, since none of the + * parts adds to its counter past its own threshold. Only the fixed tree takes + * the savings, as the text tree may only use those of the current call. + */ +static inline void debt_to_text(struct slz_stream *__restrict strm) +{ + strm->text_debt = SLZ_TEXT_CREDIT + (strm->debt > SLZ_CREDIT ? strm->debt - SLZ_CREDIT : 0); +} + +static inline void debt_to_fixed(struct slz_stream *__restrict strm) +{ + strm->debt = (strm->text_debt > SLZ_TEXT_CREDIT - SLZ_CREDIT) ? strm->text_debt - (SLZ_TEXT_CREDIT - SLZ_CREDIT) : 0; +} + +/* Makes the fixed tree the current one for the encoders that only know it, + * closing a block opened with the text tree. may be the text tree + * when they are entered: the init functions start with it from level 2, and + * the level may be changed between calls (e.g. lowered under load). + */ +static inline void leave_text_tree(struct slz_stream *__restrict strm) +{ + if (strm->cur_tree != SLZ_CT_GENERIC) { + if (strm->state == SLZ_ST_FIXED) { + send_eob(strm, strm->cur_tree); + strm->state = SLZ_ST_EOB; + } + strm->cur_tree = SLZ_CT_GENERIC; + debt_to_fixed(strm); + } +} + /* fails to build if send_text_hdr() doesn't charge SLZ_TEXT_SWITCH_COST */ typedef char slz_text_switch_cost_check[(SLZ_TEXT_SWITCH_COST == TEXT_EOB_BITS + 42 + 3 + TEXT_HDR_BITS) ? 1 : -1]; @@ -1042,6 +1177,50 @@ static inline int text_lit_stored(const struct slz_stream *strm, int32_t excess) (int32_t)strm->text_debt + excess >= SLZ_TEXT_MAX_DEBT); } +/* returns how many of the up to bytes before [] are equal to + * those before , comparing at most of them, which must be a + * constant. See the backward match in _slz_rfc1951_encode(). + */ +static inline __attribute__((always_inline)) +long backward_match(const unsigned char *in, unsigned long pos, const unsigned char *lastp, long bmax, int len) +{ + long back = 0; + +#if defined(UNALIGNED_LE_OK) + if (len == 2) { // 43.32%, best speed vs savings + uint16_t x = *(uint16_t *)&in[pos - 2] ^ *(uint16_t *)(lastp - 2); + back = x & 0xff00 ? 0 : x ? 1 : 2; + if (back > bmax) + back = bmax; + } else if (len == 3 || len == 4) { // 32 bits: 43.31% or 43.30% + uint32_t x = *(uint32_t *)&in[pos - 4] ^ *(uint32_t *)(lastp - 4); + back = x & 0xffff0000u ? + x & 0xff000000u ? 0 : 1 : + x & 0xffffff00u ? 2 : + x ? 3 : len; // 3 or 4, test eliminated at 3 + if (back > bmax) + back = bmax; + } else if (len == 8) { // 64 bits: 43.30% + uint64_t x = *(uint64_t *)&in[pos - 8] ^ *(uint64_t *)(lastp - 8); + back = x ? (__builtin_clzll(x) >> 3) : 8; + if (back > bmax) + back = bmax; + } else +#endif + if (len == 1) { // single-byte match: 43.39% + if (in[pos - 1] == lastp[-1]) + back++; + if (back > bmax) + back = bmax; + } + else { /* unaligned not supported or match != 1/2/3/4/8 */ + while (back < bmax && in[pos - back - 1] == lastp[-back - 1]) + back++; + } + + return back; +} + /* Compresses bytes from into according to RFC1951. The * output result may be up to 5 bytes larger than the input for each 65535 * nput bytes, to which 2 extra bytes may be added to send the last chunk due @@ -1052,8 +1231,8 @@ static inline int text_lit_stored(const struct slz_stream *strm, int32_t excess) * addition, the fixed huffman encoding may be behind the same data sent as * stored blocks by up to SLZ_MAX_DEBT_BYTES, which may all be reached within a * single call, so each call must also have that much room. SLZ_MAX_OVERHEAD() - * in slz.h sums all of this, including the text tree's header and debt when - * is SLZ_CT_TEXT (see SLZ_TEXT_MAX_DEBT). The caller is responsible + * in slz.h sums all of this, including the text tree's header and debt (see + * SLZ_TEXT_MAX_DEBT). The caller is responsible * for ensuring there is enough room in the output buffer for this. The amount * of output bytes is returned, and no CRC is computed. * @@ -1067,9 +1246,15 @@ static inline int text_lit_stored(const struct slz_stream *strm, int32_t excess) * better and that option degrades it by 2% again. gcc-16 is even slightly * better (+0.5%). As a general rule, nothing vectorizes well here, and the * unaligned, short memcpy() calls must not be translated to AVX512 code. + * + * is the encoder's level, not the compression level, see + * _slz_rfc1951_encode_lev1(), and the fixed tree's backward match + * length. is zero to only use the fixed tree, or non-zero to + * choose between the fixed and the text trees on the fly (see SLZ_AUTO_IN). + * All three must be constants. Each tree has its own part below. */ static inline __attribute__((always_inline)) -long _slz_rfc1951_encode(struct slz_stream *__restrict strm, unsigned char *out, const unsigned char *in, long ilen, int more, int level, int ctype) +long _slz_rfc1951_encode(struct slz_stream *__restrict strm, unsigned char *out, const unsigned char *in, long ilen, int more, int level, int bmatch, int auto_tree) { long rem = ilen; unsigned long pos = 0; @@ -1080,16 +1265,7 @@ long _slz_rfc1951_encode(struct slz_stream *__restrict strm, unsigned char *out, uint32_t h; uint64_t ent; -#if SLZ_LITERAL_SKIP - long skip_ratio = 0; -#endif uint32_t plit = 0; - uint32_t bal; /* see SLZ_MAX_DEBT, generic tree only */ - uint32_t lit_excess = 0; /* text tree only, see SLZ_TEXT_STORED_COST */ - int lit_stored = 0; /* text tree only, pending literals go to a stored block */ - uint32_t nvref = 0; /* text tree only, recorded references, see SLZ_TEXT_VREFS */ - uint32_t vplit = 0; /* text tree only, after the last recorded reference */ - struct text_vref vref[SLZ_TEXT_VREFS]; uint32_t dist, code; union ref refs[1 << SLZ_HASH_BITS]; @@ -1097,263 +1273,640 @@ long _slz_rfc1951_encode(struct slz_stream *__restrict strm, unsigned char *out, /* force to send as literals (eg to preserve CPU) */ strm->outbuf = out; if (ilen) - copy_lit(strm, in, ilen, more, ctype); + copy_lit(strm, in, ilen, more, auto_tree); goto leave; } reset_refs(refs, sizeof(refs)); - bal = strm->debt; - strm->outbuf = out; + if (auto_tree) { + /* only the savings of this call may pay for its text blocks, + * see SLZ_TEXT_CREDIT. + */ + if (strm->text_debt < SLZ_TEXT_CREDIT) + strm->text_debt = SLZ_TEXT_CREDIT; + + if (strm->tree_switch) { + /* requested by the last literals of the previous call */ + if (strm->state == SLZ_ST_FIXED) { + send_eob(strm, strm->cur_tree); + strm->state = SLZ_ST_EOB; + } + if (strm->state == SLZ_ST_EOB) { + strm->cur_tree = !strm->cur_tree; + if (strm->cur_tree == SLZ_CT_TEXT) + debt_to_text(strm); + else + debt_to_fixed(strm); + } + strm->tree_switch = 0; + strm->tree_save = 0; + } + if (strm->cur_tree == SLZ_CT_TEXT) + goto text_start; + } + + /* The fixed tree's part, with its own loop. */ + { + uint32_t bal; /* see SLZ_MAX_DEBT */ + + fixed_start: + rem = ilen - pos; + plit = 0; + bal = strm->debt; + #ifndef UNALIGNED_FASTER reload_word: - if (rem >= 4) // is only used inside the loop below, hence the test for >= 4 - word = ((uint32_t)(unsigned char)in[pos] << 8) + ((uint32_t)(unsigned char)in[pos + 1] << 16) + ((uint32_t)(unsigned char)in[pos + 2] << 24); + if (rem >= 4) // is only used inside the loop below, hence the test for >= 4 + word = ((uint32_t)(unsigned char)in[pos] << 8) + ((uint32_t)(unsigned char)in[pos + 1] << 16) + ((uint32_t)(unsigned char)in[pos + 2] << 24); #endif - while (rem >= 4) { + while (rem >= 4) { #ifndef UNALIGNED_FASTER - word = ((uint32_t)(unsigned char)in[pos + 3] << 24) + (word >> 8); + word = ((uint32_t)(unsigned char)in[pos + 3] << 24) + (word >> 8); #else - word = *(uint32_t *)&in[pos]; + word = *(uint32_t *)&in[pos]; #endif - h = slz_hash(word); - asm volatile ("" ::); // prevent gcc from trying to be smart with the prefetch - - if (sizeof(long) >= 8) { - ent = refs[h].by64; - last = (uint32_t)ent; - ent >>= 32; - refs[h].by64 = ((uint64_t)pos) + ((uint64_t)word << 32); - } else { - ent = refs[h].by32.word; - last = refs[h].by32.pos; - refs[h].by32.pos = pos; - refs[h].by32.word = word; - } + h = slz_hash(word); + asm volatile ("" ::); // prevent gcc from trying to be smart with the prefetch + + if (sizeof(long) >= 8) { + ent = refs[h].by64; + last = (uint32_t)ent; + ent >>= 32; + refs[h].by64 = ((uint64_t)pos) + ((uint64_t)word << 32); + } else { + ent = refs[h].by32.word; + last = refs[h].by32.pos; + refs[h].by32.pos = pos; + refs[h].by32.word = word; + } -#ifdef FIND_OPTIMAL_MATCH - /* Experimental code to see what could be saved with an ideal - * longest match lookup algorithm. This one is very slow but - * scans the whole window. In short, here are the savings : - * file orig fast(ratio) optimal(ratio) - * README 5185 3419 (65.9%) 3165 (61.0%) -7.5% - * index.html 76799 35662 (46.4%) 29875 (38.9%) -16.3% - * rfc1952.c 29383 13442 (45.7%) 11793 (40.1%) -12.3% - * - * Thus the savings to expect for large files is at best 16%. - * - * A non-colliding hash gives 33025 instead of 35662 (-7.4%), - * and keeping the last two entries gives 31724 (-11.0%). - */ - unsigned long scan; - int saved = 0; - int bestpos = 0; - int bestlen = 0; - int firstlen = 0; - int max_lookup = 2; // 0 = no limit + if ((uint32_t)ent != word) { + send_as_lit: + rem--; + plit++; + if (level >= 3 || !SLZ_LITERAL_SKIP) // see SLZ_MAX_DEBT + bal += ((unsigned char)word >= 144); + pos++; + + /* Incompressible data: once enough literals have accumulated + * without a single match, stop looking for a while and just + * add what follows to the pending literals, doubling the + * amount skipped each time so that a long incompressible block + * costs less and less to walk over. A match resets it. The + * skipped bytes are not emitted here, as closing a stored + * block at each step would cost 5 bytes every few kB and + * break the promise of 5 bytes per 65535. Since they were not + * inspected, is raised so that they may only leave as + * a stored block, and so that a reference is only accepted if + * it pays for the return to FIXED (see SLZ_MAX_DEBT). The EOB + * is sent by copy_lit() along with them. may thus + * exceed 65535, the pending literals are always sent with + * copy_lit() which splits them into full stored blocks. + */ +#if SLZ_LITERAL_SKIP + if (level <= 2 && !(plit & 63)) { + /* The step is derived from the literals pending since + * the last match: 0, 64, 192, 448, ... up to 8128, i.e. + * the capped exponential ramp, which keeps + * aligned on 64. Keeping it in a separate variable reset + * at each match cost 2% at level 1 even on data that + * never skips, through register pressure in the match + * path. + */ + long skip = plit - 64 > 8128 ? 8128 : plit - 64; + + skip = rem >= skip ? skip : rem; + /* Level 2 still looks up and indexes the last word of + * each whole 64-byte chunk it skips, and stops skipping + * on the first one already present within the window, + * so that the main loop resumes at the start of that + * chunk. The skipped words remain indexed for the data + * that follows. The last word rather than the first one + * gives shorter distances to what follows the skip and a + * smaller output. Only the remainder of a skip limited by + * the end of the input is skipped unchecked. Since the + * skip steps are 64 bytes apart, remains aligned + * on them. + */ + if (level == 2 && skip >= 64) { + long chunks = skip >> 6; + + skip &= 63; + do { + uint32_t w = read_le32(in + pos + 60); + uint32_t hw = slz_hash(w); + + if (sizeof(long) >= 8) { + uint64_t old = refs[hw].by64; + + if ((uint32_t)(old >> 32) == w && (unsigned long)(pos + 59 - (uint32_t)old) < 32768) { + skip = 0; + break; + } + refs[hw].by64 = ((uint64_t)(pos + 60)) + ((uint64_t)w << 32); + } else { + if (refs[hw].by32.word == w && (unsigned long)(pos + 59 - refs[hw].by32.pos) < 32768) { + skip = 0; + break; + } + refs[hw].by32.pos = pos + 60; + refs[hw].by32.word = w; + } + pos += 64; + rem -= 64; + plit += 64; + } while (--chunks); + } + pos += skip; + rem -= skip; + plit += skip; + /* skipped bytes may only leave as a stored block */ + bal += SLZ_MAX_DEBT + SLZ_CREDIT; +# ifndef UNALIGNED_FASTER + /* we'll need to update the current word since we skipped some bytes */ + goto reload_word; +# else + continue; +# endif + } + else if (__builtin_expect(level == 3 && !(plit & 2047), 0)) { + /* derived from as well: 0, 4096, 16384, then + * 28672 (the former ramp gave 0, 4096, 12288, 28672). + * is not always a multiple of 2048 here (see + * below), hence the rounding. + */ + long skip = 2 * (long)(plit & ~2047U) - 4096; + + skip = skip > 28672 ? 28672 : skip; + skip = rem >= skip ? skip : rem; + + /* Note: by checking intervals smaller than the mask above, a hit resumes + * at the start of the 64-byte chunk containing it, so is then no + * longer a multiple of 2048 and the next check comes after 64 to 1984 + * literals if no match is found in between. This shortens the scan after + * a false hit. Rewinding to the 2048 boundary instead showed almost no + * size gains but was 0.3-0.8% slower. Checking every 64 bytes instead + * of 256 makes level 4 0.05% smaller on silesia and 0.64% on a tar of + * /usr/share/doc for 0.4-1.2% of speed. + */ + if (skip >= 64) { + long chunks = skip >> 6; + + skip &= 63; + do { + uint32_t w = read_le32(in + pos + 60); + uint32_t hw = slz_hash(w); + + if (sizeof(long) >= 8) { + uint64_t old = refs[hw].by64; + + if ((uint32_t)(old >> 32) == w && (unsigned long)(pos + 59 - (uint32_t)old) < 32768) { + skip = 0; + break; + } + refs[hw].by64 = ((uint64_t)(pos + 60)) + ((uint64_t)w << 32); + } else { + if (refs[hw].by32.word == w && (unsigned long)(pos + 59 - refs[hw].by32.pos) < 32768) { + skip = 0; + break; + } + refs[hw].by32.pos = pos + 60; + refs[hw].by32.word = w; + } + pos += 64; + rem -= 64; + plit += 64; + } while (--chunks); + } + pos += skip; + rem -= skip; + plit += skip; + /* skipped bytes may only leave as a stored block. Also done + * on the first test, where is 0, like the stored + * block sent there before: it keeps the same matches. + */ + bal += SLZ_MAX_DEBT + SLZ_CREDIT; +# ifndef UNALIGNED_FASTER + /* we'll need to update the current word since we skipped some bytes */ + goto reload_word; +# else + continue; +# endif + } +#else + /* no literal skip at build time: the stored blocks are as + * large as possible. + */ + if (__builtin_expect(plit == 0xffc0, 0)) { + copy_lit_small(strm, in + pos - plit, plit, 1, SLZ_CT_GENERIC, auto_tree); + plit = 0; + bal = strm->debt + SLZ_MAX_DEBT + SLZ_CREDIT; + } +#endif + continue; + } - for (scan = pos - 1; scan < pos && (unsigned long)(pos - scan - 1) < 32768; scan--) { - int len; + /* We reject pos = last and pos > last+32768 */ + if ((unsigned long)(pos - last - 1) >= 32768) + goto send_as_lit; - if (*(uint32_t *)(in + scan) != word) - continue; + /* faster to keep a copy of this with gcc 13-16 */ + lastp = in + last; + + /* Note: cannot encode a length larger than 258 bytes */ + mlen = memmatch(in + pos + 4, lastp + 4, (rem > 258 ? 258 : rem) - 4) + 4; + + /* found a matching entry */ + + /* if the pending literals are going to be sent as a stored + * block, a short match is not worth splitting it. + */ + if (bal >= SLZ_SWITCH_COST + SLZ_CREDIT && mlen < 6) + goto send_as_lit; + + /* compute the output code, its size and the length's size in + * bits to know if the reference is cheaper than literals. + */ + code = len_code(mlen, SLZ_CT_GENERIC); - len = memmatch(in + pos, in + scan, rem); - if (!bestlen) - firstlen = len; + /* direct mapping of dist->huffman code */ + dist = fh_dist_table[pos - last - 1]; - if (len > bestlen) { - bestlen = len; - bestpos = scan; + /* If encoding the dist+length is more expensive than sending + * the equivalent as bytes, lets keep the literals. This is + * important for the compression ratio and also for standing by + * our promise that the output is no longer than input + 5 bytes + * every 65535. The explanation is that a valid stream starts in + * EOB and ends in EOB, which is the state where we can grow by + * up to 5 bytes (1 byte for BT, 2 for LEN, 2 for NLEN, then + * literal data). Sending compressed Huffman data (reference or + * fixed encoding) may only be accepted if it is guaranteed that + * it will not represent more bytes in the worst case. Switching + * from EOB to FIXED requires 3 bits of BT. Then either fixed + * data (8 or 9 bits) or references (15-33 bits). Switching back + * to EOB requires EOB (7 bits), and in order to get back to the + * situation where bytes can be added with no overhead, the BT, + * align, LEN and NLEN have to be sent as well: + * + * Before switching: + * [BT] [ALIGN] [LEN] [NLEN] [PLIT] + * bits: 3 7 16 16 8*plit + * + * In case of sending mlen bytes as literals: + * [BT] [ALIGN] [LEN] [NLEN] [PLIT] [MLEN] + * bits: 3 7 16 16 8*plit 8*mlen + * + * In case of sending mlen bytes as literals, we add: + * [MLEN] + * bits: 8*mlen + * + * In case of sending reference, we add in the worst case: + * [BT] [CODE] [DIST] [EOB] [BT] [ALIGN] [LEN] [NLEN] + * bits: 3 clen dlen 7 3 7 16 16 + * + * Thus for literals we add 8*mlen bits, and for reference we + * add clen+dlen+52 bits. If the reference encoding + 52 bits + * is shorter. Of course there are plenty of opportunities to + * be much shorter once we switch to FIXED, and a stricter + * tracking could allow to send references more often. But here + * we at least guarantee that if data fit as literals, they also + * fit using a temporary switch to FIXED. + * + * Regarding encoding size, clen is 7 for mlen 3..10, to 12 for + * mlen 131..257. dlen is 8 for distances 1..4, to 21 for + * distances 16385..32768. Thus mlen <= 130 produces clen+dlen + * <= 32 bits. This means that for mlen 4 or above, the encoded + * reference is always smaller than the data it references. This + * is what guarantees that once switched to FIXED we can stay + * in it for as long as needed. + */ + /* Note: is at least SLZ_MAX_DEBT + SLZ_CREDIT in stored + * mode, except at the beginning of a call (see SLZ_MAX_DEBT). + */ + if (bal >= SLZ_MAX_DEBT + SLZ_CREDIT && + (dist & 0x1f) + (code >> 16) + SLZ_SWITCH_COST > 8 * mlen) + goto send_as_lit; + + /* Compare queued literals in case the previous ones were not + * found due to hash collisions but still match. We don't go + * further than 8 bytes so that we can find them in a single + * 64-bit comparison. On aligned-only machines we read one byte + * at a time, as it is expected to be cheaper than composing an + * unaligned int and manually doing a ctzll() on it. Tests have + * shown that there's little to gain past 8 bytes, and anyway + * the total match length will be bounded by 258. BTW we only + * consider cases where last >= 8 because this guarantees that + * we can access both the searched sequence and the match found + * on a wole word at once. It may miss some rare matches on the + * edge of the buffer that we don't care about. + * + * It's worth noting that we *do not* fixup if a match + * is found, as this would be too complicated and counter- + * productive. This means that literals that were finally + * encoded might still count towards the budget limit, but this + * is not dramatic and keeps us on the safe side. + * + * The result of the search is to only touch that will be + * used all along in all operations. Multiple attempts have shown + * that it's both easier and faster this way than changing then + * rolling back offsets when we finally give up and send_as_lit. + * It is disabled when is zero. + */ + if (bmatch && + __builtin_expect(plit != 0, 1) && + (last >= MAX_C(sizeof(long), bmatch))) { + long bmax = MIN(plit, 8); // keep it at 8, always faster + long back; + + if (bmax > 258 - mlen) + bmax = 258 - mlen; + + back = backward_match(in, pos, lastp, bmax, bmatch); + + if (back) { + /* OK now we're certain to use the match, we must rewind the + * literals that are part of the match. + */ + pos -= back; + plit -= back; + rem += back; + mlen += back; + /* was computed from the un-extended + * length above and must follow, or we would + * consume more bytes than the length + * code tells the decoder to copy. does + * not move. The cost test above keeps using + * the un-extended length, which is safe: the + * extension only makes the match cheaper per + * byte, never more expensive. + */ + code = len_code(mlen, SLZ_CT_GENERIC); + } } - if (!--max_lookup) - break; - } - if (bestlen) { - //printf("pos=%d last=%d bestpos=%d word=%08x ent=%08x len=%d\n", - // (int)pos, (int)last, (int)bestpos, (int)word, (int)ent, bestlen); - last = bestpos; - ent = word; - saved += bestlen - firstlen; - } - //fprintf(stderr, "first=%d best=%d saved_total=%d\n", firstlen, bestlen, saved); + + /* first, copy pending literals */ + if (plit) { + /* Huffman encoding requires 9 bits for octets 144..255, so this + * is a waste of space for binary data. Switching between Huffman + * and no-comp then huffman consumes 52 bits (7 for EOB + 3 for + * block type + 7 for alignment + 32 for LEN+NLEN + 3 for next + * block). The balance already counts these literals, so once + * it is SLZ_SWITCH_COST above the credit, the literals leave + * as a stored block and are removed from it by restoring the + * value it had after the last reference. Literals sent in a + * stored block do not cost anything extra. + */ + if (bal >= SLZ_SWITCH_COST + SLZ_CREDIT) { + copy_lit(strm, in + pos - plit, plit, 1, auto_tree); + bal = strm->debt; + } + else { + if (level <= 2 && SLZ_LITERAL_SKIP) + bal += plit; // worst case: all of them take 9 bits + copy_lit_huff(strm, in + pos - plit, plit, 1, SLZ_CT_GENERIC, auto_tree); + } + + plit = 0; + } + + /* use mode 01 - fixed huffman */ + if (strm->state == SLZ_ST_EOB) { + strm->state = SLZ_ST_FIXED; + open_huff_block(strm, 0, SLZ_CT_GENERIC); + /* restart from the saved balance (drops the stored mode mark) + * plus what leaving fixed huffman will cost + */ + bal = strm->debt + SLZ_SWITCH_COST; + } + + /* copy the length first */ +#if SLZ_DIRECT_ENQUEUE56 + /* the length code and the distance code that follows it total + * 31 bits at most, which one enqueue can take in a single + * shift-or-store instead of two + */ + enqueue56(strm, (uint64_t)(code & 0xFFFF) | + ((uint64_t)(dist >> 5) << (code >> 16)), + (code >> 16) + (dist & 0x1f)); +#else + enqueue24(strm, code & 0xFFFF, code >> 16); + + /* in fixed huffman mode, dist is fixed 5 bits */ + enqueue24(strm, dist >> 5, dist & 0x1f); #endif - if ((uint32_t)ent != word) { - send_as_lit: - rem--; - plit++; - /* for the text tree, is the exact number of - * bits the pending literals cost over 8 bits each, as a - * signed value. + /* from level 3, what the text tree would save on this + * reference, see SLZ_AUTO_IN. */ - if (ctype == SLZ_CT_TEXT) - lit_excess += (text_huff[(unsigned char)word] & 15) - 8; - else if (level >= 2) // level 1 charges whole groups, see SLZ_MAX_DEBT - bal += ((unsigned char)word >= 144); - pos++; - - /* Incompressible data: once enough literals have accumulated - * without a single match, stop looking for a while and just - * add what follows to the pending literals, doubling the - * amount skipped each time so that a long incompressible block - * costs less and less to walk over. A match resets it. The - * skipped bytes are not emitted here, as closing a stored - * block at each step would cost 5 bytes every few kB and - * break the promise of 5 bytes per 65535. Since they were not - * inspected, is raised so that they may only leave as - * a stored block, and so that a reference is only accepted if - * it pays for the return to FIXED (see SLZ_MAX_DEBT). The EOB - * is sent by copy_lit() along with them. may thus - * exceed 65535, the pending literals are always sent with - * copy_lit() which splits them into full stored blocks. + if (auto_tree && level >= 2) { + int32_t save = strm->tree_save + len_tsave[mlen] + dist_tsave[(dist >> 5) & 31]; + + save = save < 0 ? 0 : save; + if (__builtin_expect(save > SLZ_AUTO_IN, 0)) { + strm->tree_switch = 1; + save = 0; + } + strm->tree_save = save; + } + + /* The reference just used (code>>16)+(dist&0x1f) bits in place + * of the 8*mlen bits these bytes would have taken as literals, + * so it saves that much on the balance, which is saturated at + * zero (no more than SLZ_CREDIT bits of savings) and saved + * there for the next group of literals and the next calls. The + * update is unconditional because a test on a value that is + * often null but not always turns into a mispredicted branch + * on binary data. */ -#if SLZ_LITERAL_SKIP - /* never on text, where it would be too expensive to - * leave the huffman block for so few literals. + code = bal + (code >> 16) + (dist & 0x1f); + bal = (code > 8 * (uint32_t)mlen) ? code - 8 * (uint32_t)mlen : 0; + strm->debt = bal; + + /* switch to the other tree after this match if the literals + * sent before it requested it, see SLZ_AUTO_IN. */ - if (ctype == SLZ_CT_GENERIC && level <= 1 && !(plit & 63)) { - long skip = rem >= skip_ratio * 64 ? skip_ratio * 64 : rem; // 0,64,192,...,8128: keeps plit aligned - skip_ratio = (skip_ratio * 2 + 1) & 127; // capped exponential: 0,1,3,7,... and stays at 127 - pos += skip; - rem -= skip; - plit += skip; - /* skipped bytes may only leave as a stored block */ - bal += SLZ_MAX_DEBT + SLZ_CREDIT; -# ifndef UNALIGNED_FASTER - /* we'll need to update the current word since we skipped some bytes */ - goto reload_word; -# else - continue; -# endif + if (auto_tree) { + if (__builtin_expect(strm->tree_switch, 0)) { + pos += mlen; + goto switch_to_text; + } } - else if (__builtin_expect(ctype == SLZ_CT_GENERIC && level == 2 && !(plit & 2047), 0)) { - long skip = rem >= skip_ratio * 4096 ? skip_ratio * 4096 : rem; - skip_ratio = (skip_ratio * 2 + 1) & 7; // capped exponential: 0,1,3,7 and stays at 7 - pos += skip; - rem -= skip; - plit += skip; - /* skipped bytes may only leave as a stored block. Also done - * on the first test, where is 0, like the stored - * block sent there before: it keeps the same matches. - */ - bal += SLZ_MAX_DEBT + SLZ_CREDIT; -# ifndef UNALIGNED_FASTER - /* we'll need to update the current word since we skipped some bytes */ - goto reload_word; -# else - continue; -# endif + rem -= mlen; + pos += mlen; + +#if SLZ_INSERT_SKIPPED + if (rem < 4) + break; + /* try to index the matched entry to improve the likelyhood + * that a similar sequence will be a hit next time. We only + * need 4 remaining bytes starting from the current one. + */ + insert_ref(refs, in, pos - mlen + 3); + insert_ref(refs, in, pos - mlen + 2); + insert_ref(refs, in, pos - mlen + 1); +#endif // SLZ_INSERT_SKIPPED + +#ifndef UNALIGNED_FASTER + /* same as before the loop, this is only used when continuing + * in the loop, so let's use the same condition (rem>=4). + */ + if (rem >= 4) { +#ifdef UNALIGNED_LE_OK + word = *(uint32_t *)&in[pos - 1]; +#else + word = ((uint32_t)(unsigned char)in[pos] << 8) + ((uint32_t)(unsigned char)in[pos + 1] << 16) + ((uint32_t)(unsigned char)in[pos + 2] << 24); +#endif } - else if (ctype != SLZ_CT_GENERIC || level >= 3) // no literal skip at levels >= 3 #endif - { - /* either no literal_skip at build time, or high enough - * level. With the text tree, the stored blocks are as - * large as possible since each split costs 5 bytes on - * top of the debt. + } + + if (__builtin_expect(rem, 0)) { + /* we're reading the 1..3 last bytes */ + plit += rem; + do { + bal += ((unsigned char)in[pos++] >= 144); + } while (--rem); + } + + /* Now copy remaining literals or mark the end. The cost of switching to + * a stored block depends on the current state and on the presence of + * data after these literals, see the SLZ_*_COST definitions above. + */ + if (plit) { + uint32_t cost; + + if (more) + cost = SLZ_SWITCH_COST; + else + cost = (strm->state == SLZ_ST_EOB) ? SLZ_LAST_COST_EOB : SLZ_LAST_COST; + + if (bal >= cost + SLZ_CREDIT) + copy_lit(strm, in + pos - plit, plit, more, auto_tree); + else { + copy_lit_huff(strm, in + pos - plit, plit, more, SLZ_CT_GENERIC, auto_tree); + strm->debt = bal; + } + + plit = 0; + } + goto leave; + } + + /* The text tree's part, with its own loop, used when it's chosen with + * the automatic tree. It is the same at all levels: no literal skip, + * debt charged per literal () and 2-byte backward match. + */ + if (auto_tree) { + uint32_t lit_excess; /* see SLZ_TEXT_STORED_COST */ + int lit_stored; /* pending literals go to a stored block */ + uint32_t nvref; /* recorded references, see SLZ_TEXT_VREFS */ + uint32_t vplit; /* after the last recorded reference */ + struct text_vref vref[SLZ_TEXT_VREFS]; + + text_start: + rem = ilen - pos; + plit = 0; + lit_excess = 0; + lit_stored = 0; + nvref = 0; + vplit = 0; + +#ifndef UNALIGNED_FASTER + if (rem >= 4) // is only used inside the loop below, hence the test for >= 4 + word = ((uint32_t)(unsigned char)in[pos] << 8) + ((uint32_t)(unsigned char)in[pos + 1] << 16) + ((uint32_t)(unsigned char)in[pos + 2] << 24); +#endif + while (rem >= 4) { + int32_t cost; + +#ifndef UNALIGNED_FASTER + word = ((uint32_t)(unsigned char)in[pos + 3] << 24) + (word >> 8); +#else + word = *(uint32_t *)&in[pos]; +#endif + h = slz_hash(word); + asm volatile ("" ::); // prevent gcc from trying to be smart with the prefetch + + if (sizeof(long) >= 8) { + ent = refs[h].by64; + last = (uint32_t)ent; + ent >>= 32; + refs[h].by64 = ((uint64_t)pos) + ((uint64_t)word << 32); + } else { + ent = refs[h].by32.word; + last = refs[h].by32.pos; + refs[h].by32.pos = pos; + refs[h].by32.word = word; + } + + if ((uint32_t)ent != word) { + text_send_as_lit: + rem--; + plit++; + /* for the text tree, is the exact number of + * bits the pending literals cost over 8 bits each, as a + * signed value. */ - if (__builtin_expect(plit == 0xffc0, 0)) { - copy_lit_small(strm, in + pos - plit, plit, 1, ctype); + lit_excess += (text_huff[(unsigned char)word] & 15) - 8; + pos++; + + /* no literal skip with the text tree, where it would be too + * expensive to leave the huffman block for so few literals. + * The stored blocks are as large as possible since each split + * costs 5 bytes on top of the debt. When choosing the tree, + * the text tree checks every 64 literals instead, which costs + * the same test, and leaves for the fixed tree if they're + * binary data, see SLZ_AUTO_IN. + */ + if (__builtin_expect(auto_tree ? !(plit & 63) : plit == 0xffc0, 0)) { + if (auto_tree) { + if (plit != 0xffc0) { + if ((int32_t)lit_excess >= (int32_t)plit) { + pos -= plit; + goto switch_to_fixed; + } + continue; + } + } + copy_lit_small(strm, in + pos - plit, plit, 1, SLZ_CT_TEXT, auto_tree); plit = 0; /* the text tree's may be negative */ - if (ctype == SLZ_CT_TEXT) { - lit_excess = 0; - nvref = vplit = 0; - } - else - bal = strm->debt + SLZ_MAX_DEBT + SLZ_CREDIT; + lit_excess = 0; + nvref = vplit = 0; } + continue; } - continue; - } - /* We reject pos = last and pos > last+32768 */ - if ((unsigned long)(pos - last - 1) >= 32768) - goto send_as_lit; + /* We reject pos = last and pos > last+32768 */ + if ((unsigned long)(pos - last - 1) >= 32768) + goto text_send_as_lit; - /* faster to keep a copy of this with gcc 13-16 */ - lastp = in + last; + /* faster to keep a copy of this with gcc 13-16 */ + lastp = in + last; - /* Note: cannot encode a length larger than 258 bytes */ - mlen = memmatch(in + pos + 4, lastp + 4, (rem > 258 ? 258 : rem) - 4) + 4; + /* Note: cannot encode a length larger than 258 bytes */ + mlen = memmatch(in + pos + 4, lastp + 4, (rem > 258 ? 258 : rem) - 4) + 4; - /* found a matching entry */ + /* found a matching entry */ - /* if the pending literals are going to be sent as a stored - * block, a short match is not worth splitting it. - */ - if (ctype == SLZ_CT_TEXT) { + /* if the pending literals are going to be sent as a stored + * block, a short match is not worth splitting it. + */ if (text_lit_stored(strm, lit_excess) && mlen < 6) - goto send_as_lit; - } - else if (bal >= SLZ_SWITCH_COST + SLZ_CREDIT && mlen < 6) - goto send_as_lit; + goto text_send_as_lit; - /* compute the output code, its size and the length's size in - * bits to know if the reference is cheaper than literals. - */ - code = len_code(mlen, ctype); + /* compute the output code, its size and the length's size in + * bits to know if the reference is cheaper than literals. + */ + code = len_code(mlen, SLZ_CT_TEXT); - /* direct mapping of dist->huffman code. Computing the text - * tree's one from the fixed one instead was measured 3 to 5% - * slower than a second table. - */ - if (ctype == SLZ_CT_TEXT) + /* direct mapping of dist->huffman code. Computing the text + * tree's one from the fixed one instead was measured 3 to 5% + * slower than a second table. + */ dist = text_dist_table[pos - last - 1]; - else - dist = fh_dist_table[pos - last - 1]; - /* If encoding the dist+length is more expensive than sending - * the equivalent as bytes, lets keep the literals. This is - * important for the compression ratio and also for standing by - * our promise that the output is no longer than input + 5 bytes - * every 65535. The explanation is that a valid stream starts in - * EOB and ends in EOB, which is the state where we can grow by - * up to 5 bytes (1 byte for BT, 2 for LEN, 2 for NLEN, then - * literal data). Sending compressed Huffman data (reference or - * fixed encoding) may only be accepted if it is guaranteed that - * it will not represent more bytes in the worst case. Switching - * from EOB to FIXED requires 3 bits of BT. Then either fixed - * data (8 or 9 bits) or references (15-33 bits). Switching back - * to EOB requires EOB (7 bits), and in order to get back to the - * situation where bytes can be added with no overhead, the BT, - * align, LEN and NLEN have to be sent as well: - * - * Before switching: - * [BT] [ALIGN] [LEN] [NLEN] [PLIT] - * bits: 3 7 16 16 8*plit - * - * In case of sending mlen bytes as literals: - * [BT] [ALIGN] [LEN] [NLEN] [PLIT] [MLEN] - * bits: 3 7 16 16 8*plit 8*mlen - * - * In case of sending mlen bytes as literals, we add: - * [MLEN] - * bits: 8*mlen - * - * In case of sending reference, we add in the worst case: - * [BT] [CODE] [DIST] [EOB] [BT] [ALIGN] [LEN] [NLEN] - * bits: 3 clen dlen 7 3 7 16 16 - * - * Thus for literals we add 8*mlen bits, and for reference we - * add clen+dlen+52 bits. If the reference encoding + 52 bits - * is shorter. Of course there are plenty of opportunities to - * be much shorter once we switch to FIXED, and a stricter - * tracking could allow to send references more often. But here - * we at least guarantee that if data fit as literals, they also - * fit using a temporary switch to FIXED. - * - * Regarding encoding size, clen is 7 for mlen 3..10, to 12 for - * mlen 131..257. dlen is 8 for distances 1..4, to 21 for - * distances 16385..32768. Thus mlen <= 130 produces clen+dlen - * <= 32 bits. This means that for mlen 4 or above, the encoded - * reference is always smaller than the data it references. This - * is what guarantees that once switched to FIXED we can stay - * in it for as long as needed. - */ - if (ctype == SLZ_CT_TEXT) { /* With the text tree, the decision is taken here for * the pending literals and the reference together, and * keeps it for when the literals are sent, @@ -1365,14 +1918,14 @@ long _slz_rfc1951_encode(struct slz_stream *__restrict strm, unsigned char *out, * SLZ_TEXT_MAX_DEBT), or if it doesn't open a block and * saves bits together with the literals. */ - int32_t cost = (int32_t)((dist & 0x1f) + (code >> 16) + lit_excess) - 8 * (int32_t)mlen; + cost = (int32_t)((dist & 0x1f) + (code >> 16) + lit_excess) - 8 * (int32_t)mlen; /* is null when there are no literals */ lit_stored = text_lit_stored(strm, lit_excess); if (__builtin_expect(strm->state != SLZ_ST_EOB && !lit_stored, 1)) { /* literals and reference in the open block */ if (cost > 0 && (int32_t)strm->text_debt + cost >= SLZ_TEXT_MAX_DEBT) - goto send_as_lit; + goto text_send_as_lit; } else { /* the reference (re)opens the block */ @@ -1382,7 +1935,7 @@ long _slz_rfc1951_encode(struct slz_stream *__restrict strm, unsigned char *out, if (strm->state != SLZ_ST_EOB || lit_stored || nvref >= SLZ_TEXT_VREFS || plit + mlen >= 0xffc0 || plit - vplit > 0xffff) - goto send_as_lit; + goto text_send_as_lit; /* record it, see SLZ_TEXT_VREFS */ vref[nvref].code = code; @@ -1393,353 +1946,612 @@ long _slz_rfc1951_encode(struct slz_stream *__restrict strm, unsigned char *out, lit_excess += (dist & 0x1f) + (code >> 16) - 8 * (uint32_t)mlen; plit += mlen; vplit = plit; - goto skip_ref; + goto text_skip_ref; } } - } - /* Note: is at least SLZ_MAX_DEBT + SLZ_CREDIT in stored - * mode, except at the beginning of a call (see SLZ_MAX_DEBT). - */ - else if (bal >= SLZ_MAX_DEBT + SLZ_CREDIT && - (dist & 0x1f) + (code >> 16) + SLZ_SWITCH_COST > 8 * mlen) - goto send_as_lit; - -#if defined(SLZ_BACKWARD_MATCH) - /* Compare queued literals in case the previous ones were not - * found due to hash collisions but still match. We don't go - * further than 8 bytes so that we can find them in a single - * 64-bit comparison. On aligned-only machines we read one byte - * at a time, as it is expected to be cheaper than composing an - * unaligned int and manually doing a ctzll() on it. Tests have - * shown that there's little to gain past 8 bytes, and anyway - * the total match length will be bounded by 258. BTW we only - * consider cases where last >= 8 because this guarantees that - * we can access both the searched sequence and the match found - * on a wole word at once. It may miss some rare matches on the - * edge of the buffer that we don't care about. - * - * It's worth noting that we *do not* fixup if a match - * is found, as this would be too complicated and counter- - * productive. This means that literals that were finally - * encoded might still count towards the budget limit, but this - * is not dramatic and keeps us on the safe side. - * - * The result of the search is to only touch that will be - * used all along in all operations. Multiple attempts have shown - * that it's both easier and faster this way than changing then - * rolling back offsets when we finally give up and send_as_lit. - */ - if (((ctype == SLZ_CT_TEXT && __builtin_expect(plit != vplit, 1)) || - (ctype != SLZ_CT_TEXT && __builtin_expect(plit != 0, 1))) && - (last >= MAX_C(sizeof(long), SLZ_BACKWARD_MATCH[level]))) { - long bmax = MIN(plit - (ctype == SLZ_CT_TEXT ? vplit : 0), 8); // keep it at 8, always faster - long back = 0; - - if (bmax > 258 - mlen) - bmax = 258 - mlen; - -#if defined(UNALIGNED_LE_OK) - if (SLZ_BACKWARD_MATCH[level] == 2) { // 43.32%, best speed vs savings - uint16_t x = *(uint16_t *)&in[pos - 2] ^ *(uint16_t *)(lastp - 2); - back = x & 0xff00 ? 0 : x ? 1 : 2; - if (back > bmax) - back = bmax; - } else if (SLZ_BACKWARD_MATCH[level] == 3 || SLZ_BACKWARD_MATCH[level] == 4) { // 32 bits: 43.31% or 43.30% - uint32_t x = *(uint32_t *)&in[pos - 4] ^ *(uint32_t *)(lastp - 4); - back = x & 0xffff0000u ? - x & 0xff000000u ? 0 : 1 : - x & 0xffffff00u ? 2 : - x ? 3 : SLZ_BACKWARD_MATCH[level]; // 3 or 4, test eliminated at 3 - if (back > bmax) - back = bmax; - } else if (SLZ_BACKWARD_MATCH[level] == 8) { // 64 bits: 43.30% - uint64_t x = *(uint64_t *)&in[pos - 8] ^ *(uint64_t *)(lastp - 8); - back = x ? (__builtin_clzll(x) >> 3) : 8; - if (back > bmax) - back = bmax; - } else -#endif - if (SLZ_BACKWARD_MATCH[level] == 1) { // single-byte match: 43.39% - if (in[pos - 1] == lastp[-1]) - back++; - if (back > bmax) - back = bmax; - } - else { /* unaligned not supported or match != 1/2/3/4/8 */ - while (back < bmax && in[pos - back - 1] == lastp[-back - 1]) - back++; - } - - if (back) { - /* OK now we're certain to use the match, we must rewind the - * literals that are part of the match. - */ - pos -= back; - plit -= back; - rem += back; - mlen += back; - /* was computed from the un-extended - * length above and must follow, or we would - * consume more bytes than the length - * code tells the decoder to copy. does - * not move. The cost test above keeps using - * the un-extended length, which is safe: the - * extension only makes the match cheaper per - * byte, never more expensive. - */ - code = len_code(mlen, ctype); - - /* Not fixing above is only safe when it - * overestimates the cost of the literals. With - * the text tree, these ones most likely saved - * bits, which would be counted twice since the - * match is credited 8 bits per byte as well. - */ - if (ctype == SLZ_CT_TEXT) { - long i; + /* Compare queued literals, see the fixed tree's loop. Only + * those after the last recorded reference may be used. + */ + if (__builtin_expect(plit != vplit, 1) && + (last >= MAX_C(sizeof(long), 2))) { + long bmax = MIN(plit - vplit, 8); // keep it at 8, always faster + long back, i; + + if (bmax > 258 - mlen) + bmax = 258 - mlen; + + /* 2 bytes, which proved best for the text tree */ + back = backward_match(in, pos, lastp, bmax, 2); + + if (back) { + /* rewind the literals that are part of the match, see + * the fixed tree's loop. + */ + pos -= back; + plit -= back; + rem += back; + mlen += back; + code = len_code(mlen, SLZ_CT_TEXT); + + /* Not fixing above is only safe when it + * overestimates the cost of the literals. With + * the text tree, these ones most likely saved + * bits, which would be counted twice since the + * match is credited 8 bits per byte as well. + */ for (i = 0; i < back; i++) lit_excess -= (text_huff[in[pos + i]] & 15) - 8; } } - } -#endif // SLZ_BACKWARD_MATCH // otherwise 43.83% - /* first, copy pending literals */ - if (plit) { - /* Huffman encoding requires 9 bits for octets 144..255, so this - * is a waste of space for binary data. Switching between Huffman - * and no-comp then huffman consumes 52 bits (7 for EOB + 3 for - * block type + 7 for alignment + 32 for LEN+NLEN + 3 for next - * block). The balance already counts these literals, so once - * it is SLZ_SWITCH_COST above the credit, the literals leave - * as a stored block and are removed from it by restoring the - * value it had after the last reference. Literals sent in a - * stored block do not cost anything extra. - */ - if (ctype == SLZ_CT_TEXT) { + /* first, copy pending literals */ + if (plit) { if (lit_stored) - copy_lit_small(strm, in + pos - plit, plit, 1, ctype); + copy_lit_small(strm, in + pos - plit, plit, 1, SLZ_CT_TEXT, auto_tree); else { int32_t debt; if (__builtin_expect(nvref != 0, 0)) text_replay(strm, in + pos - plit, plit, vref, nvref, 1); else - copy_lit_huff(strm, in + pos - plit, plit, 1, ctype); + copy_lit_huff(strm, in + pos - plit, plit, 1, SLZ_CT_TEXT, auto_tree); debt = strm->text_debt + (int32_t)lit_excess; strm->text_debt = debt > 0 ? debt : 0; } + if (auto_tree) + auto_account(strm, (int32_t)lit_excess, SLZ_CT_TEXT); nvref = vplit = 0; - } - else if (bal >= SLZ_SWITCH_COST + SLZ_CREDIT) { - copy_lit(strm, in + pos - plit, plit, 1, ctype); - bal = strm->debt; - } - else { - if (level <= 1) - bal += plit; // worst case: all of them take 9 bits - copy_lit_huff(strm, in + pos - plit, plit, 1, ctype); - } - plit = 0; - } + plit = 0; + } -#if SLZ_LITERAL_SKIP - skip_ratio = 0; -#endif - /* use mode 01 - fixed huffman */ - if (strm->state == SLZ_ST_EOB) { - strm->state = SLZ_ST_FIXED; - open_huff_block(strm, 0, ctype); - /* restart from the saved balance (drops the stored mode mark) - * plus what leaving fixed huffman will cost - */ - if (ctype == SLZ_CT_GENERIC) - bal = strm->debt + SLZ_SWITCH_COST; - } + /* open a text block (state "FIXED" means a huffman block) */ + if (strm->state == SLZ_ST_EOB) { + strm->state = SLZ_ST_FIXED; + open_huff_block(strm, 0, SLZ_CT_TEXT); + } - /* copy the length first */ + /* copy the length first */ #if SLZ_DIRECT_ENQUEUE56 - /* the length code and the distance code that follows it total - * 31 bits at most, which one enqueue can take in a single - * shift-or-store instead of two - */ - enqueue56(strm, (uint64_t)(code & 0xFFFF) | - ((uint64_t)(dist >> 5) << (code >> 16)), - (code >> 16) + (dist & 0x1f)); + /* the length code and the distance code that follows it total + * 31 bits at most, which one enqueue can take in a single + * shift-or-store instead of two + */ + enqueue56(strm, (uint64_t)(code & 0xFFFF) | + ((uint64_t)(dist >> 5) << (code >> 16)), + (code >> 16) + (dist & 0x1f)); #else - enqueue24(strm, code & 0xFFFF, code >> 16); + enqueue24(strm, code & 0xFFFF, code >> 16); - /* in fixed huffman mode, dist is fixed 5 bits */ - enqueue24(strm, dist >> 5, dist & 0x1f); + /* the distance's code and size */ + enqueue24(strm, dist >> 5, dist & 0x1f); #endif - /* The reference just used (code>>16)+(dist&0x1f) bits in place - * of the 8*mlen bits these bytes would have taken as literals, - * so it saves that much on the balance, which is saturated at - * zero (no more than SLZ_CREDIT bits of savings) and saved - * there for the next group of literals and the next calls. The - * update is unconditional because a test on a value that is - * often null but not always turns into a mispredicted branch - * on binary data. - */ - if (ctype == SLZ_CT_TEXT) { + /* the reference's savings repay the debt, see SLZ_TEXT_MAX_DEBT */ code = strm->text_debt + (code >> 16) + (dist & 0x1f); strm->text_debt = (code > 8 * (uint32_t)mlen) ? code - 8 * (uint32_t)mlen : 0; lit_excess = 0; - } else { - code = bal + (code >> 16) + (dist & 0x1f); - bal = (code > 8 * (uint32_t)mlen) ? code - 8 * (uint32_t)mlen : 0; - strm->debt = bal; - } - skip_ref: - rem -= mlen; - pos += mlen; + + /* switch to the other tree after this match if the literals + * sent before it requested it, see SLZ_AUTO_IN. + */ + if (auto_tree) { + if (__builtin_expect(strm->tree_switch, 0)) { + pos += mlen; + goto switch_to_fixed; + } + } + text_skip_ref: + rem -= mlen; + pos += mlen; #if SLZ_INSERT_SKIPPED - if (rem < 4) - break; - /* try to index the matched entry to improve the likelyhood - * that a similar sequence will be a hit next time. We only - * need 4 remaining bytes starting from the current one. - */ - insert_ref(refs, in, pos - mlen + 3); - insert_ref(refs, in, pos - mlen + 2); - insert_ref(refs, in, pos - mlen + 1); + if (rem < 4) + break; + insert_ref(refs, in, pos - mlen + 3); + insert_ref(refs, in, pos - mlen + 2); + insert_ref(refs, in, pos - mlen + 1); #endif // SLZ_INSERT_SKIPPED #ifndef UNALIGNED_FASTER - /* same as before the loop, this is only used when continuing - * in the loop, so let's use the same condition (rem>=4). - */ - if (rem >= 4) { + if (rem >= 4) { #ifdef UNALIGNED_LE_OK - word = *(uint32_t *)&in[pos - 1]; + word = *(uint32_t *)&in[pos - 1]; #else - word = ((uint32_t)(unsigned char)in[pos] << 8) + ((uint32_t)(unsigned char)in[pos + 1] << 16) + ((uint32_t)(unsigned char)in[pos + 2] << 24); + word = ((uint32_t)(unsigned char)in[pos] << 8) + ((uint32_t)(unsigned char)in[pos + 1] << 16) + ((uint32_t)(unsigned char)in[pos + 2] << 24); #endif - } + } #endif - } + } - if (__builtin_expect(rem, 0)) { - /* we're reading the 1..3 last bytes */ - plit += rem; - do { - if (ctype == SLZ_CT_TEXT) + if (__builtin_expect(rem, 0)) { + /* we're reading the 1..3 last bytes */ + plit += rem; + do { lit_excess += (text_huff[in[pos++]] & 15) - 8; - else - bal += ((unsigned char)in[pos++] >= 144); - } while (--rem); - } + } while (--rem); + } - /* Now copy remaining literals or mark the end. The cost of switching to - * a stored block depends on the current state and on the presence of - * data after these literals, see the SLZ_*_COST definitions above. - */ - if (plit && ctype == SLZ_CT_TEXT) { - int32_t lit_cost; - - /* same with the text tree, in EOB state the huffman encoding - * has to pay for the block type and the header, and the EOB - * when nothing follows. When nothing follows and a text block - * is already open, the literals stay in it, as reopening one - * only to set BFINAL would cost a second header. The stream - * is then terminated by slz_finish() with an EOB and an empty - * final fixed block (10 bits), while leaving the block for a - * final stored one costs the EOB and SLZ_TEXT_STORED_COST. + /* Now copy remaining literals or mark the end. The cost of switching to + * a stored block depends on the current state and on the presence of + * data after these literals, see the SLZ_*_COST definitions above. */ - if (more) - lit_cost = (strm->state == SLZ_ST_EOB) ? - SLZ_TEXT_STORED_COST - SLZ_TEXT_OPEN_COST : SLZ_TEXT_SWITCH_COST; - else - lit_cost = (strm->state == SLZ_ST_EOB) ? - SLZ_TEXT_STORED_COST - SLZ_TEXT_OPEN_COST - TEXT_EOB_BITS : SLZ_TEXT_STORED_COST - 10; + if (plit) { + int32_t lit_cost; + + /* same with the text tree, in EOB state the huffman encoding + * has to pay for the block type and the header, and the EOB + * when nothing follows. When nothing follows and a text block + * is already open, the literals stay in it, as reopening one + * only to set BFINAL would cost a second header. The stream + * is then terminated by slz_finish() with an EOB and an empty + * final fixed block (10 bits), while leaving the block for a + * final stored one costs the EOB and SLZ_TEXT_STORED_COST. + */ + if (more) + lit_cost = (strm->state == SLZ_ST_EOB) ? + SLZ_TEXT_STORED_COST - SLZ_TEXT_OPEN_COST : SLZ_TEXT_SWITCH_COST; + else + lit_cost = (strm->state == SLZ_ST_EOB) ? + SLZ_TEXT_STORED_COST - SLZ_TEXT_OPEN_COST - TEXT_EOB_BITS : SLZ_TEXT_STORED_COST - 10; - if ((int32_t)lit_excess >= lit_cost || - (int32_t)strm->text_debt + (int32_t)lit_excess + - (strm->state == SLZ_ST_EOB ? SLZ_TEXT_SWITCH_COST : 0) >= SLZ_TEXT_MAX_DEBT) - copy_lit_small(strm, in + pos - plit, plit, more, ctype); - else { - int32_t debt; + if ((int32_t)lit_excess >= lit_cost || + (int32_t)strm->text_debt + (int32_t)lit_excess + + (strm->state == SLZ_ST_EOB ? SLZ_TEXT_SWITCH_COST : 0) >= SLZ_TEXT_MAX_DEBT) + copy_lit_small(strm, in + pos - plit, plit, more, SLZ_CT_TEXT, auto_tree); + else { + int32_t debt; + + /* the header, if any, is charged when opening the block */ + if (nvref) + text_replay(strm, in + pos - plit, plit, vref, nvref, more); + else + copy_lit_huff(strm, in + pos - plit, plit, more || strm->state != SLZ_ST_EOB, SLZ_CT_TEXT, auto_tree); + debt = strm->text_debt + (int32_t)lit_excess; + strm->text_debt = debt > 0 ? debt : 0; + } - /* the header, if any, is charged when opening the block */ - if (nvref) - text_replay(strm, in + pos - plit, plit, vref, nvref, more); - else - copy_lit_huff(strm, in + pos - plit, plit, more || strm->state != SLZ_ST_EOB, ctype); - debt = strm->text_debt + (int32_t)lit_excess; - strm->text_debt = debt > 0 ? debt : 0; + plit = 0; } - - plit = 0; + goto leave; } - else if (plit) { - uint32_t cost; - if (more) - cost = SLZ_SWITCH_COST; - else - cost = (strm->state == SLZ_ST_EOB) ? SLZ_LAST_COST_EOB : SLZ_LAST_COST; - - if (bal >= cost + SLZ_CREDIT) - copy_lit(strm, in + pos - plit, plit, more, ctype); - else { - copy_lit_huff(strm, in + pos - plit, plit, more, ctype); - strm->debt = bal; + /* Switching trees with the automatic tree (see SLZ_AUTO_IN): close the + * block and continue from with the other tree's part. + */ + if (auto_tree) { + switch_to_text: + if (strm->state == SLZ_ST_FIXED) { + send_eob(strm, SLZ_CT_GENERIC); + strm->state = SLZ_ST_EOB; } - - plit = 0; + strm->tree_switch = 0; + strm->tree_save = 0; + strm->cur_tree = SLZ_CT_TEXT; + debt_to_text(strm); + goto text_start; + + switch_to_fixed: + if (strm->state == SLZ_ST_FIXED) { + send_eob(strm, SLZ_CT_TEXT); + strm->state = SLZ_ST_EOB; + } + strm->tree_switch = 0; + strm->tree_save = 0; + strm->cur_tree = SLZ_CT_GENERIC; + debt_to_fixed(strm); + goto fixed_start; } leave: strm->ilen += ilen; return strm->outbuf - out; } +/* The argument of the encoder is not the compression level, it sets + * the fixed tree's literal skip and debt accounting (see SLZ_MAX_DEBT and the + * literal skip in _slz_rfc1951_encode()): + * 1: blind skip ramp from 64 to 8128 bytes, debt charged per group of + * literals (per literal without SLZ_LITERAL_SKIP); + * 2: same, with a probe every 64 skipped bytes; + * 3: skip in 2048-byte steps up to 28 kB, with a probe every 64 skipped + * bytes, debt charged per literal. + * Its argument is the number of bytes a match may be grown + * backwards over the pending literals with the fixed tree (0 to disable it). + * It costs a little CPU on every match and gains about 1% of output. Tests + * show that matching up to 2 bytes brings almost optimal results at minimal + * extra cost, and that compression gains at 1 byte are higher than the CPU + * increase they cause, and tend to result in higher compression speed due to + * the savings on encoding. At compression level 4, 8 bytes save 0.03% over 2 + * on silesia for 0.5% of speed. + * The compression levels are made of: + * level 1: encoder level 1, 1-byte backward match, fixed tree only; + * level 2: encoder level 1, 1-byte backward match, automatic tree (see + * SLZ_AUTO_IN); + * level 3: encoder level 2, 1-byte backward match, automatic tree; + * level 4: encoder level 3, 8-byte backward match, automatic tree. + * The text tree's part is the same at all levels (2-byte backward match). + */ static __attribute__((noinline)) long _slz_rfc1951_encode_lev1(struct slz_stream *__restrict strm, unsigned char *out, const unsigned char *in, long ilen, int more) { - return _slz_rfc1951_encode(strm, out, in, ilen, more, 1, SLZ_CT_GENERIC); + return _slz_rfc1951_encode(strm, out, in, ilen, more, 1, 1, 0); } static __attribute__((noinline)) long _slz_rfc1951_encode_lev2(struct slz_stream *__restrict strm, unsigned char *out, const unsigned char *in, long ilen, int more) { - return _slz_rfc1951_encode(strm, out, in, ilen, more, 2, SLZ_CT_GENERIC); + return _slz_rfc1951_encode(strm, out, in, ilen, more, 1, 1, 1); } static __attribute__((noinline)) long _slz_rfc1951_encode_lev3(struct slz_stream *__restrict strm, unsigned char *out, const unsigned char *in, long ilen, int more) { - return _slz_rfc1951_encode(strm, out, in, ilen, more, 3, SLZ_CT_GENERIC); + return _slz_rfc1951_encode(strm, out, in, ilen, more, 2, 1, 1); } -/* With the text tree, we encode everything at level 2 which proves to be both - * better and faster on all tests. +static __attribute__((noinline)) +long _slz_rfc1951_encode_lev4(struct slz_stream *__restrict strm, unsigned char *out, const unsigned char *in, long ilen, int more) +{ + return _slz_rfc1951_encode(strm, out, in, ilen, more, 3, 8, 1); +} + +/* Event encoder, used with SLZ_CT_EVENT: streams made of small events flushed + * one at a time (SSE, syslog...), where slz_rfc1951_encode() cannot find much + * since it forgets everything at each call. The ring, allocated by the caller + * (slz_ring_size bytes, zeroed) and referenced by strm->ring, keeps the last + * SLZ_EV_HIST bytes passed to the stream and a reference table between calls. + * 2 kB of history and a 9-bit table of 4-byte entries were found to be a good + * trade-off between SSE and logs, and between ratio and cost with many + * concurrent streams whose ring is out of the caches at each call, see + * experiments/events. + */ +#define SLZ_EV_HIST 2048 /* history size, a power of 2 */ +#define SLZ_EV_HASH_BITS 9 /* size of the reference table */ +#define SLZ_EV_STEP 8 /* step between the matched positions indexed */ + +struct slz_ring { + /* entries: 16 low bits of the position, and a 16-bit tag of the word */ + uint32_t refs[1 << SLZ_EV_HASH_BITS]; + unsigned char hist[SLZ_EV_HIST]; /* ring of the last bytes */ + uint32_t hoff; /* next write position in */ + uint32_t hlen; /* bytes of history in , at most SLZ_EV_HIST */ + uint32_t hend; /* stream position (strm->ilen) at the history's end */ +}; + +/* size of the ring to allocate per stream for SLZ_CT_EVENT */ +const int slz_ring_size = sizeof(struct slz_ring); + +/* the hash of the event encoder's reference table, on SLZ_EV_HASH_BITS bits */ +static inline uint32_t slz_ev_hash(uint32_t a) +{ + return slz_hash_bits(a, SLZ_EV_HASH_BITS); +} + +/* 16-bit tag of word , from bits the hash mostly ignores */ +static inline uint32_t slz_ev_tag(uint32_t w) +{ + return (uint16_t)(w ^ (w >> 16)); +} + +/* inserts absolute position

whose 4 first bytes are */ +static inline void slz_ev_insert(struct slz_ring *ring, uint32_t p, uint32_t w) +{ + ring->refs[slz_ev_hash(w)] = (uint16_t)p + (slz_ev_tag(w) << 16); +} + +/* compares up to bytes at with the source at , of which only + * are contiguous, the source then continuing at . + */ +static inline long slz_ev_match(const unsigned char *a, const unsigned char *b, long bavail, const unsigned char *in, long max) +{ + long l; + + if (max <= bavail) + return memmatch(a, b, max); + l = memmatch(a, b, bavail); + if (l < bavail) + return l; + return l + memmatch(a + l, in, max - l); +} + +/* Encodes bytes from into for stream using the ring + * at strm->ring, see above. It is a copy of the fixed tree's part of + * _slz_rfc1951_encode(), whose comments apply here, adapted to the ring and + * simplified. The references may point into the history kept + * in the ring, which is matched in place: from the oldest byte to the ring's + * end, then from the ring's start, then into the input. Positions are counted + * from the start of the stream (strm->ilen), so that the entries from the + * previous calls remain valid, and every hit is verified by reading the + * source bytes, which also rejects the entries aliased by the 16-bit + * positions or by the 32-bit wrapping. The level is ignored (except 0, which + * never comes here), it always uses the fixed tree, never skips literals, and + * grows matches up to 8 bytes backwards. One matched position in SLZ_EV_STEP + * is indexed: with a step of 8, the first indexed position after the point + * where the data diverge is at most 8 bytes away, and the backward match + * recovers these bytes. The input is appended to the ring on exit. */ static __attribute__((noinline)) -long _slz_rfc1951_encode_text(struct slz_stream *__restrict strm, unsigned char *out, const unsigned char *in, long ilen, int more) +long _slz_rfc1951_ev_encode(struct slz_stream *__restrict strm, unsigned char *out, const unsigned char *in, long ilen, int more) { - /* only the savings of this call may pay for its blocks, see - * SLZ_TEXT_CREDIT. + struct slz_ring *ring = strm->ring; + const unsigned char *hist = ring->hist; + /* the history is lost if some data were passed without the ring */ + uint32_t hlen = (ring->hend == strm->ilen) ? ring->hlen : 0; + uint32_t hstart = (ring->hoff - hlen) & (SLZ_EV_HIST - 1); /* oldest byte */ + uint32_t base = strm->ilen; /* absolute position of in[0] */ + uint32_t hbase = base - hlen; /* absolute position of the oldest byte */ + long pos = 0, rem = ilen; + uint32_t plit = 0; + uint32_t bal; + const char *p; + int k; + + strm->outbuf = out; + + /* on busy proxies, nothing of the ring is left in the caches since + * the previous event, and the table is accessed randomly. */ - if (strm->text_debt < SLZ_TEXT_CREDIT) - strm->text_debt = SLZ_TEXT_CREDIT; - return _slz_rfc1951_encode(strm, out, in, ilen, more, 2, SLZ_CT_TEXT); + for (p = (const char *)ring->refs; p < (const char *)ring->refs + sizeof(ring->refs); p += 64) + __builtin_prefetch(p, 1); + + /* only the fixed tree here, see SLZ_AUTO_IN */ + leave_text_tree(strm); + strm->tree_switch = 0; + + /* the last 3 positions of the history could not be indexed before + * their 4th byte was known. + */ + for (k = 3; k >= 1; k--) { + unsigned char tmp[4]; + int j; + + if (hlen < (uint32_t)k || ilen < 4 - k) + continue; + for (j = 0; j < k; j++) + tmp[j] = hist[(hstart + hlen - k + j) & (SLZ_EV_HIST - 1)]; + for (; j < 4; j++) + tmp[j] = in[j - k]; + slz_ev_insert(ring, base - k, read_le32(tmp)); + } + + bal = strm->debt; + + while (rem >= 4) { + uint32_t word = read_le32(in + pos); + uint32_t h = slz_ev_hash(word); + uint32_t cur = base + pos; + uint32_t ent = ring->refs[h]; + uint32_t last, off, dist, code, d; + const unsigned char *src; + long savail, mlen, max, sback; + + ring->refs[h] = (uint16_t)cur + (slz_ev_tag(word) << 16); + if ((ent >> 16) != slz_ev_tag(word)) + goto send_as_lit; + + /* the most recent position with these 16 low bits, at most + * 65536 bytes back: older entries alias to it and are rejected + * by the range check or the comparison below. + */ + last = cur - (uint16_t)(cur - ent); + + /* must be within the history or the input before */ + off = last - hbase; + if (off >= hlen + (uint32_t)pos) + goto send_as_lit; + dist = hlen + pos - off; + if (dist > 32768) + goto send_as_lit; + + max = rem > 258 ? 258 : rem; + if (off < hlen) { + /* in the history: the source may run to the ring's + * end, then continue at its start, then into the + * input. is what can be read before it in + * the same segment. + */ + uint32_t ph = (hstart + off) & (SLZ_EV_HIST - 1); + long wrap = SLZ_EV_HIST - ph; + + src = hist + ph; + savail = hlen - off; + sback = off < ph ? off : ph; + if (savail > wrap) { + long l = memmatch(in + pos, src, wrap < max ? wrap : max); + + mlen = l; + if (l == wrap && l < max) + mlen += slz_ev_match(in + pos + l, hist, savail - wrap, in, max - l); + } + else + mlen = slz_ev_match(in + pos, src, savail, in, max); + } + else { + src = in + (off - hlen); + sback = off - hlen; + mlen = memmatch(in + pos, src, max); + } + + /* this also verifies the word */ + if (mlen < 4) + goto send_as_lit; + + if (bal >= SLZ_SWITCH_COST + SLZ_CREDIT && mlen < 6) + goto send_as_lit; + + code = len_code(mlen, SLZ_CT_GENERIC); + d = fh_dist_table[dist - 1]; + + /* see _slz_rfc1951_encode() */ + if (bal >= SLZ_MAX_DEBT + SLZ_CREDIT && + (d & 0x1f) + (code >> 16) + SLZ_SWITCH_COST > 8 * mlen) + goto send_as_lit; + + /* grow the match backwards over up to 8 pending literals, + * within the source's segment, see the backward match in + * _slz_rfc1951_encode(). 8 bytes are compared at once when + * available before both sides, otherwise one at a time. + */ + if (plit) { + long bmax = MIN(plit, 8); + long back; + + if (bmax > 258 - mlen) + bmax = 258 - mlen; + if (bmax > sback) + bmax = sback; + if (MIN((long)pos, sback) >= 8) + back = backward_match(in, pos, src, bmax, 8); + else + back = backward_match(in, pos, src, bmax, 0); + if (back) { + pos -= back; + plit -= back; + rem += back; + mlen += back; + code = len_code(mlen, SLZ_CT_GENERIC); + } + } + + if (plit) { + if (bal >= SLZ_SWITCH_COST + SLZ_CREDIT) { + copy_lit(strm, in + pos - plit, plit, 1, 0); + bal = strm->debt; + } + else + copy_lit_huff(strm, in + pos - plit, plit, 1, SLZ_CT_GENERIC, 0); + plit = 0; + } + + if (strm->state == SLZ_ST_EOB) { + strm->state = SLZ_ST_FIXED; + open_huff_block(strm, 0, SLZ_CT_GENERIC); + bal = strm->debt + SLZ_SWITCH_COST; + } + +#if SLZ_DIRECT_ENQUEUE56 + /* see _slz_rfc1951_encode() */ + enqueue56(strm, (uint64_t)(code & 0xFFFF) | + ((uint64_t)(d >> 5) << (code >> 16)), + (code >> 16) + (d & 0x1f)); +#else + enqueue24(strm, code & 0xFFFF, code >> 16); + enqueue24(strm, d >> 5, d & 0x1f); +#endif + + code = bal + (code >> 16) + (d & 0x1f); + bal = (code > 8 * (uint32_t)mlen) ? code - 8 * (uint32_t)mlen : 0; + strm->debt = bal; + + rem -= mlen; + pos += mlen; + + /* Index the matched positions, also when the match reaches the + * end of the input since the next calls need them. Those whose + * 4 bytes are not all there yet are indexed by the next call. + */ + { + long i = pos - mlen + 1; + long e = pos; + + if (e > ilen - 3) + e = ilen - 3; + for (; i < e; i += SLZ_EV_STEP) + slz_ev_insert(ring, base + i, read_le32(in + i)); + } + continue; + + send_as_lit: + bal += ((unsigned char)word >= 144); + plit++; + pos++; + rem--; + if (__builtin_expect(plit == 0xffc0, 0)) { + copy_lit_small(strm, in + pos - plit, plit, 1, SLZ_CT_GENERIC, 0); + plit = 0; + bal = strm->debt + SLZ_MAX_DEBT + SLZ_CREDIT; + } + } + + if (rem) { + plit += rem; + do { + bal += (in[pos++] >= 144); + } while (--rem); + } + + if (plit) { + uint32_t cost; + + if (more) + cost = SLZ_SWITCH_COST; + else + cost = (strm->state == SLZ_ST_EOB) ? SLZ_LAST_COST_EOB : SLZ_LAST_COST; + + if (bal >= cost + SLZ_CREDIT) + copy_lit(strm, in + pos - plit, plit, more, 0); + else { + copy_lit_huff(strm, in + pos - plit, plit, more, SLZ_CT_GENERIC, 0); + strm->debt = bal; + } + } + + /* append the input to the ring */ + if (ilen >= SLZ_EV_HIST) { + memcpy(ring->hist, in + ilen - SLZ_EV_HIST, SLZ_EV_HIST); + ring->hoff = 0; + ring->hlen = SLZ_EV_HIST; + } + else { + uint32_t hoff = ring->hoff; + uint32_t n1 = SLZ_EV_HIST - hoff; + + if (n1 > ilen) + n1 = ilen; + memcpy(ring->hist + hoff, in, n1); + memcpy(ring->hist, in + n1, ilen - n1); + ring->hoff = (hoff + ilen) & (SLZ_EV_HIST - 1); + ring->hlen = hlen + ilen > SLZ_EV_HIST ? SLZ_EV_HIST : hlen + ilen; + } + ring->hend = base + ilen; + + strm->ilen += ilen; + return strm->outbuf - out; } long slz_rfc1951_encode(struct slz_stream *__restrict strm, unsigned char *out, const unsigned char *in, long ilen, int more) { - if (strm->content_type == SLZ_CT_TEXT) - return _slz_rfc1951_encode_text(strm, out, in, ilen, more); + int level = strm->level; - if (strm->level <= 1) + /* events: needs a ring, otherwise the same as no hint */ + if (__builtin_expect(strm->content_type == SLZ_CT_EVENT, 0) && strm->ring && level) + return _slz_rfc1951_ev_encode(strm, out, in, ilen, more); + + /* mostly text: levels 1 and 2 work as level 3, see + * slz_set_content_type(). + */ + if (strm->content_type == SLZ_CT_TEXT && level && level < 3) + level = 3; + + if (level <= 1) { + /* level 1 (and 0) only use the fixed tree */ + if (__builtin_expect(strm->cur_tree != SLZ_CT_GENERIC, 0)) { + long n; + + strm->outbuf = out; + leave_text_tree(strm); + n = strm->outbuf - out; + return n + _slz_rfc1951_encode_lev1(strm, out + n, in, ilen, more); + } return _slz_rfc1951_encode_lev1(strm, out, in, ilen, more); - else if (strm->level <= 2) + } + else if (level <= 2) return _slz_rfc1951_encode_lev2(strm, out, in, ilen, more); - else + else if (level <= 3) return _slz_rfc1951_encode_lev3(strm, out, in, ilen, more); + else + return _slz_rfc1951_encode_lev4(strm, out, in, ilen, more); } /* Initializes stream for use with raw deflate (rfc1951). The CRC is @@ -1752,74 +2564,87 @@ int slz_rfc1951_init(struct slz_stream *strm, int level) strm->state = SLZ_ST_EOB; // no header strm->level = MIN(MAX(level, 0), 15); strm->format = SLZ_FMT_DEFLATE; - strm->content_type = SLZ_CT_GENERIC; + strm->ring = NULL; strm->crc32 = 0; strm->ilen = 0; strm->qbits = 0; strm->queue = 0; strm->debt = SLZ_CREDIT; strm->text_debt = SLZ_TEXT_CREDIT; + strm->content_type = SLZ_CT_GENERIC; + strm->cur_tree = (strm->level >= 2) ? SLZ_CT_TEXT : SLZ_CT_GENERIC; // see SLZ_AUTO_IN + strm->tree_save = 0; + strm->tree_switch = 0; return 0; } -/* Flushes any pending data for stream into buffer , then emits an - * empty literal block to byte-align the output, allowing to completely flush - * the queue. This requires that the output buffer still has the size of the - * queue (up to 31 bits when using the 64-bit queue), plus a possible EOB (7 - * bits), plus (BFINAL,BTYPE) (3 bits), which once rounded up to the next byte - * make 6 bytes, plus 4 bytes for LEN+NLEN, or a total of 10 bytes in the worst - * case. If the stream was already completed by a previous call to encode() - * with cleared, the last block is simply terminated and the output - * aligned, since nothing may be appended to a finished stream. The number of - * bytes emitted is returned. It is guaranteed that the queue is empty on - * return. This may cause some overhead by adding needless 5-byte blocks if - * called to often. +/* Emits into buffer all the bits pending in stream at the time of + * the call, so that everything passed so far is decodable from the emitted + * bytes, and returns their number. Pending bits ending on a byte boundary are + * simply emitted. Otherwise, the least is added to cross the boundary, + * depending on the state: + * - inside a block: its EOB, which is at least 7 bits long. The next encode + * call opens a new block. + * - inside the last block: its EOB, then padding to the next byte since + * nothing may follow it. The stream is then complete. + * - after the last block: padding, for the same reason. + * - between blocks: an empty fixed block (10 bits). + * Only the complete bytes are emitted, so that the bits added beyond the + * boundary may stay queued, the next call emitting them. The output is then + * not aligned on return. It costs 7 bits plus the next block's 3 header bits + * where older versions emitted an empty stored block to byte-align the + * output, or about 48 bits. This requires that the output buffer still has + * the size of the queue (up to 31 bits when using the 64-bit queue) plus up + * to 11 bits, rounded up to the next byte, or 6 bytes in the worst case. */ int slz_rfc1951_flush(struct slz_stream *strm, unsigned char *buf) { strm->outbuf = buf; - /* The queue is always empty on INIT, DONE, and END */ - if (!strm->qbits) - return 0; - - /* we may need to terminate a huffman output. Lit is always in EOB state */ - if (strm->state != SLZ_ST_EOB) { - strm->state = (strm->state == SLZ_ST_LAST) ? SLZ_ST_DONE : SLZ_ST_EOB; - send_eob(strm, strm->content_type); + if (strm->qbits & 7) { + switch (strm->state) { + case SLZ_ST_FIXED: + send_eob(strm, strm->cur_tree); + strm->state = SLZ_ST_EOB; + break; + case SLZ_ST_LAST: + send_eob(strm, strm->cur_tree); + strm->state = SLZ_ST_DONE; + /* fall through */ + case SLZ_ST_DONE: + case SLZ_ST_END: + flush_bits(strm); + return strm->outbuf - buf; + default: + /* an empty fixed block, whatever the tree + * since it's the cheapest one, as in finish(). + */ + enqueue8(strm, 2, 3); // BFINAL = 0 ; BTYPE = 01 + send_eob(strm, SLZ_CT_GENERIC); + break; + } } - if (strm->state == SLZ_ST_DONE) { - /* The block we've just terminated was already marked final, so - * the deflate stream is complete and nothing may be appended to - * it: an extra block here would be located past the end of the - * stream and would shift the gzip or zlib trailer that the - * caller is about to emit. Aligning the output is all we have - * to do, and it's all that's left to do for finish() as well. - */ - flush_bits(strm); - return strm->outbuf - buf; + /* emit the complete bytes only */ + while (strm->qbits >= 8) { + *strm->outbuf++ = strm->queue; + strm->queue >>= 8; + strm->qbits -= 8; } - - /* send BFINAL=0 and BTYPE=00 (lit) */ - enqueue8(strm, 0, 3); - flush_bits(strm); // emit pending bits - copy_32b(strm, 0xFFFF0000U); // len=0, nlen=~0 - - /* Now the queue is empty, EOB was sent, and a zero-byte block was sent - * to byte-align the output. The last state reflects all this. Let's - * just return the number of bytes added to the output buffer. - */ return strm->outbuf - buf; } +/* the EOB must cross the byte boundary in slz_rfc1951_flush() */ +typedef char slz_text_eob_bits_check[(TEXT_EOB_BITS >= 7) ? 1 : -1]; + /* Flushes any pending for stream into buffer , then sends BTYPE=1 * and BFINAL=1 if needed. The stream ends in SLZ_ST_DONE. It returns the number * of bytes emitted. The trailer consists in flushing the possibly pending bits - * from the queue (up to 31 bits when using the 64-bit queue), then possibly - * EOB (7 bits), then 3 bits, EOB, a rounding to the next byte, which amounts - * to a total of 6 bytes max, that the caller must ensure are available before - * calling the function. + * from the queue (up to 31 bits when the queue is not stored after each + * enqueue, see SLZ_DIRECT_ENQUEUE24), then possibly the EOB (7 bits with the + * fixed tree, 11 with the text one), then 3 bits, EOB, a rounding to the next + * byte, which amounts to a total of 7 bytes max, that the caller must ensure + * are available before calling the function. */ int slz_rfc1951_finish(struct slz_stream *strm, unsigned char *buf) { @@ -1827,12 +2652,12 @@ int slz_rfc1951_finish(struct slz_stream *strm, unsigned char *buf) if (strm->state == SLZ_ST_FIXED || strm->state == SLZ_ST_LAST) { strm->state = (strm->state == SLZ_ST_LAST) ? SLZ_ST_DONE : SLZ_ST_EOB; - send_eob(strm, strm->content_type); + send_eob(strm, strm->cur_tree); } if (strm->state != SLZ_ST_DONE) { /* send BTYPE=1, BFINAL=1. This is an empty fixed block whatever - * the content type since it's the cheapest one. + * the tree since it's the cheapest one. */ enqueue8(strm, 3, 3); send_eob(strm, SLZ_CT_GENERIC); @@ -2008,25 +2833,26 @@ int slz_rfc1952_init(struct slz_stream *strm, int level) strm->state = SLZ_ST_INIT; strm->level = MIN(MAX(level, 0), 15); strm->format = SLZ_FMT_GZIP; - strm->content_type = SLZ_CT_GENERIC; + strm->ring = NULL; strm->crc32 = 0; strm->ilen = 0; strm->qbits = 0; strm->queue = 0; strm->debt = SLZ_CREDIT; strm->text_debt = SLZ_TEXT_CREDIT; + strm->content_type = SLZ_CT_GENERIC; + strm->cur_tree = (strm->level >= 2) ? SLZ_CT_TEXT : SLZ_CT_GENERIC; // see SLZ_AUTO_IN + strm->tree_save = 0; + strm->tree_switch = 0; return 0; } -/* Flushes any pending data for stream into buffer , then emits an - * empty literal block to byte-align the output, allowing to completely flush - * the queue. Note that if the initial header was never sent, it will be sent - * first as well (10 extra bytes). This requires that the output buffer still - * has this plus the 6 bytes needed to flush the queue, the possible EOB and - * the (BFINAL,BTYPE) bits, plus 4 bytes for LEN+NLEN, or a total of 20 bytes - * in the worst case. The number of bytes emitted is returned. It is guaranteed - * that the queue is empty on return. This may cause some overhead by adding - * needless 5-byte blocks if called to often. +/* Makes everything passed to stream so far decodable from the bytes + * emitted into buffer , see slz_rfc1951_flush(). Note that if the initial + * header was never sent, it will be sent first as well (10 extra bytes). This + * requires that the output buffer still has this plus the 6 bytes needed to + * emit the queue and a possible EOB, or a total of 16 bytes in the worst case. + * The number of bytes emitted is returned. */ int slz_rfc1952_flush(struct slz_stream *strm, unsigned char *buf) { @@ -2044,7 +2870,7 @@ int slz_rfc1952_flush(struct slz_stream *strm, unsigned char *buf) * returns the number of bytes emitted. The trailer consists in flushing the * possibly pending bits from the queue, the possible EOB and the final empty * block, rounding to the next byte, then 4 bytes for the CRC and another 4 - * bytes for the input length. That may amount to 6+4+4 = 14 bytes, that the + * bytes for the input length. That may amount to 7+4+4 = 15 bytes, that the * caller must ensure are available before calling the function. Note that if * the initial header was never sent, it will be sent first as well (10 extra * bytes). @@ -2221,25 +3047,26 @@ int slz_rfc1950_init(struct slz_stream *strm, int level) strm->state = SLZ_ST_INIT; strm->level = MIN(MAX(level, 0), 15); strm->format = SLZ_FMT_ZLIB; - strm->content_type = SLZ_CT_GENERIC; + strm->ring = NULL; strm->crc32 = 1; // rfc1950/zlib starts with initial crc=1 strm->ilen = 0; strm->qbits = 0; strm->queue = 0; strm->debt = SLZ_CREDIT; strm->text_debt = SLZ_TEXT_CREDIT; + strm->content_type = SLZ_CT_GENERIC; + strm->cur_tree = (strm->level >= 2) ? SLZ_CT_TEXT : SLZ_CT_GENERIC; // see SLZ_AUTO_IN + strm->tree_save = 0; + strm->tree_switch = 0; return 0; } -/* Flushes any pending data for stream into buffer , then emits an - * empty literal block to byte-align the output, allowing to completely flush - * the queue. Note that if the initial header was never sent, it will be sent - * first as well (2 extra bytes). This requires that the output buffer still - * has this plus the 6 bytes needed to flush the queue, the possible EOB and - * the (BFINAL,BTYPE) bits, plus 4 bytes for LEN+NLEN, or a total of 12 bytes - * in the worst case. The number of bytes emitted is returned. It is guaranteed - * that the queue is empty on return. This may cause some overhead by adding - * needless 5-byte blocks if called to often. +/* Makes everything passed to stream so far decodable from the bytes + * emitted into buffer , see slz_rfc1951_flush(). Note that if the initial + * header was never sent, it will be sent first as well (2 extra bytes). This + * requires that the output buffer still has this plus the 6 bytes needed to + * emit the queue and a possible EOB, or a total of 8 bytes in the worst case. + * The number of bytes emitted is returned. */ int slz_rfc1950_flush(struct slz_stream *strm, unsigned char *buf) { @@ -2252,14 +3079,14 @@ int slz_rfc1950_flush(struct slz_stream *strm, unsigned char *buf) return sent; } -/* Flushes pending bits and sends the gzip trailer for stream into +/* Flushes pending bits and sends the zlib trailer for stream into * buffer . When it's done, the stream state is updated to SLZ_ST_END. It * returns the number of bytes emitted. The trailer consists in flushing the * possibly pending bits from the queue, the possible EOB and the final empty - * block, rounding to the next byte, then 4 bytes for the CRC. That may amount - * to 6+4 = 10 bytes, that the caller must ensure are available before calling - * the function. Note that if the initial header was never sent, it will be - * sent first as well (2 extra bytes). + * block, rounding to the next byte, then 4 bytes for the Adler-32. That may + * amount to 7+4 = 11 bytes, that the caller must ensure are available before + * calling the function. Note that if the initial header was never sent, it + * will be sent first as well (2 extra bytes). */ int slz_rfc1950_finish(struct slz_stream *strm, unsigned char *buf) {