From: Willy Tarreau Date: Wed, 28 Oct 2026 14:18:42 +0000 (+0100) Subject: IMPORT: slz: update slz to version 1.4-dev3 X-Git-Tag: v3.5-dev10~1 X-Git-Url: https://git.kaiwu.me/?a=commitdiff_plain;h=32582812a;p=haproxy.git IMPORT: slz: update slz to version 1.4-dev3 This refreshes the slz library to 1.4-dev3, bringing a number of improvements: - x1.4 to x20 compression speed depending on input contents, due to better handling of non-compressible data (x20, think CSS/HTML with embedded images) and many other optimizations giving at least x1.4 at the base. - slightly higher default compressed ratio (46.34% vs 46.97%) and top compressed ratio (43.03% vs 45.33%) on silesia corpus, and 23.36% vs 26.22% on linux kernel, which now compresses at 1 GB/s vs 600 MB/s before on a P-core of a Core Ultra 265k. - x2 to x3 decompression speed (mostly 2.5) depending on input stream, best gains on gzip input which was previously much slower since not as much in the target. - better support for crc32 on ARM (properly ignore older archs) - runtime detection of CPU support for PCLMULQDQ on x86_64 This version now passes all lzbench tests so it was the right moment to update it. More updates to come soon (taking content-type into account to adjust optimizations) still being worked on, but nothing critical. In order to ease updates, just like for xz or ceb, stubs were added in src/ that directly load the original files from include/import. --- diff --git a/include/import/slz-prv.h b/include/import/slz-prv.h index d4b4c840d..07719bdc0 100644 --- a/include/import/slz-prv.h +++ b/include/import/slz-prv.h @@ -48,136 +48,174 @@ #define HAVE_FAST_MULT #endif -/* Log2 of the size of the hash table used for the references table. */ -#define HASH_BITS 13 +/* returns the max of and , must be constant since evaluated twice */ +#define MIN_C(a, b) ({ \ + (a < b) ? a : b; \ +}) -/* uses the most suitable crc32 function to update crc on */ -static inline uint32_t update_crc(uint32_t crc, const void *buf, int len) +/* returns the min of and ; both must be of compatible types and are + * evaluated exactly once. + */ +#define MIN(a, b) ({ \ + typeof(a) _a = (a); \ + typeof(a) _b = (b); \ + (_a < _b) ? _a : _b; \ +}) + +/* returns the min of and ; both cast to type , are evaluated exactly + * once. + */ +#define MIN_T(t, a, b) ({ \ + t _a = (t)(a); \ + t _b = (t)(b); \ + (_a < _b) ? _a : _b; \ +}) + +/* returns the max of and , must be constant since evaluated twice */ +#define MAX_C(a, b) ({ \ + (a > b) ? a : b; \ +}) + +/* returns the max of and ; both must be of compatible types and are + * evaluated exactly once. + */ +#define MAX(a, b) ({ \ + typeof(a) _a = (a); \ + typeof(a) _b = (b); \ + (_a > _b) ? _a : _b; \ +}) + +/* returns the max of and ; both cast to type , are evaluated exactly + * once. + */ +#define MAX_T(t, a, b) ({ \ + t _a = (t)(a); \ + t _b = (t)(b); \ + (_a > _b) ? _a : _b; \ +}) + + +/* reads an unaligned 32-bit word in little endian order */ +static inline uint32_t read_le32(const void *p) { - return slz_crc32_by4(crc, buf, len); +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + const union { uint32_t u32; } __attribute__((packed)) *u = p; + return u->u32; +#else + const uint8_t *u8 = p; + return (uint32_t)u8[0] | ((uint32_t)u8[1] << 8) | + ((uint32_t)u8[2] << 16) | ((uint32_t)u8[3] << 24); +#endif } -/* This hash provides good average results on HTML contents, and is among the - * few which provide almost optimal results on various different pages. - */ -static inline uint32_t slz_hash(uint32_t a) +/* reads an unaligned 64-bit word in little endian order, same as read_le32() */ +static inline uint64_t read_le64(const void *p) { -#if defined(__ARM_FEATURE_CRC32) -# if defined(__ARM_ARCH_ISA_A64) - // 64 bit mode - __asm__ volatile("crc32w %w0,%w0,%w1" : "+r"(a) : "r"(0)); -# else - // 32 bit mode (e.g. armv7 compiler building for armv8 - __asm__ volatile("crc32w %0,%0,%1" : "+r"(a) : "r"(0)); -# endif - return a >> (32 - HASH_BITS); -#elif defined(__SSE4_2__) && defined(USE_CRC32C_HASH) - // SSE 4.2 offers CRC32C which is a bit slower than the multiply - // but provides a slightly smoother hash - __asm__ volatile("crc32l %1,%0" : "+r"(a) : "r"(0)); - return a >> (32 - HASH_BITS); -#elif defined(HAVE_FAST_MULT) - // optimal factor for HASH_BITS=12 and HASH_BITS=13 among 48k tested: 0x1af42f - return (a * 0x1af42f) >> (32 - HASH_BITS); +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + const union { uint64_t u64; } __attribute__((packed)) *u = p; + return u->u64; #else - return ((a << 19) + (a << 6) - a) >> (32 - HASH_BITS); + const uint8_t *u8 = p; + return (uint64_t)read_le32(u8) | ((uint64_t)read_le32(u8 + 4) << 32); #endif } -/* This function compares buffers and and reads 32 or 64 bits at a time - * during the approach. It makes us of unaligned little endian memory accesses - * on capable architectures. is the maximum number of bytes that can be - * read, so both and must have at least bytes ahead. may - * safely be null or negative if that simplifies computations in the caller. - */ -static inline long memmatch(const unsigned char *a, const unsigned char *b, long max) +/* reads an unaligned machine word in little endian order, same as read_le32() */ +static inline unsigned long read_lelong(const void *p) { - long len = 0; - -#ifdef UNALIGNED_LE_OK - unsigned long xor; - - while (1) { - if ((long)(len + 2 * sizeof(long)) > max) { - while (len < max) { - if (a[len] != b[len]) - break; - len++; - } - return len; - } - - xor = *(long *)&a[len] ^ *(long *)&b[len]; - if (xor) - break; - len += sizeof(long); - - xor = *(long *)&a[len] ^ *(long *)&b[len]; - if (xor) - break; - len += sizeof(long); - } - -#if defined(__x86_64__) || defined(__i386__) || defined(__i486__) || defined(__i586__) || defined(__i686__) - /* x86 has bsf. We know that xor is non-null here */ - asm("bsf %1,%0\n" : "=r"(xor) : "0" (xor)); - return len + xor / 8; +#if __SIZEOF_LONG__ > 4 + return read_le64(p); #else - if (sizeof(long) > 4 && !(xor & 0xffffffff)) { - /* This code is optimized out on 32-bit archs, but we still - * need to shift in two passes to avoid a warning. It is - * properly optimized out as a single shift. - */ - xor >>= 16; xor >>= 16; - if (xor & 0xffff) { - if (xor & 0xff) - return len + 4; - return len + 5; - } - if (xor & 0xffffff) - return len + 6; - return len + 7; - } - - if (xor & 0xffff) { - if (xor & 0xff) - return len; - return len + 1; - } - if (xor & 0xffffff) - return len + 2; - return len + 3; -#endif // x86 - -#else // UNALIGNED_LE_OK - /* This is the generic version for big endian or unaligned-incompatible - * architectures. - */ - while (len < max) { - if (a[len] != b[len]) - break; - len++; - } - return len; + return read_le32(p); +#endif +} +/* writes an unaligned 32-bit word in little endian order */ +static inline void write_le32(void *p, uint32_t v) +{ +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + union { uint32_t u32; } __attribute__((packed)) *u = p; + u->u32 = v; +#else + uint8_t *u8 = p; + u8[0] = v; + u8[1] = v >> 8; + u8[2] = v >> 16; + u8[3] = v >> 24; +#endif +} + +/* writes an unaligned 16-bit word in little order */ +static inline void write_le16(void *p, uint16_t v) +{ +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + union { uint16_t u16; } __attribute__((packed)) *u = p; + u->u16 = v; +#else + uint8_t *u8 = p; + u8[0] = v; + u8[1] = v >> 8; #endif } -/* helper function to reverse bits in which is expected to be - * bits long. +/* uses the most suitable crc32 function to update crc on */ +static inline uint32_t update_crc(uint32_t crc, const void *buf, int len) +{ + return slz_crc32_by4(crc, buf, len); +} + +/* reverses bits in 5bits len input and returns the new value. + * ie: 01100 becomes 00110. Bits 5 and above are ignored. */ -static inline short rev_short(short input, int len) +#if defined(__aarch64__) || defined(__ARM_ARCH_7A__) + +/* ARM >= v7 has a fast rbit instruction */ +static inline uint32_t rbit5(uint32_t v) { - short ret = 0; +#if defined(__ARM_ARCH_ISA_A64) + // 64 bit mode + __asm__ ("rbit %w0, %w0" : "+r" (v)); +#else + // 32 bit mode (e.g. armv7 compiler building for armv8 + __asm__ ("rbit %0, %0" : "+r" (v)); +#endif + return v >> 27; +} + +#else /* not arm */ + +#define rbit5(v) ({ \ + typeof(v) _v = (v); \ + ((0xf7b3d591e6a2c480ULL >> ((_v & 0x1e) * 2)) & 0xf) + ((_v & 1) << 4); \ +}) - while (len) { - ret <<= 1; - ret |= (input & 1); - input >>= 1; - len--; - } +#endif - return ret; +/* helper function to reverse up to 9 bits in which is expected to be + * bits long. The function might work with more than 9 bits in certain + * cases with no guarantees, so don't go over 9. + */ +static inline uint32_t rbit9(uint32_t input, int len) +{ +#if defined(__aarch64__) || defined(__ARM_ARCH_7A__) + /* ARM >= v7 has a fast rbit instruction */ + input <<= 32 - len; // align left before reversal +#if defined(__ARM_ARCH_ISA_A64) + // 64 bit mode + __asm__ ("rbit %w0, %w0" : "+r" (input)); +#else + // 32 bit mode (e.g. armv7 compiler building for armv8 + __asm__ ("rbit %0, %0" : "+r" (input)); +#endif + return input; + +#else /* not arm */ + uint32_t h, l; + h = input >> 5; + l = input; + h = rbit5(h); l = rbit5(l); + return ((l << 5) | h) >> (10 - len); /* new_h = [rev(l):5][rev(h):5] */ +#endif } #endif diff --git a/include/import/slz.h b/include/import/slz.h index 06ce35c45..440c75084 100644 --- a/include/import/slz.h +++ b/include/import/slz.h @@ -77,11 +77,12 @@ struct slz_stream { #else uint32_t queue; /* last pending bits, LSB first, faster on 32-bit archs */ #endif - uint32_t qbits; /* number of bits in queue, < 8 on 32-bit, < 32 on 64-bit */ unsigned char *outbuf; /* set by encode() */ - uint16_t state; /* one of slz_state */ - uint8_t level:1; /* 0 = no compression, 1 = compression */ + uint32_t qbits; /* number of bits in queue, < 8 on 32-bit, < 32 on 64-bit */ + uint32_t state; /* one of slz_state */ + uint8_t unused:2; /* free for future use */ uint8_t format:2; /* SLZ_FMT_* */ + uint8_t level:4; /* 0 = no compression, 1..15 = compression */ uint8_t debt; /* number of bits by which the fixed huffman encoding is * currently behind the equivalent stored blocks, see * SLZ_MAX_DEBT in slz.c @@ -90,6 +91,15 @@ struct slz_stream { uint32_t ilen; }; +/* Note about output buffer sizing: encode functions below do not take an + * output len, instead the caller must provide a "large enough" output buffer. + * When enabling direct queuing via SLZ_DIRECT_ENQUEUE24, the 3 bytes following + * the end of the buffer may be written to, and the caller must take that into + * account by allocating a buffer that is at least 3 bytes larger than needed. + * And when using SLZ_DIRECT_ENQUEUE56, it's up to 7 extra bytes that may be + * needed. + */ + /* Functions specific to rfc1951 (deflate) */ void slz_prepare_dist_table(); /* obsolete, not needed anymore */ long slz_rfc1951_encode(struct slz_stream *strm, unsigned char *out, const unsigned char *in, long ilen, int more); @@ -240,6 +250,81 @@ enum uslz_stream_state { USLZ_ST_CHECK_CKSUM, }; +/* Width, in bits, of the direct lookup tables used to decode dynamic huffman + * codes: one for the literal/length alphabet, one for the distance alphabet. + * A code no longer than the table is decoded in a single load, a longer one + * falls back to walking the tree, which costs one dependent load per bit. + * Each table costs (1 << bits) * 2 bytes in *every* stream state, so this is + * a direct trade between memory and decoding speed. Setting a width to zero + * removes the corresponding table from the state along with the code that + * uses it, which is the right thing for an application holding many + * mostly-idle streams. + * + * These are deliberately not build options. They set the size and the layout + * of struct uslz_stream, which is exported and instantiated by the calling + * application, so a -D reaching only one of the two sides would leave the + * library and the application disagreeing on the offset of every field past + * fast_lit[]: the application would allocate 1568 bytes where the library + * writes 2848, and nothing would detect it. Defining them unconditionally is + * what prevents this, the definition below wins in every translation unit + * that includes this file and a -D trying to change them only earns a + * redefinition warning. The values can still be changed, but only here, and + * doing so requires rebuilding the library *and* everything that instantiates + * a stream. + * + * The performance below was measured on 8 different machines decoding a + * silesia.tar compressed by gzip, with various combinations of literal length + * bits and distance bits. The tests were lacking several optimizations that + * were later added, so the numbers are off compared to current code, but their + * ratios are still correct. The rates are expressed in MB/s of output stream + * after decompression: + * + * lit/dist state arl spr sklx 7800 7v13 5105 atom n1 + * 0/0 1728 B 237 186 178 232 174 124 50 124 + * 0/7 1984 B 285 223 199 282 206 144 56 143 + * 9/0 2752 B 379 286 313 329 239 175 66 182 + * 8/6 2368 B 433 342 323 378 273 190 69 198 + * 9/7 3008 B 508 402 381 419 303 222 76 229 + * 10/7 4032 B 547 437 403 445 324 238 78 245 <= current + * 11/7 6080 B 556 446 409 451 325 245 79 250 + * + * It's important not to forget that decompressing gzip requires to keep a + * 32 kB sliding window, so the real sizes are those obtained by adding + * 32768 to the values above. Once taken into account, and measures reported + * as extra percentages compared to 0/0, the best choices appear more clearly: + * + * lit/dist memory arl spr sklx 7800 7v13 5105 atom n1 + * 0/0 34496 B 237 186 178 232 174 124 50 124 MB/s + * 0/7 +0.7% +20% +20% +12% +21% +18% +17% +11% +15% + * 9/0 +3.0% +60% +54% +76% +42% +37% +41% +32% +46% + * 8/6 +1.9% +83% +84% +82% +63% +57% +54% +37% +59% + * 9/7 +3.7% +114% +117% +115% +80% +74% +80% +52% +84% + * 10/7 +6.7% +131% +135% +127% +92% +86% +93% +56% +97% <= current + * 11/7 +12.6% +135% +140% +130% +94% +87% +98% +58% +101% + * + * Each step then costs a few percent of the per-stream footprint and buys + * tens of points on that scale, until it stops: + * + * 8/6 -> 9/7 memory +1.8% speed +14 to +33 points + * 9/7 -> 10/7 memory +2.9% speed +5 to +18 points + * 10/7 -> 11/7 memory +5.6% speed +1 to +5 points + * + * 10/7 is where the curve bends, and that is what is set below: against no + * table at all it buys between 56% (the Atom) and 135% (the Sapphire Rapids) + * on a dynamic stream for 6.7% more memory per stream, while the step past it + * costs twice as much memory for a tenth of the speed. + * + * An embedded application that decodes nothing but slz output is the + * exception: those streams never look these tables up, so it can set both + * widths to zero and get the 2304 bytes back. + */ +#define USLZ_FAST_BITS 10 +#define USLZ_FAST_DBITS 7 + +#if USLZ_FAST_BITS > 15 || USLZ_FAST_DBITS > 15 +#error "USLZ_FAST_BITS and USLZ_FAST_DBITS must not exceed 15, the longest deflate code" +#endif + /* decompression state */ struct uslz_stream { enum uslz_stream_state state; /* parsing state, USLZ_ST_* */ @@ -262,13 +347,13 @@ struct uslz_stream { unsigned long dec_bofs; /* offset used to read the current decoded block */ unsigned long dec_bsize; /* size of the current decoded block */ - uint64_t bit_accum; /* bit accumulator (used by GETBITS() macro) */ unsigned char num_bits; /* number of bits in the accumulator */ unsigned char block_type; /* compressed block type */ unsigned int huff_index; /* current index for pending huffman decoding * attempt */ + unsigned long bit_accum; /* bit accumulator (used by GETBITS() macro) */ unsigned int symbol; /* symbol code currently being processed */ unsigned int last_value; /* last value read for length / distance table * construction @@ -297,6 +382,17 @@ struct uslz_stream { * enough space for that alphabet. */ short literal_table[288*2-2]; + /* Direct lookup table for the literal/length alphabet, indexed by the + * next USLZ_FAST_BITS bits of the stream. Same layout as + * fixed_huff_dec_table: the symbol in the upper bits and the code + * length in the low 4 bits, a length of zero meaning the code is + * longer than the table and literal_table[] has to be walked. + */ + uint16_t fast_lit[1 << USLZ_FAST_BITS]; + + /* same for the distance alphabet */ + uint16_t fast_dist[1 << USLZ_FAST_DBITS]; + union { /* Code-to-symbol conversion table for the alphabet used for * distances. This alphabet consists of 32 symbols, 2 of @@ -310,8 +406,8 @@ struct uslz_stream { */ struct { unsigned char buf[10]; + unsigned char gzip_flags; int buf_len; - char gzip_flags; } hdr_detect; }; short codelen_table[19*2-2]; /* Code-to-symbol conversion table for the alphabet diff --git a/src/slz-prv.h b/src/slz-prv.h new file mode 100644 index 000000000..04db47e37 --- /dev/null +++ b/src/slz-prv.h @@ -0,0 +1,2 @@ +/* Only needed for slz*.c */ +#include diff --git a/src/slz.c b/src/slz.c index 4855bf1dc..b3121981f 100644 --- a/src/slz.c +++ b/src/slz.c @@ -24,9 +24,9 @@ #include #include -#include "import/slz.h" -#include "import/slz-prv.h" -#include "import/slz-tables.h" +#include "slz.h" +#include "slz-prv.h" +#include "tables.h" /* if PRECOMPUTE_TABLES not set, then tables.h does not provide fh_dist_table * table as we need to recompute it. @@ -35,6 +35,95 @@ static uint32_t fh_dist_table[32768]; #endif // ifndef PRECOMPUTE_TABLES +/* The enqueue functions have two possible implementations. The default one + * only writes to the output buffer once it holds a whole store's worth of + * bits, which costs a test but never writes a byte it does not advance over. + * The "direct" one, selected by these macros, stores unconditionally and then + * advances by however many whole bytes the queue turned out to hold, leaving + * fewer than 8 bits behind. That saves a branch, and requires the caller to + * leave room past the output pointer for the widest store used (3 bytes for + * enqueue24() and enqueue8(), 7 for enqueue56()), since the bytes beyond the + * advance are rewritten by the next call. They're only usable for unaligned + * little endian archs and generally work better on modern x86_64. Tests show + * that x86_64 and arm64 benefit greatly from this. + * + * SLZ_DIRECT_ENQUEUE56 goes further by writing a whole length+distance pair in + * a single operation instead of two. + */ + +/* directly write 32 bits at a time to output for up to 24 bits */ +#ifndef SLZ_DIRECT_ENQUEUE24 +# if defined(UNALIGNED_LE_OK) +# define SLZ_DIRECT_ENQUEUE24 1 +# else +# define SLZ_DIRECT_ENQUEUE24 0 +# endif +#endif + +/* directly write 8 bits at a time to the output */ +#ifndef SLZ_DIRECT_ENQUEUE8 +# define SLZ_DIRECT_ENQUEUE8 SLZ_DIRECT_ENQUEUE24 +#endif + +/* directly write 64 bits at a tome to output for up to 56 bits */ +#ifndef SLZ_DIRECT_ENQUEUE56 +# if defined(__SIZEOF_LONG__) && (__SIZEOF_LONG__ - 0 > 4) && defined(UNALIGNED_LE_OK) +# define SLZ_DIRECT_ENQUEUE56 1 +# else +# define SLZ_DIRECT_ENQUEUE56 0 +# endif +#endif + +/* The direct enqueue8() writes a single byte, which is only enough because it + * assumes fewer than 8 bits were already queued. That holds for every enqueue + * variant except the conditional 64-bit enqueue24(), which deliberately lets + * up to 31 bits accumulate before storing. Combining the two would advance the + * output pointer over bytes that were never written. + */ +#if SLZ_DIRECT_ENQUEUE8 && !SLZ_DIRECT_ENQUEUE24 && \ + defined(__SIZEOF_LONG__) && (__SIZEOF_LONG__ - 0 > 4) && defined(UNALIGNED_LE_OK) +# error "SLZ_DIRECT_ENQUEUE8 requires SLZ_DIRECT_ENQUEUE24 on this platform" +#endif + +/* enqueue56() needs a 64-bit queue and unaligned little-endian stores. */ +#if SLZ_DIRECT_ENQUEUE56 && !(defined(__SIZEOF_LONG__) && (__SIZEOF_LONG__ - 0 > 4) && defined(UNALIGNED_LE_OK)) +# undef SLZ_DIRECT_ENQUEUE56 +# define SLZ_DIRECT_ENQUEUE56 0 +#endif + +/* Also indexes the positions covered by the beginning of a match. This is quite + * important for example on HTML or other streams with lots of varying prefixes, + * especially when combined with the backwards match, because otherwise a likely + * repeating element will not be indexed, and when the window advances, its ref + * is likely to disappear, causing the next one to experience a miss. + */ +#ifndef SLZ_INSERT_SKIPPED +# define SLZ_INSERT_SKIPPED 1 +#endif + +/* Skip large non-compressible areas and send them as literals to avoid wasting + * CPU trying to compress them. It's faster on incompressible data, at the cost + * of a slightly reduced compression ratio. + */ +#ifndef SLZ_LITERAL_SKIP +# define SLZ_LITERAL_SKIP 1 +#endif + +/* Enables growing a match backwards over the literals that are still pending, + * see slz_rfc1951_encode(). Costs a little CPU on every match and gains about + * 1% of output. Tests show that matching up to 2 bytes brings almost optimal + * results at minimal extra cost, and that compression gains at 1 byte are + * higher than the CPU increase they cause, and tend to result in higher + * compression speed due to the savings on encoding. The only way to disable + * this optimization is to comment out this line. + */ +#define SLZ_BACKWARD_MATCH ((const int[]){ 0, 1, 2, 8 }) + +/* Log2 of the size of the hash table used for the references table. */ +#ifndef SLZ_HASH_BITS +# define SLZ_HASH_BITS 13 +#endif + /* First, RFC1951-specific declarations and extracts from the RFC. * * RFC1951 - deflate stream format @@ -158,6 +247,9 @@ Distance encoding : */ +/* for use to temporarily disable a builtin_expect */ +#define __no_builtin_expect(cond, exp) (cond) + /* back references, built in a way that is optimal for 32/64 bits */ /* format: * bit0..31 = word @@ -171,19 +263,130 @@ union ref { uint64_t by64; }; -#if defined(__SIZEOF_LONG__) && (__SIZEOF_LONG__ - 0 > 4) && defined(UNALIGNED_LE_OK) +/* This hash provides good average results on HTML contents, and is among the + * few which provide almost optimal results on various different pages. + */ +static inline uint32_t slz_hash(uint32_t a) +{ +#if defined(__ARM_FEATURE_CRC32) +# if defined(__ARM_ARCH_ISA_A64) + // 64 bit mode + __asm__ volatile("crc32w %w0,%w0,%w1" : "+r"(a) : "r"(0)); +# else + // 32 bit mode (e.g. armv7 compiler building for armv8 + __asm__ volatile("crc32w %0,%0,%1" : "+r"(a) : "r"(0)); +# endif +#elif defined(__SSE4_2__) && defined(USE_CRC32C_HASH) + // SSE 4.2 offers CRC32C which is a bit slower than the multiply + // but provides a slightly smoother hash + __asm__ volatile("crc32l %1,%0" : "+r"(a) : "r"(0)); +#elif defined(HAVE_FAST_MULT) + // optimal factor for SLZ_HASH_BITS=12 and SLZ_HASH_BITS=13 among 48k tested: 0x1af42f + a = (a * 0x1af42f); // 12: 44.12, 13: 43.35 + //return (a * 0x1aaee3) >> (32 - SLZ_HASH_BITS); // 12: 44.14, 13: 43.34 + //return (a * 0x1aae0b) >> (32 - SLZ_HASH_BITS); // 12: 44.15, 13: 43.35, -30k + //return (a * 0x17e7bd) >> (32 - SLZ_HASH_BITS); // 12: 44.15, 13: 43.36 + //return (a * 0xaae7b) >> (32 - SLZ_HASH_BITS); // 12: 44.22, 13:43.35, -1k + //return (a * 0xaaec3) >> (32 - SLZ_HASH_BITS); // 12: 44.20, 13: 43.35, -10k // 275659647 +#else + a = ((a << 19) + (a << 6) - a); +#endif + return a >> (32 - SLZ_HASH_BITS); +} + +/* This function compares buffers and and reads 32 or 64 bits at a time + * during the approach. It makes us of unaligned little endian memory accesses + * on capable architectures. is the maximum number of bytes that can be + * read, so both and must have at least bytes ahead. may + * safely be null or negative if that simplifies computations in the caller. + */ +static inline long memmatch(const unsigned char *a, const unsigned char *b, long max) +{ + long len = 0; + +#ifdef UNALIGNED_LE_OK + + while (len + 2 * (long)sizeof(long) - 1 < max) { + unsigned long xor; + + xor = *(unsigned long *)&a[len] ^ *(unsigned long *)&b[len]; + if (xor) + return len + (__builtin_ctzl(xor) >> 3); + len += (long)sizeof(long); + + xor = *(unsigned long *)&a[len] ^ *(unsigned long *)&b[len]; + if (xor) + return len + (__builtin_ctzl(xor) >> 3); + len += (long)sizeof(long); + } + + while (len + (long)sizeof(long) - 1 < max) { + unsigned long xor; + + xor = *(unsigned long *)&a[len] ^ *(unsigned long *)&b[len]; + if (xor) + return len + (__builtin_ctzl(xor) >> 3); + len += (long)sizeof(long); + } + + if (sizeof(long) > 4 && len + 3 < max) { + uint32_t xor; + + xor = *(uint32_t *)&a[len] ^ *(uint32_t *)&b[len]; + if (xor) + return len + (__builtin_ctz(xor) >> 3); + len += 4; + } + + if (len + 1 < max) { + uint16_t xor; + + xor = *(uint16_t *)&a[len] ^ *(uint16_t *)&b[len]; + if (xor) + return len + !(uint8_t)xor; + len += 2; + } + + if (len < max && a[len] == b[len]) + len++; + return len; + +#else // UNALIGNED_LE_OK + /* This is the generic version for big endian or unaligned-incompatible + * architectures. + */ + while (len < max) { + if (a[len] != b[len]) + break; + len++; + } + return len; + +#endif +} + +#if defined(__SIZEOF_LONG__) && (__SIZEOF_LONG__ - 0 > 4) /* enqueue code x of bits (LSB aligned, at most 24) and copy complete * 32-bit words into output buffer. X must not contain non-zero bits above - * xbits. + * xbits. Note that the function may write up to 3 extra bytes since it may + * write 32 bits even if only 8 are needed. */ -static inline void enqueue24(struct slz_stream *strm, uint64_t x, uint32_t xbits) +static inline void enqueue24(struct slz_stream *__restrict strm, uint64_t x, uint32_t xbits) { uint64_t queue = strm->queue + (x << strm->qbits); uint32_t qbits = strm->qbits + xbits; +#if SLZ_DIRECT_ENQUEUE24 + write_le32(strm->outbuf, queue); + queue >>= qbits & ~7U; + + strm->queue = queue; + strm->outbuf += qbits >> 3; + strm->qbits = qbits & 7; +#else if (__builtin_expect(qbits >= 32, 1)) { - *(uint32_t *)strm->outbuf = queue; + write_le32(strm->outbuf, queue); queue >>= 32; qbits -= 32; strm->outbuf += 4; @@ -191,47 +394,110 @@ static inline void enqueue24(struct slz_stream *strm, uint64_t x, uint32_t xbits strm->queue = queue; strm->qbits = qbits; +#endif } +/* enqueue code x of bits (at most 8) and copy complete bytes into out + * buf. X must not contain non-zero bits above xbits. + */ +#if SLZ_DIRECT_ENQUEUE8 +static inline void enqueue8(struct slz_stream *__restrict strm, uint32_t x, uint32_t xbits) +{ + uint64_t queue = strm->queue + ((uint64_t)x << strm->qbits); + uint32_t qbits = strm->qbits + xbits; + + *strm->outbuf = queue; + strm->outbuf += qbits >= 8; + queue >>= qbits & 8; + strm->queue = queue; + strm->qbits = qbits & 7; +} +#else #define enqueue8 enqueue24 +#endif -/* flush the queue and align to next byte */ -static inline void flush_bits(struct slz_stream *strm) +#if SLZ_DIRECT_ENQUEUE56 +/* Enqueues code of bits (LSB aligned) and stores the low 8 bytes + * of the queue, then advances over the whole bytes it holds. plus + * whatever was already queued must stay below 64, and the caller must leave 8 + * bytes of room past strm->outbuf rather than the 4 the other enqueues need. + * It exists to send a length code and the distance code that follows it in one + * go, which is at most 31 bits: 7 or 8 for the length code plus up to 5 extra, + * then 5 for the distance code plus up to 13 extra. + */ +static inline void enqueue56(struct slz_stream *__restrict strm, uint64_t x, uint32_t xbits) { - if (strm->qbits > 0) - *strm->outbuf++ = strm->queue; + uint64_t queue = strm->queue + (x << strm->qbits); + uint32_t qbits = strm->qbits + xbits; + + *(uint64_t *)strm->outbuf = queue; + queue >>= qbits & ~7U; + strm->queue = queue; + strm->outbuf += qbits >> 3; + strm->qbits = qbits & 7; +} +#endif + +/* flush the queue and align to next byte */ +static inline void flush_bits(struct slz_stream *__restrict strm) +{ + if (!strm->qbits) + return; + + *strm->outbuf++ = strm->queue; + +#if !SLZ_DIRECT_ENQUEUE24 + /* Only the conditional enqueue24() can leave more than 7 bits in the + * queue, and only it needs the three extra stores. The direct one + * masks the count with 7 on every path, as do enqueue8() and + * enqueue56(), so a single byte covers every case there. + * + * This is not free to get wrong: flush_bits() is inlined into + * copy_lit() and into all six flush/finish entry points, so the three + * dead stores cost 585 bytes of text and 3% of throughput on silesia + * once SLZ_LITERAL_SKIP starts calling copy_lit() for every skipped + * chunk. + */ if (strm->qbits > 8) *strm->outbuf++ = strm->queue >> 8; +#if !SLZ_DIRECT_ENQUEUE8 if (strm->qbits > 16) *strm->outbuf++ = strm->queue >> 16; if (strm->qbits > 24) *strm->outbuf++ = strm->queue >> 24; +#endif +#endif strm->queue = 0; strm->qbits = 0; } -#else /* non-64 bit or aligned or big endian */ +#else /* non-64 bit */ /* enqueue code x of bits (LSB aligned, at most 24) and copy complete * bytes into out buf. X must not contain non-zero bits above xbits. Prefer - * enqueue8() when xbits is known for being 8 or less. + * enqueue8() when xbits is known for being 8 or less. Note that the function + * may write up to 3 extra bytes since it may write 32 bits even if only 8 are + * needed. */ -static void enqueue24(struct slz_stream *strm, uint32_t x, uint32_t xbits) +static void enqueue24(struct slz_stream *__restrict strm, uint32_t x, uint32_t xbits) { uint32_t queue = strm->queue + (x << strm->qbits); uint32_t qbits = strm->qbits + xbits; - if (qbits >= 16) { -#ifndef UNALIGNED_LE_OK - strm->outbuf[0] = queue; - strm->outbuf[1] = queue >> 8; -#else - *(uint16_t *)strm->outbuf = queue; +#if SLZ_DIRECT_ENQUEUE24 + write_le32(strm->outbuf, queue); + queue >>= qbits & ~7U; + strm->outbuf += qbits >> 3; + strm->qbits = qbits & 7; + strm->queue = queue; + return; #endif + if (qbits >= 16) { + write_le16(strm->outbuf, queue); strm->outbuf += 2; queue >>= 16; qbits -= 16; @@ -250,11 +516,19 @@ static void enqueue24(struct slz_stream *strm, uint32_t x, uint32_t xbits) /* enqueue code x of bits (at most 8) and copy complete bytes into * out buf. X must not contain non-zero bits above xbits. */ -static inline void enqueue8(struct slz_stream *strm, uint32_t x, uint32_t xbits) +static inline void enqueue8(struct slz_stream *__restrict strm, uint32_t x, uint32_t xbits) { uint32_t queue = strm->queue + (x << strm->qbits); uint32_t qbits = strm->qbits + xbits; +#if SLZ_DIRECT_ENQUEUE8 + *strm->outbuf = queue; + strm->outbuf += qbits >= 8; + queue >>= qbits & 8; + strm->queue = queue; + strm->qbits = qbits & 7; + return; +#endif if (__builtin_expect((signed)(qbits - 8) >= 0, 1)) { qbits -= 8; *strm->outbuf++ = queue; @@ -266,13 +540,17 @@ static inline void enqueue8(struct slz_stream *strm, uint32_t x, uint32_t xbits) } /* align to next byte */ -static inline void flush_bits(struct slz_stream *strm) +static inline void flush_bits(struct slz_stream *__restrict strm) { - if (strm->qbits > 0) - *strm->outbuf++ = strm->queue; + if (!strm->qbits) + return; + + *strm->outbuf++ = strm->queue; +#if !SLZ_DIRECT_ENQUEUE8 if (strm->qbits > 8) *strm->outbuf++ = strm->queue >> 8; +#endif strm->queue = 0; strm->qbits = 0; @@ -281,31 +559,27 @@ static inline void flush_bits(struct slz_stream *strm) /* only valid if buffer is already aligned */ -static inline void copy_8b(struct slz_stream *strm, uint32_t x) +static inline void copy_8b(struct slz_stream *__restrict strm, uint32_t x) { *strm->outbuf++ = x; } /* only valid if buffer is already aligned */ -static inline void copy_16b(struct slz_stream *strm, uint32_t x) +static inline void copy_16b(struct slz_stream *__restrict strm, uint32_t x) { - strm->outbuf[0] = x; - strm->outbuf[1] = x >> 8; + write_le16(strm->outbuf, x); strm->outbuf += 2; } /* only valid if buffer is already aligned */ -static inline void copy_32b(struct slz_stream *strm, uint32_t x) +static inline void copy_32b(struct slz_stream *__restrict strm, uint32_t x) { - strm->outbuf[0] = x; - strm->outbuf[1] = x >> 8; - strm->outbuf[2] = x >> 16; - strm->outbuf[3] = x >> 24; + write_le32(strm->outbuf, x); strm->outbuf += 4; } /* Using long because faster on 64-bit (can save one shift) */ -static inline void send_huff(struct slz_stream *strm, unsigned long code) +static inline void send_huff(struct slz_stream *__restrict strm, unsigned long code) { uint32_t bits; @@ -315,44 +589,83 @@ static inline void send_huff(struct slz_stream *strm, unsigned long code) enqueue24(strm, code, bits); } -static inline void send_eob(struct slz_stream *strm) +static inline void send_eob(struct slz_stream *__restrict strm) { enqueue8(strm, 0, 7); // direct encoding of 256 = EOB (cf RFC1951) } +/* copies litterals from . at once. must not be null and must + * be lower than or equal to 65535, which can be guaranteed when using + * SLZ_LITERAL_SKIP with limited literal lengths. indicates that there + * are data past buf + . Only supports fixed encoding. + */ +__attribute__((unused)) +static void copy_lit_small(struct slz_stream *__restrict strm, const void *buf, uint32_t len, int more) +{ + int eob_shift = (strm->state == SLZ_ST_EOB) ? 0 : 7; + + /* explanation: if there is no EOB yet, we have to send one. + * It's 0 encoded over 7 bits. Then we have to copy !more into BFINAL + * (one bit), and 00 in btype (two bits). Finally we pad to the next + * byte. So we add 0 or 7 bits for EOB, followed by 3 then by 0 to 7 + * bits for alignment. That's 3 to 17 bits. + */ + enqueue24(strm, (!more) << eob_shift, 3 + eob_shift + (7 & -(3 + eob_shift + strm->qbits))); // EOB + BFINAL + BTYPE + strm->state = more ? SLZ_ST_EOB : SLZ_ST_DONE; + +#if !SLZ_DIRECT_ENQUEUE24 + /* the queue is byte-aligned now, but the conditional 64-bit enqueue24() + * may still hold up to 4 bytes of it that must be written first. + */ + flush_bits(strm); +#endif + copy_16b(strm, len); + copy_16b(strm, ~len); + memcpy(strm->outbuf, buf, len); + strm->outbuf += len; +} + /* copies litterals from . indicates that there are data past * buf + . must not be null. */ -static void copy_lit(struct slz_stream *strm, const void *buf, uint32_t len, int more) +__attribute__((unused)) +static void copy_lit(struct slz_stream *__restrict strm, const void *buf, uint32_t len, int more) { uint32_t len2; + if (len <= 65535) + return copy_lit_small(strm, buf, len, more); + + if (strm->state != SLZ_ST_EOB) + send_eob(strm); + + len2 = 65535; + len -= len2; + do { + /* wire BFINAL (!more), BTYPE (00) and round to the next byte boundary */ + enqueue24(strm, !(more || len), 3 + ((5 - strm->qbits) & 7)); +#if !SLZ_DIRECT_ENQUEUE24 + flush_bits(strm); // see copy_lit_small() +#endif + copy_32b(strm, (~len2 << 16) + len2); + memcpy(strm->outbuf, buf, len2); + buf += len2; + strm->outbuf += len2; len2 = len; if (__builtin_expect(len2 > 65535, 0)) len2 = 65535; len -= len2; + } while (__builtin_expect(len2, 0)); - if (strm->state != SLZ_ST_EOB) - send_eob(strm); - - strm->state = (more || len) ? SLZ_ST_EOB : SLZ_ST_DONE; - - enqueue8(strm, !(more || len), 3); // BFINAL = !more ; BTYPE = 00 - flush_bits(strm); - copy_16b(strm, len2); // len2 - copy_16b(strm, ~len2); // nlen2 - memcpy(strm->outbuf, buf, len2); - buf += len2; - strm->outbuf += len2; - } while (len); + strm->state = more ? SLZ_ST_EOB : SLZ_ST_DONE; } /* copies litterals from . indicates that there are data past * buf + . must not be null. */ -static void copy_lit_huff(struct slz_stream *strm, const unsigned char *buf, uint32_t len, int more) +static void copy_lit_huff(struct slz_stream *__restrict strm, const unsigned char *buf, uint32_t len, int more) { uint32_t pos; @@ -375,36 +688,43 @@ static void copy_lit_huff(struct slz_stream *strm, const unsigned char *buf, uin } while (pos < len); } -/* sets BYTES to -32769 in so that any uninitialized entry will - * verify (pos-last-1 >= 32768) and be ignored. must be a multiple of - * 128 bytes and must be at least one count in length. It's supposed to - * be applied to 64-bit aligned data exclusively, which makes it slightly - * faster than the regular memset() since no alignment check is performed. +/* Marks all bytes worth of entries in as unusable, so that any + * entry not written since is rejected by the (pos - last - 1 >= 32768) test in + * slz_rfc1951_encode(). is in bytes and must be at least that + * large. + * + * The byte value is not arbitrary. It has to leave a field that reads as + * "far in the past" for every possible , on 32-bit as well, where that + * subtraction is done on 32 bits. 0xFF wouldn't work. 0xFE gives 0xFEFEFEFE, + * hence pos + 0x01010101, which is above 32768 and does not wrap for any + * realistic input length. Note that the use of memset() here is intentional + * as it was measured to be faster on modern libcs than an unrolled loop. */ static void reset_refs(union ref *refs, long count) { - /* avoid a shift/mask by casting to void* */ - union ref *end = (void *)refs + count; + memset(refs, 0xFE, count); +} - do { - refs[ 0].by64 = -32769; - refs[ 1].by64 = -32769; - refs[ 2].by64 = -32769; - refs[ 3].by64 = -32769; - refs[ 4].by64 = -32769; - refs[ 5].by64 = -32769; - refs[ 6].by64 = -32769; - refs[ 7].by64 = -32769; - refs[ 8].by64 = -32769; - refs[ 9].by64 = -32769; - refs[10].by64 = -32769; - refs[11].by64 = -32769; - refs[12].by64 = -32769; - refs[13].by64 = -32769; - refs[14].by64 = -32769; - refs[15].by64 = -32769; - refs += 16; - } while (refs < end); +/* Inserts position

of input buffer into the reference table . + * Reads the 4 bytes at in[p], so the caller must guarantee that in[p + 3] is + * within the input buffer. + */ +static inline void insert_ref(union ref *refs, const unsigned char *in, unsigned long p) +{ +#ifdef UNALIGNED_LE_OK + uint32_t w = *(uint32_t *)&in[p]; +#else + uint32_t w = ((uint32_t)in[p]) | ((uint32_t)in[p + 1] << 8) | + ((uint32_t)in[p + 2] << 16) | ((uint32_t)in[p + 3] << 24); +#endif + uint32_t h = slz_hash(w); + + if (sizeof(long) >= 8) + refs[h].by64 = ((uint64_t)p) + ((uint64_t)w << 32); + else { + refs[h].by32.pos = p; + refs[h].by32.word = w; + } } /* Number of bits wasted by the 9-bit literals above which it becomes @@ -446,28 +766,44 @@ static void reset_refs(union ref *refs, long count) * more than its own share, which the following ones will not emit. The caller * is responsible for ensuring there is enough room in the output buffer for * this. The amount of output bytes is returned, and no CRC is computed. + * + * Note: this function uses a lot of registers due to local variables that + * hold pre-computed values that need to be kept very long. Be extremely + * careful when changing anything there, and test again on multiple compiler + * versions. For example, when building with avx512f (e.g. -march=native on + * Skylake-X and Zen-4+), gcc 13 and 14 tend to move some variables to %xmm0 + * and to read it back using vmovq multiple times, affecting performance, which + * can be recovered using -mno-avx512f (~1-2%). On gcc-15, the performance is + * better and that option degrades it by 2% again. gcc-16 is even slightly + * better (+0.5%). As a general rule, nothing vectorizes well here, and the + * unaligned, short memcpy() calls must not be translated to AVX512 code. */ -long slz_rfc1951_encode(struct slz_stream *strm, unsigned char *out, const unsigned char *in, long ilen, int more) +static inline __attribute__((always_inline)) +long _slz_rfc1951_encode(struct slz_stream *__restrict strm, unsigned char *out, const unsigned char *in, long ilen, int more, int level) { long rem = ilen; unsigned long pos = 0; unsigned long last; + const unsigned char *lastp; uint32_t word = 0; long mlen; uint32_t h; uint64_t ent; +#if SLZ_LITERAL_SKIP + long skip_ratio = 0; +#endif uint32_t plit = 0; uint32_t bit9 = 0; uint32_t dist, code; - union ref refs[1 << HASH_BITS]; + union ref refs[1 << SLZ_HASH_BITS]; - if (!strm->level) { + if (__builtin_expect(!strm->level, 0)) { /* force to send as literals (eg to preserve CPU) */ strm->outbuf = out; - plit = pos = ilen; - bit9 = 52; /* force literal dump */ - goto final_lit_dump; + if (ilen) + copy_lit(strm, in, ilen, more); + goto leave; } reset_refs(refs, sizeof(refs)); @@ -475,6 +811,7 @@ long slz_rfc1951_encode(struct slz_stream *strm, unsigned char *out, const unsig strm->outbuf = out; #ifndef UNALIGNED_FASTER + reload_word: if (rem >= 4) // is only used inside the loop below, hence the test for >= 4 word = ((uint32_t)(unsigned char)in[pos] << 8) + ((uint32_t)(unsigned char)in[pos + 1] << 16) + ((uint32_t)(unsigned char)in[pos + 2] << 24); #endif @@ -553,6 +890,54 @@ long slz_rfc1951_encode(struct slz_stream *strm, unsigned char *out, const unsig plit++; bit9 += ((unsigned char)word >= 144); pos++; + + /* Incompressible data: once enough literals have accumulated + * without a single match, stop looking for a while and just + * dump what follows as literal blocks, doubling the amount + * skipped each time so that a long incompressible block + * costs less and less to walk over. A match resets it. + */ +#if SLZ_LITERAL_SKIP + if (level <= 1 && !(plit & 63)) { + /* be careful: plit+skip must never be >= 65535 */ + long skip; + skip_ratio = (skip_ratio * 2 + 1) & 255; // capped exponential: 0,1,3,7,... and stays at 255 + skip = rem >= skip_ratio * 32 ? skip_ratio * 32 : rem; + copy_lit_small(strm, in + pos - plit, plit + skip, more || skip < rem); + pos += skip; + rem -= skip; + plit = 0; +# ifndef UNALIGNED_FASTER + /* we'll need to update the current word since we skipped some bytes */ + goto reload_word; +# else + continue; +# endif + } + else if (level == 2 && !(plit & 2047)) { + /* be careful: plit+skip must never be >= 65535 */ + long skip = rem >= skip_ratio * 4096 ? skip_ratio * 4096 : rem; + skip_ratio = (skip_ratio * 2 + 1) & 7; // capped exponential: 0,1,3,7 and stays at 7 + copy_lit_small(strm, in + pos - plit, plit + skip, more || skip < rem); + pos += skip; + rem -= skip; + plit = 0; +# ifndef UNALIGNED_FASTER + /* we'll need to update the current word since we skipped some bytes */ + goto reload_word; +# else + continue; +# endif + } + else if (level >= 3) // no literal skip at levels >= 3 +#endif + { + /* either no literal_skip at build time, or high enough level */ + if (__builtin_expect(plit == 0xffc0, 0)) { + copy_lit_small(strm, in + pos - plit, plit, 1); + plit = 0; + } + } continue; } @@ -560,8 +945,11 @@ long slz_rfc1951_encode(struct slz_stream *strm, unsigned char *out, const unsig if ((unsigned long)(pos - last - 1) >= 32768) goto send_as_lit; + /* faster to keep a copy of this with gcc 13-16 */ + lastp = in + last; + /* Note: cannot encode a length larger than 258 bytes */ - mlen = memmatch(in + pos + 4, in + last + 4, (rem > 258 ? 258 : rem) - 4) + 4; + mlen = memmatch(in + pos + 4, lastp + 4, (rem > 258 ? 258 : rem) - 4) + 4; /* found a matching entry */ @@ -628,6 +1016,93 @@ long slz_rfc1951_encode(struct slz_stream *strm, unsigned char *out, const unsig (dist & 0x1f) + (code >> 16) + SLZ_SWITCH_COST > 8 * mlen) goto send_as_lit; +#if defined(SLZ_BACKWARD_MATCH) + /* Compare queued literals in case the previous ones were not + * found due to hash collisions but still match. We don't go + * further than 8 bytes so that we can find them in a single + * 64-bit comparison. On aligned-only machines we read one byte + * at a time, as it is expected to be cheaper than composing an + * unaligned int and manually doing a ctzll() on it. Tests have + * shown that there's little to gain past 8 bytes, and anyway + * the total match length will be bounded by 258. BTW we only + * consider cases where last >= 8 because this guarantees that + * we can access both the searched sequence and the match found + * on a wole word at once. It may miss some rare matches on the + * edge of the buffer that we don't care about. + * + * It's worth noting that we *do not* fixup if a match + * is found, as this would be too complicated and counter- + * productive. This means that literals that were finally + * encoded might still count towards the budget limit, but this + * is not dramatic and keeps us on the safe side. + * + * The result of the search is to only touch that will be + * used all along in all operations. Multiple attempts have shown + * that it's both easier and faster this way than changing then + * rolling back offsets when we finally give up and send_as_lit. + */ + if (__builtin_expect(plit != 0, 1) && + (last >= MAX_C(sizeof(long), SLZ_BACKWARD_MATCH[level]))) { + long bmax = MIN(plit, 8); // keep it at 8, always faster + long back = 0; + + if (bmax > 258 - mlen) + bmax = 258 - mlen; + +#if defined(UNALIGNED_LE_OK) + if (SLZ_BACKWARD_MATCH[level] == 2) { // 43.32%, best speed vs savings + uint16_t x = *(uint16_t *)&in[pos - 2] ^ *(uint16_t *)(lastp - 2); + back = x & 0xff00 ? 0 : x ? 1 : 2; + if (back > bmax) + back = bmax; + } else if (SLZ_BACKWARD_MATCH[level] == 3 || SLZ_BACKWARD_MATCH[level] == 4) { // 32 bits: 43.31% or 43.30% + uint32_t x = *(uint32_t *)&in[pos - 4] ^ *(uint32_t *)(lastp - 4); + back = x & 0xffff0000u ? + x & 0xff000000u ? 0 : 1 : + x & 0xffffff00u ? 2 : + x ? 3 : SLZ_BACKWARD_MATCH[level]; // 3 or 4, test eliminated at 3 + if (back > bmax) + back = bmax; + } else if (SLZ_BACKWARD_MATCH[level] == 8) { // 64 bits: 43.30% + uint64_t x = *(uint64_t *)&in[pos - 8] ^ *(uint64_t *)(lastp - 8); + back = x ? (__builtin_clzll(x) >> 3) : 8; + if (back > bmax) + back = bmax; + } else +#endif + if (SLZ_BACKWARD_MATCH[level] == 1) { // single-byte match: 43.39% + if (in[pos - 1] == lastp[-1]) + back++; + if (back > bmax) + back = bmax; + } + else { /* unaligned not supported or match != 1/2/3/4/8 */ + while (back < bmax && in[pos - back - 1] == lastp[-back - 1]) + back++; + } + + if (back) { + /* OK now we're certain to use the match, we must rewind the + * literals that are part of the match. + */ + pos -= back; + plit -= back; + rem += back; + mlen += back; + /* was computed from the un-extended + * length above and must follow, or we would + * consume more bytes than the length + * code tells the decoder to copy. does + * not move. The cost test above keeps using + * the un-extended length, which is safe: the + * extension only makes the match cheaper per + * byte, never more expensive. + */ + code = len_fh[mlen]; + } + } +#endif // SLZ_BACKWARD_MATCH // otherwise 43.83% + /* first, copy pending literals */ if (plit) { /* Huffman encoding requires 9 bits for octets 144..255, so this @@ -641,7 +1116,7 @@ long slz_rfc1951_encode(struct slz_stream *strm, unsigned char *out, const unsig * only those sent in huffman mode add to the debt. */ if (bit9 >= SLZ_SWITCH_COST || strm->debt >= SLZ_MAX_DEBT) - copy_lit(strm, in + pos - plit, plit, 1); + copy_lit_small(strm, in + pos - plit, plit, 1); else { copy_lit_huff(strm, in + pos - plit, plit, 1); strm->debt += bit9; @@ -650,6 +1125,9 @@ long slz_rfc1951_encode(struct slz_stream *strm, unsigned char *out, const unsig plit = 0; } +#if SLZ_LITERAL_SKIP + skip_ratio = 0; +#endif /* use mode 01 - fixed huffman */ if (strm->state == SLZ_ST_EOB) { strm->state = SLZ_ST_FIXED; @@ -657,10 +1135,20 @@ long slz_rfc1951_encode(struct slz_stream *strm, unsigned char *out, const unsig } /* copy the length first */ +#if SLZ_DIRECT_ENQUEUE56 + /* the length code and the distance code that follows it total + * 31 bits at most, which one enqueue can take in a single + * shift-or-store instead of two + */ + enqueue56(strm, (uint64_t)(code & 0xFFFF) | + ((uint64_t)(dist >> 5) << (code >> 16)), + (code >> 16) + (dist & 0x1f)); +#else enqueue24(strm, code & 0xFFFF, code >> 16); /* in fixed huffman mode, dist is fixed 5 bits */ enqueue24(strm, dist >> 5, dist & 0x1f); +#endif /* only measures the current group of literals, which is * what the decisions above compare against, and it restarts @@ -685,6 +1173,18 @@ long slz_rfc1951_encode(struct slz_stream *strm, unsigned char *out, const unsig rem -= mlen; pos += mlen; +#if SLZ_INSERT_SKIPPED + if (rem < 4) + break; + /* try to index the matched entry to improve the likelyhood + * that a similar sequence will be a hit next time. We only + * need 4 remaining bytes starting from the current one. + */ + insert_ref(refs, in, pos - mlen + 3); + insert_ref(refs, in, pos - mlen + 2); + insert_ref(refs, in, pos - mlen + 1); +#endif // SLZ_INSERT_SKIPPED + #ifndef UNALIGNED_FASTER /* same as before the loop, this is only used when continuing * in the loop, so let's use the same condition (rem>=4). @@ -707,7 +1207,6 @@ long slz_rfc1951_encode(struct slz_stream *strm, unsigned char *out, const unsig } while (--rem); } - final_lit_dump: /* Now copy remaining literals or mark the end. The cost of switching to * a stored block depends on the current state and on the presence of * data after these literals, see the SLZ_*_COST definitions above. @@ -721,26 +1220,54 @@ long slz_rfc1951_encode(struct slz_stream *strm, unsigned char *out, const unsig cost = (strm->state == SLZ_ST_EOB) ? SLZ_LAST_COST_EOB : SLZ_LAST_COST; if (bit9 >= cost || strm->debt >= SLZ_MAX_DEBT) - copy_lit(strm, in + pos - plit, plit, more); + copy_lit_small(strm, in + pos - plit, plit, more); else copy_lit_huff(strm, in + pos - plit, plit, more); plit = 0; } - + leave: strm->ilen += ilen; return strm->outbuf - out; } +static __attribute__((noinline)) +long _slz_rfc1951_encode_lev1(struct slz_stream *__restrict strm, unsigned char *out, const unsigned char *in, long ilen, int more) +{ + return _slz_rfc1951_encode(strm, out, in, ilen, more, 1); +} + +static __attribute__((noinline)) +long _slz_rfc1951_encode_lev2(struct slz_stream *__restrict strm, unsigned char *out, const unsigned char *in, long ilen, int more) +{ + return _slz_rfc1951_encode(strm, out, in, ilen, more, 2); +} + +static __attribute__((noinline)) +long _slz_rfc1951_encode_lev3(struct slz_stream *__restrict strm, unsigned char *out, const unsigned char *in, long ilen, int more) +{ + return _slz_rfc1951_encode(strm, out, in, ilen, more, 3); +} + +long slz_rfc1951_encode(struct slz_stream *__restrict strm, unsigned char *out, const unsigned char *in, long ilen, int more) +{ + if (strm->level <= 1) + return _slz_rfc1951_encode_lev1(strm, out, in, ilen, more); + else if (strm->level <= 2) + return _slz_rfc1951_encode_lev2(strm, out, in, ilen, more); + else + return _slz_rfc1951_encode_lev3(strm, out, in, ilen, more); +} + /* Initializes stream for use with raw deflate (rfc1951). The CRC is * unused but set to zero. The compression level passed in is set. This - * value can only be 0 (no compression) or 1 (compression) and other values - * will lead to unpredictable behaviour. The function always returns 0. + * value will be capped between 0 (no compression) and 15 (max compression). + * The function always returns 0. */ int slz_rfc1951_init(struct slz_stream *strm, int level) { strm->state = SLZ_ST_EOB; // no header - strm->level = level; + strm->level = MIN(MAX(level, 0), 15); strm->format = SLZ_FMT_DEFLATE; strm->crc32 = 0; strm->ilen = 0; @@ -985,14 +1512,14 @@ long slz_rfc1952_encode(struct slz_stream *strm, unsigned char *out, const unsig } /* Initializes stream for use with the gzip format (rfc1952). The - * compression level passed in is set. This value can only be 0 (no - * compression) or 1 (compression) and other values will lead to unpredictable - * behaviour. The function always returns 0. + * compression level passed in is set. This value will be capped + * between 0 (no compression) and 15 (max compression). The function always + * returns 0. */ int slz_rfc1952_init(struct slz_stream *strm, int level) { strm->state = SLZ_ST_INIT; - strm->level = level; + strm->level = MIN(MAX(level, 0), 15); strm->format = SLZ_FMT_GZIP; strm->crc32 = 0; strm->ilen = 0; @@ -1196,14 +1723,14 @@ long slz_rfc1950_encode(struct slz_stream *strm, unsigned char *out, const unsig } /* Initializes stream for use with the zlib format (rfc1950). The - * compression level passed in is set. This value can only be 0 (no - * compression) or 1 (compression) and other values will lead to unpredictable - * behaviour. The function always returns 0. + * compression level passed in is set. This value will be capped + * between 0 (no compression) and 15 (max compression). The function always + * returns 0. */ int slz_rfc1950_init(struct slz_stream *strm, int level) { strm->state = SLZ_ST_INIT; - strm->level = level; + strm->level = MIN(MAX(level, 0), 15); strm->format = SLZ_FMT_ZLIB; strm->crc32 = 1; // rfc1950/zlib starts with initial crc=1 strm->ilen = 0; @@ -1259,6 +1786,7 @@ int slz_rfc1950_finish(struct slz_stream *strm, unsigned char *buf) return strm->outbuf - buf; } +#ifndef PRECOMPUTE_TABLES /* Returns code for lengths 1 to 32768. The bit size for the next value can be * found this way : * @@ -1306,6 +1834,7 @@ static inline uint32_t dist_to_code(uint32_t l) return code; } +#endif /* not thread-safe, must be called exactly once */ static inline void __slz_prepare_dist_table() diff --git a/src/slz.h b/src/slz.h new file mode 100644 index 000000000..0f9c1e38c --- /dev/null +++ b/src/slz.h @@ -0,0 +1,2 @@ +/* Only needed for slz*.c */ +#include diff --git a/src/slz_common.c b/src/slz_common.c index 22ec7f586..f0b818bbf 100644 --- a/src/slz_common.c +++ b/src/slz_common.c @@ -22,9 +22,20 @@ * OTHER DEALINGS IN THE SOFTWARE. */ -#include "import/slz.h" -#include "import/slz-prv.h" -#include "import/slz-tables.h" +#include "slz.h" +#include "slz-prv.h" +#include "tables.h" + +/* We'll tentatively use PCLMULQDQ for CRC calculation on x86_64 CPUs even if + * not enabled at build time, unless SLZ_NO_PCLMUL is set to disable it. This + * is done by detecting its support at boot time on supported compilers. + */ +#if !defined(SLZ_NO_PCLMUL) && (defined(__x86_64__)) && \ + (defined(__clang__) || (defined(__GNUC__) && __GNUC__ >= 6)) +#define SLZ_USE_PCLMUL +#include +static int slz_have_pclmul = 0; +#endif /* * ============================================================================= @@ -95,9 +106,10 @@ uint32_t slz_crc32_by1(uint32_t crc, const unsigned char *buf, int len) } /* This version computes the crc32 of over bytes, doing most of it - * in 32-bit chunks. + * in 32-bit chunks. It is the portable implementation, and it also finishes + * what the carry-less one below leaves behind. */ -uint32_t slz_crc32_by4(uint32_t crc, const unsigned char *buf, int len) +static uint32_t crc32_by4_tab(uint32_t crc, const unsigned char *buf, int len) { const unsigned char *end = buf + len; @@ -174,6 +186,109 @@ uint32_t slz_crc32_by4(uint32_t crc, const unsigned char *buf, int len) return crc; } +#if defined(SLZ_USE_PCLMUL) + +/* Carry-less folding implementation of the very same CRC-32, using PCLMULQDQ. + * + * The state is a 128-bit polynomial in the same reflected representation as the + * one the table code works in, which makes the final reduction trivial: it is + * just the table code run over the accumulator's 16 bytes. Folding a state + * forward by N bytes means multiplying it by x^(8N) modulo the CRC polynomial, + * which splits over the two halves of the register into two carry-less + * multiplications by a constant: + * + * fold_N(Q) = clmul(Q.lo, x^(8N+95) mod P) ^ clmul(Q.hi, x^(8N+31) mod P) + * + * The constants were derived from the CRC recurrence rather than copied from + * elsewhere, and the whole scheme was verified against the table code for every + * length and starting value before being written here; see results.md. + * + * Four accumulators are folded by 64 bytes each so that the multiplier latency, + * which is much higher than its throughput, is not what limits the loop. They + * are combined at the end by folding each into the next by 16 bytes. + */ +__attribute__((target("pclmul,sse4.1"))) /* pretend -mpclmul -msse4.1 was passed */ +static uint32_t crc32_pclmul(uint32_t crc, const unsigned char *buf, int len) +{ + /* x^159 and x^95 : fold by 16 bytes */ + const __m128i k16 = _mm_setr_epi32(0xae689191, 0, 0xccaa009e, 0); + /* x^543 and x^479 : fold by 64 bytes */ + const __m128i k64 = _mm_setr_epi32(0x8f352d95, 0, 0x1d9513d7, 0); + unsigned char tmp[16] __attribute__((aligned(16))); + __m128i q, q0, q1, q2, q3; + int blk = len & ~15; + + if (blk >= 64) { + q0 = _mm_xor_si128(_mm_loadu_si128((const __m128i *)(buf + 0)), + _mm_cvtsi32_si128(~crc)); + q1 = _mm_loadu_si128((const __m128i *)(buf + 16)); + q2 = _mm_loadu_si128((const __m128i *)(buf + 32)); + q3 = _mm_loadu_si128((const __m128i *)(buf + 48)); + buf += 64; blk -= 64; len -= 64; + + while (blk >= 64) { + q0 = _mm_xor_si128(_mm_xor_si128(_mm_clmulepi64_si128(q0, k64, 0x00), + _mm_clmulepi64_si128(q0, k64, 0x11)), + _mm_loadu_si128((const __m128i *)(buf + 0))); + q1 = _mm_xor_si128(_mm_xor_si128(_mm_clmulepi64_si128(q1, k64, 0x00), + _mm_clmulepi64_si128(q1, k64, 0x11)), + _mm_loadu_si128((const __m128i *)(buf + 16))); + q2 = _mm_xor_si128(_mm_xor_si128(_mm_clmulepi64_si128(q2, k64, 0x00), + _mm_clmulepi64_si128(q2, k64, 0x11)), + _mm_loadu_si128((const __m128i *)(buf + 32))); + q3 = _mm_xor_si128(_mm_xor_si128(_mm_clmulepi64_si128(q3, k64, 0x00), + _mm_clmulepi64_si128(q3, k64, 0x11)), + _mm_loadu_si128((const __m128i *)(buf + 48))); + buf += 64; blk -= 64; len -= 64; + } + + /* combine the four accumulators, oldest first */ + q = q0; + q = _mm_xor_si128(_mm_xor_si128(_mm_clmulepi64_si128(q, k16, 0x00), + _mm_clmulepi64_si128(q, k16, 0x11)), q1); + q = _mm_xor_si128(_mm_xor_si128(_mm_clmulepi64_si128(q, k16, 0x00), + _mm_clmulepi64_si128(q, k16, 0x11)), q2); + q = _mm_xor_si128(_mm_xor_si128(_mm_clmulepi64_si128(q, k16, 0x00), + _mm_clmulepi64_si128(q, k16, 0x11)), q3); + } + else if (blk >= 16) { + q = _mm_xor_si128(_mm_loadu_si128((const __m128i *)buf), + _mm_cvtsi32_si128(~crc)); + buf += 16; blk -= 16; len -= 16; + } + else + return crc32_by4_tab(crc, buf, len); + + while (blk >= 16) { + q = _mm_xor_si128(_mm_xor_si128(_mm_clmulepi64_si128(q, k16, 0x00), + _mm_clmulepi64_si128(q, k16, 0x11)), + _mm_loadu_si128((const __m128i *)buf)); + buf += 16; blk -= 16; len -= 16; + } + + /* The accumulator is a polynomial in exactly the representation the table + * code consumes, so reducing it is just running that code over its 16 + * bytes from an all-ones state. Then the 0 to 15 trailing bytes. + */ + _mm_store_si128((__m128i *)tmp, q); + crc = crc32_by4_tab(~0U, tmp, 16); + return crc32_by4_tab(crc, buf, len); +} + +#endif /* SLZ_USE_PCLMUL */ + +/* Computes the crc32 of over bytes, using the carry-less + * implementation when the CPU supports it. + */ +uint32_t slz_crc32_by4(uint32_t crc, const unsigned char *buf, int len) +{ +#if defined(SLZ_USE_PCLMUL) + if (__builtin_expect(slz_have_pclmul && len >= 16, 1)) + return crc32_pclmul(crc, buf, len); +#endif + return crc32_by4_tab(crc, buf, len); +} + /* Original version from RFC1950, verified and works OK */ uint32_t slz_adler32_by1(uint32_t crc, const unsigned char *buf, int len) { @@ -298,4 +413,9 @@ __attribute__((constructor)) static void __slz_common_initialize(void) { __slz_make_crc_table(); // used by both slz and uslz + +#if defined(SLZ_USE_PCLMUL) + slz_have_pclmul = __builtin_cpu_supports("pclmul") && + __builtin_cpu_supports("sse4.1"); +#endif } diff --git a/src/tables.h b/src/tables.h new file mode 100644 index 000000000..9d0eff62e --- /dev/null +++ b/src/tables.h @@ -0,0 +1,2 @@ +/* Only needed for slz*.c */ +#include diff --git a/src/uslz.c b/src/uslz.c index 043a37238..19ab7e39a 100644 --- a/src/uslz.c +++ b/src/uslz.c @@ -29,9 +29,9 @@ #include #include -#include "import/slz.h" -#include "import/slz-prv.h" -#include "import/slz-tables.h" +#include "slz.h" +#include "slz-prv.h" +#include "tables.h" /* if PRECOMPUTE_TABLES not set, then tables.h does not provide * fixed_huff_dec_table table as we need to recompute it. @@ -209,6 +209,65 @@ static int gen_huffman_table(unsigned int symbols, return 1; } +/* + * gen_fast_table: fill the direct lookup of bits from the code + * lengths in , for an alphabet of symbols. Entries have the + * same layout as fixed_huff_dec_table[]: the symbol in the upper bits and the + * code length in the low 4 bits. Codes longer than are left at zero + * and are decoded by walking the tree built by gen_huffman_table() instead. + * + * Each code is replicated over every combination of the bits above it, so + * that a lookup performed with fewer than bits still finds it, + * provided the length found is not larger than the number of bits available. + * That is what lets the decoder work at the very end of a stream, where fewer + * bits than the table width may remain. + * + * Preconditions: holds entries, all of them <= 15, and + *
is (1 << tbits) entries long. + */ +static void gen_fast_table(unsigned int symbols, const unsigned char *lengths, + uint16_t *table, unsigned int tbits) +{ + unsigned short next_code[16]; + unsigned short count[16]; + unsigned int i, len, code; + + memset(table, 0, (1U << tbits) * sizeof(*table)); + memset(count, 0, sizeof(count)); + + for (i = 0; i < symbols; i++) + count[lengths[i]]++; + + /* first code of each length, as specified in rfc1951 3.2.2 */ + count[0] = 0; + code = 0; + for (len = 1; len < 16; len++) { + code = (code + count[len - 1]) << 1; + next_code[len] = code; + } + + for (i = 0; i < symbols; i++) { + unsigned int rev, step; + + len = lengths[i]; + if (!len) + continue; + + /* codes are assigned sequentially within a length, and the + * ones too long for the table still consume code space. + */ + code = next_code[len]++; + if (len > tbits) + continue; + + /* codes appear bit-reversed in the accumulator */ + rev = rbit9(code, len); + step = 1U << len; + for (; rev < (1U << tbits); rev += step) + table[rev] = (i << 7) | len; + } +} + /* updates crc for the current inflate stream, chooses the proper * crc computation function according to the format used in the stream. */ @@ -226,11 +285,12 @@ static inline void uslz_update_crc(struct uslz_stream *state, const unsigned cha * * Returns 1 on success and 0 if more data is needed. */ -static inline int bit_accumulate(const unsigned char **in_ptr, const unsigned char *in_top, unsigned char *num_bits, uint64_t *bit_accum) +static inline int bit_accumulate(const unsigned char **in_ptr, const unsigned char *in_top, + unsigned char *num_bits, unsigned long *bit_accum) { if (*in_ptr >= in_top) return 0; - *bit_accum |= (uint64_t)(*in_ptr)[0] << *num_bits; + *bit_accum |= (unsigned long)(*in_ptr)[0] << *num_bits; *num_bits += 8; (*in_ptr)++; return 1; @@ -247,12 +307,13 @@ static inline int bit_accumulate(const unsigned char **in_ptr, const unsigned ch */ static inline int gethuff(unsigned int *huff_index, const unsigned char **in_ptr, const unsigned char *in_top, - unsigned char *num_bits, uint64_t *bit_accum, + unsigned char *num_bits, unsigned long *bit_accum, unsigned int *var, short *table) { unsigned int index; unsigned int bits; unsigned long bit_follow; + int ret = 0; index = *huff_index; bits = *num_bits; @@ -260,14 +321,10 @@ static inline int gethuff(unsigned int *huff_index, for (;;) { if (bits == 0) { - if (*in_ptr >= in_top) { - *num_bits = bits; - *bit_accum = bit_follow; - *huff_index = index; - return 0; - } - bit_follow |= (unsigned long) (*in_ptr)[0] << bits; - bits += 8; + if (*in_ptr >= in_top) + break; + bit_follow |= (unsigned long) (*in_ptr)[0]; + bits = 8; (*in_ptr)++; } @@ -275,6 +332,9 @@ static inline int gethuff(unsigned int *huff_index, bit_follow >>= 1; bits--; if (table[index] >= 0) { + *var = table[index]; + index = 0; + ret = 1; break; } index = ~(table[index]); @@ -282,10 +342,9 @@ static inline int gethuff(unsigned int *huff_index, *num_bits = bits; *bit_accum = bit_follow; - *var = table[index]; - *huff_index = 0; + *huff_index = index; - return 1; + return ret; } /* when decoding static/fixed huffman (we must be sure of that before @@ -297,7 +356,7 @@ static inline int gethuff(unsigned int *huff_index, * needs more input data. */ static inline int gethuff_fixed(const unsigned char **in_ptr, const unsigned char *in_top, - unsigned char *num_bits, uint64_t *bit_accum, + unsigned char *num_bits, unsigned long *bit_accum, unsigned int *var) { unsigned long bit_follow; @@ -330,7 +389,7 @@ static inline int gethuff_fixed(const unsigned char **in_ptr, const unsigned cha * length to prevent mismatch. */ idx = (bit_follow & ((1 << bits) - 1)); - if ((fixed_huff_dec_table[idx] & 0xF) > bits) { + if ((fixed_huff_dec_table[idx] & 0x3F) > bits) { /* no match found with , so there must be * data available for reading (errors appart) * Since we already have 7 bits and a refill is @@ -347,29 +406,178 @@ static inline int gethuff_fixed(const unsigned char **in_ptr, const unsigned cha } break; } - *num_bits = bits - (fixed_huff_dec_table[idx] & 0xF); - *bit_accum = bit_follow >> (fixed_huff_dec_table[idx] & 0xF); + *num_bits = bits - (fixed_huff_dec_table[idx] & 0x3F); + *bit_accum = bit_follow >> (fixed_huff_dec_table[idx] & 0x3F); *var = fixed_huff_dec_table[idx] >> 7; return 1; } -/* reverses bits in 5bits len input and returns the new value. - * ie: 01100 becomes 00110. Bits 5-7 are ignored. +/* Decodes one symbol using the direct lookup
of bits, which + * must have been filled by gen_fast_table(). + * + * Returns 1 when a symbol was decoded and stored in , 0 if more input + * data is needed, and -1 when the code is longer than the table, in which + * case no bit was consumed from the accumulator and the caller has to fall + * back to gethuff(). */ -static inline uint8_t rbit5(uint8_t v) +static inline int gethuff_fast(const unsigned char **in_ptr, const unsigned char *in_top, + unsigned char *num_bits, unsigned long *bit_accum, + unsigned int *var, const uint16_t *table, + unsigned int tbits) { - v = ((0xf7b3d591e6a2c480ULL >> ((v & 0x1e) * 2)) & 0xf) + ((v & 1) << 4); - return v; -} + unsigned long accum = *bit_accum; + unsigned int bits = *num_bits; + unsigned int entry, len; + + while (1) { + if (bits >= tbits) { + entry = table[accum & ((1U << tbits) - 1)]; + break; + } + + /* Not enough bits for a full lookup. The table being padded, + * looking it up with what we have is still valid as long as it + * lands on a code no longer than that; otherwise we need more + * data to tell. + */ + entry = table[accum & ((1U << bits) - 1)]; + len = entry & 0xF; + if (len && len <= bits) + break; + + if (*in_ptr >= in_top) { + *num_bits = bits; + *bit_accum = accum; + return 0; + } + accum |= (unsigned long)(*in_ptr)[0] << bits; + bits += 8; + (*in_ptr)++; + } + + len = entry & 0xF; + if (!len) { + /* too long for the table: hand the refilled accumulator back + * untouched so that the tree walk starts on the first bit of + * the code. + */ + *num_bits = bits; + *bit_accum = accum; + return -1; + } + *num_bits = bits - len; + *bit_accum = accum >> len; + *var = entry >> 7; + return 1; +} /* The set of macros below is relevant for uslz_decode_block() function */ +/* Fast path for the bit reader. Without it the accumulator is refilled one + * byte at a time, each byte costing a test against in_top and a test for + * "do I have enough bits yet", both of which depend on the data and are + * mispredicted a lot: on a silesia compressed by slz they account for about + * half of the branch misses of the whole decoder. + * + * REFILL() instead brings a whole machine word in with a single unaligned + * load and no branch beyond the one checking that the word is readable. It + * leaves the accumulator with everything but its low byte valid, that is 56 + * to 63 bits on a 64-bit platform, which covers the longest symbol any block + * type can produce (15 bits of code, 5 of length extra, 15 of distance code + * and 13 of distance extra, so 48): one call per symbol is then enough and + * every "do I have enough bits" test inside the decoders becomes a test that + * is always false, and therefore free. A 32-bit platform only gets 24 to 31 + * bits, enough for a literal or a length but not always for a whole + * length/distance pair, and the readers keep their byte at a time refill for + * what a refill did not cover, so they remain correct, they just take that + * slower path more often. + * + * The number of whole bytes the load brought in is written + * "num_bits ^ (8 * __SIZEOF_LONG__ - 1)" rather than the subtraction it is + * equal to as long as stays below the width of the accumulator, + * which it always does here. The compiler cannot know that bound, so the + * subtraction makes it emit the constant, a subtraction and a sign extension + * of a result it cannot prove positive, where the xor becomes a shift and an + * xor on the low byte. + * + * Bits above are not zero any more after this: the load brings in + * up to 7 bytes, 3 on a 32-bit platform, that are not accounted for, and + * is left pointing at them so that the next refill reads them again + * and ORs the same values at the same places. That is harmless because every + * reader masks what it takes. + * + * Nothing is ever pushed back to the input: a byte that made it into the + * accumulator counts as consumed, exactly as it did when they were pulled in + * one at a time, so a caller that is told it has fed us N bytes never has to + * present them again. The two places that want bytes rather than bits, a + * stored block and the trailer, get back on a byte boundary with ALIGN_BYTE() + * and then take those bytes from the accumulator before touching . + * + * The only condition is UNALIGNED_LE_OK. A refill of 24 bits is worth having + * as much as one of 56: built with -m32 on an x86, it decodes a fixed huffman + * stream 26% faster and a dynamic one 8% faster than the byte at a time loop + * it replaces, two data dependent branches per byte being what those bits are + * traded for. + */ +#if defined(UNALIGNED_LE_OK) +#define REFILL() \ + do { \ + if (__builtin_expect(in_top - in_ptr >= __SIZEOF_LONG__, 1)) { \ + bit_accum |= read_lelong(in_ptr) << num_bits; \ + in_ptr += (num_bits ^ (8 * __SIZEOF_LONG__ - 1)) >> 3; \ + num_bits |= 8 * __SIZEOF_LONG__ - 8; \ + } \ + } while (0) + +/* handle up to one literal at output position . This may only be used + * inside uslz_decode_block() immediately after REFILL() in the loop. The "if" + * block is used to automatically adapt to the size of the long and handle + * either 2 or 6 symbols at once. Note that num_bits may contain bit 7 set and + * will need to be masked before being consumed at the end of the calls. + */ +#define FIXED_LITERAL(ofs) \ + do { \ + if (ofs < __SIZEOF_LONG__ - 2) { \ + unsigned int entry; \ + entry = fixed_huff_dec_table[bit_accum & 0x1FF]; \ + bit_accum >>= entry & 0x3F; \ + num_bits -= entry; \ + state->symbol = entry >> 7; \ + if (state->symbol >= 256) { \ + num_bits &= 0x3F; \ + goto have_symbol; \ + } \ + *out++ = state->symbol; \ + } \ + } while (0) + +/* Drops the bits left in the byte being read so that the accumulator holds + * whole bytes only, and clears whatever REFILL() left above them. + * must be below the width of the accumulator here, which REFILL() guarantees. + */ +#define ALIGN_BYTE() \ + do { \ + bit_accum >>= num_bits & 7; \ + num_bits &= ~7; \ + bit_accum &= (1UL << num_bits) - 1; \ + } while (0) +#else +#define REFILL() do { } while (0) +#define ALIGN_BYTE() \ + do { \ + num_bits = 0; \ + bit_accum = 0; \ + } while (0) +#endif + /* The GETBITS macro retrieves the specified number of bits (n) from * the block, returning from the function if no more data is available, * and stores the value in the given variable (var). The number of - * bits to retrieve (n) must be no greater than 32. + * bits to retrieve (n) must be no greater than 24, so that the byte at a time + * refill cannot overflow even a 32-bit accumulator; the widest used below is + * 16. */ #define GETBITS(n,var) \ do { \ @@ -377,7 +585,7 @@ static inline uint8_t rbit5(uint8_t v) if (!bit_accumulate(&in_ptr, in_top, &num_bits, &bit_accum))\ goto out_of_data; \ } \ - var = bit_accum & ((1ULL << n) - 1); \ + var = bit_accum & ((1UL << n) - 1); \ bit_accum >>= n; \ num_bits -= n; \ } while (0) @@ -417,7 +625,7 @@ static inline uint8_t rbit5(uint8_t v) * the current checksum batch. */ #define SET_OUT_LIM() do { \ - unsigned long __room = out_max - 1 - dec_bsize; \ + unsigned long __room = out_max - dec_bsize; \ \ out_lim = out_end; \ if ((unsigned long)(out_end - crc_ptr) > CRC_BLOCK) \ @@ -445,6 +653,55 @@ static inline uint8_t rbit5(uint8_t v) #define CASE_STATE(s) case USLZ_ST_##s: goto state_##s +/* Called when uslz_decode_block() reported a full output. If it was waiting + * for a symbol and that symbol is an end of block, the EOB is consumed, the + * state is set so that uslz_decode_block() finishes the block when called + * again, and non-zero is returned. Otherwise nothing + * changes and zero is returned. The EOB emits nothing and must still be + * consumed, as well as the trailer that may follow, or a caller whose buffer + * is exactly the size of the output would never see the end of the stream. + * It is deliberately not done in uslz_decode_block(), where even an out of + * line call in the checkpoint costs 2 to 5% on this hot loop by disturbing + * its register allocation, and it is kept out of line and holds all the + * tests so as to add as little code as possible to uslz_decode(). + */ +static __attribute__((noinline)) +int uslz_eob_follows(struct uslz_stream *state) +{ + const unsigned char *in_ptr = state->in_ptr; + unsigned long bit_accum = state->bit_accum; + unsigned char num_bits = state->num_bits; + unsigned int hi = state->huff_index; + unsigned int sym; + int ret; + + if (state->dec_bsize < state->out_max || state->state != USLZ_ST_READ_SYMBOL) + return 0; + + if (state->block_type != 2) + ret = gethuff_fixed(&in_ptr, state->in_top, &num_bits, &bit_accum, &sym); + else + ret = gethuff(&hi, &in_ptr, state->in_top, &num_bits, &bit_accum, &sym, state->literal_table); + + if (!ret || sym != 256) + return 0; + + state->in_ptr = in_ptr; + state->bit_accum = bit_accum; + state->num_bits = num_bits; + state->huff_index = 0; + state->symbol = 256; + + /* uslz_decode_block() must now finish the block. Rather than adding a + * state for this, which costs ~1% by moving the hot loop around, we + * resume as a stored block with nothing left to copy, which goes + * straight to the end of the block. + */ + state->nread = state->len; + state->state = USLZ_ST_UNCOMPRESSED_DATA; + return 1; +} + /* main decoding function for inflate API, takes stream as parameter * and decode as much data as possible from input data (as long as output * buffer allows it), it automatically detects uncompressed, fixed or dynamic @@ -468,7 +725,7 @@ static enum uslz_decode_ret uslz_decode_block(struct uslz_stream *state) unsigned char *out = out_base + dec_total % out_max; unsigned char *crc_ptr = out; unsigned char *out_lim; - uint64_t bit_accum = state->bit_accum; + unsigned long bit_accum = state->bit_accum; unsigned char num_bits = state->num_bits; /* whether the ring already wrapped once, i.e. whether the whole * buffer is valid history for the distances. @@ -538,7 +795,8 @@ static enum uslz_decode_ret uslz_decode_block(struct uslz_stream *state) * buffer. */ if (state->block_type == 0) { - num_bits = 0; /* Skip remaining bits in the previous byte. */ + /* Skip remaining bits in the previous byte. */ + ALIGN_BYTE(); state->state = USLZ_ST_UNCOMPRESSED_LEN; state_UNCOMPRESSED_LEN: @@ -565,6 +823,18 @@ static enum uslz_decode_ret uslz_decode_block(struct uslz_stream *state) goto checkpoint; } + /* bytes already pulled into the accumulator by + * REFILL() come first, has moved past them. + */ + if (num_bits >= 8) { + *out++ = bit_accum; + bit_accum >>= 8; + num_bits -= 8; + state->nread++; + remain--; + continue; + } + if (in_ptr >= in_top) goto out_of_data; @@ -706,6 +976,21 @@ static enum uslz_decode_ret uslz_decode_block(struct uslz_stream *state) err_code = USLZ_DECODE_E_GEN_HUFF; goto error_return; } + + /* The tree above can decode any code but costs one dependent + * load per bit. Add a direct lookup table for the literal and + * length codes, which is where most of the symbols are read. + * Its cost is paid once per block and amortised over the many + * thousands of symbols a block holds. A zero width disables + * the table, and the whole thing then folds away. + */ + if (USLZ_FAST_BITS) + gen_fast_table(state->literal_count, state->literal_len, + state->fast_lit, USLZ_FAST_BITS); + + if (USLZ_FAST_DBITS) + gen_fast_table(state->distance_count, state->distance_len, + state->fast_dist, USLZ_FAST_DBITS); } /* else Static tables: we don't need to generate literal / distance table * because we directly use gethuff_fixed(). @@ -725,15 +1010,71 @@ static enum uslz_decode_ret uslz_decode_block(struct uslz_stream *state) goto checkpoint; } + /* one refill per symbol is enough for the longest one */ + REFILL(); + /* Read a compressed symbol from the block. */ if (state->block_type != 2) { +#if defined(UNALIGNED_LE_OK) + /* Literal fast path for fixed huffman blocks, which is + * what slz emits and where three symbols out of four + * are literals. As long as one refill's worth of input + * and as many output bytes as a refill covers are + * available, a literal is one table lookup and one + * store, with none of the per-symbol checks of the + * generic path: a fixed code is at most 9 bits, so a + * refill covers six of them on a 64-bit platform and + * two on a 32-bit one, which is what "__SIZEOF_LONG__ + * - 2" counts below. They are decoded with no test on + * the input, and the output room is checked once for + * all of them. The first symbol that is not a literal + * is handed to the generic path below, which still + * does all the checking on lengths and distances. + */ + while (out_lim - out >= __SIZEOF_LONG__ - 2 && + in_top - in_ptr >= __SIZEOF_LONG__) { + REFILL(); + + /* read up to (sizeof(long) - 2) symbols; + * excess calls are ignored on 32-bit machines. + */ + FIXED_LITERAL(0); FIXED_LITERAL(1); + FIXED_LITERAL(2); FIXED_LITERAL(3); + FIXED_LITERAL(4); FIXED_LITERAL(5); + num_bits &= 0x3F; + } + /* the loop may have filled the output exactly to the + * limit, and the generic path only checks that before + * decoding, at the top of the loop: go back there. + */ + if (out == out_lim) + continue; +#endif /* UNALIGNED_LE_OK */ /* fixed huffman */ if (!gethuff_fixed(&in_ptr, in_top, &num_bits, &bit_accum, &state->symbol)) goto out_of_data; } - else - GETHUFF(state->symbol, state->literal_table); + else { + /* being non-zero means a tree walk was + * interrupted by a lack of data and has to be resumed; + * the fast path cannot be used then, as some bits of + * the code have already been consumed. + */ + int ret = -1; + if (USLZ_FAST_BITS && !state->huff_index) + ret = gethuff_fast(&in_ptr, in_top, &num_bits, &bit_accum, + &state->symbol, state->fast_lit, + USLZ_FAST_BITS); + if (!ret) + goto out_of_data; + if (ret < 0) + GETHUFF(state->symbol, state->literal_table); + } + +#if defined(UNALIGNED_LE_OK) + have_symbol: +#endif /* UNALIGNED_LE_OK */ /* If the symbol is a literal, add it to the buffer and continue * with the next code. */ @@ -784,8 +1125,19 @@ static enum uslz_decode_ret uslz_decode_block(struct uslz_stream *state) bit_accum >>= 5; num_bits -= 5; } - else - GETHUFF(state->symbol, state->distance_table); + else { + /* same fast path as for the literal codes above */ + int ret = -1; + + if (USLZ_FAST_DBITS && !state->huff_index) + ret = gethuff_fast(&in_ptr, in_top, &num_bits, &bit_accum, + &state->symbol, state->fast_dist, + USLZ_FAST_DBITS); + if (!ret) + goto out_of_data; + if (ret < 0) + GETHUFF(state->symbol, state->distance_table); + } if (state->symbol <= 3) state->distance = state->symbol + 1; @@ -836,13 +1188,21 @@ static enum uslz_decode_ret uslz_decode_block(struct uslz_stream *state) * ring, which is only valid history once the * ring has wrapped at least once. This test * replaces the former distance_avail counter: - * a distance never exceeds 32K and the ring is - * at least that large, so "have we produced at - * least bytes" is exactly "has the - * ring wrapped, or is the distance within the - * current position". + * "have we produced at least bytes" + * is exactly "has the ring wrapped, or is the + * distance within the current position", as + * long as the distance doesn't exceed the ring. + * It can only do so with a ring smaller than + * 32K, where such a match would start before + * . Testing it here only costs in + * this rarely taken branch. Note the likely() + * test below that proves to improve the perf + * while an unlikely degrades them. This is + * likely due to the short exit path compared + * to the heavier memmove() that's better kept + * away from the rest of the hot path. */ - if (!wrapped) { + if (__builtin_expect(!wrapped || state->distance > out_max, 1)) { err_code = USLZ_DECODE_E_DISTANCE; goto error_return; } @@ -876,6 +1236,7 @@ static enum uslz_decode_ret uslz_decode_block(struct uslz_stream *state) do { *dst++ = *src++; + __asm__("" : "+r"(dst)); // keep gcc's vectorizer off to save 10% perf } while (--cnt); } } @@ -925,7 +1286,7 @@ static enum uslz_decode_ret uslz_decode_block(struct uslz_stream *state) /* no more room in the decoded block, ask for a drain. The state to * resume from was set by whoever jumped here. */ - if (dec_bsize + 1 >= out_max) + if (dec_bsize >= out_max) goto out_of_space; SET_OUT_LIM(); @@ -955,8 +1316,7 @@ static enum uslz_decode_ret uslz_decode_block(struct uslz_stream *state) * accumulator for that purpose since it is not used for * decoding anymore. */ - bit_accum = 0; - num_bits = 0; + ALIGN_BYTE(); /* the trailer may well not be available yet, in which case we * have to be able to come back here on the next call. */ @@ -970,20 +1330,20 @@ static enum uslz_decode_ret uslz_decode_block(struct uslz_stream *state) while (num_bits < 32) { if (in_ptr >= in_top) goto out_of_data; - /* the cast matters: in_ptr[0] is promoted to a - * signed int, so for the first byte, whose - * shift is 24, any value >= 0x80 would become - * negative and be sign-extended over the upper - * half of the 64-bit accumulator. + /* the four bytes are stored big endian while + * the accumulator holds them in the order they + * arrived, hence the swap below. */ - bit_accum |= (uint32_t)in_ptr[0] << (24 - num_bits); + bit_accum |= (unsigned long)in_ptr[0] << num_bits; in_ptr++; num_bits += 8; } - if (state->crc != bit_accum) { + if (state->crc != __builtin_bswap32((uint32_t)bit_accum)) { err_code = USLZ_DECODE_E_BAD_CRC; goto error_return; } + bit_accum = 0; + num_bits = 0; } else if ((state->flags & USLZ_FL_GZIP)) { /* The gzip trailer is 8 bytes: crc32 then isize, both @@ -996,7 +1356,9 @@ static enum uslz_decode_ret uslz_decode_block(struct uslz_stream *state) while (num_bits < 64) { if (in_ptr >= in_top) goto out_of_data; - bit_accum |= (uint64_t)in_ptr[0] << num_bits; + if (__SIZEOF_LONG__ > 4 || + num_bits < 8 * __SIZEOF_LONG__) + bit_accum |= (unsigned long)in_ptr[0] << num_bits; in_ptr++; num_bits += 8; } @@ -1084,7 +1446,7 @@ static int uslz_next_member(struct uslz_stream *state) */ return 0; } - state->bit_accum |= (uint64_t)state->in_ptr[0] << state->num_bits; + state->bit_accum |= (unsigned long)state->in_ptr[0] << state->num_bits; state->in_ptr += 1; state->num_bits += 8; } @@ -1162,7 +1524,7 @@ enum uslz_decode_ret uslz_decode(struct uslz_stream *state, unsigned char **decoded_data, long *decoded_size, long *consumed_bytes, uint32_t *crc_ret) { - enum uslz_decode_ret res; + enum uslz_decode_ret res = USLZ_DECODE_SUCCESS; /* ensure input consistency */ if (compressed_data == NULL || compressed_size < 0 || @@ -1503,6 +1865,10 @@ enum uslz_decode_ret uslz_decode(struct uslz_stream *state, case USLZ_DECODE_SUCCESS: break; case USLZ_DECODE_OUT_OF_SPACE: + /* a full output may only be waiting for an EOB */ + if (__builtin_expect(uslz_eob_follows(state), 0)) + continue; + goto drain; case USLZ_DECODE_OUT_OF_DATA: goto drain; default: @@ -1553,10 +1919,11 @@ enum uslz_decode_ret uslz_decode(struct uslz_stream *state, * It will be used by slz_inflate as a rotating buffer to decode the stream * and allow the caller to read decoded data. As such it is exclusively managed * by slz_inflate and must not be modified by the caller, only read from during - * the whole decoding process. Minimum size should be 32K, because zlib algo - * specifies that up to 32K bytes distance can be used, thus the inflate API - * should always be able to go 32K bytes in the past at any time with such a - * buffer. + * the whole decoding process. The size should be at least 32K, because deflate + * permits references up to 32K bytes in the past, which a smaller buffer + * cannot hold once it has wrapped: such references are then rejected with + * USLZ_DECODE_E_DISTANCE, even in a valid stream. A smaller buffer is thus + * only suited to a stream whose whole output fits in it. * * Returns 1 on success and 0 on failure. */ @@ -1565,7 +1932,7 @@ int uslz_init(struct uslz_stream *state, { struct uslz_stream zero = { 0 }; - if (state == NULL || output_size < 32768 || output_buffer == NULL) + if (state == NULL || output_size < 1 || output_buffer == NULL) return 0; /* zero out */ @@ -1638,54 +2005,55 @@ static inline void __uslz_prepare_fixed_huff_dec_table(void) * * symbols and len are stored on 16bits unsigned integers. We use exactly 512 indexes * (from 0 to 511 which is 9 bits maximal value) as static encoding is performed using - * at most 9 bits: + * at most 9 bits. Len is stored on 6 bits so that the compiler knows it can drop the + * mask when using them for a shift on archs that mask upper bits in shift counts. * * Each value in the array is stored this way: * * 9 bits for the symbol (A) - * 3 unused bits - * 4 bits for the len (B) + * 1 unused bit + * 6 bits for the len (B), only 4 of which are set * * ie: - * AAAAAAAA AXXXBBBB + * AAAAAAAA AXBBBBBB */ // we revese the codes in order to directly compare them as they appear - // in the bit accumulator in huffman decoding func using rev_short(), the + // in the bit accumulator in huffman decoding func using rbit9(), the // RFC doesn't specify bit order is inverted but this is the only way this // works in practise. while (it <= 143) { // 00110000X through 10111111X - fixed_huff_dec_table[rev_short((0x30 + it), 8) | 0x100] = it << 7; // symbol - fixed_huff_dec_table[rev_short((0x30 + it), 8) | 0x100] |= 8; // len - fixed_huff_dec_table[rev_short((0x30 + it), 8)] = it << 7; // symbol - fixed_huff_dec_table[rev_short((0x30 + it), 8)] |= 8; // len + fixed_huff_dec_table[rbit9((0x30 + it), 8) | 0x100] = it << 7; // symbol + fixed_huff_dec_table[rbit9((0x30 + it), 8) | 0x100] |= 8; // len + fixed_huff_dec_table[rbit9((0x30 + it), 8)] = it << 7; // symbol + fixed_huff_dec_table[rbit9((0x30 + it), 8)] |= 8; // len it += 1; } while (it <= 255) { // 110010000 through 111111111 - fixed_huff_dec_table[rev_short(0x190 + it - 144, 9)] = it << 7; // symbol - fixed_huff_dec_table[rev_short(0x190 + it - 144, 9)] |= 9; // len + fixed_huff_dec_table[rbit9(0x190 + it - 144, 9)] = it << 7; // symbol + fixed_huff_dec_table[rbit9(0x190 + it - 144, 9)] |= 9; // len it += 1; } while (it <= 279) { // 0000000XX through 0010111XX - fixed_huff_dec_table[rev_short((0x0 + it - 256), 7)] = it << 7; // symbol - fixed_huff_dec_table[rev_short((0x0 + it - 256), 7)] |= 7; // len - fixed_huff_dec_table[rev_short((0x0 + it - 256), 7) | 0x80] = it << 7; // symbol - fixed_huff_dec_table[rev_short((0x0 + it - 256), 7) | 0x80] |= 7; // len - fixed_huff_dec_table[rev_short((0x0 + it - 256), 7) | 0x100] = it << 7; // symbol - fixed_huff_dec_table[rev_short((0x0 + it - 256), 7) | 0x100] |= 7; // len - fixed_huff_dec_table[rev_short((0x0 + it - 256), 7) | 0x180] = it << 7; // symbol - fixed_huff_dec_table[rev_short((0x0 + it - 256), 7) | 0x180] |= 7; // len + fixed_huff_dec_table[rbit9((0x0 + it - 256), 7)] = it << 7; // symbol + fixed_huff_dec_table[rbit9((0x0 + it - 256), 7)] |= 7; // len + fixed_huff_dec_table[rbit9((0x0 + it - 256), 7) | 0x80] = it << 7; // symbol + fixed_huff_dec_table[rbit9((0x0 + it - 256), 7) | 0x80] |= 7; // len + fixed_huff_dec_table[rbit9((0x0 + it - 256), 7) | 0x100] = it << 7; // symbol + fixed_huff_dec_table[rbit9((0x0 + it - 256), 7) | 0x100] |= 7; // len + fixed_huff_dec_table[rbit9((0x0 + it - 256), 7) | 0x180] = it << 7; // symbol + fixed_huff_dec_table[rbit9((0x0 + it - 256), 7) | 0x180] |= 7; // len it += 1; } while (it <= 287) { // 11000000X through 11000111X - fixed_huff_dec_table[rev_short((0xC0 + it - 280), 8)] = it << 7; // symbol - fixed_huff_dec_table[rev_short((0xC0 + it - 280), 8)] |= 8; // len - fixed_huff_dec_table[rev_short((0xC0 + it - 280), 8) | 0x100] = it << 7; // symbol - fixed_huff_dec_table[rev_short((0xC0 + it - 280), 8) | 0x100] |= 8; // len + fixed_huff_dec_table[rbit9((0xC0 + it - 280), 8)] = it << 7; // symbol + fixed_huff_dec_table[rbit9((0xC0 + it - 280), 8)] |= 8; // len + fixed_huff_dec_table[rbit9((0xC0 + it - 280), 8) | 0x100] = it << 7; // symbol + fixed_huff_dec_table[rbit9((0xC0 + it - 280), 8) | 0x100] |= 8; // len it += 1; } #endif