#define HAVE_FAST_MULT
#endif
-/* Log2 of the size of the hash table used for the references table. */
-#define HASH_BITS 13
+/* returns the max of <a> and <b>, must be constant since evaluated twice */
+#define MIN_C(a, b) ({ \
+ (a < b) ? a : b; \
+})
-/* uses the most suitable crc32 function to update crc on <buf, len> */
-static inline uint32_t update_crc(uint32_t crc, const void *buf, int len)
+/* returns the min of <a> and <b>; both must be of compatible types and are
+ * evaluated exactly once.
+ */
+#define MIN(a, b) ({ \
+ typeof(a) _a = (a); \
+ typeof(a) _b = (b); \
+ (_a < _b) ? _a : _b; \
+})
+
+/* returns the min of <a> and <b>; both cast to type <t>, are evaluated exactly
+ * once.
+ */
+#define MIN_T(t, a, b) ({ \
+ t _a = (t)(a); \
+ t _b = (t)(b); \
+ (_a < _b) ? _a : _b; \
+})
+
+/* returns the max of <a> and <b>, must be constant since evaluated twice */
+#define MAX_C(a, b) ({ \
+ (a > b) ? a : b; \
+})
+
+/* returns the max of <a> and <b>; both must be of compatible types and are
+ * evaluated exactly once.
+ */
+#define MAX(a, b) ({ \
+ typeof(a) _a = (a); \
+ typeof(a) _b = (b); \
+ (_a > _b) ? _a : _b; \
+})
+
+/* returns the max of <a> and <b>; both cast to type <t>, are evaluated exactly
+ * once.
+ */
+#define MAX_T(t, a, b) ({ \
+ t _a = (t)(a); \
+ t _b = (t)(b); \
+ (_a > _b) ? _a : _b; \
+})
+
+
+/* reads an unaligned 32-bit word in little endian order */
+static inline uint32_t read_le32(const void *p)
{
- return slz_crc32_by4(crc, buf, len);
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+ const union { uint32_t u32; } __attribute__((packed)) *u = p;
+ return u->u32;
+#else
+ const uint8_t *u8 = p;
+ return (uint32_t)u8[0] | ((uint32_t)u8[1] << 8) |
+ ((uint32_t)u8[2] << 16) | ((uint32_t)u8[3] << 24);
+#endif
}
-/* This hash provides good average results on HTML contents, and is among the
- * few which provide almost optimal results on various different pages.
- */
-static inline uint32_t slz_hash(uint32_t a)
+/* reads an unaligned 64-bit word in little endian order, same as read_le32() */
+static inline uint64_t read_le64(const void *p)
{
-#if defined(__ARM_FEATURE_CRC32)
-# if defined(__ARM_ARCH_ISA_A64)
- // 64 bit mode
- __asm__ volatile("crc32w %w0,%w0,%w1" : "+r"(a) : "r"(0));
-# else
- // 32 bit mode (e.g. armv7 compiler building for armv8
- __asm__ volatile("crc32w %0,%0,%1" : "+r"(a) : "r"(0));
-# endif
- return a >> (32 - HASH_BITS);
-#elif defined(__SSE4_2__) && defined(USE_CRC32C_HASH)
- // SSE 4.2 offers CRC32C which is a bit slower than the multiply
- // but provides a slightly smoother hash
- __asm__ volatile("crc32l %1,%0" : "+r"(a) : "r"(0));
- return a >> (32 - HASH_BITS);
-#elif defined(HAVE_FAST_MULT)
- // optimal factor for HASH_BITS=12 and HASH_BITS=13 among 48k tested: 0x1af42f
- return (a * 0x1af42f) >> (32 - HASH_BITS);
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+ const union { uint64_t u64; } __attribute__((packed)) *u = p;
+ return u->u64;
#else
- return ((a << 19) + (a << 6) - a) >> (32 - HASH_BITS);
+ const uint8_t *u8 = p;
+ return (uint64_t)read_le32(u8) | ((uint64_t)read_le32(u8 + 4) << 32);
#endif
}
-/* This function compares buffers <a> and <b> and reads 32 or 64 bits at a time
- * during the approach. It makes us of unaligned little endian memory accesses
- * on capable architectures. <max> is the maximum number of bytes that can be
- * read, so both <a> and <b> must have at least <max> bytes ahead. <max> may
- * safely be null or negative if that simplifies computations in the caller.
- */
-static inline long memmatch(const unsigned char *a, const unsigned char *b, long max)
+/* reads an unaligned machine word in little endian order, same as read_le32() */
+static inline unsigned long read_lelong(const void *p)
{
- long len = 0;
-
-#ifdef UNALIGNED_LE_OK
- unsigned long xor;
-
- while (1) {
- if ((long)(len + 2 * sizeof(long)) > max) {
- while (len < max) {
- if (a[len] != b[len])
- break;
- len++;
- }
- return len;
- }
-
- xor = *(long *)&a[len] ^ *(long *)&b[len];
- if (xor)
- break;
- len += sizeof(long);
-
- xor = *(long *)&a[len] ^ *(long *)&b[len];
- if (xor)
- break;
- len += sizeof(long);
- }
-
-#if defined(__x86_64__) || defined(__i386__) || defined(__i486__) || defined(__i586__) || defined(__i686__)
- /* x86 has bsf. We know that xor is non-null here */
- asm("bsf %1,%0\n" : "=r"(xor) : "0" (xor));
- return len + xor / 8;
+#if __SIZEOF_LONG__ > 4
+ return read_le64(p);
#else
- if (sizeof(long) > 4 && !(xor & 0xffffffff)) {
- /* This code is optimized out on 32-bit archs, but we still
- * need to shift in two passes to avoid a warning. It is
- * properly optimized out as a single shift.
- */
- xor >>= 16; xor >>= 16;
- if (xor & 0xffff) {
- if (xor & 0xff)
- return len + 4;
- return len + 5;
- }
- if (xor & 0xffffff)
- return len + 6;
- return len + 7;
- }
-
- if (xor & 0xffff) {
- if (xor & 0xff)
- return len;
- return len + 1;
- }
- if (xor & 0xffffff)
- return len + 2;
- return len + 3;
-#endif // x86
-
-#else // UNALIGNED_LE_OK
- /* This is the generic version for big endian or unaligned-incompatible
- * architectures.
- */
- while (len < max) {
- if (a[len] != b[len])
- break;
- len++;
- }
- return len;
+ return read_le32(p);
+#endif
+}
+/* writes an unaligned 32-bit word in little endian order */
+static inline void write_le32(void *p, uint32_t v)
+{
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+ union { uint32_t u32; } __attribute__((packed)) *u = p;
+ u->u32 = v;
+#else
+ uint8_t *u8 = p;
+ u8[0] = v;
+ u8[1] = v >> 8;
+ u8[2] = v >> 16;
+ u8[3] = v >> 24;
+#endif
+}
+
+/* writes an unaligned 16-bit word in little order */
+static inline void write_le16(void *p, uint16_t v)
+{
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+ union { uint16_t u16; } __attribute__((packed)) *u = p;
+ u->u16 = v;
+#else
+ uint8_t *u8 = p;
+ u8[0] = v;
+ u8[1] = v >> 8;
#endif
}
-/* helper function to reverse bits in <input> which is expected to be
- * <len> bits long.
+/* uses the most suitable crc32 function to update crc on <buf, len> */
+static inline uint32_t update_crc(uint32_t crc, const void *buf, int len)
+{
+ return slz_crc32_by4(crc, buf, len);
+}
+
+/* reverses bits in 5bits len <v> input and returns the new value.
+ * ie: 01100 becomes 00110. Bits 5 and above are ignored.
*/
-static inline short rev_short(short input, int len)
+#if defined(__aarch64__) || defined(__ARM_ARCH_7A__)
+
+/* ARM >= v7 has a fast rbit instruction */
+static inline uint32_t rbit5(uint32_t v)
{
- short ret = 0;
+#if defined(__ARM_ARCH_ISA_A64)
+ // 64 bit mode
+ __asm__ ("rbit %w0, %w0" : "+r" (v));
+#else
+ // 32 bit mode (e.g. armv7 compiler building for armv8
+ __asm__ ("rbit %0, %0" : "+r" (v));
+#endif
+ return v >> 27;
+}
+
+#else /* not arm */
+
+#define rbit5(v) ({ \
+ typeof(v) _v = (v); \
+ ((0xf7b3d591e6a2c480ULL >> ((_v & 0x1e) * 2)) & 0xf) + ((_v & 1) << 4); \
+})
- while (len) {
- ret <<= 1;
- ret |= (input & 1);
- input >>= 1;
- len--;
- }
+#endif
- return ret;
+/* helper function to reverse up to 9 bits in <input> which is expected to be
+ * <len> bits long. The function might work with more than 9 bits in certain
+ * cases with no guarantees, so don't go over 9.
+ */
+static inline uint32_t rbit9(uint32_t input, int len)
+{
+#if defined(__aarch64__) || defined(__ARM_ARCH_7A__)
+ /* ARM >= v7 has a fast rbit instruction */
+ input <<= 32 - len; // align left before reversal
+#if defined(__ARM_ARCH_ISA_A64)
+ // 64 bit mode
+ __asm__ ("rbit %w0, %w0" : "+r" (input));
+#else
+ // 32 bit mode (e.g. armv7 compiler building for armv8
+ __asm__ ("rbit %0, %0" : "+r" (input));
+#endif
+ return input;
+
+#else /* not arm */
+ uint32_t h, l;
+ h = input >> 5;
+ l = input;
+ h = rbit5(h); l = rbit5(l);
+ return ((l << 5) | h) >> (10 - len); /* new_h = [rev(l):5][rev(h):5] */
+#endif
}
#endif
#else
uint32_t queue; /* last pending bits, LSB first, faster on 32-bit archs */
#endif
- uint32_t qbits; /* number of bits in queue, < 8 on 32-bit, < 32 on 64-bit */
unsigned char *outbuf; /* set by encode() */
- uint16_t state; /* one of slz_state */
- uint8_t level:1; /* 0 = no compression, 1 = compression */
+ uint32_t qbits; /* number of bits in queue, < 8 on 32-bit, < 32 on 64-bit */
+ uint32_t state; /* one of slz_state */
+ uint8_t unused:2; /* free for future use */
uint8_t format:2; /* SLZ_FMT_* */
+ uint8_t level:4; /* 0 = no compression, 1..15 = compression */
uint8_t debt; /* number of bits by which the fixed huffman encoding is
* currently behind the equivalent stored blocks, see
* SLZ_MAX_DEBT in slz.c
uint32_t ilen;
};
+/* Note about output buffer sizing: encode functions below do not take an
+ * output len, instead the caller must provide a "large enough" output buffer.
+ * When enabling direct queuing via SLZ_DIRECT_ENQUEUE24, the 3 bytes following
+ * the end of the buffer may be written to, and the caller must take that into
+ * account by allocating a buffer that is at least 3 bytes larger than needed.
+ * And when using SLZ_DIRECT_ENQUEUE56, it's up to 7 extra bytes that may be
+ * needed.
+ */
+
/* Functions specific to rfc1951 (deflate) */
void slz_prepare_dist_table(); /* obsolete, not needed anymore */
long slz_rfc1951_encode(struct slz_stream *strm, unsigned char *out, const unsigned char *in, long ilen, int more);
USLZ_ST_CHECK_CKSUM,
};
+/* Width, in bits, of the direct lookup tables used to decode dynamic huffman
+ * codes: one for the literal/length alphabet, one for the distance alphabet.
+ * A code no longer than the table is decoded in a single load, a longer one
+ * falls back to walking the tree, which costs one dependent load per bit.
+ * Each table costs (1 << bits) * 2 bytes in *every* stream state, so this is
+ * a direct trade between memory and decoding speed. Setting a width to zero
+ * removes the corresponding table from the state along with the code that
+ * uses it, which is the right thing for an application holding many
+ * mostly-idle streams.
+ *
+ * These are deliberately not build options. They set the size and the layout
+ * of struct uslz_stream, which is exported and instantiated by the calling
+ * application, so a -D reaching only one of the two sides would leave the
+ * library and the application disagreeing on the offset of every field past
+ * fast_lit[]: the application would allocate 1568 bytes where the library
+ * writes 2848, and nothing would detect it. Defining them unconditionally is
+ * what prevents this, the definition below wins in every translation unit
+ * that includes this file and a -D trying to change them only earns a
+ * redefinition warning. The values can still be changed, but only here, and
+ * doing so requires rebuilding the library *and* everything that instantiates
+ * a stream.
+ *
+ * The performance below was measured on 8 different machines decoding a
+ * silesia.tar compressed by gzip, with various combinations of literal length
+ * bits and distance bits. The tests were lacking several optimizations that
+ * were later added, so the numbers are off compared to current code, but their
+ * ratios are still correct. The rates are expressed in MB/s of output stream
+ * after decompression:
+ *
+ * lit/dist state arl spr sklx 7800 7v13 5105 atom n1
+ * 0/0 1728 B 237 186 178 232 174 124 50 124
+ * 0/7 1984 B 285 223 199 282 206 144 56 143
+ * 9/0 2752 B 379 286 313 329 239 175 66 182
+ * 8/6 2368 B 433 342 323 378 273 190 69 198
+ * 9/7 3008 B 508 402 381 419 303 222 76 229
+ * 10/7 4032 B 547 437 403 445 324 238 78 245 <= current
+ * 11/7 6080 B 556 446 409 451 325 245 79 250
+ *
+ * It's important not to forget that decompressing gzip requires to keep a
+ * 32 kB sliding window, so the real sizes are those obtained by adding
+ * 32768 to the values above. Once taken into account, and measures reported
+ * as extra percentages compared to 0/0, the best choices appear more clearly:
+ *
+ * lit/dist memory arl spr sklx 7800 7v13 5105 atom n1
+ * 0/0 34496 B 237 186 178 232 174 124 50 124 MB/s
+ * 0/7 +0.7% +20% +20% +12% +21% +18% +17% +11% +15%
+ * 9/0 +3.0% +60% +54% +76% +42% +37% +41% +32% +46%
+ * 8/6 +1.9% +83% +84% +82% +63% +57% +54% +37% +59%
+ * 9/7 +3.7% +114% +117% +115% +80% +74% +80% +52% +84%
+ * 10/7 +6.7% +131% +135% +127% +92% +86% +93% +56% +97% <= current
+ * 11/7 +12.6% +135% +140% +130% +94% +87% +98% +58% +101%
+ *
+ * Each step then costs a few percent of the per-stream footprint and buys
+ * tens of points on that scale, until it stops:
+ *
+ * 8/6 -> 9/7 memory +1.8% speed +14 to +33 points
+ * 9/7 -> 10/7 memory +2.9% speed +5 to +18 points
+ * 10/7 -> 11/7 memory +5.6% speed +1 to +5 points
+ *
+ * 10/7 is where the curve bends, and that is what is set below: against no
+ * table at all it buys between 56% (the Atom) and 135% (the Sapphire Rapids)
+ * on a dynamic stream for 6.7% more memory per stream, while the step past it
+ * costs twice as much memory for a tenth of the speed.
+ *
+ * An embedded application that decodes nothing but slz output is the
+ * exception: those streams never look these tables up, so it can set both
+ * widths to zero and get the 2304 bytes back.
+ */
+#define USLZ_FAST_BITS 10
+#define USLZ_FAST_DBITS 7
+
+#if USLZ_FAST_BITS > 15 || USLZ_FAST_DBITS > 15
+#error "USLZ_FAST_BITS and USLZ_FAST_DBITS must not exceed 15, the longest deflate code"
+#endif
+
/* decompression state */
struct uslz_stream {
enum uslz_stream_state state; /* parsing state, USLZ_ST_* */
unsigned long dec_bofs; /* offset used to read the current decoded block */
unsigned long dec_bsize; /* size of the current decoded block */
- uint64_t bit_accum; /* bit accumulator (used by GETBITS() macro) */
unsigned char num_bits; /* number of bits in the accumulator */
unsigned char block_type; /* compressed block type */
unsigned int huff_index; /* current index for pending huffman decoding
* attempt
*/
+ unsigned long bit_accum; /* bit accumulator (used by GETBITS() macro) */
unsigned int symbol; /* symbol code currently being processed */
unsigned int last_value; /* last value read for length / distance table
* construction
* enough space for that alphabet.
*/
short literal_table[288*2-2];
+ /* Direct lookup table for the literal/length alphabet, indexed by the
+ * next USLZ_FAST_BITS bits of the stream. Same layout as
+ * fixed_huff_dec_table: the symbol in the upper bits and the code
+ * length in the low 4 bits, a length of zero meaning the code is
+ * longer than the table and literal_table[] has to be walked.
+ */
+ uint16_t fast_lit[1 << USLZ_FAST_BITS];
+
+ /* same for the distance alphabet */
+ uint16_t fast_dist[1 << USLZ_FAST_DBITS];
+
union {
/* Code-to-symbol conversion table for the alphabet used for
* distances. This alphabet consists of 32 symbols, 2 of
*/
struct {
unsigned char buf[10];
+ unsigned char gzip_flags;
int buf_len;
- char gzip_flags;
} hdr_detect;
};
short codelen_table[19*2-2]; /* Code-to-symbol conversion table for the alphabet
--- /dev/null
+/* Only needed for slz*.c */
+#include <import/slz-prv.h>
#include <stdio.h>
#include <string.h>
-#include "import/slz.h"
-#include "import/slz-prv.h"
-#include "import/slz-tables.h"
+#include "slz.h"
+#include "slz-prv.h"
+#include "tables.h"
/* if PRECOMPUTE_TABLES not set, then tables.h does not provide fh_dist_table
* table as we need to recompute it.
static uint32_t fh_dist_table[32768];
#endif // ifndef PRECOMPUTE_TABLES
+/* The enqueue functions have two possible implementations. The default one
+ * only writes to the output buffer once it holds a whole store's worth of
+ * bits, which costs a test but never writes a byte it does not advance over.
+ * The "direct" one, selected by these macros, stores unconditionally and then
+ * advances by however many whole bytes the queue turned out to hold, leaving
+ * fewer than 8 bits behind. That saves a branch, and requires the caller to
+ * leave room past the output pointer for the widest store used (3 bytes for
+ * enqueue24() and enqueue8(), 7 for enqueue56()), since the bytes beyond the
+ * advance are rewritten by the next call. They're only usable for unaligned
+ * little endian archs and generally work better on modern x86_64. Tests show
+ * that x86_64 and arm64 benefit greatly from this.
+ *
+ * SLZ_DIRECT_ENQUEUE56 goes further by writing a whole length+distance pair in
+ * a single operation instead of two.
+ */
+
+/* directly write 32 bits at a time to output for up to 24 bits */
+#ifndef SLZ_DIRECT_ENQUEUE24
+# if defined(UNALIGNED_LE_OK)
+# define SLZ_DIRECT_ENQUEUE24 1
+# else
+# define SLZ_DIRECT_ENQUEUE24 0
+# endif
+#endif
+
+/* directly write 8 bits at a time to the output */
+#ifndef SLZ_DIRECT_ENQUEUE8
+# define SLZ_DIRECT_ENQUEUE8 SLZ_DIRECT_ENQUEUE24
+#endif
+
+/* directly write 64 bits at a tome to output for up to 56 bits */
+#ifndef SLZ_DIRECT_ENQUEUE56
+# if defined(__SIZEOF_LONG__) && (__SIZEOF_LONG__ - 0 > 4) && defined(UNALIGNED_LE_OK)
+# define SLZ_DIRECT_ENQUEUE56 1
+# else
+# define SLZ_DIRECT_ENQUEUE56 0
+# endif
+#endif
+
+/* The direct enqueue8() writes a single byte, which is only enough because it
+ * assumes fewer than 8 bits were already queued. That holds for every enqueue
+ * variant except the conditional 64-bit enqueue24(), which deliberately lets
+ * up to 31 bits accumulate before storing. Combining the two would advance the
+ * output pointer over bytes that were never written.
+ */
+#if SLZ_DIRECT_ENQUEUE8 && !SLZ_DIRECT_ENQUEUE24 && \
+ defined(__SIZEOF_LONG__) && (__SIZEOF_LONG__ - 0 > 4) && defined(UNALIGNED_LE_OK)
+# error "SLZ_DIRECT_ENQUEUE8 requires SLZ_DIRECT_ENQUEUE24 on this platform"
+#endif
+
+/* enqueue56() needs a 64-bit queue and unaligned little-endian stores. */
+#if SLZ_DIRECT_ENQUEUE56 && !(defined(__SIZEOF_LONG__) && (__SIZEOF_LONG__ - 0 > 4) && defined(UNALIGNED_LE_OK))
+# undef SLZ_DIRECT_ENQUEUE56
+# define SLZ_DIRECT_ENQUEUE56 0
+#endif
+
+/* Also indexes the positions covered by the beginning of a match. This is quite
+ * important for example on HTML or other streams with lots of varying prefixes,
+ * especially when combined with the backwards match, because otherwise a likely
+ * repeating element will not be indexed, and when the window advances, its ref
+ * is likely to disappear, causing the next one to experience a miss.
+ */
+#ifndef SLZ_INSERT_SKIPPED
+# define SLZ_INSERT_SKIPPED 1
+#endif
+
+/* Skip large non-compressible areas and send them as literals to avoid wasting
+ * CPU trying to compress them. It's faster on incompressible data, at the cost
+ * of a slightly reduced compression ratio.
+ */
+#ifndef SLZ_LITERAL_SKIP
+# define SLZ_LITERAL_SKIP 1
+#endif
+
+/* Enables growing a match backwards over the literals that are still pending,
+ * see slz_rfc1951_encode(). Costs a little CPU on every match and gains about
+ * 1% of output. Tests show that matching up to 2 bytes brings almost optimal
+ * results at minimal extra cost, and that compression gains at 1 byte are
+ * higher than the CPU increase they cause, and tend to result in higher
+ * compression speed due to the savings on encoding. The only way to disable
+ * this optimization is to comment out this line.
+ */
+#define SLZ_BACKWARD_MATCH ((const int[]){ 0, 1, 2, 8 })
+
+/* Log2 of the size of the hash table used for the references table. */
+#ifndef SLZ_HASH_BITS
+# define SLZ_HASH_BITS 13
+#endif
+
/* First, RFC1951-specific declarations and extracts from the RFC.
*
* RFC1951 - deflate stream format
*/
+/* for use to temporarily disable a builtin_expect */
+#define __no_builtin_expect(cond, exp) (cond)
+
/* back references, built in a way that is optimal for 32/64 bits */
/* format:
* bit0..31 = word
uint64_t by64;
};
-#if defined(__SIZEOF_LONG__) && (__SIZEOF_LONG__ - 0 > 4) && defined(UNALIGNED_LE_OK)
+/* This hash provides good average results on HTML contents, and is among the
+ * few which provide almost optimal results on various different pages.
+ */
+static inline uint32_t slz_hash(uint32_t a)
+{
+#if defined(__ARM_FEATURE_CRC32)
+# if defined(__ARM_ARCH_ISA_A64)
+ // 64 bit mode
+ __asm__ volatile("crc32w %w0,%w0,%w1" : "+r"(a) : "r"(0));
+# else
+ // 32 bit mode (e.g. armv7 compiler building for armv8
+ __asm__ volatile("crc32w %0,%0,%1" : "+r"(a) : "r"(0));
+# endif
+#elif defined(__SSE4_2__) && defined(USE_CRC32C_HASH)
+ // SSE 4.2 offers CRC32C which is a bit slower than the multiply
+ // but provides a slightly smoother hash
+ __asm__ volatile("crc32l %1,%0" : "+r"(a) : "r"(0));
+#elif defined(HAVE_FAST_MULT)
+ // optimal factor for SLZ_HASH_BITS=12 and SLZ_HASH_BITS=13 among 48k tested: 0x1af42f
+ a = (a * 0x1af42f); // 12: 44.12, 13: 43.35
+ //return (a * 0x1aaee3) >> (32 - SLZ_HASH_BITS); // 12: 44.14, 13: 43.34
+ //return (a * 0x1aae0b) >> (32 - SLZ_HASH_BITS); // 12: 44.15, 13: 43.35, -30k
+ //return (a * 0x17e7bd) >> (32 - SLZ_HASH_BITS); // 12: 44.15, 13: 43.36
+ //return (a * 0xaae7b) >> (32 - SLZ_HASH_BITS); // 12: 44.22, 13:43.35, -1k
+ //return (a * 0xaaec3) >> (32 - SLZ_HASH_BITS); // 12: 44.20, 13: 43.35, -10k // 275659647
+#else
+ a = ((a << 19) + (a << 6) - a);
+#endif
+ return a >> (32 - SLZ_HASH_BITS);
+}
+
+/* This function compares buffers <a> and <b> and reads 32 or 64 bits at a time
+ * during the approach. It makes us of unaligned little endian memory accesses
+ * on capable architectures. <max> is the maximum number of bytes that can be
+ * read, so both <a> and <b> must have at least <max> bytes ahead. <max> may
+ * safely be null or negative if that simplifies computations in the caller.
+ */
+static inline long memmatch(const unsigned char *a, const unsigned char *b, long max)
+{
+ long len = 0;
+
+#ifdef UNALIGNED_LE_OK
+
+ while (len + 2 * (long)sizeof(long) - 1 < max) {
+ unsigned long xor;
+
+ xor = *(unsigned long *)&a[len] ^ *(unsigned long *)&b[len];
+ if (xor)
+ return len + (__builtin_ctzl(xor) >> 3);
+ len += (long)sizeof(long);
+
+ xor = *(unsigned long *)&a[len] ^ *(unsigned long *)&b[len];
+ if (xor)
+ return len + (__builtin_ctzl(xor) >> 3);
+ len += (long)sizeof(long);
+ }
+
+ while (len + (long)sizeof(long) - 1 < max) {
+ unsigned long xor;
+
+ xor = *(unsigned long *)&a[len] ^ *(unsigned long *)&b[len];
+ if (xor)
+ return len + (__builtin_ctzl(xor) >> 3);
+ len += (long)sizeof(long);
+ }
+
+ if (sizeof(long) > 4 && len + 3 < max) {
+ uint32_t xor;
+
+ xor = *(uint32_t *)&a[len] ^ *(uint32_t *)&b[len];
+ if (xor)
+ return len + (__builtin_ctz(xor) >> 3);
+ len += 4;
+ }
+
+ if (len + 1 < max) {
+ uint16_t xor;
+
+ xor = *(uint16_t *)&a[len] ^ *(uint16_t *)&b[len];
+ if (xor)
+ return len + !(uint8_t)xor;
+ len += 2;
+ }
+
+ if (len < max && a[len] == b[len])
+ len++;
+ return len;
+
+#else // UNALIGNED_LE_OK
+ /* This is the generic version for big endian or unaligned-incompatible
+ * architectures.
+ */
+ while (len < max) {
+ if (a[len] != b[len])
+ break;
+ len++;
+ }
+ return len;
+
+#endif
+}
+
+#if defined(__SIZEOF_LONG__) && (__SIZEOF_LONG__ - 0 > 4)
/* enqueue code x of <xbits> bits (LSB aligned, at most 24) and copy complete
* 32-bit words into output buffer. X must not contain non-zero bits above
- * xbits.
+ * xbits. Note that the function may write up to 3 extra bytes since it may
+ * write 32 bits even if only 8 are needed.
*/
-static inline void enqueue24(struct slz_stream *strm, uint64_t x, uint32_t xbits)
+static inline void enqueue24(struct slz_stream *__restrict strm, uint64_t x, uint32_t xbits)
{
uint64_t queue = strm->queue + (x << strm->qbits);
uint32_t qbits = strm->qbits + xbits;
+#if SLZ_DIRECT_ENQUEUE24
+ write_le32(strm->outbuf, queue);
+ queue >>= qbits & ~7U;
+
+ strm->queue = queue;
+ strm->outbuf += qbits >> 3;
+ strm->qbits = qbits & 7;
+#else
if (__builtin_expect(qbits >= 32, 1)) {
- *(uint32_t *)strm->outbuf = queue;
+ write_le32(strm->outbuf, queue);
queue >>= 32;
qbits -= 32;
strm->outbuf += 4;
strm->queue = queue;
strm->qbits = qbits;
+#endif
}
+/* enqueue code x of <xbits> bits (at most 8) and copy complete bytes into out
+ * buf. X must not contain non-zero bits above xbits.
+ */
+#if SLZ_DIRECT_ENQUEUE8
+static inline void enqueue8(struct slz_stream *__restrict strm, uint32_t x, uint32_t xbits)
+{
+ uint64_t queue = strm->queue + ((uint64_t)x << strm->qbits);
+ uint32_t qbits = strm->qbits + xbits;
+
+ *strm->outbuf = queue;
+ strm->outbuf += qbits >= 8;
+ queue >>= qbits & 8;
+ strm->queue = queue;
+ strm->qbits = qbits & 7;
+}
+#else
#define enqueue8 enqueue24
+#endif
-/* flush the queue and align to next byte */
-static inline void flush_bits(struct slz_stream *strm)
+#if SLZ_DIRECT_ENQUEUE56
+/* Enqueues code <x> of <xbits> bits (LSB aligned) and stores the low 8 bytes
+ * of the queue, then advances over the whole bytes it holds. <xbits> plus
+ * whatever was already queued must stay below 64, and the caller must leave 8
+ * bytes of room past strm->outbuf rather than the 4 the other enqueues need.
+ * It exists to send a length code and the distance code that follows it in one
+ * go, which is at most 31 bits: 7 or 8 for the length code plus up to 5 extra,
+ * then 5 for the distance code plus up to 13 extra.
+ */
+static inline void enqueue56(struct slz_stream *__restrict strm, uint64_t x, uint32_t xbits)
{
- if (strm->qbits > 0)
- *strm->outbuf++ = strm->queue;
+ uint64_t queue = strm->queue + (x << strm->qbits);
+ uint32_t qbits = strm->qbits + xbits;
+
+ *(uint64_t *)strm->outbuf = queue;
+ queue >>= qbits & ~7U;
+ strm->queue = queue;
+ strm->outbuf += qbits >> 3;
+ strm->qbits = qbits & 7;
+}
+#endif
+
+/* flush the queue and align to next byte */
+static inline void flush_bits(struct slz_stream *__restrict strm)
+{
+ if (!strm->qbits)
+ return;
+
+ *strm->outbuf++ = strm->queue;
+
+#if !SLZ_DIRECT_ENQUEUE24
+ /* Only the conditional enqueue24() can leave more than 7 bits in the
+ * queue, and only it needs the three extra stores. The direct one
+ * masks the count with 7 on every path, as do enqueue8() and
+ * enqueue56(), so a single byte covers every case there.
+ *
+ * This is not free to get wrong: flush_bits() is inlined into
+ * copy_lit() and into all six flush/finish entry points, so the three
+ * dead stores cost 585 bytes of text and 3% of throughput on silesia
+ * once SLZ_LITERAL_SKIP starts calling copy_lit() for every skipped
+ * chunk.
+ */
if (strm->qbits > 8)
*strm->outbuf++ = strm->queue >> 8;
+#if !SLZ_DIRECT_ENQUEUE8
if (strm->qbits > 16)
*strm->outbuf++ = strm->queue >> 16;
if (strm->qbits > 24)
*strm->outbuf++ = strm->queue >> 24;
+#endif
+#endif
strm->queue = 0;
strm->qbits = 0;
}
-#else /* non-64 bit or aligned or big endian */
+#else /* non-64 bit */
/* enqueue code x of <xbits> bits (LSB aligned, at most 24) and copy complete
* bytes into out buf. X must not contain non-zero bits above xbits. Prefer
- * enqueue8() when xbits is known for being 8 or less.
+ * enqueue8() when xbits is known for being 8 or less. Note that the function
+ * may write up to 3 extra bytes since it may write 32 bits even if only 8 are
+ * needed.
*/
-static void enqueue24(struct slz_stream *strm, uint32_t x, uint32_t xbits)
+static void enqueue24(struct slz_stream *__restrict strm, uint32_t x, uint32_t xbits)
{
uint32_t queue = strm->queue + (x << strm->qbits);
uint32_t qbits = strm->qbits + xbits;
- if (qbits >= 16) {
-#ifndef UNALIGNED_LE_OK
- strm->outbuf[0] = queue;
- strm->outbuf[1] = queue >> 8;
-#else
- *(uint16_t *)strm->outbuf = queue;
+#if SLZ_DIRECT_ENQUEUE24
+ write_le32(strm->outbuf, queue);
+ queue >>= qbits & ~7U;
+ strm->outbuf += qbits >> 3;
+ strm->qbits = qbits & 7;
+ strm->queue = queue;
+ return;
#endif
+ if (qbits >= 16) {
+ write_le16(strm->outbuf, queue);
strm->outbuf += 2;
queue >>= 16;
qbits -= 16;
/* enqueue code x of <xbits> bits (at most 8) and copy complete bytes into
* out buf. X must not contain non-zero bits above xbits.
*/
-static inline void enqueue8(struct slz_stream *strm, uint32_t x, uint32_t xbits)
+static inline void enqueue8(struct slz_stream *__restrict strm, uint32_t x, uint32_t xbits)
{
uint32_t queue = strm->queue + (x << strm->qbits);
uint32_t qbits = strm->qbits + xbits;
+#if SLZ_DIRECT_ENQUEUE8
+ *strm->outbuf = queue;
+ strm->outbuf += qbits >= 8;
+ queue >>= qbits & 8;
+ strm->queue = queue;
+ strm->qbits = qbits & 7;
+ return;
+#endif
if (__builtin_expect((signed)(qbits - 8) >= 0, 1)) {
qbits -= 8;
*strm->outbuf++ = queue;
}
/* align to next byte */
-static inline void flush_bits(struct slz_stream *strm)
+static inline void flush_bits(struct slz_stream *__restrict strm)
{
- if (strm->qbits > 0)
- *strm->outbuf++ = strm->queue;
+ if (!strm->qbits)
+ return;
+
+ *strm->outbuf++ = strm->queue;
+#if !SLZ_DIRECT_ENQUEUE8
if (strm->qbits > 8)
*strm->outbuf++ = strm->queue >> 8;
+#endif
strm->queue = 0;
strm->qbits = 0;
/* only valid if buffer is already aligned */
-static inline void copy_8b(struct slz_stream *strm, uint32_t x)
+static inline void copy_8b(struct slz_stream *__restrict strm, uint32_t x)
{
*strm->outbuf++ = x;
}
/* only valid if buffer is already aligned */
-static inline void copy_16b(struct slz_stream *strm, uint32_t x)
+static inline void copy_16b(struct slz_stream *__restrict strm, uint32_t x)
{
- strm->outbuf[0] = x;
- strm->outbuf[1] = x >> 8;
+ write_le16(strm->outbuf, x);
strm->outbuf += 2;
}
/* only valid if buffer is already aligned */
-static inline void copy_32b(struct slz_stream *strm, uint32_t x)
+static inline void copy_32b(struct slz_stream *__restrict strm, uint32_t x)
{
- strm->outbuf[0] = x;
- strm->outbuf[1] = x >> 8;
- strm->outbuf[2] = x >> 16;
- strm->outbuf[3] = x >> 24;
+ write_le32(strm->outbuf, x);
strm->outbuf += 4;
}
/* Using long because faster on 64-bit (can save one shift) */
-static inline void send_huff(struct slz_stream *strm, unsigned long code)
+static inline void send_huff(struct slz_stream *__restrict strm, unsigned long code)
{
uint32_t bits;
enqueue24(strm, code, bits);
}
-static inline void send_eob(struct slz_stream *strm)
+static inline void send_eob(struct slz_stream *__restrict strm)
{
enqueue8(strm, 0, 7); // direct encoding of 256 = EOB (cf RFC1951)
}
+/* copies <len> litterals from <buf>. at once. <len> must not be null and must
+ * be lower than or equal to 65535, which can be guaranteed when using
+ * SLZ_LITERAL_SKIP with limited literal lengths. <more> indicates that there
+ * are data past buf + <len>. Only supports fixed encoding.
+ */
+__attribute__((unused))
+static void copy_lit_small(struct slz_stream *__restrict strm, const void *buf, uint32_t len, int more)
+{
+ int eob_shift = (strm->state == SLZ_ST_EOB) ? 0 : 7;
+
+ /* explanation: if there is no EOB yet, we have to send one.
+ * It's 0 encoded over 7 bits. Then we have to copy !more into BFINAL
+ * (one bit), and 00 in btype (two bits). Finally we pad to the next
+ * byte. So we add 0 or 7 bits for EOB, followed by 3 then by 0 to 7
+ * bits for alignment. That's 3 to 17 bits.
+ */
+ enqueue24(strm, (!more) << eob_shift, 3 + eob_shift + (7 & -(3 + eob_shift + strm->qbits))); // EOB + BFINAL + BTYPE
+ strm->state = more ? SLZ_ST_EOB : SLZ_ST_DONE;
+
+#if !SLZ_DIRECT_ENQUEUE24
+ /* the queue is byte-aligned now, but the conditional 64-bit enqueue24()
+ * may still hold up to 4 bytes of it that must be written first.
+ */
+ flush_bits(strm);
+#endif
+ copy_16b(strm, len);
+ copy_16b(strm, ~len);
+ memcpy(strm->outbuf, buf, len);
+ strm->outbuf += len;
+}
+
/* copies <len> litterals from <buf>. <more> indicates that there are data past
* buf + <len>. <len> must not be null.
*/
-static void copy_lit(struct slz_stream *strm, const void *buf, uint32_t len, int more)
+__attribute__((unused))
+static void copy_lit(struct slz_stream *__restrict strm, const void *buf, uint32_t len, int more)
{
uint32_t len2;
+ if (len <= 65535)
+ return copy_lit_small(strm, buf, len, more);
+
+ if (strm->state != SLZ_ST_EOB)
+ send_eob(strm);
+
+ len2 = 65535;
+ len -= len2;
+
do {
+ /* wire BFINAL (!more), BTYPE (00) and round to the next byte boundary */
+ enqueue24(strm, !(more || len), 3 + ((5 - strm->qbits) & 7));
+#if !SLZ_DIRECT_ENQUEUE24
+ flush_bits(strm); // see copy_lit_small()
+#endif
+ copy_32b(strm, (~len2 << 16) + len2);
+ memcpy(strm->outbuf, buf, len2);
+ buf += len2;
+ strm->outbuf += len2;
len2 = len;
if (__builtin_expect(len2 > 65535, 0))
len2 = 65535;
len -= len2;
+ } while (__builtin_expect(len2, 0));
- if (strm->state != SLZ_ST_EOB)
- send_eob(strm);
-
- strm->state = (more || len) ? SLZ_ST_EOB : SLZ_ST_DONE;
-
- enqueue8(strm, !(more || len), 3); // BFINAL = !more ; BTYPE = 00
- flush_bits(strm);
- copy_16b(strm, len2); // len2
- copy_16b(strm, ~len2); // nlen2
- memcpy(strm->outbuf, buf, len2);
- buf += len2;
- strm->outbuf += len2;
- } while (len);
+ strm->state = more ? SLZ_ST_EOB : SLZ_ST_DONE;
}
/* copies <len> litterals from <buf>. <more> indicates that there are data past
* buf + <len>. <len> must not be null.
*/
-static void copy_lit_huff(struct slz_stream *strm, const unsigned char *buf, uint32_t len, int more)
+static void copy_lit_huff(struct slz_stream *__restrict strm, const unsigned char *buf, uint32_t len, int more)
{
uint32_t pos;
} while (pos < len);
}
-/* sets <count> BYTES to -32769 in <refs> so that any uninitialized entry will
- * verify (pos-last-1 >= 32768) and be ignored. <count> must be a multiple of
- * 128 bytes and <refs> must be at least one count in length. It's supposed to
- * be applied to 64-bit aligned data exclusively, which makes it slightly
- * faster than the regular memset() since no alignment check is performed.
+/* Marks all <count> bytes worth of entries in <refs> as unusable, so that any
+ * entry not written since is rejected by the (pos - last - 1 >= 32768) test in
+ * slz_rfc1951_encode(). <count> is in bytes and <refs> must be at least that
+ * large.
+ *
+ * The byte value is not arbitrary. It has to leave a <pos> field that reads as
+ * "far in the past" for every possible <pos>, on 32-bit as well, where that
+ * subtraction is done on 32 bits. 0xFF wouldn't work. 0xFE gives 0xFEFEFEFE,
+ * hence pos + 0x01010101, which is above 32768 and does not wrap for any
+ * realistic input length. Note that the use of memset() here is intentional
+ * as it was measured to be faster on modern libcs than an unrolled loop.
*/
static void reset_refs(union ref *refs, long count)
{
- /* avoid a shift/mask by casting to void* */
- union ref *end = (void *)refs + count;
+ memset(refs, 0xFE, count);
+}
- do {
- refs[ 0].by64 = -32769;
- refs[ 1].by64 = -32769;
- refs[ 2].by64 = -32769;
- refs[ 3].by64 = -32769;
- refs[ 4].by64 = -32769;
- refs[ 5].by64 = -32769;
- refs[ 6].by64 = -32769;
- refs[ 7].by64 = -32769;
- refs[ 8].by64 = -32769;
- refs[ 9].by64 = -32769;
- refs[10].by64 = -32769;
- refs[11].by64 = -32769;
- refs[12].by64 = -32769;
- refs[13].by64 = -32769;
- refs[14].by64 = -32769;
- refs[15].by64 = -32769;
- refs += 16;
- } while (refs < end);
+/* Inserts position <p> of input buffer <in> into the reference table <refs>.
+ * Reads the 4 bytes at in[p], so the caller must guarantee that in[p + 3] is
+ * within the input buffer.
+ */
+static inline void insert_ref(union ref *refs, const unsigned char *in, unsigned long p)
+{
+#ifdef UNALIGNED_LE_OK
+ uint32_t w = *(uint32_t *)&in[p];
+#else
+ uint32_t w = ((uint32_t)in[p]) | ((uint32_t)in[p + 1] << 8) |
+ ((uint32_t)in[p + 2] << 16) | ((uint32_t)in[p + 3] << 24);
+#endif
+ uint32_t h = slz_hash(w);
+
+ if (sizeof(long) >= 8)
+ refs[h].by64 = ((uint64_t)p) + ((uint64_t)w << 32);
+ else {
+ refs[h].by32.pos = p;
+ refs[h].by32.word = w;
+ }
}
/* Number of bits wasted by the 9-bit literals above which it becomes
* more than its own share, which the following ones will not emit. The caller
* is responsible for ensuring there is enough room in the output buffer for
* this. The amount of output bytes is returned, and no CRC is computed.
+ *
+ * Note: this function uses a lot of registers due to local variables that
+ * hold pre-computed values that need to be kept very long. Be extremely
+ * careful when changing anything there, and test again on multiple compiler
+ * versions. For example, when building with avx512f (e.g. -march=native on
+ * Skylake-X and Zen-4+), gcc 13 and 14 tend to move some variables to %xmm0
+ * and to read it back using vmovq multiple times, affecting performance, which
+ * can be recovered using -mno-avx512f (~1-2%). On gcc-15, the performance is
+ * better and that option degrades it by 2% again. gcc-16 is even slightly
+ * better (+0.5%). As a general rule, nothing vectorizes well here, and the
+ * unaligned, short memcpy() calls must not be translated to AVX512 code.
*/
-long slz_rfc1951_encode(struct slz_stream *strm, unsigned char *out, const unsigned char *in, long ilen, int more)
+static inline __attribute__((always_inline))
+long _slz_rfc1951_encode(struct slz_stream *__restrict strm, unsigned char *out, const unsigned char *in, long ilen, int more, int level)
{
long rem = ilen;
unsigned long pos = 0;
unsigned long last;
+ const unsigned char *lastp;
uint32_t word = 0;
long mlen;
uint32_t h;
uint64_t ent;
+#if SLZ_LITERAL_SKIP
+ long skip_ratio = 0;
+#endif
uint32_t plit = 0;
uint32_t bit9 = 0;
uint32_t dist, code;
- union ref refs[1 << HASH_BITS];
+ union ref refs[1 << SLZ_HASH_BITS];
- if (!strm->level) {
+ if (__builtin_expect(!strm->level, 0)) {
/* force to send as literals (eg to preserve CPU) */
strm->outbuf = out;
- plit = pos = ilen;
- bit9 = 52; /* force literal dump */
- goto final_lit_dump;
+ if (ilen)
+ copy_lit(strm, in, ilen, more);
+ goto leave;
}
reset_refs(refs, sizeof(refs));
strm->outbuf = out;
#ifndef UNALIGNED_FASTER
+ reload_word:
if (rem >= 4) // <word> is only used inside the loop below, hence the test for >= 4
word = ((uint32_t)(unsigned char)in[pos] << 8) + ((uint32_t)(unsigned char)in[pos + 1] << 16) + ((uint32_t)(unsigned char)in[pos + 2] << 24);
#endif
plit++;
bit9 += ((unsigned char)word >= 144);
pos++;
+
+ /* Incompressible data: once enough literals have accumulated
+ * without a single match, stop looking for a while and just
+ * dump what follows as literal blocks, doubling the amount
+ * skipped each time so that a long incompressible block
+ * costs less and less to walk over. A match resets it.
+ */
+#if SLZ_LITERAL_SKIP
+ if (level <= 1 && !(plit & 63)) {
+ /* be careful: plit+skip must never be >= 65535 */
+ long skip;
+ skip_ratio = (skip_ratio * 2 + 1) & 255; // capped exponential: 0,1,3,7,... and stays at 255
+ skip = rem >= skip_ratio * 32 ? skip_ratio * 32 : rem;
+ copy_lit_small(strm, in + pos - plit, plit + skip, more || skip < rem);
+ pos += skip;
+ rem -= skip;
+ plit = 0;
+# ifndef UNALIGNED_FASTER
+ /* we'll need to update the current word since we skipped some bytes */
+ goto reload_word;
+# else
+ continue;
+# endif
+ }
+ else if (level == 2 && !(plit & 2047)) {
+ /* be careful: plit+skip must never be >= 65535 */
+ long skip = rem >= skip_ratio * 4096 ? skip_ratio * 4096 : rem;
+ skip_ratio = (skip_ratio * 2 + 1) & 7; // capped exponential: 0,1,3,7 and stays at 7
+ copy_lit_small(strm, in + pos - plit, plit + skip, more || skip < rem);
+ pos += skip;
+ rem -= skip;
+ plit = 0;
+# ifndef UNALIGNED_FASTER
+ /* we'll need to update the current word since we skipped some bytes */
+ goto reload_word;
+# else
+ continue;
+# endif
+ }
+ else if (level >= 3) // no literal skip at levels >= 3
+#endif
+ {
+ /* either no literal_skip at build time, or high enough level */
+ if (__builtin_expect(plit == 0xffc0, 0)) {
+ copy_lit_small(strm, in + pos - plit, plit, 1);
+ plit = 0;
+ }
+ }
continue;
}
if ((unsigned long)(pos - last - 1) >= 32768)
goto send_as_lit;
+ /* faster to keep a copy of this with gcc 13-16 */
+ lastp = in + last;
+
/* Note: cannot encode a length larger than 258 bytes */
- mlen = memmatch(in + pos + 4, in + last + 4, (rem > 258 ? 258 : rem) - 4) + 4;
+ mlen = memmatch(in + pos + 4, lastp + 4, (rem > 258 ? 258 : rem) - 4) + 4;
/* found a matching entry */
(dist & 0x1f) + (code >> 16) + SLZ_SWITCH_COST > 8 * mlen)
goto send_as_lit;
+#if defined(SLZ_BACKWARD_MATCH)
+ /* Compare queued literals in case the previous ones were not
+ * found due to hash collisions but still match. We don't go
+ * further than 8 bytes so that we can find them in a single
+ * 64-bit comparison. On aligned-only machines we read one byte
+ * at a time, as it is expected to be cheaper than composing an
+ * unaligned int and manually doing a ctzll() on it. Tests have
+ * shown that there's little to gain past 8 bytes, and anyway
+ * the total match length will be bounded by 258. BTW we only
+ * consider cases where last >= 8 because this guarantees that
+ * we can access both the searched sequence and the match found
+ * on a wole word at once. It may miss some rare matches on the
+ * edge of the buffer that we don't care about.
+ *
+ * It's worth noting that we *do not* fixup <bit9> if a match
+ * is found, as this would be too complicated and counter-
+ * productive. This means that literals that were finally
+ * encoded might still count towards the budget limit, but this
+ * is not dramatic and keeps us on the safe side.
+ *
+ * The result of the search is to only touch <back> that will be
+ * used all along in all operations. Multiple attempts have shown
+ * that it's both easier and faster this way than changing then
+ * rolling back offsets when we finally give up and send_as_lit.
+ */
+ if (__builtin_expect(plit != 0, 1) &&
+ (last >= MAX_C(sizeof(long), SLZ_BACKWARD_MATCH[level]))) {
+ long bmax = MIN(plit, 8); // keep it at 8, always faster
+ long back = 0;
+
+ if (bmax > 258 - mlen)
+ bmax = 258 - mlen;
+
+#if defined(UNALIGNED_LE_OK)
+ if (SLZ_BACKWARD_MATCH[level] == 2) { // 43.32%, best speed vs savings
+ uint16_t x = *(uint16_t *)&in[pos - 2] ^ *(uint16_t *)(lastp - 2);
+ back = x & 0xff00 ? 0 : x ? 1 : 2;
+ if (back > bmax)
+ back = bmax;
+ } else if (SLZ_BACKWARD_MATCH[level] == 3 || SLZ_BACKWARD_MATCH[level] == 4) { // 32 bits: 43.31% or 43.30%
+ uint32_t x = *(uint32_t *)&in[pos - 4] ^ *(uint32_t *)(lastp - 4);
+ back = x & 0xffff0000u ?
+ x & 0xff000000u ? 0 : 1 :
+ x & 0xffffff00u ? 2 :
+ x ? 3 : SLZ_BACKWARD_MATCH[level]; // 3 or 4, test eliminated at 3
+ if (back > bmax)
+ back = bmax;
+ } else if (SLZ_BACKWARD_MATCH[level] == 8) { // 64 bits: 43.30%
+ uint64_t x = *(uint64_t *)&in[pos - 8] ^ *(uint64_t *)(lastp - 8);
+ back = x ? (__builtin_clzll(x) >> 3) : 8;
+ if (back > bmax)
+ back = bmax;
+ } else
+#endif
+ if (SLZ_BACKWARD_MATCH[level] == 1) { // single-byte match: 43.39%
+ if (in[pos - 1] == lastp[-1])
+ back++;
+ if (back > bmax)
+ back = bmax;
+ }
+ else { /* unaligned not supported or match != 1/2/3/4/8 */
+ while (back < bmax && in[pos - back - 1] == lastp[-back - 1])
+ back++;
+ }
+
+ if (back) {
+ /* OK now we're certain to use the match, we must rewind the
+ * literals that are part of the match.
+ */
+ pos -= back;
+ plit -= back;
+ rem += back;
+ mlen += back;
+ /* <code> was computed from the un-extended
+ * length above and must follow, or we would
+ * consume <back> more bytes than the length
+ * code tells the decoder to copy. <dist> does
+ * not move. The cost test above keeps using
+ * the un-extended length, which is safe: the
+ * extension only makes the match cheaper per
+ * byte, never more expensive.
+ */
+ code = len_fh[mlen];
+ }
+ }
+#endif // SLZ_BACKWARD_MATCH // otherwise 43.83%
+
/* first, copy pending literals */
if (plit) {
/* Huffman encoding requires 9 bits for octets 144..255, so this
* only those sent in huffman mode add to the debt.
*/
if (bit9 >= SLZ_SWITCH_COST || strm->debt >= SLZ_MAX_DEBT)
- copy_lit(strm, in + pos - plit, plit, 1);
+ copy_lit_small(strm, in + pos - plit, plit, 1);
else {
copy_lit_huff(strm, in + pos - plit, plit, 1);
strm->debt += bit9;
plit = 0;
}
+#if SLZ_LITERAL_SKIP
+ skip_ratio = 0;
+#endif
/* use mode 01 - fixed huffman */
if (strm->state == SLZ_ST_EOB) {
strm->state = SLZ_ST_FIXED;
}
/* copy the length first */
+#if SLZ_DIRECT_ENQUEUE56
+ /* the length code and the distance code that follows it total
+ * 31 bits at most, which one enqueue can take in a single
+ * shift-or-store instead of two
+ */
+ enqueue56(strm, (uint64_t)(code & 0xFFFF) |
+ ((uint64_t)(dist >> 5) << (code >> 16)),
+ (code >> 16) + (dist & 0x1f));
+#else
enqueue24(strm, code & 0xFFFF, code >> 16);
/* in fixed huffman mode, dist is fixed 5 bits */
enqueue24(strm, dist >> 5, dist & 0x1f);
+#endif
/* <bit9> only measures the current group of literals, which is
* what the decisions above compare against, and it restarts
rem -= mlen;
pos += mlen;
+#if SLZ_INSERT_SKIPPED
+ if (rem < 4)
+ break;
+ /* try to index the matched entry to improve the likelyhood
+ * that a similar sequence will be a hit next time. We only
+ * need 4 remaining bytes starting from the current one.
+ */
+ insert_ref(refs, in, pos - mlen + 3);
+ insert_ref(refs, in, pos - mlen + 2);
+ insert_ref(refs, in, pos - mlen + 1);
+#endif // SLZ_INSERT_SKIPPED
+
#ifndef UNALIGNED_FASTER
/* same as before the loop, this is only used when continuing
* in the loop, so let's use the same condition (rem>=4).
} while (--rem);
}
- final_lit_dump:
/* Now copy remaining literals or mark the end. The cost of switching to
* a stored block depends on the current state and on the presence of
* data after these literals, see the SLZ_*_COST definitions above.
cost = (strm->state == SLZ_ST_EOB) ? SLZ_LAST_COST_EOB : SLZ_LAST_COST;
if (bit9 >= cost || strm->debt >= SLZ_MAX_DEBT)
- copy_lit(strm, in + pos - plit, plit, more);
+ copy_lit_small(strm, in + pos - plit, plit, more);
else
copy_lit_huff(strm, in + pos - plit, plit, more);
plit = 0;
}
-
+ leave:
strm->ilen += ilen;
return strm->outbuf - out;
}
+static __attribute__((noinline))
+long _slz_rfc1951_encode_lev1(struct slz_stream *__restrict strm, unsigned char *out, const unsigned char *in, long ilen, int more)
+{
+ return _slz_rfc1951_encode(strm, out, in, ilen, more, 1);
+}
+
+static __attribute__((noinline))
+long _slz_rfc1951_encode_lev2(struct slz_stream *__restrict strm, unsigned char *out, const unsigned char *in, long ilen, int more)
+{
+ return _slz_rfc1951_encode(strm, out, in, ilen, more, 2);
+}
+
+static __attribute__((noinline))
+long _slz_rfc1951_encode_lev3(struct slz_stream *__restrict strm, unsigned char *out, const unsigned char *in, long ilen, int more)
+{
+ return _slz_rfc1951_encode(strm, out, in, ilen, more, 3);
+}
+
+long slz_rfc1951_encode(struct slz_stream *__restrict strm, unsigned char *out, const unsigned char *in, long ilen, int more)
+{
+ if (strm->level <= 1)
+ return _slz_rfc1951_encode_lev1(strm, out, in, ilen, more);
+ else if (strm->level <= 2)
+ return _slz_rfc1951_encode_lev2(strm, out, in, ilen, more);
+ else
+ return _slz_rfc1951_encode_lev3(strm, out, in, ilen, more);
+}
+
/* Initializes stream <strm> for use with raw deflate (rfc1951). The CRC is
* unused but set to zero. The compression level passed in <level> is set. This
- * value can only be 0 (no compression) or 1 (compression) and other values
- * will lead to unpredictable behaviour. The function always returns 0.
+ * value will be capped between 0 (no compression) and 15 (max compression).
+ * The function always returns 0.
*/
int slz_rfc1951_init(struct slz_stream *strm, int level)
{
strm->state = SLZ_ST_EOB; // no header
- strm->level = level;
+ strm->level = MIN(MAX(level, 0), 15);
strm->format = SLZ_FMT_DEFLATE;
strm->crc32 = 0;
strm->ilen = 0;
}
/* Initializes stream <strm> for use with the gzip format (rfc1952). The
- * compression level passed in <level> is set. This value can only be 0 (no
- * compression) or 1 (compression) and other values will lead to unpredictable
- * behaviour. The function always returns 0.
+ * compression level passed in <level> is set. This value will be capped
+ * between 0 (no compression) and 15 (max compression). The function always
+ * returns 0.
*/
int slz_rfc1952_init(struct slz_stream *strm, int level)
{
strm->state = SLZ_ST_INIT;
- strm->level = level;
+ strm->level = MIN(MAX(level, 0), 15);
strm->format = SLZ_FMT_GZIP;
strm->crc32 = 0;
strm->ilen = 0;
}
/* Initializes stream <strm> for use with the zlib format (rfc1950). The
- * compression level passed in <level> is set. This value can only be 0 (no
- * compression) or 1 (compression) and other values will lead to unpredictable
- * behaviour. The function always returns 0.
+ * compression level passed in <level> is set. This value will be capped
+ * between 0 (no compression) and 15 (max compression). The function always
+ * returns 0.
*/
int slz_rfc1950_init(struct slz_stream *strm, int level)
{
strm->state = SLZ_ST_INIT;
- strm->level = level;
+ strm->level = MIN(MAX(level, 0), 15);
strm->format = SLZ_FMT_ZLIB;
strm->crc32 = 1; // rfc1950/zlib starts with initial crc=1
strm->ilen = 0;
return strm->outbuf - buf;
}
+#ifndef PRECOMPUTE_TABLES
/* Returns code for lengths 1 to 32768. The bit size for the next value can be
* found this way :
*
return code;
}
+#endif
/* not thread-safe, must be called exactly once */
static inline void __slz_prepare_dist_table()
--- /dev/null
+/* Only needed for slz*.c */
+#include <import/slz.h>
* OTHER DEALINGS IN THE SOFTWARE.
*/
-#include "import/slz.h"
-#include "import/slz-prv.h"
-#include "import/slz-tables.h"
+#include "slz.h"
+#include "slz-prv.h"
+#include "tables.h"
+
+/* We'll tentatively use PCLMULQDQ for CRC calculation on x86_64 CPUs even if
+ * not enabled at build time, unless SLZ_NO_PCLMUL is set to disable it. This
+ * is done by detecting its support at boot time on supported compilers.
+ */
+#if !defined(SLZ_NO_PCLMUL) && (defined(__x86_64__)) && \
+ (defined(__clang__) || (defined(__GNUC__) && __GNUC__ >= 6))
+#define SLZ_USE_PCLMUL
+#include <immintrin.h>
+static int slz_have_pclmul = 0;
+#endif
/*
* =============================================================================
}
/* This version computes the crc32 of <buf> over <len> bytes, doing most of it
- * in 32-bit chunks.
+ * in 32-bit chunks. It is the portable implementation, and it also finishes
+ * what the carry-less one below leaves behind.
*/
-uint32_t slz_crc32_by4(uint32_t crc, const unsigned char *buf, int len)
+static uint32_t crc32_by4_tab(uint32_t crc, const unsigned char *buf, int len)
{
const unsigned char *end = buf + len;
return crc;
}
+#if defined(SLZ_USE_PCLMUL)
+
+/* Carry-less folding implementation of the very same CRC-32, using PCLMULQDQ.
+ *
+ * The state is a 128-bit polynomial in the same reflected representation as the
+ * one the table code works in, which makes the final reduction trivial: it is
+ * just the table code run over the accumulator's 16 bytes. Folding a state
+ * forward by N bytes means multiplying it by x^(8N) modulo the CRC polynomial,
+ * which splits over the two halves of the register into two carry-less
+ * multiplications by a constant:
+ *
+ * fold_N(Q) = clmul(Q.lo, x^(8N+95) mod P) ^ clmul(Q.hi, x^(8N+31) mod P)
+ *
+ * The constants were derived from the CRC recurrence rather than copied from
+ * elsewhere, and the whole scheme was verified against the table code for every
+ * length and starting value before being written here; see results.md.
+ *
+ * Four accumulators are folded by 64 bytes each so that the multiplier latency,
+ * which is much higher than its throughput, is not what limits the loop. They
+ * are combined at the end by folding each into the next by 16 bytes.
+ */
+__attribute__((target("pclmul,sse4.1"))) /* pretend -mpclmul -msse4.1 was passed */
+static uint32_t crc32_pclmul(uint32_t crc, const unsigned char *buf, int len)
+{
+ /* x^159 and x^95 : fold by 16 bytes */
+ const __m128i k16 = _mm_setr_epi32(0xae689191, 0, 0xccaa009e, 0);
+ /* x^543 and x^479 : fold by 64 bytes */
+ const __m128i k64 = _mm_setr_epi32(0x8f352d95, 0, 0x1d9513d7, 0);
+ unsigned char tmp[16] __attribute__((aligned(16)));
+ __m128i q, q0, q1, q2, q3;
+ int blk = len & ~15;
+
+ if (blk >= 64) {
+ q0 = _mm_xor_si128(_mm_loadu_si128((const __m128i *)(buf + 0)),
+ _mm_cvtsi32_si128(~crc));
+ q1 = _mm_loadu_si128((const __m128i *)(buf + 16));
+ q2 = _mm_loadu_si128((const __m128i *)(buf + 32));
+ q3 = _mm_loadu_si128((const __m128i *)(buf + 48));
+ buf += 64; blk -= 64; len -= 64;
+
+ while (blk >= 64) {
+ q0 = _mm_xor_si128(_mm_xor_si128(_mm_clmulepi64_si128(q0, k64, 0x00),
+ _mm_clmulepi64_si128(q0, k64, 0x11)),
+ _mm_loadu_si128((const __m128i *)(buf + 0)));
+ q1 = _mm_xor_si128(_mm_xor_si128(_mm_clmulepi64_si128(q1, k64, 0x00),
+ _mm_clmulepi64_si128(q1, k64, 0x11)),
+ _mm_loadu_si128((const __m128i *)(buf + 16)));
+ q2 = _mm_xor_si128(_mm_xor_si128(_mm_clmulepi64_si128(q2, k64, 0x00),
+ _mm_clmulepi64_si128(q2, k64, 0x11)),
+ _mm_loadu_si128((const __m128i *)(buf + 32)));
+ q3 = _mm_xor_si128(_mm_xor_si128(_mm_clmulepi64_si128(q3, k64, 0x00),
+ _mm_clmulepi64_si128(q3, k64, 0x11)),
+ _mm_loadu_si128((const __m128i *)(buf + 48)));
+ buf += 64; blk -= 64; len -= 64;
+ }
+
+ /* combine the four accumulators, oldest first */
+ q = q0;
+ q = _mm_xor_si128(_mm_xor_si128(_mm_clmulepi64_si128(q, k16, 0x00),
+ _mm_clmulepi64_si128(q, k16, 0x11)), q1);
+ q = _mm_xor_si128(_mm_xor_si128(_mm_clmulepi64_si128(q, k16, 0x00),
+ _mm_clmulepi64_si128(q, k16, 0x11)), q2);
+ q = _mm_xor_si128(_mm_xor_si128(_mm_clmulepi64_si128(q, k16, 0x00),
+ _mm_clmulepi64_si128(q, k16, 0x11)), q3);
+ }
+ else if (blk >= 16) {
+ q = _mm_xor_si128(_mm_loadu_si128((const __m128i *)buf),
+ _mm_cvtsi32_si128(~crc));
+ buf += 16; blk -= 16; len -= 16;
+ }
+ else
+ return crc32_by4_tab(crc, buf, len);
+
+ while (blk >= 16) {
+ q = _mm_xor_si128(_mm_xor_si128(_mm_clmulepi64_si128(q, k16, 0x00),
+ _mm_clmulepi64_si128(q, k16, 0x11)),
+ _mm_loadu_si128((const __m128i *)buf));
+ buf += 16; blk -= 16; len -= 16;
+ }
+
+ /* The accumulator is a polynomial in exactly the representation the table
+ * code consumes, so reducing it is just running that code over its 16
+ * bytes from an all-ones state. Then the 0 to 15 trailing bytes.
+ */
+ _mm_store_si128((__m128i *)tmp, q);
+ crc = crc32_by4_tab(~0U, tmp, 16);
+ return crc32_by4_tab(crc, buf, len);
+}
+
+#endif /* SLZ_USE_PCLMUL */
+
+/* Computes the crc32 of <buf> over <len> bytes, using the carry-less
+ * implementation when the CPU supports it.
+ */
+uint32_t slz_crc32_by4(uint32_t crc, const unsigned char *buf, int len)
+{
+#if defined(SLZ_USE_PCLMUL)
+ if (__builtin_expect(slz_have_pclmul && len >= 16, 1))
+ return crc32_pclmul(crc, buf, len);
+#endif
+ return crc32_by4_tab(crc, buf, len);
+}
+
/* Original version from RFC1950, verified and works OK */
uint32_t slz_adler32_by1(uint32_t crc, const unsigned char *buf, int len)
{
static void __slz_common_initialize(void)
{
__slz_make_crc_table(); // used by both slz and uslz
+
+#if defined(SLZ_USE_PCLMUL)
+ slz_have_pclmul = __builtin_cpu_supports("pclmul") &&
+ __builtin_cpu_supports("sse4.1");
+#endif
}
--- /dev/null
+/* Only needed for slz*.c */
+#include <import/slz-tables.h>
#include <stdio.h>
#include <string.h>
-#include "import/slz.h"
-#include "import/slz-prv.h"
-#include "import/slz-tables.h"
+#include "slz.h"
+#include "slz-prv.h"
+#include "tables.h"
/* if PRECOMPUTE_TABLES not set, then tables.h does not provide
* fixed_huff_dec_table table as we need to recompute it.
return 1;
}
+/*
+ * gen_fast_table: fill the direct lookup <table> of <tbits> bits from the code
+ * lengths in <lengths>, for an alphabet of <symbols> symbols. Entries have the
+ * same layout as fixed_huff_dec_table[]: the symbol in the upper bits and the
+ * code length in the low 4 bits. Codes longer than <tbits> are left at zero
+ * and are decoded by walking the tree built by gen_huffman_table() instead.
+ *
+ * Each code is replicated over every combination of the bits above it, so
+ * that a lookup performed with fewer than <tbits> bits still finds it,
+ * provided the length found is not larger than the number of bits available.
+ * That is what lets the decoder work at the very end of a stream, where fewer
+ * bits than the table width may remain.
+ *
+ * Preconditions: <lengths> holds <symbols> entries, all of them <= 15, and
+ * <table> is (1 << tbits) entries long.
+ */
+static void gen_fast_table(unsigned int symbols, const unsigned char *lengths,
+ uint16_t *table, unsigned int tbits)
+{
+ unsigned short next_code[16];
+ unsigned short count[16];
+ unsigned int i, len, code;
+
+ memset(table, 0, (1U << tbits) * sizeof(*table));
+ memset(count, 0, sizeof(count));
+
+ for (i = 0; i < symbols; i++)
+ count[lengths[i]]++;
+
+ /* first code of each length, as specified in rfc1951 3.2.2 */
+ count[0] = 0;
+ code = 0;
+ for (len = 1; len < 16; len++) {
+ code = (code + count[len - 1]) << 1;
+ next_code[len] = code;
+ }
+
+ for (i = 0; i < symbols; i++) {
+ unsigned int rev, step;
+
+ len = lengths[i];
+ if (!len)
+ continue;
+
+ /* codes are assigned sequentially within a length, and the
+ * ones too long for the table still consume code space.
+ */
+ code = next_code[len]++;
+ if (len > tbits)
+ continue;
+
+ /* codes appear bit-reversed in the accumulator */
+ rev = rbit9(code, len);
+ step = 1U << len;
+ for (; rev < (1U << tbits); rev += step)
+ table[rev] = (i << 7) | len;
+ }
+}
+
/* updates crc for the current <state> inflate stream, chooses the proper
* crc computation function according to the format used in the stream.
*/
*
* Returns 1 on success and 0 if more data is needed.
*/
-static inline int bit_accumulate(const unsigned char **in_ptr, const unsigned char *in_top, unsigned char *num_bits, uint64_t *bit_accum)
+static inline int bit_accumulate(const unsigned char **in_ptr, const unsigned char *in_top,
+ unsigned char *num_bits, unsigned long *bit_accum)
{
if (*in_ptr >= in_top)
return 0;
- *bit_accum |= (uint64_t)(*in_ptr)[0] << *num_bits;
+ *bit_accum |= (unsigned long)(*in_ptr)[0] << *num_bits;
*num_bits += 8;
(*in_ptr)++;
return 1;
*/
static inline int gethuff(unsigned int *huff_index,
const unsigned char **in_ptr, const unsigned char *in_top,
- unsigned char *num_bits, uint64_t *bit_accum,
+ unsigned char *num_bits, unsigned long *bit_accum,
unsigned int *var, short *table)
{
unsigned int index;
unsigned int bits;
unsigned long bit_follow;
+ int ret = 0;
index = *huff_index;
bits = *num_bits;
for (;;) {
if (bits == 0) {
- if (*in_ptr >= in_top) {
- *num_bits = bits;
- *bit_accum = bit_follow;
- *huff_index = index;
- return 0;
- }
- bit_follow |= (unsigned long) (*in_ptr)[0] << bits;
- bits += 8;
+ if (*in_ptr >= in_top)
+ break;
+ bit_follow |= (unsigned long) (*in_ptr)[0];
+ bits = 8;
(*in_ptr)++;
}
bit_follow >>= 1;
bits--;
if (table[index] >= 0) {
+ *var = table[index];
+ index = 0;
+ ret = 1;
break;
}
index = ~(table[index]);
*num_bits = bits;
*bit_accum = bit_follow;
- *var = table[index];
- *huff_index = 0;
+ *huff_index = index;
- return 1;
+ return ret;
}
/* when decoding static/fixed huffman (we must be sure of that before
* needs more input data.
*/
static inline int gethuff_fixed(const unsigned char **in_ptr, const unsigned char *in_top,
- unsigned char *num_bits, uint64_t *bit_accum,
+ unsigned char *num_bits, unsigned long *bit_accum,
unsigned int *var)
{
unsigned long bit_follow;
* length to prevent mismatch.
*/
idx = (bit_follow & ((1 << bits) - 1));
- if ((fixed_huff_dec_table[idx] & 0xF) > bits) {
+ if ((fixed_huff_dec_table[idx] & 0x3F) > bits) {
/* no match found with <bits>, so there must be
* data available for reading (errors appart)
* Since we already have 7 bits and a refill is
}
break;
}
- *num_bits = bits - (fixed_huff_dec_table[idx] & 0xF);
- *bit_accum = bit_follow >> (fixed_huff_dec_table[idx] & 0xF);
+ *num_bits = bits - (fixed_huff_dec_table[idx] & 0x3F);
+ *bit_accum = bit_follow >> (fixed_huff_dec_table[idx] & 0x3F);
*var = fixed_huff_dec_table[idx] >> 7;
return 1;
}
-/* reverses bits in 5bits len <v> input and returns the new value.
- * ie: 01100 becomes 00110. Bits 5-7 are ignored.
+/* Decodes one symbol using the direct lookup <table> of <tbits> bits, which
+ * must have been filled by gen_fast_table().
+ *
+ * Returns 1 when a symbol was decoded and stored in <var>, 0 if more input
+ * data is needed, and -1 when the code is longer than the table, in which
+ * case no bit was consumed from the accumulator and the caller has to fall
+ * back to gethuff().
*/
-static inline uint8_t rbit5(uint8_t v)
+static inline int gethuff_fast(const unsigned char **in_ptr, const unsigned char *in_top,
+ unsigned char *num_bits, unsigned long *bit_accum,
+ unsigned int *var, const uint16_t *table,
+ unsigned int tbits)
{
- v = ((0xf7b3d591e6a2c480ULL >> ((v & 0x1e) * 2)) & 0xf) + ((v & 1) << 4);
- return v;
-}
+ unsigned long accum = *bit_accum;
+ unsigned int bits = *num_bits;
+ unsigned int entry, len;
+
+ while (1) {
+ if (bits >= tbits) {
+ entry = table[accum & ((1U << tbits) - 1)];
+ break;
+ }
+
+ /* Not enough bits for a full lookup. The table being padded,
+ * looking it up with what we have is still valid as long as it
+ * lands on a code no longer than that; otherwise we need more
+ * data to tell.
+ */
+ entry = table[accum & ((1U << bits) - 1)];
+ len = entry & 0xF;
+ if (len && len <= bits)
+ break;
+
+ if (*in_ptr >= in_top) {
+ *num_bits = bits;
+ *bit_accum = accum;
+ return 0;
+ }
+ accum |= (unsigned long)(*in_ptr)[0] << bits;
+ bits += 8;
+ (*in_ptr)++;
+ }
+
+ len = entry & 0xF;
+ if (!len) {
+ /* too long for the table: hand the refilled accumulator back
+ * untouched so that the tree walk starts on the first bit of
+ * the code.
+ */
+ *num_bits = bits;
+ *bit_accum = accum;
+ return -1;
+ }
+ *num_bits = bits - len;
+ *bit_accum = accum >> len;
+ *var = entry >> 7;
+ return 1;
+}
/* The set of macros below is relevant for uslz_decode_block() function */
+/* Fast path for the bit reader. Without it the accumulator is refilled one
+ * byte at a time, each byte costing a test against in_top and a test for
+ * "do I have enough bits yet", both of which depend on the data and are
+ * mispredicted a lot: on a silesia compressed by slz they account for about
+ * half of the branch misses of the whole decoder.
+ *
+ * REFILL() instead brings a whole machine word in with a single unaligned
+ * load and no branch beyond the one checking that the word is readable. It
+ * leaves the accumulator with everything but its low byte valid, that is 56
+ * to 63 bits on a 64-bit platform, which covers the longest symbol any block
+ * type can produce (15 bits of code, 5 of length extra, 15 of distance code
+ * and 13 of distance extra, so 48): one call per symbol is then enough and
+ * every "do I have enough bits" test inside the decoders becomes a test that
+ * is always false, and therefore free. A 32-bit platform only gets 24 to 31
+ * bits, enough for a literal or a length but not always for a whole
+ * length/distance pair, and the readers keep their byte at a time refill for
+ * what a refill did not cover, so they remain correct, they just take that
+ * slower path more often.
+ *
+ * The number of whole bytes the load brought in is written
+ * "num_bits ^ (8 * __SIZEOF_LONG__ - 1)" rather than the subtraction it is
+ * equal to as long as <num_bits> stays below the width of the accumulator,
+ * which it always does here. The compiler cannot know that bound, so the
+ * subtraction makes it emit the constant, a subtraction and a sign extension
+ * of a result it cannot prove positive, where the xor becomes a shift and an
+ * xor on the low byte.
+ *
+ * Bits above <num_bits> are not zero any more after this: the load brings in
+ * up to 7 bytes, 3 on a 32-bit platform, that are not accounted for, and
+ * <in_ptr> is left pointing at them so that the next refill reads them again
+ * and ORs the same values at the same places. That is harmless because every
+ * reader masks what it takes.
+ *
+ * Nothing is ever pushed back to the input: a byte that made it into the
+ * accumulator counts as consumed, exactly as it did when they were pulled in
+ * one at a time, so a caller that is told it has fed us N bytes never has to
+ * present them again. The two places that want bytes rather than bits, a
+ * stored block and the trailer, get back on a byte boundary with ALIGN_BYTE()
+ * and then take those bytes from the accumulator before touching <in_ptr>.
+ *
+ * The only condition is UNALIGNED_LE_OK. A refill of 24 bits is worth having
+ * as much as one of 56: built with -m32 on an x86, it decodes a fixed huffman
+ * stream 26% faster and a dynamic one 8% faster than the byte at a time loop
+ * it replaces, two data dependent branches per byte being what those bits are
+ * traded for.
+ */
+#if defined(UNALIGNED_LE_OK)
+#define REFILL() \
+ do { \
+ if (__builtin_expect(in_top - in_ptr >= __SIZEOF_LONG__, 1)) { \
+ bit_accum |= read_lelong(in_ptr) << num_bits; \
+ in_ptr += (num_bits ^ (8 * __SIZEOF_LONG__ - 1)) >> 3; \
+ num_bits |= 8 * __SIZEOF_LONG__ - 8; \
+ } \
+ } while (0)
+
+/* handle up to one literal at output position <ofs>. This may only be used
+ * inside uslz_decode_block() immediately after REFILL() in the loop. The "if"
+ * block is used to automatically adapt to the size of the long and handle
+ * either 2 or 6 symbols at once. Note that num_bits may contain bit 7 set and
+ * will need to be masked before being consumed at the end of the calls.
+ */
+#define FIXED_LITERAL(ofs) \
+ do { \
+ if (ofs < __SIZEOF_LONG__ - 2) { \
+ unsigned int entry; \
+ entry = fixed_huff_dec_table[bit_accum & 0x1FF]; \
+ bit_accum >>= entry & 0x3F; \
+ num_bits -= entry; \
+ state->symbol = entry >> 7; \
+ if (state->symbol >= 256) { \
+ num_bits &= 0x3F; \
+ goto have_symbol; \
+ } \
+ *out++ = state->symbol; \
+ } \
+ } while (0)
+
+/* Drops the bits left in the byte being read so that the accumulator holds
+ * whole bytes only, and clears whatever REFILL() left above them. <num_bits>
+ * must be below the width of the accumulator here, which REFILL() guarantees.
+ */
+#define ALIGN_BYTE() \
+ do { \
+ bit_accum >>= num_bits & 7; \
+ num_bits &= ~7; \
+ bit_accum &= (1UL << num_bits) - 1; \
+ } while (0)
+#else
+#define REFILL() do { } while (0)
+#define ALIGN_BYTE() \
+ do { \
+ num_bits = 0; \
+ bit_accum = 0; \
+ } while (0)
+#endif
+
/* The GETBITS macro retrieves the specified number of bits (n) from
* the block, returning from the function if no more data is available,
* and stores the value in the given variable (var). The number of
- * bits to retrieve (n) must be no greater than 32.
+ * bits to retrieve (n) must be no greater than 24, so that the byte at a time
+ * refill cannot overflow even a 32-bit accumulator; the widest used below is
+ * 16.
*/
#define GETBITS(n,var) \
do { \
if (!bit_accumulate(&in_ptr, in_top, &num_bits, &bit_accum))\
goto out_of_data; \
} \
- var = bit_accum & ((1ULL << n) - 1); \
+ var = bit_accum & ((1UL << n) - 1); \
bit_accum >>= n; \
num_bits -= n; \
} while (0)
* the current checksum batch.
*/
#define SET_OUT_LIM() do { \
- unsigned long __room = out_max - 1 - dec_bsize; \
+ unsigned long __room = out_max - dec_bsize; \
\
out_lim = out_end; \
if ((unsigned long)(out_end - crc_ptr) > CRC_BLOCK) \
#define CASE_STATE(s) case USLZ_ST_##s: goto state_##s
+/* Called when uslz_decode_block() reported a full output. If it was waiting
+ * for a symbol and that symbol is an end of block, the EOB is consumed, the
+ * state is set so that uslz_decode_block() finishes the block when called
+ * again, and non-zero is returned. Otherwise nothing
+ * changes and zero is returned. The EOB emits nothing and must still be
+ * consumed, as well as the trailer that may follow, or a caller whose buffer
+ * is exactly the size of the output would never see the end of the stream.
+ * It is deliberately not done in uslz_decode_block(), where even an out of
+ * line call in the checkpoint costs 2 to 5% on this hot loop by disturbing
+ * its register allocation, and it is kept out of line and holds all the
+ * tests so as to add as little code as possible to uslz_decode().
+ */
+static __attribute__((noinline))
+int uslz_eob_follows(struct uslz_stream *state)
+{
+ const unsigned char *in_ptr = state->in_ptr;
+ unsigned long bit_accum = state->bit_accum;
+ unsigned char num_bits = state->num_bits;
+ unsigned int hi = state->huff_index;
+ unsigned int sym;
+ int ret;
+
+ if (state->dec_bsize < state->out_max || state->state != USLZ_ST_READ_SYMBOL)
+ return 0;
+
+ if (state->block_type != 2)
+ ret = gethuff_fixed(&in_ptr, state->in_top, &num_bits, &bit_accum, &sym);
+ else
+ ret = gethuff(&hi, &in_ptr, state->in_top, &num_bits, &bit_accum, &sym, state->literal_table);
+
+ if (!ret || sym != 256)
+ return 0;
+
+ state->in_ptr = in_ptr;
+ state->bit_accum = bit_accum;
+ state->num_bits = num_bits;
+ state->huff_index = 0;
+ state->symbol = 256;
+
+ /* uslz_decode_block() must now finish the block. Rather than adding a
+ * state for this, which costs ~1% by moving the hot loop around, we
+ * resume as a stored block with nothing left to copy, which goes
+ * straight to the end of the block.
+ */
+ state->nread = state->len;
+ state->state = USLZ_ST_UNCOMPRESSED_DATA;
+ return 1;
+}
+
/* main decoding function for inflate API, takes <state> stream as parameter
* and decode as much data as possible from input data (as long as output
* buffer allows it), it automatically detects uncompressed, fixed or dynamic
unsigned char *out = out_base + dec_total % out_max;
unsigned char *crc_ptr = out;
unsigned char *out_lim;
- uint64_t bit_accum = state->bit_accum;
+ unsigned long bit_accum = state->bit_accum;
unsigned char num_bits = state->num_bits;
/* whether the ring already wrapped once, i.e. whether the whole
* buffer is valid history for the distances.
* buffer.
*/
if (state->block_type == 0) {
- num_bits = 0; /* Skip remaining bits in the previous byte. */
+ /* Skip remaining bits in the previous byte. */
+ ALIGN_BYTE();
state->state = USLZ_ST_UNCOMPRESSED_LEN;
state_UNCOMPRESSED_LEN:
goto checkpoint;
}
+ /* bytes already pulled into the accumulator by
+ * REFILL() come first, <in_ptr> has moved past them.
+ */
+ if (num_bits >= 8) {
+ *out++ = bit_accum;
+ bit_accum >>= 8;
+ num_bits -= 8;
+ state->nread++;
+ remain--;
+ continue;
+ }
+
if (in_ptr >= in_top)
goto out_of_data;
err_code = USLZ_DECODE_E_GEN_HUFF;
goto error_return;
}
+
+ /* The tree above can decode any code but costs one dependent
+ * load per bit. Add a direct lookup table for the literal and
+ * length codes, which is where most of the symbols are read.
+ * Its cost is paid once per block and amortised over the many
+ * thousands of symbols a block holds. A zero width disables
+ * the table, and the whole thing then folds away.
+ */
+ if (USLZ_FAST_BITS)
+ gen_fast_table(state->literal_count, state->literal_len,
+ state->fast_lit, USLZ_FAST_BITS);
+
+ if (USLZ_FAST_DBITS)
+ gen_fast_table(state->distance_count, state->distance_len,
+ state->fast_dist, USLZ_FAST_DBITS);
}
/* else Static tables: we don't need to generate literal / distance table
* because we directly use gethuff_fixed().
goto checkpoint;
}
+ /* one refill per symbol is enough for the longest one */
+ REFILL();
+
/* Read a compressed symbol from the block. */
if (state->block_type != 2) {
+#if defined(UNALIGNED_LE_OK)
+ /* Literal fast path for fixed huffman blocks, which is
+ * what slz emits and where three symbols out of four
+ * are literals. As long as one refill's worth of input
+ * and as many output bytes as a refill covers are
+ * available, a literal is one table lookup and one
+ * store, with none of the per-symbol checks of the
+ * generic path: a fixed code is at most 9 bits, so a
+ * refill covers six of them on a 64-bit platform and
+ * two on a 32-bit one, which is what "__SIZEOF_LONG__
+ * - 2" counts below. They are decoded with no test on
+ * the input, and the output room is checked once for
+ * all of them. The first symbol that is not a literal
+ * is handed to the generic path below, which still
+ * does all the checking on lengths and distances.
+ */
+ while (out_lim - out >= __SIZEOF_LONG__ - 2 &&
+ in_top - in_ptr >= __SIZEOF_LONG__) {
+ REFILL();
+
+ /* read up to (sizeof(long) - 2) symbols;
+ * excess calls are ignored on 32-bit machines.
+ */
+ FIXED_LITERAL(0); FIXED_LITERAL(1);
+ FIXED_LITERAL(2); FIXED_LITERAL(3);
+ FIXED_LITERAL(4); FIXED_LITERAL(5);
+ num_bits &= 0x3F;
+ }
+ /* the loop may have filled the output exactly to the
+ * limit, and the generic path only checks that before
+ * decoding, at the top of the loop: go back there.
+ */
+ if (out == out_lim)
+ continue;
+#endif /* UNALIGNED_LE_OK */
/* fixed huffman */
if (!gethuff_fixed(&in_ptr, in_top, &num_bits, &bit_accum, &state->symbol))
goto out_of_data;
}
- else
- GETHUFF(state->symbol, state->literal_table);
+ else {
+ /* <huff_index> being non-zero means a tree walk was
+ * interrupted by a lack of data and has to be resumed;
+ * the fast path cannot be used then, as some bits of
+ * the code have already been consumed.
+ */
+ int ret = -1;
+ if (USLZ_FAST_BITS && !state->huff_index)
+ ret = gethuff_fast(&in_ptr, in_top, &num_bits, &bit_accum,
+ &state->symbol, state->fast_lit,
+ USLZ_FAST_BITS);
+ if (!ret)
+ goto out_of_data;
+ if (ret < 0)
+ GETHUFF(state->symbol, state->literal_table);
+ }
+
+#if defined(UNALIGNED_LE_OK)
+ have_symbol:
+#endif /* UNALIGNED_LE_OK */
/* If the symbol is a literal, add it to the buffer and continue
* with the next code.
*/
bit_accum >>= 5;
num_bits -= 5;
}
- else
- GETHUFF(state->symbol, state->distance_table);
+ else {
+ /* same fast path as for the literal codes above */
+ int ret = -1;
+
+ if (USLZ_FAST_DBITS && !state->huff_index)
+ ret = gethuff_fast(&in_ptr, in_top, &num_bits, &bit_accum,
+ &state->symbol, state->fast_dist,
+ USLZ_FAST_DBITS);
+ if (!ret)
+ goto out_of_data;
+ if (ret < 0)
+ GETHUFF(state->symbol, state->distance_table);
+ }
if (state->symbol <= 3)
state->distance = state->symbol + 1;
* ring, which is only valid history once the
* ring has wrapped at least once. This test
* replaces the former distance_avail counter:
- * a distance never exceeds 32K and the ring is
- * at least that large, so "have we produced at
- * least <distance> bytes" is exactly "has the
- * ring wrapped, or is the distance within the
- * current position".
+ * "have we produced at least <distance> bytes"
+ * is exactly "has the ring wrapped, or is the
+ * distance within the current position", as
+ * long as the distance doesn't exceed the ring.
+ * It can only do so with a ring smaller than
+ * 32K, where such a match would start before
+ * <out_base>. Testing it here only costs in
+ * this rarely taken branch. Note the likely()
+ * test below that proves to improve the perf
+ * while an unlikely degrades them. This is
+ * likely due to the short exit path compared
+ * to the heavier memmove() that's better kept
+ * away from the rest of the hot path.
*/
- if (!wrapped) {
+ if (__builtin_expect(!wrapped || state->distance > out_max, 1)) {
err_code = USLZ_DECODE_E_DISTANCE;
goto error_return;
}
do {
*dst++ = *src++;
+ __asm__("" : "+r"(dst)); // keep gcc's vectorizer off to save 10% perf
} while (--cnt);
}
}
/* no more room in the decoded block, ask for a drain. The state to
* resume from was set by whoever jumped here.
*/
- if (dec_bsize + 1 >= out_max)
+ if (dec_bsize >= out_max)
goto out_of_space;
SET_OUT_LIM();
* accumulator for that purpose since it is not used for
* decoding anymore.
*/
- bit_accum = 0;
- num_bits = 0;
+ ALIGN_BYTE();
/* the trailer may well not be available yet, in which case we
* have to be able to come back here on the next call.
*/
while (num_bits < 32) {
if (in_ptr >= in_top)
goto out_of_data;
- /* the cast matters: in_ptr[0] is promoted to a
- * signed int, so for the first byte, whose
- * shift is 24, any value >= 0x80 would become
- * negative and be sign-extended over the upper
- * half of the 64-bit accumulator.
+ /* the four bytes are stored big endian while
+ * the accumulator holds them in the order they
+ * arrived, hence the swap below.
*/
- bit_accum |= (uint32_t)in_ptr[0] << (24 - num_bits);
+ bit_accum |= (unsigned long)in_ptr[0] << num_bits;
in_ptr++;
num_bits += 8;
}
- if (state->crc != bit_accum) {
+ if (state->crc != __builtin_bswap32((uint32_t)bit_accum)) {
err_code = USLZ_DECODE_E_BAD_CRC;
goto error_return;
}
+ bit_accum = 0;
+ num_bits = 0;
}
else if ((state->flags & USLZ_FL_GZIP)) {
/* The gzip trailer is 8 bytes: crc32 then isize, both
while (num_bits < 64) {
if (in_ptr >= in_top)
goto out_of_data;
- bit_accum |= (uint64_t)in_ptr[0] << num_bits;
+ if (__SIZEOF_LONG__ > 4 ||
+ num_bits < 8 * __SIZEOF_LONG__)
+ bit_accum |= (unsigned long)in_ptr[0] << num_bits;
in_ptr++;
num_bits += 8;
}
*/
return 0;
}
- state->bit_accum |= (uint64_t)state->in_ptr[0] << state->num_bits;
+ state->bit_accum |= (unsigned long)state->in_ptr[0] << state->num_bits;
state->in_ptr += 1;
state->num_bits += 8;
}
unsigned char **decoded_data, long *decoded_size,
long *consumed_bytes, uint32_t *crc_ret)
{
- enum uslz_decode_ret res;
+ enum uslz_decode_ret res = USLZ_DECODE_SUCCESS;
/* ensure input consistency */
if (compressed_data == NULL || compressed_size < 0 ||
case USLZ_DECODE_SUCCESS:
break;
case USLZ_DECODE_OUT_OF_SPACE:
+ /* a full output may only be waiting for an EOB */
+ if (__builtin_expect(uslz_eob_follows(state), 0))
+ continue;
+ goto drain;
case USLZ_DECODE_OUT_OF_DATA:
goto drain;
default:
* It will be used by slz_inflate as a rotating buffer to decode the stream
* and allow the caller to read decoded data. As such it is exclusively managed
* by slz_inflate and must not be modified by the caller, only read from during
- * the whole decoding process. Minimum size should be 32K, because zlib algo
- * specifies that up to 32K bytes distance can be used, thus the inflate API
- * should always be able to go 32K bytes in the past at any time with such a
- * buffer.
+ * the whole decoding process. The size should be at least 32K, because deflate
+ * permits references up to 32K bytes in the past, which a smaller buffer
+ * cannot hold once it has wrapped: such references are then rejected with
+ * USLZ_DECODE_E_DISTANCE, even in a valid stream. A smaller buffer is thus
+ * only suited to a stream whose whole output fits in it.
*
* Returns 1 on success and 0 on failure.
*/
{
struct uslz_stream zero = { 0 };
- if (state == NULL || output_size < 32768 || output_buffer == NULL)
+ if (state == NULL || output_size < 1 || output_buffer == NULL)
return 0;
/* zero out <state> */
*
* symbols and len are stored on 16bits unsigned integers. We use exactly 512 indexes
* (from 0 to 511 which is 9 bits maximal value) as static encoding is performed using
- * at most 9 bits:
+ * at most 9 bits. Len is stored on 6 bits so that the compiler knows it can drop the
+ * mask when using them for a shift on archs that mask upper bits in shift counts.
*
* Each value in the array is stored this way:
*
* 9 bits for the symbol (A)
- * 3 unused bits
- * 4 bits for the len (B)
+ * 1 unused bit
+ * 6 bits for the len (B), only 4 of which are set
*
* ie:
- * AAAAAAAA AXXXBBBB
+ * AAAAAAAA AXBBBBBB
*/
// we revese the codes in order to directly compare them as they appear
- // in the bit accumulator in huffman decoding func using rev_short(), the
+ // in the bit accumulator in huffman decoding func using rbit9(), the
// RFC doesn't specify bit order is inverted but this is the only way this
// works in practise.
while (it <= 143) {
// 00110000X through 10111111X
- fixed_huff_dec_table[rev_short((0x30 + it), 8) | 0x100] = it << 7; // symbol
- fixed_huff_dec_table[rev_short((0x30 + it), 8) | 0x100] |= 8; // len
- fixed_huff_dec_table[rev_short((0x30 + it), 8)] = it << 7; // symbol
- fixed_huff_dec_table[rev_short((0x30 + it), 8)] |= 8; // len
+ fixed_huff_dec_table[rbit9((0x30 + it), 8) | 0x100] = it << 7; // symbol
+ fixed_huff_dec_table[rbit9((0x30 + it), 8) | 0x100] |= 8; // len
+ fixed_huff_dec_table[rbit9((0x30 + it), 8)] = it << 7; // symbol
+ fixed_huff_dec_table[rbit9((0x30 + it), 8)] |= 8; // len
it += 1;
}
while (it <= 255) {
// 110010000 through 111111111
- fixed_huff_dec_table[rev_short(0x190 + it - 144, 9)] = it << 7; // symbol
- fixed_huff_dec_table[rev_short(0x190 + it - 144, 9)] |= 9; // len
+ fixed_huff_dec_table[rbit9(0x190 + it - 144, 9)] = it << 7; // symbol
+ fixed_huff_dec_table[rbit9(0x190 + it - 144, 9)] |= 9; // len
it += 1;
}
while (it <= 279) {
// 0000000XX through 0010111XX
- fixed_huff_dec_table[rev_short((0x0 + it - 256), 7)] = it << 7; // symbol
- fixed_huff_dec_table[rev_short((0x0 + it - 256), 7)] |= 7; // len
- fixed_huff_dec_table[rev_short((0x0 + it - 256), 7) | 0x80] = it << 7; // symbol
- fixed_huff_dec_table[rev_short((0x0 + it - 256), 7) | 0x80] |= 7; // len
- fixed_huff_dec_table[rev_short((0x0 + it - 256), 7) | 0x100] = it << 7; // symbol
- fixed_huff_dec_table[rev_short((0x0 + it - 256), 7) | 0x100] |= 7; // len
- fixed_huff_dec_table[rev_short((0x0 + it - 256), 7) | 0x180] = it << 7; // symbol
- fixed_huff_dec_table[rev_short((0x0 + it - 256), 7) | 0x180] |= 7; // len
+ fixed_huff_dec_table[rbit9((0x0 + it - 256), 7)] = it << 7; // symbol
+ fixed_huff_dec_table[rbit9((0x0 + it - 256), 7)] |= 7; // len
+ fixed_huff_dec_table[rbit9((0x0 + it - 256), 7) | 0x80] = it << 7; // symbol
+ fixed_huff_dec_table[rbit9((0x0 + it - 256), 7) | 0x80] |= 7; // len
+ fixed_huff_dec_table[rbit9((0x0 + it - 256), 7) | 0x100] = it << 7; // symbol
+ fixed_huff_dec_table[rbit9((0x0 + it - 256), 7) | 0x100] |= 7; // len
+ fixed_huff_dec_table[rbit9((0x0 + it - 256), 7) | 0x180] = it << 7; // symbol
+ fixed_huff_dec_table[rbit9((0x0 + it - 256), 7) | 0x180] |= 7; // len
it += 1;
}
while (it <= 287) {
// 11000000X through 11000111X
- fixed_huff_dec_table[rev_short((0xC0 + it - 280), 8)] = it << 7; // symbol
- fixed_huff_dec_table[rev_short((0xC0 + it - 280), 8)] |= 8; // len
- fixed_huff_dec_table[rev_short((0xC0 + it - 280), 8) | 0x100] = it << 7; // symbol
- fixed_huff_dec_table[rev_short((0xC0 + it - 280), 8) | 0x100] |= 8; // len
+ fixed_huff_dec_table[rbit9((0xC0 + it - 280), 8)] = it << 7; // symbol
+ fixed_huff_dec_table[rbit9((0xC0 + it - 280), 8)] |= 8; // len
+ fixed_huff_dec_table[rbit9((0xC0 + it - 280), 8) | 0x100] = it << 7; // symbol
+ fixed_huff_dec_table[rbit9((0xC0 + it - 280), 8) | 0x100] |= 8; // len
it += 1;
}
#endif
projects / haproxy.git / commitdiff