From 72453b868c862ed689da55dc92f4626bc605abf5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 23 Oct 2026 23:23:17 +0000 Subject: [PATCH] build(deps): bump github.com/klauspost/compress from 1.20.1 to 1.20.3 Bumps [github.com/klauspost/compress](https://github.com/klauspost/compress) from 1.20.1 to 1.20.3. - [Release notes](https://github.com/klauspost/compress/releases) - [Commits](https://github.com/klauspost/compress/compare/v1.20.1...v1.20.3) --- updated-dependencies: - dependency-name: github.com/klauspost/compress dependency-version: 1.20.3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] (cherry picked from commit c51faeab7e07ba0a08ea54e704f9e5f7acc111a7) Signed-off-by: Sebastiaan van Stijn --- go.mod | 2 +- go.sum | 4 +- .../github.com/klauspost/compress/.gitignore | 3 + .../github.com/klauspost/compress/AGENTS.md | 104 +++ .../github.com/klauspost/compress/README.md | 53 +- .../klauspost/compress/fse/compress.go | 3 +- .../klauspost/compress/fse/decompress.go | 6 + .../klauspost/compress/huff0/bitwriter.go | 93 ++- .../klauspost/compress/huff0/compress.go | 31 +- .../klauspost/compress/huff0/decompress.go | 49 +- .../compress/huff0/decompress_amd64.go | 2 +- .../compress/huff0/decompress_amd64.s | 14 +- .../compress/huff0/decompress_arm64.go | 2 +- .../compress/huff0/decompress_arm64.s | 248 +++---- .../compress/huff0/decompress_asm.go | 2 +- .../compress/huff0/decompress_generic.go | 2 +- .../internal/cpuinfo/cpuinfo_amd64.go | 2 +- .../compress/internal/cpuinfo/cpuinfo_amd64.s | 4 +- .../compress/internal/le/unsafe_disabled.go | 2 +- .../compress/internal/le/unsafe_enabled.go | 2 +- .../klauspost/compress/zstd/bitwriter.go | 39 ++ .../klauspost/compress/zstd/blockdec.go | 59 +- .../klauspost/compress/zstd/blockenc.go | 177 ++--- .../klauspost/compress/zstd/decoder.go | 30 +- .../klauspost/compress/zstd/dict.go | 14 +- .../klauspost/compress/zstd/enc_base.go | 1 + .../klauspost/compress/zstd/enc_better.go | 90 +-- .../klauspost/compress/zstd/enc_dfast.go | 67 +- .../klauspost/compress/zstd/framedec.go | 27 +- .../klauspost/compress/zstd/fse_decoder.go | 76 +- .../compress/zstd/fse_decoder_amd64.s | 110 +-- .../compress/zstd/fse_decoder_arm64.s | 124 ++-- .../compress/zstd/fse_decoder_asm.go | 8 +- .../compress/zstd/fse_decoder_generic.go | 9 +- .../klauspost/compress/zstd/fse_predefined.go | 18 +- .../klauspost/compress/zstd/hash.go | 22 + .../klauspost/compress/zstd/history.go | 89 +-- .../zstd/internal/xxhash/xxhash_amd64.s | 6 +- .../zstd/internal/xxhash/xxhash_arm64.s | 6 +- .../zstd/internal/xxhash/xxhash_asm.go | 7 +- .../zstd/internal/xxhash/xxhash_other.go | 2 +- .../klauspost/compress/zstd/matchlen_amd64.go | 2 +- .../klauspost/compress/zstd/matchlen_amd64.s | 2 +- .../compress/zstd/matchlen_generic.go | 2 +- .../klauspost/compress/zstd/seqdec.go | 41 +- .../klauspost/compress/zstd/seqdec_amd64.go | 12 +- .../klauspost/compress/zstd/seqdec_amd64.s | 610 ++++++++++++++++- .../klauspost/compress/zstd/seqdec_arm64.go | 13 +- .../klauspost/compress/zstd/seqdec_arm64.s | 648 ++++++++++++++++-- .../klauspost/compress/zstd/seqdec_asm.go | 35 +- .../klauspost/compress/zstd/seqdec_generic.go | 9 +- .../klauspost/compress/zstd/seqenc.go | 47 ++ .../zstd/{simple_go124.go => simple.go} | 2 - .../klauspost/compress/zstd/snappy.go | 12 + vendor/modules.txt | 2 +- 55 files changed, 2302 insertions(+), 744 deletions(-) create mode 100644 vendor/github.com/klauspost/compress/AGENTS.md rename vendor/github.com/klauspost/compress/zstd/{simple_go124.go => simple.go} (98%) diff --git a/go.mod b/go.mod index 4f0242a4f5..2e7f2aaf81 100644 --- a/go.mod +++ b/go.mod @@ -44,7 +44,7 @@ require ( github.com/google/uuid v1.6.0 github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.1.0 github.com/intel/goresctrl v0.13.0 - github.com/klauspost/compress v1.20.1 + github.com/klauspost/compress v1.20.3 github.com/mdlayher/vsock v1.3.0 github.com/moby/locker v1.0.1 github.com/moby/sys/mountinfo v0.7.2 diff --git a/go.sum b/go.sum index 5a456c7bea..a5f83e0fe2 100644 --- a/go.sum +++ b/go.sum @@ -185,8 +185,8 @@ github.com/jsimonetti/rtnetlink/v2 v2.0.1 h1:xda7qaHDSVOsADNouv7ukSuicKZO7GgVUCX github.com/jsimonetti/rtnetlink/v2 v2.0.1/go.mod h1:7MoNYNbb3UaDHtF8udiJo/RH6VsTKP1pqKLUTVCvToE= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ= -github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/klauspost/compress v1.20.3 h1:zWNoFqq/lSuYQRim3dUXlXeA0IqwuZGicdG5ZTfY5j4= +github.com/klauspost/compress v1.20.3/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= github.com/knqyf263/go-plugin v0.9.0 h1:CQs2+lOPIlkZVtcb835ZYDEoyyWJWLbSTWeCs0EwTwI= github.com/knqyf263/go-plugin v0.9.0/go.mod h1:2z5lCO1/pez6qGo8CvCxSlBFSEat4MEp1DrnA+f7w8Q= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= diff --git a/vendor/github.com/klauspost/compress/.gitignore b/vendor/github.com/klauspost/compress/.gitignore index d31b378152..fb06aafe83 100644 --- a/vendor/github.com/klauspost/compress/.gitignore +++ b/vendor/github.com/klauspost/compress/.gitignore @@ -30,3 +30,6 @@ perf.data.old # gdb history .gdb_history + +# Silesia corpus, fetched by testdata/fetch_silesia.sh +**/testdata/silesia.tar* diff --git a/vendor/github.com/klauspost/compress/AGENTS.md b/vendor/github.com/klauspost/compress/AGENTS.md new file mode 100644 index 0000000000..61d24b585a --- /dev/null +++ b/vendor/github.com/klauspost/compress/AGENTS.md @@ -0,0 +1,104 @@ +# AGENTS.md + +Guidance for people and coding agents changing this repository. It records +conventions and traps learned while working on the decoders and encoders; the +README describes the packages themselves. + +## Checks before sending a change + +- `gofmt`, `go vet ./...` and `go fix -diff ./...` (which should report + nothing). Code written for specific platforms must be vetted for them too: + run `go vet` with `GOOS`/`GOARCH` set to each platform it targets. `go vet` + caches results per package and does not notice edits to files excluded by + build tags; use a fresh `GOCACHE` to confirm a stale-looking error. +- `go test ./...` with and without `-tags=noasm` (and `nounsafe` if you + touched `unsafe` code), on every architecture whose assembly changed. +- Decoder changes: run the fuzzers CI runs (see `.github/workflows/go.yml`), + also with `-tags=noasm`, `-tags=nounsafe` and `-asan`, for longer than CI + does, and for zstd both `FuzzDecodeAll` and the `NoBMI2` variants. +- Raising the Go version in the root `go.mod` breaks the nested modules + (`*/_generate`, `s2/cmd/_s2sx`) until they are updated too. In a `go` + directive, `1.25` sorts before `1.25.0`, so a dependency that needs + `1.25.0` forces the full form. + +## Tests + +- Put a new test in the existing test file for the code it exercises + (`decoder_test.go` for Reader/DecodeAll behaviour, `seqdec_test.go` for + sequence execution, and so on), and reuse its helpers. Use table tests for + three or more similar cases. +- Extend the existing fuzzers (`func Fuzz…`) rather than adding parallel + harnesses; if a new harness supersedes an old one, remove the old one in + the same change. +- Prove which path a test or benchmark exercises. A benchmark once ran the + pure-Go sequence decoder instead of the assembly because of how its buffers + were set up. A `panic` planted in the path is a quick check; `println` + output is easy to miss. +- `sync.Pool` drops about a quarter of `Put`s under `-race` by design; never + assume `Get` returns the last `Put`. +- Don't add tests whose only purpose is to catch someone deliberately undoing + a performance pin (an alignment, say); a comment explaining it is enough. + +## Performance claims + +- Use real data. The maintainer's corpora are at + https://klauspost.com/files/compress/ (logs, JSON, CSV, serialized data, + tar files); put `.zst` files in `zstd/testdata/benchmark-custom/` and run + `BenchmarkDecoderWithCustomFiles`. Silesia, enwik and generated data are + fine for correctness but are not accepted as evidence for a speed-up. +- When a result depends on how the data was compressed, report the levels + separately (for example `SpeedDefault` and `SpeedBestCompression`). +- Anchor `-bench` patterns at every level with `$`: `-bench 'BenchmarkFoo$'` + also runs `BenchmarkFooBar` otherwise, which has produced fake gains and + regressions. +- Code placement alone moves results by about ±2%. For an A/B, insert + `PCALIGN $64` before every `TEXT` in the `.s` files of *both* builds + (measurement only, not committed). +- Build both variants as test binaries up front, then run them in turns: one + short `-count 1` run of each, repeated for ten or more rounds, swapping + which goes first every round (AB, BA, AB, …) so neither build always gets + the warmer or colder slot. Never run all of A and then all of B. Clock + speed, temperature, cache and page-cache state, and background load drift + over a session; alternating spreads that drift over both builds instead of + turning it into a difference between them. Pin to one core, use an + otherwise idle machine, and compare the pooled results with `benchstat`. +- Report the number of regressed cases next to the geomean; a small win with + no regressions is a different result from a larger one with some. +- No CPU-model-specific code paths or thresholds. A gate for arm64 must be + measured on several Neoverse generations (N1, V1, V2, V3), not one core. + +## Pull requests + +- One commit per independent change, with fixups squashed in. Number the + changes in the description as "(1) … (sha)" and name the test file for + each test. +- Mention each `#NNN` once in prose; refer back in words. +- Don't open many PRs against the same area at once; reviewer time is the + scarce resource. +- Keep Go comments short; generator and assembly comments can explain in + detail. +- Keep private or company-internal references (internal service names, + private links, chat or agent-session URLs) out of code, commit messages and + PR text. + +## Assembly is generated + +- zstd, huff0 and s2 assembly is generated with [avo](https://github.com/mmcloughlin/avo) + from `zstd/_generate`, `huff0/_generate` and `s2/_generate`. Never edit the + `.s` files by hand: change the generator, run `go generate -v -x` in the + `_generate` directory, and commit the regenerated output. CI's `generate` job + fails if the committed files differ from the generator's output. +- The arm64 files are lowered from the same amd64 program by the + `honeycombio/avo` fork that the `_generate/go.mod` files `replace` avo with. + A change to the generator changes both architectures; build and test both. +- When moving the avo pin, regenerate and check the output is byte-identical + unless a change is intended, so unrelated printer changes don't slip into a + review. +- Never put a label directly on a `PCALIGN` (emit the `PCALIGN` before the + label). A jump to it makes the amd64 assembler jump to the wrong place + (Go ≤ 1.25, golang/go#74648) or loop forever when it has to widen a branch + (Go ≥ 1.26, golang/go#81792). Current versions of the avo fork reject it. +- Build tags: `noasm` selects the pure-Go code and `nounsafe` the code without + `unsafe`. Every assembly or `unsafe` path needs a portable fallback, and CI + runs the default, `noasm`, `nounsafe` and `nounsafe,noasm` builds. The + library doesn't use cgo. diff --git a/vendor/github.com/klauspost/compress/README.md b/vendor/github.com/klauspost/compress/README.md index bf31b905f8..2ba61f4c33 100644 --- a/vendor/github.com/klauspost/compress/README.md +++ b/vendor/github.com/klauspost/compress/README.md @@ -29,8 +29,41 @@ Use the links above for more information on each. # changelog +* Sep 25th, 2026 [1.20.1](https://github.com/klauspost/compress/releases/tag/v1.20.1) + * zstd: Apply reset options to reused frame decoders by @pellared in https://github.com/klauspost/compress/pull/1226 + * flate: Report unexpected EOF for truncated Huffman extra bits by @rupayon123 in https://github.com/klauspost/compress/pull/1229 + * flate: Avoid FMA in EstimatedBits for portable rounding by @malt3 in https://github.com/klauspost/compress/pull/1224 + * gzhttp: Preserve caller request headers in Transport by @jakezwang in https://github.com/klauspost/compress/pull/1225 + * gzhttp: Only decompress a request when gzip is the outermost coding by @youdie006 in https://github.com/klauspost/compress/pull/1223 + * zstd/huff0: Optimizations by @lizthegrey in https://github.com/klauspost/compress/pull/1203 https://github.com/klauspost/compress/pull/1205 https://github.com/klauspost/compress/pull/1207 https://github.com/klauspost/compress/pull/1204 https://github.com/klauspost/compress/pull/1208 https://github.com/klauspost/compress/pull/1211 https://github.com/klauspost/compress/pull/1209 https://github.com/klauspost/compress/pull/1210 + * flate: Speed improvements by @alex in https://github.com/klauspost/compress/pull/1220 https://github.com/klauspost/compress/pull/1218 https://github.com/klauspost/compress/pull/1216 https://github.com/klauspost/compress/pull/1215 https://github.com/klauspost/compress/pull/1214 https://github.com/klauspost/compress/pull/1217 + +* Sep 2nd, 2026 [1.20.0](https://github.com/klauspost/compress/releases/tag/v1.20.0) + * lzw: Add lzw reader/writer in https://github.com/klauspost/compress/pull/1189 + * xpress: Add Microsoft XPRESS (MS-XCA) decompression by @MP-GOWTHAM in https://github.com/klauspost/compress/pull/1195 + * s2: arm64 assembly encoders by @lizthegrey in https://github.com/klauspost/compress/pull/1187 + * zstd, huff0: Improved arm64 assembly by @lizthegrey in https://github.com/klauspost/compress/pull/1175 https://github.com/klauspost/compress/pull/1199 https://github.com/klauspost/compress/pull/1197 + * flate: Writer L7-9 with dictionary can emit the dictionary to stream in https://github.com/klauspost/compress/pull/1201 + * gzhttp: Limit jitter hash to jitterBuffer bytes by @VedantMadane in https://github.com/klauspost/compress/pull/1192 + * s2: Clamp the Go EncodeBetter skip to match the assembly by @lizthegrey in https://github.com/klauspost/compress/pull/1190 + * xxhash: Bound input of non-preemptive assembly by @AskAlexSharov in https://github.com/klauspost/compress/pull/1196 + +* Aug 6th, 2026 [1.19.2](https://github.com/klauspost/compress/releases/tag/v1.19.2) + * huff0: Add arm64 assembly for Decompress4X/1X by @lizthegrey in https://github.com/klauspost/compress/pull/1172 + * zstd: Re-enable unsafe decodeSync memory copies by @lizthegrey in https://github.com/klauspost/compress/pull/1171 + * zstd: Fix arm64 asm frame offsets placing locals on the saved LR slot by @lizthegrey in https://github.com/klauspost/compress/pull/1176 + * zstd: Avoid racing MaxDecodedSize write on shared dict litEnc by @zanarellidev in https://github.com/klauspost/compress/pull/1182 + * zstd: Keep BuildDict recent-offsets positive and loadable by @zanarellidev in https://github.com/klauspost/compress/pull/1184 + * zstd: Handle zero-literal BuildDict corpus by @cyphercodes in https://github.com/klauspost/compress/pull/1178 + * zstd: Don't clear the registered dictionary when decoding past the window by @sueun-dev in https://github.com/klauspost/compress/pull/1177 + +* Jul 20th, 2026 [1.19.1](https://github.com/klauspost/compress/releases/tag/v1.19.1) + * zstd: Validate SnappyConverter literal copies in https://github.com/klauspost/compress/pull/1170 + * flate: Use `Peek` instead of `ReadByte` for the `bufio.Reader` decode path by @joechenrh in https://github.com/klauspost/compress/pull/1169 + * zstd: Bump avo pin, regenerate arm64 asm by @lizthegrey in https://github.com/klauspost/compress/pull/1167 + * Jul 1st, 2026 [1.19.0](https://github.com/klauspost/compress/releases/tag/v1.19.0) - * zstd: Add true concurrent stream encodingin https://github.com/klauspost/compress/pull/1136 + * zstd: Add true concurrent stream encoding in https://github.com/klauspost/compress/pull/1136 * zstd: arm64 decoder asm by @lizthegrey in https://github.com/klauspost/compress/pull/1160 * flate: Add inflate checkpoints in https://github.com/klauspost/compress/pull/1154 * zstd: avoid unused BuildDict encoder allocation by @snissn in https://github.com/klauspost/compress/pull/1147 @@ -41,6 +74,22 @@ Use the links above for more information on each. * huff0: Allow building table from oversized histogram in https://github.com/klauspost/compress/pull/1156 * s2sx: Clean symlink targets in https://github.com/klauspost/compress/pull/1163 +
+ See changes to v1.18.x + +* Jun 30th, 2026 [1.18.7](https://github.com/klauspost/compress/releases/tag/v1.18.7) + * s2: Fix `NewDict` out-of-bounds read. Security release without other changes. + +* Apr 30th, 2026 [1.18.6](https://github.com/klauspost/compress/releases/tag/v1.18.6) + * s2: Fix amd64 stack frame corruption in https://github.com/klauspost/compress/pull/1145 + * gzhttp: Canonicalize ETag header by @justinmayhew in https://github.com/klauspost/compress/pull/1139 + * Pool hash tables in Go encode paths to reduce allocations by @huynhanx03 in https://github.com/klauspost/compress/pull/1143 + +* Mar 20th, 2026 [1.18.5](https://github.com/klauspost/compress/releases/tag/v1.18.5) + * zstd: Fix crash when changing encoder dictionary with same ID in https://github.com/klauspost/compress/pull/1135 + * zstd: Default to full zero frames in https://github.com/klauspost/compress/pull/1134 + * flate: Clean up histogram order in https://github.com/klauspost/compress/pull/1133 + * Feb 9th, 2026 [1.18.4](https://github.com/klauspost/compress/releases/tag/v1.18.4) * gzhttp: Add zstandard to server handler wrapper https://github.com/klauspost/compress/pull/1121 * zstd: Add ResetWithOptions to encoder/decoder https://github.com/klauspost/compress/pull/1122 @@ -72,6 +121,8 @@ Use the links above for more information on each. * flate: Fix matchlen L5+L6 https://github.com/klauspost/compress/pull/1049 * flate: Cleanup & reduce casts https://github.com/klauspost/compress/pull/1050 +
+
See changes to v1.17.x diff --git a/vendor/github.com/klauspost/compress/fse/compress.go b/vendor/github.com/klauspost/compress/fse/compress.go index 8c8baa4fc2..2a7cd684ea 100644 --- a/vendor/github.com/klauspost/compress/fse/compress.go +++ b/vendor/github.com/klauspost/compress/fse/compress.go @@ -659,7 +659,8 @@ func (s *Scratch) validateNorm() (err error) { if v >= 0 { total += int(v) } else { - total -= int(v) + // -1 takes one cell. Adding -v would let a wrapped -32768 pass. + total++ } } defer func() { diff --git a/vendor/github.com/klauspost/compress/fse/decompress.go b/vendor/github.com/klauspost/compress/fse/decompress.go index 0c7dd4ffef..85b55db579 100644 --- a/vendor/github.com/klauspost/compress/fse/decompress.go +++ b/vendor/github.com/klauspost/compress/fse/decompress.go @@ -3,6 +3,7 @@ package fse import ( "errors" "fmt" + "math" ) const ( @@ -123,6 +124,11 @@ func (s *Scratch) readNCount() error { remaining -= count gotTotal += count } + if count > math.MaxInt16 { + // Only one symbol filling a tableLog 15 table, which cannot be decoded. + // Truncated, it would leave the decoding table unwritten. + return fmt.Errorf("corruption detected (count %d > %d)", count, math.MaxInt16) + } s.norm[charnum&0xff] = int16(count) charnum++ previous0 = count == 0 diff --git a/vendor/github.com/klauspost/compress/huff0/bitwriter.go b/vendor/github.com/klauspost/compress/huff0/bitwriter.go index 41db94cded..c48e7eade0 100644 --- a/vendor/github.com/klauspost/compress/huff0/bitwriter.go +++ b/vendor/github.com/klauspost/compress/huff0/bitwriter.go @@ -5,6 +5,50 @@ package huff0 +import ( + "github.com/klauspost/compress/internal/le" + "github.com/klauspost/compress/internal/regmask" +) + +// bitState is a bit writer that stays in registers when passed and +// returned by value. It writes into a buffer owned by the caller. +type bitState struct { + c uint64 + nb uint + pos int +} + +// flush writes the whole bytes with one 8-byte store at pos, leaving +// at most 7 bits. buf must have 8 bytes of room at pos. +func (s bitState) flush(buf []byte) bitState { + le.Store64(buf, s.pos, s.c) + s.pos += int(s.nb >> 3) + s.c >>= (s.nb &^ 7) & regmask.Shift64ByUint + s.nb &= 7 + return s +} + +// add adds n bits from v, which must have no bits set above them. +// There must be room for them. +func (s bitState) add(v uint64, n uint) bitState { + s.c |= v << (s.nb & regmask.Shift64ByUint) + s.nb += n + return s +} + +// fourSymbols returns the codes for a, b, c and d packed in that order +// from the LSB, and their total length. +func fourSymbols(a, b, c, d cTableEntry) (uint64, uint) { + bitsA := uint(a.nBits) + bitsB := bitsA + uint(b.nBits) + bitsC := bitsB + uint(c.nBits) + v := uint64(a.val) | + uint64(b.val)<<(bitsA®mask.Shift64ByUint) | + uint64(c.val)<<(bitsB®mask.Shift64ByUint) | + uint64(d.val)<<(bitsC®mask.Shift64ByUint) + return v, bitsC + uint(d.nBits) +} + // bitWriter will write bits. // First bit will be LSB of the first byte of output. type bitWriter struct { @@ -33,55 +77,6 @@ func (b *bitWriter) encSymbol(ct cTable, symbol byte) { b.nBits += enc.nBits } -// encTwoSymbols will add up to 32 bits. value may not contain more set bits than indicated. -// It will not check if there is space for them, so the caller must ensure that it has flushed recently. -func (b *bitWriter) encTwoSymbols(ct cTable, av, bv byte) { - encA := ct[av] - encB := ct[bv] - sh := b.nBits & 63 - combined := uint64(encA.val) | (uint64(encB.val) << (encA.nBits & 63)) - b.bitContainer |= combined << sh - if false { - if encA.nBits == 0 { - panic("nbitsA 0") - } - if encB.nBits == 0 { - panic("nbitsB 0") - } - } - b.nBits += encA.nBits + encB.nBits -} - -// encFourSymbols adds up to 32 bits from four symbols. -// It will not check if there is space for them, -// so the caller must ensure that b has been flushed recently. -func (b *bitWriter) encFourSymbols(encA, encB, encC, encD cTableEntry) { - bitsA := encA.nBits - bitsB := bitsA + encB.nBits - bitsC := bitsB + encC.nBits - bitsD := bitsC + encD.nBits - combined := uint64(encA.val) | - (uint64(encB.val) << (bitsA & 63)) | - (uint64(encC.val) << (bitsB & 63)) | - (uint64(encD.val) << (bitsC & 63)) - b.bitContainer |= combined << (b.nBits & 63) - b.nBits += bitsD -} - -// flush32 will flush out, so there are at least 32 bits available for writing. -func (b *bitWriter) flush32() { - if b.nBits < 32 { - return - } - b.out = append(b.out, - byte(b.bitContainer), - byte(b.bitContainer>>8), - byte(b.bitContainer>>16), - byte(b.bitContainer>>24)) - b.nBits -= 32 - b.bitContainer >>= 32 -} - // flushAlign will flush remaining full bytes and align to next byte boundary. func (b *bitWriter) flushAlign() { nbBytes := (b.nBits + 7) >> 3 diff --git a/vendor/github.com/klauspost/compress/huff0/compress.go b/vendor/github.com/klauspost/compress/huff0/compress.go index a97cf1b5d3..1406b766c4 100644 --- a/vendor/github.com/klauspost/compress/huff0/compress.go +++ b/vendor/github.com/klauspost/compress/huff0/compress.go @@ -4,6 +4,7 @@ import ( "fmt" "math" "runtime" + "slices" "sync" ) @@ -243,23 +244,27 @@ func (s *Scratch) compress1xDo(dst, src []byte) []byte { bw.encSymbol(cTable, src[n+i-1]) } n -= 4 - if s.actualTableLog <= 8 { - for ; n >= 0; n -= 4 { - tmp := src[n : n+4] - // tmp should be len 4 - bw.flush32() - bw.encFourSymbols(cTable[tmp[3]], cTable[tmp[2]], cTable[tmp[1]], cTable[tmp[0]]) + + // flush leaves at most 7 bits, so four symbols (at most 4*tableLogMax + // bits) always fit. Each chunk checks room for bytesPer4 per four + // symbols plus the 8-byte store. + const bytesPer4 = (7 + 4*tableLogMax) / 8 + w := bitState{c: bw.bitContainer, nb: uint(bw.nBits), pos: len(bw.out)} + buf := bw.out[:cap(bw.out)] + for n >= 0 { + k := (len(buf) - w.pos - 8) / bytesPer4 + if k <= 0 { + buf = slices.Grow(buf[:w.pos], (n/4+1)*bytesPer4+8) + buf = buf[:cap(buf)] + continue } - } else { - for ; n >= 0; n -= 4 { + for stop := max(n-4*k, -4); n > stop; n -= 4 { + w = w.flush(buf) tmp := src[n : n+4] - // tmp should be len 4 - bw.flush32() - bw.encTwoSymbols(cTable, tmp[3], tmp[2]) - bw.flush32() - bw.encTwoSymbols(cTable, tmp[1], tmp[0]) + w = w.add(fourSymbols(cTable[tmp[3]], cTable[tmp[2]], cTable[tmp[1]], cTable[tmp[0]])) } } + bw.bitContainer, bw.nBits, bw.out = w.c, uint8(w.nb), buf[:w.pos] bw.close() return bw.out } diff --git a/vendor/github.com/klauspost/compress/huff0/decompress.go b/vendor/github.com/klauspost/compress/huff0/decompress.go index 7d0efa8818..9358784417 100644 --- a/vendor/github.com/klauspost/compress/huff0/decompress.go +++ b/vendor/github.com/klauspost/compress/huff0/decompress.go @@ -54,6 +54,10 @@ func ReadTable(in []byte, s *Scratch) (s2 *Scratch, remain []byte, err error) { if len(in) < int(iSize) { return s, nil, fmt.Errorf("input too small for table, want %d bytes, have %d", iSize, len(in)) } + // The low nibble is the weights' FSE tableLog - 5, which the format limits to 6. + if in[0]&15+5 > 6 { + return s, nil, errors.New("corrupt input: weights tableLog too big") + } // FSE compressed weights s.fse.DecompressLimit = 255 hw := s.huffWeight[:] @@ -70,17 +74,32 @@ func ReadTable(in []byte, s *Scratch) (s2 *Scratch, remain []byte, err error) { in = in[iSize:] } - // collect weight stats + // collect weight stats. Weights repeat, so four histograms keep each + // increment from waiting on the previous store to the same counter. + var hist [4][16]uint32 + var seen uint8 + w := s.huffWeight[:s.symbolLen] + for ; len(w) >= 4; w = w[4:] { + hist[0][w[0]&15]++ + hist[1][w[1]&15]++ + hist[2][w[2]&15]++ + hist[3][w[3]&15]++ + seen |= w[0] | w[1] | w[2] | w[3] + } + for _, v := range w { + hist[0][v&15]++ + seen |= v + } var rankStats [16]uint32 weightTotal := uint32(0) - for _, v := range s.huffWeight[:s.symbolLen] { - if v > tableLogMax { - return s, nil, errors.New("corrupt input: weight too large") - } - v2 := v & 15 - rankStats[v2]++ - // (1 << (v2-1)) is slower since the compiler cannot prove that v2 isn't 0. - weightTotal += (1 << v2) >> 1 + for i := range rankStats { + rankStats[i] = hist[0][i] + hist[1][i] + hist[2][i] + hist[3][i] + } + for i := 1; i < len(rankStats); i++ { + weightTotal += rankStats[i] << (i - 1) + } + if seen > 15 || rankStats[tableLogMax+1]|rankStats[tableLogMax+2]|rankStats[tableLogMax+3]|rankStats[tableLogMax+4] != 0 { + return s, nil, errors.New("corrupt input: weight too large") } if weightTotal == 0 { return s, nil, errors.New("corrupt input: weights zero") @@ -158,9 +177,17 @@ func ReadTable(in []byte, s *Scratch) (s2 *Scratch, remain []byte, err error) { nBits: uint8(d.entry), } + // length is a power of two; runs of 4 or more are filled 4 at a time. single := s.dt.single[*rank : *rank+length] - for i := range single { - single[i] = d + if length >= 4 { + for i := 0; i < len(single); i += 4 { + s4 := single[i : i+4 : i+4] + s4[0], s4[1], s4[2], s4[3] = d, d, d, d + } + } else { + for i := range single { + single[i] = d + } } *rank += length } diff --git a/vendor/github.com/klauspost/compress/huff0/decompress_amd64.go b/vendor/github.com/klauspost/compress/huff0/decompress_amd64.go index e651f30ecb..169cf10458 100644 --- a/vendor/github.com/klauspost/compress/huff0/decompress_amd64.go +++ b/vendor/github.com/klauspost/compress/huff0/decompress_amd64.go @@ -1,4 +1,4 @@ -//go:build amd64 && !appengine && !noasm && gc +//go:build amd64 && !noasm && gc // amd64 stubs and dispatch for the asm loops used by decompress_asm.go. package huff0 diff --git a/vendor/github.com/klauspost/compress/huff0/decompress_amd64.s b/vendor/github.com/klauspost/compress/huff0/decompress_amd64.s index f072c18f04..a6d5f4b389 100644 --- a/vendor/github.com/klauspost/compress/huff0/decompress_amd64.s +++ b/vendor/github.com/klauspost/compress/huff0/decompress_amd64.s @@ -1,6 +1,6 @@ // Code generated by command: go run gen.go -out ../decompress.s -arch amd64,arm64 -pkg=huff0. DO NOT EDIT. -//go:build !appengine && !noasm && gc +//go:build !noasm && gc // func decompress4x_main_loop_amd64(ctx *decompress4xContext) // Requires: BMI, CMOV @@ -67,6 +67,7 @@ outer_loop: IMUL3Q $0x05, CX, CX ADDQ SI, CX MOVQ CX, 96(AX) + PCALIGN $0x40 inner_loop: // stream 0, symbol 0 @@ -388,6 +389,7 @@ outer_loop: IMUL3Q $0x07, CX, CX ADDQ SI, CX MOVQ CX, 96(AX) + PCALIGN $0x40 inner_loop: // stream 0, symbol 0 @@ -781,6 +783,7 @@ outer_loop: IMUL3Q $0x0e, CX, CX ADDQ SI, CX MOVQ CX, 96(AX) + PCALIGN $0x40 inner_loop: // stream 0, symbol 0 @@ -1426,6 +1429,7 @@ outer_loop: IMUL3Q $0x05, R13, R13 ADDQ BX, R13 MOVQ R13, 96(AX) + PCALIGN $0x40 inner_loop: // stream 0, symbol 0 @@ -1707,6 +1711,7 @@ outer_loop: IMUL3Q $0x07, R13, R13 ADDQ BX, R13 MOVQ R13, 96(AX) + PCALIGN $0x40 inner_loop: // stream 0, symbol 0 @@ -2044,6 +2049,7 @@ outer_loop: IMUL3Q $0x0e, R13, R13 ADDQ BX, R13 MOVQ R13, 96(AX) + PCALIGN $0x40 inner_loop: // stream 0, symbol 0 @@ -2548,6 +2554,7 @@ outer_loop: JZ done IMUL3Q $0x05, CX, R12 ADDQ SI, R12 + PCALIGN $0x40 inner_loop: // symbol 0 @@ -2666,6 +2673,7 @@ outer_loop: JZ done IMUL3Q $0x07, CX, R12 ADDQ SI, R12 + PCALIGN $0x40 inner_loop: // symbol 0 @@ -2804,6 +2812,7 @@ outer_loop: JZ done IMUL3Q $0x0e, CX, R12 ADDQ SI, R12 + PCALIGN $0x40 inner_loop: // symbol 0 @@ -3012,6 +3021,7 @@ outer_loop: JZ done IMUL3Q $0x05, R11, R11 ADDQ BX, R11 + PCALIGN $0x40 inner_loop: // symbol 0 @@ -3119,6 +3129,7 @@ outer_loop: JZ done IMUL3Q $0x07, R11, R11 ADDQ BX, R11 + PCALIGN $0x40 inner_loop: // symbol 0 @@ -3242,6 +3253,7 @@ outer_loop: JZ done IMUL3Q $0x0e, R11, R11 ADDQ BX, R11 + PCALIGN $0x40 inner_loop: // symbol 0 diff --git a/vendor/github.com/klauspost/compress/huff0/decompress_arm64.go b/vendor/github.com/klauspost/compress/huff0/decompress_arm64.go index d1988ad54d..d6bdef6a71 100644 --- a/vendor/github.com/klauspost/compress/huff0/decompress_arm64.go +++ b/vendor/github.com/klauspost/compress/huff0/decompress_arm64.go @@ -1,4 +1,4 @@ -//go:build arm64 && !appengine && !noasm && gc +//go:build arm64 && !noasm && gc // arm64 stubs and dispatch for the asm loops used by decompress_asm.go. // The asm (decompress_arm64.s) is generated by the avo arm64 lowering diff --git a/vendor/github.com/klauspost/compress/huff0/decompress_arm64.s b/vendor/github.com/klauspost/compress/huff0/decompress_arm64.s index 971b8eb8bd..2db7345f25 100644 --- a/vendor/github.com/klauspost/compress/huff0/decompress_arm64.s +++ b/vendor/github.com/klauspost/compress/huff0/decompress_arm64.s @@ -1,7 +1,7 @@ // Code generated by command: go run gen.go -out ../decompress.s -arch amd64,arm64 -pkg=huff0. DO NOT EDIT. // EXPERIMENTAL arm64 output lowered from an amd64 avo program. -//go:build arm64 && (!appengine && !noasm && gc) +//go:build arm64 && (!noasm && gc) // func decompress4x_main_loop_amd64(ctx *decompress4xContext) // Requires: BMI, CMOV @@ -51,36 +51,37 @@ outer_loop: // Iterations allowed by the input: a reload backs a pointer up by at most 7 bytes // (whatever nSyms is), so every read stays inside the block while the lowest // pointer stays above ilowest. - MOVD 64(R0), R13 - MOVD 56(R0), R16 - SUB R16, R13, R13 - LSR $0x03, R13, R13 - CMP R1, R13 - CSEL LO, R13, R1, R1 - MOVD 72(R0), R13 - MOVD 56(R0), R16 - SUB R16, R13, R13 - LSR $0x03, R13, R13 - CMP R1, R13 - CSEL LO, R13, R1, R1 - MOVD 80(R0), R13 - MOVD 56(R0), R16 - SUB R16, R13, R13 - LSR $0x03, R13, R13 - CMP R1, R13 - CSEL LO, R13, R1, R1 - MOVD 88(R0), R13 - MOVD 56(R0), R16 - SUB R16, R13, R13 - LSR $0x03, R13, R13 - CMP R1, R13 - CSEL LO, R13, R1, R1 - TST R1, R1 - BEQ done - MOVD $5, R16 - MUL R16, R1, R1 - ADD R5, R1, R1 - MOVD R1, 96(R0) + MOVD 64(R0), R13 + MOVD 56(R0), R16 + SUB R16, R13, R13 + LSR $0x03, R13, R13 + CMP R1, R13 + CSEL LO, R13, R1, R1 + MOVD 72(R0), R13 + MOVD 56(R0), R16 + SUB R16, R13, R13 + LSR $0x03, R13, R13 + CMP R1, R13 + CSEL LO, R13, R1, R1 + MOVD 80(R0), R13 + MOVD 56(R0), R16 + SUB R16, R13, R13 + LSR $0x03, R13, R13 + CMP R1, R13 + CSEL LO, R13, R1, R1 + MOVD 88(R0), R13 + MOVD 56(R0), R16 + SUB R16, R13, R13 + LSR $0x03, R13, R13 + CMP R1, R13 + CSEL LO, R13, R1, R1 + TST R1, R1 + BEQ done + MOVD $5, R16 + MUL R16, R1, R1 + ADD R5, R1, R1 + MOVD R1, 96(R0) + PCALIGN $0x40 inner_loop: // stream 0, symbol 0 @@ -355,36 +356,37 @@ outer_loop: // Iterations allowed by the input: a reload backs a pointer up by at most 7 bytes // (whatever nSyms is), so every read stays inside the block while the lowest // pointer stays above ilowest. - MOVD 64(R0), R13 - MOVD 56(R0), R16 - SUB R16, R13, R13 - LSR $0x03, R13, R13 - CMP R1, R13 - CSEL LO, R13, R1, R1 - MOVD 72(R0), R13 - MOVD 56(R0), R16 - SUB R16, R13, R13 - LSR $0x03, R13, R13 - CMP R1, R13 - CSEL LO, R13, R1, R1 - MOVD 80(R0), R13 - MOVD 56(R0), R16 - SUB R16, R13, R13 - LSR $0x03, R13, R13 - CMP R1, R13 - CSEL LO, R13, R1, R1 - MOVD 88(R0), R13 - MOVD 56(R0), R16 - SUB R16, R13, R13 - LSR $0x03, R13, R13 - CMP R1, R13 - CSEL LO, R13, R1, R1 - TST R1, R1 - BEQ done - MOVD $7, R16 - MUL R16, R1, R1 - ADD R5, R1, R1 - MOVD R1, 96(R0) + MOVD 64(R0), R13 + MOVD 56(R0), R16 + SUB R16, R13, R13 + LSR $0x03, R13, R13 + CMP R1, R13 + CSEL LO, R13, R1, R1 + MOVD 72(R0), R13 + MOVD 56(R0), R16 + SUB R16, R13, R13 + LSR $0x03, R13, R13 + CMP R1, R13 + CSEL LO, R13, R1, R1 + MOVD 80(R0), R13 + MOVD 56(R0), R16 + SUB R16, R13, R13 + LSR $0x03, R13, R13 + CMP R1, R13 + CSEL LO, R13, R1, R1 + MOVD 88(R0), R13 + MOVD 56(R0), R16 + SUB R16, R13, R13 + LSR $0x03, R13, R13 + CMP R1, R13 + CSEL LO, R13, R1, R1 + TST R1, R1 + BEQ done + MOVD $7, R16 + MUL R16, R1, R1 + ADD R5, R1, R1 + MOVD R1, 96(R0) + PCALIGN $0x40 inner_loop: // stream 0, symbol 0 @@ -715,36 +717,37 @@ outer_loop: // Iterations allowed by the input: a reload backs a pointer up by at most 7 bytes // (whatever nSyms is), so every read stays inside the block while the lowest // pointer stays above ilowest. - MOVD 64(R0), R13 - MOVD 56(R0), R16 - SUB R16, R13, R13 - LSR $0x03, R13, R13 - CMP R1, R13 - CSEL LO, R13, R1, R1 - MOVD 72(R0), R13 - MOVD 56(R0), R16 - SUB R16, R13, R13 - LSR $0x03, R13, R13 - CMP R1, R13 - CSEL LO, R13, R1, R1 - MOVD 80(R0), R13 - MOVD 56(R0), R16 - SUB R16, R13, R13 - LSR $0x03, R13, R13 - CMP R1, R13 - CSEL LO, R13, R1, R1 - MOVD 88(R0), R13 - MOVD 56(R0), R16 - SUB R16, R13, R13 - LSR $0x03, R13, R13 - CMP R1, R13 - CSEL LO, R13, R1, R1 - TST R1, R1 - BEQ done - MOVD $14, R16 - MUL R16, R1, R1 - ADD R5, R1, R1 - MOVD R1, 96(R0) + MOVD 64(R0), R13 + MOVD 56(R0), R16 + SUB R16, R13, R13 + LSR $0x03, R13, R13 + CMP R1, R13 + CSEL LO, R13, R1, R1 + MOVD 72(R0), R13 + MOVD 56(R0), R16 + SUB R16, R13, R13 + LSR $0x03, R13, R13 + CMP R1, R13 + CSEL LO, R13, R1, R1 + MOVD 80(R0), R13 + MOVD 56(R0), R16 + SUB R16, R13, R13 + LSR $0x03, R13, R13 + CMP R1, R13 + CSEL LO, R13, R1, R1 + MOVD 88(R0), R13 + MOVD 56(R0), R16 + SUB R16, R13, R13 + LSR $0x03, R13, R13 + CMP R1, R13 + CSEL LO, R13, R1, R1 + TST R1, R1 + BEQ done + MOVD $14, R16 + MUL R16, R1, R1 + ADD R5, R1, R1 + MOVD R1, 96(R0) + PCALIGN $0x40 inner_loop: // stream 0, symbol 0 @@ -1256,16 +1259,17 @@ outer_loop: // Iterations allowed by the input: a refill reads the 8 bytes below the window and // moves it down by at most 7, so every read stays inside the stream while ip stays // at least 8 above ilowest. - MOVD R7, R11 - SUB R8, R11, R11 - LSR $0x03, R11, R11 - CMP R1, R11 - CSEL LO, R11, R1, R1 - TST R1, R1 - BEQ done - MOVD $5, R16 - MUL R16, R1, R11 - ADD R5, R11, R11 + MOVD R7, R11 + SUB R8, R11, R11 + LSR $0x03, R11, R11 + CMP R1, R11 + CSEL LO, R11, R1, R1 + TST R1, R1 + BEQ done + MOVD $5, R16 + MUL R16, R1, R11 + ADD R5, R11, R11 + PCALIGN $0x40 inner_loop: // symbol 0 @@ -1367,16 +1371,17 @@ outer_loop: // Iterations allowed by the input: a refill reads the 8 bytes below the window and // moves it down by at most 7, so every read stays inside the stream while ip stays // at least 8 above ilowest. - MOVD R7, R11 - SUB R8, R11, R11 - LSR $0x03, R11, R11 - CMP R1, R11 - CSEL LO, R11, R1, R1 - TST R1, R1 - BEQ done - MOVD $7, R16 - MUL R16, R1, R11 - ADD R5, R11, R11 + MOVD R7, R11 + SUB R8, R11, R11 + LSR $0x03, R11, R11 + CMP R1, R11 + CSEL LO, R11, R1, R1 + TST R1, R1 + BEQ done + MOVD $7, R16 + MUL R16, R1, R11 + ADD R5, R11, R11 + PCALIGN $0x40 inner_loop: // symbol 0 @@ -1494,16 +1499,17 @@ outer_loop: // Iterations allowed by the input: a refill reads the 8 bytes below the window and // moves it down by at most 7, so every read stays inside the stream while ip stays // at least 8 above ilowest. - MOVD R7, R11 - SUB R8, R11, R11 - LSR $0x03, R11, R11 - CMP R1, R11 - CSEL LO, R11, R1, R1 - TST R1, R1 - BEQ done - MOVD $14, R16 - MUL R16, R1, R11 - ADD R5, R11, R11 + MOVD R7, R11 + SUB R8, R11, R11 + LSR $0x03, R11, R11 + CMP R1, R11 + CSEL LO, R11, R1, R1 + TST R1, R1 + BEQ done + MOVD $14, R16 + MUL R16, R1, R11 + ADD R5, R11, R11 + PCALIGN $0x40 inner_loop: // symbol 0 diff --git a/vendor/github.com/klauspost/compress/huff0/decompress_asm.go b/vendor/github.com/klauspost/compress/huff0/decompress_asm.go index 71a80bbd55..0d9672ea75 100644 --- a/vendor/github.com/klauspost/compress/huff0/decompress_asm.go +++ b/vendor/github.com/klauspost/compress/huff0/decompress_asm.go @@ -1,4 +1,4 @@ -//go:build (amd64 || arm64) && !appengine && !noasm && gc +//go:build (amd64 || arm64) && !noasm && gc // This file contains the specialisation of Decoder.Decompress4X // and Decoder.Decompress1X that use an asm implementation of their main loops. diff --git a/vendor/github.com/klauspost/compress/huff0/decompress_generic.go b/vendor/github.com/klauspost/compress/huff0/decompress_generic.go index e1bca5c686..8db45a1106 100644 --- a/vendor/github.com/klauspost/compress/huff0/decompress_generic.go +++ b/vendor/github.com/klauspost/compress/huff0/decompress_generic.go @@ -1,4 +1,4 @@ -//go:build (!amd64 && !arm64) || appengine || !gc || noasm +//go:build (!amd64 && !arm64) || !gc || noasm // This file contains a generic implementation of Decoder.Decompress4X. package huff0 diff --git a/vendor/github.com/klauspost/compress/internal/cpuinfo/cpuinfo_amd64.go b/vendor/github.com/klauspost/compress/internal/cpuinfo/cpuinfo_amd64.go index b97f9056f4..3d9e20f1a0 100644 --- a/vendor/github.com/klauspost/compress/internal/cpuinfo/cpuinfo_amd64.go +++ b/vendor/github.com/klauspost/compress/internal/cpuinfo/cpuinfo_amd64.go @@ -1,4 +1,4 @@ -//go:build amd64 && !appengine && !noasm && gc +//go:build amd64 && !noasm && gc package cpuinfo diff --git a/vendor/github.com/klauspost/compress/internal/cpuinfo/cpuinfo_amd64.s b/vendor/github.com/klauspost/compress/internal/cpuinfo/cpuinfo_amd64.s index 4465fbe9e9..f128c10c6f 100644 --- a/vendor/github.com/klauspost/compress/internal/cpuinfo/cpuinfo_amd64.s +++ b/vendor/github.com/klauspost/compress/internal/cpuinfo/cpuinfo_amd64.s @@ -1,6 +1,4 @@ -// +build !appengine -// +build gc -// +build !noasm +//go:build !noasm && gc #include "textflag.h" #include "funcdata.h" diff --git a/vendor/github.com/klauspost/compress/internal/le/unsafe_disabled.go b/vendor/github.com/klauspost/compress/internal/le/unsafe_disabled.go index 4f2a0d8c58..787ec9ef13 100644 --- a/vendor/github.com/klauspost/compress/internal/le/unsafe_disabled.go +++ b/vendor/github.com/klauspost/compress/internal/le/unsafe_disabled.go @@ -1,4 +1,4 @@ -//go:build !(amd64 || arm64 || ppc64le || riscv64) || nounsafe || purego || appengine +//go:build !(amd64 || arm64 || ppc64le || riscv64) || nounsafe || purego package le diff --git a/vendor/github.com/klauspost/compress/internal/le/unsafe_enabled.go b/vendor/github.com/klauspost/compress/internal/le/unsafe_enabled.go index 218a38bc4a..e60be8158e 100644 --- a/vendor/github.com/klauspost/compress/internal/le/unsafe_enabled.go +++ b/vendor/github.com/klauspost/compress/internal/le/unsafe_enabled.go @@ -1,6 +1,6 @@ // We enable 64 bit LE platforms: -//go:build (amd64 || arm64 || ppc64le || riscv64) && !nounsafe && !purego && !appengine +//go:build (amd64 || arm64 || ppc64le || riscv64) && !nounsafe && !purego package le diff --git a/vendor/github.com/klauspost/compress/zstd/bitwriter.go b/vendor/github.com/klauspost/compress/zstd/bitwriter.go index b22b297e62..668ac26108 100644 --- a/vendor/github.com/klauspost/compress/zstd/bitwriter.go +++ b/vendor/github.com/klauspost/compress/zstd/bitwriter.go @@ -5,6 +5,45 @@ package zstd +import ( + "github.com/klauspost/compress/internal/le" + "github.com/klauspost/compress/internal/regmask" +) + +// bitState is a bit writer that stays in registers when passed and +// returned by value. It writes into a buffer owned by the caller. +type bitState struct { + c uint64 + nb uint + pos int +} + +// flush writes the whole bytes with one 8-byte store at pos, leaving +// at most 7 bits. buf must have 8 bytes of room at pos. +func (s bitState) flush(buf []byte) bitState { + le.Store64(buf, s.pos, s.c) + s.pos += int(s.nb >> 3) + s.c >>= (s.nb &^ 7) & regmask.Shift64ByUint + s.nb &= 7 + return s +} + +// add adds n bits from v, which must have no bits set above them. +// There must be room for them. +func (s bitState) add(v uint64, n uint) bitState { + s.c |= v << (s.nb & regmask.Shift64ByUint) + s.nb += n + return s +} + +// addState adds the bits that FSE-encode tt from state, and returns +// the next state from tab. +func (s bitState) addState(state uint16, tab []uint16, tt symbolTransform) (bitState, uint16) { + nbBitsOut := (uint32(state) + tt.deltaNbBits) >> 16 + s = s.add(uint64(state&bitMask16[nbBitsOut&31]), uint(nbBitsOut)) + return s, tab[int32(state>>(nbBitsOut®mask.Shift16ByUint32))+int32(tt.deltaFindState)] +} + // bitWriter will write bits. // First bit will be LSB of the first byte of output. type bitWriter struct { diff --git a/vendor/github.com/klauspost/compress/zstd/blockdec.go b/vendor/github.com/klauspost/compress/zstd/blockdec.go index 89b20d6a84..74fc557019 100644 --- a/vendor/github.com/klauspost/compress/zstd/blockdec.go +++ b/vendor/github.com/klauspost/compress/zstd/blockdec.go @@ -418,6 +418,8 @@ func (b *blockDec) decodeLiterals(in []byte, hist *history) (remain []byte, err if len(literals) != litRegenSize { return in, fmt.Errorf("literal output size mismatch want %d, got %d", litRegenSize, len(literals)) } + // Re-cap to get extra size. + literals = b.literalBuf[:len(literals)] case literalsBlockCompressed: if len(in) < litCompSize { @@ -493,7 +495,20 @@ func (b *blockDec) decodeCompressed(hist *history) error { return nil } before := len(hist.decoders.out) - err = hist.decoders.decodeSync(hist.b[hist.ignoreBuffer:]) + h := hist.b[hist.ignoreBuffer:] + if hist.ext != nil { + h = hist.ext + } + if hist.dict == nil && hist.decoders.useTwoPass() { + // Two passes, as the concurrent decoder does: knowing the offsets + // up front lets executeSimple prefetch match sources. + if err := b.decodeSequences(hist); err != nil { + return err + } + err = hist.decoders.executeSimple(b.sequence, h) + } else { + err = hist.decoders.decodeSync(h) + } if err != nil { return err } @@ -604,16 +619,11 @@ func (b *blockDec) prepareSequences(in []byte, hist *history) (err error) { if seq.fse == nil || seq.fse.preDefined { seq.fse = fseDecoderPool.Get().(*fseDecoder) } - err := seq.fse.readNCount(&br, uint16(maxTableSymbol[i])) + err := seq.fse.readNCount(&br, tableIndex(i)) if err != nil { println("Read table error:", err) return err } - err = seq.fse.transform(symbolTableX[i]) - if err != nil { - println("Transform table error:", err) - return err - } if debugDecoder { println("Read table ok", "symbolLen:", seq.fse.symbolLen) } @@ -673,38 +683,41 @@ func (b *blockDec) decodeSequences(hist *history) error { return err } +// executeSequences executes the block's sequences straight into the history +// ring, as decodeCompressed does, and copies the result to b.dst for output. func (b *blockDec) executeSequences(hist *history) error { - hbytes := hist.b - if len(hbytes) > hist.windowSize { - hbytes = hbytes[len(hbytes)-hist.windowSize:] - } + hist.ensureBlockRing() + start := len(hist.b) hist.decoders.windowSize = hist.windowSize - hist.decoders.out = b.dst[:0] - err := hist.decoders.execute(b.sequence, hbytes) + hist.decoders.out = hist.b + err := hist.decoders.execute(b.sequence, hist.ext) if err != nil { return err } - return b.updateHistory(hist) + return b.updateHistory(hist, start) } -func (b *blockDec) updateHistory(hist *history) error { +// updateHistory copies the block's output, which starts at out[start], to +// b.dst and keeps it as history. +func (b *blockDec) updateHistory(hist *history, start int) error { if len(b.data) > maxCompressedBlockSize { return fmt.Errorf("compressed block size too large (%d)", len(b.data)) } - // Set output and release references. - b.dst = hist.decoders.out + out := hist.decoders.out + b.dst = append(b.dst[:0], out[start:]...) hist.recentOffsets = hist.decoders.prevOffset if b.Last { // if last block we don't care about history. - println("Last block, no history returned") + if debugDecoder { + println("Last block, no history returned") + } hist.b = hist.b[:0] return nil - } else { - hist.append(b.dst) - if debugDecoder { - println("Finished block with ", len(b.sequence), "sequences. Added", len(b.dst), "to history, now length", len(hist.b)) - } + } + hist.b = out + if debugDecoder { + println("Finished block with ", len(b.sequence), "sequences. Added", len(b.dst), "to history, now length", len(hist.b)) } hist.decoders.out, hist.decoders.literals = nil, nil diff --git a/vendor/github.com/klauspost/compress/zstd/blockenc.go b/vendor/github.com/klauspost/compress/zstd/blockenc.go index 0e33aea442..697e91f081 100644 --- a/vendor/github.com/klauspost/compress/zstd/blockenc.go +++ b/vendor/github.com/klauspost/compress/zstd/blockenc.go @@ -628,38 +628,6 @@ func (b *blockEnc) encode(org []byte, raw, rawAllLits bool) error { return err } - // Choose the best compression mode for each type. - // Will evaluate the new vs predefined and previous. - chooseComp := func(cur, prev, preDef *fseEncoder) (*fseEncoder, seqCompMode) { - // See if predefined/previous is better - hist := cur.count[:cur.symbolLen] - nSize := cur.approxSize(hist) + cur.maxHeaderSize() - predefSize := preDef.approxSize(hist) - prevSize := prev.approxSize(hist) - - // Add a small penalty for new encoders. - // Don't bother with extremely small (<2 byte gains). - nSize = nSize + (nSize+2*8*16)>>4 - switch { - case predefSize <= prevSize && predefSize <= nSize || forcePreDef: - if debugEncoder { - println("Using predefined", predefSize>>3, "<=", nSize>>3) - } - return preDef, compModePredefined - case prevSize <= nSize: - if debugEncoder { - println("Using previous", prevSize>>3, "<=", nSize>>3) - } - return prev, compModeRepeat - default: - if debugEncoder { - println("Using new, predef", predefSize>>3, ". previous:", prevSize>>3, ">", nSize>>3, "header max:", cur.maxHeaderSize()>>3, "bytes") - println("tl:", cur.actualTableLog, "symbolLen:", cur.symbolLen, "norm:", cur.norm[:cur.symbolLen], "hist", cur.count[:cur.symbolLen]) - } - return cur, compModeFSE - } - } - // Write compression mode var mode uint8 if llEnc.useRLE { @@ -670,7 +638,7 @@ func (b *blockEnc) encode(org []byte, raw, rawAllLits bool) error { } } else { var m seqCompMode - llEnc, m = chooseComp(llEnc, b.coders.llPrev, &fsePredefEnc[tableLiteralLengths]) + llEnc, m = b.chooseComp(llEnc, b.coders.llPrev, &fsePredefEnc[tableLiteralLengths]) mode |= uint8(m) << 6 } if ofEnc.useRLE { @@ -681,7 +649,7 @@ func (b *blockEnc) encode(org []byte, raw, rawAllLits bool) error { } } else { var m seqCompMode - ofEnc, m = chooseComp(ofEnc, b.coders.ofPrev, &fsePredefEnc[tableOffsets]) + ofEnc, m = b.chooseComp(ofEnc, b.coders.ofPrev, &fsePredefEnc[tableOffsets]) mode |= uint8(m) << 4 } @@ -693,7 +661,7 @@ func (b *blockEnc) encode(org []byte, raw, rawAllLits bool) error { } } else { var m seqCompMode - mlEnc, m = chooseComp(mlEnc, b.coders.mlPrev, &fsePredefEnc[tableMatchLengths]) + mlEnc, m = b.chooseComp(mlEnc, b.coders.mlPrev, &fsePredefEnc[tableMatchLengths]) mode |= uint8(m) << 2 } b.output = append(b.output, mode) @@ -753,54 +721,57 @@ func (b *blockEnc) encode(org []byte, raw, rawAllLits bool) error { println("Encoded seq", seq, s, "codes:", s.llCode, s.mlCode, s.ofCode, "states:", ll.state, ml.state, of.state, "bits:", llB, mlB, ofB) } seq-- - // Store sequences in reverse... + wr.flush32() + + // Store sequences in reverse, with the bit writer and states in locals. + // flush stores 8 bytes and advances by whole bytes, so each chunk + // checks room for seqMaxBytes per sequence plus the store's slack. + const seqMaxBytes = 12 // 26 state bits + 63 extra bits. + w := bitState{c: wr.bitContainer, nb: uint(wr.nBits), pos: len(wr.out)} + buf := wr.out[:cap(wr.out)] + ofState, mlState, llState := of.state, ml.state, ll.state + ofTab, mlTab, llTab := of.stateTable, ml.stateTable, ll.stateTable + seqs := b.sequences for seq >= 0 { - s = b.sequences[seq] - - ofB := ofTT[s.ofCode] - wr.flush32() // tablelog max is below 8 for each, so it will fill max 24 bits. - //of.encode(ofB) - nbBitsOut := (uint32(of.state) + ofB.deltaNbBits) >> 16 - dstState := int32(of.state>>(nbBitsOut&15)) + int32(ofB.deltaFindState) - wr.addBits16NC(of.state, uint8(nbBitsOut)) - of.state = of.stateTable[dstState] - - // Accumulate extra bits. - outBits := ofB.outBits & 31 - extraBits := uint64(s.offset & bitMask32[outBits]) - extraBitsN := outBits - - mlB := mlTT[s.mlCode] - //ml.encode(mlB) - nbBitsOut = (uint32(ml.state) + mlB.deltaNbBits) >> 16 - dstState = int32(ml.state>>(nbBitsOut&15)) + int32(mlB.deltaFindState) - wr.addBits16NC(ml.state, uint8(nbBitsOut)) - ml.state = ml.stateTable[dstState] - - outBits = mlB.outBits & 31 - extraBits = extraBits<> 16 - dstState = int32(ll.state>>(nbBitsOut&15)) + int32(llB.deltaFindState) - wr.addBits16NC(ll.state, uint8(nbBitsOut)) - ll.state = ll.stateTable[dstState] - - outBits = llB.outBits & 31 - extraBits = extraBits< stop; seq-- { + s := seqs[seq] + ofB, mlB, llB := ofTT[s.ofCode], mlTT[s.mlCode], llTT[s.llCode] - seq-- + // w.nb <= 37 here, and the states add at most 26 bits. + w, ofState = w.addState(ofState, ofTab, ofB) + w, mlState = w.addState(mlState, mlTab, mlB) + w, llState = w.addState(llState, llTab, llB) + + ofBits, mlBits, llBits := ofB.outBits&31, mlB.outBits&31, llB.outBits&31 + extra := uint64(s.offset&bitMask32[ofBits])<<(mlBits+llBits) | + uint64(s.matchLen&bitMask32[mlBits])< 63 { + w = w.flush(buf) + if extraN > 56 { + // Only offsets over 24 bits get here. + low := uint(mlBits + llBits) + w = w.add(extra&(1<>= low + extraN -= low + } + } + w = w.add(extra, extraN).flush(buf) + + if debugSequences { + println("Encoded seq", seq, s) + } + } } + wr.bitContainer, wr.nBits, wr.out = w.c, uint8(w.nb), buf[:w.pos] + of.state, ml.state, ll.state = ofState, mlState, llState ml.flush(mlEnc.actualTableLog) of.flush(ofEnc.actualTableLog) ll.flush(llEnc.actualTableLog) @@ -826,6 +797,54 @@ func (b *blockEnc) encode(org []byte, raw, rawAllLits bool) error { return nil } +// chooseComp chooses the cheapest of the new, predefined and previous table. +func (b *blockEnc) chooseComp(cur, prev, preDef *fseEncoder) (*fseEncoder, seqCompMode) { + hist := cur.count[:cur.symbolLen] + nSize := cur.approxSize(hist) + cur.maxHeaderSize() + predefSize := preDef.approxSize(hist) + prevSize := prev.approxSize(hist) + + // Add a small penalty for new encoders. + // Don't bother with extremely small (<2 byte gains). + nSize = nSize + (nSize+2*8*16)>>4 + // The previous table may come from a dictionary and fit poorly, + // so it is compared against the exact header size, without penalty. + nSizePrev := nSize + if prevSize != math.MaxUint32 { + // Written to spare capacity; the real header goes there later. + if hdr, err := cur.writeCount(b.output[len(b.output):]); err == nil { + nSizePrev = cur.approxSize(hist) + uint32(len(hdr))*8 + } + } + // Each stream ends with a flush of tableLog bits. + nSize += uint32(cur.actualTableLog) + nSizePrev += uint32(cur.actualTableLog) + if predefSize != math.MaxUint32 { + predefSize += uint32(preDef.actualTableLog) + } + if prevSize != math.MaxUint32 { + prevSize += uint32(prev.actualTableLog) + } + switch { + case predefSize <= prevSize && predefSize <= nSize || forcePreDef: + if debugEncoder { + println("Using predefined", predefSize>>3, "<=", nSize>>3) + } + return preDef, compModePredefined + case prevSize <= nSizePrev: + if debugEncoder { + println("Using previous", prevSize>>3, "<=", nSizePrev>>3) + } + return prev, compModeRepeat + default: + if debugEncoder { + println("Using new, predef", predefSize>>3, ". previous:", prevSize>>3, ">", nSize>>3, "header max:", cur.maxHeaderSize()>>3, "bytes") + println("tl:", cur.actualTableLog, "symbolLen:", cur.symbolLen, "norm:", cur.norm[:cur.symbolLen], "hist", cur.count[:cur.symbolLen]) + } + return cur, compModeFSE + } +} + var errIncompressible = errors.New("incompressible") func (b *blockEnc) genCodes() { diff --git a/vendor/github.com/klauspost/compress/zstd/decoder.go b/vendor/github.com/klauspost/compress/zstd/decoder.go index 90ef3bb03f..3aec681d6a 100644 --- a/vendor/github.com/klauspost/compress/zstd/decoder.go +++ b/vendor/github.com/klauspost/compress/zstd/decoder.go @@ -546,7 +546,7 @@ func (d *Decoder) nextBlockSync() (ok bool) { if d.current.err != nil { return false } - d.frame.history.ensureBlock() + d.frame.history.ensureBlockRing() if debugDecoder { println("History trimmed:", len(d.frame.history.b), "decoded already:", d.syncStream.decodedFrame) } @@ -751,7 +751,8 @@ func (d *Decoder) startStreamDecoder(ctx context.Context, r io.Reader, output ch wg.Add(1) // Async 3: Execute sequences... - frameHistCache := d.frame.history.b + // Reuse the history ring across streams. + frameRing := d.frame.history.ring go func() { var hist history var decodedFrame uint64 @@ -770,20 +771,12 @@ func (d *Decoder) startStreamDecoder(ctx context.Context, r io.Reader, output ch } hist.reset() hist.windowSize = block.async.newHist.windowSize - hist.allocFrameBuffer = block.async.newHist.allocFrameBuffer if block.async.newHist.dict != nil { hist.setDict(block.async.newHist.dict) } - - if cap(hist.b) < hist.allocFrameBuffer { - if cap(frameHistCache) >= hist.allocFrameBuffer { - hist.b = frameHistCache - } else { - hist.b = make([]byte, 0, hist.allocFrameBuffer) - println("Alloc history sized", hist.allocFrameBuffer) - } + if hist.ring == nil { + hist.ring = frameRing } - hist.b = hist.b[:0] fcs = block.async.fcs decodedFrame = 0 } @@ -806,13 +799,15 @@ func (d *Decoder) startStreamDecoder(ctx context.Context, r io.Reader, output ch for i := range block.dst { block.dst[i] = v } - hist.append(block.dst) + hist.ensureBlockRing() + hist.appendKeep(block.dst) do.b = block.dst case blockTypeRaw: if debugDecoder { println("add raw block length:", len(block.data)) } - hist.append(block.data) + hist.ensureBlockRing() + hist.appendKeep(block.data) do.b = block.data case blockTypeCompressed: if debugDecoder { @@ -845,7 +840,7 @@ func (d *Decoder) startStreamDecoder(ctx context.Context, r io.Reader, output ch output <- do } close(output) - frameHistCache = hist.b + frameRing = hist.ring wg.Done() if debugDecoder { println("decoder goroutines finished") @@ -924,9 +919,6 @@ decodeStream: err := frame.next(dec) if !historySent { h := frame.history - if debugDecoder { - println("Alloc History:", h.allocFrameBuffer) - } hist.reset() if h.dict != nil { hist.setDict(h.dict) @@ -973,7 +965,7 @@ decodeStream: close(seqDecode) wg.Wait() hist.reset() - d.frame.history.b = frameHistCache + d.frame.history.ring = frameRing } func (d *Decoder) setDict(frame *frameDec) (err error) { diff --git a/vendor/github.com/klauspost/compress/zstd/dict.go b/vendor/github.com/klauspost/compress/zstd/dict.go index 67f40e7914..7fbdedfff1 100644 --- a/vendor/github.com/klauspost/compress/zstd/dict.go +++ b/vendor/github.com/klauspost/compress/zstd/dict.go @@ -17,6 +17,7 @@ type dict struct { litEnc *huff0.Scratch llDec, ofDec, mlDec sequenceDec + llEnc, ofEnc, mlEnc *fseEncoder offsets [3]int content []byte } @@ -99,17 +100,12 @@ func loadDict(b []byte) (*dict, error) { off: 0, } readDec := func(i tableIndex, dec *fseDecoder) error { - if err := dec.readNCount(&br, uint16(maxTableSymbol[i])); err != nil { + if err := dec.readNCount(&br, i); err != nil { return err } if br.overread() { return io.ErrUnexpectedEOF } - err = dec.transform(symbolTableX[i]) - if err != nil { - println("Transform table error:", err) - return err - } if debugDecoder || debugEncoder { println("Read table ok", "symbolLen:", dec.symbolLen) } @@ -145,6 +141,9 @@ func loadDict(b []byte) (*dict, error) { if d.offsets[0] > len(d.content) || d.offsets[1] > len(d.content) || d.offsets[2] > len(d.content) { return nil, fmt.Errorf("initial offset bigger than dictionary content size %d, offsets: %v", len(d.content), d.offsets) } + d.llEnc = newDictSeqEncoder(d.llDec.fse, tableLiteralLengths) + d.ofEnc = newDictSeqEncoder(d.ofDec.fse, tableOffsets) + d.mlEnc = newDictSeqEncoder(d.mlDec.fse, tableMatchLengths) return &d, nil } @@ -219,6 +218,9 @@ func BuildDict(o BuildDictOptions) ([]byte, error) { if len(contents) == 0 { return nil, errors.New("no content provided") } + if o.Level < speedNotSet || o.Level >= speedLast { + return nil, fmt.Errorf("unknown encoder level %d", o.Level) + } d := dict{ id: o.ID, litEnc: nil, diff --git a/vendor/github.com/klauspost/compress/zstd/enc_base.go b/vendor/github.com/klauspost/compress/zstd/enc_base.go index c4fea575d6..40505ce65d 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_base.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_base.go @@ -194,6 +194,7 @@ func (e *fastBase) resetBase(d *dict, singleBlock bool) { } // Transfer litenc. e.blk.dictLitEnc = d.litEnc + e.blk.coders.seedPrevFromDict(d) e.hist = append(e.hist, d.content...) } } diff --git a/vendor/github.com/klauspost/compress/zstd/enc_better.go b/vendor/github.com/klauspost/compress/zstd/enc_better.go index 523d57f3ad..42b46a38c4 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_better.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_better.go @@ -26,6 +26,16 @@ const ( betterShortTableShardSize = betterShortTableSize / betterShortTableShardCnt // Size of an individual shard ) +// betterHashL is hashLen(u, betterLongTableBits, betterLongLen) with prime = prime8bytes. +func betterHashL(u, prime uint64) uint32 { + return uint32((u * prime) >> (64 - betterLongTableBits)) +} + +// betterHashS is hashLen(u, betterShortTableBits, betterShortLen) with prime = prime5bytes. +func betterHashS(u, prime uint64) uint32 { + return uint32(((u << (64 - 40)) * prime) >> (64 - betterShortTableBits)) +} + type prevEntry struct { offset int32 prev int32 @@ -126,6 +136,7 @@ func (e *betterFastEncoder) Encode(blk *blockEnc, src []byte) { // Override src src = e.hist sLimit := int32(len(src)) - inputMargin + primeL, primeS := hashPrimes8and5() // stepSize is the number of bytes to skip on every main loop iteration. // It should be >= 1. const stepSize = 1 @@ -163,8 +174,8 @@ encodeLoop: panic("offset0 was 0") } - nextHashL := hashLen(cv, betterLongTableBits, betterLongLen) - nextHashS := hashLen(cv, betterShortTableBits, betterShortLen) + nextHashL := betterHashL(cv, primeL) + nextHashS := betterHashS(cv, primeS) candidateL := e.longTable[nextHashL] candidateS := e.table[nextHashS] @@ -218,14 +229,13 @@ encodeLoop: break encodeLoop } // Index skipped... - for index0 < s-1 { + for end := s - 1; index0 < end; index0 += 2 { cv0 := load6432(src, index0) cv1 := cv0 >> 8 - h0 := hashLen(cv0, betterLongTableBits, betterLongLen) + h0 := betterHashL(cv0, primeL) off := index0 + e.cur + e.table[betterHashS(cv1, primeS)] = tableEntry{offset: off + 1, val: uint32(cv1)} e.longTable[h0] = prevEntry{offset: off, prev: e.longTable[h0].offset} - e.table[hashLen(cv1, betterShortTableBits, betterShortLen)] = tableEntry{offset: off + 1, val: uint32(cv1)} - index0 += 2 } cv = load6432(src, s) continue @@ -275,14 +285,13 @@ encodeLoop: } // Index skipped... - for index0 < s-1 { + for end := s - 1; index0 < end; index0 += 2 { cv0 := load6432(src, index0) cv1 := cv0 >> 8 - h0 := hashLen(cv0, betterLongTableBits, betterLongLen) + h0 := betterHashL(cv0, primeL) off := index0 + e.cur + e.table[betterHashS(cv1, primeS)] = tableEntry{offset: off + 1, val: uint32(cv1)} e.longTable[h0] = prevEntry{offset: off, prev: e.longTable[h0].offset} - e.table[hashLen(cv1, betterShortTableBits, betterShortLen)] = tableEntry{offset: off + 1, val: uint32(cv1)} - index0 += 2 } cv = load6432(src, s) // Swap offsets @@ -356,7 +365,7 @@ encodeLoop: // See if we can find a long match at s+1 const checkAt = 1 cv := load6432(src, s+checkAt) - nextHashL = hashLen(cv, betterLongTableBits, betterLongLen) + nextHashL = betterHashL(cv, primeL) candidateL = e.longTable[nextHashL] coffsetL = candidateL.offset - e.cur @@ -423,7 +432,7 @@ encodeLoop: // and still picked up as part of the match if they do. const skipBeginning = 3 - nextHashL := hashLen(load6432(src, s+matched), betterLongTableBits, betterLongLen) + nextHashL := betterHashL(load6432(src, s+matched), primeL) s2 := s + skipBeginning cv := load3232(src, s2) candidateL := e.longTable[nextHashL] @@ -501,13 +510,12 @@ encodeLoop: // Index match start+1 (long) -> s - 1 off := index0 + e.cur - for index0 < s-1 { + for end := s - 1; index0 < end; index0 += 2 { cv0 := load6432(src, index0) cv1 := cv0 >> 8 - h0 := hashLen(cv0, betterLongTableBits, betterLongLen) + e.table[betterHashS(cv1, primeS)] = tableEntry{offset: off + 1, val: uint32(cv1)} + h0 := betterHashL(cv0, primeL) e.longTable[h0] = prevEntry{offset: off, prev: e.longTable[h0].offset} - e.table[hashLen(cv1, betterShortTableBits, betterShortLen)] = tableEntry{offset: off + 1, val: uint32(cv1)} - index0 += 2 off += 2 } @@ -525,8 +533,8 @@ encodeLoop: } // Store this, since we have it. - nextHashL := hashLen(cv, betterLongTableBits, betterLongLen) - nextHashS := hashLen(cv, betterShortTableBits, betterShortLen) + nextHashL := betterHashL(cv, primeL) + nextHashS := betterHashS(cv, primeS) // We have at least 4 byte match. // No need to check backwards. We come straight from a match @@ -644,6 +652,7 @@ func (e *betterFastEncoderDict) Encode(blk *blockEnc, src []byte) { // Override src src = e.hist sLimit := int32(len(src)) - inputMargin + primeL, primeS := hashPrimes8and5() // stepSize is the number of bytes to skip on every main loop iteration. // It should be >= 1. const stepSize = 1 @@ -681,8 +690,8 @@ encodeLoop: panic("offset0 was 0") } - nextHashL := hashLen(cv, betterLongTableBits, betterLongLen) - nextHashS := hashLen(cv, betterShortTableBits, betterShortLen) + nextHashL := betterHashL(cv, primeL) + nextHashS := betterHashS(cv, primeS) candidateL := e.longTable[nextHashL] candidateS := e.table[nextHashS] @@ -737,17 +746,16 @@ encodeLoop: break encodeLoop } // Index skipped... - for index0 < s-1 { + for end := s - 1; index0 < end; index0 += 2 { cv0 := load6432(src, index0) cv1 := cv0 >> 8 - h0 := hashLen(cv0, betterLongTableBits, betterLongLen) + h0 := betterHashL(cv0, primeL) off := index0 + e.cur - e.longTable[h0] = prevEntry{offset: off, prev: e.longTable[h0].offset} - e.markLongShardDirty(h0) - h1 := hashLen(cv1, betterShortTableBits, betterShortLen) + h1 := betterHashS(cv1, primeS) e.table[h1] = tableEntry{offset: off + 1, val: uint32(cv1)} e.markShortShardDirty(h1) - index0 += 2 + e.longTable[h0] = prevEntry{offset: off, prev: e.longTable[h0].offset} + e.markLongShardDirty(h0) } cv = load6432(src, s) continue @@ -797,17 +805,16 @@ encodeLoop: } // Index skipped... - for index0 < s-1 { + for end := s - 1; index0 < end; index0 += 2 { cv0 := load6432(src, index0) cv1 := cv0 >> 8 - h0 := hashLen(cv0, betterLongTableBits, betterLongLen) + h0 := betterHashL(cv0, primeL) off := index0 + e.cur - e.longTable[h0] = prevEntry{offset: off, prev: e.longTable[h0].offset} - e.markLongShardDirty(h0) - h1 := hashLen(cv1, betterShortTableBits, betterShortLen) + h1 := betterHashS(cv1, primeS) e.table[h1] = tableEntry{offset: off + 1, val: uint32(cv1)} e.markShortShardDirty(h1) - index0 += 2 + e.longTable[h0] = prevEntry{offset: off, prev: e.longTable[h0].offset} + e.markLongShardDirty(h0) } cv = load6432(src, s) // Swap offsets @@ -881,7 +888,7 @@ encodeLoop: // See if we can find a long match at s+1 const checkAt = 1 cv := load6432(src, s+checkAt) - nextHashL = hashLen(cv, betterLongTableBits, betterLongLen) + nextHashL = betterHashL(cv, primeL) candidateL = e.longTable[nextHashL] coffsetL = candidateL.offset - e.cur @@ -942,7 +949,7 @@ encodeLoop: } // Try to find a better match by searching for a long match at the end of the current best match if s+matched < sLimit { - nextHashL := hashLen(load6432(src, s+matched), betterLongTableBits, betterLongLen) + nextHashL := betterHashL(load6432(src, s+matched), primeL) cv := load3232(src, s) candidateL := e.longTable[nextHashL] coffsetL := candidateL.offset - e.cur - matched @@ -1017,16 +1024,15 @@ encodeLoop: // Index match start+1 (long) -> s - 1 off := index0 + e.cur - for index0 < s-1 { + for end := s - 1; index0 < end; index0 += 2 { cv0 := load6432(src, index0) cv1 := cv0 >> 8 - h0 := hashLen(cv0, betterLongTableBits, betterLongLen) - e.longTable[h0] = prevEntry{offset: off, prev: e.longTable[h0].offset} - e.markLongShardDirty(h0) - h1 := hashLen(cv1, betterShortTableBits, betterShortLen) + h1 := betterHashS(cv1, primeS) e.table[h1] = tableEntry{offset: off + 1, val: uint32(cv1)} e.markShortShardDirty(h1) - index0 += 2 + h0 := betterHashL(cv0, primeL) + e.longTable[h0] = prevEntry{offset: off, prev: e.longTable[h0].offset} + e.markLongShardDirty(h0) off += 2 } @@ -1044,8 +1050,8 @@ encodeLoop: } // Store this, since we have it. - nextHashL := hashLen(cv, betterLongTableBits, betterLongLen) - nextHashS := hashLen(cv, betterShortTableBits, betterShortLen) + nextHashL := betterHashL(cv, primeL) + nextHashS := betterHashS(cv, primeS) // We have at least 4 byte match. // No need to check backwards. We come straight from a match diff --git a/vendor/github.com/klauspost/compress/zstd/enc_dfast.go b/vendor/github.com/klauspost/compress/zstd/enc_dfast.go index 712ba7ab58..aae59862aa 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_dfast.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_dfast.go @@ -22,6 +22,16 @@ const ( ) +// dFastHashL is hashLen(u, dFastLongTableBits, dFastLongLen) with prime = prime8bytes. +func dFastHashL(u, prime uint64) uint32 { + return uint32((u * prime) >> (64 - dFastLongTableBits)) +} + +// dFastHashS is hashLen(u, dFastShortTableBits, dFastShortLen) with prime = prime5bytes. +func dFastHashS(u, prime uint64) uint32 { + return uint32(((u << (64 - 40)) * prime) >> (64 - dFastShortTableBits)) +} + type doubleFastEncoder struct { fastEncoder longTable [dFastLongTableSize]tableEntry @@ -87,6 +97,7 @@ func (e *doubleFastEncoder) Encode(blk *blockEnc, src []byte) { // Override src src = e.hist sLimit := int32(len(src)) - inputMargin + primeL, primeS := hashPrimes8and5() // stepSize is the number of bytes to skip on every main loop iteration. // It should be >= 1. const stepSize = 1 @@ -123,8 +134,8 @@ encodeLoop: panic("offset0 was 0") } - nextHashL := hashLen(cv, dFastLongTableBits, dFastLongLen) - nextHashS := hashLen(cv, dFastShortTableBits, dFastShortLen) + nextHashL := dFastHashL(cv, primeL) + nextHashS := dFastHashS(cv, primeS) candidateL := e.longTable[nextHashL] candidateS := e.table[nextHashS] @@ -204,7 +215,7 @@ encodeLoop: // See if we can find a long match at s+1 const checkAt = 1 cv := load6432(src, s+checkAt) - nextHashL = hashLen(cv, dFastLongTableBits, dFastLongLen) + nextHashL = dFastHashL(cv, primeL) candidateL = e.longTable[nextHashL] coffsetL = s - (candidateL.offset - e.cur) + checkAt @@ -297,16 +308,16 @@ encodeLoop: cv1 := load6432(src, index1) te0 := tableEntry{offset: index0 + e.cur, val: uint32(cv0)} te1 := tableEntry{offset: index1 + e.cur, val: uint32(cv1)} - e.longTable[hashLen(cv0, dFastLongTableBits, dFastLongLen)] = te0 - e.longTable[hashLen(cv1, dFastLongTableBits, dFastLongLen)] = te1 + e.longTable[dFastHashL(cv0, primeL)] = te0 + e.longTable[dFastHashL(cv1, primeL)] = te1 cv0 >>= 8 cv1 >>= 8 te0.offset++ te1.offset++ te0.val = uint32(cv0) te1.val = uint32(cv1) - e.table[hashLen(cv0, dFastShortTableBits, dFastShortLen)] = te0 - e.table[hashLen(cv1, dFastShortTableBits, dFastShortLen)] = te1 + e.table[dFastHashS(cv0, primeS)] = te0 + e.table[dFastHashS(cv1, primeS)] = te1 cv = load6432(src, s) @@ -323,8 +334,8 @@ encodeLoop: } // Store this, since we have it. - nextHashS := hashLen(cv, dFastShortTableBits, dFastShortLen) - nextHashL := hashLen(cv, dFastLongTableBits, dFastLongLen) + nextHashS := dFastHashS(cv, primeS) + nextHashL := dFastHashL(cv, primeL) // We have at least 4 byte match. // No need to check backwards. We come straight from a match @@ -399,6 +410,7 @@ func (e *doubleFastEncoder) EncodeNoHist(blk *blockEnc, src []byte) { // Override src sLimit := int32(len(src)) - inputMargin + primeL, primeS := hashPrimes8and5() // stepSize is the number of bytes to skip on every main loop iteration. // It should be >= 1. const stepSize = 1 @@ -429,8 +441,8 @@ encodeLoop: var t int32 for { - nextHashL := hashLen(cv, dFastLongTableBits, dFastLongLen) - nextHashS := hashLen(cv, dFastShortTableBits, dFastShortLen) + nextHashL := dFastHashL(cv, primeL) + nextHashS := dFastHashS(cv, primeS) candidateL := e.longTable[nextHashL] candidateS := e.table[nextHashS] @@ -511,7 +523,7 @@ encodeLoop: // See if we can find a long match at s+1 const checkAt = 1 cv := load6432(src, s+checkAt) - nextHashL = hashLen(cv, dFastLongTableBits, dFastLongLen) + nextHashL = dFastHashL(cv, primeL) candidateL = e.longTable[nextHashL] coffsetL = s - (candidateL.offset - e.cur) + checkAt @@ -601,16 +613,16 @@ encodeLoop: cv1 := load6432(src, index1) te0 := tableEntry{offset: index0 + e.cur, val: uint32(cv0)} te1 := tableEntry{offset: index1 + e.cur, val: uint32(cv1)} - e.longTable[hashLen(cv0, dFastLongTableBits, dFastLongLen)] = te0 - e.longTable[hashLen(cv1, dFastLongTableBits, dFastLongLen)] = te1 + e.longTable[dFastHashL(cv0, primeL)] = te0 + e.longTable[dFastHashL(cv1, primeL)] = te1 cv0 >>= 8 cv1 >>= 8 te0.offset++ te1.offset++ te0.val = uint32(cv0) te1.val = uint32(cv1) - e.table[hashLen(cv0, dFastShortTableBits, dFastShortLen)] = te0 - e.table[hashLen(cv1, dFastShortTableBits, dFastShortLen)] = te1 + e.table[dFastHashS(cv0, primeS)] = te0 + e.table[dFastHashS(cv1, primeS)] = te1 cv = load6432(src, s) @@ -627,8 +639,8 @@ encodeLoop: } // Store this, since we have it. - nextHashS := hashLen(cv1>>8, dFastShortTableBits, dFastShortLen) - nextHashL := hashLen(cv, dFastLongTableBits, dFastLongLen) + nextHashS := dFastHashS(cv1>>8, primeS) + nextHashL := dFastHashL(cv, primeL) // We have at least 4 byte match. // No need to check backwards. We come straight from a match @@ -733,6 +745,7 @@ func (e *doubleFastEncoderDict) Encode(blk *blockEnc, src []byte) { // Override src src = e.hist sLimit := int32(len(src)) - inputMargin + primeL, primeS := hashPrimes8and5() // stepSize is the number of bytes to skip on every main loop iteration. // It should be >= 1. const stepSize = 1 @@ -769,8 +782,8 @@ encodeLoop: panic("offset0 was 0") } - nextHashL := hashLen(cv, dFastLongTableBits, dFastLongLen) - nextHashS := hashLen(cv, dFastShortTableBits, dFastShortLen) + nextHashL := dFastHashL(cv, primeL) + nextHashS := dFastHashS(cv, primeS) candidateL := e.longTable[nextHashL] candidateS := e.table[nextHashS] @@ -852,7 +865,7 @@ encodeLoop: // See if we can find a long match at s+1 const checkAt = 1 cv := load6432(src, s+checkAt) - nextHashL = hashLen(cv, dFastLongTableBits, dFastLongLen) + nextHashL = dFastHashL(cv, primeL) candidateL = e.longTable[nextHashL] coffsetL = s - (candidateL.offset - e.cur) + checkAt @@ -946,8 +959,8 @@ encodeLoop: cv1 := load6432(src, index1) te0 := tableEntry{offset: index0 + e.cur, val: uint32(cv0)} te1 := tableEntry{offset: index1 + e.cur, val: uint32(cv1)} - longHash1 := hashLen(cv0, dFastLongTableBits, dFastLongLen) - longHash2 := hashLen(cv1, dFastLongTableBits, dFastLongLen) + longHash1 := dFastHashL(cv0, primeL) + longHash2 := dFastHashL(cv1, primeL) e.longTable[longHash1] = te0 e.longTable[longHash2] = te1 e.markLongShardDirty(longHash1) @@ -958,8 +971,8 @@ encodeLoop: te1.offset++ te0.val = uint32(cv0) te1.val = uint32(cv1) - hashVal1 := hashLen(cv0, dFastShortTableBits, dFastShortLen) - hashVal2 := hashLen(cv1, dFastShortTableBits, dFastShortLen) + hashVal1 := dFastHashS(cv0, primeS) + hashVal2 := dFastHashS(cv1, primeS) e.table[hashVal1] = te0 e.markShardDirty(hashVal1) e.table[hashVal2] = te1 @@ -980,8 +993,8 @@ encodeLoop: } // Store this, since we have it. - nextHashL := hashLen(cv, dFastLongTableBits, dFastLongLen) - nextHashS := hashLen(cv, dFastShortTableBits, dFastShortLen) + nextHashL := dFastHashL(cv, primeL) + nextHashS := dFastHashS(cv, primeS) // We have at least 4 byte match. // No need to check backwards. We come straight from a match diff --git a/vendor/github.com/klauspost/compress/zstd/framedec.go b/vendor/github.com/klauspost/compress/zstd/framedec.go index 81d8b9c922..fb836c412e 100644 --- a/vendor/github.com/klauspost/compress/zstd/framedec.go +++ b/vendor/github.com/klauspost/compress/zstd/framedec.go @@ -258,18 +258,6 @@ func (d *frameDec) reset(br byteBuffer) error { return ErrWindowSizeTooSmall } d.history.windowSize = int(d.WindowSize) - if !d.o.lowMem || d.history.windowSize < maxBlockSize { - // Alloc 2x window size if not low-mem, or window size below 2MB. - d.history.allocFrameBuffer = d.history.windowSize * 2 - } else { - if d.o.lowMem { - // Alloc with 1MB extra. - d.history.allocFrameBuffer = d.history.windowSize + maxBlockSize/2 - } else { - // Alloc with 2MB extra. - d.history.allocFrameBuffer = d.history.windowSize + maxBlockSize - } - } if debugDecoder { println("Frame: Dict:", d.DictionaryID, "FrameContentSize:", d.FrameContentSize, "singleseg:", d.SingleSegment, "window:", d.WindowSize, "crc:", d.HasCheckSum) @@ -355,14 +343,12 @@ func (d *frameDec) runDecoder(dst []byte, dec *blockDec) ([]byte, error) { if debugDecoder { println("maxSyncLen:", d.history.decoders.maxSyncLen) } - if !d.o.limitToCap && uint64(cap(dst)) < d.history.decoders.maxSyncLen { - // Alloc for output - dst2 := make([]byte, len(dst), d.history.decoders.maxSyncLen+compressedBlockOverAlloc) - copy(dst2, dst) - dst = dst2 - } } var err error + // Hash each block while it is still in cache rather than the whole + // frame at the end. + hashCRC := d.HasCheckSum && !d.o.ignoreChecksum + crcDone := crcStart for { err = dec.reset(d.rawInput, d.WindowSize) if err != nil { @@ -375,6 +361,10 @@ func (d *frameDec) runDecoder(dst []byte, dec *blockDec) ([]byte, error) { if err != nil { break } + if hashCRC { + d.crc.Write(d.history.b[crcDone:]) + crcDone = len(d.history.b) + } if uint64(len(d.history.b)-crcStart) > d.o.maxDecodedSize { println("runDecoder: maxDecodedSize exceeded", uint64(len(d.history.b)-crcStart), ">", d.o.maxDecodedSize) err = ErrDecoderSizeExceeded @@ -405,7 +395,6 @@ func (d *frameDec) runDecoder(dst []byte, dec *blockDec) ([]byte, error) { if d.o.ignoreChecksum { err = d.consumeCRC() } else { - d.crc.Write(dst[crcStart:]) err = d.checkCRC() } } diff --git a/vendor/github.com/klauspost/compress/zstd/fse_decoder.go b/vendor/github.com/klauspost/compress/zstd/fse_decoder.go index 2f8860a722..12ac2b5497 100644 --- a/vendor/github.com/klauspost/compress/zstd/fse_decoder.go +++ b/vendor/github.com/klauspost/compress/zstd/fse_decoder.go @@ -43,13 +43,36 @@ type fseDecoder struct { preDefined bool } +// codeShare returns, in 1/256ths, the table's probability mass on symbols +// minCode and above: for a table read from the block, the share of its +// sequences with those codes. The predefined tables describe the format's +// defaults, not the block, and RLE tables have no distribution; both +// report -1. +func (s *fseDecoder) codeShare(minCode int) int { + if s == nil || s.preDefined || s.actualTableLog == 0 { + return -1 + } + n := 0 + for c := minCode; c < int(s.symbolLen); c++ { + switch v := int(s.norm[c]); { + case v > 0: + n += v + case v < 0: + n++ // -1 marks a probability below one slot; it occupies one + } + } + return n * 256 >> s.actualTableLog +} + // tableStep returns the next table index. func tableStep(tableSize uint32) uint32 { return (tableSize >> 1) + (tableSize >> 3) + 3 } -// readNCount will read the symbol distribution so decoding tables can be constructed. -func (s *fseDecoder) readNCount(b *byteReader, maxSymbol uint16) error { +// readNCount will read the symbol distribution for table t and build its +// decoding table. +func (s *fseDecoder) readNCount(b *byteReader, t tableIndex) error { + maxSymbol := uint16(maxTableSymbol[t]) var ( charnum uint16 previous0 bool @@ -167,7 +190,8 @@ func (s *fseDecoder) readNCount(b *byteReader, maxSymbol uint16) error { if s.symbolLen <= 1 { return fmt.Errorf("symbolLen (%d) too small", s.symbolLen) } - if s.symbolLen > maxSymbolValue+1 { + // A run of zero counts can step past maxSymbol, so check the total. + if s.symbolLen > maxSymbol+1 { return fmt.Errorf("symbolLen (%d) too big", s.symbolLen) } if remaining != 1 { @@ -180,7 +204,30 @@ func (s *fseDecoder) readNCount(b *byteReader, maxSymbol uint16) error { return fmt.Errorf("corruption detected (total %d != %d)", gotTotal, 1<> 3) - return s.buildDtable() + return s.buildDtable(&symbolExtX[t]) +} + +// symbolExt holds, per symbol, a decSymbol with only addBits and baseline set. +type symbolExt [maxMatchLengthSymbol + 1]decSymbol + +// buildDtable builds the decoding table, taking each state's addBits and +// baseline from its symbol's entry in ext. +func (s *fseDecoder) buildDtable(ext *symbolExt) error { + if int(s.symbolLen) > len(ext) { + return fmt.Errorf("symbolLen (%d) too big", s.symbolLen) + } + if err := s.fillDtable(ext); err != nil { + return err + } + // addBits never decreases with the symbol (checked in initPredefined), + // so the last symbol has the most. It may exceed the table's actual + // maximum only if the last symbol has no states, which just costs an + // extra bitreader fill. + s.maxBits = 0 + if s.symbolLen > 0 { + s.maxBits = ext[s.symbolLen-1].addBits() + } + return nil } func (s *fseDecoder) mustReadFrom(r io.Reader) { @@ -268,27 +315,6 @@ func (s *fseDecoder) setRLE(symbol decSymbol) { s.dt[0] = symbol } -// transform will transform the decoder table into a table usable for -// decoding without having to apply the transformation while decoding. -// The state will contain the base value and the number of bits to read. -func (s *fseDecoder) transform(t []baseOffset) error { - tableSize := uint16(1 << s.actualTableLog) - s.maxBits = 0 - for i, v := range s.dt[:tableSize] { - add := v.addBits() - if int(add) >= len(t) { - return fmt.Errorf("invalid decoding table entry %d, symbol %d >= max (%d)", i, v.addBits(), len(t)) - } - lu := t[add] - if lu.addBits > s.maxBits { - s.maxBits = lu.addBits - } - v.setExt(lu.addBits, lu.baseLine) - s.dt[i] = v - } - return nil -} - type fseState struct { dt []decSymbol state decSymbol diff --git a/vendor/github.com/klauspost/compress/zstd/fse_decoder_amd64.s b/vendor/github.com/klauspost/compress/zstd/fse_decoder_amd64.s index deeadc49eb..87f05503ce 100644 --- a/vendor/github.com/klauspost/compress/zstd/fse_decoder_amd64.s +++ b/vendor/github.com/klauspost/compress/zstd/fse_decoder_amd64.s @@ -1,126 +1,132 @@ // Code generated by command: go run gen_fse.go -out ../fse_decoder.s -arch amd64,arm64 -pkg=zstd. DO NOT EDIT. -//go:build !appengine && !noasm && gc && !noasm +//go:build !noasm && gc // func buildDtable_asm(s *fseDecoder, ctx *buildDtableAsmContext) int TEXT ·buildDtable_asm(SB), $0-24 MOVQ ctx+8(FP), CX - MOVQ s+0(FP), DI + MOVQ s+0(FP), R8 // Load values - MOVBQZX 4098(DI), DX + MOVBQZX 4098(R8), DX XORQ AX, AX BTSQ DX, AX MOVQ (CX), BX MOVQ 16(CX), SI - LEAQ -1(AX), R8 + MOVQ 24(CX), DI + LEAQ -1(AX), R9 MOVQ 8(CX), CX - MOVWQZX 4096(DI), DI + MOVWQZX 4096(R8), R8 // End load values // Init, lay down lowprob symbols - XORQ R9, R9 + XORQ R10, R10 JMP init_main_loop_condition init_main_loop: - MOVWQSX (CX)(R9*2), R10 - CMPW R10, $-1 + MOVWQSX (CX)(R10*2), R11 + CMPW R11, $-1 JNE do_not_update_high_threshold - MOVB R9, 1(SI)(R8*8) - DECQ R8 - MOVQ $0x0000000000000001, R10 + MOVB R10, 1(SI)(R9*8) + DECQ R9 + MOVQ $0x0000000000000001, R11 do_not_update_high_threshold: - MOVW R10, (BX)(R9*2) - INCQ R9 + MOVW R11, (BX)(R10*2) + INCQ R10 init_main_loop_condition: - CMPQ R9, DI + CMPQ R10, R8 JL init_main_loop // Spread symbols // Calculate table step - MOVQ AX, R9 - SHRQ $0x01, R9 MOVQ AX, R10 - SHRQ $0x03, R10 - LEAQ 3(R9)(R10*1), R9 + SHRQ $0x01, R10 + MOVQ AX, R11 + SHRQ $0x03, R11 + LEAQ 3(R10)(R11*1), R10 // Fill add bits values - LEAQ -1(AX), R10 - XORQ R11, R11 + LEAQ -1(AX), R11 XORQ R12, R12 + XORQ R13, R13 JMP spread_main_loop_condition spread_main_loop: - XORQ R13, R13 - MOVWQSX (CX)(R12*2), R14 + XORQ R14, R14 + MOVWQSX (CX)(R13*2), R15 JMP spread_inner_loop_condition spread_inner_loop: - MOVB R12, 1(SI)(R11*8) + MOVB R13, 1(SI)(R12*8) adjust_position: - ADDQ R9, R11 - ANDQ R10, R11 - CMPQ R11, R8 + ADDQ R10, R12 + ANDQ R11, R12 + CMPQ R12, R9 JG adjust_position - INCQ R13 + INCQ R14 spread_inner_loop_condition: - CMPQ R13, R14 + CMPQ R14, R15 JL spread_inner_loop - INCQ R12 + INCQ R13 spread_main_loop_condition: - CMPQ R12, DI + CMPQ R13, R8 JL spread_main_loop - TESTQ R11, R11 + TESTQ R12, R12 JZ spread_check_ok MOVQ ctx+8(FP), AX - MOVQ R11, 24(AX) + MOVQ R12, 32(AX) MOVQ $+1, ret+16(FP) RET spread_check_ok: // Build Decoding table - XORQ DI, DI + XORQ R8, R8 build_table_main_table: - MOVBQZX 1(SI)(DI*8), CX - MOVWQZX (BX)(CX*2), R8 - LEAQ 1(R8), R9 - MOVW R9, (BX)(CX*2) - MOVQ R8, R9 - BSRQ R9, R9 + MOVBQZX 1(SI)(R8*8), R9 + MOVWQZX (BX)(R9*2), R10 + LEAQ 1(R10), CX + MOVW CX, (BX)(R9*2) + MOVQ R10, R11 + BSRQ R11, R11 MOVQ DX, CX - SUBQ R9, CX - SHLQ CL, R8 - SUBQ AX, R8 - MOVB CL, (SI)(DI*8) - MOVW R8, 2(SI)(DI*8) - CMPQ R8, AX + SUBQ R11, CX + SHLQ CL, R10 + SUBQ AX, R10 + MOVQ (DI)(R9*8), R9 + ORQ CX, R9 + MOVQ R10, R11 + ANDQ $0x0000ffff, R11 + SHLQ $0x10, R11 + ORQ R11, R9 + MOVQ R9, (SI)(R8*8) + CMPQ R10, AX JLE build_table_check1_ok MOVQ ctx+8(FP), CX - MOVQ R8, 24(CX) - MOVQ AX, 32(CX) + MOVQ R10, 32(CX) + MOVQ AX, 40(CX) MOVQ $+2, ret+16(FP) RET build_table_check1_ok: TESTB CL, CL JNZ build_table_check2_ok - CMPW R8, DI + CMPW R10, R8 JNE build_table_check2_ok MOVQ ctx+8(FP), AX - MOVQ R8, 24(AX) - MOVQ DI, 32(AX) + MOVQ R10, 32(AX) + MOVQ R8, 40(AX) MOVQ $+3, ret+16(FP) RET build_table_check2_ok: - INCQ DI - CMPQ DI, AX + INCQ R8 + CMPQ R8, AX JL build_table_main_table MOVQ $+0, ret+16(FP) RET diff --git a/vendor/github.com/klauspost/compress/zstd/fse_decoder_arm64.s b/vendor/github.com/klauspost/compress/zstd/fse_decoder_arm64.s index 8ca913eaec..f83b76b320 100644 --- a/vendor/github.com/klauspost/compress/zstd/fse_decoder_arm64.s +++ b/vendor/github.com/klauspost/compress/zstd/fse_decoder_arm64.s @@ -1,122 +1,126 @@ // Code generated by command: go run gen_fse.go -out ../fse_decoder.s -arch amd64,arm64 -pkg=zstd. DO NOT EDIT. // EXPERIMENTAL arm64 output lowered from an amd64 avo program. -//go:build arm64 && (!appengine && !noasm && gc && !noasm) +//go:build arm64 && (!noasm && gc) // func buildDtable_asm(s *fseDecoder, ctx *buildDtableAsmContext) int TEXT ·buildDtable_asm(SB), $0-24 MOVD ctx+8(FP), R1 - MOVD s+0(FP), R6 + MOVD s+0(FP), R7 // Load values - MOVBU 4098(R6), R2 + MOVBU 4098(R7), R2 MOVD $0, R0 MOVD $1, R16 LSL R2, R16, R16 ORR R16, R0, R0 MOVD (R1), R3 MOVD 16(R1), R5 - SUB $1, R0, R7 + MOVD 24(R1), R6 + SUB $1, R0, R8 MOVD 8(R1), R1 - MOVHU 4096(R6), R6 + MOVHU 4096(R7), R7 // End load values // Init, lay down lowprob symbols - MOVD $0, R8 + MOVD $0, R9 JMP init_main_loop_condition init_main_loop: - MOVH (R1)(R8<<1), R9 + MOVH (R1)(R9<<1), R10 MOVD $-1, R16 AND $0xffff, R16, R16 - AND $0xffff, R9, R15 + AND $0xffff, R10, R15 CMP R16, R15 BNE do_not_update_high_threshold - ADD R7<<3, R5, R15 - MOVB R8, 1(R15) - SUB $1, R7, R7 - MOVD $0x0000000000000001, R9 + ADD R8<<3, R5, R15 + MOVB R9, 1(R15) + SUB $1, R8, R8 + MOVD $0x0000000000000001, R10 do_not_update_high_threshold: - MOVH R9, (R3)(R8<<1) - ADD $1, R8, R8 + MOVH R10, (R3)(R9<<1) + ADD $1, R9, R9 init_main_loop_condition: - CMP R6, R8 + CMP R7, R9 BLT init_main_loop // Spread symbols // Calculate table step - LSR $0x01, R0, R8 - LSR $0x03, R0, R9 - ADD R9, R8, R8 - ADD $3, R8, R8 + LSR $0x01, R0, R9 + LSR $0x03, R0, R10 + ADD R10, R9, R9 + ADD $3, R9, R9 // Fill add bits values - SUB $1, R0, R9 - MOVD $0, R10 + SUB $1, R0, R10 MOVD $0, R11 + MOVD $0, R12 JMP spread_main_loop_condition spread_main_loop: - MOVD $0, R12 - MOVH (R1)(R11<<1), R13 + MOVD $0, R13 + MOVH (R1)(R12<<1), R14 JMP spread_inner_loop_condition spread_inner_loop: - ADD R10<<3, R5, R15 - MOVB R11, 1(R15) + ADD R11<<3, R5, R15 + MOVB R12, 1(R15) adjust_position: - ADD R8, R10, R10 - AND R9, R10, R10 - CMP R7, R10 + ADD R9, R11, R11 + AND R10, R11, R11 + CMP R8, R11 BGT adjust_position - ADD $1, R12, R12 + ADD $1, R13, R13 spread_inner_loop_condition: - CMP R13, R12 + CMP R14, R13 BLT spread_inner_loop - ADD $1, R11, R11 + ADD $1, R12, R12 spread_main_loop_condition: - CMP R6, R11 + CMP R7, R12 BLT spread_main_loop - TST R10, R10 + TST R11, R11 BEQ spread_check_ok MOVD ctx+8(FP), R0 - MOVD R10, 24(R0) + MOVD R11, 32(R0) MOVD $+1, R16 MOVD R16, ret+16(FP) RET spread_check_ok: // Build Decoding table - MOVD $0, R6 + MOVD $0, R7 build_table_main_table: - ADD R6<<3, R5, R15 - MOVBU 1(R15), R1 - MOVHU (R3)(R1<<1), R7 - ADD $1, R7, R8 - MOVH R8, (R3)(R1<<1) - MOVD R7, R8 - CLZ R8, R16 - MOVD $63, R8 - SUB R16, R8, R8 + ADD R7<<3, R5, R15 + MOVBU 1(R15), R8 + MOVHU (R3)(R8<<1), R9 + ADD $1, R9, R1 + MOVH R1, (R3)(R8<<1) + MOVD R9, R10 + CLZ R10, R16 + MOVD $63, R10 + SUB R16, R10, R10 MOVD R2, R1 - SUB R8, R1, R1 - LSL R1, R7, R7 - SUB R0, R7, R7 - ADD R6<<3, R5, R15 - MOVB R1, (R15) - ADD R6<<3, R5, R15 - MOVH R7, 2(R15) - CMP R0, R7 + SUB R10, R1, R1 + LSL R1, R9, R9 + SUB R0, R9, R9 + MOVD (R6)(R8<<3), R8 + ORR R1, R8, R8 + MOVD R9, R10 + AND $0x0000ffff, R10, R10 + LSL $0x10, R10, R10 + ORR R10, R8, R8 + MOVD R8, (R5)(R7<<3) + CMP R0, R9 BLE build_table_check1_ok MOVD ctx+8(FP), R1 - MOVD R7, 24(R1) - MOVD R0, 32(R1) + MOVD R9, 32(R1) + MOVD R0, 40(R1) MOVD $+2, R16 MOVD R16, ret+16(FP) RET @@ -124,20 +128,20 @@ build_table_main_table: build_table_check1_ok: TST $0xff, R1 BNE build_table_check2_ok - AND $0xffff, R6, R16 - AND $0xffff, R7, R15 + AND $0xffff, R7, R16 + AND $0xffff, R9, R15 CMP R16, R15 BNE build_table_check2_ok MOVD ctx+8(FP), R0 - MOVD R7, 24(R0) - MOVD R6, 32(R0) + MOVD R9, 32(R0) + MOVD R7, 40(R0) MOVD $+3, R16 MOVD R16, ret+16(FP) RET build_table_check2_ok: - ADD $1, R6, R6 - CMP R0, R6 + ADD $1, R7, R7 + CMP R0, R7 BLT build_table_main_table MOVD $+0, R16 MOVD R16, ret+16(FP) diff --git a/vendor/github.com/klauspost/compress/zstd/fse_decoder_asm.go b/vendor/github.com/klauspost/compress/zstd/fse_decoder_asm.go index 4ffc7e3c9f..7394eefb42 100644 --- a/vendor/github.com/klauspost/compress/zstd/fse_decoder_asm.go +++ b/vendor/github.com/klauspost/compress/zstd/fse_decoder_asm.go @@ -1,4 +1,4 @@ -//go:build (amd64 || arm64) && !appengine && !noasm && gc +//go:build (amd64 || arm64) && !noasm && gc package zstd @@ -15,6 +15,7 @@ type buildDtableAsmContext struct { stateTable *uint16 norm *int16 dt *uint64 + ext *uint64 // outputs --- set by the procedure in the case of error; // for interpretation please see the error handling part below @@ -35,12 +36,13 @@ const ( errorNewStateNoBits = 3 ) -// buildDtable will build the decoding table. -func (s *fseDecoder) buildDtable() error { +// fillDtable fills the decoding table; see buildDtable. +func (s *fseDecoder) fillDtable(ext *symbolExt) error { ctx := buildDtableAsmContext{ stateTable: &s.stateTable[0], norm: &s.norm[0], dt: (*uint64)(&s.dt[0]), + ext: (*uint64)(&ext[0]), } code := buildDtable_asm(s, &ctx) diff --git a/vendor/github.com/klauspost/compress/zstd/fse_decoder_generic.go b/vendor/github.com/klauspost/compress/zstd/fse_decoder_generic.go index 38fd2ccb2a..f40353bb39 100644 --- a/vendor/github.com/klauspost/compress/zstd/fse_decoder_generic.go +++ b/vendor/github.com/klauspost/compress/zstd/fse_decoder_generic.go @@ -1,4 +1,4 @@ -//go:build (!amd64 && !arm64) || appengine || !gc || noasm +//go:build (!amd64 && !arm64) || !gc || noasm package zstd @@ -7,8 +7,8 @@ import ( "fmt" ) -// buildDtable will build the decoding table. -func (s *fseDecoder) buildDtable() error { +// fillDtable fills the decoding table; see buildDtable. +func (s *fseDecoder) fillDtable(ext *symbolExt) error { tableSize := uint32(1 << s.actualTableLog) highThreshold := tableSize - 1 symbolNext := s.stateTable[:256] @@ -56,7 +56,6 @@ func (s *fseDecoder) buildDtable() error { nextState := symbolNext[symbol] symbolNext[symbol] = nextState + 1 nBits := s.actualTableLog - byte(highBits(uint32(nextState))) - s.dt[u&maxTableMask].setNBits(nBits) newState := (nextState << nBits) - tableSize if newState > tableSize { return fmt.Errorf("newState (%d) outside table size (%d)", newState, tableSize) @@ -65,7 +64,7 @@ func (s *fseDecoder) buildDtable() error { // Seems weird that this is possible with nbits > 0. return fmt.Errorf("newState (%d) == oldState (%d) and no bits", newState, u) } - s.dt[u&maxTableMask].setNewState(newState) + s.dt[u&maxTableMask] = ext[symbol] | decSymbol(nBits) | decSymbol(newState)<<16 } } return nil diff --git a/vendor/github.com/klauspost/compress/zstd/fse_predefined.go b/vendor/github.com/klauspost/compress/zstd/fse_predefined.go index 474cb77d2b..7880b6fdfe 100644 --- a/vendor/github.com/klauspost/compress/zstd/fse_predefined.go +++ b/vendor/github.com/klauspost/compress/zstd/fse_predefined.go @@ -25,6 +25,10 @@ var ( // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#the-codes-for-literals-lengths-match-lengths-and-offsets symbolTableX [3][]baseOffset + // symbolExtX holds symbolTableX as decSymbols carrying only addBits and + // baseline, which buildDtable merges into each state. + symbolExtX [3]symbolExt + // maxTableSymbol is the biggest supported symbol for each table type // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#the-codes-for-literals-lengths-match-lengths-and-offsets maxTableSymbol = [3]uint8{tableLiteralLengths: maxLiteralLengthSymbol, tableOffsets: maxOffsetLengthSymbol, tableMatchLengths: maxMatchLengthSymbol} @@ -105,6 +109,15 @@ func initPredefined() { } fillBase(tmp[2:], 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30) symbolTableX[tableOffsets] = tmp + for i, t := range symbolTableX { + for j, lu := range t { + // fseDecoder.buildDtable takes maxBits from the last symbol. + if j > 0 && lu.addBits < t[j-1].addBits { + panic(fmt.Sprintf("table %v: addBits decreases at symbol %d", tableIndex(i), j)) + } + symbolExtX[i][j] = newDecSymbol(0, lu.addBits, 0, lu.baseLine) + } + } // Fill predefined tables and transform them. // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#default-distributions @@ -135,10 +148,7 @@ func initPredefined() { -1, -1, -1, -1, -1}) f.symbolLen = 53 } - if err := f.buildDtable(); err != nil { - panic(fmt.Errorf("building table %v: %v", tableIndex(i), err)) - } - if err := f.transform(symbolTableX[i]); err != nil { + if err := f.buildDtable(&symbolExtX[i]); err != nil { panic(fmt.Errorf("building table %v: %v", tableIndex(i), err)) } f.preDefined = true diff --git a/vendor/github.com/klauspost/compress/zstd/hash.go b/vendor/github.com/klauspost/compress/zstd/hash.go index 5d73c21ebd..2b606a3b35 100644 --- a/vendor/github.com/klauspost/compress/zstd/hash.go +++ b/vendor/github.com/klauspost/compress/zstd/hash.go @@ -4,6 +4,8 @@ package zstd +import "runtime" + const ( prime3bytes = 506832829 prime4bytes = 2654435761 @@ -13,6 +15,12 @@ const ( prime8bytes = 0xcf1bbcdcb7a56463 ) +// hashPrimesInRegs reports whether encoders should keep the 64-bit hash primes +// in registers rather than as constants. The compiler rebuilds a constant at +// every use: one instruction on amd64, but up to 4 on arm64, which also has +// the registers to spare. See hashPrimes8and5. +const hashPrimesInRegs = runtime.GOARCH == "arm64" + // hashLen returns a hash of the lowest mls bytes of with length output bits. // mls must be >=3 and <=8. Any other value will return hash for 4 bytes. // length should always be < 32. @@ -33,3 +41,17 @@ func hashLen(u uint64, length, mls uint8) uint32 { return (uint32(u) * prime4bytes) >> (32 - length) } } + +// hashPrimes85 is a var so arm64 keeps the primes in registers instead of +// rebuilding each 64-bit prime with up to 4 instructions per use. +var hashPrimes85 = [2]uint64{prime8bytes, prime5bytes} + +// hashPrimes8and5 returns prime8bytes and prime5bytes for encoders that hash +// 8 and 5 bytes. Where registers are too scarce to hold them, it returns the +// constants. +func hashPrimes8and5() (prime8, prime5 uint64) { + if hashPrimesInRegs { + return hashPrimes85[0], hashPrimes85[1] + } + return prime8bytes, prime5bytes +} diff --git a/vendor/github.com/klauspost/compress/zstd/history.go b/vendor/github.com/klauspost/compress/zstd/history.go index 09164856d2..76e4c30f8d 100644 --- a/vendor/github.com/klauspost/compress/zstd/history.go +++ b/vendor/github.com/klauspost/compress/zstd/history.go @@ -24,16 +24,26 @@ type history struct { // when checking for matches in history ignoreBuffer int - windowSize int - allocFrameBuffer int // needed? - error bool - dict *dict + windowSize int + error bool + dict *dict + + // ring backs b when streaming (see ensureBlockRing). + // ext holds the part of the previous lap of ring still in the window, + // which precedes b[0] in the stream; nil when b holds all history. + // The previous lap is ring[extStart:extEnd]. + ring []byte + ext []byte + extStart int + extEnd int } // reset will reset the history to initial state of a frame. // The history must already have been initialized to the desired size. func (h *history) reset() { h.b = h.b[:0] + h.ext = nil + h.extStart, h.extEnd = 0, 0 h.ignoreBuffer = 0 h.error = false h.recentOffsets = [3]int{1, 4, 8} @@ -67,47 +77,40 @@ func (h *history) setDict(dict *dict) { h.huffTree = dict.litEnc } -// append bytes to history. -// This function will make sure there is space for it, -// if the buffer has been allocated with enough extra space. -func (h *history) append(b []byte) { - if len(b) >= h.windowSize { - // Discard all history by simply overwriting - h.b = h.b[:h.windowSize] - copy(h.b, b[len(b)-h.windowSize:]) - return +// ensureBlockRing makes room for the next block without moving history, +// as the reference decoder does. b is a prefix of ring capped to one block +// of space. When a block no longer fits, the data written so far becomes +// ext and writing restarts at the front of ring. ext is trimmed so the +// block space ahead of b never overlaps it; with ring sized to window plus +// two blocks of space, b and ext together always hold a full window. +func (h *history) ensureBlockRing() { + // Room for the largest block plus the overrun of the fast copies. + ringBlockSpace := min(h.windowSize, maxCompressedBlockSize) + compressedBlockOverAlloc + size := h.windowSize + 2*ringBlockSpace + pos := len(h.b) + if cap(h.ring) < size || (pos > 0 && &h.b[0] != &h.ring[0]) { + // No ring yet, or b no longer lives in it: start one, keeping up + // to a window of what was decoded so far as the previous lap. + keep := min(pos, h.windowSize) + if cap(h.ring) < size { + h.ring = make([]byte, size) + } + h.ring = h.ring[:size] + copy(h.ring[size-keep:], h.b[pos-keep:]) + h.extStart, h.extEnd = size-keep, size + pos = 0 + } else if pos+ringBlockSpace > size { + // Wrap: everything written this lap becomes the previous lap. + h.extStart, h.extEnd = 0, pos + pos = 0 } - - // If there is space, append it. - if len(b) < cap(h.b)-len(h.b) { - h.b = append(h.b, b...) - return + h.ring = h.ring[:size] + end := pos + ringBlockSpace + h.b = h.ring[:pos:end] + h.ext = nil + if start := max(end, h.extStart); start < h.extEnd { + h.ext = h.ring[start:h.extEnd] } - - // Move data down so we only have window size left. - // We know we have less than window size in b at this point. - discard := len(b) + len(h.b) - h.windowSize - copy(h.b, h.b[discard:]) - h.b = h.b[:h.windowSize] - copy(h.b[h.windowSize-len(b):], b) -} - -// ensureBlock will ensure there is space for at least one block... -func (h *history) ensureBlock() { - if cap(h.b) < h.allocFrameBuffer { - h.b = make([]byte, 0, h.allocFrameBuffer) - return - } - - avail := cap(h.b) - len(h.b) - if avail >= h.windowSize || avail > maxCompressedBlockSize { - return - } - // Move data down so we only have window size left. - // We know we have less than window size in b at this point. - discard := len(h.b) - h.windowSize - copy(h.b, h.b[discard:]) - h.b = h.b[:h.windowSize] } // append bytes to history without ever discarding anything. diff --git a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_amd64.s b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_amd64.s index e4c688efd3..95d7980786 100644 --- a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_amd64.s +++ b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_amd64.s @@ -1,8 +1,4 @@ -//go:build !appengine && gc && !purego && !noasm -// +build !appengine -// +build gc -// +build !purego -// +build !noasm +//go:build gc && !purego && !noasm #include "textflag.h" diff --git a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_arm64.s b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_arm64.s index e88cc2fc91..a09d43bd9f 100644 --- a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_arm64.s +++ b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_arm64.s @@ -1,8 +1,4 @@ -//go:build !appengine && gc && !purego && !noasm -// +build !appengine -// +build gc -// +build !purego -// +build !noasm +//go:build gc && !purego && !noasm #include "textflag.h" diff --git a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_asm.go b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_asm.go index d4221edf4f..8a0170dffa 100644 --- a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_asm.go +++ b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_asm.go @@ -1,9 +1,4 @@ -//go:build (amd64 || arm64) && !appengine && gc && !purego && !noasm -// +build amd64 arm64 -// +build !appengine -// +build gc -// +build !purego -// +build !noasm +//go:build (amd64 || arm64) && gc && !purego && !noasm package xxhash diff --git a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_other.go b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_other.go index 9576426e68..83fd719123 100644 --- a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_other.go +++ b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_other.go @@ -1,4 +1,4 @@ -//go:build (!amd64 && !arm64) || appengine || !gc || purego || noasm +//go:build (!amd64 && !arm64) || !gc || purego || noasm package xxhash diff --git a/vendor/github.com/klauspost/compress/zstd/matchlen_amd64.go b/vendor/github.com/klauspost/compress/zstd/matchlen_amd64.go index 1ed18927f9..1d89a6008e 100644 --- a/vendor/github.com/klauspost/compress/zstd/matchlen_amd64.go +++ b/vendor/github.com/klauspost/compress/zstd/matchlen_amd64.go @@ -1,4 +1,4 @@ -//go:build amd64 && !appengine && !noasm && gc +//go:build amd64 && !noasm && gc // Copyright 2019+ Klaus Post. All rights reserved. // License information can be found in the LICENSE file. diff --git a/vendor/github.com/klauspost/compress/zstd/matchlen_amd64.s b/vendor/github.com/klauspost/compress/zstd/matchlen_amd64.s index 0782b86e3d..760046f0d8 100644 --- a/vendor/github.com/klauspost/compress/zstd/matchlen_amd64.s +++ b/vendor/github.com/klauspost/compress/zstd/matchlen_amd64.s @@ -1,6 +1,6 @@ // Copied from S2 implementation. -//go:build !appengine && !noasm && gc && !noasm +//go:build !noasm && gc #include "textflag.h" diff --git a/vendor/github.com/klauspost/compress/zstd/matchlen_generic.go b/vendor/github.com/klauspost/compress/zstd/matchlen_generic.go index 379746c96c..ec47656cb2 100644 --- a/vendor/github.com/klauspost/compress/zstd/matchlen_generic.go +++ b/vendor/github.com/klauspost/compress/zstd/matchlen_generic.go @@ -1,4 +1,4 @@ -//go:build !amd64 || appengine || !gc || noasm +//go:build !amd64 || !gc || noasm // Copyright 2019+ Klaus Post. All rights reserved. // License information can be found in the LICENSE file. diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec.go b/vendor/github.com/klauspost/compress/zstd/seqdec.go index d1a7732fcc..4b488e0cc6 100644 --- a/vendor/github.com/klauspost/compress/zstd/seqdec.go +++ b/vendor/github.com/klauspost/compress/zstd/seqdec.go @@ -76,6 +76,31 @@ type sequenceDecs struct { maxSyncLen uint64 } +// twoPassFarCode is the offset code from which a match counts as far: 20 is +// 1 MiB, the distance every measured arm64 core gains from. A Neoverse N1 +// (Graviton2, Ampere Altra) gains from 128 KiB too, but that needs a +// per-core threshold override; not worth the CPU-specific special case for +// a core class already being phased out of the fleet. +const twoPassFarCode = 20 + +// useTwoPass reports whether the block is decoded in two passes with the +// match-source prefetch (seqdec_arm64.go): a window of at least +// decodeTwoPassMinWindow and, unless the offset table is predefined or +// RLE, a share of at least twoPassMinFarShare of codes twoPassFarCode+. +func (s *sequenceDecs) useTwoPass() bool { + if s.windowSize < decodeTwoPassMinWindow { + return false + } + fse := s.offsets.fse + if fse != nil && !fse.preDefined && fse.actualTableLog != 0 && int(fse.symbolLen) <= twoPassFarCode { + // No symbol reaches twoPassFarCode, so codeShare can only return 0: + // skip finding it. + return false + } + share := fse.codeShare(twoPassFarCode) + return share < 0 || share >= twoPassMinFarShare +} + // initialize all 3 decoders from the stream input. func (s *sequenceDecs) initialize(br *bitReader, hist *history, out []byte) error { if err := s.litLengths.init(br); err != nil { @@ -240,11 +265,17 @@ func (s *sequenceDecs) execute(seqs []seqVals, hist []byte) error { return nil } +// decodeSyncGoOnly makes decodeSync skip the assembly decoder, so tests can +// compare it with the Go one in the same binary. +var decodeSyncGoOnly bool + // decode sequences from the stream with the provided history. func (s *sequenceDecs) decodeSync(hist []byte) error { - supported, err := s.decodeSyncSimple(hist) - if supported { - return err + if !decodeSyncGoOnly { + supported, err := s.decodeSyncSimple(hist) + if supported { + return err + } } br := s.br @@ -367,13 +398,13 @@ func (s *sequenceDecs) decodeSync(hist []byte) error { if mo > len(out)+len(hist) || mo > s.windowSize { if len(s.dict) == 0 { - return fmt.Errorf("match offset (%d) bigger than current history (%d)", mo, len(out)+len(hist)-startSize) + return fmt.Errorf("match offset (%d) bigger than current history (%d)", mo, len(out)-startSize) } // we may be in dictionary. dictO := len(s.dict) - (mo - (len(out) + len(hist))) if dictO < 0 || dictO >= len(s.dict) { - return fmt.Errorf("match offset (%d) bigger than current history (%d)", mo, len(out)+len(hist)-startSize) + return fmt.Errorf("match offset (%d) bigger than current history (%d)", mo, len(out)-startSize) } end := dictO + ml if end > len(s.dict) { diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.go b/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.go index 1281da885c..968f5b6a5e 100644 --- a/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.go +++ b/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.go @@ -1,8 +1,10 @@ -//go:build amd64 && !appengine && !noasm && gc +//go:build amd64 && !noasm && gc package zstd import ( + "math" + "github.com/klauspost/compress/internal/cpuinfo" ) @@ -10,6 +12,14 @@ import ( // in seqdec_asm.go; this file only declares the amd64 asm routines and the // dispatch helpers that pick the BMI2 / non-BMI2 (and 56-bit / safe) variant. +// See seqdec_arm64.go. Disabled here: on Sapphire Rapids, Granite Rapids +// and Zen 4 the two-pass decode costs 7-12% on real data; their caches +// already cover the offsets involved. +var ( + twoPassMinFarShare = 257 // never: the share is at most 256 + decodeTwoPassMinWindow = math.MaxInt +) + // sequenceDecs_decode implements the main loop of sequenceDecs in x86 asm. // // Please refer to seqdec_generic.go for the reference implementation. diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s b/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s index 798d940231..6895ec59d9 100644 --- a/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s +++ b/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s @@ -1,6 +1,6 @@ // Code generated by command: go run gen.go -out ../seqdec.s -arch amd64,arm64 -pkg=zstd. DO NOT EDIT. -//go:build !appengine && !noasm && gc && !noasm +//go:build !noasm && gc // func sequenceDecs_decode_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int // Requires: CMOV @@ -29,6 +29,7 @@ TEXT ·sequenceDecs_decode_amd64(SB), $40-32 MOVQ 144(AX), R11 MOVQ 152(AX), R12 MOVQ 160(AX), R13 + PCALIGN $0x40 sequenceDecs_decode_amd64_main_loop: MOVQ (SP), R14 @@ -354,6 +355,7 @@ TEXT ·sequenceDecs_decode_56_amd64(SB), $40-32 MOVQ 144(AX), R11 MOVQ 152(AX), R12 MOVQ 160(AX), R13 + PCALIGN $0x40 sequenceDecs_decode_56_amd64_main_loop: MOVQ (SP), R14 @@ -650,6 +652,7 @@ TEXT ·sequenceDecs_decode_bmi2(SB), $40-32 MOVQ 144(CX), R10 MOVQ 152(CX), R11 MOVQ 160(CX), R12 + PCALIGN $0x40 sequenceDecs_decode_bmi2_main_loop: MOVQ (SP), R13 @@ -957,6 +960,7 @@ TEXT ·sequenceDecs_decode_56_bmi2(SB), $40-32 MOVQ 144(CX), R10 MOVQ 152(CX), R11 MOVQ 160(CX), R12 + PCALIGN $0x40 sequenceDecs_decode_56_bmi2_main_loop: MOVQ (SP), R13 @@ -1209,21 +1213,33 @@ error_overread: RET // func sequenceDecs_executeSimple_amd64(ctx *executeAsmContext) bool -// Requires: SSE -TEXT ·sequenceDecs_executeSimple_amd64(SB), $8-9 +// Requires: MMX+, SSE +TEXT ·sequenceDecs_executeSimple_amd64(SB), $32-9 MOVQ ctx+0(FP), R10 MOVQ 8(R10), CX TESTQ CX, CX JZ empty_seqs MOVQ (R10), AX - MOVQ 24(R10), DX - MOVQ 32(R10), BX - MOVQ 80(R10), SI - MOVQ 104(R10), DI - MOVQ 120(R10), R8 - MOVQ 56(R10), R9 - MOVQ 64(R10), R10 - ADDQ R10, R9 + + // seqsEnd = seqsBase + 24 * len(seqs) if ctx.prefetch, else seqsBase (prefetch off) + MOVQ AX, DX + MOVBQZX 128(R10), R11 + TESTQ R11, R11 + JZ prefetch_off + LEAQ (CX)(CX*2), DX + SHLQ $0x03, DX + ADDQ AX, DX + +prefetch_off: + MOVQ DX, (SP) + MOVQ 24(R10), DX + MOVQ 32(R10), BX + MOVQ 80(R10), SI + MOVQ 104(R10), DI + MOVQ 120(R10), R8 + MOVQ 56(R10), R9 + MOVQ 64(R10), R10 + ADDQ R10, R9 // seqsBase += 24 * seqIndex LEAQ (DX)(DX*2), R11 @@ -1232,8 +1248,233 @@ TEXT ·sequenceDecs_executeSimple_amd64(SB), $8-9 // outBase += outPosition ADDQ DI, BX + MOVQ BX, 8(SP) + MOVQ BX, R11 + SUBQ DI, R11 + MOVQ R11, 16(SP) + NEGQ R11 + ADDQ R9, R11 + MOVQ R11, 24(SP) + // Prime the prefetch window + MOVQ (SP), R11 + LEAQ (AX), R12 + CMPQ R12, R11 + JAE prefetch_skip_0 + MOVQ 8(SP), R12 + MOVQ (AX), R11 + ADDQ R11, R12 + MOVQ R12, R13 + MOVQ 16(AX), R11 + SUBQ R11, R13 + MOVQ 16(SP), R11 + CMPQ R13, R11 + JAE prefetch_in_out_0 + MOVQ 24(SP), R11 + ADDQ R11, R13 + +prefetch_in_out_0: + PREFETCHT0 (R13) + PREFETCHT0 64(R13) + MOVQ 8(AX), R11 + ADDQ R11, R12 + MOVQ R12, 8(SP) + +prefetch_skip_0: + MOVQ (SP), R11 + LEAQ 24(AX), R12 + CMPQ R12, R11 + JAE prefetch_skip_1 + MOVQ 8(SP), R12 + MOVQ 24(AX), R11 + ADDQ R11, R12 + MOVQ R12, R13 + MOVQ 40(AX), R11 + SUBQ R11, R13 + MOVQ 16(SP), R11 + CMPQ R13, R11 + JAE prefetch_in_out_1 + MOVQ 24(SP), R11 + ADDQ R11, R13 + +prefetch_in_out_1: + PREFETCHT0 (R13) + PREFETCHT0 64(R13) + MOVQ 32(AX), R11 + ADDQ R11, R12 + MOVQ R12, 8(SP) + +prefetch_skip_1: + MOVQ (SP), R11 + LEAQ 48(AX), R12 + CMPQ R12, R11 + JAE prefetch_skip_2 + MOVQ 8(SP), R12 + MOVQ 48(AX), R11 + ADDQ R11, R12 + MOVQ R12, R13 + MOVQ 64(AX), R11 + SUBQ R11, R13 + MOVQ 16(SP), R11 + CMPQ R13, R11 + JAE prefetch_in_out_2 + MOVQ 24(SP), R11 + ADDQ R11, R13 + +prefetch_in_out_2: + PREFETCHT0 (R13) + PREFETCHT0 64(R13) + MOVQ 56(AX), R11 + ADDQ R11, R12 + MOVQ R12, 8(SP) + +prefetch_skip_2: + MOVQ (SP), R11 + LEAQ 72(AX), R12 + CMPQ R12, R11 + JAE prefetch_skip_3 + MOVQ 8(SP), R12 + MOVQ 72(AX), R11 + ADDQ R11, R12 + MOVQ R12, R13 + MOVQ 88(AX), R11 + SUBQ R11, R13 + MOVQ 16(SP), R11 + CMPQ R13, R11 + JAE prefetch_in_out_3 + MOVQ 24(SP), R11 + ADDQ R11, R13 + +prefetch_in_out_3: + PREFETCHT0 (R13) + PREFETCHT0 64(R13) + MOVQ 80(AX), R11 + ADDQ R11, R12 + MOVQ R12, 8(SP) + +prefetch_skip_3: + MOVQ (SP), R11 + LEAQ 96(AX), R12 + CMPQ R12, R11 + JAE prefetch_skip_4 + MOVQ 8(SP), R12 + MOVQ 96(AX), R11 + ADDQ R11, R12 + MOVQ R12, R13 + MOVQ 112(AX), R11 + SUBQ R11, R13 + MOVQ 16(SP), R11 + CMPQ R13, R11 + JAE prefetch_in_out_4 + MOVQ 24(SP), R11 + ADDQ R11, R13 + +prefetch_in_out_4: + PREFETCHT0 (R13) + PREFETCHT0 64(R13) + MOVQ 104(AX), R11 + ADDQ R11, R12 + MOVQ R12, 8(SP) + +prefetch_skip_4: + MOVQ (SP), R11 + LEAQ 120(AX), R12 + CMPQ R12, R11 + JAE prefetch_skip_5 + MOVQ 8(SP), R12 + MOVQ 120(AX), R11 + ADDQ R11, R12 + MOVQ R12, R13 + MOVQ 136(AX), R11 + SUBQ R11, R13 + MOVQ 16(SP), R11 + CMPQ R13, R11 + JAE prefetch_in_out_5 + MOVQ 24(SP), R11 + ADDQ R11, R13 + +prefetch_in_out_5: + PREFETCHT0 (R13) + PREFETCHT0 64(R13) + MOVQ 128(AX), R11 + ADDQ R11, R12 + MOVQ R12, 8(SP) + +prefetch_skip_5: + MOVQ (SP), R11 + LEAQ 144(AX), R12 + CMPQ R12, R11 + JAE prefetch_skip_6 + MOVQ 8(SP), R12 + MOVQ 144(AX), R11 + ADDQ R11, R12 + MOVQ R12, R13 + MOVQ 160(AX), R11 + SUBQ R11, R13 + MOVQ 16(SP), R11 + CMPQ R13, R11 + JAE prefetch_in_out_6 + MOVQ 24(SP), R11 + ADDQ R11, R13 + +prefetch_in_out_6: + PREFETCHT0 (R13) + PREFETCHT0 64(R13) + MOVQ 152(AX), R11 + ADDQ R11, R12 + MOVQ R12, 8(SP) + +prefetch_skip_6: + MOVQ (SP), R11 + LEAQ 168(AX), R12 + CMPQ R12, R11 + JAE prefetch_skip_7 + MOVQ 8(SP), R12 + MOVQ 168(AX), R11 + ADDQ R11, R12 + MOVQ R12, R13 + MOVQ 184(AX), R11 + SUBQ R11, R13 + MOVQ 16(SP), R11 + CMPQ R13, R11 + JAE prefetch_in_out_7 + MOVQ 24(SP), R11 + ADDQ R11, R13 + +prefetch_in_out_7: + PREFETCHT0 (R13) + PREFETCHT0 64(R13) + MOVQ 176(AX), R11 + ADDQ R11, R12 + MOVQ R12, 8(SP) + PCALIGN $0x40 + +prefetch_skip_7: main_loop: + MOVQ (SP), R11 + LEAQ 192(AX), R12 + CMPQ R12, R11 + JAE prefetch_skip_8 + MOVQ 8(SP), R12 + MOVQ 192(AX), R11 + ADDQ R11, R12 + MOVQ R12, R13 + MOVQ 208(AX), R11 + SUBQ R11, R13 + MOVQ 16(SP), R11 + CMPQ R13, R11 + JAE prefetch_in_out_8 + MOVQ 24(SP), R11 + ADDQ R11, R13 + +prefetch_in_out_8: + PREFETCHT0 (R13) + PREFETCHT0 64(R13) + MOVQ 200(AX), R11 + ADDQ R11, R12 + MOVQ R12, 8(SP) + +prefetch_skip_8: MOVQ (AX), R11 MOVQ 16(AX), R12 MOVQ 8(AX), R13 @@ -1431,6 +1672,19 @@ copy_2: // Copy overlapping match copy_overlapping_match: ADDQ R13, DI + CMPQ R12, $0x10 + JB copy_slow_3 + MOVQ BX, R12 + ADDQ R13, BX + +copy_3: + MOVUPS (R11), X0 + MOVUPS X0, (R12) + ADDQ $0x10, R11 + ADDQ $0x10, R12 + SUBQ $0x10, R13 + JHI copy_3 + JMP handle_loop copy_slow_3: MOVBQZX (R11), R12 @@ -1476,21 +1730,33 @@ empty_seqs: RET // func sequenceDecs_executeSimple_safe_amd64(ctx *executeAsmContext) bool -// Requires: SSE -TEXT ·sequenceDecs_executeSimple_safe_amd64(SB), $8-9 +// Requires: MMX+, SSE +TEXT ·sequenceDecs_executeSimple_safe_amd64(SB), $32-9 MOVQ ctx+0(FP), R10 MOVQ 8(R10), CX TESTQ CX, CX JZ empty_seqs MOVQ (R10), AX - MOVQ 24(R10), DX - MOVQ 32(R10), BX - MOVQ 80(R10), SI - MOVQ 104(R10), DI - MOVQ 120(R10), R8 - MOVQ 56(R10), R9 - MOVQ 64(R10), R10 - ADDQ R10, R9 + + // seqsEnd = seqsBase + 24 * len(seqs) if ctx.prefetch, else seqsBase (prefetch off) + MOVQ AX, DX + MOVBQZX 128(R10), R11 + TESTQ R11, R11 + JZ prefetch_off + LEAQ (CX)(CX*2), DX + SHLQ $0x03, DX + ADDQ AX, DX + +prefetch_off: + MOVQ DX, (SP) + MOVQ 24(R10), DX + MOVQ 32(R10), BX + MOVQ 80(R10), SI + MOVQ 104(R10), DI + MOVQ 120(R10), R8 + MOVQ 56(R10), R9 + MOVQ 64(R10), R10 + ADDQ R10, R9 // seqsBase += 24 * seqIndex LEAQ (DX)(DX*2), R11 @@ -1499,8 +1765,233 @@ TEXT ·sequenceDecs_executeSimple_safe_amd64(SB), $8-9 // outBase += outPosition ADDQ DI, BX + MOVQ BX, 8(SP) + MOVQ BX, R11 + SUBQ DI, R11 + MOVQ R11, 16(SP) + NEGQ R11 + ADDQ R9, R11 + MOVQ R11, 24(SP) + // Prime the prefetch window + MOVQ (SP), R11 + LEAQ (AX), R12 + CMPQ R12, R11 + JAE prefetch_skip_0 + MOVQ 8(SP), R12 + MOVQ (AX), R11 + ADDQ R11, R12 + MOVQ R12, R13 + MOVQ 16(AX), R11 + SUBQ R11, R13 + MOVQ 16(SP), R11 + CMPQ R13, R11 + JAE prefetch_in_out_0 + MOVQ 24(SP), R11 + ADDQ R11, R13 + +prefetch_in_out_0: + PREFETCHT0 (R13) + PREFETCHT0 64(R13) + MOVQ 8(AX), R11 + ADDQ R11, R12 + MOVQ R12, 8(SP) + +prefetch_skip_0: + MOVQ (SP), R11 + LEAQ 24(AX), R12 + CMPQ R12, R11 + JAE prefetch_skip_1 + MOVQ 8(SP), R12 + MOVQ 24(AX), R11 + ADDQ R11, R12 + MOVQ R12, R13 + MOVQ 40(AX), R11 + SUBQ R11, R13 + MOVQ 16(SP), R11 + CMPQ R13, R11 + JAE prefetch_in_out_1 + MOVQ 24(SP), R11 + ADDQ R11, R13 + +prefetch_in_out_1: + PREFETCHT0 (R13) + PREFETCHT0 64(R13) + MOVQ 32(AX), R11 + ADDQ R11, R12 + MOVQ R12, 8(SP) + +prefetch_skip_1: + MOVQ (SP), R11 + LEAQ 48(AX), R12 + CMPQ R12, R11 + JAE prefetch_skip_2 + MOVQ 8(SP), R12 + MOVQ 48(AX), R11 + ADDQ R11, R12 + MOVQ R12, R13 + MOVQ 64(AX), R11 + SUBQ R11, R13 + MOVQ 16(SP), R11 + CMPQ R13, R11 + JAE prefetch_in_out_2 + MOVQ 24(SP), R11 + ADDQ R11, R13 + +prefetch_in_out_2: + PREFETCHT0 (R13) + PREFETCHT0 64(R13) + MOVQ 56(AX), R11 + ADDQ R11, R12 + MOVQ R12, 8(SP) + +prefetch_skip_2: + MOVQ (SP), R11 + LEAQ 72(AX), R12 + CMPQ R12, R11 + JAE prefetch_skip_3 + MOVQ 8(SP), R12 + MOVQ 72(AX), R11 + ADDQ R11, R12 + MOVQ R12, R13 + MOVQ 88(AX), R11 + SUBQ R11, R13 + MOVQ 16(SP), R11 + CMPQ R13, R11 + JAE prefetch_in_out_3 + MOVQ 24(SP), R11 + ADDQ R11, R13 + +prefetch_in_out_3: + PREFETCHT0 (R13) + PREFETCHT0 64(R13) + MOVQ 80(AX), R11 + ADDQ R11, R12 + MOVQ R12, 8(SP) + +prefetch_skip_3: + MOVQ (SP), R11 + LEAQ 96(AX), R12 + CMPQ R12, R11 + JAE prefetch_skip_4 + MOVQ 8(SP), R12 + MOVQ 96(AX), R11 + ADDQ R11, R12 + MOVQ R12, R13 + MOVQ 112(AX), R11 + SUBQ R11, R13 + MOVQ 16(SP), R11 + CMPQ R13, R11 + JAE prefetch_in_out_4 + MOVQ 24(SP), R11 + ADDQ R11, R13 + +prefetch_in_out_4: + PREFETCHT0 (R13) + PREFETCHT0 64(R13) + MOVQ 104(AX), R11 + ADDQ R11, R12 + MOVQ R12, 8(SP) + +prefetch_skip_4: + MOVQ (SP), R11 + LEAQ 120(AX), R12 + CMPQ R12, R11 + JAE prefetch_skip_5 + MOVQ 8(SP), R12 + MOVQ 120(AX), R11 + ADDQ R11, R12 + MOVQ R12, R13 + MOVQ 136(AX), R11 + SUBQ R11, R13 + MOVQ 16(SP), R11 + CMPQ R13, R11 + JAE prefetch_in_out_5 + MOVQ 24(SP), R11 + ADDQ R11, R13 + +prefetch_in_out_5: + PREFETCHT0 (R13) + PREFETCHT0 64(R13) + MOVQ 128(AX), R11 + ADDQ R11, R12 + MOVQ R12, 8(SP) + +prefetch_skip_5: + MOVQ (SP), R11 + LEAQ 144(AX), R12 + CMPQ R12, R11 + JAE prefetch_skip_6 + MOVQ 8(SP), R12 + MOVQ 144(AX), R11 + ADDQ R11, R12 + MOVQ R12, R13 + MOVQ 160(AX), R11 + SUBQ R11, R13 + MOVQ 16(SP), R11 + CMPQ R13, R11 + JAE prefetch_in_out_6 + MOVQ 24(SP), R11 + ADDQ R11, R13 + +prefetch_in_out_6: + PREFETCHT0 (R13) + PREFETCHT0 64(R13) + MOVQ 152(AX), R11 + ADDQ R11, R12 + MOVQ R12, 8(SP) + +prefetch_skip_6: + MOVQ (SP), R11 + LEAQ 168(AX), R12 + CMPQ R12, R11 + JAE prefetch_skip_7 + MOVQ 8(SP), R12 + MOVQ 168(AX), R11 + ADDQ R11, R12 + MOVQ R12, R13 + MOVQ 184(AX), R11 + SUBQ R11, R13 + MOVQ 16(SP), R11 + CMPQ R13, R11 + JAE prefetch_in_out_7 + MOVQ 24(SP), R11 + ADDQ R11, R13 + +prefetch_in_out_7: + PREFETCHT0 (R13) + PREFETCHT0 64(R13) + MOVQ 176(AX), R11 + ADDQ R11, R12 + MOVQ R12, 8(SP) + PCALIGN $0x40 + +prefetch_skip_7: main_loop: + MOVQ (SP), R11 + LEAQ 192(AX), R12 + CMPQ R12, R11 + JAE prefetch_skip_8 + MOVQ 8(SP), R12 + MOVQ 192(AX), R11 + ADDQ R11, R12 + MOVQ R12, R13 + MOVQ 208(AX), R11 + SUBQ R11, R13 + MOVQ 16(SP), R11 + CMPQ R13, R11 + JAE prefetch_in_out_8 + MOVQ 24(SP), R11 + ADDQ R11, R13 + +prefetch_in_out_8: + PREFETCHT0 (R13) + PREFETCHT0 64(R13) + MOVQ 200(AX), R11 + ADDQ R11, R12 + MOVQ R12, 8(SP) + +prefetch_skip_8: MOVQ (AX), R11 MOVQ 16(AX), R12 MOVQ 8(AX), R13 @@ -1787,6 +2278,19 @@ copy_2_end: // Copy overlapping match copy_overlapping_match: ADDQ R13, DI + CMPQ R12, $0x10 + JB copy_slow_3 + +copy_3_blocks: + MOVUPS (R11), X0 + MOVUPS X0, (BX) + ADDQ $0x10, R11 + ADDQ $0x10, BX + SUBQ $0x10, R13 + CMPQ R13, $0x10 + JAE copy_3_blocks + TESTQ R13, R13 + JZ handle_loop copy_slow_3: MOVBQZX (R11), R12 @@ -1875,7 +2379,8 @@ TEXT ·sequenceDecs_decodeSync_amd64(SB), $96-32 ADDQ R10, 64(SP) // outBase += outPosition - ADDQ R12, R10 + ADDQ R12, R10 + PCALIGN $0x40 sequenceDecs_decodeSync_amd64_main_loop: MOVQ (SP), R13 @@ -2299,6 +2804,19 @@ copy_2: // Copy overlapping match copy_overlapping_match: ADDQ AX, R12 + CMPQ R13, $0x10 + JB copy_slow_3 + MOVQ R10, R13 + ADDQ AX, R10 + +copy_3: + MOVUPS (CX), X0 + MOVUPS X0, (R13) + ADDQ $0x10, CX + ADDQ $0x10, R13 + SUBQ $0x10, AX + JHI copy_3 + JMP handle_loop copy_slow_3: MOVBQZX (CX), R13 @@ -2429,7 +2947,8 @@ TEXT ·sequenceDecs_decodeSync_bmi2(SB), $96-32 ADDQ R9, 64(SP) // outBase += outPosition - ADDQ R11, R9 + ADDQ R11, R9 + PCALIGN $0x40 sequenceDecs_decodeSync_bmi2_main_loop: MOVQ (SP), R12 @@ -2835,6 +3354,19 @@ copy_2: // Copy overlapping match copy_overlapping_match: ADDQ CX, R11 + CMPQ R13, $0x10 + JB copy_slow_3 + MOVQ R9, R13 + ADDQ CX, R9 + +copy_3: + MOVUPS (R12), X0 + MOVUPS X0, (R13) + ADDQ $0x10, R12 + ADDQ $0x10, R13 + SUBQ $0x10, CX + JHI copy_3 + JMP handle_loop copy_slow_3: MOVBQZX (R12), R13 @@ -2965,7 +3497,8 @@ TEXT ·sequenceDecs_decodeSync_safe_amd64(SB), $96-32 ADDQ R10, 64(SP) // outBase += outPosition - ADDQ R12, R10 + ADDQ R12, R10 + PCALIGN $0x40 sequenceDecs_decodeSync_safe_amd64_main_loop: MOVQ (SP), R13 @@ -3478,6 +4011,19 @@ copy_2_end: // Copy overlapping match copy_overlapping_match: ADDQ AX, R12 + CMPQ R13, $0x10 + JB copy_slow_3 + +copy_3_blocks: + MOVUPS (CX), X0 + MOVUPS X0, (R10) + ADDQ $0x10, CX + ADDQ $0x10, R10 + SUBQ $0x10, AX + CMPQ AX, $0x10 + JAE copy_3_blocks + TESTQ AX, AX + JZ handle_loop copy_slow_3: MOVBQZX (CX), R13 @@ -3608,7 +4154,8 @@ TEXT ·sequenceDecs_decodeSync_safe_bmi2(SB), $96-32 ADDQ R9, 64(SP) // outBase += outPosition - ADDQ R11, R9 + ADDQ R11, R9 + PCALIGN $0x40 sequenceDecs_decodeSync_safe_bmi2_main_loop: MOVQ (SP), R12 @@ -4103,6 +4650,19 @@ copy_2_end: // Copy overlapping match copy_overlapping_match: ADDQ CX, R11 + CMPQ R13, $0x10 + JB copy_slow_3 + +copy_3_blocks: + MOVUPS (R12), X0 + MOVUPS X0, (R9) + ADDQ $0x10, R12 + ADDQ $0x10, R9 + SUBQ $0x10, CX + CMPQ CX, $0x10 + JAE copy_3_blocks + TESTQ CX, CX + JZ handle_loop copy_slow_3: MOVBQZX (R12), R13 diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec_arm64.go b/vendor/github.com/klauspost/compress/zstd/seqdec_arm64.go index 5ad262acff..5b911ab57e 100644 --- a/vendor/github.com/klauspost/compress/zstd/seqdec_arm64.go +++ b/vendor/github.com/klauspost/compress/zstd/seqdec_arm64.go @@ -1,4 +1,4 @@ -//go:build arm64 && !appengine && !noasm && gc +//go:build arm64 && !noasm && gc package zstd @@ -7,6 +7,17 @@ package zstd // by the avo arm64 lowering printer) and the dispatch helpers. arm64 has no // BMI2, so each helper selects only between the 56-bit / safe variants. +// The synchronous decoder takes the two-pass path, and executeSimple +// prefetches, when the window is at least decodeTwoPassMinWindow and the +// block's offset table puts at least twoPassMinFarShare/256 of its +// sequences 128 KiB or more back (sequenceDecs.useTwoPass). Thresholds +// from the per-file N1 measurement in #1212; variables so tests can force +// either path. +var ( + twoPassMinFarShare = 46 + decodeTwoPassMinWindow = 1 << 20 +) + // sequenceDecs_decode_arm64 implements the main loop of sequenceDecs in arm64 asm. // // Please refer to seqdec_generic.go for the reference implementation. diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec_arm64.s b/vendor/github.com/klauspost/compress/zstd/seqdec_arm64.s index 56391bfc18..d1d16588c4 100644 --- a/vendor/github.com/klauspost/compress/zstd/seqdec_arm64.s +++ b/vendor/github.com/klauspost/compress/zstd/seqdec_arm64.s @@ -1,35 +1,36 @@ // Code generated by command: go run gen.go -out ../seqdec.s -arch amd64,arm64 -pkg=zstd. DO NOT EDIT. // EXPERIMENTAL arm64 output lowered from an amd64 avo program. -//go:build arm64 && (!appengine && !noasm && gc && !noasm) +//go:build arm64 && (!noasm && gc) // func sequenceDecs_decode_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int // Requires: CMOV TEXT ·sequenceDecs_decode_arm64(SB), $40-32 - MOVD br+8(FP), R1 - MOVD 24(R1), R2 - MOVBU 40(R1), R3 - MOVD (R1), R0 - MOVD 32(R1), R5 - ADD R5, R0, R0 - MOVD R0, 8(RSP) - MOVD ctx+16(FP), R0 - MOVD 72(R0), R6 - MOVD 80(R0), R7 - MOVD 88(R0), R8 - MOVD (R0), R1 - MOVD R1, 16(RSP) - MOVD 24(R0), R1 - MOVD R1, 24(RSP) - MOVD 48(R0), R1 - MOVD R1, 32(RSP) - MOVD 96(R0), R1 - MOVD R1, 40(RSP) - MOVD 104(R0), R9 - MOVD s+0(FP), R0 - MOVD 144(R0), R10 - MOVD 152(R0), R11 - MOVD 160(R0), R12 + MOVD br+8(FP), R1 + MOVD 24(R1), R2 + MOVBU 40(R1), R3 + MOVD (R1), R0 + MOVD 32(R1), R5 + ADD R5, R0, R0 + MOVD R0, 8(RSP) + MOVD ctx+16(FP), R0 + MOVD 72(R0), R6 + MOVD 80(R0), R7 + MOVD 88(R0), R8 + MOVD (R0), R1 + MOVD R1, 16(RSP) + MOVD 24(R0), R1 + MOVD R1, 24(RSP) + MOVD 48(R0), R1 + MOVD R1, 32(RSP) + MOVD 96(R0), R1 + MOVD R1, 40(RSP) + MOVD 104(R0), R9 + MOVD s+0(FP), R0 + MOVD 144(R0), R10 + MOVD 152(R0), R11 + MOVD 160(R0), R12 + PCALIGN $0x40 sequenceDecs_decode_amd64_main_loop: MOVD 8(RSP), R13 @@ -326,30 +327,31 @@ error_overread: // func sequenceDecs_decode_56_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int // Requires: CMOV TEXT ·sequenceDecs_decode_56_arm64(SB), $40-32 - MOVD br+8(FP), R1 - MOVD 24(R1), R2 - MOVBU 40(R1), R3 - MOVD (R1), R0 - MOVD 32(R1), R5 - ADD R5, R0, R0 - MOVD R0, 8(RSP) - MOVD ctx+16(FP), R0 - MOVD 72(R0), R6 - MOVD 80(R0), R7 - MOVD 88(R0), R8 - MOVD (R0), R1 - MOVD R1, 16(RSP) - MOVD 24(R0), R1 - MOVD R1, 24(RSP) - MOVD 48(R0), R1 - MOVD R1, 32(RSP) - MOVD 96(R0), R1 - MOVD R1, 40(RSP) - MOVD 104(R0), R9 - MOVD s+0(FP), R0 - MOVD 144(R0), R10 - MOVD 152(R0), R11 - MOVD 160(R0), R12 + MOVD br+8(FP), R1 + MOVD 24(R1), R2 + MOVBU 40(R1), R3 + MOVD (R1), R0 + MOVD 32(R1), R5 + ADD R5, R0, R0 + MOVD R0, 8(RSP) + MOVD ctx+16(FP), R0 + MOVD 72(R0), R6 + MOVD 80(R0), R7 + MOVD 88(R0), R8 + MOVD (R0), R1 + MOVD R1, 16(RSP) + MOVD 24(R0), R1 + MOVD R1, 24(RSP) + MOVD 48(R0), R1 + MOVD R1, 32(RSP) + MOVD 96(R0), R1 + MOVD R1, 40(RSP) + MOVD 104(R0), R9 + MOVD s+0(FP), R0 + MOVD 144(R0), R10 + MOVD 152(R0), R11 + MOVD 160(R0), R12 + PCALIGN $0x40 sequenceDecs_decode_56_amd64_main_loop: MOVD 8(RSP), R13 @@ -620,13 +622,25 @@ error_overread: // skipped sequenceDecs_decode_56_bmi2 (generic twin preferred on arm64) // func sequenceDecs_executeSimple_amd64(ctx *executeAsmContext) bool -// Requires: SSE -TEXT ·sequenceDecs_executeSimple_arm64(SB), $8-9 +// Requires: MMX+, SSE +TEXT ·sequenceDecs_executeSimple_arm64(SB), $32-9 MOVD ctx+0(FP), R9 MOVD 8(R9), R1 TST R1, R1 BEQ empty_seqs MOVD (R9), R0 + + // seqsEnd = seqsBase + 24 * len(seqs) if ctx.prefetch, else seqsBase (prefetch off) + MOVD R0, R2 + MOVBU 128(R9), R10 + TST R10, R10 + BEQ prefetch_off + ADD R1<<1, R1, R2 + LSL $0x03, R2, R2 + ADD R0, R2, R2 + +prefetch_off: + MOVD R2, 8(RSP) MOVD 24(R9), R2 MOVD 32(R9), R3 MOVD 80(R9), R5 @@ -642,9 +656,234 @@ TEXT ·sequenceDecs_executeSimple_arm64(SB), $8-9 ADD R10, R0, R0 // outBase += outPosition - ADD R6, R3, R3 + ADD R6, R3, R3 + MOVD R3, 16(RSP) + MOVD R3, R10 + SUB R6, R10, R10 + MOVD R10, 24(RSP) + NEG R10, R10 + ADD R8, R10, R10 + MOVD R10, 32(RSP) + // Prime the prefetch window + MOVD 8(RSP), R10 + MOVD R0, R11 + CMP R10, R11 + BHS prefetch_skip_0 + MOVD 16(RSP), R11 + MOVD (R0), R10 + ADD R10, R11, R11 + MOVD R11, R12 + MOVD 16(R0), R10 + SUB R10, R12, R12 + MOVD 24(RSP), R10 + CMP R10, R12 + BHS prefetch_in_out_0 + MOVD 32(RSP), R10 + ADD R10, R12, R12 + +prefetch_in_out_0: + PRFM (R12), PLDL1KEEP + PRFM 64(R12), PLDL1KEEP + MOVD 8(R0), R10 + ADD R10, R11, R11 + MOVD R11, 16(RSP) + +prefetch_skip_0: + MOVD 8(RSP), R10 + ADD $24, R0, R11 + CMP R10, R11 + BHS prefetch_skip_1 + MOVD 16(RSP), R11 + MOVD 24(R0), R10 + ADD R10, R11, R11 + MOVD R11, R12 + MOVD 40(R0), R10 + SUB R10, R12, R12 + MOVD 24(RSP), R10 + CMP R10, R12 + BHS prefetch_in_out_1 + MOVD 32(RSP), R10 + ADD R10, R12, R12 + +prefetch_in_out_1: + PRFM (R12), PLDL1KEEP + PRFM 64(R12), PLDL1KEEP + MOVD 32(R0), R10 + ADD R10, R11, R11 + MOVD R11, 16(RSP) + +prefetch_skip_1: + MOVD 8(RSP), R10 + ADD $48, R0, R11 + CMP R10, R11 + BHS prefetch_skip_2 + MOVD 16(RSP), R11 + MOVD 48(R0), R10 + ADD R10, R11, R11 + MOVD R11, R12 + MOVD 64(R0), R10 + SUB R10, R12, R12 + MOVD 24(RSP), R10 + CMP R10, R12 + BHS prefetch_in_out_2 + MOVD 32(RSP), R10 + ADD R10, R12, R12 + +prefetch_in_out_2: + PRFM (R12), PLDL1KEEP + PRFM 64(R12), PLDL1KEEP + MOVD 56(R0), R10 + ADD R10, R11, R11 + MOVD R11, 16(RSP) + +prefetch_skip_2: + MOVD 8(RSP), R10 + ADD $72, R0, R11 + CMP R10, R11 + BHS prefetch_skip_3 + MOVD 16(RSP), R11 + MOVD 72(R0), R10 + ADD R10, R11, R11 + MOVD R11, R12 + MOVD 88(R0), R10 + SUB R10, R12, R12 + MOVD 24(RSP), R10 + CMP R10, R12 + BHS prefetch_in_out_3 + MOVD 32(RSP), R10 + ADD R10, R12, R12 + +prefetch_in_out_3: + PRFM (R12), PLDL1KEEP + PRFM 64(R12), PLDL1KEEP + MOVD 80(R0), R10 + ADD R10, R11, R11 + MOVD R11, 16(RSP) + +prefetch_skip_3: + MOVD 8(RSP), R10 + ADD $96, R0, R11 + CMP R10, R11 + BHS prefetch_skip_4 + MOVD 16(RSP), R11 + MOVD 96(R0), R10 + ADD R10, R11, R11 + MOVD R11, R12 + MOVD 112(R0), R10 + SUB R10, R12, R12 + MOVD 24(RSP), R10 + CMP R10, R12 + BHS prefetch_in_out_4 + MOVD 32(RSP), R10 + ADD R10, R12, R12 + +prefetch_in_out_4: + PRFM (R12), PLDL1KEEP + PRFM 64(R12), PLDL1KEEP + MOVD 104(R0), R10 + ADD R10, R11, R11 + MOVD R11, 16(RSP) + +prefetch_skip_4: + MOVD 8(RSP), R10 + ADD $120, R0, R11 + CMP R10, R11 + BHS prefetch_skip_5 + MOVD 16(RSP), R11 + MOVD 120(R0), R10 + ADD R10, R11, R11 + MOVD R11, R12 + MOVD 136(R0), R10 + SUB R10, R12, R12 + MOVD 24(RSP), R10 + CMP R10, R12 + BHS prefetch_in_out_5 + MOVD 32(RSP), R10 + ADD R10, R12, R12 + +prefetch_in_out_5: + PRFM (R12), PLDL1KEEP + PRFM 64(R12), PLDL1KEEP + MOVD 128(R0), R10 + ADD R10, R11, R11 + MOVD R11, 16(RSP) + +prefetch_skip_5: + MOVD 8(RSP), R10 + ADD $144, R0, R11 + CMP R10, R11 + BHS prefetch_skip_6 + MOVD 16(RSP), R11 + MOVD 144(R0), R10 + ADD R10, R11, R11 + MOVD R11, R12 + MOVD 160(R0), R10 + SUB R10, R12, R12 + MOVD 24(RSP), R10 + CMP R10, R12 + BHS prefetch_in_out_6 + MOVD 32(RSP), R10 + ADD R10, R12, R12 + +prefetch_in_out_6: + PRFM (R12), PLDL1KEEP + PRFM 64(R12), PLDL1KEEP + MOVD 152(R0), R10 + ADD R10, R11, R11 + MOVD R11, 16(RSP) + +prefetch_skip_6: + MOVD 8(RSP), R10 + ADD $168, R0, R11 + CMP R10, R11 + BHS prefetch_skip_7 + MOVD 16(RSP), R11 + MOVD 168(R0), R10 + ADD R10, R11, R11 + MOVD R11, R12 + MOVD 184(R0), R10 + SUB R10, R12, R12 + MOVD 24(RSP), R10 + CMP R10, R12 + BHS prefetch_in_out_7 + MOVD 32(RSP), R10 + ADD R10, R12, R12 + +prefetch_in_out_7: + PRFM (R12), PLDL1KEEP + PRFM 64(R12), PLDL1KEEP + MOVD 176(R0), R10 + ADD R10, R11, R11 + MOVD R11, 16(RSP) + PCALIGN $0x40 + +prefetch_skip_7: main_loop: + MOVD 8(RSP), R10 + ADD $192, R0, R11 + CMP R10, R11 + BHS prefetch_skip_8 + MOVD 16(RSP), R11 + MOVD 192(R0), R10 + ADD R10, R11, R11 + MOVD R11, R12 + MOVD 208(R0), R10 + SUB R10, R12, R12 + MOVD 24(RSP), R10 + CMP R10, R12 + BHS prefetch_in_out_8 + MOVD 32(RSP), R10 + ADD R10, R12, R12 + +prefetch_in_out_8: + PRFM (R12), PLDL1KEEP + PRFM 64(R12), PLDL1KEEP + MOVD 200(R0), R10 + ADD R10, R11, R11 + MOVD R11, 16(RSP) + +prefetch_skip_8: MOVD (R0), R10 MOVD 16(R0), R11 MOVD 8(R0), R12 @@ -861,7 +1100,20 @@ copy_2: // Copy overlapping match copy_overlapping_match: - ADD R12, R6, R6 + ADD R12, R6, R6 + CMP $0x10, R11 + BLO copy_slow_3 + MOVD R3, R11 + ADD R12, R3, R3 + +copy_3: + FMOVQ (R10), F0 + FMOVQ F0, (R11) + ADD $0x10, R10, R10 + ADD $0x10, R11, R11 + SUBS $0x10, R12, R12 + BHI copy_3 + JMP handle_loop copy_slow_3: MOVBU (R10), R11 @@ -912,13 +1164,25 @@ empty_seqs: RET // func sequenceDecs_executeSimple_safe_amd64(ctx *executeAsmContext) bool -// Requires: SSE -TEXT ·sequenceDecs_executeSimple_safe_arm64(SB), $8-9 +// Requires: MMX+, SSE +TEXT ·sequenceDecs_executeSimple_safe_arm64(SB), $32-9 MOVD ctx+0(FP), R9 MOVD 8(R9), R1 TST R1, R1 BEQ empty_seqs MOVD (R9), R0 + + // seqsEnd = seqsBase + 24 * len(seqs) if ctx.prefetch, else seqsBase (prefetch off) + MOVD R0, R2 + MOVBU 128(R9), R10 + TST R10, R10 + BEQ prefetch_off + ADD R1<<1, R1, R2 + LSL $0x03, R2, R2 + ADD R0, R2, R2 + +prefetch_off: + MOVD R2, 8(RSP) MOVD 24(R9), R2 MOVD 32(R9), R3 MOVD 80(R9), R5 @@ -934,9 +1198,234 @@ TEXT ·sequenceDecs_executeSimple_safe_arm64(SB), $8-9 ADD R10, R0, R0 // outBase += outPosition - ADD R6, R3, R3 + ADD R6, R3, R3 + MOVD R3, 16(RSP) + MOVD R3, R10 + SUB R6, R10, R10 + MOVD R10, 24(RSP) + NEG R10, R10 + ADD R8, R10, R10 + MOVD R10, 32(RSP) + // Prime the prefetch window + MOVD 8(RSP), R10 + MOVD R0, R11 + CMP R10, R11 + BHS prefetch_skip_0 + MOVD 16(RSP), R11 + MOVD (R0), R10 + ADD R10, R11, R11 + MOVD R11, R12 + MOVD 16(R0), R10 + SUB R10, R12, R12 + MOVD 24(RSP), R10 + CMP R10, R12 + BHS prefetch_in_out_0 + MOVD 32(RSP), R10 + ADD R10, R12, R12 + +prefetch_in_out_0: + PRFM (R12), PLDL1KEEP + PRFM 64(R12), PLDL1KEEP + MOVD 8(R0), R10 + ADD R10, R11, R11 + MOVD R11, 16(RSP) + +prefetch_skip_0: + MOVD 8(RSP), R10 + ADD $24, R0, R11 + CMP R10, R11 + BHS prefetch_skip_1 + MOVD 16(RSP), R11 + MOVD 24(R0), R10 + ADD R10, R11, R11 + MOVD R11, R12 + MOVD 40(R0), R10 + SUB R10, R12, R12 + MOVD 24(RSP), R10 + CMP R10, R12 + BHS prefetch_in_out_1 + MOVD 32(RSP), R10 + ADD R10, R12, R12 + +prefetch_in_out_1: + PRFM (R12), PLDL1KEEP + PRFM 64(R12), PLDL1KEEP + MOVD 32(R0), R10 + ADD R10, R11, R11 + MOVD R11, 16(RSP) + +prefetch_skip_1: + MOVD 8(RSP), R10 + ADD $48, R0, R11 + CMP R10, R11 + BHS prefetch_skip_2 + MOVD 16(RSP), R11 + MOVD 48(R0), R10 + ADD R10, R11, R11 + MOVD R11, R12 + MOVD 64(R0), R10 + SUB R10, R12, R12 + MOVD 24(RSP), R10 + CMP R10, R12 + BHS prefetch_in_out_2 + MOVD 32(RSP), R10 + ADD R10, R12, R12 + +prefetch_in_out_2: + PRFM (R12), PLDL1KEEP + PRFM 64(R12), PLDL1KEEP + MOVD 56(R0), R10 + ADD R10, R11, R11 + MOVD R11, 16(RSP) + +prefetch_skip_2: + MOVD 8(RSP), R10 + ADD $72, R0, R11 + CMP R10, R11 + BHS prefetch_skip_3 + MOVD 16(RSP), R11 + MOVD 72(R0), R10 + ADD R10, R11, R11 + MOVD R11, R12 + MOVD 88(R0), R10 + SUB R10, R12, R12 + MOVD 24(RSP), R10 + CMP R10, R12 + BHS prefetch_in_out_3 + MOVD 32(RSP), R10 + ADD R10, R12, R12 + +prefetch_in_out_3: + PRFM (R12), PLDL1KEEP + PRFM 64(R12), PLDL1KEEP + MOVD 80(R0), R10 + ADD R10, R11, R11 + MOVD R11, 16(RSP) + +prefetch_skip_3: + MOVD 8(RSP), R10 + ADD $96, R0, R11 + CMP R10, R11 + BHS prefetch_skip_4 + MOVD 16(RSP), R11 + MOVD 96(R0), R10 + ADD R10, R11, R11 + MOVD R11, R12 + MOVD 112(R0), R10 + SUB R10, R12, R12 + MOVD 24(RSP), R10 + CMP R10, R12 + BHS prefetch_in_out_4 + MOVD 32(RSP), R10 + ADD R10, R12, R12 + +prefetch_in_out_4: + PRFM (R12), PLDL1KEEP + PRFM 64(R12), PLDL1KEEP + MOVD 104(R0), R10 + ADD R10, R11, R11 + MOVD R11, 16(RSP) + +prefetch_skip_4: + MOVD 8(RSP), R10 + ADD $120, R0, R11 + CMP R10, R11 + BHS prefetch_skip_5 + MOVD 16(RSP), R11 + MOVD 120(R0), R10 + ADD R10, R11, R11 + MOVD R11, R12 + MOVD 136(R0), R10 + SUB R10, R12, R12 + MOVD 24(RSP), R10 + CMP R10, R12 + BHS prefetch_in_out_5 + MOVD 32(RSP), R10 + ADD R10, R12, R12 + +prefetch_in_out_5: + PRFM (R12), PLDL1KEEP + PRFM 64(R12), PLDL1KEEP + MOVD 128(R0), R10 + ADD R10, R11, R11 + MOVD R11, 16(RSP) + +prefetch_skip_5: + MOVD 8(RSP), R10 + ADD $144, R0, R11 + CMP R10, R11 + BHS prefetch_skip_6 + MOVD 16(RSP), R11 + MOVD 144(R0), R10 + ADD R10, R11, R11 + MOVD R11, R12 + MOVD 160(R0), R10 + SUB R10, R12, R12 + MOVD 24(RSP), R10 + CMP R10, R12 + BHS prefetch_in_out_6 + MOVD 32(RSP), R10 + ADD R10, R12, R12 + +prefetch_in_out_6: + PRFM (R12), PLDL1KEEP + PRFM 64(R12), PLDL1KEEP + MOVD 152(R0), R10 + ADD R10, R11, R11 + MOVD R11, 16(RSP) + +prefetch_skip_6: + MOVD 8(RSP), R10 + ADD $168, R0, R11 + CMP R10, R11 + BHS prefetch_skip_7 + MOVD 16(RSP), R11 + MOVD 168(R0), R10 + ADD R10, R11, R11 + MOVD R11, R12 + MOVD 184(R0), R10 + SUB R10, R12, R12 + MOVD 24(RSP), R10 + CMP R10, R12 + BHS prefetch_in_out_7 + MOVD 32(RSP), R10 + ADD R10, R12, R12 + +prefetch_in_out_7: + PRFM (R12), PLDL1KEEP + PRFM 64(R12), PLDL1KEEP + MOVD 176(R0), R10 + ADD R10, R11, R11 + MOVD R11, 16(RSP) + PCALIGN $0x40 + +prefetch_skip_7: main_loop: + MOVD 8(RSP), R10 + ADD $192, R0, R11 + CMP R10, R11 + BHS prefetch_skip_8 + MOVD 16(RSP), R11 + MOVD 192(R0), R10 + ADD R10, R11, R11 + MOVD R11, R12 + MOVD 208(R0), R10 + SUB R10, R12, R12 + MOVD 24(RSP), R10 + CMP R10, R12 + BHS prefetch_in_out_8 + MOVD 32(RSP), R10 + ADD R10, R12, R12 + +prefetch_in_out_8: + PRFM (R12), PLDL1KEEP + PRFM 64(R12), PLDL1KEEP + MOVD 200(R0), R10 + ADD R10, R11, R11 + MOVD R11, 16(RSP) + +prefetch_skip_8: MOVD (R0), R10 MOVD 16(R0), R11 MOVD 8(R0), R12 @@ -1263,6 +1752,19 @@ copy_2_end: // Copy overlapping match copy_overlapping_match: ADD R12, R6, R6 + CMP $0x10, R11 + BLO copy_slow_3 + +copy_3_blocks: + FMOVQ (R10), F0 + FMOVQ F0, (R3) + ADD $0x10, R10, R10 + ADD $0x10, R3, R3 + SUB $0x10, R12, R12 + CMP $0x10, R12 + BHS copy_3_blocks + TST R12, R12 + BEQ handle_loop copy_slow_3: MOVBU (R10), R11 @@ -1360,7 +1862,8 @@ TEXT ·sequenceDecs_decodeSync_arm64(SB), $96-32 MOVD R16, 72(RSP) // outBase += outPosition - ADD R11, R9, R9 + ADD R11, R9, R9 + PCALIGN $0x40 sequenceDecs_decodeSync_amd64_main_loop: MOVD 8(RSP), R12 @@ -1794,7 +2297,20 @@ copy_2: // Copy overlapping match copy_overlapping_match: - ADD R0, R11, R11 + ADD R0, R11, R11 + CMP $0x10, R12 + BLO copy_slow_3 + MOVD R9, R12 + ADD R0, R9, R9 + +copy_3: + FMOVQ (R1), F0 + FMOVQ F0, (R12) + ADD $0x10, R1, R1 + ADD $0x10, R12, R12 + SUBS $0x10, R0, R0 + BHI copy_3 + JMP handle_loop copy_slow_3: MOVBU (R1), R12 @@ -1943,7 +2459,8 @@ TEXT ·sequenceDecs_decodeSync_safe_arm64(SB), $96-32 MOVD R16, 72(RSP) // outBase += outPosition - ADD R11, R9, R9 + ADD R11, R9, R9 + PCALIGN $0x40 sequenceDecs_decodeSync_safe_amd64_main_loop: MOVD 8(RSP), R12 @@ -2486,6 +3003,19 @@ copy_2_end: // Copy overlapping match copy_overlapping_match: ADD R0, R11, R11 + CMP $0x10, R12 + BLO copy_slow_3 + +copy_3_blocks: + FMOVQ (R1), F0 + FMOVQ F0, (R9) + ADD $0x10, R1, R1 + ADD $0x10, R9, R9 + SUB $0x10, R0, R0 + CMP $0x10, R0 + BHS copy_3_blocks + TST R0, R0 + BEQ handle_loop copy_slow_3: MOVBU (R1), R12 diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec_asm.go b/vendor/github.com/klauspost/compress/zstd/seqdec_asm.go index 94559c459a..543c49074b 100644 --- a/vendor/github.com/klauspost/compress/zstd/seqdec_asm.go +++ b/vendor/github.com/klauspost/compress/zstd/seqdec_asm.go @@ -1,4 +1,4 @@ -//go:build (amd64 || arm64) && !appengine && !noasm && gc +//go:build (amd64 || arm64) && !noasm && gc package zstd @@ -55,6 +55,9 @@ type executeAsmContext struct { outPosition int litPosition int windowSize int + // prefetch enables the match-source prefetch in the execute loop; see + // sequenceDecs.useTwoPass. + prefetch bool } const noError = 0 @@ -95,10 +98,15 @@ const errorOverread = 6 // See #1168. An asan-instrumented fuzz job (see .github/workflows/go.yml) // covers the extended-copy path, which -race and plain fuzzing cannot. func (s *sequenceDecs) useSafeDecodeSync() bool { - if s.maxSyncLen == 0 && cap(s.out)-len(s.out) < maxCompressedBlockSizeAlloc { + // A block never produces more than maxBlockSize bytes. + maxBlockSize := min(s.windowSize, maxCompressedBlockSize) + if s.maxSyncLen == 0 && cap(s.out)-len(s.out) < maxBlockSize+compressedBlockOverAlloc { return true } - if s.maxSyncLen > 0 && cap(s.out)-len(s.out)-compressedBlockOverAlloc < int(s.maxSyncLen) { + // The block, not the rest of the frame, must fit with the slack: a + // buffer sized exactly to the frame needs the safe copies only for the + // final block(s). + if s.maxSyncLen > 0 && cap(s.out)-len(s.out)-compressedBlockOverAlloc < min(int(s.maxSyncLen), maxBlockSize) { return true } if cap(s.literals) < len(s.literals)+compressedBlockOverAlloc { @@ -112,7 +120,8 @@ func (s *sequenceDecs) decodeSyncSimple(hist []byte) (bool, error) { if len(s.dict) > 0 { return false, nil } - if s.maxSyncLen == 0 && cap(s.out)-len(s.out) < maxCompressedBlockSize { + maxBlockSize := min(s.windowSize, maxCompressedBlockSize) + if s.maxSyncLen == 0 && cap(s.out)-len(s.out) < maxBlockSize { return false, nil } @@ -120,8 +129,6 @@ func (s *sequenceDecs) decodeSyncSimple(hist []byte) (bool, error) { br := s.br - maxBlockSize := min(s.windowSize, maxCompressedBlockSize) - ctx := decodeSyncAsmContext{ llTable: s.litLengths.fse.dt[:maxTablesize], mlTable: s.matchLengths.fse.dt[:maxTablesize], @@ -154,7 +161,7 @@ func (s *sequenceDecs) decodeSyncSimple(hist []byte) (bool, error) { case errorMatchOffTooBig: return true, fmt.Errorf("match offset (%d) bigger than current history (%d)", - ctx.mo, ctx.outPosition+len(hist)-startSize) + ctx.mo, ctx.outPosition-startSize) case errorNotEnoughLiterals: return true, fmt.Errorf("unexpected literal count, want %d bytes, but only %d is available", @@ -267,8 +274,10 @@ func (s *sequenceDecs) decode(seqs []seqVals) error { // executeSimple handles cases when dictionary is not used. func (s *sequenceDecs) executeSimple(seqs []seqVals, hist []byte) error { - // Ensure we have enough output size... - if len(s.out)+s.seqSize+compressedBlockOverAlloc > cap(s.out) { + // Grow only if the block does not fit. Lacking just the slack is + // handled by the safe copies below, so a buffer sized exactly to the + // frame is not reallocated and copied on its last block. + if len(s.out)+s.seqSize > cap(s.out) { addBytes := s.seqSize + len(s.out) + compressedBlockOverAlloc s.out = append(s.out, make([]byte, addBytes)...) s.out = s.out[:len(s.out)-addBytes] @@ -290,10 +299,12 @@ func (s *sequenceDecs) executeSimple(seqs []seqVals, hist []byte) error { litPosition: 0, literals: s.literals, windowSize: s.windowSize, + prefetch: s.useTwoPass(), } - // useSafe avoids overwriting the output buffer when the literals slice has - // not been allocated with the required over-allocation slack. - useSafe := cap(s.literals) < len(s.literals)+compressedBlockOverAlloc + // useSafe avoids writing past either buffer when the literals or the + // output lack the extended copies' over-allocation slack. + useSafe := cap(s.literals) < len(s.literals)+compressedBlockOverAlloc || + cap(out) < len(out)+compressedBlockOverAlloc ok := executeSimpleAsm(&ctx, useSafe) if !ok { diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec_generic.go b/vendor/github.com/klauspost/compress/zstd/seqdec_generic.go index 8a3db6ba22..efeed96185 100644 --- a/vendor/github.com/klauspost/compress/zstd/seqdec_generic.go +++ b/vendor/github.com/klauspost/compress/zstd/seqdec_generic.go @@ -1,10 +1,17 @@ -//go:build (!amd64 && !arm64) || appengine || !gc || noasm +//go:build (!amd64 && !arm64) || !gc || noasm package zstd import ( "fmt" "io" + "math" +) + +// See seqdec_arm64.go; the pure Go decoder has neither path. +var ( + twoPassMinFarShare = 257 // never: the share is at most 256 + decodeTwoPassMinWindow = math.MaxInt ) // decode sequences from the stream with the provided history but without dictionary. diff --git a/vendor/github.com/klauspost/compress/zstd/seqenc.go b/vendor/github.com/klauspost/compress/zstd/seqenc.go index 65045eabdd..6fb8885992 100644 --- a/vendor/github.com/klauspost/compress/zstd/seqenc.go +++ b/vendor/github.com/klauspost/compress/zstd/seqenc.go @@ -110,3 +110,50 @@ func ofCode(offset uint32) uint8 { // A valid offset will always be > 0. return uint8(bits.Len32(offset) - 1) } + +// newDictSeqEncoder builds an encoder for a dictionary sequence table. +// Returns nil if the table is missing or unusable. +func newDictSeqEncoder(src *fseDecoder, t tableIndex) *fseEncoder { + if src == nil || src.symbolLen == 0 { + return nil + } + var e fseEncoder + e.norm = src.norm + e.symbolLen = src.symbolLen + e.actualTableLog = src.actualTableLog + if e.buildCTable() != nil { + return nil + } + e.setBits(bitTables[t]) + return &e +} + +// seedPrevFromDict loads the dictionary sequence tables as the "previous" +// encoders, so the first block can use repeat mode. +func (s *seqCoders) seedPrevFromDict(d *dict) { + seed := func(dst, src *fseEncoder) { + if src == nil { + return + } + ct := dst.ct + dst.symbolLen = src.symbolLen + dst.actualTableLog = src.actualTableLog + dst.maxBits = src.maxBits + dst.zeroBits = src.zeroBits + dst.useRLE = false + dst.preDefined = false + dst.reUsed = true + dst.norm = src.norm + // tableSymbol is only used while building, so it is not copied. + ct.stateTable = append(ct.stateTable[:0], src.ct.stateTable...) + if cap(ct.symbolTT) < len(src.ct.symbolTT) { + ct.symbolTT = make([]symbolTransform, len(src.ct.symbolTT)) + } + ct.symbolTT = ct.symbolTT[:len(src.ct.symbolTT)] + copy(ct.symbolTT, src.ct.symbolTT[:src.symbolLen]) + dst.ct = ct + } + seed(s.llPrev, d.llEnc) + seed(s.mlPrev, d.mlEnc) + seed(s.ofPrev, d.ofEnc) +} diff --git a/vendor/github.com/klauspost/compress/zstd/simple_go124.go b/vendor/github.com/klauspost/compress/zstd/simple.go similarity index 98% rename from vendor/github.com/klauspost/compress/zstd/simple_go124.go rename to vendor/github.com/klauspost/compress/zstd/simple.go index 2efc0497bf..9b6c39ac00 100644 --- a/vendor/github.com/klauspost/compress/zstd/simple_go124.go +++ b/vendor/github.com/klauspost/compress/zstd/simple.go @@ -1,8 +1,6 @@ // Copyright 2025+ Klaus Post. All rights reserved. // License information can be found in the LICENSE file. -//go:build go1.24 - package zstd import ( diff --git a/vendor/github.com/klauspost/compress/zstd/snappy.go b/vendor/github.com/klauspost/compress/zstd/snappy.go index 36c56f36d7..06ed593ea4 100644 --- a/vendor/github.com/klauspost/compress/zstd/snappy.go +++ b/vendor/github.com/klauspost/compress/zstd/snappy.go @@ -69,6 +69,10 @@ var ( // No CRC value is being generated and not all CRC values of the Snappy stream are checked. // However, it provides really fast recompression of Snappy streams. // The converter can be reused to avoid allocations, even after errors. +// +// Deprecated: Decode the stream with s2.NewReader, which reads Snappy streams, +// and compress the output with an Encoder. This validates the input and +// compresses much better. type SnappyConverter struct { r io.Reader err error @@ -373,6 +377,14 @@ func decodeSnappy(blk *blockEnc, src []byte) error { offset = uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24 } + // A zstd match is at least zstdMinMatch bytes. A shorter copy + // cannot be represented as a sequence and would underflow the + // matchLen computation below, producing an out-of-range match + // length code. + if length < zstdMinMatch { + return ErrSnappyCorrupt + } + if offset <= 0 || blk.size+lits < int(offset) /*|| length > len(blk)-d */ { println("offset <= 0 || blk.size+lits < int(offset)", offset, blk.size+lits, int(offset), blk.size, lits) diff --git a/vendor/modules.txt b/vendor/modules.txt index c7fff67c79..8a23b35bc1 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -416,7 +416,7 @@ github.com/intel/goresctrl/pkg/utils # github.com/json-iterator/go v1.1.12 ## explicit; go 1.12 github.com/json-iterator/go -# github.com/klauspost/compress v1.20.1 +# github.com/klauspost/compress v1.20.3 ## explicit; go 1.25 github.com/klauspost/compress github.com/klauspost/compress/flate