Merge pull request #14409 from containerd/dependabot/go_modules/github.com/prometheus/client_golang-1.25.0

build(deps): bump github.com/prometheus/client_golang from 1.24.1 to 1.25.0
This commit is contained in:
Maksym Pavlenko authored and GitHub committed 2026-10-17 21:57:56 +00:00
commit 6c26ce5ff7
107 files changed
+9005 -3997

No files matched your search

+5 -5
View File
@@ -44,7 +44,7 @@ require (
github.com/google/uuid v1.6.0
github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.1.0
github.com/intel/goresctrl v0.13.0
github.com/klauspost/compress v1.20.0
github.com/klauspost/compress v1.20.1
github.com/mdlayher/vsock v1.3.0
github.com/moby/locker v1.0.1
github.com/moby/sys/mountinfo v0.7.2
@@ -57,7 +57,7 @@ require (
github.com/opencontainers/runtime-spec v1.3.0
github.com/opencontainers/selinux v1.15.1
github.com/pelletier/go-toml/v2 v2.4.3
github.com/prometheus/client_golang v1.24.1
github.com/prometheus/client_golang v1.25.0
github.com/prometheus/client_model v0.6.3
github.com/sirupsen/logrus v1.10.2
github.com/stretchr/testify v1.12.1
@@ -129,8 +129,8 @@ require (
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/prometheus/common v0.70.1 // indirect
github.com/prometheus/procfs v0.21.1 // indirect
github.com/prometheus/common v0.72.0 // indirect
github.com/prometheus/procfs v0.22.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/sasha-s/go-deadlock v0.3.5 // indirect
github.com/smallstep/pkcs7 v0.2.1 // indirect
@@ -144,7 +144,7 @@ require (
go.yaml.in/yaml/v3 v3.0.5 // indirect
golang.org/x/crypto v0.57.0 // indirect
golang.org/x/net v0.60.0 // indirect
golang.org/x/oauth2 v0.36.0 // indirect
golang.org/x/oauth2 v0.37.0 // indirect
golang.org/x/term v0.46.0 // indirect
golang.org/x/text v0.42.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20260825221802-da73d73af1c5 // indirect
+10 -10
View File
@@ -185,8 +185,8 @@ github.com/jsimonetti/rtnetlink/v2 v2.0.1 h1:xda7qaHDSVOsADNouv7ukSuicKZO7GgVUCX
github.com/jsimonetti/rtnetlink/v2 v2.0.1/go.mod h1:7MoNYNbb3UaDHtF8udiJo/RH6VsTKP1pqKLUTVCvToE=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ=
github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/knqyf263/go-plugin v0.9.0 h1:CQs2+lOPIlkZVtcb835ZYDEoyyWJWLbSTWeCs0EwTwI=
github.com/knqyf263/go-plugin v0.9.0/go.mod h1:2z5lCO1/pez6qGo8CvCxSlBFSEat4MEp1DrnA+f7w8Q=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
@@ -248,14 +248,14 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU=
github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE=
github.com/prometheus/client_golang v1.25.0 h1:cDtW1TcFsMi1RJc8khwCKB4VhzwV04aKqbYtNOXuiK0=
github.com/prometheus/client_golang v1.25.0/go.mod h1:j2X/4/kaQHLPnmyEU8zjmkndo1T317VpCIQEml8LTKk=
github.com/prometheus/client_model v0.6.3 h1:O0jaTVAYNxTHYInEPFJt5I3+sN8zqBtVMPTB1qyxiEo=
github.com/prometheus/client_model v0.6.3/go.mod h1:gpN5P9S7Rr6Yr92PiQ+Ixvhf6JZEkF1dnxsYL2aPBEM=
github.com/prometheus/common v0.70.1 h1:1HvjP4D5oL3t8RsPlwxA9onvvStjtIHYE5XuuwOi/PY=
github.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc=
github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI=
github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY=
github.com/prometheus/common v0.72.0 h1:tAYsE+sPJxIncDAobm4H5aQjmox9ZxEIIqPbiffa8G4=
github.com/prometheus/common v0.72.0/go.mod h1:77NWqAQ2tXT7BIK40qjJdw5Acrsrg1TlHAnsQi3i6mk=
github.com/prometheus/procfs v0.22.0 h1:6q9+/JL9IKAPbCmBrv9n5O5Ty3NKnciV5X7YGw0oics=
github.com/prometheus/procfs v0.22.0/go.mod h1:CvmFr/GVhIjIvWJZW3tgkODBQMRIf0EyWMQLHCHab58=
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
@@ -352,8 +352,8 @@ golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU=
golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg=
golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98=
golang.org/x/oauth2 v0.37.0/go.mod h1:IxwZNxUULJmpBFf9K/9NTMSIfZZuvuTy1gGxhigP/58=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+30 -5
View File
@@ -95,6 +95,11 @@ type advancedState struct {
index int
hashMatch [maxMatchLength + minMatchLength]uint32
// litBits[i] is the number of bits needed to encode window[index:i]
// as literals, where index is the window index at the start of the
// current deflateLazy call. Only allocated when d.chain > 100 (level 9).
litBits []int32
// Input hash chains
// hashHead[hashValue] contains the largest inputIndex with the specified hash value
// If hashHead[hashValue] is within the current window, then
@@ -284,7 +289,9 @@ func (d *compressor) findMatch(pos int, prevHead int, lookahead int) (length, of
minIndex := max(pos-windowSize, 0)
offset = 0
if d.chain < 100 {
// The gain estimate below needs the tables that deflateLazy only
// generates when d.chain > 100, so the conditions must match.
if d.chain <= 100 {
for i := prevHead; tries > 0; tries-- {
if wEnd == win[i+length] {
n := matchLen(win[i:i+minMatchLook], wPos)
@@ -312,21 +319,25 @@ func (d *compressor) findMatch(pos int, prevHead int, lookahead int) (length, of
}
// Minimum gain to accept a match.
cGain := 4
cGain := int32(4)
// Some like it higher (CSV), some like it lower (JSON)
const baseCost = 3
// Base is 4 bytes at with an additional cost.
// Matches must be better than this.
s := d.state
hashPrev := &s.hashPrev
hashOffset := s.hashOffset
litBits := s.litBits[pos : pos+minMatchLook+1]
for i := prevHead; tries > 0; tries-- {
if wEnd == win[i+length] {
n := matchLen(win[i:i+minMatchLook], wPos)
if n > length {
// Calculate gain. Estimate
newGain := d.h.bitLengthRaw(wPos[:n]) - int(offsetExtraBits[offsetCode(uint32(pos-i))]) - baseCost - int(lengthExtraBits[lengthCodes[(n-3)&255]])
newGain := litBits[n] - litBits[0] - int32(offsetExtraBits[offsetCode(uint32(pos-i))]) - baseCost - int32(lengthExtraBits[lengthCodes[(n-3)&255]])
//fmt.Println("gain:", newGain, "prev:", cGain, "raw:", d.h.bitLengthRaw(wPos[:n]), "this-len:", n, "prev-len:", length)
if newGain > cGain {
length = n
offset = pos - i
@@ -344,7 +355,7 @@ func (d *compressor) findMatch(pos int, prevHead int, lookahead int) (length, of
// hashPrev[i & windowMask] has already been overwritten, so stop now.
break
}
i = int(d.state.hashPrev[i&windowMask]) - d.state.hashOffset
i = int(hashPrev[i&windowMask]) - hashOffset
if i < minIndex {
break
}
@@ -428,6 +439,20 @@ func (d *compressor) deflateLazy() {
tmp[v]++
}
d.h.generate(tmp[:], 15)
// Compute the cumulative cost of encoding the window as literals,
// so findMatch can look up the cost of any range in constant time.
if s.litBits == nil {
s.litBits = make([]int32, 2*windowSize+1)
}
codes := d.h.codes[:256]
litBits := s.litBits[s.index : d.windowEnd+1]
total := int32(0)
litBits[0] = 0
for i, v := range d.window[s.index:d.windowEnd] {
total += int32(codes[v].len())
litBits[i+1] = total
}
}
s.maxInsertIndex = d.windowEnd - (minMatchLength - 1)
+94 -128
View File
@@ -32,14 +32,12 @@ const (
// bufferFlushSize indicates the buffer size
// after which bytes are flushed to the writer.
// Should preferably be a multiple of 6, since
// we accumulate 6 bytes between writes to the buffer.
// Between checks, at most 7 bytes are added to the buffer,
// and writes are done as unconditional 8-byte stores, so the
// buffer must have at least bufferFlushSize+7+8 bytes.
bufferFlushSize = 246
)
// Minimum length code that emits bits.
const lengthExtraBitsMinCode = 8
// The number of extra bits needed by length code X - LENGTH_CODES_START.
var lengthExtraBits = [32]uint8{
/* 257 */ 0, 0, 0,
@@ -55,9 +53,6 @@ var lengthBase = [32]uint8{
64, 80, 96, 112, 128, 160, 192, 224, 255,
}
// Minimum offset code that emits bits.
const offsetExtraBitsMinCode = 4
// offset code word extra bits.
var offsetExtraBits = [32]int8{
0, 0, 0, 0, 1, 1, 2, 2, 3, 3,
@@ -67,8 +62,24 @@ var offsetExtraBits = [32]int8{
14, 14,
}
// offsetCombined combines the number of extra bits and the base offset
// of each offset code in a single table: extra bits in the low 8 bits,
// and the base offset (already reduced by baseMatchOffset) in the upper bits.
// Entries for the extended window codes 30 and 31 are unused.
var offsetCombined = [32]uint32{}
// lengthCombined combines, for each match length (reduced by
// baseMatchLength), the length code (relative to lengthCodesStart) in
// bits 0-4, the number of extra bits in bits 5-7 and the value of the
// extra bits in bits 8-12.
var lengthCombined = func() (t [256]uint32) {
for i := range t {
code := lengthCodes[i]
t[i] = uint32(code) | uint32(lengthExtraBits[code])<<5 | uint32(uint8(i)-lengthBase[code])<<8
}
return t
}()
func init() {
var offsetBase = [32]uint32{
/* normal deflate */
@@ -85,7 +96,7 @@ func init() {
for i := range offsetCombined[:] {
// Don't use extended window values...
if offsetExtraBits[i] == 0 || offsetBase[i] > 0x006000 {
if offsetBase[i] > 0x006000 {
continue
}
offsetCombined[i] = uint32(offsetExtraBits[i]) | (offsetBase[i] << 8)
@@ -115,10 +126,14 @@ type huffmanBitWriter struct {
lastHeader int
// Set between 0 (reused block can be up to 2x the size)
logNewTablePenalty uint
bytes [256 + 8]byte
literalFreq [lengthCodesStart + 32]uint16
offsetFreq [32]uint16
codegenFreq [codegenCodeCount]uint16
// bytes must hold at least bufferFlushSize+7+8 bytes (see bufferFlushSize).
// Its size is rounded up to a multiple of 8 to keep the following
// fields aligned, which is measurably faster in writeBlockHuff.
bytes [(bufferFlushSize + 7 + 8 + 7) &^ 7]byte
literalFreq [lengthCodesStart + 32]uint16
offsetFreq [32]uint16
codegenFreq [codegenCodeCount]uint16
// codegen must have an extra space for the final symbol.
codegen [literalCount + offsetCodeCount + 1]uint8
@@ -842,124 +857,75 @@ func (w *huffmanBitWriter) writeTokens(tokens []token, leCodes, oeCodes []hcode)
lengths := leCodes[lengthCodesStart:]
lengths = lengths[:32]
// Go 1.16 LOVES having these on stack.
// Keeping these on the stack instead of in w is significantly faster.
bits, nbits, nbytes := w.bits, w.nbits, w.nbytes
for _, t := range tokens {
// Flush whole bytes, so that nbits <= 7 when entering the loop below.
le.Store64(w.bytes[:], nbytes, bits)
nbytes += nbits >> 3
bits >>= nbits & 56
nbits &= 7
if nbytes >= bufferFlushSize {
_, w.err = w.writer.Write(w.bytes[:nbytes])
nbytes = 0
if w.err != nil {
return
}
}
// The loop below flushes whole bytes at the end of every iteration,
// so that nbits <= 7 at the top of every iteration.
// A literal adds at most 15 bits and a match at most
// 15+5+15+13 = 48 bits, so up to three literals or a single match
// can be added to the 64-bit accumulator without an intermediate flush.
// Unconditionally storing 8 bytes and advancing by the number of
// whole bytes avoids a poorly predicted branch per token.
for i := 0; i < len(tokens); i++ {
t := tokens[i]
if t < 256 {
//w.writeCode(lits[t.literal()])
c := lits[t]
bits |= c.code64() << (nbits & 63)
nbits += c.len()
if nbits >= 48 {
le.Store64(w.bytes[:], nbytes, bits)
bits >>= 48
nbits -= 48
nbytes += 6
if nbytes >= bufferFlushSize {
if w.err != nil {
nbytes = 0
return
}
_, w.err = w.writer.Write(w.bytes[:nbytes])
nbytes = 0
// Add up to two more literals before flushing.
if i+1 < len(tokens) && tokens[i+1] < 256 {
i++
c := lits[tokens[i]]
bits |= c.code64() << (nbits & 63)
nbits += c.len()
if i+1 < len(tokens) && tokens[i+1] < 256 {
i++
c := lits[tokens[i]]
bits |= c.code64() << (nbits & 63)
nbits += c.len()
}
}
continue
}
// Write the length
length := t.length()
lengthCode := lengthCode(length) & 31
if false {
w.writeCode(lengths[lengthCode])
} else {
// inlined
c := lengths[lengthCode]
bits |= c.code64() << (nbits & 63)
nbits += c.len()
if nbits >= 48 {
le.Store64(w.bytes[:], nbytes, bits)
bits >>= 48
nbits -= 48
nbytes += 6
if nbytes >= bufferFlushSize {
if w.err != nil {
nbytes = 0
return
}
_, w.err = w.writer.Write(w.bytes[:nbytes])
nbytes = 0
}
}
}
// Write the length code and its extra bits as one unit.
lc := lengthCombined[t.length()]
c := lengths[lc&31]
bits |= (c.code64() | uint64(lc>>8)<<(c.len()&63)) << (nbits & 63)
nbits += c.len() + uint8(lc>>5)&7
if lengthCode >= lengthExtraBitsMinCode {
extraLengthBits := lengthExtraBits[lengthCode]
//w.writeBits(extraLength, extraLengthBits)
extraLength := int32(length - lengthBase[lengthCode])
bits |= uint64(extraLength) << (nbits & 63)
nbits += extraLengthBits
if nbits >= 48 {
le.Store64(w.bytes[:], nbytes, bits)
bits >>= 48
nbits -= 48
nbytes += 6
if nbytes >= bufferFlushSize {
if w.err != nil {
nbytes = 0
return
}
_, w.err = w.writer.Write(w.bytes[:nbytes])
nbytes = 0
}
}
// Write the offset code and its extra bits as one unit.
offset := t.offset()
offCode := (offset >> 16) & 31
c = offs[offCode]
offComb := offsetCombined[offCode]
extra := (offset - (offComb >> 8)) & matchOffsetOnlyMask
bits |= (c.code64() | uint64(extra)<<(c.len()&63)) << (nbits & 63)
nbits += c.len() + uint8(offComb)
}
// Write the offset
offset := t.offset()
offsetCode := (offset >> 16) & 31
if false {
w.writeCode(offs[offsetCode])
} else {
// inlined
c := offs[offsetCode]
bits |= c.code64() << (nbits & 63)
nbits += c.len()
if nbits >= 48 {
le.Store64(w.bytes[:], nbytes, bits)
bits >>= 48
nbits -= 48
nbytes += 6
if nbytes >= bufferFlushSize {
if w.err != nil {
nbytes = 0
return
}
_, w.err = w.writer.Write(w.bytes[:nbytes])
nbytes = 0
}
}
}
if offsetCode >= offsetExtraBitsMinCode {
offsetComb := offsetCombined[offsetCode]
//w.writeBits(extraOffset, extraOffsetBits)
bits |= uint64((offset-(offsetComb>>8))&matchOffsetOnlyMask) << (nbits & 63)
nbits += uint8(offsetComb)
if nbits >= 48 {
le.Store64(w.bytes[:], nbytes, bits)
bits >>= 48
nbits -= 48
nbytes += 6
if nbytes >= bufferFlushSize {
if w.err != nil {
nbytes = 0
return
}
_, w.err = w.writer.Write(w.bytes[:nbytes])
nbytes = 0
}
le.Store64(w.bytes[:], nbytes, bits)
nbytes += nbits >> 3
bits >>= nbits & 56
nbits &= 7
if nbytes >= bufferFlushSize {
if w.err != nil {
nbytes = 0
return
}
_, w.err = w.writer.Write(w.bytes[:nbytes])
nbytes = 0
}
}
// Restore...
@@ -1094,8 +1060,8 @@ func (w *huffmanBitWriter) writeBlockHuff(eof bool, input []byte, sync bool) {
if debugDeflate {
count -= int(nbytes)*8 + int(nbits)
}
// Unroll, write 3 codes/loop.
// Fastest number of unrolls.
// Write 3 codes per iteration. The three codes are combined first,
// independently of the accumulator, and merged with a single shift.
for len(input) > 3 {
// We must have at least 48 bits free.
if nbits >= 8 {
@@ -1116,13 +1082,13 @@ func (w *huffmanBitWriter) writeBlockHuff(eof bool, input []byte, sync bool) {
_, w.err = w.writer.Write(w.bytes[:nbytes])
nbytes = 0
}
a, b := encoding[input[0]], encoding[input[1]]
bits |= a.code64() << (nbits & 63)
bits |= b.code64() << ((nbits + a.len()) & 63)
c := encoding[input[2]]
nbits += b.len() + a.len()
bits |= c.code64() << (nbits & 63)
nbits += c.len()
a, b, c := encoding[input[0]], encoding[input[1]], encoding[input[2]]
v := a.code64() | b.code64()<<(a.len()&63)
n := a.len() + b.len()
v |= c.code64() << (n & 63)
n += c.len()
bits |= v << (nbits & 63)
nbits += n
input = input[3:]
}
+102 -36
View File
@@ -38,6 +38,9 @@ type huffmanEncoder struct {
// Possible lengths are codegenCodeCount, offsetCodeCount and literalCount.
// The largest of these is literalCount, so we allocate for that case.
freqcache [literalCount + 1]literalNode
// sortcache is scratch space for sortByFreq.
sortcache [literalCount + 1]literalNode
}
type literalNode struct {
@@ -139,14 +142,6 @@ func (h *huffmanEncoder) bitLength(freq []uint16) int {
return total
}
func (h *huffmanEncoder) bitLengthRaw(b []byte) int {
var total int
for _, f := range b {
total += int(h.codes[f].len())
}
return total
}
// canReuseBits returns the number of bits or math.MaxInt32 if the encoder cannot be reused.
func (h *huffmanEncoder) canReuseBits(freq []uint16) int {
var total int
@@ -309,27 +304,93 @@ func (h *huffmanEncoder) bitCounts(list []literalNode, maxBits int32) []int32 {
}
// Look at the leaves and assign them a bit count and an encoding as specified
// in RFC 1951 3.2.2
func (h *huffmanEncoder) assignEncodingAndSize(bitCount []int32, list []literalNode) {
code := uint16(0)
// in RFC 1951 3.2.2.
// list must be sorted by increasing frequency and codes must have
// all entries for literals not in list set to zero.
func (h *huffmanEncoder) assignEncodingAndSize(bitCount []int32, list []literalNode, codes []hcode) {
// The literals list[len(list)-bits] .. list[len(list)-1]
// are encoded using "bits" bits. Record the length of each literal
// temporarily in codes.
for n, bits := range bitCount {
code <<= 1
if n == 0 || bits == 0 {
continue
}
// The literals list[len(list)-bits] .. list[len(list)-bits]
// are encoded using "bits" bits, and get the values
// code, code + 1, .... The code values are
// assigned in literal order (not frequency order).
chunk := list[len(list)-int(bits):]
sortByLiteral(chunk)
for _, node := range chunk {
h.codes[node.literal] = newhcode(reverseBits(code, uint8(n)), uint8(n))
code++
for _, node := range list[len(list)-int(bits):] {
codes[node.literal] = hcode(n)
}
list = list[0 : len(list)-int(bits)]
}
// Code values are assigned in literal order (not frequency order)
// within each code length, starting from the first code of each length.
var nextCode [maxBitsLimit]uint16
code := uint16(0)
for n := 1; n < len(bitCount); n++ {
code = (code + uint16(bitCount[n-1])) << 1
nextCode[n] = code
}
for i, c := range codes {
if c == 0 {
continue
}
// n is at most maxBitsLimit-1; the mask only elides bounds checks.
n := uint8(c)
codes[i] = newhcode(reverseBits(nextCode[n&15], n), n)
nextCode[n&15]++
}
}
// sortByFreq sorts list by increasing frequency.
// The sort is stable, so nodes with equal frequencies keep their order.
func (h *huffmanEncoder) sortByFreq(list []literalNode) {
if len(list) <= 32 {
// Insertion sort is faster for small lists.
for i := 1; i < len(list); i++ {
n := list[i]
j := i
for ; j > 0 && list[j-1].freq > n.freq; j-- {
list[j] = list[j-1]
}
list[j] = n
}
return
}
// Counting sort on the low byte and then on the high byte
// of the frequency. Both passes are stable.
tmp := h.sortcache[:len(list)]
var count [256]uint16
var maxFreq uint16
for _, n := range list {
count[n.freq&0xff]++
maxFreq |= n.freq
}
var pos uint16
for i, c := range count {
count[i] = pos
pos += c
}
for _, n := range list {
tmp[count[n.freq&0xff]] = n
count[n.freq&0xff]++
}
if maxFreq < 256 {
copy(list, tmp)
return
}
count = [256]uint16{}
for _, n := range tmp {
count[n.freq>>8]++
}
pos = 0
for i, c := range count {
count[i] = pos
pos += c
}
for _, n := range tmp {
list[count[n.freq>>8]] = n
count[n.freq>>8]++
}
}
// Update this Huffman Code object to be the minimum code for the specified frequency count.
@@ -362,12 +423,14 @@ func (h *huffmanEncoder) generate(freq []uint16, maxBits int32) {
}
return
}
sortByFreq(list)
// list is in order of increasing literal value, so a stable sort
// by frequency orders it by (frequency, literal).
h.sortByFreq(list)
// Get the number of literals for each bit count
bitCount := h.bitCounts(list, maxBits)
// And do the assignment
h.assignEncodingAndSize(bitCount, list)
h.assignEncodingAndSize(bitCount, list, codes)
}
// atLeastOne clamps the result between 1 and 15.
@@ -394,24 +457,27 @@ func histogram(b []byte, h []uint16) {
}
func histogramSplit(b []byte, h []uint16) {
// Tested, and slightly faster than 2-way.
// Writing to separate arrays and combining is also slightly slower.
h = h[:256]
// Walk four quarters in parallel, each into its own table.
// Tested to be faster than walking halves. Using separate tables
// avoids the load/store dependencies that arise when two of the
// streams hit the same byte value, which is common for text.
var h0, h1, h2, h3 [256]uint16
// Make size divisible by 4
for len(b)&3 != 0 {
h[b[0]]++
h0[b[0]]++
b = b[1:]
}
n := len(b) / 4
x, y, z, w := b[:n], b[n:], b[n+n:], b[n+n+n:]
y, z, w = y[:len(x)], z[:len(x)], w[:len(x)]
for i, t := range x {
v0 := &h[t]
v1 := &h[y[i]]
v2 := &h[z[i]]
v3 := &h[w[i]]
*v0++
*v1++
*v2++
*v3++
h0[t]++
h1[y[i]]++
h2[z[i]]++
h3[w[i]]++
}
h = h[:256]
for i := range h {
h[i] += h0[i] + h1[i] + h2[i] + h3[i]
}
}
-159
View File
@@ -1,159 +0,0 @@
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package flate
// Sort sorts data.
// It makes one call to data.Len to determine n, and O(n*log(n)) calls to
// data.Less and data.Swap. The sort is not guaranteed to be stable.
func sortByFreq(data []literalNode) {
n := len(data)
quickSortByFreq(data, 0, n, maxDepth(n))
}
func quickSortByFreq(data []literalNode, a, b, maxDepth int) {
for b-a > 12 { // Use ShellSort for slices <= 12 elements
if maxDepth == 0 {
heapSort(data, a, b)
return
}
maxDepth--
mlo, mhi := doPivotByFreq(data, a, b)
// Avoiding recursion on the larger subproblem guarantees
// a stack depth of at most lg(b-a).
if mlo-a < b-mhi {
quickSortByFreq(data, a, mlo, maxDepth)
a = mhi // i.e., quickSortByFreq(data, mhi, b)
} else {
quickSortByFreq(data, mhi, b, maxDepth)
b = mlo // i.e., quickSortByFreq(data, a, mlo)
}
}
if b-a > 1 {
// Do ShellSort pass with gap 6
// It could be written in this simplified form cause b-a <= 12
for i := a + 6; i < b; i++ {
if data[i].freq == data[i-6].freq && data[i].literal < data[i-6].literal || data[i].freq < data[i-6].freq {
data[i], data[i-6] = data[i-6], data[i]
}
}
insertionSortByFreq(data, a, b)
}
}
func doPivotByFreq(data []literalNode, lo, hi int) (midlo, midhi int) {
m := int(uint(lo+hi) >> 1) // Written like this to avoid integer overflow.
if hi-lo > 40 {
// Tukey's ``Ninther,'' median of three medians of three.
s := (hi - lo) / 8
medianOfThreeSortByFreq(data, lo, lo+s, lo+2*s)
medianOfThreeSortByFreq(data, m, m-s, m+s)
medianOfThreeSortByFreq(data, hi-1, hi-1-s, hi-1-2*s)
}
medianOfThreeSortByFreq(data, lo, m, hi-1)
// Invariants are:
// data[lo] = pivot (set up by ChoosePivot)
// data[lo < i < a] < pivot
// data[a <= i < b] <= pivot
// data[b <= i < c] unexamined
// data[c <= i < hi-1] > pivot
// data[hi-1] >= pivot
pivot := lo
a, c := lo+1, hi-1
for ; a < c && (data[a].freq == data[pivot].freq && data[a].literal < data[pivot].literal || data[a].freq < data[pivot].freq); a++ {
}
b := a
for {
for ; b < c && (data[pivot].freq == data[b].freq && data[pivot].literal > data[b].literal || data[pivot].freq > data[b].freq); b++ { // data[b] <= pivot
}
for ; b < c && (data[pivot].freq == data[c-1].freq && data[pivot].literal < data[c-1].literal || data[pivot].freq < data[c-1].freq); c-- { // data[c-1] > pivot
}
if b >= c {
break
}
// data[b] > pivot; data[c-1] <= pivot
data[b], data[c-1] = data[c-1], data[b]
b++
c--
}
// If hi-c<3 then there are duplicates (by property of median of nine).
// Let's be a bit more conservative, and set border to 5.
protect := hi-c < 5
if !protect && hi-c < (hi-lo)/4 {
// Lets test some points for equality to pivot
dups := 0
if data[pivot].freq == data[hi-1].freq && data[pivot].literal > data[hi-1].literal || data[pivot].freq > data[hi-1].freq { // data[hi-1] = pivot
data[c], data[hi-1] = data[hi-1], data[c]
c++
dups++
}
if data[b-1].freq == data[pivot].freq && data[b-1].literal > data[pivot].literal || data[b-1].freq > data[pivot].freq { // data[b-1] = pivot
b--
dups++
}
// m-lo = (hi-lo)/2 > 6
// b-lo > (hi-lo)*3/4-1 > 8
// ==> m < b ==> data[m] <= pivot
if data[m].freq == data[pivot].freq && data[m].literal > data[pivot].literal || data[m].freq > data[pivot].freq { // data[m] = pivot
data[m], data[b-1] = data[b-1], data[m]
b--
dups++
}
// if at least 2 points are equal to pivot, assume skewed distribution
protect = dups > 1
}
if protect {
// Protect against a lot of duplicates
// Add invariant:
// data[a <= i < b] unexamined
// data[b <= i < c] = pivot
for {
for ; a < b && (data[b-1].freq == data[pivot].freq && data[b-1].literal > data[pivot].literal || data[b-1].freq > data[pivot].freq); b-- { // data[b] == pivot
}
for ; a < b && (data[a].freq == data[pivot].freq && data[a].literal < data[pivot].literal || data[a].freq < data[pivot].freq); a++ { // data[a] < pivot
}
if a >= b {
break
}
// data[a] == pivot; data[b-1] < pivot
data[a], data[b-1] = data[b-1], data[a]
a++
b--
}
}
// Swap pivot into middle
data[pivot], data[b-1] = data[b-1], data[pivot]
return b - 1, c
}
// Insertion sort
func insertionSortByFreq(data []literalNode, a, b int) {
for i := a + 1; i < b; i++ {
for j := i; j > a && (data[j].freq == data[j-1].freq && data[j].literal < data[j-1].literal || data[j].freq < data[j-1].freq); j-- {
data[j], data[j-1] = data[j-1], data[j]
}
}
}
// quickSortByFreq, loosely following Bentley and McIlroy,
// ``Engineering a Sort Function,'' SP&E November 1993.
// medianOfThreeSortByFreq moves the median of the three values data[m0], data[m1], data[m2] into data[m1].
func medianOfThreeSortByFreq(data []literalNode, m1, m0, m2 int) {
// sort 3 elements
if data[m1].freq == data[m0].freq && data[m1].literal < data[m0].literal || data[m1].freq < data[m0].freq {
data[m1], data[m0] = data[m0], data[m1]
}
// data[m0] <= data[m1]
if data[m2].freq == data[m1].freq && data[m2].literal < data[m1].literal || data[m2].freq < data[m1].freq {
data[m2], data[m1] = data[m1], data[m2]
// data[m0] <= data[m2] && data[m1] < data[m2]
if data[m1].freq == data[m0].freq && data[m1].literal < data[m0].literal || data[m1].freq < data[m0].freq {
data[m1], data[m0] = data[m0], data[m1]
}
}
// now data[m0] <= data[m1] <= data[m2]
}
-201
View File
@@ -1,201 +0,0 @@
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package flate
// Sort sorts data.
// It makes one call to data.Len to determine n, and O(n*log(n)) calls to
// data.Less and data.Swap. The sort is not guaranteed to be stable.
func sortByLiteral(data []literalNode) {
n := len(data)
quickSort(data, 0, n, maxDepth(n))
}
func quickSort(data []literalNode, a, b, maxDepth int) {
for b-a > 12 { // Use ShellSort for slices <= 12 elements
if maxDepth == 0 {
heapSort(data, a, b)
return
}
maxDepth--
mlo, mhi := doPivot(data, a, b)
// Avoiding recursion on the larger subproblem guarantees
// a stack depth of at most lg(b-a).
if mlo-a < b-mhi {
quickSort(data, a, mlo, maxDepth)
a = mhi // i.e., quickSort(data, mhi, b)
} else {
quickSort(data, mhi, b, maxDepth)
b = mlo // i.e., quickSort(data, a, mlo)
}
}
if b-a > 1 {
// Do ShellSort pass with gap 6
// It could be written in this simplified form cause b-a <= 12
for i := a + 6; i < b; i++ {
if data[i].literal < data[i-6].literal {
data[i], data[i-6] = data[i-6], data[i]
}
}
insertionSort(data, a, b)
}
}
func heapSort(data []literalNode, a, b int) {
first := a
lo := 0
hi := b - a
// Build heap with greatest element at top.
for i := (hi - 1) / 2; i >= 0; i-- {
siftDown(data, i, hi, first)
}
// Pop elements, largest first, into end of data.
for i := hi - 1; i >= 0; i-- {
data[first], data[first+i] = data[first+i], data[first]
siftDown(data, lo, i, first)
}
}
// siftDown implements the heap property on data[lo, hi).
// first is an offset into the array where the root of the heap lies.
func siftDown(data []literalNode, lo, hi, first int) {
root := lo
for {
child := 2*root + 1
if child >= hi {
break
}
if child+1 < hi && data[first+child].literal < data[first+child+1].literal {
child++
}
if data[first+root].literal > data[first+child].literal {
return
}
data[first+root], data[first+child] = data[first+child], data[first+root]
root = child
}
}
func doPivot(data []literalNode, lo, hi int) (midlo, midhi int) {
m := int(uint(lo+hi) >> 1) // Written like this to avoid integer overflow.
if hi-lo > 40 {
// Tukey's ``Ninther,'' median of three medians of three.
s := (hi - lo) / 8
medianOfThree(data, lo, lo+s, lo+2*s)
medianOfThree(data, m, m-s, m+s)
medianOfThree(data, hi-1, hi-1-s, hi-1-2*s)
}
medianOfThree(data, lo, m, hi-1)
// Invariants are:
// data[lo] = pivot (set up by ChoosePivot)
// data[lo < i < a] < pivot
// data[a <= i < b] <= pivot
// data[b <= i < c] unexamined
// data[c <= i < hi-1] > pivot
// data[hi-1] >= pivot
pivot := lo
a, c := lo+1, hi-1
for ; a < c && data[a].literal < data[pivot].literal; a++ {
}
b := a
for {
for ; b < c && data[pivot].literal > data[b].literal; b++ { // data[b] <= pivot
}
for ; b < c && data[pivot].literal < data[c-1].literal; c-- { // data[c-1] > pivot
}
if b >= c {
break
}
// data[b] > pivot; data[c-1] <= pivot
data[b], data[c-1] = data[c-1], data[b]
b++
c--
}
// If hi-c<3 then there are duplicates (by property of median of nine).
// Let's be a bit more conservative, and set border to 5.
protect := hi-c < 5
if !protect && hi-c < (hi-lo)/4 {
// Lets test some points for equality to pivot
dups := 0
if data[pivot].literal > data[hi-1].literal { // data[hi-1] = pivot
data[c], data[hi-1] = data[hi-1], data[c]
c++
dups++
}
if data[b-1].literal > data[pivot].literal { // data[b-1] = pivot
b--
dups++
}
// m-lo = (hi-lo)/2 > 6
// b-lo > (hi-lo)*3/4-1 > 8
// ==> m < b ==> data[m] <= pivot
if data[m].literal > data[pivot].literal { // data[m] = pivot
data[m], data[b-1] = data[b-1], data[m]
b--
dups++
}
// if at least 2 points are equal to pivot, assume skewed distribution
protect = dups > 1
}
if protect {
// Protect against a lot of duplicates
// Add invariant:
// data[a <= i < b] unexamined
// data[b <= i < c] = pivot
for {
for ; a < b && data[b-1].literal > data[pivot].literal; b-- { // data[b] == pivot
}
for ; a < b && data[a].literal < data[pivot].literal; a++ { // data[a] < pivot
}
if a >= b {
break
}
// data[a] == pivot; data[b-1] < pivot
data[a], data[b-1] = data[b-1], data[a]
a++
b--
}
}
// Swap pivot into middle
data[pivot], data[b-1] = data[b-1], data[pivot]
return b - 1, c
}
// Insertion sort
func insertionSort(data []literalNode, a, b int) {
for i := a + 1; i < b; i++ {
for j := i; j > a && data[j].literal < data[j-1].literal; j-- {
data[j], data[j-1] = data[j-1], data[j]
}
}
}
// maxDepth returns a threshold at which quicksort should switch
// to heapsort. It returns 2*ceil(lg(n+1)).
func maxDepth(n int) int {
var depth int
for i := n; i > 0; i >>= 1 {
depth++
}
return depth * 2
}
// medianOfThree moves the median of the three values data[m0], data[m1], data[m2] into data[m1].
func medianOfThree(data []literalNode, m1, m0, m2 int) {
// sort 3 elements
if data[m1].literal < data[m0].literal {
data[m1], data[m0] = data[m0], data[m1]
}
// data[m0] <= data[m1]
if data[m2].literal < data[m1].literal {
data[m2], data[m1] = data[m1], data[m2]
// data[m0] <= data[m2] && data[m1] < data[m2]
if data[m1].literal < data[m0].literal {
data[m1], data[m0] = data[m0], data[m1]
}
}
// now data[m0] <= data[m1] <= data[m2]
}
+15 -15
View File
@@ -120,7 +120,7 @@ readLiteral:
if debugDecode {
fmt.Println("morebits n>0:", err)
}
f.err = err
f.err = noEOF(err)
return
}
f.roffset++
@@ -148,7 +148,7 @@ readLiteral:
if debugDecode {
fmt.Println("morebits f.nb<5:", err)
}
f.err = err
f.err = noEOF(err)
return
}
f.roffset++
@@ -216,7 +216,7 @@ readLiteral:
if debugDecode {
fmt.Println("morebits f.nb<nb:", err)
}
f.err = err
f.err = noEOF(err)
return
}
f.roffset++
@@ -371,7 +371,7 @@ readLiteral:
if debugDecode {
fmt.Println("morebits n>0:", err)
}
f.err = err
f.err = noEOF(err)
return
}
f.roffset++
@@ -399,7 +399,7 @@ readLiteral:
if debugDecode {
fmt.Println("morebits f.nb<5:", err)
}
f.err = err
f.err = noEOF(err)
return
}
f.roffset++
@@ -467,7 +467,7 @@ readLiteral:
if debugDecode {
fmt.Println("morebits f.nb<nb:", err)
}
f.err = err
f.err = noEOF(err)
return
}
f.roffset++
@@ -638,7 +638,7 @@ readLiteral:
if debugDecode {
fmt.Println("morebits n>0:", err)
}
f.err = err
f.err = noEOF(err)
return
}
}
@@ -674,7 +674,7 @@ readLiteral:
if debugDecode {
fmt.Println("morebits f.nb<5:", err)
}
f.err = err
f.err = noEOF(err)
return
}
}
@@ -757,7 +757,7 @@ readLiteral:
if debugDecode {
fmt.Println("morebits f.nb<nb:", err)
}
f.err = err
f.err = noEOF(err)
return
}
}
@@ -918,7 +918,7 @@ readLiteral:
if debugDecode {
fmt.Println("morebits n>0:", err)
}
f.err = err
f.err = noEOF(err)
return
}
f.roffset++
@@ -946,7 +946,7 @@ readLiteral:
if debugDecode {
fmt.Println("morebits f.nb<5:", err)
}
f.err = err
f.err = noEOF(err)
return
}
f.roffset++
@@ -1014,7 +1014,7 @@ readLiteral:
if debugDecode {
fmt.Println("morebits f.nb<nb:", err)
}
f.err = err
f.err = noEOF(err)
return
}
f.roffset++
@@ -1169,7 +1169,7 @@ readLiteral:
if debugDecode {
fmt.Println("morebits n>0:", err)
}
f.err = err
f.err = noEOF(err)
return
}
f.roffset++
@@ -1197,7 +1197,7 @@ readLiteral:
if debugDecode {
fmt.Println("morebits f.nb<5:", err)
}
f.err = err
f.err = noEOF(err)
return
}
f.roffset++
@@ -1265,7 +1265,7 @@ readLiteral:
if debugDecode {
fmt.Println("morebits f.nb<nb:", err)
}
f.err = err
f.err = noEOF(err)
return
}
f.roffset++
+6 -7
View File
@@ -12,18 +12,17 @@ import (
// matchLen returns the maximum common prefix length of a and b.
// a must be the shortest of the two.
func matchLen(a, b []byte) (n int) {
left := len(a)
for left >= 8 {
diff := le.Load64(a, n) ^ le.Load64(b, n)
// Keeping a and b the same length and reslicing them as we go
// lets the compiler prove that no bounds checks are needed.
b = b[:len(a)]
for len(a) >= 8 {
diff := le.Load64(a, 0) ^ le.Load64(b, 0)
if diff != 0 {
return n + bits.TrailingZeros64(diff)>>3
}
a, b = a[8:], b[8:]
n += 8
left -= 8
}
a = a[n:]
b = b[n:]
for i := range a {
if a[i] != b[i] {
break
+11 -7
View File
@@ -215,7 +215,11 @@ func mFastLog2(val float32) float32 {
ux &= -0x7f800001
ux += 127 << 23
uval := math.Float32frombits(uint32(ux))
log2 += ((-0.34484843)*uval+2.02466578)*uval - 0.67487759
p := float32(-0.34484843*uval) + 2.02466578
q := float32(p*uval) - 0.67487759
log2 += q
return log2
}
@@ -232,7 +236,8 @@ func (t *tokens) EstimatedBits() int {
for _, v := range t.litHist[:] {
if v > 0 {
n := float32(v)
shannon += atLeastOne(-mFastLog2(n*invTotal)) * n
w := float32(atLeastOne(-mFastLog2(n*invTotal)) * n)
shannon += w
}
}
// Just add 15 for EOB
@@ -240,7 +245,8 @@ func (t *tokens) EstimatedBits() int {
for i, v := range t.extraHist[1 : literalCount-256] {
if v > 0 {
n := float32(v)
shannon += atLeastOne(-mFastLog2(n*invTotal)) * n
w := float32(atLeastOne(-mFastLog2(n*invTotal)) * n)
shannon += w
bits += int(lengthExtraBits[i&31]) * int(v)
nMatches += int(v)
}
@@ -251,7 +257,8 @@ func (t *tokens) EstimatedBits() int {
for i, v := range t.offHist[:offsetCodeCount] {
if v > 0 {
n := float32(v)
shannon += atLeastOne(-mFastLog2(n*invTotal)) * n
w := float32(atLeastOne(-mFastLog2(n*invTotal)) * n)
shannon += w
bits += int(offsetExtraBits[i&31]) * int(v)
}
}
@@ -358,9 +365,6 @@ func (t token) offset() uint32 { return uint32(t) & offsetMask }
func (t token) length() uint8 { return uint8(t >> lengthShift) }
// Convert length to code.
func lengthCode(len uint8) uint8 { return lengthCodes[len] }
// Returns the offset code corresponding to a specific offset
func offsetCode(off uint32) uint32 {
if false {
+63
View File
@@ -9,6 +9,7 @@ import (
"errors"
"fmt"
"io"
"math/bits"
"github.com/klauspost/compress/internal/le"
)
@@ -210,6 +211,68 @@ func (b *bitReaderShifted) remaining() uint {
return b.off*8 + uint(64-b.bitsRead)
}
// canUseAsm reports whether the reader has a full 8-byte window ahead of
// its read pointer, which the Decompress4X asm loops need to take over.
func (b *bitReaderShifted) canUseAsm() bool {
return b.off >= 8
}
// prepareForAsm establishes the invariant the Decompress4X asm loops rely
// on: value == load64(in[off:off+8]) << bitsRead with bitsRead <= 7, so
// that a full group of symbols can never shift their sentinel bit out of
// the container. init leaves bitsRead == 8 when the final byte of the
// stream is exactly 0x01; that whole byte is consumed, so the same position
// is the window one byte lower with nothing consumed. Requires canUseAsm.
func (b *bitReaderShifted) prepareForAsm() {
if b.bitsRead >= 8 {
b.off--
b.value = le.Load64(b.in, b.off)
b.bitsRead -= 8
}
}
// restoreFromAsm converts the state left behind by the Decompress4X asm
// loops back to the invariant the Go code relies on: value holds the 8
// bytes at in[off:off+8] shifted left by bitsRead, with the low bitsRead
// bits zero.
//
// The asm keeps a sentinel bit in value just below the unread bits, so its
// trailing zero count is the number of consumed bits. The sentinel is ORed
// over the lowest bit of the window, which the loop never consumes before
// re-reading memory, so the window is re-read here too rather than taken
// from value. The asm reports off as the signed distance from the start of
// the stream, and it can be negative: a reload always reads a whole 8-byte
// window, so a stream that is nearly drained ends with up to 7 bytes of the
// previous stream (or the jump table) below its start inside the window.
// Those bytes sit below the stream's own bits and count as consumed.
// Anything further below, or more bits consumed than the stream holds, is
// corruption.
func (b *bitReaderShifted) restoreFromAsm() error {
off := int(b.off)
consumed := uint(bits.TrailingZeros64(b.value))
if off < 0 {
if off < -7 {
return errors.New("corruption detected: stream underrun")
}
consumed += uint(-off) * 8
if consumed > 64 {
return errors.New("corruption detected: stream underrun")
}
off = 0
}
if off+8 > len(b.in) {
return errors.New("corruption detected: stream overrun")
}
b.off = uint(off)
b.bitsRead = uint8(consumed)
if consumed >= 64 {
b.value = 0
} else {
b.value = le.Load64(b.in, b.off) << consumed
}
return nil
}
// close the bitstream and returns an error if out-of-buffer reads occurred.
func (b *bitReaderShifted) close() error {
// Release reference.
+87 -8
View File
@@ -8,36 +8,99 @@ import (
)
// decompress4x_main_loop_amd64 is an x86 assembler implementation
// of Decompress4X when tablelog > 8.
// of Decompress4X when tablelog > 8, decoding fastSymbols symbols per
// stream between bit container reloads.
//
//go:noescape
func decompress4x_main_loop_amd64(ctx *decompress4xContext)
// decompress4x_8b_main_loop_amd64 is an x86 assembler implementation
// of Decompress4X when tablelog <= 8 which decodes 4 entries
// per loop.
// of Decompress4X when tablelog <= 8, decoding fast8bSymbols symbols
// per stream between bit container reloads.
//
//go:noescape
func decompress4x_8b_main_loop_amd64(ctx *decompress4xContext)
// decompress4x_4b_main_loop_amd64 is an x86 assembler implementation
// of Decompress4X when tablelog <= 4, decoding fast4bSymbols symbols
// per stream between bit container reloads.
//
//go:noescape
func decompress4x_4b_main_loop_amd64(ctx *decompress4xContext)
// decompress4x_4b_main_loop_bmi2 is the BMI2 twin of decompress4x_4b_main_loop_amd64.
//
//go:noescape
func decompress4x_4b_main_loop_bmi2(ctx *decompress4xContext)
// decompress4x_main_loop_bmi2 is the BMI2 twin of decompress4x_main_loop_amd64.
//
//go:noescape
func decompress4x_main_loop_bmi2(ctx *decompress4xContext)
// decompress4x_8b_main_loop_bmi2 is the BMI2 twin of decompress4x_8b_main_loop_amd64.
//
//go:noescape
func decompress4x_8b_main_loop_bmi2(ctx *decompress4xContext)
// decompress1x_main_loop_amd64 is an x86 assembler implementation
// of Decompress1X when tablelog > 8.
// of Decompress1X when tablelog > 8, decoding fastSymbols symbols
// between bit container reloads.
//
//go:noescape
func decompress1x_main_loop_amd64(ctx *decompress1xContext)
// decompress1x_main_loop_bmi2 is an x86 with BMI2 assembler implementation
// of Decompress1X when tablelog > 8.
// decompress1x_8b_main_loop_amd64 is an x86 assembler implementation
// of Decompress1X when tablelog <= 8, decoding fast8bSymbols symbols
// between bit container reloads.
//
//go:noescape
func decompress1x_8b_main_loop_amd64(ctx *decompress1xContext)
// decompress1x_4b_main_loop_amd64 is an x86 assembler implementation
// of Decompress1X when tablelog <= 4, decoding fast4bSymbols symbols
// between bit container reloads.
//
//go:noescape
func decompress1x_4b_main_loop_amd64(ctx *decompress1xContext)
// decompress1x_main_loop_bmi2 is the BMI2 twin of decompress1x_main_loop_amd64.
//
//go:noescape
func decompress1x_main_loop_bmi2(ctx *decompress1xContext)
// decompress1x_8b_main_loop_bmi2 is the BMI2 twin of decompress1x_8b_main_loop_amd64.
//
//go:noescape
func decompress1x_8b_main_loop_bmi2(ctx *decompress1xContext)
// decompress1x_4b_main_loop_bmi2 is the BMI2 twin of decompress1x_4b_main_loop_amd64.
//
//go:noescape
func decompress1x_4b_main_loop_bmi2(ctx *decompress1xContext)
func decompress4x_main_loop_asm(ctx *decompress4xContext) {
decompress4x_main_loop_amd64(ctx)
if cpuinfo.HasBMI2() {
decompress4x_main_loop_bmi2(ctx)
} else {
decompress4x_main_loop_amd64(ctx)
}
}
func decompress4x_8b_main_loop_asm(ctx *decompress4xContext) {
decompress4x_8b_main_loop_amd64(ctx)
if cpuinfo.HasBMI2() {
decompress4x_8b_main_loop_bmi2(ctx)
} else {
decompress4x_8b_main_loop_amd64(ctx)
}
}
func decompress4x_4b_main_loop_asm(ctx *decompress4xContext) {
if cpuinfo.HasBMI2() {
decompress4x_4b_main_loop_bmi2(ctx)
} else {
decompress4x_4b_main_loop_amd64(ctx)
}
}
func decompress1x_main_loop_asm(ctx *decompress1xContext) {
@@ -47,3 +110,19 @@ func decompress1x_main_loop_asm(ctx *decompress1xContext) {
decompress1x_main_loop_amd64(ctx)
}
}
func decompress1x_8b_main_loop_asm(ctx *decompress1xContext) {
if cpuinfo.HasBMI2() {
decompress1x_8b_main_loop_bmi2(ctx)
} else {
decompress1x_8b_main_loop_amd64(ctx)
}
}
func decompress1x_4b_main_loop_asm(ctx *decompress1xContext) {
if cpuinfo.HasBMI2() {
decompress1x_4b_main_loop_bmi2(ctx)
} else {
decompress1x_4b_main_loop_amd64(ctx)
}
}
File diff suppressed because it is too large. Load diff
+39 -4
View File
@@ -6,24 +6,47 @@
package huff0
// decompress4x_main_loop_arm64 is an arm64 assembler implementation
// of Decompress4X when tablelog > 8.
// of Decompress4X when tablelog > 8, decoding fastSymbols symbols per
// stream between bit container reloads.
//
//go:noescape
func decompress4x_main_loop_arm64(ctx *decompress4xContext)
// decompress4x_8b_main_loop_arm64 is an arm64 assembler implementation
// of Decompress4X when tablelog <= 8 which decodes 4 entries
// per loop.
// of Decompress4X when tablelog <= 8, decoding fast8bSymbols symbols
// per stream between bit container reloads.
//
//go:noescape
func decompress4x_8b_main_loop_arm64(ctx *decompress4xContext)
// decompress4x_4b_main_loop_arm64 is an arm64 assembler implementation
// of Decompress4X when tablelog <= 4, decoding fast4bSymbols symbols
// per stream between bit container reloads.
//
//go:noescape
func decompress4x_4b_main_loop_arm64(ctx *decompress4xContext)
// decompress1x_main_loop_arm64 is an arm64 assembler implementation
// of Decompress1X when tablelog > 8.
// of Decompress1X when tablelog > 8, decoding fastSymbols symbols
// between bit container reloads.
//
//go:noescape
func decompress1x_main_loop_arm64(ctx *decompress1xContext)
// decompress1x_8b_main_loop_arm64 is an arm64 assembler implementation
// of Decompress1X when tablelog <= 8, decoding fast8bSymbols symbols
// between bit container reloads.
//
//go:noescape
func decompress1x_8b_main_loop_arm64(ctx *decompress1xContext)
// decompress1x_4b_main_loop_arm64 is an arm64 assembler implementation
// of Decompress1X when tablelog <= 4, decoding fast4bSymbols symbols
// between bit container reloads.
//
//go:noescape
func decompress1x_4b_main_loop_arm64(ctx *decompress1xContext)
func decompress4x_main_loop_asm(ctx *decompress4xContext) {
decompress4x_main_loop_arm64(ctx)
}
@@ -32,6 +55,18 @@ func decompress4x_8b_main_loop_asm(ctx *decompress4xContext) {
decompress4x_8b_main_loop_arm64(ctx)
}
func decompress4x_4b_main_loop_asm(ctx *decompress4xContext) {
decompress4x_4b_main_loop_arm64(ctx)
}
func decompress1x_main_loop_asm(ctx *decompress1xContext) {
decompress1x_main_loop_arm64(ctx)
}
func decompress1x_8b_main_loop_asm(ctx *decompress1xContext) {
decompress1x_8b_main_loop_arm64(ctx)
}
func decompress1x_4b_main_loop_asm(ctx *decompress1xContext) {
decompress1x_4b_main_loop_arm64(ctx)
}
File diff suppressed because it is too large. Load diff
+79 -19
View File
@@ -14,6 +14,10 @@ import (
// fallback8BitSize is the size where using Go version is faster.
const fallback8BitSize = 800
// decompress4xContext is the argument block of the Decompress4X asm loops.
// Go fills every field but decoded and inner; the asm advances ip, and on
// return leaves each bit reader in the form bitReaderShifted.restoreFromAsm
// expects.
type decompress4xContext struct {
pbr *[4]bitReaderShifted
peekBits uint8
@@ -21,9 +25,20 @@ type decompress4xContext struct {
dstEvery int
tbl *dEntrySingle
decoded int
limit *byte
limit *byte // stream 0's output pointer must stay below this
ilowest *byte // start of the input block; no read goes below it
ip [4]*byte // each stream's 8-byte input window, advanced by the asm
inner *byte // scratch for the asm: the current inner-loop limit
}
// Symbols decoded per stream between reloads by the 4X asm loops; must
// match the constants of the same names in _generate/gen.go.
const (
fastSymbols = 5 // tablelog 9..11
fast8bSymbols = 7 // tablelog 5..8
fast4bSymbols = 14 // tablelog <= 4
)
// Decompress4X will decompress a 4X encoded stream.
// The length of the supplied input must match the end of a block exactly.
// The *capacity* of the dst slice must match the destination size of
@@ -72,23 +87,55 @@ func (d *Decoder) Decompress4X(dst, src []byte) ([]byte, error) {
var decoded int
if len(out) > 4*4 && !(br[0].off < 4 || br[1].off < 4 || br[2].off < 4 || br[3].off < 4) {
nSyms := fastSymbols
if d.actualTableLog <= 4 {
nSyms = fast4bSymbols
} else if use8BitTables {
nSyms = fast8bSymbols
}
// The asm writes nSyms bytes per stream per iteration and only re-checks
// its bounds between batches of iterations (the batch size is derived
// from limit in _generate/gen.go), so stream 0 must stop early enough
// that stream 3 (which may be up to 3 bytes shorter than dstEvery)
// never writes past the end of out. Every stream needs a full 8-byte
// window ahead of its read pointer to enter the loop.
if limit := dstEvery - nSyms - 2; limit > 0 && br[0].canUseAsm() && br[1].canUseAsm() && br[2].canUseAsm() && br[3].canUseAsm() {
for i := range br {
br[i].prepareForAsm()
}
ctx := decompress4xContext{
pbr: &br,
peekBits: uint8((64 - d.actualTableLog) & 63), // see: bitReaderShifted.peekBitsFast()
out: &out[0],
dstEvery: dstEvery,
tbl: &single[0],
limit: &out[dstEvery-4], // Always stop decoding when first buffer gets here to avoid writing OOB on last.
limit: &out[limit],
// The 6-byte jump table sits below stream 0 inside src, so the
// lowest window may reach into it; restoreFromAsm accounts for
// bytes below a stream's start. Bounding by src rather than by
// stream 0 keeps the asm running about six bytes longer, which
// matters for streams with very short codes.
ilowest: &src[0],
}
if use8BitTables {
for i := range br {
ctx.ip[i] = &br[i].in[br[i].off]
}
switch nSyms {
case fast4bSymbols:
decompress4x_4b_main_loop_asm(&ctx)
case fast8bSymbols:
decompress4x_8b_main_loop_asm(&ctx)
} else {
default:
decompress4x_main_loop_asm(&ctx)
}
decoded = ctx.decoded
out = out[decoded/4:]
for i := range br {
if err := br[i].restoreFromAsm(); err != nil {
return nil, err
}
}
}
// Decode remaining.
@@ -128,18 +175,20 @@ func (d *Decoder) Decompress4X(dst, src []byte) ([]byte, error) {
return dst, nil
}
// decompress1xContext is the argument block of the Decompress1X asm loops.
// Go fills every field but decoded; the asm advances ip, and on return
// leaves the bit reader in the form bitReaderShifted.restoreFromAsm expects.
type decompress1xContext struct {
pbr *bitReaderShifted
peekBits uint8
out *byte
outCap int
outCap int // no write reaches out[outCap]
tbl *dEntrySingle
decoded int
ilowest *byte // start of the stream; no read goes below it
ip *byte // the 8-byte input window, advanced by the asm
}
// Error reported by asm implementations
const error_max_decoded_size_exeeded = -1
// Decompress1X will decompress a 1X encoded stream.
// The cap of the output buffer will be the maximum decompressed size.
// The length of the supplied input must match the end of a block exactly.
@@ -158,25 +207,36 @@ func (d *Decoder) Decompress1X(dst, src []byte) ([]byte, error) {
const tlSize = 1 << tableLogMax
const tlMask = tlSize - 1
if maxDecodedSize >= 4 {
// The asm decodes whole batches of symbols and only re-checks its
// bounds between them, so it needs at least one batch of room (the
// output bound rounds nSyms up to 16, see _generate/gen.go) and a full
// 8-byte window ahead of the read pointer to enter the loop. It also
// needs at most 7 bits consumed on entry so that a batch cannot run
// the container dry, which prepareForAsm establishes.
decoded := 0
if maxDecodedSize >= 16 && br.canUseAsm() {
br.prepareForAsm()
ctx := decompress1xContext{
pbr: &br,
out: &dst[0],
outCap: maxDecodedSize,
peekBits: uint8((64 - d.actualTableLog) & 63), // see: bitReaderShifted.peekBitsFast()
tbl: &d.dt.single[0],
ilowest: &src[0],
ip: &br.in[br.off],
}
decompress1x_main_loop_asm(&ctx)
if ctx.decoded == error_max_decoded_size_exeeded {
return nil, ErrMaxDecodedSizeExceeded
if d.actualTableLog <= 4 {
decompress1x_4b_main_loop_asm(&ctx)
} else if d.actualTableLog <= 8 {
decompress1x_8b_main_loop_asm(&ctx)
} else {
decompress1x_main_loop_asm(&ctx)
}
dst = dst[:ctx.decoded]
decoded = ctx.decoded
}
dst = dst[:decoded]
// br < 8, so uint8 is fine
bitsLeft := uint8(br.off)*8 + 64 - br.bitsRead
bitsLeft := br.remaining()
for bitsLeft > 0 {
br.fill()
if len(dst) >= maxDecodedSize {
@@ -186,7 +246,7 @@ func (d *Decoder) Decompress1X(dst, src []byte) ([]byte, error) {
v := d.dt.single[br.peekBitsFast(d.actualTableLog)&tlMask]
nBits := uint8(v.entry)
br.advance(nBits)
bitsLeft -= nBits
bitsLeft -= uint(nBits)
dst = append(dst, uint8(v.entry>>8))
}
return dst, br.close()
+3
View File
@@ -240,9 +240,11 @@ func (b *blockDec) decodeBuf(hist *history) error {
b.dst[i] = v
}
hist.appendKeep(b.dst)
hist.decoders.consumeSyncLen(len(b.dst))
return nil
case blockTypeRaw:
hist.appendKeep(b.data)
hist.decoders.consumeSyncLen(len(b.data))
return nil
case blockTypeCompressed:
saved := b.dst
@@ -487,6 +489,7 @@ func (b *blockDec) decodeCompressed(hist *history) error {
}
if hist.decoders.nSeqs == 0 {
b.dst = append(b.dst, hist.decoders.literals...)
hist.decoders.consumeSyncLen(len(hist.decoders.literals))
return nil
}
before := len(hist.decoders.out)
+81 -44
View File
@@ -7,8 +7,11 @@ package zstd
import (
"context"
"encoding/binary"
"errors"
"io"
"maps"
"sync"
"sync/atomic"
"github.com/klauspost/compress/zstd/internal/xxhash"
)
@@ -20,7 +23,8 @@ import (
// A decoder can safely be re-used even if the previous stream failed.
// To release the resources, you must call the Close() function on a decoder.
type Decoder struct {
o decoderOptions
// publishedOptions contains the current immutable decoder options.
publishedOptions atomic.Pointer[decoderOptions]
// Unreferenced decoders, ready for use.
decoders chan *blockDec
@@ -84,9 +88,10 @@ var (
func NewReader(r io.Reader, opts ...DOption) (*Decoder, error) {
initPredefined()
var d Decoder
d.o.setDefault()
for _, o := range opts {
err := o(&d.o)
var o decoderOptions
o.setDefault()
for _, opt := range opts {
err := opt(&o)
if err != nil {
return nil, err
}
@@ -99,17 +104,19 @@ func NewReader(r io.Reader, opts ...DOption) (*Decoder, error) {
}
// Initialize dict map if needed.
if d.o.dicts == nil {
d.o.dicts = make(map[uint32]*dict)
if o.dicts == nil {
o.dicts = make(map[uint32]*dict)
}
// Create decoders
d.decoders = make(chan *blockDec, d.o.concurrent)
for i := 0; i < d.o.concurrent; i++ {
dec := newBlockDec(d.o.lowMem)
dec.localFrame = newFrameDec(d.o)
d.decoders = make(chan *blockDec, o.concurrent)
for i := 0; i < o.concurrent; i++ {
dec := newBlockDec(o.lowMem)
dec.localFrame = newFrameDec(o)
dec.localFrame.o.dicts = nil
d.decoders <- dec
}
d.publishedOptions.Store(&o)
if r == nil {
return &d, nil
@@ -171,6 +178,9 @@ func (d *Decoder) Reset(r io.Reader) error {
d.drainOutput()
d.syncStream.br.r = nil
if d.frame != nil {
d.frame.o.dicts = nil
}
if r == nil {
d.current.err = ErrDecoderNilInput
if len(d.current.b) > 0 {
@@ -180,8 +190,10 @@ func (d *Decoder) Reset(r io.Reader) error {
return nil
}
o := *d.publishedOptions.Load()
// If bytes buffer and < 5MB, do sync decoding anyway.
if bb, ok := r.(byter); ok && bb.Len() < d.o.decodeBufsBelow && !d.o.limitToCap {
if bb, ok := r.(byter); ok && bb.Len() < o.decodeBufsBelow && !o.limitToCap {
bb2 := bb
if debugDecoder {
println("*bytes.Buffer detected, doing sync decode, len:", bb.Len())
@@ -193,18 +205,30 @@ func (d *Decoder) Reset(r io.Reader) error {
}
dst, err := d.DecodeAll(b, dst)
if err == nil {
err = io.EOF
if !errors.Is(err, ErrDecoderSizeExceeded) {
if err == nil {
err = io.EOF
}
// Save output buffer
d.syncStream.dstBuf = dst
d.current.b = dst
d.current.err = err
d.current.flushed = true
if debugDecoder {
println("sync decode to", len(dst), "bytes, err:", err)
}
return nil
}
// Save output buffer
d.syncStream.dstBuf = dst
d.current.b = dst
d.current.err = err
d.current.flushed = true
// The output does not fit WithDecoderMaxMemory, which caps the total
// decoded size when decoding in memory. A Reader follows the
// streaming contract, where the option caps the window instead, so
// decode this input as a stream after all; the shortcut has not
// consumed the reader. Drop the oversized buffer rather than keep it
// for reuse.
if debugDecoder {
println("sync decode to", len(dst), "bytes, err:", err)
println("sync decode exceeded max memory after", len(dst), "bytes; streaming instead")
}
return nil
d.syncStream.dstBuf = nil
}
// Remove current block.
d.stashDecoder()
@@ -217,14 +241,16 @@ func (d *Decoder) Reset(r io.Reader) error {
// Ensure no-one else is still running...
d.streamWg.Wait()
if d.frame == nil {
d.frame = newFrameDec(d.o)
d.frame = newFrameDec(o)
} else {
d.frame.setOptions(o)
}
if d.o.concurrent == 1 {
if o.concurrent == 1 {
return d.startSyncDecoder(r)
}
d.current.output = make(chan decodeOutput, d.o.concurrent)
d.current.output = make(chan decodeOutput, o.concurrent)
ctx, cancel := context.WithCancel(context.Background())
d.current.cancel = cancel
d.streamWg.Add(1)
@@ -238,13 +264,21 @@ func (d *Decoder) Reset(r io.Reader) error {
// Options are applied on top of the existing options.
// Some options cannot be changed on reset and will return an error.
func (d *Decoder) ResetWithOptions(r io.Reader, opts ...DOption) error {
d.o.resetOpt = true
defer func() { d.o.resetOpt = false }()
if d.current.err == ErrDecoderClosed {
return d.current.err
}
next := *d.publishedOptions.Load()
next.dicts = maps.Clone(next.dicts)
next.resetOpt = true
for _, o := range opts {
if err := o(&d.o); err != nil {
if err := o(&next); err != nil {
next.resetOpt = false
d.publishedOptions.Store(&next)
return err
}
}
next.resetOpt = false
d.publishedOptions.Store(&next)
return d.Reset(r)
}
@@ -324,6 +358,8 @@ func (d *Decoder) DecodeAll(input, dst []byte) ([]byte, error) {
// Grab a block decoder and frame decoder.
block := <-d.decoders
frame := block.localFrame
o := *d.publishedOptions.Load()
frame.setOptions(o)
initialSize := len(dst)
defer func() {
if debugDecoder {
@@ -335,6 +371,7 @@ func (d *Decoder) DecodeAll(input, dst []byte) ([]byte, error) {
frame.history.decoders.br.in = nil
frame.history.decoders.br.cursor = 0
}
frame.o.dicts = nil
d.decoders <- block
}()
frame.bBuf = input
@@ -354,20 +391,20 @@ func (d *Decoder) DecodeAll(input, dst []byte) ([]byte, error) {
if err = d.setDict(frame); err != nil {
return nil, err
}
if frame.WindowSize > d.o.maxWindowSize {
if frame.WindowSize > frame.o.maxWindowSize {
if debugDecoder {
println("window size exceeded:", frame.WindowSize, ">", d.o.maxWindowSize)
println("window size exceeded:", frame.WindowSize, ">", frame.o.maxWindowSize)
}
return dst, ErrWindowSizeExceeded
}
if frame.FrameContentSize != fcsUnknown {
if frame.FrameContentSize > d.o.maxDecodedSize-uint64(len(dst)-initialSize) {
if frame.FrameContentSize > frame.o.maxDecodedSize-uint64(len(dst)-initialSize) {
if debugDecoder {
println("decoder size exceeded; fcs:", frame.FrameContentSize, "> mcs:", d.o.maxDecodedSize-uint64(len(dst)-initialSize), "len:", len(dst))
println("decoder size exceeded; fcs:", frame.FrameContentSize, "> mcs:", frame.o.maxDecodedSize-uint64(len(dst)-initialSize), "len:", len(dst))
}
return dst, ErrDecoderSizeExceeded
}
if d.o.limitToCap && frame.FrameContentSize > uint64(cap(dst)-len(dst)) {
if frame.o.limitToCap && frame.FrameContentSize > uint64(cap(dst)-len(dst)) {
if debugDecoder {
println("decoder size exceeded; fcs:", frame.FrameContentSize, "> (cap-len)", cap(dst)-len(dst))
}
@@ -380,14 +417,14 @@ func (d *Decoder) DecodeAll(input, dst []byte) ([]byte, error) {
}
}
if cap(dst) == 0 && !d.o.limitToCap {
if cap(dst) == 0 && !frame.o.limitToCap {
// Allocate len(input) * 2 by default if nothing is provided
// and we didn't get frame content size.
size := min(
// Cap to 1 MB.
len(input)*2, 1<<20)
if uint64(size) > d.o.maxDecodedSize {
size = int(d.o.maxDecodedSize)
if uint64(size) > frame.o.maxDecodedSize {
size = int(frame.o.maxDecodedSize)
}
dst = make([]byte, 0, size)
}
@@ -396,7 +433,7 @@ func (d *Decoder) DecodeAll(input, dst []byte) ([]byte, error) {
if err != nil {
return dst, err
}
if uint64(len(dst)-initialSize) > d.o.maxDecodedSize {
if uint64(len(dst)-initialSize) > frame.o.maxDecodedSize {
return dst, ErrDecoderSizeExceeded
}
if len(frame.bBuf) == 0 {
@@ -459,7 +496,7 @@ func (d *Decoder) nextBlock(blocking bool) (ok bool) {
println("got", len(d.current.b), "bytes, error:", d.current.err, "data crc:", tmp)
}
if d.o.ignoreChecksum {
if d.frame.o.ignoreChecksum {
return true
}
@@ -497,7 +534,7 @@ func (d *Decoder) nextBlockSync() (ok bool) {
if d.current.err != nil {
return false
}
if d.frame.WindowSize > d.o.maxDecodedSize || d.frame.WindowSize > d.o.maxWindowSize {
if d.frame.WindowSize > d.frame.o.maxDecodedSize || d.frame.WindowSize > d.frame.o.maxWindowSize {
d.current.err = ErrDecoderSizeExceeded
return false
}
@@ -546,11 +583,11 @@ func (d *Decoder) nextBlockSync() (ok bool) {
// Update/Check CRC
if d.frame.HasCheckSum {
if !d.o.ignoreChecksum {
if !d.frame.o.ignoreChecksum {
d.frame.crc.Write(d.current.b)
}
if d.current.d.Last {
if !d.o.ignoreChecksum {
if !d.frame.o.ignoreChecksum {
d.current.err = d.frame.checkCRC()
} else {
d.current.err = d.frame.consumeCRC()
@@ -656,8 +693,8 @@ func (d *Decoder) startStreamDecoder(ctx context.Context, r io.Reader, output ch
defer d.streamWg.Done()
br := readerWrapper{r: r}
var seqDecode = make(chan *blockDec, d.o.concurrent)
var seqExecute = make(chan *blockDec, d.o.concurrent)
var seqDecode = make(chan *blockDec, d.frame.o.concurrent)
var seqExecute = make(chan *blockDec, d.frame.o.concurrent)
// Async 1: Decode sequences...
go func() {
@@ -858,9 +895,9 @@ decodeStream:
if err == nil {
err = d.setDict(frame)
}
if err == nil && d.frame.WindowSize > d.o.maxWindowSize {
if err == nil && d.frame.WindowSize > d.frame.o.maxWindowSize {
if debugDecoder {
println("decoder size exceeded, fws:", d.frame.WindowSize, "> mws:", d.o.maxWindowSize)
println("decoder size exceeded, fws:", d.frame.WindowSize, "> mws:", d.frame.o.maxWindowSize)
}
err = ErrDecoderSizeExceeded
@@ -940,7 +977,7 @@ decodeStream:
}
func (d *Decoder) setDict(frame *frameDec) (err error) {
dict, ok := d.o.dicts[frame.DictionaryID]
dict, ok := frame.o.dicts[frame.DictionaryID]
if ok {
if debugDecoder {
println("setting dict", frame.DictionaryID)
+7 -4
View File
@@ -49,13 +49,16 @@ const (
)
func newFrameDec(o decoderOptions) *frameDec {
d := frameDec{}
d.setOptions(o)
return &d
}
func (d *frameDec) setOptions(o decoderOptions) {
if o.maxWindowSize > o.maxDecodedSize {
o.maxWindowSize = o.maxDecodedSize
}
d := frameDec{
o: o,
}
return &d
d.o = o
}
// reset will read the frame header and prepare for block decoding.
+4 -6
View File
@@ -47,12 +47,10 @@ init_main_loop_condition:
// Spread symbols
// Calculate table step
MOVD R0, R8
LSR $0x01, R8, R8
MOVD R0, R9
LSR $0x03, R9, R9
ADD R9, R8, R8
ADD $3, R8, R8
LSR $0x01, R0, R8
LSR $0x03, R0, R9
ADD R9, R8, R8
ADD $3, R8, R8
// Fill add bits values
SUB $1, R0, R9
+23
View File
@@ -99,6 +99,29 @@ func (s *sequenceDecs) initialize(br *bitReader, hist *history, out []byte) erro
return nil
}
// consumeSyncLen accounts for n bytes of frame output produced outside
// decodeSync. maxSyncLen bounds how much the frame may still produce and
// decodeSyncSimple compares it with the output buffer's slack to decide
// whether the extended 16-byte copies are safe. Blocks with sequences
// subtracted their output; raw blocks, RLE blocks and compressed blocks
// holding only literals did not, so a frame that opened with a raw block
// overstated its remaining size for every block after it and ran the
// bounds-exact copies for the rest of the frame.
//
// Zero means "no bound" and selects the conservative paths, so a block that
// meets or exceeds the bound lands there; the frame-size checks in
// runDecoder reject the excess afterwards.
func (s *sequenceDecs) consumeSyncLen(n int) {
if s.maxSyncLen == 0 {
return
}
if uint64(n) >= s.maxSyncLen {
s.maxSyncLen = 0
return
}
s.maxSyncLen -= uint64(n)
}
func (s *sequenceDecs) freeDecoders() {
if f := s.litLengths.fse; f != nil && !f.preDefined {
fseDecoderPool.Put(f)
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
+4 -3
View File
@@ -83,9 +83,10 @@ const errorOverread = 6
// When the output and literal buffers have compressedBlockOverAlloc (16)
// bytes of slack past their logical use, the assembly may use extended
// memory copies that read and write in 16-byte blocks, overrunning the end
// of a literal run or match by up to 15 bytes. Otherwise it must use the
// bounds-exact ("safe") copies. This mirrors the analogous, always-dynamic
// selection in executeSimple below.
// of a literal run or match by up to 15 bytes; an empty literal run still
// copies one block, reading and writing the full 16 bytes of slack past
// its position. Otherwise it must use the bounds-exact ("safe") copies.
// This mirrors the analogous, always-dynamic selection in executeSimple below.
//
// The unsafe copies were disabled in #644 (2022) as a mitigation for a
// crash, but that crash's root cause — an unguarded bitReader overread that
@@ -39,7 +39,7 @@ func init() {
// token = 1*<any CHAR except CTLs or separators>
// qdtext = <any TEXT except <">>
for c := 0; c < 256; c++ {
for c := range 256 {
var t octetType
isCtl := c <= 31 || c == 127
isChar := 0 <= c && c <= 127
@@ -32,7 +32,8 @@ func NewBuildInfoCollector() Collector {
"Build information about the main Go module.",
nil, Labels{"path": path, "version": version, "checksum": sum},
),
GaugeValue, 1)}
GaugeValue, 1,
)}
c.init(c.self)
return c
}
+2 -3
View File
@@ -21,7 +21,6 @@ import (
"github.com/cespare/xxhash/v2"
dto "github.com/prometheus/client_model/go"
"github.com/prometheus/common/model"
"google.golang.org/protobuf/proto"
"github.com/prometheus/client_golang/prometheus/internal"
)
@@ -181,8 +180,8 @@ func (v2) NewDesc(fqName, help string, variableLabels ConstrainableLabels, const
d.constLabelPairs = make([]*dto.LabelPair, 0, len(constLabels))
for n, v := range constLabels {
d.constLabelPairs = append(d.constLabelPairs, &dto.LabelPair{
Name: proto.String(n),
Value: proto.String(v),
Name: new(n),
Value: new(v),
})
}
sort.Sort(internal.LabelPairSorter(d.constLabelPairs))
-1
View File
@@ -12,7 +12,6 @@
// limitations under the License.
//go:build !js || wasm
// +build !js wasm
package prometheus
@@ -12,7 +12,6 @@
// limitations under the License.
//go:build js && !wasm
// +build js,!wasm
package prometheus
+10 -5
View File
@@ -211,23 +211,28 @@ func newBaseGoCollector() baseGoCollector {
goroutinesDesc: NewDesc(
"go_goroutines",
"Number of goroutines that currently exist.",
nil, nil),
nil, nil,
),
threadsDesc: NewDesc(
"go_threads",
"Number of OS threads created.",
nil, nil),
nil, nil,
),
gcDesc: NewDesc(
"go_gc_duration_seconds",
"A summary of the wall-time pause (stop-the-world) duration in garbage collection cycles.",
nil, nil),
nil, nil,
),
gcLastTimeDesc: NewDesc(
"go_memstats_last_gc_time_seconds",
"Number of seconds since 1970 of last garbage collection.",
nil, nil),
nil, nil,
),
goInfoDesc: NewDesc(
"go_info",
"Information about the Go environment.",
nil, Labels{"version": runtime.Version()}),
nil, Labels{"version": runtime.Version()},
),
}
}
@@ -11,9 +11,6 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//go:build go1.17
// +build go1.17
package prometheus
import (
@@ -27,7 +24,6 @@ import (
"github.com/prometheus/client_golang/prometheus/internal"
dto "github.com/prometheus/client_model/go"
"google.golang.org/protobuf/proto"
)
const (
@@ -573,14 +569,14 @@ func (h *batchHistogram) Write(out *dto.Metric) error {
// by obtaining the next float64 value down, in order.
upperBound := math.Nextafter(h.buckets[i+1], h.buckets[i])
dtoBuckets = append(dtoBuckets, &dto.Bucket{
CumulativeCount: proto.Uint64(totalCount),
UpperBound: proto.Float64(upperBound),
CumulativeCount: new(totalCount),
UpperBound: new(upperBound),
})
}
out.Histogram = &dto.Histogram{
Bucket: dtoBuckets,
SampleCount: proto.Uint64(totalCount),
SampleSum: proto.Float64(sum),
SampleCount: new(totalCount),
SampleSum: new(sum),
}
return nil
}
+21 -21
View File
@@ -803,8 +803,8 @@ func (h *histogram) Write(out *dto.Metric) error {
his := &dto.Histogram{
Bucket: make([]*dto.Bucket, len(h.upperBounds)),
SampleCount: proto.Uint64(count),
SampleSum: proto.Float64(math.Float64frombits(atomic.LoadUint64(&coldCounts.sumBits))),
SampleCount: new(count),
SampleSum: new(math.Float64frombits(atomic.LoadUint64(&coldCounts.sumBits))),
CreatedTimestamp: timestamppb.New(h.lastResetTime),
}
out.Histogram = his
@@ -814,8 +814,8 @@ func (h *histogram) Write(out *dto.Metric) error {
for i, upperBound := range h.upperBounds {
cumCount += atomic.LoadUint64(&coldCounts.buckets[i])
his.Bucket[i] = &dto.Bucket{
CumulativeCount: proto.Uint64(cumCount),
UpperBound: proto.Float64(upperBound),
CumulativeCount: new(cumCount),
UpperBound: new(upperBound),
}
if e := h.exemplars[i].Load(); e != nil {
his.Bucket[i].Exemplar = e.(*dto.Exemplar)
@@ -824,15 +824,15 @@ func (h *histogram) Write(out *dto.Metric) error {
// If there is an exemplar for the +Inf bucket, we have to add that bucket explicitly.
if e := h.exemplars[len(h.upperBounds)].Load(); e != nil {
b := &dto.Bucket{
CumulativeCount: proto.Uint64(count),
UpperBound: proto.Float64(math.Inf(1)),
CumulativeCount: new(count),
UpperBound: new(math.Inf(1)),
Exemplar: e.(*dto.Exemplar),
}
his.Bucket = append(his.Bucket, b)
}
if h.nativeHistogramSchema > math.MinInt32 {
his.ZeroThreshold = proto.Float64(math.Float64frombits(atomic.LoadUint64(&coldCounts.nativeHistogramZeroThresholdBits)))
his.Schema = proto.Int32(atomic.LoadInt32(&coldCounts.nativeHistogramSchema))
his.ZeroThreshold = new(math.Float64frombits(atomic.LoadUint64(&coldCounts.nativeHistogramZeroThresholdBits)))
his.Schema = new(atomic.LoadInt32(&coldCounts.nativeHistogramSchema))
zeroBucket := atomic.LoadUint64(&coldCounts.nativeHistogramZeroBucket)
defer func() {
@@ -840,7 +840,7 @@ func (h *histogram) Write(out *dto.Metric) error {
coldCounts.nativeHistogramBucketsNegative.Range(addAndReset(&hotCounts.nativeHistogramBucketsNegative, &hotCounts.nativeHistogramBucketsNumber))
}()
his.ZeroCount = proto.Uint64(zeroBucket)
his.ZeroCount = new(zeroBucket)
his.NegativeSpan, his.NegativeDelta = makeBuckets(&coldCounts.nativeHistogramBucketsNegative)
his.PositiveSpan, his.PositiveDelta = makeBuckets(&coldCounts.nativeHistogramBucketsPositive)
@@ -1331,12 +1331,12 @@ func (h *constHistogram) Write(out *dto.Metric) error {
buckets := make([]*dto.Bucket, 0, len(h.buckets))
his.SampleCount = proto.Uint64(h.count)
his.SampleSum = proto.Float64(h.sum)
his.SampleCount = new(h.count)
his.SampleSum = new(h.sum)
for upperBound, count := range h.buckets {
buckets = append(buckets, &dto.Bucket{
CumulativeCount: proto.Uint64(count),
UpperBound: proto.Float64(upperBound),
CumulativeCount: new(count),
UpperBound: new(upperBound),
})
}
@@ -1481,7 +1481,7 @@ func pickSchema(bucketFactor float64) int32 {
func makeBuckets(buckets *sync.Map) ([]*dto.BucketSpan, []int64) {
var ii []int
buckets.Range(func(k, v any) bool {
buckets.Range(func(k, _ any) bool {
ii = append(ii, k.(int))
return true
})
@@ -1518,13 +1518,13 @@ func makeBuckets(buckets *sync.Map) ([]*dto.BucketSpan, []int64) {
// at the very beginning, or because we have found a gap
// of more than two buckets.
spans = append(spans, &dto.BucketSpan{
Offset: proto.Int32(iDelta),
Offset: new(iDelta),
Length: proto.Uint32(0),
})
} else {
// We have found a small gap (or no gap at all).
// Insert empty buckets as needed.
for j := int32(0); j < iDelta; j++ {
for range iDelta {
appendDelta(0)
}
}
@@ -1570,7 +1570,7 @@ func addAndReset(hotBuckets *sync.Map, bucketNumber *uint32) func(k, v any) bool
}
func deleteSyncMap(m *sync.Map) {
m.Range(func(k, v any) bool {
m.Range(func(k, _ any) bool {
m.Delete(k)
return true
})
@@ -1578,7 +1578,7 @@ func deleteSyncMap(m *sync.Map) {
func findSmallestKey(m *sync.Map) int {
result := math.MaxInt32
m.Range(func(k, v any) bool {
m.Range(func(k, _ any) bool {
key := k.(int)
if key < result {
result = key
@@ -1956,7 +1956,7 @@ func NewConstNativeHistogram(
PositiveSpan: PositiveSpan,
PositiveDelta: PositiveDelta,
ZeroCount: proto.Uint64(zeroBucket),
ZeroCount: new(zeroBucket),
},
labelPairs: MakeLabelPairs(desc, labelValues),
}
@@ -2044,13 +2044,13 @@ func makeBucketsFromMap(buckets map[int]int64) ([]*dto.BucketSpan, []int64) {
// at the very beginning, or because we have found a gap
// of more than two buckets.
spans = append(spans, &dto.BucketSpan{
Offset: proto.Int32(iDelta),
Offset: new(iDelta),
Length: proto.Uint32(0),
})
} else {
// We have found a small gap (or no gap at all).
// Insert empty buckets as needed.
for j := int32(0); j < iDelta; j++ {
for range iDelta {
appendDelta(0)
}
}
@@ -11,9 +11,6 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//go:build go1.17
// +build go1.17
package internal
import (
+3 -4
View File
@@ -22,7 +22,6 @@ import (
dto "github.com/prometheus/client_model/go"
"github.com/prometheus/common/model"
"google.golang.org/protobuf/proto"
)
var separatorByteSlice = []byte{model.SeparatorByte} // For convenient use with xxhash.
@@ -153,7 +152,7 @@ type timestampedMetric struct {
func (m timestampedMetric) Write(pb *dto.Metric) error {
e := m.Metric.Write(pb)
pb.TimestampMs = proto.Int64(m.t.Unix()*1000 + int64(m.t.Nanosecond()/1000000))
pb.TimestampMs = new(m.t.Unix()*1000 + int64(m.t.Nanosecond()/1000000))
return e
}
@@ -209,8 +208,8 @@ func (m *withExemplarsMetric) Write(pb *dto.Metric) error {
} else {
// The +Inf bucket should be explicitly added if there is an exemplar for it, similar to non-const histogram logic in https://github.com/prometheus/client_golang/blob/main/prometheus/histogram.go#L357-L365.
b := &dto.Bucket{
CumulativeCount: proto.Uint64(h.GetSampleCount()),
UpperBound: proto.Float64(math.Inf(1)),
CumulativeCount: new(h.GetSampleCount()),
UpperBound: new(math.Inf(1)),
Exemplar: e,
}
h.Bucket = append(h.Bucket, b)
-1
View File
@@ -12,7 +12,6 @@
// limitations under the License.
//go:build !js || wasm
// +build !js wasm
package prometheus
@@ -12,7 +12,6 @@
// limitations under the License.
//go:build js && !wasm
// +build js,!wasm
package prometheus
@@ -54,21 +54,22 @@ func getOpenFileCount() (float64, error) {
// benefits, however, include fewer system calls and not failing when at the
// open file soft limit.
if dir, err := os.Open("/dev/fd"); err != nil {
dir, err := os.Open("/dev/fd")
if err != nil {
return 0.0, err
} else {
defer dir.Close()
// Avoid ReadDir(), as it calls stat(2) on each descriptor. Not only is
// that info not used, but KQUEUE descriptors fail stat(2), which causes
// the whole method to fail.
if names, err := dir.Readdirnames(0); err != nil {
return 0.0, err
} else {
// Subtract 1 to ignore the open /dev/fd descriptor above.
return float64(len(names) - 1), nil
}
}
defer dir.Close()
// Avoid ReadDir(), as it calls stat(2) on each descriptor. Not only is
// that info not used, but KQUEUE descriptors fail stat(2), which causes
// the whole method to fail.
names, err := dir.Readdirnames(0)
if err != nil {
return 0.0, err
}
// Subtract 1 to ignore the open /dev/fd descriptor above.
return float64(len(names) - 1), nil
}
func (c *processCollector) processCollect(ch chan<- Metric) {
@@ -12,7 +12,6 @@
// limitations under the License.
//go:build wasip1 || js || ios
// +build wasip1 js ios
package prometheus
@@ -12,7 +12,6 @@
// limitations under the License.
//go:build !windows && !js && !wasip1 && !darwin
// +build !windows,!js,!wasip1,!darwin
package prometheus
+31 -7
View File
@@ -51,10 +51,12 @@ import (
)
const (
acceptHeader = "Accept"
contentTypeHeader = "Content-Type"
contentEncodingHeader = "Content-Encoding"
acceptEncodingHeader = "Accept-Encoding"
processStartTimeHeader = "Process-Start-Time-Unix"
varyHeader = "Vary"
)
// Compression represents the content encodings handlers support for the HTTP
@@ -70,9 +72,8 @@ const (
func defaultCompressionFormats() []Compression {
if internal.NewZstdWriter != nil {
return []Compression{Identity, Gzip, Zstd}
} else {
return []Compression{Identity, Gzip}
}
return []Compression{Identity, Gzip}
}
var gzipPool = sync.Pool{
@@ -315,11 +316,20 @@ func HandlerForTransactional(reg prometheus.TransactionalGatherer, opts HandlerO
}
}
var contentType expfmt.Format
if opts.EnableOpenMetrics {
contentType = expfmt.NegotiateIncludingOpenMetrics(req.Header)
} else {
contentType = expfmt.Negotiate(req.Header)
var (
contentType expfmt.Format
acceptedFormats = opts.AcceptedFormats
)
if len(acceptedFormats) == 0 {
acceptedFormats = expfmt.DefaultAcceptedFormats()
if opts.EnableOpenMetrics {
acceptedFormats = expfmt.DefaultOpenMetricsAcceptedFormats()
}
}
contentType = expfmt.NegotiateAccept(req.Header, acceptedFormats...)
rsp.Header().Add(varyHeader, acceptHeader)
if len(compressions) > 0 {
rsp.Header().Add(varyHeader, acceptEncodingHeader)
}
rsp.Header().Set(contentTypeHeader, string(contentType))
@@ -598,6 +608,8 @@ type HandlerOpts struct {
// a trailing ".0" if they would otherwise look like integer numbers
// (which changes the identity of the resulting series on the Prometheus
// server).
//
// Note that setting AcceptedFormats overrides this option.
EnableOpenMetrics bool
// EnableOpenMetricsTextCreatedSamples specifies if this handler should add, extra, synthetic
// Created Timestamps for counters, histograms and summaries, which for the current
@@ -622,6 +634,18 @@ type HandlerOpts struct {
// NOTE: This feature is experimental and not covered by OpenMetrics or Prometheus
// exposition format.
ProcessStartTime time.Time
// AcceptedFormats allows customizing the formats the handler will negotiate
// with the client based on the request's Accept header.
// The client's Accept header and q-values take precedence, but the order of
// formats in AcceptedFormats specifies the server's preference order used as a
// tie-breaker (e.g. for wildcard "*/*" requests or equal q-values) and fallback.
// If empty, the default set is used (expfmt.DefaultAcceptedFormats()).
// If empty and EnableOpenMetrics is true, stable OpenMetrics formats are negotiated
// (expfmt.DefaultOpenMetricsAcceptedFormats()).
// This is useful for testing experimental formats (e.g. OpenMetrics 2.0)
// or restricting the formats you want to allow.
// Using EnableOpenMetrics with an empty AcceptedFormats is still the recommended option for the majority of cases.
AcceptedFormats []expfmt.Format
}
// httpError removes any content-encoding header and then calls http.Error with
+1 -1
View File
@@ -41,7 +41,7 @@ type options struct {
func defaultOptions() *options {
return &options{
getExemplarFn: func(req *http.Request) prometheus.Labels { return nil },
getExemplarFn: func(_ *http.Request) prometheus.Labels { return nil },
extraLabelsFromRequest: map[string]LabelValueFromRequest{},
}
}
+13 -11
View File
@@ -17,6 +17,7 @@ import (
"bytes"
"errors"
"fmt"
"maps"
"os"
"path/filepath"
"runtime"
@@ -31,7 +32,6 @@ import (
"github.com/cespare/xxhash/v2"
dto "github.com/prometheus/client_model/go"
"github.com/prometheus/common/expfmt"
"google.golang.org/protobuf/proto"
)
const (
@@ -296,6 +296,7 @@ func (r *Registry) Register(c Collector) error {
defer func() {
// Drain channel in case of premature return to not leak a goroutine.
for range descChan {
continue
}
r.mtx.Unlock()
}()
@@ -369,9 +370,7 @@ func (r *Registry) Register(c Collector) error {
for hash := range newDescIDs {
r.descIDs[hash] = struct{}{}
}
for name, dimHash := range newDimHashesByName {
r.dimHashesByName[name] = dimHash
}
maps.Copy(r.dimHashesByName, newDimHashesByName)
return nil
}
@@ -476,7 +475,7 @@ func (r *Registry) Gather() ([]*dto.MetricFamily, error) {
for {
select {
case collector := <-checkedCollectors:
safeErrs.Append((safeCollect(collector, checkedMetricChan)))
safeErrs.Append(safeCollect(collector, checkedMetricChan))
case collector := <-uncheckedCollectors:
safeErrs.Append(safeCollect(collector, uncheckedMetricChan))
default:
@@ -502,10 +501,12 @@ func (r *Registry) Gather() ([]*dto.MetricFamily, error) {
defer func() {
if checkedMetricChan != nil {
for range checkedMetricChan {
continue
}
}
if uncheckedMetricChan != nil {
for range uncheckedMetricChan {
continue
}
}
}()
@@ -727,10 +728,10 @@ func processMetric(
}
} else { // New name.
metricFamily = &dto.MetricFamily{}
metricFamily.Name = proto.String(desc.fqName)
metricFamily.Help = proto.String(desc.help)
metricFamily.Name = new(desc.fqName)
metricFamily.Help = new(desc.help)
if desc.unit != "" {
metricFamily.Unit = proto.String(desc.unit)
metricFamily.Unit = new(desc.unit)
}
// TODO(beorn7): Simplify switch once Desc has type.
switch {
@@ -956,7 +957,8 @@ func checkMetricConsistency(
if !utf8.ValidString(labelPair.GetValue()) {
return fmt.Errorf(
"collected metric %q { %s} has a label named %q whose value is not utf8: %#v",
name, dtoMetric, labelName, labelPair.GetValue())
name, dtoMetric, labelName, labelPair.GetValue(),
)
}
previousLabelName = labelName
}
@@ -981,7 +983,7 @@ func checkMetricConsistency(
h.Write(separatorByteSlice)
}
if dtoMetric.TimestampMs != nil {
h.WriteString(strconv.FormatInt(*(dtoMetric.TimestampMs), 10))
h.WriteString(strconv.FormatInt(*dtoMetric.TimestampMs, 10))
h.Write(separatorByteSlice)
}
hSum := h.Sum64()
@@ -1013,7 +1015,7 @@ func checkDescConsistency(
copy(lpsFromDesc, desc.constLabelPairs)
for _, l := range desc.variableLabels.names {
lpsFromDesc = append(lpsFromDesc, &dto.LabelPair{
Name: proto.String(l),
Name: new(l),
})
}
if len(lpsFromDesc) != len(dtoMetric.Label) {
+10 -11
View File
@@ -25,7 +25,6 @@ import (
dto "github.com/prometheus/client_model/go"
"github.com/beorn7/perks/quantile"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/timestamppb"
)
@@ -337,8 +336,8 @@ func (s *summary) Write(out *dto.Metric) error {
s.bufMtx.Unlock()
s.flushColdBuf()
sum.SampleCount = proto.Uint64(s.cnt)
sum.SampleSum = proto.Float64(s.sum)
sum.SampleCount = new(s.cnt)
sum.SampleSum = new(s.sum)
for _, rank := range s.sortedObjectives {
var q float64
@@ -348,8 +347,8 @@ func (s *summary) Write(out *dto.Metric) error {
q = s.headStream.Query(rank)
}
qs = append(qs, &dto.Quantile{
Quantile: proto.Float64(rank),
Value: proto.Float64(q),
Quantile: new(rank),
Value: new(q),
})
}
@@ -512,8 +511,8 @@ func (s *noObjectivesSummary) Write(out *dto.Metric) error {
}
sum := &dto.Summary{
SampleCount: proto.Uint64(count),
SampleSum: proto.Float64(math.Float64frombits(atomic.LoadUint64(&coldCounts.sumBits))),
SampleCount: new(count),
SampleSum: new(math.Float64frombits(atomic.LoadUint64(&coldCounts.sumBits))),
CreatedTimestamp: s.createdTs,
}
@@ -718,13 +717,13 @@ func (s *constSummary) Write(out *dto.Metric) error {
}
qs := make([]*dto.Quantile, 0, len(s.quantiles))
sum.SampleCount = proto.Uint64(s.count)
sum.SampleSum = proto.Float64(s.sum)
sum.SampleCount = new(s.count)
sum.SampleSum = new(s.sum)
for rank, q := range s.quantiles {
qs = append(qs, &dto.Quantile{
Quantile: proto.Float64(rank),
Value: proto.Float64(q),
Quantile: new(rank),
Value: new(q),
})
}
+8 -9
View File
@@ -23,7 +23,6 @@ import (
"github.com/prometheus/client_golang/prometheus/internal"
dto "github.com/prometheus/client_model/go"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/timestamppb"
)
@@ -196,11 +195,11 @@ func populateMetric(
m.Label = labelPairs
switch t {
case CounterValue:
m.Counter = &dto.Counter{Value: proto.Float64(v), Exemplar: e, CreatedTimestamp: ct}
m.Counter = &dto.Counter{Value: new(v), Exemplar: e, CreatedTimestamp: ct}
case GaugeValue:
m.Gauge = &dto.Gauge{Value: proto.Float64(v)}
m.Gauge = &dto.Gauge{Value: new(v)}
case UntypedValue:
m.Untyped = &dto.Untyped{Value: proto.Float64(v)}
m.Untyped = &dto.Untyped{Value: new(v)}
default:
return fmt.Errorf("encountered unknown type %v", t)
}
@@ -227,8 +226,8 @@ func MakeLabelPairs(desc *Desc, labelValues []string) []*dto.LabelPair {
labelPairs := make([]*dto.LabelPair, 0, totalLen)
for i, l := range desc.variableLabels.names {
labelPairs = append(labelPairs, &dto.LabelPair{
Name: proto.String(l),
Value: proto.String(labelValues[i]),
Name: new(l),
Value: new(labelValues[i]),
})
}
labelPairs = append(labelPairs, desc.constLabelPairs...)
@@ -244,7 +243,7 @@ const ExemplarMaxRunes = 128
// number of runes in the label names and values exceeds ExemplarMaxRunes.
func newExemplar(value float64, ts time.Time, l Labels) (*dto.Exemplar, error) {
e := &dto.Exemplar{}
e.Value = proto.Float64(value)
e.Value = new(value)
tsProto := timestamppb.New(ts)
if err := tsProto.CheckValid(); err != nil {
return nil, err
@@ -262,8 +261,8 @@ func newExemplar(value float64, ts time.Time, l Labels) (*dto.Exemplar, error) {
}
runes += utf8.RuneCountInString(value)
labelPairs = append(labelPairs, &dto.LabelPair{
Name: proto.String(name),
Value: proto.String(value),
Name: new(name),
Value: new(value),
})
}
if runes > ExemplarMaxRunes {
+2 -3
View File
@@ -20,7 +20,6 @@ import (
"github.com/prometheus/client_golang/prometheus/internal"
dto "github.com/prometheus/client_model/go"
"google.golang.org/protobuf/proto"
)
// WrapRegistererWith returns a Registerer wrapping the provided
@@ -212,8 +211,8 @@ func (m *wrappingMetric) Write(out *dto.Metric) error {
}
for ln, lv := range m.labels {
out.Label = append(out.Label, &dto.LabelPair{
Name: proto.String(ln),
Value: proto.String(lv),
Name: new(ln),
Value: new(lv),
})
}
sort.Sort(internal.LabelPairSorter(out.Label))
+13 -6
View File
@@ -72,13 +72,15 @@ func ResponseFormat(h http.Header) Format {
// NewDecoder returns a new decoder based on the given input format. Metric
// names are validated based on the provided Format -- if the format requires
// escaping, raditional Prometheues validity checking is used. Otherwise, names
// escaping, traditional Prometheus validity checking is used. Otherwise, names
// are checked for UTF-8 validity. Supported formats include delimited protobuf
// and Prometheus text format. For historical reasons, this decoder fallbacks
// to classic text decoding for any other format. This decoder does not fully
// support OpenMetrics although it may often succeed due to the similarities
// between the formats. This decoder may not support the latest features of
// Prometheus text format and is not intended for high-performance applications.
// and Prometheus text format. For historical reasons, this decoder falls back
// to classic text decoding for other legacy formats, but returns an error for
// unsupported formats such as OpenMetrics 2.0. This decoder does not fully
// support OpenMetrics although it may often succeed for OpenMetrics 1.0 due to
// the similarities between the formats. This decoder may not support the latest
// features of Prometheus text format and is not intended for high-performance
// applications.
// See: https://github.com/prometheus/common/issues/812
func NewDecoder(r io.Reader, format Format) Decoder {
scheme := model.LegacyValidation
@@ -90,6 +92,11 @@ func NewDecoder(r io.Reader, format Format) Decoder {
return &protoDecoder{r: bufio.NewReader(r), s: scheme}
case TypeProtoText, TypeProtoCompact:
return &errDecoder{err: fmt.Errorf("format %s not supported for decoding", format)}
case TypeOpenMetrics:
_, params, err := mime.ParseMediaType(string(format))
if err == nil && params["version"] == OpenMetricsVersion_2_0_0 {
return &errDecoder{err: fmt.Errorf("format %s not supported for decoding", format)}
}
}
return &textDecoder{r: r, s: scheme}
}
+135 -49
View File
@@ -16,7 +16,9 @@ package expfmt
import (
"fmt"
"io"
"mime"
"net/http"
"slices"
"github.com/munnerz/goautoneg"
dto "github.com/prometheus/client_model/go"
@@ -26,6 +28,55 @@ import (
"github.com/prometheus/common/model"
)
var formatToAccept = map[Format]goautoneg.Accept{}
func init() {
for _, f := range []Format{
FmtText,
FmtProtoDelim,
FmtProtoText,
FmtProtoCompact,
FmtOpenMetrics_1_0_0,
FmtOpenMetrics_2_0_0,
FmtOpenMetrics_0_0_1,
} {
if parsed := goautoneg.ParseAccept(string(f)); len(parsed) > 0 {
formatToAccept[f] = parsed[0]
}
}
}
var (
defaultAcceptedFormats = []Format{
FmtProtoDelim,
FmtProtoText,
FmtProtoCompact,
FmtText,
}
defaultOpenMetricsAcceptedFormats = []Format{
FmtOpenMetrics_1_0_0,
FmtOpenMetrics_0_0_1,
FmtProtoDelim,
FmtProtoText,
FmtProtoCompact,
FmtText,
}
)
// DefaultAcceptedFormats returns a copy of standard accepted formats for Negotiate,
// ordered by preference (delimited protobuf, protobuf text, compact protobuf text,
// and Prometheus text format).
func DefaultAcceptedFormats() []Format {
return slices.Clone(defaultAcceptedFormats)
}
// DefaultOpenMetricsAcceptedFormats returns a copy of standard accepted formats including
// stable OpenMetrics formats, ordered by preference.
func DefaultOpenMetricsAcceptedFormats() []Format {
return slices.Clone(defaultOpenMetricsAcceptedFormats)
}
// Encoder types encode metric families into an underlying wire protocol.
type Encoder interface {
Encode(*dto.MetricFamily) error
@@ -57,11 +108,30 @@ func (ec encoderCloser) Close() error {
// Negotiate returns the Content-Type based on the given Accept header. If no
// appropriate accepted type is found, FmtText is returned (which is the
// Prometheus text format). This function will never negotiate FmtOpenMetrics,
// as the support is still experimental. To include the option to negotiate
// FmtOpenMetrics, use NegotiateIncludingOpenMetrics.
// Prometheus text format).
//
// Deprecated: Use NegotiateAccept(h, DefaultAcceptedFormats()...)
// or specify only the formats supported by your server.
func Negotiate(h http.Header) Format {
escapingScheme := Format(fmt.Sprintf("; escaping=%s", Format(model.NameEscapingScheme.String())))
return NegotiateAccept(h, defaultAcceptedFormats...)
}
// NegotiateIncludingOpenMetrics works like Negotiate but includes all stable
// FmtOpenMetrics formats as an option for the result.
//
// Deprecated: Use NegotiateAccept(h, DefaultOpenMetricsAcceptedFormats()...)
// or specify only the formats supported by your server.
func NegotiateIncludingOpenMetrics(h http.Header) Format {
return NegotiateAccept(h, defaultOpenMetricsAcceptedFormats...)
}
// NegotiateAccept returns the Content-Type based on the given Accept header
// and the list of accepted Formats provided by the caller, in order of preference.
// If no accepted format matches the Accept header, it falls back to the text
// format if present in the accepted list, or the first accepted format (or FmtText
// if accepted is empty).
func NegotiateAccept(h http.Header, accepted ...Format) Format {
escapingScheme := Format("; escaping=" + model.NameEscapingScheme.String())
for _, ac := range goautoneg.ParseAccept(h.Get(hdrAccept)) {
if escapeParam := ac.Params[model.EscapingKey]; escapeParam != "" {
switch Format(escapeParam) {
@@ -71,63 +141,66 @@ func Negotiate(h http.Header) Format {
// If the escaping parameter is unknown, ignore it.
}
}
ver := ac.Params["version"]
if ac.Type+"/"+ac.SubType == ProtoType && ac.Params["proto"] == ProtoProtocol {
switch ac.Params["encoding"] {
case "delimited":
return FmtProtoDelim + escapingScheme
case "text":
return FmtProtoText + escapingScheme
case "compact-text":
return FmtProtoCompact + escapingScheme
for _, f := range accepted {
if matchFormat(ac, f) {
return f + escapingScheme
}
}
if ac.Type == "text" && ac.SubType == "plain" && (ver == TextVersion || ver == "") {
return FmtText + escapingScheme
}
for _, f := range accepted {
if f.FormatType() == TypeTextPlain {
return f + escapingScheme
}
}
if len(accepted) > 0 {
return accepted[0] + escapingScheme
}
return FmtText + escapingScheme
}
// NegotiateIncludingOpenMetrics works like Negotiate but includes
// FmtOpenMetrics as an option for the result. Note that this function is
// temporary and will disappear once FmtOpenMetrics is fully supported and as
// such may be negotiated by the normal Negotiate function.
func NegotiateIncludingOpenMetrics(h http.Header) Format {
escapingScheme := Format(fmt.Sprintf("; escaping=%s", Format(model.NameEscapingScheme.String())))
for _, ac := range goautoneg.ParseAccept(h.Get(hdrAccept)) {
if escapeParam := ac.Params[model.EscapingKey]; escapeParam != "" {
switch Format(escapeParam) {
case model.AllowUTF8, model.EscapeUnderscores, model.EscapeDots, model.EscapeValues:
escapingScheme = Format("; escaping=" + escapeParam)
default:
// If the escaping parameter is unknown, ignore it.
}
// matchFormat checks if a parsed accept clause matches a given Format.
func matchFormat(ac goautoneg.Accept, f Format) bool {
target, ok := formatToAccept[f]
if !ok {
parsed := goautoneg.ParseAccept(string(f))
if len(parsed) == 0 {
return false
}
ver := ac.Params["version"]
if ac.Type+"/"+ac.SubType == ProtoType && ac.Params["proto"] == ProtoProtocol {
switch ac.Params["encoding"] {
case "delimited":
return FmtProtoDelim + escapingScheme
case "text":
return FmtProtoText + escapingScheme
case "compact-text":
return FmtProtoCompact + escapingScheme
}
target = parsed[0]
}
if ac.Type != "*" && ac.Type != target.Type {
return false
}
if ac.SubType != "*" && ac.SubType != target.SubType {
return false
}
// Default OpenMetrics version to OpenMetricsVersion_0_0_1.
acVersion := ac.Params["version"]
if acVersion == "" && ac.Type+"/"+ac.SubType == OpenMetricsType {
acVersion = OpenMetricsVersion_0_0_1
}
if acVersion == "" && ac.Type == "text" && ac.SubType == "plain" {
acVersion = TextVersion
}
// General param matching.
for k, v := range target.Params {
if k == "charset" {
continue
}
if ac.Type == "text" && ac.SubType == "plain" && (ver == TextVersion || ver == "") {
return FmtText + escapingScheme
acVal := ac.Params[k]
if k == "version" {
acVal = acVersion
}
if ac.Type+"/"+ac.SubType == OpenMetricsType && (ver == OpenMetricsVersion_0_0_1 || ver == OpenMetricsVersion_1_0_0 || ver == "") {
switch ver {
case OpenMetricsVersion_1_0_0:
return FmtOpenMetrics_1_0_0 + escapingScheme
default:
return FmtOpenMetrics_0_0_1 + escapingScheme
}
if acVal != v {
return false
}
}
return FmtText + escapingScheme
return true
}
// NewEncoder returns a new encoder based on content type negotiation. All
@@ -181,6 +254,19 @@ func NewEncoder(w io.Writer, format Format, options ...EncoderOption) Encoder {
close: func() error { return nil },
}
case TypeOpenMetrics:
_, params, err := mime.ParseMediaType(string(format))
if err == nil && params["version"] == OpenMetricsVersion_2_0_0 {
return encoderCloser{
encode: func(v *dto.MetricFamily) error {
_, err := MetricFamilyToOpenMetrics20(w, model.EscapeMetricFamily(v, escapingScheme), options...)
return err
},
close: func() error {
_, err := FinalizeOpenMetrics(w)
return err
},
}
}
return encoderCloser{
encode: func(v *dto.MetricFamily) error {
_, err := MetricFamilyToOpenMetrics(w, model.EscapeMetricFamily(v, escapingScheme), options...)
+64 -48
View File
@@ -16,50 +16,48 @@ package expfmt
import (
"errors"
"mime"
"strings"
"github.com/prometheus/common/model"
)
// Format specifies the HTTP content type of the different wire protocols.
type Format string
// Constants to assemble the Content-Type values for the different wire
// protocols. The Content-Type strings here are all for the legacy exposition
// formats, where valid characters for metric names and label names are limited.
// Support for arbitrary UTF-8 characters in those names is already partially
// implemented in this module (see model.ValidationScheme), but to actually use
// it on the wire, new content-type strings will have to be agreed upon and
// added here.
// protocols.
const (
TextVersion = "0.0.4"
ProtoType = `application/vnd.google.protobuf`
ProtoProtocol = `io.prometheus.client.MetricFamily`
// Deprecated: Use expfmt.NewFormat(expfmt.TypeProtoCompact) instead.
ProtoFmt = ProtoType + "; proto=" + ProtoProtocol + ";"
ProtoFmt = ProtoType + "; proto=" + ProtoProtocol + ";"
OpenMetricsType = `application/openmetrics-text`
//nolint:revive // Allow for underscores.
OpenMetricsVersion_0_0_1 = "0.0.1"
//nolint:revive // Allow for underscores.
OpenMetricsVersion_1_0_0 = "1.0.0"
//nolint:revive // Allow for underscores.
OpenMetricsVersion_2_0_0 = "2.0.0"
)
// The Content-Type values for the different wire protocols. Do not do direct
// comparisons to these constants, instead use the comparison functions.
//
// Deprecated: Use expfmt.NewFormat(expfmt.TypeUnknown) instead.
FmtUnknown Format = `<unknown>`
// Deprecated: Use expfmt.NewFormat(expfmt.TypeTextPlain) instead.
FmtText Format = `text/plain; version=` + TextVersion + `; charset=utf-8`
// Deprecated: Use expfmt.NewFormat(expfmt.TypeProtoDelim) instead.
FmtProtoDelim Format = ProtoFmt + ` encoding=delimited`
// Deprecated: Use expfmt.NewFormat(expfmt.TypeProtoText) instead.
FmtProtoText Format = ProtoFmt + ` encoding=text`
// Deprecated: Use expfmt.NewFormat(expfmt.TypeProtoCompact) instead.
// Format specifies the HTTP content type of the different wire protocols.
// The Content-Type values for the different wire protocols represent
// baseline Content-Types used for HTTP headers and content negotiation.
//
// Because Content-Types on the wire may include dynamic parameters
// (such as "; escaping=...") or whitespace variations, avoid comparing Format
// values using direct equality (==).
type Format string
const (
FmtUnknown Format = `<unknown>`
FmtText Format = `text/plain; version=` + TextVersion + `; charset=utf-8`
FmtProtoDelim Format = ProtoFmt + ` encoding=delimited`
FmtProtoText Format = ProtoFmt + ` encoding=text`
FmtProtoCompact Format = ProtoFmt + ` encoding=compact-text`
// Deprecated: Use expfmt.NewFormat(expfmt.TypeOpenMetrics) instead.
//nolint:revive // Allow for underscores.
FmtOpenMetrics_2_0_0 Format = OpenMetricsType + `; version=` + OpenMetricsVersion_2_0_0 + `; charset=utf-8`
//nolint:revive // Allow for underscores.
FmtOpenMetrics_1_0_0 Format = OpenMetricsType + `; version=` + OpenMetricsVersion_1_0_0 + `; charset=utf-8`
// Deprecated: Use expfmt.NewFormat(expfmt.TypeOpenMetrics) instead.
//nolint:revive // Allow for underscores.
FmtOpenMetrics_0_0_1 Format = OpenMetricsType + `; version=` + OpenMetricsVersion_0_0_1 + `; charset=utf-8`
)
@@ -86,7 +84,7 @@ const (
// NewFormat generates a new Format from the type provided. Mostly used for
// tests, most Formats should be generated as part of content negotiation in
// encode.go. If a type has more than one version, the latest version will be
// encode.go. If a type has more than one version, the latest stable version will be
// returned.
func NewFormat(t FormatType) Format {
switch t {
@@ -107,6 +105,9 @@ func NewFormat(t FormatType) Format {
// NewOpenMetricsFormat generates a new OpenMetrics format matching the
// specified version number.
//
// Note: OpenMetrics version 2.0.0 is experimental and encode-only (supporting
// counter, gauge, summary, untyped, histogram, and gaugehistogram metric types).
func NewOpenMetricsFormat(version string) (Format, error) {
if version == OpenMetricsVersion_0_0_1 {
return FmtOpenMetrics_0_0_1, nil
@@ -114,30 +115,22 @@ func NewOpenMetricsFormat(version string) (Format, error) {
if version == OpenMetricsVersion_1_0_0 {
return FmtOpenMetrics_1_0_0, nil
}
if version == OpenMetricsVersion_2_0_0 {
// OpenMetrics 2.0.0 is experimental and encode-only.
return FmtOpenMetrics_2_0_0, nil
}
return FmtUnknown, errors.New("unknown open metrics version string")
}
// WithEscapingScheme returns a copy of Format with the specified escaping
// scheme appended to the end. If an escaping scheme already exists it is
// removed.
func (f Format) WithEscapingScheme(s model.EscapingScheme) Format {
var terms []string
for p := range strings.SplitSeq(string(f), ";") {
toks := strings.Split(p, "=")
if len(toks) != 2 {
trimmed := strings.TrimSpace(p)
if len(trimmed) > 0 {
terms = append(terms, trimmed)
}
continue
}
key := strings.TrimSpace(toks[0])
if key != model.EscapingKey {
terms = append(terms, strings.TrimSpace(p))
}
// Version returns the version parameter without validating whether it is supported.
// It returns an empty string if the parameter is absent or the Content-Type
// cannot be parsed.
func (f Format) Version() string {
_, params, err := mime.ParseMediaType(string(f))
if err != nil {
return ""
}
terms = append(terms, model.EscapingKey+"="+s.String())
return Format(strings.Join(terms, "; "))
return params["version"]
}
// FormatType deduces an overall FormatType for the given format.
@@ -189,8 +182,31 @@ func (f Format) FormatType() FormatType {
}
}
// ToEscapingScheme returns an EscapingScheme depending on the Format. Iff the
// Format contains a escaping=allow-utf-8 term, it will select NoEscaping. If a valid
// WithEscapingScheme returns a copy of Format with the specified escaping
// scheme appended to the end. If an escaping scheme already exists it is
// removed.
func (f Format) WithEscapingScheme(s model.EscapingScheme) Format {
var terms []string
for p := range strings.SplitSeq(string(f), ";") {
toks := strings.Split(p, "=")
if len(toks) != 2 {
trimmed := strings.TrimSpace(p)
if len(trimmed) > 0 {
terms = append(terms, trimmed)
}
continue
}
key := strings.TrimSpace(toks[0])
if key != model.EscapingKey {
terms = append(terms, strings.TrimSpace(p))
}
}
terms = append(terms, model.EscapingKey+"="+s.String())
return Format(strings.Join(terms, "; "))
}
// ToEscapingScheme returns an EscapingScheme depending on the Format. IFF the
// Format contains an escaping=allow-utf-8 term, it will select NoEscaping. If a valid
// "escaping" term exists, that will be used. Otherwise, the global default will
// be returned.
func (f Format) ToEscapingScheme() model.EscapingScheme {
File diff suppressed because it is too large. Load diff
+4 -4
View File
@@ -194,6 +194,7 @@ func (p *TextParser) reset(in io.Reader) {
// start of a line (or whitespace leading up to it).
func (p *TextParser) startOfLine() stateFn {
p.lineCount++
p.currentMetric = nil
p.currentMetricIsInsideBraces = false
p.currentMetricInsideBracesIsPresent = false
if p.skipBlankTab(); p.err != nil {
@@ -339,12 +340,11 @@ func (p *TextParser) startLabelName() stateFn {
return nil // Unexpected end of input.
}
if p.currentByte == '}' {
if p.currentMF == nil {
if p.currentMetric == nil {
// The closing brace was reached before any metric name was read,
// e.g. for the input "{}". There is no metric to attach labels to,
// so this is a malformed exposition. This mirrors the guard in
// startLabelValue. currentMF (not currentMetric) is checked because
// reset only clears currentMF between parses.
// startLabelValue. currentMetric is cleared at the start of each line.
p.parseError("invalid metric name")
p.currentLabelPairs = nil
return nil
@@ -495,7 +495,7 @@ func (p *TextParser) startLabelValue() stateFn {
return p.startLabelName
case '}':
if p.currentMF == nil {
if p.currentMetric == nil {
p.parseError("invalid metric name")
return nil
}
+5 -5
View File
@@ -14,7 +14,7 @@
package model
import (
"sort"
"slices"
)
// SeparatorByte is a byte that cannot occur in valid UTF-8 sequences and is
@@ -37,7 +37,7 @@ func LabelsToSignature(labels map[string]string) uint64 {
for labelName := range labels {
labelNames = append(labelNames, labelName)
}
sort.Strings(labelNames)
slices.Sort(labelNames)
sum := hashNew()
for _, labelName := range labelNames {
@@ -60,7 +60,7 @@ func labelSetToFingerprint(ls LabelSet) Fingerprint {
for labelName := range ls {
labelNames = append(labelNames, labelName)
}
sort.Sort(labelNames)
slices.Sort(labelNames)
sum := hashNew()
for _, labelName := range labelNames {
@@ -100,7 +100,7 @@ func SignatureForLabels(m Metric, labels ...LabelName) uint64 {
return emptyLabelSignature
}
sort.Sort(LabelNames(labels))
slices.Sort(labels)
sum := hashNew()
for _, label := range labels {
@@ -129,7 +129,7 @@ func SignatureWithoutLabels(m Metric, labels map[LabelName]struct{}) uint64 {
if len(labelNames) == 0 {
return emptyLabelSignature
}
sort.Sort(labelNames)
slices.Sort(labelNames)
sum := hashNew()
for _, labelName := range labelNames {
+28
View File
@@ -164,6 +164,34 @@ func (t *Time) UnmarshalJSON(b []byte) error {
// This type should not propagate beyond the scope of input/output processing.
type Duration time.Duration
// Common durations. Day and Week are included because ParseDuration supports
// those units (unlike the standard time package).
//
// To count the number of units in a Duration, divide:
//
// second := model.Second
// fmt.Print(int64(second/model.Millisecond)) // prints 1000
//
// To convert an integer number of units to a Duration, multiply:
//
// seconds := 10
// fmt.Print(model.Duration(seconds)*model.Second) // prints 10s
const (
Nanosecond Duration = 1
Microsecond = 1000 * Nanosecond
Millisecond = 1000 * Microsecond
Second = 1000 * Millisecond
Minute = 60 * Second
Hour = 60 * Minute
Day = 24 * Hour
Week = 7 * Day
)
// Milliseconds returns the duration as an integer millisecond count.
// Prometheus stores timestamps in milliseconds; time.Duration already
// covers the other units.
func (d Duration) Milliseconds() int64 { return time.Duration(d).Milliseconds() }
// Set implements pflag/flag.Value.
func (d *Duration) Set(s string) error {
var err error
-22
View File
@@ -224,28 +224,6 @@ func (ss SampleStream) MarshalJSON() ([]byte, error) {
}
}
func (ss *SampleStream) UnmarshalJSON(b []byte) error {
v := struct {
Metric Metric `json:"metric"`
Values []SamplePair `json:"values"`
Histograms []SampleHistogramPair `json:"histograms"`
}{
Metric: ss.Metric,
Values: ss.Values,
Histograms: ss.Histograms,
}
if err := json.Unmarshal(b, &v); err != nil {
return err
}
ss.Metric = v.Metric
ss.Values = v.Values
ss.Histograms = v.Histograms
return nil
}
// Scalar is a scalar value evaluated at the set timestamp.
type Scalar struct {
Value SampleValue `json:"value"`
+70
View File
@@ -0,0 +1,70 @@
// Copyright The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//go:build go1.27
package model
import (
"bytes"
"encoding/json/jsontext"
"errors"
)
var nullBytes = []byte("null")
func (s *SamplePair) UnmarshalJSONFrom(dec *jsontext.Decoder) error {
if t, err := dec.ReadToken(); err != nil {
return err
} else if t.Kind() == jsontext.KindNull {
return nil
} else if t.Kind() != jsontext.KindBeginArray {
return errors.New("expected [")
}
// Loop until we see the end of the array (tolerate arrays of any length).
for i := 0; dec.PeekKind() != jsontext.KindEndArray; i++ {
switch i {
case 0:
if v, err := dec.ReadValue(); err != nil {
return err
} else if bytes.Equal(v, nullBytes) {
// Skip null values to match SamplePair#UnmarshalJSON behavior.
} else if err := s.Timestamp.UnmarshalJSON(v); err != nil {
return err
}
case 1:
if v, err := dec.ReadValue(); err != nil {
return err
} else if bytes.Equal(v, nullBytes) {
// Skip null values to match SamplePair#UnmarshalJSON behavior.
} else if err := s.Value.UnmarshalJSON(v); err != nil {
return err
}
default:
// Skip any remaining array items to match UnmarshalJSON behavior.
if err := dec.SkipValue(); err != nil {
return err
}
}
}
// Read the final end array token.
if t, err := dec.ReadToken(); err != nil {
return err
} else if t.Kind() != jsontext.KindEndArray {
return errors.New("expected ]")
}
return nil
}
+95
View File
@@ -0,0 +1,95 @@
// Copyright The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//go:build go1.27
package model
import (
"bytes"
"encoding/json/jsontext"
"encoding/json/v2"
"errors"
)
func (s *SampleHistogramPair) UnmarshalJSONFrom(dec *jsontext.Decoder) error {
if t, err := dec.ReadToken(); err != nil {
return err
} else if t.Kind() != jsontext.KindBeginArray {
return errors.New("expected [")
}
if v, err := dec.ReadValue(); err != nil {
return err
} else if bytes.Equal(v, nullBytes) {
// Skip null values to match SamplePair#UnmarshalJSON behavior.
} else if err := s.Timestamp.UnmarshalJSON(v); err != nil {
return err
}
if err := json.UnmarshalDecode(dec, &s.Histogram); err != nil {
return err
} else if s.Histogram == nil {
return errors.New("histogram is nil")
}
if t, err := dec.ReadToken(); err != nil {
return err
} else if t.Kind() != jsontext.KindEndArray {
return errors.New("expected ]")
}
return nil
}
func (s *HistogramBucket) UnmarshalJSONFrom(dec *jsontext.Decoder) error {
if t, err := dec.ReadToken(); err != nil {
return err
} else if t.Kind() != jsontext.KindBeginArray {
return errors.New("expected [")
}
if err := json.UnmarshalDecode(dec, &s.Boundaries); err != nil {
return err
}
if v, err := dec.ReadValue(); err != nil {
return err
} else if bytes.Equal(v, nullBytes) {
// Skip null values to match SamplePair#UnmarshalJSON behavior.
} else if err := s.Lower.UnmarshalJSON(v); err != nil {
return err
}
if v, err := dec.ReadValue(); err != nil {
return err
} else if bytes.Equal(v, nullBytes) {
// Skip null values to match SamplePair#UnmarshalJSON behavior.
} else if err := s.Upper.UnmarshalJSON(v); err != nil {
return err
}
if v, err := dec.ReadValue(); err != nil {
return err
} else if bytes.Equal(v, nullBytes) {
// Skip null values to match SamplePair#UnmarshalJSON behavior.
} else if err := s.Count.UnmarshalJSON(v); err != nil {
return err
}
if t, err := dec.ReadToken(); err != nil {
return err
} else if t.Kind() != jsontext.KindEndArray {
return errors.New("expected ]")
}
return nil
}
-8
View File
@@ -34,14 +34,6 @@ linters:
capital: true
misspell:
locale: US
revive:
rules:
- name: var-naming
# TODO(SuperQ): See: https://github.com/prometheus/prometheus/issues/17766
arguments:
- []
- []
- - skip-package-name-checks: true
exclusions:
presets:
- comments
+4 -4
View File
@@ -96,7 +96,7 @@ The `proc` and `sys` filesystems are pseudo file systems and work a bit differen
Many of the files are changing continuously and the data being read can in some cases change between subsequent
reads in the same file. Also, most of the files are relatively small (less than a few KBs), and system calls
to the `stat` function will often return the wrong size. Therefore, for most files it's recommended to read the
full file in a single operation using an internal utility function called `util.ReadFileNoStat`.
full file in a single operation using an internal utility function called `parsers.ReadFileNoStat`.
This function is similar to `os.ReadFile`, but it avoids the system call to `stat` to get the current size of
the file.
@@ -104,7 +104,7 @@ Note that parsing the file's contents can still be performed one line at a time.
the full file, and then using a scanner on the `[]byte` or `string` containing the data.
```
data, err := util.ReadFileNoStat("/proc/cpuinfo")
data, err := parsers.ReadFileNoStat("/proc/cpuinfo")
if err != nil {
return err
}
@@ -113,9 +113,9 @@ the full file, and then using a scanner on the `[]byte` or `string` containing t
```
The `/sys` filesystem contains many very small files which contain only a single numeric or text value. These files
can be read using an internal function called `util.SysReadFile` which is similar to `os.ReadFile` but does
can be read using an internal function called `parsers.SysReadFile` which is similar to `os.ReadFile` but does
not bother to check the size of the file before reading.
```
data, err := util.SysReadFile("/sys/class/power_supply/BAT0/capacity")
data, err := parsers.SysReadFile("/sys/class/power_supply/BAT0/capacity")
```
+11 -2
View File
@@ -61,7 +61,7 @@ PROMU_URL := https://github.com/prometheus/promu/releases/download/v$(PROMU_
SKIP_GOLANGCI_LINT :=
GOLANGCI_LINT :=
GOLANGCI_LINT_OPTS ?=
GOLANGCI_LINT_VERSION ?= v2.11.4
GOLANGCI_LINT_VERSION ?= v2.12.2
GOLANGCI_FMT_OPTS ?=
# golangci-lint only supports linux, darwin and windows platforms on i386/amd64/arm64.
# windows isn't included here because of the path separator being different.
@@ -165,8 +165,9 @@ common-deps:
.PHONY: update-go-deps
update-go-deps:
@echo ">> updating Go dependencies"
# GOTOOLCHAIN prevents go get from pulling deps that require a newer Go version.
@for m in $$($(GO) list -mod=readonly -m -f '{{ if and (not .Indirect) (not .Main)}}{{.Path}}{{end}}' all); do \
$(GO) get $$m; \
GOTOOLCHAIN=go$$($(GO) mod edit -json | jq -r .Go) $(GO) get $$m; \
done
$(GO) mod tidy
@@ -246,6 +247,7 @@ common-docker-repo-name:
.PHONY: common-docker $(BUILD_DOCKER_ARCHS)
common-docker: $(BUILD_DOCKER_ARCHS)
# DOCKER_ARCHS holds promu architecture names, which are not OCI platform strings.
$(BUILD_DOCKER_ARCHS): common-docker-%:
@for variant in $(DOCKERFILE_VARIANTS_WITH_NAMES); do \
dockerfile=$${variant#*:}; \
@@ -254,10 +256,16 @@ $(BUILD_DOCKER_ARCHS): common-docker-%:
if [ "$*" = "armv7" ]; then \
distroless_arch="arm"; \
fi; \
arch="$*"; \
case "$$arch" in \
armv*) docker_platform="linux/arm/v$${arch#armv}";; \
*) docker_platform="linux/$$arch";; \
esac; \
if [ "$$dockerfile" = "Dockerfile" ]; then \
echo "Building default variant ($$variant_name) for linux-$* using $$dockerfile"; \
docker build -t "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(SANITIZED_DOCKER_IMAGE_TAG)" \
-f $$dockerfile \
--platform "$$docker_platform" \
--build-arg ARCH="$*" \
--build-arg OS="linux" \
--build-arg DISTROLESS_ARCH="$$distroless_arch" \
@@ -271,6 +279,7 @@ $(BUILD_DOCKER_ARCHS): common-docker-%:
echo "Building $$variant_name variant for linux-$* using $$dockerfile"; \
docker build -t "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(SANITIZED_DOCKER_IMAGE_TAG)-$$variant_name" \
-f $$dockerfile \
--platform "$$docker_platform" \
--build-arg ARCH="$*" \
--build-arg OS="linux" \
--build-arg DISTROLESS_ARCH="$$distroless_arch" \
+2 -2
View File
@@ -16,12 +16,12 @@ package procfs
import (
"strings"
"github.com/prometheus/procfs/internal/util"
"github.com/prometheus/procfs/internal/parsers"
)
// CmdLine returns the command line of the kernel.
func (fs FS) CmdLine() ([]string, error) {
data, err := util.ReadFileNoStat(fs.proc.Path("cmdline"))
data, err := parsers.ReadFileNoStat(fs.proc.Path("cmdline"))
if err != nil {
return nil, err
}
+2 -2
View File
@@ -24,7 +24,7 @@ import (
"strconv"
"strings"
"github.com/prometheus/procfs/internal/util"
"github.com/prometheus/procfs/internal/parsers"
)
// CPUInfo contains general information about a system CPU found in /proc/cpuinfo.
@@ -65,7 +65,7 @@ var (
// CPUInfo returns information about current system CPUs.
// See https://www.kernel.org/doc/Documentation/filesystems/proc.txt
func (fs FS) CPUInfo() ([]CPUInfo, error) {
data, err := util.ReadFileNoStat(fs.proc.Path("cpuinfo"))
data, err := parsers.ReadFileNoStat(fs.proc.Path("cpuinfo"))
if err != nil {
return nil, err
}
+3 -3
View File
@@ -20,7 +20,7 @@ import (
"io"
"strings"
"github.com/prometheus/procfs/internal/util"
"github.com/prometheus/procfs/internal/parsers"
)
// Crypto holds info parsed from /proc/crypto.
@@ -55,7 +55,7 @@ var cryptoFile = "crypto"
// https://kernel.readthedocs.io/en/sphinx-samples/crypto-API.html
func (fs FS) Crypto() ([]Crypto, error) {
path := fs.proc.Path(cryptoFile)
b, err := util.ReadFileNoStat(path)
b, err := parsers.ReadFileNoStat(path)
if err != nil {
return nil, fmt.Errorf("%w: Cannot read file %v: %w", ErrFileRead, b, err)
@@ -112,7 +112,7 @@ func parseCrypto(r io.Reader) ([]Crypto, error) {
// parseKV parses a key/value pair into the appropriate field of c.
func (c *Crypto) parseKV(k, v string) error {
vp := util.NewValueParser(v)
vp := parsers.NewValueParser(v)
switch k {
case "async":
+2 -2
View File
@@ -15,8 +15,8 @@
package procfs
// isRealProc returns true on architectures that don't have a Type argument
// isRealProc returns false on architectures that don't have a Type argument
// in their Statfs_t struct.
func isRealProc(_ string) (bool, error) {
return true, nil
return false, nil
}
+2 -2
View File
@@ -21,7 +21,7 @@ import (
"strconv"
"strings"
"github.com/prometheus/procfs/internal/util"
"github.com/prometheus/procfs/internal/parsers"
)
// Fscacheinfo represents fscache statistics.
@@ -229,7 +229,7 @@ type Fscacheinfo struct {
// Fscacheinfo returns information about current fscache statistics.
// See https://www.kernel.org/doc/Documentation/filesystems/caching/fscache.txt
func (fs FS) Fscacheinfo() (Fscacheinfo, error) {
b, err := util.ReadFileNoStat(fs.proc.Path("fs/fscache/stats"))
b, err := parsers.ReadFileNoStat(fs.proc.Path("fs/fscache/stats"))
if err != nil {
return Fscacheinfo{}, err
}
+3
View File
@@ -31,6 +31,9 @@ const (
// DefaultSelinuxMountPoint is the common mount point of the selinuxfs.
DefaultSelinuxMountPoint = "/sys/fs/selinux"
// DefaultResctrlMountPoint is the common mount point of the resctrl filesystem.
DefaultResctrlMountPoint = "/sys/fs/resctrl"
)
// FS represents a pseudo-filesystem, normally /proc or /sys, which provides an
@@ -11,7 +11,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package util
package parsers
import (
"errors"
@@ -11,7 +11,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package util
package parsers
import (
"io"
@@ -13,7 +13,7 @@
//go:build (linux || darwin) && !appengine
package util
package parsers
import (
"bytes"
@@ -13,7 +13,7 @@
//go:build (linux && appengine) || (!linux && !darwin)
package util
package parsers
import (
"fmt"
@@ -11,7 +11,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package util
package parsers
import (
"strconv"
+2 -2
View File
@@ -25,7 +25,7 @@ import (
"strconv"
"strings"
"github.com/prometheus/procfs/internal/util"
"github.com/prometheus/procfs/internal/parsers"
)
// IPVSStats holds IPVS statistics, as exposed by the kernel in `/proc/net/ip_vs_stats`.
@@ -66,7 +66,7 @@ type IPVSBackendStatus struct {
// IPVSStats reads the IPVS statistics from the specified `proc` filesystem.
func (fs FS) IPVSStats() (IPVSStats, error) {
data, err := util.ReadFileNoStat(fs.proc.Path("net/ip_vs_stats"))
data, err := parsers.ReadFileNoStat(fs.proc.Path("net/ip_vs_stats"))
if err != nil {
return IPVSStats{}, err
}
+2 -2
View File
@@ -18,7 +18,7 @@ package procfs
import (
"os"
"github.com/prometheus/procfs/internal/util"
"github.com/prometheus/procfs/internal/parsers"
)
// KernelRandom contains information about to the kernel's random number generator.
@@ -48,7 +48,7 @@ func (fs FS) KernelRandom() (KernelRandom, error) {
"write_wakeup_threshold": &random.WriteWakeupThreshold,
"read_wakeup_threshold": &random.ReadWakeupThreshold,
} {
val, err := util.ReadUintFromFile(fs.proc.Path("sys", "kernel", "random", file))
val, err := parsers.ReadUintFromFile(fs.proc.Path("sys", "kernel", "random", file))
if os.IsNotExist(err) {
continue
}
+95
View File
@@ -0,0 +1,95 @@
// Copyright The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//go:build linux
package procfs
import (
"github.com/prometheus/procfs/internal/parsers"
)
// KernelTaintBit represents a single kernel taint flag.
type KernelTaintBit struct {
// Index is the zero-based bit position in the tainted bitmask.
Index int
// Flag is the letter code identifying this taint (e.g. "L" for soft lockup).
// See https://www.kernel.org/doc/html/latest/admin-guide/tainted-kernels.html
Flag string
// Description is a human-readable explanation of what this taint means.
Description string
// Set indicates whether this taint flag is currently active.
Set bool
}
// KernelTainted holds the kernel taint state read from /proc/sys/kernel/tainted.
type KernelTainted struct {
// Value is the raw integer from /proc/sys/kernel/tainted.
Value uint64
// Bits contains each known taint flag and whether it is currently set.
// Flags are listed in bit-position order (bit 0 first).
Bits []KernelTaintBit
}
// kernelTaintBitDefs lists all known kernel taint flags in bit-position order.
// Reference: https://www.kernel.org/doc/html/latest/admin-guide/tainted-kernels.html
var kernelTaintBitDefs = []struct {
flag string
description string
}{
{"P", "Proprietary module was loaded."},
{"F", "Module was force loaded."},
{"S", "SMP kernel oops on an officially SMP incapable processor."},
{"R", "Module was force unloaded."},
{"M", "Processor reported a Machine Check Exception (MCE)."},
{"B", "Bad page referenced or some unexpected page flags."},
{"U", "Taint requested by userspace application."},
{"D", "Kernel died recently, i.e. there was an OOPS or BUG."},
{"A", "An ACPI table was overridden by user."},
{"W", "Kernel issued warning."},
{"C", "Staging driver was loaded."},
{"I", "Workaround for bug in platform firmware applied."},
{"O", "Externally-built (out-of-tree) module was loaded."},
{"E", "Unsigned module was loaded."},
{"L", "Soft lockup occurred."},
{"K", "Kernel has been live patched."},
{"X", "Auxiliary taint, defined and used by distros."},
{"T", "The kernel was built with the struct randomization plugin."},
{"N", "An in-kernel test such as a KUnit test has been run."},
{"J", "Userspace used a mutating debug operation in fwctl."},
}
// KernelTainted reads /proc/sys/kernel/tainted and returns the raw taint value
// along with each known flag parsed into a KernelTainted struct.
// See https://www.kernel.org/doc/html/latest/admin-guide/tainted-kernels.html
func (fs FS) KernelTainted() (KernelTainted, error) {
value, err := parsers.SysReadUintFromFile(fs.proc.Path("sys", "kernel", "tainted"))
if err != nil {
return KernelTainted{}, err
}
return parseTainted(value), nil
}
// parseTainted builds a KernelTainted from the raw taint bitmask value.
func parseTainted(value uint64) KernelTainted {
bits := make([]KernelTaintBit, len(kernelTaintBitDefs))
for i, def := range kernelTaintBitDefs {
bits[i] = KernelTaintBit{
Index: i,
Flag: def.flag,
Description: def.description,
Set: value&(1<<uint(i)) != 0,
}
}
return KernelTainted{Value: value, Bits: bits}
}
+2 -2
View File
@@ -18,7 +18,7 @@ import (
"strconv"
"strings"
"github.com/prometheus/procfs/internal/util"
"github.com/prometheus/procfs/internal/parsers"
)
// LoadAvg represents an entry in /proc/loadavg.
@@ -32,7 +32,7 @@ type LoadAvg struct {
func (fs FS) LoadAvg() (*LoadAvg, error) {
path := fs.proc.Path("loadavg")
data, err := util.ReadFileNoStat(path)
data, err := parsers.ReadFileNoStat(path)
if err != nil {
return nil, err
}
+1 -1
View File
@@ -27,7 +27,7 @@ var (
recoveryLinePctRE = regexp.MustCompile(`= (.+)%`)
recoveryLineFinishRE = regexp.MustCompile(`finish=(.+)min`)
recoveryLineSpeedRE = regexp.MustCompile(`speed=(.+)[A-Z]`)
componentDeviceRE = regexp.MustCompile(`(.*)\[(\d+)\](\([SF]+\))?`)
componentDeviceRE = regexp.MustCompile(`(.*)\[(\d+)\]((?:\([FJRSW]\))+)?`)
personalitiesPrefix = "Personalities : "
)
+2 -2
View File
@@ -21,7 +21,7 @@ import (
"strconv"
"strings"
"github.com/prometheus/procfs/internal/util"
"github.com/prometheus/procfs/internal/parsers"
)
// Meminfo represents memory statistics.
@@ -210,7 +210,7 @@ type Meminfo struct {
// Meminfo returns an information about current kernel/system memory statistics.
// See https://www.kernel.org/doc/Documentation/filesystems/proc.txt
func (fs FS) Meminfo() (Meminfo, error) {
b, err := util.ReadFileNoStat(fs.proc.Path("meminfo"))
b, err := parsers.ReadFileNoStat(fs.proc.Path("meminfo"))
if err != nil {
return Meminfo{}, err
}
+3 -3
View File
@@ -98,11 +98,11 @@ func parseMountInfoString(mountString string) (*MountInfo, error) {
mount.MountID, err = strconv.Atoi(mountInfo[0])
if err != nil {
return nil, fmt.Errorf("%w: mount ID: %q", ErrFileParse, mount.MountID)
return nil, fmt.Errorf("%w: mount ID: %q", ErrFileParse, mountInfo[0])
}
mount.ParentID, err = strconv.Atoi(mountInfo[1])
if err != nil {
return nil, fmt.Errorf("%w: parent ID: %q", ErrFileParse, mount.ParentID)
return nil, fmt.Errorf("%w: parent ID: %q", ErrFileParse, mountInfo[1])
}
// Has optional fields, which is a space separated list of values.
// Example: shared:2 master:7
@@ -160,7 +160,7 @@ func mountOptionsParser(mountOptions string) map[string]string {
return opts
}
// readMountInfo reads a full mountinfo file (no 1 MiB cap, unlike util.ReadFileNoStat).
// readMountInfo reads a full mountinfo file (no 1 MiB cap, unlike parsers.ReadFileNoStat).
func readMountInfo(path string) ([]byte, error) {
f, err := os.Open(path)
if err != nil {
+3 -3
View File
@@ -20,7 +20,7 @@ import (
"io"
"strings"
"github.com/prometheus/procfs/internal/util"
"github.com/prometheus/procfs/internal/parsers"
)
// A ConntrackStatEntry represents one line from net/stat/nf_conntrack
@@ -49,7 +49,7 @@ func (fs FS) ConntrackStat() ([]ConntrackStatEntry, error) {
// Parses a slice of ConntrackStatEntries from the given filepath.
func readConntrackStat(path string) ([]ConntrackStatEntry, error) {
// This file is small and can be read with one syscall.
b, err := util.ReadFileNoStat(path)
b, err := parsers.ReadFileNoStat(path)
if err != nil {
// Do not wrap this error so the caller can detect os.IsNotExist and
// similar conditions.
@@ -84,7 +84,7 @@ func parseConntrackStat(r io.Reader) ([]ConntrackStatEntry, error) {
// Parses a ConntrackStatEntry from given array of fields.
func parseConntrackStatEntry(fields []string) (*ConntrackStatEntry, error) {
entries, err := util.ParseHexUint64s(fields)
entries, err := parsers.ParseHexUint64s(fields)
if err != nil {
return nil, fmt.Errorf("%w: Cannot parse entry: %d: %w", ErrFileParse, entries, err)
}
+9 -9
View File
@@ -178,7 +178,7 @@ func parseNetIPSocketLine(fields []string, isUDP bool) (*netIPSocketLine, error)
}
if line.Sl, err = strconv.ParseUint(s[0], 0, 64); err != nil {
return nil, fmt.Errorf("%w: Unable to parse sl field in %q: %w", ErrFileParse, line.Sl, err)
return nil, fmt.Errorf("%w: Unable to parse sl field in %q: %w", ErrFileParse, s[0], err)
}
// local_address
l := strings.Split(fields[1], ":")
@@ -189,7 +189,7 @@ func parseNetIPSocketLine(fields []string, isUDP bool) (*netIPSocketLine, error)
return nil, err
}
if line.LocalPort, err = strconv.ParseUint(l[1], 16, 64); err != nil {
return nil, fmt.Errorf("%w: Unable to parse local_address port value line %q: %w", ErrFileParse, line.LocalPort, err)
return nil, fmt.Errorf("%w: Unable to parse local_address port value line %q: %w", ErrFileParse, l[1], err)
}
// remote_address
@@ -201,12 +201,12 @@ func parseNetIPSocketLine(fields []string, isUDP bool) (*netIPSocketLine, error)
return nil, err
}
if line.RemPort, err = strconv.ParseUint(r[1], 16, 64); err != nil {
return nil, fmt.Errorf("%w: Cannot parse rem_address port value in %q: %w", ErrFileParse, line.RemPort, err)
return nil, fmt.Errorf("%w: Cannot parse rem_address port value in %q: %w", ErrFileParse, r[1], err)
}
// st
if line.St, err = strconv.ParseUint(fields[3], 16, 64); err != nil {
return nil, fmt.Errorf("%w: Cannot parse st value in %q: %w", ErrFileParse, line.St, err)
return nil, fmt.Errorf("%w: Cannot parse st value in %q: %w", ErrFileParse, fields[3], err)
}
// tx_queue and rx_queue
@@ -219,27 +219,27 @@ func parseNetIPSocketLine(fields []string, isUDP bool) (*netIPSocketLine, error)
)
}
if line.TxQueue, err = strconv.ParseUint(q[0], 16, 64); err != nil {
return nil, fmt.Errorf("%w: Cannot parse tx_queue value in %q: %w", ErrFileParse, line.TxQueue, err)
return nil, fmt.Errorf("%w: Cannot parse tx_queue value in %q: %w", ErrFileParse, q[0], err)
}
if line.RxQueue, err = strconv.ParseUint(q[1], 16, 64); err != nil {
return nil, fmt.Errorf("%w: Cannot parse trx_queue value in %q: %w", ErrFileParse, line.RxQueue, err)
return nil, fmt.Errorf("%w: Cannot parse trx_queue value in %q: %w", ErrFileParse, q[1], err)
}
// uid
if line.UID, err = strconv.ParseUint(fields[7], 0, 64); err != nil {
return nil, fmt.Errorf("%w: Cannot parse UID value in %q: %w", ErrFileParse, line.UID, err)
return nil, fmt.Errorf("%w: Cannot parse UID value in %q: %w", ErrFileParse, fields[7], err)
}
// inode
if line.Inode, err = strconv.ParseUint(fields[9], 0, 64); err != nil {
return nil, fmt.Errorf("%w: Cannot parse inode value in %q: %w", ErrFileParse, line.Inode, err)
return nil, fmt.Errorf("%w: Cannot parse inode value in %q: %w", ErrFileParse, fields[9], err)
}
// drops
if isUDP {
drops, err := strconv.ParseUint(fields[12], 0, 64)
if err != nil {
return nil, fmt.Errorf("%w: Cannot parse drops value in %q: %w", ErrFileParse, drops, err)
return nil, fmt.Errorf("%w: Cannot parse drops value in %q: %w", ErrFileParse, fields[12], err)
}
line.Drops = &drops
}
+2 -2
View File
@@ -20,7 +20,7 @@ import (
"strconv"
"strings"
"github.com/prometheus/procfs/internal/util"
"github.com/prometheus/procfs/internal/parsers"
)
// NetProtocolStats stores the contents from /proc/net/protocols.
@@ -71,7 +71,7 @@ type NetProtocolCapabilities struct {
// Linux 2.6.12-rc2 - https://elixir.bootlin.com/linux/v2.6.12-rc2/source/net/core/sock.c#L1452
// Linux 5.10 - https://elixir.bootlin.com/linux/v5.10.4/source/net/core/sock.c#L3586
func (fs FS) NetProtocols() (NetProtocolStats, error) {
data, err := util.ReadFileNoStat(fs.proc.Path("net/protocols"))
data, err := parsers.ReadFileNoStat(fs.proc.Path("net/protocols"))
if err != nil {
return NetProtocolStats{}, err
}
+2 -2
View File
@@ -21,7 +21,7 @@ import (
"strconv"
"strings"
"github.com/prometheus/procfs/internal/util"
"github.com/prometheus/procfs/internal/parsers"
)
const (
@@ -50,7 +50,7 @@ func (fs FS) NetRoute() ([]NetRouteLine, error) {
}
func readNetRoute(path string) ([]NetRouteLine, error) {
b, err := util.ReadFileNoStat(path)
b, err := parsers.ReadFileNoStat(path)
if err != nil {
return nil, err
}
+3 -3
View File
@@ -20,7 +20,7 @@ import (
"io"
"strings"
"github.com/prometheus/procfs/internal/util"
"github.com/prometheus/procfs/internal/parsers"
)
// A NetSockstat contains the output of /proc/net/sockstat{,6} for IPv4 or IPv6,
@@ -60,7 +60,7 @@ func (fs FS) NetSockstat6() (*NetSockstat, error) {
// readSockstat opens and parses a NetSockstat from the input file.
func readSockstat(name string) (*NetSockstat, error) {
// This file is small and can be read with one syscall.
b, err := util.ReadFileNoStat(name)
b, err := parsers.ReadFileNoStat(name)
if err != nil {
// Do not wrap this error so the caller can detect os.IsNotExist and
// similar conditions.
@@ -124,7 +124,7 @@ func parseSockstatKVs(kvs []string) (map[string]int, error) {
// Iterate two values at a time to gather key/value pairs.
out := make(map[string]int, len(kvs)/2)
for i := 0; i < len(kvs); i += 2 {
vp := util.NewValueParser(kvs[i+1])
vp := parsers.NewValueParser(kvs[i+1])
out[kvs[i]] = vp.Int()
if err := vp.Err(); err != nil {
+2 -2
View File
@@ -21,7 +21,7 @@ import (
"strconv"
"strings"
"github.com/prometheus/procfs/internal/util"
"github.com/prometheus/procfs/internal/parsers"
)
// For the proc file format details,
@@ -57,7 +57,7 @@ var softNetProcFile = "net/softnet_stat"
// NetSoftnetStat reads data from /proc/net/softnet_stat.
func (fs FS) NetSoftnetStat() ([]SoftnetStat, error) {
b, err := util.ReadFileNoStat(fs.proc.Path(softNetProcFile))
b, err := parsers.ReadFileNoStat(fs.proc.Path(softNetProcFile))
if err != nil {
return nil, err
}
+2 -2
View File
@@ -21,7 +21,7 @@ import (
"strconv"
"strings"
"github.com/prometheus/procfs/internal/util"
"github.com/prometheus/procfs/internal/parsers"
)
// Wireless models the content of /proc/net/wireless.
@@ -61,7 +61,7 @@ type Wireless struct {
// Wireless returns kernel wireless statistics.
func (fs FS) Wireless() ([]*Wireless, error) {
b, err := util.ReadFileNoStat(fs.proc.Path("net/wireless"))
b, err := parsers.ReadFileNoStat(fs.proc.Path("net/wireless"))
if err != nil {
return nil, err
}
+2 -2
View File
@@ -18,7 +18,7 @@ import (
"bytes"
"fmt"
"github.com/prometheus/procfs/internal/util"
"github.com/prometheus/procfs/internal/parsers"
)
const nfNetLinkQueueFormat = "%d %d %d %d %d %d %d %d %d"
@@ -48,7 +48,7 @@ type NFNetLinkQueue struct {
// NFNetLinkQueue returns information about current state of netfilter queues.
func (fs FS) NFNetLinkQueue() ([]NFNetLinkQueue, error) {
data, err := util.ReadFileNoStat(fs.proc.Path("net/netfilter/nfnetlink_queue"))
data, err := parsers.ReadFileNoStat(fs.proc.Path("net/netfilter/nfnetlink_queue"))
if err != nil {
return nil, err
}
+4 -4
View File
@@ -22,7 +22,7 @@ import (
"strconv"
"strings"
"github.com/prometheus/procfs/internal/util"
"github.com/prometheus/procfs/internal/parsers"
)
// Proc provides information about a running process.
@@ -128,7 +128,7 @@ func (fs FS) AllProcs() (Procs, error) {
// CmdLine returns the command line of a process.
func (p Proc) CmdLine() ([]string, error) {
data, err := util.ReadFileNoStat(p.path("cmdline"))
data, err := parsers.ReadFileNoStat(p.path("cmdline"))
if err != nil {
return nil, err
}
@@ -163,7 +163,7 @@ func (p Proc) Wchan() (string, error) {
// Comm returns the command name of a process.
func (p Proc) Comm() (string, error) {
data, err := util.ReadFileNoStat(p.path("comm"))
data, err := parsers.ReadFileNoStat(p.path("comm"))
if err != nil {
return "", err
}
@@ -281,7 +281,7 @@ func (p Proc) MountStats() ([]*Mount, error) {
// It supplies information missing in `/proc/self/mounts` and
// fixes various other problems with that file too.
func (p Proc) MountInfo() ([]*MountInfo, error) {
data, err := util.ReadFileNoStat(p.path("mountinfo"))
data, err := parsers.ReadFileNoStat(p.path("mountinfo"))
if err != nil {
return nil, err
}
+2 -2
View File
@@ -20,7 +20,7 @@ import (
"strconv"
"strings"
"github.com/prometheus/procfs/internal/util"
"github.com/prometheus/procfs/internal/parsers"
)
// Cgroup models one line from /proc/[pid]/cgroup. Each Cgroup struct describes the placement of a PID inside a
@@ -90,7 +90,7 @@ func parseCgroups(data []byte) ([]Cgroup, error) {
// control hierarchy running on this system. On every system (v1 and v2), all hierarchies contain all processes,
// so the len of the returned struct is equal to the number of active hierarchies on this system.
func (p Proc) Cgroups() ([]Cgroup, error) {
data, err := util.ReadFileNoStat(p.path("cgroup"))
data, err := parsers.ReadFileNoStat(p.path("cgroup"))
if err != nil {
return nil, err
}
+2 -2
View File
@@ -20,7 +20,7 @@ import (
"strconv"
"strings"
"github.com/prometheus/procfs/internal/util"
"github.com/prometheus/procfs/internal/parsers"
)
// CgroupSummary models one line from /proc/cgroups.
@@ -90,7 +90,7 @@ func parseCgroupSummary(data []byte) ([]CgroupSummary, error) {
// CgroupSummarys returns information about current /proc/cgroups.
func (fs FS) CgroupSummarys() ([]CgroupSummary, error) {
data, err := util.ReadFileNoStat(fs.proc.Path("cgroups"))
data, err := parsers.ReadFileNoStat(fs.proc.Path("cgroups"))
if err != nil {
return nil, err
}
+2 -2
View File
@@ -16,14 +16,14 @@ package procfs
import (
"strings"
"github.com/prometheus/procfs/internal/util"
"github.com/prometheus/procfs/internal/parsers"
)
// Environ reads process environments from `/proc/<pid>/environ`.
func (p Proc) Environ() ([]string, error) {
environments := make([]string, 0)
data, err := util.ReadFileNoStat(p.path("environ"))
data, err := parsers.ReadFileNoStat(p.path("environ"))
if err != nil {
return environments, err
}
+2 -2
View File
@@ -19,7 +19,7 @@ import (
"fmt"
"regexp"
"github.com/prometheus/procfs/internal/util"
"github.com/prometheus/procfs/internal/parsers"
)
var (
@@ -49,7 +49,7 @@ type ProcFDInfo struct {
// FDInfo constructor. On kernels older than 3.8, InotifyInfos will always be empty.
func (p Proc) FDInfo(fd string) (*ProcFDInfo, error) {
data, err := util.ReadFileNoStat(p.path("fdinfo", fd))
data, err := parsers.ReadFileNoStat(p.path("fdinfo", fd))
if err != nil {
return nil, err
}
+2 -2
View File
@@ -22,7 +22,7 @@ import (
"strconv"
"strings"
"github.com/prometheus/procfs/internal/util"
"github.com/prometheus/procfs/internal/parsers"
)
// Interrupt represents a single interrupt line.
@@ -42,7 +42,7 @@ type Interrupts map[string]Interrupt
// Interrupts creates a new instance from a given Proc instance.
func (p Proc) Interrupts() (Interrupts, error) {
data, err := util.ReadFileNoStat(p.fs.proc.Path("interrupts"))
data, err := parsers.ReadFileNoStat(p.fs.proc.Path("interrupts"))
if err != nil {
return nil, err
}
+2 -2
View File
@@ -16,7 +16,7 @@ package procfs
import (
"fmt"
"github.com/prometheus/procfs/internal/util"
"github.com/prometheus/procfs/internal/parsers"
)
// ProcIO models the content of /proc/<pid>/io.
@@ -43,7 +43,7 @@ type ProcIO struct {
func (p Proc) IO() (ProcIO, error) {
pio := ProcIO{}
data, err := util.ReadFileNoStat(p.path("io"))
data, err := parsers.ReadFileNoStat(p.path("io"))
if err != nil {
return pio, err
}
+36 -2
View File
@@ -21,7 +21,7 @@ import (
"strconv"
"strings"
"github.com/prometheus/procfs/internal/util"
"github.com/prometheus/procfs/internal/parsers"
)
// ProcNetstat models the content of /proc/<pid>/net/netstat.
@@ -146,6 +146,20 @@ type TcpExt struct { // nolint:revive
TCPMTUPFail *float64
TCPMTUPSuccess *float64
TCPWqueueTooBig *float64
// Legacy TcpExt counters dropped from recent kernels' TcpExt table;
// modeled so consumers retain metric parity with older kernels that
// still print them in /proc/net/netstat.
TCPLoss *float64
PAWSPassive *float64
TCPForwardRetrans *float64
TCPSchedulerFailed *float64
TCPPrequeued *float64
TCPDirectCopyFromBacklog *float64
TCPDirectCopyFromPrequeue *float64
TCPPrequeueDropped *float64
TCPFACKReorder *float64
TCPHPHitsToUser *float64
}
type IpExt struct { // nolint:revive
@@ -171,7 +185,7 @@ type IpExt struct { // nolint:revive
func (p Proc) Netstat() (ProcNetstat, error) {
filename := p.path("net/netstat")
data, err := util.ReadFileNoStat(filename)
data, err := parsers.ReadFileNoStat(filename)
if err != nil {
return ProcNetstat{PID: p.PID}, err
}
@@ -396,6 +410,26 @@ func parseProcNetstat(r io.Reader, fileName string) (ProcNetstat, error) {
procNetstat.TCPMTUPSuccess = &value
case "TCPWqueueTooBig":
procNetstat.TCPWqueueTooBig = &value
case "TCPLoss":
procNetstat.TCPLoss = &value
case "PAWSPassive":
procNetstat.PAWSPassive = &value
case "TCPForwardRetrans":
procNetstat.TCPForwardRetrans = &value
case "TCPSchedulerFailed":
procNetstat.TCPSchedulerFailed = &value
case "TCPPrequeued":
procNetstat.TCPPrequeued = &value
case "TCPDirectCopyFromBacklog":
procNetstat.TCPDirectCopyFromBacklog = &value
case "TCPDirectCopyFromPrequeue":
procNetstat.TCPDirectCopyFromPrequeue = &value
case "TCPPrequeueDropped":
procNetstat.TCPPrequeueDropped = &value
case "TCPFACKReorder":
procNetstat.TCPFACKReorder = &value
case "TCPHPHitsToUser":
procNetstat.TCPHPHitsToUser = &value
}
case "IpExt":
switch key {
+2 -2
View File
@@ -30,7 +30,7 @@ import (
"io"
"strings"
"github.com/prometheus/procfs/internal/util"
"github.com/prometheus/procfs/internal/parsers"
)
const lineFormat = "avg10=%f avg60=%f avg300=%f total=%d"
@@ -59,7 +59,7 @@ type PSIStats struct {
// resource from /proc/pressure/<resource>. At time of writing this can be
// either "cpu", "memory" or "io".
func (fs FS) PSIStatsForResource(resource string) (PSIStats, error) {
data, err := util.ReadFileNoStat(fs.proc.Path(fmt.Sprintf("%s/%s", "pressure", resource)))
data, err := parsers.ReadFileNoStat(fs.proc.Path(fmt.Sprintf("%s/%s", "pressure", resource)))
if err != nil {
return PSIStats{}, fmt.Errorf("%w: psi_stats: unavailable for %q: %w", ErrFileRead, resource, err)
}
+2 -2
View File
@@ -23,7 +23,7 @@ import (
"strconv"
"strings"
"github.com/prometheus/procfs/internal/util"
"github.com/prometheus/procfs/internal/parsers"
)
var (
@@ -60,7 +60,7 @@ type ProcSMapsRollup struct {
// If smaps_rollup does not exists (require kernel >= 4.15), the content of /proc/pid/smaps will
// we read and summed.
func (p Proc) ProcSMapsRollup() (ProcSMapsRollup, error) {
data, err := util.ReadFileNoStat(p.path("smaps_rollup"))
data, err := parsers.ReadFileNoStat(p.path("smaps_rollup"))
if err != nil && os.IsNotExist(err) {
return p.procSMapsRollupManual()
}
+2 -2
View File
@@ -21,7 +21,7 @@ import (
"strconv"
"strings"
"github.com/prometheus/procfs/internal/util"
"github.com/prometheus/procfs/internal/parsers"
)
// ProcSnmp models the content of /proc/<pid>/net/snmp.
@@ -135,7 +135,7 @@ type UdpLite struct { // nolint:revive
func (p Proc) Snmp() (ProcSnmp, error) {
filename := p.path("net/snmp")
data, err := util.ReadFileNoStat(filename)
data, err := parsers.ReadFileNoStat(filename)
if err != nil {
return ProcSnmp{PID: p.PID}, err
}
+2 -2
View File
@@ -22,7 +22,7 @@ import (
"strconv"
"strings"
"github.com/prometheus/procfs/internal/util"
"github.com/prometheus/procfs/internal/parsers"
)
// ProcSnmp6 models the content of /proc/<pid>/net/snmp6.
@@ -140,7 +140,7 @@ type UdpLite6 struct { // nolint:revive
func (p Proc) Snmp6() (ProcSnmp6, error) {
filename := p.path("net/snmp6")
data, err := util.ReadFileNoStat(filename)
data, err := parsers.ReadFileNoStat(filename)
if err != nil {
// On systems with IPv6 disabled, this file won't exist.
// Do nothing.
+2 -2
View File
@@ -18,7 +18,7 @@ import (
"fmt"
"os"
"github.com/prometheus/procfs/internal/util"
"github.com/prometheus/procfs/internal/parsers"
)
// Originally, this USER_HZ value was dynamically retrieved via a sysconf call
@@ -134,7 +134,7 @@ func (p Proc) NewStat() (ProcStat, error) {
// Stat returns the current status information of the process.
func (p Proc) Stat() (ProcStat, error) {
data, err := util.ReadFileNoStat(p.path("stat"))
data, err := parsers.ReadFileNoStat(p.path("stat"))
if err != nil {
return ProcStat{}, err
}
+2 -2
View File
@@ -18,7 +18,7 @@ import (
"strconv"
"strings"
"github.com/prometheus/procfs/internal/util"
"github.com/prometheus/procfs/internal/parsers"
)
// - https://man7.org/linux/man-pages/man5/proc_pid_statm.5.html
@@ -53,7 +53,7 @@ func (p Proc) NewStatm() (ProcStatm, error) {
// Statm returns the current memory usage information of the process.
func (p Proc) Statm() (ProcStatm, error) {
data, err := util.ReadFileNoStat(p.path("statm"))
data, err := parsers.ReadFileNoStat(p.path("statm"))
if err != nil {
return ProcStatm{}, err
}
+2 -2
View File
@@ -20,7 +20,7 @@ import (
"strconv"
"strings"
"github.com/prometheus/procfs/internal/util"
"github.com/prometheus/procfs/internal/parsers"
)
// ProcStatus provides status information about the process,
@@ -100,7 +100,7 @@ type ProcStatus struct {
// NewStatus returns the current status information of the process.
func (p Proc) NewStatus() (ProcStatus, error) {
data, err := util.ReadFileNoStat(p.path("status"))
data, err := parsers.ReadFileNoStat(p.path("status"))
if err != nil {
return ProcStatus{}, err
}
+3 -3
View File
@@ -17,7 +17,7 @@ import (
"fmt"
"strings"
"github.com/prometheus/procfs/internal/util"
"github.com/prometheus/procfs/internal/parsers"
)
func sysctlToPath(sysctl string) string {
@@ -25,7 +25,7 @@ func sysctlToPath(sysctl string) string {
}
func (fs FS) SysctlStrings(sysctl string) ([]string, error) {
value, err := util.SysReadFile(fs.proc.Path("sys", sysctlToPath(sysctl)))
value, err := parsers.SysReadFile(fs.proc.Path("sys", sysctlToPath(sysctl)))
if err != nil {
return nil, err
}
@@ -41,7 +41,7 @@ func (fs FS) SysctlInts(sysctl string) ([]int, error) {
values := make([]int, len(fields))
for i, f := range fields {
vp := util.NewValueParser(f)
vp := parsers.NewValueParser(f)
values[i] = vp.Int()
if err := vp.Err(); err != nil {
return nil, fmt.Errorf("%w: field %d in sysctl %s is not a valid int: %w", ErrFileParse, i, sysctl, err)
Loaded 100 of 107 files, more files were not shown because too many files have changed in this diff. Show more