Merge pull request #54132 from tonistiigi/update-buildkit-v0.35-rc1

vendor: update buildkit v0.35.0-rc1
This commit is contained in:
Paweł Gronowski authored and GitHub committed 2026-11-05 20:00:00 +01:00
commit b0e1abe292
93 files changed
+4236 -861

No files matched your search

+3 -3
View File
@@ -64,7 +64,7 @@ require (
github.com/miekg/dns v1.1.73
github.com/mistifyio/go-zfs/v4 v4.0.0
github.com/mitchellh/copystructure v1.2.0
github.com/moby/buildkit v0.34.2
github.com/moby/buildkit v0.35.0-rc1
github.com/moby/docker-image-spec v1.3.1
github.com/moby/extensions v0.0.0-20261020100352-d2819d88bcbe
github.com/moby/go-archive v0.3.3
@@ -84,7 +84,7 @@ require (
github.com/moby/sys/reexec v0.1.0
github.com/moby/sys/signal v0.7.1
github.com/moby/sys/symlink v0.3.0
github.com/moby/sys/user v0.4.1
github.com/moby/sys/user v0.5.0
github.com/moby/sys/userns v0.2.1
github.com/moby/term v0.5.2
github.com/opencontainers/cgroups v0.1.0
@@ -165,7 +165,7 @@ require (
github.com/containerd/go-runc v1.2.1 // indirect
github.com/containerd/nydus-snapshotter v0.15.15 // indirect
github.com/containerd/otelttrpc v0.1.0 // indirect
github.com/containerd/stargz-snapshotter/estargz v0.18.2 // indirect
github.com/containerd/stargz-snapshotter/estargz v0.19.1 // indirect
github.com/containernetworking/cni v1.3.1 // indirect
github.com/containernetworking/plugins v1.9.1 // indirect
github.com/coreos/go-semver v0.3.1 // indirect
+10 -10
View File
@@ -174,8 +174,8 @@ github.com/containerd/platforms v1.0.0-rc.5 h1:vXd569rDrz8LeMXzAnBsy6LADV5YtsD8o
github.com/containerd/platforms v1.0.0-rc.5/go.mod h1:lKlMXyLybmBedS/JJm11uDofzI8L2v0J2ZbYvNsbq1A=
github.com/containerd/plugin v1.1.0 h1:O+7lczNJVMy8rz0YNx3xGB8tTf5qY4i5abF041Ew19U=
github.com/containerd/plugin v1.1.0/go.mod h1:qBTum+A8lJ6lO44A19Eo7y1OlcLj4OWFH1DA/vnHmcc=
github.com/containerd/stargz-snapshotter/estargz v0.18.2 h1:yXkZFYIzz3eoLwlTUZKz2iQ4MrckBxJjkmD16ynUTrw=
github.com/containerd/stargz-snapshotter/estargz v0.18.2/go.mod h1:XyVU5tcJ3PRpkA9XS2T5us6Eg35yM0214Y+wvrZTBrY=
github.com/containerd/stargz-snapshotter/estargz v0.19.1 h1:PD/uZYqmyyEXZl3fEPM2ieCpilv7Oow9FC2OCBfkDRs=
github.com/containerd/stargz-snapshotter/estargz v0.19.1/go.mod h1:Cr0pxOkuN7Y61+Sik6X6/nPtePW7hsKWdBt+MoHQRr0=
github.com/containerd/ttrpc v1.2.10 h1:qQvy3mokhoXON7FXiOOc05O7Up6nrNj6amJF8urNvDc=
github.com/containerd/ttrpc v1.2.10/go.mod h1:YrmEQKkMbBA4EM26AzVAIwHGIYgptkL4CJ/yVTaDyBE=
github.com/containerd/typeurl/v2 v2.3.0 h1:HZHPhRWo5XMy3QGQoPrUzbW/2ckwjfweHmOwlkIrPAQ=
@@ -212,8 +212,8 @@ github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi
github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
github.com/docker/cli v29.8.2+incompatible h1:2zgdFuoFst2T80oS42vhKGxkd0JRo305eIEKTsxR7pQ=
github.com/docker/cli v29.8.2+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
github.com/docker/cli v29.9.3+incompatible h1:SBaMp7ecohadEHMDrUJg22t9I3awqsAVhnJ7GOcWpfE=
github.com/docker/cli v29.9.3+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk=
github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
github.com/docker/docker-credential-helpers v0.9.9 h1:BkydjIgZ46JnDbqyM2p2fc63KMw6y+KHL3Em/2AGJ7w=
@@ -469,8 +469,8 @@ github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyua
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ=
github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
github.com/moby/buildkit v0.34.2 h1:fXgOe79JW5zJNDumRtwTemOklLX66otQokObUXuiGJo=
github.com/moby/buildkit v0.34.2/go.mod h1:HyFnF6kIPzHvqopF1+BKeGY8Ee9MXmfR2n+0Gd4Y69s=
github.com/moby/buildkit v0.35.0-rc1 h1:wgD3pbfSTP03T3Q4IkMRRAHhxbz7177+mavHnwVjUQI=
github.com/moby/buildkit v0.35.0-rc1/go.mod h1:5L8B0JOdp1dO1ntCpfOYT5vR31kQ1nKIor/b4FwsDVQ=
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
github.com/moby/extensions v0.0.0-20261020100352-d2819d88bcbe h1:wHRo58MA+vgX2sXtL4uT/NcSYPiyJarqHZ4aDS8P0ys=
@@ -507,8 +507,8 @@ github.com/moby/sys/signal v0.7.1 h1:PrQxdvxcGijdo6UXXo/lU/TvHUWyPhj7UOpSo8tuvk0
github.com/moby/sys/signal v0.7.1/go.mod h1:Se1VGehYokAkrSQwL4tDzHvETwUZlnY7S5XtQ50mQp8=
github.com/moby/sys/symlink v0.3.0 h1:GZX89mEZ9u53f97npBy4Rc3vJKj7JBDj/PN2I22GrNU=
github.com/moby/sys/symlink v0.3.0/go.mod h1:3eNdhduHmYPcgsJtZXW1W4XUJdZGBIkttZ8xKqPUJq0=
github.com/moby/sys/user v0.4.1 h1:RgjRlaDKi/Xmyrz4t8lyzXT6v2ooFeO/7xtchmhVWE0=
github.com/moby/sys/user v0.4.1/go.mod h1:E9QsW5WRe1kUAf7kW8hXKwu1uhsZEAdPLYHYSDudF4Y=
github.com/moby/sys/user v0.5.0 h1:5+S4lyoavGyX1DGs68NW8xpx48T9+a7ogSUc0KjZg8Y=
github.com/moby/sys/user v0.5.0/go.mod h1:4mQzP5aTmaLjDLp/dc870oNkik6E9vjRtWYwI05gvow=
github.com/moby/sys/userns v0.2.1 h1:4OvdM7BcPkASbuouHsbW3aeMJSFlYDldBRnXVZhaRk8=
github.com/moby/sys/userns v0.2.1/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28=
github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ=
@@ -740,8 +740,8 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.47.0 h1:UFpxO
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.47.0/go.mod h1:YBGjxe3lt0jtXoEXxGrGhv/jM7oUBUDVK7zhAvNyEjc=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0 h1:aXZXsZ012wOgVkqaiQv+Z/d8V0xKjmg70F6m4CE94lc=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0/go.mod h1:ziu1gUIJhAaxM3EV1e3Ralk5AAyL1UOYz3lJJ4VSI38=
go.opentelemetry.io/otel/exporters/prometheus v0.68.0 h1:QOf2IftqQwITVRJpnn0M7M9ZCbgWfxz4P7i9C9yc2N4=
go.opentelemetry.io/otel/exporters/prometheus v0.68.0/go.mod h1:bgSvqu2TWGXiz7yr5UTMfObH8oqxJWHTnubQ3ef9BO4=
go.opentelemetry.io/otel/exporters/prometheus v0.69.0 h1:LQVBTGnHvzrfGJpDoCSVTY7eaLZArvaTSq5l84Skxew=
go.opentelemetry.io/otel/exporters/prometheus v0.69.0/go.mod h1:moif0nlgj/M16NRT8lDZVt8MC52+EOAJfR8EKRJ8LyU=
go.opentelemetry.io/otel/log v1.47.0 h1:cOTS1CcLbSQeZKanGJ+0JpF/+t4PELi3O3bbl2lqCcI=
go.opentelemetry.io/otel/log v1.47.0/go.mod h1:9byitSQ5pLC6PpqwGXjqdMKya6ZTswHRZh2vvXT33nw=
go.opentelemetry.io/otel/metric v1.47.0 h1:4PptaldXx3Eat1XjMZ68pPJEs5wrhlemctZE9a3UdWY=
+38 -16
View File
@@ -37,10 +37,8 @@ import (
"sync"
"sync/atomic"
"github.com/containerd/stargz-snapshotter/estargz/errorutil"
"github.com/klauspost/compress/zstd"
digest "github.com/opencontainers/go-digest"
"golang.org/x/sync/errgroup"
)
type GzipHelperFunc func(io.Reader) (io.ReadCloser, error)
@@ -53,6 +51,7 @@ type options struct {
compression Compression
ctx context.Context
minChunkSize int
parallelism int
gzipHelperFunc GzipHelperFunc
}
@@ -131,6 +130,20 @@ func WithMinChunkSize(minChunkSize int) Option {
}
}
// WithParallelism option specifies the number of workers used to build the
// layer. The tar is sliced into this many parts that are compressed
// concurrently, so the value also fixes the resulting chunk boundaries: pinning
// it makes builds reproducible across machines regardless of their CPU count.
// Zero (the default) selects runtime.GOMAXPROCS(0); a value of 1 forces a fully
// sequential build. This has no effect together with WithMinChunkSize, which is
// always built with a single worker.
func WithParallelism(workers int) Option {
return func(o *options) error {
o.parallelism = workers
return nil
}
}
// WithGzipHelperFunc option specifies a custom function to decompress gzip-compressed layers.
// When a gzip-compressed layer is detected, this function will be used instead of the
// Go standard library gzip decompression for better performance.
@@ -225,25 +238,32 @@ func Build(tarBlob *io.SectionReader, opt ...Option) (_ *Blob, rErr error) {
if err != nil {
return nil, err
}
// The worker count slices the tar and so fixes the chunk boundaries; an
// explicit value is honored verbatim, which keeps builds reproducible across
// machines. Zero falls back to GOMAXPROCS.
workers := opts.parallelism
if workers <= 0 {
workers = runtime.GOMAXPROCS(0)
}
var tarParts [][]*entry
if opts.minChunkSize > 0 {
// Each entry needs to know the size of the current gzip stream so they
// cannot be processed in parallel.
tarParts = [][]*entry{entries}
} else {
tarParts = divideEntries(entries, runtime.GOMAXPROCS(0))
tarParts = divideEntries(entries, workers)
}
writers := make([]*Writer, len(tarParts))
payloads := make([]*os.File, len(tarParts))
var mu sync.Mutex
var eg errgroup.Group
var wg sync.WaitGroup
errCh := make(chan error, len(tarParts)) // buffered to avoid goroutine leaks
for i, parts := range tarParts {
i, parts := i, parts
// builds verifiable stargz sub-blobs
eg.Go(func() error {
wg.Go(func() {
esgzFile, err := layerFiles.TempFile("", "esgzdata")
if err != nil {
return err
errCh <- err
return
}
sw := NewWriterWithCompressor(esgzFile, opts.compression)
sw.ChunkSize = opts.chunkSize
@@ -255,18 +275,20 @@ func Build(tarBlob *io.SectionReader, opt ...Option) (_ *Blob, rErr error) {
sw.needsOpenGzEntries[f] = struct{}{}
}
if err := sw.AppendTar(readerFromEntries(parts...)); err != nil {
return err
errCh <- err
return
}
mu.Lock()
writers[i] = sw
payloads[i] = esgzFile
mu.Unlock()
return nil
})
}
if err := eg.Wait(); err != nil {
rErr = err
return nil, err
wg.Wait()
close(errCh)
for err := range errCh {
if err != nil {
return nil, err
}
}
tocAndFooter, tocDgst, err := closeWithCombine(writers...)
if err != nil {
@@ -656,7 +678,7 @@ func (tf *tempFiles) cleanupAll() error {
}
}
tf.files = nil
return errorutil.Aggregate(allErr)
return errors.Join(allErr...)
}
func newCountReadSeeker(r io.ReaderAt) (*countReadSeeker, error) {
@@ -1,40 +0,0 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package errorutil
import (
"errors"
"fmt"
"strings"
)
// Aggregate combines a list of errors into a single new error.
func Aggregate(errs []error) error {
switch len(errs) {
case 0:
return nil
case 1:
return errs[0]
default:
points := make([]string, len(errs)+1)
points[0] = fmt.Sprintf("%d error(s) occurred:", len(errs))
for i, err := range errs {
points[i+1] = fmt.Sprintf("* %s", err)
}
return errors.New(strings.Join(points, "\n\t"))
}
}
+12 -8
View File
@@ -38,7 +38,6 @@ import (
"sync"
"time"
"github.com/containerd/stargz-snapshotter/estargz/errorutil"
digest "github.com/opencontainers/go-digest"
"github.com/vbatts/tar-split/archive/tar"
)
@@ -164,7 +163,7 @@ func Open(sr *io.SectionReader, opt ...OpenOption) (*Reader, error) {
allErr = append(allErr, err)
}
if !found {
return nil, errorutil.Aggregate(allErr)
return nil, errors.Join(allErr...)
}
if err := r.initFields(); err != nil {
return nil, fmt.Errorf("failed to initialize fields of entries: %v", err)
@@ -192,7 +191,7 @@ func OpenFooter(sr *io.SectionReader) (tocOffset int64, footerSize int64, rErr e
}
allErr = append(allErr, err)
}
return 0, 0, errorutil.Aggregate(allErr)
return 0, 0, errors.Join(allErr...)
}
// initFields populates the Reader from r.toc after decoding it from
@@ -286,7 +285,7 @@ func (r *Reader) initFields() error {
pdir := r.getOrCreateDir(pdirName)
ent.NumLink++ // at least one name(ent.Name) references this entry.
if ent.Type == "hardlink" {
org, err := r.getSource(ent)
org, err := r.getSource(ent, make(map[string]struct{}))
if err != nil {
return err
}
@@ -319,13 +318,18 @@ func (r *Reader) initFields() error {
return nil
}
func (r *Reader) getSource(ent *TOCEntry) (_ *TOCEntry, err error) {
func (r *Reader) getSource(ent *TOCEntry, visited map[string]struct{}) (_ *TOCEntry, err error) {
if ent.Type == "hardlink" {
org, ok := r.m[cleanEntryName(ent.LinkName)]
entName := cleanEntryName(ent.LinkName)
if _, ok := visited[entName]; ok {
return nil, fmt.Errorf("cyclic hardlink detected")
}
visited[entName] = struct{}{}
org, ok := r.m[entName]
if !ok {
return nil, fmt.Errorf("%q is a hardlink but the linkname %q isn't found", ent.Name, ent.LinkName)
}
ent, err = r.getSource(org)
ent, err = r.getSource(org, visited)
if err != nil {
return nil, err
}
@@ -493,7 +497,7 @@ func (r *Reader) Lookup(path string) (e *TOCEntry, ok bool) {
e, ok = r.m[path]
if ok && e.Type == "hardlink" {
var err error
e, err = r.getSource(e)
e, err = r.getSource(e, make(map[string]struct{}))
if err != nil {
return nil, false
}
+42 -8
View File
@@ -100,21 +100,52 @@ func (gc *GzipCompressor) WriteTOCAndFooter(w io.Writer, off int64, toc *JTOC, d
// gzipFooterBytes returns the 51 bytes footer.
func gzipFooterBytes(tocOff int64) []byte {
buf := bytes.NewBuffer(make([]byte, 0, FooterSize))
gz, _ := gzip.NewWriterLevel(buf, gzip.NoCompression) // MUST be NoCompression to keep 51 bytes
// Extra header indicating the offset of TOCJSON
// https://tools.ietf.org/html/rfc1952#section-2.3.1.1
header := make([]byte, 4)
header[0], header[1] = 'S', 'G'
subfield := fmt.Sprintf("%016xSTARGZ", tocOff)
binary.LittleEndian.PutUint16(header[2:4], uint16(len(subfield))) // little-endian per RFC1952
gz.Extra = append(header, []byte(subfield)...)
gz.Close()
if buf.Len() != FooterSize {
panic(fmt.Sprintf("footer buffer = %d, not %d", buf.Len(), FooterSize))
extra := append(header, []byte(subfield)...)
buf := CreateGzipFooter(extra)
if len(buf) != FooterSize {
panic(fmt.Sprintf("footer buffer = %d, not %d", len(buf), FooterSize))
}
return buf.Bytes()
return buf
}
// CreateGzipFooter creates eStargz's footer with the specified extra field.
func CreateGzipFooter(extra []byte) []byte {
// Gzip header (10 bytes)
// https://datatracker.ietf.org/doc/html/rfc1952
buf := make([]byte, 10)
buf[0] = 0x1f // ID1
buf[1] = 0x8b // ID2
buf[2] = 8 // "deflate"
buf[3] = 0x4 // FEXTRA
// MTIME = 0
// XFL = 0
buf[9] = 255 // unknown
// Extra field
xlen := make([]byte, 2)
binary.LittleEndian.PutUint16(xlen, uint16(len(extra)))
buf = append(buf, append(xlen, extra...)...)
// Flate header (5 bytes)
// https://datatracker.ietf.org/doc/html/rfc1951
flateHeader := []byte{
1, // BFINAL = 1 (last block), BFTYPE = 0 (no compression), remaining bits are ignored
0, 0, // LEN = 0
0xff, 0xff, // NLEN (the one's complement of LEN)
}
buf = append(buf, flateHeader...)
// Gzip footer (8 bytes): CRC32 = 0, ISIZE = 0
buf = append(buf, make([]byte, 8)...)
return buf
}
type GzipDecompressor struct{}
@@ -137,6 +168,9 @@ func (gz *GzipDecompressor) ParseFooter(p []byte) (blobPayloadSize, tocOffset, t
}
defer zr.Close()
extra := zr.Extra
if len(extra) < 4 {
return 0, 0, 0, fmt.Errorf("invalid extra field length %d; expected >= 4", len(extra))
}
si1, si2, subfieldlen, subfield := extra[0], extra[1], extra[2:4], extra[4:]
if si1 != 'S' || si2 != 'G' {
return 0, 0, 0, fmt.Errorf("invalid subfield IDs: %q, %q; want E, S", si1, si2)
+45 -43
View File
@@ -40,7 +40,6 @@ import (
"strings"
"time"
"github.com/containerd/stargz-snapshotter/estargz/errorutil"
"github.com/klauspost/compress/zstd"
digest "github.com/opencontainers/go-digest"
)
@@ -187,15 +186,10 @@ func testBuild(t *TestRunner, controllers ...TestingControllerFactory) {
tt.minChunkSize = []int{0}
}
for _, srcCompression := range srcCompressions {
srcCompression := srcCompression
for _, newCL := range controllers {
newCL := newCL
for _, srcTarFormat := range []tar.Format{tar.FormatUSTAR, tar.FormatPAX, tar.FormatGNU} {
srcTarFormat := srcTarFormat
for _, prefix := range allowedPrefix {
prefix := prefix
for _, minChunkSize := range tt.minChunkSize {
minChunkSize := minChunkSize
t.Run(tt.name+"-"+fmt.Sprintf("compression=%v,prefix=%q,src=%d,format=%s,minChunkSize=%d", newCL(), prefix, srcCompression, srcTarFormat, minChunkSize), func(t *TestRunner) {
tarBlob := buildTar(t, tt.in, prefix, srcTarFormat)
// Test divideEntries()
@@ -675,15 +669,10 @@ func testDigestAndVerify(t *TestRunner, controllers ...TestingControllerFactory)
tt.minChunkSize = []int{0}
}
for _, srcCompression := range srcCompressions {
srcCompression := srcCompression
for _, newCL := range controllers {
newCL := newCL
for _, prefix := range allowedPrefix {
prefix := prefix
for _, srcTarFormat := range []tar.Format{tar.FormatUSTAR, tar.FormatPAX, tar.FormatGNU} {
srcTarFormat := srcTarFormat
for _, minChunkSize := range tt.minChunkSize {
minChunkSize := minChunkSize
t.Run(tt.name+"-"+fmt.Sprintf("compression=%v,prefix=%q,format=%s,minChunkSize=%d", newCL(), prefix, srcTarFormat, minChunkSize), func(t *TestRunner) {
// Get original tar file and chunk digests
dgstMap := make(map[string]digest.Digest)
@@ -1170,6 +1159,8 @@ func testWriteAndOpen(t *TestRunner, controllers ...TestingControllerFactory) {
wantNumGzLossLess int // expected number of streams (> 0) in lossless mode if it's different from wantNumGz
wantFailOnLossLess bool
wantTOCVersion int // default = 1
wantOpenError bool // opening this blob should fail with returning an error
}{
{
name: "empty",
@@ -1484,15 +1475,19 @@ func testWriteAndOpen(t *TestRunner, controllers ...TestingControllerFactory) {
hasFileContentsRange("foo3", len(data64KB)-1, data64KB[len(data64KB)-1:]),
),
},
{
name: "cyclic_hardlink",
in: tarOf(
link("foo", "foo"),
),
wantOpenError: true,
},
}
for _, tt := range tests {
for _, newCL := range controllers {
newCL := newCL
for _, prefix := range allowedPrefix {
prefix := prefix
for _, srcTarFormat := range []tar.Format{tar.FormatUSTAR, tar.FormatPAX, tar.FormatGNU} {
srcTarFormat := srcTarFormat
for _, lossless := range []bool{true, false} {
t.Run(tt.name+"-"+fmt.Sprintf("compression=%v,prefix=%q,lossless=%v,format=%s", newCL(), prefix, lossless, srcTarFormat), func(t *TestRunner) {
var tr io.Reader = buildTar(t, tt.in, prefix, srcTarFormat)
@@ -1558,6 +1553,9 @@ func testWriteAndOpen(t *TestRunner, controllers ...TestingControllerFactory) {
WithTelemetry(telemetry),
)
if err != nil {
if tt.wantOpenError {
return
}
t.Fatalf("stargz.Open: %v", err)
}
if _, ok := r.Lookup(""); !ok {
@@ -1634,24 +1632,24 @@ func newCalledTelemetry() (telemetry *Telemetry, check func(needsGetTOC bool) er
var getTocLatencyCalled bool
var deserializeTocLatencyCalled bool
return &Telemetry{
func(time.Time) { getFooterLatencyCalled = true },
func(time.Time) { getTocLatencyCalled = true },
func(time.Time) { deserializeTocLatencyCalled = true },
}, func(needsGetTOC bool) error {
var allErr []error
if !getFooterLatencyCalled {
allErr = append(allErr, fmt.Errorf("metrics GetFooterLatency isn't called"))
}
if needsGetTOC {
if !getTocLatencyCalled {
allErr = append(allErr, fmt.Errorf("metrics GetTocLatency isn't called"))
}
}
if !deserializeTocLatencyCalled {
allErr = append(allErr, fmt.Errorf("metrics DeserializeTocLatency isn't called"))
}
return errorutil.Aggregate(allErr)
func(time.Time) { getFooterLatencyCalled = true },
func(time.Time) { getTocLatencyCalled = true },
func(time.Time) { deserializeTocLatencyCalled = true },
}, func(needsGetTOC bool) error {
var allErr []error
if !getFooterLatencyCalled {
allErr = append(allErr, fmt.Errorf("metrics GetFooterLatency isn't called"))
}
if needsGetTOC {
if !getTocLatencyCalled {
allErr = append(allErr, fmt.Errorf("metrics GetTocLatency isn't called"))
}
}
if !deserializeTocLatencyCalled {
allErr = append(allErr, fmt.Errorf("metrics DeserializeTocLatency isn't called"))
}
return errors.Join(allErr...)
}
}
func digestFor(content string) string {
@@ -2072,7 +2070,7 @@ func (f tarEntryFunc) appendTar(tw *tar.Writer, prefix string, format tar.Format
return f(tw, prefix, format)
}
func buildTar(t TestingT, ents []tarEntry, prefix string, opts ...interface{}) *io.SectionReader {
func buildTar(t TestingT, ents []tarEntry, prefix string, opts ...any) *io.SectionReader {
format := tar.FormatUnknown
for _, opt := range opts {
switch v := opt.(type) {
@@ -2096,7 +2094,7 @@ func buildTar(t TestingT, ents []tarEntry, prefix string, opts ...interface{}) *
return io.NewSectionReader(bytes.NewReader(data), 0, int64(len(data)))
}
func dir(name string, opts ...interface{}) tarEntry {
func dir(name string, opts ...any) tarEntry {
return tarEntryFunc(func(tw *tar.Writer, prefix string, format tar.Format) error {
var o owner
mode := os.FileMode(0755)
@@ -2137,7 +2135,7 @@ type owner struct {
gid int
}
func file(name, contents string, opts ...interface{}) tarEntry {
func file(name, contents string, opts ...any) tarEntry {
return tarEntryFunc(func(tw *tar.Writer, prefix string, format tar.Format) error {
var xattrs xAttr
var o owner
@@ -2164,15 +2162,19 @@ func file(name, contents string, opts ...interface{}) tarEntry {
if len(xattrs) > 0 {
format = tar.FormatPAX // only PAX supports xattrs
}
xattrsPAXRecords := make(map[string]string)
for k, v := range xattrs {
xattrsPAXRecords["SCHILY.xattr."+k] = v
}
if err := tw.WriteHeader(&tar.Header{
Typeflag: tar.TypeReg,
Name: prefix + name,
Mode: tm,
Xattrs: xattrs,
Size: int64(len(contents)),
Uid: o.uid,
Gid: o.gid,
Format: format,
Typeflag: tar.TypeReg,
Name: prefix + name,
Mode: tm,
PAXRecords: xattrsPAXRecords,
Size: int64(len(contents)),
Uid: o.uid,
Gid: o.gid,
Format: format,
}); err != nil {
return err
}
@@ -2349,7 +2351,7 @@ func (f fileInfoOnlyMode) Size() int64 { return 0 }
func (f fileInfoOnlyMode) Mode() os.FileMode { return os.FileMode(f) }
func (f fileInfoOnlyMode) ModTime() time.Time { return time.Now() }
func (f fileInfoOnlyMode) IsDir() bool { return os.FileMode(f).IsDir() }
func (f fileInfoOnlyMode) Sys() interface{} { return nil }
func (f fileInfoOnlyMode) Sys() any { return nil }
func CheckGzipHasStreams(t TestingT, b []byte, streams []int64) {
if len(streams) == 0 {
+1 -1
View File
@@ -246,7 +246,7 @@ func (fi fileInfo) Name() string { return path.Base(fi.e.Name) }
func (fi fileInfo) IsDir() bool { return fi.e.Type == "dir" }
func (fi fileInfo) Size() int64 { return fi.e.Size }
func (fi fileInfo) ModTime() time.Time { return fi.e.ModTime() }
func (fi fileInfo) Sys() interface{} { return fi.e }
func (fi fileInfo) Sys() any { return fi.e }
func (fi fileInfo) Mode() (m os.FileMode) {
// TOCEntry.Mode is tar.Header.Mode so we can understand the these bits using `tar` pkg.
m = (&tar.Header{Mode: fi.e.Mode}).FileInfo().Mode() &
+11 -3
View File
@@ -394,6 +394,11 @@ func (cm *cacheManager) init(ctx context.Context) error {
cm.LeaseManager.Delete(ctx, leases.Lease{ID: si.ID() + "-variants"})
}
}
if n, err := cm.MetadataStore.ClearOrphans(); err != nil {
bklog.G(ctx).Warnf("failed to remove external metadata of deleted cache records: %v", err)
} else if n > 0 {
bklog.G(ctx).Infof("removed external metadata of %d deleted cache records", n)
}
return nil
}
@@ -470,7 +475,10 @@ func (cm *cacheManager) getRecord(ctx context.Context, id string, opts ...RefOpt
}
if rec, ok := cm.records[id]; ok {
if rec.isDead() {
rec.mu.Lock()
isDead := rec.isDead()
rec.mu.Unlock()
if isDead {
return nil, errors.Wrapf(errNotFound, "failed to get dead record %s", id)
}
if err := checkLazyProviders(rec); err != nil {
@@ -498,7 +506,7 @@ func (cm *cacheManager) getRecord(ctx context.Context, id string, opts ...RefOpt
mutable, err := cm.getRecord(ctx, mutableID)
if err == nil {
rec := &cacheRecord{
mu: &sync.Mutex{},
mu: mutable.mu, // share the mutex as in mutableRef.commit
cm: cm,
refs: make(map[ref]struct{}),
parentRefs: parents,
@@ -570,10 +578,10 @@ func (cm *cacheManager) getRecord(ctx context.Context, id string, opts ...RefOpt
return nil, errors.Wrapf(err, "failed to append image ref metadata to ref %s", rec.ID())
}
cm.records[id] = rec
if err := checkLazyProviders(rec); err != nil {
return nil, err
}
cm.records[id] = rec
return rec, nil
}
+38
View File
@@ -190,6 +190,40 @@ func (s *Store) Clear(id string) error {
}))
}
func (s *Store) ClearOrphans() (int, error) {
var n int
err := s.db.Update(func(tx *bolt.Tx) error {
external := tx.Bucket([]byte(externalBucket))
if external == nil {
return nil
}
var records *bolt.Cursor
if main := tx.Bucket([]byte(mainBucket)); main != nil {
records = main.Cursor()
}
var orphans [][]byte
if err := external.ForEach(func(k, _ []byte) error {
if records != nil {
if id, v := records.Seek(k); bytes.Equal(id, k) && v == nil {
return nil
}
}
orphans = append(orphans, append([]byte(nil), k...))
return nil
}); err != nil {
return err
}
for _, id := range orphans {
if err := external.DeleteBucket(id); err != nil {
return err
}
}
n = len(orphans)
return nil
})
return n, errors.WithStack(err)
}
func (s *Store) Update(id string, fn func(b *bolt.Bucket) error) error {
return errors.WithStack(s.db.Update(func(tx *bolt.Tx) error {
b, err := tx.CreateBucketIfNotExists([]byte(mainBucket))
@@ -330,6 +364,10 @@ func (s *StorageItem) GetExternal(k string) ([]byte, error) {
func (s *StorageItem) SetExternal(k string, dt []byte) error {
return errors.WithStack(s.storage.db.Update(func(tx *bolt.Tx) error {
main := tx.Bucket([]byte(mainBucket))
if main == nil || main.Bucket([]byte(s.id)) == nil {
return errors.Wrapf(errNotFound, "record %s", s.id)
}
b, err := tx.CreateBucketIfNotExists([]byte(externalBucket))
if err != nil {
return errors.WithStack(err)
+79 -30
View File
@@ -103,6 +103,10 @@ type cacheRecord struct {
equalMutable *mutableRef
equalImmutable *immutableRef
// layerDigestChainCache memoizes layerDigestChain. It is only set when
// every layer has a blob, as a blob can't be replaced after it is set.
// Access is serialized by cacheManager.mu, as computing the chain of a
// record also fills the memoized chains of its parents.
layerDigestChainCache []digest.Digest
}
@@ -554,18 +558,10 @@ func (sr *immutableRef) layerWalk(f func(*immutableRef)) {
parent.layerWalk(f)
}
case Diff:
lower := sr.diffParents.lower
upper := sr.diffParents.upper
// If upper is only one blob different from lower, then re-use that blob
switch {
case upper != nil && lower == nil && upper.kind() == BaseLayer:
// upper is a single layer being diffed with scratch
f(upper)
case upper != nil && lower != nil && upper.kind() == Layer && upper.layerParent.ID() == lower.ID():
// upper is a single layer on top of lower
f(upper)
default:
// otherwise, the diff will be computed and turned into its own single blob
if sr.diffReusesUpper() {
f(sr.diffParents.upper)
} else {
// the diff will be computed and turned into its own single blob
f(sr)
}
case Layer:
@@ -576,30 +572,65 @@ func (sr *immutableRef) layerWalk(f func(*immutableRef)) {
}
}
// hold cacheRecord.mu lock before calling
// diffReusesUpper reports whether the Diff record cr is exported as the blob
// of its upper, because upper is only one blob different from lower. Such a
// diff never gets a blob of its own.
func (cr *cacheRecord) diffReusesUpper() bool {
lower := cr.diffParents.lower
upper := cr.diffParents.upper
switch {
case upper != nil && lower == nil && upper.kind() == BaseLayer:
// upper is a single layer being diffed with scratch
return true
case upper != nil && lower != nil && upper.kind() == Layer && upper.layerParent.ID() == lower.ID():
// upper is a single layer on top of lower
return true
default:
return false
}
}
// layerDigestChain returns the blob digests of the layers of cr, or nil if any
// of them has no blob yet. Such a record has never been exported, so no image
// can reference it. The caller must hold cacheManager.mu and cacheRecord.mu.
func (cr *cacheRecord) layerDigestChain() []digest.Digest {
if cr.layerDigestChainCache != nil {
return cr.layerDigestChainCache
}
// The own blob of a record is checked before its parents, so that a
// record without a blob doesn't walk its ancestors on every call.
var chain []digest.Digest
switch cr.kind() {
case Diff:
if cr.getBlob() == "" && cr.diffParents.upper != nil {
// this diff just reuses the upper blob
cr.layerDigestChainCache = cr.diffParents.upper.layerDigestChain()
} else {
cr.layerDigestChainCache = append(cr.layerDigestChainCache, cr.getBlob())
if blob := cr.getBlob(); blob != "" {
chain = []digest.Digest{blob}
} else if cr.diffReusesUpper() {
// export uses the blob of upper as the only layer of the diff
if blob := cr.diffParents.upper.getBlob(); blob != "" {
chain = []digest.Digest{blob}
}
}
case Merge:
for _, parent := range cr.mergeParents {
cr.layerDigestChainCache = append(cr.layerDigestChainCache, parent.layerDigestChain()...)
parentChain := parent.layerDigestChain()
if parentChain == nil {
return nil
}
chain = append(chain, parentChain...)
}
case Layer:
cr.layerDigestChainCache = append(cr.layerDigestChainCache, cr.layerParent.layerDigestChain()...)
fallthrough
if blob := cr.getBlob(); blob != "" {
if parentChain := cr.layerParent.layerDigestChain(); parentChain != nil {
chain = slices.Concat(parentChain, []digest.Digest{blob})
}
}
case BaseLayer:
cr.layerDigestChainCache = append(cr.layerDigestChainCache, cr.getBlob())
if blob := cr.getBlob(); blob != "" {
chain = []digest.Digest{blob}
}
}
return cr.layerDigestChainCache
cr.layerDigestChainCache = chain
return chain
}
type RefList []ImmutableRef
@@ -967,7 +998,9 @@ func fieldsFromLabels(labels map[string]string) (fields []string) {
}
func (sr *immutableRef) Mount(ctx context.Context, readonly bool, s session.Group) (_ snapshot.Mountable, rerr error) {
if sr.equalMutable != nil && !readonly {
// Finalize is a no-op if there is no equalMutable. Don't check it here,
// equalMutable is protected by sr.mu.
if !readonly {
if err := sr.Finalize(ctx); err != nil {
return nil, err
}
@@ -1536,6 +1569,11 @@ func (cr *cacheRecord) finalize(ctx context.Context) error {
return nil
}
// The snapshotter may apply the commit even when the caller sees a
// cancellation (e.g. a containerd RPC canceled in flight), leaving the
// mutable snapshot gone while the record still points to it.
ctx = context.WithoutCancel(ctx)
_, err := cr.cm.LeaseManager.Create(ctx, func(l *leases.Lease) error {
l.ID = cr.ID()
l.Labels = map[string]string{
@@ -1553,13 +1591,23 @@ func (cr *cacheRecord) finalize(ctx context.Context) error {
ID: cr.getSnapshotID(),
Type: "snapshots/" + cr.cm.Snapshotter.Name(),
}); err != nil {
_ = cr.cm.LeaseManager.Delete(context.WithoutCancel(ctx), leases.Lease{ID: cr.ID()})
_ = cr.cm.LeaseManager.Delete(ctx, leases.Lease{ID: cr.ID()})
return errors.Wrapf(err, "failed to add snapshot %s to lease", cr.getSnapshotID())
}
if err := cr.cm.Snapshotter.Commit(ctx, cr.getSnapshotID(), mutable.getSnapshotID()); err != nil {
_ = cr.cm.LeaseManager.Delete(context.WithoutCancel(ctx), leases.Lease{ID: cr.ID()})
return errors.Wrapf(err, "failed to commit %s to %s during finalize", mutable.getSnapshotID(), cr.getSnapshotID())
// the commit may still have been applied, e.g. if the response was lost
info, statErr := cr.cm.Snapshotter.Stat(ctx, cr.getSnapshotID())
if statErr != nil || info.Kind != snapshots.KindCommitted {
// keep the lease if it is unknown whether the commit was applied
if statErr == nil || errors.Is(statErr, cerrdefs.ErrNotFound) {
_ = cr.cm.LeaseManager.Delete(ctx, leases.Lease{ID: cr.ID()})
} else {
bklog.G(ctx).WithError(statErr).Warnf("failed to check snapshot %s after commit error, keeping lease", cr.getSnapshotID())
}
return errors.Wrapf(err, "failed to commit %s to %s during finalize", mutable.getSnapshotID(), cr.getSnapshotID())
}
bklog.G(ctx).WithError(err).Warnf("snapshot %s was committed to %s despite commit error", mutable.getSnapshotID(), cr.getSnapshotID())
}
cr.mountCache = nil
@@ -1567,9 +1615,8 @@ func (cr *cacheRecord) finalize(ctx context.Context) error {
go func() {
cr.cm.mu.Lock()
defer cr.cm.mu.Unlock()
cleanupCtx := context.WithoutCancel(ctx)
if err := mutable.remove(cleanupCtx, true); err != nil {
bklog.G(cleanupCtx).Error(err)
if err := mutable.remove(ctx, true); err != nil {
bklog.G(ctx).Error(err)
}
}()
@@ -1856,6 +1903,8 @@ func (sm *sharableMountable) Mount() (_ []mount.Mount, _ func() error, retErr er
return nil, nil, err
}
}
mounts = mount.RemoveVolatileOption(mounts)
if err := mount.All(mounts, dir); err != nil {
return nil, nil, err
}
+21 -117
View File
@@ -7,7 +7,6 @@ import (
"fmt"
"io"
"os"
"os/exec"
"strconv"
"strings"
"sync"
@@ -17,7 +16,7 @@ import (
"github.com/containerd/containerd/v2/pkg/labels"
cerrdefs "github.com/containerd/errdefs"
"github.com/moby/buildkit/cache/remotecache"
"github.com/moby/buildkit/cache/remotecache/gha/ghatypes"
"github.com/moby/buildkit/cache/remotecache/signing"
v1 "github.com/moby/buildkit/cache/remotecache/v1"
cacheimporttypes "github.com/moby/buildkit/cache/remotecache/v1/types"
"github.com/moby/buildkit/session"
@@ -28,11 +27,9 @@ import (
"github.com/moby/buildkit/util/tracing"
bkversion "github.com/moby/buildkit/version"
"github.com/moby/buildkit/worker"
policy "github.com/moby/policy-helpers"
digest "github.com/opencontainers/go-digest"
ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
"github.com/sigstore/sigstore-go/pkg/fulcio/certificate"
actionscache "github.com/tonistiigi/go-actions-cache"
"golang.org/x/sync/errgroup"
)
@@ -55,8 +52,6 @@ const (
defaultTimeout = 10 * time.Minute
)
type VerifierProvider func() (*policy.Verifier, error)
type Config struct {
Scope string
URL string
@@ -66,11 +61,11 @@ type Config struct {
Version int
Timeout time.Duration
*ghatypes.CacheConfig
verifier VerifierProvider
*signing.Config
verifier signing.Verifier
}
func getConfig(conf *ghatypes.CacheConfig, v VerifierProvider, attrs map[string]string) (*Config, error) {
func getConfig(conf *signing.Config, v signing.VerifierProvider, attrs map[string]string) (*Config, error) {
scope, ok := attrs[attrScope]
if !ok {
scope = "buildkit"
@@ -120,24 +115,24 @@ func getConfig(conf *ghatypes.CacheConfig, v VerifierProvider, attrs map[string]
}
if conf == nil {
conf = &ghatypes.CacheConfig{}
conf = &signing.Config{}
}
return &Config{
Scope: scope,
URL: url,
Token: token,
Timeout: timeout,
GHToken: attrs[attrGHToken],
Repository: attrs[attrRepository],
Version: apiVersionInt,
CacheConfig: conf,
verifier: v,
Scope: scope,
URL: url,
Token: token,
Timeout: timeout,
GHToken: attrs[attrGHToken],
Repository: attrs[attrRepository],
Version: apiVersionInt,
Config: conf,
verifier: signing.NewVerifier(conf.Verify.Policy, v),
}, nil
}
// ResolveCacheExporterFunc for Github actions cache exporter.
func ResolveCacheExporterFunc(conf *ghatypes.CacheConfig, v VerifierProvider) remotecache.ResolveCacheExporterFunc {
func ResolveCacheExporterFunc(conf *signing.Config, v signing.VerifierProvider) remotecache.ResolveCacheExporterFunc {
return func(ctx context.Context, g session.Group, attrs map[string]string) (remotecache.Exporter, error) {
cfg, err := getConfig(conf, v, attrs)
if err != nil {
@@ -330,98 +325,23 @@ func (ce *exporter) Finalize(ctx context.Context) (_ map[string]string, err erro
dgst := digest.FromBytes(dt)
signDone := progress.OneOff(ctx, fmt.Sprintf("signing cache index %s", dgst))
defer signDone(err)
defer func() { signDone(err) }()
cmd := exec.CommandContext(ctx, args[0], args[1:]...) //nolint:gosec // defined in toml config
cmd.Stdin = bytes.NewReader(dt)
var out bytes.Buffer
cmd.Stdout = &out
var stderr bytes.Buffer
cmd.Stderr = &stderr
if err := cmd.Run(); err != nil {
return nil, errors.Wrapf(err, "signing command failed: %s", stderr.String())
}
// validate signature before uploading
if err := verifySignature(ctx, dgst, out.Bytes(), ce.config); err != nil {
bundle, err := signing.NewSigner(args, ce.config.verifier).Sign(ctx, dt)
if err != nil {
return nil, err
}
key := blobKey(dgst + "-sig")
if err := ce.cache.Save(ctx, key, actionscache.NewBlob(out.Bytes())); err != nil {
if err := ce.cache.Save(ctx, key, actionscache.NewBlob(bundle)); err != nil {
return nil, err
}
return nil, nil
}
func verifySignature(ctx context.Context, dgst digest.Digest, bundle []byte, config *Config) error {
v, err := config.verifier()
if err != nil {
return err
}
if v == nil {
return errors.New("no verifier available for signed github actions cache")
}
sig, err := v.VerifyArtifact(ctx, dgst, bundle, policy.WithSLSANotRequired())
if err != nil {
return err
}
if sig.Signer == nil {
return errors.New("signature verification failed: no signer found")
}
numTimestamps := len(sig.Timestamps)
numTlog := 0
for _, t := range sig.Timestamps {
if t.Type == "Tlog" {
numTlog++
}
}
policyRules := config.Verify.Policy
if policyRules.TimestampThreshold > numTimestamps {
return errors.Errorf("signature verification failed: not enough timestamp authorities: have %d, need %d", numTimestamps, policyRules.TimestampThreshold)
}
if policyRules.TlogThreshold > numTlog {
return errors.Errorf("signature verification failed: not enough tlog authorities: have %d, need %d", numTlog, policyRules.TlogThreshold)
}
certRules, err := certToStringMap(&config.Verify.Policy.Summary)
if err != nil {
return err
}
certFields, err := certToStringMap(sig.Signer)
if err != nil {
return err
}
bklog.G(ctx).Debugf("signature verification: %+v", sig)
bklog.G(ctx).Debugf("signer: %+v", sig.Signer)
for k, v := range certRules {
if v == "" {
continue
}
if !simplePatternMatch(v, certFields[k]) {
return errors.Errorf("signature verification failed: certificate field %q does not match policy (%q != %q)", k, certFields[k], v)
}
bklog.G(ctx).Debugf("certificate field %q matches policy (%q)", k, certFields[k])
}
return nil
}
func certToStringMap(cert *certificate.Summary) (map[string]string, error) {
dt, err := json.Marshal(cert)
if err != nil {
return nil, err
}
m := map[string]string{}
if err := json.Unmarshal(dt, &m); err != nil {
return nil, err
}
return m, nil
}
// ResolveCacheImporterFunc for Github actions cache importer.
func ResolveCacheImporterFunc(conf *ghatypes.CacheConfig, v VerifierProvider) remotecache.ResolveCacheImporterFunc {
func ResolveCacheImporterFunc(conf *signing.Config, v signing.VerifierProvider) remotecache.ResolveCacheImporterFunc {
return func(ctx context.Context, g session.Group, attrs map[string]string) (remotecache.Importer, ocispecs.Descriptor, error) {
cfg, err := getConfig(conf, v, attrs)
if err != nil {
@@ -518,7 +438,7 @@ func (ci *importer) loadScope(ctx context.Context, scope string) (*v1.CacheChain
verifyDone(err)
return nil, err
}
if err := verifySignature(ctx, dgst, sigBuf.Bytes(), ci.config); err != nil {
if err := ci.config.verifier.Verify(ctx, dgst, sigBuf.Bytes()); err != nil {
verifyDone(err)
return nil, err
}
@@ -648,19 +568,3 @@ func (r *readerAt) ReadAt(p []byte, off int64) (int, error) {
func (r *readerAt) Size() int64 {
return r.desc.Size
}
func simplePatternMatch(pat, s string) bool {
if pat == "*" {
return true
}
if strings.HasPrefix(pat, "*") && strings.HasSuffix(pat, "*") {
return strings.Contains(s, pat[1:len(pat)-1])
}
if strings.HasPrefix(pat, "*") {
return strings.HasSuffix(s, pat[1:])
}
if strings.HasSuffix(pat, "*") {
return strings.HasPrefix(s, pat[:len(pat)-1])
}
return s == pat
}
@@ -1,23 +0,0 @@
package ghatypes
import "github.com/sigstore/sigstore-go/pkg/fulcio/certificate"
type CacheConfig struct {
Sign *SignConfig `toml:"sign"`
Verify VerifyConfig `toml:"verify"`
}
type SignConfig struct {
Command []string `toml:"command"`
}
type VerifyConfig struct {
Required bool `toml:"required"`
Policy VerifyPolicy `toml:"policy"`
}
type VerifyPolicy struct {
TimestampThreshold int `toml:"timestampThreshold"`
TlogThreshold int `toml:"tlogThreshold"`
certificate.Summary
}
+27
View File
@@ -0,0 +1,27 @@
package signing
import "github.com/sigstore/sigstore-go/pkg/fulcio/certificate"
// Config controls signing and verification of remote cache metadata.
type Config struct {
Sign *SignConfig `toml:"sign"`
Verify VerifyConfig `toml:"verify"`
}
// SignConfig specifies the command that reads a payload and writes its bundle.
type SignConfig struct {
Command []string `toml:"command"`
}
// VerifyConfig controls whether imported cache metadata must be verified.
type VerifyConfig struct {
Required bool `toml:"required"`
Policy VerifyPolicy `toml:"policy"`
}
// VerifyPolicy constrains the verified timestamps and signing certificate.
type VerifyPolicy struct {
TimestampThreshold int `toml:"timestampThreshold"`
TlogThreshold *int `toml:"tlogThreshold"`
certificate.Summary
}
+48
View File
@@ -0,0 +1,48 @@
package signing
import (
"bytes"
"context"
"os/exec"
"slices"
digest "github.com/opencontainers/go-digest"
"github.com/pkg/errors"
)
// Signer signs the exact payload bytes and verifies the resulting bundle.
type Signer interface {
Sign(context.Context, []byte) ([]byte, error)
}
// NewSigner returns a command-based signer using verifier to check its output.
// The command is an argv array configured by the daemon administrator.
func NewSigner(command []string, verifier Verifier) Signer {
return &commandSigner{command: slices.Clone(command), verifier: verifier}
}
type commandSigner struct {
command []string
verifier Verifier
}
func (s *commandSigner) Sign(ctx context.Context, payload []byte) ([]byte, error) {
if len(s.command) == 0 {
return nil, errors.New("no signing command configured")
}
if s.verifier == nil {
return nil, errors.New("no verifier available for signed cache")
}
cmd := exec.CommandContext(ctx, s.command[0], s.command[1:]...) //nolint:gosec // defined in toml config
cmd.Stdin = bytes.NewReader(payload)
var out, stderr bytes.Buffer
cmd.Stdout = &out
cmd.Stderr = &stderr
if err := cmd.Run(); err != nil {
return nil, errors.Wrapf(err, "signing command failed: %s", stderr.String())
}
if err := s.verifier.Verify(ctx, digest.FromBytes(payload), out.Bytes()); err != nil {
return nil, err
}
return out.Bytes(), nil
}
+114
View File
@@ -0,0 +1,114 @@
package signing
import (
"context"
"encoding/json"
"strings"
"github.com/moby/buildkit/util/bklog"
policy "github.com/moby/policy-helpers"
"github.com/moby/policy-helpers/types"
digest "github.com/opencontainers/go-digest"
"github.com/pkg/errors"
"github.com/sigstore/sigstore-go/pkg/fulcio/certificate"
)
// Verifier authenticates a bundle against the digest and the configured policy.
type Verifier interface {
Verify(context.Context, digest.Digest, []byte) error
}
// VerifierProvider lazily supplies the daemon's shared Sigstore verifier.
type VerifierProvider func() (*policy.Verifier, error)
// NewVerifier returns a verifier applying the given policy to cache signatures.
func NewVerifier(rules VerifyPolicy, provider VerifierProvider) Verifier {
return &verifier{rules: rules, provider: provider}
}
type verifier struct {
rules VerifyPolicy
provider VerifierProvider
}
func (v *verifier) Verify(ctx context.Context, dgst digest.Digest, bundle []byte) error {
if v.provider == nil {
return errors.New("no verifier available for signed cache")
}
artifactVerifier, err := v.provider()
if err != nil {
return err
}
if artifactVerifier == nil {
return errors.New("no verifier available for signed cache")
}
opts := []policy.ArtifactVerifyOpt{policy.WithSLSANotRequired()}
if v.rules.TlogThreshold != nil {
opts = append(opts, policy.WithTlogThreshold(*v.rules.TlogThreshold))
}
sig, err := artifactVerifier.VerifyArtifact(ctx, dgst, bundle, opts...)
if err != nil {
return err
}
return verifyPolicy(ctx, sig, v.rules)
}
// verifyPolicy checks the authenticated certificate and timestamps.
func verifyPolicy(ctx context.Context, sig *types.SignatureInfo, policyRules VerifyPolicy) error {
if sig.Signer == nil {
return errors.New("signature verification failed: no signer found")
}
numTimestamps := len(sig.Timestamps)
if policyRules.TimestampThreshold > numTimestamps {
return errors.Errorf("signature verification failed: not enough timestamp authorities: have %d, need %d", numTimestamps, policyRules.TimestampThreshold)
}
certRules, err := certToStringMap(&policyRules.Summary)
if err != nil {
return err
}
certFields, err := certToStringMap(sig.Signer)
if err != nil {
return err
}
bklog.G(ctx).Debugf("signature verification: %+v", sig)
bklog.G(ctx).Debugf("signer: %+v", sig.Signer)
for k, v := range certRules {
if v == "" {
continue
}
if !simplePatternMatch(v, certFields[k]) {
return errors.Errorf("signature verification failed: certificate field %q does not match policy (%q != %q)", k, certFields[k], v)
}
bklog.G(ctx).Debugf("certificate field %q matches policy (%q)", k, certFields[k])
}
return nil
}
func certToStringMap(cert *certificate.Summary) (map[string]string, error) {
dt, err := json.Marshal(cert)
if err != nil {
return nil, err
}
m := map[string]string{}
if err := json.Unmarshal(dt, &m); err != nil {
return nil, err
}
return m, nil
}
func simplePatternMatch(pat, s string) bool {
if pat == "*" {
return true
}
if strings.HasPrefix(pat, "*") && strings.HasSuffix(pat, "*") {
return strings.Contains(s, pat[1:len(pat)-1])
}
if strings.HasPrefix(pat, "*") {
return strings.HasSuffix(s, pat[1:])
}
if strings.HasSuffix(pat, "*") {
return strings.HasPrefix(s, pat[:len(pat)-1])
}
return s == pat
}
+25 -32
View File
@@ -10,6 +10,7 @@ import (
cerrdefs "github.com/containerd/errdefs"
cacheimporttypes "github.com/moby/buildkit/cache/remotecache/v1/types"
"github.com/moby/buildkit/solver"
"github.com/moby/buildkit/util/bklog"
"github.com/moby/buildkit/util/compression"
digest "github.com/opencontainers/go-digest"
"github.com/pkg/errors"
@@ -136,48 +137,40 @@ func marshalRemote(ctx context.Context, r *solver.Remote, state *marshalState) s
if r.Provider != nil {
for _, d := range r.Descriptors {
if _, err := r.Provider.Info(ctx, d.Digest); err != nil {
if !cerrdefs.IsNotImplemented(err) {
return ""
}
if _, err := r.Provider.Info(ctx, d.Digest); err != nil && !cerrdefs.IsNotImplemented(err) {
bklog.G(ctx).WithError(err).Warnf("skipping cache result: blob %s is not available", d.Digest)
return ""
}
}
}
var parentID string
if len(r.Descriptors) > 1 {
r2 := &solver.Remote{
Descriptors: r.Descriptors[:len(r.Descriptors)-1],
Provider: r.Provider,
for _, desc := range r.Descriptors {
if desc.MediaType != "" {
desc = compression.ConvertAllLayerMediaTypes(ctx, true, desc)[0]
}
parentID = marshalRemote(ctx, r2, state)
}
desc := r.Descriptors[len(r.Descriptors)-1]
if desc.MediaType != "" {
desc = compression.ConvertAllLayerMediaTypes(ctx, true, desc)[0]
}
state.descriptors[desc.Digest] = DescriptorProviderPair{
Descriptor: desc,
Provider: r.Provider,
}
state.descriptors[desc.Digest] = DescriptorProviderPair{
Descriptor: desc,
Provider: r.Provider,
}
id := desc.Digest.String() + parentID
id := desc.Digest.String() + parentID
if _, ok := state.chainsByID[id]; ok {
return id
if _, ok := state.chainsByID[id]; !ok {
state.chainsByID[id] = len(state.layers)
l := cacheimporttypes.CacheLayer{
Blob: desc.Digest,
ParentIndex: -1,
}
if parentID != "" {
l.ParentIndex = state.chainsByID[parentID]
}
state.layers = append(state.layers, l)
}
parentID = id
}
state.chainsByID[id] = len(state.layers)
l := cacheimporttypes.CacheLayer{
Blob: desc.Digest,
ParentIndex: -1,
}
if parentID != "" {
l.ParentIndex = state.chainsByID[parentID]
}
state.layers = append(state.layers, l)
return id
return parentID
}
func marshalItem(ctx context.Context, it *item, state *marshalState) error {
+4 -1
View File
@@ -262,7 +262,10 @@ func parseUID(str string) (int, error) {
if str == "root" {
return 0, nil
}
uid, err := strconv.ParseInt(str, 10, 32)
// uid_t/gid_t span 0..2^32-1, matching the daemon-side parser in
// executor/oci. ParseInt here would reject the upper half of that range and
// silently turn a numeric ID into a name lookup.
uid, err := strconv.ParseUint(str, 10, 32)
if err != nil {
return 0, err
}
+15
View File
@@ -61,6 +61,21 @@ func addEnvf(key, value string, replace bool, v ...any) StateOption {
}
}
// DelEnv returns a [StateOption] which removes an environment variable from the state.
// This is the equivalent of [State.DelEnv]
func DelEnv(key string) StateOption {
return func(s State) State {
return s.withValue(keyEnv, func(ctx context.Context, c *Constraints) (any, error) {
env, err := getEnv(s)(ctx, c)
if err != nil {
return nil, err
}
deleted := env.Delete(key)
return &deleted, nil
})
}
}
// Dir returns a [StateOption] sets the working directory for the state which will be used to resolve
// relative paths as well as the working directory for [State.Run].
// See [State.With] for where to use this.
+12
View File
@@ -477,6 +477,9 @@ func Git(url, fragment string, opts ...GitOption) State {
attrs[pb.AttrGitSkipSubmodules] = "true"
addCap(&gi.Constraints, pb.CapSourceGitSkipSubmodules)
}
if gi.DebugCommands {
attrs[pb.AttrGitDebugCommands] = "true"
}
if gi.MTime != "" {
attrs[pb.AttrGitMTime] = gi.MTime
@@ -536,6 +539,7 @@ type GitInfo struct {
SkipSubmodules bool
MTime string
Advice bool
DebugCommands bool
Bundle string
BundleOCISessionID string
BundleOCIStoreID string
@@ -583,6 +587,14 @@ func GitAdvice(enabled bool) GitOption {
})
}
// GitDebugCommands controls whether executed Git commands are logged while
// resolving this git source.
func GitDebugCommands(enabled bool) GitOption {
return gitOptionFunc(func(gi *GitInfo) {
gi.DebugCommands = enabled
})
}
func KeepGitDir() GitOption {
return gitOptionFunc(func(gi *GitInfo) {
gi.KeepGitDir = true
+6
View File
@@ -326,6 +326,12 @@ func (s State) AddEnvf(key, value string, v ...any) State {
return AddEnvf(key, value, v...)(s)
}
// DelEnv returns a new [State] with the provided environment variable removed.
// See [DelEnv]
func (s State) DelEnv(key string) State {
return DelEnv(key)(s)
}
// Dir returns a new [State] with the provided working directory set.
// See [Dir]
func (s State) Dir(str string) State {
+113 -8
View File
@@ -17,6 +17,7 @@ import (
"github.com/containerd/containerd/v2/core/content"
"github.com/containerd/containerd/v2/core/images"
contentlocal "github.com/containerd/containerd/v2/plugins/content/local"
cerrdefs "github.com/containerd/errdefs"
"github.com/gofrs/flock"
controlapi "github.com/moby/buildkit/api/services/control"
"github.com/moby/buildkit/client/llb"
@@ -425,9 +426,13 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG
if err = json.Unmarshal([]byte(manifestDescJSON), &manifestDesc); err != nil {
return nil, err
}
for storePath, tag := range cacheOpt.storesToUpdate {
for storePath, store := range cacheOpt.storesToUpdate {
idx := ociindex.NewStoreIndex(storePath)
if err := idx.Put(manifestDesc, ociindex.Tag(tag)); err != nil {
if err := idx.Put(manifestDesc, ociindex.Tag(store.tag)); err != nil {
if store.ignoreError {
bklog.G(callerCtx).WithError(err).WithField("cache_store_path", storePath).Warn("failed to update cache index, ignoring")
continue
}
return nil, err
}
}
@@ -605,11 +610,16 @@ type cacheStoreRef struct {
store content.Store
}
type cacheStoreUpdate struct {
tag string
ignoreError bool
}
type cacheOptions struct {
options controlapi.CacheOptions
contentStores map[string]content.Store // key: ID of content store ("local:" + csDir)
storesToUpdate map[string]string // key: path to content store, value: tag
storesToReset []cacheStoreRef // cache stores with reset=true
storesToUpdate map[string]cacheStoreUpdate
storesToReset []cacheStoreRef // cache stores with reset=true
frontendAttrs map[string]string
}
@@ -620,7 +630,7 @@ func parseCacheOptions(ctx context.Context, isGateway bool, opt SolveOpt) (*cach
)
var storesToReset []cacheStoreRef
contentStores := make(map[string]content.Store)
storesToUpdate := make(map[string]string)
storesToUpdate := make(map[string]cacheStoreUpdate)
frontendAttrs := make(map[string]string)
for _, ex := range opt.CacheExports {
if ex.Type == "local" {
@@ -631,7 +641,7 @@ func parseCacheOptions(ctx context.Context, isGateway bool, opt SolveOpt) (*cach
if err := os.MkdirAll(csDir, 0755); err != nil {
return nil, err
}
cs, err := contentlocal.NewStore(csDir)
cs, err := newLocalCacheStore(csDir)
if err != nil {
return nil, err
}
@@ -641,8 +651,14 @@ func parseCacheOptions(ctx context.Context, isGateway bool, opt SolveOpt) (*cach
if t, ok := ex.Attrs["tag"]; ok {
tag = t
}
update := cacheStoreUpdate{tag: tag}
if v, ok := ex.Attrs["ignore-error"]; ok {
if b, err := strconv.ParseBool(v); err == nil {
update.ignoreError = b
}
}
// TODO(AkihiroSuda): support custom index JSON path and tag
storesToUpdate[csDir] = tag
storesToUpdate[csDir] = update
if v, ok := ex.Attrs["reset"]; ok {
b, err := strconv.ParseBool(v)
@@ -671,7 +687,7 @@ func parseCacheOptions(ctx context.Context, isGateway bool, opt SolveOpt) (*cach
if csDir == "" {
return nil, errors.New("local cache importer requires src")
}
cs, err := contentlocal.NewStore(csDir)
cs, err := newLocalCacheStore(csDir)
if err != nil {
bklog.G(ctx).Warning("local cache import at " + csDir + " not found due to err: " + err.Error())
continue
@@ -737,3 +753,92 @@ func parseCacheOptions(ctx context.Context, isGateway bool, opt SolveOpt) (*cach
}
return &res, nil
}
// localCacheStore coordinates ingests across clients sharing a cache directory.
// The underlying local store only locks references within one store instance.
type localCacheStore struct {
content.Store
dir string
}
func newLocalCacheStore(dir string) (content.Store, error) {
store, err := contentlocal.NewStore(dir)
if err != nil {
return nil, err
}
return &localCacheStore{Store: store, dir: dir}, nil
}
func (s *localCacheStore) lock(ref string) (*flock.Flock, error) {
dir := filepath.Join(s.dir, "ingest-locks")
if err := os.MkdirAll(dir, 0755); err != nil {
return nil, err
}
// Keep lock files after unlocking: deleting one could let another client
// lock a different inode while an existing waiter still uses the old one.
// This retains one empty file per ref, even after cache reset. The reset
// lock cannot protect their removal: not all users of this store hold it.
lock := flock.New(filepath.Join(dir, digest.FromString(ref).Encoded()))
locked, err := lock.TryLock()
if err != nil {
return nil, err
}
if !locked {
return nil, errors.Wrapf(cerrdefs.ErrUnavailable, "ingest ref %q is locked", ref)
}
return lock, nil
}
func (s *localCacheStore) Writer(ctx context.Context, opts ...content.WriterOpt) (content.Writer, error) {
var wo content.WriterOpts
for _, opt := range opts {
if err := opt(&wo); err != nil {
return nil, err
}
}
if wo.Ref == "" {
return nil, errors.Wrap(cerrdefs.ErrInvalidArgument, "ref must not be empty")
}
lock, err := s.lock(wo.Ref)
if err != nil {
return nil, err
}
w, err := s.Store.Writer(ctx, content.WithRef(wo.Ref), content.WithDescriptor(wo.Desc))
if err != nil {
lock.Close()
return nil, err
}
return &localCacheWriter{Writer: w, lock: lock}, nil
}
func (s *localCacheStore) Abort(ctx context.Context, ref string) error {
lock, err := s.lock(ref)
if err != nil {
return err
}
defer lock.Close()
return s.Store.Abort(ctx, ref)
}
type localCacheWriter struct {
content.Writer
lock *flock.Flock
}
func (w *localCacheWriter) Commit(ctx context.Context, size int64, expected digest.Digest, opts ...content.Opt) error {
err := w.Writer.Commit(ctx, size, expected, opts...)
closeErr := w.Close()
if err != nil {
return err
}
return closeErr
}
func (w *localCacheWriter) Close() error {
err := w.Writer.Close()
lockErr := w.lock.Close()
if err != nil {
return err
}
return lockErr
}
+75 -61
View File
@@ -5,15 +5,11 @@ import (
controlapi "github.com/moby/buildkit/api/services/control"
digest "github.com/opencontainers/go-digest"
"google.golang.org/protobuf/encoding/protowire"
"google.golang.org/protobuf/types/known/timestamppb"
)
var emptyLogVertexSize int
func init() {
emptyLogVertex := controlapi.VertexLog{}
emptyLogVertexSize = emptyLogVertex.SizeVT()
}
const statusResponseSize = 1024 * 1024
func NewSolveStatus(resp *controlapi.StatusResponse) *SolveStatus {
s := &SolveStatus{}
@@ -63,68 +59,86 @@ func NewSolveStatus(resp *controlapi.StatusResponse) *SolveStatus {
return s
}
// Marshal batches status updates by encoded size without modifying SolveStatus.
// Logs may be split across responses. Individual metadata entries larger than
// the batch target are emitted intact in their own response.
func (ss *SolveStatus) Marshal() (out []*controlapi.StatusResponse) {
logSize := 0
for {
retry := false
sr := controlapi.StatusResponse{}
for _, v := range ss.Vertexes {
sr.Vertexes = append(sr.Vertexes, &controlapi.Vertex{
Digest: string(v.Digest),
Inputs: digestSliceToPB(v.Inputs),
Name: v.Name,
Started: timestampToPB(v.Started),
Completed: timestampToPB(v.Completed),
Error: v.Error,
Cached: v.Cached,
ProgressGroup: v.ProgressGroup,
})
sr := &controlapi.StatusResponse{}
size := 0
// Each repeated field has a one-byte tag and a length-prefixed message.
// Keep indivisible metadata entries intact even if they exceed the target.
reserve := func(n int) {
n = 1 + protowire.SizeBytes(n)
if size > 0 && size+n > statusResponseSize {
out = append(out, sr)
sr = &controlapi.StatusResponse{}
size = 0
}
for _, v := range ss.Statuses {
sr.Statuses = append(sr.Statuses, &controlapi.VertexStatus{
ID: v.ID,
Vertex: string(v.Vertex),
Name: v.Name,
Current: v.Current,
Total: v.Total,
Timestamp: timestamppb.New(v.Timestamp),
Started: timestampToPB(v.Started),
Completed: timestampToPB(v.Completed),
})
size += n
}
for _, v := range ss.Vertexes {
p := &controlapi.Vertex{
Digest: string(v.Digest),
Inputs: digestSliceToPB(v.Inputs),
Name: v.Name,
Started: timestampToPB(v.Started),
Completed: timestampToPB(v.Completed),
Error: v.Error,
Cached: v.Cached,
ProgressGroup: v.ProgressGroup,
}
for i, v := range ss.Logs {
sr.Logs = append(sr.Logs, &controlapi.VertexLog{
Vertex: string(v.Vertex),
Stream: int64(v.Stream),
Msg: v.Data,
Timestamp: timestamppb.New(v.Timestamp),
})
logSize += len(v.Data) + emptyLogVertexSize
// avoid logs growing big and split apart if they do
if logSize > 1024*1024 {
ss.Vertexes = nil
ss.Statuses = nil
ss.Logs = ss.Logs[i+1:]
retry = true
reserve(p.SizeVT())
sr.Vertexes = append(sr.Vertexes, p)
}
for _, v := range ss.Statuses {
p := &controlapi.VertexStatus{
ID: v.ID,
Vertex: string(v.Vertex),
Name: v.Name,
Current: v.Current,
Total: v.Total,
Timestamp: timestamppb.New(v.Timestamp),
Started: timestampToPB(v.Started),
Completed: timestampToPB(v.Completed),
}
reserve(p.SizeVT())
sr.Statuses = append(sr.Statuses, p)
}
for _, v := range ss.Logs {
p := &controlapi.VertexLog{
Vertex: string(v.Vertex),
Stream: int64(v.Stream),
Timestamp: timestamppb.New(v.Timestamp),
}
const maxData = statusResponseSize / 2
data := v.Data
for {
p.Msg = data[:min(len(data), maxData)]
reserve(p.SizeVT())
sr.Logs = append(sr.Logs, p)
data = data[len(p.Msg):]
if len(data) == 0 {
break
}
}
for _, v := range ss.Warnings {
sr.Warnings = append(sr.Warnings, &controlapi.VertexWarning{
Vertex: string(v.Vertex),
Level: int64(v.Level),
Short: v.Short,
Detail: v.Detail,
Info: v.SourceInfo,
Ranges: v.Range,
Url: v.URL,
})
}
out = append(out, &sr)
if !retry {
break
p = &controlapi.VertexLog{Vertex: p.Vertex, Stream: p.Stream, Timestamp: p.Timestamp}
}
}
for _, v := range ss.Warnings {
p := &controlapi.VertexWarning{
Vertex: string(v.Vertex),
Level: int64(v.Level),
Short: v.Short,
Detail: v.Detail,
Info: v.SourceInfo,
Ranges: v.Range,
Url: v.URL,
}
reserve(p.SizeVT())
sr.Warnings = append(sr.Warnings, p)
}
out = append(out, sr)
return
}
+2 -2
View File
@@ -1,7 +1,7 @@
package config
import (
"github.com/moby/buildkit/cache/remotecache/gha/ghatypes"
"github.com/moby/buildkit/cache/remotecache/signing"
"github.com/moby/buildkit/util/db/compaction"
resolverconfig "github.com/moby/buildkit/util/resolver/config"
)
@@ -105,7 +105,7 @@ func (c CompactionConfig) Policy() (compaction.Config, error) {
}
type CacheConfig struct {
GHA *ghatypes.CacheConfig `toml:"gha"`
GHA *signing.Config `toml:"gha"`
}
type SystemConfig struct {
+3 -2
View File
@@ -245,7 +245,8 @@ func (c *Controller) Prune(req *controlapi.PruneRequest, stream controlapi.Contr
ch := make(chan client.UsageInfo, 32)
eg, ctx := errgroup.WithContext(stream.Context())
ctx := stream.Context()
eg, ectx := errgroup.WithContext(ctx)
workers, err := c.opt.WorkerController.List()
if err != nil {
return errors.Wrap(err, "failed to list workers for prune")
@@ -267,7 +268,7 @@ func (c *Controller) Prune(req *controlapi.PruneRequest, stream controlapi.Contr
for _, w := range workers {
func(w worker.Worker) {
eg.Go(func() error {
return w.Prune(ctx, ch, client.PruneInfo{
return w.Prune(ectx, ch, client.PruneInfo{
Filter: req.Filter,
All: req.All,
KeepDuration: time.Duration(req.KeepDuration),
+2 -4
View File
@@ -426,8 +426,7 @@ func (w *containerdExecutor) runProcess(ctx context.Context, p ctd.Process, resi
if !ok {
return // chan closed
}
err = p.Resize(eventCtx, size.Cols, size.Rows)
if err != nil {
if err := p.Resize(eventCtx, size.Cols, size.Rows); err != nil {
bklog.G(eventCtx).Warnf("Failed to resize %s: %s", p.ID(), err)
}
}
@@ -442,8 +441,7 @@ func (w *containerdExecutor) runProcess(ctx context.Context, p ctd.Process, resi
if !ok {
return // chan closed
}
err = p.Kill(eventCtx, sig)
if err != nil {
if err := p.Kill(eventCtx, sig); err != nil {
bklog.G(eventCtx).Warnf("Failed to signal %s: %s", p.ID(), err)
}
}
+7 -7
View File
@@ -4,6 +4,7 @@ import (
"context"
"net/netip"
"os"
"sync/atomic"
"github.com/moby/buildkit/solver/pb"
"github.com/moby/buildkit/util/bklog"
@@ -23,9 +24,10 @@ const (
)
var (
g flightcontrol.Group[struct{}]
notFirstRun bool
lastNotEmpty bool
g flightcontrol.Group[struct{}]
// accessed concurrently by calls with different keys
notFirstRun atomic.Bool
lastNotEmpty atomic.Bool
)
// overridden by tests
@@ -67,8 +69,7 @@ func GetResolvConf(ctx context.Context, root *os.Root, idmap *user.IdentityMappi
}
_, err := g.Do(ctx, root.Name()+"/"+name, func(ctx context.Context) (struct{}, error) {
generate := !notFirstRun
notFirstRun = true
generate := !notFirstRun.Swap(true)
if !generate {
fi, err := root.Stat(name)
@@ -84,9 +85,8 @@ func GetResolvConf(ctx context.Context, root *os.Root, idmap *user.IdentityMappi
if !errors.Is(err, os.ErrNotExist) {
return struct{}{}, err
}
if lastNotEmpty {
if lastNotEmpty.CompareAndSwap(true, false) {
generate = true
lastNotEmpty = false
}
} else if fi.ModTime().Before(fiMain.ModTime()) {
generate = true
+3 -1
View File
@@ -40,6 +40,8 @@ import (
"github.com/pkg/errors"
)
const runcStartTimeout = 15 * time.Second
type Opt struct {
// root directory
Root string
@@ -748,7 +750,7 @@ func (p *procHandle) WaitForReady(ctx context.Context) error {
// callback is non-nil it will be called after receiving the pid.
func (p *procHandle) WaitForStart(ctx context.Context, startedCh <-chan int, started func()) error {
ctx, cancel := context.WithCancelCause(ctx)
ctx, _ = context.WithTimeoutCause(ctx, 10*time.Second, errors.WithStack(context.DeadlineExceeded)) //nolint:govet
ctx, _ = context.WithTimeoutCause(ctx, runcStartTimeout, errors.WithStack(context.DeadlineExceeded)) //nolint:govet
defer func() { cancel(errors.WithStack(context.Canceled)) }()
select {
case <-ctx.Done():
+17 -4
View File
@@ -13,6 +13,7 @@ import (
"github.com/containerd/console"
runc "github.com/containerd/go-runc"
"github.com/moby/buildkit/executor"
"github.com/moby/buildkit/executor/oci"
gatewayapi "github.com/moby/buildkit/frontend/gateway/pb"
"github.com/moby/buildkit/util/bklog"
"github.com/moby/sys/signal"
@@ -34,10 +35,14 @@ func (w *runcExecutor) run(ctx context.Context, id, bundle string, process execu
extraArgs = append(extraArgs, "--keep")
}
_, err := w.runc.Run(ctx, id, bundle, &runc.CreateOpts{
NoPivot: w.noPivot,
Started: started,
IO: io,
ExtraArgs: extraArgs,
NoPivot: w.noPivot,
// Without a PID namespace, processes left behind by a step outlive the
// container and keep its session keyring, until the per-user key quota
// is exhausted and every new container fails to start.
NoNewKeyring: w.processMode == oci.NoProcessSandbox,
Started: started,
IO: io,
ExtraArgs: extraArgs,
})
return err
})
@@ -164,7 +169,9 @@ func (w *runcExecutor) callWithIO(ctx context.Context, process executor.ProcessI
})
}
resizeDone := make(chan struct{})
eg.Go(func() error {
defer close(resizeDone)
err := runcProcess.WaitForReady(ctx)
if err != nil {
return err
@@ -190,6 +197,12 @@ func (w *runcExecutor) callWithIO(ctx context.Context, process executor.ProcessI
}
}
})
// Runs before the cleanup defer above: the resize loop uses ptm's fd, so
// stop it before ptm is closed.
defer func() {
runcProcess.Shutdown()
<-resizeDone
}()
runcIO := &forwardIO{}
if process.Stdin != nil {
+1 -1
View File
@@ -35,7 +35,7 @@ const (
)
func Build(ctx context.Context, c client.Client) (_ *client.Result, err error) {
c = &withResolveCache{Client: c}
c = newWithResolveCache(c)
bc, err := dockerui.NewClient(c)
if err != nil {
return nil, err
@@ -26,8 +26,14 @@ type resolveResult struct {
var _ client.Client = &withResolveCache{}
func newWithResolveCache(c client.Client) *withResolveCache {
return &withResolveCache{
Client: c,
g: flightcontrol.CachedGroup[*resolveResult]{CacheError: true},
}
}
func (c *withResolveCache) ResolveImageConfig(ctx context.Context, ref string, opt sourceresolver.Opt) (string, digest.Digest, []byte, error) {
c.g.CacheError = true
optHash, err := hashstructure.Hash(opt, nil)
if err != nil {
return "", "", nil, err
@@ -20,6 +20,7 @@ import (
"github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/client/llb/imagemetaresolver"
"github.com/moby/buildkit/client/llb/sourceresolver"
dfcommand "github.com/moby/buildkit/frontend/dockerfile/command"
"github.com/moby/buildkit/frontend/dockerfile/instructions"
"github.com/moby/buildkit/frontend/dockerfile/linter"
"github.com/moby/buildkit/frontend/dockerfile/parser"
@@ -47,6 +48,7 @@ const (
sbomScanContext = "BUILDKIT_SBOM_SCAN_CONTEXT"
sbomScanStage = "BUILDKIT_SBOM_SCAN_STAGE"
disableOnbuild = "BUILDKIT_DISABLE_ONBUILD"
)
var nonEnvArgs = map[string]struct{}{
@@ -216,6 +218,7 @@ type dispatchContext struct {
allDispatchStates *dispatchStates
proxyEnv *llb.ProxyEnv
namedContext func(string, dockerui.ContextOpt) (*dockerui.NamedContext, error)
disableOnbuild bool
}
func namedContextFunc(opt ConvertOpt) func(string, dockerui.ContextOpt) (*dockerui.NamedContext, error) {
@@ -310,6 +313,14 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS
globalArgs = setBuildArgValue(opt.BuildArgs, globalArgs, "SOURCE_DATE_EPOCH", formatSourceDateEpochValue(resolvedEpoch))
}
var skipOnbuild bool
if v, ok := getBuildArgValue(opt.BuildArgs, globalArgs, disableOnbuild); ok && v != "" {
skipOnbuild, err = strconv.ParseBool(v)
if err != nil {
return nil, errors.Wrapf(err, "invalid %s value %q", disableOnbuild, v)
}
}
metaResolver := opt.MetaResolver
if metaResolver == nil {
metaResolver = imagemetaresolver.Default()
@@ -327,6 +338,7 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS
allDispatchStates: newDispatchStates(),
proxyEnv: proxyEnvFromBuildArgs(opt.BuildArgs),
namedContext: namedContextFunc(opt),
disableOnbuild: skipOnbuild,
}
if err := dctx.buildDispatchStates(stages); err != nil {
@@ -568,6 +580,10 @@ func (dctx *dispatchContext) resolveStages(ctx context.Context, target *dispatch
}
d.onBuildInit = true
}
if dctx.disableOnbuild || slices.Contains(d.stage.Reset, dfcommand.Onbuild) {
onbuilds = nil
d.image.Config.OnBuild = nil
}
if len(onbuilds) > 0 {
if b, err := initOnBuildTriggers(d, onbuilds, dctx.allDispatchStates, dctx.shlex); err != nil {
@@ -789,6 +805,8 @@ func (dctx *dispatchContext) dispatchStages(ctx context.Context, allReachable ma
d.platform = &d.opt.targetPlatform
}
resetBaseConfig(d)
// make sure that PATH is always set
if _, ok := shell.EnvsFromSlice(d.image.Config.Env).Get("PATH"); !ok {
var osName string
@@ -834,6 +852,7 @@ func (dctx *dispatchContext) dispatchStages(ctx context.Context, allReachable ma
targetPlatform: dctx.platformOpt.targetPlatform,
extraHosts: dctx.opt.ExtraHosts,
gitAdvice: dctx.opt.GitAdvice,
gitDebugCommands: dctx.opt.GitDebugCommands,
shmSize: dctx.opt.ShmSize,
ulimit: dctx.opt.Ulimits,
devices: dctx.opt.Devices,
@@ -991,6 +1010,7 @@ type dispatchOpt struct {
buildPlatforms []ocispecs.Platform
extraHosts []llb.HostIP
gitAdvice bool
gitDebugCommands bool
shmSize int64
ulimit []*pb.Ulimit
devices []*pb.CDIDevice
@@ -1329,6 +1349,17 @@ func initOnBuildTriggers(d *dispatchState, triggers []string, allDispatchStates
}
func dispatchEnv(d *dispatchState, c *instructions.EnvCommand, lint *linter.Linter) error {
if c.Unset {
keys := make([]string, 0, len(c.Env))
for _, e := range c.Env {
if e.Key == "" {
return errors.New("ENV --unset names can not be blank")
}
unsetEnv(d, e.Key)
keys = append(keys, e.Key)
}
return commitToHistory(&d.image, "ENV --unset "+strings.Join(keys, " "), false, nil, d.epoch)
}
commitMessage := bytes.NewBufferString("ENV")
for _, e := range c.Env {
if e.NoDelim {
@@ -1343,6 +1374,61 @@ func dispatchEnv(d *dispatchState, c *instructions.EnvCommand, lint *linter.Lint
return commitToHistory(&d.image, commitMessage.String(), false, nil, d.epoch)
}
// unsetEnv removes an environment variable and a build arg with the same name.
func unsetEnv(d *dispatchState, key string) {
for _, k := range getEnv(d.state).Keys() {
if shell.EqualEnvKeys(k, key) {
d.state = d.state.DelEnv(k)
}
}
d.image.Config.Env = delEnv(d.image.Config.Env, key)
d.buildArgs = slices.DeleteFunc(d.buildArgs, func(arg instructions.KeyValuePairOptional) bool {
return shell.EqualEnvKeys(arg.Key, key)
})
}
// resetBaseConfig clears the base image config fields set with FROM --reset.
// ONBUILD triggers are skipped when the stages are resolved.
func resetBaseConfig(d *dispatchState) {
for _, k := range d.stage.Reset {
switch k {
case dfcommand.Cmd:
d.image.Config.Cmd = nil
case dfcommand.Entrypoint:
d.image.Config.Entrypoint = nil
case dfcommand.Env:
for _, env := range slices.Clone(d.image.Config.Env) {
key, _ := parseKeyValue(env)
unsetEnv(d, key)
}
case dfcommand.Expose:
d.image.Config.ExposedPorts = nil
case dfcommand.Healthcheck:
d.image.Config.Healthcheck = nil
case dfcommand.Label:
d.image.Config.Labels = nil
case dfcommand.Shell:
d.image.Config.Shell = nil
case dfcommand.StopSignal:
d.image.Config.StopSignal = ""
case dfcommand.User:
d.image.Config.User = ""
d.state = d.state.User("")
case dfcommand.Volume:
d.image.Config.Volumes = nil
case dfcommand.Workdir:
d.image.Config.WorkingDir = ""
d.state = d.state.Dir("/")
// workdir no longer depends on the base image
d.workdirSet = true
}
}
// ArgsEscaped applies to both cmd and entrypoint
if len(d.stage.Reset) > 0 && len(d.image.Config.Cmd) == 0 && len(d.image.Config.Entrypoint) == 0 {
d.image.Config.ArgsEscaped = false //nolint:staticcheck // ignore SA1019: field is deprecated in OCI Image spec, but used for backward-compatibility with Docker image spec.
}
}
func dispatchRun(d *dispatchState, c *instructions.RunCommand, proxy *llb.ProxyEnv, sources []*dispatchState, dopt dispatchOpt) error {
var opt []llb.RunOption
@@ -1573,6 +1659,17 @@ func dispatchMaintainer(d *dispatchState, c *instructions.MaintainerCommand) err
}
func dispatchLabel(d *dispatchState, c *instructions.LabelCommand, lint *linter.Linter) error {
if c.Unset {
keys := make([]string, 0, len(c.Labels))
for _, v := range c.Labels {
if v.Key == "" {
return errors.New("LABEL --unset names can not be blank")
}
delete(d.image.Config.Labels, v.Key)
keys = append(keys, v.Key)
}
return commitToHistory(&d.image, "LABEL --unset "+strings.Join(keys, " "), false, nil, d.epoch)
}
commitMessage := bytes.NewBufferString("LABEL")
if d.image.Config.Labels == nil {
d.image.Config.Labels = make(map[string]string, len(c.Labels))
@@ -1650,13 +1747,24 @@ func dispatchUser(d *dispatchState, c *instructions.UserCommand, commit bool) er
}
func dispatchVolume(d *dispatchState, c *instructions.VolumeCommand) error {
if slices.Contains(c.Volumes, "") {
return errors.New("VOLUME specified can not be an empty string")
}
if c.Unset {
for _, v := range c.Volumes {
v = path.Clean(v)
for k := range d.image.Config.Volumes {
if path.Clean(k) == v {
delete(d.image.Config.Volumes, k)
}
}
}
return commitToHistory(&d.image, fmt.Sprintf("VOLUME --unset %v", c.Volumes), false, nil, d.epoch)
}
if d.image.Config.Volumes == nil {
d.image.Config.Volumes = map[string]struct{}{}
}
for _, v := range c.Volumes {
if v == "" {
return errors.New("VOLUME specified can not be an empty string")
}
d.image.Config.Volumes[v] = struct{}{}
}
return commitToHistory(&d.image, fmt.Sprintf("VOLUME %v", c.Volumes), false, nil, d.epoch)
@@ -1768,6 +1876,13 @@ func addEnv(env []string, k, v string) []string {
return env
}
func delEnv(env []string, k string) []string {
return slices.DeleteFunc(env, func(envVar string) bool {
key, _ := parseKeyValue(envVar)
return shell.EqualEnvKeys(key, k)
})
}
func parseKeyValue(env string) (string, string) {
parts := strings.SplitN(env, "=", 2)
v := ""
@@ -145,6 +145,9 @@ func dispatchCopy(d *dispatchState, cfg copyConfig) error {
if cfg.opt.gitAdvice {
gitOptions = append(gitOptions, llb.GitAdvice(true))
}
if cfg.opt.gitDebugCommands {
gitOptions = append(gitOptions, llb.GitDebugCommands(true))
}
if cfg.keepGitDir != nil && gitRef.KeepGitDir != nil {
if *cfg.keepGitDir != *gitRef.KeepGitDir {
return errors.New("inconsistent keep-git-dir configuration")
@@ -41,7 +41,7 @@ func dispatchExpose(d *dispatchState, c *instructions.ExposeCommand, opt *dispat
d.image.Config.ExposedPorts[p] = struct{}{}
}
return commitToHistory(&d.image, fmt.Sprintf("EXPOSE %v", psp), false, nil, d.epoch)
return commitToHistory(&d.image, fmt.Sprintf("EXPOSE %s", strings.Join(psp, " ")), false, nil, d.epoch)
}
type portSpecs struct {
@@ -98,7 +98,7 @@ func resolveSourceDateEpochState(ctx context.Context, value string, opt ConvertO
args = &updated
}
sourceState, err := sourceDateEpochStageSource(stages[i], opt.BuildArgs, args, shlex, opt.GitAdvice)
sourceState, err := sourceDateEpochStageSource(stages[i], opt.BuildArgs, args, shlex, opt.GitAdvice, opt.GitDebugCommands)
if err != nil {
return nil, sourceDateEpochStateOpt{}, parser.WithLocation(err, stages[i].Location)
}
@@ -110,7 +110,7 @@ func resolveSourceDateEpochState(ctx context.Context, value string, opt ConvertO
return nil, sourceDateEpochStateOpt{}, errors.Errorf("invalid SOURCE_DATE_EPOCH: %s", value)
}
func sourceDateEpochStageSource(stage instructions.Stage, buildArgs map[string]string, globalArgs *llb.EnvList, shlex *shell.Lex, gitAdvice bool) (*llb.State, error) {
func sourceDateEpochStageSource(stage instructions.Stage, buildArgs map[string]string, globalArgs *llb.EnvList, shlex *shell.Lex, gitAdvice, gitDebugCommands bool) (*llb.State, error) {
stageBaseName, _, err := shlex.ProcessWord(stage.BaseName, globalArgs)
if err != nil {
return nil, errors.Wrapf(err, "failed to process source stage base name %q", stage.BaseName)
@@ -133,7 +133,7 @@ func sourceDateEpochStageSource(stage instructions.Stage, buildArgs map[string]s
if sourceState != nil {
return nil, errors.New("SOURCE_DATE_EPOCH stage must contain exactly one remote ADD")
}
sourceState, err = sourceDateEpochAddSource(c, env, shlex, gitAdvice)
sourceState, err = sourceDateEpochAddSource(c, env, shlex, gitAdvice, gitDebugCommands)
if err != nil {
return nil, err
}
@@ -167,7 +167,7 @@ func applySourceDateEpochStageArgs(args []instructions.KeyValuePairOptional, env
return env, nil
}
func sourceDateEpochAddSource(cmd *instructions.AddCommand, env *llb.EnvList, shlex *shell.Lex, gitAdvice bool) (*llb.State, error) {
func sourceDateEpochAddSource(cmd *instructions.AddCommand, env *llb.EnvList, shlex *shell.Lex, gitAdvice, gitDebugCommands bool) (*llb.State, error) {
if len(cmd.SourceContents) != 0 || len(cmd.SourcePaths) != 1 {
return nil, errors.New("SOURCE_DATE_EPOCH stage must contain exactly one remote ADD source")
}
@@ -204,6 +204,9 @@ func sourceDateEpochAddSource(cmd *instructions.AddCommand, env *llb.EnvList, sh
if gitAdvice {
gitOptions = append(gitOptions, llb.GitAdvice(true))
}
if gitDebugCommands {
gitOptions = append(gitOptions, llb.GitDebugCommands(true))
}
if cmd.KeepGitDir != nil && *cmd.KeepGitDir {
gitOptions = append(gitOptions, llb.KeepGitDir())
}
@@ -311,6 +311,7 @@ func getSecretsRegex() (*regexp.Regexp, *regexp.Regexp) {
"password",
"pword",
"passwd",
"psw",
"secret",
"token",
}
@@ -148,9 +148,11 @@ func expandSliceInPlace(values []string, expander SingleWordExpander) error {
// EnvCommand allows setting an variable in the container's environment.
//
// ENV key1 value1 [keyN valueN...]
// ENV --unset key1 [keyN...]
type EnvCommand struct {
withNameAndCode
Env KeyValuePairs
Env KeyValuePairs
Unset bool // Env keys are removed, values are empty
}
func (c *EnvCommand) Expand(expander SingleWordExpander) error {
@@ -185,9 +187,11 @@ func NewLabelCommand(k string, v string, noExp bool) *LabelCommand {
// LabelCommand sets an image label in the output
//
// LABEL some json data describing the image
// LABEL --unset key1 [keyN...]
type LabelCommand struct {
withNameAndCode
Labels KeyValuePairs
Unset bool // Labels keys are removed, values are empty
noExpand bool
}
@@ -436,9 +440,11 @@ func (c *UserCommand) Expand(expander SingleWordExpander) error {
// VolumeCommand exposes the specified volume for use in the build environment.
//
// VOLUME /foo
// VOLUME --unset /foo
type VolumeCommand struct {
withNameAndCode
Volumes []string
Unset bool // Volumes are removed
}
func (c *VolumeCommand) Expand(expander SingleWordExpander) error {
@@ -521,6 +527,7 @@ type Stage struct {
OrigCmd string // original FROM command, used for rule checks
BaseName string // name of the base stage or source
Platform string // platform of base source to use
Reset []string // base image config fields to reset, see ResetKeys
DocComment string // doc-comment directly above the stage
@@ -36,6 +36,31 @@ var (
parseRunPostHooks []func(*RunCommand, parseRequest) error
)
var (
unsetEnabled = false
resetEnabled = false
)
// ResetAll is the FROM --reset value that resets all ResetKeys.
const ResetAll = "all"
// ResetKeys are the base image config fields that can be reset with
// FROM --reset. Each field is named after the command that sets it.
var ResetKeys = []string{
command.Cmd,
command.Entrypoint,
command.Env,
command.Expose,
command.Healthcheck,
command.Label,
command.Onbuild,
command.Shell,
command.StopSignal,
command.User,
command.Volume,
command.Workdir,
}
func nodeArgs(node *parser.Node) []string {
result := []string{}
for ; node.Next != nil; node = node.Next {
@@ -230,10 +255,48 @@ func parseKvps(args []string, cmdName string) (KeyValuePairs, error) {
return res, nil
}
// addUnsetFlag adds the --unset flag if it is enabled in this build.
func addUnsetFlag(req parseRequest) *Flag {
if !unsetEnabled {
return nil
}
return req.flags.AddBool("unset", false)
}
// parseUnsetKeys returns the keys for the --unset form of a command.
func parseUnsetKeys(args []string, cmdName string) (KeyValuePairs, error) {
if len(args) == 0 {
return nil, errAtLeastOneArgument(cmdName + " --unset")
}
var res KeyValuePairs
for _, name := range args {
if name == "" {
return nil, errBlankCommandNames(cmdName)
}
if strings.Contains(name, "=") {
return nil, errors.Errorf("%s --unset does not accept values: %s", cmdName, name)
}
res = append(res, KeyValuePair{Key: name})
}
return res, nil
}
func parseEnv(req parseRequest) (*EnvCommand, error) {
flUnset := addUnsetFlag(req)
if err := req.flags.Parse(); err != nil {
return nil, err
}
if flUnset != nil && flUnset.Value == "true" {
envs, err := parseUnsetKeys(req.args, "ENV")
if err != nil {
return nil, err
}
return &EnvCommand{
Env: envs,
Unset: true,
withNameAndCode: newWithNameAndCode(req),
}, nil
}
envs, err := parseKvps(req.args, "ENV")
if err != nil {
return nil, err
@@ -259,9 +322,21 @@ func parseMaintainer(req parseRequest) (*MaintainerCommand, error) {
}
func parseLabel(req parseRequest) (*LabelCommand, error) {
flUnset := addUnsetFlag(req)
if err := req.flags.Parse(); err != nil {
return nil, err
}
if flUnset != nil && flUnset.Value == "true" {
labels, err := parseUnsetKeys(req.args, "LABEL")
if err != nil {
return nil, err
}
return &LabelCommand{
Labels: labels,
Unset: true,
withNameAndCode: newWithNameAndCode(req),
}, nil
}
labels, err := parseKvps(req.args, "LABEL")
if err != nil {
@@ -399,10 +474,22 @@ func parseFrom(req parseRequest) (*Stage, error) {
}
flPlatform := req.flags.AddString("platform", "")
var flReset *Flag
if resetEnabled {
flReset = req.flags.AddStrings("reset")
}
if err := req.flags.Parse(); err != nil {
return nil, err
}
var reset []string
if flReset != nil {
reset, err = parseResetKeys(flReset.StringValues)
if err != nil {
return nil, err
}
}
code := strings.TrimSpace(req.original)
return &Stage{
BaseName: req.args[0],
@@ -411,12 +498,35 @@ func parseFrom(req parseRequest) (*Stage, error) {
SourceCode: code,
Commands: []Command{},
Platform: flPlatform.Value,
Reset: reset,
Location: req.location,
Comments: req.comments,
DocComment: getDocComment(req.comments, stageName),
}, nil
}
// parseResetKeys returns the sorted list of fields for FROM --reset values.
func parseResetKeys(values []string) ([]string, error) {
var keys []string
for _, v := range values {
for k := range strings.SplitSeq(v, ",") {
switch {
case k == ResetAll:
keys = append(keys, ResetKeys...)
case slices.Contains(ResetKeys, k):
keys = append(keys, k)
case strings.ToLower(k) == ResetAll || slices.Contains(ResetKeys, strings.ToLower(k)):
return nil, errors.Errorf("invalid FROM --reset value: %q (did you mean %s?)", k, strings.ToLower(k))
default:
err := errors.Errorf("invalid FROM --reset value: %q", k)
return nil, suggest.WrapError(err, k, append([]string{ResetAll}, ResetKeys...), true)
}
}
}
slices.Sort(keys)
return slices.Compact(keys), nil
}
var validStageName = regexp.MustCompile("^[a-z][a-z0-9-_.]*$")
func parseBuildStageName(args []string) (stageName string, err error) {
@@ -714,11 +824,13 @@ func parseVolume(req parseRequest) (*VolumeCommand, error) {
return nil, errAtLeastOneArgument("VOLUME")
}
flUnset := addUnsetFlag(req)
if err := req.flags.Parse(); err != nil {
return nil, err
}
cmd := &VolumeCommand{
Unset: flUnset != nil && flUnset.Value == "true",
withNameAndCode: newWithNameAndCode(req),
}
@@ -736,6 +848,9 @@ func parseStopSignal(req parseRequest) (*StopSignalCommand, error) {
if len(req.args) != 1 {
return nil, errExactlyOneArgument("STOPSIGNAL")
}
if err := req.flags.Parse(); err != nil {
return nil, err
}
sig := req.args[0]
cmd := &StopSignalCommand{
@@ -750,6 +865,10 @@ func parseArg(req parseRequest) (*ArgCommand, error) {
return nil, errAtLeastOneArgument("ARG")
}
if err := req.flags.Parse(); err != nil {
return nil, err
}
pairs := make([]KeyValuePairOptional, len(req.args))
for i, arg := range req.args {
@@ -0,0 +1,7 @@
//go:build dfreset
package instructions
func init() {
resetEnabled = true
}
@@ -0,0 +1,7 @@
//go:build dfunset
package instructions
func init() {
unsetEnabled = true
}
+22
View File
@@ -137,6 +137,13 @@ var (
heredocCompoundDirectives = map[string]bool{
command.Onbuild: true,
}
// Directives that parse key-value pairs but take a list of names
// when --unset flag is set
unsetCommands = map[string]bool{
command.Env: true,
command.Label: true,
}
)
// directives is the structure used during a build run to hold the state of
@@ -234,6 +241,10 @@ func newNodeFromLine(line string, d *directives, comments []string) (*Node, erro
if fn == nil {
fn = parseIgnore
}
// --unset form takes a list of names instead of key-value pairs
if unsetCommands[strings.ToLower(cmd)] && hasUnsetFlag(flags) {
fn = parseNameOrNameVal
}
next, attrs, err := fn(args, d)
if err != nil {
return nil, err
@@ -249,6 +260,17 @@ func newNodeFromLine(line string, d *directives, comments []string) (*Node, erro
}, nil
}
func hasUnsetFlag(flags []string) bool {
for _, f := range flags {
name, value, _ := strings.Cut(f, "=")
if name == "--unset" {
// invalid values are reported by the flag parser
return !strings.EqualFold(value, "false")
}
}
return false
}
// Result contains the bundled outputs from parsing a Dockerfile.
type Result struct {
AST *Node
+1 -1
View File
@@ -1,3 +1,3 @@
package version
const version = "1.28"
const version = "1.29"
+8
View File
@@ -58,6 +58,7 @@ const (
keyDockerfileLintArg = "build-arg:BUILDKIT_DOCKERFILE_CHECK"
keyContextKeepGitDirArg = "build-arg:BUILDKIT_CONTEXT_KEEP_GIT_DIR"
keyGitAdviceArg = "build-arg:BUILDKIT_GIT_ADVICE"
keyGitDebugCommandsArg = "build-arg:BUILDKIT_GIT_DEBUG_COMMANDS"
)
type Config struct {
@@ -70,6 +71,7 @@ type Config struct {
Labels map[string]string
NetworkMode pb.NetMode
GitAdvice bool
GitDebugCommands bool
ShmSize int64
Target string
Ulimits []*pb.Ulimit
@@ -319,6 +321,12 @@ func (bc *Client) init() error {
return errors.Wrapf(err, "failed to parse %s", keyGitAdviceArg)
}
}
if v := opts[keyGitDebugCommandsArg]; v != "" {
bc.GitDebugCommands, err = strconv.ParseBool(v)
if err != nil {
return errors.Wrapf(err, "failed to parse %s", keyGitDebugCommandsArg)
}
}
bc.localsSessionIDs = parseLocalSessionIDs(opts)
+9 -5
View File
@@ -91,7 +91,7 @@ func (bc *Client) initContext(ctx context.Context) (*buildContext, error) {
if opts[buildArgPrefix+"SOURCE_DATE_EPOCH"] != "" {
extraGitOpts = append(extraGitOpts, llb.GitMTimeCommit())
}
extraGitOpts = append(extraGitOpts, gitAdviceOpts(bc.GitAdvice)...)
extraGitOpts = append(extraGitOpts, gitSourceOpts(bc.GitAdvice, bc.GitDebugCommands)...)
if st, ok, err := DetectGitContext(opts[localNameContext], keepGit, extraGitOpts...); ok {
if err != nil {
return nil, err
@@ -325,11 +325,15 @@ func DetectGitContext(ref string, keepGit *bool, opts ...llb.GitOption) (*llb.St
return &st, true, nil
}
func gitAdviceOpts(enabled bool) []llb.GitOption {
if enabled {
return []llb.GitOption{llb.GitAdvice(true)}
func gitSourceOpts(advice, debugCommands bool) []llb.GitOption {
var opts []llb.GitOption
if advice {
opts = append(opts, llb.GitAdvice(true))
}
return nil
if debugCommands {
opts = append(opts, llb.GitDebugCommands(true))
}
return opts
}
func DetectHTTPContext(ref string) (*llb.State, string, bool) {
+2 -2
View File
@@ -141,7 +141,7 @@ func (nc *NamedContext) load(ctx context.Context, count int) (*llb.State, *docke
}
return &st, &img, nil
case "git":
st, ok, err := DetectGitContext(nc.input, nil, gitAdviceOpts(nc.bc.GitAdvice)...)
st, ok, err := DetectGitContext(nc.input, nil, gitSourceOpts(nc.bc.GitAdvice, nc.bc.GitDebugCommands)...)
if !ok {
return nil, nil, errors.Errorf("invalid git context %s", nc.input)
}
@@ -150,7 +150,7 @@ func (nc *NamedContext) load(ctx context.Context, count int) (*llb.State, *docke
}
return st, nil, nil
case "http", "https":
st, ok, err := DetectGitContext(nc.input, nil, gitAdviceOpts(nc.bc.GitAdvice)...)
st, ok, err := DetectGitContext(nc.input, nil, gitSourceOpts(nc.bc.GitAdvice, nc.bc.GitDebugCommands)...)
if ok {
if err != nil {
return nil, nil, err
+18 -29
View File
@@ -68,6 +68,7 @@ type BridgeClient struct {
workers worker.Infos
resultByID map[string]solver.Result
discarded bool
execErrors []*llberrdefs.ExecError
buildOpts client.BuildOpts
ctrs []client.Container
executor executor.Executor
@@ -159,6 +160,13 @@ func (c *BridgeClient) wrapSolveError(solveErr error) error {
subject errdefs.IsSolve_Subject
)
if errors.As(solveErr, &ee) {
c.mu.Lock()
if c.discarded {
ee.Release()
} else if !slices.Contains(c.execErrors, ee) {
c.execErrors = append(c.execErrors, ee)
}
c.mu.Unlock()
var err error
inputIDs, err = c.registerResultIDs(ee.Inputs...)
if err != nil {
@@ -174,7 +182,7 @@ func (c *BridgeClient) wrapSolveError(solveErr error) error {
}
if errors.As(solveErr, &sce) {
var err error
inputIDs, err = c.registerSlowCacheResult(sce.Result)
inputIDs, err = c.registerResultIDs(sce.Result)
if err != nil {
return err
}
@@ -183,32 +191,14 @@ func (c *BridgeClient) wrapSolveError(solveErr error) error {
return errdefs.WithSolveError(solveErr, subject, inputIDs, mountIDs)
}
// registerResultIDs retains gateway-owned clones of error refs until discard.
func (c *BridgeClient) registerResultIDs(results ...solver.Result) (ids []string, err error) {
c.mu.Lock()
defer c.mu.Unlock()
return c.registerResultIDsLocked(results...)
}
// registerSlowCacheResult registers a clone of a result borrowed from the
// solver job. The clone is only taken before the client is discarded, as the
// job may be released after that.
func (c *BridgeClient) registerSlowCacheResult(res solver.Result) ([]string, error) {
c.mu.Lock()
defer c.mu.Unlock()
if c.discarded {
return nil, nil
}
// Validate before cloning so a type error can't leak the clone. Once
// cloned, registerResultIDsLocked either keeps or releases it.
if _, ok := res.Sys().(*worker.WorkerRef); !ok {
return nil, errors.Errorf("unexpected type for result, got %T", res.Sys())
}
return c.registerResultIDsLocked(res.Clone())
}
func (c *BridgeClient) registerResultIDsLocked(results ...solver.Result) (ids []string, err error) {
ids = make([]string, len(results))
for i, res := range results {
if res == nil {
@@ -220,16 +210,11 @@ func (c *BridgeClient) registerResultIDsLocked(results ...solver.Result) (ids []
}
id := workerRef.ID()
ids[i] = id
if existing, ok := c.resultByID[id]; ok {
if existing != res {
if err := res.Release(context.TODO()); err != nil {
return ids, errors.WithStack(err)
}
}
continue
if _, ok := c.resultByID[id]; !ok {
// Cached sharedOp.Exec errors have snapshots pinned by the active
// solver state, even after the original error refs are released.
c.resultByID[id] = res.Clone()
}
// Keep the result wrapper so Release balances split-result ownership.
c.resultByID[id] = res
}
return ids, nil
}
@@ -281,6 +266,10 @@ func (c *BridgeClient) discard(err error) {
c.mu.Lock()
c.discarded = true
for _, ee := range c.execErrors {
ee.Release()
}
c.execErrors = nil
for id, res := range c.resultByID {
res.Release(context.TODO())
delete(c.resultByID, id)
+74 -45
View File
@@ -1,6 +1,7 @@
package gateway
import (
"bufio"
"context"
"encoding/json"
"fmt"
@@ -280,6 +281,10 @@ func (gf *gatewayFrontend) Solve(ctx context.Context, llbBridge frontend.Fronten
lbf, ctx := serveLLBBridgeForwarder(ctx, llbBridge, exec, gf.workers, inputs, sid, sm)
defer lbf.Discard()
defer lbf.conn.Close()
// The executor may pass these fds directly to the frontend process, so
// close them only after Run has returned and no fork can still use them.
defer lbf.Stdin.Close()
defer lbf.Stdout.Close()
mdmnt, release, err := metadataMount(frontendDef)
if err != nil {
@@ -327,12 +332,12 @@ func metadataMount(def *opspb.Definition) (*executor.Mount, func(), error) {
}
return &executor.Mount{
Src: &bind{dir},
Dest: "/run/config/buildkit/metadata",
Readonly: true,
}, func() {
os.RemoveAll(dir)
}, nil
Src: &bind{dir},
Dest: "/run/config/buildkit/metadata",
Readonly: true,
}, func() {
os.RemoveAll(dir)
}, nil
}
type bind struct {
@@ -394,6 +399,10 @@ func (lbf *llbBridgeForwarder) discard() {
lbf.mountsMu.Unlock()
lbf.discarded = true
for _, ee := range lbf.execErrors {
ee.Release()
}
lbf.execErrors = nil
for id, res := range lbf.resultByID {
res.Release(context.TODO())
delete(lbf.resultByID, id)
@@ -474,7 +483,6 @@ func newBridgeForwarder(ctx context.Context, llbBridge frontend.FrontendLLBBridg
refs: map[string]solver.ResultProxy{},
resultByID: map[string]solver.Result{},
doneCh: make(chan struct{}),
pipe: newPipe(),
workers: workers,
inputs: inputs,
sid: sid,
@@ -489,6 +497,7 @@ func newBridgeForwarder(ctx context.Context, llbBridge frontend.FrontendLLBBridg
func serveLLBBridgeForwarder(ctx context.Context, llbBridge frontend.FrontendLLBBridge, exec executor.Executor, workers worker.Infos, inputs map[string]*opspb.Definition, sid string, sm *session.Manager) (*llbBridgeForwarder, context.Context) {
ctx, cancel := context.WithCancelCause(ctx)
lbf := newBridgeForwarder(ctx, llbBridge, exec, workers, inputs, sid, sm)
lbf.pipe = newPipe()
serverOpt := []grpc.ServerOption{
grpc.UnaryInterceptor(grpcerrors.UnaryServerInterceptor),
grpc.StreamInterceptor(grpcerrors.StreamServerInterceptor),
@@ -525,9 +534,9 @@ func newPipe() *pipe {
Stdin: pr1,
Stdout: pw2,
conn: &conn{
Reader: pr2,
Writer: pw1,
Closer: pw2,
Reader: pr2,
Writer: pw1,
closers: []io.Closer{pr2, pw1},
},
}
}
@@ -535,7 +544,17 @@ func newPipe() *pipe {
type conn struct {
io.Reader
io.Writer
io.Closer
closers []io.Closer
}
func (s *conn) Close() error {
var err error
for _, closer := range s.closers {
if closeErr := closer.Close(); err == nil {
err = closeErr
}
}
return errors.WithStack(err)
}
func (s *conn) LocalAddr() net.Addr {
@@ -589,6 +608,7 @@ type llbBridgeForwarder struct {
refs map[string]solver.ResultProxy
resultByID map[string]solver.Result
discarded bool
execErrors []*llberrdefs.ExecError
// lastRef solver.CachedResult
// lastRefs map[string]solver.CachedResult
// err error
@@ -715,6 +735,13 @@ func (lbf *llbBridgeForwarder) wrapSolveError(solveErr error) error {
subject errdefs.IsSolve_Subject
)
if errors.As(solveErr, &ee) {
lbf.mu.Lock()
if lbf.discarded {
ee.Release()
} else if !slices.Contains(lbf.execErrors, ee) {
lbf.execErrors = append(lbf.execErrors, ee)
}
lbf.mu.Unlock()
var err error
inputIDs, err = lbf.registerResultIDs(ee.Inputs...)
if err != nil {
@@ -730,7 +757,7 @@ func (lbf *llbBridgeForwarder) wrapSolveError(solveErr error) error {
}
if errors.As(solveErr, &sce) {
var err error
inputIDs, err = lbf.registerSlowCacheResult(sce.Result)
inputIDs, err = lbf.registerResultIDs(sce.Result)
if err != nil {
return err
}
@@ -739,32 +766,14 @@ func (lbf *llbBridgeForwarder) wrapSolveError(solveErr error) error {
return errdefs.WithSolveError(solveErr, subject, inputIDs, mountIDs)
}
// registerResultIDs retains gateway-owned clones of error refs until discard.
func (lbf *llbBridgeForwarder) registerResultIDs(results ...solver.Result) (ids []string, err error) {
lbf.mu.Lock()
defer lbf.mu.Unlock()
return lbf.registerResultIDsLocked(results...)
}
// registerSlowCacheResult registers a clone of a result borrowed from the
// solver job. The clone is only taken before the forwarder is discarded, as the
// job may be released after that.
func (lbf *llbBridgeForwarder) registerSlowCacheResult(res solver.Result) ([]string, error) {
lbf.mu.Lock()
defer lbf.mu.Unlock()
if lbf.discarded {
return nil, nil
}
// Validate before cloning so a type error can't leak the clone. Once
// cloned, registerResultIDsLocked either keeps or releases it.
if _, ok := res.Sys().(*worker.WorkerRef); !ok {
return nil, errors.Errorf("unexpected type for result, got %T", res.Sys())
}
return lbf.registerResultIDsLocked(res.Clone())
}
func (lbf *llbBridgeForwarder) registerResultIDsLocked(results ...solver.Result) (ids []string, err error) {
ids = make([]string, len(results))
for i, res := range results {
if res == nil {
@@ -776,16 +785,11 @@ func (lbf *llbBridgeForwarder) registerResultIDsLocked(results ...solver.Result)
}
id := workerRef.ID()
ids[i] = id
if existing, ok := lbf.resultByID[id]; ok {
if existing != res {
if err := res.Release(context.TODO()); err != nil {
return ids, errors.WithStack(err)
}
}
continue
if _, ok := lbf.resultByID[id]; !ok {
// Cached sharedOp.Exec errors have snapshots pinned by the active
// solver state, even after the original error refs are released.
lbf.resultByID[id] = res.Clone()
}
// Keep the result wrapper so Release balances split-result ownership.
lbf.resultByID[id] = res
}
return ids, nil
}
@@ -834,6 +838,16 @@ func (lbf *llbBridgeForwarder) Solve(ctx context.Context, req *pb.SolveRequest)
lbf.mu.Lock()
if lbf.discarded {
// A solve that finished after Discard has no one to release its refs.
lbf.mu.Unlock()
res.EachRef(func(r solver.ResultProxy) error {
r.Release(context.TODO())
return nil
})
return nil, stack.Enable(status.Error(codes.Unavailable, "gateway forwarder is closing"))
}
if res.Refs != nil {
ids := make(map[string]string, len(res.Refs))
defs := make(map[string]*opspb.Definition, len(res.Refs))
@@ -1869,13 +1883,28 @@ func (lbf *llbBridgeForwarder) cloneRef(id string) (solver.ResultProxy, error) {
return s2, nil
}
type bufferedConn struct {
net.Conn
reader *bufio.Reader
}
func (c *bufferedConn) Read(p []byte) (int, error) {
return c.reader.Read(p)
}
func serve(ctx context.Context, grpcServer *grpc.Server, conn net.Conn) {
go func() {
<-ctx.Done()
conn.Close()
}()
stop := context.AfterFunc(ctx, func() { conn.Close() })
defer stop()
defer conn.Close()
// The frontend container may take a while to start. Start HTTP/2's
// client preface timeout only once the frontend begins communicating.
reader := bufio.NewReader(conn)
if _, err := reader.Peek(1); err != nil {
return
}
bklog.G(ctx).Debugf("serving grpc connection")
(&http2.Server{}).ServeConn(conn, &http2.ServeConnOpts{Handler: grpcServer}) //nolint:staticcheck // This existing HTTP/2 connection is already established.
(&http2.Server{}).ServeConn(&bufferedConn{Conn: conn, reader: reader}, &http2.ServeConnOpts{Handler: grpcServer}) //nolint:staticcheck // This existing HTTP/2 connection is already established.
}
type markTypeFrontend struct{}
+38 -26
View File
@@ -97,7 +97,8 @@ func convertRef(ref client.Reference) (*pb.Ref, error) {
if !ok {
return nil, errors.Errorf("invalid return reference type %T", ref)
}
return &pb.Ref{Id: r.id, Def: r.def}, nil
// Return only needs the ID of the reference held by the server.
return &pb.Ref{Id: r.id}, nil
}
func RunFromEnvironment(ctx context.Context, f client.BuildFunc) error {
@@ -821,12 +822,14 @@ func (c *grpcClient) Inputs(ctx context.Context) (map[string]llb.State, error) {
// communication channel between the process and the ExecProcess message
// stream.
type procMessageForwarder struct {
ctx context.Context
done chan struct{}
msgs chan *pb.ExecMessage
}
func newProcMessageForwarder() *procMessageForwarder {
func newProcMessageForwarder(ctx context.Context) *procMessageForwarder {
return &procMessageForwarder{
ctx: ctx,
done: make(chan struct{}),
msgs: make(chan *pb.ExecMessage),
}
@@ -840,14 +843,16 @@ func (b *procMessageForwarder) Send(ctx context.Context, m *pb.ExecMessage) {
}
}
func (b *procMessageForwarder) Recv(ctx context.Context) (m *pb.ExecMessage, ok bool) {
func (b *procMessageForwarder) Recv(ctx context.Context) (*pb.ExecMessage, error) {
select {
case <-ctx.Done():
return nil, true
return nil, context.Cause(ctx)
case <-b.ctx.Done():
return nil, context.Cause(b.ctx)
case <-b.done:
return nil, false
case m = <-b.msgs:
return m, true
return nil, nil
case m := <-b.msgs:
return m, nil
}
}
@@ -908,6 +913,11 @@ func (m *messageForwarder) Start() (err error) {
m.eg.Go(func() error {
for {
msg, err := m.stream.Recv()
if err != nil {
// EOF and cancellation also end delivery to every process,
// even though they are not errors for Release.
m.cancel(errors.Wrap(err, "remote exec stream closed"))
}
if errors.Is(err, io.EOF) || grpcerrors.Code(err) == codes.Canceled {
return nil
}
@@ -976,7 +986,7 @@ func (m *messageForwarder) Release() error {
func (m *messageForwarder) Register(pid string) *procMessageForwarder {
m.mu.Lock()
defer m.mu.Unlock()
sender := newProcMessageForwarder()
sender := newProcMessageForwarder(m.ctx)
m.pids[pid] = sender
return sender
}
@@ -1120,7 +1130,11 @@ func (ctr *container) Start(ctx context.Context, req client.StartRequest) (clien
return nil, err
}
msg, _ := msgs.Recv(ctx)
msg, err := msgs.Recv(ctx)
if err != nil {
ctr.execMsgs.Deregister(pid)
return nil, err
}
if msg == nil {
return nil, errors.New("failed to receive started message")
}
@@ -1165,13 +1179,11 @@ func (ctr *container) Start(ctx context.Context, req client.StartRequest) (clien
processID: pid,
fd: 0,
}
_, err := io.Copy(m, stdinReader)
// ignore ErrClosedPipe, it is EOF for our usage.
if err != nil && !errors.Is(err, io.ErrClosedPipe) {
return err
_, copyErr := io.Copy(m, stdinReader)
if copyErr != nil && !errors.Is(copyErr, io.ErrClosedPipe) {
return copyErr
}
// not an error so must be eof
return ctr.execMsgs.Send(&pb.ExecMessage{
err := ctr.execMsgs.Send(&pb.ExecMessage{
ProcessID: pid,
Input: &pb.ExecMessage_File{
File: &pb.FdMessage{
@@ -1180,6 +1192,12 @@ func (ctr *container) Start(ctx context.Context, req client.StartRequest) (clien
},
},
})
// Cleanup still sends EOF to unblock remote stdin, but a failed send
// must not replace the process result that triggered cleanup.
if errors.Is(copyErr, io.ErrClosedPipe) {
return nil
}
return err
})
}
@@ -1188,21 +1206,15 @@ func (ctr *container) Start(ctx context.Context, req client.StartRequest) (clien
defer closeDoneOnce.Do(func() { close(done) })
var exitError error
for {
msg, ok := msgs.Recv(ctx)
if !ok {
msg, err := msgs.Recv(ctx)
if err != nil {
return err
}
if msg == nil {
// no more messages, return
return exitError
}
if msg == nil {
// empty message from ctx cancel, so just start shutting down
// input
closeDoneOnce.Do(func() {
close(done)
})
return context.Cause(ctx)
}
if file := msg.GetFile(); file != nil {
var out io.WriteCloser
switch file.Fd {
+8
View File
@@ -3,6 +3,7 @@ package imagerefchecker
import (
"context"
"encoding/json"
"slices"
"strings"
"sync"
@@ -39,6 +40,13 @@ func (c *Checker) Exists(key string, blobs []digest.Digest) bool {
return false
}
// An empty digest means that a layer in the chain has no blob, so it was
// never exported and no image can reference it. Without this check, the
// key of such a chain would be the same as the key of its parent chain.
if slices.Contains(blobs, "") {
return false
}
c.once.Do(c.init)
if b, ok := c.cache[key]; ok {
+2
View File
@@ -37,6 +37,8 @@ func (lm *localMounter) Mount() (string, error) {
}
}
lm.mounts = mount.RemoveVolatileOption(lm.mounts)
var isFile bool
if len(lm.mounts) == 1 && (lm.mounts[0].Type == "bind" || lm.mounts[0].Type == "rbind") {
if !lm.forceRemount {
+28 -30
View File
@@ -409,42 +409,40 @@ func (c *cacheManager) ensurePersistentKey(k *CacheKey) error {
func (c *cacheManager) getIDFromDeps(k *CacheKey) string {
matches := map[string]struct{}{}
loop0:
for i, deps := range k.deps {
if i == 0 || len(matches) > 0 {
for _, ck := range deps {
m2 := make(map[string]struct{})
if err := c.backend.WalkLinks(c.getID(ck.CacheKey.CacheKey), CacheInfoLink{
Input: Index(i),
Output: k.Output(),
Digest: k.Digest(),
Selector: ck.Selector,
}, func(id string) error {
if i == 0 {
matches[id] = struct{}{}
} else {
m2[id] = struct{}{}
}
return nil
}); err != nil {
matches = map[string]struct{}{}
break
}
if i != 0 {
for id := range matches {
if _, ok := m2[id]; !ok {
delete(matches, id)
}
}
if i > 0 && len(matches) == 0 {
break
}
// Keys within one input are alternatives; different inputs must all match.
candidates := map[string]struct{}{}
for _, ck := range deps {
if err := c.backend.WalkLinks(c.getID(ck.CacheKey.CacheKey), CacheInfoLink{
Input: Index(i),
Output: k.Output(),
Digest: k.Digest(),
Selector: ck.Selector,
}, func(id string) error {
candidates[id] = struct{}{}
return nil
}); err != nil {
matches = nil
break loop0
}
}
if i == 0 {
matches = candidates
} else {
for id := range matches {
if _, ok := candidates[id]; !ok {
delete(matches, id)
}
}
}
}
for k := range matches {
return k
for id := range matches {
return id
}
return identity.NewID()
}
+1 -1
View File
@@ -36,7 +36,7 @@ func withAncestorCacheOpts(ctx context.Context, start *state) context.Context {
}
values := make(map[any]any)
walkAncestors(ctx, start, func(st *state) bool {
if st.clientVertex.Error != "" {
if st.vertexFailed() {
// don't use values from cancelled or otherwise error'd vertexes
return false
}
+18 -7
View File
@@ -69,7 +69,10 @@ type edge struct {
keysDidChange bool
index *edgeIndex
secondaryExporters []expDep
// secondaryExportersMu protects secondaryExporters, which the scheduler
// appends to on merge while cache exporters read it from other goroutines
secondaryExportersMu sync.Mutex
secondaryExporters []expDep
failedOnce sync.Once
debug bool
@@ -729,6 +732,14 @@ func (e *edge) processDepSlowCacheReq(index int, dep *dep) {
// respondToIncoming responds to all incoming requests. completing or
// updating them when possible
func (e *edge) respondToIncoming(incoming []pipeSender, allPipes []pipeReceiver) (edgeStatusType, bool) {
// Requests can be canceled concurrently. Decide on one snapshot so a late
// cancel cannot finish the last request while outgoing work is active; it
// is handled on the next dispatch.
canceled := make([]bool, len(incoming))
for i, req := range incoming {
canceled[i] = req.Request().Canceled
}
// detect the result state for the requests
allIncomingCanComplete := true
desiredState := e.state
@@ -737,8 +748,8 @@ func (e *edge) respondToIncoming(incoming []pipeSender, allPipes []pipeReceiver)
// check incoming requests
// check if all requests can be either answered or canceled
if !e.isComplete() {
for _, req := range incoming {
if !req.Request().Canceled {
for i, req := range incoming {
if !canceled[i] {
allCanceled = false
if r := req.Request().Payload; desiredState < r.desiredState {
desiredState = r.desiredState
@@ -765,8 +776,8 @@ func (e *edge) respondToIncoming(incoming []pipeSender, allPipes []pipeReceiver)
// can close all but one requests
var leaveOpen pipeSender
for _, req := range incoming {
if !req.Request().Canceled {
for i, req := range incoming {
if !canceled[i] {
leaveOpen = req
break
}
@@ -790,9 +801,9 @@ func (e *edge) respondToIncoming(incoming []pipeSender, allPipes []pipeReceiver)
}
// update incoming based on current state
for _, req := range incoming {
for i, req := range incoming {
r := req.Request().Payload
if req.Request().Canceled {
if canceled[i] {
e.finishIncoming(req)
} else if !e.hasActiveOutgoing && e.state >= r.desiredState {
e.finishIncoming(req)
+4 -1
View File
@@ -246,7 +246,10 @@ func (e *exporter) ExportTo(ctx context.Context, t CacheExporterTarget, opt Cach
}
if e.edge != nil {
for _, de := range e.edge.secondaryExporters {
e.edge.secondaryExportersMu.Lock()
secondaryExporters := slices.Clone(e.edge.secondaryExporters)
e.edge.secondaryExportersMu.Unlock()
for _, de := range secondaryExporters {
recs, err := de.cacheKey.CacheKey.Exporter.ExportTo(mainCtx, t, opt)
if err != nil {
continue
+3
View File
@@ -160,11 +160,14 @@ func (ei *edgeIndex) enforceIndexID(k *CacheKey) {
matches := ei.getAllMatches(k)
// TraceFields reads indexIDs under the cache-key lock.
k.mu.Lock()
if len(matches) > 0 {
k.indexIDs = matches
} else {
k.indexIDs = []string{identity.NewID()}
}
k.mu.Unlock()
for _, id := range k.indexIDs {
ei.enforceLinked(id, k)
+1 -1
View File
@@ -153,7 +153,7 @@ func (pw *sender[Payload, Value]) Finalize(v Value, err error) {
pw.status.Value = v
pw.status.Err = err
pw.status.Completed = true
if errors.Is(err, context.Canceled) && pw.req.Canceled {
if errors.Is(err, context.Canceled) && pw.Request().Canceled {
pw.status.Canceled = true
}
pw.sendChannel.Send(pw.status)
+91 -34
View File
@@ -61,9 +61,10 @@ type state struct {
mspan *tracing.MultiSpan
execSpan trace.Span
vtx Vertex
clientVertex client.Vertex
origDigest digest.Digest // original LLB digest. TODO: probably better to use string ID so this isn't needed
vtx Vertex
clientVertex client.Vertex
clientVertexMu sync.Mutex // protects mutable clientVertex fields
origDigest digest.Digest // original LLB digest. TODO: probably better to use string ID so this isn't needed
mu sync.RWMutex
op *sharedOp
@@ -329,6 +330,51 @@ func (s *state) combinedCacheManager() CacheManager {
return NewCombinedCacheManager(cms, mainCache)
}
// writeVertexSnapshot can read progress concurrently with
// notifyStarted, which is the only writer of the mutable vertex fields.
func (s *state) writeVertexSnapshot(pw progress.Writer) {
s.clientVertexMu.Lock()
defer s.clientVertexMu.Unlock()
// Keep the snapshot and write together so a newer update cannot be
// published before this snapshot.
pw.Write(identity.NewID(), s.clientVertex)
}
// vertexFailed lets the ancestor walk read the error concurrently with
// progress updates from notifyStarted.
func (s *state) vertexFailed() bool {
s.clientVertexMu.Lock()
defer s.clientVertexMu.Unlock()
return s.clientVertex.Error != ""
}
func (s *state) notifyStarted(ctx context.Context, cached bool) func(err error, cached bool) {
pw, _, _ := progress.NewFromContext(ctx)
s.clientVertexMu.Lock()
v := &s.clientVertex
start := time.Now()
v.Started = &start
v.Completed = nil
v.Cached = cached
id := identity.NewID()
pw.Write(id, *v)
s.clientVertexMu.Unlock()
return func(err error, cached bool) {
defer pw.Close()
s.clientVertexMu.Lock()
defer s.clientVertexMu.Unlock()
stop := time.Now()
v.Completed = &stop
v.Cached = cached
if err != nil {
v.Error = err.Error()
} else {
v.Error = ""
}
pw.Write(id, *v)
}
}
func (s *state) Release() {
for _, e := range s.edges {
for e.owner != nil {
@@ -388,7 +434,9 @@ func (sb *subBuilder) EachValue(ctx context.Context, key string, fn func(any) er
}
type Job struct {
mu sync.Mutex // protects completedTime, pw, span
mu sync.Mutex // protects completedTime, pw, span, builds, discarded
builds int
discarded bool
list *Solver
pr *progress.MultiReader
pw progress.Writer
@@ -689,7 +737,7 @@ func (jl *Solver) connectProgressFromState(target, src *state) {
if _, ok := target.allPw[pw]; !ok {
target.mpw.Add(pw)
target.allPw[pw] = struct{}{}
pw.Write(identity.NewID(), target.clientVertex)
target.writeVertexSnapshot(pw)
if span != nil && span.SpanContext().IsValid() {
target.mspan.Add(span)
}
@@ -800,11 +848,17 @@ func (jl *Solver) deleteIfUnreferenced(k digest.Digest, st *state) {
}
func (j *Job) Build(ctx context.Context, e Edge) (CachedResultWithProvenance, error) {
if span := trace.SpanFromContext(ctx); span.SpanContext().IsValid() {
j.mu.Lock()
j.span = span
j.mu.Lock()
if j.discarded {
j.mu.Unlock()
return nil, errors.Errorf("build on discarded job %s", j.id)
}
j.builds++
if span := trace.SpanFromContext(ctx); span.SpanContext().IsValid() {
j.span = span
}
j.mu.Unlock()
defer j.buildDone()
v, err := j.list.load(ctx, e.Vertex, nil, j)
if err != nil {
@@ -873,7 +927,32 @@ func (j *Job) CloseProgress() {
j.pw.Close()
}
// buildDone finishes a pending Discard when the last running build returns.
func (j *Job) buildDone() {
j.mu.Lock()
j.builds--
last := j.discarded && j.builds == 0
j.mu.Unlock()
if last {
j.discard()
}
}
// Discard releases the job's states. While builds are still running the states
// are released when the last one returns, so results that complete after
// Discard are released with their state.
func (j *Job) Discard() error {
j.mu.Lock()
j.discarded = true
running := j.builds > 0
j.mu.Unlock()
if running {
return nil
}
return j.discard()
}
func (j *Job) discard() error {
j.list.mu.Lock()
defer j.list.mu.Unlock()
@@ -1058,7 +1137,7 @@ func (s *sharedOp) LoadCache(ctx context.Context, rec *CacheRecord) (Result, fun
// no cache hit. start evaluating the node
span, ctx := tracing.StartSpan(ctx, "load cache: "+s.st.vtx.Name(), trace.WithAttributes(attribute.String("vertex", s.st.vtx.Digest().String())))
s.st.execSpan = span
notifyCompleted := notifyStarted(ctx, &s.st.clientVertex, true)
notifyCompleted := s.st.notifyStarted(ctx, true)
res, err := s.Cache().Load(withAncestorCacheOpts(ctx, s.st), rec)
tracing.FinishWithError(span, err)
notifyCompleted(err, true)
@@ -1145,7 +1224,7 @@ func (s *sharedOp) CalcSlowCache(ctx context.Context, index Index, p PreprocessF
if s.st.mspan.Span != nil {
ctx = trace.ContextWithSpan(ctx, s.st.mspan)
}
notifyCompleted := notifyStarted(ctx, &s.st.clientVertex, false)
notifyCompleted := s.st.notifyStarted(ctx, false)
notifyCompleted(err, false)
return "", err
}
@@ -1177,7 +1256,7 @@ func (s *sharedOp) CacheMap(ctx context.Context, index int) (resp *cacheMapResp,
if len(s.st.vtx.Inputs()) == 0 {
// no cache hit. start evaluating the node
span, ctx := tracing.StartSpan(ctx, "cache request: "+s.st.vtx.Name(), trace.WithAttributes(attribute.String("vertex", s.st.vtx.Digest().String())))
notifyCompleted := notifyStarted(ctx, &s.st.clientVertex, false)
notifyCompleted := s.st.notifyStarted(ctx, false)
defer func() {
tracing.FinishWithError(span, retErr)
notifyCompleted(retErr, false)
@@ -1263,7 +1342,7 @@ func (s *sharedOp) Exec(ctx context.Context, inputs []Result) (outputs []Result,
// no cache hit. start evaluating the node
span, ctx := tracing.StartSpan(ctx, s.st.vtx.Name(), trace.WithAttributes(attribute.String("vertex", s.st.vtx.Digest().String())))
s.st.execSpan = span
notifyCompleted := notifyStarted(ctx, &s.st.clientVertex, false)
notifyCompleted := s.st.notifyStarted(ctx, false)
defer func() {
tracing.FinishWithError(span, retErr)
notifyCompleted(retErr, false)
@@ -1390,28 +1469,6 @@ func (v *vertexWithCacheOptions) Inputs() []Edge {
return v.inputs
}
func notifyStarted(ctx context.Context, v *client.Vertex, cached bool) func(err error, cached bool) {
pw, _, _ := progress.NewFromContext(ctx)
start := time.Now()
v.Started = &start
v.Completed = nil
v.Cached = cached
id := identity.NewID()
pw.Write(id, *v)
return func(err error, cached bool) {
defer pw.Close()
stop := time.Now()
v.Completed = &stop
v.Cached = cached
if err != nil {
v.Error = err.Error()
} else {
v.Error = ""
}
pw.Write(id, *v)
}
}
type SlowCacheError struct {
error
Index Index
+28 -10
View File
@@ -5,6 +5,7 @@ import (
"maps"
"strconv"
"strings"
"sync"
"github.com/moby/buildkit/solver/pb"
"github.com/moby/buildkit/util/bklog"
@@ -40,9 +41,10 @@ type Device struct {
}
type Manager struct {
cache *cdi.Cache
locker *locker.Locker
autoAllowed map[string]struct{}
cache *cdi.Cache
locker *locker.Locker
autoAllowedMu sync.RWMutex
autoAllowed map[string]struct{}
}
func NewManager(cache *cdi.Cache, autoAllowed []string) *Manager {
@@ -58,6 +60,8 @@ func NewManager(cache *cdi.Cache, autoAllowed []string) *Manager {
}
func (m *Manager) isAutoAllowed(kind, name string, annotations map[string]string) bool {
m.autoAllowedMu.RLock()
defer m.autoAllowedMu.RUnlock()
if _, ok := m.autoAllowed[name]; ok {
return true
}
@@ -152,7 +156,25 @@ func (m *Manager) FindDevices(devs ...*pb.CDIDevice) ([]string, error) {
return dedupSlice(out), nil
}
// MatchDevices returns the registered devices that the request resolves to,
// including matches by class annotation. Unlike FindDevices it does not fail
// or warn when nothing matches.
func (m *Manager) MatchDevices(dev *pb.CDIDevice) []string {
return m.matchDevice(dev, m.cache.ListDevices())
}
func (m *Manager) parseDevice(dev *pb.CDIDevice, all []string) ([]string, error) {
out := m.matchDevice(dev, all)
if len(out) == 0 {
if !dev.Optional {
return nil, errors.Errorf("required device %q is not registered", dev.Name)
}
bklog.G(context.TODO()).Warnf("Optional device %q is not registered", dev.Name)
}
return out, nil
}
func (m *Manager) matchDevice(dev *pb.CDIDevice, all []string) []string {
var out []string
kind, name, _ := strings.Cut(dev.Name, "=")
@@ -197,13 +219,7 @@ func (m *Manager) parseDevice(dev *pb.CDIDevice, all []string) ([]string, error)
}
}
if len(out) == 0 {
if !dev.Optional {
return nil, errors.Errorf("required device %q is not registered", dev.Name)
}
bklog.G(context.TODO()).Warnf("Optional device %q is not registered", dev.Name)
}
return out, nil
return out
}
func (m *Manager) hasDevice(name string) bool {
@@ -249,7 +265,9 @@ func (m *Manager) OnDemandInstaller(name string) (func(context.Context) error, b
}
// TODO: this needs to be set as annotation to survive reboot
m.autoAllowedMu.Lock()
m.autoAllowed[name] = struct{}{}
m.autoAllowedMu.Unlock()
return nil
}, true
+28 -7
View File
@@ -3,17 +3,27 @@ package errdefs
import (
"context"
"runtime"
"sync"
"github.com/moby/buildkit/solver"
"github.com/moby/buildkit/util/bklog"
)
// ExecError will be returned when an error is encountered when evaluating an op.
// The error owns its original refs until Borrow or the first Release. Proxies
// and gateways retain the error; gateways also keep independent clones. They may
// clone cached originals after Release, which native result handles support
// while another ref pins the snapshot. When sharedOp.Exec completes with an
// active solver state, that state pins error refs until its last job leaves.
// Pinning follows a single Unwrap chain, not errors.Join branches; CacheMap and
// Acquire errors do not get that pin.
type ExecError struct {
error
Inputs []solver.Result
Mounts []solver.Result
OwnerBorrowed bool
mu sync.Mutex // protects OwnerBorrowed
}
func (e *ExecError) Unwrap() error {
@@ -49,16 +59,27 @@ func (e *ExecError) EachRef(fn func(solver.Result) error) (err error) {
return err
}
func (e *ExecError) Release() error {
// Borrow marks the error's refs as owned by a caller, so the finalizer and
// Release do not release them. The same error can be shared by multiple
// callers; Borrow returns false if the refs were already borrowed.
// The successful caller must release the refs; later callers acquire no ownership.
func (e *ExecError) Borrow() bool {
e.mu.Lock()
defer e.mu.Unlock()
if e.OwnerBorrowed {
return false
}
e.OwnerBorrowed = true
return true
}
func (e *ExecError) Release() error {
if !e.Borrow() {
return nil
}
err := e.EachRef(func(r solver.Result) error {
r.Release(context.TODO())
return nil
return e.EachRef(func(r solver.Result) error {
return r.Release(context.TODO())
})
e.OwnerBorrowed = true
return err
}
func WithExecError(err error, inputs, mounts []solver.Result) error {
@@ -75,7 +96,7 @@ func WithExecErrorWithContext(ctx context.Context, err error, inputs, mounts []s
Mounts: mounts,
}
runtime.SetFinalizer(ee, func(e *ExecError) {
if !e.OwnerBorrowed {
if e.Borrow() {
e.EachRef(func(r solver.Result) error {
bklog.G(ctx).Warn("leaked execError detected and released")
r.Release(context.TODO())
+1 -5
View File
@@ -137,11 +137,7 @@ func (s *Solver) recordBuildHistory(ctx context.Context, id string, req frontend
if err != nil {
return nil, nil, err
}
pr, err := pc.Predicate(ctx)
if err != nil {
return nil, nil, err
}
dt, err := json.MarshalIndent(pr, "", " ")
dt, err := pc.PredicateJSON(ctx)
if err != nil {
return nil, nil, err
}
+6 -6
View File
@@ -71,12 +71,12 @@ func (m *staticEmulatorMount) Mount() ([]mount.Mount, func() error, error) {
ret = true
return []mount.Mount{{
Type: "bind",
Source: filepath.Join(tmpdir, qemuMountName),
Options: []string{"ro", "bind"},
}}, func() error {
return os.RemoveAll(tmpdir)
}, nil
Type: "bind",
Source: filepath.Join(tmpdir, qemuMountName),
Options: []string{"ro", "bind"},
}}, func() error {
return os.RemoveAll(tmpdir)
}, nil
}
func (m *staticEmulatorMount) IdentityMapping() *user.IdentityMapping {
+2 -9
View File
@@ -2,7 +2,6 @@ package proc
import (
"context"
"encoding/json"
"strconv"
"github.com/moby/buildkit/executor/resources"
@@ -65,14 +64,8 @@ func ProvenanceProcessor(slsaVersion provenancetypes.ProvenanceSLSA, attrs map[s
InToto: result.InTotoAttestation{
PredicateType: pc.PredicateType(),
},
Path: filename,
ContentFunc: func(ctx context.Context) ([]byte, error) {
pr, err := pc.Predicate(ctx)
if err != nil {
return nil, err
}
return json.MarshalIndent(pr, "", " ")
},
Path: filename,
ContentFunc: pc.PredicateJSON,
})
}
+20 -1
View File
@@ -2,6 +2,7 @@ package llbsolver
import (
"context"
"encoding/json"
"fmt"
"slices"
"strconv"
@@ -384,6 +385,7 @@ func captureProvenance(ctx context.Context, res solver.CachedResultWithProvenanc
}
type ProvenanceCreator struct {
mu sync.Mutex // serializes predicate updates and encoding of pr
pr *provenancetypes.ProvenancePredicateSLSA1
slsaVersion provenancetypes.ProvenanceSLSA
j *solver.Job
@@ -643,7 +645,24 @@ func (p *ProvenanceCreator) PredicateType() string {
return slsa1.PredicateSLSAProvenance
}
func (p *ProvenanceCreator) Predicate(ctx context.Context) (any, error) {
// PredicateJSON finalizes the predicate and returns it JSON encoded. It may be
// called concurrently, e.g. by parallel exporters sharing one attestation.
func (p *ProvenanceCreator) PredicateJSON(ctx context.Context) ([]byte, error) {
p.mu.Lock()
defer p.mu.Unlock()
pr, err := p.predicate(ctx)
if err != nil {
return nil, err
}
dt, err := json.MarshalIndent(pr, "", " ")
if err != nil {
return nil, errors.WithStack(err)
}
return dt, nil
}
func (p *ProvenanceCreator) predicate(ctx context.Context) (any, error) {
end := p.j.RegisterCompleteTime()
if p.pr.RunDetails.Metadata == nil {
p.pr.RunDetails.Metadata = &provenancetypes.ProvenanceMetadataSLSA1{}
+13 -12
View File
@@ -2,6 +2,7 @@ package llbsolver
import (
"context"
"slices"
"sync"
cacheconfig "github.com/moby/buildkit/cache/config"
@@ -46,7 +47,7 @@ type resultProxy struct {
released bool
v solver.CachedResult
err error
errResults []solver.Result
execErrors []*llberrdefs.ExecError
provenance *provenance.Capture
}
@@ -72,8 +73,8 @@ func (rp *resultProxy) Provenance() any {
func (rp *resultProxy) Release(ctx context.Context) (err error) {
rp.mu.Lock()
defer rp.mu.Unlock()
for _, res := range rp.errResults {
rerr := res.Release(ctx)
for _, ee := range rp.execErrors {
rerr := ee.Release()
if rerr != nil {
err = rerr
}
@@ -82,9 +83,8 @@ func (rp *resultProxy) Release(ctx context.Context) (err error) {
if rp.released {
bklog.G(ctx).Warnf("release of already released result")
}
rerr := rp.v.Release(ctx)
if err != nil {
return rerr
if rerr := rp.v.Release(ctx); rerr != nil {
err = rerr
}
}
rp.released = true
@@ -118,12 +118,13 @@ func (rp *resultProxy) loadResult(ctx context.Context) (solver.CachedResultWithP
res, err := rp.b.loadResult(ctx, rp.req.Definition, rp.req.CacheImports, rp.req.SourcePolicies)
var ee *llberrdefs.ExecError
if errors.As(err, &ee) {
ee.EachRef(func(res solver.Result) error {
rp.errResults = append(rp.errResults, res)
return nil
})
// acquire ownership so ExecError finalizer doesn't attempt to release as well
ee.OwnerBorrowed = true
rp.mu.Lock()
if rp.released {
ee.Release()
} else if !slices.Contains(rp.execErrors, ee) {
rp.execErrors = append(rp.execErrors, ee)
}
rp.mu.Unlock()
}
return res, err
}
+24 -9
View File
@@ -167,26 +167,37 @@ func ValidateEntitlements(ent entitlements.Set, cdiManager *cdidevices.Manager)
}
var allowedDevices []*pb.CDIDevice
var nonAliasedDevices []*pb.CDIDevice
var devicesToResolve []*pb.CDIDevice
onDemand := make(map[string]struct{})
for _, d := range cdiManager.ListDevices() {
if d.OnDemand && d.AutoAllow {
allowedDevices = append(allowedDevices, &pb.CDIDevice{Name: d.Name})
onDemand[d.Name] = struct{}{}
}
}
if cfg != nil {
for _, d := range op.Exec.CdiDevices {
for _, d := range op.Exec.CdiDevices {
if d == nil {
continue
}
if cfg != nil {
if newName, ok := cfg.Devices[d.Name]; ok && newName != "" {
d.Name = newName
allowedDevices = append(allowedDevices, d)
} else {
nonAliasedDevices = append(nonAliasedDevices, d)
continue
}
}
} else {
nonAliasedDevices = op.Exec.CdiDevices
kind, _, _ := strings.Cut(d.Name, "=")
// A kind with an on-demand installer has no registered devices
// yet, so a match here can only come from a class annotation
// on another device. That must go through the permission check.
if _, ok := onDemand[kind]; ok && len(cdiManager.MatchDevices(d)) == 0 {
// Resolve the requested name after the installer has registered its devices.
allowedDevices = append(allowedDevices, d)
} else {
devicesToResolve = append(devicesToResolve, d)
}
}
mountedDevices, err := cdiManager.FindDevices(nonAliasedDevices...)
mountedDevices, err := cdiManager.FindDevices(devicesToResolve...)
if err != nil {
return err
}
@@ -198,6 +209,10 @@ func ValidateEntitlements(ent entitlements.Set, cdiManager *cdidevices.Manager)
grantedDevices := make(map[string]struct{})
if cfg != nil {
for d := range cfg.Devices {
kind, _, _ := strings.Cut(d, "=")
if _, ok := onDemand[kind]; ok && len(cdiManager.MatchDevices(&pb.CDIDevice{Name: d})) == 0 {
continue
}
resolved, err := cdiManager.FindDevices(&pb.CDIDevice{Name: d})
if err != nil {
return err
+1
View File
@@ -13,6 +13,7 @@ const AttrGitFetchByCommit = "git.fetchbycommit"
const AttrGitBundle = "git.bundle"
const AttrGitCheckoutBundle = "git.checkoutbundle"
const AttrGitAdvice = "git.advice"
const AttrGitDebugCommands = "git.debugcommands"
const AttrGitSignatureVerifyPubKey = "git.sig.pubkey"
const AttrGitSignatureVerifyRejectExpired = "git.sig.rejectexpired"
+7 -3
View File
@@ -309,19 +309,23 @@ func (s *scheduler) mergeTo(target, src *edge) bool {
delete(s.outgoing, src)
s.signal(target)
var exps []expDep
for i, d := range src.deps {
for _, k := range d.keys {
target.secondaryExporters = append(target.secondaryExporters, expDep{i, CacheKeyWithSelector{CacheKey: k, Selector: src.cacheMap.Deps[i].Selector}})
exps = append(exps, expDep{i, CacheKeyWithSelector{CacheKey: k, Selector: src.cacheMap.Deps[i].Selector}})
}
if d.slowCacheKey != nil {
target.secondaryExporters = append(target.secondaryExporters, expDep{i, CacheKeyWithSelector{CacheKey: *d.slowCacheKey}})
exps = append(exps, expDep{i, CacheKeyWithSelector{CacheKey: *d.slowCacheKey}})
}
if d.result != nil {
for _, dk := range d.result.CacheKeys() {
target.secondaryExporters = append(target.secondaryExporters, expDep{i, CacheKeyWithSelector{CacheKey: dk, Selector: src.cacheMap.Deps[i].Selector}})
exps = append(exps, expDep{i, CacheKeyWithSelector{CacheKey: dk, Selector: src.cacheMap.Deps[i].Selector}})
}
}
}
target.secondaryExportersMu.Lock()
target.secondaryExporters = append(target.secondaryExporters, exps...)
target.secondaryExportersMu.Unlock()
// TODO(tonistiigi): merge cache providers
+24 -11
View File
@@ -49,13 +49,14 @@ type puller struct {
ResolverType
store sourceresolver.ResolveImageConfigOptStore
g flightcontrol.Group[struct{}]
cacheKeyErr error
cacheKeyDone bool
releaseTmpLeases func(context.Context) error
descHandlers cache.DescHandlers
manifest *pull.PulledManifests
manifestKey string
g flightcontrol.Group[struct{}]
cacheKeyErr error
cacheKeyDone bool
releaseTmpLeases func(context.Context) error
descHandlers cache.DescHandlers
manifest *pull.PulledManifests
manifestKey string
platformManifestKey string
*pull.Puller
}
@@ -132,8 +133,8 @@ func (p *puller) CacheKey(ctx context.Context, jobCtx solver.JobContext, index i
return struct{}{}, err
}
if p.checksum != "" && p.manifest.MainManifestDesc.Digest != p.checksum {
return struct{}{}, errors.Errorf("image digest %s for %s does not match expected checksum %s", p.manifest.MainManifestDesc.Digest, p.Ref, p.checksum)
if p.checksum != "" && p.manifest.RootManifestDesc.Digest != p.checksum {
return struct{}{}, errors.Errorf("image digest %s for %s does not match expected checksum %s", p.manifest.RootManifestDesc.Digest, p.Ref, p.checksum)
}
if ll := p.layerLimit; ll != nil {
@@ -172,12 +173,17 @@ func (p *puller) CacheKey(ctx context.Context, jobCtx solver.JobContext, index i
}
}
desc := p.manifest.MainManifestDesc
desc := p.manifest.RootManifestDesc
k, err := mainManifestKey(desc, p.Platform, p.layerLimit)
if err != nil {
return struct{}{}, err
}
p.manifestKey = k.String()
k, err = mainManifestKey(p.manifest.ImageManifestDesc, p.Platform, p.layerLimit)
if err != nil {
return struct{}{}, err
}
p.platformManifestKey = k.String()
// Do not use RootFS DiffIDs from the image config as an alternate
// cache key. They are verified only when layers are applied, and a
// failed malicious pull must not poison another image source.
@@ -193,7 +199,14 @@ func (p *puller) CacheKey(ctx context.Context, jobCtx solver.JobContext, index i
cacheOpts[cache.DescHandlerKey(dgst)] = descHandler
}
return p.manifestKey, p.manifest.MainManifestDesc.Digest.String(), cacheOpts, true, nil
// Keep the index key for existing caches, but also match the selected
// manifest so changes to attestations or other platforms do not invalidate
// this source. Both keys identify verified manifest content, not DiffIDs
// advertised by an unverified image config.
if index == 0 {
return p.manifestKey, p.manifest.RootManifestDesc.Digest.String(), cacheOpts, p.manifestKey == p.platformManifestKey, nil
}
return p.platformManifestKey, p.manifest.RootManifestDesc.Digest.String(), cacheOpts, true, nil
}
func (p *puller) Snapshot(ctx context.Context, jobCtx solver.JobContext) (ir cache.ImmutableRef, err error) {
+4 -4
View File
@@ -72,8 +72,8 @@ func bundleTargetRef(ref string) string {
// sha256 object IDs. git ls-remote reads bundle files directly, so this works
// before a destination repo exists and lets callers initialize the right object
// format for a subsequent fetch/import.
func detectBundleSHA256(ctx context.Context, bundlePath string, gitAdvice bool) (bool, error) {
buf, err := gitCLI(gitutil.WithGitAdvice(gitAdvice)).Run(ctx, "ls-remote", "--", bundlePath)
func detectBundleSHA256(ctx context.Context, bundlePath string, gitAdvice, debugCommands bool) (bool, error) {
buf, err := gitCLI(gitutil.WithGitAdvice(gitAdvice), gitutil.WithGitDebugCommands(debugCommands)).Run(ctx, "ls-remote", "--", bundlePath)
if err != nil {
return false, errors.Wrapf(err, "failed to inspect git bundle %s", bundlePath)
}
@@ -120,7 +120,7 @@ func (gs *gitSourceHandler) stageBundle(ctx context.Context, g session.Group) (_
return "", nil, err
}
bundlePath := filepath.Join(tmpDir, bundleImportFileName)
sha256, err := detectBundleSHA256(ctx, bundlePath, gs.src.Advice)
sha256, err := detectBundleSHA256(ctx, bundlePath, gs.src.Advice, gs.src.DebugCommands)
if err != nil {
return "", nil, err
}
@@ -133,7 +133,7 @@ func (gs *gitSourceHandler) stageBundle(ctx context.Context, g session.Group) (_
if err := os.Mkdir(tmpRepoDir, 0700); err != nil {
return "", nil, errors.Wrap(err, "failed to create temp bare repo dir")
}
tmpGit := gitCLI(gitutil.WithGitAdvice(gs.src.Advice), gitutil.WithGitDir(tmpRepoDir))
tmpGit := gitCLI(gitutil.WithGitAdvice(gs.src.Advice), gitutil.WithGitDebugCommands(gs.src.DebugCommands), gitutil.WithGitDir(tmpRepoDir))
initArgs := []string{"-c", "init.defaultBranch=master", "init", "--bare"}
if sha256 {
initArgs = append(initArgs, "--object-format=sha256")
+1
View File
@@ -29,6 +29,7 @@ type GitIdentifier struct {
SkipSubmodules bool
MTime string // "checkout" (default) or "commit"
Advice bool
DebugCommands bool
FetchByCommit bool
// Bundle, when non-empty, instructs the git source to fetch commits from a
+6 -2
View File
@@ -150,6 +150,8 @@ func (gs *Source) Identifier(scheme, ref string, attrs map[string]string, platfo
id.MTime = v
case pb.AttrGitAdvice:
id.Advice = v == "true"
case pb.AttrGitDebugCommands:
id.DebugCommands = v == "true"
case pb.AttrGitFetchByCommit:
id.FetchByCommit = v == "true"
case pb.AttrGitBundle:
@@ -173,7 +175,7 @@ func (gs *Source) Identifier(scheme, ref string, attrs map[string]string, platfo
}
// needs to be called with repo lock
func (gs *Source) mountRemote(ctx context.Context, remote string, authArgs []string, gitAdvice bool, sha256 bool, reset bool, g session.Group) (target string, release func() error, retErr error) {
func (gs *Source) mountRemote(ctx context.Context, remote string, authArgs []string, gitAdvice bool, debugCommands bool, sha256 bool, reset bool, g session.Group) (target string, release func() error, retErr error) {
sis, err := searchGitRemote(ctx, gs.cache, remote)
if err != nil {
return "", nil, errors.Wrapf(err, "failed to search metadata for %s", urlutil.RedactCredentials(remote))
@@ -237,6 +239,7 @@ func (gs *Source) mountRemote(ctx context.Context, remote string, authArgs []str
git := gitCLI(
gitutil.WithGitAdvice(gitAdvice),
gitutil.WithGitDebugCommands(debugCommands),
gitutil.WithGitDir(dir),
gitutil.WithArgs(authArgs...),
)
@@ -976,7 +979,7 @@ func (gs *gitSourceHandler) tryRemoteFetch(ctx context.Context, jobCtx solver.Jo
}
}
gitDir, unmountGitDir, err := gs.mountRemote(ctx, gs.src.Remote, authArgs, gs.src.Advice, gs.sha256, reset, g)
gitDir, unmountGitDir, err := gs.mountRemote(ctx, gs.src.Remote, authArgs, gs.src.Advice, gs.src.DebugCommands, gs.sha256, reset, g)
if err != nil {
return nil, err
}
@@ -1534,6 +1537,7 @@ func (gs *gitSourceHandler) emptyGitCli(ctx context.Context, g session.Group, op
opts = append([]gitutil.Option{
gitutil.WithGitAdvice(gs.src.Advice),
gitutil.WithGitDebugCommands(gs.src.DebugCommands),
gitutil.WithArgs(gs.authArgs...),
gitutil.WithSSHAuthSock(sock),
gitutil.WithSSHKnownHosts(knownHosts),
+44 -2
View File
@@ -8,6 +8,7 @@ import (
"encoding/json"
"fmt"
"io"
"math"
"mime"
"net/http"
"net/url"
@@ -31,6 +32,7 @@ import (
"github.com/moby/buildkit/util/bklog"
"github.com/moby/buildkit/util/cachedigest"
"github.com/moby/buildkit/util/pgpsign"
"github.com/moby/buildkit/util/resolver/retryhandler"
"github.com/moby/buildkit/util/tracing"
"github.com/moby/buildkit/util/urlutil"
"github.com/moby/buildkit/version"
@@ -215,6 +217,45 @@ func (hs *httpSourceHandler) client(g session.Group) *http.Client {
return &http.Client{Transport: newTransport(hs.transport, hs.sm, g)}
}
// doRequest retries transient network failures and HTTP 429/503 responses with
// jittered exponential backoff, honoring Retry-After as a minimum delay.
// Requests have no body and are safe to replay. Failures reading a successful
// response body and other HTTP statuses are left to the caller.
func doRequest(ctx context.Context, client *http.Client, req *http.Request) (*http.Response, error) {
return retryhandler.WithRetry(ctx, nil, func(ctx context.Context) (*http.Response, error) {
resp, err := client.Do(req.Clone(ctx))
if err != nil {
return nil, err
}
if resp.StatusCode != http.StatusTooManyRequests && resp.StatusCode != http.StatusServiceUnavailable {
return resp, nil
}
// Do not drain an error body: it may stall independently of the headers.
resp.Body.Close()
return nil, &retryhandler.RetryAfterError{
Err: errors.Errorf("invalid response status %d", resp.StatusCode),
Delay: retryAfterDelay(resp.Header.Get("Retry-After")),
}
})
}
// retryAfterDelay parses either delay-seconds or an HTTP-date. Invalid and past
// values fall back to the normal backoff. Saturate large values to avoid overflow.
func retryAfterDelay(value string) time.Duration {
value = strings.TrimSpace(value)
seconds, err := strconv.ParseUint(value, 10, 64)
if err == nil || errors.Is(err, strconv.ErrRange) {
if seconds > uint64(math.MaxInt64/int64(time.Second)) {
return time.Duration(math.MaxInt64)
}
return time.Duration(seconds) * time.Second
}
if date, err := http.ParseTime(value); err == nil {
return max(time.Until(date), 0)
}
return 0
}
// urlHash is internal hash the etag is stored by that doesn't leak outside
// this package.
func (hs *httpSourceHandler) urlHash() (digest.Digest, error) {
@@ -408,6 +449,7 @@ func (hs *httpSourceHandler) resolveMetadataRef(ctx context.Context, jobCtx solv
// we need to add accept-encoding header manually because stdlib only adds it to GET requests
// some servers will return different etags if Accept-Encoding header is different
req.Header.Set("Accept-Encoding", "gzip")
// Best-effort: on any error we fall through to the GET below, which is retried.
resp, err := client.Do(req)
if err == nil {
if resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusNotModified {
@@ -454,7 +496,7 @@ func (hs *httpSourceHandler) resolveMetadataRef(ctx context.Context, jobCtx solv
req.Header.Del("Accept-Encoding")
}
resp, err := client.Do(req)
resp, err := doRequest(ctx, client, req)
if err != nil {
return nil, err
}
@@ -889,7 +931,7 @@ func (hs *httpSourceHandler) Snapshot(ctx context.Context, jobCtx solver.JobCont
client := hs.client(g)
resp, err := client.Do(req)
resp, err := doRequest(ctx, client, req)
if err != nil {
return nil, err
}
+27
View File
@@ -0,0 +1,27 @@
Copyright 2009 The Go Authors.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google LLC nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+747
View File
@@ -0,0 +1,747 @@
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Code generated from Go 1.26.5 commit c19862e5f8415b4f24b189d065ed739517c548ba; DO NOT EDIT.
package go126flate
import (
"errors"
"fmt"
"io"
"math"
)
const (
NoCompression = 0
BestSpeed = 1
BestCompression = 9
DefaultCompression = -1
// HuffmanOnly disables Lempel-Ziv match searching and only performs Huffman
// entropy encoding. This mode is useful in compressing data that has
// already been compressed with an LZ style algorithm (e.g. Snappy or LZ4)
// that lacks an entropy encoder. Compression gains are achieved when
// certain bytes in the input stream occur more frequently than others.
//
// Note that HuffmanOnly produces a compressed output that is
// RFC 1951 compliant. That is, any valid DEFLATE decompressor will
// continue to be able to decompress this output.
HuffmanOnly = -2
)
const (
logWindowSize = 15
windowSize = 1 << logWindowSize
windowMask = windowSize - 1
// The LZ77 step produces a sequence of literal tokens and <length, offset>
// pair tokens. The offset is also known as distance. The underlying wire
// format limits the range of lengths and offsets. For example, there are
// 256 legitimate lengths: those in the range [3, 258]. This package's
// compressor uses a higher minimum match length, enabling optimizations
// such as finding matches via 32-bit loads and compares.
baseMatchLength = 3 // The smallest match length per the RFC section 3.2.5
minMatchLength = 4 // The smallest match length that the compressor actually emits
maxMatchLength = 258 // The largest match length
baseMatchOffset = 1 // The smallest match offset
maxMatchOffset = 1 << 15 // The largest match offset
// The maximum number of tokens we put into a single flate block, just to
// stop things from getting too large.
maxFlateBlockTokens = 1 << 14
maxStoreBlockSize = 65535
hashBits = 17 // After 17 performance degrades
hashSize = 1 << hashBits
hashMask = (1 << hashBits) - 1
maxHashOffset = 1 << 24
skipNever = math.MaxInt32
)
type compressionLevel struct {
level, good, lazy, nice, chain, fastSkipHashing int
}
var levels = []compressionLevel{
{0, 0, 0, 0, 0, 0}, // NoCompression.
{1, 0, 0, 0, 0, 0}, // BestSpeed uses a custom algorithm; see deflatefast.go.
// For levels 2-3 we don't bother trying with lazy matches.
{2, 4, 0, 16, 8, 5},
{3, 4, 0, 32, 32, 6},
// Levels 4-9 use increasingly more lazy matching
// and increasingly stringent conditions for "good enough".
{4, 4, 4, 16, 16, skipNever},
{5, 8, 16, 32, 32, skipNever},
{6, 8, 16, 128, 128, skipNever},
{7, 8, 32, 128, 256, skipNever},
{8, 32, 128, 258, 1024, skipNever},
{9, 32, 258, 258, 4096, skipNever},
}
type compressor struct {
compressionLevel
w *huffmanBitWriter
bulkHasher func([]byte, []uint32)
// compression algorithm
fill func(*compressor, []byte) int // copy data to window
step func(*compressor) // process window
bestSpeed *deflateFast // Encoder for BestSpeed
// input window: unprocessed data is window[index:windowEnd]
index int
window []byte
windowEnd int
blockStart int // window index where current tokens start
byteAvailable bool // if true, still need to process window[index-1].
sync bool // requesting flush
// queued output tokens
tokens []token
// deflate state
length int
offset int
maxInsertIndex int
err error
// Input hash chains
// hashHead[hashValue] contains the largest inputIndex with the specified hash value
// If hashHead[hashValue] is within the current window, then
// hashPrev[hashHead[hashValue] & windowMask] contains the previous index
// with the same hash value.
// These are large and do not contain pointers, so put them
// near the end of the struct so the GC has to scan less.
chainHead int
hashHead [hashSize]uint32
hashPrev [windowSize]uint32
hashOffset int
// hashMatch must be able to contain hashes for the maximum match length.
hashMatch [maxMatchLength - 1]uint32
}
func (d *compressor) fillDeflate(b []byte) int {
if d.index >= 2*windowSize-(minMatchLength+maxMatchLength) {
// shift the window by windowSize
copy(d.window, d.window[windowSize:2*windowSize])
d.index -= windowSize
d.windowEnd -= windowSize
if d.blockStart >= windowSize {
d.blockStart -= windowSize
} else {
d.blockStart = math.MaxInt32
}
d.hashOffset += windowSize
if d.hashOffset > maxHashOffset {
delta := d.hashOffset - 1
d.hashOffset -= delta
d.chainHead -= delta
// Iterate over slices instead of arrays to avoid copying
// the entire table onto the stack (Issue #18625).
for i, v := range d.hashPrev[:] {
if int(v) > delta {
d.hashPrev[i] = uint32(int(v) - delta)
} else {
d.hashPrev[i] = 0
}
}
for i, v := range d.hashHead[:] {
if int(v) > delta {
d.hashHead[i] = uint32(int(v) - delta)
} else {
d.hashHead[i] = 0
}
}
}
}
n := copy(d.window[d.windowEnd:], b)
d.windowEnd += n
return n
}
func (d *compressor) writeBlock(tokens []token, index int) error {
if index > 0 {
var window []byte
if d.blockStart <= index {
window = d.window[d.blockStart:index]
}
d.blockStart = index
d.w.writeBlock(tokens, false, window)
return d.w.err
}
return nil
}
// fillWindow will fill the current window with the supplied
// dictionary and calculate all hashes.
// This is much faster than doing a full encode.
// Should only be used after a reset.
func (d *compressor) fillWindow(b []byte) {
// Do not fill window if we are in store-only mode.
if d.compressionLevel.level < 2 {
return
}
if d.index != 0 || d.windowEnd != 0 {
panic("internal error: fillWindow called with stale data")
}
// If we are given too much, cut it.
if len(b) > windowSize {
b = b[len(b)-windowSize:]
}
// Add all to window.
n := copy(d.window, b)
// Calculate 256 hashes at the time (more L1 cache hits)
loops := (n + 256 - minMatchLength) / 256
for j := 0; j < loops; j++ {
index := j * 256
end := index + 256 + minMatchLength - 1
if end > n {
end = n
}
toCheck := d.window[index:end]
dstSize := len(toCheck) - minMatchLength + 1
if dstSize <= 0 {
continue
}
dst := d.hashMatch[:dstSize]
d.bulkHasher(toCheck, dst)
for i, val := range dst {
di := i + index
hh := &d.hashHead[val&hashMask]
// Get previous value with the same hash.
// Our chain should point to the previous value.
d.hashPrev[di&windowMask] = *hh
// Set the head of the hash chain to us.
*hh = uint32(di + d.hashOffset)
}
}
// Update window information.
d.windowEnd = n
d.index = n
}
// Try to find a match starting at index whose length is greater than prevSize.
// We only look at chainCount possibilities before giving up.
func (d *compressor) findMatch(pos int, prevHead int, prevLength int, lookahead int) (length, offset int, ok bool) {
minMatchLook := maxMatchLength
if lookahead < minMatchLook {
minMatchLook = lookahead
}
win := d.window[0 : pos+minMatchLook]
// We quit when we get a match that's at least nice long
nice := len(win) - pos
if d.nice < nice {
nice = d.nice
}
// If we've got a match that's good enough, only look in 1/4 the chain.
tries := d.chain
length = prevLength
if length >= d.good {
tries >>= 2
}
wEnd := win[pos+length]
wPos := win[pos:]
minIndex := pos - windowSize
for i := prevHead; tries > 0; tries-- {
if wEnd == win[i+length] {
n := matchLen(win[i:], wPos, minMatchLook)
if n > length && (n > minMatchLength || pos-i <= 4096) {
length = n
offset = pos - i
ok = true
if n >= nice {
// The match is good enough that we don't try to find a better one.
break
}
wEnd = win[pos+n]
}
}
if i == minIndex {
// hashPrev[i & windowMask] has already been overwritten, so stop now.
break
}
i = int(d.hashPrev[i&windowMask]) - d.hashOffset
if i < minIndex || i < 0 {
break
}
}
return
}
func (d *compressor) writeStoredBlock(buf []byte) error {
if d.w.writeStoredHeader(len(buf), false); d.w.err != nil {
return d.w.err
}
d.w.writeBytes(buf)
return d.w.err
}
const hashmul = 0x1e35a7bd
// hash4 returns a hash representation of the first 4 bytes
// of the supplied slice.
// The caller must ensure that len(b) >= 4.
func hash4(b []byte) uint32 {
return ((uint32(b[3]) | uint32(b[2])<<8 | uint32(b[1])<<16 | uint32(b[0])<<24) * hashmul) >> (32 - hashBits)
}
// bulkHash4 will compute hashes using the same
// algorithm as hash4.
func bulkHash4(b []byte, dst []uint32) {
if len(b) < minMatchLength {
return
}
hb := uint32(b[3]) | uint32(b[2])<<8 | uint32(b[1])<<16 | uint32(b[0])<<24
dst[0] = (hb * hashmul) >> (32 - hashBits)
end := len(b) - minMatchLength + 1
for i := 1; i < end; i++ {
hb = (hb << 8) | uint32(b[i+3])
dst[i] = (hb * hashmul) >> (32 - hashBits)
}
}
// matchLen returns the number of matching bytes in a and b
// up to length 'max'. Both slices must be at least 'max'
// bytes in size.
func matchLen(a, b []byte, max int) int {
a = a[:max]
b = b[:len(a)]
for i, av := range a {
if b[i] != av {
return i
}
}
return max
}
// encSpeed will compress and store the currently added data,
// if enough has been accumulated or we at the end of the stream.
// Any error that occurred will be in d.err
func (d *compressor) encSpeed() {
// We only compress if we have maxStoreBlockSize.
if d.windowEnd < maxStoreBlockSize {
if !d.sync {
return
}
// Handle small sizes.
if d.windowEnd < 128 {
switch {
case d.windowEnd == 0:
return
case d.windowEnd <= 16:
d.err = d.writeStoredBlock(d.window[:d.windowEnd])
default:
d.w.writeBlockHuff(false, d.window[:d.windowEnd])
d.err = d.w.err
}
d.windowEnd = 0
d.bestSpeed.reset()
return
}
}
// Encode the block.
d.tokens = d.bestSpeed.encode(d.tokens[:0], d.window[:d.windowEnd])
// If we removed less than 1/16th, Huffman compress the block.
if len(d.tokens) > d.windowEnd-(d.windowEnd>>4) {
d.w.writeBlockHuff(false, d.window[:d.windowEnd])
} else {
d.w.writeBlockDynamic(d.tokens, false, d.window[:d.windowEnd])
}
d.err = d.w.err
d.windowEnd = 0
}
func (d *compressor) initDeflate() {
d.window = make([]byte, 2*windowSize)
d.hashOffset = 1
d.tokens = make([]token, 0, maxFlateBlockTokens+1)
d.length = minMatchLength - 1
d.offset = 0
d.byteAvailable = false
d.index = 0
d.chainHead = -1
d.bulkHasher = bulkHash4
}
func (d *compressor) deflate() {
if d.windowEnd-d.index < minMatchLength+maxMatchLength && !d.sync {
return
}
d.maxInsertIndex = d.windowEnd - (minMatchLength - 1)
Loop:
for {
if d.index > d.windowEnd {
panic("index > windowEnd")
}
lookahead := d.windowEnd - d.index
if lookahead < minMatchLength+maxMatchLength {
if !d.sync {
break Loop
}
if d.index > d.windowEnd {
panic("index > windowEnd")
}
if lookahead == 0 {
// Flush current output block if any.
if d.byteAvailable {
// There is still one pending token that needs to be flushed
d.tokens = append(d.tokens, literalToken(uint32(d.window[d.index-1])))
d.byteAvailable = false
}
if len(d.tokens) > 0 {
if d.err = d.writeBlock(d.tokens, d.index); d.err != nil {
return
}
d.tokens = d.tokens[:0]
}
break Loop
}
}
if d.index < d.maxInsertIndex {
// Update the hash
hash := hash4(d.window[d.index : d.index+minMatchLength])
hh := &d.hashHead[hash&hashMask]
d.chainHead = int(*hh)
d.hashPrev[d.index&windowMask] = uint32(d.chainHead)
*hh = uint32(d.index + d.hashOffset)
}
prevLength := d.length
prevOffset := d.offset
d.length = minMatchLength - 1
d.offset = 0
minIndex := d.index - windowSize
if minIndex < 0 {
minIndex = 0
}
if d.chainHead-d.hashOffset >= minIndex &&
(d.fastSkipHashing != skipNever && lookahead > minMatchLength-1 ||
d.fastSkipHashing == skipNever && lookahead > prevLength && prevLength < d.lazy) {
if newLength, newOffset, ok := d.findMatch(d.index, d.chainHead-d.hashOffset, minMatchLength-1, lookahead); ok {
d.length = newLength
d.offset = newOffset
}
}
if d.fastSkipHashing != skipNever && d.length >= minMatchLength ||
d.fastSkipHashing == skipNever && prevLength >= minMatchLength && d.length <= prevLength {
// There was a match at the previous step, and the current match is
// not better. Output the previous match.
if d.fastSkipHashing != skipNever {
d.tokens = append(d.tokens, matchToken(uint32(d.length-baseMatchLength), uint32(d.offset-baseMatchOffset)))
} else {
d.tokens = append(d.tokens, matchToken(uint32(prevLength-baseMatchLength), uint32(prevOffset-baseMatchOffset)))
}
// Insert in the hash table all strings up to the end of the match.
// index and index-1 are already inserted. If there is not enough
// lookahead, the last two strings are not inserted into the hash
// table.
if d.length <= d.fastSkipHashing {
var newIndex int
if d.fastSkipHashing != skipNever {
newIndex = d.index + d.length
} else {
newIndex = d.index + prevLength - 1
}
index := d.index
for index++; index < newIndex; index++ {
if index < d.maxInsertIndex {
hash := hash4(d.window[index : index+minMatchLength])
// Get previous value with the same hash.
// Our chain should point to the previous value.
hh := &d.hashHead[hash&hashMask]
d.hashPrev[index&windowMask] = *hh
// Set the head of the hash chain to us.
*hh = uint32(index + d.hashOffset)
}
}
d.index = index
if d.fastSkipHashing == skipNever {
d.byteAvailable = false
d.length = minMatchLength - 1
}
} else {
// For matches this long, we don't bother inserting each individual
// item into the table.
d.index += d.length
}
if len(d.tokens) == maxFlateBlockTokens {
// The block includes the current character
if d.err = d.writeBlock(d.tokens, d.index); d.err != nil {
return
}
d.tokens = d.tokens[:0]
}
} else {
if d.fastSkipHashing != skipNever || d.byteAvailable {
i := d.index - 1
if d.fastSkipHashing != skipNever {
i = d.index
}
d.tokens = append(d.tokens, literalToken(uint32(d.window[i])))
if len(d.tokens) == maxFlateBlockTokens {
if d.err = d.writeBlock(d.tokens, i+1); d.err != nil {
return
}
d.tokens = d.tokens[:0]
}
}
d.index++
if d.fastSkipHashing == skipNever {
d.byteAvailable = true
}
}
}
}
func (d *compressor) fillStore(b []byte) int {
n := copy(d.window[d.windowEnd:], b)
d.windowEnd += n
return n
}
func (d *compressor) store() {
if d.windowEnd > 0 && (d.windowEnd == maxStoreBlockSize || d.sync) {
d.err = d.writeStoredBlock(d.window[:d.windowEnd])
d.windowEnd = 0
}
}
// storeHuff compresses and stores the currently added data
// when the d.window is full or we are at the end of the stream.
// Any error that occurred will be in d.err
func (d *compressor) storeHuff() {
if d.windowEnd < len(d.window) && !d.sync || d.windowEnd == 0 {
return
}
d.w.writeBlockHuff(false, d.window[:d.windowEnd])
d.err = d.w.err
d.windowEnd = 0
}
func (d *compressor) write(b []byte) (n int, err error) {
if d.err != nil {
return 0, d.err
}
n = len(b)
for len(b) > 0 {
d.step(d)
b = b[d.fill(d, b):]
if d.err != nil {
return 0, d.err
}
}
return n, nil
}
func (d *compressor) syncFlush() error {
if d.err != nil {
return d.err
}
d.sync = true
d.step(d)
if d.err == nil {
d.w.writeStoredHeader(0, false)
d.w.flush()
d.err = d.w.err
}
d.sync = false
return d.err
}
func (d *compressor) init(w io.Writer, level int) (err error) {
d.w = newHuffmanBitWriter(w)
switch {
case level == NoCompression:
d.window = make([]byte, maxStoreBlockSize)
d.fill = (*compressor).fillStore
d.step = (*compressor).store
case level == HuffmanOnly:
d.window = make([]byte, maxStoreBlockSize)
d.fill = (*compressor).fillStore
d.step = (*compressor).storeHuff
case level == BestSpeed:
d.compressionLevel = levels[level]
d.window = make([]byte, maxStoreBlockSize)
d.fill = (*compressor).fillStore
d.step = (*compressor).encSpeed
d.bestSpeed = newDeflateFast()
d.tokens = make([]token, maxStoreBlockSize)
case level == DefaultCompression:
level = 6
fallthrough
case 2 <= level && level <= 9:
d.compressionLevel = levels[level]
d.initDeflate()
d.fill = (*compressor).fillDeflate
d.step = (*compressor).deflate
default:
return fmt.Errorf("flate: invalid compression level %d: want value in range [-2, 9]", level)
}
return nil
}
func (d *compressor) reset(w io.Writer) {
d.w.reset(w)
d.sync = false
d.err = nil
switch d.compressionLevel.level {
case NoCompression:
d.windowEnd = 0
case BestSpeed:
d.windowEnd = 0
d.tokens = d.tokens[:0]
d.bestSpeed.reset()
default:
d.chainHead = -1
clear(d.hashHead[:])
clear(d.hashPrev[:])
d.hashOffset = 1
d.index, d.windowEnd = 0, 0
d.blockStart, d.byteAvailable = 0, false
d.tokens = d.tokens[:0]
d.length = minMatchLength - 1
d.offset = 0
d.maxInsertIndex = 0
}
}
func (d *compressor) close() error {
if d.err == errWriterClosed {
return nil
}
if d.err != nil {
return d.err
}
d.sync = true
d.step(d)
if d.err != nil {
return d.err
}
if d.w.writeStoredHeader(0, true); d.w.err != nil {
return d.w.err
}
d.w.flush()
if d.w.err != nil {
return d.w.err
}
d.err = errWriterClosed
return nil
}
// NewWriter returns a new [Writer] compressing data at the given level.
// Following zlib, levels range from 1 ([BestSpeed]) to 9 ([BestCompression]);
// higher levels typically run slower but compress more. Level 0
// ([NoCompression]) does not attempt any compression; it only adds the
// necessary DEFLATE framing.
// Level -1 ([DefaultCompression]) uses the default compression level.
// Level -2 ([HuffmanOnly]) will use Huffman compression only, giving
// a very fast compression for all types of input, but sacrificing considerable
// compression efficiency.
//
// If level is in the range [-2, 9] then the error returned will be nil.
// Otherwise the error returned will be non-nil.
func NewWriter(w io.Writer, level int) (*Writer, error) {
var dw Writer
if err := dw.d.init(w, level); err != nil {
return nil, err
}
return &dw, nil
}
// NewWriterDict is like [NewWriter] but initializes the new
// [Writer] with a preset dictionary. The returned [Writer] behaves
// as if the dictionary had been written to it without producing
// any compressed output. The compressed data written to w
// can only be decompressed by a reader initialized with the
// same dictionary (see [NewReaderDict]).
func NewWriterDict(w io.Writer, level int, dict []byte) (*Writer, error) {
dw := &dictWriter{w}
zw, err := NewWriter(dw, level)
if err != nil {
return nil, err
}
zw.d.fillWindow(dict)
zw.dict = append(zw.dict, dict...) // duplicate dictionary for Reset method.
return zw, nil
}
type dictWriter struct {
w io.Writer
}
func (w *dictWriter) Write(b []byte) (n int, err error) {
return w.w.Write(b)
}
var errWriterClosed = errors.New("flate: closed writer")
// A Writer takes data written to it and writes the compressed
// form of that data to an underlying writer (see [NewWriter]).
type Writer struct {
d compressor
dict []byte
}
// Write writes data to w, which will eventually write the
// compressed form of data to its underlying writer.
func (w *Writer) Write(data []byte) (n int, err error) {
return w.d.write(data)
}
// Flush flushes any pending data to the underlying writer.
// It is useful mainly in compressed network protocols, to ensure that
// a remote reader has enough data to reconstruct a packet.
// Flush does not return until the data has been written.
// Calling Flush when there is no pending data still causes the [Writer]
// to emit a sync marker of at least 4 bytes.
// If the underlying writer returns an error, Flush returns that error.
//
// In the terminology of the zlib library, Flush is equivalent to Z_SYNC_FLUSH.
func (w *Writer) Flush() error {
// For more about flushing:
// https://www.bolet.org/~pornin/deflate-flush.html
return w.d.syncFlush()
}
// Close flushes and closes the writer.
func (w *Writer) Close() error {
return w.d.close()
}
// Reset discards the writer's state and makes it equivalent to
// the result of [NewWriter] or [NewWriterDict] called with dst
// and w's level and dictionary.
func (w *Writer) Reset(dst io.Writer) {
if dw, ok := w.d.w.writer.(*dictWriter); ok {
// w was created with NewWriterDict
dw.w = dst
w.d.reset(dw)
w.d.fillWindow(w.dict)
} else {
// w was created with NewWriter
w.d.reset(dst)
}
}
@@ -0,0 +1,309 @@
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Code generated from Go 1.26.5 commit c19862e5f8415b4f24b189d065ed739517c548ba; DO NOT EDIT.
package go126flate
import "math"
// This encoding algorithm, which prioritizes speed over output size, is
// based on Snappy's LZ77-style encoder: github.com/golang/snappy
const (
tableBits = 14 // Bits used in the table.
tableSize = 1 << tableBits // Size of the table.
tableMask = tableSize - 1 // Mask for table indices. Redundant, but can eliminate bounds checks.
tableShift = 32 - tableBits // Right-shift to get the tableBits most significant bits of a uint32.
// Reset the buffer offset when reaching this.
// Offsets are stored between blocks as int32 values.
// Since the offset we are checking against is at the beginning
// of the buffer, we need to subtract the current and input
// buffer to not risk overflowing the int32.
bufferReset = math.MaxInt32 - maxStoreBlockSize*2
)
func load32(b []byte, i int32) uint32 {
b = b[i : i+4 : len(b)] // Help the compiler eliminate bounds checks on the next line.
return uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24
}
func load64(b []byte, i int32) uint64 {
b = b[i : i+8 : len(b)] // Help the compiler eliminate bounds checks on the next line.
return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 |
uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56
}
func hash(u uint32) uint32 {
return (u * 0x1e35a7bd) >> tableShift
}
// These constants are defined by the Snappy implementation so that its
// assembly implementation can fast-path some 16-bytes-at-a-time copies. They
// aren't necessary in the pure Go implementation, as we don't use those same
// optimizations, but using the same thresholds doesn't really hurt.
const (
inputMargin = 16 - 1
minNonLiteralBlockSize = 1 + 1 + inputMargin
)
type tableEntry struct {
val uint32 // Value at destination
offset int32
}
// deflateFast maintains the table for matches,
// and the previous byte block for cross block matching.
type deflateFast struct {
table [tableSize]tableEntry
prev []byte // Previous block, zero length if unknown.
cur int32 // Current match offset.
}
func newDeflateFast() *deflateFast {
return &deflateFast{cur: maxStoreBlockSize, prev: make([]byte, 0, maxStoreBlockSize)}
}
// encode encodes a block given in src and appends tokens
// to dst and returns the result.
func (e *deflateFast) encode(dst []token, src []byte) []token {
// Ensure that e.cur doesn't wrap.
if e.cur >= bufferReset {
e.shiftOffsets()
}
// This check isn't in the Snappy implementation, but there, the caller
// instead of the callee handles this case.
if len(src) < minNonLiteralBlockSize {
e.cur += maxStoreBlockSize
e.prev = e.prev[:0]
return emitLiteral(dst, src)
}
// sLimit is when to stop looking for offset/length copies. The inputMargin
// lets us use a fast path for emitLiteral in the main loop, while we are
// looking for copies.
sLimit := int32(len(src) - inputMargin)
// nextEmit is where in src the next emitLiteral should start from.
nextEmit := int32(0)
s := int32(0)
cv := load32(src, s)
nextHash := hash(cv)
for {
// Copied from the C++ snappy implementation:
//
// Heuristic match skipping: If 32 bytes are scanned with no matches
// found, start looking only at every other byte. If 32 more bytes are
// scanned (or skipped), look at every third byte, etc.. When a match
// is found, immediately go back to looking at every byte. This is a
// small loss (~5% performance, ~0.1% density) for compressible data
// due to more bookkeeping, but for non-compressible data (such as
// JPEG) it's a huge win since the compressor quickly "realizes" the
// data is incompressible and doesn't bother looking for matches
// everywhere.
//
// The "skip" variable keeps track of how many bytes there are since
// the last match; dividing it by 32 (ie. right-shifting by five) gives
// the number of bytes to move ahead for each iteration.
skip := int32(32)
nextS := s
var candidate tableEntry
for {
s = nextS
bytesBetweenHashLookups := skip >> 5
nextS = s + bytesBetweenHashLookups
skip += bytesBetweenHashLookups
if nextS > sLimit {
goto emitRemainder
}
candidate = e.table[nextHash&tableMask]
now := load32(src, nextS)
e.table[nextHash&tableMask] = tableEntry{offset: s + e.cur, val: cv}
nextHash = hash(now)
offset := s - (candidate.offset - e.cur)
if offset > maxMatchOffset || cv != candidate.val {
// Out of range or not matched.
cv = now
continue
}
break
}
// A 4-byte match has been found. We'll later see if more than 4 bytes
// match. But, prior to the match, src[nextEmit:s] are unmatched. Emit
// them as literal bytes.
dst = emitLiteral(dst, src[nextEmit:s])
// Call emitCopy, and then see if another emitCopy could be our next
// move. Repeat until we find no match for the input immediately after
// what was consumed by the last emitCopy call.
//
// If we exit this loop normally then we need to call emitLiteral next,
// though we don't yet know how big the literal will be. We handle that
// by proceeding to the next iteration of the main loop. We also can
// exit this loop via goto if we get close to exhausting the input.
for {
// Invariant: we have a 4-byte match at s, and no need to emit any
// literal bytes prior to s.
// Extend the 4-byte match as long as possible.
//
s += 4
t := candidate.offset - e.cur + 4
l := e.matchLen(s, t, src)
// matchToken is flate's equivalent of Snappy's emitCopy. (length,offset)
dst = append(dst, matchToken(uint32(l+4-baseMatchLength), uint32(s-t-baseMatchOffset)))
s += l
nextEmit = s
if s >= sLimit {
goto emitRemainder
}
// We could immediately start working at s now, but to improve
// compression we first update the hash table at s-1 and at s. If
// another emitCopy is not our next move, also calculate nextHash
// at s+1. At least on GOARCH=amd64, these three hash calculations
// are faster as one load64 call (with some shifts) instead of
// three load32 calls.
x := load64(src, s-1)
prevHash := hash(uint32(x))
e.table[prevHash&tableMask] = tableEntry{offset: e.cur + s - 1, val: uint32(x)}
x >>= 8
currHash := hash(uint32(x))
candidate = e.table[currHash&tableMask]
e.table[currHash&tableMask] = tableEntry{offset: e.cur + s, val: uint32(x)}
offset := s - (candidate.offset - e.cur)
if offset > maxMatchOffset || uint32(x) != candidate.val {
cv = uint32(x >> 8)
nextHash = hash(cv)
s++
break
}
}
}
emitRemainder:
if int(nextEmit) < len(src) {
dst = emitLiteral(dst, src[nextEmit:])
}
e.cur += int32(len(src))
e.prev = e.prev[:len(src)]
copy(e.prev, src)
return dst
}
func emitLiteral(dst []token, lit []byte) []token {
for _, v := range lit {
dst = append(dst, literalToken(uint32(v)))
}
return dst
}
// matchLen returns the match length between src[s:] and src[t:].
// t can be negative to indicate the match is starting in e.prev.
// We assume that src[s-4:s] and src[t-4:t] already match.
func (e *deflateFast) matchLen(s, t int32, src []byte) int32 {
s1 := int(s) + maxMatchLength - 4
if s1 > len(src) {
s1 = len(src)
}
// If we are inside the current block
if t >= 0 {
b := src[t:]
a := src[s:s1]
b = b[:len(a)]
// Extend the match to be as long as possible.
for i := range a {
if a[i] != b[i] {
return int32(i)
}
}
return int32(len(a))
}
// We found a match in the previous block.
tp := int32(len(e.prev)) + t
if tp < 0 {
return 0
}
// Extend the match to be as long as possible.
a := src[s:s1]
b := e.prev[tp:]
if len(b) > len(a) {
b = b[:len(a)]
}
a = a[:len(b)]
for i := range b {
if a[i] != b[i] {
return int32(i)
}
}
// If we reached our limit, we matched everything we are
// allowed to in the previous block and we return.
n := int32(len(b))
if int(s+n) == s1 {
return n
}
// Continue looking for more matches in the current block.
a = src[s+n : s1]
b = src[:len(a)]
for i := range a {
if a[i] != b[i] {
return int32(i) + n
}
}
return int32(len(a)) + n
}
// Reset resets the encoding history.
// This ensures that no matches are made to the previous block.
func (e *deflateFast) reset() {
e.prev = e.prev[:0]
// Bump the offset, so all matches will fail distance check.
// Nothing should be >= e.cur in the table.
e.cur += maxMatchOffset
// Protect against e.cur wraparound.
if e.cur >= bufferReset {
e.shiftOffsets()
}
}
// shiftOffsets will shift down all match offset.
// This is only called in rare situations to prevent integer overflow.
//
// See https://golang.org/issue/18636 and https://github.com/golang/go/issues/34121.
func (e *deflateFast) shiftOffsets() {
if len(e.prev) == 0 {
// We have no history; just clear the table.
clear(e.table[:])
e.cur = maxMatchOffset + 1
return
}
// Shift down everything in the table that isn't already too far away.
for i := range e.table[:] {
v := e.table[i].offset - e.cur + maxMatchOffset + 1
if v < 0 {
// We want to reset e.cur to maxMatchOffset + 1, so we need to shift
// all table entries down by (e.cur - (maxMatchOffset + 1)).
// Because we ignore matches > maxMatchOffset, we can cap
// any negative offsets at 0.
v = 0
}
e.table[i].offset = v
}
e.cur = maxMatchOffset + 1
}
+20
View File
@@ -0,0 +1,20 @@
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package go126flate contains the DEFLATE writer from Go 1.26.5.
//
// The sources are copied from Go commit
// c19862e5f8415b4f24b189d065ed739517c548ba. Go 1.27 changed the encoded
// output while improving compression speed in CL 707355 for golang/go#75532.
// BuildKit keeps this writer to preserve the digests of gzip-compressed layers.
package go126flate
// maxNumLit comes from RFC 1951 section 3.2.7. It normally lives in the Go
// flate decoder, which is intentionally not included in this writer-only copy.
const maxNumLit = 286
// InternalError reports an error in the flate code itself.
type InternalError string
func (e InternalError) Error() string { return "flate: internal error: " + string(e) }
@@ -0,0 +1,695 @@
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Code generated from Go 1.26.5 commit c19862e5f8415b4f24b189d065ed739517c548ba; DO NOT EDIT.
package go126flate
import (
"io"
)
const (
// The largest offset code.
offsetCodeCount = 30
// The special code used to mark the end of a block.
endBlockMarker = 256
// The first length code.
lengthCodesStart = 257
// The number of codegen codes.
codegenCodeCount = 19
badCode = 255
// bufferFlushSize indicates the buffer size
// after which bytes are flushed to the writer.
// Should preferably be a multiple of 6, since
// we accumulate 6 bytes between writes to the buffer.
bufferFlushSize = 240
// bufferSize is the actual output byte buffer size.
// It must have additional headroom for a flush
// which can contain up to 8 bytes.
bufferSize = bufferFlushSize + 8
)
// The number of extra bits needed by length code X - LENGTH_CODES_START.
var lengthExtraBits = []int8{
/* 257 */ 0, 0, 0,
/* 260 */ 0, 0, 0, 0, 0, 1, 1, 1, 1, 2,
/* 270 */ 2, 2, 2, 3, 3, 3, 3, 4, 4, 4,
/* 280 */ 4, 5, 5, 5, 5, 0,
}
// The length indicated by length code X - LENGTH_CODES_START.
var lengthBase = []uint32{
0, 1, 2, 3, 4, 5, 6, 7, 8, 10,
12, 14, 16, 20, 24, 28, 32, 40, 48, 56,
64, 80, 96, 112, 128, 160, 192, 224, 255,
}
// offset code word extra bits.
var offsetExtraBits = []int8{
0, 0, 0, 0, 1, 1, 2, 2, 3, 3,
4, 4, 5, 5, 6, 6, 7, 7, 8, 8,
9, 9, 10, 10, 11, 11, 12, 12, 13, 13,
}
var offsetBase = []uint32{
0x000000, 0x000001, 0x000002, 0x000003, 0x000004,
0x000006, 0x000008, 0x00000c, 0x000010, 0x000018,
0x000020, 0x000030, 0x000040, 0x000060, 0x000080,
0x0000c0, 0x000100, 0x000180, 0x000200, 0x000300,
0x000400, 0x000600, 0x000800, 0x000c00, 0x001000,
0x001800, 0x002000, 0x003000, 0x004000, 0x006000,
}
// The odd order in which the codegen code sizes are written.
var codegenOrder = []uint32{16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15}
type huffmanBitWriter struct {
// writer is the underlying writer.
// Do not use it directly; use the write method, which ensures
// that Write errors are sticky.
writer io.Writer
// Data waiting to be written is bytes[0:nbytes]
// and then the low nbits of bits. Data is always written
// sequentially into the bytes array.
bits uint64
nbits uint
bytes [bufferSize]byte
codegenFreq [codegenCodeCount]int32
nbytes int
literalFreq []int32
offsetFreq []int32
codegen []uint8
literalEncoding *huffmanEncoder
offsetEncoding *huffmanEncoder
codegenEncoding *huffmanEncoder
err error
}
func newHuffmanBitWriter(w io.Writer) *huffmanBitWriter {
return &huffmanBitWriter{
writer: w,
literalFreq: make([]int32, maxNumLit),
offsetFreq: make([]int32, offsetCodeCount),
codegen: make([]uint8, maxNumLit+offsetCodeCount+1),
literalEncoding: newHuffmanEncoder(maxNumLit),
codegenEncoding: newHuffmanEncoder(codegenCodeCount),
offsetEncoding: newHuffmanEncoder(offsetCodeCount),
}
}
func (w *huffmanBitWriter) reset(writer io.Writer) {
w.writer = writer
w.bits, w.nbits, w.nbytes, w.err = 0, 0, 0, nil
}
func (w *huffmanBitWriter) flush() {
if w.err != nil {
w.nbits = 0
return
}
n := w.nbytes
for w.nbits != 0 {
w.bytes[n] = byte(w.bits)
w.bits >>= 8
if w.nbits > 8 { // Avoid underflow
w.nbits -= 8
} else {
w.nbits = 0
}
n++
}
w.bits = 0
w.write(w.bytes[:n])
w.nbytes = 0
}
func (w *huffmanBitWriter) write(b []byte) {
if w.err != nil {
return
}
_, w.err = w.writer.Write(b)
}
func (w *huffmanBitWriter) writeBits(b int32, nb uint) {
if w.err != nil {
return
}
w.bits |= uint64(b) << w.nbits
w.nbits += nb
if w.nbits >= 48 {
bits := w.bits
w.bits >>= 48
w.nbits -= 48
n := w.nbytes
bytes := w.bytes[n : n+6]
bytes[0] = byte(bits)
bytes[1] = byte(bits >> 8)
bytes[2] = byte(bits >> 16)
bytes[3] = byte(bits >> 24)
bytes[4] = byte(bits >> 32)
bytes[5] = byte(bits >> 40)
n += 6
if n >= bufferFlushSize {
w.write(w.bytes[:n])
n = 0
}
w.nbytes = n
}
}
func (w *huffmanBitWriter) writeBytes(bytes []byte) {
if w.err != nil {
return
}
n := w.nbytes
if w.nbits&7 != 0 {
w.err = InternalError("writeBytes with unfinished bits")
return
}
for w.nbits != 0 {
w.bytes[n] = byte(w.bits)
w.bits >>= 8
w.nbits -= 8
n++
}
if n != 0 {
w.write(w.bytes[:n])
}
w.nbytes = 0
w.write(bytes)
}
// RFC 1951 3.2.7 specifies a special run-length encoding for specifying
// the literal and offset lengths arrays (which are concatenated into a single
// array). This method generates that run-length encoding.
//
// The result is written into the codegen array, and the frequencies
// of each code is written into the codegenFreq array.
// Codes 0-15 are single byte codes. Codes 16-18 are followed by additional
// information. Code badCode is an end marker
//
// numLiterals The number of literals in literalEncoding
// numOffsets The number of offsets in offsetEncoding
// litenc, offenc The literal and offset encoder to use
func (w *huffmanBitWriter) generateCodegen(numLiterals int, numOffsets int, litEnc, offEnc *huffmanEncoder) {
clear(w.codegenFreq[:])
// Note that we are using codegen both as a temporary variable for holding
// a copy of the frequencies, and as the place where we put the result.
// This is fine because the output is always shorter than the input used
// so far.
codegen := w.codegen // cache
// Copy the concatenated code sizes to codegen. Put a marker at the end.
cgnl := codegen[:numLiterals]
for i := range cgnl {
cgnl[i] = uint8(litEnc.codes[i].len)
}
cgnl = codegen[numLiterals : numLiterals+numOffsets]
for i := range cgnl {
cgnl[i] = uint8(offEnc.codes[i].len)
}
codegen[numLiterals+numOffsets] = badCode
size := codegen[0]
count := 1
outIndex := 0
for inIndex := 1; size != badCode; inIndex++ {
// INVARIANT: We have seen "count" copies of size that have not yet
// had output generated for them.
nextSize := codegen[inIndex]
if nextSize == size {
count++
continue
}
// We need to generate codegen indicating "count" of size.
if size != 0 {
codegen[outIndex] = size
outIndex++
w.codegenFreq[size]++
count--
for count >= 3 {
n := 6
if n > count {
n = count
}
codegen[outIndex] = 16
outIndex++
codegen[outIndex] = uint8(n - 3)
outIndex++
w.codegenFreq[16]++
count -= n
}
} else {
for count >= 11 {
n := 138
if n > count {
n = count
}
codegen[outIndex] = 18
outIndex++
codegen[outIndex] = uint8(n - 11)
outIndex++
w.codegenFreq[18]++
count -= n
}
if count >= 3 {
// count >= 3 && count <= 10
codegen[outIndex] = 17
outIndex++
codegen[outIndex] = uint8(count - 3)
outIndex++
w.codegenFreq[17]++
count = 0
}
}
count--
for ; count >= 0; count-- {
codegen[outIndex] = size
outIndex++
w.codegenFreq[size]++
}
// Set up invariant for next time through the loop.
size = nextSize
count = 1
}
// Marker indicating the end of the codegen.
codegen[outIndex] = badCode
}
// dynamicSize returns the size of dynamically encoded data in bits.
func (w *huffmanBitWriter) dynamicSize(litEnc, offEnc *huffmanEncoder, extraBits int) (size, numCodegens int) {
numCodegens = len(w.codegenFreq)
for numCodegens > 4 && w.codegenFreq[codegenOrder[numCodegens-1]] == 0 {
numCodegens--
}
header := 3 + 5 + 5 + 4 + (3 * numCodegens) +
w.codegenEncoding.bitLength(w.codegenFreq[:]) +
int(w.codegenFreq[16])*2 +
int(w.codegenFreq[17])*3 +
int(w.codegenFreq[18])*7
size = header +
litEnc.bitLength(w.literalFreq) +
offEnc.bitLength(w.offsetFreq) +
extraBits
return size, numCodegens
}
// fixedSize returns the size of dynamically encoded data in bits.
func (w *huffmanBitWriter) fixedSize(extraBits int) int {
return 3 +
fixedLiteralEncoding.bitLength(w.literalFreq) +
fixedOffsetEncoding.bitLength(w.offsetFreq) +
extraBits
}
// storedSize calculates the stored size, including header.
// The function returns the size in bits and whether the block
// fits inside a single block.
func (w *huffmanBitWriter) storedSize(in []byte) (int, bool) {
if in == nil {
return 0, false
}
if len(in) <= maxStoreBlockSize {
return (len(in) + 5) * 8, true
}
return 0, false
}
func (w *huffmanBitWriter) writeCode(c hcode) {
if w.err != nil {
return
}
w.bits |= uint64(c.code) << w.nbits
w.nbits += uint(c.len)
if w.nbits >= 48 {
bits := w.bits
w.bits >>= 48
w.nbits -= 48
n := w.nbytes
bytes := w.bytes[n : n+6]
bytes[0] = byte(bits)
bytes[1] = byte(bits >> 8)
bytes[2] = byte(bits >> 16)
bytes[3] = byte(bits >> 24)
bytes[4] = byte(bits >> 32)
bytes[5] = byte(bits >> 40)
n += 6
if n >= bufferFlushSize {
w.write(w.bytes[:n])
n = 0
}
w.nbytes = n
}
}
// Write the header of a dynamic Huffman block to the output stream.
//
// numLiterals The number of literals specified in codegen
// numOffsets The number of offsets specified in codegen
// numCodegens The number of codegens used in codegen
func (w *huffmanBitWriter) writeDynamicHeader(numLiterals int, numOffsets int, numCodegens int, isEof bool) {
if w.err != nil {
return
}
var firstBits int32 = 4
if isEof {
firstBits = 5
}
w.writeBits(firstBits, 3)
w.writeBits(int32(numLiterals-257), 5)
w.writeBits(int32(numOffsets-1), 5)
w.writeBits(int32(numCodegens-4), 4)
for i := 0; i < numCodegens; i++ {
value := uint(w.codegenEncoding.codes[codegenOrder[i]].len)
w.writeBits(int32(value), 3)
}
i := 0
for {
var codeWord int = int(w.codegen[i])
i++
if codeWord == badCode {
break
}
w.writeCode(w.codegenEncoding.codes[uint32(codeWord)])
switch codeWord {
case 16:
w.writeBits(int32(w.codegen[i]), 2)
i++
case 17:
w.writeBits(int32(w.codegen[i]), 3)
i++
case 18:
w.writeBits(int32(w.codegen[i]), 7)
i++
}
}
}
func (w *huffmanBitWriter) writeStoredHeader(length int, isEof bool) {
if w.err != nil {
return
}
var flag int32
if isEof {
flag = 1
}
w.writeBits(flag, 3)
w.flush()
w.writeBits(int32(length), 16)
w.writeBits(int32(^uint16(length)), 16)
}
func (w *huffmanBitWriter) writeFixedHeader(isEof bool) {
if w.err != nil {
return
}
// Indicate that we are a fixed Huffman block
var value int32 = 2
if isEof {
value = 3
}
w.writeBits(value, 3)
}
// writeBlock will write a block of tokens with the smallest encoding.
// The original input can be supplied, and if the huffman encoded data
// is larger than the original bytes, the data will be written as a
// stored block.
// If the input is nil, the tokens will always be Huffman encoded.
func (w *huffmanBitWriter) writeBlock(tokens []token, eof bool, input []byte) {
if w.err != nil {
return
}
tokens = append(tokens, endBlockMarker)
numLiterals, numOffsets := w.indexTokens(tokens)
var extraBits int
storedSize, storable := w.storedSize(input)
if storable {
// We only bother calculating the costs of the extra bits required by
// the length of offset fields (which will be the same for both fixed
// and dynamic encoding), if we need to compare those two encodings
// against stored encoding.
for lengthCode := lengthCodesStart + 8; lengthCode < numLiterals; lengthCode++ {
// First eight length codes have extra size = 0.
extraBits += int(w.literalFreq[lengthCode]) * int(lengthExtraBits[lengthCode-lengthCodesStart])
}
for offsetCode := 4; offsetCode < numOffsets; offsetCode++ {
// First four offset codes have extra size = 0.
extraBits += int(w.offsetFreq[offsetCode]) * int(offsetExtraBits[offsetCode])
}
}
// Figure out smallest code.
// Fixed Huffman baseline.
var literalEncoding = fixedLiteralEncoding
var offsetEncoding = fixedOffsetEncoding
var size = w.fixedSize(extraBits)
// Dynamic Huffman?
var numCodegens int
// Generate codegen and codegenFrequencies, which indicates how to encode
// the literalEncoding and the offsetEncoding.
w.generateCodegen(numLiterals, numOffsets, w.literalEncoding, w.offsetEncoding)
w.codegenEncoding.generate(w.codegenFreq[:], 7)
dynamicSize, numCodegens := w.dynamicSize(w.literalEncoding, w.offsetEncoding, extraBits)
if dynamicSize < size {
size = dynamicSize
literalEncoding = w.literalEncoding
offsetEncoding = w.offsetEncoding
}
// Stored bytes?
if storable && storedSize < size {
w.writeStoredHeader(len(input), eof)
w.writeBytes(input)
return
}
// Huffman.
if literalEncoding == fixedLiteralEncoding {
w.writeFixedHeader(eof)
} else {
w.writeDynamicHeader(numLiterals, numOffsets, numCodegens, eof)
}
// Write the tokens.
w.writeTokens(tokens, literalEncoding.codes, offsetEncoding.codes)
}
// writeBlockDynamic encodes a block using a dynamic Huffman table.
// This should be used if the symbols used have a disproportionate
// histogram distribution.
// If input is supplied and the compression savings are below 1/16th of the
// input size the block is stored.
func (w *huffmanBitWriter) writeBlockDynamic(tokens []token, eof bool, input []byte) {
if w.err != nil {
return
}
tokens = append(tokens, endBlockMarker)
numLiterals, numOffsets := w.indexTokens(tokens)
// Generate codegen and codegenFrequencies, which indicates how to encode
// the literalEncoding and the offsetEncoding.
w.generateCodegen(numLiterals, numOffsets, w.literalEncoding, w.offsetEncoding)
w.codegenEncoding.generate(w.codegenFreq[:], 7)
size, numCodegens := w.dynamicSize(w.literalEncoding, w.offsetEncoding, 0)
// Store bytes, if we don't get a reasonable improvement.
if ssize, storable := w.storedSize(input); storable && ssize < (size+size>>4) {
w.writeStoredHeader(len(input), eof)
w.writeBytes(input)
return
}
// Write Huffman table.
w.writeDynamicHeader(numLiterals, numOffsets, numCodegens, eof)
// Write the tokens.
w.writeTokens(tokens, w.literalEncoding.codes, w.offsetEncoding.codes)
}
// indexTokens indexes a slice of tokens, and updates
// literalFreq and offsetFreq, and generates literalEncoding
// and offsetEncoding.
// The number of literal and offset tokens is returned.
func (w *huffmanBitWriter) indexTokens(tokens []token) (numLiterals, numOffsets int) {
clear(w.literalFreq)
clear(w.offsetFreq)
for _, t := range tokens {
if t < matchType {
w.literalFreq[t.literal()]++
continue
}
length := t.length()
offset := t.offset()
w.literalFreq[lengthCodesStart+lengthCode(length)]++
w.offsetFreq[offsetCode(offset)]++
}
// get the number of literals
numLiterals = len(w.literalFreq)
for w.literalFreq[numLiterals-1] == 0 {
numLiterals--
}
// get the number of offsets
numOffsets = len(w.offsetFreq)
for numOffsets > 0 && w.offsetFreq[numOffsets-1] == 0 {
numOffsets--
}
if numOffsets == 0 {
// We haven't found a single match. If we want to go with the dynamic encoding,
// we should count at least one offset to be sure that the offset huffman tree could be encoded.
w.offsetFreq[0] = 1
numOffsets = 1
}
w.literalEncoding.generate(w.literalFreq, 15)
w.offsetEncoding.generate(w.offsetFreq, 15)
return
}
// writeTokens writes a slice of tokens to the output.
// codes for literal and offset encoding must be supplied.
func (w *huffmanBitWriter) writeTokens(tokens []token, leCodes, oeCodes []hcode) {
if w.err != nil {
return
}
for _, t := range tokens {
if t < matchType {
w.writeCode(leCodes[t.literal()])
continue
}
// Write the length
length := t.length()
lengthCode := lengthCode(length)
w.writeCode(leCodes[lengthCode+lengthCodesStart])
extraLengthBits := uint(lengthExtraBits[lengthCode])
if extraLengthBits > 0 {
extraLength := int32(length - lengthBase[lengthCode])
w.writeBits(extraLength, extraLengthBits)
}
// Write the offset
offset := t.offset()
offsetCode := offsetCode(offset)
w.writeCode(oeCodes[offsetCode])
extraOffsetBits := uint(offsetExtraBits[offsetCode])
if extraOffsetBits > 0 {
extraOffset := int32(offset - offsetBase[offsetCode])
w.writeBits(extraOffset, extraOffsetBits)
}
}
}
// huffOffset is a static offset encoder used for huffman only encoding.
// It can be reused since we will not be encoding offset values.
var huffOffset *huffmanEncoder
func init() {
offsetFreq := make([]int32, offsetCodeCount)
offsetFreq[0] = 1
huffOffset = newHuffmanEncoder(offsetCodeCount)
huffOffset.generate(offsetFreq, 15)
}
// writeBlockHuff encodes a block of bytes as either
// Huffman encoded literals or uncompressed bytes if the
// results only gains very little from compression.
func (w *huffmanBitWriter) writeBlockHuff(eof bool, input []byte) {
if w.err != nil {
return
}
// Clear histogram
clear(w.literalFreq)
// Add everything as literals
histogram(input, w.literalFreq)
w.literalFreq[endBlockMarker] = 1
const numLiterals = endBlockMarker + 1
w.offsetFreq[0] = 1
const numOffsets = 1
w.literalEncoding.generate(w.literalFreq, 15)
// Figure out smallest code.
// Always use dynamic Huffman or Store
var numCodegens int
// Generate codegen and codegenFrequencies, which indicates how to encode
// the literalEncoding and the offsetEncoding.
w.generateCodegen(numLiterals, numOffsets, w.literalEncoding, huffOffset)
w.codegenEncoding.generate(w.codegenFreq[:], 7)
size, numCodegens := w.dynamicSize(w.literalEncoding, huffOffset, 0)
// Store bytes, if we don't get a reasonable improvement.
if ssize, storable := w.storedSize(input); storable && ssize < (size+size>>4) {
w.writeStoredHeader(len(input), eof)
w.writeBytes(input)
return
}
// Huffman.
w.writeDynamicHeader(numLiterals, numOffsets, numCodegens, eof)
encoding := w.literalEncoding.codes[:257]
n := w.nbytes
for _, t := range input {
// Bitwriting inlined, ~30% speedup
c := encoding[t]
w.bits |= uint64(c.code) << w.nbits
w.nbits += uint(c.len)
if w.nbits < 48 {
continue
}
// Store 6 bytes
bits := w.bits
w.bits >>= 48
w.nbits -= 48
bytes := w.bytes[n : n+6]
bytes[0] = byte(bits)
bytes[1] = byte(bits >> 8)
bytes[2] = byte(bits >> 16)
bytes[3] = byte(bits >> 24)
bytes[4] = byte(bits >> 32)
bytes[5] = byte(bits >> 40)
n += 6
if n < bufferFlushSize {
continue
}
w.write(w.bytes[:n])
if w.err != nil {
return // Return early in the event of write failures
}
n = 0
}
w.nbytes = n
w.writeCode(encoding[endBlockMarker])
}
// histogram accumulates a histogram of b in h.
//
// len(h) must be >= 256, and h's elements must be all zeroes.
func histogram(b []byte, h []int32) {
h = h[:256]
for _, t := range b {
h[t]++
}
}
@@ -0,0 +1,347 @@
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Code generated from Go 1.26.5 commit c19862e5f8415b4f24b189d065ed739517c548ba; DO NOT EDIT.
package go126flate
import (
"math"
"math/bits"
"sort"
)
// hcode is a huffman code with a bit code and bit length.
type hcode struct {
code, len uint16
}
type huffmanEncoder struct {
codes []hcode
freqcache []literalNode
bitCount [17]int32
lns byLiteral // stored to avoid repeated allocation in generate
lfs byFreq // stored to avoid repeated allocation in generate
}
type literalNode struct {
literal uint16
freq int32
}
// A levelInfo describes the state of the constructed tree for a given depth.
type levelInfo struct {
// Our level. for better printing
level int32
// The frequency of the last node at this level
lastFreq int32
// The frequency of the next character to add to this level
nextCharFreq int32
// The frequency of the next pair (from level below) to add to this level.
// Only valid if the "needed" value of the next lower level is 0.
nextPairFreq int32
// The number of chains remaining to generate for this level before moving
// up to the next level
needed int32
}
// set sets the code and length of an hcode.
func (h *hcode) set(code uint16, length uint16) {
h.len = length
h.code = code
}
func maxNode() literalNode { return literalNode{math.MaxUint16, math.MaxInt32} }
func newHuffmanEncoder(size int) *huffmanEncoder {
return &huffmanEncoder{codes: make([]hcode, size)}
}
// Generates a HuffmanCode corresponding to the fixed literal table.
func generateFixedLiteralEncoding() *huffmanEncoder {
h := newHuffmanEncoder(maxNumLit)
codes := h.codes
var ch uint16
for ch = 0; ch < maxNumLit; ch++ {
var bits uint16
var size uint16
switch {
case ch < 144:
// size 8, 000110000 .. 10111111
bits = ch + 48
size = 8
case ch < 256:
// size 9, 110010000 .. 111111111
bits = ch + 400 - 144
size = 9
case ch < 280:
// size 7, 0000000 .. 0010111
bits = ch - 256
size = 7
default:
// size 8, 11000000 .. 11000111
bits = ch + 192 - 280
size = 8
}
codes[ch] = hcode{code: reverseBits(bits, byte(size)), len: size}
}
return h
}
func generateFixedOffsetEncoding() *huffmanEncoder {
h := newHuffmanEncoder(30)
codes := h.codes
for ch := range codes {
codes[ch] = hcode{code: reverseBits(uint16(ch), 5), len: 5}
}
return h
}
var fixedLiteralEncoding *huffmanEncoder = generateFixedLiteralEncoding()
var fixedOffsetEncoding *huffmanEncoder = generateFixedOffsetEncoding()
func (h *huffmanEncoder) bitLength(freq []int32) int {
var total int
for i, f := range freq {
if f != 0 {
total += int(f) * int(h.codes[i].len)
}
}
return total
}
const maxBitsLimit = 16
// bitCounts computes the number of literals assigned to each bit size in the Huffman encoding.
// It is only called when list.length >= 3.
// The cases of 0, 1, and 2 literals are handled by special case code.
//
// list is an array of the literals with non-zero frequencies
// and their associated frequencies. The array is in order of increasing
// frequency and has as its last element a special element with frequency
// MaxInt32.
//
// maxBits is the maximum number of bits that should be used to encode any literal.
// It must be less than 16.
//
// bitCounts returns an integer slice in which slice[i] indicates the number of literals
// that should be encoded in i bits.
func (h *huffmanEncoder) bitCounts(list []literalNode, maxBits int32) []int32 {
if maxBits >= maxBitsLimit {
panic("flate: maxBits too large")
}
n := int32(len(list))
list = list[0 : n+1]
list[n] = maxNode()
// The tree can't have greater depth than n - 1, no matter what. This
// saves a little bit of work in some small cases
if maxBits > n-1 {
maxBits = n - 1
}
// Create information about each of the levels.
// A bogus "Level 0" whose sole purpose is so that
// level1.prev.needed==0. This makes level1.nextPairFreq
// be a legitimate value that never gets chosen.
var levels [maxBitsLimit]levelInfo
// leafCounts[i] counts the number of literals at the left
// of ancestors of the rightmost node at level i.
// leafCounts[i][j] is the number of literals at the left
// of the level j ancestor.
var leafCounts [maxBitsLimit][maxBitsLimit]int32
for level := int32(1); level <= maxBits; level++ {
// For every level, the first two items are the first two characters.
// We initialize the levels as if we had already figured this out.
levels[level] = levelInfo{
level: level,
lastFreq: list[1].freq,
nextCharFreq: list[2].freq,
nextPairFreq: list[0].freq + list[1].freq,
}
leafCounts[level][level] = 2
if level == 1 {
levels[level].nextPairFreq = math.MaxInt32
}
}
// We need a total of 2*n - 2 items at top level and have already generated 2.
levels[maxBits].needed = 2*n - 4
level := maxBits
for {
l := &levels[level]
if l.nextPairFreq == math.MaxInt32 && l.nextCharFreq == math.MaxInt32 {
// We've run out of both leaves and pairs.
// End all calculations for this level.
// To make sure we never come back to this level or any lower level,
// set nextPairFreq impossibly large.
l.needed = 0
levels[level+1].nextPairFreq = math.MaxInt32
level++
continue
}
prevFreq := l.lastFreq
if l.nextCharFreq < l.nextPairFreq {
// The next item on this row is a leaf node.
n := leafCounts[level][level] + 1
l.lastFreq = l.nextCharFreq
// Lower leafCounts are the same of the previous node.
leafCounts[level][level] = n
l.nextCharFreq = list[n].freq
} else {
// The next item on this row is a pair from the previous row.
// nextPairFreq isn't valid until we generate two
// more values in the level below
l.lastFreq = l.nextPairFreq
// Take leaf counts from the lower level, except counts[level] remains the same.
copy(leafCounts[level][:level], leafCounts[level-1][:level])
levels[l.level-1].needed = 2
}
if l.needed--; l.needed == 0 {
// We've done everything we need to do for this level.
// Continue calculating one level up. Fill in nextPairFreq
// of that level with the sum of the two nodes we've just calculated on
// this level.
if l.level == maxBits {
// All done!
break
}
levels[l.level+1].nextPairFreq = prevFreq + l.lastFreq
level++
} else {
// If we stole from below, move down temporarily to replenish it.
for levels[level-1].needed > 0 {
level--
}
}
}
// Somethings is wrong if at the end, the top level is null or hasn't used
// all of the leaves.
if leafCounts[maxBits][maxBits] != n {
panic("leafCounts[maxBits][maxBits] != n")
}
bitCount := h.bitCount[:maxBits+1]
bits := 1
counts := &leafCounts[maxBits]
for level := maxBits; level > 0; level-- {
// chain.leafCount gives the number of literals requiring at least "bits"
// bits to encode.
bitCount[bits] = counts[level] - counts[level-1]
bits++
}
return bitCount
}
// Look at the leaves and assign them a bit count and an encoding as specified
// in RFC 1951 3.2.2
func (h *huffmanEncoder) assignEncodingAndSize(bitCount []int32, list []literalNode) {
code := uint16(0)
for n, bits := range bitCount {
code <<= 1
if n == 0 || bits == 0 {
continue
}
// The literals list[len(list)-bits] .. list[len(list)-bits]
// are encoded using "bits" bits, and get the values
// code, code + 1, .... The code values are
// assigned in literal order (not frequency order).
chunk := list[len(list)-int(bits):]
h.lns.sort(chunk)
for _, node := range chunk {
h.codes[node.literal] = hcode{code: reverseBits(code, uint8(n)), len: uint16(n)}
code++
}
list = list[0 : len(list)-int(bits)]
}
}
// Update this Huffman Code object to be the minimum code for the specified frequency count.
//
// freq is an array of frequencies, in which freq[i] gives the frequency of literal i.
// maxBits The maximum number of bits to use for any literal.
func (h *huffmanEncoder) generate(freq []int32, maxBits int32) {
if h.freqcache == nil {
// Allocate a reusable buffer with the longest possible frequency table.
// Possible lengths are codegenCodeCount, offsetCodeCount and maxNumLit.
// The largest of these is maxNumLit, so we allocate for that case.
h.freqcache = make([]literalNode, maxNumLit+1)
}
list := h.freqcache[:len(freq)+1]
// Number of non-zero literals
count := 0
// Set list to be the set of all non-zero literals and their frequencies
for i, f := range freq {
if f != 0 {
list[count] = literalNode{uint16(i), f}
count++
} else {
h.codes[i].len = 0
}
}
list = list[:count]
if count <= 2 {
// Handle the small cases here, because they are awkward for the general case code. With
// two or fewer literals, everything has bit length 1.
for i, node := range list {
// "list" is in order of increasing literal value.
h.codes[node.literal].set(uint16(i), 1)
}
return
}
h.lfs.sort(list)
// Get the number of literals for each bit count
bitCount := h.bitCounts(list, maxBits)
// And do the assignment
h.assignEncodingAndSize(bitCount, list)
}
type byLiteral []literalNode
func (s *byLiteral) sort(a []literalNode) {
*s = byLiteral(a)
sort.Sort(s)
}
func (s byLiteral) Len() int { return len(s) }
func (s byLiteral) Less(i, j int) bool {
return s[i].literal < s[j].literal
}
func (s byLiteral) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
type byFreq []literalNode
func (s *byFreq) sort(a []literalNode) {
*s = byFreq(a)
sort.Sort(s)
}
func (s byFreq) Len() int { return len(s) }
func (s byFreq) Less(i, j int) bool {
if s[i].freq == s[j].freq {
return s[i].literal < s[j].literal
}
return s[i].freq < s[j].freq
}
func (s byFreq) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func reverseBits(number uint16, bitLength byte) uint16 {
return bits.Reverse16(number << (16 - bitLength))
}
+99
View File
@@ -0,0 +1,99 @@
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Code generated from Go 1.26.5 commit c19862e5f8415b4f24b189d065ed739517c548ba; DO NOT EDIT.
package go126flate
const (
// 2 bits: type 0 = literal 1=EOF 2=Match 3=Unused
// 8 bits: xlength = length - MIN_MATCH_LENGTH
// 22 bits xoffset = offset - MIN_OFFSET_SIZE, or literal
lengthShift = 22
offsetMask = 1<<lengthShift - 1
typeMask = 3 << 30
literalType = 0 << 30
matchType = 1 << 30
)
// The length code for length X (MIN_MATCH_LENGTH <= X <= MAX_MATCH_LENGTH)
// is lengthCodes[length - MIN_MATCH_LENGTH]
var lengthCodes = [...]uint32{
0, 1, 2, 3, 4, 5, 6, 7, 8, 8,
9, 9, 10, 10, 11, 11, 12, 12, 12, 12,
13, 13, 13, 13, 14, 14, 14, 14, 15, 15,
15, 15, 16, 16, 16, 16, 16, 16, 16, 16,
17, 17, 17, 17, 17, 17, 17, 17, 18, 18,
18, 18, 18, 18, 18, 18, 19, 19, 19, 19,
19, 19, 19, 19, 20, 20, 20, 20, 20, 20,
20, 20, 20, 20, 20, 20, 20, 20, 20, 20,
21, 21, 21, 21, 21, 21, 21, 21, 21, 21,
21, 21, 21, 21, 21, 21, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 24, 24,
24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
25, 25, 25, 25, 25, 25, 25, 25, 25, 25,
25, 25, 25, 25, 25, 25, 25, 25, 25, 25,
25, 25, 25, 25, 25, 25, 25, 25, 25, 25,
25, 25, 26, 26, 26, 26, 26, 26, 26, 26,
26, 26, 26, 26, 26, 26, 26, 26, 26, 26,
26, 26, 26, 26, 26, 26, 26, 26, 26, 26,
26, 26, 26, 26, 27, 27, 27, 27, 27, 27,
27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
27, 27, 27, 27, 27, 28,
}
var offsetCodes = [...]uint32{
0, 1, 2, 3, 4, 4, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7,
8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9,
10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,
11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,
13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,
14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
}
type token uint32
// Convert a literal into a literal token.
func literalToken(literal uint32) token { return token(literalType + literal) }
// Convert a < xlength, xoffset > pair into a match token.
func matchToken(xlength uint32, xoffset uint32) token {
return token(matchType + xlength<<lengthShift + xoffset)
}
// Returns the literal of a literal token.
func (t token) literal() uint32 { return uint32(t - literalType) }
// Returns the extra offset of a match token.
func (t token) offset() uint32 { return uint32(t) & offsetMask }
func (t token) length() uint32 { return uint32((t - matchType) >> lengthShift) }
func lengthCode(len uint32) uint32 { return lengthCodes[len] }
// Returns the offset code corresponding to a specific offset.
func offsetCode(off uint32) uint32 {
if off < uint32(len(offsetCodes)) {
return offsetCodes[off]
}
if off>>7 < uint32(len(offsetCodes)) {
return offsetCodes[off>>7] + 14
}
return offsetCodes[off>>14] + 28
}
+92
View File
@@ -0,0 +1,92 @@
package compression
import (
"encoding/binary"
"hash/crc32"
"io"
"github.com/moby/buildkit/util/compression/go126flate"
"github.com/pkg/errors"
)
const go126GzipDefaultCompression = go126flate.DefaultCompression
// go126GzipWriter writes the fixed gzip framing used for image layers around
// the Go 1.26 DEFLATE stream. Image layers do not use optional gzip headers.
type go126GzipWriter struct {
w io.Writer
level int
wroteHeader bool
closed bool
buf [10]byte
compressor *go126flate.Writer
digest uint32
size uint32
err error
}
func newGo126GzipWriterLevel(w io.Writer, level int) (io.WriteCloser, error) {
if level < go126flate.HuffmanOnly || level > go126flate.BestCompression {
return nil, errors.Errorf("gzip: invalid compression level: %d", level)
}
return &go126GzipWriter{w: w, level: level}, nil
}
func (z *go126GzipWriter) writeHeader() error {
z.wroteHeader = true
z.buf = [10]byte{0: 0x1f, 1: 0x8b, 2: 8, 9: 255}
switch z.level {
case go126flate.BestCompression:
z.buf[8] = 2
case go126flate.BestSpeed:
z.buf[8] = 4
}
if _, err := z.w.Write(z.buf[:10]); err != nil {
return err
}
compressor, err := go126flate.NewWriter(z.w, z.level)
if err != nil {
return errors.WithStack(err)
}
z.compressor = compressor
return nil
}
func (z *go126GzipWriter) Write(p []byte) (int, error) {
if z.err != nil {
return 0, z.err
}
if !z.wroteHeader {
z.err = z.writeHeader()
if z.err != nil {
return 0, z.err
}
}
z.size += uint32(len(p))
z.digest = crc32.Update(z.digest, crc32.IEEETable, p)
n, err := z.compressor.Write(p)
z.err = err
return n, err
}
func (z *go126GzipWriter) Close() error {
if z.err != nil {
return z.err
}
if z.closed {
return nil
}
z.closed = true
if !z.wroteHeader {
if z.err = z.writeHeader(); z.err != nil {
return z.err
}
}
if z.err = z.compressor.Close(); z.err != nil {
return z.err
}
binary.LittleEndian.PutUint32(z.buf[:4], z.digest)
binary.LittleEndian.PutUint32(z.buf[4:8], z.size)
_, z.err = z.w.Write(z.buf[:8])
return z.err
}
+5 -3
View File
@@ -1,7 +1,6 @@
package compression
import (
"compress/gzip"
"context"
"io"
@@ -57,10 +56,13 @@ func (c gzipType) String() string {
func gzipWriter(comp Config) func(io.Writer) (io.WriteCloser, error) {
return func(dest io.Writer) (io.WriteCloser, error) {
level := gzip.DefaultCompression
level := go126GzipDefaultCompression
if comp.Level != nil {
level = *comp.Level
}
return gzip.NewWriterLevel(dest, level)
// Go 1.27 changed compress/flate output in CL 707355 for
// golang/go#75532. Keep Go 1.26 encoding so existing compatibility
// versions continue to produce the same compressed layer digests.
return newGo126GzipWriterLevel(dest, level)
}
}
+74 -11
View File
@@ -7,8 +7,11 @@ import (
"os"
"os/exec"
"slices"
"strconv"
"strings"
"unicode/utf8"
"github.com/moby/buildkit/util/urlutil"
"github.com/pkg/errors"
)
@@ -18,10 +21,11 @@ type GitCLI struct {
git string
exec func(context.Context, *exec.Cmd) error
args []string
dir string
streams StreamFunc
advice *bool
args []string
dir string
streams StreamFunc
advice *bool
debugCommands bool
workTree string
gitDir string
@@ -62,6 +66,13 @@ func WithGitAdvice(enabled bool) Option {
}
}
// WithGitDebugCommands controls whether executed Git commands are logged.
func WithGitDebugCommands(debugCommands bool) Option {
return func(b *GitCLI) {
b.debugCommands = debugCommands
}
}
// WithDir sets working directory.
//
// This should be a path to any directory within a standard git repository.
@@ -190,22 +201,32 @@ func (cli *GitCLI) Run(ctx context.Context, args ...string) (_ []byte, err error
cmd.Stdin = nil
cmd.Stdout = buf
cmd.Stderr = errbuf
var stdoutStream io.WriteCloser
var stderrStream io.WriteCloser
flush := func() {}
if cli.streams != nil {
stdout, stderr, flush := cli.streams(ctx)
if stdout != nil {
cmd.Stdout = io.MultiWriter(stdout, cmd.Stdout)
var flushStream func()
stdoutStream, stderrStream, flushStream = cli.streams(ctx)
if flushStream != nil {
flush = flushStream
}
if stderr != nil {
cmd.Stderr = io.MultiWriter(stderr, cmd.Stderr)
if stdoutStream != nil {
cmd.Stdout = io.MultiWriter(stdoutStream, cmd.Stdout)
defer stdoutStream.Close()
}
if stderrStream != nil {
cmd.Stderr = io.MultiWriter(stderrStream, cmd.Stderr)
defer stderrStream.Close()
}
defer stdout.Close()
defer stderr.Close()
defer func() {
if err != nil {
flush()
}
}()
}
if cli.debugCommands && stderrStream != nil {
_, _ = io.WriteString(stderrStream, "+ "+formatDebugCommand(cmd.Args)+"\n")
}
cmd.Env = []string{
"PATH=" + os.Getenv("PATH"),
@@ -297,6 +318,48 @@ func (cli *GitCLI) Run(ctx context.Context, args ...string) (_ []byte, err error
}
}
func formatDebugCommand(args []string) string {
formatted := make([]string, 0, len(args))
for i := 0; i < len(args); i++ {
arg := args[i]
if arg == "-c" && i+1 < len(args) {
formatted = append(formatted, arg, quoteDebugCommandArg(redactGitConfig(args[i+1])))
i++
continue
}
formatted = append(formatted, quoteDebugCommandArg(redactURLCredentials(arg)))
}
return strings.Join(formatted, " ")
}
func redactGitConfig(arg string) string {
// URL-scoped keys can contain '=' before the header value separator.
for i := range arg {
if arg[i] == '=' && strings.HasSuffix(strings.ToLower(arg[:i]), ".extraheader") {
return redactURLCredentials(arg[:i]) + "=xxxxx"
}
}
return redactURLCredentials(arg)
}
func redactURLCredentials(arg string) string {
if !strings.Contains(arg, "://") {
return arg
}
return urlutil.RedactCredentials(arg)
}
// quoteDebugCommandArg preserves argument boundaries and escapes nonprintable
// characters for display. The output is not intended for shell re-execution.
func quoteDebugCommandArg(arg string) string {
if arg == "" || !utf8.ValidString(arg) || strings.ContainsFunc(arg, func(r rune) bool {
return !strconv.IsPrint(r) || strings.ContainsRune(" \"'\\", r)
}) {
return strconv.Quote(arg)
}
return arg
}
func getGitSSHCommand(knownHosts string) string {
gitSSHCommand := "ssh -F " + os.DevNull
if knownHosts != "" {
+4 -3
View File
@@ -6,6 +6,7 @@ import (
"maps"
"slices"
"sync"
"sync/atomic"
"time"
"github.com/pkg/errors"
@@ -217,13 +218,13 @@ func newWriter(pw *progressWriter) *progressWriter {
}
type progressWriter struct {
done bool
done atomic.Bool // set by Close, may race with writes from other goroutines
reader *progressReader
meta map[string]any
}
func (pw *progressWriter) Write(id string, v any) error {
if pw.done {
if pw.done.Load() {
return errors.Errorf("writing %s to closed progress writer", id)
}
return pw.writeRawProgress(&Progress{
@@ -251,7 +252,7 @@ func (pw *progressWriter) Close() error {
delete(pw.reader.writers, pw)
pw.reader.mu.Unlock()
pw.reader.cond.Broadcast()
pw.done = true
pw.done.Store(true)
return nil
}
+26 -11
View File
@@ -45,12 +45,13 @@ type Puller struct {
var _ content.Provider = &provider{}
type PulledManifests struct {
Ref string
MainManifestDesc ocispecs.Descriptor
ConfigDesc ocispecs.Descriptor
Nonlayers []ocispecs.Descriptor
Descriptors []ocispecs.Descriptor
Provider func(session.Group) content.Provider
Ref string
RootManifestDesc ocispecs.Descriptor
ImageManifestDesc ocispecs.Descriptor // Platform-specific manifest selected from RootManifestDesc.
ConfigDesc ocispecs.Descriptor
Nonlayers []ocispecs.Descriptor
Descriptors []ocispecs.Descriptor
Provider func(session.Group) content.Provider
}
func (p *Puller) resolve(ctx context.Context, resolver remotes.Resolver) error {
@@ -144,6 +145,18 @@ func (p *Puller) PullManifests(ctx context.Context, getResolver SessionResolver)
// Limit manifests pulled to the best match in an index
childrenHandler = images.LimitManifests(childrenHandler, platform, 1)
var manifestDesc ocispecs.Descriptor
var manifestOnce sync.Once
manifestHandler := images.HandlerFunc(func(ctx context.Context, desc ocispecs.Descriptor) ([]ocispecs.Descriptor, error) {
if images.IsManifestType(desc.MediaType) {
// Each index yields one child, so the first manifest is the
// selected image, including through nested indexes. Do not let
// malformed config or layer descriptors replace it later.
manifestOnce.Do(func() { manifestDesc = desc })
}
return nil, nil
})
dslHandler, err := docker.AppendDistributionSourceLabel(p.ContentStore, p.ref)
if err != nil {
return nil, err
@@ -152,6 +165,7 @@ func (p *Puller) PullManifests(ctx context.Context, getResolver SessionResolver)
filterLayerBlobs(metadata, &mu),
retryhandler.New(limited.FetchHandler(p.ContentStore, fetcher, p.ref), logs.LoggerFromContext(ctx)),
childrenHandler,
manifestHandler,
dslHandler,
)
@@ -174,11 +188,12 @@ func (p *Puller) PullManifests(ctx context.Context, getResolver SessionResolver)
}
return &PulledManifests{
Ref: p.ref,
MainManifestDesc: p.desc,
ConfigDesc: p.configDesc,
Nonlayers: p.nonlayers,
Descriptors: p.layers,
Ref: p.ref,
RootManifestDesc: p.desc,
ImageManifestDesc: manifestDesc,
ConfigDesc: p.configDesc,
Nonlayers: p.nonlayers,
Descriptors: p.layers,
Provider: func(g session.Group) content.Provider {
return &provider{puller: p, resolver: getResolver(g)}
},
+7
View File
@@ -147,6 +147,13 @@ func (a *dockerAuthorizer) Authorize(ctx context.Context, req *http.Request) err
authHeader, err := ah.authorize(ctx, a.sm, a.session)
if err != nil {
// containerd attaches a streaming request body (the pipe of a blob
// upload) before authorizing, and does not close it if authorization
// fails. Nothing would then unblock the writer feeding the pipe, and
// the registry concurrency slot held by it would never be released.
if req.Body != nil {
req.Body.Close()
}
return err
}
+37 -3
View File
@@ -5,6 +5,7 @@ import (
stderrors "errors"
"fmt"
"io"
"math/rand/v2"
"net"
"syscall"
"time"
@@ -19,6 +20,27 @@ import (
// variable so that code which embeds BuildKit can override the default value.
var MaxRetryBackoff = 8 * time.Second
// MaxRetryAfter is the longest server-requested delay accepted for a retry.
// It is independent of the exponential backoff budget. Code which embeds
// BuildKit can override the default value.
var MaxRetryAfter = 30 * time.Second
// RetryAfterError marks a failure as retryable after at least Delay.
// A zero Delay uses the normal backoff. Delays above MaxRetryAfter
// stop retries rather than retrying before the requested delay has elapsed.
type RetryAfterError struct {
Err error
Delay time.Duration
}
func (e *RetryAfterError) Error() string {
return e.Err.Error()
}
func (e *RetryAfterError) Unwrap() error {
return e.Err
}
func New(f images.HandlerFunc, logger func([]byte)) images.HandlerFunc {
return func(ctx context.Context, desc ocispecs.Descriptor) ([]ocispecs.Descriptor, error) {
descs, err := WithRetry(ctx, logger, func(ctx context.Context) ([]ocispecs.Descriptor, error) {
@@ -33,7 +55,7 @@ func New(f images.HandlerFunc, logger func([]byte)) images.HandlerFunc {
type retryContextKey struct{}
// WithRetry retries transient failures with exponential backoff. Nested calls
// WithRetry retries transient failures with jittered exponential backoff. Nested calls
// using the supplied context leave retries to the outer call, including New.
// f must be safe to retry and preserve retryable errors when wrapping them.
func WithRetry[T any](ctx context.Context, logger func([]byte), f func(context.Context) (T, error)) (T, error) {
@@ -66,13 +88,21 @@ func WithRetryIf[T any](ctx context.Context, logger func([]byte), f func(context
if backoff >= MaxRetryBackoff {
return v, err
}
// Keep the retry budget independent of jitter, including the first retry.
delay := backoff/2 + rand.N(backoff/2) //nolint:gosec // Jitter spreads load; it does not need cryptographic randomness.
if retryAfter, ok := stderrors.AsType[*RetryAfterError](err); ok {
if retryAfter.Delay > MaxRetryAfter {
return v, err
}
delay = max(delay, retryAfter.Delay)
}
if logger != nil {
logger(fmt.Appendf(nil, "retrying in %v\n", backoff))
logger(fmt.Appendf(nil, "retrying in %v\n", delay))
}
select {
case <-ctx.Done():
return v, contextFailure(ctx)
case <-time.After(backoff):
case <-time.After(delay):
}
backoff *= 2
}
@@ -88,6 +118,10 @@ func contextFailure(ctx context.Context) error {
}
func retryError(err error) bool {
if _, ok := stderrors.AsType[*RetryAfterError](err); ok {
return true
}
// Retry on 5xx errors
var errUnexpectedStatus remoteserrors.ErrUnexpectedStatus
if errors.As(err, &errUnexpectedStatus) &&
+1 -1
View File
@@ -743,7 +743,7 @@ func ID(root string) (string, error) {
if err != nil {
if errors.Is(err, os.ErrNotExist) {
id := identity.NewID()
err := os.WriteFile(f, []byte(id), 0400)
err := os.WriteFile(f, []byte(id), 0o600)
return id, err
}
return "", err
+31 -37
View File
@@ -12,37 +12,39 @@ type mkdirOptions struct {
onlyNew bool
}
// WithOnlyNew is an option for MkdirAllAndChown that will only change ownership and permissions
// on newly created directories. If the directory already exists, it will not be modified
// WithOnlyNew configures [MkdirAllAndChown] and [MkdirAndChown] to leave
// the requested directory unchanged if it already exists. Newly created
// directories are still assigned the requested ownership and permissions.
func WithOnlyNew(o *mkdirOptions) {
o.onlyNew = true
}
// MkdirAllAndChown creates a directory (include any along the path) and then modifies
// ownership to the requested uid/gid. By default, if the directory already exists, this
// function will still change ownership and permissions. If WithOnlyNew is passed as an
// option, then only the newly created directories will have ownership and permissions changed.
// MkdirAllAndChown creates a directory named path, along with any necessary
// parents, and applies the requested ownership and permission bits to all
// directories it creates.
//
// If path already exists as a directory, its ownership and permission bits
// are updated unless [WithOnlyNew] is specified. Existing parent directories
// are not modified.
//
// MkdirAllAndChown is similar to [os.MkdirAll], but also applies ownership and
// reapplies the requested permission bits after creation, so the resulting
// permissions are not affected by the process umask.
func MkdirAllAndChown(path string, mode os.FileMode, uid, gid int, opts ...MkdirOpt) error {
var options mkdirOptions
for _, opt := range opts {
opt(&options)
}
return mkdirAs(path, mode, uid, gid, true, options.onlyNew)
return mkdirAs(path, mode, uid, gid, true, opts...)
}
// MkdirAndChown creates a directory and then modifies ownership to the requested uid/gid.
// By default, if the directory already exists, this function still changes ownership and permissions.
// If WithOnlyNew is passed as an option, then only the newly created directory will have ownership
// and permissions changed.
// Note that unlike os.Mkdir(), this function does not return IsExist error
// in case path already exists.
// MkdirAndChown creates a directory named path and applies the requested
// ownership and permission bits.
//
// If path already exists as a directory, its ownership and permission bits
// are updated unless [WithOnlyNew] is specified.
//
// MkdirAndChown is similar to [os.Mkdir], but returns nil if path already
// exists as a directory. The resulting permission bits are not affected by
// the process umask.
func MkdirAndChown(path string, mode os.FileMode, uid, gid int, opts ...MkdirOpt) error {
var options mkdirOptions
for _, opt := range opts {
opt(&options)
}
return mkdirAs(path, mode, uid, gid, false, options.onlyNew)
return mkdirAs(path, mode, uid, gid, false, opts...)
}
// getRootUIDGID retrieves the remapped root uid/gid pair from the set of maps.
@@ -106,22 +108,14 @@ func (i IdentityMapping) RootPair() (int, int) {
return uid, gid
}
// ToHost returns the host UID and GID for the container uid, gid.
// Remapping is only performed if the ids aren't already the remapped root ids
// ToHost returns the host UID and GID for the container uid and gid.
// An empty mapping is treated as identity.
func (i IdentityMapping) ToHost(uid, gid int) (int, int, error) {
var err error
ruid, rgid := i.RootPair()
if uid != ruid {
ruid, err = toHost(uid, i.UIDMaps)
if err != nil {
return ruid, rgid, err
}
}
if gid != rgid {
rgid, err = toHost(gid, i.GIDMaps)
ruid, err := toHost(uid, i.UIDMaps)
if err != nil {
return -1, -1, err
}
rgid, err := toHost(gid, i.GIDMaps)
return ruid, rgid, err
}
+7 -2
View File
@@ -10,7 +10,12 @@ import (
"syscall"
)
func mkdirAs(path string, mode os.FileMode, uid, gid int, mkAll, onlyNew bool) error {
func mkdirAs(path string, mode os.FileMode, uid, gid int, mkAll bool, opts ...MkdirOpt) error {
var options mkdirOptions
for _, opt := range opts {
opt(&options)
}
path, err := filepath.Abs(path)
if err != nil {
return err
@@ -21,7 +26,7 @@ func mkdirAs(path string, mode os.FileMode, uid, gid int, mkAll, onlyNew bool) e
if !stat.IsDir() {
return &os.PathError{Op: "mkdir", Path: path, Err: syscall.ENOTDIR}
}
if onlyNew {
if options.onlyNew {
return nil
}
+19 -6
View File
@@ -2,12 +2,25 @@ package user
import (
"os"
"syscall"
)
// This is currently a wrapper around [os.MkdirAll] since currently
// permissions aren't set through this path, the identity isn't utilized.
// Ownership is handled elsewhere, but in the future could be support here
// too.
func mkdirAs(path string, _ os.FileMode, _, _ int, _, _ bool) error {
return os.MkdirAll(path, 0)
// mkdirAs creates path, optionally creating any missing parent directories.
//
// On Windows this is currently a thin wrapper around os.Mkdir and
// os.MkdirAll. Unlike the Unix implementation, ownership and permission
// bits are not applied.
func mkdirAs(path string, _ os.FileMode, _, _ int, mkAll bool, _ ...MkdirOpt) error {
if mkAll {
return os.MkdirAll(path, 0)
}
stat, err := os.Stat(path)
if err == nil {
if !stat.IsDir() {
return &os.PathError{Op: "mkdir", Path: path, Err: syscall.ENOTDIR}
}
return nil
}
return os.Mkdir(path, 0)
}
+31 -79
View File
@@ -7,6 +7,7 @@ import (
"fmt"
"io"
"os"
"slices"
"strconv"
"strings"
)
@@ -17,9 +18,9 @@ const (
)
var (
// ErrNoPasswdEntries is returned if no matching entries were found in /etc/group.
// ErrNoPasswdEntries is returned if no matching entries were found in /etc/passwd.
ErrNoPasswdEntries = errors.New("no matching entries in passwd file")
// ErrNoGroupEntries is returned if no matching entries were found in /etc/passwd.
// ErrNoGroupEntries is returned if no matching entries were found in /etc/group.
ErrNoGroupEntries = errors.New("no matching entries in group file")
// ErrRange is returned if a UID or GID is outside of the valid range.
ErrRange = fmt.Errorf("uids and gids must be in range %d-%d", minID, maxID)
@@ -243,9 +244,6 @@ func GetExecUserPath(userSpec string, defaults *ExecUser, passwdPath, groupPath
//
// See https://github.com/containerd/containerd/commit/de1341c201ffb0effebbf51d00376181968c8779
func parseNumeric(val string) (int, bool, error) {
if val == "" {
return 0, false, nil
}
id, err := strconv.Atoi(val)
if err != nil {
if errors.Is(err, strconv.ErrSyntax) {
@@ -296,51 +294,40 @@ func GetExecUser(userSpec string, defaults *ExecUser, passwd, group io.Reader) (
user := &ExecUser{
Uid: defaults.Uid,
Gid: defaults.Gid,
Sgids: defaults.Sgids,
Sgids: append([]int{}, defaults.Sgids...), // Sgids slice *cannot* be nil.
Home: defaults.Home,
}
// Sgids slice *cannot* be nil.
if user.Sgids == nil {
user.Sgids = []int{}
}
// Allow for userSpec to have either "user", or optionally "user:group" syntax.
usr, grp, _ := strings.Cut(userSpec, ":")
// Allow for userArg to have either "user" syntax, or optionally "user:group" syntax
var userArg, groupArg string
parseLine([]byte(userSpec), &userArg, &groupArg)
// Convert userArg and groupArg to be numeric, so we don't have to execute
// Convert usr and grp to be numeric, so we don't have to execute
// Atoi *twice* for each iteration over lines.
uidArg, isUID, err := parseNumeric(userArg)
userArg, err := parseUserArg(usr)
if err != nil {
return nil, err
}
gidArg, isGID, err := parseNumeric(groupArg)
groupArg, err := parseGroupArg(grp)
if err != nil {
return nil, err
}
// Find the matching user.
users, err := ParsePasswdFilter(passwd, func(u User) bool {
if userArg == "" {
if userArg == nil {
// Default to current state of the user.
return u.Uid == user.Uid
}
if isUID {
// If the userArg is a valid numeric value, always treat it as a UID.
return uidArg == u.Uid
}
return u.Name == userArg
return userArg.matches(u)
})
// If we can't find the user, we have to bail.
if err != nil && passwd != nil {
if userArg == "" {
userArg = strconv.Itoa(user.Uid)
name := usr
if userArg == nil {
name = strconv.Itoa(user.Uid)
}
return nil, fmt.Errorf("unable to find user %s: %w", userArg, err)
return nil, fmt.Errorf("unable to find user %s: %w", name, err)
}
var matchedUserName string
@@ -350,57 +337,42 @@ func GetExecUser(userSpec string, defaults *ExecUser, passwd, group io.Reader) (
user.Uid = users[0].Uid
user.Gid = users[0].Gid
user.Home = users[0].Home
} else if userArg != "" {
} else if userArg != nil {
// If we can't find a user with the given username, the only other valid
// option is if it's a numeric username with no associated entry in passwd.
if !isUID {
// Not numeric.
return nil, fmt.Errorf("unable to find user %s: %w", userArg, ErrNoPasswdEntries)
if !userArg.isNumeric {
return nil, fmt.Errorf("unable to find user %s: %w", userArg.name, ErrNoPasswdEntries)
}
user.Uid = uidArg
user.Uid = userArg.uid
}
// On to the groups. If we matched a username, we need to do this because of
// the supplementary group IDs.
if groupArg != "" || matchedUserName != "" {
if groupArg != nil || matchedUserName != "" {
groups, err := ParseGroupFilter(group, func(g Group) bool {
// If the group argument isn't explicit, we'll just search for it.
if groupArg == "" {
if groupArg == nil {
// Check if user is a member of this group.
for _, u := range g.List {
if u == matchedUserName {
return true
}
}
return false
return slices.Contains(g.List, matchedUserName)
}
if isGID {
// If the groupArg is numeric, always treat it as a GID.
return gidArg == g.Gid
}
return g.Name == groupArg
return groupArg.matches(g)
})
if err != nil && group != nil {
return nil, fmt.Errorf("unable to find groups for spec %v: %w", matchedUserName, err)
}
// Only start modifying user.Gid if it is in explicit form.
if groupArg != "" {
if groupArg != nil {
if len(groups) > 0 {
// First match wins, even if there's more than one matching entry.
user.Gid = groups[0].Gid
} else {
// If we can't find a group with the given name, the only other valid
// option is if it's a numeric group name with no associated entry in group.
if !isGID {
// Not numeric.
return nil, fmt.Errorf("unable to find group %s: %w", groupArg, ErrNoGroupEntries)
if !groupArg.isNumeric {
return nil, fmt.Errorf("unable to find group %s: %w", groupArg.name, ErrNoGroupEntries)
}
user.Gid = gidArg
user.Gid = groupArg.gid
}
} else if len(groups) > 0 {
// Supplementary group ids only make sense if in the implicit form.
@@ -414,39 +386,19 @@ func GetExecUser(userSpec string, defaults *ExecUser, passwd, group io.Reader) (
return user, nil
}
// groupArg is a parsed group argument for [GetAdditionalGroups].
type groupArg struct {
name string
gid int
isNumeric bool
}
// matches reports whether group g satisfies the argument. Numeric arguments
// are matched by GID only, others by name.
func (ag groupArg) matches(g Group) bool {
if ag.isNumeric {
return g.Gid == ag.gid
}
return g.Name == ag.name
}
// GetAdditionalGroups looks up a list of groups by name or group id
// against the given /etc/group formatted data. If a group name cannot
// be found, an error will be returned. If a group id cannot be found,
// or the given group data is nil, the id will be returned as-is
// provided it is in the legal range.
func GetAdditionalGroups(additionalGroups []string, group io.Reader) ([]int, error) {
addtlGroups := make([]groupArg, len(additionalGroups))
for i, ag := range additionalGroups {
gid, ok, err := parseNumeric(ag)
addtlGroups := make([]groupArg, 0, len(additionalGroups))
for _, ag := range additionalGroups {
groupArg, err := parseGroupArg(ag)
if err != nil {
return nil, err
}
addtlGroups[i] = groupArg{
name: ag,
gid: gid,
isNumeric: ok,
}
addtlGroups = append(addtlGroups, *groupArg)
}
groups := []Group{}
+62
View File
@@ -62,3 +62,65 @@ func (l *limitedFile) Read(p []byte) (int, error) {
}
return n, err
}
// userArg is a parsed user argument for [GetExecUser].
type userArg struct {
name string
uid int
isNumeric bool
}
// parseUserArg parses a user argument as either a user name or UID.
//
// If name is empty, parseUserArg returns "nil, nil" to indicate that no user
// argument was specified.
func parseUserArg(name string) (*userArg, error) {
if name == "" {
return nil, nil
}
uid, isNumeric, err := parseNumeric(name)
if err != nil {
return nil, err
}
return &userArg{name: name, uid: uid, isNumeric: isNumeric}, nil
}
// matches reports whether user u satisfies the argument. Numeric arguments
// are matched by UID only, others by name.
func (ua *userArg) matches(u User) bool {
if ua.isNumeric {
return u.Uid == ua.uid
}
return u.Name == ua.name
}
// groupArg is a parsed group argument for [GetAdditionalGroups] or [GetExecUser].
type groupArg struct {
name string
gid int
isNumeric bool
}
// parseGroupArg parses a group argument as either a group name or GID.
//
// If name is empty, parseGroupArg returns "nil, nil" to indicate that no group
// argument was specified.
func parseGroupArg(name string) (*groupArg, error) {
if name == "" {
return nil, nil
}
gid, isNumeric, err := parseNumeric(name)
if err != nil {
return nil, err
}
return &groupArg{name: name, gid: gid, isNumeric: isNumeric}, nil
}
// matches reports whether group g satisfies the argument. Numeric arguments
// are matched by GID only, others by name.
func (ag groupArg) matches(g Group) bool {
if ag.isNumeric {
return g.Gid == ag.gid
}
return g.Name == ag.name
}
+7 -7
View File
@@ -626,10 +626,9 @@ github.com/containerd/platforms
## explicit; go 1.22
github.com/containerd/plugin
github.com/containerd/plugin/registry
# github.com/containerd/stargz-snapshotter/estargz v0.18.2
## explicit; go 1.24.0
# github.com/containerd/stargz-snapshotter/estargz v0.19.1
## explicit; go 1.25.0
github.com/containerd/stargz-snapshotter/estargz
github.com/containerd/stargz-snapshotter/estargz/errorutil
# github.com/containerd/ttrpc v1.2.10
## explicit; go 1.25.0
github.com/containerd/ttrpc
@@ -1078,7 +1077,7 @@ github.com/mitchellh/copystructure
# github.com/mitchellh/reflectwalk v1.0.2
## explicit
github.com/mitchellh/reflectwalk
# github.com/moby/buildkit v0.34.2
# github.com/moby/buildkit v0.35.0-rc1
## explicit; go 1.26.8
github.com/moby/buildkit/api/services/control
github.com/moby/buildkit/api/types
@@ -1088,10 +1087,10 @@ github.com/moby/buildkit/cache/contenthash
github.com/moby/buildkit/cache/metadata
github.com/moby/buildkit/cache/remotecache
github.com/moby/buildkit/cache/remotecache/gha
github.com/moby/buildkit/cache/remotecache/gha/ghatypes
github.com/moby/buildkit/cache/remotecache/inline
github.com/moby/buildkit/cache/remotecache/local
github.com/moby/buildkit/cache/remotecache/registry
github.com/moby/buildkit/cache/remotecache/signing
github.com/moby/buildkit/cache/remotecache/v1
github.com/moby/buildkit/cache/remotecache/v1/types
github.com/moby/buildkit/cache/util
@@ -1204,6 +1203,7 @@ github.com/moby/buildkit/util/bkmaps
github.com/moby/buildkit/util/bkslices
github.com/moby/buildkit/util/cachedigest
github.com/moby/buildkit/util/compression
github.com/moby/buildkit/util/compression/go126flate
github.com/moby/buildkit/util/cond
github.com/moby/buildkit/util/contentutil
github.com/moby/buildkit/util/converter
@@ -1456,8 +1456,8 @@ github.com/moby/sys/signal
# github.com/moby/sys/symlink v0.3.0
## explicit; go 1.17
github.com/moby/sys/symlink
# github.com/moby/sys/user v0.4.1
## explicit; go 1.18
# github.com/moby/sys/user v0.5.0
## explicit; go 1.24
github.com/moby/sys/user
# github.com/moby/sys/userns v0.2.1
## explicit; go 1.21