Merge pull request #54132 from tonistiigi/update-buildkit-v0.35-rc1
vendor: update buildkit v0.35.0-rc1
This commit is contained in:
93 files changed
+4236
-861
No files matched your search
@@ -64,7 +64,7 @@ require (
|
||||
github.com/miekg/dns v1.1.73
|
||||
github.com/mistifyio/go-zfs/v4 v4.0.0
|
||||
github.com/mitchellh/copystructure v1.2.0
|
||||
github.com/moby/buildkit v0.34.2
|
||||
github.com/moby/buildkit v0.35.0-rc1
|
||||
github.com/moby/docker-image-spec v1.3.1
|
||||
github.com/moby/extensions v0.0.0-20261020100352-d2819d88bcbe
|
||||
github.com/moby/go-archive v0.3.3
|
||||
@@ -84,7 +84,7 @@ require (
|
||||
github.com/moby/sys/reexec v0.1.0
|
||||
github.com/moby/sys/signal v0.7.1
|
||||
github.com/moby/sys/symlink v0.3.0
|
||||
github.com/moby/sys/user v0.4.1
|
||||
github.com/moby/sys/user v0.5.0
|
||||
github.com/moby/sys/userns v0.2.1
|
||||
github.com/moby/term v0.5.2
|
||||
github.com/opencontainers/cgroups v0.1.0
|
||||
@@ -165,7 +165,7 @@ require (
|
||||
github.com/containerd/go-runc v1.2.1 // indirect
|
||||
github.com/containerd/nydus-snapshotter v0.15.15 // indirect
|
||||
github.com/containerd/otelttrpc v0.1.0 // indirect
|
||||
github.com/containerd/stargz-snapshotter/estargz v0.18.2 // indirect
|
||||
github.com/containerd/stargz-snapshotter/estargz v0.19.1 // indirect
|
||||
github.com/containernetworking/cni v1.3.1 // indirect
|
||||
github.com/containernetworking/plugins v1.9.1 // indirect
|
||||
github.com/coreos/go-semver v0.3.1 // indirect
|
||||
|
||||
@@ -174,8 +174,8 @@ github.com/containerd/platforms v1.0.0-rc.5 h1:vXd569rDrz8LeMXzAnBsy6LADV5YtsD8o
|
||||
github.com/containerd/platforms v1.0.0-rc.5/go.mod h1:lKlMXyLybmBedS/JJm11uDofzI8L2v0J2ZbYvNsbq1A=
|
||||
github.com/containerd/plugin v1.1.0 h1:O+7lczNJVMy8rz0YNx3xGB8tTf5qY4i5abF041Ew19U=
|
||||
github.com/containerd/plugin v1.1.0/go.mod h1:qBTum+A8lJ6lO44A19Eo7y1OlcLj4OWFH1DA/vnHmcc=
|
||||
github.com/containerd/stargz-snapshotter/estargz v0.18.2 h1:yXkZFYIzz3eoLwlTUZKz2iQ4MrckBxJjkmD16ynUTrw=
|
||||
github.com/containerd/stargz-snapshotter/estargz v0.18.2/go.mod h1:XyVU5tcJ3PRpkA9XS2T5us6Eg35yM0214Y+wvrZTBrY=
|
||||
github.com/containerd/stargz-snapshotter/estargz v0.19.1 h1:PD/uZYqmyyEXZl3fEPM2ieCpilv7Oow9FC2OCBfkDRs=
|
||||
github.com/containerd/stargz-snapshotter/estargz v0.19.1/go.mod h1:Cr0pxOkuN7Y61+Sik6X6/nPtePW7hsKWdBt+MoHQRr0=
|
||||
github.com/containerd/ttrpc v1.2.10 h1:qQvy3mokhoXON7FXiOOc05O7Up6nrNj6amJF8urNvDc=
|
||||
github.com/containerd/ttrpc v1.2.10/go.mod h1:YrmEQKkMbBA4EM26AzVAIwHGIYgptkL4CJ/yVTaDyBE=
|
||||
github.com/containerd/typeurl/v2 v2.3.0 h1:HZHPhRWo5XMy3QGQoPrUzbW/2ckwjfweHmOwlkIrPAQ=
|
||||
@@ -212,8 +212,8 @@ github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi
|
||||
github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
|
||||
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
|
||||
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
|
||||
github.com/docker/cli v29.8.2+incompatible h1:2zgdFuoFst2T80oS42vhKGxkd0JRo305eIEKTsxR7pQ=
|
||||
github.com/docker/cli v29.8.2+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
|
||||
github.com/docker/cli v29.9.3+incompatible h1:SBaMp7ecohadEHMDrUJg22t9I3awqsAVhnJ7GOcWpfE=
|
||||
github.com/docker/cli v29.9.3+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
|
||||
github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk=
|
||||
github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
|
||||
github.com/docker/docker-credential-helpers v0.9.9 h1:BkydjIgZ46JnDbqyM2p2fc63KMw6y+KHL3Em/2AGJ7w=
|
||||
@@ -469,8 +469,8 @@ github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyua
|
||||
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
|
||||
github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ=
|
||||
github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
|
||||
github.com/moby/buildkit v0.34.2 h1:fXgOe79JW5zJNDumRtwTemOklLX66otQokObUXuiGJo=
|
||||
github.com/moby/buildkit v0.34.2/go.mod h1:HyFnF6kIPzHvqopF1+BKeGY8Ee9MXmfR2n+0Gd4Y69s=
|
||||
github.com/moby/buildkit v0.35.0-rc1 h1:wgD3pbfSTP03T3Q4IkMRRAHhxbz7177+mavHnwVjUQI=
|
||||
github.com/moby/buildkit v0.35.0-rc1/go.mod h1:5L8B0JOdp1dO1ntCpfOYT5vR31kQ1nKIor/b4FwsDVQ=
|
||||
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
|
||||
github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
|
||||
github.com/moby/extensions v0.0.0-20261020100352-d2819d88bcbe h1:wHRo58MA+vgX2sXtL4uT/NcSYPiyJarqHZ4aDS8P0ys=
|
||||
@@ -507,8 +507,8 @@ github.com/moby/sys/signal v0.7.1 h1:PrQxdvxcGijdo6UXXo/lU/TvHUWyPhj7UOpSo8tuvk0
|
||||
github.com/moby/sys/signal v0.7.1/go.mod h1:Se1VGehYokAkrSQwL4tDzHvETwUZlnY7S5XtQ50mQp8=
|
||||
github.com/moby/sys/symlink v0.3.0 h1:GZX89mEZ9u53f97npBy4Rc3vJKj7JBDj/PN2I22GrNU=
|
||||
github.com/moby/sys/symlink v0.3.0/go.mod h1:3eNdhduHmYPcgsJtZXW1W4XUJdZGBIkttZ8xKqPUJq0=
|
||||
github.com/moby/sys/user v0.4.1 h1:RgjRlaDKi/Xmyrz4t8lyzXT6v2ooFeO/7xtchmhVWE0=
|
||||
github.com/moby/sys/user v0.4.1/go.mod h1:E9QsW5WRe1kUAf7kW8hXKwu1uhsZEAdPLYHYSDudF4Y=
|
||||
github.com/moby/sys/user v0.5.0 h1:5+S4lyoavGyX1DGs68NW8xpx48T9+a7ogSUc0KjZg8Y=
|
||||
github.com/moby/sys/user v0.5.0/go.mod h1:4mQzP5aTmaLjDLp/dc870oNkik6E9vjRtWYwI05gvow=
|
||||
github.com/moby/sys/userns v0.2.1 h1:4OvdM7BcPkASbuouHsbW3aeMJSFlYDldBRnXVZhaRk8=
|
||||
github.com/moby/sys/userns v0.2.1/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28=
|
||||
github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ=
|
||||
@@ -740,8 +740,8 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.47.0 h1:UFpxO
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.47.0/go.mod h1:YBGjxe3lt0jtXoEXxGrGhv/jM7oUBUDVK7zhAvNyEjc=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0 h1:aXZXsZ012wOgVkqaiQv+Z/d8V0xKjmg70F6m4CE94lc=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0/go.mod h1:ziu1gUIJhAaxM3EV1e3Ralk5AAyL1UOYz3lJJ4VSI38=
|
||||
go.opentelemetry.io/otel/exporters/prometheus v0.68.0 h1:QOf2IftqQwITVRJpnn0M7M9ZCbgWfxz4P7i9C9yc2N4=
|
||||
go.opentelemetry.io/otel/exporters/prometheus v0.68.0/go.mod h1:bgSvqu2TWGXiz7yr5UTMfObH8oqxJWHTnubQ3ef9BO4=
|
||||
go.opentelemetry.io/otel/exporters/prometheus v0.69.0 h1:LQVBTGnHvzrfGJpDoCSVTY7eaLZArvaTSq5l84Skxew=
|
||||
go.opentelemetry.io/otel/exporters/prometheus v0.69.0/go.mod h1:moif0nlgj/M16NRT8lDZVt8MC52+EOAJfR8EKRJ8LyU=
|
||||
go.opentelemetry.io/otel/log v1.47.0 h1:cOTS1CcLbSQeZKanGJ+0JpF/+t4PELi3O3bbl2lqCcI=
|
||||
go.opentelemetry.io/otel/log v1.47.0/go.mod h1:9byitSQ5pLC6PpqwGXjqdMKya6ZTswHRZh2vvXT33nw=
|
||||
go.opentelemetry.io/otel/metric v1.47.0 h1:4PptaldXx3Eat1XjMZ68pPJEs5wrhlemctZE9a3UdWY=
|
||||
|
||||
+38
-16
@@ -37,10 +37,8 @@ import (
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/containerd/stargz-snapshotter/estargz/errorutil"
|
||||
"github.com/klauspost/compress/zstd"
|
||||
digest "github.com/opencontainers/go-digest"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
type GzipHelperFunc func(io.Reader) (io.ReadCloser, error)
|
||||
@@ -53,6 +51,7 @@ type options struct {
|
||||
compression Compression
|
||||
ctx context.Context
|
||||
minChunkSize int
|
||||
parallelism int
|
||||
gzipHelperFunc GzipHelperFunc
|
||||
}
|
||||
|
||||
@@ -131,6 +130,20 @@ func WithMinChunkSize(minChunkSize int) Option {
|
||||
}
|
||||
}
|
||||
|
||||
// WithParallelism option specifies the number of workers used to build the
|
||||
// layer. The tar is sliced into this many parts that are compressed
|
||||
// concurrently, so the value also fixes the resulting chunk boundaries: pinning
|
||||
// it makes builds reproducible across machines regardless of their CPU count.
|
||||
// Zero (the default) selects runtime.GOMAXPROCS(0); a value of 1 forces a fully
|
||||
// sequential build. This has no effect together with WithMinChunkSize, which is
|
||||
// always built with a single worker.
|
||||
func WithParallelism(workers int) Option {
|
||||
return func(o *options) error {
|
||||
o.parallelism = workers
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// WithGzipHelperFunc option specifies a custom function to decompress gzip-compressed layers.
|
||||
// When a gzip-compressed layer is detected, this function will be used instead of the
|
||||
// Go standard library gzip decompression for better performance.
|
||||
@@ -225,25 +238,32 @@ func Build(tarBlob *io.SectionReader, opt ...Option) (_ *Blob, rErr error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// The worker count slices the tar and so fixes the chunk boundaries; an
|
||||
// explicit value is honored verbatim, which keeps builds reproducible across
|
||||
// machines. Zero falls back to GOMAXPROCS.
|
||||
workers := opts.parallelism
|
||||
if workers <= 0 {
|
||||
workers = runtime.GOMAXPROCS(0)
|
||||
}
|
||||
var tarParts [][]*entry
|
||||
if opts.minChunkSize > 0 {
|
||||
// Each entry needs to know the size of the current gzip stream so they
|
||||
// cannot be processed in parallel.
|
||||
tarParts = [][]*entry{entries}
|
||||
} else {
|
||||
tarParts = divideEntries(entries, runtime.GOMAXPROCS(0))
|
||||
tarParts = divideEntries(entries, workers)
|
||||
}
|
||||
writers := make([]*Writer, len(tarParts))
|
||||
payloads := make([]*os.File, len(tarParts))
|
||||
var mu sync.Mutex
|
||||
var eg errgroup.Group
|
||||
var wg sync.WaitGroup
|
||||
errCh := make(chan error, len(tarParts)) // buffered to avoid goroutine leaks
|
||||
for i, parts := range tarParts {
|
||||
i, parts := i, parts
|
||||
// builds verifiable stargz sub-blobs
|
||||
eg.Go(func() error {
|
||||
wg.Go(func() {
|
||||
esgzFile, err := layerFiles.TempFile("", "esgzdata")
|
||||
if err != nil {
|
||||
return err
|
||||
errCh <- err
|
||||
return
|
||||
}
|
||||
sw := NewWriterWithCompressor(esgzFile, opts.compression)
|
||||
sw.ChunkSize = opts.chunkSize
|
||||
@@ -255,18 +275,20 @@ func Build(tarBlob *io.SectionReader, opt ...Option) (_ *Blob, rErr error) {
|
||||
sw.needsOpenGzEntries[f] = struct{}{}
|
||||
}
|
||||
if err := sw.AppendTar(readerFromEntries(parts...)); err != nil {
|
||||
return err
|
||||
errCh <- err
|
||||
return
|
||||
}
|
||||
mu.Lock()
|
||||
writers[i] = sw
|
||||
payloads[i] = esgzFile
|
||||
mu.Unlock()
|
||||
return nil
|
||||
})
|
||||
}
|
||||
if err := eg.Wait(); err != nil {
|
||||
rErr = err
|
||||
return nil, err
|
||||
wg.Wait()
|
||||
close(errCh)
|
||||
|
||||
for err := range errCh {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
tocAndFooter, tocDgst, err := closeWithCombine(writers...)
|
||||
if err != nil {
|
||||
@@ -656,7 +678,7 @@ func (tf *tempFiles) cleanupAll() error {
|
||||
}
|
||||
}
|
||||
tf.files = nil
|
||||
return errorutil.Aggregate(allErr)
|
||||
return errors.Join(allErr...)
|
||||
}
|
||||
|
||||
func newCountReadSeeker(r io.ReaderAt) (*countReadSeeker, error) {
|
||||
|
||||
-40
@@ -1,40 +0,0 @@
|
||||
/*
|
||||
Copyright The containerd Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package errorutil
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Aggregate combines a list of errors into a single new error.
|
||||
func Aggregate(errs []error) error {
|
||||
switch len(errs) {
|
||||
case 0:
|
||||
return nil
|
||||
case 1:
|
||||
return errs[0]
|
||||
default:
|
||||
points := make([]string, len(errs)+1)
|
||||
points[0] = fmt.Sprintf("%d error(s) occurred:", len(errs))
|
||||
for i, err := range errs {
|
||||
points[i+1] = fmt.Sprintf("* %s", err)
|
||||
}
|
||||
return errors.New(strings.Join(points, "\n\t"))
|
||||
}
|
||||
}
|
||||
+12
-8
@@ -38,7 +38,6 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/containerd/stargz-snapshotter/estargz/errorutil"
|
||||
digest "github.com/opencontainers/go-digest"
|
||||
"github.com/vbatts/tar-split/archive/tar"
|
||||
)
|
||||
@@ -164,7 +163,7 @@ func Open(sr *io.SectionReader, opt ...OpenOption) (*Reader, error) {
|
||||
allErr = append(allErr, err)
|
||||
}
|
||||
if !found {
|
||||
return nil, errorutil.Aggregate(allErr)
|
||||
return nil, errors.Join(allErr...)
|
||||
}
|
||||
if err := r.initFields(); err != nil {
|
||||
return nil, fmt.Errorf("failed to initialize fields of entries: %v", err)
|
||||
@@ -192,7 +191,7 @@ func OpenFooter(sr *io.SectionReader) (tocOffset int64, footerSize int64, rErr e
|
||||
}
|
||||
allErr = append(allErr, err)
|
||||
}
|
||||
return 0, 0, errorutil.Aggregate(allErr)
|
||||
return 0, 0, errors.Join(allErr...)
|
||||
}
|
||||
|
||||
// initFields populates the Reader from r.toc after decoding it from
|
||||
@@ -286,7 +285,7 @@ func (r *Reader) initFields() error {
|
||||
pdir := r.getOrCreateDir(pdirName)
|
||||
ent.NumLink++ // at least one name(ent.Name) references this entry.
|
||||
if ent.Type == "hardlink" {
|
||||
org, err := r.getSource(ent)
|
||||
org, err := r.getSource(ent, make(map[string]struct{}))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -319,13 +318,18 @@ func (r *Reader) initFields() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Reader) getSource(ent *TOCEntry) (_ *TOCEntry, err error) {
|
||||
func (r *Reader) getSource(ent *TOCEntry, visited map[string]struct{}) (_ *TOCEntry, err error) {
|
||||
if ent.Type == "hardlink" {
|
||||
org, ok := r.m[cleanEntryName(ent.LinkName)]
|
||||
entName := cleanEntryName(ent.LinkName)
|
||||
if _, ok := visited[entName]; ok {
|
||||
return nil, fmt.Errorf("cyclic hardlink detected")
|
||||
}
|
||||
visited[entName] = struct{}{}
|
||||
org, ok := r.m[entName]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("%q is a hardlink but the linkname %q isn't found", ent.Name, ent.LinkName)
|
||||
}
|
||||
ent, err = r.getSource(org)
|
||||
ent, err = r.getSource(org, visited)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -493,7 +497,7 @@ func (r *Reader) Lookup(path string) (e *TOCEntry, ok bool) {
|
||||
e, ok = r.m[path]
|
||||
if ok && e.Type == "hardlink" {
|
||||
var err error
|
||||
e, err = r.getSource(e)
|
||||
e, err = r.getSource(e, make(map[string]struct{}))
|
||||
if err != nil {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
+42
-8
@@ -100,21 +100,52 @@ func (gc *GzipCompressor) WriteTOCAndFooter(w io.Writer, off int64, toc *JTOC, d
|
||||
|
||||
// gzipFooterBytes returns the 51 bytes footer.
|
||||
func gzipFooterBytes(tocOff int64) []byte {
|
||||
buf := bytes.NewBuffer(make([]byte, 0, FooterSize))
|
||||
gz, _ := gzip.NewWriterLevel(buf, gzip.NoCompression) // MUST be NoCompression to keep 51 bytes
|
||||
|
||||
// Extra header indicating the offset of TOCJSON
|
||||
// https://tools.ietf.org/html/rfc1952#section-2.3.1.1
|
||||
header := make([]byte, 4)
|
||||
header[0], header[1] = 'S', 'G'
|
||||
subfield := fmt.Sprintf("%016xSTARGZ", tocOff)
|
||||
binary.LittleEndian.PutUint16(header[2:4], uint16(len(subfield))) // little-endian per RFC1952
|
||||
gz.Extra = append(header, []byte(subfield)...)
|
||||
gz.Close()
|
||||
if buf.Len() != FooterSize {
|
||||
panic(fmt.Sprintf("footer buffer = %d, not %d", buf.Len(), FooterSize))
|
||||
extra := append(header, []byte(subfield)...)
|
||||
|
||||
buf := CreateGzipFooter(extra)
|
||||
if len(buf) != FooterSize {
|
||||
panic(fmt.Sprintf("footer buffer = %d, not %d", len(buf), FooterSize))
|
||||
}
|
||||
return buf.Bytes()
|
||||
return buf
|
||||
}
|
||||
|
||||
// CreateGzipFooter creates eStargz's footer with the specified extra field.
|
||||
func CreateGzipFooter(extra []byte) []byte {
|
||||
// Gzip header (10 bytes)
|
||||
// https://datatracker.ietf.org/doc/html/rfc1952
|
||||
buf := make([]byte, 10)
|
||||
buf[0] = 0x1f // ID1
|
||||
buf[1] = 0x8b // ID2
|
||||
buf[2] = 8 // "deflate"
|
||||
buf[3] = 0x4 // FEXTRA
|
||||
// MTIME = 0
|
||||
// XFL = 0
|
||||
buf[9] = 255 // unknown
|
||||
|
||||
// Extra field
|
||||
xlen := make([]byte, 2)
|
||||
binary.LittleEndian.PutUint16(xlen, uint16(len(extra)))
|
||||
buf = append(buf, append(xlen, extra...)...)
|
||||
|
||||
// Flate header (5 bytes)
|
||||
// https://datatracker.ietf.org/doc/html/rfc1951
|
||||
flateHeader := []byte{
|
||||
1, // BFINAL = 1 (last block), BFTYPE = 0 (no compression), remaining bits are ignored
|
||||
0, 0, // LEN = 0
|
||||
0xff, 0xff, // NLEN (the one's complement of LEN)
|
||||
}
|
||||
buf = append(buf, flateHeader...)
|
||||
|
||||
// Gzip footer (8 bytes): CRC32 = 0, ISIZE = 0
|
||||
buf = append(buf, make([]byte, 8)...)
|
||||
|
||||
return buf
|
||||
}
|
||||
|
||||
type GzipDecompressor struct{}
|
||||
@@ -137,6 +168,9 @@ func (gz *GzipDecompressor) ParseFooter(p []byte) (blobPayloadSize, tocOffset, t
|
||||
}
|
||||
defer zr.Close()
|
||||
extra := zr.Extra
|
||||
if len(extra) < 4 {
|
||||
return 0, 0, 0, fmt.Errorf("invalid extra field length %d; expected >= 4", len(extra))
|
||||
}
|
||||
si1, si2, subfieldlen, subfield := extra[0], extra[1], extra[2:4], extra[4:]
|
||||
if si1 != 'S' || si2 != 'G' {
|
||||
return 0, 0, 0, fmt.Errorf("invalid subfield IDs: %q, %q; want E, S", si1, si2)
|
||||
|
||||
+45
-43
@@ -40,7 +40,6 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/containerd/stargz-snapshotter/estargz/errorutil"
|
||||
"github.com/klauspost/compress/zstd"
|
||||
digest "github.com/opencontainers/go-digest"
|
||||
)
|
||||
@@ -187,15 +186,10 @@ func testBuild(t *TestRunner, controllers ...TestingControllerFactory) {
|
||||
tt.minChunkSize = []int{0}
|
||||
}
|
||||
for _, srcCompression := range srcCompressions {
|
||||
srcCompression := srcCompression
|
||||
for _, newCL := range controllers {
|
||||
newCL := newCL
|
||||
for _, srcTarFormat := range []tar.Format{tar.FormatUSTAR, tar.FormatPAX, tar.FormatGNU} {
|
||||
srcTarFormat := srcTarFormat
|
||||
for _, prefix := range allowedPrefix {
|
||||
prefix := prefix
|
||||
for _, minChunkSize := range tt.minChunkSize {
|
||||
minChunkSize := minChunkSize
|
||||
t.Run(tt.name+"-"+fmt.Sprintf("compression=%v,prefix=%q,src=%d,format=%s,minChunkSize=%d", newCL(), prefix, srcCompression, srcTarFormat, minChunkSize), func(t *TestRunner) {
|
||||
tarBlob := buildTar(t, tt.in, prefix, srcTarFormat)
|
||||
// Test divideEntries()
|
||||
@@ -675,15 +669,10 @@ func testDigestAndVerify(t *TestRunner, controllers ...TestingControllerFactory)
|
||||
tt.minChunkSize = []int{0}
|
||||
}
|
||||
for _, srcCompression := range srcCompressions {
|
||||
srcCompression := srcCompression
|
||||
for _, newCL := range controllers {
|
||||
newCL := newCL
|
||||
for _, prefix := range allowedPrefix {
|
||||
prefix := prefix
|
||||
for _, srcTarFormat := range []tar.Format{tar.FormatUSTAR, tar.FormatPAX, tar.FormatGNU} {
|
||||
srcTarFormat := srcTarFormat
|
||||
for _, minChunkSize := range tt.minChunkSize {
|
||||
minChunkSize := minChunkSize
|
||||
t.Run(tt.name+"-"+fmt.Sprintf("compression=%v,prefix=%q,format=%s,minChunkSize=%d", newCL(), prefix, srcTarFormat, minChunkSize), func(t *TestRunner) {
|
||||
// Get original tar file and chunk digests
|
||||
dgstMap := make(map[string]digest.Digest)
|
||||
@@ -1170,6 +1159,8 @@ func testWriteAndOpen(t *TestRunner, controllers ...TestingControllerFactory) {
|
||||
wantNumGzLossLess int // expected number of streams (> 0) in lossless mode if it's different from wantNumGz
|
||||
wantFailOnLossLess bool
|
||||
wantTOCVersion int // default = 1
|
||||
|
||||
wantOpenError bool // opening this blob should fail with returning an error
|
||||
}{
|
||||
{
|
||||
name: "empty",
|
||||
@@ -1484,15 +1475,19 @@ func testWriteAndOpen(t *TestRunner, controllers ...TestingControllerFactory) {
|
||||
hasFileContentsRange("foo3", len(data64KB)-1, data64KB[len(data64KB)-1:]),
|
||||
),
|
||||
},
|
||||
{
|
||||
name: "cyclic_hardlink",
|
||||
in: tarOf(
|
||||
link("foo", "foo"),
|
||||
),
|
||||
wantOpenError: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
for _, newCL := range controllers {
|
||||
newCL := newCL
|
||||
for _, prefix := range allowedPrefix {
|
||||
prefix := prefix
|
||||
for _, srcTarFormat := range []tar.Format{tar.FormatUSTAR, tar.FormatPAX, tar.FormatGNU} {
|
||||
srcTarFormat := srcTarFormat
|
||||
for _, lossless := range []bool{true, false} {
|
||||
t.Run(tt.name+"-"+fmt.Sprintf("compression=%v,prefix=%q,lossless=%v,format=%s", newCL(), prefix, lossless, srcTarFormat), func(t *TestRunner) {
|
||||
var tr io.Reader = buildTar(t, tt.in, prefix, srcTarFormat)
|
||||
@@ -1558,6 +1553,9 @@ func testWriteAndOpen(t *TestRunner, controllers ...TestingControllerFactory) {
|
||||
WithTelemetry(telemetry),
|
||||
)
|
||||
if err != nil {
|
||||
if tt.wantOpenError {
|
||||
return
|
||||
}
|
||||
t.Fatalf("stargz.Open: %v", err)
|
||||
}
|
||||
if _, ok := r.Lookup(""); !ok {
|
||||
@@ -1634,24 +1632,24 @@ func newCalledTelemetry() (telemetry *Telemetry, check func(needsGetTOC bool) er
|
||||
var getTocLatencyCalled bool
|
||||
var deserializeTocLatencyCalled bool
|
||||
return &Telemetry{
|
||||
func(time.Time) { getFooterLatencyCalled = true },
|
||||
func(time.Time) { getTocLatencyCalled = true },
|
||||
func(time.Time) { deserializeTocLatencyCalled = true },
|
||||
}, func(needsGetTOC bool) error {
|
||||
var allErr []error
|
||||
if !getFooterLatencyCalled {
|
||||
allErr = append(allErr, fmt.Errorf("metrics GetFooterLatency isn't called"))
|
||||
}
|
||||
if needsGetTOC {
|
||||
if !getTocLatencyCalled {
|
||||
allErr = append(allErr, fmt.Errorf("metrics GetTocLatency isn't called"))
|
||||
}
|
||||
}
|
||||
if !deserializeTocLatencyCalled {
|
||||
allErr = append(allErr, fmt.Errorf("metrics DeserializeTocLatency isn't called"))
|
||||
}
|
||||
return errorutil.Aggregate(allErr)
|
||||
func(time.Time) { getFooterLatencyCalled = true },
|
||||
func(time.Time) { getTocLatencyCalled = true },
|
||||
func(time.Time) { deserializeTocLatencyCalled = true },
|
||||
}, func(needsGetTOC bool) error {
|
||||
var allErr []error
|
||||
if !getFooterLatencyCalled {
|
||||
allErr = append(allErr, fmt.Errorf("metrics GetFooterLatency isn't called"))
|
||||
}
|
||||
if needsGetTOC {
|
||||
if !getTocLatencyCalled {
|
||||
allErr = append(allErr, fmt.Errorf("metrics GetTocLatency isn't called"))
|
||||
}
|
||||
}
|
||||
if !deserializeTocLatencyCalled {
|
||||
allErr = append(allErr, fmt.Errorf("metrics DeserializeTocLatency isn't called"))
|
||||
}
|
||||
return errors.Join(allErr...)
|
||||
}
|
||||
}
|
||||
|
||||
func digestFor(content string) string {
|
||||
@@ -2072,7 +2070,7 @@ func (f tarEntryFunc) appendTar(tw *tar.Writer, prefix string, format tar.Format
|
||||
return f(tw, prefix, format)
|
||||
}
|
||||
|
||||
func buildTar(t TestingT, ents []tarEntry, prefix string, opts ...interface{}) *io.SectionReader {
|
||||
func buildTar(t TestingT, ents []tarEntry, prefix string, opts ...any) *io.SectionReader {
|
||||
format := tar.FormatUnknown
|
||||
for _, opt := range opts {
|
||||
switch v := opt.(type) {
|
||||
@@ -2096,7 +2094,7 @@ func buildTar(t TestingT, ents []tarEntry, prefix string, opts ...interface{}) *
|
||||
return io.NewSectionReader(bytes.NewReader(data), 0, int64(len(data)))
|
||||
}
|
||||
|
||||
func dir(name string, opts ...interface{}) tarEntry {
|
||||
func dir(name string, opts ...any) tarEntry {
|
||||
return tarEntryFunc(func(tw *tar.Writer, prefix string, format tar.Format) error {
|
||||
var o owner
|
||||
mode := os.FileMode(0755)
|
||||
@@ -2137,7 +2135,7 @@ type owner struct {
|
||||
gid int
|
||||
}
|
||||
|
||||
func file(name, contents string, opts ...interface{}) tarEntry {
|
||||
func file(name, contents string, opts ...any) tarEntry {
|
||||
return tarEntryFunc(func(tw *tar.Writer, prefix string, format tar.Format) error {
|
||||
var xattrs xAttr
|
||||
var o owner
|
||||
@@ -2164,15 +2162,19 @@ func file(name, contents string, opts ...interface{}) tarEntry {
|
||||
if len(xattrs) > 0 {
|
||||
format = tar.FormatPAX // only PAX supports xattrs
|
||||
}
|
||||
xattrsPAXRecords := make(map[string]string)
|
||||
for k, v := range xattrs {
|
||||
xattrsPAXRecords["SCHILY.xattr."+k] = v
|
||||
}
|
||||
if err := tw.WriteHeader(&tar.Header{
|
||||
Typeflag: tar.TypeReg,
|
||||
Name: prefix + name,
|
||||
Mode: tm,
|
||||
Xattrs: xattrs,
|
||||
Size: int64(len(contents)),
|
||||
Uid: o.uid,
|
||||
Gid: o.gid,
|
||||
Format: format,
|
||||
Typeflag: tar.TypeReg,
|
||||
Name: prefix + name,
|
||||
Mode: tm,
|
||||
PAXRecords: xattrsPAXRecords,
|
||||
Size: int64(len(contents)),
|
||||
Uid: o.uid,
|
||||
Gid: o.gid,
|
||||
Format: format,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -2349,7 +2351,7 @@ func (f fileInfoOnlyMode) Size() int64 { return 0 }
|
||||
func (f fileInfoOnlyMode) Mode() os.FileMode { return os.FileMode(f) }
|
||||
func (f fileInfoOnlyMode) ModTime() time.Time { return time.Now() }
|
||||
func (f fileInfoOnlyMode) IsDir() bool { return os.FileMode(f).IsDir() }
|
||||
func (f fileInfoOnlyMode) Sys() interface{} { return nil }
|
||||
func (f fileInfoOnlyMode) Sys() any { return nil }
|
||||
|
||||
func CheckGzipHasStreams(t TestingT, b []byte, streams []int64) {
|
||||
if len(streams) == 0 {
|
||||
|
||||
+1
-1
@@ -246,7 +246,7 @@ func (fi fileInfo) Name() string { return path.Base(fi.e.Name) }
|
||||
func (fi fileInfo) IsDir() bool { return fi.e.Type == "dir" }
|
||||
func (fi fileInfo) Size() int64 { return fi.e.Size }
|
||||
func (fi fileInfo) ModTime() time.Time { return fi.e.ModTime() }
|
||||
func (fi fileInfo) Sys() interface{} { return fi.e }
|
||||
func (fi fileInfo) Sys() any { return fi.e }
|
||||
func (fi fileInfo) Mode() (m os.FileMode) {
|
||||
// TOCEntry.Mode is tar.Header.Mode so we can understand the these bits using `tar` pkg.
|
||||
m = (&tar.Header{Mode: fi.e.Mode}).FileInfo().Mode() &
|
||||
|
||||
+11
-3
@@ -394,6 +394,11 @@ func (cm *cacheManager) init(ctx context.Context) error {
|
||||
cm.LeaseManager.Delete(ctx, leases.Lease{ID: si.ID() + "-variants"})
|
||||
}
|
||||
}
|
||||
if n, err := cm.MetadataStore.ClearOrphans(); err != nil {
|
||||
bklog.G(ctx).Warnf("failed to remove external metadata of deleted cache records: %v", err)
|
||||
} else if n > 0 {
|
||||
bklog.G(ctx).Infof("removed external metadata of %d deleted cache records", n)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -470,7 +475,10 @@ func (cm *cacheManager) getRecord(ctx context.Context, id string, opts ...RefOpt
|
||||
}
|
||||
|
||||
if rec, ok := cm.records[id]; ok {
|
||||
if rec.isDead() {
|
||||
rec.mu.Lock()
|
||||
isDead := rec.isDead()
|
||||
rec.mu.Unlock()
|
||||
if isDead {
|
||||
return nil, errors.Wrapf(errNotFound, "failed to get dead record %s", id)
|
||||
}
|
||||
if err := checkLazyProviders(rec); err != nil {
|
||||
@@ -498,7 +506,7 @@ func (cm *cacheManager) getRecord(ctx context.Context, id string, opts ...RefOpt
|
||||
mutable, err := cm.getRecord(ctx, mutableID)
|
||||
if err == nil {
|
||||
rec := &cacheRecord{
|
||||
mu: &sync.Mutex{},
|
||||
mu: mutable.mu, // share the mutex as in mutableRef.commit
|
||||
cm: cm,
|
||||
refs: make(map[ref]struct{}),
|
||||
parentRefs: parents,
|
||||
@@ -570,10 +578,10 @@ func (cm *cacheManager) getRecord(ctx context.Context, id string, opts ...RefOpt
|
||||
return nil, errors.Wrapf(err, "failed to append image ref metadata to ref %s", rec.ID())
|
||||
}
|
||||
|
||||
cm.records[id] = rec
|
||||
if err := checkLazyProviders(rec); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cm.records[id] = rec
|
||||
return rec, nil
|
||||
}
|
||||
|
||||
|
||||
+38
@@ -190,6 +190,40 @@ func (s *Store) Clear(id string) error {
|
||||
}))
|
||||
}
|
||||
|
||||
func (s *Store) ClearOrphans() (int, error) {
|
||||
var n int
|
||||
err := s.db.Update(func(tx *bolt.Tx) error {
|
||||
external := tx.Bucket([]byte(externalBucket))
|
||||
if external == nil {
|
||||
return nil
|
||||
}
|
||||
var records *bolt.Cursor
|
||||
if main := tx.Bucket([]byte(mainBucket)); main != nil {
|
||||
records = main.Cursor()
|
||||
}
|
||||
var orphans [][]byte
|
||||
if err := external.ForEach(func(k, _ []byte) error {
|
||||
if records != nil {
|
||||
if id, v := records.Seek(k); bytes.Equal(id, k) && v == nil {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
orphans = append(orphans, append([]byte(nil), k...))
|
||||
return nil
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, id := range orphans {
|
||||
if err := external.DeleteBucket(id); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
n = len(orphans)
|
||||
return nil
|
||||
})
|
||||
return n, errors.WithStack(err)
|
||||
}
|
||||
|
||||
func (s *Store) Update(id string, fn func(b *bolt.Bucket) error) error {
|
||||
return errors.WithStack(s.db.Update(func(tx *bolt.Tx) error {
|
||||
b, err := tx.CreateBucketIfNotExists([]byte(mainBucket))
|
||||
@@ -330,6 +364,10 @@ func (s *StorageItem) GetExternal(k string) ([]byte, error) {
|
||||
|
||||
func (s *StorageItem) SetExternal(k string, dt []byte) error {
|
||||
return errors.WithStack(s.storage.db.Update(func(tx *bolt.Tx) error {
|
||||
main := tx.Bucket([]byte(mainBucket))
|
||||
if main == nil || main.Bucket([]byte(s.id)) == nil {
|
||||
return errors.Wrapf(errNotFound, "record %s", s.id)
|
||||
}
|
||||
b, err := tx.CreateBucketIfNotExists([]byte(externalBucket))
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
|
||||
+79
-30
@@ -103,6 +103,10 @@ type cacheRecord struct {
|
||||
equalMutable *mutableRef
|
||||
equalImmutable *immutableRef
|
||||
|
||||
// layerDigestChainCache memoizes layerDigestChain. It is only set when
|
||||
// every layer has a blob, as a blob can't be replaced after it is set.
|
||||
// Access is serialized by cacheManager.mu, as computing the chain of a
|
||||
// record also fills the memoized chains of its parents.
|
||||
layerDigestChainCache []digest.Digest
|
||||
}
|
||||
|
||||
@@ -554,18 +558,10 @@ func (sr *immutableRef) layerWalk(f func(*immutableRef)) {
|
||||
parent.layerWalk(f)
|
||||
}
|
||||
case Diff:
|
||||
lower := sr.diffParents.lower
|
||||
upper := sr.diffParents.upper
|
||||
// If upper is only one blob different from lower, then re-use that blob
|
||||
switch {
|
||||
case upper != nil && lower == nil && upper.kind() == BaseLayer:
|
||||
// upper is a single layer being diffed with scratch
|
||||
f(upper)
|
||||
case upper != nil && lower != nil && upper.kind() == Layer && upper.layerParent.ID() == lower.ID():
|
||||
// upper is a single layer on top of lower
|
||||
f(upper)
|
||||
default:
|
||||
// otherwise, the diff will be computed and turned into its own single blob
|
||||
if sr.diffReusesUpper() {
|
||||
f(sr.diffParents.upper)
|
||||
} else {
|
||||
// the diff will be computed and turned into its own single blob
|
||||
f(sr)
|
||||
}
|
||||
case Layer:
|
||||
@@ -576,30 +572,65 @@ func (sr *immutableRef) layerWalk(f func(*immutableRef)) {
|
||||
}
|
||||
}
|
||||
|
||||
// hold cacheRecord.mu lock before calling
|
||||
// diffReusesUpper reports whether the Diff record cr is exported as the blob
|
||||
// of its upper, because upper is only one blob different from lower. Such a
|
||||
// diff never gets a blob of its own.
|
||||
func (cr *cacheRecord) diffReusesUpper() bool {
|
||||
lower := cr.diffParents.lower
|
||||
upper := cr.diffParents.upper
|
||||
switch {
|
||||
case upper != nil && lower == nil && upper.kind() == BaseLayer:
|
||||
// upper is a single layer being diffed with scratch
|
||||
return true
|
||||
case upper != nil && lower != nil && upper.kind() == Layer && upper.layerParent.ID() == lower.ID():
|
||||
// upper is a single layer on top of lower
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// layerDigestChain returns the blob digests of the layers of cr, or nil if any
|
||||
// of them has no blob yet. Such a record has never been exported, so no image
|
||||
// can reference it. The caller must hold cacheManager.mu and cacheRecord.mu.
|
||||
func (cr *cacheRecord) layerDigestChain() []digest.Digest {
|
||||
if cr.layerDigestChainCache != nil {
|
||||
return cr.layerDigestChainCache
|
||||
}
|
||||
// The own blob of a record is checked before its parents, so that a
|
||||
// record without a blob doesn't walk its ancestors on every call.
|
||||
var chain []digest.Digest
|
||||
switch cr.kind() {
|
||||
case Diff:
|
||||
if cr.getBlob() == "" && cr.diffParents.upper != nil {
|
||||
// this diff just reuses the upper blob
|
||||
cr.layerDigestChainCache = cr.diffParents.upper.layerDigestChain()
|
||||
} else {
|
||||
cr.layerDigestChainCache = append(cr.layerDigestChainCache, cr.getBlob())
|
||||
if blob := cr.getBlob(); blob != "" {
|
||||
chain = []digest.Digest{blob}
|
||||
} else if cr.diffReusesUpper() {
|
||||
// export uses the blob of upper as the only layer of the diff
|
||||
if blob := cr.diffParents.upper.getBlob(); blob != "" {
|
||||
chain = []digest.Digest{blob}
|
||||
}
|
||||
}
|
||||
case Merge:
|
||||
for _, parent := range cr.mergeParents {
|
||||
cr.layerDigestChainCache = append(cr.layerDigestChainCache, parent.layerDigestChain()...)
|
||||
parentChain := parent.layerDigestChain()
|
||||
if parentChain == nil {
|
||||
return nil
|
||||
}
|
||||
chain = append(chain, parentChain...)
|
||||
}
|
||||
case Layer:
|
||||
cr.layerDigestChainCache = append(cr.layerDigestChainCache, cr.layerParent.layerDigestChain()...)
|
||||
fallthrough
|
||||
if blob := cr.getBlob(); blob != "" {
|
||||
if parentChain := cr.layerParent.layerDigestChain(); parentChain != nil {
|
||||
chain = slices.Concat(parentChain, []digest.Digest{blob})
|
||||
}
|
||||
}
|
||||
case BaseLayer:
|
||||
cr.layerDigestChainCache = append(cr.layerDigestChainCache, cr.getBlob())
|
||||
if blob := cr.getBlob(); blob != "" {
|
||||
chain = []digest.Digest{blob}
|
||||
}
|
||||
}
|
||||
return cr.layerDigestChainCache
|
||||
cr.layerDigestChainCache = chain
|
||||
return chain
|
||||
}
|
||||
|
||||
type RefList []ImmutableRef
|
||||
@@ -967,7 +998,9 @@ func fieldsFromLabels(labels map[string]string) (fields []string) {
|
||||
}
|
||||
|
||||
func (sr *immutableRef) Mount(ctx context.Context, readonly bool, s session.Group) (_ snapshot.Mountable, rerr error) {
|
||||
if sr.equalMutable != nil && !readonly {
|
||||
// Finalize is a no-op if there is no equalMutable. Don't check it here,
|
||||
// equalMutable is protected by sr.mu.
|
||||
if !readonly {
|
||||
if err := sr.Finalize(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -1536,6 +1569,11 @@ func (cr *cacheRecord) finalize(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// The snapshotter may apply the commit even when the caller sees a
|
||||
// cancellation (e.g. a containerd RPC canceled in flight), leaving the
|
||||
// mutable snapshot gone while the record still points to it.
|
||||
ctx = context.WithoutCancel(ctx)
|
||||
|
||||
_, err := cr.cm.LeaseManager.Create(ctx, func(l *leases.Lease) error {
|
||||
l.ID = cr.ID()
|
||||
l.Labels = map[string]string{
|
||||
@@ -1553,13 +1591,23 @@ func (cr *cacheRecord) finalize(ctx context.Context) error {
|
||||
ID: cr.getSnapshotID(),
|
||||
Type: "snapshots/" + cr.cm.Snapshotter.Name(),
|
||||
}); err != nil {
|
||||
_ = cr.cm.LeaseManager.Delete(context.WithoutCancel(ctx), leases.Lease{ID: cr.ID()})
|
||||
_ = cr.cm.LeaseManager.Delete(ctx, leases.Lease{ID: cr.ID()})
|
||||
return errors.Wrapf(err, "failed to add snapshot %s to lease", cr.getSnapshotID())
|
||||
}
|
||||
|
||||
if err := cr.cm.Snapshotter.Commit(ctx, cr.getSnapshotID(), mutable.getSnapshotID()); err != nil {
|
||||
_ = cr.cm.LeaseManager.Delete(context.WithoutCancel(ctx), leases.Lease{ID: cr.ID()})
|
||||
return errors.Wrapf(err, "failed to commit %s to %s during finalize", mutable.getSnapshotID(), cr.getSnapshotID())
|
||||
// the commit may still have been applied, e.g. if the response was lost
|
||||
info, statErr := cr.cm.Snapshotter.Stat(ctx, cr.getSnapshotID())
|
||||
if statErr != nil || info.Kind != snapshots.KindCommitted {
|
||||
// keep the lease if it is unknown whether the commit was applied
|
||||
if statErr == nil || errors.Is(statErr, cerrdefs.ErrNotFound) {
|
||||
_ = cr.cm.LeaseManager.Delete(ctx, leases.Lease{ID: cr.ID()})
|
||||
} else {
|
||||
bklog.G(ctx).WithError(statErr).Warnf("failed to check snapshot %s after commit error, keeping lease", cr.getSnapshotID())
|
||||
}
|
||||
return errors.Wrapf(err, "failed to commit %s to %s during finalize", mutable.getSnapshotID(), cr.getSnapshotID())
|
||||
}
|
||||
bklog.G(ctx).WithError(err).Warnf("snapshot %s was committed to %s despite commit error", mutable.getSnapshotID(), cr.getSnapshotID())
|
||||
}
|
||||
cr.mountCache = nil
|
||||
|
||||
@@ -1567,9 +1615,8 @@ func (cr *cacheRecord) finalize(ctx context.Context) error {
|
||||
go func() {
|
||||
cr.cm.mu.Lock()
|
||||
defer cr.cm.mu.Unlock()
|
||||
cleanupCtx := context.WithoutCancel(ctx)
|
||||
if err := mutable.remove(cleanupCtx, true); err != nil {
|
||||
bklog.G(cleanupCtx).Error(err)
|
||||
if err := mutable.remove(ctx, true); err != nil {
|
||||
bklog.G(ctx).Error(err)
|
||||
}
|
||||
}()
|
||||
|
||||
@@ -1856,6 +1903,8 @@ func (sm *sharableMountable) Mount() (_ []mount.Mount, _ func() error, retErr er
|
||||
return nil, nil, err
|
||||
}
|
||||
}
|
||||
|
||||
mounts = mount.RemoveVolatileOption(mounts)
|
||||
if err := mount.All(mounts, dir); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
+21
-117
@@ -7,7 +7,6 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -17,7 +16,7 @@ import (
|
||||
"github.com/containerd/containerd/v2/pkg/labels"
|
||||
cerrdefs "github.com/containerd/errdefs"
|
||||
"github.com/moby/buildkit/cache/remotecache"
|
||||
"github.com/moby/buildkit/cache/remotecache/gha/ghatypes"
|
||||
"github.com/moby/buildkit/cache/remotecache/signing"
|
||||
v1 "github.com/moby/buildkit/cache/remotecache/v1"
|
||||
cacheimporttypes "github.com/moby/buildkit/cache/remotecache/v1/types"
|
||||
"github.com/moby/buildkit/session"
|
||||
@@ -28,11 +27,9 @@ import (
|
||||
"github.com/moby/buildkit/util/tracing"
|
||||
bkversion "github.com/moby/buildkit/version"
|
||||
"github.com/moby/buildkit/worker"
|
||||
policy "github.com/moby/policy-helpers"
|
||||
digest "github.com/opencontainers/go-digest"
|
||||
ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/sigstore/sigstore-go/pkg/fulcio/certificate"
|
||||
actionscache "github.com/tonistiigi/go-actions-cache"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
@@ -55,8 +52,6 @@ const (
|
||||
defaultTimeout = 10 * time.Minute
|
||||
)
|
||||
|
||||
type VerifierProvider func() (*policy.Verifier, error)
|
||||
|
||||
type Config struct {
|
||||
Scope string
|
||||
URL string
|
||||
@@ -66,11 +61,11 @@ type Config struct {
|
||||
Version int
|
||||
Timeout time.Duration
|
||||
|
||||
*ghatypes.CacheConfig
|
||||
verifier VerifierProvider
|
||||
*signing.Config
|
||||
verifier signing.Verifier
|
||||
}
|
||||
|
||||
func getConfig(conf *ghatypes.CacheConfig, v VerifierProvider, attrs map[string]string) (*Config, error) {
|
||||
func getConfig(conf *signing.Config, v signing.VerifierProvider, attrs map[string]string) (*Config, error) {
|
||||
scope, ok := attrs[attrScope]
|
||||
if !ok {
|
||||
scope = "buildkit"
|
||||
@@ -120,24 +115,24 @@ func getConfig(conf *ghatypes.CacheConfig, v VerifierProvider, attrs map[string]
|
||||
}
|
||||
|
||||
if conf == nil {
|
||||
conf = &ghatypes.CacheConfig{}
|
||||
conf = &signing.Config{}
|
||||
}
|
||||
|
||||
return &Config{
|
||||
Scope: scope,
|
||||
URL: url,
|
||||
Token: token,
|
||||
Timeout: timeout,
|
||||
GHToken: attrs[attrGHToken],
|
||||
Repository: attrs[attrRepository],
|
||||
Version: apiVersionInt,
|
||||
CacheConfig: conf,
|
||||
verifier: v,
|
||||
Scope: scope,
|
||||
URL: url,
|
||||
Token: token,
|
||||
Timeout: timeout,
|
||||
GHToken: attrs[attrGHToken],
|
||||
Repository: attrs[attrRepository],
|
||||
Version: apiVersionInt,
|
||||
Config: conf,
|
||||
verifier: signing.NewVerifier(conf.Verify.Policy, v),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ResolveCacheExporterFunc for Github actions cache exporter.
|
||||
func ResolveCacheExporterFunc(conf *ghatypes.CacheConfig, v VerifierProvider) remotecache.ResolveCacheExporterFunc {
|
||||
func ResolveCacheExporterFunc(conf *signing.Config, v signing.VerifierProvider) remotecache.ResolveCacheExporterFunc {
|
||||
return func(ctx context.Context, g session.Group, attrs map[string]string) (remotecache.Exporter, error) {
|
||||
cfg, err := getConfig(conf, v, attrs)
|
||||
if err != nil {
|
||||
@@ -330,98 +325,23 @@ func (ce *exporter) Finalize(ctx context.Context) (_ map[string]string, err erro
|
||||
|
||||
dgst := digest.FromBytes(dt)
|
||||
signDone := progress.OneOff(ctx, fmt.Sprintf("signing cache index %s", dgst))
|
||||
defer signDone(err)
|
||||
defer func() { signDone(err) }()
|
||||
|
||||
cmd := exec.CommandContext(ctx, args[0], args[1:]...) //nolint:gosec // defined in toml config
|
||||
cmd.Stdin = bytes.NewReader(dt)
|
||||
var out bytes.Buffer
|
||||
cmd.Stdout = &out
|
||||
var stderr bytes.Buffer
|
||||
cmd.Stderr = &stderr
|
||||
if err := cmd.Run(); err != nil {
|
||||
return nil, errors.Wrapf(err, "signing command failed: %s", stderr.String())
|
||||
}
|
||||
|
||||
// validate signature before uploading
|
||||
if err := verifySignature(ctx, dgst, out.Bytes(), ce.config); err != nil {
|
||||
bundle, err := signing.NewSigner(args, ce.config.verifier).Sign(ctx, dt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
key := blobKey(dgst + "-sig")
|
||||
if err := ce.cache.Save(ctx, key, actionscache.NewBlob(out.Bytes())); err != nil {
|
||||
if err := ce.cache.Save(ctx, key, actionscache.NewBlob(bundle)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func verifySignature(ctx context.Context, dgst digest.Digest, bundle []byte, config *Config) error {
|
||||
v, err := config.verifier()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if v == nil {
|
||||
return errors.New("no verifier available for signed github actions cache")
|
||||
}
|
||||
|
||||
sig, err := v.VerifyArtifact(ctx, dgst, bundle, policy.WithSLSANotRequired())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if sig.Signer == nil {
|
||||
return errors.New("signature verification failed: no signer found")
|
||||
}
|
||||
numTimestamps := len(sig.Timestamps)
|
||||
numTlog := 0
|
||||
for _, t := range sig.Timestamps {
|
||||
if t.Type == "Tlog" {
|
||||
numTlog++
|
||||
}
|
||||
}
|
||||
policyRules := config.Verify.Policy
|
||||
if policyRules.TimestampThreshold > numTimestamps {
|
||||
return errors.Errorf("signature verification failed: not enough timestamp authorities: have %d, need %d", numTimestamps, policyRules.TimestampThreshold)
|
||||
}
|
||||
if policyRules.TlogThreshold > numTlog {
|
||||
return errors.Errorf("signature verification failed: not enough tlog authorities: have %d, need %d", numTlog, policyRules.TlogThreshold)
|
||||
}
|
||||
|
||||
certRules, err := certToStringMap(&config.Verify.Policy.Summary)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
certFields, err := certToStringMap(sig.Signer)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
bklog.G(ctx).Debugf("signature verification: %+v", sig)
|
||||
bklog.G(ctx).Debugf("signer: %+v", sig.Signer)
|
||||
for k, v := range certRules {
|
||||
if v == "" {
|
||||
continue
|
||||
}
|
||||
if !simplePatternMatch(v, certFields[k]) {
|
||||
return errors.Errorf("signature verification failed: certificate field %q does not match policy (%q != %q)", k, certFields[k], v)
|
||||
}
|
||||
bklog.G(ctx).Debugf("certificate field %q matches policy (%q)", k, certFields[k])
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func certToStringMap(cert *certificate.Summary) (map[string]string, error) {
|
||||
dt, err := json.Marshal(cert)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m := map[string]string{}
|
||||
if err := json.Unmarshal(dt, &m); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
// ResolveCacheImporterFunc for Github actions cache importer.
|
||||
func ResolveCacheImporterFunc(conf *ghatypes.CacheConfig, v VerifierProvider) remotecache.ResolveCacheImporterFunc {
|
||||
func ResolveCacheImporterFunc(conf *signing.Config, v signing.VerifierProvider) remotecache.ResolveCacheImporterFunc {
|
||||
return func(ctx context.Context, g session.Group, attrs map[string]string) (remotecache.Importer, ocispecs.Descriptor, error) {
|
||||
cfg, err := getConfig(conf, v, attrs)
|
||||
if err != nil {
|
||||
@@ -518,7 +438,7 @@ func (ci *importer) loadScope(ctx context.Context, scope string) (*v1.CacheChain
|
||||
verifyDone(err)
|
||||
return nil, err
|
||||
}
|
||||
if err := verifySignature(ctx, dgst, sigBuf.Bytes(), ci.config); err != nil {
|
||||
if err := ci.config.verifier.Verify(ctx, dgst, sigBuf.Bytes()); err != nil {
|
||||
verifyDone(err)
|
||||
return nil, err
|
||||
}
|
||||
@@ -648,19 +568,3 @@ func (r *readerAt) ReadAt(p []byte, off int64) (int, error) {
|
||||
func (r *readerAt) Size() int64 {
|
||||
return r.desc.Size
|
||||
}
|
||||
|
||||
func simplePatternMatch(pat, s string) bool {
|
||||
if pat == "*" {
|
||||
return true
|
||||
}
|
||||
if strings.HasPrefix(pat, "*") && strings.HasSuffix(pat, "*") {
|
||||
return strings.Contains(s, pat[1:len(pat)-1])
|
||||
}
|
||||
if strings.HasPrefix(pat, "*") {
|
||||
return strings.HasSuffix(s, pat[1:])
|
||||
}
|
||||
if strings.HasSuffix(pat, "*") {
|
||||
return strings.HasPrefix(s, pat[:len(pat)-1])
|
||||
}
|
||||
return s == pat
|
||||
}
|
||||
-23
@@ -1,23 +0,0 @@
|
||||
package ghatypes
|
||||
|
||||
import "github.com/sigstore/sigstore-go/pkg/fulcio/certificate"
|
||||
|
||||
type CacheConfig struct {
|
||||
Sign *SignConfig `toml:"sign"`
|
||||
Verify VerifyConfig `toml:"verify"`
|
||||
}
|
||||
|
||||
type SignConfig struct {
|
||||
Command []string `toml:"command"`
|
||||
}
|
||||
|
||||
type VerifyConfig struct {
|
||||
Required bool `toml:"required"`
|
||||
Policy VerifyPolicy `toml:"policy"`
|
||||
}
|
||||
|
||||
type VerifyPolicy struct {
|
||||
TimestampThreshold int `toml:"timestampThreshold"`
|
||||
TlogThreshold int `toml:"tlogThreshold"`
|
||||
certificate.Summary
|
||||
}
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
package signing
|
||||
|
||||
import "github.com/sigstore/sigstore-go/pkg/fulcio/certificate"
|
||||
|
||||
// Config controls signing and verification of remote cache metadata.
|
||||
type Config struct {
|
||||
Sign *SignConfig `toml:"sign"`
|
||||
Verify VerifyConfig `toml:"verify"`
|
||||
}
|
||||
|
||||
// SignConfig specifies the command that reads a payload and writes its bundle.
|
||||
type SignConfig struct {
|
||||
Command []string `toml:"command"`
|
||||
}
|
||||
|
||||
// VerifyConfig controls whether imported cache metadata must be verified.
|
||||
type VerifyConfig struct {
|
||||
Required bool `toml:"required"`
|
||||
Policy VerifyPolicy `toml:"policy"`
|
||||
}
|
||||
|
||||
// VerifyPolicy constrains the verified timestamps and signing certificate.
|
||||
type VerifyPolicy struct {
|
||||
TimestampThreshold int `toml:"timestampThreshold"`
|
||||
TlogThreshold *int `toml:"tlogThreshold"`
|
||||
certificate.Summary
|
||||
}
|
||||
+48
@@ -0,0 +1,48 @@
|
||||
package signing
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"os/exec"
|
||||
"slices"
|
||||
|
||||
digest "github.com/opencontainers/go-digest"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// Signer signs the exact payload bytes and verifies the resulting bundle.
|
||||
type Signer interface {
|
||||
Sign(context.Context, []byte) ([]byte, error)
|
||||
}
|
||||
|
||||
// NewSigner returns a command-based signer using verifier to check its output.
|
||||
// The command is an argv array configured by the daemon administrator.
|
||||
func NewSigner(command []string, verifier Verifier) Signer {
|
||||
return &commandSigner{command: slices.Clone(command), verifier: verifier}
|
||||
}
|
||||
|
||||
type commandSigner struct {
|
||||
command []string
|
||||
verifier Verifier
|
||||
}
|
||||
|
||||
func (s *commandSigner) Sign(ctx context.Context, payload []byte) ([]byte, error) {
|
||||
if len(s.command) == 0 {
|
||||
return nil, errors.New("no signing command configured")
|
||||
}
|
||||
if s.verifier == nil {
|
||||
return nil, errors.New("no verifier available for signed cache")
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, s.command[0], s.command[1:]...) //nolint:gosec // defined in toml config
|
||||
cmd.Stdin = bytes.NewReader(payload)
|
||||
var out, stderr bytes.Buffer
|
||||
cmd.Stdout = &out
|
||||
cmd.Stderr = &stderr
|
||||
if err := cmd.Run(); err != nil {
|
||||
return nil, errors.Wrapf(err, "signing command failed: %s", stderr.String())
|
||||
}
|
||||
if err := s.verifier.Verify(ctx, digest.FromBytes(payload), out.Bytes()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out.Bytes(), nil
|
||||
}
|
||||
+114
@@ -0,0 +1,114 @@
|
||||
package signing
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
|
||||
"github.com/moby/buildkit/util/bklog"
|
||||
policy "github.com/moby/policy-helpers"
|
||||
"github.com/moby/policy-helpers/types"
|
||||
digest "github.com/opencontainers/go-digest"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/sigstore/sigstore-go/pkg/fulcio/certificate"
|
||||
)
|
||||
|
||||
// Verifier authenticates a bundle against the digest and the configured policy.
|
||||
type Verifier interface {
|
||||
Verify(context.Context, digest.Digest, []byte) error
|
||||
}
|
||||
|
||||
// VerifierProvider lazily supplies the daemon's shared Sigstore verifier.
|
||||
type VerifierProvider func() (*policy.Verifier, error)
|
||||
|
||||
// NewVerifier returns a verifier applying the given policy to cache signatures.
|
||||
func NewVerifier(rules VerifyPolicy, provider VerifierProvider) Verifier {
|
||||
return &verifier{rules: rules, provider: provider}
|
||||
}
|
||||
|
||||
type verifier struct {
|
||||
rules VerifyPolicy
|
||||
provider VerifierProvider
|
||||
}
|
||||
|
||||
func (v *verifier) Verify(ctx context.Context, dgst digest.Digest, bundle []byte) error {
|
||||
if v.provider == nil {
|
||||
return errors.New("no verifier available for signed cache")
|
||||
}
|
||||
artifactVerifier, err := v.provider()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if artifactVerifier == nil {
|
||||
return errors.New("no verifier available for signed cache")
|
||||
}
|
||||
opts := []policy.ArtifactVerifyOpt{policy.WithSLSANotRequired()}
|
||||
if v.rules.TlogThreshold != nil {
|
||||
opts = append(opts, policy.WithTlogThreshold(*v.rules.TlogThreshold))
|
||||
}
|
||||
sig, err := artifactVerifier.VerifyArtifact(ctx, dgst, bundle, opts...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return verifyPolicy(ctx, sig, v.rules)
|
||||
}
|
||||
|
||||
// verifyPolicy checks the authenticated certificate and timestamps.
|
||||
func verifyPolicy(ctx context.Context, sig *types.SignatureInfo, policyRules VerifyPolicy) error {
|
||||
if sig.Signer == nil {
|
||||
return errors.New("signature verification failed: no signer found")
|
||||
}
|
||||
numTimestamps := len(sig.Timestamps)
|
||||
if policyRules.TimestampThreshold > numTimestamps {
|
||||
return errors.Errorf("signature verification failed: not enough timestamp authorities: have %d, need %d", numTimestamps, policyRules.TimestampThreshold)
|
||||
}
|
||||
|
||||
certRules, err := certToStringMap(&policyRules.Summary)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
certFields, err := certToStringMap(sig.Signer)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
bklog.G(ctx).Debugf("signature verification: %+v", sig)
|
||||
bklog.G(ctx).Debugf("signer: %+v", sig.Signer)
|
||||
for k, v := range certRules {
|
||||
if v == "" {
|
||||
continue
|
||||
}
|
||||
if !simplePatternMatch(v, certFields[k]) {
|
||||
return errors.Errorf("signature verification failed: certificate field %q does not match policy (%q != %q)", k, certFields[k], v)
|
||||
}
|
||||
bklog.G(ctx).Debugf("certificate field %q matches policy (%q)", k, certFields[k])
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func certToStringMap(cert *certificate.Summary) (map[string]string, error) {
|
||||
dt, err := json.Marshal(cert)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m := map[string]string{}
|
||||
if err := json.Unmarshal(dt, &m); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func simplePatternMatch(pat, s string) bool {
|
||||
if pat == "*" {
|
||||
return true
|
||||
}
|
||||
if strings.HasPrefix(pat, "*") && strings.HasSuffix(pat, "*") {
|
||||
return strings.Contains(s, pat[1:len(pat)-1])
|
||||
}
|
||||
if strings.HasPrefix(pat, "*") {
|
||||
return strings.HasSuffix(s, pat[1:])
|
||||
}
|
||||
if strings.HasSuffix(pat, "*") {
|
||||
return strings.HasPrefix(s, pat[:len(pat)-1])
|
||||
}
|
||||
return s == pat
|
||||
}
|
||||
+25
-32
@@ -10,6 +10,7 @@ import (
|
||||
cerrdefs "github.com/containerd/errdefs"
|
||||
cacheimporttypes "github.com/moby/buildkit/cache/remotecache/v1/types"
|
||||
"github.com/moby/buildkit/solver"
|
||||
"github.com/moby/buildkit/util/bklog"
|
||||
"github.com/moby/buildkit/util/compression"
|
||||
digest "github.com/opencontainers/go-digest"
|
||||
"github.com/pkg/errors"
|
||||
@@ -136,48 +137,40 @@ func marshalRemote(ctx context.Context, r *solver.Remote, state *marshalState) s
|
||||
|
||||
if r.Provider != nil {
|
||||
for _, d := range r.Descriptors {
|
||||
if _, err := r.Provider.Info(ctx, d.Digest); err != nil {
|
||||
if !cerrdefs.IsNotImplemented(err) {
|
||||
return ""
|
||||
}
|
||||
if _, err := r.Provider.Info(ctx, d.Digest); err != nil && !cerrdefs.IsNotImplemented(err) {
|
||||
bklog.G(ctx).WithError(err).Warnf("skipping cache result: blob %s is not available", d.Digest)
|
||||
return ""
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var parentID string
|
||||
if len(r.Descriptors) > 1 {
|
||||
r2 := &solver.Remote{
|
||||
Descriptors: r.Descriptors[:len(r.Descriptors)-1],
|
||||
Provider: r.Provider,
|
||||
for _, desc := range r.Descriptors {
|
||||
if desc.MediaType != "" {
|
||||
desc = compression.ConvertAllLayerMediaTypes(ctx, true, desc)[0]
|
||||
}
|
||||
parentID = marshalRemote(ctx, r2, state)
|
||||
}
|
||||
desc := r.Descriptors[len(r.Descriptors)-1]
|
||||
if desc.MediaType != "" {
|
||||
desc = compression.ConvertAllLayerMediaTypes(ctx, true, desc)[0]
|
||||
}
|
||||
|
||||
state.descriptors[desc.Digest] = DescriptorProviderPair{
|
||||
Descriptor: desc,
|
||||
Provider: r.Provider,
|
||||
}
|
||||
state.descriptors[desc.Digest] = DescriptorProviderPair{
|
||||
Descriptor: desc,
|
||||
Provider: r.Provider,
|
||||
}
|
||||
|
||||
id := desc.Digest.String() + parentID
|
||||
id := desc.Digest.String() + parentID
|
||||
|
||||
if _, ok := state.chainsByID[id]; ok {
|
||||
return id
|
||||
if _, ok := state.chainsByID[id]; !ok {
|
||||
state.chainsByID[id] = len(state.layers)
|
||||
l := cacheimporttypes.CacheLayer{
|
||||
Blob: desc.Digest,
|
||||
ParentIndex: -1,
|
||||
}
|
||||
if parentID != "" {
|
||||
l.ParentIndex = state.chainsByID[parentID]
|
||||
}
|
||||
state.layers = append(state.layers, l)
|
||||
}
|
||||
parentID = id
|
||||
}
|
||||
|
||||
state.chainsByID[id] = len(state.layers)
|
||||
l := cacheimporttypes.CacheLayer{
|
||||
Blob: desc.Digest,
|
||||
ParentIndex: -1,
|
||||
}
|
||||
if parentID != "" {
|
||||
l.ParentIndex = state.chainsByID[parentID]
|
||||
}
|
||||
state.layers = append(state.layers, l)
|
||||
return id
|
||||
return parentID
|
||||
}
|
||||
|
||||
func marshalItem(ctx context.Context, it *item, state *marshalState) error {
|
||||
|
||||
+4
-1
@@ -262,7 +262,10 @@ func parseUID(str string) (int, error) {
|
||||
if str == "root" {
|
||||
return 0, nil
|
||||
}
|
||||
uid, err := strconv.ParseInt(str, 10, 32)
|
||||
// uid_t/gid_t span 0..2^32-1, matching the daemon-side parser in
|
||||
// executor/oci. ParseInt here would reject the upper half of that range and
|
||||
// silently turn a numeric ID into a name lookup.
|
||||
uid, err := strconv.ParseUint(str, 10, 32)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
+15
@@ -61,6 +61,21 @@ func addEnvf(key, value string, replace bool, v ...any) StateOption {
|
||||
}
|
||||
}
|
||||
|
||||
// DelEnv returns a [StateOption] which removes an environment variable from the state.
|
||||
// This is the equivalent of [State.DelEnv]
|
||||
func DelEnv(key string) StateOption {
|
||||
return func(s State) State {
|
||||
return s.withValue(keyEnv, func(ctx context.Context, c *Constraints) (any, error) {
|
||||
env, err := getEnv(s)(ctx, c)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
deleted := env.Delete(key)
|
||||
return &deleted, nil
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Dir returns a [StateOption] sets the working directory for the state which will be used to resolve
|
||||
// relative paths as well as the working directory for [State.Run].
|
||||
// See [State.With] for where to use this.
|
||||
|
||||
+12
@@ -477,6 +477,9 @@ func Git(url, fragment string, opts ...GitOption) State {
|
||||
attrs[pb.AttrGitSkipSubmodules] = "true"
|
||||
addCap(&gi.Constraints, pb.CapSourceGitSkipSubmodules)
|
||||
}
|
||||
if gi.DebugCommands {
|
||||
attrs[pb.AttrGitDebugCommands] = "true"
|
||||
}
|
||||
|
||||
if gi.MTime != "" {
|
||||
attrs[pb.AttrGitMTime] = gi.MTime
|
||||
@@ -536,6 +539,7 @@ type GitInfo struct {
|
||||
SkipSubmodules bool
|
||||
MTime string
|
||||
Advice bool
|
||||
DebugCommands bool
|
||||
Bundle string
|
||||
BundleOCISessionID string
|
||||
BundleOCIStoreID string
|
||||
@@ -583,6 +587,14 @@ func GitAdvice(enabled bool) GitOption {
|
||||
})
|
||||
}
|
||||
|
||||
// GitDebugCommands controls whether executed Git commands are logged while
|
||||
// resolving this git source.
|
||||
func GitDebugCommands(enabled bool) GitOption {
|
||||
return gitOptionFunc(func(gi *GitInfo) {
|
||||
gi.DebugCommands = enabled
|
||||
})
|
||||
}
|
||||
|
||||
func KeepGitDir() GitOption {
|
||||
return gitOptionFunc(func(gi *GitInfo) {
|
||||
gi.KeepGitDir = true
|
||||
|
||||
+6
@@ -326,6 +326,12 @@ func (s State) AddEnvf(key, value string, v ...any) State {
|
||||
return AddEnvf(key, value, v...)(s)
|
||||
}
|
||||
|
||||
// DelEnv returns a new [State] with the provided environment variable removed.
|
||||
// See [DelEnv]
|
||||
func (s State) DelEnv(key string) State {
|
||||
return DelEnv(key)(s)
|
||||
}
|
||||
|
||||
// Dir returns a new [State] with the provided working directory set.
|
||||
// See [Dir]
|
||||
func (s State) Dir(str string) State {
|
||||
|
||||
+113
-8
@@ -17,6 +17,7 @@ import (
|
||||
"github.com/containerd/containerd/v2/core/content"
|
||||
"github.com/containerd/containerd/v2/core/images"
|
||||
contentlocal "github.com/containerd/containerd/v2/plugins/content/local"
|
||||
cerrdefs "github.com/containerd/errdefs"
|
||||
"github.com/gofrs/flock"
|
||||
controlapi "github.com/moby/buildkit/api/services/control"
|
||||
"github.com/moby/buildkit/client/llb"
|
||||
@@ -425,9 +426,13 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG
|
||||
if err = json.Unmarshal([]byte(manifestDescJSON), &manifestDesc); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for storePath, tag := range cacheOpt.storesToUpdate {
|
||||
for storePath, store := range cacheOpt.storesToUpdate {
|
||||
idx := ociindex.NewStoreIndex(storePath)
|
||||
if err := idx.Put(manifestDesc, ociindex.Tag(tag)); err != nil {
|
||||
if err := idx.Put(manifestDesc, ociindex.Tag(store.tag)); err != nil {
|
||||
if store.ignoreError {
|
||||
bklog.G(callerCtx).WithError(err).WithField("cache_store_path", storePath).Warn("failed to update cache index, ignoring")
|
||||
continue
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
@@ -605,11 +610,16 @@ type cacheStoreRef struct {
|
||||
store content.Store
|
||||
}
|
||||
|
||||
type cacheStoreUpdate struct {
|
||||
tag string
|
||||
ignoreError bool
|
||||
}
|
||||
|
||||
type cacheOptions struct {
|
||||
options controlapi.CacheOptions
|
||||
contentStores map[string]content.Store // key: ID of content store ("local:" + csDir)
|
||||
storesToUpdate map[string]string // key: path to content store, value: tag
|
||||
storesToReset []cacheStoreRef // cache stores with reset=true
|
||||
storesToUpdate map[string]cacheStoreUpdate
|
||||
storesToReset []cacheStoreRef // cache stores with reset=true
|
||||
frontendAttrs map[string]string
|
||||
}
|
||||
|
||||
@@ -620,7 +630,7 @@ func parseCacheOptions(ctx context.Context, isGateway bool, opt SolveOpt) (*cach
|
||||
)
|
||||
var storesToReset []cacheStoreRef
|
||||
contentStores := make(map[string]content.Store)
|
||||
storesToUpdate := make(map[string]string)
|
||||
storesToUpdate := make(map[string]cacheStoreUpdate)
|
||||
frontendAttrs := make(map[string]string)
|
||||
for _, ex := range opt.CacheExports {
|
||||
if ex.Type == "local" {
|
||||
@@ -631,7 +641,7 @@ func parseCacheOptions(ctx context.Context, isGateway bool, opt SolveOpt) (*cach
|
||||
if err := os.MkdirAll(csDir, 0755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cs, err := contentlocal.NewStore(csDir)
|
||||
cs, err := newLocalCacheStore(csDir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -641,8 +651,14 @@ func parseCacheOptions(ctx context.Context, isGateway bool, opt SolveOpt) (*cach
|
||||
if t, ok := ex.Attrs["tag"]; ok {
|
||||
tag = t
|
||||
}
|
||||
update := cacheStoreUpdate{tag: tag}
|
||||
if v, ok := ex.Attrs["ignore-error"]; ok {
|
||||
if b, err := strconv.ParseBool(v); err == nil {
|
||||
update.ignoreError = b
|
||||
}
|
||||
}
|
||||
// TODO(AkihiroSuda): support custom index JSON path and tag
|
||||
storesToUpdate[csDir] = tag
|
||||
storesToUpdate[csDir] = update
|
||||
|
||||
if v, ok := ex.Attrs["reset"]; ok {
|
||||
b, err := strconv.ParseBool(v)
|
||||
@@ -671,7 +687,7 @@ func parseCacheOptions(ctx context.Context, isGateway bool, opt SolveOpt) (*cach
|
||||
if csDir == "" {
|
||||
return nil, errors.New("local cache importer requires src")
|
||||
}
|
||||
cs, err := contentlocal.NewStore(csDir)
|
||||
cs, err := newLocalCacheStore(csDir)
|
||||
if err != nil {
|
||||
bklog.G(ctx).Warning("local cache import at " + csDir + " not found due to err: " + err.Error())
|
||||
continue
|
||||
@@ -737,3 +753,92 @@ func parseCacheOptions(ctx context.Context, isGateway bool, opt SolveOpt) (*cach
|
||||
}
|
||||
return &res, nil
|
||||
}
|
||||
|
||||
// localCacheStore coordinates ingests across clients sharing a cache directory.
|
||||
// The underlying local store only locks references within one store instance.
|
||||
type localCacheStore struct {
|
||||
content.Store
|
||||
dir string
|
||||
}
|
||||
|
||||
func newLocalCacheStore(dir string) (content.Store, error) {
|
||||
store, err := contentlocal.NewStore(dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &localCacheStore{Store: store, dir: dir}, nil
|
||||
}
|
||||
|
||||
func (s *localCacheStore) lock(ref string) (*flock.Flock, error) {
|
||||
dir := filepath.Join(s.dir, "ingest-locks")
|
||||
if err := os.MkdirAll(dir, 0755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Keep lock files after unlocking: deleting one could let another client
|
||||
// lock a different inode while an existing waiter still uses the old one.
|
||||
// This retains one empty file per ref, even after cache reset. The reset
|
||||
// lock cannot protect their removal: not all users of this store hold it.
|
||||
lock := flock.New(filepath.Join(dir, digest.FromString(ref).Encoded()))
|
||||
locked, err := lock.TryLock()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !locked {
|
||||
return nil, errors.Wrapf(cerrdefs.ErrUnavailable, "ingest ref %q is locked", ref)
|
||||
}
|
||||
return lock, nil
|
||||
}
|
||||
|
||||
func (s *localCacheStore) Writer(ctx context.Context, opts ...content.WriterOpt) (content.Writer, error) {
|
||||
var wo content.WriterOpts
|
||||
for _, opt := range opts {
|
||||
if err := opt(&wo); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if wo.Ref == "" {
|
||||
return nil, errors.Wrap(cerrdefs.ErrInvalidArgument, "ref must not be empty")
|
||||
}
|
||||
lock, err := s.lock(wo.Ref)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
w, err := s.Store.Writer(ctx, content.WithRef(wo.Ref), content.WithDescriptor(wo.Desc))
|
||||
if err != nil {
|
||||
lock.Close()
|
||||
return nil, err
|
||||
}
|
||||
return &localCacheWriter{Writer: w, lock: lock}, nil
|
||||
}
|
||||
|
||||
func (s *localCacheStore) Abort(ctx context.Context, ref string) error {
|
||||
lock, err := s.lock(ref)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer lock.Close()
|
||||
return s.Store.Abort(ctx, ref)
|
||||
}
|
||||
|
||||
type localCacheWriter struct {
|
||||
content.Writer
|
||||
lock *flock.Flock
|
||||
}
|
||||
|
||||
func (w *localCacheWriter) Commit(ctx context.Context, size int64, expected digest.Digest, opts ...content.Opt) error {
|
||||
err := w.Writer.Commit(ctx, size, expected, opts...)
|
||||
closeErr := w.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return closeErr
|
||||
}
|
||||
|
||||
func (w *localCacheWriter) Close() error {
|
||||
err := w.Writer.Close()
|
||||
lockErr := w.lock.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return lockErr
|
||||
}
|
||||
+75
-61
@@ -5,15 +5,11 @@ import (
|
||||
|
||||
controlapi "github.com/moby/buildkit/api/services/control"
|
||||
digest "github.com/opencontainers/go-digest"
|
||||
"google.golang.org/protobuf/encoding/protowire"
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
)
|
||||
|
||||
var emptyLogVertexSize int
|
||||
|
||||
func init() {
|
||||
emptyLogVertex := controlapi.VertexLog{}
|
||||
emptyLogVertexSize = emptyLogVertex.SizeVT()
|
||||
}
|
||||
const statusResponseSize = 1024 * 1024
|
||||
|
||||
func NewSolveStatus(resp *controlapi.StatusResponse) *SolveStatus {
|
||||
s := &SolveStatus{}
|
||||
@@ -63,68 +59,86 @@ func NewSolveStatus(resp *controlapi.StatusResponse) *SolveStatus {
|
||||
return s
|
||||
}
|
||||
|
||||
// Marshal batches status updates by encoded size without modifying SolveStatus.
|
||||
// Logs may be split across responses. Individual metadata entries larger than
|
||||
// the batch target are emitted intact in their own response.
|
||||
func (ss *SolveStatus) Marshal() (out []*controlapi.StatusResponse) {
|
||||
logSize := 0
|
||||
for {
|
||||
retry := false
|
||||
sr := controlapi.StatusResponse{}
|
||||
for _, v := range ss.Vertexes {
|
||||
sr.Vertexes = append(sr.Vertexes, &controlapi.Vertex{
|
||||
Digest: string(v.Digest),
|
||||
Inputs: digestSliceToPB(v.Inputs),
|
||||
Name: v.Name,
|
||||
Started: timestampToPB(v.Started),
|
||||
Completed: timestampToPB(v.Completed),
|
||||
Error: v.Error,
|
||||
Cached: v.Cached,
|
||||
ProgressGroup: v.ProgressGroup,
|
||||
})
|
||||
sr := &controlapi.StatusResponse{}
|
||||
size := 0
|
||||
|
||||
// Each repeated field has a one-byte tag and a length-prefixed message.
|
||||
// Keep indivisible metadata entries intact even if they exceed the target.
|
||||
reserve := func(n int) {
|
||||
n = 1 + protowire.SizeBytes(n)
|
||||
if size > 0 && size+n > statusResponseSize {
|
||||
out = append(out, sr)
|
||||
sr = &controlapi.StatusResponse{}
|
||||
size = 0
|
||||
}
|
||||
for _, v := range ss.Statuses {
|
||||
sr.Statuses = append(sr.Statuses, &controlapi.VertexStatus{
|
||||
ID: v.ID,
|
||||
Vertex: string(v.Vertex),
|
||||
Name: v.Name,
|
||||
Current: v.Current,
|
||||
Total: v.Total,
|
||||
Timestamp: timestamppb.New(v.Timestamp),
|
||||
Started: timestampToPB(v.Started),
|
||||
Completed: timestampToPB(v.Completed),
|
||||
})
|
||||
size += n
|
||||
}
|
||||
|
||||
for _, v := range ss.Vertexes {
|
||||
p := &controlapi.Vertex{
|
||||
Digest: string(v.Digest),
|
||||
Inputs: digestSliceToPB(v.Inputs),
|
||||
Name: v.Name,
|
||||
Started: timestampToPB(v.Started),
|
||||
Completed: timestampToPB(v.Completed),
|
||||
Error: v.Error,
|
||||
Cached: v.Cached,
|
||||
ProgressGroup: v.ProgressGroup,
|
||||
}
|
||||
for i, v := range ss.Logs {
|
||||
sr.Logs = append(sr.Logs, &controlapi.VertexLog{
|
||||
Vertex: string(v.Vertex),
|
||||
Stream: int64(v.Stream),
|
||||
Msg: v.Data,
|
||||
Timestamp: timestamppb.New(v.Timestamp),
|
||||
})
|
||||
logSize += len(v.Data) + emptyLogVertexSize
|
||||
// avoid logs growing big and split apart if they do
|
||||
if logSize > 1024*1024 {
|
||||
ss.Vertexes = nil
|
||||
ss.Statuses = nil
|
||||
ss.Logs = ss.Logs[i+1:]
|
||||
retry = true
|
||||
reserve(p.SizeVT())
|
||||
sr.Vertexes = append(sr.Vertexes, p)
|
||||
}
|
||||
for _, v := range ss.Statuses {
|
||||
p := &controlapi.VertexStatus{
|
||||
ID: v.ID,
|
||||
Vertex: string(v.Vertex),
|
||||
Name: v.Name,
|
||||
Current: v.Current,
|
||||
Total: v.Total,
|
||||
Timestamp: timestamppb.New(v.Timestamp),
|
||||
Started: timestampToPB(v.Started),
|
||||
Completed: timestampToPB(v.Completed),
|
||||
}
|
||||
reserve(p.SizeVT())
|
||||
sr.Statuses = append(sr.Statuses, p)
|
||||
}
|
||||
for _, v := range ss.Logs {
|
||||
p := &controlapi.VertexLog{
|
||||
Vertex: string(v.Vertex),
|
||||
Stream: int64(v.Stream),
|
||||
Timestamp: timestamppb.New(v.Timestamp),
|
||||
}
|
||||
const maxData = statusResponseSize / 2
|
||||
data := v.Data
|
||||
for {
|
||||
p.Msg = data[:min(len(data), maxData)]
|
||||
reserve(p.SizeVT())
|
||||
sr.Logs = append(sr.Logs, p)
|
||||
data = data[len(p.Msg):]
|
||||
if len(data) == 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
for _, v := range ss.Warnings {
|
||||
sr.Warnings = append(sr.Warnings, &controlapi.VertexWarning{
|
||||
Vertex: string(v.Vertex),
|
||||
Level: int64(v.Level),
|
||||
Short: v.Short,
|
||||
Detail: v.Detail,
|
||||
Info: v.SourceInfo,
|
||||
Ranges: v.Range,
|
||||
Url: v.URL,
|
||||
})
|
||||
}
|
||||
out = append(out, &sr)
|
||||
if !retry {
|
||||
break
|
||||
p = &controlapi.VertexLog{Vertex: p.Vertex, Stream: p.Stream, Timestamp: p.Timestamp}
|
||||
}
|
||||
}
|
||||
for _, v := range ss.Warnings {
|
||||
p := &controlapi.VertexWarning{
|
||||
Vertex: string(v.Vertex),
|
||||
Level: int64(v.Level),
|
||||
Short: v.Short,
|
||||
Detail: v.Detail,
|
||||
Info: v.SourceInfo,
|
||||
Ranges: v.Range,
|
||||
Url: v.URL,
|
||||
}
|
||||
reserve(p.SizeVT())
|
||||
sr.Warnings = append(sr.Warnings, p)
|
||||
}
|
||||
out = append(out, sr)
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
+2
-2
@@ -1,7 +1,7 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"github.com/moby/buildkit/cache/remotecache/gha/ghatypes"
|
||||
"github.com/moby/buildkit/cache/remotecache/signing"
|
||||
"github.com/moby/buildkit/util/db/compaction"
|
||||
resolverconfig "github.com/moby/buildkit/util/resolver/config"
|
||||
)
|
||||
@@ -105,7 +105,7 @@ func (c CompactionConfig) Policy() (compaction.Config, error) {
|
||||
}
|
||||
|
||||
type CacheConfig struct {
|
||||
GHA *ghatypes.CacheConfig `toml:"gha"`
|
||||
GHA *signing.Config `toml:"gha"`
|
||||
}
|
||||
|
||||
type SystemConfig struct {
|
||||
|
||||
+3
-2
@@ -245,7 +245,8 @@ func (c *Controller) Prune(req *controlapi.PruneRequest, stream controlapi.Contr
|
||||
|
||||
ch := make(chan client.UsageInfo, 32)
|
||||
|
||||
eg, ctx := errgroup.WithContext(stream.Context())
|
||||
ctx := stream.Context()
|
||||
eg, ectx := errgroup.WithContext(ctx)
|
||||
workers, err := c.opt.WorkerController.List()
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to list workers for prune")
|
||||
@@ -267,7 +268,7 @@ func (c *Controller) Prune(req *controlapi.PruneRequest, stream controlapi.Contr
|
||||
for _, w := range workers {
|
||||
func(w worker.Worker) {
|
||||
eg.Go(func() error {
|
||||
return w.Prune(ctx, ch, client.PruneInfo{
|
||||
return w.Prune(ectx, ch, client.PruneInfo{
|
||||
Filter: req.Filter,
|
||||
All: req.All,
|
||||
KeepDuration: time.Duration(req.KeepDuration),
|
||||
|
||||
+2
-4
@@ -426,8 +426,7 @@ func (w *containerdExecutor) runProcess(ctx context.Context, p ctd.Process, resi
|
||||
if !ok {
|
||||
return // chan closed
|
||||
}
|
||||
err = p.Resize(eventCtx, size.Cols, size.Rows)
|
||||
if err != nil {
|
||||
if err := p.Resize(eventCtx, size.Cols, size.Rows); err != nil {
|
||||
bklog.G(eventCtx).Warnf("Failed to resize %s: %s", p.ID(), err)
|
||||
}
|
||||
}
|
||||
@@ -442,8 +441,7 @@ func (w *containerdExecutor) runProcess(ctx context.Context, p ctd.Process, resi
|
||||
if !ok {
|
||||
return // chan closed
|
||||
}
|
||||
err = p.Kill(eventCtx, sig)
|
||||
if err != nil {
|
||||
if err := p.Kill(eventCtx, sig); err != nil {
|
||||
bklog.G(eventCtx).Warnf("Failed to signal %s: %s", p.ID(), err)
|
||||
}
|
||||
}
|
||||
|
||||
+7
-7
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"net/netip"
|
||||
"os"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/moby/buildkit/solver/pb"
|
||||
"github.com/moby/buildkit/util/bklog"
|
||||
@@ -23,9 +24,10 @@ const (
|
||||
)
|
||||
|
||||
var (
|
||||
g flightcontrol.Group[struct{}]
|
||||
notFirstRun bool
|
||||
lastNotEmpty bool
|
||||
g flightcontrol.Group[struct{}]
|
||||
// accessed concurrently by calls with different keys
|
||||
notFirstRun atomic.Bool
|
||||
lastNotEmpty atomic.Bool
|
||||
)
|
||||
|
||||
// overridden by tests
|
||||
@@ -67,8 +69,7 @@ func GetResolvConf(ctx context.Context, root *os.Root, idmap *user.IdentityMappi
|
||||
}
|
||||
|
||||
_, err := g.Do(ctx, root.Name()+"/"+name, func(ctx context.Context) (struct{}, error) {
|
||||
generate := !notFirstRun
|
||||
notFirstRun = true
|
||||
generate := !notFirstRun.Swap(true)
|
||||
|
||||
if !generate {
|
||||
fi, err := root.Stat(name)
|
||||
@@ -84,9 +85,8 @@ func GetResolvConf(ctx context.Context, root *os.Root, idmap *user.IdentityMappi
|
||||
if !errors.Is(err, os.ErrNotExist) {
|
||||
return struct{}{}, err
|
||||
}
|
||||
if lastNotEmpty {
|
||||
if lastNotEmpty.CompareAndSwap(true, false) {
|
||||
generate = true
|
||||
lastNotEmpty = false
|
||||
}
|
||||
} else if fi.ModTime().Before(fiMain.ModTime()) {
|
||||
generate = true
|
||||
|
||||
+3
-1
@@ -40,6 +40,8 @@ import (
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
const runcStartTimeout = 15 * time.Second
|
||||
|
||||
type Opt struct {
|
||||
// root directory
|
||||
Root string
|
||||
@@ -748,7 +750,7 @@ func (p *procHandle) WaitForReady(ctx context.Context) error {
|
||||
// callback is non-nil it will be called after receiving the pid.
|
||||
func (p *procHandle) WaitForStart(ctx context.Context, startedCh <-chan int, started func()) error {
|
||||
ctx, cancel := context.WithCancelCause(ctx)
|
||||
ctx, _ = context.WithTimeoutCause(ctx, 10*time.Second, errors.WithStack(context.DeadlineExceeded)) //nolint:govet
|
||||
ctx, _ = context.WithTimeoutCause(ctx, runcStartTimeout, errors.WithStack(context.DeadlineExceeded)) //nolint:govet
|
||||
defer func() { cancel(errors.WithStack(context.Canceled)) }()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
|
||||
+17
-4
@@ -13,6 +13,7 @@ import (
|
||||
"github.com/containerd/console"
|
||||
runc "github.com/containerd/go-runc"
|
||||
"github.com/moby/buildkit/executor"
|
||||
"github.com/moby/buildkit/executor/oci"
|
||||
gatewayapi "github.com/moby/buildkit/frontend/gateway/pb"
|
||||
"github.com/moby/buildkit/util/bklog"
|
||||
"github.com/moby/sys/signal"
|
||||
@@ -34,10 +35,14 @@ func (w *runcExecutor) run(ctx context.Context, id, bundle string, process execu
|
||||
extraArgs = append(extraArgs, "--keep")
|
||||
}
|
||||
_, err := w.runc.Run(ctx, id, bundle, &runc.CreateOpts{
|
||||
NoPivot: w.noPivot,
|
||||
Started: started,
|
||||
IO: io,
|
||||
ExtraArgs: extraArgs,
|
||||
NoPivot: w.noPivot,
|
||||
// Without a PID namespace, processes left behind by a step outlive the
|
||||
// container and keep its session keyring, until the per-user key quota
|
||||
// is exhausted and every new container fails to start.
|
||||
NoNewKeyring: w.processMode == oci.NoProcessSandbox,
|
||||
Started: started,
|
||||
IO: io,
|
||||
ExtraArgs: extraArgs,
|
||||
})
|
||||
return err
|
||||
})
|
||||
@@ -164,7 +169,9 @@ func (w *runcExecutor) callWithIO(ctx context.Context, process executor.ProcessI
|
||||
})
|
||||
}
|
||||
|
||||
resizeDone := make(chan struct{})
|
||||
eg.Go(func() error {
|
||||
defer close(resizeDone)
|
||||
err := runcProcess.WaitForReady(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -190,6 +197,12 @@ func (w *runcExecutor) callWithIO(ctx context.Context, process executor.ProcessI
|
||||
}
|
||||
}
|
||||
})
|
||||
// Runs before the cleanup defer above: the resize loop uses ptm's fd, so
|
||||
// stop it before ptm is closed.
|
||||
defer func() {
|
||||
runcProcess.Shutdown()
|
||||
<-resizeDone
|
||||
}()
|
||||
|
||||
runcIO := &forwardIO{}
|
||||
if process.Stdin != nil {
|
||||
|
||||
+1
-1
@@ -35,7 +35,7 @@ const (
|
||||
)
|
||||
|
||||
func Build(ctx context.Context, c client.Client) (_ *client.Result, err error) {
|
||||
c = &withResolveCache{Client: c}
|
||||
c = newWithResolveCache(c)
|
||||
bc, err := dockerui.NewClient(c)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
+7
-1
@@ -26,8 +26,14 @@ type resolveResult struct {
|
||||
|
||||
var _ client.Client = &withResolveCache{}
|
||||
|
||||
func newWithResolveCache(c client.Client) *withResolveCache {
|
||||
return &withResolveCache{
|
||||
Client: c,
|
||||
g: flightcontrol.CachedGroup[*resolveResult]{CacheError: true},
|
||||
}
|
||||
}
|
||||
|
||||
func (c *withResolveCache) ResolveImageConfig(ctx context.Context, ref string, opt sourceresolver.Opt) (string, digest.Digest, []byte, error) {
|
||||
c.g.CacheError = true
|
||||
optHash, err := hashstructure.Hash(opt, nil)
|
||||
if err != nil {
|
||||
return "", "", nil, err
|
||||
|
||||
+118
-3
@@ -20,6 +20,7 @@ import (
|
||||
"github.com/moby/buildkit/client/llb"
|
||||
"github.com/moby/buildkit/client/llb/imagemetaresolver"
|
||||
"github.com/moby/buildkit/client/llb/sourceresolver"
|
||||
dfcommand "github.com/moby/buildkit/frontend/dockerfile/command"
|
||||
"github.com/moby/buildkit/frontend/dockerfile/instructions"
|
||||
"github.com/moby/buildkit/frontend/dockerfile/linter"
|
||||
"github.com/moby/buildkit/frontend/dockerfile/parser"
|
||||
@@ -47,6 +48,7 @@ const (
|
||||
|
||||
sbomScanContext = "BUILDKIT_SBOM_SCAN_CONTEXT"
|
||||
sbomScanStage = "BUILDKIT_SBOM_SCAN_STAGE"
|
||||
disableOnbuild = "BUILDKIT_DISABLE_ONBUILD"
|
||||
)
|
||||
|
||||
var nonEnvArgs = map[string]struct{}{
|
||||
@@ -216,6 +218,7 @@ type dispatchContext struct {
|
||||
allDispatchStates *dispatchStates
|
||||
proxyEnv *llb.ProxyEnv
|
||||
namedContext func(string, dockerui.ContextOpt) (*dockerui.NamedContext, error)
|
||||
disableOnbuild bool
|
||||
}
|
||||
|
||||
func namedContextFunc(opt ConvertOpt) func(string, dockerui.ContextOpt) (*dockerui.NamedContext, error) {
|
||||
@@ -310,6 +313,14 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS
|
||||
globalArgs = setBuildArgValue(opt.BuildArgs, globalArgs, "SOURCE_DATE_EPOCH", formatSourceDateEpochValue(resolvedEpoch))
|
||||
}
|
||||
|
||||
var skipOnbuild bool
|
||||
if v, ok := getBuildArgValue(opt.BuildArgs, globalArgs, disableOnbuild); ok && v != "" {
|
||||
skipOnbuild, err = strconv.ParseBool(v)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "invalid %s value %q", disableOnbuild, v)
|
||||
}
|
||||
}
|
||||
|
||||
metaResolver := opt.MetaResolver
|
||||
if metaResolver == nil {
|
||||
metaResolver = imagemetaresolver.Default()
|
||||
@@ -327,6 +338,7 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS
|
||||
allDispatchStates: newDispatchStates(),
|
||||
proxyEnv: proxyEnvFromBuildArgs(opt.BuildArgs),
|
||||
namedContext: namedContextFunc(opt),
|
||||
disableOnbuild: skipOnbuild,
|
||||
}
|
||||
|
||||
if err := dctx.buildDispatchStates(stages); err != nil {
|
||||
@@ -568,6 +580,10 @@ func (dctx *dispatchContext) resolveStages(ctx context.Context, target *dispatch
|
||||
}
|
||||
d.onBuildInit = true
|
||||
}
|
||||
if dctx.disableOnbuild || slices.Contains(d.stage.Reset, dfcommand.Onbuild) {
|
||||
onbuilds = nil
|
||||
d.image.Config.OnBuild = nil
|
||||
}
|
||||
|
||||
if len(onbuilds) > 0 {
|
||||
if b, err := initOnBuildTriggers(d, onbuilds, dctx.allDispatchStates, dctx.shlex); err != nil {
|
||||
@@ -789,6 +805,8 @@ func (dctx *dispatchContext) dispatchStages(ctx context.Context, allReachable ma
|
||||
d.platform = &d.opt.targetPlatform
|
||||
}
|
||||
|
||||
resetBaseConfig(d)
|
||||
|
||||
// make sure that PATH is always set
|
||||
if _, ok := shell.EnvsFromSlice(d.image.Config.Env).Get("PATH"); !ok {
|
||||
var osName string
|
||||
@@ -834,6 +852,7 @@ func (dctx *dispatchContext) dispatchStages(ctx context.Context, allReachable ma
|
||||
targetPlatform: dctx.platformOpt.targetPlatform,
|
||||
extraHosts: dctx.opt.ExtraHosts,
|
||||
gitAdvice: dctx.opt.GitAdvice,
|
||||
gitDebugCommands: dctx.opt.GitDebugCommands,
|
||||
shmSize: dctx.opt.ShmSize,
|
||||
ulimit: dctx.opt.Ulimits,
|
||||
devices: dctx.opt.Devices,
|
||||
@@ -991,6 +1010,7 @@ type dispatchOpt struct {
|
||||
buildPlatforms []ocispecs.Platform
|
||||
extraHosts []llb.HostIP
|
||||
gitAdvice bool
|
||||
gitDebugCommands bool
|
||||
shmSize int64
|
||||
ulimit []*pb.Ulimit
|
||||
devices []*pb.CDIDevice
|
||||
@@ -1329,6 +1349,17 @@ func initOnBuildTriggers(d *dispatchState, triggers []string, allDispatchStates
|
||||
}
|
||||
|
||||
func dispatchEnv(d *dispatchState, c *instructions.EnvCommand, lint *linter.Linter) error {
|
||||
if c.Unset {
|
||||
keys := make([]string, 0, len(c.Env))
|
||||
for _, e := range c.Env {
|
||||
if e.Key == "" {
|
||||
return errors.New("ENV --unset names can not be blank")
|
||||
}
|
||||
unsetEnv(d, e.Key)
|
||||
keys = append(keys, e.Key)
|
||||
}
|
||||
return commitToHistory(&d.image, "ENV --unset "+strings.Join(keys, " "), false, nil, d.epoch)
|
||||
}
|
||||
commitMessage := bytes.NewBufferString("ENV")
|
||||
for _, e := range c.Env {
|
||||
if e.NoDelim {
|
||||
@@ -1343,6 +1374,61 @@ func dispatchEnv(d *dispatchState, c *instructions.EnvCommand, lint *linter.Lint
|
||||
return commitToHistory(&d.image, commitMessage.String(), false, nil, d.epoch)
|
||||
}
|
||||
|
||||
// unsetEnv removes an environment variable and a build arg with the same name.
|
||||
func unsetEnv(d *dispatchState, key string) {
|
||||
for _, k := range getEnv(d.state).Keys() {
|
||||
if shell.EqualEnvKeys(k, key) {
|
||||
d.state = d.state.DelEnv(k)
|
||||
}
|
||||
}
|
||||
d.image.Config.Env = delEnv(d.image.Config.Env, key)
|
||||
d.buildArgs = slices.DeleteFunc(d.buildArgs, func(arg instructions.KeyValuePairOptional) bool {
|
||||
return shell.EqualEnvKeys(arg.Key, key)
|
||||
})
|
||||
}
|
||||
|
||||
// resetBaseConfig clears the base image config fields set with FROM --reset.
|
||||
// ONBUILD triggers are skipped when the stages are resolved.
|
||||
func resetBaseConfig(d *dispatchState) {
|
||||
for _, k := range d.stage.Reset {
|
||||
switch k {
|
||||
case dfcommand.Cmd:
|
||||
d.image.Config.Cmd = nil
|
||||
case dfcommand.Entrypoint:
|
||||
d.image.Config.Entrypoint = nil
|
||||
case dfcommand.Env:
|
||||
for _, env := range slices.Clone(d.image.Config.Env) {
|
||||
key, _ := parseKeyValue(env)
|
||||
unsetEnv(d, key)
|
||||
}
|
||||
case dfcommand.Expose:
|
||||
d.image.Config.ExposedPorts = nil
|
||||
case dfcommand.Healthcheck:
|
||||
d.image.Config.Healthcheck = nil
|
||||
case dfcommand.Label:
|
||||
d.image.Config.Labels = nil
|
||||
case dfcommand.Shell:
|
||||
d.image.Config.Shell = nil
|
||||
case dfcommand.StopSignal:
|
||||
d.image.Config.StopSignal = ""
|
||||
case dfcommand.User:
|
||||
d.image.Config.User = ""
|
||||
d.state = d.state.User("")
|
||||
case dfcommand.Volume:
|
||||
d.image.Config.Volumes = nil
|
||||
case dfcommand.Workdir:
|
||||
d.image.Config.WorkingDir = ""
|
||||
d.state = d.state.Dir("/")
|
||||
// workdir no longer depends on the base image
|
||||
d.workdirSet = true
|
||||
}
|
||||
}
|
||||
// ArgsEscaped applies to both cmd and entrypoint
|
||||
if len(d.stage.Reset) > 0 && len(d.image.Config.Cmd) == 0 && len(d.image.Config.Entrypoint) == 0 {
|
||||
d.image.Config.ArgsEscaped = false //nolint:staticcheck // ignore SA1019: field is deprecated in OCI Image spec, but used for backward-compatibility with Docker image spec.
|
||||
}
|
||||
}
|
||||
|
||||
func dispatchRun(d *dispatchState, c *instructions.RunCommand, proxy *llb.ProxyEnv, sources []*dispatchState, dopt dispatchOpt) error {
|
||||
var opt []llb.RunOption
|
||||
|
||||
@@ -1573,6 +1659,17 @@ func dispatchMaintainer(d *dispatchState, c *instructions.MaintainerCommand) err
|
||||
}
|
||||
|
||||
func dispatchLabel(d *dispatchState, c *instructions.LabelCommand, lint *linter.Linter) error {
|
||||
if c.Unset {
|
||||
keys := make([]string, 0, len(c.Labels))
|
||||
for _, v := range c.Labels {
|
||||
if v.Key == "" {
|
||||
return errors.New("LABEL --unset names can not be blank")
|
||||
}
|
||||
delete(d.image.Config.Labels, v.Key)
|
||||
keys = append(keys, v.Key)
|
||||
}
|
||||
return commitToHistory(&d.image, "LABEL --unset "+strings.Join(keys, " "), false, nil, d.epoch)
|
||||
}
|
||||
commitMessage := bytes.NewBufferString("LABEL")
|
||||
if d.image.Config.Labels == nil {
|
||||
d.image.Config.Labels = make(map[string]string, len(c.Labels))
|
||||
@@ -1650,13 +1747,24 @@ func dispatchUser(d *dispatchState, c *instructions.UserCommand, commit bool) er
|
||||
}
|
||||
|
||||
func dispatchVolume(d *dispatchState, c *instructions.VolumeCommand) error {
|
||||
if slices.Contains(c.Volumes, "") {
|
||||
return errors.New("VOLUME specified can not be an empty string")
|
||||
}
|
||||
if c.Unset {
|
||||
for _, v := range c.Volumes {
|
||||
v = path.Clean(v)
|
||||
for k := range d.image.Config.Volumes {
|
||||
if path.Clean(k) == v {
|
||||
delete(d.image.Config.Volumes, k)
|
||||
}
|
||||
}
|
||||
}
|
||||
return commitToHistory(&d.image, fmt.Sprintf("VOLUME --unset %v", c.Volumes), false, nil, d.epoch)
|
||||
}
|
||||
if d.image.Config.Volumes == nil {
|
||||
d.image.Config.Volumes = map[string]struct{}{}
|
||||
}
|
||||
for _, v := range c.Volumes {
|
||||
if v == "" {
|
||||
return errors.New("VOLUME specified can not be an empty string")
|
||||
}
|
||||
d.image.Config.Volumes[v] = struct{}{}
|
||||
}
|
||||
return commitToHistory(&d.image, fmt.Sprintf("VOLUME %v", c.Volumes), false, nil, d.epoch)
|
||||
@@ -1768,6 +1876,13 @@ func addEnv(env []string, k, v string) []string {
|
||||
return env
|
||||
}
|
||||
|
||||
func delEnv(env []string, k string) []string {
|
||||
return slices.DeleteFunc(env, func(envVar string) bool {
|
||||
key, _ := parseKeyValue(envVar)
|
||||
return shell.EqualEnvKeys(key, k)
|
||||
})
|
||||
}
|
||||
|
||||
func parseKeyValue(env string) (string, string) {
|
||||
parts := strings.SplitN(env, "=", 2)
|
||||
v := ""
|
||||
|
||||
Generated
Vendored
+3
@@ -145,6 +145,9 @@ func dispatchCopy(d *dispatchState, cfg copyConfig) error {
|
||||
if cfg.opt.gitAdvice {
|
||||
gitOptions = append(gitOptions, llb.GitAdvice(true))
|
||||
}
|
||||
if cfg.opt.gitDebugCommands {
|
||||
gitOptions = append(gitOptions, llb.GitDebugCommands(true))
|
||||
}
|
||||
if cfg.keepGitDir != nil && gitRef.KeepGitDir != nil {
|
||||
if *cfg.keepGitDir != *gitRef.KeepGitDir {
|
||||
return errors.New("inconsistent keep-git-dir configuration")
|
||||
|
||||
Generated
Vendored
+1
-1
@@ -41,7 +41,7 @@ func dispatchExpose(d *dispatchState, c *instructions.ExposeCommand, opt *dispat
|
||||
d.image.Config.ExposedPorts[p] = struct{}{}
|
||||
}
|
||||
|
||||
return commitToHistory(&d.image, fmt.Sprintf("EXPOSE %v", psp), false, nil, d.epoch)
|
||||
return commitToHistory(&d.image, fmt.Sprintf("EXPOSE %s", strings.Join(psp, " ")), false, nil, d.epoch)
|
||||
}
|
||||
|
||||
type portSpecs struct {
|
||||
|
||||
+7
-4
@@ -98,7 +98,7 @@ func resolveSourceDateEpochState(ctx context.Context, value string, opt ConvertO
|
||||
args = &updated
|
||||
}
|
||||
|
||||
sourceState, err := sourceDateEpochStageSource(stages[i], opt.BuildArgs, args, shlex, opt.GitAdvice)
|
||||
sourceState, err := sourceDateEpochStageSource(stages[i], opt.BuildArgs, args, shlex, opt.GitAdvice, opt.GitDebugCommands)
|
||||
if err != nil {
|
||||
return nil, sourceDateEpochStateOpt{}, parser.WithLocation(err, stages[i].Location)
|
||||
}
|
||||
@@ -110,7 +110,7 @@ func resolveSourceDateEpochState(ctx context.Context, value string, opt ConvertO
|
||||
return nil, sourceDateEpochStateOpt{}, errors.Errorf("invalid SOURCE_DATE_EPOCH: %s", value)
|
||||
}
|
||||
|
||||
func sourceDateEpochStageSource(stage instructions.Stage, buildArgs map[string]string, globalArgs *llb.EnvList, shlex *shell.Lex, gitAdvice bool) (*llb.State, error) {
|
||||
func sourceDateEpochStageSource(stage instructions.Stage, buildArgs map[string]string, globalArgs *llb.EnvList, shlex *shell.Lex, gitAdvice, gitDebugCommands bool) (*llb.State, error) {
|
||||
stageBaseName, _, err := shlex.ProcessWord(stage.BaseName, globalArgs)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "failed to process source stage base name %q", stage.BaseName)
|
||||
@@ -133,7 +133,7 @@ func sourceDateEpochStageSource(stage instructions.Stage, buildArgs map[string]s
|
||||
if sourceState != nil {
|
||||
return nil, errors.New("SOURCE_DATE_EPOCH stage must contain exactly one remote ADD")
|
||||
}
|
||||
sourceState, err = sourceDateEpochAddSource(c, env, shlex, gitAdvice)
|
||||
sourceState, err = sourceDateEpochAddSource(c, env, shlex, gitAdvice, gitDebugCommands)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -167,7 +167,7 @@ func applySourceDateEpochStageArgs(args []instructions.KeyValuePairOptional, env
|
||||
return env, nil
|
||||
}
|
||||
|
||||
func sourceDateEpochAddSource(cmd *instructions.AddCommand, env *llb.EnvList, shlex *shell.Lex, gitAdvice bool) (*llb.State, error) {
|
||||
func sourceDateEpochAddSource(cmd *instructions.AddCommand, env *llb.EnvList, shlex *shell.Lex, gitAdvice, gitDebugCommands bool) (*llb.State, error) {
|
||||
if len(cmd.SourceContents) != 0 || len(cmd.SourcePaths) != 1 {
|
||||
return nil, errors.New("SOURCE_DATE_EPOCH stage must contain exactly one remote ADD source")
|
||||
}
|
||||
@@ -204,6 +204,9 @@ func sourceDateEpochAddSource(cmd *instructions.AddCommand, env *llb.EnvList, sh
|
||||
if gitAdvice {
|
||||
gitOptions = append(gitOptions, llb.GitAdvice(true))
|
||||
}
|
||||
if gitDebugCommands {
|
||||
gitOptions = append(gitOptions, llb.GitDebugCommands(true))
|
||||
}
|
||||
if cmd.KeepGitDir != nil && *cmd.KeepGitDir {
|
||||
gitOptions = append(gitOptions, llb.KeepGitDir())
|
||||
}
|
||||
|
||||
Generated
Vendored
+1
@@ -311,6 +311,7 @@ func getSecretsRegex() (*regexp.Regexp, *regexp.Regexp) {
|
||||
"password",
|
||||
"pword",
|
||||
"passwd",
|
||||
"psw",
|
||||
"secret",
|
||||
"token",
|
||||
}
|
||||
|
||||
+8
-1
@@ -148,9 +148,11 @@ func expandSliceInPlace(values []string, expander SingleWordExpander) error {
|
||||
// EnvCommand allows setting an variable in the container's environment.
|
||||
//
|
||||
// ENV key1 value1 [keyN valueN...]
|
||||
// ENV --unset key1 [keyN...]
|
||||
type EnvCommand struct {
|
||||
withNameAndCode
|
||||
Env KeyValuePairs
|
||||
Env KeyValuePairs
|
||||
Unset bool // Env keys are removed, values are empty
|
||||
}
|
||||
|
||||
func (c *EnvCommand) Expand(expander SingleWordExpander) error {
|
||||
@@ -185,9 +187,11 @@ func NewLabelCommand(k string, v string, noExp bool) *LabelCommand {
|
||||
// LabelCommand sets an image label in the output
|
||||
//
|
||||
// LABEL some json data describing the image
|
||||
// LABEL --unset key1 [keyN...]
|
||||
type LabelCommand struct {
|
||||
withNameAndCode
|
||||
Labels KeyValuePairs
|
||||
Unset bool // Labels keys are removed, values are empty
|
||||
noExpand bool
|
||||
}
|
||||
|
||||
@@ -436,9 +440,11 @@ func (c *UserCommand) Expand(expander SingleWordExpander) error {
|
||||
// VolumeCommand exposes the specified volume for use in the build environment.
|
||||
//
|
||||
// VOLUME /foo
|
||||
// VOLUME --unset /foo
|
||||
type VolumeCommand struct {
|
||||
withNameAndCode
|
||||
Volumes []string
|
||||
Unset bool // Volumes are removed
|
||||
}
|
||||
|
||||
func (c *VolumeCommand) Expand(expander SingleWordExpander) error {
|
||||
@@ -521,6 +527,7 @@ type Stage struct {
|
||||
OrigCmd string // original FROM command, used for rule checks
|
||||
BaseName string // name of the base stage or source
|
||||
Platform string // platform of base source to use
|
||||
Reset []string // base image config fields to reset, see ResetKeys
|
||||
|
||||
DocComment string // doc-comment directly above the stage
|
||||
|
||||
|
||||
+119
@@ -36,6 +36,31 @@ var (
|
||||
parseRunPostHooks []func(*RunCommand, parseRequest) error
|
||||
)
|
||||
|
||||
var (
|
||||
unsetEnabled = false
|
||||
resetEnabled = false
|
||||
)
|
||||
|
||||
// ResetAll is the FROM --reset value that resets all ResetKeys.
|
||||
const ResetAll = "all"
|
||||
|
||||
// ResetKeys are the base image config fields that can be reset with
|
||||
// FROM --reset. Each field is named after the command that sets it.
|
||||
var ResetKeys = []string{
|
||||
command.Cmd,
|
||||
command.Entrypoint,
|
||||
command.Env,
|
||||
command.Expose,
|
||||
command.Healthcheck,
|
||||
command.Label,
|
||||
command.Onbuild,
|
||||
command.Shell,
|
||||
command.StopSignal,
|
||||
command.User,
|
||||
command.Volume,
|
||||
command.Workdir,
|
||||
}
|
||||
|
||||
func nodeArgs(node *parser.Node) []string {
|
||||
result := []string{}
|
||||
for ; node.Next != nil; node = node.Next {
|
||||
@@ -230,10 +255,48 @@ func parseKvps(args []string, cmdName string) (KeyValuePairs, error) {
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// addUnsetFlag adds the --unset flag if it is enabled in this build.
|
||||
func addUnsetFlag(req parseRequest) *Flag {
|
||||
if !unsetEnabled {
|
||||
return nil
|
||||
}
|
||||
return req.flags.AddBool("unset", false)
|
||||
}
|
||||
|
||||
// parseUnsetKeys returns the keys for the --unset form of a command.
|
||||
func parseUnsetKeys(args []string, cmdName string) (KeyValuePairs, error) {
|
||||
if len(args) == 0 {
|
||||
return nil, errAtLeastOneArgument(cmdName + " --unset")
|
||||
}
|
||||
var res KeyValuePairs
|
||||
for _, name := range args {
|
||||
if name == "" {
|
||||
return nil, errBlankCommandNames(cmdName)
|
||||
}
|
||||
if strings.Contains(name, "=") {
|
||||
return nil, errors.Errorf("%s --unset does not accept values: %s", cmdName, name)
|
||||
}
|
||||
res = append(res, KeyValuePair{Key: name})
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func parseEnv(req parseRequest) (*EnvCommand, error) {
|
||||
flUnset := addUnsetFlag(req)
|
||||
if err := req.flags.Parse(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if flUnset != nil && flUnset.Value == "true" {
|
||||
envs, err := parseUnsetKeys(req.args, "ENV")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &EnvCommand{
|
||||
Env: envs,
|
||||
Unset: true,
|
||||
withNameAndCode: newWithNameAndCode(req),
|
||||
}, nil
|
||||
}
|
||||
envs, err := parseKvps(req.args, "ENV")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -259,9 +322,21 @@ func parseMaintainer(req parseRequest) (*MaintainerCommand, error) {
|
||||
}
|
||||
|
||||
func parseLabel(req parseRequest) (*LabelCommand, error) {
|
||||
flUnset := addUnsetFlag(req)
|
||||
if err := req.flags.Parse(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if flUnset != nil && flUnset.Value == "true" {
|
||||
labels, err := parseUnsetKeys(req.args, "LABEL")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &LabelCommand{
|
||||
Labels: labels,
|
||||
Unset: true,
|
||||
withNameAndCode: newWithNameAndCode(req),
|
||||
}, nil
|
||||
}
|
||||
|
||||
labels, err := parseKvps(req.args, "LABEL")
|
||||
if err != nil {
|
||||
@@ -399,10 +474,22 @@ func parseFrom(req parseRequest) (*Stage, error) {
|
||||
}
|
||||
|
||||
flPlatform := req.flags.AddString("platform", "")
|
||||
var flReset *Flag
|
||||
if resetEnabled {
|
||||
flReset = req.flags.AddStrings("reset")
|
||||
}
|
||||
if err := req.flags.Parse(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var reset []string
|
||||
if flReset != nil {
|
||||
reset, err = parseResetKeys(flReset.StringValues)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
code := strings.TrimSpace(req.original)
|
||||
return &Stage{
|
||||
BaseName: req.args[0],
|
||||
@@ -411,12 +498,35 @@ func parseFrom(req parseRequest) (*Stage, error) {
|
||||
SourceCode: code,
|
||||
Commands: []Command{},
|
||||
Platform: flPlatform.Value,
|
||||
Reset: reset,
|
||||
Location: req.location,
|
||||
Comments: req.comments,
|
||||
DocComment: getDocComment(req.comments, stageName),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// parseResetKeys returns the sorted list of fields for FROM --reset values.
|
||||
func parseResetKeys(values []string) ([]string, error) {
|
||||
var keys []string
|
||||
for _, v := range values {
|
||||
for k := range strings.SplitSeq(v, ",") {
|
||||
switch {
|
||||
case k == ResetAll:
|
||||
keys = append(keys, ResetKeys...)
|
||||
case slices.Contains(ResetKeys, k):
|
||||
keys = append(keys, k)
|
||||
case strings.ToLower(k) == ResetAll || slices.Contains(ResetKeys, strings.ToLower(k)):
|
||||
return nil, errors.Errorf("invalid FROM --reset value: %q (did you mean %s?)", k, strings.ToLower(k))
|
||||
default:
|
||||
err := errors.Errorf("invalid FROM --reset value: %q", k)
|
||||
return nil, suggest.WrapError(err, k, append([]string{ResetAll}, ResetKeys...), true)
|
||||
}
|
||||
}
|
||||
}
|
||||
slices.Sort(keys)
|
||||
return slices.Compact(keys), nil
|
||||
}
|
||||
|
||||
var validStageName = regexp.MustCompile("^[a-z][a-z0-9-_.]*$")
|
||||
|
||||
func parseBuildStageName(args []string) (stageName string, err error) {
|
||||
@@ -714,11 +824,13 @@ func parseVolume(req parseRequest) (*VolumeCommand, error) {
|
||||
return nil, errAtLeastOneArgument("VOLUME")
|
||||
}
|
||||
|
||||
flUnset := addUnsetFlag(req)
|
||||
if err := req.flags.Parse(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cmd := &VolumeCommand{
|
||||
Unset: flUnset != nil && flUnset.Value == "true",
|
||||
withNameAndCode: newWithNameAndCode(req),
|
||||
}
|
||||
|
||||
@@ -736,6 +848,9 @@ func parseStopSignal(req parseRequest) (*StopSignalCommand, error) {
|
||||
if len(req.args) != 1 {
|
||||
return nil, errExactlyOneArgument("STOPSIGNAL")
|
||||
}
|
||||
if err := req.flags.Parse(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sig := req.args[0]
|
||||
|
||||
cmd := &StopSignalCommand{
|
||||
@@ -750,6 +865,10 @@ func parseArg(req parseRequest) (*ArgCommand, error) {
|
||||
return nil, errAtLeastOneArgument("ARG")
|
||||
}
|
||||
|
||||
if err := req.flags.Parse(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
pairs := make([]KeyValuePairOptional, len(req.args))
|
||||
|
||||
for i, arg := range req.args {
|
||||
|
||||
+7
@@ -0,0 +1,7 @@
|
||||
//go:build dfreset
|
||||
|
||||
package instructions
|
||||
|
||||
func init() {
|
||||
resetEnabled = true
|
||||
}
|
||||
+7
@@ -0,0 +1,7 @@
|
||||
//go:build dfunset
|
||||
|
||||
package instructions
|
||||
|
||||
func init() {
|
||||
unsetEnabled = true
|
||||
}
|
||||
+22
@@ -137,6 +137,13 @@ var (
|
||||
heredocCompoundDirectives = map[string]bool{
|
||||
command.Onbuild: true,
|
||||
}
|
||||
|
||||
// Directives that parse key-value pairs but take a list of names
|
||||
// when --unset flag is set
|
||||
unsetCommands = map[string]bool{
|
||||
command.Env: true,
|
||||
command.Label: true,
|
||||
}
|
||||
)
|
||||
|
||||
// directives is the structure used during a build run to hold the state of
|
||||
@@ -234,6 +241,10 @@ func newNodeFromLine(line string, d *directives, comments []string) (*Node, erro
|
||||
if fn == nil {
|
||||
fn = parseIgnore
|
||||
}
|
||||
// --unset form takes a list of names instead of key-value pairs
|
||||
if unsetCommands[strings.ToLower(cmd)] && hasUnsetFlag(flags) {
|
||||
fn = parseNameOrNameVal
|
||||
}
|
||||
next, attrs, err := fn(args, d)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -249,6 +260,17 @@ func newNodeFromLine(line string, d *directives, comments []string) (*Node, erro
|
||||
}, nil
|
||||
}
|
||||
|
||||
func hasUnsetFlag(flags []string) bool {
|
||||
for _, f := range flags {
|
||||
name, value, _ := strings.Cut(f, "=")
|
||||
if name == "--unset" {
|
||||
// invalid values are reported by the flag parser
|
||||
return !strings.EqualFold(value, "false")
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Result contains the bundled outputs from parsing a Dockerfile.
|
||||
type Result struct {
|
||||
AST *Node
|
||||
|
||||
+1
-1
@@ -1,3 +1,3 @@
|
||||
package version
|
||||
|
||||
const version = "1.28"
|
||||
const version = "1.29"
|
||||
+8
@@ -58,6 +58,7 @@ const (
|
||||
keyDockerfileLintArg = "build-arg:BUILDKIT_DOCKERFILE_CHECK"
|
||||
keyContextKeepGitDirArg = "build-arg:BUILDKIT_CONTEXT_KEEP_GIT_DIR"
|
||||
keyGitAdviceArg = "build-arg:BUILDKIT_GIT_ADVICE"
|
||||
keyGitDebugCommandsArg = "build-arg:BUILDKIT_GIT_DEBUG_COMMANDS"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
@@ -70,6 +71,7 @@ type Config struct {
|
||||
Labels map[string]string
|
||||
NetworkMode pb.NetMode
|
||||
GitAdvice bool
|
||||
GitDebugCommands bool
|
||||
ShmSize int64
|
||||
Target string
|
||||
Ulimits []*pb.Ulimit
|
||||
@@ -319,6 +321,12 @@ func (bc *Client) init() error {
|
||||
return errors.Wrapf(err, "failed to parse %s", keyGitAdviceArg)
|
||||
}
|
||||
}
|
||||
if v := opts[keyGitDebugCommandsArg]; v != "" {
|
||||
bc.GitDebugCommands, err = strconv.ParseBool(v)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to parse %s", keyGitDebugCommandsArg)
|
||||
}
|
||||
}
|
||||
|
||||
bc.localsSessionIDs = parseLocalSessionIDs(opts)
|
||||
|
||||
|
||||
+9
-5
@@ -91,7 +91,7 @@ func (bc *Client) initContext(ctx context.Context) (*buildContext, error) {
|
||||
if opts[buildArgPrefix+"SOURCE_DATE_EPOCH"] != "" {
|
||||
extraGitOpts = append(extraGitOpts, llb.GitMTimeCommit())
|
||||
}
|
||||
extraGitOpts = append(extraGitOpts, gitAdviceOpts(bc.GitAdvice)...)
|
||||
extraGitOpts = append(extraGitOpts, gitSourceOpts(bc.GitAdvice, bc.GitDebugCommands)...)
|
||||
if st, ok, err := DetectGitContext(opts[localNameContext], keepGit, extraGitOpts...); ok {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -325,11 +325,15 @@ func DetectGitContext(ref string, keepGit *bool, opts ...llb.GitOption) (*llb.St
|
||||
return &st, true, nil
|
||||
}
|
||||
|
||||
func gitAdviceOpts(enabled bool) []llb.GitOption {
|
||||
if enabled {
|
||||
return []llb.GitOption{llb.GitAdvice(true)}
|
||||
func gitSourceOpts(advice, debugCommands bool) []llb.GitOption {
|
||||
var opts []llb.GitOption
|
||||
if advice {
|
||||
opts = append(opts, llb.GitAdvice(true))
|
||||
}
|
||||
return nil
|
||||
if debugCommands {
|
||||
opts = append(opts, llb.GitDebugCommands(true))
|
||||
}
|
||||
return opts
|
||||
}
|
||||
|
||||
func DetectHTTPContext(ref string) (*llb.State, string, bool) {
|
||||
|
||||
+2
-2
@@ -141,7 +141,7 @@ func (nc *NamedContext) load(ctx context.Context, count int) (*llb.State, *docke
|
||||
}
|
||||
return &st, &img, nil
|
||||
case "git":
|
||||
st, ok, err := DetectGitContext(nc.input, nil, gitAdviceOpts(nc.bc.GitAdvice)...)
|
||||
st, ok, err := DetectGitContext(nc.input, nil, gitSourceOpts(nc.bc.GitAdvice, nc.bc.GitDebugCommands)...)
|
||||
if !ok {
|
||||
return nil, nil, errors.Errorf("invalid git context %s", nc.input)
|
||||
}
|
||||
@@ -150,7 +150,7 @@ func (nc *NamedContext) load(ctx context.Context, count int) (*llb.State, *docke
|
||||
}
|
||||
return st, nil, nil
|
||||
case "http", "https":
|
||||
st, ok, err := DetectGitContext(nc.input, nil, gitAdviceOpts(nc.bc.GitAdvice)...)
|
||||
st, ok, err := DetectGitContext(nc.input, nil, gitSourceOpts(nc.bc.GitAdvice, nc.bc.GitDebugCommands)...)
|
||||
if ok {
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
|
||||
+18
-29
@@ -68,6 +68,7 @@ type BridgeClient struct {
|
||||
workers worker.Infos
|
||||
resultByID map[string]solver.Result
|
||||
discarded bool
|
||||
execErrors []*llberrdefs.ExecError
|
||||
buildOpts client.BuildOpts
|
||||
ctrs []client.Container
|
||||
executor executor.Executor
|
||||
@@ -159,6 +160,13 @@ func (c *BridgeClient) wrapSolveError(solveErr error) error {
|
||||
subject errdefs.IsSolve_Subject
|
||||
)
|
||||
if errors.As(solveErr, &ee) {
|
||||
c.mu.Lock()
|
||||
if c.discarded {
|
||||
ee.Release()
|
||||
} else if !slices.Contains(c.execErrors, ee) {
|
||||
c.execErrors = append(c.execErrors, ee)
|
||||
}
|
||||
c.mu.Unlock()
|
||||
var err error
|
||||
inputIDs, err = c.registerResultIDs(ee.Inputs...)
|
||||
if err != nil {
|
||||
@@ -174,7 +182,7 @@ func (c *BridgeClient) wrapSolveError(solveErr error) error {
|
||||
}
|
||||
if errors.As(solveErr, &sce) {
|
||||
var err error
|
||||
inputIDs, err = c.registerSlowCacheResult(sce.Result)
|
||||
inputIDs, err = c.registerResultIDs(sce.Result)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -183,32 +191,14 @@ func (c *BridgeClient) wrapSolveError(solveErr error) error {
|
||||
return errdefs.WithSolveError(solveErr, subject, inputIDs, mountIDs)
|
||||
}
|
||||
|
||||
// registerResultIDs retains gateway-owned clones of error refs until discard.
|
||||
func (c *BridgeClient) registerResultIDs(results ...solver.Result) (ids []string, err error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
return c.registerResultIDsLocked(results...)
|
||||
}
|
||||
|
||||
// registerSlowCacheResult registers a clone of a result borrowed from the
|
||||
// solver job. The clone is only taken before the client is discarded, as the
|
||||
// job may be released after that.
|
||||
func (c *BridgeClient) registerSlowCacheResult(res solver.Result) ([]string, error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
if c.discarded {
|
||||
return nil, nil
|
||||
}
|
||||
// Validate before cloning so a type error can't leak the clone. Once
|
||||
// cloned, registerResultIDsLocked either keeps or releases it.
|
||||
if _, ok := res.Sys().(*worker.WorkerRef); !ok {
|
||||
return nil, errors.Errorf("unexpected type for result, got %T", res.Sys())
|
||||
}
|
||||
return c.registerResultIDsLocked(res.Clone())
|
||||
}
|
||||
|
||||
func (c *BridgeClient) registerResultIDsLocked(results ...solver.Result) (ids []string, err error) {
|
||||
ids = make([]string, len(results))
|
||||
for i, res := range results {
|
||||
if res == nil {
|
||||
@@ -220,16 +210,11 @@ func (c *BridgeClient) registerResultIDsLocked(results ...solver.Result) (ids []
|
||||
}
|
||||
id := workerRef.ID()
|
||||
ids[i] = id
|
||||
if existing, ok := c.resultByID[id]; ok {
|
||||
if existing != res {
|
||||
if err := res.Release(context.TODO()); err != nil {
|
||||
return ids, errors.WithStack(err)
|
||||
}
|
||||
}
|
||||
continue
|
||||
if _, ok := c.resultByID[id]; !ok {
|
||||
// Cached sharedOp.Exec errors have snapshots pinned by the active
|
||||
// solver state, even after the original error refs are released.
|
||||
c.resultByID[id] = res.Clone()
|
||||
}
|
||||
// Keep the result wrapper so Release balances split-result ownership.
|
||||
c.resultByID[id] = res
|
||||
}
|
||||
return ids, nil
|
||||
}
|
||||
@@ -281,6 +266,10 @@ func (c *BridgeClient) discard(err error) {
|
||||
|
||||
c.mu.Lock()
|
||||
c.discarded = true
|
||||
for _, ee := range c.execErrors {
|
||||
ee.Release()
|
||||
}
|
||||
c.execErrors = nil
|
||||
for id, res := range c.resultByID {
|
||||
res.Release(context.TODO())
|
||||
delete(c.resultByID, id)
|
||||
|
||||
+74
-45
@@ -1,6 +1,7 @@
|
||||
package gateway
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
@@ -280,6 +281,10 @@ func (gf *gatewayFrontend) Solve(ctx context.Context, llbBridge frontend.Fronten
|
||||
lbf, ctx := serveLLBBridgeForwarder(ctx, llbBridge, exec, gf.workers, inputs, sid, sm)
|
||||
defer lbf.Discard()
|
||||
defer lbf.conn.Close()
|
||||
// The executor may pass these fds directly to the frontend process, so
|
||||
// close them only after Run has returned and no fork can still use them.
|
||||
defer lbf.Stdin.Close()
|
||||
defer lbf.Stdout.Close()
|
||||
|
||||
mdmnt, release, err := metadataMount(frontendDef)
|
||||
if err != nil {
|
||||
@@ -327,12 +332,12 @@ func metadataMount(def *opspb.Definition) (*executor.Mount, func(), error) {
|
||||
}
|
||||
|
||||
return &executor.Mount{
|
||||
Src: &bind{dir},
|
||||
Dest: "/run/config/buildkit/metadata",
|
||||
Readonly: true,
|
||||
}, func() {
|
||||
os.RemoveAll(dir)
|
||||
}, nil
|
||||
Src: &bind{dir},
|
||||
Dest: "/run/config/buildkit/metadata",
|
||||
Readonly: true,
|
||||
}, func() {
|
||||
os.RemoveAll(dir)
|
||||
}, nil
|
||||
}
|
||||
|
||||
type bind struct {
|
||||
@@ -394,6 +399,10 @@ func (lbf *llbBridgeForwarder) discard() {
|
||||
lbf.mountsMu.Unlock()
|
||||
|
||||
lbf.discarded = true
|
||||
for _, ee := range lbf.execErrors {
|
||||
ee.Release()
|
||||
}
|
||||
lbf.execErrors = nil
|
||||
for id, res := range lbf.resultByID {
|
||||
res.Release(context.TODO())
|
||||
delete(lbf.resultByID, id)
|
||||
@@ -474,7 +483,6 @@ func newBridgeForwarder(ctx context.Context, llbBridge frontend.FrontendLLBBridg
|
||||
refs: map[string]solver.ResultProxy{},
|
||||
resultByID: map[string]solver.Result{},
|
||||
doneCh: make(chan struct{}),
|
||||
pipe: newPipe(),
|
||||
workers: workers,
|
||||
inputs: inputs,
|
||||
sid: sid,
|
||||
@@ -489,6 +497,7 @@ func newBridgeForwarder(ctx context.Context, llbBridge frontend.FrontendLLBBridg
|
||||
func serveLLBBridgeForwarder(ctx context.Context, llbBridge frontend.FrontendLLBBridge, exec executor.Executor, workers worker.Infos, inputs map[string]*opspb.Definition, sid string, sm *session.Manager) (*llbBridgeForwarder, context.Context) {
|
||||
ctx, cancel := context.WithCancelCause(ctx)
|
||||
lbf := newBridgeForwarder(ctx, llbBridge, exec, workers, inputs, sid, sm)
|
||||
lbf.pipe = newPipe()
|
||||
serverOpt := []grpc.ServerOption{
|
||||
grpc.UnaryInterceptor(grpcerrors.UnaryServerInterceptor),
|
||||
grpc.StreamInterceptor(grpcerrors.StreamServerInterceptor),
|
||||
@@ -525,9 +534,9 @@ func newPipe() *pipe {
|
||||
Stdin: pr1,
|
||||
Stdout: pw2,
|
||||
conn: &conn{
|
||||
Reader: pr2,
|
||||
Writer: pw1,
|
||||
Closer: pw2,
|
||||
Reader: pr2,
|
||||
Writer: pw1,
|
||||
closers: []io.Closer{pr2, pw1},
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -535,7 +544,17 @@ func newPipe() *pipe {
|
||||
type conn struct {
|
||||
io.Reader
|
||||
io.Writer
|
||||
io.Closer
|
||||
closers []io.Closer
|
||||
}
|
||||
|
||||
func (s *conn) Close() error {
|
||||
var err error
|
||||
for _, closer := range s.closers {
|
||||
if closeErr := closer.Close(); err == nil {
|
||||
err = closeErr
|
||||
}
|
||||
}
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
func (s *conn) LocalAddr() net.Addr {
|
||||
@@ -589,6 +608,7 @@ type llbBridgeForwarder struct {
|
||||
refs map[string]solver.ResultProxy
|
||||
resultByID map[string]solver.Result
|
||||
discarded bool
|
||||
execErrors []*llberrdefs.ExecError
|
||||
// lastRef solver.CachedResult
|
||||
// lastRefs map[string]solver.CachedResult
|
||||
// err error
|
||||
@@ -715,6 +735,13 @@ func (lbf *llbBridgeForwarder) wrapSolveError(solveErr error) error {
|
||||
subject errdefs.IsSolve_Subject
|
||||
)
|
||||
if errors.As(solveErr, &ee) {
|
||||
lbf.mu.Lock()
|
||||
if lbf.discarded {
|
||||
ee.Release()
|
||||
} else if !slices.Contains(lbf.execErrors, ee) {
|
||||
lbf.execErrors = append(lbf.execErrors, ee)
|
||||
}
|
||||
lbf.mu.Unlock()
|
||||
var err error
|
||||
inputIDs, err = lbf.registerResultIDs(ee.Inputs...)
|
||||
if err != nil {
|
||||
@@ -730,7 +757,7 @@ func (lbf *llbBridgeForwarder) wrapSolveError(solveErr error) error {
|
||||
}
|
||||
if errors.As(solveErr, &sce) {
|
||||
var err error
|
||||
inputIDs, err = lbf.registerSlowCacheResult(sce.Result)
|
||||
inputIDs, err = lbf.registerResultIDs(sce.Result)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -739,32 +766,14 @@ func (lbf *llbBridgeForwarder) wrapSolveError(solveErr error) error {
|
||||
return errdefs.WithSolveError(solveErr, subject, inputIDs, mountIDs)
|
||||
}
|
||||
|
||||
// registerResultIDs retains gateway-owned clones of error refs until discard.
|
||||
func (lbf *llbBridgeForwarder) registerResultIDs(results ...solver.Result) (ids []string, err error) {
|
||||
lbf.mu.Lock()
|
||||
defer lbf.mu.Unlock()
|
||||
|
||||
return lbf.registerResultIDsLocked(results...)
|
||||
}
|
||||
|
||||
// registerSlowCacheResult registers a clone of a result borrowed from the
|
||||
// solver job. The clone is only taken before the forwarder is discarded, as the
|
||||
// job may be released after that.
|
||||
func (lbf *llbBridgeForwarder) registerSlowCacheResult(res solver.Result) ([]string, error) {
|
||||
lbf.mu.Lock()
|
||||
defer lbf.mu.Unlock()
|
||||
|
||||
if lbf.discarded {
|
||||
return nil, nil
|
||||
}
|
||||
// Validate before cloning so a type error can't leak the clone. Once
|
||||
// cloned, registerResultIDsLocked either keeps or releases it.
|
||||
if _, ok := res.Sys().(*worker.WorkerRef); !ok {
|
||||
return nil, errors.Errorf("unexpected type for result, got %T", res.Sys())
|
||||
}
|
||||
return lbf.registerResultIDsLocked(res.Clone())
|
||||
}
|
||||
|
||||
func (lbf *llbBridgeForwarder) registerResultIDsLocked(results ...solver.Result) (ids []string, err error) {
|
||||
ids = make([]string, len(results))
|
||||
for i, res := range results {
|
||||
if res == nil {
|
||||
@@ -776,16 +785,11 @@ func (lbf *llbBridgeForwarder) registerResultIDsLocked(results ...solver.Result)
|
||||
}
|
||||
id := workerRef.ID()
|
||||
ids[i] = id
|
||||
if existing, ok := lbf.resultByID[id]; ok {
|
||||
if existing != res {
|
||||
if err := res.Release(context.TODO()); err != nil {
|
||||
return ids, errors.WithStack(err)
|
||||
}
|
||||
}
|
||||
continue
|
||||
if _, ok := lbf.resultByID[id]; !ok {
|
||||
// Cached sharedOp.Exec errors have snapshots pinned by the active
|
||||
// solver state, even after the original error refs are released.
|
||||
lbf.resultByID[id] = res.Clone()
|
||||
}
|
||||
// Keep the result wrapper so Release balances split-result ownership.
|
||||
lbf.resultByID[id] = res
|
||||
}
|
||||
return ids, nil
|
||||
}
|
||||
@@ -834,6 +838,16 @@ func (lbf *llbBridgeForwarder) Solve(ctx context.Context, req *pb.SolveRequest)
|
||||
|
||||
lbf.mu.Lock()
|
||||
|
||||
if lbf.discarded {
|
||||
// A solve that finished after Discard has no one to release its refs.
|
||||
lbf.mu.Unlock()
|
||||
res.EachRef(func(r solver.ResultProxy) error {
|
||||
r.Release(context.TODO())
|
||||
return nil
|
||||
})
|
||||
return nil, stack.Enable(status.Error(codes.Unavailable, "gateway forwarder is closing"))
|
||||
}
|
||||
|
||||
if res.Refs != nil {
|
||||
ids := make(map[string]string, len(res.Refs))
|
||||
defs := make(map[string]*opspb.Definition, len(res.Refs))
|
||||
@@ -1869,13 +1883,28 @@ func (lbf *llbBridgeForwarder) cloneRef(id string) (solver.ResultProxy, error) {
|
||||
return s2, nil
|
||||
}
|
||||
|
||||
type bufferedConn struct {
|
||||
net.Conn
|
||||
reader *bufio.Reader
|
||||
}
|
||||
|
||||
func (c *bufferedConn) Read(p []byte) (int, error) {
|
||||
return c.reader.Read(p)
|
||||
}
|
||||
|
||||
func serve(ctx context.Context, grpcServer *grpc.Server, conn net.Conn) {
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
conn.Close()
|
||||
}()
|
||||
stop := context.AfterFunc(ctx, func() { conn.Close() })
|
||||
defer stop()
|
||||
defer conn.Close()
|
||||
|
||||
// The frontend container may take a while to start. Start HTTP/2's
|
||||
// client preface timeout only once the frontend begins communicating.
|
||||
reader := bufio.NewReader(conn)
|
||||
if _, err := reader.Peek(1); err != nil {
|
||||
return
|
||||
}
|
||||
bklog.G(ctx).Debugf("serving grpc connection")
|
||||
(&http2.Server{}).ServeConn(conn, &http2.ServeConnOpts{Handler: grpcServer}) //nolint:staticcheck // This existing HTTP/2 connection is already established.
|
||||
(&http2.Server{}).ServeConn(&bufferedConn{Conn: conn, reader: reader}, &http2.ServeConnOpts{Handler: grpcServer}) //nolint:staticcheck // This existing HTTP/2 connection is already established.
|
||||
}
|
||||
|
||||
type markTypeFrontend struct{}
|
||||
|
||||
+38
-26
@@ -97,7 +97,8 @@ func convertRef(ref client.Reference) (*pb.Ref, error) {
|
||||
if !ok {
|
||||
return nil, errors.Errorf("invalid return reference type %T", ref)
|
||||
}
|
||||
return &pb.Ref{Id: r.id, Def: r.def}, nil
|
||||
// Return only needs the ID of the reference held by the server.
|
||||
return &pb.Ref{Id: r.id}, nil
|
||||
}
|
||||
|
||||
func RunFromEnvironment(ctx context.Context, f client.BuildFunc) error {
|
||||
@@ -821,12 +822,14 @@ func (c *grpcClient) Inputs(ctx context.Context) (map[string]llb.State, error) {
|
||||
// communication channel between the process and the ExecProcess message
|
||||
// stream.
|
||||
type procMessageForwarder struct {
|
||||
ctx context.Context
|
||||
done chan struct{}
|
||||
msgs chan *pb.ExecMessage
|
||||
}
|
||||
|
||||
func newProcMessageForwarder() *procMessageForwarder {
|
||||
func newProcMessageForwarder(ctx context.Context) *procMessageForwarder {
|
||||
return &procMessageForwarder{
|
||||
ctx: ctx,
|
||||
done: make(chan struct{}),
|
||||
msgs: make(chan *pb.ExecMessage),
|
||||
}
|
||||
@@ -840,14 +843,16 @@ func (b *procMessageForwarder) Send(ctx context.Context, m *pb.ExecMessage) {
|
||||
}
|
||||
}
|
||||
|
||||
func (b *procMessageForwarder) Recv(ctx context.Context) (m *pb.ExecMessage, ok bool) {
|
||||
func (b *procMessageForwarder) Recv(ctx context.Context) (*pb.ExecMessage, error) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, true
|
||||
return nil, context.Cause(ctx)
|
||||
case <-b.ctx.Done():
|
||||
return nil, context.Cause(b.ctx)
|
||||
case <-b.done:
|
||||
return nil, false
|
||||
case m = <-b.msgs:
|
||||
return m, true
|
||||
return nil, nil
|
||||
case m := <-b.msgs:
|
||||
return m, nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -908,6 +913,11 @@ func (m *messageForwarder) Start() (err error) {
|
||||
m.eg.Go(func() error {
|
||||
for {
|
||||
msg, err := m.stream.Recv()
|
||||
if err != nil {
|
||||
// EOF and cancellation also end delivery to every process,
|
||||
// even though they are not errors for Release.
|
||||
m.cancel(errors.Wrap(err, "remote exec stream closed"))
|
||||
}
|
||||
if errors.Is(err, io.EOF) || grpcerrors.Code(err) == codes.Canceled {
|
||||
return nil
|
||||
}
|
||||
@@ -976,7 +986,7 @@ func (m *messageForwarder) Release() error {
|
||||
func (m *messageForwarder) Register(pid string) *procMessageForwarder {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
sender := newProcMessageForwarder()
|
||||
sender := newProcMessageForwarder(m.ctx)
|
||||
m.pids[pid] = sender
|
||||
return sender
|
||||
}
|
||||
@@ -1120,7 +1130,11 @@ func (ctr *container) Start(ctx context.Context, req client.StartRequest) (clien
|
||||
return nil, err
|
||||
}
|
||||
|
||||
msg, _ := msgs.Recv(ctx)
|
||||
msg, err := msgs.Recv(ctx)
|
||||
if err != nil {
|
||||
ctr.execMsgs.Deregister(pid)
|
||||
return nil, err
|
||||
}
|
||||
if msg == nil {
|
||||
return nil, errors.New("failed to receive started message")
|
||||
}
|
||||
@@ -1165,13 +1179,11 @@ func (ctr *container) Start(ctx context.Context, req client.StartRequest) (clien
|
||||
processID: pid,
|
||||
fd: 0,
|
||||
}
|
||||
_, err := io.Copy(m, stdinReader)
|
||||
// ignore ErrClosedPipe, it is EOF for our usage.
|
||||
if err != nil && !errors.Is(err, io.ErrClosedPipe) {
|
||||
return err
|
||||
_, copyErr := io.Copy(m, stdinReader)
|
||||
if copyErr != nil && !errors.Is(copyErr, io.ErrClosedPipe) {
|
||||
return copyErr
|
||||
}
|
||||
// not an error so must be eof
|
||||
return ctr.execMsgs.Send(&pb.ExecMessage{
|
||||
err := ctr.execMsgs.Send(&pb.ExecMessage{
|
||||
ProcessID: pid,
|
||||
Input: &pb.ExecMessage_File{
|
||||
File: &pb.FdMessage{
|
||||
@@ -1180,6 +1192,12 @@ func (ctr *container) Start(ctx context.Context, req client.StartRequest) (clien
|
||||
},
|
||||
},
|
||||
})
|
||||
// Cleanup still sends EOF to unblock remote stdin, but a failed send
|
||||
// must not replace the process result that triggered cleanup.
|
||||
if errors.Is(copyErr, io.ErrClosedPipe) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1188,21 +1206,15 @@ func (ctr *container) Start(ctx context.Context, req client.StartRequest) (clien
|
||||
defer closeDoneOnce.Do(func() { close(done) })
|
||||
var exitError error
|
||||
for {
|
||||
msg, ok := msgs.Recv(ctx)
|
||||
if !ok {
|
||||
msg, err := msgs.Recv(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if msg == nil {
|
||||
// no more messages, return
|
||||
return exitError
|
||||
}
|
||||
|
||||
if msg == nil {
|
||||
// empty message from ctx cancel, so just start shutting down
|
||||
// input
|
||||
closeDoneOnce.Do(func() {
|
||||
close(done)
|
||||
})
|
||||
return context.Cause(ctx)
|
||||
}
|
||||
|
||||
if file := msg.GetFile(); file != nil {
|
||||
var out io.WriteCloser
|
||||
switch file.Fd {
|
||||
|
||||
+8
@@ -3,6 +3,7 @@ package imagerefchecker
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
@@ -39,6 +40,13 @@ func (c *Checker) Exists(key string, blobs []digest.Digest) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// An empty digest means that a layer in the chain has no blob, so it was
|
||||
// never exported and no image can reference it. Without this check, the
|
||||
// key of such a chain would be the same as the key of its parent chain.
|
||||
if slices.Contains(blobs, "") {
|
||||
return false
|
||||
}
|
||||
|
||||
c.once.Do(c.init)
|
||||
|
||||
if b, ok := c.cache[key]; ok {
|
||||
|
||||
+2
@@ -37,6 +37,8 @@ func (lm *localMounter) Mount() (string, error) {
|
||||
}
|
||||
}
|
||||
|
||||
lm.mounts = mount.RemoveVolatileOption(lm.mounts)
|
||||
|
||||
var isFile bool
|
||||
if len(lm.mounts) == 1 && (lm.mounts[0].Type == "bind" || lm.mounts[0].Type == "rbind") {
|
||||
if !lm.forceRemount {
|
||||
|
||||
+28
-30
@@ -409,42 +409,40 @@ func (c *cacheManager) ensurePersistentKey(k *CacheKey) error {
|
||||
|
||||
func (c *cacheManager) getIDFromDeps(k *CacheKey) string {
|
||||
matches := map[string]struct{}{}
|
||||
|
||||
loop0:
|
||||
for i, deps := range k.deps {
|
||||
if i == 0 || len(matches) > 0 {
|
||||
for _, ck := range deps {
|
||||
m2 := make(map[string]struct{})
|
||||
if err := c.backend.WalkLinks(c.getID(ck.CacheKey.CacheKey), CacheInfoLink{
|
||||
Input: Index(i),
|
||||
Output: k.Output(),
|
||||
Digest: k.Digest(),
|
||||
Selector: ck.Selector,
|
||||
}, func(id string) error {
|
||||
if i == 0 {
|
||||
matches[id] = struct{}{}
|
||||
} else {
|
||||
m2[id] = struct{}{}
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
matches = map[string]struct{}{}
|
||||
break
|
||||
}
|
||||
if i != 0 {
|
||||
for id := range matches {
|
||||
if _, ok := m2[id]; !ok {
|
||||
delete(matches, id)
|
||||
}
|
||||
}
|
||||
if i > 0 && len(matches) == 0 {
|
||||
break
|
||||
}
|
||||
// Keys within one input are alternatives; different inputs must all match.
|
||||
candidates := map[string]struct{}{}
|
||||
for _, ck := range deps {
|
||||
if err := c.backend.WalkLinks(c.getID(ck.CacheKey.CacheKey), CacheInfoLink{
|
||||
Input: Index(i),
|
||||
Output: k.Output(),
|
||||
Digest: k.Digest(),
|
||||
Selector: ck.Selector,
|
||||
}, func(id string) error {
|
||||
candidates[id] = struct{}{}
|
||||
return nil
|
||||
}); err != nil {
|
||||
matches = nil
|
||||
break loop0
|
||||
}
|
||||
}
|
||||
if i == 0 {
|
||||
matches = candidates
|
||||
} else {
|
||||
for id := range matches {
|
||||
if _, ok := candidates[id]; !ok {
|
||||
delete(matches, id)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for k := range matches {
|
||||
return k
|
||||
for id := range matches {
|
||||
return id
|
||||
}
|
||||
|
||||
return identity.NewID()
|
||||
}
|
||||
|
||||
|
||||
+1
-1
@@ -36,7 +36,7 @@ func withAncestorCacheOpts(ctx context.Context, start *state) context.Context {
|
||||
}
|
||||
values := make(map[any]any)
|
||||
walkAncestors(ctx, start, func(st *state) bool {
|
||||
if st.clientVertex.Error != "" {
|
||||
if st.vertexFailed() {
|
||||
// don't use values from cancelled or otherwise error'd vertexes
|
||||
return false
|
||||
}
|
||||
|
||||
+18
-7
@@ -69,7 +69,10 @@ type edge struct {
|
||||
keysDidChange bool
|
||||
index *edgeIndex
|
||||
|
||||
secondaryExporters []expDep
|
||||
// secondaryExportersMu protects secondaryExporters, which the scheduler
|
||||
// appends to on merge while cache exporters read it from other goroutines
|
||||
secondaryExportersMu sync.Mutex
|
||||
secondaryExporters []expDep
|
||||
|
||||
failedOnce sync.Once
|
||||
debug bool
|
||||
@@ -729,6 +732,14 @@ func (e *edge) processDepSlowCacheReq(index int, dep *dep) {
|
||||
// respondToIncoming responds to all incoming requests. completing or
|
||||
// updating them when possible
|
||||
func (e *edge) respondToIncoming(incoming []pipeSender, allPipes []pipeReceiver) (edgeStatusType, bool) {
|
||||
// Requests can be canceled concurrently. Decide on one snapshot so a late
|
||||
// cancel cannot finish the last request while outgoing work is active; it
|
||||
// is handled on the next dispatch.
|
||||
canceled := make([]bool, len(incoming))
|
||||
for i, req := range incoming {
|
||||
canceled[i] = req.Request().Canceled
|
||||
}
|
||||
|
||||
// detect the result state for the requests
|
||||
allIncomingCanComplete := true
|
||||
desiredState := e.state
|
||||
@@ -737,8 +748,8 @@ func (e *edge) respondToIncoming(incoming []pipeSender, allPipes []pipeReceiver)
|
||||
// check incoming requests
|
||||
// check if all requests can be either answered or canceled
|
||||
if !e.isComplete() {
|
||||
for _, req := range incoming {
|
||||
if !req.Request().Canceled {
|
||||
for i, req := range incoming {
|
||||
if !canceled[i] {
|
||||
allCanceled = false
|
||||
if r := req.Request().Payload; desiredState < r.desiredState {
|
||||
desiredState = r.desiredState
|
||||
@@ -765,8 +776,8 @@ func (e *edge) respondToIncoming(incoming []pipeSender, allPipes []pipeReceiver)
|
||||
|
||||
// can close all but one requests
|
||||
var leaveOpen pipeSender
|
||||
for _, req := range incoming {
|
||||
if !req.Request().Canceled {
|
||||
for i, req := range incoming {
|
||||
if !canceled[i] {
|
||||
leaveOpen = req
|
||||
break
|
||||
}
|
||||
@@ -790,9 +801,9 @@ func (e *edge) respondToIncoming(incoming []pipeSender, allPipes []pipeReceiver)
|
||||
}
|
||||
|
||||
// update incoming based on current state
|
||||
for _, req := range incoming {
|
||||
for i, req := range incoming {
|
||||
r := req.Request().Payload
|
||||
if req.Request().Canceled {
|
||||
if canceled[i] {
|
||||
e.finishIncoming(req)
|
||||
} else if !e.hasActiveOutgoing && e.state >= r.desiredState {
|
||||
e.finishIncoming(req)
|
||||
|
||||
+4
-1
@@ -246,7 +246,10 @@ func (e *exporter) ExportTo(ctx context.Context, t CacheExporterTarget, opt Cach
|
||||
}
|
||||
|
||||
if e.edge != nil {
|
||||
for _, de := range e.edge.secondaryExporters {
|
||||
e.edge.secondaryExportersMu.Lock()
|
||||
secondaryExporters := slices.Clone(e.edge.secondaryExporters)
|
||||
e.edge.secondaryExportersMu.Unlock()
|
||||
for _, de := range secondaryExporters {
|
||||
recs, err := de.cacheKey.CacheKey.Exporter.ExportTo(mainCtx, t, opt)
|
||||
if err != nil {
|
||||
continue
|
||||
|
||||
+3
@@ -160,11 +160,14 @@ func (ei *edgeIndex) enforceIndexID(k *CacheKey) {
|
||||
|
||||
matches := ei.getAllMatches(k)
|
||||
|
||||
// TraceFields reads indexIDs under the cache-key lock.
|
||||
k.mu.Lock()
|
||||
if len(matches) > 0 {
|
||||
k.indexIDs = matches
|
||||
} else {
|
||||
k.indexIDs = []string{identity.NewID()}
|
||||
}
|
||||
k.mu.Unlock()
|
||||
|
||||
for _, id := range k.indexIDs {
|
||||
ei.enforceLinked(id, k)
|
||||
|
||||
+1
-1
@@ -153,7 +153,7 @@ func (pw *sender[Payload, Value]) Finalize(v Value, err error) {
|
||||
pw.status.Value = v
|
||||
pw.status.Err = err
|
||||
pw.status.Completed = true
|
||||
if errors.Is(err, context.Canceled) && pw.req.Canceled {
|
||||
if errors.Is(err, context.Canceled) && pw.Request().Canceled {
|
||||
pw.status.Canceled = true
|
||||
}
|
||||
pw.sendChannel.Send(pw.status)
|
||||
|
||||
+91
-34
@@ -61,9 +61,10 @@ type state struct {
|
||||
mspan *tracing.MultiSpan
|
||||
execSpan trace.Span
|
||||
|
||||
vtx Vertex
|
||||
clientVertex client.Vertex
|
||||
origDigest digest.Digest // original LLB digest. TODO: probably better to use string ID so this isn't needed
|
||||
vtx Vertex
|
||||
clientVertex client.Vertex
|
||||
clientVertexMu sync.Mutex // protects mutable clientVertex fields
|
||||
origDigest digest.Digest // original LLB digest. TODO: probably better to use string ID so this isn't needed
|
||||
|
||||
mu sync.RWMutex
|
||||
op *sharedOp
|
||||
@@ -329,6 +330,51 @@ func (s *state) combinedCacheManager() CacheManager {
|
||||
return NewCombinedCacheManager(cms, mainCache)
|
||||
}
|
||||
|
||||
// writeVertexSnapshot can read progress concurrently with
|
||||
// notifyStarted, which is the only writer of the mutable vertex fields.
|
||||
func (s *state) writeVertexSnapshot(pw progress.Writer) {
|
||||
s.clientVertexMu.Lock()
|
||||
defer s.clientVertexMu.Unlock()
|
||||
// Keep the snapshot and write together so a newer update cannot be
|
||||
// published before this snapshot.
|
||||
pw.Write(identity.NewID(), s.clientVertex)
|
||||
}
|
||||
|
||||
// vertexFailed lets the ancestor walk read the error concurrently with
|
||||
// progress updates from notifyStarted.
|
||||
func (s *state) vertexFailed() bool {
|
||||
s.clientVertexMu.Lock()
|
||||
defer s.clientVertexMu.Unlock()
|
||||
return s.clientVertex.Error != ""
|
||||
}
|
||||
|
||||
func (s *state) notifyStarted(ctx context.Context, cached bool) func(err error, cached bool) {
|
||||
pw, _, _ := progress.NewFromContext(ctx)
|
||||
s.clientVertexMu.Lock()
|
||||
v := &s.clientVertex
|
||||
start := time.Now()
|
||||
v.Started = &start
|
||||
v.Completed = nil
|
||||
v.Cached = cached
|
||||
id := identity.NewID()
|
||||
pw.Write(id, *v)
|
||||
s.clientVertexMu.Unlock()
|
||||
return func(err error, cached bool) {
|
||||
defer pw.Close()
|
||||
s.clientVertexMu.Lock()
|
||||
defer s.clientVertexMu.Unlock()
|
||||
stop := time.Now()
|
||||
v.Completed = &stop
|
||||
v.Cached = cached
|
||||
if err != nil {
|
||||
v.Error = err.Error()
|
||||
} else {
|
||||
v.Error = ""
|
||||
}
|
||||
pw.Write(id, *v)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *state) Release() {
|
||||
for _, e := range s.edges {
|
||||
for e.owner != nil {
|
||||
@@ -388,7 +434,9 @@ func (sb *subBuilder) EachValue(ctx context.Context, key string, fn func(any) er
|
||||
}
|
||||
|
||||
type Job struct {
|
||||
mu sync.Mutex // protects completedTime, pw, span
|
||||
mu sync.Mutex // protects completedTime, pw, span, builds, discarded
|
||||
builds int
|
||||
discarded bool
|
||||
list *Solver
|
||||
pr *progress.MultiReader
|
||||
pw progress.Writer
|
||||
@@ -689,7 +737,7 @@ func (jl *Solver) connectProgressFromState(target, src *state) {
|
||||
if _, ok := target.allPw[pw]; !ok {
|
||||
target.mpw.Add(pw)
|
||||
target.allPw[pw] = struct{}{}
|
||||
pw.Write(identity.NewID(), target.clientVertex)
|
||||
target.writeVertexSnapshot(pw)
|
||||
if span != nil && span.SpanContext().IsValid() {
|
||||
target.mspan.Add(span)
|
||||
}
|
||||
@@ -800,11 +848,17 @@ func (jl *Solver) deleteIfUnreferenced(k digest.Digest, st *state) {
|
||||
}
|
||||
|
||||
func (j *Job) Build(ctx context.Context, e Edge) (CachedResultWithProvenance, error) {
|
||||
if span := trace.SpanFromContext(ctx); span.SpanContext().IsValid() {
|
||||
j.mu.Lock()
|
||||
j.span = span
|
||||
j.mu.Lock()
|
||||
if j.discarded {
|
||||
j.mu.Unlock()
|
||||
return nil, errors.Errorf("build on discarded job %s", j.id)
|
||||
}
|
||||
j.builds++
|
||||
if span := trace.SpanFromContext(ctx); span.SpanContext().IsValid() {
|
||||
j.span = span
|
||||
}
|
||||
j.mu.Unlock()
|
||||
defer j.buildDone()
|
||||
|
||||
v, err := j.list.load(ctx, e.Vertex, nil, j)
|
||||
if err != nil {
|
||||
@@ -873,7 +927,32 @@ func (j *Job) CloseProgress() {
|
||||
j.pw.Close()
|
||||
}
|
||||
|
||||
// buildDone finishes a pending Discard when the last running build returns.
|
||||
func (j *Job) buildDone() {
|
||||
j.mu.Lock()
|
||||
j.builds--
|
||||
last := j.discarded && j.builds == 0
|
||||
j.mu.Unlock()
|
||||
if last {
|
||||
j.discard()
|
||||
}
|
||||
}
|
||||
|
||||
// Discard releases the job's states. While builds are still running the states
|
||||
// are released when the last one returns, so results that complete after
|
||||
// Discard are released with their state.
|
||||
func (j *Job) Discard() error {
|
||||
j.mu.Lock()
|
||||
j.discarded = true
|
||||
running := j.builds > 0
|
||||
j.mu.Unlock()
|
||||
if running {
|
||||
return nil
|
||||
}
|
||||
return j.discard()
|
||||
}
|
||||
|
||||
func (j *Job) discard() error {
|
||||
j.list.mu.Lock()
|
||||
defer j.list.mu.Unlock()
|
||||
|
||||
@@ -1058,7 +1137,7 @@ func (s *sharedOp) LoadCache(ctx context.Context, rec *CacheRecord) (Result, fun
|
||||
// no cache hit. start evaluating the node
|
||||
span, ctx := tracing.StartSpan(ctx, "load cache: "+s.st.vtx.Name(), trace.WithAttributes(attribute.String("vertex", s.st.vtx.Digest().String())))
|
||||
s.st.execSpan = span
|
||||
notifyCompleted := notifyStarted(ctx, &s.st.clientVertex, true)
|
||||
notifyCompleted := s.st.notifyStarted(ctx, true)
|
||||
res, err := s.Cache().Load(withAncestorCacheOpts(ctx, s.st), rec)
|
||||
tracing.FinishWithError(span, err)
|
||||
notifyCompleted(err, true)
|
||||
@@ -1145,7 +1224,7 @@ func (s *sharedOp) CalcSlowCache(ctx context.Context, index Index, p PreprocessF
|
||||
if s.st.mspan.Span != nil {
|
||||
ctx = trace.ContextWithSpan(ctx, s.st.mspan)
|
||||
}
|
||||
notifyCompleted := notifyStarted(ctx, &s.st.clientVertex, false)
|
||||
notifyCompleted := s.st.notifyStarted(ctx, false)
|
||||
notifyCompleted(err, false)
|
||||
return "", err
|
||||
}
|
||||
@@ -1177,7 +1256,7 @@ func (s *sharedOp) CacheMap(ctx context.Context, index int) (resp *cacheMapResp,
|
||||
if len(s.st.vtx.Inputs()) == 0 {
|
||||
// no cache hit. start evaluating the node
|
||||
span, ctx := tracing.StartSpan(ctx, "cache request: "+s.st.vtx.Name(), trace.WithAttributes(attribute.String("vertex", s.st.vtx.Digest().String())))
|
||||
notifyCompleted := notifyStarted(ctx, &s.st.clientVertex, false)
|
||||
notifyCompleted := s.st.notifyStarted(ctx, false)
|
||||
defer func() {
|
||||
tracing.FinishWithError(span, retErr)
|
||||
notifyCompleted(retErr, false)
|
||||
@@ -1263,7 +1342,7 @@ func (s *sharedOp) Exec(ctx context.Context, inputs []Result) (outputs []Result,
|
||||
// no cache hit. start evaluating the node
|
||||
span, ctx := tracing.StartSpan(ctx, s.st.vtx.Name(), trace.WithAttributes(attribute.String("vertex", s.st.vtx.Digest().String())))
|
||||
s.st.execSpan = span
|
||||
notifyCompleted := notifyStarted(ctx, &s.st.clientVertex, false)
|
||||
notifyCompleted := s.st.notifyStarted(ctx, false)
|
||||
defer func() {
|
||||
tracing.FinishWithError(span, retErr)
|
||||
notifyCompleted(retErr, false)
|
||||
@@ -1390,28 +1469,6 @@ func (v *vertexWithCacheOptions) Inputs() []Edge {
|
||||
return v.inputs
|
||||
}
|
||||
|
||||
func notifyStarted(ctx context.Context, v *client.Vertex, cached bool) func(err error, cached bool) {
|
||||
pw, _, _ := progress.NewFromContext(ctx)
|
||||
start := time.Now()
|
||||
v.Started = &start
|
||||
v.Completed = nil
|
||||
v.Cached = cached
|
||||
id := identity.NewID()
|
||||
pw.Write(id, *v)
|
||||
return func(err error, cached bool) {
|
||||
defer pw.Close()
|
||||
stop := time.Now()
|
||||
v.Completed = &stop
|
||||
v.Cached = cached
|
||||
if err != nil {
|
||||
v.Error = err.Error()
|
||||
} else {
|
||||
v.Error = ""
|
||||
}
|
||||
pw.Write(id, *v)
|
||||
}
|
||||
}
|
||||
|
||||
type SlowCacheError struct {
|
||||
error
|
||||
Index Index
|
||||
|
||||
+28
-10
@@ -5,6 +5,7 @@ import (
|
||||
"maps"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/moby/buildkit/solver/pb"
|
||||
"github.com/moby/buildkit/util/bklog"
|
||||
@@ -40,9 +41,10 @@ type Device struct {
|
||||
}
|
||||
|
||||
type Manager struct {
|
||||
cache *cdi.Cache
|
||||
locker *locker.Locker
|
||||
autoAllowed map[string]struct{}
|
||||
cache *cdi.Cache
|
||||
locker *locker.Locker
|
||||
autoAllowedMu sync.RWMutex
|
||||
autoAllowed map[string]struct{}
|
||||
}
|
||||
|
||||
func NewManager(cache *cdi.Cache, autoAllowed []string) *Manager {
|
||||
@@ -58,6 +60,8 @@ func NewManager(cache *cdi.Cache, autoAllowed []string) *Manager {
|
||||
}
|
||||
|
||||
func (m *Manager) isAutoAllowed(kind, name string, annotations map[string]string) bool {
|
||||
m.autoAllowedMu.RLock()
|
||||
defer m.autoAllowedMu.RUnlock()
|
||||
if _, ok := m.autoAllowed[name]; ok {
|
||||
return true
|
||||
}
|
||||
@@ -152,7 +156,25 @@ func (m *Manager) FindDevices(devs ...*pb.CDIDevice) ([]string, error) {
|
||||
return dedupSlice(out), nil
|
||||
}
|
||||
|
||||
// MatchDevices returns the registered devices that the request resolves to,
|
||||
// including matches by class annotation. Unlike FindDevices it does not fail
|
||||
// or warn when nothing matches.
|
||||
func (m *Manager) MatchDevices(dev *pb.CDIDevice) []string {
|
||||
return m.matchDevice(dev, m.cache.ListDevices())
|
||||
}
|
||||
|
||||
func (m *Manager) parseDevice(dev *pb.CDIDevice, all []string) ([]string, error) {
|
||||
out := m.matchDevice(dev, all)
|
||||
if len(out) == 0 {
|
||||
if !dev.Optional {
|
||||
return nil, errors.Errorf("required device %q is not registered", dev.Name)
|
||||
}
|
||||
bklog.G(context.TODO()).Warnf("Optional device %q is not registered", dev.Name)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (m *Manager) matchDevice(dev *pb.CDIDevice, all []string) []string {
|
||||
var out []string
|
||||
|
||||
kind, name, _ := strings.Cut(dev.Name, "=")
|
||||
@@ -197,13 +219,7 @@ func (m *Manager) parseDevice(dev *pb.CDIDevice, all []string) ([]string, error)
|
||||
}
|
||||
}
|
||||
|
||||
if len(out) == 0 {
|
||||
if !dev.Optional {
|
||||
return nil, errors.Errorf("required device %q is not registered", dev.Name)
|
||||
}
|
||||
bklog.G(context.TODO()).Warnf("Optional device %q is not registered", dev.Name)
|
||||
}
|
||||
return out, nil
|
||||
return out
|
||||
}
|
||||
|
||||
func (m *Manager) hasDevice(name string) bool {
|
||||
@@ -249,7 +265,9 @@ func (m *Manager) OnDemandInstaller(name string) (func(context.Context) error, b
|
||||
}
|
||||
|
||||
// TODO: this needs to be set as annotation to survive reboot
|
||||
m.autoAllowedMu.Lock()
|
||||
m.autoAllowed[name] = struct{}{}
|
||||
m.autoAllowedMu.Unlock()
|
||||
|
||||
return nil
|
||||
}, true
|
||||
|
||||
+28
-7
@@ -3,17 +3,27 @@ package errdefs
|
||||
import (
|
||||
"context"
|
||||
"runtime"
|
||||
"sync"
|
||||
|
||||
"github.com/moby/buildkit/solver"
|
||||
"github.com/moby/buildkit/util/bklog"
|
||||
)
|
||||
|
||||
// ExecError will be returned when an error is encountered when evaluating an op.
|
||||
// The error owns its original refs until Borrow or the first Release. Proxies
|
||||
// and gateways retain the error; gateways also keep independent clones. They may
|
||||
// clone cached originals after Release, which native result handles support
|
||||
// while another ref pins the snapshot. When sharedOp.Exec completes with an
|
||||
// active solver state, that state pins error refs until its last job leaves.
|
||||
// Pinning follows a single Unwrap chain, not errors.Join branches; CacheMap and
|
||||
// Acquire errors do not get that pin.
|
||||
type ExecError struct {
|
||||
error
|
||||
Inputs []solver.Result
|
||||
Mounts []solver.Result
|
||||
OwnerBorrowed bool
|
||||
|
||||
mu sync.Mutex // protects OwnerBorrowed
|
||||
}
|
||||
|
||||
func (e *ExecError) Unwrap() error {
|
||||
@@ -49,16 +59,27 @@ func (e *ExecError) EachRef(fn func(solver.Result) error) (err error) {
|
||||
return err
|
||||
}
|
||||
|
||||
func (e *ExecError) Release() error {
|
||||
// Borrow marks the error's refs as owned by a caller, so the finalizer and
|
||||
// Release do not release them. The same error can be shared by multiple
|
||||
// callers; Borrow returns false if the refs were already borrowed.
|
||||
// The successful caller must release the refs; later callers acquire no ownership.
|
||||
func (e *ExecError) Borrow() bool {
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
if e.OwnerBorrowed {
|
||||
return false
|
||||
}
|
||||
e.OwnerBorrowed = true
|
||||
return true
|
||||
}
|
||||
|
||||
func (e *ExecError) Release() error {
|
||||
if !e.Borrow() {
|
||||
return nil
|
||||
}
|
||||
err := e.EachRef(func(r solver.Result) error {
|
||||
r.Release(context.TODO())
|
||||
return nil
|
||||
return e.EachRef(func(r solver.Result) error {
|
||||
return r.Release(context.TODO())
|
||||
})
|
||||
e.OwnerBorrowed = true
|
||||
return err
|
||||
}
|
||||
|
||||
func WithExecError(err error, inputs, mounts []solver.Result) error {
|
||||
@@ -75,7 +96,7 @@ func WithExecErrorWithContext(ctx context.Context, err error, inputs, mounts []s
|
||||
Mounts: mounts,
|
||||
}
|
||||
runtime.SetFinalizer(ee, func(e *ExecError) {
|
||||
if !e.OwnerBorrowed {
|
||||
if e.Borrow() {
|
||||
e.EachRef(func(r solver.Result) error {
|
||||
bklog.G(ctx).Warn("leaked execError detected and released")
|
||||
r.Release(context.TODO())
|
||||
|
||||
+1
-5
@@ -137,11 +137,7 @@ func (s *Solver) recordBuildHistory(ctx context.Context, id string, req frontend
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
pr, err := pc.Predicate(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
dt, err := json.MarshalIndent(pr, "", " ")
|
||||
dt, err := pc.PredicateJSON(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
+6
-6
@@ -71,12 +71,12 @@ func (m *staticEmulatorMount) Mount() ([]mount.Mount, func() error, error) {
|
||||
|
||||
ret = true
|
||||
return []mount.Mount{{
|
||||
Type: "bind",
|
||||
Source: filepath.Join(tmpdir, qemuMountName),
|
||||
Options: []string{"ro", "bind"},
|
||||
}}, func() error {
|
||||
return os.RemoveAll(tmpdir)
|
||||
}, nil
|
||||
Type: "bind",
|
||||
Source: filepath.Join(tmpdir, qemuMountName),
|
||||
Options: []string{"ro", "bind"},
|
||||
}}, func() error {
|
||||
return os.RemoveAll(tmpdir)
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (m *staticEmulatorMount) IdentityMapping() *user.IdentityMapping {
|
||||
|
||||
+2
-9
@@ -2,7 +2,6 @@ package proc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strconv"
|
||||
|
||||
"github.com/moby/buildkit/executor/resources"
|
||||
@@ -65,14 +64,8 @@ func ProvenanceProcessor(slsaVersion provenancetypes.ProvenanceSLSA, attrs map[s
|
||||
InToto: result.InTotoAttestation{
|
||||
PredicateType: pc.PredicateType(),
|
||||
},
|
||||
Path: filename,
|
||||
ContentFunc: func(ctx context.Context) ([]byte, error) {
|
||||
pr, err := pc.Predicate(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return json.MarshalIndent(pr, "", " ")
|
||||
},
|
||||
Path: filename,
|
||||
ContentFunc: pc.PredicateJSON,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
+20
-1
@@ -2,6 +2,7 @@ package llbsolver
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strconv"
|
||||
@@ -384,6 +385,7 @@ func captureProvenance(ctx context.Context, res solver.CachedResultWithProvenanc
|
||||
}
|
||||
|
||||
type ProvenanceCreator struct {
|
||||
mu sync.Mutex // serializes predicate updates and encoding of pr
|
||||
pr *provenancetypes.ProvenancePredicateSLSA1
|
||||
slsaVersion provenancetypes.ProvenanceSLSA
|
||||
j *solver.Job
|
||||
@@ -643,7 +645,24 @@ func (p *ProvenanceCreator) PredicateType() string {
|
||||
return slsa1.PredicateSLSAProvenance
|
||||
}
|
||||
|
||||
func (p *ProvenanceCreator) Predicate(ctx context.Context) (any, error) {
|
||||
// PredicateJSON finalizes the predicate and returns it JSON encoded. It may be
|
||||
// called concurrently, e.g. by parallel exporters sharing one attestation.
|
||||
func (p *ProvenanceCreator) PredicateJSON(ctx context.Context) ([]byte, error) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
|
||||
pr, err := p.predicate(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dt, err := json.MarshalIndent(pr, "", " ")
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
return dt, nil
|
||||
}
|
||||
|
||||
func (p *ProvenanceCreator) predicate(ctx context.Context) (any, error) {
|
||||
end := p.j.RegisterCompleteTime()
|
||||
if p.pr.RunDetails.Metadata == nil {
|
||||
p.pr.RunDetails.Metadata = &provenancetypes.ProvenanceMetadataSLSA1{}
|
||||
|
||||
+13
-12
@@ -2,6 +2,7 @@ package llbsolver
|
||||
|
||||
import (
|
||||
"context"
|
||||
"slices"
|
||||
"sync"
|
||||
|
||||
cacheconfig "github.com/moby/buildkit/cache/config"
|
||||
@@ -46,7 +47,7 @@ type resultProxy struct {
|
||||
released bool
|
||||
v solver.CachedResult
|
||||
err error
|
||||
errResults []solver.Result
|
||||
execErrors []*llberrdefs.ExecError
|
||||
provenance *provenance.Capture
|
||||
}
|
||||
|
||||
@@ -72,8 +73,8 @@ func (rp *resultProxy) Provenance() any {
|
||||
func (rp *resultProxy) Release(ctx context.Context) (err error) {
|
||||
rp.mu.Lock()
|
||||
defer rp.mu.Unlock()
|
||||
for _, res := range rp.errResults {
|
||||
rerr := res.Release(ctx)
|
||||
for _, ee := range rp.execErrors {
|
||||
rerr := ee.Release()
|
||||
if rerr != nil {
|
||||
err = rerr
|
||||
}
|
||||
@@ -82,9 +83,8 @@ func (rp *resultProxy) Release(ctx context.Context) (err error) {
|
||||
if rp.released {
|
||||
bklog.G(ctx).Warnf("release of already released result")
|
||||
}
|
||||
rerr := rp.v.Release(ctx)
|
||||
if err != nil {
|
||||
return rerr
|
||||
if rerr := rp.v.Release(ctx); rerr != nil {
|
||||
err = rerr
|
||||
}
|
||||
}
|
||||
rp.released = true
|
||||
@@ -118,12 +118,13 @@ func (rp *resultProxy) loadResult(ctx context.Context) (solver.CachedResultWithP
|
||||
res, err := rp.b.loadResult(ctx, rp.req.Definition, rp.req.CacheImports, rp.req.SourcePolicies)
|
||||
var ee *llberrdefs.ExecError
|
||||
if errors.As(err, &ee) {
|
||||
ee.EachRef(func(res solver.Result) error {
|
||||
rp.errResults = append(rp.errResults, res)
|
||||
return nil
|
||||
})
|
||||
// acquire ownership so ExecError finalizer doesn't attempt to release as well
|
||||
ee.OwnerBorrowed = true
|
||||
rp.mu.Lock()
|
||||
if rp.released {
|
||||
ee.Release()
|
||||
} else if !slices.Contains(rp.execErrors, ee) {
|
||||
rp.execErrors = append(rp.execErrors, ee)
|
||||
}
|
||||
rp.mu.Unlock()
|
||||
}
|
||||
return res, err
|
||||
}
|
||||
|
||||
+24
-9
@@ -167,26 +167,37 @@ func ValidateEntitlements(ent entitlements.Set, cdiManager *cdidevices.Manager)
|
||||
}
|
||||
|
||||
var allowedDevices []*pb.CDIDevice
|
||||
var nonAliasedDevices []*pb.CDIDevice
|
||||
var devicesToResolve []*pb.CDIDevice
|
||||
onDemand := make(map[string]struct{})
|
||||
for _, d := range cdiManager.ListDevices() {
|
||||
if d.OnDemand && d.AutoAllow {
|
||||
allowedDevices = append(allowedDevices, &pb.CDIDevice{Name: d.Name})
|
||||
onDemand[d.Name] = struct{}{}
|
||||
}
|
||||
}
|
||||
if cfg != nil {
|
||||
for _, d := range op.Exec.CdiDevices {
|
||||
for _, d := range op.Exec.CdiDevices {
|
||||
if d == nil {
|
||||
continue
|
||||
}
|
||||
if cfg != nil {
|
||||
if newName, ok := cfg.Devices[d.Name]; ok && newName != "" {
|
||||
d.Name = newName
|
||||
allowedDevices = append(allowedDevices, d)
|
||||
} else {
|
||||
nonAliasedDevices = append(nonAliasedDevices, d)
|
||||
continue
|
||||
}
|
||||
}
|
||||
} else {
|
||||
nonAliasedDevices = op.Exec.CdiDevices
|
||||
kind, _, _ := strings.Cut(d.Name, "=")
|
||||
// A kind with an on-demand installer has no registered devices
|
||||
// yet, so a match here can only come from a class annotation
|
||||
// on another device. That must go through the permission check.
|
||||
if _, ok := onDemand[kind]; ok && len(cdiManager.MatchDevices(d)) == 0 {
|
||||
// Resolve the requested name after the installer has registered its devices.
|
||||
allowedDevices = append(allowedDevices, d)
|
||||
} else {
|
||||
devicesToResolve = append(devicesToResolve, d)
|
||||
}
|
||||
}
|
||||
|
||||
mountedDevices, err := cdiManager.FindDevices(nonAliasedDevices...)
|
||||
mountedDevices, err := cdiManager.FindDevices(devicesToResolve...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -198,6 +209,10 @@ func ValidateEntitlements(ent entitlements.Set, cdiManager *cdidevices.Manager)
|
||||
grantedDevices := make(map[string]struct{})
|
||||
if cfg != nil {
|
||||
for d := range cfg.Devices {
|
||||
kind, _, _ := strings.Cut(d, "=")
|
||||
if _, ok := onDemand[kind]; ok && len(cdiManager.MatchDevices(&pb.CDIDevice{Name: d})) == 0 {
|
||||
continue
|
||||
}
|
||||
resolved, err := cdiManager.FindDevices(&pb.CDIDevice{Name: d})
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
+1
@@ -13,6 +13,7 @@ const AttrGitFetchByCommit = "git.fetchbycommit"
|
||||
const AttrGitBundle = "git.bundle"
|
||||
const AttrGitCheckoutBundle = "git.checkoutbundle"
|
||||
const AttrGitAdvice = "git.advice"
|
||||
const AttrGitDebugCommands = "git.debugcommands"
|
||||
|
||||
const AttrGitSignatureVerifyPubKey = "git.sig.pubkey"
|
||||
const AttrGitSignatureVerifyRejectExpired = "git.sig.rejectexpired"
|
||||
|
||||
+7
-3
@@ -309,19 +309,23 @@ func (s *scheduler) mergeTo(target, src *edge) bool {
|
||||
delete(s.outgoing, src)
|
||||
s.signal(target)
|
||||
|
||||
var exps []expDep
|
||||
for i, d := range src.deps {
|
||||
for _, k := range d.keys {
|
||||
target.secondaryExporters = append(target.secondaryExporters, expDep{i, CacheKeyWithSelector{CacheKey: k, Selector: src.cacheMap.Deps[i].Selector}})
|
||||
exps = append(exps, expDep{i, CacheKeyWithSelector{CacheKey: k, Selector: src.cacheMap.Deps[i].Selector}})
|
||||
}
|
||||
if d.slowCacheKey != nil {
|
||||
target.secondaryExporters = append(target.secondaryExporters, expDep{i, CacheKeyWithSelector{CacheKey: *d.slowCacheKey}})
|
||||
exps = append(exps, expDep{i, CacheKeyWithSelector{CacheKey: *d.slowCacheKey}})
|
||||
}
|
||||
if d.result != nil {
|
||||
for _, dk := range d.result.CacheKeys() {
|
||||
target.secondaryExporters = append(target.secondaryExporters, expDep{i, CacheKeyWithSelector{CacheKey: dk, Selector: src.cacheMap.Deps[i].Selector}})
|
||||
exps = append(exps, expDep{i, CacheKeyWithSelector{CacheKey: dk, Selector: src.cacheMap.Deps[i].Selector}})
|
||||
}
|
||||
}
|
||||
}
|
||||
target.secondaryExportersMu.Lock()
|
||||
target.secondaryExporters = append(target.secondaryExporters, exps...)
|
||||
target.secondaryExportersMu.Unlock()
|
||||
|
||||
// TODO(tonistiigi): merge cache providers
|
||||
|
||||
|
||||
+24
-11
@@ -49,13 +49,14 @@ type puller struct {
|
||||
ResolverType
|
||||
store sourceresolver.ResolveImageConfigOptStore
|
||||
|
||||
g flightcontrol.Group[struct{}]
|
||||
cacheKeyErr error
|
||||
cacheKeyDone bool
|
||||
releaseTmpLeases func(context.Context) error
|
||||
descHandlers cache.DescHandlers
|
||||
manifest *pull.PulledManifests
|
||||
manifestKey string
|
||||
g flightcontrol.Group[struct{}]
|
||||
cacheKeyErr error
|
||||
cacheKeyDone bool
|
||||
releaseTmpLeases func(context.Context) error
|
||||
descHandlers cache.DescHandlers
|
||||
manifest *pull.PulledManifests
|
||||
manifestKey string
|
||||
platformManifestKey string
|
||||
*pull.Puller
|
||||
}
|
||||
|
||||
@@ -132,8 +133,8 @@ func (p *puller) CacheKey(ctx context.Context, jobCtx solver.JobContext, index i
|
||||
return struct{}{}, err
|
||||
}
|
||||
|
||||
if p.checksum != "" && p.manifest.MainManifestDesc.Digest != p.checksum {
|
||||
return struct{}{}, errors.Errorf("image digest %s for %s does not match expected checksum %s", p.manifest.MainManifestDesc.Digest, p.Ref, p.checksum)
|
||||
if p.checksum != "" && p.manifest.RootManifestDesc.Digest != p.checksum {
|
||||
return struct{}{}, errors.Errorf("image digest %s for %s does not match expected checksum %s", p.manifest.RootManifestDesc.Digest, p.Ref, p.checksum)
|
||||
}
|
||||
|
||||
if ll := p.layerLimit; ll != nil {
|
||||
@@ -172,12 +173,17 @@ func (p *puller) CacheKey(ctx context.Context, jobCtx solver.JobContext, index i
|
||||
}
|
||||
}
|
||||
|
||||
desc := p.manifest.MainManifestDesc
|
||||
desc := p.manifest.RootManifestDesc
|
||||
k, err := mainManifestKey(desc, p.Platform, p.layerLimit)
|
||||
if err != nil {
|
||||
return struct{}{}, err
|
||||
}
|
||||
p.manifestKey = k.String()
|
||||
k, err = mainManifestKey(p.manifest.ImageManifestDesc, p.Platform, p.layerLimit)
|
||||
if err != nil {
|
||||
return struct{}{}, err
|
||||
}
|
||||
p.platformManifestKey = k.String()
|
||||
// Do not use RootFS DiffIDs from the image config as an alternate
|
||||
// cache key. They are verified only when layers are applied, and a
|
||||
// failed malicious pull must not poison another image source.
|
||||
@@ -193,7 +199,14 @@ func (p *puller) CacheKey(ctx context.Context, jobCtx solver.JobContext, index i
|
||||
cacheOpts[cache.DescHandlerKey(dgst)] = descHandler
|
||||
}
|
||||
|
||||
return p.manifestKey, p.manifest.MainManifestDesc.Digest.String(), cacheOpts, true, nil
|
||||
// Keep the index key for existing caches, but also match the selected
|
||||
// manifest so changes to attestations or other platforms do not invalidate
|
||||
// this source. Both keys identify verified manifest content, not DiffIDs
|
||||
// advertised by an unverified image config.
|
||||
if index == 0 {
|
||||
return p.manifestKey, p.manifest.RootManifestDesc.Digest.String(), cacheOpts, p.manifestKey == p.platformManifestKey, nil
|
||||
}
|
||||
return p.platformManifestKey, p.manifest.RootManifestDesc.Digest.String(), cacheOpts, true, nil
|
||||
}
|
||||
|
||||
func (p *puller) Snapshot(ctx context.Context, jobCtx solver.JobContext) (ir cache.ImmutableRef, err error) {
|
||||
|
||||
+4
-4
@@ -72,8 +72,8 @@ func bundleTargetRef(ref string) string {
|
||||
// sha256 object IDs. git ls-remote reads bundle files directly, so this works
|
||||
// before a destination repo exists and lets callers initialize the right object
|
||||
// format for a subsequent fetch/import.
|
||||
func detectBundleSHA256(ctx context.Context, bundlePath string, gitAdvice bool) (bool, error) {
|
||||
buf, err := gitCLI(gitutil.WithGitAdvice(gitAdvice)).Run(ctx, "ls-remote", "--", bundlePath)
|
||||
func detectBundleSHA256(ctx context.Context, bundlePath string, gitAdvice, debugCommands bool) (bool, error) {
|
||||
buf, err := gitCLI(gitutil.WithGitAdvice(gitAdvice), gitutil.WithGitDebugCommands(debugCommands)).Run(ctx, "ls-remote", "--", bundlePath)
|
||||
if err != nil {
|
||||
return false, errors.Wrapf(err, "failed to inspect git bundle %s", bundlePath)
|
||||
}
|
||||
@@ -120,7 +120,7 @@ func (gs *gitSourceHandler) stageBundle(ctx context.Context, g session.Group) (_
|
||||
return "", nil, err
|
||||
}
|
||||
bundlePath := filepath.Join(tmpDir, bundleImportFileName)
|
||||
sha256, err := detectBundleSHA256(ctx, bundlePath, gs.src.Advice)
|
||||
sha256, err := detectBundleSHA256(ctx, bundlePath, gs.src.Advice, gs.src.DebugCommands)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
@@ -133,7 +133,7 @@ func (gs *gitSourceHandler) stageBundle(ctx context.Context, g session.Group) (_
|
||||
if err := os.Mkdir(tmpRepoDir, 0700); err != nil {
|
||||
return "", nil, errors.Wrap(err, "failed to create temp bare repo dir")
|
||||
}
|
||||
tmpGit := gitCLI(gitutil.WithGitAdvice(gs.src.Advice), gitutil.WithGitDir(tmpRepoDir))
|
||||
tmpGit := gitCLI(gitutil.WithGitAdvice(gs.src.Advice), gitutil.WithGitDebugCommands(gs.src.DebugCommands), gitutil.WithGitDir(tmpRepoDir))
|
||||
initArgs := []string{"-c", "init.defaultBranch=master", "init", "--bare"}
|
||||
if sha256 {
|
||||
initArgs = append(initArgs, "--object-format=sha256")
|
||||
|
||||
+1
@@ -29,6 +29,7 @@ type GitIdentifier struct {
|
||||
SkipSubmodules bool
|
||||
MTime string // "checkout" (default) or "commit"
|
||||
Advice bool
|
||||
DebugCommands bool
|
||||
FetchByCommit bool
|
||||
|
||||
// Bundle, when non-empty, instructs the git source to fetch commits from a
|
||||
|
||||
+6
-2
@@ -150,6 +150,8 @@ func (gs *Source) Identifier(scheme, ref string, attrs map[string]string, platfo
|
||||
id.MTime = v
|
||||
case pb.AttrGitAdvice:
|
||||
id.Advice = v == "true"
|
||||
case pb.AttrGitDebugCommands:
|
||||
id.DebugCommands = v == "true"
|
||||
case pb.AttrGitFetchByCommit:
|
||||
id.FetchByCommit = v == "true"
|
||||
case pb.AttrGitBundle:
|
||||
@@ -173,7 +175,7 @@ func (gs *Source) Identifier(scheme, ref string, attrs map[string]string, platfo
|
||||
}
|
||||
|
||||
// needs to be called with repo lock
|
||||
func (gs *Source) mountRemote(ctx context.Context, remote string, authArgs []string, gitAdvice bool, sha256 bool, reset bool, g session.Group) (target string, release func() error, retErr error) {
|
||||
func (gs *Source) mountRemote(ctx context.Context, remote string, authArgs []string, gitAdvice bool, debugCommands bool, sha256 bool, reset bool, g session.Group) (target string, release func() error, retErr error) {
|
||||
sis, err := searchGitRemote(ctx, gs.cache, remote)
|
||||
if err != nil {
|
||||
return "", nil, errors.Wrapf(err, "failed to search metadata for %s", urlutil.RedactCredentials(remote))
|
||||
@@ -237,6 +239,7 @@ func (gs *Source) mountRemote(ctx context.Context, remote string, authArgs []str
|
||||
|
||||
git := gitCLI(
|
||||
gitutil.WithGitAdvice(gitAdvice),
|
||||
gitutil.WithGitDebugCommands(debugCommands),
|
||||
gitutil.WithGitDir(dir),
|
||||
gitutil.WithArgs(authArgs...),
|
||||
)
|
||||
@@ -976,7 +979,7 @@ func (gs *gitSourceHandler) tryRemoteFetch(ctx context.Context, jobCtx solver.Jo
|
||||
}
|
||||
}
|
||||
|
||||
gitDir, unmountGitDir, err := gs.mountRemote(ctx, gs.src.Remote, authArgs, gs.src.Advice, gs.sha256, reset, g)
|
||||
gitDir, unmountGitDir, err := gs.mountRemote(ctx, gs.src.Remote, authArgs, gs.src.Advice, gs.src.DebugCommands, gs.sha256, reset, g)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -1534,6 +1537,7 @@ func (gs *gitSourceHandler) emptyGitCli(ctx context.Context, g session.Group, op
|
||||
|
||||
opts = append([]gitutil.Option{
|
||||
gitutil.WithGitAdvice(gs.src.Advice),
|
||||
gitutil.WithGitDebugCommands(gs.src.DebugCommands),
|
||||
gitutil.WithArgs(gs.authArgs...),
|
||||
gitutil.WithSSHAuthSock(sock),
|
||||
gitutil.WithSSHKnownHosts(knownHosts),
|
||||
|
||||
+44
-2
@@ -8,6 +8,7 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"mime"
|
||||
"net/http"
|
||||
"net/url"
|
||||
@@ -31,6 +32,7 @@ import (
|
||||
"github.com/moby/buildkit/util/bklog"
|
||||
"github.com/moby/buildkit/util/cachedigest"
|
||||
"github.com/moby/buildkit/util/pgpsign"
|
||||
"github.com/moby/buildkit/util/resolver/retryhandler"
|
||||
"github.com/moby/buildkit/util/tracing"
|
||||
"github.com/moby/buildkit/util/urlutil"
|
||||
"github.com/moby/buildkit/version"
|
||||
@@ -215,6 +217,45 @@ func (hs *httpSourceHandler) client(g session.Group) *http.Client {
|
||||
return &http.Client{Transport: newTransport(hs.transport, hs.sm, g)}
|
||||
}
|
||||
|
||||
// doRequest retries transient network failures and HTTP 429/503 responses with
|
||||
// jittered exponential backoff, honoring Retry-After as a minimum delay.
|
||||
// Requests have no body and are safe to replay. Failures reading a successful
|
||||
// response body and other HTTP statuses are left to the caller.
|
||||
func doRequest(ctx context.Context, client *http.Client, req *http.Request) (*http.Response, error) {
|
||||
return retryhandler.WithRetry(ctx, nil, func(ctx context.Context) (*http.Response, error) {
|
||||
resp, err := client.Do(req.Clone(ctx))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp.StatusCode != http.StatusTooManyRequests && resp.StatusCode != http.StatusServiceUnavailable {
|
||||
return resp, nil
|
||||
}
|
||||
// Do not drain an error body: it may stall independently of the headers.
|
||||
resp.Body.Close()
|
||||
return nil, &retryhandler.RetryAfterError{
|
||||
Err: errors.Errorf("invalid response status %d", resp.StatusCode),
|
||||
Delay: retryAfterDelay(resp.Header.Get("Retry-After")),
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// retryAfterDelay parses either delay-seconds or an HTTP-date. Invalid and past
|
||||
// values fall back to the normal backoff. Saturate large values to avoid overflow.
|
||||
func retryAfterDelay(value string) time.Duration {
|
||||
value = strings.TrimSpace(value)
|
||||
seconds, err := strconv.ParseUint(value, 10, 64)
|
||||
if err == nil || errors.Is(err, strconv.ErrRange) {
|
||||
if seconds > uint64(math.MaxInt64/int64(time.Second)) {
|
||||
return time.Duration(math.MaxInt64)
|
||||
}
|
||||
return time.Duration(seconds) * time.Second
|
||||
}
|
||||
if date, err := http.ParseTime(value); err == nil {
|
||||
return max(time.Until(date), 0)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// urlHash is internal hash the etag is stored by that doesn't leak outside
|
||||
// this package.
|
||||
func (hs *httpSourceHandler) urlHash() (digest.Digest, error) {
|
||||
@@ -408,6 +449,7 @@ func (hs *httpSourceHandler) resolveMetadataRef(ctx context.Context, jobCtx solv
|
||||
// we need to add accept-encoding header manually because stdlib only adds it to GET requests
|
||||
// some servers will return different etags if Accept-Encoding header is different
|
||||
req.Header.Set("Accept-Encoding", "gzip")
|
||||
// Best-effort: on any error we fall through to the GET below, which is retried.
|
||||
resp, err := client.Do(req)
|
||||
if err == nil {
|
||||
if resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusNotModified {
|
||||
@@ -454,7 +496,7 @@ func (hs *httpSourceHandler) resolveMetadataRef(ctx context.Context, jobCtx solv
|
||||
req.Header.Del("Accept-Encoding")
|
||||
}
|
||||
|
||||
resp, err := client.Do(req)
|
||||
resp, err := doRequest(ctx, client, req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -889,7 +931,7 @@ func (hs *httpSourceHandler) Snapshot(ctx context.Context, jobCtx solver.JobCont
|
||||
|
||||
client := hs.client(g)
|
||||
|
||||
resp, err := client.Do(req)
|
||||
resp, err := doRequest(ctx, client, req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
Copyright 2009 The Go Authors.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
* Neither the name of Google LLC nor the names of its
|
||||
contributors may be used to endorse or promote products derived from
|
||||
this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
+747
@@ -0,0 +1,747 @@
|
||||
// Copyright 2009 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// Code generated from Go 1.26.5 commit c19862e5f8415b4f24b189d065ed739517c548ba; DO NOT EDIT.
|
||||
|
||||
package go126flate
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
)
|
||||
|
||||
const (
|
||||
NoCompression = 0
|
||||
BestSpeed = 1
|
||||
BestCompression = 9
|
||||
DefaultCompression = -1
|
||||
|
||||
// HuffmanOnly disables Lempel-Ziv match searching and only performs Huffman
|
||||
// entropy encoding. This mode is useful in compressing data that has
|
||||
// already been compressed with an LZ style algorithm (e.g. Snappy or LZ4)
|
||||
// that lacks an entropy encoder. Compression gains are achieved when
|
||||
// certain bytes in the input stream occur more frequently than others.
|
||||
//
|
||||
// Note that HuffmanOnly produces a compressed output that is
|
||||
// RFC 1951 compliant. That is, any valid DEFLATE decompressor will
|
||||
// continue to be able to decompress this output.
|
||||
HuffmanOnly = -2
|
||||
)
|
||||
|
||||
const (
|
||||
logWindowSize = 15
|
||||
windowSize = 1 << logWindowSize
|
||||
windowMask = windowSize - 1
|
||||
|
||||
// The LZ77 step produces a sequence of literal tokens and <length, offset>
|
||||
// pair tokens. The offset is also known as distance. The underlying wire
|
||||
// format limits the range of lengths and offsets. For example, there are
|
||||
// 256 legitimate lengths: those in the range [3, 258]. This package's
|
||||
// compressor uses a higher minimum match length, enabling optimizations
|
||||
// such as finding matches via 32-bit loads and compares.
|
||||
baseMatchLength = 3 // The smallest match length per the RFC section 3.2.5
|
||||
minMatchLength = 4 // The smallest match length that the compressor actually emits
|
||||
maxMatchLength = 258 // The largest match length
|
||||
baseMatchOffset = 1 // The smallest match offset
|
||||
maxMatchOffset = 1 << 15 // The largest match offset
|
||||
|
||||
// The maximum number of tokens we put into a single flate block, just to
|
||||
// stop things from getting too large.
|
||||
maxFlateBlockTokens = 1 << 14
|
||||
maxStoreBlockSize = 65535
|
||||
hashBits = 17 // After 17 performance degrades
|
||||
hashSize = 1 << hashBits
|
||||
hashMask = (1 << hashBits) - 1
|
||||
maxHashOffset = 1 << 24
|
||||
|
||||
skipNever = math.MaxInt32
|
||||
)
|
||||
|
||||
type compressionLevel struct {
|
||||
level, good, lazy, nice, chain, fastSkipHashing int
|
||||
}
|
||||
|
||||
var levels = []compressionLevel{
|
||||
{0, 0, 0, 0, 0, 0}, // NoCompression.
|
||||
{1, 0, 0, 0, 0, 0}, // BestSpeed uses a custom algorithm; see deflatefast.go.
|
||||
// For levels 2-3 we don't bother trying with lazy matches.
|
||||
{2, 4, 0, 16, 8, 5},
|
||||
{3, 4, 0, 32, 32, 6},
|
||||
// Levels 4-9 use increasingly more lazy matching
|
||||
// and increasingly stringent conditions for "good enough".
|
||||
{4, 4, 4, 16, 16, skipNever},
|
||||
{5, 8, 16, 32, 32, skipNever},
|
||||
{6, 8, 16, 128, 128, skipNever},
|
||||
{7, 8, 32, 128, 256, skipNever},
|
||||
{8, 32, 128, 258, 1024, skipNever},
|
||||
{9, 32, 258, 258, 4096, skipNever},
|
||||
}
|
||||
|
||||
type compressor struct {
|
||||
compressionLevel
|
||||
|
||||
w *huffmanBitWriter
|
||||
bulkHasher func([]byte, []uint32)
|
||||
|
||||
// compression algorithm
|
||||
fill func(*compressor, []byte) int // copy data to window
|
||||
step func(*compressor) // process window
|
||||
bestSpeed *deflateFast // Encoder for BestSpeed
|
||||
|
||||
// input window: unprocessed data is window[index:windowEnd]
|
||||
index int
|
||||
window []byte
|
||||
windowEnd int
|
||||
blockStart int // window index where current tokens start
|
||||
byteAvailable bool // if true, still need to process window[index-1].
|
||||
|
||||
sync bool // requesting flush
|
||||
|
||||
// queued output tokens
|
||||
tokens []token
|
||||
|
||||
// deflate state
|
||||
length int
|
||||
offset int
|
||||
maxInsertIndex int
|
||||
err error
|
||||
|
||||
// Input hash chains
|
||||
// hashHead[hashValue] contains the largest inputIndex with the specified hash value
|
||||
// If hashHead[hashValue] is within the current window, then
|
||||
// hashPrev[hashHead[hashValue] & windowMask] contains the previous index
|
||||
// with the same hash value.
|
||||
// These are large and do not contain pointers, so put them
|
||||
// near the end of the struct so the GC has to scan less.
|
||||
chainHead int
|
||||
hashHead [hashSize]uint32
|
||||
hashPrev [windowSize]uint32
|
||||
hashOffset int
|
||||
|
||||
// hashMatch must be able to contain hashes for the maximum match length.
|
||||
hashMatch [maxMatchLength - 1]uint32
|
||||
}
|
||||
|
||||
func (d *compressor) fillDeflate(b []byte) int {
|
||||
if d.index >= 2*windowSize-(minMatchLength+maxMatchLength) {
|
||||
// shift the window by windowSize
|
||||
copy(d.window, d.window[windowSize:2*windowSize])
|
||||
d.index -= windowSize
|
||||
d.windowEnd -= windowSize
|
||||
if d.blockStart >= windowSize {
|
||||
d.blockStart -= windowSize
|
||||
} else {
|
||||
d.blockStart = math.MaxInt32
|
||||
}
|
||||
d.hashOffset += windowSize
|
||||
if d.hashOffset > maxHashOffset {
|
||||
delta := d.hashOffset - 1
|
||||
d.hashOffset -= delta
|
||||
d.chainHead -= delta
|
||||
|
||||
// Iterate over slices instead of arrays to avoid copying
|
||||
// the entire table onto the stack (Issue #18625).
|
||||
for i, v := range d.hashPrev[:] {
|
||||
if int(v) > delta {
|
||||
d.hashPrev[i] = uint32(int(v) - delta)
|
||||
} else {
|
||||
d.hashPrev[i] = 0
|
||||
}
|
||||
}
|
||||
for i, v := range d.hashHead[:] {
|
||||
if int(v) > delta {
|
||||
d.hashHead[i] = uint32(int(v) - delta)
|
||||
} else {
|
||||
d.hashHead[i] = 0
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
n := copy(d.window[d.windowEnd:], b)
|
||||
d.windowEnd += n
|
||||
return n
|
||||
}
|
||||
|
||||
func (d *compressor) writeBlock(tokens []token, index int) error {
|
||||
if index > 0 {
|
||||
var window []byte
|
||||
if d.blockStart <= index {
|
||||
window = d.window[d.blockStart:index]
|
||||
}
|
||||
d.blockStart = index
|
||||
d.w.writeBlock(tokens, false, window)
|
||||
return d.w.err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// fillWindow will fill the current window with the supplied
|
||||
// dictionary and calculate all hashes.
|
||||
// This is much faster than doing a full encode.
|
||||
// Should only be used after a reset.
|
||||
func (d *compressor) fillWindow(b []byte) {
|
||||
// Do not fill window if we are in store-only mode.
|
||||
if d.compressionLevel.level < 2 {
|
||||
return
|
||||
}
|
||||
if d.index != 0 || d.windowEnd != 0 {
|
||||
panic("internal error: fillWindow called with stale data")
|
||||
}
|
||||
|
||||
// If we are given too much, cut it.
|
||||
if len(b) > windowSize {
|
||||
b = b[len(b)-windowSize:]
|
||||
}
|
||||
// Add all to window.
|
||||
n := copy(d.window, b)
|
||||
|
||||
// Calculate 256 hashes at the time (more L1 cache hits)
|
||||
loops := (n + 256 - minMatchLength) / 256
|
||||
for j := 0; j < loops; j++ {
|
||||
index := j * 256
|
||||
end := index + 256 + minMatchLength - 1
|
||||
if end > n {
|
||||
end = n
|
||||
}
|
||||
toCheck := d.window[index:end]
|
||||
dstSize := len(toCheck) - minMatchLength + 1
|
||||
|
||||
if dstSize <= 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
dst := d.hashMatch[:dstSize]
|
||||
d.bulkHasher(toCheck, dst)
|
||||
for i, val := range dst {
|
||||
di := i + index
|
||||
hh := &d.hashHead[val&hashMask]
|
||||
// Get previous value with the same hash.
|
||||
// Our chain should point to the previous value.
|
||||
d.hashPrev[di&windowMask] = *hh
|
||||
// Set the head of the hash chain to us.
|
||||
*hh = uint32(di + d.hashOffset)
|
||||
}
|
||||
}
|
||||
// Update window information.
|
||||
d.windowEnd = n
|
||||
d.index = n
|
||||
}
|
||||
|
||||
// Try to find a match starting at index whose length is greater than prevSize.
|
||||
// We only look at chainCount possibilities before giving up.
|
||||
func (d *compressor) findMatch(pos int, prevHead int, prevLength int, lookahead int) (length, offset int, ok bool) {
|
||||
minMatchLook := maxMatchLength
|
||||
if lookahead < minMatchLook {
|
||||
minMatchLook = lookahead
|
||||
}
|
||||
|
||||
win := d.window[0 : pos+minMatchLook]
|
||||
|
||||
// We quit when we get a match that's at least nice long
|
||||
nice := len(win) - pos
|
||||
if d.nice < nice {
|
||||
nice = d.nice
|
||||
}
|
||||
|
||||
// If we've got a match that's good enough, only look in 1/4 the chain.
|
||||
tries := d.chain
|
||||
length = prevLength
|
||||
if length >= d.good {
|
||||
tries >>= 2
|
||||
}
|
||||
|
||||
wEnd := win[pos+length]
|
||||
wPos := win[pos:]
|
||||
minIndex := pos - windowSize
|
||||
|
||||
for i := prevHead; tries > 0; tries-- {
|
||||
if wEnd == win[i+length] {
|
||||
n := matchLen(win[i:], wPos, minMatchLook)
|
||||
|
||||
if n > length && (n > minMatchLength || pos-i <= 4096) {
|
||||
length = n
|
||||
offset = pos - i
|
||||
ok = true
|
||||
if n >= nice {
|
||||
// The match is good enough that we don't try to find a better one.
|
||||
break
|
||||
}
|
||||
wEnd = win[pos+n]
|
||||
}
|
||||
}
|
||||
if i == minIndex {
|
||||
// hashPrev[i & windowMask] has already been overwritten, so stop now.
|
||||
break
|
||||
}
|
||||
i = int(d.hashPrev[i&windowMask]) - d.hashOffset
|
||||
if i < minIndex || i < 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (d *compressor) writeStoredBlock(buf []byte) error {
|
||||
if d.w.writeStoredHeader(len(buf), false); d.w.err != nil {
|
||||
return d.w.err
|
||||
}
|
||||
d.w.writeBytes(buf)
|
||||
return d.w.err
|
||||
}
|
||||
|
||||
const hashmul = 0x1e35a7bd
|
||||
|
||||
// hash4 returns a hash representation of the first 4 bytes
|
||||
// of the supplied slice.
|
||||
// The caller must ensure that len(b) >= 4.
|
||||
func hash4(b []byte) uint32 {
|
||||
return ((uint32(b[3]) | uint32(b[2])<<8 | uint32(b[1])<<16 | uint32(b[0])<<24) * hashmul) >> (32 - hashBits)
|
||||
}
|
||||
|
||||
// bulkHash4 will compute hashes using the same
|
||||
// algorithm as hash4.
|
||||
func bulkHash4(b []byte, dst []uint32) {
|
||||
if len(b) < minMatchLength {
|
||||
return
|
||||
}
|
||||
hb := uint32(b[3]) | uint32(b[2])<<8 | uint32(b[1])<<16 | uint32(b[0])<<24
|
||||
dst[0] = (hb * hashmul) >> (32 - hashBits)
|
||||
end := len(b) - minMatchLength + 1
|
||||
for i := 1; i < end; i++ {
|
||||
hb = (hb << 8) | uint32(b[i+3])
|
||||
dst[i] = (hb * hashmul) >> (32 - hashBits)
|
||||
}
|
||||
}
|
||||
|
||||
// matchLen returns the number of matching bytes in a and b
|
||||
// up to length 'max'. Both slices must be at least 'max'
|
||||
// bytes in size.
|
||||
func matchLen(a, b []byte, max int) int {
|
||||
a = a[:max]
|
||||
b = b[:len(a)]
|
||||
for i, av := range a {
|
||||
if b[i] != av {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return max
|
||||
}
|
||||
|
||||
// encSpeed will compress and store the currently added data,
|
||||
// if enough has been accumulated or we at the end of the stream.
|
||||
// Any error that occurred will be in d.err
|
||||
func (d *compressor) encSpeed() {
|
||||
// We only compress if we have maxStoreBlockSize.
|
||||
if d.windowEnd < maxStoreBlockSize {
|
||||
if !d.sync {
|
||||
return
|
||||
}
|
||||
|
||||
// Handle small sizes.
|
||||
if d.windowEnd < 128 {
|
||||
switch {
|
||||
case d.windowEnd == 0:
|
||||
return
|
||||
case d.windowEnd <= 16:
|
||||
d.err = d.writeStoredBlock(d.window[:d.windowEnd])
|
||||
default:
|
||||
d.w.writeBlockHuff(false, d.window[:d.windowEnd])
|
||||
d.err = d.w.err
|
||||
}
|
||||
d.windowEnd = 0
|
||||
d.bestSpeed.reset()
|
||||
return
|
||||
}
|
||||
|
||||
}
|
||||
// Encode the block.
|
||||
d.tokens = d.bestSpeed.encode(d.tokens[:0], d.window[:d.windowEnd])
|
||||
|
||||
// If we removed less than 1/16th, Huffman compress the block.
|
||||
if len(d.tokens) > d.windowEnd-(d.windowEnd>>4) {
|
||||
d.w.writeBlockHuff(false, d.window[:d.windowEnd])
|
||||
} else {
|
||||
d.w.writeBlockDynamic(d.tokens, false, d.window[:d.windowEnd])
|
||||
}
|
||||
d.err = d.w.err
|
||||
d.windowEnd = 0
|
||||
}
|
||||
|
||||
func (d *compressor) initDeflate() {
|
||||
d.window = make([]byte, 2*windowSize)
|
||||
d.hashOffset = 1
|
||||
d.tokens = make([]token, 0, maxFlateBlockTokens+1)
|
||||
d.length = minMatchLength - 1
|
||||
d.offset = 0
|
||||
d.byteAvailable = false
|
||||
d.index = 0
|
||||
d.chainHead = -1
|
||||
d.bulkHasher = bulkHash4
|
||||
}
|
||||
|
||||
func (d *compressor) deflate() {
|
||||
if d.windowEnd-d.index < minMatchLength+maxMatchLength && !d.sync {
|
||||
return
|
||||
}
|
||||
|
||||
d.maxInsertIndex = d.windowEnd - (minMatchLength - 1)
|
||||
|
||||
Loop:
|
||||
for {
|
||||
if d.index > d.windowEnd {
|
||||
panic("index > windowEnd")
|
||||
}
|
||||
lookahead := d.windowEnd - d.index
|
||||
if lookahead < minMatchLength+maxMatchLength {
|
||||
if !d.sync {
|
||||
break Loop
|
||||
}
|
||||
if d.index > d.windowEnd {
|
||||
panic("index > windowEnd")
|
||||
}
|
||||
if lookahead == 0 {
|
||||
// Flush current output block if any.
|
||||
if d.byteAvailable {
|
||||
// There is still one pending token that needs to be flushed
|
||||
d.tokens = append(d.tokens, literalToken(uint32(d.window[d.index-1])))
|
||||
d.byteAvailable = false
|
||||
}
|
||||
if len(d.tokens) > 0 {
|
||||
if d.err = d.writeBlock(d.tokens, d.index); d.err != nil {
|
||||
return
|
||||
}
|
||||
d.tokens = d.tokens[:0]
|
||||
}
|
||||
break Loop
|
||||
}
|
||||
}
|
||||
if d.index < d.maxInsertIndex {
|
||||
// Update the hash
|
||||
hash := hash4(d.window[d.index : d.index+minMatchLength])
|
||||
hh := &d.hashHead[hash&hashMask]
|
||||
d.chainHead = int(*hh)
|
||||
d.hashPrev[d.index&windowMask] = uint32(d.chainHead)
|
||||
*hh = uint32(d.index + d.hashOffset)
|
||||
}
|
||||
prevLength := d.length
|
||||
prevOffset := d.offset
|
||||
d.length = minMatchLength - 1
|
||||
d.offset = 0
|
||||
minIndex := d.index - windowSize
|
||||
if minIndex < 0 {
|
||||
minIndex = 0
|
||||
}
|
||||
|
||||
if d.chainHead-d.hashOffset >= minIndex &&
|
||||
(d.fastSkipHashing != skipNever && lookahead > minMatchLength-1 ||
|
||||
d.fastSkipHashing == skipNever && lookahead > prevLength && prevLength < d.lazy) {
|
||||
if newLength, newOffset, ok := d.findMatch(d.index, d.chainHead-d.hashOffset, minMatchLength-1, lookahead); ok {
|
||||
d.length = newLength
|
||||
d.offset = newOffset
|
||||
}
|
||||
}
|
||||
if d.fastSkipHashing != skipNever && d.length >= minMatchLength ||
|
||||
d.fastSkipHashing == skipNever && prevLength >= minMatchLength && d.length <= prevLength {
|
||||
// There was a match at the previous step, and the current match is
|
||||
// not better. Output the previous match.
|
||||
if d.fastSkipHashing != skipNever {
|
||||
d.tokens = append(d.tokens, matchToken(uint32(d.length-baseMatchLength), uint32(d.offset-baseMatchOffset)))
|
||||
} else {
|
||||
d.tokens = append(d.tokens, matchToken(uint32(prevLength-baseMatchLength), uint32(prevOffset-baseMatchOffset)))
|
||||
}
|
||||
// Insert in the hash table all strings up to the end of the match.
|
||||
// index and index-1 are already inserted. If there is not enough
|
||||
// lookahead, the last two strings are not inserted into the hash
|
||||
// table.
|
||||
if d.length <= d.fastSkipHashing {
|
||||
var newIndex int
|
||||
if d.fastSkipHashing != skipNever {
|
||||
newIndex = d.index + d.length
|
||||
} else {
|
||||
newIndex = d.index + prevLength - 1
|
||||
}
|
||||
index := d.index
|
||||
for index++; index < newIndex; index++ {
|
||||
if index < d.maxInsertIndex {
|
||||
hash := hash4(d.window[index : index+minMatchLength])
|
||||
// Get previous value with the same hash.
|
||||
// Our chain should point to the previous value.
|
||||
hh := &d.hashHead[hash&hashMask]
|
||||
d.hashPrev[index&windowMask] = *hh
|
||||
// Set the head of the hash chain to us.
|
||||
*hh = uint32(index + d.hashOffset)
|
||||
}
|
||||
}
|
||||
d.index = index
|
||||
|
||||
if d.fastSkipHashing == skipNever {
|
||||
d.byteAvailable = false
|
||||
d.length = minMatchLength - 1
|
||||
}
|
||||
} else {
|
||||
// For matches this long, we don't bother inserting each individual
|
||||
// item into the table.
|
||||
d.index += d.length
|
||||
}
|
||||
if len(d.tokens) == maxFlateBlockTokens {
|
||||
// The block includes the current character
|
||||
if d.err = d.writeBlock(d.tokens, d.index); d.err != nil {
|
||||
return
|
||||
}
|
||||
d.tokens = d.tokens[:0]
|
||||
}
|
||||
} else {
|
||||
if d.fastSkipHashing != skipNever || d.byteAvailable {
|
||||
i := d.index - 1
|
||||
if d.fastSkipHashing != skipNever {
|
||||
i = d.index
|
||||
}
|
||||
d.tokens = append(d.tokens, literalToken(uint32(d.window[i])))
|
||||
if len(d.tokens) == maxFlateBlockTokens {
|
||||
if d.err = d.writeBlock(d.tokens, i+1); d.err != nil {
|
||||
return
|
||||
}
|
||||
d.tokens = d.tokens[:0]
|
||||
}
|
||||
}
|
||||
d.index++
|
||||
if d.fastSkipHashing == skipNever {
|
||||
d.byteAvailable = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (d *compressor) fillStore(b []byte) int {
|
||||
n := copy(d.window[d.windowEnd:], b)
|
||||
d.windowEnd += n
|
||||
return n
|
||||
}
|
||||
|
||||
func (d *compressor) store() {
|
||||
if d.windowEnd > 0 && (d.windowEnd == maxStoreBlockSize || d.sync) {
|
||||
d.err = d.writeStoredBlock(d.window[:d.windowEnd])
|
||||
d.windowEnd = 0
|
||||
}
|
||||
}
|
||||
|
||||
// storeHuff compresses and stores the currently added data
|
||||
// when the d.window is full or we are at the end of the stream.
|
||||
// Any error that occurred will be in d.err
|
||||
func (d *compressor) storeHuff() {
|
||||
if d.windowEnd < len(d.window) && !d.sync || d.windowEnd == 0 {
|
||||
return
|
||||
}
|
||||
d.w.writeBlockHuff(false, d.window[:d.windowEnd])
|
||||
d.err = d.w.err
|
||||
d.windowEnd = 0
|
||||
}
|
||||
|
||||
func (d *compressor) write(b []byte) (n int, err error) {
|
||||
if d.err != nil {
|
||||
return 0, d.err
|
||||
}
|
||||
n = len(b)
|
||||
for len(b) > 0 {
|
||||
d.step(d)
|
||||
b = b[d.fill(d, b):]
|
||||
if d.err != nil {
|
||||
return 0, d.err
|
||||
}
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func (d *compressor) syncFlush() error {
|
||||
if d.err != nil {
|
||||
return d.err
|
||||
}
|
||||
d.sync = true
|
||||
d.step(d)
|
||||
if d.err == nil {
|
||||
d.w.writeStoredHeader(0, false)
|
||||
d.w.flush()
|
||||
d.err = d.w.err
|
||||
}
|
||||
d.sync = false
|
||||
return d.err
|
||||
}
|
||||
|
||||
func (d *compressor) init(w io.Writer, level int) (err error) {
|
||||
d.w = newHuffmanBitWriter(w)
|
||||
|
||||
switch {
|
||||
case level == NoCompression:
|
||||
d.window = make([]byte, maxStoreBlockSize)
|
||||
d.fill = (*compressor).fillStore
|
||||
d.step = (*compressor).store
|
||||
case level == HuffmanOnly:
|
||||
d.window = make([]byte, maxStoreBlockSize)
|
||||
d.fill = (*compressor).fillStore
|
||||
d.step = (*compressor).storeHuff
|
||||
case level == BestSpeed:
|
||||
d.compressionLevel = levels[level]
|
||||
d.window = make([]byte, maxStoreBlockSize)
|
||||
d.fill = (*compressor).fillStore
|
||||
d.step = (*compressor).encSpeed
|
||||
d.bestSpeed = newDeflateFast()
|
||||
d.tokens = make([]token, maxStoreBlockSize)
|
||||
case level == DefaultCompression:
|
||||
level = 6
|
||||
fallthrough
|
||||
case 2 <= level && level <= 9:
|
||||
d.compressionLevel = levels[level]
|
||||
d.initDeflate()
|
||||
d.fill = (*compressor).fillDeflate
|
||||
d.step = (*compressor).deflate
|
||||
default:
|
||||
return fmt.Errorf("flate: invalid compression level %d: want value in range [-2, 9]", level)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *compressor) reset(w io.Writer) {
|
||||
d.w.reset(w)
|
||||
d.sync = false
|
||||
d.err = nil
|
||||
switch d.compressionLevel.level {
|
||||
case NoCompression:
|
||||
d.windowEnd = 0
|
||||
case BestSpeed:
|
||||
d.windowEnd = 0
|
||||
d.tokens = d.tokens[:0]
|
||||
d.bestSpeed.reset()
|
||||
default:
|
||||
d.chainHead = -1
|
||||
clear(d.hashHead[:])
|
||||
clear(d.hashPrev[:])
|
||||
d.hashOffset = 1
|
||||
d.index, d.windowEnd = 0, 0
|
||||
d.blockStart, d.byteAvailable = 0, false
|
||||
d.tokens = d.tokens[:0]
|
||||
d.length = minMatchLength - 1
|
||||
d.offset = 0
|
||||
d.maxInsertIndex = 0
|
||||
}
|
||||
}
|
||||
|
||||
func (d *compressor) close() error {
|
||||
if d.err == errWriterClosed {
|
||||
return nil
|
||||
}
|
||||
if d.err != nil {
|
||||
return d.err
|
||||
}
|
||||
d.sync = true
|
||||
d.step(d)
|
||||
if d.err != nil {
|
||||
return d.err
|
||||
}
|
||||
if d.w.writeStoredHeader(0, true); d.w.err != nil {
|
||||
return d.w.err
|
||||
}
|
||||
d.w.flush()
|
||||
if d.w.err != nil {
|
||||
return d.w.err
|
||||
}
|
||||
d.err = errWriterClosed
|
||||
return nil
|
||||
}
|
||||
|
||||
// NewWriter returns a new [Writer] compressing data at the given level.
|
||||
// Following zlib, levels range from 1 ([BestSpeed]) to 9 ([BestCompression]);
|
||||
// higher levels typically run slower but compress more. Level 0
|
||||
// ([NoCompression]) does not attempt any compression; it only adds the
|
||||
// necessary DEFLATE framing.
|
||||
// Level -1 ([DefaultCompression]) uses the default compression level.
|
||||
// Level -2 ([HuffmanOnly]) will use Huffman compression only, giving
|
||||
// a very fast compression for all types of input, but sacrificing considerable
|
||||
// compression efficiency.
|
||||
//
|
||||
// If level is in the range [-2, 9] then the error returned will be nil.
|
||||
// Otherwise the error returned will be non-nil.
|
||||
func NewWriter(w io.Writer, level int) (*Writer, error) {
|
||||
var dw Writer
|
||||
if err := dw.d.init(w, level); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &dw, nil
|
||||
}
|
||||
|
||||
// NewWriterDict is like [NewWriter] but initializes the new
|
||||
// [Writer] with a preset dictionary. The returned [Writer] behaves
|
||||
// as if the dictionary had been written to it without producing
|
||||
// any compressed output. The compressed data written to w
|
||||
// can only be decompressed by a reader initialized with the
|
||||
// same dictionary (see [NewReaderDict]).
|
||||
func NewWriterDict(w io.Writer, level int, dict []byte) (*Writer, error) {
|
||||
dw := &dictWriter{w}
|
||||
zw, err := NewWriter(dw, level)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
zw.d.fillWindow(dict)
|
||||
zw.dict = append(zw.dict, dict...) // duplicate dictionary for Reset method.
|
||||
return zw, nil
|
||||
}
|
||||
|
||||
type dictWriter struct {
|
||||
w io.Writer
|
||||
}
|
||||
|
||||
func (w *dictWriter) Write(b []byte) (n int, err error) {
|
||||
return w.w.Write(b)
|
||||
}
|
||||
|
||||
var errWriterClosed = errors.New("flate: closed writer")
|
||||
|
||||
// A Writer takes data written to it and writes the compressed
|
||||
// form of that data to an underlying writer (see [NewWriter]).
|
||||
type Writer struct {
|
||||
d compressor
|
||||
dict []byte
|
||||
}
|
||||
|
||||
// Write writes data to w, which will eventually write the
|
||||
// compressed form of data to its underlying writer.
|
||||
func (w *Writer) Write(data []byte) (n int, err error) {
|
||||
return w.d.write(data)
|
||||
}
|
||||
|
||||
// Flush flushes any pending data to the underlying writer.
|
||||
// It is useful mainly in compressed network protocols, to ensure that
|
||||
// a remote reader has enough data to reconstruct a packet.
|
||||
// Flush does not return until the data has been written.
|
||||
// Calling Flush when there is no pending data still causes the [Writer]
|
||||
// to emit a sync marker of at least 4 bytes.
|
||||
// If the underlying writer returns an error, Flush returns that error.
|
||||
//
|
||||
// In the terminology of the zlib library, Flush is equivalent to Z_SYNC_FLUSH.
|
||||
func (w *Writer) Flush() error {
|
||||
// For more about flushing:
|
||||
// https://www.bolet.org/~pornin/deflate-flush.html
|
||||
return w.d.syncFlush()
|
||||
}
|
||||
|
||||
// Close flushes and closes the writer.
|
||||
func (w *Writer) Close() error {
|
||||
return w.d.close()
|
||||
}
|
||||
|
||||
// Reset discards the writer's state and makes it equivalent to
|
||||
// the result of [NewWriter] or [NewWriterDict] called with dst
|
||||
// and w's level and dictionary.
|
||||
func (w *Writer) Reset(dst io.Writer) {
|
||||
if dw, ok := w.d.w.writer.(*dictWriter); ok {
|
||||
// w was created with NewWriterDict
|
||||
dw.w = dst
|
||||
w.d.reset(dw)
|
||||
w.d.fillWindow(w.dict)
|
||||
} else {
|
||||
// w was created with NewWriter
|
||||
w.d.reset(dst)
|
||||
}
|
||||
}
|
||||
+309
@@ -0,0 +1,309 @@
|
||||
// Copyright 2016 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// Code generated from Go 1.26.5 commit c19862e5f8415b4f24b189d065ed739517c548ba; DO NOT EDIT.
|
||||
|
||||
package go126flate
|
||||
|
||||
import "math"
|
||||
|
||||
// This encoding algorithm, which prioritizes speed over output size, is
|
||||
// based on Snappy's LZ77-style encoder: github.com/golang/snappy
|
||||
|
||||
const (
|
||||
tableBits = 14 // Bits used in the table.
|
||||
tableSize = 1 << tableBits // Size of the table.
|
||||
tableMask = tableSize - 1 // Mask for table indices. Redundant, but can eliminate bounds checks.
|
||||
tableShift = 32 - tableBits // Right-shift to get the tableBits most significant bits of a uint32.
|
||||
|
||||
// Reset the buffer offset when reaching this.
|
||||
// Offsets are stored between blocks as int32 values.
|
||||
// Since the offset we are checking against is at the beginning
|
||||
// of the buffer, we need to subtract the current and input
|
||||
// buffer to not risk overflowing the int32.
|
||||
bufferReset = math.MaxInt32 - maxStoreBlockSize*2
|
||||
)
|
||||
|
||||
func load32(b []byte, i int32) uint32 {
|
||||
b = b[i : i+4 : len(b)] // Help the compiler eliminate bounds checks on the next line.
|
||||
return uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24
|
||||
}
|
||||
|
||||
func load64(b []byte, i int32) uint64 {
|
||||
b = b[i : i+8 : len(b)] // Help the compiler eliminate bounds checks on the next line.
|
||||
return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 |
|
||||
uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56
|
||||
}
|
||||
|
||||
func hash(u uint32) uint32 {
|
||||
return (u * 0x1e35a7bd) >> tableShift
|
||||
}
|
||||
|
||||
// These constants are defined by the Snappy implementation so that its
|
||||
// assembly implementation can fast-path some 16-bytes-at-a-time copies. They
|
||||
// aren't necessary in the pure Go implementation, as we don't use those same
|
||||
// optimizations, but using the same thresholds doesn't really hurt.
|
||||
const (
|
||||
inputMargin = 16 - 1
|
||||
minNonLiteralBlockSize = 1 + 1 + inputMargin
|
||||
)
|
||||
|
||||
type tableEntry struct {
|
||||
val uint32 // Value at destination
|
||||
offset int32
|
||||
}
|
||||
|
||||
// deflateFast maintains the table for matches,
|
||||
// and the previous byte block for cross block matching.
|
||||
type deflateFast struct {
|
||||
table [tableSize]tableEntry
|
||||
prev []byte // Previous block, zero length if unknown.
|
||||
cur int32 // Current match offset.
|
||||
}
|
||||
|
||||
func newDeflateFast() *deflateFast {
|
||||
return &deflateFast{cur: maxStoreBlockSize, prev: make([]byte, 0, maxStoreBlockSize)}
|
||||
}
|
||||
|
||||
// encode encodes a block given in src and appends tokens
|
||||
// to dst and returns the result.
|
||||
func (e *deflateFast) encode(dst []token, src []byte) []token {
|
||||
// Ensure that e.cur doesn't wrap.
|
||||
if e.cur >= bufferReset {
|
||||
e.shiftOffsets()
|
||||
}
|
||||
|
||||
// This check isn't in the Snappy implementation, but there, the caller
|
||||
// instead of the callee handles this case.
|
||||
if len(src) < minNonLiteralBlockSize {
|
||||
e.cur += maxStoreBlockSize
|
||||
e.prev = e.prev[:0]
|
||||
return emitLiteral(dst, src)
|
||||
}
|
||||
|
||||
// sLimit is when to stop looking for offset/length copies. The inputMargin
|
||||
// lets us use a fast path for emitLiteral in the main loop, while we are
|
||||
// looking for copies.
|
||||
sLimit := int32(len(src) - inputMargin)
|
||||
|
||||
// nextEmit is where in src the next emitLiteral should start from.
|
||||
nextEmit := int32(0)
|
||||
s := int32(0)
|
||||
cv := load32(src, s)
|
||||
nextHash := hash(cv)
|
||||
|
||||
for {
|
||||
// Copied from the C++ snappy implementation:
|
||||
//
|
||||
// Heuristic match skipping: If 32 bytes are scanned with no matches
|
||||
// found, start looking only at every other byte. If 32 more bytes are
|
||||
// scanned (or skipped), look at every third byte, etc.. When a match
|
||||
// is found, immediately go back to looking at every byte. This is a
|
||||
// small loss (~5% performance, ~0.1% density) for compressible data
|
||||
// due to more bookkeeping, but for non-compressible data (such as
|
||||
// JPEG) it's a huge win since the compressor quickly "realizes" the
|
||||
// data is incompressible and doesn't bother looking for matches
|
||||
// everywhere.
|
||||
//
|
||||
// The "skip" variable keeps track of how many bytes there are since
|
||||
// the last match; dividing it by 32 (ie. right-shifting by five) gives
|
||||
// the number of bytes to move ahead for each iteration.
|
||||
skip := int32(32)
|
||||
|
||||
nextS := s
|
||||
var candidate tableEntry
|
||||
for {
|
||||
s = nextS
|
||||
bytesBetweenHashLookups := skip >> 5
|
||||
nextS = s + bytesBetweenHashLookups
|
||||
skip += bytesBetweenHashLookups
|
||||
if nextS > sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
candidate = e.table[nextHash&tableMask]
|
||||
now := load32(src, nextS)
|
||||
e.table[nextHash&tableMask] = tableEntry{offset: s + e.cur, val: cv}
|
||||
nextHash = hash(now)
|
||||
|
||||
offset := s - (candidate.offset - e.cur)
|
||||
if offset > maxMatchOffset || cv != candidate.val {
|
||||
// Out of range or not matched.
|
||||
cv = now
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
// A 4-byte match has been found. We'll later see if more than 4 bytes
|
||||
// match. But, prior to the match, src[nextEmit:s] are unmatched. Emit
|
||||
// them as literal bytes.
|
||||
dst = emitLiteral(dst, src[nextEmit:s])
|
||||
|
||||
// Call emitCopy, and then see if another emitCopy could be our next
|
||||
// move. Repeat until we find no match for the input immediately after
|
||||
// what was consumed by the last emitCopy call.
|
||||
//
|
||||
// If we exit this loop normally then we need to call emitLiteral next,
|
||||
// though we don't yet know how big the literal will be. We handle that
|
||||
// by proceeding to the next iteration of the main loop. We also can
|
||||
// exit this loop via goto if we get close to exhausting the input.
|
||||
for {
|
||||
// Invariant: we have a 4-byte match at s, and no need to emit any
|
||||
// literal bytes prior to s.
|
||||
|
||||
// Extend the 4-byte match as long as possible.
|
||||
//
|
||||
s += 4
|
||||
t := candidate.offset - e.cur + 4
|
||||
l := e.matchLen(s, t, src)
|
||||
|
||||
// matchToken is flate's equivalent of Snappy's emitCopy. (length,offset)
|
||||
dst = append(dst, matchToken(uint32(l+4-baseMatchLength), uint32(s-t-baseMatchOffset)))
|
||||
s += l
|
||||
nextEmit = s
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
|
||||
// We could immediately start working at s now, but to improve
|
||||
// compression we first update the hash table at s-1 and at s. If
|
||||
// another emitCopy is not our next move, also calculate nextHash
|
||||
// at s+1. At least on GOARCH=amd64, these three hash calculations
|
||||
// are faster as one load64 call (with some shifts) instead of
|
||||
// three load32 calls.
|
||||
x := load64(src, s-1)
|
||||
prevHash := hash(uint32(x))
|
||||
e.table[prevHash&tableMask] = tableEntry{offset: e.cur + s - 1, val: uint32(x)}
|
||||
x >>= 8
|
||||
currHash := hash(uint32(x))
|
||||
candidate = e.table[currHash&tableMask]
|
||||
e.table[currHash&tableMask] = tableEntry{offset: e.cur + s, val: uint32(x)}
|
||||
|
||||
offset := s - (candidate.offset - e.cur)
|
||||
if offset > maxMatchOffset || uint32(x) != candidate.val {
|
||||
cv = uint32(x >> 8)
|
||||
nextHash = hash(cv)
|
||||
s++
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
emitRemainder:
|
||||
if int(nextEmit) < len(src) {
|
||||
dst = emitLiteral(dst, src[nextEmit:])
|
||||
}
|
||||
e.cur += int32(len(src))
|
||||
e.prev = e.prev[:len(src)]
|
||||
copy(e.prev, src)
|
||||
return dst
|
||||
}
|
||||
|
||||
func emitLiteral(dst []token, lit []byte) []token {
|
||||
for _, v := range lit {
|
||||
dst = append(dst, literalToken(uint32(v)))
|
||||
}
|
||||
return dst
|
||||
}
|
||||
|
||||
// matchLen returns the match length between src[s:] and src[t:].
|
||||
// t can be negative to indicate the match is starting in e.prev.
|
||||
// We assume that src[s-4:s] and src[t-4:t] already match.
|
||||
func (e *deflateFast) matchLen(s, t int32, src []byte) int32 {
|
||||
s1 := int(s) + maxMatchLength - 4
|
||||
if s1 > len(src) {
|
||||
s1 = len(src)
|
||||
}
|
||||
|
||||
// If we are inside the current block
|
||||
if t >= 0 {
|
||||
b := src[t:]
|
||||
a := src[s:s1]
|
||||
b = b[:len(a)]
|
||||
// Extend the match to be as long as possible.
|
||||
for i := range a {
|
||||
if a[i] != b[i] {
|
||||
return int32(i)
|
||||
}
|
||||
}
|
||||
return int32(len(a))
|
||||
}
|
||||
|
||||
// We found a match in the previous block.
|
||||
tp := int32(len(e.prev)) + t
|
||||
if tp < 0 {
|
||||
return 0
|
||||
}
|
||||
|
||||
// Extend the match to be as long as possible.
|
||||
a := src[s:s1]
|
||||
b := e.prev[tp:]
|
||||
if len(b) > len(a) {
|
||||
b = b[:len(a)]
|
||||
}
|
||||
a = a[:len(b)]
|
||||
for i := range b {
|
||||
if a[i] != b[i] {
|
||||
return int32(i)
|
||||
}
|
||||
}
|
||||
|
||||
// If we reached our limit, we matched everything we are
|
||||
// allowed to in the previous block and we return.
|
||||
n := int32(len(b))
|
||||
if int(s+n) == s1 {
|
||||
return n
|
||||
}
|
||||
|
||||
// Continue looking for more matches in the current block.
|
||||
a = src[s+n : s1]
|
||||
b = src[:len(a)]
|
||||
for i := range a {
|
||||
if a[i] != b[i] {
|
||||
return int32(i) + n
|
||||
}
|
||||
}
|
||||
return int32(len(a)) + n
|
||||
}
|
||||
|
||||
// Reset resets the encoding history.
|
||||
// This ensures that no matches are made to the previous block.
|
||||
func (e *deflateFast) reset() {
|
||||
e.prev = e.prev[:0]
|
||||
// Bump the offset, so all matches will fail distance check.
|
||||
// Nothing should be >= e.cur in the table.
|
||||
e.cur += maxMatchOffset
|
||||
|
||||
// Protect against e.cur wraparound.
|
||||
if e.cur >= bufferReset {
|
||||
e.shiftOffsets()
|
||||
}
|
||||
}
|
||||
|
||||
// shiftOffsets will shift down all match offset.
|
||||
// This is only called in rare situations to prevent integer overflow.
|
||||
//
|
||||
// See https://golang.org/issue/18636 and https://github.com/golang/go/issues/34121.
|
||||
func (e *deflateFast) shiftOffsets() {
|
||||
if len(e.prev) == 0 {
|
||||
// We have no history; just clear the table.
|
||||
clear(e.table[:])
|
||||
e.cur = maxMatchOffset + 1
|
||||
return
|
||||
}
|
||||
|
||||
// Shift down everything in the table that isn't already too far away.
|
||||
for i := range e.table[:] {
|
||||
v := e.table[i].offset - e.cur + maxMatchOffset + 1
|
||||
if v < 0 {
|
||||
// We want to reset e.cur to maxMatchOffset + 1, so we need to shift
|
||||
// all table entries down by (e.cur - (maxMatchOffset + 1)).
|
||||
// Because we ignore matches > maxMatchOffset, we can cap
|
||||
// any negative offsets at 0.
|
||||
v = 0
|
||||
}
|
||||
e.table[i].offset = v
|
||||
}
|
||||
e.cur = maxMatchOffset + 1
|
||||
}
|
||||
+20
@@ -0,0 +1,20 @@
|
||||
// Copyright 2009 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// Package go126flate contains the DEFLATE writer from Go 1.26.5.
|
||||
//
|
||||
// The sources are copied from Go commit
|
||||
// c19862e5f8415b4f24b189d065ed739517c548ba. Go 1.27 changed the encoded
|
||||
// output while improving compression speed in CL 707355 for golang/go#75532.
|
||||
// BuildKit keeps this writer to preserve the digests of gzip-compressed layers.
|
||||
package go126flate
|
||||
|
||||
// maxNumLit comes from RFC 1951 section 3.2.7. It normally lives in the Go
|
||||
// flate decoder, which is intentionally not included in this writer-only copy.
|
||||
const maxNumLit = 286
|
||||
|
||||
// InternalError reports an error in the flate code itself.
|
||||
type InternalError string
|
||||
|
||||
func (e InternalError) Error() string { return "flate: internal error: " + string(e) }
|
||||
Generated
Vendored
+695
@@ -0,0 +1,695 @@
|
||||
// Copyright 2009 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// Code generated from Go 1.26.5 commit c19862e5f8415b4f24b189d065ed739517c548ba; DO NOT EDIT.
|
||||
|
||||
package go126flate
|
||||
|
||||
import (
|
||||
"io"
|
||||
)
|
||||
|
||||
const (
|
||||
// The largest offset code.
|
||||
offsetCodeCount = 30
|
||||
|
||||
// The special code used to mark the end of a block.
|
||||
endBlockMarker = 256
|
||||
|
||||
// The first length code.
|
||||
lengthCodesStart = 257
|
||||
|
||||
// The number of codegen codes.
|
||||
codegenCodeCount = 19
|
||||
badCode = 255
|
||||
|
||||
// bufferFlushSize indicates the buffer size
|
||||
// after which bytes are flushed to the writer.
|
||||
// Should preferably be a multiple of 6, since
|
||||
// we accumulate 6 bytes between writes to the buffer.
|
||||
bufferFlushSize = 240
|
||||
|
||||
// bufferSize is the actual output byte buffer size.
|
||||
// It must have additional headroom for a flush
|
||||
// which can contain up to 8 bytes.
|
||||
bufferSize = bufferFlushSize + 8
|
||||
)
|
||||
|
||||
// The number of extra bits needed by length code X - LENGTH_CODES_START.
|
||||
var lengthExtraBits = []int8{
|
||||
/* 257 */ 0, 0, 0,
|
||||
/* 260 */ 0, 0, 0, 0, 0, 1, 1, 1, 1, 2,
|
||||
/* 270 */ 2, 2, 2, 3, 3, 3, 3, 4, 4, 4,
|
||||
/* 280 */ 4, 5, 5, 5, 5, 0,
|
||||
}
|
||||
|
||||
// The length indicated by length code X - LENGTH_CODES_START.
|
||||
var lengthBase = []uint32{
|
||||
0, 1, 2, 3, 4, 5, 6, 7, 8, 10,
|
||||
12, 14, 16, 20, 24, 28, 32, 40, 48, 56,
|
||||
64, 80, 96, 112, 128, 160, 192, 224, 255,
|
||||
}
|
||||
|
||||
// offset code word extra bits.
|
||||
var offsetExtraBits = []int8{
|
||||
0, 0, 0, 0, 1, 1, 2, 2, 3, 3,
|
||||
4, 4, 5, 5, 6, 6, 7, 7, 8, 8,
|
||||
9, 9, 10, 10, 11, 11, 12, 12, 13, 13,
|
||||
}
|
||||
|
||||
var offsetBase = []uint32{
|
||||
0x000000, 0x000001, 0x000002, 0x000003, 0x000004,
|
||||
0x000006, 0x000008, 0x00000c, 0x000010, 0x000018,
|
||||
0x000020, 0x000030, 0x000040, 0x000060, 0x000080,
|
||||
0x0000c0, 0x000100, 0x000180, 0x000200, 0x000300,
|
||||
0x000400, 0x000600, 0x000800, 0x000c00, 0x001000,
|
||||
0x001800, 0x002000, 0x003000, 0x004000, 0x006000,
|
||||
}
|
||||
|
||||
// The odd order in which the codegen code sizes are written.
|
||||
var codegenOrder = []uint32{16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15}
|
||||
|
||||
type huffmanBitWriter struct {
|
||||
// writer is the underlying writer.
|
||||
// Do not use it directly; use the write method, which ensures
|
||||
// that Write errors are sticky.
|
||||
writer io.Writer
|
||||
|
||||
// Data waiting to be written is bytes[0:nbytes]
|
||||
// and then the low nbits of bits. Data is always written
|
||||
// sequentially into the bytes array.
|
||||
bits uint64
|
||||
nbits uint
|
||||
bytes [bufferSize]byte
|
||||
codegenFreq [codegenCodeCount]int32
|
||||
nbytes int
|
||||
literalFreq []int32
|
||||
offsetFreq []int32
|
||||
codegen []uint8
|
||||
literalEncoding *huffmanEncoder
|
||||
offsetEncoding *huffmanEncoder
|
||||
codegenEncoding *huffmanEncoder
|
||||
err error
|
||||
}
|
||||
|
||||
func newHuffmanBitWriter(w io.Writer) *huffmanBitWriter {
|
||||
return &huffmanBitWriter{
|
||||
writer: w,
|
||||
literalFreq: make([]int32, maxNumLit),
|
||||
offsetFreq: make([]int32, offsetCodeCount),
|
||||
codegen: make([]uint8, maxNumLit+offsetCodeCount+1),
|
||||
literalEncoding: newHuffmanEncoder(maxNumLit),
|
||||
codegenEncoding: newHuffmanEncoder(codegenCodeCount),
|
||||
offsetEncoding: newHuffmanEncoder(offsetCodeCount),
|
||||
}
|
||||
}
|
||||
|
||||
func (w *huffmanBitWriter) reset(writer io.Writer) {
|
||||
w.writer = writer
|
||||
w.bits, w.nbits, w.nbytes, w.err = 0, 0, 0, nil
|
||||
}
|
||||
|
||||
func (w *huffmanBitWriter) flush() {
|
||||
if w.err != nil {
|
||||
w.nbits = 0
|
||||
return
|
||||
}
|
||||
n := w.nbytes
|
||||
for w.nbits != 0 {
|
||||
w.bytes[n] = byte(w.bits)
|
||||
w.bits >>= 8
|
||||
if w.nbits > 8 { // Avoid underflow
|
||||
w.nbits -= 8
|
||||
} else {
|
||||
w.nbits = 0
|
||||
}
|
||||
n++
|
||||
}
|
||||
w.bits = 0
|
||||
w.write(w.bytes[:n])
|
||||
w.nbytes = 0
|
||||
}
|
||||
|
||||
func (w *huffmanBitWriter) write(b []byte) {
|
||||
if w.err != nil {
|
||||
return
|
||||
}
|
||||
_, w.err = w.writer.Write(b)
|
||||
}
|
||||
|
||||
func (w *huffmanBitWriter) writeBits(b int32, nb uint) {
|
||||
if w.err != nil {
|
||||
return
|
||||
}
|
||||
w.bits |= uint64(b) << w.nbits
|
||||
w.nbits += nb
|
||||
if w.nbits >= 48 {
|
||||
bits := w.bits
|
||||
w.bits >>= 48
|
||||
w.nbits -= 48
|
||||
n := w.nbytes
|
||||
bytes := w.bytes[n : n+6]
|
||||
bytes[0] = byte(bits)
|
||||
bytes[1] = byte(bits >> 8)
|
||||
bytes[2] = byte(bits >> 16)
|
||||
bytes[3] = byte(bits >> 24)
|
||||
bytes[4] = byte(bits >> 32)
|
||||
bytes[5] = byte(bits >> 40)
|
||||
n += 6
|
||||
if n >= bufferFlushSize {
|
||||
w.write(w.bytes[:n])
|
||||
n = 0
|
||||
}
|
||||
w.nbytes = n
|
||||
}
|
||||
}
|
||||
|
||||
func (w *huffmanBitWriter) writeBytes(bytes []byte) {
|
||||
if w.err != nil {
|
||||
return
|
||||
}
|
||||
n := w.nbytes
|
||||
if w.nbits&7 != 0 {
|
||||
w.err = InternalError("writeBytes with unfinished bits")
|
||||
return
|
||||
}
|
||||
for w.nbits != 0 {
|
||||
w.bytes[n] = byte(w.bits)
|
||||
w.bits >>= 8
|
||||
w.nbits -= 8
|
||||
n++
|
||||
}
|
||||
if n != 0 {
|
||||
w.write(w.bytes[:n])
|
||||
}
|
||||
w.nbytes = 0
|
||||
w.write(bytes)
|
||||
}
|
||||
|
||||
// RFC 1951 3.2.7 specifies a special run-length encoding for specifying
|
||||
// the literal and offset lengths arrays (which are concatenated into a single
|
||||
// array). This method generates that run-length encoding.
|
||||
//
|
||||
// The result is written into the codegen array, and the frequencies
|
||||
// of each code is written into the codegenFreq array.
|
||||
// Codes 0-15 are single byte codes. Codes 16-18 are followed by additional
|
||||
// information. Code badCode is an end marker
|
||||
//
|
||||
// numLiterals The number of literals in literalEncoding
|
||||
// numOffsets The number of offsets in offsetEncoding
|
||||
// litenc, offenc The literal and offset encoder to use
|
||||
func (w *huffmanBitWriter) generateCodegen(numLiterals int, numOffsets int, litEnc, offEnc *huffmanEncoder) {
|
||||
clear(w.codegenFreq[:])
|
||||
// Note that we are using codegen both as a temporary variable for holding
|
||||
// a copy of the frequencies, and as the place where we put the result.
|
||||
// This is fine because the output is always shorter than the input used
|
||||
// so far.
|
||||
codegen := w.codegen // cache
|
||||
// Copy the concatenated code sizes to codegen. Put a marker at the end.
|
||||
cgnl := codegen[:numLiterals]
|
||||
for i := range cgnl {
|
||||
cgnl[i] = uint8(litEnc.codes[i].len)
|
||||
}
|
||||
|
||||
cgnl = codegen[numLiterals : numLiterals+numOffsets]
|
||||
for i := range cgnl {
|
||||
cgnl[i] = uint8(offEnc.codes[i].len)
|
||||
}
|
||||
codegen[numLiterals+numOffsets] = badCode
|
||||
|
||||
size := codegen[0]
|
||||
count := 1
|
||||
outIndex := 0
|
||||
for inIndex := 1; size != badCode; inIndex++ {
|
||||
// INVARIANT: We have seen "count" copies of size that have not yet
|
||||
// had output generated for them.
|
||||
nextSize := codegen[inIndex]
|
||||
if nextSize == size {
|
||||
count++
|
||||
continue
|
||||
}
|
||||
// We need to generate codegen indicating "count" of size.
|
||||
if size != 0 {
|
||||
codegen[outIndex] = size
|
||||
outIndex++
|
||||
w.codegenFreq[size]++
|
||||
count--
|
||||
for count >= 3 {
|
||||
n := 6
|
||||
if n > count {
|
||||
n = count
|
||||
}
|
||||
codegen[outIndex] = 16
|
||||
outIndex++
|
||||
codegen[outIndex] = uint8(n - 3)
|
||||
outIndex++
|
||||
w.codegenFreq[16]++
|
||||
count -= n
|
||||
}
|
||||
} else {
|
||||
for count >= 11 {
|
||||
n := 138
|
||||
if n > count {
|
||||
n = count
|
||||
}
|
||||
codegen[outIndex] = 18
|
||||
outIndex++
|
||||
codegen[outIndex] = uint8(n - 11)
|
||||
outIndex++
|
||||
w.codegenFreq[18]++
|
||||
count -= n
|
||||
}
|
||||
if count >= 3 {
|
||||
// count >= 3 && count <= 10
|
||||
codegen[outIndex] = 17
|
||||
outIndex++
|
||||
codegen[outIndex] = uint8(count - 3)
|
||||
outIndex++
|
||||
w.codegenFreq[17]++
|
||||
count = 0
|
||||
}
|
||||
}
|
||||
count--
|
||||
for ; count >= 0; count-- {
|
||||
codegen[outIndex] = size
|
||||
outIndex++
|
||||
w.codegenFreq[size]++
|
||||
}
|
||||
// Set up invariant for next time through the loop.
|
||||
size = nextSize
|
||||
count = 1
|
||||
}
|
||||
// Marker indicating the end of the codegen.
|
||||
codegen[outIndex] = badCode
|
||||
}
|
||||
|
||||
// dynamicSize returns the size of dynamically encoded data in bits.
|
||||
func (w *huffmanBitWriter) dynamicSize(litEnc, offEnc *huffmanEncoder, extraBits int) (size, numCodegens int) {
|
||||
numCodegens = len(w.codegenFreq)
|
||||
for numCodegens > 4 && w.codegenFreq[codegenOrder[numCodegens-1]] == 0 {
|
||||
numCodegens--
|
||||
}
|
||||
header := 3 + 5 + 5 + 4 + (3 * numCodegens) +
|
||||
w.codegenEncoding.bitLength(w.codegenFreq[:]) +
|
||||
int(w.codegenFreq[16])*2 +
|
||||
int(w.codegenFreq[17])*3 +
|
||||
int(w.codegenFreq[18])*7
|
||||
size = header +
|
||||
litEnc.bitLength(w.literalFreq) +
|
||||
offEnc.bitLength(w.offsetFreq) +
|
||||
extraBits
|
||||
|
||||
return size, numCodegens
|
||||
}
|
||||
|
||||
// fixedSize returns the size of dynamically encoded data in bits.
|
||||
func (w *huffmanBitWriter) fixedSize(extraBits int) int {
|
||||
return 3 +
|
||||
fixedLiteralEncoding.bitLength(w.literalFreq) +
|
||||
fixedOffsetEncoding.bitLength(w.offsetFreq) +
|
||||
extraBits
|
||||
}
|
||||
|
||||
// storedSize calculates the stored size, including header.
|
||||
// The function returns the size in bits and whether the block
|
||||
// fits inside a single block.
|
||||
func (w *huffmanBitWriter) storedSize(in []byte) (int, bool) {
|
||||
if in == nil {
|
||||
return 0, false
|
||||
}
|
||||
if len(in) <= maxStoreBlockSize {
|
||||
return (len(in) + 5) * 8, true
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
func (w *huffmanBitWriter) writeCode(c hcode) {
|
||||
if w.err != nil {
|
||||
return
|
||||
}
|
||||
w.bits |= uint64(c.code) << w.nbits
|
||||
w.nbits += uint(c.len)
|
||||
if w.nbits >= 48 {
|
||||
bits := w.bits
|
||||
w.bits >>= 48
|
||||
w.nbits -= 48
|
||||
n := w.nbytes
|
||||
bytes := w.bytes[n : n+6]
|
||||
bytes[0] = byte(bits)
|
||||
bytes[1] = byte(bits >> 8)
|
||||
bytes[2] = byte(bits >> 16)
|
||||
bytes[3] = byte(bits >> 24)
|
||||
bytes[4] = byte(bits >> 32)
|
||||
bytes[5] = byte(bits >> 40)
|
||||
n += 6
|
||||
if n >= bufferFlushSize {
|
||||
w.write(w.bytes[:n])
|
||||
n = 0
|
||||
}
|
||||
w.nbytes = n
|
||||
}
|
||||
}
|
||||
|
||||
// Write the header of a dynamic Huffman block to the output stream.
|
||||
//
|
||||
// numLiterals The number of literals specified in codegen
|
||||
// numOffsets The number of offsets specified in codegen
|
||||
// numCodegens The number of codegens used in codegen
|
||||
func (w *huffmanBitWriter) writeDynamicHeader(numLiterals int, numOffsets int, numCodegens int, isEof bool) {
|
||||
if w.err != nil {
|
||||
return
|
||||
}
|
||||
var firstBits int32 = 4
|
||||
if isEof {
|
||||
firstBits = 5
|
||||
}
|
||||
w.writeBits(firstBits, 3)
|
||||
w.writeBits(int32(numLiterals-257), 5)
|
||||
w.writeBits(int32(numOffsets-1), 5)
|
||||
w.writeBits(int32(numCodegens-4), 4)
|
||||
|
||||
for i := 0; i < numCodegens; i++ {
|
||||
value := uint(w.codegenEncoding.codes[codegenOrder[i]].len)
|
||||
w.writeBits(int32(value), 3)
|
||||
}
|
||||
|
||||
i := 0
|
||||
for {
|
||||
var codeWord int = int(w.codegen[i])
|
||||
i++
|
||||
if codeWord == badCode {
|
||||
break
|
||||
}
|
||||
w.writeCode(w.codegenEncoding.codes[uint32(codeWord)])
|
||||
|
||||
switch codeWord {
|
||||
case 16:
|
||||
w.writeBits(int32(w.codegen[i]), 2)
|
||||
i++
|
||||
case 17:
|
||||
w.writeBits(int32(w.codegen[i]), 3)
|
||||
i++
|
||||
case 18:
|
||||
w.writeBits(int32(w.codegen[i]), 7)
|
||||
i++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w *huffmanBitWriter) writeStoredHeader(length int, isEof bool) {
|
||||
if w.err != nil {
|
||||
return
|
||||
}
|
||||
var flag int32
|
||||
if isEof {
|
||||
flag = 1
|
||||
}
|
||||
w.writeBits(flag, 3)
|
||||
w.flush()
|
||||
w.writeBits(int32(length), 16)
|
||||
w.writeBits(int32(^uint16(length)), 16)
|
||||
}
|
||||
|
||||
func (w *huffmanBitWriter) writeFixedHeader(isEof bool) {
|
||||
if w.err != nil {
|
||||
return
|
||||
}
|
||||
// Indicate that we are a fixed Huffman block
|
||||
var value int32 = 2
|
||||
if isEof {
|
||||
value = 3
|
||||
}
|
||||
w.writeBits(value, 3)
|
||||
}
|
||||
|
||||
// writeBlock will write a block of tokens with the smallest encoding.
|
||||
// The original input can be supplied, and if the huffman encoded data
|
||||
// is larger than the original bytes, the data will be written as a
|
||||
// stored block.
|
||||
// If the input is nil, the tokens will always be Huffman encoded.
|
||||
func (w *huffmanBitWriter) writeBlock(tokens []token, eof bool, input []byte) {
|
||||
if w.err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
tokens = append(tokens, endBlockMarker)
|
||||
numLiterals, numOffsets := w.indexTokens(tokens)
|
||||
|
||||
var extraBits int
|
||||
storedSize, storable := w.storedSize(input)
|
||||
if storable {
|
||||
// We only bother calculating the costs of the extra bits required by
|
||||
// the length of offset fields (which will be the same for both fixed
|
||||
// and dynamic encoding), if we need to compare those two encodings
|
||||
// against stored encoding.
|
||||
for lengthCode := lengthCodesStart + 8; lengthCode < numLiterals; lengthCode++ {
|
||||
// First eight length codes have extra size = 0.
|
||||
extraBits += int(w.literalFreq[lengthCode]) * int(lengthExtraBits[lengthCode-lengthCodesStart])
|
||||
}
|
||||
for offsetCode := 4; offsetCode < numOffsets; offsetCode++ {
|
||||
// First four offset codes have extra size = 0.
|
||||
extraBits += int(w.offsetFreq[offsetCode]) * int(offsetExtraBits[offsetCode])
|
||||
}
|
||||
}
|
||||
|
||||
// Figure out smallest code.
|
||||
// Fixed Huffman baseline.
|
||||
var literalEncoding = fixedLiteralEncoding
|
||||
var offsetEncoding = fixedOffsetEncoding
|
||||
var size = w.fixedSize(extraBits)
|
||||
|
||||
// Dynamic Huffman?
|
||||
var numCodegens int
|
||||
|
||||
// Generate codegen and codegenFrequencies, which indicates how to encode
|
||||
// the literalEncoding and the offsetEncoding.
|
||||
w.generateCodegen(numLiterals, numOffsets, w.literalEncoding, w.offsetEncoding)
|
||||
w.codegenEncoding.generate(w.codegenFreq[:], 7)
|
||||
dynamicSize, numCodegens := w.dynamicSize(w.literalEncoding, w.offsetEncoding, extraBits)
|
||||
|
||||
if dynamicSize < size {
|
||||
size = dynamicSize
|
||||
literalEncoding = w.literalEncoding
|
||||
offsetEncoding = w.offsetEncoding
|
||||
}
|
||||
|
||||
// Stored bytes?
|
||||
if storable && storedSize < size {
|
||||
w.writeStoredHeader(len(input), eof)
|
||||
w.writeBytes(input)
|
||||
return
|
||||
}
|
||||
|
||||
// Huffman.
|
||||
if literalEncoding == fixedLiteralEncoding {
|
||||
w.writeFixedHeader(eof)
|
||||
} else {
|
||||
w.writeDynamicHeader(numLiterals, numOffsets, numCodegens, eof)
|
||||
}
|
||||
|
||||
// Write the tokens.
|
||||
w.writeTokens(tokens, literalEncoding.codes, offsetEncoding.codes)
|
||||
}
|
||||
|
||||
// writeBlockDynamic encodes a block using a dynamic Huffman table.
|
||||
// This should be used if the symbols used have a disproportionate
|
||||
// histogram distribution.
|
||||
// If input is supplied and the compression savings are below 1/16th of the
|
||||
// input size the block is stored.
|
||||
func (w *huffmanBitWriter) writeBlockDynamic(tokens []token, eof bool, input []byte) {
|
||||
if w.err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
tokens = append(tokens, endBlockMarker)
|
||||
numLiterals, numOffsets := w.indexTokens(tokens)
|
||||
|
||||
// Generate codegen and codegenFrequencies, which indicates how to encode
|
||||
// the literalEncoding and the offsetEncoding.
|
||||
w.generateCodegen(numLiterals, numOffsets, w.literalEncoding, w.offsetEncoding)
|
||||
w.codegenEncoding.generate(w.codegenFreq[:], 7)
|
||||
size, numCodegens := w.dynamicSize(w.literalEncoding, w.offsetEncoding, 0)
|
||||
|
||||
// Store bytes, if we don't get a reasonable improvement.
|
||||
if ssize, storable := w.storedSize(input); storable && ssize < (size+size>>4) {
|
||||
w.writeStoredHeader(len(input), eof)
|
||||
w.writeBytes(input)
|
||||
return
|
||||
}
|
||||
|
||||
// Write Huffman table.
|
||||
w.writeDynamicHeader(numLiterals, numOffsets, numCodegens, eof)
|
||||
|
||||
// Write the tokens.
|
||||
w.writeTokens(tokens, w.literalEncoding.codes, w.offsetEncoding.codes)
|
||||
}
|
||||
|
||||
// indexTokens indexes a slice of tokens, and updates
|
||||
// literalFreq and offsetFreq, and generates literalEncoding
|
||||
// and offsetEncoding.
|
||||
// The number of literal and offset tokens is returned.
|
||||
func (w *huffmanBitWriter) indexTokens(tokens []token) (numLiterals, numOffsets int) {
|
||||
clear(w.literalFreq)
|
||||
clear(w.offsetFreq)
|
||||
|
||||
for _, t := range tokens {
|
||||
if t < matchType {
|
||||
w.literalFreq[t.literal()]++
|
||||
continue
|
||||
}
|
||||
length := t.length()
|
||||
offset := t.offset()
|
||||
w.literalFreq[lengthCodesStart+lengthCode(length)]++
|
||||
w.offsetFreq[offsetCode(offset)]++
|
||||
}
|
||||
|
||||
// get the number of literals
|
||||
numLiterals = len(w.literalFreq)
|
||||
for w.literalFreq[numLiterals-1] == 0 {
|
||||
numLiterals--
|
||||
}
|
||||
// get the number of offsets
|
||||
numOffsets = len(w.offsetFreq)
|
||||
for numOffsets > 0 && w.offsetFreq[numOffsets-1] == 0 {
|
||||
numOffsets--
|
||||
}
|
||||
if numOffsets == 0 {
|
||||
// We haven't found a single match. If we want to go with the dynamic encoding,
|
||||
// we should count at least one offset to be sure that the offset huffman tree could be encoded.
|
||||
w.offsetFreq[0] = 1
|
||||
numOffsets = 1
|
||||
}
|
||||
w.literalEncoding.generate(w.literalFreq, 15)
|
||||
w.offsetEncoding.generate(w.offsetFreq, 15)
|
||||
return
|
||||
}
|
||||
|
||||
// writeTokens writes a slice of tokens to the output.
|
||||
// codes for literal and offset encoding must be supplied.
|
||||
func (w *huffmanBitWriter) writeTokens(tokens []token, leCodes, oeCodes []hcode) {
|
||||
if w.err != nil {
|
||||
return
|
||||
}
|
||||
for _, t := range tokens {
|
||||
if t < matchType {
|
||||
w.writeCode(leCodes[t.literal()])
|
||||
continue
|
||||
}
|
||||
// Write the length
|
||||
length := t.length()
|
||||
lengthCode := lengthCode(length)
|
||||
w.writeCode(leCodes[lengthCode+lengthCodesStart])
|
||||
extraLengthBits := uint(lengthExtraBits[lengthCode])
|
||||
if extraLengthBits > 0 {
|
||||
extraLength := int32(length - lengthBase[lengthCode])
|
||||
w.writeBits(extraLength, extraLengthBits)
|
||||
}
|
||||
// Write the offset
|
||||
offset := t.offset()
|
||||
offsetCode := offsetCode(offset)
|
||||
w.writeCode(oeCodes[offsetCode])
|
||||
extraOffsetBits := uint(offsetExtraBits[offsetCode])
|
||||
if extraOffsetBits > 0 {
|
||||
extraOffset := int32(offset - offsetBase[offsetCode])
|
||||
w.writeBits(extraOffset, extraOffsetBits)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// huffOffset is a static offset encoder used for huffman only encoding.
|
||||
// It can be reused since we will not be encoding offset values.
|
||||
var huffOffset *huffmanEncoder
|
||||
|
||||
func init() {
|
||||
offsetFreq := make([]int32, offsetCodeCount)
|
||||
offsetFreq[0] = 1
|
||||
huffOffset = newHuffmanEncoder(offsetCodeCount)
|
||||
huffOffset.generate(offsetFreq, 15)
|
||||
}
|
||||
|
||||
// writeBlockHuff encodes a block of bytes as either
|
||||
// Huffman encoded literals or uncompressed bytes if the
|
||||
// results only gains very little from compression.
|
||||
func (w *huffmanBitWriter) writeBlockHuff(eof bool, input []byte) {
|
||||
if w.err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Clear histogram
|
||||
clear(w.literalFreq)
|
||||
|
||||
// Add everything as literals
|
||||
histogram(input, w.literalFreq)
|
||||
|
||||
w.literalFreq[endBlockMarker] = 1
|
||||
|
||||
const numLiterals = endBlockMarker + 1
|
||||
w.offsetFreq[0] = 1
|
||||
const numOffsets = 1
|
||||
|
||||
w.literalEncoding.generate(w.literalFreq, 15)
|
||||
|
||||
// Figure out smallest code.
|
||||
// Always use dynamic Huffman or Store
|
||||
var numCodegens int
|
||||
|
||||
// Generate codegen and codegenFrequencies, which indicates how to encode
|
||||
// the literalEncoding and the offsetEncoding.
|
||||
w.generateCodegen(numLiterals, numOffsets, w.literalEncoding, huffOffset)
|
||||
w.codegenEncoding.generate(w.codegenFreq[:], 7)
|
||||
size, numCodegens := w.dynamicSize(w.literalEncoding, huffOffset, 0)
|
||||
|
||||
// Store bytes, if we don't get a reasonable improvement.
|
||||
if ssize, storable := w.storedSize(input); storable && ssize < (size+size>>4) {
|
||||
w.writeStoredHeader(len(input), eof)
|
||||
w.writeBytes(input)
|
||||
return
|
||||
}
|
||||
|
||||
// Huffman.
|
||||
w.writeDynamicHeader(numLiterals, numOffsets, numCodegens, eof)
|
||||
encoding := w.literalEncoding.codes[:257]
|
||||
n := w.nbytes
|
||||
for _, t := range input {
|
||||
// Bitwriting inlined, ~30% speedup
|
||||
c := encoding[t]
|
||||
w.bits |= uint64(c.code) << w.nbits
|
||||
w.nbits += uint(c.len)
|
||||
if w.nbits < 48 {
|
||||
continue
|
||||
}
|
||||
// Store 6 bytes
|
||||
bits := w.bits
|
||||
w.bits >>= 48
|
||||
w.nbits -= 48
|
||||
bytes := w.bytes[n : n+6]
|
||||
bytes[0] = byte(bits)
|
||||
bytes[1] = byte(bits >> 8)
|
||||
bytes[2] = byte(bits >> 16)
|
||||
bytes[3] = byte(bits >> 24)
|
||||
bytes[4] = byte(bits >> 32)
|
||||
bytes[5] = byte(bits >> 40)
|
||||
n += 6
|
||||
if n < bufferFlushSize {
|
||||
continue
|
||||
}
|
||||
w.write(w.bytes[:n])
|
||||
if w.err != nil {
|
||||
return // Return early in the event of write failures
|
||||
}
|
||||
n = 0
|
||||
}
|
||||
w.nbytes = n
|
||||
w.writeCode(encoding[endBlockMarker])
|
||||
}
|
||||
|
||||
// histogram accumulates a histogram of b in h.
|
||||
//
|
||||
// len(h) must be >= 256, and h's elements must be all zeroes.
|
||||
func histogram(b []byte, h []int32) {
|
||||
h = h[:256]
|
||||
for _, t := range b {
|
||||
h[t]++
|
||||
}
|
||||
}
|
||||
+347
@@ -0,0 +1,347 @@
|
||||
// Copyright 2009 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// Code generated from Go 1.26.5 commit c19862e5f8415b4f24b189d065ed739517c548ba; DO NOT EDIT.
|
||||
|
||||
package go126flate
|
||||
|
||||
import (
|
||||
"math"
|
||||
"math/bits"
|
||||
"sort"
|
||||
)
|
||||
|
||||
// hcode is a huffman code with a bit code and bit length.
|
||||
type hcode struct {
|
||||
code, len uint16
|
||||
}
|
||||
|
||||
type huffmanEncoder struct {
|
||||
codes []hcode
|
||||
freqcache []literalNode
|
||||
bitCount [17]int32
|
||||
lns byLiteral // stored to avoid repeated allocation in generate
|
||||
lfs byFreq // stored to avoid repeated allocation in generate
|
||||
}
|
||||
|
||||
type literalNode struct {
|
||||
literal uint16
|
||||
freq int32
|
||||
}
|
||||
|
||||
// A levelInfo describes the state of the constructed tree for a given depth.
|
||||
type levelInfo struct {
|
||||
// Our level. for better printing
|
||||
level int32
|
||||
|
||||
// The frequency of the last node at this level
|
||||
lastFreq int32
|
||||
|
||||
// The frequency of the next character to add to this level
|
||||
nextCharFreq int32
|
||||
|
||||
// The frequency of the next pair (from level below) to add to this level.
|
||||
// Only valid if the "needed" value of the next lower level is 0.
|
||||
nextPairFreq int32
|
||||
|
||||
// The number of chains remaining to generate for this level before moving
|
||||
// up to the next level
|
||||
needed int32
|
||||
}
|
||||
|
||||
// set sets the code and length of an hcode.
|
||||
func (h *hcode) set(code uint16, length uint16) {
|
||||
h.len = length
|
||||
h.code = code
|
||||
}
|
||||
|
||||
func maxNode() literalNode { return literalNode{math.MaxUint16, math.MaxInt32} }
|
||||
|
||||
func newHuffmanEncoder(size int) *huffmanEncoder {
|
||||
return &huffmanEncoder{codes: make([]hcode, size)}
|
||||
}
|
||||
|
||||
// Generates a HuffmanCode corresponding to the fixed literal table.
|
||||
func generateFixedLiteralEncoding() *huffmanEncoder {
|
||||
h := newHuffmanEncoder(maxNumLit)
|
||||
codes := h.codes
|
||||
var ch uint16
|
||||
for ch = 0; ch < maxNumLit; ch++ {
|
||||
var bits uint16
|
||||
var size uint16
|
||||
switch {
|
||||
case ch < 144:
|
||||
// size 8, 000110000 .. 10111111
|
||||
bits = ch + 48
|
||||
size = 8
|
||||
case ch < 256:
|
||||
// size 9, 110010000 .. 111111111
|
||||
bits = ch + 400 - 144
|
||||
size = 9
|
||||
case ch < 280:
|
||||
// size 7, 0000000 .. 0010111
|
||||
bits = ch - 256
|
||||
size = 7
|
||||
default:
|
||||
// size 8, 11000000 .. 11000111
|
||||
bits = ch + 192 - 280
|
||||
size = 8
|
||||
}
|
||||
codes[ch] = hcode{code: reverseBits(bits, byte(size)), len: size}
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
func generateFixedOffsetEncoding() *huffmanEncoder {
|
||||
h := newHuffmanEncoder(30)
|
||||
codes := h.codes
|
||||
for ch := range codes {
|
||||
codes[ch] = hcode{code: reverseBits(uint16(ch), 5), len: 5}
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
var fixedLiteralEncoding *huffmanEncoder = generateFixedLiteralEncoding()
|
||||
var fixedOffsetEncoding *huffmanEncoder = generateFixedOffsetEncoding()
|
||||
|
||||
func (h *huffmanEncoder) bitLength(freq []int32) int {
|
||||
var total int
|
||||
for i, f := range freq {
|
||||
if f != 0 {
|
||||
total += int(f) * int(h.codes[i].len)
|
||||
}
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
const maxBitsLimit = 16
|
||||
|
||||
// bitCounts computes the number of literals assigned to each bit size in the Huffman encoding.
|
||||
// It is only called when list.length >= 3.
|
||||
// The cases of 0, 1, and 2 literals are handled by special case code.
|
||||
//
|
||||
// list is an array of the literals with non-zero frequencies
|
||||
// and their associated frequencies. The array is in order of increasing
|
||||
// frequency and has as its last element a special element with frequency
|
||||
// MaxInt32.
|
||||
//
|
||||
// maxBits is the maximum number of bits that should be used to encode any literal.
|
||||
// It must be less than 16.
|
||||
//
|
||||
// bitCounts returns an integer slice in which slice[i] indicates the number of literals
|
||||
// that should be encoded in i bits.
|
||||
func (h *huffmanEncoder) bitCounts(list []literalNode, maxBits int32) []int32 {
|
||||
if maxBits >= maxBitsLimit {
|
||||
panic("flate: maxBits too large")
|
||||
}
|
||||
n := int32(len(list))
|
||||
list = list[0 : n+1]
|
||||
list[n] = maxNode()
|
||||
|
||||
// The tree can't have greater depth than n - 1, no matter what. This
|
||||
// saves a little bit of work in some small cases
|
||||
if maxBits > n-1 {
|
||||
maxBits = n - 1
|
||||
}
|
||||
|
||||
// Create information about each of the levels.
|
||||
// A bogus "Level 0" whose sole purpose is so that
|
||||
// level1.prev.needed==0. This makes level1.nextPairFreq
|
||||
// be a legitimate value that never gets chosen.
|
||||
var levels [maxBitsLimit]levelInfo
|
||||
// leafCounts[i] counts the number of literals at the left
|
||||
// of ancestors of the rightmost node at level i.
|
||||
// leafCounts[i][j] is the number of literals at the left
|
||||
// of the level j ancestor.
|
||||
var leafCounts [maxBitsLimit][maxBitsLimit]int32
|
||||
|
||||
for level := int32(1); level <= maxBits; level++ {
|
||||
// For every level, the first two items are the first two characters.
|
||||
// We initialize the levels as if we had already figured this out.
|
||||
levels[level] = levelInfo{
|
||||
level: level,
|
||||
lastFreq: list[1].freq,
|
||||
nextCharFreq: list[2].freq,
|
||||
nextPairFreq: list[0].freq + list[1].freq,
|
||||
}
|
||||
leafCounts[level][level] = 2
|
||||
if level == 1 {
|
||||
levels[level].nextPairFreq = math.MaxInt32
|
||||
}
|
||||
}
|
||||
|
||||
// We need a total of 2*n - 2 items at top level and have already generated 2.
|
||||
levels[maxBits].needed = 2*n - 4
|
||||
|
||||
level := maxBits
|
||||
for {
|
||||
l := &levels[level]
|
||||
if l.nextPairFreq == math.MaxInt32 && l.nextCharFreq == math.MaxInt32 {
|
||||
// We've run out of both leaves and pairs.
|
||||
// End all calculations for this level.
|
||||
// To make sure we never come back to this level or any lower level,
|
||||
// set nextPairFreq impossibly large.
|
||||
l.needed = 0
|
||||
levels[level+1].nextPairFreq = math.MaxInt32
|
||||
level++
|
||||
continue
|
||||
}
|
||||
|
||||
prevFreq := l.lastFreq
|
||||
if l.nextCharFreq < l.nextPairFreq {
|
||||
// The next item on this row is a leaf node.
|
||||
n := leafCounts[level][level] + 1
|
||||
l.lastFreq = l.nextCharFreq
|
||||
// Lower leafCounts are the same of the previous node.
|
||||
leafCounts[level][level] = n
|
||||
l.nextCharFreq = list[n].freq
|
||||
} else {
|
||||
// The next item on this row is a pair from the previous row.
|
||||
// nextPairFreq isn't valid until we generate two
|
||||
// more values in the level below
|
||||
l.lastFreq = l.nextPairFreq
|
||||
// Take leaf counts from the lower level, except counts[level] remains the same.
|
||||
copy(leafCounts[level][:level], leafCounts[level-1][:level])
|
||||
levels[l.level-1].needed = 2
|
||||
}
|
||||
|
||||
if l.needed--; l.needed == 0 {
|
||||
// We've done everything we need to do for this level.
|
||||
// Continue calculating one level up. Fill in nextPairFreq
|
||||
// of that level with the sum of the two nodes we've just calculated on
|
||||
// this level.
|
||||
if l.level == maxBits {
|
||||
// All done!
|
||||
break
|
||||
}
|
||||
levels[l.level+1].nextPairFreq = prevFreq + l.lastFreq
|
||||
level++
|
||||
} else {
|
||||
// If we stole from below, move down temporarily to replenish it.
|
||||
for levels[level-1].needed > 0 {
|
||||
level--
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Somethings is wrong if at the end, the top level is null or hasn't used
|
||||
// all of the leaves.
|
||||
if leafCounts[maxBits][maxBits] != n {
|
||||
panic("leafCounts[maxBits][maxBits] != n")
|
||||
}
|
||||
|
||||
bitCount := h.bitCount[:maxBits+1]
|
||||
bits := 1
|
||||
counts := &leafCounts[maxBits]
|
||||
for level := maxBits; level > 0; level-- {
|
||||
// chain.leafCount gives the number of literals requiring at least "bits"
|
||||
// bits to encode.
|
||||
bitCount[bits] = counts[level] - counts[level-1]
|
||||
bits++
|
||||
}
|
||||
return bitCount
|
||||
}
|
||||
|
||||
// Look at the leaves and assign them a bit count and an encoding as specified
|
||||
// in RFC 1951 3.2.2
|
||||
func (h *huffmanEncoder) assignEncodingAndSize(bitCount []int32, list []literalNode) {
|
||||
code := uint16(0)
|
||||
for n, bits := range bitCount {
|
||||
code <<= 1
|
||||
if n == 0 || bits == 0 {
|
||||
continue
|
||||
}
|
||||
// The literals list[len(list)-bits] .. list[len(list)-bits]
|
||||
// are encoded using "bits" bits, and get the values
|
||||
// code, code + 1, .... The code values are
|
||||
// assigned in literal order (not frequency order).
|
||||
chunk := list[len(list)-int(bits):]
|
||||
|
||||
h.lns.sort(chunk)
|
||||
for _, node := range chunk {
|
||||
h.codes[node.literal] = hcode{code: reverseBits(code, uint8(n)), len: uint16(n)}
|
||||
code++
|
||||
}
|
||||
list = list[0 : len(list)-int(bits)]
|
||||
}
|
||||
}
|
||||
|
||||
// Update this Huffman Code object to be the minimum code for the specified frequency count.
|
||||
//
|
||||
// freq is an array of frequencies, in which freq[i] gives the frequency of literal i.
|
||||
// maxBits The maximum number of bits to use for any literal.
|
||||
func (h *huffmanEncoder) generate(freq []int32, maxBits int32) {
|
||||
if h.freqcache == nil {
|
||||
// Allocate a reusable buffer with the longest possible frequency table.
|
||||
// Possible lengths are codegenCodeCount, offsetCodeCount and maxNumLit.
|
||||
// The largest of these is maxNumLit, so we allocate for that case.
|
||||
h.freqcache = make([]literalNode, maxNumLit+1)
|
||||
}
|
||||
list := h.freqcache[:len(freq)+1]
|
||||
// Number of non-zero literals
|
||||
count := 0
|
||||
// Set list to be the set of all non-zero literals and their frequencies
|
||||
for i, f := range freq {
|
||||
if f != 0 {
|
||||
list[count] = literalNode{uint16(i), f}
|
||||
count++
|
||||
} else {
|
||||
h.codes[i].len = 0
|
||||
}
|
||||
}
|
||||
|
||||
list = list[:count]
|
||||
if count <= 2 {
|
||||
// Handle the small cases here, because they are awkward for the general case code. With
|
||||
// two or fewer literals, everything has bit length 1.
|
||||
for i, node := range list {
|
||||
// "list" is in order of increasing literal value.
|
||||
h.codes[node.literal].set(uint16(i), 1)
|
||||
}
|
||||
return
|
||||
}
|
||||
h.lfs.sort(list)
|
||||
|
||||
// Get the number of literals for each bit count
|
||||
bitCount := h.bitCounts(list, maxBits)
|
||||
// And do the assignment
|
||||
h.assignEncodingAndSize(bitCount, list)
|
||||
}
|
||||
|
||||
type byLiteral []literalNode
|
||||
|
||||
func (s *byLiteral) sort(a []literalNode) {
|
||||
*s = byLiteral(a)
|
||||
sort.Sort(s)
|
||||
}
|
||||
|
||||
func (s byLiteral) Len() int { return len(s) }
|
||||
|
||||
func (s byLiteral) Less(i, j int) bool {
|
||||
return s[i].literal < s[j].literal
|
||||
}
|
||||
|
||||
func (s byLiteral) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
|
||||
|
||||
type byFreq []literalNode
|
||||
|
||||
func (s *byFreq) sort(a []literalNode) {
|
||||
*s = byFreq(a)
|
||||
sort.Sort(s)
|
||||
}
|
||||
|
||||
func (s byFreq) Len() int { return len(s) }
|
||||
|
||||
func (s byFreq) Less(i, j int) bool {
|
||||
if s[i].freq == s[j].freq {
|
||||
return s[i].literal < s[j].literal
|
||||
}
|
||||
return s[i].freq < s[j].freq
|
||||
}
|
||||
|
||||
func (s byFreq) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
|
||||
|
||||
func reverseBits(number uint16, bitLength byte) uint16 {
|
||||
return bits.Reverse16(number << (16 - bitLength))
|
||||
}
|
||||
+99
@@ -0,0 +1,99 @@
|
||||
// Copyright 2009 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// Code generated from Go 1.26.5 commit c19862e5f8415b4f24b189d065ed739517c548ba; DO NOT EDIT.
|
||||
|
||||
package go126flate
|
||||
|
||||
const (
|
||||
// 2 bits: type 0 = literal 1=EOF 2=Match 3=Unused
|
||||
// 8 bits: xlength = length - MIN_MATCH_LENGTH
|
||||
// 22 bits xoffset = offset - MIN_OFFSET_SIZE, or literal
|
||||
lengthShift = 22
|
||||
offsetMask = 1<<lengthShift - 1
|
||||
typeMask = 3 << 30
|
||||
literalType = 0 << 30
|
||||
matchType = 1 << 30
|
||||
)
|
||||
|
||||
// The length code for length X (MIN_MATCH_LENGTH <= X <= MAX_MATCH_LENGTH)
|
||||
// is lengthCodes[length - MIN_MATCH_LENGTH]
|
||||
var lengthCodes = [...]uint32{
|
||||
0, 1, 2, 3, 4, 5, 6, 7, 8, 8,
|
||||
9, 9, 10, 10, 11, 11, 12, 12, 12, 12,
|
||||
13, 13, 13, 13, 14, 14, 14, 14, 15, 15,
|
||||
15, 15, 16, 16, 16, 16, 16, 16, 16, 16,
|
||||
17, 17, 17, 17, 17, 17, 17, 17, 18, 18,
|
||||
18, 18, 18, 18, 18, 18, 19, 19, 19, 19,
|
||||
19, 19, 19, 19, 20, 20, 20, 20, 20, 20,
|
||||
20, 20, 20, 20, 20, 20, 20, 20, 20, 20,
|
||||
21, 21, 21, 21, 21, 21, 21, 21, 21, 21,
|
||||
21, 21, 21, 21, 21, 21, 22, 22, 22, 22,
|
||||
22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
|
||||
22, 22, 23, 23, 23, 23, 23, 23, 23, 23,
|
||||
23, 23, 23, 23, 23, 23, 23, 23, 24, 24,
|
||||
24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
|
||||
24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
|
||||
24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
|
||||
25, 25, 25, 25, 25, 25, 25, 25, 25, 25,
|
||||
25, 25, 25, 25, 25, 25, 25, 25, 25, 25,
|
||||
25, 25, 25, 25, 25, 25, 25, 25, 25, 25,
|
||||
25, 25, 26, 26, 26, 26, 26, 26, 26, 26,
|
||||
26, 26, 26, 26, 26, 26, 26, 26, 26, 26,
|
||||
26, 26, 26, 26, 26, 26, 26, 26, 26, 26,
|
||||
26, 26, 26, 26, 27, 27, 27, 27, 27, 27,
|
||||
27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
|
||||
27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
|
||||
27, 27, 27, 27, 27, 28,
|
||||
}
|
||||
|
||||
var offsetCodes = [...]uint32{
|
||||
0, 1, 2, 3, 4, 4, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7,
|
||||
8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9,
|
||||
10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,
|
||||
11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
|
||||
12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
|
||||
12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
|
||||
13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,
|
||||
13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,
|
||||
14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
|
||||
14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
|
||||
14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
|
||||
14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
|
||||
15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
|
||||
15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
|
||||
15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
|
||||
15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
|
||||
}
|
||||
|
||||
type token uint32
|
||||
|
||||
// Convert a literal into a literal token.
|
||||
func literalToken(literal uint32) token { return token(literalType + literal) }
|
||||
|
||||
// Convert a < xlength, xoffset > pair into a match token.
|
||||
func matchToken(xlength uint32, xoffset uint32) token {
|
||||
return token(matchType + xlength<<lengthShift + xoffset)
|
||||
}
|
||||
|
||||
// Returns the literal of a literal token.
|
||||
func (t token) literal() uint32 { return uint32(t - literalType) }
|
||||
|
||||
// Returns the extra offset of a match token.
|
||||
func (t token) offset() uint32 { return uint32(t) & offsetMask }
|
||||
|
||||
func (t token) length() uint32 { return uint32((t - matchType) >> lengthShift) }
|
||||
|
||||
func lengthCode(len uint32) uint32 { return lengthCodes[len] }
|
||||
|
||||
// Returns the offset code corresponding to a specific offset.
|
||||
func offsetCode(off uint32) uint32 {
|
||||
if off < uint32(len(offsetCodes)) {
|
||||
return offsetCodes[off]
|
||||
}
|
||||
if off>>7 < uint32(len(offsetCodes)) {
|
||||
return offsetCodes[off>>7] + 14
|
||||
}
|
||||
return offsetCodes[off>>14] + 28
|
||||
}
|
||||
+92
@@ -0,0 +1,92 @@
|
||||
package compression
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"hash/crc32"
|
||||
"io"
|
||||
|
||||
"github.com/moby/buildkit/util/compression/go126flate"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
const go126GzipDefaultCompression = go126flate.DefaultCompression
|
||||
|
||||
// go126GzipWriter writes the fixed gzip framing used for image layers around
|
||||
// the Go 1.26 DEFLATE stream. Image layers do not use optional gzip headers.
|
||||
type go126GzipWriter struct {
|
||||
w io.Writer
|
||||
level int
|
||||
wroteHeader bool
|
||||
closed bool
|
||||
buf [10]byte
|
||||
compressor *go126flate.Writer
|
||||
digest uint32
|
||||
size uint32
|
||||
err error
|
||||
}
|
||||
|
||||
func newGo126GzipWriterLevel(w io.Writer, level int) (io.WriteCloser, error) {
|
||||
if level < go126flate.HuffmanOnly || level > go126flate.BestCompression {
|
||||
return nil, errors.Errorf("gzip: invalid compression level: %d", level)
|
||||
}
|
||||
return &go126GzipWriter{w: w, level: level}, nil
|
||||
}
|
||||
|
||||
func (z *go126GzipWriter) writeHeader() error {
|
||||
z.wroteHeader = true
|
||||
z.buf = [10]byte{0: 0x1f, 1: 0x8b, 2: 8, 9: 255}
|
||||
switch z.level {
|
||||
case go126flate.BestCompression:
|
||||
z.buf[8] = 2
|
||||
case go126flate.BestSpeed:
|
||||
z.buf[8] = 4
|
||||
}
|
||||
if _, err := z.w.Write(z.buf[:10]); err != nil {
|
||||
return err
|
||||
}
|
||||
compressor, err := go126flate.NewWriter(z.w, z.level)
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
z.compressor = compressor
|
||||
return nil
|
||||
}
|
||||
|
||||
func (z *go126GzipWriter) Write(p []byte) (int, error) {
|
||||
if z.err != nil {
|
||||
return 0, z.err
|
||||
}
|
||||
if !z.wroteHeader {
|
||||
z.err = z.writeHeader()
|
||||
if z.err != nil {
|
||||
return 0, z.err
|
||||
}
|
||||
}
|
||||
z.size += uint32(len(p))
|
||||
z.digest = crc32.Update(z.digest, crc32.IEEETable, p)
|
||||
n, err := z.compressor.Write(p)
|
||||
z.err = err
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (z *go126GzipWriter) Close() error {
|
||||
if z.err != nil {
|
||||
return z.err
|
||||
}
|
||||
if z.closed {
|
||||
return nil
|
||||
}
|
||||
z.closed = true
|
||||
if !z.wroteHeader {
|
||||
if z.err = z.writeHeader(); z.err != nil {
|
||||
return z.err
|
||||
}
|
||||
}
|
||||
if z.err = z.compressor.Close(); z.err != nil {
|
||||
return z.err
|
||||
}
|
||||
binary.LittleEndian.PutUint32(z.buf[:4], z.digest)
|
||||
binary.LittleEndian.PutUint32(z.buf[4:8], z.size)
|
||||
_, z.err = z.w.Write(z.buf[:8])
|
||||
return z.err
|
||||
}
|
||||
+5
-3
@@ -1,7 +1,6 @@
|
||||
package compression
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"io"
|
||||
|
||||
@@ -57,10 +56,13 @@ func (c gzipType) String() string {
|
||||
|
||||
func gzipWriter(comp Config) func(io.Writer) (io.WriteCloser, error) {
|
||||
return func(dest io.Writer) (io.WriteCloser, error) {
|
||||
level := gzip.DefaultCompression
|
||||
level := go126GzipDefaultCompression
|
||||
if comp.Level != nil {
|
||||
level = *comp.Level
|
||||
}
|
||||
return gzip.NewWriterLevel(dest, level)
|
||||
// Go 1.27 changed compress/flate output in CL 707355 for
|
||||
// golang/go#75532. Keep Go 1.26 encoding so existing compatibility
|
||||
// versions continue to produce the same compressed layer digests.
|
||||
return newGo126GzipWriterLevel(dest, level)
|
||||
}
|
||||
}
|
||||
+74
-11
@@ -7,8 +7,11 @@ import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/moby/buildkit/util/urlutil"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
@@ -18,10 +21,11 @@ type GitCLI struct {
|
||||
git string
|
||||
exec func(context.Context, *exec.Cmd) error
|
||||
|
||||
args []string
|
||||
dir string
|
||||
streams StreamFunc
|
||||
advice *bool
|
||||
args []string
|
||||
dir string
|
||||
streams StreamFunc
|
||||
advice *bool
|
||||
debugCommands bool
|
||||
|
||||
workTree string
|
||||
gitDir string
|
||||
@@ -62,6 +66,13 @@ func WithGitAdvice(enabled bool) Option {
|
||||
}
|
||||
}
|
||||
|
||||
// WithGitDebugCommands controls whether executed Git commands are logged.
|
||||
func WithGitDebugCommands(debugCommands bool) Option {
|
||||
return func(b *GitCLI) {
|
||||
b.debugCommands = debugCommands
|
||||
}
|
||||
}
|
||||
|
||||
// WithDir sets working directory.
|
||||
//
|
||||
// This should be a path to any directory within a standard git repository.
|
||||
@@ -190,22 +201,32 @@ func (cli *GitCLI) Run(ctx context.Context, args ...string) (_ []byte, err error
|
||||
cmd.Stdin = nil
|
||||
cmd.Stdout = buf
|
||||
cmd.Stderr = errbuf
|
||||
var stdoutStream io.WriteCloser
|
||||
var stderrStream io.WriteCloser
|
||||
flush := func() {}
|
||||
if cli.streams != nil {
|
||||
stdout, stderr, flush := cli.streams(ctx)
|
||||
if stdout != nil {
|
||||
cmd.Stdout = io.MultiWriter(stdout, cmd.Stdout)
|
||||
var flushStream func()
|
||||
stdoutStream, stderrStream, flushStream = cli.streams(ctx)
|
||||
if flushStream != nil {
|
||||
flush = flushStream
|
||||
}
|
||||
if stderr != nil {
|
||||
cmd.Stderr = io.MultiWriter(stderr, cmd.Stderr)
|
||||
if stdoutStream != nil {
|
||||
cmd.Stdout = io.MultiWriter(stdoutStream, cmd.Stdout)
|
||||
defer stdoutStream.Close()
|
||||
}
|
||||
if stderrStream != nil {
|
||||
cmd.Stderr = io.MultiWriter(stderrStream, cmd.Stderr)
|
||||
defer stderrStream.Close()
|
||||
}
|
||||
defer stdout.Close()
|
||||
defer stderr.Close()
|
||||
defer func() {
|
||||
if err != nil {
|
||||
flush()
|
||||
}
|
||||
}()
|
||||
}
|
||||
if cli.debugCommands && stderrStream != nil {
|
||||
_, _ = io.WriteString(stderrStream, "+ "+formatDebugCommand(cmd.Args)+"\n")
|
||||
}
|
||||
|
||||
cmd.Env = []string{
|
||||
"PATH=" + os.Getenv("PATH"),
|
||||
@@ -297,6 +318,48 @@ func (cli *GitCLI) Run(ctx context.Context, args ...string) (_ []byte, err error
|
||||
}
|
||||
}
|
||||
|
||||
func formatDebugCommand(args []string) string {
|
||||
formatted := make([]string, 0, len(args))
|
||||
for i := 0; i < len(args); i++ {
|
||||
arg := args[i]
|
||||
if arg == "-c" && i+1 < len(args) {
|
||||
formatted = append(formatted, arg, quoteDebugCommandArg(redactGitConfig(args[i+1])))
|
||||
i++
|
||||
continue
|
||||
}
|
||||
formatted = append(formatted, quoteDebugCommandArg(redactURLCredentials(arg)))
|
||||
}
|
||||
return strings.Join(formatted, " ")
|
||||
}
|
||||
|
||||
func redactGitConfig(arg string) string {
|
||||
// URL-scoped keys can contain '=' before the header value separator.
|
||||
for i := range arg {
|
||||
if arg[i] == '=' && strings.HasSuffix(strings.ToLower(arg[:i]), ".extraheader") {
|
||||
return redactURLCredentials(arg[:i]) + "=xxxxx"
|
||||
}
|
||||
}
|
||||
return redactURLCredentials(arg)
|
||||
}
|
||||
|
||||
func redactURLCredentials(arg string) string {
|
||||
if !strings.Contains(arg, "://") {
|
||||
return arg
|
||||
}
|
||||
return urlutil.RedactCredentials(arg)
|
||||
}
|
||||
|
||||
// quoteDebugCommandArg preserves argument boundaries and escapes nonprintable
|
||||
// characters for display. The output is not intended for shell re-execution.
|
||||
func quoteDebugCommandArg(arg string) string {
|
||||
if arg == "" || !utf8.ValidString(arg) || strings.ContainsFunc(arg, func(r rune) bool {
|
||||
return !strconv.IsPrint(r) || strings.ContainsRune(" \"'\\", r)
|
||||
}) {
|
||||
return strconv.Quote(arg)
|
||||
}
|
||||
return arg
|
||||
}
|
||||
|
||||
func getGitSSHCommand(knownHosts string) string {
|
||||
gitSSHCommand := "ssh -F " + os.DevNull
|
||||
if knownHosts != "" {
|
||||
|
||||
+4
-3
@@ -6,6 +6,7 @@ import (
|
||||
"maps"
|
||||
"slices"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
@@ -217,13 +218,13 @@ func newWriter(pw *progressWriter) *progressWriter {
|
||||
}
|
||||
|
||||
type progressWriter struct {
|
||||
done bool
|
||||
done atomic.Bool // set by Close, may race with writes from other goroutines
|
||||
reader *progressReader
|
||||
meta map[string]any
|
||||
}
|
||||
|
||||
func (pw *progressWriter) Write(id string, v any) error {
|
||||
if pw.done {
|
||||
if pw.done.Load() {
|
||||
return errors.Errorf("writing %s to closed progress writer", id)
|
||||
}
|
||||
return pw.writeRawProgress(&Progress{
|
||||
@@ -251,7 +252,7 @@ func (pw *progressWriter) Close() error {
|
||||
delete(pw.reader.writers, pw)
|
||||
pw.reader.mu.Unlock()
|
||||
pw.reader.cond.Broadcast()
|
||||
pw.done = true
|
||||
pw.done.Store(true)
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
+26
-11
@@ -45,12 +45,13 @@ type Puller struct {
|
||||
var _ content.Provider = &provider{}
|
||||
|
||||
type PulledManifests struct {
|
||||
Ref string
|
||||
MainManifestDesc ocispecs.Descriptor
|
||||
ConfigDesc ocispecs.Descriptor
|
||||
Nonlayers []ocispecs.Descriptor
|
||||
Descriptors []ocispecs.Descriptor
|
||||
Provider func(session.Group) content.Provider
|
||||
Ref string
|
||||
RootManifestDesc ocispecs.Descriptor
|
||||
ImageManifestDesc ocispecs.Descriptor // Platform-specific manifest selected from RootManifestDesc.
|
||||
ConfigDesc ocispecs.Descriptor
|
||||
Nonlayers []ocispecs.Descriptor
|
||||
Descriptors []ocispecs.Descriptor
|
||||
Provider func(session.Group) content.Provider
|
||||
}
|
||||
|
||||
func (p *Puller) resolve(ctx context.Context, resolver remotes.Resolver) error {
|
||||
@@ -144,6 +145,18 @@ func (p *Puller) PullManifests(ctx context.Context, getResolver SessionResolver)
|
||||
// Limit manifests pulled to the best match in an index
|
||||
childrenHandler = images.LimitManifests(childrenHandler, platform, 1)
|
||||
|
||||
var manifestDesc ocispecs.Descriptor
|
||||
var manifestOnce sync.Once
|
||||
manifestHandler := images.HandlerFunc(func(ctx context.Context, desc ocispecs.Descriptor) ([]ocispecs.Descriptor, error) {
|
||||
if images.IsManifestType(desc.MediaType) {
|
||||
// Each index yields one child, so the first manifest is the
|
||||
// selected image, including through nested indexes. Do not let
|
||||
// malformed config or layer descriptors replace it later.
|
||||
manifestOnce.Do(func() { manifestDesc = desc })
|
||||
}
|
||||
return nil, nil
|
||||
})
|
||||
|
||||
dslHandler, err := docker.AppendDistributionSourceLabel(p.ContentStore, p.ref)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -152,6 +165,7 @@ func (p *Puller) PullManifests(ctx context.Context, getResolver SessionResolver)
|
||||
filterLayerBlobs(metadata, &mu),
|
||||
retryhandler.New(limited.FetchHandler(p.ContentStore, fetcher, p.ref), logs.LoggerFromContext(ctx)),
|
||||
childrenHandler,
|
||||
manifestHandler,
|
||||
dslHandler,
|
||||
)
|
||||
|
||||
@@ -174,11 +188,12 @@ func (p *Puller) PullManifests(ctx context.Context, getResolver SessionResolver)
|
||||
}
|
||||
|
||||
return &PulledManifests{
|
||||
Ref: p.ref,
|
||||
MainManifestDesc: p.desc,
|
||||
ConfigDesc: p.configDesc,
|
||||
Nonlayers: p.nonlayers,
|
||||
Descriptors: p.layers,
|
||||
Ref: p.ref,
|
||||
RootManifestDesc: p.desc,
|
||||
ImageManifestDesc: manifestDesc,
|
||||
ConfigDesc: p.configDesc,
|
||||
Nonlayers: p.nonlayers,
|
||||
Descriptors: p.layers,
|
||||
Provider: func(g session.Group) content.Provider {
|
||||
return &provider{puller: p, resolver: getResolver(g)}
|
||||
},
|
||||
|
||||
+7
@@ -147,6 +147,13 @@ func (a *dockerAuthorizer) Authorize(ctx context.Context, req *http.Request) err
|
||||
|
||||
authHeader, err := ah.authorize(ctx, a.sm, a.session)
|
||||
if err != nil {
|
||||
// containerd attaches a streaming request body (the pipe of a blob
|
||||
// upload) before authorizing, and does not close it if authorization
|
||||
// fails. Nothing would then unblock the writer feeding the pipe, and
|
||||
// the registry concurrency slot held by it would never be released.
|
||||
if req.Body != nil {
|
||||
req.Body.Close()
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
|
||||
+37
-3
@@ -5,6 +5,7 @@ import (
|
||||
stderrors "errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/rand/v2"
|
||||
"net"
|
||||
"syscall"
|
||||
"time"
|
||||
@@ -19,6 +20,27 @@ import (
|
||||
// variable so that code which embeds BuildKit can override the default value.
|
||||
var MaxRetryBackoff = 8 * time.Second
|
||||
|
||||
// MaxRetryAfter is the longest server-requested delay accepted for a retry.
|
||||
// It is independent of the exponential backoff budget. Code which embeds
|
||||
// BuildKit can override the default value.
|
||||
var MaxRetryAfter = 30 * time.Second
|
||||
|
||||
// RetryAfterError marks a failure as retryable after at least Delay.
|
||||
// A zero Delay uses the normal backoff. Delays above MaxRetryAfter
|
||||
// stop retries rather than retrying before the requested delay has elapsed.
|
||||
type RetryAfterError struct {
|
||||
Err error
|
||||
Delay time.Duration
|
||||
}
|
||||
|
||||
func (e *RetryAfterError) Error() string {
|
||||
return e.Err.Error()
|
||||
}
|
||||
|
||||
func (e *RetryAfterError) Unwrap() error {
|
||||
return e.Err
|
||||
}
|
||||
|
||||
func New(f images.HandlerFunc, logger func([]byte)) images.HandlerFunc {
|
||||
return func(ctx context.Context, desc ocispecs.Descriptor) ([]ocispecs.Descriptor, error) {
|
||||
descs, err := WithRetry(ctx, logger, func(ctx context.Context) ([]ocispecs.Descriptor, error) {
|
||||
@@ -33,7 +55,7 @@ func New(f images.HandlerFunc, logger func([]byte)) images.HandlerFunc {
|
||||
|
||||
type retryContextKey struct{}
|
||||
|
||||
// WithRetry retries transient failures with exponential backoff. Nested calls
|
||||
// WithRetry retries transient failures with jittered exponential backoff. Nested calls
|
||||
// using the supplied context leave retries to the outer call, including New.
|
||||
// f must be safe to retry and preserve retryable errors when wrapping them.
|
||||
func WithRetry[T any](ctx context.Context, logger func([]byte), f func(context.Context) (T, error)) (T, error) {
|
||||
@@ -66,13 +88,21 @@ func WithRetryIf[T any](ctx context.Context, logger func([]byte), f func(context
|
||||
if backoff >= MaxRetryBackoff {
|
||||
return v, err
|
||||
}
|
||||
// Keep the retry budget independent of jitter, including the first retry.
|
||||
delay := backoff/2 + rand.N(backoff/2) //nolint:gosec // Jitter spreads load; it does not need cryptographic randomness.
|
||||
if retryAfter, ok := stderrors.AsType[*RetryAfterError](err); ok {
|
||||
if retryAfter.Delay > MaxRetryAfter {
|
||||
return v, err
|
||||
}
|
||||
delay = max(delay, retryAfter.Delay)
|
||||
}
|
||||
if logger != nil {
|
||||
logger(fmt.Appendf(nil, "retrying in %v\n", backoff))
|
||||
logger(fmt.Appendf(nil, "retrying in %v\n", delay))
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return v, contextFailure(ctx)
|
||||
case <-time.After(backoff):
|
||||
case <-time.After(delay):
|
||||
}
|
||||
backoff *= 2
|
||||
}
|
||||
@@ -88,6 +118,10 @@ func contextFailure(ctx context.Context) error {
|
||||
}
|
||||
|
||||
func retryError(err error) bool {
|
||||
if _, ok := stderrors.AsType[*RetryAfterError](err); ok {
|
||||
return true
|
||||
}
|
||||
|
||||
// Retry on 5xx errors
|
||||
var errUnexpectedStatus remoteserrors.ErrUnexpectedStatus
|
||||
if errors.As(err, &errUnexpectedStatus) &&
|
||||
|
||||
+1
-1
@@ -743,7 +743,7 @@ func ID(root string) (string, error) {
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
id := identity.NewID()
|
||||
err := os.WriteFile(f, []byte(id), 0400)
|
||||
err := os.WriteFile(f, []byte(id), 0o600)
|
||||
return id, err
|
||||
}
|
||||
return "", err
|
||||
|
||||
+31
-37
@@ -12,37 +12,39 @@ type mkdirOptions struct {
|
||||
onlyNew bool
|
||||
}
|
||||
|
||||
// WithOnlyNew is an option for MkdirAllAndChown that will only change ownership and permissions
|
||||
// on newly created directories. If the directory already exists, it will not be modified
|
||||
// WithOnlyNew configures [MkdirAllAndChown] and [MkdirAndChown] to leave
|
||||
// the requested directory unchanged if it already exists. Newly created
|
||||
// directories are still assigned the requested ownership and permissions.
|
||||
func WithOnlyNew(o *mkdirOptions) {
|
||||
o.onlyNew = true
|
||||
}
|
||||
|
||||
// MkdirAllAndChown creates a directory (include any along the path) and then modifies
|
||||
// ownership to the requested uid/gid. By default, if the directory already exists, this
|
||||
// function will still change ownership and permissions. If WithOnlyNew is passed as an
|
||||
// option, then only the newly created directories will have ownership and permissions changed.
|
||||
// MkdirAllAndChown creates a directory named path, along with any necessary
|
||||
// parents, and applies the requested ownership and permission bits to all
|
||||
// directories it creates.
|
||||
//
|
||||
// If path already exists as a directory, its ownership and permission bits
|
||||
// are updated unless [WithOnlyNew] is specified. Existing parent directories
|
||||
// are not modified.
|
||||
//
|
||||
// MkdirAllAndChown is similar to [os.MkdirAll], but also applies ownership and
|
||||
// reapplies the requested permission bits after creation, so the resulting
|
||||
// permissions are not affected by the process umask.
|
||||
func MkdirAllAndChown(path string, mode os.FileMode, uid, gid int, opts ...MkdirOpt) error {
|
||||
var options mkdirOptions
|
||||
for _, opt := range opts {
|
||||
opt(&options)
|
||||
}
|
||||
|
||||
return mkdirAs(path, mode, uid, gid, true, options.onlyNew)
|
||||
return mkdirAs(path, mode, uid, gid, true, opts...)
|
||||
}
|
||||
|
||||
// MkdirAndChown creates a directory and then modifies ownership to the requested uid/gid.
|
||||
// By default, if the directory already exists, this function still changes ownership and permissions.
|
||||
// If WithOnlyNew is passed as an option, then only the newly created directory will have ownership
|
||||
// and permissions changed.
|
||||
// Note that unlike os.Mkdir(), this function does not return IsExist error
|
||||
// in case path already exists.
|
||||
// MkdirAndChown creates a directory named path and applies the requested
|
||||
// ownership and permission bits.
|
||||
//
|
||||
// If path already exists as a directory, its ownership and permission bits
|
||||
// are updated unless [WithOnlyNew] is specified.
|
||||
//
|
||||
// MkdirAndChown is similar to [os.Mkdir], but returns nil if path already
|
||||
// exists as a directory. The resulting permission bits are not affected by
|
||||
// the process umask.
|
||||
func MkdirAndChown(path string, mode os.FileMode, uid, gid int, opts ...MkdirOpt) error {
|
||||
var options mkdirOptions
|
||||
for _, opt := range opts {
|
||||
opt(&options)
|
||||
}
|
||||
return mkdirAs(path, mode, uid, gid, false, options.onlyNew)
|
||||
return mkdirAs(path, mode, uid, gid, false, opts...)
|
||||
}
|
||||
|
||||
// getRootUIDGID retrieves the remapped root uid/gid pair from the set of maps.
|
||||
@@ -106,22 +108,14 @@ func (i IdentityMapping) RootPair() (int, int) {
|
||||
return uid, gid
|
||||
}
|
||||
|
||||
// ToHost returns the host UID and GID for the container uid, gid.
|
||||
// Remapping is only performed if the ids aren't already the remapped root ids
|
||||
// ToHost returns the host UID and GID for the container uid and gid.
|
||||
// An empty mapping is treated as identity.
|
||||
func (i IdentityMapping) ToHost(uid, gid int) (int, int, error) {
|
||||
var err error
|
||||
ruid, rgid := i.RootPair()
|
||||
|
||||
if uid != ruid {
|
||||
ruid, err = toHost(uid, i.UIDMaps)
|
||||
if err != nil {
|
||||
return ruid, rgid, err
|
||||
}
|
||||
}
|
||||
|
||||
if gid != rgid {
|
||||
rgid, err = toHost(gid, i.GIDMaps)
|
||||
ruid, err := toHost(uid, i.UIDMaps)
|
||||
if err != nil {
|
||||
return -1, -1, err
|
||||
}
|
||||
rgid, err := toHost(gid, i.GIDMaps)
|
||||
return ruid, rgid, err
|
||||
}
|
||||
|
||||
|
||||
+7
-2
@@ -10,7 +10,12 @@ import (
|
||||
"syscall"
|
||||
)
|
||||
|
||||
func mkdirAs(path string, mode os.FileMode, uid, gid int, mkAll, onlyNew bool) error {
|
||||
func mkdirAs(path string, mode os.FileMode, uid, gid int, mkAll bool, opts ...MkdirOpt) error {
|
||||
var options mkdirOptions
|
||||
for _, opt := range opts {
|
||||
opt(&options)
|
||||
}
|
||||
|
||||
path, err := filepath.Abs(path)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -21,7 +26,7 @@ func mkdirAs(path string, mode os.FileMode, uid, gid int, mkAll, onlyNew bool) e
|
||||
if !stat.IsDir() {
|
||||
return &os.PathError{Op: "mkdir", Path: path, Err: syscall.ENOTDIR}
|
||||
}
|
||||
if onlyNew {
|
||||
if options.onlyNew {
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
+19
-6
@@ -2,12 +2,25 @@ package user
|
||||
|
||||
import (
|
||||
"os"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
// This is currently a wrapper around [os.MkdirAll] since currently
|
||||
// permissions aren't set through this path, the identity isn't utilized.
|
||||
// Ownership is handled elsewhere, but in the future could be support here
|
||||
// too.
|
||||
func mkdirAs(path string, _ os.FileMode, _, _ int, _, _ bool) error {
|
||||
return os.MkdirAll(path, 0)
|
||||
// mkdirAs creates path, optionally creating any missing parent directories.
|
||||
//
|
||||
// On Windows this is currently a thin wrapper around os.Mkdir and
|
||||
// os.MkdirAll. Unlike the Unix implementation, ownership and permission
|
||||
// bits are not applied.
|
||||
func mkdirAs(path string, _ os.FileMode, _, _ int, mkAll bool, _ ...MkdirOpt) error {
|
||||
if mkAll {
|
||||
return os.MkdirAll(path, 0)
|
||||
}
|
||||
stat, err := os.Stat(path)
|
||||
if err == nil {
|
||||
if !stat.IsDir() {
|
||||
return &os.PathError{Op: "mkdir", Path: path, Err: syscall.ENOTDIR}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
return os.Mkdir(path, 0)
|
||||
}
|
||||
+31
-79
@@ -7,6 +7,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
@@ -17,9 +18,9 @@ const (
|
||||
)
|
||||
|
||||
var (
|
||||
// ErrNoPasswdEntries is returned if no matching entries were found in /etc/group.
|
||||
// ErrNoPasswdEntries is returned if no matching entries were found in /etc/passwd.
|
||||
ErrNoPasswdEntries = errors.New("no matching entries in passwd file")
|
||||
// ErrNoGroupEntries is returned if no matching entries were found in /etc/passwd.
|
||||
// ErrNoGroupEntries is returned if no matching entries were found in /etc/group.
|
||||
ErrNoGroupEntries = errors.New("no matching entries in group file")
|
||||
// ErrRange is returned if a UID or GID is outside of the valid range.
|
||||
ErrRange = fmt.Errorf("uids and gids must be in range %d-%d", minID, maxID)
|
||||
@@ -243,9 +244,6 @@ func GetExecUserPath(userSpec string, defaults *ExecUser, passwdPath, groupPath
|
||||
//
|
||||
// See https://github.com/containerd/containerd/commit/de1341c201ffb0effebbf51d00376181968c8779
|
||||
func parseNumeric(val string) (int, bool, error) {
|
||||
if val == "" {
|
||||
return 0, false, nil
|
||||
}
|
||||
id, err := strconv.Atoi(val)
|
||||
if err != nil {
|
||||
if errors.Is(err, strconv.ErrSyntax) {
|
||||
@@ -296,51 +294,40 @@ func GetExecUser(userSpec string, defaults *ExecUser, passwd, group io.Reader) (
|
||||
user := &ExecUser{
|
||||
Uid: defaults.Uid,
|
||||
Gid: defaults.Gid,
|
||||
Sgids: defaults.Sgids,
|
||||
Sgids: append([]int{}, defaults.Sgids...), // Sgids slice *cannot* be nil.
|
||||
Home: defaults.Home,
|
||||
}
|
||||
|
||||
// Sgids slice *cannot* be nil.
|
||||
if user.Sgids == nil {
|
||||
user.Sgids = []int{}
|
||||
}
|
||||
// Allow for userSpec to have either "user", or optionally "user:group" syntax.
|
||||
usr, grp, _ := strings.Cut(userSpec, ":")
|
||||
|
||||
// Allow for userArg to have either "user" syntax, or optionally "user:group" syntax
|
||||
var userArg, groupArg string
|
||||
parseLine([]byte(userSpec), &userArg, &groupArg)
|
||||
|
||||
// Convert userArg and groupArg to be numeric, so we don't have to execute
|
||||
// Convert usr and grp to be numeric, so we don't have to execute
|
||||
// Atoi *twice* for each iteration over lines.
|
||||
uidArg, isUID, err := parseNumeric(userArg)
|
||||
userArg, err := parseUserArg(usr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
gidArg, isGID, err := parseNumeric(groupArg)
|
||||
groupArg, err := parseGroupArg(grp)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Find the matching user.
|
||||
users, err := ParsePasswdFilter(passwd, func(u User) bool {
|
||||
if userArg == "" {
|
||||
if userArg == nil {
|
||||
// Default to current state of the user.
|
||||
return u.Uid == user.Uid
|
||||
}
|
||||
|
||||
if isUID {
|
||||
// If the userArg is a valid numeric value, always treat it as a UID.
|
||||
return uidArg == u.Uid
|
||||
}
|
||||
|
||||
return u.Name == userArg
|
||||
return userArg.matches(u)
|
||||
})
|
||||
|
||||
// If we can't find the user, we have to bail.
|
||||
if err != nil && passwd != nil {
|
||||
if userArg == "" {
|
||||
userArg = strconv.Itoa(user.Uid)
|
||||
name := usr
|
||||
if userArg == nil {
|
||||
name = strconv.Itoa(user.Uid)
|
||||
}
|
||||
return nil, fmt.Errorf("unable to find user %s: %w", userArg, err)
|
||||
return nil, fmt.Errorf("unable to find user %s: %w", name, err)
|
||||
}
|
||||
|
||||
var matchedUserName string
|
||||
@@ -350,57 +337,42 @@ func GetExecUser(userSpec string, defaults *ExecUser, passwd, group io.Reader) (
|
||||
user.Uid = users[0].Uid
|
||||
user.Gid = users[0].Gid
|
||||
user.Home = users[0].Home
|
||||
} else if userArg != "" {
|
||||
} else if userArg != nil {
|
||||
// If we can't find a user with the given username, the only other valid
|
||||
// option is if it's a numeric username with no associated entry in passwd.
|
||||
|
||||
if !isUID {
|
||||
// Not numeric.
|
||||
return nil, fmt.Errorf("unable to find user %s: %w", userArg, ErrNoPasswdEntries)
|
||||
if !userArg.isNumeric {
|
||||
return nil, fmt.Errorf("unable to find user %s: %w", userArg.name, ErrNoPasswdEntries)
|
||||
}
|
||||
user.Uid = uidArg
|
||||
user.Uid = userArg.uid
|
||||
}
|
||||
|
||||
// On to the groups. If we matched a username, we need to do this because of
|
||||
// the supplementary group IDs.
|
||||
if groupArg != "" || matchedUserName != "" {
|
||||
if groupArg != nil || matchedUserName != "" {
|
||||
groups, err := ParseGroupFilter(group, func(g Group) bool {
|
||||
// If the group argument isn't explicit, we'll just search for it.
|
||||
if groupArg == "" {
|
||||
if groupArg == nil {
|
||||
// Check if user is a member of this group.
|
||||
for _, u := range g.List {
|
||||
if u == matchedUserName {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
return slices.Contains(g.List, matchedUserName)
|
||||
}
|
||||
|
||||
if isGID {
|
||||
// If the groupArg is numeric, always treat it as a GID.
|
||||
return gidArg == g.Gid
|
||||
}
|
||||
|
||||
return g.Name == groupArg
|
||||
return groupArg.matches(g)
|
||||
})
|
||||
if err != nil && group != nil {
|
||||
return nil, fmt.Errorf("unable to find groups for spec %v: %w", matchedUserName, err)
|
||||
}
|
||||
|
||||
// Only start modifying user.Gid if it is in explicit form.
|
||||
if groupArg != "" {
|
||||
if groupArg != nil {
|
||||
if len(groups) > 0 {
|
||||
// First match wins, even if there's more than one matching entry.
|
||||
user.Gid = groups[0].Gid
|
||||
} else {
|
||||
// If we can't find a group with the given name, the only other valid
|
||||
// option is if it's a numeric group name with no associated entry in group.
|
||||
|
||||
if !isGID {
|
||||
// Not numeric.
|
||||
return nil, fmt.Errorf("unable to find group %s: %w", groupArg, ErrNoGroupEntries)
|
||||
if !groupArg.isNumeric {
|
||||
return nil, fmt.Errorf("unable to find group %s: %w", groupArg.name, ErrNoGroupEntries)
|
||||
}
|
||||
user.Gid = gidArg
|
||||
user.Gid = groupArg.gid
|
||||
}
|
||||
} else if len(groups) > 0 {
|
||||
// Supplementary group ids only make sense if in the implicit form.
|
||||
@@ -414,39 +386,19 @@ func GetExecUser(userSpec string, defaults *ExecUser, passwd, group io.Reader) (
|
||||
return user, nil
|
||||
}
|
||||
|
||||
// groupArg is a parsed group argument for [GetAdditionalGroups].
|
||||
type groupArg struct {
|
||||
name string
|
||||
gid int
|
||||
isNumeric bool
|
||||
}
|
||||
|
||||
// matches reports whether group g satisfies the argument. Numeric arguments
|
||||
// are matched by GID only, others by name.
|
||||
func (ag groupArg) matches(g Group) bool {
|
||||
if ag.isNumeric {
|
||||
return g.Gid == ag.gid
|
||||
}
|
||||
return g.Name == ag.name
|
||||
}
|
||||
|
||||
// GetAdditionalGroups looks up a list of groups by name or group id
|
||||
// against the given /etc/group formatted data. If a group name cannot
|
||||
// be found, an error will be returned. If a group id cannot be found,
|
||||
// or the given group data is nil, the id will be returned as-is
|
||||
// provided it is in the legal range.
|
||||
func GetAdditionalGroups(additionalGroups []string, group io.Reader) ([]int, error) {
|
||||
addtlGroups := make([]groupArg, len(additionalGroups))
|
||||
for i, ag := range additionalGroups {
|
||||
gid, ok, err := parseNumeric(ag)
|
||||
addtlGroups := make([]groupArg, 0, len(additionalGroups))
|
||||
for _, ag := range additionalGroups {
|
||||
groupArg, err := parseGroupArg(ag)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
addtlGroups[i] = groupArg{
|
||||
name: ag,
|
||||
gid: gid,
|
||||
isNumeric: ok,
|
||||
}
|
||||
addtlGroups = append(addtlGroups, *groupArg)
|
||||
}
|
||||
|
||||
groups := []Group{}
|
||||
|
||||
+62
@@ -62,3 +62,65 @@ func (l *limitedFile) Read(p []byte) (int, error) {
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
// userArg is a parsed user argument for [GetExecUser].
|
||||
type userArg struct {
|
||||
name string
|
||||
uid int
|
||||
isNumeric bool
|
||||
}
|
||||
|
||||
// parseUserArg parses a user argument as either a user name or UID.
|
||||
//
|
||||
// If name is empty, parseUserArg returns "nil, nil" to indicate that no user
|
||||
// argument was specified.
|
||||
func parseUserArg(name string) (*userArg, error) {
|
||||
if name == "" {
|
||||
return nil, nil
|
||||
}
|
||||
uid, isNumeric, err := parseNumeric(name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &userArg{name: name, uid: uid, isNumeric: isNumeric}, nil
|
||||
}
|
||||
|
||||
// matches reports whether user u satisfies the argument. Numeric arguments
|
||||
// are matched by UID only, others by name.
|
||||
func (ua *userArg) matches(u User) bool {
|
||||
if ua.isNumeric {
|
||||
return u.Uid == ua.uid
|
||||
}
|
||||
return u.Name == ua.name
|
||||
}
|
||||
|
||||
// groupArg is a parsed group argument for [GetAdditionalGroups] or [GetExecUser].
|
||||
type groupArg struct {
|
||||
name string
|
||||
gid int
|
||||
isNumeric bool
|
||||
}
|
||||
|
||||
// parseGroupArg parses a group argument as either a group name or GID.
|
||||
//
|
||||
// If name is empty, parseGroupArg returns "nil, nil" to indicate that no group
|
||||
// argument was specified.
|
||||
func parseGroupArg(name string) (*groupArg, error) {
|
||||
if name == "" {
|
||||
return nil, nil
|
||||
}
|
||||
gid, isNumeric, err := parseNumeric(name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &groupArg{name: name, gid: gid, isNumeric: isNumeric}, nil
|
||||
}
|
||||
|
||||
// matches reports whether group g satisfies the argument. Numeric arguments
|
||||
// are matched by GID only, others by name.
|
||||
func (ag groupArg) matches(g Group) bool {
|
||||
if ag.isNumeric {
|
||||
return g.Gid == ag.gid
|
||||
}
|
||||
return g.Name == ag.name
|
||||
}
|
||||
Vendored
+7
-7
@@ -626,10 +626,9 @@ github.com/containerd/platforms
|
||||
## explicit; go 1.22
|
||||
github.com/containerd/plugin
|
||||
github.com/containerd/plugin/registry
|
||||
# github.com/containerd/stargz-snapshotter/estargz v0.18.2
|
||||
## explicit; go 1.24.0
|
||||
# github.com/containerd/stargz-snapshotter/estargz v0.19.1
|
||||
## explicit; go 1.25.0
|
||||
github.com/containerd/stargz-snapshotter/estargz
|
||||
github.com/containerd/stargz-snapshotter/estargz/errorutil
|
||||
# github.com/containerd/ttrpc v1.2.10
|
||||
## explicit; go 1.25.0
|
||||
github.com/containerd/ttrpc
|
||||
@@ -1078,7 +1077,7 @@ github.com/mitchellh/copystructure
|
||||
# github.com/mitchellh/reflectwalk v1.0.2
|
||||
## explicit
|
||||
github.com/mitchellh/reflectwalk
|
||||
# github.com/moby/buildkit v0.34.2
|
||||
# github.com/moby/buildkit v0.35.0-rc1
|
||||
## explicit; go 1.26.8
|
||||
github.com/moby/buildkit/api/services/control
|
||||
github.com/moby/buildkit/api/types
|
||||
@@ -1088,10 +1087,10 @@ github.com/moby/buildkit/cache/contenthash
|
||||
github.com/moby/buildkit/cache/metadata
|
||||
github.com/moby/buildkit/cache/remotecache
|
||||
github.com/moby/buildkit/cache/remotecache/gha
|
||||
github.com/moby/buildkit/cache/remotecache/gha/ghatypes
|
||||
github.com/moby/buildkit/cache/remotecache/inline
|
||||
github.com/moby/buildkit/cache/remotecache/local
|
||||
github.com/moby/buildkit/cache/remotecache/registry
|
||||
github.com/moby/buildkit/cache/remotecache/signing
|
||||
github.com/moby/buildkit/cache/remotecache/v1
|
||||
github.com/moby/buildkit/cache/remotecache/v1/types
|
||||
github.com/moby/buildkit/cache/util
|
||||
@@ -1204,6 +1203,7 @@ github.com/moby/buildkit/util/bkmaps
|
||||
github.com/moby/buildkit/util/bkslices
|
||||
github.com/moby/buildkit/util/cachedigest
|
||||
github.com/moby/buildkit/util/compression
|
||||
github.com/moby/buildkit/util/compression/go126flate
|
||||
github.com/moby/buildkit/util/cond
|
||||
github.com/moby/buildkit/util/contentutil
|
||||
github.com/moby/buildkit/util/converter
|
||||
@@ -1456,8 +1456,8 @@ github.com/moby/sys/signal
|
||||
# github.com/moby/sys/symlink v0.3.0
|
||||
## explicit; go 1.17
|
||||
github.com/moby/sys/symlink
|
||||
# github.com/moby/sys/user v0.4.1
|
||||
## explicit; go 1.18
|
||||
# github.com/moby/sys/user v0.5.0
|
||||
## explicit; go 1.24
|
||||
github.com/moby/sys/user
|
||||
# github.com/moby/sys/userns v0.2.1
|
||||
## explicit; go 1.21
|
||||
|
||||
Reference in new issue
Block a user