From 72e79221cb1074bacec68fe6515c94c2973c5db3 Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Wed, 4 Nov 2026 14:25:06 -0800 Subject: [PATCH] vendor: update buildkit v0.35.0-rc1 Signed-off-by: Tonis Tiigi --- go.mod | 8 +- go.sum | 24 +- vendor/github.com/aws/smithy-go/CHANGELOG.md | 9 + .../aws/smithy-go/go_module_metadata.go | 2 +- .../http/protocol/internal/json/parser.go | 4 +- .../stargz-snapshotter/estargz/build.go | 54 +- .../estargz/errorutil/errors.go | 40 - .../stargz-snapshotter/estargz/estargz.go | 20 +- .../stargz-snapshotter/estargz/gzip.go | 50 +- .../stargz-snapshotter/estargz/testutil.go | 88 ++- .../stargz-snapshotter/estargz/types.go | 2 +- .../github.com/moby/buildkit/cache/manager.go | 14 +- .../moby/buildkit/cache/metadata/metadata.go | 38 + vendor/github.com/moby/buildkit/cache/refs.go | 109 ++- .../buildkit/cache/remotecache/gha/gha.go | 138 +--- .../cache/remotecache/gha/ghatypes/config.go | 23 - .../cache/remotecache/signing/config.go | 27 + .../cache/remotecache/signing/sign.go | 48 ++ .../cache/remotecache/signing/verify.go | 114 +++ .../buildkit/cache/remotecache/v1/utils.go | 57 +- .../moby/buildkit/client/llb/fileop.go | 5 +- .../moby/buildkit/client/llb/meta.go | 15 + .../moby/buildkit/client/llb/source.go | 12 + .../moby/buildkit/client/llb/state.go | 6 + .../github.com/moby/buildkit/client/solve.go | 121 ++- .../github.com/moby/buildkit/client/status.go | 136 ++-- .../buildkit/cmd/buildkitd/config/config.go | 4 +- .../moby/buildkit/control/control.go | 5 +- .../executor/containerdexecutor/executor.go | 6 +- .../moby/buildkit/executor/oci/resolvconf.go | 14 +- .../executor/runcexecutor/executor.go | 4 +- .../executor/runcexecutor/executor_linux.go | 21 +- .../frontend/dockerfile/builder/build.go | 2 +- .../dockerfile/builder/resolvecache.go | 8 +- .../dockerfile/dockerfile2llb/convert.go | 121 ++- .../dockerfile/dockerfile2llb/convert_copy.go | 3 + .../dockerfile2llb/convert_expose.go | 2 +- .../dockerfile/dockerfile2llb/epoch.go | 11 +- .../dockerfile/dockerfile2llb/validations.go | 1 + .../dockerfile/instructions/commands.go | 9 +- .../frontend/dockerfile/instructions/parse.go | 119 +++ .../dockerfile/instructions/parse_reset.go | 7 + .../dockerfile/instructions/parse_unset.go | 7 + .../frontend/dockerfile/parser/parser.go | 22 + .../frontend/dockerfile/version/version.go | 2 +- .../moby/buildkit/frontend/dockerui/config.go | 8 + .../buildkit/frontend/dockerui/context.go | 14 +- .../frontend/dockerui/namedcontext.go | 4 +- .../frontend/gateway/forwarder/forward.go | 47 +- .../moby/buildkit/frontend/gateway/gateway.go | 119 +-- .../frontend/gateway/grpcclient/client.go | 64 +- .../snapshot/imagerefchecker/checker.go | 8 + .../buildkit/snapshot/localmounter_linux.go | 2 + .../moby/buildkit/solver/cachemanager.go | 58 +- .../moby/buildkit/solver/cacheopts.go | 2 +- .../github.com/moby/buildkit/solver/edge.go | 25 +- .../moby/buildkit/solver/exporter.go | 5 +- .../github.com/moby/buildkit/solver/index.go | 3 + .../buildkit/solver/internal/pipe/pipe.go | 2 +- .../github.com/moby/buildkit/solver/jobs.go | 125 ++- .../solver/llbsolver/cdidevices/manager.go | 38 +- .../buildkit/solver/llbsolver/errdefs/exec.go | 35 +- .../moby/buildkit/solver/llbsolver/history.go | 6 +- .../solver/llbsolver/ops/exec_binfmt.go | 12 +- .../solver/llbsolver/proc/provenance.go | 11 +- .../buildkit/solver/llbsolver/provenance.go | 21 +- .../moby/buildkit/solver/llbsolver/result.go | 25 +- .../moby/buildkit/solver/llbsolver/vertex.go | 33 +- .../moby/buildkit/solver/pb/attr.go | 1 + .../moby/buildkit/solver/scheduler.go | 10 +- .../buildkit/source/containerimage/pull.go | 35 +- .../moby/buildkit/source/git/bundle.go | 8 +- .../moby/buildkit/source/git/identifier.go | 1 + .../moby/buildkit/source/git/source.go | 8 +- .../moby/buildkit/source/http/source.go | 46 +- .../util/compression/go126flate/LICENSE | 27 + .../util/compression/go126flate/deflate.go | 747 ++++++++++++++++++ .../compression/go126flate/deflatefast.go | 309 ++++++++ .../util/compression/go126flate/doc.go | 20 + .../go126flate/huffman_bit_writer.go | 695 ++++++++++++++++ .../compression/go126flate/huffman_code.go | 347 ++++++++ .../util/compression/go126flate/token.go | 99 +++ .../buildkit/util/compression/go126gzip.go | 92 +++ .../moby/buildkit/util/compression/gzip.go | 8 +- .../moby/buildkit/util/gitutil/git_cli.go | 85 +- .../moby/buildkit/util/progress/progress.go | 7 +- .../moby/buildkit/util/pull/pull.go | 37 +- .../moby/buildkit/util/resolver/authorizer.go | 7 + .../util/resolver/retryhandler/retry.go | 40 +- .../moby/buildkit/worker/base/worker.go | 2 +- vendor/github.com/moby/sys/user/idtools.go | 68 +- .../github.com/moby/sys/user/idtools_unix.go | 9 +- .../moby/sys/user/idtools_windows.go | 25 +- vendor/github.com/moby/sys/user/user.go | 110 +-- vendor/github.com/moby/sys/user/user_utils.go | 62 ++ vendor/modules.txt | 16 +- 96 files changed, 4252 insertions(+), 868 deletions(-) delete mode 100644 vendor/github.com/containerd/stargz-snapshotter/estargz/errorutil/errors.go delete mode 100644 vendor/github.com/moby/buildkit/cache/remotecache/gha/ghatypes/config.go create mode 100644 vendor/github.com/moby/buildkit/cache/remotecache/signing/config.go create mode 100644 vendor/github.com/moby/buildkit/cache/remotecache/signing/sign.go create mode 100644 vendor/github.com/moby/buildkit/cache/remotecache/signing/verify.go create mode 100644 vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/parse_reset.go create mode 100644 vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/parse_unset.go create mode 100644 vendor/github.com/moby/buildkit/util/compression/go126flate/LICENSE create mode 100644 vendor/github.com/moby/buildkit/util/compression/go126flate/deflate.go create mode 100644 vendor/github.com/moby/buildkit/util/compression/go126flate/deflatefast.go create mode 100644 vendor/github.com/moby/buildkit/util/compression/go126flate/doc.go create mode 100644 vendor/github.com/moby/buildkit/util/compression/go126flate/huffman_bit_writer.go create mode 100644 vendor/github.com/moby/buildkit/util/compression/go126flate/huffman_code.go create mode 100644 vendor/github.com/moby/buildkit/util/compression/go126flate/token.go create mode 100644 vendor/github.com/moby/buildkit/util/compression/go126gzip.go diff --git a/go.mod b/go.mod index 92bcba1947..f42fefdf6a 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/aws/aws-sdk-go-v2/credentials v1.20.13 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.5 github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs v1.91.0 - github.com/aws/smithy-go v1.28.7 + github.com/aws/smithy-go v1.28.8 github.com/cloudflare/cfssl v1.6.5 github.com/containerd/cgroups/v3 v3.1.3 github.com/containerd/containerd/api v1.12.0 @@ -64,7 +64,7 @@ require ( github.com/miekg/dns v1.1.73 github.com/mistifyio/go-zfs/v4 v4.0.0 github.com/mitchellh/copystructure v1.2.0 - github.com/moby/buildkit v0.34.2 + github.com/moby/buildkit v0.35.0-rc1 github.com/moby/docker-image-spec v1.3.1 github.com/moby/extensions v0.0.0-20261020100352-d2819d88bcbe github.com/moby/go-archive v0.3.3 @@ -84,7 +84,7 @@ require ( github.com/moby/sys/reexec v0.1.0 github.com/moby/sys/signal v0.7.1 github.com/moby/sys/symlink v0.3.0 - github.com/moby/sys/user v0.4.1 + github.com/moby/sys/user v0.5.0 github.com/moby/sys/userns v0.2.1 github.com/moby/term v0.5.2 github.com/opencontainers/cgroups v0.1.0 @@ -165,7 +165,7 @@ require ( github.com/containerd/go-runc v1.2.1 // indirect github.com/containerd/nydus-snapshotter v0.15.15 // indirect github.com/containerd/otelttrpc v0.1.0 // indirect - github.com/containerd/stargz-snapshotter/estargz v0.18.2 // indirect + github.com/containerd/stargz-snapshotter/estargz v0.19.1 // indirect github.com/containernetworking/cni v1.3.1 // indirect github.com/containernetworking/plugins v1.9.1 // indirect github.com/coreos/go-semver v0.3.1 // indirect diff --git a/go.sum b/go.sum index e3a7a70448..d61aee9838 100644 --- a/go.sum +++ b/go.sum @@ -108,8 +108,8 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.1 h1:w8yui2ZqdHmGwVBDGvazoIvK github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.1/go.mod h1:gJSdIY6YUzDvqQLcMtPVZgBRHrFwzHryvpE8Igzbxps= github.com/aws/aws-sdk-go-v2/service/sts v1.51.6 h1:BVjKrtRUtSYKztBCCfE1oAB+d3ViKcW2cKZfserOqMQ= github.com/aws/aws-sdk-go-v2/service/sts v1.51.6/go.mod h1:eSukFLeLkPMm+1OxDR+F0joauZdyBkF47Ow7MqjZiJw= -github.com/aws/smithy-go v1.28.7 h1:h/n1SHrwVtaDYLz8mR1/Kq4Xh4RaFbFW7bOYckg9mfE= -github.com/aws/smithy-go v1.28.7/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/aws/smithy-go v1.28.8 h1:aEmdAq7gVCoGCnvf1NnELlu3I6D/nhqrC321FyFiQkQ= +github.com/aws/smithy-go v1.28.8/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bits-and-blooms/bitset v1.25.0 h1:0Ro0qF4abCkM6SqWPVj29sFhAbMPAZpaDD7xhJ10beM= @@ -174,8 +174,8 @@ github.com/containerd/platforms v1.0.0-rc.5 h1:vXd569rDrz8LeMXzAnBsy6LADV5YtsD8o github.com/containerd/platforms v1.0.0-rc.5/go.mod h1:lKlMXyLybmBedS/JJm11uDofzI8L2v0J2ZbYvNsbq1A= github.com/containerd/plugin v1.1.0 h1:O+7lczNJVMy8rz0YNx3xGB8tTf5qY4i5abF041Ew19U= github.com/containerd/plugin v1.1.0/go.mod h1:qBTum+A8lJ6lO44A19Eo7y1OlcLj4OWFH1DA/vnHmcc= -github.com/containerd/stargz-snapshotter/estargz v0.18.2 h1:yXkZFYIzz3eoLwlTUZKz2iQ4MrckBxJjkmD16ynUTrw= -github.com/containerd/stargz-snapshotter/estargz v0.18.2/go.mod h1:XyVU5tcJ3PRpkA9XS2T5us6Eg35yM0214Y+wvrZTBrY= +github.com/containerd/stargz-snapshotter/estargz v0.19.1 h1:PD/uZYqmyyEXZl3fEPM2ieCpilv7Oow9FC2OCBfkDRs= +github.com/containerd/stargz-snapshotter/estargz v0.19.1/go.mod h1:Cr0pxOkuN7Y61+Sik6X6/nPtePW7hsKWdBt+MoHQRr0= github.com/containerd/ttrpc v1.2.10 h1:qQvy3mokhoXON7FXiOOc05O7Up6nrNj6amJF8urNvDc= github.com/containerd/ttrpc v1.2.10/go.mod h1:YrmEQKkMbBA4EM26AzVAIwHGIYgptkL4CJ/yVTaDyBE= github.com/containerd/typeurl/v2 v2.3.0 h1:HZHPhRWo5XMy3QGQoPrUzbW/2ckwjfweHmOwlkIrPAQ= @@ -212,8 +212,8 @@ github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/docker/cli v29.8.2+incompatible h1:2zgdFuoFst2T80oS42vhKGxkd0JRo305eIEKTsxR7pQ= -github.com/docker/cli v29.8.2+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/cli v29.9.3+incompatible h1:SBaMp7ecohadEHMDrUJg22t9I3awqsAVhnJ7GOcWpfE= +github.com/docker/cli v29.9.3+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk= github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker-credential-helpers v0.9.9 h1:BkydjIgZ46JnDbqyM2p2fc63KMw6y+KHL3Em/2AGJ7w= @@ -469,8 +469,8 @@ github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyua github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= -github.com/moby/buildkit v0.34.2 h1:fXgOe79JW5zJNDumRtwTemOklLX66otQokObUXuiGJo= -github.com/moby/buildkit v0.34.2/go.mod h1:HyFnF6kIPzHvqopF1+BKeGY8Ee9MXmfR2n+0Gd4Y69s= +github.com/moby/buildkit v0.35.0-rc1 h1:wgD3pbfSTP03T3Q4IkMRRAHhxbz7177+mavHnwVjUQI= +github.com/moby/buildkit v0.35.0-rc1/go.mod h1:5L8B0JOdp1dO1ntCpfOYT5vR31kQ1nKIor/b4FwsDVQ= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/extensions v0.0.0-20261020100352-d2819d88bcbe h1:wHRo58MA+vgX2sXtL4uT/NcSYPiyJarqHZ4aDS8P0ys= @@ -507,8 +507,8 @@ github.com/moby/sys/signal v0.7.1 h1:PrQxdvxcGijdo6UXXo/lU/TvHUWyPhj7UOpSo8tuvk0 github.com/moby/sys/signal v0.7.1/go.mod h1:Se1VGehYokAkrSQwL4tDzHvETwUZlnY7S5XtQ50mQp8= github.com/moby/sys/symlink v0.3.0 h1:GZX89mEZ9u53f97npBy4Rc3vJKj7JBDj/PN2I22GrNU= github.com/moby/sys/symlink v0.3.0/go.mod h1:3eNdhduHmYPcgsJtZXW1W4XUJdZGBIkttZ8xKqPUJq0= -github.com/moby/sys/user v0.4.1 h1:RgjRlaDKi/Xmyrz4t8lyzXT6v2ooFeO/7xtchmhVWE0= -github.com/moby/sys/user v0.4.1/go.mod h1:E9QsW5WRe1kUAf7kW8hXKwu1uhsZEAdPLYHYSDudF4Y= +github.com/moby/sys/user v0.5.0 h1:5+S4lyoavGyX1DGs68NW8xpx48T9+a7ogSUc0KjZg8Y= +github.com/moby/sys/user v0.5.0/go.mod h1:4mQzP5aTmaLjDLp/dc870oNkik6E9vjRtWYwI05gvow= github.com/moby/sys/userns v0.2.1 h1:4OvdM7BcPkASbuouHsbW3aeMJSFlYDldBRnXVZhaRk8= github.com/moby/sys/userns v0.2.1/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ= @@ -740,8 +740,8 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.47.0 h1:UFpxO go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.47.0/go.mod h1:YBGjxe3lt0jtXoEXxGrGhv/jM7oUBUDVK7zhAvNyEjc= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0 h1:aXZXsZ012wOgVkqaiQv+Z/d8V0xKjmg70F6m4CE94lc= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0/go.mod h1:ziu1gUIJhAaxM3EV1e3Ralk5AAyL1UOYz3lJJ4VSI38= -go.opentelemetry.io/otel/exporters/prometheus v0.68.0 h1:QOf2IftqQwITVRJpnn0M7M9ZCbgWfxz4P7i9C9yc2N4= -go.opentelemetry.io/otel/exporters/prometheus v0.68.0/go.mod h1:bgSvqu2TWGXiz7yr5UTMfObH8oqxJWHTnubQ3ef9BO4= +go.opentelemetry.io/otel/exporters/prometheus v0.69.0 h1:LQVBTGnHvzrfGJpDoCSVTY7eaLZArvaTSq5l84Skxew= +go.opentelemetry.io/otel/exporters/prometheus v0.69.0/go.mod h1:moif0nlgj/M16NRT8lDZVt8MC52+EOAJfR8EKRJ8LyU= go.opentelemetry.io/otel/log v1.47.0 h1:cOTS1CcLbSQeZKanGJ+0JpF/+t4PELi3O3bbl2lqCcI= go.opentelemetry.io/otel/log v1.47.0/go.mod h1:9byitSQ5pLC6PpqwGXjqdMKya6ZTswHRZh2vvXT33nw= go.opentelemetry.io/otel/metric v1.47.0 h1:4PptaldXx3Eat1XjMZ68pPJEs5wrhlemctZE9a3UdWY= diff --git a/vendor/github.com/aws/smithy-go/CHANGELOG.md b/vendor/github.com/aws/smithy-go/CHANGELOG.md index 158e05a481..648e978e96 100644 --- a/vendor/github.com/aws/smithy-go/CHANGELOG.md +++ b/vendor/github.com/aws/smithy-go/CHANGELOG.md @@ -1,3 +1,12 @@ +# Release (2026-11-03) + +## General Highlights +* **Dependency Update**: Updated to the latest SDK module versions + +## Module Highlights +* `github.com/aws/smithy-go`: v1.28.8 + * **Bug Fix**: Preserve the exact value of numbers in JSON codec instead of automatically parsing to float64. + # Release (2026-10-27) ## General Highlights diff --git a/vendor/github.com/aws/smithy-go/go_module_metadata.go b/vendor/github.com/aws/smithy-go/go_module_metadata.go index 184e469ea1..f3cfc27034 100644 --- a/vendor/github.com/aws/smithy-go/go_module_metadata.go +++ b/vendor/github.com/aws/smithy-go/go_module_metadata.go @@ -3,4 +3,4 @@ package smithy // goModuleVersion is the tagged release for this module -const goModuleVersion = "1.28.7" +const goModuleVersion = "1.28.8" diff --git a/vendor/github.com/aws/smithy-go/transport/http/protocol/internal/json/parser.go b/vendor/github.com/aws/smithy-go/transport/http/protocol/internal/json/parser.go index 2a4e9d173c..796236d48a 100644 --- a/vendor/github.com/aws/smithy-go/transport/http/protocol/internal/json/parser.go +++ b/vendor/github.com/aws/smithy-go/transport/http/protocol/internal/json/parser.go @@ -1,10 +1,10 @@ package json import ( + "encoding/json" "errors" "fmt" "io" - "strconv" "unsafe" "github.com/aws/smithy-go/internal/serde" @@ -275,7 +275,7 @@ func (p *parser) value(tok []byte) (any, error) { list = append(list, val) } default: - return strconv.ParseFloat(string(tok), 64) + return json.Number(tok), nil } } diff --git a/vendor/github.com/containerd/stargz-snapshotter/estargz/build.go b/vendor/github.com/containerd/stargz-snapshotter/estargz/build.go index a9e1b72ba7..413cfd0c75 100644 --- a/vendor/github.com/containerd/stargz-snapshotter/estargz/build.go +++ b/vendor/github.com/containerd/stargz-snapshotter/estargz/build.go @@ -37,10 +37,8 @@ import ( "sync" "sync/atomic" - "github.com/containerd/stargz-snapshotter/estargz/errorutil" "github.com/klauspost/compress/zstd" digest "github.com/opencontainers/go-digest" - "golang.org/x/sync/errgroup" ) type GzipHelperFunc func(io.Reader) (io.ReadCloser, error) @@ -53,6 +51,7 @@ type options struct { compression Compression ctx context.Context minChunkSize int + parallelism int gzipHelperFunc GzipHelperFunc } @@ -131,6 +130,20 @@ func WithMinChunkSize(minChunkSize int) Option { } } +// WithParallelism option specifies the number of workers used to build the +// layer. The tar is sliced into this many parts that are compressed +// concurrently, so the value also fixes the resulting chunk boundaries: pinning +// it makes builds reproducible across machines regardless of their CPU count. +// Zero (the default) selects runtime.GOMAXPROCS(0); a value of 1 forces a fully +// sequential build. This has no effect together with WithMinChunkSize, which is +// always built with a single worker. +func WithParallelism(workers int) Option { + return func(o *options) error { + o.parallelism = workers + return nil + } +} + // WithGzipHelperFunc option specifies a custom function to decompress gzip-compressed layers. // When a gzip-compressed layer is detected, this function will be used instead of the // Go standard library gzip decompression for better performance. @@ -225,25 +238,32 @@ func Build(tarBlob *io.SectionReader, opt ...Option) (_ *Blob, rErr error) { if err != nil { return nil, err } + // The worker count slices the tar and so fixes the chunk boundaries; an + // explicit value is honored verbatim, which keeps builds reproducible across + // machines. Zero falls back to GOMAXPROCS. + workers := opts.parallelism + if workers <= 0 { + workers = runtime.GOMAXPROCS(0) + } var tarParts [][]*entry if opts.minChunkSize > 0 { // Each entry needs to know the size of the current gzip stream so they // cannot be processed in parallel. tarParts = [][]*entry{entries} } else { - tarParts = divideEntries(entries, runtime.GOMAXPROCS(0)) + tarParts = divideEntries(entries, workers) } writers := make([]*Writer, len(tarParts)) payloads := make([]*os.File, len(tarParts)) - var mu sync.Mutex - var eg errgroup.Group + var wg sync.WaitGroup + errCh := make(chan error, len(tarParts)) // buffered to avoid goroutine leaks for i, parts := range tarParts { - i, parts := i, parts // builds verifiable stargz sub-blobs - eg.Go(func() error { + wg.Go(func() { esgzFile, err := layerFiles.TempFile("", "esgzdata") if err != nil { - return err + errCh <- err + return } sw := NewWriterWithCompressor(esgzFile, opts.compression) sw.ChunkSize = opts.chunkSize @@ -255,18 +275,20 @@ func Build(tarBlob *io.SectionReader, opt ...Option) (_ *Blob, rErr error) { sw.needsOpenGzEntries[f] = struct{}{} } if err := sw.AppendTar(readerFromEntries(parts...)); err != nil { - return err + errCh <- err + return } - mu.Lock() writers[i] = sw payloads[i] = esgzFile - mu.Unlock() - return nil }) } - if err := eg.Wait(); err != nil { - rErr = err - return nil, err + wg.Wait() + close(errCh) + + for err := range errCh { + if err != nil { + return nil, err + } } tocAndFooter, tocDgst, err := closeWithCombine(writers...) if err != nil { @@ -656,7 +678,7 @@ func (tf *tempFiles) cleanupAll() error { } } tf.files = nil - return errorutil.Aggregate(allErr) + return errors.Join(allErr...) } func newCountReadSeeker(r io.ReaderAt) (*countReadSeeker, error) { diff --git a/vendor/github.com/containerd/stargz-snapshotter/estargz/errorutil/errors.go b/vendor/github.com/containerd/stargz-snapshotter/estargz/errorutil/errors.go deleted file mode 100644 index 6de78b02dc..0000000000 --- a/vendor/github.com/containerd/stargz-snapshotter/estargz/errorutil/errors.go +++ /dev/null @@ -1,40 +0,0 @@ -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package errorutil - -import ( - "errors" - "fmt" - "strings" -) - -// Aggregate combines a list of errors into a single new error. -func Aggregate(errs []error) error { - switch len(errs) { - case 0: - return nil - case 1: - return errs[0] - default: - points := make([]string, len(errs)+1) - points[0] = fmt.Sprintf("%d error(s) occurred:", len(errs)) - for i, err := range errs { - points[i+1] = fmt.Sprintf("* %s", err) - } - return errors.New(strings.Join(points, "\n\t")) - } -} diff --git a/vendor/github.com/containerd/stargz-snapshotter/estargz/estargz.go b/vendor/github.com/containerd/stargz-snapshotter/estargz/estargz.go index 693730420c..0503ff68a8 100644 --- a/vendor/github.com/containerd/stargz-snapshotter/estargz/estargz.go +++ b/vendor/github.com/containerd/stargz-snapshotter/estargz/estargz.go @@ -38,7 +38,6 @@ import ( "sync" "time" - "github.com/containerd/stargz-snapshotter/estargz/errorutil" digest "github.com/opencontainers/go-digest" "github.com/vbatts/tar-split/archive/tar" ) @@ -164,7 +163,7 @@ func Open(sr *io.SectionReader, opt ...OpenOption) (*Reader, error) { allErr = append(allErr, err) } if !found { - return nil, errorutil.Aggregate(allErr) + return nil, errors.Join(allErr...) } if err := r.initFields(); err != nil { return nil, fmt.Errorf("failed to initialize fields of entries: %v", err) @@ -192,7 +191,7 @@ func OpenFooter(sr *io.SectionReader) (tocOffset int64, footerSize int64, rErr e } allErr = append(allErr, err) } - return 0, 0, errorutil.Aggregate(allErr) + return 0, 0, errors.Join(allErr...) } // initFields populates the Reader from r.toc after decoding it from @@ -286,7 +285,7 @@ func (r *Reader) initFields() error { pdir := r.getOrCreateDir(pdirName) ent.NumLink++ // at least one name(ent.Name) references this entry. if ent.Type == "hardlink" { - org, err := r.getSource(ent) + org, err := r.getSource(ent, make(map[string]struct{})) if err != nil { return err } @@ -319,13 +318,18 @@ func (r *Reader) initFields() error { return nil } -func (r *Reader) getSource(ent *TOCEntry) (_ *TOCEntry, err error) { +func (r *Reader) getSource(ent *TOCEntry, visited map[string]struct{}) (_ *TOCEntry, err error) { if ent.Type == "hardlink" { - org, ok := r.m[cleanEntryName(ent.LinkName)] + entName := cleanEntryName(ent.LinkName) + if _, ok := visited[entName]; ok { + return nil, fmt.Errorf("cyclic hardlink detected") + } + visited[entName] = struct{}{} + org, ok := r.m[entName] if !ok { return nil, fmt.Errorf("%q is a hardlink but the linkname %q isn't found", ent.Name, ent.LinkName) } - ent, err = r.getSource(org) + ent, err = r.getSource(org, visited) if err != nil { return nil, err } @@ -493,7 +497,7 @@ func (r *Reader) Lookup(path string) (e *TOCEntry, ok bool) { e, ok = r.m[path] if ok && e.Type == "hardlink" { var err error - e, err = r.getSource(e) + e, err = r.getSource(e, make(map[string]struct{})) if err != nil { return nil, false } diff --git a/vendor/github.com/containerd/stargz-snapshotter/estargz/gzip.go b/vendor/github.com/containerd/stargz-snapshotter/estargz/gzip.go index 88fa13b191..dbf1458d32 100644 --- a/vendor/github.com/containerd/stargz-snapshotter/estargz/gzip.go +++ b/vendor/github.com/containerd/stargz-snapshotter/estargz/gzip.go @@ -100,21 +100,52 @@ func (gc *GzipCompressor) WriteTOCAndFooter(w io.Writer, off int64, toc *JTOC, d // gzipFooterBytes returns the 51 bytes footer. func gzipFooterBytes(tocOff int64) []byte { - buf := bytes.NewBuffer(make([]byte, 0, FooterSize)) - gz, _ := gzip.NewWriterLevel(buf, gzip.NoCompression) // MUST be NoCompression to keep 51 bytes - // Extra header indicating the offset of TOCJSON // https://tools.ietf.org/html/rfc1952#section-2.3.1.1 header := make([]byte, 4) header[0], header[1] = 'S', 'G' subfield := fmt.Sprintf("%016xSTARGZ", tocOff) binary.LittleEndian.PutUint16(header[2:4], uint16(len(subfield))) // little-endian per RFC1952 - gz.Extra = append(header, []byte(subfield)...) - gz.Close() - if buf.Len() != FooterSize { - panic(fmt.Sprintf("footer buffer = %d, not %d", buf.Len(), FooterSize)) + extra := append(header, []byte(subfield)...) + + buf := CreateGzipFooter(extra) + if len(buf) != FooterSize { + panic(fmt.Sprintf("footer buffer = %d, not %d", len(buf), FooterSize)) } - return buf.Bytes() + return buf +} + +// CreateGzipFooter creates eStargz's footer with the specified extra field. +func CreateGzipFooter(extra []byte) []byte { + // Gzip header (10 bytes) + // https://datatracker.ietf.org/doc/html/rfc1952 + buf := make([]byte, 10) + buf[0] = 0x1f // ID1 + buf[1] = 0x8b // ID2 + buf[2] = 8 // "deflate" + buf[3] = 0x4 // FEXTRA + // MTIME = 0 + // XFL = 0 + buf[9] = 255 // unknown + + // Extra field + xlen := make([]byte, 2) + binary.LittleEndian.PutUint16(xlen, uint16(len(extra))) + buf = append(buf, append(xlen, extra...)...) + + // Flate header (5 bytes) + // https://datatracker.ietf.org/doc/html/rfc1951 + flateHeader := []byte{ + 1, // BFINAL = 1 (last block), BFTYPE = 0 (no compression), remaining bits are ignored + 0, 0, // LEN = 0 + 0xff, 0xff, // NLEN (the one's complement of LEN) + } + buf = append(buf, flateHeader...) + + // Gzip footer (8 bytes): CRC32 = 0, ISIZE = 0 + buf = append(buf, make([]byte, 8)...) + + return buf } type GzipDecompressor struct{} @@ -137,6 +168,9 @@ func (gz *GzipDecompressor) ParseFooter(p []byte) (blobPayloadSize, tocOffset, t } defer zr.Close() extra := zr.Extra + if len(extra) < 4 { + return 0, 0, 0, fmt.Errorf("invalid extra field length %d; expected >= 4", len(extra)) + } si1, si2, subfieldlen, subfield := extra[0], extra[1], extra[2:4], extra[4:] if si1 != 'S' || si2 != 'G' { return 0, 0, 0, fmt.Errorf("invalid subfield IDs: %q, %q; want E, S", si1, si2) diff --git a/vendor/github.com/containerd/stargz-snapshotter/estargz/testutil.go b/vendor/github.com/containerd/stargz-snapshotter/estargz/testutil.go index ff165e090e..2dc9dae602 100644 --- a/vendor/github.com/containerd/stargz-snapshotter/estargz/testutil.go +++ b/vendor/github.com/containerd/stargz-snapshotter/estargz/testutil.go @@ -40,7 +40,6 @@ import ( "strings" "time" - "github.com/containerd/stargz-snapshotter/estargz/errorutil" "github.com/klauspost/compress/zstd" digest "github.com/opencontainers/go-digest" ) @@ -187,15 +186,10 @@ func testBuild(t *TestRunner, controllers ...TestingControllerFactory) { tt.minChunkSize = []int{0} } for _, srcCompression := range srcCompressions { - srcCompression := srcCompression for _, newCL := range controllers { - newCL := newCL for _, srcTarFormat := range []tar.Format{tar.FormatUSTAR, tar.FormatPAX, tar.FormatGNU} { - srcTarFormat := srcTarFormat for _, prefix := range allowedPrefix { - prefix := prefix for _, minChunkSize := range tt.minChunkSize { - minChunkSize := minChunkSize t.Run(tt.name+"-"+fmt.Sprintf("compression=%v,prefix=%q,src=%d,format=%s,minChunkSize=%d", newCL(), prefix, srcCompression, srcTarFormat, minChunkSize), func(t *TestRunner) { tarBlob := buildTar(t, tt.in, prefix, srcTarFormat) // Test divideEntries() @@ -675,15 +669,10 @@ func testDigestAndVerify(t *TestRunner, controllers ...TestingControllerFactory) tt.minChunkSize = []int{0} } for _, srcCompression := range srcCompressions { - srcCompression := srcCompression for _, newCL := range controllers { - newCL := newCL for _, prefix := range allowedPrefix { - prefix := prefix for _, srcTarFormat := range []tar.Format{tar.FormatUSTAR, tar.FormatPAX, tar.FormatGNU} { - srcTarFormat := srcTarFormat for _, minChunkSize := range tt.minChunkSize { - minChunkSize := minChunkSize t.Run(tt.name+"-"+fmt.Sprintf("compression=%v,prefix=%q,format=%s,minChunkSize=%d", newCL(), prefix, srcTarFormat, minChunkSize), func(t *TestRunner) { // Get original tar file and chunk digests dgstMap := make(map[string]digest.Digest) @@ -1170,6 +1159,8 @@ func testWriteAndOpen(t *TestRunner, controllers ...TestingControllerFactory) { wantNumGzLossLess int // expected number of streams (> 0) in lossless mode if it's different from wantNumGz wantFailOnLossLess bool wantTOCVersion int // default = 1 + + wantOpenError bool // opening this blob should fail with returning an error }{ { name: "empty", @@ -1484,15 +1475,19 @@ func testWriteAndOpen(t *TestRunner, controllers ...TestingControllerFactory) { hasFileContentsRange("foo3", len(data64KB)-1, data64KB[len(data64KB)-1:]), ), }, + { + name: "cyclic_hardlink", + in: tarOf( + link("foo", "foo"), + ), + wantOpenError: true, + }, } for _, tt := range tests { for _, newCL := range controllers { - newCL := newCL for _, prefix := range allowedPrefix { - prefix := prefix for _, srcTarFormat := range []tar.Format{tar.FormatUSTAR, tar.FormatPAX, tar.FormatGNU} { - srcTarFormat := srcTarFormat for _, lossless := range []bool{true, false} { t.Run(tt.name+"-"+fmt.Sprintf("compression=%v,prefix=%q,lossless=%v,format=%s", newCL(), prefix, lossless, srcTarFormat), func(t *TestRunner) { var tr io.Reader = buildTar(t, tt.in, prefix, srcTarFormat) @@ -1558,6 +1553,9 @@ func testWriteAndOpen(t *TestRunner, controllers ...TestingControllerFactory) { WithTelemetry(telemetry), ) if err != nil { + if tt.wantOpenError { + return + } t.Fatalf("stargz.Open: %v", err) } if _, ok := r.Lookup(""); !ok { @@ -1634,24 +1632,24 @@ func newCalledTelemetry() (telemetry *Telemetry, check func(needsGetTOC bool) er var getTocLatencyCalled bool var deserializeTocLatencyCalled bool return &Telemetry{ - func(time.Time) { getFooterLatencyCalled = true }, - func(time.Time) { getTocLatencyCalled = true }, - func(time.Time) { deserializeTocLatencyCalled = true }, - }, func(needsGetTOC bool) error { - var allErr []error - if !getFooterLatencyCalled { - allErr = append(allErr, fmt.Errorf("metrics GetFooterLatency isn't called")) - } - if needsGetTOC { - if !getTocLatencyCalled { - allErr = append(allErr, fmt.Errorf("metrics GetTocLatency isn't called")) - } - } - if !deserializeTocLatencyCalled { - allErr = append(allErr, fmt.Errorf("metrics DeserializeTocLatency isn't called")) - } - return errorutil.Aggregate(allErr) + func(time.Time) { getFooterLatencyCalled = true }, + func(time.Time) { getTocLatencyCalled = true }, + func(time.Time) { deserializeTocLatencyCalled = true }, + }, func(needsGetTOC bool) error { + var allErr []error + if !getFooterLatencyCalled { + allErr = append(allErr, fmt.Errorf("metrics GetFooterLatency isn't called")) } + if needsGetTOC { + if !getTocLatencyCalled { + allErr = append(allErr, fmt.Errorf("metrics GetTocLatency isn't called")) + } + } + if !deserializeTocLatencyCalled { + allErr = append(allErr, fmt.Errorf("metrics DeserializeTocLatency isn't called")) + } + return errors.Join(allErr...) + } } func digestFor(content string) string { @@ -2072,7 +2070,7 @@ func (f tarEntryFunc) appendTar(tw *tar.Writer, prefix string, format tar.Format return f(tw, prefix, format) } -func buildTar(t TestingT, ents []tarEntry, prefix string, opts ...interface{}) *io.SectionReader { +func buildTar(t TestingT, ents []tarEntry, prefix string, opts ...any) *io.SectionReader { format := tar.FormatUnknown for _, opt := range opts { switch v := opt.(type) { @@ -2096,7 +2094,7 @@ func buildTar(t TestingT, ents []tarEntry, prefix string, opts ...interface{}) * return io.NewSectionReader(bytes.NewReader(data), 0, int64(len(data))) } -func dir(name string, opts ...interface{}) tarEntry { +func dir(name string, opts ...any) tarEntry { return tarEntryFunc(func(tw *tar.Writer, prefix string, format tar.Format) error { var o owner mode := os.FileMode(0755) @@ -2137,7 +2135,7 @@ type owner struct { gid int } -func file(name, contents string, opts ...interface{}) tarEntry { +func file(name, contents string, opts ...any) tarEntry { return tarEntryFunc(func(tw *tar.Writer, prefix string, format tar.Format) error { var xattrs xAttr var o owner @@ -2164,15 +2162,19 @@ func file(name, contents string, opts ...interface{}) tarEntry { if len(xattrs) > 0 { format = tar.FormatPAX // only PAX supports xattrs } + xattrsPAXRecords := make(map[string]string) + for k, v := range xattrs { + xattrsPAXRecords["SCHILY.xattr."+k] = v + } if err := tw.WriteHeader(&tar.Header{ - Typeflag: tar.TypeReg, - Name: prefix + name, - Mode: tm, - Xattrs: xattrs, - Size: int64(len(contents)), - Uid: o.uid, - Gid: o.gid, - Format: format, + Typeflag: tar.TypeReg, + Name: prefix + name, + Mode: tm, + PAXRecords: xattrsPAXRecords, + Size: int64(len(contents)), + Uid: o.uid, + Gid: o.gid, + Format: format, }); err != nil { return err } @@ -2349,7 +2351,7 @@ func (f fileInfoOnlyMode) Size() int64 { return 0 } func (f fileInfoOnlyMode) Mode() os.FileMode { return os.FileMode(f) } func (f fileInfoOnlyMode) ModTime() time.Time { return time.Now() } func (f fileInfoOnlyMode) IsDir() bool { return os.FileMode(f).IsDir() } -func (f fileInfoOnlyMode) Sys() interface{} { return nil } +func (f fileInfoOnlyMode) Sys() any { return nil } func CheckGzipHasStreams(t TestingT, b []byte, streams []int64) { if len(streams) == 0 { diff --git a/vendor/github.com/containerd/stargz-snapshotter/estargz/types.go b/vendor/github.com/containerd/stargz-snapshotter/estargz/types.go index 57e0aa614e..41eb6571fb 100644 --- a/vendor/github.com/containerd/stargz-snapshotter/estargz/types.go +++ b/vendor/github.com/containerd/stargz-snapshotter/estargz/types.go @@ -246,7 +246,7 @@ func (fi fileInfo) Name() string { return path.Base(fi.e.Name) } func (fi fileInfo) IsDir() bool { return fi.e.Type == "dir" } func (fi fileInfo) Size() int64 { return fi.e.Size } func (fi fileInfo) ModTime() time.Time { return fi.e.ModTime() } -func (fi fileInfo) Sys() interface{} { return fi.e } +func (fi fileInfo) Sys() any { return fi.e } func (fi fileInfo) Mode() (m os.FileMode) { // TOCEntry.Mode is tar.Header.Mode so we can understand the these bits using `tar` pkg. m = (&tar.Header{Mode: fi.e.Mode}).FileInfo().Mode() & diff --git a/vendor/github.com/moby/buildkit/cache/manager.go b/vendor/github.com/moby/buildkit/cache/manager.go index 7a02d192b9..07e2e8ad84 100644 --- a/vendor/github.com/moby/buildkit/cache/manager.go +++ b/vendor/github.com/moby/buildkit/cache/manager.go @@ -394,6 +394,11 @@ func (cm *cacheManager) init(ctx context.Context) error { cm.LeaseManager.Delete(ctx, leases.Lease{ID: si.ID() + "-variants"}) } } + if n, err := cm.MetadataStore.ClearOrphans(); err != nil { + bklog.G(ctx).Warnf("failed to remove external metadata of deleted cache records: %v", err) + } else if n > 0 { + bklog.G(ctx).Infof("removed external metadata of %d deleted cache records", n) + } return nil } @@ -470,7 +475,10 @@ func (cm *cacheManager) getRecord(ctx context.Context, id string, opts ...RefOpt } if rec, ok := cm.records[id]; ok { - if rec.isDead() { + rec.mu.Lock() + isDead := rec.isDead() + rec.mu.Unlock() + if isDead { return nil, errors.Wrapf(errNotFound, "failed to get dead record %s", id) } if err := checkLazyProviders(rec); err != nil { @@ -498,7 +506,7 @@ func (cm *cacheManager) getRecord(ctx context.Context, id string, opts ...RefOpt mutable, err := cm.getRecord(ctx, mutableID) if err == nil { rec := &cacheRecord{ - mu: &sync.Mutex{}, + mu: mutable.mu, // share the mutex as in mutableRef.commit cm: cm, refs: make(map[ref]struct{}), parentRefs: parents, @@ -570,10 +578,10 @@ func (cm *cacheManager) getRecord(ctx context.Context, id string, opts ...RefOpt return nil, errors.Wrapf(err, "failed to append image ref metadata to ref %s", rec.ID()) } - cm.records[id] = rec if err := checkLazyProviders(rec); err != nil { return nil, err } + cm.records[id] = rec return rec, nil } diff --git a/vendor/github.com/moby/buildkit/cache/metadata/metadata.go b/vendor/github.com/moby/buildkit/cache/metadata/metadata.go index 01c6608e72..fd90c4111b 100644 --- a/vendor/github.com/moby/buildkit/cache/metadata/metadata.go +++ b/vendor/github.com/moby/buildkit/cache/metadata/metadata.go @@ -190,6 +190,40 @@ func (s *Store) Clear(id string) error { })) } +func (s *Store) ClearOrphans() (int, error) { + var n int + err := s.db.Update(func(tx *bolt.Tx) error { + external := tx.Bucket([]byte(externalBucket)) + if external == nil { + return nil + } + var records *bolt.Cursor + if main := tx.Bucket([]byte(mainBucket)); main != nil { + records = main.Cursor() + } + var orphans [][]byte + if err := external.ForEach(func(k, _ []byte) error { + if records != nil { + if id, v := records.Seek(k); bytes.Equal(id, k) && v == nil { + return nil + } + } + orphans = append(orphans, append([]byte(nil), k...)) + return nil + }); err != nil { + return err + } + for _, id := range orphans { + if err := external.DeleteBucket(id); err != nil { + return err + } + } + n = len(orphans) + return nil + }) + return n, errors.WithStack(err) +} + func (s *Store) Update(id string, fn func(b *bolt.Bucket) error) error { return errors.WithStack(s.db.Update(func(tx *bolt.Tx) error { b, err := tx.CreateBucketIfNotExists([]byte(mainBucket)) @@ -330,6 +364,10 @@ func (s *StorageItem) GetExternal(k string) ([]byte, error) { func (s *StorageItem) SetExternal(k string, dt []byte) error { return errors.WithStack(s.storage.db.Update(func(tx *bolt.Tx) error { + main := tx.Bucket([]byte(mainBucket)) + if main == nil || main.Bucket([]byte(s.id)) == nil { + return errors.Wrapf(errNotFound, "record %s", s.id) + } b, err := tx.CreateBucketIfNotExists([]byte(externalBucket)) if err != nil { return errors.WithStack(err) diff --git a/vendor/github.com/moby/buildkit/cache/refs.go b/vendor/github.com/moby/buildkit/cache/refs.go index 4b0005de47..7f20320803 100644 --- a/vendor/github.com/moby/buildkit/cache/refs.go +++ b/vendor/github.com/moby/buildkit/cache/refs.go @@ -103,6 +103,10 @@ type cacheRecord struct { equalMutable *mutableRef equalImmutable *immutableRef + // layerDigestChainCache memoizes layerDigestChain. It is only set when + // every layer has a blob, as a blob can't be replaced after it is set. + // Access is serialized by cacheManager.mu, as computing the chain of a + // record also fills the memoized chains of its parents. layerDigestChainCache []digest.Digest } @@ -554,18 +558,10 @@ func (sr *immutableRef) layerWalk(f func(*immutableRef)) { parent.layerWalk(f) } case Diff: - lower := sr.diffParents.lower - upper := sr.diffParents.upper - // If upper is only one blob different from lower, then re-use that blob - switch { - case upper != nil && lower == nil && upper.kind() == BaseLayer: - // upper is a single layer being diffed with scratch - f(upper) - case upper != nil && lower != nil && upper.kind() == Layer && upper.layerParent.ID() == lower.ID(): - // upper is a single layer on top of lower - f(upper) - default: - // otherwise, the diff will be computed and turned into its own single blob + if sr.diffReusesUpper() { + f(sr.diffParents.upper) + } else { + // the diff will be computed and turned into its own single blob f(sr) } case Layer: @@ -576,30 +572,65 @@ func (sr *immutableRef) layerWalk(f func(*immutableRef)) { } } -// hold cacheRecord.mu lock before calling +// diffReusesUpper reports whether the Diff record cr is exported as the blob +// of its upper, because upper is only one blob different from lower. Such a +// diff never gets a blob of its own. +func (cr *cacheRecord) diffReusesUpper() bool { + lower := cr.diffParents.lower + upper := cr.diffParents.upper + switch { + case upper != nil && lower == nil && upper.kind() == BaseLayer: + // upper is a single layer being diffed with scratch + return true + case upper != nil && lower != nil && upper.kind() == Layer && upper.layerParent.ID() == lower.ID(): + // upper is a single layer on top of lower + return true + default: + return false + } +} + +// layerDigestChain returns the blob digests of the layers of cr, or nil if any +// of them has no blob yet. Such a record has never been exported, so no image +// can reference it. The caller must hold cacheManager.mu and cacheRecord.mu. func (cr *cacheRecord) layerDigestChain() []digest.Digest { if cr.layerDigestChainCache != nil { return cr.layerDigestChainCache } + // The own blob of a record is checked before its parents, so that a + // record without a blob doesn't walk its ancestors on every call. + var chain []digest.Digest switch cr.kind() { case Diff: - if cr.getBlob() == "" && cr.diffParents.upper != nil { - // this diff just reuses the upper blob - cr.layerDigestChainCache = cr.diffParents.upper.layerDigestChain() - } else { - cr.layerDigestChainCache = append(cr.layerDigestChainCache, cr.getBlob()) + if blob := cr.getBlob(); blob != "" { + chain = []digest.Digest{blob} + } else if cr.diffReusesUpper() { + // export uses the blob of upper as the only layer of the diff + if blob := cr.diffParents.upper.getBlob(); blob != "" { + chain = []digest.Digest{blob} + } } case Merge: for _, parent := range cr.mergeParents { - cr.layerDigestChainCache = append(cr.layerDigestChainCache, parent.layerDigestChain()...) + parentChain := parent.layerDigestChain() + if parentChain == nil { + return nil + } + chain = append(chain, parentChain...) } case Layer: - cr.layerDigestChainCache = append(cr.layerDigestChainCache, cr.layerParent.layerDigestChain()...) - fallthrough + if blob := cr.getBlob(); blob != "" { + if parentChain := cr.layerParent.layerDigestChain(); parentChain != nil { + chain = slices.Concat(parentChain, []digest.Digest{blob}) + } + } case BaseLayer: - cr.layerDigestChainCache = append(cr.layerDigestChainCache, cr.getBlob()) + if blob := cr.getBlob(); blob != "" { + chain = []digest.Digest{blob} + } } - return cr.layerDigestChainCache + cr.layerDigestChainCache = chain + return chain } type RefList []ImmutableRef @@ -967,7 +998,9 @@ func fieldsFromLabels(labels map[string]string) (fields []string) { } func (sr *immutableRef) Mount(ctx context.Context, readonly bool, s session.Group) (_ snapshot.Mountable, rerr error) { - if sr.equalMutable != nil && !readonly { + // Finalize is a no-op if there is no equalMutable. Don't check it here, + // equalMutable is protected by sr.mu. + if !readonly { if err := sr.Finalize(ctx); err != nil { return nil, err } @@ -1536,6 +1569,11 @@ func (cr *cacheRecord) finalize(ctx context.Context) error { return nil } + // The snapshotter may apply the commit even when the caller sees a + // cancellation (e.g. a containerd RPC canceled in flight), leaving the + // mutable snapshot gone while the record still points to it. + ctx = context.WithoutCancel(ctx) + _, err := cr.cm.LeaseManager.Create(ctx, func(l *leases.Lease) error { l.ID = cr.ID() l.Labels = map[string]string{ @@ -1553,13 +1591,23 @@ func (cr *cacheRecord) finalize(ctx context.Context) error { ID: cr.getSnapshotID(), Type: "snapshots/" + cr.cm.Snapshotter.Name(), }); err != nil { - _ = cr.cm.LeaseManager.Delete(context.WithoutCancel(ctx), leases.Lease{ID: cr.ID()}) + _ = cr.cm.LeaseManager.Delete(ctx, leases.Lease{ID: cr.ID()}) return errors.Wrapf(err, "failed to add snapshot %s to lease", cr.getSnapshotID()) } if err := cr.cm.Snapshotter.Commit(ctx, cr.getSnapshotID(), mutable.getSnapshotID()); err != nil { - _ = cr.cm.LeaseManager.Delete(context.WithoutCancel(ctx), leases.Lease{ID: cr.ID()}) - return errors.Wrapf(err, "failed to commit %s to %s during finalize", mutable.getSnapshotID(), cr.getSnapshotID()) + // the commit may still have been applied, e.g. if the response was lost + info, statErr := cr.cm.Snapshotter.Stat(ctx, cr.getSnapshotID()) + if statErr != nil || info.Kind != snapshots.KindCommitted { + // keep the lease if it is unknown whether the commit was applied + if statErr == nil || errors.Is(statErr, cerrdefs.ErrNotFound) { + _ = cr.cm.LeaseManager.Delete(ctx, leases.Lease{ID: cr.ID()}) + } else { + bklog.G(ctx).WithError(statErr).Warnf("failed to check snapshot %s after commit error, keeping lease", cr.getSnapshotID()) + } + return errors.Wrapf(err, "failed to commit %s to %s during finalize", mutable.getSnapshotID(), cr.getSnapshotID()) + } + bklog.G(ctx).WithError(err).Warnf("snapshot %s was committed to %s despite commit error", mutable.getSnapshotID(), cr.getSnapshotID()) } cr.mountCache = nil @@ -1567,9 +1615,8 @@ func (cr *cacheRecord) finalize(ctx context.Context) error { go func() { cr.cm.mu.Lock() defer cr.cm.mu.Unlock() - cleanupCtx := context.WithoutCancel(ctx) - if err := mutable.remove(cleanupCtx, true); err != nil { - bklog.G(cleanupCtx).Error(err) + if err := mutable.remove(ctx, true); err != nil { + bklog.G(ctx).Error(err) } }() @@ -1856,6 +1903,8 @@ func (sm *sharableMountable) Mount() (_ []mount.Mount, _ func() error, retErr er return nil, nil, err } } + + mounts = mount.RemoveVolatileOption(mounts) if err := mount.All(mounts, dir); err != nil { return nil, nil, err } diff --git a/vendor/github.com/moby/buildkit/cache/remotecache/gha/gha.go b/vendor/github.com/moby/buildkit/cache/remotecache/gha/gha.go index a340d378ec..a57da3ada7 100644 --- a/vendor/github.com/moby/buildkit/cache/remotecache/gha/gha.go +++ b/vendor/github.com/moby/buildkit/cache/remotecache/gha/gha.go @@ -7,7 +7,6 @@ import ( "fmt" "io" "os" - "os/exec" "strconv" "strings" "sync" @@ -17,7 +16,7 @@ import ( "github.com/containerd/containerd/v2/pkg/labels" cerrdefs "github.com/containerd/errdefs" "github.com/moby/buildkit/cache/remotecache" - "github.com/moby/buildkit/cache/remotecache/gha/ghatypes" + "github.com/moby/buildkit/cache/remotecache/signing" v1 "github.com/moby/buildkit/cache/remotecache/v1" cacheimporttypes "github.com/moby/buildkit/cache/remotecache/v1/types" "github.com/moby/buildkit/session" @@ -28,11 +27,9 @@ import ( "github.com/moby/buildkit/util/tracing" bkversion "github.com/moby/buildkit/version" "github.com/moby/buildkit/worker" - policy "github.com/moby/policy-helpers" digest "github.com/opencontainers/go-digest" ocispecs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" - "github.com/sigstore/sigstore-go/pkg/fulcio/certificate" actionscache "github.com/tonistiigi/go-actions-cache" "golang.org/x/sync/errgroup" ) @@ -55,8 +52,6 @@ const ( defaultTimeout = 10 * time.Minute ) -type VerifierProvider func() (*policy.Verifier, error) - type Config struct { Scope string URL string @@ -66,11 +61,11 @@ type Config struct { Version int Timeout time.Duration - *ghatypes.CacheConfig - verifier VerifierProvider + *signing.Config + verifier signing.Verifier } -func getConfig(conf *ghatypes.CacheConfig, v VerifierProvider, attrs map[string]string) (*Config, error) { +func getConfig(conf *signing.Config, v signing.VerifierProvider, attrs map[string]string) (*Config, error) { scope, ok := attrs[attrScope] if !ok { scope = "buildkit" @@ -120,24 +115,24 @@ func getConfig(conf *ghatypes.CacheConfig, v VerifierProvider, attrs map[string] } if conf == nil { - conf = &ghatypes.CacheConfig{} + conf = &signing.Config{} } return &Config{ - Scope: scope, - URL: url, - Token: token, - Timeout: timeout, - GHToken: attrs[attrGHToken], - Repository: attrs[attrRepository], - Version: apiVersionInt, - CacheConfig: conf, - verifier: v, + Scope: scope, + URL: url, + Token: token, + Timeout: timeout, + GHToken: attrs[attrGHToken], + Repository: attrs[attrRepository], + Version: apiVersionInt, + Config: conf, + verifier: signing.NewVerifier(conf.Verify.Policy, v), }, nil } // ResolveCacheExporterFunc for Github actions cache exporter. -func ResolveCacheExporterFunc(conf *ghatypes.CacheConfig, v VerifierProvider) remotecache.ResolveCacheExporterFunc { +func ResolveCacheExporterFunc(conf *signing.Config, v signing.VerifierProvider) remotecache.ResolveCacheExporterFunc { return func(ctx context.Context, g session.Group, attrs map[string]string) (remotecache.Exporter, error) { cfg, err := getConfig(conf, v, attrs) if err != nil { @@ -330,98 +325,23 @@ func (ce *exporter) Finalize(ctx context.Context) (_ map[string]string, err erro dgst := digest.FromBytes(dt) signDone := progress.OneOff(ctx, fmt.Sprintf("signing cache index %s", dgst)) - defer signDone(err) + defer func() { signDone(err) }() - cmd := exec.CommandContext(ctx, args[0], args[1:]...) //nolint:gosec // defined in toml config - cmd.Stdin = bytes.NewReader(dt) - var out bytes.Buffer - cmd.Stdout = &out - var stderr bytes.Buffer - cmd.Stderr = &stderr - if err := cmd.Run(); err != nil { - return nil, errors.Wrapf(err, "signing command failed: %s", stderr.String()) - } - - // validate signature before uploading - if err := verifySignature(ctx, dgst, out.Bytes(), ce.config); err != nil { + bundle, err := signing.NewSigner(args, ce.config.verifier).Sign(ctx, dt) + if err != nil { return nil, err } key := blobKey(dgst + "-sig") - if err := ce.cache.Save(ctx, key, actionscache.NewBlob(out.Bytes())); err != nil { + if err := ce.cache.Save(ctx, key, actionscache.NewBlob(bundle)); err != nil { return nil, err } return nil, nil } -func verifySignature(ctx context.Context, dgst digest.Digest, bundle []byte, config *Config) error { - v, err := config.verifier() - if err != nil { - return err - } - if v == nil { - return errors.New("no verifier available for signed github actions cache") - } - - sig, err := v.VerifyArtifact(ctx, dgst, bundle, policy.WithSLSANotRequired()) - if err != nil { - return err - } - if sig.Signer == nil { - return errors.New("signature verification failed: no signer found") - } - numTimestamps := len(sig.Timestamps) - numTlog := 0 - for _, t := range sig.Timestamps { - if t.Type == "Tlog" { - numTlog++ - } - } - policyRules := config.Verify.Policy - if policyRules.TimestampThreshold > numTimestamps { - return errors.Errorf("signature verification failed: not enough timestamp authorities: have %d, need %d", numTimestamps, policyRules.TimestampThreshold) - } - if policyRules.TlogThreshold > numTlog { - return errors.Errorf("signature verification failed: not enough tlog authorities: have %d, need %d", numTlog, policyRules.TlogThreshold) - } - - certRules, err := certToStringMap(&config.Verify.Policy.Summary) - if err != nil { - return err - } - certFields, err := certToStringMap(sig.Signer) - if err != nil { - return err - } - bklog.G(ctx).Debugf("signature verification: %+v", sig) - bklog.G(ctx).Debugf("signer: %+v", sig.Signer) - for k, v := range certRules { - if v == "" { - continue - } - if !simplePatternMatch(v, certFields[k]) { - return errors.Errorf("signature verification failed: certificate field %q does not match policy (%q != %q)", k, certFields[k], v) - } - bklog.G(ctx).Debugf("certificate field %q matches policy (%q)", k, certFields[k]) - } - return nil -} - -func certToStringMap(cert *certificate.Summary) (map[string]string, error) { - dt, err := json.Marshal(cert) - if err != nil { - return nil, err - } - m := map[string]string{} - if err := json.Unmarshal(dt, &m); err != nil { - return nil, err - } - return m, nil -} - // ResolveCacheImporterFunc for Github actions cache importer. -func ResolveCacheImporterFunc(conf *ghatypes.CacheConfig, v VerifierProvider) remotecache.ResolveCacheImporterFunc { +func ResolveCacheImporterFunc(conf *signing.Config, v signing.VerifierProvider) remotecache.ResolveCacheImporterFunc { return func(ctx context.Context, g session.Group, attrs map[string]string) (remotecache.Importer, ocispecs.Descriptor, error) { cfg, err := getConfig(conf, v, attrs) if err != nil { @@ -518,7 +438,7 @@ func (ci *importer) loadScope(ctx context.Context, scope string) (*v1.CacheChain verifyDone(err) return nil, err } - if err := verifySignature(ctx, dgst, sigBuf.Bytes(), ci.config); err != nil { + if err := ci.config.verifier.Verify(ctx, dgst, sigBuf.Bytes()); err != nil { verifyDone(err) return nil, err } @@ -648,19 +568,3 @@ func (r *readerAt) ReadAt(p []byte, off int64) (int, error) { func (r *readerAt) Size() int64 { return r.desc.Size } - -func simplePatternMatch(pat, s string) bool { - if pat == "*" { - return true - } - if strings.HasPrefix(pat, "*") && strings.HasSuffix(pat, "*") { - return strings.Contains(s, pat[1:len(pat)-1]) - } - if strings.HasPrefix(pat, "*") { - return strings.HasSuffix(s, pat[1:]) - } - if strings.HasSuffix(pat, "*") { - return strings.HasPrefix(s, pat[:len(pat)-1]) - } - return s == pat -} diff --git a/vendor/github.com/moby/buildkit/cache/remotecache/gha/ghatypes/config.go b/vendor/github.com/moby/buildkit/cache/remotecache/gha/ghatypes/config.go deleted file mode 100644 index 770ef624ed..0000000000 --- a/vendor/github.com/moby/buildkit/cache/remotecache/gha/ghatypes/config.go +++ /dev/null @@ -1,23 +0,0 @@ -package ghatypes - -import "github.com/sigstore/sigstore-go/pkg/fulcio/certificate" - -type CacheConfig struct { - Sign *SignConfig `toml:"sign"` - Verify VerifyConfig `toml:"verify"` -} - -type SignConfig struct { - Command []string `toml:"command"` -} - -type VerifyConfig struct { - Required bool `toml:"required"` - Policy VerifyPolicy `toml:"policy"` -} - -type VerifyPolicy struct { - TimestampThreshold int `toml:"timestampThreshold"` - TlogThreshold int `toml:"tlogThreshold"` - certificate.Summary -} diff --git a/vendor/github.com/moby/buildkit/cache/remotecache/signing/config.go b/vendor/github.com/moby/buildkit/cache/remotecache/signing/config.go new file mode 100644 index 0000000000..e333819040 --- /dev/null +++ b/vendor/github.com/moby/buildkit/cache/remotecache/signing/config.go @@ -0,0 +1,27 @@ +package signing + +import "github.com/sigstore/sigstore-go/pkg/fulcio/certificate" + +// Config controls signing and verification of remote cache metadata. +type Config struct { + Sign *SignConfig `toml:"sign"` + Verify VerifyConfig `toml:"verify"` +} + +// SignConfig specifies the command that reads a payload and writes its bundle. +type SignConfig struct { + Command []string `toml:"command"` +} + +// VerifyConfig controls whether imported cache metadata must be verified. +type VerifyConfig struct { + Required bool `toml:"required"` + Policy VerifyPolicy `toml:"policy"` +} + +// VerifyPolicy constrains the verified timestamps and signing certificate. +type VerifyPolicy struct { + TimestampThreshold int `toml:"timestampThreshold"` + TlogThreshold *int `toml:"tlogThreshold"` + certificate.Summary +} diff --git a/vendor/github.com/moby/buildkit/cache/remotecache/signing/sign.go b/vendor/github.com/moby/buildkit/cache/remotecache/signing/sign.go new file mode 100644 index 0000000000..29f98dc612 --- /dev/null +++ b/vendor/github.com/moby/buildkit/cache/remotecache/signing/sign.go @@ -0,0 +1,48 @@ +package signing + +import ( + "bytes" + "context" + "os/exec" + "slices" + + digest "github.com/opencontainers/go-digest" + "github.com/pkg/errors" +) + +// Signer signs the exact payload bytes and verifies the resulting bundle. +type Signer interface { + Sign(context.Context, []byte) ([]byte, error) +} + +// NewSigner returns a command-based signer using verifier to check its output. +// The command is an argv array configured by the daemon administrator. +func NewSigner(command []string, verifier Verifier) Signer { + return &commandSigner{command: slices.Clone(command), verifier: verifier} +} + +type commandSigner struct { + command []string + verifier Verifier +} + +func (s *commandSigner) Sign(ctx context.Context, payload []byte) ([]byte, error) { + if len(s.command) == 0 { + return nil, errors.New("no signing command configured") + } + if s.verifier == nil { + return nil, errors.New("no verifier available for signed cache") + } + cmd := exec.CommandContext(ctx, s.command[0], s.command[1:]...) //nolint:gosec // defined in toml config + cmd.Stdin = bytes.NewReader(payload) + var out, stderr bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + return nil, errors.Wrapf(err, "signing command failed: %s", stderr.String()) + } + if err := s.verifier.Verify(ctx, digest.FromBytes(payload), out.Bytes()); err != nil { + return nil, err + } + return out.Bytes(), nil +} diff --git a/vendor/github.com/moby/buildkit/cache/remotecache/signing/verify.go b/vendor/github.com/moby/buildkit/cache/remotecache/signing/verify.go new file mode 100644 index 0000000000..fc39d46e1d --- /dev/null +++ b/vendor/github.com/moby/buildkit/cache/remotecache/signing/verify.go @@ -0,0 +1,114 @@ +package signing + +import ( + "context" + "encoding/json" + "strings" + + "github.com/moby/buildkit/util/bklog" + policy "github.com/moby/policy-helpers" + "github.com/moby/policy-helpers/types" + digest "github.com/opencontainers/go-digest" + "github.com/pkg/errors" + "github.com/sigstore/sigstore-go/pkg/fulcio/certificate" +) + +// Verifier authenticates a bundle against the digest and the configured policy. +type Verifier interface { + Verify(context.Context, digest.Digest, []byte) error +} + +// VerifierProvider lazily supplies the daemon's shared Sigstore verifier. +type VerifierProvider func() (*policy.Verifier, error) + +// NewVerifier returns a verifier applying the given policy to cache signatures. +func NewVerifier(rules VerifyPolicy, provider VerifierProvider) Verifier { + return &verifier{rules: rules, provider: provider} +} + +type verifier struct { + rules VerifyPolicy + provider VerifierProvider +} + +func (v *verifier) Verify(ctx context.Context, dgst digest.Digest, bundle []byte) error { + if v.provider == nil { + return errors.New("no verifier available for signed cache") + } + artifactVerifier, err := v.provider() + if err != nil { + return err + } + if artifactVerifier == nil { + return errors.New("no verifier available for signed cache") + } + opts := []policy.ArtifactVerifyOpt{policy.WithSLSANotRequired()} + if v.rules.TlogThreshold != nil { + opts = append(opts, policy.WithTlogThreshold(*v.rules.TlogThreshold)) + } + sig, err := artifactVerifier.VerifyArtifact(ctx, dgst, bundle, opts...) + if err != nil { + return err + } + return verifyPolicy(ctx, sig, v.rules) +} + +// verifyPolicy checks the authenticated certificate and timestamps. +func verifyPolicy(ctx context.Context, sig *types.SignatureInfo, policyRules VerifyPolicy) error { + if sig.Signer == nil { + return errors.New("signature verification failed: no signer found") + } + numTimestamps := len(sig.Timestamps) + if policyRules.TimestampThreshold > numTimestamps { + return errors.Errorf("signature verification failed: not enough timestamp authorities: have %d, need %d", numTimestamps, policyRules.TimestampThreshold) + } + + certRules, err := certToStringMap(&policyRules.Summary) + if err != nil { + return err + } + certFields, err := certToStringMap(sig.Signer) + if err != nil { + return err + } + bklog.G(ctx).Debugf("signature verification: %+v", sig) + bklog.G(ctx).Debugf("signer: %+v", sig.Signer) + for k, v := range certRules { + if v == "" { + continue + } + if !simplePatternMatch(v, certFields[k]) { + return errors.Errorf("signature verification failed: certificate field %q does not match policy (%q != %q)", k, certFields[k], v) + } + bklog.G(ctx).Debugf("certificate field %q matches policy (%q)", k, certFields[k]) + } + return nil +} + +func certToStringMap(cert *certificate.Summary) (map[string]string, error) { + dt, err := json.Marshal(cert) + if err != nil { + return nil, err + } + m := map[string]string{} + if err := json.Unmarshal(dt, &m); err != nil { + return nil, err + } + return m, nil +} + +func simplePatternMatch(pat, s string) bool { + if pat == "*" { + return true + } + if strings.HasPrefix(pat, "*") && strings.HasSuffix(pat, "*") { + return strings.Contains(s, pat[1:len(pat)-1]) + } + if strings.HasPrefix(pat, "*") { + return strings.HasSuffix(s, pat[1:]) + } + if strings.HasSuffix(pat, "*") { + return strings.HasPrefix(s, pat[:len(pat)-1]) + } + return s == pat +} diff --git a/vendor/github.com/moby/buildkit/cache/remotecache/v1/utils.go b/vendor/github.com/moby/buildkit/cache/remotecache/v1/utils.go index 8e004d8625..dba07903c9 100644 --- a/vendor/github.com/moby/buildkit/cache/remotecache/v1/utils.go +++ b/vendor/github.com/moby/buildkit/cache/remotecache/v1/utils.go @@ -10,6 +10,7 @@ import ( cerrdefs "github.com/containerd/errdefs" cacheimporttypes "github.com/moby/buildkit/cache/remotecache/v1/types" "github.com/moby/buildkit/solver" + "github.com/moby/buildkit/util/bklog" "github.com/moby/buildkit/util/compression" digest "github.com/opencontainers/go-digest" "github.com/pkg/errors" @@ -136,48 +137,40 @@ func marshalRemote(ctx context.Context, r *solver.Remote, state *marshalState) s if r.Provider != nil { for _, d := range r.Descriptors { - if _, err := r.Provider.Info(ctx, d.Digest); err != nil { - if !cerrdefs.IsNotImplemented(err) { - return "" - } + if _, err := r.Provider.Info(ctx, d.Digest); err != nil && !cerrdefs.IsNotImplemented(err) { + bklog.G(ctx).WithError(err).Warnf("skipping cache result: blob %s is not available", d.Digest) + return "" } } } var parentID string - if len(r.Descriptors) > 1 { - r2 := &solver.Remote{ - Descriptors: r.Descriptors[:len(r.Descriptors)-1], - Provider: r.Provider, + for _, desc := range r.Descriptors { + if desc.MediaType != "" { + desc = compression.ConvertAllLayerMediaTypes(ctx, true, desc)[0] } - parentID = marshalRemote(ctx, r2, state) - } - desc := r.Descriptors[len(r.Descriptors)-1] - if desc.MediaType != "" { - desc = compression.ConvertAllLayerMediaTypes(ctx, true, desc)[0] - } - state.descriptors[desc.Digest] = DescriptorProviderPair{ - Descriptor: desc, - Provider: r.Provider, - } + state.descriptors[desc.Digest] = DescriptorProviderPair{ + Descriptor: desc, + Provider: r.Provider, + } - id := desc.Digest.String() + parentID + id := desc.Digest.String() + parentID - if _, ok := state.chainsByID[id]; ok { - return id + if _, ok := state.chainsByID[id]; !ok { + state.chainsByID[id] = len(state.layers) + l := cacheimporttypes.CacheLayer{ + Blob: desc.Digest, + ParentIndex: -1, + } + if parentID != "" { + l.ParentIndex = state.chainsByID[parentID] + } + state.layers = append(state.layers, l) + } + parentID = id } - - state.chainsByID[id] = len(state.layers) - l := cacheimporttypes.CacheLayer{ - Blob: desc.Digest, - ParentIndex: -1, - } - if parentID != "" { - l.ParentIndex = state.chainsByID[parentID] - } - state.layers = append(state.layers, l) - return id + return parentID } func marshalItem(ctx context.Context, it *item, state *marshalState) error { diff --git a/vendor/github.com/moby/buildkit/client/llb/fileop.go b/vendor/github.com/moby/buildkit/client/llb/fileop.go index 928d8503ba..ddaf5b7b8f 100644 --- a/vendor/github.com/moby/buildkit/client/llb/fileop.go +++ b/vendor/github.com/moby/buildkit/client/llb/fileop.go @@ -262,7 +262,10 @@ func parseUID(str string) (int, error) { if str == "root" { return 0, nil } - uid, err := strconv.ParseInt(str, 10, 32) + // uid_t/gid_t span 0..2^32-1, matching the daemon-side parser in + // executor/oci. ParseInt here would reject the upper half of that range and + // silently turn a numeric ID into a name lookup. + uid, err := strconv.ParseUint(str, 10, 32) if err != nil { return 0, err } diff --git a/vendor/github.com/moby/buildkit/client/llb/meta.go b/vendor/github.com/moby/buildkit/client/llb/meta.go index 4491fc7acc..19178a38d4 100644 --- a/vendor/github.com/moby/buildkit/client/llb/meta.go +++ b/vendor/github.com/moby/buildkit/client/llb/meta.go @@ -61,6 +61,21 @@ func addEnvf(key, value string, replace bool, v ...any) StateOption { } } +// DelEnv returns a [StateOption] which removes an environment variable from the state. +// This is the equivalent of [State.DelEnv] +func DelEnv(key string) StateOption { + return func(s State) State { + return s.withValue(keyEnv, func(ctx context.Context, c *Constraints) (any, error) { + env, err := getEnv(s)(ctx, c) + if err != nil { + return nil, err + } + deleted := env.Delete(key) + return &deleted, nil + }) + } +} + // Dir returns a [StateOption] sets the working directory for the state which will be used to resolve // relative paths as well as the working directory for [State.Run]. // See [State.With] for where to use this. diff --git a/vendor/github.com/moby/buildkit/client/llb/source.go b/vendor/github.com/moby/buildkit/client/llb/source.go index cf79fa5668..a3dd6af8dc 100644 --- a/vendor/github.com/moby/buildkit/client/llb/source.go +++ b/vendor/github.com/moby/buildkit/client/llb/source.go @@ -477,6 +477,9 @@ func Git(url, fragment string, opts ...GitOption) State { attrs[pb.AttrGitSkipSubmodules] = "true" addCap(&gi.Constraints, pb.CapSourceGitSkipSubmodules) } + if gi.DebugCommands { + attrs[pb.AttrGitDebugCommands] = "true" + } if gi.MTime != "" { attrs[pb.AttrGitMTime] = gi.MTime @@ -536,6 +539,7 @@ type GitInfo struct { SkipSubmodules bool MTime string Advice bool + DebugCommands bool Bundle string BundleOCISessionID string BundleOCIStoreID string @@ -583,6 +587,14 @@ func GitAdvice(enabled bool) GitOption { }) } +// GitDebugCommands controls whether executed Git commands are logged while +// resolving this git source. +func GitDebugCommands(enabled bool) GitOption { + return gitOptionFunc(func(gi *GitInfo) { + gi.DebugCommands = enabled + }) +} + func KeepGitDir() GitOption { return gitOptionFunc(func(gi *GitInfo) { gi.KeepGitDir = true diff --git a/vendor/github.com/moby/buildkit/client/llb/state.go b/vendor/github.com/moby/buildkit/client/llb/state.go index 370012c88c..b2df7b60c9 100644 --- a/vendor/github.com/moby/buildkit/client/llb/state.go +++ b/vendor/github.com/moby/buildkit/client/llb/state.go @@ -326,6 +326,12 @@ func (s State) AddEnvf(key, value string, v ...any) State { return AddEnvf(key, value, v...)(s) } +// DelEnv returns a new [State] with the provided environment variable removed. +// See [DelEnv] +func (s State) DelEnv(key string) State { + return DelEnv(key)(s) +} + // Dir returns a new [State] with the provided working directory set. // See [Dir] func (s State) Dir(str string) State { diff --git a/vendor/github.com/moby/buildkit/client/solve.go b/vendor/github.com/moby/buildkit/client/solve.go index 77ba6f097a..9eb4319f63 100644 --- a/vendor/github.com/moby/buildkit/client/solve.go +++ b/vendor/github.com/moby/buildkit/client/solve.go @@ -17,6 +17,7 @@ import ( "github.com/containerd/containerd/v2/core/content" "github.com/containerd/containerd/v2/core/images" contentlocal "github.com/containerd/containerd/v2/plugins/content/local" + cerrdefs "github.com/containerd/errdefs" "github.com/gofrs/flock" controlapi "github.com/moby/buildkit/api/services/control" "github.com/moby/buildkit/client/llb" @@ -425,9 +426,13 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG if err = json.Unmarshal([]byte(manifestDescJSON), &manifestDesc); err != nil { return nil, err } - for storePath, tag := range cacheOpt.storesToUpdate { + for storePath, store := range cacheOpt.storesToUpdate { idx := ociindex.NewStoreIndex(storePath) - if err := idx.Put(manifestDesc, ociindex.Tag(tag)); err != nil { + if err := idx.Put(manifestDesc, ociindex.Tag(store.tag)); err != nil { + if store.ignoreError { + bklog.G(callerCtx).WithError(err).WithField("cache_store_path", storePath).Warn("failed to update cache index, ignoring") + continue + } return nil, err } } @@ -605,11 +610,16 @@ type cacheStoreRef struct { store content.Store } +type cacheStoreUpdate struct { + tag string + ignoreError bool +} + type cacheOptions struct { options controlapi.CacheOptions contentStores map[string]content.Store // key: ID of content store ("local:" + csDir) - storesToUpdate map[string]string // key: path to content store, value: tag - storesToReset []cacheStoreRef // cache stores with reset=true + storesToUpdate map[string]cacheStoreUpdate + storesToReset []cacheStoreRef // cache stores with reset=true frontendAttrs map[string]string } @@ -620,7 +630,7 @@ func parseCacheOptions(ctx context.Context, isGateway bool, opt SolveOpt) (*cach ) var storesToReset []cacheStoreRef contentStores := make(map[string]content.Store) - storesToUpdate := make(map[string]string) + storesToUpdate := make(map[string]cacheStoreUpdate) frontendAttrs := make(map[string]string) for _, ex := range opt.CacheExports { if ex.Type == "local" { @@ -631,7 +641,7 @@ func parseCacheOptions(ctx context.Context, isGateway bool, opt SolveOpt) (*cach if err := os.MkdirAll(csDir, 0755); err != nil { return nil, err } - cs, err := contentlocal.NewStore(csDir) + cs, err := newLocalCacheStore(csDir) if err != nil { return nil, err } @@ -641,8 +651,14 @@ func parseCacheOptions(ctx context.Context, isGateway bool, opt SolveOpt) (*cach if t, ok := ex.Attrs["tag"]; ok { tag = t } + update := cacheStoreUpdate{tag: tag} + if v, ok := ex.Attrs["ignore-error"]; ok { + if b, err := strconv.ParseBool(v); err == nil { + update.ignoreError = b + } + } // TODO(AkihiroSuda): support custom index JSON path and tag - storesToUpdate[csDir] = tag + storesToUpdate[csDir] = update if v, ok := ex.Attrs["reset"]; ok { b, err := strconv.ParseBool(v) @@ -671,7 +687,7 @@ func parseCacheOptions(ctx context.Context, isGateway bool, opt SolveOpt) (*cach if csDir == "" { return nil, errors.New("local cache importer requires src") } - cs, err := contentlocal.NewStore(csDir) + cs, err := newLocalCacheStore(csDir) if err != nil { bklog.G(ctx).Warning("local cache import at " + csDir + " not found due to err: " + err.Error()) continue @@ -737,3 +753,92 @@ func parseCacheOptions(ctx context.Context, isGateway bool, opt SolveOpt) (*cach } return &res, nil } + +// localCacheStore coordinates ingests across clients sharing a cache directory. +// The underlying local store only locks references within one store instance. +type localCacheStore struct { + content.Store + dir string +} + +func newLocalCacheStore(dir string) (content.Store, error) { + store, err := contentlocal.NewStore(dir) + if err != nil { + return nil, err + } + return &localCacheStore{Store: store, dir: dir}, nil +} + +func (s *localCacheStore) lock(ref string) (*flock.Flock, error) { + dir := filepath.Join(s.dir, "ingest-locks") + if err := os.MkdirAll(dir, 0755); err != nil { + return nil, err + } + // Keep lock files after unlocking: deleting one could let another client + // lock a different inode while an existing waiter still uses the old one. + // This retains one empty file per ref, even after cache reset. The reset + // lock cannot protect their removal: not all users of this store hold it. + lock := flock.New(filepath.Join(dir, digest.FromString(ref).Encoded())) + locked, err := lock.TryLock() + if err != nil { + return nil, err + } + if !locked { + return nil, errors.Wrapf(cerrdefs.ErrUnavailable, "ingest ref %q is locked", ref) + } + return lock, nil +} + +func (s *localCacheStore) Writer(ctx context.Context, opts ...content.WriterOpt) (content.Writer, error) { + var wo content.WriterOpts + for _, opt := range opts { + if err := opt(&wo); err != nil { + return nil, err + } + } + if wo.Ref == "" { + return nil, errors.Wrap(cerrdefs.ErrInvalidArgument, "ref must not be empty") + } + lock, err := s.lock(wo.Ref) + if err != nil { + return nil, err + } + w, err := s.Store.Writer(ctx, content.WithRef(wo.Ref), content.WithDescriptor(wo.Desc)) + if err != nil { + lock.Close() + return nil, err + } + return &localCacheWriter{Writer: w, lock: lock}, nil +} + +func (s *localCacheStore) Abort(ctx context.Context, ref string) error { + lock, err := s.lock(ref) + if err != nil { + return err + } + defer lock.Close() + return s.Store.Abort(ctx, ref) +} + +type localCacheWriter struct { + content.Writer + lock *flock.Flock +} + +func (w *localCacheWriter) Commit(ctx context.Context, size int64, expected digest.Digest, opts ...content.Opt) error { + err := w.Writer.Commit(ctx, size, expected, opts...) + closeErr := w.Close() + if err != nil { + return err + } + return closeErr +} + +func (w *localCacheWriter) Close() error { + err := w.Writer.Close() + lockErr := w.lock.Close() + if err != nil { + return err + } + return lockErr +} diff --git a/vendor/github.com/moby/buildkit/client/status.go b/vendor/github.com/moby/buildkit/client/status.go index 92044e72a7..bff8f18114 100644 --- a/vendor/github.com/moby/buildkit/client/status.go +++ b/vendor/github.com/moby/buildkit/client/status.go @@ -5,15 +5,11 @@ import ( controlapi "github.com/moby/buildkit/api/services/control" digest "github.com/opencontainers/go-digest" + "google.golang.org/protobuf/encoding/protowire" "google.golang.org/protobuf/types/known/timestamppb" ) -var emptyLogVertexSize int - -func init() { - emptyLogVertex := controlapi.VertexLog{} - emptyLogVertexSize = emptyLogVertex.SizeVT() -} +const statusResponseSize = 1024 * 1024 func NewSolveStatus(resp *controlapi.StatusResponse) *SolveStatus { s := &SolveStatus{} @@ -63,68 +59,86 @@ func NewSolveStatus(resp *controlapi.StatusResponse) *SolveStatus { return s } +// Marshal batches status updates by encoded size without modifying SolveStatus. +// Logs may be split across responses. Individual metadata entries larger than +// the batch target are emitted intact in their own response. func (ss *SolveStatus) Marshal() (out []*controlapi.StatusResponse) { - logSize := 0 - for { - retry := false - sr := controlapi.StatusResponse{} - for _, v := range ss.Vertexes { - sr.Vertexes = append(sr.Vertexes, &controlapi.Vertex{ - Digest: string(v.Digest), - Inputs: digestSliceToPB(v.Inputs), - Name: v.Name, - Started: timestampToPB(v.Started), - Completed: timestampToPB(v.Completed), - Error: v.Error, - Cached: v.Cached, - ProgressGroup: v.ProgressGroup, - }) + sr := &controlapi.StatusResponse{} + size := 0 + + // Each repeated field has a one-byte tag and a length-prefixed message. + // Keep indivisible metadata entries intact even if they exceed the target. + reserve := func(n int) { + n = 1 + protowire.SizeBytes(n) + if size > 0 && size+n > statusResponseSize { + out = append(out, sr) + sr = &controlapi.StatusResponse{} + size = 0 } - for _, v := range ss.Statuses { - sr.Statuses = append(sr.Statuses, &controlapi.VertexStatus{ - ID: v.ID, - Vertex: string(v.Vertex), - Name: v.Name, - Current: v.Current, - Total: v.Total, - Timestamp: timestamppb.New(v.Timestamp), - Started: timestampToPB(v.Started), - Completed: timestampToPB(v.Completed), - }) + size += n + } + + for _, v := range ss.Vertexes { + p := &controlapi.Vertex{ + Digest: string(v.Digest), + Inputs: digestSliceToPB(v.Inputs), + Name: v.Name, + Started: timestampToPB(v.Started), + Completed: timestampToPB(v.Completed), + Error: v.Error, + Cached: v.Cached, + ProgressGroup: v.ProgressGroup, } - for i, v := range ss.Logs { - sr.Logs = append(sr.Logs, &controlapi.VertexLog{ - Vertex: string(v.Vertex), - Stream: int64(v.Stream), - Msg: v.Data, - Timestamp: timestamppb.New(v.Timestamp), - }) - logSize += len(v.Data) + emptyLogVertexSize - // avoid logs growing big and split apart if they do - if logSize > 1024*1024 { - ss.Vertexes = nil - ss.Statuses = nil - ss.Logs = ss.Logs[i+1:] - retry = true + reserve(p.SizeVT()) + sr.Vertexes = append(sr.Vertexes, p) + } + for _, v := range ss.Statuses { + p := &controlapi.VertexStatus{ + ID: v.ID, + Vertex: string(v.Vertex), + Name: v.Name, + Current: v.Current, + Total: v.Total, + Timestamp: timestamppb.New(v.Timestamp), + Started: timestampToPB(v.Started), + Completed: timestampToPB(v.Completed), + } + reserve(p.SizeVT()) + sr.Statuses = append(sr.Statuses, p) + } + for _, v := range ss.Logs { + p := &controlapi.VertexLog{ + Vertex: string(v.Vertex), + Stream: int64(v.Stream), + Timestamp: timestamppb.New(v.Timestamp), + } + const maxData = statusResponseSize / 2 + data := v.Data + for { + p.Msg = data[:min(len(data), maxData)] + reserve(p.SizeVT()) + sr.Logs = append(sr.Logs, p) + data = data[len(p.Msg):] + if len(data) == 0 { break } - } - for _, v := range ss.Warnings { - sr.Warnings = append(sr.Warnings, &controlapi.VertexWarning{ - Vertex: string(v.Vertex), - Level: int64(v.Level), - Short: v.Short, - Detail: v.Detail, - Info: v.SourceInfo, - Ranges: v.Range, - Url: v.URL, - }) - } - out = append(out, &sr) - if !retry { - break + p = &controlapi.VertexLog{Vertex: p.Vertex, Stream: p.Stream, Timestamp: p.Timestamp} } } + for _, v := range ss.Warnings { + p := &controlapi.VertexWarning{ + Vertex: string(v.Vertex), + Level: int64(v.Level), + Short: v.Short, + Detail: v.Detail, + Info: v.SourceInfo, + Ranges: v.Range, + Url: v.URL, + } + reserve(p.SizeVT()) + sr.Warnings = append(sr.Warnings, p) + } + out = append(out, sr) return } diff --git a/vendor/github.com/moby/buildkit/cmd/buildkitd/config/config.go b/vendor/github.com/moby/buildkit/cmd/buildkitd/config/config.go index e363b3e1a1..740d4b40dc 100644 --- a/vendor/github.com/moby/buildkit/cmd/buildkitd/config/config.go +++ b/vendor/github.com/moby/buildkit/cmd/buildkitd/config/config.go @@ -1,7 +1,7 @@ package config import ( - "github.com/moby/buildkit/cache/remotecache/gha/ghatypes" + "github.com/moby/buildkit/cache/remotecache/signing" "github.com/moby/buildkit/util/db/compaction" resolverconfig "github.com/moby/buildkit/util/resolver/config" ) @@ -105,7 +105,7 @@ func (c CompactionConfig) Policy() (compaction.Config, error) { } type CacheConfig struct { - GHA *ghatypes.CacheConfig `toml:"gha"` + GHA *signing.Config `toml:"gha"` } type SystemConfig struct { diff --git a/vendor/github.com/moby/buildkit/control/control.go b/vendor/github.com/moby/buildkit/control/control.go index 78adaadd8f..594c2c03b9 100644 --- a/vendor/github.com/moby/buildkit/control/control.go +++ b/vendor/github.com/moby/buildkit/control/control.go @@ -245,7 +245,8 @@ func (c *Controller) Prune(req *controlapi.PruneRequest, stream controlapi.Contr ch := make(chan client.UsageInfo, 32) - eg, ctx := errgroup.WithContext(stream.Context()) + ctx := stream.Context() + eg, ectx := errgroup.WithContext(ctx) workers, err := c.opt.WorkerController.List() if err != nil { return errors.Wrap(err, "failed to list workers for prune") @@ -267,7 +268,7 @@ func (c *Controller) Prune(req *controlapi.PruneRequest, stream controlapi.Contr for _, w := range workers { func(w worker.Worker) { eg.Go(func() error { - return w.Prune(ctx, ch, client.PruneInfo{ + return w.Prune(ectx, ch, client.PruneInfo{ Filter: req.Filter, All: req.All, KeepDuration: time.Duration(req.KeepDuration), diff --git a/vendor/github.com/moby/buildkit/executor/containerdexecutor/executor.go b/vendor/github.com/moby/buildkit/executor/containerdexecutor/executor.go index bbe585499c..e8ebc725d2 100644 --- a/vendor/github.com/moby/buildkit/executor/containerdexecutor/executor.go +++ b/vendor/github.com/moby/buildkit/executor/containerdexecutor/executor.go @@ -426,8 +426,7 @@ func (w *containerdExecutor) runProcess(ctx context.Context, p ctd.Process, resi if !ok { return // chan closed } - err = p.Resize(eventCtx, size.Cols, size.Rows) - if err != nil { + if err := p.Resize(eventCtx, size.Cols, size.Rows); err != nil { bklog.G(eventCtx).Warnf("Failed to resize %s: %s", p.ID(), err) } } @@ -442,8 +441,7 @@ func (w *containerdExecutor) runProcess(ctx context.Context, p ctd.Process, resi if !ok { return // chan closed } - err = p.Kill(eventCtx, sig) - if err != nil { + if err := p.Kill(eventCtx, sig); err != nil { bklog.G(eventCtx).Warnf("Failed to signal %s: %s", p.ID(), err) } } diff --git a/vendor/github.com/moby/buildkit/executor/oci/resolvconf.go b/vendor/github.com/moby/buildkit/executor/oci/resolvconf.go index 313f01909a..2d87056f75 100644 --- a/vendor/github.com/moby/buildkit/executor/oci/resolvconf.go +++ b/vendor/github.com/moby/buildkit/executor/oci/resolvconf.go @@ -4,6 +4,7 @@ import ( "context" "net/netip" "os" + "sync/atomic" "github.com/moby/buildkit/solver/pb" "github.com/moby/buildkit/util/bklog" @@ -23,9 +24,10 @@ const ( ) var ( - g flightcontrol.Group[struct{}] - notFirstRun bool - lastNotEmpty bool + g flightcontrol.Group[struct{}] + // accessed concurrently by calls with different keys + notFirstRun atomic.Bool + lastNotEmpty atomic.Bool ) // overridden by tests @@ -67,8 +69,7 @@ func GetResolvConf(ctx context.Context, root *os.Root, idmap *user.IdentityMappi } _, err := g.Do(ctx, root.Name()+"/"+name, func(ctx context.Context) (struct{}, error) { - generate := !notFirstRun - notFirstRun = true + generate := !notFirstRun.Swap(true) if !generate { fi, err := root.Stat(name) @@ -84,9 +85,8 @@ func GetResolvConf(ctx context.Context, root *os.Root, idmap *user.IdentityMappi if !errors.Is(err, os.ErrNotExist) { return struct{}{}, err } - if lastNotEmpty { + if lastNotEmpty.CompareAndSwap(true, false) { generate = true - lastNotEmpty = false } } else if fi.ModTime().Before(fiMain.ModTime()) { generate = true diff --git a/vendor/github.com/moby/buildkit/executor/runcexecutor/executor.go b/vendor/github.com/moby/buildkit/executor/runcexecutor/executor.go index c7fbf58c38..2542c01961 100644 --- a/vendor/github.com/moby/buildkit/executor/runcexecutor/executor.go +++ b/vendor/github.com/moby/buildkit/executor/runcexecutor/executor.go @@ -40,6 +40,8 @@ import ( "github.com/pkg/errors" ) +const runcStartTimeout = 15 * time.Second + type Opt struct { // root directory Root string @@ -748,7 +750,7 @@ func (p *procHandle) WaitForReady(ctx context.Context) error { // callback is non-nil it will be called after receiving the pid. func (p *procHandle) WaitForStart(ctx context.Context, startedCh <-chan int, started func()) error { ctx, cancel := context.WithCancelCause(ctx) - ctx, _ = context.WithTimeoutCause(ctx, 10*time.Second, errors.WithStack(context.DeadlineExceeded)) //nolint:govet + ctx, _ = context.WithTimeoutCause(ctx, runcStartTimeout, errors.WithStack(context.DeadlineExceeded)) //nolint:govet defer func() { cancel(errors.WithStack(context.Canceled)) }() select { case <-ctx.Done(): diff --git a/vendor/github.com/moby/buildkit/executor/runcexecutor/executor_linux.go b/vendor/github.com/moby/buildkit/executor/runcexecutor/executor_linux.go index f5097e59a5..d4a0e64a2b 100644 --- a/vendor/github.com/moby/buildkit/executor/runcexecutor/executor_linux.go +++ b/vendor/github.com/moby/buildkit/executor/runcexecutor/executor_linux.go @@ -13,6 +13,7 @@ import ( "github.com/containerd/console" runc "github.com/containerd/go-runc" "github.com/moby/buildkit/executor" + "github.com/moby/buildkit/executor/oci" gatewayapi "github.com/moby/buildkit/frontend/gateway/pb" "github.com/moby/buildkit/util/bklog" "github.com/moby/sys/signal" @@ -34,10 +35,14 @@ func (w *runcExecutor) run(ctx context.Context, id, bundle string, process execu extraArgs = append(extraArgs, "--keep") } _, err := w.runc.Run(ctx, id, bundle, &runc.CreateOpts{ - NoPivot: w.noPivot, - Started: started, - IO: io, - ExtraArgs: extraArgs, + NoPivot: w.noPivot, + // Without a PID namespace, processes left behind by a step outlive the + // container and keep its session keyring, until the per-user key quota + // is exhausted and every new container fails to start. + NoNewKeyring: w.processMode == oci.NoProcessSandbox, + Started: started, + IO: io, + ExtraArgs: extraArgs, }) return err }) @@ -164,7 +169,9 @@ func (w *runcExecutor) callWithIO(ctx context.Context, process executor.ProcessI }) } + resizeDone := make(chan struct{}) eg.Go(func() error { + defer close(resizeDone) err := runcProcess.WaitForReady(ctx) if err != nil { return err @@ -190,6 +197,12 @@ func (w *runcExecutor) callWithIO(ctx context.Context, process executor.ProcessI } } }) + // Runs before the cleanup defer above: the resize loop uses ptm's fd, so + // stop it before ptm is closed. + defer func() { + runcProcess.Shutdown() + <-resizeDone + }() runcIO := &forwardIO{} if process.Stdin != nil { diff --git a/vendor/github.com/moby/buildkit/frontend/dockerfile/builder/build.go b/vendor/github.com/moby/buildkit/frontend/dockerfile/builder/build.go index 1804c69abd..874ecce978 100644 --- a/vendor/github.com/moby/buildkit/frontend/dockerfile/builder/build.go +++ b/vendor/github.com/moby/buildkit/frontend/dockerfile/builder/build.go @@ -35,7 +35,7 @@ const ( ) func Build(ctx context.Context, c client.Client) (_ *client.Result, err error) { - c = &withResolveCache{Client: c} + c = newWithResolveCache(c) bc, err := dockerui.NewClient(c) if err != nil { return nil, err diff --git a/vendor/github.com/moby/buildkit/frontend/dockerfile/builder/resolvecache.go b/vendor/github.com/moby/buildkit/frontend/dockerfile/builder/resolvecache.go index 4d0b27802b..55d31e7a94 100644 --- a/vendor/github.com/moby/buildkit/frontend/dockerfile/builder/resolvecache.go +++ b/vendor/github.com/moby/buildkit/frontend/dockerfile/builder/resolvecache.go @@ -26,8 +26,14 @@ type resolveResult struct { var _ client.Client = &withResolveCache{} +func newWithResolveCache(c client.Client) *withResolveCache { + return &withResolveCache{ + Client: c, + g: flightcontrol.CachedGroup[*resolveResult]{CacheError: true}, + } +} + func (c *withResolveCache) ResolveImageConfig(ctx context.Context, ref string, opt sourceresolver.Opt) (string, digest.Digest, []byte, error) { - c.g.CacheError = true optHash, err := hashstructure.Hash(opt, nil) if err != nil { return "", "", nil, err diff --git a/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert.go b/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert.go index 4c9f0a4c8f..12b83f364b 100644 --- a/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert.go +++ b/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert.go @@ -20,6 +20,7 @@ import ( "github.com/moby/buildkit/client/llb" "github.com/moby/buildkit/client/llb/imagemetaresolver" "github.com/moby/buildkit/client/llb/sourceresolver" + dfcommand "github.com/moby/buildkit/frontend/dockerfile/command" "github.com/moby/buildkit/frontend/dockerfile/instructions" "github.com/moby/buildkit/frontend/dockerfile/linter" "github.com/moby/buildkit/frontend/dockerfile/parser" @@ -47,6 +48,7 @@ const ( sbomScanContext = "BUILDKIT_SBOM_SCAN_CONTEXT" sbomScanStage = "BUILDKIT_SBOM_SCAN_STAGE" + disableOnbuild = "BUILDKIT_DISABLE_ONBUILD" ) var nonEnvArgs = map[string]struct{}{ @@ -216,6 +218,7 @@ type dispatchContext struct { allDispatchStates *dispatchStates proxyEnv *llb.ProxyEnv namedContext func(string, dockerui.ContextOpt) (*dockerui.NamedContext, error) + disableOnbuild bool } func namedContextFunc(opt ConvertOpt) func(string, dockerui.ContextOpt) (*dockerui.NamedContext, error) { @@ -310,6 +313,14 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS globalArgs = setBuildArgValue(opt.BuildArgs, globalArgs, "SOURCE_DATE_EPOCH", formatSourceDateEpochValue(resolvedEpoch)) } + var skipOnbuild bool + if v, ok := getBuildArgValue(opt.BuildArgs, globalArgs, disableOnbuild); ok && v != "" { + skipOnbuild, err = strconv.ParseBool(v) + if err != nil { + return nil, errors.Wrapf(err, "invalid %s value %q", disableOnbuild, v) + } + } + metaResolver := opt.MetaResolver if metaResolver == nil { metaResolver = imagemetaresolver.Default() @@ -327,6 +338,7 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS allDispatchStates: newDispatchStates(), proxyEnv: proxyEnvFromBuildArgs(opt.BuildArgs), namedContext: namedContextFunc(opt), + disableOnbuild: skipOnbuild, } if err := dctx.buildDispatchStates(stages); err != nil { @@ -568,6 +580,10 @@ func (dctx *dispatchContext) resolveStages(ctx context.Context, target *dispatch } d.onBuildInit = true } + if dctx.disableOnbuild || slices.Contains(d.stage.Reset, dfcommand.Onbuild) { + onbuilds = nil + d.image.Config.OnBuild = nil + } if len(onbuilds) > 0 { if b, err := initOnBuildTriggers(d, onbuilds, dctx.allDispatchStates, dctx.shlex); err != nil { @@ -789,6 +805,8 @@ func (dctx *dispatchContext) dispatchStages(ctx context.Context, allReachable ma d.platform = &d.opt.targetPlatform } + resetBaseConfig(d) + // make sure that PATH is always set if _, ok := shell.EnvsFromSlice(d.image.Config.Env).Get("PATH"); !ok { var osName string @@ -834,6 +852,7 @@ func (dctx *dispatchContext) dispatchStages(ctx context.Context, allReachable ma targetPlatform: dctx.platformOpt.targetPlatform, extraHosts: dctx.opt.ExtraHosts, gitAdvice: dctx.opt.GitAdvice, + gitDebugCommands: dctx.opt.GitDebugCommands, shmSize: dctx.opt.ShmSize, ulimit: dctx.opt.Ulimits, devices: dctx.opt.Devices, @@ -991,6 +1010,7 @@ type dispatchOpt struct { buildPlatforms []ocispecs.Platform extraHosts []llb.HostIP gitAdvice bool + gitDebugCommands bool shmSize int64 ulimit []*pb.Ulimit devices []*pb.CDIDevice @@ -1329,6 +1349,17 @@ func initOnBuildTriggers(d *dispatchState, triggers []string, allDispatchStates } func dispatchEnv(d *dispatchState, c *instructions.EnvCommand, lint *linter.Linter) error { + if c.Unset { + keys := make([]string, 0, len(c.Env)) + for _, e := range c.Env { + if e.Key == "" { + return errors.New("ENV --unset names can not be blank") + } + unsetEnv(d, e.Key) + keys = append(keys, e.Key) + } + return commitToHistory(&d.image, "ENV --unset "+strings.Join(keys, " "), false, nil, d.epoch) + } commitMessage := bytes.NewBufferString("ENV") for _, e := range c.Env { if e.NoDelim { @@ -1343,6 +1374,61 @@ func dispatchEnv(d *dispatchState, c *instructions.EnvCommand, lint *linter.Lint return commitToHistory(&d.image, commitMessage.String(), false, nil, d.epoch) } +// unsetEnv removes an environment variable and a build arg with the same name. +func unsetEnv(d *dispatchState, key string) { + for _, k := range getEnv(d.state).Keys() { + if shell.EqualEnvKeys(k, key) { + d.state = d.state.DelEnv(k) + } + } + d.image.Config.Env = delEnv(d.image.Config.Env, key) + d.buildArgs = slices.DeleteFunc(d.buildArgs, func(arg instructions.KeyValuePairOptional) bool { + return shell.EqualEnvKeys(arg.Key, key) + }) +} + +// resetBaseConfig clears the base image config fields set with FROM --reset. +// ONBUILD triggers are skipped when the stages are resolved. +func resetBaseConfig(d *dispatchState) { + for _, k := range d.stage.Reset { + switch k { + case dfcommand.Cmd: + d.image.Config.Cmd = nil + case dfcommand.Entrypoint: + d.image.Config.Entrypoint = nil + case dfcommand.Env: + for _, env := range slices.Clone(d.image.Config.Env) { + key, _ := parseKeyValue(env) + unsetEnv(d, key) + } + case dfcommand.Expose: + d.image.Config.ExposedPorts = nil + case dfcommand.Healthcheck: + d.image.Config.Healthcheck = nil + case dfcommand.Label: + d.image.Config.Labels = nil + case dfcommand.Shell: + d.image.Config.Shell = nil + case dfcommand.StopSignal: + d.image.Config.StopSignal = "" + case dfcommand.User: + d.image.Config.User = "" + d.state = d.state.User("") + case dfcommand.Volume: + d.image.Config.Volumes = nil + case dfcommand.Workdir: + d.image.Config.WorkingDir = "" + d.state = d.state.Dir("/") + // workdir no longer depends on the base image + d.workdirSet = true + } + } + // ArgsEscaped applies to both cmd and entrypoint + if len(d.stage.Reset) > 0 && len(d.image.Config.Cmd) == 0 && len(d.image.Config.Entrypoint) == 0 { + d.image.Config.ArgsEscaped = false //nolint:staticcheck // ignore SA1019: field is deprecated in OCI Image spec, but used for backward-compatibility with Docker image spec. + } +} + func dispatchRun(d *dispatchState, c *instructions.RunCommand, proxy *llb.ProxyEnv, sources []*dispatchState, dopt dispatchOpt) error { var opt []llb.RunOption @@ -1573,6 +1659,17 @@ func dispatchMaintainer(d *dispatchState, c *instructions.MaintainerCommand) err } func dispatchLabel(d *dispatchState, c *instructions.LabelCommand, lint *linter.Linter) error { + if c.Unset { + keys := make([]string, 0, len(c.Labels)) + for _, v := range c.Labels { + if v.Key == "" { + return errors.New("LABEL --unset names can not be blank") + } + delete(d.image.Config.Labels, v.Key) + keys = append(keys, v.Key) + } + return commitToHistory(&d.image, "LABEL --unset "+strings.Join(keys, " "), false, nil, d.epoch) + } commitMessage := bytes.NewBufferString("LABEL") if d.image.Config.Labels == nil { d.image.Config.Labels = make(map[string]string, len(c.Labels)) @@ -1650,13 +1747,24 @@ func dispatchUser(d *dispatchState, c *instructions.UserCommand, commit bool) er } func dispatchVolume(d *dispatchState, c *instructions.VolumeCommand) error { + if slices.Contains(c.Volumes, "") { + return errors.New("VOLUME specified can not be an empty string") + } + if c.Unset { + for _, v := range c.Volumes { + v = path.Clean(v) + for k := range d.image.Config.Volumes { + if path.Clean(k) == v { + delete(d.image.Config.Volumes, k) + } + } + } + return commitToHistory(&d.image, fmt.Sprintf("VOLUME --unset %v", c.Volumes), false, nil, d.epoch) + } if d.image.Config.Volumes == nil { d.image.Config.Volumes = map[string]struct{}{} } for _, v := range c.Volumes { - if v == "" { - return errors.New("VOLUME specified can not be an empty string") - } d.image.Config.Volumes[v] = struct{}{} } return commitToHistory(&d.image, fmt.Sprintf("VOLUME %v", c.Volumes), false, nil, d.epoch) @@ -1768,6 +1876,13 @@ func addEnv(env []string, k, v string) []string { return env } +func delEnv(env []string, k string) []string { + return slices.DeleteFunc(env, func(envVar string) bool { + key, _ := parseKeyValue(envVar) + return shell.EqualEnvKeys(key, k) + }) +} + func parseKeyValue(env string) (string, string) { parts := strings.SplitN(env, "=", 2) v := "" diff --git a/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert_copy.go b/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert_copy.go index f98c49f0da..f13b6b2886 100644 --- a/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert_copy.go +++ b/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert_copy.go @@ -145,6 +145,9 @@ func dispatchCopy(d *dispatchState, cfg copyConfig) error { if cfg.opt.gitAdvice { gitOptions = append(gitOptions, llb.GitAdvice(true)) } + if cfg.opt.gitDebugCommands { + gitOptions = append(gitOptions, llb.GitDebugCommands(true)) + } if cfg.keepGitDir != nil && gitRef.KeepGitDir != nil { if *cfg.keepGitDir != *gitRef.KeepGitDir { return errors.New("inconsistent keep-git-dir configuration") diff --git a/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert_expose.go b/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert_expose.go index 87beabbeef..943d28c6fd 100644 --- a/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert_expose.go +++ b/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert_expose.go @@ -41,7 +41,7 @@ func dispatchExpose(d *dispatchState, c *instructions.ExposeCommand, opt *dispat d.image.Config.ExposedPorts[p] = struct{}{} } - return commitToHistory(&d.image, fmt.Sprintf("EXPOSE %v", psp), false, nil, d.epoch) + return commitToHistory(&d.image, fmt.Sprintf("EXPOSE %s", strings.Join(psp, " ")), false, nil, d.epoch) } type portSpecs struct { diff --git a/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/epoch.go b/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/epoch.go index 07fd63dc16..978b0b6c95 100644 --- a/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/epoch.go +++ b/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/epoch.go @@ -98,7 +98,7 @@ func resolveSourceDateEpochState(ctx context.Context, value string, opt ConvertO args = &updated } - sourceState, err := sourceDateEpochStageSource(stages[i], opt.BuildArgs, args, shlex, opt.GitAdvice) + sourceState, err := sourceDateEpochStageSource(stages[i], opt.BuildArgs, args, shlex, opt.GitAdvice, opt.GitDebugCommands) if err != nil { return nil, sourceDateEpochStateOpt{}, parser.WithLocation(err, stages[i].Location) } @@ -110,7 +110,7 @@ func resolveSourceDateEpochState(ctx context.Context, value string, opt ConvertO return nil, sourceDateEpochStateOpt{}, errors.Errorf("invalid SOURCE_DATE_EPOCH: %s", value) } -func sourceDateEpochStageSource(stage instructions.Stage, buildArgs map[string]string, globalArgs *llb.EnvList, shlex *shell.Lex, gitAdvice bool) (*llb.State, error) { +func sourceDateEpochStageSource(stage instructions.Stage, buildArgs map[string]string, globalArgs *llb.EnvList, shlex *shell.Lex, gitAdvice, gitDebugCommands bool) (*llb.State, error) { stageBaseName, _, err := shlex.ProcessWord(stage.BaseName, globalArgs) if err != nil { return nil, errors.Wrapf(err, "failed to process source stage base name %q", stage.BaseName) @@ -133,7 +133,7 @@ func sourceDateEpochStageSource(stage instructions.Stage, buildArgs map[string]s if sourceState != nil { return nil, errors.New("SOURCE_DATE_EPOCH stage must contain exactly one remote ADD") } - sourceState, err = sourceDateEpochAddSource(c, env, shlex, gitAdvice) + sourceState, err = sourceDateEpochAddSource(c, env, shlex, gitAdvice, gitDebugCommands) if err != nil { return nil, err } @@ -167,7 +167,7 @@ func applySourceDateEpochStageArgs(args []instructions.KeyValuePairOptional, env return env, nil } -func sourceDateEpochAddSource(cmd *instructions.AddCommand, env *llb.EnvList, shlex *shell.Lex, gitAdvice bool) (*llb.State, error) { +func sourceDateEpochAddSource(cmd *instructions.AddCommand, env *llb.EnvList, shlex *shell.Lex, gitAdvice, gitDebugCommands bool) (*llb.State, error) { if len(cmd.SourceContents) != 0 || len(cmd.SourcePaths) != 1 { return nil, errors.New("SOURCE_DATE_EPOCH stage must contain exactly one remote ADD source") } @@ -204,6 +204,9 @@ func sourceDateEpochAddSource(cmd *instructions.AddCommand, env *llb.EnvList, sh if gitAdvice { gitOptions = append(gitOptions, llb.GitAdvice(true)) } + if gitDebugCommands { + gitOptions = append(gitOptions, llb.GitDebugCommands(true)) + } if cmd.KeepGitDir != nil && *cmd.KeepGitDir { gitOptions = append(gitOptions, llb.KeepGitDir()) } diff --git a/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/validations.go b/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/validations.go index 9a3523a7ef..29ec1dc4db 100644 --- a/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/validations.go +++ b/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/validations.go @@ -311,6 +311,7 @@ func getSecretsRegex() (*regexp.Regexp, *regexp.Regexp) { "password", "pword", "passwd", + "psw", "secret", "token", } diff --git a/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/commands.go b/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/commands.go index e7fbcf7a67..d3d390e7f9 100644 --- a/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/commands.go +++ b/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/commands.go @@ -148,9 +148,11 @@ func expandSliceInPlace(values []string, expander SingleWordExpander) error { // EnvCommand allows setting an variable in the container's environment. // // ENV key1 value1 [keyN valueN...] +// ENV --unset key1 [keyN...] type EnvCommand struct { withNameAndCode - Env KeyValuePairs + Env KeyValuePairs + Unset bool // Env keys are removed, values are empty } func (c *EnvCommand) Expand(expander SingleWordExpander) error { @@ -185,9 +187,11 @@ func NewLabelCommand(k string, v string, noExp bool) *LabelCommand { // LabelCommand sets an image label in the output // // LABEL some json data describing the image +// LABEL --unset key1 [keyN...] type LabelCommand struct { withNameAndCode Labels KeyValuePairs + Unset bool // Labels keys are removed, values are empty noExpand bool } @@ -436,9 +440,11 @@ func (c *UserCommand) Expand(expander SingleWordExpander) error { // VolumeCommand exposes the specified volume for use in the build environment. // // VOLUME /foo +// VOLUME --unset /foo type VolumeCommand struct { withNameAndCode Volumes []string + Unset bool // Volumes are removed } func (c *VolumeCommand) Expand(expander SingleWordExpander) error { @@ -521,6 +527,7 @@ type Stage struct { OrigCmd string // original FROM command, used for rule checks BaseName string // name of the base stage or source Platform string // platform of base source to use + Reset []string // base image config fields to reset, see ResetKeys DocComment string // doc-comment directly above the stage diff --git a/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/parse.go b/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/parse.go index b5e4a62f53..635e0288e7 100644 --- a/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/parse.go +++ b/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/parse.go @@ -36,6 +36,31 @@ var ( parseRunPostHooks []func(*RunCommand, parseRequest) error ) +var ( + unsetEnabled = false + resetEnabled = false +) + +// ResetAll is the FROM --reset value that resets all ResetKeys. +const ResetAll = "all" + +// ResetKeys are the base image config fields that can be reset with +// FROM --reset. Each field is named after the command that sets it. +var ResetKeys = []string{ + command.Cmd, + command.Entrypoint, + command.Env, + command.Expose, + command.Healthcheck, + command.Label, + command.Onbuild, + command.Shell, + command.StopSignal, + command.User, + command.Volume, + command.Workdir, +} + func nodeArgs(node *parser.Node) []string { result := []string{} for ; node.Next != nil; node = node.Next { @@ -230,10 +255,48 @@ func parseKvps(args []string, cmdName string) (KeyValuePairs, error) { return res, nil } +// addUnsetFlag adds the --unset flag if it is enabled in this build. +func addUnsetFlag(req parseRequest) *Flag { + if !unsetEnabled { + return nil + } + return req.flags.AddBool("unset", false) +} + +// parseUnsetKeys returns the keys for the --unset form of a command. +func parseUnsetKeys(args []string, cmdName string) (KeyValuePairs, error) { + if len(args) == 0 { + return nil, errAtLeastOneArgument(cmdName + " --unset") + } + var res KeyValuePairs + for _, name := range args { + if name == "" { + return nil, errBlankCommandNames(cmdName) + } + if strings.Contains(name, "=") { + return nil, errors.Errorf("%s --unset does not accept values: %s", cmdName, name) + } + res = append(res, KeyValuePair{Key: name}) + } + return res, nil +} + func parseEnv(req parseRequest) (*EnvCommand, error) { + flUnset := addUnsetFlag(req) if err := req.flags.Parse(); err != nil { return nil, err } + if flUnset != nil && flUnset.Value == "true" { + envs, err := parseUnsetKeys(req.args, "ENV") + if err != nil { + return nil, err + } + return &EnvCommand{ + Env: envs, + Unset: true, + withNameAndCode: newWithNameAndCode(req), + }, nil + } envs, err := parseKvps(req.args, "ENV") if err != nil { return nil, err @@ -259,9 +322,21 @@ func parseMaintainer(req parseRequest) (*MaintainerCommand, error) { } func parseLabel(req parseRequest) (*LabelCommand, error) { + flUnset := addUnsetFlag(req) if err := req.flags.Parse(); err != nil { return nil, err } + if flUnset != nil && flUnset.Value == "true" { + labels, err := parseUnsetKeys(req.args, "LABEL") + if err != nil { + return nil, err + } + return &LabelCommand{ + Labels: labels, + Unset: true, + withNameAndCode: newWithNameAndCode(req), + }, nil + } labels, err := parseKvps(req.args, "LABEL") if err != nil { @@ -399,10 +474,22 @@ func parseFrom(req parseRequest) (*Stage, error) { } flPlatform := req.flags.AddString("platform", "") + var flReset *Flag + if resetEnabled { + flReset = req.flags.AddStrings("reset") + } if err := req.flags.Parse(); err != nil { return nil, err } + var reset []string + if flReset != nil { + reset, err = parseResetKeys(flReset.StringValues) + if err != nil { + return nil, err + } + } + code := strings.TrimSpace(req.original) return &Stage{ BaseName: req.args[0], @@ -411,12 +498,35 @@ func parseFrom(req parseRequest) (*Stage, error) { SourceCode: code, Commands: []Command{}, Platform: flPlatform.Value, + Reset: reset, Location: req.location, Comments: req.comments, DocComment: getDocComment(req.comments, stageName), }, nil } +// parseResetKeys returns the sorted list of fields for FROM --reset values. +func parseResetKeys(values []string) ([]string, error) { + var keys []string + for _, v := range values { + for k := range strings.SplitSeq(v, ",") { + switch { + case k == ResetAll: + keys = append(keys, ResetKeys...) + case slices.Contains(ResetKeys, k): + keys = append(keys, k) + case strings.ToLower(k) == ResetAll || slices.Contains(ResetKeys, strings.ToLower(k)): + return nil, errors.Errorf("invalid FROM --reset value: %q (did you mean %s?)", k, strings.ToLower(k)) + default: + err := errors.Errorf("invalid FROM --reset value: %q", k) + return nil, suggest.WrapError(err, k, append([]string{ResetAll}, ResetKeys...), true) + } + } + } + slices.Sort(keys) + return slices.Compact(keys), nil +} + var validStageName = regexp.MustCompile("^[a-z][a-z0-9-_.]*$") func parseBuildStageName(args []string) (stageName string, err error) { @@ -714,11 +824,13 @@ func parseVolume(req parseRequest) (*VolumeCommand, error) { return nil, errAtLeastOneArgument("VOLUME") } + flUnset := addUnsetFlag(req) if err := req.flags.Parse(); err != nil { return nil, err } cmd := &VolumeCommand{ + Unset: flUnset != nil && flUnset.Value == "true", withNameAndCode: newWithNameAndCode(req), } @@ -736,6 +848,9 @@ func parseStopSignal(req parseRequest) (*StopSignalCommand, error) { if len(req.args) != 1 { return nil, errExactlyOneArgument("STOPSIGNAL") } + if err := req.flags.Parse(); err != nil { + return nil, err + } sig := req.args[0] cmd := &StopSignalCommand{ @@ -750,6 +865,10 @@ func parseArg(req parseRequest) (*ArgCommand, error) { return nil, errAtLeastOneArgument("ARG") } + if err := req.flags.Parse(); err != nil { + return nil, err + } + pairs := make([]KeyValuePairOptional, len(req.args)) for i, arg := range req.args { diff --git a/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/parse_reset.go b/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/parse_reset.go new file mode 100644 index 0000000000..ca9a867efd --- /dev/null +++ b/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/parse_reset.go @@ -0,0 +1,7 @@ +//go:build dfreset + +package instructions + +func init() { + resetEnabled = true +} diff --git a/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/parse_unset.go b/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/parse_unset.go new file mode 100644 index 0000000000..e9944b188a --- /dev/null +++ b/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/parse_unset.go @@ -0,0 +1,7 @@ +//go:build dfunset + +package instructions + +func init() { + unsetEnabled = true +} diff --git a/vendor/github.com/moby/buildkit/frontend/dockerfile/parser/parser.go b/vendor/github.com/moby/buildkit/frontend/dockerfile/parser/parser.go index d07df3412c..fca295bfa9 100644 --- a/vendor/github.com/moby/buildkit/frontend/dockerfile/parser/parser.go +++ b/vendor/github.com/moby/buildkit/frontend/dockerfile/parser/parser.go @@ -137,6 +137,13 @@ var ( heredocCompoundDirectives = map[string]bool{ command.Onbuild: true, } + + // Directives that parse key-value pairs but take a list of names + // when --unset flag is set + unsetCommands = map[string]bool{ + command.Env: true, + command.Label: true, + } ) // directives is the structure used during a build run to hold the state of @@ -234,6 +241,10 @@ func newNodeFromLine(line string, d *directives, comments []string) (*Node, erro if fn == nil { fn = parseIgnore } + // --unset form takes a list of names instead of key-value pairs + if unsetCommands[strings.ToLower(cmd)] && hasUnsetFlag(flags) { + fn = parseNameOrNameVal + } next, attrs, err := fn(args, d) if err != nil { return nil, err @@ -249,6 +260,17 @@ func newNodeFromLine(line string, d *directives, comments []string) (*Node, erro }, nil } +func hasUnsetFlag(flags []string) bool { + for _, f := range flags { + name, value, _ := strings.Cut(f, "=") + if name == "--unset" { + // invalid values are reported by the flag parser + return !strings.EqualFold(value, "false") + } + } + return false +} + // Result contains the bundled outputs from parsing a Dockerfile. type Result struct { AST *Node diff --git a/vendor/github.com/moby/buildkit/frontend/dockerfile/version/version.go b/vendor/github.com/moby/buildkit/frontend/dockerfile/version/version.go index 802f78369b..5df5f124da 100644 --- a/vendor/github.com/moby/buildkit/frontend/dockerfile/version/version.go +++ b/vendor/github.com/moby/buildkit/frontend/dockerfile/version/version.go @@ -1,3 +1,3 @@ package version -const version = "1.28" +const version = "1.29" diff --git a/vendor/github.com/moby/buildkit/frontend/dockerui/config.go b/vendor/github.com/moby/buildkit/frontend/dockerui/config.go index 09f8e51a4b..7f7ddd8f42 100644 --- a/vendor/github.com/moby/buildkit/frontend/dockerui/config.go +++ b/vendor/github.com/moby/buildkit/frontend/dockerui/config.go @@ -58,6 +58,7 @@ const ( keyDockerfileLintArg = "build-arg:BUILDKIT_DOCKERFILE_CHECK" keyContextKeepGitDirArg = "build-arg:BUILDKIT_CONTEXT_KEEP_GIT_DIR" keyGitAdviceArg = "build-arg:BUILDKIT_GIT_ADVICE" + keyGitDebugCommandsArg = "build-arg:BUILDKIT_GIT_DEBUG_COMMANDS" ) type Config struct { @@ -70,6 +71,7 @@ type Config struct { Labels map[string]string NetworkMode pb.NetMode GitAdvice bool + GitDebugCommands bool ShmSize int64 Target string Ulimits []*pb.Ulimit @@ -319,6 +321,12 @@ func (bc *Client) init() error { return errors.Wrapf(err, "failed to parse %s", keyGitAdviceArg) } } + if v := opts[keyGitDebugCommandsArg]; v != "" { + bc.GitDebugCommands, err = strconv.ParseBool(v) + if err != nil { + return errors.Wrapf(err, "failed to parse %s", keyGitDebugCommandsArg) + } + } bc.localsSessionIDs = parseLocalSessionIDs(opts) diff --git a/vendor/github.com/moby/buildkit/frontend/dockerui/context.go b/vendor/github.com/moby/buildkit/frontend/dockerui/context.go index 8842c22dba..91127a712f 100644 --- a/vendor/github.com/moby/buildkit/frontend/dockerui/context.go +++ b/vendor/github.com/moby/buildkit/frontend/dockerui/context.go @@ -91,7 +91,7 @@ func (bc *Client) initContext(ctx context.Context) (*buildContext, error) { if opts[buildArgPrefix+"SOURCE_DATE_EPOCH"] != "" { extraGitOpts = append(extraGitOpts, llb.GitMTimeCommit()) } - extraGitOpts = append(extraGitOpts, gitAdviceOpts(bc.GitAdvice)...) + extraGitOpts = append(extraGitOpts, gitSourceOpts(bc.GitAdvice, bc.GitDebugCommands)...) if st, ok, err := DetectGitContext(opts[localNameContext], keepGit, extraGitOpts...); ok { if err != nil { return nil, err @@ -325,11 +325,15 @@ func DetectGitContext(ref string, keepGit *bool, opts ...llb.GitOption) (*llb.St return &st, true, nil } -func gitAdviceOpts(enabled bool) []llb.GitOption { - if enabled { - return []llb.GitOption{llb.GitAdvice(true)} +func gitSourceOpts(advice, debugCommands bool) []llb.GitOption { + var opts []llb.GitOption + if advice { + opts = append(opts, llb.GitAdvice(true)) } - return nil + if debugCommands { + opts = append(opts, llb.GitDebugCommands(true)) + } + return opts } func DetectHTTPContext(ref string) (*llb.State, string, bool) { diff --git a/vendor/github.com/moby/buildkit/frontend/dockerui/namedcontext.go b/vendor/github.com/moby/buildkit/frontend/dockerui/namedcontext.go index b125761880..d858e32fc4 100644 --- a/vendor/github.com/moby/buildkit/frontend/dockerui/namedcontext.go +++ b/vendor/github.com/moby/buildkit/frontend/dockerui/namedcontext.go @@ -141,7 +141,7 @@ func (nc *NamedContext) load(ctx context.Context, count int) (*llb.State, *docke } return &st, &img, nil case "git": - st, ok, err := DetectGitContext(nc.input, nil, gitAdviceOpts(nc.bc.GitAdvice)...) + st, ok, err := DetectGitContext(nc.input, nil, gitSourceOpts(nc.bc.GitAdvice, nc.bc.GitDebugCommands)...) if !ok { return nil, nil, errors.Errorf("invalid git context %s", nc.input) } @@ -150,7 +150,7 @@ func (nc *NamedContext) load(ctx context.Context, count int) (*llb.State, *docke } return st, nil, nil case "http", "https": - st, ok, err := DetectGitContext(nc.input, nil, gitAdviceOpts(nc.bc.GitAdvice)...) + st, ok, err := DetectGitContext(nc.input, nil, gitSourceOpts(nc.bc.GitAdvice, nc.bc.GitDebugCommands)...) if ok { if err != nil { return nil, nil, err diff --git a/vendor/github.com/moby/buildkit/frontend/gateway/forwarder/forward.go b/vendor/github.com/moby/buildkit/frontend/gateway/forwarder/forward.go index 2c40846ccc..a04671e9f9 100644 --- a/vendor/github.com/moby/buildkit/frontend/gateway/forwarder/forward.go +++ b/vendor/github.com/moby/buildkit/frontend/gateway/forwarder/forward.go @@ -68,6 +68,7 @@ type BridgeClient struct { workers worker.Infos resultByID map[string]solver.Result discarded bool + execErrors []*llberrdefs.ExecError buildOpts client.BuildOpts ctrs []client.Container executor executor.Executor @@ -159,6 +160,13 @@ func (c *BridgeClient) wrapSolveError(solveErr error) error { subject errdefs.IsSolve_Subject ) if errors.As(solveErr, &ee) { + c.mu.Lock() + if c.discarded { + ee.Release() + } else if !slices.Contains(c.execErrors, ee) { + c.execErrors = append(c.execErrors, ee) + } + c.mu.Unlock() var err error inputIDs, err = c.registerResultIDs(ee.Inputs...) if err != nil { @@ -174,7 +182,7 @@ func (c *BridgeClient) wrapSolveError(solveErr error) error { } if errors.As(solveErr, &sce) { var err error - inputIDs, err = c.registerSlowCacheResult(sce.Result) + inputIDs, err = c.registerResultIDs(sce.Result) if err != nil { return err } @@ -183,32 +191,14 @@ func (c *BridgeClient) wrapSolveError(solveErr error) error { return errdefs.WithSolveError(solveErr, subject, inputIDs, mountIDs) } +// registerResultIDs retains gateway-owned clones of error refs until discard. func (c *BridgeClient) registerResultIDs(results ...solver.Result) (ids []string, err error) { c.mu.Lock() defer c.mu.Unlock() - return c.registerResultIDsLocked(results...) -} - -// registerSlowCacheResult registers a clone of a result borrowed from the -// solver job. The clone is only taken before the client is discarded, as the -// job may be released after that. -func (c *BridgeClient) registerSlowCacheResult(res solver.Result) ([]string, error) { - c.mu.Lock() - defer c.mu.Unlock() - if c.discarded { return nil, nil } - // Validate before cloning so a type error can't leak the clone. Once - // cloned, registerResultIDsLocked either keeps or releases it. - if _, ok := res.Sys().(*worker.WorkerRef); !ok { - return nil, errors.Errorf("unexpected type for result, got %T", res.Sys()) - } - return c.registerResultIDsLocked(res.Clone()) -} - -func (c *BridgeClient) registerResultIDsLocked(results ...solver.Result) (ids []string, err error) { ids = make([]string, len(results)) for i, res := range results { if res == nil { @@ -220,16 +210,11 @@ func (c *BridgeClient) registerResultIDsLocked(results ...solver.Result) (ids [] } id := workerRef.ID() ids[i] = id - if existing, ok := c.resultByID[id]; ok { - if existing != res { - if err := res.Release(context.TODO()); err != nil { - return ids, errors.WithStack(err) - } - } - continue + if _, ok := c.resultByID[id]; !ok { + // Cached sharedOp.Exec errors have snapshots pinned by the active + // solver state, even after the original error refs are released. + c.resultByID[id] = res.Clone() } - // Keep the result wrapper so Release balances split-result ownership. - c.resultByID[id] = res } return ids, nil } @@ -281,6 +266,10 @@ func (c *BridgeClient) discard(err error) { c.mu.Lock() c.discarded = true + for _, ee := range c.execErrors { + ee.Release() + } + c.execErrors = nil for id, res := range c.resultByID { res.Release(context.TODO()) delete(c.resultByID, id) diff --git a/vendor/github.com/moby/buildkit/frontend/gateway/gateway.go b/vendor/github.com/moby/buildkit/frontend/gateway/gateway.go index f4bf29e095..dab20f4893 100644 --- a/vendor/github.com/moby/buildkit/frontend/gateway/gateway.go +++ b/vendor/github.com/moby/buildkit/frontend/gateway/gateway.go @@ -1,6 +1,7 @@ package gateway import ( + "bufio" "context" "encoding/json" "fmt" @@ -280,6 +281,10 @@ func (gf *gatewayFrontend) Solve(ctx context.Context, llbBridge frontend.Fronten lbf, ctx := serveLLBBridgeForwarder(ctx, llbBridge, exec, gf.workers, inputs, sid, sm) defer lbf.Discard() defer lbf.conn.Close() + // The executor may pass these fds directly to the frontend process, so + // close them only after Run has returned and no fork can still use them. + defer lbf.Stdin.Close() + defer lbf.Stdout.Close() mdmnt, release, err := metadataMount(frontendDef) if err != nil { @@ -327,12 +332,12 @@ func metadataMount(def *opspb.Definition) (*executor.Mount, func(), error) { } return &executor.Mount{ - Src: &bind{dir}, - Dest: "/run/config/buildkit/metadata", - Readonly: true, - }, func() { - os.RemoveAll(dir) - }, nil + Src: &bind{dir}, + Dest: "/run/config/buildkit/metadata", + Readonly: true, + }, func() { + os.RemoveAll(dir) + }, nil } type bind struct { @@ -394,6 +399,10 @@ func (lbf *llbBridgeForwarder) discard() { lbf.mountsMu.Unlock() lbf.discarded = true + for _, ee := range lbf.execErrors { + ee.Release() + } + lbf.execErrors = nil for id, res := range lbf.resultByID { res.Release(context.TODO()) delete(lbf.resultByID, id) @@ -474,7 +483,6 @@ func newBridgeForwarder(ctx context.Context, llbBridge frontend.FrontendLLBBridg refs: map[string]solver.ResultProxy{}, resultByID: map[string]solver.Result{}, doneCh: make(chan struct{}), - pipe: newPipe(), workers: workers, inputs: inputs, sid: sid, @@ -489,6 +497,7 @@ func newBridgeForwarder(ctx context.Context, llbBridge frontend.FrontendLLBBridg func serveLLBBridgeForwarder(ctx context.Context, llbBridge frontend.FrontendLLBBridge, exec executor.Executor, workers worker.Infos, inputs map[string]*opspb.Definition, sid string, sm *session.Manager) (*llbBridgeForwarder, context.Context) { ctx, cancel := context.WithCancelCause(ctx) lbf := newBridgeForwarder(ctx, llbBridge, exec, workers, inputs, sid, sm) + lbf.pipe = newPipe() serverOpt := []grpc.ServerOption{ grpc.UnaryInterceptor(grpcerrors.UnaryServerInterceptor), grpc.StreamInterceptor(grpcerrors.StreamServerInterceptor), @@ -525,9 +534,9 @@ func newPipe() *pipe { Stdin: pr1, Stdout: pw2, conn: &conn{ - Reader: pr2, - Writer: pw1, - Closer: pw2, + Reader: pr2, + Writer: pw1, + closers: []io.Closer{pr2, pw1}, }, } } @@ -535,7 +544,17 @@ func newPipe() *pipe { type conn struct { io.Reader io.Writer - io.Closer + closers []io.Closer +} + +func (s *conn) Close() error { + var err error + for _, closer := range s.closers { + if closeErr := closer.Close(); err == nil { + err = closeErr + } + } + return errors.WithStack(err) } func (s *conn) LocalAddr() net.Addr { @@ -589,6 +608,7 @@ type llbBridgeForwarder struct { refs map[string]solver.ResultProxy resultByID map[string]solver.Result discarded bool + execErrors []*llberrdefs.ExecError // lastRef solver.CachedResult // lastRefs map[string]solver.CachedResult // err error @@ -715,6 +735,13 @@ func (lbf *llbBridgeForwarder) wrapSolveError(solveErr error) error { subject errdefs.IsSolve_Subject ) if errors.As(solveErr, &ee) { + lbf.mu.Lock() + if lbf.discarded { + ee.Release() + } else if !slices.Contains(lbf.execErrors, ee) { + lbf.execErrors = append(lbf.execErrors, ee) + } + lbf.mu.Unlock() var err error inputIDs, err = lbf.registerResultIDs(ee.Inputs...) if err != nil { @@ -730,7 +757,7 @@ func (lbf *llbBridgeForwarder) wrapSolveError(solveErr error) error { } if errors.As(solveErr, &sce) { var err error - inputIDs, err = lbf.registerSlowCacheResult(sce.Result) + inputIDs, err = lbf.registerResultIDs(sce.Result) if err != nil { return err } @@ -739,32 +766,14 @@ func (lbf *llbBridgeForwarder) wrapSolveError(solveErr error) error { return errdefs.WithSolveError(solveErr, subject, inputIDs, mountIDs) } +// registerResultIDs retains gateway-owned clones of error refs until discard. func (lbf *llbBridgeForwarder) registerResultIDs(results ...solver.Result) (ids []string, err error) { lbf.mu.Lock() defer lbf.mu.Unlock() - return lbf.registerResultIDsLocked(results...) -} - -// registerSlowCacheResult registers a clone of a result borrowed from the -// solver job. The clone is only taken before the forwarder is discarded, as the -// job may be released after that. -func (lbf *llbBridgeForwarder) registerSlowCacheResult(res solver.Result) ([]string, error) { - lbf.mu.Lock() - defer lbf.mu.Unlock() - if lbf.discarded { return nil, nil } - // Validate before cloning so a type error can't leak the clone. Once - // cloned, registerResultIDsLocked either keeps or releases it. - if _, ok := res.Sys().(*worker.WorkerRef); !ok { - return nil, errors.Errorf("unexpected type for result, got %T", res.Sys()) - } - return lbf.registerResultIDsLocked(res.Clone()) -} - -func (lbf *llbBridgeForwarder) registerResultIDsLocked(results ...solver.Result) (ids []string, err error) { ids = make([]string, len(results)) for i, res := range results { if res == nil { @@ -776,16 +785,11 @@ func (lbf *llbBridgeForwarder) registerResultIDsLocked(results ...solver.Result) } id := workerRef.ID() ids[i] = id - if existing, ok := lbf.resultByID[id]; ok { - if existing != res { - if err := res.Release(context.TODO()); err != nil { - return ids, errors.WithStack(err) - } - } - continue + if _, ok := lbf.resultByID[id]; !ok { + // Cached sharedOp.Exec errors have snapshots pinned by the active + // solver state, even after the original error refs are released. + lbf.resultByID[id] = res.Clone() } - // Keep the result wrapper so Release balances split-result ownership. - lbf.resultByID[id] = res } return ids, nil } @@ -834,6 +838,16 @@ func (lbf *llbBridgeForwarder) Solve(ctx context.Context, req *pb.SolveRequest) lbf.mu.Lock() + if lbf.discarded { + // A solve that finished after Discard has no one to release its refs. + lbf.mu.Unlock() + res.EachRef(func(r solver.ResultProxy) error { + r.Release(context.TODO()) + return nil + }) + return nil, stack.Enable(status.Error(codes.Unavailable, "gateway forwarder is closing")) + } + if res.Refs != nil { ids := make(map[string]string, len(res.Refs)) defs := make(map[string]*opspb.Definition, len(res.Refs)) @@ -1869,13 +1883,28 @@ func (lbf *llbBridgeForwarder) cloneRef(id string) (solver.ResultProxy, error) { return s2, nil } +type bufferedConn struct { + net.Conn + reader *bufio.Reader +} + +func (c *bufferedConn) Read(p []byte) (int, error) { + return c.reader.Read(p) +} + func serve(ctx context.Context, grpcServer *grpc.Server, conn net.Conn) { - go func() { - <-ctx.Done() - conn.Close() - }() + stop := context.AfterFunc(ctx, func() { conn.Close() }) + defer stop() + defer conn.Close() + + // The frontend container may take a while to start. Start HTTP/2's + // client preface timeout only once the frontend begins communicating. + reader := bufio.NewReader(conn) + if _, err := reader.Peek(1); err != nil { + return + } bklog.G(ctx).Debugf("serving grpc connection") - (&http2.Server{}).ServeConn(conn, &http2.ServeConnOpts{Handler: grpcServer}) //nolint:staticcheck // This existing HTTP/2 connection is already established. + (&http2.Server{}).ServeConn(&bufferedConn{Conn: conn, reader: reader}, &http2.ServeConnOpts{Handler: grpcServer}) //nolint:staticcheck // This existing HTTP/2 connection is already established. } type markTypeFrontend struct{} diff --git a/vendor/github.com/moby/buildkit/frontend/gateway/grpcclient/client.go b/vendor/github.com/moby/buildkit/frontend/gateway/grpcclient/client.go index 0188e2b22e..bd03077e1d 100644 --- a/vendor/github.com/moby/buildkit/frontend/gateway/grpcclient/client.go +++ b/vendor/github.com/moby/buildkit/frontend/gateway/grpcclient/client.go @@ -97,7 +97,8 @@ func convertRef(ref client.Reference) (*pb.Ref, error) { if !ok { return nil, errors.Errorf("invalid return reference type %T", ref) } - return &pb.Ref{Id: r.id, Def: r.def}, nil + // Return only needs the ID of the reference held by the server. + return &pb.Ref{Id: r.id}, nil } func RunFromEnvironment(ctx context.Context, f client.BuildFunc) error { @@ -821,12 +822,14 @@ func (c *grpcClient) Inputs(ctx context.Context) (map[string]llb.State, error) { // communication channel between the process and the ExecProcess message // stream. type procMessageForwarder struct { + ctx context.Context done chan struct{} msgs chan *pb.ExecMessage } -func newProcMessageForwarder() *procMessageForwarder { +func newProcMessageForwarder(ctx context.Context) *procMessageForwarder { return &procMessageForwarder{ + ctx: ctx, done: make(chan struct{}), msgs: make(chan *pb.ExecMessage), } @@ -840,14 +843,16 @@ func (b *procMessageForwarder) Send(ctx context.Context, m *pb.ExecMessage) { } } -func (b *procMessageForwarder) Recv(ctx context.Context) (m *pb.ExecMessage, ok bool) { +func (b *procMessageForwarder) Recv(ctx context.Context) (*pb.ExecMessage, error) { select { case <-ctx.Done(): - return nil, true + return nil, context.Cause(ctx) + case <-b.ctx.Done(): + return nil, context.Cause(b.ctx) case <-b.done: - return nil, false - case m = <-b.msgs: - return m, true + return nil, nil + case m := <-b.msgs: + return m, nil } } @@ -908,6 +913,11 @@ func (m *messageForwarder) Start() (err error) { m.eg.Go(func() error { for { msg, err := m.stream.Recv() + if err != nil { + // EOF and cancellation also end delivery to every process, + // even though they are not errors for Release. + m.cancel(errors.Wrap(err, "remote exec stream closed")) + } if errors.Is(err, io.EOF) || grpcerrors.Code(err) == codes.Canceled { return nil } @@ -976,7 +986,7 @@ func (m *messageForwarder) Release() error { func (m *messageForwarder) Register(pid string) *procMessageForwarder { m.mu.Lock() defer m.mu.Unlock() - sender := newProcMessageForwarder() + sender := newProcMessageForwarder(m.ctx) m.pids[pid] = sender return sender } @@ -1120,7 +1130,11 @@ func (ctr *container) Start(ctx context.Context, req client.StartRequest) (clien return nil, err } - msg, _ := msgs.Recv(ctx) + msg, err := msgs.Recv(ctx) + if err != nil { + ctr.execMsgs.Deregister(pid) + return nil, err + } if msg == nil { return nil, errors.New("failed to receive started message") } @@ -1165,13 +1179,11 @@ func (ctr *container) Start(ctx context.Context, req client.StartRequest) (clien processID: pid, fd: 0, } - _, err := io.Copy(m, stdinReader) - // ignore ErrClosedPipe, it is EOF for our usage. - if err != nil && !errors.Is(err, io.ErrClosedPipe) { - return err + _, copyErr := io.Copy(m, stdinReader) + if copyErr != nil && !errors.Is(copyErr, io.ErrClosedPipe) { + return copyErr } - // not an error so must be eof - return ctr.execMsgs.Send(&pb.ExecMessage{ + err := ctr.execMsgs.Send(&pb.ExecMessage{ ProcessID: pid, Input: &pb.ExecMessage_File{ File: &pb.FdMessage{ @@ -1180,6 +1192,12 @@ func (ctr *container) Start(ctx context.Context, req client.StartRequest) (clien }, }, }) + // Cleanup still sends EOF to unblock remote stdin, but a failed send + // must not replace the process result that triggered cleanup. + if errors.Is(copyErr, io.ErrClosedPipe) { + return nil + } + return err }) } @@ -1188,21 +1206,15 @@ func (ctr *container) Start(ctx context.Context, req client.StartRequest) (clien defer closeDoneOnce.Do(func() { close(done) }) var exitError error for { - msg, ok := msgs.Recv(ctx) - if !ok { + msg, err := msgs.Recv(ctx) + if err != nil { + return err + } + if msg == nil { // no more messages, return return exitError } - if msg == nil { - // empty message from ctx cancel, so just start shutting down - // input - closeDoneOnce.Do(func() { - close(done) - }) - return context.Cause(ctx) - } - if file := msg.GetFile(); file != nil { var out io.WriteCloser switch file.Fd { diff --git a/vendor/github.com/moby/buildkit/snapshot/imagerefchecker/checker.go b/vendor/github.com/moby/buildkit/snapshot/imagerefchecker/checker.go index 6a596a5e6b..44e6fed900 100644 --- a/vendor/github.com/moby/buildkit/snapshot/imagerefchecker/checker.go +++ b/vendor/github.com/moby/buildkit/snapshot/imagerefchecker/checker.go @@ -3,6 +3,7 @@ package imagerefchecker import ( "context" "encoding/json" + "slices" "strings" "sync" @@ -39,6 +40,13 @@ func (c *Checker) Exists(key string, blobs []digest.Digest) bool { return false } + // An empty digest means that a layer in the chain has no blob, so it was + // never exported and no image can reference it. Without this check, the + // key of such a chain would be the same as the key of its parent chain. + if slices.Contains(blobs, "") { + return false + } + c.once.Do(c.init) if b, ok := c.cache[key]; ok { diff --git a/vendor/github.com/moby/buildkit/snapshot/localmounter_linux.go b/vendor/github.com/moby/buildkit/snapshot/localmounter_linux.go index a8132c9a7d..c17556c7ed 100644 --- a/vendor/github.com/moby/buildkit/snapshot/localmounter_linux.go +++ b/vendor/github.com/moby/buildkit/snapshot/localmounter_linux.go @@ -37,6 +37,8 @@ func (lm *localMounter) Mount() (string, error) { } } + lm.mounts = mount.RemoveVolatileOption(lm.mounts) + var isFile bool if len(lm.mounts) == 1 && (lm.mounts[0].Type == "bind" || lm.mounts[0].Type == "rbind") { if !lm.forceRemount { diff --git a/vendor/github.com/moby/buildkit/solver/cachemanager.go b/vendor/github.com/moby/buildkit/solver/cachemanager.go index bdf2f9cf55..0b7dfc1413 100644 --- a/vendor/github.com/moby/buildkit/solver/cachemanager.go +++ b/vendor/github.com/moby/buildkit/solver/cachemanager.go @@ -409,42 +409,40 @@ func (c *cacheManager) ensurePersistentKey(k *CacheKey) error { func (c *cacheManager) getIDFromDeps(k *CacheKey) string { matches := map[string]struct{}{} - +loop0: for i, deps := range k.deps { - if i == 0 || len(matches) > 0 { - for _, ck := range deps { - m2 := make(map[string]struct{}) - if err := c.backend.WalkLinks(c.getID(ck.CacheKey.CacheKey), CacheInfoLink{ - Input: Index(i), - Output: k.Output(), - Digest: k.Digest(), - Selector: ck.Selector, - }, func(id string) error { - if i == 0 { - matches[id] = struct{}{} - } else { - m2[id] = struct{}{} - } - return nil - }); err != nil { - matches = map[string]struct{}{} - break - } - if i != 0 { - for id := range matches { - if _, ok := m2[id]; !ok { - delete(matches, id) - } - } + if i > 0 && len(matches) == 0 { + break + } + // Keys within one input are alternatives; different inputs must all match. + candidates := map[string]struct{}{} + for _, ck := range deps { + if err := c.backend.WalkLinks(c.getID(ck.CacheKey.CacheKey), CacheInfoLink{ + Input: Index(i), + Output: k.Output(), + Digest: k.Digest(), + Selector: ck.Selector, + }, func(id string) error { + candidates[id] = struct{}{} + return nil + }); err != nil { + matches = nil + break loop0 + } + } + if i == 0 { + matches = candidates + } else { + for id := range matches { + if _, ok := candidates[id]; !ok { + delete(matches, id) } } } } - - for k := range matches { - return k + for id := range matches { + return id } - return identity.NewID() } diff --git a/vendor/github.com/moby/buildkit/solver/cacheopts.go b/vendor/github.com/moby/buildkit/solver/cacheopts.go index 67099c2423..6aa93514ad 100644 --- a/vendor/github.com/moby/buildkit/solver/cacheopts.go +++ b/vendor/github.com/moby/buildkit/solver/cacheopts.go @@ -36,7 +36,7 @@ func withAncestorCacheOpts(ctx context.Context, start *state) context.Context { } values := make(map[any]any) walkAncestors(ctx, start, func(st *state) bool { - if st.clientVertex.Error != "" { + if st.vertexFailed() { // don't use values from cancelled or otherwise error'd vertexes return false } diff --git a/vendor/github.com/moby/buildkit/solver/edge.go b/vendor/github.com/moby/buildkit/solver/edge.go index ed364ef0e1..ab9ff3b01c 100644 --- a/vendor/github.com/moby/buildkit/solver/edge.go +++ b/vendor/github.com/moby/buildkit/solver/edge.go @@ -69,7 +69,10 @@ type edge struct { keysDidChange bool index *edgeIndex - secondaryExporters []expDep + // secondaryExportersMu protects secondaryExporters, which the scheduler + // appends to on merge while cache exporters read it from other goroutines + secondaryExportersMu sync.Mutex + secondaryExporters []expDep failedOnce sync.Once debug bool @@ -729,6 +732,14 @@ func (e *edge) processDepSlowCacheReq(index int, dep *dep) { // respondToIncoming responds to all incoming requests. completing or // updating them when possible func (e *edge) respondToIncoming(incoming []pipeSender, allPipes []pipeReceiver) (edgeStatusType, bool) { + // Requests can be canceled concurrently. Decide on one snapshot so a late + // cancel cannot finish the last request while outgoing work is active; it + // is handled on the next dispatch. + canceled := make([]bool, len(incoming)) + for i, req := range incoming { + canceled[i] = req.Request().Canceled + } + // detect the result state for the requests allIncomingCanComplete := true desiredState := e.state @@ -737,8 +748,8 @@ func (e *edge) respondToIncoming(incoming []pipeSender, allPipes []pipeReceiver) // check incoming requests // check if all requests can be either answered or canceled if !e.isComplete() { - for _, req := range incoming { - if !req.Request().Canceled { + for i, req := range incoming { + if !canceled[i] { allCanceled = false if r := req.Request().Payload; desiredState < r.desiredState { desiredState = r.desiredState @@ -765,8 +776,8 @@ func (e *edge) respondToIncoming(incoming []pipeSender, allPipes []pipeReceiver) // can close all but one requests var leaveOpen pipeSender - for _, req := range incoming { - if !req.Request().Canceled { + for i, req := range incoming { + if !canceled[i] { leaveOpen = req break } @@ -790,9 +801,9 @@ func (e *edge) respondToIncoming(incoming []pipeSender, allPipes []pipeReceiver) } // update incoming based on current state - for _, req := range incoming { + for i, req := range incoming { r := req.Request().Payload - if req.Request().Canceled { + if canceled[i] { e.finishIncoming(req) } else if !e.hasActiveOutgoing && e.state >= r.desiredState { e.finishIncoming(req) diff --git a/vendor/github.com/moby/buildkit/solver/exporter.go b/vendor/github.com/moby/buildkit/solver/exporter.go index 6ade2f44ce..04574ffff7 100644 --- a/vendor/github.com/moby/buildkit/solver/exporter.go +++ b/vendor/github.com/moby/buildkit/solver/exporter.go @@ -246,7 +246,10 @@ func (e *exporter) ExportTo(ctx context.Context, t CacheExporterTarget, opt Cach } if e.edge != nil { - for _, de := range e.edge.secondaryExporters { + e.edge.secondaryExportersMu.Lock() + secondaryExporters := slices.Clone(e.edge.secondaryExporters) + e.edge.secondaryExportersMu.Unlock() + for _, de := range secondaryExporters { recs, err := de.cacheKey.CacheKey.Exporter.ExportTo(mainCtx, t, opt) if err != nil { continue diff --git a/vendor/github.com/moby/buildkit/solver/index.go b/vendor/github.com/moby/buildkit/solver/index.go index c640e3b107..3cfaedc8ed 100644 --- a/vendor/github.com/moby/buildkit/solver/index.go +++ b/vendor/github.com/moby/buildkit/solver/index.go @@ -160,11 +160,14 @@ func (ei *edgeIndex) enforceIndexID(k *CacheKey) { matches := ei.getAllMatches(k) + // TraceFields reads indexIDs under the cache-key lock. + k.mu.Lock() if len(matches) > 0 { k.indexIDs = matches } else { k.indexIDs = []string{identity.NewID()} } + k.mu.Unlock() for _, id := range k.indexIDs { ei.enforceLinked(id, k) diff --git a/vendor/github.com/moby/buildkit/solver/internal/pipe/pipe.go b/vendor/github.com/moby/buildkit/solver/internal/pipe/pipe.go index 55309c2a90..32b712f412 100644 --- a/vendor/github.com/moby/buildkit/solver/internal/pipe/pipe.go +++ b/vendor/github.com/moby/buildkit/solver/internal/pipe/pipe.go @@ -153,7 +153,7 @@ func (pw *sender[Payload, Value]) Finalize(v Value, err error) { pw.status.Value = v pw.status.Err = err pw.status.Completed = true - if errors.Is(err, context.Canceled) && pw.req.Canceled { + if errors.Is(err, context.Canceled) && pw.Request().Canceled { pw.status.Canceled = true } pw.sendChannel.Send(pw.status) diff --git a/vendor/github.com/moby/buildkit/solver/jobs.go b/vendor/github.com/moby/buildkit/solver/jobs.go index 5dec5daa7f..c2c6153c1a 100644 --- a/vendor/github.com/moby/buildkit/solver/jobs.go +++ b/vendor/github.com/moby/buildkit/solver/jobs.go @@ -61,9 +61,10 @@ type state struct { mspan *tracing.MultiSpan execSpan trace.Span - vtx Vertex - clientVertex client.Vertex - origDigest digest.Digest // original LLB digest. TODO: probably better to use string ID so this isn't needed + vtx Vertex + clientVertex client.Vertex + clientVertexMu sync.Mutex // protects mutable clientVertex fields + origDigest digest.Digest // original LLB digest. TODO: probably better to use string ID so this isn't needed mu sync.RWMutex op *sharedOp @@ -329,6 +330,51 @@ func (s *state) combinedCacheManager() CacheManager { return NewCombinedCacheManager(cms, mainCache) } +// writeVertexSnapshot can read progress concurrently with +// notifyStarted, which is the only writer of the mutable vertex fields. +func (s *state) writeVertexSnapshot(pw progress.Writer) { + s.clientVertexMu.Lock() + defer s.clientVertexMu.Unlock() + // Keep the snapshot and write together so a newer update cannot be + // published before this snapshot. + pw.Write(identity.NewID(), s.clientVertex) +} + +// vertexFailed lets the ancestor walk read the error concurrently with +// progress updates from notifyStarted. +func (s *state) vertexFailed() bool { + s.clientVertexMu.Lock() + defer s.clientVertexMu.Unlock() + return s.clientVertex.Error != "" +} + +func (s *state) notifyStarted(ctx context.Context, cached bool) func(err error, cached bool) { + pw, _, _ := progress.NewFromContext(ctx) + s.clientVertexMu.Lock() + v := &s.clientVertex + start := time.Now() + v.Started = &start + v.Completed = nil + v.Cached = cached + id := identity.NewID() + pw.Write(id, *v) + s.clientVertexMu.Unlock() + return func(err error, cached bool) { + defer pw.Close() + s.clientVertexMu.Lock() + defer s.clientVertexMu.Unlock() + stop := time.Now() + v.Completed = &stop + v.Cached = cached + if err != nil { + v.Error = err.Error() + } else { + v.Error = "" + } + pw.Write(id, *v) + } +} + func (s *state) Release() { for _, e := range s.edges { for e.owner != nil { @@ -388,7 +434,9 @@ func (sb *subBuilder) EachValue(ctx context.Context, key string, fn func(any) er } type Job struct { - mu sync.Mutex // protects completedTime, pw, span + mu sync.Mutex // protects completedTime, pw, span, builds, discarded + builds int + discarded bool list *Solver pr *progress.MultiReader pw progress.Writer @@ -689,7 +737,7 @@ func (jl *Solver) connectProgressFromState(target, src *state) { if _, ok := target.allPw[pw]; !ok { target.mpw.Add(pw) target.allPw[pw] = struct{}{} - pw.Write(identity.NewID(), target.clientVertex) + target.writeVertexSnapshot(pw) if span != nil && span.SpanContext().IsValid() { target.mspan.Add(span) } @@ -800,11 +848,17 @@ func (jl *Solver) deleteIfUnreferenced(k digest.Digest, st *state) { } func (j *Job) Build(ctx context.Context, e Edge) (CachedResultWithProvenance, error) { - if span := trace.SpanFromContext(ctx); span.SpanContext().IsValid() { - j.mu.Lock() - j.span = span + j.mu.Lock() + if j.discarded { j.mu.Unlock() + return nil, errors.Errorf("build on discarded job %s", j.id) } + j.builds++ + if span := trace.SpanFromContext(ctx); span.SpanContext().IsValid() { + j.span = span + } + j.mu.Unlock() + defer j.buildDone() v, err := j.list.load(ctx, e.Vertex, nil, j) if err != nil { @@ -873,7 +927,32 @@ func (j *Job) CloseProgress() { j.pw.Close() } +// buildDone finishes a pending Discard when the last running build returns. +func (j *Job) buildDone() { + j.mu.Lock() + j.builds-- + last := j.discarded && j.builds == 0 + j.mu.Unlock() + if last { + j.discard() + } +} + +// Discard releases the job's states. While builds are still running the states +// are released when the last one returns, so results that complete after +// Discard are released with their state. func (j *Job) Discard() error { + j.mu.Lock() + j.discarded = true + running := j.builds > 0 + j.mu.Unlock() + if running { + return nil + } + return j.discard() +} + +func (j *Job) discard() error { j.list.mu.Lock() defer j.list.mu.Unlock() @@ -1058,7 +1137,7 @@ func (s *sharedOp) LoadCache(ctx context.Context, rec *CacheRecord) (Result, fun // no cache hit. start evaluating the node span, ctx := tracing.StartSpan(ctx, "load cache: "+s.st.vtx.Name(), trace.WithAttributes(attribute.String("vertex", s.st.vtx.Digest().String()))) s.st.execSpan = span - notifyCompleted := notifyStarted(ctx, &s.st.clientVertex, true) + notifyCompleted := s.st.notifyStarted(ctx, true) res, err := s.Cache().Load(withAncestorCacheOpts(ctx, s.st), rec) tracing.FinishWithError(span, err) notifyCompleted(err, true) @@ -1145,7 +1224,7 @@ func (s *sharedOp) CalcSlowCache(ctx context.Context, index Index, p PreprocessF if s.st.mspan.Span != nil { ctx = trace.ContextWithSpan(ctx, s.st.mspan) } - notifyCompleted := notifyStarted(ctx, &s.st.clientVertex, false) + notifyCompleted := s.st.notifyStarted(ctx, false) notifyCompleted(err, false) return "", err } @@ -1177,7 +1256,7 @@ func (s *sharedOp) CacheMap(ctx context.Context, index int) (resp *cacheMapResp, if len(s.st.vtx.Inputs()) == 0 { // no cache hit. start evaluating the node span, ctx := tracing.StartSpan(ctx, "cache request: "+s.st.vtx.Name(), trace.WithAttributes(attribute.String("vertex", s.st.vtx.Digest().String()))) - notifyCompleted := notifyStarted(ctx, &s.st.clientVertex, false) + notifyCompleted := s.st.notifyStarted(ctx, false) defer func() { tracing.FinishWithError(span, retErr) notifyCompleted(retErr, false) @@ -1263,7 +1342,7 @@ func (s *sharedOp) Exec(ctx context.Context, inputs []Result) (outputs []Result, // no cache hit. start evaluating the node span, ctx := tracing.StartSpan(ctx, s.st.vtx.Name(), trace.WithAttributes(attribute.String("vertex", s.st.vtx.Digest().String()))) s.st.execSpan = span - notifyCompleted := notifyStarted(ctx, &s.st.clientVertex, false) + notifyCompleted := s.st.notifyStarted(ctx, false) defer func() { tracing.FinishWithError(span, retErr) notifyCompleted(retErr, false) @@ -1390,28 +1469,6 @@ func (v *vertexWithCacheOptions) Inputs() []Edge { return v.inputs } -func notifyStarted(ctx context.Context, v *client.Vertex, cached bool) func(err error, cached bool) { - pw, _, _ := progress.NewFromContext(ctx) - start := time.Now() - v.Started = &start - v.Completed = nil - v.Cached = cached - id := identity.NewID() - pw.Write(id, *v) - return func(err error, cached bool) { - defer pw.Close() - stop := time.Now() - v.Completed = &stop - v.Cached = cached - if err != nil { - v.Error = err.Error() - } else { - v.Error = "" - } - pw.Write(id, *v) - } -} - type SlowCacheError struct { error Index Index diff --git a/vendor/github.com/moby/buildkit/solver/llbsolver/cdidevices/manager.go b/vendor/github.com/moby/buildkit/solver/llbsolver/cdidevices/manager.go index 7df61f371a..19c78bb05f 100644 --- a/vendor/github.com/moby/buildkit/solver/llbsolver/cdidevices/manager.go +++ b/vendor/github.com/moby/buildkit/solver/llbsolver/cdidevices/manager.go @@ -5,6 +5,7 @@ import ( "maps" "strconv" "strings" + "sync" "github.com/moby/buildkit/solver/pb" "github.com/moby/buildkit/util/bklog" @@ -40,9 +41,10 @@ type Device struct { } type Manager struct { - cache *cdi.Cache - locker *locker.Locker - autoAllowed map[string]struct{} + cache *cdi.Cache + locker *locker.Locker + autoAllowedMu sync.RWMutex + autoAllowed map[string]struct{} } func NewManager(cache *cdi.Cache, autoAllowed []string) *Manager { @@ -58,6 +60,8 @@ func NewManager(cache *cdi.Cache, autoAllowed []string) *Manager { } func (m *Manager) isAutoAllowed(kind, name string, annotations map[string]string) bool { + m.autoAllowedMu.RLock() + defer m.autoAllowedMu.RUnlock() if _, ok := m.autoAllowed[name]; ok { return true } @@ -152,7 +156,25 @@ func (m *Manager) FindDevices(devs ...*pb.CDIDevice) ([]string, error) { return dedupSlice(out), nil } +// MatchDevices returns the registered devices that the request resolves to, +// including matches by class annotation. Unlike FindDevices it does not fail +// or warn when nothing matches. +func (m *Manager) MatchDevices(dev *pb.CDIDevice) []string { + return m.matchDevice(dev, m.cache.ListDevices()) +} + func (m *Manager) parseDevice(dev *pb.CDIDevice, all []string) ([]string, error) { + out := m.matchDevice(dev, all) + if len(out) == 0 { + if !dev.Optional { + return nil, errors.Errorf("required device %q is not registered", dev.Name) + } + bklog.G(context.TODO()).Warnf("Optional device %q is not registered", dev.Name) + } + return out, nil +} + +func (m *Manager) matchDevice(dev *pb.CDIDevice, all []string) []string { var out []string kind, name, _ := strings.Cut(dev.Name, "=") @@ -197,13 +219,7 @@ func (m *Manager) parseDevice(dev *pb.CDIDevice, all []string) ([]string, error) } } - if len(out) == 0 { - if !dev.Optional { - return nil, errors.Errorf("required device %q is not registered", dev.Name) - } - bklog.G(context.TODO()).Warnf("Optional device %q is not registered", dev.Name) - } - return out, nil + return out } func (m *Manager) hasDevice(name string) bool { @@ -249,7 +265,9 @@ func (m *Manager) OnDemandInstaller(name string) (func(context.Context) error, b } // TODO: this needs to be set as annotation to survive reboot + m.autoAllowedMu.Lock() m.autoAllowed[name] = struct{}{} + m.autoAllowedMu.Unlock() return nil }, true diff --git a/vendor/github.com/moby/buildkit/solver/llbsolver/errdefs/exec.go b/vendor/github.com/moby/buildkit/solver/llbsolver/errdefs/exec.go index f70bc95bf6..d4f7bb292c 100644 --- a/vendor/github.com/moby/buildkit/solver/llbsolver/errdefs/exec.go +++ b/vendor/github.com/moby/buildkit/solver/llbsolver/errdefs/exec.go @@ -3,17 +3,27 @@ package errdefs import ( "context" "runtime" + "sync" "github.com/moby/buildkit/solver" "github.com/moby/buildkit/util/bklog" ) // ExecError will be returned when an error is encountered when evaluating an op. +// The error owns its original refs until Borrow or the first Release. Proxies +// and gateways retain the error; gateways also keep independent clones. They may +// clone cached originals after Release, which native result handles support +// while another ref pins the snapshot. When sharedOp.Exec completes with an +// active solver state, that state pins error refs until its last job leaves. +// Pinning follows a single Unwrap chain, not errors.Join branches; CacheMap and +// Acquire errors do not get that pin. type ExecError struct { error Inputs []solver.Result Mounts []solver.Result OwnerBorrowed bool + + mu sync.Mutex // protects OwnerBorrowed } func (e *ExecError) Unwrap() error { @@ -49,16 +59,27 @@ func (e *ExecError) EachRef(fn func(solver.Result) error) (err error) { return err } -func (e *ExecError) Release() error { +// Borrow marks the error's refs as owned by a caller, so the finalizer and +// Release do not release them. The same error can be shared by multiple +// callers; Borrow returns false if the refs were already borrowed. +// The successful caller must release the refs; later callers acquire no ownership. +func (e *ExecError) Borrow() bool { + e.mu.Lock() + defer e.mu.Unlock() if e.OwnerBorrowed { + return false + } + e.OwnerBorrowed = true + return true +} + +func (e *ExecError) Release() error { + if !e.Borrow() { return nil } - err := e.EachRef(func(r solver.Result) error { - r.Release(context.TODO()) - return nil + return e.EachRef(func(r solver.Result) error { + return r.Release(context.TODO()) }) - e.OwnerBorrowed = true - return err } func WithExecError(err error, inputs, mounts []solver.Result) error { @@ -75,7 +96,7 @@ func WithExecErrorWithContext(ctx context.Context, err error, inputs, mounts []s Mounts: mounts, } runtime.SetFinalizer(ee, func(e *ExecError) { - if !e.OwnerBorrowed { + if e.Borrow() { e.EachRef(func(r solver.Result) error { bklog.G(ctx).Warn("leaked execError detected and released") r.Release(context.TODO()) diff --git a/vendor/github.com/moby/buildkit/solver/llbsolver/history.go b/vendor/github.com/moby/buildkit/solver/llbsolver/history.go index 8fd7486949..2bdb075cf8 100644 --- a/vendor/github.com/moby/buildkit/solver/llbsolver/history.go +++ b/vendor/github.com/moby/buildkit/solver/llbsolver/history.go @@ -137,11 +137,7 @@ func (s *Solver) recordBuildHistory(ctx context.Context, id string, req frontend if err != nil { return nil, nil, err } - pr, err := pc.Predicate(ctx) - if err != nil { - return nil, nil, err - } - dt, err := json.MarshalIndent(pr, "", " ") + dt, err := pc.PredicateJSON(ctx) if err != nil { return nil, nil, err } diff --git a/vendor/github.com/moby/buildkit/solver/llbsolver/ops/exec_binfmt.go b/vendor/github.com/moby/buildkit/solver/llbsolver/ops/exec_binfmt.go index 460ec4014b..5e8d0fdb38 100644 --- a/vendor/github.com/moby/buildkit/solver/llbsolver/ops/exec_binfmt.go +++ b/vendor/github.com/moby/buildkit/solver/llbsolver/ops/exec_binfmt.go @@ -71,12 +71,12 @@ func (m *staticEmulatorMount) Mount() ([]mount.Mount, func() error, error) { ret = true return []mount.Mount{{ - Type: "bind", - Source: filepath.Join(tmpdir, qemuMountName), - Options: []string{"ro", "bind"}, - }}, func() error { - return os.RemoveAll(tmpdir) - }, nil + Type: "bind", + Source: filepath.Join(tmpdir, qemuMountName), + Options: []string{"ro", "bind"}, + }}, func() error { + return os.RemoveAll(tmpdir) + }, nil } func (m *staticEmulatorMount) IdentityMapping() *user.IdentityMapping { diff --git a/vendor/github.com/moby/buildkit/solver/llbsolver/proc/provenance.go b/vendor/github.com/moby/buildkit/solver/llbsolver/proc/provenance.go index c2bcb55fc2..8d6b43bd21 100644 --- a/vendor/github.com/moby/buildkit/solver/llbsolver/proc/provenance.go +++ b/vendor/github.com/moby/buildkit/solver/llbsolver/proc/provenance.go @@ -2,7 +2,6 @@ package proc import ( "context" - "encoding/json" "strconv" "github.com/moby/buildkit/executor/resources" @@ -65,14 +64,8 @@ func ProvenanceProcessor(slsaVersion provenancetypes.ProvenanceSLSA, attrs map[s InToto: result.InTotoAttestation{ PredicateType: pc.PredicateType(), }, - Path: filename, - ContentFunc: func(ctx context.Context) ([]byte, error) { - pr, err := pc.Predicate(ctx) - if err != nil { - return nil, err - } - return json.MarshalIndent(pr, "", " ") - }, + Path: filename, + ContentFunc: pc.PredicateJSON, }) } diff --git a/vendor/github.com/moby/buildkit/solver/llbsolver/provenance.go b/vendor/github.com/moby/buildkit/solver/llbsolver/provenance.go index 1bb7c2a4aa..ae32023947 100644 --- a/vendor/github.com/moby/buildkit/solver/llbsolver/provenance.go +++ b/vendor/github.com/moby/buildkit/solver/llbsolver/provenance.go @@ -2,6 +2,7 @@ package llbsolver import ( "context" + "encoding/json" "fmt" "slices" "strconv" @@ -384,6 +385,7 @@ func captureProvenance(ctx context.Context, res solver.CachedResultWithProvenanc } type ProvenanceCreator struct { + mu sync.Mutex // serializes predicate updates and encoding of pr pr *provenancetypes.ProvenancePredicateSLSA1 slsaVersion provenancetypes.ProvenanceSLSA j *solver.Job @@ -643,7 +645,24 @@ func (p *ProvenanceCreator) PredicateType() string { return slsa1.PredicateSLSAProvenance } -func (p *ProvenanceCreator) Predicate(ctx context.Context) (any, error) { +// PredicateJSON finalizes the predicate and returns it JSON encoded. It may be +// called concurrently, e.g. by parallel exporters sharing one attestation. +func (p *ProvenanceCreator) PredicateJSON(ctx context.Context) ([]byte, error) { + p.mu.Lock() + defer p.mu.Unlock() + + pr, err := p.predicate(ctx) + if err != nil { + return nil, err + } + dt, err := json.MarshalIndent(pr, "", " ") + if err != nil { + return nil, errors.WithStack(err) + } + return dt, nil +} + +func (p *ProvenanceCreator) predicate(ctx context.Context) (any, error) { end := p.j.RegisterCompleteTime() if p.pr.RunDetails.Metadata == nil { p.pr.RunDetails.Metadata = &provenancetypes.ProvenanceMetadataSLSA1{} diff --git a/vendor/github.com/moby/buildkit/solver/llbsolver/result.go b/vendor/github.com/moby/buildkit/solver/llbsolver/result.go index ccce9e6af7..f5ce9d9478 100644 --- a/vendor/github.com/moby/buildkit/solver/llbsolver/result.go +++ b/vendor/github.com/moby/buildkit/solver/llbsolver/result.go @@ -2,6 +2,7 @@ package llbsolver import ( "context" + "slices" "sync" cacheconfig "github.com/moby/buildkit/cache/config" @@ -46,7 +47,7 @@ type resultProxy struct { released bool v solver.CachedResult err error - errResults []solver.Result + execErrors []*llberrdefs.ExecError provenance *provenance.Capture } @@ -72,8 +73,8 @@ func (rp *resultProxy) Provenance() any { func (rp *resultProxy) Release(ctx context.Context) (err error) { rp.mu.Lock() defer rp.mu.Unlock() - for _, res := range rp.errResults { - rerr := res.Release(ctx) + for _, ee := range rp.execErrors { + rerr := ee.Release() if rerr != nil { err = rerr } @@ -82,9 +83,8 @@ func (rp *resultProxy) Release(ctx context.Context) (err error) { if rp.released { bklog.G(ctx).Warnf("release of already released result") } - rerr := rp.v.Release(ctx) - if err != nil { - return rerr + if rerr := rp.v.Release(ctx); rerr != nil { + err = rerr } } rp.released = true @@ -118,12 +118,13 @@ func (rp *resultProxy) loadResult(ctx context.Context) (solver.CachedResultWithP res, err := rp.b.loadResult(ctx, rp.req.Definition, rp.req.CacheImports, rp.req.SourcePolicies) var ee *llberrdefs.ExecError if errors.As(err, &ee) { - ee.EachRef(func(res solver.Result) error { - rp.errResults = append(rp.errResults, res) - return nil - }) - // acquire ownership so ExecError finalizer doesn't attempt to release as well - ee.OwnerBorrowed = true + rp.mu.Lock() + if rp.released { + ee.Release() + } else if !slices.Contains(rp.execErrors, ee) { + rp.execErrors = append(rp.execErrors, ee) + } + rp.mu.Unlock() } return res, err } diff --git a/vendor/github.com/moby/buildkit/solver/llbsolver/vertex.go b/vendor/github.com/moby/buildkit/solver/llbsolver/vertex.go index 3a661d94d3..99dd9f4230 100644 --- a/vendor/github.com/moby/buildkit/solver/llbsolver/vertex.go +++ b/vendor/github.com/moby/buildkit/solver/llbsolver/vertex.go @@ -167,26 +167,37 @@ func ValidateEntitlements(ent entitlements.Set, cdiManager *cdidevices.Manager) } var allowedDevices []*pb.CDIDevice - var nonAliasedDevices []*pb.CDIDevice + var devicesToResolve []*pb.CDIDevice + onDemand := make(map[string]struct{}) for _, d := range cdiManager.ListDevices() { if d.OnDemand && d.AutoAllow { - allowedDevices = append(allowedDevices, &pb.CDIDevice{Name: d.Name}) + onDemand[d.Name] = struct{}{} } } - if cfg != nil { - for _, d := range op.Exec.CdiDevices { + for _, d := range op.Exec.CdiDevices { + if d == nil { + continue + } + if cfg != nil { if newName, ok := cfg.Devices[d.Name]; ok && newName != "" { d.Name = newName allowedDevices = append(allowedDevices, d) - } else { - nonAliasedDevices = append(nonAliasedDevices, d) + continue } } - } else { - nonAliasedDevices = op.Exec.CdiDevices + kind, _, _ := strings.Cut(d.Name, "=") + // A kind with an on-demand installer has no registered devices + // yet, so a match here can only come from a class annotation + // on another device. That must go through the permission check. + if _, ok := onDemand[kind]; ok && len(cdiManager.MatchDevices(d)) == 0 { + // Resolve the requested name after the installer has registered its devices. + allowedDevices = append(allowedDevices, d) + } else { + devicesToResolve = append(devicesToResolve, d) + } } - mountedDevices, err := cdiManager.FindDevices(nonAliasedDevices...) + mountedDevices, err := cdiManager.FindDevices(devicesToResolve...) if err != nil { return err } @@ -198,6 +209,10 @@ func ValidateEntitlements(ent entitlements.Set, cdiManager *cdidevices.Manager) grantedDevices := make(map[string]struct{}) if cfg != nil { for d := range cfg.Devices { + kind, _, _ := strings.Cut(d, "=") + if _, ok := onDemand[kind]; ok && len(cdiManager.MatchDevices(&pb.CDIDevice{Name: d})) == 0 { + continue + } resolved, err := cdiManager.FindDevices(&pb.CDIDevice{Name: d}) if err != nil { return err diff --git a/vendor/github.com/moby/buildkit/solver/pb/attr.go b/vendor/github.com/moby/buildkit/solver/pb/attr.go index 7a0a68d4ab..5b93672d22 100644 --- a/vendor/github.com/moby/buildkit/solver/pb/attr.go +++ b/vendor/github.com/moby/buildkit/solver/pb/attr.go @@ -13,6 +13,7 @@ const AttrGitFetchByCommit = "git.fetchbycommit" const AttrGitBundle = "git.bundle" const AttrGitCheckoutBundle = "git.checkoutbundle" const AttrGitAdvice = "git.advice" +const AttrGitDebugCommands = "git.debugcommands" const AttrGitSignatureVerifyPubKey = "git.sig.pubkey" const AttrGitSignatureVerifyRejectExpired = "git.sig.rejectexpired" diff --git a/vendor/github.com/moby/buildkit/solver/scheduler.go b/vendor/github.com/moby/buildkit/solver/scheduler.go index 825542647b..91ae832486 100644 --- a/vendor/github.com/moby/buildkit/solver/scheduler.go +++ b/vendor/github.com/moby/buildkit/solver/scheduler.go @@ -309,19 +309,23 @@ func (s *scheduler) mergeTo(target, src *edge) bool { delete(s.outgoing, src) s.signal(target) + var exps []expDep for i, d := range src.deps { for _, k := range d.keys { - target.secondaryExporters = append(target.secondaryExporters, expDep{i, CacheKeyWithSelector{CacheKey: k, Selector: src.cacheMap.Deps[i].Selector}}) + exps = append(exps, expDep{i, CacheKeyWithSelector{CacheKey: k, Selector: src.cacheMap.Deps[i].Selector}}) } if d.slowCacheKey != nil { - target.secondaryExporters = append(target.secondaryExporters, expDep{i, CacheKeyWithSelector{CacheKey: *d.slowCacheKey}}) + exps = append(exps, expDep{i, CacheKeyWithSelector{CacheKey: *d.slowCacheKey}}) } if d.result != nil { for _, dk := range d.result.CacheKeys() { - target.secondaryExporters = append(target.secondaryExporters, expDep{i, CacheKeyWithSelector{CacheKey: dk, Selector: src.cacheMap.Deps[i].Selector}}) + exps = append(exps, expDep{i, CacheKeyWithSelector{CacheKey: dk, Selector: src.cacheMap.Deps[i].Selector}}) } } } + target.secondaryExportersMu.Lock() + target.secondaryExporters = append(target.secondaryExporters, exps...) + target.secondaryExportersMu.Unlock() // TODO(tonistiigi): merge cache providers diff --git a/vendor/github.com/moby/buildkit/source/containerimage/pull.go b/vendor/github.com/moby/buildkit/source/containerimage/pull.go index ca4e7b6270..a89dddfd8b 100644 --- a/vendor/github.com/moby/buildkit/source/containerimage/pull.go +++ b/vendor/github.com/moby/buildkit/source/containerimage/pull.go @@ -49,13 +49,14 @@ type puller struct { ResolverType store sourceresolver.ResolveImageConfigOptStore - g flightcontrol.Group[struct{}] - cacheKeyErr error - cacheKeyDone bool - releaseTmpLeases func(context.Context) error - descHandlers cache.DescHandlers - manifest *pull.PulledManifests - manifestKey string + g flightcontrol.Group[struct{}] + cacheKeyErr error + cacheKeyDone bool + releaseTmpLeases func(context.Context) error + descHandlers cache.DescHandlers + manifest *pull.PulledManifests + manifestKey string + platformManifestKey string *pull.Puller } @@ -132,8 +133,8 @@ func (p *puller) CacheKey(ctx context.Context, jobCtx solver.JobContext, index i return struct{}{}, err } - if p.checksum != "" && p.manifest.MainManifestDesc.Digest != p.checksum { - return struct{}{}, errors.Errorf("image digest %s for %s does not match expected checksum %s", p.manifest.MainManifestDesc.Digest, p.Ref, p.checksum) + if p.checksum != "" && p.manifest.RootManifestDesc.Digest != p.checksum { + return struct{}{}, errors.Errorf("image digest %s for %s does not match expected checksum %s", p.manifest.RootManifestDesc.Digest, p.Ref, p.checksum) } if ll := p.layerLimit; ll != nil { @@ -172,12 +173,17 @@ func (p *puller) CacheKey(ctx context.Context, jobCtx solver.JobContext, index i } } - desc := p.manifest.MainManifestDesc + desc := p.manifest.RootManifestDesc k, err := mainManifestKey(desc, p.Platform, p.layerLimit) if err != nil { return struct{}{}, err } p.manifestKey = k.String() + k, err = mainManifestKey(p.manifest.ImageManifestDesc, p.Platform, p.layerLimit) + if err != nil { + return struct{}{}, err + } + p.platformManifestKey = k.String() // Do not use RootFS DiffIDs from the image config as an alternate // cache key. They are verified only when layers are applied, and a // failed malicious pull must not poison another image source. @@ -193,7 +199,14 @@ func (p *puller) CacheKey(ctx context.Context, jobCtx solver.JobContext, index i cacheOpts[cache.DescHandlerKey(dgst)] = descHandler } - return p.manifestKey, p.manifest.MainManifestDesc.Digest.String(), cacheOpts, true, nil + // Keep the index key for existing caches, but also match the selected + // manifest so changes to attestations or other platforms do not invalidate + // this source. Both keys identify verified manifest content, not DiffIDs + // advertised by an unverified image config. + if index == 0 { + return p.manifestKey, p.manifest.RootManifestDesc.Digest.String(), cacheOpts, p.manifestKey == p.platformManifestKey, nil + } + return p.platformManifestKey, p.manifest.RootManifestDesc.Digest.String(), cacheOpts, true, nil } func (p *puller) Snapshot(ctx context.Context, jobCtx solver.JobContext) (ir cache.ImmutableRef, err error) { diff --git a/vendor/github.com/moby/buildkit/source/git/bundle.go b/vendor/github.com/moby/buildkit/source/git/bundle.go index ff9e7afde8..d39af82940 100644 --- a/vendor/github.com/moby/buildkit/source/git/bundle.go +++ b/vendor/github.com/moby/buildkit/source/git/bundle.go @@ -72,8 +72,8 @@ func bundleTargetRef(ref string) string { // sha256 object IDs. git ls-remote reads bundle files directly, so this works // before a destination repo exists and lets callers initialize the right object // format for a subsequent fetch/import. -func detectBundleSHA256(ctx context.Context, bundlePath string, gitAdvice bool) (bool, error) { - buf, err := gitCLI(gitutil.WithGitAdvice(gitAdvice)).Run(ctx, "ls-remote", "--", bundlePath) +func detectBundleSHA256(ctx context.Context, bundlePath string, gitAdvice, debugCommands bool) (bool, error) { + buf, err := gitCLI(gitutil.WithGitAdvice(gitAdvice), gitutil.WithGitDebugCommands(debugCommands)).Run(ctx, "ls-remote", "--", bundlePath) if err != nil { return false, errors.Wrapf(err, "failed to inspect git bundle %s", bundlePath) } @@ -120,7 +120,7 @@ func (gs *gitSourceHandler) stageBundle(ctx context.Context, g session.Group) (_ return "", nil, err } bundlePath := filepath.Join(tmpDir, bundleImportFileName) - sha256, err := detectBundleSHA256(ctx, bundlePath, gs.src.Advice) + sha256, err := detectBundleSHA256(ctx, bundlePath, gs.src.Advice, gs.src.DebugCommands) if err != nil { return "", nil, err } @@ -133,7 +133,7 @@ func (gs *gitSourceHandler) stageBundle(ctx context.Context, g session.Group) (_ if err := os.Mkdir(tmpRepoDir, 0700); err != nil { return "", nil, errors.Wrap(err, "failed to create temp bare repo dir") } - tmpGit := gitCLI(gitutil.WithGitAdvice(gs.src.Advice), gitutil.WithGitDir(tmpRepoDir)) + tmpGit := gitCLI(gitutil.WithGitAdvice(gs.src.Advice), gitutil.WithGitDebugCommands(gs.src.DebugCommands), gitutil.WithGitDir(tmpRepoDir)) initArgs := []string{"-c", "init.defaultBranch=master", "init", "--bare"} if sha256 { initArgs = append(initArgs, "--object-format=sha256") diff --git a/vendor/github.com/moby/buildkit/source/git/identifier.go b/vendor/github.com/moby/buildkit/source/git/identifier.go index 0ee27ed585..a41b35e227 100644 --- a/vendor/github.com/moby/buildkit/source/git/identifier.go +++ b/vendor/github.com/moby/buildkit/source/git/identifier.go @@ -29,6 +29,7 @@ type GitIdentifier struct { SkipSubmodules bool MTime string // "checkout" (default) or "commit" Advice bool + DebugCommands bool FetchByCommit bool // Bundle, when non-empty, instructs the git source to fetch commits from a diff --git a/vendor/github.com/moby/buildkit/source/git/source.go b/vendor/github.com/moby/buildkit/source/git/source.go index 73a08ac908..f798b6fba8 100644 --- a/vendor/github.com/moby/buildkit/source/git/source.go +++ b/vendor/github.com/moby/buildkit/source/git/source.go @@ -150,6 +150,8 @@ func (gs *Source) Identifier(scheme, ref string, attrs map[string]string, platfo id.MTime = v case pb.AttrGitAdvice: id.Advice = v == "true" + case pb.AttrGitDebugCommands: + id.DebugCommands = v == "true" case pb.AttrGitFetchByCommit: id.FetchByCommit = v == "true" case pb.AttrGitBundle: @@ -173,7 +175,7 @@ func (gs *Source) Identifier(scheme, ref string, attrs map[string]string, platfo } // needs to be called with repo lock -func (gs *Source) mountRemote(ctx context.Context, remote string, authArgs []string, gitAdvice bool, sha256 bool, reset bool, g session.Group) (target string, release func() error, retErr error) { +func (gs *Source) mountRemote(ctx context.Context, remote string, authArgs []string, gitAdvice bool, debugCommands bool, sha256 bool, reset bool, g session.Group) (target string, release func() error, retErr error) { sis, err := searchGitRemote(ctx, gs.cache, remote) if err != nil { return "", nil, errors.Wrapf(err, "failed to search metadata for %s", urlutil.RedactCredentials(remote)) @@ -237,6 +239,7 @@ func (gs *Source) mountRemote(ctx context.Context, remote string, authArgs []str git := gitCLI( gitutil.WithGitAdvice(gitAdvice), + gitutil.WithGitDebugCommands(debugCommands), gitutil.WithGitDir(dir), gitutil.WithArgs(authArgs...), ) @@ -976,7 +979,7 @@ func (gs *gitSourceHandler) tryRemoteFetch(ctx context.Context, jobCtx solver.Jo } } - gitDir, unmountGitDir, err := gs.mountRemote(ctx, gs.src.Remote, authArgs, gs.src.Advice, gs.sha256, reset, g) + gitDir, unmountGitDir, err := gs.mountRemote(ctx, gs.src.Remote, authArgs, gs.src.Advice, gs.src.DebugCommands, gs.sha256, reset, g) if err != nil { return nil, err } @@ -1534,6 +1537,7 @@ func (gs *gitSourceHandler) emptyGitCli(ctx context.Context, g session.Group, op opts = append([]gitutil.Option{ gitutil.WithGitAdvice(gs.src.Advice), + gitutil.WithGitDebugCommands(gs.src.DebugCommands), gitutil.WithArgs(gs.authArgs...), gitutil.WithSSHAuthSock(sock), gitutil.WithSSHKnownHosts(knownHosts), diff --git a/vendor/github.com/moby/buildkit/source/http/source.go b/vendor/github.com/moby/buildkit/source/http/source.go index 858805358d..5495c55a6e 100644 --- a/vendor/github.com/moby/buildkit/source/http/source.go +++ b/vendor/github.com/moby/buildkit/source/http/source.go @@ -8,6 +8,7 @@ import ( "encoding/json" "fmt" "io" + "math" "mime" "net/http" "net/url" @@ -31,6 +32,7 @@ import ( "github.com/moby/buildkit/util/bklog" "github.com/moby/buildkit/util/cachedigest" "github.com/moby/buildkit/util/pgpsign" + "github.com/moby/buildkit/util/resolver/retryhandler" "github.com/moby/buildkit/util/tracing" "github.com/moby/buildkit/util/urlutil" "github.com/moby/buildkit/version" @@ -215,6 +217,45 @@ func (hs *httpSourceHandler) client(g session.Group) *http.Client { return &http.Client{Transport: newTransport(hs.transport, hs.sm, g)} } +// doRequest retries transient network failures and HTTP 429/503 responses with +// jittered exponential backoff, honoring Retry-After as a minimum delay. +// Requests have no body and are safe to replay. Failures reading a successful +// response body and other HTTP statuses are left to the caller. +func doRequest(ctx context.Context, client *http.Client, req *http.Request) (*http.Response, error) { + return retryhandler.WithRetry(ctx, nil, func(ctx context.Context) (*http.Response, error) { + resp, err := client.Do(req.Clone(ctx)) + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusTooManyRequests && resp.StatusCode != http.StatusServiceUnavailable { + return resp, nil + } + // Do not drain an error body: it may stall independently of the headers. + resp.Body.Close() + return nil, &retryhandler.RetryAfterError{ + Err: errors.Errorf("invalid response status %d", resp.StatusCode), + Delay: retryAfterDelay(resp.Header.Get("Retry-After")), + } + }) +} + +// retryAfterDelay parses either delay-seconds or an HTTP-date. Invalid and past +// values fall back to the normal backoff. Saturate large values to avoid overflow. +func retryAfterDelay(value string) time.Duration { + value = strings.TrimSpace(value) + seconds, err := strconv.ParseUint(value, 10, 64) + if err == nil || errors.Is(err, strconv.ErrRange) { + if seconds > uint64(math.MaxInt64/int64(time.Second)) { + return time.Duration(math.MaxInt64) + } + return time.Duration(seconds) * time.Second + } + if date, err := http.ParseTime(value); err == nil { + return max(time.Until(date), 0) + } + return 0 +} + // urlHash is internal hash the etag is stored by that doesn't leak outside // this package. func (hs *httpSourceHandler) urlHash() (digest.Digest, error) { @@ -408,6 +449,7 @@ func (hs *httpSourceHandler) resolveMetadataRef(ctx context.Context, jobCtx solv // we need to add accept-encoding header manually because stdlib only adds it to GET requests // some servers will return different etags if Accept-Encoding header is different req.Header.Set("Accept-Encoding", "gzip") + // Best-effort: on any error we fall through to the GET below, which is retried. resp, err := client.Do(req) if err == nil { if resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusNotModified { @@ -454,7 +496,7 @@ func (hs *httpSourceHandler) resolveMetadataRef(ctx context.Context, jobCtx solv req.Header.Del("Accept-Encoding") } - resp, err := client.Do(req) + resp, err := doRequest(ctx, client, req) if err != nil { return nil, err } @@ -889,7 +931,7 @@ func (hs *httpSourceHandler) Snapshot(ctx context.Context, jobCtx solver.JobCont client := hs.client(g) - resp, err := client.Do(req) + resp, err := doRequest(ctx, client, req) if err != nil { return nil, err } diff --git a/vendor/github.com/moby/buildkit/util/compression/go126flate/LICENSE b/vendor/github.com/moby/buildkit/util/compression/go126flate/LICENSE new file mode 100644 index 0000000000..2a7cf70da6 --- /dev/null +++ b/vendor/github.com/moby/buildkit/util/compression/go126flate/LICENSE @@ -0,0 +1,27 @@ +Copyright 2009 The Go Authors. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google LLC nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/moby/buildkit/util/compression/go126flate/deflate.go b/vendor/github.com/moby/buildkit/util/compression/go126flate/deflate.go new file mode 100644 index 0000000000..1d173de887 --- /dev/null +++ b/vendor/github.com/moby/buildkit/util/compression/go126flate/deflate.go @@ -0,0 +1,747 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Code generated from Go 1.26.5 commit c19862e5f8415b4f24b189d065ed739517c548ba; DO NOT EDIT. + +package go126flate + +import ( + "errors" + "fmt" + "io" + "math" +) + +const ( + NoCompression = 0 + BestSpeed = 1 + BestCompression = 9 + DefaultCompression = -1 + + // HuffmanOnly disables Lempel-Ziv match searching and only performs Huffman + // entropy encoding. This mode is useful in compressing data that has + // already been compressed with an LZ style algorithm (e.g. Snappy or LZ4) + // that lacks an entropy encoder. Compression gains are achieved when + // certain bytes in the input stream occur more frequently than others. + // + // Note that HuffmanOnly produces a compressed output that is + // RFC 1951 compliant. That is, any valid DEFLATE decompressor will + // continue to be able to decompress this output. + HuffmanOnly = -2 +) + +const ( + logWindowSize = 15 + windowSize = 1 << logWindowSize + windowMask = windowSize - 1 + + // The LZ77 step produces a sequence of literal tokens and + // pair tokens. The offset is also known as distance. The underlying wire + // format limits the range of lengths and offsets. For example, there are + // 256 legitimate lengths: those in the range [3, 258]. This package's + // compressor uses a higher minimum match length, enabling optimizations + // such as finding matches via 32-bit loads and compares. + baseMatchLength = 3 // The smallest match length per the RFC section 3.2.5 + minMatchLength = 4 // The smallest match length that the compressor actually emits + maxMatchLength = 258 // The largest match length + baseMatchOffset = 1 // The smallest match offset + maxMatchOffset = 1 << 15 // The largest match offset + + // The maximum number of tokens we put into a single flate block, just to + // stop things from getting too large. + maxFlateBlockTokens = 1 << 14 + maxStoreBlockSize = 65535 + hashBits = 17 // After 17 performance degrades + hashSize = 1 << hashBits + hashMask = (1 << hashBits) - 1 + maxHashOffset = 1 << 24 + + skipNever = math.MaxInt32 +) + +type compressionLevel struct { + level, good, lazy, nice, chain, fastSkipHashing int +} + +var levels = []compressionLevel{ + {0, 0, 0, 0, 0, 0}, // NoCompression. + {1, 0, 0, 0, 0, 0}, // BestSpeed uses a custom algorithm; see deflatefast.go. + // For levels 2-3 we don't bother trying with lazy matches. + {2, 4, 0, 16, 8, 5}, + {3, 4, 0, 32, 32, 6}, + // Levels 4-9 use increasingly more lazy matching + // and increasingly stringent conditions for "good enough". + {4, 4, 4, 16, 16, skipNever}, + {5, 8, 16, 32, 32, skipNever}, + {6, 8, 16, 128, 128, skipNever}, + {7, 8, 32, 128, 256, skipNever}, + {8, 32, 128, 258, 1024, skipNever}, + {9, 32, 258, 258, 4096, skipNever}, +} + +type compressor struct { + compressionLevel + + w *huffmanBitWriter + bulkHasher func([]byte, []uint32) + + // compression algorithm + fill func(*compressor, []byte) int // copy data to window + step func(*compressor) // process window + bestSpeed *deflateFast // Encoder for BestSpeed + + // input window: unprocessed data is window[index:windowEnd] + index int + window []byte + windowEnd int + blockStart int // window index where current tokens start + byteAvailable bool // if true, still need to process window[index-1]. + + sync bool // requesting flush + + // queued output tokens + tokens []token + + // deflate state + length int + offset int + maxInsertIndex int + err error + + // Input hash chains + // hashHead[hashValue] contains the largest inputIndex with the specified hash value + // If hashHead[hashValue] is within the current window, then + // hashPrev[hashHead[hashValue] & windowMask] contains the previous index + // with the same hash value. + // These are large and do not contain pointers, so put them + // near the end of the struct so the GC has to scan less. + chainHead int + hashHead [hashSize]uint32 + hashPrev [windowSize]uint32 + hashOffset int + + // hashMatch must be able to contain hashes for the maximum match length. + hashMatch [maxMatchLength - 1]uint32 +} + +func (d *compressor) fillDeflate(b []byte) int { + if d.index >= 2*windowSize-(minMatchLength+maxMatchLength) { + // shift the window by windowSize + copy(d.window, d.window[windowSize:2*windowSize]) + d.index -= windowSize + d.windowEnd -= windowSize + if d.blockStart >= windowSize { + d.blockStart -= windowSize + } else { + d.blockStart = math.MaxInt32 + } + d.hashOffset += windowSize + if d.hashOffset > maxHashOffset { + delta := d.hashOffset - 1 + d.hashOffset -= delta + d.chainHead -= delta + + // Iterate over slices instead of arrays to avoid copying + // the entire table onto the stack (Issue #18625). + for i, v := range d.hashPrev[:] { + if int(v) > delta { + d.hashPrev[i] = uint32(int(v) - delta) + } else { + d.hashPrev[i] = 0 + } + } + for i, v := range d.hashHead[:] { + if int(v) > delta { + d.hashHead[i] = uint32(int(v) - delta) + } else { + d.hashHead[i] = 0 + } + } + } + } + n := copy(d.window[d.windowEnd:], b) + d.windowEnd += n + return n +} + +func (d *compressor) writeBlock(tokens []token, index int) error { + if index > 0 { + var window []byte + if d.blockStart <= index { + window = d.window[d.blockStart:index] + } + d.blockStart = index + d.w.writeBlock(tokens, false, window) + return d.w.err + } + return nil +} + +// fillWindow will fill the current window with the supplied +// dictionary and calculate all hashes. +// This is much faster than doing a full encode. +// Should only be used after a reset. +func (d *compressor) fillWindow(b []byte) { + // Do not fill window if we are in store-only mode. + if d.compressionLevel.level < 2 { + return + } + if d.index != 0 || d.windowEnd != 0 { + panic("internal error: fillWindow called with stale data") + } + + // If we are given too much, cut it. + if len(b) > windowSize { + b = b[len(b)-windowSize:] + } + // Add all to window. + n := copy(d.window, b) + + // Calculate 256 hashes at the time (more L1 cache hits) + loops := (n + 256 - minMatchLength) / 256 + for j := 0; j < loops; j++ { + index := j * 256 + end := index + 256 + minMatchLength - 1 + if end > n { + end = n + } + toCheck := d.window[index:end] + dstSize := len(toCheck) - minMatchLength + 1 + + if dstSize <= 0 { + continue + } + + dst := d.hashMatch[:dstSize] + d.bulkHasher(toCheck, dst) + for i, val := range dst { + di := i + index + hh := &d.hashHead[val&hashMask] + // Get previous value with the same hash. + // Our chain should point to the previous value. + d.hashPrev[di&windowMask] = *hh + // Set the head of the hash chain to us. + *hh = uint32(di + d.hashOffset) + } + } + // Update window information. + d.windowEnd = n + d.index = n +} + +// Try to find a match starting at index whose length is greater than prevSize. +// We only look at chainCount possibilities before giving up. +func (d *compressor) findMatch(pos int, prevHead int, prevLength int, lookahead int) (length, offset int, ok bool) { + minMatchLook := maxMatchLength + if lookahead < minMatchLook { + minMatchLook = lookahead + } + + win := d.window[0 : pos+minMatchLook] + + // We quit when we get a match that's at least nice long + nice := len(win) - pos + if d.nice < nice { + nice = d.nice + } + + // If we've got a match that's good enough, only look in 1/4 the chain. + tries := d.chain + length = prevLength + if length >= d.good { + tries >>= 2 + } + + wEnd := win[pos+length] + wPos := win[pos:] + minIndex := pos - windowSize + + for i := prevHead; tries > 0; tries-- { + if wEnd == win[i+length] { + n := matchLen(win[i:], wPos, minMatchLook) + + if n > length && (n > minMatchLength || pos-i <= 4096) { + length = n + offset = pos - i + ok = true + if n >= nice { + // The match is good enough that we don't try to find a better one. + break + } + wEnd = win[pos+n] + } + } + if i == minIndex { + // hashPrev[i & windowMask] has already been overwritten, so stop now. + break + } + i = int(d.hashPrev[i&windowMask]) - d.hashOffset + if i < minIndex || i < 0 { + break + } + } + return +} + +func (d *compressor) writeStoredBlock(buf []byte) error { + if d.w.writeStoredHeader(len(buf), false); d.w.err != nil { + return d.w.err + } + d.w.writeBytes(buf) + return d.w.err +} + +const hashmul = 0x1e35a7bd + +// hash4 returns a hash representation of the first 4 bytes +// of the supplied slice. +// The caller must ensure that len(b) >= 4. +func hash4(b []byte) uint32 { + return ((uint32(b[3]) | uint32(b[2])<<8 | uint32(b[1])<<16 | uint32(b[0])<<24) * hashmul) >> (32 - hashBits) +} + +// bulkHash4 will compute hashes using the same +// algorithm as hash4. +func bulkHash4(b []byte, dst []uint32) { + if len(b) < minMatchLength { + return + } + hb := uint32(b[3]) | uint32(b[2])<<8 | uint32(b[1])<<16 | uint32(b[0])<<24 + dst[0] = (hb * hashmul) >> (32 - hashBits) + end := len(b) - minMatchLength + 1 + for i := 1; i < end; i++ { + hb = (hb << 8) | uint32(b[i+3]) + dst[i] = (hb * hashmul) >> (32 - hashBits) + } +} + +// matchLen returns the number of matching bytes in a and b +// up to length 'max'. Both slices must be at least 'max' +// bytes in size. +func matchLen(a, b []byte, max int) int { + a = a[:max] + b = b[:len(a)] + for i, av := range a { + if b[i] != av { + return i + } + } + return max +} + +// encSpeed will compress and store the currently added data, +// if enough has been accumulated or we at the end of the stream. +// Any error that occurred will be in d.err +func (d *compressor) encSpeed() { + // We only compress if we have maxStoreBlockSize. + if d.windowEnd < maxStoreBlockSize { + if !d.sync { + return + } + + // Handle small sizes. + if d.windowEnd < 128 { + switch { + case d.windowEnd == 0: + return + case d.windowEnd <= 16: + d.err = d.writeStoredBlock(d.window[:d.windowEnd]) + default: + d.w.writeBlockHuff(false, d.window[:d.windowEnd]) + d.err = d.w.err + } + d.windowEnd = 0 + d.bestSpeed.reset() + return + } + + } + // Encode the block. + d.tokens = d.bestSpeed.encode(d.tokens[:0], d.window[:d.windowEnd]) + + // If we removed less than 1/16th, Huffman compress the block. + if len(d.tokens) > d.windowEnd-(d.windowEnd>>4) { + d.w.writeBlockHuff(false, d.window[:d.windowEnd]) + } else { + d.w.writeBlockDynamic(d.tokens, false, d.window[:d.windowEnd]) + } + d.err = d.w.err + d.windowEnd = 0 +} + +func (d *compressor) initDeflate() { + d.window = make([]byte, 2*windowSize) + d.hashOffset = 1 + d.tokens = make([]token, 0, maxFlateBlockTokens+1) + d.length = minMatchLength - 1 + d.offset = 0 + d.byteAvailable = false + d.index = 0 + d.chainHead = -1 + d.bulkHasher = bulkHash4 +} + +func (d *compressor) deflate() { + if d.windowEnd-d.index < minMatchLength+maxMatchLength && !d.sync { + return + } + + d.maxInsertIndex = d.windowEnd - (minMatchLength - 1) + +Loop: + for { + if d.index > d.windowEnd { + panic("index > windowEnd") + } + lookahead := d.windowEnd - d.index + if lookahead < minMatchLength+maxMatchLength { + if !d.sync { + break Loop + } + if d.index > d.windowEnd { + panic("index > windowEnd") + } + if lookahead == 0 { + // Flush current output block if any. + if d.byteAvailable { + // There is still one pending token that needs to be flushed + d.tokens = append(d.tokens, literalToken(uint32(d.window[d.index-1]))) + d.byteAvailable = false + } + if len(d.tokens) > 0 { + if d.err = d.writeBlock(d.tokens, d.index); d.err != nil { + return + } + d.tokens = d.tokens[:0] + } + break Loop + } + } + if d.index < d.maxInsertIndex { + // Update the hash + hash := hash4(d.window[d.index : d.index+minMatchLength]) + hh := &d.hashHead[hash&hashMask] + d.chainHead = int(*hh) + d.hashPrev[d.index&windowMask] = uint32(d.chainHead) + *hh = uint32(d.index + d.hashOffset) + } + prevLength := d.length + prevOffset := d.offset + d.length = minMatchLength - 1 + d.offset = 0 + minIndex := d.index - windowSize + if minIndex < 0 { + minIndex = 0 + } + + if d.chainHead-d.hashOffset >= minIndex && + (d.fastSkipHashing != skipNever && lookahead > minMatchLength-1 || + d.fastSkipHashing == skipNever && lookahead > prevLength && prevLength < d.lazy) { + if newLength, newOffset, ok := d.findMatch(d.index, d.chainHead-d.hashOffset, minMatchLength-1, lookahead); ok { + d.length = newLength + d.offset = newOffset + } + } + if d.fastSkipHashing != skipNever && d.length >= minMatchLength || + d.fastSkipHashing == skipNever && prevLength >= minMatchLength && d.length <= prevLength { + // There was a match at the previous step, and the current match is + // not better. Output the previous match. + if d.fastSkipHashing != skipNever { + d.tokens = append(d.tokens, matchToken(uint32(d.length-baseMatchLength), uint32(d.offset-baseMatchOffset))) + } else { + d.tokens = append(d.tokens, matchToken(uint32(prevLength-baseMatchLength), uint32(prevOffset-baseMatchOffset))) + } + // Insert in the hash table all strings up to the end of the match. + // index and index-1 are already inserted. If there is not enough + // lookahead, the last two strings are not inserted into the hash + // table. + if d.length <= d.fastSkipHashing { + var newIndex int + if d.fastSkipHashing != skipNever { + newIndex = d.index + d.length + } else { + newIndex = d.index + prevLength - 1 + } + index := d.index + for index++; index < newIndex; index++ { + if index < d.maxInsertIndex { + hash := hash4(d.window[index : index+minMatchLength]) + // Get previous value with the same hash. + // Our chain should point to the previous value. + hh := &d.hashHead[hash&hashMask] + d.hashPrev[index&windowMask] = *hh + // Set the head of the hash chain to us. + *hh = uint32(index + d.hashOffset) + } + } + d.index = index + + if d.fastSkipHashing == skipNever { + d.byteAvailable = false + d.length = minMatchLength - 1 + } + } else { + // For matches this long, we don't bother inserting each individual + // item into the table. + d.index += d.length + } + if len(d.tokens) == maxFlateBlockTokens { + // The block includes the current character + if d.err = d.writeBlock(d.tokens, d.index); d.err != nil { + return + } + d.tokens = d.tokens[:0] + } + } else { + if d.fastSkipHashing != skipNever || d.byteAvailable { + i := d.index - 1 + if d.fastSkipHashing != skipNever { + i = d.index + } + d.tokens = append(d.tokens, literalToken(uint32(d.window[i]))) + if len(d.tokens) == maxFlateBlockTokens { + if d.err = d.writeBlock(d.tokens, i+1); d.err != nil { + return + } + d.tokens = d.tokens[:0] + } + } + d.index++ + if d.fastSkipHashing == skipNever { + d.byteAvailable = true + } + } + } +} + +func (d *compressor) fillStore(b []byte) int { + n := copy(d.window[d.windowEnd:], b) + d.windowEnd += n + return n +} + +func (d *compressor) store() { + if d.windowEnd > 0 && (d.windowEnd == maxStoreBlockSize || d.sync) { + d.err = d.writeStoredBlock(d.window[:d.windowEnd]) + d.windowEnd = 0 + } +} + +// storeHuff compresses and stores the currently added data +// when the d.window is full or we are at the end of the stream. +// Any error that occurred will be in d.err +func (d *compressor) storeHuff() { + if d.windowEnd < len(d.window) && !d.sync || d.windowEnd == 0 { + return + } + d.w.writeBlockHuff(false, d.window[:d.windowEnd]) + d.err = d.w.err + d.windowEnd = 0 +} + +func (d *compressor) write(b []byte) (n int, err error) { + if d.err != nil { + return 0, d.err + } + n = len(b) + for len(b) > 0 { + d.step(d) + b = b[d.fill(d, b):] + if d.err != nil { + return 0, d.err + } + } + return n, nil +} + +func (d *compressor) syncFlush() error { + if d.err != nil { + return d.err + } + d.sync = true + d.step(d) + if d.err == nil { + d.w.writeStoredHeader(0, false) + d.w.flush() + d.err = d.w.err + } + d.sync = false + return d.err +} + +func (d *compressor) init(w io.Writer, level int) (err error) { + d.w = newHuffmanBitWriter(w) + + switch { + case level == NoCompression: + d.window = make([]byte, maxStoreBlockSize) + d.fill = (*compressor).fillStore + d.step = (*compressor).store + case level == HuffmanOnly: + d.window = make([]byte, maxStoreBlockSize) + d.fill = (*compressor).fillStore + d.step = (*compressor).storeHuff + case level == BestSpeed: + d.compressionLevel = levels[level] + d.window = make([]byte, maxStoreBlockSize) + d.fill = (*compressor).fillStore + d.step = (*compressor).encSpeed + d.bestSpeed = newDeflateFast() + d.tokens = make([]token, maxStoreBlockSize) + case level == DefaultCompression: + level = 6 + fallthrough + case 2 <= level && level <= 9: + d.compressionLevel = levels[level] + d.initDeflate() + d.fill = (*compressor).fillDeflate + d.step = (*compressor).deflate + default: + return fmt.Errorf("flate: invalid compression level %d: want value in range [-2, 9]", level) + } + return nil +} + +func (d *compressor) reset(w io.Writer) { + d.w.reset(w) + d.sync = false + d.err = nil + switch d.compressionLevel.level { + case NoCompression: + d.windowEnd = 0 + case BestSpeed: + d.windowEnd = 0 + d.tokens = d.tokens[:0] + d.bestSpeed.reset() + default: + d.chainHead = -1 + clear(d.hashHead[:]) + clear(d.hashPrev[:]) + d.hashOffset = 1 + d.index, d.windowEnd = 0, 0 + d.blockStart, d.byteAvailable = 0, false + d.tokens = d.tokens[:0] + d.length = minMatchLength - 1 + d.offset = 0 + d.maxInsertIndex = 0 + } +} + +func (d *compressor) close() error { + if d.err == errWriterClosed { + return nil + } + if d.err != nil { + return d.err + } + d.sync = true + d.step(d) + if d.err != nil { + return d.err + } + if d.w.writeStoredHeader(0, true); d.w.err != nil { + return d.w.err + } + d.w.flush() + if d.w.err != nil { + return d.w.err + } + d.err = errWriterClosed + return nil +} + +// NewWriter returns a new [Writer] compressing data at the given level. +// Following zlib, levels range from 1 ([BestSpeed]) to 9 ([BestCompression]); +// higher levels typically run slower but compress more. Level 0 +// ([NoCompression]) does not attempt any compression; it only adds the +// necessary DEFLATE framing. +// Level -1 ([DefaultCompression]) uses the default compression level. +// Level -2 ([HuffmanOnly]) will use Huffman compression only, giving +// a very fast compression for all types of input, but sacrificing considerable +// compression efficiency. +// +// If level is in the range [-2, 9] then the error returned will be nil. +// Otherwise the error returned will be non-nil. +func NewWriter(w io.Writer, level int) (*Writer, error) { + var dw Writer + if err := dw.d.init(w, level); err != nil { + return nil, err + } + return &dw, nil +} + +// NewWriterDict is like [NewWriter] but initializes the new +// [Writer] with a preset dictionary. The returned [Writer] behaves +// as if the dictionary had been written to it without producing +// any compressed output. The compressed data written to w +// can only be decompressed by a reader initialized with the +// same dictionary (see [NewReaderDict]). +func NewWriterDict(w io.Writer, level int, dict []byte) (*Writer, error) { + dw := &dictWriter{w} + zw, err := NewWriter(dw, level) + if err != nil { + return nil, err + } + zw.d.fillWindow(dict) + zw.dict = append(zw.dict, dict...) // duplicate dictionary for Reset method. + return zw, nil +} + +type dictWriter struct { + w io.Writer +} + +func (w *dictWriter) Write(b []byte) (n int, err error) { + return w.w.Write(b) +} + +var errWriterClosed = errors.New("flate: closed writer") + +// A Writer takes data written to it and writes the compressed +// form of that data to an underlying writer (see [NewWriter]). +type Writer struct { + d compressor + dict []byte +} + +// Write writes data to w, which will eventually write the +// compressed form of data to its underlying writer. +func (w *Writer) Write(data []byte) (n int, err error) { + return w.d.write(data) +} + +// Flush flushes any pending data to the underlying writer. +// It is useful mainly in compressed network protocols, to ensure that +// a remote reader has enough data to reconstruct a packet. +// Flush does not return until the data has been written. +// Calling Flush when there is no pending data still causes the [Writer] +// to emit a sync marker of at least 4 bytes. +// If the underlying writer returns an error, Flush returns that error. +// +// In the terminology of the zlib library, Flush is equivalent to Z_SYNC_FLUSH. +func (w *Writer) Flush() error { + // For more about flushing: + // https://www.bolet.org/~pornin/deflate-flush.html + return w.d.syncFlush() +} + +// Close flushes and closes the writer. +func (w *Writer) Close() error { + return w.d.close() +} + +// Reset discards the writer's state and makes it equivalent to +// the result of [NewWriter] or [NewWriterDict] called with dst +// and w's level and dictionary. +func (w *Writer) Reset(dst io.Writer) { + if dw, ok := w.d.w.writer.(*dictWriter); ok { + // w was created with NewWriterDict + dw.w = dst + w.d.reset(dw) + w.d.fillWindow(w.dict) + } else { + // w was created with NewWriter + w.d.reset(dst) + } +} diff --git a/vendor/github.com/moby/buildkit/util/compression/go126flate/deflatefast.go b/vendor/github.com/moby/buildkit/util/compression/go126flate/deflatefast.go new file mode 100644 index 0000000000..dd25cff8fd --- /dev/null +++ b/vendor/github.com/moby/buildkit/util/compression/go126flate/deflatefast.go @@ -0,0 +1,309 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Code generated from Go 1.26.5 commit c19862e5f8415b4f24b189d065ed739517c548ba; DO NOT EDIT. + +package go126flate + +import "math" + +// This encoding algorithm, which prioritizes speed over output size, is +// based on Snappy's LZ77-style encoder: github.com/golang/snappy + +const ( + tableBits = 14 // Bits used in the table. + tableSize = 1 << tableBits // Size of the table. + tableMask = tableSize - 1 // Mask for table indices. Redundant, but can eliminate bounds checks. + tableShift = 32 - tableBits // Right-shift to get the tableBits most significant bits of a uint32. + + // Reset the buffer offset when reaching this. + // Offsets are stored between blocks as int32 values. + // Since the offset we are checking against is at the beginning + // of the buffer, we need to subtract the current and input + // buffer to not risk overflowing the int32. + bufferReset = math.MaxInt32 - maxStoreBlockSize*2 +) + +func load32(b []byte, i int32) uint32 { + b = b[i : i+4 : len(b)] // Help the compiler eliminate bounds checks on the next line. + return uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24 +} + +func load64(b []byte, i int32) uint64 { + b = b[i : i+8 : len(b)] // Help the compiler eliminate bounds checks on the next line. + return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | + uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56 +} + +func hash(u uint32) uint32 { + return (u * 0x1e35a7bd) >> tableShift +} + +// These constants are defined by the Snappy implementation so that its +// assembly implementation can fast-path some 16-bytes-at-a-time copies. They +// aren't necessary in the pure Go implementation, as we don't use those same +// optimizations, but using the same thresholds doesn't really hurt. +const ( + inputMargin = 16 - 1 + minNonLiteralBlockSize = 1 + 1 + inputMargin +) + +type tableEntry struct { + val uint32 // Value at destination + offset int32 +} + +// deflateFast maintains the table for matches, +// and the previous byte block for cross block matching. +type deflateFast struct { + table [tableSize]tableEntry + prev []byte // Previous block, zero length if unknown. + cur int32 // Current match offset. +} + +func newDeflateFast() *deflateFast { + return &deflateFast{cur: maxStoreBlockSize, prev: make([]byte, 0, maxStoreBlockSize)} +} + +// encode encodes a block given in src and appends tokens +// to dst and returns the result. +func (e *deflateFast) encode(dst []token, src []byte) []token { + // Ensure that e.cur doesn't wrap. + if e.cur >= bufferReset { + e.shiftOffsets() + } + + // This check isn't in the Snappy implementation, but there, the caller + // instead of the callee handles this case. + if len(src) < minNonLiteralBlockSize { + e.cur += maxStoreBlockSize + e.prev = e.prev[:0] + return emitLiteral(dst, src) + } + + // sLimit is when to stop looking for offset/length copies. The inputMargin + // lets us use a fast path for emitLiteral in the main loop, while we are + // looking for copies. + sLimit := int32(len(src) - inputMargin) + + // nextEmit is where in src the next emitLiteral should start from. + nextEmit := int32(0) + s := int32(0) + cv := load32(src, s) + nextHash := hash(cv) + + for { + // Copied from the C++ snappy implementation: + // + // Heuristic match skipping: If 32 bytes are scanned with no matches + // found, start looking only at every other byte. If 32 more bytes are + // scanned (or skipped), look at every third byte, etc.. When a match + // is found, immediately go back to looking at every byte. This is a + // small loss (~5% performance, ~0.1% density) for compressible data + // due to more bookkeeping, but for non-compressible data (such as + // JPEG) it's a huge win since the compressor quickly "realizes" the + // data is incompressible and doesn't bother looking for matches + // everywhere. + // + // The "skip" variable keeps track of how many bytes there are since + // the last match; dividing it by 32 (ie. right-shifting by five) gives + // the number of bytes to move ahead for each iteration. + skip := int32(32) + + nextS := s + var candidate tableEntry + for { + s = nextS + bytesBetweenHashLookups := skip >> 5 + nextS = s + bytesBetweenHashLookups + skip += bytesBetweenHashLookups + if nextS > sLimit { + goto emitRemainder + } + candidate = e.table[nextHash&tableMask] + now := load32(src, nextS) + e.table[nextHash&tableMask] = tableEntry{offset: s + e.cur, val: cv} + nextHash = hash(now) + + offset := s - (candidate.offset - e.cur) + if offset > maxMatchOffset || cv != candidate.val { + // Out of range or not matched. + cv = now + continue + } + break + } + + // A 4-byte match has been found. We'll later see if more than 4 bytes + // match. But, prior to the match, src[nextEmit:s] are unmatched. Emit + // them as literal bytes. + dst = emitLiteral(dst, src[nextEmit:s]) + + // Call emitCopy, and then see if another emitCopy could be our next + // move. Repeat until we find no match for the input immediately after + // what was consumed by the last emitCopy call. + // + // If we exit this loop normally then we need to call emitLiteral next, + // though we don't yet know how big the literal will be. We handle that + // by proceeding to the next iteration of the main loop. We also can + // exit this loop via goto if we get close to exhausting the input. + for { + // Invariant: we have a 4-byte match at s, and no need to emit any + // literal bytes prior to s. + + // Extend the 4-byte match as long as possible. + // + s += 4 + t := candidate.offset - e.cur + 4 + l := e.matchLen(s, t, src) + + // matchToken is flate's equivalent of Snappy's emitCopy. (length,offset) + dst = append(dst, matchToken(uint32(l+4-baseMatchLength), uint32(s-t-baseMatchOffset))) + s += l + nextEmit = s + if s >= sLimit { + goto emitRemainder + } + + // We could immediately start working at s now, but to improve + // compression we first update the hash table at s-1 and at s. If + // another emitCopy is not our next move, also calculate nextHash + // at s+1. At least on GOARCH=amd64, these three hash calculations + // are faster as one load64 call (with some shifts) instead of + // three load32 calls. + x := load64(src, s-1) + prevHash := hash(uint32(x)) + e.table[prevHash&tableMask] = tableEntry{offset: e.cur + s - 1, val: uint32(x)} + x >>= 8 + currHash := hash(uint32(x)) + candidate = e.table[currHash&tableMask] + e.table[currHash&tableMask] = tableEntry{offset: e.cur + s, val: uint32(x)} + + offset := s - (candidate.offset - e.cur) + if offset > maxMatchOffset || uint32(x) != candidate.val { + cv = uint32(x >> 8) + nextHash = hash(cv) + s++ + break + } + } + } + +emitRemainder: + if int(nextEmit) < len(src) { + dst = emitLiteral(dst, src[nextEmit:]) + } + e.cur += int32(len(src)) + e.prev = e.prev[:len(src)] + copy(e.prev, src) + return dst +} + +func emitLiteral(dst []token, lit []byte) []token { + for _, v := range lit { + dst = append(dst, literalToken(uint32(v))) + } + return dst +} + +// matchLen returns the match length between src[s:] and src[t:]. +// t can be negative to indicate the match is starting in e.prev. +// We assume that src[s-4:s] and src[t-4:t] already match. +func (e *deflateFast) matchLen(s, t int32, src []byte) int32 { + s1 := int(s) + maxMatchLength - 4 + if s1 > len(src) { + s1 = len(src) + } + + // If we are inside the current block + if t >= 0 { + b := src[t:] + a := src[s:s1] + b = b[:len(a)] + // Extend the match to be as long as possible. + for i := range a { + if a[i] != b[i] { + return int32(i) + } + } + return int32(len(a)) + } + + // We found a match in the previous block. + tp := int32(len(e.prev)) + t + if tp < 0 { + return 0 + } + + // Extend the match to be as long as possible. + a := src[s:s1] + b := e.prev[tp:] + if len(b) > len(a) { + b = b[:len(a)] + } + a = a[:len(b)] + for i := range b { + if a[i] != b[i] { + return int32(i) + } + } + + // If we reached our limit, we matched everything we are + // allowed to in the previous block and we return. + n := int32(len(b)) + if int(s+n) == s1 { + return n + } + + // Continue looking for more matches in the current block. + a = src[s+n : s1] + b = src[:len(a)] + for i := range a { + if a[i] != b[i] { + return int32(i) + n + } + } + return int32(len(a)) + n +} + +// Reset resets the encoding history. +// This ensures that no matches are made to the previous block. +func (e *deflateFast) reset() { + e.prev = e.prev[:0] + // Bump the offset, so all matches will fail distance check. + // Nothing should be >= e.cur in the table. + e.cur += maxMatchOffset + + // Protect against e.cur wraparound. + if e.cur >= bufferReset { + e.shiftOffsets() + } +} + +// shiftOffsets will shift down all match offset. +// This is only called in rare situations to prevent integer overflow. +// +// See https://golang.org/issue/18636 and https://github.com/golang/go/issues/34121. +func (e *deflateFast) shiftOffsets() { + if len(e.prev) == 0 { + // We have no history; just clear the table. + clear(e.table[:]) + e.cur = maxMatchOffset + 1 + return + } + + // Shift down everything in the table that isn't already too far away. + for i := range e.table[:] { + v := e.table[i].offset - e.cur + maxMatchOffset + 1 + if v < 0 { + // We want to reset e.cur to maxMatchOffset + 1, so we need to shift + // all table entries down by (e.cur - (maxMatchOffset + 1)). + // Because we ignore matches > maxMatchOffset, we can cap + // any negative offsets at 0. + v = 0 + } + e.table[i].offset = v + } + e.cur = maxMatchOffset + 1 +} diff --git a/vendor/github.com/moby/buildkit/util/compression/go126flate/doc.go b/vendor/github.com/moby/buildkit/util/compression/go126flate/doc.go new file mode 100644 index 0000000000..32958954cc --- /dev/null +++ b/vendor/github.com/moby/buildkit/util/compression/go126flate/doc.go @@ -0,0 +1,20 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package go126flate contains the DEFLATE writer from Go 1.26.5. +// +// The sources are copied from Go commit +// c19862e5f8415b4f24b189d065ed739517c548ba. Go 1.27 changed the encoded +// output while improving compression speed in CL 707355 for golang/go#75532. +// BuildKit keeps this writer to preserve the digests of gzip-compressed layers. +package go126flate + +// maxNumLit comes from RFC 1951 section 3.2.7. It normally lives in the Go +// flate decoder, which is intentionally not included in this writer-only copy. +const maxNumLit = 286 + +// InternalError reports an error in the flate code itself. +type InternalError string + +func (e InternalError) Error() string { return "flate: internal error: " + string(e) } diff --git a/vendor/github.com/moby/buildkit/util/compression/go126flate/huffman_bit_writer.go b/vendor/github.com/moby/buildkit/util/compression/go126flate/huffman_bit_writer.go new file mode 100644 index 0000000000..62b48cc1ce --- /dev/null +++ b/vendor/github.com/moby/buildkit/util/compression/go126flate/huffman_bit_writer.go @@ -0,0 +1,695 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Code generated from Go 1.26.5 commit c19862e5f8415b4f24b189d065ed739517c548ba; DO NOT EDIT. + +package go126flate + +import ( + "io" +) + +const ( + // The largest offset code. + offsetCodeCount = 30 + + // The special code used to mark the end of a block. + endBlockMarker = 256 + + // The first length code. + lengthCodesStart = 257 + + // The number of codegen codes. + codegenCodeCount = 19 + badCode = 255 + + // bufferFlushSize indicates the buffer size + // after which bytes are flushed to the writer. + // Should preferably be a multiple of 6, since + // we accumulate 6 bytes between writes to the buffer. + bufferFlushSize = 240 + + // bufferSize is the actual output byte buffer size. + // It must have additional headroom for a flush + // which can contain up to 8 bytes. + bufferSize = bufferFlushSize + 8 +) + +// The number of extra bits needed by length code X - LENGTH_CODES_START. +var lengthExtraBits = []int8{ + /* 257 */ 0, 0, 0, + /* 260 */ 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, + /* 270 */ 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, + /* 280 */ 4, 5, 5, 5, 5, 0, +} + +// The length indicated by length code X - LENGTH_CODES_START. +var lengthBase = []uint32{ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 10, + 12, 14, 16, 20, 24, 28, 32, 40, 48, 56, + 64, 80, 96, 112, 128, 160, 192, 224, 255, +} + +// offset code word extra bits. +var offsetExtraBits = []int8{ + 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, + 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, + 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, +} + +var offsetBase = []uint32{ + 0x000000, 0x000001, 0x000002, 0x000003, 0x000004, + 0x000006, 0x000008, 0x00000c, 0x000010, 0x000018, + 0x000020, 0x000030, 0x000040, 0x000060, 0x000080, + 0x0000c0, 0x000100, 0x000180, 0x000200, 0x000300, + 0x000400, 0x000600, 0x000800, 0x000c00, 0x001000, + 0x001800, 0x002000, 0x003000, 0x004000, 0x006000, +} + +// The odd order in which the codegen code sizes are written. +var codegenOrder = []uint32{16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15} + +type huffmanBitWriter struct { + // writer is the underlying writer. + // Do not use it directly; use the write method, which ensures + // that Write errors are sticky. + writer io.Writer + + // Data waiting to be written is bytes[0:nbytes] + // and then the low nbits of bits. Data is always written + // sequentially into the bytes array. + bits uint64 + nbits uint + bytes [bufferSize]byte + codegenFreq [codegenCodeCount]int32 + nbytes int + literalFreq []int32 + offsetFreq []int32 + codegen []uint8 + literalEncoding *huffmanEncoder + offsetEncoding *huffmanEncoder + codegenEncoding *huffmanEncoder + err error +} + +func newHuffmanBitWriter(w io.Writer) *huffmanBitWriter { + return &huffmanBitWriter{ + writer: w, + literalFreq: make([]int32, maxNumLit), + offsetFreq: make([]int32, offsetCodeCount), + codegen: make([]uint8, maxNumLit+offsetCodeCount+1), + literalEncoding: newHuffmanEncoder(maxNumLit), + codegenEncoding: newHuffmanEncoder(codegenCodeCount), + offsetEncoding: newHuffmanEncoder(offsetCodeCount), + } +} + +func (w *huffmanBitWriter) reset(writer io.Writer) { + w.writer = writer + w.bits, w.nbits, w.nbytes, w.err = 0, 0, 0, nil +} + +func (w *huffmanBitWriter) flush() { + if w.err != nil { + w.nbits = 0 + return + } + n := w.nbytes + for w.nbits != 0 { + w.bytes[n] = byte(w.bits) + w.bits >>= 8 + if w.nbits > 8 { // Avoid underflow + w.nbits -= 8 + } else { + w.nbits = 0 + } + n++ + } + w.bits = 0 + w.write(w.bytes[:n]) + w.nbytes = 0 +} + +func (w *huffmanBitWriter) write(b []byte) { + if w.err != nil { + return + } + _, w.err = w.writer.Write(b) +} + +func (w *huffmanBitWriter) writeBits(b int32, nb uint) { + if w.err != nil { + return + } + w.bits |= uint64(b) << w.nbits + w.nbits += nb + if w.nbits >= 48 { + bits := w.bits + w.bits >>= 48 + w.nbits -= 48 + n := w.nbytes + bytes := w.bytes[n : n+6] + bytes[0] = byte(bits) + bytes[1] = byte(bits >> 8) + bytes[2] = byte(bits >> 16) + bytes[3] = byte(bits >> 24) + bytes[4] = byte(bits >> 32) + bytes[5] = byte(bits >> 40) + n += 6 + if n >= bufferFlushSize { + w.write(w.bytes[:n]) + n = 0 + } + w.nbytes = n + } +} + +func (w *huffmanBitWriter) writeBytes(bytes []byte) { + if w.err != nil { + return + } + n := w.nbytes + if w.nbits&7 != 0 { + w.err = InternalError("writeBytes with unfinished bits") + return + } + for w.nbits != 0 { + w.bytes[n] = byte(w.bits) + w.bits >>= 8 + w.nbits -= 8 + n++ + } + if n != 0 { + w.write(w.bytes[:n]) + } + w.nbytes = 0 + w.write(bytes) +} + +// RFC 1951 3.2.7 specifies a special run-length encoding for specifying +// the literal and offset lengths arrays (which are concatenated into a single +// array). This method generates that run-length encoding. +// +// The result is written into the codegen array, and the frequencies +// of each code is written into the codegenFreq array. +// Codes 0-15 are single byte codes. Codes 16-18 are followed by additional +// information. Code badCode is an end marker +// +// numLiterals The number of literals in literalEncoding +// numOffsets The number of offsets in offsetEncoding +// litenc, offenc The literal and offset encoder to use +func (w *huffmanBitWriter) generateCodegen(numLiterals int, numOffsets int, litEnc, offEnc *huffmanEncoder) { + clear(w.codegenFreq[:]) + // Note that we are using codegen both as a temporary variable for holding + // a copy of the frequencies, and as the place where we put the result. + // This is fine because the output is always shorter than the input used + // so far. + codegen := w.codegen // cache + // Copy the concatenated code sizes to codegen. Put a marker at the end. + cgnl := codegen[:numLiterals] + for i := range cgnl { + cgnl[i] = uint8(litEnc.codes[i].len) + } + + cgnl = codegen[numLiterals : numLiterals+numOffsets] + for i := range cgnl { + cgnl[i] = uint8(offEnc.codes[i].len) + } + codegen[numLiterals+numOffsets] = badCode + + size := codegen[0] + count := 1 + outIndex := 0 + for inIndex := 1; size != badCode; inIndex++ { + // INVARIANT: We have seen "count" copies of size that have not yet + // had output generated for them. + nextSize := codegen[inIndex] + if nextSize == size { + count++ + continue + } + // We need to generate codegen indicating "count" of size. + if size != 0 { + codegen[outIndex] = size + outIndex++ + w.codegenFreq[size]++ + count-- + for count >= 3 { + n := 6 + if n > count { + n = count + } + codegen[outIndex] = 16 + outIndex++ + codegen[outIndex] = uint8(n - 3) + outIndex++ + w.codegenFreq[16]++ + count -= n + } + } else { + for count >= 11 { + n := 138 + if n > count { + n = count + } + codegen[outIndex] = 18 + outIndex++ + codegen[outIndex] = uint8(n - 11) + outIndex++ + w.codegenFreq[18]++ + count -= n + } + if count >= 3 { + // count >= 3 && count <= 10 + codegen[outIndex] = 17 + outIndex++ + codegen[outIndex] = uint8(count - 3) + outIndex++ + w.codegenFreq[17]++ + count = 0 + } + } + count-- + for ; count >= 0; count-- { + codegen[outIndex] = size + outIndex++ + w.codegenFreq[size]++ + } + // Set up invariant for next time through the loop. + size = nextSize + count = 1 + } + // Marker indicating the end of the codegen. + codegen[outIndex] = badCode +} + +// dynamicSize returns the size of dynamically encoded data in bits. +func (w *huffmanBitWriter) dynamicSize(litEnc, offEnc *huffmanEncoder, extraBits int) (size, numCodegens int) { + numCodegens = len(w.codegenFreq) + for numCodegens > 4 && w.codegenFreq[codegenOrder[numCodegens-1]] == 0 { + numCodegens-- + } + header := 3 + 5 + 5 + 4 + (3 * numCodegens) + + w.codegenEncoding.bitLength(w.codegenFreq[:]) + + int(w.codegenFreq[16])*2 + + int(w.codegenFreq[17])*3 + + int(w.codegenFreq[18])*7 + size = header + + litEnc.bitLength(w.literalFreq) + + offEnc.bitLength(w.offsetFreq) + + extraBits + + return size, numCodegens +} + +// fixedSize returns the size of dynamically encoded data in bits. +func (w *huffmanBitWriter) fixedSize(extraBits int) int { + return 3 + + fixedLiteralEncoding.bitLength(w.literalFreq) + + fixedOffsetEncoding.bitLength(w.offsetFreq) + + extraBits +} + +// storedSize calculates the stored size, including header. +// The function returns the size in bits and whether the block +// fits inside a single block. +func (w *huffmanBitWriter) storedSize(in []byte) (int, bool) { + if in == nil { + return 0, false + } + if len(in) <= maxStoreBlockSize { + return (len(in) + 5) * 8, true + } + return 0, false +} + +func (w *huffmanBitWriter) writeCode(c hcode) { + if w.err != nil { + return + } + w.bits |= uint64(c.code) << w.nbits + w.nbits += uint(c.len) + if w.nbits >= 48 { + bits := w.bits + w.bits >>= 48 + w.nbits -= 48 + n := w.nbytes + bytes := w.bytes[n : n+6] + bytes[0] = byte(bits) + bytes[1] = byte(bits >> 8) + bytes[2] = byte(bits >> 16) + bytes[3] = byte(bits >> 24) + bytes[4] = byte(bits >> 32) + bytes[5] = byte(bits >> 40) + n += 6 + if n >= bufferFlushSize { + w.write(w.bytes[:n]) + n = 0 + } + w.nbytes = n + } +} + +// Write the header of a dynamic Huffman block to the output stream. +// +// numLiterals The number of literals specified in codegen +// numOffsets The number of offsets specified in codegen +// numCodegens The number of codegens used in codegen +func (w *huffmanBitWriter) writeDynamicHeader(numLiterals int, numOffsets int, numCodegens int, isEof bool) { + if w.err != nil { + return + } + var firstBits int32 = 4 + if isEof { + firstBits = 5 + } + w.writeBits(firstBits, 3) + w.writeBits(int32(numLiterals-257), 5) + w.writeBits(int32(numOffsets-1), 5) + w.writeBits(int32(numCodegens-4), 4) + + for i := 0; i < numCodegens; i++ { + value := uint(w.codegenEncoding.codes[codegenOrder[i]].len) + w.writeBits(int32(value), 3) + } + + i := 0 + for { + var codeWord int = int(w.codegen[i]) + i++ + if codeWord == badCode { + break + } + w.writeCode(w.codegenEncoding.codes[uint32(codeWord)]) + + switch codeWord { + case 16: + w.writeBits(int32(w.codegen[i]), 2) + i++ + case 17: + w.writeBits(int32(w.codegen[i]), 3) + i++ + case 18: + w.writeBits(int32(w.codegen[i]), 7) + i++ + } + } +} + +func (w *huffmanBitWriter) writeStoredHeader(length int, isEof bool) { + if w.err != nil { + return + } + var flag int32 + if isEof { + flag = 1 + } + w.writeBits(flag, 3) + w.flush() + w.writeBits(int32(length), 16) + w.writeBits(int32(^uint16(length)), 16) +} + +func (w *huffmanBitWriter) writeFixedHeader(isEof bool) { + if w.err != nil { + return + } + // Indicate that we are a fixed Huffman block + var value int32 = 2 + if isEof { + value = 3 + } + w.writeBits(value, 3) +} + +// writeBlock will write a block of tokens with the smallest encoding. +// The original input can be supplied, and if the huffman encoded data +// is larger than the original bytes, the data will be written as a +// stored block. +// If the input is nil, the tokens will always be Huffman encoded. +func (w *huffmanBitWriter) writeBlock(tokens []token, eof bool, input []byte) { + if w.err != nil { + return + } + + tokens = append(tokens, endBlockMarker) + numLiterals, numOffsets := w.indexTokens(tokens) + + var extraBits int + storedSize, storable := w.storedSize(input) + if storable { + // We only bother calculating the costs of the extra bits required by + // the length of offset fields (which will be the same for both fixed + // and dynamic encoding), if we need to compare those two encodings + // against stored encoding. + for lengthCode := lengthCodesStart + 8; lengthCode < numLiterals; lengthCode++ { + // First eight length codes have extra size = 0. + extraBits += int(w.literalFreq[lengthCode]) * int(lengthExtraBits[lengthCode-lengthCodesStart]) + } + for offsetCode := 4; offsetCode < numOffsets; offsetCode++ { + // First four offset codes have extra size = 0. + extraBits += int(w.offsetFreq[offsetCode]) * int(offsetExtraBits[offsetCode]) + } + } + + // Figure out smallest code. + // Fixed Huffman baseline. + var literalEncoding = fixedLiteralEncoding + var offsetEncoding = fixedOffsetEncoding + var size = w.fixedSize(extraBits) + + // Dynamic Huffman? + var numCodegens int + + // Generate codegen and codegenFrequencies, which indicates how to encode + // the literalEncoding and the offsetEncoding. + w.generateCodegen(numLiterals, numOffsets, w.literalEncoding, w.offsetEncoding) + w.codegenEncoding.generate(w.codegenFreq[:], 7) + dynamicSize, numCodegens := w.dynamicSize(w.literalEncoding, w.offsetEncoding, extraBits) + + if dynamicSize < size { + size = dynamicSize + literalEncoding = w.literalEncoding + offsetEncoding = w.offsetEncoding + } + + // Stored bytes? + if storable && storedSize < size { + w.writeStoredHeader(len(input), eof) + w.writeBytes(input) + return + } + + // Huffman. + if literalEncoding == fixedLiteralEncoding { + w.writeFixedHeader(eof) + } else { + w.writeDynamicHeader(numLiterals, numOffsets, numCodegens, eof) + } + + // Write the tokens. + w.writeTokens(tokens, literalEncoding.codes, offsetEncoding.codes) +} + +// writeBlockDynamic encodes a block using a dynamic Huffman table. +// This should be used if the symbols used have a disproportionate +// histogram distribution. +// If input is supplied and the compression savings are below 1/16th of the +// input size the block is stored. +func (w *huffmanBitWriter) writeBlockDynamic(tokens []token, eof bool, input []byte) { + if w.err != nil { + return + } + + tokens = append(tokens, endBlockMarker) + numLiterals, numOffsets := w.indexTokens(tokens) + + // Generate codegen and codegenFrequencies, which indicates how to encode + // the literalEncoding and the offsetEncoding. + w.generateCodegen(numLiterals, numOffsets, w.literalEncoding, w.offsetEncoding) + w.codegenEncoding.generate(w.codegenFreq[:], 7) + size, numCodegens := w.dynamicSize(w.literalEncoding, w.offsetEncoding, 0) + + // Store bytes, if we don't get a reasonable improvement. + if ssize, storable := w.storedSize(input); storable && ssize < (size+size>>4) { + w.writeStoredHeader(len(input), eof) + w.writeBytes(input) + return + } + + // Write Huffman table. + w.writeDynamicHeader(numLiterals, numOffsets, numCodegens, eof) + + // Write the tokens. + w.writeTokens(tokens, w.literalEncoding.codes, w.offsetEncoding.codes) +} + +// indexTokens indexes a slice of tokens, and updates +// literalFreq and offsetFreq, and generates literalEncoding +// and offsetEncoding. +// The number of literal and offset tokens is returned. +func (w *huffmanBitWriter) indexTokens(tokens []token) (numLiterals, numOffsets int) { + clear(w.literalFreq) + clear(w.offsetFreq) + + for _, t := range tokens { + if t < matchType { + w.literalFreq[t.literal()]++ + continue + } + length := t.length() + offset := t.offset() + w.literalFreq[lengthCodesStart+lengthCode(length)]++ + w.offsetFreq[offsetCode(offset)]++ + } + + // get the number of literals + numLiterals = len(w.literalFreq) + for w.literalFreq[numLiterals-1] == 0 { + numLiterals-- + } + // get the number of offsets + numOffsets = len(w.offsetFreq) + for numOffsets > 0 && w.offsetFreq[numOffsets-1] == 0 { + numOffsets-- + } + if numOffsets == 0 { + // We haven't found a single match. If we want to go with the dynamic encoding, + // we should count at least one offset to be sure that the offset huffman tree could be encoded. + w.offsetFreq[0] = 1 + numOffsets = 1 + } + w.literalEncoding.generate(w.literalFreq, 15) + w.offsetEncoding.generate(w.offsetFreq, 15) + return +} + +// writeTokens writes a slice of tokens to the output. +// codes for literal and offset encoding must be supplied. +func (w *huffmanBitWriter) writeTokens(tokens []token, leCodes, oeCodes []hcode) { + if w.err != nil { + return + } + for _, t := range tokens { + if t < matchType { + w.writeCode(leCodes[t.literal()]) + continue + } + // Write the length + length := t.length() + lengthCode := lengthCode(length) + w.writeCode(leCodes[lengthCode+lengthCodesStart]) + extraLengthBits := uint(lengthExtraBits[lengthCode]) + if extraLengthBits > 0 { + extraLength := int32(length - lengthBase[lengthCode]) + w.writeBits(extraLength, extraLengthBits) + } + // Write the offset + offset := t.offset() + offsetCode := offsetCode(offset) + w.writeCode(oeCodes[offsetCode]) + extraOffsetBits := uint(offsetExtraBits[offsetCode]) + if extraOffsetBits > 0 { + extraOffset := int32(offset - offsetBase[offsetCode]) + w.writeBits(extraOffset, extraOffsetBits) + } + } +} + +// huffOffset is a static offset encoder used for huffman only encoding. +// It can be reused since we will not be encoding offset values. +var huffOffset *huffmanEncoder + +func init() { + offsetFreq := make([]int32, offsetCodeCount) + offsetFreq[0] = 1 + huffOffset = newHuffmanEncoder(offsetCodeCount) + huffOffset.generate(offsetFreq, 15) +} + +// writeBlockHuff encodes a block of bytes as either +// Huffman encoded literals or uncompressed bytes if the +// results only gains very little from compression. +func (w *huffmanBitWriter) writeBlockHuff(eof bool, input []byte) { + if w.err != nil { + return + } + + // Clear histogram + clear(w.literalFreq) + + // Add everything as literals + histogram(input, w.literalFreq) + + w.literalFreq[endBlockMarker] = 1 + + const numLiterals = endBlockMarker + 1 + w.offsetFreq[0] = 1 + const numOffsets = 1 + + w.literalEncoding.generate(w.literalFreq, 15) + + // Figure out smallest code. + // Always use dynamic Huffman or Store + var numCodegens int + + // Generate codegen and codegenFrequencies, which indicates how to encode + // the literalEncoding and the offsetEncoding. + w.generateCodegen(numLiterals, numOffsets, w.literalEncoding, huffOffset) + w.codegenEncoding.generate(w.codegenFreq[:], 7) + size, numCodegens := w.dynamicSize(w.literalEncoding, huffOffset, 0) + + // Store bytes, if we don't get a reasonable improvement. + if ssize, storable := w.storedSize(input); storable && ssize < (size+size>>4) { + w.writeStoredHeader(len(input), eof) + w.writeBytes(input) + return + } + + // Huffman. + w.writeDynamicHeader(numLiterals, numOffsets, numCodegens, eof) + encoding := w.literalEncoding.codes[:257] + n := w.nbytes + for _, t := range input { + // Bitwriting inlined, ~30% speedup + c := encoding[t] + w.bits |= uint64(c.code) << w.nbits + w.nbits += uint(c.len) + if w.nbits < 48 { + continue + } + // Store 6 bytes + bits := w.bits + w.bits >>= 48 + w.nbits -= 48 + bytes := w.bytes[n : n+6] + bytes[0] = byte(bits) + bytes[1] = byte(bits >> 8) + bytes[2] = byte(bits >> 16) + bytes[3] = byte(bits >> 24) + bytes[4] = byte(bits >> 32) + bytes[5] = byte(bits >> 40) + n += 6 + if n < bufferFlushSize { + continue + } + w.write(w.bytes[:n]) + if w.err != nil { + return // Return early in the event of write failures + } + n = 0 + } + w.nbytes = n + w.writeCode(encoding[endBlockMarker]) +} + +// histogram accumulates a histogram of b in h. +// +// len(h) must be >= 256, and h's elements must be all zeroes. +func histogram(b []byte, h []int32) { + h = h[:256] + for _, t := range b { + h[t]++ + } +} diff --git a/vendor/github.com/moby/buildkit/util/compression/go126flate/huffman_code.go b/vendor/github.com/moby/buildkit/util/compression/go126flate/huffman_code.go new file mode 100644 index 0000000000..1af0b06e8f --- /dev/null +++ b/vendor/github.com/moby/buildkit/util/compression/go126flate/huffman_code.go @@ -0,0 +1,347 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Code generated from Go 1.26.5 commit c19862e5f8415b4f24b189d065ed739517c548ba; DO NOT EDIT. + +package go126flate + +import ( + "math" + "math/bits" + "sort" +) + +// hcode is a huffman code with a bit code and bit length. +type hcode struct { + code, len uint16 +} + +type huffmanEncoder struct { + codes []hcode + freqcache []literalNode + bitCount [17]int32 + lns byLiteral // stored to avoid repeated allocation in generate + lfs byFreq // stored to avoid repeated allocation in generate +} + +type literalNode struct { + literal uint16 + freq int32 +} + +// A levelInfo describes the state of the constructed tree for a given depth. +type levelInfo struct { + // Our level. for better printing + level int32 + + // The frequency of the last node at this level + lastFreq int32 + + // The frequency of the next character to add to this level + nextCharFreq int32 + + // The frequency of the next pair (from level below) to add to this level. + // Only valid if the "needed" value of the next lower level is 0. + nextPairFreq int32 + + // The number of chains remaining to generate for this level before moving + // up to the next level + needed int32 +} + +// set sets the code and length of an hcode. +func (h *hcode) set(code uint16, length uint16) { + h.len = length + h.code = code +} + +func maxNode() literalNode { return literalNode{math.MaxUint16, math.MaxInt32} } + +func newHuffmanEncoder(size int) *huffmanEncoder { + return &huffmanEncoder{codes: make([]hcode, size)} +} + +// Generates a HuffmanCode corresponding to the fixed literal table. +func generateFixedLiteralEncoding() *huffmanEncoder { + h := newHuffmanEncoder(maxNumLit) + codes := h.codes + var ch uint16 + for ch = 0; ch < maxNumLit; ch++ { + var bits uint16 + var size uint16 + switch { + case ch < 144: + // size 8, 000110000 .. 10111111 + bits = ch + 48 + size = 8 + case ch < 256: + // size 9, 110010000 .. 111111111 + bits = ch + 400 - 144 + size = 9 + case ch < 280: + // size 7, 0000000 .. 0010111 + bits = ch - 256 + size = 7 + default: + // size 8, 11000000 .. 11000111 + bits = ch + 192 - 280 + size = 8 + } + codes[ch] = hcode{code: reverseBits(bits, byte(size)), len: size} + } + return h +} + +func generateFixedOffsetEncoding() *huffmanEncoder { + h := newHuffmanEncoder(30) + codes := h.codes + for ch := range codes { + codes[ch] = hcode{code: reverseBits(uint16(ch), 5), len: 5} + } + return h +} + +var fixedLiteralEncoding *huffmanEncoder = generateFixedLiteralEncoding() +var fixedOffsetEncoding *huffmanEncoder = generateFixedOffsetEncoding() + +func (h *huffmanEncoder) bitLength(freq []int32) int { + var total int + for i, f := range freq { + if f != 0 { + total += int(f) * int(h.codes[i].len) + } + } + return total +} + +const maxBitsLimit = 16 + +// bitCounts computes the number of literals assigned to each bit size in the Huffman encoding. +// It is only called when list.length >= 3. +// The cases of 0, 1, and 2 literals are handled by special case code. +// +// list is an array of the literals with non-zero frequencies +// and their associated frequencies. The array is in order of increasing +// frequency and has as its last element a special element with frequency +// MaxInt32. +// +// maxBits is the maximum number of bits that should be used to encode any literal. +// It must be less than 16. +// +// bitCounts returns an integer slice in which slice[i] indicates the number of literals +// that should be encoded in i bits. +func (h *huffmanEncoder) bitCounts(list []literalNode, maxBits int32) []int32 { + if maxBits >= maxBitsLimit { + panic("flate: maxBits too large") + } + n := int32(len(list)) + list = list[0 : n+1] + list[n] = maxNode() + + // The tree can't have greater depth than n - 1, no matter what. This + // saves a little bit of work in some small cases + if maxBits > n-1 { + maxBits = n - 1 + } + + // Create information about each of the levels. + // A bogus "Level 0" whose sole purpose is so that + // level1.prev.needed==0. This makes level1.nextPairFreq + // be a legitimate value that never gets chosen. + var levels [maxBitsLimit]levelInfo + // leafCounts[i] counts the number of literals at the left + // of ancestors of the rightmost node at level i. + // leafCounts[i][j] is the number of literals at the left + // of the level j ancestor. + var leafCounts [maxBitsLimit][maxBitsLimit]int32 + + for level := int32(1); level <= maxBits; level++ { + // For every level, the first two items are the first two characters. + // We initialize the levels as if we had already figured this out. + levels[level] = levelInfo{ + level: level, + lastFreq: list[1].freq, + nextCharFreq: list[2].freq, + nextPairFreq: list[0].freq + list[1].freq, + } + leafCounts[level][level] = 2 + if level == 1 { + levels[level].nextPairFreq = math.MaxInt32 + } + } + + // We need a total of 2*n - 2 items at top level and have already generated 2. + levels[maxBits].needed = 2*n - 4 + + level := maxBits + for { + l := &levels[level] + if l.nextPairFreq == math.MaxInt32 && l.nextCharFreq == math.MaxInt32 { + // We've run out of both leaves and pairs. + // End all calculations for this level. + // To make sure we never come back to this level or any lower level, + // set nextPairFreq impossibly large. + l.needed = 0 + levels[level+1].nextPairFreq = math.MaxInt32 + level++ + continue + } + + prevFreq := l.lastFreq + if l.nextCharFreq < l.nextPairFreq { + // The next item on this row is a leaf node. + n := leafCounts[level][level] + 1 + l.lastFreq = l.nextCharFreq + // Lower leafCounts are the same of the previous node. + leafCounts[level][level] = n + l.nextCharFreq = list[n].freq + } else { + // The next item on this row is a pair from the previous row. + // nextPairFreq isn't valid until we generate two + // more values in the level below + l.lastFreq = l.nextPairFreq + // Take leaf counts from the lower level, except counts[level] remains the same. + copy(leafCounts[level][:level], leafCounts[level-1][:level]) + levels[l.level-1].needed = 2 + } + + if l.needed--; l.needed == 0 { + // We've done everything we need to do for this level. + // Continue calculating one level up. Fill in nextPairFreq + // of that level with the sum of the two nodes we've just calculated on + // this level. + if l.level == maxBits { + // All done! + break + } + levels[l.level+1].nextPairFreq = prevFreq + l.lastFreq + level++ + } else { + // If we stole from below, move down temporarily to replenish it. + for levels[level-1].needed > 0 { + level-- + } + } + } + + // Somethings is wrong if at the end, the top level is null or hasn't used + // all of the leaves. + if leafCounts[maxBits][maxBits] != n { + panic("leafCounts[maxBits][maxBits] != n") + } + + bitCount := h.bitCount[:maxBits+1] + bits := 1 + counts := &leafCounts[maxBits] + for level := maxBits; level > 0; level-- { + // chain.leafCount gives the number of literals requiring at least "bits" + // bits to encode. + bitCount[bits] = counts[level] - counts[level-1] + bits++ + } + return bitCount +} + +// Look at the leaves and assign them a bit count and an encoding as specified +// in RFC 1951 3.2.2 +func (h *huffmanEncoder) assignEncodingAndSize(bitCount []int32, list []literalNode) { + code := uint16(0) + for n, bits := range bitCount { + code <<= 1 + if n == 0 || bits == 0 { + continue + } + // The literals list[len(list)-bits] .. list[len(list)-bits] + // are encoded using "bits" bits, and get the values + // code, code + 1, .... The code values are + // assigned in literal order (not frequency order). + chunk := list[len(list)-int(bits):] + + h.lns.sort(chunk) + for _, node := range chunk { + h.codes[node.literal] = hcode{code: reverseBits(code, uint8(n)), len: uint16(n)} + code++ + } + list = list[0 : len(list)-int(bits)] + } +} + +// Update this Huffman Code object to be the minimum code for the specified frequency count. +// +// freq is an array of frequencies, in which freq[i] gives the frequency of literal i. +// maxBits The maximum number of bits to use for any literal. +func (h *huffmanEncoder) generate(freq []int32, maxBits int32) { + if h.freqcache == nil { + // Allocate a reusable buffer with the longest possible frequency table. + // Possible lengths are codegenCodeCount, offsetCodeCount and maxNumLit. + // The largest of these is maxNumLit, so we allocate for that case. + h.freqcache = make([]literalNode, maxNumLit+1) + } + list := h.freqcache[:len(freq)+1] + // Number of non-zero literals + count := 0 + // Set list to be the set of all non-zero literals and their frequencies + for i, f := range freq { + if f != 0 { + list[count] = literalNode{uint16(i), f} + count++ + } else { + h.codes[i].len = 0 + } + } + + list = list[:count] + if count <= 2 { + // Handle the small cases here, because they are awkward for the general case code. With + // two or fewer literals, everything has bit length 1. + for i, node := range list { + // "list" is in order of increasing literal value. + h.codes[node.literal].set(uint16(i), 1) + } + return + } + h.lfs.sort(list) + + // Get the number of literals for each bit count + bitCount := h.bitCounts(list, maxBits) + // And do the assignment + h.assignEncodingAndSize(bitCount, list) +} + +type byLiteral []literalNode + +func (s *byLiteral) sort(a []literalNode) { + *s = byLiteral(a) + sort.Sort(s) +} + +func (s byLiteral) Len() int { return len(s) } + +func (s byLiteral) Less(i, j int) bool { + return s[i].literal < s[j].literal +} + +func (s byLiteral) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +type byFreq []literalNode + +func (s *byFreq) sort(a []literalNode) { + *s = byFreq(a) + sort.Sort(s) +} + +func (s byFreq) Len() int { return len(s) } + +func (s byFreq) Less(i, j int) bool { + if s[i].freq == s[j].freq { + return s[i].literal < s[j].literal + } + return s[i].freq < s[j].freq +} + +func (s byFreq) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +func reverseBits(number uint16, bitLength byte) uint16 { + return bits.Reverse16(number << (16 - bitLength)) +} diff --git a/vendor/github.com/moby/buildkit/util/compression/go126flate/token.go b/vendor/github.com/moby/buildkit/util/compression/go126flate/token.go new file mode 100644 index 0000000000..2f56c25305 --- /dev/null +++ b/vendor/github.com/moby/buildkit/util/compression/go126flate/token.go @@ -0,0 +1,99 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Code generated from Go 1.26.5 commit c19862e5f8415b4f24b189d065ed739517c548ba; DO NOT EDIT. + +package go126flate + +const ( + // 2 bits: type 0 = literal 1=EOF 2=Match 3=Unused + // 8 bits: xlength = length - MIN_MATCH_LENGTH + // 22 bits xoffset = offset - MIN_OFFSET_SIZE, or literal + lengthShift = 22 + offsetMask = 1< pair into a match token. +func matchToken(xlength uint32, xoffset uint32) token { + return token(matchType + xlength<> lengthShift) } + +func lengthCode(len uint32) uint32 { return lengthCodes[len] } + +// Returns the offset code corresponding to a specific offset. +func offsetCode(off uint32) uint32 { + if off < uint32(len(offsetCodes)) { + return offsetCodes[off] + } + if off>>7 < uint32(len(offsetCodes)) { + return offsetCodes[off>>7] + 14 + } + return offsetCodes[off>>14] + 28 +} diff --git a/vendor/github.com/moby/buildkit/util/compression/go126gzip.go b/vendor/github.com/moby/buildkit/util/compression/go126gzip.go new file mode 100644 index 0000000000..dba942c5e8 --- /dev/null +++ b/vendor/github.com/moby/buildkit/util/compression/go126gzip.go @@ -0,0 +1,92 @@ +package compression + +import ( + "encoding/binary" + "hash/crc32" + "io" + + "github.com/moby/buildkit/util/compression/go126flate" + "github.com/pkg/errors" +) + +const go126GzipDefaultCompression = go126flate.DefaultCompression + +// go126GzipWriter writes the fixed gzip framing used for image layers around +// the Go 1.26 DEFLATE stream. Image layers do not use optional gzip headers. +type go126GzipWriter struct { + w io.Writer + level int + wroteHeader bool + closed bool + buf [10]byte + compressor *go126flate.Writer + digest uint32 + size uint32 + err error +} + +func newGo126GzipWriterLevel(w io.Writer, level int) (io.WriteCloser, error) { + if level < go126flate.HuffmanOnly || level > go126flate.BestCompression { + return nil, errors.Errorf("gzip: invalid compression level: %d", level) + } + return &go126GzipWriter{w: w, level: level}, nil +} + +func (z *go126GzipWriter) writeHeader() error { + z.wroteHeader = true + z.buf = [10]byte{0: 0x1f, 1: 0x8b, 2: 8, 9: 255} + switch z.level { + case go126flate.BestCompression: + z.buf[8] = 2 + case go126flate.BestSpeed: + z.buf[8] = 4 + } + if _, err := z.w.Write(z.buf[:10]); err != nil { + return err + } + compressor, err := go126flate.NewWriter(z.w, z.level) + if err != nil { + return errors.WithStack(err) + } + z.compressor = compressor + return nil +} + +func (z *go126GzipWriter) Write(p []byte) (int, error) { + if z.err != nil { + return 0, z.err + } + if !z.wroteHeader { + z.err = z.writeHeader() + if z.err != nil { + return 0, z.err + } + } + z.size += uint32(len(p)) + z.digest = crc32.Update(z.digest, crc32.IEEETable, p) + n, err := z.compressor.Write(p) + z.err = err + return n, err +} + +func (z *go126GzipWriter) Close() error { + if z.err != nil { + return z.err + } + if z.closed { + return nil + } + z.closed = true + if !z.wroteHeader { + if z.err = z.writeHeader(); z.err != nil { + return z.err + } + } + if z.err = z.compressor.Close(); z.err != nil { + return z.err + } + binary.LittleEndian.PutUint32(z.buf[:4], z.digest) + binary.LittleEndian.PutUint32(z.buf[4:8], z.size) + _, z.err = z.w.Write(z.buf[:8]) + return z.err +} diff --git a/vendor/github.com/moby/buildkit/util/compression/gzip.go b/vendor/github.com/moby/buildkit/util/compression/gzip.go index b6135e782c..c8f0180ba5 100644 --- a/vendor/github.com/moby/buildkit/util/compression/gzip.go +++ b/vendor/github.com/moby/buildkit/util/compression/gzip.go @@ -1,7 +1,6 @@ package compression import ( - "compress/gzip" "context" "io" @@ -57,10 +56,13 @@ func (c gzipType) String() string { func gzipWriter(comp Config) func(io.Writer) (io.WriteCloser, error) { return func(dest io.Writer) (io.WriteCloser, error) { - level := gzip.DefaultCompression + level := go126GzipDefaultCompression if comp.Level != nil { level = *comp.Level } - return gzip.NewWriterLevel(dest, level) + // Go 1.27 changed compress/flate output in CL 707355 for + // golang/go#75532. Keep Go 1.26 encoding so existing compatibility + // versions continue to produce the same compressed layer digests. + return newGo126GzipWriterLevel(dest, level) } } diff --git a/vendor/github.com/moby/buildkit/util/gitutil/git_cli.go b/vendor/github.com/moby/buildkit/util/gitutil/git_cli.go index 2e7c658205..e8a00b3265 100644 --- a/vendor/github.com/moby/buildkit/util/gitutil/git_cli.go +++ b/vendor/github.com/moby/buildkit/util/gitutil/git_cli.go @@ -7,8 +7,11 @@ import ( "os" "os/exec" "slices" + "strconv" "strings" + "unicode/utf8" + "github.com/moby/buildkit/util/urlutil" "github.com/pkg/errors" ) @@ -18,10 +21,11 @@ type GitCLI struct { git string exec func(context.Context, *exec.Cmd) error - args []string - dir string - streams StreamFunc - advice *bool + args []string + dir string + streams StreamFunc + advice *bool + debugCommands bool workTree string gitDir string @@ -62,6 +66,13 @@ func WithGitAdvice(enabled bool) Option { } } +// WithGitDebugCommands controls whether executed Git commands are logged. +func WithGitDebugCommands(debugCommands bool) Option { + return func(b *GitCLI) { + b.debugCommands = debugCommands + } +} + // WithDir sets working directory. // // This should be a path to any directory within a standard git repository. @@ -190,22 +201,32 @@ func (cli *GitCLI) Run(ctx context.Context, args ...string) (_ []byte, err error cmd.Stdin = nil cmd.Stdout = buf cmd.Stderr = errbuf + var stdoutStream io.WriteCloser + var stderrStream io.WriteCloser + flush := func() {} if cli.streams != nil { - stdout, stderr, flush := cli.streams(ctx) - if stdout != nil { - cmd.Stdout = io.MultiWriter(stdout, cmd.Stdout) + var flushStream func() + stdoutStream, stderrStream, flushStream = cli.streams(ctx) + if flushStream != nil { + flush = flushStream } - if stderr != nil { - cmd.Stderr = io.MultiWriter(stderr, cmd.Stderr) + if stdoutStream != nil { + cmd.Stdout = io.MultiWriter(stdoutStream, cmd.Stdout) + defer stdoutStream.Close() + } + if stderrStream != nil { + cmd.Stderr = io.MultiWriter(stderrStream, cmd.Stderr) + defer stderrStream.Close() } - defer stdout.Close() - defer stderr.Close() defer func() { if err != nil { flush() } }() } + if cli.debugCommands && stderrStream != nil { + _, _ = io.WriteString(stderrStream, "+ "+formatDebugCommand(cmd.Args)+"\n") + } cmd.Env = []string{ "PATH=" + os.Getenv("PATH"), @@ -297,6 +318,48 @@ func (cli *GitCLI) Run(ctx context.Context, args ...string) (_ []byte, err error } } +func formatDebugCommand(args []string) string { + formatted := make([]string, 0, len(args)) + for i := 0; i < len(args); i++ { + arg := args[i] + if arg == "-c" && i+1 < len(args) { + formatted = append(formatted, arg, quoteDebugCommandArg(redactGitConfig(args[i+1]))) + i++ + continue + } + formatted = append(formatted, quoteDebugCommandArg(redactURLCredentials(arg))) + } + return strings.Join(formatted, " ") +} + +func redactGitConfig(arg string) string { + // URL-scoped keys can contain '=' before the header value separator. + for i := range arg { + if arg[i] == '=' && strings.HasSuffix(strings.ToLower(arg[:i]), ".extraheader") { + return redactURLCredentials(arg[:i]) + "=xxxxx" + } + } + return redactURLCredentials(arg) +} + +func redactURLCredentials(arg string) string { + if !strings.Contains(arg, "://") { + return arg + } + return urlutil.RedactCredentials(arg) +} + +// quoteDebugCommandArg preserves argument boundaries and escapes nonprintable +// characters for display. The output is not intended for shell re-execution. +func quoteDebugCommandArg(arg string) string { + if arg == "" || !utf8.ValidString(arg) || strings.ContainsFunc(arg, func(r rune) bool { + return !strconv.IsPrint(r) || strings.ContainsRune(" \"'\\", r) + }) { + return strconv.Quote(arg) + } + return arg +} + func getGitSSHCommand(knownHosts string) string { gitSSHCommand := "ssh -F " + os.DevNull if knownHosts != "" { diff --git a/vendor/github.com/moby/buildkit/util/progress/progress.go b/vendor/github.com/moby/buildkit/util/progress/progress.go index 2c2d517dba..3ff26455c5 100644 --- a/vendor/github.com/moby/buildkit/util/progress/progress.go +++ b/vendor/github.com/moby/buildkit/util/progress/progress.go @@ -6,6 +6,7 @@ import ( "maps" "slices" "sync" + "sync/atomic" "time" "github.com/pkg/errors" @@ -217,13 +218,13 @@ func newWriter(pw *progressWriter) *progressWriter { } type progressWriter struct { - done bool + done atomic.Bool // set by Close, may race with writes from other goroutines reader *progressReader meta map[string]any } func (pw *progressWriter) Write(id string, v any) error { - if pw.done { + if pw.done.Load() { return errors.Errorf("writing %s to closed progress writer", id) } return pw.writeRawProgress(&Progress{ @@ -251,7 +252,7 @@ func (pw *progressWriter) Close() error { delete(pw.reader.writers, pw) pw.reader.mu.Unlock() pw.reader.cond.Broadcast() - pw.done = true + pw.done.Store(true) return nil } diff --git a/vendor/github.com/moby/buildkit/util/pull/pull.go b/vendor/github.com/moby/buildkit/util/pull/pull.go index 1f5926c8a4..359ac4f0fc 100644 --- a/vendor/github.com/moby/buildkit/util/pull/pull.go +++ b/vendor/github.com/moby/buildkit/util/pull/pull.go @@ -45,12 +45,13 @@ type Puller struct { var _ content.Provider = &provider{} type PulledManifests struct { - Ref string - MainManifestDesc ocispecs.Descriptor - ConfigDesc ocispecs.Descriptor - Nonlayers []ocispecs.Descriptor - Descriptors []ocispecs.Descriptor - Provider func(session.Group) content.Provider + Ref string + RootManifestDesc ocispecs.Descriptor + ImageManifestDesc ocispecs.Descriptor // Platform-specific manifest selected from RootManifestDesc. + ConfigDesc ocispecs.Descriptor + Nonlayers []ocispecs.Descriptor + Descriptors []ocispecs.Descriptor + Provider func(session.Group) content.Provider } func (p *Puller) resolve(ctx context.Context, resolver remotes.Resolver) error { @@ -144,6 +145,18 @@ func (p *Puller) PullManifests(ctx context.Context, getResolver SessionResolver) // Limit manifests pulled to the best match in an index childrenHandler = images.LimitManifests(childrenHandler, platform, 1) + var manifestDesc ocispecs.Descriptor + var manifestOnce sync.Once + manifestHandler := images.HandlerFunc(func(ctx context.Context, desc ocispecs.Descriptor) ([]ocispecs.Descriptor, error) { + if images.IsManifestType(desc.MediaType) { + // Each index yields one child, so the first manifest is the + // selected image, including through nested indexes. Do not let + // malformed config or layer descriptors replace it later. + manifestOnce.Do(func() { manifestDesc = desc }) + } + return nil, nil + }) + dslHandler, err := docker.AppendDistributionSourceLabel(p.ContentStore, p.ref) if err != nil { return nil, err @@ -152,6 +165,7 @@ func (p *Puller) PullManifests(ctx context.Context, getResolver SessionResolver) filterLayerBlobs(metadata, &mu), retryhandler.New(limited.FetchHandler(p.ContentStore, fetcher, p.ref), logs.LoggerFromContext(ctx)), childrenHandler, + manifestHandler, dslHandler, ) @@ -174,11 +188,12 @@ func (p *Puller) PullManifests(ctx context.Context, getResolver SessionResolver) } return &PulledManifests{ - Ref: p.ref, - MainManifestDesc: p.desc, - ConfigDesc: p.configDesc, - Nonlayers: p.nonlayers, - Descriptors: p.layers, + Ref: p.ref, + RootManifestDesc: p.desc, + ImageManifestDesc: manifestDesc, + ConfigDesc: p.configDesc, + Nonlayers: p.nonlayers, + Descriptors: p.layers, Provider: func(g session.Group) content.Provider { return &provider{puller: p, resolver: getResolver(g)} }, diff --git a/vendor/github.com/moby/buildkit/util/resolver/authorizer.go b/vendor/github.com/moby/buildkit/util/resolver/authorizer.go index b1392a8b5f..808c92632b 100644 --- a/vendor/github.com/moby/buildkit/util/resolver/authorizer.go +++ b/vendor/github.com/moby/buildkit/util/resolver/authorizer.go @@ -147,6 +147,13 @@ func (a *dockerAuthorizer) Authorize(ctx context.Context, req *http.Request) err authHeader, err := ah.authorize(ctx, a.sm, a.session) if err != nil { + // containerd attaches a streaming request body (the pipe of a blob + // upload) before authorizing, and does not close it if authorization + // fails. Nothing would then unblock the writer feeding the pipe, and + // the registry concurrency slot held by it would never be released. + if req.Body != nil { + req.Body.Close() + } return err } diff --git a/vendor/github.com/moby/buildkit/util/resolver/retryhandler/retry.go b/vendor/github.com/moby/buildkit/util/resolver/retryhandler/retry.go index 6d2d9c63ac..80e46b94c8 100644 --- a/vendor/github.com/moby/buildkit/util/resolver/retryhandler/retry.go +++ b/vendor/github.com/moby/buildkit/util/resolver/retryhandler/retry.go @@ -5,6 +5,7 @@ import ( stderrors "errors" "fmt" "io" + "math/rand/v2" "net" "syscall" "time" @@ -19,6 +20,27 @@ import ( // variable so that code which embeds BuildKit can override the default value. var MaxRetryBackoff = 8 * time.Second +// MaxRetryAfter is the longest server-requested delay accepted for a retry. +// It is independent of the exponential backoff budget. Code which embeds +// BuildKit can override the default value. +var MaxRetryAfter = 30 * time.Second + +// RetryAfterError marks a failure as retryable after at least Delay. +// A zero Delay uses the normal backoff. Delays above MaxRetryAfter +// stop retries rather than retrying before the requested delay has elapsed. +type RetryAfterError struct { + Err error + Delay time.Duration +} + +func (e *RetryAfterError) Error() string { + return e.Err.Error() +} + +func (e *RetryAfterError) Unwrap() error { + return e.Err +} + func New(f images.HandlerFunc, logger func([]byte)) images.HandlerFunc { return func(ctx context.Context, desc ocispecs.Descriptor) ([]ocispecs.Descriptor, error) { descs, err := WithRetry(ctx, logger, func(ctx context.Context) ([]ocispecs.Descriptor, error) { @@ -33,7 +55,7 @@ func New(f images.HandlerFunc, logger func([]byte)) images.HandlerFunc { type retryContextKey struct{} -// WithRetry retries transient failures with exponential backoff. Nested calls +// WithRetry retries transient failures with jittered exponential backoff. Nested calls // using the supplied context leave retries to the outer call, including New. // f must be safe to retry and preserve retryable errors when wrapping them. func WithRetry[T any](ctx context.Context, logger func([]byte), f func(context.Context) (T, error)) (T, error) { @@ -66,13 +88,21 @@ func WithRetryIf[T any](ctx context.Context, logger func([]byte), f func(context if backoff >= MaxRetryBackoff { return v, err } + // Keep the retry budget independent of jitter, including the first retry. + delay := backoff/2 + rand.N(backoff/2) //nolint:gosec // Jitter spreads load; it does not need cryptographic randomness. + if retryAfter, ok := stderrors.AsType[*RetryAfterError](err); ok { + if retryAfter.Delay > MaxRetryAfter { + return v, err + } + delay = max(delay, retryAfter.Delay) + } if logger != nil { - logger(fmt.Appendf(nil, "retrying in %v\n", backoff)) + logger(fmt.Appendf(nil, "retrying in %v\n", delay)) } select { case <-ctx.Done(): return v, contextFailure(ctx) - case <-time.After(backoff): + case <-time.After(delay): } backoff *= 2 } @@ -88,6 +118,10 @@ func contextFailure(ctx context.Context) error { } func retryError(err error) bool { + if _, ok := stderrors.AsType[*RetryAfterError](err); ok { + return true + } + // Retry on 5xx errors var errUnexpectedStatus remoteserrors.ErrUnexpectedStatus if errors.As(err, &errUnexpectedStatus) && diff --git a/vendor/github.com/moby/buildkit/worker/base/worker.go b/vendor/github.com/moby/buildkit/worker/base/worker.go index b25fd685ab..84d95f7873 100644 --- a/vendor/github.com/moby/buildkit/worker/base/worker.go +++ b/vendor/github.com/moby/buildkit/worker/base/worker.go @@ -743,7 +743,7 @@ func ID(root string) (string, error) { if err != nil { if errors.Is(err, os.ErrNotExist) { id := identity.NewID() - err := os.WriteFile(f, []byte(id), 0400) + err := os.WriteFile(f, []byte(id), 0o600) return id, err } return "", err diff --git a/vendor/github.com/moby/sys/user/idtools.go b/vendor/github.com/moby/sys/user/idtools.go index 595b7a9272..1478153111 100644 --- a/vendor/github.com/moby/sys/user/idtools.go +++ b/vendor/github.com/moby/sys/user/idtools.go @@ -12,37 +12,39 @@ type mkdirOptions struct { onlyNew bool } -// WithOnlyNew is an option for MkdirAllAndChown that will only change ownership and permissions -// on newly created directories. If the directory already exists, it will not be modified +// WithOnlyNew configures [MkdirAllAndChown] and [MkdirAndChown] to leave +// the requested directory unchanged if it already exists. Newly created +// directories are still assigned the requested ownership and permissions. func WithOnlyNew(o *mkdirOptions) { o.onlyNew = true } -// MkdirAllAndChown creates a directory (include any along the path) and then modifies -// ownership to the requested uid/gid. By default, if the directory already exists, this -// function will still change ownership and permissions. If WithOnlyNew is passed as an -// option, then only the newly created directories will have ownership and permissions changed. +// MkdirAllAndChown creates a directory named path, along with any necessary +// parents, and applies the requested ownership and permission bits to all +// directories it creates. +// +// If path already exists as a directory, its ownership and permission bits +// are updated unless [WithOnlyNew] is specified. Existing parent directories +// are not modified. +// +// MkdirAllAndChown is similar to [os.MkdirAll], but also applies ownership and +// reapplies the requested permission bits after creation, so the resulting +// permissions are not affected by the process umask. func MkdirAllAndChown(path string, mode os.FileMode, uid, gid int, opts ...MkdirOpt) error { - var options mkdirOptions - for _, opt := range opts { - opt(&options) - } - - return mkdirAs(path, mode, uid, gid, true, options.onlyNew) + return mkdirAs(path, mode, uid, gid, true, opts...) } -// MkdirAndChown creates a directory and then modifies ownership to the requested uid/gid. -// By default, if the directory already exists, this function still changes ownership and permissions. -// If WithOnlyNew is passed as an option, then only the newly created directory will have ownership -// and permissions changed. -// Note that unlike os.Mkdir(), this function does not return IsExist error -// in case path already exists. +// MkdirAndChown creates a directory named path and applies the requested +// ownership and permission bits. +// +// If path already exists as a directory, its ownership and permission bits +// are updated unless [WithOnlyNew] is specified. +// +// MkdirAndChown is similar to [os.Mkdir], but returns nil if path already +// exists as a directory. The resulting permission bits are not affected by +// the process umask. func MkdirAndChown(path string, mode os.FileMode, uid, gid int, opts ...MkdirOpt) error { - var options mkdirOptions - for _, opt := range opts { - opt(&options) - } - return mkdirAs(path, mode, uid, gid, false, options.onlyNew) + return mkdirAs(path, mode, uid, gid, false, opts...) } // getRootUIDGID retrieves the remapped root uid/gid pair from the set of maps. @@ -106,22 +108,14 @@ func (i IdentityMapping) RootPair() (int, int) { return uid, gid } -// ToHost returns the host UID and GID for the container uid, gid. -// Remapping is only performed if the ids aren't already the remapped root ids +// ToHost returns the host UID and GID for the container uid and gid. +// An empty mapping is treated as identity. func (i IdentityMapping) ToHost(uid, gid int) (int, int, error) { - var err error - ruid, rgid := i.RootPair() - - if uid != ruid { - ruid, err = toHost(uid, i.UIDMaps) - if err != nil { - return ruid, rgid, err - } - } - - if gid != rgid { - rgid, err = toHost(gid, i.GIDMaps) + ruid, err := toHost(uid, i.UIDMaps) + if err != nil { + return -1, -1, err } + rgid, err := toHost(gid, i.GIDMaps) return ruid, rgid, err } diff --git a/vendor/github.com/moby/sys/user/idtools_unix.go b/vendor/github.com/moby/sys/user/idtools_unix.go index 4e39d2446b..679aaa3a26 100644 --- a/vendor/github.com/moby/sys/user/idtools_unix.go +++ b/vendor/github.com/moby/sys/user/idtools_unix.go @@ -10,7 +10,12 @@ import ( "syscall" ) -func mkdirAs(path string, mode os.FileMode, uid, gid int, mkAll, onlyNew bool) error { +func mkdirAs(path string, mode os.FileMode, uid, gid int, mkAll bool, opts ...MkdirOpt) error { + var options mkdirOptions + for _, opt := range opts { + opt(&options) + } + path, err := filepath.Abs(path) if err != nil { return err @@ -21,7 +26,7 @@ func mkdirAs(path string, mode os.FileMode, uid, gid int, mkAll, onlyNew bool) e if !stat.IsDir() { return &os.PathError{Op: "mkdir", Path: path, Err: syscall.ENOTDIR} } - if onlyNew { + if options.onlyNew { return nil } diff --git a/vendor/github.com/moby/sys/user/idtools_windows.go b/vendor/github.com/moby/sys/user/idtools_windows.go index 9de730cafb..3973d5593a 100644 --- a/vendor/github.com/moby/sys/user/idtools_windows.go +++ b/vendor/github.com/moby/sys/user/idtools_windows.go @@ -2,12 +2,25 @@ package user import ( "os" + "syscall" ) -// This is currently a wrapper around [os.MkdirAll] since currently -// permissions aren't set through this path, the identity isn't utilized. -// Ownership is handled elsewhere, but in the future could be support here -// too. -func mkdirAs(path string, _ os.FileMode, _, _ int, _, _ bool) error { - return os.MkdirAll(path, 0) +// mkdirAs creates path, optionally creating any missing parent directories. +// +// On Windows this is currently a thin wrapper around os.Mkdir and +// os.MkdirAll. Unlike the Unix implementation, ownership and permission +// bits are not applied. +func mkdirAs(path string, _ os.FileMode, _, _ int, mkAll bool, _ ...MkdirOpt) error { + if mkAll { + return os.MkdirAll(path, 0) + } + stat, err := os.Stat(path) + if err == nil { + if !stat.IsDir() { + return &os.PathError{Op: "mkdir", Path: path, Err: syscall.ENOTDIR} + } + return nil + } + + return os.Mkdir(path, 0) } diff --git a/vendor/github.com/moby/sys/user/user.go b/vendor/github.com/moby/sys/user/user.go index 857711ebfd..48c34a8b23 100644 --- a/vendor/github.com/moby/sys/user/user.go +++ b/vendor/github.com/moby/sys/user/user.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "os" + "slices" "strconv" "strings" ) @@ -17,9 +18,9 @@ const ( ) var ( - // ErrNoPasswdEntries is returned if no matching entries were found in /etc/group. + // ErrNoPasswdEntries is returned if no matching entries were found in /etc/passwd. ErrNoPasswdEntries = errors.New("no matching entries in passwd file") - // ErrNoGroupEntries is returned if no matching entries were found in /etc/passwd. + // ErrNoGroupEntries is returned if no matching entries were found in /etc/group. ErrNoGroupEntries = errors.New("no matching entries in group file") // ErrRange is returned if a UID or GID is outside of the valid range. ErrRange = fmt.Errorf("uids and gids must be in range %d-%d", minID, maxID) @@ -243,9 +244,6 @@ func GetExecUserPath(userSpec string, defaults *ExecUser, passwdPath, groupPath // // See https://github.com/containerd/containerd/commit/de1341c201ffb0effebbf51d00376181968c8779 func parseNumeric(val string) (int, bool, error) { - if val == "" { - return 0, false, nil - } id, err := strconv.Atoi(val) if err != nil { if errors.Is(err, strconv.ErrSyntax) { @@ -296,51 +294,40 @@ func GetExecUser(userSpec string, defaults *ExecUser, passwd, group io.Reader) ( user := &ExecUser{ Uid: defaults.Uid, Gid: defaults.Gid, - Sgids: defaults.Sgids, + Sgids: append([]int{}, defaults.Sgids...), // Sgids slice *cannot* be nil. Home: defaults.Home, } - // Sgids slice *cannot* be nil. - if user.Sgids == nil { - user.Sgids = []int{} - } + // Allow for userSpec to have either "user", or optionally "user:group" syntax. + usr, grp, _ := strings.Cut(userSpec, ":") - // Allow for userArg to have either "user" syntax, or optionally "user:group" syntax - var userArg, groupArg string - parseLine([]byte(userSpec), &userArg, &groupArg) - - // Convert userArg and groupArg to be numeric, so we don't have to execute + // Convert usr and grp to be numeric, so we don't have to execute // Atoi *twice* for each iteration over lines. - uidArg, isUID, err := parseNumeric(userArg) + userArg, err := parseUserArg(usr) if err != nil { return nil, err } - gidArg, isGID, err := parseNumeric(groupArg) + groupArg, err := parseGroupArg(grp) if err != nil { return nil, err } // Find the matching user. users, err := ParsePasswdFilter(passwd, func(u User) bool { - if userArg == "" { + if userArg == nil { // Default to current state of the user. return u.Uid == user.Uid } - - if isUID { - // If the userArg is a valid numeric value, always treat it as a UID. - return uidArg == u.Uid - } - - return u.Name == userArg + return userArg.matches(u) }) // If we can't find the user, we have to bail. if err != nil && passwd != nil { - if userArg == "" { - userArg = strconv.Itoa(user.Uid) + name := usr + if userArg == nil { + name = strconv.Itoa(user.Uid) } - return nil, fmt.Errorf("unable to find user %s: %w", userArg, err) + return nil, fmt.Errorf("unable to find user %s: %w", name, err) } var matchedUserName string @@ -350,57 +337,42 @@ func GetExecUser(userSpec string, defaults *ExecUser, passwd, group io.Reader) ( user.Uid = users[0].Uid user.Gid = users[0].Gid user.Home = users[0].Home - } else if userArg != "" { + } else if userArg != nil { // If we can't find a user with the given username, the only other valid // option is if it's a numeric username with no associated entry in passwd. - - if !isUID { - // Not numeric. - return nil, fmt.Errorf("unable to find user %s: %w", userArg, ErrNoPasswdEntries) + if !userArg.isNumeric { + return nil, fmt.Errorf("unable to find user %s: %w", userArg.name, ErrNoPasswdEntries) } - user.Uid = uidArg + user.Uid = userArg.uid } // On to the groups. If we matched a username, we need to do this because of // the supplementary group IDs. - if groupArg != "" || matchedUserName != "" { + if groupArg != nil || matchedUserName != "" { groups, err := ParseGroupFilter(group, func(g Group) bool { // If the group argument isn't explicit, we'll just search for it. - if groupArg == "" { + if groupArg == nil { // Check if user is a member of this group. - for _, u := range g.List { - if u == matchedUserName { - return true - } - } - return false + return slices.Contains(g.List, matchedUserName) } - - if isGID { - // If the groupArg is numeric, always treat it as a GID. - return gidArg == g.Gid - } - - return g.Name == groupArg + return groupArg.matches(g) }) if err != nil && group != nil { return nil, fmt.Errorf("unable to find groups for spec %v: %w", matchedUserName, err) } // Only start modifying user.Gid if it is in explicit form. - if groupArg != "" { + if groupArg != nil { if len(groups) > 0 { // First match wins, even if there's more than one matching entry. user.Gid = groups[0].Gid } else { // If we can't find a group with the given name, the only other valid // option is if it's a numeric group name with no associated entry in group. - - if !isGID { - // Not numeric. - return nil, fmt.Errorf("unable to find group %s: %w", groupArg, ErrNoGroupEntries) + if !groupArg.isNumeric { + return nil, fmt.Errorf("unable to find group %s: %w", groupArg.name, ErrNoGroupEntries) } - user.Gid = gidArg + user.Gid = groupArg.gid } } else if len(groups) > 0 { // Supplementary group ids only make sense if in the implicit form. @@ -414,39 +386,19 @@ func GetExecUser(userSpec string, defaults *ExecUser, passwd, group io.Reader) ( return user, nil } -// groupArg is a parsed group argument for [GetAdditionalGroups]. -type groupArg struct { - name string - gid int - isNumeric bool -} - -// matches reports whether group g satisfies the argument. Numeric arguments -// are matched by GID only, others by name. -func (ag groupArg) matches(g Group) bool { - if ag.isNumeric { - return g.Gid == ag.gid - } - return g.Name == ag.name -} - // GetAdditionalGroups looks up a list of groups by name or group id // against the given /etc/group formatted data. If a group name cannot // be found, an error will be returned. If a group id cannot be found, // or the given group data is nil, the id will be returned as-is // provided it is in the legal range. func GetAdditionalGroups(additionalGroups []string, group io.Reader) ([]int, error) { - addtlGroups := make([]groupArg, len(additionalGroups)) - for i, ag := range additionalGroups { - gid, ok, err := parseNumeric(ag) + addtlGroups := make([]groupArg, 0, len(additionalGroups)) + for _, ag := range additionalGroups { + groupArg, err := parseGroupArg(ag) if err != nil { return nil, err } - addtlGroups[i] = groupArg{ - name: ag, - gid: gid, - isNumeric: ok, - } + addtlGroups = append(addtlGroups, *groupArg) } groups := []Group{} diff --git a/vendor/github.com/moby/sys/user/user_utils.go b/vendor/github.com/moby/sys/user/user_utils.go index 978aa927ed..8084d3b071 100644 --- a/vendor/github.com/moby/sys/user/user_utils.go +++ b/vendor/github.com/moby/sys/user/user_utils.go @@ -62,3 +62,65 @@ func (l *limitedFile) Read(p []byte) (int, error) { } return n, err } + +// userArg is a parsed user argument for [GetExecUser]. +type userArg struct { + name string + uid int + isNumeric bool +} + +// parseUserArg parses a user argument as either a user name or UID. +// +// If name is empty, parseUserArg returns "nil, nil" to indicate that no user +// argument was specified. +func parseUserArg(name string) (*userArg, error) { + if name == "" { + return nil, nil + } + uid, isNumeric, err := parseNumeric(name) + if err != nil { + return nil, err + } + return &userArg{name: name, uid: uid, isNumeric: isNumeric}, nil +} + +// matches reports whether user u satisfies the argument. Numeric arguments +// are matched by UID only, others by name. +func (ua *userArg) matches(u User) bool { + if ua.isNumeric { + return u.Uid == ua.uid + } + return u.Name == ua.name +} + +// groupArg is a parsed group argument for [GetAdditionalGroups] or [GetExecUser]. +type groupArg struct { + name string + gid int + isNumeric bool +} + +// parseGroupArg parses a group argument as either a group name or GID. +// +// If name is empty, parseGroupArg returns "nil, nil" to indicate that no group +// argument was specified. +func parseGroupArg(name string) (*groupArg, error) { + if name == "" { + return nil, nil + } + gid, isNumeric, err := parseNumeric(name) + if err != nil { + return nil, err + } + return &groupArg{name: name, gid: gid, isNumeric: isNumeric}, nil +} + +// matches reports whether group g satisfies the argument. Numeric arguments +// are matched by GID only, others by name. +func (ag groupArg) matches(g Group) bool { + if ag.isNumeric { + return g.Gid == ag.gid + } + return g.Name == ag.name +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 00f553627d..658430a61a 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -287,7 +287,7 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc/types github.com/aws/aws-sdk-go-v2/service/sts github.com/aws/aws-sdk-go-v2/service/sts/internal/endpoints github.com/aws/aws-sdk-go-v2/service/sts/types -# github.com/aws/smithy-go v1.28.7 +# github.com/aws/smithy-go v1.28.8 ## explicit; go 1.24 github.com/aws/smithy-go github.com/aws/smithy-go/auth @@ -626,10 +626,9 @@ github.com/containerd/platforms ## explicit; go 1.22 github.com/containerd/plugin github.com/containerd/plugin/registry -# github.com/containerd/stargz-snapshotter/estargz v0.18.2 -## explicit; go 1.24.0 +# github.com/containerd/stargz-snapshotter/estargz v0.19.1 +## explicit; go 1.25.0 github.com/containerd/stargz-snapshotter/estargz -github.com/containerd/stargz-snapshotter/estargz/errorutil # github.com/containerd/ttrpc v1.2.10 ## explicit; go 1.25.0 github.com/containerd/ttrpc @@ -1078,7 +1077,7 @@ github.com/mitchellh/copystructure # github.com/mitchellh/reflectwalk v1.0.2 ## explicit github.com/mitchellh/reflectwalk -# github.com/moby/buildkit v0.34.2 +# github.com/moby/buildkit v0.35.0-rc1 ## explicit; go 1.26.8 github.com/moby/buildkit/api/services/control github.com/moby/buildkit/api/types @@ -1088,10 +1087,10 @@ github.com/moby/buildkit/cache/contenthash github.com/moby/buildkit/cache/metadata github.com/moby/buildkit/cache/remotecache github.com/moby/buildkit/cache/remotecache/gha -github.com/moby/buildkit/cache/remotecache/gha/ghatypes github.com/moby/buildkit/cache/remotecache/inline github.com/moby/buildkit/cache/remotecache/local github.com/moby/buildkit/cache/remotecache/registry +github.com/moby/buildkit/cache/remotecache/signing github.com/moby/buildkit/cache/remotecache/v1 github.com/moby/buildkit/cache/remotecache/v1/types github.com/moby/buildkit/cache/util @@ -1204,6 +1203,7 @@ github.com/moby/buildkit/util/bkmaps github.com/moby/buildkit/util/bkslices github.com/moby/buildkit/util/cachedigest github.com/moby/buildkit/util/compression +github.com/moby/buildkit/util/compression/go126flate github.com/moby/buildkit/util/cond github.com/moby/buildkit/util/contentutil github.com/moby/buildkit/util/converter @@ -1456,8 +1456,8 @@ github.com/moby/sys/signal # github.com/moby/sys/symlink v0.3.0 ## explicit; go 1.17 github.com/moby/sys/symlink -# github.com/moby/sys/user v0.4.1 -## explicit; go 1.18 +# github.com/moby/sys/user v0.5.0 +## explicit; go 1.24 github.com/moby/sys/user # github.com/moby/sys/userns v0.2.1 ## explicit; go 1.21