vendor: update buildkit to v0.32.0-rc1

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
This commit is contained in:
Tonis Tiigi authored and Sebastiaan van Stijn committed 2026-07-23 15:15:24 +02:00
1 parent 181391d73e
commit 2f89623c55
74 files changed
+736 -305

No files matched your search

@@ -131,6 +131,12 @@ func (is *Source) registryIdentifier(ref string, attrs map[string]string, platfo
return nil, errors.Errorf("invalid layer limit %s", v)
}
id.LayerLimit = &l
case pb.AttrImageChecksum:
dgst, err := digest.Parse(v)
if err != nil {
return nil, errors.Wrapf(err, "invalid image checksum %s", v)
}
id.Checksum = dgst
}
}
@@ -328,7 +334,7 @@ func (p *puller) resolveLocal() {
}
}
if p.src.ResolveMode == resolver.ResolveModeDefault || p.src.ResolveMode == resolver.ResolveModePreferLocal {
if p.src.Checksum == "" && (p.src.ResolveMode == resolver.ResolveModeDefault || p.src.ResolveMode == resolver.ResolveModePreferLocal) {
ref := p.src.Reference.String()
img, err := p.is.resolveLocal(ref)
if err == nil {
@@ -371,6 +377,10 @@ func (p *puller) resolve(ctx context.Context, jobCtx solver.JobContext) error {
p.ref = origRef
}
if p.src.Checksum != "" && p.desc.Digest != p.src.Checksum {
return struct{}{}, errors.Errorf("image digest %s for %s does not match expected checksum %s", p.desc.Digest, p.src.Reference.String(), p.src.Checksum)
}
// Schema 1 manifests cannot be resolved to an image config
// since the conversion must take place after all the content
// has been read.
@@ -0,0 +1,28 @@
package containerimage
import (
"testing"
"github.com/moby/buildkit/solver/pb"
bkcontainerimage "github.com/moby/buildkit/source/containerimage"
"github.com/opencontainers/go-digest"
"gotest.tools/v3/assert"
)
func TestRegistryIdentifierImageChecksum(t *testing.T) {
checksum := digest.FromString("test")
id, err := (&Source{}).registryIdentifier("docker.io/library/alpine:latest", map[string]string{
pb.AttrImageChecksum: checksum.String(),
}, nil)
assert.NilError(t, err)
assert.Equal(t, id.(*bkcontainerimage.ImageIdentifier).Checksum, checksum)
}
func TestRegistryIdentifierInvalidImageChecksum(t *testing.T) {
_, err := (&Source{}).registryIdentifier("docker.io/library/alpine:latest", map[string]string{
pb.AttrImageChecksum: "invalid",
}, nil)
assert.ErrorContains(t, err, "invalid image checksum")
}
@@ -258,6 +258,10 @@ func (w *Worker) LoadRef(ctx context.Context, id string, hidden bool) (cache.Imm
return w.CacheManager().Get(ctx, id, nil, opts...)
}
func (w *Worker) ParseSource(op *pb.SourceOp, platform *pb.Platform) (source.Identifier, error) {
return w.SourceManager.Identifier(&pb.Op_Source{Source: op}, platform)
}
func (w *Worker) ResolveSourceMetadata(ctx context.Context, op *pb.SourceOp, opt sourceresolver.Opt, sm *session.Manager, jobCtx solver.JobContext) (*sourceresolver.MetaResponse, error) {
if opt.SourcePolicies != nil {
return nil, errors.New("source policies can not be set for worker")
@@ -279,7 +283,7 @@ func (w *Worker) ResolveSourceMetadata(ctx context.Context, op *pb.SourceOp, opt
}
}
id, err := w.SourceManager.Identifier(&pb.Op_Source{Source: op}, platform)
id, err := w.ParseSource(op, platform)
if err != nil {
return nil, err
}
+2 -2
View File
@@ -60,7 +60,7 @@ require (
github.com/miekg/dns v1.1.72
github.com/mistifyio/go-zfs/v4 v4.0.0
github.com/mitchellh/copystructure v1.2.0
github.com/moby/buildkit v0.31.2
github.com/moby/buildkit v0.32.0-rc1
github.com/moby/docker-image-spec v1.3.1
github.com/moby/go-archive v0.2.0
github.com/moby/ipvs v1.1.0
@@ -206,6 +206,7 @@ require (
github.com/go-openapi/validate v0.26.0 // indirect
github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
github.com/gofrs/flock v0.13.0 // indirect
github.com/gohugoio/hashstructure v0.6.0 // indirect
github.com/golang-jwt/jwt/v5 v5.3.1 // indirect
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
github.com/golang/protobuf v1.5.4 // indirect
@@ -232,7 +233,6 @@ require (
github.com/jmoiron/sqlx v1.4.0 // indirect
github.com/klauspost/compress v1.19.1 // indirect
github.com/knqyf263/go-plugin v0.9.0 // indirect
github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect
github.com/mitchellh/reflectwalk v1.0.2 // indirect
github.com/moby/sys/capability v0.4.0 // indirect
github.com/morikuni/aec v1.1.0 // indirect
+6 -6
View File
@@ -227,8 +227,8 @@ github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi
github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
github.com/docker/cli v29.5.3+incompatible h1:nbEFfz774vBwQ5KRYv7c/AghjReqnGISvrRhzjV0evs=
github.com/docker/cli v29.5.3+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
github.com/docker/cli v29.6.2+incompatible h1:/bjePvcbbFTnRrMfWJBY7AjfICdsiLVgHn6LwTVOcqw=
github.com/docker/cli v29.6.2+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk=
github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
github.com/docker/docker-credential-helpers v0.9.8 h1:bIREROb7So6PRlq6KTtdS9MPEjC29OQRkFNlvK2OX8Q=
@@ -349,6 +349,8 @@ github.com/gofrs/flock v0.13.0/go.mod h1:jxeyy9R1auM5S6JYDBhDt+E2TCo7DkratH4Pgi8
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
github.com/gohugoio/hashstructure v0.6.0 h1:7wMB/2CfXoThFYhdWRGv3u3rUM761Cq29CxUW+NltUg=
github.com/gohugoio/hashstructure v0.6.0/go.mod h1:lapVLk9XidheHG1IQ4ZSbyYrXcaILU1ZEP/+vno5rBQ=
github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY=
github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=
github.com/golang/gddo v0.0.0-20210115222349-20d68f94ee1f h1:16RtHeWGkJMc80Etb8RPCcKevXGldr57+LOyZt8zOlg=
@@ -537,15 +539,13 @@ github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa1
github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s=
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4=
github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE=
github.com/mitchellh/mapstructure v0.0.0-20170523030023-d0303fe80992/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ=
github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
github.com/moby/buildkit v0.31.2 h1:UsaoSa0z45ycj+8DqWwmiXKnszDSNvv3wYPGLVtcEKE=
github.com/moby/buildkit v0.31.2/go.mod h1:q1QO35x5YryiXLONScL2IybwMIziz6Rq3FBznz8fmGc=
github.com/moby/buildkit v0.32.0-rc1 h1:gYtsqXOSA43i9zTf2w+0KzTxpZYao0ZYr/AaGx0RJhs=
github.com/moby/buildkit v0.32.0-rc1/go.mod h1:0GB/EJ1d+4VIVqIAgy3asaoGkVXy7IrDfVy7mPhOvg8=
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8=
+1
View File
@@ -0,0 +1 @@
*.test
File renamed without changes.
@@ -30,18 +30,9 @@ sending data across the network, caching values locally (de-dup), and so on.
Standard `go get`:
```
$ go get github.com/mitchellh/hashstructure/v2
$ go get github.com/gohugoio/hashstructure
```
**Note on v2:** It is highly recommended you use the "v2" release since this
fixes some significant hash collisions issues from v1. In practice, we used
v1 for many years in real projects at HashiCorp and never had issues, but it
is highly dependent on the shape of the data you're hashing and how you use
those hashes.
When using v2+, you can still generate weaker v1 hashes by using the
`FormatV1` format when calling `Hash`.
## Usage & Example
For usage and examples see the [Godoc](http://godoc.org/github.com/mitchellh/hashstructure).
@@ -65,7 +56,7 @@ v := ComplexStruct{
},
}
hash, err := hashstructure.Hash(v, hashstructure.FormatV2, nil)
hash, err := hashstructure.Hash(v, nil)
if err != nil {
panic(err)
}
File renamed without changes.
@@ -5,8 +5,10 @@ import (
"fmt"
"hash"
"hash/fnv"
"math"
"reflect"
"time"
"unsafe"
)
// HashOptions are options that are available for hashing.
@@ -39,25 +41,6 @@ type HashOptions struct {
UseStringer bool
}
// Format specifies the hashing process used. Different formats typically
// generate different hashes for the same value and have different properties.
type Format uint
const (
// To disallow the zero value
formatInvalid Format = iota
// FormatV1 is the format used in v1.x of this library. This has the
// downsides noted in issue #18 but allows simultaneous v1/v2 usage.
FormatV1
// FormatV2 is the current recommended format and fixes the issues
// noted in FormatV1.
FormatV2
formatMax // so we can easily find the end
)
// Hash returns the hash value of an arbitrary value.
//
// If opts is nil, then default options will be used. See HashOptions
@@ -72,35 +55,29 @@ const (
//
// Notes on the value:
//
// * Unexported fields on structs are ignored and do not affect the
// - Unexported fields on structs are ignored and do not affect the
// hash value.
//
// * Adding an exported field to a struct with the zero value will change
// - Adding an exported field to a struct with the zero value will change
// the hash value.
//
// For structs, the hashing can be controlled using tags. For example:
//
// struct {
// Name string
// UUID string `hash:"ignore"`
// }
// struct {
// Name string
// UUID string `hash:"ignore"`
// }
//
// The available tag values are:
//
// * "ignore" or "-" - The field will be ignored and not affect the hash code.
// - "ignore" or "-" - The field will be ignored and not affect the hash code.
//
// * "set" - The field will be treated as a set, where ordering doesn't
// affect the hash code. This only works for slices.
// - "set" - The field will be treated as a set, where ordering doesn't
// affect the hash code. This only works for slices.
//
// * "string" - The field will be hashed as a string, only works when the
// field implements fmt.Stringer
//
func Hash(v interface{}, format Format, opts *HashOptions) (uint64, error) {
// Validate our format
if format <= formatInvalid || format >= formatMax {
return 0, &ErrFormat{}
}
// - "string" - The field will be hashed as a string, only works when the
// field implements fmt.Stringer
func Hash(v interface{}, opts *HashOptions) (uint64, error) {
// Create default options
if opts == nil {
opts = &HashOptions{}
@@ -115,9 +92,13 @@ func Hash(v interface{}, format Format, opts *HashOptions) (uint64, error) {
// Reset the hash
opts.Hasher.Reset()
// Fast path for strings.
if s, ok := v.(string); ok {
return hashString(opts.Hasher, s)
}
// Create our walker and walk the structure
w := &walker{
format: format,
h: opts.Hasher,
tag: opts.TagName,
zeronil: opts.ZeroNil,
@@ -129,13 +110,13 @@ func Hash(v interface{}, format Format, opts *HashOptions) (uint64, error) {
}
type walker struct {
format Format
h hash.Hash64
tag string
zeronil bool
ignorezerovalue bool
sets bool
stringer bool
buf [16]byte // Reusable buffer for binary encoding
}
type visitOpts struct {
@@ -149,6 +130,76 @@ type visitOpts struct {
var timeType = reflect.TypeOf(time.Time{})
// A direct hash calculation used for numeric and bool values.
func (w *walker) hashDirect(v any) (uint64, error) {
w.h.Reset()
// Use direct byte manipulation for numbers instead of binary.Write to avoid allocations
switch val := v.(type) {
case int64:
binary.LittleEndian.PutUint64(w.buf[:8], uint64(val))
w.h.Write(w.buf[:8])
case uint64:
binary.LittleEndian.PutUint64(w.buf[:8], val)
w.h.Write(w.buf[:8])
case int8:
w.buf[0] = byte(val)
w.h.Write(w.buf[:1])
case uint8:
w.buf[0] = val
w.h.Write(w.buf[:1])
case int16:
binary.LittleEndian.PutUint16(w.buf[:2], uint16(val))
w.h.Write(w.buf[:2])
case uint16:
binary.LittleEndian.PutUint16(w.buf[:2], val)
w.h.Write(w.buf[:2])
case int32:
binary.LittleEndian.PutUint32(w.buf[:4], uint32(val))
w.h.Write(w.buf[:4])
case uint32:
binary.LittleEndian.PutUint32(w.buf[:4], val)
w.h.Write(w.buf[:4])
case float32:
binary.LittleEndian.PutUint32(w.buf[:4], math.Float32bits(val))
w.h.Write(w.buf[:4])
case float64:
binary.LittleEndian.PutUint64(w.buf[:8], math.Float64bits(val))
w.h.Write(w.buf[:8])
case complex64:
binary.LittleEndian.PutUint32(w.buf[:4], math.Float32bits(real(val)))
binary.LittleEndian.PutUint32(w.buf[4:8], math.Float32bits(imag(val)))
w.h.Write(w.buf[:8])
case complex128:
binary.LittleEndian.PutUint64(w.buf[:8], math.Float64bits(real(val)))
binary.LittleEndian.PutUint64(w.buf[8:16], math.Float64bits(imag(val)))
w.h.Write(w.buf[:16])
default:
// Fallback to binary.Write for unsupported types, for instance enums
err := binary.Write(w.h, binary.LittleEndian, v)
return w.h.Sum64(), err
}
return w.h.Sum64(), nil
}
// A direct hash calculation used for strings.
func (w *walker) hashString(s string) (uint64, error) {
return hashString(w.h, s)
}
// A direct hash calculation used for strings.
func hashString(h hash.Hash64, s string) (uint64, error) {
h.Reset()
// Use zero-copy conversion from string to []byte using unsafe
if len(s) > 0 {
b := unsafe.Slice(unsafe.StringData(s), len(s))
h.Write(b)
}
return h.Sum64(), nil
}
func (w *walker) visit(v reflect.Value, opts *visitOpts) (uint64, error) {
t := reflect.TypeOf(0)
@@ -179,29 +230,66 @@ func (w *walker) visit(v reflect.Value, opts *visitOpts) (uint64, error) {
v = reflect.Zero(t)
}
// Binary writing can use raw ints, we have to convert to
// a sized-int, we'll choose the largest...
switch v.Kind() {
case reflect.Int:
v = reflect.ValueOf(int64(v.Int()))
case reflect.Uint:
v = reflect.ValueOf(uint64(v.Uint()))
case reflect.Bool:
var tmp int8
if v.Bool() {
tmp = 1
if v.CanInt() {
i := v.Int()
switch v.Kind() {
case reflect.Int:
return w.hashDirect(i)
case reflect.Int8:
return w.hashDirect(int8(i))
case reflect.Int16:
return w.hashDirect(int16(i))
case reflect.Int32:
return w.hashDirect(int32(i))
case reflect.Int64:
return w.hashDirect(i)
}
}
if v.CanUint() {
u := v.Uint()
switch v.Kind() {
case reflect.Uint:
return w.hashDirect(u)
case reflect.Uint8:
return w.hashDirect(uint8(u))
case reflect.Uint16:
return w.hashDirect(uint16(u))
case reflect.Uint32:
return w.hashDirect(uint32(u))
case reflect.Uint64:
return w.hashDirect(u)
}
}
if v.CanFloat() {
f := v.Float()
switch v.Kind() {
case reflect.Float32:
return w.hashDirect(float32(f))
case reflect.Float64:
return w.hashDirect(f)
}
}
if v.CanComplex() {
c := v.Complex()
switch v.Kind() {
case reflect.Complex64:
return w.hashDirect(complex64(c))
case reflect.Complex128:
return w.hashDirect(c)
}
v = reflect.ValueOf(tmp)
}
k := v.Kind()
// We can shortcut numeric values by directly binary writing them
if k >= reflect.Int && k <= reflect.Complex64 {
// A direct hash calculation
w.h.Reset()
err := binary.Write(w.h, binary.LittleEndian, v.Interface())
return w.h.Sum64(), err
if k == reflect.Bool {
var tmp int8
if v.Bool() {
tmp = 1
}
return w.hashDirect(tmp)
}
switch v.Type() {
@@ -212,8 +300,8 @@ func (w *walker) visit(v reflect.Value, opts *visitOpts) (uint64, error) {
return 0, err
}
err = binary.Write(w.h, binary.LittleEndian, b)
return w.h.Sum64(), err
w.h.Write(b)
return w.h.Sum64(), nil
}
switch k {
@@ -233,20 +321,30 @@ func (w *walker) visit(v reflect.Value, opts *visitOpts) (uint64, error) {
case reflect.Map:
var includeMap IncludableMap
if opts != nil && opts.Struct != nil {
var field string
if v, ok := v.Interface().(IncludableMap); ok {
includeMap = v
} else if opts != nil && opts.Struct != nil {
if v, ok := opts.Struct.(IncludableMap); ok {
includeMap = v
includeMap, field = v, opts.StructField
}
}
// Build the hash for the map. We do this by XOR-ing all the key
// and value hashes. This makes it deterministic despite ordering.
var h uint64
for _, k := range v.MapKeys() {
v := v.MapIndex(k)
k := reflect.New(v.Type().Key()).Elem()
vv := reflect.New(v.Type().Elem()).Elem()
iter := v.MapRange()
for iter.Next() {
k.SetIterKey(iter)
vv.SetIterValue(iter)
if includeMap != nil {
incl, err := includeMap.HashIncludeMap(
opts.StructField, k.Interface(), v.Interface())
incl, err := includeMap.HashIncludeMap(field, k.Interface(), vv.Interface())
if err != nil {
return 0, err
}
@@ -259,7 +357,7 @@ func (w *walker) visit(v reflect.Value, opts *visitOpts) (uint64, error) {
if err != nil {
return 0, err
}
vh, err := w.visit(v, nil)
vh, err := w.visit(vv, nil)
if err != nil {
return 0, err
}
@@ -268,26 +366,16 @@ func (w *walker) visit(v reflect.Value, opts *visitOpts) (uint64, error) {
h = hashUpdateUnordered(h, fieldHash)
}
if w.format != FormatV1 {
// Important: read the docs for hashFinishUnordered
h = hashFinishUnordered(w.h, h)
}
// Important: read the docs for hashFinishUnordered
h = hashFinishUnordered(w.h, h)
return h, nil
case reflect.Struct:
parent := v.Interface()
var include Includable
if impl, ok := parent.(Includable); ok {
include = impl
}
var parent interface{}
if impl, ok := parent.(Hashable); ok {
return impl.Hash()
}
// If we can address this value, check if the pointer value
// implements our interfaces and use that if so.
// Check if we can address this value first (more common case for pointer receivers)
if v.CanAddr() {
vptr := v.Addr()
parentptr := vptr.Interface()
@@ -298,18 +386,38 @@ func (w *walker) visit(v reflect.Value, opts *visitOpts) (uint64, error) {
if impl, ok := parentptr.(Hashable); ok {
return impl.Hash()
}
// Only set parent if we'll need it for IncludableMap
parent = parentptr
}
// Only box the value if we haven't already found an implementation via pointer
if include == nil && parent == nil {
parent = v.Interface()
if impl, ok := parent.(Includable); ok {
include = impl
}
if impl, ok := parent.(Hashable); ok {
return impl.Hash()
}
}
t := v.Type()
h, err := w.visit(reflect.ValueOf(t.Name()), nil)
h, err := w.hashString(t.Name())
if err != nil {
return 0, err
}
l := v.NumField()
var fieldOpts visitOpts
// Defer boxing parent until we know we need it
if parent == nil {
parent = v.Interface()
}
fieldOpts.Struct = parent
for i := 0; i < l; i++ {
if innerV := v.Field(i); v.CanSet() || t.Field(i).Name != "_" {
var f visitFlag
fieldType := t.Field(i)
if fieldType.PkgPath != "" {
// Unexported
@@ -352,21 +460,18 @@ func (w *walker) visit(v reflect.Value, opts *visitOpts) (uint64, error) {
}
}
switch tag {
case "set":
f |= visitFlagSet
fieldOpts.Flags = 0
if tag == "set" {
fieldOpts.Flags |= visitFlagSet
}
kh, err := w.visit(reflect.ValueOf(fieldType.Name), nil)
kh, err := w.hashString(fieldType.Name)
if err != nil {
return 0, err
}
vh, err := w.visit(innerV, &visitOpts{
Flags: f,
Struct: parent,
StructField: fieldType.Name,
})
fieldOpts.StructField = fieldType.Name
vh, err := w.visit(innerV, &fieldOpts)
if err != nil {
return 0, err
}
@@ -374,11 +479,8 @@ func (w *walker) visit(v reflect.Value, opts *visitOpts) (uint64, error) {
fieldHash := hashUpdateOrdered(w.h, kh, vh)
h = hashUpdateUnordered(h, fieldHash)
}
if w.format != FormatV1 {
// Important: read the docs for hashFinishUnordered
h = hashFinishUnordered(w.h, h)
}
// Important: read the docs for hashFinishUnordered
h = hashFinishUnordered(w.h, h)
}
return h, nil
@@ -406,7 +508,7 @@ func (w *walker) visit(v reflect.Value, opts *visitOpts) (uint64, error) {
}
}
if set && w.format != FormatV1 {
if set {
// Important: read the docs for hashFinishUnordered
h = hashFinishUnordered(w.h, h)
}
@@ -414,31 +516,20 @@ func (w *walker) visit(v reflect.Value, opts *visitOpts) (uint64, error) {
return h, nil
case reflect.String:
// Directly hash
w.h.Reset()
_, err := w.h.Write([]byte(v.String()))
return w.h.Sum64(), err
return w.hashString(v.String())
default:
return 0, fmt.Errorf("unknown kind to hash: %s", k)
}
}
func hashUpdateOrdered(h hash.Hash64, a, b uint64) uint64 {
// For ordered updates, use a real hash function
h.Reset()
// We just panic if the binary writes fail because we are writing
// an int64 which should never be fail-able.
e1 := binary.Write(h, binary.LittleEndian, a)
e2 := binary.Write(h, binary.LittleEndian, b)
if e1 != nil {
panic(e1)
}
if e2 != nil {
panic(e2)
}
var buf [16]byte
binary.LittleEndian.PutUint64(buf[0:8], a)
binary.LittleEndian.PutUint64(buf[8:16], b)
h.Write(buf[:])
return h.Sum64()
}
@@ -453,22 +544,20 @@ func hashUpdateUnordered(a, b uint64) uint64 {
// hashUpdateUnordered can effectively cancel out a previous change to the hash
// result if the same hash value appears later on. For example, consider:
//
// hashUpdateUnordered(hashUpdateUnordered("A", "B"), hashUpdateUnordered("A", "C")) =
// H("A") ^ H("B")) ^ (H("A") ^ H("C")) =
// (H("A") ^ H("A")) ^ (H("B") ^ H(C)) =
// H(B) ^ H(C) =
// hashUpdateUnordered(hashUpdateUnordered("Z", "B"), hashUpdateUnordered("Z", "C"))
// hashUpdateUnordered(hashUpdateUnordered("A", "B"), hashUpdateUnordered("A", "C")) =
// H("A") ^ H("B")) ^ (H("A") ^ H("C")) =
// (H("A") ^ H("A")) ^ (H("B") ^ H(C)) =
// H(B) ^ H(C) =
// hashUpdateUnordered(hashUpdateUnordered("Z", "B"), hashUpdateUnordered("Z", "C"))
//
// hashFinishUnordered "hardens" the result, so that encountering partially
// overlapping input data later on in a different context won't cancel out.
func hashFinishUnordered(h hash.Hash64, a uint64) uint64 {
h.Reset()
// We just panic if the writes fail
e1 := binary.Write(h, binary.LittleEndian, a)
if e1 != nil {
panic(e1)
}
var buf [8]byte
binary.LittleEndian.PutUint64(buf[:], a)
h.Write(buf[:])
return h.Sum64()
}
+1 -1
View File
@@ -1,6 +1,6 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.36.11
// protoc-gen-go v1.36.11-devel
// protoc v3.14.0
// source: github.com/moby/buildkit/api/services/control/control.proto
+1 -1
View File
@@ -1,6 +1,6 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.36.11
// protoc-gen-go v1.36.11-devel
// protoc v3.14.0
// source: github.com/moby/buildkit/api/types/worker.proto
+1 -1
View File
@@ -1,6 +1,6 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.36.11
// protoc-gen-go v1.36.11-devel
// protoc v3.14.0
// source: github.com/moby/buildkit/cache/contenthash/checksum.proto
+3 -3
View File
@@ -162,12 +162,12 @@ func (cm *cacheManager) GetByBlob(ctx context.Context, desc ocispecs.Descriptor,
p = p2.(*immutableRef)
if err := p.Finalize(ctx); err != nil {
p.Release(context.TODO())
_ = p.Release(context.WithoutCancel(ctx))
return nil, err
}
if p.getChainID() == "" || p.getBlobChainID() == "" {
p.Release(context.TODO())
_ = p.Release(context.WithoutCancel(ctx))
return nil, errors.Errorf("failed to get ref by blob on non-addressable parent")
}
chainID = imagespecidentity.ChainID([]digest.Digest{p.getChainID(), chainID})
@@ -177,7 +177,7 @@ func (cm *cacheManager) GetByBlob(ctx context.Context, desc ocispecs.Descriptor,
releaseParent := false
defer func() {
if releaseParent || rerr != nil && p != nil {
p.Release(context.WithoutCancel(ctx))
_ = p.Release(context.WithoutCancel(ctx))
}
}()
+3 -1
View File
@@ -41,7 +41,9 @@ func NewStore(dbPath string) (*Store, error) {
)
}
}
db, err := boltutil.Open(dbPath, 0600, nil)
db, err := boltutil.Open(dbPath, 0600, &bolt.Options{
FreelistType: bolt.FreelistMapType,
})
if err != nil {
return nil, errors.Wrapf(err, "failed to open database file %s", dbPath)
}
+2 -2
View File
@@ -1508,12 +1508,12 @@ func (cr *cacheRecord) finalize(ctx context.Context) error {
ID: cr.getSnapshotID(),
Type: "snapshots/" + cr.cm.Snapshotter.Name(),
}); err != nil {
cr.cm.LeaseManager.Delete(context.TODO(), leases.Lease{ID: cr.ID()})
_ = cr.cm.LeaseManager.Delete(context.WithoutCancel(ctx), leases.Lease{ID: cr.ID()})
return errors.Wrapf(err, "failed to add snapshot %s to lease", cr.getSnapshotID())
}
if err := cr.cm.Snapshotter.Commit(ctx, cr.getSnapshotID(), mutable.getSnapshotID()); err != nil {
cr.cm.LeaseManager.Delete(context.TODO(), leases.Lease{ID: cr.ID()})
_ = cr.cm.LeaseManager.Delete(context.WithoutCancel(ctx), leases.Lease{ID: cr.ID()})
return errors.Wrapf(err, "failed to commit %s to %s during finalize", mutable.getSnapshotID(), cr.getSnapshotID())
}
cr.mountCache = nil
+4
View File
@@ -10,6 +10,7 @@ import (
cerrdefs "github.com/containerd/errdefs"
cacheimporttypes "github.com/moby/buildkit/cache/remotecache/v1/types"
"github.com/moby/buildkit/solver"
"github.com/moby/buildkit/util/compression"
digest "github.com/opencontainers/go-digest"
"github.com/pkg/errors"
)
@@ -152,6 +153,9 @@ func marshalRemote(ctx context.Context, r *solver.Remote, state *marshalState) s
parentID = marshalRemote(ctx, r2, state)
}
desc := r.Descriptors[len(r.Descriptors)-1]
if desc.MediaType != "" {
desc = compression.ConvertAllLayerMediaTypes(ctx, true, desc)[0]
}
state.descriptors[desc.Digest] = DescriptorProviderPair{
Descriptor: desc,
+8
View File
@@ -41,6 +41,14 @@ func ReadFile(ctx context.Context, root string, req ReadRequest) ([]byte, error)
}
defer f.Close()
info, err := f.Stat()
if err != nil {
return nil, errors.WithStack(err)
}
if !info.Mode().IsRegular() {
return nil, errors.Errorf("%s is not a regular file", req.Filename)
}
var rdr io.Reader = f
if req.Range != nil {
rdr = io.NewSectionReader(f, int64(req.Range.Offset), int64(req.Range.Length))
+8 -3
View File
@@ -13,8 +13,9 @@ import (
contentapi "github.com/containerd/containerd/api/services/content/v1"
"github.com/containerd/containerd/v2/core/content"
"github.com/containerd/containerd/v2/plugins/services/content/contentserver"
"github.com/containerd/platforms"
"github.com/distribution/reference"
"github.com/mitchellh/hashstructure/v2"
"github.com/gohugoio/hashstructure"
controlapi "github.com/moby/buildkit/api/services/control"
apitypes "github.com/moby/buildkit/api/types"
"github.com/moby/buildkit/cache/remotecache"
@@ -548,7 +549,11 @@ func (c *Controller) Solve(ctx context.Context, req *controlapi.SolveRequest) (*
resolveMode = v
}
procs = append(procs, proc.SBOMProcessor(ref.String(), useCache, resolveMode, params))
scannerPlatform := platforms.Normalize(platforms.DefaultSpec())
if ps := w.Platforms(false); len(ps) > 0 {
scannerPlatform = ps[0]
}
procs = append(procs, proc.SBOMProcessor(ref.String(), scannerPlatform, useCache, resolveMode, params))
}
if attrs, ok := attests["provenance"]; ok {
@@ -811,7 +816,7 @@ func cacheOptKey(opt *controlapi.CacheOptionsEntry) (string, error) {
Type: opt.Type,
Attrs: opt.Attrs,
}
hash, err := hashstructure.Hash(rawOpt, hashstructure.FormatV2, nil)
hash, err := hashstructure.Hash(rawOpt, nil)
if err != nil {
return "", err
}
+28 -3
View File
@@ -13,7 +13,7 @@ import (
"github.com/containerd/containerd/v2/core/mount"
"github.com/containerd/containerd/v2/pkg/namespaces"
"github.com/containerd/containerd/v2/pkg/oci"
"github.com/mitchellh/hashstructure/v2"
"github.com/gohugoio/hashstructure"
"github.com/moby/buildkit/executor"
"github.com/moby/buildkit/snapshot"
"github.com/moby/buildkit/solver/llbsolver/cdidevices"
@@ -46,6 +46,29 @@ var tracingEnvVars = []string{
"OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=grpc",
}
// envName returns the variable name of a "NAME=VALUE" environment entry.
func envName(env string) string {
name, _, _ := strings.Cut(env, "=")
return name
}
// appendMissingTracingEnv appends the OpenTelemetry trace-exporter variables
// that the build has not already set. This lets a build override or opt out of
// them (for example with `--opt env.OTEL_TRACES_EXPORTER=none`) instead of
// BuildKit unconditionally forcing its own values.
func appendMissingTracingEnv(env []string) []string {
existing := make(map[string]struct{}, len(env))
for _, e := range env {
existing[envName(e)] = struct{}{}
}
for _, e := range tracingEnvVars {
if _, ok := existing[envName(e)]; !ok {
env = append(env, e)
}
}
return env
}
func (pm ProcessMode) String() string {
switch pm {
case ProcessSandbox:
@@ -125,7 +148,9 @@ func GenerateSpec(ctx context.Context, meta executor.Meta, mounts []executor.Mou
if tracingSocket != "" {
// https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/exporter.md
meta.Env = append(meta.Env, tracingEnvVars...)
// Only inject the trace-exporter variables the build has not already set,
// so it can override or opt out of them (e.g. `--opt env.OTEL_TRACES_EXPORTER=none`).
meta.Env = appendMissingTracingEnv(meta.Env)
meta.Env = append(meta.Env, childprocess.Environ(ctx)...)
}
@@ -265,7 +290,7 @@ func (s *submounts) subMount(m mount.Mount, subPath string) (mount.Mount, error)
if s.m == nil {
s.m = map[uint64]mountRef{}
}
h, err := hashstructure.Hash(m, hashstructure.FormatV2, nil)
h, err := hashstructure.Hash(m, nil)
if err != nil {
return mount.Mount{}, errors.WithStack(err)
}
+34 -13
View File
@@ -7,6 +7,7 @@ import (
"crypto/sha256"
"crypto/x509"
"encoding/pem"
"io"
"os"
"path/filepath"
"syscall"
@@ -29,6 +30,8 @@ var (
proxyCAEnd = []byte("# buildkit proxy CA end\n")
)
const maxCertBundleBytes = 10 << 20
// InjectProxyCA appends caPEM to the rootfs trust bundle used by common Linux
// TLS stacks and returns a cleanup that removes only the injected CA.
func InjectProxyCA(rootfsPath string, caPEM []byte) (func() error, error) {
@@ -47,7 +50,7 @@ func InjectProxyCA(rootfsPath string, caPEM []byte) (func() error, error) {
if err != nil {
return nil, errors.Wrapf(err, "failed to resolve certificate bundle %s", name)
}
if st, err := os.Stat(p); err == nil && !st.IsDir() {
if st, err := os.Stat(p); err == nil && st.Mode().IsRegular() {
bundle = p
break
}
@@ -56,17 +59,13 @@ func InjectProxyCA(rootfsPath string, caPEM []byte) (func() error, error) {
return func() error { return nil }, nil
}
original, err := os.ReadFile(bundle)
original, st, err := readCertBundle(bundle)
if err != nil {
return nil, errors.WithStack(err)
return nil, err
}
if containsCertificate(original, certSum) {
return func() error { return nil }, nil
}
st, err := os.Stat(bundle)
if err != nil {
return nil, errors.WithStack(err)
}
next := append([]byte{}, original...)
if len(next) > 0 && next[len(next)-1] != '\n' {
next = append(next, '\n')
@@ -82,25 +81,47 @@ func InjectProxyCA(rootfsPath string, caPEM []byte) (func() error, error) {
}
return func() error {
current, err := os.ReadFile(bundle)
current, st, err := readCertBundle(bundle)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return nil
}
return errors.WithStack(err)
return err
}
cleaned := removeInjectedCA(current, certSum)
if bytes.Equal(current, cleaned) {
return nil
}
st, err := os.Stat(bundle)
if err != nil {
return errors.WithStack(err)
}
return writeCertBundle(bundle, cleaned, st)
}, nil
}
func readCertBundle(path string) ([]byte, os.FileInfo, error) {
f, err := os.Open(path)
if err != nil {
return nil, nil, errors.WithStack(err)
}
defer f.Close()
st, err := f.Stat()
if err != nil {
return nil, nil, errors.WithStack(err)
}
if !st.Mode().IsRegular() {
return nil, nil, errors.Errorf("%s is not a regular file", path)
}
limited := &io.LimitedReader{R: f, N: maxCertBundleBytes + 1}
dt, err := io.ReadAll(limited)
if err != nil {
return nil, nil, errors.WithStack(err)
}
if limited.N == 0 {
return nil, nil, errors.Errorf("%s exceeds %d bytes", path, maxCertBundleBytes)
}
return dt, st, nil
}
func firstCertificate(dt []byte) (*x509.Certificate, error) {
for {
block, rest := pem.Decode(dt)
+4 -4
View File
@@ -51,11 +51,11 @@ func getCgroupCPUStat(cgroupPath string) (*resourcestypes.CPUStat, error) {
switch key {
case cpuUsageUsec:
cpuStat.UsageNanos = uint64Ptr(value * 1000)
cpuStat.UsageNanos = new(value * 1000)
case cpuUserUsec:
cpuStat.UserNanos = uint64Ptr(value * 1000)
cpuStat.UserNanos = new(value * 1000)
case cpuSystemUsec:
cpuStat.SystemNanos = uint64Ptr(value * 1000)
cpuStat.SystemNanos = new(value * 1000)
case cpuNrPeriods:
cpuStat.NrPeriods = new(uint32)
*cpuStat.NrPeriods = uint32(value)
@@ -63,7 +63,7 @@ func getCgroupCPUStat(cgroupPath string) (*resourcestypes.CPUStat, error) {
cpuStat.NrThrottled = new(uint32)
*cpuStat.NrThrottled = uint32(value)
case cpuThrottledUsec:
cpuStat.ThrottledNanos = uint64Ptr(value * 1000)
cpuStat.ThrottledNanos = new(value * 1000)
}
}
+6 -10
View File
@@ -53,37 +53,37 @@ func getCgroupIOStat(cgroupPath string) (*resourcestypes.IOStat, error) {
if ioStat.ReadBytes != nil {
*ioStat.ReadBytes += value
} else {
ioStat.ReadBytes = uint64Ptr(value)
ioStat.ReadBytes = new(value)
}
case ioWriteBytes:
if ioStat.WriteBytes != nil {
*ioStat.WriteBytes += value
} else {
ioStat.WriteBytes = uint64Ptr(value)
ioStat.WriteBytes = new(value)
}
case ioDiscardBytes:
if ioStat.DiscardBytes != nil {
*ioStat.DiscardBytes += value
} else {
ioStat.DiscardBytes = uint64Ptr(value)
ioStat.DiscardBytes = new(value)
}
case ioReadIOs:
if ioStat.ReadIOs != nil {
*ioStat.ReadIOs += value
} else {
ioStat.ReadIOs = uint64Ptr(value)
ioStat.ReadIOs = new(value)
}
case ioWriteIOs:
if ioStat.WriteIOs != nil {
*ioStat.WriteIOs += value
} else {
ioStat.WriteIOs = uint64Ptr(value)
ioStat.WriteIOs = new(value)
}
case ioDiscardIOs:
if ioStat.DiscardIOs != nil {
*ioStat.DiscardIOs += value
} else {
ioStat.DiscardIOs = uint64Ptr(value)
ioStat.DiscardIOs = new(value)
}
}
}
@@ -111,7 +111,3 @@ func parseKeyValue(kv string) (key string, value uint64) {
}
return key, value
}
func uint64Ptr(v uint64) *uint64 {
return &v
}
+49 -1
View File
@@ -3,6 +3,7 @@ package attestation
import (
"context"
"encoding/json"
"io"
"os"
"github.com/containerd/continuity/fs"
@@ -16,6 +17,8 @@ import (
"golang.org/x/sync/errgroup"
)
const maxAttestationBytes int64 = 40 << 20
// ReadAll reads the content of an attestation.
func ReadAll(ctx context.Context, s session.Group, att exporter.Attestation) ([]byte, error) {
var content []byte
@@ -41,7 +44,7 @@ func ReadAll(ctx context.Context, s session.Group, att exporter.Attestation) ([]
if err != nil {
return nil, err
}
content, err = os.ReadFile(p)
content, err = readRegularFile(p)
if err != nil {
return nil, errors.Wrap(err, "cannot read in-toto attestation")
}
@@ -54,6 +57,51 @@ func ReadAll(ctx context.Context, s session.Group, att exporter.Attestation) ([]
return content, nil
}
func readRegularFile(p string) ([]byte, error) {
f, err := openRegularFile(p)
if err != nil {
return nil, err
}
defer f.Close()
dt, err := readAllLimited(f, p, maxAttestationBytes)
if err != nil {
return nil, err
}
return dt, nil
}
func readAllLimited(r io.Reader, name string, limit int64) ([]byte, error) {
limited := &io.LimitedReader{R: r, N: limit + 1}
dt, err := io.ReadAll(limited)
if err != nil {
return nil, errors.WithStack(err)
}
if limited.N == 0 {
return nil, errors.Errorf("%s exceeds %d bytes", name, limit)
}
return dt, nil
}
func openRegularFile(p string) (*os.File, error) {
f, err := os.Open(p)
if err != nil {
return nil, errors.WithStack(err)
}
st, err := f.Stat()
if err != nil {
f.Close()
return nil, errors.WithStack(err)
}
if !st.Mode().IsRegular() {
f.Close()
return nil, errors.Errorf("%s is not a regular file", p)
}
return f, nil
}
// MakeInTotoStatements iterates over all provided result attestations and
// generates intoto attestation statements.
func MakeInTotoStatements(ctx context.Context, s session.Group, attestations []exporter.Attestation, defaultSubjects []intoto.Subject) ([]intoto.Statement, error) {
+28 -7
View File
@@ -132,17 +132,15 @@ func unbundle(root string, bundle exporter.Attestation) ([]exporter.Attestation,
if err != nil {
return nil, err
}
f, err := os.Open(p)
f, err := openRegularFile(p)
if err != nil {
return nil, err
}
dec := json.NewDecoder(f)
var stmt intoto.Statement
if err := dec.Decode(&stmt); err != nil {
return nil, errors.Wrap(err, "cannot decode in-toto statement")
}
if _, err := dec.Token(); !errors.Is(err, io.EOF) {
return nil, errors.New("in-toto statement is not a single JSON object")
stmt, err = decodeStatement(f, p)
f.Close()
if err != nil {
return nil, err
}
if bundle.InToto.PredicateType != "" && stmt.PredicateType != bundle.InToto.PredicateType {
return nil, errors.Errorf("bundle entry %s does not match required predicate type %s", stmt.PredicateType, bundle.InToto.PredicateType)
@@ -175,6 +173,29 @@ func unbundle(root string, bundle exporter.Attestation) ([]exporter.Attestation,
return unbundled, nil
}
func decodeStatement(r io.Reader, name string) (intoto.Statement, error) {
limited := &io.LimitedReader{R: r, N: maxAttestationBytes + 1}
dec := json.NewDecoder(limited)
var stmt intoto.Statement
if err := dec.Decode(&stmt); err != nil {
if limited.N == 0 {
return stmt, errors.Errorf("%s exceeds %d bytes", name, maxAttestationBytes)
}
return stmt, errors.Wrap(err, "cannot decode in-toto statement")
}
if _, err := dec.Token(); !errors.Is(err, io.EOF) {
if limited.N == 0 {
return stmt, errors.Errorf("%s exceeds %d bytes", name, maxAttestationBytes)
}
return stmt, errors.New("in-toto statement is not a single JSON object")
}
if limited.N == 0 {
return stmt, errors.Errorf("%s exceeds %d bytes", name, maxAttestationBytes)
}
return stmt, nil
}
func Validate(atts []exporter.Attestation) error {
for _, att := range atts {
if err := validate(att); err != nil {
+6
View File
@@ -234,6 +234,7 @@ func (e *imageExporterInstance) Export(ctx context.Context, src *exporter.Source
}
opts.Annotations = opts.Annotations.Merge(as)
opts.SetOCITypesDefault(DefaultOCITypes(buildInfo.CompatibilityVersion, src))
opts.SetOCIArtifactDefault(DefaultOCIArtifact(buildInfo.CompatibilityVersion, &opts))
if err := opts.Validate(); err != nil {
return nil, nil, nil, err
}
@@ -590,6 +591,11 @@ func DefaultOCITypes(compatibilityVersion int, src *exporter.Source) bool {
return len(src.Attestations) > 0
}
// DefaultOCIArtifact returns the default attestation manifest format.
func DefaultOCIArtifact(compatibilityVersion int, opts *ImageCommitOpts) bool {
return compatibilityVersion >= compat.CompatibilityVersion031 && opts.OCITypesEnabled()
}
func NewDescriptorReference(desc ocispecs.Descriptor, release func(context.Context) error) exporter.DescriptorReference {
return &descriptorReference{
desc: desc,
+15 -3
View File
@@ -15,7 +15,7 @@ type ImageCommitOpts struct {
ImageName string
RefCfg cacheconfig.RefConfig
OCITypes *bool
OCIArtifact bool
OCIArtifact *bool
Annotations AnnotationsGroup
Epoch *epoch.Epoch
@@ -51,7 +51,9 @@ func (c *ImageCommitOpts) Load(ctx context.Context, opt map[string]string) (map[
err = parseBool(&b, k, v)
c.OCITypes = &b
case exptypes.OptKeyOCIArtifact:
err = parseBool(&c.OCIArtifact, k, v)
var b bool
err = parseBool(&b, k, v)
c.OCIArtifact = &b
case exptypes.OptKeyForceInlineAttestations:
err = parseBool(&c.ForceInlineAttestations, k, v)
case exptypes.OptKeyPreferNondistLayers:
@@ -83,7 +85,7 @@ func (c *ImageCommitOpts) Validate() error {
if c.RefCfg.Compression.Type.OnlySupportOCITypes() && !c.OCITypesEnabled() {
return errors.Errorf("exporter option \"compression=%s\" conflicts with \"oci-mediatypes=false\"", c.RefCfg.Compression.Type)
}
if c.OCIArtifact && !c.OCITypesEnabled() {
if c.OCIArtifactEnabled() && !c.OCITypesEnabled() {
return errors.New("exporter option \"oci-artifact=true\" conflicts with \"oci-mediatypes=false\"")
}
return nil
@@ -99,6 +101,16 @@ func (c *ImageCommitOpts) OCITypesEnabled() bool {
return c.OCITypes != nil && *c.OCITypes
}
func (c *ImageCommitOpts) SetOCIArtifactDefault(v bool) {
if c.OCIArtifact == nil {
c.OCIArtifact = &v
}
}
func (c *ImageCommitOpts) OCIArtifactEnabled() bool {
return c.OCIArtifact != nil && *c.OCIArtifact
}
func parseBool(dest *bool, key string, value string) error {
b, err := strconv.ParseBool(value)
if err != nil {
+1 -1
View File
@@ -342,7 +342,7 @@ func (ic *ImageWriter) Commit(ctx context.Context, inp *exporter.Source, session
return nil, err
}
desc, err := ic.commitAttestationsManifest(ctx, opts, *desc, stmts, opts.OCIArtifact)
desc, err := ic.commitAttestationsManifest(ctx, opts, *desc, stmts, opts.OCIArtifactEnabled())
if err != nil {
return nil, err
}
+1
View File
@@ -148,6 +148,7 @@ func (e *imageExporterInstance) Export(ctx context.Context, src *exporter.Source
}
opts.Annotations = opts.Annotations.Merge(as)
opts.SetOCITypesDefault(e.defaultOCITypes(buildInfo.CompatibilityVersion, src))
opts.SetOCIArtifactDefault(containerimage.DefaultOCIArtifact(buildInfo.CompatibilityVersion, &opts))
if err := opts.Validate(); err != nil {
return nil, nil, nil, err
}
+15 -3
View File
@@ -34,7 +34,7 @@ const (
// attestation.
type Scanner func(ctx context.Context, name string, ref llb.State, extras map[string]llb.State, opts ...llb.ConstraintsOpt) (result.Attestation[*llb.State], error)
func CreateSBOMScanner(ctx context.Context, resolver sourceresolver.MetaResolver, scanner string, resolveOpt sourceresolver.Opt, params map[string]string) (Scanner, error) {
func CreateSBOMScanner(ctx context.Context, resolver sourceresolver.MetaResolver, scanner string, scannerPlatform ocispecs.Platform, resolveOpt sourceresolver.Opt, params map[string]string) (Scanner, error) {
if scanner == "" {
return nil, nil
}
@@ -83,8 +83,10 @@ func CreateSBOMScanner(ctx context.Context, resolver sourceresolver.MetaResolver
runOpts = append(runOpts, llb.AddEnv(k, v))
}
runscan := llb.Image(scanner).Run(runOpts...)
runscan.AddMount("/tmp", llb.Scratch(), llb.Tmpfs())
scannerImage := llb.Image(scanner, llb.Platform(scannerPlatform))
runscan := scannerImage.Run(runOpts...)
tmpState, tmpMountOpt := scannerTmpMount(scannerImage, scannerPlatform)
runscan.AddMount("/tmp", tmpState, tmpMountOpt...)
runscan.AddMount(path.Join(srcDir, "core", CoreSBOMName), ref, llb.Readonly)
for k, extra := range extras {
@@ -106,6 +108,16 @@ func CreateSBOMScanner(ctx context.Context, resolver sourceresolver.MetaResolver
}, nil
}
func scannerTmpMount(scannerImage llb.State, platform ocispecs.Platform) (llb.State, []llb.MountOption) {
if platform.OS == "windows" {
return scannerImage.File(llb.Mkdir("/tmp", 0777, llb.WithParents(true))), []llb.MountOption{
llb.SourcePath("/tmp"),
llb.ForceNoOutput,
}
}
return llb.Scratch(), []llb.MountOption{llb.Tmpfs()}
}
func HasSBOM[T comparable](res *result.Result[T]) bool {
for _, as := range res.Attestations {
for _, a := range as {
+3 -1
View File
@@ -118,9 +118,11 @@ func Build(ctx context.Context, c client.Client) (_ *client.Result, err error) {
var scanner sbom.Scanner
if bc.SBOM != nil {
// TODO: scanner should pass policy
scanner, err = sbom.CreateSBOMScanner(ctx, c, bc.SBOM.Generator, sourceresolver.Opt{
scannerPlatform := bc.BuildPlatforms[0]
scanner, err = sbom.CreateSBOMScanner(ctx, c, bc.SBOM.Generator, scannerPlatform, sourceresolver.Opt{
ImageOpt: &sourceresolver.ResolveImageOpt{
ResolveMode: opts["image-resolve-mode"],
Platform: &scannerPlatform,
},
}, bc.SBOM.Parameters)
if err != nil {
@@ -4,7 +4,7 @@ import (
"context"
"fmt"
"github.com/mitchellh/hashstructure/v2"
"github.com/gohugoio/hashstructure"
"github.com/moby/buildkit/client/llb/sourceresolver"
"github.com/moby/buildkit/frontend/gateway/client"
"github.com/moby/buildkit/util/flightcontrol"
@@ -28,7 +28,7 @@ var _ client.Client = &withResolveCache{}
func (c *withResolveCache) ResolveImageConfig(ctx context.Context, ref string, opt sourceresolver.Opt) (string, digest.Digest, []byte, error) {
c.g.CacheError = true
optHash, err := hashstructure.Hash(opt, hashstructure.FormatV2, nil)
optHash, err := hashstructure.Hash(opt, nil)
if err != nil {
return "", "", nil, err
}
@@ -15,6 +15,7 @@ import (
"github.com/moby/buildkit/frontend/dockerfile/parser"
"github.com/moby/buildkit/frontend/dockerfile/shell"
"github.com/moby/buildkit/util/suggest"
"github.com/moby/patternmatcher"
ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
)
@@ -46,6 +47,13 @@ func validateCopySourcePath(src string, cfg *copyConfig) error {
}
src = filepath.ToSlash(filepath.Clean(src))
if src == "." || src == "/" {
// "." and "/" are context roots, not real paths that can be excluded.
// Only keep the warning for patterns that exclude all root entries.
if !copySourceRootIgnored(cfg.ignoreMatcher) {
return nil
}
}
ok, err := cfg.ignoreMatcher.MatchesOrParentMatches(src)
if err != nil {
return err
@@ -58,6 +66,19 @@ func validateCopySourcePath(src string, cfg *copyConfig) error {
return nil
}
func copySourceRootIgnored(matcher *patternmatcher.PatternMatcher) bool {
for _, pattern := range matcher.Patterns() {
if pattern.Exclusion() {
continue
}
switch pattern.String() {
case "*", "**", "**/*":
return true
}
}
return false
}
func validateCircularDependency(states []*dispatchState) error {
var visit func(*dispatchState, []instructions.Command) []instructions.Command
if states == nil {
+3 -2
View File
@@ -14,6 +14,7 @@ import (
"syscall"
"time"
"github.com/containerd/containerd/v2/defaults"
distreference "github.com/distribution/reference"
"github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/client/llb/sourceresolver"
@@ -1451,8 +1452,8 @@ func grpcClientConn(ctx context.Context) (context.Context, *grpc.ClientConn, err
grpc.WithTransportCredentials(insecure.NewCredentials()),
grpc.WithUnaryInterceptor(grpcerrors.UnaryClientInterceptor),
grpc.WithStreamInterceptor(grpcerrors.StreamClientInterceptor),
grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(16 << 20)),
grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(16 << 20)),
grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(defaults.DefaultMaxRecvMsgSize)),
grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(defaults.DefaultMaxSendMsgSize)),
}
//nolint:staticcheck // ignore SA1019 NewClient has different behavior and needs to be tested
+1 -1
View File
@@ -1,6 +1,6 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.36.11
// protoc-gen-go v1.36.11-devel
// protoc v3.14.0
// source: github.com/moby/buildkit/frontend/gateway/pb/gateway.proto
+1 -1
View File
@@ -1,6 +1,6 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.36.11
// protoc-gen-go v1.36.11-devel
// protoc v3.14.0
// source: github.com/moby/buildkit/session/auth/auth.proto
+1 -1
View File
@@ -1,6 +1,6 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.36.11
// protoc-gen-go v1.36.11-devel
// protoc v3.14.0
// source: github.com/moby/buildkit/session/exporter/exporter.proto
+1 -1
View File
@@ -1,6 +1,6 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.36.11
// protoc-gen-go v1.36.11-devel
// protoc v3.14.0
// source: github.com/moby/buildkit/session/filesync/filesync.proto
+1 -1
View File
@@ -1,6 +1,6 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.36.11
// protoc-gen-go v1.36.11-devel
// protoc v3.14.0
// source: github.com/moby/buildkit/session/secrets/secrets.proto
+1 -1
View File
@@ -1,6 +1,6 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.36.11
// protoc-gen-go v1.36.11-devel
// protoc v3.14.0
// source: github.com/moby/buildkit/session/sshforward/ssh.proto
+1 -1
View File
@@ -1,6 +1,6 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.36.11
// protoc-gen-go v1.36.11-devel
// protoc v3.14.0
// source: github.com/moby/buildkit/session/upload/upload.proto
+2 -1
View File
@@ -26,7 +26,8 @@ type Store struct {
func NewStore(dbPath string) (*Store, error) {
db, err := boltutil.SafeOpen(dbPath, 0600, &bolt.Options{
NoSync: true,
NoSync: true,
FreelistType: bolt.FreelistMapType,
})
if err != nil {
return nil, err
+46
View File
@@ -311,3 +311,49 @@ func debugSchedulerSkipInputRequestBasedOnDepState(e *edge, dep *dep, desiredSta
Debug("skip input request based on dep state")
}
}
// debugSchedulerNoCacheMatchPossible records the moment an edge latches
// noCacheMatchPossible because a dependency exposed no probeable key
// (len(dep.keyMap)==0) once past cache-slow. When the dep nonetheless carries a
// non-empty result cache key (dep_result_keys>0 with dep_keymap==0), that is the
// stale/complete shared-dependency starvation: the key lives on the result but
// was never delivered via edgeState.keys, so probeCache/Query never ran.
func debugSchedulerNoCacheMatchPossible(e *edge, dep *dep, depHasSlowCache bool) {
if e.debug {
depResultKeys := 0
if dep.result != nil {
depResultKeys = len(dep.result.CacheKeys())
}
bklog.G(context.TODO()).
WithField("vtx", e.edge.Vertex.Digest()).
WithField("dep", dep.index).
WithField("dep_state", dep.state).
WithField("dep_keys", len(dep.keys)).
WithField("dep_keymap", len(dep.keyMap)).
WithField("dep_has_slow_cache", depHasSlowCache).
WithField("dep_result", dep.result != nil).
WithField("dep_result_keys", depResultKeys).
Debug("noCacheMatchPossible set: dep yields no probeable cache key")
}
}
// debugSchedulerDepDelivery records what a dependency hands over once it is
// key-bearing (cache-slow) or terminal (complete). The starvation shows up as a
// dep that arrives with dep_edgestate_keys=0 while dep_result_keys>0 -- keys on
// the result but none in edgeState.keys, so probeCache is fed an empty slice.
func debugSchedulerDepDelivery(e *edge, dep *dep, state *edgeState) {
if e.debug {
depResultKeys := 0
if state.result != nil {
depResultKeys = len(state.result.CacheKeys())
}
bklog.G(context.TODO()).
WithField("vtx", e.edge.Vertex.Digest()).
WithField("dep", dep.index).
WithField("dep_state", state.state).
WithField("dep_edgestate_keys", len(state.keys)).
WithField("dep_result", state.result != nil).
WithField("dep_result_keys", depResultKeys).
Debug("dep delivery: edgeState keys vs result keys")
}
}
+16
View File
@@ -220,6 +220,7 @@ func (e *edge) checkDepMatchPossible(dep *dep) {
depHasSlowCache := e.cacheMap.Deps[dep.index].ComputeDigestFunc != nil
if !e.noCacheMatchPossible && (((!dep.slowCacheFoundKey && dep.slowCacheComplete && depHasSlowCache) || (!depHasSlowCache && dep.state >= edgeStatusCacheSlow)) && len(dep.keyMap) == 0) {
e.noCacheMatchPossible = true
debugSchedulerNoCacheMatchPossible(e, dep, depHasSlowCache)
}
}
@@ -633,6 +634,17 @@ func (e *edge) processExecReq() {
e.result = NewSharedCachedResult(upt.Status().Value.(CachedResult))
e.state = edgeStatusComplete
// The keys committed by the execution have so far only existed on the
// result. Add them to e.keys so that a consumer subscribing after this
// edge has completed (a shared edge kept alive by a concurrent build)
// still receives them and can probe the cache. Skipped on cache load
// where the loaded record's key is in e.keys already, and for
// ignore-cache so that consumers cannot match cache records through a
// dependency that was itself forced to re-run.
if !e.execCacheLoad && !e.op.IgnoreCache() {
e.keys = append(e.keys, e.result.CacheKeys()...)
}
}
func (e *edge) processDepReq(dep *dep) (depChanged bool) {
@@ -654,6 +666,10 @@ func (e *edge) processDepReq(dep *dep) (depChanged bool) {
return false
}
if state.state >= edgeStatusCacheSlow {
debugSchedulerDepDelivery(e, dep, state)
}
if len(dep.keys) < len(state.keys) {
newKeys := state.keys[len(dep.keys):]
if e.cacheMap != nil {
+1 -1
View File
@@ -1,6 +1,6 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.36.11
// protoc-gen-go v1.36.11-devel
// protoc v3.14.0
// source: github.com/moby/buildkit/solver/errdefs/errdefs.proto
+15 -14
View File
@@ -65,7 +65,7 @@ type state struct {
clientVertex client.Vertex
origDigest digest.Digest // original LLB digest. TODO: probably better to use string ID so this isn't needed
mu sync.Mutex
mu sync.RWMutex
op *sharedOp
edges map[Index]*edge
opts SolverOpt
@@ -93,12 +93,12 @@ func (s *state) ResolverCache() ResolverCache {
}
func (s *state) CompatibilityVersion() (int, error) {
s.mu.Lock()
s.mu.RLock()
jobs := make([]*Job, 0, len(s.jobs))
for j := range s.jobs {
jobs = append(jobs, j)
}
s.mu.Unlock()
s.mu.RUnlock()
version := 0
for _, j := range jobs {
@@ -122,11 +122,11 @@ func (s *state) CompatibilityVersion() (int, error) {
func (s *state) Lock(key any) (values []any, release func(any) error, err error) {
var rcs []ResolverCache
s.mu.Lock()
s.mu.RLock()
for j := range s.jobs {
rcs = append(rcs, j.resolverCache)
}
s.mu.Unlock()
s.mu.RUnlock()
return combinedResolverCache(rcs).Lock(key)
}
@@ -296,19 +296,20 @@ func (s *state) addJobs(srcState *state, memo map[*state]struct{}) {
}
func (s *state) combinedCacheManager() CacheManager {
s.mu.Lock()
s.mu.RLock()
cms := make([]CacheManager, 0, len(s.cache)+1)
cms = append(cms, s.mainCache)
mainCache := s.mainCache
cms = append(cms, mainCache)
for _, cm := range s.cache {
cms = append(cms, cm)
}
s.mu.Unlock()
s.mu.RUnlock()
if len(cms) == 1 {
return s.mainCache
return mainCache
}
return NewCombinedCacheManager(cms, s.mainCache)
return NewCombinedCacheManager(cms, mainCache)
}
func (s *state) Release() {
@@ -354,12 +355,12 @@ func (sb *subBuilder) InContext(ctx context.Context, f func(context.Context, Job
}
func (sb *subBuilder) EachValue(ctx context.Context, key string, fn func(any) error) error {
sb.state.mu.Lock()
sb.state.mu.RLock()
jobs := make([]*Job, 0, len(sb.jobs))
for j := range sb.jobs {
jobs = append(jobs, j)
}
sb.state.mu.Unlock()
sb.state.mu.RUnlock()
for _, j := range jobs {
if err := j.EachValue(ctx, key, fn); err != nil {
@@ -1287,9 +1288,9 @@ func (s *sharedOp) Exec(ctx context.Context, inputs []Result) (outputs []Result,
func (s *sharedOp) getOp() (Op, error) {
s.opOnce.Do(func() {
s.subBuilder = s.st.builder()
s.st.mu.Lock()
s.st.mu.RLock()
metadata := s.st.metadata
s.st.mu.Unlock()
s.st.mu.RUnlock()
v := s.st.vtx
if metadata != nil {
v = &vertexWithMetadata{Vertex: v, metadata: metadata}
+6 -2
View File
@@ -7,7 +7,7 @@ import (
"time"
"github.com/containerd/platforms"
"github.com/mitchellh/hashstructure/v2"
"github.com/gohugoio/hashstructure"
"github.com/moby/buildkit/cache/remotecache"
"github.com/moby/buildkit/client"
"github.com/moby/buildkit/client/llb/sourceresolver"
@@ -163,6 +163,9 @@ func (b *llbBridge) loadResult(ctx context.Context, def *pb.Definition, cacheImp
}
func (b *llbBridge) policy(engine *sourcepolicy.Engine) SourcePolicyEvaluator {
if engine == nil {
return nil
}
return &policyEvaluator{
llbBridge: b,
engine: engine,
@@ -279,6 +282,7 @@ func (b *llbBridge) resolveSourceMetadata(ctx context.Context, op *pb.SourceOp,
engine := sourcepolicy.NewEngine(opt.SourcePolicies)
if !withPolicy {
op.Identifier = normalizedSourceIdentifier(w, op)
if _, err := engine.Evaluate(ctx, op); err != nil {
return nil, errors.Wrap(err, "could not resolve image due to policy")
}
@@ -382,7 +386,7 @@ func cmKey(im gw.CacheOptionsEntry) (string, error) {
if im.Type == "registry" && im.Attrs["ref"] != "" {
return im.Attrs["ref"], nil
}
i, err := hashstructure.Hash(im, hashstructure.FormatV2, nil)
i, err := hashstructure.Hash(im, nil)
if err != nil {
return "", err
}
+9
View File
@@ -120,6 +120,15 @@ func (b *BuildOp) Exec(ctx context.Context, job solver.JobContext, inputs []solv
if err != nil {
return nil, errors.Wrapf(err, "failed to open %s", newfn)
}
st, err := f.Stat()
if err != nil {
f.Close()
return nil, errors.WithStack(err)
}
if !st.Mode().IsRegular() {
f.Close()
return nil, errors.Errorf("%s is not a regular file", newfn)
}
def, err := llb.ReadFrom(f)
if err != nil {
+17
View File
@@ -13,6 +13,7 @@ import (
"github.com/moby/buildkit/sourcepolicy"
spb "github.com/moby/buildkit/sourcepolicy/pb"
"github.com/moby/buildkit/sourcepolicy/policysession"
"github.com/moby/buildkit/worker"
ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
)
@@ -32,6 +33,17 @@ type policyEvaluator struct {
engine *sourcepolicy.Engine
}
func normalizedSourceIdentifier(w worker.Worker, op *pb.SourceOp) string {
if w == nil {
return op.GetIdentifier()
}
id, err := w.ParseSource(op, nil)
if err != nil {
return op.GetIdentifier()
}
return id.String()
}
func (p *policyEvaluator) Evaluate(ctx context.Context, op *pb.Op) (bool, error) {
return p.evaluate(ctx, op, 10)
}
@@ -41,6 +53,11 @@ func (p *policyEvaluator) evaluate(ctx context.Context, op *pb.Op, max int) (boo
if source == nil {
return false, nil
}
w, err := p.resolveWorker()
if err != nil {
return false, err
}
source.Identifier = normalizedSourceIdentifier(w, source)
ok, err := p.engine.Evaluate(ctx, source)
if err != nil {
return false, err
+4 -2
View File
@@ -13,10 +13,11 @@ import (
"github.com/moby/buildkit/solver/llbsolver"
"github.com/moby/buildkit/solver/result"
"github.com/moby/buildkit/util/tracing"
ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
)
func SBOMProcessor(scannerRef string, useCache bool, resolveMode string, params map[string]string) llbsolver.Processor {
func SBOMProcessor(scannerRef string, scannerPlatform ocispecs.Platform, useCache bool, resolveMode string, params map[string]string) llbsolver.Processor {
return func(ctx context.Context, res *llbsolver.Result, s *llbsolver.Solver, j *solver.Job, usage *resources.SysSampler) (*llbsolver.Result, error) {
// skip sbom generation if we already have an sbom
if sbom.HasSBOM(res.Result) {
@@ -31,9 +32,10 @@ func SBOMProcessor(scannerRef string, useCache bool, resolveMode string, params
return nil, err
}
scanner, err := sbom.CreateSBOMScanner(ctx, s.Bridge(j), scannerRef, sourceresolver.Opt{
scanner, err := sbom.CreateSBOMScanner(ctx, s.Bridge(j), scannerRef, scannerPlatform, sourceresolver.Opt{
ImageOpt: &sourceresolver.ResolveImageOpt{
ResolveMode: resolveMode,
Platform: &scannerPlatform,
},
}, params)
if err != nil {
+1 -1
View File
@@ -1,6 +1,6 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.36.11
// protoc-gen-go v1.36.11-devel
// protoc v3.14.0
// source: github.com/moby/buildkit/solver/pb/ops.proto
+4
View File
@@ -42,6 +42,10 @@ func NewImageBlobIdentifier(str string, scheme string) (*ImageBlobIdentifier, er
var _ source.Identifier = (*ImageBlobIdentifier)(nil)
func (id *ImageBlobIdentifier) String() string {
return id.Scheme() + "://" + id.Reference.String()
}
func (id *ImageBlobIdentifier) Scheme() string {
if id.SchemeName == "" {
return srctypes.DockerImageBlobScheme
+8
View File
@@ -36,6 +36,10 @@ func NewImageIdentifier(str string) (*ImageIdentifier, error) {
var _ source.Identifier = (*ImageIdentifier)(nil)
func (id *ImageIdentifier) String() string {
return srctypes.DockerImageScheme + "://" + id.Reference.String()
}
func (*ImageIdentifier) Scheme() string {
return srctypes.DockerImageScheme
}
@@ -75,6 +79,10 @@ func NewOCIIdentifier(str string) (*OCIIdentifier, error) {
var _ source.Identifier = (*OCIIdentifier)(nil)
func (id *OCIIdentifier) String() string {
return srctypes.OCIScheme + "://" + id.Reference.String()
}
func (*OCIIdentifier) Scheme() string {
return srctypes.OCIScheme
}
+17
View File
@@ -1,6 +1,7 @@
package git
import (
"path"
"strings"
"github.com/containerd/containerd/v2/pkg/reference"
@@ -71,6 +72,22 @@ func NewGitIdentifier(remoteURL string) (*GitIdentifier, error) {
return &repo, nil
}
func (id *GitIdentifier) String() string {
remote := id.Remote
if u, err := gitutil.ParseURL(remote); err == nil {
remote = u.Host + path.Join("/", u.Path)
}
ref := srctypes.GitScheme + "://" + remote
if id.Ref != "" || id.Subdir != "" {
ref += "#" + id.Ref
if id.Subdir != "" {
ref += ":" + id.Subdir
}
}
return ref
}
func (GitIdentifier) Scheme() string {
return srctypes.GitScheme
}
+4
View File
@@ -42,6 +42,10 @@ type HeaderField struct {
var _ source.Identifier = (*HTTPIdentifier)(nil)
func (id *HTTPIdentifier) String() string {
return id.URL
}
func (id *HTTPIdentifier) Scheme() string {
if id.TLS {
return srctypes.HTTPSScheme
+3
View File
@@ -11,6 +11,9 @@ var (
)
type Identifier interface {
// String returns the canonical source operation identifier.
String() string
// Scheme returns the scheme of the identifier so that it can be routed back
// to an appropriate Source.
Scheme() string
+4
View File
@@ -24,6 +24,10 @@ func NewLocalIdentifier(str string) (*LocalIdentifier, error) {
return &LocalIdentifier{Name: str}, nil
}
func (id *LocalIdentifier) String() string {
return srctypes.LocalScheme + "://" + id.Name
}
func (*LocalIdentifier) Scheme() string {
return srctypes.LocalScheme
}
+1 -1
View File
@@ -57,7 +57,7 @@ func (e *Engine) selectorCache(src *spb.Selector) *selectorCache {
// Evaluate evaluates a source operation against the policy.
//
// Policies are re-evaluated for each convert rule.
// Evaluate will error if the there are too many converts for a single source op to prevent infinite loops.
// Evaluate will error if there are too many converts for a single source op to prevent infinite loops.
// This function may error out even if the op was mutated, in which case `true` will be returned along with the error.
//
// An error is returned when the source is denied by the policy.
+1 -1
View File
@@ -1,6 +1,6 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.36.11
// protoc-gen-go v1.36.11-devel
// protoc v3.14.0
// source: github.com/moby/buildkit/sourcepolicy/pb/policy.proto
@@ -1,6 +1,6 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.36.11
// protoc-gen-go v1.36.11-devel
// protoc v3.14.0
// source: github.com/moby/buildkit/sourcepolicy/policysession/policysession.proto
+1 -1
View File
@@ -1,6 +1,6 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.36.11
// protoc-gen-go v1.36.11-devel
// protoc v3.14.0
// source: github.com/moby/buildkit/util/apicaps/pb/caps.proto
+8 -6
View File
@@ -7,7 +7,7 @@ import (
digest "github.com/opencontainers/go-digest"
"github.com/pkg/errors"
"go.etcd.io/bbolt"
bolt "go.etcd.io/bbolt"
)
var ErrInvalidEncoding = errors.Errorf("invalid encoding")
@@ -16,7 +16,7 @@ var ErrNotFound = errors.Errorf("not found")
const bucketName = "byhash"
type DB struct {
db *bbolt.DB
db *bolt.DB
wg sync.WaitGroup
}
@@ -31,7 +31,9 @@ func GetDefaultDB() *DB {
}
func NewDB(path string) (*DB, error) {
db, err := bbolt.Open(path, 0600, nil)
db, err := bolt.Open(path, 0600, &bolt.Options{
FreelistType: bolt.FreelistMapType,
})
if err != nil {
return nil, err
}
@@ -74,7 +76,7 @@ func (d *DB) saveFrames(key string, frames []Frame) {
// Optionally log error
return
}
_ = d.db.Update(func(tx *bbolt.Tx) error {
_ = d.db.Update(func(tx *bolt.Tx) error {
b, err := tx.CreateBucketIfNotExists([]byte(bucketName))
if err != nil {
return err
@@ -94,7 +96,7 @@ func (d *DB) Get(ctx context.Context, dgst string) (Type, []Frame, error) {
}
var typ Type
var resultFrames []Frame
err = d.db.View(func(tx *bbolt.Tx) error {
err = d.db.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(bucketName))
if b == nil {
return errors.WithStack(ErrNotFound)
@@ -127,7 +129,7 @@ func (d *DB) All(ctx context.Context, cb func(key string, typ Type, frames []Fra
if d.db == nil {
return nil
}
return d.db.View(func(tx *bbolt.Tx) error {
return d.db.View(func(tx *bolt.Tx) error {
select {
case <-ctx.Done():
return context.Cause(ctx)
+5 -26
View File
@@ -12,7 +12,11 @@ import (
func (c zstdType) Compress(ctx context.Context, comp Config) (compressorFunc Compressor, finalize Finalizer) {
return func(dest io.Writer, _ string) (io.WriteCloser, error) {
return zstdWriter(comp)(dest)
var opts []zstd.EOption
if comp.Level != nil {
opts = append(opts, zstd.WithEncoderLevel(zstd.EncoderLevelFromZstd(*comp.Level)))
}
return zstd.NewWriter(dest, opts...)
}, nil
}
@@ -49,28 +53,3 @@ func (c zstdType) MediaType() string {
func (c zstdType) String() string {
return "zstd"
}
func zstdWriter(comp Config) func(io.Writer) (io.WriteCloser, error) {
return func(dest io.Writer) (io.WriteCloser, error) {
level := zstd.SpeedDefault
if comp.Level != nil {
level = toZstdEncoderLevel(*comp.Level)
}
return zstd.NewWriter(dest, zstd.WithEncoderLevel(level))
}
}
func toZstdEncoderLevel(level int) zstd.EncoderLevel {
// map zstd compression levels to go-zstd levels
// once we also have c based implementation move this to helper pkg
if level < 0 {
return zstd.SpeedDefault
} else if level < 3 {
return zstd.SpeedFastest
} else if level < 7 {
return zstd.SpeedDefault
} else if level < 9 {
return zstd.SpeedBetterCompression
}
return zstd.SpeedBestCompression
}
+7
View File
@@ -212,6 +212,13 @@ func (cli *GitCLI) Run(ctx context.Context, args ...string) (_ []byte, err error
"HOMEPATH",
"GIT_CONFIG_GLOBAL",
"GIT_CONFIG_SYSTEM",
// When git runs as root under sudo it consults SUDO_UID to trust
// repositories owned by the user who invoked sudo, so client-side
// callers such as `sudo docker build` don't trip git's "detected
// dubious ownership" check and silently lose commit provenance.
// This mirrors git's own default behavior and does not disable
// safe.directory checks. See docker/buildx#3855.
"SUDO_UID",
} {
if v, ok := os.LookupEnv(ev); ok {
cmd.Env = append(cmd.Env, ev+"="+v)
+3 -3
View File
@@ -33,7 +33,7 @@ func (p *ProviderWithProgress) ReaderAt(ctx context.Context, desc ocispecs.Descr
progressCtx, cancel := context.WithCancelCause(context.WithoutCancel(ctx))
doneCh := make(chan struct{})
go trackProgress(progressCtx, desc, p.Manager, doneCh) //nolint:gosec // Progress tracking is tied to the reader lifetime and canceled by Close.
go trackProgress(progressCtx, desc, p.Manager, doneCh) // Progress tracking is tied to the reader lifetime and canceled by Close.
return readerAtWithCancel{ReaderAt: ra, cancel: cancel, doneCh: doneCh, logger: bklog.G(ctx)}, nil
}
@@ -67,7 +67,7 @@ func (f *FetcherWithProgress) Fetch(ctx context.Context, desc ocispecs.Descripto
progressCtx, cancel := context.WithCancelCause(context.WithoutCancel(ctx))
doneCh := make(chan struct{})
go trackProgress(progressCtx, desc, f.Manager, doneCh) //nolint:gosec // Progress tracking is tied to the reader lifetime and canceled by Close.
go trackProgress(progressCtx, desc, f.Manager, doneCh) // Progress tracking is tied to the reader lifetime and canceled by Close.
return readerWithCancel{ReadCloser: rc, cancel: cancel, doneCh: doneCh, logger: bklog.G(ctx)}, nil
}
@@ -110,7 +110,7 @@ func trackProgress(ctx context.Context, desc ocispecs.Descriptor, manager PullMa
case <-ctx.Done():
onFinalStatus = true
// we need a context for the manager.Status() calls to pass once. after that this function will exit
ctx = context.TODO()
ctx = context.WithoutCancel(ctx) //nolint:fatcontext // ignore "nested context in loop"
case <-ticker.C:
}
+14 -2
View File
@@ -14,6 +14,17 @@ const defaultPort = 22
var errCallbackDone = errors.New("callback failed on purpose")
// knownHostsServerID formats a host identifier for a known_hosts entry. A
// non-standard port must be rendered as "[host]:port" so that ssh matches the
// entry when connecting (see the SSH_KNOWN_HOSTS format in sshd(8)); the
// default port is rendered as a bare hostname.
func knownHostsServerID(hostname, port string) string {
if port == "" || port == strconv.Itoa(defaultPort) {
return hostname
}
return fmt.Sprintf("[%s]:%s", hostname, port)
}
// addDefaultPort appends a default port if hostport doesn't contain one
func addDefaultPort(hostport string, defaultPort int) string {
_, _, err := net.SplitHostPort(hostport)
@@ -28,11 +39,12 @@ func addDefaultPort(hostport string, defaultPort int) string {
func SSHKeyScan(server string) (string, error) {
var key string
KeyScanCallback := func(hostport string, remote net.Addr, pubKey ssh.PublicKey) error {
hostname, _, err := net.SplitHostPort(hostport)
hostname, port, err := net.SplitHostPort(hostport)
if err != nil {
return err
}
key = strings.TrimSpace(fmt.Sprintf("%s %s", hostname, string(ssh.MarshalAuthorizedKey(pubKey))))
serverID := knownHostsServerID(hostname, port)
key = strings.TrimSpace(fmt.Sprintf("%s %s", serverID, string(ssh.MarshalAuthorizedKey(pubKey))))
return errCallbackDone
}
config := &ssh.ClientConfig{
+1 -1
View File
@@ -1,6 +1,6 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.36.11
// protoc-gen-go v1.36.11-devel
// protoc v3.14.0
// source: github.com/moby/buildkit/util/stack/stack.proto
+3 -13
View File
@@ -14,23 +14,13 @@ func init() {
func initContext(ctx context.Context) context.Context {
// open-telemetry/opentelemetry-specification#740
parent := os.Getenv("TRACEPARENT")
state := os.Getenv("TRACESTATE")
parent := os.Getenv("TRACEPARENT") // https://www.w3.org/TR/trace-context/#traceparent-header
state := os.Getenv("TRACESTATE") // https://www.w3.org/TR/trace-context/#tracestate-header
if parent != "" {
tc := propagation.TraceContext{}
return tc.Extract(ctx, &textMap{parent: parent, state: state})
}
// deprecated: removed in v0.11.0
// previously defined in https://github.com/open-telemetry/opentelemetry-swift/blob/4ea467ed4b881d7329bf2254ca7ed7f2d9d6e1eb/Sources/OpenTelemetrySdk/Trace/Propagation/EnvironmentContextPropagator.swift#L14-L15
parent = os.Getenv("OTEL_TRACE_PARENT")
state = os.Getenv("OTEL_TRACE_STATE")
if parent == "" {
return ctx
}
tc := propagation.TraceContext{}
return tc.Extract(ctx, &textMap{parent: parent, state: state})
return ctx
}
@@ -15,15 +15,6 @@ func Environ(ctx context.Context) []string {
var env []string
// deprecated: removed in v0.11.0
// previously defined in https://github.com/open-telemetry/opentelemetry-swift/blob/4ea467ed4b881d7329bf2254ca7ed7f2d9d6e1eb/Sources/OpenTelemetrySdk/Trace/Propagation/EnvironmentContextPropagator.swift#L14-L15
if tm.parent != "" {
env = append(env, "OTEL_TRACE_PARENT="+tm.parent)
}
if tm.state != "" {
env = append(env, "OTEL_TRACE_STATE="+tm.state)
}
// open-telemetry/opentelemetry-specification#740
if tm.parent != "" {
env = append(env, "TRACEPARENT="+tm.parent)
+1 -1
View File
@@ -45,7 +45,7 @@ func (fn TraceExporterDetector) DetectMetricExporter() (sdkmetric.Exporter, erro
}
func detectExporter[T any](envVar string, fn func(d ExporterDetector) (T, bool, error)) (exp T, err error) {
ignoreErrors, _ := strconv.ParseBool("OTEL_IGNORE_ERROR")
ignoreErrors, _ := strconv.ParseBool(os.Getenv("OTEL_IGNORE_ERROR"))
if n := os.Getenv(envVar); n != "" {
d, ok := detectors[n]
+5 -1
View File
@@ -465,6 +465,10 @@ func (w *Worker) PruneCacheMounts(ctx context.Context, ids map[string]bool) erro
return nil
}
func (w *Worker) ParseSource(op *pb.SourceOp, platform *pb.Platform) (source.Identifier, error) {
return w.SourceManager.Identifier(&pb.Op_Source{Source: op}, platform)
}
func (w *Worker) ResolveSourceMetadata(ctx context.Context, op *pb.SourceOp, opt sourceresolver.Opt, sm *session.Manager, jobCtx solver.JobContext) (*sourceresolver.MetaResponse, error) {
if opt.SourcePolicies != nil {
return nil, errors.New("source policies can not be set for worker")
@@ -487,7 +491,7 @@ func (w *Worker) ResolveSourceMetadata(ctx context.Context, op *pb.SourceOp, opt
}
}
id, err := w.SourceManager.Identifier(&pb.Op_Source{Source: op}, platform)
id, err := w.ParseSource(op, platform)
if err != nil {
return nil, err
}
+2
View File
@@ -15,6 +15,7 @@ import (
"github.com/moby/buildkit/solver"
"github.com/moby/buildkit/solver/llbsolver/cdidevices"
"github.com/moby/buildkit/solver/pb"
"github.com/moby/buildkit/source"
"github.com/moby/buildkit/util/leaseutil"
"github.com/moby/buildkit/util/network"
ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
@@ -37,6 +38,7 @@ type Worker interface {
LoadRef(ctx context.Context, id string, hidden bool) (cache.ImmutableRef, error)
// ResolveOp resolves Vertex.Sys() to Op implementation.
ResolveOp(v solver.Vertex, s frontend.FrontendLLBBridge, sm *session.Manager, proxyOpt ProxyOpt) (solver.Op, error)
ParseSource(op *pb.SourceOp, platform *pb.Platform) (source.Identifier, error)
ResolveSourceMetadata(ctx context.Context, op *pb.SourceOp, opt sourceresolver.Opt, sm *session.Manager, jobCtx solver.JobContext) (*sourceresolver.MetaResponse, error)
DiskUsage(ctx context.Context, opt client.DiskUsageInfo) ([]*client.UsageInfo, error)
Exporter(name string, sm *session.Manager) (exporter.Exporter, error)
+5 -5
View File
@@ -795,6 +795,9 @@ github.com/gogo/protobuf/sortkeys
github.com/gogo/protobuf/types
github.com/gogo/protobuf/vanity
github.com/gogo/protobuf/vanity/command
# github.com/gohugoio/hashstructure v0.6.0
## explicit; go 1.20
github.com/gohugoio/hashstructure
# github.com/golang-jwt/jwt/v5 v5.3.1
## explicit; go 1.21
github.com/golang-jwt/jwt/v5
@@ -978,14 +981,11 @@ github.com/mistifyio/go-zfs/v4
# github.com/mitchellh/copystructure v1.2.0
## explicit; go 1.15
github.com/mitchellh/copystructure
# github.com/mitchellh/hashstructure/v2 v2.0.2
## explicit; go 1.14
github.com/mitchellh/hashstructure/v2
# github.com/mitchellh/reflectwalk v1.0.2
## explicit
github.com/mitchellh/reflectwalk
# github.com/moby/buildkit v0.31.2
## explicit; go 1.25.9
# github.com/moby/buildkit v0.32.0-rc1
## explicit; go 1.26.3
github.com/moby/buildkit/api/services/control
github.com/moby/buildkit/api/types
github.com/moby/buildkit/cache