diff --git a/hack/buildkit-ref b/hack/buildkit-ref index 280bb0e25c..f51a489593 100755 --- a/hack/buildkit-ref +++ b/hack/buildkit-ref @@ -19,6 +19,9 @@ if [[ "${buildkit_ref}" == *-*-* ]]; then buildkit_ref=$(curl -s "https://api.github.com/repos/${buildkit_repo}/commits/${buildkit_ref}" | jq -r .sha) fi +# https://github.com/moby/buildkit/pull/6278 +buildkit_ref="1030099b27bd3455bf7e5d5fe73b6be5dbec3c1f" + cat << EOF BUILDKIT_REPO=$buildkit_repo BUILDKIT_REF=$buildkit_ref diff --git a/hack/cpexp/cpexp.go b/hack/cpexp/cpexp.go new file mode 100644 index 0000000000..c58d062871 --- /dev/null +++ b/hack/cpexp/cpexp.go @@ -0,0 +1,127 @@ +package main + +import ( + "log" + "os" + "path/filepath" + "strings" + + "github.com/moby/buildkit/identity" + "github.com/moby/sys/mountinfo" + "github.com/pkg/errors" +) + +const volumePath = "/abc/a" + +var rootfs string + +func main() { + if err := run(); err != nil { + log.Printf("error: %+v", err) + } +} + +func run() error { + infos, err := mountinfo.GetMounts(nil) + if err != nil { + return err + } + hasVolume := false + for _, info := range infos { + if info.Mountpoint == "/" { + v, err := getOverlayRootfs(info) + if err != nil { + return err + } + rootfs = v + } + + if info.Mountpoint == volumePath { + hasVolume = true + } + log.Printf("mount: %+v", info) + } + if !hasVolume { + return errors.Errorf("volume not found: %s", volumePath) + } + + log.Printf("rootfs: %s", rootfs) + + base := filepath.Base(rootfs) + if err := os.Symlink("/", filepath.Join(volumePath, base)); err != nil { + return err + } + + // create duplicate volume path with symlink target + p := "/" + var volumeRoot string + for _, c := range strings.Split(filepath.Dir(volumePath), string(filepath.Separator)) { + if c == "" { + continue + } + if volumeRoot == "" { + volumeRoot = "/" + c + c += "_target" + } + p = filepath.Join(p, c) + if err := os.Mkdir(p, 0755); err != nil { + if os.IsExist(err) { + continue + } + return err + } + log.Printf("created: %s", p) + } + if err := os.Symlink(filepath.Dir(rootfs), filepath.Join(p, filepath.Base(volumePath))); err != nil { + return err + } + + if err := os.Rename(volumeRoot, volumeRoot+"_old"); err != nil { + return err + } + + for { + if _, err := os.Stat(volumeRoot); err != nil { + if os.IsNotExist(err) { + continue + } + return err + } + // log.Printf("detected: %s", volumeRoot) + break + } + + for { + if err := os.Rename(volumeRoot+"_target", volumeRoot); err != nil { + if os.IsExist(err) { + if err := os.Rename(volumeRoot, volumeRoot+"_"+identity.NewID()); err != nil { + return err + } + continue + } + log.Printf("failed to rename: %s", err) + } + break + } + + return nil +} + +func getOverlayRootfs(info *mountinfo.Info) (string, error) { + if info.FSType != "overlay" { + return "", errors.Errorf("not overlay: %s", info.FSType) + } + for _, opt := range strings.Split(info.VFSOptions, ",") { + parts := strings.SplitN(opt, "=", 2) + if parts[0] == "workdir" { + return filepath.Join(filepath.Dir(parts[1]), "merged"), nil + } + } + return "", errors.Errorf("workdir not found: %s", info.VFSOptions) +} + +// /var/lib/docker/overlay2/86fcb18db0e3774cfe3b9ed6fa526d4dffcb45ac3b26cbe99db6c2d08e6dfc0e/merged + +// merged//dir1/dir2/ + +// volume diff --git a/vendor.mod b/vendor.mod index a151677726..d1ccaee16c 100644 --- a/vendor.mod +++ b/vendor.mod @@ -62,7 +62,7 @@ require ( github.com/miekg/dns v1.1.66 github.com/mistifyio/go-zfs/v3 v3.0.1 github.com/mitchellh/copystructure v1.2.0 - github.com/moby/buildkit v0.25.0 + github.com/moby/buildkit v0.25.1 github.com/moby/docker-image-spec v1.3.1 github.com/moby/go-archive v0.1.0 github.com/moby/ipvs v1.1.0 diff --git a/vendor.sum b/vendor.sum index 5ddc82a192..dff7f36e35 100644 --- a/vendor.sum +++ b/vendor.sum @@ -383,8 +383,8 @@ github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:F github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mndrix/tap-go v0.0.0-20171203230836-629fa407e90b/go.mod h1:pzzDgJWZ34fGzaAZGFW22KVZDfyrYW+QABMrWnJBnSs= -github.com/moby/buildkit v0.25.0 h1:cRgh74ymzyHxS5a/lsYT4OCyVU8iC3UgkwasIEUi0og= -github.com/moby/buildkit v0.25.0/go.mod h1:phM8sdqnvgK2y1dPDnbwI6veUCXHOZ6KFSl6E164tkc= +github.com/moby/buildkit v0.25.1 h1:j7IlVkeNbEo+ZLoxdudYCHpmTsbwKvhgc/6UJ/mY/o8= +github.com/moby/buildkit v0.25.1/go.mod h1:phM8sdqnvgK2y1dPDnbwI6veUCXHOZ6KFSl6E164tkc= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/go-archive v0.1.0 h1:Kk/5rdW/g+H8NHdJW2gsXyZ7UnzvJNOy6VKJqueWdcQ= diff --git a/vendor/github.com/moby/buildkit/cache/remotecache/gha/gha.go b/vendor/github.com/moby/buildkit/cache/remotecache/gha/gha.go index c7ef08be73..5e2e019144 100644 --- a/vendor/github.com/moby/buildkit/cache/remotecache/gha/gha.go +++ b/vendor/github.com/moby/buildkit/cache/remotecache/gha/gha.go @@ -14,6 +14,7 @@ import ( "github.com/containerd/containerd/v2/core/content" "github.com/containerd/containerd/v2/pkg/labels" + cerrdefs "github.com/containerd/errdefs" "github.com/moby/buildkit/cache/remotecache" v1 "github.com/moby/buildkit/cache/remotecache/v1" "github.com/moby/buildkit/session" @@ -440,7 +441,7 @@ type ciProvider struct { func (p *ciProvider) Info(ctx context.Context, dgst digest.Digest) (content.Info, error) { if dgst != p.desc.Digest { - return content.Info{}, errors.Errorf("content not found %s", dgst) + return content.Info{}, errors.Wrapf(cerrdefs.ErrNotFound, "blob %s", dgst) } if _, err := p.loadEntry(ctx, p.desc); err != nil { @@ -465,7 +466,7 @@ func (p *ciProvider) loadEntry(ctx context.Context, desc ocispecs.Descriptor) (* return nil, err } if ce == nil { - return nil, errors.Errorf("blob %s not found", desc.Digest) + return nil, errors.Wrapf(cerrdefs.ErrNotFound, "blob %s", desc.Digest) } if p.entries == nil { p.entries = make(map[digest.Digest]*actionscache.Entry) diff --git a/vendor/github.com/moby/buildkit/cache/remotecache/v1/chains.go b/vendor/github.com/moby/buildkit/cache/remotecache/v1/chains.go index 7f80399e53..1a89400b5c 100644 --- a/vendor/github.com/moby/buildkit/cache/remotecache/v1/chains.go +++ b/vendor/github.com/moby/buildkit/cache/remotecache/v1/chains.go @@ -10,6 +10,7 @@ import ( "github.com/cespare/xxhash/v2" "github.com/containerd/containerd/v2/core/content" + cerrdefs "github.com/containerd/errdefs" "github.com/moby/buildkit/session" "github.com/moby/buildkit/solver" digest "github.com/opencontainers/go-digest" @@ -246,7 +247,7 @@ func (p DescriptorProviderPair) Info(ctx context.Context, dgst digest.Digest) (c return p.InfoProvider.Info(ctx, dgst) } if dgst != p.Descriptor.Digest { - return content.Info{}, errors.Errorf("content not found %s", dgst) + return content.Info{}, errors.Wrapf(cerrdefs.ErrNotFound, "blob %s", dgst) } return content.Info{ Digest: p.Descriptor.Digest, diff --git a/vendor/github.com/moby/buildkit/solver/exporter.go b/vendor/github.com/moby/buildkit/solver/exporter.go index 3f6dc1ccf5..6ade2f44ce 100644 --- a/vendor/github.com/moby/buildkit/solver/exporter.go +++ b/vendor/github.com/moby/buildkit/solver/exporter.go @@ -5,6 +5,7 @@ import ( "errors" "slices" + cerrdefs "github.com/containerd/errdefs" digest "github.com/opencontainers/go-digest" ) @@ -189,24 +190,28 @@ func (e *exporter) ExportTo(ctx context.Context, t CacheExporterTarget, opt Cach if (remote == nil || opt.CompressionOpt != nil) && opt.Mode != CacheExportModeRemoteOnly { res, err := cm.results.Load(ctx, res) if err != nil { - return nil, err - } - remotes, err := opt.ResolveRemotes(ctx, res) - if err != nil { - return nil, err - } - res.Release(context.TODO()) - if remote == nil && len(remotes) > 0 { - remote, remotes = remotes[0], remotes[1:] // pop the first element - } - if opt.CompressionOpt != nil { - for _, r := range remotes { // record all remaining remotes as well - results = append(results, CacheExportResult{ - CreatedAt: v.CreatedAt, - Result: r, - EdgeVertex: k.vtx, - EdgeIndex: k.output, - }) + if !errors.Is(err, cerrdefs.ErrNotFound) { + return nil, err + } + remote = nil + } else { + remotes, err := opt.ResolveRemotes(ctx, res) + if err != nil { + return nil, err + } + res.Release(context.TODO()) + if remote == nil && len(remotes) > 0 { + remote, remotes = remotes[0], remotes[1:] // pop the first element + } + if opt.CompressionOpt != nil { + for _, r := range remotes { // record all remaining remotes as well + results = append(results, CacheExportResult{ + CreatedAt: v.CreatedAt, + Result: r, + EdgeVertex: k.vtx, + EdgeIndex: k.output, + }) + } } } } @@ -232,7 +237,7 @@ func (e *exporter) ExportTo(ctx context.Context, t CacheExporterTarget, opt Cach for _, dep := range deps { rec, err := dep.CacheKey.Exporter.ExportTo(ctx, t, opt) if err != nil { - return nil, err + continue } for _, r := range rec { srcs[i] = append(srcs[i], CacheLink{Src: r, Selector: string(dep.Selector)}) @@ -244,7 +249,7 @@ func (e *exporter) ExportTo(ctx context.Context, t CacheExporterTarget, opt Cach for _, de := range e.edge.secondaryExporters { recs, err := de.cacheKey.CacheKey.Exporter.ExportTo(mainCtx, t, opt) if err != nil { - return nil, nil + continue } for _, r := range recs { srcs[de.index] = append(srcs[de.index], CacheLink{Src: r, Selector: de.cacheKey.Selector.String()}) @@ -261,6 +266,14 @@ func (e *exporter) ExportTo(ctx context.Context, t CacheExporterTarget, opt Cach } } + // validate deps are present + for _, deps := range srcs { + if len(deps) == 0 { + res[e] = nil + return res[e], nil + } + } + if v != nil && len(deps) == 0 { cm := v.cacheManager key := cm.getID(v.key) diff --git a/vendor/github.com/moby/buildkit/solver/llbsolver/provenance/types/types.go b/vendor/github.com/moby/buildkit/solver/llbsolver/provenance/types/types.go index bf0efabc85..382a34b054 100644 --- a/vendor/github.com/moby/buildkit/solver/llbsolver/provenance/types/types.go +++ b/vendor/github.com/moby/buildkit/solver/llbsolver/provenance/types/types.go @@ -1,6 +1,8 @@ package types import ( + "encoding/json" + "maps" "slices" slsa "github.com/in-toto/in-toto-golang/in_toto/slsa_provenance/common" @@ -311,3 +313,95 @@ func (p *ProvenancePredicateSLSA02) ConvertToSLSA1() *ProvenancePredicateSLSA1 { RunDetails: runDetails, } } + +// MarshalJSON flattens ProvenanceCustomEnv into top level. +func (p ProvenanceInternalParametersSLSA1) MarshalJSON() ([]byte, error) { + type Alias ProvenanceInternalParametersSLSA1 + base, err := json.Marshal(Alias(p)) + if err != nil { + return nil, err + } + var m map[string]any + if err := json.Unmarshal(base, &m); err != nil { + return nil, err + } + maps.Copy(m, p.ProvenanceCustomEnv) + delete(m, "ProvenanceCustomEnv") + return json.Marshal(m) +} + +// UnmarshalJSON fills both struct fields and flattened custom env. +func (p *ProvenanceInternalParametersSLSA1) UnmarshalJSON(data []byte) error { + var m map[string]any + if err := json.Unmarshal(data, &m); err != nil { + return err + } + + type Alias ProvenanceInternalParametersSLSA1 + var a Alias + if err := json.Unmarshal(data, &a); err != nil { + return err + } + + // Unmarshal known struct again to identify its keys + structBytes, err := json.Marshal(a) + if err != nil { + return err + } + var known map[string]any + if err := json.Unmarshal(structBytes, &known); err != nil { + return err + } + + for k := range known { + delete(m, k) + } + + *p = ProvenanceInternalParametersSLSA1(a) + p.ProvenanceCustomEnv = m + return nil +} + +func (p Environment) MarshalJSON() ([]byte, error) { + type Alias Environment + base, err := json.Marshal(Alias(p)) + if err != nil { + return nil, err + } + var m map[string]any + if err := json.Unmarshal(base, &m); err != nil { + return nil, err + } + maps.Copy(m, p.ProvenanceCustomEnv) + delete(m, "ProvenanceCustomEnv") + return json.Marshal(m) +} + +func (p *Environment) UnmarshalJSON(data []byte) error { + var m map[string]any + if err := json.Unmarshal(data, &m); err != nil { + return err + } + + type Alias Environment + var a Alias + if err := json.Unmarshal(data, &a); err != nil { + return err + } + // Unmarshal known struct again to identify its keys + structBytes, err := json.Marshal(a) + if err != nil { + return err + } + var known map[string]any + if err := json.Unmarshal(structBytes, &known); err != nil { + return err + } + + for k := range known { + delete(m, k) + } + *p = Environment(a) + p.ProvenanceCustomEnv = m + return nil +} diff --git a/vendor/github.com/moby/buildkit/solver/llbsolver/solver.go b/vendor/github.com/moby/buildkit/solver/llbsolver/solver.go index d1b3c7994e..c5f0e02f63 100644 --- a/vendor/github.com/moby/buildkit/solver/llbsolver/solver.go +++ b/vendor/github.com/moby/buildkit/solver/llbsolver/solver.go @@ -847,6 +847,7 @@ func (s *Solver) runExporters(ctx context.Context, exporters []exporter.Exporter eg, ctx := errgroup.WithContext(ctx) resps := make([]map[string]string, len(exporters)) descs := make([]exporter.DescriptorReference, len(exporters)) + var inlineCacheMu sync.Mutex for i, exp := range exporters { i, exp := i, exp eg.Go(func() error { @@ -865,6 +866,8 @@ func (s *Solver) runExporters(ctx context.Context, exporters []exporter.Exporter } } inlineCache := exptypes.InlineCache(func(ctx context.Context) (*result.Result[*exptypes.InlineCacheEntry], error) { + inlineCacheMu.Lock() // ensure only one inline cache exporter runs at a time + defer inlineCacheMu.Unlock() return runInlineCacheExporter(ctx, exp, inlineCacheExporter, job, cached) }) diff --git a/vendor/modules.txt b/vendor/modules.txt index e23faa4ed4..6f28e8a682 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -757,7 +757,7 @@ github.com/mitchellh/hashstructure/v2 # github.com/mitchellh/reflectwalk v1.0.2 ## explicit github.com/mitchellh/reflectwalk -# github.com/moby/buildkit v0.25.0 +# github.com/moby/buildkit v0.25.1 ## explicit; go 1.24.0 github.com/moby/buildkit/api/services/control github.com/moby/buildkit/api/types