diff --git a/vendor.conf b/vendor.conf index eea53c5d85..c50dcf5899 100644 --- a/vendor.conf +++ b/vendor.conf @@ -33,7 +33,7 @@ github.com/imdario/mergo 1afb36080aec31e0d1528973ebe6 golang.org/x/sync cd5d95a43a6e21273425c7ae415d3df9ea832eeb # buildkit -github.com/moby/buildkit fcb87e6b8ccf3631a65799cc56caa76f9117816e # v0.8.0-rc2 +github.com/moby/buildkit 950603da215ae03b843f3f66fbe86c4876a6f5a1 github.com/tonistiigi/fsutil 0834f99b7b85462efb69b4f571a4fa3ca7da5ac9 github.com/tonistiigi/units 6950e57a87eaf136bbe44ef2ec8e75b9e3569de2 github.com/grpc-ecosystem/grpc-opentracing 8e809c8a86450a29b90dcc9efbf062d0fe6d9746 diff --git a/vendor/github.com/moby/buildkit/README.md b/vendor/github.com/moby/buildkit/README.md index 0af9e2e126..4532dd7cde 100644 --- a/vendor/github.com/moby/buildkit/README.md +++ b/vendor/github.com/moby/buildkit/README.md @@ -3,8 +3,9 @@ # BuildKit [![GoDoc](https://godoc.org/github.com/moby/buildkit?status.svg)](https://godoc.org/github.com/moby/buildkit/client/llb) -[![Build Status](https://travis-ci.org/moby/buildkit.svg?branch=master)](https://travis-ci.org/moby/buildkit) +[![Build Status](https://travis-ci.com/moby/buildkit.svg?branch=master)](https://travis-ci.com/moby/buildkit) [![Go Report Card](https://goreportcard.com/badge/github.com/moby/buildkit)](https://goreportcard.com/report/github.com/moby/buildkit) +[![codecov](https://codecov.io/gh/moby/buildkit/branch/master/graph/badge.svg)](https://codecov.io/gh/moby/buildkit) BuildKit is a toolkit for converting source code to build artifacts in an efficient, expressive and repeatable manner. diff --git a/vendor/github.com/moby/buildkit/cache/refs.go b/vendor/github.com/moby/buildkit/cache/refs.go index 2b25235e44..a17f6e10de 100644 --- a/vendor/github.com/moby/buildkit/cache/refs.go +++ b/vendor/github.com/moby/buildkit/cache/refs.go @@ -399,8 +399,10 @@ func (sr *immutableRef) Extract(ctx context.Context, s session.Group) (rerr erro ctx = winlayers.UseWindowsLayerMode(ctx) } - if _, err := sr.prepareRemoteSnapshots(ctx, sr.descHandlers); err != nil { - return err + if sr.cm.Snapshotter.Name() == "stargz" { + if _, err := sr.prepareRemoteSnapshots(ctx, sr.descHandlers); err != nil { + return err + } } return sr.extract(ctx, sr.descHandlers, s) @@ -454,6 +456,9 @@ func (sr *immutableRef) prepareRemoteSnapshots(ctx context.Context, dhs DescHand // This layer cannot be prepared without unlazying. return false, nil }) + if err != nil { + return false, err + } return ok.(bool), err } diff --git a/vendor/github.com/moby/buildkit/frontend/dockerfile/builder/build.go b/vendor/github.com/moby/buildkit/frontend/dockerfile/builder/build.go index f182ae9663..345e789849 100644 --- a/vendor/github.com/moby/buildkit/frontend/dockerfile/builder/build.go +++ b/vendor/github.com/moby/buildkit/frontend/dockerfile/builder/build.go @@ -130,8 +130,15 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) { name := "load build definition from " + filename + filenames := []string{filename, filename + ".dockerignore"} + + // dockerfile is also supported casing moby/moby#10858 + if path.Base(filename) == defaultDockerfileName { + filenames = append(filenames, path.Join(path.Dir(filename), strings.ToLower(defaultDockerfileName))) + } + src := llb.Local(localNameDockerfile, - llb.FollowPaths([]string{filename, filename + ".dockerignore"}), + llb.FollowPaths(filenames), llb.SessionID(c.BuildOpts().SessionID), llb.SharedKeyHint(localNameDockerfile), dockerfile2llb.WithInternalName(name), @@ -258,7 +265,19 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) { Filename: filename, }) if err != nil { - return errors.Wrapf(err, "failed to read dockerfile") + fallback := false + if path.Base(filename) == defaultDockerfileName { + var err1 error + dtDockerfile, err1 = ref.ReadFile(ctx2, client.ReadRequest{ + Filename: path.Join(path.Dir(filename), strings.ToLower(defaultDockerfileName)), + }) + if err1 == nil { + fallback = true + } + } + if !fallback { + return errors.Wrapf(err, "failed to read dockerfile") + } } sourceMap = llb.NewSourceMap(&src, filename, dtDockerfile) diff --git a/vendor/github.com/moby/buildkit/frontend/gateway/gateway.go b/vendor/github.com/moby/buildkit/frontend/gateway/gateway.go index 4067f87f10..029468b974 100644 --- a/vendor/github.com/moby/buildkit/frontend/gateway/gateway.go +++ b/vendor/github.com/moby/buildkit/frontend/gateway/gateway.go @@ -255,7 +255,7 @@ func (gf *gatewayFrontend) Solve(ctx context.Context, llbBridge frontend.Fronten err = w.Executor().Run(ctx, "", mountWithSession(rootFS, session.NewGroup(sid)), nil, executor.ProcessInfo{Meta: meta, Stdin: lbf.Stdin, Stdout: lbf.Stdout, Stderr: os.Stderr}, nil) if err != nil { - if errors.Is(err, context.Canceled) && lbf.isErrServerClosed { + if errdefs.IsCanceled(err) && lbf.isErrServerClosed { err = errors.Errorf("frontend grpc server closed unexpectedly") } // An existing error (set via Return rpc) takes diff --git a/vendor/github.com/moby/buildkit/solver/edge.go b/vendor/github.com/moby/buildkit/solver/edge.go index 6ebf3a74db..125c55fcfe 100644 --- a/vendor/github.com/moby/buildkit/solver/edge.go +++ b/vendor/github.com/moby/buildkit/solver/edge.go @@ -359,6 +359,7 @@ func (e *edge) unpark(incoming []pipe.Sender, updates, allPipes []pipe.Receiver, if e.execReq == nil { if added := e.createInputRequests(desiredState, f, false); !added && !e.hasActiveOutgoing && !cacheMapReq { logrus.Errorf("buildkit scheluding error: leaving incoming open. forcing solve. Please report this with BUILDKIT_SCHEDULER_DEBUG=1") + debugSchedulerPreUnpark(e, incoming, updates, allPipes) e.createInputRequests(desiredState, f, true) } } diff --git a/vendor/github.com/moby/buildkit/solver/errdefs/context.go b/vendor/github.com/moby/buildkit/solver/errdefs/context.go new file mode 100644 index 0000000000..ea6bdfbf09 --- /dev/null +++ b/vendor/github.com/moby/buildkit/solver/errdefs/context.go @@ -0,0 +1,13 @@ +package errdefs + +import ( + "context" + "errors" + + "github.com/moby/buildkit/util/grpcerrors" + "google.golang.org/grpc/codes" +) + +func IsCanceled(err error) bool { + return errors.Is(err, context.Canceled) || grpcerrors.Code(err) == codes.Canceled +} diff --git a/vendor/github.com/moby/buildkit/solver/jobs.go b/vendor/github.com/moby/buildkit/solver/jobs.go index 932d9d0f1e..67f1a30f94 100644 --- a/vendor/github.com/moby/buildkit/solver/jobs.go +++ b/vendor/github.com/moby/buildkit/solver/jobs.go @@ -659,6 +659,7 @@ func (s *sharedOp) CalcSlowCache(ctx context.Context, index Index, p PreprocessF case <-ctx.Done(): if strings.Contains(err.Error(), context.Canceled.Error()) { complete = false + releaseError(err) err = errors.Wrap(ctx.Err(), err.Error()) } default: @@ -717,6 +718,7 @@ func (s *sharedOp) CacheMap(ctx context.Context, index int) (resp *cacheMapResp, case <-ctx.Done(): if strings.Contains(err.Error(), context.Canceled.Error()) { complete = false + releaseError(err) err = errors.Wrap(ctx.Err(), err.Error()) } default: @@ -774,6 +776,7 @@ func (s *sharedOp) Exec(ctx context.Context, inputs []Result) (outputs []Result, case <-ctx.Done(): if strings.Contains(err.Error(), context.Canceled.Error()) { complete = false + releaseError(err) err = errors.Wrap(ctx.Err(), err.Error()) } default: @@ -911,3 +914,15 @@ func WrapSlowCache(err error, index Index, res Result) error { } return &SlowCacheError{Index: index, Result: res, error: err} } + +func releaseError(err error) { + if err == nil { + return + } + if re, ok := err.(interface { + Release() error + }); ok { + re.Release() + } + releaseError(errors.Unwrap(err)) +} diff --git a/vendor/github.com/moby/buildkit/solver/llbsolver/bridge.go b/vendor/github.com/moby/buildkit/solver/llbsolver/bridge.go index 52a88a2364..e6a5850fef 100644 --- a/vendor/github.com/moby/buildkit/solver/llbsolver/bridge.go +++ b/vendor/github.com/moby/buildkit/solver/llbsolver/bridge.go @@ -16,6 +16,7 @@ import ( "github.com/moby/buildkit/session" "github.com/moby/buildkit/solver" "github.com/moby/buildkit/solver/errdefs" + llberrdefs "github.com/moby/buildkit/solver/llbsolver/errdefs" "github.com/moby/buildkit/solver/pb" "github.com/moby/buildkit/util/flightcontrol" "github.com/moby/buildkit/worker" @@ -144,41 +145,60 @@ func (b *llbBridge) Solve(ctx context.Context, req frontend.SolveRequest, sid st } type resultProxy struct { - cb func(context.Context) (solver.CachedResult, error) - def *pb.Definition - g flightcontrol.Group - mu sync.Mutex - released bool - v solver.CachedResult - err error + cb func(context.Context) (solver.CachedResult, error) + def *pb.Definition + g flightcontrol.Group + mu sync.Mutex + released bool + v solver.CachedResult + err error + errResults []solver.Result } func newResultProxy(b *llbBridge, req frontend.SolveRequest) *resultProxy { - return &resultProxy{ + rp := &resultProxy{ def: req.Definition, - cb: func(ctx context.Context) (solver.CachedResult, error) { - return b.loadResult(ctx, req.Definition, req.CacheImports) - }, } + rp.cb = func(ctx context.Context) (solver.CachedResult, error) { + res, err := b.loadResult(ctx, req.Definition, req.CacheImports) + var ee *llberrdefs.ExecError + if errors.As(err, &ee) { + ee.EachRef(func(res solver.Result) error { + rp.errResults = append(rp.errResults, res) + return nil + }) + // acquire ownership so ExecError finalizer doesn't attempt to release as well + ee.OwnerBorrowed = true + } + return res, err + } + return rp } func (rp *resultProxy) Definition() *pb.Definition { return rp.def } -func (rp *resultProxy) Release(ctx context.Context) error { +func (rp *resultProxy) Release(ctx context.Context) (err error) { rp.mu.Lock() defer rp.mu.Unlock() + for _, res := range rp.errResults { + rerr := res.Release(ctx) + if rerr != nil { + err = rerr + } + } if rp.v != nil { if rp.released { logrus.Warnf("release of already released result") } - if err := rp.v.Release(ctx); err != nil { - return err + rerr := rp.v.Release(ctx) + if err != nil { + return rerr } } rp.released = true - return nil + return } func (rp *resultProxy) wrapError(err error) error { diff --git a/vendor/github.com/moby/buildkit/solver/llbsolver/errdefs/exec.go b/vendor/github.com/moby/buildkit/solver/llbsolver/errdefs/exec.go index 0a41792db2..f60757104f 100644 --- a/vendor/github.com/moby/buildkit/solver/llbsolver/errdefs/exec.go +++ b/vendor/github.com/moby/buildkit/solver/llbsolver/errdefs/exec.go @@ -1,14 +1,19 @@ package errdefs import ( + "context" + "runtime" + "github.com/moby/buildkit/solver" + "github.com/sirupsen/logrus" ) // ExecError will be returned when an error is encountered when evaluating an op. type ExecError struct { error - Inputs []solver.Result - Mounts []solver.Result + Inputs []solver.Result + Mounts []solver.Result + OwnerBorrowed bool } func (e *ExecError) Unwrap() error { @@ -35,13 +40,35 @@ func (e *ExecError) EachRef(fn func(solver.Result) error) (err error) { return err } +func (e *ExecError) Release() error { + if e.OwnerBorrowed { + return nil + } + err := e.EachRef(func(r solver.Result) error { + r.Release(context.TODO()) + return nil + }) + e.OwnerBorrowed = true + return err +} + func WithExecError(err error, inputs, mounts []solver.Result) error { if err == nil { return nil } - return &ExecError{ + ee := &ExecError{ error: err, Inputs: inputs, Mounts: mounts, } + runtime.SetFinalizer(ee, func(e *ExecError) { + if !e.OwnerBorrowed { + e.EachRef(func(r solver.Result) error { + logrus.Warn("leaked execError detected and released") + r.Release(context.TODO()) + return nil + }) + } + }) + return ee } diff --git a/vendor/github.com/moby/buildkit/solver/llbsolver/mounts/mount.go b/vendor/github.com/moby/buildkit/solver/llbsolver/mounts/mount.go index f9e3db8995..05703acab5 100644 --- a/vendor/github.com/moby/buildkit/solver/llbsolver/mounts/mount.go +++ b/vendor/github.com/moby/buildkit/solver/llbsolver/mounts/mount.go @@ -178,6 +178,9 @@ func (mm *MountManager) getSSHMountable(ctx context.Context, m *pb.Mount, g sess if err != nil { return nil, err } + if caller == nil { + return nil, nil + } // because ssh socket remains active, to actually handle session disconnecting ssh error // should restart the whole exec with new session return &sshMount{mount: m, caller: caller, idmap: mm.cm.IdentityMapping()}, nil diff --git a/vendor/github.com/moby/buildkit/solver/llbsolver/solver.go b/vendor/github.com/moby/buildkit/solver/llbsolver/solver.go index 1413bc8c06..d3ef585284 100644 --- a/vendor/github.com/moby/buildkit/solver/llbsolver/solver.go +++ b/vendor/github.com/moby/buildkit/solver/llbsolver/solver.go @@ -16,7 +16,6 @@ import ( "github.com/moby/buildkit/frontend/gateway" "github.com/moby/buildkit/session" "github.com/moby/buildkit/solver" - "github.com/moby/buildkit/solver/llbsolver/errdefs" "github.com/moby/buildkit/util/compression" "github.com/moby/buildkit/util/entitlements" "github.com/moby/buildkit/util/progress" @@ -143,20 +142,6 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro res.EachRef(func(ref solver.ResultProxy) error { eg.Go(func() error { _, err := ref.Result(ctx2) - if err != nil { - // Also release any results referenced by exec errors. - var ee *errdefs.ExecError - if errors.As(err, &ee) { - ee.EachRef(func(res solver.Result) error { - - workerRef, ok := res.Sys().(*worker.WorkerRef) - if !ok { - return nil - } - return workerRef.ImmutableRef.Release(ctx) - }) - } - } return err }) return nil diff --git a/vendor/github.com/moby/buildkit/solver/scheduler.go b/vendor/github.com/moby/buildkit/solver/scheduler.go index 7fe981eff4..5b598bd6b9 100644 --- a/vendor/github.com/moby/buildkit/solver/scheduler.go +++ b/vendor/github.com/moby/buildkit/solver/scheduler.go @@ -130,9 +130,13 @@ func (s *scheduler) dispatch(e *edge) { pf := &pipeFactory{s: s, e: e} // unpark the edge - debugSchedulerPreUnpark(e, inc, updates, out) + if debugScheduler { + debugSchedulerPreUnpark(e, inc, updates, out) + } e.unpark(inc, updates, out, pf) - debugSchedulerPostUnpark(e, inc) + if debugScheduler { + debugSchedulerPostUnpark(e, inc) + } postUnpark: // set up new requests that didn't complete/were added by this run @@ -361,9 +365,6 @@ func (pf *pipeFactory) NewFuncRequest(f func(context.Context) (interface{}, erro } func debugSchedulerPreUnpark(e *edge, inc []pipe.Sender, updates, allPipes []pipe.Receiver) { - if !debugScheduler { - return - } logrus.Debugf(">> unpark %s req=%d upt=%d out=%d state=%s %s", e.edge.Vertex.Name(), len(inc), len(updates), len(allPipes), e.state, e.edge.Vertex.Digest()) for i, dep := range e.deps { @@ -371,7 +372,7 @@ func debugSchedulerPreUnpark(e *edge, inc []pipe.Sender, updates, allPipes []pip if dep.req != nil { des = dep.req.Request().(*edgeRequest).desiredState } - logrus.Debugf(":: dep%d %s state=%s des=%s keys=%d hasslowcache=%v", i, e.edge.Vertex.Inputs()[i].Vertex.Name(), dep.state, des, len(dep.keys), e.slowCacheFunc(dep) != nil) + logrus.Debugf(":: dep%d %s state=%s des=%s keys=%d hasslowcache=%v preprocessfunc=%v", i, e.edge.Vertex.Inputs()[i].Vertex.Name(), dep.state, des, len(dep.keys), e.slowCacheFunc(dep) != nil, e.preprocessFunc(dep) != nil) } for i, in := range inc { @@ -400,9 +401,6 @@ func debugSchedulerPreUnpark(e *edge, inc []pipe.Sender, updates, allPipes []pip } func debugSchedulerPostUnpark(e *edge, inc []pipe.Sender) { - if !debugScheduler { - return - } for i, in := range inc { logrus.Debugf("< incoming-%d: %p completed=%v", i, in, in.Status().Completed) }