From 81a5fa5a2e7b8bfc100379eff48c93a49e4d5ff4 Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Wed, 27 Feb 2019 14:40:45 -0800 Subject: [PATCH] llbsolver: fileop implementation Signed-off-by: Tonis Tiigi --- cache/refs.go | 2 +- client/client_test.go | 99 +++++++++++++++ client/llb/fileop.go | 4 +- client/llb/fileop_test.go | 66 ++++++++-- solver/llbsolver/file/backend.go | 54 +++++--- solver/llbsolver/file/refmanager.go | 8 +- solver/llbsolver/ops/exec.go | 10 +- solver/llbsolver/ops/file.go | 185 +++++++++++++++++++++++++++- solver/llbsolver/result.go | 11 +- worker/base/worker.go | 2 + 10 files changed, 402 insertions(+), 39 deletions(-) diff --git a/cache/refs.go b/cache/refs.go index 7521e0aa9..4c8fa27d4 100644 --- a/cache/refs.go +++ b/cache/refs.go @@ -311,7 +311,7 @@ func (sr *mutableRef) updateLastUsed() bool { func (sr *mutableRef) commit(ctx context.Context) (ImmutableRef, error) { if !sr.mutable || len(sr.refs) == 0 { - return nil, errors.Wrapf(errInvalid, "invalid mutable ref") + return nil, errors.Wrapf(errInvalid, "invalid mutable ref %p", sr) } id := identity.NewID() diff --git a/client/client_test.go b/client/client_test.go index 568e1306e..14c03680b 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -51,6 +51,8 @@ func (nopWriteCloser) Close() error { return nil } func TestClientIntegration(t *testing.T) { integration.Run(t, []integration.Test{ testRelativeWorkDir, + testFileOpMkdirMkfile, + testFileOpCopyRm, testCallDiskUsage, testBuildMultiMount, testBuildHTTPSource, @@ -653,6 +655,103 @@ func testRelativeWorkDir(t *testing.T, sb integration.Sandbox) { require.Equal(t, []byte("/test1/test2\n"), dt) } +func testFileOpMkdirMkfile(t *testing.T, sb integration.Sandbox) { + requiresLinux(t) + c, err := New(context.TODO(), sb.Address()) + require.NoError(t, err) + defer c.Close() + + st := llb.Scratch(). + File(llb.Mkdir("/foo", 0700).Mkfile("bar", 0600, []byte("contents"))) + + def, err := st.Marshal() + require.NoError(t, err) + + destDir, err := ioutil.TempDir("", "buildkit") + require.NoError(t, err) + defer os.RemoveAll(destDir) + + _, err = c.Solve(context.TODO(), def, SolveOpt{ + Exporter: ExporterLocal, + ExporterOutputDir: destDir, + }, nil) + require.NoError(t, err) + + fi, err := os.Stat(filepath.Join(destDir, "foo")) + require.NoError(t, err) + require.Equal(t, true, fi.IsDir()) + + dt, err := ioutil.ReadFile(filepath.Join(destDir, "bar")) + require.NoError(t, err) + require.Equal(t, []byte("contents"), dt) +} + +func testFileOpCopyRm(t *testing.T, sb integration.Sandbox) { + requiresLinux(t) + c, err := New(context.TODO(), sb.Address()) + require.NoError(t, err) + defer c.Close() + + dir, err := tmpdir( + fstest.CreateFile("myfile", []byte("data0"), 0600), + fstest.CreateDir("sub", 0700), + fstest.CreateFile("sub/foo", []byte("foo0"), 0600), + fstest.CreateFile("sub/bar", []byte("bar0"), 0600), + ) + require.NoError(t, err) + defer os.RemoveAll(dir) + + dir2, err := tmpdir( + fstest.CreateFile("file2", []byte("file2"), 0600), + ) + require.NoError(t, err) + defer os.RemoveAll(dir) + + st := llb.Scratch(). + File( + llb.Copy(llb.Local("mylocal"), "myfile", "myfile2"). + Copy(llb.Local("mylocal"), "sub", "out"). + Rm("out/foo"). + Copy(llb.Local("mylocal2"), "file2", "/")) + + def, err := st.Marshal() + require.NoError(t, err) + + destDir, err := ioutil.TempDir("", "buildkit") + require.NoError(t, err) + defer os.RemoveAll(destDir) + + _, err = c.Solve(context.TODO(), def, SolveOpt{ + Exporter: ExporterLocal, + ExporterOutputDir: destDir, + LocalDirs: map[string]string{ + "mylocal": dir, + "mylocal2": dir2, + }, + }, nil) + require.NoError(t, err) + + dt, err := ioutil.ReadFile(filepath.Join(destDir, "myfile2")) + require.NoError(t, err) + require.Equal(t, []byte("data0"), dt) + + fi, err := os.Stat(filepath.Join(destDir, "out")) + require.NoError(t, err) + require.Equal(t, true, fi.IsDir()) + + dt, err = ioutil.ReadFile(filepath.Join(destDir, "out/bar")) + require.NoError(t, err) + require.Equal(t, []byte("bar0"), dt) + + _, err = os.Stat(filepath.Join(destDir, "out/foo")) + require.Equal(t, true, os.IsNotExist(err)) + + dt, err = ioutil.ReadFile(filepath.Join(destDir, "file2")) + require.NoError(t, err) + require.Equal(t, []byte("file2"), dt) + +} + func testCallDiskUsage(t *testing.T, sb integration.Sandbox) { c, err := New(context.TODO(), sb.Address()) require.NoError(t, err) diff --git a/client/llb/fileop.go b/client/llb/fileop.go index 074fb35ed..18d1d68e1 100644 --- a/client/llb/fileop.go +++ b/client/llb/fileop.go @@ -634,8 +634,8 @@ func (f *FileOp) Marshal(c *Constraints) (digest.Digest, []byte, *pb.OpMetadata, } pfo.Actions = append(pfo.Actions, &pb.FileAction{ - Input: getIndex(st.input, len(state.actions), st.inputRelative), - SecondaryInput: getIndex(st.input2, len(state.actions), st.input2Relative), + Input: getIndex(st.input, len(state.inputs), st.inputRelative), + SecondaryInput: getIndex(st.input2, len(state.inputs), st.input2Relative), Output: output, Action: st.action.toProtoAction(parent, st.base), }) diff --git a/client/llb/fileop_test.go b/client/llb/fileop_test.go index 657653043..b32928f35 100644 --- a/client/llb/fileop_test.go +++ b/client/llb/fileop_test.go @@ -76,7 +76,7 @@ func TestFileMkdirChain(t *testing.T) { require.Nil(t, mkdir.Owner) action = f.Actions[1] - require.Equal(t, 3, int(action.Input)) + require.Equal(t, 1, int(action.Input)) require.Equal(t, -1, int(action.SecondaryInput)) require.Equal(t, -1, int(action.Output)) mkdir = action.Action.(*pb.FileAction_Mkdir).Mkdir @@ -86,7 +86,7 @@ func TestFileMkdirChain(t *testing.T) { require.Nil(t, mkdir.Owner) action = f.Actions[2] - require.Equal(t, 4, int(action.Input)) + require.Equal(t, 2, int(action.Input)) require.Equal(t, -1, int(action.SecondaryInput)) require.Equal(t, 0, int(action.Output)) mkdir = action.Action.(*pb.FileAction_Mkdir).Mkdir @@ -96,6 +96,52 @@ func TestFileMkdirChain(t *testing.T) { require.Nil(t, mkdir.Owner) } +func TestFileMkdirMkfile(t *testing.T) { + t.Parallel() + + st := Scratch().File(Mkdir("/foo", 0700).Mkfile("bar", 0700, []byte("data"))) + def, err := st.Marshal() + + require.NoError(t, err) + + m, arr := parseDef(t, def.Def) + require.Equal(t, 2, len(arr)) + + dgst, idx := last(t, arr) + require.Equal(t, 0, idx) + require.Equal(t, m[dgst], arr[0]) + + f := arr[0].Op.(*pb.Op_File).File + require.Equal(t, len(arr[1].Inputs), 1) + require.Equal(t, m[arr[1].Inputs[0].Digest], arr[0]) + require.Equal(t, 0, int(arr[1].Inputs[0].Index)) + + require.Equal(t, 2, len(f.Actions)) + + action := f.Actions[0] + require.Equal(t, -1, int(action.Input)) + require.Equal(t, -1, int(action.SecondaryInput)) + require.Equal(t, -1, int(action.Output)) + + mkdir := action.Action.(*pb.FileAction_Mkdir).Mkdir + + require.Equal(t, "/foo", mkdir.Path) + require.Equal(t, 0700, int(mkdir.Mode)) + require.Equal(t, int64(-1), mkdir.Timestamp) + + action = f.Actions[1] + require.Equal(t, 0, int(action.Input)) + require.Equal(t, -1, int(action.SecondaryInput)) + require.Equal(t, 0, int(action.Output)) + + mkfile := action.Action.(*pb.FileAction_Mkfile).Mkfile + + require.Equal(t, "/bar", mkfile.Path) + require.Equal(t, 0700, int(mkfile.Mode)) + require.Equal(t, "data", string(mkfile.Data)) + require.Equal(t, int64(-1), mkfile.Timestamp) +} + func TestFileMkfile(t *testing.T) { t.Parallel() @@ -202,7 +248,7 @@ func TestFileSimpleChains(t *testing.T) { require.Equal(t, "/tmp/sub/foo", rm.Path) action = f.Actions[1] - require.Equal(t, 2, int(action.Input)) + require.Equal(t, 1, int(action.Input)) require.Equal(t, -1, int(action.SecondaryInput)) require.Equal(t, 0, int(action.Output)) @@ -224,7 +270,7 @@ func TestFileSimpleChains(t *testing.T) { require.Equal(t, "/tmp/foo/bar", mkdir.Path) action = f.Actions[1] - require.Equal(t, 3, int(action.Input)) + require.Equal(t, 1, int(action.Input)) require.Equal(t, -1, int(action.SecondaryInput)) require.Equal(t, -1, int(action.Output)) @@ -232,7 +278,7 @@ func TestFileSimpleChains(t *testing.T) { require.Equal(t, "/tmp/abc", rm.Path) action = f.Actions[2] - require.Equal(t, 4, int(action.Input)) + require.Equal(t, 2, int(action.Input)) require.Equal(t, -1, int(action.SecondaryInput)) require.Equal(t, 0, int(action.Output)) @@ -314,7 +360,7 @@ func TestFileCopyFromAction(t *testing.T) { require.Equal(t, 0700, int(mkdir.Mode)) action = f.Actions[1] - require.Equal(t, 3, int(action.Input)) + require.Equal(t, 1, int(action.Input)) require.Equal(t, -1, int(action.SecondaryInput)) require.Equal(t, -1, int(action.Output)) @@ -326,7 +372,7 @@ func TestFileCopyFromAction(t *testing.T) { action = f.Actions[2] require.Equal(t, 0, int(action.Input)) - require.Equal(t, 4, int(action.SecondaryInput)) + require.Equal(t, 2, int(action.SecondaryInput)) require.Equal(t, 0, int(action.Output)) copy := action.Action.(*pb.FileAction_Copy).Copy @@ -420,7 +466,7 @@ func TestFilePipeline(t *testing.T) { require.Equal(t, 0700, int(mkdir.Mode)) action = f.Actions[1] - require.Equal(t, 4, int(action.Input)) + require.Equal(t, 2, int(action.Input)) require.Equal(t, -1, int(action.SecondaryInput)) require.Equal(t, -1, int(action.Output)) @@ -432,7 +478,7 @@ func TestFilePipeline(t *testing.T) { action = f.Actions[2] require.Equal(t, 0, int(action.Input)) - require.Equal(t, 5, int(action.SecondaryInput)) + require.Equal(t, 3, int(action.SecondaryInput)) require.Equal(t, -1, int(action.Output)) require.Equal(t, arr[4].Inputs[1].Digest, op.Inputs[0].Digest) @@ -442,7 +488,7 @@ func TestFilePipeline(t *testing.T) { require.Equal(t, "/out/baz", copy.Dest) action = f.Actions[3] - require.Equal(t, 6, int(action.Input)) + require.Equal(t, 4, int(action.Input)) require.Equal(t, -1, int(action.SecondaryInput)) require.Equal(t, 0, int(action.Output)) diff --git a/solver/llbsolver/file/backend.go b/solver/llbsolver/file/backend.go index 02dec5f78..d212fa13d 100644 --- a/solver/llbsolver/file/backend.go +++ b/solver/llbsolver/file/backend.go @@ -118,10 +118,10 @@ func docopy(ctx context.Context, src, dest string, action pb.FileActionCopy) err return nil } -type FileBackend struct { +type Backend struct { } -func (fb *FileBackend) Mkdir(ctx context.Context, m fileoptypes.Mount, action pb.FileActionMkDir) error { +func (fb *Backend) Mkdir(ctx context.Context, m fileoptypes.Mount, action pb.FileActionMkDir) error { mnt, ok := m.(*Mount) if !ok { return errors.Errorf("invalid mount type %T", m) @@ -137,26 +137,37 @@ func (fb *FileBackend) Mkdir(ctx context.Context, m fileoptypes.Mount, action pb return mkdir(ctx, dir, action) } -func (fb *FileBackend) Mkfile(ctx context.Context, m fileoptypes.Mount, action pb.FileActionMkFile) error { +func (fb *Backend) Mkfile(ctx context.Context, m fileoptypes.Mount, action pb.FileActionMkFile) error { mnt, ok := m.(*Mount) if !ok { return errors.Errorf("invalid mount type %T", m) } - _ = mnt + lm := snapshot.LocalMounter(mnt.m) + dir, err := lm.Mount() + if err != nil { + return err + } + defer lm.Unmount() - return errors.Errorf("mkfile not implemented") + return mkfile(ctx, dir, action) } -func (fb *FileBackend) Rm(ctx context.Context, m fileoptypes.Mount, action pb.FileActionRm) error { +func (fb *Backend) Rm(ctx context.Context, m fileoptypes.Mount, action pb.FileActionRm) error { mnt, ok := m.(*Mount) if !ok { return errors.Errorf("invalid mount type %T", m) } - _ = mnt - return errors.Errorf("rm not implemented") + lm := snapshot.LocalMounter(mnt.m) + dir, err := lm.Mount() + if err != nil { + return err + } + defer lm.Unmount() + + return rm(ctx, dir, action) } -func (fb *FileBackend) Copy(ctx context.Context, m1 fileoptypes.Mount, m2 fileoptypes.Mount, action pb.FileActionCopy) error { +func (fb *Backend) Copy(ctx context.Context, m1 fileoptypes.Mount, m2 fileoptypes.Mount, action pb.FileActionCopy) error { mnt1, ok := m1.(*Mount) if !ok { return errors.Errorf("invalid mount type %T", m1) @@ -166,14 +177,19 @@ func (fb *FileBackend) Copy(ctx context.Context, m1 fileoptypes.Mount, m2 fileop return errors.Errorf("invalid mount type %T", m2) } - _ = mnt1 - _ = mnt2 - return errors.Errorf("copy not implemented") -} + lm := snapshot.LocalMounter(mnt1.m) + src, err := lm.Mount() + if err != nil { + return err + } + defer lm.Unmount() -// type Backend interface { -// Mkdir(context.Context, Mount, pb.FileActionMkDir) error -// Mkfile(context.Context, Mount, pb.FileActionMkFile) error -// Rm(context.Context, Mount, pb.FileActionRm) error -// Copy(context.Context, Mount, Mount, pb.FileActionCopy) error -// } + lm2 := snapshot.LocalMounter(mnt2.m) + dest, err := lm2.Mount() + if err != nil { + return err + } + defer lm2.Unmount() + + return docopy(ctx, src, dest, action) +} diff --git a/solver/llbsolver/file/refmanager.go b/solver/llbsolver/file/refmanager.go index aeb048756..bfee9f85b 100644 --- a/solver/llbsolver/file/refmanager.go +++ b/solver/llbsolver/file/refmanager.go @@ -9,13 +9,17 @@ import ( "github.com/pkg/errors" ) +func NewRefManager(cm cache.Manager) *RefManager { + return &RefManager{cm: cm} +} + type RefManager struct { cm cache.Manager } func (rm *RefManager) Prepare(ctx context.Context, ref fileoptypes.Ref, readonly bool) (fileoptypes.Mount, error) { ir, ok := ref.(cache.ImmutableRef) - if !ok { + if !ok && ref != nil { return nil, errors.Errorf("invalid ref type: %T", ref) } @@ -43,7 +47,7 @@ func (rm *RefManager) Commit(ctx context.Context, mount fileoptypes.Mount) (file if !ok { return nil, errors.Errorf("invalid mount type %T", mount) } - if err := m.Release(context.TODO()); err != nil { + if err := m.m.Release(); err != nil { return nil, err } if m.mr == nil { diff --git a/solver/llbsolver/ops/exec.go b/solver/llbsolver/ops/exec.go index bbc744703..ca9b0b9ed 100644 --- a/solver/llbsolver/ops/exec.go +++ b/solver/llbsolver/ops/exec.go @@ -149,7 +149,7 @@ func (e *execOp) CacheMap(ctx context.Context, index int) (*solver.CacheMap, boo cm.Deps[i].Selector = digest.FromBytes(bytes.Join(dgsts, []byte{0})) } if !dep.NoContentBasedHash { - cm.Deps[i].ComputeDigestFunc = llbsolver.NewContentHashFunc(dedupePaths(dep.Selectors)) + cm.Deps[i].ComputeDigestFunc = llbsolver.NewContentHashFunc(toSelectors(dedupePaths(dep.Selectors))) } } @@ -180,6 +180,14 @@ func dedupePaths(inp []string) []string { return paths } +func toSelectors(p []string) []llbsolver.Selector { + sel := make([]llbsolver.Selector, 0, len(p)) + for _, p := range p { + sel = append(sel, llbsolver.Selector{Path: p}) + } + return sel +} + type dep struct { Selectors []string NoContentBasedHash bool diff --git a/solver/llbsolver/ops/file.go b/solver/llbsolver/ops/file.go index 12513daab..ada18391c 100644 --- a/solver/llbsolver/ops/file.go +++ b/solver/llbsolver/ops/file.go @@ -1,17 +1,199 @@ package ops import ( + "bytes" "context" + "encoding/json" "fmt" + "path" + "runtime" + "sort" "sync" + "github.com/moby/buildkit/cache" + "github.com/moby/buildkit/cache/metadata" + "github.com/moby/buildkit/solver" + "github.com/moby/buildkit/solver/llbsolver" + "github.com/moby/buildkit/solver/llbsolver/file" "github.com/moby/buildkit/solver/llbsolver/ops/fileoptypes" "github.com/moby/buildkit/solver/pb" "github.com/moby/buildkit/util/flightcontrol" + "github.com/moby/buildkit/worker" + digest "github.com/opencontainers/go-digest" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "golang.org/x/sync/errgroup" ) +const fileCacheType = "buildkit.exec.v0" + +type fileOp struct { + op *pb.FileOp + md *metadata.Store + w worker.Worker + solver *FileOpSolver + numInputs int +} + +func NewFileOp(v solver.Vertex, op *pb.Op_File, cm cache.Manager, md *metadata.Store, w worker.Worker) (solver.Op, error) { + return &fileOp{ + op: op.File, + md: md, + numInputs: len(v.Inputs()), + w: w, + solver: NewFileOpSolver(&file.Backend{}, file.NewRefManager(cm)), + }, nil +} + +func (f *fileOp) CacheMap(ctx context.Context, index int) (*solver.CacheMap, bool, error) { + selectors := map[int]map[llbsolver.Selector]struct{}{} + + digester := digest.Canonical.Digester() + + for _, action := range f.op.Actions { + var dt []byte + var err error + switch a := action.Action.(type) { + case *pb.FileAction_Mkdir: + p := *a.Mkdir + p.Owner = nil + dt, err = json.Marshal(p) + if err != nil { + return nil, false, err + } + case *pb.FileAction_Mkfile: + p := *a.Mkfile + p.Owner = nil + dt, err = json.Marshal(p) + if err != nil { + return nil, false, err + } + case *pb.FileAction_Rm: + p := *a.Rm + dt, err = json.Marshal(p) + if err != nil { + return nil, false, err + } + case *pb.FileAction_Copy: + p := *a.Copy + p.Owner = nil + if action.SecondaryInput != -1 && int(action.SecondaryInput) < f.numInputs { + p.Src = path.Base(p.Src) + addSelector(selectors, int(action.SecondaryInput), p.Src, p.AllowWildcard) + } + dt, err = json.Marshal(p) + if err != nil { + return nil, false, err + } + } + + if _, err = digester.Hash().Write(dt); err != nil { + return nil, false, err + } + } + + cm := &solver.CacheMap{ + Digest: digester.Digest(), + Deps: make([]struct { + Selector digest.Digest + ComputeDigestFunc solver.ResultBasedCacheFunc + }, f.numInputs), + } + + for idx, m := range selectors { + dgsts := make([][]byte, 0, len(m)) + for k := range m { + dgsts = append(dgsts, []byte(k.Path)) + } + sort.Slice(dgsts, func(i, j int) bool { + return bytes.Compare(dgsts[i], dgsts[j]) > 0 + }) + cm.Deps[idx].Selector = digest.FromBytes(bytes.Join(dgsts, []byte{0})) + + cm.Deps[idx].ComputeDigestFunc = llbsolver.NewContentHashFunc(dedupeSelectors(m)) + } + + return cm, true, nil +} + +func (f *fileOp) Exec(ctx context.Context, inputs []solver.Result) ([]solver.Result, error) { + + inpRefs := make([]fileoptypes.Ref, 0, len(inputs)) + for i, inp := range inputs { + workerRef, ok := inp.Sys().(*worker.WorkerRef) + if !ok { + return nil, errors.Errorf("invalid reference for exec %T", inp.Sys()) + } + inpRefs = append(inpRefs, workerRef.ImmutableRef) + logrus.Debugf("inp %d : %+v", i, workerRef.ImmutableRef) + } + + outs, err := f.solver.Solve(ctx, inpRefs, f.op.Actions) + if err != nil { + return nil, err + } + + outResults := make([]solver.Result, 0, len(outs)) + for _, out := range outs { + outResults = append(outResults, worker.NewWorkerRefResult(out.(cache.ImmutableRef), f.w)) + } + + return outResults, nil +} + +func addSelector(m map[int]map[llbsolver.Selector]struct{}, idx int, sel string, wildcard bool) { + mm, ok := m[idx] + if !ok { + mm = map[llbsolver.Selector]struct{}{} + m[idx] = mm + } + if wildcard && containsWildcards(sel) { + mm[llbsolver.Selector{Path: sel, Wildcard: wildcard}] = struct{}{} + } else { + mm[llbsolver.Selector{Path: sel}] = struct{}{} + } +} + +func containsWildcards(name string) bool { + isWindows := runtime.GOOS == "windows" + for i := 0; i < len(name); i++ { + ch := name[i] + if ch == '\\' && !isWindows { + i++ + } else if ch == '*' || ch == '?' || ch == '[' { + return true + } + } + return false +} + +func dedupeSelectors(m map[llbsolver.Selector]struct{}) []llbsolver.Selector { + paths := make([]string, 0, len(m)) + for sel := range m { + if !sel.Wildcard { + paths = append(paths, sel.Path) + } + } + paths = dedupePaths(paths) + selectors := make([]llbsolver.Selector, 0, len(m)) + + for _, p := range paths { + selectors = append(selectors, llbsolver.Selector{Path: p}) + } + + for sel := range m { + if sel.Wildcard { + selectors = append(selectors, sel) + } + } + + sort.Slice(selectors, func(i, j int) bool { + return selectors[i].Path < selectors[j].Path + }) + + return selectors +} + func NewFileOpSolver(b fileoptypes.Backend, r fileoptypes.RefManager) *FileOpSolver { return &FileOpSolver{ b: b, @@ -39,8 +221,9 @@ type input struct { func (s *FileOpSolver) Solve(ctx context.Context, inputs []fileoptypes.Ref, actions []*pb.FileAction) ([]fileoptypes.Ref, error) { for i, a := range actions { + logrus.Debugf("action: %+v", a) if int(a.Input) < -1 || int(a.Input) >= len(inputs)+len(actions) { - return nil, errors.Errorf("invalid input index %d, %d provided", a.Input, len(inputs)) + return nil, errors.Errorf("invalid input index %d, %d provided", a.Input, len(inputs)+len(actions)) } if int(a.SecondaryInput) < -1 || int(a.SecondaryInput) >= len(inputs)+len(actions) { return nil, errors.Errorf("invalid secondary input index %d, %d provided", a.Input, len(inputs)) diff --git a/solver/llbsolver/result.go b/solver/llbsolver/result.go index 19431fe0a..54fa6fc92 100644 --- a/solver/llbsolver/result.go +++ b/solver/llbsolver/result.go @@ -13,7 +13,12 @@ import ( "golang.org/x/sync/errgroup" ) -func NewContentHashFunc(selectors []string) solver.ResultBasedCacheFunc { +type Selector struct { + Path string + Wildcard bool +} + +func NewContentHashFunc(selectors []Selector) solver.ResultBasedCacheFunc { return func(ctx context.Context, res solver.Result) (digest.Digest, error) { ref, ok := res.Sys().(*worker.WorkerRef) if !ok { @@ -21,7 +26,7 @@ func NewContentHashFunc(selectors []string) solver.ResultBasedCacheFunc { } if len(selectors) == 0 { - selectors = []string{""} + selectors = []Selector{Selector{}} } dgsts := make([][]byte, len(selectors)) @@ -32,7 +37,7 @@ func NewContentHashFunc(selectors []string) solver.ResultBasedCacheFunc { // FIXME(tonistiigi): enabling this parallelization seems to create wrong results for some big inputs(like gobuild) // func(i int) { // eg.Go(func() error { - dgst, err := contenthash.Checksum(ctx, ref.ImmutableRef, path.Join("/", sel), true) + dgst, err := contenthash.Checksum(ctx, ref.ImmutableRef, path.Join("/", sel.Path), true) if err != nil { return "", err } diff --git a/worker/base/worker.go b/worker/base/worker.go index ee93d6e34..b051390d3 100644 --- a/worker/base/worker.go +++ b/worker/base/worker.go @@ -198,6 +198,8 @@ func (w *Worker) ResolveOp(v solver.Vertex, s frontend.FrontendLLBBridge, sm *se return ops.NewSourceOp(v, op, baseOp.Platform, w.SourceManager, sm, w) case *pb.Op_Exec: return ops.NewExecOp(v, op, baseOp.Platform, w.CacheManager, sm, w.MetadataStore, w.Executor, w) + case *pb.Op_File: + return ops.NewFileOp(v, op, w.CacheManager, w.MetadataStore, w) case *pb.Op_Build: return ops.NewBuildOp(v, op, s, w) default: