From 08e1c2990cf86da0ffc2a6c45e38d8992634f41a Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Wed, 13 Dec 2017 18:49:14 -0800 Subject: [PATCH] dockerfile: add dockerignore support Signed-off-by: Tonis Tiigi --- client/llb/source.go | 19 +++++ frontend/dockerfile/builder/build.go | 50 +++++++++++-- frontend/dockerfile/dockerfile2llb/convert.go | 4 +- frontend/dockerfile/dockerfile_test.go | 73 +++++++++++++++++++ session/filesync/filesync.go | 14 +++- solver/pb/attr.go | 1 + source/identifier.go | 7 ++ source/local/local.go | 1 + .../builder/dockerignore/dockerignore.go | 64 ++++++++++++++++ 9 files changed, 221 insertions(+), 12 deletions(-) create mode 100644 vendor/github.com/docker/docker/builder/dockerignore/dockerignore.go diff --git a/client/llb/source.go b/client/llb/source.go index 519e3d9ae..0767dce42 100644 --- a/client/llb/source.go +++ b/client/llb/source.go @@ -193,6 +193,9 @@ func Local(name string, opts ...LocalOption) State { if gi.IncludePatterns != "" { attrs[pb.AttrIncludePatterns] = gi.IncludePatterns } + if gi.ExcludePatterns != "" { + attrs[pb.AttrExcludePatterns] = gi.ExcludePatterns + } source := NewSource("local://"+name, attrs, gi.Metadata()) return NewState(source.Output()) @@ -216,15 +219,31 @@ func SessionID(id string) LocalOption { func IncludePatterns(p []string) LocalOption { return localOptionFunc(func(li *LocalInfo) { + if len(p) == 0 { + li.IncludePatterns = "" + return + } dt, _ := json.Marshal(p) // empty on error li.IncludePatterns = string(dt) }) } +func ExcludePatterns(p []string) LocalOption { + return localOptionFunc(func(li *LocalInfo) { + if len(p) == 0 { + li.ExcludePatterns = "" + return + } + dt, _ := json.Marshal(p) // empty on error + li.ExcludePatterns = string(dt) + }) +} + type LocalInfo struct { opMetaWrapper SessionID string IncludePatterns string + ExcludePatterns string } func HTTP(url string, opts ...HTTPOption) State { diff --git a/frontend/dockerfile/builder/build.go b/frontend/dockerfile/builder/build.go index ebb868403..cc3031805 100644 --- a/frontend/dockerfile/builder/build.go +++ b/frontend/dockerfile/builder/build.go @@ -1,15 +1,18 @@ package builder import ( + "bytes" "context" "encoding/json" "path" "strings" + "github.com/docker/docker/builder/dockerignore" "github.com/moby/buildkit/client/llb" "github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb" "github.com/moby/buildkit/frontend/gateway/client" "github.com/pkg/errors" + "golang.org/x/sync/errgroup" ) const ( @@ -19,6 +22,7 @@ const ( keyFilename = "filename" exporterImageConfig = "containerimage.config" defaultDockerfileName = "Dockerfile" + dockerignoreFilename = ".dockerignore" buildArgPrefix = "build-arg:" gitPrefix = "git://" ) @@ -48,13 +52,46 @@ func Build(ctx context.Context, c client.Client) error { return err } - ref, err := c.Solve(ctx, def.ToPB(), "", nil, false) - if err != nil { - return err - } + eg, ctx2 := errgroup.WithContext(ctx) + var dtDockerfile []byte + eg.Go(func() error { + ref, err := c.Solve(ctx2, def.ToPB(), "", nil, false) + if err != nil { + return err + } - dtDockerfile, err := ref.ReadFile(ctx, filename) - if err != nil { + dtDockerfile, err = ref.ReadFile(ctx2, filename) + if err != nil { + return err + } + return nil + }) + var excludes []string + eg.Go(func() error { + dockerignoreState := buildContext + if dockerignoreState == nil { + st := llb.Local(LocalNameContext, llb.SessionID(c.SessionID()), llb.IncludePatterns([]string{dockerignoreFilename})) + dockerignoreState = &st + } + def, err := dockerignoreState.Marshal() + if err != nil { + return err + } + ref, err := c.Solve(ctx2, def.ToPB(), "", nil, false) + if err != nil { + return err + } + dtDockerignore, err := ref.ReadFile(ctx2, dockerignoreFilename) + if err == nil { + excludes, err = dockerignore.ReadAll(bytes.NewBuffer(dtDockerignore)) + if err != nil { + return errors.Wrap(err, "failed to parse dockerignore") + } + } + return nil + }) + + if err := eg.Wait(); err != nil { return err } @@ -64,6 +101,7 @@ func Build(ctx context.Context, c client.Client) error { BuildArgs: filterBuildArgs(opts), SessionID: c.SessionID(), BuildContext: buildContext, + Excludes: excludes, }) if err != nil { diff --git a/frontend/dockerfile/dockerfile2llb/convert.go b/frontend/dockerfile/dockerfile2llb/convert.go index e524a3f60..878cf8e81 100644 --- a/frontend/dockerfile/dockerfile2llb/convert.go +++ b/frontend/dockerfile/dockerfile2llb/convert.go @@ -38,6 +38,7 @@ type ConvertOpt struct { BuildArgs map[string]string SessionID string BuildContext *llb.State + Excludes []string } func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State, *Image, error) { @@ -166,8 +167,7 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State, if err := eg.Wait(); err != nil { return nil, nil, err } - - buildContext := llb.Local(localNameContext, llb.SessionID(opt.SessionID)) + buildContext := llb.Local(localNameContext, llb.SessionID(opt.SessionID), llb.ExcludePatterns(opt.Excludes)) if opt.BuildContext != nil { buildContext = *opt.BuildContext } diff --git a/frontend/dockerfile/dockerfile_test.go b/frontend/dockerfile/dockerfile_test.go index 3407420f5..823e41012 100644 --- a/frontend/dockerfile/dockerfile_test.go +++ b/frontend/dockerfile/dockerfile_test.go @@ -40,6 +40,7 @@ func TestIntegration(t *testing.T) { testExportedHistory, testExposeExpansion, testUser, + testDockerignore, }) } @@ -503,6 +504,78 @@ EXPOSE 5000 require.Equal(t, "5000/tcp", ports[2]) } +func testDockerignore(t *testing.T, sb integration.Sandbox) { + t.Parallel() + + dockerfile := []byte(` +FROM scratch +COPY . . +`) + + dockerignore := []byte(` +ba* +Dockerfile +!bay +.dockerignore +`) + + dir, err := tmpdir( + fstest.CreateFile("Dockerfile", dockerfile, 0600), + fstest.CreateFile("foo", []byte(`foo-contents`), 0600), + fstest.CreateFile("bar", []byte(`bar-contents`), 0600), + fstest.CreateFile("baz", []byte(`baz-contents`), 0600), + fstest.CreateFile("bay", []byte(`bay-contents`), 0600), + fstest.CreateFile(".dockerignore", dockerignore, 0600), + ) + require.NoError(t, err) + defer os.RemoveAll(dir) + + c, err := client.New(sb.Address()) + require.NoError(t, err) + defer c.Close() + + destDir, err := ioutil.TempDir("", "buildkit") + require.NoError(t, err) + defer os.RemoveAll(destDir) + + err = c.Solve(context.TODO(), nil, client.SolveOpt{ + Frontend: "dockerfile.v0", + Exporter: client.ExporterLocal, + ExporterAttrs: map[string]string{ + "output": destDir, + }, + LocalDirs: map[string]string{ + builder.LocalNameDockerfile: dir, + builder.LocalNameContext: dir, + }, + }, nil) + require.NoError(t, err) + + dt, err := ioutil.ReadFile(filepath.Join(destDir, "foo")) + require.NoError(t, err) + require.Equal(t, "foo-contents", string(dt)) + + _, err = os.Stat(filepath.Join(destDir, ".dockerignore")) + require.Error(t, err) + require.True(t, os.IsNotExist(err)) + + _, err = os.Stat(filepath.Join(destDir, "Dockerfile")) + require.Error(t, err) + require.True(t, os.IsNotExist(err)) + + _, err = os.Stat(filepath.Join(destDir, "bar")) + require.Error(t, err) + require.True(t, os.IsNotExist(err)) + + _, err = os.Stat(filepath.Join(destDir, "baz")) + require.Error(t, err) + require.True(t, os.IsNotExist(err)) + + dt, err = ioutil.ReadFile(filepath.Join(destDir, "bay")) + require.NoError(t, err) + require.Equal(t, "bay-contents", string(dt)) +} + func testExportedHistory(t *testing.T, sb integration.Sandbox) { t.Parallel() diff --git a/session/filesync/filesync.go b/session/filesync/filesync.go index 0b82a03f2..2f8223645 100644 --- a/session/filesync/filesync.go +++ b/session/filesync/filesync.go @@ -17,6 +17,7 @@ import ( const ( keyOverrideExcludes = "override-excludes" keyIncludePatterns = "include-patterns" + keyExcludePatterns = "exclude-patterns" keyDirName = "dir-name" ) @@ -55,7 +56,7 @@ func (sp *fsSyncProvider) TarStream(stream FileSync_TarStreamServer) error { return sp.handle("tarstream", stream) } -func (sp *fsSyncProvider) handle(method string, stream grpc.ServerStream) error { +func (sp *fsSyncProvider) handle(method string, stream grpc.ServerStream) (retErr error) { var pr *protocol for _, p := range supportedProtocols { if method == p.name && isProtoSupported(p.name) { @@ -80,8 +81,8 @@ func (sp *fsSyncProvider) handle(method string, stream grpc.ServerStream) error return errors.Errorf("no access allowed to dir %q", dirName) } - var excludes []string - if len(opts[keyOverrideExcludes]) == 0 || opts[keyOverrideExcludes][0] != "true" { + excludes := opts[keyExcludePatterns] + if len(dir.Excludes) != 0 && (len(opts[keyOverrideExcludes]) == 0 || opts[keyOverrideExcludes][0] != "true") { excludes = dir.Excludes } includes := opts[keyIncludePatterns] @@ -140,7 +141,8 @@ var supportedProtocols = []protocol{ type FSSendRequestOpt struct { Name string IncludePatterns []string - OverrideExcludes bool + ExcludePatterns []string + OverrideExcludes bool // deprecated: this is used by docker/cli for automatically loading .dockerignore from the directory DestDir string CacheUpdater CacheUpdater ProgressCb func(int, bool) @@ -175,6 +177,10 @@ func FSSync(ctx context.Context, c session.Caller, opt FSSendRequestOpt) error { opts[keyIncludePatterns] = opt.IncludePatterns } + if opt.ExcludePatterns != nil { + opts[keyExcludePatterns] = opt.ExcludePatterns + } + opts[keyDirName] = []string{opt.Name} ctx, cancel := context.WithCancel(ctx) diff --git a/solver/pb/attr.go b/solver/pb/attr.go index f71b85f51..f507a2529 100644 --- a/solver/pb/attr.go +++ b/solver/pb/attr.go @@ -3,6 +3,7 @@ package pb const AttrKeepGitDir = "git.keepgitdir" const AttrLocalSessionID = "local.session" const AttrIncludePatterns = "local.includepattern" +const AttrExcludePatterns = "local.excludepatterns" const AttrLLBDefinitionFilename = "llbbuild.filename" const AttrHTTPChecksum = "http.checksum" diff --git a/source/identifier.go b/source/identifier.go index a93f4723a..59ff0cb02 100644 --- a/source/identifier.go +++ b/source/identifier.go @@ -80,6 +80,12 @@ func FromLLB(op *pb.Op_Source) (Identifier, error) { return nil, err } id.IncludePatterns = patterns + case pb.AttrExcludePatterns: + var patterns []string + if err := json.Unmarshal([]byte(v), &patterns); err != nil { + return nil, err + } + id.ExcludePatterns = patterns } } } @@ -142,6 +148,7 @@ type LocalIdentifier struct { Name string SessionID string IncludePatterns []string + ExcludePatterns []string } func NewLocalIdentifier(str string) (*LocalIdentifier, error) { diff --git a/source/local/local.go b/source/local/local.go index 1e374cd10..3565f8ecd 100644 --- a/source/local/local.go +++ b/source/local/local.go @@ -157,6 +157,7 @@ func (ls *localSourceHandler) Snapshot(ctx context.Context) (out cache.Immutable opt := filesync.FSSendRequestOpt{ Name: ls.src.Name, IncludePatterns: ls.src.IncludePatterns, + ExcludePatterns: ls.src.ExcludePatterns, OverrideExcludes: false, DestDir: dest, CacheUpdater: &cacheUpdater{cc}, diff --git a/vendor/github.com/docker/docker/builder/dockerignore/dockerignore.go b/vendor/github.com/docker/docker/builder/dockerignore/dockerignore.go new file mode 100644 index 000000000..cc2238133 --- /dev/null +++ b/vendor/github.com/docker/docker/builder/dockerignore/dockerignore.go @@ -0,0 +1,64 @@ +package dockerignore + +import ( + "bufio" + "bytes" + "fmt" + "io" + "path/filepath" + "strings" +) + +// ReadAll reads a .dockerignore file and returns the list of file patterns +// to ignore. Note this will trim whitespace from each line as well +// as use GO's "clean" func to get the shortest/cleanest path for each. +func ReadAll(reader io.Reader) ([]string, error) { + if reader == nil { + return nil, nil + } + + scanner := bufio.NewScanner(reader) + var excludes []string + currentLine := 0 + + utf8bom := []byte{0xEF, 0xBB, 0xBF} + for scanner.Scan() { + scannedBytes := scanner.Bytes() + // We trim UTF8 BOM + if currentLine == 0 { + scannedBytes = bytes.TrimPrefix(scannedBytes, utf8bom) + } + pattern := string(scannedBytes) + currentLine++ + // Lines starting with # (comments) are ignored before processing + if strings.HasPrefix(pattern, "#") { + continue + } + pattern = strings.TrimSpace(pattern) + if pattern == "" { + continue + } + // normalize absolute paths to paths relative to the context + // (taking care of '!' prefix) + invert := pattern[0] == '!' + if invert { + pattern = strings.TrimSpace(pattern[1:]) + } + if len(pattern) > 0 { + pattern = filepath.Clean(pattern) + pattern = filepath.ToSlash(pattern) + if len(pattern) > 1 && pattern[0] == '/' { + pattern = pattern[1:] + } + } + if invert { + pattern = "!" + pattern + } + + excludes = append(excludes, pattern) + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("Error reading .dockerignore: %v", err) + } + return excludes, nil +}