Merge pull request #3001 from DYefimov/feature/copy_parents

docker-bake.hcl

@@ -123,7 +123,7 @@ target "lint" {
   matrix = {
     buildtags = [
       { name = "default", tags = "", target = "golangci-lint" },
-      { name = "labs", tags = "dfrunsecurity", target = "golangci-lint" },
+      { name = "labs", tags = "dfrunsecurity dfparents", target = "golangci-lint" },
       { name = "nydus", tags = "nydus", target = "golangci-lint" },
       { name = "yaml", tags = "", target = "yamllint" },
       { name = "proto", tags = "", target = "protolint" },

frontend/dockerfile/dockerfile2llb/convert.go

@@ -761,6 +761,7 @@ func dispatch(d *dispatchState, cmd command, opt dispatchOpt) error {
 			chown:        c.Chown,
 			chmod:        c.Chmod,
 			link:         c.Link,
+			parents:      c.Parents,
 			location:     c.Location(),
 			opt:          opt,
 		})
@@ -1167,6 +1168,14 @@ func dispatchCopy(d *dispatchState, cfg copyConfig) error {
 				AllowEmptyWildcard:  true,
 			}}, copyOpt...)
 
+			if cfg.parents {
+				path := strings.TrimPrefix(src, "/")
+				opts = append(opts, &llb.CopyInfo{
+					IncludePatterns: []string{path},
+				})
+				src = "/"
+			}
+
 			if a == nil {
 				a = llb.Copy(cfg.source, src, dest, opts...)
 			} else {
@@ -1257,6 +1266,7 @@ type copyConfig struct {
 	link         bool
 	keepGitDir   bool
 	checksum     digest.Digest
+	parents      bool
 	location     []parser.Range
 	opt          dispatchOpt
 }

frontend/dockerfile/dockerfile_parents_test.go

@@ -0,0 +1,76 @@
+//go:build dfparents
+// +build dfparents
+
+package dockerfile
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/containerd/continuity/fs/fstest"
+	"github.com/moby/buildkit/client"
+	"github.com/moby/buildkit/frontend/dockerui"
+	"github.com/moby/buildkit/util/testutil/integration"
+	"github.com/stretchr/testify/require"
+)
+
+var parentsTests = integration.TestFuncs(
+	testCopyParents,
+)
+
+func init() {
+	allTests = append(allTests, parentsTests...)
+}
+
+func testCopyParents(t *testing.T, sb integration.Sandbox) {
+	f := getFrontend(t, sb)
+
+	dockerfile := []byte(`
+FROM scratch
+COPY --parents foo1/foo2/bar /
+
+WORKDIR /test
+COPY --parents foo1/foo2/ba* .
+`)
+
+	dir := integration.Tmpdir(
+		t,
+		fstest.CreateFile("Dockerfile", dockerfile, 0600),
+		fstest.CreateDir("foo1", 0700),
+		fstest.CreateDir("foo1/foo2", 0700),
+		fstest.CreateFile("foo1/foo2/bar", []byte(`testing`), 0600),
+		fstest.CreateFile("foo1/foo2/baz", []byte(`testing2`), 0600),
+	)
+
+	c, err := client.New(sb.Context(), sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	destDir := t.TempDir()
+
+	_, err = f.Solve(sb.Context(), c, client.SolveOpt{
+		Exports: []client.ExportEntry{
+			{
+				Type:      client.ExporterLocal,
+				OutputDir: destDir,
+			},
+		},
+		LocalDirs: map[string]string{
+			dockerui.DefaultLocalNameDockerfile: dir,
+			dockerui.DefaultLocalNameContext:    dir,
+		},
+	}, nil)
+	require.NoError(t, err)
+
+	dt, err := os.ReadFile(filepath.Join(destDir, "foo1/foo2/bar"))
+	require.NoError(t, err)
+	require.Equal(t, "testing", string(dt))
+
+	dt, err = os.ReadFile(filepath.Join(destDir, "test/foo1/foo2/bar"))
+	require.NoError(t, err)
+	require.Equal(t, "testing", string(dt))
+	dt, err = os.ReadFile(filepath.Join(destDir, "test/foo1/foo2/baz"))
+	require.NoError(t, err)
+	require.Equal(t, "testing2", string(dt))
+}

frontend/dockerfile/docs/reference.md

@@ -1457,6 +1457,46 @@ path, using `--link` is always recommended. The performance of `--link` is
 equivalent or better than the default behavior and, it creates much better
 conditions for cache reuse.
 
+
+## COPY --parents
+
+> **Note**
+>
+> Available in [`docker/dockerfile-upstream:master-labs`](#syntax).
+> Will be included in `docker/dockerfile:1.6-labs`.
+
+```dockerfile
+COPY [--parents[=<boolean>]] <src>... <dest>
+```
+
+The `--parents` flag preserves parent directories for `src` entries. This flag defaults to `false`.
+
+```dockerfile
+# syntax=docker/dockerfile-upstream:master-labs
+FROM scratch
+
+COPY ./x/a.txt ./y/a.txt /no_parents/
+COPY --parents ./x/a.txt ./y/a.txt /parents/
+
+# /no_parents/a.txt
+# /parents/x/a.txt
+# /parents/y/a.txt
+```
+
+This behavior is analogous to the [Linux `cp` utility's](https://www.man7.org/linux/man-pages/man1/cp.1.html)
+`--parents` flag.
+
+Note that, without the `--parents` flag specified, any filename collision will
+fail the Linux `cp` operation with an explicit error message
+(`cp: will not overwrite just-created './x/a.txt' with './y/a.txt'`), where the
+Buildkit will silently overwrite the target file at the destination.
+
+While it is possible to preserve the directory structure for `COPY`
+instructions consisting of only one `src` entry, usually it is more beneficial
+to keep the layer count in the resulting image as low as possible. Therefore,
+with the `--parents` flag, the Buildkit is capable of packing multiple
+`COPY` instructions together, keeping the directory structure intact.
+
 ## ENTRYPOINT
 
 ENTRYPOINT has two forms:

frontend/dockerfile/instructions/commands.go

@@ -270,10 +270,11 @@ func (c *AddCommand) Expand(expander SingleWordExpander) error {
 type CopyCommand struct {
 	withNameAndCode
 	SourcesAndDest
-	From  string
-	Chown string
-	Chmod string
-	Link  bool
+	From    string
+	Chown   string
+	Chmod   string
+	Link    bool
+	Parents bool // parents preserves directory structure
 }
 
 func (c *CopyCommand) Expand(expander SingleWordExpander) error {

frontend/dockerfile/instructions/parse.go

@@ -34,6 +34,8 @@ type parseRequest struct {
 var parseRunPreHooks []func(*RunCommand, parseRequest) error
 var parseRunPostHooks []func(*RunCommand, parseRequest) error
 
+var parentsEnabled = false
+
 func nodeArgs(node *parser.Node) []string {
 	result := []string{}
 	for ; node.Next != nil; node = node.Next {
@@ -315,6 +317,7 @@ func parseCopy(req parseRequest) (*CopyCommand, error) {
 	flFrom := req.flags.AddString("from", "")
 	flChmod := req.flags.AddString("chmod", "")
 	flLink := req.flags.AddBool("link", false)
+	flParents := req.flags.AddBool("parents", false)
 	if err := req.flags.Parse(); err != nil {
 		return nil, err
 	}
@@ -331,6 +334,7 @@ func parseCopy(req parseRequest) (*CopyCommand, error) {
 		Chown:           flChown.Value,
 		Chmod:           flChmod.Value,
 		Link:            flLink.Value == "true",
+		Parents:         (flParents.Value == "true") && parentsEnabled, // silently ignore if not -labs
 	}, nil
 }
 

frontend/dockerfile/instructions/parse_parents.go

@@ -0,0 +1,8 @@
+//go:build dfparents
+// +build dfparents
+
+package instructions
+
+func init() {
+	parentsEnabled = true
+}

frontend/dockerfile/release/labs/tags

@@ -1 +1 @@
-dfrunsecurity
+dfrunsecurity dfparents