Merge pull request #6229 from jsternberg/copy-parents

frontend: add required paths to LLB and use it with --parents

cache/contenthash/checksum.go

@@ -12,6 +12,7 @@ import (
 	"sync"
 	"sync/atomic"
 
+	cerrdefs "github.com/containerd/errdefs"
 	iradix "github.com/hashicorp/go-immutable-radix/v2"
 	simplelru "github.com/hashicorp/golang-lru/v2/simplelru"
 	"github.com/moby/buildkit/cache"
@@ -59,6 +60,7 @@ type ChecksumOpts struct {
 	Wildcard        bool
 	IncludePatterns []string
 	ExcludePatterns []string
+	RequiredPaths   []string
 }
 
 func Checksum(ctx context.Context, ref cache.ImmutableRef, path string, opts ChecksumOpts, s session.Group) (digest.Digest, error) {
@@ -690,6 +692,17 @@ func (cc *cacheContext) includedPaths(ctx context.Context, m *mount, p string, o
 	cc.tree = txn.Commit()
 	cc.dirty = updated
 
+	// Validate that all required paths exist.
+	for _, requiredPath := range opts.RequiredPaths {
+		found := slices.ContainsFunc(includedPaths, func(includedPath *includedPath) bool {
+			return strings.HasPrefix(includedPath.path, requiredPath)
+		})
+
+		if !found {
+			return "", nil, errors.Wrapf(cerrdefs.ErrNotFound, "%q", requiredPath)
+		}
+	}
+
 	return origPrefix, includedPaths, nil
 }
 

client/llb/fileop.go

@@ -569,6 +569,7 @@ type CopyInfo struct {
 	CopyDirContentsOnly            bool
 	IncludePatterns                []string
 	ExcludePatterns                []string
+	RequiredPaths                  []string
 	AttemptUnpack                  bool
 	CreateDestPath                 bool
 	AllowWildcard                  bool
@@ -603,6 +604,7 @@ func (a *fileActionCopy) toProtoAction(ctx context.Context, parent string, base
 		Owner:                            a.info.ChownOpt.marshal(base),
 		IncludePatterns:                  a.info.IncludePatterns,
 		ExcludePatterns:                  a.info.ExcludePatterns,
+		RequiredPaths:                    a.info.RequiredPaths,
 		AllowWildcard:                    a.info.AllowWildcard,
 		AllowEmptyWildcard:               a.info.AllowEmptyWildcard,
 		FollowSymlink:                    a.info.FollowSymlinks,
@@ -647,6 +649,9 @@ func (a *fileActionCopy) addCaps(f *FileOp) {
 	if len(a.info.IncludePatterns) != 0 || len(a.info.ExcludePatterns) != 0 {
 		addCap(&f.constraints, pb.CapFileCopyIncludeExcludePatterns)
 	}
+	if len(a.info.RequiredPaths) != 0 {
+		addCap(&f.constraints, pb.CapFileCopyRequiredPaths)
+	}
 	if a.info.AlwaysReplaceExistingDestPaths {
 		addCap(&f.constraints, pb.CapFileCopyAlwaysReplaceExistingDestPaths)
 	}

frontend/dockerfile/dockerfile2llb/convert.go

@@ -1223,7 +1223,7 @@ func dispatchRun(d *dispatchState, c *instructions.RunCommand, proxy *llb.ProxyE
 	// Run command can potentially access any file. Mark the full filesystem as used.
 	d.paths["/"] = struct{}{}
 
-	var args = c.CmdLine
+	args := c.CmdLine
 	if len(c.Files) > 0 {
 		if len(args) != 1 || !c.PrependShell {
 			return errors.Errorf("parsing produced an invalid run command: %v", args)
@@ -1606,6 +1606,7 @@ func dispatchCopy(d *dispatchState, cfg copyConfig) error {
 		} else {
 			validateCopySourcePath(src, &cfg)
 			var patterns []string
+			var requiredPaths []string
 			if cfg.parents {
 				// detect optional pivot point
 				parent, pattern, ok := strings.Cut(src, "/./")
@@ -1622,6 +1623,12 @@ func dispatchCopy(d *dispatchState, cfg copyConfig) error {
 				}
 
 				patterns = []string{strings.TrimPrefix(pattern, "/")}
+
+				// determine if we want to require any paths to exist.
+				// we only require a path to exist if wildcards aren't present.
+				if !containsWildcards(src) && !containsWildcards(pattern) {
+					requiredPaths = []string{filepath.Join(src, pattern)}
+				}
 			}
 
 			src, err = system.NormalizePath("/", src, d.platform.OS, false)
@@ -1639,6 +1646,7 @@ func dispatchCopy(d *dispatchState, cfg copyConfig) error {
 				FollowSymlinks:      true,
 				CopyDirContentsOnly: true,
 				IncludePatterns:     patterns,
+				RequiredPaths:       requiredPaths,
 				AttemptUnpack:       unpack,
 				CreateDestPath:      true,
 				AllowWildcard:       true,
@@ -1760,7 +1768,7 @@ func dispatchOnbuild(d *dispatchState, c *instructions.OnbuildCommand) error {
 func dispatchCmd(d *dispatchState, c *instructions.CmdCommand, lint *linter.Linter) error {
 	validateUsedOnce(c, &d.cmd, lint)
 
-	var args = c.CmdLine
+	args := c.CmdLine
 	if c.PrependShell {
 		if len(d.image.Config.Shell) == 0 {
 			msg := linter.RuleJSONArgsRecommended.Format(c.Name())
@@ -1776,7 +1784,7 @@ func dispatchCmd(d *dispatchState, c *instructions.CmdCommand, lint *linter.Lint
 func dispatchEntrypoint(d *dispatchState, c *instructions.EntrypointCommand, lint *linter.Linter) error {
 	validateUsedOnce(c, &d.entrypoint, lint)
 
-	var args = c.CmdLine
+	args := c.CmdLine
 	if c.PrependShell {
 		if len(d.image.Config.Shell) == 0 {
 			msg := linter.RuleJSONArgsRecommended.Format(c.Name())
@@ -2692,3 +2700,15 @@ func (emptyEnvs) Get(string) (string, bool) {
 func (emptyEnvs) Keys() []string {
 	return nil
 }
+
+func containsWildcards(name string) bool {
+	for i := 0; i < len(name); i++ {
+		switch name[i] {
+		case '*', '?', '[':
+			return true
+		case '\\':
+			i++
+		}
+	}
+	return false
+}

frontend/dockerfile/dockerfile_parents_test.go

@@ -18,6 +18,7 @@ import (
 var parentsTests = integration.TestFuncs(
 	testCopyParents,
 	testCopyRelativeParents,
+	testCopyParentsMissingDirectory,
 )
 
 func init() {
@@ -180,3 +181,100 @@ eot
 		require.NoError(t, err)
 	}
 }
+
+func testCopyParentsMissingDirectory(t *testing.T, sb integration.Sandbox) {
+	f := getFrontend(t, sb)
+
+	dockerfile := []byte(`
+FROM alpine AS base
+WORKDIR /test
+RUN <<eot
+	set -ex
+	mkdir -p a/b/c/d/e
+	touch a/b/c/d/foo
+	touch a/b/c/d/e/bay
+eot
+
+FROM alpine AS normal
+COPY --from=base --parents /test/a/b/c/d /out/
+RUN <<eot
+	set -ex
+	[ -d /out/test/a/b/c/d/e ]
+	[ -f /out/test/a/b/c/d/e/bay ]
+	[ ! -d /out/e ]
+	[ ! -d /out/a ]
+eot
+
+FROM alpine AS withpivot
+COPY --from=base --parents /test/a/b/./c/d /out/
+RUN <<eot
+	set -ex
+	[ -d /out/c/d/e ]
+	[ -f /out/c/d/foo ]
+	[ ! -d /out/a ]
+	[ ! -d /out/e ]
+eot
+
+FROM alpine AS nonexistentfile
+COPY --from=base --parents /test/nonexistent-file /out/
+
+FROM alpine AS wildcard-nonexistent
+COPY --from=base --parents /test/a/b2*/c /out/
+RUN <<eot
+	set -ex
+	[ -d /out ]
+	[ ! -d /out/a ]
+eot
+
+FROM alpine AS wildcard-afterpivot
+COPY --from=base --parents /test/a/b/./c2* /out/
+RUN <<eot
+	set -ex
+	[ -d /out ]
+	[ ! -d /out/a ]
+	[ ! -d /out/c* ]
+eot
+`)
+
+	dir := integration.Tmpdir(
+		t,
+		fstest.CreateFile("Dockerfile", dockerfile, 0600),
+	)
+
+	c, err := client.New(sb.Context(), sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	type test struct {
+		target     string
+		errorRegex any
+	}
+
+	tests := []test{
+		{"normal", nil},
+		{"withpivot", nil},
+		{"nonexistentfile", `failed to calculate checksum of ref.*: "/test/nonexistent-file": not found`},
+		{"wildcard-nonexistent", nil},
+		{"wildcard-afterpivot", nil},
+	}
+
+	for _, tt := range tests {
+		t.Logf("target: %s", tt.target)
+		_, err = f.Solve(sb.Context(), c, client.SolveOpt{
+			FrontendAttrs: map[string]string{
+				"target": tt.target,
+			},
+			LocalMounts: map[string]fsutil.FS{
+				dockerui.DefaultLocalNameDockerfile: dir,
+				dockerui.DefaultLocalNameContext:    dir,
+			},
+		}, nil)
+
+		if tt.errorRegex != nil {
+			require.Error(t, err)
+			require.Regexp(t, tt.errorRegex, err.Error())
+		} else {
+			require.NoError(t, err)
+		}
+	}
+}

solver/llbsolver/ops/file.go

@@ -105,7 +105,7 @@ func (f *fileOp) CacheMap(ctx context.Context, jobCtx solver.JobContext, index i
 			markInvalid(action.Input)
 			processOwner(p.Owner, selectors)
 			if action.SecondaryInput != -1 && int(action.SecondaryInput) < f.numInputs {
-				addSelector(selectors, int(action.SecondaryInput), p.Src, p.AllowWildcard, p.FollowSymlink, p.IncludePatterns, p.ExcludePatterns)
+				addSelector(selectors, int(action.SecondaryInput), p.Src, p.AllowWildcard, p.FollowSymlink, p.IncludePatterns, p.ExcludePatterns, p.RequiredPaths)
 				p.Src = path.Base(p.Src)
 			}
 			dt, err = json.Marshal(p)
@@ -215,13 +215,14 @@ func (f *fileOp) Acquire(ctx context.Context) (solver.ReleaseFunc, error) {
 	}, nil
 }
 
-func addSelector(m map[int][]opsutils.Selector, idx int, sel string, wildcard, followLinks bool, includePatterns, excludePatterns []string) {
+func addSelector(m map[int][]opsutils.Selector, idx int, sel string, wildcard, followLinks bool, includePatterns, excludePatterns, requiredPaths []string) {
 	s := opsutils.Selector{
 		Path:            sel,
 		FollowLinks:     followLinks,
 		Wildcard:        wildcard && containsWildcards(sel),
 		IncludePatterns: includePatterns,
 		ExcludePatterns: excludePatterns,
+		RequiredPaths:   requiredPaths,
 	}
 
 	m[idx] = append(m[idx], s)
@@ -284,7 +285,7 @@ func processOwner(chopt *pb.ChownOpt, selectors map[int][]opsutils.Selector) err
 			if u.ByName.Input < 0 {
 				return errors.Errorf("invalid user index %d", u.ByName.Input)
 			}
-			addSelector(selectors, int(u.ByName.Input), "/etc/passwd", false, true, nil, nil)
+			addSelector(selectors, int(u.ByName.Input), "/etc/passwd", false, true, nil, nil, nil)
 		}
 	}
 	if chopt.Group != nil {
@@ -292,7 +293,7 @@ func processOwner(chopt *pb.ChownOpt, selectors map[int][]opsutils.Selector) err
 			if u.ByName.Input < 0 {
 				return errors.Errorf("invalid user index %d", u.ByName.Input)
 			}
-			addSelector(selectors, int(u.ByName.Input), "/etc/group", false, true, nil, nil)
+			addSelector(selectors, int(u.ByName.Input), "/etc/group", false, true, nil, nil, nil)
 		}
 	}
 	return nil

solver/llbsolver/ops/opsutils/contenthash.go

@@ -21,6 +21,7 @@ type Selector struct {
 	FollowLinks     bool
 	IncludePatterns []string
 	ExcludePatterns []string
+	RequiredPaths   []string
 }
 
 func (sel Selector) HasWildcardOrFilters() bool {
@@ -52,6 +53,7 @@ func NewContentHashFunc(selectors []Selector) solver.ResultBasedCacheFunc {
 						FollowLinks:     sel.FollowLinks,
 						IncludePatterns: sel.IncludePatterns,
 						ExcludePatterns: sel.ExcludePatterns,
+						RequiredPaths:   sel.RequiredPaths,
 					},
 					s,
 				)

solver/pb/caps.go

@@ -73,6 +73,7 @@ const (
 	CapFileBase                               apicaps.CapID = "file.base"
 	CapFileRmWildcard                         apicaps.CapID = "file.rm.wildcard"
 	CapFileCopyIncludeExcludePatterns         apicaps.CapID = "file.copy.includeexcludepatterns"
+	CapFileCopyRequiredPaths                  apicaps.CapID = "file.copy.requiredpaths"
 	CapFileRmNoFollowSymlink                  apicaps.CapID = "file.rm.nofollowsymlink"
 	CapFileCopyAlwaysReplaceExistingDestPaths apicaps.CapID = "file.copy.alwaysreplaceexistingdestpaths"
 	CapFileCopyModeStringFormat               apicaps.CapID = "file.copy.modestring"
@@ -451,6 +452,12 @@ func init() {
 		Status:  apicaps.CapStatusExperimental,
 	})
 
+	Caps.Init(apicaps.Cap{
+		ID:      CapFileCopyRequiredPaths,
+		Enabled: true,
+		Status:  apicaps.CapStatusExperimental,
+	})
+
 	Caps.Init(apicaps.Cap{
 		ID:      CapFileCopyAlwaysReplaceExistingDestPaths,
 		Enabled: true,

solver/pb/ops.pb.go

@@ -2527,7 +2527,10 @@ type FileActionCopy struct {
 	// alwaysReplaceExistingDestPaths results in an existing dest path that differs in type from the src path being replaced rather than the default of returning an error
 	AlwaysReplaceExistingDestPaths bool `protobuf:"varint,14,opt,name=alwaysReplaceExistingDestPaths,proto3" json:"alwaysReplaceExistingDestPaths,omitempty"`
 	// mode in non-octal format
-	ModeStr       string `protobuf:"bytes,15,opt,name=modeStr,proto3" json:"modeStr,omitempty"`
+	ModeStr string `protobuf:"bytes,15,opt,name=modeStr,proto3" json:"modeStr,omitempty"`
+	// required paths that must be included in the copy. This is only used when
+	// include_patterns has at least one pattern.
+	RequiredPaths []string `protobuf:"bytes,16,rep,name=required_paths,json=requiredPaths,proto3" json:"required_paths,omitempty"`
 	unknownFields protoimpl.UnknownFields
 	sizeCache     protoimpl.SizeCache
 }
@@ -2667,6 +2670,13 @@ func (x *FileActionCopy) GetModeStr() string {
 	return ""
 }
 
+func (x *FileActionCopy) GetRequiredPaths() []string {
+	if x != nil {
+		return x.RequiredPaths
+	}
+	return nil
+}
+
 type FileActionMkFile struct {
 	state protoimpl.MessageState `protogen:"open.v1"`
 	// path for the new file
@@ -3581,7 +3591,7 @@ const file_github_com_moby_buildkit_solver_pb_ops_proto_rawDesc = "" +
 	"\x05mkdir\x18\x06 \x01(\v2\x13.pb.FileActionMkDirH\x00R\x05mkdir\x12\"\n" +
 	"\x02rm\x18\a \x01(\v2\x10.pb.FileActionRmH\x00R\x02rm\x121\n" +
 	"\asymlink\x18\b \x01(\v2\x15.pb.FileActionSymlinkH\x00R\asymlinkB\b\n" +
-	"\x06action\"\xde\x04\n" +
+	"\x06action\"\x85\x05\n" +
 	"\x0eFileActionCopy\x12\x10\n" +
 	"\x03src\x18\x01 \x01(\tR\x03src\x12\x12\n" +
 	"\x04dest\x18\x02 \x01(\tR\x04dest\x12\"\n" +
@@ -3598,7 +3608,8 @@ const file_github_com_moby_buildkit_solver_pb_ops_proto_rawDesc = "" +
 	"\x10include_patterns\x18\f \x03(\tR\x0fincludePatterns\x12)\n" +
 	"\x10exclude_patterns\x18\r \x03(\tR\x0fexcludePatterns\x12F\n" +
 	"\x1ealwaysReplaceExistingDestPaths\x18\x0e \x01(\bR\x1ealwaysReplaceExistingDestPaths\x12\x18\n" +
-	"\amodeStr\x18\x0f \x01(\tR\amodeStr\"\x90\x01\n" +
+	"\amodeStr\x18\x0f \x01(\tR\amodeStr\x12%\n" +
+	"\x0erequired_paths\x18\x10 \x03(\tR\rrequiredPaths\"\x90\x01\n" +
 	"\x10FileActionMkFile\x12\x12\n" +
 	"\x04path\x18\x01 \x01(\tR\x04path\x12\x12\n" +
 	"\x04mode\x18\x02 \x01(\x05R\x04mode\x12\x12\n" +

solver/pb/ops.proto

@@ -355,6 +355,9 @@ message FileActionCopy {
 	bool alwaysReplaceExistingDestPaths = 14;
 	// mode in non-octal format
 	string modeStr = 15;
+	// required paths that must be included in the copy. This is only used when
+	// include_patterns has at least one pattern.
+	repeated string required_paths = 16;
 }
 
 message FileActionMkFile {

solver/pb/ops_vtproto.pb.go

@@ -886,6 +886,11 @@ func (m *FileActionCopy) CloneVT() *FileActionCopy {
 		copy(tmpContainer, rhs)
 		r.ExcludePatterns = tmpContainer
 	}
+	if rhs := m.RequiredPaths; rhs != nil {
+		tmpContainer := make([]string, len(rhs))
+		copy(tmpContainer, rhs)
+		r.RequiredPaths = tmpContainer
+	}
 	if len(m.unknownFields) > 0 {
 		r.unknownFields = make([]byte, len(m.unknownFields))
 		copy(r.unknownFields, m.unknownFields)
@@ -2578,6 +2583,15 @@ func (this *FileActionCopy) EqualVT(that *FileActionCopy) bool {
 	if this.ModeStr != that.ModeStr {
 		return false
 	}
+	if len(this.RequiredPaths) != len(that.RequiredPaths) {
+		return false
+	}
+	for i, vx := range this.RequiredPaths {
+		vy := that.RequiredPaths[i]
+		if vx != vy {
+			return false
+		}
+	}
 	return string(this.unknownFields) == string(that.unknownFields)
 }
 
@@ -5183,6 +5197,17 @@ func (m *FileActionCopy) MarshalToSizedBufferVT(dAtA []byte) (int, error) {
 		i -= len(m.unknownFields)
 		copy(dAtA[i:], m.unknownFields)
 	}
+	if len(m.RequiredPaths) > 0 {
+		for iNdEx := len(m.RequiredPaths) - 1; iNdEx >= 0; iNdEx-- {
+			i -= len(m.RequiredPaths[iNdEx])
+			copy(dAtA[i:], m.RequiredPaths[iNdEx])
+			i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.RequiredPaths[iNdEx])))
+			i--
+			dAtA[i] = 0x1
+			i--
+			dAtA[i] = 0x82
+		}
+	}
 	if len(m.ModeStr) > 0 {
 		i -= len(m.ModeStr)
 		copy(dAtA[i:], m.ModeStr)
@@ -6950,6 +6975,12 @@ func (m *FileActionCopy) SizeVT() (n int) {
 	if l > 0 {
 		n += 1 + l + protohelpers.SizeOfVarint(uint64(l))
 	}
+	if len(m.RequiredPaths) > 0 {
+		for _, s := range m.RequiredPaths {
+			l = len(s)
+			n += 2 + l + protohelpers.SizeOfVarint(uint64(l))
+		}
+	}
 	n += len(m.unknownFields)
 	return n
 }
@@ -13313,6 +13344,38 @@ func (m *FileActionCopy) UnmarshalVT(dAtA []byte) error {
 			}
 			m.ModeStr = string(dAtA[iNdEx:postIndex])
 			iNdEx = postIndex
+		case 16:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field RequiredPaths", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return protohelpers.ErrIntOverflow
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				stringLen |= uint64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return protohelpers.ErrInvalidLength
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex < 0 {
+				return protohelpers.ErrInvalidLength
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.RequiredPaths = append(m.RequiredPaths, string(dAtA[iNdEx:postIndex]))
+			iNdEx = postIndex
 		default:
 			iNdEx = preIndex
 			skippy, err := protohelpers.Skip(dAtA[iNdEx:])