diff --git a/client/llb/source.go b/client/llb/source.go index 66f30cb3c..ecc6734ff 100644 --- a/client/llb/source.go +++ b/client/llb/source.go @@ -478,6 +478,11 @@ func Git(url, fragment string, opts ...GitOption) State { addCap(&gi.Constraints, pb.CapSourceGitSkipSubmodules) } + if gi.MTime != "" { + attrs[pb.AttrGitMTime] = gi.MTime + addCap(&gi.Constraints, pb.CapSourceGitMTime) + } + addCap(&gi.Constraints, pb.CapSourceGit) source := NewSource("git://"+id, attrs, gi.Constraints) @@ -505,6 +510,7 @@ type GitInfo struct { Ref string SubDir string SkipSubmodules bool + MTime string } func GitRef(v string) GitOption { @@ -525,6 +531,20 @@ func GitSkipSubmodules() GitOption { }) } +// GitMTimeCommit sets file modification times to the commit timestamp +// of the resolved commit, rather than the checkout time. +func GitMTimeCommit() GitOption { + return GitMTime("commit") +} + +// GitMTime sets the file modification time policy for git sources. +// Valid values are "checkout" (default) and "commit". +func GitMTime(v string) GitOption { + return gitOptionFunc(func(gi *GitInfo) { + gi.MTime = v + }) +} + func KeepGitDir() GitOption { return gitOptionFunc(func(gi *GitInfo) { gi.KeepGitDir = true diff --git a/frontend/dockerfile/dfgitutil/git_ref.go b/frontend/dockerfile/dfgitutil/git_ref.go index a45c9fb7d..26abb16cb 100644 --- a/frontend/dockerfile/dfgitutil/git_ref.go +++ b/frontend/dockerfile/dfgitutil/git_ref.go @@ -56,6 +56,9 @@ type GitRef struct { // Submodules is true for URL that controls whether to fetch git submodules. Submodules *bool + + // MTime controls file modification time policy: "checkout" (default) or "commit". + MTime string } // ParseGitRef parses a git ref. @@ -182,6 +185,13 @@ func (gf *GitRef) loadQuery(query url.Values) error { } } gf.Submodules = &vv + case "mtime": + switch v[0] { + case "checkout", "commit": + gf.MTime = v[0] + default: + return errors.Errorf("invalid mtime value: %q (must be \"checkout\" or \"commit\")", v[0]) + } default: return errors.Errorf("unexpected query %q", k) } diff --git a/frontend/dockerfile/dfgitutil/git_ref_test.go b/frontend/dockerfile/dfgitutil/git_ref_test.go index c767e5f1c..a03128bd8 100644 --- a/frontend/dockerfile/dfgitutil/git_ref_test.go +++ b/frontend/dockerfile/dfgitutil/git_ref_test.go @@ -214,6 +214,26 @@ func TestParseGitRef(t *testing.T) { SubDir: "/subdir", }, }, + { + ref: "https://github.com/moby/buildkit.git?mtime=commit", + expected: &GitRef{ + Remote: "https://github.com/moby/buildkit.git", + ShortName: "buildkit", + MTime: "commit", + }, + }, + { + ref: "https://github.com/moby/buildkit.git?mtime=checkout", + expected: &GitRef{ + Remote: "https://github.com/moby/buildkit.git", + ShortName: "buildkit", + MTime: "checkout", + }, + }, + { + ref: "https://github.com/moby/buildkit.git?mtime=invalid", + err: "invalid mtime value", + }, { ref: "https://github.com/moby/buildkit.git?invalid=123", err: "unexpected query \"invalid\"", diff --git a/frontend/dockerui/context.go b/frontend/dockerui/context.go index 88d68569b..3bd5094f6 100644 --- a/frontend/dockerui/context.go +++ b/frontend/dockerui/context.go @@ -6,6 +6,7 @@ import ( "context" "path/filepath" "regexp" + "slices" "strconv" "github.com/moby/buildkit/client/llb" @@ -73,7 +74,11 @@ func (bc *Client) initContext(ctx context.Context) (*buildContext, error) { if v, err := strconv.ParseBool(opts[keyContextKeepGitDirArg]); err == nil { keepGit = &v } - if st, ok, err := DetectGitContext(opts[localNameContext], keepGit); ok { + var extraGitOpts []llb.GitOption + if opts[keySourceDateEpoch] != "" { + extraGitOpts = append(extraGitOpts, llb.GitMTimeCommit()) + } + if st, ok, err := DetectGitContext(opts[localNameContext], keepGit, extraGitOpts...); ok { if err != nil { return nil, err } @@ -143,15 +148,15 @@ func (bc *Client) initContext(ctx context.Context) (*buildContext, error) { return bctx, nil } -func DetectGitContext(ref string, keepGit *bool) (*llb.State, bool, error) { +func DetectGitContext(ref string, keepGit *bool, opts ...llb.GitOption) (*llb.State, bool, error) { g, isGit, err := dfgitutil.ParseGitRef(ref) if err != nil { return nil, isGit, err } - gitOpts := []llb.GitOption{ + gitOpts := slices.Concat(opts, []llb.GitOption{ llb.GitRef(g.Ref), WithInternalName("load git source " + ref), - } + }) if g.KeepGitDir != nil && *g.KeepGitDir { gitOpts = append(gitOpts, llb.KeepGitDir()) } @@ -167,6 +172,9 @@ func DetectGitContext(ref string, keepGit *bool) (*llb.State, bool, error) { if g.Submodules != nil && !*g.Submodules { gitOpts = append(gitOpts, llb.GitSkipSubmodules()) } + if g.MTime != "" { + gitOpts = append(gitOpts, llb.GitMTime(g.MTime)) + } st := llb.Git(g.Remote, "", gitOpts...) return &st, true, nil diff --git a/solver/pb/attr.go b/solver/pb/attr.go index 0db767341..53aa4eafc 100644 --- a/solver/pb/attr.go +++ b/solver/pb/attr.go @@ -8,6 +8,7 @@ const AttrKnownSSHHosts = "git.knownsshhosts" const AttrMountSSHSock = "git.mountsshsock" const AttrGitChecksum = "git.checksum" const AttrGitSkipSubmodules = "git.skipsubmodules" +const AttrGitMTime = "git.mtime" const AttrGitSignatureVerifyPubKey = "git.sig.pubkey" const AttrGitSignatureVerifyRejectExpired = "git.sig.rejectexpired" diff --git a/solver/pb/caps.go b/solver/pb/caps.go index d869b9f81..d62c5e2c2 100644 --- a/solver/pb/caps.go +++ b/solver/pb/caps.go @@ -34,6 +34,7 @@ const ( CapSourceGitChecksum apicaps.CapID = "source.git.checksum" CapSourceGitSkipSubmodules apicaps.CapID = "source.git.skipsubmodules" CapSourceGitSignatureVerify apicaps.CapID = "source.git.signatureverify" + CapSourceGitMTime apicaps.CapID = "source.git.mtime" CapSourceHTTP apicaps.CapID = "source.http" CapSourceHTTPAuth apicaps.CapID = "source.http.auth" @@ -255,6 +256,12 @@ func init() { Status: apicaps.CapStatusExperimental, }) + Caps.Init(apicaps.Cap{ + ID: CapSourceGitMTime, + Enabled: true, + Status: apicaps.CapStatusExperimental, + }) + Caps.Init(apicaps.Cap{ ID: CapSourceHTTP, Enabled: true, diff --git a/source/git/identifier.go b/source/git/identifier.go index 60fc3e6d4..7dda7f476 100644 --- a/source/git/identifier.go +++ b/source/git/identifier.go @@ -19,6 +19,7 @@ type GitIdentifier struct { MountSSHSock string KnownSSHHosts string SkipSubmodules bool + MTime string // "checkout" (default) or "commit" VerifySignature *GitSignatureVerifyOptions } diff --git a/source/git/mtime_unix.go b/source/git/mtime_unix.go new file mode 100644 index 000000000..9857da7c2 --- /dev/null +++ b/source/git/mtime_unix.go @@ -0,0 +1,14 @@ +//go:build !windows + +package git + +import ( + "time" + + "golang.org/x/sys/unix" +) + +func lchtimes(path string, t time.Time) error { + ts := unix.NsecToTimespec(t.UnixNano()) + return unix.UtimesNanoAt(unix.AT_FDCWD, path, []unix.Timespec{ts, ts}, unix.AT_SYMLINK_NOFOLLOW) +} diff --git a/source/git/mtime_windows.go b/source/git/mtime_windows.go new file mode 100644 index 000000000..7b17bb7e3 --- /dev/null +++ b/source/git/mtime_windows.go @@ -0,0 +1,9 @@ +//go:build windows + +package git + +import "time" + +func lchtimes(_ string, _ time.Time) error { + return nil +} diff --git a/source/git/source.go b/source/git/source.go index 10b905eaa..c3a20ca60 100644 --- a/source/git/source.go +++ b/source/git/source.go @@ -14,6 +14,7 @@ import ( "regexp" "strconv" "strings" + "time" "github.com/moby/buildkit/cache" "github.com/moby/buildkit/client" @@ -134,6 +135,8 @@ func (gs *Source) Identifier(scheme, ref string, attrs map[string]string, platfo id.VerifySignature = &GitSignatureVerifyOptions{} } id.VerifySignature.IgnoreSignedTag = v == "true" + case pb.AttrGitMTime: + id.MTime = v } } if err := validateGitRef(id.Ref); err != nil { @@ -267,6 +270,9 @@ func (gs *gitSourceHandler) shaToCacheKey(sha, ref string) string { if gs.src.SkipSubmodules { key += "(skip-submodules)" } + if gs.src.MTime != "" && gs.src.MTime != "checkout" { + key += "(mtime=" + gs.src.MTime + ")" + } return key } @@ -1132,6 +1138,16 @@ func (gs *gitSourceHandler) checkout(ctx context.Context, repo *gitRepo, g sessi } } + if gs.src.MTime == "commit" { + commitTime, err := getCommitTime(ctx, git, ref) + if err != nil { + return nil, errors.Wrapf(err, "failed to get commit time for %s", urlutil.RedactCredentials(gs.src.Remote)) + } + if err := resetSnapshotMtimes(checkoutDir, commitTime); err != nil { + return nil, errors.Wrapf(err, "failed to normalize mtimes for %s", urlutil.RedactCredentials(gs.src.Remote)) + } + } + if idmap := mount.IdentityMapping(); idmap != nil { uid, gid := idmap.RootPair() err := filepath.WalkDir(gitDir, func(p string, _ os.DirEntry, _ error) error { @@ -1160,6 +1176,50 @@ func (gs *gitSourceHandler) checkout(ctx context.Context, repo *gitRepo, g sessi return snap, nil } +// getCommitTime returns the committer timestamp of the resolved commit. +// For annotated tags, it peels to the underlying commit. +func getCommitTime(ctx context.Context, git *gitutil.GitCLI, ref string) (time.Time, error) { + // %ct = committer date, UNIX timestamp; ^{commit} peels tags + buf, err := git.Run(ctx, "log", "-1", "--format=%ct", ref+"^{commit}") + if err != nil { + return time.Time{}, err + } + ts, err := strconv.ParseInt(strings.TrimSpace(string(buf)), 10, 64) + if err != nil { + return time.Time{}, errors.Wrapf(err, "failed to parse commit timestamp %q", string(buf)) + } + return time.Unix(ts, 0), nil +} + +// resetSnapshotMtimes walks dir and sets the mtime of every file, +// symlink, and directory to t. Directories are set bottom-up so that +// a parent's mtime is not invalidated by a later child write. +func resetSnapshotMtimes(dir string, t time.Time) error { + var dirs []string + err := filepath.WalkDir(dir, func(p string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + dirs = append(dirs, p) + return nil + } + if d.Type()&os.ModeSymlink != 0 { + return lchtimes(p, t) + } + return os.Chtimes(p, t, t) + }) + if err != nil { + return err + } + for i := len(dirs) - 1; i >= 0; i-- { + if err := os.Chtimes(dirs[i], t, t); err != nil { + return err + } + } + return nil +} + type wouldClobberExistingTagError struct { error } diff --git a/source/git/source_test.go b/source/git/source_test.go index 94346f1ca..079caa708 100644 --- a/source/git/source_test.go +++ b/source/git/source_test.go @@ -2433,3 +2433,137 @@ func logProgressStreams(ctx context.Context, t *testing.T) context.Context { }() return ctx } + +func TestResetSnapshotMtimes(t *testing.T) { + t.Parallel() + dir := t.TempDir() + + // Create a tree: + // dir/ + // dir/file.txt + // dir/subdir/ + // dir/subdir/nested.txt + // dir/link -> file.txt (symlink) + require.NoError(t, os.MkdirAll(filepath.Join(dir, "subdir"), 0755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "file.txt"), []byte("hello"), 0644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "subdir", "nested.txt"), []byte("world"), 0644)) + require.NoError(t, os.Symlink("file.txt", filepath.Join(dir, "link"))) + + target := time.Date(2023, 6, 15, 12, 0, 0, 0, time.UTC) + require.NoError(t, resetSnapshotMtimes(dir, target)) + + // Regular files should have mtime set + fi, err := os.Lstat(filepath.Join(dir, "file.txt")) + require.NoError(t, err) + require.Equal(t, target, fi.ModTime().UTC()) + + fi, err = os.Lstat(filepath.Join(dir, "subdir", "nested.txt")) + require.NoError(t, err) + require.Equal(t, target, fi.ModTime().UTC()) + + // Directories should have mtime set + fi, err = os.Lstat(dir) + require.NoError(t, err) + require.Equal(t, target, fi.ModTime().UTC()) + + fi, err = os.Lstat(filepath.Join(dir, "subdir")) + require.NoError(t, err) + require.Equal(t, target, fi.ModTime().UTC()) + + // Symlink itself should have mtime set via lutimes + fi, err = os.Lstat(filepath.Join(dir, "link")) + require.NoError(t, err) + require.Equal(t, target, fi.ModTime().UTC()) + + // Verify symlink itself still exists and points correctly + linkTarget, err := os.Readlink(filepath.Join(dir, "link")) + require.NoError(t, err) + require.Equal(t, "file.txt", linkTarget) +} + +func TestCommitTimeMtimesSHA1(t *testing.T) { + testCommitTimeMtimes(t, "sha1", false) +} + +func TestCommitTimeMtimesKeepGitDirSHA1(t *testing.T) { + testCommitTimeMtimes(t, "sha1", true) +} + +func TestCommitTimeMtimesSHA256(t *testing.T) { + testCommitTimeMtimes(t, "sha256", false) +} + +func TestCommitTimeMtimesKeepGitDirSHA256(t *testing.T) { + testCommitTimeMtimes(t, "sha256", true) +} + +func testCommitTimeMtimes(t *testing.T, format string, keepGitDir bool) { + if runtime.GOOS == "windows" { + t.Skip("Depends on unimplemented containerd bind-mount support on Windows") + } + + t.Parallel() + ctx := logProgressStreams(context.Background(), t) + + gs := setupGitSource(t, t.TempDir()) + repo := setupGitRepo(t, format) + + // Get the commit timestamp of the master branch + cmd := exec.CommandContext(ctx, "git", "log", "-1", "--format=%ct", "master^{commit}") + cmd.Dir = repo.mainPath + out, err := cmd.Output() + require.NoError(t, err) + ts, err := strconv.ParseInt(strings.TrimSpace(string(out)), 10, 64) + require.NoError(t, err) + expectedTime := time.Unix(ts, 0) + + // Snapshot without MTime=commit + idDefault := &GitIdentifier{Remote: repo.mainURL, KeepGitDir: keepGitDir} + gDefault, err := gs.Resolve(ctx, idDefault, nil, nil) + require.NoError(t, err) + keyDefault, _, _, _, err := gDefault.CacheKey(ctx, nil, 0) + require.NoError(t, err) + + // Snapshot with MTime=commit + idCommit := &GitIdentifier{Remote: repo.mainURL, KeepGitDir: keepGitDir, MTime: "commit"} + gCommit, err := gs.Resolve(ctx, idCommit, nil, nil) + require.NoError(t, err) + keyCommit, _, _, _, err := gCommit.CacheKey(ctx, nil, 0) + require.NoError(t, err) + + // Cache keys must differ + require.NotEqual(t, keyDefault, keyCommit) + require.Contains(t, keyCommit, "(mtime=commit)") + + refCommit, err := gCommit.Snapshot(ctx, nil) + require.NoError(t, err) + defer refCommit.Release(context.TODO()) + + mount, err := refCommit.Mount(ctx, true, nil) + require.NoError(t, err) + lm := snapshot.LocalMounter(mount) + dir, err := lm.Mount() + require.NoError(t, err) + defer lm.Unmount() + + // Verify file mtimes match the commit timestamp + fi, err := os.Lstat(filepath.Join(dir, "abc")) + require.NoError(t, err) + require.Equal(t, expectedTime.Unix(), fi.ModTime().Unix(), "file mtime should match commit time") + + fi, err = os.Lstat(filepath.Join(dir, "def")) + require.NoError(t, err) + require.Equal(t, expectedTime.Unix(), fi.ModTime().Unix(), "file mtime should match commit time") + + // Verify directory mtime + fi, err = os.Lstat(dir) + require.NoError(t, err) + require.Equal(t, expectedTime.Unix(), fi.ModTime().Unix(), "dir mtime should match commit time") + + if keepGitDir { + // .git directory should also have normalized mtime + fi, err = os.Lstat(filepath.Join(dir, ".git")) + require.NoError(t, err) + require.Equal(t, expectedTime.Unix(), fi.ModTime().Unix(), ".git dir mtime should match commit time") + } +}