Merge pull request #6600 from tonistiigi/git-mtime-commit

source: add git.mtime attr for commit-time mtimes

client/llb/source.go

@@ -478,6 +478,11 @@ func Git(url, fragment string, opts ...GitOption) State {
 		addCap(&gi.Constraints, pb.CapSourceGitSkipSubmodules)
 	}
 
+	if gi.MTime != "" {
+		attrs[pb.AttrGitMTime] = gi.MTime
+		addCap(&gi.Constraints, pb.CapSourceGitMTime)
+	}
+
 	addCap(&gi.Constraints, pb.CapSourceGit)
 
 	source := NewSource("git://"+id, attrs, gi.Constraints)
@@ -505,6 +510,7 @@ type GitInfo struct {
 	Ref              string
 	SubDir           string
 	SkipSubmodules   bool
+	MTime            string
 }
 
 func GitRef(v string) GitOption {
@@ -525,6 +531,20 @@ func GitSkipSubmodules() GitOption {
 	})
 }
 
+// GitMTimeCommit sets file modification times to the commit timestamp
+// of the resolved commit, rather than the checkout time.
+func GitMTimeCommit() GitOption {
+	return GitMTime("commit")
+}
+
+// GitMTime sets the file modification time policy for git sources.
+// Valid values are "checkout" (default) and "commit".
+func GitMTime(v string) GitOption {
+	return gitOptionFunc(func(gi *GitInfo) {
+		gi.MTime = v
+	})
+}
+
 func KeepGitDir() GitOption {
 	return gitOptionFunc(func(gi *GitInfo) {
 		gi.KeepGitDir = true

frontend/dockerfile/dfgitutil/git_ref.go

@@ -56,6 +56,9 @@ type GitRef struct {
 
 	// Submodules is true for URL that controls whether to fetch git submodules.
 	Submodules *bool
+
+	// MTime controls file modification time policy: "checkout" (default) or "commit".
+	MTime string
 }
 
 // ParseGitRef parses a git ref.
@@ -182,6 +185,13 @@ func (gf *GitRef) loadQuery(query url.Values) error {
 				}
 			}
 			gf.Submodules = &vv
+		case "mtime":
+			switch v[0] {
+			case "checkout", "commit":
+				gf.MTime = v[0]
+			default:
+				return errors.Errorf("invalid mtime value: %q (must be \"checkout\" or \"commit\")", v[0])
+			}
 		default:
 			return errors.Errorf("unexpected query %q", k)
 		}

frontend/dockerfile/dfgitutil/git_ref_test.go

@@ -214,6 +214,26 @@ func TestParseGitRef(t *testing.T) {
 				SubDir:    "/subdir",
 			},
 		},
+		{
+			ref: "https://github.com/moby/buildkit.git?mtime=commit",
+			expected: &GitRef{
+				Remote:    "https://github.com/moby/buildkit.git",
+				ShortName: "buildkit",
+				MTime:     "commit",
+			},
+		},
+		{
+			ref: "https://github.com/moby/buildkit.git?mtime=checkout",
+			expected: &GitRef{
+				Remote:    "https://github.com/moby/buildkit.git",
+				ShortName: "buildkit",
+				MTime:     "checkout",
+			},
+		},
+		{
+			ref: "https://github.com/moby/buildkit.git?mtime=invalid",
+			err: "invalid mtime value",
+		},
 		{
 			ref: "https://github.com/moby/buildkit.git?invalid=123",
 			err: "unexpected query \"invalid\"",

frontend/dockerui/context.go

@@ -6,6 +6,7 @@ import (
 	"context"
 	"path/filepath"
 	"regexp"
+	"slices"
 	"strconv"
 
 	"github.com/moby/buildkit/client/llb"
@@ -73,7 +74,11 @@ func (bc *Client) initContext(ctx context.Context) (*buildContext, error) {
 	if v, err := strconv.ParseBool(opts[keyContextKeepGitDirArg]); err == nil {
 		keepGit = &v
 	}
-	if st, ok, err := DetectGitContext(opts[localNameContext], keepGit); ok {
+	var extraGitOpts []llb.GitOption
+	if opts[keySourceDateEpoch] != "" {
+		extraGitOpts = append(extraGitOpts, llb.GitMTimeCommit())
+	}
+	if st, ok, err := DetectGitContext(opts[localNameContext], keepGit, extraGitOpts...); ok {
 		if err != nil {
 			return nil, err
 		}
@@ -143,15 +148,15 @@ func (bc *Client) initContext(ctx context.Context) (*buildContext, error) {
 	return bctx, nil
 }
 
-func DetectGitContext(ref string, keepGit *bool) (*llb.State, bool, error) {
+func DetectGitContext(ref string, keepGit *bool, opts ...llb.GitOption) (*llb.State, bool, error) {
 	g, isGit, err := dfgitutil.ParseGitRef(ref)
 	if err != nil {
 		return nil, isGit, err
 	}
-	gitOpts := []llb.GitOption{
+	gitOpts := slices.Concat(opts, []llb.GitOption{
 		llb.GitRef(g.Ref),
 		WithInternalName("load git source " + ref),
-	}
+	})
 	if g.KeepGitDir != nil && *g.KeepGitDir {
 		gitOpts = append(gitOpts, llb.KeepGitDir())
 	}
@@ -167,6 +172,9 @@ func DetectGitContext(ref string, keepGit *bool) (*llb.State, bool, error) {
 	if g.Submodules != nil && !*g.Submodules {
 		gitOpts = append(gitOpts, llb.GitSkipSubmodules())
 	}
+	if g.MTime != "" {
+		gitOpts = append(gitOpts, llb.GitMTime(g.MTime))
+	}
 
 	st := llb.Git(g.Remote, "", gitOpts...)
 	return &st, true, nil

solver/pb/attr.go

@@ -8,6 +8,7 @@ const AttrKnownSSHHosts = "git.knownsshhosts"
 const AttrMountSSHSock = "git.mountsshsock"
 const AttrGitChecksum = "git.checksum"
 const AttrGitSkipSubmodules = "git.skipsubmodules"
+const AttrGitMTime = "git.mtime"
 
 const AttrGitSignatureVerifyPubKey = "git.sig.pubkey"
 const AttrGitSignatureVerifyRejectExpired = "git.sig.rejectexpired"

solver/pb/caps.go

@@ -34,6 +34,7 @@ const (
 	CapSourceGitChecksum        apicaps.CapID = "source.git.checksum"
 	CapSourceGitSkipSubmodules  apicaps.CapID = "source.git.skipsubmodules"
 	CapSourceGitSignatureVerify apicaps.CapID = "source.git.signatureverify"
+	CapSourceGitMTime           apicaps.CapID = "source.git.mtime"
 
 	CapSourceHTTP         apicaps.CapID = "source.http"
 	CapSourceHTTPAuth     apicaps.CapID = "source.http.auth"
@@ -255,6 +256,12 @@ func init() {
 		Status:  apicaps.CapStatusExperimental,
 	})
 
+	Caps.Init(apicaps.Cap{
+		ID:      CapSourceGitMTime,
+		Enabled: true,
+		Status:  apicaps.CapStatusExperimental,
+	})
+
 	Caps.Init(apicaps.Cap{
 		ID:      CapSourceHTTP,
 		Enabled: true,

source/git/identifier.go

@@ -19,6 +19,7 @@ type GitIdentifier struct {
 	MountSSHSock     string
 	KnownSSHHosts    string
 	SkipSubmodules   bool
+	MTime            string // "checkout" (default) or "commit"
 
 	VerifySignature *GitSignatureVerifyOptions
 }

source/git/mtime_unix.go

@@ -0,0 +1,14 @@
+//go:build !windows
+
+package git
+
+import (
+	"time"
+
+	"golang.org/x/sys/unix"
+)
+
+func lchtimes(path string, t time.Time) error {
+	ts := unix.NsecToTimespec(t.UnixNano())
+	return unix.UtimesNanoAt(unix.AT_FDCWD, path, []unix.Timespec{ts, ts}, unix.AT_SYMLINK_NOFOLLOW)
+}

source/git/mtime_windows.go

@@ -0,0 +1,9 @@
+//go:build windows
+
+package git
+
+import "time"
+
+func lchtimes(_ string, _ time.Time) error {
+	return nil
+}

source/git/source.go

@@ -14,6 +14,7 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/moby/buildkit/cache"
 	"github.com/moby/buildkit/client"
@@ -134,6 +135,8 @@ func (gs *Source) Identifier(scheme, ref string, attrs map[string]string, platfo
 				id.VerifySignature = &GitSignatureVerifyOptions{}
 			}
 			id.VerifySignature.IgnoreSignedTag = v == "true"
+		case pb.AttrGitMTime:
+			id.MTime = v
 		}
 	}
 	if err := validateGitRef(id.Ref); err != nil {
@@ -267,6 +270,9 @@ func (gs *gitSourceHandler) shaToCacheKey(sha, ref string) string {
 	if gs.src.SkipSubmodules {
 		key += "(skip-submodules)"
 	}
+	if gs.src.MTime != "" && gs.src.MTime != "checkout" {
+		key += "(mtime=" + gs.src.MTime + ")"
+	}
 	return key
 }
 
@@ -1132,6 +1138,16 @@ func (gs *gitSourceHandler) checkout(ctx context.Context, repo *gitRepo, g sessi
 		}
 	}
 
+	if gs.src.MTime == "commit" {
+		commitTime, err := getCommitTime(ctx, git, ref)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to get commit time for %s", urlutil.RedactCredentials(gs.src.Remote))
+		}
+		if err := resetSnapshotMtimes(checkoutDir, commitTime); err != nil {
+			return nil, errors.Wrapf(err, "failed to normalize mtimes for %s", urlutil.RedactCredentials(gs.src.Remote))
+		}
+	}
+
 	if idmap := mount.IdentityMapping(); idmap != nil {
 		uid, gid := idmap.RootPair()
 		err := filepath.WalkDir(gitDir, func(p string, _ os.DirEntry, _ error) error {
@@ -1160,6 +1176,50 @@ func (gs *gitSourceHandler) checkout(ctx context.Context, repo *gitRepo, g sessi
 	return snap, nil
 }
 
+// getCommitTime returns the committer timestamp of the resolved commit.
+// For annotated tags, it peels to the underlying commit.
+func getCommitTime(ctx context.Context, git *gitutil.GitCLI, ref string) (time.Time, error) {
+	// %ct = committer date, UNIX timestamp; ^{commit} peels tags
+	buf, err := git.Run(ctx, "log", "-1", "--format=%ct", ref+"^{commit}")
+	if err != nil {
+		return time.Time{}, err
+	}
+	ts, err := strconv.ParseInt(strings.TrimSpace(string(buf)), 10, 64)
+	if err != nil {
+		return time.Time{}, errors.Wrapf(err, "failed to parse commit timestamp %q", string(buf))
+	}
+	return time.Unix(ts, 0), nil
+}
+
+// resetSnapshotMtimes walks dir and sets the mtime of every file,
+// symlink, and directory to t. Directories are set bottom-up so that
+// a parent's mtime is not invalidated by a later child write.
+func resetSnapshotMtimes(dir string, t time.Time) error {
+	var dirs []string
+	err := filepath.WalkDir(dir, func(p string, d os.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if d.IsDir() {
+			dirs = append(dirs, p)
+			return nil
+		}
+		if d.Type()&os.ModeSymlink != 0 {
+			return lchtimes(p, t)
+		}
+		return os.Chtimes(p, t, t)
+	})
+	if err != nil {
+		return err
+	}
+	for i := len(dirs) - 1; i >= 0; i-- {
+		if err := os.Chtimes(dirs[i], t, t); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 type wouldClobberExistingTagError struct {
 	error
 }

source/git/source_test.go

@@ -2433,3 +2433,137 @@ func logProgressStreams(ctx context.Context, t *testing.T) context.Context {
 	}()
 	return ctx
 }
+
+func TestResetSnapshotMtimes(t *testing.T) {
+	t.Parallel()
+	dir := t.TempDir()
+
+	// Create a tree:
+	//   dir/
+	//   dir/file.txt
+	//   dir/subdir/
+	//   dir/subdir/nested.txt
+	//   dir/link -> file.txt  (symlink)
+	require.NoError(t, os.MkdirAll(filepath.Join(dir, "subdir"), 0755))
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "file.txt"), []byte("hello"), 0644))
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "subdir", "nested.txt"), []byte("world"), 0644))
+	require.NoError(t, os.Symlink("file.txt", filepath.Join(dir, "link")))
+
+	target := time.Date(2023, 6, 15, 12, 0, 0, 0, time.UTC)
+	require.NoError(t, resetSnapshotMtimes(dir, target))
+
+	// Regular files should have mtime set
+	fi, err := os.Lstat(filepath.Join(dir, "file.txt"))
+	require.NoError(t, err)
+	require.Equal(t, target, fi.ModTime().UTC())
+
+	fi, err = os.Lstat(filepath.Join(dir, "subdir", "nested.txt"))
+	require.NoError(t, err)
+	require.Equal(t, target, fi.ModTime().UTC())
+
+	// Directories should have mtime set
+	fi, err = os.Lstat(dir)
+	require.NoError(t, err)
+	require.Equal(t, target, fi.ModTime().UTC())
+
+	fi, err = os.Lstat(filepath.Join(dir, "subdir"))
+	require.NoError(t, err)
+	require.Equal(t, target, fi.ModTime().UTC())
+
+	// Symlink itself should have mtime set via lutimes
+	fi, err = os.Lstat(filepath.Join(dir, "link"))
+	require.NoError(t, err)
+	require.Equal(t, target, fi.ModTime().UTC())
+
+	// Verify symlink itself still exists and points correctly
+	linkTarget, err := os.Readlink(filepath.Join(dir, "link"))
+	require.NoError(t, err)
+	require.Equal(t, "file.txt", linkTarget)
+}
+
+func TestCommitTimeMtimesSHA1(t *testing.T) {
+	testCommitTimeMtimes(t, "sha1", false)
+}
+
+func TestCommitTimeMtimesKeepGitDirSHA1(t *testing.T) {
+	testCommitTimeMtimes(t, "sha1", true)
+}
+
+func TestCommitTimeMtimesSHA256(t *testing.T) {
+	testCommitTimeMtimes(t, "sha256", false)
+}
+
+func TestCommitTimeMtimesKeepGitDirSHA256(t *testing.T) {
+	testCommitTimeMtimes(t, "sha256", true)
+}
+
+func testCommitTimeMtimes(t *testing.T, format string, keepGitDir bool) {
+	if runtime.GOOS == "windows" {
+		t.Skip("Depends on unimplemented containerd bind-mount support on Windows")
+	}
+
+	t.Parallel()
+	ctx := logProgressStreams(context.Background(), t)
+
+	gs := setupGitSource(t, t.TempDir())
+	repo := setupGitRepo(t, format)
+
+	// Get the commit timestamp of the master branch
+	cmd := exec.CommandContext(ctx, "git", "log", "-1", "--format=%ct", "master^{commit}")
+	cmd.Dir = repo.mainPath
+	out, err := cmd.Output()
+	require.NoError(t, err)
+	ts, err := strconv.ParseInt(strings.TrimSpace(string(out)), 10, 64)
+	require.NoError(t, err)
+	expectedTime := time.Unix(ts, 0)
+
+	// Snapshot without MTime=commit
+	idDefault := &GitIdentifier{Remote: repo.mainURL, KeepGitDir: keepGitDir}
+	gDefault, err := gs.Resolve(ctx, idDefault, nil, nil)
+	require.NoError(t, err)
+	keyDefault, _, _, _, err := gDefault.CacheKey(ctx, nil, 0)
+	require.NoError(t, err)
+
+	// Snapshot with MTime=commit
+	idCommit := &GitIdentifier{Remote: repo.mainURL, KeepGitDir: keepGitDir, MTime: "commit"}
+	gCommit, err := gs.Resolve(ctx, idCommit, nil, nil)
+	require.NoError(t, err)
+	keyCommit, _, _, _, err := gCommit.CacheKey(ctx, nil, 0)
+	require.NoError(t, err)
+
+	// Cache keys must differ
+	require.NotEqual(t, keyDefault, keyCommit)
+	require.Contains(t, keyCommit, "(mtime=commit)")
+
+	refCommit, err := gCommit.Snapshot(ctx, nil)
+	require.NoError(t, err)
+	defer refCommit.Release(context.TODO())
+
+	mount, err := refCommit.Mount(ctx, true, nil)
+	require.NoError(t, err)
+	lm := snapshot.LocalMounter(mount)
+	dir, err := lm.Mount()
+	require.NoError(t, err)
+	defer lm.Unmount()
+
+	// Verify file mtimes match the commit timestamp
+	fi, err := os.Lstat(filepath.Join(dir, "abc"))
+	require.NoError(t, err)
+	require.Equal(t, expectedTime.Unix(), fi.ModTime().Unix(), "file mtime should match commit time")
+
+	fi, err = os.Lstat(filepath.Join(dir, "def"))
+	require.NoError(t, err)
+	require.Equal(t, expectedTime.Unix(), fi.ModTime().Unix(), "file mtime should match commit time")
+
+	// Verify directory mtime
+	fi, err = os.Lstat(dir)
+	require.NoError(t, err)
+	require.Equal(t, expectedTime.Unix(), fi.ModTime().Unix(), "dir mtime should match commit time")
+
+	if keepGitDir {
+		// .git directory should also have normalized mtime
+		fi, err = os.Lstat(filepath.Join(dir, ".git"))
+		require.NoError(t, err)
+		require.Equal(t, expectedTime.Unix(), fi.ModTime().Unix(), ".git dir mtime should match commit time")
+	}
+}