source: add keepgitdir option for git source

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>

source/git/gitsource.go

@@ -3,7 +3,9 @@ package git
 import (
 	"bytes"
 	"io"
+	"os"
 	"os/exec"
+	"path/filepath"
 	"regexp"
 	"strings"
 
@@ -12,6 +14,7 @@ import (
 	"github.com/boltdb/bolt"
 	"github.com/moby/buildkit/cache"
 	"github.com/moby/buildkit/cache/metadata"
+	"github.com/moby/buildkit/identity"
 	"github.com/moby/buildkit/snapshot"
 	"github.com/moby/buildkit/source"
 	"github.com/moby/buildkit/util/progress/logs"
@@ -130,7 +133,7 @@ func (gs *gitSource) mountRemote(ctx context.Context, remote string) (target str
 		}
 
 		if err := si.Update(func(b *bolt.Bucket) error {
-			return si.SetValue(b, "git-repo", *v)
+			return si.SetValue(b, "git-remote", *v)
 		}); err != nil {
 			return "", nil, err
 		}
@@ -143,7 +146,7 @@ func (gs *gitSource) mountRemote(ctx context.Context, remote string) (target str
 
 type gitSourceHandler struct {
 	*gitSource
-	src      *source.GitIdentifier
+	src      source.GitIdentifier
 	cacheKey string
 }
 
@@ -154,7 +157,7 @@ func (gs *gitSource) Resolve(ctx context.Context, id source.Identifier) (source.
 	}
 
 	return &gitSourceHandler{
-		src:       gitIdentifier,
+		src:       *gitIdentifier,
 		gitSource: gs,
 	}, nil
 }
@@ -165,7 +168,6 @@ func (gs *gitSourceHandler) CacheKey(ctx context.Context) (string, error) {
 	if ref == "" {
 		ref = "master"
 	}
-
 	gs.locker.Lock(remote)
 	defer gs.locker.Unlock(remote)
 
@@ -196,7 +198,7 @@ func (gs *gitSourceHandler) CacheKey(ctx context.Context) (string, error) {
 	if !isCommitSHA(sha) {
 		return "", errors.Errorf("invalid commit sha %q", sha)
 	}
-	gs.cacheKey = ref
+	gs.cacheKey = sha
 	return sha, nil
 }
 
@@ -245,10 +247,14 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context) (out cache.ImmutableRe
 		args := []string{"fetch", "--recurse-submodules=yes"}
 		if !isCommitSHA(ref) { // TODO: find a branch from ls-remote?
 			args = append(args, "--depth=1", "--no-tags")
+		} else {
+			if _, err := os.Lstat(filepath.Join(gitDir, "shallow")); err == nil {
+				args = append(args, "--unshallow")
+			}
 		}
 		args = append(args, "origin")
 		if !isCommitSHA(ref) {
-			args = append(args, ref+":"+ref)
+			args = append(args, ref+":tags/"+ref)
 			// local refs are needed so they would be advertised on next fetches
 			// TODO: is there a better way to do this?
 		}
@@ -280,12 +286,45 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context) (out cache.ImmutableRe
 	if err != nil {
 		return nil, err
 	}
+	defer func() {
+		if retErr != nil && lm != nil {
+			lm.Unmount()
+		}
+	}()
 
-	_, err = gitWithinDir(ctx, gitDir, checkoutDir, "checkout", ref, "--", ".")
-	lm.Unmount()
-	if err != nil {
-		return nil, errors.Wrapf(err, "failed to checkout remote %s", gs.src.Remote)
+	if gs.src.KeepGitDir {
+		_, err = gitWithinDir(ctx, checkoutDir, "", "init")
+		if err != nil {
+			return nil, err
+		}
+		_, err = gitWithinDir(ctx, checkoutDir, "", "remote", "add", "origin", gitDir)
+		if err != nil {
+			return nil, err
+		}
+		pullref := ref
+		if isCommitSHA(ref) {
+			pullref = "refs/buildkit/" + identity.NewID()
+			_, err = gitWithinDir(ctx, gitDir, "", "update-ref", pullref, ref)
+			if err != nil {
+				return nil, err
+			}
+		}
+		_, err = gitWithinDir(ctx, checkoutDir, "", "fetch", "--recurse-submodules=yes", "--depth=1", "origin", pullref)
+		if err != nil {
+			return nil, err
+		}
+		_, err = gitWithinDir(ctx, checkoutDir, checkoutDir, "checkout", "FETCH_HEAD")
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to checkout remote %s", gs.src.Remote)
+		}
+	} else {
+		_, err = gitWithinDir(ctx, gitDir, checkoutDir, "checkout", ref, "--", ".")
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to checkout remote %s", gs.src.Remote)
+		}
 	}
+	lm.Unmount()
+	lm = nil
 
 	snap, err := checkoutRef.ReleaseAndCommit(ctx)
 	if err != nil {

source/git/gitsource_test.go

@@ -20,6 +20,13 @@ import (
 )
 
 func TestRepeatedFetch(t *testing.T) {
+	testRepeatedFetch(t, false)
+}
+func TestRepeatedFetchKeepGitDir(t *testing.T) {
+	testRepeatedFetch(t, true)
+}
+
+func testRepeatedFetch(t *testing.T, keepGitDir bool) {
 	ctx := namespaces.WithNamespace(context.Background(), "buildkit-test")
 
 	tmpdir, err := ioutil.TempDir("", "buildkit-state")
@@ -34,7 +41,7 @@ func TestRepeatedFetch(t *testing.T) {
 
 	setupGitRepo(t, repodir)
 
-	id := &source.GitIdentifier{Remote: repodir}
+	id := &source.GitIdentifier{Remote: repodir, KeepGitDir: keepGitDir}
 
 	g, err := gs.Resolve(ctx, id)
 	require.NoError(t, err)
@@ -66,7 +73,7 @@ func TestRepeatedFetch(t *testing.T) {
 	require.True(t, os.IsNotExist(err))
 
 	// second fetch returns same dir
-	id = &source.GitIdentifier{Remote: repodir, Ref: "master"}
+	id = &source.GitIdentifier{Remote: repodir, Ref: "master", KeepGitDir: keepGitDir}
 
 	g, err = gs.Resolve(ctx, id)
 	require.NoError(t, err)
@@ -82,7 +89,7 @@ func TestRepeatedFetch(t *testing.T) {
 
 	require.Equal(t, ref1.ID(), ref2.ID())
 
-	id = &source.GitIdentifier{Remote: repodir, Ref: "feature"}
+	id = &source.GitIdentifier{Remote: repodir, Ref: "feature", KeepGitDir: keepGitDir}
 
 	g, err = gs.Resolve(ctx, id)
 	require.NoError(t, err)
@@ -110,6 +117,13 @@ func TestRepeatedFetch(t *testing.T) {
 }
 
 func TestFetchBySHA(t *testing.T) {
+	testFetchBySHA(t, false)
+}
+func TestFetchBySHAKeepGitDir(t *testing.T) {
+	testFetchBySHA(t, true)
+}
+
+func testFetchBySHA(t *testing.T, keepGitDir bool) {
 	ctx := namespaces.WithNamespace(context.Background(), "buildkit-test")
 
 	tmpdir, err := ioutil.TempDir("", "buildkit-state")
@@ -133,7 +147,7 @@ func TestFetchBySHA(t *testing.T) {
 	sha := strings.TrimSpace(string(out))
 	require.Equal(t, 40, len(sha))
 
-	id := &source.GitIdentifier{Remote: repodir, Ref: sha}
+	id := &source.GitIdentifier{Remote: repodir, Ref: sha, KeepGitDir: keepGitDir}
 
 	g, err := gs.Resolve(ctx, id)
 	require.NoError(t, err)
@@ -161,6 +175,96 @@ func TestFetchBySHA(t *testing.T) {
 	require.Equal(t, "baz\n", string(dt))
 }
 
+func TestMultipleRepos(t *testing.T) {
+	testMultipleRepos(t, false)
+}
+
+func TestMultipleReposKeepGitDir(t *testing.T) {
+	testMultipleRepos(t, true)
+}
+
+func testMultipleRepos(t *testing.T, keepGitDir bool) {
+	ctx := namespaces.WithNamespace(context.Background(), "buildkit-test")
+
+	tmpdir, err := ioutil.TempDir("", "buildkit-state")
+	require.NoError(t, err)
+	defer os.RemoveAll(tmpdir)
+
+	gs := setupGitSource(t, tmpdir)
+
+	repodir, err := ioutil.TempDir("", "buildkit-gitsource")
+	require.NoError(t, err)
+	defer os.RemoveAll(repodir)
+
+	setupGitRepo(t, repodir)
+
+	repodir2, err := ioutil.TempDir("", "buildkit-gitsource")
+	require.NoError(t, err)
+	defer os.RemoveAll(repodir2)
+
+	runShell(t, repodir2,
+		"git init",
+		"git config --local user.email test",
+		"git config --local user.name test",
+		"echo xyz > xyz",
+		"git add xyz",
+		"git commit -m initial",
+	)
+
+	id := &source.GitIdentifier{Remote: repodir, KeepGitDir: keepGitDir}
+	id2 := &source.GitIdentifier{Remote: repodir2, KeepGitDir: keepGitDir}
+
+	g, err := gs.Resolve(ctx, id)
+	require.NoError(t, err)
+
+	g2, err := gs.Resolve(ctx, id2)
+	require.NoError(t, err)
+
+	key1, err := g.CacheKey(ctx)
+	require.NoError(t, err)
+	require.Equal(t, 40, len(key1))
+
+	key2, err := g2.CacheKey(ctx)
+	require.NoError(t, err)
+	require.Equal(t, 40, len(key2))
+
+	require.NotEqual(t, key1, key2)
+
+	ref1, err := g.Snapshot(ctx)
+	require.NoError(t, err)
+	defer ref1.Release(context.TODO())
+
+	mount, err := ref1.Mount(ctx)
+	require.NoError(t, err)
+
+	lm := snapshot.LocalMounter(mount)
+	dir, err := lm.Mount()
+	require.NoError(t, err)
+	defer lm.Unmount()
+
+	ref2, err := g2.Snapshot(ctx)
+	require.NoError(t, err)
+	defer ref2.Release(context.TODO())
+
+	mount, err = ref2.Mount(ctx)
+	require.NoError(t, err)
+
+	lm = snapshot.LocalMounter(mount)
+	dir2, err := lm.Mount()
+	require.NoError(t, err)
+	defer lm.Unmount()
+
+	dt, err := ioutil.ReadFile(filepath.Join(dir, "def"))
+	require.NoError(t, err)
+
+	require.Equal(t, "bar\n", string(dt))
+
+	dt, err = ioutil.ReadFile(filepath.Join(dir2, "xyz"))
+	require.NoError(t, err)
+
+	require.Equal(t, "xyz\n", string(dt))
+}
+
 func setupGitSource(t *testing.T, tmpdir string) source.Source {
 	snapshotter, err := naive.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
 	assert.NoError(t, err)

source/gitidentifier.go

@@ -9,9 +9,10 @@ import (
 )
 
 type GitIdentifier struct {
-	Remote string
-	Ref    string
-	Subdir string
+	Remote     string
+	Ref        string
+	Subdir     string
+	KeepGitDir bool
 }
 
 func NewGitIdentifier(remoteURL string) (*GitIdentifier, error) {