source: add keepgitdir option for git source

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>

client/llb/llb.go

@@ -17,12 +17,13 @@ type RunOption func(m Meta) (Meta, error)
 func Source(id string) *State {
 	return &State{
 		metaNext: NewMeta(),
-		source:   &source{id: id},
+		source:   &source{id: id, attrs: map[string]string{}},
 	}
 }
 
 type source struct {
-	id string
+	id    string
+	attrs map[string]string
 }
 
 func (so *source) Validate() error {
@@ -39,7 +40,7 @@ func (so *source) marshalTo(list [][]byte, cache map[digest.Digest]struct{}) (di
 	}
 	po := &pb.Op{
 		Op: &pb.Op_Source{
-			Source: &pb.SourceOp{Identifier: so.id},
+			Source: &pb.SourceOp{Identifier: so.id, Attrs: so.attrs},
 		},
 	}
 	return appendResult(po, list, cache)
@@ -49,12 +50,24 @@ func Image(ref string) *State {
 	return Source("docker-image://" + ref) // controversial
 }
 
-func Git(remote, ref string) *State {
+func Git(remote, ref string, opts ...GitOption) *State {
 	id := remote
 	if ref != "" {
 		id += "#" + ref
 	}
-	return Source("git://" + id)
+	state := Source("git://" + id)
+	for _, opt := range opts {
+		opt(state.source)
+	}
+	return state
+}
+
+type GitOption func(*source)
+
+func KeepGitDir() GitOption {
+	return func(s *source) {
+		s.attrs[pb.AttrKeepGitDir] = "true"
+	}
 }
 
 type exec struct {

examples/buildkit/buildkit.go

@@ -50,10 +50,12 @@ func runc(version string) *llb.State {
 }
 
 func containerd(version string) *llb.State {
-	return goBuildBase().
+	repo := "github.com/containerd/containerd"
+	build := goBuildBase().
 		Run(llb.Shlex("apk add --no-cache btrfs-progs-dev")).
-		With(goFromGit("github.com/containerd/containerd", version)).
-		Run(llb.Shlex("make bin/containerd")).Root()
+		Dir("/go/src/" + repo).
+		Run(llb.Shlex("make bin/containerd"))
+	return build.AddMount("/go/src/"+repo, llb.Git(repo, version, llb.KeepGitDir()))
 }
 
 func buildkit(opt buildOpt) *llb.State {
@@ -75,7 +77,7 @@ func buildkit(opt buildOpt) *llb.State {
 
 	if opt.target == "containerd" {
 		return r.With(
-			copyFrom(containerd(opt.containerd), "/go/src/github.com/containerd/containerd/bin/containerd", "/bin/"),
+			copyFrom(containerd(opt.containerd), "/bin/containerd", "/bin/"),
 			copyFrom(builddContainerd, "/bin/buildd-containerd", "/bin/"))
 	}
 	return r.With(copyFrom(builddStandalone, "/bin/buildd-standalone", "/bin/"))

gometalinter.json

@@ -1,7 +1,7 @@
 {
   "Vendor": true,
   "Deadline": "8m",
-
+  "Exclude": ["solver/pb/ops.pb.go"],
   "DisableAll": true,
   "Enable": [
     "gofmt",

solver/pb/attr.go

@@ -0,0 +1,3 @@
+package pb
+
+const AttrKeepGitDir = "git.keepgitdir"

solver/pb/ops.pb.go

@@ -248,13 +248,22 @@ func (m *CopySource) String() string { return proto.CompactTextString(m) }
 func (*CopySource) ProtoMessage()    {}
 
 type SourceOp struct {
-	Identifier string `protobuf:"bytes,1,opt,name=identifier,proto3" json:"identifier,omitempty"`
+	// source type?
+	Identifier string            `protobuf:"bytes,1,opt,name=identifier,proto3" json:"identifier,omitempty"`
+	Attrs      map[string]string `protobuf:"bytes,2,rep,name=attrs" json:"attrs,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
 }
 
 func (m *SourceOp) Reset()         { *m = SourceOp{} }
 func (m *SourceOp) String() string { return proto.CompactTextString(m) }
 func (*SourceOp) ProtoMessage()    {}
 
+func (m *SourceOp) GetAttrs() map[string]string {
+	if m != nil {
+		return m.Attrs
+	}
+	return nil
+}
+
 func init() {
 	proto.RegisterType((*Op)(nil), "pb.Op")
 	proto.RegisterType((*Input)(nil), "pb.Input")
@@ -593,6 +602,23 @@ func (m *SourceOp) MarshalTo(data []byte) (int, error) {
 		i = encodeVarintOps(data, i, uint64(len(m.Identifier)))
 		i += copy(data[i:], m.Identifier)
 	}
+	if len(m.Attrs) > 0 {
+		for k, _ := range m.Attrs {
+			data[i] = 0x12
+			i++
+			v := m.Attrs[k]
+			mapSize := 1 + len(k) + sovOps(uint64(len(k))) + 1 + len(v) + sovOps(uint64(len(v)))
+			i = encodeVarintOps(data, i, uint64(mapSize))
+			data[i] = 0xa
+			i++
+			i = encodeVarintOps(data, i, uint64(len(k)))
+			i += copy(data[i:], k)
+			data[i] = 0x12
+			i++
+			i = encodeVarintOps(data, i, uint64(len(v)))
+			i += copy(data[i:], v)
+		}
+	}
 	return i, nil
 }
 
@@ -772,6 +798,14 @@ func (m *SourceOp) Size() (n int) {
 	if l > 0 {
 		n += 1 + l + sovOps(uint64(l))
 	}
+	if len(m.Attrs) > 0 {
+		for k, v := range m.Attrs {
+			_ = k
+			_ = v
+			mapEntrySize := 1 + len(k) + sovOps(uint64(len(k))) + 1 + len(v) + sovOps(uint64(len(v)))
+			n += mapEntrySize + 1 + sovOps(uint64(mapEntrySize))
+		}
+	}
 	return n
 }
 
@@ -1726,6 +1760,117 @@ func (m *SourceOp) Unmarshal(data []byte) error {
 			}
 			m.Identifier = string(data[iNdEx:postIndex])
 			iNdEx = postIndex
+		case 2:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Attrs", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowOps
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				msglen |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthOps
+			}
+			postIndex := iNdEx + msglen
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			var keykey uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowOps
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				keykey |= (uint64(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			var stringLenmapkey uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowOps
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				stringLenmapkey |= (uint64(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLenmapkey := int(stringLenmapkey)
+			if intStringLenmapkey < 0 {
+				return ErrInvalidLengthOps
+			}
+			postStringIndexmapkey := iNdEx + intStringLenmapkey
+			if postStringIndexmapkey > l {
+				return io.ErrUnexpectedEOF
+			}
+			mapkey := string(data[iNdEx:postStringIndexmapkey])
+			iNdEx = postStringIndexmapkey
+			var valuekey uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowOps
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				valuekey |= (uint64(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			var stringLenmapvalue uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowOps
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				stringLenmapvalue |= (uint64(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLenmapvalue := int(stringLenmapvalue)
+			if intStringLenmapvalue < 0 {
+				return ErrInvalidLengthOps
+			}
+			postStringIndexmapvalue := iNdEx + intStringLenmapvalue
+			if postStringIndexmapvalue > l {
+				return io.ErrUnexpectedEOF
+			}
+			mapvalue := string(data[iNdEx:postStringIndexmapvalue])
+			iNdEx = postStringIndexmapvalue
+			if m.Attrs == nil {
+				m.Attrs = make(map[string]string)
+			}
+			m.Attrs[mapkey] = mapvalue
+			iNdEx = postIndex
 		default:
 			iNdEx = preIndex
 			skippy, err := skipOps(data[iNdEx:])

solver/pb/ops.proto

@@ -47,5 +47,7 @@ message CopySource {
 }
 
 message SourceOp {
+	// source type?
 	string identifier = 1;
+	map<string, string> attrs = 2;
 }

solver/source.go

@@ -32,6 +32,16 @@ func (s *sourceOp) instance(ctx context.Context) (source.SourceInstance, error)
 	if err != nil {
 		return nil, err
 	}
+	if id, ok := id.(*source.GitIdentifier); ok {
+		for k, v := range s.op.Source.Attrs {
+			switch k {
+			case pb.AttrKeepGitDir:
+				if v == "true" {
+					id.KeepGitDir = true
+				}
+			}
+		}
+	}
 	src, err := s.sm.Resolve(ctx, id)
 	if err != nil {
 		return nil, err

source/git/gitsource.go

@@ -3,7 +3,9 @@ package git
 import (
 	"bytes"
 	"io"
+	"os"
 	"os/exec"
+	"path/filepath"
 	"regexp"
 	"strings"
 
@@ -12,6 +14,7 @@ import (
 	"github.com/boltdb/bolt"
 	"github.com/moby/buildkit/cache"
 	"github.com/moby/buildkit/cache/metadata"
+	"github.com/moby/buildkit/identity"
 	"github.com/moby/buildkit/snapshot"
 	"github.com/moby/buildkit/source"
 	"github.com/moby/buildkit/util/progress/logs"
@@ -130,7 +133,7 @@ func (gs *gitSource) mountRemote(ctx context.Context, remote string) (target str
 		}
 
 		if err := si.Update(func(b *bolt.Bucket) error {
-			return si.SetValue(b, "git-repo", *v)
+			return si.SetValue(b, "git-remote", *v)
 		}); err != nil {
 			return "", nil, err
 		}
@@ -143,7 +146,7 @@ func (gs *gitSource) mountRemote(ctx context.Context, remote string) (target str
 
 type gitSourceHandler struct {
 	*gitSource
-	src      *source.GitIdentifier
+	src      source.GitIdentifier
 	cacheKey string
 }
 
@@ -154,7 +157,7 @@ func (gs *gitSource) Resolve(ctx context.Context, id source.Identifier) (source.
 	}
 
 	return &gitSourceHandler{
-		src:       gitIdentifier,
+		src:       *gitIdentifier,
 		gitSource: gs,
 	}, nil
 }
@@ -165,7 +168,6 @@ func (gs *gitSourceHandler) CacheKey(ctx context.Context) (string, error) {
 	if ref == "" {
 		ref = "master"
 	}
-
 	gs.locker.Lock(remote)
 	defer gs.locker.Unlock(remote)
 
@@ -196,7 +198,7 @@ func (gs *gitSourceHandler) CacheKey(ctx context.Context) (string, error) {
 	if !isCommitSHA(sha) {
 		return "", errors.Errorf("invalid commit sha %q", sha)
 	}
-	gs.cacheKey = ref
+	gs.cacheKey = sha
 	return sha, nil
 }
 
@@ -245,10 +247,14 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context) (out cache.ImmutableRe
 		args := []string{"fetch", "--recurse-submodules=yes"}
 		if !isCommitSHA(ref) { // TODO: find a branch from ls-remote?
 			args = append(args, "--depth=1", "--no-tags")
+		} else {
+			if _, err := os.Lstat(filepath.Join(gitDir, "shallow")); err == nil {
+				args = append(args, "--unshallow")
+			}
 		}
 		args = append(args, "origin")
 		if !isCommitSHA(ref) {
-			args = append(args, ref+":"+ref)
+			args = append(args, ref+":tags/"+ref)
 			// local refs are needed so they would be advertised on next fetches
 			// TODO: is there a better way to do this?
 		}
@@ -280,12 +286,45 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context) (out cache.ImmutableRe
 	if err != nil {
 		return nil, err
 	}
+	defer func() {
+		if retErr != nil && lm != nil {
+			lm.Unmount()
+		}
+	}()
 
-	_, err = gitWithinDir(ctx, gitDir, checkoutDir, "checkout", ref, "--", ".")
-	lm.Unmount()
-	if err != nil {
-		return nil, errors.Wrapf(err, "failed to checkout remote %s", gs.src.Remote)
+	if gs.src.KeepGitDir {
+		_, err = gitWithinDir(ctx, checkoutDir, "", "init")
+		if err != nil {
+			return nil, err
+		}
+		_, err = gitWithinDir(ctx, checkoutDir, "", "remote", "add", "origin", gitDir)
+		if err != nil {
+			return nil, err
+		}
+		pullref := ref
+		if isCommitSHA(ref) {
+			pullref = "refs/buildkit/" + identity.NewID()
+			_, err = gitWithinDir(ctx, gitDir, "", "update-ref", pullref, ref)
+			if err != nil {
+				return nil, err
+			}
+		}
+		_, err = gitWithinDir(ctx, checkoutDir, "", "fetch", "--recurse-submodules=yes", "--depth=1", "origin", pullref)
+		if err != nil {
+			return nil, err
+		}
+		_, err = gitWithinDir(ctx, checkoutDir, checkoutDir, "checkout", "FETCH_HEAD")
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to checkout remote %s", gs.src.Remote)
+		}
+	} else {
+		_, err = gitWithinDir(ctx, gitDir, checkoutDir, "checkout", ref, "--", ".")
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to checkout remote %s", gs.src.Remote)
+		}
 	}
+	lm.Unmount()
+	lm = nil
 
 	snap, err := checkoutRef.ReleaseAndCommit(ctx)
 	if err != nil {

source/git/gitsource_test.go

@@ -20,6 +20,13 @@ import (
 )
 
 func TestRepeatedFetch(t *testing.T) {
+	testRepeatedFetch(t, false)
+}
+func TestRepeatedFetchKeepGitDir(t *testing.T) {
+	testRepeatedFetch(t, true)
+}
+
+func testRepeatedFetch(t *testing.T, keepGitDir bool) {
 	ctx := namespaces.WithNamespace(context.Background(), "buildkit-test")
 
 	tmpdir, err := ioutil.TempDir("", "buildkit-state")
@@ -34,7 +41,7 @@ func TestRepeatedFetch(t *testing.T) {
 
 	setupGitRepo(t, repodir)
 
-	id := &source.GitIdentifier{Remote: repodir}
+	id := &source.GitIdentifier{Remote: repodir, KeepGitDir: keepGitDir}
 
 	g, err := gs.Resolve(ctx, id)
 	require.NoError(t, err)
@@ -66,7 +73,7 @@ func TestRepeatedFetch(t *testing.T) {
 	require.True(t, os.IsNotExist(err))
 
 	// second fetch returns same dir
-	id = &source.GitIdentifier{Remote: repodir, Ref: "master"}
+	id = &source.GitIdentifier{Remote: repodir, Ref: "master", KeepGitDir: keepGitDir}
 
 	g, err = gs.Resolve(ctx, id)
 	require.NoError(t, err)
@@ -82,7 +89,7 @@ func TestRepeatedFetch(t *testing.T) {
 
 	require.Equal(t, ref1.ID(), ref2.ID())
 
-	id = &source.GitIdentifier{Remote: repodir, Ref: "feature"}
+	id = &source.GitIdentifier{Remote: repodir, Ref: "feature", KeepGitDir: keepGitDir}
 
 	g, err = gs.Resolve(ctx, id)
 	require.NoError(t, err)
@@ -110,6 +117,13 @@ func TestRepeatedFetch(t *testing.T) {
 }
 
 func TestFetchBySHA(t *testing.T) {
+	testFetchBySHA(t, false)
+}
+func TestFetchBySHAKeepGitDir(t *testing.T) {
+	testFetchBySHA(t, true)
+}
+
+func testFetchBySHA(t *testing.T, keepGitDir bool) {
 	ctx := namespaces.WithNamespace(context.Background(), "buildkit-test")
 
 	tmpdir, err := ioutil.TempDir("", "buildkit-state")
@@ -133,7 +147,7 @@ func TestFetchBySHA(t *testing.T) {
 	sha := strings.TrimSpace(string(out))
 	require.Equal(t, 40, len(sha))
 
-	id := &source.GitIdentifier{Remote: repodir, Ref: sha}
+	id := &source.GitIdentifier{Remote: repodir, Ref: sha, KeepGitDir: keepGitDir}
 
 	g, err := gs.Resolve(ctx, id)
 	require.NoError(t, err)
@@ -161,6 +175,96 @@ func TestFetchBySHA(t *testing.T) {
 	require.Equal(t, "baz\n", string(dt))
 }
 
+func TestMultipleRepos(t *testing.T) {
+	testMultipleRepos(t, false)
+}
+
+func TestMultipleReposKeepGitDir(t *testing.T) {
+	testMultipleRepos(t, true)
+}
+
+func testMultipleRepos(t *testing.T, keepGitDir bool) {
+	ctx := namespaces.WithNamespace(context.Background(), "buildkit-test")
+
+	tmpdir, err := ioutil.TempDir("", "buildkit-state")
+	require.NoError(t, err)
+	defer os.RemoveAll(tmpdir)
+
+	gs := setupGitSource(t, tmpdir)
+
+	repodir, err := ioutil.TempDir("", "buildkit-gitsource")
+	require.NoError(t, err)
+	defer os.RemoveAll(repodir)
+
+	setupGitRepo(t, repodir)
+
+	repodir2, err := ioutil.TempDir("", "buildkit-gitsource")
+	require.NoError(t, err)
+	defer os.RemoveAll(repodir2)
+
+	runShell(t, repodir2,
+		"git init",
+		"git config --local user.email test",
+		"git config --local user.name test",
+		"echo xyz > xyz",
+		"git add xyz",
+		"git commit -m initial",
+	)
+
+	id := &source.GitIdentifier{Remote: repodir, KeepGitDir: keepGitDir}
+	id2 := &source.GitIdentifier{Remote: repodir2, KeepGitDir: keepGitDir}
+
+	g, err := gs.Resolve(ctx, id)
+	require.NoError(t, err)
+
+	g2, err := gs.Resolve(ctx, id2)
+	require.NoError(t, err)
+
+	key1, err := g.CacheKey(ctx)
+	require.NoError(t, err)
+	require.Equal(t, 40, len(key1))
+
+	key2, err := g2.CacheKey(ctx)
+	require.NoError(t, err)
+	require.Equal(t, 40, len(key2))
+
+	require.NotEqual(t, key1, key2)
+
+	ref1, err := g.Snapshot(ctx)
+	require.NoError(t, err)
+	defer ref1.Release(context.TODO())
+
+	mount, err := ref1.Mount(ctx)
+	require.NoError(t, err)
+
+	lm := snapshot.LocalMounter(mount)
+	dir, err := lm.Mount()
+	require.NoError(t, err)
+	defer lm.Unmount()
+
+	ref2, err := g2.Snapshot(ctx)
+	require.NoError(t, err)
+	defer ref2.Release(context.TODO())
+
+	mount, err = ref2.Mount(ctx)
+	require.NoError(t, err)
+
+	lm = snapshot.LocalMounter(mount)
+	dir2, err := lm.Mount()
+	require.NoError(t, err)
+	defer lm.Unmount()
+
+	dt, err := ioutil.ReadFile(filepath.Join(dir, "def"))
+	require.NoError(t, err)
+
+	require.Equal(t, "bar\n", string(dt))
+
+	dt, err = ioutil.ReadFile(filepath.Join(dir2, "xyz"))
+	require.NoError(t, err)
+
+	require.Equal(t, "xyz\n", string(dt))
+}
+
 func setupGitSource(t *testing.T, tmpdir string) source.Source {
 	snapshotter, err := naive.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
 	assert.NoError(t, err)

source/gitidentifier.go

@@ -9,9 +9,10 @@ import (
 )
 
 type GitIdentifier struct {
-	Remote string
-	Ref    string
-	Subdir string
+	Remote     string
+	Ref        string
+	Subdir     string
+	KeepGitDir bool
 }
 
 func NewGitIdentifier(remoteURL string) (*GitIdentifier, error) {