diff --git a/client/llb/llb.go b/client/llb/llb.go index 1974a2c76..e77aea156 100644 --- a/client/llb/llb.go +++ b/client/llb/llb.go @@ -17,12 +17,13 @@ type RunOption func(m Meta) (Meta, error) func Source(id string) *State { return &State{ metaNext: NewMeta(), - source: &source{id: id}, + source: &source{id: id, attrs: map[string]string{}}, } } type source struct { - id string + id string + attrs map[string]string } func (so *source) Validate() error { @@ -39,7 +40,7 @@ func (so *source) marshalTo(list [][]byte, cache map[digest.Digest]struct{}) (di } po := &pb.Op{ Op: &pb.Op_Source{ - Source: &pb.SourceOp{Identifier: so.id}, + Source: &pb.SourceOp{Identifier: so.id, Attrs: so.attrs}, }, } return appendResult(po, list, cache) @@ -49,12 +50,24 @@ func Image(ref string) *State { return Source("docker-image://" + ref) // controversial } -func Git(remote, ref string) *State { +func Git(remote, ref string, opts ...GitOption) *State { id := remote if ref != "" { id += "#" + ref } - return Source("git://" + id) + state := Source("git://" + id) + for _, opt := range opts { + opt(state.source) + } + return state +} + +type GitOption func(*source) + +func KeepGitDir() GitOption { + return func(s *source) { + s.attrs[pb.AttrKeepGitDir] = "true" + } } type exec struct { diff --git a/examples/buildkit/buildkit.go b/examples/buildkit/buildkit.go index dbfa4fb16..279be7172 100644 --- a/examples/buildkit/buildkit.go +++ b/examples/buildkit/buildkit.go @@ -50,10 +50,12 @@ func runc(version string) *llb.State { } func containerd(version string) *llb.State { - return goBuildBase(). + repo := "github.com/containerd/containerd" + build := goBuildBase(). Run(llb.Shlex("apk add --no-cache btrfs-progs-dev")). - With(goFromGit("github.com/containerd/containerd", version)). - Run(llb.Shlex("make bin/containerd")).Root() + Dir("/go/src/" + repo). + Run(llb.Shlex("make bin/containerd")) + return build.AddMount("/go/src/"+repo, llb.Git(repo, version, llb.KeepGitDir())) } func buildkit(opt buildOpt) *llb.State { @@ -75,7 +77,7 @@ func buildkit(opt buildOpt) *llb.State { if opt.target == "containerd" { return r.With( - copyFrom(containerd(opt.containerd), "/go/src/github.com/containerd/containerd/bin/containerd", "/bin/"), + copyFrom(containerd(opt.containerd), "/bin/containerd", "/bin/"), copyFrom(builddContainerd, "/bin/buildd-containerd", "/bin/")) } return r.With(copyFrom(builddStandalone, "/bin/buildd-standalone", "/bin/")) diff --git a/gometalinter.json b/gometalinter.json index 0625ea9ab..af1c0632b 100644 --- a/gometalinter.json +++ b/gometalinter.json @@ -1,7 +1,7 @@ { "Vendor": true, "Deadline": "8m", - + "Exclude": ["solver/pb/ops.pb.go"], "DisableAll": true, "Enable": [ "gofmt", diff --git a/solver/pb/attr.go b/solver/pb/attr.go new file mode 100644 index 000000000..64ac3886d --- /dev/null +++ b/solver/pb/attr.go @@ -0,0 +1,3 @@ +package pb + +const AttrKeepGitDir = "git.keepgitdir" diff --git a/solver/pb/ops.pb.go b/solver/pb/ops.pb.go index 4e61296d1..3f60292c6 100644 --- a/solver/pb/ops.pb.go +++ b/solver/pb/ops.pb.go @@ -248,13 +248,22 @@ func (m *CopySource) String() string { return proto.CompactTextString(m) } func (*CopySource) ProtoMessage() {} type SourceOp struct { - Identifier string `protobuf:"bytes,1,opt,name=identifier,proto3" json:"identifier,omitempty"` + // source type? + Identifier string `protobuf:"bytes,1,opt,name=identifier,proto3" json:"identifier,omitempty"` + Attrs map[string]string `protobuf:"bytes,2,rep,name=attrs" json:"attrs,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` } func (m *SourceOp) Reset() { *m = SourceOp{} } func (m *SourceOp) String() string { return proto.CompactTextString(m) } func (*SourceOp) ProtoMessage() {} +func (m *SourceOp) GetAttrs() map[string]string { + if m != nil { + return m.Attrs + } + return nil +} + func init() { proto.RegisterType((*Op)(nil), "pb.Op") proto.RegisterType((*Input)(nil), "pb.Input") @@ -593,6 +602,23 @@ func (m *SourceOp) MarshalTo(data []byte) (int, error) { i = encodeVarintOps(data, i, uint64(len(m.Identifier))) i += copy(data[i:], m.Identifier) } + if len(m.Attrs) > 0 { + for k, _ := range m.Attrs { + data[i] = 0x12 + i++ + v := m.Attrs[k] + mapSize := 1 + len(k) + sovOps(uint64(len(k))) + 1 + len(v) + sovOps(uint64(len(v))) + i = encodeVarintOps(data, i, uint64(mapSize)) + data[i] = 0xa + i++ + i = encodeVarintOps(data, i, uint64(len(k))) + i += copy(data[i:], k) + data[i] = 0x12 + i++ + i = encodeVarintOps(data, i, uint64(len(v))) + i += copy(data[i:], v) + } + } return i, nil } @@ -772,6 +798,14 @@ func (m *SourceOp) Size() (n int) { if l > 0 { n += 1 + l + sovOps(uint64(l)) } + if len(m.Attrs) > 0 { + for k, v := range m.Attrs { + _ = k + _ = v + mapEntrySize := 1 + len(k) + sovOps(uint64(len(k))) + 1 + len(v) + sovOps(uint64(len(v))) + n += mapEntrySize + 1 + sovOps(uint64(mapEntrySize)) + } + } return n } @@ -1726,6 +1760,117 @@ func (m *SourceOp) Unmarshal(data []byte) error { } m.Identifier = string(data[iNdEx:postIndex]) iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Attrs", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowOps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthOps + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + var keykey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowOps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + keykey |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + var stringLenmapkey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowOps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + stringLenmapkey |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLenmapkey := int(stringLenmapkey) + if intStringLenmapkey < 0 { + return ErrInvalidLengthOps + } + postStringIndexmapkey := iNdEx + intStringLenmapkey + if postStringIndexmapkey > l { + return io.ErrUnexpectedEOF + } + mapkey := string(data[iNdEx:postStringIndexmapkey]) + iNdEx = postStringIndexmapkey + var valuekey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowOps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + valuekey |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + var stringLenmapvalue uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowOps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + stringLenmapvalue |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLenmapvalue := int(stringLenmapvalue) + if intStringLenmapvalue < 0 { + return ErrInvalidLengthOps + } + postStringIndexmapvalue := iNdEx + intStringLenmapvalue + if postStringIndexmapvalue > l { + return io.ErrUnexpectedEOF + } + mapvalue := string(data[iNdEx:postStringIndexmapvalue]) + iNdEx = postStringIndexmapvalue + if m.Attrs == nil { + m.Attrs = make(map[string]string) + } + m.Attrs[mapkey] = mapvalue + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipOps(data[iNdEx:]) diff --git a/solver/pb/ops.proto b/solver/pb/ops.proto index fa9020757..ed5b069de 100644 --- a/solver/pb/ops.proto +++ b/solver/pb/ops.proto @@ -47,5 +47,7 @@ message CopySource { } message SourceOp { + // source type? string identifier = 1; + map attrs = 2; } diff --git a/solver/source.go b/solver/source.go index 911cc547d..378aa25fa 100644 --- a/solver/source.go +++ b/solver/source.go @@ -32,6 +32,16 @@ func (s *sourceOp) instance(ctx context.Context) (source.SourceInstance, error) if err != nil { return nil, err } + if id, ok := id.(*source.GitIdentifier); ok { + for k, v := range s.op.Source.Attrs { + switch k { + case pb.AttrKeepGitDir: + if v == "true" { + id.KeepGitDir = true + } + } + } + } src, err := s.sm.Resolve(ctx, id) if err != nil { return nil, err diff --git a/source/git/gitsource.go b/source/git/gitsource.go index 9f5a924ef..1ed72b3dd 100644 --- a/source/git/gitsource.go +++ b/source/git/gitsource.go @@ -3,7 +3,9 @@ package git import ( "bytes" "io" + "os" "os/exec" + "path/filepath" "regexp" "strings" @@ -12,6 +14,7 @@ import ( "github.com/boltdb/bolt" "github.com/moby/buildkit/cache" "github.com/moby/buildkit/cache/metadata" + "github.com/moby/buildkit/identity" "github.com/moby/buildkit/snapshot" "github.com/moby/buildkit/source" "github.com/moby/buildkit/util/progress/logs" @@ -130,7 +133,7 @@ func (gs *gitSource) mountRemote(ctx context.Context, remote string) (target str } if err := si.Update(func(b *bolt.Bucket) error { - return si.SetValue(b, "git-repo", *v) + return si.SetValue(b, "git-remote", *v) }); err != nil { return "", nil, err } @@ -143,7 +146,7 @@ func (gs *gitSource) mountRemote(ctx context.Context, remote string) (target str type gitSourceHandler struct { *gitSource - src *source.GitIdentifier + src source.GitIdentifier cacheKey string } @@ -154,7 +157,7 @@ func (gs *gitSource) Resolve(ctx context.Context, id source.Identifier) (source. } return &gitSourceHandler{ - src: gitIdentifier, + src: *gitIdentifier, gitSource: gs, }, nil } @@ -165,7 +168,6 @@ func (gs *gitSourceHandler) CacheKey(ctx context.Context) (string, error) { if ref == "" { ref = "master" } - gs.locker.Lock(remote) defer gs.locker.Unlock(remote) @@ -196,7 +198,7 @@ func (gs *gitSourceHandler) CacheKey(ctx context.Context) (string, error) { if !isCommitSHA(sha) { return "", errors.Errorf("invalid commit sha %q", sha) } - gs.cacheKey = ref + gs.cacheKey = sha return sha, nil } @@ -245,10 +247,14 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context) (out cache.ImmutableRe args := []string{"fetch", "--recurse-submodules=yes"} if !isCommitSHA(ref) { // TODO: find a branch from ls-remote? args = append(args, "--depth=1", "--no-tags") + } else { + if _, err := os.Lstat(filepath.Join(gitDir, "shallow")); err == nil { + args = append(args, "--unshallow") + } } args = append(args, "origin") if !isCommitSHA(ref) { - args = append(args, ref+":"+ref) + args = append(args, ref+":tags/"+ref) // local refs are needed so they would be advertised on next fetches // TODO: is there a better way to do this? } @@ -280,12 +286,45 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context) (out cache.ImmutableRe if err != nil { return nil, err } + defer func() { + if retErr != nil && lm != nil { + lm.Unmount() + } + }() - _, err = gitWithinDir(ctx, gitDir, checkoutDir, "checkout", ref, "--", ".") - lm.Unmount() - if err != nil { - return nil, errors.Wrapf(err, "failed to checkout remote %s", gs.src.Remote) + if gs.src.KeepGitDir { + _, err = gitWithinDir(ctx, checkoutDir, "", "init") + if err != nil { + return nil, err + } + _, err = gitWithinDir(ctx, checkoutDir, "", "remote", "add", "origin", gitDir) + if err != nil { + return nil, err + } + pullref := ref + if isCommitSHA(ref) { + pullref = "refs/buildkit/" + identity.NewID() + _, err = gitWithinDir(ctx, gitDir, "", "update-ref", pullref, ref) + if err != nil { + return nil, err + } + } + _, err = gitWithinDir(ctx, checkoutDir, "", "fetch", "--recurse-submodules=yes", "--depth=1", "origin", pullref) + if err != nil { + return nil, err + } + _, err = gitWithinDir(ctx, checkoutDir, checkoutDir, "checkout", "FETCH_HEAD") + if err != nil { + return nil, errors.Wrapf(err, "failed to checkout remote %s", gs.src.Remote) + } + } else { + _, err = gitWithinDir(ctx, gitDir, checkoutDir, "checkout", ref, "--", ".") + if err != nil { + return nil, errors.Wrapf(err, "failed to checkout remote %s", gs.src.Remote) + } } + lm.Unmount() + lm = nil snap, err := checkoutRef.ReleaseAndCommit(ctx) if err != nil { diff --git a/source/git/gitsource_test.go b/source/git/gitsource_test.go index 8e1009c26..f9d8da6e7 100644 --- a/source/git/gitsource_test.go +++ b/source/git/gitsource_test.go @@ -20,6 +20,13 @@ import ( ) func TestRepeatedFetch(t *testing.T) { + testRepeatedFetch(t, false) +} +func TestRepeatedFetchKeepGitDir(t *testing.T) { + testRepeatedFetch(t, true) +} + +func testRepeatedFetch(t *testing.T, keepGitDir bool) { ctx := namespaces.WithNamespace(context.Background(), "buildkit-test") tmpdir, err := ioutil.TempDir("", "buildkit-state") @@ -34,7 +41,7 @@ func TestRepeatedFetch(t *testing.T) { setupGitRepo(t, repodir) - id := &source.GitIdentifier{Remote: repodir} + id := &source.GitIdentifier{Remote: repodir, KeepGitDir: keepGitDir} g, err := gs.Resolve(ctx, id) require.NoError(t, err) @@ -66,7 +73,7 @@ func TestRepeatedFetch(t *testing.T) { require.True(t, os.IsNotExist(err)) // second fetch returns same dir - id = &source.GitIdentifier{Remote: repodir, Ref: "master"} + id = &source.GitIdentifier{Remote: repodir, Ref: "master", KeepGitDir: keepGitDir} g, err = gs.Resolve(ctx, id) require.NoError(t, err) @@ -82,7 +89,7 @@ func TestRepeatedFetch(t *testing.T) { require.Equal(t, ref1.ID(), ref2.ID()) - id = &source.GitIdentifier{Remote: repodir, Ref: "feature"} + id = &source.GitIdentifier{Remote: repodir, Ref: "feature", KeepGitDir: keepGitDir} g, err = gs.Resolve(ctx, id) require.NoError(t, err) @@ -110,6 +117,13 @@ func TestRepeatedFetch(t *testing.T) { } func TestFetchBySHA(t *testing.T) { + testFetchBySHA(t, false) +} +func TestFetchBySHAKeepGitDir(t *testing.T) { + testFetchBySHA(t, true) +} + +func testFetchBySHA(t *testing.T, keepGitDir bool) { ctx := namespaces.WithNamespace(context.Background(), "buildkit-test") tmpdir, err := ioutil.TempDir("", "buildkit-state") @@ -133,7 +147,7 @@ func TestFetchBySHA(t *testing.T) { sha := strings.TrimSpace(string(out)) require.Equal(t, 40, len(sha)) - id := &source.GitIdentifier{Remote: repodir, Ref: sha} + id := &source.GitIdentifier{Remote: repodir, Ref: sha, KeepGitDir: keepGitDir} g, err := gs.Resolve(ctx, id) require.NoError(t, err) @@ -161,6 +175,96 @@ func TestFetchBySHA(t *testing.T) { require.Equal(t, "baz\n", string(dt)) } +func TestMultipleRepos(t *testing.T) { + testMultipleRepos(t, false) +} + +func TestMultipleReposKeepGitDir(t *testing.T) { + testMultipleRepos(t, true) +} + +func testMultipleRepos(t *testing.T, keepGitDir bool) { + ctx := namespaces.WithNamespace(context.Background(), "buildkit-test") + + tmpdir, err := ioutil.TempDir("", "buildkit-state") + require.NoError(t, err) + defer os.RemoveAll(tmpdir) + + gs := setupGitSource(t, tmpdir) + + repodir, err := ioutil.TempDir("", "buildkit-gitsource") + require.NoError(t, err) + defer os.RemoveAll(repodir) + + setupGitRepo(t, repodir) + + repodir2, err := ioutil.TempDir("", "buildkit-gitsource") + require.NoError(t, err) + defer os.RemoveAll(repodir2) + + runShell(t, repodir2, + "git init", + "git config --local user.email test", + "git config --local user.name test", + "echo xyz > xyz", + "git add xyz", + "git commit -m initial", + ) + + id := &source.GitIdentifier{Remote: repodir, KeepGitDir: keepGitDir} + id2 := &source.GitIdentifier{Remote: repodir2, KeepGitDir: keepGitDir} + + g, err := gs.Resolve(ctx, id) + require.NoError(t, err) + + g2, err := gs.Resolve(ctx, id2) + require.NoError(t, err) + + key1, err := g.CacheKey(ctx) + require.NoError(t, err) + require.Equal(t, 40, len(key1)) + + key2, err := g2.CacheKey(ctx) + require.NoError(t, err) + require.Equal(t, 40, len(key2)) + + require.NotEqual(t, key1, key2) + + ref1, err := g.Snapshot(ctx) + require.NoError(t, err) + defer ref1.Release(context.TODO()) + + mount, err := ref1.Mount(ctx) + require.NoError(t, err) + + lm := snapshot.LocalMounter(mount) + dir, err := lm.Mount() + require.NoError(t, err) + defer lm.Unmount() + + ref2, err := g2.Snapshot(ctx) + require.NoError(t, err) + defer ref2.Release(context.TODO()) + + mount, err = ref2.Mount(ctx) + require.NoError(t, err) + + lm = snapshot.LocalMounter(mount) + dir2, err := lm.Mount() + require.NoError(t, err) + defer lm.Unmount() + + dt, err := ioutil.ReadFile(filepath.Join(dir, "def")) + require.NoError(t, err) + + require.Equal(t, "bar\n", string(dt)) + + dt, err = ioutil.ReadFile(filepath.Join(dir2, "xyz")) + require.NoError(t, err) + + require.Equal(t, "xyz\n", string(dt)) +} + func setupGitSource(t *testing.T, tmpdir string) source.Source { snapshotter, err := naive.NewSnapshotter(filepath.Join(tmpdir, "snapshots")) assert.NoError(t, err) diff --git a/source/gitidentifier.go b/source/gitidentifier.go index 940df84d8..aecd06b38 100644 --- a/source/gitidentifier.go +++ b/source/gitidentifier.go @@ -9,9 +9,10 @@ import ( ) type GitIdentifier struct { - Remote string - Ref string - Subdir string + Remote string + Ref string + Subdir string + KeepGitDir bool } func NewGitIdentifier(remoteURL string) (*GitIdentifier, error) {