From ce6a428195d5c23f140bbbff4abefdfece0da367 Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Sun, 23 Jul 2017 10:33:19 -0700 Subject: [PATCH] rework llb package Signed-off-by: Tonis Tiigi --- client/llb/exec.go | 227 ++++++++++++++++++++++ client/llb/llb.go | 234 ----------------------- client/llb/meta.go | 141 +++++++------- client/llb/meta_test.go | 61 ------ client/llb/source.go | 121 ++++++++++++ client/llb/state.go | 340 ++++++++++++++------------------- examples/buildkit0/buildkit.go | 10 +- examples/buildkit1/buildkit.go | 16 +- examples/buildkit2/buildkit.go | 20 +- examples/buildkit3/buildkit.go | 20 +- 10 files changed, 589 insertions(+), 601 deletions(-) create mode 100644 client/llb/exec.go delete mode 100644 client/llb/llb.go delete mode 100644 client/llb/meta_test.go create mode 100644 client/llb/source.go diff --git a/client/llb/exec.go b/client/llb/exec.go new file mode 100644 index 000000000..7af4082f8 --- /dev/null +++ b/client/llb/exec.go @@ -0,0 +1,227 @@ +package llb + +import ( + _ "crypto/sha256" + "sort" + + "github.com/moby/buildkit/solver/pb" + "github.com/pkg/errors" +) + +type Meta struct { + Args []string + Env EnvList + Cwd string +} + +func NewExecOp(root Output, meta Meta) *ExecOp { + e := &ExecOp{meta: meta} + rootMount := &mount{ + target: pb.RootMount, + source: root, + } + e.mounts = append(e.mounts, rootMount) + e.root = &output{vertex: e, getIndex: e.getMountIndexFn(rootMount)} + rootMount.output = e.root + + return e +} + +type mount struct { + target string + readonly bool + source Output + output Output + // hasOutput bool +} + +type ExecOp struct { + root Output + mounts []*mount + meta Meta +} + +func (e *ExecOp) AddMount(target string, source Output, opt ...MountOption) Output { + m := &mount{ + target: target, + source: source, + } + for _, o := range opt { + o(m) + } + e.mounts = append(e.mounts, m) + m.output = &output{vertex: e, getIndex: e.getMountIndexFn(m)} + return m.output +} + +func (e *ExecOp) GetMount(target string) Output { + for _, m := range e.mounts { + if m.target == target { + return m.output + } + } + return nil +} + +func (e *ExecOp) Validate() error { + if len(e.meta.Args) == 0 { + return errors.Errorf("arguments are required") + } + if e.meta.Cwd == "" { + return errors.Errorf("working directory is required") + } + for _, m := range e.mounts { + if m.source != nil { + if err := m.source.Vertex().Validate(); err != nil { + return nil + } + } + } + return nil +} + +func (e *ExecOp) Marshal() ([]byte, error) { + if err := e.Validate(); err != nil { + return nil, err + } + // make sure mounts are sorted + sort.Slice(e.mounts, func(i, j int) bool { + return e.mounts[i].target < e.mounts[j].target + }) + + peo := &pb.ExecOp{ + Meta: &pb.Meta{ + Args: e.meta.Args, + Env: e.meta.Env.ToArray(), + Cwd: e.meta.Cwd, + }, + } + + pop := &pb.Op{ + Op: &pb.Op_Exec{ + Exec: peo, + }, + } + + for i, m := range e.mounts { + inputIndex := pb.InputIndex(len(pop.Inputs)) + if m.source != nil { + inp, err := m.source.ToInput() + if err != nil { + return nil, err + } + + newInput := true + + for i, inp2 := range pop.Inputs { + if *inp == *inp2 { + inputIndex = pb.InputIndex(i) + newInput = false + break + } + } + + if newInput { + pop.Inputs = append(pop.Inputs, inp) + } + } else { + inputIndex = 0 + } + + pm := &pb.Mount{ + Input: inputIndex, + Dest: m.target, + Readonly: m.readonly, + Output: pb.OutputIndex(i), + } + peo.Mounts = append(peo.Mounts, pm) + } + + return pop.Marshal() +} + +func (e *ExecOp) Output() Output { + return e.root +} + +func (e *ExecOp) Inputs() (inputs []Output) { + mm := map[Output]struct{}{} + for _, m := range e.mounts { + if m.source != nil { + mm[m.source] = struct{}{} + } + } + for o := range mm { + inputs = append(inputs, o) + } + return +} + +func (e *ExecOp) getMountIndexFn(m *mount) func() (pb.OutputIndex, error) { + return func() (pb.OutputIndex, error) { + // make sure mounts are sorted + sort.Slice(e.mounts, func(i, j int) bool { + return e.mounts[i].target < e.mounts[j].target + }) + + for i, m2 := range e.mounts { + if m == m2 { + return pb.OutputIndex(i), nil + } + } + return pb.OutputIndex(0), errors.Errorf("invalid mount") + } +} + +type ExecState struct { + State + exec *ExecOp +} + +func (e ExecState) AddMount(target string, source State, opt ...MountOption) State { + return NewState(e.exec.AddMount(target, source.Output(), opt...)) +} + +func (e ExecState) GetMount(target string) State { + return NewState(e.exec.GetMount(target)) +} + +func (e ExecState) Root() State { + return e.State +} + +type MountOption func(*mount) + +func Readonly(m *mount) { + m.readonly = true +} + +type RunOption func(es ExecInfo) ExecInfo + +func Shlex(str string) RunOption { + return Shlexf(str) +} +func Shlexf(str string, v ...interface{}) RunOption { + return func(ei ExecInfo) ExecInfo { + ei.State = shlexf(str, v...)(ei.State) + return ei + } +} + +func AddMount(dest string, mountState *State, opts ...MountOption) RunOption { + return func(ei ExecInfo) ExecInfo { + ei.Mounts = append(ei.Mounts, MountInfo{dest, mountState.Output(), opts}) + return ei + } +} + +type ExecInfo struct { + State State + Mounts []MountInfo +} + +type MountInfo struct { + Target string + Source Output + Opts []MountOption +} diff --git a/client/llb/llb.go b/client/llb/llb.go deleted file mode 100644 index 50696c792..000000000 --- a/client/llb/llb.go +++ /dev/null @@ -1,234 +0,0 @@ -package llb - -import ( - _ "crypto/sha256" - "sort" - - "github.com/gogo/protobuf/proto" - "github.com/moby/buildkit/solver/pb" - digest "github.com/opencontainers/go-digest" - "github.com/pkg/errors" -) - -var errNotFound = errors.Errorf("not found") - -func Source(id string) *State { - return &State{ - metaNext: NewMeta(), - source: &source{id: id, attrs: map[string]string{}}, - } -} - -type source struct { - id string - attrs map[string]string - scratch bool -} - -func (so *source) Validate() error { - // TODO: basic identifier validation - if so.id == "" { - return errors.Errorf("source identifier can't be empty") - } - return nil -} - -func (so *source) marshalTo(list [][]byte, cache map[digest.Digest]struct{}) (digest.Digest, [][]byte, error) { - if so.scratch { - return "", list, nil - } - if err := so.Validate(); err != nil { - return "", nil, err - } - po := &pb.Op{ - Op: &pb.Op_Source{ - Source: &pb.SourceOp{Identifier: so.id, Attrs: so.attrs}, - }, - } - return appendResult(po, list, cache) -} - -func Image(ref string) *State { - return Source("docker-image://" + ref) // controversial -} - -func Git(remote, ref string, opts ...GitOption) *State { - id := remote - if ref != "" { - id += "#" + ref - } - state := Source("git://" + id) - for _, opt := range opts { - opt(state.source) - } - return state -} - -type GitOption func(*source) - -func KeepGitDir() GitOption { - return func(s *source) { - s.attrs[pb.AttrKeepGitDir] = "true" - } -} - -func Scratch() *State { - s := Source("scratch") - s.source.scratch = true - return s -} - -func Local(name string) *State { - return Source("local://" + name) -} - -type LocalOption func(*source) - -func SessionID(id string) LocalOption { - return func(s *source) { - s.attrs[pb.AttrLocalSessionID] = id - } -} - -type exec struct { - meta Meta - mounts []*mount - root *mount -} - -func (eo *exec) Validate() error { - for _, m := range eo.mounts { - if m.source != nil { - if err := m.source.Validate(); err != nil { - return err - } - } - if m.parent != nil { - if err := m.parent.execState.exec.Validate(); err != nil { - return err - } - } - } - // TODO: validate meta - return nil -} - -func (eo *exec) marshalTo(list [][]byte, cache map[digest.Digest]struct{}) (digest.Digest, [][]byte, error) { - peo := &pb.ExecOp{ - Meta: &pb.Meta{ - Args: eo.meta.args, - Env: eo.meta.env.ToArray(), - Cwd: eo.meta.cwd, - }, - } - - pop := &pb.Op{ - Op: &pb.Op_Exec{ - Exec: peo, - }, - } - - sort.Slice(eo.mounts, func(i, j int) bool { - return eo.mounts[i].dest < eo.mounts[j].dest - }) - - var outputIndex pb.OutputIndex - - for _, m := range eo.mounts { - var dgst digest.Digest - var err error - if m.source != nil { - dgst, list, err = m.source.marshalTo(list, cache) - } else { - dgst, list, err = m.parent.execState.exec.marshalTo(list, cache) - } - if err != nil { - return "", list, err - } - - var mountIndex pb.OutputIndex - if m.parent != nil { - mountIndex = m.parent.outputIndex - } - - inputIndex := pb.InputIndex(len(pop.Inputs)) - for i, inp := range pop.Inputs { - if inp.Digest == dgst && inp.Index == mountIndex { - inputIndex = pb.InputIndex(i) - break - } - } - if dgst == "" { - inputIndex = pb.Empty - } - if inputIndex == pb.InputIndex(len(pop.Inputs)) { - pop.Inputs = append(pop.Inputs, &pb.Input{ - Digest: dgst, - Index: mountIndex, - }) - } - - pm := &pb.Mount{ - Input: inputIndex, - Dest: m.dest, - Readonly: m.readonly, - } - if m.hasOutput { - pm.Output = outputIndex - outputIndex++ - } else { - pm.Output = pb.SkipOutput - } - m.outputIndex = pm.Output - peo.Mounts = append(peo.Mounts, pm) - } - - return appendResult(pop, list, cache) -} - -type mount struct { - execState *ExecState - dest string - readonly bool - // either parent or source has to be set - parent *mount - source *source - hasOutput bool // TODO: remove - outputIndex pb.OutputIndex // filled in after marshal -} - -func (m *mount) marshalTo(list [][]byte, cache map[digest.Digest]struct{}) (digest.Digest, [][]byte, error) { - if m.execState == nil { - return "", nil, errors.Errorf("invalid mount") - } - var dgst digest.Digest - dgst, list, err := m.execState.exec.marshalTo(list, cache) - if err != nil { - return "", list, err - } - for _, m2 := range m.execState.exec.mounts { - if m2 == m { - po := &pb.Op{} - po.Inputs = append(po.Inputs, &pb.Input{ - Digest: dgst, - Index: m.outputIndex, - }) - return appendResult(po, list, cache) - } - } - return "", nil, errors.Errorf("invalid mount") -} - -func appendResult(p proto.Marshaler, list [][]byte, cache map[digest.Digest]struct{}) (dgst digest.Digest, out [][]byte, err error) { - dt, err := p.Marshal() - if err != nil { - return "", nil, err - } - dgst = digest.FromBytes(dt) - if _, ok := cache[dgst]; ok { - return dgst, list, nil - } - list = append(list, dt) - cache[dgst] = struct{}{} - return dgst, list, nil -} diff --git a/client/llb/meta.go b/client/llb/meta.go index fdd3f60d1..34fb6d8fe 100644 --- a/client/llb/meta.go +++ b/client/llb/meta.go @@ -4,126 +4,121 @@ import ( "fmt" "github.com/google/shlex" - "github.com/moby/buildkit/util/system" ) -func NewMeta(arg ...string) Meta { - m := Meta{} - m, _ = addEnv("PATH", system.DefaultPathEnv)(m) - m, _ = args(arg...)(m) - m, _ = dir("/")(m) - return m -} +type contextKeyT string -type Meta struct { - args []string - env envList - cwd string -} +var ( + keyArgs = contextKeyT("llb.exec.args") + keyDir = contextKeyT("llb.exec.dir") + keyEnv = contextKeyT("llb.exec.env") +) -type metaOption func(Meta) (Meta, error) - -func addEnv(key, value string) metaOption { +func addEnv(key, value string) StateOption { return addEnvf(key, value) } -func addEnvf(key, value string, v ...interface{}) metaOption { - return func(m Meta) (Meta, error) { - m.env = m.env.AddOrReplace(key, fmt.Sprintf(value, v...)) - return m, nil +func addEnvf(key, value string, v ...interface{}) StateOption { + return func(s State) State { + return s.WithValue(keyEnv, getEnv(s).AddOrReplace(key, fmt.Sprintf(value, v...))) + } +} +func clearEnv() StateOption { + return func(s State) State { + return s.WithValue(keyEnv, EnvList{}) + } +} +func delEnv(key string) StateOption { + return func(s State) State { + return s.WithValue(keyEnv, getEnv(s).Delete(key)) } } -func clearEnv() metaOption { - return func(m Meta) (Meta, error) { - m.env = NewMeta().env - return m, nil - } -} - -func delEnv(key string) metaOption { - return func(m Meta) (Meta, error) { - m.env = m.env.Delete(key) - return m, nil - } -} - -func args(args ...string) metaOption { - return func(m Meta) (Meta, error) { - m.args = args - return m, nil - } -} - -func dir(str string) metaOption { +func dir(str string) StateOption { return dirf(str) } -func dirf(str string, v ...interface{}) metaOption { - return func(m Meta) (Meta, error) { - m.cwd = fmt.Sprintf(str, v...) - return m, nil +func dirf(str string, v ...interface{}) StateOption { + return func(s State) State { + return s.WithValue(keyDir, fmt.Sprintf(str, v...)) } } -func reset(s *State) metaOption { - return func(m Meta) (Meta, error) { - if s == nil { - return NewMeta(), nil - } - return s.metaNext, nil +func reset(s_ State) StateOption { + return func(s State) State { + s = NewState(s.Output()) + s.ctx = s_.ctx + return s } } -func (m Meta) Env(key string) (string, bool) { - return m.env.Get(key) +func getEnv(s State) EnvList { + v := s.Value(keyEnv) + if v != nil { + return v.(EnvList) + } + return EnvList{} } -func (m Meta) Dir() string { - return m.cwd +func getDir(s State) string { + v := s.Value(keyDir) + if v != nil { + return v.(string) + } + return "" } -func (m Meta) Args() []string { - return append([]string{}, m.args...) +func getArgs(s State) []string { + v := s.Value(keyArgs) + if v != nil { + return v.([]string) + } + return nil } -func shlexf(str string, v ...interface{}) metaOption { - return func(m Meta) (Meta, error) { +func args(args ...string) StateOption { + return func(s State) State { + return s.WithValue(keyArgs, args) + } +} + +func shlexf(str string, v ...interface{}) StateOption { + return func(s State) State { arg, err := shlex.Split(fmt.Sprintf(str, v...)) if err != nil { - return m, err + // TODO: handle error } - return args(arg...)(m) + return args(arg...)(s) } } -type envList []keyValue +type EnvList []KeyValue -type keyValue struct { +type KeyValue struct { key string value string } -func (e envList) AddOrReplace(k, v string) envList { +func (e EnvList) AddOrReplace(k, v string) EnvList { e = e.Delete(k) - e = append(e, keyValue{key: k, value: v}) + e = append(e, KeyValue{key: k, value: v}) return e } -func (e envList) Delete(k string) envList { - e = append([]keyValue(nil), e...) - if i, ok := e.index(k); ok { +func (e EnvList) Delete(k string) EnvList { + e = append([]KeyValue(nil), e...) + if i, ok := e.Index(k); ok { return append(e[:i], e[i+1:]...) } return e } -func (e envList) Get(k string) (string, bool) { - if index, ok := e.index(k); ok { +func (e EnvList) Get(k string) (string, bool) { + if index, ok := e.Index(k); ok { return e[index].value, true } return "", false } -func (e envList) index(k string) (int, bool) { +func (e EnvList) Index(k string) (int, bool) { for i, kv := range e { if kv.key == k { return i, true @@ -132,7 +127,7 @@ func (e envList) index(k string) (int, bool) { return -1, false } -func (e envList) ToArray() []string { +func (e EnvList) ToArray() []string { out := make([]string, 0, len(e)) for _, kv := range e { out = append(out, kv.key+"="+kv.value) diff --git a/client/llb/meta_test.go b/client/llb/meta_test.go deleted file mode 100644 index 3b1ff23d4..000000000 --- a/client/llb/meta_test.go +++ /dev/null @@ -1,61 +0,0 @@ -package llb - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestDefaultMeta(t *testing.T) { - m := NewMeta() - path, ok := m.Env("PATH") - assert.True(t, ok) - assert.NotEmpty(t, path) - - cwd := m.Dir() - assert.NotEmpty(t, cwd) -} - -func TestReset(t *testing.T) { - m := NewMeta() - wd := m.Dir() - path, _ := m.Env("PATH") - m, _ = dir("/foo")(m) - m, _ = addEnv("FOO", "bar")(m) - - m, _ = reset(nil)(m) - assert.Equal(t, wd, m.Dir()) - path2, _ := m.Env("PATH") - assert.Equal(t, path, path2) -} - -func TestEnv(t *testing.T) { - m := NewMeta() - m, _ = addEnv("FOO", "bar")(m) - m2, _ := addEnv("FOO", "baz")(m) - m2, _ = addEnv("BAR", "abc")(m2) - - v, ok := m.Env("FOO") - assert.True(t, ok) - assert.Equal(t, "bar", v) - - _, ok = m.Env("BAR") - assert.False(t, ok) - - v, ok = m2.Env("FOO") - assert.True(t, ok) - assert.Equal(t, "baz", v) - - v, ok = m2.Env("BAR") - assert.True(t, ok) - assert.Equal(t, "abc", v) -} - -func TestShlex(t *testing.T) { - m, err := shlexf("echo foo")(Meta{}) - assert.Nil(t, err) - assert.Equal(t, []string{"echo", "foo"}, m.Args()) - - _, err = shlexf("echo \"foo")(Meta{}) - assert.Error(t, err) -} diff --git a/client/llb/source.go b/client/llb/source.go new file mode 100644 index 000000000..973737142 --- /dev/null +++ b/client/llb/source.go @@ -0,0 +1,121 @@ +package llb + +import ( + _ "crypto/sha256" + + "github.com/moby/buildkit/solver/pb" + "github.com/pkg/errors" +) + +type SourceOp struct { + id string + attrs map[string]string + output Output +} + +func NewSource(id string, attrs map[string]string) *SourceOp { + s := &SourceOp{ + id: id, + attrs: attrs, + } + s.output = &output{vertex: s} + return s +} + +func (s *SourceOp) Validate() error { + if s.id == "" { + return errors.Errorf("source identifier can't be empty") + } + return nil +} + +func (s *SourceOp) Marshal() ([]byte, error) { + if err := s.Validate(); err != nil { + return nil, err + } + + proto := &pb.Op{ + Op: &pb.Op_Source{ + Source: &pb.SourceOp{Identifier: s.id, Attrs: s.attrs}, + }, + } + return proto.Marshal() +} + +func (s *SourceOp) Output() Output { + return s.output +} + +func (s *SourceOp) Inputs() []Output { + return nil +} + +func Source(id string) State { + return NewState(NewSource(id, nil).Output()) +} + +func Image(ref string) State { + return Source("docker-image://" + ref) // controversial +} + +func Git(remote, ref string, opts ...GitOption) State { + id := remote + if ref != "" { + id += "#" + ref + } + + gi := &GitInfo{} + for _, o := range opts { + o(gi) + } + attrs := map[string]string{} + if gi.KeepGitDir { + attrs[pb.AttrKeepGitDir] = "true" + } + + source := NewSource("git://"+id, attrs) + return NewState(source.Output()) +} + +type GitOption func(*GitInfo) + +type GitInfo struct { + KeepGitDir bool +} + +func KeepGitDir() GitOption { + return func(gi *GitInfo) { + gi.KeepGitDir = true + } +} + +func Scratch() State { + return NewState(nil) +} + +func Local(name string, opts ...LocalOption) State { + gi := &LocalInfo{} + + for _, o := range opts { + o(gi) + } + attrs := map[string]string{} + if gi.SessionID != "" { + attrs[pb.AttrLocalSessionID] = gi.SessionID + } + + source := NewSource("local://"+name, attrs) + return NewState(source.Output()) +} + +type LocalOption func(*LocalInfo) + +func SessionID(id string) LocalOption { + return func(li *LocalInfo) { + li.SessionID = id + } +} + +type LocalInfo struct { + SessionID string +} diff --git a/client/llb/state.go b/client/llb/state.go index 4dc557a0e..d956b14f4 100644 --- a/client/llb/state.go +++ b/client/llb/state.go @@ -1,249 +1,189 @@ package llb import ( - _ "crypto/sha256" + "context" "github.com/moby/buildkit/solver/pb" + "github.com/moby/buildkit/util/system" digest "github.com/opencontainers/go-digest" - "github.com/pkg/errors" ) -type StateOption func(s *State) *State +type StateOption func(State) State + +type Output interface { + ToInput() (*pb.Input, error) + Vertex() Vertex +} + +type Vertex interface { + Validate() error + Marshal() ([]byte, error) + Output() Output + Inputs() []Output +} + +func NewState(o Output) State { + s := State{ + out: o, + ctx: context.Background(), + } + s = dir("/")(s) + s = addEnv("PATH", system.DefaultPathEnv)(s) + return s +} -// State represents modifiable llb state type State struct { - source *source - exec *exec - meta Meta - mount *mount - metaNext Meta // this meta will be used for the next Run() - err error + out Output + ctx context.Context } -// TODO: add state.Reset() state.Save() state.Restore() +func (s State) WithValue(k, v interface{}) State { + return State{ + out: s.out, + ctx: context.WithValue(s.ctx, k, v), + } +} -// Validate checks that every node has been set up properly -func (s *State) Validate() error { - if s.source != nil { - if err := s.source.Validate(); err != nil { - return err +func (s State) Value(k interface{}) interface{} { + return s.ctx.Value(k) +} + +func (s State) Marshal() ([][]byte, error) { + list, err := marshal(s.Output().Vertex(), nil, map[digest.Digest]struct{}{}) + if err != nil { + return nil, err + } + inp, err := s.Output().ToInput() + if err != nil { + return nil, err + } + proto := &pb.Op{Inputs: []*pb.Input{inp}} + dt, err := proto.Marshal() + if err != nil { + return nil, err + } + list = append(list, dt) + return list, nil +} + +func marshal(v Vertex, list [][]byte, cache map[digest.Digest]struct{}) (out [][]byte, err error) { + for _, inp := range v.Inputs() { + var err error + list, err = marshal(inp.Vertex(), list, cache) + if err != nil { + return nil, err } } - if s.exec != nil { - if err := s.exec.Validate(); err != nil { - return err - } + dt, err := v.Marshal() + if err != nil { + return nil, err } - return nil + dgst := digest.FromBytes(dt) + if _, ok := cache[dgst]; ok { + return list, nil + } + list = append(list, dt) + cache[dgst] = struct{}{} + return list, nil } -func (s *State) Run(opts ...RunOption) *ExecState { - var es ExecState - meta := s.metaNext - - exec := &exec{ - meta: meta, - mounts: []*mount{}, - root: &mount{ - dest: pb.RootMount, - source: s.source, - parent: s.mount, - hasOutput: true, - }, - } - exec.root.execState = &es - exec.mounts = append(exec.mounts, exec.root) - - es.exec = exec - es.mount = exec.root - es.meta = meta - es.metaNext = es.meta - var err error - for _, o := range opts { - es = *o(&es) - } - es.exec.meta = es.meta - es.err = err - return &es +func (s State) Validate() error { + return s.Output().Vertex().Validate() } -func (s *State) AddEnv(key, value string) *State { +func (s State) Output() Output { + return s.out +} + +func (s State) WithOutput(o Output) State { + return State{ + out: o, + ctx: s.ctx, + } +} + +func (s State) Run(ro ...RunOption) ExecState { + ei := ExecInfo{State: s} + for _, o := range ro { + ei = o(ei) + } + meta := Meta{ + Args: getArgs(ei.State), + Cwd: getDir(ei.State), + Env: getEnv(ei.State), + } + + exec := NewExecOp(s.Output(), meta) + for _, m := range ei.Mounts { + exec.AddMount(m.Target, m.Source, m.Opts...) + } + + return ExecState{ + State: s.WithOutput(exec.Output()), + exec: exec, + } +} + +func (s State) AddEnv(key, value string) State { return s.AddEnvf(key, value) } -func (s *State) AddEnvf(key, value string, v ...interface{}) *State { - s.metaNext, _ = addEnvf(key, value, v...)(s.metaNext) - return s + +func (s State) AddEnvf(key, value string, v ...interface{}) State { + return addEnvf(key, value, v...)(s) } -func (s *State) DelEnv(key string) *State { - s.metaNext, _ = delEnv(key)(s.metaNext) - return s -} -func (s *State) ClearEnv() *State { - s.metaNext, _ = clearEnv()(s.metaNext) - return s -} -func (s *State) GetEnv(key string) (string, bool) { - return s.metaNext.Env(key) -} - -func (s *State) Dir(str string) *State { +func (s State) Dir(str string) State { return s.Dirf(str) } -func (s *State) Dirf(str string, v ...interface{}) *State { - s.metaNext, _ = dirf(str, v...)(s.metaNext) - return s +func (s State) Dirf(str string, v ...interface{}) State { + return dirf(str, v...)(s) } -func (s *State) GetDir() string { - return s.metaNext.Dir() +func (s State) GetEnv(key string) (string, bool) { + return getEnv(s).Get(key) } -func (s *State) Args(arg ...string) *State { - s.metaNext, _ = args(arg...)(s.metaNext) - return s +func (s State) GetDir() string { + return getDir(s) } -func (s *State) Reset(src *State) *State { - copy := *s - copy.metaNext, _ = reset(src)(s.metaNext) - return © +func (s State) GetArgs() []string { + return getArgs(s) } -func (s *State) With(so ...StateOption) *State { +func (s State) Reset(s2 State) State { + return reset(s2)(s) +} + +func (s State) With(so ...StateOption) State { for _, o := range so { s = o(s) } return s } -func (s *State) Marshal() (list [][]byte, err error) { - if err := s.Validate(); err != nil { - return nil, err - } - cache := make(map[digest.Digest]struct{}) - if s.source != nil { - _, list, err = s.source.marshalTo(nil, cache) - } else if s.exec != nil { - _, list, err = s.exec.root.marshalTo(nil, cache) - } else { - _, list, err = s.mount.marshalTo(nil, cache) - } - return +type output struct { + vertex Vertex + getIndex func() (pb.OutputIndex, error) } -// ExecState is a state with a active leaf pointing to a run exec command. -// Mounts can be added only to this state. -type ExecState struct { - State -} - -func (s *ExecState) AddMount(dest string, mountState *State, opts ...MountOption) *State { - m := &mount{ - dest: dest, - source: mountState.source, - parent: mountState.mount, - execState: s, - hasOutput: true, // TODO: should be set only if something inherits - } - for _, opt := range opts { - opt(m) - } - var newState State - newState.meta = s.meta - newState.metaNext = s.metaNext - newState.mount = m - s.exec.mounts = append(s.exec.mounts, m) - return &newState -} - -func (s *ExecState) Root() *State { - return &s.State -} - -func (s *ExecState) GetMount(target string) (*State, error) { - for _, m := range s.exec.mounts { - if m.dest == target { - var newState State - newState.meta = m.execState.meta - newState.metaNext = m.execState.metaNext - newState.mount = m - return &newState, nil +func (o *output) ToInput() (*pb.Input, error) { + var index pb.OutputIndex + if o.getIndex != nil { + var err error + index, err = o.getIndex() + if err != nil { + return nil, err } } - return nil, errors.WithStack(errNotFound) -} - -func (s *ExecState) updateMeta(fn metaOption) *ExecState { - meta, err := fn(s.meta) - s.meta = meta + dt, err := o.vertex.Marshal() if err != nil { - s.err = err + return nil, err } - return s + dgst := digest.FromBytes(dt) + return &pb.Input{Digest: dgst, Index: index}, nil } -type RunOption func(es *ExecState) *ExecState - -type MountOption func(*mount) - -func Readonly(m *mount) { - m.readonly = true -} - -func AddMount(dest string, mountState *State, opts ...MountOption) RunOption { - return func(es *ExecState) *ExecState { - es.AddMount(dest, mountState, opts...) - return nil - } -} - -func Shlex(str string) RunOption { - return Shlexf(str) -} -func Shlexf(str string, v ...interface{}) RunOption { - return func(es *ExecState) *ExecState { - return es.updateMeta(shlexf(str, v...)) - } -} - -func AddEnv(key, value string) RunOption { - return AddEnvf(key, value) -} -func AddEnvf(key, value string, v ...interface{}) RunOption { - return func(es *ExecState) *ExecState { - return es.updateMeta(addEnvf(key, value, v...)) - } -} - -func DelEnv(key string) RunOption { - return func(es *ExecState) *ExecState { - return es.updateMeta(delEnv(key)) - } -} -func ClearEnv() RunOption { - return func(es *ExecState) *ExecState { - return es.updateMeta(clearEnv()) - } -} - -func Dir(str string) RunOption { - return Dirf(str) -} -func Dirf(str string, v ...interface{}) RunOption { - return func(es *ExecState) *ExecState { - return es.updateMeta(dirf(str, v...)) - } -} - -func Args(arg ...string) RunOption { - return func(es *ExecState) *ExecState { - return es.updateMeta(args(arg...)) - } -} - -func Reset(src *State) RunOption { - return func(es *ExecState) *ExecState { - return es.updateMeta(reset(src)) - } +func (o *output) Vertex() Vertex { + return o.vertex } diff --git a/examples/buildkit0/buildkit.go b/examples/buildkit0/buildkit.go index 21100636e..960de07fa 100644 --- a/examples/buildkit0/buildkit.go +++ b/examples/buildkit0/buildkit.go @@ -31,7 +31,7 @@ func main() { llb.WriteTo(dt, os.Stdout) } -func goBuildBase() *llb.State { +func goBuildBase() llb.State { goAlpine := llb.Image("docker.io/library/golang:1.8-alpine") return goAlpine. AddEnv("PATH", "/usr/local/go/bin:"+system.DefaultPathEnv). @@ -40,7 +40,7 @@ func goBuildBase() *llb.State { Run(llb.Shlex("apk add --no-cache git make")).Root() } -func runc(version string) *llb.State { +func runc(version string) llb.State { return goBuildBase(). Run(llb.Shlex("git clone https://github.com/opencontainers/runc.git /go/src/github.com/opencontainers/runc")). Dir("/go/src/github.com/opencontainers/runc"). @@ -48,7 +48,7 @@ func runc(version string) *llb.State { Run(llb.Shlex("go build -o /usr/bin/runc ./")).Root() } -func containerd(version string) *llb.State { +func containerd(version string) llb.State { return goBuildBase(). Run(llb.Shlex("apk add --no-cache btrfs-progs-dev")). Run(llb.Shlex("git clone https://github.com/containerd/containerd.git /go/src/github.com/containerd/containerd")). @@ -57,7 +57,7 @@ func containerd(version string) *llb.State { Run(llb.Shlex("make bin/containerd")).Root() } -func buildkit(opt buildOpt) *llb.State { +func buildkit(opt buildOpt) llb.State { src := goBuildBase(). Run(llb.Shlex("git clone https://github.com/moby/buildkit.git /go/src/github.com/moby/buildkit")). Dir("/go/src/github.com/moby/buildkit") @@ -83,7 +83,7 @@ func buildkit(opt buildOpt) *llb.State { return r } -func copy(src *llb.State, srcPath string, dest *llb.State, destPath string) *llb.State { +func copy(src llb.State, srcPath string, dest llb.State, destPath string) llb.State { cpImage := llb.Image("docker.io/library/alpine:latest") cp := cpImage.Run(llb.Shlexf("cp -a /src%s /dest%s", srcPath, destPath)) cp.AddMount("/src", src) diff --git a/examples/buildkit1/buildkit.go b/examples/buildkit1/buildkit.go index 69d15adb1..259ae8c31 100644 --- a/examples/buildkit1/buildkit.go +++ b/examples/buildkit1/buildkit.go @@ -31,7 +31,7 @@ func main() { llb.WriteTo(dt, os.Stdout) } -func goBuildBase() *llb.State { +func goBuildBase() llb.State { goAlpine := llb.Image("docker.io/library/golang:1.8-alpine") return goAlpine. AddEnv("PATH", "/usr/local/go/bin:"+system.DefaultPathEnv). @@ -40,21 +40,21 @@ func goBuildBase() *llb.State { Run(llb.Shlex("apk add --no-cache git make")).Root() } -func runc(version string) *llb.State { +func runc(version string) llb.State { return goBuildBase(). With(goFromGit("github.com/opencontainers/runc", version)). Run(llb.Shlex("go build -o /usr/bin/runc ./")). Root() } -func containerd(version string) *llb.State { +func containerd(version string) llb.State { return goBuildBase(). Run(llb.Shlex("apk add --no-cache btrfs-progs-dev")). With(goFromGit("github.com/containerd/containerd", version)). Run(llb.Shlex("make bin/containerd")).Root() } -func buildkit(opt buildOpt) *llb.State { +func buildkit(opt buildOpt) llb.State { src := goBuildBase().With(goFromGit("github.com/moby/buildkit", "master")) builddStandalone := src. @@ -87,20 +87,20 @@ func goFromGit(repo, tag string) llb.StateOption { Run(llb.Shlexf("git clone https://%[1]s.git /go/src/%[1]s", repo)). Dirf("/go/src/%s", repo). Run(llb.Shlexf("git checkout -q %s", tag)).Root() - return func(s *llb.State) *llb.State { + return func(s llb.State) llb.State { return s.With(copyFrom(src, "/go", "/")).Reset(s).Dir(src.GetDir()) } } // copyFrom has similar semantics as `COPY --from` -func copyFrom(src *llb.State, srcPath, destPath string) llb.StateOption { - return func(s *llb.State) *llb.State { +func copyFrom(src llb.State, srcPath, destPath string) llb.StateOption { + return func(s llb.State) llb.State { return copy(src, srcPath, s, destPath) } } // copy copies files between 2 states using cp until there is no copyOp -func copy(src *llb.State, srcPath string, dest *llb.State, destPath string) *llb.State { +func copy(src llb.State, srcPath string, dest llb.State, destPath string) llb.State { cpImage := llb.Image("docker.io/library/alpine:latest") cp := cpImage.Run(llb.Shlexf("cp -a /src%s /dest%s", srcPath, destPath)) cp.AddMount("/src", src) diff --git a/examples/buildkit2/buildkit.go b/examples/buildkit2/buildkit.go index 17ba6fe52..c7edef9d4 100644 --- a/examples/buildkit2/buildkit.go +++ b/examples/buildkit2/buildkit.go @@ -31,7 +31,7 @@ func main() { llb.WriteTo(dt, os.Stdout) } -func goBuildBase() *llb.State { +func goBuildBase() llb.State { goAlpine := llb.Image("docker.io/library/golang:1.8-alpine") return goAlpine. AddEnv("PATH", "/usr/local/go/bin:"+system.DefaultPathEnv). @@ -39,22 +39,22 @@ func goBuildBase() *llb.State { Run(llb.Shlex("apk add --no-cache g++ linux-headers make")).Root() } -func goRepo(s *llb.State, repo, ref string, g ...llb.GitOption) func(ro ...llb.RunOption) *llb.State { +func goRepo(s llb.State, repo, ref string, g ...llb.GitOption) func(ro ...llb.RunOption) llb.State { dir := "/go/src/" + repo - return func(ro ...llb.RunOption) *llb.State { + return func(ro ...llb.RunOption) llb.State { es := s.Dir(dir).Run(ro...) es.AddMount(dir, llb.Git(repo, ref, g...)) return es.AddMount(dir+"/bin", llb.Scratch()) } } -func runc(version string) *llb.State { +func runc(version string) llb.State { return goRepo(goBuildBase(), "github.com/opencontainers/runc", version)( llb.Shlex("go build -o ./bin/runc ./"), ) } -func containerd(version string) *llb.State { +func containerd(version string) llb.State { return goRepo( goBuildBase(). Run(llb.Shlex("apk add --no-cache btrfs-progs-dev")).Root(), @@ -63,7 +63,7 @@ func containerd(version string) *llb.State { ) } -func buildkit(opt buildOpt) *llb.State { +func buildkit(opt buildOpt) llb.State { run := goRepo(goBuildBase(), "github.com/moby/buildkit", "master") builddStandalone := run(llb.Shlex("go build -o ./bin/buildd-standalone -tags standalone ./cmd/buildd")) @@ -85,19 +85,19 @@ func buildkit(opt buildOpt) *llb.State { return r.With(copyAll(builddStandalone, "/bin")) } -func copyAll(src *llb.State, destPath string) llb.StateOption { +func copyAll(src llb.State, destPath string) llb.StateOption { return copyFrom(src, "/.", destPath) } // copyFrom has similar semantics as `COPY --from` -func copyFrom(src *llb.State, srcPath, destPath string) llb.StateOption { - return func(s *llb.State) *llb.State { +func copyFrom(src llb.State, srcPath, destPath string) llb.StateOption { + return func(s llb.State) llb.State { return copy(src, srcPath, s, destPath) } } // copy copies files between 2 states using cp until there is no copyOp -func copy(src *llb.State, srcPath string, dest *llb.State, destPath string) *llb.State { +func copy(src llb.State, srcPath string, dest llb.State, destPath string) llb.State { cpImage := llb.Image("docker.io/library/alpine:latest") cp := cpImage.Run(llb.Shlexf("cp -a /src%s /dest%s", srcPath, destPath)) cp.AddMount("/src", src) diff --git a/examples/buildkit3/buildkit.go b/examples/buildkit3/buildkit.go index f23c4b003..d8163af8f 100644 --- a/examples/buildkit3/buildkit.go +++ b/examples/buildkit3/buildkit.go @@ -33,7 +33,7 @@ func main() { llb.WriteTo(dt, os.Stdout) } -func goBuildBase() *llb.State { +func goBuildBase() llb.State { goAlpine := llb.Image("docker.io/library/golang:1.8-alpine") return goAlpine. AddEnv("PATH", "/usr/local/go/bin:"+system.DefaultPathEnv). @@ -41,16 +41,16 @@ func goBuildBase() *llb.State { Run(llb.Shlex("apk add --no-cache g++ linux-headers make")).Root() } -func goRepo(s *llb.State, repo string, src *llb.State) func(ro ...llb.RunOption) *llb.State { +func goRepo(s llb.State, repo string, src llb.State) func(ro ...llb.RunOption) llb.State { dir := "/go/src/" + repo - return func(ro ...llb.RunOption) *llb.State { + return func(ro ...llb.RunOption) llb.State { es := s.Dir(dir).Run(ro...) es.AddMount(dir, src, llb.Readonly) return es.AddMount("/out", llb.Scratch()) } } -func runc(version string) *llb.State { +func runc(version string) llb.State { repo := "github.com/opencontainers/runc" src := llb.Git(repo, version) if version == "local" { @@ -61,7 +61,7 @@ func runc(version string) *llb.State { ) } -func containerd(version string) *llb.State { +func containerd(version string) llb.State { repo := "github.com/containerd/containerd" src := llb.Git(repo, version, llb.KeepGitDir()) if version == "local" { @@ -75,7 +75,7 @@ func containerd(version string) *llb.State { ) } -func buildkit(opt buildOpt) *llb.State { +func buildkit(opt buildOpt) llb.State { repo := "github.com/moby/buildkit" src := llb.Git(repo, "master") if opt.buildkit == "local" { @@ -102,19 +102,19 @@ func buildkit(opt buildOpt) *llb.State { return r.With(copyAll(builddStandalone, "/bin")) } -func copyAll(src *llb.State, destPath string) llb.StateOption { +func copyAll(src llb.State, destPath string) llb.StateOption { return copyFrom(src, "/.", destPath) } // copyFrom has similar semantics as `COPY --from` -func copyFrom(src *llb.State, srcPath, destPath string) llb.StateOption { - return func(s *llb.State) *llb.State { +func copyFrom(src llb.State, srcPath, destPath string) llb.StateOption { + return func(s llb.State) llb.State { return copy(src, srcPath, s, destPath) } } // copy copies files between 2 states using cp until there is no copyOp -func copy(src *llb.State, srcPath string, dest *llb.State, destPath string) *llb.State { +func copy(src llb.State, srcPath string, dest llb.State, destPath string) llb.State { cpImage := llb.Image("docker.io/library/alpine:latest") cp := cpImage.Run(llb.Shlexf("cp -a /src%s /dest%s", srcPath, destPath)) cp.AddMount("/src", src, llb.Readonly)