dilim/buildkit
1
0
Fork 0
mirror of https://github.com/moby/buildkit.git synced 2026-06-30 19:57:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

provenance: add layers support

Browse Source 
Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
This commit is contained in:
Tonis Tiigi
2022-09-25 23:02:38 -07:00
parent 915d7dcf7a
commit a20e48f36d
14 changed files with 217 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
cache/remotecache/v1/chains.go vendored
View File

@@ -146,7 +146,7 @@ func (c *item) removeLink(src *item) bool {
return found
}
func (c *item) AddResult(createdAt time.Time, result *solver.Remote) {
func (c *item) AddResult(_ digest.Digest, _ int, createdAt time.Time, result *solver.Remote) {
c.resultTime = createdAt
c.result = result
}
@@ -214,7 +214,7 @@ func (c *item) walkAllResults(fn func(i *item) error, visited map[*item]struct{}
type nopRecord struct {
}
func (c *nopRecord) AddResult(createdAt time.Time, result *solver.Remote) {
func (c *nopRecord) AddResult(_ digest.Digest, _ int, createdAt time.Time, result *solver.Remote) {
}
func (c *nopRecord) LinkFrom(rec solver.CacheExporterRecord, index int, selector string) {

2
cache/remotecache/v1/chains_test.go vendored
View File

@@ -29,7 +29,7 @@ func TestSimpleMarshal(t *testing.T) {
Digest: dgst("d1"),
}},
}
baz.AddResult(time.Now(), r0)
baz.AddResult("", 0, time.Now(), r0)
}
addRecords()

4
cache/remotecache/v1/parse.go vendored
View File

@@ -61,7 +61,7 @@ func parseRecord(cc CacheConfig, idx int, provider DescriptorProvider, t solver.
return nil, err
}
if remote != nil {
r.AddResult(res.CreatedAt, remote)
r.AddResult("", 0, res.CreatedAt, remote)
}
}
@@ -86,7 +86,7 @@ func parseRecord(cc CacheConfig, idx int, provider DescriptorProvider, t solver.
}
if remote != nil {
remote.Provider = mp
r.AddResult(res.CreatedAt, remote)
r.AddResult("", 0, res.CreatedAt, remote)
}
}

6
exporter/containerimage/writer.go
View File

@@ -519,7 +519,7 @@ func (ic *ImageWriter) commitDistributionManifest(ctx context.Context, opts *Ima
}
for i, desc := range remote.Descriptors {
removeInternalLayerAnnotations(&desc, opts.OCITypes)
RemoveInternalLayerAnnotations(&desc, opts.OCITypes)
mfst.Layers = append(mfst.Layers, desc)
labels[fmt.Sprintf("containerd.io/gc.ref.content.%d", i+1)] = desc.Digest.String()
}
@@ -631,7 +631,7 @@ func (ic *ImageWriter) commitAttestationsManifest(ctx context.Context, opts *Ima
"containerd.io/gc.ref.content.0": configDigest.String(),
}
for i, desc := range layers {
removeInternalLayerAnnotations(&desc, opts.OCITypes)
RemoveInternalLayerAnnotations(&desc, opts.OCITypes)
mfst.Layers = append(mfst.Layers, desc)
labels[fmt.Sprintf("containerd.io/gc.ref.content.%d", i+1)] = desc.Digest.String()
}
@@ -889,7 +889,7 @@ func normalizeLayersAndHistory(ctx context.Context, remote *solver.Remote, histo
return remote, history
}
func removeInternalLayerAnnotations(desc *ocispecs.Descriptor, oci bool) {
func RemoveInternalLayerAnnotations(desc *ocispecs.Descriptor, oci bool) {
if oci {
// oci supports annotations but don't export internal annotations
delete(desc.Annotations, "containerd.io/uncompressed")

30
solver/cache_test.go
View File

@@ -19,7 +19,7 @@ func depKeys(cks ...ExportableCacheKey) []CacheKeyWithSelector {
}
func testCacheKey(dgst digest.Digest, output Index, deps ...ExportableCacheKey) *CacheKey {
k := NewCacheKey(dgst, output)
k := NewCacheKey(dgst, "", output)
k.deps = make([][]CacheKeyWithSelector, len(deps))
for i, dep := range deps {
k.deps[i] = depKeys(dep)
@@ -28,7 +28,7 @@ func testCacheKey(dgst digest.Digest, output Index, deps ...ExportableCacheKey)
}
func testCacheKeyWithDeps(dgst digest.Digest, output Index, deps [][]CacheKeyWithSelector) *CacheKey {
k := NewCacheKey(dgst, output)
k := NewCacheKey(dgst, "", output)
k.deps = deps
return k
}
@@ -42,7 +42,7 @@ func TestInMemoryCache(t *testing.T) {
m := NewInMemoryCacheManager()
cacheFoo, err := m.Save(NewCacheKey(dgst("foo"), 0), testResult("result0"), time.Now())
cacheFoo, err := m.Save(NewCacheKey(dgst("foo"), "", 0), testResult("result0"), time.Now())
require.NoError(t, err)
keys, err := m.Query(nil, 0, dgst("foo"), 0)
@@ -58,7 +58,7 @@ func TestInMemoryCache(t *testing.T) {
require.Equal(t, "result0", unwrap(res))
// another record
cacheBar, err := m.Save(NewCacheKey(dgst("bar"), 0), testResult("result1"), time.Now())
cacheBar, err := m.Save(NewCacheKey(dgst("bar"), "", 0), testResult("result1"), time.Now())
require.NoError(t, err)
keys, err = m.Query(nil, 0, dgst("bar"), 0)
@@ -155,7 +155,7 @@ func TestInMemoryCacheSelector(t *testing.T) {
m := NewInMemoryCacheManager()
cacheFoo, err := m.Save(NewCacheKey(dgst("foo"), 0), testResult("result0"), time.Now())
cacheFoo, err := m.Save(NewCacheKey(dgst("foo"), "", 0), testResult("result0"), time.Now())
require.NoError(t, err)
_, err = m.Save(testCacheKeyWithDeps(dgst("bar"), 0, [][]CacheKeyWithSelector{
@@ -189,11 +189,11 @@ func TestInMemoryCacheSelectorNested(t *testing.T) {
m := NewInMemoryCacheManager()
cacheFoo, err := m.Save(NewCacheKey(dgst("foo"), 0), testResult("result0"), time.Now())
cacheFoo, err := m.Save(NewCacheKey(dgst("foo"), "", 0), testResult("result0"), time.Now())
require.NoError(t, err)
_, err = m.Save(testCacheKeyWithDeps(dgst("bar"), 0, [][]CacheKeyWithSelector{
{{CacheKey: *cacheFoo, Selector: dgst("sel0")}, {CacheKey: expKey(NewCacheKey(dgst("second"), 0))}},
{{CacheKey: *cacheFoo, Selector: dgst("sel0")}, {CacheKey: expKey(NewCacheKey(dgst("second"), "", 0))}},
}), testResult("result1"), time.Now())
require.NoError(t, err)
@@ -219,7 +219,7 @@ func TestInMemoryCacheSelectorNested(t *testing.T) {
require.NoError(t, err)
require.Equal(t, len(keys), 0)
keys, err = m.Query(depKeys(expKey(NewCacheKey(dgst("second"), 0))), 0, dgst("bar"), 0)
keys, err = m.Query(depKeys(expKey(NewCacheKey(dgst("second"), "", 0))), 0, dgst("bar"), 0)
require.NoError(t, err)
require.Equal(t, len(keys), 1)
@@ -231,7 +231,7 @@ func TestInMemoryCacheSelectorNested(t *testing.T) {
require.NoError(t, err)
require.Equal(t, "result1", unwrap(res))
keys, err = m.Query(depKeys(expKey(NewCacheKey(dgst("second"), 0))), 0, dgst("bar"), 0)
keys, err = m.Query(depKeys(expKey(NewCacheKey(dgst("second"), "", 0))), 0, dgst("bar"), 0)
require.NoError(t, err)
require.Equal(t, len(keys), 1)
}
@@ -242,7 +242,7 @@ func TestInMemoryCacheReleaseParent(t *testing.T) {
m := NewCacheManager(context.TODO(), identity.NewID(), storage, results)
res0 := testResult("result0")
cacheFoo, err := m.Save(NewCacheKey(dgst("foo"), 0), res0, time.Now())
cacheFoo, err := m.Save(NewCacheKey(dgst("foo"), "", 0), res0, time.Now())
require.NoError(t, err)
res1 := testResult("result1")
@@ -294,7 +294,7 @@ func TestInMemoryCacheRestoreOfflineDeletion(t *testing.T) {
m := NewCacheManager(context.TODO(), identity.NewID(), storage, results)
res0 := testResult("result0")
cacheFoo, err := m.Save(NewCacheKey(dgst("foo"), 0), res0, time.Now())
cacheFoo, err := m.Save(NewCacheKey(dgst("foo"), "", 0), res0, time.Now())
require.NoError(t, err)
res1 := testResult("result1")
@@ -329,20 +329,20 @@ func TestCarryOverFromSublink(t *testing.T) {
results := NewInMemoryResultStorage()
m := NewCacheManager(context.TODO(), identity.NewID(), storage, results)
cacheFoo, err := m.Save(NewCacheKey(dgst("foo"), 0), testResult("resultFoo"), time.Now())
cacheFoo, err := m.Save(NewCacheKey(dgst("foo"), "", 0), testResult("resultFoo"), time.Now())
require.NoError(t, err)
_, err = m.Save(testCacheKeyWithDeps(dgst("res"), 0, [][]CacheKeyWithSelector{
{{CacheKey: *cacheFoo, Selector: dgst("sel0")}, {CacheKey: expKey(NewCacheKey(dgst("content0"), 0))}},
{{CacheKey: *cacheFoo, Selector: dgst("sel0")}, {CacheKey: expKey(NewCacheKey(dgst("content0"), "", 0))}},
}), testResult("result0"), time.Now())
require.NoError(t, err)
cacheBar, err := m.Save(NewCacheKey(dgst("bar"), 0), testResult("resultBar"), time.Now())
cacheBar, err := m.Save(NewCacheKey(dgst("bar"), "", 0), testResult("resultBar"), time.Now())
require.NoError(t, err)
keys, err := m.Query([]CacheKeyWithSelector{
{CacheKey: *cacheBar, Selector: dgst("sel0")},
{CacheKey: expKey(NewCacheKey(dgst("content0"), 0))},
{CacheKey: expKey(NewCacheKey(dgst("content0"), "", 0))},
}, 0, dgst("res"), 0)
require.NoError(t, err)
require.Equal(t, len(keys), 1)

5
solver/cachekey.go
View File

@@ -7,10 +7,11 @@ import (
)
// NewCacheKey creates a new cache key for a specific output index
func NewCacheKey(dgst digest.Digest, output Index) *CacheKey {
func NewCacheKey(dgst, vtx digest.Digest, output Index) *CacheKey {
return &CacheKey{
ID: rootKey(dgst, output).String(),
digest: dgst,
vtx: vtx,
output: output,
ids: map[*cacheManager]string{},
}
@@ -29,6 +30,7 @@ type CacheKey struct {
ID string
deps [][]CacheKeyWithSelector // only [][]*inMemoryCacheKey
digest digest.Digest
vtx digest.Digest
output Index
ids map[*cacheManager]string
@@ -56,6 +58,7 @@ func (ck *CacheKey) clone() *CacheKey {
nk := &CacheKey{
ID: ck.ID,
digest: ck.digest,
vtx: ck.vtx,
output: ck.output,
ids: map[*cacheManager]string{},
}

10
solver/edge.go
View File

@@ -136,11 +136,11 @@ func (e *edge) release() {
// commitOptions returns parameters for the op execution
func (e *edge) commitOptions() ([]*CacheKey, []CachedResult) {
k := NewCacheKey(e.cacheMap.Digest, e.edge.Index)
k := NewCacheKey(e.cacheMap.Digest, e.edge.Vertex.Digest(), e.edge.Index)
if len(e.deps) == 0 {
keys := make([]*CacheKey, 0, len(e.cacheMapDigests))
for _, dgst := range e.cacheMapDigests {
keys = append(keys, NewCacheKey(dgst, e.edge.Index))
keys = append(keys, NewCacheKey(dgst, e.edge.Vertex.Digest(), e.edge.Index))
}
return keys, nil
}
@@ -201,6 +201,7 @@ func (e *edge) probeCache(d *dep, depKeys []CacheKeyWithSelector) bool {
}
found := false
for _, k := range keys {
k.vtx = e.edge.Vertex.Digest()
if _, ok := d.keyMap[k.ID]; !ok {
d.keyMap[k.ID] = k
found = true
@@ -275,7 +276,7 @@ func (e *edge) currentIndexKey() *CacheKey {
}
}
k := NewCacheKey(e.cacheMap.Digest, e.edge.Index)
k := NewCacheKey(e.cacheMap.Digest, e.edge.Vertex.Digest(), e.edge.Index)
k.deps = keys
return k
@@ -403,6 +404,7 @@ func (e *edge) processUpdate(upt pipe.Receiver) (depChanged bool) {
bklog.G(context.TODO()).Error(errors.Wrap(err, "invalid query response")) // make the build fail for this error
} else {
for _, k := range keys {
k.vtx = e.edge.Vertex.Digest()
records, err := e.op.Cache().Records(k)
if err != nil {
bklog.G(context.TODO()).Errorf("error receiving cache records: %v", err)
@@ -508,7 +510,7 @@ func (e *edge) processUpdate(upt pipe.Receiver) (depChanged bool) {
} else if !dep.slowCacheComplete {
dgst := upt.Status().Value.(digest.Digest)
if e.cacheMap.Deps[int(dep.index)].ComputeDigestFunc != nil && dgst != "" {
k := NewCacheKey(dgst, -1)
k := NewCacheKey(dgst, "", -1)
dep.slowCacheKey = &ExportableCacheKey{CacheKey: k, Exporter: &exporter{k: k}}
slowKeyExp := CacheKeyWithSelector{CacheKey: *dep.slowCacheKey}
defKeys := make([]CacheKeyWithSelector, 0, len(dep.result.CacheKeys()))

15
solver/exporter.go
View File

@@ -96,12 +96,17 @@ func (e *exporter) ExportTo(ctx context.Context, t CacheExporterTarget, opt Cach
addRecord = *e.override
}
if e.record == nil && len(e.k.Deps()) > 0 {
exportRecord := opt.ExportRoots
if len(e.k.Deps()) > 0 {
exportRecord = true
}
if e.record == nil && exportRecord {
e.record = getBestResult(e.records)
}
var remote *Remote
if v := e.record; v != nil && len(e.k.Deps()) > 0 && addRecord {
if v := e.record; v != nil && exportRecord && addRecord {
var variants []CacheExporterRecord
cm := v.cacheManager
@@ -121,7 +126,7 @@ func (e *exporter) ExportTo(ctx context.Context, t CacheExporterTarget, opt Cach
if opt.CompressionOpt != nil {
for _, r := range remotes { // record all remaining remotes as well
rec := t.Add(recKey)
rec.AddResult(v.CreatedAt, r)
rec.AddResult(e.k.vtx, int(e.k.output), v.CreatedAt, r)
variants = append(variants, rec)
}
}
@@ -142,7 +147,7 @@ func (e *exporter) ExportTo(ctx context.Context, t CacheExporterTarget, opt Cach
if opt.CompressionOpt != nil {
for _, r := range remotes { // record all remaining remotes as well
rec := t.Add(recKey)
rec.AddResult(v.CreatedAt, r)
rec.AddResult(e.k.vtx, int(e.k.output), v.CreatedAt, r)
variants = append(variants, rec)
}
}
@@ -150,7 +155,7 @@ func (e *exporter) ExportTo(ctx context.Context, t CacheExporterTarget, opt Cach
if remote != nil {
for _, rec := range allRec {
rec.AddResult(v.CreatedAt, remote)
rec.AddResult(e.k.vtx, int(e.k.output), v.CreatedAt, remote)
}
}
allRec = append(allRec, variants...)

74
solver/index_test.go
View File

@@ -18,21 +18,21 @@ func TestIndexSimple(t *testing.T) {
e2 := &edge{}
e3 := &edge{}
k1 := NewCacheKey(dgst("foo"), 0)
k1 := NewCacheKey(dgst("foo"), "", 0)
v := idx.LoadOrStore(k1, e1)
require.Nil(t, v)
k2 := NewCacheKey(dgst("bar"), 0)
k2 := NewCacheKey(dgst("bar"), "", 0)
v = idx.LoadOrStore(k2, e2)
require.Nil(t, v)
v = idx.LoadOrStore(NewCacheKey(dgst("bar"), 0), e3)
v = idx.LoadOrStore(NewCacheKey(dgst("bar"), "", 0), e3)
require.Equal(t, v, e2)
v = idx.LoadOrStore(NewCacheKey(dgst("bar"), 0), e3)
v = idx.LoadOrStore(NewCacheKey(dgst("bar"), "", 0), e3)
require.Equal(t, v, e2)
v = idx.LoadOrStore(NewCacheKey(dgst("foo"), 0), e3)
v = idx.LoadOrStore(NewCacheKey(dgst("foo"), "", 0), e3)
require.Equal(t, v, e1)
idx.Release(e1)
@@ -48,16 +48,16 @@ func TestIndexMultiLevelSimple(t *testing.T) {
e3 := &edge{}
k1 := testCacheKeyWithDeps(dgst("foo"), 1, [][]CacheKeyWithSelector{
{{CacheKey: expKey(NewCacheKey("f0", 0))}},
{{CacheKey: expKey(NewCacheKey("s0", 0)), Selector: dgst("s0")}},
{{CacheKey: expKey(NewCacheKey("f0", "", 0))}},
{{CacheKey: expKey(NewCacheKey("s0", "", 0)), Selector: dgst("s0")}},
})
v := idx.LoadOrStore(k1, e1)
require.Nil(t, v)
k2 := testCacheKeyWithDeps(dgst("foo"), 1, [][]CacheKeyWithSelector{
{{CacheKey: expKey(NewCacheKey("f0", 0))}},
{{CacheKey: expKey(NewCacheKey("s0", 0)), Selector: dgst("s0")}},
{{CacheKey: expKey(NewCacheKey("f0", "", 0))}},
{{CacheKey: expKey(NewCacheKey("s0", "", 0)), Selector: dgst("s0")}},
})
v = idx.LoadOrStore(k2, e2)
@@ -72,18 +72,18 @@ func TestIndexMultiLevelSimple(t *testing.T) {
// update selector
k2 = testCacheKeyWithDeps(dgst("foo"), 1, [][]CacheKeyWithSelector{
{{CacheKey: expKey(NewCacheKey("f0", 0))}},
{{CacheKey: expKey(NewCacheKey("s0", 0))}},
{{CacheKey: expKey(NewCacheKey("f0", "", 0))}},
{{CacheKey: expKey(NewCacheKey("s0", "", 0))}},
})
v = idx.LoadOrStore(k2, e2)
require.Nil(t, v)
// add one dep to e1
k2 = testCacheKeyWithDeps(dgst("foo"), 1, [][]CacheKeyWithSelector{
{{CacheKey: expKey(NewCacheKey("f0", 0))}},
{{CacheKey: expKey(NewCacheKey("f0", "", 0))}},
{
{CacheKey: expKey(NewCacheKey("s0", 0)), Selector: dgst("s0")},
{CacheKey: expKey(NewCacheKey("s1", 1))},
{CacheKey: expKey(NewCacheKey("s0", "", 0)), Selector: dgst("s0")},
{CacheKey: expKey(NewCacheKey("s1", "", 1))},
},
})
v = idx.LoadOrStore(k2, e2)
@@ -91,9 +91,9 @@ func TestIndexMultiLevelSimple(t *testing.T) {
// recheck with only the new dep key
k2 = testCacheKeyWithDeps(dgst("foo"), 1, [][]CacheKeyWithSelector{
{{CacheKey: expKey(NewCacheKey("f0", 0))}},
{{CacheKey: expKey(NewCacheKey("f0", "", 0))}},
{
{CacheKey: expKey(NewCacheKey("s1", 1))},
{CacheKey: expKey(NewCacheKey("s1", "", 1))},
},
})
v = idx.LoadOrStore(k2, e2)
@@ -101,10 +101,10 @@ func TestIndexMultiLevelSimple(t *testing.T) {
// combine e1 and e2
k2 = testCacheKeyWithDeps(dgst("foo"), 1, [][]CacheKeyWithSelector{
{{CacheKey: expKey(NewCacheKey("f0", 0))}},
{{CacheKey: expKey(NewCacheKey("f0", "", 0))}},
{
{CacheKey: expKey(NewCacheKey("s0", 0))},
{CacheKey: expKey(NewCacheKey("s1", 1))},
{CacheKey: expKey(NewCacheKey("s0", "", 0))},
{CacheKey: expKey(NewCacheKey("s1", "", 1))},
},
})
v = idx.LoadOrStore(k2, e2)
@@ -112,8 +112,8 @@ func TestIndexMultiLevelSimple(t *testing.T) {
// initial e2 now points to e1
k2 = testCacheKeyWithDeps(dgst("foo"), 1, [][]CacheKeyWithSelector{
{{CacheKey: expKey(NewCacheKey("f0", 0))}},
{{CacheKey: expKey(NewCacheKey("s0", 0))}},
{{CacheKey: expKey(NewCacheKey("f0", "", 0))}},
{{CacheKey: expKey(NewCacheKey("s0", "", 0))}},
})
v = idx.LoadOrStore(k2, e2)
require.Equal(t, v, e1)
@@ -122,8 +122,8 @@ func TestIndexMultiLevelSimple(t *testing.T) {
// e2 still remains after e1 is gone
k2 = testCacheKeyWithDeps(dgst("foo"), 1, [][]CacheKeyWithSelector{
{{CacheKey: expKey(NewCacheKey("f0", 0))}},
{{CacheKey: expKey(NewCacheKey("s0", 0))}},
{{CacheKey: expKey(NewCacheKey("f0", "", 0))}},
{{CacheKey: expKey(NewCacheKey("s0", "", 0))}},
})
v = idx.LoadOrStore(k2, e3)
require.Equal(t, v, e2)
@@ -140,8 +140,8 @@ func TestIndexThreeLevels(t *testing.T) {
e3 := &edge{}
k1 := testCacheKeyWithDeps(dgst("foo"), 1, [][]CacheKeyWithSelector{
{{CacheKey: expKey(NewCacheKey("f0", 0))}},
{{CacheKey: expKey(NewCacheKey("s0", 0)), Selector: dgst("s0")}},
{{CacheKey: expKey(NewCacheKey("f0", "", 0))}},
{{CacheKey: expKey(NewCacheKey("s0", "", 0)), Selector: dgst("s0")}},
})
v := idx.LoadOrStore(k1, e1)
@@ -151,26 +151,26 @@ func TestIndexThreeLevels(t *testing.T) {
require.Equal(t, v, e1)
k2 := testCacheKeyWithDeps(dgst("bar"), 0, [][]CacheKeyWithSelector{
{{CacheKey: expKey(NewCacheKey("f0", 0))}},
{{CacheKey: expKey(NewCacheKey("f0", "", 0))}},
{{CacheKey: expKey(k1)}},
})
v = idx.LoadOrStore(k2, e2)
require.Nil(t, v)
k2 = testCacheKeyWithDeps(dgst("bar"), 0, [][]CacheKeyWithSelector{
{{CacheKey: expKey(NewCacheKey("f0", 0))}},
{{CacheKey: expKey(NewCacheKey("f0", "", 0))}},
{
{CacheKey: expKey(k1)},
{CacheKey: expKey(NewCacheKey("alt", 0))},
{CacheKey: expKey(NewCacheKey("alt", "", 0))},
},
})
v = idx.LoadOrStore(k2, e2)
require.Nil(t, v)
k2 = testCacheKeyWithDeps(dgst("bar"), 0, [][]CacheKeyWithSelector{
{{CacheKey: expKey(NewCacheKey("f0", 0))}},
{{CacheKey: expKey(NewCacheKey("f0", "", 0))}},
{
{CacheKey: expKey(NewCacheKey("alt", 0))},
{CacheKey: expKey(NewCacheKey("alt", "", 0))},
},
})
v = idx.LoadOrStore(k2, e3)
@@ -179,13 +179,13 @@ func TestIndexThreeLevels(t *testing.T) {
// change dep in a low key
k1 = testCacheKeyWithDeps(dgst("foo"), 1, [][]CacheKeyWithSelector{
{
{CacheKey: expKey(NewCacheKey("f0", 0))},
{CacheKey: expKey(NewCacheKey("f0_", 0))},
{CacheKey: expKey(NewCacheKey("f0", "", 0))},
{CacheKey: expKey(NewCacheKey("f0_", "", 0))},
},
{{CacheKey: expKey(NewCacheKey("s0", 0)), Selector: dgst("s0")}},
{{CacheKey: expKey(NewCacheKey("s0", "", 0)), Selector: dgst("s0")}},
})
k2 = testCacheKeyWithDeps(dgst("bar"), 0, [][]CacheKeyWithSelector{
{{CacheKey: expKey(NewCacheKey("f0", 0))}},
{{CacheKey: expKey(NewCacheKey("f0", "", 0))}},
{{CacheKey: expKey(k1)}},
})
v = idx.LoadOrStore(k2, e3)
@@ -194,12 +194,12 @@ func TestIndexThreeLevels(t *testing.T) {
// reload with only f0_ still matches
k1 = testCacheKeyWithDeps(dgst("foo"), 1, [][]CacheKeyWithSelector{
{
{CacheKey: expKey(NewCacheKey("f0_", 0))},
{CacheKey: expKey(NewCacheKey("f0_", "", 0))},
},
{{CacheKey: expKey(NewCacheKey("s0", 0)), Selector: dgst("s0")}},
{{CacheKey: expKey(NewCacheKey("s0", "", 0)), Selector: dgst("s0")}},
})
k2 = testCacheKeyWithDeps(dgst("bar"), 0, [][]CacheKeyWithSelector{
{{CacheKey: expKey(NewCacheKey("f0", 0))}},
{{CacheKey: expKey(NewCacheKey("f0", "", 0))}},
{{CacheKey: expKey(k1)}},
})
v = idx.LoadOrStore(k2, e3)

124
solver/llbsolver/proc/provenance.go
View File

@@ -3,11 +3,15 @@ package proc
import (
"context"
"encoding/json"
"fmt"
"strconv"
"strings"
"time"
slsa "github.com/in-toto/in-toto-golang/in_toto/slsa_provenance/v0.2"
"github.com/moby/buildkit/cache"
"github.com/moby/buildkit/cache/config"
"github.com/moby/buildkit/exporter/containerimage"
"github.com/moby/buildkit/exporter/containerimage/exptypes"
"github.com/moby/buildkit/frontend"
gatewaypb "github.com/moby/buildkit/frontend/gateway/pb"
@@ -17,6 +21,9 @@ import (
"github.com/moby/buildkit/solver/result"
binfotypes "github.com/moby/buildkit/util/buildinfo/types"
provenance "github.com/moby/buildkit/util/provenance"
"github.com/moby/buildkit/worker"
digest "github.com/opencontainers/go-digest"
ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
)
@@ -87,6 +94,8 @@ func ProvenanceProcessor(attrs map[string]string) llbsolver.Processor {
pr.Metadata.Reproducible = reproducible
pr.Metadata.BuildInvocationID = buildID
var addLayers func() error
if mode != "max" {
param := make(map[string]*string)
for k, v := range pr.Invocation.Parameters.(map[string]*string) {
@@ -98,9 +107,43 @@ func ProvenanceProcessor(attrs map[string]string) llbsolver.Processor {
}
pr.Invocation.Parameters = param
} else {
if err := provenance.AddBuildConfig(ctx, pr, res.Refs[p.ID]); err != nil {
dgsts, err := provenance.AddBuildConfig(ctx, pr, res.Refs[p.ID])
if err != nil {
return nil, err
}
r, err := res.Refs[p.ID].Result(ctx)
if err != nil {
return nil, err
}
addLayers = func() error {
e := newCacheExporter()
if _, err := r.CacheKeys()[0].Exporter.ExportTo(ctx, e, solver.CacheExportOpt{
ResolveRemotes: resolveRemotes,
Mode: solver.CacheExportModeRemoteOnly,
ExportRoots: true,
}); err != nil {
return err
}
m := map[string][][]ocispecs.Descriptor{}
for l, descs := range e.layers {
idx, ok := dgsts[l.digest]
if !ok {
continue
}
m[fmt.Sprintf("step%d:%d", idx, l.index)] = descs
}
if len(m) != 0 {
pr.Layers = m
}
return nil
}
}
res.AddAttestation(p.ID, result.Attestation{
@@ -111,6 +154,13 @@ func ProvenanceProcessor(attrs map[string]string) llbsolver.Processor {
ContentFunc: func() ([]byte, error) {
end := time.Now()
pr.Metadata.BuildFinishedOn = &end
if addLayers != nil {
if err := addLayers(); err != nil {
return nil, err
}
}
// TODO: pass indent to json.Marshal
return json.MarshalIndent(pr, "", " ")
},
@@ -120,3 +170,75 @@ func ProvenanceProcessor(attrs map[string]string) llbsolver.Processor {
return res, nil
}
}
func resolveRemotes(ctx context.Context, res solver.Result) ([]*solver.Remote, error) {
ref, ok := res.Sys().(*worker.WorkerRef)
if !ok {
return nil, errors.Errorf("invalid result: %T", res.Sys())
}
remotes, err := ref.GetRemotes(ctx, false, config.RefConfig{}, true, nil)
if err != nil {
if errors.Is(err, cache.ErrNoBlobs) {
return nil, nil
}
return nil, err
}
return remotes, nil
}
type edge struct {
digest digest.Digest
index int
}
func newCacheExporter() *cacheExporter {
return &cacheExporter{
m: map[interface{}]struct{}{},
layers: map[edge][][]ocispecs.Descriptor{},
}
}
type cacheExporter struct {
layers map[edge][][]ocispecs.Descriptor
m map[interface{}]struct{}
}
func (ce *cacheExporter) Add(dgst digest.Digest) solver.CacheExporterRecord {
return &cacheRecord{
ce: ce,
}
}
func (ce *cacheExporter) Visit(v interface{}) {
ce.m[v] = struct{}{}
}
func (ce *cacheExporter) Visited(v interface{}) bool {
_, ok := ce.m[v]
return ok
}
type cacheRecord struct {
ce *cacheExporter
}
func (c *cacheRecord) AddResult(dgst digest.Digest, idx int, createdAt time.Time, result *solver.Remote) {
if result == nil || dgst == "" {
return
}
e := edge{
digest: dgst,
index: idx,
}
descs := make([]ocispecs.Descriptor, len(result.Descriptors))
for i, desc := range result.Descriptors {
d := desc
containerimage.RemoveInternalLayerAnnotations(&d, true)
descs[i] = d
}
c.ce.layers[e] = append(c.ce.layers[e], descs)
}
func (c *cacheRecord) LinkFrom(rec solver.CacheExporterRecord, index int, selector string) {
}

2
solver/scheduler_test.go
View File

@@ -3844,7 +3844,7 @@ type testExporterRecord struct {
linkMap map[digest.Digest]struct{}
}
func (r *testExporterRecord) AddResult(createdAt time.Time, result *Remote) {
func (r *testExporterRecord) AddResult(_ digest.Digest, _ int, createdAt time.Time, result *Remote) {
r.results++
}

4
solver/types.go
View File

@@ -104,6 +104,8 @@ type CacheExportOpt struct {
// CompressionOpt is an option to specify the compression of the object to load.
// If specified, all objects that meet the option will be cached.
CompressionOpt *compression.Config
// ExportRoots defines if records for root vertexes should be exported.
ExportRoots bool
}
// CacheExporter can export the artifacts of the build chain
@@ -120,7 +122,7 @@ type CacheExporterTarget interface {
// CacheExporterRecord is a single object being exported
type CacheExporterRecord interface {
AddResult(createdAt time.Time, result *Remote)
AddResult(vtx digest.Digest, index int, createdAt time.Time, result *Remote)
LinkFrom(src CacheExporterRecord, index int, selector string)
}

11
util/provenance/buildconfig.go
View File

@@ -31,11 +31,11 @@ type SourceInfo struct {
Definition []BuildStep `json:"llbDefinition,omitempty"`
}
func AddBuildConfig(ctx context.Context, p *ProvenancePredicate, rp solver.ResultProxy) error {
func AddBuildConfig(ctx context.Context, p *ProvenancePredicate, rp solver.ResultProxy) (map[digest.Digest]int, error) {
def := rp.Definition()
steps, indexes, err := toBuildSteps(def)
if err != nil {
return err
return nil, err
}
bc := &BuildConfig{
@@ -49,7 +49,7 @@ func AddBuildConfig(ctx context.Context, p *ProvenancePredicate, rp solver.Resul
for i, si := range def.Source.Infos {
steps, _, err := toBuildSteps(si.Definition)
if err != nil {
return err
return nil, err
}
s := SourceInfo{
Filename: si.Filename,
@@ -70,12 +70,13 @@ func AddBuildConfig(ctx context.Context, p *ProvenancePredicate, rp solver.Resul
}
p.Source = &Source{
Infos: sis,
Infos: sis,
Locations: locs,
}
}
}
return nil
return indexes, nil
}
func toBuildSteps(def *pb.Definition) ([]BuildStep, map[digest.Digest]int, error) {

6
util/provenance/buildinfo.go
View File

@@ -8,6 +8,7 @@ import (
slsa "github.com/in-toto/in-toto-golang/in_toto/slsa_provenance/v0.2"
binfotypes "github.com/moby/buildkit/util/buildinfo/types"
digest "github.com/opencontainers/go-digest"
ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
)
@@ -15,8 +16,9 @@ var BuildKitBuildType = "https://mobyproject.org/buildkit@v1"
type ProvenancePredicate struct {
slsa.ProvenancePredicate
Metadata *ProvenanceMetadata `json:"metadata,omitempty"`
Source *Source `json:"buildSource,omitempty"`
Metadata *ProvenanceMetadata `json:"metadata,omitempty"`
Source *Source `json:"buildSource,omitempty"`
Layers map[string][][]ocispecs.Descriptor `json:"buildLayers,omitempty"`
}
type ProvenanceMetadata struct {