dilim/buildkit
1
0
Fork 0
mirror of https://github.com/moby/buildkit.git synced 2026-08-08 00:30:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

solver: keep original cache creation date when copying between stores

Browse Source 
Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
This commit is contained in:
Tonis Tiigi
2019-01-10 08:43:27 -08:00
parent 1ec543e384
commit ac4037a1d9
10 changed files with 38 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
cache/remotecache/v1/cachestorage.go vendored
View File

@@ -2,6 +2,7 @@ package cacheimport
import (
"context"
"time"
"github.com/moby/buildkit/identity"
"github.com/moby/buildkit/solver"
@@ -191,7 +192,7 @@ type cacheResultStorage struct {
byResult map[string]map[string]struct{}
}
func (cs *cacheResultStorage) Save(res solver.Result) (solver.CacheResult, error) {
func (cs *cacheResultStorage) Save(res solver.Result, createdAt time.Time) (solver.CacheResult, error) {
return solver.CacheResult{}, errors.Errorf("importer is immutable")
}

35
solver/cache_test.go
View File

@@ -3,6 +3,7 @@ package solver
import (
"context"
"testing"
"time"
"github.com/moby/buildkit/identity"
digest "github.com/opencontainers/go-digest"
@@ -41,7 +42,7 @@ func TestInMemoryCache(t *testing.T) {
m := NewInMemoryCacheManager()
cacheFoo, err := m.Save(NewCacheKey(dgst("foo"), 0), testResult("result0"))
cacheFoo, err := m.Save(NewCacheKey(dgst("foo"), 0), testResult("result0"), time.Now())
require.NoError(t, err)
keys, err := m.Query(nil, 0, dgst("foo"), 0)
@@ -57,7 +58,7 @@ func TestInMemoryCache(t *testing.T) {
require.Equal(t, "result0", unwrap(res))
// another record
cacheBar, err := m.Save(NewCacheKey(dgst("bar"), 0), testResult("result1"))
cacheBar, err := m.Save(NewCacheKey(dgst("bar"), 0), testResult("result1"), time.Now())
require.NoError(t, err)
keys, err = m.Query(nil, 0, dgst("bar"), 0)
@@ -79,7 +80,7 @@ func TestInMemoryCache(t *testing.T) {
// second level
k := testCacheKey(dgst("baz"), Index(1), *cacheFoo, *cacheBar)
cacheBaz, err := m.Save(k, testResult("result2"))
cacheBaz, err := m.Save(k, testResult("result2"), time.Now())
require.NoError(t, err)
keys, err = m.Query(nil, 0, dgst("baz"), 0)
@@ -113,7 +114,7 @@ func TestInMemoryCache(t *testing.T) {
require.Equal(t, keys[0].ID, keys2[0].ID)
k = testCacheKey(dgst("baz"), Index(1), *cacheFoo)
_, err = m.Save(k, testResult("result3"))
_, err = m.Save(k, testResult("result3"), time.Now())
require.NoError(t, err)
keys, err = m.Query(depKeys(*cacheFoo), 0, dgst("baz"), Index(1))
@@ -128,7 +129,7 @@ func TestInMemoryCache(t *testing.T) {
{{CacheKey: *cacheFoo}, {CacheKey: *cacheBaz}},
{{CacheKey: *cacheBar}},
})
_, err = m.Save(k, testResult("result4"))
_, err = m.Save(k, testResult("result4"), time.Now())
require.NoError(t, err)
// foo, bar, baz should all point to result4
@@ -154,12 +155,12 @@ func TestInMemoryCacheSelector(t *testing.T) {
m := NewInMemoryCacheManager()
cacheFoo, err := m.Save(NewCacheKey(dgst("foo"), 0), testResult("result0"))
cacheFoo, err := m.Save(NewCacheKey(dgst("foo"), 0), testResult("result0"), time.Now())
require.NoError(t, err)
_, err = m.Save(testCacheKeyWithDeps(dgst("bar"), 0, [][]CacheKeyWithSelector{
{{CacheKey: *cacheFoo, Selector: dgst("sel0")}},
}), testResult("result1"))
}), testResult("result1"), time.Now())
require.NoError(t, err)
keys, err := m.Query(depKeys(*cacheFoo), 0, dgst("bar"), 0)
@@ -188,12 +189,12 @@ func TestInMemoryCacheSelectorNested(t *testing.T) {
m := NewInMemoryCacheManager()
cacheFoo, err := m.Save(NewCacheKey(dgst("foo"), 0), testResult("result0"))
cacheFoo, err := m.Save(NewCacheKey(dgst("foo"), 0), testResult("result0"), time.Now())
require.NoError(t, err)
_, err = m.Save(testCacheKeyWithDeps(dgst("bar"), 0, [][]CacheKeyWithSelector{
{{CacheKey: *cacheFoo, Selector: dgst("sel0")}, {CacheKey: expKey(NewCacheKey(dgst("second"), 0))}},
}), testResult("result1"))
}), testResult("result1"), time.Now())
require.NoError(t, err)
keys, err := m.Query(
@@ -241,11 +242,11 @@ func TestInMemoryCacheReleaseParent(t *testing.T) {
m := NewCacheManager(identity.NewID(), storage, results)
res0 := testResult("result0")
cacheFoo, err := m.Save(NewCacheKey(dgst("foo"), 0), res0)
cacheFoo, err := m.Save(NewCacheKey(dgst("foo"), 0), res0, time.Now())
require.NoError(t, err)
res1 := testResult("result1")
_, err = m.Save(testCacheKey(dgst("bar"), 0, *cacheFoo), res1)
_, err = m.Save(testCacheKey(dgst("bar"), 0, *cacheFoo), res1, time.Now())
require.NoError(t, err)
keys, err := m.Query(nil, 0, dgst("foo"), 0)
@@ -293,15 +294,15 @@ func TestInMemoryCacheRestoreOfflineDeletion(t *testing.T) {
m := NewCacheManager(identity.NewID(), storage, results)
res0 := testResult("result0")
cacheFoo, err := m.Save(NewCacheKey(dgst("foo"), 0), res0)
cacheFoo, err := m.Save(NewCacheKey(dgst("foo"), 0), res0, time.Now())
require.NoError(t, err)
res1 := testResult("result1")
_, err = m.Save(testCacheKey(dgst("bar"), 0, *cacheFoo), res1)
_, err = m.Save(testCacheKey(dgst("bar"), 0, *cacheFoo), res1, time.Now())
require.NoError(t, err)
results2 := NewInMemoryResultStorage()
_, err = results2.Save(res1) // only add bar
_, err = results2.Save(res1, time.Now()) // only add bar
require.NoError(t, err)
m = NewCacheManager(identity.NewID(), storage, results2)
@@ -328,15 +329,15 @@ func TestCarryOverFromSublink(t *testing.T) {
results := NewInMemoryResultStorage()
m := NewCacheManager(identity.NewID(), storage, results)
cacheFoo, err := m.Save(NewCacheKey(dgst("foo"), 0), testResult("resultFoo"))
cacheFoo, err := m.Save(NewCacheKey(dgst("foo"), 0), testResult("resultFoo"), time.Now())
require.NoError(t, err)
_, err = m.Save(testCacheKeyWithDeps(dgst("res"), 0, [][]CacheKeyWithSelector{
{{CacheKey: *cacheFoo, Selector: dgst("sel0")}, {CacheKey: expKey(NewCacheKey(dgst("content0"), 0))}},
}), testResult("result0"))
}), testResult("result0"), time.Now())
require.NoError(t, err)
cacheBar, err := m.Save(NewCacheKey(dgst("bar"), 0), testResult("resultBar"))
cacheBar, err := m.Save(NewCacheKey(dgst("bar"), 0), testResult("resultBar"), time.Now())
require.NoError(t, err)
keys, err := m.Query([]CacheKeyWithSelector{

6
solver/cachemanager.go
View File

@@ -5,6 +5,7 @@ import (
"fmt"
"strings"
"sync"
"time"
"github.com/moby/buildkit/identity"
digest "github.com/opencontainers/go-digest"
@@ -143,15 +144,14 @@ func (c *cacheManager) Load(ctx context.Context, rec *CacheRecord) (Result, erro
return c.results.Load(ctx, res)
}
func (c *cacheManager) Save(k *CacheKey, r Result) (*ExportableCacheKey, error) {
func (c *cacheManager) Save(k *CacheKey, r Result, createdAt time.Time) (*ExportableCacheKey, error) {
c.mu.Lock()
defer c.mu.Unlock()
res, err := c.results.Save(r)
res, err := c.results.Save(r, createdAt)
if err != nil {
return nil, err
}
if err := c.backend.AddResult(c.getID(k), res); err != nil {
return nil, err
}

2
solver/cachestorage.go
View File

@@ -44,7 +44,7 @@ type CacheInfoLink struct {
// CacheResultStorage is interface for converting cache metadata result to
// actual solve result
type CacheResultStorage interface {
Save(Result) (CacheResult, error)
Save(Result, time.Time) (CacheResult, error)
Load(ctx context.Context, res CacheResult) (Result, error)
LoadRemote(ctx context.Context, res CacheResult) (*Remote, error)
Exists(id string) bool

7
solver/combinedcache.go
View File

@@ -4,6 +4,7 @@ import (
"context"
"strings"
"sync"
"time"
digest "github.com/opencontainers/go-digest"
"github.com/pkg/errors"
@@ -71,14 +72,14 @@ func (cm *combinedCacheManager) Load(ctx context.Context, rec *CacheRecord) (Res
if err != nil {
return nil, err
}
if _, err := cm.main.Save(rec.key, res); err != nil {
if _, err := cm.main.Save(rec.key, res, rec.CreatedAt); err != nil {
return nil, err
}
return res, nil
}
func (cm *combinedCacheManager) Save(key *CacheKey, s Result) (*ExportableCacheKey, error) {
return cm.main.Save(key, s)
func (cm *combinedCacheManager) Save(key *CacheKey, s Result, createdAt time.Time) (*ExportableCacheKey, error) {
return cm.main.Save(key, s, createdAt)
}
func (cm *combinedCacheManager) Records(ck *CacheKey) ([]*CacheRecord, error) {

3
solver/edge.go
View File

@@ -3,6 +3,7 @@ package solver
import (
"context"
"sync"
"time"
"github.com/moby/buildkit/solver/internal/pipe"
digest "github.com/opencontainers/go-digest"
@@ -845,7 +846,7 @@ func (e *edge) execOp(ctx context.Context) (interface{}, error) {
var exporters []CacheExporter
for _, cacheKey := range cacheKeys {
ck, err := e.op.Cache().Save(cacheKey, res)
ck, err := e.op.Cache().Save(cacheKey, res, time.Now())
if err != nil {
return nil, err
}

5
solver/llbsolver/bridge.go
View File

@@ -6,6 +6,7 @@ import (
"io"
"strings"
"sync"
"time"
"github.com/containerd/containerd/platforms"
"github.com/moby/buildkit/cache"
@@ -190,11 +191,11 @@ func (lcm *lazyCacheManager) Load(ctx context.Context, rec *solver.CacheRecord)
}
return lcm.main.Load(ctx, rec)
}
func (lcm *lazyCacheManager) Save(key *solver.CacheKey, s solver.Result) (*solver.ExportableCacheKey, error) {
func (lcm *lazyCacheManager) Save(key *solver.CacheKey, s solver.Result, createdAt time.Time) (*solver.ExportableCacheKey, error) {
if err := lcm.wait(); err != nil {
return nil, err
}
return lcm.main.Save(key, s)
return lcm.main.Save(key, s, createdAt)
}
func (lcm *lazyCacheManager) wait() error {

4
solver/memorycachestorage.go
View File

@@ -284,9 +284,9 @@ type inMemoryResultStore struct {
m *sync.Map
}
func (s *inMemoryResultStore) Save(r Result) (CacheResult, error) {
func (s *inMemoryResultStore) Save(r Result, createdAt time.Time) (CacheResult, error) {
s.m.Store(r.ID(), r)
return CacheResult{ID: r.ID(), CreatedAt: time.Now()}, nil
return CacheResult{ID: r.ID(), CreatedAt: createdAt}, nil
}
func (s *inMemoryResultStore) Load(ctx context.Context, res CacheResult) (Result, error) {

2
solver/types.go
View File

@@ -164,5 +164,5 @@ type CacheManager interface {
// Load pulls and returns the cached result
Load(ctx context.Context, rec *CacheRecord) (Result, error)
// Save saves a result based on a cache key
Save(key *CacheKey, s Result) (*ExportableCacheKey, error)
Save(key *CacheKey, s Result, createdAt time.Time) (*ExportableCacheKey, error)
}

4
worker/cacheresult.go
View File

@@ -20,7 +20,7 @@ type cacheResultStorage struct {
wc *Controller
}
func (s *cacheResultStorage) Save(res solver.Result) (solver.CacheResult, error) {
func (s *cacheResultStorage) Save(res solver.Result, createdAt time.Time) (solver.CacheResult, error) {
ref, ok := res.Sys().(*WorkerRef)
if !ok {
return solver.CacheResult{}, errors.Errorf("invalid result: %T", res.Sys())
@@ -33,7 +33,7 @@ func (s *cacheResultStorage) Save(res solver.Result) (solver.CacheResult, error)
ref.ImmutableRef.Metadata().Commit()
}
}
return solver.CacheResult{ID: ref.ID(), CreatedAt: time.Now()}, nil
return solver.CacheResult{ID: ref.ID(), CreatedAt: createdAt}, nil
}
func (s *cacheResultStorage) Load(ctx context.Context, res solver.CacheResult) (solver.Result, error) {
return s.load(res.ID, false)