solver: handle not found error on cache export

When a cleanup happens while a build is running it
may cause boltdb record to be deleted while that record
would have been used for cache export of the running build,
causing the cache export to fail with Not Found error. This
is possible because when cache is loaded to snapshots then
snapshots are reference counted, protecting against them being
released or deleted, but when cache key is just intermediary,
the query result just gets saved and is used later.

It should be safe to ignore this error and skip over the result
if it doesn't exist anymore.

Note that cache export also gets called on provenance
creation because cache chains are used for finding remote
descriptors for build steps. This was the more likely
codepath for hitting this error.

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>

solver/exporter.go

@@ -2,6 +2,8 @@ package solver
 
 import (
 	"context"
+	"errors"
+	"slices"
 
 	digest "github.com/opencontainers/go-digest"
 )
@@ -102,18 +104,29 @@ func (e *exporter) ExportTo(ctx context.Context, t CacheExporterTarget, opt Cach
 		exportRecord = true
 	}
 
-	if e.record == nil && exportRecord {
-		e.record = getBestResult(e.records)
-	}
+	records := slices.Clone(e.records)
+	slices.SortStableFunc(records, compareCacheRecord)
 
 	var remote *Remote
-	if v := e.record; v != nil && exportRecord && addRecord {
+	var i int
+	for exportRecord && addRecord {
 		var variants []CacheExporterRecord
-
+		v := e.record
+		if v == nil {
+			if i < len(records) {
+				v = records[i]
+				i++
+			} else {
+				break
+			}
+		}
 		cm := v.cacheManager
 		key := cm.getID(v.key)
 		res, err := cm.backend.Load(key, v.ID)
 		if err != nil {
+			if errors.Is(err, ErrNotFound) {
+				continue
+			}
 			return nil, err
 		}
 
@@ -160,6 +173,7 @@ func (e *exporter) ExportTo(ctx context.Context, t CacheExporterTarget, opt Cach
 			}
 		}
 		allRec = append(allRec, variants...)
+		break
 	}
 
 	if remote != nil && opt.Mode == CacheExportModeMin {
@@ -228,13 +242,28 @@ func (e *exporter) ExportTo(ctx context.Context, t CacheExporterTarget, opt Cach
 }
 
 func getBestResult(records []*CacheRecord) *CacheRecord {
-	var rec *CacheRecord
-	for _, r := range records {
-		if rec == nil || rec.CreatedAt.Before(r.CreatedAt) || (rec.CreatedAt.Equal(r.CreatedAt) && rec.Priority < r.Priority) {
-			rec = r
-		}
+	records = slices.Clone(records)
+	slices.SortStableFunc(records, compareCacheRecord)
+	if len(records) == 0 {
+		return nil
 	}
-	return rec
+	return records[0]
+}
+
+func compareCacheRecord(a, b *CacheRecord) int {
+	if a == nil && b == nil {
+		return 0
+	}
+	if a == nil {
+		return 1
+	}
+	if b == nil {
+		return -1
+	}
+	if v := b.CreatedAt.Compare(a.CreatedAt); v != 0 {
+		return v
+	}
+	return a.Priority - b.Priority
 }
 
 type mergedExporter struct {

solver/exporter_test.go

@@ -0,0 +1,34 @@
+package solver
+
+import (
+	"slices"
+	"testing"
+	"time"
+)
+
+func TestCompareCacheRecord(t *testing.T) {
+	now := time.Now()
+	a := &CacheRecord{CreatedAt: now, Priority: 1}
+	b := &CacheRecord{CreatedAt: now, Priority: 2}
+	c := &CacheRecord{CreatedAt: now.Add(1 * time.Second), Priority: 1}
+	d := &CacheRecord{CreatedAt: now.Add(-1 * time.Second), Priority: 1}
+
+	records := []*CacheRecord{b, nil, d, a, c, nil}
+	slices.SortFunc(records, compareCacheRecord)
+
+	names := map[*CacheRecord]string{
+		a:   "a",
+		b:   "b",
+		c:   "c",
+		d:   "d",
+		nil: "nil",
+	}
+	var got []string
+	for _, r := range records {
+		got = append(got, names[r])
+	}
+	want := []string{"c", "a", "b", "d", "nil", "nil"}
+	if !slices.Equal(got, want) {
+		t.Fatalf("unexpected order: got %v, want %v", got, want)
+	}
+}