From 854eace665d6895ee04b0e774a501cacb2d5717a Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Thu, 10 Jul 2025 17:24:45 -0700 Subject: [PATCH] debug: combine links from parents intead of children Grouping the link digests to parent side (links pointing to current node) makes more sense than children side (links pointing to next nodes). This also reduces the duplicates in the output as previously same digest link could need plaintext under multiple nodes. Signed-off-by: Tonis Tiigi --- cmd/buildkitd/debug.go | 29 +++++++++++-------- util/cachedigest/db_test.go | 2 +- util/cachestore/store.go | 57 +++++++++++++++++++++---------------- 3 files changed, 51 insertions(+), 37 deletions(-) diff --git a/cmd/buildkitd/debug.go b/cmd/buildkitd/debug.go index 1f67a0565..bebc0197b 100644 --- a/cmd/buildkitd/debug.go +++ b/cmd/buildkitd/debug.go @@ -1,6 +1,7 @@ package main import ( + "cmp" "context" "encoding/binary" "encoding/json" @@ -309,10 +310,23 @@ func writeCacheRecordsResponse(w http.ResponseWriter, r *http.Request, recs []*r if rec.Digest != "" { fmt.Fprintf(w, "Digest: %s\n", rec.Digest) } + if len(rec.Parents) > 0 { fmt.Fprintln(w, "Parents:") - for input := range rec.Parents { - ids := slices.Collect(maps.Keys(rec.ParentIDs[input])) + slices.SortStableFunc(rec.Parents, func(i, j cachestore.Link) int { + return cmp.Or(cmp.Compare(i.Input, j.Input), cmp.Compare(i.Digest, j.Digest)) + }) + for _, parent := range rec.Parents { + fmt.Fprintf(w, " Input %d:\t%d\t%s\n", parent.Input, parent.Record.ID, parent.Digest) + if parent.Selector != "" { + fmt.Fprintf(w, " Selector: %s\n", parent.Selector) + } + } + } + if len(rec.Children) > 0 { + fmt.Fprintln(w, "Children:") + for input := range rec.Children { + ids := slices.Collect(maps.Keys(rec.ChildIDs[input])) s := make([]string, len(ids)) for i, id := range ids { s[i] = fmt.Sprintf("%d", id) @@ -320,15 +334,6 @@ func writeCacheRecordsResponse(w http.ResponseWriter, r *http.Request, recs []*r fmt.Fprintf(w, " Input %d:\t %s\n", input, strings.Join(s, ", ")) } } - if len(rec.Children) > 0 { - fmt.Fprintln(w, "Children:") - for _, child := range rec.Children { - fmt.Fprintf(w, " %d %s (input %d, output %d)\n", child.Record.ID, child.Digest, child.Input, child.Output) - if child.Selector != "" { - fmt.Fprintf(w, " Selector: %s\n", child.Selector) - } - } - } if len(rec.Debug) > 0 { fmt.Fprintln(w, "Plaintexts:") for _, debugRec := range rec.Debug { @@ -361,7 +366,7 @@ func debugCacheStore(ctx context.Context, store solver.CacheKeyStorage) ([]*reco if rec.Digest != "" { m[rec.Digest] = nil } - for _, link := range rec.Children { + for _, link := range rec.Parents { m[link.Digest] = nil if link.Selector != "" { m[link.Selector] = nil diff --git a/util/cachedigest/db_test.go b/util/cachedigest/db_test.go index f5a6fbdd0..36d046812 100644 --- a/util/cachedigest/db_test.go +++ b/util/cachedigest/db_test.go @@ -95,7 +95,7 @@ func TestNewHashAndGet(t *testing.T) { dataFrames = append(dataFrames, f.Data) case FrameIDSkip: require.Len(t, f.Data, 4) - skipLens = append(skipLens, uint32(f.Data[0])<<24|uint32(f.Data[1])<<16|uint32(f.Data[2])<<8|uint32(f.Data[3])) + skipLens = append(skipLens, uint32(f.Data[3])<<24|uint32(f.Data[2])<<16|uint32(f.Data[1])<<8|uint32(f.Data[0])) } } require.Len(t, dataFrames, len(inputs)) diff --git a/util/cachestore/store.go b/util/cachestore/store.go index e4165dfc4..ac445e203 100644 --- a/util/cachestore/store.go +++ b/util/cachestore/store.go @@ -1,7 +1,10 @@ package cachestore import ( + "cmp" "context" + "maps" + "slices" "strings" "github.com/moby/buildkit/solver" @@ -10,12 +13,12 @@ import ( ) type Record struct { - ID int `json:"id"` - Parents map[int]map[*Record]struct{} `json:"-"` - Children []Link `json:"children,omitempty"` - Digest digest.Digest `json:"digest,omitempty"` - Random bool `json:"random,omitempty"` - ParentIDs map[int]map[int]struct{} `json:"parents,omitempty"` + ID int `json:"id"` + Parents []Link `json:"parents,omitempty"` + Children map[int]map[*Record]struct{} `json:"-"` + Digest digest.Digest `json:"digest,omitempty"` + Random bool `json:"random,omitempty"` + ChildIDs map[int]map[int]struct{} `json:"children,omitempty"` } type Link struct { @@ -72,16 +75,16 @@ func Records(ctx context.Context, store solver.CacheKeyStorage) ([]*Record, erro } func setLinkIDs(rec *Record) { - for i, child := range rec.Children { - child.ID = child.Record.ID - rec.Children[i] = child + for i, parent := range rec.Parents { + parent.ID = parent.Record.ID + rec.Parents[i] = parent } - if rec.Parents != nil { - rec.ParentIDs = make(map[int]map[int]struct{}) - for input, m := range rec.Parents { - rec.ParentIDs[input] = make(map[int]struct{}) - for parent := range m { - rec.ParentIDs[input][parent.ID] = struct{}{} + if rec.Children != nil { + rec.ChildIDs = make(map[int]map[int]struct{}) + for input, m := range rec.Children { + rec.ChildIDs[input] = make(map[int]struct{}) + for child := range m { + rec.ChildIDs[input][child.ID] = struct{}{} } } } @@ -93,8 +96,14 @@ func setIndex(rec *Record, arr []*Record) []*Record { } arr = append(arr, rec) rec.ID = len(arr) - for _, child := range rec.Children { - arr = setIndex(child.Record, arr) + for _, links := range rec.Children { + recs := slices.Collect(maps.Keys(links)) + slices.SortFunc(recs, func(i, j *Record) int { + return cmp.Compare(i.Digest, j.Digest) + }) + for _, child := range recs { + arr = setIndex(child, arr) + } } return arr } @@ -122,23 +131,23 @@ func loadRecord(ctx context.Context, store storeWithLinks, id string, out map[st if err != nil { return errors.Wrapf(err, "failed to load link %s for %s", linkID, id) } - rec.Children = append(rec.Children, Link{ + child.Parents = append(child.Parents, Link{ Input: int(link.Input), Output: int(link.Output), Selector: link.Selector, - Record: child, + Record: rec, Digest: link.Digest, }) - if child.Parents == nil { - child.Parents = make(map[int]map[*Record]struct{}) + if rec.Children == nil { + rec.Children = make(map[int]map[*Record]struct{}) } - m, ok := child.Parents[int(link.Input)] + m, ok := rec.Children[int(link.Output)] if !ok { m = make(map[*Record]struct{}) - child.Parents[int(link.Input)] = m + rec.Children[int(link.Output)] = m } - m[rec] = struct{}{} + m[child] = struct{}{} return nil }) if err != nil {