From d14a00065fab56e0d6c79215e71263284e8c6485 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Morav=C4=8D=C3=ADk?= Date: Wed, 25 Mar 2026 14:17:58 +0100 Subject: [PATCH] client: add `reset` option for local cache exporter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add `reset=true` attribute to the local cache exporter that removes unreferenced blobs from the cache directory after export, preventing unbounded growth. Signed-off-by: Jiří Moravčík --- README.md | 1 + client/client_test.go | 88 ++++++++++ client/solve.go | 67 ++++++++ client/solve_resetcache_test.go | 285 ++++++++++++++++++++++++++++++++ 4 files changed, 441 insertions(+) create mode 100644 client/solve_resetcache_test.go diff --git a/README.md b/README.md index 81370f759..6ccba2a1c 100644 --- a/README.md +++ b/README.md @@ -497,6 +497,7 @@ The directory layout conforms to OCI Image Spec v1.0. * `compression-level=`: compression level for gzip, estargz (0-9) and zstd (0-22) * `force-compression=true`: forcibly apply `compression` option to all layers * `ignore-error=`: specify if error is ignored in case cache export fails (default: `false`) +* `reset=`: remove any blobs in the cache directory that are not referenced by the current manifests in `index.json` (default: `false`). This is useful for keeping the local cache directory from growing indefinitely. `--import-cache` options: * `type=local` diff --git a/client/client_test.go b/client/client_test.go index 09a63a2e2..cd976a79a 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -202,6 +202,7 @@ var allTests = []func(t *testing.T, sb integration.Sandbox){ testZstdLocalCacheExport, testCacheExportIgnoreError, testCacheExportCacheDeletedContent, + testLocalCacheExportReset, testZstdRegistryCacheImportExport, testZstdLocalCacheImportExport, testUncompressedLocalCacheImportExport, @@ -521,6 +522,93 @@ func testCacheExportCacheDeletedContent(t *testing.T, sb integration.Sandbox) { } } +func testLocalCacheExportReset(t *testing.T, sb integration.Sandbox) { + integration.SkipOnPlatform(t, "windows") + workers.CheckFeatureCompat(t, sb, workers.FeatureCacheExport, workers.FeatureCacheBackendLocal) + c, err := New(sb.Context(), sb.Address()) + require.NoError(t, err) + defer c.Close() + + tmpdir := integration.Tmpdir(t) + cacheDir := filepath.Join(tmpdir.Name, "cache") + + // Build 1: export cache for a simple build + st1 := llb.Image("alpine:latest").Run(llb.Shlex(`sh -c "echo build1 > /out"`)).Root() + def1, err := llb.Scratch().File(llb.Copy(st1, "/out", "/out")).Marshal(sb.Context()) + require.NoError(t, err) + + _, err = c.Solve(sb.Context(), def1, SolveOpt{ + CacheExports: []CacheOptionsEntry{ + { + Type: "local", + Attrs: map[string]string{ + "dest": cacheDir, + }, + }, + }, + }, nil) + require.NoError(t, err) + + // Verify blobs were written + blobDir := filepath.Join(cacheDir, "blobs", "sha256") + entries1, err := os.ReadDir(blobDir) + require.NoError(t, err) + require.Greater(t, len(entries1), 0) + + // Build 2: different build, export with reset=true + st2 := llb.Image("alpine:latest").Run(llb.Shlex(`sh -c "echo build2-different > /out2"`)).Root() + def2, err := llb.Scratch().File(llb.Copy(st2, "/out2", "/out2")).Marshal(sb.Context()) + require.NoError(t, err) + + _, err = c.Solve(sb.Context(), def2, SolveOpt{ + CacheExports: []CacheOptionsEntry{ + { + Type: "local", + Attrs: map[string]string{ + "dest": cacheDir, + "reset": "true", + }, + }, + }, + }, nil) + require.NoError(t, err) + + // Read the manifest to determine which blobs should be referenced + dt, err := os.ReadFile(filepath.Join(cacheDir, "index.json")) + require.NoError(t, err) + + var index ocispecs.Index + err = json.Unmarshal(dt, &index) + require.NoError(t, err) + require.Len(t, index.Manifests, 1) + + dt, err = os.ReadFile(filepath.Join(blobDir, index.Manifests[0].Digest.Hex())) + require.NoError(t, err) + + var mfst ocispecs.Manifest + err = json.Unmarshal(dt, &mfst) + require.NoError(t, err) + + // Build the set of expected blobs: manifest + config + all layers + expectedBlobs := map[string]struct{}{ + index.Manifests[0].Digest.Hex(): {}, + mfst.Config.Digest.Hex(): {}, + } + for _, l := range mfst.Layers { + expectedBlobs[l.Digest.Hex()] = struct{}{} + } + + // Verify only referenced blobs remain + entries2, err := os.ReadDir(blobDir) + require.NoError(t, err) + + actualBlobs := map[string]struct{}{} + for _, e := range entries2 { + actualBlobs[e.Name()] = struct{}{} + } + require.Equal(t, expectedBlobs, actualBlobs, "after reset, only blobs referenced by the current manifest should remain") +} + func testBridgeNetworking(t *testing.T, sb integration.Sandbox) { if os.Getenv("BUILDKIT_RUN_NETWORK_INTEGRATION_TESTS") == "" { t.SkipNow() diff --git a/client/solve.go b/client/solve.go index da13d015c..cb281b090 100644 --- a/client/solve.go +++ b/client/solve.go @@ -8,10 +8,13 @@ import ( "maps" "os" "slices" + "strconv" "strings" + "sync" "time" "github.com/containerd/containerd/v2/core/content" + "github.com/containerd/containerd/v2/core/images" contentlocal "github.com/containerd/containerd/v2/plugins/content/local" controlapi "github.com/moby/buildkit/api/services/control" "github.com/moby/buildkit/client/llb" @@ -417,9 +420,55 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG } } } + // Reset cache stores that have reset=true — delete unreferenced blobs + for _, ref := range cacheOpt.storesToReset { + if err := resetCacheStore(ctx, ref.store, ref.path); err != nil { + bklog.G(ctx).WithError(err).Warn("failed to reset cache store") + } + } return res, nil } +// resetCacheStore deletes all blobs not referenced by any manifest in +// index.json. Referenced blobs are always preserved. +func resetCacheStore(ctx context.Context, cs content.Store, storePath string) error { + idx := ociindex.NewStoreIndex(storePath) + index, err := idx.Read() + if err != nil { + return errors.Wrap(err, "reset: failed to read index.json") + } + + var mu sync.Mutex + referenced := make(map[digest.Digest]struct{}) + childrenHandler := images.ChildrenHandler(cs) + handler := images.HandlerFunc(func(ctx context.Context, desc ocispecs.Descriptor) ([]ocispecs.Descriptor, error) { + mu.Lock() + referenced[desc.Digest] = struct{}{} + mu.Unlock() + return childrenHandler(ctx, desc) + }) + if err := images.Dispatch(ctx, handler, nil, index.Manifests...); err != nil { + return errors.Wrap(err, "reset: failed to collect referenced blobs") + } + + var toDelete []digest.Digest + if err := cs.Walk(ctx, func(info content.Info) error { + if _, ok := referenced[info.Digest]; !ok { + toDelete = append(toDelete, info.Digest) + } + return nil + }); err != nil { + return errors.Wrap(err, "reset: failed to walk content store") + } + + for _, dgst := range toDelete { + if err := cs.Delete(ctx, dgst); err != nil { + bklog.G(ctx).WithError(err).Warnf("reset: failed to delete blob %s", dgst) + } + } + return nil +} + func prepareSyncedFiles(def *llb.Definition, localMounts map[string]fsutil.FS) (filesync.StaticDirSource, error) { resetUIDAndGID := func(p string, st *fstypes.Stat) fsutil.MapResult { st.Uid = 0 @@ -464,10 +513,16 @@ func prepareSyncedFiles(def *llb.Definition, localMounts map[string]fsutil.FS) ( return result, nil } +type cacheStoreRef struct { + path string + store content.Store +} + type cacheOptions struct { options controlapi.CacheOptions contentStores map[string]content.Store // key: ID of content store ("local:" + csDir) storesToUpdate map[string]string // key: path to content store, value: tag + storesToReset []cacheStoreRef // cache stores with reset=true frontendAttrs map[string]string } @@ -476,6 +531,7 @@ func parseCacheOptions(ctx context.Context, isGateway bool, opt SolveOpt) (*cach cacheExports []*controlapi.CacheOptionsEntry cacheImports []*controlapi.CacheOptionsEntry ) + var storesToReset []cacheStoreRef contentStores := make(map[string]content.Store) storesToUpdate := make(map[string]string) frontendAttrs := make(map[string]string) @@ -500,6 +556,16 @@ func parseCacheOptions(ctx context.Context, isGateway bool, opt SolveOpt) (*cach } // TODO(AkihiroSuda): support custom index JSON path and tag storesToUpdate[csDir] = tag + + if v, ok := ex.Attrs["reset"]; ok { + b, err := strconv.ParseBool(v) + if err != nil { + return nil, errors.Wrapf(err, "failed to parse reset attribute") + } + if b { + storesToReset = append(storesToReset, cacheStoreRef{path: csDir, store: cs}) + } + } } if ex.Type == "registry" { regRef := ex.Attrs["ref"] @@ -579,6 +645,7 @@ func parseCacheOptions(ctx context.Context, isGateway bool, opt SolveOpt) (*cach }, contentStores: contentStores, storesToUpdate: storesToUpdate, + storesToReset: storesToReset, frontendAttrs: frontendAttrs, } return &res, nil diff --git a/client/solve_resetcache_test.go b/client/solve_resetcache_test.go new file mode 100644 index 000000000..352209b04 --- /dev/null +++ b/client/solve_resetcache_test.go @@ -0,0 +1,285 @@ +package client + +import ( + "bytes" + "context" + "encoding/json" + "testing" + + "github.com/containerd/containerd/v2/core/content" + "github.com/containerd/containerd/v2/core/images" + contentlocal "github.com/containerd/containerd/v2/plugins/content/local" + "github.com/moby/buildkit/client/ociindex" + digest "github.com/opencontainers/go-digest" + ocispecs "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/stretchr/testify/require" +) + +func writeBlob(ctx context.Context, t *testing.T, cs content.Store, data []byte) digest.Digest { + t.Helper() + dgst := digest.FromBytes(data) + desc := ocispecs.Descriptor{Digest: dgst, Size: int64(len(data))} + err := content.WriteBlob(ctx, cs, dgst.String(), bytes.NewReader(data), desc) + require.NoError(t, err) + return dgst +} + +func listDigests(ctx context.Context, t *testing.T, cs content.Store) map[digest.Digest]struct{} { + t.Helper() + result := map[digest.Digest]struct{}{} + err := cs.Walk(ctx, func(info content.Info) error { + result[info.Digest] = struct{}{} + return nil + }) + require.NoError(t, err) + return result +} + +// setupCacheStore creates a content store with an index.json and a manifest +// referencing the given config and layers. Returns the store path, content +// store, and manifest digest. +func setupCacheStore(ctx context.Context, t *testing.T, configData []byte, layersData [][]byte, tag string) (string, content.Store, digest.Digest) { + t.Helper() + dir := t.TempDir() + cs, err := contentlocal.NewStore(dir) + require.NoError(t, err) + + configDgst := writeBlob(ctx, t, cs, configData) + + layers := make([]ocispecs.Descriptor, len(layersData)) + for i, ld := range layersData { + dgst := writeBlob(ctx, t, cs, ld) + layers[i] = ocispecs.Descriptor{Digest: dgst, Size: int64(len(ld))} + } + + manifest := ocispecs.Manifest{ + MediaType: ocispecs.MediaTypeImageManifest, + Config: ocispecs.Descriptor{ + Digest: configDgst, + Size: int64(len(configData)), + MediaType: "application/vnd.buildkit.cacheconfig.v0", + }, + Layers: layers, + } + manifestData, err := json.Marshal(manifest) + require.NoError(t, err) + manifestDgst := writeBlob(ctx, t, cs, manifestData) + + idx := ociindex.NewStoreIndex(dir) + err = idx.Put(ocispecs.Descriptor{ + Digest: manifestDgst, + Size: int64(len(manifestData)), + MediaType: ocispecs.MediaTypeImageManifest, + }, ociindex.Tag(tag)) + require.NoError(t, err) + + return dir, cs, manifestDgst +} + +func TestResetCacheStoreImageManifest(t *testing.T) { + ctx := t.Context() + dir, cs, manifestDgst := setupCacheStore(ctx, t, + []byte(`{"test":"config"}`), + [][]byte{[]byte("layer1-data")}, + "latest", + ) + + // Write orphan blob + orphanDgst := writeBlob(ctx, t, cs, []byte("orphan-old-layer")) + + // Verify 4 blobs exist + require.Len(t, listDigests(ctx, t, cs), 4) + + err := resetCacheStore(ctx, cs, dir) + require.NoError(t, err) + + remaining := listDigests(ctx, t, cs) + require.Len(t, remaining, 3) // manifest + config + layer + require.Contains(t, remaining, manifestDgst) + require.NotContains(t, remaining, orphanDgst) +} + +func TestResetCacheStoreMultipleTags(t *testing.T) { + ctx := t.Context() + dir := t.TempDir() + cs, err := contentlocal.NewStore(dir) + require.NoError(t, err) + + // Manifest 1 (tag=v1) + config1Dgst := writeBlob(ctx, t, cs, []byte(`{"tag":"v1"}`)) + layer1Dgst := writeBlob(ctx, t, cs, []byte("layer1-only-in-v1")) + m1 := ocispecs.Manifest{ + MediaType: ocispecs.MediaTypeImageManifest, + Config: ocispecs.Descriptor{Digest: config1Dgst, Size: 12, MediaType: "application/vnd.buildkit.cacheconfig.v0"}, + Layers: []ocispecs.Descriptor{{Digest: layer1Dgst, Size: 17}}, + } + m1Data, err := json.Marshal(m1) + require.NoError(t, err) + m1Dgst := writeBlob(ctx, t, cs, m1Data) + + // Manifest 2 (tag=v2) + config2Dgst := writeBlob(ctx, t, cs, []byte(`{"tag":"v2"}`)) + layer2Dgst := writeBlob(ctx, t, cs, []byte("layer2-only-in-v2")) + m2 := ocispecs.Manifest{ + MediaType: ocispecs.MediaTypeImageManifest, + Config: ocispecs.Descriptor{Digest: config2Dgst, Size: 12, MediaType: "application/vnd.buildkit.cacheconfig.v0"}, + Layers: []ocispecs.Descriptor{{Digest: layer2Dgst, Size: 17}}, + } + m2Data, err := json.Marshal(m2) + require.NoError(t, err) + m2Dgst := writeBlob(ctx, t, cs, m2Data) + + // Orphan blob + orphanDgst := writeBlob(ctx, t, cs, []byte("orphan-blob")) + + // Write index.json with both tags + idx := ociindex.NewStoreIndex(dir) + require.NoError(t, idx.Put(ocispecs.Descriptor{Digest: m1Dgst, Size: int64(len(m1Data)), MediaType: ocispecs.MediaTypeImageManifest}, ociindex.Tag("v1"))) + require.NoError(t, idx.Put(ocispecs.Descriptor{Digest: m2Dgst, Size: int64(len(m2Data)), MediaType: ocispecs.MediaTypeImageManifest}, ociindex.Tag("v2"))) + + require.Len(t, listDigests(ctx, t, cs), 7) + + err = resetCacheStore(ctx, cs, dir) + require.NoError(t, err) + + remaining := listDigests(ctx, t, cs) + require.Len(t, remaining, 6) + require.Contains(t, remaining, m1Dgst) + require.Contains(t, remaining, config1Dgst) + require.Contains(t, remaining, layer1Dgst) + require.Contains(t, remaining, m2Dgst) + require.Contains(t, remaining, config2Dgst) + require.Contains(t, remaining, layer2Dgst) + require.NotContains(t, remaining, orphanDgst) +} + +func TestResetCacheStoreNoOrphans(t *testing.T) { + ctx := t.Context() + dir, cs, _ := setupCacheStore(ctx, t, + []byte(`{"test":"config"}`), + nil, + "latest", + ) + + err := resetCacheStore(ctx, cs, dir) + require.NoError(t, err) + + require.Len(t, listDigests(ctx, t, cs), 2) // manifest + config +} + +func TestResetCacheStoreNestedIndex(t *testing.T) { + ctx := t.Context() + dir := t.TempDir() + cs, err := contentlocal.NewStore(dir) + require.NoError(t, err) + + // Build a manifest with config + layer + configData := []byte(`{"nested":"config"}`) + layerData := []byte("nested-layer-data") + configDgst := writeBlob(ctx, t, cs, configData) + layerDgst := writeBlob(ctx, t, cs, layerData) + manifest := ocispecs.Manifest{ + MediaType: ocispecs.MediaTypeImageManifest, + Config: ocispecs.Descriptor{Digest: configDgst, Size: int64(len(configData)), MediaType: "application/vnd.buildkit.cacheconfig.v0"}, + Layers: []ocispecs.Descriptor{{Digest: layerDgst, Size: int64(len(layerData)), MediaType: ocispecs.MediaTypeImageLayerGzip}}, + } + manifestData, err := json.Marshal(manifest) + require.NoError(t, err) + manifestDgst := writeBlob(ctx, t, cs, manifestData) + + // A cache config blob referenced directly by the sub-index (no manifest wrapper) + cacheConfigData := []byte(`{"cache":"config"}`) + cacheConfigDgst := writeBlob(ctx, t, cs, cacheConfigData) + + // Build a sub-index that references both the manifest and the cache config directly + subIndex := ocispecs.Index{ + MediaType: ocispecs.MediaTypeImageIndex, + Manifests: []ocispecs.Descriptor{ + {Digest: manifestDgst, Size: int64(len(manifestData)), MediaType: ocispecs.MediaTypeImageManifest}, + {Digest: cacheConfigDgst, Size: int64(len(cacheConfigData)), MediaType: "application/vnd.buildkit.cacheconfig.v0"}, + }, + } + subIndexData, err := json.Marshal(subIndex) + require.NoError(t, err) + subIndexDgst := writeBlob(ctx, t, cs, subIndexData) + + orphanDgst := writeBlob(ctx, t, cs, []byte("orphan-data")) + + // Top-level index.json references the sub-index + idx := ociindex.NewStoreIndex(dir) + require.NoError(t, idx.Put(ocispecs.Descriptor{ + Digest: subIndexDgst, + Size: int64(len(subIndexData)), + MediaType: ocispecs.MediaTypeImageIndex, + }, ociindex.Tag("latest"))) + + require.Len(t, listDigests(ctx, t, cs), 6) // sub-index + manifest + config + layer + cacheConfig + orphan + + err = resetCacheStore(ctx, cs, dir) + require.NoError(t, err) + + remaining := listDigests(ctx, t, cs) + require.Len(t, remaining, 5) // sub-index + manifest + config + layer + cacheConfig + require.Contains(t, remaining, subIndexDgst) + require.Contains(t, remaining, manifestDgst) + require.Contains(t, remaining, configDgst) + require.Contains(t, remaining, layerDgst) + require.Contains(t, remaining, cacheConfigDgst) + require.NotContains(t, remaining, orphanDgst) +} + +func TestResetCacheStoreDockerMediaTypes(t *testing.T) { + ctx := t.Context() + dir := t.TempDir() + cs, err := contentlocal.NewStore(dir) + require.NoError(t, err) + + // Build a manifest using Docker schema2 media types + configData := []byte(`{"docker":"config"}`) + layerData := []byte("docker-layer") + configDgst := writeBlob(ctx, t, cs, configData) + layerDgst := writeBlob(ctx, t, cs, layerData) + manifest := ocispecs.Manifest{ + MediaType: images.MediaTypeDockerSchema2Manifest, + Config: ocispecs.Descriptor{Digest: configDgst, Size: int64(len(configData)), MediaType: images.MediaTypeDockerSchema2Config}, + Layers: []ocispecs.Descriptor{{Digest: layerDgst, Size: int64(len(layerData)), MediaType: images.MediaTypeDockerSchema2LayerGzip}}, + } + manifestData, err := json.Marshal(manifest) + require.NoError(t, err) + manifestDgst := writeBlob(ctx, t, cs, manifestData) + + // Build a manifest list using Docker schema2 media type + manifestList := ocispecs.Index{ + MediaType: images.MediaTypeDockerSchema2ManifestList, + Manifests: []ocispecs.Descriptor{{ + Digest: manifestDgst, + Size: int64(len(manifestData)), + MediaType: images.MediaTypeDockerSchema2Manifest, + }}, + } + manifestListData, err := json.Marshal(manifestList) + require.NoError(t, err) + manifestListDgst := writeBlob(ctx, t, cs, manifestListData) + + orphanDgst := writeBlob(ctx, t, cs, []byte("docker-orphan")) + + idx := ociindex.NewStoreIndex(dir) + require.NoError(t, idx.Put(ocispecs.Descriptor{ + Digest: manifestListDgst, + Size: int64(len(manifestListData)), + MediaType: images.MediaTypeDockerSchema2ManifestList, + }, ociindex.Tag("latest"))) + + require.Len(t, listDigests(ctx, t, cs), 5) + + err = resetCacheStore(ctx, cs, dir) + require.NoError(t, err) + + remaining := listDigests(ctx, t, cs) + require.Len(t, remaining, 4) // manifest list + manifest + config + layer + require.Contains(t, remaining, manifestListDgst) + require.Contains(t, remaining, manifestDgst) + require.Contains(t, remaining, configDgst) + require.Contains(t, remaining, layerDgst) + require.NotContains(t, remaining, orphanDgst) +}