client: add referrers support to client

Allows management of referrer objects when performing
pull, archive export and archive import.

Referrer objects are linked to their subjects via GC
labels. The label is based on sha256 checksum of the
object instead of incremental numbers as referrers are
not immutable and don't have any strict order.

In OCI-layout referrers that are not already in the exported
tree are added to the main index.json with
io.containerd.manifest.subject annotation.

On import such descriptors with that annotation
do not create digest-based images in the image store.

Note that this does not mean all the referrer objects in
the registry are now pulled/exported/imported by default.
The caller of the client pkg functions can choose which
referrer objects should also be handled.

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>

client/client.go

@@ -466,6 +466,10 @@ type RemoteContext struct {
 	// ChildLabelMap sets the labels used to reference child objects in the content
 	// store. By default, all GC reference labels will be set for all fetched content.
 	ChildLabelMap func(ocispec.Descriptor) []string
+
+	// ReferrersProvider provides a way to lookup additional referrers for a given
+	// descriptor. For example pulling them with remotes.ReferrerFetcher.
+	ReferrersProvider content.ReferrersProvider
 }
 
 func defaultRemoteContext() *RemoteContext {

client/client_opts.go

@@ -19,13 +19,13 @@ package client
 import (
 	"time"
 
+	"github.com/containerd/containerd/v2/core/content"
 	"github.com/containerd/containerd/v2/core/images"
 	"github.com/containerd/containerd/v2/core/remotes"
 	"github.com/containerd/containerd/v2/core/snapshots"
 	"github.com/containerd/platforms"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"golang.org/x/sync/semaphore"
-
 	"google.golang.org/grpc"
 )
 
@@ -284,3 +284,11 @@ func WithAllMetadata() RemoteOpt {
 		return nil
 	}
 }
+
+// WithReferrersProvider sets a referrers provider to resolve referrer objects.
+func WithReferrersProvider(r content.ReferrersProvider) RemoteOpt {
+	return func(_ *Client, c *RemoteContext) error {
+		c.ReferrersProvider = r
+		return nil
+	}
+}

client/import.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"io"
 
+	"github.com/containerd/containerd/v2/core/content"
 	"github.com/containerd/containerd/v2/core/images"
 	"github.com/containerd/containerd/v2/core/images/archive"
 	"github.com/containerd/errdefs"
@@ -39,6 +40,7 @@ type importOpts struct {
 	discardLayers   bool
 	skipMissing     bool
 	imageLabels     map[string]string
+	referrers       content.ReferrersProvider
 }
 
 // ImportOpt allows the caller to specify import specific options
@@ -132,6 +134,15 @@ func WithSkipMissing() ImportOpt {
 	}
 }
 
+// WithImportReferrers allows to set a custom referrers provider
+// for setting referrer objects during import.
+func WithImportReferrers(r content.ReferrersProvider) ImportOpt {
+	return func(c *importOpts) error {
+		c.referrers = r
+		return nil
+	}
+}
+
 // Import imports an image from a Tar stream using reader.
 // Caller needs to specify importer. Future version may use oci.v1 as the default.
 // Note that unreferenced blobs may be imported to the content store as well.
@@ -202,6 +213,10 @@ func (c *Client) Import(ctx context.Context, reader io.Reader, opts ...ImportOpt
 					Target: m,
 				})
 			}
+			// don't create images for referrer descriptors
+			if _, ok := m.Annotations[images.AnnotationManifestSubject]; ok {
+				continue
+			}
 			if iopts.skipDgstRef != nil {
 				if iopts.skipDgstRef(name) {
 					continue
@@ -222,6 +237,9 @@ func (c *Client) Import(ctx context.Context, reader io.Reader, opts ...ImportOpt
 	}
 
 	handler = images.FilterPlatforms(handler, platformMatcher)
+	if iopts.referrers != nil {
+		handler = images.SetReferrers(iopts.referrers, handler)
+	}
 	if iopts.discardLayers {
 		handler = images.SetChildrenMappedLabels(cs, handler, images.ChildGCLabelsFilterLayers)
 	} else {

client/pull.go

@@ -207,6 +207,9 @@ func (c *Client) fetch(ctx context.Context, rCtx *RemoteContext, ref string, lim
 	}
 	// Get all the children for a descriptor
 	childrenHandler := images.ChildrenHandler(store)
+	if rCtx.ReferrersProvider != nil {
+		childrenHandler = images.SetReferrers(rCtx.ReferrersProvider, childrenHandler)
+	}
 	// Set any children labels for that content
 	childrenHandler = images.SetChildrenMappedLabels(store, childrenHandler, rCtx.ChildLabelMap)
 	if rCtx.AllMetadata {

core/content/content.go

@@ -170,6 +170,11 @@ type Syncer interface {
 	Sync() error
 }
 
+// ReferrersProvider handles looking up additional referrer objects for a given descriptor.
+type ReferrersProvider interface {
+	Referrers(context.Context, ocispec.Descriptor) ([]ocispec.Descriptor, error)
+}
+
 // Opt is used to alter the mutable properties of content
 type Opt func(*Info) error
 

core/images/annotations.go

@@ -20,4 +20,10 @@ const (
 	// AnnotationImageName is an annotation on a Descriptor in an index.json
 	// containing the `Name` value as used by an `Image` struct
 	AnnotationImageName = "io.containerd.image.name"
+
+	// AnnotationManifestSubject is an annotation on a Descriptor that means
+	// that current descriptor is a referrer to the subject manifest.
+	// If descriptor in image.json has this annotation, it will not create
+	// a new image.
+	AnnotationManifestSubject = "io.containerd.manifest.subject"
 )

core/images/archive/exporter.go

@@ -23,6 +23,7 @@ import (
 	"fmt"
 	"io"
 	"path"
+	"slices"
 	"sort"
 	"strings"
 
@@ -43,6 +44,7 @@ type exportOptions struct {
 	allPlatforms       bool
 	skipDockerManifest bool
 	blobRecordOptions  blobRecordOptions
+	referrers          content.ReferrersProvider
 }
 
 // ExportOpt defines options for configuring exported descriptors
@@ -143,6 +145,15 @@ func WithSkipNonDistributableBlobs() ExportOpt {
 	return WithBlobFilter(f)
 }
 
+// WithReferrersProvider sets a referrers provider to resolve referrer objects
+// that are included in the exported archive.
+func WithReferrersProvider(p content.ReferrersProvider) ExportOpt {
+	return func(ctx context.Context, o *exportOptions) error {
+		o.referrers = p
+		return nil
+	}
+}
+
 // WithSkipMissing excludes blobs referenced by manifests if not all blobs
 // would be included in the archive.
 // The manifest itself is excluded only if it's not present locally.
@@ -242,7 +253,7 @@ func Export(ctx context.Context, store content.InfoReaderProvider, writer io.Wri
 			mt, ok := dManifests[desc.Digest]
 			if !ok {
 				// TODO(containerd): Skip if already added
-				r, err := getRecords(ctx, store, desc, algorithms, &eo.blobRecordOptions)
+				r, err := getRecords(ctx, store, desc, algorithms, &eo.blobRecordOptions, eo.referrers)
 				if err != nil {
 					return err
 				}
@@ -286,7 +297,7 @@ func Export(ctx context.Context, store content.InfoReaderProvider, writer io.Wri
 						}
 					}
 
-					r, err := getRecords(ctx, store, m, algorithms, &eo.blobRecordOptions)
+					r, err := getRecords(ctx, store, m, algorithms, &eo.blobRecordOptions, eo.referrers)
 					if err != nil {
 						return err
 					}
@@ -327,7 +338,7 @@ func Export(ctx context.Context, store content.InfoReaderProvider, writer io.Wri
 		}
 	}
 
-	records = append(records, ociIndexRecord(manifests))
+	records = append(records, ociIndexRecord(slices.Concat(manifests, filterReferrers(records))))
 
 	if !eo.skipDockerManifest && len(dManifests) > 0 {
 		tr, err := manifestsRecord(ctx, store, dManifests)
@@ -350,7 +361,7 @@ func Export(ctx context.Context, store content.InfoReaderProvider, writer io.Wri
 	return writeTar(ctx, tw, records)
 }
 
-func getRecords(ctx context.Context, store content.Provider, desc ocispec.Descriptor, algorithms map[string]struct{}, brOpts *blobRecordOptions) ([]tarRecord, error) {
+func getRecords(ctx context.Context, store content.Provider, desc ocispec.Descriptor, algorithms map[string]struct{}, brOpts *blobRecordOptions, referrers content.ReferrersProvider) ([]tarRecord, error) {
 	var records []tarRecord
 	exportHandler := func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
 		if err := desc.Digest.Validate(); err != nil {
@@ -366,6 +377,36 @@ func getRecords(ctx context.Context, store content.Provider, desc ocispec.Descri
 		childrenHandler = images.ChildrenHandler(store)
 	}
 
+	if referrers != nil {
+		ch := childrenHandler
+		childrenHandler = func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
+			children, err := ch(ctx, desc)
+			if err != nil {
+				return nil, err
+			}
+			if !images.IsManifestType(desc.MediaType) && !images.IsIndexType(desc.MediaType) {
+				return children, nil
+			}
+
+			refs, err := referrers.Referrers(ctx, desc)
+			if err != nil {
+				if errdefs.IsNotFound(err) {
+					return children, nil
+				}
+				return nil, err
+			}
+			for _, r := range refs {
+				if r.Annotations == nil {
+					r.Annotations = map[string]string{}
+				}
+				r.Annotations[images.AnnotationManifestSubject] = desc.Digest.String()
+				children = append(children, r)
+			}
+
+			return children, nil
+		}
+	}
+
 	handlers := images.Handlers(
 		childrenHandler,
 		images.HandlerFunc(exportHandler),
@@ -380,9 +421,38 @@ func getRecords(ctx context.Context, store content.Provider, desc ocispec.Descri
 	return records, nil
 }
 
+// filterReferrers filters records with referrer annotations that are not
+// exported in another way already
+func filterReferrers(records []tarRecord) []ocispec.Descriptor {
+	inTree := map[digest.Digest]struct{}{}
+	referrersMap := map[digest.Digest]ocispec.Descriptor{}
+	for _, r := range records {
+		if r.Descriptor != nil {
+			if _, ok := r.Descriptor.Annotations[images.AnnotationManifestSubject]; ok {
+				referrersMap[r.Descriptor.Digest] = *r.Descriptor
+			} else {
+				inTree[r.Descriptor.Digest] = struct{}{}
+			}
+		}
+	}
+
+	var referrers []ocispec.Descriptor
+	for dgst, desc := range referrersMap {
+		if _, ok := inTree[dgst]; ok {
+			continue
+		}
+		referrers = append(referrers, desc)
+	}
+	slices.SortFunc(referrers, func(a, b ocispec.Descriptor) int {
+		return strings.Compare(a.Digest.String(), b.Digest.String())
+	})
+	return referrers
+}
+
 type tarRecord struct {
-	Header *tar.Header
-	CopyTo func(context.Context, io.Writer) (int64, error)
+	Header     *tar.Header
+	CopyTo     func(context.Context, io.Writer) (int64, error)
+	Descriptor *ocispec.Descriptor
 }
 
 type blobRecordOptions struct {
@@ -395,6 +465,7 @@ func blobRecord(cs content.Provider, desc ocispec.Descriptor, opts *blobRecordOp
 		return tarRecord{}
 	}
 	return tarRecord{
+		Descriptor: &desc,
 		Header: &tar.Header{
 			Name:     path.Join(ocispec.ImageBlobsDir, desc.Digest.Algorithm().String(), desc.Digest.Encoded()),
 			Mode:     0444,

core/images/handlers.go

@@ -20,7 +20,9 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"slices"
 	"sort"
+	"strings"
 
 	"github.com/containerd/containerd/v2/core/content"
 	"github.com/containerd/errdefs"
@@ -198,6 +200,35 @@ func ChildrenHandler(provider content.Provider) HandlerFunc {
 	}
 }
 
+// SetReferrers is a handler wrapper which adds referrer descriptors
+// from the provided ReferrersProvider and adds them to the children
+// returned by the handler. The referrers will have a container-specific
+// annotation added to indicate the subject descriptor.
+func SetReferrers(refProvider content.ReferrersProvider, f HandlerFunc) HandlerFunc {
+	return func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
+		children, err := f(ctx, desc)
+		if err != nil {
+			return children, err
+		}
+		if !IsManifestType(desc.MediaType) && !IsIndexType(desc.MediaType) {
+			return children, nil
+		}
+		refs, err := refProvider.Referrers(ctx, desc)
+		if err != nil {
+			return children, err
+		}
+		children = slices.Grow(children, len(refs))
+		for _, ref := range refs {
+			if ref.Annotations == nil {
+				ref.Annotations = map[string]string{}
+			}
+			ref.Annotations[AnnotationManifestSubject] = desc.Digest.String()
+			children = append(children, ref)
+		}
+		return children, nil
+	}
+}
+
 // SetChildrenLabels is a handler wrapper which sets labels for the content on
 // the children returned by the handler and passes through the children.
 // Must follow a handler that returns the children to be labeled.
@@ -235,7 +266,9 @@ func SetChildrenMappedLabels(manager content.Manager, f HandlerFunc, labelMap fu
 				for _, key := range labelKeys {
 					idx := keys[key]
 					keys[key] = idx + 1
-					if idx > 0 || key[len(key)-1] == '.' {
+					if strings.HasSuffix(key, ".sha256.") {
+						key = fmt.Sprintf("%s%s", key, ch.Digest.Hex()[:12])
+					} else if idx > 0 || key[len(key)-1] == '.' {
 						key = fmt.Sprintf("%s%d", key, idx)
 					}
 

core/images/mediatypes.go

@@ -208,6 +208,9 @@ func IsAttestationType(mt string) bool {
 
 // ChildGCLabels returns the label for a given descriptor to reference it
 func ChildGCLabels(desc ocispec.Descriptor) []string {
+	if _, ok := desc.Annotations[AnnotationManifestSubject]; ok {
+		return []string{"containerd.io/gc.ref.content.referrer.sha256."}
+	}
 	mt := desc.MediaType
 	if IsKnownConfig(mt) {
 		return []string{"containerd.io/gc.ref.content.config"}