diff --git a/exporter/containerimage/export.go b/exporter/containerimage/export.go index cdbe3469f..6be45b03f 100644 --- a/exporter/containerimage/export.go +++ b/exporter/containerimage/export.go @@ -8,11 +8,9 @@ import ( "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/images" "github.com/moby/buildkit/exporter" - "github.com/moby/buildkit/exporter/containerimage/exptypes" "github.com/moby/buildkit/session" "github.com/moby/buildkit/util/push" "github.com/pkg/errors" - "github.com/sirupsen/logrus" ) const ( @@ -67,8 +65,6 @@ func (e *imageExporter) Resolve(ctx context.Context, opt map[string]string) (exp return nil, errors.Wrapf(err, "non-bool value specified for %s", k) } i.insecure = b - case exptypes.ExporterImageConfigKey: - i.config = []byte(v) case ociTypes: if v == "" { i.ociTypes = true @@ -80,7 +76,10 @@ func (e *imageExporter) Resolve(ctx context.Context, opt map[string]string) (exp } i.ociTypes = b default: - logrus.Warnf("image exporter: unknown option %s", k) + if i.meta == nil { + i.meta = make(map[string][]byte) + i.meta[k] = []byte(v) + } } } return i, nil @@ -92,7 +91,7 @@ type imageExporterInstance struct { push bool insecure bool ociTypes bool - config []byte + meta map[string][]byte } func (e *imageExporterInstance) Name() string { @@ -100,12 +99,10 @@ func (e *imageExporterInstance) Name() string { } func (e *imageExporterInstance) Export(ctx context.Context, src exporter.Source) (map[string]string, error) { - ref := src.Ref - opt := src.Metadata - if config, ok := opt[exptypes.ExporterImageConfigKey]; ok { - e.config = config + for k, v := range e.meta { + src.Metadata[k] = v } - desc, err := e.opt.ImageWriter.Commit(ctx, ref, e.config, e.ociTypes) + desc, err := e.opt.ImageWriter.Commit(ctx, src, e.ociTypes) if err != nil { return nil, err } diff --git a/exporter/containerimage/writer.go b/exporter/containerimage/writer.go index 8570ebef2..40eeab6a9 100644 --- a/exporter/containerimage/writer.go +++ b/exporter/containerimage/writer.go @@ -13,6 +13,8 @@ import ( "github.com/containerd/containerd/images" "github.com/moby/buildkit/cache" "github.com/moby/buildkit/cache/blobs" + "github.com/moby/buildkit/exporter" + "github.com/moby/buildkit/exporter/containerimage/exptypes" "github.com/moby/buildkit/snapshot" "github.com/moby/buildkit/util/progress" "github.com/moby/buildkit/util/system" @@ -21,6 +23,7 @@ import ( ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" + "golang.org/x/sync/errgroup" ) const ( @@ -41,15 +44,138 @@ type ImageWriter struct { opt WriterOpt } -func (ic *ImageWriter) Commit(ctx context.Context, ref cache.ImmutableRef, config []byte, oci bool) (*ocispec.Descriptor, error) { - layersDone := oneOffProgress(ctx, "exporting layers") - diffPairs, err := blobs.GetDiffPairs(ctx, ic.opt.ContentStore, ic.opt.Snapshotter, ic.opt.Differ, ref, true) - if err != nil { - return nil, errors.Wrap(err, "failed calculaing diff pairs for exported snapshot") - } - layersDone(nil) +func (ic *ImageWriter) Commit(ctx context.Context, inp exporter.Source, oci bool) (*ocispec.Descriptor, error) { + platformsBytes, ok := inp.Metadata[exptypes.ExporterPlatformsKey] + if len(inp.Refs) > 0 && !ok { + return nil, errors.Errorf("unable to export multiple refs, missing platforms mapping") + } + + if len(inp.Refs) == 0 { + layers, err := ic.exportLayers(ctx, inp.Ref) + if err != nil { + return nil, err + } + return ic.commitDistributionManifest(ctx, inp.Ref, inp.Metadata[exptypes.ExporterImageConfigKey], layers[0], oci) + } + + var p exptypes.Platforms + if err := json.Unmarshal(platformsBytes, &p); err != nil { + return nil, errors.Wrapf(err, "failed to parse platforms passed to exporter") + } + + if len(p.Platforms) != len(inp.Refs) { + return nil, errors.Errorf("number of platforms does not match references %d %d", len(p.Platforms), len(inp.Refs)) + } + + refs := make([]cache.ImmutableRef, 0, len(inp.Refs)) + layersMap := make(map[string]int, len(inp.Refs)) + for id, r := range inp.Refs { + layersMap[id] = len(refs) + refs = append(refs, r) + } + + layers, err := ic.exportLayers(ctx, refs...) + if err != nil { + return nil, err + } + + idx := struct { + // MediaType is reserved in the OCI spec but + // excluded from go types. + MediaType string `json:"mediaType,omitempty"` + + ocispec.Index + }{ + MediaType: ocispec.MediaTypeImageIndex, + Index: ocispec.Index{ + Versioned: specs.Versioned{ + SchemaVersion: 2, + }, + }, + } + + if !oci { + idx.MediaType = images.MediaTypeDockerSchema2ManifestList + } + + labels := map[string]string{} + + for i, p := range p.Platforms { + r, ok := inp.Refs[p.ID] + if !ok { + return nil, errors.Errorf("failed to find ref for ID %s", p.ID) + } + config := inp.Metadata[fmt.Sprintf("%s/%s", exptypes.ExporterImageConfigKey, p.ID)] + + desc, err := ic.commitDistributionManifest(ctx, r, config, layers[layersMap[p.ID]], oci) + if err != nil { + return nil, err + } + dp := p.Platform + desc.Platform = &dp + idx.Manifests = append(idx.Manifests, *desc) + + labels[fmt.Sprintf("containerd.io/gc.ref.content.%d", i)] = desc.Digest.String() + } + + idxBytes, err := json.MarshalIndent(idx, "", " ") + if err != nil { + return nil, errors.Wrap(err, "failed to marshal index") + } + + idxDigest := digest.FromBytes(idxBytes) + idxDesc := ocispec.Descriptor{ + Digest: idxDigest, + Size: int64(len(idxBytes)), + MediaType: idx.MediaType, + } + idxDone := oneOffProgress(ctx, "exporting manifest list "+idxDigest.String()) + + if err := content.WriteBlob(ctx, ic.opt.ContentStore, idxDigest.String(), bytes.NewReader(idxBytes), idxDesc, content.WithLabels(labels)); err != nil { + return nil, idxDone(errors.Wrapf(err, "error writing manifest list blob %s", idxDigest)) + } + idxDone(nil) + + for _, desc := range idx.Manifests { + // delete manifest root. manifest will remain linked to the index + if err := ic.opt.ContentStore.Delete(context.TODO(), desc.Digest); err != nil { + return nil, errors.Wrap(err, "error removing manifest root") + } + } + + return &idxDesc, nil +} + +func (ic *ImageWriter) exportLayers(ctx context.Context, refs ...cache.ImmutableRef) ([][]blobs.DiffPair, error) { + eg, ctx := errgroup.WithContext(ctx) + layersDone := oneOffProgress(ctx, "exporting layers") + + out := make([][]blobs.DiffPair, len(refs)) + + for i, ref := range refs { + func(i int, ref cache.ImmutableRef) { + eg.Go(func() error { + diffPairs, err := blobs.GetDiffPairs(ctx, ic.opt.ContentStore, ic.opt.Snapshotter, ic.opt.Differ, ref, true) + if err != nil { + return errors.Wrap(err, "failed calculaing diff pairs for exported snapshot") + } + out[i] = diffPairs + return nil + }) + }(i, ref) + } + + if err := layersDone(eg.Wait()); err != nil { + return nil, err + } + + return out, nil +} + +func (ic *ImageWriter) commitDistributionManifest(ctx context.Context, ref cache.ImmutableRef, config []byte, layers []blobs.DiffPair, oci bool) (*ocispec.Descriptor, error) { if len(config) == 0 { + var err error config, err = emptyImageConfig() if err != nil { return nil, err @@ -61,7 +187,7 @@ func (ic *ImageWriter) Commit(ctx context.Context, ref cache.ImmutableRef, confi return nil, err } - diffPairs, history = normalizeLayersAndHistory(diffPairs, history, ref) + diffPairs, history := normalizeLayersAndHistory(layers, history, ref) config, err = patchImageConfig(config, diffPairs, history) if err != nil { diff --git a/exporter/oci/export.go b/exporter/oci/export.go index 70256347b..5e5e41ad8 100644 --- a/exporter/oci/export.go +++ b/exporter/oci/export.go @@ -10,14 +10,12 @@ import ( "github.com/docker/distribution/reference" "github.com/moby/buildkit/exporter" "github.com/moby/buildkit/exporter/containerimage" - "github.com/moby/buildkit/exporter/containerimage/exptypes" "github.com/moby/buildkit/session" "github.com/moby/buildkit/session/filesync" "github.com/moby/buildkit/util/dockerexporter" "github.com/moby/buildkit/util/progress" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" - "github.com/sirupsen/logrus" ) type ExporterVariant string @@ -62,8 +60,6 @@ func (e *imageExporter) Resolve(ctx context.Context, opt map[string]string) (exp i := &imageExporterInstance{imageExporter: e, caller: caller} for k, v := range opt { switch k { - case exptypes.ExporterImageConfigKey: - i.config = []byte(v) case keyImageName: parsed, err := reference.ParseNormalizedNamed(v) if err != nil { @@ -82,7 +78,10 @@ func (e *imageExporter) Resolve(ctx context.Context, opt map[string]string) (exp } *ot = b default: - logrus.Warnf("oci exporter: unknown option %s", k) + if i.meta == nil { + i.meta = make(map[string][]byte) + i.meta[k] = []byte(v) + } } } if ot == nil { @@ -95,7 +94,7 @@ func (e *imageExporter) Resolve(ctx context.Context, opt map[string]string) (exp type imageExporterInstance struct { *imageExporter - config []byte + meta map[string][]byte caller session.Caller name string ociTypes bool @@ -106,12 +105,15 @@ func (e *imageExporterInstance) Name() string { } func (e *imageExporterInstance) Export(ctx context.Context, src exporter.Source) (map[string]string, error) { - ref := src.Ref - opt := src.Metadata - if config, ok := opt[exptypes.ExporterImageConfigKey]; ok { - e.config = config + if e.opt.Variant == VariantDocker && len(src.Refs) > 0 { + return nil, errors.Errorf("docker exporter does not currently support exporting manifest lists") } - desc, err := e.opt.ImageWriter.Commit(ctx, ref, e.config, e.ociTypes) + + for k, v := range e.meta { + src.Metadata[k] = v + } + + desc, err := e.opt.ImageWriter.Commit(ctx, src, e.ociTypes) if err != nil { return nil, err } diff --git a/frontend/dockerfile/dockerfile2llb/convert.go b/frontend/dockerfile/dockerfile2llb/convert.go index 3ecaee636..288f0d9a1 100644 --- a/frontend/dockerfile/dockerfile2llb/convert.go +++ b/frontend/dockerfile/dockerfile2llb/convert.go @@ -217,6 +217,7 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State, } else { d.state = llb.Image(d.stage.BaseName, dfCmd(d.stage.SourceCode), llb.Platform(*platform)) } + d.platform = platform return nil }) }(i, d) @@ -236,6 +237,7 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State, } if d.base != nil { d.state = d.base.state + d.platform = d.base.platform d.image = clone(d.base.image) } @@ -646,6 +648,10 @@ func dispatchCopy(d *dispatchState, c instructions.SourcesAndDest, sourceState l run := img.Run(append(runOpt, mounts...)...) d.state = run.AddMount("/dest", d.state).Platform(opt.targetPlatform) + if d.platform != nil { + d.state = d.state.Platform(*d.platform) + } + return commitToHistory(&d.image, commitMessage.String(), true, &d.state) } diff --git a/source/containerimage/pull.go b/source/containerimage/pull.go index 706a57665..728ffa3c7 100644 --- a/source/containerimage/pull.go +++ b/source/containerimage/pull.go @@ -58,7 +58,11 @@ func (is *imageSource) ResolveImageConfig(ctx context.Context, ref string, platf dgst digest.Digest dt []byte } - res, err := is.g.Do(ctx, ref, func(ctx context.Context) (interface{}, error) { + key := ref + if platform != nil { + key += platforms.Format(*platform) + } + res, err := is.g.Do(ctx, key, func(ctx context.Context) (interface{}, error) { dgst, dt, err := imageutil.Config(ctx, ref, pull.NewResolver(ctx, is.SessionManager, is.ImageStore), is.ContentStore, platform) if err != nil { return nil, err diff --git a/util/dockerexporter/dockerexporter.go b/util/dockerexporter/dockerexporter.go index da8cce781..f135febcf 100644 --- a/util/dockerexporter/dockerexporter.go +++ b/util/dockerexporter/dockerexporter.go @@ -10,7 +10,6 @@ import ( "github.com/containerd/containerd/content" "github.com/containerd/containerd/images" - "github.com/containerd/containerd/platforms" ocispecs "github.com/opencontainers/image-spec/specs-go" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" @@ -51,8 +50,6 @@ func (de *DockerExporter) Export(ctx context.Context, store content.Provider, de // Get all the children for a descriptor childrenHandler := images.ChildrenHandler(store) - // Filter the childen by the platform - childrenHandler = images.FilterPlatforms(childrenHandler, platforms.Default()) handlers := images.Handlers( childrenHandler, diff --git a/util/pull/pull.go b/util/pull/pull.go index bd7d90641..622dd5212 100644 --- a/util/pull/pull.go +++ b/util/pull/pull.go @@ -182,7 +182,7 @@ func (p *Puller) Pull(ctx context.Context) (*Pulled, error) { var layerBlobs []ocispec.Descriptor for _, j := range usedBlobs { switch j.MediaType { - case ocispec.MediaTypeImageLayer, images.MediaTypeDockerSchema2Layer, ocispec.MediaTypeImageLayerGzip, images.MediaTypeDockerSchema2LayerGzip: + case ocispec.MediaTypeImageLayer, images.MediaTypeDockerSchema2Layer, ocispec.MediaTypeImageLayerGzip, images.MediaTypeDockerSchema2LayerGzip, images.MediaTypeDockerSchema2LayerForeign, images.MediaTypeDockerSchema2LayerForeignGzip: layerBlobs = append(layerBlobs, j) default: notLayerBlobs = append(notLayerBlobs, j)