dilim/buildkit
1
0
Fork 0
mirror of https://github.com/moby/buildkit.git synced 2026-08-07 16:20:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

export: manifest list support

Browse Source 
Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
This commit is contained in:
Tonis Tiigi
2018-07-13 11:28:36 -07:00
parent 8c065401f2
commit 6be1257f5d
7 changed files with 167 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
exporter/containerimage/export.go
View File

@@ -8,11 +8,9 @@ import (
"github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/images"
"github.com/moby/buildkit/exporter"
"github.com/moby/buildkit/exporter/containerimage/exptypes"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/util/push"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)
const (
@@ -67,8 +65,6 @@ func (e *imageExporter) Resolve(ctx context.Context, opt map[string]string) (exp
return nil, errors.Wrapf(err, "non-bool value specified for %s", k)
}
i.insecure = b
case exptypes.ExporterImageConfigKey:
i.config = []byte(v)
case ociTypes:
if v == "" {
i.ociTypes = true
@@ -80,7 +76,10 @@ func (e *imageExporter) Resolve(ctx context.Context, opt map[string]string) (exp
}
i.ociTypes = b
default:
logrus.Warnf("image exporter: unknown option %s", k)
if i.meta == nil {
i.meta = make(map[string][]byte)
i.meta[k] = []byte(v)
}
}
}
return i, nil
@@ -92,7 +91,7 @@ type imageExporterInstance struct {
push bool
insecure bool
ociTypes bool
config []byte
meta map[string][]byte
}
func (e *imageExporterInstance) Name() string {
@@ -100,12 +99,10 @@ func (e *imageExporterInstance) Name() string {
}
func (e *imageExporterInstance) Export(ctx context.Context, src exporter.Source) (map[string]string, error) {
ref := src.Ref
opt := src.Metadata
if config, ok := opt[exptypes.ExporterImageConfigKey]; ok {
e.config = config
for k, v := range e.meta {
src.Metadata[k] = v
}
desc, err := e.opt.ImageWriter.Commit(ctx, ref, e.config, e.ociTypes)
desc, err := e.opt.ImageWriter.Commit(ctx, src, e.ociTypes)
if err != nil {
return nil, err
}

142
exporter/containerimage/writer.go
View File

@@ -13,6 +13,8 @@ import (
"github.com/containerd/containerd/images"
"github.com/moby/buildkit/cache"
"github.com/moby/buildkit/cache/blobs"
"github.com/moby/buildkit/exporter"
"github.com/moby/buildkit/exporter/containerimage/exptypes"
"github.com/moby/buildkit/snapshot"
"github.com/moby/buildkit/util/progress"
"github.com/moby/buildkit/util/system"
@@ -21,6 +23,7 @@ import (
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"golang.org/x/sync/errgroup"
)
const (
@@ -41,15 +44,138 @@ type ImageWriter struct {
opt WriterOpt
}
func (ic *ImageWriter) Commit(ctx context.Context, ref cache.ImmutableRef, config []byte, oci bool) (*ocispec.Descriptor, error) {
layersDone := oneOffProgress(ctx, "exporting layers")
diffPairs, err := blobs.GetDiffPairs(ctx, ic.opt.ContentStore, ic.opt.Snapshotter, ic.opt.Differ, ref, true)
if err != nil {
return nil, errors.Wrap(err, "failed calculaing diff pairs for exported snapshot")
}
layersDone(nil)
func (ic *ImageWriter) Commit(ctx context.Context, inp exporter.Source, oci bool) (*ocispec.Descriptor, error) {
platformsBytes, ok := inp.Metadata[exptypes.ExporterPlatformsKey]
if len(inp.Refs) > 0 && !ok {
return nil, errors.Errorf("unable to export multiple refs, missing platforms mapping")
}
if len(inp.Refs) == 0 {
layers, err := ic.exportLayers(ctx, inp.Ref)
if err != nil {
return nil, err
}
return ic.commitDistributionManifest(ctx, inp.Ref, inp.Metadata[exptypes.ExporterImageConfigKey], layers[0], oci)
}
var p exptypes.Platforms
if err := json.Unmarshal(platformsBytes, &p); err != nil {
return nil, errors.Wrapf(err, "failed to parse platforms passed to exporter")
}
if len(p.Platforms) != len(inp.Refs) {
return nil, errors.Errorf("number of platforms does not match references %d %d", len(p.Platforms), len(inp.Refs))
}
refs := make([]cache.ImmutableRef, 0, len(inp.Refs))
layersMap := make(map[string]int, len(inp.Refs))
for id, r := range inp.Refs {
layersMap[id] = len(refs)
refs = append(refs, r)
}
layers, err := ic.exportLayers(ctx, refs...)
if err != nil {
return nil, err
}
idx := struct {
// MediaType is reserved in the OCI spec but
// excluded from go types.
MediaType string `json:"mediaType,omitempty"`
ocispec.Index
}{
MediaType: ocispec.MediaTypeImageIndex,
Index: ocispec.Index{
Versioned: specs.Versioned{
SchemaVersion: 2,
},
},
}
if !oci {
idx.MediaType = images.MediaTypeDockerSchema2ManifestList
}
labels := map[string]string{}
for i, p := range p.Platforms {
r, ok := inp.Refs[p.ID]
if !ok {
return nil, errors.Errorf("failed to find ref for ID %s", p.ID)
}
config := inp.Metadata[fmt.Sprintf("%s/%s", exptypes.ExporterImageConfigKey, p.ID)]
desc, err := ic.commitDistributionManifest(ctx, r, config, layers[layersMap[p.ID]], oci)
if err != nil {
return nil, err
}
dp := p.Platform
desc.Platform = &dp
idx.Manifests = append(idx.Manifests, *desc)
labels[fmt.Sprintf("containerd.io/gc.ref.content.%d", i)] = desc.Digest.String()
}
idxBytes, err := json.MarshalIndent(idx, "", " ")
if err != nil {
return nil, errors.Wrap(err, "failed to marshal index")
}
idxDigest := digest.FromBytes(idxBytes)
idxDesc := ocispec.Descriptor{
Digest: idxDigest,
Size: int64(len(idxBytes)),
MediaType: idx.MediaType,
}
idxDone := oneOffProgress(ctx, "exporting manifest list "+idxDigest.String())
if err := content.WriteBlob(ctx, ic.opt.ContentStore, idxDigest.String(), bytes.NewReader(idxBytes), idxDesc, content.WithLabels(labels)); err != nil {
return nil, idxDone(errors.Wrapf(err, "error writing manifest list blob %s", idxDigest))
}
idxDone(nil)
for _, desc := range idx.Manifests {
// delete manifest root. manifest will remain linked to the index
if err := ic.opt.ContentStore.Delete(context.TODO(), desc.Digest); err != nil {
return nil, errors.Wrap(err, "error removing manifest root")
}
}
return &idxDesc, nil
}
func (ic *ImageWriter) exportLayers(ctx context.Context, refs ...cache.ImmutableRef) ([][]blobs.DiffPair, error) {
eg, ctx := errgroup.WithContext(ctx)
layersDone := oneOffProgress(ctx, "exporting layers")
out := make([][]blobs.DiffPair, len(refs))
for i, ref := range refs {
func(i int, ref cache.ImmutableRef) {
eg.Go(func() error {
diffPairs, err := blobs.GetDiffPairs(ctx, ic.opt.ContentStore, ic.opt.Snapshotter, ic.opt.Differ, ref, true)
if err != nil {
return errors.Wrap(err, "failed calculaing diff pairs for exported snapshot")
}
out[i] = diffPairs
return nil
})
}(i, ref)
}
if err := layersDone(eg.Wait()); err != nil {
return nil, err
}
return out, nil
}
func (ic *ImageWriter) commitDistributionManifest(ctx context.Context, ref cache.ImmutableRef, config []byte, layers []blobs.DiffPair, oci bool) (*ocispec.Descriptor, error) {
if len(config) == 0 {
var err error
config, err = emptyImageConfig()
if err != nil {
return nil, err
@@ -61,7 +187,7 @@ func (ic *ImageWriter) Commit(ctx context.Context, ref cache.ImmutableRef, confi
return nil, err
}
diffPairs, history = normalizeLayersAndHistory(diffPairs, history, ref)
diffPairs, history := normalizeLayersAndHistory(layers, history, ref)
config, err = patchImageConfig(config, diffPairs, history)
if err != nil {

24
exporter/oci/export.go
View File

@@ -10,14 +10,12 @@ import (
"github.com/docker/distribution/reference"
"github.com/moby/buildkit/exporter"
"github.com/moby/buildkit/exporter/containerimage"
"github.com/moby/buildkit/exporter/containerimage/exptypes"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/session/filesync"
"github.com/moby/buildkit/util/dockerexporter"
"github.com/moby/buildkit/util/progress"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)
type ExporterVariant string
@@ -62,8 +60,6 @@ func (e *imageExporter) Resolve(ctx context.Context, opt map[string]string) (exp
i := &imageExporterInstance{imageExporter: e, caller: caller}
for k, v := range opt {
switch k {
case exptypes.ExporterImageConfigKey:
i.config = []byte(v)
case keyImageName:
parsed, err := reference.ParseNormalizedNamed(v)
if err != nil {
@@ -82,7 +78,10 @@ func (e *imageExporter) Resolve(ctx context.Context, opt map[string]string) (exp
}
*ot = b
default:
logrus.Warnf("oci exporter: unknown option %s", k)
if i.meta == nil {
i.meta = make(map[string][]byte)
i.meta[k] = []byte(v)
}
}
}
if ot == nil {
@@ -95,7 +94,7 @@ func (e *imageExporter) Resolve(ctx context.Context, opt map[string]string) (exp
type imageExporterInstance struct {
*imageExporter
config []byte
meta map[string][]byte
caller session.Caller
name string
ociTypes bool
@@ -106,12 +105,15 @@ func (e *imageExporterInstance) Name() string {
}
func (e *imageExporterInstance) Export(ctx context.Context, src exporter.Source) (map[string]string, error) {
ref := src.Ref
opt := src.Metadata
if config, ok := opt[exptypes.ExporterImageConfigKey]; ok {
e.config = config
if e.opt.Variant == VariantDocker && len(src.Refs) > 0 {
return nil, errors.Errorf("docker exporter does not currently support exporting manifest lists")
}
desc, err := e.opt.ImageWriter.Commit(ctx, ref, e.config, e.ociTypes)
for k, v := range e.meta {
src.Metadata[k] = v
}
desc, err := e.opt.ImageWriter.Commit(ctx, src, e.ociTypes)
if err != nil {
return nil, err
}

6
frontend/dockerfile/dockerfile2llb/convert.go
View File

@@ -217,6 +217,7 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
} else {
d.state = llb.Image(d.stage.BaseName, dfCmd(d.stage.SourceCode), llb.Platform(*platform))
}
d.platform = platform
return nil
})
}(i, d)
@@ -236,6 +237,7 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
}
if d.base != nil {
d.state = d.base.state
d.platform = d.base.platform
d.image = clone(d.base.image)
}
@@ -646,6 +648,10 @@ func dispatchCopy(d *dispatchState, c instructions.SourcesAndDest, sourceState l
run := img.Run(append(runOpt, mounts...)...)
d.state = run.AddMount("/dest", d.state).Platform(opt.targetPlatform)
if d.platform != nil {
d.state = d.state.Platform(*d.platform)
}
return commitToHistory(&d.image, commitMessage.String(), true, &d.state)
}

6
source/containerimage/pull.go
View File

@@ -58,7 +58,11 @@ func (is *imageSource) ResolveImageConfig(ctx context.Context, ref string, platf
dgst digest.Digest
dt []byte
}
res, err := is.g.Do(ctx, ref, func(ctx context.Context) (interface{}, error) {
key := ref
if platform != nil {
key += platforms.Format(*platform)
}
res, err := is.g.Do(ctx, key, func(ctx context.Context) (interface{}, error) {
dgst, dt, err := imageutil.Config(ctx, ref, pull.NewResolver(ctx, is.SessionManager, is.ImageStore), is.ContentStore, platform)
if err != nil {
return nil, err

3
util/dockerexporter/dockerexporter.go
View File

@@ -10,7 +10,6 @@ import (
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/images"
"github.com/containerd/containerd/platforms"
ocispecs "github.com/opencontainers/image-spec/specs-go"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
@@ -51,8 +50,6 @@ func (de *DockerExporter) Export(ctx context.Context, store content.Provider, de
// Get all the children for a descriptor
childrenHandler := images.ChildrenHandler(store)
// Filter the childen by the platform
childrenHandler = images.FilterPlatforms(childrenHandler, platforms.Default())
handlers := images.Handlers(
childrenHandler,

2
util/pull/pull.go
View File

@@ -182,7 +182,7 @@ func (p *Puller) Pull(ctx context.Context) (*Pulled, error) {
var layerBlobs []ocispec.Descriptor
for _, j := range usedBlobs {
switch j.MediaType {
case ocispec.MediaTypeImageLayer, images.MediaTypeDockerSchema2Layer, ocispec.MediaTypeImageLayerGzip, images.MediaTypeDockerSchema2LayerGzip:
case ocispec.MediaTypeImageLayer, images.MediaTypeDockerSchema2Layer, ocispec.MediaTypeImageLayerGzip, images.MediaTypeDockerSchema2LayerGzip, images.MediaTypeDockerSchema2LayerForeign, images.MediaTypeDockerSchema2LayerForeignGzip:
layerBlobs = append(layerBlobs, j)
default:
notLayerBlobs = append(notLayerBlobs, j)