diff --git a/examples/dockerfile2llb/main.go b/examples/dockerfile2llb/main.go index 1de468ed6..b5a6d7d2d 100644 --- a/examples/dockerfile2llb/main.go +++ b/examples/dockerfile2llb/main.go @@ -42,7 +42,7 @@ func xmain() error { caps := pb.Caps.CapSet(pb.Caps.All()) - state, img, baseImg, _, err := dockerfile2llb.Dockerfile2LLB(appcontext.Context(), df, dockerfile2llb.ConvertOpt{ + res, err := dockerfile2llb.Dockerfile2LLB(appcontext.Context(), df, dockerfile2llb.ConvertOpt{ MetaResolver: imagemetaresolver.Default(), LLBCaps: &caps, Config: dockerui.Config{ @@ -53,7 +53,7 @@ func xmain() error { return err } - dt, err := state.Marshal(context.TODO()) + dt, err := res.State.Marshal(context.TODO()) if err != nil { return err } @@ -61,12 +61,12 @@ func xmain() error { return err } if opt.partialImageConfigFile != "" { - if err := writeJSON(opt.partialImageConfigFile, img); err != nil { + if err := writeJSON(opt.partialImageConfigFile, res.Image); err != nil { return err } } if opt.baseImageConfigFile != "" { - if err := writeJSON(opt.baseImageConfigFile, baseImg); err != nil { + if err := writeJSON(opt.baseImageConfigFile, res.BaseImage); err != nil { return err } } diff --git a/exporter/containerimage/export.go b/exporter/containerimage/export.go index 918b55e7c..7966d61b3 100644 --- a/exporter/containerimage/export.go +++ b/exporter/containerimage/export.go @@ -283,8 +283,8 @@ func (e *imageExporterInstance) Export(ctx context.Context, src *exporter.Source // However, due to a bug of containerd, we are temporarily stuck with this workaround. // https://github.com/containerd/containerd/issues/8322 imageClientCtx := ctx - if e.opts.Epoch != nil { - imageClientCtx = epoch.WithSourceDateEpoch(imageClientCtx, e.opts.Epoch) + if e.opts.Epoch != nil && e.opts.Epoch.Value != nil { + imageClientCtx = epoch.WithSourceDateEpoch(imageClientCtx, e.opts.Epoch.Value) } img := images.Image{ Target: *desc, diff --git a/exporter/containerimage/opts.go b/exporter/containerimage/opts.go index 1ccadbd94..5f268229f 100644 --- a/exporter/containerimage/opts.go +++ b/exporter/containerimage/opts.go @@ -3,7 +3,6 @@ package containerimage import ( "context" "strconv" - "time" cacheconfig "github.com/moby/buildkit/cache/config" "github.com/moby/buildkit/exporter/containerimage/exptypes" @@ -19,7 +18,7 @@ type ImageCommitOpts struct { OCITypes bool OCIArtifact bool Annotations AnnotationsGroup - Epoch *time.Time + Epoch *epoch.Epoch ForceInlineAttestations bool // force inline attestations to be attached RewriteTimestamp bool // rewrite timestamps in layers to match the epoch diff --git a/exporter/containerimage/writer.go b/exporter/containerimage/writer.go index f0cddeab9..d5363c714 100644 --- a/exporter/containerimage/writer.go +++ b/exporter/containerimage/writer.go @@ -94,10 +94,10 @@ func (ic *ImageWriter) Commit(ctx context.Context, inp *exporter.Source, session } } if opts.Epoch == nil { - if tm, ok, err := epoch.ParseSource(inp); err != nil { + if tm, err := epoch.ParseSource(inp, nil); err != nil { return nil, err - } else if ok { - opts.Epoch = tm + } else if tm != nil { + opts.Epoch = &epoch.Epoch{Value: tm} } } @@ -119,14 +119,24 @@ func (ic *ImageWriter) Commit(ctx context.Context, inp *exporter.Source, session var ref cache.ImmutableRef var p *exptypes.Platform + var expEpoch *time.Time if len(ps.Platforms) > 0 { p = &ps.Platforms[0] if r, ok := inp.FindRef(p.ID); ok { ref = r } + if opts.Epoch == nil { + expEpoch, err = epoch.ParseSource(inp, p) + if err != nil { + return nil, err + } + } } else { ref = inp.Ref } + if opts.Epoch != nil { + expEpoch = opts.Epoch.Value + } config := exptypes.ParseKey(inp.Metadata, exptypes.ExporterImageConfigKey, p) baseImgConfig := exptypes.ParseKey(inp.Metadata, exptypes.ExporterImageBaseConfigKey, p) var baseImg *dockerspec.DockerOCIImage @@ -144,7 +154,7 @@ func (ic *ImageWriter) Commit(ctx context.Context, inp *exporter.Source, session } remote := &remotes[0] if opts.RewriteTimestamp { - remote, err = ic.rewriteRemoteWithEpoch(ctx, opts, remote, baseImg) + remote, err = ic.rewriteRemoteWithEpoch(ctx, opts, remote, baseImg, expEpoch) if err != nil { return nil, err } @@ -170,7 +180,7 @@ func (ic *ImageWriter) Commit(ctx context.Context, inp *exporter.Source, session } } - mfstDesc, configDesc, err := ic.commitDistributionManifest(ctx, opts, ref, config, remote, annotations, inlineCacheEntry, opts.Epoch, session.NewGroup(sessionID), baseImg) + mfstDesc, configDesc, err := ic.commitDistributionManifest(ctx, opts, ref, config, remote, annotations, inlineCacheEntry, expEpoch, session.NewGroup(sessionID), baseImg) if err != nil { return nil, err } @@ -245,6 +255,16 @@ func (ic *ImageWriter) Commit(ctx context.Context, inp *exporter.Source, session baseImg = &baseImgX } + var expEpoch *time.Time + if opts.Epoch == nil { + expEpoch, err = epoch.ParseSource(inp, &p) + if err != nil { + return nil, err + } + } else { + expEpoch = opts.Epoch.Value + } + remote := &remotes[remotesMap[p.ID]] if remote == nil { remote = &solver.Remote{ @@ -252,7 +272,7 @@ func (ic *ImageWriter) Commit(ctx context.Context, inp *exporter.Source, session } } if opts.RewriteTimestamp { - remote, err = ic.rewriteRemoteWithEpoch(ctx, opts, remote, baseImg) + remote, err = ic.rewriteRemoteWithEpoch(ctx, opts, remote, baseImg, expEpoch) if err != nil { return nil, err } @@ -263,7 +283,7 @@ func (ic *ImageWriter) Commit(ctx context.Context, inp *exporter.Source, session inlineCacheEntry, _ = inlineCacheResult.FindRef(p.ID) } - desc, _, err := ic.commitDistributionManifest(ctx, opts, r, config, remote, opts.Annotations.Platform(&p.Platform), inlineCacheEntry, opts.Epoch, session.NewGroup(sessionID), baseImg) + desc, _, err := ic.commitDistributionManifest(ctx, opts, r, config, remote, opts.Annotations.Platform(&p.Platform), inlineCacheEntry, expEpoch, session.NewGroup(sessionID), baseImg) if err != nil { return nil, err } @@ -407,8 +427,8 @@ func rewriteImageLayerWithEpoch(ctx context.Context, cs content.Store, desc ocis return converterFn(ctx, cs, desc) } -func (ic *ImageWriter) rewriteRemoteWithEpoch(ctx context.Context, opts *ImageCommitOpts, remote *solver.Remote, baseImg *dockerspec.DockerOCIImage) (*solver.Remote, error) { - if opts.Epoch == nil { +func (ic *ImageWriter) rewriteRemoteWithEpoch(ctx context.Context, opts *ImageCommitOpts, remote *solver.Remote, baseImg *dockerspec.DockerOCIImage, expEpoch *time.Time) (*solver.Remote, error) { + if expEpoch == nil { bklog.G(ctx).Warn("rewrite-timestamp is specified, but no source-date-epoch was found") return remote, nil } @@ -416,7 +436,7 @@ func (ic *ImageWriter) rewriteRemoteWithEpoch(ctx context.Context, opts *ImageCo cs := contentutil.NewStoreWithProvider(ic.opt.ContentStore, remote.Provider) eg, ctx := errgroup.WithContext(ctx) rewriteDone := progress.OneOff(ctx, - fmt.Sprintf("rewriting layers with source-date-epoch %d (%s)", opts.Epoch.Unix(), opts.Epoch.String())) + fmt.Sprintf("rewriting layers with source-date-epoch %d (%s)", expEpoch.Unix(), expEpoch.String())) var divergedFromBase bool for i, desc := range remoteDescriptors { // Usually we get non-empty diffID here, but if the content was ingested via a third-party containerd client, @@ -439,9 +459,9 @@ func (ic *ImageWriter) rewriteRemoteWithEpoch(ctx context.Context, opts *ImageCo divergedFromBase = true } eg.Go(func() error { - if rewrittenDesc, err := rewriteImageLayerWithEpoch(ctx, cs, desc, opts.RefCfg.Compression, opts.Epoch, immDiffID); err != nil { + if rewrittenDesc, err := rewriteImageLayerWithEpoch(ctx, cs, desc, opts.RefCfg.Compression, expEpoch, immDiffID); err != nil { bklog.G(ctx).WithError(err).Warnf("failed to rewrite layer %d/%d to match source-date-epoch %d (%s)", - i+1, len(remoteDescriptors), opts.Epoch.Unix(), opts.Epoch.String()) + i+1, len(remoteDescriptors), expEpoch.Unix(), expEpoch.String()) } else if rewrittenDesc != nil { remoteDescriptors[i] = *rewrittenDesc } diff --git a/exporter/local/export.go b/exporter/local/export.go index f2cdc8915..ff043143d 100644 --- a/exporter/local/export.go +++ b/exporter/local/export.go @@ -84,10 +84,10 @@ func (e *localExporterInstance) Export(ctx context.Context, inp *exporter.Source defer func() { cancel(errors.WithStack(context.Canceled)) }() if e.opts.Epoch == nil { - if tm, ok, err := epoch.ParseSource(inp); err != nil { + if tm, err := epoch.ParseSource(inp, nil); err != nil { return nil, nil, nil, err - } else if ok { - e.opts.Epoch = tm + } else if tm != nil { + e.opts.Epoch = &epoch.Epoch{Value: tm} } } @@ -115,9 +115,9 @@ func (e *localExporterInstance) Export(ctx context.Context, inp *exporter.Source visitedPath := map[string]string{} var visitedMu sync.Mutex - export := func(ctx context.Context, k string, ref cache.ImmutableRef, attestations []exporter.Attestation) func() error { + export := func(ctx context.Context, k string, ref cache.ImmutableRef, attestations []exporter.Attestation, opt CreateFSOpts) func() error { return func() error { - outputFS, cleanup, err := CreateFS(ctx, buildInfo.SessionID, k, ref, attestations, now, isMap, e.opts) + outputFS, cleanup, err := CreateFS(ctx, buildInfo.SessionID, k, ref, attestations, now, isMap, opt) if err != nil { return err } @@ -152,8 +152,8 @@ func (e *localExporterInstance) Export(ctx context.Context, inp *exporter.Source Mode: uint32(os.ModeDir | 0755), Path: strings.ReplaceAll(k, "/", "_"), } - if e.opts.Epoch != nil { - st.ModTime = e.opts.Epoch.UnixNano() + if opt.Epoch != nil && opt.Epoch.Value != nil { + st.ModTime = opt.Epoch.Value.UnixNano() } outputFS, err = fsutil.SubDirFS([]fsutil.Dir{{FS: outputFS, Stat: st}}) if err != nil { @@ -177,10 +177,18 @@ func (e *localExporterInstance) Export(ctx context.Context, inp *exporter.Source if !ok { return nil, nil, nil, errors.Errorf("failed to find ref for ID %s", p.ID) } - eg.Go(export(ctx, p.ID, r, inp.Attestations[p.ID])) + opt := e.opts + if e.opts.Epoch == nil { + tm, err := epoch.ParseSource(inp, &p) + if err != nil { + return nil, nil, nil, err + } + opt.Epoch = &epoch.Epoch{Value: tm} + } + eg.Go(export(ctx, p.ID, r, inp.Attestations[p.ID], opt)) } } else { - eg.Go(export(ctx, "", inp.Ref, nil)) + eg.Go(export(ctx, "", inp.Ref, nil, e.opts)) } if err := eg.Wait(); err != nil { diff --git a/exporter/local/fs.go b/exporter/local/fs.go index ccbbf7e64..199003a26 100644 --- a/exporter/local/fs.go +++ b/exporter/local/fs.go @@ -36,7 +36,7 @@ const ( ) type CreateFSOpts struct { - Epoch *time.Time + Epoch *epoch.Epoch AttestationPrefix string PlatformSplit *bool } @@ -129,9 +129,9 @@ func CreateFS(ctx context.Context, sessionID string, k string, ref cache.Immutab // apply host uid/gid res = idMapFunc(p, st) } - if opt.Epoch != nil { + if opt.Epoch != nil && opt.Epoch.Value != nil { // apply used-specified epoch time - st.ModTime = opt.Epoch.UnixNano() + st.ModTime = opt.Epoch.Value.UnixNano() } return res } @@ -205,8 +205,8 @@ func CreateFS(ctx context.Context, sessionID string, k string, ref cache.Immutab Path: name, ModTime: defaultTime.UnixNano(), } - if opt.Epoch != nil { - st.ModTime = opt.Epoch.UnixNano() + if opt.Epoch != nil && opt.Epoch.Value != nil { + st.ModTime = opt.Epoch.Value.UnixNano() } stmtFS.Add(name, st, dt) } diff --git a/exporter/oci/export.go b/exporter/oci/export.go index 24b96a338..839f2edf5 100644 --- a/exporter/oci/export.go +++ b/exporter/oci/export.go @@ -174,8 +174,8 @@ func (e *imageExporterInstance) Export(ctx context.Context, src *exporter.Source } if _, ok := desc.Annotations[ocispecs.AnnotationCreated]; !ok { tm := time.Now() - if opts.Epoch != nil { - tm = *opts.Epoch + if opts.Epoch != nil && opts.Epoch.Value != nil { + tm = *opts.Epoch.Value } desc.Annotations[ocispecs.AnnotationCreated] = tm.UTC().Format(time.RFC3339) } diff --git a/exporter/tar/export.go b/exporter/tar/export.go index 921cf616f..02000593b 100644 --- a/exporter/tar/export.go +++ b/exporter/tar/export.go @@ -87,18 +87,18 @@ func (e *localExporterInstance) Export(ctx context.Context, inp *exporter.Source }() if e.opts.Epoch == nil { - if tm, ok, err := epoch.ParseSource(inp); err != nil { + if tm, err := epoch.ParseSource(inp, nil); err != nil { return nil, nil, nil, err - } else if ok { - e.opts.Epoch = tm + } else if tm != nil { + e.opts.Epoch = &epoch.Epoch{Value: tm} } } now := time.Now().Truncate(time.Second) isMap := len(inp.Refs) > 0 - getDir := func(ctx context.Context, k string, ref cache.ImmutableRef, attestations []exporter.Attestation) (*fsutil.Dir, error) { - outputFS, cleanup, err := local.CreateFS(ctx, buildInfo.SessionID, k, ref, attestations, now, isMap, e.opts) + getDir := func(ctx context.Context, k string, ref cache.ImmutableRef, attestations []exporter.Attestation, opt local.CreateFSOpts) (*fsutil.Dir, error) { + outputFS, cleanup, err := local.CreateFS(ctx, buildInfo.SessionID, k, ref, attestations, now, isMap, opt) if err != nil { return nil, err } @@ -110,8 +110,8 @@ func (e *localExporterInstance) Export(ctx context.Context, inp *exporter.Source Mode: uint32(os.ModeDir | 0755), Path: strings.ReplaceAll(k, "/", "_"), } - if e.opts.Epoch != nil { - st.ModTime = e.opts.Epoch.UnixNano() + if opt.Epoch != nil && opt.Epoch.Value != nil { + st.ModTime = opt.Epoch.Value.UnixNano() } return &fsutil.Dir{ @@ -140,7 +140,15 @@ func (e *localExporterInstance) Export(ctx context.Context, inp *exporter.Source if !ok { return nil, nil, nil, errors.Errorf("failed to find ref for ID %s", p.ID) } - d, err := getDir(ctx, p.ID, r, inp.Attestations[p.ID]) + opt := e.opts + if e.opts.Epoch == nil { + tm, err := epoch.ParseSource(inp, &p) + if err != nil { + return nil, nil, nil, err + } + opt.Epoch = &epoch.Epoch{Value: tm} + } + d, err := getDir(ctx, p.ID, r, inp.Attestations[p.ID], opt) if err != nil { return nil, nil, nil, err } @@ -156,7 +164,7 @@ func (e *localExporterInstance) Export(ctx context.Context, inp *exporter.Source fs = dirs[0].FS } } else { - d, err := getDir(ctx, "", inp.Ref, nil) + d, err := getDir(ctx, "", inp.Ref, nil, e.opts) if err != nil { return nil, nil, nil, err } diff --git a/exporter/util/epoch/parse.go b/exporter/util/epoch/parse.go index d5a014608..aa32b16b8 100644 --- a/exporter/util/epoch/parse.go +++ b/exporter/util/epoch/parse.go @@ -1,10 +1,12 @@ package epoch import ( + "fmt" "strconv" "time" "github.com/moby/buildkit/exporter" + imageexptypes "github.com/moby/buildkit/exporter/containerimage/exptypes" commonexptypes "github.com/moby/buildkit/exporter/exptypes" "github.com/pkg/errors" ) @@ -13,24 +15,29 @@ const ( frontendSourceDateEpochArg = "build-arg:SOURCE_DATE_EPOCH" ) +type Epoch struct { + Value *time.Time +} + func ParseBuildArgs(opt map[string]string) (string, bool) { v, ok := opt[frontendSourceDateEpochArg] return v, ok } -func ParseExporterAttrs(opt map[string]string) (*time.Time, map[string]string, error) { +func ParseExporterAttrs(opt map[string]string) (*Epoch, map[string]string, error) { rest := make(map[string]string, len(opt)) - var tm *time.Time + var tm *Epoch for k, v := range opt { switch k { case string(commonexptypes.OptKeySourceDateEpoch): var err error - tm, err = parseTime(k, v) + value, err := parseTime(k, v) if err != nil { return nil, nil, err } + tm = &Epoch{Value: value} default: rest[k] = v } @@ -39,15 +46,25 @@ func ParseExporterAttrs(opt map[string]string) (*time.Time, map[string]string, e return tm, rest, nil } -func ParseSource(inp *exporter.Source) (*time.Time, bool, error) { - if v, ok := inp.Metadata[commonexptypes.ExporterEpochKey]; ok { +func ParseSource(inp *exporter.Source, p *imageexptypes.Platform) (*time.Time, error) { + key := commonexptypes.ExporterEpochKey + if p != nil { + if v, ok := inp.Metadata[fmt.Sprintf("%s/%s", key, p.ID)]; ok { + epoch, err := parseTime("", string(v)) + if err != nil { + return nil, errors.Wrapf(err, "invalid SOURCE_DATE_EPOCH from frontend: %q", v) + } + return epoch, nil + } + } + if v, ok := inp.Metadata[key]; ok { epoch, err := parseTime("", string(v)) if err != nil { - return nil, false, errors.Wrapf(err, "invalid SOURCE_DATE_EPOCH from frontend: %q", v) + return nil, errors.Wrapf(err, "invalid SOURCE_DATE_EPOCH from frontend: %q", v) } - return epoch, true, nil + return epoch, nil } - return nil, false, nil + return nil, nil } func parseTime(key, value string) (*time.Time, error) { diff --git a/frontend/dockerfile/builder/build.go b/frontend/dockerfile/builder/build.go index a9ecb820e..33a8c9294 100644 --- a/frontend/dockerfile/builder/build.go +++ b/frontend/dockerfile/builder/build.go @@ -23,7 +23,6 @@ import ( "github.com/moby/buildkit/solver/pb" "github.com/moby/buildkit/solver/result" "github.com/moby/buildkit/util/bkmaps" - dockerspec "github.com/moby/docker-image-spec/specs-go/v1" ocispecs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" ) @@ -128,23 +127,23 @@ func Build(ctx context.Context, c client.Client) (_ *client.Result, err error) { } } - scanTargets := bkmaps.SyncMap[string, *dockerfile2llb.SBOMTargets]{} + scanTargets := bkmaps.SyncMap[string, *dockerfile2llb.Result]{} - rb, err := bc.Build(ctx, func(ctx context.Context, platform *ocispecs.Platform, idx int) (client.Reference, *dockerspec.DockerOCIImage, *dockerspec.DockerOCIImage, error) { + rb, err := bc.Build(ctx, func(ctx context.Context, platform *ocispecs.Platform, idx int) (*dockerui.BuildResult, error) { opt := convertOpt opt.TargetPlatform = platform if idx != 0 { opt.Warn = nil } - st, img, baseImg, scanTarget, err := dockerfile2llb.Dockerfile2LLB(ctx, src.Data, opt) + dfRes, err := dockerfile2llb.Dockerfile2LLB(ctx, src.Data, opt) if err != nil { - return nil, nil, nil, err + return nil, err } - def, err := st.Marshal(ctx) + def, err := dfRes.State.Marshal(ctx) if err != nil { - return nil, nil, nil, errors.Wrapf(err, "failed to marshal LLB definition") + return nil, errors.Wrapf(err, "failed to marshal LLB definition") } r, err := c.Solve(ctx, client.SolveRequest{ @@ -152,12 +151,12 @@ func Build(ctx context.Context, c client.Client) (_ *client.Result, err error) { CacheImports: bc.CacheImports, }) if err != nil { - return nil, nil, nil, err + return nil, err } ref, err := r.SingleRef() if err != nil { - return nil, nil, nil, err + return nil, err } var p ocispecs.Platform @@ -166,9 +165,14 @@ func Build(ctx context.Context, c client.Client) (_ *client.Result, err error) { } else { p = platforms.DefaultSpec() } - scanTargets.Store(platforms.FormatAll(platforms.Normalize(p)), scanTarget) - - return ref, img, baseImg, nil + id := platforms.FormatAll(platforms.Normalize(p)) + scanTargets.Store(id, dfRes) + return &dockerui.BuildResult{ + Reference: ref, + Image: dfRes.Image, + BaseImage: dfRes.BaseImage, + Epoch: dfRes.Epoch, + }, nil }) if err != nil { return nil, err @@ -182,10 +186,10 @@ func Build(ctx context.Context, c client.Client) (_ *client.Result, err error) { } var opts []llb.ConstraintsOpt - if target.IgnoreCache { + if target.IsIgnoreCache { opts = append(opts, llb.IgnoreCache) } - att, err := scanner(ctx, id, target.Core, target.Extras, opts...) + att, err := scanner(ctx, id, target.SBOM.Core, target.SBOM.Extras, opts...) if err != nil { return err } diff --git a/frontend/dockerfile/dockerfile2llb/convert.go b/frontend/dockerfile/dockerfile2llb/convert.go index 81576d197..6c81ad24b 100644 --- a/frontend/dockerfile/dockerfile2llb/convert.go +++ b/frontend/dockerfile/dockerfile2llb/convert.go @@ -69,36 +69,50 @@ type ConvertOpt struct { type SBOMTargets struct { Core llb.State Extras map[string]llb.State - - IgnoreCache bool } -func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (st *llb.State, img, baseImg *dockerspec.DockerOCIImage, sbom *SBOMTargets, err error) { +type Result struct { + State llb.State + Image *dockerspec.DockerOCIImage + BaseImage *dockerspec.DockerOCIImage + SBOM *SBOMTargets + Epoch *time.Time + + IsIgnoreCache bool +} + +func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*Result, error) { ds, err := toDispatchState(ctx, dt, opt) if err != nil { - return nil, nil, nil, nil, err + return nil, err } - sbom = &SBOMTargets{ - Core: ds.state, - Extras: map[string]llb.State{}, + res := &Result{ + State: ds.state, + Image: &ds.image, + BaseImage: ds.baseImg, + SBOM: &SBOMTargets{ + Core: ds.state, + Extras: map[string]llb.State{}, + }, + Epoch: ds.epoch, } if ds.scanContext { - sbom.Extras["context"] = ds.opt.buildContext + res.SBOM.Extras["context"] = ds.opt.buildContext } if ds.ignoreCache { - sbom.IgnoreCache = true + res.IsIgnoreCache = true } for dsi := range allReachableStages(ds) { if ds != dsi && dsi.scanStage { - sbom.Extras[dsi.stageName] = dsi.state + res.SBOM.Extras[dsi.stageName] = dsi.state if dsi.ignoreCache { - sbom.IgnoreCache = true + res.IsIgnoreCache = true } } } - return &ds.state, &ds.image, ds.baseImg, sbom, nil + return res, nil } func Dockerfile2Outline(ctx context.Context, dt []byte, opt ConvertOpt) (*outline.Outline, error) { diff --git a/frontend/dockerfile/dockerfile2llb/convert_test.go b/frontend/dockerfile/dockerfile2llb/convert_test.go index 51962696b..6574913b0 100644 --- a/frontend/dockerfile/dockerfile2llb/convert_test.go +++ b/frontend/dockerfile/dockerfile2llb/convert_test.go @@ -41,7 +41,7 @@ ENV FOO bar COPY f1 f2 /sub/ RUN ls -l ` - _, _, _, _, err := Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{}) + _, err := Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{}) require.NoError(t, err) df = `FROM scratch AS foo @@ -50,7 +50,7 @@ FROM foo COPY --from=foo f1 / COPY --from=0 f2 / ` - _, _, _, _, err = Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{}) + _, err = Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{}) require.NoError(t, err) df = `FROM scratch AS foo @@ -59,14 +59,14 @@ FROM foo COPY --from=foo f1 / COPY --from=0 f2 / ` - _, _, _, _, err = Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{ + _, err = Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{ Config: dockerui.Config{ Target: "Foo", }, }) require.NoError(t, err) - _, _, _, _, err = Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{ + _, err = Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{ Config: dockerui.Config{ Target: "nosuch", }, @@ -76,21 +76,21 @@ COPY --from=0 f2 / df = `FROM scratch ADD http://github.com/moby/buildkit/blob/master/README.md / ` - _, _, _, _, err = Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{}) + _, err = Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{}) require.NoError(t, err) df = `FROM scratch COPY http://github.com/moby/buildkit/blob/master/README.md / ` - _, _, _, _, err = Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{}) + _, err = Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{}) require.EqualError(t, err, "source can't be a URL for COPY") df = `FROM "" AS foo` - _, _, _, _, err = Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{}) + _, err = Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{}) require.Error(t, err) df = `FROM ${BLANK} AS foo` - _, _, _, _, err = Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{}) + _, err = Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{}) require.Error(t, err) } @@ -101,10 +101,10 @@ ENV FOO bar COPY f1 f2 /sub/ RUN ls -l ` - state, _, _, _, err := Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{}) + res, err := Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{}) require.NoError(t, err) - _, err = state.Marshal(context.TODO()) + _, err = res.State.Marshal(context.TODO()) require.NoError(t, err) } @@ -225,7 +225,7 @@ func TestDockerfileCircularDependencies(t *testing.T) { df := `FROM busybox AS stage0 COPY --from=stage0 f1 /sub/ ` - _, _, _, _, err := Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{}) + _, err := Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{}) require.EqualError(t, err, "circular dependency detected on stage: stage0") // multiple stages with circular dependency @@ -236,7 +236,7 @@ COPY --from=stage0 f2 /sub/ FROM busybox AS stage2 COPY --from=stage1 f2 /sub/ ` - _, _, _, _, err = Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{}) + _, err = Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{}) require.EqualError(t, err, "circular dependency detected on stage: stage0") } @@ -248,9 +248,9 @@ RUN echo foo FROM foo AS bar RUN echo bar ` - _, _, baseImg, _, err := Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{}) + res, err := Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{}) require.NoError(t, err) - t.Logf("baseImg=%+v", baseImg) - assert.Equal(t, []digest.Digest{"sha256:2e112031b4b923a873c8b3d685d48037e4d5ccd967b658743d93a6e56c3064b9"}, baseImg.RootFS.DiffIDs) - assert.Equal(t, "2024-01-17 21:49:12 +0000 UTC", baseImg.Created.String()) + t.Logf("baseImg=%+v", res.BaseImage) + assert.Equal(t, []digest.Digest{"sha256:2e112031b4b923a873c8b3d685d48037e4d5ccd967b658743d93a6e56c3064b9"}, res.BaseImage.RootFS.DiffIDs) + assert.Equal(t, "2024-01-17 21:49:12 +0000 UTC", res.BaseImage.Created.String()) } diff --git a/frontend/dockerfile/dockerfile2llb/exclude_patterns_test.go b/frontend/dockerfile/dockerfile2llb/exclude_patterns_test.go index 2f59da861..836dac792 100644 --- a/frontend/dockerfile/dockerfile2llb/exclude_patterns_test.go +++ b/frontend/dockerfile/dockerfile2llb/exclude_patterns_test.go @@ -11,7 +11,7 @@ func TestDockerfileCopyExcludePatterns(t *testing.T) { df := `FROM scratch COPY --exclude=src/*.go --exclude=tmp/*.txt dir /sub/ ` - _, _, _, _, err := Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{}) + _, err := Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{}) assert.NoError(t, err) } @@ -19,6 +19,6 @@ func TestDockerfileAddExcludePatterns(t *testing.T) { df := `FROM scratch ADD --exclude=src/*.go --exclude=tmp/*.txt dir /sub/ ` - _, _, _, _, err := Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{}) + _, err := Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{}) assert.NoError(t, err) } diff --git a/frontend/dockerfile/dockerfile_test.go b/frontend/dockerfile/dockerfile_test.go index 93ff17523..038cac794 100644 --- a/frontend/dockerfile/dockerfile_test.go +++ b/frontend/dockerfile/dockerfile_test.go @@ -231,6 +231,7 @@ var reproTests = integration.TestFuncs( testWorkdirSourceDateEpochReproducible, testSourceDateEpochDockerfileDefault, testSourceDateEpochDockerfileDefaultOverride, + testSourceDateEpochDockerfileDefaultReset, testSourceDateEpochDockerfileDefaultInvalid, ) @@ -1029,7 +1030,7 @@ WORKDIR /mydir require.Equal(t, index1, index2) } -func readOCIImageCreated(t *testing.T, dt []byte) time.Time { +func readOCIManifest(t *testing.T, dt []byte) ocispecs.Manifest { t.Helper() m, err := testutil.ReadTarToMap(dt, false) @@ -1043,12 +1044,7 @@ func readOCIImageCreated(t *testing.T, dt []byte) time.Time { err = json.Unmarshal(m[ocispecs.ImageBlobsDir+"/sha256/"+idx.Manifests[0].Digest.Hex()].Data, &mfst) require.NoError(t, err) - var img ocispecs.Image - err = json.Unmarshal(m[ocispecs.ImageBlobsDir+"/sha256/"+mfst.Config.Digest.Hex()].Data, &img) - require.NoError(t, err) - require.NotNil(t, img.Created) - - return *img.Created + return mfst } func testCacheReleased(t *testing.T, sb integration.Sandbox) { @@ -8945,11 +8941,11 @@ func testSourceDateEpochDockerfileDefault(t *testing.T, sb integration.Sandbox) f := getFrontend(t, sb) tm := time.Date(2015, time.October, 21, 7, 28, 0, 0, time.UTC) - dockerfile := []byte(fmt.Sprintf(` + dockerfile := fmt.Appendf(nil, ` ARG SOURCE_DATE_EPOCH=%d FROM scratch COPY Dockerfile . -`, tm.Unix())) +`, tm.Unix()) dir := integration.Tmpdir( t, @@ -8974,7 +8970,7 @@ COPY Dockerfile . { Type: client.ExporterOCI, Attrs: map[string]string{ - "source-date-epoch": "", + "rewrite-timestamp": "true", }, Output: fixedWriteCloser(outW), }, @@ -8985,7 +8981,9 @@ COPY Dockerfile . dt, err := os.ReadFile(out) require.NoError(t, err) - require.Equal(t, tm.Unix(), readOCIImageCreated(t, dt).Unix()) + mfst := readOCIManifest(t, dt) + require.Len(t, mfst.Layers, 1) + require.Equal(t, fmt.Sprintf("%d", tm.Unix()), mfst.Layers[0].Annotations["buildkit/rewritten-timestamp"]) } func testSourceDateEpochDockerfileDefaultOverride(t *testing.T, sb integration.Sandbox) { @@ -8995,11 +8993,11 @@ func testSourceDateEpochDockerfileDefaultOverride(t *testing.T, sb integration.S defaultTM := time.Unix(1700000001, 0).UTC() overrideTM := time.Unix(1700000002, 0).UTC() - dockerfile := []byte(fmt.Sprintf(` + dockerfile := fmt.Appendf(nil, ` ARG SOURCE_DATE_EPOCH=%d FROM scratch COPY Dockerfile . -`, defaultTM.Unix())) +`, defaultTM.Unix()) dir := integration.Tmpdir( t, @@ -9027,7 +9025,7 @@ COPY Dockerfile . { Type: client.ExporterOCI, Attrs: map[string]string{ - "source-date-epoch": "", + "rewrite-timestamp": "true", }, Output: fixedWriteCloser(outW), }, @@ -9038,7 +9036,61 @@ COPY Dockerfile . dt, err := os.ReadFile(out) require.NoError(t, err) - require.Equal(t, overrideTM.Unix(), readOCIImageCreated(t, dt).Unix()) + mfst := readOCIManifest(t, dt) + require.Len(t, mfst.Layers, 1) + require.Equal(t, fmt.Sprintf("%d", overrideTM.Unix()), mfst.Layers[0].Annotations["buildkit/rewritten-timestamp"]) +} + +func testSourceDateEpochDockerfileDefaultReset(t *testing.T, sb integration.Sandbox) { + integration.SkipOnPlatform(t, "windows") + workers.CheckFeatureCompat(t, sb, workers.FeatureOCIExporter, workers.FeatureSourceDateEpoch) + f := getFrontend(t, sb) + + tm := time.Date(2015, time.October, 21, 7, 28, 0, 0, time.UTC) + dockerfile := fmt.Appendf(nil, ` +ARG SOURCE_DATE_EPOCH=%d +FROM scratch +COPY Dockerfile . +`, tm.Unix()) + + dir := integration.Tmpdir( + t, + fstest.CreateFile("Dockerfile", dockerfile, 0600), + ) + + c, err := client.New(sb.Context(), sb.Address()) + require.NoError(t, err) + defer c.Close() + + destDir := t.TempDir() + out := filepath.Join(destDir, "out.tar") + outW, err := os.Create(out) + require.NoError(t, err) + + _, err = f.Solve(sb.Context(), c, client.SolveOpt{ + LocalMounts: map[string]fsutil.FS{ + dockerui.DefaultLocalNameDockerfile: dir, + dockerui.DefaultLocalNameContext: dir, + }, + Exports: []client.ExportEntry{ + { + Type: client.ExporterOCI, + Attrs: map[string]string{ + "source-date-epoch": "", + "rewrite-timestamp": "true", + }, + Output: fixedWriteCloser(outW), + }, + }, + }, nil) + require.NoError(t, err) + + dt, err := os.ReadFile(out) + require.NoError(t, err) + + mfst := readOCIManifest(t, dt) + require.Len(t, mfst.Layers, 1) + require.Empty(t, mfst.Layers[0].Annotations["buildkit/rewritten-timestamp"]) } func testSourceDateEpochDockerfileDefaultInvalid(t *testing.T, sb integration.Sandbox) { diff --git a/frontend/dockerui/build.go b/frontend/dockerui/build.go index 02a966ab4..06b940091 100644 --- a/frontend/dockerui/build.go +++ b/frontend/dockerui/build.go @@ -5,9 +5,12 @@ import ( "encoding/json" "fmt" "slices" + "strconv" + "time" "github.com/containerd/platforms" "github.com/moby/buildkit/exporter/containerimage/exptypes" + commonexptypes "github.com/moby/buildkit/exporter/exptypes" "github.com/moby/buildkit/frontend/gateway/client" dockerspec "github.com/moby/docker-image-spec/specs-go/v1" ocispecs "github.com/opencontainers/image-spec/specs-go/v1" @@ -15,7 +18,14 @@ import ( "golang.org/x/sync/errgroup" ) -type BuildFunc func(ctx context.Context, platform *ocispecs.Platform, idx int) (r client.Reference, img, baseImg *dockerspec.DockerOCIImage, err error) +type BuildResult struct { + Reference client.Reference + Image *dockerspec.DockerOCIImage + BaseImage *dockerspec.DockerOCIImage + Epoch *time.Time +} + +type BuildFunc func(ctx context.Context, platform *ocispecs.Platform, idx int) (*BuildResult, error) func (bc *Client) Build(ctx context.Context, fn BuildFunc) (*ResultBuilder, error) { res := client.NewResult() @@ -35,10 +45,11 @@ func (bc *Client) Build(ctx context.Context, fn BuildFunc) (*ResultBuilder, erro for i, tp := range targets { eg.Go(func() error { - ref, img, baseImg, err := fn(ctx, tp, i) + buildRes, err := fn(ctx, tp, i) if err != nil { return err } + ref, img, baseImg := buildRes.Reference, buildRes.Image, buildRes.BaseImage config, err := json.Marshal(img) if err != nil { @@ -66,12 +77,18 @@ func (bc *Client) Build(ctx context.Context, fn BuildFunc) (*ResultBuilder, erro if len(baseConfig) > 0 { res.AddMeta(fmt.Sprintf("%s/%s", exptypes.ExporterImageBaseConfigKey, expPlat.ID), baseConfig) } + if buildRes.Epoch != nil { + res.AddMeta(fmt.Sprintf("%s/%s", commonexptypes.ExporterEpochKey, expPlat.ID), []byte(strconv.FormatInt(buildRes.Epoch.Unix(), 10))) + } } else { res.SetRef(ref) res.AddMeta(exptypes.ExporterImageConfigKey, config) if len(baseConfig) > 0 { res.AddMeta(exptypes.ExporterImageBaseConfigKey, baseConfig) } + if buildRes.Epoch != nil { + res.AddMeta(commonexptypes.ExporterEpochKey, []byte(strconv.FormatInt(buildRes.Epoch.Unix(), 10))) + } } expPlatforms.Platforms[i] = expPlat return nil