From 4eadeaf0f2eef00280d00e27e418cd18ba4f1925 Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Sun, 12 Dec 2021 20:39:36 -0800 Subject: [PATCH 1/3] dockerfile: add support for named contexts Stages and implicit stages from image names can be redefined with build options. This enables using more that one source directory and reusing results from other builds. This can also be used to use a local image from other build without including a registry. Contexts need to be defined as `context:name=` frontend options. The value can be image, git repository, URL, local directory or a frontend input. Signed-off-by: Tonis Tiigi --- frontend/dockerfile/builder/build.go | 75 ++++++ frontend/dockerfile/builder/caps.go | 1 + .../cmd/dockerfile-frontend/Dockerfile | 2 +- frontend/dockerfile/dockerfile2llb/convert.go | 50 +++- frontend/dockerfile/dockerfile_test.go | 242 ++++++++++++++++++ 5 files changed, 366 insertions(+), 4 deletions(-) diff --git a/frontend/dockerfile/builder/build.go b/frontend/dockerfile/builder/build.go index cf4cc4717..7f7e8b42c 100644 --- a/frontend/dockerfile/builder/build.go +++ b/frontend/dockerfile/builder/build.go @@ -14,6 +14,7 @@ import ( "strings" "github.com/containerd/containerd/platforms" + "github.com/docker/distribution/reference" "github.com/docker/go-units" controlapi "github.com/moby/buildkit/api/services/control" "github.com/moby/buildkit/client/llb" @@ -461,6 +462,7 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) { } c.Warn(ctx, defVtx, msg, warnOpts(sourceMap, location, detail, url)) }, + ContextByName: contextByName(c, tp), }) if err != nil { @@ -802,6 +804,79 @@ func warnOpts(sm *llb.SourceMap, r *parser.Range, detail [][]byte, url string) c return opts } +func contextByName(c client.Client, p *ocispecs.Platform) func(context.Context, string) (*llb.State, *dockerfile2llb.Image, error) { + return func(ctx context.Context, name string) (*llb.State, *dockerfile2llb.Image, error) { + named, err := reference.ParseNormalizedNamed(name) + if err != nil { + return nil, nil, errors.Wrapf(err, "invalid context name %s", name) + } + name = strings.TrimSuffix(reference.FamiliarString(named), ":latest") + + opts := c.BuildOpts().Opts + v, ok := opts["context:"+name] + if !ok { + return nil, nil, nil + } + + vv := strings.SplitN(v, ":", 2) + if len(vv) != 2 { + return nil, nil, errors.Errorf("invalid context specifier %s for %s", v, name) + } + switch vv[0] { + case "docker-image": + st := llb.Image(strings.TrimPrefix(vv[1], "//"), llb.WithCustomName("[context "+name+"] "+vv[1]), llb.WithMetaResolver(c)) + return &st, nil, nil + case "git": + st, ok := detectGitContext(v, "") + if !ok { + return nil, nil, errors.Errorf("invalid git context %s", v) + } + return st, nil, nil + case "http", "https": + st, ok := detectGitContext(v, "") + if !ok { + httpst := llb.HTTP(v, llb.WithCustomName("[context "+name+"] "+v)) + st = &httpst + } + return st, nil, nil + case "local": + st := llb.Local(vv[1], llb.WithCustomName("[context "+name+"] load from client"), llb.SessionID(c.BuildOpts().SessionID), llb.SharedKeyHint("context:"+name)) + return &st, nil, nil + case "input": + inputs, err := c.Inputs(ctx) + if err != nil { + return nil, nil, err + } + st, ok := inputs[vv[1]] + if !ok { + return nil, nil, errors.Errorf("invalid input %s for %s", vv[1], name) + } + md, ok := opts["input-metadata:"+vv[1]] + if ok { + m := make(map[string][]byte) + if err := json.Unmarshal([]byte(md), &m); err != nil { + return nil, nil, errors.Wrapf(err, "failed to parse input metadata %s", md) + } + dt, ok := m["containerimage.config"] + if ok { + st, err = st.WithImageConfig([]byte(dt)) + if err != nil { + return nil, nil, err + } + var img dockerfile2llb.Image + if err := json.Unmarshal(dt, &img); err != nil { + return nil, nil, errors.Wrapf(err, "failed to parse image config for %s", name) + } + return &st, &img, nil + } + } + return &st, nil, nil + default: + return nil, nil, errors.Errorf("unsupported context source %s for %s", vv[0], name) + } + } +} + func wrapSource(err error, sm *llb.SourceMap, ranges []parser.Range) error { if sm == nil { return err diff --git a/frontend/dockerfile/builder/caps.go b/frontend/dockerfile/builder/caps.go index 279701154..3c78cd56c 100644 --- a/frontend/dockerfile/builder/caps.go +++ b/frontend/dockerfile/builder/caps.go @@ -12,6 +12,7 @@ import ( var enabledCaps = map[string]struct{}{ "moby.buildkit.frontend.inputs": {}, "moby.buildkit.frontend.subrequests": {}, + "moby.buildkit.frontend.contexts": {}, } func validateCaps(req string) (forward bool, err error) { diff --git a/frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile b/frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile index b8f4e0e85..f97d44315 100644 --- a/frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile +++ b/frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile @@ -29,7 +29,7 @@ RUN --mount=target=. --mount=type=cache,target=/root/.cache \ FROM scratch AS release LABEL moby.buildkit.frontend.network.none="true" -LABEL moby.buildkit.frontend.caps="moby.buildkit.frontend.inputs,moby.buildkit.frontend.subrequests" +LABEL moby.buildkit.frontend.caps="moby.buildkit.frontend.inputs,moby.buildkit.frontend.subrequests,moby.buildkit.frontend.contexts" COPY --from=build /dockerfile-frontend /bin/dockerfile-frontend ENTRYPOINT ["/bin/dockerfile-frontend"] diff --git a/frontend/dockerfile/dockerfile2llb/convert.go b/frontend/dockerfile/dockerfile2llb/convert.go index 0a521621c..218b38225 100644 --- a/frontend/dockerfile/dockerfile2llb/convert.go +++ b/frontend/dockerfile/dockerfile2llb/convert.go @@ -69,9 +69,20 @@ type ConvertOpt struct { SourceMap *llb.SourceMap Hostname string Warn func(short, url string, detail [][]byte, location *parser.Range) + ContextByName func(context.Context, string) (*llb.State, *Image, error) } func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State, *Image, error) { + contextByName := opt.ContextByName + opt.ContextByName = func(ctx context.Context, name string) (*llb.State, *Image, error) { + if !strings.EqualFold(name, "scratch") && !strings.EqualFold(name, "context") { + if contextByName != nil { + return contextByName(ctx, name) + } + } + return nil, nil, nil + } + if len(dt) == 0 { return nil, nil, errors.Errorf("the Dockerfile cannot be empty") } @@ -133,13 +144,30 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State, st.BaseName = name ds := &dispatchState{ - stage: st, deps: make(map[*dispatchState]struct{}), ctxPaths: make(map[string]struct{}), stageName: st.Name, prefixPlatform: opt.PrefixPlatform, } + if st.Name != "" { + s, img, err := opt.ContextByName(ctx, st.Name) + if err != nil { + return nil, nil, err + } + if s != nil { + ds.noinit = true + ds.state = *s + if img != nil { + ds.image = *img + } + allDispatchStates.addState(ds) + continue + } + } + + ds.stage = st + if st.Name == "" { ds.stageName = fmt.Sprintf("stage-%d", i) } @@ -237,7 +265,7 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State, for i, d := range allDispatchStates.states { reachable := isReachable(target, d) // resolve image config for every stage - if d.base == nil { + if d.base == nil && !d.noinit { if d.stage.BaseName == emptyImageName { d.state = llb.Scratch() d.image = emptyImage(platformOpt.targetPlatform) @@ -260,8 +288,23 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State, platform = &platformOpt.targetPlatform } d.stage.BaseName = reference.TagNameOnly(ref).String() + var isScratch bool - if metaResolver != nil && reachable { + st, img, err := opt.ContextByName(ctx, d.stage.BaseName) + if err != nil { + return err + } + if st != nil { + if img != nil { + d.image = *img + } else { + d.image = emptyImage(platformOpt.targetPlatform) + } + d.state = *st + d.platform = platform + return nil + } + if reachable { prefix := "[" if opt.PrefixPlatform && platform != nil { prefix += platforms.Format(*platform) + " " @@ -615,6 +658,7 @@ type dispatchState struct { platform *ocispecs.Platform stage instructions.Stage base *dispatchState + noinit bool deps map[*dispatchState]struct{} buildArgs []instructions.KeyValuePairOptional commands []command diff --git a/frontend/dockerfile/dockerfile_test.go b/frontend/dockerfile/dockerfile_test.go index 66856008f..fa67fa5e7 100644 --- a/frontend/dockerfile/dockerfile_test.go +++ b/frontend/dockerfile/dockerfile_test.go @@ -39,6 +39,7 @@ import ( "github.com/moby/buildkit/session" "github.com/moby/buildkit/session/upload/uploadprovider" "github.com/moby/buildkit/solver/errdefs" + "github.com/moby/buildkit/solver/pb" "github.com/moby/buildkit/util/contentutil" "github.com/moby/buildkit/util/testutil" "github.com/moby/buildkit/util/testutil/httpserver" @@ -118,6 +119,9 @@ var allTests = integration.TestFuncs( testShmSize, testUlimit, testCgroupParent, + testNamedImageContext, + testNamedLocalContext, + testNamedInputContext, ) var fileOpTests = integration.TestFuncs( @@ -159,6 +163,7 @@ var securityOpts []integration.TestOpt type frontend interface { Solve(context.Context, *client.Client, client.SolveOpt, chan *client.SolveStatus) (*client.SolveResponse, error) + SolveGateway(context.Context, gateway.Client, gateway.SolveRequest) (*gateway.Result, error) DFCmdArgs(string, string) (string, string) RequiresBuildctl(t *testing.T) } @@ -5406,6 +5411,222 @@ COPY --from=base /out / require.Contains(t, strings.TrimSpace(string(dt)), `/foocgroup/buildkit/`) } +func testNamedImageContext(t *testing.T, sb integration.Sandbox) { + ctx := sb.Context() + + c, err := client.New(ctx, sb.Address()) + require.NoError(t, err) + defer c.Close() + + dockerfile := []byte(` +FROM busybox AS base +RUN cat /etc/alpine-release > /out +FROM scratch +COPY --from=base /out / +`) + + dir, err := tmpdir( + fstest.CreateFile("Dockerfile", dockerfile, 0600), + ) + require.NoError(t, err) + defer os.RemoveAll(dir) + + f := getFrontend(t, sb) + + destDir, err := ioutil.TempDir("", "buildkit") + require.NoError(t, err) + defer os.RemoveAll(destDir) + + _, err = f.Solve(sb.Context(), c, client.SolveOpt{ + FrontendAttrs: map[string]string{ + "context:busybox": "docker-image://alpine", + }, + LocalDirs: map[string]string{ + builder.DefaultLocalNameDockerfile: dir, + builder.DefaultLocalNameContext: dir, + }, + Exports: []client.ExportEntry{ + { + Type: client.ExporterLocal, + OutputDir: destDir, + }, + }, + }, nil) + require.NoError(t, err) + + dt, err := ioutil.ReadFile(filepath.Join(destDir, "out")) + require.NoError(t, err) + require.True(t, len(dt) > 0) +} + +func testNamedLocalContext(t *testing.T, sb integration.Sandbox) { + ctx := sb.Context() + + c, err := client.New(ctx, sb.Address()) + require.NoError(t, err) + defer c.Close() + + dockerfile := []byte(` +FROM busybox AS base +RUN cat /etc/alpine-release > /out +FROM scratch +COPY --from=base /out / +`) + + dir, err := tmpdir( + fstest.CreateFile("Dockerfile", dockerfile, 0600), + ) + require.NoError(t, err) + defer os.RemoveAll(dir) + + outf := []byte(`dummy-result`) + + dir2, err := tmpdir( + fstest.CreateFile("out", outf, 0600), + ) + require.NoError(t, err) + defer os.RemoveAll(dir2) + + f := getFrontend(t, sb) + + destDir, err := ioutil.TempDir("", "buildkit") + require.NoError(t, err) + defer os.RemoveAll(destDir) + + _, err = f.Solve(sb.Context(), c, client.SolveOpt{ + FrontendAttrs: map[string]string{ + "context:base": "local:basedir", + }, + LocalDirs: map[string]string{ + builder.DefaultLocalNameDockerfile: dir, + builder.DefaultLocalNameContext: dir, + "basedir": dir2, + }, + Exports: []client.ExportEntry{ + { + Type: client.ExporterLocal, + OutputDir: destDir, + }, + }, + }, nil) + require.NoError(t, err) + + dt, err := ioutil.ReadFile(filepath.Join(destDir, "out")) + require.NoError(t, err) + require.True(t, len(dt) > 0) +} + +func testNamedInputContext(t *testing.T, sb integration.Sandbox) { + ctx := sb.Context() + + c, err := client.New(ctx, sb.Address()) + require.NoError(t, err) + defer c.Close() + + dockerfile := []byte(` +FROM alpine +ENV FOO=bar +RUN echo first > /out +`) + + dir, err := tmpdir( + fstest.CreateFile("Dockerfile", dockerfile, 0600), + ) + require.NoError(t, err) + defer os.RemoveAll(dir) + + dockerfile2 := []byte(` +FROM base AS build +RUN echo "foo is $FOO" > /foo +FROM scratch +COPY --from=build /foo /out / +`) + + dir2, err := tmpdir( + fstest.CreateFile("Dockerfile", dockerfile2, 0600), + ) + require.NoError(t, err) + defer os.RemoveAll(dir) + + f := getFrontend(t, sb) + + b := func(ctx context.Context, c gateway.Client) (*gateway.Result, error) { + res, err := f.SolveGateway(ctx, c, gateway.SolveRequest{}) + if err != nil { + return nil, err + } + ref, err := res.SingleRef() + if err != nil { + return nil, err + } + st, err := ref.ToState() + if err != nil { + return nil, err + } + + def, err := st.Marshal(ctx) + if err != nil { + return nil, err + } + + dt, ok := res.Metadata["containerimage.config"] + if !ok { + return nil, errors.Errorf("no containerimage.config in metadata") + } + + dt, err = json.Marshal(map[string][]byte{ + "containerimage.config": dt, + }) + if err != nil { + return nil, err + } + + res, err = f.SolveGateway(ctx, c, gateway.SolveRequest{ + FrontendOpt: map[string]string{ + "dockerfilekey": builder.DefaultLocalNameDockerfile + "2", + "context:base": "input:base", + "input-metadata:base": string(dt), + }, + FrontendInputs: map[string]*pb.Definition{ + "base": def.ToPB(), + }, + }) + if err != nil { + return nil, err + } + return res, nil + } + + product := "buildkit_test" + + destDir, err := ioutil.TempDir("", "buildkit") + require.NoError(t, err) + defer os.RemoveAll(destDir) + + _, err = c.Build(ctx, client.SolveOpt{ + LocalDirs: map[string]string{ + builder.DefaultLocalNameDockerfile: dir, + builder.DefaultLocalNameContext: dir, + builder.DefaultLocalNameDockerfile + "2": dir2, + }, + Exports: []client.ExportEntry{ + { + Type: client.ExporterLocal, + OutputDir: destDir, + }, + }, + }, product, b, nil) + require.NoError(t, err) + + dt, err := ioutil.ReadFile(filepath.Join(destDir, "out")) + require.NoError(t, err) + require.Equal(t, "first\n", string(dt)) + + dt, err = ioutil.ReadFile(filepath.Join(destDir, "foo")) + require.NoError(t, err) + require.Equal(t, "foo is bar\n", string(dt)) +} + func tmpdir(appliers ...fstest.Applier) (string, error) { tmpdir, err := ioutil.TempDir("", "buildkit-dockerfile") if err != nil { @@ -5542,6 +5763,11 @@ func (f *builtinFrontend) Solve(ctx context.Context, c *client.Client, opt clien return c.Solve(ctx, nil, opt, statusChan) } +func (f *builtinFrontend) SolveGateway(ctx context.Context, c gateway.Client, req gateway.SolveRequest) (*gateway.Result, error) { + req.Frontend = "dockerfile.v0" + return c.Solve(ctx, req) +} + func (f *builtinFrontend) DFCmdArgs(ctx, dockerfile string) (string, string) { return dfCmdArgs(ctx, dockerfile, "--frontend dockerfile.v0") } @@ -5556,6 +5782,13 @@ func (f *clientFrontend) Solve(ctx context.Context, c *client.Client, opt client return c.Build(ctx, opt, "", builder.Build, statusChan) } +func (f *clientFrontend) SolveGateway(ctx context.Context, c gateway.Client, req gateway.SolveRequest) (*gateway.Result, error) { + if req.Frontend == "" && req.Definition == nil { + req.Frontend = "dockerfile.v0" + } + return c.Solve(ctx, req) +} + func (f *clientFrontend) DFCmdArgs(ctx, dockerfile string) (string, string) { return "", "" } @@ -5578,6 +5811,15 @@ func (f *gatewayFrontend) Solve(ctx context.Context, c *client.Client, opt clien return c.Solve(ctx, nil, opt, statusChan) } +func (f *gatewayFrontend) SolveGateway(ctx context.Context, c gateway.Client, req gateway.SolveRequest) (*gateway.Result, error) { + req.Frontend = "gateway.v0" + if req.FrontendOpt == nil { + req.FrontendOpt = make(map[string]string) + } + req.FrontendOpt["source"] = f.gw + return c.Solve(ctx, req) +} + func (f *gatewayFrontend) DFCmdArgs(ctx, dockerfile string) (string, string) { return dfCmdArgs(ctx, dockerfile, "--frontend gateway.v0 --opt=source="+f.gw) } From 20285bb53eca9c7f0b91be0375f680ae5fbfdf7f Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Mon, 13 Dec 2021 23:26:10 -0800 Subject: [PATCH 2/3] dockerfile: test named contexts with multi-platform Signed-off-by: Tonis Tiigi --- frontend/dockerfile/builder/build.go | 132 ++++++++++++---------- frontend/dockerfile/dockerfile_test.go | 150 +++++++++++++++++++++++++ 2 files changed, 223 insertions(+), 59 deletions(-) diff --git a/frontend/dockerfile/builder/build.go b/frontend/dockerfile/builder/build.go index 7f7e8b42c..18d175ce4 100644 --- a/frontend/dockerfile/builder/build.go +++ b/frontend/dockerfile/builder/build.go @@ -462,7 +462,7 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) { } c.Warn(ctx, defVtx, msg, warnOpts(sourceMap, location, detail, url)) }, - ContextByName: contextByName(c, tp), + ContextByName: contextByNameFunc(c, tp), }) if err != nil { @@ -804,7 +804,7 @@ func warnOpts(sm *llb.SourceMap, r *parser.Range, detail [][]byte, url string) c return opts } -func contextByName(c client.Client, p *ocispecs.Platform) func(context.Context, string) (*llb.State, *dockerfile2llb.Image, error) { +func contextByNameFunc(c client.Client, p *ocispecs.Platform) func(context.Context, string) (*llb.State, *dockerfile2llb.Image, error) { return func(ctx context.Context, name string) (*llb.State, *dockerfile2llb.Image, error) { named, err := reference.ParseNormalizedNamed(name) if err != nil { @@ -812,68 +812,82 @@ func contextByName(c client.Client, p *ocispecs.Platform) func(context.Context, } name = strings.TrimSuffix(reference.FamiliarString(named), ":latest") - opts := c.BuildOpts().Opts - v, ok := opts["context:"+name] - if !ok { - return nil, nil, nil - } - - vv := strings.SplitN(v, ":", 2) - if len(vv) != 2 { - return nil, nil, errors.Errorf("invalid context specifier %s for %s", v, name) - } - switch vv[0] { - case "docker-image": - st := llb.Image(strings.TrimPrefix(vv[1], "//"), llb.WithCustomName("[context "+name+"] "+vv[1]), llb.WithMetaResolver(c)) - return &st, nil, nil - case "git": - st, ok := detectGitContext(v, "") - if !ok { - return nil, nil, errors.Errorf("invalid git context %s", v) - } - return st, nil, nil - case "http", "https": - st, ok := detectGitContext(v, "") - if !ok { - httpst := llb.HTTP(v, llb.WithCustomName("[context "+name+"] "+v)) - st = &httpst - } - return st, nil, nil - case "local": - st := llb.Local(vv[1], llb.WithCustomName("[context "+name+"] load from client"), llb.SessionID(c.BuildOpts().SessionID), llb.SharedKeyHint("context:"+name)) - return &st, nil, nil - case "input": - inputs, err := c.Inputs(ctx) + if p != nil { + name := name + "::" + platforms.Format(platforms.Normalize(*p)) + st, img, err := contextByName(ctx, c, name) if err != nil { return nil, nil, err } - st, ok := inputs[vv[1]] - if !ok { - return nil, nil, errors.Errorf("invalid input %s for %s", vv[1], name) + if st != nil { + return st, img, nil } - md, ok := opts["input-metadata:"+vv[1]] - if ok { - m := make(map[string][]byte) - if err := json.Unmarshal([]byte(md), &m); err != nil { - return nil, nil, errors.Wrapf(err, "failed to parse input metadata %s", md) - } - dt, ok := m["containerimage.config"] - if ok { - st, err = st.WithImageConfig([]byte(dt)) - if err != nil { - return nil, nil, err - } - var img dockerfile2llb.Image - if err := json.Unmarshal(dt, &img); err != nil { - return nil, nil, errors.Wrapf(err, "failed to parse image config for %s", name) - } - return &st, &img, nil - } - } - return &st, nil, nil - default: - return nil, nil, errors.Errorf("unsupported context source %s for %s", vv[0], name) } + return contextByName(ctx, c, name) + } +} + +func contextByName(ctx context.Context, c client.Client, name string) (*llb.State, *dockerfile2llb.Image, error) { + opts := c.BuildOpts().Opts + v, ok := opts["context:"+name] + if !ok { + return nil, nil, nil + } + + vv := strings.SplitN(v, ":", 2) + if len(vv) != 2 { + return nil, nil, errors.Errorf("invalid context specifier %s for %s", v, name) + } + switch vv[0] { + case "docker-image": + st := llb.Image(strings.TrimPrefix(vv[1], "//"), llb.WithCustomName("[context "+name+"] "+vv[1]), llb.WithMetaResolver(c)) + return &st, nil, nil + case "git": + st, ok := detectGitContext(v, "1") + if !ok { + return nil, nil, errors.Errorf("invalid git context %s", v) + } + return st, nil, nil + case "http", "https": + st, ok := detectGitContext(v, "1") + if !ok { + httpst := llb.HTTP(v, llb.WithCustomName("[context "+name+"] "+v)) + st = &httpst + } + return st, nil, nil + case "local": + st := llb.Local(vv[1], llb.WithCustomName("[context "+name+"] load from client"), llb.SessionID(c.BuildOpts().SessionID), llb.SharedKeyHint("context:"+name)) + return &st, nil, nil + case "input": + inputs, err := c.Inputs(ctx) + if err != nil { + return nil, nil, err + } + st, ok := inputs[vv[1]] + if !ok { + return nil, nil, errors.Errorf("invalid input %s for %s", vv[1], name) + } + md, ok := opts["input-metadata:"+vv[1]] + if ok { + m := make(map[string][]byte) + if err := json.Unmarshal([]byte(md), &m); err != nil { + return nil, nil, errors.Wrapf(err, "failed to parse input metadata %s", md) + } + dt, ok := m["containerimage.config"] + if ok { + st, err = st.WithImageConfig([]byte(dt)) + if err != nil { + return nil, nil, err + } + var img dockerfile2llb.Image + if err := json.Unmarshal(dt, &img); err != nil { + return nil, nil, errors.Wrapf(err, "failed to parse image config for %s", name) + } + return &st, &img, nil + } + } + return &st, nil, nil + default: + return nil, nil, errors.Errorf("unsupported context source %s for %s", vv[0], name) } } diff --git a/frontend/dockerfile/dockerfile_test.go b/frontend/dockerfile/dockerfile_test.go index fa67fa5e7..3aaa9f0e3 100644 --- a/frontend/dockerfile/dockerfile_test.go +++ b/frontend/dockerfile/dockerfile_test.go @@ -122,6 +122,7 @@ var allTests = integration.TestFuncs( testNamedImageContext, testNamedLocalContext, testNamedInputContext, + testNamedMultiplatformInputContext, ) var fileOpTests = integration.TestFuncs( @@ -5627,6 +5628,155 @@ COPY --from=build /foo /out / require.Equal(t, "foo is bar\n", string(dt)) } +func testNamedMultiplatformInputContext(t *testing.T, sb integration.Sandbox) { + ctx := sb.Context() + + c, err := client.New(ctx, sb.Address()) + require.NoError(t, err) + defer c.Close() + + dockerfile := []byte(` +FROM --platform=$BUILDPLATFORM alpine +ARG TARGETARCH +ENV FOO=bar-$TARGETARCH +RUN echo "foo $TARGETARCH" > /out +`) + + dir, err := tmpdir( + fstest.CreateFile("Dockerfile", dockerfile, 0600), + ) + require.NoError(t, err) + defer os.RemoveAll(dir) + + dockerfile2 := []byte(` +FROM base AS build +RUN echo "foo is $FOO" > /foo +FROM scratch +COPY --from=build /foo /out / +`) + + dir2, err := tmpdir( + fstest.CreateFile("Dockerfile", dockerfile2, 0600), + ) + require.NoError(t, err) + defer os.RemoveAll(dir) + + f := getFrontend(t, sb) + + b := func(ctx context.Context, c gateway.Client) (*gateway.Result, error) { + res, err := f.SolveGateway(ctx, c, gateway.SolveRequest{ + FrontendOpt: map[string]string{ + "platform": "linux/amd64,linux/arm64", + }, + }) + if err != nil { + return nil, err + } + + if len(res.Refs) != 2 { + return nil, errors.Errorf("expected 2 refs, got %d", len(res.Refs)) + } + + inputs := map[string]*pb.Definition{} + st, err := res.Refs["linux/amd64"].ToState() + if err != nil { + return nil, err + } + def, err := st.Marshal(ctx) + if err != nil { + return nil, err + } + inputs["base::linux/amd64"] = def.ToPB() + + st, err = res.Refs["linux/arm64"].ToState() + if err != nil { + return nil, err + } + def, err = st.Marshal(ctx) + if err != nil { + return nil, err + } + inputs["base::linux/arm64"] = def.ToPB() + + frontendOpt := map[string]string{ + "dockerfilekey": builder.DefaultLocalNameDockerfile + "2", + "context:base::linux/amd64": "input:base::linux/amd64", + "context:base::linux/arm64": "input:base::linux/arm64", + "platform": "linux/amd64,linux/arm64", + } + + dt, ok := res.Metadata["containerimage.config/linux/amd64"] + if !ok { + return nil, errors.Errorf("no containerimage.config in metadata") + } + dt, err = json.Marshal(map[string][]byte{ + "containerimage.config": dt, + }) + if err != nil { + return nil, err + } + frontendOpt["input-metadata:base::linux/amd64"] = string(dt) + + dt, ok = res.Metadata["containerimage.config/linux/arm64"] + if !ok { + return nil, errors.Errorf("no containerimage.config in metadata") + } + dt, err = json.Marshal(map[string][]byte{ + "containerimage.config": dt, + }) + if err != nil { + return nil, err + } + frontendOpt["input-metadata:base::linux/arm64"] = string(dt) + + res, err = f.SolveGateway(ctx, c, gateway.SolveRequest{ + FrontendOpt: frontendOpt, + FrontendInputs: inputs, + }) + if err != nil { + return nil, err + } + return res, nil + } + + product := "buildkit_test" + + destDir, err := ioutil.TempDir("", "buildkit") + require.NoError(t, err) + defer os.RemoveAll(destDir) + + _, err = c.Build(ctx, client.SolveOpt{ + LocalDirs: map[string]string{ + builder.DefaultLocalNameDockerfile: dir, + builder.DefaultLocalNameContext: dir, + builder.DefaultLocalNameDockerfile + "2": dir2, + }, + Exports: []client.ExportEntry{ + { + Type: client.ExporterLocal, + OutputDir: destDir, + }, + }, + }, product, b, nil) + require.NoError(t, err) + + dt, err := ioutil.ReadFile(filepath.Join(destDir, "linux_amd64/out")) + require.NoError(t, err) + require.Equal(t, "foo amd64\n", string(dt)) + + dt, err = ioutil.ReadFile(filepath.Join(destDir, "linux_amd64/foo")) + require.NoError(t, err) + require.Equal(t, "foo is bar-amd64\n", string(dt)) + + dt, err = ioutil.ReadFile(filepath.Join(destDir, "linux_arm64/out")) + require.NoError(t, err) + require.Equal(t, "foo arm64\n", string(dt)) + + dt, err = ioutil.ReadFile(filepath.Join(destDir, "linux_arm64/foo")) + require.NoError(t, err) + require.Equal(t, "foo is bar-arm64\n", string(dt)) +} + func tmpdir(appliers ...fstest.Applier) (string, error) { tmpdir, err := ioutil.TempDir("", "buildkit-dockerfile") if err != nil { From 23709ef3163b660caef6056e19ddd31c11cc97b6 Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Mon, 13 Dec 2021 23:36:53 -0800 Subject: [PATCH 3/3] dockerfile: set current platform on image based named context Signed-off-by: Tonis Tiigi --- frontend/dockerfile/builder/build.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/frontend/dockerfile/builder/build.go b/frontend/dockerfile/builder/build.go index 18d175ce4..5ea0eae1b 100644 --- a/frontend/dockerfile/builder/build.go +++ b/frontend/dockerfile/builder/build.go @@ -814,7 +814,7 @@ func contextByNameFunc(c client.Client, p *ocispecs.Platform) func(context.Conte if p != nil { name := name + "::" + platforms.Format(platforms.Normalize(*p)) - st, img, err := contextByName(ctx, c, name) + st, img, err := contextByName(ctx, c, name, p) if err != nil { return nil, nil, err } @@ -822,11 +822,11 @@ func contextByNameFunc(c client.Client, p *ocispecs.Platform) func(context.Conte return st, img, nil } } - return contextByName(ctx, c, name) + return contextByName(ctx, c, name, p) } } -func contextByName(ctx context.Context, c client.Client, name string) (*llb.State, *dockerfile2llb.Image, error) { +func contextByName(ctx context.Context, c client.Client, name string, platform *ocispecs.Platform) (*llb.State, *dockerfile2llb.Image, error) { opts := c.BuildOpts().Opts v, ok := opts["context:"+name] if !ok { @@ -839,7 +839,14 @@ func contextByName(ctx context.Context, c client.Client, name string) (*llb.Stat } switch vv[0] { case "docker-image": - st := llb.Image(strings.TrimPrefix(vv[1], "//"), llb.WithCustomName("[context "+name+"] "+vv[1]), llb.WithMetaResolver(c)) + imgOpt := []llb.ImageOption{ + llb.WithCustomName("[context " + name + "] " + vv[1]), + llb.WithMetaResolver(c), + } + if platform != nil { + imgOpt = append(imgOpt, llb.Platform(*platform)) + } + st := llb.Image(strings.TrimPrefix(vv[1], "//"), imgOpt...) return &st, nil, nil case "git": st, ok := detectGitContext(v, "1")