Add options to specify containerd runtime

Co-authored-by: Marat Radchenko <marat@slonopotamus.org>
Signed-off-by: Justin Chadwell <me@jedevc.com>
This commit is contained in:
Justin Chadwell
2023-08-13 15:34:26 +03:00
parent 5e6497c943
commit 8bfd280ab7
11 changed files with 305 additions and 13 deletions

View File

@@ -112,6 +112,7 @@ type ContainerdConfig struct {
Labels map[string]string `toml:"labels"`
Platforms []string `toml:"platforms"`
Namespace string `toml:"namespace"`
Runtime ContainerdRuntime `toml:"runtime"`
GCConfig
NetworkConfig
Snapshotter string `toml:"snapshotter"`
@@ -128,6 +129,11 @@ type ContainerdConfig struct {
Rootless bool `toml:"rootless"`
}
type ContainerdRuntime struct {
Name string `toml:"name"`
Options map[string]interface{} `toml:"options"`
}
type GCPolicy struct {
All bool `toml:"all"`
KeepBytes DiskSpace `toml:"keepBytes"`

View File

@@ -42,6 +42,9 @@ foo="bar"
namespace="non-default"
platforms=["linux/amd64"]
address="containerd.sock"
[worker.containerd.runtime]
name="exotic"
options.foo="bar"
[[worker.containerd.gcpolicy]]
all=true
filters=["foo==bar"]
@@ -103,6 +106,8 @@ searchDomains=["example.com"]
require.Equal(t, 0, len(cfg.Workers.OCI.GCPolicy))
require.Equal(t, "non-default", cfg.Workers.Containerd.Namespace)
require.Equal(t, "exotic", cfg.Workers.Containerd.Runtime.Name)
require.Equal(t, "bar", cfg.Workers.Containerd.Runtime.Options["foo"])
require.Equal(t, 3, len(cfg.Workers.Containerd.GCPolicy))
require.Nil(t, cfg.Workers.Containerd.GC)

View File

@@ -6,12 +6,17 @@ package main
import (
"context"
"os"
"runtime"
"strconv"
"strings"
"time"
ctd "github.com/containerd/containerd"
"github.com/containerd/containerd/defaults"
runtimeoptions "github.com/containerd/containerd/pkg/runtimeoptions/v1"
"github.com/containerd/containerd/pkg/userns"
"github.com/containerd/containerd/plugin"
runcoptions "github.com/containerd/containerd/runtime/v2/runc/options"
"github.com/moby/buildkit/cmd/buildkitd/config"
"github.com/moby/buildkit/util/bklog"
"github.com/moby/buildkit/util/network/cniprovider"
@@ -19,6 +24,7 @@ import (
"github.com/moby/buildkit/worker"
"github.com/moby/buildkit/worker/base"
"github.com/moby/buildkit/worker/containerd"
"github.com/pelletier/go-toml"
"github.com/pkg/errors"
"github.com/urfave/cli"
"golang.org/x/sync/semaphore"
@@ -46,6 +52,14 @@ func init() {
defaultConf.Workers.Containerd.Namespace = defaultContainerdNamespace
}
if defaultConf.Workers.Containerd.Runtime.Name == "" {
if runtime.GOOS == "freebsd" {
defaultConf.Workers.Containerd.Runtime.Name = "wtf.sbk.runj.v1"
} else {
defaultConf.Workers.Containerd.Runtime.Name = defaults.DefaultRuntime
}
}
flags := []cli.Flag{
cli.StringFlag{
Name: "containerd-worker",
@@ -74,6 +88,12 @@ func init() {
Value: defaultConf.Workers.Containerd.Namespace,
Hidden: true,
},
cli.StringFlag{
Name: "containerd-worker-runtime",
Usage: "override containerd runtime",
Value: defaultConf.Workers.Containerd.Runtime.Name,
Hidden: true,
},
cli.StringFlag{
Name: "containerd-worker-net",
Usage: "worker network type (auto, cni or host)",
@@ -202,6 +222,12 @@ func applyContainerdFlags(c *cli.Context, cfg *config.Config) error {
cfg.Workers.Containerd.Namespace = c.GlobalString("containerd-worker-namespace")
}
if c.GlobalIsSet("containerd-worker-runtime") || cfg.Workers.Containerd.Runtime.Name == "" {
cfg.Workers.Containerd.Runtime = config.ContainerdRuntime{
Name: c.GlobalString("containerd-worker-runtime"),
}
}
if c.GlobalIsSet("containerd-worker-gc") {
v := c.GlobalBool("containerd-worker-gc")
cfg.Workers.Containerd.GC = &v
@@ -275,7 +301,26 @@ func containerdWorkerInitializer(c *cli.Context, common workerInitializerOpt) ([
if cfg.Snapshotter != "" {
snapshotter = cfg.Snapshotter
}
opt, err := containerd.NewWorkerOpt(common.config.Root, cfg.Address, snapshotter, cfg.Namespace, cfg.Rootless, cfg.Labels, dns, nc, common.config.Workers.Containerd.ApparmorProfile, common.config.Workers.Containerd.SELinux, parallelismSem, common.traceSocket, ctd.WithTimeout(60*time.Second))
var runtime *containerd.RuntimeInfo
if cfg.Runtime.Name != "" {
opts := getRuntimeOptionsType(cfg.Runtime.Name)
t, err := toml.TreeFromMap(cfg.Runtime.Options)
if err != nil {
return nil, errors.Wrapf(err, "failed to parse runtime options config")
}
err = t.Unmarshal(opts)
if err != nil {
return nil, errors.Wrapf(err, "failed to parse runtime options config")
}
runtime = &containerd.RuntimeInfo{
Name: cfg.Runtime.Name,
Options: opts,
}
}
opt, err := containerd.NewWorkerOpt(common.config.Root, cfg.Address, snapshotter, cfg.Namespace, cfg.Rootless, cfg.Labels, dns, nc, common.config.Workers.Containerd.ApparmorProfile, common.config.Workers.Containerd.SELinux, parallelismSem, common.traceSocket, runtime, ctd.WithTimeout(60*time.Second))
if err != nil {
return nil, err
}
@@ -320,3 +365,13 @@ func validContainerdSocket(cfg config.ContainerdConfig) bool {
}
return true
}
// getRuntimeOptionsType gets empty runtime options by the runtime type name.
func getRuntimeOptionsType(t string) interface{} {
switch t {
case plugin.RuntimeRuncV2:
return &runcoptions.Options{}
default:
return &runtimeoptions.Options{}
}
}

View File

@@ -106,6 +106,11 @@ insecure-entitlements = [ "network.host", "security.insecure" ]
[worker.containerd.labels]
"foo" = "bar"
# configure the containerd runtime
[worker.containerd.runtime]
runtime = "io.containerd.runc.v2"
options = { BinaryName = "runc" }
[[worker.containerd.gcpolicy]]
keepBytes = 512000000
keepDuration = 172800

View File

@@ -38,6 +38,7 @@ type containerdExecutor struct {
selinux bool
traceSocket string
rootless bool
runtime *RuntimeInfo
}
// OnCreateRuntimer provides an alternative to OCI hooks for applying network
@@ -53,8 +54,13 @@ type OnCreateRuntimer interface {
OnCreateRuntime(pid uint32) error
}
type RuntimeInfo struct {
Name string
Options any
}
// New creates a new executor backed by connection to containerd API
func New(client *containerd.Client, root, cgroup string, networkProviders map[pb.NetMode]network.Provider, dnsConfig *oci.DNSConfig, apparmorProfile string, selinux bool, traceSocket string, rootless bool) executor.Executor {
func New(client *containerd.Client, root, cgroup string, networkProviders map[pb.NetMode]network.Provider, dnsConfig *oci.DNSConfig, apparmorProfile string, selinux bool, traceSocket string, rootless bool, runtime *RuntimeInfo) executor.Executor {
// clean up old hosts/resolv.conf file. ignore errors
os.RemoveAll(filepath.Join(root, "hosts"))
os.RemoveAll(filepath.Join(root, "resolv.conf"))
@@ -70,6 +76,7 @@ func New(client *containerd.Client, root, cgroup string, networkProviders map[pb
selinux: selinux,
traceSocket: traceSocket,
rootless: rootless,
runtime: runtime,
}
}
@@ -145,9 +152,13 @@ func (w *containerdExecutor) Run(ctx context.Context, id string, root executor.M
defer releaseSpec()
}
container, err := w.client.NewContainer(ctx, id,
opts := []containerd.NewContainerOpts{
containerd.WithSpec(spec),
)
}
if w.runtime != nil {
opts = append(opts, containerd.WithRuntime(w.runtime.Name, w.runtime.Options))
}
container, err := w.client.NewContainer(ctx, id, opts...)
if err != nil {
return nil, err
}

View File

@@ -0,0 +1,177 @@
// To regenerate api.pb.go run `make protos`
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.28.1
// protoc v3.20.1
// source: github.com/containerd/containerd/pkg/runtimeoptions/v1/api.proto
package runtimeoptions_v1
import (
protoreflect "google.golang.org/protobuf/reflect/protoreflect"
protoimpl "google.golang.org/protobuf/runtime/protoimpl"
reflect "reflect"
sync "sync"
)
const (
// Verify that this generated code is sufficiently up-to-date.
_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
// Verify that runtime/protoimpl is sufficiently up-to-date.
_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
)
type Options struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
// TypeUrl specifies the type of the content inside the config file.
TypeUrl string `protobuf:"bytes,1,opt,name=type_url,json=typeUrl,proto3" json:"type_url,omitempty"`
// ConfigPath specifies the filesystem location of the config file
// used by the runtime.
ConfigPath string `protobuf:"bytes,2,opt,name=config_path,json=configPath,proto3" json:"config_path,omitempty"`
// Blob specifies an in-memory TOML blob passed from containerd's configuration section
// for this runtime. This will be used if config_path is not specified.
ConfigBody []byte `protobuf:"bytes,3,opt,name=config_body,json=configBody,proto3" json:"config_body,omitempty"`
}
func (x *Options) Reset() {
*x = Options{}
if protoimpl.UnsafeEnabled {
mi := &file_github_com_containerd_containerd_pkg_runtimeoptions_v1_api_proto_msgTypes[0]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *Options) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*Options) ProtoMessage() {}
func (x *Options) ProtoReflect() protoreflect.Message {
mi := &file_github_com_containerd_containerd_pkg_runtimeoptions_v1_api_proto_msgTypes[0]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use Options.ProtoReflect.Descriptor instead.
func (*Options) Descriptor() ([]byte, []int) {
return file_github_com_containerd_containerd_pkg_runtimeoptions_v1_api_proto_rawDescGZIP(), []int{0}
}
func (x *Options) GetTypeUrl() string {
if x != nil {
return x.TypeUrl
}
return ""
}
func (x *Options) GetConfigPath() string {
if x != nil {
return x.ConfigPath
}
return ""
}
func (x *Options) GetConfigBody() []byte {
if x != nil {
return x.ConfigBody
}
return nil
}
var File_github_com_containerd_containerd_pkg_runtimeoptions_v1_api_proto protoreflect.FileDescriptor
var file_github_com_containerd_containerd_pkg_runtimeoptions_v1_api_proto_rawDesc = []byte{
0x0a, 0x40, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x6f, 0x6e,
0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65,
0x72, 0x64, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x70,
0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2f, 0x76, 0x31, 0x2f, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x72, 0x6f,
0x74, 0x6f, 0x12, 0x11, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x70, 0x74, 0x69, 0x6f,
0x6e, 0x73, 0x2e, 0x76, 0x31, 0x22, 0x66, 0x0a, 0x07, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73,
0x12, 0x19, 0x0a, 0x08, 0x74, 0x79, 0x70, 0x65, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01,
0x28, 0x09, 0x52, 0x07, 0x74, 0x79, 0x70, 0x65, 0x55, 0x72, 0x6c, 0x12, 0x1f, 0x0a, 0x0b, 0x63,
0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09,
0x52, 0x0a, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x50, 0x61, 0x74, 0x68, 0x12, 0x1f, 0x0a, 0x0b,
0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28,
0x0c, 0x52, 0x0a, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x42, 0x6f, 0x64, 0x79, 0x42, 0x4a, 0x5a,
0x48, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x6f, 0x6e, 0x74,
0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72,
0x64, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x70, 0x74,
0x69, 0x6f, 0x6e, 0x73, 0x2f, 0x76, 0x31, 0x3b, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x6f,
0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x5f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f,
0x33,
}
var (
file_github_com_containerd_containerd_pkg_runtimeoptions_v1_api_proto_rawDescOnce sync.Once
file_github_com_containerd_containerd_pkg_runtimeoptions_v1_api_proto_rawDescData = file_github_com_containerd_containerd_pkg_runtimeoptions_v1_api_proto_rawDesc
)
func file_github_com_containerd_containerd_pkg_runtimeoptions_v1_api_proto_rawDescGZIP() []byte {
file_github_com_containerd_containerd_pkg_runtimeoptions_v1_api_proto_rawDescOnce.Do(func() {
file_github_com_containerd_containerd_pkg_runtimeoptions_v1_api_proto_rawDescData = protoimpl.X.CompressGZIP(file_github_com_containerd_containerd_pkg_runtimeoptions_v1_api_proto_rawDescData)
})
return file_github_com_containerd_containerd_pkg_runtimeoptions_v1_api_proto_rawDescData
}
var file_github_com_containerd_containerd_pkg_runtimeoptions_v1_api_proto_msgTypes = make([]protoimpl.MessageInfo, 1)
var file_github_com_containerd_containerd_pkg_runtimeoptions_v1_api_proto_goTypes = []interface{}{
(*Options)(nil), // 0: runtimeoptions.v1.Options
}
var file_github_com_containerd_containerd_pkg_runtimeoptions_v1_api_proto_depIdxs = []int32{
0, // [0:0] is the sub-list for method output_type
0, // [0:0] is the sub-list for method input_type
0, // [0:0] is the sub-list for extension type_name
0, // [0:0] is the sub-list for extension extendee
0, // [0:0] is the sub-list for field type_name
}
func init() { file_github_com_containerd_containerd_pkg_runtimeoptions_v1_api_proto_init() }
func file_github_com_containerd_containerd_pkg_runtimeoptions_v1_api_proto_init() {
if File_github_com_containerd_containerd_pkg_runtimeoptions_v1_api_proto != nil {
return
}
if !protoimpl.UnsafeEnabled {
file_github_com_containerd_containerd_pkg_runtimeoptions_v1_api_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*Options); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
}
type x struct{}
out := protoimpl.TypeBuilder{
File: protoimpl.DescBuilder{
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: file_github_com_containerd_containerd_pkg_runtimeoptions_v1_api_proto_rawDesc,
NumEnums: 0,
NumMessages: 1,
NumExtensions: 0,
NumServices: 0,
},
GoTypes: file_github_com_containerd_containerd_pkg_runtimeoptions_v1_api_proto_goTypes,
DependencyIndexes: file_github_com_containerd_containerd_pkg_runtimeoptions_v1_api_proto_depIdxs,
MessageInfos: file_github_com_containerd_containerd_pkg_runtimeoptions_v1_api_proto_msgTypes,
}.Build()
File_github_com_containerd_containerd_pkg_runtimeoptions_v1_api_proto = out.File
file_github_com_containerd_containerd_pkg_runtimeoptions_v1_api_proto_rawDesc = nil
file_github_com_containerd_containerd_pkg_runtimeoptions_v1_api_proto_goTypes = nil
file_github_com_containerd_containerd_pkg_runtimeoptions_v1_api_proto_depIdxs = nil
}

View File

@@ -0,0 +1,17 @@
// To regenerate api.pb.go run `make protos`
syntax = "proto3";
package runtimeoptions.v1;
option go_package = "github.com/containerd/containerd/pkg/runtimeoptions/v1;runtimeoptions_v1";
message Options {
// TypeUrl specifies the type of the content inside the config file.
string type_url = 1;
// ConfigPath specifies the filesystem location of the config file
// used by the runtime.
string config_path = 2;
// Blob specifies an in-memory TOML blob passed from containerd's configuration section
// for this runtime. This will be used if config_path is not specified.
bytes config_body = 3;
}

View File

@@ -0,0 +1,17 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package runtimeoptions_v1 //nolint:revive // Ignore var-naming: don't use an underscore in package name (revive)

1
vendor/modules.txt vendored
View File

@@ -303,6 +303,7 @@ github.com/containerd/containerd/pkg/dialer
github.com/containerd/containerd/pkg/epoch
github.com/containerd/containerd/pkg/kmutex
github.com/containerd/containerd/pkg/randutil
github.com/containerd/containerd/pkg/runtimeoptions/v1
github.com/containerd/containerd/pkg/seccomp
github.com/containerd/containerd/pkg/seed
github.com/containerd/containerd/pkg/snapshotters

View File

@@ -4,7 +4,6 @@ import (
"context"
"os"
"path/filepath"
"runtime"
"strconv"
"strings"
@@ -27,21 +26,20 @@ import (
"golang.org/x/sync/semaphore"
)
type RuntimeInfo = containerdexecutor.RuntimeInfo
// NewWorkerOpt creates a WorkerOpt.
func NewWorkerOpt(root string, address, snapshotterName, ns string, rootless bool, labels map[string]string, dns *oci.DNSConfig, nopt netproviders.Opt, apparmorProfile string, selinux bool, parallelismSem *semaphore.Weighted, traceSocket string, opts ...containerd.ClientOpt) (base.WorkerOpt, error) {
func NewWorkerOpt(root string, address, snapshotterName, ns string, rootless bool, labels map[string]string, dns *oci.DNSConfig, nopt netproviders.Opt, apparmorProfile string, selinux bool, parallelismSem *semaphore.Weighted, traceSocket string, runtime *RuntimeInfo, opts ...containerd.ClientOpt) (base.WorkerOpt, error) {
opts = append(opts, containerd.WithDefaultNamespace(ns))
if runtime.GOOS == "freebsd" {
opts = append(opts, containerd.WithDefaultRuntime("wtf.sbk.runj.v1"))
}
client, err := containerd.New(address, opts...)
if err != nil {
return base.WorkerOpt{}, errors.Wrapf(err, "failed to connect client to %q . make sure containerd is running", address)
}
return newContainerd(root, client, snapshotterName, ns, rootless, labels, dns, nopt, apparmorProfile, selinux, parallelismSem, traceSocket)
return newContainerd(root, client, snapshotterName, ns, rootless, labels, dns, nopt, apparmorProfile, selinux, parallelismSem, traceSocket, runtime)
}
func newContainerd(root string, client *containerd.Client, snapshotterName, ns string, rootless bool, labels map[string]string, dns *oci.DNSConfig, nopt netproviders.Opt, apparmorProfile string, selinux bool, parallelismSem *semaphore.Weighted, traceSocket string) (base.WorkerOpt, error) {
func newContainerd(root string, client *containerd.Client, snapshotterName, ns string, rootless bool, labels map[string]string, dns *oci.DNSConfig, nopt netproviders.Opt, apparmorProfile string, selinux bool, parallelismSem *semaphore.Weighted, traceSocket string, runtime *RuntimeInfo) (base.WorkerOpt, error) {
if strings.Contains(snapshotterName, "/") {
return base.WorkerOpt{}, errors.Errorf("bad snapshotter name: %q", snapshotterName)
}
@@ -142,7 +140,7 @@ func newContainerd(root string, client *containerd.Client, snapshotterName, ns s
Labels: xlabels,
MetadataStore: md,
NetworkProviders: np,
Executor: containerdexecutor.New(client, root, "", np, dns, apparmorProfile, selinux, traceSocket, rootless),
Executor: containerdexecutor.New(client, root, "", np, dns, apparmorProfile, selinux, traceSocket, rootless, runtime),
Snapshotter: snap,
ContentStore: cs,
Applier: winlayers.NewFileSystemApplierWithWindows(cs, df),

View File

@@ -32,7 +32,7 @@ func TestContainerdWorkerIntegration(t *testing.T) {
func newWorkerOpt(t *testing.T, addr string) base.WorkerOpt {
tmpdir := t.TempDir()
rootless := false
workerOpt, err := NewWorkerOpt(tmpdir, addr, "overlayfs", "buildkit-test", rootless, nil, nil, netproviders.Opt{Mode: "host"}, "", false, nil, "")
workerOpt, err := NewWorkerOpt(tmpdir, addr, "overlayfs", "buildkit-test", rootless, nil, nil, netproviders.Opt{Mode: "host"}, "", false, nil, "", nil)
require.NoError(t, err)
return workerOpt
}